From ad2b6db5f10d9ae5e413d5e287a4efaa8a42d72f Mon Sep 17 00:00:00 2001 From: Joseph Lynch Date: Mon, 10 Sep 2018 20:40:40 -0700 Subject: [PATCH 001/228] Fix 3.11 Bootstrap (#715) * Fix handling of cluster startup. Previously we would provide the local node as a seed even if auto_bootstrap was set to true, which meant that when doubling we could potentially stream no data. Providing the local node as a seed was added to work around the (imo buggy) behaviour introduced in CASSANDRA-10134 where the shadow round fails unless the node is a seed which even if autobootstrap was turned off. * Test for InstanceIdentity properly handling auto bootstrap conditions Previously we could double and have nodes join without data, now when auto bootstrap is true we do not return the current node as a seed ever. * Provide an overwrite method to force Priam to replace a particular ip This allows us to work around the A->B->C replacement problem where Priam get's confused about who to replace --- .../priam/identity/InstanceIdentity.java | 22 ++++++++++--- .../priam/resources/CassandraConfig.java | 13 ++++++++ .../backup/identity/InstanceIdentityTest.java | 33 ++++++++++++++++--- .../priam/config/FakeConfiguration.java | 17 +++++++--- .../identity/FakePriamInstanceFactory.java | 4 ++- 5 files changed, 75 insertions(+), 14 deletions(-) diff --git a/priam/src/main/java/com/netflix/priam/identity/InstanceIdentity.java b/priam/src/main/java/com/netflix/priam/identity/InstanceIdentity.java index 7071e8511..1360ac9d5 100644 --- a/priam/src/main/java/com/netflix/priam/identity/InstanceIdentity.java +++ b/priam/src/main/java/com/netflix/priam/identity/InstanceIdentity.java @@ -63,12 +63,18 @@ public List get() { private final Predicate differentHostPredicate = new Predicate() { @Override - /** - * This is used to provide the list of seed providers. - * Since 3.x backported the @see CASSANDRA-10134 we need to ensure that seed list contains all the seed(including itself) or cluster would never come up. - */ public boolean apply(PriamInstance instance) { - return (!instance.getInstanceId().equalsIgnoreCase(DUMMY_INSTANCE_ID)); + if (config.getAutoBoostrap()) { + // auto_bootstrap = true indicates that the cluster is up and running normally, in such a case + // we cannot provide the local instance as a seed otherwise we can bootstrap nodes with no data + return (!instance.getInstanceId().equalsIgnoreCase(DUMMY_INSTANCE_ID) && !instance.getHostName().equals(myInstance.getHostName())); + } else { + // auto_bootstrap = false indicates a freshly provisioned cluster. Some nodes in such a cluster must + // provide itself as a seed due to the changes in CASSANDRA-10134 which made it so the cluster would + // not start up when auto_bootstrap was false. This is because in 3.11 failing the shadow round + // (which will happen on bootup by definition) is acceptable for seeds, but not for non seeds + return (!instance.getInstanceId().equalsIgnoreCase(DUMMY_INSTANCE_ID)); + } } @Override @@ -301,6 +307,12 @@ public String getReplacedIp() { return replacedIp; } + public void setReplacedIp(String replacedIp) { + this.replacedIp = replacedIp; + if (!replacedIp.isEmpty()) + this.isReplace = true; + } + private static boolean isInstanceDummy(PriamInstance instance) { return instance.getInstanceId().equals(DUMMY_INSTANCE_ID); } diff --git a/priam/src/main/java/com/netflix/priam/resources/CassandraConfig.java b/priam/src/main/java/com/netflix/priam/resources/CassandraConfig.java index 13e076107..432e1ccda 100644 --- a/priam/src/main/java/com/netflix/priam/resources/CassandraConfig.java +++ b/priam/src/main/java/com/netflix/priam/resources/CassandraConfig.java @@ -25,8 +25,10 @@ import org.slf4j.LoggerFactory; import javax.ws.rs.GET; +import javax.ws.rs.POST; import javax.ws.rs.Path; import javax.ws.rs.Produces; +import javax.ws.rs.QueryParam; import javax.ws.rs.core.MediaType; import javax.ws.rs.core.Response; import java.io.IOException; @@ -109,6 +111,17 @@ public Response getReplacedIp() { } } + @POST + @Path("/set_replaced_ip") + public Response setReplacedIp(@QueryParam("ip") String ip) { + try { + priamServer.getId().setReplacedIp(ip); + return Response.ok().build(); + } catch (Exception e) { + logger.error("Error while overriding replacement ip", e); + return Response.serverError().build(); + } + } @GET @Path("/get_extra_env_params") diff --git a/priam/src/test/java/com/netflix/priam/backup/identity/InstanceIdentityTest.java b/priam/src/test/java/com/netflix/priam/backup/identity/InstanceIdentityTest.java index 203fcf86f..eb3bbfdb2 100644 --- a/priam/src/test/java/com/netflix/priam/backup/identity/InstanceIdentityTest.java +++ b/priam/src/test/java/com/netflix/priam/backup/identity/InstanceIdentityTest.java @@ -26,6 +26,8 @@ import java.util.List; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.assertFalse; public class InstanceIdentityTest extends InstanceTestUtils { @@ -66,11 +68,34 @@ public void testCreateToken() throws Exception } @Test - public void testGetSeeds() throws Exception + public void testGetSeedsAutobootstrapTrue() throws Exception { - createInstances(); - identity = createInstanceIdentity("az1", "fakeinstance1"); - assertEquals(3, identity.getSeeds().size()); + boolean previous = (Boolean) config.getFakeConfig("auto_bootstrap"); + try { + config.setFakeConfig("auto_bootstrap", true); + createInstances(); + identity = createInstanceIdentity("az1", "fakeinstance1"); + assertEquals(3, identity.getSeeds().size()); + assertFalse(identity.getSeeds().contains("fakeinstance1")); + } finally { + config.setFakeConfig("auto_bootstrap", previous); + } + + } + + @Test + public void testGetSeedsAutobootstrapFalse() throws Exception + { + boolean previous = (Boolean) config.getFakeConfig("auto_bootstrap"); + try { + config.setFakeConfig("auto_bootstrap", false); + createInstances(); + identity = createInstanceIdentity("az1", "fakeinstance1"); + assertEquals(3, identity.getSeeds().size()); + assertTrue(identity.getSeeds().contains("fakeinstance1")); + } finally { + config.setFakeConfig("auto_bootstrap", previous); + } } @Test diff --git a/priam/src/test/java/com/netflix/priam/config/FakeConfiguration.java b/priam/src/test/java/com/netflix/priam/config/FakeConfiguration.java index c17fc2f03..a1c7462b3 100644 --- a/priam/src/test/java/com/netflix/priam/config/FakeConfiguration.java +++ b/priam/src/test/java/com/netflix/priam/config/FakeConfiguration.java @@ -19,9 +19,7 @@ import com.google.common.collect.Lists; import com.google.inject.Singleton; -import com.netflix.priam.config.IConfiguration; import com.netflix.priam.tuner.JVMOption; -import com.netflix.priam.config.PriamConfiguration; import com.netflix.priam.tuner.GCType; import com.netflix.priam.identity.config.InstanceDataRetriever; import com.netflix.priam.identity.config.LocalInstanceDataRetriever; @@ -30,6 +28,7 @@ import java.io.File; import java.util.Arrays; +import java.util.HashMap; import java.util.List; import java.util.Map; @@ -44,6 +43,7 @@ public class FakeConfiguration implements IConfiguration public String zone; public String instance_id; public String restorePrefix; + public Map fakeConfig; public FakeConfiguration() { @@ -57,6 +57,16 @@ public FakeConfiguration(String region, String appName, String zone, String ins_ this.zone = zone; this.instance_id = ins_id; this.restorePrefix = ""; + fakeConfig = new HashMap<>(); + fakeConfig.put("auto_bootstrap", false); + } + + public Object getFakeConfig(String key) { + return fakeConfig.get(key); + } + + public void setFakeConfig(String key, Object value) { + fakeConfig.put(key, value); } @Override @@ -673,8 +683,7 @@ public String getCassYamlVal(String priamKey) { @Override public boolean getAutoBoostrap() { - // TODO Auto-generated method stub - return false; + return (Boolean) fakeConfig.getOrDefault("auto_bootstrap", false); } @Override diff --git a/priam/src/test/java/com/netflix/priam/identity/FakePriamInstanceFactory.java b/priam/src/test/java/com/netflix/priam/identity/FakePriamInstanceFactory.java index 5e8c83326..0a94a3f2e 100644 --- a/priam/src/test/java/com/netflix/priam/identity/FakePriamInstanceFactory.java +++ b/priam/src/test/java/com/netflix/priam/identity/FakePriamInstanceFactory.java @@ -39,7 +39,9 @@ public FakePriamInstanceFactory(IConfiguration config) @Override public List getAllIds(String appName) { - return new ArrayList(instances.values()); + List result = new ArrayList<>(instances.values()); + sort(result); + return result; } @Override From 261abdccbb964c2904b2ed97f169d3e4ba4137d7 Mon Sep 17 00:00:00 2001 From: Josh Snyder Date: Thu, 20 Sep 2018 18:29:16 -0700 Subject: [PATCH 002/228] Convert CRLF -> LF --- .../com/netflix/priam/aws/S3FileSystem.java | 376 ++++----- .../netflix/priam/aws/S3FileSystemBase.java | 754 +++++++++--------- 2 files changed, 565 insertions(+), 565 deletions(-) diff --git a/priam/src/main/java/com/netflix/priam/aws/S3FileSystem.java b/priam/src/main/java/com/netflix/priam/aws/S3FileSystem.java index 42ff0cb7f..a11a1f27e 100644 --- a/priam/src/main/java/com/netflix/priam/aws/S3FileSystem.java +++ b/priam/src/main/java/com/netflix/priam/aws/S3FileSystem.java @@ -1,189 +1,189 @@ -/* - * Copyright 2013 Netflix, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ -package com.netflix.priam.aws; - -import com.amazonaws.services.s3.AmazonS3Client; -import com.amazonaws.services.s3.S3ResponseMetadata; -import com.amazonaws.services.s3.model.*; -import com.google.inject.Inject; -import com.google.inject.Provider; -import com.google.inject.Singleton; -import com.google.inject.name.Named; -import com.netflix.priam.config.IConfiguration; -import com.netflix.priam.aws.auth.IS3Credential; -import com.netflix.priam.backup.AbstractBackupPath; -import com.netflix.priam.backup.BackupRestoreException; -import com.netflix.priam.backup.RangeReadInputStream; -import com.netflix.priam.compress.ICompression; -import com.netflix.priam.merics.BackupMetrics; -import com.netflix.priam.notification.BackupNotificationMgr; -import com.netflix.priam.utils.BoundedExponentialRetryCallable; -import org.apache.commons.io.IOUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import javax.management.MBeanServer; -import javax.management.ObjectName; -import java.io.*; -import java.lang.management.ManagementFactory; -import java.util.ArrayList; -import java.util.Collections; -import java.util.Iterator; -import java.util.List; -import java.util.concurrent.atomic.AtomicInteger; - -/** - * Implementation of IBackupFileSystem for S3 - */ -@Singleton -public class S3FileSystem extends S3FileSystemBase implements S3FileSystemMBean { - private static final Logger logger = LoggerFactory.getLogger(S3FileSystem.class); - - @Inject - public S3FileSystem(@Named("awss3roleassumption") IS3Credential cred, Provider pathProvider, - ICompression compress, - final IConfiguration config, - BackupMetrics backupMetrics, - BackupNotificationMgr backupNotificationMgr) { - super(pathProvider, compress, config, backupMetrics, backupNotificationMgr); - - MBeanServer mbs = ManagementFactory.getPlatformMBeanServer(); - try { - mbs.registerMBean(this, new ObjectName(MBEAN_NAME)); - } catch (Exception e) { - throw new RuntimeException(e); - } - - s3Client = AmazonS3Client.builder().withCredentials(cred.getAwsCredentialProvider()).withRegion(config.getDC()).build(); - } - - @Override - public void downloadFile(AbstractBackupPath path, OutputStream os) throws BackupRestoreException { - try { - RangeReadInputStream rris = new RangeReadInputStream(s3Client, getPrefix(this.config), path); - final long bufSize = MAX_BUFFERED_IN_STREAM_SIZE > path.getSize() ? path.getSize() : MAX_BUFFERED_IN_STREAM_SIZE; - compress.decompressAndClose(new BufferedInputStream(rris, (int) bufSize), os); - } catch (Exception e) { - throw new BackupRestoreException("Exception encountered downloading " + path.getRemotePath() + " from S3 bucket " + getPrefix(config) - + ", Msg: " + e.getMessage(), e); - } - } - - private void uploadMultipart(AbstractBackupPath path, InputStream in, long chunkSize) throws BackupRestoreException { - InitiateMultipartUploadRequest initRequest = new InitiateMultipartUploadRequest(config.getBackupPrefix(), path.getRemotePath()); - InitiateMultipartUploadResult initResponse = s3Client.initiateMultipartUpload(initRequest); - DataPart part = new DataPart(config.getBackupPrefix(), path.getRemotePath(), initResponse.getUploadId()); - List partETags = Collections.synchronizedList(new ArrayList()); - - try { - Iterator chunks = compress.compress(in, chunkSize); - // Upload parts. - int partNum = 0; - AtomicInteger partsUploaded = new AtomicInteger(0); - - while (chunks.hasNext()) { - byte[] chunk = chunks.next(); - rateLimiter.acquire(chunk.length); - DataPart dp = new DataPart(++partNum, chunk, config.getBackupPrefix(), path.getRemotePath(), initResponse.getUploadId()); - S3PartUploader partUploader = new S3PartUploader(s3Client, dp, partETags, partsUploaded); - executor.submit(partUploader); - bytesUploaded.addAndGet(chunk.length); - } - executor.sleepTillEmpty(); - logger.info("All chunks uploaded for file {}, num of expected parts:{}, num of actual uploaded parts: {}", path.getFileName(), partNum, partsUploaded.get()); - - if (partNum != partETags.size()) - throw new BackupRestoreException("Number of parts(" + partNum + ") does not match the uploaded parts(" + partETags.size() + ")"); - - CompleteMultipartUploadResult resultS3MultiPartUploadComplete = new S3PartUploader(s3Client, part, partETags).completeUpload(); - checkSuccessfulUpload(resultS3MultiPartUploadComplete, path); - - if (logger.isDebugEnabled()) { - final S3ResponseMetadata responseMetadata = s3Client.getCachedResponseMetadata(initRequest); - final String requestId = responseMetadata.getRequestId(); // "x-amz-request-id" header - final String hostId = responseMetadata.getHostId(); // "x-amz-id-2" header - logger.debug("S3 AWS x-amz-request-id[" + requestId + "], and x-amz-id-2[" + hostId + "]"); - } - - } catch (Exception e) { - throw encounterError(path, new S3PartUploader(s3Client, part, partETags), e); - } finally { - IOUtils.closeQuietly(in); - } - } - - @Override - public void uploadFile(AbstractBackupPath path, InputStream in, long chunkSize) throws BackupRestoreException { - - if (path.getSize() < chunkSize) { - //Upload file without using multipart upload as it will be more efficient. - if (logger.isDebugEnabled()) - logger.debug("Uploading file using put: {}", path.getRemotePath()); - - try (ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream()) { - Iterator chunkedStream = compress.compress(in, chunkSize); - while (chunkedStream.hasNext()) { - byteArrayOutputStream.write(chunkedStream.next()); - } - byte[] chunk = byteArrayOutputStream.toByteArray(); - rateLimiter.acquire(chunk.length); - ObjectMetadata objectMetadata = new ObjectMetadata(); - objectMetadata.setContentLength(chunk.length); - PutObjectRequest putObjectRequest = new PutObjectRequest(config.getBackupPrefix(), path.getRemotePath(), new ByteArrayInputStream(chunk), objectMetadata); - //Retry if failed. - PutObjectResult upload = new BoundedExponentialRetryCallable() { - @Override - public PutObjectResult retriableCall() throws Exception { - return s3Client.putObject(putObjectRequest); - } - }.retriableCall(); - - bytesUploaded.addAndGet(chunk.length); - - if (logger.isDebugEnabled()) - logger.debug("Successfully uploaded file with putObject: {} and etag: {}", path.getRemotePath(), upload.getETag()); - } catch (Exception e) { - throw encounterError(path, e); - } finally { - IOUtils.closeQuietly(in); - } - } else - uploadMultipart(path, in, chunkSize); - } - - - @Override - public int getActivecount() { - return executor.getActiveCount(); - } - - - @Override - /* - Note: provides same information as getBytesUploaded() but it's meant for S3FileSystemMBean object types. - */ - public long bytesUploaded() { - return super.bytesUploaded.get(); - } - - - @Override - public long bytesDownloaded() { - return bytesDownloaded.get(); - } - +/* + * Copyright 2013 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package com.netflix.priam.aws; + +import com.amazonaws.services.s3.AmazonS3Client; +import com.amazonaws.services.s3.S3ResponseMetadata; +import com.amazonaws.services.s3.model.*; +import com.google.inject.Inject; +import com.google.inject.Provider; +import com.google.inject.Singleton; +import com.google.inject.name.Named; +import com.netflix.priam.config.IConfiguration; +import com.netflix.priam.aws.auth.IS3Credential; +import com.netflix.priam.backup.AbstractBackupPath; +import com.netflix.priam.backup.BackupRestoreException; +import com.netflix.priam.backup.RangeReadInputStream; +import com.netflix.priam.compress.ICompression; +import com.netflix.priam.merics.BackupMetrics; +import com.netflix.priam.notification.BackupNotificationMgr; +import com.netflix.priam.utils.BoundedExponentialRetryCallable; +import org.apache.commons.io.IOUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.management.MBeanServer; +import javax.management.ObjectName; +import java.io.*; +import java.lang.management.ManagementFactory; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Iterator; +import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; + +/** + * Implementation of IBackupFileSystem for S3 + */ +@Singleton +public class S3FileSystem extends S3FileSystemBase implements S3FileSystemMBean { + private static final Logger logger = LoggerFactory.getLogger(S3FileSystem.class); + + @Inject + public S3FileSystem(@Named("awss3roleassumption") IS3Credential cred, Provider pathProvider, + ICompression compress, + final IConfiguration config, + BackupMetrics backupMetrics, + BackupNotificationMgr backupNotificationMgr) { + super(pathProvider, compress, config, backupMetrics, backupNotificationMgr); + + MBeanServer mbs = ManagementFactory.getPlatformMBeanServer(); + try { + mbs.registerMBean(this, new ObjectName(MBEAN_NAME)); + } catch (Exception e) { + throw new RuntimeException(e); + } + + s3Client = AmazonS3Client.builder().withCredentials(cred.getAwsCredentialProvider()).withRegion(config.getDC()).build(); + } + + @Override + public void downloadFile(AbstractBackupPath path, OutputStream os) throws BackupRestoreException { + try { + RangeReadInputStream rris = new RangeReadInputStream(s3Client, getPrefix(this.config), path); + final long bufSize = MAX_BUFFERED_IN_STREAM_SIZE > path.getSize() ? path.getSize() : MAX_BUFFERED_IN_STREAM_SIZE; + compress.decompressAndClose(new BufferedInputStream(rris, (int) bufSize), os); + } catch (Exception e) { + throw new BackupRestoreException("Exception encountered downloading " + path.getRemotePath() + " from S3 bucket " + getPrefix(config) + + ", Msg: " + e.getMessage(), e); + } + } + + private void uploadMultipart(AbstractBackupPath path, InputStream in, long chunkSize) throws BackupRestoreException { + InitiateMultipartUploadRequest initRequest = new InitiateMultipartUploadRequest(config.getBackupPrefix(), path.getRemotePath()); + InitiateMultipartUploadResult initResponse = s3Client.initiateMultipartUpload(initRequest); + DataPart part = new DataPart(config.getBackupPrefix(), path.getRemotePath(), initResponse.getUploadId()); + List partETags = Collections.synchronizedList(new ArrayList()); + + try { + Iterator chunks = compress.compress(in, chunkSize); + // Upload parts. + int partNum = 0; + AtomicInteger partsUploaded = new AtomicInteger(0); + + while (chunks.hasNext()) { + byte[] chunk = chunks.next(); + rateLimiter.acquire(chunk.length); + DataPart dp = new DataPart(++partNum, chunk, config.getBackupPrefix(), path.getRemotePath(), initResponse.getUploadId()); + S3PartUploader partUploader = new S3PartUploader(s3Client, dp, partETags, partsUploaded); + executor.submit(partUploader); + bytesUploaded.addAndGet(chunk.length); + } + executor.sleepTillEmpty(); + logger.info("All chunks uploaded for file {}, num of expected parts:{}, num of actual uploaded parts: {}", path.getFileName(), partNum, partsUploaded.get()); + + if (partNum != partETags.size()) + throw new BackupRestoreException("Number of parts(" + partNum + ") does not match the uploaded parts(" + partETags.size() + ")"); + + CompleteMultipartUploadResult resultS3MultiPartUploadComplete = new S3PartUploader(s3Client, part, partETags).completeUpload(); + checkSuccessfulUpload(resultS3MultiPartUploadComplete, path); + + if (logger.isDebugEnabled()) { + final S3ResponseMetadata responseMetadata = s3Client.getCachedResponseMetadata(initRequest); + final String requestId = responseMetadata.getRequestId(); // "x-amz-request-id" header + final String hostId = responseMetadata.getHostId(); // "x-amz-id-2" header + logger.debug("S3 AWS x-amz-request-id[" + requestId + "], and x-amz-id-2[" + hostId + "]"); + } + + } catch (Exception e) { + throw encounterError(path, new S3PartUploader(s3Client, part, partETags), e); + } finally { + IOUtils.closeQuietly(in); + } + } + + @Override + public void uploadFile(AbstractBackupPath path, InputStream in, long chunkSize) throws BackupRestoreException { + + if (path.getSize() < chunkSize) { + //Upload file without using multipart upload as it will be more efficient. + if (logger.isDebugEnabled()) + logger.debug("Uploading file using put: {}", path.getRemotePath()); + + try (ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream()) { + Iterator chunkedStream = compress.compress(in, chunkSize); + while (chunkedStream.hasNext()) { + byteArrayOutputStream.write(chunkedStream.next()); + } + byte[] chunk = byteArrayOutputStream.toByteArray(); + rateLimiter.acquire(chunk.length); + ObjectMetadata objectMetadata = new ObjectMetadata(); + objectMetadata.setContentLength(chunk.length); + PutObjectRequest putObjectRequest = new PutObjectRequest(config.getBackupPrefix(), path.getRemotePath(), new ByteArrayInputStream(chunk), objectMetadata); + //Retry if failed. + PutObjectResult upload = new BoundedExponentialRetryCallable() { + @Override + public PutObjectResult retriableCall() throws Exception { + return s3Client.putObject(putObjectRequest); + } + }.retriableCall(); + + bytesUploaded.addAndGet(chunk.length); + + if (logger.isDebugEnabled()) + logger.debug("Successfully uploaded file with putObject: {} and etag: {}", path.getRemotePath(), upload.getETag()); + } catch (Exception e) { + throw encounterError(path, e); + } finally { + IOUtils.closeQuietly(in); + } + } else + uploadMultipart(path, in, chunkSize); + } + + + @Override + public int getActivecount() { + return executor.getActiveCount(); + } + + + @Override + /* + Note: provides same information as getBytesUploaded() but it's meant for S3FileSystemMBean object types. + */ + public long bytesUploaded() { + return super.bytesUploaded.get(); + } + + + @Override + public long bytesDownloaded() { + return bytesDownloaded.get(); + } + } \ No newline at end of file diff --git a/priam/src/main/java/com/netflix/priam/aws/S3FileSystemBase.java b/priam/src/main/java/com/netflix/priam/aws/S3FileSystemBase.java index 6d651ac53..9ea13cbae 100755 --- a/priam/src/main/java/com/netflix/priam/aws/S3FileSystemBase.java +++ b/priam/src/main/java/com/netflix/priam/aws/S3FileSystemBase.java @@ -1,377 +1,377 @@ -/** - * Copyright 2017 Netflix, Inc. - *

- * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.netflix.priam.aws; - -import com.amazonaws.services.s3.AmazonS3; -import com.amazonaws.services.s3.model.AmazonS3Exception; -import com.amazonaws.services.s3.model.BucketLifecycleConfiguration; -import com.amazonaws.services.s3.model.BucketLifecycleConfiguration.Rule; -import com.amazonaws.services.s3.model.CompleteMultipartUploadResult; -import com.google.common.collect.Lists; -import com.google.common.util.concurrent.RateLimiter; -import com.google.inject.Provider; -import com.netflix.priam.config.IConfiguration; -import com.netflix.priam.backup.AbstractBackupPath; -import com.netflix.priam.backup.BackupRestoreException; -import com.netflix.priam.backup.IBackupFileSystem; -import com.netflix.priam.compress.ICompression; -import com.netflix.priam.merics.BackupMetrics; -import com.netflix.priam.notification.BackupEvent; -import com.netflix.priam.notification.BackupNotificationMgr; -import com.netflix.priam.notification.EventGenerator; -import com.netflix.priam.notification.EventObserver; -import com.netflix.priam.scheduler.BlockingSubmitThreadPoolExecutor; -import org.apache.commons.lang3.StringUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.File; -import java.io.InputStream; -import java.io.OutputStream; -import java.util.Date; -import java.util.Iterator; -import java.util.List; -import java.util.concurrent.CopyOnWriteArrayList; -import java.util.concurrent.LinkedBlockingQueue; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicLong; - -public abstract class S3FileSystemBase implements IBackupFileSystem, EventGenerator { - protected static final int MAX_CHUNKS = 10000; - protected static final long MAX_BUFFERED_IN_STREAM_SIZE = 5 * 1024 * 1024; - protected static final long UPLOAD_TIMEOUT = (2 * 60 * 60 * 1000L); - private static final Logger logger = LoggerFactory.getLogger(S3FileSystemBase.class); - //protected AtomicInteger uploadCount = new AtomicInteger(); - protected AtomicLong bytesUploaded = new AtomicLong(); //bytes uploaded per file - //protected AtomicInteger downloadCount = new AtomicInteger(); - protected AtomicLong bytesDownloaded = new AtomicLong(); - protected BackupMetrics backupMetrics; - protected AmazonS3 s3Client; - protected IConfiguration config; - protected Provider pathProvider; - protected ICompression compress; - protected BlockingSubmitThreadPoolExecutor executor; - protected RateLimiter rateLimiter; //a throttling mechanism, we can limit the amount of bytes uploaded to endpoint per second. - private final CopyOnWriteArrayList> observers = new CopyOnWriteArrayList<>(); - - public S3FileSystemBase(Provider pathProvider, - ICompression compress, - final IConfiguration config, - BackupMetrics backupMetrics, - BackupNotificationMgr backupNotificationMgr) { - this.pathProvider = pathProvider; - this.compress = compress; - this.config = config; - this.backupMetrics = backupMetrics; - - int threads = config.getMaxBackupUploadThreads(); - LinkedBlockingQueue queue = new LinkedBlockingQueue(threads); - this.executor = new BlockingSubmitThreadPoolExecutor(threads, queue, UPLOAD_TIMEOUT); - - double throttleLimit = config.getUploadThrottle(); - this.rateLimiter = RateLimiter.create(throttleLimit < 1 ? Double.MAX_VALUE : throttleLimit); - this.addObserver(backupNotificationMgr); - } - - public AmazonS3 getS3Client() { - return s3Client; - } - - /* - * A means to change the default handle to the S3 client. - */ - public void setS3Client(AmazonS3 client) { - s3Client = client; - } - - /** - * Get S3 prefix which will be used to locate S3 files - */ - protected String getPrefix(IConfiguration config) { - String prefix; - if (StringUtils.isNotBlank(config.getRestorePrefix())) - prefix = config.getRestorePrefix(); - else - prefix = config.getBackupPrefix(); - - String[] paths = prefix.split(String.valueOf(S3BackupPath.PATH_SEP)); - return paths[0]; - } - - @Override - public void cleanup() { - - AmazonS3 s3Client = getS3Client(); - String clusterPath = pathProvider.get().clusterPrefix(""); - logger.debug("Bucket: {}", config.getBackupPrefix()); - BucketLifecycleConfiguration lifeConfig = s3Client.getBucketLifecycleConfiguration(config.getBackupPrefix()); - logger.debug("Got bucket:{} lifecycle.{}", config.getBackupPrefix(), lifeConfig); - if (lifeConfig == null) { - lifeConfig = new BucketLifecycleConfiguration(); - List rules = Lists.newArrayList(); - lifeConfig.setRules(rules); - } - List rules = lifeConfig.getRules(); - if (updateLifecycleRule(config, rules, clusterPath)) { - if (rules.size() > 0) { - lifeConfig.setRules(rules); - s3Client.setBucketLifecycleConfiguration(config.getBackupPrefix(), lifeConfig); - } else - s3Client.deleteBucketLifecycleConfiguration(config.getBackupPrefix()); - } - - } - - private boolean updateLifecycleRule(IConfiguration config, List rules, String prefix) { - Rule rule = null; - for (BucketLifecycleConfiguration.Rule lcRule : rules) { - if (lcRule.getPrefix().equals(prefix)) { - rule = lcRule; - break; - } - } - if (rule == null && config.getBackupRetentionDays() <= 0) - return false; - if (rule != null && rule.getExpirationInDays() == config.getBackupRetentionDays()) { - logger.info("Cleanup rule already set"); - return false; - } - if (rule == null) { - // Create a new rule - rule = new BucketLifecycleConfiguration.Rule().withExpirationInDays(config.getBackupRetentionDays()).withPrefix(prefix); - rule.setStatus(BucketLifecycleConfiguration.ENABLED); - rule.setId(prefix); - rules.add(rule); - logger.info("Setting cleanup for {} to {} days", rule.getPrefix(), rule.getExpirationInDays()); - } else if (config.getBackupRetentionDays() > 0) { - logger.info("Setting cleanup for {} to {} days", rule.getPrefix(), config.getBackupRetentionDays()); - rule.setExpirationInDays(config.getBackupRetentionDays()); - } else { - logger.info("Removing cleanup rule for {}", rule.getPrefix()); - rules.remove(rule); - } - return true; - } - - /* - @param path - representation of the file uploaded - @param start time of upload, in millisecs - @param completion time of upload, in millsecs - */ - private void postProcessingPerFile(AbstractBackupPath path, long startTimeInMilliSecs, long completedTimeInMilliSecs) { - //Publish upload rate for each uploaded file - try { - long sizeInBytes = path.getSize(); - long elapseTimeInMillisecs = completedTimeInMilliSecs - startTimeInMilliSecs; - long elapseTimeInSecs = elapseTimeInMillisecs / 1000; //converting millis to seconds as 1000m in 1 second - long bytesReadPerSec = 0; - Double speedInKBps = 0.0; - if (elapseTimeInSecs > 0 && sizeInBytes > 0) { - bytesReadPerSec = sizeInBytes / elapseTimeInSecs; - speedInKBps = bytesReadPerSec / 1024D; - } else { - bytesReadPerSec = sizeInBytes; //we uploaded the whole file in less than a sec - speedInKBps = (double) sizeInBytes; - } - - logger.info("Upload rate for file: {}" - + ", elapsse time in sec(s): {}" - + ", KB per sec: {}", - path.getFileName(), elapseTimeInSecs, speedInKBps); - backupMetrics.recordUploadRate(sizeInBytes); - } catch (Exception e) { - logger.error("Post processing of file {} failed, not fatal.", path.getFileName(), e); - } - } - - /* - Reinitializtion which should be performed before uploading a file - */ - protected void reinitialize() { - bytesUploaded = new AtomicLong(0); //initialize - } - - /* - @param file uploaded to S3 - @param a list of unique parts uploaded to S3 for file - */ - protected void logDiagnosticInfo(AbstractBackupPath fileUploaded, CompleteMultipartUploadResult res) { - File f = fileUploaded.getBackupFile(); - String fName = f.getAbsolutePath(); - logger.info("Uploaded file: {}, object eTag: {}", fName, res.getETag()); - } - - @Override - public void upload(AbstractBackupPath path, InputStream in) throws BackupRestoreException { - reinitialize(); //perform before file upload - long chunkSize = config.getBackupChunkSize(); - if (path.getSize() > 0) - chunkSize = (path.getSize() / chunkSize >= MAX_CHUNKS) ? (path.getSize() / (MAX_CHUNKS - 1)) : chunkSize; //compute the size of each block we will upload to endpoint - - logger.info("Uploading to {}/{} with chunk size {}", config.getBackupPrefix(), path.getRemotePath(), chunkSize); - - long startTime = System.nanoTime(); //initialize for each file upload - notifyEventStart(new BackupEvent(path)); - uploadFile(path, in, chunkSize); - long completedTime = System.nanoTime(); - postProcessingPerFile(path, TimeUnit.NANOSECONDS.toMillis(startTime), TimeUnit.NANOSECONDS.toMillis(completedTime)); - notifyEventSuccess(new BackupEvent(path)); - backupMetrics.incrementValidUploads(); - } - - protected void checkSuccessfulUpload(CompleteMultipartUploadResult resultS3MultiPartUploadComplete, AbstractBackupPath path) throws BackupRestoreException { - if (null != resultS3MultiPartUploadComplete && null != resultS3MultiPartUploadComplete.getETag()) { - String eTagObjectId = resultS3MultiPartUploadComplete.getETag(); //unique id of the whole object - logDiagnosticInfo(path, resultS3MultiPartUploadComplete); - } else { - this.backupMetrics.incrementInvalidUploads(); - throw new BackupRestoreException("Error uploading file as ETag or CompleteMultipartUploadResult is NULL -" + path.getFileName()); - } - } - - - protected BackupRestoreException encounterError(AbstractBackupPath path, S3PartUploader s3PartUploader, Exception e) { - s3PartUploader.abortUpload(); - return encounterError(path, e); - } - - protected BackupRestoreException encounterError(AbstractBackupPath path, Exception e) { - this.backupMetrics.incrementInvalidUploads(); - if (e instanceof AmazonS3Exception) { - AmazonS3Exception a = (AmazonS3Exception) e; - String amazoneErrorCode = a.getErrorCode(); - if (amazoneErrorCode != null && !amazoneErrorCode.isEmpty()) { - if (amazoneErrorCode.equalsIgnoreCase("slowdown")) { - backupMetrics.incrementAwsSlowDownException(1); - logger.warn("Received slow down from AWS when uploading file: {}", path.getFileName()); - } - } - } - - logger.error("Error uploading file {}, a datapart was not uploaded.", path.getFileName(), e); - notifyEventFailure(new BackupEvent(path)); - return new BackupRestoreException("Error uploading file " + path.getFileName(), e); - } - - abstract void uploadFile(AbstractBackupPath path, InputStream in, long chunkSize) throws BackupRestoreException; - - /** - * This method does exactly as other download method.(Supposed to be overridden) - * filePath parameter provides the diskPath of the downloaded file. - * This path can be used to correlate the files which are Streamed In - * during Incremental Restores - */ - @Override - public void download(AbstractBackupPath path, OutputStream os, - String filePath) throws BackupRestoreException { - try { - // Calling original Download method - download(path, os); - } catch (Exception e) { - throw new BackupRestoreException(e.getMessage(), e); - } - - } - - @Override - public void download(AbstractBackupPath path, OutputStream os) throws BackupRestoreException { - logger.info("Downloading {} from S3 bucket {}", path.getRemotePath(), getPrefix(this.config)); - long contentLen = s3Client.getObjectMetadata(getPrefix(config), path.getRemotePath()).getContentLength(); - path.setSize(contentLen); - try { - downloadFile(path, os); - bytesDownloaded.addAndGet(contentLen); - backupMetrics.incrementValidDownloads(); - } catch (BackupRestoreException e) { - backupMetrics.incrementInvalidDownloads(); - throw e; - } - } - - protected abstract void downloadFile(AbstractBackupPath path, OutputStream os) throws BackupRestoreException; - - @Override - public long getBytesUploaded() { - return bytesUploaded.get(); - } - - @Override - public long getAWSSlowDownExceptionCounter() { - return backupMetrics.getAwsSlowDownException(); - } - - public long downloadCount() { - return backupMetrics.getValidDownloads(); - } - - public long uploadCount() { - return backupMetrics.getValidUploads(); - } - - @Override - public void shutdown() { - if (executor != null) - executor.shutdown(); - - } - - @Override - public Iterator listPrefixes(Date date) { - return new S3PrefixIterator(config, pathProvider, s3Client, date); - } - - @Override - public Iterator list(String path, Date start, Date till) { - return new S3FileIterator(pathProvider, s3Client, path, start, till); - } - - - @Override - public final void addObserver(EventObserver observer) { - if (observer == null) - throw new NullPointerException("observer must not be null."); - - observers.addIfAbsent(observer); - } - - @Override - public void removeObserver(EventObserver observer) { - if (observer == null) - throw new NullPointerException("observer must not be null."); - - observers.remove(observer); - } - - @Override - public void notifyEventStart(BackupEvent event) { - observers.forEach(eventObserver -> eventObserver.updateEventStart(event)); - } - - @Override - public void notifyEventSuccess(BackupEvent event) { - observers.forEach(eventObserver -> eventObserver.updateEventSuccess(event)); - } - - @Override - public void notifyEventFailure(BackupEvent event) { - observers.forEach(eventObserver -> eventObserver.updateEventFailure(event)); - } - - @Override - public void notifyEventStop(BackupEvent event) { - observers.forEach(eventObserver -> eventObserver.updateEventStop(event)); - } -} +/** + * Copyright 2017 Netflix, Inc. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.netflix.priam.aws; + +import com.amazonaws.services.s3.AmazonS3; +import com.amazonaws.services.s3.model.AmazonS3Exception; +import com.amazonaws.services.s3.model.BucketLifecycleConfiguration; +import com.amazonaws.services.s3.model.BucketLifecycleConfiguration.Rule; +import com.amazonaws.services.s3.model.CompleteMultipartUploadResult; +import com.google.common.collect.Lists; +import com.google.common.util.concurrent.RateLimiter; +import com.google.inject.Provider; +import com.netflix.priam.config.IConfiguration; +import com.netflix.priam.backup.AbstractBackupPath; +import com.netflix.priam.backup.BackupRestoreException; +import com.netflix.priam.backup.IBackupFileSystem; +import com.netflix.priam.compress.ICompression; +import com.netflix.priam.merics.BackupMetrics; +import com.netflix.priam.notification.BackupEvent; +import com.netflix.priam.notification.BackupNotificationMgr; +import com.netflix.priam.notification.EventGenerator; +import com.netflix.priam.notification.EventObserver; +import com.netflix.priam.scheduler.BlockingSubmitThreadPoolExecutor; +import org.apache.commons.lang3.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.io.InputStream; +import java.io.OutputStream; +import java.util.Date; +import java.util.Iterator; +import java.util.List; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; + +public abstract class S3FileSystemBase implements IBackupFileSystem, EventGenerator { + protected static final int MAX_CHUNKS = 10000; + protected static final long MAX_BUFFERED_IN_STREAM_SIZE = 5 * 1024 * 1024; + protected static final long UPLOAD_TIMEOUT = (2 * 60 * 60 * 1000L); + private static final Logger logger = LoggerFactory.getLogger(S3FileSystemBase.class); + //protected AtomicInteger uploadCount = new AtomicInteger(); + protected AtomicLong bytesUploaded = new AtomicLong(); //bytes uploaded per file + //protected AtomicInteger downloadCount = new AtomicInteger(); + protected AtomicLong bytesDownloaded = new AtomicLong(); + protected BackupMetrics backupMetrics; + protected AmazonS3 s3Client; + protected IConfiguration config; + protected Provider pathProvider; + protected ICompression compress; + protected BlockingSubmitThreadPoolExecutor executor; + protected RateLimiter rateLimiter; //a throttling mechanism, we can limit the amount of bytes uploaded to endpoint per second. + private final CopyOnWriteArrayList> observers = new CopyOnWriteArrayList<>(); + + public S3FileSystemBase(Provider pathProvider, + ICompression compress, + final IConfiguration config, + BackupMetrics backupMetrics, + BackupNotificationMgr backupNotificationMgr) { + this.pathProvider = pathProvider; + this.compress = compress; + this.config = config; + this.backupMetrics = backupMetrics; + + int threads = config.getMaxBackupUploadThreads(); + LinkedBlockingQueue queue = new LinkedBlockingQueue(threads); + this.executor = new BlockingSubmitThreadPoolExecutor(threads, queue, UPLOAD_TIMEOUT); + + double throttleLimit = config.getUploadThrottle(); + this.rateLimiter = RateLimiter.create(throttleLimit < 1 ? Double.MAX_VALUE : throttleLimit); + this.addObserver(backupNotificationMgr); + } + + public AmazonS3 getS3Client() { + return s3Client; + } + + /* + * A means to change the default handle to the S3 client. + */ + public void setS3Client(AmazonS3 client) { + s3Client = client; + } + + /** + * Get S3 prefix which will be used to locate S3 files + */ + protected String getPrefix(IConfiguration config) { + String prefix; + if (StringUtils.isNotBlank(config.getRestorePrefix())) + prefix = config.getRestorePrefix(); + else + prefix = config.getBackupPrefix(); + + String[] paths = prefix.split(String.valueOf(S3BackupPath.PATH_SEP)); + return paths[0]; + } + + @Override + public void cleanup() { + + AmazonS3 s3Client = getS3Client(); + String clusterPath = pathProvider.get().clusterPrefix(""); + logger.debug("Bucket: {}", config.getBackupPrefix()); + BucketLifecycleConfiguration lifeConfig = s3Client.getBucketLifecycleConfiguration(config.getBackupPrefix()); + logger.debug("Got bucket:{} lifecycle.{}", config.getBackupPrefix(), lifeConfig); + if (lifeConfig == null) { + lifeConfig = new BucketLifecycleConfiguration(); + List rules = Lists.newArrayList(); + lifeConfig.setRules(rules); + } + List rules = lifeConfig.getRules(); + if (updateLifecycleRule(config, rules, clusterPath)) { + if (rules.size() > 0) { + lifeConfig.setRules(rules); + s3Client.setBucketLifecycleConfiguration(config.getBackupPrefix(), lifeConfig); + } else + s3Client.deleteBucketLifecycleConfiguration(config.getBackupPrefix()); + } + + } + + private boolean updateLifecycleRule(IConfiguration config, List rules, String prefix) { + Rule rule = null; + for (BucketLifecycleConfiguration.Rule lcRule : rules) { + if (lcRule.getPrefix().equals(prefix)) { + rule = lcRule; + break; + } + } + if (rule == null && config.getBackupRetentionDays() <= 0) + return false; + if (rule != null && rule.getExpirationInDays() == config.getBackupRetentionDays()) { + logger.info("Cleanup rule already set"); + return false; + } + if (rule == null) { + // Create a new rule + rule = new BucketLifecycleConfiguration.Rule().withExpirationInDays(config.getBackupRetentionDays()).withPrefix(prefix); + rule.setStatus(BucketLifecycleConfiguration.ENABLED); + rule.setId(prefix); + rules.add(rule); + logger.info("Setting cleanup for {} to {} days", rule.getPrefix(), rule.getExpirationInDays()); + } else if (config.getBackupRetentionDays() > 0) { + logger.info("Setting cleanup for {} to {} days", rule.getPrefix(), config.getBackupRetentionDays()); + rule.setExpirationInDays(config.getBackupRetentionDays()); + } else { + logger.info("Removing cleanup rule for {}", rule.getPrefix()); + rules.remove(rule); + } + return true; + } + + /* + @param path - representation of the file uploaded + @param start time of upload, in millisecs + @param completion time of upload, in millsecs + */ + private void postProcessingPerFile(AbstractBackupPath path, long startTimeInMilliSecs, long completedTimeInMilliSecs) { + //Publish upload rate for each uploaded file + try { + long sizeInBytes = path.getSize(); + long elapseTimeInMillisecs = completedTimeInMilliSecs - startTimeInMilliSecs; + long elapseTimeInSecs = elapseTimeInMillisecs / 1000; //converting millis to seconds as 1000m in 1 second + long bytesReadPerSec = 0; + Double speedInKBps = 0.0; + if (elapseTimeInSecs > 0 && sizeInBytes > 0) { + bytesReadPerSec = sizeInBytes / elapseTimeInSecs; + speedInKBps = bytesReadPerSec / 1024D; + } else { + bytesReadPerSec = sizeInBytes; //we uploaded the whole file in less than a sec + speedInKBps = (double) sizeInBytes; + } + + logger.info("Upload rate for file: {}" + + ", elapsse time in sec(s): {}" + + ", KB per sec: {}", + path.getFileName(), elapseTimeInSecs, speedInKBps); + backupMetrics.recordUploadRate(sizeInBytes); + } catch (Exception e) { + logger.error("Post processing of file {} failed, not fatal.", path.getFileName(), e); + } + } + + /* + Reinitializtion which should be performed before uploading a file + */ + protected void reinitialize() { + bytesUploaded = new AtomicLong(0); //initialize + } + + /* + @param file uploaded to S3 + @param a list of unique parts uploaded to S3 for file + */ + protected void logDiagnosticInfo(AbstractBackupPath fileUploaded, CompleteMultipartUploadResult res) { + File f = fileUploaded.getBackupFile(); + String fName = f.getAbsolutePath(); + logger.info("Uploaded file: {}, object eTag: {}", fName, res.getETag()); + } + + @Override + public void upload(AbstractBackupPath path, InputStream in) throws BackupRestoreException { + reinitialize(); //perform before file upload + long chunkSize = config.getBackupChunkSize(); + if (path.getSize() > 0) + chunkSize = (path.getSize() / chunkSize >= MAX_CHUNKS) ? (path.getSize() / (MAX_CHUNKS - 1)) : chunkSize; //compute the size of each block we will upload to endpoint + + logger.info("Uploading to {}/{} with chunk size {}", config.getBackupPrefix(), path.getRemotePath(), chunkSize); + + long startTime = System.nanoTime(); //initialize for each file upload + notifyEventStart(new BackupEvent(path)); + uploadFile(path, in, chunkSize); + long completedTime = System.nanoTime(); + postProcessingPerFile(path, TimeUnit.NANOSECONDS.toMillis(startTime), TimeUnit.NANOSECONDS.toMillis(completedTime)); + notifyEventSuccess(new BackupEvent(path)); + backupMetrics.incrementValidUploads(); + } + + protected void checkSuccessfulUpload(CompleteMultipartUploadResult resultS3MultiPartUploadComplete, AbstractBackupPath path) throws BackupRestoreException { + if (null != resultS3MultiPartUploadComplete && null != resultS3MultiPartUploadComplete.getETag()) { + String eTagObjectId = resultS3MultiPartUploadComplete.getETag(); //unique id of the whole object + logDiagnosticInfo(path, resultS3MultiPartUploadComplete); + } else { + this.backupMetrics.incrementInvalidUploads(); + throw new BackupRestoreException("Error uploading file as ETag or CompleteMultipartUploadResult is NULL -" + path.getFileName()); + } + } + + + protected BackupRestoreException encounterError(AbstractBackupPath path, S3PartUploader s3PartUploader, Exception e) { + s3PartUploader.abortUpload(); + return encounterError(path, e); + } + + protected BackupRestoreException encounterError(AbstractBackupPath path, Exception e) { + this.backupMetrics.incrementInvalidUploads(); + if (e instanceof AmazonS3Exception) { + AmazonS3Exception a = (AmazonS3Exception) e; + String amazoneErrorCode = a.getErrorCode(); + if (amazoneErrorCode != null && !amazoneErrorCode.isEmpty()) { + if (amazoneErrorCode.equalsIgnoreCase("slowdown")) { + backupMetrics.incrementAwsSlowDownException(1); + logger.warn("Received slow down from AWS when uploading file: {}", path.getFileName()); + } + } + } + + logger.error("Error uploading file {}, a datapart was not uploaded.", path.getFileName(), e); + notifyEventFailure(new BackupEvent(path)); + return new BackupRestoreException("Error uploading file " + path.getFileName(), e); + } + + abstract void uploadFile(AbstractBackupPath path, InputStream in, long chunkSize) throws BackupRestoreException; + + /** + * This method does exactly as other download method.(Supposed to be overridden) + * filePath parameter provides the diskPath of the downloaded file. + * This path can be used to correlate the files which are Streamed In + * during Incremental Restores + */ + @Override + public void download(AbstractBackupPath path, OutputStream os, + String filePath) throws BackupRestoreException { + try { + // Calling original Download method + download(path, os); + } catch (Exception e) { + throw new BackupRestoreException(e.getMessage(), e); + } + + } + + @Override + public void download(AbstractBackupPath path, OutputStream os) throws BackupRestoreException { + logger.info("Downloading {} from S3 bucket {}", path.getRemotePath(), getPrefix(this.config)); + long contentLen = s3Client.getObjectMetadata(getPrefix(config), path.getRemotePath()).getContentLength(); + path.setSize(contentLen); + try { + downloadFile(path, os); + bytesDownloaded.addAndGet(contentLen); + backupMetrics.incrementValidDownloads(); + } catch (BackupRestoreException e) { + backupMetrics.incrementInvalidDownloads(); + throw e; + } + } + + protected abstract void downloadFile(AbstractBackupPath path, OutputStream os) throws BackupRestoreException; + + @Override + public long getBytesUploaded() { + return bytesUploaded.get(); + } + + @Override + public long getAWSSlowDownExceptionCounter() { + return backupMetrics.getAwsSlowDownException(); + } + + public long downloadCount() { + return backupMetrics.getValidDownloads(); + } + + public long uploadCount() { + return backupMetrics.getValidUploads(); + } + + @Override + public void shutdown() { + if (executor != null) + executor.shutdown(); + + } + + @Override + public Iterator listPrefixes(Date date) { + return new S3PrefixIterator(config, pathProvider, s3Client, date); + } + + @Override + public Iterator list(String path, Date start, Date till) { + return new S3FileIterator(pathProvider, s3Client, path, start, till); + } + + + @Override + public final void addObserver(EventObserver observer) { + if (observer == null) + throw new NullPointerException("observer must not be null."); + + observers.addIfAbsent(observer); + } + + @Override + public void removeObserver(EventObserver observer) { + if (observer == null) + throw new NullPointerException("observer must not be null."); + + observers.remove(observer); + } + + @Override + public void notifyEventStart(BackupEvent event) { + observers.forEach(eventObserver -> eventObserver.updateEventStart(event)); + } + + @Override + public void notifyEventSuccess(BackupEvent event) { + observers.forEach(eventObserver -> eventObserver.updateEventSuccess(event)); + } + + @Override + public void notifyEventFailure(BackupEvent event) { + observers.forEach(eventObserver -> eventObserver.updateEventFailure(event)); + } + + @Override + public void notifyEventStop(BackupEvent event) { + observers.forEach(eventObserver -> eventObserver.updateEventStop(event)); + } +} From ca98a639a65f1875c5160437062efd96620629cd Mon Sep 17 00:00:00 2001 From: Josh Snyder Date: Fri, 7 Sep 2018 15:56:55 -0700 Subject: [PATCH 003/228] Add last modified time to uploaded S3 object metadata (port of #718 onto 3.11) --- .../com/netflix/priam/aws/S3FileSystem.java | 19 +++++++++++- .../priam/backup/AbstractBackupPath.java | 29 ++++++++++++++++++- 2 files changed, 46 insertions(+), 2 deletions(-) diff --git a/priam/src/main/java/com/netflix/priam/aws/S3FileSystem.java b/priam/src/main/java/com/netflix/priam/aws/S3FileSystem.java index a11a1f27e..f5cc08239 100644 --- a/priam/src/main/java/com/netflix/priam/aws/S3FileSystem.java +++ b/priam/src/main/java/com/netflix/priam/aws/S3FileSystem.java @@ -83,8 +83,25 @@ public void downloadFile(AbstractBackupPath path, OutputStream os) throws Backup } } + private ObjectMetadata getObjectMetadata(AbstractBackupPath path) { + ObjectMetadata ret = new ObjectMetadata(); + long lastModified = path.getLastModified(); + + if (lastModified != 0) { + ret.addUserMetadata("local-modification-time", Long.toString(lastModified)); + } + + long fileSize = path.getSize(); + if (fileSize != 0) { + ret.addUserMetadata("local-size", Long.toString(fileSize)); + } + return ret; + } + private void uploadMultipart(AbstractBackupPath path, InputStream in, long chunkSize) throws BackupRestoreException { InitiateMultipartUploadRequest initRequest = new InitiateMultipartUploadRequest(config.getBackupPrefix(), path.getRemotePath()); + + initRequest.withObjectMetadata(getObjectMetadata(path)); InitiateMultipartUploadResult initResponse = s3Client.initiateMultipartUpload(initRequest); DataPart part = new DataPart(config.getBackupPrefix(), path.getRemotePath(), initResponse.getUploadId()); List partETags = Collections.synchronizedList(new ArrayList()); @@ -141,7 +158,7 @@ public void uploadFile(AbstractBackupPath path, InputStream in, long chunkSize) } byte[] chunk = byteArrayOutputStream.toByteArray(); rateLimiter.acquire(chunk.length); - ObjectMetadata objectMetadata = new ObjectMetadata(); + ObjectMetadata objectMetadata = getObjectMetadata(path); objectMetadata.setContentLength(chunk.length); PutObjectRequest putObjectRequest = new PutObjectRequest(config.getBackupPrefix(), path.getRemotePath(), new ByteArrayInputStream(chunk), objectMetadata); //Retry if failed. diff --git a/priam/src/main/java/com/netflix/priam/backup/AbstractBackupPath.java b/priam/src/main/java/com/netflix/priam/backup/AbstractBackupPath.java index 28b049e51..a4315fc76 100644 --- a/priam/src/main/java/com/netflix/priam/backup/AbstractBackupPath.java +++ b/priam/src/main/java/com/netflix/priam/backup/AbstractBackupPath.java @@ -69,6 +69,7 @@ public static boolean isDataFile(BackupFileType type){ protected final InstanceIdentity factory; protected final IConfiguration config; protected File backupFile; + protected long lastModified = 0; protected Date uploadedTs; public AbstractBackupPath(IConfiguration config, InstanceIdentity factory) { @@ -86,7 +87,29 @@ public Date parseDate(String s) { public InputStream localReader() throws IOException { assert backupFile != null; - return new RafInputStream(new RandomAccessFile(backupFile, "r")); + + InputStream ret = null; + + while (true) { + if (ret != null) { + ret.close(); + } + + lastModified = backupFile.lastModified(); + ret = new RafInputStream(new RandomAccessFile(backupFile, "r")); + + // Verify that the file hasn't changed since we opened it. + // We could avoid this flow by using the fstat() system call, + // but I see no way to do that (easily) from the JVM. + // The JVM returns the last modified time in milliseconds, + // but on Linux systems tested, it appears to be using the + // stat.st_mtime result, which is accurate only to seconds. + if (backupFile.lastModified() == lastModified) { + break; + } + } + + return ret; } public void parseLocal(File file, BackupFileType type) throws ParseException { @@ -280,6 +303,10 @@ public Date getUploadedTs() { return this.uploadedTs; } + public long getLastModified() { + return lastModified; + } + public static class RafInputStream extends InputStream { private RandomAccessFile raf; From 8c9dc48ee3d33fcc14b206cd59dd7c009c05e992 Mon Sep 17 00:00:00 2001 From: Arun Agrawal Date: Mon, 1 Oct 2018 16:46:18 -0700 Subject: [PATCH 004/228] changelog update --- CHANGELOG.md | 4 ++++ build.gradle | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b6576c162..16d4df3db 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,8 @@ # Changelog + +## 2018/09/10 3.11.31 +* (#715) Bug Fix: Fix the bootstrap issue. Do not provide yourself as seed node if cluster is already up and running as it will lead to data loss. + ## 2018/08/20 3.11.30 ***WARNING*** THIS IS A BREAKING RELEASE ### New Feature diff --git a/build.gradle b/build.gradle index 5e0628a90..c430a4620 100644 --- a/build.gradle +++ b/build.gradle @@ -1,5 +1,5 @@ plugins { - id 'nebula.netflixoss' version '5.1.1' + id 'nebula.netflixoss' version '5.2.0' } ext.githubProjectName = 'Priam' From 2ca90d9aad7469117c5c40eb5decd0cfaa29ee42 Mon Sep 17 00:00:00 2001 From: Arun Agrawal Date: Mon, 1 Oct 2018 18:01:01 -0700 Subject: [PATCH 005/228] include root project name --- settings.gradle | 1 + 1 file changed, 1 insertion(+) diff --git a/settings.gradle b/settings.gradle index b772ee1bd..dfe759d98 100644 --- a/settings.gradle +++ b/settings.gradle @@ -1 +1,2 @@ +rootProject.name = 'Priam' include 'priam','priam-web','priam-cass-extensions','priam-dse-extensions' From 107662b9b5d9b61f61918986dcfd1081db39c1b2 Mon Sep 17 00:00:00 2001 From: Arun Agrawal Date: Tue, 2 Oct 2018 13:28:03 -0700 Subject: [PATCH 006/228] change git depth to 100 for PR to succeed. --- .travis.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.travis.yml b/.travis.yml index 60fe9e378..2a1b23979 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,6 +2,8 @@ language: java jdk: - oraclejdk8 sudo: false +git: + depth: 100 install: ./installViaTravis.sh script: ./buildViaTravis.sh cache: From d615e6a2c786a940fee8d9b0ff468f51d75293c4 Mon Sep 17 00:00:00 2001 From: Joseph Lynch Date: Tue, 2 Oct 2018 13:56:07 -0700 Subject: [PATCH 007/228] Continue trying to upload incrementals all the time (#727) We have reports of missing incrementals. The incremental upload path is somewhat of a maze of retries and it turns out that we ultimately were not uploading with retries and we wouldn't re-enqueue because the hash that shadowed the queue didn't get updated. We should ... not do those things. This cherry-picks 3f21bc7 --- .../com/netflix/priam/aws/S3FileSystem.java | 4 +- .../com/netflix/priam/aws/S3PartUploader.java | 10 +++-- .../netflix/priam/backup/AbstractBackup.java | 44 +++++++++---------- .../parallel/CassandraBackupQueueMgr.java | 44 ++++--------------- .../priam/backup/parallel/ITaskQueueMgr.java | 18 +++++--- .../parallel/IncrementalBackupProducer.java | 3 +- .../backup/parallel/IncrementalConsumer.java | 22 +++++++--- .../parallel/IncrementalConsumerMgr.java | 37 ++++++++++++---- .../netflix/priam/config/IConfiguration.java | 28 +++++++----- .../priam/config/PriamConfiguration.java | 2 +- .../BoundedExponentialRetryCallable.java | 6 +-- .../priam/config/FakeConfiguration.java | 4 +- 12 files changed, 120 insertions(+), 102 deletions(-) diff --git a/priam/src/main/java/com/netflix/priam/aws/S3FileSystem.java b/priam/src/main/java/com/netflix/priam/aws/S3FileSystem.java index f5cc08239..a86e495c1 100644 --- a/priam/src/main/java/com/netflix/priam/aws/S3FileSystem.java +++ b/priam/src/main/java/com/netflix/priam/aws/S3FileSystem.java @@ -162,12 +162,12 @@ public void uploadFile(AbstractBackupPath path, InputStream in, long chunkSize) objectMetadata.setContentLength(chunk.length); PutObjectRequest putObjectRequest = new PutObjectRequest(config.getBackupPrefix(), path.getRemotePath(), new ByteArrayInputStream(chunk), objectMetadata); //Retry if failed. - PutObjectResult upload = new BoundedExponentialRetryCallable() { + PutObjectResult upload = new BoundedExponentialRetryCallable(1000, 10000, 5) { @Override public PutObjectResult retriableCall() throws Exception { return s3Client.putObject(putObjectRequest); } - }.retriableCall(); + }.call(); bytesUploaded.addAndGet(chunk.length); diff --git a/priam/src/main/java/com/netflix/priam/aws/S3PartUploader.java b/priam/src/main/java/com/netflix/priam/aws/S3PartUploader.java index 1d7c1250d..8c15b7c06 100644 --- a/priam/src/main/java/com/netflix/priam/aws/S3PartUploader.java +++ b/priam/src/main/java/com/netflix/priam/aws/S3PartUploader.java @@ -20,7 +20,7 @@ import com.amazonaws.services.s3.AmazonS3; import com.amazonaws.services.s3.model.*; import com.netflix.priam.backup.BackupRestoreException; -import com.netflix.priam.utils.RetryableCallable; +import com.netflix.priam.utils.BoundedExponentialRetryCallable; import com.netflix.priam.utils.SystemUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -29,7 +29,8 @@ import java.util.List; import java.util.concurrent.atomic.AtomicInteger; -public class S3PartUploader extends RetryableCallable { +public class S3PartUploader extends BoundedExponentialRetryCallable +{ private final AmazonS3 client; private DataPart dataPart; private List partETags; @@ -37,16 +38,17 @@ public class S3PartUploader extends RetryableCallable { private static final Logger logger = LoggerFactory.getLogger(S3PartUploader.class); private static final int MAX_RETRIES = 5; + private static final int DEFAULT_MIN_SLEEP_MS = 200; public S3PartUploader(AmazonS3 client, DataPart dp, List partETags) { - super(MAX_RETRIES, RetryableCallable.DEFAULT_WAIT_TIME); + super(DEFAULT_MIN_SLEEP_MS, BoundedExponentialRetryCallable.MAX_SLEEP, MAX_RETRIES); this.client = client; this.dataPart = dp; this.partETags = partETags; } public S3PartUploader(AmazonS3 client, DataPart dp, List partETags, AtomicInteger partsUploaded) { - super(MAX_RETRIES, RetryableCallable.DEFAULT_WAIT_TIME); + super(DEFAULT_MIN_SLEEP_MS, BoundedExponentialRetryCallable.MAX_SLEEP, MAX_RETRIES); this.client = client; this.dataPart = dp; this.partETags = partETags; diff --git a/priam/src/main/java/com/netflix/priam/backup/AbstractBackup.java b/priam/src/main/java/com/netflix/priam/backup/AbstractBackup.java index 611aa6ab8..adcceded7 100644 --- a/priam/src/main/java/com/netflix/priam/backup/AbstractBackup.java +++ b/priam/src/main/java/com/netflix/priam/backup/AbstractBackup.java @@ -76,7 +76,12 @@ List upload(File parent, final BackupFileType type) throws E try { logger.info("About to upload file {} for backup", file.getCanonicalFile()); - AbstractBackupPath abp = new RetryableCallable(3, RetryableCallable.DEFAULT_WAIT_TIME) { + // Allow up to 30s of arbitrary failures at the top level. The upload call itself typically has retries + // as well so this top level retry is on top of those retries. Assuming that each call to upload has + // ~30s maximum of retries this yields about 3.5 minutes of retries at the top level since + // (6 * (5 + 30) = 210 seconds). Even if this fails, however, higher level schedulers (e.g. in + // incremental) will hopefully re-enqueue. + AbstractBackupPath abp = new RetryableCallable(6, 5000) { public AbstractBackupPath retriableCall() throws Exception { upload(bp); file.delete(); @@ -99,33 +104,26 @@ public AbstractBackupPath retriableCall() throws Exception { /** - * Upload specified file (RandomAccessFile) with retries + * Upload specified file (RandomAccessFile) * * @param bp backup path to be uploaded. */ protected void upload(final AbstractBackupPath bp) throws Exception { - new RetryableCallable() { - @Override - public Void retriableCall() throws Exception { - java.io.InputStream is = null; - try { - is = bp.localReader(); - if (is == null) { - throw new NullPointerException("Unable to get handle on file: " + bp.fileName); - } - fs.upload(bp, is); - bp.setCompressedFileSize(fs.getBytesUploaded()); - return null; - } catch (Exception e) { - logger.error("Exception uploading local file {}, releasing handle, and will retry.", bp.backupFile.getCanonicalFile()); - if (is != null) { - is.close(); - } - throw e; - } - + java.io.InputStream is = null; + try { + is = bp.localReader(); + if (is == null) { + throw new NullPointerException("Unable to get handle on file: " + bp.fileName); + } + fs.upload(bp, is); + bp.setCompressedFileSize(fs.getBytesUploaded()); + } catch (Exception e) { + logger.error("Exception uploading local file {}, releasing handle, and will retry.", bp.backupFile.getCanonicalFile()); + if (is != null) { + is.close(); } - }.call(); + throw e; + } } protected final void initiateBackup(String monitoringFolder, BackupRestoreUtil backupRestoreUtil) throws Exception { diff --git a/priam/src/main/java/com/netflix/priam/backup/parallel/CassandraBackupQueueMgr.java b/priam/src/main/java/com/netflix/priam/backup/parallel/CassandraBackupQueueMgr.java index 1940a706c..2d734760c 100644 --- a/priam/src/main/java/com/netflix/priam/backup/parallel/CassandraBackupQueueMgr.java +++ b/priam/src/main/java/com/netflix/priam/backup/parallel/CassandraBackupQueueMgr.java @@ -48,39 +48,30 @@ public class CassandraBackupQueueMgr implements ITaskQueueMgr(config.getUncrementalBkupQueueSize()); - tasksQueued = new HashSet(config.getUncrementalBkupQueueSize()); //Key to task is the S3 absolute path (BASE/REGION/CLUSTER/TOKEN/[yyyymmddhhmm]/[SST|SNP|META]/KEYSPACE/COLUMNFAMILY/FILE + tasks = new ArrayBlockingQueue(config.getIncrementalBkupQueueSize()); + // Key to task is the S3 absolute path (BASE/REGION/CLUSTER/TOKEN/[yyyymmddhhmm]/[SST|SNP|META]/KEYSPACE/COLUMNFAMILY/FILE + tasksQueued = new HashSet(config.getIncrementalBkupQueueSize()); } @Override - /* - * Add task to queue if it does not already exist. For performance reasons, this behavior does not acquire a lock on the queue hence - * it is up to the caller to handle possible duplicate tasks. - * - * Note: will block until there is space in the queue. - */ - public void add(AbstractBackupPath task) { + public void add(AbstractBackupPath task) + { if (!tasksQueued.contains(task.getRemotePath())) { tasksQueued.add(task.getRemotePath()); try { - tasks.put(task); //block until space becomes available in queue + // block until space becomes available in queue + tasks.put(task); logger.debug("Queued file {} within CF {}", task.getFileName(), task.getColumnFamily()); - } catch (InterruptedException e) { logger.warn("Interrupted waiting for the task queue to have free space, not fatal will just move on. Error Msg: {}", e.getLocalizedMessage()); + tasksQueued.remove(task.getRemotePath()); } } else { logger.debug("Already in queue, no-op. File: {}", task.getRemotePath()); } - } @Override - /* - * Guarantee delivery of a task to only one consumer. - * - * @return task, null if task in queue. - */ public AbstractBackupPath take() throws InterruptedException { AbstractBackupPath task = null; if (!tasks.isEmpty()) { @@ -94,34 +85,17 @@ public AbstractBackupPath take() throws InterruptedException { } @Override - /* - * @return true if there are more tasks. - * - * Note: this is a best effort so the caller should call me again just before taking a task. - * We anticipate this method will be invoked at a high frequency hence, making it thread-safe will slow down the appliation or - * worse yet, create a deadlock. For example, caller blocks to determine if there are more tasks and also blocks waiting to dequeue - * the task. - */ public Boolean hasTasks() { return !tasks.isEmpty(); } @Override - /* - * A means to perform any post processing once the task has been completed. If post processing is needed, - * the consumer should notify this behavior via callback once the task is completed. - * - * *Note: "completed" here can mean success or failure. - */ public void taskPostProcessing(AbstractBackupPath completedTask) { this.tasksQueued.remove(completedTask.getRemotePath()); } @Override - /* - * @return num of pending tasks. Note, the result is a best guess, don't rely on it to be 100% accurate. - */ - public Integer getNumOfTasksToBeProessed() { + public Integer getNumOfTasksToBeProcessed() { return tasks.size(); } diff --git a/priam/src/main/java/com/netflix/priam/backup/parallel/ITaskQueueMgr.java b/priam/src/main/java/com/netflix/priam/backup/parallel/ITaskQueueMgr.java index ee5ceb135..e09198d6d 100644 --- a/priam/src/main/java/com/netflix/priam/backup/parallel/ITaskQueueMgr.java +++ b/priam/src/main/java/com/netflix/priam/backup/parallel/ITaskQueueMgr.java @@ -26,19 +26,26 @@ */ public interface ITaskQueueMgr { + /** + * Adds the provided task into the queue if it does not already exist. For performance reasons + * this is best effort and therefore callers are responsible for handling duplicate tasks. + * + * This method will block if the queue of tasks is full + * @param task The task to put onto the queue + */ void add(E task); - /* + /** * @return task, null if none is available. */ E take() throws InterruptedException; - /* + /** * @return true if there are tasks within queue to be processed; false otherwise. */ Boolean hasTasks(); - /* + /** * A means to perform any post processing once the task has been completed. If post processing is needed, * the consumer should notify this behavior via callback once the task is completed. * @@ -46,10 +53,9 @@ public interface ITaskQueueMgr { */ void taskPostProcessing(E completedTask); + Integer getNumOfTasksToBeProcessed(); - Integer getNumOfTasksToBeProessed(); - - /* + /** * @return true if all tasks completed (includes failures) for a date; false, if at least 1 task is still in queue. */ Boolean tasksCompleted(java.util.Date date); diff --git a/priam/src/main/java/com/netflix/priam/backup/parallel/IncrementalBackupProducer.java b/priam/src/main/java/com/netflix/priam/backup/parallel/IncrementalBackupProducer.java index d6c8d2c7e..7e8c33f19 100644 --- a/priam/src/main/java/com/netflix/priam/backup/parallel/IncrementalBackupProducer.java +++ b/priam/src/main/java/com/netflix/priam/backup/parallel/IncrementalBackupProducer.java @@ -70,7 +70,8 @@ protected void processColumnFamily(String keyspace, String columnFamily, File ba try { final AbstractBackupPath bp = pathFactory.get(); bp.parseLocal(file, BackupFileType.SST); - this.taskQueueMgr.add(bp); //producer -- populate the queue of files. *Note: producer will block if queue is full. + // producer -- populate the queue of files. *Note: producer will block if queue is full. + this.taskQueueMgr.add(bp); } catch (Exception e) { logger.warn("Unable to queue incremental file, treating as non-fatal and moving on to next. Msg: {} Fail to queue file: {}", e.getLocalizedMessage(), file.getAbsolutePath()); diff --git a/priam/src/main/java/com/netflix/priam/backup/parallel/IncrementalConsumer.java b/priam/src/main/java/com/netflix/priam/backup/parallel/IncrementalConsumer.java index e205f303f..bc1a5509d 100644 --- a/priam/src/main/java/com/netflix/priam/backup/parallel/IncrementalConsumer.java +++ b/priam/src/main/java/com/netflix/priam/backup/parallel/IncrementalConsumer.java @@ -48,7 +48,7 @@ public IncrementalConsumer(AbstractBackupPath bp, IBackupFileSystem fs @Override /* - * Upload specified file, with retries logic. + * Upload specified file, with retries logic. * File will be deleted only if uploaded successfully. */ public void run() { @@ -56,8 +56,11 @@ public void run() { logger.info("Consumer - about to upload file: {}", this.bp.getFileName()); try { - - new RetryableCallable() { + // Allow up to 30s of arbitrary failures at the top level. The upload call itself typically has retries + // as well so this top level retry is on top of those retries. Assuming that each call to upload has + // ~30s maximum of retries this yields about 3.5 minutes of retries at the top level since + // (6 * (5 + 30) = 210 seconds). Even if this fails, however, the upload will be re-enqueued + new RetryableCallable(6, 5000) { @Override public Void retriableCall() throws Exception { @@ -76,6 +79,9 @@ public Void retriableCall() throws Exception { if (is == null) { throw new NullPointerException("Unable to get handle on file: " + bp.getFileName()); } + // Important context: this upload call typically has internal retries but those are only + // to cover over very temporary (<10s) network partitions. For larger partitions re rely on + // higher up retries and re-enqueues. fs.upload(bp, is); bp.setCompressedFileSize(fs.getBytesUploaded()); return null; @@ -88,16 +94,20 @@ public Void retriableCall() throws Exception { } } }.call(); - - this.bp.getBackupFile().delete(); //resource cleanup - this.callback.postProcessing(bp); //post processing + // Clean up the underlying file. + bp.getBackupFile().delete(); } catch (Exception e) { if (e instanceof java.util.concurrent.CancellationException) { logger.debug("Failed to upload local file {}. Ignoring to continue with rest of backup. Msg: {}", this.bp.getFileName(), e.getLocalizedMessage()); } else { logger.error("Failed to upload local file {}. Ignoring to continue with rest of backup. Msg: {}", this.bp.getFileName(), e.getLocalizedMessage()); } + } finally { + // post processing must happen regardless of the outcome of the upload. Otherwise we can + // leak tasks into the underlying taskMgr queue and prevent re-enqueues of the upload itself + this.callback.postProcessing(bp); } + logger.info("Consumer - done with upload file: {}", this.bp.getFileName()); } } \ No newline at end of file diff --git a/priam/src/main/java/com/netflix/priam/backup/parallel/IncrementalConsumerMgr.java b/priam/src/main/java/com/netflix/priam/backup/parallel/IncrementalConsumerMgr.java index c887bd974..df1f7c651 100644 --- a/priam/src/main/java/com/netflix/priam/backup/parallel/IncrementalConsumerMgr.java +++ b/priam/src/main/java/com/netflix/priam/backup/parallel/IncrementalConsumerMgr.java @@ -25,6 +25,14 @@ import java.util.concurrent.*; import java.util.concurrent.atomic.AtomicBoolean; +import javax.annotation.Nullable; + +import com.google.common.util.concurrent.FutureCallback; +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.ListeningExecutorService; +import com.google.common.util.concurrent.MoreExecutors; + /* * Monitors files to be uploaded and assigns each file to a worker */ @@ -33,7 +41,7 @@ public class IncrementalConsumerMgr implements Runnable { private static final Logger logger = LoggerFactory.getLogger(IncrementalConsumerMgr.class); private AtomicBoolean run = new AtomicBoolean(true); - private ThreadPoolExecutor executor; + private ListeningExecutorService executor; private IBackupFileSystem fs; private ITaskQueueMgr taskQueueMgr; private BackupPostProcessingCallback callback; @@ -58,8 +66,8 @@ public IncrementalConsumerMgr(ITaskQueueMgr taskQueueMgr, IB * worker queue. Specifically, the calling will continue to perform the upload unless a worker is avaialble. */ RejectedExecutionHandler rejectedExecutionHandler = new ThreadPoolExecutor.CallerRunsPolicy(); - executor = new ThreadPoolExecutor(maxWorkers, maxWorkers, 60, TimeUnit.SECONDS, - workQueue, rejectedExecutionHandler); + executor = MoreExecutors.listeningDecorator(new ThreadPoolExecutor(maxWorkers, maxWorkers, 60, TimeUnit.SECONDS, + workQueue, rejectedExecutionHandler)); callback = new IncrementalBkupPostProcessing(this.taskQueueMgr); } @@ -78,20 +86,31 @@ public void run() { while (this.taskQueueMgr.hasTasks()) { try { - AbstractBackupPath bp = this.taskQueueMgr.take(); + final AbstractBackupPath bp = this.taskQueueMgr.take(); IncrementalConsumer task = new IncrementalConsumer(bp, this.fs, this.callback); - executor.submit(task); //non-blocking, will be rejected if the task cannot be scheduled - - + ListenableFuture upload = executor.submit(task); //non-blocking, will be rejected if the task cannot be scheduled + Futures.addCallback(upload, new FutureCallback() + { + public void onSuccess(@Nullable Object result) { } + + public void onFailure(Throwable t) { + // The post processing hook is responsible for removing the task from the de-duplicating + // HashSet, so we want to do the safe thing here and remove it just in case so the + // producers can re-enqueue this file in the next iteration. + // Note that this should be an abundance of caution as the IncrementalConsumer _should_ + // have deleted the task from the queue when it internally failed. + taskQueueMgr.taskPostProcessing(bp); + } + }); } catch (InterruptedException e) { logger.warn("Was interrupted while wating to dequeued a task. Msgl: {}", e.getLocalizedMessage()); } } - //Lets not overwhelmend the node hence we will pause before checking the work queue again. + // Lets not overwhelm the node hence we will pause before checking the work queue again. try { - Thread.currentThread().sleep(IIncrementalBackup.INCREMENTAL_INTERVAL_IN_MILLISECS); + Thread.sleep(IIncrementalBackup.INCREMENTAL_INTERVAL_IN_MILLISECS); } catch (InterruptedException e) { logger.warn("Was interrupted while sleeping until next interval run. Msgl: {}", e.getLocalizedMessage()); } diff --git a/priam/src/main/java/com/netflix/priam/config/IConfiguration.java b/priam/src/main/java/com/netflix/priam/config/IConfiguration.java index 829ebabda..1691c91cf 100644 --- a/priam/src/main/java/com/netflix/priam/config/IConfiguration.java +++ b/priam/src/main/java/com/netflix/priam/config/IConfiguration.java @@ -444,7 +444,6 @@ default String getCompactionExcludeCFList(){ */ int getHintedHandoffThrottleKb(); - /** * @return Size of Cassandra max direct memory */ @@ -470,7 +469,6 @@ default String getCompactionExcludeCFList(){ */ int getStreamingThroughputMB(); - /** * Get the paritioner for this cassandra cluster/node. * @@ -646,7 +644,7 @@ default String getCompactionExcludeCFList(){ String getPgpPasswordPhrase(); /* - * @return public key use by PGP cryptography. This information is to be use within the restore and backup functionality when encryption is enabled. + * @return key use by PGP cryptography. This information is to be use within the restore and backup functionality when encryption is enabled. * Note: for backward compatibility, this property should be optional. Specifically, if it does not exist, it should not cause an adverse impact on current functionality. */ String getPgpPublicKeyLoc(); @@ -688,7 +686,7 @@ default String getCompactionExcludeCFList(){ /* * The max number of files queued to be uploaded. */ - int getUncrementalBkupQueueSize(); + int getIncrementalBkupQueueSize(); /** * @return tombstone_warn_threshold in C* yaml @@ -760,33 +758,43 @@ default String getFlushCronExpression(){ * Post restore hook enabled state. If enabled, jar represented by getPostRepairHook is called once download of files is complete, before starting Cassandra. * @return if post restore hook is enabled */ - boolean isPostRestoreHookEnabled(); + + default boolean isPostRestoreHookEnabled() { + return false; + } /** * Post restore hook to be executed * @return post restore hook to be executed once restore is complete */ - String getPostRestoreHook(); - + default String getPostRestoreHook() { + return ""; + } /** * HeartBeat file of post restore hook * @return file that indicates heartbeat of post restore hook */ - String getPostRestoreHookHeartbeatFileName(); + default String getPostRestoreHookHeartbeatFileName() { + return "postrestorehook_heartbeat"; + } /** * Done file for post restore hook * @return file that indicates completion of post restore hook */ - String getPostRestoreHookDoneFileName(); + default String getPostRestoreHookDoneFileName() { + return "postrestorehook_done"; + } /** * Maximum time Priam has to wait for post restore hook sub-process to complete successfully * @return time out for post restore hook in days */ - int getPostRestoreHookTimeOutInDays(); + default int getPostRestoreHookTimeOutInDays() { + return 2; + } /** * Heartbeat timeout (in ms) for post restore hook diff --git a/priam/src/main/java/com/netflix/priam/config/PriamConfiguration.java b/priam/src/main/java/com/netflix/priam/config/PriamConfiguration.java index 140943db4..a58d5a9d9 100644 --- a/priam/src/main/java/com/netflix/priam/config/PriamConfiguration.java +++ b/priam/src/main/java/com/netflix/priam/config/PriamConfiguration.java @@ -1081,7 +1081,7 @@ public int getIncrementalBkupMaxConsumers() { } @Override - public int getUncrementalBkupQueueSize() { + public int getIncrementalBkupQueueSize() { return config.get(PRIAM_PRE + ".incremental.bkup.queue.size", 100000); } diff --git a/priam/src/main/java/com/netflix/priam/utils/BoundedExponentialRetryCallable.java b/priam/src/main/java/com/netflix/priam/utils/BoundedExponentialRetryCallable.java index b5cde9163..b6619c4d6 100644 --- a/priam/src/main/java/com/netflix/priam/utils/BoundedExponentialRetryCallable.java +++ b/priam/src/main/java/com/netflix/priam/utils/BoundedExponentialRetryCallable.java @@ -23,9 +23,9 @@ import java.util.concurrent.CancellationException; public abstract class BoundedExponentialRetryCallable extends RetryableCallable { - private final static long MAX_SLEEP = 10000; - private final static long MIN_SLEEP = 1000; - private final static int MAX_RETRIES = 10; + protected final static long MAX_SLEEP = 10000; + protected final static long MIN_SLEEP = 1000; + protected final static int MAX_RETRIES = 10; private static final Logger logger = LoggerFactory.getLogger(BoundedExponentialRetryCallable.class); private long max; diff --git a/priam/src/test/java/com/netflix/priam/config/FakeConfiguration.java b/priam/src/test/java/com/netflix/priam/config/FakeConfiguration.java index a1c7462b3..bbf6bf6f7 100644 --- a/priam/src/test/java/com/netflix/priam/config/FakeConfiguration.java +++ b/priam/src/test/java/com/netflix/priam/config/FakeConfiguration.java @@ -814,7 +814,7 @@ public int getIncrementalBkupMaxConsumers() { } @Override - public int getUncrementalBkupQueueSize() { + public int getIncrementalBkupQueueSize() { return 100; } @@ -884,7 +884,7 @@ public boolean isPostRestoreHookEnabled() { @Override public String getPostRestoreHook() { - return "iostat -d 2 10"; + return "echo"; } @Override From 3ecdc13710f7f21b107293bda358f5cca9111bfb Mon Sep 17 00:00:00 2001 From: Arun Agrawal Date: Fri, 31 Aug 2018 09:43:02 -0700 Subject: [PATCH 008/228] Some code cleanup of AbstractBackupPath and SnappyCompression --- .../java/com/netflix/priam/backup/AbstractBackupPath.java | 3 ++- .../com/netflix/priam/compress/SnappyCompression.java | 8 ++------ 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/priam/src/main/java/com/netflix/priam/backup/AbstractBackupPath.java b/priam/src/main/java/com/netflix/priam/backup/AbstractBackupPath.java index a4315fc76..6958fa97c 100644 --- a/priam/src/main/java/com/netflix/priam/backup/AbstractBackupPath.java +++ b/priam/src/main/java/com/netflix/priam/backup/AbstractBackupPath.java @@ -307,7 +307,8 @@ public long getLastModified() { return lastModified; } - public static class RafInputStream extends InputStream { + + public static class RafInputStream extends InputStream implements AutoCloseable { private RandomAccessFile raf; public RafInputStream(RandomAccessFile raf) { diff --git a/priam/src/main/java/com/netflix/priam/compress/SnappyCompression.java b/priam/src/main/java/com/netflix/priam/compress/SnappyCompression.java index a510a966a..cdb22e468 100644 --- a/priam/src/main/java/com/netflix/priam/compress/SnappyCompression.java +++ b/priam/src/main/java/com/netflix/priam/compress/SnappyCompression.java @@ -45,17 +45,13 @@ public void decompressAndClose(InputStream input, OutputStream output) throws IO } private void decompress(InputStream input, OutputStream output) throws IOException { - SnappyInputStream is = new SnappyInputStream(new BufferedInputStream(input)); byte data[] = new byte[BUFFER]; - BufferedOutputStream dest1 = new BufferedOutputStream(output, BUFFER); - try { + try(BufferedOutputStream dest1 = new BufferedOutputStream(output, BUFFER); + SnappyInputStream is = new SnappyInputStream(new BufferedInputStream(input))) { int c; while ((c = is.read(data, 0, BUFFER)) != -1) { dest1.write(data, 0, c); } - } finally { - IOUtils.closeQuietly(dest1); - IOUtils.closeQuietly(is); } } } From 43be5fabbd842f30745741677acfe50c98aa8898 Mon Sep 17 00:00:00 2001 From: Arun Agrawal Date: Thu, 6 Sep 2018 16:06:23 -0700 Subject: [PATCH 009/228] Add LocalDBReaderWriter for creating local database on filesystem. --- .../priam/backupv2/FileUploadResult.java | 22 +- .../priam/backupv2/LocalDBReaderWriter.java | 245 ++++++++++++++++++ .../priam/backupv2/PrefixGenerator.java | 2 +- .../backupv2/TestLocalDBReaderWriter.java | 244 +++++++++++++++++ .../services/TestSnapshotMetaService.java | 2 - 5 files changed, 501 insertions(+), 14 deletions(-) create mode 100644 priam/src/main/java/com/netflix/priam/backupv2/LocalDBReaderWriter.java create mode 100644 priam/src/test/java/com/netflix/priam/backupv2/TestLocalDBReaderWriter.java diff --git a/priam/src/main/java/com/netflix/priam/backupv2/FileUploadResult.java b/priam/src/main/java/com/netflix/priam/backupv2/FileUploadResult.java index 8d0268bb1..55b6a7bc4 100644 --- a/priam/src/main/java/com/netflix/priam/backupv2/FileUploadResult.java +++ b/priam/src/main/java/com/netflix/priam/backupv2/FileUploadResult.java @@ -110,17 +110,17 @@ public void setBackupPath(Path backupPath) { this.backupPath = backupPath; } - // - public JSONObject getJSONObject() throws Exception { - JSONObject result = new JSONObject(); - result.put("file", fileName.toFile().getName()); - result.put("modify", lastModifiedTime.toEpochMilli()); - result.put("creation", fileCreationTime.toEpochMilli()); - result.put("size", fileSizeOnDisk); - result.put("compression", compression.name()); - result.put("uploaded", isUploaded); - result.put("loc", backupPath); - return result; + public void setLastModifiedTime(Instant lastModifiedTime) { + this.lastModifiedTime = lastModifiedTime; + } + + public void setKeyspaceName(String keyspaceName) { + + this.keyspaceName = keyspaceName; + } + + public void setColumnFamilyName(String columnFamilyName) { + this.columnFamilyName = columnFamilyName; } @Override diff --git a/priam/src/main/java/com/netflix/priam/backupv2/LocalDBReaderWriter.java b/priam/src/main/java/com/netflix/priam/backupv2/LocalDBReaderWriter.java new file mode 100644 index 000000000..4be4d4e93 --- /dev/null +++ b/priam/src/main/java/com/netflix/priam/backupv2/LocalDBReaderWriter.java @@ -0,0 +1,245 @@ +/* + * Copyright 2018 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.netflix.priam.backupv2; + +import com.google.inject.Inject; +import com.netflix.priam.config.IConfiguration; +import com.netflix.priam.utils.GsonJsonSerializer; +import com.netflix.priam.utils.RetryableCallable; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.FileReader; +import java.io.FileWriter; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.time.Instant; +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; + +/** + * This class is used to create a local DB entry for SSTable components. This will be used to identify when a version + * of a SSTable component was last uploaded or last referenced. + * This db entry will be used to enforce the TTL of a version of a SSTable component as we will not be relying on + * backup file system level TTL. + * Local DB should be copied over to new instance replacing this token in case of instance replacement. If no local DB + * is found then Priam will try to re-create the local DB using meta files uploaded to backup file system. + * The check operation for local DB is done at every start of Priam and when operator requests to re-build the local DB. + * Created by aagrawal on 8/31/18. + */ +public class LocalDBReaderWriter { + private static final Logger logger = LoggerFactory.getLogger(LocalDBReaderWriter.class); + private IConfiguration configuration; + public static final String LOCAL_DB = "localdb"; + + @Inject + public LocalDBReaderWriter(IConfiguration configuration) { + this.configuration = configuration; + } + + public synchronized LocalDB upsertLocalDBEntry(final LocalDBEntry localDBEntry) throws Exception { + //validate the localDBEntry first + if (localDBEntry.getTimeLastReferenced() == null) + throw new Exception("Time last referenced in localDB can never be null"); + + if (localDBEntry.getBackupTime() == null) + throw new Exception("Backup time for localDB can never be null"); + + final Path localDBFile = getLocalDBPath(localDBEntry.getFileUploadResult()); + + if (!localDBFile.getParent().toFile().exists()) + localDBFile.getParent().toFile().mkdirs(); + + LocalDB localDB = getLocalDB(localDBEntry.getFileUploadResult()); + + if (localDB == null) + localDB = new LocalDB(new ArrayList<>()); + + //Verify again if someone beat you to write the entry. + LocalDBEntry entry = getLocalDBEntry(localDBEntry.getFileUploadResult(), localDB); + //Write entry as it might be either - + //1. new component entry + //2. new version of file (change in compression type or file is modified e.g. stats file) + if (entry == null) { + localDB.getLocalDb().add(localDBEntry); + writeLocalDB(localDBFile, localDB); + }else{ + //An entry already exists. Maybe last referenced time or backup time changed. We want to write the last time referenced. + entry.setBackupTime(localDBEntry.getBackupTime()); + entry.setTimeLastReferenced(localDBEntry.getTimeLastReferenced()); + writeLocalDB(localDBFile, localDB); + } + + return localDB; + } + + private LocalDB getLocalDB(final FileUploadResult fileUploadResult) throws Exception { + final Path localDbPath = getLocalDBPath(fileUploadResult); + //Retry for reading. + return new RetryableCallable(5, 1000) { + @Override + public LocalDB retriableCall() throws Exception { + return readLocalDB(localDbPath); + } + }.call(); + } + + public LocalDBEntry getLocalDBEntry(final FileUploadResult fileUploadResult) throws Exception { + LocalDB localDB = getLocalDB(fileUploadResult); + + if (localDB == null || localDB.getLocalDb() == null || localDB.getLocalDb().isEmpty()) + return null; + + return getLocalDBEntry(fileUploadResult, localDB); + } + + private LocalDBEntry getLocalDBEntry(final FileUploadResult fileUploadResult, final LocalDB localDB) throws Exception { + if (localDB == null || localDB.getLocalDb().isEmpty()) + return null; + + //Get local db entry for same file and version. + List localDBEntries = localDB.getLocalDb().stream().filter(localDBEntry -> + (localDBEntry.getFileUploadResult().getFileName().toFile().getName().toLowerCase().equals(fileUploadResult.getFileName().toFile().getName().toLowerCase()))) + .filter(localDBEntry -> (localDBEntry.getFileUploadResult().getLastModifiedTime().equals(fileUploadResult.getLastModifiedTime()))) + .filter(localDBEntry -> (localDBEntry.getFileUploadResult().getCompression().equals(fileUploadResult.getCompression()))) + .collect(Collectors.toList()); + + if (localDBEntries == null || localDBEntries.isEmpty()) + return null; + + if (localDBEntries.size() == 1) { + if (logger.isDebugEnabled()) + logger.debug("Local entry found: {}", localDBEntries.get(0)); + + return localDBEntries.get(0); + } + + throw new Exception("Unexpected behavior: More than one entry found in local database for the same file. FileUploadResult: " + fileUploadResult); + } + + public Path getLocalDBPath(final FileUploadResult fileUploadResult) { + return Paths.get(configuration.getDataFileLocation(), LOCAL_DB, fileUploadResult.getKeyspaceName(), fileUploadResult.getColumnFamilyName(), PrefixGenerator.getSSTFileBase(fileUploadResult.getFileName().toFile().getName()) + ".localdb"); + } + + public void writeLocalDB(final Path localDBFile, final LocalDB localDB) throws Exception { + if (localDB == null || localDBFile == null || localDBFile.toFile().isDirectory()) + throw new Exception("Invalid Arguments: localDbFile: " + localDBFile + ", localDB: " + localDB); + + if (!localDBFile.getParent().toFile().exists()) + localDBFile.getParent().toFile().mkdirs(); + + try (FileWriter writer = new FileWriter(localDBFile.toFile())) { + writer.write(localDB.toString()); + } + } + + public LocalDB readLocalDB(final Path localDBFile) throws Exception { + //Verify it is file and it exists. + if (localDBFile.toFile().isDirectory()) + throw new Exception("Invalid Arguments: Path provided is directory and not a file: " + localDBFile.toString()); + + if (!localDBFile.toFile().exists()) + return new LocalDB(new ArrayList<>()); + + try (FileReader reader = new FileReader(localDBFile.toFile())) { + LocalDB localDB = GsonJsonSerializer.getGson().fromJson(reader, LocalDB.class); + String columnfamilyName = localDBFile.getParent().toFile().getName(); + String keyspaceName = localDBFile.getParent().getParent().toFile().getName(); + localDB.getLocalDb().stream().forEach(localDBEntry -> { + localDBEntry.getFileUploadResult().setColumnFamilyName(columnfamilyName); + localDBEntry.getFileUploadResult().setKeyspaceName(keyspaceName); + }); + + if (logger.isDebugEnabled()) + logger.debug("Local DB: {}", localDB); + return localDB; + } + } + + static class LocalDB { + private List localDb; + + public LocalDB(List localDb) { + this.localDb = localDb; + } + + public List getLocalDb() { + + return localDb; + } + + public void setLocalDb(List localDb) { + this.localDb = localDb; + } + + @Override + public String toString() { + return GsonJsonSerializer.getGson().toJson(this); + } + } + + static class LocalDBEntry { + private FileUploadResult fileUploadResult; + private Instant timeLastReferenced; + private Instant backupTime; + + public LocalDBEntry(FileUploadResult fileUploadResult) { + this.fileUploadResult = fileUploadResult; + } + + public FileUploadResult getFileUploadResult() { + return fileUploadResult; + } + + public void setFileUploadResult(FileUploadResult fileUploadResult) { + this.fileUploadResult = fileUploadResult; + } + + public Instant getTimeLastReferenced() { + return timeLastReferenced; + } + + public void setTimeLastReferenced(Instant timeLastReferenced) { + this.timeLastReferenced = timeLastReferenced; + } + + public Instant getBackupTime() { + return backupTime; + } + + public void setBackupTime(Instant backupTime) { + this.backupTime = backupTime; + } + + public LocalDBEntry(FileUploadResult fileUploadResult, Instant timeLastReferenced, Instant backupTime) { + + this.fileUploadResult = fileUploadResult; + this.timeLastReferenced = timeLastReferenced; + this.backupTime = backupTime; + } + + @Override + public String toString() { + return GsonJsonSerializer.getGson().toJson(this); + } + + } + + +} diff --git a/priam/src/main/java/com/netflix/priam/backupv2/PrefixGenerator.java b/priam/src/main/java/com/netflix/priam/backupv2/PrefixGenerator.java index e88e7a05c..631e2a9f7 100644 --- a/priam/src/main/java/com/netflix/priam/backupv2/PrefixGenerator.java +++ b/priam/src/main/java/com/netflix/priam/backupv2/PrefixGenerator.java @@ -27,7 +27,7 @@ /** * This is a utility class to get the backup location of the SSTables/Meta files with backup version 2.0. - * TODO: All this functinality will be used when we have BackupUploadDownloadService. + * TODO: All this functionality will be used when we have BackupUploadDownloadService. * Created by aagrawal on 6/5/18. */ public class PrefixGenerator { diff --git a/priam/src/test/java/com/netflix/priam/backupv2/TestLocalDBReaderWriter.java b/priam/src/test/java/com/netflix/priam/backupv2/TestLocalDBReaderWriter.java new file mode 100644 index 000000000..bcdaa3e29 --- /dev/null +++ b/priam/src/test/java/com/netflix/priam/backupv2/TestLocalDBReaderWriter.java @@ -0,0 +1,244 @@ +/* + * Copyright 2018 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.netflix.priam.backupv2; + +import com.google.inject.Guice; +import com.google.inject.Injector; +import com.netflix.priam.backup.BRTestModule; +import com.netflix.priam.config.IConfiguration; +import com.netflix.priam.utils.DateUtil; +import org.apache.cassandra.io.sstable.Component; +import org.apache.commons.io.FileUtils; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.EnumSet; +import java.util.List; +import java.util.Random; +import java.util.concurrent.Callable; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; + +/** + * Created by aagrawal on 9/4/18. + */ +public class TestLocalDBReaderWriter { + private static final Logger logger = LoggerFactory.getLogger(TestLocalDBReaderWriter.class.getName()); + private static IConfiguration configuration; + private static LocalDBReaderWriter localDBReaderWriter; + private static Path dummyDataDirectoryLocation; + + @Before + public void setUp() { + Injector injector = Guice.createInjector(new BRTestModule()); + + if (configuration == null) + configuration = injector.getInstance(IConfiguration.class); + + if (localDBReaderWriter == null) + localDBReaderWriter = injector.getInstance(LocalDBReaderWriter.class); + + dummyDataDirectoryLocation = Paths.get(configuration.getDataFileLocation()); + cleanupDir(dummyDataDirectoryLocation); + } + + @After + public void destroy(){ + cleanupDir(dummyDataDirectoryLocation); + } + + private void cleanupDir(Path dir) { + if (dir.toFile().exists()) + try { + FileUtils.cleanDirectory(dir.toFile()); + } catch (IOException e) { + e.printStackTrace(); + } + } + + @Test + public void readWriteLocalDB() throws Exception { + int noOfKeyspaces = 2; + int noOfCf = 1; + int noOfSstables = 2; + Path localDbPath = Paths.get(dummyDataDirectoryLocation.toString(), LocalDBReaderWriter.LOCAL_DB); + List localDBList = generateDummyLocalDB(noOfKeyspaces, noOfCf, noOfSstables); + + localDBList.stream().forEach(localDB -> { + FileUploadResult fileUploadResult = localDB.getLocalDb().get(0).getFileUploadResult(); + final Path localDBPath = localDBReaderWriter.getLocalDBPath(fileUploadResult); + try { + localDBReaderWriter.writeLocalDB(localDBPath, localDB); + } catch (Exception e) { + logger.error("Error while writing to local DB: " + e.getMessage(), e); + } + }); + + //Verify the write succeeded for each KS/CF/SStable. + Assert.assertEquals(localDbPath.toFile().listFiles().length, noOfKeyspaces); + Path cfLocalDBPath = localDbPath.toFile().listFiles()[0].listFiles()[0].toPath(); + Assert.assertEquals(noOfSstables, cfLocalDBPath.toFile().listFiles().length); + + //Read the database. + LocalDBReaderWriter.LocalDB localDB = localDBReaderWriter.readLocalDB(cfLocalDBPath.toFile().listFiles()[0].toPath()); + Assert.assertEquals(EnumSet.allOf(Component.Type.class).size(), localDB.getLocalDb().size()); + } + + @Test + public void upsertLocalDB() throws Exception{ + LocalDBReaderWriter.LocalDB localDB = generateDummyLocalDB(1, 1, 1).get(0); + + //Lets do write with each LocalDBEntry first. + localDB.getLocalDb().stream().forEach(localDBEntry -> { + try { + localDBReaderWriter.upsertLocalDBEntry(localDBEntry); + } catch (Exception e) { + e.printStackTrace(); + } + }); + + //Verify the write has happened. + LocalDBReaderWriter.LocalDBEntry localDBEntry = localDBReaderWriter.getLocalDBEntry(localDB.getLocalDb().get(0).getFileUploadResult()); + Assert.assertNotNull(localDBEntry); + + //Now lets see if we can write the same entry again?? + LocalDBReaderWriter.LocalDB localDBUpsert = localDBReaderWriter.upsertLocalDBEntry(localDBEntry); + Assert.assertEquals(localDB.getLocalDb().size(), localDBUpsert.getLocalDb().size()); + + //Now lets change the localDBEntry and see if upsert succeeds. + localDBEntry.setTimeLastReferenced(DateUtil.getInstant()); + localDBUpsert = localDBReaderWriter.upsertLocalDBEntry(localDBEntry); + LocalDBReaderWriter.LocalDBEntry localDBEntryUpsert = localDBReaderWriter.getLocalDBEntry(localDBEntry.getFileUploadResult()); + Assert.assertEquals(localDBEntryUpsert.getTimeLastReferenced(),localDBEntry.getTimeLastReferenced()); + Assert.assertEquals(localDB.getLocalDb().size(), localDBUpsert.getLocalDb().size()); + + //Now change the file modification time. This should end up creating a new DB Entry. + localDBEntry.getFileUploadResult().setLastModifiedTime(DateUtil.getInstant()); + localDBUpsert = localDBReaderWriter.upsertLocalDBEntry(localDBEntry); + localDBEntryUpsert = localDBReaderWriter.getLocalDBEntry(localDBEntry.getFileUploadResult()); + Assert.assertEquals(localDB.getLocalDb().size() + 1, localDBUpsert.getLocalDb().size()); + Assert.assertEquals(localDBEntry.getFileUploadResult().getLastModifiedTime(), localDBEntryUpsert.getFileUploadResult().getLastModifiedTime()); + } + + @Test + public void readConcurrentLocalDB() throws Exception{ + List localDBList = generateDummyLocalDB(1, 1, 1); + localDBList.stream().forEach(localDB -> { + FileUploadResult fileUploadResult = localDB.getLocalDb().get(0).getFileUploadResult(); + final Path localDBPath = localDBReaderWriter.getLocalDBPath(fileUploadResult); + try { + localDBReaderWriter.writeLocalDB(localDBPath, localDB); + } catch (Exception e) { + logger.error("Error while writing to local DB: " + e.getMessage(), e); + } + }); + + FileUploadResult sample = localDBList.get(0).getLocalDb().get(0).getFileUploadResult(); + int size = 5; + + ExecutorService threads = Executors.newFixedThreadPool(size); + List> torun = new ArrayList<>(size); + for (int i = 0; i < size; i++) { + torun.add(() -> localDBReaderWriter.getLocalDBEntry(sample) != null); + } + + // all tasks executed in different threads, at 'once'. + List> futures = threads.invokeAll(torun); + + // no more need for the threadpool + threads.shutdown(); + // check the results of the tasks. + int noOfBadRun = 0; + for (Future fut : futures) { + if (!fut.get()) noOfBadRun ++; + } + + Assert.assertEquals(0, noOfBadRun); + } + + @Test + public void writeConcurrentLocalDB() throws Exception{ + LocalDBReaderWriter.LocalDB localDB = generateDummyLocalDB(1, 1, 1).get(0); + int size = localDB.getLocalDb().size(); + ExecutorService threads = Executors.newFixedThreadPool(size); + List> torun = new ArrayList<>(size); + for (int i = 0; i < size; i++) { + int finalI = i; + torun.add(() -> localDBReaderWriter.upsertLocalDBEntry(localDB.getLocalDb().get(finalI))); + } + // all tasks executed in different threads, at 'once'. + List> futures = threads.invokeAll(torun); + + // no more need for the threadpool + threads.shutdown(); + // check the results of the tasks. + int noOfBadRun = 0; + for (Future fut : futures) { + //We expect exception here. + try{ + fut.get(); + }catch(Exception e){ + noOfBadRun++; + } + } + + Assert.assertEquals(0, noOfBadRun); + LocalDBReaderWriter.LocalDB localDBRead = localDBReaderWriter.readLocalDB(localDBReaderWriter.getLocalDBPath(localDB.getLocalDb().get(0).getFileUploadResult())); + Assert.assertEquals(localDB.getLocalDb().size(), localDBRead.getLocalDb().size()); + } + + private List generateDummyLocalDB(int noOfKeyspaces, int noOfCf, int noOfSstables) throws Exception { + + //Clean the dummy directory + cleanupDir(dummyDataDirectoryLocation); + List localDBList = new ArrayList<>(); + + Random random = new Random(); + + for (int i = 1; i <= noOfKeyspaces; i++) { + String keyspaceName = "sample" + i; + + for (int j = 1; j <= noOfCf; j++) { + String columnfamilyname = "cf" + j; + + for (int k = 1; k <= noOfSstables; k++) { + String prefixName = "mc-" + k + "-big"; + LocalDBReaderWriter.LocalDB localDB = new LocalDBReaderWriter.LocalDB(new ArrayList<>()); + localDBList.add(localDB); + for (Component.Type type : EnumSet.allOf(Component.Type.class)) { + Path componentPath = Paths.get(dummyDataDirectoryLocation.toFile().getAbsolutePath(), keyspaceName, columnfamilyname, prefixName + "-" + type.name() + ".db"); + FileUploadResult fileUploadResult = new FileUploadResult(componentPath, keyspaceName, columnfamilyname, DateUtil.getInstant(), DateUtil.getInstant(), random.nextLong()); + LocalDBReaderWriter.LocalDBEntry localDBEntry = new LocalDBReaderWriter.LocalDBEntry(fileUploadResult, DateUtil.getInstant(), DateUtil.getInstant()); + localDB.getLocalDb().add(localDBEntry); + } + } + } + } + + return localDBList; + } +} diff --git a/priam/src/test/java/com/netflix/priam/services/TestSnapshotMetaService.java b/priam/src/test/java/com/netflix/priam/services/TestSnapshotMetaService.java index 5d40da666..1e2d00851 100644 --- a/priam/src/test/java/com/netflix/priam/services/TestSnapshotMetaService.java +++ b/priam/src/test/java/com/netflix/priam/services/TestSnapshotMetaService.java @@ -149,8 +149,6 @@ private void generateDummyFiles(Path dummyDir, int noOfKeyspaces, int noOfCf, in if (dummyDir.toFile().exists()) FileUtils.cleanDirectory(dummyDir.toFile()); - Random random = new Random(); - for (int i = 1; i <= noOfKeyspaces; i++) { String keyspaceName = "sample" + i; From 0ea8f2980e8bec9fea0400a3c83336b0138a8db6 Mon Sep 17 00:00:00 2001 From: Arun Agrawal Date: Fri, 7 Sep 2018 10:53:32 -0700 Subject: [PATCH 010/228] Cleanup of S3EncryptedFileSystem, S3FileSystem, and RangeReadInputStream --- .../priam/aws/S3EncryptedFileSystem.java | 6 +-- .../com/netflix/priam/aws/S3FileSystem.java | 10 ++-- .../netflix/priam/aws/S3FileSystemBase.java | 23 +++------ .../priam/backup/BackupFileSystemAdapter.java | 48 ------------------- .../priam/backup/RangeReadInputStream.java | 25 ++++------ 5 files changed, 22 insertions(+), 90 deletions(-) delete mode 100644 priam/src/main/java/com/netflix/priam/backup/BackupFileSystemAdapter.java diff --git a/priam/src/main/java/com/netflix/priam/aws/S3EncryptedFileSystem.java b/priam/src/main/java/com/netflix/priam/aws/S3EncryptedFileSystem.java index f36a2d3a1..4b08b1d33 100755 --- a/priam/src/main/java/com/netflix/priam/aws/S3EncryptedFileSystem.java +++ b/priam/src/main/java/com/netflix/priam/aws/S3EncryptedFileSystem.java @@ -92,10 +92,10 @@ public long bytesDownloaded() { } @Override - public void downloadFile(AbstractBackupPath path, OutputStream os) throws BackupRestoreException { + void downloadFileImpl(AbstractBackupPath path, OutputStream os) throws BackupRestoreException { try { - RangeReadInputStream rris = new RangeReadInputStream(s3Client, getPrefix(config), path); + RangeReadInputStream rris = new RangeReadInputStream(s3Client, getPrefix(config), path.getSize(), path.getRemotePath()); /* * To handle use cases where decompression should be done outside of the download. For example, the file have been compressed and then encrypted. @@ -121,7 +121,7 @@ public void downloadFile(AbstractBackupPath path, OutputStream os) throws Backup @Override - public void uploadFile(AbstractBackupPath path, InputStream in, long chunkSize) throws BackupRestoreException { + void uploadFileImpl(AbstractBackupPath path, InputStream in, long chunkSize) throws BackupRestoreException { InitiateMultipartUploadRequest initRequest = new InitiateMultipartUploadRequest(config.getBackupPrefix(), path.getRemotePath()); //initialize chunking request to aws InitiateMultipartUploadResult initResponse = s3Client.initiateMultipartUpload(initRequest); //Fetch the aws generated upload id for this chunking request diff --git a/priam/src/main/java/com/netflix/priam/aws/S3FileSystem.java b/priam/src/main/java/com/netflix/priam/aws/S3FileSystem.java index a86e495c1..af39be67d 100644 --- a/priam/src/main/java/com/netflix/priam/aws/S3FileSystem.java +++ b/priam/src/main/java/com/netflix/priam/aws/S3FileSystem.java @@ -72,9 +72,9 @@ public S3FileSystem(@Named("awss3roleassumption") IS3Credential cred, Provider path.getSize() ? path.getSize() : MAX_BUFFERED_IN_STREAM_SIZE; compress.decompressAndClose(new BufferedInputStream(rris, (int) bufSize), os); } catch (Exception e) { @@ -99,6 +99,7 @@ private ObjectMetadata getObjectMetadata(AbstractBackupPath path) { } private void uploadMultipart(AbstractBackupPath path, InputStream in, long chunkSize) throws BackupRestoreException { + logger.info("Uploading to {}/{} with chunk size {}", config.getBackupPrefix(), path.getRemotePath(), chunkSize); InitiateMultipartUploadRequest initRequest = new InitiateMultipartUploadRequest(config.getBackupPrefix(), path.getRemotePath()); initRequest.withObjectMetadata(getObjectMetadata(path)); @@ -144,12 +145,11 @@ private void uploadMultipart(AbstractBackupPath path, InputStream in, long chunk } @Override - public void uploadFile(AbstractBackupPath path, InputStream in, long chunkSize) throws BackupRestoreException { + void uploadFileImpl(AbstractBackupPath path, InputStream in, long chunkSize) throws BackupRestoreException { if (path.getSize() < chunkSize) { //Upload file without using multipart upload as it will be more efficient. - if (logger.isDebugEnabled()) - logger.debug("Uploading file using put: {}", path.getRemotePath()); + logger.info("Uploading to {}/{} using PUT operation", config.getBackupPrefix(), path.getRemotePath()); try (ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream()) { Iterator chunkedStream = compress.compress(in, chunkSize); diff --git a/priam/src/main/java/com/netflix/priam/aws/S3FileSystemBase.java b/priam/src/main/java/com/netflix/priam/aws/S3FileSystemBase.java index 9ea13cbae..e4f644b9b 100755 --- a/priam/src/main/java/com/netflix/priam/aws/S3FileSystemBase.java +++ b/priam/src/main/java/com/netflix/priam/aws/S3FileSystemBase.java @@ -23,11 +23,11 @@ import com.google.common.collect.Lists; import com.google.common.util.concurrent.RateLimiter; import com.google.inject.Provider; -import com.netflix.priam.config.IConfiguration; import com.netflix.priam.backup.AbstractBackupPath; import com.netflix.priam.backup.BackupRestoreException; import com.netflix.priam.backup.IBackupFileSystem; import com.netflix.priam.compress.ICompression; +import com.netflix.priam.config.IConfiguration; import com.netflix.priam.merics.BackupMetrics; import com.netflix.priam.notification.BackupEvent; import com.netflix.priam.notification.BackupNotificationMgr; @@ -54,9 +54,7 @@ public abstract class S3FileSystemBase implements IBackupFileSystem, EventGenera protected static final long MAX_BUFFERED_IN_STREAM_SIZE = 5 * 1024 * 1024; protected static final long UPLOAD_TIMEOUT = (2 * 60 * 60 * 1000L); private static final Logger logger = LoggerFactory.getLogger(S3FileSystemBase.class); - //protected AtomicInteger uploadCount = new AtomicInteger(); protected AtomicLong bytesUploaded = new AtomicLong(); //bytes uploaded per file - //protected AtomicInteger downloadCount = new AtomicInteger(); protected AtomicLong bytesDownloaded = new AtomicLong(); protected BackupMetrics backupMetrics; protected AmazonS3 s3Client; @@ -86,6 +84,7 @@ public S3FileSystemBase(Provider pathProvider, this.addObserver(backupNotificationMgr); } + public AmazonS3 getS3Client() { return s3Client; } @@ -221,11 +220,9 @@ public void upload(AbstractBackupPath path, InputStream in) throws BackupRestore if (path.getSize() > 0) chunkSize = (path.getSize() / chunkSize >= MAX_CHUNKS) ? (path.getSize() / (MAX_CHUNKS - 1)) : chunkSize; //compute the size of each block we will upload to endpoint - logger.info("Uploading to {}/{} with chunk size {}", config.getBackupPrefix(), path.getRemotePath(), chunkSize); - long startTime = System.nanoTime(); //initialize for each file upload notifyEventStart(new BackupEvent(path)); - uploadFile(path, in, chunkSize); + uploadFileImpl(path, in, chunkSize); long completedTime = System.nanoTime(); postProcessingPerFile(path, TimeUnit.NANOSECONDS.toMillis(startTime), TimeUnit.NANOSECONDS.toMillis(completedTime)); notifyEventSuccess(new BackupEvent(path)); @@ -237,12 +234,10 @@ protected void checkSuccessfulUpload(CompleteMultipartUploadResult resultS3Multi String eTagObjectId = resultS3MultiPartUploadComplete.getETag(); //unique id of the whole object logDiagnosticInfo(path, resultS3MultiPartUploadComplete); } else { - this.backupMetrics.incrementInvalidUploads(); throw new BackupRestoreException("Error uploading file as ETag or CompleteMultipartUploadResult is NULL -" + path.getFileName()); } } - protected BackupRestoreException encounterError(AbstractBackupPath path, S3PartUploader s3PartUploader, Exception e) { s3PartUploader.abortUpload(); return encounterError(path, e); @@ -266,7 +261,7 @@ protected BackupRestoreException encounterError(AbstractBackupPath path, Excepti return new BackupRestoreException("Error uploading file " + path.getFileName(), e); } - abstract void uploadFile(AbstractBackupPath path, InputStream in, long chunkSize) throws BackupRestoreException; + abstract void uploadFileImpl(AbstractBackupPath path, InputStream in, long chunkSize) throws BackupRestoreException; /** * This method does exactly as other download method.(Supposed to be overridden) @@ -277,13 +272,7 @@ protected BackupRestoreException encounterError(AbstractBackupPath path, Excepti @Override public void download(AbstractBackupPath path, OutputStream os, String filePath) throws BackupRestoreException { - try { - // Calling original Download method download(path, os); - } catch (Exception e) { - throw new BackupRestoreException(e.getMessage(), e); - } - } @Override @@ -292,7 +281,7 @@ public void download(AbstractBackupPath path, OutputStream os) throws BackupRest long contentLen = s3Client.getObjectMetadata(getPrefix(config), path.getRemotePath()).getContentLength(); path.setSize(contentLen); try { - downloadFile(path, os); + downloadFileImpl(path, os); bytesDownloaded.addAndGet(contentLen); backupMetrics.incrementValidDownloads(); } catch (BackupRestoreException e) { @@ -301,7 +290,7 @@ public void download(AbstractBackupPath path, OutputStream os) throws BackupRest } } - protected abstract void downloadFile(AbstractBackupPath path, OutputStream os) throws BackupRestoreException; + abstract void downloadFileImpl(AbstractBackupPath path, OutputStream os) throws BackupRestoreException; @Override public long getBytesUploaded() { diff --git a/priam/src/main/java/com/netflix/priam/backup/BackupFileSystemAdapter.java b/priam/src/main/java/com/netflix/priam/backup/BackupFileSystemAdapter.java deleted file mode 100644 index 075677523..000000000 --- a/priam/src/main/java/com/netflix/priam/backup/BackupFileSystemAdapter.java +++ /dev/null @@ -1,48 +0,0 @@ -/** - * Copyright 2017 Netflix, Inc. - *

- * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.netflix.priam.backup; - -import java.io.InputStream; -import java.io.OutputStream; -import java.util.Date; -import java.util.Iterator; - -public abstract class BackupFileSystemAdapter implements IBackupFileSystem { - - public void download(AbstractBackupPath path, OutputStream os) throws BackupRestoreException { - } - - public void upload(AbstractBackupPath path, InputStream in) throws BackupRestoreException { - } - - public Iterator list(String path, Date start, Date till) { - return null; - } - - public Iterator listPrefixes(Date date) { - return null; - } - - public void cleanup() { - } - - public int getActivecount() { - return 0; - } - - public void shutdown() { - } -} diff --git a/priam/src/main/java/com/netflix/priam/backup/RangeReadInputStream.java b/priam/src/main/java/com/netflix/priam/backup/RangeReadInputStream.java index d2e41af46..263782435 100644 --- a/priam/src/main/java/com/netflix/priam/backup/RangeReadInputStream.java +++ b/priam/src/main/java/com/netflix/priam/backup/RangeReadInputStream.java @@ -37,19 +37,18 @@ public class RangeReadInputStream extends InputStream { private final AmazonS3 s3Client; private final String bucketName; - private final AbstractBackupPath path; + private final long fileSize; + private final String remotePath; private long offset; - public RangeReadInputStream(AmazonS3 s3Client, String bucketName, AbstractBackupPath path) { + public RangeReadInputStream(AmazonS3 s3Client, String bucketName, long fileSize, String remotePath) { this.s3Client = s3Client; this.bucketName = bucketName; - this.path = path; + this.fileSize = fileSize; + this.remotePath = remotePath; } public int read(final byte b[], final int off, final int len) throws IOException { -// logger.info(String.format("incoming buf req's size = %d, off = %d, len to read = %d, on file size %d, cur offset = %d path = %s", -// b.length, off, len, path.getSize(), offset, path.getRemotePath())); - final long fileSize = path.getSize(); if (fileSize > 0 && offset >= fileSize) return -1; final long firstByte = offset; @@ -59,16 +58,12 @@ public int read(final byte b[], final int off, final int len) throws IOException //need to subtract one as the call to getRange is inclusive //meaning if you want to download the first 10 bytes of a file, request bytes 0..9 final long endByte = curEndByte - 1; -// logger.info(String.format("start byte = %d, end byte = %d", firstByte, endByte)); try { Integer cnt = new RetryableCallable() { public Integer retriableCall() throws IOException { - GetObjectRequest req = new GetObjectRequest(bucketName, path.getRemotePath()); + GetObjectRequest req = new GetObjectRequest(bucketName, remotePath); req.setRange(firstByte, endByte); - S3ObjectInputStream is = null; - try { - is = s3Client.getObject(req).getObjectContent(); - + try(S3ObjectInputStream is = s3Client.getObject(req).getObjectContent()) { byte[] readBuf = new byte[4092]; int rCnt; int readTotal = 0; @@ -77,22 +72,18 @@ public Integer retriableCall() throws IOException { System.arraycopy(readBuf, 0, b, incomingOffet, rCnt); readTotal += rCnt; incomingOffet += rCnt; -// logger.info(" local read cnt = " + rCnt + "Current Thread Name = "+Thread.currentThread().getName()); } if (readTotal == 0 && rCnt == -1) return -1; offset += readTotal; return Integer.valueOf(readTotal); - } finally { - IOUtils.closeQuietly(is); } } }.call(); -// logger.info("read cnt = " + cnt); return cnt.intValue(); } catch (Exception e) { String msg = String.format("failed to read offset range %d-%d of file %s whose size is %d", - firstByte, endByte, path.getRemotePath(), path.getSize()); + firstByte, endByte, remotePath, fileSize); throw new IOException(msg, e); } } From 1c8ff57b995cc42dd794ef773be7a4aa21d0ae82 Mon Sep 17 00:00:00 2001 From: Arun Agrawal Date: Thu, 13 Sep 2018 16:59:25 -0700 Subject: [PATCH 011/228] Refactor of file system, abstraction of common components in AbstractFileSystem. Removal of file system mbeans. --- .../com/netflix/priam/aws/S3BackupPath.java | 23 +-- .../priam/aws/S3EncryptedFileSystem.java | 121 +++-------- .../priam/aws/S3EncryptedFileSystemMBean.java | 31 --- .../com/netflix/priam/aws/S3FileSystem.java | 110 ++++------ .../netflix/priam/aws/S3FileSystemBase.java | 192 ++---------------- .../netflix/priam/aws/S3FileSystemMBean.java | 31 --- .../netflix/priam/backup/AbstractBackup.java | 14 +- .../priam/backup/AbstractBackupPath.java | 19 +- .../priam/backup/AbstractFileSystem.java | 128 ++++++++++++ .../priam/backup/BackupVerification.java | 3 +- .../netflix/priam/backup/CommitLogBackup.java | 3 +- .../priam/backup/IBackupFileSystem.java | 43 ++-- .../com/netflix/priam/backup/MetaData.java | 7 +- .../backup/parallel/IncrementalConsumer.java | 25 +-- .../priam/backupv2/MetaFileWriterBuilder.java | 3 +- .../google/GoogleEncryptedFileSystem.java | 137 ++----------- .../GoogleEncryptedFileSystemMBean.java | 31 --- .../netflix/priam/merics/BackupMetrics.java | 49 ++--- .../priam/resources/BackupServlet.java | 3 - .../priam/restore/EncryptedRestoreBase.java | 3 +- .../com/netflix/priam/restore/Restore.java | 3 +- .../netflix/priam/backup/BRTestModule.java | 2 +- .../priam/backup/FakeBackupFileSystem.java | 105 ++++------ .../backup/FakedS3EncryptedFileSystem.java | 105 ---------- .../priam/backup/NullBackupFileSystem.java | 57 ++---- .../priam/backup/TestS3FileSystem.java | 182 ++++++++--------- 26 files changed, 454 insertions(+), 976 deletions(-) delete mode 100755 priam/src/main/java/com/netflix/priam/aws/S3EncryptedFileSystemMBean.java delete mode 100644 priam/src/main/java/com/netflix/priam/aws/S3FileSystemMBean.java create mode 100644 priam/src/main/java/com/netflix/priam/backup/AbstractFileSystem.java delete mode 100755 priam/src/main/java/com/netflix/priam/google/GoogleEncryptedFileSystemMBean.java delete mode 100755 priam/src/test/java/com/netflix/priam/backup/FakedS3EncryptedFileSystem.java diff --git a/priam/src/main/java/com/netflix/priam/aws/S3BackupPath.java b/priam/src/main/java/com/netflix/priam/aws/S3BackupPath.java index 292de2082..90ecda6ee 100644 --- a/priam/src/main/java/com/netflix/priam/aws/S3BackupPath.java +++ b/priam/src/main/java/com/netflix/priam/aws/S3BackupPath.java @@ -29,12 +29,6 @@ * Represents an S3 object key */ public class S3BackupPath extends AbstractBackupPath { - /* - * Checking if request came from Cassandra 1.0 or 1.1 - * In Cassandra 1.0, Number of path elements = 8 - * In Cassandra 1.1, Number of path elements = 9 - */ - private static final int NUM_PATH_ELEMENTS_CASS_1_0 = 8; @Inject public S3BackupPath(IConfiguration config, InstanceIdentity factory) { @@ -43,9 +37,7 @@ public S3BackupPath(IConfiguration config, InstanceIdentity factory) { /** * Format of backup path: - * Cassandra 1.0 - * BASE/REGION/CLUSTER/TOKEN/[SNAPSHOTTIME]/[SST|SNP|META]/KEYSPACE/FILE - * Cassandra 1.1 + * Cassandra > 1.1 * BASE/REGION/CLUSTER/TOKEN/[SNAPSHOTTIME]/[SST|SNP|META]/KEYSPACE/COLUMNFAMILY/FILE */ @Override @@ -57,12 +49,8 @@ public String getRemotePath() { buff.append(token).append(S3BackupPath.PATH_SEP); buff.append(formatDate(time)).append(S3BackupPath.PATH_SEP); buff.append(type).append(S3BackupPath.PATH_SEP); - if (BackupFileType.isDataFile(type)) { - if (isCassandra1_0) - buff.append(keyspace).append(S3BackupPath.PATH_SEP); - else - buff.append(keyspace).append(S3BackupPath.PATH_SEP).append(columnFamily).append(S3BackupPath.PATH_SEP); - } + if (BackupFileType.isDataFile(type)) + buff.append(keyspace).append(S3BackupPath.PATH_SEP).append(columnFamily).append(S3BackupPath.PATH_SEP); buff.append(fileName); return buff.toString(); } @@ -78,8 +66,6 @@ public void parseRemote(String remoteFilePath) { pieces.add(ele); } assert pieces.size() >= 7 : "Too few elements in path " + remoteFilePath; - if (pieces.size() == NUM_PATH_ELEMENTS_CASS_1_0) - setCassandra1_0(true); baseDir = pieces.get(0); region = pieces.get(1); clusterName = pieces.get(2); @@ -88,8 +74,7 @@ public void parseRemote(String remoteFilePath) { type = BackupFileType.valueOf(pieces.get(5)); if (BackupFileType.isDataFile(type)) { keyspace = pieces.get(6); - if (!isCassandra1_0) - columnFamily = pieces.get(7); + columnFamily = pieces.get(7); } // append the rest fileName = pieces.get(pieces.size() - 1); diff --git a/priam/src/main/java/com/netflix/priam/aws/S3EncryptedFileSystem.java b/priam/src/main/java/com/netflix/priam/aws/S3EncryptedFileSystem.java index 4b08b1d33..22db02abb 100755 --- a/priam/src/main/java/com/netflix/priam/aws/S3EncryptedFileSystem.java +++ b/priam/src/main/java/com/netflix/priam/aws/S3EncryptedFileSystem.java @@ -38,10 +38,8 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import javax.management.MBeanServer; -import javax.management.ObjectName; import java.io.*; -import java.lang.management.ManagementFactory; +import java.nio.file.Path; import java.util.Iterator; import java.util.List; import java.util.concurrent.atomic.AtomicInteger; @@ -50,7 +48,7 @@ * Implementation of IBackupFileSystem for S3. The upload/download will work with ciphertext. */ @Singleton -public class S3EncryptedFileSystem extends S3FileSystemBase implements S3EncryptedFileSystemMBean { +public class S3EncryptedFileSystem extends S3FileSystemBase { private static final Logger logger = LoggerFactory.getLogger(S3EncryptedFileSystem.class); private AtomicInteger uploadCount = new AtomicInteger(); @@ -65,122 +63,67 @@ public S3EncryptedFileSystem(Provider pathProvider, ICompres super(pathProvider, compress, config, backupMetrics, backupNotificationMgr); this.encryptor = fileCryptography; - - MBeanServer mbs = ManagementFactory.getPlatformMBeanServer(); - String mbeanName = ENCRYPTED_FILE_SYSTEM_MBEAN_NAME; - try { - mbs.registerMBean(this, new ObjectName(mbeanName)); - } catch (Exception e) { - throw new RuntimeException("Unable to regiser JMX bean: " + mbeanName + " to JMX server. Msg: " + e.getLocalizedMessage(), e); - } - super.s3Client = AmazonS3Client.builder().withCredentials(cred.getAwsCredentialProvider()).withRegion(config.getDC()).build(); } - @Override - /* - Note: provides same information as getBytesUploaded() but it's meant for S3FileSystemMBean object types. - */ - public long bytesUploaded() { - return bytesUploaded.get(); - } - @Override - public long bytesDownloaded() { - return bytesDownloaded.get(); - } - - @Override - void downloadFileImpl(AbstractBackupPath path, OutputStream os) throws BackupRestoreException { - try { - - RangeReadInputStream rris = new RangeReadInputStream(s3Client, getPrefix(config), path.getSize(), path.getRemotePath()); - - /* + protected void downloadFileImpl(Path remotePath, Path localPath) throws BackupRestoreException { + try (OutputStream os = new FileOutputStream(localPath.toFile()); + RangeReadInputStream rris = new RangeReadInputStream(s3Client, getPrefix(config), super.getFileSize(remotePath), remotePath.toString()); + ) { + /* * To handle use cases where decompression should be done outside of the download. For example, the file have been compressed and then encrypted. * Hence, decompressing it here would compromise the decryption. */ - try { - IOUtils.copyLarge(rris, os); - - } catch (Exception ex) { - - throw new BackupRestoreException("Exception encountered when copying bytes from input to output during download", ex); - - } finally { - IOUtils.closeQuietly(rris); - IOUtils.closeQuietly(os); - } - + IOUtils.copyLarge(rris, os); } catch (Exception e) { - throw new BackupRestoreException("Exception encountered downloading " + path.getRemotePath() + " from S3 bucket " + getPrefix(config) + throw new BackupRestoreException("Exception encountered downloading " + remotePath + " from S3 bucket " + getPrefix(config) + ", Msg: " + e.getMessage(), e); } } @Override - void uploadFileImpl(AbstractBackupPath path, InputStream in, long chunkSize) throws BackupRestoreException { - - InitiateMultipartUploadRequest initRequest = new InitiateMultipartUploadRequest(config.getBackupPrefix(), path.getRemotePath()); //initialize chunking request to aws + protected long uploadFileImpl(Path localPath, Path remotePath) throws BackupRestoreException { + long chunkSize = getChunkSize(localPath); + InitiateMultipartUploadRequest initRequest = new InitiateMultipartUploadRequest(config.getBackupPrefix(), remotePath.toString()); //initialize chunking request to aws InitiateMultipartUploadResult initResponse = s3Client.initiateMultipartUpload(initRequest); //Fetch the aws generated upload id for this chunking request - DataPart part = new DataPart(config.getBackupPrefix(), path.getRemotePath(), initResponse.getUploadId()); + DataPart part = new DataPart(config.getBackupPrefix(), remotePath.toString(), initResponse.getUploadId()); List partETags = Lists.newArrayList(); //Metadata on number of parts to be uploaded - //== Read chunks from src, compress it, and write to temp file - String compressedFileName = path.newRestoreFile() + ".compressed"; - logger.debug("Compressing {} with chunk size {}", compressedFileName, chunkSize); - File compressedDstFile = null; - FileOutputStream compressedDstFileOs = null; - BufferedOutputStream compressedBos = null; - try { - - compressedDstFile = new File(compressedFileName); - compressedDstFileOs = new FileOutputStream(compressedDstFile); - compressedBos = new BufferedOutputStream(compressedDstFileOs); - - } catch (FileNotFoundException e) { - throw new BackupRestoreException("Not able to find temporary compressed file: " + compressedFileName); - } - - try { + File compressedDstFile = new File(localPath.toString() + ".compressed"); + logger.debug("Compressing {} with chunk size {}", compressedDstFile.getAbsolutePath(), chunkSize); + try (InputStream in = new FileInputStream(localPath.toFile()); + BufferedOutputStream compressedBos = new BufferedOutputStream(new FileOutputStream(compressedDstFile))) { Iterator compressedChunks = this.compress.compress(in, chunkSize); while (compressedChunks.hasNext()) { byte[] compressedChunk = compressedChunks.next(); compressedBos.write(compressedChunk); } - - } catch (IOException e) { + } catch (Exception e) { String message = String.format("Exception in compressing the input data during upload to EncryptedStore Msg: " + e.getMessage()); logger.error(message, e); throw new BackupRestoreException(message); - } finally { - IOUtils.closeQuietly(in); - IOUtils.closeQuietly(compressedBos); } //== Read compressed data, encrypt each chunk, upload it to aws - FileInputStream compressedFileIs = null; - BufferedInputStream compressedBis = null; - try { - - compressedFileIs = new FileInputStream(new File(compressedFileName)); - compressedBis = new BufferedInputStream(compressedFileIs); - Iterator chunks = this.encryptor.encryptStream(compressedBis, path.getRemotePath()); + try (BufferedInputStream compressedBis = new BufferedInputStream(new FileInputStream(compressedDstFile))) { + Iterator chunks = this.encryptor.encryptStream(compressedBis, remotePath.toString()); int partNum = 0; //identifies this part position in the object we are uploading + long encryptedFileSize = 0; + while (chunks.hasNext()) { byte[] chunk = chunks.next(); rateLimiter.acquire(chunk.length); //throttle upload to endpoint - DataPart dp = new DataPart(++partNum, chunk, config.getBackupPrefix(), path.getRemotePath(), initResponse.getUploadId()); + DataPart dp = new DataPart(++partNum, chunk, config.getBackupPrefix(), remotePath.toString(), initResponse.getUploadId()); S3PartUploader partUploader = new S3PartUploader(s3Client, dp, partETags); + encryptedFileSize += chunk.length; executor.submit(partUploader); - - bytesUploaded.addAndGet(chunk.length); } executor.sleepTillEmpty(); @@ -189,23 +132,15 @@ void uploadFileImpl(AbstractBackupPath path, InputStream in, long chunkSize) thr } CompleteMultipartUploadResult resultS3MultiPartUploadComplete = new S3PartUploader(s3Client, part, partETags).completeUpload(); //complete the aws chunking upload by providing to aws the ETag that uniquely identifies the combined object data - checkSuccessfulUpload(resultS3MultiPartUploadComplete, path); - + checkSuccessfulUpload(resultS3MultiPartUploadComplete, localPath); + return encryptedFileSize; } catch (Exception e) { - throw encounterError(path, new S3PartUploader(s3Client, part, partETags), e); + new S3PartUploader(s3Client, part, partETags).abortUpload(); + throw new BackupRestoreException("Error uploading file: " + localPath, e); } finally { - IOUtils.closeQuietly(compressedBis); if (compressedDstFile.exists()) compressedDstFile.delete(); } } - - - @Override - public int getActivecount() { - return executor.getActiveCount(); - } - - } \ No newline at end of file diff --git a/priam/src/main/java/com/netflix/priam/aws/S3EncryptedFileSystemMBean.java b/priam/src/main/java/com/netflix/priam/aws/S3EncryptedFileSystemMBean.java deleted file mode 100755 index a60a61103..000000000 --- a/priam/src/main/java/com/netflix/priam/aws/S3EncryptedFileSystemMBean.java +++ /dev/null @@ -1,31 +0,0 @@ -/** - * Copyright 2017 Netflix, Inc. - *

- * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.netflix.priam.aws; - -public interface S3EncryptedFileSystemMBean { - - String ENCRYPTED_FILE_SYSTEM_MBEAN_NAME = "com.priam.aws.S3EncryptedFileSystemMBean:name=S3EncryptedFileSystemMBean"; - - long downloadCount(); - - long uploadCount(); - - int getActivecount(); - - long bytesUploaded(); - - long bytesDownloaded(); -} \ No newline at end of file diff --git a/priam/src/main/java/com/netflix/priam/aws/S3FileSystem.java b/priam/src/main/java/com/netflix/priam/aws/S3FileSystem.java index af39be67d..09697a97d 100644 --- a/priam/src/main/java/com/netflix/priam/aws/S3FileSystem.java +++ b/priam/src/main/java/com/netflix/priam/aws/S3FileSystem.java @@ -40,6 +40,7 @@ import javax.management.ObjectName; import java.io.*; import java.lang.management.ManagementFactory; +import java.nio.file.Path; import java.util.ArrayList; import java.util.Collections; import java.util.Iterator; @@ -50,7 +51,7 @@ * Implementation of IBackupFileSystem for S3 */ @Singleton -public class S3FileSystem extends S3FileSystemBase implements S3FileSystemMBean { +public class S3FileSystem extends S3FileSystemBase{ private static final Logger logger = LoggerFactory.getLogger(S3FileSystem.class); @Inject @@ -60,75 +61,69 @@ public S3FileSystem(@Named("awss3roleassumption") IS3Credential cred, Provider path.getSize() ? path.getSize() : MAX_BUFFERED_IN_STREAM_SIZE; - compress.decompressAndClose(new BufferedInputStream(rris, (int) bufSize), os); + long remoteFileSize = super.getFileSize(remotePath); + RangeReadInputStream rris = new RangeReadInputStream(s3Client, getPrefix(this.config), remoteFileSize, remotePath.toString()); + final long bufSize = MAX_BUFFERED_IN_STREAM_SIZE > remoteFileSize ? remoteFileSize : MAX_BUFFERED_IN_STREAM_SIZE; + compress.decompressAndClose(new BufferedInputStream(rris, (int) bufSize), new BufferedOutputStream(new FileOutputStream(localPath.toFile()))); } catch (Exception e) { - throw new BackupRestoreException("Exception encountered downloading " + path.getRemotePath() + " from S3 bucket " + getPrefix(config) + throw new BackupRestoreException("Exception encountered downloading " + remotePath + " from S3 bucket " + getPrefix(config) + ", Msg: " + e.getMessage(), e); } } - private ObjectMetadata getObjectMetadata(AbstractBackupPath path) { + private ObjectMetadata getObjectMetadata(Path path) { ObjectMetadata ret = new ObjectMetadata(); - long lastModified = path.getLastModified(); + long lastModified = path.toFile().lastModified(); if (lastModified != 0) { ret.addUserMetadata("local-modification-time", Long.toString(lastModified)); } - long fileSize = path.getSize(); + long fileSize = path.toFile().length(); if (fileSize != 0) { ret.addUserMetadata("local-size", Long.toString(fileSize)); } return ret; } - private void uploadMultipart(AbstractBackupPath path, InputStream in, long chunkSize) throws BackupRestoreException { - logger.info("Uploading to {}/{} with chunk size {}", config.getBackupPrefix(), path.getRemotePath(), chunkSize); - InitiateMultipartUploadRequest initRequest = new InitiateMultipartUploadRequest(config.getBackupPrefix(), path.getRemotePath()); - - initRequest.withObjectMetadata(getObjectMetadata(path)); + private long uploadMultipart(Path localPath, Path remotePath) throws BackupRestoreException { + long chunkSize = getChunkSize(localPath); + logger.info("Uploading to {}/{} with chunk size {}", config.getBackupPrefix(), remotePath, chunkSize); + InitiateMultipartUploadRequest initRequest = new InitiateMultipartUploadRequest(config.getBackupPrefix(), remotePath.toString()); + initRequest.withObjectMetadata(getObjectMetadata(localPath)); InitiateMultipartUploadResult initResponse = s3Client.initiateMultipartUpload(initRequest); - DataPart part = new DataPart(config.getBackupPrefix(), path.getRemotePath(), initResponse.getUploadId()); + DataPart part = new DataPart(config.getBackupPrefix(), remotePath.toString(), initResponse.getUploadId()); List partETags = Collections.synchronizedList(new ArrayList()); - try { + try(InputStream in = new FileInputStream(localPath.toFile())) { Iterator chunks = compress.compress(in, chunkSize); // Upload parts. int partNum = 0; AtomicInteger partsUploaded = new AtomicInteger(0); + long compressedFileSize = 0; while (chunks.hasNext()) { byte[] chunk = chunks.next(); rateLimiter.acquire(chunk.length); - DataPart dp = new DataPart(++partNum, chunk, config.getBackupPrefix(), path.getRemotePath(), initResponse.getUploadId()); + DataPart dp = new DataPart(++partNum, chunk, config.getBackupPrefix(), remotePath.toString(), initResponse.getUploadId()); S3PartUploader partUploader = new S3PartUploader(s3Client, dp, partETags, partsUploaded); + compressedFileSize += chunk.length; executor.submit(partUploader); - bytesUploaded.addAndGet(chunk.length); } executor.sleepTillEmpty(); - logger.info("All chunks uploaded for file {}, num of expected parts:{}, num of actual uploaded parts: {}", path.getFileName(), partNum, partsUploaded.get()); + logger.info("All chunks uploaded for file {}, num of expected parts:{}, num of actual uploaded parts: {}", localPath.toFile().getName(), partNum, partsUploaded.get()); if (partNum != partETags.size()) throw new BackupRestoreException("Number of parts(" + partNum + ") does not match the uploaded parts(" + partETags.size() + ")"); CompleteMultipartUploadResult resultS3MultiPartUploadComplete = new S3PartUploader(s3Client, part, partETags).completeUpload(); - checkSuccessfulUpload(resultS3MultiPartUploadComplete, path); + checkSuccessfulUpload(resultS3MultiPartUploadComplete, localPath); if (logger.isDebugEnabled()) { final S3ResponseMetadata responseMetadata = s3Client.getCachedResponseMetadata(initRequest); @@ -137,30 +132,33 @@ private void uploadMultipart(AbstractBackupPath path, InputStream in, long chunk logger.debug("S3 AWS x-amz-request-id[" + requestId + "], and x-amz-id-2[" + hostId + "]"); } + return compressedFileSize; } catch (Exception e) { - throw encounterError(path, new S3PartUploader(s3Client, part, partETags), e); - } finally { - IOUtils.closeQuietly(in); + new S3PartUploader(s3Client, part, partETags).abortUpload(); + throw new BackupRestoreException("Error uploading file: " + localPath.toString(), e); } } - @Override - void uploadFileImpl(AbstractBackupPath path, InputStream in, long chunkSize) throws BackupRestoreException { + protected long uploadFileImpl(Path localPath, Path remotePath) throws BackupRestoreException{ + long chunkSize = config.getBackupChunkSize(); + long fileSize = localPath.toFile().length(); - if (path.getSize() < chunkSize) { + if (fileSize < chunkSize) { //Upload file without using multipart upload as it will be more efficient. - logger.info("Uploading to {}/{} using PUT operation", config.getBackupPrefix(), path.getRemotePath()); + logger.info("Uploading to {}/{} using PUT operation", config.getBackupPrefix(), remotePath); - try (ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream()) { + try (ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); + InputStream in = new BufferedInputStream(new FileInputStream(localPath.toFile()))) { Iterator chunkedStream = compress.compress(in, chunkSize); while (chunkedStream.hasNext()) { byteArrayOutputStream.write(chunkedStream.next()); } byte[] chunk = byteArrayOutputStream.toByteArray(); + long compressedFileSize = chunk.length; rateLimiter.acquire(chunk.length); - ObjectMetadata objectMetadata = getObjectMetadata(path); + ObjectMetadata objectMetadata = getObjectMetadata(localPath); objectMetadata.setContentLength(chunk.length); - PutObjectRequest putObjectRequest = new PutObjectRequest(config.getBackupPrefix(), path.getRemotePath(), new ByteArrayInputStream(chunk), objectMetadata); + PutObjectRequest putObjectRequest = new PutObjectRequest(config.getBackupPrefix(), remotePath.toString(), new ByteArrayInputStream(chunk), objectMetadata); //Retry if failed. PutObjectResult upload = new BoundedExponentialRetryCallable(1000, 10000, 5) { @Override @@ -169,38 +167,14 @@ public PutObjectResult retriableCall() throws Exception { } }.call(); - bytesUploaded.addAndGet(chunk.length); - if (logger.isDebugEnabled()) - logger.debug("Successfully uploaded file with putObject: {} and etag: {}", path.getRemotePath(), upload.getETag()); + logger.debug("Successfully uploaded file with putObject: {} and etag: {}", remotePath, upload.getETag()); + + return compressedFileSize; } catch (Exception e) { - throw encounterError(path, e); - } finally { - IOUtils.closeQuietly(in); + throw new BackupRestoreException("Error uploading file: " + localPath.toFile().getName(), e); } } else - uploadMultipart(path, in, chunkSize); + return uploadMultipart(localPath, remotePath); } - - - @Override - public int getActivecount() { - return executor.getActiveCount(); - } - - - @Override - /* - Note: provides same information as getBytesUploaded() but it's meant for S3FileSystemMBean object types. - */ - public long bytesUploaded() { - return super.bytesUploaded.get(); - } - - - @Override - public long bytesDownloaded() { - return bytesDownloaded.get(); - } - -} \ No newline at end of file +} diff --git a/priam/src/main/java/com/netflix/priam/aws/S3FileSystemBase.java b/priam/src/main/java/com/netflix/priam/aws/S3FileSystemBase.java index e4f644b9b..d9dda3c04 100755 --- a/priam/src/main/java/com/netflix/priam/aws/S3FileSystemBase.java +++ b/priam/src/main/java/com/netflix/priam/aws/S3FileSystemBase.java @@ -24,6 +24,7 @@ import com.google.common.util.concurrent.RateLimiter; import com.google.inject.Provider; import com.netflix.priam.backup.AbstractBackupPath; +import com.netflix.priam.backup.AbstractFileSystem; import com.netflix.priam.backup.BackupRestoreException; import com.netflix.priam.backup.IBackupFileSystem; import com.netflix.priam.compress.ICompression; @@ -41,6 +42,7 @@ import java.io.File; import java.io.InputStream; import java.io.OutputStream; +import java.nio.file.Path; import java.util.Date; import java.util.Iterator; import java.util.List; @@ -49,31 +51,27 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLong; -public abstract class S3FileSystemBase implements IBackupFileSystem, EventGenerator { +public abstract class S3FileSystemBase extends AbstractFileSystem { protected static final int MAX_CHUNKS = 10000; protected static final long MAX_BUFFERED_IN_STREAM_SIZE = 5 * 1024 * 1024; protected static final long UPLOAD_TIMEOUT = (2 * 60 * 60 * 1000L); private static final Logger logger = LoggerFactory.getLogger(S3FileSystemBase.class); - protected AtomicLong bytesUploaded = new AtomicLong(); //bytes uploaded per file - protected AtomicLong bytesDownloaded = new AtomicLong(); - protected BackupMetrics backupMetrics; protected AmazonS3 s3Client; protected IConfiguration config; protected Provider pathProvider; protected ICompression compress; protected BlockingSubmitThreadPoolExecutor executor; protected RateLimiter rateLimiter; //a throttling mechanism, we can limit the amount of bytes uploaded to endpoint per second. - private final CopyOnWriteArrayList> observers = new CopyOnWriteArrayList<>(); public S3FileSystemBase(Provider pathProvider, ICompression compress, final IConfiguration config, BackupMetrics backupMetrics, BackupNotificationMgr backupNotificationMgr) { + super(backupMetrics, backupNotificationMgr); this.pathProvider = pathProvider; this.compress = compress; this.config = config; - this.backupMetrics = backupMetrics; int threads = config.getMaxBackupUploadThreads(); LinkedBlockingQueue queue = new LinkedBlockingQueue(threads); @@ -81,7 +79,6 @@ public S3FileSystemBase(Provider pathProvider, double throttleLimit = config.getUploadThrottle(); this.rateLimiter = RateLimiter.create(throttleLimit < 1 ? Double.MAX_VALUE : throttleLimit); - this.addObserver(backupNotificationMgr); } @@ -165,149 +162,17 @@ private boolean updateLifecycleRule(IConfiguration config, List rules, Str return true; } - /* - @param path - representation of the file uploaded - @param start time of upload, in millisecs - @param completion time of upload, in millsecs - */ - private void postProcessingPerFile(AbstractBackupPath path, long startTimeInMilliSecs, long completedTimeInMilliSecs) { - //Publish upload rate for each uploaded file - try { - long sizeInBytes = path.getSize(); - long elapseTimeInMillisecs = completedTimeInMilliSecs - startTimeInMilliSecs; - long elapseTimeInSecs = elapseTimeInMillisecs / 1000; //converting millis to seconds as 1000m in 1 second - long bytesReadPerSec = 0; - Double speedInKBps = 0.0; - if (elapseTimeInSecs > 0 && sizeInBytes > 0) { - bytesReadPerSec = sizeInBytes / elapseTimeInSecs; - speedInKBps = bytesReadPerSec / 1024D; - } else { - bytesReadPerSec = sizeInBytes; //we uploaded the whole file in less than a sec - speedInKBps = (double) sizeInBytes; - } - - logger.info("Upload rate for file: {}" - + ", elapsse time in sec(s): {}" - + ", KB per sec: {}", - path.getFileName(), elapseTimeInSecs, speedInKBps); - backupMetrics.recordUploadRate(sizeInBytes); - } catch (Exception e) { - logger.error("Post processing of file {} failed, not fatal.", path.getFileName(), e); - } - } - - /* - Reinitializtion which should be performed before uploading a file - */ - protected void reinitialize() { - bytesUploaded = new AtomicLong(0); //initialize - } - - /* - @param file uploaded to S3 - @param a list of unique parts uploaded to S3 for file - */ - protected void logDiagnosticInfo(AbstractBackupPath fileUploaded, CompleteMultipartUploadResult res) { - File f = fileUploaded.getBackupFile(); - String fName = f.getAbsolutePath(); - logger.info("Uploaded file: {}, object eTag: {}", fName, res.getETag()); - } - - @Override - public void upload(AbstractBackupPath path, InputStream in) throws BackupRestoreException { - reinitialize(); //perform before file upload - long chunkSize = config.getBackupChunkSize(); - if (path.getSize() > 0) - chunkSize = (path.getSize() / chunkSize >= MAX_CHUNKS) ? (path.getSize() / (MAX_CHUNKS - 1)) : chunkSize; //compute the size of each block we will upload to endpoint - - long startTime = System.nanoTime(); //initialize for each file upload - notifyEventStart(new BackupEvent(path)); - uploadFileImpl(path, in, chunkSize); - long completedTime = System.nanoTime(); - postProcessingPerFile(path, TimeUnit.NANOSECONDS.toMillis(startTime), TimeUnit.NANOSECONDS.toMillis(completedTime)); - notifyEventSuccess(new BackupEvent(path)); - backupMetrics.incrementValidUploads(); - } - - protected void checkSuccessfulUpload(CompleteMultipartUploadResult resultS3MultiPartUploadComplete, AbstractBackupPath path) throws BackupRestoreException { + protected void checkSuccessfulUpload(CompleteMultipartUploadResult resultS3MultiPartUploadComplete, Path localPath) throws BackupRestoreException { if (null != resultS3MultiPartUploadComplete && null != resultS3MultiPartUploadComplete.getETag()) { - String eTagObjectId = resultS3MultiPartUploadComplete.getETag(); //unique id of the whole object - logDiagnosticInfo(path, resultS3MultiPartUploadComplete); + logger.info("Uploaded file: {}, object eTag: {}", localPath, resultS3MultiPartUploadComplete.getETag()); } else { - throw new BackupRestoreException("Error uploading file as ETag or CompleteMultipartUploadResult is NULL -" + path.getFileName()); - } - } - - protected BackupRestoreException encounterError(AbstractBackupPath path, S3PartUploader s3PartUploader, Exception e) { - s3PartUploader.abortUpload(); - return encounterError(path, e); - } - - protected BackupRestoreException encounterError(AbstractBackupPath path, Exception e) { - this.backupMetrics.incrementInvalidUploads(); - if (e instanceof AmazonS3Exception) { - AmazonS3Exception a = (AmazonS3Exception) e; - String amazoneErrorCode = a.getErrorCode(); - if (amazoneErrorCode != null && !amazoneErrorCode.isEmpty()) { - if (amazoneErrorCode.equalsIgnoreCase("slowdown")) { - backupMetrics.incrementAwsSlowDownException(1); - logger.warn("Received slow down from AWS when uploading file: {}", path.getFileName()); - } - } - } - - logger.error("Error uploading file {}, a datapart was not uploaded.", path.getFileName(), e); - notifyEventFailure(new BackupEvent(path)); - return new BackupRestoreException("Error uploading file " + path.getFileName(), e); - } - - abstract void uploadFileImpl(AbstractBackupPath path, InputStream in, long chunkSize) throws BackupRestoreException; - - /** - * This method does exactly as other download method.(Supposed to be overridden) - * filePath parameter provides the diskPath of the downloaded file. - * This path can be used to correlate the files which are Streamed In - * during Incremental Restores - */ - @Override - public void download(AbstractBackupPath path, OutputStream os, - String filePath) throws BackupRestoreException { - download(path, os); - } - - @Override - public void download(AbstractBackupPath path, OutputStream os) throws BackupRestoreException { - logger.info("Downloading {} from S3 bucket {}", path.getRemotePath(), getPrefix(this.config)); - long contentLen = s3Client.getObjectMetadata(getPrefix(config), path.getRemotePath()).getContentLength(); - path.setSize(contentLen); - try { - downloadFileImpl(path, os); - bytesDownloaded.addAndGet(contentLen); - backupMetrics.incrementValidDownloads(); - } catch (BackupRestoreException e) { - backupMetrics.incrementInvalidDownloads(); - throw e; + throw new BackupRestoreException("Error uploading file as ETag or CompleteMultipartUploadResult is NULL -" + localPath); } } - abstract void downloadFileImpl(AbstractBackupPath path, OutputStream os) throws BackupRestoreException; - - @Override - public long getBytesUploaded() { - return bytesUploaded.get(); - } - @Override - public long getAWSSlowDownExceptionCounter() { - return backupMetrics.getAwsSlowDownException(); - } - - public long downloadCount() { - return backupMetrics.getValidDownloads(); - } - - public long uploadCount() { - return backupMetrics.getValidUploads(); + public long getFileSize(Path remotePath) throws BackupRestoreException{ + return s3Client.getObjectMetadata(getPrefix(config), remotePath.toString()).getContentLength(); } @Override @@ -327,40 +192,15 @@ public Iterator list(String path, Date start, Date till) { return new S3FileIterator(pathProvider, s3Client, path, start, till); } + final long getChunkSize(Path localPath) throws BackupRestoreException{ + long chunkSize = config.getBackupChunkSize(); + long fileSize = localPath.toFile().length(); - @Override - public final void addObserver(EventObserver observer) { - if (observer == null) - throw new NullPointerException("observer must not be null."); - - observers.addIfAbsent(observer); - } - - @Override - public void removeObserver(EventObserver observer) { - if (observer == null) - throw new NullPointerException("observer must not be null."); - - observers.remove(observer); - } - - @Override - public void notifyEventStart(BackupEvent event) { - observers.forEach(eventObserver -> eventObserver.updateEventStart(event)); - } - - @Override - public void notifyEventSuccess(BackupEvent event) { - observers.forEach(eventObserver -> eventObserver.updateEventSuccess(event)); - } + //compute the size of each block we will upload to endpoint + if (fileSize > 0) + chunkSize = (fileSize / chunkSize >= MAX_CHUNKS) ? (fileSize / (MAX_CHUNKS - 1)) : chunkSize; - @Override - public void notifyEventFailure(BackupEvent event) { - observers.forEach(eventObserver -> eventObserver.updateEventFailure(event)); + return chunkSize; } - @Override - public void notifyEventStop(BackupEvent event) { - observers.forEach(eventObserver -> eventObserver.updateEventStop(event)); - } } diff --git a/priam/src/main/java/com/netflix/priam/aws/S3FileSystemMBean.java b/priam/src/main/java/com/netflix/priam/aws/S3FileSystemMBean.java deleted file mode 100644 index 5466d6a39..000000000 --- a/priam/src/main/java/com/netflix/priam/aws/S3FileSystemMBean.java +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Copyright 2013 Netflix, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ -package com.netflix.priam.aws; - -public interface S3FileSystemMBean { - String MBEAN_NAME = "com.priam.aws.S3FileSystemMBean:name=S3FileSystemMBean"; - - long downloadCount(); - - long uploadCount(); - - int getActivecount(); - - long bytesUploaded(); - - long bytesDownloaded(); -} diff --git a/priam/src/main/java/com/netflix/priam/backup/AbstractBackup.java b/priam/src/main/java/com/netflix/priam/backup/AbstractBackup.java index adcceded7..ce9450619 100644 --- a/priam/src/main/java/com/netflix/priam/backup/AbstractBackup.java +++ b/priam/src/main/java/com/netflix/priam/backup/AbstractBackup.java @@ -28,6 +28,7 @@ import org.slf4j.LoggerFactory; import java.io.File; +import java.nio.file.Paths; import java.util.List; /** @@ -108,20 +109,11 @@ public AbstractBackupPath retriableCall() throws Exception { * * @param bp backup path to be uploaded. */ - protected void upload(final AbstractBackupPath bp) throws Exception { - java.io.InputStream is = null; + private void upload(final AbstractBackupPath bp) throws Exception { try { - is = bp.localReader(); - if (is == null) { - throw new NullPointerException("Unable to get handle on file: " + bp.fileName); - } - fs.upload(bp, is); - bp.setCompressedFileSize(fs.getBytesUploaded()); + fs.uploadFile(Paths.get(bp.getBackupFile().getAbsolutePath()), Paths.get(bp.getRemotePath()), bp); } catch (Exception e) { logger.error("Exception uploading local file {}, releasing handle, and will retry.", bp.backupFile.getCanonicalFile()); - if (is != null) { - is.close(); - } throw e; } } diff --git a/priam/src/main/java/com/netflix/priam/backup/AbstractBackupPath.java b/priam/src/main/java/com/netflix/priam/backup/AbstractBackupPath.java index 6958fa97c..a2f66a1a2 100644 --- a/priam/src/main/java/com/netflix/priam/backup/AbstractBackupPath.java +++ b/priam/src/main/java/com/netflix/priam/backup/AbstractBackupPath.java @@ -64,8 +64,6 @@ public static boolean isDataFile(BackupFileType type){ protected Date time; protected long size; //uncompressed file size protected long compressedFileSize = 0; - protected boolean isCassandra1_0; - protected final InstanceIdentity factory; protected final IConfiguration config; protected File backupFile; @@ -125,7 +123,6 @@ public void parseLocal(File file, BackupFileType type) throws ParseException { this.type = type; if (BackupFileType.isDataFile(type)) { this.keyspace = elements[0]; - if (!isCassandra1_0) this.columnFamily = elements[1]; } if (type == BackupFileType.SNAP) @@ -157,15 +154,10 @@ public File newRestoreFile() { if (type == BackupFileType.CL) { buff.append(config.getBackupCommitLogLocation()).append(PATH_SEP); } else { - buff.append(config.getDataFileLocation()).append(PATH_SEP); - if (type != BackupFileType.META && type != BackupFileType.META_V2) { - if (isCassandra1_0) - buff.append(keyspace).append(PATH_SEP); - else + if (type != BackupFileType.META && type != BackupFileType.META_V2) buff.append(keyspace).append(PATH_SEP).append(columnFamily).append(PATH_SEP); } - } buff.append(fileName); @@ -177,7 +169,6 @@ public File newRestoreFile() { } - @Override public int compareTo(AbstractBackupPath o) { return getRemotePath().compareTo(o.getRemotePath()); @@ -279,14 +270,6 @@ public File getBackupFile() { return backupFile; } - public boolean isCassandra1_0() { - return isCassandra1_0; - } - - public void setCassandra1_0(boolean isCassandra1_0) { - this.isCassandra1_0 = isCassandra1_0; - } - public void setFileName(String fileName) { this.fileName = fileName; } diff --git a/priam/src/main/java/com/netflix/priam/backup/AbstractFileSystem.java b/priam/src/main/java/com/netflix/priam/backup/AbstractFileSystem.java new file mode 100644 index 000000000..e0770b36b --- /dev/null +++ b/priam/src/main/java/com/netflix/priam/backup/AbstractFileSystem.java @@ -0,0 +1,128 @@ +/* + * Copyright 2018 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.netflix.priam.backup; + +import com.google.inject.Inject; +import com.netflix.priam.backupv2.FileUploadResult; +import com.netflix.priam.merics.BackupMetrics; +import com.netflix.priam.notification.BackupEvent; +import com.netflix.priam.notification.BackupNotificationMgr; +import com.netflix.priam.notification.EventGenerator; +import com.netflix.priam.notification.EventObserver; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.file.Path; +import java.util.Date; +import java.util.Iterator; +import java.util.concurrent.CopyOnWriteArrayList; + +/** + * Created by aagrawal on 8/30/18. + */ +public abstract class AbstractFileSystem implements IBackupFileSystem, EventGenerator { + private static final Logger logger = LoggerFactory.getLogger(AbstractFileSystem.class); + private final CopyOnWriteArrayList> observers = new CopyOnWriteArrayList<>(); + protected BackupMetrics backupMetrics; + + @Inject + public AbstractFileSystem(BackupMetrics backupMetrics, + BackupNotificationMgr backupNotificationMgr){ + this.backupMetrics = backupMetrics; + //Add notifications. + this.addObserver(backupNotificationMgr); + } + + @Override + public void downloadFile(Path remotePath, Path localPath) throws BackupRestoreException{ + logger.info("Downloading file: {} to location: {}", remotePath, localPath); + try{ + //TODO: Retries should ideally go here. + downloadFileImpl(remotePath, localPath); + backupMetrics.recordDownloadRate(getFileSize(remotePath)); + logger.info("Successfully downloaded file: {} to location: {}", remotePath, localPath); + }catch (BackupRestoreException e){ + backupMetrics.incrementInvalidDownloads(); + logger.error("Error while downloading file: {} to location: {}", remotePath, localPath); + throw e; + } + } + + protected abstract void downloadFileImpl(Path remotePath, Path localPath) throws BackupRestoreException; + + @Override + public void uploadFile(Path localPath, Path remotePath, AbstractBackupPath path) throws BackupRestoreException{ + if (!localPath.toFile().exists()) + throw new BackupRestoreException("File do not exist: {}" + localPath); + + logger.info("Uploading file: {} to location: {}", localPath, remotePath); + try{ + notifyEventStart(new BackupEvent(path)); + //TODO: Retries should ideally go here. + long uploadedFileSize = uploadFileImpl(localPath, remotePath); + backupMetrics.recordUploadRate(uploadedFileSize); + notifyEventSuccess(new BackupEvent(path)); + logger.info("Successfully uploaded file: {} to location: {}", localPath, remotePath); + }catch (BackupRestoreException e){ + backupMetrics.incrementInvalidUploads(); + notifyEventFailure(new BackupEvent(path)); + logger.error("Error while uploading file: {} to location: {}", localPath, remotePath); + throw e; + } + } + + protected abstract long uploadFileImpl(Path localPath, Path remotePath) throws BackupRestoreException; + + @Override + public final void addObserver(EventObserver observer) { + if (observer == null) + throw new NullPointerException("observer must not be null."); + + observers.addIfAbsent(observer); + } + + @Override + public void removeObserver(EventObserver observer) { + if (observer == null) + throw new NullPointerException("observer must not be null."); + + observers.remove(observer); + } + + @Override + public void notifyEventStart(BackupEvent event) { + observers.forEach(eventObserver -> eventObserver.updateEventStart(event)); + } + + @Override + public void notifyEventSuccess(BackupEvent event) { + observers.forEach(eventObserver -> eventObserver.updateEventSuccess(event)); + } + + @Override + public void notifyEventFailure(BackupEvent event) { + observers.forEach(eventObserver -> eventObserver.updateEventFailure(event)); + } + + @Override + public void notifyEventStop(BackupEvent event) { + observers.forEach(eventObserver -> eventObserver.updateEventStop(event)); + } +} diff --git a/priam/src/main/java/com/netflix/priam/backup/BackupVerification.java b/priam/src/main/java/com/netflix/priam/backup/BackupVerification.java index 03c896f3e..f89a4cb69 100644 --- a/priam/src/main/java/com/netflix/priam/backup/BackupVerification.java +++ b/priam/src/main/java/com/netflix/priam/backup/BackupVerification.java @@ -29,6 +29,7 @@ import java.io.FileReader; import java.nio.file.FileSystems; import java.nio.file.Path; +import java.nio.file.Paths; import java.util.*; import java.util.stream.Collectors; @@ -117,7 +118,7 @@ public BackupVerificationResult verifyBackup(List metadata, Date List metaFileList = new ArrayList<>(); try { Path metaFileLocation = FileSystems.getDefault().getPath(config.getDataFileLocation(), "tmp_meta.json"); - bkpStatusFs.download(metas.get(0), new FileOutputStream(metaFileLocation.toFile())); + bkpStatusFs.downloadFile(Paths.get(metas.get(0).getRemotePath()), metaFileLocation); logger.info("Meta file successfully downloaded to localhost: {}", metaFileLocation.toString()); JSONParser jsonParser = new JSONParser(); diff --git a/priam/src/main/java/com/netflix/priam/backup/CommitLogBackup.java b/priam/src/main/java/com/netflix/priam/backup/CommitLogBackup.java index 2cacb6ff8..c5f995034 100644 --- a/priam/src/main/java/com/netflix/priam/backup/CommitLogBackup.java +++ b/priam/src/main/java/com/netflix/priam/backup/CommitLogBackup.java @@ -26,6 +26,7 @@ import org.slf4j.LoggerFactory; import java.io.File; +import java.nio.file.Paths; import java.util.ArrayList; import java.util.List; @@ -94,7 +95,7 @@ private void upload(final AbstractBackupPath bp) new RetryableCallable() { public Void retriableCall() throws Exception { - fs.upload(bp, bp.localReader()); + fs.uploadFile(Paths.get(bp.getBackupFile().getAbsolutePath()), Paths.get(bp.getRemotePath()), bp); return null; } } diff --git a/priam/src/main/java/com/netflix/priam/backup/IBackupFileSystem.java b/priam/src/main/java/com/netflix/priam/backup/IBackupFileSystem.java index 0036d6736..1261201b6 100644 --- a/priam/src/main/java/com/netflix/priam/backup/IBackupFileSystem.java +++ b/priam/src/main/java/com/netflix/priam/backup/IBackupFileSystem.java @@ -16,8 +16,7 @@ */ package com.netflix.priam.backup; -import java.io.InputStream; -import java.io.OutputStream; +import java.nio.file.Path; import java.util.Date; import java.util.Iterator; @@ -26,22 +25,23 @@ */ public interface IBackupFileSystem { /** - * Write the contents of the specified remote path to the output stream and - * close + * Download the file denoted by remotePath to the local file system denoted by local path. + * + * @param remotePath fully qualified location of the file on remote file system. + * @param localPath location on the local file sytem where remote file should be downloaded. + * @throws BackupRestoreException if file is not available, downloadable or any other error from remote file system. */ - void download(AbstractBackupPath path, OutputStream os) throws BackupRestoreException; + void downloadFile(Path remotePath, Path localPath) throws BackupRestoreException; /** - * Write the contents of the specified remote path to the output stream and close. - * filePath denotes the diskPath of the downloaded file + * Upload the local file denoted by localPath to the remote file system at location denoted by remotePath. + * + * @param localPath Path of the local file that needs to be uploaded. + * @param remotePath Fully qualified path on the remote file system where file should be uploaded. + * @param path AbstractBackupPath to be used to send backup notifications only. + * @throws BackupRestoreException in case of failure to upload for any reason including file not available, readable or remote file system errors. */ - void download(AbstractBackupPath path, OutputStream os, String filePath) throws BackupRestoreException; - - /** - * Upload/Backup to the specified location with contents from the input - * stream. Closes the InputStream after its done. - */ - void upload(AbstractBackupPath path, InputStream in) throws BackupRestoreException; + void uploadFile(Path localPath, Path remotePath, AbstractBackupPath path) throws BackupRestoreException; /** * List all files in the backup location for the specified time range. @@ -58,17 +58,18 @@ public interface IBackupFileSystem { */ void cleanup(); - /** - * Get number of active upload or downloads - */ - int getActivecount(); - /** * Give the file system a chance to terminate any thread pools, etc. */ void shutdown(); - long getBytesUploaded(); + /** + * Get the size of the remote object + * + * @param remotePath Location of the object on the remote file system. + * @return size of the object on the remote filesystem. + * @throws BackupRestoreException in case of failure to read object denoted by remotePath or any other error. + */ + long getFileSize(Path remotePath) throws BackupRestoreException; - long getAWSSlowDownExceptionCounter(); } diff --git a/priam/src/main/java/com/netflix/priam/backup/MetaData.java b/priam/src/main/java/com/netflix/priam/backup/MetaData.java index 17f172f5b..df0cb85da 100644 --- a/priam/src/main/java/com/netflix/priam/backup/MetaData.java +++ b/priam/src/main/java/com/netflix/priam/backup/MetaData.java @@ -30,6 +30,7 @@ import org.slf4j.LoggerFactory; import java.io.*; +import java.nio.file.Paths; import java.text.ParseException; import java.util.ArrayList; import java.util.List; @@ -99,7 +100,7 @@ public Boolean doesExist(final AbstractBackupPath meta) { new RetryableCallable() { @Override public Void retriableCall() throws Exception { - fs.download(meta, new FileOutputStream(meta.newRestoreFile())); //download actual file to disk + fs.downloadFile(Paths.get(meta.getRemotePath()),Paths.get(meta.newRestoreFile().getAbsolutePath())); //download actual file to disk return null; } }.call(); @@ -116,12 +117,10 @@ private void upload(final AbstractBackupPath bp) throws Exception { new RetryableCallable() { @Override public Void retriableCall() throws Exception { - fs.upload(bp, bp.localReader()); + fs.uploadFile(Paths.get(bp.getBackupFile().getAbsolutePath()), Paths.get(bp.getRemotePath()), bp); return null; } }.call(); - - bp.setCompressedFileSize(fs.getBytesUploaded()); } public File createTmpMetaFile() throws IOException { diff --git a/priam/src/main/java/com/netflix/priam/backup/parallel/IncrementalConsumer.java b/priam/src/main/java/com/netflix/priam/backup/parallel/IncrementalConsumer.java index bc1a5509d..098c3a6b4 100644 --- a/priam/src/main/java/com/netflix/priam/backup/parallel/IncrementalConsumer.java +++ b/priam/src/main/java/com/netflix/priam/backup/parallel/IncrementalConsumer.java @@ -21,6 +21,8 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.nio.file.Paths; + /* * Performs an upload of a file, with retries. */ @@ -64,32 +66,15 @@ public void run() { @Override public Void retriableCall() throws Exception { - java.io.InputStream is = null; - try { - is = bp.localReader(); - } catch (java.io.FileNotFoundException | RuntimeException e) { - if (is != null) { - is.close(); - } + if (!bp.getBackupFile().exists()) throw new java.util.concurrent.CancellationException("Someone beat me to uploading this file" + ", no need to retry. Most likely not needed but to be safe, checked and released handle to file if appropriate."); - } try { - if (is == null) { - throw new NullPointerException("Unable to get handle on file: " + bp.getFileName()); - } - // Important context: this upload call typically has internal retries but those are only - // to cover over very temporary (<10s) network partitions. For larger partitions re rely on - // higher up retries and re-enqueues. - fs.upload(bp, is); - bp.setCompressedFileSize(fs.getBytesUploaded()); + fs.uploadFile(Paths.get(bp.getBackupFile().getAbsolutePath()), Paths.get(bp.getRemotePath()), bp); return null; } catch (Exception e) { logger.error("Exception uploading local file {}, releasing handle, and will retry.", bp.getFileName()); - if (is != null) { - is.close(); - } throw e; } } @@ -110,4 +95,4 @@ public Void retriableCall() throws Exception { logger.info("Consumer - done with upload file: {}", this.bp.getFileName()); } -} \ No newline at end of file +} diff --git a/priam/src/main/java/com/netflix/priam/backupv2/MetaFileWriterBuilder.java b/priam/src/main/java/com/netflix/priam/backupv2/MetaFileWriterBuilder.java index ffccdeec1..b2ca7e2ef 100644 --- a/priam/src/main/java/com/netflix/priam/backupv2/MetaFileWriterBuilder.java +++ b/priam/src/main/java/com/netflix/priam/backupv2/MetaFileWriterBuilder.java @@ -163,8 +163,7 @@ public void uploadMetaFile(boolean deleteOnSuccess) throws Exception { new RetryableCallable(6, 5000) { @Override public Void retriableCall() throws Exception { - backupFileSystem.upload(abstractBackupPath, abstractBackupPath.localReader()); - abstractBackupPath.setCompressedFileSize(backupFileSystem.getBytesUploaded()); + backupFileSystem.uploadFile(metaFilePath, Paths.get(abstractBackupPath.getRemotePath()), abstractBackupPath); return null; } }.call(); diff --git a/priam/src/main/java/com/netflix/priam/google/GoogleEncryptedFileSystem.java b/priam/src/main/java/com/netflix/priam/google/GoogleEncryptedFileSystem.java index 9ce1957c3..2e50a1298 100755 --- a/priam/src/main/java/com/netflix/priam/google/GoogleEncryptedFileSystem.java +++ b/priam/src/main/java/com/netflix/priam/google/GoogleEncryptedFileSystem.java @@ -31,17 +31,17 @@ import com.netflix.priam.cred.ICredentialGeneric.KEY; import com.netflix.priam.aws.S3BackupPath; import com.netflix.priam.backup.AbstractBackupPath; +import com.netflix.priam.backup.AbstractFileSystem; import com.netflix.priam.backup.BackupRestoreException; -import com.netflix.priam.backup.IBackupFileSystem; +import com.netflix.priam.merics.BackupMetrics; +import com.netflix.priam.notification.BackupNotificationMgr; import org.apache.commons.io.IOUtils; import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import javax.management.MBeanServer; -import javax.management.ObjectName; import java.io.*; -import java.lang.management.ManagementFactory; +import java.nio.file.Path; import java.util.ArrayList; import java.util.Collection; import java.util.Date; @@ -49,7 +49,7 @@ import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; -public class GoogleEncryptedFileSystem implements IBackupFileSystem, GoogleEncryptedFileSystemMBean { +public class GoogleEncryptedFileSystem extends AbstractFileSystem { private static final Logger logger = LoggerFactory.getLogger(GoogleEncryptedFileSystem.class); @@ -64,46 +64,34 @@ public class GoogleEncryptedFileSystem implements IBackupFileSystem, GoogleEncry private Provider pathProvider; private String srcBucketName; private IConfiguration config; - private AtomicInteger downloadCount = new AtomicInteger(); - protected AtomicLong bytesDownloaded = new AtomicLong(); private ICredentialGeneric gcsCredential; + private BackupMetrics backupMetrics; @Inject public GoogleEncryptedFileSystem(Provider pathProvider, final IConfiguration config - , @Named("gcscredential") ICredentialGeneric credential) { - + , @Named("gcscredential") ICredentialGeneric credential, BackupMetrics backupMetrics, BackupNotificationMgr backupNotificationManager) { + super(backupMetrics, backupNotificationManager); + this.backupMetrics = backupMetrics; this.pathProvider = pathProvider; this.config = config; this.gcsCredential = credential; try { - this.httpTransport = GoogleNetHttpTransport.newTrustedTransport(); - } catch (Exception e) { throw new IllegalStateException("Unable to create a handle to the Google Http tranport", e); } this.srcBucketName = getSourcebucket(getPathPrefix()); - - MBeanServer mbs = ManagementFactory.getPlatformMBeanServer(); - String mbeanName = MBEAN_NAME; - try { - mbs.registerMBean(this, new ObjectName(mbeanName)); - } catch (Exception e) { - throw new RuntimeException("Unable to regiser JMX bean: " + mbeanName + " to JMX server. Msg: " + e.getLocalizedMessage(), e); - } } /* * @param pathprefix - the absolute path (including bucket name) to the object. */ private String getSourcebucket(String pathPrefix) { - String[] paths = pathPrefix.split(String.valueOf(S3BackupPath.PATH_SEP)); return paths[0]; - } private Storage.Objects constructObjectResourceHandle() { @@ -112,9 +100,7 @@ private Storage.Objects constructObjectResourceHandle() { } constructGcsStorageHandle(); - this.objectsResoruceHandle = this.gcsStorageHandle.objects(); - return this.objectsResoruceHandle; } @@ -124,16 +110,13 @@ private Storage.Objects constructObjectResourceHandle() { * * Note: GCS storage will use our credential to do auto-refresh of expired tokens */ - private Storage constructGcsStorageHandle() { if (this.gcsStorageHandle != null) { return this.gcsStorageHandle; } try { - constructGcsCredential(); - } catch (Exception e) { throw new IllegalStateException("Exception during GCS authorization", e); } @@ -164,26 +147,14 @@ private Credential constructGcsCredential() throws Exception { //Take the encrypted private key, decrypted into an in-transit file which is passed to GCS File gcsPrivateKeyHandle = new File(this.config.getGcsServiceAccountPrivateKeyLoc() + ".output"); - OutputStream os = new FileOutputStream(gcsPrivateKeyHandle); - BufferedOutputStream bos = new BufferedOutputStream(os); ByteArrayOutputStream byteos = new ByteArrayOutputStream(); byte[] gcsPrivateKeyPlainText = this.gcsCredential.getValue(KEY.GCS_PRIVATE_KEY_LOC); - try { - + try(BufferedOutputStream bos = new BufferedOutputStream(new FileOutputStream(gcsPrivateKeyHandle))) { byteos.write(gcsPrivateKeyPlainText); byteos.writeTo(bos); - } catch (IOException e) { - throw new IOException("Exception when writing decrypted gcs private key value to disk.", e); - - } finally { - try { - bos.close(); - } catch (IOException e) { - throw new IOException("Exception when closing decrypted gcs private key value to disk.", e); - } } Collection scopes = new ArrayList(1); @@ -195,68 +166,37 @@ private Credential constructGcsCredential() throws Exception { .setServiceAccountPrivateKeyFromP12File(gcsPrivateKeyHandle) //Cryptex decrypted service account key derive from the GCS console .build(); } - } return this.credential; } @Override - public void download(AbstractBackupPath path, OutputStream os) throws BackupRestoreException { - - logger.info("Downloading {} from GCS bucket {}", path.getRemotePath(), this.srcBucketName); - this.downloadCount.incrementAndGet(); - + protected void downloadFileImpl(Path remotePath, Path localPath) throws BackupRestoreException{ + logger.info("Downloading {} from GCS bucket {}", remotePath, this.srcBucketName); String objectName = parseObjectname(getPathPrefix()); - com.google.api.services.storage.Storage.Objects.Get get = null; - try { - - get = constructObjectResourceHandle().get(this.srcBucketName, path.getRemotePath()); + try { + get = constructObjectResourceHandle().get(this.srcBucketName, remotePath.toString()); } catch (IOException e) { throw new BackupRestoreException("IO error retrieving metadata for: " + objectName + " from bucket: " + this.srcBucketName, e); } get.getMediaHttpDownloader().setDirectDownloadEnabled(true); // If you're not using GCS' AppEngine, download the whole thing (instead of chunks) in one request, if possible. InputStream is = null; - try { - + try(OutputStream os = new FileOutputStream(localPath.toFile())) { is = get.executeMediaAsInputStream(); IOUtils.copyLarge(is, os); - - } catch (IOException e) { throw new BackupRestoreException("IO error during streaming of object: " + objectName + " from bucket: " + this.srcBucketName, e); } catch (Exception ex) { - throw new BackupRestoreException("Exception encountered when copying bytes from input to output", ex); - } finally { IOUtils.closeQuietly(is); - IOUtils.closeQuietly(os); } - bytesDownloaded.addAndGet(get.getLastResponseHeaders().getContentLength()); - - } - - @Override - public void download(AbstractBackupPath path, OutputStream os, String filePath) throws BackupRestoreException { - try { - - download(path, os); - - } catch (Exception e) { - throw new BackupRestoreException(e.getMessage(), e); - } - } - - @Override - public void upload(AbstractBackupPath path, InputStream in) - throws BackupRestoreException { - throw new UnsupportedOperationException(); - + backupMetrics.recordDownloadRate(get.getLastResponseHeaders().getContentLength()); } @Override @@ -273,59 +213,27 @@ public Iterator listPrefixes(Date date) { @Override public void cleanup() { // TODO Auto-generated method stub - - } - - @Override - public int getActivecount() { - // TODO Auto-generated method stub - return 0; - } @Override public void shutdown() { // TODO Auto-generated method stub - } @Override - public int downloadCount() { - return this.downloadCount.get(); - } - - @Override - public int uploadCount() { - // TODO Auto-generated method stub - return 0; - } - - @Override - public long bytesUploaded() { - // TODO Auto-generated method stub - return 0; - } - - @Override - public long getBytesUploaded() { - return 0; + protected long uploadFileImpl(Path localPath, Path remotePath) throws BackupRestoreException { + throw new UnsupportedOperationException(); } @Override - public long getAWSSlowDownExceptionCounter() { + public long getFileSize(Path remotePath) throws BackupRestoreException { return 0; } - @Override - public long bytesDownloaded() { - return this.bytesDownloaded.get(); - } - /** * Get restore prefix which will be used to locate GVS files */ - public String getPathPrefix() { - + private String getPathPrefix() { String prefix; if (StringUtils.isNotBlank(config.getRestorePrefix())) prefix = config.getRestorePrefix(); @@ -339,11 +247,8 @@ public String getPathPrefix() { * @param pathPrefix * @return objectName */ - public static String parseObjectname(String pathPrefix) { + static String parseObjectname(String pathPrefix) { int offset = pathPrefix.lastIndexOf(0x2f); return pathPrefix.substring(offset + 1); - } - - } \ No newline at end of file diff --git a/priam/src/main/java/com/netflix/priam/google/GoogleEncryptedFileSystemMBean.java b/priam/src/main/java/com/netflix/priam/google/GoogleEncryptedFileSystemMBean.java deleted file mode 100755 index a235316d3..000000000 --- a/priam/src/main/java/com/netflix/priam/google/GoogleEncryptedFileSystemMBean.java +++ /dev/null @@ -1,31 +0,0 @@ -/** - * Copyright 2017 Netflix, Inc. - *

- * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.netflix.priam.google; - -public interface GoogleEncryptedFileSystemMBean { - - String MBEAN_NAME = "com.priam.google.GoogleEncryptedFileSystemMBean:name=GoogleEncryptedFileSystemMBean"; - - int downloadCount(); - - int uploadCount(); - - int getActivecount(); - - long bytesUploaded(); - - long bytesDownloaded(); -} \ No newline at end of file diff --git a/priam/src/main/java/com/netflix/priam/merics/BackupMetrics.java b/priam/src/main/java/com/netflix/priam/merics/BackupMetrics.java index 8b12b53aa..5a8c079f4 100644 --- a/priam/src/main/java/com/netflix/priam/merics/BackupMetrics.java +++ b/priam/src/main/java/com/netflix/priam/merics/BackupMetrics.java @@ -30,54 +30,48 @@ public class BackupMetrics { /** * Distribution summary will provide the metric like count (how many uploads were made), max no. of bytes uploaded and total amount of bytes uploaded. */ - private final DistributionSummary uploadRate; - private final Counter validUploads, invalidUploads, validDownloads, invalidDownloads, awsSlowDownException, snsNotificationSuccess, snsNotificationFailure; + private final DistributionSummary uploadRate, downloadRate; + private final Counter invalidUploads, invalidDownloads, snsNotificationSuccess, snsNotificationFailure, forgottenFiles; @Inject public BackupMetrics(Registry registry) { - validDownloads = registry.counter(Metrics.METRIC_PREFIX + "download.valid"); invalidDownloads = registry.counter(Metrics.METRIC_PREFIX + "download.invalid"); - validUploads = registry.counter(Metrics.METRIC_PREFIX + "upload.valid"); invalidUploads = registry.counter(Metrics.METRIC_PREFIX + "upload.invalid"); uploadRate = registry.distributionSummary(Metrics.METRIC_PREFIX + "upload.rate"); - awsSlowDownException = registry.counter(Metrics.METRIC_PREFIX + "aws.slowDown"); + downloadRate = registry.distributionSummary(Metrics.METRIC_PREFIX + "download.rate"); snsNotificationSuccess = registry.counter(Metrics.METRIC_PREFIX + "sns.notification.success"); snsNotificationFailure = registry.counter(Metrics.METRIC_PREFIX + "sns.notification.failure"); + forgottenFiles = registry.counter(Metrics.METRIC_PREFIX + "forgotten.files"); } - public void incrementValidUploads() { - this.validUploads.increment(); + public DistributionSummary getUploadRate() { + return uploadRate; } - public void incrementInvalidUploads() { - this.invalidUploads.increment(); + public Counter getInvalidUploads() { + return invalidUploads; } - public long getValidUploads() { - return this.validUploads.count(); + public Counter getInvalidDownloads() { + return invalidDownloads; } - public long getValidDownloads() { - return this.validDownloads.count(); + public Counter getSnsNotificationSuccess() { + return snsNotificationSuccess; } - public void incrementValidDownloads() { - this.invalidDownloads.increment(); + public Counter getSnsNotificationFailure() { + return snsNotificationFailure; } + public void incrementInvalidUploads() { + this.invalidUploads.increment(); + } public void incrementInvalidDownloads() { this.invalidDownloads.increment(); } - public long getAwsSlowDownException() { - return awsSlowDownException.count(); - } - - public void incrementAwsSlowDownException(int awsSlowDown) { - awsSlowDownException.increment(awsSlowDown); - } - public void incrementSnsNotificationSuccess() { snsNotificationSuccess.increment(); } @@ -90,4 +84,13 @@ public void recordUploadRate(long sizeInBytes) { uploadRate.record(sizeInBytes); } + public void incrementForgottenFiles(long forgottenFilesVal) {forgottenFiles.increment(forgottenFilesVal);} + + public void recordDownloadRate(long sizeInBytes) { + downloadRate.record(sizeInBytes); + } + + public DistributionSummary getDownloadRate() { + return downloadRate; + } } diff --git a/priam/src/main/java/com/netflix/priam/resources/BackupServlet.java b/priam/src/main/java/com/netflix/priam/resources/BackupServlet.java index 88754fc72..14d71db3e 100644 --- a/priam/src/main/java/com/netflix/priam/resources/BackupServlet.java +++ b/priam/src/main/java/com/netflix/priam/resources/BackupServlet.java @@ -166,10 +166,7 @@ public Response list(@QueryParam(REST_HEADER_RANGE) String daterange, @QueryPara public Response status() throws Exception { int restoreTCount = restoreObj.getActiveCount(); //Active threads performing the restore logger.debug("Thread counts for restore is: {}", restoreTCount); - int backupTCount = backupFs.getActivecount(); - logger.debug("Thread counts for snapshot backup is: {}", backupTCount); JSONObject object = new JSONObject(); - object.put("ThreadCount", new Integer(backupTCount)); //Number of active threads performing the snapshot backups object.put("SnapshotStatus", snapshotBackup.state().toString()); return Response.ok(object.toString(), MediaType.APPLICATION_JSON).build(); } diff --git a/priam/src/main/java/com/netflix/priam/restore/EncryptedRestoreBase.java b/priam/src/main/java/com/netflix/priam/restore/EncryptedRestoreBase.java index a9fd1fc88..18d3bb3f9 100755 --- a/priam/src/main/java/com/netflix/priam/restore/EncryptedRestoreBase.java +++ b/priam/src/main/java/com/netflix/priam/restore/EncryptedRestoreBase.java @@ -32,6 +32,7 @@ import org.slf4j.LoggerFactory; import java.io.*; +import java.nio.file.Paths; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.atomic.AtomicInteger; @@ -80,7 +81,7 @@ public Integer retriableCall() throws Exception { try { logger.info("Downloading file from: {} to: {}", path.getRemotePath(), tempFile.getAbsolutePath()); - fs.download(path, new FileOutputStream(tempFile), tempFile.getAbsolutePath()); + fs.downloadFile(Paths.get(path.getRemotePath()), Paths.get(tempFile.getAbsolutePath())); tracker.adjustAndAdd(path); logger.info("Completed downloading file from: {} to: {}", path.getRemotePath(), tempFile.getAbsolutePath()); diff --git a/priam/src/main/java/com/netflix/priam/restore/Restore.java b/priam/src/main/java/com/netflix/priam/restore/Restore.java index 7ed6d2aed..640c3d4ed 100644 --- a/priam/src/main/java/com/netflix/priam/restore/Restore.java +++ b/priam/src/main/java/com/netflix/priam/restore/Restore.java @@ -37,6 +37,7 @@ import java.io.File; import java.io.FileOutputStream; +import java.nio.file.Paths; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.atomic.AtomicInteger; @@ -66,7 +67,7 @@ protected final void downloadFile(final AbstractBackupPath path, final File rest @Override public Integer retriableCall() throws Exception { logger.info("Downloading file: {} to: {}", path.getRemotePath(), restoreLocation.getAbsolutePath()); - fs.download(path, new FileOutputStream(restoreLocation), restoreLocation.getAbsolutePath()); + fs.downloadFile(Paths.get(path.getRemotePath()), Paths.get(restoreLocation.getAbsolutePath())); tracker.adjustAndAdd(path); // TODO: fix me -> if there is exception the why hang? logger.info("Completed download of file: {} to: {}", path.getRemotePath(), restoreLocation.getAbsolutePath()); diff --git a/priam/src/test/java/com/netflix/priam/backup/BRTestModule.java b/priam/src/test/java/com/netflix/priam/backup/BRTestModule.java index 7c5e0ceb3..3afe0325d 100644 --- a/priam/src/test/java/com/netflix/priam/backup/BRTestModule.java +++ b/priam/src/test/java/com/netflix/priam/backup/BRTestModule.java @@ -62,7 +62,7 @@ protected void configure() bind(IS3Credential.class).annotatedWith(Names.named("awss3roleassumption")).to(S3RoleAssumptionCredential.class); - bind(IBackupFileSystem.class).annotatedWith(Names.named("encryptedbackup")).to(FakedS3EncryptedFileSystem.class); + bind(IBackupFileSystem.class).annotatedWith(Names.named("encryptedbackup")).to(NullBackupFileSystem.class); bind(IFileCryptography.class).annotatedWith(Names.named("filecryptoalgorithm")).to(PgpCryptography.class); bind(IIncrementalBackup.class).to(IncrementalBackup.class); bind(InstanceEnvIdentity.class).to(FakeInstanceEnvIdentity.class); diff --git a/priam/src/test/java/com/netflix/priam/backup/FakeBackupFileSystem.java b/priam/src/test/java/com/netflix/priam/backup/FakeBackupFileSystem.java index c653ad295..ab5e2f9e5 100644 --- a/priam/src/test/java/com/netflix/priam/backup/FakeBackupFileSystem.java +++ b/priam/src/test/java/com/netflix/priam/backup/FakeBackupFileSystem.java @@ -22,18 +22,19 @@ import com.google.inject.Singleton; import com.netflix.priam.aws.S3BackupPath; import com.netflix.priam.backup.AbstractBackupPath.BackupFileType; -import org.apache.commons.io.IOUtils; +import com.netflix.priam.merics.BackupMetrics; +import com.netflix.priam.notification.BackupNotificationMgr; import org.json.simple.JSONArray; import java.io.FileWriter; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; +import java.nio.file.Path; import java.util.*; @Singleton -public class FakeBackupFileSystem implements IBackupFileSystem -{ +public class FakeBackupFileSystem extends AbstractFileSystem { private List flist; public Set downloadedFiles; public Set uploadedFiles; @@ -42,8 +43,13 @@ public class FakeBackupFileSystem implements IBackupFileSystem @Inject Provider pathProvider; - public void setupTest(List files) - { + @Inject + public FakeBackupFileSystem(BackupMetrics backupMetrics, + BackupNotificationMgr backupNotificationMgr){ + super(backupMetrics, backupNotificationMgr); + } + + public void setupTest(List files) { clearTest(); flist = new ArrayList(); for (String file : files) @@ -81,47 +87,7 @@ public void addFile(String file) @SuppressWarnings("unchecked") @Override - public void download(AbstractBackupPath path, OutputStream os) throws BackupRestoreException - { - try - { - if (path.type == BackupFileType.META) - { - // List all files and generate the file - FileWriter fr = new FileWriter(path.newRestoreFile()); - try - { - JSONArray jsonObj = new JSONArray(); - for (AbstractBackupPath filePath : flist) - { - if (filePath.type == BackupFileType.SNAP) - jsonObj.add(filePath.getRemotePath()); - } - fr.write(jsonObj.toJSONString()); - } - finally - { - IOUtils.closeQuietly(fr); - } - } - downloadedFiles.add(path.getRemotePath()); - System.out.println("Downloading " + path.getRemotePath()); - } - catch (IOException io) - { - throw new BackupRestoreException(io.getMessage(), io); - } - } - - @Override - public void upload(AbstractBackupPath path, InputStream in) throws BackupRestoreException - { - uploadedFiles.add(path.backupFile.getAbsolutePath()); - } - - @Override - public Iterator list(String bucket, Date start, Date till) - { + public Iterator list(String bucket, Date start, Date till) { String[] paths = bucket.split(String.valueOf(S3BackupPath.PATH_SEP)); if( paths.length > 1){ @@ -143,25 +109,13 @@ public Iterator list(String bucket, Date start, Date till) return tmpList.iterator(); } - @Override - public int getActivecount() - { - // TODO Auto-generated method stub - return 0; - } - public void shutdown() - { + public void shutdown() { //nop } @Override - public long getBytesUploaded() { - return 0; - } - - @Override - public long getAWSSlowDownExceptionCounter() { + public long getFileSize(Path remotePath) throws BackupRestoreException { return 0; } @@ -176,13 +130,32 @@ public Iterator listPrefixes(Date date) public void cleanup() { // TODO Auto-generated method stub - } - @Override - public void download(AbstractBackupPath path, OutputStream os, - String diskPath) throws BackupRestoreException { - download(path, os); - } + @Override + protected void downloadFileImpl(Path remotePath, Path localPath) throws BackupRestoreException { + //if (path.type == BackupFileType.META) + { + // List all files and generate the file + try (FileWriter fr = new FileWriter(localPath.toFile())) { + JSONArray jsonObj = new JSONArray(); + for (AbstractBackupPath filePath : flist) { + if (filePath.type == BackupFileType.SNAP) + jsonObj.add(filePath.getRemotePath()); + } + fr.write(jsonObj.toJSONString()); + fr.flush(); + } catch (IOException io) { + throw new BackupRestoreException(io.getMessage(), io); + } + } + downloadedFiles.add(remotePath.toString()); + System.out.println("Downloading " + remotePath.toString()); + } + @Override + protected long uploadFileImpl(Path localPath, Path remotePath) throws BackupRestoreException { + uploadedFiles.add(localPath.toFile().getAbsolutePath()); + return localPath.toFile().length(); + } } diff --git a/priam/src/test/java/com/netflix/priam/backup/FakedS3EncryptedFileSystem.java b/priam/src/test/java/com/netflix/priam/backup/FakedS3EncryptedFileSystem.java deleted file mode 100755 index 50ea79fb1..000000000 --- a/priam/src/test/java/com/netflix/priam/backup/FakedS3EncryptedFileSystem.java +++ /dev/null @@ -1,105 +0,0 @@ -/* - * Copyright 2017 Netflix, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.netflix.priam.backup; - -import java.io.InputStream; -import java.io.OutputStream; -import java.util.Date; -import java.util.Iterator; - -import com.google.inject.Inject; -import com.google.inject.Provider; -import com.google.inject.Singleton; -import com.google.inject.name.Named; -import com.netflix.priam.config.IConfiguration; -import com.netflix.priam.cred.ICredential; -import com.netflix.priam.compress.ICompression; -import com.netflix.priam.cryptography.IFileCryptography; - -@Singleton -public class FakedS3EncryptedFileSystem implements IBackupFileSystem { - - @Inject - public FakedS3EncryptedFileSystem( Provider pathProvider, ICompression compress, final IConfiguration config, ICredential cred - , @Named("filecryptoalgorithm") IFileCryptography fileCryptography - ) { - - } - - @Override - public void download(AbstractBackupPath path, OutputStream os) - throws BackupRestoreException { - // TODO Auto-generated method stub - - } - - @Override - public void download(AbstractBackupPath path, OutputStream os, - String filePath) throws BackupRestoreException { - // TODO Auto-generated method stub - - } - - @Override - public void upload(AbstractBackupPath path, InputStream in) - throws BackupRestoreException { - // TODO Auto-generated method stub - - } - - @Override - public Iterator list(String path, Date start, Date till) { - // TODO Auto-generated method stub - return null; - } - - @Override - public Iterator listPrefixes(Date date) { - // TODO Auto-generated method stub - return null; - } - - @Override - public void cleanup() { - // TODO Auto-generated method stub - - } - - @Override - public int getActivecount() { - // TODO Auto-generated method stub - return 0; - } - - @Override - public void shutdown() { - // TODO Auto-generated method stub - - } - - @Override - public long getBytesUploaded() { - return 0; - } - - @Override - public long getAWSSlowDownExceptionCounter() { - return 0; - } - -} diff --git a/priam/src/test/java/com/netflix/priam/backup/NullBackupFileSystem.java b/priam/src/test/java/com/netflix/priam/backup/NullBackupFileSystem.java index a177b24b2..31f69b385 100644 --- a/priam/src/test/java/com/netflix/priam/backup/NullBackupFileSystem.java +++ b/priam/src/test/java/com/netflix/priam/backup/NullBackupFileSystem.java @@ -17,28 +17,26 @@ package com.netflix.priam.backup; -import java.io.InputStream; -import java.io.OutputStream; +import com.google.inject.Inject; +import com.netflix.priam.merics.BackupMetrics; +import com.netflix.priam.notification.BackupNotificationMgr; + +import java.nio.file.Path; import java.util.Date; import java.util.Iterator; -import com.netflix.priam.backup.AbstractBackupPath; -import com.netflix.priam.backup.BackupRestoreException; -import com.netflix.priam.backup.IBackupFileSystem; -public class NullBackupFileSystem implements IBackupFileSystem -{ +public class NullBackupFileSystem extends AbstractFileSystem { - @Override - public Iterator list(String bucket, Date start, Date till) - { - return null; + @Inject + public NullBackupFileSystem(BackupMetrics backupMetrics, + BackupNotificationMgr backupNotificationMgr){ + super(backupMetrics, backupNotificationMgr); } @Override - public int getActivecount() - { - return 0; + public Iterator list(String bucket, Date start, Date till) { + return null; } public void shutdown() @@ -47,42 +45,29 @@ public void shutdown() } @Override - public long getBytesUploaded() { + public long getFileSize(Path remotePath) throws BackupRestoreException { return 0; } @Override - public long getAWSSlowDownExceptionCounter() { - return 0; - } - @Override - public void download(AbstractBackupPath path, OutputStream os) throws BackupRestoreException - { + public Iterator listPrefixes(Date date) { + return null; } @Override - public void upload(AbstractBackupPath path, InputStream in) throws BackupRestoreException + public void cleanup() { + // TODO Auto-generated method stub } @Override - public Iterator listPrefixes(Date date) - { - return null; + protected void downloadFileImpl(Path remotePath, Path localPath) throws BackupRestoreException { + } @Override - public void cleanup() - { - // TODO Auto-generated method stub - + protected long uploadFileImpl(Path localPath, Path remotePath) throws BackupRestoreException { + return 0; } - - @Override - public void download(AbstractBackupPath path, OutputStream os, - String filePath) throws BackupRestoreException { - // TODO Auto-generated method stub - - } } \ No newline at end of file diff --git a/priam/src/test/java/com/netflix/priam/backup/TestS3FileSystem.java b/priam/src/test/java/com/netflix/priam/backup/TestS3FileSystem.java index 28d5d861a..830c22ab3 100644 --- a/priam/src/test/java/com/netflix/priam/backup/TestS3FileSystem.java +++ b/priam/src/test/java/com/netflix/priam/backup/TestS3FileSystem.java @@ -17,50 +17,56 @@ package com.netflix.priam.backup; -import java.io.BufferedOutputStream; -import java.io.File; -import java.io.FileOutputStream; -import java.io.IOException; -import java.util.List; - -import com.amazonaws.services.s3.model.*; -import org.junit.Assert; -import mockit.Mock; -import mockit.MockUp; - -import org.junit.AfterClass; -import org.junit.BeforeClass; -import org.junit.Ignore; -import org.junit.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import com.amazonaws.AmazonClientException; +import com.amazonaws.AmazonServiceException; +import com.amazonaws.SdkClientException; import com.amazonaws.services.s3.AmazonS3; import com.amazonaws.services.s3.AmazonS3Client; +import com.amazonaws.services.s3.model.*; import com.google.common.collect.Lists; import com.google.inject.Guice; import com.google.inject.Injector; -import com.netflix.priam.config.FakeConfiguration; import com.netflix.priam.aws.DataPart; import com.netflix.priam.aws.S3BackupPath; import com.netflix.priam.aws.S3FileSystem; import com.netflix.priam.aws.S3PartUploader; import com.netflix.priam.backup.AbstractBackupPath.BackupFileType; +import com.netflix.priam.config.FakeConfiguration; +import com.netflix.priam.merics.BackupMetrics; +import mockit.Mock; +import mockit.MockUp; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.BufferedOutputStream; +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.nio.file.Paths; +import java.util.List; -public class TestS3FileSystem -{ +public class TestS3FileSystem { private static Injector injector; private static final Logger logger = LoggerFactory.getLogger(TestS3FileSystem.class); - private static final String FILE_PATH = "target/data/Keyspace1/Standard1/backups/201108082320/Keyspace1-Standard1-ia-1-Data.db"; + private static String FILE_PATH = "target/data/Keyspace1/Standard1/backups/201108082320/Keyspace1-Standard1-ia-1-Data.db"; + private static BackupMetrics backupMetrics; - @BeforeClass - public static void setup() throws InterruptedException, IOException - { - new MockS3PartUploader(); - new MockAmazonS3Client(); + public TestS3FileSystem() { + if (injector == null) + injector = Guice.createInjector(new BRTestModule()); + + if (backupMetrics == null) + backupMetrics = injector.getInstance(BackupMetrics.class); + } - injector = Guice.createInjector(new BRTestModule()); + @BeforeClass + public static void setup() throws InterruptedException, IOException { + new MockS3PartUploader(); + new MockAmazonS3Client(); File dir1 = new File("target/data/Keyspace1/Standard1/backups/201108082320"); if (!dir1.exists()) @@ -69,68 +75,59 @@ public static void setup() throws InterruptedException, IOException long fiveKB = (5L * 1024); byte b = 8; BufferedOutputStream bos1 = new BufferedOutputStream(new FileOutputStream(file)); - for (long i = 0; i < fiveKB; i++) - { + for (long i = 0; i < fiveKB; i++) { bos1.write(b); } bos1.close(); } @AfterClass - public static void cleanup() - { + public static void cleanup() { File file = new File(FILE_PATH); file.delete(); } @Test - public void testFileUpload() throws Exception - { + public void testFileUpload() throws Exception { MockS3PartUploader.setup(); - S3FileSystem fs = injector.getInstance(S3FileSystem.class); - // String snapshotfile = "target/data/Keyspace1/Standard1/backups/201108082320/Keyspace1-Standard1-ia-1-Data.db"; + IBackupFileSystem fs = injector.getInstance(NullBackupFileSystem.class); S3BackupPath backupfile = injector.getInstance(S3BackupPath.class); backupfile.parseLocal(new File(FILE_PATH), BackupFileType.SNAP); - //fs.upload(backupfile, backupfile.localReader()); - //Assert.assertEquals(1, MockS3PartUploader.compattempts); + long noOfFilesUploaded = backupMetrics.getUploadRate().count(); + fs.uploadFile(Paths.get(backupfile.getBackupFile().getAbsolutePath()), Paths.get(backupfile.getRemotePath()), backupfile); + Assert.assertEquals(1, backupMetrics.getUploadRate().count() - noOfFilesUploaded); } @Test - public void testFileUploadFailures() throws Exception - { + public void testFileUploadFailures() throws Exception { MockS3PartUploader.setup(); MockS3PartUploader.partFailure = true; + long noOfFailures = backupMetrics.getInvalidUploads().count(); S3FileSystem fs = injector.getInstance(S3FileSystem.class); String snapshotfile = "target/data/Keyspace1/Standard1/backups/201108082320/Keyspace1-Standard1-ia-1-Data.db"; S3BackupPath backupfile = injector.getInstance(S3BackupPath.class); backupfile.parseLocal(new File(snapshotfile), BackupFileType.SNAP); - try - { - fs.upload(backupfile, backupfile.localReader()); - } - catch (BackupRestoreException e) - { + try { + fs.uploadFile(Paths.get(backupfile.getBackupFile().getAbsolutePath()), Paths.get(backupfile.getRemotePath()), backupfile); + } catch (BackupRestoreException e) { // ignore } //Assert.assertEquals(RetryableCallable.DEFAULT_NUMBER_OF_RETRIES, MockS3PartUploader.partAttempts); Assert.assertEquals(0, MockS3PartUploader.compattempts); + Assert.assertEquals(1, backupMetrics.getInvalidUploads().count() - noOfFailures); } @Test - public void testFileUploadCompleteFailure() throws Exception - { + public void testFileUploadCompleteFailure() throws Exception { MockS3PartUploader.setup(); MockS3PartUploader.completionFailure = true; S3FileSystem fs = injector.getInstance(S3FileSystem.class); String snapshotfile = "target/data/Keyspace1/Standard1/backups/201108082320/Keyspace1-Standard1-ia-1-Data.db"; S3BackupPath backupfile = injector.getInstance(S3BackupPath.class); backupfile.parseLocal(new File(snapshotfile), BackupFileType.SNAP); - try - { - fs.upload(backupfile, backupfile.localReader()); - } - catch (BackupRestoreException e) - { + try { + fs.uploadFile(Paths.get(backupfile.getBackupFile().getAbsolutePath()), Paths.get(backupfile.getRemotePath()), backupfile); + } catch (BackupRestoreException e) { // ignore } //Assert.assertEquals(1, MockS3PartUploader.partAttempts); @@ -139,51 +136,45 @@ public void testFileUploadCompleteFailure() throws Exception } @Test - public void testCleanupAdd() throws Exception - { + public void testCleanupAdd() throws Exception { MockAmazonS3Client.ruleAvailable = false; S3FileSystem fs = injector.getInstance(S3FileSystem.class); fs.cleanup(); Assert.assertEquals(1, MockAmazonS3Client.bconf.getRules().size()); BucketLifecycleConfiguration.Rule rule = MockAmazonS3Client.bconf.getRules().get(0); logger.info(rule.getPrefix()); - Assert.assertEquals("casstestbackup/"+FakeConfiguration.FAKE_REGION+"/fake-app/", rule.getPrefix()); + Assert.assertEquals("casstestbackup/" + FakeConfiguration.FAKE_REGION + "/fake-app/", rule.getPrefix()); Assert.assertEquals(5, rule.getExpirationInDays()); } @Test - public void testCleanupIgnore() throws Exception - { + public void testCleanupIgnore() throws Exception { MockAmazonS3Client.ruleAvailable = true; S3FileSystem fs = injector.getInstance(S3FileSystem.class); fs.cleanup(); Assert.assertEquals(1, MockAmazonS3Client.bconf.getRules().size()); BucketLifecycleConfiguration.Rule rule = MockAmazonS3Client.bconf.getRules().get(0); logger.info(rule.getPrefix()); - Assert.assertEquals("casstestbackup/"+FakeConfiguration.FAKE_REGION+"/fake-app/", rule.getPrefix()); + Assert.assertEquals("casstestbackup/" + FakeConfiguration.FAKE_REGION + "/fake-app/", rule.getPrefix()); Assert.assertEquals(5, rule.getExpirationInDays()); } - + // Mock Nodeprobe class - @Ignore - public static class MockS3PartUploader extends MockUp - { - public static int compattempts = 0; - public static int partAttempts = 0; - public static boolean partFailure = false; - public static boolean completionFailure = false; + static class MockS3PartUploader extends MockUp { + static int compattempts = 0; + static int partAttempts = 0; + static boolean partFailure = false; + static boolean completionFailure = false; private static List partETags; @Mock - public void $init(AmazonS3 client, DataPart dp, List partETags) - { + public void $init(AmazonS3 client, DataPart dp, List partETags) { MockS3PartUploader.partETags = partETags; } @Mock - private Void uploadPart() throws AmazonClientException, BackupRestoreException - { + private Void uploadPart() throws AmazonClientException, BackupRestoreException { ++partAttempts; if (partFailure) throw new BackupRestoreException("Test exception"); @@ -192,8 +183,7 @@ private Void uploadPart() throws AmazonClientException, BackupRestoreException } @Mock - public CompleteMultipartUploadResult completeUpload() throws BackupRestoreException - { + public CompleteMultipartUploadResult completeUpload() throws BackupRestoreException { ++compattempts; if (completionFailure) throw new BackupRestoreException("Test exception"); @@ -202,19 +192,16 @@ public CompleteMultipartUploadResult completeUpload() throws BackupRestoreExcept } @Mock - public void abortUpload() - { + public void abortUpload() { } @Mock - public Void retriableCall() throws AmazonClientException, BackupRestoreException - { + public Void retriableCall() throws AmazonClientException, BackupRestoreException { logger.info("MOCK UPLOADING..."); return uploadPart(); } - public static void setup() - { + public static void setup() { compattempts = 0; partAttempts = 0; partFailure = false; @@ -222,41 +209,42 @@ public static void setup() } } - @Ignore - public static class MockAmazonS3Client extends MockUp - { - public static boolean ruleAvailable = false; - public static BucketLifecycleConfiguration bconf = new BucketLifecycleConfiguration(); + static class MockAmazonS3Client extends MockUp { + static boolean ruleAvailable = false; + static BucketLifecycleConfiguration bconf = new BucketLifecycleConfiguration(); + @Mock - public void $init() - { + public void $init() { } @Mock public InitiateMultipartUploadResult initiateMultipartUpload(InitiateMultipartUploadRequest initiateMultipartUploadRequest) throws AmazonClientException { return new InitiateMultipartUploadResult(); } - + + public PutObjectResult putObject(PutObjectRequest putObjectRequest) throws SdkClientException, AmazonServiceException { + PutObjectResult result = new PutObjectResult(); + result.setETag("ad"); + return result; + } + @Mock - public BucketLifecycleConfiguration getBucketLifecycleConfiguration(String bucketName) - { + public BucketLifecycleConfiguration getBucketLifecycleConfiguration(String bucketName) { List rules = Lists.newArrayList(); - if( ruleAvailable ) - { - String clusterPath = "casstestbackup/"+FakeConfiguration.FAKE_REGION+"/fake-app/"; + if (ruleAvailable) { + String clusterPath = "casstestbackup/" + FakeConfiguration.FAKE_REGION + "/fake-app/"; BucketLifecycleConfiguration.Rule rule = new BucketLifecycleConfiguration.Rule().withExpirationInDays(5).withPrefix(clusterPath); rule.setStatus(BucketLifecycleConfiguration.ENABLED); rule.setId(clusterPath); rules.add(rule); - + } bconf.setRules(rules); return bconf; } - + @Mock - public void setBucketLifecycleConfiguration(String bucketName, BucketLifecycleConfiguration bucketLifecycleConfiguration) - { + public void setBucketLifecycleConfiguration(String bucketName, BucketLifecycleConfiguration bucketLifecycleConfiguration) { bconf = bucketLifecycleConfiguration; } From f881fb38e6e62a092cdc4626f805d8a29d06bdb8 Mon Sep 17 00:00:00 2001 From: Arun Agrawal Date: Thu, 13 Sep 2018 17:37:16 -0700 Subject: [PATCH 012/228] Remove unnecessary annotations for IBackupFileSystem. --- .../priam/backup/BackupVerification.java | 2 +- .../priam/defaultimpl/PriamGuiceModule.java | 2 -- .../google/GoogleEncryptedFileSystem.java | 1 - .../priam/resources/BackupServlet.java | 13 +++++------- .../priam/restore/EncryptedRestoreBase.java | 5 ----- .../com/netflix/priam/restore/Restore.java | 2 -- .../netflix/priam/backup/BRTestModule.java | 1 - .../com/netflix/priam/backup/TestRestore.java | 2 +- .../priam/resources/BackupServletTest.java | 21 ++++++++++++------- 9 files changed, 20 insertions(+), 29 deletions(-) diff --git a/priam/src/main/java/com/netflix/priam/backup/BackupVerification.java b/priam/src/main/java/com/netflix/priam/backup/BackupVerification.java index f89a4cb69..8f002bb00 100644 --- a/priam/src/main/java/com/netflix/priam/backup/BackupVerification.java +++ b/priam/src/main/java/com/netflix/priam/backup/BackupVerification.java @@ -48,7 +48,7 @@ public class BackupVerification { private IConfiguration config; @Inject - BackupVerification(@Named("backup_status") IBackupFileSystem bkpStatusFs, IConfiguration config) { + BackupVerification(@Named("backup") IBackupFileSystem bkpStatusFs, IConfiguration config) { this.bkpStatusFs = bkpStatusFs; this.config = config; } diff --git a/priam/src/main/java/com/netflix/priam/defaultimpl/PriamGuiceModule.java b/priam/src/main/java/com/netflix/priam/defaultimpl/PriamGuiceModule.java index a6fc032f0..046815482 100644 --- a/priam/src/main/java/com/netflix/priam/defaultimpl/PriamGuiceModule.java +++ b/priam/src/main/java/com/netflix/priam/defaultimpl/PriamGuiceModule.java @@ -47,8 +47,6 @@ protected void configure() { bind(IBackupFileSystem.class).annotatedWith(Names.named("backup")).to(S3FileSystem.class); bind(IBackupFileSystem.class).annotatedWith(Names.named("encryptedbackup")).to(S3EncryptedFileSystem.class); - bind(IBackupFileSystem.class).annotatedWith(Names.named("incr_restore")).to(S3FileSystem.class); - bind(IBackupFileSystem.class).annotatedWith(Names.named("backup_status")).to(S3FileSystem.class); bind(S3CrossAccountFileSystem.class); diff --git a/priam/src/main/java/com/netflix/priam/google/GoogleEncryptedFileSystem.java b/priam/src/main/java/com/netflix/priam/google/GoogleEncryptedFileSystem.java index 2e50a1298..4a0bfb5d4 100755 --- a/priam/src/main/java/com/netflix/priam/google/GoogleEncryptedFileSystem.java +++ b/priam/src/main/java/com/netflix/priam/google/GoogleEncryptedFileSystem.java @@ -173,7 +173,6 @@ private Credential constructGcsCredential() throws Exception { @Override protected void downloadFileImpl(Path remotePath, Path localPath) throws BackupRestoreException{ - logger.info("Downloading {} from GCS bucket {}", remotePath, this.srcBucketName); String objectName = parseObjectname(getPathPrefix()); com.google.api.services.storage.Storage.Objects.Get get = null; diff --git a/priam/src/main/java/com/netflix/priam/resources/BackupServlet.java b/priam/src/main/java/com/netflix/priam/resources/BackupServlet.java index 14d71db3e..9af452fab 100644 --- a/priam/src/main/java/com/netflix/priam/resources/BackupServlet.java +++ b/priam/src/main/java/com/netflix/priam/resources/BackupServlet.java @@ -76,7 +76,6 @@ public class BackupServlet { private PriamServer priamServer; private IConfiguration config; private IBackupFileSystem backupFs; - private IBackupFileSystem bkpStatusFs; private Restore restoreObj; private Provider pathProvider; private ICassandraTuner tuner; @@ -93,14 +92,12 @@ public class BackupServlet { private IBackupStatusMgr completedBkups; @Inject - public BackupServlet(PriamServer priamServer, IConfiguration config, @Named("backup")IBackupFileSystem backupFs,@Named("backup_status")IBackupFileSystem bkpStatusFs, Restore restoreObj, Provider pathProvider, ICassandraTuner tuner, - SnapshotBackup snapshotBackup, IPriamInstanceFactory factory, ITokenManager tokenManager, ICassandraProcess cassProcess - ,IBackupStatusMgr completedBkups, BackupVerification backupVerification) - { + public BackupServlet(PriamServer priamServer, IConfiguration config, @Named("backup") IBackupFileSystem backupFs, Restore restoreObj, Provider pathProvider, ICassandraTuner tuner, + SnapshotBackup snapshotBackup, IPriamInstanceFactory factory, ITokenManager tokenManager, ICassandraProcess cassProcess + , IBackupStatusMgr completedBkups, BackupVerification backupVerification) { this.priamServer = priamServer; this.config = config; this.backupFs = backupFs; - this.bkpStatusFs = bkpStatusFs; this.restoreObj = restoreObj; this.pathProvider = pathProvider; this.tuner = tuner; @@ -133,7 +130,7 @@ public Response backupIncrementals() throws Exception { * Fetch the list of files for the requested date range. * * @param date range - * @param filter. The type of data files fetched. E.g. META will only fetch the dailsy snapshot meta data file (meta.json). + * @param filter. The type of data files fetched. E.g. META will only fetch the daily snapshot meta data file (meta.json). * @return the list of files in json format as part of the Http response body. */ public Response list(@QueryParam(REST_HEADER_RANGE) String daterange, @QueryParam(REST_HEADER_FILTER) @DefaultValue("") String filter) throws Exception { @@ -153,7 +150,7 @@ public Response list(@QueryParam(REST_HEADER_RANGE) String daterange, @QueryPara logger.info("Parameters: {backupPrefix: [{}], daterange: [{}], filter: [{}]}", config.getBackupPrefix(), daterange, filter); - Iterator it = bkpStatusFs.list(config.getBackupPrefix(), startTime, endTime); + Iterator it = backupFs.list(config.getBackupPrefix(), startTime, endTime); JSONObject object = new JSONObject(); object = constructJsonResponse(object, it, filter); return Response.ok(object.toString(2), MediaType.APPLICATION_JSON).build(); diff --git a/priam/src/main/java/com/netflix/priam/restore/EncryptedRestoreBase.java b/priam/src/main/java/com/netflix/priam/restore/EncryptedRestoreBase.java index 18d3bb3f9..633cee63a 100755 --- a/priam/src/main/java/com/netflix/priam/restore/EncryptedRestoreBase.java +++ b/priam/src/main/java/com/netflix/priam/restore/EncryptedRestoreBase.java @@ -79,13 +79,8 @@ public Integer retriableCall() throws Exception { //== download object from source bucket try { - - logger.info("Downloading file from: {} to: {}", path.getRemotePath(), tempFile.getAbsolutePath()); fs.downloadFile(Paths.get(path.getRemotePath()), Paths.get(tempFile.getAbsolutePath())); tracker.adjustAndAdd(path); - logger.info("Completed downloading file from: {} to: {}", path.getRemotePath(), tempFile.getAbsolutePath()); - - } catch (Exception ex) { //This behavior is retryable; therefore, lets get to a clean state before each retry. if (tempFile.exists()) { diff --git a/priam/src/main/java/com/netflix/priam/restore/Restore.java b/priam/src/main/java/com/netflix/priam/restore/Restore.java index 640c3d4ed..dd8e21378 100644 --- a/priam/src/main/java/com/netflix/priam/restore/Restore.java +++ b/priam/src/main/java/com/netflix/priam/restore/Restore.java @@ -66,11 +66,9 @@ protected final void downloadFile(final AbstractBackupPath path, final File rest executor.submit(new RetryableCallable() { @Override public Integer retriableCall() throws Exception { - logger.info("Downloading file: {} to: {}", path.getRemotePath(), restoreLocation.getAbsolutePath()); fs.downloadFile(Paths.get(path.getRemotePath()), Paths.get(restoreLocation.getAbsolutePath())); tracker.adjustAndAdd(path); // TODO: fix me -> if there is exception the why hang? - logger.info("Completed download of file: {} to: {}", path.getRemotePath(), restoreLocation.getAbsolutePath()); return count.decrementAndGet(); } }); diff --git a/priam/src/test/java/com/netflix/priam/backup/BRTestModule.java b/priam/src/test/java/com/netflix/priam/backup/BRTestModule.java index 3afe0325d..e48e36e12 100644 --- a/priam/src/test/java/com/netflix/priam/backup/BRTestModule.java +++ b/priam/src/test/java/com/netflix/priam/backup/BRTestModule.java @@ -57,7 +57,6 @@ protected void configure() bind(IMembership.class).toInstance(new FakeMembership(Arrays.asList("fakeInstance1"))); bind(ICredential.class).to(FakeNullCredential.class).in(Scopes.SINGLETON); bind(IBackupFileSystem.class).annotatedWith(Names.named("backup")).to(FakeBackupFileSystem.class).in(Scopes.SINGLETON); - bind(IBackupFileSystem.class).annotatedWith(Names.named("incr_restore")).to(FakeBackupFileSystem.class).in(Scopes.SINGLETON); bind(Sleeper.class).to(FakeSleeper.class); bind(IS3Credential.class).annotatedWith(Names.named("awss3roleassumption")).to(S3RoleAssumptionCredential.class); diff --git a/priam/src/test/java/com/netflix/priam/backup/TestRestore.java b/priam/src/test/java/com/netflix/priam/backup/TestRestore.java index 5fa11351f..4caf5a17f 100644 --- a/priam/src/test/java/com/netflix/priam/backup/TestRestore.java +++ b/priam/src/test/java/com/netflix/priam/backup/TestRestore.java @@ -48,7 +48,7 @@ public class TestRestore public static void setup() throws InterruptedException, IOException { injector = Guice.createInjector(new BRTestModule()); - filesystem = (FakeBackupFileSystem) injector.getInstance(Key.get(IBackupFileSystem.class,Names.named("incr_restore"))); + filesystem = (FakeBackupFileSystem) injector.getInstance(Key.get(IBackupFileSystem.class, Names.named("backup"))); conf = injector.getInstance(IConfiguration.class); fileList = new ArrayList(); File cassdir = new File(conf.getDataFileLocation()); diff --git a/priam/src/test/java/com/netflix/priam/resources/BackupServletTest.java b/priam/src/test/java/com/netflix/priam/resources/BackupServletTest.java index 8fb87d5e8..70158849a 100644 --- a/priam/src/test/java/com/netflix/priam/resources/BackupServletTest.java +++ b/priam/src/test/java/com/netflix/priam/resources/BackupServletTest.java @@ -54,11 +54,17 @@ public class BackupServletTest { private @Mocked PriamServer priamServer; private IConfiguration config; - private @Mocked IBackupFileSystem bkpFs; - private @Mocked IBackupFileSystem bkpStatusFs; - private @Mocked Restore restoreObj; - private @Mocked Provider pathProvider; - private @Mocked + private + @Mocked + IBackupFileSystem bkpFs; + private + @Mocked + Restore restoreObj; + private + @Mocked + Provider pathProvider; + private + @Mocked ICassandraTuner tuner; private @Mocked SnapshotBackup snapshotBackup; private @Mocked IPriamInstanceFactory factory; @@ -75,9 +81,8 @@ public void setUp() config = injector.getInstance(IConfiguration.class); InstanceState instanceState = injector.getInstance(InstanceState.class); ITokenManager tokenManager = new TokenManager(config); - resource = new BackupServlet(priamServer, config, bkpFs, bkpStatusFs, restoreObj, pathProvider, - tuner, snapshotBackup, factory, tokenManager, cassProcess, bkupStatusMgr,backupVerification); - + resource = new BackupServlet(priamServer, config, bkpFs, restoreObj, pathProvider, + tuner, snapshotBackup, factory, tokenManager, cassProcess, bkupStatusMgr, backupVerification); restoreResource = new RestoreServlet(config, restoreObj, pathProvider,priamServer, factory, tuner, cassProcess , tokenManager, instanceState); } From cea9d75f5e233274aaa24f0e5fad9f7f04116ce4 Mon Sep 17 00:00:00 2001 From: Arun Agrawal Date: Fri, 21 Sep 2018 11:15:35 -0700 Subject: [PATCH 013/228] Backup 2.0 Milestone 2 Implementation Move the retry logic and file deletion logic while doing upload to AbstractFileSystem. Also removes the old Incremental implementation and adds backupv2 Meta file changes --- .../priam/aws/S3EncryptedFileSystem.java | 3 +- .../com/netflix/priam/aws/S3FileSystem.java | 13 +- .../netflix/priam/aws/S3FileSystemBase.java | 17 +- .../netflix/priam/backup/AbstractBackup.java | 44 +-- .../priam/backup/AbstractBackupPath.java | 1 - .../priam/backup/AbstractFileSystem.java | 133 ++++++-- .../priam/backup/BackupVerification.java | 2 +- .../netflix/priam/backup/CommitLogBackup.java | 35 +- .../priam/backup/IBackupFileSystem.java | 61 +++- .../com/netflix/priam/backup/MetaData.java | 39 +-- .../backup/parallel/IncrementalConsumer.java | 24 +- .../priam/backupv2/MetaFileWriterBuilder.java | 11 +- .../google/GoogleEncryptedFileSystem.java | 2 +- .../netflix/priam/merics/BackupMetrics.java | 31 +- .../priam/restore/EncryptedRestoreBase.java | 3 +- .../com/netflix/priam/restore/Restore.java | 12 +- .../priam/backup/FakeBackupFileSystem.java | 5 +- .../priam/backup/NullBackupFileSystem.java | 5 +- .../priam/backup/TestAbstractFileSystem.java | 318 ++++++++++++++++++ .../priam/backup/TestS3FileSystem.java | 6 +- .../services/TestSnapshotMetaService.java | 50 +-- .../netflix/priam/utils/BackupFileUtils.java | 72 ++++ 22 files changed, 631 insertions(+), 256 deletions(-) create mode 100644 priam/src/test/java/com/netflix/priam/backup/TestAbstractFileSystem.java create mode 100644 priam/src/test/java/com/netflix/priam/utils/BackupFileUtils.java diff --git a/priam/src/main/java/com/netflix/priam/aws/S3EncryptedFileSystem.java b/priam/src/main/java/com/netflix/priam/aws/S3EncryptedFileSystem.java index 22db02abb..354e502bb 100755 --- a/priam/src/main/java/com/netflix/priam/aws/S3EncryptedFileSystem.java +++ b/priam/src/main/java/com/netflix/priam/aws/S3EncryptedFileSystem.java @@ -94,7 +94,8 @@ protected long uploadFileImpl(Path localPath, Path remotePath) throws BackupRest //== Read chunks from src, compress it, and write to temp file File compressedDstFile = new File(localPath.toString() + ".compressed"); - logger.debug("Compressing {} with chunk size {}", compressedDstFile.getAbsolutePath(), chunkSize); + if (logger.isDebugEnabled()) + logger.debug("Compressing {} with chunk size {}", compressedDstFile.getAbsolutePath(), chunkSize); try (InputStream in = new FileInputStream(localPath.toFile()); BufferedOutputStream compressedBos = new BufferedOutputStream(new FileOutputStream(compressedDstFile))) { diff --git a/priam/src/main/java/com/netflix/priam/aws/S3FileSystem.java b/priam/src/main/java/com/netflix/priam/aws/S3FileSystem.java index 09697a97d..e1d41c7a4 100644 --- a/priam/src/main/java/com/netflix/priam/aws/S3FileSystem.java +++ b/priam/src/main/java/com/netflix/priam/aws/S3FileSystem.java @@ -45,6 +45,7 @@ import java.util.Collections; import java.util.Iterator; import java.util.List; +import java.util.concurrent.Future; import java.util.concurrent.atomic.AtomicInteger; /** @@ -94,7 +95,8 @@ private ObjectMetadata getObjectMetadata(Path path) { private long uploadMultipart(Path localPath, Path remotePath) throws BackupRestoreException { long chunkSize = getChunkSize(localPath); - logger.info("Uploading to {}/{} with chunk size {}", config.getBackupPrefix(), remotePath, chunkSize); + if (logger.isDebugEnabled()) + logger.debug("Uploading to {}/{} with chunk size {}", config.getBackupPrefix(), remotePath, chunkSize); InitiateMultipartUploadRequest initRequest = new InitiateMultipartUploadRequest(config.getBackupPrefix(), remotePath.toString()); initRequest.withObjectMetadata(getObjectMetadata(localPath)); InitiateMultipartUploadResult initResponse = s3Client.initiateMultipartUpload(initRequest); @@ -114,8 +116,11 @@ private long uploadMultipart(Path localPath, Path remotePath) throws BackupResto DataPart dp = new DataPart(++partNum, chunk, config.getBackupPrefix(), remotePath.toString(), initResponse.getUploadId()); S3PartUploader partUploader = new S3PartUploader(s3Client, dp, partETags, partsUploaded); compressedFileSize += chunk.length; - executor.submit(partUploader); + //TODO: Get the future over here and create a new arraylist. + Future future = executor.submit(partUploader); } + + //TODO: Instead of waiting for executor thread to be empty we should wait for all the futures to finish. executor.sleepTillEmpty(); logger.info("All chunks uploaded for file {}, num of expected parts:{}, num of actual uploaded parts: {}", localPath.toFile().getName(), partNum, partsUploaded.get()); @@ -145,7 +150,9 @@ protected long uploadFileImpl(Path localPath, Path remotePath) throws BackupRest if (fileSize < chunkSize) { //Upload file without using multipart upload as it will be more efficient. - logger.info("Uploading to {}/{} using PUT operation", config.getBackupPrefix(), remotePath); + + if (logger.isDebugEnabled()) + logger.debug("Uploading to {}/{} using PUT operation", config.getBackupPrefix(), remotePath); try (ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); InputStream in = new BufferedInputStream(new FileInputStream(localPath.toFile()))) { diff --git a/priam/src/main/java/com/netflix/priam/aws/S3FileSystemBase.java b/priam/src/main/java/com/netflix/priam/aws/S3FileSystemBase.java index d9dda3c04..2a1936ce6 100755 --- a/priam/src/main/java/com/netflix/priam/aws/S3FileSystemBase.java +++ b/priam/src/main/java/com/netflix/priam/aws/S3FileSystemBase.java @@ -16,7 +16,6 @@ package com.netflix.priam.aws; import com.amazonaws.services.s3.AmazonS3; -import com.amazonaws.services.s3.model.AmazonS3Exception; import com.amazonaws.services.s3.model.BucketLifecycleConfiguration; import com.amazonaws.services.s3.model.BucketLifecycleConfiguration.Rule; import com.amazonaws.services.s3.model.CompleteMultipartUploadResult; @@ -26,30 +25,20 @@ import com.netflix.priam.backup.AbstractBackupPath; import com.netflix.priam.backup.AbstractFileSystem; import com.netflix.priam.backup.BackupRestoreException; -import com.netflix.priam.backup.IBackupFileSystem; import com.netflix.priam.compress.ICompression; import com.netflix.priam.config.IConfiguration; import com.netflix.priam.merics.BackupMetrics; -import com.netflix.priam.notification.BackupEvent; import com.netflix.priam.notification.BackupNotificationMgr; -import com.netflix.priam.notification.EventGenerator; -import com.netflix.priam.notification.EventObserver; import com.netflix.priam.scheduler.BlockingSubmitThreadPoolExecutor; import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.File; -import java.io.InputStream; -import java.io.OutputStream; import java.nio.file.Path; import java.util.Date; import java.util.Iterator; import java.util.List; -import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.LinkedBlockingQueue; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicLong; public abstract class S3FileSystemBase extends AbstractFileSystem { protected static final int MAX_CHUNKS = 10000; @@ -68,7 +57,7 @@ public S3FileSystemBase(Provider pathProvider, final IConfiguration config, BackupMetrics backupMetrics, BackupNotificationMgr backupNotificationMgr) { - super(backupMetrics, backupNotificationMgr); + super(config, backupMetrics, backupNotificationMgr); this.pathProvider = pathProvider; this.compress = compress; this.config = config; @@ -171,7 +160,7 @@ protected void checkSuccessfulUpload(CompleteMultipartUploadResult resultS3Multi } @Override - public long getFileSize(Path remotePath) throws BackupRestoreException{ + public long getFileSize(Path remotePath) throws BackupRestoreException { return s3Client.getObjectMetadata(getPrefix(config), remotePath.toString()).getContentLength(); } @@ -192,7 +181,7 @@ public Iterator list(String path, Date start, Date till) { return new S3FileIterator(pathProvider, s3Client, path, start, till); } - final long getChunkSize(Path localPath) throws BackupRestoreException{ + final long getChunkSize(Path localPath) throws BackupRestoreException { long chunkSize = config.getBackupChunkSize(); long fileSize = localPath.toFile().length(); diff --git a/priam/src/main/java/com/netflix/priam/backup/AbstractBackup.java b/priam/src/main/java/com/netflix/priam/backup/AbstractBackup.java index ce9450619..e0a167f44 100644 --- a/priam/src/main/java/com/netflix/priam/backup/AbstractBackup.java +++ b/priam/src/main/java/com/netflix/priam/backup/AbstractBackup.java @@ -73,51 +73,13 @@ List upload(File parent, final BackupFileType type) throws E //== decorate file with metadata final AbstractBackupPath bp = pathFactory.get(); bp.parseLocal(file, type); - - try { - logger.info("About to upload file {} for backup", file.getCanonicalFile()); - - // Allow up to 30s of arbitrary failures at the top level. The upload call itself typically has retries - // as well so this top level retry is on top of those retries. Assuming that each call to upload has - // ~30s maximum of retries this yields about 3.5 minutes of retries at the top level since - // (6 * (5 + 30) = 210 seconds). Even if this fails, however, higher level schedulers (e.g. in - // incremental) will hopefully re-enqueue. - AbstractBackupPath abp = new RetryableCallable(6, 5000) { - public AbstractBackupPath retriableCall() throws Exception { - upload(bp); - file.delete(); - return bp; - } - }.call(); - - if (abp != null) - bps.add(abp); - - addToRemotePath(abp.getRemotePath()); - } catch (Exception e) { - //Throw exception to the caller. This will allow them to take appropriate decision. - logger.error("Failed to upload local file {} within CF {}.", file.getCanonicalFile(), parent.getAbsolutePath(), e); - throw e; - } + fs.uploadFile(Paths.get(bp.getBackupFile().getAbsolutePath()), Paths.get(bp.getRemotePath()), bp, 10, true); + bps.add(bp); + addToRemotePath(bp.getRemotePath()); } return bps; } - - /** - * Upload specified file (RandomAccessFile) - * - * @param bp backup path to be uploaded. - */ - private void upload(final AbstractBackupPath bp) throws Exception { - try { - fs.uploadFile(Paths.get(bp.getBackupFile().getAbsolutePath()), Paths.get(bp.getRemotePath()), bp); - } catch (Exception e) { - logger.error("Exception uploading local file {}, releasing handle, and will retry.", bp.backupFile.getCanonicalFile()); - throw e; - } - } - protected final void initiateBackup(String monitoringFolder, BackupRestoreUtil backupRestoreUtil) throws Exception { File dataDir = new File(config.getDataFileLocation()); diff --git a/priam/src/main/java/com/netflix/priam/backup/AbstractBackupPath.java b/priam/src/main/java/com/netflix/priam/backup/AbstractBackupPath.java index a2f66a1a2..dc14a187e 100644 --- a/priam/src/main/java/com/netflix/priam/backup/AbstractBackupPath.java +++ b/priam/src/main/java/com/netflix/priam/backup/AbstractBackupPath.java @@ -290,7 +290,6 @@ public long getLastModified() { return lastModified; } - public static class RafInputStream extends InputStream implements AutoCloseable { private RandomAccessFile raf; diff --git a/priam/src/main/java/com/netflix/priam/backup/AbstractFileSystem.java b/priam/src/main/java/com/netflix/priam/backup/AbstractFileSystem.java index e0770b36b..3f9f914c2 100644 --- a/priam/src/main/java/com/netflix/priam/backup/AbstractFileSystem.java +++ b/priam/src/main/java/com/netflix/priam/backup/AbstractFileSystem.java @@ -18,21 +18,24 @@ package com.netflix.priam.backup; import com.google.inject.Inject; -import com.netflix.priam.backupv2.FileUploadResult; +import com.netflix.priam.config.IConfiguration; import com.netflix.priam.merics.BackupMetrics; import com.netflix.priam.notification.BackupEvent; import com.netflix.priam.notification.BackupNotificationMgr; import com.netflix.priam.notification.EventGenerator; import com.netflix.priam.notification.EventObserver; +import com.netflix.priam.scheduler.BlockingSubmitThreadPoolExecutor; +import com.netflix.priam.utils.BoundedExponentialRetryCallable; +import com.netflix.spectator.api.patterns.PolledMeter; +import org.apache.commons.io.FileUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.InputStream; -import java.io.OutputStream; +import java.io.FileNotFoundException; import java.nio.file.Path; -import java.util.Date; -import java.util.Iterator; -import java.util.concurrent.CopyOnWriteArrayList; +import java.util.HashSet; +import java.util.Set; +import java.util.concurrent.*; /** * Created by aagrawal on 8/30/18. @@ -41,54 +44,111 @@ public abstract class AbstractFileSystem implements IBackupFileSystem, EventGene private static final Logger logger = LoggerFactory.getLogger(AbstractFileSystem.class); private final CopyOnWriteArrayList> observers = new CopyOnWriteArrayList<>(); protected BackupMetrics backupMetrics; + private Set tasksQueued; + private ThreadPoolExecutor fileUploadExecutor; + private ThreadPoolExecutor fileDownloadExecutor; + private static final long UPLOAD_TIMEOUT = (2 * 60 * 60 * 1000L); //2 minutes. @Inject - public AbstractFileSystem(BackupMetrics backupMetrics, - BackupNotificationMgr backupNotificationMgr){ + public AbstractFileSystem(IConfiguration configuration, BackupMetrics backupMetrics, + BackupNotificationMgr backupNotificationMgr) { this.backupMetrics = backupMetrics; //Add notifications. this.addObserver(backupNotificationMgr); + tasksQueued = new HashSet<>(configuration.getUncrementalBkupQueueSize()); + BlockingQueue queue = new ArrayBlockingQueue(configuration.getUncrementalBkupQueueSize()); + PolledMeter.using(backupMetrics.getRegistry()).withName(backupMetrics.uploadDownloadQueueSize).monitorSize(queue); + this.fileUploadExecutor = new BlockingSubmitThreadPoolExecutor(configuration.getMaxBackupUploadThreads(), queue, UPLOAD_TIMEOUT); + this.fileDownloadExecutor = new BlockingSubmitThreadPoolExecutor(configuration.getMaxBackupDownloadThreads(), queue, UPLOAD_TIMEOUT); } @Override - public void downloadFile(Path remotePath, Path localPath) throws BackupRestoreException{ + public Future downloadFileAsync(final Path remotePath, final Path localPath, final int retry) throws BackupRestoreException, RejectedExecutionException { + return fileDownloadExecutor.submit(() -> { + downloadFile(remotePath, localPath, retry); + return remotePath; + }); + } + + @Override + public void downloadFile(final Path remotePath, final Path localPath, final int retry) throws BackupRestoreException { + //TODO: Should we download the file if localPath already exists? + if (remotePath == null) + return; + logger.info("Downloading file: {} to location: {}", remotePath, localPath); - try{ - //TODO: Retries should ideally go here. - downloadFileImpl(remotePath, localPath); + try { + new BoundedExponentialRetryCallable(500, 10000, retry) { + @Override + public Void retriableCall() throws Exception { + downloadFileImpl(remotePath, localPath); + return null; + } + }.call(); + //Note we only downloaded the bytes which are represented on file system (they are compressed and maybe encrypted). + //File size after decompression or decryption might be more/less. backupMetrics.recordDownloadRate(getFileSize(remotePath)); + backupMetrics.incrementValidDownloads(); logger.info("Successfully downloaded file: {} to location: {}", remotePath, localPath); - }catch (BackupRestoreException e){ + } catch (Exception e) { backupMetrics.incrementInvalidDownloads(); logger.error("Error while downloading file: {} to location: {}", remotePath, localPath); - throw e; + throw new BackupRestoreException(e.getMessage()); } } - protected abstract void downloadFileImpl(Path remotePath, Path localPath) throws BackupRestoreException; + protected abstract void downloadFileImpl(final Path remotePath, final Path localPath) throws BackupRestoreException; @Override - public void uploadFile(Path localPath, Path remotePath, AbstractBackupPath path) throws BackupRestoreException{ - if (!localPath.toFile().exists()) - throw new BackupRestoreException("File do not exist: {}" + localPath); - - logger.info("Uploading file: {} to location: {}", localPath, remotePath); - try{ - notifyEventStart(new BackupEvent(path)); - //TODO: Retries should ideally go here. - long uploadedFileSize = uploadFileImpl(localPath, remotePath); - backupMetrics.recordUploadRate(uploadedFileSize); - notifyEventSuccess(new BackupEvent(path)); - logger.info("Successfully uploaded file: {} to location: {}", localPath, remotePath); - }catch (BackupRestoreException e){ - backupMetrics.incrementInvalidUploads(); - notifyEventFailure(new BackupEvent(path)); - logger.error("Error while uploading file: {} to location: {}", localPath, remotePath); - throw e; - } + public Future asyncUploadFile(final Path localPath, final Path remotePath, final AbstractBackupPath path, final int retry, final boolean deleteAfterSuccessfulUpload) throws FileNotFoundException, RejectedExecutionException, BackupRestoreException { + return fileUploadExecutor.submit(() -> { + uploadFile(localPath, remotePath, path, retry, deleteAfterSuccessfulUpload); + return localPath; + }); + } + + @Override + public void uploadFile(final Path localPath, final Path remotePath, final AbstractBackupPath path, final int retry, final boolean deleteAfterSuccessfulUpload) throws FileNotFoundException, BackupRestoreException { + if (!localPath.toFile().exists() || localPath.toFile().isDirectory()) + throw new FileNotFoundException("File do not exist or is a directory: {}" + localPath); + + if (!tasksQueued.contains(localPath)) { + //Add file to local memory + tasksQueued.add(localPath); + + logger.info("Uploading file: {} to location: {}", localPath, remotePath); + try { + notifyEventStart(new BackupEvent(path)); + long uploadedFileSize = new BoundedExponentialRetryCallable(500, 10000, retry) { + @Override + public Long retriableCall() throws Exception { + return uploadFileImpl(localPath, remotePath); + } + }.call(); + backupMetrics.recordUploadRate(uploadedFileSize); + backupMetrics.incrementValidUploads(); + path.setCompressedFileSize(uploadedFileSize); + notifyEventSuccess(new BackupEvent(path)); + logger.info("Successfully uploaded file: {} to location: {}", localPath, remotePath); + + if (deleteAfterSuccessfulUpload) + FileUtils.deleteQuietly(localPath.toFile()); + + } catch (Exception e) { + backupMetrics.incrementInvalidUploads(); + notifyEventFailure(new BackupEvent(path)); + logger.error("Error while uploading file: {} to location: {}", localPath, remotePath); + throw new BackupRestoreException(e.getMessage()); + } finally { + //Remove the task from the list so if we try to upload file ever again, we can. + tasksQueued.remove(localPath); + } + } else + logger.info("Already in queue, no-op. File: {}", localPath); + } - protected abstract long uploadFileImpl(Path localPath, Path remotePath) throws BackupRestoreException; + protected abstract long uploadFileImpl(final Path localPath, final Path remotePath) throws BackupRestoreException; @Override public final void addObserver(EventObserver observer) { @@ -125,4 +185,9 @@ public void notifyEventFailure(BackupEvent event) { public void notifyEventStop(BackupEvent event) { observers.forEach(eventObserver -> eventObserver.updateEventStop(event)); } + + @Override + public int getTasksQueued(){ + return tasksQueued.size(); + } } diff --git a/priam/src/main/java/com/netflix/priam/backup/BackupVerification.java b/priam/src/main/java/com/netflix/priam/backup/BackupVerification.java index 8f002bb00..afecbcb6f 100644 --- a/priam/src/main/java/com/netflix/priam/backup/BackupVerification.java +++ b/priam/src/main/java/com/netflix/priam/backup/BackupVerification.java @@ -118,7 +118,7 @@ public BackupVerificationResult verifyBackup(List metadata, Date List metaFileList = new ArrayList<>(); try { Path metaFileLocation = FileSystems.getDefault().getPath(config.getDataFileLocation(), "tmp_meta.json"); - bkpStatusFs.downloadFile(Paths.get(metas.get(0).getRemotePath()), metaFileLocation); + bkpStatusFs.downloadFile(Paths.get(metas.get(0).getRemotePath()), metaFileLocation, 5); logger.info("Meta file successfully downloaded to localhost: {}", metaFileLocation.toString()); JSONParser jsonParser = new JSONParser(); diff --git a/priam/src/main/java/com/netflix/priam/backup/CommitLogBackup.java b/priam/src/main/java/com/netflix/priam/backup/CommitLogBackup.java index c5f995034..97c5dc1ac 100644 --- a/priam/src/main/java/com/netflix/priam/backup/CommitLogBackup.java +++ b/priam/src/main/java/com/netflix/priam/backup/CommitLogBackup.java @@ -65,24 +65,15 @@ public List upload(String archivedDir, final String snapshot for (final File file : archivedCommitLogDir.listFiles()) { logger.debug("Uploading commit log {} for backup", file.getCanonicalFile()); try { - AbstractBackupPath abp = (AbstractBackupPath) new RetryableCallable(3, 100L) { - public AbstractBackupPath retriableCall() throws Exception { + AbstractBackupPath bp = pathFactory.get(); + bp.parseLocal(file, BackupFileType.CL); - AbstractBackupPath bp = pathFactory.get(); - bp.parseLocal(file, BackupFileType.CL); - if (snapshotName != null) - bp.time = bp.parseDate(snapshotName); - upload(bp); - file.delete(); //TODO: should we put delete call here? We don't want to delete if the upload operation fails - return bp; - } - } - .call(); + if (snapshotName != null) + bp.time = bp.parseDate(snapshotName); - if (abp != null) { - bps.add(abp); - } - addToRemotePath(abp.getRemotePath()); + fs.uploadFile(Paths.get(bp.getBackupFile().getAbsolutePath()), Paths.get(bp.getRemotePath()), bp, 10, true); + bps.add(bp); + addToRemotePath(bp.getRemotePath()); } catch (Exception e) { logger.error("Failed to upload local file {}. Ignoring to continue with rest of backup.", file, e); } @@ -90,18 +81,6 @@ public AbstractBackupPath retriableCall() throws Exception { return bps; } - private void upload(final AbstractBackupPath bp) - throws Exception { - new RetryableCallable() { - public Void retriableCall() - throws Exception { - fs.uploadFile(Paths.get(bp.getBackupFile().getAbsolutePath()), Paths.get(bp.getRemotePath()), bp); - return null; - } - } - .call(); - } - public static void addObserver(IMessageObserver observer) { observers.add(observer); } diff --git a/priam/src/main/java/com/netflix/priam/backup/IBackupFileSystem.java b/priam/src/main/java/com/netflix/priam/backup/IBackupFileSystem.java index 1261201b6..b1535a0e3 100644 --- a/priam/src/main/java/com/netflix/priam/backup/IBackupFileSystem.java +++ b/priam/src/main/java/com/netflix/priam/backup/IBackupFileSystem.java @@ -16,9 +16,12 @@ */ package com.netflix.priam.backup; +import java.io.FileNotFoundException; import java.nio.file.Path; import java.util.Date; import java.util.Iterator; +import java.util.concurrent.Future; +import java.util.concurrent.RejectedExecutionException; /** * Interface representing a backup storage as a file system @@ -29,22 +32,63 @@ public interface IBackupFileSystem { * * @param remotePath fully qualified location of the file on remote file system. * @param localPath location on the local file sytem where remote file should be downloaded. + * @param retry No. of times to retry to download a file from remote file system. If <=0, it will try to download file exactly once. * @throws BackupRestoreException if file is not available, downloadable or any other error from remote file system. */ - void downloadFile(Path remotePath, Path localPath) throws BackupRestoreException; + void downloadFile(Path remotePath, Path localPath, int retry) throws BackupRestoreException; + + /** + * Download the file denoted by remotePath in an async fashion to the local file system denoted by local path. + * + * @param remotePath fully qualified location of the file on remote file system. + * @param localPath location on the local file sytem where remote file should be downloaded. + * @param retry No. of times to retry to download a file from remote file system. If <=0, it will try to download file exactly once. + * @return The future of the async job to monitor the progress of the job. + * @throws BackupRestoreException if file is not available, downloadable or any other error from remote file system. + * @throws RejectedExecutionException if the queue is full and TIMEOUT is reached while trying to add the work to the queue. + */ + Future downloadFileAsync(final Path remotePath, final Path localPath, final int retry) throws BackupRestoreException, RejectedExecutionException; + /** * Upload the local file denoted by localPath to the remote file system at location denoted by remotePath. + * De-duping of the file to upload will always be done by comparing the files-in-progress to be uploaded. + * This may result in this particular request to not to be executed e.g. if any other thread has given the same file + * to upload and that file is in internal queue. + * Note that de-duping is best effort and is not always guaranteed as we try to avoid lock on read/write of the files-in-progress. * - * @param localPath Path of the local file that needs to be uploaded. - * @param remotePath Fully qualified path on the remote file system where file should be uploaded. - * @param path AbstractBackupPath to be used to send backup notifications only. - * @throws BackupRestoreException in case of failure to upload for any reason including file not available, readable or remote file system errors. + * @param localPath Path of the local file that needs to be uploaded. + * @param remotePath Fully qualified path on the remote file system where file should be uploaded. + * @param path AbstractBackupPath to be used to send backup notifications only. + * @param retry No of times to retry to upload a file. If <=0, it will try to upload file exactly once. + * @param deleteAfterSuccessfulUpload If true, delete the file denoted by localPath after it is successfully uploaded to the filesystem. If there is any failure, file will not be deleted. + * @throws BackupRestoreException in case of failure to upload for any reason including file not readable or remote file system errors. + * @throws FileNotFoundException If a file as denoted by localPath is not available or is a directory. */ - void uploadFile(Path localPath, Path remotePath, AbstractBackupPath path) throws BackupRestoreException; + void uploadFile(Path localPath, Path remotePath, AbstractBackupPath path, int retry, boolean deleteAfterSuccessfulUpload) throws FileNotFoundException, BackupRestoreException; + + /** + * Upload the local file denoted by localPath in async fashion to the remote file system at location denoted by remotePath. + * + * @param localPath Path of the local file that needs to be uploaded. + * @param remotePath Fully qualified path on the remote file system where file should be uploaded. + * @param path AbstractBackupPath to be used to send backup notifications only. + * @param retry No of times to retry to upload a file. If <=0, it will try to upload file exactly once. + * @param deleteAfterSuccessfulUpload If true, delete the file denoted by localPath after it is successfully uploaded to the filesystem. If there is any failure, file will not be deleted. + * @return The future of the async job to monitor the progress of the job. This will be null if file was de-duped for upload. + * @throws BackupRestoreException in case of failure to upload for any reason including file not readable or remote file system errors. + * @throws FileNotFoundException If a file as denoted by localPath is not available or is a directory. + * @throws RejectedExecutionException if the queue is full and TIMEOUT is reached while trying to add the work to the queue. + */ + Future asyncUploadFile(final Path localPath, final Path remotePath, final AbstractBackupPath path, final int retry, final boolean deleteAfterSuccessfulUpload) throws FileNotFoundException, RejectedExecutionException, BackupRestoreException; /** * List all files in the backup location for the specified time range. + * + * @param path This is used as the `prefix` for listing files in the filesystem. All the files that start with this prefix will be returned. + * @param start Start date of the file upload. + * @param till End date of the file upload. + * @return Iterator of the AbstractBackupPath matching the criteria. */ Iterator list(String path, Date start, Date till); @@ -72,4 +116,9 @@ public interface IBackupFileSystem { */ long getFileSize(Path remotePath) throws BackupRestoreException; + /** + * Get the number of tasks en-queue in the filesystem for download/upload. + * @return the total no. of tasks to be executed. + */ + int getTasksQueued(); } diff --git a/priam/src/main/java/com/netflix/priam/backup/MetaData.java b/priam/src/main/java/com/netflix/priam/backup/MetaData.java index df0cb85da..8cd6cab95 100644 --- a/priam/src/main/java/com/netflix/priam/backup/MetaData.java +++ b/priam/src/main/java/com/netflix/priam/backup/MetaData.java @@ -29,7 +29,10 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.*; +import java.io.File; +import java.io.FileReader; +import java.io.FileWriter; +import java.io.IOException; import java.nio.file.Paths; import java.text.ParseException; import java.util.ArrayList; @@ -56,22 +59,17 @@ public MetaData(Provider pathFactory, IFileSystemContext bac public AbstractBackupPath set(List bps, String snapshotName) throws Exception { File metafile = createTmpMetaFile(); - try(FileWriter fr = new FileWriter(metafile)) { + try (FileWriter fr = new FileWriter(metafile)) { JSONArray jsonObj = new JSONArray(); for (AbstractBackupPath filePath : bps) jsonObj.add(filePath.getRemotePath()); fr.write(jsonObj.toJSONString()); } AbstractBackupPath backupfile = decorateMetaJson(metafile, snapshotName); - try { - upload(backupfile); - - addToRemotePath(backupfile.getRemotePath()); - if (metaRemotePaths.size() > 0) { - notifyObservers(); - } - } finally { - FileUtils.deleteQuietly(metafile); + fs.uploadFile(Paths.get(backupfile.getBackupFile().getAbsolutePath()), Paths.get(backupfile.getRemotePath()), backupfile, 10, true); + addToRemotePath(backupfile.getRemotePath()); + if (metaRemotePaths.size() > 0) { + notifyObservers(); } return backupfile; @@ -97,14 +95,7 @@ public AbstractBackupPath decorateMetaJson(File metafile, String snapshotName) t */ public Boolean doesExist(final AbstractBackupPath meta) { try { - new RetryableCallable() { - @Override - public Void retriableCall() throws Exception { - fs.downloadFile(Paths.get(meta.getRemotePath()),Paths.get(meta.newRestoreFile().getAbsolutePath())); //download actual file to disk - return null; - } - }.call(); - + fs.downloadFile(Paths.get(meta.getRemotePath()), Paths.get(meta.newRestoreFile().getAbsolutePath()), 5); //download actual file to disk } catch (Exception e) { logger.error("Error downloading the Meta data try with a different date...", e); } @@ -113,16 +104,6 @@ public Void retriableCall() throws Exception { } - private void upload(final AbstractBackupPath bp) throws Exception { - new RetryableCallable() { - @Override - public Void retriableCall() throws Exception { - fs.uploadFile(Paths.get(bp.getBackupFile().getAbsolutePath()), Paths.get(bp.getRemotePath()), bp); - return null; - } - }.call(); - } - public File createTmpMetaFile() throws IOException { File metafile = File.createTempFile("meta", ".json"); File destFile = new File(metafile.getParent(), "meta.json"); diff --git a/priam/src/main/java/com/netflix/priam/backup/parallel/IncrementalConsumer.java b/priam/src/main/java/com/netflix/priam/backup/parallel/IncrementalConsumer.java index 098c3a6b4..7becd0fb1 100644 --- a/priam/src/main/java/com/netflix/priam/backup/parallel/IncrementalConsumer.java +++ b/priam/src/main/java/com/netflix/priam/backup/parallel/IncrementalConsumer.java @@ -58,33 +58,13 @@ public void run() { logger.info("Consumer - about to upload file: {}", this.bp.getFileName()); try { - // Allow up to 30s of arbitrary failures at the top level. The upload call itself typically has retries - // as well so this top level retry is on top of those retries. Assuming that each call to upload has - // ~30s maximum of retries this yields about 3.5 minutes of retries at the top level since - // (6 * (5 + 30) = 210 seconds). Even if this fails, however, the upload will be re-enqueued - new RetryableCallable(6, 5000) { - @Override - public Void retriableCall() throws Exception { - if (!bp.getBackupFile().exists()) throw new java.util.concurrent.CancellationException("Someone beat me to uploading this file" + ", no need to retry. Most likely not needed but to be safe, checked and released handle to file if appropriate."); - try { - fs.uploadFile(Paths.get(bp.getBackupFile().getAbsolutePath()), Paths.get(bp.getRemotePath()), bp); - return null; - } catch (Exception e) { - logger.error("Exception uploading local file {}, releasing handle, and will retry.", bp.getFileName()); - throw e; - } - } - }.call(); - // Clean up the underlying file. - bp.getBackupFile().delete(); + fs.uploadFile(Paths.get(bp.getBackupFile().getAbsolutePath()), Paths.get(bp.getRemotePath()), bp, 10, true); + this.callback.postProcessing(bp); //post processing } catch (Exception e) { - if (e instanceof java.util.concurrent.CancellationException) { - logger.debug("Failed to upload local file {}. Ignoring to continue with rest of backup. Msg: {}", this.bp.getFileName(), e.getLocalizedMessage()); - } else { logger.error("Failed to upload local file {}. Ignoring to continue with rest of backup. Msg: {}", this.bp.getFileName(), e.getLocalizedMessage()); } } finally { diff --git a/priam/src/main/java/com/netflix/priam/backupv2/MetaFileWriterBuilder.java b/priam/src/main/java/com/netflix/priam/backupv2/MetaFileWriterBuilder.java index b2ca7e2ef..73c44dec6 100644 --- a/priam/src/main/java/com/netflix/priam/backupv2/MetaFileWriterBuilder.java +++ b/priam/src/main/java/com/netflix/priam/backupv2/MetaFileWriterBuilder.java @@ -160,16 +160,7 @@ public MetaFileWriterBuilder.UploadStep endMetaFileGeneration() throws IOExcepti public void uploadMetaFile(boolean deleteOnSuccess) throws Exception { AbstractBackupPath abstractBackupPath = pathFactory.get(); abstractBackupPath.parseLocal(metaFilePath.toFile(), AbstractBackupPath.BackupFileType.META_V2); - new RetryableCallable(6, 5000) { - @Override - public Void retriableCall() throws Exception { - backupFileSystem.uploadFile(metaFilePath, Paths.get(abstractBackupPath.getRemotePath()), abstractBackupPath); - return null; - } - }.call(); - - if (deleteOnSuccess) - FileUtils.deleteQuietly(metaFilePath.toFile()); + backupFileSystem.uploadFile(metaFilePath, Paths.get(abstractBackupPath.getRemotePath()), abstractBackupPath, 10, deleteOnSuccess); } public Path getMetaFilePath(){ diff --git a/priam/src/main/java/com/netflix/priam/google/GoogleEncryptedFileSystem.java b/priam/src/main/java/com/netflix/priam/google/GoogleEncryptedFileSystem.java index 4a0bfb5d4..0a837160b 100755 --- a/priam/src/main/java/com/netflix/priam/google/GoogleEncryptedFileSystem.java +++ b/priam/src/main/java/com/netflix/priam/google/GoogleEncryptedFileSystem.java @@ -71,7 +71,7 @@ public class GoogleEncryptedFileSystem extends AbstractFileSystem { @Inject public GoogleEncryptedFileSystem(Provider pathProvider, final IConfiguration config , @Named("gcscredential") ICredentialGeneric credential, BackupMetrics backupMetrics, BackupNotificationMgr backupNotificationManager) { - super(backupMetrics, backupNotificationManager); + super(config, backupMetrics, backupNotificationManager); this.backupMetrics = backupMetrics; this.pathProvider = pathProvider; this.config = config; diff --git a/priam/src/main/java/com/netflix/priam/merics/BackupMetrics.java b/priam/src/main/java/com/netflix/priam/merics/BackupMetrics.java index 5a8c079f4..52ea387d1 100644 --- a/priam/src/main/java/com/netflix/priam/merics/BackupMetrics.java +++ b/priam/src/main/java/com/netflix/priam/merics/BackupMetrics.java @@ -27,15 +27,20 @@ */ @Singleton public class BackupMetrics { + private Registry registry; /** * Distribution summary will provide the metric like count (how many uploads were made), max no. of bytes uploaded and total amount of bytes uploaded. */ private final DistributionSummary uploadRate, downloadRate; - private final Counter invalidUploads, invalidDownloads, snsNotificationSuccess, snsNotificationFailure, forgottenFiles; + private final Counter validUploads, validDownloads, invalidUploads, invalidDownloads, snsNotificationSuccess, snsNotificationFailure, forgottenFiles; + public static final String uploadDownloadQueueSize = Metrics.METRIC_PREFIX + "upload.download.queue.size"; @Inject public BackupMetrics(Registry registry) { + this.registry = registry; + validDownloads = registry.counter(Metrics.METRIC_PREFIX + "download.valid"); invalidDownloads = registry.counter(Metrics.METRIC_PREFIX + "download.invalid"); + validUploads = registry.counter(Metrics.METRIC_PREFIX + "upload.valid"); invalidUploads = registry.counter(Metrics.METRIC_PREFIX + "upload.invalid"); uploadRate = registry.distributionSummary(Metrics.METRIC_PREFIX + "upload.rate"); downloadRate = registry.distributionSummary(Metrics.METRIC_PREFIX + "download.rate"); @@ -84,7 +89,9 @@ public void recordUploadRate(long sizeInBytes) { uploadRate.record(sizeInBytes); } - public void incrementForgottenFiles(long forgottenFilesVal) {forgottenFiles.increment(forgottenFilesVal);} + public void incrementForgottenFiles(long forgottenFilesVal) { + forgottenFiles.increment(forgottenFilesVal); + } public void recordDownloadRate(long sizeInBytes) { downloadRate.record(sizeInBytes); @@ -93,4 +100,24 @@ public void recordDownloadRate(long sizeInBytes) { public DistributionSummary getDownloadRate() { return downloadRate; } + + public Counter getValidUploads() { + return validUploads; + } + + public Counter getValidDownloads() { + return validDownloads; + } + + public void incrementValidUploads() { + this.validUploads.increment(); + } + + public void incrementValidDownloads() { + this.validDownloads.increment(); + } + + public Registry getRegistry() { + return registry; + } } diff --git a/priam/src/main/java/com/netflix/priam/restore/EncryptedRestoreBase.java b/priam/src/main/java/com/netflix/priam/restore/EncryptedRestoreBase.java index 633cee63a..b7bf444ae 100755 --- a/priam/src/main/java/com/netflix/priam/restore/EncryptedRestoreBase.java +++ b/priam/src/main/java/com/netflix/priam/restore/EncryptedRestoreBase.java @@ -79,7 +79,8 @@ public Integer retriableCall() throws Exception { //== download object from source bucket try { - fs.downloadFile(Paths.get(path.getRemotePath()), Paths.get(tempFile.getAbsolutePath())); + //Not retrying to download file here as it is already in RetryCallable. + fs.downloadFile(Paths.get(path.getRemotePath()), Paths.get(tempFile.getAbsolutePath()), 0); tracker.adjustAndAdd(path); } catch (Exception ex) { //This behavior is retryable; therefore, lets get to a clean state before each retry. diff --git a/priam/src/main/java/com/netflix/priam/restore/Restore.java b/priam/src/main/java/com/netflix/priam/restore/Restore.java index dd8e21378..9fbbd68d8 100644 --- a/priam/src/main/java/com/netflix/priam/restore/Restore.java +++ b/priam/src/main/java/com/netflix/priam/restore/Restore.java @@ -63,15 +63,9 @@ public Restore(IConfiguration config, @Named("backup") IBackupFileSystem fs, Sle @Override protected final void downloadFile(final AbstractBackupPath path, final File restoreLocation) throws Exception { count.incrementAndGet(); - executor.submit(new RetryableCallable() { - @Override - public Integer retriableCall() throws Exception { - fs.downloadFile(Paths.get(path.getRemotePath()), Paths.get(restoreLocation.getAbsolutePath())); - tracker.adjustAndAdd(path); - // TODO: fix me -> if there is exception the why hang? - return count.decrementAndGet(); - } - }); + fs.downloadFile(Paths.get(path.getRemotePath()), Paths.get(restoreLocation.getAbsolutePath()), 5); + tracker.adjustAndAdd(path); + count.decrementAndGet(); } @Override diff --git a/priam/src/test/java/com/netflix/priam/backup/FakeBackupFileSystem.java b/priam/src/test/java/com/netflix/priam/backup/FakeBackupFileSystem.java index ab5e2f9e5..d8fb37a97 100644 --- a/priam/src/test/java/com/netflix/priam/backup/FakeBackupFileSystem.java +++ b/priam/src/test/java/com/netflix/priam/backup/FakeBackupFileSystem.java @@ -22,6 +22,7 @@ import com.google.inject.Singleton; import com.netflix.priam.aws.S3BackupPath; import com.netflix.priam.backup.AbstractBackupPath.BackupFileType; +import com.netflix.priam.config.IConfiguration; import com.netflix.priam.merics.BackupMetrics; import com.netflix.priam.notification.BackupNotificationMgr; import org.json.simple.JSONArray; @@ -44,9 +45,9 @@ public class FakeBackupFileSystem extends AbstractFileSystem { Provider pathProvider; @Inject - public FakeBackupFileSystem(BackupMetrics backupMetrics, + public FakeBackupFileSystem(IConfiguration configuration, BackupMetrics backupMetrics, BackupNotificationMgr backupNotificationMgr){ - super(backupMetrics, backupNotificationMgr); + super(configuration, backupMetrics, backupNotificationMgr); } public void setupTest(List files) { diff --git a/priam/src/test/java/com/netflix/priam/backup/NullBackupFileSystem.java b/priam/src/test/java/com/netflix/priam/backup/NullBackupFileSystem.java index 31f69b385..7ba439ec6 100644 --- a/priam/src/test/java/com/netflix/priam/backup/NullBackupFileSystem.java +++ b/priam/src/test/java/com/netflix/priam/backup/NullBackupFileSystem.java @@ -18,6 +18,7 @@ package com.netflix.priam.backup; import com.google.inject.Inject; +import com.netflix.priam.config.IConfiguration; import com.netflix.priam.merics.BackupMetrics; import com.netflix.priam.notification.BackupNotificationMgr; @@ -29,9 +30,9 @@ public class NullBackupFileSystem extends AbstractFileSystem { @Inject - public NullBackupFileSystem(BackupMetrics backupMetrics, + public NullBackupFileSystem(IConfiguration configuration, BackupMetrics backupMetrics, BackupNotificationMgr backupNotificationMgr){ - super(backupMetrics, backupNotificationMgr); + super(configuration, backupMetrics, backupNotificationMgr); } @Override diff --git a/priam/src/test/java/com/netflix/priam/backup/TestAbstractFileSystem.java b/priam/src/test/java/com/netflix/priam/backup/TestAbstractFileSystem.java new file mode 100644 index 000000000..6bb76d29d --- /dev/null +++ b/priam/src/test/java/com/netflix/priam/backup/TestAbstractFileSystem.java @@ -0,0 +1,318 @@ +/* + * Copyright 2018 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.netflix.priam.backup; + +import com.google.inject.Guice; +import com.google.inject.Inject; +import com.google.inject.Injector; +import com.netflix.priam.config.IConfiguration; +import com.netflix.priam.merics.BackupMetrics; +import com.netflix.priam.notification.BackupNotificationMgr; +import com.netflix.priam.utils.BackupFileUtils; +import org.apache.commons.io.FileUtils; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.text.ParseException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.Random; +import java.util.concurrent.Callable; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; + +/** + * The goal of this class is to test common functionality which are encapsulated in AbstractFileSystem. + * The actual upload/download of a file to remote file system is beyond the scope of this class. + * Created by aagrawal on 9/22/18. + */ +public class TestAbstractFileSystem { + private static final Logger logger = LoggerFactory.getLogger(TestAbstractFileSystem.class.getName()); + private Injector injector; + private IConfiguration configuration; + private BackupMetrics backupMetrics; + private BackupNotificationMgr backupNotificationMgr; + private AbstractFileSystem failureFileSystem; + private MyFileSystem myFileSystem; + + @Before + public void setBackupMetrics() { + if (injector == null) + injector = Guice.createInjector(new BRTestModule()); + + if (configuration == null) + configuration = injector.getInstance(IConfiguration.class); + + if (backupNotificationMgr == null) + backupNotificationMgr = injector.getInstance(BackupNotificationMgr.class); + + backupMetrics = injector.getInstance(BackupMetrics.class); + + if (failureFileSystem == null) + failureFileSystem = new FailureFileSystem(configuration, backupMetrics, backupNotificationMgr); + + if (myFileSystem == null) + myFileSystem = new MyFileSystem(configuration, backupMetrics, backupNotificationMgr); + + BackupFileUtils.cleanupDir(Paths.get(configuration.getDataFileLocation())); + } + + + @Test + public void testFailedRetriesUpload() throws Exception { + try { + Collection files = generateFiles(1, 1, 1); + for (File file : files) { + failureFileSystem.uploadFile(file.toPath(), null, getDummyPath(file.toPath()), 2, true); + } + } catch (BackupRestoreException e) { + //Verify the failure metric for upload is incremented. + Assert.assertEquals(1, (int) backupMetrics.getInvalidUploads().count()); + } + } + + private AbstractBackupPath getDummyPath(Path localPath) throws ParseException { + AbstractBackupPath path = injector.getInstance(AbstractBackupPath.class); + path.parseLocal(localPath.toFile(), AbstractBackupPath.BackupFileType.SNAP); + return path; + } + + private Collection generateFiles(int noOfKeyspaces, int noOfCf, int noOfSstables) throws Exception { + Path dataDir = Paths.get(configuration.getDataFileLocation()); + BackupFileUtils.generateDummyFiles(dataDir, noOfKeyspaces, noOfCf, noOfSstables, "snapshot", "201812310000"); + String[] ext = {"db"}; + return FileUtils.listFiles(dataDir.toFile(), ext, true); + } + + @Test + public void testFailedRetriesDownload() { + try { + failureFileSystem.downloadFile(Paths.get(""), null, 2); + } catch (BackupRestoreException e) { + //Verify the failure metric for download is incremented. + Assert.assertEquals(1, (int) backupMetrics.getInvalidDownloads().count()); + } + } + + @Test + public void testUpload() throws Exception { + Collection files = generateFiles(1, 1, 1); + //Dummy upload with compressed size. + for (File file : files) { + myFileSystem.uploadFile(file.toPath(), null, getDummyPath(file.toPath()), 2, true); + //Verify the success metric for upload is incremented. + Assert.assertEquals(1, (int) backupMetrics.getValidUploads().actualCount()); + + //Verify delete of the original file if flag provided. + Assert.assertFalse(file.exists()); + break; + } + } + + @Test + public void testDownload() throws Exception { + //Dummy download + myFileSystem.downloadFile(Paths.get(""), null, 2); + //Verify the success metric for download is incremented. + Assert.assertEquals(1, (int) backupMetrics.getValidDownloads().actualCount()); + } + + @Test + public void testAsyncUpload() throws Exception { + //Testing single async upload. + Collection files = generateFiles(1, 1, 1); + for (File file : files) { + myFileSystem.asyncUploadFile(file.toPath(), null, getDummyPath(file.toPath()), 2, true).get(); + //1. Verify the success metric for upload is incremented. + Assert.assertEquals(1, (int) backupMetrics.getValidUploads().actualCount()); + //2. The task queue is empty after upload is finished. + Assert.assertEquals(0, myFileSystem.getTasksQueued()); + break; + } + } + + @Test + public void testAsyncUploadBulk() throws Exception { + //Testing the queue feature works. + //1. Give 1000 dummy files to upload. File upload takes some random time to upload + Collection files = generateFiles(1, 1, 20); + List> futures = new ArrayList<>(); + for (File file : files) { + futures.add(myFileSystem.asyncUploadFile(file.toPath(), null, getDummyPath(file.toPath()), 2, true)); + } + + //Verify all the work is finished. + for (Future future : futures) { + future.get(); + } + //2. Success metric is incremented correctly + Assert.assertEquals(files.size(), (int) backupMetrics.getValidUploads().actualCount()); + + //3. The task queue is empty after upload is finished. + Assert.assertEquals(0, myFileSystem.getTasksQueued()); + } + + @Test + public void testUploadDedup() throws Exception { + //Testing the de-duping works. + Collection files = generateFiles(1, 1, 1); + File file = files.iterator().next(); + AbstractBackupPath abstractBackupPath = getDummyPath(file.toPath()); + //1. Give same file to upload x times. Only one request will be entertained. + int size = 10; + ExecutorService threads = Executors.newFixedThreadPool(size); + List> torun = new ArrayList<>(size); + for (int i = 0; i < size; i++) { + torun.add(() -> { + myFileSystem.uploadFile(file.toPath(), null, abstractBackupPath, 2, true); + return Boolean.TRUE; + }); + } + + // all tasks executed in different threads, at 'once'. + List> futures = threads.invokeAll(torun); + + // no more need for the threadpool + threads.shutdown(); + for (Future future : futures) { + future.get(); + } + //2. Verify the success metric for upload is not same as size, i.e. some amount of de-duping happened. + Assert.assertNotEquals(size, (int) backupMetrics.getValidUploads().actualCount()); + } + + @Test + public void testAsyncUploadFailure() throws Exception { + //Testing single async upload. + Collection files = generateFiles(1, 1, 1); + for (File file : files) { + Future future = failureFileSystem.asyncUploadFile(file.toPath(), null, getDummyPath(file.toPath()), 2, true); + try { + future.get(); + } catch (Exception e) { + //1. Future get returns error message. + + //2. Verify the failure metric for upload is incremented. + Assert.assertEquals(1, (int) backupMetrics.getInvalidUploads().count()); + + //3. The task queue is empty after upload is finished. + Assert.assertEquals(0, failureFileSystem.getTasksQueued()); + break; + } + } + } + + @Test + public void testAsyncDownload() throws Exception { + //Testing single async download. + Future future = myFileSystem.downloadFileAsync(Paths.get(""), null, 2); + future.get(); + //1. Verify the success metric for download is incremented. + Assert.assertEquals(1, (int) backupMetrics.getValidDownloads().actualCount()); + //2. Verify the queue size is '0' after success. + Assert.assertEquals(0, myFileSystem.getTasksQueued()); + } + + @Test + public void testAsyncDownloadBulk() throws Exception { + //Testing the queue feature works. + //1. Give 1000 dummy files to download. File download takes some random time to download. + int totalFiles = 1000; + List> futureList = new ArrayList<>(); + for (int i = 0; i < totalFiles; i++) + futureList.add(myFileSystem.downloadFileAsync(Paths.get("" + i), null, 2)); + + //Ensure processing is finished. + for (Future future1 : futureList) { + future1.get(); + } + + //2. Success metric is incremented correctly -> exactly 1000 times. + Assert.assertEquals(totalFiles, (int) backupMetrics.getValidDownloads().actualCount()); + + //3. The task queue is empty after download is finished. + Assert.assertEquals(0, myFileSystem.getTasksQueued()); + } + + @Test + public void testAsyncDownloadFailure() throws Exception { + Future future = failureFileSystem.downloadFileAsync(Paths.get(""), null, 2); + try { + future.get(); + } catch (Exception e) { + //Verify the failure metric for upload is incremented. + Assert.assertEquals(1, (int) backupMetrics.getInvalidDownloads().count()); + } + } + + class FailureFileSystem extends NullBackupFileSystem { + + @Inject + public FailureFileSystem(IConfiguration configuration, BackupMetrics backupMetrics, BackupNotificationMgr backupNotificationMgr) { + super(configuration, backupMetrics, backupNotificationMgr); + } + + @Override + protected void downloadFileImpl(Path remotePath, Path localPath) throws BackupRestoreException { + throw new BackupRestoreException("User injected failure file system error for testing download. Remote path: " + remotePath); + } + + @Override + protected long uploadFileImpl(Path localPath, Path remotePath) throws BackupRestoreException { + throw new BackupRestoreException("User injected failure file system error for testing upload. Local path: " + localPath); + } + } + + class MyFileSystem extends NullBackupFileSystem { + + private Random random = new Random(); + + @Inject + public MyFileSystem(IConfiguration configuration, BackupMetrics backupMetrics, BackupNotificationMgr backupNotificationMgr) { + super(configuration, backupMetrics, backupNotificationMgr); + } + + @Override + protected void downloadFileImpl(Path remotePath, Path localPath) throws BackupRestoreException { + try { + Thread.sleep(random.nextInt(20)); + } catch (InterruptedException e) { + e.printStackTrace(); + } + } + + @Override + protected long uploadFileImpl(Path localPath, Path remotePath) throws BackupRestoreException { + try { + Thread.sleep(random.nextInt(20)); + } catch (InterruptedException e) { + e.printStackTrace(); + } + + return 0; + } + } +} diff --git a/priam/src/test/java/com/netflix/priam/backup/TestS3FileSystem.java b/priam/src/test/java/com/netflix/priam/backup/TestS3FileSystem.java index 830c22ab3..ae3fa327a 100644 --- a/priam/src/test/java/com/netflix/priam/backup/TestS3FileSystem.java +++ b/priam/src/test/java/com/netflix/priam/backup/TestS3FileSystem.java @@ -94,7 +94,7 @@ public void testFileUpload() throws Exception { S3BackupPath backupfile = injector.getInstance(S3BackupPath.class); backupfile.parseLocal(new File(FILE_PATH), BackupFileType.SNAP); long noOfFilesUploaded = backupMetrics.getUploadRate().count(); - fs.uploadFile(Paths.get(backupfile.getBackupFile().getAbsolutePath()), Paths.get(backupfile.getRemotePath()), backupfile); + fs.uploadFile(Paths.get(backupfile.getBackupFile().getAbsolutePath()), Paths.get(backupfile.getRemotePath()), backupfile, 0, false); Assert.assertEquals(1, backupMetrics.getUploadRate().count() - noOfFilesUploaded); } @@ -108,7 +108,7 @@ public void testFileUploadFailures() throws Exception { S3BackupPath backupfile = injector.getInstance(S3BackupPath.class); backupfile.parseLocal(new File(snapshotfile), BackupFileType.SNAP); try { - fs.uploadFile(Paths.get(backupfile.getBackupFile().getAbsolutePath()), Paths.get(backupfile.getRemotePath()), backupfile); + fs.uploadFile(Paths.get(backupfile.getBackupFile().getAbsolutePath()), Paths.get(backupfile.getRemotePath()), backupfile, 0, false); } catch (BackupRestoreException e) { // ignore } @@ -126,7 +126,7 @@ public void testFileUploadCompleteFailure() throws Exception { S3BackupPath backupfile = injector.getInstance(S3BackupPath.class); backupfile.parseLocal(new File(snapshotfile), BackupFileType.SNAP); try { - fs.uploadFile(Paths.get(backupfile.getBackupFile().getAbsolutePath()), Paths.get(backupfile.getRemotePath()), backupfile); + fs.uploadFile(Paths.get(backupfile.getBackupFile().getAbsolutePath()), Paths.get(backupfile.getRemotePath()), backupfile, 0, false); } catch (BackupRestoreException e) { // ignore } diff --git a/priam/src/test/java/com/netflix/priam/services/TestSnapshotMetaService.java b/priam/src/test/java/com/netflix/priam/services/TestSnapshotMetaService.java index 1e2d00851..78a484406 100644 --- a/priam/src/test/java/com/netflix/priam/services/TestSnapshotMetaService.java +++ b/priam/src/test/java/com/netflix/priam/services/TestSnapshotMetaService.java @@ -24,6 +24,7 @@ import com.netflix.priam.backupv2.*; import com.netflix.priam.config.IBackupRestoreConfig; import com.netflix.priam.scheduler.TaskTimer; +import com.netflix.priam.utils.BackupFileUtils; import com.netflix.priam.utils.DateUtil; import org.apache.cassandra.io.sstable.Component; import org.apache.commons.io.FileUtils; @@ -72,7 +73,7 @@ public void setUp() { prefixGenerator = injector.getInstance(PrefixGenerator.class); dummyDataDirectoryLocation = Paths.get(configuration.getDataFileLocation()); - cleanupDir(dummyDataDirectoryLocation); + BackupFileUtils.cleanupDir(dummyDataDirectoryLocation); } @@ -100,7 +101,7 @@ public void testMetaFileName() throws Exception { private void test(int noOfSstables, int noOfKeyspaces, int noOfCf) throws Exception{ Instant snapshotInstant = DateUtil.getInstant(); String snapshotName = snapshotMetaService.generateSnapshotName(snapshotInstant); - generateDummyFiles(dummyDataDirectoryLocation, noOfKeyspaces, noOfCf, noOfSstables, AbstractBackup.SNAPSHOT_FOLDER, snapshotName); + BackupFileUtils.generateDummyFiles(dummyDataDirectoryLocation, noOfKeyspaces, noOfCf, noOfSstables, AbstractBackup.SNAPSHOT_FOLDER, snapshotName); snapshotMetaService.setSnapshotName(snapshotName); Path metaFileLocation = snapshotMetaService.processSnapshot(snapshotInstant).getMetaFilePath(); Assert.assertNotNull(metaFileLocation); @@ -119,7 +120,7 @@ private void test(int noOfSstables, int noOfKeyspaces, int noOfCf) throws Except //Cleanup metaFileLocation.toFile().delete(); - cleanupDir(dummyDataDirectoryLocation); + BackupFileUtils.cleanupDir(dummyDataDirectoryLocation); } @Test @@ -127,54 +128,11 @@ public void testMetaFile() throws Exception { test(5, 1,1); } - private void cleanupDir(Path dir){ - if (dir.toFile().exists()) - try { - FileUtils.cleanDirectory(dir.toFile()); - } catch (IOException e) { - e.printStackTrace(); - } - } - @Test public void testSize() throws Exception { test (1000, 2,2); } - private void generateDummyFiles(Path dummyDir, int noOfKeyspaces, int noOfCf, int noOfSstables, String backupDir, String snapshotName) throws Exception { - if (dummyDir == null) - dummyDir = dummyDataDirectoryLocation; - - //Clean the dummy directory - if (dummyDir.toFile().exists()) - FileUtils.cleanDirectory(dummyDir.toFile()); - - for (int i = 1; i <= noOfKeyspaces; i++) { - String keyspaceName = "sample" + i; - - for (int j = 1; j <= noOfCf; j++) { - String columnfamilyname = "cf" + j; - - for (int k = 1; k <= noOfSstables; k++) { - String prefixName = "mc-" + k + "-big"; - - for (Component.Type type : EnumSet.allOf(Component.Type.class)) { - Path componentPath = Paths.get(dummyDir.toFile().getAbsolutePath(), keyspaceName, columnfamilyname, backupDir, snapshotName, prefixName + "-" + type.name() + ".db"); - componentPath.getParent().toFile().mkdirs(); - try (FileWriter fileWriter = new FileWriter(componentPath.toFile())) { - fileWriter.write(""); - } - - } - } - - Path componentPath = Paths.get(dummyDir.toFile().getAbsolutePath(), keyspaceName, columnfamilyname, backupDir, snapshotName, "manifest.json"); - try(FileWriter fileWriter = new FileWriter(componentPath.toFile())){ - fileWriter.write(""); - } - } - } - } public static class TestMetaFileReader extends MetaFileReader { diff --git a/priam/src/test/java/com/netflix/priam/utils/BackupFileUtils.java b/priam/src/test/java/com/netflix/priam/utils/BackupFileUtils.java new file mode 100644 index 000000000..3f7fdcb4d --- /dev/null +++ b/priam/src/test/java/com/netflix/priam/utils/BackupFileUtils.java @@ -0,0 +1,72 @@ +/* + * Copyright 2018 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.netflix.priam.utils; + +import org.apache.cassandra.io.sstable.Component; +import org.apache.commons.io.FileUtils; + +import java.io.FileWriter; +import java.io.IOException; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.EnumSet; + +/** + * Created by aagrawal on 9/23/18. + */ +public class BackupFileUtils { + public static void cleanupDir(Path dir){ + if (dir.toFile().exists()) + try { + FileUtils.cleanDirectory(dir.toFile()); + } catch (IOException e) { + e.printStackTrace(); + } + } + + public static void generateDummyFiles(Path dummyDir, int noOfKeyspaces, int noOfCf, int noOfSstables, String backupDir, String snapshotName) throws Exception { + //Clean the dummy directory + cleanupDir(dummyDir); + + for (int i = 1; i <= noOfKeyspaces; i++) { + String keyspaceName = "sample" + i; + + for (int j = 1; j <= noOfCf; j++) { + String columnfamilyname = "cf" + j; + + for (int k = 1; k <= noOfSstables; k++) { + String prefixName = "mc-" + k + "-big"; + + for (Component.Type type : EnumSet.allOf(Component.Type.class)) { + Path componentPath = Paths.get(dummyDir.toFile().getAbsolutePath(), keyspaceName, columnfamilyname, backupDir, snapshotName, prefixName + "-" + type.name() + ".db"); + componentPath.getParent().toFile().mkdirs(); + try (FileWriter fileWriter = new FileWriter(componentPath.toFile())) { + fileWriter.write(""); + } + + } + } + + Path componentPath = Paths.get(dummyDir.toFile().getAbsolutePath(), keyspaceName, columnfamilyname, backupDir, snapshotName, "manifest.json"); + try(FileWriter fileWriter = new FileWriter(componentPath.toFile())){ + fileWriter.write(""); + } + } + } + } +} From 34dd8e348debde5d3e445f91262b22bf8a1858b3 Mon Sep 17 00:00:00 2001 From: Arun Agrawal Date: Tue, 25 Sep 2018 20:28:35 -0700 Subject: [PATCH 014/228] Finish async snapshots and new path for backup and downloads The old incremental path is removed and unified with the snapshot path --- .../java/com/netflix/priam/PriamServer.java | 8 +- .../com/netflix/priam/aws/AWSMembership.java | 22 +- .../com/netflix/priam/aws/S3BackupPath.java | 1 - .../priam/aws/S3EncryptedFileSystem.java | 6 +- .../com/netflix/priam/aws/S3FileSystem.java | 1 - .../netflix/priam/aws/S3FileSystemBase.java | 18 +- .../netflix/priam/aws/SDBInstanceData.java | 15 +- .../netflix/priam/aws/SDBInstanceFactory.java | 2 +- .../priam/aws/UpdateSecuritySettings.java | 2 +- .../priam/aws/auth/S3InstanceCredential.java | 2 +- .../netflix/priam/backup/AbstractBackup.java | 43 ++- .../priam/backup/AbstractFileSystem.java | 32 +- .../priam/backup/BackupRestoreUtil.java | 171 +++------ .../priam/backup/BackupVerification.java | 1 - .../netflix/priam/backup/CommitLogBackup.java | 1 - .../priam/backup/CommitLogBackupTask.java | 4 +- .../priam/backup/IBackupFileSystem.java | 26 +- .../priam/backup/IncrementalBackup.java | 8 +- .../com/netflix/priam/backup/MetaData.java | 5 +- .../priam/backup/RangeReadInputStream.java | 5 +- .../netflix/priam/backup/SnapshotBackup.java | 55 ++- .../BackupPostProcessingCallback.java | 24 -- .../parallel/CassandraBackupQueueMgr.java | 107 ------ .../priam/backup/parallel/ITaskQueueMgr.java | 62 --- .../parallel/IncrementalBackupProducer.java | 151 -------- .../IncrementalBkupPostProcessing.java | 33 -- .../backup/parallel/IncrementalConsumer.java | 78 ---- .../parallel/IncrementalConsumerMgr.java | 121 ------ .../priam/backupv2/FileUploadResult.java | 3 +- .../priam/backupv2/LocalDBReaderWriter.java | 104 +++--- .../priam/backupv2/MetaFileManager.java | 2 +- .../priam/backupv2/MetaFileWriterBuilder.java | 2 - .../com/netflix/priam/cli/Application.java | 2 +- .../priam/cluster/management/Compaction.java | 45 +-- .../priam/cluster/management/Flush.java | 7 +- .../priam/config/IBackupRestoreConfig.java | 2 +- .../netflix/priam/config/IConfiguration.java | 352 +++++++++++++----- .../priam/config/PriamConfiguration.java | 144 +++---- .../configSource/SimpleDBConfigSource.java | 4 +- .../priam/cryptography/pgp/PgpUtil.java | 18 +- .../defaultimpl/CassandraProcessManager.java | 2 +- .../defaultimpl/InjectedWebListener.java | 4 +- .../priam/defaultimpl/PriamGuiceModule.java | 3 - .../google/GoogleEncryptedFileSystem.java | 11 +- .../priam/identity/InstanceIdentity.java | 7 +- .../identity/token/DeadTokenRetriever.java | 6 +- .../netflix/priam/merics/BackupMetrics.java | 5 +- .../priam/resources/BackupServlet.java | 26 +- .../priam/resources/CassandraConfig.java | 2 +- .../priam/resources/RestoreServlet.java | 13 +- .../priam/restore/AbstractRestore.java | 85 +++-- .../priam/restore/EncryptedRestoreBase.java | 34 +- .../com/netflix/priam/restore/Restore.java | 33 +- .../priam/restore/RestoreTokenSelector.java | 2 +- .../scheduler/NamedThreadPoolExecutor.java | 2 +- .../priam/scheduler/PriamScheduler.java | 18 +- .../com/netflix/priam/scheduler/Task.java | 4 - .../priam/services/SnapshotMetaService.java | 8 +- .../netflix/priam/tuner/StandardTuner.java | 6 +- .../com/netflix/priam/utils/FifoQueue.java | 7 +- .../com/netflix/priam/utils/JMXNodeTool.java | 12 +- .../com/netflix/priam/utils/SystemUtils.java | 13 +- .../backup/TestBackupRestoreUtil.groovy | 65 ++-- .../priam/backup/FakeBackupFileSystem.java | 2 - .../priam/backup/TestAbstractFileSystem.java | 39 +- .../netflix/priam/backup/TestCompression.java | 59 ++- .../priam/backup/TestSnapshotStatusMgr.java | 2 +- .../backupv2/TestLocalDBReaderWriter.java | 28 +- .../priam/config/FakeConfiguration.java | 227 ++--------- .../priam/resources/BackupServletTest.java | 8 +- .../services/TestSnapshotMetaService.java | 5 - 71 files changed, 844 insertions(+), 1583 deletions(-) delete mode 100644 priam/src/main/java/com/netflix/priam/backup/parallel/BackupPostProcessingCallback.java delete mode 100644 priam/src/main/java/com/netflix/priam/backup/parallel/CassandraBackupQueueMgr.java delete mode 100644 priam/src/main/java/com/netflix/priam/backup/parallel/ITaskQueueMgr.java delete mode 100644 priam/src/main/java/com/netflix/priam/backup/parallel/IncrementalBackupProducer.java delete mode 100644 priam/src/main/java/com/netflix/priam/backup/parallel/IncrementalBkupPostProcessing.java delete mode 100644 priam/src/main/java/com/netflix/priam/backup/parallel/IncrementalConsumer.java delete mode 100644 priam/src/main/java/com/netflix/priam/backup/parallel/IncrementalConsumerMgr.java diff --git a/priam/src/main/java/com/netflix/priam/PriamServer.java b/priam/src/main/java/com/netflix/priam/PriamServer.java index 7840ff649..7d4e0365b 100644 --- a/priam/src/main/java/com/netflix/priam/PriamServer.java +++ b/priam/src/main/java/com/netflix/priam/PriamServer.java @@ -23,7 +23,6 @@ import com.netflix.priam.backup.CommitLogBackupTask; import com.netflix.priam.backup.IncrementalBackup; import com.netflix.priam.backup.SnapshotBackup; -import com.netflix.priam.backup.parallel.IncrementalBackupProducer; import com.netflix.priam.cluster.management.Compaction; import com.netflix.priam.cluster.management.Flush; import com.netflix.priam.cluster.management.IClusterManagement; @@ -98,13 +97,8 @@ else if (UpdateSecuritySettings.firstTimeUpdated) // Start the Incremental backup schedule if enabled if (config.isIncrBackup()) { - if (!config.isIncrBackupParallelEnabled()) { scheduler.addTask(IncrementalBackup.JOBNAME, IncrementalBackup.class, IncrementalBackup.getTimer()); - logger.info("Added incremental synchronous bkup"); - } else { - scheduler.addTask(IncrementalBackupProducer.JOBNAME, IncrementalBackupProducer.class, IncrementalBackupProducer.getTimer()); - logger.info("Added incremental async-synchronous bkup, next fired time: {}", IncrementalBackupProducer.getTimer().getTrigger().getNextFireTime()); - } + logger.info("Added incremental backup job"); } } diff --git a/priam/src/main/java/com/netflix/priam/aws/AWSMembership.java b/priam/src/main/java/com/netflix/priam/aws/AWSMembership.java index 66c4226e4..4c9d9dd1b 100644 --- a/priam/src/main/java/com/netflix/priam/aws/AWSMembership.java +++ b/priam/src/main/java/com/netflix/priam/aws/AWSMembership.java @@ -17,11 +17,11 @@ package com.netflix.priam.aws; import com.amazonaws.services.autoscaling.AmazonAutoScaling; -import com.amazonaws.services.autoscaling.AmazonAutoScalingClient; +import com.amazonaws.services.autoscaling.AmazonAutoScalingClientBuilder; import com.amazonaws.services.autoscaling.model.*; import com.amazonaws.services.autoscaling.model.Instance; import com.amazonaws.services.ec2.AmazonEC2; -import com.amazonaws.services.ec2.AmazonEC2Client; +import com.amazonaws.services.ec2.AmazonEC2ClientBuilder; import com.amazonaws.services.ec2.model.*; import com.amazonaws.services.ec2.model.Filter; import com.google.common.collect.Lists; @@ -149,7 +149,7 @@ public void addACL(Collection listIPs, int from, int to) { AmazonEC2 client = null; try { client = getEc2Client(); - List ipPermissions = new ArrayList(); + List ipPermissions = new ArrayList<>(); ipPermissions.add(new IpPermission().withFromPort(from).withIpProtocol("tcp").withIpRanges(listIPs).withToPort(to)); if (this.insEnvIdentity.isClassic()) { @@ -204,7 +204,7 @@ public void removeACL(Collection listIPs, int from, int to) { AmazonEC2 client = null; try { client = getEc2Client(); - List ipPermissions = new ArrayList(); + List ipPermissions = new ArrayList<>(); ipPermissions.add(new IpPermission().withFromPort(from).withIpProtocol("tcp").withIpRanges(listIPs).withToPort(to)); if (this.insEnvIdentity.isClassic()) { @@ -235,7 +235,7 @@ public List listACL(int from, int to) { AmazonEC2 client = null; try { client = getEc2Client(); - List ipPermissions = new ArrayList(); + List ipPermissions = new ArrayList<>(); if (this.insEnvIdentity.isClassic()) { @@ -295,20 +295,14 @@ public void expandRacMembership(int count) { } protected AmazonAutoScaling getAutoScalingClient() { - AmazonAutoScaling client = new AmazonAutoScalingClient(provider.getAwsCredentialProvider()); - client.setEndpoint("autoscaling." + config.getDC() + ".amazonaws.com"); - return client; + return AmazonAutoScalingClientBuilder.standard().withCredentials(provider.getAwsCredentialProvider()).withRegion(config.getDC()).build(); } protected AmazonAutoScaling getCrossAccountAutoScalingClient() { - AmazonAutoScaling client = new AmazonAutoScalingClient(crossAccountProvider.getAwsCredentialProvider()); - client.setEndpoint("autoscaling." + config.getDC() + ".amazonaws.com"); - return client; + return AmazonAutoScalingClientBuilder.standard().withCredentials(crossAccountProvider.getAwsCredentialProvider()).withRegion(config.getDC()).build(); } protected AmazonEC2 getEc2Client() { - AmazonEC2 client = new AmazonEC2Client(provider.getAwsCredentialProvider()); - client.setEndpoint("ec2." + config.getDC() + ".amazonaws.com"); - return client; + return AmazonEC2ClientBuilder.standard().withCredentials(provider.getAwsCredentialProvider()).withRegion(config.getDC()).build(); } } \ No newline at end of file diff --git a/priam/src/main/java/com/netflix/priam/aws/S3BackupPath.java b/priam/src/main/java/com/netflix/priam/aws/S3BackupPath.java index 90ecda6ee..7d1402baf 100644 --- a/priam/src/main/java/com/netflix/priam/aws/S3BackupPath.java +++ b/priam/src/main/java/com/netflix/priam/aws/S3BackupPath.java @@ -37,7 +37,6 @@ public S3BackupPath(IConfiguration config, InstanceIdentity factory) { /** * Format of backup path: - * Cassandra > 1.1 * BASE/REGION/CLUSTER/TOKEN/[SNAPSHOTTIME]/[SST|SNP|META]/KEYSPACE/COLUMNFAMILY/FILE */ @Override diff --git a/priam/src/main/java/com/netflix/priam/aws/S3EncryptedFileSystem.java b/priam/src/main/java/com/netflix/priam/aws/S3EncryptedFileSystem.java index 354e502bb..be4043001 100755 --- a/priam/src/main/java/com/netflix/priam/aws/S3EncryptedFileSystem.java +++ b/priam/src/main/java/com/netflix/priam/aws/S3EncryptedFileSystem.java @@ -42,7 +42,6 @@ import java.nio.file.Path; import java.util.Iterator; import java.util.List; -import java.util.concurrent.atomic.AtomicInteger; /** * Implementation of IBackupFileSystem for S3. The upload/download will work with ciphertext. @@ -51,8 +50,7 @@ public class S3EncryptedFileSystem extends S3FileSystemBase { private static final Logger logger = LoggerFactory.getLogger(S3EncryptedFileSystem.class); - private AtomicInteger uploadCount = new AtomicInteger(); - private IFileCryptography encryptor; + private final IFileCryptography encryptor; @Inject public S3EncryptedFileSystem(Provider pathProvider, ICompression compress, final IConfiguration config, ICredential cred @@ -70,7 +68,7 @@ public S3EncryptedFileSystem(Provider pathProvider, ICompres @Override protected void downloadFileImpl(Path remotePath, Path localPath) throws BackupRestoreException { try (OutputStream os = new FileOutputStream(localPath.toFile()); - RangeReadInputStream rris = new RangeReadInputStream(s3Client, getPrefix(config), super.getFileSize(remotePath), remotePath.toString()); + RangeReadInputStream rris = new RangeReadInputStream(s3Client, getPrefix(config), super.getFileSize(remotePath), remotePath.toString()) ) { /* * To handle use cases where decompression should be done outside of the download. For example, the file have been compressed and then encrypted. diff --git a/priam/src/main/java/com/netflix/priam/aws/S3FileSystem.java b/priam/src/main/java/com/netflix/priam/aws/S3FileSystem.java index e1d41c7a4..172406f9f 100644 --- a/priam/src/main/java/com/netflix/priam/aws/S3FileSystem.java +++ b/priam/src/main/java/com/netflix/priam/aws/S3FileSystem.java @@ -150,7 +150,6 @@ protected long uploadFileImpl(Path localPath, Path remotePath) throws BackupRest if (fileSize < chunkSize) { //Upload file without using multipart upload as it will be more efficient. - if (logger.isDebugEnabled()) logger.debug("Uploading to {}/{} using PUT operation", config.getBackupPrefix(), remotePath); diff --git a/priam/src/main/java/com/netflix/priam/aws/S3FileSystemBase.java b/priam/src/main/java/com/netflix/priam/aws/S3FileSystemBase.java index 2a1936ce6..e3fc34701 100755 --- a/priam/src/main/java/com/netflix/priam/aws/S3FileSystemBase.java +++ b/priam/src/main/java/com/netflix/priam/aws/S3FileSystemBase.java @@ -46,11 +46,11 @@ public abstract class S3FileSystemBase extends AbstractFileSystem { protected static final long UPLOAD_TIMEOUT = (2 * 60 * 60 * 1000L); private static final Logger logger = LoggerFactory.getLogger(S3FileSystemBase.class); protected AmazonS3 s3Client; - protected IConfiguration config; - protected Provider pathProvider; - protected ICompression compress; - protected BlockingSubmitThreadPoolExecutor executor; - protected RateLimiter rateLimiter; //a throttling mechanism, we can limit the amount of bytes uploaded to endpoint per second. + protected final IConfiguration config; + protected final Provider pathProvider; + protected final ICompression compress; + protected final BlockingSubmitThreadPoolExecutor executor; + protected final RateLimiter rateLimiter; //a throttling mechanism, we can limit the amount of bytes uploaded to endpoint per second. public S3FileSystemBase(Provider pathProvider, ICompression compress, @@ -62,8 +62,8 @@ public S3FileSystemBase(Provider pathProvider, this.compress = compress; this.config = config; - int threads = config.getMaxBackupUploadThreads(); - LinkedBlockingQueue queue = new LinkedBlockingQueue(threads); + int threads = config.getBackupThreads(); + LinkedBlockingQueue queue = new LinkedBlockingQueue<>(threads); this.executor = new BlockingSubmitThreadPoolExecutor(threads, queue, UPLOAD_TIMEOUT); double throttleLimit = config.getUploadThrottle(); @@ -160,7 +160,7 @@ protected void checkSuccessfulUpload(CompleteMultipartUploadResult resultS3Multi } @Override - public long getFileSize(Path remotePath) throws BackupRestoreException { + public long getFileSize(Path remotePath) throws BackupRestoreException{ return s3Client.getObjectMetadata(getPrefix(config), remotePath.toString()).getContentLength(); } @@ -181,7 +181,7 @@ public Iterator list(String path, Date start, Date till) { return new S3FileIterator(pathProvider, s3Client, path, start, till); } - final long getChunkSize(Path localPath) throws BackupRestoreException { + final long getChunkSize(Path localPath) throws BackupRestoreException{ long chunkSize = config.getBackupChunkSize(); long fileSize = localPath.toFile().length(); diff --git a/priam/src/main/java/com/netflix/priam/aws/SDBInstanceData.java b/priam/src/main/java/com/netflix/priam/aws/SDBInstanceData.java index 128042435..6c07c7bd8 100644 --- a/priam/src/main/java/com/netflix/priam/aws/SDBInstanceData.java +++ b/priam/src/main/java/com/netflix/priam/aws/SDBInstanceData.java @@ -82,16 +82,15 @@ public PriamInstance getInstance(String app, String dc, int id) { */ public Set getAllIds(String app) { AmazonSimpleDB simpleDBClient = getSimpleDBClient(); - Set inslist = new HashSet(); + Set inslist = new HashSet<>(); String nextToken = null; do { SelectRequest request = new SelectRequest(String.format(ALL_QUERY, app)); request.setNextToken(nextToken); SelectResult result = simpleDBClient.select(request); nextToken = result.getNextToken(); - Iterator itemiter = result.getItems().iterator(); - while (itemiter.hasNext()) { - inslist.add(transform(itemiter.next())); + for (Item item : result.getItems()) { + inslist.add(transform(item)); } } while (nextToken != null); @@ -140,7 +139,7 @@ public void deregisterInstance(PriamInstance instance) throws AmazonServiceExcep protected List createAttributesToRegister(PriamInstance instance) { instance.setUpdatetime(new Date().getTime()); - List attrs = new ArrayList(); + List attrs = new ArrayList<>(); attrs.add(new ReplaceableAttribute(Attributes.INSTANCE_ID, instance.getInstanceId(), false)); attrs.add(new ReplaceableAttribute(Attributes.TOKEN, instance.getToken(), true)); attrs.add(new ReplaceableAttribute(Attributes.APP_ID, instance.getApp(), true)); @@ -154,7 +153,7 @@ protected List createAttributesToRegister(PriamInstance in } protected List createAttributesToDeRegister(PriamInstance instance) { - List attrs = new ArrayList(); + List attrs = new ArrayList<>(); attrs.add(new Attribute(Attributes.INSTANCE_ID, instance.getInstanceId())); attrs.add(new Attribute(Attributes.TOKEN, instance.getToken())); attrs.add(new Attribute(Attributes.APP_ID, instance.getApp())); @@ -175,9 +174,7 @@ protected List createAttributesToDeRegister(PriamInstance instance) { */ public PriamInstance transform(Item item) { PriamInstance ins = new PriamInstance(); - Iterator attrs = item.getAttributes().iterator(); - while (attrs.hasNext()) { - Attribute att = attrs.next(); + for (Attribute att : item.getAttributes()) { if (att.getName().equals(Attributes.INSTANCE_ID)) ins.setInstanceId(att.getValue()); else if (att.getName().equals(Attributes.TOKEN)) diff --git a/priam/src/main/java/com/netflix/priam/aws/SDBInstanceFactory.java b/priam/src/main/java/com/netflix/priam/aws/SDBInstanceFactory.java index a10cd5878..b6e1228e3 100644 --- a/priam/src/main/java/com/netflix/priam/aws/SDBInstanceFactory.java +++ b/priam/src/main/java/com/netflix/priam/aws/SDBInstanceFactory.java @@ -46,7 +46,7 @@ public SDBInstanceFactory(IConfiguration config, SDBInstanceData dao) { @Override public List getAllIds(String appName) { - List return_ = new ArrayList(); + List return_ = new ArrayList<>(); for (PriamInstance instance : dao.getAllIds(appName)) { return_.add(instance); } diff --git a/priam/src/main/java/com/netflix/priam/aws/UpdateSecuritySettings.java b/priam/src/main/java/com/netflix/priam/aws/UpdateSecuritySettings.java index 21c407223..3de271198 100644 --- a/priam/src/main/java/com/netflix/priam/aws/UpdateSecuritySettings.java +++ b/priam/src/main/java/com/netflix/priam/aws/UpdateSecuritySettings.java @@ -80,7 +80,7 @@ public void execute() { List instances = factory.getAllIds(config.getAppName()); // iterate to add... - Set add = new HashSet(); + Set add = new HashSet<>(); List allInstances = factory.getAllIds(config.getAppName()); for (PriamInstance instance : allInstances) { String range = instance.getHostIP() + "/32"; diff --git a/priam/src/main/java/com/netflix/priam/aws/auth/S3InstanceCredential.java b/priam/src/main/java/com/netflix/priam/aws/auth/S3InstanceCredential.java index 09f172ef1..0ef0fc0f9 100755 --- a/priam/src/main/java/com/netflix/priam/aws/auth/S3InstanceCredential.java +++ b/priam/src/main/java/com/netflix/priam/aws/auth/S3InstanceCredential.java @@ -27,7 +27,7 @@ public class S3InstanceCredential implements IS3Credential { private InstanceProfileCredentialsProvider credentialsProvider; public S3InstanceCredential() { - this.credentialsProvider = new InstanceProfileCredentialsProvider(); + this.credentialsProvider = InstanceProfileCredentialsProvider.getInstance(); } @Override diff --git a/priam/src/main/java/com/netflix/priam/backup/AbstractBackup.java b/priam/src/main/java/com/netflix/priam/backup/AbstractBackup.java index e0a167f44..c923d38e3 100644 --- a/priam/src/main/java/com/netflix/priam/backup/AbstractBackup.java +++ b/priam/src/main/java/com/netflix/priam/backup/AbstractBackup.java @@ -22,14 +22,15 @@ import com.netflix.priam.config.IConfiguration; import com.netflix.priam.backup.AbstractBackupPath.BackupFileType; import com.netflix.priam.scheduler.Task; -import com.netflix.priam.utils.RetryableCallable; import com.netflix.priam.utils.SystemUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.File; +import java.nio.file.Path; import java.nio.file.Paths; import java.util.List; +import java.util.concurrent.Future; /** * Abstract Backup class for uploading files to backup location @@ -58,25 +59,47 @@ protected void setFileSystem(IBackupFileSystem fs) { this.fs = fs; } + private AbstractBackupPath getAbstractBackupPath(final File file, final BackupFileType type) throws Exception { + final AbstractBackupPath bp = pathFactory.get(); + bp.parseLocal(file, type); + return bp; + } + + /** * Upload files in the specified dir. Does not delete the file in case of - * error. The files are uploaded serially. + * error. The files are uploaded serially or async based on flag provided. * * @param parent Parent dir * @param type Type of file (META, SST, SNAP etc) + * @param async Upload the file(s) in async fashion if enabled. * @return List of files that are successfully uploaded as part of backup * @throws Exception when there is failure in uploading files. */ - List upload(File parent, final BackupFileType type) throws Exception { + protected List upload(final File parent, final BackupFileType type, boolean async) throws Exception { final List bps = Lists.newArrayList(); - for (final File file : parent.listFiles()) { - //== decorate file with metadata - final AbstractBackupPath bp = pathFactory.get(); - bp.parseLocal(file, type); - fs.uploadFile(Paths.get(bp.getBackupFile().getAbsolutePath()), Paths.get(bp.getRemotePath()), bp, 10, true); - bps.add(bp); - addToRemotePath(bp.getRemotePath()); + final List> futures = Lists.newArrayList(); + + for (File file : parent.listFiles()) { + if (file.isFile() && file.exists()) { + AbstractBackupPath bp = getAbstractBackupPath(file, type); + + if (async) + futures.add(fs.asyncUploadFile(Paths.get(bp.getBackupFile().getAbsolutePath()), Paths.get(bp.getRemotePath()), bp, 10, true)); + else + fs.uploadFile(Paths.get(bp.getBackupFile().getAbsolutePath()), Paths.get(bp.getRemotePath()), bp, 10, true); + + bps.add(bp); + addToRemotePath(bp.getRemotePath()); + } + } + + //Wait for all files to be uploaded. + if (async) { + for (Future future : futures) + future.get(); //This might throw exception if there is any error } + return bps; } diff --git a/priam/src/main/java/com/netflix/priam/backup/AbstractFileSystem.java b/priam/src/main/java/com/netflix/priam/backup/AbstractFileSystem.java index 3f9f914c2..bdfd8e1e7 100644 --- a/priam/src/main/java/com/netflix/priam/backup/AbstractFileSystem.java +++ b/priam/src/main/java/com/netflix/priam/backup/AbstractFileSystem.java @@ -47,7 +47,6 @@ public abstract class AbstractFileSystem implements IBackupFileSystem, EventGene private Set tasksQueued; private ThreadPoolExecutor fileUploadExecutor; private ThreadPoolExecutor fileDownloadExecutor; - private static final long UPLOAD_TIMEOUT = (2 * 60 * 60 * 1000L); //2 minutes. @Inject public AbstractFileSystem(IConfiguration configuration, BackupMetrics backupMetrics, @@ -55,15 +54,23 @@ public AbstractFileSystem(IConfiguration configuration, BackupMetrics backupMetr this.backupMetrics = backupMetrics; //Add notifications. this.addObserver(backupNotificationMgr); - tasksQueued = new HashSet<>(configuration.getUncrementalBkupQueueSize()); - BlockingQueue queue = new ArrayBlockingQueue(configuration.getUncrementalBkupQueueSize()); - PolledMeter.using(backupMetrics.getRegistry()).withName(backupMetrics.uploadDownloadQueueSize).monitorSize(queue); - this.fileUploadExecutor = new BlockingSubmitThreadPoolExecutor(configuration.getMaxBackupUploadThreads(), queue, UPLOAD_TIMEOUT); - this.fileDownloadExecutor = new BlockingSubmitThreadPoolExecutor(configuration.getMaxBackupDownloadThreads(), queue, UPLOAD_TIMEOUT); + tasksQueued = new HashSet<>(configuration.getBackupQueueSize()); + /* + Note: We are using different queue for upload and download as with Backup V2.0 we might download all the meta + files for "sync" feature which might compete with backups for scheduling. + Also, we may want to have different TIMEOUT for each kind of operation (upload/download) based on our file system choices. + */ + BlockingQueue uploadQueue = new ArrayBlockingQueue<>(configuration.getBackupQueueSize()); + PolledMeter.using(backupMetrics.getRegistry()).withName(backupMetrics.uploadQueueSize).monitorSize(uploadQueue); + this.fileUploadExecutor = new BlockingSubmitThreadPoolExecutor(configuration.getBackupThreads(), uploadQueue, configuration.getUploadTimeout()); + + BlockingQueue downloadQueue = new ArrayBlockingQueue<>(configuration.getDownloadQueueSize()); + PolledMeter.using(backupMetrics.getRegistry()).withName(backupMetrics.downloadQueueSize).monitorSize(downloadQueue); + this.fileDownloadExecutor = new BlockingSubmitThreadPoolExecutor(configuration.getRestoreThreads(), downloadQueue, configuration.getDownloadTimeout()); } @Override - public Future downloadFileAsync(final Path remotePath, final Path localPath, final int retry) throws BackupRestoreException, RejectedExecutionException { + public Future asyncDownloadFile(final Path remotePath, final Path localPath, final int retry) throws BackupRestoreException, RejectedExecutionException { return fileDownloadExecutor.submit(() -> { downloadFile(remotePath, localPath, retry); return remotePath; @@ -109,8 +116,8 @@ public Future asyncUploadFile(final Path localPath, final Path remotePath, @Override public void uploadFile(final Path localPath, final Path remotePath, final AbstractBackupPath path, final int retry, final boolean deleteAfterSuccessfulUpload) throws FileNotFoundException, BackupRestoreException { - if (!localPath.toFile().exists() || localPath.toFile().isDirectory()) - throw new FileNotFoundException("File do not exist or is a directory: {}" + localPath); + if (localPath == null || remotePath == null || !localPath.toFile().exists() || localPath.toFile().isDirectory()) + throw new FileNotFoundException("File do not exist or is a directory. localPath: " + localPath + ", remotePath: " + remotePath); if (!tasksQueued.contains(localPath)) { //Add file to local memory @@ -187,7 +194,12 @@ public void notifyEventStop(BackupEvent event) { } @Override - public int getTasksQueued(){ + public int getUploadTasksQueued(){ return tasksQueued.size(); } + + @Override + public int getDownloadTasksQueued(){ + return fileDownloadExecutor.getQueue().size(); + } } diff --git a/priam/src/main/java/com/netflix/priam/backup/BackupRestoreUtil.java b/priam/src/main/java/com/netflix/priam/backup/BackupRestoreUtil.java index 37fcc9179..5c33d6725 100644 --- a/priam/src/main/java/com/netflix/priam/backup/BackupRestoreUtil.java +++ b/priam/src/main/java/com/netflix/priam/backup/BackupRestoreUtil.java @@ -24,7 +24,6 @@ import org.slf4j.LoggerFactory; import java.util.*; -import java.util.regex.Matcher; import java.util.regex.Pattern; /** @@ -32,103 +31,64 @@ */ public class BackupRestoreUtil { private static final Logger logger = LoggerFactory.getLogger(BackupRestoreUtil.class); - private static final String JOBNAME = "BackupRestoreUtil"; - - private final Map> columnFamilyFilter = new HashMap<>(); //key: keyspace, value: a list of CFs within the keyspace - private final Map keyspaceFilter = new HashMap<>(); //key: keyspace, value: null - - private final Pattern columnFamilyFilterPattern = Pattern.compile(".\\.."); - private String configKeyspaceFilter; - private String configColumnfamilyFilter; + private static Pattern columnFamilyFilterPattern = Pattern.compile(".\\.."); + private Map> includeFilter; + private Map> excludeFilter; public static final List FILTER_KEYSPACE = Arrays.asList("OpsCenter"); - private static final Map> FILTER_COLUMN_FAMILY = ImmutableMap.of("system", Arrays.asList("local", "peers", "compactions_in_progress", "LocationInfo")); + private static final Map> FILTER_COLUMN_FAMILY = ImmutableMap.of("system", Arrays.asList("local", "peers", "hints", "compactions_in_progress", "LocationInfo")); @Inject - public BackupRestoreUtil(String configKeyspaceFilter, String configColumnfamilyFilter) { - setFilters(configKeyspaceFilter, configColumnfamilyFilter); + public BackupRestoreUtil(String configIncludeFilter, String configExcludeFilter) { + setFilters(configIncludeFilter, configExcludeFilter); } - private BackupRestoreUtil setFilters(String configKeyspaceFilter, String configColumnfamilyFilter) { - this.configColumnfamilyFilter = configColumnfamilyFilter; - this.configKeyspaceFilter = configKeyspaceFilter; - populateFilters(); + public BackupRestoreUtil setFilters(String configIncludeFilter, String configExcludeFilter) { + includeFilter = getFilter(configIncludeFilter); + excludeFilter = getFilter(configExcludeFilter); + logger.info("Exclude filter set: {}", configExcludeFilter); + logger.info("Include filter set: {}", configIncludeFilter); return this; } - /** - * Search for "1:* alphanumeric chars including special chars""literal period"" 1:* alphanumeric chars including special chars" - * - * @param cfFilter input string - * @return true if input string matches search pattern; otherwise, false - */ - private final boolean isValidCFFilterFormat(String cfFilter) { - return columnFamilyFilterPattern.matcher(cfFilter).find(); - } - /** - * Populate the filters for backup/restore as configured for internal use. - */ - private final void populateFilters() { - //Clear the filters as we will (re)populate the filters. - keyspaceFilter.clear(); - columnFamilyFilter.clear(); - - if (configKeyspaceFilter == null || configKeyspaceFilter.isEmpty()) { - logger.info("No keyspace filter set for {}.", JOBNAME); - } else { - String[] keyspaces = configKeyspaceFilter.split(","); - for (int i = 0; i < keyspaces.length; i++) { - logger.info("Adding {} keyspace filter: {}", JOBNAME, keyspaces[i]); - this.keyspaceFilter.put(keyspaces[i], null); - } + public static final Map> getFilter(String inputFilter) throws IllegalArgumentException { + if (StringUtils.isEmpty(inputFilter)) + return null; - } - - if (configColumnfamilyFilter == null || configColumnfamilyFilter.isEmpty()) { - - logger.info("No column family filter set for {}.", JOBNAME); - - } else { + final Map> columnFamilyFilter = new HashMap<>(); //key: keyspace, value: a list of CFs within the keyspace - String[] filters = configColumnfamilyFilter.split(","); - for (int i = 0; i < filters.length; i++) { //process each filter - if (isValidCFFilterFormat(filters[i])) { + String[] filters = inputFilter.split(","); + for (int i = 0; i < filters.length; i++) { //process each filter + if (columnFamilyFilterPattern.matcher(filters[i]).find()) { - String[] filter = filters[i].split("\\."); - String ksName = filter[0]; - String cfName = filter[1]; - logger.info("Adding {} CF filter: {}.{}", JOBNAME, ksName, cfName); + String[] filter = filters[i].split("\\."); + String keyspaceName = filter[0]; + String columnFamilyName = filter[1]; - if (this.columnFamilyFilter.containsKey(ksName)) { - //add cf to existing filter - List columnfamilies = this.columnFamilyFilter.get(ksName); - columnfamilies.add(cfName); - this.columnFamilyFilter.put(ksName, columnfamilies); + if (columnFamilyName.contains("-")) + columnFamilyName = columnFamilyName.substring(0, columnFamilyName.indexOf("-")); - } else { - - List cfs = new ArrayList(); - cfs.add(cfName); - this.columnFamilyFilter.put(ksName, cfs); - - } - - } else { - throw new IllegalArgumentException("Column family filter format is not valid. Format needs to be \"keyspace.columnfamily\". Invalid input: " + filters[i]); - } - } //end processing each filter + List existingCfs = columnFamilyFilter.getOrDefault(keyspaceName, new ArrayList<>()); + if (!columnFamilyName.equalsIgnoreCase("*")) + existingCfs.add(columnFamilyName); + columnFamilyFilter.put(keyspaceName, existingCfs); + } else { + throw new IllegalArgumentException("Column family filter format is not valid. Format needs to be \"keyspace.columnfamily\". Invalid input: " + filters[i]); + } } + return columnFamilyFilter; } /** * Returns if provided keyspace and/or columnfamily is filtered for backup or restore. - * @param keyspace name of the keyspace in consideration + * + * @param keyspace name of the keyspace in consideration * @param columnFamilyDir name of the columnfamily directory in consideration * @return true if directory should be filter from processing; otherwise, false. */ - public final boolean isFiltered(String keyspace, String columnFamilyDir){ + public final boolean isFiltered(String keyspace, String columnFamilyDir) { if (StringUtils.isEmpty(keyspace) || StringUtils.isEmpty(columnFamilyDir)) return false; @@ -137,61 +97,20 @@ public final boolean isFiltered(String keyspace, String columnFamilyDir){ if (FILTER_COLUMN_FAMILY.containsKey(keyspace) && FILTER_COLUMN_FAMILY.get(keyspace).contains(columnFamilyName)) return true; - if (isFiltered(BackupRestoreUtil.DIRECTORYTYPE.KEYSPACE, keyspace) || //keyspace is filtered - isFiltered(BackupRestoreUtil.DIRECTORYTYPE.CF, keyspace, columnFamilyDir) //columnfamily is filtered - ) { - logger.debug("Skipping: keyspace: {}, CF: {} is part of filter list.", keyspace, columnFamilyName); - return true; - } - - return false; - } - - /** - * @param directoryType keyspace or columnfamily directory type. - * @return true if directory should be filter from processing; otherwise, false. - */ - private final boolean isFiltered(DIRECTORYTYPE directoryType, String... args) { - - if (directoryType.equals(DIRECTORYTYPE.KEYSPACE)) { //start with filtering the parent (keyspace) - //Apply each keyspace filter to input string - String keyspaceName = args[0]; - - java.util.Set ksFilters = keyspaceFilter.keySet(); - Iterator it = ksFilters.iterator(); - while (it.hasNext()) { - String ksFilter = it.next(); - Pattern pattern = Pattern.compile(ksFilter); - Matcher matcher = pattern.matcher(keyspaceName); - if (matcher.find()) { - logger.debug("Keyspace: {} matched filter: {}", keyspaceName, ksFilter); - return true; - } - } - } - - if (directoryType.equals(DIRECTORYTYPE.CF)) { //parent (keyspace) is not filtered, now see if the child (CF) is filtered - String keyspaceName = args[0]; - if (!columnFamilyFilter.containsKey(keyspaceName)) { - return false; + if (excludeFilter != null) + if (excludeFilter.containsKey(keyspace) && + (excludeFilter.get(keyspace).isEmpty() || excludeFilter.get(keyspace).contains(columnFamilyName))) { + logger.debug("Skipping: keyspace: {}, CF: {} is part of exclude list.", keyspace, columnFamilyName); + return true; } - String cfName = args[1]; - List cfsFilter = columnFamilyFilter.get(keyspaceName); - for (int i = 0; i < cfsFilter.size(); i++) { - Pattern pattern = Pattern.compile(cfsFilter.get(i)); - Matcher matcher = pattern.matcher(cfName); - if (matcher.find()) { - logger.debug("{}.{} matched filter", keyspaceName, cfName); - return true; - } + if (includeFilter != null) + if (!(includeFilter.containsKey(keyspace) && + (includeFilter.get(keyspace).isEmpty() || includeFilter.get(keyspace).contains(columnFamilyName)))) { + logger.debug("Skipping: keyspace: {}, CF: {} is not part of include list.", keyspace, columnFamilyName); + return true; } - } - - return false; //if here, current input are not part of keyspae and cf filters - } - public enum DIRECTORYTYPE { - KEYSPACE, CF + return false; } } diff --git a/priam/src/main/java/com/netflix/priam/backup/BackupVerification.java b/priam/src/main/java/com/netflix/priam/backup/BackupVerification.java index afecbcb6f..f3d976005 100644 --- a/priam/src/main/java/com/netflix/priam/backup/BackupVerification.java +++ b/priam/src/main/java/com/netflix/priam/backup/BackupVerification.java @@ -25,7 +25,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.FileOutputStream; import java.io.FileReader; import java.nio.file.FileSystems; import java.nio.file.Path; diff --git a/priam/src/main/java/com/netflix/priam/backup/CommitLogBackup.java b/priam/src/main/java/com/netflix/priam/backup/CommitLogBackup.java index 97c5dc1ac..9c9c5d86a 100644 --- a/priam/src/main/java/com/netflix/priam/backup/CommitLogBackup.java +++ b/priam/src/main/java/com/netflix/priam/backup/CommitLogBackup.java @@ -20,7 +20,6 @@ import com.google.inject.Provider; import com.google.inject.name.Named; import com.netflix.priam.backup.AbstractBackupPath.BackupFileType; -import com.netflix.priam.utils.RetryableCallable; import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/priam/src/main/java/com/netflix/priam/backup/CommitLogBackupTask.java b/priam/src/main/java/com/netflix/priam/backup/CommitLogBackupTask.java index 36dff0a5e..85950beeb 100644 --- a/priam/src/main/java/com/netflix/priam/backup/CommitLogBackupTask.java +++ b/priam/src/main/java/com/netflix/priam/backup/CommitLogBackupTask.java @@ -37,8 +37,8 @@ public class CommitLogBackupTask extends AbstractBackup { public static String JOBNAME = "CommitLogBackup"; private static final Logger logger = LoggerFactory.getLogger(CommitLogBackupTask.class); - private final List clRemotePaths = new ArrayList(); - private static List observers = new ArrayList(); + private final List clRemotePaths = new ArrayList<>(); + private static List observers = new ArrayList<>(); private final CommitLogBackup clBackup; diff --git a/priam/src/main/java/com/netflix/priam/backup/IBackupFileSystem.java b/priam/src/main/java/com/netflix/priam/backup/IBackupFileSystem.java index b1535a0e3..e4a3d8a6d 100644 --- a/priam/src/main/java/com/netflix/priam/backup/IBackupFileSystem.java +++ b/priam/src/main/java/com/netflix/priam/backup/IBackupFileSystem.java @@ -32,7 +32,7 @@ public interface IBackupFileSystem { * * @param remotePath fully qualified location of the file on remote file system. * @param localPath location on the local file sytem where remote file should be downloaded. - * @param retry No. of times to retry to download a file from remote file system. If <=0, it will try to download file exactly once. + * @param retry No. of times to retry to download a file from remote file system. If <1, it will try to download file exactly once. * @throws BackupRestoreException if file is not available, downloadable or any other error from remote file system. */ void downloadFile(Path remotePath, Path localPath, int retry) throws BackupRestoreException; @@ -42,12 +42,12 @@ public interface IBackupFileSystem { * * @param remotePath fully qualified location of the file on remote file system. * @param localPath location on the local file sytem where remote file should be downloaded. - * @param retry No. of times to retry to download a file from remote file system. If <=0, it will try to download file exactly once. + * @param retry No. of times to retry to download a file from remote file system. If <1, it will try to download file exactly once. * @return The future of the async job to monitor the progress of the job. * @throws BackupRestoreException if file is not available, downloadable or any other error from remote file system. * @throws RejectedExecutionException if the queue is full and TIMEOUT is reached while trying to add the work to the queue. */ - Future downloadFileAsync(final Path remotePath, final Path localPath, final int retry) throws BackupRestoreException, RejectedExecutionException; + Future asyncDownloadFile(final Path remotePath, final Path localPath, final int retry) throws BackupRestoreException, RejectedExecutionException; /** @@ -60,7 +60,7 @@ public interface IBackupFileSystem { * @param localPath Path of the local file that needs to be uploaded. * @param remotePath Fully qualified path on the remote file system where file should be uploaded. * @param path AbstractBackupPath to be used to send backup notifications only. - * @param retry No of times to retry to upload a file. If <=0, it will try to upload file exactly once. + * @param retry No of times to retry to upload a file. If <1, it will try to upload file exactly once. * @param deleteAfterSuccessfulUpload If true, delete the file denoted by localPath after it is successfully uploaded to the filesystem. If there is any failure, file will not be deleted. * @throws BackupRestoreException in case of failure to upload for any reason including file not readable or remote file system errors. * @throws FileNotFoundException If a file as denoted by localPath is not available or is a directory. @@ -73,11 +73,11 @@ public interface IBackupFileSystem { * @param localPath Path of the local file that needs to be uploaded. * @param remotePath Fully qualified path on the remote file system where file should be uploaded. * @param path AbstractBackupPath to be used to send backup notifications only. - * @param retry No of times to retry to upload a file. If <=0, it will try to upload file exactly once. + * @param retry No of times to retry to upload a file. If <1, it will try to upload file exactly once. * @param deleteAfterSuccessfulUpload If true, delete the file denoted by localPath after it is successfully uploaded to the filesystem. If there is any failure, file will not be deleted. * @return The future of the async job to monitor the progress of the job. This will be null if file was de-duped for upload. - * @throws BackupRestoreException in case of failure to upload for any reason including file not readable or remote file system errors. - * @throws FileNotFoundException If a file as denoted by localPath is not available or is a directory. + * @throws BackupRestoreException in case of failure to upload for any reason including file not readable or remote file system errors. + * @throws FileNotFoundException If a file as denoted by localPath is not available or is a directory. * @throws RejectedExecutionException if the queue is full and TIMEOUT is reached while trying to add the work to the queue. */ Future asyncUploadFile(final Path localPath, final Path remotePath, final AbstractBackupPath path, final int retry, final boolean deleteAfterSuccessfulUpload) throws FileNotFoundException, RejectedExecutionException, BackupRestoreException; @@ -117,8 +117,16 @@ public interface IBackupFileSystem { long getFileSize(Path remotePath) throws BackupRestoreException; /** - * Get the number of tasks en-queue in the filesystem for download/upload. + * Get the number of tasks en-queue in the filesystem for upload. + * + * @return the total no. of tasks to be executed. + */ + int getUploadTasksQueued(); + + /** + * Get the number of tasks en-queue in the filesystem for download. + * * @return the total no. of tasks to be executed. */ - int getTasksQueued(); + int getDownloadTasksQueued(); } diff --git a/priam/src/main/java/com/netflix/priam/backup/IncrementalBackup.java b/priam/src/main/java/com/netflix/priam/backup/IncrementalBackup.java index 388803220..ee570dff8 100644 --- a/priam/src/main/java/com/netflix/priam/backup/IncrementalBackup.java +++ b/priam/src/main/java/com/netflix/priam/backup/IncrementalBackup.java @@ -38,17 +38,17 @@ public class IncrementalBackup extends AbstractBackup implements IIncrementalBackup { private static final Logger logger = LoggerFactory.getLogger(IncrementalBackup.class); public static final String JOBNAME = "IncrementalBackup"; - private final List incrementalRemotePaths = new ArrayList(); + private final List incrementalRemotePaths = new ArrayList<>(); private IncrementalMetaData metaData; private BackupRestoreUtil backupRestoreUtil; - private static List observers = new ArrayList(); + private static List observers = new ArrayList<>(); @Inject public IncrementalBackup(IConfiguration config, Provider pathFactory, IFileSystemContext backupFileSystemCtx , IncrementalMetaData metaData) { super(config, backupFileSystemCtx, pathFactory); this.metaData = metaData; //a means to upload audit trail (via meta_cf_yyyymmddhhmm.json) of files successfully uploaded) - backupRestoreUtil = new BackupRestoreUtil(config.getIncrementalKeyspaceFilters(), config.getIncrementalCFFilter()); + backupRestoreUtil = new BackupRestoreUtil(config.getIncrementalIncludeCFList(), config.getIncrementalExcludeCFList()); } @Override @@ -94,7 +94,7 @@ private void notifyObservers() { @Override protected void processColumnFamily(String keyspace, String columnFamily, File backupDir) throws Exception { - List uploadedFiles = upload(backupDir, BackupFileType.SST); + List uploadedFiles = upload(backupDir, BackupFileType.SST, config.enableAsyncIncremental()); if (!uploadedFiles.isEmpty()) { String incrementalUploadTime = AbstractBackupPath.formatDate(uploadedFiles.get(0).getTime()); //format of yyyymmddhhmm (e.g. 201505060901) diff --git a/priam/src/main/java/com/netflix/priam/backup/MetaData.java b/priam/src/main/java/com/netflix/priam/backup/MetaData.java index 8cd6cab95..6a56ac606 100644 --- a/priam/src/main/java/com/netflix/priam/backup/MetaData.java +++ b/priam/src/main/java/com/netflix/priam/backup/MetaData.java @@ -22,7 +22,6 @@ import com.netflix.priam.config.IConfiguration; import com.netflix.priam.backup.AbstractBackupPath.BackupFileType; import com.netflix.priam.backup.IMessageObserver.BACKUP_MESSAGE_TYPE; -import com.netflix.priam.utils.RetryableCallable; import org.apache.commons.io.FileUtils; import org.json.simple.JSONArray; import org.json.simple.parser.JSONParser; @@ -45,8 +44,8 @@ public class MetaData { private static final Logger logger = LoggerFactory.getLogger(MetaData.class); private final Provider pathFactory; - private static List observers = new ArrayList(); - private final List metaRemotePaths = new ArrayList(); + private static List observers = new ArrayList<>(); + private final List metaRemotePaths = new ArrayList<>(); private final IBackupFileSystem fs; @Inject diff --git a/priam/src/main/java/com/netflix/priam/backup/RangeReadInputStream.java b/priam/src/main/java/com/netflix/priam/backup/RangeReadInputStream.java index 263782435..76f888573 100644 --- a/priam/src/main/java/com/netflix/priam/backup/RangeReadInputStream.java +++ b/priam/src/main/java/com/netflix/priam/backup/RangeReadInputStream.java @@ -20,7 +20,6 @@ import com.amazonaws.services.s3.model.GetObjectRequest; import com.amazonaws.services.s3.model.S3ObjectInputStream; import com.netflix.priam.utils.RetryableCallable; -import org.apache.commons.io.IOUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -76,11 +75,11 @@ public Integer retriableCall() throws IOException { if (readTotal == 0 && rCnt == -1) return -1; offset += readTotal; - return Integer.valueOf(readTotal); + return readTotal; } } }.call(); - return cnt.intValue(); + return cnt; } catch (Exception e) { String msg = String.format("failed to read offset range %d-%d of file %s whose size is %d", firstByte, endByte, remotePath, fileSize); diff --git a/priam/src/main/java/com/netflix/priam/backup/SnapshotBackup.java b/priam/src/main/java/com/netflix/priam/backup/SnapshotBackup.java index 58cb4aade..abed6626a 100644 --- a/priam/src/main/java/com/netflix/priam/backup/SnapshotBackup.java +++ b/priam/src/main/java/com/netflix/priam/backup/SnapshotBackup.java @@ -43,8 +43,8 @@ public class SnapshotBackup extends AbstractBackup { private static final Logger logger = LoggerFactory.getLogger(SnapshotBackup.class); public static final String JOBNAME = "SnapshotBackup"; private final MetaData metaData; - private final List snapshotRemotePaths = new ArrayList(); - private static List observers = new ArrayList(); + private final List snapshotRemotePaths = new ArrayList<>(); + private static List observers = new ArrayList<>(); private final ThreadSleeper sleeper = new ThreadSleeper(); private static final long WAIT_TIME_MS = 60 * 1000 * 10; private InstanceIdentity instanceIdentity; @@ -64,7 +64,7 @@ public SnapshotBackup(IConfiguration config, Provider pathFa this.snapshotStatusMgr = snapshotStatusMgr; this.instanceIdentity = instanceIdentity; this.cassandraOperations = cassandraOperations; - backupRestoreUtil = new BackupRestoreUtil(config.getSnapshotKeyspaceFilters(), config.getSnapshotCFFilter()); + backupRestoreUtil = new BackupRestoreUtil(config.getSnapshotIncludeCFList(), config.getSnapshotExcludeCFList()); } @Override @@ -183,12 +183,53 @@ protected void processColumnFamily(String keyspace, String columnFamily, File ba File snapshotDir = getValidSnapshot(backupDir, snapshotName); // Add files to this dir - if (null != snapshotDir) - abstractBackupPaths.addAll(upload(snapshotDir, BackupFileType.SNAP)); - else - logger.warn("{} folder does not contain {} snapshots", backupDir, snapshotName); + abstractBackupPaths.addAll(upload(snapshotDir, BackupFileType.SNAP, config.enableAsyncSnapshot())); } +// private void findForgottenFiles(File snapshotDir) { +// try { +// Collection snapshotFiles = FileUtils.listFiles(snapshotDir, FileFilterUtils.fileFileFilter(), null); +// File columnfamilyDir = snapshotDir.getParentFile().getParentFile(); +// +// //Find all the files in columnfamily folder which is : +// // 1. Not a temp file. +// // 2. Is a file. (we don't care about directories) +// // 3. Is older than snapshot time, as new files keep getting created after taking a snapshot. +// IOFileFilter tmpFileFilter1 = FileFilterUtils.suffixFileFilter(TMP_EXT); +// IOFileFilter tmpFileFilter2 = FileFilterUtils.asFileFilter(pathname -> tmpFilePattern.matcher(pathname.getName()).matches()); +// IOFileFilter tmpFileFilter = FileFilterUtils.or(tmpFileFilter1, tmpFileFilter2); +// // Here we are allowing files which were more than @link{IConfiguration#getForgottenFileGracePeriodDays}. We do this to allow cassandra to +// // clean up any files which were generated as part of repair/compaction and cleanup thread has not already deleted. +// // Refer to https://issues.apache.org/jira/browse/CASSANDRA-6756 and https://issues.apache.org/jira/browse/CASSANDRA-7066 +// // for more information. +// IOFileFilter ageFilter = FileFilterUtils.ageFileFilter(snapshotInstant.minus(config.getForgottenFileGracePeriodDays(), ChronoUnit.DAYS).toEpochMilli()); +// IOFileFilter fileFilter = FileFilterUtils.and(FileFilterUtils.notFileFilter(tmpFileFilter), FileFilterUtils.fileFileFilter(), ageFilter); +// +// Collection columnfamilyFiles = FileUtils.listFiles(columnfamilyDir, fileFilter, null); +// +// //Remove the SSTable(s) which are part of snapshot from the CF file list. +// //This cannot be a simple removeAll as snapshot files have "different" file folder prefix. +// for (File file : snapshotFiles) { +// //Get its parent directory file based on this file. +// File originalFile = new File(columnfamilyDir, file.getName()); +// columnfamilyFiles.remove(originalFile); +// } +// +// //If there are no "extra" SSTables in CF data folder, we are done. +// if (columnfamilyFiles.size() == 0) +// return; +// +// columnfamilyFiles.parallelStream().forEach(file -> logger.info("Forgotten file: {} found for CF: {}", file.getAbsolutePath(), columnfamilyDir.getName())); +// +// //TODO: The eventual plan is to move the forgotten files to a lost+found directory and clean the directory after 'x' amount of time. This behavior should be configurable. +// backupMetrics.incrementForgottenFiles(columnfamilyFiles.size()); +// logger.warn("# of forgotten files: {} found for CF: {}", columnfamilyFiles.size(), columnfamilyDir.getName()); +// } catch (Exception e) { +// //Eat the exception, if there, for any reason. This should not stop the snapshot for any reason. +// logger.error("Exception occurred while trying to find forgottenFile. Ignoring the error and continuing with remaining backup", e); +// } +// } + @Override protected void addToRemotePath(String remotePath) { snapshotRemotePaths.add(remotePath); diff --git a/priam/src/main/java/com/netflix/priam/backup/parallel/BackupPostProcessingCallback.java b/priam/src/main/java/com/netflix/priam/backup/parallel/BackupPostProcessingCallback.java deleted file mode 100644 index eee1661ea..000000000 --- a/priam/src/main/java/com/netflix/priam/backup/parallel/BackupPostProcessingCallback.java +++ /dev/null @@ -1,24 +0,0 @@ -/** - * Copyright 2017 Netflix, Inc. - *

- * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.netflix.priam.backup.parallel; - -/* - * Encapsules one to many steps needed once an upload is completed. - */ -public interface BackupPostProcessingCallback { - - void postProcessing(E completedTask); -} \ No newline at end of file diff --git a/priam/src/main/java/com/netflix/priam/backup/parallel/CassandraBackupQueueMgr.java b/priam/src/main/java/com/netflix/priam/backup/parallel/CassandraBackupQueueMgr.java deleted file mode 100644 index 2d734760c..000000000 --- a/priam/src/main/java/com/netflix/priam/backup/parallel/CassandraBackupQueueMgr.java +++ /dev/null @@ -1,107 +0,0 @@ -/** - * Copyright 2017 Netflix, Inc. - *

- * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.netflix.priam.backup.parallel; - -import com.google.inject.Inject; -import com.google.inject.Singleton; -import com.netflix.priam.config.IConfiguration; -import com.netflix.priam.backup.AbstractBackupPath; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.AbstractSet; -import java.util.Date; -import java.util.HashSet; -import java.util.concurrent.ArrayBlockingQueue; -import java.util.concurrent.BlockingQueue; - -/* - * - * Represents a queue of files (incrementals, snapshots, meta files) to be uploaded. - * - * Duties of the mgr include: - * - Mechanism to add a task, including de-duplication of tasks before adding to queue. - * - Guarantee delivery of task to only one consumer. - * - Provide relevant metrics including number of tasks in queue, etc. - */ - -@Singleton -public class CassandraBackupQueueMgr implements ITaskQueueMgr { - - private static final Logger logger = LoggerFactory.getLogger(CassandraBackupQueueMgr.class); - - private BlockingQueue tasks; //A queue of files to be uploaded - private AbstractSet tasksQueued; //A queue to determine what files have been queued, used for deduplication - - @Inject - public CassandraBackupQueueMgr(IConfiguration config) { - tasks = new ArrayBlockingQueue(config.getIncrementalBkupQueueSize()); - // Key to task is the S3 absolute path (BASE/REGION/CLUSTER/TOKEN/[yyyymmddhhmm]/[SST|SNP|META]/KEYSPACE/COLUMNFAMILY/FILE - tasksQueued = new HashSet(config.getIncrementalBkupQueueSize()); - } - - @Override - public void add(AbstractBackupPath task) - { - if (!tasksQueued.contains(task.getRemotePath())) { - tasksQueued.add(task.getRemotePath()); - try { - // block until space becomes available in queue - tasks.put(task); - logger.debug("Queued file {} within CF {}", task.getFileName(), task.getColumnFamily()); - } catch (InterruptedException e) { - logger.warn("Interrupted waiting for the task queue to have free space, not fatal will just move on. Error Msg: {}", e.getLocalizedMessage()); - tasksQueued.remove(task.getRemotePath()); - } - } else { - logger.debug("Already in queue, no-op. File: {}", task.getRemotePath()); - } - } - - @Override - public AbstractBackupPath take() throws InterruptedException { - AbstractBackupPath task = null; - if (!tasks.isEmpty()) { - - synchronized (tasks) { - task = tasks.poll(); //non-blocking call - } - } - - return task; - } - - @Override - public Boolean hasTasks() { - return !tasks.isEmpty(); - } - - @Override - public void taskPostProcessing(AbstractBackupPath completedTask) { - this.tasksQueued.remove(completedTask.getRemotePath()); - } - - @Override - public Integer getNumOfTasksToBeProcessed() { - return tasks.size(); - } - - @Override - public Boolean tasksCompleted(Date date) { - throw new UnsupportedOperationException(); - } - -} \ No newline at end of file diff --git a/priam/src/main/java/com/netflix/priam/backup/parallel/ITaskQueueMgr.java b/priam/src/main/java/com/netflix/priam/backup/parallel/ITaskQueueMgr.java deleted file mode 100644 index e09198d6d..000000000 --- a/priam/src/main/java/com/netflix/priam/backup/parallel/ITaskQueueMgr.java +++ /dev/null @@ -1,62 +0,0 @@ -/** - * Copyright 2017 Netflix, Inc. - *

- * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.netflix.priam.backup.parallel; - -/* - * - * Represents a queue of tasks to be completed. - * - * Duties of the mgr include: - * - Mechanism to add a task, including deduplication of tasks before adding to queue. - * - Guarantee delivery of task to only one consumer. - * - Provide relevant metrics including number of tasks in queue, number of tasks processed. - */ -public interface ITaskQueueMgr { - - /** - * Adds the provided task into the queue if it does not already exist. For performance reasons - * this is best effort and therefore callers are responsible for handling duplicate tasks. - * - * This method will block if the queue of tasks is full - * @param task The task to put onto the queue - */ - void add(E task); - - /** - * @return task, null if none is available. - */ - E take() throws InterruptedException; - - /** - * @return true if there are tasks within queue to be processed; false otherwise. - */ - Boolean hasTasks(); - - /** - * A means to perform any post processing once the task has been completed. If post processing is needed, - * the consumer should notify this behavior via callback once the task is completed. - * - * *Note: "completed" here can mean success or failure. - */ - void taskPostProcessing(E completedTask); - - Integer getNumOfTasksToBeProcessed(); - - /** - * @return true if all tasks completed (includes failures) for a date; false, if at least 1 task is still in queue. - */ - Boolean tasksCompleted(java.util.Date date); -} \ No newline at end of file diff --git a/priam/src/main/java/com/netflix/priam/backup/parallel/IncrementalBackupProducer.java b/priam/src/main/java/com/netflix/priam/backup/parallel/IncrementalBackupProducer.java deleted file mode 100644 index 7e8c33f19..000000000 --- a/priam/src/main/java/com/netflix/priam/backup/parallel/IncrementalBackupProducer.java +++ /dev/null @@ -1,151 +0,0 @@ -/** - * Copyright 2017 Netflix, Inc. - *

- * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.netflix.priam.backup.parallel; - -import com.google.inject.Inject; -import com.google.inject.Provider; -import com.google.inject.Singleton; -import com.google.inject.name.Named; -import com.netflix.priam.config.IConfiguration; -import com.netflix.priam.backup.*; -import com.netflix.priam.backup.AbstractBackupPath.BackupFileType; -import com.netflix.priam.scheduler.SimpleTimer; -import com.netflix.priam.scheduler.TaskTimer; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.File; -import java.util.ArrayList; -import java.util.List; - -@Singleton -public class IncrementalBackupProducer extends AbstractBackup implements IIncrementalBackup { - - public static final String JOBNAME = "ParallelIncremental"; - private static final Logger logger = LoggerFactory.getLogger(IncrementalBackupProducer.class); - - private final List incrementalRemotePaths = new ArrayList(); - private IncrementalMetaData metaData; - private IncrementalConsumerMgr incrementalConsumerMgr; - private ITaskQueueMgr taskQueueMgr; - private BackupRestoreUtil backupRestoreUtil; - - @Inject - public IncrementalBackupProducer(IConfiguration config, Provider pathFactory, IFileSystemContext backupFileSystemCtx - , IncrementalMetaData metaData - , @Named("backup") ITaskQueueMgr taskQueueMgr ) { - - super(config, backupFileSystemCtx, pathFactory); - this.taskQueueMgr = taskQueueMgr; - this.metaData = metaData; - - init(backupFileSystemCtx); - } - - private void init(IFileSystemContext backupFileSystemCtx) { - backupRestoreUtil = new BackupRestoreUtil(config.getIncrementalKeyspaceFilters(), config.getIncrementalCFFilter()); - //"this" is a producer, lets wake up the "consumers" - this.incrementalConsumerMgr = new IncrementalConsumerMgr(this.taskQueueMgr, backupFileSystemCtx.getFileStrategy(config), super.config); - Thread consumerMgr = new Thread(this.incrementalConsumerMgr); - consumerMgr.start(); - - } - - @Override - protected void processColumnFamily(String keyspace, String columnFamily, File backupDir) throws Exception { - for (final File file : backupDir.listFiles()) { - try { - final AbstractBackupPath bp = pathFactory.get(); - bp.parseLocal(file, BackupFileType.SST); - // producer -- populate the queue of files. *Note: producer will block if queue is full. - this.taskQueueMgr.add(bp); - } catch (Exception e) { - logger.warn("Unable to queue incremental file, treating as non-fatal and moving on to next. Msg: {} Fail to queue file: {}", - e.getLocalizedMessage(), file.getAbsolutePath()); - } - - } //end enqueuing all incremental files for a CF - } - - @Override - /* - * Keeping track of successfully uploaded files. - */ - protected void addToRemotePath(String remotePath) { - incrementalRemotePaths.add(remotePath); - } - - - @Override - public void execute() throws Exception { - //Clearing remotePath List - incrementalRemotePaths.clear(); - initiateBackup(INCREMENTAL_BACKUP_FOLDER, backupRestoreUtil); - } - - public void postProcessing() { - /* - * - * Upload the audit file of completed uploads - * - List uploadedFiles = upload(backupDir, BackupFileType.SST); - if ( ! uploadedFiles.isEmpty() ) { - String incrementalUploadTime = AbstractBackupPath.formatDate(uploadedFiles.get(0).getTime()); //format of yyyymmddhhmm (e.g. 201505060901) - String metaFileName = "meta_" + columnFamilyDir.getName() + "_" + incrementalUploadTime; - logger.info("Uploading meta file for incremental backup: " + metaFileName); - this.metaData.setMetaFileName(metaFileName); - this.metaData.set(uploadedFiles, incrementalUploadTime); - logger.info("Uploaded meta file for incremental backup: " + metaFileName); - } - */ - - /* * - * Notify observers once all incrremental uploads completed - * - if(incrementalRemotePaths.size() > 0) - { - notifyObservers(); - } - * */ - - } - - @Override - /* - * @return an identifier of purpose of the task. - */ - public String getName() { - return JOBNAME; - } - - @Override - public long getNumPendingFiles() { - throw new UnsupportedOperationException(); - } - - /** - * @return Timer that run every 10 Sec - */ - public static TaskTimer getTimer() { - return new SimpleTimer(JOBNAME, INCREMENTAL_INTERVAL_IN_MILLISECS); - } - - @Override - public String getJobName() { - return JOBNAME; - } - -} \ No newline at end of file diff --git a/priam/src/main/java/com/netflix/priam/backup/parallel/IncrementalBkupPostProcessing.java b/priam/src/main/java/com/netflix/priam/backup/parallel/IncrementalBkupPostProcessing.java deleted file mode 100644 index 37a5d9878..000000000 --- a/priam/src/main/java/com/netflix/priam/backup/parallel/IncrementalBkupPostProcessing.java +++ /dev/null @@ -1,33 +0,0 @@ -/** - * Copyright 2017 Netflix, Inc. - *

- * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.netflix.priam.backup.parallel; - -import com.netflix.priam.backup.AbstractBackupPath; - -public class IncrementalBkupPostProcessing implements BackupPostProcessingCallback { - - private ITaskQueueMgr taskQueueMgr; - - public IncrementalBkupPostProcessing(ITaskQueueMgr taskQueueMgr) { - this.taskQueueMgr = taskQueueMgr; - } - - @Override - public void postProcessing(AbstractBackupPath completedTask) { - this.taskQueueMgr.taskPostProcessing(completedTask); - } - -} \ No newline at end of file diff --git a/priam/src/main/java/com/netflix/priam/backup/parallel/IncrementalConsumer.java b/priam/src/main/java/com/netflix/priam/backup/parallel/IncrementalConsumer.java deleted file mode 100644 index 7becd0fb1..000000000 --- a/priam/src/main/java/com/netflix/priam/backup/parallel/IncrementalConsumer.java +++ /dev/null @@ -1,78 +0,0 @@ -/** - * Copyright 2017 Netflix, Inc. - *

- * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.netflix.priam.backup.parallel; - -import com.netflix.priam.backup.AbstractBackupPath; -import com.netflix.priam.backup.IBackupFileSystem; -import com.netflix.priam.utils.RetryableCallable; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.nio.file.Paths; - -/* - * Performs an upload of a file, with retries. - */ -public class IncrementalConsumer implements Runnable { - private static final Logger logger = LoggerFactory.getLogger(IncrementalConsumer.class); - - private AbstractBackupPath bp; - private IBackupFileSystem fs; - private BackupPostProcessingCallback callback; - - /** - * Upload files. Does not delete the file in case of - * error. - * - * @param bp - the file to upload with additional metada - */ - public IncrementalConsumer(AbstractBackupPath bp, IBackupFileSystem fs - , BackupPostProcessingCallback callback - ) { - this.bp = bp; - this.bp.setType(AbstractBackupPath.BackupFileType.SST); //Tag this is an incremental upload, not snapshot - this.fs = fs; - this.callback = callback; - } - - @Override - /* - * Upload specified file, with retries logic. - * File will be deleted only if uploaded successfully. - */ - public void run() { - - logger.info("Consumer - about to upload file: {}", this.bp.getFileName()); - - try { - if (!bp.getBackupFile().exists()) - throw new java.util.concurrent.CancellationException("Someone beat me to uploading this file" - + ", no need to retry. Most likely not needed but to be safe, checked and released handle to file if appropriate."); - - fs.uploadFile(Paths.get(bp.getBackupFile().getAbsolutePath()), Paths.get(bp.getRemotePath()), bp, 10, true); - this.callback.postProcessing(bp); //post processing - } catch (Exception e) { - logger.error("Failed to upload local file {}. Ignoring to continue with rest of backup. Msg: {}", this.bp.getFileName(), e.getLocalizedMessage()); - } - } finally { - // post processing must happen regardless of the outcome of the upload. Otherwise we can - // leak tasks into the underlying taskMgr queue and prevent re-enqueues of the upload itself - this.callback.postProcessing(bp); - } - logger.info("Consumer - done with upload file: {}", this.bp.getFileName()); - } - -} diff --git a/priam/src/main/java/com/netflix/priam/backup/parallel/IncrementalConsumerMgr.java b/priam/src/main/java/com/netflix/priam/backup/parallel/IncrementalConsumerMgr.java deleted file mode 100644 index df1f7c651..000000000 --- a/priam/src/main/java/com/netflix/priam/backup/parallel/IncrementalConsumerMgr.java +++ /dev/null @@ -1,121 +0,0 @@ -/** - * Copyright 2017 Netflix, Inc. - *

- * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.netflix.priam.backup.parallel; - -import com.netflix.priam.config.IConfiguration; -import com.netflix.priam.backup.AbstractBackupPath; -import com.netflix.priam.backup.IBackupFileSystem; -import com.netflix.priam.backup.IIncrementalBackup; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.concurrent.*; -import java.util.concurrent.atomic.AtomicBoolean; - -import javax.annotation.Nullable; - -import com.google.common.util.concurrent.FutureCallback; -import com.google.common.util.concurrent.Futures; -import com.google.common.util.concurrent.ListenableFuture; -import com.google.common.util.concurrent.ListeningExecutorService; -import com.google.common.util.concurrent.MoreExecutors; - -/* - * Monitors files to be uploaded and assigns each file to a worker - */ -public class IncrementalConsumerMgr implements Runnable { - - private static final Logger logger = LoggerFactory.getLogger(IncrementalConsumerMgr.class); - - private AtomicBoolean run = new AtomicBoolean(true); - private ListeningExecutorService executor; - private IBackupFileSystem fs; - private ITaskQueueMgr taskQueueMgr; - private BackupPostProcessingCallback callback; - - public IncrementalConsumerMgr(ITaskQueueMgr taskQueueMgr, IBackupFileSystem fs - , IConfiguration config - ) { - this.taskQueueMgr = taskQueueMgr; - this.fs = fs; - - /* - * Too few threads, the queue will build up, consuming a lot of memory. - * Too many threads on the other hand will slow down the whole system due to excessive context switches - and lead to same symptoms. - */ - int maxWorkers = config.getIncrementalBkupMaxConsumers(); - /* - * ThreadPoolExecutor will move the file to be uploaded as a Runnable task in the work queue. - */ - BlockingQueue workQueue = new ArrayBlockingQueue(config.getIncrementalBkupMaxConsumers() * 2); - /* - * If there all workers are busy, the calling thread for the submit() will itself upload the file. This is a way to throttle how many files are moved to the - * worker queue. Specifically, the calling will continue to perform the upload unless a worker is avaialble. - */ - RejectedExecutionHandler rejectedExecutionHandler = new ThreadPoolExecutor.CallerRunsPolicy(); - executor = MoreExecutors.listeningDecorator(new ThreadPoolExecutor(maxWorkers, maxWorkers, 60, TimeUnit.SECONDS, - workQueue, rejectedExecutionHandler)); - - callback = new IncrementalBkupPostProcessing(this.taskQueueMgr); - } - - /* - * Stop looking for files to upload - */ - public void shutdown() { - this.run.set(false); - this.executor.shutdown(); //will not accept new task and waits for active threads to be completed before shutdown. - } - - @Override - public void run() { - while (this.run.get()) { - - while (this.taskQueueMgr.hasTasks()) { - try { - final AbstractBackupPath bp = this.taskQueueMgr.take(); - - IncrementalConsumer task = new IncrementalConsumer(bp, this.fs, this.callback); - ListenableFuture upload = executor.submit(task); //non-blocking, will be rejected if the task cannot be scheduled - Futures.addCallback(upload, new FutureCallback() - { - public void onSuccess(@Nullable Object result) { } - - public void onFailure(Throwable t) { - // The post processing hook is responsible for removing the task from the de-duplicating - // HashSet, so we want to do the safe thing here and remove it just in case so the - // producers can re-enqueue this file in the next iteration. - // Note that this should be an abundance of caution as the IncrementalConsumer _should_ - // have deleted the task from the queue when it internally failed. - taskQueueMgr.taskPostProcessing(bp); - } - }); - } catch (InterruptedException e) { - logger.warn("Was interrupted while wating to dequeued a task. Msgl: {}", e.getLocalizedMessage()); - } - } - - // Lets not overwhelm the node hence we will pause before checking the work queue again. - try { - Thread.sleep(IIncrementalBackup.INCREMENTAL_INTERVAL_IN_MILLISECS); - } catch (InterruptedException e) { - logger.warn("Was interrupted while sleeping until next interval run. Msgl: {}", e.getLocalizedMessage()); - } - } - - } - -} \ No newline at end of file diff --git a/priam/src/main/java/com/netflix/priam/backupv2/FileUploadResult.java b/priam/src/main/java/com/netflix/priam/backupv2/FileUploadResult.java index 55b6a7bc4..94e169711 100644 --- a/priam/src/main/java/com/netflix/priam/backupv2/FileUploadResult.java +++ b/priam/src/main/java/com/netflix/priam/backupv2/FileUploadResult.java @@ -18,7 +18,6 @@ import com.netflix.priam.compress.ICompression; import com.netflix.priam.utils.GsonJsonSerializer; -import org.codehaus.jettison.json.JSONObject; import java.io.File; import java.nio.file.Files; @@ -31,7 +30,7 @@ * Created by aagrawal on 6/20/18. */ public class FileUploadResult { - private Path fileName; + private final Path fileName; @GsonJsonSerializer.PriamAnnotation.GsonIgnore private String keyspaceName; @GsonJsonSerializer.PriamAnnotation.GsonIgnore diff --git a/priam/src/main/java/com/netflix/priam/backupv2/LocalDBReaderWriter.java b/priam/src/main/java/com/netflix/priam/backupv2/LocalDBReaderWriter.java index 4be4d4e93..398a63350 100644 --- a/priam/src/main/java/com/netflix/priam/backupv2/LocalDBReaderWriter.java +++ b/priam/src/main/java/com/netflix/priam/backupv2/LocalDBReaderWriter.java @@ -20,12 +20,13 @@ import com.google.inject.Inject; import com.netflix.priam.config.IConfiguration; import com.netflix.priam.utils.GsonJsonSerializer; -import com.netflix.priam.utils.RetryableCallable; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.File; import java.io.FileReader; import java.io.FileWriter; +import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; import java.time.Instant; @@ -63,10 +64,9 @@ public synchronized LocalDB upsertLocalDBEntry(final LocalDBEntry localDBEntry) final Path localDBFile = getLocalDBPath(localDBEntry.getFileUploadResult()); - if (!localDBFile.getParent().toFile().exists()) - localDBFile.getParent().toFile().mkdirs(); + localDBFile.getParent().toFile().mkdirs(); - LocalDB localDB = getLocalDB(localDBEntry.getFileUploadResult()); + LocalDB localDB = readAndGetLocalDB(localDBEntry.getFileUploadResult()); if (localDB == null) localDB = new LocalDB(new ArrayList<>()); @@ -77,9 +77,9 @@ public synchronized LocalDB upsertLocalDBEntry(final LocalDBEntry localDBEntry) //1. new component entry //2. new version of file (change in compression type or file is modified e.g. stats file) if (entry == null) { - localDB.getLocalDb().add(localDBEntry); + localDB.getLocalDBEntries().add(localDBEntry); writeLocalDB(localDBFile, localDB); - }else{ + } else { //An entry already exists. Maybe last referenced time or backup time changed. We want to write the last time referenced. entry.setBackupTime(localDBEntry.getBackupTime()); entry.setTimeLastReferenced(localDBEntry.getTimeLastReferenced()); @@ -89,38 +89,35 @@ public synchronized LocalDB upsertLocalDBEntry(final LocalDBEntry localDBEntry) return localDB; } - private LocalDB getLocalDB(final FileUploadResult fileUploadResult) throws Exception { + private LocalDB readAndGetLocalDB(final FileUploadResult fileUploadResult) throws Exception { final Path localDbPath = getLocalDBPath(fileUploadResult); - //Retry for reading. - return new RetryableCallable(5, 1000) { - @Override - public LocalDB retriableCall() throws Exception { - return readLocalDB(localDbPath); - } - }.call(); + return readLocalDB(localDbPath); } + /** + * Get the local database entry for a given file upload result. + * + * @param fileUploadResult File upload result for which local db is required. + * @return LocalDBEntry if one exists. + * @throws Exception if there is any error in getting local db file. + */ public LocalDBEntry getLocalDBEntry(final FileUploadResult fileUploadResult) throws Exception { - LocalDB localDB = getLocalDB(fileUploadResult); - - if (localDB == null || localDB.getLocalDb() == null || localDB.getLocalDb().isEmpty()) - return null; - + LocalDB localDB = readAndGetLocalDB(fileUploadResult); return getLocalDBEntry(fileUploadResult, localDB); } private LocalDBEntry getLocalDBEntry(final FileUploadResult fileUploadResult, final LocalDB localDB) throws Exception { - if (localDB == null || localDB.getLocalDb().isEmpty()) + if (localDB == null || localDB.getLocalDBEntries() == null || localDB.getLocalDBEntries().isEmpty()) return null; //Get local db entry for same file and version. - List localDBEntries = localDB.getLocalDb().stream().filter(localDBEntry -> + List localDBEntries = localDB.getLocalDBEntries().stream().filter(localDBEntry -> (localDBEntry.getFileUploadResult().getFileName().toFile().getName().toLowerCase().equals(fileUploadResult.getFileName().toFile().getName().toLowerCase()))) .filter(localDBEntry -> (localDBEntry.getFileUploadResult().getLastModifiedTime().equals(fileUploadResult.getLastModifiedTime()))) .filter(localDBEntry -> (localDBEntry.getFileUploadResult().getCompression().equals(fileUploadResult.getCompression()))) .collect(Collectors.toList()); - if (localDBEntries == null || localDBEntries.isEmpty()) + if (localDBEntries.isEmpty()) return null; if (localDBEntries.size() == 1) { @@ -133,10 +130,26 @@ private LocalDBEntry getLocalDBEntry(final FileUploadResult fileUploadResult, fi throw new Exception("Unexpected behavior: More than one entry found in local database for the same file. FileUploadResult: " + fileUploadResult); } + /** + * Gets the local db path on the local file system. + * + * @param fileUploadResult This contains the SSTable component for which local db path is required. + * @return the local db path on local file system. + */ public Path getLocalDBPath(final FileUploadResult fileUploadResult) { - return Paths.get(configuration.getDataFileLocation(), LOCAL_DB, fileUploadResult.getKeyspaceName(), fileUploadResult.getColumnFamilyName(), PrefixGenerator.getSSTFileBase(fileUploadResult.getFileName().toFile().getName()) + ".localdb"); + return Paths.get(configuration.getDataFileLocation(), LOCAL_DB, + fileUploadResult.getKeyspaceName(), + fileUploadResult.getColumnFamilyName(), + PrefixGenerator.getSSTFileBase(fileUploadResult.getFileName().toFile().getName()) + ".localdb"); } + /** + * Writes the local database to the local db file. This will do a complete replace of existing local database if any. + * + * @param localDBFile path to the local database file. + * @param localDB local database containing all the entries to the local database. + * @throws Exception If the path denoted is directory, write permission issues or any other exceptions. + */ public void writeLocalDB(final Path localDBFile, final LocalDB localDB) throws Exception { if (localDB == null || localDBFile == null || localDBFile.toFile().isDirectory()) throw new Exception("Invalid Arguments: localDbFile: " + localDBFile + ", localDB: " + localDB); @@ -144,16 +157,29 @@ public void writeLocalDB(final Path localDBFile, final LocalDB localDB) throws E if (!localDBFile.getParent().toFile().exists()) localDBFile.getParent().toFile().mkdirs(); - try (FileWriter writer = new FileWriter(localDBFile.toFile())) { + File tmpFile = File.createTempFile(localDBFile.toFile().getName(), ".tmp", localDBFile.getParent().toFile()); + try (FileWriter writer = new FileWriter(tmpFile)) { writer.write(localDB.toString()); + + // Atomically swap out the new local db file for the old local db file. + if (!tmpFile.renameTo(localDBFile.toFile())) + logger.error("Failed to persist local db: {}", localDB); + } finally { + if (tmpFile != null) + Files.deleteIfExists(tmpFile.toPath()); } + } + /** + * Reads the local database file stored on local file system. + * + * @param localDBFile path the local database. + * @return local database if file exists or empty local database. + * @throws Exception If there is any error in de-serializing the object or any other file system errors. + */ public LocalDB readLocalDB(final Path localDBFile) throws Exception { - //Verify it is file and it exists. - if (localDBFile.toFile().isDirectory()) - throw new Exception("Invalid Arguments: Path provided is directory and not a file: " + localDBFile.toString()); - + //Verify file exists if (!localDBFile.toFile().exists()) return new LocalDB(new ArrayList<>()); @@ -161,7 +187,7 @@ public LocalDB readLocalDB(final Path localDBFile) throws Exception { LocalDB localDB = GsonJsonSerializer.getGson().fromJson(reader, LocalDB.class); String columnfamilyName = localDBFile.getParent().toFile().getName(); String keyspaceName = localDBFile.getParent().getParent().toFile().getName(); - localDB.getLocalDb().stream().forEach(localDBEntry -> { + localDB.getLocalDBEntries().forEach(localDBEntry -> { localDBEntry.getFileUploadResult().setColumnFamilyName(columnfamilyName); localDBEntry.getFileUploadResult().setKeyspaceName(keyspaceName); }); @@ -173,19 +199,15 @@ public LocalDB readLocalDB(final Path localDBFile) throws Exception { } static class LocalDB { - private List localDb; + private final List localDBEntries; - public LocalDB(List localDb) { - this.localDb = localDb; + public LocalDB(List localDBEntries) { + this.localDBEntries = localDBEntries; } - public List getLocalDb() { - - return localDb; - } + public List getLocalDBEntries() { - public void setLocalDb(List localDb) { - this.localDb = localDb; + return localDBEntries; } @Override @@ -195,7 +217,7 @@ public String toString() { } static class LocalDBEntry { - private FileUploadResult fileUploadResult; + private final FileUploadResult fileUploadResult; private Instant timeLastReferenced; private Instant backupTime; @@ -207,10 +229,6 @@ public FileUploadResult getFileUploadResult() { return fileUploadResult; } - public void setFileUploadResult(FileUploadResult fileUploadResult) { - this.fileUploadResult = fileUploadResult; - } - public Instant getTimeLastReferenced() { return timeLastReferenced; } diff --git a/priam/src/main/java/com/netflix/priam/backupv2/MetaFileManager.java b/priam/src/main/java/com/netflix/priam/backupv2/MetaFileManager.java index 85b8a17fa..723aa5a8b 100644 --- a/priam/src/main/java/com/netflix/priam/backupv2/MetaFileManager.java +++ b/priam/src/main/java/com/netflix/priam/backupv2/MetaFileManager.java @@ -56,7 +56,7 @@ public void cleanupOldMetaFiles() { FileFilterUtils.or(FileFilterUtils.suffixFileFilter(MetaFileInfo.META_FILE_SUFFIX), FileFilterUtils.suffixFileFilter(MetaFileInfo.META_FILE_SUFFIX + ".tmp"))); Collection files = FileUtils.listFiles(metaFileDirectory.toFile(), fileNameFilter, null); - files.stream().filter(file -> file.isFile()).forEach(file -> { + files.stream().filter(File::isFile).forEach(file -> { logger.debug("Deleting old META_V2 file found: {}", file.getAbsolutePath()); file.delete(); }); diff --git a/priam/src/main/java/com/netflix/priam/backupv2/MetaFileWriterBuilder.java b/priam/src/main/java/com/netflix/priam/backupv2/MetaFileWriterBuilder.java index 73c44dec6..b8546bae4 100644 --- a/priam/src/main/java/com/netflix/priam/backupv2/MetaFileWriterBuilder.java +++ b/priam/src/main/java/com/netflix/priam/backupv2/MetaFileWriterBuilder.java @@ -23,8 +23,6 @@ import com.netflix.priam.backup.IBackupFileSystem; import com.netflix.priam.backup.IFileSystemContext; import com.netflix.priam.identity.InstanceIdentity; -import com.netflix.priam.utils.RetryableCallable; -import org.apache.commons.io.FileUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/priam/src/main/java/com/netflix/priam/cli/Application.java b/priam/src/main/java/com/netflix/priam/cli/Application.java index 24fea63c3..fa2f49266 100644 --- a/priam/src/main/java/com/netflix/priam/cli/Application.java +++ b/priam/src/main/java/com/netflix/priam/cli/Application.java @@ -32,7 +32,7 @@ static Injector getInjector() { static void initialize() { IConfiguration conf = getInjector().getInstance(IConfiguration.class); - conf.intialize(); + conf.initialize(); } static void shutdownAdditionalThreads() { diff --git a/priam/src/main/java/com/netflix/priam/cluster/management/Compaction.java b/priam/src/main/java/com/netflix/priam/cluster/management/Compaction.java index 271f7b546..cdd727c00 100644 --- a/priam/src/main/java/com/netflix/priam/cluster/management/Compaction.java +++ b/priam/src/main/java/com/netflix/priam/cluster/management/Compaction.java @@ -15,23 +15,22 @@ */ package com.netflix.priam.cluster.management; +import com.netflix.priam.backup.BackupRestoreUtil; import com.netflix.priam.config.IConfiguration; import com.netflix.priam.defaultimpl.CassandraOperations; import com.netflix.priam.merics.CompactionMeasurement; import com.netflix.priam.scheduler.CronTimer; import com.netflix.priam.scheduler.TaskTimer; import org.apache.commons.collections4.CollectionUtils; -import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import javax.inject.Inject; import javax.inject.Singleton; -import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.regex.Pattern; + /** * Utility class to compact the keyspaces/columnfamilies * Created by aagrawal on 1/25/18. @@ -40,7 +39,6 @@ public class Compaction extends IClusterManagement { private static final Logger logger = LoggerFactory.getLogger(Compaction.class); private final IConfiguration config; - private static final Pattern columnFamilyFilterPattern = Pattern.compile(".\\.."); private final CassandraOperations cassandraOperations; @Inject public Compaction(IConfiguration config, CassandraOperations cassandraOperations, CompactionMeasurement compactionMeasurement) { @@ -48,39 +46,15 @@ public Compaction(IConfiguration config, CassandraOperations cassandraOperations this.config = config; this.cassandraOperations = cassandraOperations; } - private final Map> getCompactionFilter(String compactionFilter) throws IllegalArgumentException { - if (StringUtils.isEmpty(compactionFilter)) - return null; - final Map> columnFamilyFilter = new HashMap<>(); //key: keyspace, value: a list of CFs within the keyspace - String[] filters = compactionFilter.split(","); - for (int i = 0; i < filters.length; i++) { //process each filter - if (columnFamilyFilterPattern.matcher(filters[i]).find()) { - String[] filter = filters[i].split("\\."); - String keyspaceName = filter[0]; - String columnFamilyName = filter[1]; - if (columnFamilyName.indexOf("-") != -1) - columnFamilyName = columnFamilyName.substring(0, columnFamilyName.indexOf("-")); - List existingCfs = columnFamilyFilter.getOrDefault(keyspaceName, new ArrayList<>()); - if (!columnFamilyName.equalsIgnoreCase("*")) - existingCfs.add(columnFamilyName); - columnFamilyFilter.put(keyspaceName, existingCfs); - } else { - throw new IllegalArgumentException("Column family filter format is not valid. Format needs to be \"keyspace.columnfamily\". Invalid input: " + filters[i]); - } - } - return columnFamilyFilter; - } + + final Map> getCompactionIncludeFilter(IConfiguration config) throws Exception { - if (StringUtils.isEmpty(config.getCompactionIncludeCFList())) - return null; - Map> columnFamilyFilter = getCompactionFilter(config.getCompactionIncludeCFList()); + Map> columnFamilyFilter = BackupRestoreUtil.getFilter(config.getCompactionIncludeCFList()); logger.info("Compaction: Override for include CF provided by user: {}", columnFamilyFilter); return columnFamilyFilter; } final Map> getCompactionExcludeFilter(IConfiguration config) throws Exception { - if (StringUtils.isEmpty(config.getCompactionExcludeCFList())) - return null; - Map> columnFamilyFilter = getCompactionFilter(config.getCompactionExcludeCFList()); + Map> columnFamilyFilter = BackupRestoreUtil.getFilter(config.getCompactionExcludeCFList()); logger.info("Compaction: Override for exclude CF provided by user: {}", columnFamilyFilter); return columnFamilyFilter; } @@ -89,11 +63,11 @@ final Map> getCompactionFilterCfs(IConfiguration config) th final Map> excludeFilter = getCompactionExcludeFilter(config); final Map> allColumnfamilies = cassandraOperations.getColumnfamilies(); Map> result = new HashMap<>(); - allColumnfamilies.entrySet().forEach(entry -> { - String keyspaceName = entry.getKey(); + + allColumnfamilies.forEach((keyspaceName, columnfamilies) -> { if (SchemaConstant.isSystemKeyspace(keyspaceName)) //no need to compact system keyspaces. return; - List columnfamilies = entry.getValue(); + if (excludeFilter != null && excludeFilter.containsKey(keyspaceName)) { List excludeCFFilter = excludeFilter.get(keyspaceName); //Is CF list null/empty? If yes, then exclude all CF's for this keyspace. @@ -132,6 +106,7 @@ protected String runTask() throws Exception { * * @param config {@link IConfiguration} to get configuration details from priam. * @return the timer to be used for compaction interval from {@link IConfiguration#getCompactionCronExpression()} + * @throws Exception If the cron expression is invalid. */ public static TaskTimer getTimer(IConfiguration config) throws Exception { return CronTimer.getCronTimer(Task.COMPACTION.name(), config.getCompactionCronExpression()); diff --git a/priam/src/main/java/com/netflix/priam/cluster/management/Flush.java b/priam/src/main/java/com/netflix/priam/cluster/management/Flush.java index 6d6d97b9c..17947ebf8 100644 --- a/priam/src/main/java/com/netflix/priam/cluster/management/Flush.java +++ b/priam/src/main/java/com/netflix/priam/cluster/management/Flush.java @@ -41,7 +41,7 @@ public class Flush extends IClusterManagement { private final IConfiguration config; private final CassandraOperations cassandraOperations; - private List keyspaces = new ArrayList(); + private List keyspaces = new ArrayList<>(); @Inject public Flush(IConfiguration config, CassandraOperations cassandraOperations, NodeToolFlushMeasurement nodeToolFlushMeasurement) { @@ -55,7 +55,7 @@ public Flush(IConfiguration config, CassandraOperations cassandraOperations, Nod * @return the keyspace(s) flushed. List can be empty but never null. */ protected String runTask() throws Exception { - List flushed = new ArrayList(); + List flushed = new ArrayList<>(); //Get keyspaces to flush deriveKeyspaces(); @@ -88,7 +88,7 @@ protected String runTask() throws Exception { /* Derive keyspace(s) to flush in the following order: explicit list provided by caller, property, or all keyspaces. */ - private void deriveKeyspaces() throws Exception{ + private void deriveKeyspaces() throws Exception { //== get value from property String raw = this.config.getFlushKeyspaces(); if (raw != null && !raw.isEmpty()) { @@ -113,6 +113,7 @@ private void deriveKeyspaces() throws Exception{ * If {@link IConfiguration#getFlushSchedulerType()} is {@link com.netflix.priam.scheduler.SchedulerType#HOUR} then it expects {@link IConfiguration#getFlushInterval()} in the format of hour=x or daily=x *

* If {@link IConfiguration#getFlushSchedulerType()} is {@link com.netflix.priam.scheduler.SchedulerType#CRON} then it expects a valid CRON expression from {@link IConfiguration#getFlushCronExpression()} + * @throws Exception if the configurations are wrong. .e.g invalid cron expression. */ public static TaskTimer getTimer(IConfiguration config) throws Exception { diff --git a/priam/src/main/java/com/netflix/priam/config/IBackupRestoreConfig.java b/priam/src/main/java/com/netflix/priam/config/IBackupRestoreConfig.java index c7d2c4d3d..84f301b4d 100644 --- a/priam/src/main/java/com/netflix/priam/config/IBackupRestoreConfig.java +++ b/priam/src/main/java/com/netflix/priam/config/IBackupRestoreConfig.java @@ -29,7 +29,7 @@ public interface IBackupRestoreConfig { * * @return Snapshot Meta Service cron expression for generating manifest.json * @see quartz-scheduler - * @see http://www.cronmaker.com To build new cron timer + * @see http://www.cronmaker.com */ default String getSnapshotMetaServiceCronExpression(){ return "-1"; diff --git a/priam/src/main/java/com/netflix/priam/config/IConfiguration.java b/priam/src/main/java/com/netflix/priam/config/IConfiguration.java index 1691c91cf..a2bd43163 100644 --- a/priam/src/main/java/com/netflix/priam/config/IConfiguration.java +++ b/priam/src/main/java/com/netflix/priam/config/IConfiguration.java @@ -16,6 +16,8 @@ */ package com.netflix.priam.config; +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.databind.ObjectMapper; import com.google.inject.ImplementedBy; import com.netflix.priam.tuner.JVMOption; import com.netflix.priam.config.PriamConfiguration; @@ -33,7 +35,7 @@ @ImplementedBy(PriamConfiguration.class) public interface IConfiguration { - void intialize(); + void initialize(); /** * @return Path to the home dir of Cassandra @@ -114,7 +116,7 @@ public interface IConfiguration { String getBackupPrefix(); /** - * Location containing backup files. Typically bucket name followed by path + * @return Location containing backup files. Typically bucket name followed by path * to the clusters backup */ String getRestorePrefix(); @@ -124,12 +126,6 @@ public interface IConfiguration { */ void setRestorePrefix(String prefix); - /** - * @return List of keyspaces to restore. If none, all keyspaces are - * restored. - */ - List getRestoreKeySpaces(); - /** * @return Location of the local data dir */ @@ -168,26 +164,33 @@ public interface IConfiguration { /** * @return Cassandra's JMX port */ - int getJmxPort(); + default int getJmxPort() { + return 7199; + } /** * @return Cassandra's JMX username */ - String getJmxUsername(); + default String getJmxUsername() { + return null; + } /** * @return Cassandra's JMX password */ - String getJmxPassword(); + default String getJmxPassword() { + return null; + } /** * @return Enables Remote JMX connections n C* */ - boolean enableRemoteJMX(); - + default boolean enableRemoteJMX() { + return false; + } /** - * Cassandra storage/cluster communication port + * @return Cassandra storage/cluster communication port */ int getStoragePort(); @@ -265,7 +268,6 @@ default String getCompactionIncludeCFList(){ return null; } - /** * Column family(ies), comma delimited, to exclude while starting compaction (user-initiated or on CRON). * Note 1: The expected format is keyspace.cfname. If no value is provided then compaction is scheduled for all KS,CF(s) @@ -298,47 +300,83 @@ default String getCompactionExcludeCFList(){ * * @return Type of scheduler to use for backup. Note the default is TIMER based i.e. to use {@link #getBackupHour()}. * If value of "CRON" is provided it starts using {@link #getBackupCronExpression()}. + * @throws UnsupportedTypeException if the scheduler type is not CRON/HOUR. */ SchedulerType getBackupSchedulerType() throws UnsupportedTypeException; - /* - * @return key spaces, comma delimited, to filter from restore. If no filter is applied, returns null or empty string. + + /** + * Column Family(ies), comma delimited, to include during snapshot backup. + * Note 1: The expected format is keyspace.cfname. If no value is provided then snapshot contains all KS,CF(s) + * Note 2: CF name allows special character "*" to denote all the columnfamilies in a given keyspace. e.g. keyspace1.* denotes all the CFs in keyspace1. + * Note 3: {@link #getSnapshotExcludeCFList()} is applied first to exclude CF/keyspace and then {@link #getSnapshotIncludeCFList()} is applied to include the CF's/keyspaces. + * + * @return Column Family(ies), comma delimited, to include in snapshot backup. If no filter is applied, returns null. */ - String getSnapshotKeyspaceFilters(); + default String getSnapshotIncludeCFList() { + return null; + } - /* - * Column Family(ies), comma delimited, to filter from backup. - * *Note: the expected format is keyspace.cfname + /** + * Column family(ies), comma delimited, to exclude during snapshot backup. + * Note 1: The expected format is keyspace.cfname. If no value is provided then snapshot is scheduled for all KS,CF(s) + * Note 2: CF name allows special character "*" to denote all the columnfamilies in a given keyspace. e.g. keyspace1.* denotes all the CFs in keyspace1. + * Note 3: {@link #getSnapshotExcludeCFList()} is applied first to exclude CF/keyspace and then {@link #getSnapshotIncludeCFList()} is applied to include the CF's/keyspaces. * - * @return Column Family(ies), comma delimited, to filter from backup. If no filter is applied, returns null. + * @return Column Family(ies), comma delimited, to exclude from snapshot backup. If no filter is applied, returns null. */ - String getSnapshotCFFilter(); + default String getSnapshotExcludeCFList() { + return null; + } - /* - * @return key spaces, comma delimited, to filter from restore. If no filter is applied, returns null or empty string. + /** + * Column Family(ies), comma delimited, to include during incremental backup. + * Note 1: The expected format is keyspace.cfname. If no value is provided then incremental contains all KS,CF(s) + * Note 2: CF name allows special character "*" to denote all the columnfamilies in a given keyspace. e.g. keyspace1.* denotes all the CFs in keyspace1. + * Note 3: {@link #getIncrementalExcludeCFList()} is applied first to exclude CF/keyspace and then {@link #getIncrementalIncludeCFList()} is applied to include the CF's/keyspaces. + * + * @return Column Family(ies), comma delimited, to include in incremental backup. If no filter is applied, returns null. */ - String getIncrementalKeyspaceFilters(); + default String getIncrementalIncludeCFList() { + return null; + } - /* - * Column Family(ies), comma delimited, to filter from backup. - * *Note: the expected format is keyspace.cfname + /** + * Column family(ies), comma delimited, to exclude during incremental backup. + * Note 1: The expected format is keyspace.cfname. If no value is provided then incremental is scheduled for all KS,CF(s) + * Note 2: CF name allows special character "*" to denote all the columnfamilies in a given keyspace. e.g. keyspace1.* denotes all the CFs in keyspace1. + * Note 3: {@link #getIncrementalExcludeCFList()} is applied first to exclude CF/keyspace and then {@link #getIncrementalIncludeCFList()} is applied to include the CF's/keyspaces. * - * @return Column Family(ies), comma delimited, to filter from backup. If no filter is applied, returns null. + * @return Column Family(ies), comma delimited, to exclude from incremental backup. If no filter is applied, returns null. */ - String getIncrementalCFFilter(); + default String getIncrementalExcludeCFList() { + return null; + } - /* - * @return key spaces, comma delimited, to filter from restore. If no filter is applied, returns null or empty string. + /** + * Column Family(ies), comma delimited, to include during restore. + * Note 1: The expected format is keyspace.cfname. If no value is provided then restore contains all KS,CF(s) + * Note 2: CF name allows special character "*" to denote all the columnfamilies in a given keyspace. e.g. keyspace1.* denotes all the CFs in keyspace1. + * Note 3: {@link #getRestoreExcludeCFList()} is applied first to exclude CF/keyspace and then {@link #getRestoreIncludeCFList()} is applied to include the CF's/keyspaces. + * + * @return Column Family(ies), comma delimited, to include in restore. If no filter is applied, returns null. */ - String getRestoreKeyspaceFilter(); + default String getRestoreIncludeCFList() { + return null; + } - /* - * Column Family(ies), comma delimited, to filter from backup. - * Note: the expected format is keyspace.cfname + /** + * Column family(ies), comma delimited, to exclude during restore. + * Note 1: The expected format is keyspace.cfname. If no value is provided then restore is scheduled for all KS,CF(s) + * Note 2: CF name allows special character "*" to denote all the columnfamilies in a given keyspace. e.g. keyspace1.* denotes all the CFs in keyspace1. + * Note 3: {@link #getRestoreExcludeCFList()} is applied first to exclude CF/keyspace and then {@link #getRestoreIncludeCFList()} is applied to include the CF's/keyspaces. * - * @return Column Family(ies), comma delimited, to filter from restore. If no filter is applied, returns null or empty string. + * @return Column Family(ies), comma delimited, to exclude from restore. If no filter is applied, returns null. */ - String getRestoreCFFilter(); + default String getRestoreExcludeCFList() { + return null; + } + /** * Specifies the start and end time used for restoring data (yyyyMMddHHmm @@ -369,14 +407,18 @@ default String getCompactionExcludeCFList(){ boolean isMultiDC(); /** - * @return Number of backup threads for uploading + * @return Number of backup threads for uploading files when using async feature */ - int getMaxBackupUploadThreads(); + default int getBackupThreads() { + return 2; + } /** - * @return Number of download threads + * @return Number of download threads for downloading files when using async feature */ - int getMaxBackupDownloadThreads(); + default int getRestoreThreads() { + return 8; + } /** * @return true if restore should search for nearest token if current token @@ -424,11 +466,6 @@ default String getCompactionExcludeCFList(){ */ boolean isLocalBootstrapEnabled(); - /** - * @return In memory compaction limit - */ - int getInMemoryCompactionLimit(); - /** * @return Compaction throughput */ @@ -460,9 +497,13 @@ default String getCompactionExcludeCFList(){ String getSeedProviderName(); /** + * memtable_cleanup_threshold defaults to 1 / (memtable_flush_writers + 1) = 0.11 + * * @return memtable_cleanup_threshold in C* yaml */ - double getMemtableCleanupThreshold(); + default double getMemtableCleanupThreshold() { + return 0.11; + } /** * @return stream_throughput_outbound_megabits_per_sec in yaml @@ -519,13 +560,17 @@ default String getCompactionExcludeCFList(){ /** * @return possible values: all, dc, none */ - String getInternodeCompression(); + default String getInternodeCompression() { + return "all"; + } /** * Enable/disable backup/restore of commit logs. * @return boolean value true if commit log backup/restore is enabled, false otherwise. Default: false. */ - boolean isBackingUpCommitLogs(); + default boolean isBackingUpCommitLogs() { + return false; + } String getCommitLogBackupPropsFile(); @@ -544,8 +589,6 @@ default String getCompactionExcludeCFList(){ */ boolean isVpcRing(); - void setRestoreKeySpaces(List keyspaces); - boolean isClientSslEnabled(); String getInternodeEncryption(); @@ -562,13 +605,17 @@ default String getCompactionExcludeCFList(){ int getConcurrentCompactorsCnt(); - String getRpcServerType(); - - int getRpcMinThreads(); + default String getRpcServerType() { + return "hsha"; + } - int getRpcMaxThreads(); + default int getRpcMinThreads() { + return 16; + } - int getIndexInterval(); + default int getRpcMaxThreads() { + return 2048; + } /* * @return the warning threshold in MB's for large partitions encountered during compaction. @@ -582,9 +629,6 @@ default String getCompactionExcludeCFList(){ boolean getAutoBoostrap(); - //if using with Datastax Enterprise - String getDseClusterType(); - boolean isCreateNewTokenEnable(); /* @@ -592,59 +636,64 @@ default String getCompactionExcludeCFList(){ */ String getPrivateKeyLocation(); - /* + /** * @return the type of source for the restore. Valid values are: AWSCROSSACCT or GOOGLE. * Note: for backward compatibility, this property should be optional. Specifically, if it does not exist, it should not cause an adverse impact on current functionality. - * + *

* AWSCROSSACCT * - You are restoring from an AWS account which requires cross account assumption where an IAM user in one account is allowed to access resources that belong * to a different account. - * + *

* GOOGLE * - You are restoring from Google Cloud Storage - * */ String getRestoreSourceType(); - /* + /** + * Should backups be encrypted. If this is on, then all the files uploaded will be compressed and encrypted before being uploaded to remote file system. + * * @return true to enable encryption of backup (snapshots, incrementals, commit logs). * Note: for backward compatibility, this property should be optional. Specifically, if it does not exist, it should not cause an adverse impact on current functionality. */ - boolean isEncryptBackupEnabled(); + default boolean isEncryptBackupEnabled() { + return false; + } /** * Data that needs to be restored is encrypted? * @return true if data that needs to be restored is encrypted. Note that setting this value does not play any role until {@link #getRestoreSnapshot()} is set to a non-null value. */ - boolean isRestoreEncrypted(); + default boolean isRestoreEncrypted() { + return false; + } - /* + /** * @return the Amazon Resource Name (ARN). This is applicable when restoring from an AWS account which requires cross account assumption. * Note: for backward compatibility, this property should be optional. Specifically, if it does not exist, it should not cause an adverse impact on current functionality. */ String getAWSRoleAssumptionArn(); - /* + /** * @return Google Cloud Storage service account id to be use within the restore functionality. * Note: for backward compatibility, this property should be optional. Specifically, if it does not exist, it should not cause an adverse impact on current functionality. */ String getGcsServiceAccountId(); - /* + /** * @return the absolute path on disk for the Google Cloud Storage PFX file (i.e. the combined format of the private key and certificate). * This information is to be use within the restore functionality. * Note: for backward compatibility, this property should be optional. Specifically, if it does not exist, it should not cause an adverse impact on current functionality. */ String getGcsServiceAccountPrivateKeyLoc(); - /* + /** * @return the pass phrase use by PGP cryptography. This information is to be use within the restore and backup functionality when encryption is enabled. * Note: for backward compatibility, this property should be optional. Specifically, if it does not exist, it should not cause an adverse impact on current functionality. */ String getPgpPasswordPhrase(); - /* - * @return key use by PGP cryptography. This information is to be use within the restore and backup functionality when encryption is enabled. + /** + * @return public key use by PGP cryptography. This information is to be use within the restore and backup functionality when encryption is enabled. * Note: for backward compatibility, this property should be optional. Specifically, if it does not exist, it should not cause an adverse impact on current functionality. */ String getPgpPublicKeyLoc(); @@ -652,11 +701,11 @@ default String getCompactionExcludeCFList(){ /** * Use this method for adding extra/ dynamic cassandra startup options or env properties * - * @return + * @return A map of extra paramaters. */ Map getExtraEnvParams(); - /* + /** * @return the vpc id of the running instance. */ String getVpcId(); @@ -671,37 +720,93 @@ default String getCompactionExcludeCFList(){ */ String getVpcEC2RoleAssumptionArn(); - /* + /** + * Is cassandra cluster spanning more than one account. This may be true if you are migrating your cluster from one account to another. + * * @return if the dual account support */ - boolean isDualAccount(); + default boolean isDualAccount() { + return false; + } + + /** + * Should incremental backup be uploaded in async fashion? If this is false, then incrementals will be in sync fashion. + * + * @return enable async incrementals for backup + */ + default boolean enableAsyncIncremental() { + return false; + } - Boolean isIncrBackupParallelEnabled(); + /** + * Should snapshot backup be uploaded in async fashion? If this is false, then snapshot will be in sync fashion. + * + * @return enable async snapshot for backup + */ + default boolean enableAsyncSnapshot() { + return false; + } - /* - * The number of workers for parallel uploads. + /** + * Queue size to be used for backup uploads. Note that once queue is full, we would wait for {@link #getUploadTimeout()} + * to add any new item before declining the request and throwing exception. + * + * @return size of the queue for uploads. */ - int getIncrementalBkupMaxConsumers(); + default int getBackupQueueSize() { + return 100000; + } - /* - * The max number of files queued to be uploaded. + /** + * Queue size to be used for file downloads. Note that once queue is full, we would wait for {@link #getDownloadTimeout()} + * to add any new item before declining the request and throwing exception. + * + * @return size of the queue for downloads. + */ + default int getDownloadQueueSize() { + return 100000; + } + + /** + * Uploads are scheduled in {@link #getBackupQueueSize()}. If queue is full then we wait for {@link #getUploadTimeout()} + * for the queue to have an entry available for queueing the current task after which we throw RejectedExecutionException. + * + * @return timeout for uploads to wait to blocking queue + */ + default long getUploadTimeout() { + return (2 * 60 * 60 * 1000L); //2 minutes. + } + + /** + * Downloads are scheduled in {@link #getDownloadQueueSize()}. If queue is full then we wait for {@link #getDownloadTimeout()} + * for the queue to have an entry available for queueing the current task after which we throw RejectedExecutionException. + * + * @return timeout for downloads to wait to blocking queue */ - int getIncrementalBkupQueueSize(); + default long getDownloadTimeout() { + return (10 * 60 * 60 * 1000L); //10 minutes. + } /** * @return tombstone_warn_threshold in C* yaml */ - int getTombstoneWarnThreshold(); + default int getTombstoneWarnThreshold() { + return 1000; + } /** * @return tombstone_failure_threshold in C* yaml */ - int getTombstoneFailureThreshold(); + default int getTombstoneFailureThreshold() { + return 100000; + } /** * @return streaming_socket_timeout_in_ms in C* yaml */ - int getStreamingSocketTimeoutInMS(); + default int getStreamingSocketTimeoutInMS() { + return 86400000; + } /** * List of keyspaces to flush. Default: all keyspaces. @@ -721,10 +826,11 @@ default String getCompactionExcludeCFList(){ String getFlushInterval(); /** - * Scheduler type to use for flush. + * Scheduler type to use for flush. Default: HOUR. * * @return Type of scheduler to use for flush. Note the default is TIMER based i.e. to use {@link #getFlushInterval()}. * If value of "CRON" is provided it starts using {@link #getFlushCronExpression()}. + * @throws UnsupportedTypeException if the scheduler type is not HOUR/CRON. */ SchedulerType getFlushSchedulerType() throws UnsupportedTypeException; @@ -744,7 +850,12 @@ default String getFlushCronExpression(){ */ String getBackupStatusFileLoc(); - boolean useSudo(); + /** + * @return Decides whether to use sudo to start C* or not + */ + default boolean useSudo() { + return true; + } /** * SNS Notification topic to be used for sending backup event notifications. @@ -811,4 +922,65 @@ default int getPostRestoreHookHeartBeatTimeoutInMs() { default int getPostRestoreHookHeartbeatCheckFrequencyInMs() { return 120000; } + + /** + * Grace period for the file(that should have been deleted by cassandra) that are considered to be forgotten. Only required for cassandra 2.x. + * + * @return grace period for the forgotten files. + */ + default int getForgottenFileGracePeriodDays() { + return 1; + } + + /** + * A method for allowing access to outside programs to Priam configuration when paired with the Priam configuration + * HTTP endpoint at /v1/config/structured/all/property + * @param group The group of configuration options to return, currently just returns everything no matter what + * @return A Map representation of this configuration, or null if the method doesn't exist + */ + @SuppressWarnings("unchecked") + @JsonIgnore + default Map getStructuredConfiguration(String group) { + ObjectMapper objectMapper = new ObjectMapper(); + return objectMapper.convertValue(this, Map.class); + } + + /** + * Cron expression to be used for persisting Priam merged configuration to disk. Use "-1" to disable the CRON. + * This will persist the fully merged value of Priam's configuration to the {@link #getMergedConfigurationDirectory()} + * as two JSON files: structured.json and unstructured.json which persist structured config and + * unstructured config respectively. We recommend you only rely on unstructured for the time being until the + * structured interface is finalized. + * + * Default: every minute + * + * @return Cron expression for merged configuration writing + * @see quartz-scheduler + * @see http://www.cronmaker.com To build new cron timer + */ + default String getMergedConfigurationCronExpression() { + // Every minute on the top of the minute. + return "0 * * * * ? *"; + } + + /** + * Returns the path to the directory that Priam should write merged configuration to. Note that if you disable + * the merged configuration cron above {@link #getMergedConfigurationCronExpression()} then this directory is + * not created or used + * @return A string representation of the path to the merged priam configuration directory. + */ + default String getMergedConfigurationDirectory() { + return "/tmp/priam_configuration"; + } + + /** + * Escape hatch for getting any arbitrary property by key + * This is useful so we don't have to keep adding methods to this interface for every single configuration + * option ever. Also exposed via HTTP at v1/config/unstructured/X + * + * @param key The arbitrary configuration property to look up + * @param defaultValue The default value to return if the key is not found. + * @return The result for the property, or the defaultValue if provided (null otherwise) + */ + String getProperty(String key, String defaultValue); } diff --git a/priam/src/main/java/com/netflix/priam/config/PriamConfiguration.java b/priam/src/main/java/com/netflix/priam/config/PriamConfiguration.java index a58d5a9d9..bcb682a2a 100644 --- a/priam/src/main/java/com/netflix/priam/config/PriamConfiguration.java +++ b/priam/src/main/java/com/netflix/priam/config/PriamConfiguration.java @@ -17,7 +17,7 @@ package com.netflix.priam.config; import com.amazonaws.services.ec2.AmazonEC2; -import com.amazonaws.services.ec2.AmazonEC2Client; +import com.amazonaws.services.ec2.AmazonEC2ClientBuilder; import com.amazonaws.services.ec2.model.*; import com.google.common.collect.ImmutableList; import com.google.common.collect.Lists; @@ -75,21 +75,16 @@ public class PriamConfiguration implements IConfiguration { private static final String CONFIG_SSL_STORAGE_LISTERN_PORT_NAME = PRIAM_PRE + ".ssl.storage.port"; private static final String CONFIG_CL_BK_LOCATION = PRIAM_PRE + ".backup.commitlog.location"; private static final String CONFIG_THROTTLE_UPLOAD_PER_SECOND = PRIAM_PRE + ".upload.throttle"; - private static final String CONFIG_IN_MEMORY_COMPACTION_LIMIT = PRIAM_PRE + ".memory.compaction.limit"; private static final String CONFIG_COMPACTION_THROUHPUT = PRIAM_PRE + ".compaction.throughput"; private static final String CONFIG_MAX_HINT_WINDOW_IN_MS = PRIAM_PRE + ".hint.window"; - private static final String CONFIG_HINT_DELAY = PRIAM_PRE + ".hint.delay"; private static final String CONFIG_BOOTCLUSTER_NAME = PRIAM_PRE + ".bootcluster"; private static final String CONFIG_ENDPOINT_SNITCH = PRIAM_PRE + ".endpoint_snitch"; - private static final String CONFIG_MEMTABLE_TOTAL_SPACE = PRIAM_PRE + ".memtabletotalspace"; private static final String CONFIG_MEMTABLE_CLEANUP_THRESHOLD = PRIAM_PRE + ".memtable.cleanup.threshold"; private static final String CONFIG_CASS_PROCESS_NAME = PRIAM_PRE + ".cass.process"; private static final String CONFIG_VNODE_NUM_TOKENS = PRIAM_PRE + ".vnodes.numTokens"; private static final String CONFIG_YAML_LOCATION = PRIAM_PRE + ".yamlLocation"; private static final String CONFIG_AUTHENTICATOR = PRIAM_PRE + ".authenticator"; private static final String CONFIG_AUTHORIZER = PRIAM_PRE + ".authorizer"; - private static final String CONFIG_TARGET_KEYSPACE_NAME = PRIAM_PRE + ".target.keyspace"; - private static final String CONFIG_TARGET_COLUMN_FAMILY_NAME = PRIAM_PRE + ".target.columnfamily"; private static final String CONFIG_CASS_MANUAL_START_ENABLE = PRIAM_PRE + ".cass.manual.start.enable"; private static final String CONFIG_REMEDIATE_DEAD_CASSANDRA_RATE_S = PRIAM_PRE + ".remediate.dead.cassandra.rate"; private static final String CONFIG_CREATE_NEW_TOKEN_ENABLE = PRIAM_PRE + ".create.new.token.enable"; @@ -98,14 +93,7 @@ public class PriamConfiguration implements IConfiguration { private static final String CONFIG_BACKUP_THREADS = PRIAM_PRE + ".backup.threads"; private static final String CONFIG_RESTORE_PREFIX = PRIAM_PRE + ".restore.prefix"; private static final String CONFIG_INCR_BK_ENABLE = PRIAM_PRE + ".backup.incremental.enable"; - private static final String CONFIG_SNAPSHOT_KEYSPACE_FILTER = PRIAM_PRE + ".snapshot.keyspace.filter"; - private static final String CONFIG_SNAPSHOT_CF_FILTER = PRIAM_PRE + ".snapshot.cf.filter"; - private static final String CONFIG_INCREMENTAL_KEYSPACE_FILTER = PRIAM_PRE + ".incremental.keyspace.filter"; - private static final String CONFIG_INCREMENTAL_CF_FILTER = PRIAM_PRE + ".incremental.cf.filter"; - private static final String CONFIG_RESTORE_KEYSPACE_FILTER = PRIAM_PRE + ".restore.keyspace.filter"; - private static final String CONFIG_RESTORE_CF_FILTER = PRIAM_PRE + ".restore.cf.filter"; - - private static final String CONFIG_CL_BK_ENABLE = PRIAM_PRE + ".backup.commitlog.enable"; + private static final String CONFIG_AUTO_RESTORE_SNAPSHOTNAME = PRIAM_PRE + ".restore.snapshot"; private static final String CONFIG_BUCKET_NAME = PRIAM_PRE + ".s3.bucket"; private static final String CONFIG_BACKUP_SCHEDULE_TYPE = PRIAM_PRE + ".backup.schedule.type"; @@ -114,12 +102,10 @@ public class PriamConfiguration implements IConfiguration { private static final String CONFIG_S3_BASE_DIR = PRIAM_PRE + ".s3.base_dir"; private static final String CONFIG_RESTORE_THREADS = PRIAM_PRE + ".restore.threads"; private static final String CONFIG_RESTORE_CLOSEST_TOKEN = PRIAM_PRE + ".restore.closesttoken"; - private static final String CONFIG_RESTORE_KEYSPACES = PRIAM_PRE + ".restore.keyspaces"; private static final String CONFIG_BACKUP_CHUNK_SIZE = PRIAM_PRE + ".backup.chunksizemb"; private static final String CONFIG_BACKUP_RETENTION = PRIAM_PRE + ".backup.retention"; private static final String CONFIG_BACKUP_RACS = PRIAM_PRE + ".backup.racs"; private static final String CONFIG_BACKUP_STATUS_FILE_LOCATION = PRIAM_PRE + ".backup.status.location"; - private static final String CONFIG_MULTITHREADED_COMPACTION = PRIAM_PRE + ".multithreaded.compaction"; private static final String CONFIG_STREAMING_THROUGHPUT_MB = PRIAM_PRE + ".streaming.throughput.mb"; private static final String CONFIG_STREAMING_SOCKET_TIMEOUT_IN_MS = PRIAM_PRE + ".streaming.socket.timeout.ms"; private static final String CONFIG_TOMBSTONE_FAILURE_THRESHOLD = PRIAM_PRE + ".tombstone.failure.threshold"; @@ -153,10 +139,8 @@ public class PriamConfiguration implements IConfiguration { private static final String CONFIG_RPC_SERVER_TYPE = PRIAM_PRE + ".rpc.server.type"; private static final String CONFIG_RPC_MIN_THREADS = PRIAM_PRE + ".rpc.min.threads"; private static final String CONFIG_RPC_MAX_THREADS = PRIAM_PRE + ".rpc.max.threads"; - private static final String CONFIG_INDEX_INTERVAL = PRIAM_PRE + ".index.interval"; private static final String CONFIG_EXTRA_PARAMS = PRIAM_PRE + ".extra.params"; private static final String CONFIG_AUTO_BOOTSTRAP = PRIAM_PRE + ".auto.bootstrap"; - private static final String CONFIG_DSE_CLUSTER_TYPE = PRIAM_PRE + ".dse.cluster.type"; private static final String CONFIG_EXTRA_ENV_PARAMS = PRIAM_PRE + ".extra.env.params"; private static final String CONFIG_RESTORE_SOURCE_TYPE = PRIAM_PRE + ".restore.source.type"; //the type of source for the restore. Valid values are: AWSCROSSACCT or GOOGLE. @@ -249,7 +233,6 @@ public class PriamConfiguration implements IConfiguration { private static final String DEFAULT_RPC_SERVER_TYPE = "hsha"; private static final int DEFAULT_RPC_MIN_THREADS = 16; private static final int DEFAULT_RPC_MAX_THREADS = 2048; - private static final int DEFAULT_INDEX_INTERVAL = 256; private static final int DEFAULT_STREAMING_SOCKET_TIMEOUT_IN_MS = 86400000; // 24 Hours private static final int DEFAULT_TOMBSTONE_WARNING_THRESHOLD = 1000; // C* defaults private static final int DEFAULT_TOMBSTONE_FAILURE_THRESHOLD = 100000;// C* defaults @@ -272,7 +255,7 @@ public PriamConfiguration(ICredential provider, IConfigSource config, InstanceEn } @Override - public void intialize() { + public void initialize() { try { if (this.insEnvIdentity.isClassic()) { this.instanceDataRetriever = (InstanceDataRetriever) Class.forName("com.netflix.priam.identity.config.AwsClassicInstanceDataRetriever").newInstance(); @@ -344,8 +327,7 @@ public GetASGName(String region, String instanceId) { super(NUMBER_OF_RETRIES, WAIT_TIME); this.region = region; this.instanceId = instanceId; - client = new AmazonEC2Client(provider.getAwsCredentialProvider()); - client.setEndpoint("ec2." + region + ".amazonaws.com"); + client = AmazonEC2ClientBuilder.standard().withCredentials(provider.getAwsCredentialProvider()).withRegion(region).build(); } @Override @@ -371,8 +353,7 @@ public String retriableCall() throws IllegalStateException { * Get the fist 3 available zones in the region */ public void setDefaultRACList(String region) { - AmazonEC2 client = new AmazonEC2Client(provider.getAwsCredentialProvider()); - client.setEndpoint("ec2." + region + ".amazonaws.com"); + AmazonEC2 client = AmazonEC2ClientBuilder.standard().withCredentials(provider.getAwsCredentialProvider()).withRegion(region).build(); DescribeAvailabilityZonesResult res = client.describeAvailabilityZones(); List zone = Lists.newArrayList(); for (AvailabilityZone reg : res.getAvailabilityZones()) { @@ -443,11 +424,6 @@ public String getRestorePrefix() { return config.get(CONFIG_RESTORE_PREFIX); } - @Override - public List getRestoreKeySpaces() { - return config.getList(CONFIG_RESTORE_KEYSPACES); - } - @Override public String getDataFileLocation() { return config.get(CONFIG_DATA_LOCATION, DEFAULT_DATA_LOCATION); @@ -627,33 +603,33 @@ public String getCompactionExcludeCFList() { } @Override - public String getSnapshotKeyspaceFilters() { - return config.get(CONFIG_SNAPSHOT_KEYSPACE_FILTER); + public String getSnapshotIncludeCFList() { + return config.get(PRIAM_PRE + ".snapshot.cf.include"); } @Override - public String getSnapshotCFFilter() throws IllegalArgumentException { - return config.get(CONFIG_SNAPSHOT_CF_FILTER); + public String getSnapshotExcludeCFList() { + return config.get(PRIAM_PRE + ".snapshot.cf.exclude"); } @Override - public String getIncrementalKeyspaceFilters() { - return config.get(CONFIG_INCREMENTAL_KEYSPACE_FILTER); + public String getIncrementalIncludeCFList() { + return config.get(PRIAM_PRE + ".incremental.cf.include"); } @Override - public String getIncrementalCFFilter() { - return config.get(CONFIG_INCREMENTAL_CF_FILTER); + public String getIncrementalExcludeCFList() { + return config.get(PRIAM_PRE + ".incremental.cf.exclude"); } @Override - public String getRestoreKeyspaceFilter() { - return config.get(CONFIG_RESTORE_KEYSPACE_FILTER); + public String getRestoreIncludeCFList() { + return config.get(PRIAM_PRE + ".restore.cf.include"); } @Override - public String getRestoreCFFilter() { - return config.get(CONFIG_RESTORE_CF_FILTER); + public String getRestoreExcludeCFList() { + return config.get(PRIAM_PRE + ".restore.cf.exclude"); } @Override @@ -687,13 +663,12 @@ public boolean isMultiDC() { } @Override - public int getMaxBackupUploadThreads() { - + public int getBackupThreads() { return config.get(CONFIG_BACKUP_THREADS, DEFAULT_BACKUP_THREADS); } @Override - public int getMaxBackupDownloadThreads() { + public int getRestoreThreads() { return config.get(CONFIG_RESTORE_THREADS, DEFAULT_RESTORE_THREADS); } @@ -741,11 +716,6 @@ public boolean isLocalBootstrapEnabled() { return config.get(CONFIG_LOAD_LOCAL_PROPERTIES, false); } - @Override - public int getInMemoryCompactionLimit() { - return config.get(CONFIG_IN_MEMORY_COMPACTION_LIMIT, 128); - } - @Override public int getCompactionThroughput() { return config.get(CONFIG_COMPACTION_THROUHPUT, 8); @@ -760,10 +730,6 @@ public int getHintedHandoffThrottleKb() { return config.get(CONFIG_HINTS_THROTTLE_KB, DEFAULT_HINTS_THROTTLE_KB); } - public int getMaxHintThreads() { - return config.get(CONFIG_MAX_HINT_THREADS, DEFAULT_HINTS_MAX_THREADS); - } - @Override public String getBootClusterName() { return config.get(CONFIG_BOOTCLUSTER_NAME, ""); @@ -774,9 +740,6 @@ public String getSeedProviderName() { return config.get(CONFIG_SEED_PROVIDER_NAME, DEFAULT_SEED_PROVIDER); } - /** - * memtable_cleanup_threshold defaults to 1 / (memtable_flush_writers + 1) = 0.11 - */ public double getMemtableCleanupThreshold() { return config.get(CONFIG_MEMTABLE_CLEANUP_THRESHOLD, 0.11); } @@ -888,22 +851,6 @@ public boolean isVpcRing() { return config.get(CONFIG_VPC_RING, false); } - @Override - public void setRestoreKeySpaces(List keyspaces) { - if (keyspaces == null) - return; - - StringBuilder sb = new StringBuilder(); - for (int i = 0; i < keyspaces.size(); i++) { - if (i > 0) - sb.append(","); - - sb.append(keyspaces.get(i)); - } - - config.set(CONFIG_RESTORE_KEYSPACES, sb.toString()); - } - public boolean isClientSslEnabled() { return config.get(CONFIG_CLIENT_SSL_ENABLED, false); } @@ -949,10 +896,6 @@ public int getRpcMaxThreads() { return config.get(CONFIG_RPC_MAX_THREADS, DEFAULT_RPC_MAX_THREADS); } - public int getIndexInterval() { - return config.get(CONFIG_INDEX_INTERVAL, DEFAULT_INDEX_INTERVAL); - } - @Override public int getCompactionLargePartitionWarnThresholdInMB() { return config.get(PRIAM_PRE + ".compaction.large.partition.warn.threshold", 100); @@ -969,7 +912,7 @@ public Map getExtraEnvParams() { logger.info("getExtraEnvParams: No extra env params"); return null; } - Map extraEnvParamsMap = new HashMap(); + Map extraEnvParamsMap = new HashMap<>(); String[] pairs = envParams.split(","); logger.info("getExtraEnvParams: Extra cass params. From config :{}", envParams); for (int i = 0; i < pairs.length; i++) { @@ -996,11 +939,6 @@ public boolean getAutoBoostrap() { return config.get(CONFIG_AUTO_BOOTSTRAP, true); } - //values are cassandra, solr, hadoop, spark or hadoop-spark - public String getDseClusterType() { - return config.get(CONFIG_DSE_CLUSTER_TYPE + "." + ASG_NAME, "cassandra"); - } - @Override public boolean isCreateNewTokenEnable() { return config.get(CONFIG_CREATE_NEW_TOKEN_ENABLE, true); @@ -1071,39 +1009,44 @@ public String getVpcId() { } @Override - public Boolean isIncrBackupParallelEnabled() { - return config.get(PRIAM_PRE + ".incremental.bkup.parallel", false); + public boolean enableAsyncIncremental() { + return config.get(PRIAM_PRE + ".async.incremental", false); } @Override - public int getIncrementalBkupMaxConsumers() { - return config.get(PRIAM_PRE + ".incremental.bkup.max.consumers", 4); + public boolean enableAsyncSnapshot() { + return config.get(PRIAM_PRE + ".async.snapshot", false); } @Override - public int getIncrementalBkupQueueSize() { - return config.get(PRIAM_PRE + ".incremental.bkup.queue.size", 100000); + public int getBackupQueueSize() { + return config.get(PRIAM_PRE + ".backup.queue.size", 100000); + } + + @Override + public int getDownloadQueueSize() { + return config.get(PRIAM_PRE + ".download.queue.size", 100000); + } + + @Override + public long getUploadTimeout() { + return config.get(PRIAM_PRE + ".upload.timeout", (2 * 60 * 60 * 1000L)); + } + + public long getDownloadTimeout() { + return config.get(PRIAM_PRE + ".download.timeout", (10 * 60 * 60 * 1000L)); } - /** - * @return tombstone_warn_threshold in yaml - */ @Override public int getTombstoneWarnThreshold() { return config.get(CONFIG_TOMBSTONE_WARNING_THRESHOLD, DEFAULT_TOMBSTONE_WARNING_THRESHOLD); } - /** - * @return tombstone_failure_threshold in yaml - */ @Override public int getTombstoneFailureThreshold() { return config.get(CONFIG_TOMBSTONE_FAILURE_THRESHOLD, DEFAULT_TOMBSTONE_FAILURE_THRESHOLD); } - /** - * @return streaming_socket_timeout_in_ms in yaml - */ @Override public int getStreamingSocketTimeoutInMS() { return config.get(CONFIG_STREAMING_SOCKET_TIMEOUT_IN_MS, DEFAULT_STREAMING_SOCKET_TIMEOUT_IN_MS); @@ -1163,8 +1106,15 @@ public int getPostRestoreHookTimeOutInDays() { public int getPostRestoreHookHeartBeatTimeoutInMs() { return config.get(CONFIG_POST_RESTORE_HOOK_HEARTBEAT_TIMEOUT_MS, 120000); } + @Override public int getPostRestoreHookHeartbeatCheckFrequencyInMs() { return config.get(CONFIG_POST_RESTORE_HOOK_HEARTBEAT_CHECK_FREQUENCY_MS, 120000); } + + @Override + public String getProperty(String key, String defaultValue) + { + return config.get(key, defaultValue); + } } diff --git a/priam/src/main/java/com/netflix/priam/configSource/SimpleDBConfigSource.java b/priam/src/main/java/com/netflix/priam/configSource/SimpleDBConfigSource.java index 11d68ddd6..f2d4446dd 100644 --- a/priam/src/main/java/com/netflix/priam/configSource/SimpleDBConfigSource.java +++ b/priam/src/main/java/com/netflix/priam/configSource/SimpleDBConfigSource.java @@ -75,9 +75,7 @@ public void intialize(final String asgName, final String region) { request.setNextToken(nextToken); SelectResult result = simpleDBClient.select(request); nextToken = result.getNextToken(); - Iterator itemiter = result.getItems().iterator(); - while (itemiter.hasNext()) - addProperty(itemiter.next()); + for (Item item : result.getItems()) addProperty(item); } while (nextToken != null); diff --git a/priam/src/main/java/com/netflix/priam/cryptography/pgp/PgpUtil.java b/priam/src/main/java/com/netflix/priam/cryptography/pgp/PgpUtil.java index 05e5b9740..633d2f7b4 100755 --- a/priam/src/main/java/com/netflix/priam/cryptography/pgp/PgpUtil.java +++ b/priam/src/main/java/com/netflix/priam/cryptography/pgp/PgpUtil.java @@ -32,11 +32,11 @@ public class PgpUtil { * exists. * * @param pgpSec a secret key ring collection. - * @param keyID keyID we want. - * @param pass passphrase to decrypt secret key with. - * @return - * @throws PGPException - * @throws NoSuchProviderException + * @param keyID keyID we want. + * @param pass passphrase to decrypt secret key with. + * @return secret or private key corresponding to the keyID. + * @throws PGPException if there is any exception in getting the PGP key corresponding to the ID provided. + * @throws NoSuchProviderException If PGP Provider is not available. */ public static PGPPrivateKey findSecretKey(PGPSecretKeyRingCollection pgpSec, long keyID, char[] pass) throws PGPException, NoSuchProviderException { @@ -60,10 +60,10 @@ public static PGPPublicKey readPublicKey(String fileName) throws IOException, PG * A simple routine that opens a key ring file and loads the first available key * suitable for encryption. * - * @param input - * @return - * @throws IOException - * @throws PGPException + * @param input inputstream to the pgp file key ring. + * @return PGP key from the key ring. + * @throws IOException If any error in reading from the input stream. + * @throws PGPException if there is any error in getting key from key ring. */ @SuppressWarnings("rawtypes") public static PGPPublicKey readPublicKey(InputStream input) throws IOException, PGPException { diff --git a/priam/src/main/java/com/netflix/priam/defaultimpl/CassandraProcessManager.java b/priam/src/main/java/com/netflix/priam/defaultimpl/CassandraProcessManager.java index 1e647bdae..ad04774f2 100644 --- a/priam/src/main/java/com/netflix/priam/defaultimpl/CassandraProcessManager.java +++ b/priam/src/main/java/com/netflix/priam/defaultimpl/CassandraProcessManager.java @@ -109,7 +109,7 @@ public void start(boolean join_ring) throws IOException { } protected List getStartCommand() { - List startCmd = new LinkedList(); + List startCmd = new LinkedList<>(); for (String param : config.getCassStartupScript().split(" ")) { if (StringUtils.isNotBlank(param)) startCmd.add(param); diff --git a/priam/src/main/java/com/netflix/priam/defaultimpl/InjectedWebListener.java b/priam/src/main/java/com/netflix/priam/defaultimpl/InjectedWebListener.java index 8ebbe7303..a596ac1f3 100644 --- a/priam/src/main/java/com/netflix/priam/defaultimpl/InjectedWebListener.java +++ b/priam/src/main/java/com/netflix/priam/defaultimpl/InjectedWebListener.java @@ -48,7 +48,7 @@ protected Injector getInjector() { moduleList.add(new PriamGuiceModule()); injector = Guice.createInjector(moduleList); try { - injector.getInstance(IConfiguration.class).intialize(); + injector.getInstance(IConfiguration.class).initialize(); injector.getInstance(PriamServer.class).initialize(); } catch (Exception e) { logger.error(e.getMessage(), e); @@ -75,7 +75,7 @@ public void contextDestroyed(ServletContextEvent servletContextEvent) { public static class JaxServletModule extends ServletModule { @Override protected void configureServlets() { - Map params = new HashMap(); + Map params = new HashMap<>(); params.put(PackagesResourceConfig.PROPERTY_PACKAGES, "unbound"); params.put("com.sun.jersey.config.property.packages", "com.netflix.priam.resources"); params.put(ServletContainer.PROPERTY_FILTER_CONTEXT_PATH, "/REST"); diff --git a/priam/src/main/java/com/netflix/priam/defaultimpl/PriamGuiceModule.java b/priam/src/main/java/com/netflix/priam/defaultimpl/PriamGuiceModule.java index 046815482..10e8092a6 100644 --- a/priam/src/main/java/com/netflix/priam/defaultimpl/PriamGuiceModule.java +++ b/priam/src/main/java/com/netflix/priam/defaultimpl/PriamGuiceModule.java @@ -27,8 +27,6 @@ import com.netflix.priam.aws.auth.IS3Credential; import com.netflix.priam.aws.auth.S3RoleAssumptionCredential; import com.netflix.priam.backup.IBackupFileSystem; -import com.netflix.priam.backup.parallel.CassandraBackupQueueMgr; -import com.netflix.priam.backup.parallel.ITaskQueueMgr; import com.netflix.priam.cryptography.IFileCryptography; import com.netflix.priam.cryptography.pgp.PgpCredential; import com.netflix.priam.cryptography.pgp.PgpCryptography; @@ -56,7 +54,6 @@ protected void configure() { bind(IFileCryptography.class).annotatedWith(Names.named("filecryptoalgorithm")).to(PgpCryptography.class); bind(ICredentialGeneric.class).annotatedWith(Names.named("gcscredential")).to(GcsCredential.class); bind(ICredentialGeneric.class).annotatedWith(Names.named("pgpcredential")).to(PgpCredential.class); - bind(ITaskQueueMgr.class).annotatedWith(Names.named("backup")).to(CassandraBackupQueueMgr.class); bind(Registry.class).toInstance(new NoopRegistry()); } } diff --git a/priam/src/main/java/com/netflix/priam/google/GoogleEncryptedFileSystem.java b/priam/src/main/java/com/netflix/priam/google/GoogleEncryptedFileSystem.java index 0a837160b..e6e4f39aa 100755 --- a/priam/src/main/java/com/netflix/priam/google/GoogleEncryptedFileSystem.java +++ b/priam/src/main/java/com/netflix/priam/google/GoogleEncryptedFileSystem.java @@ -46,8 +46,6 @@ import java.util.Collection; import java.util.Date; import java.util.Iterator; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.concurrent.atomic.AtomicLong; public class GoogleEncryptedFileSystem extends AbstractFileSystem { @@ -157,7 +155,7 @@ private Credential constructGcsCredential() throws Exception { throw new IOException("Exception when writing decrypted gcs private key value to disk.", e); } - Collection scopes = new ArrayList(1); + Collection scopes = new ArrayList<>(1); scopes.add(StorageScopes.DEVSTORAGE_READ_ONLY); this.credential = new GoogleCredential.Builder().setTransport(this.httpTransport) .setJsonFactory(JSON_FACTORY) @@ -183,16 +181,13 @@ protected void downloadFileImpl(Path remotePath, Path localPath) throws BackupRe } get.getMediaHttpDownloader().setDirectDownloadEnabled(true); // If you're not using GCS' AppEngine, download the whole thing (instead of chunks) in one request, if possible. - InputStream is = null; - try(OutputStream os = new FileOutputStream(localPath.toFile())) { - is = get.executeMediaAsInputStream(); + try(OutputStream os = new FileOutputStream(localPath.toFile()); + InputStream is = get.executeMediaAsInputStream()) { IOUtils.copyLarge(is, os); } catch (IOException e) { throw new BackupRestoreException("IO error during streaming of object: " + objectName + " from bucket: " + this.srcBucketName, e); } catch (Exception ex) { throw new BackupRestoreException("Exception encountered when copying bytes from input to output", ex); - } finally { - IOUtils.closeQuietly(is); } backupMetrics.recordDownloadRate(get.getLastResponseHeaders().getContentLength()); diff --git a/priam/src/main/java/com/netflix/priam/identity/InstanceIdentity.java b/priam/src/main/java/com/netflix/priam/identity/InstanceIdentity.java index 1360ac9d5..ca565c323 100644 --- a/priam/src/main/java/com/netflix/priam/identity/InstanceIdentity.java +++ b/priam/src/main/java/com/netflix/priam/identity/InstanceIdentity.java @@ -17,7 +17,6 @@ package com.netflix.priam.identity; import com.google.common.base.Predicate; -import com.google.common.base.Supplier; import com.google.common.collect.Iterables; import com.google.common.collect.ListMultimap; import com.google.common.collect.Lists; @@ -50,11 +49,7 @@ public class InstanceIdentity { private static final Logger logger = LoggerFactory.getLogger(InstanceIdentity.class); public static final String DUMMY_INSTANCE_ID = "new_slot"; - private final ListMultimap locMap = Multimaps.newListMultimap(new HashMap>(), new Supplier>() { - public List get() { - return Lists.newArrayList(); - } - }); + private final ListMultimap locMap = Multimaps.newListMultimap(new HashMap>(), () -> Lists.newArrayList()); private final IPriamInstanceFactory factory; private final IMembership membership; private final IConfiguration config; diff --git a/priam/src/main/java/com/netflix/priam/identity/token/DeadTokenRetriever.java b/priam/src/main/java/com/netflix/priam/identity/token/DeadTokenRetriever.java index 75bf4b730..41085ada5 100755 --- a/priam/src/main/java/com/netflix/priam/identity/token/DeadTokenRetriever.java +++ b/priam/src/main/java/com/netflix/priam/identity/token/DeadTokenRetriever.java @@ -37,7 +37,6 @@ import javax.ws.rs.core.MediaType; import java.util.Arrays; -import java.util.Iterator; import java.util.List; import java.util.Random; @@ -184,10 +183,7 @@ private String getIp(String host, String token) throws ParseException { JSONObject jsonObject = (JSONObject) obj; - Iterator iter = jsonObject.keySet().iterator(); - - while (iter.hasNext()) { - Object key = iter.next(); + for (Object key : jsonObject.keySet()) { JSONObject msg = (JSONObject) jsonObject.get(key); if (msg.get("Token") == null) { continue; diff --git a/priam/src/main/java/com/netflix/priam/merics/BackupMetrics.java b/priam/src/main/java/com/netflix/priam/merics/BackupMetrics.java index 52ea387d1..0b926b785 100644 --- a/priam/src/main/java/com/netflix/priam/merics/BackupMetrics.java +++ b/priam/src/main/java/com/netflix/priam/merics/BackupMetrics.java @@ -27,13 +27,14 @@ */ @Singleton public class BackupMetrics { - private Registry registry; + private final Registry registry; /** * Distribution summary will provide the metric like count (how many uploads were made), max no. of bytes uploaded and total amount of bytes uploaded. */ private final DistributionSummary uploadRate, downloadRate; private final Counter validUploads, validDownloads, invalidUploads, invalidDownloads, snsNotificationSuccess, snsNotificationFailure, forgottenFiles; - public static final String uploadDownloadQueueSize = Metrics.METRIC_PREFIX + "upload.download.queue.size"; + public static final String uploadQueueSize = Metrics.METRIC_PREFIX + "upload.queue.size"; + public static final String downloadQueueSize = Metrics.METRIC_PREFIX + "download.queue.size"; @Inject public BackupMetrics(Registry registry) { diff --git a/priam/src/main/java/com/netflix/priam/resources/BackupServlet.java b/priam/src/main/java/com/netflix/priam/resources/BackupServlet.java index 9af452fab..4b8e8ee38 100644 --- a/priam/src/main/java/com/netflix/priam/resources/BackupServlet.java +++ b/priam/src/main/java/com/netflix/priam/resources/BackupServlet.java @@ -161,8 +161,8 @@ public Response list(@QueryParam(REST_HEADER_RANGE) String daterange, @QueryPara @Path("/status") @Produces(MediaType.APPLICATION_JSON) public Response status() throws Exception { - int restoreTCount = restoreObj.getActiveCount(); //Active threads performing the restore - logger.debug("Thread counts for restore is: {}", restoreTCount); + int restoreQueueSize = restoreObj.getDownloadTasksQueued(); //Items left to restore from the filesystem. + logger.info("Thread counts for restore is: {}. Items in queue: {}", config.getRestoreThreads(), restoreQueueSize); JSONObject object = new JSONObject(); object.put("SnapshotStatus", snapshotBackup.state().toString()); return Response.ok(object.toString(), MediaType.APPLICATION_JSON).build(); @@ -228,7 +228,7 @@ public Response statusByDate(@PathParam("date") String date) throws Exception { public Response snapshotsByDate(@PathParam("date") String date) throws Exception { List metadata = this.completedBkups.locate(date); JSONObject object = new JSONObject(); - List snapshots = new ArrayList(); + List snapshots = new ArrayList<>(); if (metadata != null && !metadata.isEmpty()) snapshots.addAll(metadata.stream().map(backupMetadata -> DateUtil.formatyyyyMMddHHmm(backupMetadata.getStart())).collect(Collectors.toList())); @@ -417,7 +417,7 @@ private void restore(String token, String region, Date startTime, Date endTime, logger.info("Restore will use token {}", priamServer.getId().getInstance().getToken()); } - setRestoreKeyspaces(keyspaces); + restoreObj.setRestoreConfiguration(keyspaces, null); try { restoreObj.restore(startTime, endTime); @@ -442,17 +442,6 @@ private String closestToken(String token, String region) { return tokenManager.findClosestToken(new BigInteger(token), tokenList).toString(); } - /* - * TODO: decouple the servlet, config, and restorer. this should not rely on a side - * effect of a list mutation on the config object (treating it as global var). - */ - private void setRestoreKeyspaces(String keyspaces) { - if (StringUtils.isNotBlank(keyspaces)) { - List newKeyspaces = Lists.newArrayList(keyspaces.split(",")); - config.setRestoreKeySpaces(newKeyspaces); - } - } - /* * A list of files for requested filter. Currently, the only supported filter is META, all others will be ignore. * For filter of META, ONLY the daily snapshot meta file (meta.json) are accounted for, not the incremental meta file. @@ -521,12 +510,7 @@ public void checkSSTablesForKey(String rowkey, String keyspace, String cf, Strin .getRuntime() .exec(cmd); - Callable callable = new Callable() { - @Override - public Integer call() throws Exception { - return p.waitFor(); - } - }; + Callable callable = () -> p.waitFor(); ExecutorService exeService = Executors.newSingleThreadExecutor(); try { diff --git a/priam/src/main/java/com/netflix/priam/resources/CassandraConfig.java b/priam/src/main/java/com/netflix/priam/resources/CassandraConfig.java index 432e1ccda..4b30101d1 100644 --- a/priam/src/main/java/com/netflix/priam/resources/CassandraConfig.java +++ b/priam/src/main/java/com/netflix/priam/resources/CassandraConfig.java @@ -130,7 +130,7 @@ public Response getExtraEnvParams() { Map returnMap; returnMap = priamServer.getConfiguration().getExtraEnvParams(); if (returnMap == null) { - returnMap = new HashMap(); + returnMap = new HashMap<>(); } String extraEnvParamsJson = JSONValue.toJSONString(returnMap); return Response.ok(extraEnvParamsJson).build(); diff --git a/priam/src/main/java/com/netflix/priam/resources/RestoreServlet.java b/priam/src/main/java/com/netflix/priam/resources/RestoreServlet.java index b3ff5ab7f..bcd6c1cdb 100644 --- a/priam/src/main/java/com/netflix/priam/resources/RestoreServlet.java +++ b/priam/src/main/java/com/netflix/priam/resources/RestoreServlet.java @@ -154,7 +154,7 @@ private void restore(String token, String region, Date startTime, Date endTime, logger.info("Restore will use token {}", priamServer.getId().getInstance().getToken()); } - setRestoreKeyspaces(keyspaces); + restoreObj.setRestoreConfiguration(keyspaces, null); try { restoreObj.restore(startTime, endTime); @@ -179,15 +179,4 @@ private String closestToken(String token, String region) { return tokenManager.findClosestToken(new BigInteger(token), tokenList).toString(); } - /* - * TODO: decouple the servlet, config, and restorer. this should not rely on a side - * effect of a list mutation on the config object (treating it as global var). - */ - private void setRestoreKeyspaces(String keyspaces) { - if (StringUtils.isNotBlank(keyspaces)) { - List newKeyspaces = Lists.newArrayList(keyspaces.split(",")); - config.setRestoreKeySpaces(newKeyspaces); - } - } - } \ No newline at end of file diff --git a/priam/src/main/java/com/netflix/priam/restore/AbstractRestore.java b/priam/src/main/java/com/netflix/priam/restore/AbstractRestore.java index c86950d2e..e2fa8234a 100644 --- a/priam/src/main/java/com/netflix/priam/restore/AbstractRestore.java +++ b/priam/src/main/java/com/netflix/priam/restore/AbstractRestore.java @@ -28,6 +28,7 @@ import com.netflix.priam.scheduler.Task; import com.netflix.priam.utils.*; import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.io.FileUtils; import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -35,8 +36,10 @@ import java.io.File; import java.io.IOException; import java.math.BigInteger; +import java.nio.file.Path; import java.time.LocalDateTime; import java.util.*; +import java.util.concurrent.Future; /** * A means to perform a restore. This class contains the following characteristics: @@ -44,10 +47,10 @@ * - This class can be scheduled, i.e. it is a "Task". * - When this class is executed, it uses its own thread pool to execute the restores. */ -public abstract class AbstractRestore extends Task implements IRestoreStrategy{ +public abstract class AbstractRestore extends Task implements IRestoreStrategy { // keeps track of the last few download which was executed. // TODO fix the magic number of 1000 => the idea of 80% of 1000 files limit per s3 query - static final FifoQueue tracker = new FifoQueue(800); + protected static final FifoQueue tracker = new FifoQueue<>(800); private static final Logger logger = LoggerFactory.getLogger(AbstractRestore.class); private static final String JOBNAME = "AbstractRestore"; private static final String SYSTEM_KEYSPACE = "system"; @@ -76,7 +79,7 @@ public abstract class AbstractRestore extends Task implements IRestoreStrategy{ this.cassProcess = cassProcess; this.metaData = metaData; this.instanceState = instanceState; - backupRestoreUtil = new BackupRestoreUtil(config.getRestoreKeyspaceFilter(), config.getRestoreCFFilter()); + backupRestoreUtil = new BackupRestoreUtil(config.getRestoreIncludeCFList(), config.getRestoreExcludeCFList()); this.postRestoreHook = postRestoreHook; } @@ -86,7 +89,12 @@ public static final boolean isRestoreEnabled(IConfiguration conf) { return (isRestoreMode && isBackedupRac); } - private final void download(Iterator fsIterator, BackupFileType bkupFileType) throws Exception { + public void setRestoreConfiguration(String restoreIncludeCFList, String restoreExcludeCFList) { + backupRestoreUtil.setFilters(restoreIncludeCFList, restoreExcludeCFList); + } + + private final List> download(Iterator fsIterator, BackupFileType bkupFileType, boolean waitForCompletion) throws Exception { + List> futureList = new ArrayList<>(); while (fsIterator.hasNext()) { AbstractBackupPath temp = fsIterator.next(); if (temp.getType() == BackupFileType.SST && tracker.contains(temp)) @@ -98,27 +106,29 @@ private final void download(Iterator fsIterator, BackupFileT continue; } - if (config.getRestoreKeySpaces().size() != 0 && (!config.getRestoreKeySpaces().contains(temp.getKeyspace()) || temp.getKeyspace().equals(SYSTEM_KEYSPACE))) { - logger.info("Bypassing restoring file \"{}\" as it is system keyspace", temp.newRestoreFile()); - continue; - } - - if (temp.getType() == bkupFileType) - { + if (temp.getType() == bkupFileType) { File localFileHandler = temp.newRestoreFile(); if (logger.isDebugEnabled()) logger.debug("Created local file name: " + localFileHandler.getAbsolutePath() + File.pathSeparator + localFileHandler.getName()); - downloadFile(temp, localFileHandler); + futureList.add(downloadFile(temp, localFileHandler)); } } //Wait for all download to finish that were started from this method. - waitToComplete(); + if (waitForCompletion) + waitForCompletion(futureList); + + return futureList; + } + + private void waitForCompletion(List> futureList) throws Exception { + for (Future future : futureList) + future.get(); } - private final void downloadCommitLogs(Iterator fsIterator, BackupFileType filter, int lastN) throws Exception { + private final List> downloadCommitLogs(Iterator fsIterator, BackupFileType filter, int lastN, boolean waitForCompletion) throws Exception { if (fsIterator == null) - return; + return null; BoundedList bl = new BoundedList(lastN); while (fsIterator.hasNext()) { @@ -131,13 +141,11 @@ private final void downloadCommitLogs(Iterator fsIterator, B } } - download(bl.iterator(), filter); + return download(bl.iterator(), filter, waitForCompletion); } - - private void stopCassProcess() throws IOException { - if (config.getRestoreKeySpaces().size() == 0) - cassProcess.stop(true); + private final void stopCassProcess() throws IOException { + cassProcess.stop(true); } private String getRestorePrefix() { @@ -198,7 +206,7 @@ public Void retriableCall() throws Exception { public void restore(Date startTime, Date endTime) throws Exception { //fail early if post restore hook has invalid parameters - if(!postRestoreHook.hasValidParameters()) { + if (!postRestoreHook.hasValidParameters()) { throw new PostRestoreHookException("Invalid PostRestoreHook parameters"); } @@ -216,11 +224,13 @@ public void restore(Date startTime, Date endTime) throws Exception { id.getInstance().setToken(restoreToken.toString()); } - // Stop cassandra if its running and restoring all keyspaces + // Stop cassandra if its running stopCassProcess(); // Cleanup local data - SystemUtils.cleanupDir(config.getDataFileLocation(), config.getRestoreKeySpaces()); + File dataDir = new File(config.getDataFileLocation()); + if (dataDir.exists() && dataDir.isDirectory()) + FileUtils.cleanDirectory(dataDir); // Try and read the Meta file. List metas = Lists.newArrayList(); @@ -242,19 +252,19 @@ public void restore(Date startTime, Date endTime) throws Exception { //Download the meta.json file. ArrayList metaFile = new ArrayList<>(); metaFile.add(meta); - download(metaFile.iterator(), BackupFileType.META); - waitToComplete(); + download(metaFile.iterator(), BackupFileType.META, true); + List> futureList = new ArrayList<>(); //Parse meta.json file to find the files required to download from this snapshot. List snapshots = metaData.toJson(meta.newRestoreFile()); // Download snapshot which is listed in the meta file. - download(snapshots.iterator(), BackupFileType.SNAP); + futureList.addAll(download(snapshots.iterator(), BackupFileType.SNAP, false)); logger.info("Downloading incrementals"); // Download incrementals (SST) after the snapshot meta file. Iterator incrementals = fs.list(prefix, meta.getTime(), endTime); - download(incrementals, BackupFileType.SST); + futureList.addAll(download(incrementals, BackupFileType.SST, false)); //Downloading CommitLogs if (config.isBackingUpCommitLogs()) { @@ -265,11 +275,11 @@ public void restore(Date startTime, Date endTime) throws Exception { SystemUtils.cleanupDir(config.getCommitLogLocation(), null); Iterator commitLogPathIterator = fs.list(prefix, meta.getTime(), endTime); - downloadCommitLogs(commitLogPathIterator, BackupFileType.CL, config.maxCommitLogsRestore()); + futureList.addAll(downloadCommitLogs(commitLogPathIterator, BackupFileType.CL, config.maxCommitLogsRestore(), false)); } - // Ensure all the files are downloaded. - waitToComplete(); + //Wait for all the futures to finish. + waitForCompletion(futureList); // Given that files are restored now, kick off post restore hook logger.info("Starting post restore hook"); @@ -297,16 +307,13 @@ public void restore(Date startTime, Date endTime) throws Exception { /** * Download file to the location specified. After downloading the file will be decrypted(optionally) and decompressed before saving to final location. + * * @param path - path of object to download from source S3/GCS. * @param restoreLocation - path to the final location of the decompressed and/or decrypted file. + * @return Future of the job to track the progress of the job. + * @throws Exception If there is any error in downloading file from the remote file system. */ - protected abstract void downloadFile(final AbstractBackupPath path, final File restoreLocation) throws Exception; - - /** - * A means to wait until until all threads have completed. It blocks calling thread - * until all tasks are completed. - */ - protected abstract void waitToComplete(); + protected abstract Future downloadFile(final AbstractBackupPath path, final File restoreLocation) throws Exception; final class BoundedList extends LinkedList { @@ -325,4 +332,8 @@ public boolean add(E o) { return true; } } + + public final int getDownloadTasksQueued() { + return fs.getDownloadTasksQueued(); + } } diff --git a/priam/src/main/java/com/netflix/priam/restore/EncryptedRestoreBase.java b/priam/src/main/java/com/netflix/priam/restore/EncryptedRestoreBase.java index b7bf444ae..4d8c514c5 100755 --- a/priam/src/main/java/com/netflix/priam/restore/EncryptedRestoreBase.java +++ b/priam/src/main/java/com/netflix/priam/restore/EncryptedRestoreBase.java @@ -32,7 +32,9 @@ import org.slf4j.LoggerFactory; import java.io.*; +import java.nio.file.Path; import java.nio.file.Paths; +import java.util.concurrent.Future; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.atomic.AtomicInteger; @@ -47,7 +49,6 @@ public abstract class EncryptedRestoreBase extends AbstractRestore{ private IFileCryptography fileCryptography; private ICompression compress; private final ThreadPoolExecutor executor; - private AtomicInteger count = new AtomicInteger(); protected EncryptedRestoreBase(IConfiguration config, IBackupFileSystem fs, String jobName, Sleeper sleeper, ICassandraProcess cassProcess, Provider pathProvider, @@ -59,23 +60,21 @@ protected EncryptedRestoreBase(IConfiguration config, IBackupFileSystem fs, Stri this.pgpCredential = pgpCredential; this.fileCryptography = fileCryptography; this.compress = compress; - executor = new NamedThreadPoolExecutor(config.getMaxBackupDownloadThreads(), jobName); + executor = new NamedThreadPoolExecutor(config.getRestoreThreads(), jobName); executor.allowCoreThreadTimeOut(true); logger.info("Trying to restore cassandra cluster with filesystem: {}, RestoreStrategy: {}, Encryption: ON, Compression: {}", fs.getClass(), jobName, compress.getClass()); } @Override - protected final void downloadFile(final AbstractBackupPath path, final File restoreLocation) throws Exception{ + protected final Future downloadFile(final AbstractBackupPath path, final File restoreLocation) throws Exception{ final char[] passPhrase = new String(this.pgpCredential.getValue(ICredentialGeneric.KEY.PGP_PASSWORD)).toCharArray(); File tempFile = new File(restoreLocation.getAbsolutePath() + ".tmp"); - count.incrementAndGet(); - try { - executor.submit(new RetryableCallable() { + return executor.submit(new RetryableCallable() { @Override - public Integer retriableCall() throws Exception { + public Path retriableCall() throws Exception { //== download object from source bucket try { @@ -132,27 +131,10 @@ public Integer retriableCall() throws Exception { decryptedFile.delete(); } - return count.decrementAndGet(); + return Paths.get(path.getRemotePath()); } - }); - }catch (Exception e){ - throw new Exception("Exception in download of: " + path.getFileName() + ", msg: " + e.getLocalizedMessage(), e); - } - - } - - @Override - protected final void waitToComplete() { - while (count.get() != 0) { - try { - sleeper.sleep(1000); - } catch (InterruptedException e) { - logger.error("Interrupted: ", e); - Thread.currentThread().interrupt(); - } - } - } + }); } @Override public String getName() { diff --git a/priam/src/main/java/com/netflix/priam/restore/Restore.java b/priam/src/main/java/com/netflix/priam/restore/Restore.java index 9fbbd68d8..41baf7b80 100644 --- a/priam/src/main/java/com/netflix/priam/restore/Restore.java +++ b/priam/src/main/java/com/netflix/priam/restore/Restore.java @@ -27,19 +27,16 @@ import com.netflix.priam.backup.MetaData; import com.netflix.priam.health.InstanceState; import com.netflix.priam.identity.InstanceIdentity; -import com.netflix.priam.scheduler.NamedThreadPoolExecutor; import com.netflix.priam.scheduler.SimpleTimer; import com.netflix.priam.scheduler.TaskTimer; -import com.netflix.priam.utils.RetryableCallable; import com.netflix.priam.utils.Sleeper; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.File; -import java.io.FileOutputStream; +import java.nio.file.Path; import java.nio.file.Paths; -import java.util.concurrent.ThreadPoolExecutor; -import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.Future; /** * Main class for restoring data from backup. Backup restored using this way are not encrypted. @@ -48,36 +45,18 @@ public class Restore extends AbstractRestore { public static final String JOBNAME = "AUTO_RESTORE_JOB"; private static final Logger logger = LoggerFactory.getLogger(Restore.class); - private final ThreadPoolExecutor executor; - private AtomicInteger count = new AtomicInteger(); @Inject public Restore(IConfiguration config, @Named("backup") IBackupFileSystem fs, Sleeper sleeper, ICassandraProcess cassProcess, Provider pathProvider, InstanceIdentity instanceIdentity, RestoreTokenSelector tokenSelector, MetaData metaData, InstanceState instanceState, IPostRestoreHook postRestoreHook) { super(config, fs, JOBNAME, sleeper, pathProvider, instanceIdentity, tokenSelector, cassProcess, metaData, instanceState, postRestoreHook); - executor = new NamedThreadPoolExecutor(config.getMaxBackupDownloadThreads(), JOBNAME); - executor.allowCoreThreadTimeOut(true); } @Override - protected final void downloadFile(final AbstractBackupPath path, final File restoreLocation) throws Exception { - count.incrementAndGet(); - fs.downloadFile(Paths.get(path.getRemotePath()), Paths.get(restoreLocation.getAbsolutePath()), 5); + protected final Future downloadFile(final AbstractBackupPath path, final File restoreLocation) throws Exception { tracker.adjustAndAdd(path); - count.decrementAndGet(); - } - - @Override - protected final void waitToComplete() { - while (count.get() != 0) { - try { - sleeper.sleep(1000); - } catch (InterruptedException e) { - logger.error("Interrupted: ", e); - Thread.currentThread().interrupt(); - } - } + return fs.asyncDownloadFile(Paths.get(path.getRemotePath()), Paths.get(restoreLocation.getAbsolutePath()), 5); } public static TaskTimer getTimer() { @@ -88,8 +67,4 @@ public static TaskTimer getTimer() { public String getName() { return JOBNAME; } - - public int getActiveCount() { - return (executor == null) ? 0 : executor.getActiveCount(); - } } \ No newline at end of file diff --git a/priam/src/main/java/com/netflix/priam/restore/RestoreTokenSelector.java b/priam/src/main/java/com/netflix/priam/restore/RestoreTokenSelector.java index 6787fbe5b..9d80bb472 100644 --- a/priam/src/main/java/com/netflix/priam/restore/RestoreTokenSelector.java +++ b/priam/src/main/java/com/netflix/priam/restore/RestoreTokenSelector.java @@ -55,7 +55,7 @@ public RestoreTokenSelector(ITokenManager tokenManager, @Named("backup") IBackup * @return Token as BigInteger */ public BigInteger getClosestToken(BigInteger tokenToSearch, Date startDate) { - List tokenList = new ArrayList(); + List tokenList = new ArrayList<>(); Iterator iter = fs.listPrefixes(startDate); while (iter.hasNext()) tokenList.add(new BigInteger(iter.next().getToken())); diff --git a/priam/src/main/java/com/netflix/priam/scheduler/NamedThreadPoolExecutor.java b/priam/src/main/java/com/netflix/priam/scheduler/NamedThreadPoolExecutor.java index d078b5dae..2a6f282df 100644 --- a/priam/src/main/java/com/netflix/priam/scheduler/NamedThreadPoolExecutor.java +++ b/priam/src/main/java/com/netflix/priam/scheduler/NamedThreadPoolExecutor.java @@ -22,7 +22,7 @@ public class NamedThreadPoolExecutor extends ThreadPoolExecutor { public NamedThreadPoolExecutor(int poolSize, String poolName) { - this(poolSize, poolName, new LinkedBlockingQueue()); + this(poolSize, poolName, new LinkedBlockingQueue<>()); } public NamedThreadPoolExecutor(int poolSize, String poolName, BlockingQueue queue) { diff --git a/priam/src/main/java/com/netflix/priam/scheduler/PriamScheduler.java b/priam/src/main/java/com/netflix/priam/scheduler/PriamScheduler.java index f6bfce851..08339c07f 100644 --- a/priam/src/main/java/com/netflix/priam/scheduler/PriamScheduler.java +++ b/priam/src/main/java/com/netflix/priam/scheduler/PriamScheduler.java @@ -71,16 +71,14 @@ public void addTaskWithDelay(final String name, Class taskclass, final JobDetail job = JobBuilder.newJob().withIdentity(name, Scheduler.DEFAULT_GROUP).ofType(taskclass).build();//new JobDetail(name, Scheduler.DEFAULT_GROUP, taskclass); //we know Priam doesn't do too many new tasks, so this is probably easy/safe/simple - new Thread(new Runnable() { - public void run() { - try { - sleeper.sleepQuietly(delayInSeconds * 1000L); - scheduler.scheduleJob(job, timer.getTrigger()); - } catch (SchedulerException e) { - logger.warn("problem occurred while scheduling a job with name {}", name, e); - } catch (ParseException e) { - logger.warn("problem occurred while parsing a job with name {}", name, e); - } + new Thread(() -> { + try { + sleeper.sleepQuietly(delayInSeconds * 1000L); + scheduler.scheduleJob(job, timer.getTrigger()); + } catch (SchedulerException e) { + logger.warn("problem occurred while scheduling a job with name {}", name, e); + } catch (ParseException e) { + logger.warn("problem occurred while parsing a job with name {}", name, e); } }).start(); } diff --git a/priam/src/main/java/com/netflix/priam/scheduler/Task.java b/priam/src/main/java/com/netflix/priam/scheduler/Task.java index a8e9d6c80..f9e4e0248 100644 --- a/priam/src/main/java/com/netflix/priam/scheduler/Task.java +++ b/priam/src/main/java/com/netflix/priam/scheduler/Task.java @@ -85,10 +85,6 @@ public void execute(JobExecutionContext context) throws JobExecutionException { status = STATE.RUNNING; execute(); - } catch (Exception e) { - status = STATE.ERROR; - logger.error("Couldnt execute the task because of {}", e.getMessage(), e); - errors.incrementAndGet(); } catch (Throwable e) { status = STATE.ERROR; logger.error("Couldnt execute the task because of {}", e.getMessage(), e); diff --git a/priam/src/main/java/com/netflix/priam/services/SnapshotMetaService.java b/priam/src/main/java/com/netflix/priam/services/SnapshotMetaService.java index 3e75f7fda..0f311af96 100644 --- a/priam/src/main/java/com/netflix/priam/services/SnapshotMetaService.java +++ b/priam/src/main/java/com/netflix/priam/services/SnapshotMetaService.java @@ -74,7 +74,7 @@ public class SnapshotMetaService extends AbstractBackup { MetaFileWriterBuilder metaFileWriter, MetaFileManager metaFileManager, CassandraOperations cassandraOperations) { super(config, backupFileSystemCtx, pathFactory); this.cassandraOperations = cassandraOperations; - backupRestoreUtil = new BackupRestoreUtil(config.getSnapshotKeyspaceFilters(), config.getSnapshotCFFilter()); + backupRestoreUtil = new BackupRestoreUtil(config.getSnapshotIncludeCFList(), config.getSnapshotExcludeCFList()); this.metaFileWriter = metaFileWriter; this.metaFileManager = metaFileManager; } @@ -162,10 +162,10 @@ public String getName() { private ColumnfamilyResult convertToColumnFamilyResult(String keyspace, String columnFamilyName, Map> filePrefixToFileMap) { ColumnfamilyResult columnfamilyResult = new ColumnfamilyResult(keyspace, columnFamilyName); - filePrefixToFileMap.entrySet().forEach(sstableEntry -> { + filePrefixToFileMap.forEach((key, value) -> { ColumnfamilyResult.SSTableResult ssTableResult = new ColumnfamilyResult.SSTableResult(); - ssTableResult.setPrefix(sstableEntry.getKey()); - ssTableResult.setSstableComponents(sstableEntry.getValue()); + ssTableResult.setPrefix(key); + ssTableResult.setSstableComponents(value); columnfamilyResult.addSstable(ssTableResult); }); return columnfamilyResult; diff --git a/priam/src/main/java/com/netflix/priam/tuner/StandardTuner.java b/priam/src/main/java/com/netflix/priam/tuner/StandardTuner.java index c7636afa9..b60166088 100644 --- a/priam/src/main/java/com/netflix/priam/tuner/StandardTuner.java +++ b/priam/src/main/java/com/netflix/priam/tuner/StandardTuner.java @@ -201,12 +201,8 @@ protected void configureCommitLogBackups() throws IOException { props.put("restore_directories", config.getCommitLogBackupRestoreFromDirs()); props.put("restore_point_in_time", config.getCommitLogBackupRestorePointInTime()); - FileOutputStream fos = null; - try { - fos = new FileOutputStream(new File(config.getCommitLogBackupPropsFile())); + try(FileOutputStream fos = new FileOutputStream(new File(config.getCommitLogBackupPropsFile()))) { props.store(fos, "cassandra commit log archive props, as written by priam"); - } finally { - IOUtils.closeQuietly(fos); } } diff --git a/priam/src/main/java/com/netflix/priam/utils/FifoQueue.java b/priam/src/main/java/com/netflix/priam/utils/FifoQueue.java index 5e21a1515..aa31d26f8 100644 --- a/priam/src/main/java/com/netflix/priam/utils/FifoQueue.java +++ b/priam/src/main/java/com/netflix/priam/utils/FifoQueue.java @@ -24,12 +24,7 @@ public class FifoQueue> extends TreeSet { private int capacity; public FifoQueue(int capacity) { - super(new Comparator() { - @Override - public int compare(E o1, E o2) { - return o1.compareTo(o2); - } - }); + super(Comparator.naturalOrder()); this.capacity = capacity; } diff --git a/priam/src/main/java/com/netflix/priam/utils/JMXNodeTool.java b/priam/src/main/java/com/netflix/priam/utils/JMXNodeTool.java index c393481fa..b4f745e2e 100644 --- a/priam/src/main/java/com/netflix/priam/utils/JMXNodeTool.java +++ b/priam/src/main/java/com/netflix/priam/utils/JMXNodeTool.java @@ -53,7 +53,7 @@ public class JMXNodeTool extends NodeProbe implements INodeToolObservable { private static volatile JMXNodeTool tool = null; private MBeanServerConnection mbeanServerConn = null; - private static Set observers = new HashSet(); + private static Set observers = new HashSet<>(); /** * Hostname and Port to talk to will be same server for now optionally we @@ -200,9 +200,7 @@ public JMXNodeTool retriableCall() throws Exception { } logger.info("Connected to remote jmx agent, will notify interested parties!"); - Iterator it = observers.iterator(); - while (it.hasNext()) { - INodeToolObserver observer = it.next(); + for (INodeToolObserver observer : observers) { observer.nodeToolHasChanged(tool); } @@ -248,7 +246,7 @@ public JSONObject info() throws JSONException { public JSONArray ring(String keyspace) throws JSONException { JSONArray ring = new JSONArray(); Map tokenToEndpoint = getTokenToEndpointMap(); - List sortedTokens = new ArrayList(tokenToEndpoint.keySet()); + List sortedTokens = new ArrayList<>(tokenToEndpoint.keySet()); Collection liveNodes = getLiveNodes(); Collection deadNodes = getUnreachableNodes(); @@ -296,9 +294,7 @@ else if (leavingNodes.contains(primaryEndpoint)) else if (movingNodes.contains(primaryEndpoint)) state = "Moving"; - String load = loadMap.containsKey(primaryEndpoint) - ? loadMap.get(primaryEndpoint) - : "?"; + String load = loadMap.getOrDefault(primaryEndpoint, "?"); String owns = new DecimalFormat("##0.00%").format(ownerships.get(token) == null ? 0.0F : ownerships.get(token)); ring.put(createJson(primaryEndpoint, dataCenter, rack, status, state, load, owns, token)); } diff --git a/priam/src/main/java/com/netflix/priam/utils/SystemUtils.java b/priam/src/main/java/com/netflix/priam/utils/SystemUtils.java index 87bfd2a2f..3c9c48fbe 100644 --- a/priam/src/main/java/com/netflix/priam/utils/SystemUtils.java +++ b/priam/src/main/java/com/netflix/priam/utils/SystemUtils.java @@ -61,10 +61,12 @@ public static String getDataFromUrl(String url) { } - /** - * delete all the files/dirs in the given Directory but dont delete the dir - * itself. + * delete all the files/dirs in the given Directory but dont delete the dir itself. + * + * @param dirPath The directory path where all the child directories exist. + * @param childdirs List of child directories to be cleaned up in the dirPath + * @throws IOException If there is any error encountered during cleanup. */ public static void cleanupDir(String dirPath, List childdirs) throws IOException { if (childdirs == null || childdirs.size() == 0) @@ -95,7 +97,10 @@ public static byte[] md5(byte[] buf) { } /** - * Get a Md5 string which is similar to OS Md5sum + * Calculate the MD5 hashsum of the given file. + * + * @param file File for which md5 checksum should be calculated. + * @return Get a Md5 string which is similar to OS Md5sum */ public static String md5(File file) { try { diff --git a/priam/src/test/groovy/com.netflix.priam/backup/TestBackupRestoreUtil.groovy b/priam/src/test/groovy/com.netflix.priam/backup/TestBackupRestoreUtil.groovy index a70c02ce6..c4582e22b 100644 --- a/priam/src/test/groovy/com.netflix.priam/backup/TestBackupRestoreUtil.groovy +++ b/priam/src/test/groovy/com.netflix.priam/backup/TestBackupRestoreUtil.groovy @@ -8,44 +8,45 @@ import spock.lang.Unroll */ @Unroll class TestBackupRestoreUtil extends Specification { - def "IsFilter for KS #keyspace with configuration #configKeyspaceFilter is #result"() { + def "IsFilter for KS #keyspace and CF #columnfamily with configuration include #configIncludeFilter and exclude #configExcludeFilter is #result"() { expect: - new BackupRestoreUtil(configKeyspaceFilter, configCFFilter).isFiltered(BackupRestoreUtil.DIRECTORYTYPE.KEYSPACE, keyspace, columnfamily) == result + new BackupRestoreUtil(configIncludeFilter, configExcludeFilter).isFiltered(keyspace, columnfamily) == result where: - configKeyspaceFilter | configCFFilter | keyspace | columnfamily || result - "abc" | null | "abc" | null || true - "abc" | "ab.ab" | "ab" | null || false - "abc" | null | "ab" | null || false - "abc,def" | null | "abc" | null || true - "abc,def" | null | "def" | null || true - "abc,def" | null | "ab" | null || false - "abc,def" | null | "df" | null || false - "ab.*" | null | "ab" | null || true - "ab.*,def" | null | "ab" | null || true - "ab.*,de.*" | null | "ab" | null || true - "ab.*,de.*" | null | "abab" | null || true - "ab.*,de.*" | null | "defg" | null || true - null | null | "defg" | null || false + configIncludeFilter | configExcludeFilter | keyspace | columnfamily || result + null | null | "defg" | "gh" || false + "abc.*" | null | "abc" | "cd" || false + "abc.*" | null | "ab" | "cd" || true + null | "abc.de" | "abc" | "def" || false + null | "abc.de" | "abc" | "de" || true + "abc.*,def.*" | null | "abc" | "cd" || false + "abc.*,def.*" | null | "def" | "ab" || false + "abc.*,def.*" | null | "ab" | "cd" || true + "abc.*,def.*" | null | "df" | "ab" || true + null | "abc.de,fg.hi" | "abc" | "def" || false + null | "abc.de,fg.hi" | "abc" | "de" || true + null | "abc.de,fg.hi" | "fg" | "hijk" || false + null | "abc.de,fg.hi" | "fg" | "hi" || true + "abc.*" | "ab.ab" | "ab" | "cd" || true + "abc.*" | "ab.ab" | "ab" | "ab" || true + "abc.*" | "abc.ab" | "abc" | "ab" || true + "abc.*" | "abc.ab" | "abc" | "cd" || false + "abc.cd" | "abc.*" | "abc" | "cd" || true + "abc.*" | "abc.*" | "abc" | "cd" || true + "abc.*,def.*" | "abc.*" | "def" | "ab" || false } - def "IsFilter for CF #columnfamily with configuration #configCFFilter is #result"() { - expect: - new BackupRestoreUtil(configKeyspaceFilter, configCFFilter).isFiltered(BackupRestoreUtil.DIRECTORYTYPE.CF, keyspace, columnfamily) == result - where: - configKeyspaceFilter | configCFFilter | keyspace | columnfamily || result - "abc" | null | "abc" | null || false - "abc" | "ab.ab" | "ks" | "ab" || false - "abc" | "ab.ab" | "ab" | "ab.ab" || true - "abc" | "ab.ab,de.fg" | "ab" | "ab.ab" || true - "abc" | "ab.ab,de.fg" | "de" | "fg" || true - null | "abc.de.*" | "abc" | "def" || true - null | "abc.de.*" | "abc" | "abc.def" || true - null | "abc.de.*,fg.hi.*" | "abc" | "def" || true - null | "abc.de.*,fg.hi.*" | "abc" | "abc.def" || true - null | "abc.de.*,fg.hi.*" | "fg" | "hijk" || true - null | "abc.de.*,fg.hi.*" | "fg" | "fg.hijk" || true + def "Expected exception KS #keyspace and CF #columnfamily with configuration include #configIncludeFilter and exclude #configExcludeFilter"() { + when: + new BackupRestoreUtil(configIncludeFilter, configExcludeFilter).isFiltered(keyspace, columnfamily) + then: + thrown(ExcpectedException) + + where: + configIncludeFilter | configExcludeFilter | keyspace | columnfamily || ExcpectedException + null | "def" | "defg" | null || IllegalArgumentException + "abc" | null | null | "cd" || IllegalArgumentException } } diff --git a/priam/src/test/java/com/netflix/priam/backup/FakeBackupFileSystem.java b/priam/src/test/java/com/netflix/priam/backup/FakeBackupFileSystem.java index d8fb37a97..b5adcbe88 100644 --- a/priam/src/test/java/com/netflix/priam/backup/FakeBackupFileSystem.java +++ b/priam/src/test/java/com/netflix/priam/backup/FakeBackupFileSystem.java @@ -29,8 +29,6 @@ import java.io.FileWriter; import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; import java.nio.file.Path; import java.util.*; diff --git a/priam/src/test/java/com/netflix/priam/backup/TestAbstractFileSystem.java b/priam/src/test/java/com/netflix/priam/backup/TestAbstractFileSystem.java index 6bb76d29d..f66403c7c 100644 --- a/priam/src/test/java/com/netflix/priam/backup/TestAbstractFileSystem.java +++ b/priam/src/test/java/com/netflix/priam/backup/TestAbstractFileSystem.java @@ -39,10 +39,7 @@ import java.util.Collection; import java.util.List; import java.util.Random; -import java.util.concurrent.Callable; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.Future; +import java.util.concurrent.*; /** * The goal of this class is to test common functionality which are encapsulated in AbstractFileSystem. @@ -86,7 +83,7 @@ public void testFailedRetriesUpload() throws Exception { try { Collection files = generateFiles(1, 1, 1); for (File file : files) { - failureFileSystem.uploadFile(file.toPath(), null, getDummyPath(file.toPath()), 2, true); + failureFileSystem.uploadFile(file.toPath(), file.toPath(), getDummyPath(file.toPath()), 2, true); } } catch (BackupRestoreException e) { //Verify the failure metric for upload is incremented. @@ -122,7 +119,7 @@ public void testUpload() throws Exception { Collection files = generateFiles(1, 1, 1); //Dummy upload with compressed size. for (File file : files) { - myFileSystem.uploadFile(file.toPath(), null, getDummyPath(file.toPath()), 2, true); + myFileSystem.uploadFile(file.toPath(), file.toPath(), getDummyPath(file.toPath()), 2, true); //Verify the success metric for upload is incremented. Assert.assertEquals(1, (int) backupMetrics.getValidUploads().actualCount()); @@ -145,11 +142,11 @@ public void testAsyncUpload() throws Exception { //Testing single async upload. Collection files = generateFiles(1, 1, 1); for (File file : files) { - myFileSystem.asyncUploadFile(file.toPath(), null, getDummyPath(file.toPath()), 2, true).get(); + myFileSystem.asyncUploadFile(file.toPath(), file.toPath(), getDummyPath(file.toPath()), 2, true).get(); //1. Verify the success metric for upload is incremented. Assert.assertEquals(1, (int) backupMetrics.getValidUploads().actualCount()); //2. The task queue is empty after upload is finished. - Assert.assertEquals(0, myFileSystem.getTasksQueued()); + Assert.assertEquals(0, myFileSystem.getUploadTasksQueued()); break; } } @@ -161,7 +158,7 @@ public void testAsyncUploadBulk() throws Exception { Collection files = generateFiles(1, 1, 20); List> futures = new ArrayList<>(); for (File file : files) { - futures.add(myFileSystem.asyncUploadFile(file.toPath(), null, getDummyPath(file.toPath()), 2, true)); + futures.add(myFileSystem.asyncUploadFile(file.toPath(), file.toPath(), getDummyPath(file.toPath()), 2, true)); } //Verify all the work is finished. @@ -172,7 +169,7 @@ public void testAsyncUploadBulk() throws Exception { Assert.assertEquals(files.size(), (int) backupMetrics.getValidUploads().actualCount()); //3. The task queue is empty after upload is finished. - Assert.assertEquals(0, myFileSystem.getTasksQueued()); + Assert.assertEquals(0, myFileSystem.getUploadTasksQueued()); } @Test @@ -187,7 +184,7 @@ public void testUploadDedup() throws Exception { List> torun = new ArrayList<>(size); for (int i = 0; i < size; i++) { torun.add(() -> { - myFileSystem.uploadFile(file.toPath(), null, abstractBackupPath, 2, true); + myFileSystem.uploadFile(file.toPath(), file.toPath(), abstractBackupPath, 2, true); return Boolean.TRUE; }); } @@ -198,7 +195,11 @@ public void testUploadDedup() throws Exception { // no more need for the threadpool threads.shutdown(); for (Future future : futures) { - future.get(); + try { + future.get(); + }catch (InterruptedException | ExecutionException e){ + //Do nothing. + } } //2. Verify the success metric for upload is not same as size, i.e. some amount of de-duping happened. Assert.assertNotEquals(size, (int) backupMetrics.getValidUploads().actualCount()); @@ -209,7 +210,7 @@ public void testAsyncUploadFailure() throws Exception { //Testing single async upload. Collection files = generateFiles(1, 1, 1); for (File file : files) { - Future future = failureFileSystem.asyncUploadFile(file.toPath(), null, getDummyPath(file.toPath()), 2, true); + Future future = failureFileSystem.asyncUploadFile(file.toPath(), file.toPath(), getDummyPath(file.toPath()), 2, true); try { future.get(); } catch (Exception e) { @@ -219,7 +220,7 @@ public void testAsyncUploadFailure() throws Exception { Assert.assertEquals(1, (int) backupMetrics.getInvalidUploads().count()); //3. The task queue is empty after upload is finished. - Assert.assertEquals(0, failureFileSystem.getTasksQueued()); + Assert.assertEquals(0, failureFileSystem.getUploadTasksQueued()); break; } } @@ -228,12 +229,12 @@ public void testAsyncUploadFailure() throws Exception { @Test public void testAsyncDownload() throws Exception { //Testing single async download. - Future future = myFileSystem.downloadFileAsync(Paths.get(""), null, 2); + Future future = myFileSystem.asyncDownloadFile(Paths.get(""), null, 2); future.get(); //1. Verify the success metric for download is incremented. Assert.assertEquals(1, (int) backupMetrics.getValidDownloads().actualCount()); //2. Verify the queue size is '0' after success. - Assert.assertEquals(0, myFileSystem.getTasksQueued()); + Assert.assertEquals(0, myFileSystem.getDownloadTasksQueued()); } @Test @@ -243,7 +244,7 @@ public void testAsyncDownloadBulk() throws Exception { int totalFiles = 1000; List> futureList = new ArrayList<>(); for (int i = 0; i < totalFiles; i++) - futureList.add(myFileSystem.downloadFileAsync(Paths.get("" + i), null, 2)); + futureList.add(myFileSystem.asyncDownloadFile(Paths.get("" + i), null, 2)); //Ensure processing is finished. for (Future future1 : futureList) { @@ -254,12 +255,12 @@ public void testAsyncDownloadBulk() throws Exception { Assert.assertEquals(totalFiles, (int) backupMetrics.getValidDownloads().actualCount()); //3. The task queue is empty after download is finished. - Assert.assertEquals(0, myFileSystem.getTasksQueued()); + Assert.assertEquals(0, myFileSystem.getDownloadTasksQueued()); } @Test public void testAsyncDownloadFailure() throws Exception { - Future future = failureFileSystem.downloadFileAsync(Paths.get(""), null, 2); + Future future = failureFileSystem.asyncDownloadFile(Paths.get(""), null, 2); try { future.get(); } catch (Exception e) { diff --git a/priam/src/test/java/com/netflix/priam/backup/TestCompression.java b/priam/src/test/java/com/netflix/priam/backup/TestCompression.java index 30149be43..a899d45a6 100644 --- a/priam/src/test/java/com/netflix/priam/backup/TestCompression.java +++ b/priam/src/test/java/com/netflix/priam/backup/TestCompression.java @@ -45,13 +45,12 @@ public class TestCompression public void setup() throws IOException { File f = new File("/tmp/compress-test.txt"); - FileOutputStream stream = new FileOutputStream(f); - for (int i = 0; i < (1000 * 1000); i++) - { - stream.write("This is a test... Random things happen... and you are responsible for it...\n".getBytes("UTF-8")); - stream.write("The quick brown fox jumps over the lazy dog.The quick brown fox jumps over the lazy dog.The quick brown fox jumps over the lazy dog.\n".getBytes("UTF-8")); + try(FileOutputStream stream = new FileOutputStream(f)) { + for (int i = 0; i < (1000 * 1000); i++) { + stream.write("This is a test... Random things happen... and you are responsible for it...\n".getBytes("UTF-8")); + stream.write("The quick brown fox jumps over the lazy dog.The quick brown fox jumps over the lazy dog.The quick brown fox jumps over the lazy dog.\n".getBytes("UTF-8")); + } } - IOUtils.closeQuietly(stream); } @After @@ -73,44 +72,38 @@ void validateCompression(String uncompress, String compress) public void zip() throws IOException { BufferedInputStream source = null; - FileOutputStream dest = new FileOutputStream("/tmp/compressed.zip"); - ZipOutputStream out = new ZipOutputStream(new BufferedOutputStream(dest)); - byte data[] = new byte[2048]; - File file = new File("/tmp/compress-test.txt"); - FileInputStream fi = new FileInputStream(file); - source = new BufferedInputStream(fi, 2048); - ZipEntry entry = new ZipEntry(file.getName()); - out.putNextEntry(entry); - int count; - while ((count = source.read(data, 0, 2048)) != -1) - { - out.write(data, 0, count); + try(ZipOutputStream out = new ZipOutputStream(new BufferedOutputStream(new FileOutputStream("/tmp/compressed.zip")))) { + byte data[] = new byte[2048]; + File file = new File("/tmp/compress-test.txt"); + FileInputStream fi = new FileInputStream(file); + source = new BufferedInputStream(fi, 2048); + ZipEntry entry = new ZipEntry(file.getName()); + out.putNextEntry(entry); + int count; + while ((count = source.read(data, 0, 2048)) != -1) { + out.write(data, 0, count); + } } - IOUtils.closeQuietly(out); validateCompression("/tmp/compress-test.txt", "/tmp/compressed.zip"); } @Test - public void unzip() throws IOException - { - BufferedOutputStream dest1 = null; - BufferedInputStream is = null; + public void unzip() throws IOException { ZipFile zipfile = new ZipFile("/tmp/compressed.zip"); Enumeration e = zipfile.entries(); while (e.hasMoreElements()) { ZipEntry entry = (ZipEntry) e.nextElement(); - is = new BufferedInputStream(zipfile.getInputStream(entry)); - int c; - byte d[] = new byte[2048]; - FileOutputStream fos = new FileOutputStream("/tmp/compress-test-out-0.txt"); - dest1 = new BufferedOutputStream(fos, 2048); - while ((c = is.read(d, 0, 2048)) != -1) - { - dest1.write(d, 0, c); + try(BufferedInputStream is = new BufferedInputStream(zipfile.getInputStream(entry)); + BufferedOutputStream dest1 = new BufferedOutputStream(new FileOutputStream("/tmp/compress-test-out-0.txt"), 2048)) { + ; + int c; + byte d[] = new byte[2048]; + + while ((c = is.read(d, 0, 2048)) != -1) { + dest1.write(d, 0, c); + } } - IOUtils.closeQuietly(dest1); - IOUtils.closeQuietly(is); } String md1 = SystemUtils.md5(new File("/tmp/compress-test.txt")); String md2 = SystemUtils.md5(new File("/tmp/compress-test-out-0.txt")); diff --git a/priam/src/test/java/com/netflix/priam/backup/TestSnapshotStatusMgr.java b/priam/src/test/java/com/netflix/priam/backup/TestSnapshotStatusMgr.java index a04fd190c..7d6a26cd5 100644 --- a/priam/src/test/java/com/netflix/priam/backup/TestSnapshotStatusMgr.java +++ b/priam/src/test/java/com/netflix/priam/backup/TestSnapshotStatusMgr.java @@ -21,8 +21,8 @@ import com.google.inject.Injector; import com.netflix.priam.config.IConfiguration; import com.netflix.priam.utils.DateUtil; -import junit.framework.Assert; import org.joda.time.DateTime; +import org.junit.Assert; import org.junit.BeforeClass; import org.junit.Test; import org.slf4j.Logger; diff --git a/priam/src/test/java/com/netflix/priam/backupv2/TestLocalDBReaderWriter.java b/priam/src/test/java/com/netflix/priam/backupv2/TestLocalDBReaderWriter.java index bcdaa3e29..8585828b0 100644 --- a/priam/src/test/java/com/netflix/priam/backupv2/TestLocalDBReaderWriter.java +++ b/priam/src/test/java/com/netflix/priam/backupv2/TestLocalDBReaderWriter.java @@ -89,7 +89,7 @@ public void readWriteLocalDB() throws Exception { List localDBList = generateDummyLocalDB(noOfKeyspaces, noOfCf, noOfSstables); localDBList.stream().forEach(localDB -> { - FileUploadResult fileUploadResult = localDB.getLocalDb().get(0).getFileUploadResult(); + FileUploadResult fileUploadResult = localDB.getLocalDBEntries().get(0).getFileUploadResult(); final Path localDBPath = localDBReaderWriter.getLocalDBPath(fileUploadResult); try { localDBReaderWriter.writeLocalDB(localDBPath, localDB); @@ -105,7 +105,7 @@ public void readWriteLocalDB() throws Exception { //Read the database. LocalDBReaderWriter.LocalDB localDB = localDBReaderWriter.readLocalDB(cfLocalDBPath.toFile().listFiles()[0].toPath()); - Assert.assertEquals(EnumSet.allOf(Component.Type.class).size(), localDB.getLocalDb().size()); + Assert.assertEquals(EnumSet.allOf(Component.Type.class).size(), localDB.getLocalDBEntries().size()); } @Test @@ -113,7 +113,7 @@ public void upsertLocalDB() throws Exception{ LocalDBReaderWriter.LocalDB localDB = generateDummyLocalDB(1, 1, 1).get(0); //Lets do write with each LocalDBEntry first. - localDB.getLocalDb().stream().forEach(localDBEntry -> { + localDB.getLocalDBEntries().stream().forEach(localDBEntry -> { try { localDBReaderWriter.upsertLocalDBEntry(localDBEntry); } catch (Exception e) { @@ -122,25 +122,25 @@ public void upsertLocalDB() throws Exception{ }); //Verify the write has happened. - LocalDBReaderWriter.LocalDBEntry localDBEntry = localDBReaderWriter.getLocalDBEntry(localDB.getLocalDb().get(0).getFileUploadResult()); + LocalDBReaderWriter.LocalDBEntry localDBEntry = localDBReaderWriter.getLocalDBEntry(localDB.getLocalDBEntries().get(0).getFileUploadResult()); Assert.assertNotNull(localDBEntry); //Now lets see if we can write the same entry again?? LocalDBReaderWriter.LocalDB localDBUpsert = localDBReaderWriter.upsertLocalDBEntry(localDBEntry); - Assert.assertEquals(localDB.getLocalDb().size(), localDBUpsert.getLocalDb().size()); + Assert.assertEquals(localDB.getLocalDBEntries().size(), localDBUpsert.getLocalDBEntries().size()); //Now lets change the localDBEntry and see if upsert succeeds. localDBEntry.setTimeLastReferenced(DateUtil.getInstant()); localDBUpsert = localDBReaderWriter.upsertLocalDBEntry(localDBEntry); LocalDBReaderWriter.LocalDBEntry localDBEntryUpsert = localDBReaderWriter.getLocalDBEntry(localDBEntry.getFileUploadResult()); Assert.assertEquals(localDBEntryUpsert.getTimeLastReferenced(),localDBEntry.getTimeLastReferenced()); - Assert.assertEquals(localDB.getLocalDb().size(), localDBUpsert.getLocalDb().size()); + Assert.assertEquals(localDB.getLocalDBEntries().size(), localDBUpsert.getLocalDBEntries().size()); //Now change the file modification time. This should end up creating a new DB Entry. localDBEntry.getFileUploadResult().setLastModifiedTime(DateUtil.getInstant()); localDBUpsert = localDBReaderWriter.upsertLocalDBEntry(localDBEntry); localDBEntryUpsert = localDBReaderWriter.getLocalDBEntry(localDBEntry.getFileUploadResult()); - Assert.assertEquals(localDB.getLocalDb().size() + 1, localDBUpsert.getLocalDb().size()); + Assert.assertEquals(localDB.getLocalDBEntries().size() + 1, localDBUpsert.getLocalDBEntries().size()); Assert.assertEquals(localDBEntry.getFileUploadResult().getLastModifiedTime(), localDBEntryUpsert.getFileUploadResult().getLastModifiedTime()); } @@ -148,7 +148,7 @@ public void upsertLocalDB() throws Exception{ public void readConcurrentLocalDB() throws Exception{ List localDBList = generateDummyLocalDB(1, 1, 1); localDBList.stream().forEach(localDB -> { - FileUploadResult fileUploadResult = localDB.getLocalDb().get(0).getFileUploadResult(); + FileUploadResult fileUploadResult = localDB.getLocalDBEntries().get(0).getFileUploadResult(); final Path localDBPath = localDBReaderWriter.getLocalDBPath(fileUploadResult); try { localDBReaderWriter.writeLocalDB(localDBPath, localDB); @@ -157,7 +157,7 @@ public void readConcurrentLocalDB() throws Exception{ } }); - FileUploadResult sample = localDBList.get(0).getLocalDb().get(0).getFileUploadResult(); + FileUploadResult sample = localDBList.get(0).getLocalDBEntries().get(0).getFileUploadResult(); int size = 5; ExecutorService threads = Executors.newFixedThreadPool(size); @@ -183,12 +183,12 @@ public void readConcurrentLocalDB() throws Exception{ @Test public void writeConcurrentLocalDB() throws Exception{ LocalDBReaderWriter.LocalDB localDB = generateDummyLocalDB(1, 1, 1).get(0); - int size = localDB.getLocalDb().size(); + int size = localDB.getLocalDBEntries().size(); ExecutorService threads = Executors.newFixedThreadPool(size); List> torun = new ArrayList<>(size); for (int i = 0; i < size; i++) { int finalI = i; - torun.add(() -> localDBReaderWriter.upsertLocalDBEntry(localDB.getLocalDb().get(finalI))); + torun.add(() -> localDBReaderWriter.upsertLocalDBEntry(localDB.getLocalDBEntries().get(finalI))); } // all tasks executed in different threads, at 'once'. List> futures = threads.invokeAll(torun); @@ -207,8 +207,8 @@ public void writeConcurrentLocalDB() throws Exception{ } Assert.assertEquals(0, noOfBadRun); - LocalDBReaderWriter.LocalDB localDBRead = localDBReaderWriter.readLocalDB(localDBReaderWriter.getLocalDBPath(localDB.getLocalDb().get(0).getFileUploadResult())); - Assert.assertEquals(localDB.getLocalDb().size(), localDBRead.getLocalDb().size()); + LocalDBReaderWriter.LocalDB localDBRead = localDBReaderWriter.readLocalDB(localDBReaderWriter.getLocalDBPath(localDB.getLocalDBEntries().get(0).getFileUploadResult())); + Assert.assertEquals(localDB.getLocalDBEntries().size(), localDBRead.getLocalDBEntries().size()); } private List generateDummyLocalDB(int noOfKeyspaces, int noOfCf, int noOfSstables) throws Exception { @@ -233,7 +233,7 @@ private List generateDummyLocalDB(int noOfKeyspaces Path componentPath = Paths.get(dummyDataDirectoryLocation.toFile().getAbsolutePath(), keyspaceName, columnfamilyname, prefixName + "-" + type.name() + ".db"); FileUploadResult fileUploadResult = new FileUploadResult(componentPath, keyspaceName, columnfamilyname, DateUtil.getInstant(), DateUtil.getInstant(), random.nextLong()); LocalDBReaderWriter.LocalDBEntry localDBEntry = new LocalDBReaderWriter.LocalDBEntry(fileUploadResult, DateUtil.getInstant(), DateUtil.getInstant()); - localDB.getLocalDb().add(localDBEntry); + localDB.getLocalDBEntries().add(localDBEntry); } } } diff --git a/priam/src/test/java/com/netflix/priam/config/FakeConfiguration.java b/priam/src/test/java/com/netflix/priam/config/FakeConfiguration.java index bbf6bf6f7..37d2e1a86 100644 --- a/priam/src/test/java/com/netflix/priam/config/FakeConfiguration.java +++ b/priam/src/test/java/com/netflix/priam/config/FakeConfiguration.java @@ -44,6 +44,8 @@ public class FakeConfiguration implements IConfiguration public String instance_id; public String restorePrefix; public Map fakeConfig; + public Map fakeProperties = new HashMap<>(); + public FakeConfiguration() { @@ -70,8 +72,7 @@ public void setFakeConfig(String key, Object value) { } @Override - public void intialize() - { + public void initialize() { // TODO Auto-generated method stub } @@ -128,34 +129,7 @@ public List getRacs() } @Override - public int getJmxPort() - { - return 7199; - } - - @Override - public String getJmxUsername() - { - return null; - } - - @Override - public String getJmxPassword() - { - return null; - } - - /** - * @return Enables Remote JMX connections n C* - */ - @Override - public boolean enableRemoteJMX() { - return false; - } - - @Override - public int getThriftPort() - { + public int getThriftPort() { return 9160; } @@ -229,8 +203,7 @@ public String getRestoreSnapshot() } @Override - public String getAppName() - { + public String getAppName() { return appName; } @@ -241,31 +214,19 @@ public String getACLGroupName() } @Override - public int getMaxBackupUploadThreads() - { - // TODO Auto-generated method stub - return 2; - } - - @Override - public String getSDBInstanceIdentityRegion() - { + public String getSDBInstanceIdentityRegion() { // TODO Auto-generated method stub return null; } @Override - public String getDC() - { - // TODO Auto-generated method stub + public String getDC() { return this.region; } @Override - public int getMaxBackupDownloadThreads() - { - // TODO Auto-generated method stub - return 3; + public int getRestoreThreads() { + return 2; } public void setRestorePrefix(String prefix) @@ -341,16 +302,11 @@ public boolean isLocalBootstrapEnabled() { return false; } - @Override - public int getInMemoryCompactionLimit() { - return 8; - } - - @Override - public int getCompactionThroughput() { - // TODO Auto-generated method stub - return 0; - } + @Override + public int getCompactionThroughput() { + // TODO Auto-generated method stub + return 0; + } @Override public String getMaxDirectMemory() @@ -378,15 +334,8 @@ public String getCassStartupScript() } @Override - public List getRestoreKeySpaces() - { - return Lists.newArrayList(); - } - - @Override - public long getBackupChunkSize() - { - return 5L*1024*1024; + public long getBackupChunkSize() { + return 5L * 1024 * 1024; } @Override @@ -558,17 +507,6 @@ public boolean isVpcRing() { return false; } - public String getInternodeCompression() - { - return "all"; - } - - @Override - public boolean isBackingUpCommitLogs() - { - return false; - } - @Override public String getCommitLogBackupPropsFile() { @@ -648,26 +586,6 @@ public int getConcurrentCompactorsCnt() return 1; } - @Override - public String getRpcServerType() { - return "hsha"; - } - - @Override - public int getRpcMinThreads() { - return 16; - } - - @Override - public int getRpcMaxThreads() { - return 2048; - } - - @Override - public int getIndexInterval() { - return 0; - } - @Override public int getCompactionLargePartitionWarnThresholdInMB() { return 100; @@ -687,15 +605,10 @@ public boolean getAutoBoostrap() { } @Override - public String getDseClusterType() { - // TODO Auto-generated method stub - return "cassandra"; - } - @Override - public boolean isCreateNewTokenEnable() { - return true; //allow Junit test to create new tokens - } + public boolean isCreateNewTokenEnable() { + return true; //allow Junit test to create new tokens + } @Override public String getPrivateKeyLocation() { @@ -713,15 +626,11 @@ public boolean isEncryptBackupEnabled() { } @Override - public boolean isRestoreEncrypted() { - return false; + + public String getAWSRoleAssumptionArn() { + return null; } - @Override - public String getAWSRoleAssumptionArn() { - return null; - } - @Override public String getClassicEC2RoleAssumptionArn() { return null; @@ -732,11 +641,6 @@ public String getVpcEC2RoleAssumptionArn() { return null; } - @Override - public boolean isDualAccount(){ - return false; - } - @Override public String getGcsServiceAccountId() { return null; @@ -767,79 +671,14 @@ public Map getExtraEnvParams() { return null; } - @Override - public String getRestoreKeyspaceFilter() - { - return null; - } - - @Override - public String getRestoreCFFilter() { - return null; - } - - @Override - public String getIncrementalKeyspaceFilters() { - return null; - } - - @Override - public String getIncrementalCFFilter() { - return null; - } - - @Override - public String getSnapshotKeyspaceFilters() { - return null; - } - - @Override - public String getSnapshotCFFilter() { - return null; - } - @Override public String getVpcId() { return ""; } - @Override - public Boolean isIncrBackupParallelEnabled() { - return false; - } - - @Override - public int getIncrementalBkupMaxConsumers() { - return 2; - } - - @Override - public int getIncrementalBkupQueueSize() { - return 100; - } - - /** - * @return tombstone_warn_threshold in yaml - */ - @Override - public int getTombstoneWarnThreshold() { - return 1000; - } - - /** - * @return tombstone_failure_threshold in yaml - */ @Override - public int getTombstoneFailureThreshold() { - return 100000; - } - - /** - * @return streaming_socket_timeout_in_ms in yaml - */ - @Override - public int getStreamingSocketTimeoutInMS() { - return 86400000; + public int getBackupQueueSize() { + return 100; } @Override @@ -857,11 +696,6 @@ public String getBackupStatusFileLoc() { return "backupstatus.ser"; } - @Override - public boolean useSudo() { - return true; - } - @Override public String getBackupNotificationTopicArn() { return null; @@ -897,8 +731,19 @@ public String getPostRestoreHookDoneFileName() { return System.getProperty("java.io.tmpdir") + File.separator + "postrestorehook.done"; } - @Override public int getPostRestoreHookTimeOutInDays() { return 2; } + + @Override + public String getProperty(String key, String defaultValue) + { + return fakeProperties.getOrDefault(key, defaultValue); + } + + @Override + public String getMergedConfigurationDirectory() + { + return fakeProperties.getOrDefault("priam_test_config", "/tmp/priam_test_config"); + } } diff --git a/priam/src/test/java/com/netflix/priam/resources/BackupServletTest.java b/priam/src/test/java/com/netflix/priam/resources/BackupServletTest.java index 70158849a..6277929a3 100644 --- a/priam/src/test/java/com/netflix/priam/resources/BackupServletTest.java +++ b/priam/src/test/java/com/netflix/priam/resources/BackupServletTest.java @@ -310,9 +310,11 @@ public void restore_withKeyspaces(@Mocked final InstanceIdentity identity, restoreKeyspaces.clear(); restoreKeyspaces.addAll(ImmutableList.of("keyspace1", "keyspace2")); - config.getRestoreKeySpaces(); result = restoreKeyspaces; - config.setDC(oldRegion); - priamServer.getId(); result = identity; times = 2; + result = restoreKeyspaces; + config.setDC(oldRegion); + priamServer.getId(); + result = identity; + times = 2; } }; new Expectations() { diff --git a/priam/src/test/java/com/netflix/priam/services/TestSnapshotMetaService.java b/priam/src/test/java/com/netflix/priam/services/TestSnapshotMetaService.java index 78a484406..89a0da984 100644 --- a/priam/src/test/java/com/netflix/priam/services/TestSnapshotMetaService.java +++ b/priam/src/test/java/com/netflix/priam/services/TestSnapshotMetaService.java @@ -26,20 +26,15 @@ import com.netflix.priam.scheduler.TaskTimer; import com.netflix.priam.utils.BackupFileUtils; import com.netflix.priam.utils.DateUtil; -import org.apache.cassandra.io.sstable.Component; -import org.apache.commons.io.FileUtils; import org.junit.Assert; import org.junit.Before; import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.FileWriter; -import java.io.IOException; import java.nio.file.Path; import java.nio.file.Paths; import java.time.Instant; -import java.util.*; /** * Created by aagrawal on 6/20/18. From e5d6e174c83aa6df5e3af27bce337ecb782f0399 Mon Sep 17 00:00:00 2001 From: Arun Agrawal Date: Thu, 4 Oct 2018 22:49:18 -0700 Subject: [PATCH 015/228] Fix based on PR feedback --- .../java/com/netflix/priam/PriamServer.java | 6 +- .../netflix/priam/backup/AbstractBackup.java | 3 +- .../priam/backup/AbstractFileSystem.java | 20 +-- .../priam/backup/BackupRestoreUtil.java | 4 +- .../priam/backupv2/LocalDBReaderWriter.java | 14 ++- .../config/PriamConfigurationPersister.java | 115 ++++++++++++++++++ 6 files changed, 142 insertions(+), 20 deletions(-) create mode 100644 priam/src/main/java/com/netflix/priam/config/PriamConfigurationPersister.java diff --git a/priam/src/main/java/com/netflix/priam/PriamServer.java b/priam/src/main/java/com/netflix/priam/PriamServer.java index 7d4e0365b..89467653d 100644 --- a/priam/src/main/java/com/netflix/priam/PriamServer.java +++ b/priam/src/main/java/com/netflix/priam/PriamServer.java @@ -97,7 +97,7 @@ else if (UpdateSecuritySettings.firstTimeUpdated) // Start the Incremental backup schedule if enabled if (config.isIncrBackup()) { - scheduler.addTask(IncrementalBackup.JOBNAME, IncrementalBackup.class, IncrementalBackup.getTimer()); + scheduler.addTask(IncrementalBackup.JOBNAME, IncrementalBackup.class, IncrementalBackup.getTimer()); logger.info("Added incremental backup job"); } @@ -109,7 +109,7 @@ else if (UpdateSecuritySettings.firstTimeUpdated) // Determine if we need to restore from backup else start cassandra. - if (restoreContext.isRestoreEnabled()){ + if (restoreContext.isRestoreEnabled()) { restoreContext.restore(); } else { //no restores needed logger.info("No restore needed, task not scheduled"); @@ -149,7 +149,7 @@ else if (UpdateSecuritySettings.firstTimeUpdated) setUpSnapshotService(); } - private void setUpSnapshotService() throws Exception{ + private void setUpSnapshotService() throws Exception { TaskTimer snapshotMetaServiceTimer = SnapshotMetaService.getTimer(backupRestoreConfig); if (snapshotMetaServiceTimer != null) { scheduler.addTask(SnapshotMetaService.JOBNAME, SnapshotMetaService.class, snapshotMetaServiceTimer); diff --git a/priam/src/main/java/com/netflix/priam/backup/AbstractBackup.java b/priam/src/main/java/com/netflix/priam/backup/AbstractBackup.java index c923d38e3..b89d1e7a6 100644 --- a/priam/src/main/java/com/netflix/priam/backup/AbstractBackup.java +++ b/priam/src/main/java/com/netflix/priam/backup/AbstractBackup.java @@ -29,6 +29,7 @@ import java.io.File; import java.nio.file.Path; import java.nio.file.Paths; +import java.text.ParseException; import java.util.List; import java.util.concurrent.Future; @@ -59,7 +60,7 @@ protected void setFileSystem(IBackupFileSystem fs) { this.fs = fs; } - private AbstractBackupPath getAbstractBackupPath(final File file, final BackupFileType type) throws Exception { + private AbstractBackupPath getAbstractBackupPath(final File file, final BackupFileType type) throws ParseException { final AbstractBackupPath bp = pathFactory.get(); bp.parseLocal(file, type); return bp; diff --git a/priam/src/main/java/com/netflix/priam/backup/AbstractFileSystem.java b/priam/src/main/java/com/netflix/priam/backup/AbstractFileSystem.java index bdfd8e1e7..7be1ed6a7 100644 --- a/priam/src/main/java/com/netflix/priam/backup/AbstractFileSystem.java +++ b/priam/src/main/java/com/netflix/priam/backup/AbstractFileSystem.java @@ -38,6 +38,9 @@ import java.util.concurrent.*; /** + * This class is responsible for managing parallelism and orchestrating the upload and download, but the subclasses + * actually implement the details of uploading a file. + * * Created by aagrawal on 8/30/18. */ public abstract class AbstractFileSystem implements IBackupFileSystem, EventGenerator { @@ -52,9 +55,9 @@ public abstract class AbstractFileSystem implements IBackupFileSystem, EventGene public AbstractFileSystem(IConfiguration configuration, BackupMetrics backupMetrics, BackupNotificationMgr backupNotificationMgr) { this.backupMetrics = backupMetrics; - //Add notifications. + // Add notifications. this.addObserver(backupNotificationMgr); - tasksQueued = new HashSet<>(configuration.getBackupQueueSize()); + tasksQueued = new ConcurrentHashMap<>().newKeySet(); /* Note: We are using different queue for upload and download as with Backup V2.0 we might download all the meta files for "sync" feature which might compete with backups for scheduling. @@ -92,8 +95,8 @@ public Void retriableCall() throws Exception { return null; } }.call(); - //Note we only downloaded the bytes which are represented on file system (they are compressed and maybe encrypted). - //File size after decompression or decryption might be more/less. + // Note we only downloaded the bytes which are represented on file system (they are compressed and maybe encrypted). + // File size after decompression or decryption might be more/less. backupMetrics.recordDownloadRate(getFileSize(remotePath)); backupMetrics.incrementValidDownloads(); logger.info("Successfully downloaded file: {} to location: {}", remotePath, localPath); @@ -119,10 +122,7 @@ public void uploadFile(final Path localPath, final Path remotePath, final Abstra if (localPath == null || remotePath == null || !localPath.toFile().exists() || localPath.toFile().isDirectory()) throw new FileNotFoundException("File do not exist or is a directory. localPath: " + localPath + ", remotePath: " + remotePath); - if (!tasksQueued.contains(localPath)) { - //Add file to local memory - tasksQueued.add(localPath); - + if (tasksQueued.add(localPath)) { logger.info("Uploading file: {} to location: {}", localPath, remotePath); try { notifyEventStart(new BackupEvent(path)); @@ -194,12 +194,12 @@ public void notifyEventStop(BackupEvent event) { } @Override - public int getUploadTasksQueued(){ + public int getUploadTasksQueued() { return tasksQueued.size(); } @Override - public int getDownloadTasksQueued(){ + public int getDownloadTasksQueued() { return fileDownloadExecutor.getQueue().size(); } } diff --git a/priam/src/main/java/com/netflix/priam/backup/BackupRestoreUtil.java b/priam/src/main/java/com/netflix/priam/backup/BackupRestoreUtil.java index 5c33d6725..c10a3cc30 100644 --- a/priam/src/main/java/com/netflix/priam/backup/BackupRestoreUtil.java +++ b/priam/src/main/java/com/netflix/priam/backup/BackupRestoreUtil.java @@ -59,7 +59,7 @@ public static final Map> getFilter(String inputFilter) thro final Map> columnFamilyFilter = new HashMap<>(); //key: keyspace, value: a list of CFs within the keyspace String[] filters = inputFilter.split(","); - for (int i = 0; i < filters.length; i++) { //process each filter + for (int i = 0; i < filters.length; i++) { // process filter of form keyspace.* or keyspace.columnfamily if (columnFamilyFilterPattern.matcher(filters[i]).find()) { String[] filter = filters[i].split("\\."); @@ -93,7 +93,7 @@ public final boolean isFiltered(String keyspace, String columnFamilyDir) { return false; String columnFamilyName = columnFamilyDir.split("-")[0]; - //column family is in list of global CF filter + // column family is in list of global CF filter if (FILTER_COLUMN_FAMILY.containsKey(keyspace) && FILTER_COLUMN_FAMILY.get(keyspace).contains(columnFamilyName)) return true; diff --git a/priam/src/main/java/com/netflix/priam/backupv2/LocalDBReaderWriter.java b/priam/src/main/java/com/netflix/priam/backupv2/LocalDBReaderWriter.java index 398a63350..8ab25f49c 100644 --- a/priam/src/main/java/com/netflix/priam/backupv2/LocalDBReaderWriter.java +++ b/priam/src/main/java/com/netflix/priam/backupv2/LocalDBReaderWriter.java @@ -110,11 +110,17 @@ private LocalDBEntry getLocalDBEntry(final FileUploadResult fileUploadResult, fi if (localDB == null || localDB.getLocalDBEntries() == null || localDB.getLocalDBEntries().isEmpty()) return null; - //Get local db entry for same file and version. + // Get local db entry for same file and version. List localDBEntries = localDB.getLocalDBEntries().stream().filter(localDBEntry -> - (localDBEntry.getFileUploadResult().getFileName().toFile().getName().toLowerCase().equals(fileUploadResult.getFileName().toFile().getName().toLowerCase()))) - .filter(localDBEntry -> (localDBEntry.getFileUploadResult().getLastModifiedTime().equals(fileUploadResult.getLastModifiedTime()))) - .filter(localDBEntry -> (localDBEntry.getFileUploadResult().getCompression().equals(fileUploadResult.getCompression()))) + // Name of the file should be same. + (localDBEntry.getFileUploadResult().getFileName().toFile().getName().toLowerCase(). + equals(fileUploadResult.getFileName().toFile().getName().toLowerCase()))) + // Should be same version (same last modified time) + .filter(localDBEntry -> (localDBEntry.getFileUploadResult().getLastModifiedTime(). + equals(fileUploadResult.getLastModifiedTime()))) + // Same compression as before. If we switch compression technique we can upload again. + .filter(localDBEntry -> (localDBEntry.getFileUploadResult().getCompression(). + equals(fileUploadResult.getCompression()))) .collect(Collectors.toList()); if (localDBEntries.isEmpty()) diff --git a/priam/src/main/java/com/netflix/priam/config/PriamConfigurationPersister.java b/priam/src/main/java/com/netflix/priam/config/PriamConfigurationPersister.java new file mode 100644 index 000000000..7c4882567 --- /dev/null +++ b/priam/src/main/java/com/netflix/priam/config/PriamConfigurationPersister.java @@ -0,0 +1,115 @@ +/* + * Copyright 2018 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package com.netflix.priam.config; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.attribute.PosixFilePermissions; +import java.util.Map; +import javax.inject.Inject; +import javax.inject.Singleton; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.fasterxml.jackson.core.util.MinimalPrettyPrinter; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.ObjectWriter; +import com.netflix.priam.scheduler.CronTimer; +import com.netflix.priam.scheduler.Task; +import com.netflix.priam.scheduler.TaskTimer; + +/** + * Task that persists structured and merged priam configuration to disk. + */ +@Singleton +public class PriamConfigurationPersister extends Task +{ + public static final String NAME = "PriamConfigurationPersister"; + + private static final Logger logger = LoggerFactory.getLogger(PriamConfigurationPersister.class); + + private final Path mergedConfigDirectory; + private final Path structuredPath; + + @Inject + public PriamConfigurationPersister(IConfiguration config) { + super(config); + + mergedConfigDirectory = Paths.get(config.getMergedConfigurationDirectory()); + structuredPath = Paths.get(config.getMergedConfigurationDirectory(), "structured.json"); + } + + private synchronized void ensurePaths() throws IOException + { + File directory = mergedConfigDirectory.toFile(); + + if (directory.mkdirs()) { + Files.setPosixFilePermissions(mergedConfigDirectory, PosixFilePermissions.fromString("rwx------")); + logger.info("Set up PriamConfigurationPersister directory successfully"); + } + } + + + @Override + public void execute() throws Exception + { + ensurePaths(); + Path tempPath = null; + try { + File output = File.createTempFile(structuredPath.getFileName().toString(), ".tmp", mergedConfigDirectory.toFile()); + tempPath = output.toPath(); + + // The configuration might contain sensitive information, so ... don't let non Priam users read it + // Theoretically createTempFile creates the file with the right permissions, but I want to be explicit + Files.setPosixFilePermissions(tempPath, PosixFilePermissions.fromString("rw-------")); + + Map structuredConfiguration = config.getStructuredConfiguration("all"); + + ObjectMapper mapper = new ObjectMapper(); + ObjectWriter structuredPathTmpWriter = mapper.writer(new MinimalPrettyPrinter()); + structuredPathTmpWriter.writeValue(output, structuredConfiguration); + + // Atomically swap out the new config for the old config. + if (!output.renameTo(structuredPath.toFile())) + logger.error("Failed to persist structured Priam configuration"); + } finally { + if (tempPath != null) + Files.deleteIfExists(tempPath); + } + } + + @Override + public String getName() + { + return NAME; + } + + /** + * Timer to be used for configuration writing. + * + * @param config {@link IConfiguration} to get configuration details from priam. + * @return the timer to be used for Configuration Persisting from {@link IConfiguration#getMergedConfigurationCronExpression()} + */ + public static TaskTimer getTimer(IConfiguration config) { + return CronTimer.getCronTimer(NAME, config.getMergedConfigurationCronExpression()); + } + +} From 9cdf781d92ab69d5c8714408aed7771ead36158e Mon Sep 17 00:00:00 2001 From: Joseph Lynch Date: Fri, 28 Sep 2018 15:29:24 -0700 Subject: [PATCH 016/228] Expose Priam config via HTTP * Expose Priam config via HTTP and persist to disk This provides two new HTTP APIs for external tools to access Priam configuration. These APIs allow external tools to work with Priam to manage Cassandra while still using Priam's flat configuration as a source of truth for the Cassandra instance GET /v1/config/structured/all -> returns all Priam IConfiguration fields GET /v1/config/unstructured/X?default=Y -> returns the property X, or Y if it is not found (404 will be raised if no default and the value doesn't exist) In addition to the HTTP apis there is also now a persister thread which makes it so that every minute (by default) configuration is written out to a directory in tmp by Priam. This way tools can access config even if Priam is dead. The file permissions are 600 so that only the Priam user can access it. --- .../java/com/netflix/priam/PriamServer.java | 16 ++- .../netflix/priam/config/IConfiguration.java | 5 + .../priam/config/PriamConfiguration.java | 15 ++- .../netflix/priam/resources/PriamConfig.java | 103 ++++++++++++++++++ .../priam/config/FakeConfiguration.java | 4 +- .../PriamConfigurationPersisterTest.java | 66 +++++++++++ .../priam/resources/PriamConfigTest.java | 85 +++++++++++++++ 7 files changed, 288 insertions(+), 6 deletions(-) create mode 100644 priam/src/main/java/com/netflix/priam/resources/PriamConfig.java create mode 100644 priam/src/test/java/com/netflix/priam/config/PriamConfigurationPersisterTest.java create mode 100644 priam/src/test/java/com/netflix/priam/resources/PriamConfigTest.java diff --git a/priam/src/main/java/com/netflix/priam/PriamServer.java b/priam/src/main/java/com/netflix/priam/PriamServer.java index 89467653d..603a82850 100644 --- a/priam/src/main/java/com/netflix/priam/PriamServer.java +++ b/priam/src/main/java/com/netflix/priam/PriamServer.java @@ -28,6 +28,7 @@ import com.netflix.priam.cluster.management.IClusterManagement; import com.netflix.priam.config.IBackupRestoreConfig; import com.netflix.priam.config.IConfiguration; +import com.netflix.priam.config.PriamConfigurationPersister; import com.netflix.priam.defaultimpl.ICassandraProcess; import com.netflix.priam.identity.InstanceIdentity; import com.netflix.priam.restore.RestoreContext; @@ -128,23 +129,32 @@ else if (UpdateSecuritySettings.firstTimeUpdated) scheduler.addTaskWithDelay(CassandraMonitor.JOBNAME, CassandraMonitor.class, CassandraMonitor.getTimer(), CASSANDRA_MONITORING_INITIAL_DELAY); - //Set cleanup + // Set cleanup scheduler.addTask(UpdateCleanupPolicy.JOBNAME, UpdateCleanupPolicy.class, UpdateCleanupPolicy.getTimer()); - //Set up nodetool flush task + // Set up nodetool flush task TaskTimer flushTaskTimer = Flush.getTimer(config); if (flushTaskTimer != null) { scheduler.addTask(IClusterManagement.Task.FLUSH.name(), Flush.class, flushTaskTimer); logger.info("Added nodetool flush task."); } - //Set up compaction task + // Set up compaction task TaskTimer compactionTimer = Compaction.getTimer(config); if (compactionTimer != null) { scheduler.addTask(IClusterManagement.Task.COMPACTION.name(), Compaction.class, compactionTimer); logger.info("Added compaction task."); } + // Set up the background configuration dumping thread + TaskTimer configurationPersisterTimer = PriamConfigurationPersister.getTimer(config); + if (configurationPersisterTimer != null) { + scheduler.addTask(PriamConfigurationPersister.NAME, PriamConfigurationPersister.class, configurationPersisterTimer); + logger.info("Added configuration persister task with schedule [{}]", configurationPersisterTimer.getCronExpression()); + } else { + logger.warn("Priam configuration persister disabled!"); + } + //Set up the SnapshotService setUpSnapshotService(); } diff --git a/priam/src/main/java/com/netflix/priam/config/IConfiguration.java b/priam/src/main/java/com/netflix/priam/config/IConfiguration.java index a2bd43163..d51dc4348 100644 --- a/priam/src/main/java/com/netflix/priam/config/IConfiguration.java +++ b/priam/src/main/java/com/netflix/priam/config/IConfiguration.java @@ -26,6 +26,11 @@ import com.netflix.priam.scheduler.SchedulerType; import com.netflix.priam.scheduler.UnsupportedTypeException; +import java.io.File; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.util.Collections; +import java.util.HashMap; import java.util.List; import java.util.Map; diff --git a/priam/src/main/java/com/netflix/priam/config/PriamConfiguration.java b/priam/src/main/java/com/netflix/priam/config/PriamConfiguration.java index bcb682a2a..75d8d179d 100644 --- a/priam/src/main/java/com/netflix/priam/config/PriamConfiguration.java +++ b/priam/src/main/java/com/netflix/priam/config/PriamConfiguration.java @@ -21,6 +21,8 @@ import com.amazonaws.services.ec2.model.*; import com.google.common.collect.ImmutableList; import com.google.common.collect.Lists; + +import com.fasterxml.jackson.annotation.JsonIgnore; import com.google.inject.Inject; import com.google.inject.Singleton; import com.netflix.priam.configSource.IConfigSource; @@ -180,6 +182,7 @@ public class PriamConfiguration implements IConfiguration { //Running instance meta data private String RAC; + private String INSTANCE_ID; //== vpc specific private String NETWORK_VPC; //Fetch the vpc id of running instance @@ -244,7 +247,9 @@ public class PriamConfiguration implements IConfiguration { private static final Logger logger = LoggerFactory.getLogger(PriamConfiguration.class); private final ICredential provider; + @JsonIgnore private InstanceEnvIdentity insEnvIdentity; + @JsonIgnore private InstanceDataRetriever instanceDataRetriever; @Inject @@ -270,6 +275,7 @@ public void initialize() { } RAC = instanceDataRetriever.getRac(); + INSTANCE_ID = instanceDataRetriever.getInstanceId(); NETWORK_VPC = instanceDataRetriever.getVpcId(); @@ -371,7 +377,7 @@ private void populateProps() { } public String getInstanceName(){ - return instanceDataRetriever.getInstanceId(); + return INSTANCE_ID; } @Override @@ -523,6 +529,7 @@ public List getRacs() { return config.getList(CONFIG_AVAILABILITY_ZONES, DEFAULT_AVAILABILITY_ZONES); } + @JsonIgnore @Override public String getHostname() { if (this.isVpcRing()) return getInstanceDataRetriever().getPrivateIP(); @@ -1117,4 +1124,10 @@ public String getProperty(String key, String defaultValue) { return config.get(key, defaultValue); } + + @Override + public String getMergedConfigurationCronExpression() { + // Every minute on the top of the minute. + return config.get(PRIAM_PRE + ".configMerge.cron", "0 * * * * ? *"); + } } diff --git a/priam/src/main/java/com/netflix/priam/resources/PriamConfig.java b/priam/src/main/java/com/netflix/priam/resources/PriamConfig.java new file mode 100644 index 000000000..a20c94bd7 --- /dev/null +++ b/priam/src/main/java/com/netflix/priam/resources/PriamConfig.java @@ -0,0 +1,103 @@ +/* + * Copyright 2018 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package com.netflix.priam.resources; + +import java.util.HashMap; +import java.util.Map; +import javax.ws.rs.GET; +import javax.ws.rs.Path; +import javax.ws.rs.PathParam; +import javax.ws.rs.Produces; +import javax.ws.rs.QueryParam; +import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.inject.Inject; +import com.netflix.priam.PriamServer; + +/** + * This servlet will provide the configuration API service for use by external scripts and tooling + */ +@Path("/v1/config") +@Produces(MediaType.APPLICATION_JSON) +public class PriamConfig +{ + private static final Logger logger = LoggerFactory.getLogger(PriamConfig.class); + private PriamServer priamServer; + + @Inject + public PriamConfig(PriamServer server) { + this.priamServer = server; + } + + private Response doGetPriamConfig(String group, String name) { + try { + final Map result = new HashMap<>(); + final Map value = priamServer.getConfiguration().getStructuredConfiguration(group); + if (name != null && value.containsKey(name)) { + result.put(name, value.get(name)); + return Response.ok(result, MediaType.APPLICATION_JSON).build(); + } else if (name != null) { + result.put("message", String.format("No such structured config: [%s]", name)); + logger.error(String.format("No such structured config: [%s]", name)); + return Response.status(404).entity(result).type(MediaType.APPLICATION_JSON).build(); + } else { + result.putAll(value); + return Response.ok(result, MediaType.APPLICATION_JSON).build(); + } + } catch (Exception e) { + logger.error("Error while executing getPriamConfig", e); + return Response.serverError().build(); + } + } + + + @GET + @Path("/structured/{group}") + public Response getPriamConfig(@PathParam("group") String group, @PathParam("name") String name) { + return doGetPriamConfig(group, null); + } + + @GET + @Path("/structured/{group}/{name}") + public Response getPriamConfigByName(@PathParam("group") String group, @PathParam("name") String name) { + return doGetPriamConfig(group, name); + } + + @GET + @Path("/unstructured/{name}") + public Response getProperty(@PathParam("name") String name, @QueryParam("default") String defaultValue) { + Map result = new HashMap<>(); + try { + String value = priamServer.getConfiguration().getProperty(name, defaultValue); + if (value != null) { + result.put(name, value); + return Response.ok(result, MediaType.APPLICATION_JSON).build(); + } else { + result.put("message", String.format("No such property: [%s]", name)); + logger.error(String.format("No such property: [%s]", name)); + return Response.status(404).entity(result).type(MediaType.APPLICATION_JSON).build(); + } + } catch (Exception e) { + logger.error("Error while executing getPriamConfig", e); + return Response.serverError().build(); + } + } +} \ No newline at end of file diff --git a/priam/src/test/java/com/netflix/priam/config/FakeConfiguration.java b/priam/src/test/java/com/netflix/priam/config/FakeConfiguration.java index 37d2e1a86..32b2eb1b4 100644 --- a/priam/src/test/java/com/netflix/priam/config/FakeConfiguration.java +++ b/priam/src/test/java/com/netflix/priam/config/FakeConfiguration.java @@ -31,6 +31,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.stream.Collectors; @Singleton public class FakeConfiguration implements IConfiguration @@ -45,8 +46,7 @@ public class FakeConfiguration implements IConfiguration public String restorePrefix; public Map fakeConfig; public Map fakeProperties = new HashMap<>(); - - + public FakeConfiguration() { this(FAKE_REGION, "my_fake_cluster", "my_zone", "i-01234567"); diff --git a/priam/src/test/java/com/netflix/priam/config/PriamConfigurationPersisterTest.java b/priam/src/test/java/com/netflix/priam/config/PriamConfigurationPersisterTest.java new file mode 100644 index 000000000..6388d4a6f --- /dev/null +++ b/priam/src/test/java/com/netflix/priam/config/PriamConfigurationPersisterTest.java @@ -0,0 +1,66 @@ +package com.netflix.priam.config; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.HashMap; +import java.util.Map; + +import org.junit.After; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.inject.Guice; +import com.google.inject.Injector; +import com.netflix.priam.TestModule; + +import static junit.framework.TestCase.assertTrue; +import static org.junit.Assert.assertEquals; + +public class PriamConfigurationPersisterTest +{ + private static PriamConfigurationPersister persister; + + @Rule + public TemporaryFolder folder = new TemporaryFolder(); + + private FakeConfiguration fakeConfiguration; + + @Before + public void setUp() { + Injector injector = Guice.createInjector(new TestModule()); + fakeConfiguration = (FakeConfiguration) injector.getInstance(IConfiguration.class); + fakeConfiguration.fakeProperties.put("priam_test_config", folder.getRoot().getPath()); + + if (persister == null) + persister = injector.getInstance(PriamConfigurationPersister.class); + } + + @After + public void cleanUp() { + fakeConfiguration.fakeProperties.clear(); + } + + @Test + @SuppressWarnings("unchecked") + public void execute() throws Exception + { + Path structuredJson = Paths.get(folder.getRoot().getPath(), "structured.json"); + + persister.execute(); + assertTrue(structuredJson.toFile().exists()); + + ObjectMapper objectMapper = new ObjectMapper(); + Map myMap = objectMapper.readValue(Files.readAllBytes(structuredJson), HashMap.class); + assertEquals(myMap.get("backupLocation"), fakeConfiguration.getBackupLocation()); + } + + @Test + public void getTimer() + { + assertEquals("0 * * * * ? *", PriamConfigurationPersister.getTimer(fakeConfiguration).getCronExpression()); + } +} \ No newline at end of file diff --git a/priam/src/test/java/com/netflix/priam/resources/PriamConfigTest.java b/priam/src/test/java/com/netflix/priam/resources/PriamConfigTest.java new file mode 100644 index 000000000..9c5d76cb1 --- /dev/null +++ b/priam/src/test/java/com/netflix/priam/resources/PriamConfigTest.java @@ -0,0 +1,85 @@ +package com.netflix.priam.resources; + +import java.util.HashMap; +import java.util.Map; +import javax.ws.rs.core.Response; + +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; + +import com.google.inject.Inject; +import com.netflix.priam.PriamServer; +import com.netflix.priam.config.FakeConfiguration; +import com.netflix.priam.config.IConfiguration; +import mockit.Expectations; +import mockit.Mocked; +import mockit.integration.junit4.JMockit; + +import static org.junit.Assert.*; + +@RunWith(JMockit.class) +public class PriamConfigTest +{ + private @Mocked PriamServer priamServer; + + private PriamConfig resource; + + private FakeConfiguration fakeConfiguration; + + @Before + public void setUp() { + resource = new PriamConfig(priamServer); + fakeConfiguration = new FakeConfiguration("test", "cass_test", "test", "12345"); + fakeConfiguration.fakeProperties.put("test.prop", "test_value"); + } + + + @Test + public void getPriamConfig() + { + final Map expected = new HashMap<>(); + expected.put("backupLocation", "casstestbackup"); + new Expectations() { + { + priamServer.getConfiguration(); + result = fakeConfiguration; + times = 2; + } + }; + + Response response = resource.getPriamConfigByName("all", "backupLocation"); + assertEquals(200, response.getStatus()); + assertEquals(expected, response.getEntity()); + + Response badResponse = resource.getPriamConfigByName("all", "getUnrealThing"); + assertEquals(404, badResponse.getStatus()); + } + + @Test + public void getProperty() + { + final Map expected = new HashMap<>(); + expected.put("test.prop", "test_value"); + new Expectations() { + { + priamServer.getConfiguration(); + result = fakeConfiguration; + times = 3; + } + }; + + Response response = resource.getProperty("test.prop", null); + assertEquals(200, response.getStatus()); + assertEquals(expected, response.getEntity()); + + Response defaultResponse = resource.getProperty("not.a.property", "NOVALUE"); + expected.clear(); + expected.put("not.a.property", "NOVALUE"); + assertEquals(200, defaultResponse.getStatus()); + assertEquals(expected, defaultResponse.getEntity()); + + Response badResponse = resource.getProperty("not.a.property", null); + assertEquals(404, badResponse.getStatus()); + } +} \ No newline at end of file From 5b6b48f7c6ff99cce34c999a2ad9ef3d932450d7 Mon Sep 17 00:00:00 2001 From: Arun Agrawal Date: Mon, 8 Oct 2018 13:54:42 -0700 Subject: [PATCH 017/228] changelog message and code fix for SnapshotBackup (merge removed it) --- CHANGELOG.md | 19 +++++++++++++++++++ .../netflix/priam/backup/SnapshotBackup.java | 6 ++++++ 2 files changed, 25 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 16d4df3db..aa4077bdb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,23 @@ # Changelog +## 2018/10/08 3.11.33 +***WARNING*** THIS IS A BREAKING RELEASE +### New Feature +* (#731) Restores will be async in nature by default. +* (#731) Support for async snapshots via configuration - `priam.async.snapshot`. Similar support for async incrementals via configuration - `priam.async.incremental`. +* (#731) Better metrics for upload and download to/from remote file system. +* (#731) Better support for include/exclude keyspaces/columnfamilies from backup, incremental backup and restores. +* (#731) Expose priam configuration over HTTP and persist at regular interval (CRON) to local file system for automation/tooling. +### Bug fix +* (#731) Metrics are incremented only once and in a central location at AbstractFileSystem. +* (#731) Remove deprecated AWS API Calls. +### Breaking changes +* (#731) Removal of MBeans to collect metrics from S3FileSystem. They were unreliable and incorrect. +* (#731) Update to backup configurations :- isIncrBackupParallelEnabled, getIncrementalBkupMaxConsumers, getIncrementalBkupQueueSize. They are renamed to ensure naming consistency. Refer to wiki for more details. +* (#731) Changes to backup/restore configuration :- getSnapshotKeyspaceFilters, getSnapshotCFFilter, getIncrementalKeyspaceFilters, getIncrementalCFFilter, getRestoreKeyspaceFilter, getRestoreCFFilter. They are now centralized to ensure that we can support both include and exclude keyspaces/CF. Refer to wiki for more details. + +## 2018/10/02 3.11.32 +* (#727) Bug Fix: Continue uploading incrementals when parallel incrementals is enabled and file fails to upload. +* (#718) Add last modified time to S3 Object Metadata. ## 2018/09/10 3.11.31 * (#715) Bug Fix: Fix the bootstrap issue. Do not provide yourself as seed node if cluster is already up and running as it will lead to data loss. diff --git a/priam/src/main/java/com/netflix/priam/backup/SnapshotBackup.java b/priam/src/main/java/com/netflix/priam/backup/SnapshotBackup.java index abed6626a..2b52b71c9 100644 --- a/priam/src/main/java/com/netflix/priam/backup/SnapshotBackup.java +++ b/priam/src/main/java/com/netflix/priam/backup/SnapshotBackup.java @@ -182,6 +182,12 @@ private void notifyObservers() { protected void processColumnFamily(String keyspace, String columnFamily, File backupDir) throws Exception { File snapshotDir = getValidSnapshot(backupDir, snapshotName); + + if (snapshotDir == null) { + logger.warn("{} folder does not contain {} snapshots", backupDir, snapshotName); + return; + } + // Add files to this dir abstractBackupPaths.addAll(upload(snapshotDir, BackupFileType.SNAP, config.enableAsyncSnapshot())); } From af1a91ec9c07e6c0ab72f4ad259193142638744f Mon Sep 17 00:00:00 2001 From: Arun Agrawal Date: Mon, 8 Oct 2018 15:10:02 -0700 Subject: [PATCH 018/228] 3.11 Supports C* 3.x (i.e. 3.0.x and 3.x) --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 598f6d600..475036d64 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ [![Build Status](https://travis-ci.org/Netflix/Priam.svg?branch=3.x)](https://travis-ci.org/Netflix/Priam) -### Priam 3.11 branch supports Cassandra 3.11.x and 3.11+ +### Priam 3.11 branch supports Cassandra 3.x Priam is a process/tool that runs alongside Apache Cassandra to automate the following: - Backup and recovery (Complete and incremental) From 88806ba46de84fa2d133b2f2d1474460a181c35e Mon Sep 17 00:00:00 2001 From: Arun Agrawal Date: Fri, 12 Oct 2018 09:45:57 -0700 Subject: [PATCH 019/228] fix imports, fixes, remove deprecated calls, code cleanup (#735) * fix imports, fixes, remove deprecated calls, code cleanup * Use default properties from PriamConfiguration. --- .../com/netflix/priam/aws/AWSMembership.java | 7 +- .../com/netflix/priam/aws/S3BackupPath.java | 6 +- .../priam/aws/S3CrossAccountFileSystem.java | 6 +- .../priam/aws/S3EncryptedFileSystem.java | 2 +- .../com/netflix/priam/aws/S3FileSystem.java | 4 - .../netflix/priam/aws/S3FileSystemBase.java | 37 +++--- .../com/netflix/priam/aws/S3PartUploader.java | 4 +- .../netflix/priam/aws/S3PrefixIterator.java | 8 +- .../netflix/priam/aws/SDBInstanceData.java | 8 +- .../netflix/priam/aws/SDBInstanceFactory.java | 20 ++-- .../priam/aws/UpdateCleanupPolicy.java | 2 +- .../aws/auth/EC2RoleAssumptionCredential.java | 8 +- .../priam/aws/auth/S3InstanceCredential.java | 2 +- .../aws/auth/S3RoleAssumptionCredential.java | 4 +- .../priam/backup/AbstractBackupPath.java | 8 +- .../priam/backup/AbstractFileSystem.java | 13 +-- .../netflix/priam/backup/BackupMetadata.java | 4 +- .../priam/backup/BackupRestoreUtil.java | 12 +- .../netflix/priam/backup/BackupStatusMgr.java | 13 +-- .../priam/backup/BackupVerification.java | 7 +- .../netflix/priam/backup/CommitLogBackup.java | 2 +- .../priam/backup/CommitLogBackupTask.java | 4 +- .../priam/backup/FileSnapshotStatusMgr.java | 2 +- .../priam/backup/IncrementalBackup.java | 6 +- .../priam/backup/IncrementalMetaData.java | 2 +- .../com/netflix/priam/backup/MetaData.java | 6 +- .../netflix/priam/backup/SnapshotBackup.java | 10 +- .../priam/backupv2/FileUploadResult.java | 4 +- .../priam/backupv2/LocalDBReaderWriter.java | 2 +- .../priam/backupv2/MetaFileReader.java | 5 +- .../priam/backupv2/MetaFileWriterBuilder.java | 12 +- .../priam/backupv2/PrefixGenerator.java | 4 +- .../netflix/priam/cli/LightGuiceModule.java | 2 +- .../netflix/priam/cli/StaticMembership.java | 2 +- .../management/IClusterManagement.java | 6 +- .../netflix/priam/compress/ChunkedStream.java | 10 +- .../netflix/priam/config/IConfiguration.java | 5 - .../priam/config/PriamConfiguration.java | 110 ++++++------------ .../priam/cryptography/pgp/PgpCredential.java | 2 +- .../cryptography/pgp/PgpCryptography.java | 10 +- .../defaultimpl/CassandraOperations.java | 4 +- .../defaultimpl/CassandraProcessManager.java | 4 +- .../netflix/priam/google/GcsCredential.java | 2 +- .../google/GoogleEncryptedFileSystem.java | 10 +- .../priam/google/GoogleFileIterator.java | 6 +- .../netflix/priam/health/InstanceState.java | 2 +- .../netflix/priam/identity/DoubleRing.java | 17 +-- .../priam/identity/InstanceIdentity.java | 14 +-- .../identity/token/DeadTokenRetriever.java | 14 +-- .../identity/token/NewTokenRetriever.java | 12 +- .../token/PreGeneratedTokenRetriever.java | 8 +- .../identity/token/TokenRetrieverBase.java | 2 +- .../netflix/priam/merics/BackupMetrics.java | 1 - .../AWSSnsNotificationService.java | 8 +- .../notification/BackupNotificationMgr.java | 2 +- .../priam/resources/BackupServlet.java | 28 ++--- .../priam/resources/CassandraConfig.java | 4 +- .../netflix/priam/resources/PriamConfig.java | 2 +- .../priam/resources/RestoreServlet.java | 18 +-- .../priam/restore/AbstractRestore.java | 26 ++--- .../priam/restore/EncryptedRestoreBase.java | 9 +- .../priam/restore/PostRestoreHook.java | 8 +- .../netflix/priam/restore/RestoreContext.java | 2 +- .../BlockingSubmitThreadPoolExecutor.java | 6 +- .../netflix/priam/scheduler/CronTimer.java | 2 +- .../priam/scheduler/SchedulerType.java | 2 +- .../netflix/priam/scheduler/SimpleTimer.java | 2 +- .../com/netflix/priam/scheduler/Task.java | 3 +- .../priam/services/SnapshotMetaService.java | 8 +- .../java/com/netflix/priam/tuner/GCType.java | 2 +- .../com/netflix/priam/tuner/JVMOption.java | 2 +- .../netflix/priam/tuner/JVMOptionsTuner.java | 8 +- .../netflix/priam/tuner/StandardTuner.java | 9 +- .../netflix/priam/tuner/TuneCassandra.java | 2 +- .../priam/tuner/dse/AuditLogTunerLog4J.java | 6 +- .../priam/tuner/dse/AuditLogTunerYaml.java | 4 +- .../BoundedExponentialRetryCallable.java | 6 +- .../netflix/priam/utils/CassandraMonitor.java | 8 +- .../priam/utils/ExponentialRetryCallable.java | 4 +- .../com/netflix/priam/utils/FifoQueue.java | 2 +- .../com/netflix/priam/utils/JMXNodeTool.java | 10 +- .../com/netflix/priam/utils/SystemUtils.java | 10 +- .../com/netflix/priam/utils/TokenManager.java | 5 +- priam/src/main/resources/Priam.properties | 52 --------- 84 files changed, 304 insertions(+), 433 deletions(-) diff --git a/priam/src/main/java/com/netflix/priam/aws/AWSMembership.java b/priam/src/main/java/com/netflix/priam/aws/AWSMembership.java index 4c9d9dd1b..c4e5ba932 100644 --- a/priam/src/main/java/com/netflix/priam/aws/AWSMembership.java +++ b/priam/src/main/java/com/netflix/priam/aws/AWSMembership.java @@ -35,10 +35,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.List; +import java.util.*; /** * Class to query amazon ASG for its members to provide - Number of valid nodes @@ -239,7 +236,7 @@ public List listACL(int from, int to) { if (this.insEnvIdentity.isClassic()) { - DescribeSecurityGroupsRequest req = new DescribeSecurityGroupsRequest().withGroupNames(Arrays.asList(config.getACLGroupName())); + DescribeSecurityGroupsRequest req = new DescribeSecurityGroupsRequest().withGroupNames(Collections.singletonList(config.getACLGroupName())); DescribeSecurityGroupsResult result = client.describeSecurityGroups(req); for (SecurityGroup group : result.getSecurityGroups()) for (IpPermission perm : group.getIpPermissions()) diff --git a/priam/src/main/java/com/netflix/priam/aws/S3BackupPath.java b/priam/src/main/java/com/netflix/priam/aws/S3BackupPath.java index 7d1402baf..5bf4805a5 100644 --- a/priam/src/main/java/com/netflix/priam/aws/S3BackupPath.java +++ b/priam/src/main/java/com/netflix/priam/aws/S3BackupPath.java @@ -41,7 +41,7 @@ public S3BackupPath(IConfiguration config, InstanceIdentity factory) { */ @Override public String getRemotePath() { - StringBuffer buff = new StringBuffer(); + StringBuilder buff = new StringBuilder(); buff.append(baseDir).append(S3BackupPath.PATH_SEP); // Base dir buff.append(region).append(S3BackupPath.PATH_SEP); buff.append(clusterName).append(S3BackupPath.PATH_SEP);// Cluster name @@ -98,7 +98,7 @@ public void parsePartialPrefix(String remoteFilePath) { @Override public String remotePrefix(Date start, Date end, String location) { - StringBuffer buff = new StringBuffer(clusterPrefix(location)); + StringBuilder buff = new StringBuilder(clusterPrefix(location)); token = factory.getInstance().getToken(); buff.append(token).append(S3BackupPath.PATH_SEP); // match the common characters to prefix. @@ -108,7 +108,7 @@ public String remotePrefix(Date start, Date end, String location) { @Override public String clusterPrefix(String location) { - StringBuffer buff = new StringBuffer(); + StringBuilder buff = new StringBuilder(); String[] elements = location.split(String.valueOf(S3BackupPath.PATH_SEP)); if (elements.length <= 1) { baseDir = config.getBackupLocation(); diff --git a/priam/src/main/java/com/netflix/priam/aws/S3CrossAccountFileSystem.java b/priam/src/main/java/com/netflix/priam/aws/S3CrossAccountFileSystem.java index af058ae95..354d29903 100755 --- a/priam/src/main/java/com/netflix/priam/aws/S3CrossAccountFileSystem.java +++ b/priam/src/main/java/com/netflix/priam/aws/S3CrossAccountFileSystem.java @@ -40,9 +40,9 @@ public class S3CrossAccountFileSystem { private static final Logger logger = LoggerFactory.getLogger(S3CrossAccountFileSystem.class); private AmazonS3 s3Client; - private S3FileSystem s3fs; - private IConfiguration config; - private IS3Credential s3Credential; + private final S3FileSystem s3fs; + private final IConfiguration config; + private final IS3Credential s3Credential; @Inject public S3CrossAccountFileSystem(@Named("backup") IBackupFileSystem fs, @Named("awss3roleassumption") IS3Credential s3Credential, IConfiguration config) { diff --git a/priam/src/main/java/com/netflix/priam/aws/S3EncryptedFileSystem.java b/priam/src/main/java/com/netflix/priam/aws/S3EncryptedFileSystem.java index be4043001..9d7168b05 100755 --- a/priam/src/main/java/com/netflix/priam/aws/S3EncryptedFileSystem.java +++ b/priam/src/main/java/com/netflix/priam/aws/S3EncryptedFileSystem.java @@ -103,7 +103,7 @@ protected long uploadFileImpl(Path localPath, Path remotePath) throws BackupRest compressedBos.write(compressedChunk); } } catch (Exception e) { - String message = String.format("Exception in compressing the input data during upload to EncryptedStore Msg: " + e.getMessage()); + String message = "Exception in compressing the input data during upload to EncryptedStore Msg: " + e.getMessage(); logger.error(message, e); throw new BackupRestoreException(message); } diff --git a/priam/src/main/java/com/netflix/priam/aws/S3FileSystem.java b/priam/src/main/java/com/netflix/priam/aws/S3FileSystem.java index 172406f9f..d4114f5c7 100644 --- a/priam/src/main/java/com/netflix/priam/aws/S3FileSystem.java +++ b/priam/src/main/java/com/netflix/priam/aws/S3FileSystem.java @@ -32,14 +32,10 @@ import com.netflix.priam.merics.BackupMetrics; import com.netflix.priam.notification.BackupNotificationMgr; import com.netflix.priam.utils.BoundedExponentialRetryCallable; -import org.apache.commons.io.IOUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import javax.management.MBeanServer; -import javax.management.ObjectName; import java.io.*; -import java.lang.management.ManagementFactory; import java.nio.file.Path; import java.util.ArrayList; import java.util.Collections; diff --git a/priam/src/main/java/com/netflix/priam/aws/S3FileSystemBase.java b/priam/src/main/java/com/netflix/priam/aws/S3FileSystemBase.java index e3fc34701..c0cb1021e 100755 --- a/priam/src/main/java/com/netflix/priam/aws/S3FileSystemBase.java +++ b/priam/src/main/java/com/netflix/priam/aws/S3FileSystemBase.java @@ -41,22 +41,21 @@ import java.util.concurrent.LinkedBlockingQueue; public abstract class S3FileSystemBase extends AbstractFileSystem { - protected static final int MAX_CHUNKS = 10000; - protected static final long MAX_BUFFERED_IN_STREAM_SIZE = 5 * 1024 * 1024; - protected static final long UPLOAD_TIMEOUT = (2 * 60 * 60 * 1000L); + private static final int MAX_CHUNKS = 10000; + static final long MAX_BUFFERED_IN_STREAM_SIZE = 5 * 1024 * 1024; private static final Logger logger = LoggerFactory.getLogger(S3FileSystemBase.class); - protected AmazonS3 s3Client; - protected final IConfiguration config; - protected final Provider pathProvider; - protected final ICompression compress; - protected final BlockingSubmitThreadPoolExecutor executor; - protected final RateLimiter rateLimiter; //a throttling mechanism, we can limit the amount of bytes uploaded to endpoint per second. - - public S3FileSystemBase(Provider pathProvider, - ICompression compress, - final IConfiguration config, - BackupMetrics backupMetrics, - BackupNotificationMgr backupNotificationMgr) { + AmazonS3 s3Client; + final IConfiguration config; + private final Provider pathProvider; + final ICompression compress; + final BlockingSubmitThreadPoolExecutor executor; + final RateLimiter rateLimiter; //a throttling mechanism, we can limit the amount of bytes uploaded to endpoint per second. + + S3FileSystemBase(Provider pathProvider, + ICompression compress, + final IConfiguration config, + BackupMetrics backupMetrics, + BackupNotificationMgr backupNotificationMgr) { super(config, backupMetrics, backupNotificationMgr); this.pathProvider = pathProvider; this.compress = compress; @@ -64,14 +63,14 @@ public S3FileSystemBase(Provider pathProvider, int threads = config.getBackupThreads(); LinkedBlockingQueue queue = new LinkedBlockingQueue<>(threads); - this.executor = new BlockingSubmitThreadPoolExecutor(threads, queue, UPLOAD_TIMEOUT); + this.executor = new BlockingSubmitThreadPoolExecutor(threads, queue, config.getUploadTimeout()); double throttleLimit = config.getUploadThrottle(); this.rateLimiter = RateLimiter.create(throttleLimit < 1 ? Double.MAX_VALUE : throttleLimit); } - public AmazonS3 getS3Client() { + private AmazonS3 getS3Client() { return s3Client; } @@ -85,7 +84,7 @@ public void setS3Client(AmazonS3 client) { /** * Get S3 prefix which will be used to locate S3 files */ - protected String getPrefix(IConfiguration config) { + String getPrefix(IConfiguration config) { String prefix; if (StringUtils.isNotBlank(config.getRestorePrefix())) prefix = config.getRestorePrefix(); @@ -151,7 +150,7 @@ private boolean updateLifecycleRule(IConfiguration config, List rules, Str return true; } - protected void checkSuccessfulUpload(CompleteMultipartUploadResult resultS3MultiPartUploadComplete, Path localPath) throws BackupRestoreException { + void checkSuccessfulUpload(CompleteMultipartUploadResult resultS3MultiPartUploadComplete, Path localPath) throws BackupRestoreException { if (null != resultS3MultiPartUploadComplete && null != resultS3MultiPartUploadComplete.getETag()) { logger.info("Uploaded file: {}, object eTag: {}", localPath, resultS3MultiPartUploadComplete.getETag()); } else { diff --git a/priam/src/main/java/com/netflix/priam/aws/S3PartUploader.java b/priam/src/main/java/com/netflix/priam/aws/S3PartUploader.java index 8c15b7c06..b36835f67 100644 --- a/priam/src/main/java/com/netflix/priam/aws/S3PartUploader.java +++ b/priam/src/main/java/com/netflix/priam/aws/S3PartUploader.java @@ -32,8 +32,8 @@ public class S3PartUploader extends BoundedExponentialRetryCallable { private final AmazonS3 client; - private DataPart dataPart; - private List partETags; + private final DataPart dataPart; + private final List partETags; private AtomicInteger partsUploaded = null; //num of data parts successfully uploaded private static final Logger logger = LoggerFactory.getLogger(S3PartUploader.class); diff --git a/priam/src/main/java/com/netflix/priam/aws/S3PrefixIterator.java b/priam/src/main/java/com/netflix/priam/aws/S3PrefixIterator.java index 649486b33..079baa82c 100644 --- a/priam/src/main/java/com/netflix/priam/aws/S3PrefixIterator.java +++ b/priam/src/main/java/com/netflix/priam/aws/S3PrefixIterator.java @@ -47,9 +47,9 @@ public class S3PrefixIterator implements Iterator { private String bucket = ""; private String clusterPath = ""; - private SimpleDateFormat datefmt = new SimpleDateFormat("yyyyMMdd"); + private final SimpleDateFormat datefmt = new SimpleDateFormat("yyyyMMdd"); private ObjectListing objectListing = null; - Date date; + final Date date; @Inject public S3PrefixIterator(IConfiguration config, Provider pathProvider, AmazonS3 s3Client, Date date) { @@ -57,7 +57,7 @@ public S3PrefixIterator(IConfiguration config, Provider path this.pathProvider = pathProvider; this.s3Client = s3Client; this.date = date; - String path = ""; + String path; if (StringUtils.isNotBlank(config.getRestorePrefix())) path = config.getRestorePrefix(); else @@ -120,7 +120,7 @@ public void remove() { * Get remote prefix upto the token */ private String remotePrefix(String location) { - StringBuffer buff = new StringBuffer(); + StringBuilder buff = new StringBuilder(); String[] elements = location.split(String.valueOf(S3BackupPath.PATH_SEP)); if (elements.length <= 1) { buff.append(config.getBackupLocation()).append(S3BackupPath.PATH_SEP); diff --git a/priam/src/main/java/com/netflix/priam/aws/SDBInstanceData.java b/priam/src/main/java/com/netflix/priam/aws/SDBInstanceData.java index 6c07c7bd8..997c7c677 100644 --- a/priam/src/main/java/com/netflix/priam/aws/SDBInstanceData.java +++ b/priam/src/main/java/com/netflix/priam/aws/SDBInstanceData.java @@ -100,8 +100,8 @@ public Set getAllIds(String app) { /** * Create a new instance entry in SimpleDB * - * @param instance - * @throws AmazonServiceException + * @param instance Instance entry to be created. + * @throws AmazonServiceException If unable to write to Simple DB because of any error. */ public void createInstance(PriamInstance instance) throws AmazonServiceException { AmazonSimpleDB simpleDBClient = getSimpleDBClient(); @@ -112,8 +112,8 @@ public void createInstance(PriamInstance instance) throws AmazonServiceException /** * Register a new instance. Registration will fail if a prior entry exists * - * @param instance - * @throws AmazonServiceException + * @param instance Instance entry to be registered. + * @throws AmazonServiceException If unable to write to Simple DB because of any error. */ public void registerInstance(PriamInstance instance) throws AmazonServiceException { AmazonSimpleDB simpleDBClient = getSimpleDBClient(); diff --git a/priam/src/main/java/com/netflix/priam/aws/SDBInstanceFactory.java b/priam/src/main/java/com/netflix/priam/aws/SDBInstanceFactory.java index b6e1228e3..148f82af8 100644 --- a/priam/src/main/java/com/netflix/priam/aws/SDBInstanceFactory.java +++ b/priam/src/main/java/com/netflix/priam/aws/SDBInstanceFactory.java @@ -47,9 +47,7 @@ public SDBInstanceFactory(IConfiguration config, SDBInstanceData dao) { @Override public List getAllIds(String appName) { List return_ = new ArrayList<>(); - for (PriamInstance instance : dao.getAllIds(appName)) { - return_.add(instance); - } + return_.addAll(dao.getAllIds(appName)); sort(return_); return return_; } @@ -103,17 +101,13 @@ public void update(PriamInstance inst) { @Override public void sort(List return_) { - Comparator comparator = new Comparator() { - - @Override - public int compare(PriamInstance o1, PriamInstance o2) { + Comparator comparator = (Comparator) (o1, o2) -> { - Integer c1 = o1.getId(); - Integer c2 = o2.getId(); - return c1.compareTo(c2); - } + Integer c1 = o1.getId(); + Integer c2 = o2.getId(); + return c1.compareTo(c2); }; - Collections.sort(return_, comparator); + return_.sort(comparator); } @Override @@ -122,7 +116,7 @@ public void attachVolumes(PriamInstance instance, String mountPath, String devic } private PriamInstance makePriamInstance(String app, int id, String instanceID, String hostname, String ip, String rac, Map volumes, String token) { - Map v = (volumes == null) ? new HashMap() : volumes; + Map v = (volumes == null) ? new HashMap<>() : volumes; PriamInstance ins = new PriamInstance(); ins.setApp(app); ins.setRac(rac); diff --git a/priam/src/main/java/com/netflix/priam/aws/UpdateCleanupPolicy.java b/priam/src/main/java/com/netflix/priam/aws/UpdateCleanupPolicy.java index 41034a5f2..813eba350 100644 --- a/priam/src/main/java/com/netflix/priam/aws/UpdateCleanupPolicy.java +++ b/priam/src/main/java/com/netflix/priam/aws/UpdateCleanupPolicy.java @@ -33,7 +33,7 @@ @Singleton public class UpdateCleanupPolicy extends Task { public static final String JOBNAME = "UpdateCleanupPolicy"; - private IBackupFileSystem fs; + private final IBackupFileSystem fs; @Inject public UpdateCleanupPolicy(IConfiguration config, @Named("backup") IBackupFileSystem fs) { diff --git a/priam/src/main/java/com/netflix/priam/aws/auth/EC2RoleAssumptionCredential.java b/priam/src/main/java/com/netflix/priam/aws/auth/EC2RoleAssumptionCredential.java index f19805256..279c80ad9 100644 --- a/priam/src/main/java/com/netflix/priam/aws/auth/EC2RoleAssumptionCredential.java +++ b/priam/src/main/java/com/netflix/priam/aws/auth/EC2RoleAssumptionCredential.java @@ -24,9 +24,9 @@ public class EC2RoleAssumptionCredential implements ICredential { private static final String AWS_ROLE_ASSUMPTION_SESSION_NAME = "AwsRoleAssumptionSession"; - private ICredential cred; - private IConfiguration config; - private InstanceEnvIdentity insEnvIdentity; + private final ICredential cred; + private final IConfiguration config; + private final InstanceEnvIdentity insEnvIdentity; private AWSCredentialsProvider stsSessionCredentialsProvider; @Inject @@ -42,7 +42,7 @@ public AWSCredentialsProvider getAwsCredentialProvider() { synchronized (this) { if (this.stsSessionCredentialsProvider == null) { - String roleArn = null; + String roleArn; /** * Create the assumed IAM role based on the environment. * For example, if the current environment is VPC, diff --git a/priam/src/main/java/com/netflix/priam/aws/auth/S3InstanceCredential.java b/priam/src/main/java/com/netflix/priam/aws/auth/S3InstanceCredential.java index 0ef0fc0f9..654c13ce2 100755 --- a/priam/src/main/java/com/netflix/priam/aws/auth/S3InstanceCredential.java +++ b/priam/src/main/java/com/netflix/priam/aws/auth/S3InstanceCredential.java @@ -24,7 +24,7 @@ */ public class S3InstanceCredential implements IS3Credential { - private InstanceProfileCredentialsProvider credentialsProvider; + private final InstanceProfileCredentialsProvider credentialsProvider; public S3InstanceCredential() { this.credentialsProvider = InstanceProfileCredentialsProvider.getInstance(); diff --git a/priam/src/main/java/com/netflix/priam/aws/auth/S3RoleAssumptionCredential.java b/priam/src/main/java/com/netflix/priam/aws/auth/S3RoleAssumptionCredential.java index 1296f7ea7..cb96b9d1f 100755 --- a/priam/src/main/java/com/netflix/priam/aws/auth/S3RoleAssumptionCredential.java +++ b/priam/src/main/java/com/netflix/priam/aws/auth/S3RoleAssumptionCredential.java @@ -30,8 +30,8 @@ public class S3RoleAssumptionCredential implements IS3Credential { private static final String AWS_ROLE_ASSUMPTION_SESSION_NAME = "S3RoleAssumptionSession"; private static final Logger logger = LoggerFactory.getLogger(S3RoleAssumptionCredential.class); - private ICredential cred; - private IConfiguration config; + private final ICredential cred; + private final IConfiguration config; private AWSCredentialsProvider stsSessionCredentialsProvider; @Inject diff --git a/priam/src/main/java/com/netflix/priam/backup/AbstractBackupPath.java b/priam/src/main/java/com/netflix/priam/backup/AbstractBackupPath.java index dc14a187e..89906daef 100644 --- a/priam/src/main/java/com/netflix/priam/backup/AbstractBackupPath.java +++ b/priam/src/main/java/com/netflix/priam/backup/AbstractBackupPath.java @@ -150,7 +150,7 @@ public String match(Date start, Date end) { * Local restore file */ public File newRestoreFile() { - StringBuffer buff = new StringBuffer(); + StringBuilder buff = new StringBuilder(); if (type == BackupFileType.CL) { buff.append(config.getBackupCommitLogLocation()).append(PATH_SEP); } else { @@ -176,9 +176,7 @@ public int compareTo(AbstractBackupPath o) { @Override public boolean equals(Object obj) { - if (!obj.getClass().equals(this.getClass())) - return false; - return getRemotePath().equals(((AbstractBackupPath) obj).getRemotePath()); + return obj.getClass().equals(this.getClass()) && getRemotePath().equals(((AbstractBackupPath) obj).getRemotePath()); } /** @@ -291,7 +289,7 @@ public long getLastModified() { } public static class RafInputStream extends InputStream implements AutoCloseable { - private RandomAccessFile raf; + private final RandomAccessFile raf; public RafInputStream(RandomAccessFile raf) { this.raf = raf; diff --git a/priam/src/main/java/com/netflix/priam/backup/AbstractFileSystem.java b/priam/src/main/java/com/netflix/priam/backup/AbstractFileSystem.java index 7be1ed6a7..0c63f5172 100644 --- a/priam/src/main/java/com/netflix/priam/backup/AbstractFileSystem.java +++ b/priam/src/main/java/com/netflix/priam/backup/AbstractFileSystem.java @@ -33,7 +33,6 @@ import java.io.FileNotFoundException; import java.nio.file.Path; -import java.util.HashSet; import java.util.Set; import java.util.concurrent.*; @@ -46,10 +45,10 @@ public abstract class AbstractFileSystem implements IBackupFileSystem, EventGenerator { private static final Logger logger = LoggerFactory.getLogger(AbstractFileSystem.class); private final CopyOnWriteArrayList> observers = new CopyOnWriteArrayList<>(); - protected BackupMetrics backupMetrics; - private Set tasksQueued; - private ThreadPoolExecutor fileUploadExecutor; - private ThreadPoolExecutor fileDownloadExecutor; + protected final BackupMetrics backupMetrics; + private final Set tasksQueued; + private final ThreadPoolExecutor fileUploadExecutor; + private final ThreadPoolExecutor fileDownloadExecutor; @Inject public AbstractFileSystem(IConfiguration configuration, BackupMetrics backupMetrics, @@ -138,8 +137,8 @@ public Long retriableCall() throws Exception { notifyEventSuccess(new BackupEvent(path)); logger.info("Successfully uploaded file: {} to location: {}", localPath, remotePath); - if (deleteAfterSuccessfulUpload) - FileUtils.deleteQuietly(localPath.toFile()); + if (deleteAfterSuccessfulUpload && !FileUtils.deleteQuietly(localPath.toFile())) + logger.warn(String.format("Failed to delete local file %s.", localPath.toFile().getAbsolutePath())); } catch (Exception e) { backupMetrics.incrementInvalidUploads(); diff --git a/priam/src/main/java/com/netflix/priam/backup/BackupMetadata.java b/priam/src/main/java/com/netflix/priam/backup/BackupMetadata.java index e6101c796..d7eb5aefc 100644 --- a/priam/src/main/java/com/netflix/priam/backup/BackupMetadata.java +++ b/priam/src/main/java/com/netflix/priam/backup/BackupMetadata.java @@ -56,9 +56,7 @@ public boolean equals(Object o) { BackupMetadata that = (BackupMetadata) o; - if (!this.snapshotDate.equals(that.snapshotDate)) return false; - if (!this.token.equals(that.token)) return false; - return this.start.equals(that.start); + return this.snapshotDate.equals(that.snapshotDate) && this.token.equals(that.token) && this.start.equals(that.start); } @Override diff --git a/priam/src/main/java/com/netflix/priam/backup/BackupRestoreUtil.java b/priam/src/main/java/com/netflix/priam/backup/BackupRestoreUtil.java index c10a3cc30..bf150477e 100644 --- a/priam/src/main/java/com/netflix/priam/backup/BackupRestoreUtil.java +++ b/priam/src/main/java/com/netflix/priam/backup/BackupRestoreUtil.java @@ -31,11 +31,11 @@ */ public class BackupRestoreUtil { private static final Logger logger = LoggerFactory.getLogger(BackupRestoreUtil.class); - private static Pattern columnFamilyFilterPattern = Pattern.compile(".\\.."); + private static final Pattern columnFamilyFilterPattern = Pattern.compile(".\\.."); private Map> includeFilter; private Map> excludeFilter; - public static final List FILTER_KEYSPACE = Arrays.asList("OpsCenter"); + public static final List FILTER_KEYSPACE = Collections.singletonList("OpsCenter"); private static final Map> FILTER_COLUMN_FAMILY = ImmutableMap.of("system", Arrays.asList("local", "peers", "hints", "compactions_in_progress", "LocationInfo")); @Inject @@ -59,10 +59,10 @@ public static final Map> getFilter(String inputFilter) thro final Map> columnFamilyFilter = new HashMap<>(); //key: keyspace, value: a list of CFs within the keyspace String[] filters = inputFilter.split(","); - for (int i = 0; i < filters.length; i++) { // process filter of form keyspace.* or keyspace.columnfamily - if (columnFamilyFilterPattern.matcher(filters[i]).find()) { + for (String cfFilter : filters) { // process filter of form keyspace.* or keyspace.columnfamily + if (columnFamilyFilterPattern.matcher(cfFilter).find()) { - String[] filter = filters[i].split("\\."); + String[] filter = cfFilter.split("\\."); String keyspaceName = filter[0]; String columnFamilyName = filter[1]; @@ -75,7 +75,7 @@ public static final Map> getFilter(String inputFilter) thro columnFamilyFilter.put(keyspaceName, existingCfs); } else { - throw new IllegalArgumentException("Column family filter format is not valid. Format needs to be \"keyspace.columnfamily\". Invalid input: " + filters[i]); + throw new IllegalArgumentException("Column family filter format is not valid. Format needs to be \"keyspace.columnfamily\". Invalid input: " + cfFilter); } } return columnFamilyFilter; diff --git a/priam/src/main/java/com/netflix/priam/backup/BackupStatusMgr.java b/priam/src/main/java/com/netflix/priam/backup/BackupStatusMgr.java index f645901a3..bb35a8306 100644 --- a/priam/src/main/java/com/netflix/priam/backup/BackupStatusMgr.java +++ b/priam/src/main/java/com/netflix/priam/backup/BackupStatusMgr.java @@ -40,8 +40,8 @@ public abstract class BackupStatusMgr implements IBackupStatusMgr { * Note: A {@link LinkedList} was chosen for fastest retrieval of latest snapshot. */ Map> backupMetadataMap; - int capacity; - private InstanceState instanceState; + final int capacity; + private final InstanceState instanceState; /** * @param capacity Capacity to hold in-memory snapshot status days. @@ -176,10 +176,9 @@ public void failed(BackupMetadata backupMetadata) { @Override public String toString() { - final StringBuffer sb = new StringBuffer("BackupStatusMgr{"); - sb.append("backupMetadataMap=").append(backupMetadataMap); - sb.append(", capacity=").append(capacity); - sb.append('}'); - return sb.toString(); + String sb = "BackupStatusMgr{" + "backupMetadataMap=" + backupMetadataMap + + ", capacity=" + capacity + + '}'; + return sb; } } diff --git a/priam/src/main/java/com/netflix/priam/backup/BackupVerification.java b/priam/src/main/java/com/netflix/priam/backup/BackupVerification.java index f3d976005..9cb69d438 100644 --- a/priam/src/main/java/com/netflix/priam/backup/BackupVerification.java +++ b/priam/src/main/java/com/netflix/priam/backup/BackupVerification.java @@ -43,8 +43,8 @@ public class BackupVerification { private static final Logger logger = LoggerFactory.getLogger(BackupVerification.class); - private IBackupFileSystem bkpStatusFs; - private IConfiguration config; + private final IBackupFileSystem bkpStatusFs; + private final IConfiguration config; @Inject BackupVerification(@Named("backup") IBackupFileSystem bkpStatusFs, IConfiguration config) { @@ -122,8 +122,7 @@ public BackupVerificationResult verifyBackup(List metadata, Date JSONParser jsonParser = new JSONParser(); org.json.simple.JSONArray fileList = (org.json.simple.JSONArray) jsonParser.parse(new FileReader(metaFileLocation.toFile())); - for (int i = 0; i < fileList.size(); i++) - metaFileList.add(fileList.get(i).toString()); + for (Object aFileList : fileList) metaFileList.add(aFileList.toString()); } catch (Exception e) { logger.error("Error while fetching meta.json from path: {}", metas.get(0), e); diff --git a/priam/src/main/java/com/netflix/priam/backup/CommitLogBackup.java b/priam/src/main/java/com/netflix/priam/backup/CommitLogBackup.java index 9c9c5d86a..849d5700b 100644 --- a/priam/src/main/java/com/netflix/priam/backup/CommitLogBackup.java +++ b/priam/src/main/java/com/netflix/priam/backup/CommitLogBackup.java @@ -34,7 +34,7 @@ public class CommitLogBackup { private static final Logger logger = LoggerFactory.getLogger(CommitLogBackup.class); private final Provider pathFactory; - static List observers = new ArrayList(); + static final List observers = new ArrayList(); private final List clRemotePaths = new ArrayList(); private final IBackupFileSystem fs; diff --git a/priam/src/main/java/com/netflix/priam/backup/CommitLogBackupTask.java b/priam/src/main/java/com/netflix/priam/backup/CommitLogBackupTask.java index 85950beeb..1b7e208f2 100644 --- a/priam/src/main/java/com/netflix/priam/backup/CommitLogBackupTask.java +++ b/priam/src/main/java/com/netflix/priam/backup/CommitLogBackupTask.java @@ -34,11 +34,11 @@ //Provide this to be run as a Quart job @Singleton public class CommitLogBackupTask extends AbstractBackup { - public static String JOBNAME = "CommitLogBackup"; + public static final String JOBNAME = "CommitLogBackup"; private static final Logger logger = LoggerFactory.getLogger(CommitLogBackupTask.class); private final List clRemotePaths = new ArrayList<>(); - private static List observers = new ArrayList<>(); + private static final List observers = new ArrayList<>(); private final CommitLogBackup clBackup; diff --git a/priam/src/main/java/com/netflix/priam/backup/FileSnapshotStatusMgr.java b/priam/src/main/java/com/netflix/priam/backup/FileSnapshotStatusMgr.java index 721c66fe3..f01a2320c 100644 --- a/priam/src/main/java/com/netflix/priam/backup/FileSnapshotStatusMgr.java +++ b/priam/src/main/java/com/netflix/priam/backup/FileSnapshotStatusMgr.java @@ -37,7 +37,7 @@ public class FileSnapshotStatusMgr extends BackupStatusMgr { private static final Logger logger = LoggerFactory.getLogger(FileSnapshotStatusMgr.class); private static final int IN_MEMORY_SNAPSHOT_CAPACITY = 60; - private String filename; + private final String filename; /** * Constructor to initialize the file based snapshot status manager. diff --git a/priam/src/main/java/com/netflix/priam/backup/IncrementalBackup.java b/priam/src/main/java/com/netflix/priam/backup/IncrementalBackup.java index ee570dff8..398749822 100644 --- a/priam/src/main/java/com/netflix/priam/backup/IncrementalBackup.java +++ b/priam/src/main/java/com/netflix/priam/backup/IncrementalBackup.java @@ -39,9 +39,9 @@ public class IncrementalBackup extends AbstractBackup implements IIncrementalBac private static final Logger logger = LoggerFactory.getLogger(IncrementalBackup.class); public static final String JOBNAME = "IncrementalBackup"; private final List incrementalRemotePaths = new ArrayList<>(); - private IncrementalMetaData metaData; - private BackupRestoreUtil backupRestoreUtil; - private static List observers = new ArrayList<>(); + private final IncrementalMetaData metaData; + private final BackupRestoreUtil backupRestoreUtil; + private static final List observers = new ArrayList<>(); @Inject public IncrementalBackup(IConfiguration config, Provider pathFactory, IFileSystemContext backupFileSystemCtx diff --git a/priam/src/main/java/com/netflix/priam/backup/IncrementalMetaData.java b/priam/src/main/java/com/netflix/priam/backup/IncrementalMetaData.java index 2d301062c..ed6175929 100644 --- a/priam/src/main/java/com/netflix/priam/backup/IncrementalMetaData.java +++ b/priam/src/main/java/com/netflix/priam/backup/IncrementalMetaData.java @@ -38,7 +38,7 @@ public void setMetaFileName(String name) { @Override public File createTmpMetaFile() throws IOException { - File metafile = null, destFile = null; + File metafile, destFile; if (this.metaFileName == null) { diff --git a/priam/src/main/java/com/netflix/priam/backup/MetaData.java b/priam/src/main/java/com/netflix/priam/backup/MetaData.java index 6a56ac606..6afa5d8fc 100644 --- a/priam/src/main/java/com/netflix/priam/backup/MetaData.java +++ b/priam/src/main/java/com/netflix/priam/backup/MetaData.java @@ -44,7 +44,7 @@ public class MetaData { private static final Logger logger = LoggerFactory.getLogger(MetaData.class); private final Provider pathFactory; - private static List observers = new ArrayList<>(); + private static final List observers = new ArrayList<>(); private final List metaRemotePaths = new ArrayList<>(); private final IBackupFileSystem fs; @@ -138,9 +138,9 @@ public List toJson(File input) { List files = Lists.newArrayList(); try { JSONArray jsonObj = (JSONArray) new JSONParser().parse(new FileReader(input)); - for (int i = 0; i < jsonObj.size(); i++) { + for (Object aJsonObj : jsonObj) { AbstractBackupPath p = pathFactory.get(); - p.parseRemote((String) jsonObj.get(i)); + p.parseRemote((String) aJsonObj); files.add(p); } diff --git a/priam/src/main/java/com/netflix/priam/backup/SnapshotBackup.java b/priam/src/main/java/com/netflix/priam/backup/SnapshotBackup.java index 2b52b71c9..186744cea 100644 --- a/priam/src/main/java/com/netflix/priam/backup/SnapshotBackup.java +++ b/priam/src/main/java/com/netflix/priam/backup/SnapshotBackup.java @@ -44,15 +44,15 @@ public class SnapshotBackup extends AbstractBackup { public static final String JOBNAME = "SnapshotBackup"; private final MetaData metaData; private final List snapshotRemotePaths = new ArrayList<>(); - private static List observers = new ArrayList<>(); + private static final List observers = new ArrayList<>(); private final ThreadSleeper sleeper = new ThreadSleeper(); private static final long WAIT_TIME_MS = 60 * 1000 * 10; - private InstanceIdentity instanceIdentity; - private IBackupStatusMgr snapshotStatusMgr; - private BackupRestoreUtil backupRestoreUtil; + private final InstanceIdentity instanceIdentity; + private final IBackupStatusMgr snapshotStatusMgr; + private final BackupRestoreUtil backupRestoreUtil; private String snapshotName = null; private List abstractBackupPaths = null; - private CassandraOperations cassandraOperations; + private final CassandraOperations cassandraOperations; @Inject public SnapshotBackup(IConfiguration config, Provider pathFactory, diff --git a/priam/src/main/java/com/netflix/priam/backupv2/FileUploadResult.java b/priam/src/main/java/com/netflix/priam/backupv2/FileUploadResult.java index 94e169711..3f8859fe2 100644 --- a/priam/src/main/java/com/netflix/priam/backupv2/FileUploadResult.java +++ b/priam/src/main/java/com/netflix/priam/backupv2/FileUploadResult.java @@ -36,8 +36,8 @@ public class FileUploadResult { @GsonJsonSerializer.PriamAnnotation.GsonIgnore private String columnFamilyName; private Instant lastModifiedTime; - private Instant fileCreationTime; - private long fileSizeOnDisk; //Size on disk in bytes + private final Instant fileCreationTime; + private final long fileSizeOnDisk; //Size on disk in bytes private Boolean isUploaded; //Valid compression technique for now is SNAPPY only. Future we need to support LZ4 and NONE private ICompression.CompressionAlgorithm compression = ICompression.CompressionAlgorithm.SNAPPY; diff --git a/priam/src/main/java/com/netflix/priam/backupv2/LocalDBReaderWriter.java b/priam/src/main/java/com/netflix/priam/backupv2/LocalDBReaderWriter.java index 8ab25f49c..163b08dd5 100644 --- a/priam/src/main/java/com/netflix/priam/backupv2/LocalDBReaderWriter.java +++ b/priam/src/main/java/com/netflix/priam/backupv2/LocalDBReaderWriter.java @@ -46,7 +46,7 @@ */ public class LocalDBReaderWriter { private static final Logger logger = LoggerFactory.getLogger(LocalDBReaderWriter.class); - private IConfiguration configuration; + private final IConfiguration configuration; public static final String LOCAL_DB = "localdb"; @Inject diff --git a/priam/src/main/java/com/netflix/priam/backupv2/MetaFileReader.java b/priam/src/main/java/com/netflix/priam/backupv2/MetaFileReader.java index a8df6c6ab..4b6068a18 100644 --- a/priam/src/main/java/com/netflix/priam/backupv2/MetaFileReader.java +++ b/priam/src/main/java/com/netflix/priam/backupv2/MetaFileReader.java @@ -34,7 +34,6 @@ public abstract class MetaFileReader { private static final Logger logger = LoggerFactory.getLogger(MetaFileReader.class); - private JsonReader jsonReader; private MetaFileInfo metaFileInfo; public MetaFileInfo getMetaFileInfo() { @@ -45,7 +44,7 @@ public MetaFileInfo getMetaFileInfo() { * Reads the local meta file as denoted by metaFilePath. * * @param metaFilePath local file path for the meta file. - * @throws IOException + * @throws IOException if not enough permissions or file is not valid format. */ public void readMeta(Path metaFilePath) throws IOException { //Validate if meta file exists and is right file name. @@ -55,7 +54,7 @@ public void readMeta(Path metaFilePath) throws IOException { //Read the meta file. logger.info("Trying to read the meta file: {}", metaFilePath); - jsonReader = new JsonReader(new FileReader(metaFilePath.toFile())); + JsonReader jsonReader = new JsonReader(new FileReader(metaFilePath.toFile())); jsonReader.beginObject(); while (jsonReader.hasNext()) { switch (jsonReader.nextName()) { diff --git a/priam/src/main/java/com/netflix/priam/backupv2/MetaFileWriterBuilder.java b/priam/src/main/java/com/netflix/priam/backupv2/MetaFileWriterBuilder.java index b8546bae4..d2c9d8cc4 100644 --- a/priam/src/main/java/com/netflix/priam/backupv2/MetaFileWriterBuilder.java +++ b/priam/src/main/java/com/netflix/priam/backupv2/MetaFileWriterBuilder.java @@ -44,7 +44,7 @@ * Created by aagrawal on 6/12/18. */ public class MetaFileWriterBuilder { - private MetaFileWriter metaFileWriter; + private final MetaFileWriter metaFileWriter; private static final Logger logger = LoggerFactory.getLogger(MetaFileWriterBuilder.class); @Inject @@ -74,8 +74,8 @@ public static class MetaFileWriter implements StartStep, DataStep, UploadStep { private final Provider pathFactory; private final IBackupFileSystem backupFileSystem; - private MetaFileInfo metaFileInfo; - private MetaFileManager metaFileManager; + private final MetaFileInfo metaFileInfo; + private final MetaFileManager metaFileManager; private JsonWriter jsonWriter; private Path metaFilePath; @@ -92,7 +92,7 @@ private MetaFileWriter(IConfiguration configuration, InstanceIdentity instanceId /** * Start the generation of meta file. * - * @throws IOException + * @throws IOException if unable to write to meta file (permissions, disk full etc) */ public DataStep startMetaFileGeneration(Instant snapshotInstant) throws IOException { //Compute meta file name. @@ -115,7 +115,7 @@ public DataStep startMetaFileGeneration(Instant snapshotInstant) throws IOExcept * Add {@link ColumnfamilyResult} after it has been processed so it can be streamed to meta.json. Streaming write to meta.json is required so we don't get Priam OOM. * * @param columnfamilyResult a POJO encapsulating the column family result - * @throws IOException + * @throws IOException if unable to write to the file or if JSON is not valid */ public MetaFileWriterBuilder.DataStep addColumnfamilyResult(ColumnfamilyResult columnfamilyResult) throws IOException { if (jsonWriter == null) @@ -130,7 +130,7 @@ public MetaFileWriterBuilder.DataStep addColumnfamilyResult(ColumnfamilyResult c * Finish the generation of meta.json file and save it on local media. * * @return {@link Path} to the local meta.json produced. - * @throws IOException + * @throws IOException if unable to write to file or if JSON is not valid */ public MetaFileWriterBuilder.UploadStep endMetaFileGeneration() throws IOException { if (jsonWriter == null) diff --git a/priam/src/main/java/com/netflix/priam/backupv2/PrefixGenerator.java b/priam/src/main/java/com/netflix/priam/backupv2/PrefixGenerator.java index 631e2a9f7..af62ec325 100644 --- a/priam/src/main/java/com/netflix/priam/backupv2/PrefixGenerator.java +++ b/priam/src/main/java/com/netflix/priam/backupv2/PrefixGenerator.java @@ -32,8 +32,8 @@ */ public class PrefixGenerator { - private IConfiguration configuration; - private InstanceIdentity instanceIdentity; + private final IConfiguration configuration; + private final InstanceIdentity instanceIdentity; @Inject PrefixGenerator(IConfiguration configuration, InstanceIdentity instanceIdentity) { diff --git a/priam/src/main/java/com/netflix/priam/cli/LightGuiceModule.java b/priam/src/main/java/com/netflix/priam/cli/LightGuiceModule.java index ff5fb0650..8287beb1a 100644 --- a/priam/src/main/java/com/netflix/priam/cli/LightGuiceModule.java +++ b/priam/src/main/java/com/netflix/priam/cli/LightGuiceModule.java @@ -26,7 +26,7 @@ class LightGuiceModule extends AbstractModule { @Override protected void configure() { - bind(IConfiguration.class).to(PriamConfiguration.class).asEagerSingleton(); + bind(IConfiguration.class).asEagerSingleton(); bind(IMembership.class).to(StaticMembership.class); bind(IBackupFileSystem.class).to(S3FileSystem.class); } diff --git a/priam/src/main/java/com/netflix/priam/cli/StaticMembership.java b/priam/src/main/java/com/netflix/priam/cli/StaticMembership.java index e6a739f2e..928c49547 100644 --- a/priam/src/main/java/com/netflix/priam/cli/StaticMembership.java +++ b/priam/src/main/java/com/netflix/priam/cli/StaticMembership.java @@ -57,7 +57,7 @@ public StaticMembership() throws IOException { for (String name : config.stringPropertyNames()) { if (name.startsWith(INSTANCES_PRE)) { racCount += 1; - if (name == INSTANCES_PRE + racName) + if (name.equals(INSTANCES_PRE + racName)) racMembership = Arrays.asList(config.getProperty(name).split(",")); } } diff --git a/priam/src/main/java/com/netflix/priam/cluster/management/IClusterManagement.java b/priam/src/main/java/com/netflix/priam/cluster/management/IClusterManagement.java index 3ba49c454..be8737517 100644 --- a/priam/src/main/java/com/netflix/priam/cluster/management/IClusterManagement.java +++ b/priam/src/main/java/com/netflix/priam/cluster/management/IClusterManagement.java @@ -33,9 +33,9 @@ public abstract class IClusterManagement extends Task { public enum Task {FLUSH, COMPACTION} private static final Logger logger = LoggerFactory.getLogger(IClusterManagement.class); - private Task taskType; - private IMeasurement measurement; - private static Lock lock = new ReentrantLock(); + private final Task taskType; + private final IMeasurement measurement; + private static final Lock lock = new ReentrantLock(); protected IClusterManagement(IConfiguration config, Task taskType, IMeasurement measurement) { super(config); diff --git a/priam/src/main/java/com/netflix/priam/compress/ChunkedStream.java b/priam/src/main/java/com/netflix/priam/compress/ChunkedStream.java index 4aff039c3..8bfd4635f 100644 --- a/priam/src/main/java/com/netflix/priam/compress/ChunkedStream.java +++ b/priam/src/main/java/com/netflix/priam/compress/ChunkedStream.java @@ -30,11 +30,11 @@ */ public class ChunkedStream implements Iterator { private boolean hasnext = true; - private ByteArrayOutputStream bos; - private SnappyOutputStream compress; - private InputStream origin; - private long chunkSize; - private static int BYTES_TO_READ = 2048; + private final ByteArrayOutputStream bos; + private final SnappyOutputStream compress; + private final InputStream origin; + private final long chunkSize; + private static final int BYTES_TO_READ = 2048; public ChunkedStream(InputStream is, long chunkSize) throws IOException { this.origin = is; diff --git a/priam/src/main/java/com/netflix/priam/config/IConfiguration.java b/priam/src/main/java/com/netflix/priam/config/IConfiguration.java index d51dc4348..a2bd43163 100644 --- a/priam/src/main/java/com/netflix/priam/config/IConfiguration.java +++ b/priam/src/main/java/com/netflix/priam/config/IConfiguration.java @@ -26,11 +26,6 @@ import com.netflix.priam.scheduler.SchedulerType; import com.netflix.priam.scheduler.UnsupportedTypeException; -import java.io.File; -import java.lang.reflect.InvocationTargetException; -import java.lang.reflect.Method; -import java.util.Collections; -import java.util.HashMap; import java.util.List; import java.util.Map; diff --git a/priam/src/main/java/com/netflix/priam/config/PriamConfiguration.java b/priam/src/main/java/com/netflix/priam/config/PriamConfiguration.java index 75d8d179d..76ae9699a 100644 --- a/priam/src/main/java/com/netflix/priam/config/PriamConfiguration.java +++ b/priam/src/main/java/com/netflix/priam/config/PriamConfiguration.java @@ -187,51 +187,15 @@ public class PriamConfiguration implements IConfiguration { //== vpc specific private String NETWORK_VPC; //Fetch the vpc id of running instance - // Defaults - private final String DEFAULT_CLUSTER_NAME = "cass_cluster"; private final String CASS_BASE_DATA_DIR = "/var/lib/cassandra"; - private final String DEFAULT_DATA_LOCATION = CASS_BASE_DATA_DIR + "/data"; - private final String DEFAULT_LOGS_LOCATION = CASS_BASE_DATA_DIR +"/logs"; - private final String DEFAULT_COMMIT_LOG_LOCATION = "/var/lib/cassandra/commitlog"; - private final String DEFAULT_CACHE_LOCATION = "/var/lib/cassandra/saved_caches"; - private final String DEFAULT_HINTS_DIR_LOCATION = "/var/lib/cassandra/hints"; - private final String DEFAULT_ENDPOINT_SNITCH = "org.apache.cassandra.locator.Ec2Snitch"; - private final String DEFAULT_SEED_PROVIDER = "com.netflix.priam.cassandra.extensions.NFSeedProvider"; - private final String DEFAULT_PARTITIONER = "org.apache.cassandra.dht.RandomPartitioner"; public static final String DEFAULT_AUTHENTICATOR = "org.apache.cassandra.auth.AllowAllAuthenticator"; public static final String DEFAULT_AUTHORIZER = "org.apache.cassandra.auth.AllowAllAuthorizer"; public static final String DEFAULT_COMMITLOG_PROPS_FILE = "/conf/commitlog_archiving.properties"; - // rpm based. Can be modified for tar based. - private final String DEFAULT_CASS_HOME_DIR = "/etc/cassandra"; - private final String DEFAULT_CASS_START_SCRIPT = "/etc/init.d/cassandra start"; - private final String DEFAULT_CASS_STOP_SCRIPT = "/etc/init.d/cassandra stop"; - private final String DEFAULT_BACKUP_LOCATION = "backup"; - private final String DEFAULT_BUCKET_NAME = "cassandra-archive"; // private String DEFAULT_AVAILABILITY_ZONES = ""; private List DEFAULT_AVAILABILITY_ZONES = ImmutableList.of(); - private final String DEFAULT_CASS_PROCESS_NAME = "CassandraDaemon"; - - private final String DEFAULT_MAX_DIRECT_MEM = "50G"; - private final String DEFAULT_MAX_HEAP = "8G"; - private final String DEFAULT_MAX_NEWGEN_HEAP = "2G"; - private final int DEFAULT_JMX_PORT = 7199; - private final int DEFAULT_THRIFT_PORT = 9160; - private final int DEFAULT_NATIVE_PROTOCOL_PORT = 9042; - private final int DEFAULT_STORAGE_PORT = 7000; - private final int DEFAULT_SSL_STORAGE_PORT = 7001; - private final int DEFAULT_BACKUP_HOUR = 12; - private final String DEFAULT_BACKUP_CRON_EXPRESSION = "0 0 12 1/1 * ? *"; //Backup daily at 12. - private final int DEFAULT_BACKUP_THREADS = 2; - private final int DEFAULT_RESTORE_THREADS = 8; - private final int DEFAULT_BACKUP_CHUNK_SIZE = 10; - private final int DEFAULT_BACKUP_RETENTION = 0; - private final int DEFAULT_VNODE_NUM_TOKENS = 1; + private final int DEFAULT_HINTS_MAX_THREADS = 2; //default value from 1.2 yaml - private final int DEFAULT_HINTS_THROTTLE_KB = 1024; //default value from 1.2 yaml - private final String DEFAULT_INTERNODE_COMPRESSION = "all"; //default value from 1.2 yaml - // Default to restarting Cassandra automatically once per hour. - private final int DEFAULT_REMEDIATE_DEAD_CASSANDRA_RATE_S = 60 * 60; private static final String DEFAULT_RPC_SERVER_TYPE = "hsha"; private static final int DEFAULT_RPC_MIN_THREADS = 16; @@ -248,7 +212,7 @@ public class PriamConfiguration implements IConfiguration { private final ICredential provider; @JsonIgnore - private InstanceEnvIdentity insEnvIdentity; + private final InstanceEnvIdentity insEnvIdentity; @JsonIgnore private InstanceDataRetriever instanceDataRetriever; @@ -382,12 +346,12 @@ public String getInstanceName(){ @Override public String getCassStartupScript() { - return config.get(CONFIG_CASS_START_SCRIPT, DEFAULT_CASS_START_SCRIPT); + return config.get(CONFIG_CASS_START_SCRIPT, "/etc/init.d/cassandra start"); } @Override public String getCassStopScript() { - return config.get(CONFIG_CASS_STOP_SCRIPT, DEFAULT_CASS_STOP_SCRIPT); + return config.get(CONFIG_CASS_STOP_SCRIPT, "/etc/init.d/cassandra stop"); } @Override @@ -397,27 +361,27 @@ public int getGracefulDrainHealthWaitSeconds() { @Override public int getRemediateDeadCassandraRate() { - return config.get(CONFIG_REMEDIATE_DEAD_CASSANDRA_RATE_S, DEFAULT_REMEDIATE_DEAD_CASSANDRA_RATE_S); + return config.get(CONFIG_REMEDIATE_DEAD_CASSANDRA_RATE_S, 3600); //Default to once per hour } @Override public String getCassHome() { - return config.get(CONFIG_CASS_HOME_DIR, DEFAULT_CASS_HOME_DIR); + return config.get(CONFIG_CASS_HOME_DIR, "/etc/cassandra"); } @Override public String getBackupLocation() { - return config.get(CONFIG_S3_BASE_DIR, DEFAULT_BACKUP_LOCATION); + return config.get(CONFIG_S3_BASE_DIR, "backup"); } @Override public String getBackupPrefix() { - return config.get(CONFIG_BUCKET_NAME, DEFAULT_BUCKET_NAME); + return config.get(CONFIG_BUCKET_NAME, "cassandra-archive"); } @Override public int getBackupRetentionDays() { - return config.get(CONFIG_BACKUP_RETENTION, DEFAULT_BACKUP_RETENTION); + return config.get(CONFIG_BACKUP_RETENTION, 0); } @Override @@ -432,28 +396,28 @@ public String getRestorePrefix() { @Override public String getDataFileLocation() { - return config.get(CONFIG_DATA_LOCATION, DEFAULT_DATA_LOCATION); + return config.get(CONFIG_DATA_LOCATION, CASS_BASE_DATA_DIR + "/data"); } @Override public String getLogDirLocation() { - return config.get(CONFIG_LOGS_LOCATION, DEFAULT_LOGS_LOCATION); + return config.get(CONFIG_LOGS_LOCATION, CASS_BASE_DATA_DIR + "/logs"); } @Override public String getHintsLocation() { - return config.get(PRIAM_PRE + ".hints.location", DEFAULT_HINTS_DIR_LOCATION); + return config.get(PRIAM_PRE + ".hints.location", CASS_BASE_DATA_DIR + "/hints"); } @Override public String getCacheLocation() { - return config.get(CONFIG_SAVE_CACHE_LOCATION, DEFAULT_CACHE_LOCATION); + return config.get(CONFIG_SAVE_CACHE_LOCATION, CASS_BASE_DATA_DIR + "/saved_caches"); } @Override public String getCommitLogLocation() { - return config.get(CONFIG_CL_LOCATION, DEFAULT_COMMIT_LOG_LOCATION); + return config.get(CONFIG_CL_LOCATION, CASS_BASE_DATA_DIR + "/commitlog"); } @Override @@ -463,13 +427,13 @@ public String getBackupCommitLogLocation() { @Override public long getBackupChunkSize() { - long size = config.get(CONFIG_BACKUP_CHUNK_SIZE, DEFAULT_BACKUP_CHUNK_SIZE); + long size = config.get(CONFIG_BACKUP_CHUNK_SIZE, 10); return size * 1024 * 1024L; } @Override public int getJmxPort() { - return config.get(CONFIG_JMX_LISTERN_PORT_NAME, DEFAULT_JMX_PORT); + return config.get(CONFIG_JMX_LISTERN_PORT_NAME, 7199); } @Override @@ -491,32 +455,32 @@ public boolean enableRemoteJMX() { } public int getNativeTransportPort() { - return config.get(CONFIG_NATIVE_PROTOCOL_PORT, DEFAULT_NATIVE_PROTOCOL_PORT); + return config.get(CONFIG_NATIVE_PROTOCOL_PORT, 9042); } @Override public int getThriftPort() { - return config.get(CONFIG_THRIFT_LISTEN_PORT_NAME, DEFAULT_THRIFT_PORT); + return config.get(CONFIG_THRIFT_LISTEN_PORT_NAME, 9160); } @Override public int getStoragePort() { - return config.get(CONFIG_STORAGE_LISTERN_PORT_NAME, DEFAULT_STORAGE_PORT); + return config.get(CONFIG_STORAGE_LISTERN_PORT_NAME, 7000); } @Override public int getSSLStoragePort() { - return config.get(CONFIG_SSL_STORAGE_LISTERN_PORT_NAME, DEFAULT_SSL_STORAGE_PORT); + return config.get(CONFIG_SSL_STORAGE_LISTERN_PORT_NAME, 7001); } @Override public String getSnitch() { - return config.get(CONFIG_ENDPOINT_SNITCH, DEFAULT_ENDPOINT_SNITCH); + return config.get(CONFIG_ENDPOINT_SNITCH, "org.apache.cassandra.locator.Ec2Snitch"); } @Override public String getAppName() { - return config.get(CONFIG_CLUSTER_NAME, DEFAULT_CLUSTER_NAME); + return config.get(CONFIG_CLUSTER_NAME, "cass_cluster"); } @Override @@ -538,27 +502,27 @@ public String getHostname() { @Override public String getHeapSize() { - return config.get(CONFIG_MAX_HEAP_SIZE + getInstanceDataRetriever().getInstanceType(), DEFAULT_MAX_HEAP); + return config.get(CONFIG_MAX_HEAP_SIZE + getInstanceDataRetriever().getInstanceType(), "8G"); } @Override public String getHeapNewSize() { - return config.get(CONFIG_NEW_MAX_HEAP_SIZE + getInstanceDataRetriever().getInstanceType(), DEFAULT_MAX_NEWGEN_HEAP); + return config.get(CONFIG_NEW_MAX_HEAP_SIZE + getInstanceDataRetriever().getInstanceType(), "2G"); } @Override public String getMaxDirectMemory() { - return config.get(CONFIG_DIRECT_MAX_HEAP_SIZE + getInstanceDataRetriever().getInstanceType(), DEFAULT_MAX_DIRECT_MEM); + return config.get(CONFIG_DIRECT_MAX_HEAP_SIZE + getInstanceDataRetriever().getInstanceType(), "50G"); } @Override public int getBackupHour() { - return config.get(CONFIG_BACKUP_HOUR, DEFAULT_BACKUP_HOUR); + return config.get(CONFIG_BACKUP_HOUR, 12); } @Override public String getBackupCronExpression() { - return config.get(CONFIG_BACKUP_CRON_EXPRESSION, DEFAULT_BACKUP_CRON_EXPRESSION); + return config.get(CONFIG_BACKUP_CRON_EXPRESSION, "0 0 12 1/1 * ? *"); //Backup daily at 12 } @Override @@ -671,12 +635,12 @@ public boolean isMultiDC() { @Override public int getBackupThreads() { - return config.get(CONFIG_BACKUP_THREADS, DEFAULT_BACKUP_THREADS); + return config.get(CONFIG_BACKUP_THREADS, 2); } @Override public int getRestoreThreads() { - return config.get(CONFIG_RESTORE_THREADS, DEFAULT_RESTORE_THREADS); + return config.get(CONFIG_RESTORE_THREADS, 8); } @Override @@ -734,7 +698,7 @@ public int getMaxHintWindowInMS() { } public int getHintedHandoffThrottleKb() { - return config.get(CONFIG_HINTS_THROTTLE_KB, DEFAULT_HINTS_THROTTLE_KB); + return config.get(CONFIG_HINTS_THROTTLE_KB, 1024); } @Override @@ -744,7 +708,7 @@ public String getBootClusterName() { @Override public String getSeedProviderName() { - return config.get(CONFIG_SEED_PROVIDER_NAME, DEFAULT_SEED_PROVIDER); + return config.get(CONFIG_SEED_PROVIDER_NAME, "com.netflix.priam.cassandra.extensions.NFSeedProvider"); } public double getMemtableCleanupThreshold() { @@ -757,7 +721,7 @@ public int getStreamingThroughputMB() { } public String getPartitioner() { - return config.get(CONFIG_PARTITIONER, DEFAULT_PARTITIONER); + return config.get(CONFIG_PARTITIONER, "org.apache.cassandra.dht.RandomPartitioner"); } public String getKeyCacheSizeInMB() { @@ -778,11 +742,11 @@ public String getRowCacheKeysToSave() { @Override public String getCassProcessName() { - return config.get(CONFIG_CASS_PROCESS_NAME, DEFAULT_CASS_PROCESS_NAME); + return config.get(CONFIG_CASS_PROCESS_NAME, "CassandraDaemon"); } public int getNumTokens() { - return config.get(CONFIG_VNODE_NUM_TOKENS, DEFAULT_VNODE_NUM_TOKENS); + return config.get(CONFIG_VNODE_NUM_TOKENS, 1); } public String getYamlLocation() { @@ -809,7 +773,7 @@ public boolean doesCassandraStartManually() { } public String getInternodeCompression() { - return config.get(CONFIG_INTERNODE_COMPRESSION, DEFAULT_INTERNODE_COMPRESSION); + return config.get(CONFIG_INTERNODE_COMPRESSION, "all"); } @Override @@ -922,8 +886,8 @@ public Map getExtraEnvParams() { Map extraEnvParamsMap = new HashMap<>(); String[] pairs = envParams.split(","); logger.info("getExtraEnvParams: Extra cass params. From config :{}", envParams); - for (int i = 0; i < pairs.length; i++) { - String[] pair = pairs[i].split("="); + for (String pair1 : pairs) { + String[] pair = pair1.split("="); if (pair.length > 1) { String priamKey = pair[0]; String cassKey = pair[1]; diff --git a/priam/src/main/java/com/netflix/priam/cryptography/pgp/PgpCredential.java b/priam/src/main/java/com/netflix/priam/cryptography/pgp/PgpCredential.java index 174de3cd0..acc2caac9 100755 --- a/priam/src/main/java/com/netflix/priam/cryptography/pgp/PgpCredential.java +++ b/priam/src/main/java/com/netflix/priam/cryptography/pgp/PgpCredential.java @@ -26,7 +26,7 @@ */ public class PgpCredential implements ICredentialGeneric { - private IConfiguration config; + private final IConfiguration config; @Inject public PgpCredential(IConfiguration config) { diff --git a/priam/src/main/java/com/netflix/priam/cryptography/pgp/PgpCryptography.java b/priam/src/main/java/com/netflix/priam/cryptography/pgp/PgpCryptography.java index 26e30ed8f..f3a4f063a 100755 --- a/priam/src/main/java/com/netflix/priam/cryptography/pgp/PgpCryptography.java +++ b/priam/src/main/java/com/netflix/priam/cryptography/pgp/PgpCryptography.java @@ -180,10 +180,10 @@ public class ChunkEncryptorStream implements Iterator { private static final int MAX_CHUNK = 10 * 1024 * 1024; private boolean hasnext = true; - private InputStream is; - private InputStream encryptedSrc; - private ByteArrayOutputStream bos; - private BufferedOutputStream pgout; + private final InputStream is; + private final InputStream encryptedSrc; + private final ByteArrayOutputStream bos; + private final BufferedOutputStream pgout; public ChunkEncryptorStream(InputStream is, String fileName, PGPPublicKey pubKey) { this.is = is; @@ -257,7 +257,7 @@ private byte[] done() throws IOException { public class EncryptedInputStream extends InputStream { - private InputStream srcHandle; //handle to the source stream + private final InputStream srcHandle; //handle to the source stream private ByteArrayOutputStream bos = null; //Handle to encrypted stream private int bosOff = 0; //current position within encrypted stream private OutputStream pgpBosWrapper; //wrapper around the buffer which will contain the encrypted data. diff --git a/priam/src/main/java/com/netflix/priam/defaultimpl/CassandraOperations.java b/priam/src/main/java/com/netflix/priam/defaultimpl/CassandraOperations.java index 5b2eebf22..fec12705a 100644 --- a/priam/src/main/java/com/netflix/priam/defaultimpl/CassandraOperations.java +++ b/priam/src/main/java/com/netflix/priam/defaultimpl/CassandraOperations.java @@ -31,7 +31,7 @@ */ public class CassandraOperations { private static final Logger logger = LoggerFactory.getLogger(CassandraOperations.class); - private IConfiguration configuration; + private final IConfiguration configuration; @Inject CassandraOperations(IConfiguration configuration) @@ -128,7 +128,7 @@ public void forceKeyspaceFlush(String keyspaceName) throws Exception{ new RetryableCallable(){ public Void retriableCall() throws Exception{ try(JMXNodeTool nodeTool = JMXNodeTool.instance(configuration)) { - nodeTool.forceKeyspaceFlush(keyspaceName, new String[0]); + nodeTool.forceKeyspaceFlush(keyspaceName); return null; } } diff --git a/priam/src/main/java/com/netflix/priam/defaultimpl/CassandraProcessManager.java b/priam/src/main/java/com/netflix/priam/defaultimpl/CassandraProcessManager.java index ad04774f2..6dbaeaf52 100644 --- a/priam/src/main/java/com/netflix/priam/defaultimpl/CassandraProcessManager.java +++ b/priam/src/main/java/com/netflix/priam/defaultimpl/CassandraProcessManager.java @@ -39,8 +39,8 @@ public class CassandraProcessManager implements ICassandraProcess { private static final String SUDO_STRING = "/usr/bin/sudo"; private static final int SCRIPT_EXECUTE_WAIT_TIME_MS = 5000; protected final IConfiguration config; - private InstanceState instanceState; - private CassMonitorMetrics cassMonitorMetrics; + private final InstanceState instanceState; + private final CassMonitorMetrics cassMonitorMetrics; @Inject public CassandraProcessManager(IConfiguration config, InstanceState instanceState, CassMonitorMetrics cassMonitorMetrics) { diff --git a/priam/src/main/java/com/netflix/priam/google/GcsCredential.java b/priam/src/main/java/com/netflix/priam/google/GcsCredential.java index cfb3f2b79..7416e2804 100755 --- a/priam/src/main/java/com/netflix/priam/google/GcsCredential.java +++ b/priam/src/main/java/com/netflix/priam/google/GcsCredential.java @@ -26,7 +26,7 @@ */ public class GcsCredential implements ICredentialGeneric { - private IConfiguration config; + private final IConfiguration config; @Inject public GcsCredential(IConfiguration config) { diff --git a/priam/src/main/java/com/netflix/priam/google/GoogleEncryptedFileSystem.java b/priam/src/main/java/com/netflix/priam/google/GoogleEncryptedFileSystem.java index e6e4f39aa..f2dfdeb5d 100755 --- a/priam/src/main/java/com/netflix/priam/google/GoogleEncryptedFileSystem.java +++ b/priam/src/main/java/com/netflix/priam/google/GoogleEncryptedFileSystem.java @@ -59,12 +59,12 @@ public class GoogleEncryptedFileSystem extends AbstractFileSystem { private Storage gcsStorageHandle; private Storage.Objects objectsResoruceHandle = null; - private Provider pathProvider; + private final Provider pathProvider; private String srcBucketName; - private IConfiguration config; + private final IConfiguration config; - private ICredentialGeneric gcsCredential; - private BackupMetrics backupMetrics; + private final ICredentialGeneric gcsCredential; + private final BackupMetrics backupMetrics; @Inject public GoogleEncryptedFileSystem(Provider pathProvider, final IConfiguration config @@ -172,7 +172,7 @@ private Credential constructGcsCredential() throws Exception { @Override protected void downloadFileImpl(Path remotePath, Path localPath) throws BackupRestoreException{ String objectName = parseObjectname(getPathPrefix()); - com.google.api.services.storage.Storage.Objects.Get get = null; + com.google.api.services.storage.Storage.Objects.Get get; try { get = constructObjectResourceHandle().get(this.srcBucketName, remotePath.toString()); diff --git a/priam/src/main/java/com/netflix/priam/google/GoogleFileIterator.java b/priam/src/main/java/com/netflix/priam/google/GoogleFileIterator.java index 851d3289e..1d608c0c2 100755 --- a/priam/src/main/java/com/netflix/priam/google/GoogleFileIterator.java +++ b/priam/src/main/java/com/netflix/priam/google/GoogleFileIterator.java @@ -35,10 +35,10 @@ public class GoogleFileIterator implements Iterator { private static final Logger logger = LoggerFactory.getLogger(GoogleFileIterator.class); - private Date start; - private Date till; + private final Date start; + private final Date till; private Iterator iterator; - private Provider pathProvider; + private final Provider pathProvider; private String bucketName; private Storage.Objects objectsResoruceHandle = null; diff --git a/priam/src/main/java/com/netflix/priam/health/InstanceState.java b/priam/src/main/java/com/netflix/priam/health/InstanceState.java index 13d23aa49..24a26bf19 100644 --- a/priam/src/main/java/com/netflix/priam/health/InstanceState.java +++ b/priam/src/main/java/com/netflix/priam/health/InstanceState.java @@ -64,7 +64,7 @@ public enum NODE_STATE { private BackupMetadata backupStatus; //Restore status - private RestoreStatus restoreStatus; + private final RestoreStatus restoreStatus; @Inject InstanceState(RestoreStatus restoreStatus){ diff --git a/priam/src/main/java/com/netflix/priam/identity/DoubleRing.java b/priam/src/main/java/com/netflix/priam/identity/DoubleRing.java index 2271d44d4..2de127c5e 100644 --- a/priam/src/main/java/com/netflix/priam/identity/DoubleRing.java +++ b/priam/src/main/java/com/netflix/priam/identity/DoubleRing.java @@ -20,7 +20,6 @@ import com.google.inject.Inject; import com.netflix.priam.config.IConfiguration; import com.netflix.priam.utils.ITokenManager; -import org.apache.commons.io.IOUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -88,14 +87,9 @@ private List filteredRemote(List lst) { public void backup() throws IOException { // writing to the backup file. TMP_BACKUP_FILE = File.createTempFile("Backup-instance-data", ".dat"); - OutputStream out = new FileOutputStream(TMP_BACKUP_FILE); - ObjectOutputStream stream = new ObjectOutputStream(out); - try { + try (ObjectOutputStream stream = new ObjectOutputStream(new FileOutputStream(TMP_BACKUP_FILE))) { stream.writeObject(filteredRemote(factory.getAllIds(config.getAppName()))); logger.info("Wrote the backup of the instances to: {}", TMP_BACKUP_FILE.getAbsolutePath()); - } finally { - IOUtils.closeQuietly(stream); - IOUtils.closeQuietly(out); } } @@ -110,17 +104,12 @@ public void restore() throws IOException, ClassNotFoundException { factory.delete(data); // read from the file. - InputStream in = new FileInputStream(TMP_BACKUP_FILE); - ObjectInputStream stream = new ObjectInputStream(in); - try { + try (ObjectInputStream stream = new ObjectInputStream(new FileInputStream(TMP_BACKUP_FILE))) { @SuppressWarnings("unchecked") List allInstances = (List) stream.readObject(); for (PriamInstance data : allInstances) factory.create(data.getApp(), data.getId(), data.getInstanceId(), data.getHostName(), data.getHostIP(), data.getRac(), data.getVolumes(), data.getToken()); - logger.info("Sucecsfully restored the Instances from the backup: {}", TMP_BACKUP_FILE.getAbsolutePath()); - } finally { - IOUtils.closeQuietly(stream); - IOUtils.closeQuietly(in); + logger.info("Successfully restored the Instances from the backup: {}", TMP_BACKUP_FILE.getAbsolutePath()); } } diff --git a/priam/src/main/java/com/netflix/priam/identity/InstanceIdentity.java b/priam/src/main/java/com/netflix/priam/identity/InstanceIdentity.java index ca565c323..1ec9e67ac 100644 --- a/priam/src/main/java/com/netflix/priam/identity/InstanceIdentity.java +++ b/priam/src/main/java/com/netflix/priam/identity/InstanceIdentity.java @@ -49,7 +49,7 @@ public class InstanceIdentity { private static final Logger logger = LoggerFactory.getLogger(InstanceIdentity.class); public static final String DUMMY_INSTANCE_ID = "new_slot"; - private final ListMultimap locMap = Multimaps.newListMultimap(new HashMap>(), () -> Lists.newArrayList()); + private final ListMultimap locMap = Multimaps.newListMultimap(new HashMap>(), Lists::newArrayList); private final IPriamInstanceFactory factory; private final IMembership membership; private final IConfiguration config; @@ -82,9 +82,9 @@ public boolean test(PriamInstance input) { private boolean isReplace = false; private boolean isTokenPregenerated = false; private String replacedIp = ""; - private IDeadTokenRetriever deadTokenRetriever; - private IPreGeneratedTokenRetriever preGeneratedTokenRetriever; - private INewTokenRetriever newTokenRetriever; + private final IDeadTokenRetriever deadTokenRetriever; + private final IPreGeneratedTokenRetriever preGeneratedTokenRetriever; + private final INewTokenRetriever newTokenRetriever; @Inject //Note: do not parameterized the generic type variable to an implementation as it confuses Guice in the binding. @@ -163,7 +163,7 @@ public PriamInstance retriableCall() throws Exception { @Override public PriamInstance retriableCall() throws Exception { - PriamInstance result = null; + PriamInstance result; result = deadTokenRetriever.get(); if (result != null) { @@ -195,7 +195,7 @@ public void forEachExecution() { @Override public PriamInstance retriableCall() throws Exception { - PriamInstance result = null; + PriamInstance result; result = preGeneratedTokenRetriever.get(); if (result != null) { isTokenPregenerated = true; @@ -255,7 +255,7 @@ private void populateRacMap() { public List getSeeds() throws UnknownHostException { populateRacMap(); - List seeds = new LinkedList(); + List seeds = new LinkedList<>(); // Handle single zone deployment if (config.getRacs().size() == 1) { // Return empty list if all nodes are not up diff --git a/priam/src/main/java/com/netflix/priam/identity/token/DeadTokenRetriever.java b/priam/src/main/java/com/netflix/priam/identity/token/DeadTokenRetriever.java index 41085ada5..cac643d81 100755 --- a/priam/src/main/java/com/netflix/priam/identity/token/DeadTokenRetriever.java +++ b/priam/src/main/java/com/netflix/priam/identity/token/DeadTokenRetriever.java @@ -42,13 +42,13 @@ public class DeadTokenRetriever extends TokenRetrieverBase implements IDeadTokenRetriever { private static final Logger logger = LoggerFactory.getLogger(DeadTokenRetriever.class); - private IPriamInstanceFactory factory; - private IMembership membership; - private IConfiguration config; - private Sleeper sleeper; + private final IPriamInstanceFactory factory; + private final IMembership membership; + private final IConfiguration config; + private final Sleeper sleeper; private String replacedIp; //The IP address of the dead instance to which we will acquire its token private ListMultimap locMap; - private InstanceEnvIdentity insEnvIdentity; + private final InstanceEnvIdentity insEnvIdentity; @Inject @@ -130,7 +130,7 @@ public String getReplaceIp() { } private String findReplaceIp(List allIds, String token, String location) { - String ip = null; + String ip; for (PriamInstance ins : allIds) { logger.info("Calling getIp on hostname[{}] and token[{}]", ins.getHostName(), token); if (ins.getToken().equals(token) || !ins.getDC().equals(location)) { //avoid using dead instance and other regions' instances @@ -159,7 +159,7 @@ private String getIp(String host, String token) throws ParseException { WebResource service = client.resource(baseURI); ClientResponse clientResp; - String textEntity = null; + String textEntity; try { clientResp = service.path("Priam/REST/v1/cassadmin/gossipinfo").accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); diff --git a/priam/src/main/java/com/netflix/priam/identity/token/NewTokenRetriever.java b/priam/src/main/java/com/netflix/priam/identity/token/NewTokenRetriever.java index 189e8bd19..6fde4e53e 100755 --- a/priam/src/main/java/com/netflix/priam/identity/token/NewTokenRetriever.java +++ b/priam/src/main/java/com/netflix/priam/identity/token/NewTokenRetriever.java @@ -32,11 +32,11 @@ public class NewTokenRetriever extends TokenRetrieverBase implements INewTokenRetriever { private static final Logger logger = LoggerFactory.getLogger(NewTokenRetriever.class); - private IPriamInstanceFactory factory; - private IMembership membership; - private IConfiguration config; - private Sleeper sleeper; - private ITokenManager tokenManager; + private final IPriamInstanceFactory factory; + private final IMembership membership; + private final IConfiguration config; + private final Sleeper sleeper; + private final ITokenManager tokenManager; private ListMultimap locMap; @Inject @@ -64,7 +64,7 @@ public PriamInstance get() throws Exception { for (PriamInstance data : allInstances) max = (data.getRac().equals(config.getRac()) && (data.getId() > max)) ? data.getId() : max; int maxSlot = max - hash; - int my_slot = 0; + int my_slot; if (hash == max && locMap.get(config.getRac()).size() == 0) { int idx = config.getRacs().indexOf(config.getRac()); diff --git a/priam/src/main/java/com/netflix/priam/identity/token/PreGeneratedTokenRetriever.java b/priam/src/main/java/com/netflix/priam/identity/token/PreGeneratedTokenRetriever.java index 98f32ad78..a65c19682 100755 --- a/priam/src/main/java/com/netflix/priam/identity/token/PreGeneratedTokenRetriever.java +++ b/priam/src/main/java/com/netflix/priam/identity/token/PreGeneratedTokenRetriever.java @@ -31,10 +31,10 @@ public class PreGeneratedTokenRetriever extends TokenRetrieverBase implements IPreGeneratedTokenRetriever { private static final Logger logger = LoggerFactory.getLogger(PreGeneratedTokenRetriever.class); - private IPriamInstanceFactory factory; - private IMembership membership; - private IConfiguration config; - private Sleeper sleeper; + private final IPriamInstanceFactory factory; + private final IMembership membership; + private final IConfiguration config; + private final Sleeper sleeper; private ListMultimap locMap; @Inject diff --git a/priam/src/main/java/com/netflix/priam/identity/token/TokenRetrieverBase.java b/priam/src/main/java/com/netflix/priam/identity/token/TokenRetrieverBase.java index 1153cbbef..3a929f903 100755 --- a/priam/src/main/java/com/netflix/priam/identity/token/TokenRetrieverBase.java +++ b/priam/src/main/java/com/netflix/priam/identity/token/TokenRetrieverBase.java @@ -23,7 +23,7 @@ public class TokenRetrieverBase { public static final String DUMMY_INSTANCE_ID = "new_slot"; private static final int MAX_VALUE_IN_MILISECS = 300000; //sleep up to 5 minutes - protected Random randomizer; + protected final Random randomizer; public TokenRetrieverBase() { this.randomizer = new Random(); diff --git a/priam/src/main/java/com/netflix/priam/merics/BackupMetrics.java b/priam/src/main/java/com/netflix/priam/merics/BackupMetrics.java index 0b926b785..27a96c910 100644 --- a/priam/src/main/java/com/netflix/priam/merics/BackupMetrics.java +++ b/priam/src/main/java/com/netflix/priam/merics/BackupMetrics.java @@ -19,7 +19,6 @@ import com.google.inject.Singleton; import com.netflix.spectator.api.Counter; import com.netflix.spectator.api.DistributionSummary; -import com.netflix.spectator.api.Gauge; import com.netflix.spectator.api.Registry; /** diff --git a/priam/src/main/java/com/netflix/priam/notification/AWSSnsNotificationService.java b/priam/src/main/java/com/netflix/priam/notification/AWSSnsNotificationService.java index dec091b14..624121976 100644 --- a/priam/src/main/java/com/netflix/priam/notification/AWSSnsNotificationService.java +++ b/priam/src/main/java/com/netflix/priam/notification/AWSSnsNotificationService.java @@ -40,9 +40,9 @@ public class AWSSnsNotificationService implements INotificationService { private static final Logger logger = LoggerFactory.getLogger(AWSSnsNotificationService.class); - private IConfiguration configuration; - private AmazonSNS snsClient; - private BackupMetrics backupMetrics; + private final IConfiguration configuration; + private final AmazonSNS snsClient; + private final BackupMetrics backupMetrics; @Inject public AWSSnsNotificationService(IConfiguration config, IAMCredential iamCredential @@ -62,7 +62,7 @@ public void notify(final String msg, final Map me return; } - PublishResult publishResult = null; + PublishResult publishResult; try { publishResult = new BoundedExponentialRetryCallable() { @Override diff --git a/priam/src/main/java/com/netflix/priam/notification/BackupNotificationMgr.java b/priam/src/main/java/com/netflix/priam/notification/BackupNotificationMgr.java index 4246c384a..c2c4d9f7e 100644 --- a/priam/src/main/java/com/netflix/priam/notification/BackupNotificationMgr.java +++ b/priam/src/main/java/com/netflix/priam/notification/BackupNotificationMgr.java @@ -39,7 +39,7 @@ public class BackupNotificationMgr implements EventObserver { private static final String STARTED = "started"; private static final Logger logger = LoggerFactory.getLogger(BackupNotificationMgr.class); private final IConfiguration config; - private INotificationService notificationService; + private final INotificationService notificationService; @Inject public BackupNotificationMgr(IConfiguration config, INotificationService notificationService) { diff --git a/priam/src/main/java/com/netflix/priam/resources/BackupServlet.java b/priam/src/main/java/com/netflix/priam/resources/BackupServlet.java index 4b8e8ee38..52d9864f2 100644 --- a/priam/src/main/java/com/netflix/priam/resources/BackupServlet.java +++ b/priam/src/main/java/com/netflix/priam/resources/BackupServlet.java @@ -73,23 +73,23 @@ public class BackupServlet { private static final String SSTABLE2JSON_DIR_LOCATION = "/tmp/priam_sstables"; private static final String SSTABLE2JSON_COMMAND_FROM_CASSHOME = "/bin/sstable2json"; - private PriamServer priamServer; - private IConfiguration config; - private IBackupFileSystem backupFs; - private Restore restoreObj; - private Provider pathProvider; - private ICassandraTuner tuner; - private SnapshotBackup snapshotBackup; - private IPriamInstanceFactory factory; + private final PriamServer priamServer; + private final IConfiguration config; + private final IBackupFileSystem backupFs; + private final Restore restoreObj; + private final Provider pathProvider; + private final ICassandraTuner tuner; + private final SnapshotBackup snapshotBackup; + private final IPriamInstanceFactory factory; private final ITokenManager tokenManager; private final ICassandraProcess cassProcess; - private BackupVerification backupVerification; + private final BackupVerification backupVerification; @Inject private PriamScheduler scheduler; @Inject private MetaData metaData; - private IBackupStatusMgr completedBkups; + private final IBackupStatusMgr completedBkups; @Inject public BackupServlet(PriamServer priamServer, IConfiguration config, @Named("backup") IBackupFileSystem backupFs, Restore restoreObj, Provider pathProvider, ICassandraTuner tuner, @@ -345,7 +345,7 @@ public Response restore_verify_key( Date endTime; //Creating Dir for Json storage SystemUtils.createDirs(SSTABLE2JSON_DIR_LOCATION); - String JSON_FILE_PATH = ""; + String JSON_FILE_PATH; try { @@ -510,7 +510,7 @@ public void checkSSTablesForKey(String rowkey, String keyspace, String cf, Strin .getRuntime() .exec(cmd); - Callable callable = () -> p.waitFor(); + Callable callable = p::waitFor; ExecutorService exeService = Executors.newSingleThreadExecutor(); try { @@ -536,12 +536,12 @@ public void checkSSTablesForKey(String rowkey, String keyspace, String cf, Strin public String formulateCommandToRun(String rowkey, String keyspace, String cf, String fileExtension, String jsonFilePath) { StringBuffer sbuff = new StringBuffer(); - sbuff.append("for i in $(ls " + config.getDataFileLocation() + File.separator + keyspace + File.separator + cf + File.separator + fileExtension + "-*-Data.db); do " + config.getCassHome() + SSTABLE2JSON_COMMAND_FROM_CASSHOME + " $i -k "); + sbuff.append("for i in $(ls ").append(config.getDataFileLocation()).append(File.separator).append(keyspace).append(File.separator).append(cf).append(File.separator).append(fileExtension).append("-*-Data.db); do ").append(config.getCassHome()).append(SSTABLE2JSON_COMMAND_FROM_CASSHOME).append(" $i -k "); sbuff.append(rowkey); sbuff.append(" | grep "); sbuff.append(rowkey); sbuff.append(" >> "); - sbuff.append(SSTABLE2JSON_DIR_LOCATION + File.separator + jsonFilePath); + sbuff.append(SSTABLE2JSON_DIR_LOCATION).append(File.separator).append(jsonFilePath); sbuff.append(" ; done"); logger.info("SSTable2JSON location <" + SSTABLE2JSON_DIR_LOCATION + "{}{}>", File.separator, jsonFilePath); diff --git a/priam/src/main/java/com/netflix/priam/resources/CassandraConfig.java b/priam/src/main/java/com/netflix/priam/resources/CassandraConfig.java index 4b30101d1..292269d15 100644 --- a/priam/src/main/java/com/netflix/priam/resources/CassandraConfig.java +++ b/priam/src/main/java/com/netflix/priam/resources/CassandraConfig.java @@ -44,8 +44,8 @@ @Produces(MediaType.TEXT_PLAIN) public class CassandraConfig { private static final Logger logger = LoggerFactory.getLogger(CassandraConfig.class); - private PriamServer priamServer; - private DoubleRing doubleRing; + private final PriamServer priamServer; + private final DoubleRing doubleRing; @Inject public CassandraConfig(PriamServer server, DoubleRing doubleRing) { diff --git a/priam/src/main/java/com/netflix/priam/resources/PriamConfig.java b/priam/src/main/java/com/netflix/priam/resources/PriamConfig.java index a20c94bd7..103a54041 100644 --- a/priam/src/main/java/com/netflix/priam/resources/PriamConfig.java +++ b/priam/src/main/java/com/netflix/priam/resources/PriamConfig.java @@ -40,7 +40,7 @@ public class PriamConfig { private static final Logger logger = LoggerFactory.getLogger(PriamConfig.class); - private PriamServer priamServer; + private final PriamServer priamServer; @Inject public PriamConfig(PriamServer server) { diff --git a/priam/src/main/java/com/netflix/priam/resources/RestoreServlet.java b/priam/src/main/java/com/netflix/priam/resources/RestoreServlet.java index bcd6c1cdb..403638438 100644 --- a/priam/src/main/java/com/netflix/priam/resources/RestoreServlet.java +++ b/priam/src/main/java/com/netflix/priam/resources/RestoreServlet.java @@ -55,15 +55,15 @@ public class RestoreServlet { private static final String REST_RESTORE_PREFIX = "restoreprefix"; private static final String REST_SUCCESS = "[\"ok\"]"; - private IConfiguration config; - private Restore restoreObj; - private Provider pathProvider; - private PriamServer priamServer; - private IPriamInstanceFactory factory; - private ICassandraTuner tuner; - private ICassandraProcess cassProcess; - private ITokenManager tokenManager; - private InstanceState instanceState; + private final IConfiguration config; + private final Restore restoreObj; + private final Provider pathProvider; + private final PriamServer priamServer; + private final IPriamInstanceFactory factory; + private final ICassandraTuner tuner; + private final ICassandraProcess cassProcess; + private final ITokenManager tokenManager; + private final InstanceState instanceState; @Inject public RestoreServlet(IConfiguration config, Restore restoreObj, Provider pathProvider, PriamServer priamServer diff --git a/priam/src/main/java/com/netflix/priam/restore/AbstractRestore.java b/priam/src/main/java/com/netflix/priam/restore/AbstractRestore.java index e2fa8234a..14e1ca83d 100644 --- a/priam/src/main/java/com/netflix/priam/restore/AbstractRestore.java +++ b/priam/src/main/java/com/netflix/priam/restore/AbstractRestore.java @@ -57,14 +57,14 @@ public abstract class AbstractRestore extends Task implements IRestoreStrategy { private static BigInteger restoreToken; final IBackupFileSystem fs; final Sleeper sleeper; - private BackupRestoreUtil backupRestoreUtil; - private Provider pathProvider; - private InstanceIdentity id; - private RestoreTokenSelector tokenSelector; - private ICassandraProcess cassProcess; - private InstanceState instanceState; - private MetaData metaData; - private IPostRestoreHook postRestoreHook; + private final BackupRestoreUtil backupRestoreUtil; + private final Provider pathProvider; + private final InstanceIdentity id; + private final RestoreTokenSelector tokenSelector; + private final ICassandraProcess cassProcess; + private final InstanceState instanceState; + private final MetaData metaData; + private final IPostRestoreHook postRestoreHook; AbstractRestore(IConfiguration config, IBackupFileSystem fs, String name, Sleeper sleeper, Provider pathProvider, @@ -93,7 +93,7 @@ public void setRestoreConfiguration(String restoreIncludeCFList, String restoreE backupRestoreUtil.setFilters(restoreIncludeCFList, restoreExcludeCFList); } - private final List> download(Iterator fsIterator, BackupFileType bkupFileType, boolean waitForCompletion) throws Exception { + private List> download(Iterator fsIterator, BackupFileType bkupFileType, boolean waitForCompletion) throws Exception { List> futureList = new ArrayList<>(); while (fsIterator.hasNext()) { AbstractBackupPath temp = fsIterator.next(); @@ -126,7 +126,7 @@ private void waitForCompletion(List> futureList) throws Exception { future.get(); } - private final List> downloadCommitLogs(Iterator fsIterator, BackupFileType filter, int lastN, boolean waitForCompletion) throws Exception { + private List> downloadCommitLogs(Iterator fsIterator, BackupFileType filter, int lastN, boolean waitForCompletion) throws Exception { if (fsIterator == null) return null; @@ -144,12 +144,12 @@ private final List> downloadCommitLogs(Iterator return download(bl.iterator(), filter, waitForCompletion); } - private final void stopCassProcess() throws IOException { + private void stopCassProcess() throws IOException { cassProcess.stop(true); } private String getRestorePrefix() { - String prefix = ""; + String prefix; if (StringUtils.isNotBlank(config.getRestorePrefix())) prefix = config.getRestorePrefix(); @@ -162,7 +162,7 @@ private String getRestorePrefix() { /* * Fetches meta.json used to store snapshots metadata. */ - private final void fetchSnapshotMetaFile(String restorePrefix, List out, Date startTime, Date endTime) throws IllegalStateException { + private void fetchSnapshotMetaFile(String restorePrefix, List out, Date startTime, Date endTime) throws IllegalStateException { logger.debug("Looking for snapshot meta file within restore prefix: {}", restorePrefix); Iterator backupfiles = fs.list(restorePrefix, startTime, endTime); diff --git a/priam/src/main/java/com/netflix/priam/restore/EncryptedRestoreBase.java b/priam/src/main/java/com/netflix/priam/restore/EncryptedRestoreBase.java index 4d8c514c5..86c8f8b6e 100755 --- a/priam/src/main/java/com/netflix/priam/restore/EncryptedRestoreBase.java +++ b/priam/src/main/java/com/netflix/priam/restore/EncryptedRestoreBase.java @@ -36,7 +36,6 @@ import java.nio.file.Paths; import java.util.concurrent.Future; import java.util.concurrent.ThreadPoolExecutor; -import java.util.concurrent.atomic.AtomicInteger; /** * Provides common functionality applicable to all restore strategies @@ -44,10 +43,10 @@ public abstract class EncryptedRestoreBase extends AbstractRestore{ private static final Logger logger = LoggerFactory.getLogger(EncryptedRestoreBase.class); - private String jobName; - private ICredentialGeneric pgpCredential; - private IFileCryptography fileCryptography; - private ICompression compress; + private final String jobName; + private final ICredentialGeneric pgpCredential; + private final IFileCryptography fileCryptography; + private final ICompression compress; private final ThreadPoolExecutor executor; protected EncryptedRestoreBase(IConfiguration config, IBackupFileSystem fs, String jobName, Sleeper sleeper, diff --git a/priam/src/main/java/com/netflix/priam/restore/PostRestoreHook.java b/priam/src/main/java/com/netflix/priam/restore/PostRestoreHook.java index 0330c5cc2..a9b12b81e 100644 --- a/priam/src/main/java/com/netflix/priam/restore/PostRestoreHook.java +++ b/priam/src/main/java/com/netflix/priam/restore/PostRestoreHook.java @@ -41,10 +41,10 @@ public class PostRestoreHook implements IPostRestoreHook { private static final Logger logger = LoggerFactory.getLogger(PostRestoreHook.class); private final IConfiguration config; private final Sleeper sleeper; - private static String PostRestoreHookCommandDelimiter = " "; - private static String PriamPostRestoreHookFilePrefix = "PriamFileForPostRestoreHook"; - private static String PriamPostRestoreHookFileSuffix = ".tmp"; - private static String PriamPostRestoreHookFileOptionName = "--parentHookFilePath="; + private static final String PostRestoreHookCommandDelimiter = " "; + private static final String PriamPostRestoreHookFilePrefix = "PriamFileForPostRestoreHook"; + private static final String PriamPostRestoreHookFileSuffix = ".tmp"; + private static final String PriamPostRestoreHookFileOptionName = "--parentHookFilePath="; @Inject public PostRestoreHook(IConfiguration config, Sleeper sleeper) { diff --git a/priam/src/main/java/com/netflix/priam/restore/RestoreContext.java b/priam/src/main/java/com/netflix/priam/restore/RestoreContext.java index 2b839d76c..284a1cebf 100755 --- a/priam/src/main/java/com/netflix/priam/restore/RestoreContext.java +++ b/priam/src/main/java/com/netflix/priam/restore/RestoreContext.java @@ -113,7 +113,7 @@ public static SourceType lookup(String sourceType, boolean acceptNullOrEmpty, bo private static String getSupportedValues() { - StringBuffer supportedValues = new StringBuffer(); + StringBuilder supportedValues = new StringBuilder(); boolean first = true; for (SourceType type : SourceType.values()) { if (!first) diff --git a/priam/src/main/java/com/netflix/priam/scheduler/BlockingSubmitThreadPoolExecutor.java b/priam/src/main/java/com/netflix/priam/scheduler/BlockingSubmitThreadPoolExecutor.java index 5fa23c770..9f8a4cad4 100644 --- a/priam/src/main/java/com/netflix/priam/scheduler/BlockingSubmitThreadPoolExecutor.java +++ b/priam/src/main/java/com/netflix/priam/scheduler/BlockingSubmitThreadPoolExecutor.java @@ -30,9 +30,9 @@ public class BlockingSubmitThreadPoolExecutor extends ThreadPoolExecutor { private static final long DEFAULT_SLEEP = 100; private static final long DEFAULT_KEEP_ALIVE = 100; private static final Logger logger = LoggerFactory.getLogger(BlockingSubmitThreadPoolExecutor.class); - private BlockingQueue queue; - private long giveupTime; - private AtomicInteger active; + private final BlockingQueue queue; + private final long giveupTime; + private final AtomicInteger active; public BlockingSubmitThreadPoolExecutor(int maximumPoolSize, BlockingQueue workQueue, long timeoutAdding) { super(maximumPoolSize, maximumPoolSize, DEFAULT_KEEP_ALIVE, TimeUnit.SECONDS, workQueue); diff --git a/priam/src/main/java/com/netflix/priam/scheduler/CronTimer.java b/priam/src/main/java/com/netflix/priam/scheduler/CronTimer.java index 4872015a0..d5997d0b6 100644 --- a/priam/src/main/java/com/netflix/priam/scheduler/CronTimer.java +++ b/priam/src/main/java/com/netflix/priam/scheduler/CronTimer.java @@ -28,7 +28,7 @@ */ public class CronTimer implements TaskTimer { private static final Logger logger = LoggerFactory.getLogger(CronTimer.class); - private String cronExpression; + private final String cronExpression; private String name; public enum DayOfWeek { diff --git a/priam/src/main/java/com/netflix/priam/scheduler/SchedulerType.java b/priam/src/main/java/com/netflix/priam/scheduler/SchedulerType.java index 2627c7ec8..6f3221e9e 100644 --- a/priam/src/main/java/com/netflix/priam/scheduler/SchedulerType.java +++ b/priam/src/main/java/com/netflix/priam/scheduler/SchedulerType.java @@ -75,7 +75,7 @@ public static SchedulerType lookup(String schedulerType, boolean acceptNullOrEmp } private static String getSupportedValues() { - StringBuffer supportedValues = new StringBuffer(); + StringBuilder supportedValues = new StringBuilder(); boolean first = true; for (SchedulerType type : SchedulerType.values()) { if (!first) diff --git a/priam/src/main/java/com/netflix/priam/scheduler/SimpleTimer.java b/priam/src/main/java/com/netflix/priam/scheduler/SimpleTimer.java index fcecba9bf..eaf8a416d 100644 --- a/priam/src/main/java/com/netflix/priam/scheduler/SimpleTimer.java +++ b/priam/src/main/java/com/netflix/priam/scheduler/SimpleTimer.java @@ -29,7 +29,7 @@ * regular frequency's. Frequency of the execution timestamp since epoch. */ public class SimpleTimer implements TaskTimer { - private Trigger trigger; + private final Trigger trigger; public SimpleTimer(String name, long interval) { this.trigger = TriggerBuilder.newTrigger() diff --git a/priam/src/main/java/com/netflix/priam/scheduler/Task.java b/priam/src/main/java/com/netflix/priam/scheduler/Task.java index f9e4e0248..16c5130da 100644 --- a/priam/src/main/java/com/netflix/priam/scheduler/Task.java +++ b/priam/src/main/java/com/netflix/priam/scheduler/Task.java @@ -16,7 +16,6 @@ */ package com.netflix.priam.scheduler; -import com.google.common.base.Throwables; import com.netflix.priam.config.IConfiguration; import org.quartz.Job; import org.quartz.JobExecutionContext; @@ -60,7 +59,7 @@ protected Task(IConfiguration config, MBeanServer mBeanServer) { mBeanServer.registerMBean(this, new ObjectName(mbeanName)); initialize(); } catch (Exception e) { - throw Throwables.propagate(e); + throw new RuntimeException(e); } } diff --git a/priam/src/main/java/com/netflix/priam/services/SnapshotMetaService.java b/priam/src/main/java/com/netflix/priam/services/SnapshotMetaService.java index 0f311af96..e97570ed9 100644 --- a/priam/src/main/java/com/netflix/priam/services/SnapshotMetaService.java +++ b/priam/src/main/java/com/netflix/priam/services/SnapshotMetaService.java @@ -62,11 +62,11 @@ public class SnapshotMetaService extends AbstractBackup { private static final Logger logger = LoggerFactory.getLogger(SnapshotMetaService.class); private static final String SNAPSHOT_PREFIX = "snap_v2_"; private static final String CASSANDRA_MANIFEST_FILE = "manifest.json"; - private BackupRestoreUtil backupRestoreUtil; - private MetaFileWriterBuilder metaFileWriter; + private final BackupRestoreUtil backupRestoreUtil; + private final MetaFileWriterBuilder metaFileWriter; private MetaFileWriterBuilder.DataStep dataStep; - private MetaFileManager metaFileManager; - private CassandraOperations cassandraOperations; + private final MetaFileManager metaFileManager; + private final CassandraOperations cassandraOperations; private String snapshotName = null; @Inject diff --git a/priam/src/main/java/com/netflix/priam/tuner/GCType.java b/priam/src/main/java/com/netflix/priam/tuner/GCType.java index acc57bda4..112072e84 100644 --- a/priam/src/main/java/com/netflix/priam/tuner/GCType.java +++ b/priam/src/main/java/com/netflix/priam/tuner/GCType.java @@ -81,7 +81,7 @@ public static GCType lookup(String gcType, boolean acceptNullOrEmpty, boolean ac private static String getSupportedValues() { - StringBuffer supportedValues = new StringBuffer(); + StringBuilder supportedValues = new StringBuilder(); boolean first = true; for (GCType type : GCType.values()) { if (!first) diff --git a/priam/src/main/java/com/netflix/priam/tuner/JVMOption.java b/priam/src/main/java/com/netflix/priam/tuner/JVMOption.java index 3cf54bc1e..01273e951 100644 --- a/priam/src/main/java/com/netflix/priam/tuner/JVMOption.java +++ b/priam/src/main/java/com/netflix/priam/tuner/JVMOption.java @@ -47,7 +47,7 @@ public JVMOption(String jvmOption, String value, boolean isCommented, boolean is } public String toJVMOptionString() { - final StringBuffer sb = new StringBuffer(); + final StringBuilder sb = new StringBuilder(); if (isCommented) sb.append("#"); sb.append(jvmOption); diff --git a/priam/src/main/java/com/netflix/priam/tuner/JVMOptionsTuner.java b/priam/src/main/java/com/netflix/priam/tuner/JVMOptionsTuner.java index 2998fff80..a7ea57a7f 100644 --- a/priam/src/main/java/com/netflix/priam/tuner/JVMOptionsTuner.java +++ b/priam/src/main/java/com/netflix/priam/tuner/JVMOptionsTuner.java @@ -105,7 +105,7 @@ protected List updateJVMOptions() throws Exception { configuredOptions.add("#################"); configuredOptions.addAll(upsertSet.values().stream() - .map(jvmOption -> jvmOption.toJVMOptionString()).collect(Collectors.toList())); + .map(JVMOption::toJVMOptionString).collect(Collectors.toList())); } return configuredOptions; @@ -131,7 +131,7 @@ private String updateConfigurationValue(final String line, GCType configuredGC, //Is parameter for heap setting. if (option.isHeapJVMOption()) { - String configuredValue = null; + String configuredValue; switch (option.getJvmOption()) { //Special handling for heap new size ("Xmn") case "-Xmn": @@ -192,8 +192,8 @@ private void validate(File jvmOptionsFile) throws Exception { public static final Map parseJVMOptions(String property) { if (StringUtils.isEmpty(property)) return null; - return new HashSet(Arrays.asList(property.split(","))).stream() - .map(line -> JVMOption.parse(line)).filter(jvmOption -> jvmOption != null).collect(Collectors.toMap(jvmOption -> jvmOption.getJvmOption(), jvmOption -> jvmOption)); + return new HashSet<>(Arrays.asList(property.split(","))).stream() + .map(JVMOption::parse).filter(Objects::nonNull).collect(Collectors.toMap(JVMOption::getJvmOption, jvmOption -> jvmOption)); } } diff --git a/priam/src/main/java/com/netflix/priam/tuner/StandardTuner.java b/priam/src/main/java/com/netflix/priam/tuner/StandardTuner.java index b60166088..294abc945 100644 --- a/priam/src/main/java/com/netflix/priam/tuner/StandardTuner.java +++ b/priam/src/main/java/com/netflix/priam/tuner/StandardTuner.java @@ -21,7 +21,6 @@ import com.netflix.priam.backup.SnapshotBackup; import com.netflix.priam.restore.Restore; import org.apache.commons.collections4.CollectionUtils; -import org.apache.commons.io.IOUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.yaml.snakeyaml.DumperOptions; @@ -49,7 +48,7 @@ public void writeAllProperties(String yamlLocation, String hostname, String seed options.setDefaultFlowStyle(DumperOptions.FlowStyle.BLOCK); Yaml yaml = new Yaml(options); File yamlFile = new File(yamlLocation); - Map map = (Map) yaml.load(new FileInputStream(yamlFile)); + Map map = yaml.load(new FileInputStream(yamlFile)); map.put("cluster_name", config.getAppName()); map.put("storage_port", config.getStoragePort()); map.put("ssl_storage_port", config.getSSLStoragePort()); @@ -211,7 +210,7 @@ public void updateAutoBootstrap(String yamlFile, boolean autobootstrap) throws I options.setDefaultFlowStyle(DumperOptions.FlowStyle.BLOCK); Yaml yaml = new Yaml(options); @SuppressWarnings("rawtypes") - Map map = (Map) yaml.load(new FileInputStream(yamlFile)); + Map map = yaml.load(new FileInputStream(yamlFile)); //Dont bootstrap in restore mode map.put("auto_bootstrap", autobootstrap); if (logger.isInfoEnabled()) { @@ -236,8 +235,8 @@ public void addExtraCassParams(Map map) { String[] pairs = params.split(","); logger.info("Updating yaml: adding extra cass params"); - for (int i = 0; i < pairs.length; i++) { - String[] pair = pairs[i].split("="); + for (String pair1 : pairs) { + String[] pair = pair1.split("="); String priamKey = pair[0]; String cassKey = pair[1]; String cassVal = config.getCassYamlVal(priamKey); diff --git a/priam/src/main/java/com/netflix/priam/tuner/TuneCassandra.java b/priam/src/main/java/com/netflix/priam/tuner/TuneCassandra.java index bf586cca1..aa0b9257d 100644 --- a/priam/src/main/java/com/netflix/priam/tuner/TuneCassandra.java +++ b/priam/src/main/java/com/netflix/priam/tuner/TuneCassandra.java @@ -35,7 +35,7 @@ public class TuneCassandra extends Task { private static final String JOBNAME = "Tune-Cassandra"; private static final Logger LOGGER = LoggerFactory.getLogger(TuneCassandra.class); private final ICassandraTuner tuner; - private InstanceState instanceState; + private final InstanceState instanceState; @Inject public TuneCassandra(IConfiguration config, ICassandraTuner tuner, InstanceState instanceState) { diff --git a/priam/src/main/java/com/netflix/priam/tuner/dse/AuditLogTunerLog4J.java b/priam/src/main/java/com/netflix/priam/tuner/dse/AuditLogTunerLog4J.java index 66cbc9d4b..60f248de0 100644 --- a/priam/src/main/java/com/netflix/priam/tuner/dse/AuditLogTunerLog4J.java +++ b/priam/src/main/java/com/netflix/priam/tuner/dse/AuditLogTunerLog4J.java @@ -37,8 +37,8 @@ */ public class AuditLogTunerLog4J implements IAuditLogTuner { - private IConfiguration config; - private IDseConfiguration dseConfig; + private final IConfiguration config; + private final IDseConfiguration dseConfig; protected static final String AUDIT_LOG_ADDITIVE_ENTRY = "log4j.additivity.DataAudit"; protected static final String AUDIT_LOG_FILE = "/conf/log4j-server.properties"; protected static final String PRIMARY_AUDIT_LOG_ENTRY = "log4j.logger.DataAudit"; @@ -112,7 +112,7 @@ public void tuneAuditLog() { } - private final String findAuditLoggerName(List lines) throws IllegalStateException { + private String findAuditLoggerName(List lines) throws IllegalStateException { for (final String l : lines) { if (l.contains(PRIMARY_AUDIT_LOG_ENTRY)) { final String[] valTokens = l.split(","); diff --git a/priam/src/main/java/com/netflix/priam/tuner/dse/AuditLogTunerYaml.java b/priam/src/main/java/com/netflix/priam/tuner/dse/AuditLogTunerYaml.java index 45fccea95..9e775e0b8 100644 --- a/priam/src/main/java/com/netflix/priam/tuner/dse/AuditLogTunerYaml.java +++ b/priam/src/main/java/com/netflix/priam/tuner/dse/AuditLogTunerYaml.java @@ -35,7 +35,7 @@ */ public class AuditLogTunerYaml implements IAuditLogTuner { - private IDseConfiguration dseConfig; + private final IDseConfiguration dseConfig; private static final String AUDIT_LOG_DSE_ENTRY = "audit_logging_options"; private static final Logger logger = LoggerFactory.getLogger(AuditLogTunerYaml.class); @@ -50,7 +50,7 @@ public void tuneAuditLog() { Yaml yaml = new Yaml(options); String dseYaml = dseConfig.getDseYamlLocation(); try { - Map map = (Map) yaml.load(new FileInputStream(dseYaml)); + Map map = yaml.load(new FileInputStream(dseYaml)); if (map.containsKey(AUDIT_LOG_DSE_ENTRY)) { Boolean isEnabled = (Boolean) ((Map) map.get(AUDIT_LOG_DSE_ENTRY)).get("enabled"); diff --git a/priam/src/main/java/com/netflix/priam/utils/BoundedExponentialRetryCallable.java b/priam/src/main/java/com/netflix/priam/utils/BoundedExponentialRetryCallable.java index b6619c4d6..ec38df3a4 100644 --- a/priam/src/main/java/com/netflix/priam/utils/BoundedExponentialRetryCallable.java +++ b/priam/src/main/java/com/netflix/priam/utils/BoundedExponentialRetryCallable.java @@ -28,9 +28,9 @@ public abstract class BoundedExponentialRetryCallable extends RetryableCallab protected final static int MAX_RETRIES = 10; private static final Logger logger = LoggerFactory.getLogger(BoundedExponentialRetryCallable.class); - private long max; - private long min; - private int maxRetries; + private final long max; + private final long min; + private final int maxRetries; private final ThreadSleeper sleeper = new ThreadSleeper(); public BoundedExponentialRetryCallable() { diff --git a/priam/src/main/java/com/netflix/priam/utils/CassandraMonitor.java b/priam/src/main/java/com/netflix/priam/utils/CassandraMonitor.java index 9c13f3bd0..ab1ddecba 100644 --- a/priam/src/main/java/com/netflix/priam/utils/CassandraMonitor.java +++ b/priam/src/main/java/com/netflix/priam/utils/CassandraMonitor.java @@ -45,9 +45,9 @@ public class CassandraMonitor extends Task { public static final String JOBNAME = "CASS_MONITOR_THREAD"; private static final Logger logger = LoggerFactory.getLogger(CassandraMonitor.class); private static final AtomicBoolean isCassandraStarted = new AtomicBoolean(false); - private InstanceState instanceState; - private ICassandraProcess cassProcess; - private CassMonitorMetrics cassMonitorMetrics; + private final InstanceState instanceState; + private final ICassandraProcess cassProcess; + private final CassMonitorMetrics cassMonitorMetrics; @Inject protected CassandraMonitor(IConfiguration config, InstanceState instanceState, ICassandraProcess cassProcess, CassMonitorMetrics cassMonitorMetrics) { @@ -140,7 +140,7 @@ private void checkDirectory(File directory) { throw new IllegalStateException(String.format("Directory: {} does not exist", directory)); if (!directory.canRead() || !directory.canWrite()) - throw new IllegalStateException(String.format("Directory: {} does not have read/write permissions.")); + throw new IllegalStateException(String.format("Directory: {} does not have read/write permissions.", directory)); } public static TaskTimer getTimer() { diff --git a/priam/src/main/java/com/netflix/priam/utils/ExponentialRetryCallable.java b/priam/src/main/java/com/netflix/priam/utils/ExponentialRetryCallable.java index a551781d4..02f4e8691 100644 --- a/priam/src/main/java/com/netflix/priam/utils/ExponentialRetryCallable.java +++ b/priam/src/main/java/com/netflix/priam/utils/ExponentialRetryCallable.java @@ -26,8 +26,8 @@ public abstract class ExponentialRetryCallable extends RetryableCallable { public final static long MIN_SLEEP = 200; private static final Logger logger = LoggerFactory.getLogger(ExponentialRetryCallable.class); - private long max; - private long min; + private final long max; + private final long min; public ExponentialRetryCallable() { this.max = MAX_SLEEP; diff --git a/priam/src/main/java/com/netflix/priam/utils/FifoQueue.java b/priam/src/main/java/com/netflix/priam/utils/FifoQueue.java index aa31d26f8..bb1417375 100644 --- a/priam/src/main/java/com/netflix/priam/utils/FifoQueue.java +++ b/priam/src/main/java/com/netflix/priam/utils/FifoQueue.java @@ -21,7 +21,7 @@ public class FifoQueue> extends TreeSet { private static final long serialVersionUID = -7388604551920505669L; - private int capacity; + private final int capacity; public FifoQueue(int capacity) { super(Comparator.naturalOrder()); diff --git a/priam/src/main/java/com/netflix/priam/utils/JMXNodeTool.java b/priam/src/main/java/com/netflix/priam/utils/JMXNodeTool.java index b4f745e2e..54d723d30 100644 --- a/priam/src/main/java/com/netflix/priam/utils/JMXNodeTool.java +++ b/priam/src/main/java/com/netflix/priam/utils/JMXNodeTool.java @@ -53,7 +53,7 @@ public class JMXNodeTool extends NodeProbe implements INodeToolObservable { private static volatile JMXNodeTool tool = null; private MBeanServerConnection mbeanServerConn = null; - private static Set observers = new HashSet<>(); + private static final Set observers = new HashSet<>(); /** * Hostname and Port to talk to will be same server for now optionally we @@ -183,11 +183,11 @@ public JMXNodeTool retriableCall() throws Exception { } Field fields[] = NodeProbe.class.getDeclaredFields(); - for (int i = 0; i < fields.length; i++) { - if (!fields[i].getName().equals("mbeanServerConn")) + for (Field field : fields) { + if (!field.getName().equals("mbeanServerConn")) continue; - fields[i].setAccessible(true); - nodetool.mbeanServerConn = (MBeanServerConnection) fields[i].get(nodetool); + field.setAccessible(true); + nodetool.mbeanServerConn = (MBeanServerConnection) field.get(nodetool); } return nodetool; diff --git a/priam/src/main/java/com/netflix/priam/utils/SystemUtils.java b/priam/src/main/java/com/netflix/priam/utils/SystemUtils.java index 3c9c48fbe..a1a5a81a6 100644 --- a/priam/src/main/java/com/netflix/priam/utils/SystemUtils.java +++ b/priam/src/main/java/com/netflix/priam/utils/SystemUtils.java @@ -48,7 +48,7 @@ public static String getDataFromUrl(String url) { byte[] b = new byte[2048]; ByteArrayOutputStream bos = new ByteArrayOutputStream(); DataInputStream d = new DataInputStream((FilterInputStream) conn.getContent()); - int c = 0; + int c; while ((c = d.read(b, 0, b.length)) != -1) bos.write(b, 0, c); String return_ = new String(bos.toByteArray(), Charsets.UTF_8); @@ -112,9 +112,9 @@ public static String md5(File file) { } public static String toHex(byte[] digest) { - StringBuffer sb = new StringBuffer(digest.length * 2); - for (int i = 0; i < digest.length; i++) { - String hex = Integer.toHexString(digest[i]); + StringBuilder sb = new StringBuilder(digest.length * 2); + for (byte aDigest : digest) { + String hex = Integer.toHexString(aDigest); if (hex.length() == 1) { sb.append("0"); } else if (hex.length() == 8) { @@ -165,7 +165,7 @@ public static BufferedReader readFile(String absPathToFile) throws IOException { public static void writeToFile(String filename, String line) { File f = new File(filename); PrintWriter pw = null; - FileWriter fw = null; + FileWriter fw; try { if (!f.exists()) { f.createNewFile(); diff --git a/priam/src/main/java/com/netflix/priam/utils/TokenManager.java b/priam/src/main/java/com/netflix/priam/utils/TokenManager.java index a3f5debbb..e7f1bdb9f 100644 --- a/priam/src/main/java/com/netflix/priam/utils/TokenManager.java +++ b/priam/src/main/java/com/netflix/priam/utils/TokenManager.java @@ -23,6 +23,7 @@ import com.netflix.priam.config.IConfiguration; import java.math.BigInteger; +import java.util.Collections; import java.util.List; public class TokenManager implements ITokenManager { @@ -36,7 +37,7 @@ public class TokenManager implements ITokenManager { private final BigInteger maximumToken; private final BigInteger tokenRangeSize; - private IConfiguration config; + private final IConfiguration config; @Inject public TokenManager(IConfiguration config) { @@ -103,7 +104,7 @@ public String createToken(int my_slot, int totalCount, String region) { public BigInteger findClosestToken(BigInteger tokenToSearch, List tokenList) { Preconditions.checkArgument(!tokenList.isEmpty(), "token list must not be empty"); List sortedTokens = Ordering.natural().sortedCopy(tokenList); - int index = Ordering.natural().binarySearch(sortedTokens, tokenToSearch); + int index = Collections.binarySearch(sortedTokens, tokenToSearch, Ordering.natural()); if (index < 0) { int i = Math.abs(index) - 1; if ((i >= sortedTokens.size()) || (i > 0 && sortedTokens.get(i).subtract(tokenToSearch) diff --git a/priam/src/main/resources/Priam.properties b/priam/src/main/resources/Priam.properties index a6c72ddca..8c1ccd42e 100644 --- a/priam/src/main/resources/Priam.properties +++ b/priam/src/main/resources/Priam.properties @@ -1,55 +1,3 @@ -priam.authenticator=org.apache.cassandra.auth.AllowAllAuthenticator -priam.authorizer=org.apache.cassandra.auth.AllowAllAuthorizer -priam.backup.chunksizemb=10 -priam.backup.commitlog.enable= -priam.backup.commitlog.location= -priam.backup.hour=12 -priam.backup.incremental.enable= -priam.backup.racs= -priam.backup.retention= -priam.backup.threads=2 -priam.bootcluster= -priam.cache.location=/var/lib/cassandra/saved_caches -priam.cass.home=/mnt/cassandra -priam.cass.manual.start.enable= -priam.cass.process= -priam.cass.startscript=/mnt/cassandra/bin/cassandra -priam.cass.stopscript=/mnt/cassandra/bin/cassandra -priam.clustername=cass_cluster -priam.commitlog.location=/var/lib/cassandra/commitlog -priam.compaction.throughput= -priam.data.location=/var/lib/cassandra/data priam.direct.memory.size.m1.large=1G -priam.endpoint_snitch=org.apache.cassandra.locator.Ec2Snitch priam.heap.newgen.size.m1.large=2G priam.heap.size.m1.large=4G -priam.hint.delay= -priam.hint.window= -priam.jmx.port=7199 -priam.keyCache.count= -priam.keyCache.size= -priam.localbootstrap.enable= -priam.memory.compaction.limit= -priam.memtabletotalspace= -priam.multiregion.enable=false -priam.multithreaded.compaction= -priam.partitioner=org.apache.cassandra.dht.RandomPartitioner -priam.restore.closesttoken= -priam.restore.keyspaces= -priam.restore.prefix= -priam.restore.snapshot= -priam.restore.threads=8 -priam.rowCache.count= -priam.rowCache.size= -priam.s3.base_dir=mdo-backup -priam.s3.bucket=mdo-cassandra-archive -priam.seed.provider=com.netflix.priam.cassandra.extensions.NFSeedProvider -priam.ssl.storage.port=7001 -priam.storage.port=7000 -priam.streaming.throughput.mb= -priam.target.columnfamily= -priam.target.keyspace= -priam.thrift.port=9160 -priam.upload.throttle= -priam.yamlLocation= -priam.zones.available= From b5a4d8638c84c619256ef46a40b100515884d2ca Mon Sep 17 00:00:00 2001 From: Arun Agrawal Date: Fri, 12 Oct 2018 16:36:15 -0700 Subject: [PATCH 020/228] Move to google java style. (#737) * Move to google java style. --- build.gradle | 9 + .../cassandra/extensions/DataFetcher.java | 34 +- .../cassandra/extensions/NFSeedProvider.java | 28 +- .../extensions/PriamStartupAgent.java | 108 ++- .../java/com/netflix/priam/PriamServer.java | 97 ++- .../com/netflix/priam/aws/AWSMembership.java | 193 ++++-- .../java/com/netflix/priam/aws/DataPart.java | 5 +- .../com/netflix/priam/aws/S3BackupPath.java | 21 +- .../priam/aws/S3CrossAccountFileSystem.java | 58 +- .../priam/aws/S3EncryptedFileSystem.java | 152 +++-- .../com/netflix/priam/aws/S3FileIterator.java | 28 +- .../com/netflix/priam/aws/S3FileSystem.java | 168 +++-- .../netflix/priam/aws/S3FileSystemBase.java | 117 ++-- .../com/netflix/priam/aws/S3PartUploader.java | 34 +- .../netflix/priam/aws/S3PrefixIterator.java | 42 +- .../netflix/priam/aws/SDBInstanceData.java | 90 +-- .../netflix/priam/aws/SDBInstanceFactory.java | 50 +- .../priam/aws/UpdateCleanupPolicy.java | 8 +- .../priam/aws/UpdateSecuritySettings.java | 49 +- .../aws/auth/EC2RoleAssumptionCredential.java | 60 +- .../netflix/priam/aws/auth/IS3Credential.java | 20 +- .../priam/aws/auth/S3InstanceCredential.java | 24 +- .../aws/auth/S3RoleAssumptionCredential.java | 48 +- .../netflix/priam/backup/AbstractBackup.java | 102 +-- .../priam/backup/AbstractBackupPath.java | 80 +-- .../priam/backup/AbstractFileSystem.java | 160 +++-- .../priam/backup/BackupFileSystemContext.java | 26 +- .../netflix/priam/backup/BackupMetadata.java | 48 +- .../priam/backup/BackupRestoreException.java | 1 - .../priam/backup/BackupRestoreUtil.java | 67 +- .../netflix/priam/backup/BackupStatusMgr.java | 98 +-- .../priam/backup/BackupVerification.java | 108 +-- .../backup/BackupVerificationResult.java | 27 +- .../netflix/priam/backup/CommitLogBackup.java | 53 +- .../priam/backup/CommitLogBackupTask.java | 54 +- .../priam/backup/FileSnapshotStatusMgr.java | 65 +- .../priam/backup/IBackupFileSystem.java | 126 ++-- .../priam/backup/IBackupStatusMgr.java | 40 +- .../priam/backup/IFileSystemContext.java | 20 +- .../priam/backup/IIncrementalBackup.java | 23 +- .../priam/backup/IMessageObserver.java | 37 +- .../priam/backup/IncrementalBackup.java | 46 +- .../priam/backup/IncrementalMetaData.java | 35 +- .../com/netflix/priam/backup/MetaData.java | 68 +- .../priam/backup/RangeReadInputStream.java | 69 +- .../netflix/priam/backup/SnapshotBackup.java | 180 +++-- .../java/com/netflix/priam/backup/Status.java | 15 +- .../priam/backupv2/ColumnfamilyResult.java | 32 +- .../priam/backupv2/FileUploadResult.java | 39 +- .../priam/backupv2/LocalDBReaderWriter.java | 163 +++-- .../netflix/priam/backupv2/MetaFileInfo.java | 12 +- .../priam/backupv2/MetaFileManager.java | 50 +- .../priam/backupv2/MetaFileReader.java | 42 +- .../priam/backupv2/MetaFileWriterBuilder.java | 77 ++- .../priam/backupv2/PrefixGenerator.java | 50 +- .../com/netflix/priam/cli/Application.java | 7 +- .../java/com/netflix/priam/cli/Backuper.java | 2 +- .../priam/cli/IncrementalBackuper.java | 5 +- .../netflix/priam/cli/LightGuiceModule.java | 4 +- .../java/com/netflix/priam/cli/Restorer.java | 8 +- .../netflix/priam/cli/StaticMembership.java | 21 +- .../priam/cluster/management/Compaction.java | 125 ++-- .../priam/cluster/management/Flush.java | 90 +-- .../management/IClusterManagement.java | 44 +- .../cluster/management/SchemaConstant.java | 17 +- .../netflix/priam/compress/ChunkedStream.java | 17 +- .../netflix/priam/compress/ICompression.java | 13 +- .../priam/compress/SnappyCompression.java | 14 +- .../priam/config/BackupRestoreConfig.java | 30 +- .../priam/config/IBackupRestoreConfig.java | 33 +- .../netflix/priam/config/IConfiguration.java | 630 ++++++++---------- .../priam/config/PriamConfiguration.java | 272 +++++--- .../config/PriamConfigurationPersister.java | 53 +- .../configSource/AbstractConfigSource.java | 12 +- .../configSource/CompositeConfigSource.java | 27 +- .../priam/configSource/IConfigSource.java | 34 +- .../configSource/MemoryConfigSource.java | 1 - .../priam/configSource/PriamConfigSource.java | 23 +- .../configSource/PropertiesConfigSource.java | 19 +- .../configSource/SimpleDBConfigSource.java | 68 +- .../SystemPropertiesConfigSource.java | 9 +- .../netflix/priam/cred/ClearCredential.java | 18 +- .../com/netflix/priam/cred/ICredential.java | 9 +- .../priam/cred/ICredentialGeneric.java | 8 +- .../priam/cryptography/IFileCryptography.java | 27 +- .../priam/cryptography/pgp/PgpCredential.java | 25 +- .../cryptography/pgp/PgpCryptography.java | 285 ++++---- .../priam/cryptography/pgp/PgpUtil.java | 52 +- .../defaultimpl/CassandraOperations.java | 126 ++-- .../defaultimpl/CassandraProcessManager.java | 111 +-- .../priam/defaultimpl/ICassandraProcess.java | 2 - .../defaultimpl/InjectedWebListener.java | 25 +- .../priam/defaultimpl/PriamGuiceModule.java | 33 +- .../netflix/priam/google/GcsCredential.java | 23 +- .../google/GoogleEncryptedFileSystem.java | 145 ++-- .../priam/google/GoogleFileIterator.java | 95 ++- .../netflix/priam/health/InstanceState.java | 74 +- .../identity/AwsInstanceEnvIdentity.java | 24 +- .../netflix/priam/identity/DoubleRing.java | 77 ++- .../netflix/priam/identity/IMembership.java | 11 +- .../priam/identity/IPriamInstanceFactory.java | 24 +- .../priam/identity/InstanceEnvIdentity.java | 25 +- .../priam/identity/InstanceIdentity.java | 332 ++++----- .../netflix/priam/identity/PriamInstance.java | 16 +- .../config/AWSVpcInstanceDataRetriever.java | 41 +- .../AwsClassicInstanceDataRetriever.java | 31 +- .../config/InstanceDataRetriever.java | 45 +- .../config/InstanceDataRetrieverBase.java | 45 +- .../config/LocalInstanceDataRetriever.java | 26 +- .../identity/token/DeadTokenRetriever.java | 122 ++-- .../identity/token/IDeadTokenRetriever.java | 20 +- .../identity/token/INewTokenRetriever.java | 20 +- .../token/IPreGeneratedTokenRetriever.java | 20 +- .../identity/token/NewTokenRetriever.java | 96 ++- .../token/PreGeneratedTokenRetriever.java | 69 +- .../identity/token/TokenRetrieverBase.java | 27 +- .../netflix/priam/merics/BackupMetrics.java | 42 +- .../priam/merics/CassMonitorMetrics.java | 25 +- .../priam/merics/CompactionMeasurement.java | 7 +- .../netflix/priam/merics/IMeasurement.java | 23 +- .../com/netflix/priam/merics/Metrics.java | 4 +- .../merics/NodeToolFlushMeasurement.java | 25 +- .../AWSSnsNotificationService.java | 118 ++-- .../priam/notification/BackupEvent.java | 7 +- .../notification/BackupNotificationMgr.java | 50 +- .../priam/notification/EventGenerator.java | 14 +- .../priam/notification/EventObserver.java | 4 +- .../notification/INotificationService.java | 28 +- .../priam/resources/BackupServlet.java | 293 ++++---- .../priam/resources/CassandraAdmin.java | 181 ++--- .../priam/resources/CassandraConfig.java | 27 +- .../netflix/priam/resources/PriamConfig.java | 24 +- .../resources/PriamInstanceResource.java | 47 +- .../priam/resources/RestoreServlet.java | 112 ++-- .../priam/resources/SecurityGroupAdmin.java | 48 +- .../priam/restore/AbstractRestore.java | 163 +++-- ...ossAccountCryptographyRestoreStrategy.java | 75 ++- .../priam/restore/EncryptedRestoreBase.java | 235 ++++--- .../restore/EncryptedRestoreStrategy.java | 66 +- .../GoogleCryptographyRestoreStrategy.java | 76 ++- .../priam/restore/IPostRestoreHook.java | 5 +- .../priam/restore/IRestoreStrategy.java | 22 +- .../priam/restore/PostRestoreHook.java | 114 ++-- .../restore/PostRestoreHookException.java | 7 +- .../com/netflix/priam/restore/Restore.java | 48 +- .../netflix/priam/restore/RestoreContext.java | 85 ++- .../priam/restore/RestoreTokenSelector.java | 21 +- .../BlockingSubmitThreadPoolExecutor.java | 28 +- .../netflix/priam/scheduler/CronTimer.java | 56 +- .../scheduler/NamedThreadPoolExecutor.java | 13 +- .../priam/scheduler/PriamScheduler.java | 73 +- .../priam/scheduler/SchedulerType.java | 33 +- .../netflix/priam/scheduler/SimpleTimer.java | 63 +- .../com/netflix/priam/scheduler/Task.java | 39 +- .../netflix/priam/scheduler/TaskMBean.java | 5 +- .../netflix/priam/scheduler/TaskTimer.java | 7 +- .../scheduler/UnsupportedTypeException.java | 25 +- .../priam/services/SnapshotMetaService.java | 220 +++--- .../java/com/netflix/priam/tuner/GCTuner.java | 121 ++-- .../java/com/netflix/priam/tuner/GCType.java | 52 +- .../netflix/priam/tuner/ICassandraTuner.java | 27 +- .../com/netflix/priam/tuner/JVMOption.java | 44 +- .../netflix/priam/tuner/JVMOptionsTuner.java | 106 +-- .../netflix/priam/tuner/StandardTuner.java | 104 +-- .../netflix/priam/tuner/TuneCassandra.java | 13 +- .../priam/tuner/dse/AuditLogTunerLog4J.java | 54 +- .../priam/tuner/dse/AuditLogTunerYaml.java | 28 +- .../priam/tuner/dse/DseProcessManager.java | 40 +- .../com/netflix/priam/tuner/dse/DseTuner.java | 54 +- .../priam/tuner/dse/IAuditLogTuner.java | 5 +- .../priam/tuner/dse/IDseConfiguration.java | 37 +- .../BoundedExponentialRetryCallable.java | 20 +- .../netflix/priam/utils/CassandraMonitor.java | 60 +- .../com/netflix/priam/utils/DateUtil.java | 90 ++- .../priam/utils/ExponentialRetryCallable.java | 10 +- .../com/netflix/priam/utils/FifoQueue.java | 3 +- .../priam/utils/GsonJsonSerializer.java | 58 +- .../priam/utils/INodeToolObservable.java | 20 +- .../priam/utils/INodeToolObserver.java | 20 +- .../netflix/priam/utils/ITokenManager.java | 1 - .../priam/utils/JMXConnectionException.java | 21 +- .../netflix/priam/utils/JMXConnectorMgr.java | 32 +- .../com/netflix/priam/utils/JMXNodeTool.java | 256 +++---- .../netflix/priam/utils/MaxSizeHashMap.java | 9 +- .../priam/utils/RetryableCallable.java | 7 +- .../java/com/netflix/priam/utils/Sleeper.java | 4 +- .../com/netflix/priam/utils/SystemUtils.java | 28 +- .../netflix/priam/utils/ThreadSleeper.java | 11 +- .../com/netflix/priam/utils/TokenManager.java | 37 +- .../java/com/netflix/priam/TestModule.java | 20 +- .../netflix/priam/backup/BRTestModule.java | 32 +- .../priam/backup/FakeBackupFileSystem.java | 65 +- .../netflix/priam/backup/FakeCredentials.java | 11 +- .../priam/backup/FakeNullCredential.java | 11 +- .../priam/backup/FakePostRestoreHook.java | 2 +- .../priam/backup/NullBackupFileSystem.java | 24 +- .../priam/backup/TestAbstractFileSystem.java | 164 +++-- .../com/netflix/priam/backup/TestBackup.java | 188 +++--- .../netflix/priam/backup/TestBackupFile.java | 73 +- .../netflix/priam/backup/TestCompression.java | 101 +-- .../priam/backup/TestCustomizedTPE.java | 71 +- .../priam/backup/TestFileIterator.java | 383 +++++++---- .../com/netflix/priam/backup/TestRestore.java | 97 +-- .../priam/backup/TestS3FileSystem.java | 100 +-- .../priam/backup/TestSnapshotStatusMgr.java | 50 +- .../priam/backup/identity/DoubleRingTest.java | 32 +- .../identity/FakeInstanceEnvIdentity.java | 27 +- .../backup/identity/InstanceIdentityTest.java | 44 +- .../backup/identity/InstanceTestUtils.java | 48 +- .../token/FakeDeadTokenRetriever.java | 29 +- .../identity/token/FakeNewTokenRetriever.java | 19 +- .../token/FakePreGeneratedTokenRetriever.java | 22 +- .../backupv2/TestLocalDBReaderWriter.java | 219 +++--- .../cassandra/token/TestDoublingLogic.java | 87 +-- .../priam/config/FakeBackupRestoreConfig.java | 28 +- .../priam/config/FakeConfiguration.java | 324 ++++----- .../PriamConfigurationPersisterTest.java | 40 +- .../AbstractConfigSourceTest.java | 18 +- .../CompositeConfigSourceTest.java | 12 +- .../PropertiesConfigSourceTest.java | 22 +- .../SystemPropertiesConfigSourceTest.java | 9 +- .../CassandraProcessManagerTest.java | 40 +- .../defaultimpl/FakeCassandraProcess.java | 10 +- .../priam/health/TestInstanceStatus.java | 102 ++- .../priam/identity/FakeMembership.java | 48 +- .../identity/FakePriamInstanceFactory.java | 63 +- .../priam/resources/BackupServletTest.java | 433 +++++++----- .../priam/resources/CassandraConfigTest.java | 204 +++--- .../priam/resources/PriamConfigTest.java | 30 +- .../resources/PriamInstanceResourceTest.java | 140 ++-- .../priam/restore/TestPostRestoreHook.java | 85 ++- .../priam/scheduler/TestGuiceSingleton.java | 30 +- .../priam/scheduler/TestScheduler.java | 64 +- .../services/TestSnapshotMetaService.java | 52 +- .../netflix/priam/stream/StreamingTest.java | 110 +-- .../priam/tuner/JVMOptionTunerTest.java | 173 +++-- .../priam/tuner/StandardTunerTest.java | 32 +- .../priam/tuner/dse/DseConfigStub.java | 31 +- .../netflix/priam/tuner/dse/DseTunerTest.java | 55 +- .../netflix/priam/utils/BackupFileUtils.java | 43 +- .../com/netflix/priam/utils/FakeSleeper.java | 17 +- .../priam/utils/Murmur3TokenManagerTest.java | 98 ++- .../priam/utils/RandomTokenManagerTest.java | 157 +++-- .../priam/utils/TestCassandraMonitor.java | 86 ++- .../priam/utils/TestGsonJsonSerializer.java | 21 +- 245 files changed, 8468 insertions(+), 7121 deletions(-) diff --git a/build.gradle b/build.gradle index c430a4620..e94f09413 100644 --- a/build.gradle +++ b/build.gradle @@ -1,5 +1,14 @@ plugins { id 'nebula.netflixoss' version '5.2.0' + id 'com.github.sherter.google-java-format' version '0.7.1' +} + +repositories { + jcenter() +} + +googleJavaFormat { + options style: 'AOSP' } ext.githubProjectName = 'Priam' diff --git a/priam-cass-extensions/src/main/java/com/netflix/priam/cassandra/extensions/DataFetcher.java b/priam-cass-extensions/src/main/java/com/netflix/priam/cassandra/extensions/DataFetcher.java index 81e6136ba..751d47b07 100644 --- a/priam-cass-extensions/src/main/java/com/netflix/priam/cassandra/extensions/DataFetcher.java +++ b/priam-cass-extensions/src/main/java/com/netflix/priam/cassandra/extensions/DataFetcher.java @@ -16,25 +16,21 @@ */ package com.netflix.priam.cassandra.extensions; +import com.google.common.base.Charsets; import java.io.ByteArrayOutputStream; import java.io.DataInputStream; import java.io.FilterInputStream; import java.net.HttpURLConnection; import java.net.URL; - -import com.google.common.base.Charsets; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -public class DataFetcher -{ +public class DataFetcher { private static final Logger logger = LoggerFactory.getLogger(DataFetcher.class); - public static String fetchData(String url) - { + public static String fetchData(String url) { DataInputStream responseStream = null; - try - { + try { HttpURLConnection conn = (HttpURLConnection) new URL(url).openConnection(); conn.setConnectTimeout(1000); conn.setReadTimeout(10000); @@ -46,29 +42,19 @@ public static String fetchData(String url) ByteArrayOutputStream bos = new ByteArrayOutputStream(); responseStream = new DataInputStream((FilterInputStream) conn.getContent()); int c = 0; - while ((c = responseStream.read(b, 0, b.length)) != -1) - bos.write(b, 0, c); + while ((c = responseStream.read(b, 0, b.length)) != -1) bos.write(b, 0, c); String return_ = new String(bos.toByteArray(), Charsets.UTF_8); logger.info("Calling URL API: {} returns: {}", url, return_); conn.disconnect(); return return_; - } - catch (Exception ex) - { + } catch (Exception ex) { throw new RuntimeException(ex); - } - finally - { - try - { - if(responseStream != null) - responseStream.close(); - } - catch (Exception e) - { + } finally { + try { + if (responseStream != null) responseStream.close(); + } catch (Exception e) { logger.warn("Failed to close response stream from priam", e); } } } - } diff --git a/priam-cass-extensions/src/main/java/com/netflix/priam/cassandra/extensions/NFSeedProvider.java b/priam-cass-extensions/src/main/java/com/netflix/priam/cassandra/extensions/NFSeedProvider.java index 784b5d2c7..3a5102541 100644 --- a/priam-cass-extensions/src/main/java/com/netflix/priam/cassandra/extensions/NFSeedProvider.java +++ b/priam-cass-extensions/src/main/java/com/netflix/priam/cassandra/extensions/NFSeedProvider.java @@ -20,33 +20,25 @@ import java.util.ArrayList; import java.util.List; import java.util.Map; - import org.apache.cassandra.locator.SeedProvider; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -/** - * Retrieves the list of seeds from Priam. - */ -public class NFSeedProvider implements SeedProvider -{ +/** Retrieves the list of seeds from Priam. */ +public class NFSeedProvider implements SeedProvider { private static final Logger logger = LoggerFactory.getLogger(NFSeedProvider.class); - public NFSeedProvider(Map args) - { } + public NFSeedProvider(Map args) {} @Override - public List getSeeds() - { + public List getSeeds() { List seeds = new ArrayList(); - try - { - String priamSeeds = DataFetcher.fetchData("http://127.0.0.1:8080/Priam/REST/v1/cassconfig/get_seeds"); - for (String seed : priamSeeds.split(",")) - seeds.add(InetAddress.getByName(seed)); - } - catch (Exception e) - { + try { + String priamSeeds = + DataFetcher.fetchData( + "http://127.0.0.1:8080/Priam/REST/v1/cassconfig/get_seeds"); + for (String seed : priamSeeds.split(",")) seeds.add(InetAddress.getByName(seed)); + } catch (Exception e) { logger.error("Failed to load seed data", e); } return seeds; diff --git a/priam-cass-extensions/src/main/java/com/netflix/priam/cassandra/extensions/PriamStartupAgent.java b/priam-cass-extensions/src/main/java/com/netflix/priam/cassandra/extensions/PriamStartupAgent.java index ad644cb14..50e340a10 100644 --- a/priam-cass-extensions/src/main/java/com/netflix/priam/cassandra/extensions/PriamStartupAgent.java +++ b/priam-cass-extensions/src/main/java/com/netflix/priam/cassandra/extensions/PriamStartupAgent.java @@ -16,91 +16,89 @@ */ package com.netflix.priam.cassandra.extensions; +import java.lang.instrument.Instrumentation; +import java.util.Iterator; import org.apache.cassandra.utils.FBUtilities; import org.apache.commons.lang3.StringUtils; import org.json.simple.JSONObject; import org.json.simple.parser.JSONParser; -import java.lang.instrument.Instrumentation; -import java.util.Iterator; - - /** - * A PreMain class - * to run inside of the cassandra process. Contacts Priam for essential cassandra startup information - * like token and seeds. + * A PreMain + * class to run inside of the cassandra process. Contacts Priam for essential cassandra startup + * information like token and seeds. */ -public class PriamStartupAgent -{ - public static String REPLACED_ADDRESS_MIN_VER = "1.2.11"; - public static void premain(String agentArgs, Instrumentation inst) - { +public class PriamStartupAgent { + public static String REPLACED_ADDRESS_MIN_VER = "1.2.11"; + + public static void premain(String agentArgs, Instrumentation inst) { PriamStartupAgent agent = new PriamStartupAgent(); agent.setPriamProperties(); } - private void setPriamProperties() - { + private void setPriamProperties() { String token = null; String seeds = null; boolean isReplace = false; String replacedIp = ""; String extraEnvParams = null; - - while (true) - { - try - { - token = DataFetcher.fetchData("http://127.0.0.1:8080/Priam/REST/v1/cassconfig/get_token"); - seeds = DataFetcher.fetchData("http://127.0.0.1:8080/Priam/REST/v1/cassconfig/get_seeds"); - isReplace = Boolean.parseBoolean(DataFetcher.fetchData("http://127.0.0.1:8080/Priam/REST/v1/cassconfig/is_replace_token")); - replacedIp = DataFetcher.fetchData("http://127.0.0.1:8080/Priam/REST/v1/cassconfig/get_replaced_ip"); - extraEnvParams = DataFetcher.fetchData("http://127.0.0.1:8080/Priam/REST/v1/cassconfig/get_extra_env_params"); - } - catch (Exception e) - { - System.out.println("Failed to obtain startup data from priam, can not start yet. will retry shortly"); + while (true) { + try { + token = + DataFetcher.fetchData( + "http://127.0.0.1:8080/Priam/REST/v1/cassconfig/get_token"); + seeds = + DataFetcher.fetchData( + "http://127.0.0.1:8080/Priam/REST/v1/cassconfig/get_seeds"); + isReplace = + Boolean.parseBoolean( + DataFetcher.fetchData( + "http://127.0.0.1:8080/Priam/REST/v1/cassconfig/is_replace_token")); + replacedIp = + DataFetcher.fetchData( + "http://127.0.0.1:8080/Priam/REST/v1/cassconfig/get_replaced_ip"); + extraEnvParams = + DataFetcher.fetchData( + "http://127.0.0.1:8080/Priam/REST/v1/cassconfig/get_extra_env_params"); + + } catch (Exception e) { + System.out.println( + "Failed to obtain startup data from priam, can not start yet. will retry shortly"); e.printStackTrace(); } - - if (token != null && seeds != null) - break; - try - { + + if (token != null && seeds != null) break; + try { Thread.sleep(5 * 1000); - } - catch (InterruptedException e1) - { + } catch (InterruptedException e1) { // do nothing. } } - + System.setProperty("cassandra.initial_token", token); setExtraEnvParams(extraEnvParams); - if (isReplace) - { - System.out.println("Detect cassandra version : " + FBUtilities.getReleaseVersionString()); - if (FBUtilities.getReleaseVersionString().compareTo(REPLACED_ADDRESS_MIN_VER) < 0) - { - System.setProperty("cassandra.replace_token", token); - } else - { - System.setProperty("cassandra.replace_address", replacedIp); - } + if (isReplace) { + System.out.println( + "Detect cassandra version : " + FBUtilities.getReleaseVersionString()); + if (FBUtilities.getReleaseVersionString().compareTo(REPLACED_ADDRESS_MIN_VER) < 0) { + System.setProperty("cassandra.replace_token", token); + } else { + System.setProperty("cassandra.replace_address", replacedIp); + } } - } - private void setExtraEnvParams(String extraEnvParams) { + private void setExtraEnvParams(String extraEnvParams) { try { if (null != extraEnvParams && extraEnvParams.length() > 0) { JSONParser parser = new JSONParser(); Object obj = parser.parse(extraEnvParams); JSONObject jsonObj = (JSONObject) obj; - if(jsonObj.size()>0) { + if (jsonObj.size() > 0) { for (Iterator iterator = jsonObj.keySet().iterator(); iterator.hasNext(); ) { String key = (String) iterator.next(); String val = (String) jsonObj.get(key); @@ -110,12 +108,12 @@ private void setExtraEnvParams(String extraEnvParams) { } } } - } - catch (Exception e) - { - System.out.println("Failed to parse extra env params: "+extraEnvParams+". However, ignoring the exception."); + } catch (Exception e) { + System.out.println( + "Failed to parse extra env params: " + + extraEnvParams + + ". However, ignoring the exception."); e.printStackTrace(); } } - } diff --git a/priam/src/main/java/com/netflix/priam/PriamServer.java b/priam/src/main/java/com/netflix/priam/PriamServer.java index 603a82850..ec795168c 100644 --- a/priam/src/main/java/com/netflix/priam/PriamServer.java +++ b/priam/src/main/java/com/netflix/priam/PriamServer.java @@ -42,10 +42,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -/** - * Start all tasks here - Property update task - Backup task - Restore task - - * Incremental backup - */ +/** Start all tasks here - Property update task - Backup task - Restore task - Incremental backup */ @Singleton public class PriamServer { private final PriamScheduler scheduler; @@ -59,7 +56,14 @@ public class PriamServer { private static final Logger logger = LoggerFactory.getLogger(PriamServer.class); @Inject - public PriamServer(IConfiguration config, IBackupRestoreConfig backupRestoreConfig, PriamScheduler scheduler, InstanceIdentity id, Sleeper sleeper, ICassandraProcess cassProcess, RestoreContext restoreContext) { + public PriamServer( + IConfiguration config, + IBackupRestoreConfig backupRestoreConfig, + PriamScheduler scheduler, + InstanceIdentity id, + Sleeper sleeper, + ICassandraProcess cassProcess, + RestoreContext restoreContext) { this.config = config; this.backupRestoreConfig = backupRestoreConfig; this.scheduler = scheduler; @@ -70,8 +74,7 @@ public PriamServer(IConfiguration config, IBackupRestoreConfig backupRestoreConf } public void initialize() throws Exception { - if (id.getInstance().isOutOfService()) - return; + if (id.getInstance().isOutOfService()) return; // start to schedule jobs scheduler.start(); @@ -79,13 +82,15 @@ public void initialize() throws Exception { // update security settings. if (config.isMultiDC()) { scheduler.runTaskNow(UpdateSecuritySettings.class); - // sleep for 150 sec if this is a new node with new IP for SG to be updated by other seed nodes - if (id.isReplace() || id.isTokenPregenerated()) - sleeper.sleep(150 * 1000); - else if (UpdateSecuritySettings.firstTimeUpdated) - sleeper.sleep(60 * 1000); - - scheduler.addTask(UpdateSecuritySettings.JOBNAME, UpdateSecuritySettings.class, UpdateSecuritySettings.getTimer(id)); + // sleep for 150 sec if this is a new node with new IP for SG to be updated by other + // seed nodes + if (id.isReplace() || id.isTokenPregenerated()) sleeper.sleep(150 * 1000); + else if (UpdateSecuritySettings.firstTimeUpdated) sleeper.sleep(60 * 1000); + + scheduler.addTask( + UpdateSecuritySettings.JOBNAME, + UpdateSecuritySettings.class, + UpdateSecuritySettings.getTimer(id)); } // Run the task to tune Cassandra @@ -93,44 +98,56 @@ else if (UpdateSecuritySettings.firstTimeUpdated) // Start the snapshot backup schedule - Always run this. (If you want to // set it off, set backup hour to -1) or set backup cron to "-1" - if (SnapshotBackup.getTimer(config) != null && (CollectionUtils.isEmpty(config.getBackupRacs()) || config.getBackupRacs().contains(config.getRac()))) { - scheduler.addTask(SnapshotBackup.JOBNAME, SnapshotBackup.class, SnapshotBackup.getTimer(config)); + if (SnapshotBackup.getTimer(config) != null + && (CollectionUtils.isEmpty(config.getBackupRacs()) + || config.getBackupRacs().contains(config.getRac()))) { + scheduler.addTask( + SnapshotBackup.JOBNAME, SnapshotBackup.class, SnapshotBackup.getTimer(config)); // Start the Incremental backup schedule if enabled if (config.isIncrBackup()) { - scheduler.addTask(IncrementalBackup.JOBNAME, IncrementalBackup.class, IncrementalBackup.getTimer()); + scheduler.addTask( + IncrementalBackup.JOBNAME, + IncrementalBackup.class, + IncrementalBackup.getTimer()); logger.info("Added incremental backup job"); } - } if (config.isBackingUpCommitLogs()) { - scheduler.addTask(CommitLogBackupTask.JOBNAME, CommitLogBackupTask.class, CommitLogBackupTask.getTimer(config)); + scheduler.addTask( + CommitLogBackupTask.JOBNAME, + CommitLogBackupTask.class, + CommitLogBackupTask.getTimer(config)); } - // Determine if we need to restore from backup else start cassandra. if (restoreContext.isRestoreEnabled()) { restoreContext.restore(); - } else { //no restores needed + } else { // no restores needed logger.info("No restore needed, task not scheduled"); - if (!config.doesCassandraStartManually()) - cassProcess.start(true); // Start cassandra. + if (!config.doesCassandraStartManually()) cassProcess.start(true); // Start cassandra. else - logger.info("config.doesCassandraStartManually() is set to True, hence Cassandra needs to be started manually ..."); + logger.info( + "config.doesCassandraStartManually() is set to True, hence Cassandra needs to be started manually ..."); } - /* * Run the delayed task (after 10 seconds) to Monitor Cassandra - * If Restore option is chosen, then Running Cassandra instance is stopped + * If Restore option is chosen, then Running Cassandra instance is stopped * Hence waiting for Cassandra to stop */ - scheduler.addTaskWithDelay(CassandraMonitor.JOBNAME, CassandraMonitor.class, CassandraMonitor.getTimer(), CASSANDRA_MONITORING_INITIAL_DELAY); - + scheduler.addTaskWithDelay( + CassandraMonitor.JOBNAME, + CassandraMonitor.class, + CassandraMonitor.getTimer(), + CASSANDRA_MONITORING_INITIAL_DELAY); // Set cleanup - scheduler.addTask(UpdateCleanupPolicy.JOBNAME, UpdateCleanupPolicy.class, UpdateCleanupPolicy.getTimer()); + scheduler.addTask( + UpdateCleanupPolicy.JOBNAME, + UpdateCleanupPolicy.class, + UpdateCleanupPolicy.getTimer()); // Set up nodetool flush task TaskTimer flushTaskTimer = Flush.getTimer(config); @@ -142,27 +159,36 @@ else if (UpdateSecuritySettings.firstTimeUpdated) // Set up compaction task TaskTimer compactionTimer = Compaction.getTimer(config); if (compactionTimer != null) { - scheduler.addTask(IClusterManagement.Task.COMPACTION.name(), Compaction.class, compactionTimer); + scheduler.addTask( + IClusterManagement.Task.COMPACTION.name(), Compaction.class, compactionTimer); logger.info("Added compaction task."); } // Set up the background configuration dumping thread TaskTimer configurationPersisterTimer = PriamConfigurationPersister.getTimer(config); if (configurationPersisterTimer != null) { - scheduler.addTask(PriamConfigurationPersister.NAME, PriamConfigurationPersister.class, configurationPersisterTimer); - logger.info("Added configuration persister task with schedule [{}]", configurationPersisterTimer.getCronExpression()); + scheduler.addTask( + PriamConfigurationPersister.NAME, + PriamConfigurationPersister.class, + configurationPersisterTimer); + logger.info( + "Added configuration persister task with schedule [{}]", + configurationPersisterTimer.getCronExpression()); } else { logger.warn("Priam configuration persister disabled!"); } - //Set up the SnapshotService + // Set up the SnapshotService setUpSnapshotService(); } private void setUpSnapshotService() throws Exception { TaskTimer snapshotMetaServiceTimer = SnapshotMetaService.getTimer(backupRestoreConfig); if (snapshotMetaServiceTimer != null) { - scheduler.addTask(SnapshotMetaService.JOBNAME, SnapshotMetaService.class, snapshotMetaServiceTimer); + scheduler.addTask( + SnapshotMetaService.JOBNAME, + SnapshotMetaService.class, + snapshotMetaServiceTimer); logger.info("Added SnapshotMetaService Task."); } } @@ -178,5 +204,4 @@ public PriamScheduler getScheduler() { public IConfiguration getConfiguration() { return config; } - -} \ No newline at end of file +} diff --git a/priam/src/main/java/com/netflix/priam/aws/AWSMembership.java b/priam/src/main/java/com/netflix/priam/aws/AWSMembership.java index c4e5ba932..0e9726d9b 100644 --- a/priam/src/main/java/com/netflix/priam/aws/AWSMembership.java +++ b/priam/src/main/java/com/netflix/priam/aws/AWSMembership.java @@ -31,15 +31,14 @@ import com.netflix.priam.cred.ICredential; import com.netflix.priam.identity.IMembership; import com.netflix.priam.identity.InstanceEnvIdentity; +import java.util.*; import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.*; - /** - * Class to query amazon ASG for its members to provide - Number of valid nodes - * in the ASG - Number of zones - Methods for adding ACLs for the nodes + * Class to query amazon ASG for its members to provide - Number of valid nodes in the ASG - Number + * of zones - Methods for adding ACLs for the nodes */ public class AWSMembership implements IMembership { private static final Logger logger = LoggerFactory.getLogger(AWSMembership.class); @@ -49,7 +48,11 @@ public class AWSMembership implements IMembership { private final ICredential crossAccountProvider; @Inject - public AWSMembership(IConfiguration config, ICredential provider, @Named("awsec2roleassumption") ICredential crossAccountProvider, InstanceEnvIdentity insEnvIdentity) { + public AWSMembership( + IConfiguration config, + ICredential provider, + @Named("awsec2roleassumption") ICredential crossAccountProvider, + InstanceEnvIdentity insEnvIdentity) { this.config = config; this.provider = provider; this.insEnvIdentity = insEnvIdentity; @@ -64,35 +67,43 @@ public List getRacMembership() { asgNames.add(config.getASGName()); asgNames.addAll(Arrays.asList(config.getSiblingASGNames().split("\\s*,\\s*"))); client = getAutoScalingClient(); - DescribeAutoScalingGroupsRequest asgReq = new DescribeAutoScalingGroupsRequest().withAutoScalingGroupNames(asgNames.toArray(new String[asgNames.size()])); + DescribeAutoScalingGroupsRequest asgReq = + new DescribeAutoScalingGroupsRequest() + .withAutoScalingGroupNames( + asgNames.toArray(new String[asgNames.size()])); DescribeAutoScalingGroupsResult res = client.describeAutoScalingGroups(asgReq); List instanceIds = Lists.newArrayList(); for (AutoScalingGroup asg : res.getAutoScalingGroups()) { for (Instance ins : asg.getInstances()) - if (!(ins.getLifecycleState().equalsIgnoreCase("Terminating") || ins.getLifecycleState().equalsIgnoreCase("shutting-down") || ins.getLifecycleState() - .equalsIgnoreCase("Terminated"))) + if (!(ins.getLifecycleState().equalsIgnoreCase("Terminating") + || ins.getLifecycleState().equalsIgnoreCase("shutting-down") + || ins.getLifecycleState().equalsIgnoreCase("Terminated"))) instanceIds.add(ins.getInstanceId()); } if (logger.isInfoEnabled()) { - logger.info(String.format("Querying Amazon returned following instance in the RAC: %s, ASGs: %s --> %s", config.getRac(), StringUtils.join(asgNames, ","), StringUtils.join(instanceIds, ","))); + logger.info( + String.format( + "Querying Amazon returned following instance in the RAC: %s, ASGs: %s --> %s", + config.getRac(), + StringUtils.join(asgNames, ","), + StringUtils.join(instanceIds, ","))); } return instanceIds; } finally { - if (client != null) - client.shutdown(); + if (client != null) client.shutdown(); } } - /** - * Actual membership AWS source of truth... - */ + /** Actual membership AWS source of truth... */ @Override public int getRacMembershipSize() { AmazonAutoScaling client = null; try { client = getAutoScalingClient(); - DescribeAutoScalingGroupsRequest asgReq = new DescribeAutoScalingGroupsRequest().withAutoScalingGroupNames(config.getASGName()); + DescribeAutoScalingGroupsRequest asgReq = + new DescribeAutoScalingGroupsRequest() + .withAutoScalingGroupNames(config.getASGName()); DescribeAutoScalingGroupsResult res = client.describeAutoScalingGroups(asgReq); int size = 0; for (AutoScalingGroup asg : res.getAutoScalingGroups()) { @@ -101,8 +112,7 @@ public int getRacMembershipSize() { logger.info("Query on ASG returning {} instances", size); return size; } finally { - if (client != null) - client.shutdown(); + if (client != null) client.shutdown(); } } @@ -114,23 +124,29 @@ public List getCrossAccountRacMembership() { asgNames.add(config.getASGName()); asgNames.addAll(Arrays.asList(config.getSiblingASGNames().split("\\s*,\\s*"))); client = getCrossAccountAutoScalingClient(); - DescribeAutoScalingGroupsRequest asgReq = new DescribeAutoScalingGroupsRequest().withAutoScalingGroupNames(asgNames.toArray(new String[asgNames.size()])); + DescribeAutoScalingGroupsRequest asgReq = + new DescribeAutoScalingGroupsRequest() + .withAutoScalingGroupNames( + asgNames.toArray(new String[asgNames.size()])); DescribeAutoScalingGroupsResult res = client.describeAutoScalingGroups(asgReq); List instanceIds = Lists.newArrayList(); for (AutoScalingGroup asg : res.getAutoScalingGroups()) { for (Instance ins : asg.getInstances()) - if (!(ins.getLifecycleState().equalsIgnoreCase("Terminating") || ins.getLifecycleState().equalsIgnoreCase("shutting-down") || ins.getLifecycleState() - .equalsIgnoreCase("Terminated"))) + if (!(ins.getLifecycleState().equalsIgnoreCase("Terminating") + || ins.getLifecycleState().equalsIgnoreCase("shutting-down") + || ins.getLifecycleState().equalsIgnoreCase("Terminated"))) instanceIds.add(ins.getInstanceId()); } if (logger.isInfoEnabled()) { - logger.info(String.format("Querying Amazon returned following instance in the cross-account ASG: %s --> %s", config.getRac(), StringUtils.join(instanceIds, ","))); + logger.info( + String.format( + "Querying Amazon returned following instance in the cross-account ASG: %s --> %s", + config.getRac(), StringUtils.join(instanceIds, ","))); } return instanceIds; } finally { - if (client != null) - client.shutdown(); + if (client != null) client.shutdown(); } } @@ -140,33 +156,44 @@ public int getRacCount() { } /** - * Adding peers' IPs as ingress to the running instance SG. The running instance could be in "classic" or "vpc" + * Adding peers' IPs as ingress to the running instance SG. The running instance could be in + * "classic" or "vpc" */ public void addACL(Collection listIPs, int from, int to) { AmazonEC2 client = null; try { client = getEc2Client(); List ipPermissions = new ArrayList<>(); - ipPermissions.add(new IpPermission().withFromPort(from).withIpProtocol("tcp").withIpRanges(listIPs).withToPort(to)); + ipPermissions.add( + new IpPermission() + .withFromPort(from) + .withIpProtocol("tcp") + .withIpRanges(listIPs) + .withToPort(to)); if (this.insEnvIdentity.isClassic()) { - client.authorizeSecurityGroupIngress(new AuthorizeSecurityGroupIngressRequest(config.getACLGroupName(), ipPermissions)); + client.authorizeSecurityGroupIngress( + new AuthorizeSecurityGroupIngressRequest( + config.getACLGroupName(), ipPermissions)); if (logger.isInfoEnabled()) { logger.info("Done adding ACL to classic: " + StringUtils.join(listIPs, ",")); } } else { - AuthorizeSecurityGroupIngressRequest sgIngressRequest = new AuthorizeSecurityGroupIngressRequest(); - sgIngressRequest.withGroupId(getVpcGoupId()); //fetch SG group id for vpc account of the running instance. - client.authorizeSecurityGroupIngress(sgIngressRequest.withIpPermissions(ipPermissions)); //Adding peers' IPs as ingress to the running instance SG + AuthorizeSecurityGroupIngressRequest sgIngressRequest = + new AuthorizeSecurityGroupIngressRequest(); + sgIngressRequest.withGroupId(getVpcGoupId()); + // fetch SG group id for vpc account of the running instance. + client.authorizeSecurityGroupIngress( + sgIngressRequest.withIpPermissions( + ipPermissions)); // Adding peers' IPs as ingress to the running + // instance SG if (logger.isInfoEnabled()) { logger.info("Done adding ACL to vpc: " + StringUtils.join(listIPs, ",")); } } - } finally { - if (client != null) - client.shutdown(); + if (client != null) client.shutdown(); } } @@ -177,57 +204,72 @@ protected String getVpcGoupId() { AmazonEC2 client = null; try { client = getEc2Client(); - Filter nameFilter = new Filter().withName("group-name").withValues(config.getACLGroupName()); //SG + Filter nameFilter = + new Filter().withName("group-name").withValues(config.getACLGroupName()); // SG Filter vpcFilter = new Filter().withName("vpc-id").withValues(config.getVpcId()); - DescribeSecurityGroupsRequest req = new DescribeSecurityGroupsRequest().withFilters(nameFilter, vpcFilter); + DescribeSecurityGroupsRequest req = + new DescribeSecurityGroupsRequest().withFilters(nameFilter, vpcFilter); DescribeSecurityGroupsResult result = client.describeSecurityGroups(req); for (SecurityGroup group : result.getSecurityGroups()) { - logger.debug("got group-id:{} for group-name:{},vpc-id:{}", group.getGroupId(), config.getACLGroupName(), config.getVpcId()); + logger.debug( + "got group-id:{} for group-name:{},vpc-id:{}", + group.getGroupId(), + config.getACLGroupName(), + config.getVpcId()); return group.getGroupId(); } - logger.error("unable to get group-id for group-name={} vpc-id={}", config.getACLGroupName(), config.getVpcId()); + logger.error( + "unable to get group-id for group-name={} vpc-id={}", + config.getACLGroupName(), + config.getVpcId()); return ""; } finally { - if (client != null) - client.shutdown(); + if (client != null) client.shutdown(); } } - /** - * removes a iplist from the SG - */ + /** removes a iplist from the SG */ public void removeACL(Collection listIPs, int from, int to) { AmazonEC2 client = null; try { client = getEc2Client(); List ipPermissions = new ArrayList<>(); - ipPermissions.add(new IpPermission().withFromPort(from).withIpProtocol("tcp").withIpRanges(listIPs).withToPort(to)); + ipPermissions.add( + new IpPermission() + .withFromPort(from) + .withIpProtocol("tcp") + .withIpRanges(listIPs) + .withToPort(to)); if (this.insEnvIdentity.isClassic()) { - client.revokeSecurityGroupIngress(new RevokeSecurityGroupIngressRequest(config.getACLGroupName(), ipPermissions)); + client.revokeSecurityGroupIngress( + new RevokeSecurityGroupIngressRequest( + config.getACLGroupName(), ipPermissions)); if (logger.isInfoEnabled()) { - logger.info("Done removing from ACL within classic env for running instance: " + StringUtils.join(listIPs, ",")); + logger.info( + "Done removing from ACL within classic env for running instance: " + + StringUtils.join(listIPs, ",")); } } else { RevokeSecurityGroupIngressRequest req = new RevokeSecurityGroupIngressRequest(); - req.withGroupId(getVpcGoupId()); //fetch SG group id for vpc account of the running instance. - client.revokeSecurityGroupIngress(req.withIpPermissions(ipPermissions)); //Adding peers' IPs as ingress to the running instance SG + // fetch SG group id for vpc account of the running instance. + req.withGroupId(getVpcGoupId()); + // Adding peers' IPs as ingress to the running instance SG + client.revokeSecurityGroupIngress(req.withIpPermissions(ipPermissions)); if (logger.isInfoEnabled()) { - logger.info("Done removing from ACL within vpc env for running instance: " + StringUtils.join(listIPs, ",")); + logger.info( + "Done removing from ACL within vpc env for running instance: " + + StringUtils.join(listIPs, ",")); } } - } finally { - if (client != null) - client.shutdown(); + if (client != null) client.shutdown(); } } - /** - * List SG ACL's - */ + /** List SG ACL's */ public List listACL(int from, int to) { AmazonEC2 client = null; try { @@ -236,7 +278,10 @@ public List listACL(int from, int to) { if (this.insEnvIdentity.isClassic()) { - DescribeSecurityGroupsRequest req = new DescribeSecurityGroupsRequest().withGroupNames(Collections.singletonList(config.getACLGroupName())); + DescribeSecurityGroupsRequest req = + new DescribeSecurityGroupsRequest() + .withGroupNames( + Collections.singletonList(config.getACLGroupName())); DescribeSecurityGroupsResult result = client.describeSecurityGroups(req); for (SecurityGroup group : result.getSecurityGroups()) for (IpPermission perm : group.getIpPermissions()) @@ -246,14 +291,18 @@ public List listACL(int from, int to) { logger.debug("Fetch current permissions for classic env of running instance"); } else { - Filter nameFilter = new Filter().withName("group-name").withValues(config.getACLGroupName()); + Filter nameFilter = + new Filter().withName("group-name").withValues(config.getACLGroupName()); String vpcid = config.getVpcId(); if (vpcid == null || vpcid.isEmpty()) { - throw new IllegalStateException("vpcid is null even though instance is running in vpc."); + throw new IllegalStateException( + "vpcid is null even though instance is running in vpc."); } - Filter vpcFilter = new Filter().withName("vpc-id").withValues(vpcid); //only fetch SG for the vpc id of the running instance - DescribeSecurityGroupsRequest req = new DescribeSecurityGroupsRequest().withFilters(nameFilter, vpcFilter); + // only fetch SG for the vpc id of the running instance + Filter vpcFilter = new Filter().withName("vpc-id").withValues(vpcid); + DescribeSecurityGroupsRequest req = + new DescribeSecurityGroupsRequest().withFilters(nameFilter, vpcFilter); DescribeSecurityGroupsResult result = client.describeSecurityGroups(req); for (SecurityGroup group : result.getSecurityGroups()) for (IpPermission perm : group.getIpPermissions()) @@ -263,11 +312,9 @@ public List listACL(int from, int to) { logger.debug("Fetch current permissions for vpc env of running instance"); } - return ipPermissions; } finally { - if (client != null) - client.shutdown(); + if (client != null) client.shutdown(); } } @@ -276,7 +323,9 @@ public void expandRacMembership(int count) { AmazonAutoScaling client = null; try { client = getAutoScalingClient(); - DescribeAutoScalingGroupsRequest asgReq = new DescribeAutoScalingGroupsRequest().withAutoScalingGroupNames(config.getASGName()); + DescribeAutoScalingGroupsRequest asgReq = + new DescribeAutoScalingGroupsRequest() + .withAutoScalingGroupNames(config.getASGName()); DescribeAutoScalingGroupsResult res = client.describeAutoScalingGroups(asgReq); AutoScalingGroup asg = res.getAutoScalingGroups().get(0); UpdateAutoScalingGroupRequest ureq = new UpdateAutoScalingGroupRequest(); @@ -286,20 +335,28 @@ public void expandRacMembership(int count) { ureq.setDesiredCapacity(asg.getMinSize() + 1); client.updateAutoScalingGroup(ureq); } finally { - if (client != null) - client.shutdown(); + if (client != null) client.shutdown(); } } protected AmazonAutoScaling getAutoScalingClient() { - return AmazonAutoScalingClientBuilder.standard().withCredentials(provider.getAwsCredentialProvider()).withRegion(config.getDC()).build(); + return AmazonAutoScalingClientBuilder.standard() + .withCredentials(provider.getAwsCredentialProvider()) + .withRegion(config.getDC()) + .build(); } protected AmazonAutoScaling getCrossAccountAutoScalingClient() { - return AmazonAutoScalingClientBuilder.standard().withCredentials(crossAccountProvider.getAwsCredentialProvider()).withRegion(config.getDC()).build(); + return AmazonAutoScalingClientBuilder.standard() + .withCredentials(crossAccountProvider.getAwsCredentialProvider()) + .withRegion(config.getDC()) + .build(); } protected AmazonEC2 getEc2Client() { - return AmazonEC2ClientBuilder.standard().withCredentials(provider.getAwsCredentialProvider()).withRegion(config.getDC()).build(); + return AmazonEC2ClientBuilder.standard() + .withCredentials(provider.getAwsCredentialProvider()) + .withRegion(config.getDC()) + .build(); } -} \ No newline at end of file +} diff --git a/priam/src/main/java/com/netflix/priam/aws/DataPart.java b/priam/src/main/java/com/netflix/priam/aws/DataPart.java index 9db9a6e38..e7e36ede6 100644 --- a/priam/src/main/java/com/netflix/priam/aws/DataPart.java +++ b/priam/src/main/java/com/netflix/priam/aws/DataPart.java @@ -18,10 +18,7 @@ import com.netflix.priam.utils.SystemUtils; -/** - * Class for holding part data of a backup file, - * which will be used for multi-part uploading - */ +/** Class for holding part data of a backup file, which will be used for multi-part uploading */ public class DataPart { private final String bucketName; private final String uploadID; diff --git a/priam/src/main/java/com/netflix/priam/aws/S3BackupPath.java b/priam/src/main/java/com/netflix/priam/aws/S3BackupPath.java index 5bf4805a5..6000e09cc 100644 --- a/priam/src/main/java/com/netflix/priam/aws/S3BackupPath.java +++ b/priam/src/main/java/com/netflix/priam/aws/S3BackupPath.java @@ -18,16 +18,13 @@ import com.google.common.collect.Lists; import com.google.inject.Inject; -import com.netflix.priam.config.IConfiguration; import com.netflix.priam.backup.AbstractBackupPath; +import com.netflix.priam.config.IConfiguration; import com.netflix.priam.identity.InstanceIdentity; - import java.util.Date; import java.util.List; -/** - * Represents an S3 object key - */ +/** Represents an S3 object key */ public class S3BackupPath extends AbstractBackupPath { @Inject @@ -44,12 +41,15 @@ public String getRemotePath() { StringBuilder buff = new StringBuilder(); buff.append(baseDir).append(S3BackupPath.PATH_SEP); // Base dir buff.append(region).append(S3BackupPath.PATH_SEP); - buff.append(clusterName).append(S3BackupPath.PATH_SEP);// Cluster name + buff.append(clusterName).append(S3BackupPath.PATH_SEP); // Cluster name buff.append(token).append(S3BackupPath.PATH_SEP); buff.append(formatDate(time)).append(S3BackupPath.PATH_SEP); buff.append(type).append(S3BackupPath.PATH_SEP); if (BackupFileType.isDataFile(type)) - buff.append(keyspace).append(S3BackupPath.PATH_SEP).append(columnFamily).append(S3BackupPath.PATH_SEP); + buff.append(keyspace) + .append(S3BackupPath.PATH_SEP) + .append(columnFamily) + .append(S3BackupPath.PATH_SEP); buff.append(fileName); return buff.toString(); } @@ -60,8 +60,7 @@ public void parseRemote(String remoteFilePath) { // parse out things which are empty List pieces = Lists.newArrayList(); for (String ele : elements) { - if (ele.equals("")) - continue; + if (ele.equals("")) continue; pieces.add(ele); } assert pieces.size() >= 7 : "Too few elements in path " + remoteFilePath; @@ -85,8 +84,7 @@ public void parsePartialPrefix(String remoteFilePath) { // parse out things which are empty List pieces = Lists.newArrayList(); for (String ele : elements) { - if (ele.equals("")) - continue; + if (ele.equals("")) continue; pieces.add(ele); } assert pieces.size() >= 4 : "Too few elements in path " + remoteFilePath; @@ -126,5 +124,4 @@ public String clusterPrefix(String location) { return buff.toString(); } - } diff --git a/priam/src/main/java/com/netflix/priam/aws/S3CrossAccountFileSystem.java b/priam/src/main/java/com/netflix/priam/aws/S3CrossAccountFileSystem.java index 354d29903..0f0befead 100755 --- a/priam/src/main/java/com/netflix/priam/aws/S3CrossAccountFileSystem.java +++ b/priam/src/main/java/com/netflix/priam/aws/S3CrossAccountFileSystem.java @@ -1,16 +1,14 @@ /** * Copyright 2017 Netflix, Inc. - *

- * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and + * + *

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file + * except in compliance with the License. You may obtain a copy of the License at + * + *

http://www.apache.org/licenses/LICENSE-2.0 + * + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing permissions and * limitations under the License. */ package com.netflix.priam.aws; @@ -20,20 +18,20 @@ import com.google.inject.Inject; import com.google.inject.Singleton; import com.google.inject.name.Named; -import com.netflix.priam.config.IConfiguration; import com.netflix.priam.aws.auth.IS3Credential; import com.netflix.priam.backup.IBackupFileSystem; +import com.netflix.priam.config.IConfiguration; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /* * A version of S3FileSystem which allows it api access across different AWS accounts. - * + * * *Note: ideally, this object should extend S3FileSystem but could not be done because: * - S3FileSystem is a singleton and it uses DI. To follow the DI pattern, the best way to get this singleton is via injection. - * - S3FileSystem registers a MBean to JMX which must be only once per JVM. If not, you get + * - S3FileSystem registers a MBean to JMX which must be only once per JVM. If not, you get * java.lang.RuntimeException: javax.management.InstanceAlreadyExistsException: com.priam.aws.S3FileSystemMBean:name=S3FileSystemMBean - * - + * - */ @Singleton public class S3CrossAccountFileSystem { @@ -45,13 +43,14 @@ public class S3CrossAccountFileSystem { private final IS3Credential s3Credential; @Inject - public S3CrossAccountFileSystem(@Named("backup") IBackupFileSystem fs, @Named("awss3roleassumption") IS3Credential s3Credential, IConfiguration config) { - + public S3CrossAccountFileSystem( + @Named("backup") IBackupFileSystem fs, + @Named("awss3roleassumption") IS3Credential s3Credential, + IConfiguration config) { this.s3fs = (S3FileSystem) fs; this.config = config; this.s3Credential = s3Credential; - } public IBackupFileSystem getBackupFileSystem() { @@ -62,29 +61,30 @@ public AmazonS3 getCrossAcctS3Client() { if (this.s3Client == null) { synchronized (this) { - if (this.s3Client == null) { try { - this.s3Client = AmazonS3Client.builder().withCredentials(s3Credential.getAwsCredentialProvider()).withRegion(config.getDC()).build(); + this.s3Client = + AmazonS3Client.builder() + .withCredentials(s3Credential.getAwsCredentialProvider()) + .withRegion(config.getDC()) + .build(); } catch (Exception e) { - throw new IllegalStateException("Exception in getting handle to s3 client. Msg: " + e.getLocalizedMessage(), e); - + throw new IllegalStateException( + "Exception in getting handle to s3 client. Msg: " + + e.getLocalizedMessage(), + e); } - //Lets leverage the IBackupFileSystem behaviors except we want it to use our amazon S3 client which has cross AWS account api capability. + // Lets leverage the IBackupFileSystem behaviors except we want it to use our + // amazon S3 client which has cross AWS account api capability. this.s3fs.setS3Client(s3Client); - } - } - } - return this.s3Client; } - -} \ No newline at end of file +} diff --git a/priam/src/main/java/com/netflix/priam/aws/S3EncryptedFileSystem.java b/priam/src/main/java/com/netflix/priam/aws/S3EncryptedFileSystem.java index 9d7168b05..91645b7ef 100755 --- a/priam/src/main/java/com/netflix/priam/aws/S3EncryptedFileSystem.java +++ b/priam/src/main/java/com/netflix/priam/aws/S3EncryptedFileSystem.java @@ -1,16 +1,14 @@ /** * Copyright 2017 Netflix, Inc. - *

- * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and + * + *

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file + * except in compliance with the License. You may obtain a copy of the License at + * + *

http://www.apache.org/licenses/LICENSE-2.0 + * + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing permissions and * limitations under the License. */ package com.netflix.priam.aws; @@ -25,27 +23,24 @@ import com.google.inject.Provider; import com.google.inject.Singleton; import com.google.inject.name.Named; -import com.netflix.priam.config.IConfiguration; -import com.netflix.priam.cred.ICredential; import com.netflix.priam.backup.AbstractBackupPath; import com.netflix.priam.backup.BackupRestoreException; import com.netflix.priam.backup.RangeReadInputStream; import com.netflix.priam.compress.ICompression; +import com.netflix.priam.config.IConfiguration; +import com.netflix.priam.cred.ICredential; import com.netflix.priam.cryptography.IFileCryptography; import com.netflix.priam.merics.BackupMetrics; import com.netflix.priam.notification.BackupNotificationMgr; -import org.apache.commons.io.IOUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import java.io.*; import java.nio.file.Path; import java.util.Iterator; import java.util.List; +import org.apache.commons.io.IOUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; -/** - * Implementation of IBackupFileSystem for S3. The upload/download will work with ciphertext. - */ +/** Implementation of IBackupFileSystem for S3. The upload/download will work with ciphertext. */ @Singleton public class S3EncryptedFileSystem extends S3FileSystemBase { @@ -53,73 +48,112 @@ public class S3EncryptedFileSystem extends S3FileSystemBase { private final IFileCryptography encryptor; @Inject - public S3EncryptedFileSystem(Provider pathProvider, ICompression compress, final IConfiguration config, ICredential cred - , @Named("filecryptoalgorithm") IFileCryptography fileCryptography - , BackupMetrics backupMetrics, - BackupNotificationMgr backupNotificationMgr - ) { + public S3EncryptedFileSystem( + Provider pathProvider, + ICompression compress, + final IConfiguration config, + ICredential cred, + @Named("filecryptoalgorithm") IFileCryptography fileCryptography, + BackupMetrics backupMetrics, + BackupNotificationMgr backupNotificationMgr) { super(pathProvider, compress, config, backupMetrics, backupNotificationMgr); this.encryptor = fileCryptography; - super.s3Client = AmazonS3Client.builder().withCredentials(cred.getAwsCredentialProvider()).withRegion(config.getDC()).build(); + super.s3Client = + AmazonS3Client.builder() + .withCredentials(cred.getAwsCredentialProvider()) + .withRegion(config.getDC()) + .build(); } - @Override protected void downloadFileImpl(Path remotePath, Path localPath) throws BackupRestoreException { try (OutputStream os = new FileOutputStream(localPath.toFile()); - RangeReadInputStream rris = new RangeReadInputStream(s3Client, getPrefix(config), super.getFileSize(remotePath), remotePath.toString()) - ) { + RangeReadInputStream rris = + new RangeReadInputStream( + s3Client, + getPrefix(config), + super.getFileSize(remotePath), + remotePath.toString())) { /* * To handle use cases where decompression should be done outside of the download. For example, the file have been compressed and then encrypted. - * Hence, decompressing it here would compromise the decryption. - */ + * Hence, decompressing it here would compromise the decryption. + */ IOUtils.copyLarge(rris, os); } catch (Exception e) { - throw new BackupRestoreException("Exception encountered downloading " + remotePath + " from S3 bucket " + getPrefix(config) - + ", Msg: " + e.getMessage(), e); + throw new BackupRestoreException( + "Exception encountered downloading " + + remotePath + + " from S3 bucket " + + getPrefix(config) + + ", Msg: " + + e.getMessage(), + e); } } - @Override protected long uploadFileImpl(Path localPath, Path remotePath) throws BackupRestoreException { long chunkSize = getChunkSize(localPath); - InitiateMultipartUploadRequest initRequest = new InitiateMultipartUploadRequest(config.getBackupPrefix(), remotePath.toString()); //initialize chunking request to aws - InitiateMultipartUploadResult initResponse = s3Client.initiateMultipartUpload(initRequest); //Fetch the aws generated upload id for this chunking request - DataPart part = new DataPart(config.getBackupPrefix(), remotePath.toString(), initResponse.getUploadId()); - List partETags = Lists.newArrayList(); //Metadata on number of parts to be uploaded - - //== Read chunks from src, compress it, and write to temp file + // initialize chunking request to aws + InitiateMultipartUploadRequest initRequest = + new InitiateMultipartUploadRequest(config.getBackupPrefix(), remotePath.toString()); + // Fetch the aws generated upload id for this chunking request + InitiateMultipartUploadResult initResponse = s3Client.initiateMultipartUpload(initRequest); + DataPart part = + new DataPart( + config.getBackupPrefix(), + remotePath.toString(), + initResponse.getUploadId()); + // Metadata on number of parts to be uploaded + List partETags = Lists.newArrayList(); + + // Read chunks from src, compress it, and write to temp file File compressedDstFile = new File(localPath.toString() + ".compressed"); if (logger.isDebugEnabled()) - logger.debug("Compressing {} with chunk size {}", compressedDstFile.getAbsolutePath(), chunkSize); + logger.debug( + "Compressing {} with chunk size {}", + compressedDstFile.getAbsolutePath(), + chunkSize); try (InputStream in = new FileInputStream(localPath.toFile()); - BufferedOutputStream compressedBos = new BufferedOutputStream(new FileOutputStream(compressedDstFile))) { + BufferedOutputStream compressedBos = + new BufferedOutputStream(new FileOutputStream(compressedDstFile))) { Iterator compressedChunks = this.compress.compress(in, chunkSize); while (compressedChunks.hasNext()) { byte[] compressedChunk = compressedChunks.next(); compressedBos.write(compressedChunk); } } catch (Exception e) { - String message = "Exception in compressing the input data during upload to EncryptedStore Msg: " + e.getMessage(); + String message = + "Exception in compressing the input data during upload to EncryptedStore Msg: " + + e.getMessage(); logger.error(message, e); throw new BackupRestoreException(message); } - //== Read compressed data, encrypt each chunk, upload it to aws - try (BufferedInputStream compressedBis = new BufferedInputStream(new FileInputStream(compressedDstFile))) { - Iterator chunks = this.encryptor.encryptStream(compressedBis, remotePath.toString()); + // == Read compressed data, encrypt each chunk, upload it to aws + try (BufferedInputStream compressedBis = + new BufferedInputStream(new FileInputStream(compressedDstFile))) { + Iterator chunks = + this.encryptor.encryptStream(compressedBis, remotePath.toString()); - int partNum = 0; //identifies this part position in the object we are uploading + // identifies this part position in the object we are uploading + int partNum = 0; long encryptedFileSize = 0; while (chunks.hasNext()) { byte[] chunk = chunks.next(); - rateLimiter.acquire(chunk.length); //throttle upload to endpoint - - DataPart dp = new DataPart(++partNum, chunk, config.getBackupPrefix(), remotePath.toString(), initResponse.getUploadId()); + // throttle upload to endpoint + rateLimiter.acquire(chunk.length); + + DataPart dp = + new DataPart( + ++partNum, + chunk, + config.getBackupPrefix(), + remotePath.toString(), + initResponse.getUploadId()); S3PartUploader partUploader = new S3PartUploader(s3Client, dp, partETags); encryptedFileSize += chunk.length; executor.submit(partUploader); @@ -127,19 +161,25 @@ protected long uploadFileImpl(Path localPath, Path remotePath) throws BackupRest executor.sleepTillEmpty(); if (partNum != partETags.size()) { - throw new BackupRestoreException("Number of parts(" + partNum + ") does not match the expected number of uploaded parts(" + partETags.size() + ")"); + throw new BackupRestoreException( + "Number of parts(" + + partNum + + ") does not match the expected number of uploaded parts(" + + partETags.size() + + ")"); } - CompleteMultipartUploadResult resultS3MultiPartUploadComplete = new S3PartUploader(s3Client, part, partETags).completeUpload(); //complete the aws chunking upload by providing to aws the ETag that uniquely identifies the combined object data + // complete the aws chunking upload by providing to aws the ETag that uniquely + // identifies the combined object datav + CompleteMultipartUploadResult resultS3MultiPartUploadComplete = + new S3PartUploader(s3Client, part, partETags).completeUpload(); checkSuccessfulUpload(resultS3MultiPartUploadComplete, localPath); return encryptedFileSize; } catch (Exception e) { new S3PartUploader(s3Client, part, partETags).abortUpload(); throw new BackupRestoreException("Error uploading file: " + localPath, e); } finally { - if (compressedDstFile.exists()) - compressedDstFile.delete(); + if (compressedDstFile.exists()) compressedDstFile.delete(); } - } -} \ No newline at end of file +} diff --git a/priam/src/main/java/com/netflix/priam/aws/S3FileIterator.java b/priam/src/main/java/com/netflix/priam/aws/S3FileIterator.java index a803216c8..a3c237645 100644 --- a/priam/src/main/java/com/netflix/priam/aws/S3FileIterator.java +++ b/priam/src/main/java/com/netflix/priam/aws/S3FileIterator.java @@ -23,16 +23,13 @@ import com.google.common.collect.Lists; import com.google.inject.Provider; import com.netflix.priam.backup.AbstractBackupPath; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import java.util.Date; import java.util.Iterator; import java.util.List; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; -/** - * Iterator representing list of backup files available on S3 - */ +/** Iterator representing list of backup files available on S3 */ public class S3FileIterator implements Iterator { private static final Logger logger = LoggerFactory.getLogger(S3FileIterator.class); private final Provider pathProvider; @@ -42,7 +39,12 @@ public class S3FileIterator implements Iterator { private Iterator iterator; private ObjectListing objectListing; - public S3FileIterator(Provider pathProvider, AmazonS3 s3Client, String path, Date start, Date till) { + public S3FileIterator( + Provider pathProvider, + AmazonS3 s3Client, + String path, + Date start, + Date till) { this.start = start; this.till = till; this.pathProvider = pathProvider; @@ -64,7 +66,6 @@ public boolean hasNext() { objectListing = s3Client.listNextBatchOfObjects(objectListing); iterator = createIterator(); } - } return iterator.hasNext(); } @@ -74,8 +75,15 @@ private Iterator createIterator() { for (S3ObjectSummary summary : objectListing.getObjectSummaries()) { AbstractBackupPath path = pathProvider.get(); path.parseRemote(summary.getKey()); - logger.debug("New key {} path = {} start: {} end: {} my {}", summary.getKey(), path.getRemotePath(), start, till, path.getTime()); - if ((path.getTime().after(start) && path.getTime().before(till)) || path.getTime().equals(start)) { + logger.debug( + "New key {} path = {} start: {} end: {} my {}", + summary.getKey(), + path.getRemotePath(), + start, + till, + path.getTime()); + if ((path.getTime().after(start) && path.getTime().before(till)) + || path.getTime().equals(start)) { temp.add(path); logger.debug("Added key {}", summary.getKey()); } diff --git a/priam/src/main/java/com/netflix/priam/aws/S3FileSystem.java b/priam/src/main/java/com/netflix/priam/aws/S3FileSystem.java index d4114f5c7..51df86653 100644 --- a/priam/src/main/java/com/netflix/priam/aws/S3FileSystem.java +++ b/priam/src/main/java/com/netflix/priam/aws/S3FileSystem.java @@ -23,18 +23,15 @@ import com.google.inject.Provider; import com.google.inject.Singleton; import com.google.inject.name.Named; -import com.netflix.priam.config.IConfiguration; import com.netflix.priam.aws.auth.IS3Credential; import com.netflix.priam.backup.AbstractBackupPath; import com.netflix.priam.backup.BackupRestoreException; import com.netflix.priam.backup.RangeReadInputStream; import com.netflix.priam.compress.ICompression; +import com.netflix.priam.config.IConfiguration; import com.netflix.priam.merics.BackupMetrics; import com.netflix.priam.notification.BackupNotificationMgr; import com.netflix.priam.utils.BoundedExponentialRetryCallable; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import java.io.*; import java.nio.file.Path; import java.util.ArrayList; @@ -43,34 +40,56 @@ import java.util.List; import java.util.concurrent.Future; import java.util.concurrent.atomic.AtomicInteger; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; -/** - * Implementation of IBackupFileSystem for S3 - */ +/** Implementation of IBackupFileSystem for S3 */ @Singleton -public class S3FileSystem extends S3FileSystemBase{ +public class S3FileSystem extends S3FileSystemBase { private static final Logger logger = LoggerFactory.getLogger(S3FileSystem.class); @Inject - public S3FileSystem(@Named("awss3roleassumption") IS3Credential cred, Provider pathProvider, - ICompression compress, - final IConfiguration config, - BackupMetrics backupMetrics, - BackupNotificationMgr backupNotificationMgr) { + public S3FileSystem( + @Named("awss3roleassumption") IS3Credential cred, + Provider pathProvider, + ICompression compress, + final IConfiguration config, + BackupMetrics backupMetrics, + BackupNotificationMgr backupNotificationMgr) { super(pathProvider, compress, config, backupMetrics, backupNotificationMgr); - s3Client = AmazonS3Client.builder().withCredentials(cred.getAwsCredentialProvider()).withRegion(config.getDC()).build(); + s3Client = + AmazonS3Client.builder() + .withCredentials(cred.getAwsCredentialProvider()) + .withRegion(config.getDC()) + .build(); } @Override - protected void downloadFileImpl(Path remotePath, Path localPath) throws BackupRestoreException{ + protected void downloadFileImpl(Path remotePath, Path localPath) throws BackupRestoreException { try { long remoteFileSize = super.getFileSize(remotePath); - RangeReadInputStream rris = new RangeReadInputStream(s3Client, getPrefix(this.config), remoteFileSize, remotePath.toString()); - final long bufSize = MAX_BUFFERED_IN_STREAM_SIZE > remoteFileSize ? remoteFileSize : MAX_BUFFERED_IN_STREAM_SIZE; - compress.decompressAndClose(new BufferedInputStream(rris, (int) bufSize), new BufferedOutputStream(new FileOutputStream(localPath.toFile()))); + RangeReadInputStream rris = + new RangeReadInputStream( + s3Client, + getPrefix(this.config), + remoteFileSize, + remotePath.toString()); + final long bufSize = + MAX_BUFFERED_IN_STREAM_SIZE > remoteFileSize + ? remoteFileSize + : MAX_BUFFERED_IN_STREAM_SIZE; + compress.decompressAndClose( + new BufferedInputStream(rris, (int) bufSize), + new BufferedOutputStream(new FileOutputStream(localPath.toFile()))); } catch (Exception e) { - throw new BackupRestoreException("Exception encountered downloading " + remotePath + " from S3 bucket " + getPrefix(config) - + ", Msg: " + e.getMessage(), e); + throw new BackupRestoreException( + "Exception encountered downloading " + + remotePath + + " from S3 bucket " + + getPrefix(config) + + ", Msg: " + + e.getMessage(), + e); } } @@ -92,14 +111,23 @@ private ObjectMetadata getObjectMetadata(Path path) { private long uploadMultipart(Path localPath, Path remotePath) throws BackupRestoreException { long chunkSize = getChunkSize(localPath); if (logger.isDebugEnabled()) - logger.debug("Uploading to {}/{} with chunk size {}", config.getBackupPrefix(), remotePath, chunkSize); - InitiateMultipartUploadRequest initRequest = new InitiateMultipartUploadRequest(config.getBackupPrefix(), remotePath.toString()); + logger.debug( + "Uploading to {}/{} with chunk size {}", + config.getBackupPrefix(), + remotePath, + chunkSize); + InitiateMultipartUploadRequest initRequest = + new InitiateMultipartUploadRequest(config.getBackupPrefix(), remotePath.toString()); initRequest.withObjectMetadata(getObjectMetadata(localPath)); InitiateMultipartUploadResult initResponse = s3Client.initiateMultipartUpload(initRequest); - DataPart part = new DataPart(config.getBackupPrefix(), remotePath.toString(), initResponse.getUploadId()); + DataPart part = + new DataPart( + config.getBackupPrefix(), + remotePath.toString(), + initResponse.getUploadId()); List partETags = Collections.synchronizedList(new ArrayList()); - try(InputStream in = new FileInputStream(localPath.toFile())) { + try (InputStream in = new FileInputStream(localPath.toFile())) { Iterator chunks = compress.compress(in, chunkSize); // Upload parts. int partNum = 0; @@ -109,28 +137,53 @@ private long uploadMultipart(Path localPath, Path remotePath) throws BackupResto while (chunks.hasNext()) { byte[] chunk = chunks.next(); rateLimiter.acquire(chunk.length); - DataPart dp = new DataPart(++partNum, chunk, config.getBackupPrefix(), remotePath.toString(), initResponse.getUploadId()); - S3PartUploader partUploader = new S3PartUploader(s3Client, dp, partETags, partsUploaded); + DataPart dp = + new DataPart( + ++partNum, + chunk, + config.getBackupPrefix(), + remotePath.toString(), + initResponse.getUploadId()); + S3PartUploader partUploader = + new S3PartUploader(s3Client, dp, partETags, partsUploaded); compressedFileSize += chunk.length; - //TODO: Get the future over here and create a new arraylist. + // TODO: Get the future over here and create a new arraylist. Future future = executor.submit(partUploader); } - //TODO: Instead of waiting for executor thread to be empty we should wait for all the futures to finish. + // TODO: Instead of waiting for executor thread to be empty we should wait for all the + // futures to finish. executor.sleepTillEmpty(); - logger.info("All chunks uploaded for file {}, num of expected parts:{}, num of actual uploaded parts: {}", localPath.toFile().getName(), partNum, partsUploaded.get()); + logger.info( + "All chunks uploaded for file {}, num of expected parts:{}, num of actual uploaded parts: {}", + localPath.toFile().getName(), + partNum, + partsUploaded.get()); if (partNum != partETags.size()) - throw new BackupRestoreException("Number of parts(" + partNum + ") does not match the uploaded parts(" + partETags.size() + ")"); - - CompleteMultipartUploadResult resultS3MultiPartUploadComplete = new S3PartUploader(s3Client, part, partETags).completeUpload(); + throw new BackupRestoreException( + "Number of parts(" + + partNum + + ") does not match the uploaded parts(" + + partETags.size() + + ")"); + + CompleteMultipartUploadResult resultS3MultiPartUploadComplete = + new S3PartUploader(s3Client, part, partETags).completeUpload(); checkSuccessfulUpload(resultS3MultiPartUploadComplete, localPath); if (logger.isDebugEnabled()) { - final S3ResponseMetadata responseMetadata = s3Client.getCachedResponseMetadata(initRequest); - final String requestId = responseMetadata.getRequestId(); // "x-amz-request-id" header + final S3ResponseMetadata responseMetadata = + s3Client.getCachedResponseMetadata(initRequest); + final String requestId = + responseMetadata.getRequestId(); // "x-amz-request-id" header final String hostId = responseMetadata.getHostId(); // "x-amz-id-2" header - logger.debug("S3 AWS x-amz-request-id[" + requestId + "], and x-amz-id-2[" + hostId + "]"); + logger.debug( + "S3 AWS x-amz-request-id[" + + requestId + + "], and x-amz-id-2[" + + hostId + + "]"); } return compressedFileSize; @@ -140,17 +193,21 @@ private long uploadMultipart(Path localPath, Path remotePath) throws BackupResto } } - protected long uploadFileImpl(Path localPath, Path remotePath) throws BackupRestoreException{ + protected long uploadFileImpl(Path localPath, Path remotePath) throws BackupRestoreException { long chunkSize = config.getBackupChunkSize(); long fileSize = localPath.toFile().length(); if (fileSize < chunkSize) { - //Upload file without using multipart upload as it will be more efficient. + // Upload file without using multipart upload as it will be more efficient. if (logger.isDebugEnabled()) - logger.debug("Uploading to {}/{} using PUT operation", config.getBackupPrefix(), remotePath); + logger.debug( + "Uploading to {}/{} using PUT operation", + config.getBackupPrefix(), + remotePath); try (ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); - InputStream in = new BufferedInputStream(new FileInputStream(localPath.toFile()))) { + InputStream in = + new BufferedInputStream(new FileInputStream(localPath.toFile()))) { Iterator chunkedStream = compress.compress(in, chunkSize); while (chunkedStream.hasNext()) { byteArrayOutputStream.write(chunkedStream.next()); @@ -160,23 +217,32 @@ protected long uploadFileImpl(Path localPath, Path remotePath) throws BackupRest rateLimiter.acquire(chunk.length); ObjectMetadata objectMetadata = getObjectMetadata(localPath); objectMetadata.setContentLength(chunk.length); - PutObjectRequest putObjectRequest = new PutObjectRequest(config.getBackupPrefix(), remotePath.toString(), new ByteArrayInputStream(chunk), objectMetadata); - //Retry if failed. - PutObjectResult upload = new BoundedExponentialRetryCallable(1000, 10000, 5) { - @Override - public PutObjectResult retriableCall() throws Exception { - return s3Client.putObject(putObjectRequest); - } - }.call(); + PutObjectRequest putObjectRequest = + new PutObjectRequest( + config.getBackupPrefix(), + remotePath.toString(), + new ByteArrayInputStream(chunk), + objectMetadata); + // Retry if failed. + PutObjectResult upload = + new BoundedExponentialRetryCallable(1000, 10000, 5) { + @Override + public PutObjectResult retriableCall() throws Exception { + return s3Client.putObject(putObjectRequest); + } + }.call(); if (logger.isDebugEnabled()) - logger.debug("Successfully uploaded file with putObject: {} and etag: {}", remotePath, upload.getETag()); + logger.debug( + "Successfully uploaded file with putObject: {} and etag: {}", + remotePath, + upload.getETag()); return compressedFileSize; } catch (Exception e) { - throw new BackupRestoreException("Error uploading file: " + localPath.toFile().getName(), e); + throw new BackupRestoreException( + "Error uploading file: " + localPath.toFile().getName(), e); } - } else - return uploadMultipart(localPath, remotePath); + } else return uploadMultipart(localPath, remotePath); } } diff --git a/priam/src/main/java/com/netflix/priam/aws/S3FileSystemBase.java b/priam/src/main/java/com/netflix/priam/aws/S3FileSystemBase.java index c0cb1021e..eb2533c17 100755 --- a/priam/src/main/java/com/netflix/priam/aws/S3FileSystemBase.java +++ b/priam/src/main/java/com/netflix/priam/aws/S3FileSystemBase.java @@ -1,16 +1,14 @@ /** * Copyright 2017 Netflix, Inc. - *

- * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and + * + *

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file + * except in compliance with the License. You may obtain a copy of the License at + * + *

http://www.apache.org/licenses/LICENSE-2.0 + * + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing permissions and * limitations under the License. */ package com.netflix.priam.aws; @@ -30,15 +28,14 @@ import com.netflix.priam.merics.BackupMetrics; import com.netflix.priam.notification.BackupNotificationMgr; import com.netflix.priam.scheduler.BlockingSubmitThreadPoolExecutor; -import org.apache.commons.lang3.StringUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import java.nio.file.Path; import java.util.Date; import java.util.Iterator; import java.util.List; import java.util.concurrent.LinkedBlockingQueue; +import org.apache.commons.lang3.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public abstract class S3FileSystemBase extends AbstractFileSystem { private static final int MAX_CHUNKS = 10000; @@ -49,13 +46,15 @@ public abstract class S3FileSystemBase extends AbstractFileSystem { private final Provider pathProvider; final ICompression compress; final BlockingSubmitThreadPoolExecutor executor; - final RateLimiter rateLimiter; //a throttling mechanism, we can limit the amount of bytes uploaded to endpoint per second. - - S3FileSystemBase(Provider pathProvider, - ICompression compress, - final IConfiguration config, - BackupMetrics backupMetrics, - BackupNotificationMgr backupNotificationMgr) { + // a throttling mechanism, we can limit the amount of bytes uploaded to endpoint per second. + final RateLimiter rateLimiter; + + S3FileSystemBase( + Provider pathProvider, + ICompression compress, + final IConfiguration config, + BackupMetrics backupMetrics, + BackupNotificationMgr backupNotificationMgr) { super(config, backupMetrics, backupNotificationMgr); this.pathProvider = pathProvider; this.compress = compress; @@ -63,13 +62,13 @@ public abstract class S3FileSystemBase extends AbstractFileSystem { int threads = config.getBackupThreads(); LinkedBlockingQueue queue = new LinkedBlockingQueue<>(threads); - this.executor = new BlockingSubmitThreadPoolExecutor(threads, queue, config.getUploadTimeout()); + this.executor = + new BlockingSubmitThreadPoolExecutor(threads, queue, config.getUploadTimeout()); double throttleLimit = config.getUploadThrottle(); this.rateLimiter = RateLimiter.create(throttleLimit < 1 ? Double.MAX_VALUE : throttleLimit); } - private AmazonS3 getS3Client() { return s3Client; } @@ -81,15 +80,11 @@ public void setS3Client(AmazonS3 client) { s3Client = client; } - /** - * Get S3 prefix which will be used to locate S3 files - */ + /** Get S3 prefix which will be used to locate S3 files */ String getPrefix(IConfiguration config) { String prefix; - if (StringUtils.isNotBlank(config.getRestorePrefix())) - prefix = config.getRestorePrefix(); - else - prefix = config.getBackupPrefix(); + if (StringUtils.isNotBlank(config.getRestorePrefix())) prefix = config.getRestorePrefix(); + else prefix = config.getBackupPrefix(); String[] paths = prefix.split(String.valueOf(S3BackupPath.PATH_SEP)); return paths[0]; @@ -101,7 +96,8 @@ public void cleanup() { AmazonS3 s3Client = getS3Client(); String clusterPath = pathProvider.get().clusterPrefix(""); logger.debug("Bucket: {}", config.getBackupPrefix()); - BucketLifecycleConfiguration lifeConfig = s3Client.getBucketLifecycleConfiguration(config.getBackupPrefix()); + BucketLifecycleConfiguration lifeConfig = + s3Client.getBucketLifecycleConfiguration(config.getBackupPrefix()); logger.debug("Got bucket:{} lifecycle.{}", config.getBackupPrefix(), lifeConfig); if (lifeConfig == null) { lifeConfig = new BucketLifecycleConfiguration(); @@ -113,10 +109,8 @@ public void cleanup() { if (rules.size() > 0) { lifeConfig.setRules(rules); s3Client.setBucketLifecycleConfiguration(config.getBackupPrefix(), lifeConfig); - } else - s3Client.deleteBucketLifecycleConfiguration(config.getBackupPrefix()); + } else s3Client.deleteBucketLifecycleConfiguration(config.getBackupPrefix()); } - } private boolean updateLifecycleRule(IConfiguration config, List rules, String prefix) { @@ -127,21 +121,29 @@ private boolean updateLifecycleRule(IConfiguration config, List rules, Str break; } } - if (rule == null && config.getBackupRetentionDays() <= 0) - return false; + if (rule == null && config.getBackupRetentionDays() <= 0) return false; if (rule != null && rule.getExpirationInDays() == config.getBackupRetentionDays()) { logger.info("Cleanup rule already set"); return false; } if (rule == null) { // Create a new rule - rule = new BucketLifecycleConfiguration.Rule().withExpirationInDays(config.getBackupRetentionDays()).withPrefix(prefix); + rule = + new BucketLifecycleConfiguration.Rule() + .withExpirationInDays(config.getBackupRetentionDays()) + .withPrefix(prefix); rule.setStatus(BucketLifecycleConfiguration.ENABLED); rule.setId(prefix); rules.add(rule); - logger.info("Setting cleanup for {} to {} days", rule.getPrefix(), rule.getExpirationInDays()); + logger.info( + "Setting cleanup for {} to {} days", + rule.getPrefix(), + rule.getExpirationInDays()); } else if (config.getBackupRetentionDays() > 0) { - logger.info("Setting cleanup for {} to {} days", rule.getPrefix(), config.getBackupRetentionDays()); + logger.info( + "Setting cleanup for {} to {} days", + rule.getPrefix(), + config.getBackupRetentionDays()); rule.setExpirationInDays(config.getBackupRetentionDays()); } else { logger.info("Removing cleanup rule for {}", rule.getPrefix()); @@ -150,24 +152,31 @@ private boolean updateLifecycleRule(IConfiguration config, List rules, Str return true; } - void checkSuccessfulUpload(CompleteMultipartUploadResult resultS3MultiPartUploadComplete, Path localPath) throws BackupRestoreException { - if (null != resultS3MultiPartUploadComplete && null != resultS3MultiPartUploadComplete.getETag()) { - logger.info("Uploaded file: {}, object eTag: {}", localPath, resultS3MultiPartUploadComplete.getETag()); + void checkSuccessfulUpload( + CompleteMultipartUploadResult resultS3MultiPartUploadComplete, Path localPath) + throws BackupRestoreException { + if (null != resultS3MultiPartUploadComplete + && null != resultS3MultiPartUploadComplete.getETag()) { + logger.info( + "Uploaded file: {}, object eTag: {}", + localPath, + resultS3MultiPartUploadComplete.getETag()); } else { - throw new BackupRestoreException("Error uploading file as ETag or CompleteMultipartUploadResult is NULL -" + localPath); + throw new BackupRestoreException( + "Error uploading file as ETag or CompleteMultipartUploadResult is NULL -" + + localPath); } } @Override - public long getFileSize(Path remotePath) throws BackupRestoreException{ - return s3Client.getObjectMetadata(getPrefix(config), remotePath.toString()).getContentLength(); + public long getFileSize(Path remotePath) throws BackupRestoreException { + return s3Client.getObjectMetadata(getPrefix(config), remotePath.toString()) + .getContentLength(); } @Override public void shutdown() { - if (executor != null) - executor.shutdown(); - + if (executor != null) executor.shutdown(); } @Override @@ -180,15 +189,17 @@ public Iterator list(String path, Date start, Date till) { return new S3FileIterator(pathProvider, s3Client, path, start, till); } - final long getChunkSize(Path localPath) throws BackupRestoreException{ + final long getChunkSize(Path localPath) throws BackupRestoreException { long chunkSize = config.getBackupChunkSize(); long fileSize = localPath.toFile().length(); - //compute the size of each block we will upload to endpoint + // compute the size of each block we will upload to endpoint if (fileSize > 0) - chunkSize = (fileSize / chunkSize >= MAX_CHUNKS) ? (fileSize / (MAX_CHUNKS - 1)) : chunkSize; + chunkSize = + (fileSize / chunkSize >= MAX_CHUNKS) + ? (fileSize / (MAX_CHUNKS - 1)) + : chunkSize; return chunkSize; } - } diff --git a/priam/src/main/java/com/netflix/priam/aws/S3PartUploader.java b/priam/src/main/java/com/netflix/priam/aws/S3PartUploader.java index b36835f67..5a6a04ae6 100644 --- a/priam/src/main/java/com/netflix/priam/aws/S3PartUploader.java +++ b/priam/src/main/java/com/netflix/priam/aws/S3PartUploader.java @@ -22,19 +22,17 @@ import com.netflix.priam.backup.BackupRestoreException; import com.netflix.priam.utils.BoundedExponentialRetryCallable; import com.netflix.priam.utils.SystemUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import java.io.ByteArrayInputStream; import java.util.List; import java.util.concurrent.atomic.AtomicInteger; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; -public class S3PartUploader extends BoundedExponentialRetryCallable -{ +public class S3PartUploader extends BoundedExponentialRetryCallable { private final AmazonS3 client; private final DataPart dataPart; private final List partETags; - private AtomicInteger partsUploaded = null; //num of data parts successfully uploaded + private AtomicInteger partsUploaded = null; // num of data parts successfully uploaded private static final Logger logger = LoggerFactory.getLogger(S3PartUploader.class); private static final int MAX_RETRIES = 5; @@ -47,7 +45,8 @@ public S3PartUploader(AmazonS3 client, DataPart dp, List partETags) { this.partETags = partETags; } - public S3PartUploader(AmazonS3 client, DataPart dp, List partETags, AtomicInteger partsUploaded) { + public S3PartUploader( + AmazonS3 client, DataPart dp, List partETags, AtomicInteger partsUploaded) { super(DEFAULT_MIN_SLEEP_MS, BoundedExponentialRetryCallable.MAX_SLEEP, MAX_RETRIES); this.client = client; this.dataPart = dp; @@ -55,7 +54,6 @@ public S3PartUploader(AmazonS3 client, DataPart dp, List partETags, At this.partsUploaded = partsUploaded; } - private Void uploadPart() throws AmazonClientException, BackupRestoreException { UploadPartRequest req = new UploadPartRequest(); req.setBucketName(dataPart.getBucketName()); @@ -68,27 +66,35 @@ private Void uploadPart() throws AmazonClientException, BackupRestoreException { UploadPartResult res = client.uploadPart(req); PartETag partETag = res.getPartETag(); if (!partETag.getETag().equals(SystemUtils.toHex(dataPart.getMd5()))) - throw new BackupRestoreException("Unable to match MD5 for part " + dataPart.getPartNo()); + throw new BackupRestoreException( + "Unable to match MD5 for part " + dataPart.getPartNo()); partETags.add(partETag); - if (this.partsUploaded != null) - this.partsUploaded.incrementAndGet(); + if (this.partsUploaded != null) this.partsUploaded.incrementAndGet(); return null; } public CompleteMultipartUploadResult completeUpload() throws BackupRestoreException { - CompleteMultipartUploadRequest compRequest = new CompleteMultipartUploadRequest(dataPart.getBucketName(), dataPart.getS3key(), dataPart.getUploadID(), partETags); + CompleteMultipartUploadRequest compRequest = + new CompleteMultipartUploadRequest( + dataPart.getBucketName(), + dataPart.getS3key(), + dataPart.getUploadID(), + partETags); return client.completeMultipartUpload(compRequest); } // Abort public void abortUpload() { - AbortMultipartUploadRequest abortRequest = new AbortMultipartUploadRequest(dataPart.getBucketName(), dataPart.getS3key(), dataPart.getUploadID()); + AbortMultipartUploadRequest abortRequest = + new AbortMultipartUploadRequest( + dataPart.getBucketName(), dataPart.getS3key(), dataPart.getUploadID()); client.abortMultipartUpload(abortRequest); } @Override public Void retriableCall() throws AmazonClientException, BackupRestoreException { - logger.debug("Picked up part {} size {}", dataPart.getPartNo(), dataPart.getPartData().length); + logger.debug( + "Picked up part {} size {}", dataPart.getPartNo(), dataPart.getPartData().length); return uploadPart(); } } diff --git a/priam/src/main/java/com/netflix/priam/aws/S3PrefixIterator.java b/priam/src/main/java/com/netflix/priam/aws/S3PrefixIterator.java index 079baa82c..2c6d55254 100644 --- a/priam/src/main/java/com/netflix/priam/aws/S3PrefixIterator.java +++ b/priam/src/main/java/com/netflix/priam/aws/S3PrefixIterator.java @@ -22,21 +22,19 @@ import com.google.common.collect.Lists; import com.google.inject.Inject; import com.google.inject.Provider; -import com.netflix.priam.config.IConfiguration; import com.netflix.priam.backup.AbstractBackupPath; -import org.apache.commons.lang3.StringUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - +import com.netflix.priam.config.IConfiguration; import java.text.SimpleDateFormat; import java.util.Date; import java.util.Iterator; import java.util.List; +import org.apache.commons.lang3.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** - * Class to iterate over prefixes (S3 Common prefixes) upto - * the token element in the path. The abstract path generated by this class - * is partial (does not have all data). + * Class to iterate over prefixes (S3 Common prefixes) upto the token element in the path. The + * abstract path generated by this class is partial (does not have all data). */ public class S3PrefixIterator implements Iterator { private static final Logger logger = LoggerFactory.getLogger(S3PrefixIterator.class); @@ -52,16 +50,18 @@ public class S3PrefixIterator implements Iterator { final Date date; @Inject - public S3PrefixIterator(IConfiguration config, Provider pathProvider, AmazonS3 s3Client, Date date) { + public S3PrefixIterator( + IConfiguration config, + Provider pathProvider, + AmazonS3 s3Client, + Date date) { this.config = config; this.pathProvider = pathProvider; this.s3Client = s3Client; this.date = date; String path; - if (StringUtils.isNotBlank(config.getRestorePrefix())) - path = config.getRestorePrefix(); - else - path = config.getBackupPrefix(); + if (StringUtils.isNotBlank(config.getRestorePrefix())) path = config.getRestorePrefix(); + else path = config.getBackupPrefix(); String[] paths = path.split(String.valueOf(S3BackupPath.PATH_SEP)); bucket = paths[0]; @@ -77,12 +77,10 @@ private void initListing() { listReq.setDelimiter(String.valueOf(AbstractBackupPath.PATH_SEP)); logger.info("Using cluster prefix for searching tokens: {}", clusterPath); objectListing = s3Client.listObjects(listReq); - } private Iterator createIterator() { - if (objectListing == null) - initListing(); + if (objectListing == null) initListing(); List temp = Lists.newArrayList(); for (String summary : objectListing.getCommonPrefixes()) { if (pathExistsForDate(summary, datefmt.format(date))) { @@ -113,12 +111,9 @@ public AbstractBackupPath next() { } @Override - public void remove() { - } + public void remove() {} - /** - * Get remote prefix upto the token - */ + /** Get remote prefix upto the token */ private String remotePrefix(String location) { StringBuilder buff = new StringBuilder(); String[] elements = location.split(String.valueOf(S3BackupPath.PATH_SEP)); @@ -135,9 +130,7 @@ private String remotePrefix(String location) { return buff.toString(); } - /** - * Check to see if the path exists for the date - */ + /** Check to see if the path exists for the date */ private boolean pathExistsForDate(String tprefix, String datestr) { ListObjectsRequest listReq = new ListObjectsRequest(); // Get list of tokens @@ -147,5 +140,4 @@ private boolean pathExistsForDate(String tprefix, String datestr) { listing = s3Client.listObjects(listReq); return listing.getObjectSummaries().size() > 0; } - } diff --git a/priam/src/main/java/com/netflix/priam/aws/SDBInstanceData.java b/priam/src/main/java/com/netflix/priam/aws/SDBInstanceData.java index 997c7c677..3c035c3d3 100644 --- a/priam/src/main/java/com/netflix/priam/aws/SDBInstanceData.java +++ b/priam/src/main/java/com/netflix/priam/aws/SDBInstanceData.java @@ -25,29 +25,36 @@ import com.netflix.priam.config.IConfiguration; import com.netflix.priam.cred.ICredential; import com.netflix.priam.identity.PriamInstance; - import java.util.*; -/** - * DAO for handling Instance identity information such as token, zone, region - */ +/** DAO for handling Instance identity information such as token, zone, region */ @Singleton public class SDBInstanceData { public static class Attributes { - public final static String APP_ID = "appId"; - public final static String ID = "id"; - public final static String INSTANCE_ID = "instanceId"; - public final static String TOKEN = "token"; - public final static String AVAILABILITY_ZONE = "availabilityZone"; - public final static String ELASTIC_IP = "elasticIP"; - public final static String UPDATE_TS = "updateTimestamp"; - public final static String LOCATION = "location"; - public final static String HOSTNAME = "hostname"; + public static final String APP_ID = "appId"; + public static final String ID = "id"; + public static final String INSTANCE_ID = "instanceId"; + public static final String TOKEN = "token"; + public static final String AVAILABILITY_ZONE = "availabilityZone"; + public static final String ELASTIC_IP = "elasticIP"; + public static final String UPDATE_TS = "updateTimestamp"; + public static final String LOCATION = "location"; + public static final String HOSTNAME = "hostname"; } public static final String DOMAIN = "InstanceIdentity"; - public static final String ALL_QUERY = "select * from " + DOMAIN + " where " + Attributes.APP_ID + "='%s'"; - public static final String INSTANCE_QUERY = "select * from " + DOMAIN + " where " + Attributes.APP_ID + "='%s' and " + Attributes.LOCATION + "='%s' and " + Attributes.ID + "='%d'"; + public static final String ALL_QUERY = + "select * from " + DOMAIN + " where " + Attributes.APP_ID + "='%s'"; + public static final String INSTANCE_QUERY = + "select * from " + + DOMAIN + + " where " + + Attributes.APP_ID + + "='%s' and " + + Attributes.LOCATION + + "='%s' and " + + Attributes.ID + + "='%d'"; private final ICredential provider; private final IConfiguration configuration; @@ -69,8 +76,7 @@ public PriamInstance getInstance(String app, String dc, int id) { AmazonSimpleDB simpleDBClient = getSimpleDBClient(); SelectRequest request = new SelectRequest(String.format(INSTANCE_QUERY, app, dc, id)); SelectResult result = simpleDBClient.select(request); - if (result.getItems().size() == 0) - return null; + if (result.getItems().size() == 0) return null; return transform(result.getItems().get(0)); } @@ -105,7 +111,9 @@ public Set getAllIds(String app) { */ public void createInstance(PriamInstance instance) throws AmazonServiceException { AmazonSimpleDB simpleDBClient = getSimpleDBClient(); - PutAttributesRequest putReq = new PutAttributesRequest(DOMAIN, getKey(instance), createAttributesToRegister(instance)); + PutAttributesRequest putReq = + new PutAttributesRequest( + DOMAIN, getKey(instance), createAttributesToRegister(instance)); simpleDBClient.putAttributes(putReq); } @@ -117,7 +125,9 @@ public void createInstance(PriamInstance instance) throws AmazonServiceException */ public void registerInstance(PriamInstance instance) throws AmazonServiceException { AmazonSimpleDB simpleDBClient = getSimpleDBClient(); - PutAttributesRequest putReq = new PutAttributesRequest(DOMAIN, getKey(instance), createAttributesToRegister(instance)); + PutAttributesRequest putReq = + new PutAttributesRequest( + DOMAIN, getKey(instance), createAttributesToRegister(instance)); UpdateCondition expected = new UpdateCondition(); expected.setName(Attributes.INSTANCE_ID); expected.setExists(false); @@ -133,22 +143,28 @@ public void registerInstance(PriamInstance instance) throws AmazonServiceExcepti */ public void deregisterInstance(PriamInstance instance) throws AmazonServiceException { AmazonSimpleDB simpleDBClient = getSimpleDBClient(); - DeleteAttributesRequest delReq = new DeleteAttributesRequest(DOMAIN, getKey(instance), createAttributesToDeRegister(instance)); + DeleteAttributesRequest delReq = + new DeleteAttributesRequest( + DOMAIN, getKey(instance), createAttributesToDeRegister(instance)); simpleDBClient.deleteAttributes(delReq); } protected List createAttributesToRegister(PriamInstance instance) { instance.setUpdatetime(new Date().getTime()); List attrs = new ArrayList<>(); - attrs.add(new ReplaceableAttribute(Attributes.INSTANCE_ID, instance.getInstanceId(), false)); + attrs.add( + new ReplaceableAttribute(Attributes.INSTANCE_ID, instance.getInstanceId(), false)); attrs.add(new ReplaceableAttribute(Attributes.TOKEN, instance.getToken(), true)); attrs.add(new ReplaceableAttribute(Attributes.APP_ID, instance.getApp(), true)); - attrs.add(new ReplaceableAttribute(Attributes.ID, Integer.toString(instance.getId()), true)); + attrs.add( + new ReplaceableAttribute(Attributes.ID, Integer.toString(instance.getId()), true)); attrs.add(new ReplaceableAttribute(Attributes.AVAILABILITY_ZONE, instance.getRac(), true)); attrs.add(new ReplaceableAttribute(Attributes.ELASTIC_IP, instance.getHostIP(), true)); attrs.add(new ReplaceableAttribute(Attributes.HOSTNAME, instance.getHostName(), true)); attrs.add(new ReplaceableAttribute(Attributes.LOCATION, instance.getDC(), true)); - attrs.add(new ReplaceableAttribute(Attributes.UPDATE_TS, Long.toString(instance.getUpdatetime()), true)); + attrs.add( + new ReplaceableAttribute( + Attributes.UPDATE_TS, Long.toString(instance.getUpdatetime()), true)); return attrs; } @@ -175,22 +191,15 @@ protected List createAttributesToDeRegister(PriamInstance instance) { public PriamInstance transform(Item item) { PriamInstance ins = new PriamInstance(); for (Attribute att : item.getAttributes()) { - if (att.getName().equals(Attributes.INSTANCE_ID)) - ins.setInstanceId(att.getValue()); - else if (att.getName().equals(Attributes.TOKEN)) - ins.setToken(att.getValue()); - else if (att.getName().equals(Attributes.APP_ID)) - ins.setApp(att.getValue()); + if (att.getName().equals(Attributes.INSTANCE_ID)) ins.setInstanceId(att.getValue()); + else if (att.getName().equals(Attributes.TOKEN)) ins.setToken(att.getValue()); + else if (att.getName().equals(Attributes.APP_ID)) ins.setApp(att.getValue()); else if (att.getName().equals(Attributes.ID)) ins.setId(Integer.parseInt(att.getValue())); - else if (att.getName().equals(Attributes.AVAILABILITY_ZONE)) - ins.setRac(att.getValue()); - else if (att.getName().equals(Attributes.ELASTIC_IP)) - ins.setHostIP(att.getValue()); - else if (att.getName().equals(Attributes.HOSTNAME)) - ins.setHost(att.getValue()); - else if (att.getName().equals(Attributes.LOCATION)) - ins.setDC(att.getValue()); + else if (att.getName().equals(Attributes.AVAILABILITY_ZONE)) ins.setRac(att.getValue()); + else if (att.getName().equals(Attributes.ELASTIC_IP)) ins.setHostIP(att.getValue()); + else if (att.getName().equals(Attributes.HOSTNAME)) ins.setHost(att.getValue()); + else if (att.getName().equals(Attributes.LOCATION)) ins.setDC(att.getValue()); else if (att.getName().equals(Attributes.UPDATE_TS)) ins.setUpdatetime(Long.parseLong(att.getValue())); } @@ -202,7 +211,10 @@ private String getKey(PriamInstance instance) { } private AmazonSimpleDB getSimpleDBClient() { - //Create per request - return AmazonSimpleDBClient.builder().withCredentials(provider.getAwsCredentialProvider()).withRegion(configuration.getSDBInstanceIdentityRegion()).build(); + // Create per request + return AmazonSimpleDBClient.builder() + .withCredentials(provider.getAwsCredentialProvider()) + .withRegion(configuration.getSDBInstanceIdentityRegion()) + .build(); } } diff --git a/priam/src/main/java/com/netflix/priam/aws/SDBInstanceFactory.java b/priam/src/main/java/com/netflix/priam/aws/SDBInstanceFactory.java index 148f82af8..bf4795068 100644 --- a/priam/src/main/java/com/netflix/priam/aws/SDBInstanceFactory.java +++ b/priam/src/main/java/com/netflix/priam/aws/SDBInstanceFactory.java @@ -22,15 +22,11 @@ import com.netflix.priam.config.IConfiguration; import com.netflix.priam.identity.IPriamInstanceFactory; import com.netflix.priam.identity.PriamInstance; +import java.util.*; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.*; - -/** - * SimpleDB based instance factory. Requires 'InstanceIdentity' domain to be - * created ahead - */ +/** SimpleDB based instance factory. Requires 'InstanceIdentity' domain to be created ahead */ @Singleton public class SDBInstanceFactory implements IPriamInstanceFactory { private static final Logger logger = LoggerFactory.getLogger(SDBInstanceFactory.class); @@ -58,18 +54,29 @@ public PriamInstance getInstance(String appName, String dc, int id) { } @Override - public PriamInstance create(String app, int id, String instanceID, String hostname, String ip, String rac, Map volumes, String token) { + public PriamInstance create( + String app, + int id, + String instanceID, + String hostname, + String ip, + String rac, + Map volumes, + String token) { try { - PriamInstance ins = makePriamInstance(app, id, instanceID, hostname, ip, rac, volumes, token); + PriamInstance ins = + makePriamInstance(app, id, instanceID, hostname, ip, rac, volumes, token); // remove old data node which are dead. if (app.endsWith("-dead")) { try { PriamInstance oldData = dao.getInstance(app, config.getDC(), id); // clean up a very old data... - if (null != oldData && oldData.getUpdatetime() < (System.currentTimeMillis() - (3 * 60 * 1000))) + if (null != oldData + && oldData.getUpdatetime() + < (System.currentTimeMillis() - (3 * 60 * 1000))) dao.deregisterInstance(oldData); } catch (Exception ex) { - //Do nothing + // Do nothing logger.error(ex.getMessage(), ex); } } @@ -101,12 +108,13 @@ public void update(PriamInstance inst) { @Override public void sort(List return_) { - Comparator comparator = (Comparator) (o1, o2) -> { - - Integer c1 = o1.getId(); - Integer c2 = o2.getId(); - return c1.compareTo(c2); - }; + Comparator comparator = + (Comparator) + (o1, o2) -> { + Integer c1 = o1.getId(); + Integer c2 = o2.getId(); + return c1.compareTo(c2); + }; return_.sort(comparator); } @@ -115,7 +123,15 @@ public void attachVolumes(PriamInstance instance, String mountPath, String devic // TODO Auto-generated method stub } - private PriamInstance makePriamInstance(String app, int id, String instanceID, String hostname, String ip, String rac, Map volumes, String token) { + private PriamInstance makePriamInstance( + String app, + int id, + String instanceID, + String hostname, + String ip, + String rac, + Map volumes, + String token) { Map v = (volumes == null) ? new HashMap<>() : volumes; PriamInstance ins = new PriamInstance(); ins.setApp(app); diff --git a/priam/src/main/java/com/netflix/priam/aws/UpdateCleanupPolicy.java b/priam/src/main/java/com/netflix/priam/aws/UpdateCleanupPolicy.java index 813eba350..56ca053ba 100644 --- a/priam/src/main/java/com/netflix/priam/aws/UpdateCleanupPolicy.java +++ b/priam/src/main/java/com/netflix/priam/aws/UpdateCleanupPolicy.java @@ -19,17 +19,14 @@ import com.google.inject.Inject; import com.google.inject.Singleton; import com.google.inject.name.Named; -import com.netflix.priam.config.IConfiguration; import com.netflix.priam.backup.IBackupFileSystem; +import com.netflix.priam.config.IConfiguration; import com.netflix.priam.scheduler.SimpleTimer; import com.netflix.priam.scheduler.Task; import com.netflix.priam.scheduler.TaskTimer; import com.netflix.priam.utils.RetryableCallable; -/** - * Updates the cleanup policy for the bucket - * - */ +/** Updates the cleanup policy for the bucket */ @Singleton public class UpdateCleanupPolicy extends Task { public static final String JOBNAME = "UpdateCleanupPolicy"; @@ -61,5 +58,4 @@ public String getName() { public static TaskTimer getTimer() { return new SimpleTimer(JOBNAME); } - } diff --git a/priam/src/main/java/com/netflix/priam/aws/UpdateSecuritySettings.java b/priam/src/main/java/com/netflix/priam/aws/UpdateSecuritySettings.java index 3de271198..6a0c4f51b 100644 --- a/priam/src/main/java/com/netflix/priam/aws/UpdateSecuritySettings.java +++ b/priam/src/main/java/com/netflix/priam/aws/UpdateSecuritySettings.java @@ -27,27 +27,22 @@ import com.netflix.priam.scheduler.SimpleTimer; import com.netflix.priam.scheduler.Task; import com.netflix.priam.scheduler.TaskTimer; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import java.util.HashSet; import java.util.List; import java.util.Random; import java.util.Set; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** - * this class will associate an Public IP's with a new instance so they can talk - * across the regions. - * - * Requirement: 1) Nodes in the same region needs to be able to talk to each - * other. 2) Nodes in other regions needs to be able to talk to t`he others in - * the other region. + * this class will associate an Public IP's with a new instance so they can talk across the regions. * - * Assumption: 1) IPriamInstanceFactory will provide the membership... and will - * be visible across the regions 2) IMembership amazon or any other - * implementation which can tell if the instance is part of the group (ASG in - * amazons case). + *

Requirement: 1) Nodes in the same region needs to be able to talk to each other. 2) Nodes in + * other regions needs to be able to talk to t`he others in the other region. * + *

Assumption: 1) IPriamInstanceFactory will provide the membership... and will be visible across + * the regions 2) IMembership amazon or any other implementation which can tell if the instance is + * part of the group (ASG in amazons case). */ @Singleton public class UpdateSecuritySettings extends Task { @@ -60,17 +55,18 @@ public class UpdateSecuritySettings extends Task { private final IPriamInstanceFactory factory; @Inject - //Note: do not parameterized the generic type variable to an implementation as it confuses Guice in the binding. - public UpdateSecuritySettings(IConfiguration config, IMembership membership, IPriamInstanceFactory factory) { + // Note: do not parameterized the generic type variable to an implementation as it confuses + // Guice in the binding. + public UpdateSecuritySettings( + IConfiguration config, IMembership membership, IPriamInstanceFactory factory) { super(config); this.membership = membership; this.factory = factory; } /** - * Seeds nodes execute this at the specifed interval. - * Other nodes run only on startup. - * Seeds in cassandra are the first node in each Availablity Zone. + * Seeds nodes execute this at the specifed interval. Other nodes run only on startup. Seeds in + * cassandra are the first node in each Availablity Zone. */ @Override public void execute() { @@ -84,8 +80,7 @@ public void execute() { List allInstances = factory.getAllIds(config.getAppName()); for (PriamInstance instance : allInstances) { String range = instance.getHostIP() + "/32"; - if (!acls.contains(range)) - add.add(range); + if (!acls.contains(range)) add.add(range); } if (add.size() > 0) { membership.addACL(add, port, port); @@ -103,7 +98,7 @@ public void execute() { List remove = Lists.newArrayList(); for (String acl : acls) if (!currentRanges.contains(acl)) // if not found then remove.... - remove.add(acl); + remove.add(acl); if (remove.size() > 0) { membership.removeACL(remove, port, port); firstTimeUpdated = true; @@ -113,13 +108,13 @@ public void execute() { public static TaskTimer getTimer(InstanceIdentity id) { SimpleTimer return_; if (id.isSeed()) { - logger.info("Seed node. Instance id: {}" - + ", host ip: {}" - + ", host name: {}", - id.getInstance().getInstanceId(), id.getInstance().getHostIP(), id.getInstance().getHostName()); + logger.info( + "Seed node. Instance id: {}" + ", host ip: {}" + ", host name: {}", + id.getInstance().getInstanceId(), + id.getInstance().getHostIP(), + id.getInstance().getHostName()); return_ = new SimpleTimer(JOBNAME, 120 * 1000 + ran.nextInt(120 * 1000)); - } else - return_ = new SimpleTimer(JOBNAME); + } else return_ = new SimpleTimer(JOBNAME); return return_; } diff --git a/priam/src/main/java/com/netflix/priam/aws/auth/EC2RoleAssumptionCredential.java b/priam/src/main/java/com/netflix/priam/aws/auth/EC2RoleAssumptionCredential.java index 279c80ad9..744e0903c 100644 --- a/priam/src/main/java/com/netflix/priam/aws/auth/EC2RoleAssumptionCredential.java +++ b/priam/src/main/java/com/netflix/priam/aws/auth/EC2RoleAssumptionCredential.java @@ -1,16 +1,14 @@ /** * Copyright 2017 Netflix, Inc. - *

- * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and + * + *

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file + * except in compliance with the License. You may obtain a copy of the License at + * + *

http://www.apache.org/licenses/LICENSE-2.0 + * + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing permissions and * limitations under the License. */ package com.netflix.priam.aws.auth; @@ -30,7 +28,8 @@ public class EC2RoleAssumptionCredential implements ICredential { private AWSCredentialsProvider stsSessionCredentialsProvider; @Inject - public EC2RoleAssumptionCredential(ICredential cred, IConfiguration config, InstanceEnvIdentity insEnvIdentity) { + public EC2RoleAssumptionCredential( + ICredential cred, IConfiguration config, InstanceEnvIdentity insEnvIdentity) { this.cred = cred; this.config = config; this.insEnvIdentity = insEnvIdentity; @@ -44,38 +43,45 @@ public AWSCredentialsProvider getAwsCredentialProvider() { String roleArn; /** - * Create the assumed IAM role based on the environment. - * For example, if the current environment is VPC, - * then the assumed role is for EC2 classic, and vice versa. + * Create the assumed IAM role based on the environment. For example, if the + * current environment is VPC, then the assumed role is for EC2 classic, and + * vice versa. */ if (this.insEnvIdentity.isClassic()) { - roleArn = this.config.getClassicEC2RoleAssumptionArn(); // Env is EC2 classic --> IAM assumed role for VPC created + roleArn = this.config.getClassicEC2RoleAssumptionArn(); + // Env is EC2 classic --> IAM assumed role for VPC created } else { - roleArn = this.config.getVpcEC2RoleAssumptionArn(); // Env is VPC --> IAM assumed role for EC2 classic created + roleArn = this.config.getVpcEC2RoleAssumptionArn(); + // Env is VPC --> IAM assumed role for EC2 classic created. } // if (roleArn == null || roleArn.isEmpty()) - throw new NullPointerException("Role ARN is null or empty probably due to missing config entry"); - + throw new NullPointerException( + "Role ARN is null or empty probably due to missing config entry"); /** - * Get handle to an implementation that uses AWS Security Token Service (STS) to create temporary, - * short-lived session with explicit refresh for session/token expiration. + * Get handle to an implementation that uses AWS Security Token Service (STS) to + * create temporary, short-lived session with explicit refresh for session/token + * expiration. */ try { - this.stsSessionCredentialsProvider = new STSAssumeRoleSessionCredentialsProvider(this.cred.getAwsCredentialProvider(), roleArn, AWS_ROLE_ASSUMPTION_SESSION_NAME); + this.stsSessionCredentialsProvider = + new STSAssumeRoleSessionCredentialsProvider( + this.cred.getAwsCredentialProvider(), + roleArn, + AWS_ROLE_ASSUMPTION_SESSION_NAME); } catch (Exception ex) { - throw new IllegalStateException("Exception in getting handle to AWS Security Token Service (STS). Msg: " + ex.getLocalizedMessage(), ex); + throw new IllegalStateException( + "Exception in getting handle to AWS Security Token Service (STS). Msg: " + + ex.getLocalizedMessage(), + ex); } - } - } } return this.stsSessionCredentialsProvider; - } -} \ No newline at end of file +} diff --git a/priam/src/main/java/com/netflix/priam/aws/auth/IS3Credential.java b/priam/src/main/java/com/netflix/priam/aws/auth/IS3Credential.java index 266cf1eb3..86ede4744 100755 --- a/priam/src/main/java/com/netflix/priam/aws/auth/IS3Credential.java +++ b/priam/src/main/java/com/netflix/priam/aws/auth/IS3Credential.java @@ -1,16 +1,14 @@ /** * Copyright 2017 Netflix, Inc. - *

- * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and + * + *

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file + * except in compliance with the License. You may obtain a copy of the License at + * + *

http://www.apache.org/licenses/LICENSE-2.0 + * + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing permissions and * limitations under the License. */ package com.netflix.priam.aws.auth; diff --git a/priam/src/main/java/com/netflix/priam/aws/auth/S3InstanceCredential.java b/priam/src/main/java/com/netflix/priam/aws/auth/S3InstanceCredential.java index 654c13ce2..03b4bf939 100755 --- a/priam/src/main/java/com/netflix/priam/aws/auth/S3InstanceCredential.java +++ b/priam/src/main/java/com/netflix/priam/aws/auth/S3InstanceCredential.java @@ -1,16 +1,14 @@ /** * Copyright 2017 Netflix, Inc. - *

- * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and + * + *

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file + * except in compliance with the License. You may obtain a copy of the License at + * + *

http://www.apache.org/licenses/LICENSE-2.0 + * + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing permissions and * limitations under the License. */ package com.netflix.priam.aws.auth; @@ -39,6 +37,4 @@ public AWSCredentials getCredentials() throws Exception { public AWSCredentialsProvider getAwsCredentialProvider() { return this.credentialsProvider; } - - -} \ No newline at end of file +} diff --git a/priam/src/main/java/com/netflix/priam/aws/auth/S3RoleAssumptionCredential.java b/priam/src/main/java/com/netflix/priam/aws/auth/S3RoleAssumptionCredential.java index cb96b9d1f..21befc17b 100755 --- a/priam/src/main/java/com/netflix/priam/aws/auth/S3RoleAssumptionCredential.java +++ b/priam/src/main/java/com/netflix/priam/aws/auth/S3RoleAssumptionCredential.java @@ -1,16 +1,14 @@ /** * Copyright 2017 Netflix, Inc. - *

- * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and + * + *

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file + * except in compliance with the License. You may obtain a copy of the License at + * + *

http://www.apache.org/licenses/LICENSE-2.0 + * + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing permissions and * limitations under the License. */ package com.netflix.priam.aws.auth; @@ -68,27 +66,37 @@ public AWSCredentialsProvider getAwsCredentialProvider() { synchronized (this) { if (this.stsSessionCredentialsProvider == null) { - final String roleArn = this.config.getAWSRoleAssumptionArn(); //IAM role created for bucket own by account "awsprodbackup" + final String roleArn = this.config.getAWSRoleAssumptionArn(); + // IAM role created for bucket own by account "awsprodbackup" if (roleArn == null || roleArn.isEmpty()) { - logger.warn("Role ARN is null or empty probably due to missing config entry. Falling back to instance level credentials"); + logger.warn( + "Role ARN is null or empty probably due to missing config entry. Falling back to instance level credentials"); this.stsSessionCredentialsProvider = this.cred.getAwsCredentialProvider(); - //throw new NullPointerException("Role ARN is null or empty probably due to missing config entry"); + // throw new NullPointerException("Role ARN is null or empty probably due to + // missing config entry"); } else { - //== Get handle to an implementation that uses AWS Security Token Service (STS) to create temporary, short-lived session with explicit refresh for session/token expiration. + // Get handle to an implementation that uses AWS Security Token Service + // (STS) to create temporary, short-lived session with explicit refresh for + // session/token expiration. try { - this.stsSessionCredentialsProvider = new STSAssumeRoleSessionCredentialsProvider(this.cred.getAwsCredentialProvider(), roleArn, AWS_ROLE_ASSUMPTION_SESSION_NAME); + this.stsSessionCredentialsProvider = + new STSAssumeRoleSessionCredentialsProvider( + this.cred.getAwsCredentialProvider(), + roleArn, + AWS_ROLE_ASSUMPTION_SESSION_NAME); } catch (Exception ex) { - throw new IllegalStateException("Exception in getting handle to AWS Security Token Service (STS). Msg: " + ex.getLocalizedMessage(), ex); + throw new IllegalStateException( + "Exception in getting handle to AWS Security Token Service (STS). Msg: " + + ex.getLocalizedMessage(), + ex); } } - } } } return this.stsSessionCredentialsProvider; } - -} \ No newline at end of file +} diff --git a/priam/src/main/java/com/netflix/priam/backup/AbstractBackup.java b/priam/src/main/java/com/netflix/priam/backup/AbstractBackup.java index b89d1e7a6..f0ecebc3a 100644 --- a/priam/src/main/java/com/netflix/priam/backup/AbstractBackup.java +++ b/priam/src/main/java/com/netflix/priam/backup/AbstractBackup.java @@ -19,24 +19,21 @@ import com.google.common.collect.Lists; import com.google.inject.Inject; import com.google.inject.Provider; -import com.netflix.priam.config.IConfiguration; import com.netflix.priam.backup.AbstractBackupPath.BackupFileType; +import com.netflix.priam.config.IConfiguration; import com.netflix.priam.scheduler.Task; import com.netflix.priam.utils.SystemUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import java.io.File; import java.nio.file.Path; import java.nio.file.Paths; import java.text.ParseException; import java.util.List; import java.util.concurrent.Future; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; -/** - * Abstract Backup class for uploading files to backup location - */ -public abstract class AbstractBackup extends Task{ +/** Abstract Backup class for uploading files to backup location */ +public abstract class AbstractBackup extends Task { private static final Logger logger = LoggerFactory.getLogger(AbstractBackup.class); public static final String INCREMENTAL_BACKUP_FOLDER = "backups"; public static final String SNAPSHOT_FOLDER = "snapshots"; @@ -46,38 +43,39 @@ public abstract class AbstractBackup extends Task{ protected IBackupFileSystem fs; @Inject - public AbstractBackup(IConfiguration config, IFileSystemContext backupFileSystemCtx, - Provider pathFactory) { + public AbstractBackup( + IConfiguration config, + IFileSystemContext backupFileSystemCtx, + Provider pathFactory) { super(config); this.pathFactory = pathFactory; this.fs = backupFileSystemCtx.getFileStrategy(config); } - /** - * A means to override the type of backup strategy chosen via BackupFileSystemContext - */ + /** A means to override the type of backup strategy chosen via BackupFileSystemContext */ protected void setFileSystem(IBackupFileSystem fs) { this.fs = fs; } - private AbstractBackupPath getAbstractBackupPath(final File file, final BackupFileType type) throws ParseException { + private AbstractBackupPath getAbstractBackupPath(final File file, final BackupFileType type) + throws ParseException { final AbstractBackupPath bp = pathFactory.get(); bp.parseLocal(file, type); return bp; } - /** - * Upload files in the specified dir. Does not delete the file in case of - * error. The files are uploaded serially or async based on flag provided. + * Upload files in the specified dir. Does not delete the file in case of error. The files are + * uploaded serially or async based on flag provided. * * @param parent Parent dir - * @param type Type of file (META, SST, SNAP etc) - * @param async Upload the file(s) in async fashion if enabled. + * @param type Type of file (META, SST, SNAP etc) + * @param async Upload the file(s) in async fashion if enabled. * @return List of files that are successfully uploaded as part of backup * @throws Exception when there is failure in uploading files. */ - protected List upload(final File parent, final BackupFileType type, boolean async) throws Exception { + protected List upload( + final File parent, final BackupFileType type, boolean async) throws Exception { final List bps = Lists.newArrayList(); final List> futures = Lists.newArrayList(); @@ -86,35 +84,47 @@ protected List upload(final File parent, final BackupFileTyp AbstractBackupPath bp = getAbstractBackupPath(file, type); if (async) - futures.add(fs.asyncUploadFile(Paths.get(bp.getBackupFile().getAbsolutePath()), Paths.get(bp.getRemotePath()), bp, 10, true)); + futures.add( + fs.asyncUploadFile( + Paths.get(bp.getBackupFile().getAbsolutePath()), + Paths.get(bp.getRemotePath()), + bp, + 10, + true)); else - fs.uploadFile(Paths.get(bp.getBackupFile().getAbsolutePath()), Paths.get(bp.getRemotePath()), bp, 10, true); + fs.uploadFile( + Paths.get(bp.getBackupFile().getAbsolutePath()), + Paths.get(bp.getRemotePath()), + bp, + 10, + true); bps.add(bp); addToRemotePath(bp.getRemotePath()); } } - //Wait for all files to be uploaded. + // Wait for all files to be uploaded. if (async) { for (Future future : futures) - future.get(); //This might throw exception if there is any error + future.get(); // This might throw exception if there is any error } return bps; } - protected final void initiateBackup(String monitoringFolder, BackupRestoreUtil backupRestoreUtil) throws Exception { + protected final void initiateBackup( + String monitoringFolder, BackupRestoreUtil backupRestoreUtil) throws Exception { File dataDir = new File(config.getDataFileLocation()); if (!dataDir.exists()) { - throw new IllegalArgumentException("The configured 'data file location' does not exist: " - + config.getDataFileLocation()); + throw new IllegalArgumentException( + "The configured 'data file location' does not exist: " + + config.getDataFileLocation()); } logger.debug("Scanning for backup in: {}", dataDir.getAbsolutePath()); for (File keyspaceDir : dataDir.listFiles()) { - if (keyspaceDir.isFile()) - continue; + if (keyspaceDir.isFile()) continue; logger.debug("Entering {} keyspace..", keyspaceDir.getName()); @@ -126,47 +136,43 @@ protected final void initiateBackup(String monitoringFolder, BackupRestoreUtil b } String columnFamilyName = columnFamilyDir.getName().split("-")[0]; - if (backupRestoreUtil.isFiltered(keyspaceDir.getName(), columnFamilyDir.getName())) { - //Clean the backup/snapshot directory else files will keep getting accumulated. + if (backupRestoreUtil.isFiltered( + keyspaceDir.getName(), columnFamilyDir.getName())) { + // Clean the backup/snapshot directory else files will keep getting accumulated. SystemUtils.cleanupDir(backupDir.getAbsolutePath(), null); continue; } processColumnFamily(keyspaceDir.getName(), columnFamilyName, backupDir); - - } //end processing all CFs for keyspace - } //end processing keyspaces under the C* data dir - + } // end processing all CFs for keyspace + } // end processing keyspaces under the C* data dir } /** * Process the columnfamily in a given snapshot/backup directory. * - * @param keyspace Name of the keyspace + * @param keyspace Name of the keyspace * @param columnFamily Name of the columnfamily - * @param backupDir Location of the backup/snapshot directory in that columnfamily. + * @param backupDir Location of the backup/snapshot directory in that columnfamily. * @throws Exception throws exception if there is any error in process the directory. */ - protected abstract void processColumnFamily(String keyspace, String columnFamily, File backupDir) throws Exception; + protected abstract void processColumnFamily( + String keyspace, String columnFamily, File backupDir) throws Exception; - - /** - * Filters unwanted keyspaces - */ + /** Filters unwanted keyspaces */ private boolean isValidBackupDir(File keyspaceDir, File backupDir) { - if (!backupDir.isDirectory() && !backupDir.exists()) - return false; + if (!backupDir.isDirectory() && !backupDir.exists()) return false; String keyspaceName = keyspaceDir.getName(); if (BackupRestoreUtil.FILTER_KEYSPACE.contains(keyspaceName)) { - logger.debug("{} is not consider a valid keyspace backup directory, will be bypass.", keyspaceName); + logger.debug( + "{} is not consider a valid keyspace backup directory, will be bypass.", + keyspaceName); return false; } return true; } - /** - * Adds Remote path to the list of Remote Paths - */ + /** Adds Remote path to the list of Remote Paths */ protected abstract void addToRemotePath(String remotePath); } diff --git a/priam/src/main/java/com/netflix/priam/backup/AbstractBackupPath.java b/priam/src/main/java/com/netflix/priam/backup/AbstractBackupPath.java index 89906daef..b1dd6f852 100644 --- a/priam/src/main/java/com/netflix/priam/backup/AbstractBackupPath.java +++ b/priam/src/main/java/com/netflix/priam/backup/AbstractBackupPath.java @@ -17,9 +17,15 @@ package com.netflix.priam.backup; import com.google.inject.ImplementedBy; -import com.netflix.priam.config.IConfiguration; import com.netflix.priam.aws.S3BackupPath; +import com.netflix.priam.config.IConfiguration; import com.netflix.priam.identity.InstanceIdentity; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.io.RandomAccessFile; +import java.text.ParseException; +import java.util.Date; import org.apache.commons.io.IOUtils; import org.apache.commons.lang3.StringUtils; import org.joda.time.DateTime; @@ -28,13 +34,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.File; -import java.io.IOException; -import java.io.InputStream; -import java.io.RandomAccessFile; -import java.text.ParseException; -import java.util.Date; - @ImplementedBy(S3BackupPath.class) public abstract class AbstractBackupPath implements Comparable { private static final Logger logger = LoggerFactory.getLogger(AbstractBackupPath.class); @@ -43,11 +42,16 @@ public abstract class AbstractBackupPath implements ComparableCreated by aagrawal on 8/30/18. */ public abstract class AbstractFileSystem implements IBackupFileSystem, EventGenerator { private static final Logger logger = LoggerFactory.getLogger(AbstractFileSystem.class); - private final CopyOnWriteArrayList> observers = new CopyOnWriteArrayList<>(); + private final CopyOnWriteArrayList> observers = + new CopyOnWriteArrayList<>(); protected final BackupMetrics backupMetrics; private final Set tasksQueued; private final ThreadPoolExecutor fileUploadExecutor; private final ThreadPoolExecutor fileDownloadExecutor; @Inject - public AbstractFileSystem(IConfiguration configuration, BackupMetrics backupMetrics, - BackupNotificationMgr backupNotificationMgr) { + public AbstractFileSystem( + IConfiguration configuration, + BackupMetrics backupMetrics, + BackupNotificationMgr backupNotificationMgr) { this.backupMetrics = backupMetrics; // Add notifications. this.addObserver(backupNotificationMgr); tasksQueued = new ConcurrentHashMap<>().newKeySet(); /* - Note: We are using different queue for upload and download as with Backup V2.0 we might download all the meta - files for "sync" feature which might compete with backups for scheduling. - Also, we may want to have different TIMEOUT for each kind of operation (upload/download) based on our file system choices. - */ - BlockingQueue uploadQueue = new ArrayBlockingQueue<>(configuration.getBackupQueueSize()); - PolledMeter.using(backupMetrics.getRegistry()).withName(backupMetrics.uploadQueueSize).monitorSize(uploadQueue); - this.fileUploadExecutor = new BlockingSubmitThreadPoolExecutor(configuration.getBackupThreads(), uploadQueue, configuration.getUploadTimeout()); - - BlockingQueue downloadQueue = new ArrayBlockingQueue<>(configuration.getDownloadQueueSize()); - PolledMeter.using(backupMetrics.getRegistry()).withName(backupMetrics.downloadQueueSize).monitorSize(downloadQueue); - this.fileDownloadExecutor = new BlockingSubmitThreadPoolExecutor(configuration.getRestoreThreads(), downloadQueue, configuration.getDownloadTimeout()); + Note: We are using different queue for upload and download as with Backup V2.0 we might download all the meta + files for "sync" feature which might compete with backups for scheduling. + Also, we may want to have different TIMEOUT for each kind of operation (upload/download) based on our file system choices. + */ + BlockingQueue uploadQueue = + new ArrayBlockingQueue<>(configuration.getBackupQueueSize()); + PolledMeter.using(backupMetrics.getRegistry()) + .withName(backupMetrics.uploadQueueSize) + .monitorSize(uploadQueue); + this.fileUploadExecutor = + new BlockingSubmitThreadPoolExecutor( + configuration.getBackupThreads(), + uploadQueue, + configuration.getUploadTimeout()); + + BlockingQueue downloadQueue = + new ArrayBlockingQueue<>(configuration.getDownloadQueueSize()); + PolledMeter.using(backupMetrics.getRegistry()) + .withName(backupMetrics.downloadQueueSize) + .monitorSize(downloadQueue); + this.fileDownloadExecutor = + new BlockingSubmitThreadPoolExecutor( + configuration.getRestoreThreads(), + downloadQueue, + configuration.getDownloadTimeout()); } @Override - public Future asyncDownloadFile(final Path remotePath, final Path localPath, final int retry) throws BackupRestoreException, RejectedExecutionException { - return fileDownloadExecutor.submit(() -> { - downloadFile(remotePath, localPath, retry); - return remotePath; - }); + public Future asyncDownloadFile( + final Path remotePath, final Path localPath, final int retry) + throws BackupRestoreException, RejectedExecutionException { + return fileDownloadExecutor.submit( + () -> { + downloadFile(remotePath, localPath, retry); + return remotePath; + }); } @Override - public void downloadFile(final Path remotePath, final Path localPath, final int retry) throws BackupRestoreException { - //TODO: Should we download the file if localPath already exists? - if (remotePath == null) - return; + public void downloadFile(final Path remotePath, final Path localPath, final int retry) + throws BackupRestoreException { + // TODO: Should we download the file if localPath already exists? + if (remotePath == null) return; logger.info("Downloading file: {} to location: {}", remotePath, localPath); try { @@ -94,7 +113,8 @@ public Void retriableCall() throws Exception { return null; } }.call(); - // Note we only downloaded the bytes which are represented on file system (they are compressed and maybe encrypted). + // Note we only downloaded the bytes which are represented on file system (they are + // compressed and maybe encrypted). // File size after decompression or decryption might be more/less. backupMetrics.recordDownloadRate(getFileSize(remotePath)); backupMetrics.incrementValidDownloads(); @@ -106,68 +126,92 @@ public Void retriableCall() throws Exception { } } - protected abstract void downloadFileImpl(final Path remotePath, final Path localPath) throws BackupRestoreException; + protected abstract void downloadFileImpl(final Path remotePath, final Path localPath) + throws BackupRestoreException; @Override - public Future asyncUploadFile(final Path localPath, final Path remotePath, final AbstractBackupPath path, final int retry, final boolean deleteAfterSuccessfulUpload) throws FileNotFoundException, RejectedExecutionException, BackupRestoreException { - return fileUploadExecutor.submit(() -> { - uploadFile(localPath, remotePath, path, retry, deleteAfterSuccessfulUpload); - return localPath; - }); + public Future asyncUploadFile( + final Path localPath, + final Path remotePath, + final AbstractBackupPath path, + final int retry, + final boolean deleteAfterSuccessfulUpload) + throws FileNotFoundException, RejectedExecutionException, BackupRestoreException { + return fileUploadExecutor.submit( + () -> { + uploadFile(localPath, remotePath, path, retry, deleteAfterSuccessfulUpload); + return localPath; + }); } @Override - public void uploadFile(final Path localPath, final Path remotePath, final AbstractBackupPath path, final int retry, final boolean deleteAfterSuccessfulUpload) throws FileNotFoundException, BackupRestoreException { - if (localPath == null || remotePath == null || !localPath.toFile().exists() || localPath.toFile().isDirectory()) - throw new FileNotFoundException("File do not exist or is a directory. localPath: " + localPath + ", remotePath: " + remotePath); + public void uploadFile( + final Path localPath, + final Path remotePath, + final AbstractBackupPath path, + final int retry, + final boolean deleteAfterSuccessfulUpload) + throws FileNotFoundException, BackupRestoreException { + if (localPath == null + || remotePath == null + || !localPath.toFile().exists() + || localPath.toFile().isDirectory()) + throw new FileNotFoundException( + "File do not exist or is a directory. localPath: " + + localPath + + ", remotePath: " + + remotePath); if (tasksQueued.add(localPath)) { logger.info("Uploading file: {} to location: {}", localPath, remotePath); try { notifyEventStart(new BackupEvent(path)); - long uploadedFileSize = new BoundedExponentialRetryCallable(500, 10000, retry) { - @Override - public Long retriableCall() throws Exception { - return uploadFileImpl(localPath, remotePath); - } - }.call(); + long uploadedFileSize = + new BoundedExponentialRetryCallable(500, 10000, retry) { + @Override + public Long retriableCall() throws Exception { + return uploadFileImpl(localPath, remotePath); + } + }.call(); backupMetrics.recordUploadRate(uploadedFileSize); backupMetrics.incrementValidUploads(); path.setCompressedFileSize(uploadedFileSize); notifyEventSuccess(new BackupEvent(path)); - logger.info("Successfully uploaded file: {} to location: {}", localPath, remotePath); + logger.info( + "Successfully uploaded file: {} to location: {}", localPath, remotePath); if (deleteAfterSuccessfulUpload && !FileUtils.deleteQuietly(localPath.toFile())) - logger.warn(String.format("Failed to delete local file %s.", localPath.toFile().getAbsolutePath())); + logger.warn( + String.format( + "Failed to delete local file %s.", + localPath.toFile().getAbsolutePath())); } catch (Exception e) { backupMetrics.incrementInvalidUploads(); notifyEventFailure(new BackupEvent(path)); - logger.error("Error while uploading file: {} to location: {}", localPath, remotePath); + logger.error( + "Error while uploading file: {} to location: {}", localPath, remotePath); throw new BackupRestoreException(e.getMessage()); } finally { - //Remove the task from the list so if we try to upload file ever again, we can. + // Remove the task from the list so if we try to upload file ever again, we can. tasksQueued.remove(localPath); } - } else - logger.info("Already in queue, no-op. File: {}", localPath); - + } else logger.info("Already in queue, no-op. File: {}", localPath); } - protected abstract long uploadFileImpl(final Path localPath, final Path remotePath) throws BackupRestoreException; + protected abstract long uploadFileImpl(final Path localPath, final Path remotePath) + throws BackupRestoreException; @Override public final void addObserver(EventObserver observer) { - if (observer == null) - throw new NullPointerException("observer must not be null."); + if (observer == null) throw new NullPointerException("observer must not be null."); observers.addIfAbsent(observer); } @Override public void removeObserver(EventObserver observer) { - if (observer == null) - throw new NullPointerException("observer must not be null."); + if (observer == null) throw new NullPointerException("observer must not be null."); observers.remove(observer); } diff --git a/priam/src/main/java/com/netflix/priam/backup/BackupFileSystemContext.java b/priam/src/main/java/com/netflix/priam/backup/BackupFileSystemContext.java index 0db3d72df..c8b319258 100755 --- a/priam/src/main/java/com/netflix/priam/backup/BackupFileSystemContext.java +++ b/priam/src/main/java/com/netflix/priam/backup/BackupFileSystemContext.java @@ -1,16 +1,14 @@ /** * Copyright 2017 Netflix, Inc. - *

- * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and + * + *

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file + * except in compliance with the License. You may obtain a copy of the License at + * + *

http://www.apache.org/licenses/LICENSE-2.0 + * + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing permissions and * limitations under the License. */ package com.netflix.priam.backup; @@ -24,7 +22,9 @@ public class BackupFileSystemContext implements IFileSystemContext { private IBackupFileSystem fs = null, encryptedFs = null; @Inject - public BackupFileSystemContext(@Named("backup") IBackupFileSystem fs, @Named("encryptedbackup") IBackupFileSystem encryptedFs) { + public BackupFileSystemContext( + @Named("backup") IBackupFileSystem fs, + @Named("encryptedbackup") IBackupFileSystem encryptedFs) { this.fs = fs; this.encryptedFs = encryptedFs; @@ -41,4 +41,4 @@ public IBackupFileSystem getFileStrategy(IConfiguration config) { return this.encryptedFs; } } -} \ No newline at end of file +} diff --git a/priam/src/main/java/com/netflix/priam/backup/BackupMetadata.java b/priam/src/main/java/com/netflix/priam/backup/BackupMetadata.java index d7eb5aefc..74cf5c587 100644 --- a/priam/src/main/java/com/netflix/priam/backup/BackupMetadata.java +++ b/priam/src/main/java/com/netflix/priam/backup/BackupMetadata.java @@ -1,36 +1,28 @@ /** * Copyright 2017 Netflix, Inc. - *

- * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and + * + *

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file + * except in compliance with the License. You may obtain a copy of the License at + * + *

http://www.apache.org/licenses/LICENSE-2.0 + * + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing permissions and * limitations under the License. */ package com.netflix.priam.backup; - import com.netflix.priam.utils.DateUtil; import com.netflix.priam.utils.GsonJsonSerializer; +import java.io.Serializable; +import java.util.Date; import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.Serializable; -import java.util.Date; - -/** - * POJO to encapsulate the metadata for a snapshot - * Created by aagrawal on 1/31/17. - */ - -final public class BackupMetadata implements Serializable { +/** POJO to encapsulate the metadata for a snapshot Created by aagrawal on 1/31/17. */ +public final class BackupMetadata implements Serializable { private static final Logger logger = LoggerFactory.getLogger(BackupMetadata.class); private String snapshotDate; @@ -41,7 +33,11 @@ final public class BackupMetadata implements Serializable { public BackupMetadata(String token, Date start) throws Exception { if (start == null || token == null || StringUtils.isEmpty(token)) - throw new Exception(String.format("Invalid Input: Token: {} or start date:{} is null or empty.", token, start)); + throw new Exception( + String.format( + "Invalid Input: Token: {} or start date:{} is null or empty.", + token, + start)); this.snapshotDate = DateUtil.formatyyyyMMdd(start); this.token = token; @@ -56,7 +52,9 @@ public boolean equals(Object o) { BackupMetadata that = (BackupMetadata) o; - return this.snapshotDate.equals(that.snapshotDate) && this.token.equals(that.token) && this.start.equals(that.start); + return this.snapshotDate.equals(that.snapshotDate) + && this.token.equals(that.token) + && this.start.equals(that.start); } @Override @@ -99,7 +97,6 @@ public Date getStart() { * * @return completion date of snapshot. */ - public Date getCompleted() { return this.completed; } @@ -150,8 +147,7 @@ public void setSnapshotLocation(String snapshotLocation) { } @Override - public String toString() - { + public String toString() { return GsonJsonSerializer.getGson().toJson(this); } } diff --git a/priam/src/main/java/com/netflix/priam/backup/BackupRestoreException.java b/priam/src/main/java/com/netflix/priam/backup/BackupRestoreException.java index 8f5321154..356ad2d7a 100644 --- a/priam/src/main/java/com/netflix/priam/backup/BackupRestoreException.java +++ b/priam/src/main/java/com/netflix/priam/backup/BackupRestoreException.java @@ -27,5 +27,4 @@ public BackupRestoreException(String message) { public BackupRestoreException(String message, Exception e) { super(message, e); } - } diff --git a/priam/src/main/java/com/netflix/priam/backup/BackupRestoreUtil.java b/priam/src/main/java/com/netflix/priam/backup/BackupRestoreUtil.java index bf150477e..ef21aff8d 100644 --- a/priam/src/main/java/com/netflix/priam/backup/BackupRestoreUtil.java +++ b/priam/src/main/java/com/netflix/priam/backup/BackupRestoreUtil.java @@ -19,16 +19,13 @@ import com.google.common.collect.ImmutableMap; import com.google.inject.Inject; +import java.util.*; +import java.util.regex.Pattern; import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.*; -import java.util.regex.Pattern; - -/** - * Created by aagrawal on 8/14/17. - */ +/** Created by aagrawal on 8/14/17. */ public class BackupRestoreUtil { private static final Logger logger = LoggerFactory.getLogger(BackupRestoreUtil.class); private static final Pattern columnFamilyFilterPattern = Pattern.compile(".\\.."); @@ -36,7 +33,11 @@ public class BackupRestoreUtil { private Map> excludeFilter; public static final List FILTER_KEYSPACE = Collections.singletonList("OpsCenter"); - private static final Map> FILTER_COLUMN_FAMILY = ImmutableMap.of("system", Arrays.asList("local", "peers", "hints", "compactions_in_progress", "LocationInfo")); + private static final Map> FILTER_COLUMN_FAMILY = + ImmutableMap.of( + "system", + Arrays.asList( + "local", "peers", "hints", "compactions_in_progress", "LocationInfo")); @Inject public BackupRestoreUtil(String configIncludeFilter, String configExcludeFilter) { @@ -51,15 +52,16 @@ public BackupRestoreUtil setFilters(String configIncludeFilter, String configExc return this; } + public static final Map> getFilter(String inputFilter) + throws IllegalArgumentException { + if (StringUtils.isEmpty(inputFilter)) return null; - public static final Map> getFilter(String inputFilter) throws IllegalArgumentException { - if (StringUtils.isEmpty(inputFilter)) - return null; - - final Map> columnFamilyFilter = new HashMap<>(); //key: keyspace, value: a list of CFs within the keyspace + final Map> columnFamilyFilter = + new HashMap<>(); // key: keyspace, value: a list of CFs within the keyspace String[] filters = inputFilter.split(","); - for (String cfFilter : filters) { // process filter of form keyspace.* or keyspace.columnfamily + for (String cfFilter : + filters) { // process filter of form keyspace.* or keyspace.columnfamily if (columnFamilyFilterPattern.matcher(cfFilter).find()) { String[] filter = cfFilter.split("\\."); @@ -69,13 +71,15 @@ public static final Map> getFilter(String inputFilter) thro if (columnFamilyName.contains("-")) columnFamilyName = columnFamilyName.substring(0, columnFamilyName.indexOf("-")); - List existingCfs = columnFamilyFilter.getOrDefault(keyspaceName, new ArrayList<>()); - if (!columnFamilyName.equalsIgnoreCase("*")) - existingCfs.add(columnFamilyName); + List existingCfs = + columnFamilyFilter.getOrDefault(keyspaceName, new ArrayList<>()); + if (!columnFamilyName.equalsIgnoreCase("*")) existingCfs.add(columnFamilyName); columnFamilyFilter.put(keyspaceName, existingCfs); } else { - throw new IllegalArgumentException("Column family filter format is not valid. Format needs to be \"keyspace.columnfamily\". Invalid input: " + cfFilter); + throw new IllegalArgumentException( + "Column family filter format is not valid. Format needs to be \"keyspace.columnfamily\". Invalid input: " + + cfFilter); } } return columnFamilyFilter; @@ -84,30 +88,37 @@ public static final Map> getFilter(String inputFilter) thro /** * Returns if provided keyspace and/or columnfamily is filtered for backup or restore. * - * @param keyspace name of the keyspace in consideration + * @param keyspace name of the keyspace in consideration * @param columnFamilyDir name of the columnfamily directory in consideration * @return true if directory should be filter from processing; otherwise, false. */ public final boolean isFiltered(String keyspace, String columnFamilyDir) { - if (StringUtils.isEmpty(keyspace) || StringUtils.isEmpty(columnFamilyDir)) - return false; + if (StringUtils.isEmpty(keyspace) || StringUtils.isEmpty(columnFamilyDir)) return false; String columnFamilyName = columnFamilyDir.split("-")[0]; // column family is in list of global CF filter - if (FILTER_COLUMN_FAMILY.containsKey(keyspace) && FILTER_COLUMN_FAMILY.get(keyspace).contains(columnFamilyName)) - return true; + if (FILTER_COLUMN_FAMILY.containsKey(keyspace) + && FILTER_COLUMN_FAMILY.get(keyspace).contains(columnFamilyName)) return true; if (excludeFilter != null) - if (excludeFilter.containsKey(keyspace) && - (excludeFilter.get(keyspace).isEmpty() || excludeFilter.get(keyspace).contains(columnFamilyName))) { - logger.debug("Skipping: keyspace: {}, CF: {} is part of exclude list.", keyspace, columnFamilyName); + if (excludeFilter.containsKey(keyspace) + && (excludeFilter.get(keyspace).isEmpty() + || excludeFilter.get(keyspace).contains(columnFamilyName))) { + logger.debug( + "Skipping: keyspace: {}, CF: {} is part of exclude list.", + keyspace, + columnFamilyName); return true; } if (includeFilter != null) - if (!(includeFilter.containsKey(keyspace) && - (includeFilter.get(keyspace).isEmpty() || includeFilter.get(keyspace).contains(columnFamilyName)))) { - logger.debug("Skipping: keyspace: {}, CF: {} is not part of include list.", keyspace, columnFamilyName); + if (!(includeFilter.containsKey(keyspace) + && (includeFilter.get(keyspace).isEmpty() + || includeFilter.get(keyspace).contains(columnFamilyName)))) { + logger.debug( + "Skipping: keyspace: {}, CF: {} is not part of include list.", + keyspace, + columnFamilyName); return true; } diff --git a/priam/src/main/java/com/netflix/priam/backup/BackupStatusMgr.java b/priam/src/main/java/com/netflix/priam/backup/BackupStatusMgr.java index bb35a8306..d600a3f4b 100644 --- a/priam/src/main/java/com/netflix/priam/backup/BackupStatusMgr.java +++ b/priam/src/main/java/com/netflix/priam/backup/BackupStatusMgr.java @@ -1,16 +1,14 @@ /** * Copyright 2017 Netflix, Inc. - *

- * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and + * + *

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file + * except in compliance with the License. You may obtain a copy of the License at + * + *

http://www.apache.org/licenses/LICENSE-2.0 + * + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing permissions and * limitations under the License. */ package com.netflix.priam.backup; @@ -20,12 +18,11 @@ import com.netflix.priam.health.InstanceState; import com.netflix.priam.utils.DateUtil; import com.netflix.priam.utils.MaxSizeHashMap; +import java.util.*; import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.*; - /* * A means to manage metadata for various types of backups (snapshots, incrementals) */ @@ -35,17 +32,19 @@ public abstract class BackupStatusMgr implements IBackupStatusMgr { private static final Logger logger = LoggerFactory.getLogger(BackupStatusMgr.class); /** - * Map: Map of completed snapshots represented by its snapshot day (yyyymmdd) - * and a list of snapshots started on that day - * Note: A {@link LinkedList} was chosen for fastest retrieval of latest snapshot. + * Map: Map of completed snapshots represented by its + * snapshot day (yyyymmdd) and a list of snapshots started on that day Note: A {@link + * LinkedList} was chosen for fastest retrieval of latest snapshot. */ Map> backupMetadataMap; + final int capacity; private final InstanceState instanceState; /** * @param capacity Capacity to hold in-memory snapshot status days. - * @param instanceState Status of the instance encapsulating health and other metadata of Priam and Cassandra. + * @param instanceState Status of the instance encapsulating health and other metadata of Priam + * and Cassandra. */ @Inject public BackupStatusMgr(int capacity, InstanceState instanceState) { @@ -73,16 +72,14 @@ public LinkedList locate(Date snapshotDate) { @Override public LinkedList locate(String snapshotDate) { - if (StringUtils.isEmpty(snapshotDate)) - return null; + if (StringUtils.isEmpty(snapshotDate)) return null; // See if in memory - if (backupMetadataMap.containsKey(snapshotDate)) - return backupMetadataMap.get(snapshotDate); + if (backupMetadataMap.containsKey(snapshotDate)) return backupMetadataMap.get(snapshotDate); LinkedList metadataLinkedList = fetch(snapshotDate); - //Save the result in local cache so we don't hit data store/file. + // Save the result in local cache so we don't hit data store/file. backupMetadataMap.put(snapshotDate, metadataLinkedList); return metadataLinkedList; @@ -99,63 +96,65 @@ public void start(BackupMetadata backupMetadata) { metadataLinkedList.addFirst(backupMetadata); backupMetadataMap.put(backupMetadata.getSnapshotDate(), metadataLinkedList); instanceState.setBackupStatus(backupMetadata); - //Save the backupMetaDataMap + // Save the backupMetaDataMap save(backupMetadata); } @Override public void finish(BackupMetadata backupMetadata) { - //validate that it has actually finished. If not, then set the status and current date. + // validate that it has actually finished. If not, then set the status and current date. if (backupMetadata.getStatus() != Status.FINISHED) backupMetadata.setStatus(Status.FINISHED); if (backupMetadata.getCompleted() == null) - backupMetadata.setCompleted(Calendar.getInstance(TimeZone.getTimeZone("GMT")).getTime()); + backupMetadata.setCompleted( + Calendar.getInstance(TimeZone.getTimeZone("GMT")).getTime()); instanceState.setBackupStatus(backupMetadata); - //Retrieve the snapshot metadata and then update the finish date/status. + // Retrieve the snapshot metadata and then update the finish date/status. retrieveAndUpdate(backupMetadata); - //Save the backupMetaDataMap + // Save the backupMetaDataMap save(backupMetadata); - } private void retrieveAndUpdate(final BackupMetadata backupMetadata) { - //Retrieve the snapshot metadata and then update the date/status. + // Retrieve the snapshot metadata and then update the date/status. LinkedList metadataLinkedList = locate(backupMetadata.getSnapshotDate()); if (metadataLinkedList == null || metadataLinkedList.isEmpty()) { - logger.error("No previous backupMetaData found. This should not happen. Creating new to ensure app keeps running."); + logger.error( + "No previous backupMetaData found. This should not happen. Creating new to ensure app keeps running."); metadataLinkedList = new LinkedList<>(); metadataLinkedList.addFirst(backupMetadata); } - metadataLinkedList.forEach(backupMetadata1 -> { - if (backupMetadata1.equals(backupMetadata)) { - backupMetadata1.setCompleted(backupMetadata.getCompleted()); - backupMetadata1.setStatus(backupMetadata.getStatus()); - } - }); + metadataLinkedList.forEach( + backupMetadata1 -> { + if (backupMetadata1.equals(backupMetadata)) { + backupMetadata1.setCompleted(backupMetadata.getCompleted()); + backupMetadata1.setStatus(backupMetadata.getStatus()); + } + }); } @Override public void failed(BackupMetadata backupMetadata) { - //validate that it has actually failed. If not, then set the status and current date. + // validate that it has actually failed. If not, then set the status and current date. if (backupMetadata.getCompleted() == null) - backupMetadata.setCompleted(Calendar.getInstance(TimeZone.getTimeZone("GMT")).getTime()); + backupMetadata.setCompleted( + Calendar.getInstance(TimeZone.getTimeZone("GMT")).getTime()); - //Set this later to ensure the status - if (backupMetadata.getStatus() != Status.FAILED) - backupMetadata.setStatus(Status.FAILED); + // Set this later to ensure the status + if (backupMetadata.getStatus() != Status.FAILED) backupMetadata.setStatus(Status.FAILED); instanceState.setBackupStatus(backupMetadata); - //Retrieve the snapshot metadata and then update the failure date/status. + // Retrieve the snapshot metadata and then update the failure date/status. retrieveAndUpdate(backupMetadata); - //Save the backupMetaDataMap + // Save the backupMetaDataMap save(backupMetadata); } @@ -170,15 +169,20 @@ public void failed(BackupMetadata backupMetadata) { * Implementation on how to retrieve the backup metadata(s) for a given date from store. * * @param snapshotDate Snapshot date to be retrieved from datastore in format of yyyyMMdd - * @return The list of snapshots started on the snapshot day in descending order of snapshot start time. + * @return The list of snapshots started on the snapshot day in descending order of snapshot + * start time. */ protected abstract LinkedList fetch(String snapshotDate); @Override public String toString() { - String sb = "BackupStatusMgr{" + "backupMetadataMap=" + backupMetadataMap + - ", capacity=" + capacity + - '}'; + String sb = + "BackupStatusMgr{" + + "backupMetadataMap=" + + backupMetadataMap + + ", capacity=" + + capacity + + '}'; return sb; } } diff --git a/priam/src/main/java/com/netflix/priam/backup/BackupVerification.java b/priam/src/main/java/com/netflix/priam/backup/BackupVerification.java index 9cb69d438..0c7dbf45d 100644 --- a/priam/src/main/java/com/netflix/priam/backup/BackupVerification.java +++ b/priam/src/main/java/com/netflix/priam/backup/BackupVerification.java @@ -1,16 +1,14 @@ /** * Copyright 2017 Netflix, Inc. - *

- * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and + * + *

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file + * except in compliance with the License. You may obtain a copy of the License at + * + *

http://www.apache.org/licenses/LICENSE-2.0 + * + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing permissions and * limitations under the License. */ package com.netflix.priam.backup; @@ -20,24 +18,23 @@ import com.google.inject.name.Named; import com.netflix.priam.config.IConfiguration; import com.netflix.priam.utils.DateUtil; -import org.apache.commons.collections4.CollectionUtils; -import org.json.simple.parser.JSONParser; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import java.io.FileReader; import java.nio.file.FileSystems; import java.nio.file.Path; import java.nio.file.Paths; import java.util.*; import java.util.stream.Collectors; +import org.apache.commons.collections4.CollectionUtils; +import org.json.simple.parser.JSONParser; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** - * Created by aagrawal on 2/16/17. - * This class validates the backup by doing listing of files in the backup destination and comparing with meta.json by downloading from the location. - * Input: BackupMetadata that needs to be verified. - * Since one backupmetadata can have multiple start time, provide one startTime if interested in verifying one particular backup. - * Leave startTime as null to get the latest snapshot for the provided BackupMetadata. + * Created by aagrawal on 2/16/17. This class validates the backup by doing listing of files in the + * backup destination and comparing with meta.json by downloading from the location. Input: + * BackupMetadata that needs to be verified. Since one backupmetadata can have multiple start time, + * provide one startTime if interested in verifying one particular backup. Leave startTime as null + * to get the latest snapshot for the provided BackupMetadata. */ @Singleton public class BackupVerification { @@ -55,42 +52,51 @@ public class BackupVerification { public BackupVerificationResult verifyBackup(List metadata, Date startTime) { BackupVerificationResult result = new BackupVerificationResult(); - if (metadata == null || metadata.isEmpty()) - return result; + if (metadata == null || metadata.isEmpty()) return result; result.snapshotAvailable = true; // All the dates should be same. result.selectedDate = metadata.get(0).getSnapshotDate(); - List backups = metadata.stream().map(backupMetadata -> - DateUtil.formatyyyyMMddHHmm(backupMetadata.getStart())).collect(Collectors.toList()); + List backups = + metadata.stream() + .map( + backupMetadata -> + DateUtil.formatyyyyMMddHHmm(backupMetadata.getStart())) + .collect(Collectors.toList()); logger.info("Snapshots found for {} : [{}]", result.selectedDate, backups); - //find the latest date (default) or verify if one provided + // find the latest date (default) or verify if one provided Date latestDate = null; for (BackupMetadata backupMetadata : metadata) { if (latestDate == null || latestDate.before(backupMetadata.getStart())) latestDate = backupMetadata.getStart(); - if (startTime != null && - DateUtil.formatyyyyMMddHHmm(backupMetadata.getStart()).equals(DateUtil.formatyyyyMMddHHmm(startTime))) { + if (startTime != null + && DateUtil.formatyyyyMMddHHmm(backupMetadata.getStart()) + .equals(DateUtil.formatyyyyMMddHHmm(startTime))) { latestDate = startTime; break; } } result.snapshotTime = DateUtil.formatyyyyMMddHHmm(latestDate); - logger.info("Latest/Requested snapshot date found: {}, for selected/provided date: {}", result.snapshotTime, result.selectedDate); + logger.info( + "Latest/Requested snapshot date found: {}, for selected/provided date: {}", + result.snapshotTime, + result.selectedDate); - //Get Backup File Iterator + // Get Backup File Iterator String prefix = config.getBackupPrefix(); logger.info("Looking for meta file in the location: {}", prefix); Date strippedMsSnapshotTime = DateUtil.getDate(result.snapshotTime); - Iterator backupfiles = bkpStatusFs.list(prefix, strippedMsSnapshotTime, strippedMsSnapshotTime); - //Return validation fail if backup filesystem listing failed. + Iterator backupfiles = + bkpStatusFs.list(prefix, strippedMsSnapshotTime, strippedMsSnapshotTime); + // Return validation fail if backup filesystem listing failed. if (!backupfiles.hasNext()) { - logger.warn("ERROR: No files available while doing backup filesystem listing. Declaring the verification failed."); + logger.warn( + "ERROR: No files available while doing backup filesystem listing. Declaring the verification failed."); return result; } @@ -101,27 +107,31 @@ public BackupVerificationResult verifyBackup(List metadata, Date while (backupfiles.hasNext()) { AbstractBackupPath path = backupfiles.next(); - if (path.getFileName().equalsIgnoreCase("meta.json")) - metas.add(path); - else - s3Listing.add(path.getRemotePath()); + if (path.getFileName().equalsIgnoreCase("meta.json")) metas.add(path); + else s3Listing.add(path.getRemotePath()); } if (metas.size() == 0) { - logger.error("No meta found for snapshotdate: {}", DateUtil.formatyyyyMMddHHmm(latestDate)); + logger.error( + "No meta found for snapshotdate: {}", DateUtil.formatyyyyMMddHHmm(latestDate)); return result; } result.metaFileFound = true; - //Download meta.json from backup location and uncompress it. + // Download meta.json from backup location and uncompress it. List metaFileList = new ArrayList<>(); try { - Path metaFileLocation = FileSystems.getDefault().getPath(config.getDataFileLocation(), "tmp_meta.json"); + Path metaFileLocation = + FileSystems.getDefault().getPath(config.getDataFileLocation(), "tmp_meta.json"); bkpStatusFs.downloadFile(Paths.get(metas.get(0).getRemotePath()), metaFileLocation, 5); - logger.info("Meta file successfully downloaded to localhost: {}", metaFileLocation.toString()); + logger.info( + "Meta file successfully downloaded to localhost: {}", + metaFileLocation.toString()); JSONParser jsonParser = new JSONParser(); - org.json.simple.JSONArray fileList = (org.json.simple.JSONArray) jsonParser.parse(new FileReader(metaFileLocation.toFile())); + org.json.simple.JSONArray fileList = + (org.json.simple.JSONArray) + jsonParser.parse(new FileReader(metaFileLocation.toFile())); for (Object aFileList : fileList) metaFileList.add(aFileList.toString()); } catch (Exception e) { @@ -130,21 +140,23 @@ public BackupVerificationResult verifyBackup(List metadata, Date } if (metaFileList.isEmpty() && s3Listing.isEmpty()) { - logger.info("Uncommon Scenario: Both meta file and backup filesystem listing is empty. Considering this as success"); + logger.info( + "Uncommon Scenario: Both meta file and backup filesystem listing is empty. Considering this as success"); result.valid = true; return result; } - //Atleast meta file or s3 listing contains some file. + // Atleast meta file or s3 listing contains some file. result.filesInS3Only = new ArrayList<>(s3Listing); result.filesInS3Only.removeAll(metaFileList); result.filesInMetaOnly = new ArrayList<>(metaFileList); result.filesInMetaOnly.removeAll(s3Listing); - result.filesMatched = (ArrayList) CollectionUtils.intersection(metaFileList, s3Listing); + result.filesMatched = + (ArrayList) CollectionUtils.intersection(metaFileList, s3Listing); - //There could be a scenario that backupfilesystem has more files than meta file. e.g. some leftover objects - if (result.filesInMetaOnly.size() == 0) - result.valid = true; + // There could be a scenario that backupfilesystem has more files than meta file. e.g. some + // leftover objects + if (result.filesInMetaOnly.size() == 0) result.valid = true; return result; } diff --git a/priam/src/main/java/com/netflix/priam/backup/BackupVerificationResult.java b/priam/src/main/java/com/netflix/priam/backup/BackupVerificationResult.java index 4f1ce4c97..2bb0dd948 100644 --- a/priam/src/main/java/com/netflix/priam/backup/BackupVerificationResult.java +++ b/priam/src/main/java/com/netflix/priam/backup/BackupVerificationResult.java @@ -1,16 +1,14 @@ /** * Copyright 2017 Netflix, Inc. - *

- * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and + * + *

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file + * except in compliance with the License. You may obtain a copy of the License at + * + *

http://www.apache.org/licenses/LICENSE-2.0 + * + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing permissions and * limitations under the License. */ package com.netflix.priam.backup; @@ -18,10 +16,9 @@ import java.util.List; /** - * Created by aagrawal on 2/16/17. - * This class holds the result from BackupVerification. The default are all null and false. + * Created by aagrawal on 2/16/17. This class holds the result from BackupVerification. The default + * are all null and false. */ - public class BackupVerificationResult { public boolean snapshotAvailable = false; public boolean valid = false; @@ -32,4 +29,4 @@ public class BackupVerificationResult { public List filesInMetaOnly = null; public List filesInS3Only = null; public List filesMatched = null; -} \ No newline at end of file +} diff --git a/priam/src/main/java/com/netflix/priam/backup/CommitLogBackup.java b/priam/src/main/java/com/netflix/priam/backup/CommitLogBackup.java index 849d5700b..6e691501f 100644 --- a/priam/src/main/java/com/netflix/priam/backup/CommitLogBackup.java +++ b/priam/src/main/java/com/netflix/priam/backup/CommitLogBackup.java @@ -1,16 +1,14 @@ /** * Copyright 2017 Netflix, Inc. - *

- * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and + * + *

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file + * except in compliance with the License. You may obtain a copy of the License at + * + *

http://www.apache.org/licenses/LICENSE-2.0 + * + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing permissions and * limitations under the License. */ package com.netflix.priam.backup; @@ -20,17 +18,15 @@ import com.google.inject.Provider; import com.google.inject.name.Named; import com.netflix.priam.backup.AbstractBackupPath.BackupFileType; -import org.apache.commons.lang3.StringUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import java.io.File; import java.nio.file.Paths; import java.util.ArrayList; import java.util.List; +import org.apache.commons.lang3.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; - -//Providing this if we want to use it outside Quart +// Providing this if we want to use it outside Quart public class CommitLogBackup { private static final Logger logger = LoggerFactory.getLogger(CommitLogBackup.class); private final Provider pathFactory; @@ -39,7 +35,8 @@ public class CommitLogBackup { private final IBackupFileSystem fs; @Inject - public CommitLogBackup(Provider pathFactory, @Named("backup") IBackupFileSystem fs) { + public CommitLogBackup( + Provider pathFactory, @Named("backup") IBackupFileSystem fs) { this.pathFactory = pathFactory; this.fs = fs; } @@ -54,7 +51,8 @@ public List upload(String archivedDir, final String snapshot File archivedCommitLogDir = new File(archivedDir); if (!archivedCommitLogDir.exists()) { - throw new IllegalArgumentException("The archived commitlog director does not exist: " + archivedDir); + throw new IllegalArgumentException( + "The archived commitlog director does not exist: " + archivedDir); } if (logger.isDebugEnabled()) { @@ -67,14 +65,21 @@ public List upload(String archivedDir, final String snapshot AbstractBackupPath bp = pathFactory.get(); bp.parseLocal(file, BackupFileType.CL); - if (snapshotName != null) - bp.time = bp.parseDate(snapshotName); + if (snapshotName != null) bp.time = bp.parseDate(snapshotName); - fs.uploadFile(Paths.get(bp.getBackupFile().getAbsolutePath()), Paths.get(bp.getRemotePath()), bp, 10, true); + fs.uploadFile( + Paths.get(bp.getBackupFile().getAbsolutePath()), + Paths.get(bp.getRemotePath()), + bp, + 10, + true); bps.add(bp); addToRemotePath(bp.getRemotePath()); } catch (Exception e) { - logger.error("Failed to upload local file {}. Ignoring to continue with rest of backup.", file, e); + logger.error( + "Failed to upload local file {}. Ignoring to continue with rest of backup.", + file, + e); } } return bps; @@ -101,4 +106,4 @@ public void notifyObservers() { protected void addToRemotePath(String remotePath) { this.clRemotePaths.add(remotePath); } -} \ No newline at end of file +} diff --git a/priam/src/main/java/com/netflix/priam/backup/CommitLogBackupTask.java b/priam/src/main/java/com/netflix/priam/backup/CommitLogBackupTask.java index 1b7e208f2..1f7ba4766 100644 --- a/priam/src/main/java/com/netflix/priam/backup/CommitLogBackupTask.java +++ b/priam/src/main/java/com/netflix/priam/backup/CommitLogBackupTask.java @@ -1,37 +1,32 @@ /** * Copyright 2017 Netflix, Inc. - *

- * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and + * + *

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file + * except in compliance with the License. You may obtain a copy of the License at + * + *

http://www.apache.org/licenses/LICENSE-2.0 + * + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing permissions and * limitations under the License. */ package com.netflix.priam.backup; - import com.google.inject.Inject; import com.google.inject.Provider; import com.google.inject.Singleton; -import com.netflix.priam.config.IConfiguration; import com.netflix.priam.backup.IMessageObserver.BACKUP_MESSAGE_TYPE; +import com.netflix.priam.config.IConfiguration; import com.netflix.priam.scheduler.SimpleTimer; import com.netflix.priam.scheduler.TaskTimer; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import java.io.File; import java.util.ArrayList; import java.util.List; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; - -//Provide this to be run as a Quart job +// Provide this to be run as a Quart job @Singleton public class CommitLogBackupTask extends AbstractBackup { public static final String JOBNAME = "CommitLogBackup"; @@ -41,20 +36,21 @@ public class CommitLogBackupTask extends AbstractBackup { private static final List observers = new ArrayList<>(); private final CommitLogBackup clBackup; - @Inject - public CommitLogBackupTask(IConfiguration config, Provider pathFactory, - CommitLogBackup clBackup, IFileSystemContext backupFileSystemCtx) { + public CommitLogBackupTask( + IConfiguration config, + Provider pathFactory, + CommitLogBackup clBackup, + IFileSystemContext backupFileSystemCtx) { super(config, backupFileSystemCtx, pathFactory); this.clBackup = clBackup; } - @Override public void execute() throws Exception { try { logger.debug("Checking for any archived commitlogs"); - //double-check the permission + // double-check the permission if (config.isBackingUpCommitLogs()) clBackup.upload(config.getCommitLogBackupRestoreFromDirs(), null); } catch (Exception e) { @@ -62,17 +58,15 @@ public void execute() throws Exception { } } - @Override public String getName() { return JOBNAME; } public static TaskTimer getTimer(IConfiguration config) { - return new SimpleTimer(JOBNAME, 60L * 1000); //every 1 min + return new SimpleTimer(JOBNAME, 60L * 1000); // every 1 min } - public static void addObserver(IMessageObserver observer) { observers.add(observer); } @@ -86,14 +80,14 @@ public void notifyObservers() { if (observer != null) { logger.debug("Updating CL observers now ..."); observer.update(BACKUP_MESSAGE_TYPE.COMMITLOG, clRemotePaths); - } else - logger.info("Observer is Null, hence can not notify ..."); + } else logger.info("Observer is Null, hence can not notify ..."); } } @Override - protected void processColumnFamily(String keyspace, String columnFamily, File backupDir) throws Exception { - //Do nothing. + protected void processColumnFamily(String keyspace, String columnFamily, File backupDir) + throws Exception { + // Do nothing. } @Override diff --git a/priam/src/main/java/com/netflix/priam/backup/FileSnapshotStatusMgr.java b/priam/src/main/java/com/netflix/priam/backup/FileSnapshotStatusMgr.java index f01a2320c..0f6c94e4f 100644 --- a/priam/src/main/java/com/netflix/priam/backup/FileSnapshotStatusMgr.java +++ b/priam/src/main/java/com/netflix/priam/backup/FileSnapshotStatusMgr.java @@ -21,17 +21,16 @@ import com.netflix.priam.config.IConfiguration; import com.netflix.priam.health.InstanceState; import com.netflix.priam.utils.MaxSizeHashMap; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import javax.inject.Singleton; import java.io.*; import java.util.LinkedList; import java.util.Map; +import javax.inject.Singleton; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** - * Default implementation for {@link IBackupStatusMgr}. This will save the snapshot status in local file. - * Created by aagrawal on 7/11/17. + * Default implementation for {@link IBackupStatusMgr}. This will save the snapshot status in local + * file. Created by aagrawal on 7/11/17. */ @Singleton public class FileSnapshotStatusMgr extends BackupStatusMgr { @@ -43,57 +42,77 @@ public class FileSnapshotStatusMgr extends BackupStatusMgr { * Constructor to initialize the file based snapshot status manager. * * @param config {@link IConfiguration} of priam to find where file should be saved/read from. - * @param instanceState Status of the instance encapsulating health and other metadata of Priam and Cassandra. + * @param instanceState Status of the instance encapsulating health and other metadata of Priam + * and Cassandra. */ @Inject public FileSnapshotStatusMgr(IConfiguration config, InstanceState instanceState) { - super(IN_MEMORY_SNAPSHOT_CAPACITY, instanceState); //Fetch capacity from properties, if required. + super( + IN_MEMORY_SNAPSHOT_CAPACITY, + instanceState); // Fetch capacity from properties, if required. this.filename = config.getBackupStatusFileLoc(); init(); } private void init() { - //Retrieve entire file and re-populate the list. + // Retrieve entire file and re-populate the list. File snapshotFile = new File(filename); if (!snapshotFile.exists()) { - logger.info("Snapshot status file do not exist on system. Bypassing initilization phase."); + logger.info( + "Snapshot status file do not exist on system. Bypassing initilization phase."); return; } - try (final ObjectInputStream inputStream = new ObjectInputStream(new FileInputStream(snapshotFile))) { + try (final ObjectInputStream inputStream = + new ObjectInputStream(new FileInputStream(snapshotFile))) { backupMetadataMap = (Map>) inputStream.readObject(); - logger.info("Snapshot status of size {} fetched successfully from {}", backupMetadataMap.size(), filename); + logger.info( + "Snapshot status of size {} fetched successfully from {}", + backupMetadataMap.size(), + filename); } catch (IOException e) { - logger.error("Error while trying to fetch snapshot status from {}. Error: {}. If this is first time after upgrading Priam, ignore this.", filename, e.getLocalizedMessage()); + logger.error( + "Error while trying to fetch snapshot status from {}. Error: {}. If this is first time after upgrading Priam, ignore this.", + filename, + e.getLocalizedMessage()); e.printStackTrace(); } catch (Exception e) { - logger.error("Error while trying to fetch snapshot status from {}. Error: {}.", filename, e.getLocalizedMessage()); + logger.error( + "Error while trying to fetch snapshot status from {}. Error: {}.", + filename, + e.getLocalizedMessage()); e.printStackTrace(); } - if (backupMetadataMap == null) - backupMetadataMap = new MaxSizeHashMap<>(capacity); + if (backupMetadataMap == null) backupMetadataMap = new MaxSizeHashMap<>(capacity); } @Override public void save(BackupMetadata backupMetadata) { File snapshotFile = new File(filename); - if (!snapshotFile.exists()) - snapshotFile.mkdirs(); + if (!snapshotFile.exists()) snapshotFile.mkdirs(); - //Will save entire list to file. - try (final ObjectOutputStream out = new ObjectOutputStream(new FileOutputStream(filename))) { + // Will save entire list to file. + try (final ObjectOutputStream out = + new ObjectOutputStream(new FileOutputStream(filename))) { out.writeObject(backupMetadataMap); out.flush(); - logger.info("Snapshot status of size {} is saved to {}", backupMetadataMap.size(), filename); + logger.info( + "Snapshot status of size {} is saved to {}", + backupMetadataMap.size(), + filename); } catch (IOException e) { - logger.error("Error while trying to persist snapshot status to {}. Error: {}", filename, e.getLocalizedMessage()); + logger.error( + "Error while trying to persist snapshot status to {}. Error: {}", + filename, + e.getLocalizedMessage()); } } @Override public LinkedList fetch(String snapshotDate) { - //No need to fetch from local machine as it was read once at start. No point reading again and again. + // No need to fetch from local machine as it was read once at start. No point reading again + // and again. return backupMetadataMap.get(snapshotDate); } } diff --git a/priam/src/main/java/com/netflix/priam/backup/IBackupFileSystem.java b/priam/src/main/java/com/netflix/priam/backup/IBackupFileSystem.java index e4a3d8a6d..cff60e6fe 100644 --- a/priam/src/main/java/com/netflix/priam/backup/IBackupFileSystem.java +++ b/priam/src/main/java/com/netflix/priam/backup/IBackupFileSystem.java @@ -23,88 +23,115 @@ import java.util.concurrent.Future; import java.util.concurrent.RejectedExecutionException; -/** - * Interface representing a backup storage as a file system - */ +/** Interface representing a backup storage as a file system */ public interface IBackupFileSystem { /** * Download the file denoted by remotePath to the local file system denoted by local path. * * @param remotePath fully qualified location of the file on remote file system. - * @param localPath location on the local file sytem where remote file should be downloaded. - * @param retry No. of times to retry to download a file from remote file system. If <1, it will try to download file exactly once. - * @throws BackupRestoreException if file is not available, downloadable or any other error from remote file system. + * @param localPath location on the local file sytem where remote file should be downloaded. + * @param retry No. of times to retry to download a file from remote file system. If <1, it + * will try to download file exactly once. + * @throws BackupRestoreException if file is not available, downloadable or any other error from + * remote file system. */ void downloadFile(Path remotePath, Path localPath, int retry) throws BackupRestoreException; /** - * Download the file denoted by remotePath in an async fashion to the local file system denoted by local path. + * Download the file denoted by remotePath in an async fashion to the local file system denoted + * by local path. * * @param remotePath fully qualified location of the file on remote file system. - * @param localPath location on the local file sytem where remote file should be downloaded. - * @param retry No. of times to retry to download a file from remote file system. If <1, it will try to download file exactly once. + * @param localPath location on the local file sytem where remote file should be downloaded. + * @param retry No. of times to retry to download a file from remote file system. If <1, it + * will try to download file exactly once. * @return The future of the async job to monitor the progress of the job. - * @throws BackupRestoreException if file is not available, downloadable or any other error from remote file system. - * @throws RejectedExecutionException if the queue is full and TIMEOUT is reached while trying to add the work to the queue. + * @throws BackupRestoreException if file is not available, downloadable or any other error from + * remote file system. + * @throws RejectedExecutionException if the queue is full and TIMEOUT is reached while trying + * to add the work to the queue. */ - Future asyncDownloadFile(final Path remotePath, final Path localPath, final int retry) throws BackupRestoreException, RejectedExecutionException; - + Future asyncDownloadFile(final Path remotePath, final Path localPath, final int retry) + throws BackupRestoreException, RejectedExecutionException; /** - * Upload the local file denoted by localPath to the remote file system at location denoted by remotePath. - * De-duping of the file to upload will always be done by comparing the files-in-progress to be uploaded. - * This may result in this particular request to not to be executed e.g. if any other thread has given the same file - * to upload and that file is in internal queue. - * Note that de-duping is best effort and is not always guaranteed as we try to avoid lock on read/write of the files-in-progress. + * Upload the local file denoted by localPath to the remote file system at location denoted by + * remotePath. De-duping of the file to upload will always be done by comparing the + * files-in-progress to be uploaded. This may result in this particular request to not to be + * executed e.g. if any other thread has given the same file to upload and that file is in + * internal queue. Note that de-duping is best effort and is not always guaranteed as we try to + * avoid lock on read/write of the files-in-progress. * - * @param localPath Path of the local file that needs to be uploaded. - * @param remotePath Fully qualified path on the remote file system where file should be uploaded. - * @param path AbstractBackupPath to be used to send backup notifications only. - * @param retry No of times to retry to upload a file. If <1, it will try to upload file exactly once. - * @param deleteAfterSuccessfulUpload If true, delete the file denoted by localPath after it is successfully uploaded to the filesystem. If there is any failure, file will not be deleted. - * @throws BackupRestoreException in case of failure to upload for any reason including file not readable or remote file system errors. - * @throws FileNotFoundException If a file as denoted by localPath is not available or is a directory. + * @param localPath Path of the local file that needs to be uploaded. + * @param remotePath Fully qualified path on the remote file system where file should be + * uploaded. + * @param path AbstractBackupPath to be used to send backup notifications only. + * @param retry No of times to retry to upload a file. If <1, it will try to upload file + * exactly once. + * @param deleteAfterSuccessfulUpload If true, delete the file denoted by localPath after it is + * successfully uploaded to the filesystem. If there is any failure, file will not be + * deleted. + * @throws BackupRestoreException in case of failure to upload for any reason including file not + * readable or remote file system errors. + * @throws FileNotFoundException If a file as denoted by localPath is not available or is a + * directory. */ - void uploadFile(Path localPath, Path remotePath, AbstractBackupPath path, int retry, boolean deleteAfterSuccessfulUpload) throws FileNotFoundException, BackupRestoreException; + void uploadFile( + Path localPath, + Path remotePath, + AbstractBackupPath path, + int retry, + boolean deleteAfterSuccessfulUpload) + throws FileNotFoundException, BackupRestoreException; /** - * Upload the local file denoted by localPath in async fashion to the remote file system at location denoted by remotePath. + * Upload the local file denoted by localPath in async fashion to the remote file system at + * location denoted by remotePath. * - * @param localPath Path of the local file that needs to be uploaded. - * @param remotePath Fully qualified path on the remote file system where file should be uploaded. - * @param path AbstractBackupPath to be used to send backup notifications only. - * @param retry No of times to retry to upload a file. If <1, it will try to upload file exactly once. - * @param deleteAfterSuccessfulUpload If true, delete the file denoted by localPath after it is successfully uploaded to the filesystem. If there is any failure, file will not be deleted. - * @return The future of the async job to monitor the progress of the job. This will be null if file was de-duped for upload. - * @throws BackupRestoreException in case of failure to upload for any reason including file not readable or remote file system errors. - * @throws FileNotFoundException If a file as denoted by localPath is not available or is a directory. - * @throws RejectedExecutionException if the queue is full and TIMEOUT is reached while trying to add the work to the queue. + * @param localPath Path of the local file that needs to be uploaded. + * @param remotePath Fully qualified path on the remote file system where file should be + * uploaded. + * @param path AbstractBackupPath to be used to send backup notifications only. + * @param retry No of times to retry to upload a file. If <1, it will try to upload file + * exactly once. + * @param deleteAfterSuccessfulUpload If true, delete the file denoted by localPath after it is + * successfully uploaded to the filesystem. If there is any failure, file will not be + * deleted. + * @return The future of the async job to monitor the progress of the job. This will be null if + * file was de-duped for upload. + * @throws BackupRestoreException in case of failure to upload for any reason including file not + * readable or remote file system errors. + * @throws FileNotFoundException If a file as denoted by localPath is not available or is a + * directory. + * @throws RejectedExecutionException if the queue is full and TIMEOUT is reached while trying + * to add the work to the queue. */ - Future asyncUploadFile(final Path localPath, final Path remotePath, final AbstractBackupPath path, final int retry, final boolean deleteAfterSuccessfulUpload) throws FileNotFoundException, RejectedExecutionException, BackupRestoreException; + Future asyncUploadFile( + final Path localPath, + final Path remotePath, + final AbstractBackupPath path, + final int retry, + final boolean deleteAfterSuccessfulUpload) + throws FileNotFoundException, RejectedExecutionException, BackupRestoreException; /** * List all files in the backup location for the specified time range. * - * @param path This is used as the `prefix` for listing files in the filesystem. All the files that start with this prefix will be returned. + * @param path This is used as the `prefix` for listing files in the filesystem. All the files + * that start with this prefix will be returned. * @param start Start date of the file upload. - * @param till End date of the file upload. + * @param till End date of the file upload. * @return Iterator of the AbstractBackupPath matching the criteria. */ Iterator list(String path, Date start, Date till); - /** - * Get a list of prefixes for the cluster available in backup for the specified date - */ + /** Get a list of prefixes for the cluster available in backup for the specified date */ Iterator listPrefixes(Date date); - /** - * Runs cleanup or set retention - */ + /** Runs cleanup or set retention */ void cleanup(); - /** - * Give the file system a chance to terminate any thread pools, etc. - */ + /** Give the file system a chance to terminate any thread pools, etc. */ void shutdown(); /** @@ -112,7 +139,8 @@ public interface IBackupFileSystem { * * @param remotePath Location of the object on the remote file system. * @return size of the object on the remote filesystem. - * @throws BackupRestoreException in case of failure to read object denoted by remotePath or any other error. + * @throws BackupRestoreException in case of failure to read object denoted by remotePath or any + * other error. */ long getFileSize(Path remotePath) throws BackupRestoreException; diff --git a/priam/src/main/java/com/netflix/priam/backup/IBackupStatusMgr.java b/priam/src/main/java/com/netflix/priam/backup/IBackupStatusMgr.java index 97054fe35..11f016516 100644 --- a/priam/src/main/java/com/netflix/priam/backup/IBackupStatusMgr.java +++ b/priam/src/main/java/com/netflix/priam/backup/IBackupStatusMgr.java @@ -1,31 +1,27 @@ /** * Copyright 2017 Netflix, Inc. - *

- * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and + * + *

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file + * except in compliance with the License. You may obtain a copy of the License at + * + *

http://www.apache.org/licenses/LICENSE-2.0 + * + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing permissions and * limitations under the License. */ package com.netflix.priam.backup; import com.google.inject.ImplementedBy; - import java.util.Date; import java.util.LinkedList; import java.util.List; import java.util.Map; /** - * This will store the status of snapshots as they start, fail or finish. By default they will save the snapshot - * status of last 60 days on instance. - * Created by aagrawal on 1/30/17. + * This will store the status of snapshots as they start, fail or finish. By default they will save + * the snapshot status of last 60 days on instance. Created by aagrawal on 1/30/17. */ @ImplementedBy(FileSnapshotStatusMgr.class) public interface IBackupStatusMgr { @@ -40,20 +36,23 @@ public interface IBackupStatusMgr { /** * Return the list of snapshot executed on provided day or null if not present. * - * @param snapshotDate date on which snapshot was started in the format of yyyyMMdd or yyyyMMddHHmm. + * @param snapshotDate date on which snapshot was started in the format of yyyyMMdd or + * yyyyMMddHHmm. * @return List of snapshots started on that day in descending order of snapshot start time. */ List locate(String snapshotDate); /** - * Save the status of snapshot BackupMetadata which started in-memory and other implementations, if any. + * Save the status of snapshot BackupMetadata which started in-memory and other implementations, + * if any. * * @param backupMetadata backupmetadata that started */ void start(BackupMetadata backupMetadata); /** - * Save the status of successfully finished snapshot BackupMetadata in-memory and other implementations, if any. + * Save the status of successfully finished snapshot BackupMetadata in-memory and other + * implementations, if any. * * @param backupMetadata backupmetadata that finished successfully */ @@ -76,8 +75,9 @@ public interface IBackupStatusMgr { /** * Get the entire map of snapshot status hold in-memory * - * @return The map of snapshot status in-memory in format. - * Key is snapshot day in format of yyyyMMdd (start date of snapshot) with a list of snapshots in the descending order of snapshot start time. + * @return The map of snapshot status in-memory in format. Key is snapshot day in format of + * yyyyMMdd (start date of snapshot) with a list of snapshots in the descending order of + * snapshot start time. */ Map> getAllSnapshotStatus(); } diff --git a/priam/src/main/java/com/netflix/priam/backup/IFileSystemContext.java b/priam/src/main/java/com/netflix/priam/backup/IFileSystemContext.java index 8c9d9f961..380a98103 100755 --- a/priam/src/main/java/com/netflix/priam/backup/IFileSystemContext.java +++ b/priam/src/main/java/com/netflix/priam/backup/IFileSystemContext.java @@ -1,16 +1,14 @@ /** * Copyright 2017 Netflix, Inc. - *

- * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and + * + *

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file + * except in compliance with the License. You may obtain a copy of the License at + * + *

http://www.apache.org/licenses/LICENSE-2.0 + * + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing permissions and * limitations under the License. */ package com.netflix.priam.backup; diff --git a/priam/src/main/java/com/netflix/priam/backup/IIncrementalBackup.java b/priam/src/main/java/com/netflix/priam/backup/IIncrementalBackup.java index 3dd48caf6..62c1d5e09 100644 --- a/priam/src/main/java/com/netflix/priam/backup/IIncrementalBackup.java +++ b/priam/src/main/java/com/netflix/priam/backup/IIncrementalBackup.java @@ -1,16 +1,14 @@ /** * Copyright 2017 Netflix, Inc. - *

- * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and + * + *

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file + * except in compliance with the License. You may obtain a copy of the License at + * + *

http://www.apache.org/licenses/LICENSE-2.0 + * + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing permissions and * limitations under the License. */ package com.netflix.priam.backup; @@ -26,5 +24,4 @@ public interface IIncrementalBackup { long getNumPendingFiles(); String getJobName(); - -} \ No newline at end of file +} diff --git a/priam/src/main/java/com/netflix/priam/backup/IMessageObserver.java b/priam/src/main/java/com/netflix/priam/backup/IMessageObserver.java index 62da1a744..58af87dca 100644 --- a/priam/src/main/java/com/netflix/priam/backup/IMessageObserver.java +++ b/priam/src/main/java/com/netflix/priam/backup/IMessageObserver.java @@ -18,19 +18,38 @@ import java.util.List; - public interface IMessageObserver { - enum BACKUP_MESSAGE_TYPE {SNAPSHOT, INCREMENTAL, COMMITLOG, META} - - enum RESTORE_MESSAGE_TYPE {SNAPSHOT, INCREMENTAL, COMMITLOG, META} - - enum RESTORE_MESSAGE_STATUS {UPLOADED, DOWNLOADED, STREAMED} + enum BACKUP_MESSAGE_TYPE { + SNAPSHOT, + INCREMENTAL, + COMMITLOG, + META + } + + enum RESTORE_MESSAGE_TYPE { + SNAPSHOT, + INCREMENTAL, + COMMITLOG, + META + } + + enum RESTORE_MESSAGE_STATUS { + UPLOADED, + DOWNLOADED, + STREAMED + } void update(BACKUP_MESSAGE_TYPE bkpMsgType, List remotePathNames); - void update(RESTORE_MESSAGE_TYPE rstMsgType, List remotePathNames, RESTORE_MESSAGE_STATUS rstMsgStatus); - - void update(RESTORE_MESSAGE_TYPE rstMsgType, String remotePath, String fileDiskPath, RESTORE_MESSAGE_STATUS rstMsgStatus); + void update( + RESTORE_MESSAGE_TYPE rstMsgType, + List remotePathNames, + RESTORE_MESSAGE_STATUS rstMsgStatus); + void update( + RESTORE_MESSAGE_TYPE rstMsgType, + String remotePath, + String fileDiskPath, + RESTORE_MESSAGE_STATUS rstMsgStatus); } diff --git a/priam/src/main/java/com/netflix/priam/backup/IncrementalBackup.java b/priam/src/main/java/com/netflix/priam/backup/IncrementalBackup.java index 398749822..09a139da9 100644 --- a/priam/src/main/java/com/netflix/priam/backup/IncrementalBackup.java +++ b/priam/src/main/java/com/netflix/priam/backup/IncrementalBackup.java @@ -19,17 +19,16 @@ import com.google.inject.Inject; import com.google.inject.Provider; import com.google.inject.Singleton; -import com.netflix.priam.config.IConfiguration; import com.netflix.priam.backup.AbstractBackupPath.BackupFileType; import com.netflix.priam.backup.IMessageObserver.BACKUP_MESSAGE_TYPE; +import com.netflix.priam.config.IConfiguration; import com.netflix.priam.scheduler.SimpleTimer; import com.netflix.priam.scheduler.TaskTimer; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import java.io.File; import java.util.ArrayList; import java.util.List; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /* * Incremental/SSTable backup @@ -44,16 +43,23 @@ public class IncrementalBackup extends AbstractBackup implements IIncrementalBac private static final List observers = new ArrayList<>(); @Inject - public IncrementalBackup(IConfiguration config, Provider pathFactory, IFileSystemContext backupFileSystemCtx - , IncrementalMetaData metaData) { + public IncrementalBackup( + IConfiguration config, + Provider pathFactory, + IFileSystemContext backupFileSystemCtx, + IncrementalMetaData metaData) { super(config, backupFileSystemCtx, pathFactory); - this.metaData = metaData; //a means to upload audit trail (via meta_cf_yyyymmddhhmm.json) of files successfully uploaded) - backupRestoreUtil = new BackupRestoreUtil(config.getIncrementalIncludeCFList(), config.getIncrementalExcludeCFList()); + // a means to upload audit trail (via meta_cf_yyyymmddhhmm.json) of files successfully + // uploaded) + this.metaData = metaData; + backupRestoreUtil = + new BackupRestoreUtil( + config.getIncrementalIncludeCFList(), config.getIncrementalExcludeCFList()); } @Override public void execute() throws Exception { - //Clearing remotePath List + // Clearing remotePath List incrementalRemotePaths.clear(); initiateBackup(INCREMENTAL_BACKUP_FOLDER, backupRestoreUtil); if (incrementalRemotePaths.size() > 0) { @@ -61,10 +67,7 @@ public void execute() throws Exception { } } - - /** - * Run every 10 Sec - */ + /** Run every 10 Sec */ public static TaskTimer getTimer() { return new SimpleTimer(JOBNAME, 10L * 1000); } @@ -87,24 +90,26 @@ private void notifyObservers() { if (observer != null) { logger.debug("Updating incremental observers now ..."); observer.update(BACKUP_MESSAGE_TYPE.INCREMENTAL, incrementalRemotePaths); - } else - logger.info("Observer is Null, hence can not notify ..."); + } else logger.info("Observer is Null, hence can not notify ..."); } } @Override - protected void processColumnFamily(String keyspace, String columnFamily, File backupDir) throws Exception { - List uploadedFiles = upload(backupDir, BackupFileType.SST, config.enableAsyncIncremental()); + protected void processColumnFamily(String keyspace, String columnFamily, File backupDir) + throws Exception { + List uploadedFiles = + upload(backupDir, BackupFileType.SST, config.enableAsyncIncremental()); if (!uploadedFiles.isEmpty()) { - String incrementalUploadTime = AbstractBackupPath.formatDate(uploadedFiles.get(0).getTime()); //format of yyyymmddhhmm (e.g. 201505060901) + // format of yyyymmddhhmm (e.g. 201505060901) + String incrementalUploadTime = + AbstractBackupPath.formatDate(uploadedFiles.get(0).getTime()); String metaFileName = "meta_" + backupDir.getParent() + "_" + incrementalUploadTime; logger.info("Uploading meta file for incremental backup: {}", metaFileName); this.metaData.setMetaFileName(metaFileName); this.metaData.set(uploadedFiles, incrementalUploadTime); logger.info("Uploaded meta file for incremental backup: {}", metaFileName); } - } @Override @@ -121,5 +126,4 @@ public long getNumPendingFiles() { public String getJobName() { return JOBNAME; } - -} \ No newline at end of file +} diff --git a/priam/src/main/java/com/netflix/priam/backup/IncrementalMetaData.java b/priam/src/main/java/com/netflix/priam/backup/IncrementalMetaData.java index ed6175929..99aaf31c2 100644 --- a/priam/src/main/java/com/netflix/priam/backup/IncrementalMetaData.java +++ b/priam/src/main/java/com/netflix/priam/backup/IncrementalMetaData.java @@ -1,16 +1,14 @@ /** * Copyright 2017 Netflix, Inc. - *

- * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and + * + *

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file + * except in compliance with the License. You may obtain a copy of the License at + * + *

http://www.apache.org/licenses/LICENSE-2.0 + * + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing permissions and * limitations under the License. */ package com.netflix.priam.backup; @@ -18,17 +16,19 @@ import com.google.inject.Inject; import com.google.inject.Provider; import com.netflix.priam.config.IConfiguration; -import org.apache.commons.io.FileUtils; - import java.io.File; import java.io.IOException; +import org.apache.commons.io.FileUtils; public class IncrementalMetaData extends MetaData { - private String metaFileName = null; //format meta_cf_time (e.g. + private String metaFileName = null; // format meta_cf_time (e.g. @Inject - public IncrementalMetaData(IConfiguration config, Provider pathFactory, IFileSystemContext backupFileSystemCtx) { + public IncrementalMetaData( + IConfiguration config, + Provider pathFactory, + IFileSystemContext backupFileSystemCtx) { super(pathFactory, backupFileSystemCtx, config); } @@ -50,15 +50,14 @@ public File createTmpMetaFile() throws IOException { destFile = new File(metafile.getParent(), this.metaFileName + ".json"); } - if (destFile.exists()) - destFile.delete(); + if (destFile.exists()) destFile.delete(); try { FileUtils.moveFile(metafile, destFile); } finally { - if (metafile != null && metafile.exists()) { //clean up resource + if (metafile != null && metafile.exists()) { // clean up resource FileUtils.deleteQuietly(metafile); } } diff --git a/priam/src/main/java/com/netflix/priam/backup/MetaData.java b/priam/src/main/java/com/netflix/priam/backup/MetaData.java index 6afa5d8fc..210913212 100644 --- a/priam/src/main/java/com/netflix/priam/backup/MetaData.java +++ b/priam/src/main/java/com/netflix/priam/backup/MetaData.java @@ -19,15 +19,9 @@ import com.google.common.collect.Lists; import com.google.inject.Inject; import com.google.inject.Provider; -import com.netflix.priam.config.IConfiguration; import com.netflix.priam.backup.AbstractBackupPath.BackupFileType; import com.netflix.priam.backup.IMessageObserver.BACKUP_MESSAGE_TYPE; -import org.apache.commons.io.FileUtils; -import org.json.simple.JSONArray; -import org.json.simple.parser.JSONParser; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - +import com.netflix.priam.config.IConfiguration; import java.io.File; import java.io.FileReader; import java.io.FileWriter; @@ -36,10 +30,15 @@ import java.text.ParseException; import java.util.ArrayList; import java.util.List; +import org.apache.commons.io.FileUtils; +import org.json.simple.JSONArray; +import org.json.simple.parser.JSONParser; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** - * Class to create a meta data file with a list of snapshot files. Also list the - * contents of a meta data file. + * Class to create a meta data file with a list of snapshot files. Also list the contents of a meta + * data file. */ public class MetaData { private static final Logger logger = LoggerFactory.getLogger(MetaData.class); @@ -49,23 +48,30 @@ public class MetaData { private final IBackupFileSystem fs; @Inject - public MetaData(Provider pathFactory, IFileSystemContext backupFileSystemCtx, IConfiguration config) + public MetaData( + Provider pathFactory, + IFileSystemContext backupFileSystemCtx, + IConfiguration config) { - { this.pathFactory = pathFactory; this.fs = backupFileSystemCtx.getFileStrategy(config); } - public AbstractBackupPath set(List bps, String snapshotName) throws Exception { + public AbstractBackupPath set(List bps, String snapshotName) + throws Exception { File metafile = createTmpMetaFile(); try (FileWriter fr = new FileWriter(metafile)) { JSONArray jsonObj = new JSONArray(); - for (AbstractBackupPath filePath : bps) - jsonObj.add(filePath.getRemotePath()); + for (AbstractBackupPath filePath : bps) jsonObj.add(filePath.getRemotePath()); fr.write(jsonObj.toJSONString()); } AbstractBackupPath backupfile = decorateMetaJson(metafile, snapshotName); - fs.uploadFile(Paths.get(backupfile.getBackupFile().getAbsolutePath()), Paths.get(backupfile.getRemotePath()), backupfile, 10, true); + fs.uploadFile( + Paths.get(backupfile.getBackupFile().getAbsolutePath()), + Paths.get(backupfile.getRemotePath()), + backupfile, + 10, + true); addToRemotePath(backupfile.getRemotePath()); if (metaRemotePaths.size() > 0) { notifyObservers(); @@ -77,37 +83,38 @@ public AbstractBackupPath set(List bps, String snapshotName) /* From the meta.json to be created, populate its meta data for the backup file. */ - public AbstractBackupPath decorateMetaJson(File metafile, String snapshotName) throws ParseException { + public AbstractBackupPath decorateMetaJson(File metafile, String snapshotName) + throws ParseException { AbstractBackupPath backupfile = pathFactory.get(); backupfile.parseLocal(metafile, BackupFileType.META); backupfile.time = backupfile.parseDate(snapshotName); return backupfile; } - /* - * Determines the existence of the backup meta file. This meta file could be snapshot (meta.json) or + * Determines the existence of the backup meta file. This meta file could be snapshot (meta.json) or * incrementals (meta_keyspace_cf..json). - * + * * @param backup meta file to search * @return true if backup meta file exist, false otherwise. */ public Boolean doesExist(final AbstractBackupPath meta) { try { - fs.downloadFile(Paths.get(meta.getRemotePath()), Paths.get(meta.newRestoreFile().getAbsolutePath()), 5); //download actual file to disk + fs.downloadFile( + Paths.get(meta.getRemotePath()), + Paths.get(meta.newRestoreFile().getAbsolutePath()), + 5); // download actual file to disk } catch (Exception e) { logger.error("Error downloading the Meta data try with a different date...", e); } return meta.newRestoreFile().exists(); - } public File createTmpMetaFile() throws IOException { File metafile = File.createTempFile("meta", ".json"); File destFile = new File(metafile.getParent(), "meta.json"); - if (destFile.exists()) - destFile.delete(); + if (destFile.exists()) destFile.delete(); FileUtils.moveFile(metafile, destFile); return destFile; } @@ -125,8 +132,7 @@ private void notifyObservers() { if (observer != null) { logger.debug("Updating snapshot observers now ..."); observer.update(BACKUP_MESSAGE_TYPE.META, metaRemotePaths); - } else - logger.info("Observer is Null, hence can not notify ..."); + } else logger.info("Observer is Null, hence can not notify ..."); } } @@ -145,10 +151,18 @@ public List toJson(File input) { } } catch (Exception ex) { - throw new RuntimeException("Error transforming file " + input.getAbsolutePath() + " to JSON format. Msg:" + ex.getLocalizedMessage(), ex); + throw new RuntimeException( + "Error transforming file " + + input.getAbsolutePath() + + " to JSON format. Msg:" + + ex.getLocalizedMessage(), + ex); } - logger.debug("Transformed file {} to JSON. Number of JSON elements: {}", input.getAbsolutePath(), files.size()); + logger.debug( + "Transformed file {} to JSON. Number of JSON elements: {}", + input.getAbsolutePath(), + files.size()); return files; } } diff --git a/priam/src/main/java/com/netflix/priam/backup/RangeReadInputStream.java b/priam/src/main/java/com/netflix/priam/backup/RangeReadInputStream.java index 76f888573..aa1fafa03 100644 --- a/priam/src/main/java/com/netflix/priam/backup/RangeReadInputStream.java +++ b/priam/src/main/java/com/netflix/priam/backup/RangeReadInputStream.java @@ -20,16 +20,15 @@ import com.amazonaws.services.s3.model.GetObjectRequest; import com.amazonaws.services.s3.model.S3ObjectInputStream; import com.netflix.priam.utils.RetryableCallable; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import java.io.IOException; import java.io.InputStream; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** - * An implementation of InputStream that will request explicit byte ranges of the target file. - * This will make it easier to retry a failed read - which is important if we don't want to \ - * throw away a 100Gb file and restart after reading 99Gb and failing. + * An implementation of InputStream that will request explicit byte ranges of the target file. This + * will make it easier to retry a failed read - which is important if we don't want to \ throw away + * a 100Gb file and restart after reading 99Gb and failing. */ public class RangeReadInputStream extends InputStream { private static final Logger logger = LoggerFactory.getLogger(RangeReadInputStream.class); @@ -40,7 +39,8 @@ public class RangeReadInputStream extends InputStream { private final String remotePath; private long offset; - public RangeReadInputStream(AmazonS3 s3Client, String bucketName, long fileSize, String remotePath) { + public RangeReadInputStream( + AmazonS3 s3Client, String bucketName, long fileSize, String remotePath) { this.s3Client = s3Client; this.bucketName = bucketName; this.fileSize = fileSize; @@ -48,41 +48,43 @@ public RangeReadInputStream(AmazonS3 s3Client, String bucketName, long fileSize, } public int read(final byte b[], final int off, final int len) throws IOException { - if (fileSize > 0 && offset >= fileSize) - return -1; + if (fileSize > 0 && offset >= fileSize) return -1; final long firstByte = offset; long curEndByte = firstByte + len; curEndByte = curEndByte <= fileSize ? curEndByte : fileSize; - //need to subtract one as the call to getRange is inclusive - //meaning if you want to download the first 10 bytes of a file, request bytes 0..9 + // need to subtract one as the call to getRange is inclusive + // meaning if you want to download the first 10 bytes of a file, request bytes 0..9 final long endByte = curEndByte - 1; try { - Integer cnt = new RetryableCallable() { - public Integer retriableCall() throws IOException { - GetObjectRequest req = new GetObjectRequest(bucketName, remotePath); - req.setRange(firstByte, endByte); - try(S3ObjectInputStream is = s3Client.getObject(req).getObjectContent()) { - byte[] readBuf = new byte[4092]; - int rCnt; - int readTotal = 0; - int incomingOffet = off; - while ((rCnt = is.read(readBuf, 0, readBuf.length)) >= 0) { - System.arraycopy(readBuf, 0, b, incomingOffet, rCnt); - readTotal += rCnt; - incomingOffet += rCnt; + Integer cnt = + new RetryableCallable() { + public Integer retriableCall() throws IOException { + GetObjectRequest req = new GetObjectRequest(bucketName, remotePath); + req.setRange(firstByte, endByte); + try (S3ObjectInputStream is = + s3Client.getObject(req).getObjectContent()) { + byte[] readBuf = new byte[4092]; + int rCnt; + int readTotal = 0; + int incomingOffet = off; + while ((rCnt = is.read(readBuf, 0, readBuf.length)) >= 0) { + System.arraycopy(readBuf, 0, b, incomingOffet, rCnt); + readTotal += rCnt; + incomingOffet += rCnt; + } + if (readTotal == 0 && rCnt == -1) return -1; + offset += readTotal; + return readTotal; + } } - if (readTotal == 0 && rCnt == -1) - return -1; - offset += readTotal; - return readTotal; - } - } - }.call(); + }.call(); return cnt; } catch (Exception e) { - String msg = String.format("failed to read offset range %d-%d of file %s whose size is %d", - firstByte, endByte, remotePath, fileSize); + String msg = + String.format( + "failed to read offset range %d-%d of file %s whose size is %d", + firstByte, endByte, remotePath, fileSize); throw new IOException(msg, e); } } @@ -92,4 +94,3 @@ public int read() throws IOException { return -1; } } - diff --git a/priam/src/main/java/com/netflix/priam/backup/SnapshotBackup.java b/priam/src/main/java/com/netflix/priam/backup/SnapshotBackup.java index 186744cea..e25bc9c78 100644 --- a/priam/src/main/java/com/netflix/priam/backup/SnapshotBackup.java +++ b/priam/src/main/java/com/netflix/priam/backup/SnapshotBackup.java @@ -20,24 +20,21 @@ import com.google.inject.Inject; import com.google.inject.Provider; import com.google.inject.Singleton; -import com.netflix.priam.config.IConfiguration; import com.netflix.priam.backup.AbstractBackupPath.BackupFileType; import com.netflix.priam.backup.IMessageObserver.BACKUP_MESSAGE_TYPE; +import com.netflix.priam.config.IConfiguration; import com.netflix.priam.defaultimpl.CassandraOperations; import com.netflix.priam.identity.InstanceIdentity; import com.netflix.priam.scheduler.CronTimer; import com.netflix.priam.scheduler.TaskTimer; import com.netflix.priam.utils.CassandraMonitor; import com.netflix.priam.utils.ThreadSleeper; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import java.io.File; import java.util.*; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; -/** - * Task for running daily snapshots - */ +/** Task for running daily snapshots */ @Singleton public class SnapshotBackup extends AbstractBackup { private static final Logger logger = LoggerFactory.getLogger(SnapshotBackup.class); @@ -55,23 +52,32 @@ public class SnapshotBackup extends AbstractBackup { private final CassandraOperations cassandraOperations; @Inject - public SnapshotBackup(IConfiguration config, Provider pathFactory, - MetaData metaData, IFileSystemContext backupFileSystemCtx - , IBackupStatusMgr snapshotStatusMgr - , InstanceIdentity instanceIdentity, CassandraOperations cassandraOperations) { + public SnapshotBackup( + IConfiguration config, + Provider pathFactory, + MetaData metaData, + IFileSystemContext backupFileSystemCtx, + IBackupStatusMgr snapshotStatusMgr, + InstanceIdentity instanceIdentity, + CassandraOperations cassandraOperations) { super(config, backupFileSystemCtx, pathFactory); this.metaData = metaData; this.snapshotStatusMgr = snapshotStatusMgr; this.instanceIdentity = instanceIdentity; this.cassandraOperations = cassandraOperations; - backupRestoreUtil = new BackupRestoreUtil(config.getSnapshotIncludeCFList(), config.getSnapshotExcludeCFList()); + backupRestoreUtil = + new BackupRestoreUtil( + config.getSnapshotIncludeCFList(), config.getSnapshotExcludeCFList()); } @Override public void execute() throws Exception { - //If Cassandra is started then only start Snapshot Backup + // If Cassandra is started then only start Snapshot Backup while (!CassandraMonitor.hasCassadraStarted()) { - logger.debug("Cassandra has not yet started, hence Snapshot Backup will start after [" + WAIT_TIME_MS / 1000 + "] secs ..."); + logger.debug( + "Cassandra has not yet started, hence Snapshot Backup will start after [" + + WAIT_TIME_MS / 1000 + + "] secs ..."); sleeper.sleep(WAIT_TIME_MS); } @@ -85,27 +91,29 @@ public void execute() throws Exception { try { logger.info("Starting snapshot {}", snapshotName); - //Clearing remotePath List + // Clearing remotePath List snapshotRemotePaths.clear(); cassandraOperations.takeSnapshot(snapshotName); // Collect all snapshot dir's under keyspace dir's abstractBackupPaths = Lists.newArrayList(); - // Try to upload all the files as part of snapshot. If there is any error, there will be an exception and snapshot will be considered as failure. + // Try to upload all the files as part of snapshot. If there is any error, there will be + // an exception and snapshot will be considered as failure. initiateBackup(SNAPSHOT_FOLDER, backupRestoreUtil); - // All the files are uploaded successfully as part of snapshot. - //pre condition notify of meta.json upload - File tmpMetaFile = metaData.createTmpMetaFile(); //Note: no need to remove this temp as it is done within createTmpMetaFile() + // pre condition notify of meta.json upload + File tmpMetaFile = + metaData.createTmpMetaFile(); // Note: no need to remove this temp as it is done + // within createTmpMetaFile() AbstractBackupPath metaJsonAbp = metaData.decorateMetaJson(tmpMetaFile, snapshotName); - // Upload meta file AbstractBackupPath metaJson = metaData.set(abstractBackupPaths, snapshotName); logger.info("Snapshot upload complete for {}", snapshotName); - backupMetadata.setSnapshotLocation(config.getBackupPrefix() + File.separator + metaJson.getRemotePath()); + backupMetadata.setSnapshotLocation( + config.getBackupPrefix() + File.separator + metaJson.getRemotePath()); snapshotStatusMgr.finish(backupMetadata); if (snapshotRemotePaths.size() > 0) { @@ -113,7 +121,10 @@ public void execute() throws Exception { } } catch (Exception e) { - logger.error("Exception occurred while taking snapshot: {}. Exception: {}", snapshotName, e.getLocalizedMessage()); + logger.error( + "Exception occurred while taking snapshot: {}. Exception: {}", + snapshotName, + e.getLocalizedMessage()); snapshotStatusMgr.failed(backupMetadata); throw e; } finally { @@ -127,12 +138,10 @@ public void execute() throws Exception { private File getValidSnapshot(File snpDir, String snapshotName) { for (File snapshotDir : snpDir.listFiles()) - if (snapshotDir.getName().matches(snapshotName)) - return snapshotDir; + if (snapshotDir.getName().matches(snapshotName)) return snapshotDir; return null; } - @Override public String getName() { return JOBNAME; @@ -147,10 +156,15 @@ public static TaskTimer getTimer(IConfiguration config) throws Exception { switch (config.getBackupSchedulerType()) { case HOUR: if (config.getBackupHour() < 0) - logger.info("Skipping {} as it is disabled via backup hour: {}", JOBNAME, config.getBackupHour()); + logger.info( + "Skipping {} as it is disabled via backup hour: {}", + JOBNAME, + config.getBackupHour()); else { cronTimer = new CronTimer(JOBNAME, config.getBackupHour(), 1, 0); - logger.info("Starting snapshot backup with backup hour: {}", config.getBackupHour()); + logger.info( + "Starting snapshot backup with backup hour: {}", + config.getBackupHour()); } break; case CRON: @@ -173,13 +187,13 @@ private void notifyObservers() { if (observer != null) { logger.debug("Updating snapshot observers now ..."); observer.update(BACKUP_MESSAGE_TYPE.SNAPSHOT, snapshotRemotePaths); - } else - logger.info("Observer is Null, hence can not notify ..."); + } else logger.info("Observer is Null, hence can not notify ..."); } } @Override - protected void processColumnFamily(String keyspace, String columnFamily, File backupDir) throws Exception { + protected void processColumnFamily(String keyspace, String columnFamily, File backupDir) + throws Exception { File snapshotDir = getValidSnapshot(backupDir, snapshotName); @@ -189,52 +203,70 @@ protected void processColumnFamily(String keyspace, String columnFamily, File ba } // Add files to this dir - abstractBackupPaths.addAll(upload(snapshotDir, BackupFileType.SNAP, config.enableAsyncSnapshot())); + abstractBackupPaths.addAll( + upload(snapshotDir, BackupFileType.SNAP, config.enableAsyncSnapshot())); } -// private void findForgottenFiles(File snapshotDir) { -// try { -// Collection snapshotFiles = FileUtils.listFiles(snapshotDir, FileFilterUtils.fileFileFilter(), null); -// File columnfamilyDir = snapshotDir.getParentFile().getParentFile(); -// -// //Find all the files in columnfamily folder which is : -// // 1. Not a temp file. -// // 2. Is a file. (we don't care about directories) -// // 3. Is older than snapshot time, as new files keep getting created after taking a snapshot. -// IOFileFilter tmpFileFilter1 = FileFilterUtils.suffixFileFilter(TMP_EXT); -// IOFileFilter tmpFileFilter2 = FileFilterUtils.asFileFilter(pathname -> tmpFilePattern.matcher(pathname.getName()).matches()); -// IOFileFilter tmpFileFilter = FileFilterUtils.or(tmpFileFilter1, tmpFileFilter2); -// // Here we are allowing files which were more than @link{IConfiguration#getForgottenFileGracePeriodDays}. We do this to allow cassandra to -// // clean up any files which were generated as part of repair/compaction and cleanup thread has not already deleted. -// // Refer to https://issues.apache.org/jira/browse/CASSANDRA-6756 and https://issues.apache.org/jira/browse/CASSANDRA-7066 -// // for more information. -// IOFileFilter ageFilter = FileFilterUtils.ageFileFilter(snapshotInstant.minus(config.getForgottenFileGracePeriodDays(), ChronoUnit.DAYS).toEpochMilli()); -// IOFileFilter fileFilter = FileFilterUtils.and(FileFilterUtils.notFileFilter(tmpFileFilter), FileFilterUtils.fileFileFilter(), ageFilter); -// -// Collection columnfamilyFiles = FileUtils.listFiles(columnfamilyDir, fileFilter, null); -// -// //Remove the SSTable(s) which are part of snapshot from the CF file list. -// //This cannot be a simple removeAll as snapshot files have "different" file folder prefix. -// for (File file : snapshotFiles) { -// //Get its parent directory file based on this file. -// File originalFile = new File(columnfamilyDir, file.getName()); -// columnfamilyFiles.remove(originalFile); -// } -// -// //If there are no "extra" SSTables in CF data folder, we are done. -// if (columnfamilyFiles.size() == 0) -// return; -// -// columnfamilyFiles.parallelStream().forEach(file -> logger.info("Forgotten file: {} found for CF: {}", file.getAbsolutePath(), columnfamilyDir.getName())); -// -// //TODO: The eventual plan is to move the forgotten files to a lost+found directory and clean the directory after 'x' amount of time. This behavior should be configurable. -// backupMetrics.incrementForgottenFiles(columnfamilyFiles.size()); -// logger.warn("# of forgotten files: {} found for CF: {}", columnfamilyFiles.size(), columnfamilyDir.getName()); -// } catch (Exception e) { -// //Eat the exception, if there, for any reason. This should not stop the snapshot for any reason. -// logger.error("Exception occurred while trying to find forgottenFile. Ignoring the error and continuing with remaining backup", e); -// } -// } + // private void findForgottenFiles(File snapshotDir) { + // try { + // Collection snapshotFiles = FileUtils.listFiles(snapshotDir, + // FileFilterUtils.fileFileFilter(), null); + // File columnfamilyDir = snapshotDir.getParentFile().getParentFile(); + // + // //Find all the files in columnfamily folder which is : + // // 1. Not a temp file. + // // 2. Is a file. (we don't care about directories) + // // 3. Is older than snapshot time, as new files keep getting created after taking + // a snapshot. + // IOFileFilter tmpFileFilter1 = FileFilterUtils.suffixFileFilter(TMP_EXT); + // IOFileFilter tmpFileFilter2 = FileFilterUtils.asFileFilter(pathname -> + // tmpFilePattern.matcher(pathname.getName()).matches()); + // IOFileFilter tmpFileFilter = FileFilterUtils.or(tmpFileFilter1, tmpFileFilter2); + // // Here we are allowing files which were more than + // @link{IConfiguration#getForgottenFileGracePeriodDays}. We do this to allow cassandra to + // // clean up any files which were generated as part of repair/compaction and + // cleanup thread has not already deleted. + // // Refer to https://issues.apache.org/jira/browse/CASSANDRA-6756 and + // https://issues.apache.org/jira/browse/CASSANDRA-7066 + // // for more information. + // IOFileFilter ageFilter = + // FileFilterUtils.ageFileFilter(snapshotInstant.minus(config.getForgottenFileGracePeriodDays(), + // ChronoUnit.DAYS).toEpochMilli()); + // IOFileFilter fileFilter = + // FileFilterUtils.and(FileFilterUtils.notFileFilter(tmpFileFilter), + // FileFilterUtils.fileFileFilter(), ageFilter); + // + // Collection columnfamilyFiles = FileUtils.listFiles(columnfamilyDir, + // fileFilter, null); + // + // //Remove the SSTable(s) which are part of snapshot from the CF file list. + // //This cannot be a simple removeAll as snapshot files have "different" file folder + // prefix. + // for (File file : snapshotFiles) { + // //Get its parent directory file based on this file. + // File originalFile = new File(columnfamilyDir, file.getName()); + // columnfamilyFiles.remove(originalFile); + // } + // + // //If there are no "extra" SSTables in CF data folder, we are done. + // if (columnfamilyFiles.size() == 0) + // return; + // + // columnfamilyFiles.parallelStream().forEach(file -> logger.info("Forgotten file: {} + // found for CF: {}", file.getAbsolutePath(), columnfamilyDir.getName())); + // + // //TODO: The eventual plan is to move the forgotten files to a lost+found directory + // and clean the directory after 'x' amount of time. This behavior should be configurable. + // backupMetrics.incrementForgottenFiles(columnfamilyFiles.size()); + // logger.warn("# of forgotten files: {} found for CF: {}", columnfamilyFiles.size(), + // columnfamilyDir.getName()); + // } catch (Exception e) { + // //Eat the exception, if there, for any reason. This should not stop the snapshot + // for any reason. + // logger.error("Exception occurred while trying to find forgottenFile. Ignoring the + // error and continuing with remaining backup", e); + // } + // } @Override protected void addToRemotePath(String remotePath) { diff --git a/priam/src/main/java/com/netflix/priam/backup/Status.java b/priam/src/main/java/com/netflix/priam/backup/Status.java index ea2eba1f3..57cd1768b 100644 --- a/priam/src/main/java/com/netflix/priam/backup/Status.java +++ b/priam/src/main/java/com/netflix/priam/backup/Status.java @@ -16,20 +16,15 @@ */ package com.netflix.priam.backup; -/** - * Enum to describe the status of the snapshot/restore. - */ +/** Enum to describe the status of the snapshot/restore. */ public enum Status { - /** - * Denotes snapshot/restore has started successfully and is running. - */ + /** Denotes snapshot/restore has started successfully and is running. */ STARTED, - /** - * Denotes snapshot/restore has finished successfully. - */ + /** Denotes snapshot/restore has finished successfully. */ FINISHED, /** - * Denotes snapshot/restore has failed to upload/restore successfully or there was a failure marking the snapshot/restore as failure. + * Denotes snapshot/restore has failed to upload/restore successfully or there was a failure + * marking the snapshot/restore as failure. */ FAILED } diff --git a/priam/src/main/java/com/netflix/priam/backupv2/ColumnfamilyResult.java b/priam/src/main/java/com/netflix/priam/backupv2/ColumnfamilyResult.java index e97eb17ca..a0d4242da 100644 --- a/priam/src/main/java/com/netflix/priam/backupv2/ColumnfamilyResult.java +++ b/priam/src/main/java/com/netflix/priam/backupv2/ColumnfamilyResult.java @@ -1,28 +1,25 @@ /** * Copyright 2018 Netflix, Inc. - *

- * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and + * + *

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file + * except in compliance with the License. You may obtain a copy of the License at + * + *

http://www.apache.org/licenses/LICENSE-2.0 + * + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing permissions and * limitations under the License. */ package com.netflix.priam.backupv2; import com.netflix.priam.utils.GsonJsonSerializer; - import java.util.ArrayList; import java.util.List; /** - * This is a POJO to encapsulate all the SSTables for a given column family. - * Created by aagrawal on 7/1/18. + * This is a POJO to encapsulate all the SSTables for a given column family. Created by aagrawal on + * 7/1/18. */ public class ColumnfamilyResult { private String keyspaceName; @@ -59,8 +56,7 @@ public void setSstables(List sstables) { } public void addSstable(SSTableResult sstable) { - if (sstables == null) - sstables = new ArrayList<>(); + if (sstables == null) sstables = new ArrayList<>(); sstables.add(sstable); } @@ -69,9 +65,7 @@ public String toString() { return GsonJsonSerializer.getGson().toJson(this); } - /** - * This is a POJO to encapsulate a SSTable and all its components. - */ + /** This is a POJO to encapsulate a SSTable and all its components. */ public static class SSTableResult { private String prefix; private List sstableComponents; diff --git a/priam/src/main/java/com/netflix/priam/backupv2/FileUploadResult.java b/priam/src/main/java/com/netflix/priam/backupv2/FileUploadResult.java index 3f8859fe2..b356f0229 100644 --- a/priam/src/main/java/com/netflix/priam/backupv2/FileUploadResult.java +++ b/priam/src/main/java/com/netflix/priam/backupv2/FileUploadResult.java @@ -18,7 +18,6 @@ import com.netflix.priam.compress.ICompression; import com.netflix.priam.utils.GsonJsonSerializer; - import java.io.File; import java.nio.file.Files; import java.nio.file.Path; @@ -26,24 +25,28 @@ import java.time.Instant; /** - * This is a POJO that will encapsulate the result of file upload. - * Created by aagrawal on 6/20/18. + * This is a POJO that will encapsulate the result of file upload. Created by aagrawal on 6/20/18. */ public class FileUploadResult { private final Path fileName; - @GsonJsonSerializer.PriamAnnotation.GsonIgnore - private String keyspaceName; - @GsonJsonSerializer.PriamAnnotation.GsonIgnore - private String columnFamilyName; + @GsonJsonSerializer.PriamAnnotation.GsonIgnore private String keyspaceName; + @GsonJsonSerializer.PriamAnnotation.GsonIgnore private String columnFamilyName; private Instant lastModifiedTime; private final Instant fileCreationTime; - private final long fileSizeOnDisk; //Size on disk in bytes + private final long fileSizeOnDisk; // Size on disk in bytes private Boolean isUploaded; - //Valid compression technique for now is SNAPPY only. Future we need to support LZ4 and NONE - private ICompression.CompressionAlgorithm compression = ICompression.CompressionAlgorithm.SNAPPY; + // Valid compression technique for now is SNAPPY only. Future we need to support LZ4 and NONE + private ICompression.CompressionAlgorithm compression = + ICompression.CompressionAlgorithm.SNAPPY; private Path backupPath; - public FileUploadResult(Path fileName, String keyspaceName, String columnFamilyName, Instant lastModifiedTime, Instant fileCreationTime, long fileSizeOnDisk) { + public FileUploadResult( + Path fileName, + String keyspaceName, + String columnFamilyName, + Instant lastModifiedTime, + Instant fileCreationTime, + long fileSizeOnDisk) { this.fileName = fileName; this.keyspaceName = keyspaceName; this.columnFamilyName = columnFamilyName; @@ -52,12 +55,20 @@ public FileUploadResult(Path fileName, String keyspaceName, String columnFamilyN this.fileSizeOnDisk = fileSizeOnDisk; } - public static FileUploadResult getFileUploadResult(String keyspaceName, String columnFamilyName, Path file) throws Exception { + public static FileUploadResult getFileUploadResult( + String keyspaceName, String columnFamilyName, Path file) throws Exception { BasicFileAttributes fileAttributes = Files.readAttributes(file, BasicFileAttributes.class); - return new FileUploadResult(file, keyspaceName, columnFamilyName, fileAttributes.lastModifiedTime().toInstant(), fileAttributes.creationTime().toInstant(), fileAttributes.size()); + return new FileUploadResult( + file, + keyspaceName, + columnFamilyName, + fileAttributes.lastModifiedTime().toInstant(), + fileAttributes.creationTime().toInstant(), + fileAttributes.size()); } - public static FileUploadResult getFileUploadResult(String keyspaceName, String columnFamilyName, File file) throws Exception { + public static FileUploadResult getFileUploadResult( + String keyspaceName, String columnFamilyName, File file) throws Exception { return getFileUploadResult(keyspaceName, columnFamilyName, file.toPath()); } diff --git a/priam/src/main/java/com/netflix/priam/backupv2/LocalDBReaderWriter.java b/priam/src/main/java/com/netflix/priam/backupv2/LocalDBReaderWriter.java index 163b08dd5..154c25af1 100644 --- a/priam/src/main/java/com/netflix/priam/backupv2/LocalDBReaderWriter.java +++ b/priam/src/main/java/com/netflix/priam/backupv2/LocalDBReaderWriter.java @@ -20,9 +20,6 @@ import com.google.inject.Inject; import com.netflix.priam.config.IConfiguration; import com.netflix.priam.utils.GsonJsonSerializer; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import java.io.File; import java.io.FileReader; import java.io.FileWriter; @@ -33,15 +30,17 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** - * This class is used to create a local DB entry for SSTable components. This will be used to identify when a version - * of a SSTable component was last uploaded or last referenced. - * This db entry will be used to enforce the TTL of a version of a SSTable component as we will not be relying on - * backup file system level TTL. - * Local DB should be copied over to new instance replacing this token in case of instance replacement. If no local DB - * is found then Priam will try to re-create the local DB using meta files uploaded to backup file system. - * The check operation for local DB is done at every start of Priam and when operator requests to re-build the local DB. + * This class is used to create a local DB entry for SSTable components. This will be used to + * identify when a version of a SSTable component was last uploaded or last referenced. This db + * entry will be used to enforce the TTL of a version of a SSTable component as we will not be + * relying on backup file system level TTL. Local DB should be copied over to new instance replacing + * this token in case of instance replacement. If no local DB is found then Priam will try to + * re-create the local DB using meta files uploaded to backup file system. The check operation for + * local DB is done at every start of Priam and when operator requests to re-build the local DB. * Created by aagrawal on 8/31/18. */ public class LocalDBReaderWriter { @@ -54,8 +53,9 @@ public LocalDBReaderWriter(IConfiguration configuration) { this.configuration = configuration; } - public synchronized LocalDB upsertLocalDBEntry(final LocalDBEntry localDBEntry) throws Exception { - //validate the localDBEntry first + public synchronized LocalDB upsertLocalDBEntry(final LocalDBEntry localDBEntry) + throws Exception { + // validate the localDBEntry first if (localDBEntry.getTimeLastReferenced() == null) throw new Exception("Time last referenced in localDB can never be null"); @@ -68,19 +68,19 @@ public synchronized LocalDB upsertLocalDBEntry(final LocalDBEntry localDBEntry) LocalDB localDB = readAndGetLocalDB(localDBEntry.getFileUploadResult()); - if (localDB == null) - localDB = new LocalDB(new ArrayList<>()); + if (localDB == null) localDB = new LocalDB(new ArrayList<>()); - //Verify again if someone beat you to write the entry. + // Verify again if someone beat you to write the entry. LocalDBEntry entry = getLocalDBEntry(localDBEntry.getFileUploadResult(), localDB); - //Write entry as it might be either - - //1. new component entry - //2. new version of file (change in compression type or file is modified e.g. stats file) + // Write entry as it might be either - + // 1. new component entry + // 2. new version of file (change in compression type or file is modified e.g. stats file) if (entry == null) { localDB.getLocalDBEntries().add(localDBEntry); writeLocalDB(localDBFile, localDB); } else { - //An entry already exists. Maybe last referenced time or backup time changed. We want to write the last time referenced. + // An entry already exists. Maybe last referenced time or backup time changed. We want + // to write the last time referenced. entry.setBackupTime(localDBEntry.getBackupTime()); entry.setTimeLastReferenced(localDBEntry.getTimeLastReferenced()); writeLocalDB(localDBFile, localDB); @@ -106,25 +106,49 @@ public LocalDBEntry getLocalDBEntry(final FileUploadResult fileUploadResult) thr return getLocalDBEntry(fileUploadResult, localDB); } - private LocalDBEntry getLocalDBEntry(final FileUploadResult fileUploadResult, final LocalDB localDB) throws Exception { - if (localDB == null || localDB.getLocalDBEntries() == null || localDB.getLocalDBEntries().isEmpty()) - return null; + private LocalDBEntry getLocalDBEntry( + final FileUploadResult fileUploadResult, final LocalDB localDB) throws Exception { + if (localDB == null + || localDB.getLocalDBEntries() == null + || localDB.getLocalDBEntries().isEmpty()) return null; // Get local db entry for same file and version. - List localDBEntries = localDB.getLocalDBEntries().stream().filter(localDBEntry -> - // Name of the file should be same. - (localDBEntry.getFileUploadResult().getFileName().toFile().getName().toLowerCase(). - equals(fileUploadResult.getFileName().toFile().getName().toLowerCase()))) - // Should be same version (same last modified time) - .filter(localDBEntry -> (localDBEntry.getFileUploadResult().getLastModifiedTime(). - equals(fileUploadResult.getLastModifiedTime()))) - // Same compression as before. If we switch compression technique we can upload again. - .filter(localDBEntry -> (localDBEntry.getFileUploadResult().getCompression(). - equals(fileUploadResult.getCompression()))) - .collect(Collectors.toList()); - - if (localDBEntries.isEmpty()) - return null; + List localDBEntries = + localDB.getLocalDBEntries() + .stream() + .filter( + localDBEntry -> + // Name of the file should be same. + (localDBEntry + .getFileUploadResult() + .getFileName() + .toFile() + .getName() + .toLowerCase() + .equals( + fileUploadResult + .getFileName() + .toFile() + .getName() + .toLowerCase()))) + // Should be same version (same last modified time) + .filter( + localDBEntry -> + (localDBEntry + .getFileUploadResult() + .getLastModifiedTime() + .equals(fileUploadResult.getLastModifiedTime()))) + // Same compression as before. If we switch compression technique we can + // upload again. + .filter( + localDBEntry -> + (localDBEntry + .getFileUploadResult() + .getCompression() + .equals(fileUploadResult.getCompression()))) + .collect(Collectors.toList()); + + if (localDBEntries.isEmpty()) return null; if (localDBEntries.size() == 1) { if (logger.isDebugEnabled()) @@ -133,37 +157,47 @@ private LocalDBEntry getLocalDBEntry(final FileUploadResult fileUploadResult, fi return localDBEntries.get(0); } - throw new Exception("Unexpected behavior: More than one entry found in local database for the same file. FileUploadResult: " + fileUploadResult); + throw new Exception( + "Unexpected behavior: More than one entry found in local database for the same file. FileUploadResult: " + + fileUploadResult); } /** * Gets the local db path on the local file system. * - * @param fileUploadResult This contains the SSTable component for which local db path is required. + * @param fileUploadResult This contains the SSTable component for which local db path is + * required. * @return the local db path on local file system. */ public Path getLocalDBPath(final FileUploadResult fileUploadResult) { - return Paths.get(configuration.getDataFileLocation(), LOCAL_DB, + return Paths.get( + configuration.getDataFileLocation(), + LOCAL_DB, fileUploadResult.getKeyspaceName(), fileUploadResult.getColumnFamilyName(), - PrefixGenerator.getSSTFileBase(fileUploadResult.getFileName().toFile().getName()) + ".localdb"); + PrefixGenerator.getSSTFileBase(fileUploadResult.getFileName().toFile().getName()) + + ".localdb"); } /** - * Writes the local database to the local db file. This will do a complete replace of existing local database if any. + * Writes the local database to the local db file. This will do a complete replace of existing + * local database if any. * * @param localDBFile path to the local database file. - * @param localDB local database containing all the entries to the local database. - * @throws Exception If the path denoted is directory, write permission issues or any other exceptions. + * @param localDB local database containing all the entries to the local database. + * @throws Exception If the path denoted is directory, write permission issues or any other + * exceptions. */ public void writeLocalDB(final Path localDBFile, final LocalDB localDB) throws Exception { if (localDB == null || localDBFile == null || localDBFile.toFile().isDirectory()) - throw new Exception("Invalid Arguments: localDbFile: " + localDBFile + ", localDB: " + localDB); + throw new Exception( + "Invalid Arguments: localDbFile: " + localDBFile + ", localDB: " + localDB); - if (!localDBFile.getParent().toFile().exists()) - localDBFile.getParent().toFile().mkdirs(); + if (!localDBFile.getParent().toFile().exists()) localDBFile.getParent().toFile().mkdirs(); - File tmpFile = File.createTempFile(localDBFile.toFile().getName(), ".tmp", localDBFile.getParent().toFile()); + File tmpFile = + File.createTempFile( + localDBFile.toFile().getName(), ".tmp", localDBFile.getParent().toFile()); try (FileWriter writer = new FileWriter(tmpFile)) { writer.write(localDB.toString()); @@ -171,10 +205,8 @@ public void writeLocalDB(final Path localDBFile, final LocalDB localDB) throws E if (!tmpFile.renameTo(localDBFile.toFile())) logger.error("Failed to persist local db: {}", localDB); } finally { - if (tmpFile != null) - Files.deleteIfExists(tmpFile.toPath()); + if (tmpFile != null) Files.deleteIfExists(tmpFile.toPath()); } - } /** @@ -182,24 +214,27 @@ public void writeLocalDB(final Path localDBFile, final LocalDB localDB) throws E * * @param localDBFile path the local database. * @return local database if file exists or empty local database. - * @throws Exception If there is any error in de-serializing the object or any other file system errors. + * @throws Exception If there is any error in de-serializing the object or any other file system + * errors. */ public LocalDB readLocalDB(final Path localDBFile) throws Exception { - //Verify file exists - if (!localDBFile.toFile().exists()) - return new LocalDB(new ArrayList<>()); + // Verify file exists + if (!localDBFile.toFile().exists()) return new LocalDB(new ArrayList<>()); try (FileReader reader = new FileReader(localDBFile.toFile())) { LocalDB localDB = GsonJsonSerializer.getGson().fromJson(reader, LocalDB.class); String columnfamilyName = localDBFile.getParent().toFile().getName(); String keyspaceName = localDBFile.getParent().getParent().toFile().getName(); - localDB.getLocalDBEntries().forEach(localDBEntry -> { - localDBEntry.getFileUploadResult().setColumnFamilyName(columnfamilyName); - localDBEntry.getFileUploadResult().setKeyspaceName(keyspaceName); - }); - - if (logger.isDebugEnabled()) - logger.debug("Local DB: {}", localDB); + localDB.getLocalDBEntries() + .forEach( + localDBEntry -> { + localDBEntry + .getFileUploadResult() + .setColumnFamilyName(columnfamilyName); + localDBEntry.getFileUploadResult().setKeyspaceName(keyspaceName); + }); + + if (logger.isDebugEnabled()) logger.debug("Local DB: {}", localDB); return localDB; } } @@ -251,7 +286,8 @@ public void setBackupTime(Instant backupTime) { this.backupTime = backupTime; } - public LocalDBEntry(FileUploadResult fileUploadResult, Instant timeLastReferenced, Instant backupTime) { + public LocalDBEntry( + FileUploadResult fileUploadResult, Instant timeLastReferenced, Instant backupTime) { this.fileUploadResult = fileUploadResult; this.timeLastReferenced = timeLastReferenced; @@ -262,8 +298,5 @@ public LocalDBEntry(FileUploadResult fileUploadResult, Instant timeLastReference public String toString() { return GsonJsonSerializer.getGson().toJson(this); } - } - - } diff --git a/priam/src/main/java/com/netflix/priam/backupv2/MetaFileInfo.java b/priam/src/main/java/com/netflix/priam/backupv2/MetaFileInfo.java index 72d76c23d..8a734811a 100644 --- a/priam/src/main/java/com/netflix/priam/backupv2/MetaFileInfo.java +++ b/priam/src/main/java/com/netflix/priam/backupv2/MetaFileInfo.java @@ -18,20 +18,20 @@ import com.netflix.priam.utils.DateUtil; import com.netflix.priam.utils.GsonJsonSerializer; - import java.time.Instant; import java.util.List; -/** - * This POJO class encapsulates the information for a meta file. - */ +/** This POJO class encapsulates the information for a meta file. */ public class MetaFileInfo { @GsonJsonSerializer.PriamAnnotation.GsonIgnore public static final String META_FILE_PREFIX = "meta_v2_"; + @GsonJsonSerializer.PriamAnnotation.GsonIgnore public static final String META_FILE_SUFFIX = ".json"; + @GsonJsonSerializer.PriamAnnotation.GsonIgnore public static final String META_FILE_INFO = "info"; + @GsonJsonSerializer.PriamAnnotation.GsonIgnore public static final String META_FILE_DATA = "data"; @@ -95,6 +95,8 @@ public String toString() { } public static String getMetaFileName(Instant instant) { - return MetaFileInfo.META_FILE_PREFIX + DateUtil.formatInstant(DateUtil.yyyyMMddHHmm, instant) + MetaFileInfo.META_FILE_SUFFIX; + return MetaFileInfo.META_FILE_PREFIX + + DateUtil.formatInstant(DateUtil.yyyyMMddHHmm, instant) + + MetaFileInfo.META_FILE_SUFFIX; } } diff --git a/priam/src/main/java/com/netflix/priam/backupv2/MetaFileManager.java b/priam/src/main/java/com/netflix/priam/backupv2/MetaFileManager.java index 723aa5a8b..415dd5e97 100644 --- a/priam/src/main/java/com/netflix/priam/backupv2/MetaFileManager.java +++ b/priam/src/main/java/com/netflix/priam/backupv2/MetaFileManager.java @@ -18,48 +18,50 @@ package com.netflix.priam.backupv2; import com.netflix.priam.config.IConfiguration; +import java.io.File; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Collection; +import javax.inject.Inject; import org.apache.commons.io.FileUtils; import org.apache.commons.io.filefilter.FileFilterUtils; import org.apache.commons.io.filefilter.IOFileFilter; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import javax.inject.Inject; -import java.io.File; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.Collection; - -/** - * Do any management task for meta files. - * Created by aagrawal on 8/2/18. - */ +/** Do any management task for meta files. Created by aagrawal on 8/2/18. */ public class MetaFileManager { private static final Logger logger = LoggerFactory.getLogger(MetaFileManager.class); private final Path metaFileDirectory; @Inject - MetaFileManager(IConfiguration configuration){ + MetaFileManager(IConfiguration configuration) { metaFileDirectory = Paths.get(configuration.getDataFileLocation()); } - public Path getMetaFileDirectory(){ + public Path getMetaFileDirectory() { return metaFileDirectory; } - /** - * Delete the old meta files, if any present in the metaFileDirectory - */ + /** Delete the old meta files, if any present in the metaFileDirectory */ public void cleanupOldMetaFiles() { logger.info("Deleting any old META_V2 files if any"); - IOFileFilter fileNameFilter = FileFilterUtils.and(FileFilterUtils.prefixFileFilter(MetaFileInfo.META_FILE_PREFIX), - FileFilterUtils.or(FileFilterUtils.suffixFileFilter(MetaFileInfo.META_FILE_SUFFIX), - FileFilterUtils.suffixFileFilter(MetaFileInfo.META_FILE_SUFFIX + ".tmp"))); - Collection files = FileUtils.listFiles(metaFileDirectory.toFile(), fileNameFilter, null); - files.stream().filter(File::isFile).forEach(file -> { - logger.debug("Deleting old META_V2 file found: {}", file.getAbsolutePath()); - file.delete(); - }); + IOFileFilter fileNameFilter = + FileFilterUtils.and( + FileFilterUtils.prefixFileFilter(MetaFileInfo.META_FILE_PREFIX), + FileFilterUtils.or( + FileFilterUtils.suffixFileFilter(MetaFileInfo.META_FILE_SUFFIX), + FileFilterUtils.suffixFileFilter( + MetaFileInfo.META_FILE_SUFFIX + ".tmp"))); + Collection files = + FileUtils.listFiles(metaFileDirectory.toFile(), fileNameFilter, null); + files.stream() + .filter(File::isFile) + .forEach( + file -> { + logger.debug( + "Deleting old META_V2 file found: {}", file.getAbsolutePath()); + file.delete(); + }); } - } diff --git a/priam/src/main/java/com/netflix/priam/backupv2/MetaFileReader.java b/priam/src/main/java/com/netflix/priam/backupv2/MetaFileReader.java index 4b6068a18..17382f79f 100644 --- a/priam/src/main/java/com/netflix/priam/backupv2/MetaFileReader.java +++ b/priam/src/main/java/com/netflix/priam/backupv2/MetaFileReader.java @@ -19,17 +19,16 @@ import com.google.gson.stream.JsonReader; import com.netflix.priam.utils.DateUtil; import com.netflix.priam.utils.GsonJsonSerializer; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import java.io.FileNotFoundException; import java.io.FileReader; import java.io.IOException; import java.nio.file.Path; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** - * This abstract class encapsulates the reading of meta file in streaming fashion. This is required as we could have a meta file which cannot fit in memory. - * Created by aagrawal on 7/3/18. + * This abstract class encapsulates the reading of meta file in streaming fashion. This is required + * as we could have a meta file which cannot fit in memory. Created by aagrawal on 7/3/18. */ public abstract class MetaFileReader { @@ -47,24 +46,31 @@ public MetaFileInfo getMetaFileInfo() { * @throws IOException if not enough permissions or file is not valid format. */ public void readMeta(Path metaFilePath) throws IOException { - //Validate if meta file exists and is right file name. - if (metaFilePath == null || !metaFilePath.toFile().exists() || !metaFilePath.toFile().isFile() || !isValidMetaFile(metaFilePath)) { - throw new FileNotFoundException("MetaFilePath: " + metaFilePath + " do not exist or is not valid meta file."); + // Validate if meta file exists and is right file name. + if (metaFilePath == null + || !metaFilePath.toFile().exists() + || !metaFilePath.toFile().isFile() + || !isValidMetaFile(metaFilePath)) { + throw new FileNotFoundException( + "MetaFilePath: " + metaFilePath + " do not exist or is not valid meta file."); } - //Read the meta file. + // Read the meta file. logger.info("Trying to read the meta file: {}", metaFilePath); JsonReader jsonReader = new JsonReader(new FileReader(metaFilePath.toFile())); jsonReader.beginObject(); while (jsonReader.hasNext()) { switch (jsonReader.nextName()) { case MetaFileInfo.META_FILE_INFO: - metaFileInfo = GsonJsonSerializer.getGson().fromJson(jsonReader, MetaFileInfo.class); + metaFileInfo = + GsonJsonSerializer.getGson().fromJson(jsonReader, MetaFileInfo.class); break; case MetaFileInfo.META_FILE_DATA: jsonReader.beginArray(); while (jsonReader.hasNext()) - process(GsonJsonSerializer.getGson().fromJson(jsonReader, ColumnfamilyResult.class)); + process( + GsonJsonSerializer.getGson() + .fromJson(jsonReader, ColumnfamilyResult.class)); jsonReader.endArray(); } } @@ -73,11 +79,11 @@ public void readMeta(Path metaFilePath) throws IOException { logger.info("Finished reading the meta file: {}", metaFilePath); } - /** * Process the columnfamily result obtained after reading meta file. * - * @param columnfamilyResult {@link ColumnfamilyResult} POJO containing the column family data (all SSTables references) obtained from meta.json. + * @param columnfamilyResult {@link ColumnfamilyResult} POJO containing the column family data + * (all SSTables references) obtained from meta.json. */ public abstract void process(ColumnfamilyResult columnfamilyResult); @@ -89,9 +95,13 @@ public void readMeta(Path metaFilePath) throws IOException { */ public boolean isValidMetaFile(Path metaFilePath) { String fileName = metaFilePath.toFile().getName(); - if (fileName.startsWith(MetaFileInfo.META_FILE_PREFIX) && fileName.endsWith(MetaFileInfo.META_FILE_SUFFIX)) { - //is valid date? - String dateString = fileName.substring(MetaFileInfo.META_FILE_PREFIX.length(), fileName.length() - MetaFileInfo.META_FILE_SUFFIX.length()); + if (fileName.startsWith(MetaFileInfo.META_FILE_PREFIX) + && fileName.endsWith(MetaFileInfo.META_FILE_SUFFIX)) { + // is valid date? + String dateString = + fileName.substring( + MetaFileInfo.META_FILE_PREFIX.length(), + fileName.length() - MetaFileInfo.META_FILE_SUFFIX.length()); DateUtil.parseInstant(dateString); return true; } diff --git a/priam/src/main/java/com/netflix/priam/backupv2/MetaFileWriterBuilder.java b/priam/src/main/java/com/netflix/priam/backupv2/MetaFileWriterBuilder.java index d2c9d8cc4..00ec5d83e 100644 --- a/priam/src/main/java/com/netflix/priam/backupv2/MetaFileWriterBuilder.java +++ b/priam/src/main/java/com/netflix/priam/backupv2/MetaFileWriterBuilder.java @@ -18,15 +18,11 @@ import com.google.gson.stream.JsonWriter; import com.google.inject.Provider; -import com.netflix.priam.config.IConfiguration; import com.netflix.priam.backup.AbstractBackupPath; import com.netflix.priam.backup.IBackupFileSystem; import com.netflix.priam.backup.IFileSystemContext; +import com.netflix.priam.config.IConfiguration; import com.netflix.priam.identity.InstanceIdentity; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import javax.inject.Inject; import java.io.FileWriter; import java.io.IOException; import java.nio.file.Path; @@ -34,13 +30,16 @@ import java.time.Instant; import java.util.ArrayList; import java.util.List; +import javax.inject.Inject; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** - * This class will help in generation of meta.json files. This will encapsulate all the SSTables that were there - * on the file system. This will write the meta.json file as a JSON blob. - * NOTE: We want to ensure that it is done via streaming JSON write to ensure we do not consume memory to load all - * these objects in memory. With multi-tenant clusters or LCS enabled on large number of CF's it is easy to have 1000's - * of SSTables (thus 1000's of SSTable components) across CF's. + * This class will help in generation of meta.json files. This will encapsulate all the SSTables + * that were there on the file system. This will write the meta.json file as a JSON blob. NOTE: We + * want to ensure that it is done via streaming JSON write to ensure we do not consume memory to + * load all these objects in memory. With multi-tenant clusters or LCS enabled on large number of + * CF's it is easy to have 1000's of SSTables (thus 1000's of SSTable components) across CF's. * Created by aagrawal on 6/12/18. */ public class MetaFileWriterBuilder { @@ -62,11 +61,13 @@ public interface StartStep { public interface DataStep { DataStep addColumnfamilyResult(ColumnfamilyResult columnfamilyResult) throws IOException; + UploadStep endMetaFileGeneration() throws IOException; } public interface UploadStep { void uploadMetaFile(boolean deleteOnSuccess) throws Exception; + Path getMetaFilePath(); } @@ -80,13 +81,23 @@ public static class MetaFileWriter implements StartStep, DataStep, UploadStep { private Path metaFilePath; @Inject - private MetaFileWriter(IConfiguration configuration, InstanceIdentity instanceIdentity, Provider pathFactory, IFileSystemContext backupFileSystemCtx, MetaFileManager metaFileManager) { + private MetaFileWriter( + IConfiguration configuration, + InstanceIdentity instanceIdentity, + Provider pathFactory, + IFileSystemContext backupFileSystemCtx, + MetaFileManager metaFileManager) { this.pathFactory = pathFactory; this.backupFileSystem = backupFileSystemCtx.getFileStrategy(configuration); this.metaFileManager = metaFileManager; List backupIdentifier = new ArrayList<>(); backupIdentifier.add(instanceIdentity.getInstance().getToken()); - metaFileInfo = new MetaFileInfo(configuration.getAppName(), configuration.getDC(), configuration.getRac(), backupIdentifier); + metaFileInfo = + new MetaFileInfo( + configuration.getAppName(), + configuration.getDC(), + configuration.getRac(), + backupIdentifier); } /** @@ -95,10 +106,11 @@ private MetaFileWriter(IConfiguration configuration, InstanceIdentity instanceId * @throws IOException if unable to write to meta file (permissions, disk full etc) */ public DataStep startMetaFileGeneration(Instant snapshotInstant) throws IOException { - //Compute meta file name. + // Compute meta file name. String fileName = MetaFileInfo.getMetaFileName(snapshotInstant); metaFilePath = Paths.get(metaFileManager.getMetaFileDirectory().toString(), fileName); - Path tempMetaFilePath = Paths.get(metaFileManager.getMetaFileDirectory().toString(), fileName + ".tmp"); + Path tempMetaFilePath = + Paths.get(metaFileManager.getMetaFileDirectory().toString(), fileName + ".tmp"); logger.info("Starting to write a new meta file: {}", metaFilePath); @@ -112,16 +124,20 @@ public DataStep startMetaFileGeneration(Instant snapshotInstant) throws IOExcept } /** - * Add {@link ColumnfamilyResult} after it has been processed so it can be streamed to meta.json. Streaming write to meta.json is required so we don't get Priam OOM. + * Add {@link ColumnfamilyResult} after it has been processed so it can be streamed to + * meta.json. Streaming write to meta.json is required so we don't get Priam OOM. * * @param columnfamilyResult a POJO encapsulating the column family result * @throws IOException if unable to write to the file or if JSON is not valid */ - public MetaFileWriterBuilder.DataStep addColumnfamilyResult(ColumnfamilyResult columnfamilyResult) throws IOException { + public MetaFileWriterBuilder.DataStep addColumnfamilyResult( + ColumnfamilyResult columnfamilyResult) throws IOException { if (jsonWriter == null) - throw new NullPointerException("addColumnfamilyResult: Json Writer in MetaFileWriter is null. This should not happen!"); + throw new NullPointerException( + "addColumnfamilyResult: Json Writer in MetaFileWriter is null. This should not happen!"); if (columnfamilyResult == null) - throw new NullPointerException("Column family result is null in MetaFileWriter. This should not happen!"); + throw new NullPointerException( + "Column family result is null in MetaFileWriter. This should not happen!"); jsonWriter.jsonValue(columnfamilyResult.toString()); return this; } @@ -134,15 +150,19 @@ public MetaFileWriterBuilder.DataStep addColumnfamilyResult(ColumnfamilyResult c */ public MetaFileWriterBuilder.UploadStep endMetaFileGeneration() throws IOException { if (jsonWriter == null) - throw new NullPointerException("endMetaFileGeneration: Json Writer in MetaFileWriter is null. This should not happen!"); + throw new NullPointerException( + "endMetaFileGeneration: Json Writer in MetaFileWriter is null. This should not happen!"); jsonWriter.endArray(); jsonWriter.endObject(); jsonWriter.close(); - Path tempMetaFilePath = Paths.get(metaFileManager.getMetaFileDirectory().toString(), metaFilePath.toFile().getName() + ".tmp"); + Path tempMetaFilePath = + Paths.get( + metaFileManager.getMetaFileDirectory().toString(), + metaFilePath.toFile().getName() + ".tmp"); - //Rename the tmp file. + // Rename the tmp file. tempMetaFilePath.toFile().renameTo(metaFilePath.toFile()); logger.info("Finished writing to meta file: {}", metaFilePath); @@ -152,16 +172,23 @@ public MetaFileWriterBuilder.UploadStep endMetaFileGeneration() throws IOExcepti /** * Upload the meta file generated to backup file system. * - * @param deleteOnSuccess delete the meta file from local file system if backup is successful. Useful for testing purposes + * @param deleteOnSuccess delete the meta file from local file system if backup is + * successful. Useful for testing purposes * @throws Exception when unable to upload the meta file. */ public void uploadMetaFile(boolean deleteOnSuccess) throws Exception { AbstractBackupPath abstractBackupPath = pathFactory.get(); - abstractBackupPath.parseLocal(metaFilePath.toFile(), AbstractBackupPath.BackupFileType.META_V2); - backupFileSystem.uploadFile(metaFilePath, Paths.get(abstractBackupPath.getRemotePath()), abstractBackupPath, 10, deleteOnSuccess); + abstractBackupPath.parseLocal( + metaFilePath.toFile(), AbstractBackupPath.BackupFileType.META_V2); + backupFileSystem.uploadFile( + metaFilePath, + Paths.get(abstractBackupPath.getRemotePath()), + abstractBackupPath, + 10, + deleteOnSuccess); } - public Path getMetaFilePath(){ + public Path getMetaFilePath() { return metaFilePath; } } diff --git a/priam/src/main/java/com/netflix/priam/backupv2/PrefixGenerator.java b/priam/src/main/java/com/netflix/priam/backupv2/PrefixGenerator.java index af62ec325..881beb5e5 100644 --- a/priam/src/main/java/com/netflix/priam/backupv2/PrefixGenerator.java +++ b/priam/src/main/java/com/netflix/priam/backupv2/PrefixGenerator.java @@ -19,16 +19,15 @@ import com.netflix.priam.config.IConfiguration; import com.netflix.priam.identity.InstanceIdentity; import com.netflix.priam.utils.DateUtil; - -import javax.inject.Inject; import java.nio.file.Path; import java.nio.file.Paths; import java.time.Instant; +import javax.inject.Inject; /** - * This is a utility class to get the backup location of the SSTables/Meta files with backup version 2.0. - * TODO: All this functionality will be used when we have BackupUploadDownloadService. - * Created by aagrawal on 6/5/18. + * This is a utility class to get the backup location of the SSTables/Meta files with backup version + * 2.0. TODO: All this functionality will be used when we have BackupUploadDownloadService. Created + * by aagrawal on 6/5/18. */ public class PrefixGenerator { @@ -42,15 +41,30 @@ public class PrefixGenerator { } public Path getPrefix() { - return Paths.get(configuration.getBackupLocation(), configuration.getBackupPrefix(), getAppNameReverse(), instanceIdentity.getInstance().getToken()); + return Paths.get( + configuration.getBackupLocation(), + configuration.getBackupPrefix(), + getAppNameReverse(), + instanceIdentity.getInstance().getToken()); } public Path getSSTPrefix() { return getPrefix(); } - public Path getSSTLocation(Instant instant, String keyspaceName, String columnfamilyName, String prefix, String fileName) { - return Paths.get(getPrefix().toString(), DateUtil.formatInstant(DateUtil.ddMMyyyyHHmm, instant), keyspaceName, columnfamilyName, prefix, fileName); + public Path getSSTLocation( + Instant instant, + String keyspaceName, + String columnfamilyName, + String prefix, + String fileName) { + return Paths.get( + getPrefix().toString(), + DateUtil.formatInstant(DateUtil.ddMMyyyyHHmm, instant), + keyspaceName, + columnfamilyName, + prefix, + fileName); } public Path getMetaPrefix() { @@ -58,28 +72,32 @@ public Path getMetaPrefix() { } public Path getMetaLocation(Instant instant, String metaFileName) { - return Paths.get(getMetaPrefix().toString(), DateUtil.formatInstant(DateUtil.ddMMyyyyHHmm, instant), metaFileName); + return Paths.get( + getMetaPrefix().toString(), + DateUtil.formatInstant(DateUtil.ddMMyyyyHHmm, instant), + metaFileName); } private String getAppNameReverse() { return new StringBuilder(configuration.getAppName()).reverse().toString(); } - //e.g. mc-3-big-Data.db or sample_cf-ka-7213-Index.db + // e.g. mc-3-big-Data.db or sample_cf-ka-7213-Index.db /** - * Gives the prefix (common name) of the sstable components. Returns null if it is not sstable component - * e.g. mc-3-big-Data.db or ks-cf-ka-7213-Index.db will return mc-3-big or ks-cf-ka-7213 + * Gives the prefix (common name) of the sstable components. Returns null if it is not sstable + * component e.g. mc-3-big-Data.db or ks-cf-ka-7213-Index.db will return mc-3-big or + * ks-cf-ka-7213 + * * @param fileName name of the file for common prefix * @return common prefix of the file, or null, if not identified as sstable component. */ public static final String getSSTFileBase(String fileName) { String prefix = null; - try{ + try { prefix = fileName.substring(0, fileName.lastIndexOf("-")); - }catch (IndexOutOfBoundsException e) - { - //Do nothing + } catch (IndexOutOfBoundsException e) { + // Do nothing } return prefix; diff --git a/priam/src/main/java/com/netflix/priam/cli/Application.java b/priam/src/main/java/com/netflix/priam/cli/Application.java index fa2f49266..6eea39c20 100644 --- a/priam/src/main/java/com/netflix/priam/cli/Application.java +++ b/priam/src/main/java/com/netflix/priam/cli/Application.java @@ -18,15 +18,14 @@ import com.google.inject.Guice; import com.google.inject.Injector; -import com.netflix.priam.config.IConfiguration; import com.netflix.priam.backup.IBackupFileSystem; +import com.netflix.priam.config.IConfiguration; public class Application { - static private Injector injector; + private static Injector injector; static Injector getInjector() { - if (injector == null) - injector = Guice.createInjector(new LightGuiceModule()); + if (injector == null) injector = Guice.createInjector(new LightGuiceModule()); return injector; } diff --git a/priam/src/main/java/com/netflix/priam/cli/Backuper.java b/priam/src/main/java/com/netflix/priam/cli/Backuper.java index 005d7dae6..1b4bb374c 100644 --- a/priam/src/main/java/com/netflix/priam/cli/Backuper.java +++ b/priam/src/main/java/com/netflix/priam/cli/Backuper.java @@ -36,4 +36,4 @@ public static void main(String[] args) { Application.shutdownAdditionalThreads(); } } -} \ No newline at end of file +} diff --git a/priam/src/main/java/com/netflix/priam/cli/IncrementalBackuper.java b/priam/src/main/java/com/netflix/priam/cli/IncrementalBackuper.java index 87f6574de..5075e4f58 100644 --- a/priam/src/main/java/com/netflix/priam/cli/IncrementalBackuper.java +++ b/priam/src/main/java/com/netflix/priam/cli/IncrementalBackuper.java @@ -26,7 +26,8 @@ public class IncrementalBackuper { public static void main(String[] args) { try { Application.initialize(); - IncrementalBackup backuper = Application.getInjector().getInstance(IncrementalBackup.class); + IncrementalBackup backuper = + Application.getInjector().getInstance(IncrementalBackup.class); try { backuper.execute(); } catch (Exception e) { @@ -36,4 +37,4 @@ public static void main(String[] args) { Application.shutdownAdditionalThreads(); } } -} \ No newline at end of file +} diff --git a/priam/src/main/java/com/netflix/priam/cli/LightGuiceModule.java b/priam/src/main/java/com/netflix/priam/cli/LightGuiceModule.java index 8287beb1a..1f59b05b1 100644 --- a/priam/src/main/java/com/netflix/priam/cli/LightGuiceModule.java +++ b/priam/src/main/java/com/netflix/priam/cli/LightGuiceModule.java @@ -17,10 +17,9 @@ package com.netflix.priam.cli; import com.google.inject.AbstractModule; -import com.netflix.priam.config.IConfiguration; import com.netflix.priam.aws.S3FileSystem; import com.netflix.priam.backup.IBackupFileSystem; -import com.netflix.priam.config.PriamConfiguration; +import com.netflix.priam.config.IConfiguration; import com.netflix.priam.identity.IMembership; class LightGuiceModule extends AbstractModule { @@ -31,4 +30,3 @@ protected void configure() { bind(IBackupFileSystem.class).to(S3FileSystem.class); } } - diff --git a/priam/src/main/java/com/netflix/priam/cli/Restorer.java b/priam/src/main/java/com/netflix/priam/cli/Restorer.java index 8fed49ee3..77cb6113e 100644 --- a/priam/src/main/java/com/netflix/priam/cli/Restorer.java +++ b/priam/src/main/java/com/netflix/priam/cli/Restorer.java @@ -18,11 +18,10 @@ import com.netflix.priam.backup.AbstractBackupPath; import com.netflix.priam.restore.Restore; +import java.util.Date; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.Date; - public class Restorer { private static final Logger logger = LoggerFactory.getLogger(Restorer.class); @@ -39,7 +38,8 @@ public static void main(String[] args) { displayHelp(); return; } - AbstractBackupPath path = Application.getInjector().getInstance(AbstractBackupPath.class); + AbstractBackupPath path = + Application.getInjector().getInstance(AbstractBackupPath.class); startTime = path.parseDate(args[0]); endTime = path.parseDate(args[1]); @@ -53,4 +53,4 @@ public static void main(String[] args) { Application.shutdownAdditionalThreads(); } } -} \ No newline at end of file +} diff --git a/priam/src/main/java/com/netflix/priam/cli/StaticMembership.java b/priam/src/main/java/com/netflix/priam/cli/StaticMembership.java index 928c49547..3e6244d27 100644 --- a/priam/src/main/java/com/netflix/priam/cli/StaticMembership.java +++ b/priam/src/main/java/com/netflix/priam/cli/StaticMembership.java @@ -17,16 +17,15 @@ package com.netflix.priam.cli; import com.netflix.priam.identity.IMembership; -import org.apache.cassandra.io.util.FileUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import java.io.FileInputStream; import java.io.IOException; import java.util.Arrays; import java.util.Collection; import java.util.List; import java.util.Properties; +import org.apache.cassandra.io.util.FileUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class StaticMembership implements IMembership { private static final String MEMBERSHIP_PRE = "membership."; @@ -75,8 +74,7 @@ public List getCrossAccountRacMembership() { @Override public int getRacMembershipSize() { - if (racMembership == null) - return 0; + if (racMembership == null) return 0; return racMembership.size(); } @@ -86,12 +84,10 @@ public int getRacCount() { } @Override - public void addACL(Collection listIPs, int from, int to) { - } + public void addACL(Collection listIPs, int from, int to) {} @Override - public void removeACL(Collection listIPs, int from, int to) { - } + public void removeACL(Collection listIPs, int from, int to) {} @Override public List listACL(int from, int to) { @@ -99,6 +95,5 @@ public List listACL(int from, int to) { } @Override - public void expandRacMembership(int count) { - } -} \ No newline at end of file + public void expandRacMembership(int count) {} +} diff --git a/priam/src/main/java/com/netflix/priam/cluster/management/Compaction.java b/priam/src/main/java/com/netflix/priam/cluster/management/Compaction.java index cdd727c00..73ea3e257 100644 --- a/priam/src/main/java/com/netflix/priam/cluster/management/Compaction.java +++ b/priam/src/main/java/com/netflix/priam/cluster/management/Compaction.java @@ -1,16 +1,14 @@ /** * Copyright 2018 Netflix, Inc. - *

- * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and + * + *

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file + * except in compliance with the License. You may obtain a copy of the License at + * + *

http://www.apache.org/licenses/LICENSE-2.0 + * + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing permissions and * limitations under the License. */ package com.netflix.priam.cluster.management; @@ -21,83 +19,99 @@ import com.netflix.priam.merics.CompactionMeasurement; import com.netflix.priam.scheduler.CronTimer; import com.netflix.priam.scheduler.TaskTimer; -import org.apache.commons.collections4.CollectionUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import javax.inject.Inject; -import javax.inject.Singleton; import java.util.HashMap; import java.util.List; import java.util.Map; +import javax.inject.Inject; +import javax.inject.Singleton; +import org.apache.commons.collections4.CollectionUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; -/** - * Utility class to compact the keyspaces/columnfamilies - * Created by aagrawal on 1/25/18. - */ +/** Utility class to compact the keyspaces/columnfamilies Created by aagrawal on 1/25/18. */ @Singleton public class Compaction extends IClusterManagement { private static final Logger logger = LoggerFactory.getLogger(Compaction.class); private final IConfiguration config; private final CassandraOperations cassandraOperations; + @Inject - public Compaction(IConfiguration config, CassandraOperations cassandraOperations, CompactionMeasurement compactionMeasurement) { + public Compaction( + IConfiguration config, + CassandraOperations cassandraOperations, + CompactionMeasurement compactionMeasurement) { super(config, Task.COMPACTION, compactionMeasurement); this.config = config; this.cassandraOperations = cassandraOperations; } - - final Map> getCompactionIncludeFilter(IConfiguration config) throws Exception { - Map> columnFamilyFilter = BackupRestoreUtil.getFilter(config.getCompactionIncludeCFList()); + final Map> getCompactionIncludeFilter(IConfiguration config) + throws Exception { + Map> columnFamilyFilter = + BackupRestoreUtil.getFilter(config.getCompactionIncludeCFList()); logger.info("Compaction: Override for include CF provided by user: {}", columnFamilyFilter); return columnFamilyFilter; } - final Map> getCompactionExcludeFilter(IConfiguration config) throws Exception { - Map> columnFamilyFilter = BackupRestoreUtil.getFilter(config.getCompactionExcludeCFList()); + + final Map> getCompactionExcludeFilter(IConfiguration config) + throws Exception { + Map> columnFamilyFilter = + BackupRestoreUtil.getFilter(config.getCompactionExcludeCFList()); logger.info("Compaction: Override for exclude CF provided by user: {}", columnFamilyFilter); return columnFamilyFilter; } + final Map> getCompactionFilterCfs(IConfiguration config) throws Exception { final Map> includeFilter = getCompactionIncludeFilter(config); final Map> excludeFilter = getCompactionExcludeFilter(config); final Map> allColumnfamilies = cassandraOperations.getColumnfamilies(); Map> result = new HashMap<>(); - allColumnfamilies.forEach((keyspaceName, columnfamilies) -> { - if (SchemaConstant.isSystemKeyspace(keyspaceName)) //no need to compact system keyspaces. - return; - - if (excludeFilter != null && excludeFilter.containsKey(keyspaceName)) { - List excludeCFFilter = excludeFilter.get(keyspaceName); - //Is CF list null/empty? If yes, then exclude all CF's for this keyspace. - if (excludeCFFilter == null || excludeCFFilter.isEmpty()) + allColumnfamilies.forEach( + (keyspaceName, columnfamilies) -> { + if (SchemaConstant.isSystemKeyspace( + keyspaceName)) // no need to compact system keyspaces. return; - columnfamilies = (List) CollectionUtils.removeAll(columnfamilies, excludeCFFilter); - } - if (includeFilter != null) { - //Include filter is not empty and this keyspace is not provided in include filter. Ignore processing of this keyspace. - if (!includeFilter.containsKey(keyspaceName)) - return; - List includeCFFilter = includeFilter.get(keyspaceName); - //If include filter is empty or null, it means include all. - //If not, then we need to find intersection of CF's which are present and one which are configured to compact. - if (includeCFFilter != null && !includeCFFilter.isEmpty()) //If include filter is empty or null, it means include all. - columnfamilies = (List) CollectionUtils.intersection(columnfamilies, includeCFFilter); - } - if (columnfamilies != null && !columnfamilies.isEmpty()) - result.put(keyspaceName, columnfamilies); - }); + + if (excludeFilter != null && excludeFilter.containsKey(keyspaceName)) { + List excludeCFFilter = excludeFilter.get(keyspaceName); + // Is CF list null/empty? If yes, then exclude all CF's for this keyspace. + if (excludeCFFilter == null || excludeCFFilter.isEmpty()) return; + columnfamilies = + (List) + CollectionUtils.removeAll(columnfamilies, excludeCFFilter); + } + if (includeFilter != null) { + // Include filter is not empty and this keyspace is not provided in include + // filter. Ignore processing of this keyspace. + if (!includeFilter.containsKey(keyspaceName)) return; + List includeCFFilter = includeFilter.get(keyspaceName); + // If include filter is empty or null, it means include all. + // If not, then we need to find intersection of CF's which are present and + // one which are configured to compact. + if (includeCFFilter != null + && !includeCFFilter + .isEmpty()) // If include filter is empty or null, it means + // include all. + columnfamilies = + (List) + CollectionUtils.intersection( + columnfamilies, includeCFFilter); + } + if (columnfamilies != null && !columnfamilies.isEmpty()) + result.put(keyspaceName, columnfamilies); + }); return result; } /* - * @return the keyspace(s) compacted. List can be empty but never null. - */ + * @return the keyspace(s) compacted. List can be empty but never null. + */ protected String runTask() throws Exception { final Map> columnfamilies = getCompactionFilterCfs(config); if (!columnfamilies.isEmpty()) for (Map.Entry> entry : columnfamilies.entrySet()) { - cassandraOperations.forceKeyspaceCompaction(entry.getKey(), entry.getValue().toArray(new String[0])); + cassandraOperations.forceKeyspaceCompaction( + entry.getKey(), entry.getValue().toArray(new String[0])); } return columnfamilies.toString(); } @@ -105,10 +119,11 @@ protected String runTask() throws Exception { * Timer to be used for compaction interval. * * @param config {@link IConfiguration} to get configuration details from priam. - * @return the timer to be used for compaction interval from {@link IConfiguration#getCompactionCronExpression()} + * @return the timer to be used for compaction interval from {@link + * IConfiguration#getCompactionCronExpression()} * @throws Exception If the cron expression is invalid. */ public static TaskTimer getTimer(IConfiguration config) throws Exception { return CronTimer.getCronTimer(Task.COMPACTION.name(), config.getCompactionCronExpression()); } -} \ No newline at end of file +} diff --git a/priam/src/main/java/com/netflix/priam/cluster/management/Flush.java b/priam/src/main/java/com/netflix/priam/cluster/management/Flush.java index 17947ebf8..3379772de 100644 --- a/priam/src/main/java/com/netflix/priam/cluster/management/Flush.java +++ b/priam/src/main/java/com/netflix/priam/cluster/management/Flush.java @@ -1,16 +1,14 @@ /** * Copyright 2017 Netflix, Inc. - *

- * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and + * + *

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file + * except in compliance with the License. You may obtain a copy of the License at + * + *

http://www.apache.org/licenses/LICENSE-2.0 + * + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing permissions and * limitations under the License. */ package com.netflix.priam.cluster.management; @@ -21,20 +19,16 @@ import com.netflix.priam.scheduler.CronTimer; import com.netflix.priam.scheduler.TaskTimer; import com.netflix.priam.scheduler.UnsupportedTypeException; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import javax.inject.Inject; -import javax.inject.Singleton; import java.io.IOException; import java.util.ArrayList; import java.util.List; import java.util.concurrent.ExecutionException; +import javax.inject.Inject; +import javax.inject.Singleton; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; -/** - * Utility to flush Keyspaces from memtable to disk - * Created by vinhn on 10/12/16. - */ +/** Utility to flush Keyspaces from memtable to disk Created by vinhn on 10/12/16. */ @Singleton public class Flush extends IClusterManagement { private static final Logger logger = LoggerFactory.getLogger(Flush.class); @@ -44,7 +38,10 @@ public class Flush extends IClusterManagement { private List keyspaces = new ArrayList<>(); @Inject - public Flush(IConfiguration config, CassandraOperations cassandraOperations, NodeToolFlushMeasurement nodeToolFlushMeasurement) { + public Flush( + IConfiguration config, + CassandraOperations cassandraOperations, + NodeToolFlushMeasurement nodeToolFlushMeasurement) { super(config, Task.FLUSH, nodeToolFlushMeasurement); this.config = config; this.cassandraOperations = cassandraOperations; @@ -57,7 +54,7 @@ public Flush(IConfiguration config, CassandraOperations cassandraOperations, Nod protected String runTask() throws Exception { List flushed = new ArrayList<>(); - //Get keyspaces to flush + // Get keyspaces to flush deriveKeyspaces(); if (this.keyspaces == null || this.keyspaces.isEmpty()) { @@ -65,14 +62,14 @@ protected String runTask() throws Exception { return flushed.toString(); } - //If flush is for certain keyspaces, validate keyspace exist + // If flush is for certain keyspaces, validate keyspace exist for (String keyspace : keyspaces) { if (!cassandraOperations.getKeyspaces().contains(keyspace)) { throw new IllegalArgumentException("Keyspace [" + keyspace + "] does not exist."); } - if (SchemaConstant.isSystemKeyspace(keyspace)) //no need to flush system keyspaces. - continue; + if (SchemaConstant.isSystemKeyspace(keyspace)) // no need to flush system keyspaces. + continue; try { cassandraOperations.forceKeyspaceFlush(keyspace); @@ -89,7 +86,7 @@ protected String runTask() throws Exception { Derive keyspace(s) to flush in the following order: explicit list provided by caller, property, or all keyspaces. */ private void deriveKeyspaces() throws Exception { - //== get value from property + // == get value from property String raw = this.config.getFlushKeyspaces(); if (raw != null && !raw.isEmpty()) { String k[] = raw.split(","); @@ -100,7 +97,7 @@ private void deriveKeyspaces() throws Exception { return; } - //== no override via FP, default to all keyspaces + // == no override via FP, default to all keyspaces this.keyspaces = cassandraOperations.getKeyspaces(); } @@ -109,10 +106,12 @@ private void deriveKeyspaces() throws Exception { * * @param config {@link IConfiguration} to get configuration details from priam. * @return the timer to be used for flush interval. - *

- * If {@link IConfiguration#getFlushSchedulerType()} is {@link com.netflix.priam.scheduler.SchedulerType#HOUR} then it expects {@link IConfiguration#getFlushInterval()} in the format of hour=x or daily=x - *

- * If {@link IConfiguration#getFlushSchedulerType()} is {@link com.netflix.priam.scheduler.SchedulerType#CRON} then it expects a valid CRON expression from {@link IConfiguration#getFlushCronExpression()} + *

If {@link IConfiguration#getFlushSchedulerType()} is {@link + * com.netflix.priam.scheduler.SchedulerType#HOUR} then it expects {@link + * IConfiguration#getFlushInterval()} in the format of hour=x or daily=x + *

If {@link IConfiguration#getFlushSchedulerType()} is {@link + * com.netflix.priam.scheduler.SchedulerType#CRON} then it expects a valid CRON expression + * from {@link IConfiguration#getFlushCronExpression()} * @throws Exception if the configurations are wrong. .e.g invalid cron expression. */ public static TaskTimer getTimer(IConfiguration config) throws Exception { @@ -120,31 +119,42 @@ public static TaskTimer getTimer(IConfiguration config) throws Exception { CronTimer cronTimer = null; switch (config.getFlushSchedulerType()) { case HOUR: - String timerVal = config.getFlushInterval(); //e.g. hour=0 or daily=10 - if (timerVal == null) - return null; + String timerVal = config.getFlushInterval(); // e.g. hour=0 or daily=10 + if (timerVal == null) return null; String s[] = timerVal.split("="); if (s.length != 2) { - throw new IllegalArgumentException("Flush interval format is invalid. Expecting name=value, received: " + timerVal); + throw new IllegalArgumentException( + "Flush interval format is invalid. Expecting name=value, received: " + + timerVal); } String name = s[0].toUpperCase(); Integer time = new Integer(s[1]); switch (name) { case "HOUR": - cronTimer = new CronTimer(Task.FLUSH.name(), time, 0); //minute, sec after each hour + cronTimer = + new CronTimer( + Task.FLUSH.name(), time, 0); // minute, sec after each hour break; case "DAILY": - cronTimer = new CronTimer(Task.FLUSH.name(), time, 0, 0); //hour, minute, sec to run on a daily basis + cronTimer = + new CronTimer( + Task.FLUSH.name(), + time, + 0, + 0); // hour, minute, sec to run on a daily basis break; default: - throw new UnsupportedTypeException("Flush interval type is invalid. Expecting \"hour, daily\", received: " + name); + throw new UnsupportedTypeException( + "Flush interval type is invalid. Expecting \"hour, daily\", received: " + + name); } break; case CRON: - cronTimer = CronTimer.getCronTimer(Task.FLUSH.name(), config.getFlushCronExpression()); + cronTimer = + CronTimer.getCronTimer(Task.FLUSH.name(), config.getFlushCronExpression()); break; } return cronTimer; } -} \ No newline at end of file +} diff --git a/priam/src/main/java/com/netflix/priam/cluster/management/IClusterManagement.java b/priam/src/main/java/com/netflix/priam/cluster/management/IClusterManagement.java index be8737517..a6df274c2 100644 --- a/priam/src/main/java/com/netflix/priam/cluster/management/IClusterManagement.java +++ b/priam/src/main/java/com/netflix/priam/cluster/management/IClusterManagement.java @@ -1,16 +1,14 @@ /** * Copyright 2017 Netflix, Inc. - *

- * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and + * + *

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file + * except in compliance with the License. You may obtain a copy of the License at + * + *

http://www.apache.org/licenses/LICENSE-2.0 + * + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing permissions and * limitations under the License. */ package com.netflix.priam.cluster.management; @@ -19,18 +17,18 @@ import com.netflix.priam.merics.IMeasurement; import com.netflix.priam.scheduler.Task; import com.netflix.priam.utils.CassandraMonitor; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; -/** - * Created by vinhn on 10/12/16. - */ +/** Created by vinhn on 10/12/16. */ public abstract class IClusterManagement extends Task { - public enum Task {FLUSH, COMPACTION} + public enum Task { + FLUSH, + COMPACTION + } private static final Logger logger = LoggerFactory.getLogger(IClusterManagement.class); private final Task taskType; @@ -58,11 +56,15 @@ public void execute() throws Exception { try { String result = runTask(); measurement.incrementSuccess(); - logger.info("Successfully finished executing the cluster management task: {} with result: {}", taskType, result); + logger.info( + "Successfully finished executing the cluster management task: {} with result: {}", + taskType, + result); if (result.isEmpty()) { - logger.warn("{} task completed successfully but no action was done.", taskType.name()); + logger.warn( + "{} task completed successfully but no action was done.", taskType.name()); } - } catch (Exception e){ + } catch (Exception e) { measurement.incrementFailure(); throw new Exception("Exception during execution of operation: " + taskType.name(), e); } finally { diff --git a/priam/src/main/java/com/netflix/priam/cluster/management/SchemaConstant.java b/priam/src/main/java/com/netflix/priam/cluster/management/SchemaConstant.java index 02b937d7c..48029ce9e 100644 --- a/priam/src/main/java/com/netflix/priam/cluster/management/SchemaConstant.java +++ b/priam/src/main/java/com/netflix/priam/cluster/management/SchemaConstant.java @@ -17,12 +17,9 @@ package com.netflix.priam.cluster.management; import com.google.common.collect.ImmutableSet; - import java.util.Set; -/** - * Created by aagrawal on 3/6/18. - */ +/** Created by aagrawal on 3/6/18. */ class SchemaConstant { private static final String SYSTEM_KEYSPACE_NAME = "system"; private static final String SCHEMA_KEYSPACE_NAME = "system_schema"; @@ -32,9 +29,15 @@ class SchemaConstant { private static final String DSE_SYSTEM = "dse_system"; private static final Set SYSTEM_KEYSPACE_NAMES = - ImmutableSet.of(SYSTEM_KEYSPACE_NAME, SCHEMA_KEYSPACE_NAME, TRACE_KEYSPACE_NAME, AUTH_KEYSPACE_NAME, DISTRIBUTED_KEYSPACE_NAME, DSE_SYSTEM); + ImmutableSet.of( + SYSTEM_KEYSPACE_NAME, + SCHEMA_KEYSPACE_NAME, + TRACE_KEYSPACE_NAME, + AUTH_KEYSPACE_NAME, + DISTRIBUTED_KEYSPACE_NAME, + DSE_SYSTEM); - public static final boolean isSystemKeyspace(String keyspace){ + public static final boolean isSystemKeyspace(String keyspace) { return SYSTEM_KEYSPACE_NAMES.contains(keyspace.toLowerCase()); } -} \ No newline at end of file +} diff --git a/priam/src/main/java/com/netflix/priam/compress/ChunkedStream.java b/priam/src/main/java/com/netflix/priam/compress/ChunkedStream.java index 8bfd4635f..b23b29dd4 100644 --- a/priam/src/main/java/com/netflix/priam/compress/ChunkedStream.java +++ b/priam/src/main/java/com/netflix/priam/compress/ChunkedStream.java @@ -16,18 +16,14 @@ */ package com.netflix.priam.compress; -import org.apache.commons.io.IOUtils; -import org.xerial.snappy.SnappyOutputStream; - import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.InputStream; import java.util.Iterator; +import org.apache.commons.io.IOUtils; +import org.xerial.snappy.SnappyOutputStream; -/** - * Byte iterator representing compressed data. - * Uses snappy compression - */ +/** Byte iterator representing compressed data. Uses snappy compression */ public class ChunkedStream implements Iterator { private boolean hasnext = true; private final ByteArrayOutputStream bos; @@ -55,8 +51,7 @@ public byte[] next() { int count; while ((count = origin.read(data, 0, data.length)) != -1) { compress.write(data, 0, count); - if (bos.size() >= chunkSize) - return returnSafe(); + if (bos.size() >= chunkSize) return returnSafe(); } // We don't have anything else to read hence set to false. return done(); @@ -82,7 +77,5 @@ private byte[] returnSafe() throws IOException { } @Override - public void remove() { - } - + public void remove() {} } diff --git a/priam/src/main/java/com/netflix/priam/compress/ICompression.java b/priam/src/main/java/com/netflix/priam/compress/ICompression.java index 48139ef8c..bab9b0f0f 100644 --- a/priam/src/main/java/com/netflix/priam/compress/ICompression.java +++ b/priam/src/main/java/com/netflix/priam/compress/ICompression.java @@ -17,7 +17,6 @@ package com.netflix.priam.compress; import com.google.inject.ImplementedBy; - import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; @@ -27,17 +26,17 @@ public interface ICompression { enum CompressionAlgorithm { - SNAPPY, LZ4, NONE + SNAPPY, + LZ4, + NONE } /** - * Uncompress the input stream and write to the output stream. - * Closes both input and output streams + * Uncompress the input stream and write to the output stream. Closes both input and output + * streams */ void decompressAndClose(InputStream input, OutputStream output) throws IOException; - /** - * Produces chunks of compressed data. - */ + /** Produces chunks of compressed data. */ Iterator compress(InputStream is, long chunkSize) throws IOException; } diff --git a/priam/src/main/java/com/netflix/priam/compress/SnappyCompression.java b/priam/src/main/java/com/netflix/priam/compress/SnappyCompression.java index cdb22e468..084bcf04e 100644 --- a/priam/src/main/java/com/netflix/priam/compress/SnappyCompression.java +++ b/priam/src/main/java/com/netflix/priam/compress/SnappyCompression.java @@ -16,16 +16,12 @@ */ package com.netflix.priam.compress; -import org.apache.commons.io.IOUtils; -import org.xerial.snappy.SnappyInputStream; - import java.io.*; import java.util.Iterator; +import org.apache.commons.io.IOUtils; +import org.xerial.snappy.SnappyInputStream; -/** - * Class to generate compressed chunks of data from an input stream using - * SnappyCompression - */ +/** Class to generate compressed chunks of data from an input stream using SnappyCompression */ public class SnappyCompression implements ICompression { private static final int BUFFER = 2 * 1024; @@ -46,8 +42,8 @@ public void decompressAndClose(InputStream input, OutputStream output) throws IO private void decompress(InputStream input, OutputStream output) throws IOException { byte data[] = new byte[BUFFER]; - try(BufferedOutputStream dest1 = new BufferedOutputStream(output, BUFFER); - SnappyInputStream is = new SnappyInputStream(new BufferedInputStream(input))) { + try (BufferedOutputStream dest1 = new BufferedOutputStream(output, BUFFER); + SnappyInputStream is = new SnappyInputStream(new BufferedInputStream(input))) { int c; while ((c = is.read(data, 0, BUFFER)) != -1) { dest1.write(data, 0, c); diff --git a/priam/src/main/java/com/netflix/priam/config/BackupRestoreConfig.java b/priam/src/main/java/com/netflix/priam/config/BackupRestoreConfig.java index 7046a0a2f..a7675594a 100644 --- a/priam/src/main/java/com/netflix/priam/config/BackupRestoreConfig.java +++ b/priam/src/main/java/com/netflix/priam/config/BackupRestoreConfig.java @@ -1,29 +1,23 @@ /** * Copyright 2018 Netflix, Inc. - *

- * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and + * + *

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file + * except in compliance with the License. You may obtain a copy of the License at + * + *

http://www.apache.org/licenses/LICENSE-2.0 + * + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing permissions and * limitations under the License. */ package com.netflix.priam.config; import com.netflix.priam.configSource.IConfigSource; - import javax.inject.Inject; -/** - * Implementation of IBackupRestoreConfig. - * Created by aagrawal on 6/26/18. - */ -public class BackupRestoreConfig implements IBackupRestoreConfig{ +/** Implementation of IBackupRestoreConfig. Created by aagrawal on 6/26/18. */ +public class BackupRestoreConfig implements IBackupRestoreConfig { private final IConfigSource config; @@ -32,7 +26,7 @@ public BackupRestoreConfig(IConfigSource config) { this.config = config; } - public String getSnapshotMetaServiceCronExpression(){ + public String getSnapshotMetaServiceCronExpression() { return config.get("priam.snapshot.meta.cron", "-1"); } } diff --git a/priam/src/main/java/com/netflix/priam/config/IBackupRestoreConfig.java b/priam/src/main/java/com/netflix/priam/config/IBackupRestoreConfig.java index 84f301b4d..391ec4898 100644 --- a/priam/src/main/java/com/netflix/priam/config/IBackupRestoreConfig.java +++ b/priam/src/main/java/com/netflix/priam/config/IBackupRestoreConfig.java @@ -1,16 +1,14 @@ /** * Copyright 2018 Netflix, Inc. - *

- * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and + * + *

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file + * except in compliance with the License. You may obtain a copy of the License at + * + *

http://www.apache.org/licenses/LICENSE-2.0 + * + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing permissions and * limitations under the License. */ package com.netflix.priam.config; @@ -18,8 +16,8 @@ import com.google.inject.ImplementedBy; /** - * This interface is to abstract out the backup and restore configuration used by Priam. Goal is to eventually have each module/functionality to have its own Config. - * Created by aagrawal on 6/26/18. + * This interface is to abstract out the backup and restore configuration used by Priam. Goal is to + * eventually have each module/functionality to have its own Config. Created by aagrawal on 6/26/18. */ @ImplementedBy(BackupRestoreConfig.class) public interface IBackupRestoreConfig { @@ -28,10 +26,11 @@ public interface IBackupRestoreConfig { * Cron expression to be used for snapshot meta service. Use "-1" to disable the service. * * @return Snapshot Meta Service cron expression for generating manifest.json - * @see quartz-scheduler + * @see quartz-scheduler * @see http://www.cronmaker.com */ - default String getSnapshotMetaServiceCronExpression(){ - return "-1"; - } + default String getSnapshotMetaServiceCronExpression() { + return "-1"; + } } diff --git a/priam/src/main/java/com/netflix/priam/config/IConfiguration.java b/priam/src/main/java/com/netflix/priam/config/IConfiguration.java index a2bd43163..031b484f4 100644 --- a/priam/src/main/java/com/netflix/priam/config/IConfiguration.java +++ b/priam/src/main/java/com/netflix/priam/config/IConfiguration.java @@ -19,75 +19,62 @@ import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.inject.ImplementedBy; -import com.netflix.priam.tuner.JVMOption; -import com.netflix.priam.config.PriamConfiguration; -import com.netflix.priam.tuner.GCType; import com.netflix.priam.identity.config.InstanceDataRetriever; import com.netflix.priam.scheduler.SchedulerType; import com.netflix.priam.scheduler.UnsupportedTypeException; - +import com.netflix.priam.tuner.GCType; +import com.netflix.priam.tuner.JVMOption; import java.util.List; import java.util.Map; -/** - * Interface for Priam's configuration - */ +/** Interface for Priam's configuration */ @ImplementedBy(PriamConfiguration.class) public interface IConfiguration { void initialize(); - /** - * @return Path to the home dir of Cassandra - */ + /** @return Path to the home dir of Cassandra */ String getCassHome(); String getYamlLocation(); - /** - * @return Path to jvm.options file. This is used to pass JVM options to Cassandra. - */ + /** @return Path to jvm.options file. This is used to pass JVM options to Cassandra. */ String getJVMOptionsFileLocation(); /** - * @return Type of garbage collection mechanism to use for Cassandra. Supported values are CMS,G1GC + * @return Type of garbage collection mechanism to use for Cassandra. Supported values are + * CMS,G1GC */ GCType getGCType() throws UnsupportedTypeException; - /** - * @return Set of JVM options to exclude/comment. - */ + /** @return Set of JVM options to exclude/comment. */ Map getJVMExcludeSet(); - /** - * @return Set of JMV options to add/upsert - */ + /** @return Set of JMV options to add/upsert */ Map getJVMUpsertSet(); - /** - * @return Path to Cassandra startup script - */ + /** @return Path to Cassandra startup script */ String getCassStartupScript(); - /** - * @return Path to Cassandra stop sript - */ + /** @return Path to Cassandra stop sript */ String getCassStopScript(); /** - * @return int representing how many seconds Priam should fail healthchecks for before gracefully draining (nodetool drain) - * cassandra prior to stop. If this number is negative then no draining occurs and Priam immediately stops Cassanddra - * using the provided stop script. If this number is >= 0 then Priam will fail healthchecks for this number of - * seconds before gracefully draining cassandra (nodetool drain) and stopping cassandra with the stop script. + * @return int representing how many seconds Priam should fail healthchecks for before + * gracefully draining (nodetool drain) cassandra prior to stop. If this number is negative + * then no draining occurs and Priam immediately stops Cassanddra using the provided stop + * script. If this number is >= 0 then Priam will fail healthchecks for this number of + * seconds before gracefully draining cassandra (nodetool drain) and stopping cassandra with + * the stop script. */ int getGracefulDrainHealthWaitSeconds(); /** - * @return int representing how often (in seconds) Priam should auto-remediate Cassandra process crash - * If zero, Priam will restart Cassandra whenever it notices it is crashed - * If a positive number, Priam will restart cassandra no more than once in that number of seconds. For example a - * value of 60 means that Priam will only restart Cassandra once per 60 seconds - * If a negative number, Priam will not restart Cassandra due to crash at all + * @return int representing how often (in seconds) Priam should auto-remediate Cassandra process + * crash If zero, Priam will restart Cassandra whenever it notices it is crashed If a + * positive number, Priam will restart cassandra no more than once in that number of + * seconds. For example a value of 60 means that Priam will only restart Cassandra once per + * 60 seconds If a negative number, Priam will not restart Cassandra due to crash at all */ int getRemediateDeadCassandraRate(); @@ -98,14 +85,10 @@ public interface IConfiguration { */ String getBackupLocation(); - /** - * @return Get Backup retention in days - */ + /** @return Get Backup retention in days */ int getBackupRetentionDays(); - /** - * @return Get list of racs to backup. Backup all racs if empty - */ + /** @return Get list of racs to backup. Backup all racs if empty */ List getBackupRacs(); /** @@ -116,166 +99,135 @@ public interface IConfiguration { String getBackupPrefix(); /** - * @return Location containing backup files. Typically bucket name followed by path - * to the clusters backup + * @return Location containing backup files. Typically bucket name followed by path to the + * clusters backup */ String getRestorePrefix(); - /** - * @param prefix Set the current restore prefix - */ + /** @param prefix Set the current restore prefix */ void setRestorePrefix(String prefix); - /** - * @return Location of the local data dir - */ + /** @return Location of the local data dir */ String getDataFileLocation(); /** - * Path where cassandra logs should be stored. This is passed to Cassandra as where to store logs. + * Path where cassandra logs should be stored. This is passed to Cassandra as where to store + * logs. + * * @return Path to cassandra logs. */ String getLogDirLocation(); - /** - * @return Location of the hints data directory - */ + /** @return Location of the hints data directory */ String getHintsLocation(); - /** - * @return Location of local cache - */ + /** @return Location of local cache */ String getCacheLocation(); - /** - * @return Location of local commit log dir - */ + /** @return Location of local commit log dir */ String getCommitLogLocation(); - /** - * @return Remote commit log location for backups - */ + /** @return Remote commit log location for backups */ String getBackupCommitLogLocation(); - /** - * @return Preferred data part size for multi part uploads - */ + /** @return Preferred data part size for multi part uploads */ long getBackupChunkSize(); - /** - * @return Cassandra's JMX port - */ + /** @return Cassandra's JMX port */ default int getJmxPort() { return 7199; } - /** - * @return Cassandra's JMX username - */ + /** @return Cassandra's JMX username */ default String getJmxUsername() { return null; } - /** - * @return Cassandra's JMX password - */ + /** @return Cassandra's JMX password */ default String getJmxPassword() { return null; } - /** - * @return Enables Remote JMX connections n C* - */ + /** @return Enables Remote JMX connections n C* */ default boolean enableRemoteJMX() { return false; } - /** - * @return Cassandra storage/cluster communication port - */ + /** @return Cassandra storage/cluster communication port */ int getStoragePort(); int getSSLStoragePort(); - /** - * @return Cassandra's thrift port - */ + /** @return Cassandra's thrift port */ int getThriftPort(); - /** - * @return Port for CQL binary transport. - */ + /** @return Port for CQL binary transport. */ int getNativeTransportPort(); - /** - * @return Snitch to be used in cassandra.yaml - */ + /** @return Snitch to be used in cassandra.yaml */ String getSnitch(); - /** - * @return Cluster name - */ + /** @return Cluster name */ String getAppName(); - /** - * @return RAC (or zone for AWS) - */ + /** @return RAC (or zone for AWS) */ String getRac(); - /** - * @return List of all RAC used for the cluster - */ + /** @return List of all RAC used for the cluster */ List getRacs(); - /** - * @return Local hostmame - */ + /** @return Local hostmame */ String getHostname(); - /** - * @return Get instance name (for AWS) - */ + /** @return Get instance name (for AWS) */ String getInstanceName(); - /** - * @return Max heap size be used for Cassandra - */ + /** @return Max heap size be used for Cassandra */ String getHeapSize(); - /** - * @return New heap size for Cassandra - */ + /** @return New heap size for Cassandra */ String getHeapNewSize(); /** - * Cron expression to be used to schedule regular compactions. Use "-1" to disable the CRON. Default: -1 + * Cron expression to be used to schedule regular compactions. Use "-1" to disable the CRON. + * Default: -1 * * @return Compaction cron expression. - * @see quartz-scheduler + * @see quartz-scheduler * @see http://www.cronmaker.com To build new cron timer */ - default String getCompactionCronExpression(){ + default String getCompactionCronExpression() { return "-1"; } - /** - * Column Family(ies), comma delimited, to start compactions (user-initiated or on CRON). - * Note 1: The expected format is keyspace.cfname. If no value is provided then compaction is scheduled for all KS,CF(s) - * Note 2: CF name allows special character "*" to denote all the columnfamilies in a given keyspace. e.g. keyspace1.* denotes all the CFs in keyspace1. - * Note 3: {@link #getCompactionExcludeCFList()} is applied first to exclude CF/keyspace and then {@link #getCompactionIncludeCFList()} is applied to include the CF's/keyspaces. - * @return Column Family(ies), comma delimited, to start compactions. If no filter is applied, returns null. - */ - default String getCompactionIncludeCFList(){ + /** + * Column Family(ies), comma delimited, to start compactions (user-initiated or on CRON). Note + * 1: The expected format is keyspace.cfname. If no value is provided then compaction is + * scheduled for all KS,CF(s) Note 2: CF name allows special character "*" to denote all the + * columnfamilies in a given keyspace. e.g. keyspace1.* denotes all the CFs in keyspace1. Note + * 3: {@link #getCompactionExcludeCFList()} is applied first to exclude CF/keyspace and then + * {@link #getCompactionIncludeCFList()} is applied to include the CF's/keyspaces. + * + * @return Column Family(ies), comma delimited, to start compactions. If no filter is applied, + * returns null. + */ + default String getCompactionIncludeCFList() { return null; } /** - * Column family(ies), comma delimited, to exclude while starting compaction (user-initiated or on CRON). - * Note 1: The expected format is keyspace.cfname. If no value is provided then compaction is scheduled for all KS,CF(s) - * Note 2: CF name allows special character "*" to denote all the columnfamilies in a given keyspace. e.g. keyspace1.* denotes all the CFs in keyspace1. - * Note 3: {@link #getCompactionExcludeCFList()} is applied first to exclude CF/keyspace and then {@link #getCompactionIncludeCFList()} is applied to include the CF's/keyspaces. - * @return Column Family(ies), comma delimited, to exclude from compactions. If no filter is applied, returns null. + * Column family(ies), comma delimited, to exclude while starting compaction (user-initiated or + * on CRON). Note 1: The expected format is keyspace.cfname. If no value is provided then + * compaction is scheduled for all KS,CF(s) Note 2: CF name allows special character "*" to + * denote all the columnfamilies in a given keyspace. e.g. keyspace1.* denotes all the CFs in + * keyspace1. Note 3: {@link #getCompactionExcludeCFList()} is applied first to exclude + * CF/keyspace and then {@link #getCompactionIncludeCFList()} is applied to include the + * CF's/keyspaces. + * + * @return Column Family(ies), comma delimited, to exclude from compactions. If no filter is + * applied, returns null. */ - default String getCompactionExcludeCFList(){ + default String getCompactionExcludeCFList() { return null; } @@ -290,7 +242,8 @@ default String getCompactionExcludeCFList(){ * Cron expression to be used for snapshot backups. * * @return Backup cron expression for snapshots - * @see quartz-scheduler + * @see quartz-scheduler * @see http://www.cronmaker.com To build new cron timer */ String getBackupCronExpression(); @@ -298,202 +251,183 @@ default String getCompactionExcludeCFList(){ /** * Backup scheduler type to use for backup. * - * @return Type of scheduler to use for backup. Note the default is TIMER based i.e. to use {@link #getBackupHour()}. - * If value of "CRON" is provided it starts using {@link #getBackupCronExpression()}. + * @return Type of scheduler to use for backup. Note the default is TIMER based i.e. to use + * {@link #getBackupHour()}. If value of "CRON" is provided it starts using {@link + * #getBackupCronExpression()}. * @throws UnsupportedTypeException if the scheduler type is not CRON/HOUR. */ SchedulerType getBackupSchedulerType() throws UnsupportedTypeException; - /** - * Column Family(ies), comma delimited, to include during snapshot backup. - * Note 1: The expected format is keyspace.cfname. If no value is provided then snapshot contains all KS,CF(s) - * Note 2: CF name allows special character "*" to denote all the columnfamilies in a given keyspace. e.g. keyspace1.* denotes all the CFs in keyspace1. - * Note 3: {@link #getSnapshotExcludeCFList()} is applied first to exclude CF/keyspace and then {@link #getSnapshotIncludeCFList()} is applied to include the CF's/keyspaces. + * Column Family(ies), comma delimited, to include during snapshot backup. Note 1: The expected + * format is keyspace.cfname. If no value is provided then snapshot contains all KS,CF(s) Note + * 2: CF name allows special character "*" to denote all the columnfamilies in a given keyspace. + * e.g. keyspace1.* denotes all the CFs in keyspace1. Note 3: {@link + * #getSnapshotExcludeCFList()} is applied first to exclude CF/keyspace and then {@link + * #getSnapshotIncludeCFList()} is applied to include the CF's/keyspaces. * - * @return Column Family(ies), comma delimited, to include in snapshot backup. If no filter is applied, returns null. + * @return Column Family(ies), comma delimited, to include in snapshot backup. If no filter is + * applied, returns null. */ default String getSnapshotIncludeCFList() { return null; } /** - * Column family(ies), comma delimited, to exclude during snapshot backup. - * Note 1: The expected format is keyspace.cfname. If no value is provided then snapshot is scheduled for all KS,CF(s) - * Note 2: CF name allows special character "*" to denote all the columnfamilies in a given keyspace. e.g. keyspace1.* denotes all the CFs in keyspace1. - * Note 3: {@link #getSnapshotExcludeCFList()} is applied first to exclude CF/keyspace and then {@link #getSnapshotIncludeCFList()} is applied to include the CF's/keyspaces. - * - * @return Column Family(ies), comma delimited, to exclude from snapshot backup. If no filter is applied, returns null. + * Column family(ies), comma delimited, to exclude during snapshot backup. Note 1: The expected + * format is keyspace.cfname. If no value is provided then snapshot is scheduled for all + * KS,CF(s) Note 2: CF name allows special character "*" to denote all the columnfamilies in a + * given keyspace. e.g. keyspace1.* denotes all the CFs in keyspace1. Note 3: {@link + * #getSnapshotExcludeCFList()} is applied first to exclude CF/keyspace and then {@link + * #getSnapshotIncludeCFList()} is applied to include the CF's/keyspaces. + * + * @return Column Family(ies), comma delimited, to exclude from snapshot backup. If no filter is + * applied, returns null. */ default String getSnapshotExcludeCFList() { return null; } /** - * Column Family(ies), comma delimited, to include during incremental backup. - * Note 1: The expected format is keyspace.cfname. If no value is provided then incremental contains all KS,CF(s) - * Note 2: CF name allows special character "*" to denote all the columnfamilies in a given keyspace. e.g. keyspace1.* denotes all the CFs in keyspace1. - * Note 3: {@link #getIncrementalExcludeCFList()} is applied first to exclude CF/keyspace and then {@link #getIncrementalIncludeCFList()} is applied to include the CF's/keyspaces. + * Column Family(ies), comma delimited, to include during incremental backup. Note 1: The + * expected format is keyspace.cfname. If no value is provided then incremental contains all + * KS,CF(s) Note 2: CF name allows special character "*" to denote all the columnfamilies in a + * given keyspace. e.g. keyspace1.* denotes all the CFs in keyspace1. Note 3: {@link + * #getIncrementalExcludeCFList()} is applied first to exclude CF/keyspace and then {@link + * #getIncrementalIncludeCFList()} is applied to include the CF's/keyspaces. * - * @return Column Family(ies), comma delimited, to include in incremental backup. If no filter is applied, returns null. + * @return Column Family(ies), comma delimited, to include in incremental backup. If no filter + * is applied, returns null. */ default String getIncrementalIncludeCFList() { return null; } /** - * Column family(ies), comma delimited, to exclude during incremental backup. - * Note 1: The expected format is keyspace.cfname. If no value is provided then incremental is scheduled for all KS,CF(s) - * Note 2: CF name allows special character "*" to denote all the columnfamilies in a given keyspace. e.g. keyspace1.* denotes all the CFs in keyspace1. - * Note 3: {@link #getIncrementalExcludeCFList()} is applied first to exclude CF/keyspace and then {@link #getIncrementalIncludeCFList()} is applied to include the CF's/keyspaces. - * - * @return Column Family(ies), comma delimited, to exclude from incremental backup. If no filter is applied, returns null. + * Column family(ies), comma delimited, to exclude during incremental backup. Note 1: The + * expected format is keyspace.cfname. If no value is provided then incremental is scheduled for + * all KS,CF(s) Note 2: CF name allows special character "*" to denote all the columnfamilies in + * a given keyspace. e.g. keyspace1.* denotes all the CFs in keyspace1. Note 3: {@link + * #getIncrementalExcludeCFList()} is applied first to exclude CF/keyspace and then {@link + * #getIncrementalIncludeCFList()} is applied to include the CF's/keyspaces. + * + * @return Column Family(ies), comma delimited, to exclude from incremental backup. If no filter + * is applied, returns null. */ default String getIncrementalExcludeCFList() { return null; } /** - * Column Family(ies), comma delimited, to include during restore. - * Note 1: The expected format is keyspace.cfname. If no value is provided then restore contains all KS,CF(s) - * Note 2: CF name allows special character "*" to denote all the columnfamilies in a given keyspace. e.g. keyspace1.* denotes all the CFs in keyspace1. - * Note 3: {@link #getRestoreExcludeCFList()} is applied first to exclude CF/keyspace and then {@link #getRestoreIncludeCFList()} is applied to include the CF's/keyspaces. + * Column Family(ies), comma delimited, to include during restore. Note 1: The expected format + * is keyspace.cfname. If no value is provided then restore contains all KS,CF(s) Note 2: CF + * name allows special character "*" to denote all the columnfamilies in a given keyspace. e.g. + * keyspace1.* denotes all the CFs in keyspace1. Note 3: {@link #getRestoreExcludeCFList()} is + * applied first to exclude CF/keyspace and then {@link #getRestoreIncludeCFList()} is applied + * to include the CF's/keyspaces. * - * @return Column Family(ies), comma delimited, to include in restore. If no filter is applied, returns null. + * @return Column Family(ies), comma delimited, to include in restore. If no filter is applied, + * returns null. */ default String getRestoreIncludeCFList() { return null; } /** - * Column family(ies), comma delimited, to exclude during restore. - * Note 1: The expected format is keyspace.cfname. If no value is provided then restore is scheduled for all KS,CF(s) - * Note 2: CF name allows special character "*" to denote all the columnfamilies in a given keyspace. e.g. keyspace1.* denotes all the CFs in keyspace1. - * Note 3: {@link #getRestoreExcludeCFList()} is applied first to exclude CF/keyspace and then {@link #getRestoreIncludeCFList()} is applied to include the CF's/keyspaces. - * - * @return Column Family(ies), comma delimited, to exclude from restore. If no filter is applied, returns null. + * Column family(ies), comma delimited, to exclude during restore. Note 1: The expected format + * is keyspace.cfname. If no value is provided then restore is scheduled for all KS,CF(s) Note + * 2: CF name allows special character "*" to denote all the columnfamilies in a given keyspace. + * e.g. keyspace1.* denotes all the CFs in keyspace1. Note 3: {@link #getRestoreExcludeCFList()} + * is applied first to exclude CF/keyspace and then {@link #getRestoreIncludeCFList()} is + * applied to include the CF's/keyspaces. + * + * @return Column Family(ies), comma delimited, to exclude from restore. If no filter is + * applied, returns null. */ default String getRestoreExcludeCFList() { return null; } - /** - * Specifies the start and end time used for restoring data (yyyyMMddHHmm - * format) Eg: 201201132030,201201142030 + * Specifies the start and end time used for restoring data (yyyyMMddHHmm format) Eg: + * 201201132030,201201142030 * * @return Snapshot to be searched and restored */ String getRestoreSnapshot(); - /** - * @return Get the region to connect to SDB for instance identity - */ + /** @return Get the region to connect to SDB for instance identity */ String getSDBInstanceIdentityRegion(); - /** - * @return Get the Data Center name (or region for AWS) - */ + /** @return Get the Data Center name (or region for AWS) */ String getDC(); - /** - * @param region Set the current data center - */ + /** @param region Set the current data center */ void setDC(String region); - /** - * @return true if it is a multi regional cluster - */ + /** @return true if it is a multi regional cluster */ boolean isMultiDC(); - /** - * @return Number of backup threads for uploading files when using async feature - */ + /** @return Number of backup threads for uploading files when using async feature */ default int getBackupThreads() { return 2; } - /** - * @return Number of download threads for downloading files when using async feature - */ + /** @return Number of download threads for downloading files when using async feature */ default int getRestoreThreads() { return 8; } - /** - * @return true if restore should search for nearest token if current token - * is not found - */ + /** @return true if restore should search for nearest token if current token is not found */ boolean isRestoreClosestToken(); - /** - * Amazon specific setting to query ASG Membership - */ + /** Amazon specific setting to query ASG Membership */ String getASGName(); /** - * Amazon specific setting to query Additional/ Sibling ASG Memberships in csv format to consider while calculating RAC membership + * Amazon specific setting to query Additional/ Sibling ASG Memberships in csv format to + * consider while calculating RAC membership */ String getSiblingASGNames(); - /** - * Get the security group associated with nodes in this cluster - */ + /** Get the security group associated with nodes in this cluster */ String getACLGroupName(); - /** - * @return true if incremental backups are enabled - */ + /** @return true if incremental backups are enabled */ boolean isIncrBackup(); - /** - * @return Get host IP - */ + /** @return Get host IP */ String getHostIP(); - /** - * @return Bytes per second to throttle for backups - */ + /** @return Bytes per second to throttle for backups */ int getUploadThrottle(); /** - * @return InstanceDataRetriever which encapsulates meta-data about the running instance like region, RAC, name, ip address etc. + * @return InstanceDataRetriever which encapsulates meta-data about the running instance like + * region, RAC, name, ip address etc. */ - InstanceDataRetriever getInstanceDataRetriever() throws InstantiationException, IllegalAccessException, ClassNotFoundException; + InstanceDataRetriever getInstanceDataRetriever() + throws InstantiationException, IllegalAccessException, ClassNotFoundException; - /** - * @return true if Priam should local config file for tokens and seeds - */ + /** @return true if Priam should local config file for tokens and seeds */ boolean isLocalBootstrapEnabled(); - /** - * @return Compaction throughput - */ + /** @return Compaction throughput */ int getCompactionThroughput(); - /** - * @return compaction_throughput_mb_per_sec - */ + /** @return compaction_throughput_mb_per_sec */ int getMaxHintWindowInMS(); - /** - * @return hinted_handoff_throttle_in_kb - */ + /** @return hinted_handoff_throttle_in_kb */ int getHintedHandoffThrottleKb(); - /** - * @return Size of Cassandra max direct memory - */ + /** @return Size of Cassandra max direct memory */ String getMaxDirectMemory(); - /** - * @return Bootstrap cluster name (depends on another cass cluster) - */ + /** @return Bootstrap cluster name (depends on another cass cluster) */ String getBootClusterName(); - /** - * @return Get the name of seed provider - */ + /** @return Get the name of seed provider */ String getSeedProviderName(); /** @@ -505,9 +439,7 @@ default double getMemtableCleanupThreshold() { return 0.11; } - /** - * @return stream_throughput_outbound_megabits_per_sec in yaml - */ + /** @return stream_throughput_outbound_megabits_per_sec in yaml */ int getStreamingThroughputMB(); /** @@ -517,56 +449,40 @@ default double getMemtableCleanupThreshold() { */ String getPartitioner(); - /** - * Support for c* 1.1 global key cache size - */ + /** Support for c* 1.1 global key cache size */ String getKeyCacheSizeInMB(); - /** - * Support for limiting the total number of keys in c* 1.1 global key cache. - */ + /** Support for limiting the total number of keys in c* 1.1 global key cache. */ String getKeyCacheKeysToSave(); - /** - * Support for c* 1.1 global row cache size - */ + /** Support for c* 1.1 global row cache size */ String getRowCacheSizeInMB(); - /** - * Support for limiting the total number of rows in c* 1.1 global row cache. - */ + /** Support for limiting the total number of rows in c* 1.1 global row cache. */ String getRowCacheKeysToSave(); - /** - * @return C* Process Name - */ + /** @return C* Process Name */ String getCassProcessName(); - /** - * Defaults to 'allow all'. - */ + /** Defaults to 'allow all'. */ String getAuthenticator(); - /** - * Defaults to 'allow all'. - */ + /** Defaults to 'allow all'. */ String getAuthorizer(); - /** - * @return true/false, if Cassandra needs to be started manually - */ + /** @return true/false, if Cassandra needs to be started manually */ boolean doesCassandraStartManually(); - /** - * @return possible values: all, dc, none - */ + /** @return possible values: all, dc, none */ default String getInternodeCompression() { return "all"; } /** * Enable/disable backup/restore of commit logs. - * @return boolean value true if commit log backup/restore is enabled, false otherwise. Default: false. + * + * @return boolean value true if commit log backup/restore is enabled, false otherwise. Default: + * false. */ default boolean isBackingUpCommitLogs() { return false; @@ -584,9 +500,7 @@ default boolean isBackingUpCommitLogs() { int maxCommitLogsRestore(); - /** - * @return true/false, if Cassandra is running in a VPC environment - */ + /** @return true/false, if Cassandra is running in a VPC environment */ boolean isVpcRing(); boolean isClientSslEnabled(); @@ -637,23 +551,23 @@ default int getRpcMaxThreads() { String getPrivateKeyLocation(); /** - * @return the type of source for the restore. Valid values are: AWSCROSSACCT or GOOGLE. - * Note: for backward compatibility, this property should be optional. Specifically, if it does not exist, it should not cause an adverse impact on current functionality. - *

- * AWSCROSSACCT - * - You are restoring from an AWS account which requires cross account assumption where an IAM user in one account is allowed to access resources that belong - * to a different account. - *

- * GOOGLE - * - You are restoring from Google Cloud Storage + * @return the type of source for the restore. Valid values are: AWSCROSSACCT or GOOGLE. Note: + * for backward compatibility, this property should be optional. Specifically, if it does + * not exist, it should not cause an adverse impact on current functionality. + *

AWSCROSSACCT - You are restoring from an AWS account which requires cross account + * assumption where an IAM user in one account is allowed to access resources that belong to + * a different account. + *

GOOGLE - You are restoring from Google Cloud Storage */ String getRestoreSourceType(); /** - * Should backups be encrypted. If this is on, then all the files uploaded will be compressed and encrypted before being uploaded to remote file system. + * Should backups be encrypted. If this is on, then all the files uploaded will be compressed + * and encrypted before being uploaded to remote file system. * - * @return true to enable encryption of backup (snapshots, incrementals, commit logs). - * Note: for backward compatibility, this property should be optional. Specifically, if it does not exist, it should not cause an adverse impact on current functionality. + * @return true to enable encryption of backup (snapshots, incrementals, commit logs). Note: for + * backward compatibility, this property should be optional. Specifically, if it does not + * exist, it should not cause an adverse impact on current functionality. */ default boolean isEncryptBackupEnabled() { return false; @@ -661,40 +575,51 @@ default boolean isEncryptBackupEnabled() { /** * Data that needs to be restored is encrypted? - * @return true if data that needs to be restored is encrypted. Note that setting this value does not play any role until {@link #getRestoreSnapshot()} is set to a non-null value. + * + * @return true if data that needs to be restored is encrypted. Note that setting this value + * does not play any role until {@link #getRestoreSnapshot()} is set to a non-null value. */ default boolean isRestoreEncrypted() { return false; } /** - * @return the Amazon Resource Name (ARN). This is applicable when restoring from an AWS account which requires cross account assumption. - * Note: for backward compatibility, this property should be optional. Specifically, if it does not exist, it should not cause an adverse impact on current functionality. + * @return the Amazon Resource Name (ARN). This is applicable when restoring from an AWS account + * which requires cross account assumption. Note: for backward compatibility, this property + * should be optional. Specifically, if it does not exist, it should not cause an adverse + * impact on current functionality. */ String getAWSRoleAssumptionArn(); /** * @return Google Cloud Storage service account id to be use within the restore functionality. - * Note: for backward compatibility, this property should be optional. Specifically, if it does not exist, it should not cause an adverse impact on current functionality. + * Note: for backward compatibility, this property should be optional. Specifically, if it + * does not exist, it should not cause an adverse impact on current functionality. */ String getGcsServiceAccountId(); /** - * @return the absolute path on disk for the Google Cloud Storage PFX file (i.e. the combined format of the private key and certificate). - * This information is to be use within the restore functionality. - * Note: for backward compatibility, this property should be optional. Specifically, if it does not exist, it should not cause an adverse impact on current functionality. + * @return the absolute path on disk for the Google Cloud Storage PFX file (i.e. the combined + * format of the private key and certificate). This information is to be use within the + * restore functionality. Note: for backward compatibility, this property should be + * optional. Specifically, if it does not exist, it should not cause an adverse impact on + * current functionality. */ String getGcsServiceAccountPrivateKeyLoc(); /** - * @return the pass phrase use by PGP cryptography. This information is to be use within the restore and backup functionality when encryption is enabled. - * Note: for backward compatibility, this property should be optional. Specifically, if it does not exist, it should not cause an adverse impact on current functionality. + * @return the pass phrase use by PGP cryptography. This information is to be use within the + * restore and backup functionality when encryption is enabled. Note: for backward + * compatibility, this property should be optional. Specifically, if it does not exist, it + * should not cause an adverse impact on current functionality. */ String getPgpPasswordPhrase(); /** - * @return public key use by PGP cryptography. This information is to be use within the restore and backup functionality when encryption is enabled. - * Note: for backward compatibility, this property should be optional. Specifically, if it does not exist, it should not cause an adverse impact on current functionality. + * @return public key use by PGP cryptography. This information is to be use within the restore + * and backup functionality when encryption is enabled. Note: for backward compatibility, + * this property should be optional. Specifically, if it does not exist, it should not cause + * an adverse impact on current functionality. */ String getPgpPublicKeyLoc(); @@ -705,23 +630,22 @@ default boolean isRestoreEncrypted() { */ Map getExtraEnvParams(); - /** - * @return the vpc id of the running instance. - */ + /** @return the vpc id of the running instance. */ String getVpcId(); /* - * @return the Amazon Resource Name (ARN) for EC2 classic. + * @return the Amazon Resource Name (ARN) for EC2 classic. */ String getClassicEC2RoleAssumptionArn(); /* - * @return the Amazon Resource Name (ARN) for VPC. + * @return the Amazon Resource Name (ARN) for VPC. */ String getVpcEC2RoleAssumptionArn(); /** - * Is cassandra cluster spanning more than one account. This may be true if you are migrating your cluster from one account to another. + * Is cassandra cluster spanning more than one account. This may be true if you are migrating + * your cluster from one account to another. * * @return if the dual account support */ @@ -730,7 +654,8 @@ default boolean isDualAccount() { } /** - * Should incremental backup be uploaded in async fashion? If this is false, then incrementals will be in sync fashion. + * Should incremental backup be uploaded in async fashion? If this is false, then incrementals + * will be in sync fashion. * * @return enable async incrementals for backup */ @@ -739,7 +664,8 @@ default boolean enableAsyncIncremental() { } /** - * Should snapshot backup be uploaded in async fashion? If this is false, then snapshot will be in sync fashion. + * Should snapshot backup be uploaded in async fashion? If this is false, then snapshot will be + * in sync fashion. * * @return enable async snapshot for backup */ @@ -748,8 +674,9 @@ default boolean enableAsyncSnapshot() { } /** - * Queue size to be used for backup uploads. Note that once queue is full, we would wait for {@link #getUploadTimeout()} - * to add any new item before declining the request and throwing exception. + * Queue size to be used for backup uploads. Note that once queue is full, we would wait for + * {@link #getUploadTimeout()} to add any new item before declining the request and throwing + * exception. * * @return size of the queue for uploads. */ @@ -758,8 +685,9 @@ default int getBackupQueueSize() { } /** - * Queue size to be used for file downloads. Note that once queue is full, we would wait for {@link #getDownloadTimeout()} - * to add any new item before declining the request and throwing exception. + * Queue size to be used for file downloads. Note that once queue is full, we would wait for + * {@link #getDownloadTimeout()} to add any new item before declining the request and throwing + * exception. * * @return size of the queue for downloads. */ @@ -768,42 +696,38 @@ default int getDownloadQueueSize() { } /** - * Uploads are scheduled in {@link #getBackupQueueSize()}. If queue is full then we wait for {@link #getUploadTimeout()} - * for the queue to have an entry available for queueing the current task after which we throw RejectedExecutionException. + * Uploads are scheduled in {@link #getBackupQueueSize()}. If queue is full then we wait for + * {@link #getUploadTimeout()} for the queue to have an entry available for queueing the current + * task after which we throw RejectedExecutionException. * * @return timeout for uploads to wait to blocking queue */ default long getUploadTimeout() { - return (2 * 60 * 60 * 1000L); //2 minutes. + return (2 * 60 * 60 * 1000L); // 2 minutes. } /** - * Downloads are scheduled in {@link #getDownloadQueueSize()}. If queue is full then we wait for {@link #getDownloadTimeout()} - * for the queue to have an entry available for queueing the current task after which we throw RejectedExecutionException. + * Downloads are scheduled in {@link #getDownloadQueueSize()}. If queue is full then we wait for + * {@link #getDownloadTimeout()} for the queue to have an entry available for queueing the + * current task after which we throw RejectedExecutionException. * * @return timeout for downloads to wait to blocking queue */ default long getDownloadTimeout() { - return (10 * 60 * 60 * 1000L); //10 minutes. + return (10 * 60 * 60 * 1000L); // 10 minutes. } - /** - * @return tombstone_warn_threshold in C* yaml - */ + /** @return tombstone_warn_threshold in C* yaml */ default int getTombstoneWarnThreshold() { return 1000; } - /** - * @return tombstone_failure_threshold in C* yaml - */ + /** @return tombstone_failure_threshold in C* yaml */ default int getTombstoneFailureThreshold() { return 100000; } - /** - * @return streaming_socket_timeout_in_ms in C* yaml - */ + /** @return streaming_socket_timeout_in_ms in C* yaml */ default int getStreamingSocketTimeoutInMS() { return 86400000; } @@ -818,8 +742,8 @@ default int getStreamingSocketTimeoutInMS() { /** * Interval to be used for flush. * - * @return the interval to run the flush task. Format is name=value where - * “name” is an enum of hour, daily, value is ... + * @return the interval to run the flush task. Format is name=value where “name” is an enum of + * hour, daily, value is ... * @deprecated Use the {{@link #getFlushCronExpression()} instead. */ @Deprecated @@ -828,8 +752,9 @@ default int getStreamingSocketTimeoutInMS() { /** * Scheduler type to use for flush. Default: HOUR. * - * @return Type of scheduler to use for flush. Note the default is TIMER based i.e. to use {@link #getFlushInterval()}. - * If value of "CRON" is provided it starts using {@link #getFlushCronExpression()}. + * @return Type of scheduler to use for flush. Note the default is TIMER based i.e. to use + * {@link #getFlushInterval()}. If value of "CRON" is provided it starts using {@link + * #getFlushCronExpression()}. * @throws UnsupportedTypeException if the scheduler type is not HOUR/CRON. */ SchedulerType getFlushSchedulerType() throws UnsupportedTypeException; @@ -838,44 +763,45 @@ default int getStreamingSocketTimeoutInMS() { * Cron expression to be used for flush. Use "-1" to disable the CRON. Default: -1 * * @return Cron expression for flush - * @see quartz-scheduler + * @see quartz-scheduler * @see http://www.cronmaker.com To build new cron timer */ - default String getFlushCronExpression(){ + default String getFlushCronExpression() { return "-1"; } - /** - * @return the absolute path to store the backup status on disk - */ + /** @return the absolute path to store the backup status on disk */ String getBackupStatusFileLoc(); - /** - * @return Decides whether to use sudo to start C* or not - */ + /** @return Decides whether to use sudo to start C* or not */ default boolean useSudo() { return true; } /** - * SNS Notification topic to be used for sending backup event notifications. - * One start event is sent before uploading any file and one complete/failure event is sent after the file is uploaded/failed. This applies to both incremental and snapshot. - * Default: no notifications i.e. this value is set to EMPTY VALUE + * SNS Notification topic to be used for sending backup event notifications. One start event is + * sent before uploading any file and one complete/failure event is sent after the file is + * uploaded/failed. This applies to both incremental and snapshot. Default: no notifications + * i.e. this value is set to EMPTY VALUE + * * @return SNS Topic ARN to be used to send notification. */ String getBackupNotificationTopicArn(); /** - * Post restore hook enabled state. If enabled, jar represented by getPostRepairHook is called once download of files is complete, before starting Cassandra. + * Post restore hook enabled state. If enabled, jar represented by getPostRepairHook is called + * once download of files is complete, before starting Cassandra. + * * @return if post restore hook is enabled */ - default boolean isPostRestoreHookEnabled() { return false; } /** * Post restore hook to be executed + * * @return post restore hook to be executed once restore is complete */ default String getPostRestoreHook() { @@ -884,15 +810,16 @@ default String getPostRestoreHook() { /** * HeartBeat file of post restore hook + * * @return file that indicates heartbeat of post restore hook */ default String getPostRestoreHookHeartbeatFileName() { return "postrestorehook_heartbeat"; } - /** * Done file for post restore hook + * * @return file that indicates completion of post restore hook */ default String getPostRestoreHookDoneFileName() { @@ -901,6 +828,7 @@ default String getPostRestoreHookDoneFileName() { /** * Maximum time Priam has to wait for post restore hook sub-process to complete successfully + * * @return time out for post restore hook in days */ default int getPostRestoreHookTimeOutInDays() { @@ -909,6 +837,7 @@ default int getPostRestoreHookTimeOutInDays() { /** * Heartbeat timeout (in ms) for post restore hook + * * @return heartbeat timeout for post restore hook */ default int getPostRestoreHookHeartBeatTimeoutInMs() { @@ -917,6 +846,7 @@ default int getPostRestoreHookHeartBeatTimeoutInMs() { /** * Heartbeat check frequency (in ms) for post restore hook + * * @return heart beat check frequency for post restore hook */ default int getPostRestoreHookHeartbeatCheckFrequencyInMs() { @@ -924,7 +854,8 @@ default int getPostRestoreHookHeartbeatCheckFrequencyInMs() { } /** - * Grace period for the file(that should have been deleted by cassandra) that are considered to be forgotten. Only required for cassandra 2.x. + * Grace period for the file(that should have been deleted by cassandra) that are considered to + * be forgotten. Only required for cassandra 2.x. * * @return grace period for the forgotten files. */ @@ -933,9 +864,11 @@ default int getForgottenFileGracePeriodDays() { } /** - * A method for allowing access to outside programs to Priam configuration when paired with the Priam configuration - * HTTP endpoint at /v1/config/structured/all/property - * @param group The group of configuration options to return, currently just returns everything no matter what + * A method for allowing access to outside programs to Priam configuration when paired with the + * Priam configuration HTTP endpoint at /v1/config/structured/all/property + * + * @param group The group of configuration options to return, currently just returns everything + * no matter what * @return A Map representation of this configuration, or null if the method doesn't exist */ @SuppressWarnings("unchecked") @@ -946,16 +879,18 @@ default Map getStructuredConfiguration(String group) { } /** - * Cron expression to be used for persisting Priam merged configuration to disk. Use "-1" to disable the CRON. - * This will persist the fully merged value of Priam's configuration to the {@link #getMergedConfigurationDirectory()} - * as two JSON files: structured.json and unstructured.json which persist structured config and - * unstructured config respectively. We recommend you only rely on unstructured for the time being until the - * structured interface is finalized. + * Cron expression to be used for persisting Priam merged configuration to disk. Use "-1" to + * disable the CRON. This will persist the fully merged value of Priam's configuration to the + * {@link #getMergedConfigurationDirectory()} as two JSON files: structured.json and + * unstructured.json which persist structured config and unstructured config respectively. We + * recommend you only rely on unstructured for the time being until the structured interface is + * finalized. * - * Default: every minute + *

Default: every minute * * @return Cron expression for merged configuration writing - * @see quartz-scheduler + * @see quartz-scheduler * @see http://www.cronmaker.com To build new cron timer */ default String getMergedConfigurationCronExpression() { @@ -964,9 +899,10 @@ default String getMergedConfigurationCronExpression() { } /** - * Returns the path to the directory that Priam should write merged configuration to. Note that if you disable - * the merged configuration cron above {@link #getMergedConfigurationCronExpression()} then this directory is - * not created or used + * Returns the path to the directory that Priam should write merged configuration to. Note that + * if you disable the merged configuration cron above {@link + * #getMergedConfigurationCronExpression()} then this directory is not created or used + * * @return A string representation of the path to the merged priam configuration directory. */ default String getMergedConfigurationDirectory() { @@ -974,9 +910,9 @@ default String getMergedConfigurationDirectory() { } /** - * Escape hatch for getting any arbitrary property by key - * This is useful so we don't have to keep adding methods to this interface for every single configuration - * option ever. Also exposed via HTTP at v1/config/unstructured/X + * Escape hatch for getting any arbitrary property by key This is useful so we don't have to + * keep adding methods to this interface for every single configuration option ever. Also + * exposed via HTTP at v1/config/unstructured/X * * @param key The arbitrary configuration property to look up * @param defaultValue The default value to return if the key is not found. diff --git a/priam/src/main/java/com/netflix/priam/config/PriamConfiguration.java b/priam/src/main/java/com/netflix/priam/config/PriamConfiguration.java index 76ae9699a..b0d4d0e4d 100644 --- a/priam/src/main/java/com/netflix/priam/config/PriamConfiguration.java +++ b/priam/src/main/java/com/netflix/priam/config/PriamConfiguration.java @@ -19,10 +19,9 @@ import com.amazonaws.services.ec2.AmazonEC2; import com.amazonaws.services.ec2.AmazonEC2ClientBuilder; import com.amazonaws.services.ec2.model.*; +import com.fasterxml.jackson.annotation.JsonIgnore; import com.google.common.collect.ImmutableList; import com.google.common.collect.Lists; - -import com.fasterxml.jackson.annotation.JsonIgnore; import com.google.inject.Inject; import com.google.inject.Singleton; import com.netflix.priam.configSource.IConfigSource; @@ -36,14 +35,13 @@ import com.netflix.priam.tuner.JVMOptionsTuner; import com.netflix.priam.utils.RetryableCallable; import com.netflix.priam.utils.SystemUtils; -import org.apache.commons.lang3.StringUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import java.io.File; import java.util.HashMap; import java.util.List; import java.util.Map; +import org.apache.commons.lang3.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; @Singleton public class PriamConfiguration implements IConfiguration { @@ -72,24 +70,30 @@ public class PriamConfiguration implements IConfiguration { private static final String CONFIG_THRIFT_LISTEN_PORT_NAME = PRIAM_PRE + ".thrift.port"; private static final String CONFIG_THRIFT_ENABLED = PRIAM_PRE + ".thrift.enabled"; private static final String CONFIG_NATIVE_PROTOCOL_PORT = PRIAM_PRE + ".nativeTransport.port"; - private static final String CONFIG_NATIVE_PROTOCOL_ENABLED = PRIAM_PRE + ".nativeTransport.enabled"; + private static final String CONFIG_NATIVE_PROTOCOL_ENABLED = + PRIAM_PRE + ".nativeTransport.enabled"; private static final String CONFIG_STORAGE_LISTERN_PORT_NAME = PRIAM_PRE + ".storage.port"; - private static final String CONFIG_SSL_STORAGE_LISTERN_PORT_NAME = PRIAM_PRE + ".ssl.storage.port"; + private static final String CONFIG_SSL_STORAGE_LISTERN_PORT_NAME = + PRIAM_PRE + ".ssl.storage.port"; private static final String CONFIG_CL_BK_LOCATION = PRIAM_PRE + ".backup.commitlog.location"; private static final String CONFIG_THROTTLE_UPLOAD_PER_SECOND = PRIAM_PRE + ".upload.throttle"; private static final String CONFIG_COMPACTION_THROUHPUT = PRIAM_PRE + ".compaction.throughput"; private static final String CONFIG_MAX_HINT_WINDOW_IN_MS = PRIAM_PRE + ".hint.window"; private static final String CONFIG_BOOTCLUSTER_NAME = PRIAM_PRE + ".bootcluster"; private static final String CONFIG_ENDPOINT_SNITCH = PRIAM_PRE + ".endpoint_snitch"; - private static final String CONFIG_MEMTABLE_CLEANUP_THRESHOLD = PRIAM_PRE + ".memtable.cleanup.threshold"; + private static final String CONFIG_MEMTABLE_CLEANUP_THRESHOLD = + PRIAM_PRE + ".memtable.cleanup.threshold"; private static final String CONFIG_CASS_PROCESS_NAME = PRIAM_PRE + ".cass.process"; private static final String CONFIG_VNODE_NUM_TOKENS = PRIAM_PRE + ".vnodes.numTokens"; private static final String CONFIG_YAML_LOCATION = PRIAM_PRE + ".yamlLocation"; private static final String CONFIG_AUTHENTICATOR = PRIAM_PRE + ".authenticator"; private static final String CONFIG_AUTHORIZER = PRIAM_PRE + ".authorizer"; - private static final String CONFIG_CASS_MANUAL_START_ENABLE = PRIAM_PRE + ".cass.manual.start.enable"; - private static final String CONFIG_REMEDIATE_DEAD_CASSANDRA_RATE_S = PRIAM_PRE + ".remediate.dead.cassandra.rate"; - private static final String CONFIG_CREATE_NEW_TOKEN_ENABLE = PRIAM_PRE + ".create.new.token.enable"; + private static final String CONFIG_CASS_MANUAL_START_ENABLE = + PRIAM_PRE + ".cass.manual.start.enable"; + private static final String CONFIG_REMEDIATE_DEAD_CASSANDRA_RATE_S = + PRIAM_PRE + ".remediate.dead.cassandra.rate"; + private static final String CONFIG_CREATE_NEW_TOKEN_ENABLE = + PRIAM_PRE + ".create.new.token.enable"; // Backup and Restore private static final String CONFIG_BACKUP_THREADS = PRIAM_PRE + ".backup.threads"; @@ -107,11 +111,16 @@ public class PriamConfiguration implements IConfiguration { private static final String CONFIG_BACKUP_CHUNK_SIZE = PRIAM_PRE + ".backup.chunksizemb"; private static final String CONFIG_BACKUP_RETENTION = PRIAM_PRE + ".backup.retention"; private static final String CONFIG_BACKUP_RACS = PRIAM_PRE + ".backup.racs"; - private static final String CONFIG_BACKUP_STATUS_FILE_LOCATION = PRIAM_PRE + ".backup.status.location"; - private static final String CONFIG_STREAMING_THROUGHPUT_MB = PRIAM_PRE + ".streaming.throughput.mb"; - private static final String CONFIG_STREAMING_SOCKET_TIMEOUT_IN_MS = PRIAM_PRE + ".streaming.socket.timeout.ms"; - private static final String CONFIG_TOMBSTONE_FAILURE_THRESHOLD = PRIAM_PRE + ".tombstone.failure.threshold"; - private static final String CONFIG_TOMBSTONE_WARNING_THRESHOLD = PRIAM_PRE + ".tombstone.warning.threshold"; + private static final String CONFIG_BACKUP_STATUS_FILE_LOCATION = + PRIAM_PRE + ".backup.status.location"; + private static final String CONFIG_STREAMING_THROUGHPUT_MB = + PRIAM_PRE + ".streaming.throughput.mb"; + private static final String CONFIG_STREAMING_SOCKET_TIMEOUT_IN_MS = + PRIAM_PRE + ".streaming.socket.timeout.ms"; + private static final String CONFIG_TOMBSTONE_FAILURE_THRESHOLD = + PRIAM_PRE + ".tombstone.failure.threshold"; + private static final String CONFIG_TOMBSTONE_WARNING_THRESHOLD = + PRIAM_PRE + ".tombstone.warning.threshold"; private static final String CONFIG_PARTITIONER = PRIAM_PRE + ".partitioner"; private static final String CONFIG_KEYCACHE_SIZE = PRIAM_PRE + ".keyCache.size"; @@ -128,7 +137,8 @@ public class PriamConfiguration implements IConfiguration { private static final String CONFIG_COMMITLOG_ARCHIVE_CMD = PRIAM_PRE + ".clbackup.archiveCmd"; private static final String CONFIG_COMMITLOG_RESTORE_CMD = PRIAM_PRE + ".clbackup.restoreCmd"; private static final String CONFIG_COMMITLOG_RESTORE_DIRS = PRIAM_PRE + ".clbackup.restoreDirs"; - private static final String CONFIG_COMMITLOG_RESTORE_POINT_IN_TIME = PRIAM_PRE + ".clbackup.restoreTime"; + private static final String CONFIG_COMMITLOG_RESTORE_POINT_IN_TIME = + PRIAM_PRE + ".clbackup.restoreTime"; private static final String CONFIG_COMMITLOG_RESTORE_MAX = PRIAM_PRE + ".clrestore.max"; private static final String CONFIG_CLIENT_SSL_ENABLED = PRIAM_PRE + ".client.sslEnabled"; private static final String CONFIG_INTERNODE_ENCRYPTION = PRIAM_PRE + ".internodeEncryption"; @@ -145,64 +155,91 @@ public class PriamConfiguration implements IConfiguration { private static final String CONFIG_AUTO_BOOTSTRAP = PRIAM_PRE + ".auto.bootstrap"; private static final String CONFIG_EXTRA_ENV_PARAMS = PRIAM_PRE + ".extra.env.params"; - private static final String CONFIG_RESTORE_SOURCE_TYPE = PRIAM_PRE + ".restore.source.type"; //the type of source for the restore. Valid values are: AWSCROSSACCT or GOOGLE. - private static final String CONFIG_ENCRYPTED_BACKUP_ENABLED = PRIAM_PRE + ".encrypted.backup.enabled"; //enable encryption of backup (snapshots, incrementals, commit logs). - - //Backup and restore cryptography - private static final String CONFIG_PRIKEY_LOC = PRIAM_PRE + ".private.key.location"; //the location on disk of the private key used by the cryptography algorithm - private static final String CONFIG_PGP_PASSWORD_PHRASE = PRIAM_PRE + ".pgp.password.phrase"; //pass phrase used by the cryptography algorithm + private static final String CONFIG_RESTORE_SOURCE_TYPE = + PRIAM_PRE + ".restore.source.type"; // the type of source for the restore. Valid values + // are: AWSCROSSACCT or GOOGLE. + private static final String CONFIG_ENCRYPTED_BACKUP_ENABLED = + PRIAM_PRE + ".encrypted.backup.enabled"; // enable encryption of backup (snapshots, + // incrementals, commit logs). + + // Backup and restore cryptography + private static final String CONFIG_PRIKEY_LOC = + PRIAM_PRE + ".private.key.location"; // the location on disk of the private key used by + // the cryptography algorithm + private static final String CONFIG_PGP_PASSWORD_PHRASE = + PRIAM_PRE + ".pgp.password.phrase"; // pass phrase used by the cryptography algorithm private static final String CONFIG_PGP_PUB_KEY_LOC = PRIAM_PRE + ".pgp.pubkey.file.location"; - //Restore from Google Cloud Storage - private static final String CONFIG_GCS_SERVICE_ACCT_ID = PRIAM_PRE + ".gcs.service.acct.id"; //Google Cloud Storage service account id - private static final String CONFIG_GCS_SERVICE_ACCT_PRIVATE_KEY_LOC = PRIAM_PRE + ".gcs.service.acct.private.key"; //the absolute path on disk for the Google Cloud Storage PFX file (i.e. the combined format of the private key and certificate). + // Restore from Google Cloud Storage + private static final String CONFIG_GCS_SERVICE_ACCT_ID = + PRIAM_PRE + ".gcs.service.acct.id"; // Google Cloud Storage service account id + private static final String CONFIG_GCS_SERVICE_ACCT_PRIVATE_KEY_LOC = + PRIAM_PRE + ".gcs.service.acct.private.key"; // the absolute path on disk for the Google + // Cloud Storage PFX file (i.e. the combined + // format of the private key and + // certificate). // Amazon specific private static final String CONFIG_ASG_NAME = PRIAM_PRE + ".az.asgname"; private static final String CONFIG_SIBLING_ASG_NAMES = PRIAM_PRE + ".az.sibling.asgnames"; private static final String CONFIG_REGION_NAME = PRIAM_PRE + ".az.region"; - private static final String SDB_INSTANCE_INDENTITY_REGION_NAME = PRIAM_PRE + ".sdb.instanceIdentity.region"; + private static final String SDB_INSTANCE_INDENTITY_REGION_NAME = + PRIAM_PRE + ".sdb.instanceIdentity.region"; private static final String CONFIG_ACL_GROUP_NAME = PRIAM_PRE + ".acl.groupname"; private static String ASG_NAME = System.getenv("ASG_NAME"); private static String REGION = System.getenv("EC2_REGION"); private static final String CONFIG_VPC_RING = PRIAM_PRE + ".vpc"; - private static final String CONFIG_S3_ROLE_ASSUMPTION_ARN = PRIAM_PRE + ".roleassumption.arn"; //Restore from AWS. This is applicable when restoring from an AWS account which requires cross account assumption. - private static final String CONFIG_EC2_ROLE_ASSUMPTION_ARN = PRIAM_PRE + ".ec2.roleassumption.arn"; - private static final String CONFIG_VPC_ROLE_ASSUMPTION_ARN = PRIAM_PRE + ".vpc.roleassumption.arn"; + private static final String CONFIG_S3_ROLE_ASSUMPTION_ARN = + PRIAM_PRE + + ".roleassumption.arn"; // Restore from AWS. This is applicable when restoring + // from an AWS account which requires cross account + // assumption. + private static final String CONFIG_EC2_ROLE_ASSUMPTION_ARN = + PRIAM_PRE + ".ec2.roleassumption.arn"; + private static final String CONFIG_VPC_ROLE_ASSUMPTION_ARN = + PRIAM_PRE + ".vpc.roleassumption.arn"; private static final String CONFIG_DUAL_ACCOUNT = PRIAM_PRE + ".roleassumption.dualaccount"; - //Post Restore Hook - private static final String CONFIG_POST_RESTORE_HOOK_ENABLED = PRIAM_PRE + ".postrestorehook.enabled"; + // Post Restore Hook + private static final String CONFIG_POST_RESTORE_HOOK_ENABLED = + PRIAM_PRE + ".postrestorehook.enabled"; private static final String CONFIG_POST_RESTORE_HOOK = PRIAM_PRE + ".postrestorehook"; - private static final String CONFIG_POST_RESTORE_HOOK_HEARTBEAT_FILENAME = PRIAM_PRE + ".postrestorehook.heartbeat.filename"; - private static final String CONFIG_POST_RESTORE_HOOK_DONE_FILENAME = PRIAM_PRE + ".postrestorehook.done.filename"; - private static final String CONFIG_POST_RESTORE_HOOK_TIMEOUT_IN_DAYS = PRIAM_PRE + ".postrestorehook.timeout.in.days"; - private static final String CONFIG_POST_RESTORE_HOOK_HEARTBEAT_TIMEOUT_MS = PRIAM_PRE + ".postrestorehook.heartbeat.timeout"; - private static final String CONFIG_POST_RESTORE_HOOK_HEARTBEAT_CHECK_FREQUENCY_MS = PRIAM_PRE + ".postrestorehook.heartbeat.check.frequency"; - - //Running instance meta data + private static final String CONFIG_POST_RESTORE_HOOK_HEARTBEAT_FILENAME = + PRIAM_PRE + ".postrestorehook.heartbeat.filename"; + private static final String CONFIG_POST_RESTORE_HOOK_DONE_FILENAME = + PRIAM_PRE + ".postrestorehook.done.filename"; + private static final String CONFIG_POST_RESTORE_HOOK_TIMEOUT_IN_DAYS = + PRIAM_PRE + ".postrestorehook.timeout.in.days"; + private static final String CONFIG_POST_RESTORE_HOOK_HEARTBEAT_TIMEOUT_MS = + PRIAM_PRE + ".postrestorehook.heartbeat.timeout"; + private static final String CONFIG_POST_RESTORE_HOOK_HEARTBEAT_CHECK_FREQUENCY_MS = + PRIAM_PRE + ".postrestorehook.heartbeat.check.frequency"; + + // Running instance meta data private String RAC; private String INSTANCE_ID; - //== vpc specific - private String NETWORK_VPC; //Fetch the vpc id of running instance + // == vpc specific + private String NETWORK_VPC; // Fetch the vpc id of running instance private final String CASS_BASE_DATA_DIR = "/var/lib/cassandra"; - public static final String DEFAULT_AUTHENTICATOR = "org.apache.cassandra.auth.AllowAllAuthenticator"; + public static final String DEFAULT_AUTHENTICATOR = + "org.apache.cassandra.auth.AllowAllAuthenticator"; public static final String DEFAULT_AUTHORIZER = "org.apache.cassandra.auth.AllowAllAuthorizer"; - public static final String DEFAULT_COMMITLOG_PROPS_FILE = "/conf/commitlog_archiving.properties"; + public static final String DEFAULT_COMMITLOG_PROPS_FILE = + "/conf/commitlog_archiving.properties"; // private String DEFAULT_AVAILABILITY_ZONES = ""; private List DEFAULT_AVAILABILITY_ZONES = ImmutableList.of(); - private final int DEFAULT_HINTS_MAX_THREADS = 2; //default value from 1.2 yaml + private final int DEFAULT_HINTS_MAX_THREADS = 2; // default value from 1.2 yaml private static final String DEFAULT_RPC_SERVER_TYPE = "hsha"; private static final int DEFAULT_RPC_MIN_THREADS = 16; private static final int DEFAULT_RPC_MAX_THREADS = 2048; private static final int DEFAULT_STREAMING_SOCKET_TIMEOUT_IN_MS = 86400000; // 24 Hours private static final int DEFAULT_TOMBSTONE_WARNING_THRESHOLD = 1000; // C* defaults - private static final int DEFAULT_TOMBSTONE_FAILURE_THRESHOLD = 100000;// C* defaults + private static final int DEFAULT_TOMBSTONE_FAILURE_THRESHOLD = 100000; // C* defaults // AWS EC2 Dual Account private static final boolean DEFAULT_DUAL_ACCOUNT = false; @@ -211,13 +248,12 @@ public class PriamConfiguration implements IConfiguration { private static final Logger logger = LoggerFactory.getLogger(PriamConfiguration.class); private final ICredential provider; - @JsonIgnore - private final InstanceEnvIdentity insEnvIdentity; - @JsonIgnore - private InstanceDataRetriever instanceDataRetriever; + @JsonIgnore private final InstanceEnvIdentity insEnvIdentity; + @JsonIgnore private InstanceDataRetriever instanceDataRetriever; @Inject - public PriamConfiguration(ICredential provider, IConfigSource config, InstanceEnvIdentity insEnvIdentity) { + public PriamConfiguration( + ICredential provider, IConfigSource config, InstanceEnvIdentity insEnvIdentity) { this.provider = provider; this.config = config; this.insEnvIdentity = insEnvIdentity; @@ -227,15 +263,26 @@ public PriamConfiguration(ICredential provider, IConfigSource config, InstanceEn public void initialize() { try { if (this.insEnvIdentity.isClassic()) { - this.instanceDataRetriever = (InstanceDataRetriever) Class.forName("com.netflix.priam.identity.config.AwsClassicInstanceDataRetriever").newInstance(); + this.instanceDataRetriever = + (InstanceDataRetriever) + Class.forName( + "com.netflix.priam.identity.config.AwsClassicInstanceDataRetriever") + .newInstance(); } else if (this.insEnvIdentity.isNonDefaultVpc()) { - this.instanceDataRetriever = (InstanceDataRetriever) Class.forName("com.netflix.priam.identity.config.AWSVpcInstanceDataRetriever").newInstance(); + this.instanceDataRetriever = + (InstanceDataRetriever) + Class.forName( + "com.netflix.priam.identity.config.AWSVpcInstanceDataRetriever") + .newInstance(); } else { - throw new IllegalStateException("Unable to determine environemt (vpc, classic) for running instance."); + throw new IllegalStateException( + "Unable to determine environemt (vpc, classic) for running instance."); } } catch (Exception e) { - throw new IllegalStateException("Exception when instantiating the instance data retriever. Msg: " + e.getLocalizedMessage()); + throw new IllegalStateException( + "Exception when instantiating the instance data retriever. Msg: " + + e.getLocalizedMessage()); } RAC = instanceDataRetriever.getRac(); @@ -255,26 +302,22 @@ public void initialize() { SystemUtils.createDirs(getLogDirLocation()); } - public InstanceDataRetriever getInstanceDataRetriever() { - return instanceDataRetriever; + public InstanceDataRetriever getInstanceDataRetriever() { + return instanceDataRetriever; } private void setupEnvVars() { // Search in java opt properties REGION = StringUtils.isBlank(REGION) ? System.getProperty("EC2_REGION") : REGION; // Infer from zone - if (StringUtils.isBlank(REGION)) - REGION = RAC.substring(0, RAC.length() - 1); + if (StringUtils.isBlank(REGION)) REGION = RAC.substring(0, RAC.length() - 1); ASG_NAME = StringUtils.isBlank(ASG_NAME) ? System.getProperty("ASG_NAME") : ASG_NAME; if (StringUtils.isBlank(ASG_NAME)) ASG_NAME = populateASGName(REGION, getInstanceDataRetriever().getInstanceId()); logger.info("REGION set to {}, ASG Name set to {}", REGION, ASG_NAME); } - /** - * Query amazon to get ASG name. Currently not available as part of instance - * info api. - */ + /** Query amazon to get ASG name. Currently not available as part of instance info api. */ private String populateASGName(String region, String instanceId) { GetASGName getASGName = new GetASGName(region, instanceId); @@ -297,19 +340,23 @@ public GetASGName(String region, String instanceId) { super(NUMBER_OF_RETRIES, WAIT_TIME); this.region = region; this.instanceId = instanceId; - client = AmazonEC2ClientBuilder.standard().withCredentials(provider.getAwsCredentialProvider()).withRegion(region).build(); + client = + AmazonEC2ClientBuilder.standard() + .withCredentials(provider.getAwsCredentialProvider()) + .withRegion(region) + .build(); } @Override public String retriableCall() throws IllegalStateException { - DescribeInstancesRequest desc = new DescribeInstancesRequest().withInstanceIds(instanceId); + DescribeInstancesRequest desc = + new DescribeInstancesRequest().withInstanceIds(instanceId); DescribeInstancesResult res = client.describeInstances(desc); for (Reservation resr : res.getReservations()) { for (Instance ins : resr.getInstances()) { for (com.amazonaws.services.ec2.model.Tag tag : ins.getTags()) { - if (tag.getKey().equals("aws:autoscaling:groupName")) - return tag.getValue(); + if (tag.getKey().equals("aws:autoscaling:groupName")) return tag.getValue(); } } } @@ -319,18 +366,18 @@ public String retriableCall() throws IllegalStateException { } } - /** - * Get the fist 3 available zones in the region - */ + /** Get the fist 3 available zones in the region */ public void setDefaultRACList(String region) { - AmazonEC2 client = AmazonEC2ClientBuilder.standard().withCredentials(provider.getAwsCredentialProvider()).withRegion(region).build(); + AmazonEC2 client = + AmazonEC2ClientBuilder.standard() + .withCredentials(provider.getAwsCredentialProvider()) + .withRegion(region) + .build(); DescribeAvailabilityZonesResult res = client.describeAvailabilityZones(); List zone = Lists.newArrayList(); for (AvailabilityZone reg : res.getAvailabilityZones()) { - if (reg.getState().equals("available")) - zone.add(reg.getZoneName()); - if (zone.size() == 3) - break; + if (reg.getState().equals("available")) zone.add(reg.getZoneName()); + if (zone.size() == 3) break; } DEFAULT_AVAILABILITY_ZONES = ImmutableList.copyOf(zone); } @@ -340,7 +387,7 @@ private void populateProps() { config.set(CONFIG_REGION_NAME, REGION); } - public String getInstanceName(){ + public String getInstanceName() { return INSTANCE_ID; } @@ -361,7 +408,7 @@ public int getGracefulDrainHealthWaitSeconds() { @Override public int getRemediateDeadCassandraRate() { - return config.get(CONFIG_REMEDIATE_DEAD_CASSANDRA_RATE_S, 3600); //Default to once per hour + return config.get(CONFIG_REMEDIATE_DEAD_CASSANDRA_RATE_S, 3600); // Default to once per hour } @Override @@ -410,8 +457,7 @@ public String getHintsLocation() { } @Override - public String getCacheLocation() - { + public String getCacheLocation() { return config.get(CONFIG_SAVE_CACHE_LOCATION, CASS_BASE_DATA_DIR + "/saved_caches"); } @@ -446,9 +492,7 @@ public String getJmxPassword() { return config.get(CONFIG_JMX_PASSWORD, ""); } - /** - * @return Enables Remote JMX connections n C* - */ + /** @return Enables Remote JMX connections n C* */ @Override public boolean enableRemoteJMX() { return config.get(CONFIG_JMX_ENABLE_REMOTE, false); @@ -502,17 +546,20 @@ public String getHostname() { @Override public String getHeapSize() { - return config.get(CONFIG_MAX_HEAP_SIZE + getInstanceDataRetriever().getInstanceType(), "8G"); + return config.get( + CONFIG_MAX_HEAP_SIZE + getInstanceDataRetriever().getInstanceType(), "8G"); } @Override public String getHeapNewSize() { - return config.get(CONFIG_NEW_MAX_HEAP_SIZE + getInstanceDataRetriever().getInstanceType(), "2G"); + return config.get( + CONFIG_NEW_MAX_HEAP_SIZE + getInstanceDataRetriever().getInstanceType(), "2G"); } @Override public String getMaxDirectMemory() { - return config.get(CONFIG_DIRECT_MAX_HEAP_SIZE + getInstanceDataRetriever().getInstanceType(), "50G"); + return config.get( + CONFIG_DIRECT_MAX_HEAP_SIZE + getInstanceDataRetriever().getInstanceType(), "50G"); } @Override @@ -522,17 +569,18 @@ public int getBackupHour() { @Override public String getBackupCronExpression() { - return config.get(CONFIG_BACKUP_CRON_EXPRESSION, "0 0 12 1/1 * ? *"); //Backup daily at 12 + return config.get(CONFIG_BACKUP_CRON_EXPRESSION, "0 0 12 1/1 * ? *"); // Backup daily at 12 } @Override public SchedulerType getBackupSchedulerType() throws UnsupportedTypeException { - String schedulerType = config.get(CONFIG_BACKUP_SCHEDULE_TYPE, SchedulerType.HOUR.getSchedulerType()); + String schedulerType = + config.get(CONFIG_BACKUP_SCHEDULE_TYPE, SchedulerType.HOUR.getSchedulerType()); return SchedulerType.lookup(schedulerType); } @Override - public GCType getGCType() throws UnsupportedTypeException{ + public GCType getGCType() throws UnsupportedTypeException { String gcType = config.get(PRIAM_PRE + ".gc.type", GCType.CMS.getGcType()); return GCType.lookup(gcType); } @@ -549,7 +597,9 @@ public Map getJVMUpsertSet() { @Override public SchedulerType getFlushSchedulerType() throws UnsupportedTypeException { - String schedulerType = config.get(PRIAM_PRE + ".flush.schedule.type", SchedulerType.HOUR.getSchedulerType()); + String schedulerType = + config.get( + PRIAM_PRE + ".flush.schedule.type", SchedulerType.HOUR.getSchedulerType()); return SchedulerType.lookup(schedulerType); } @@ -609,7 +659,7 @@ public String getRestoreSnapshot() { } @Override - public boolean isRestoreEncrypted(){ + public boolean isRestoreEncrypted() { return config.get(PRIAM_PRE + ".encrypted.restore.enabled", false); } @@ -654,7 +704,8 @@ public String getASGName() { } /** - * Amazon specific setting to query Additional/ Sibling ASG Memberships in csv format to consider while calculating RAC membership + * Amazon specific setting to query Additional/ Sibling ASG Memberships in csv format to + * consider while calculating RAC membership */ @Override public String getSiblingASGNames() { @@ -708,7 +759,8 @@ public String getBootClusterName() { @Override public String getSeedProviderName() { - return config.get(CONFIG_SEED_PROVIDER_NAME, "com.netflix.priam.cassandra.extensions.NFSeedProvider"); + return config.get( + CONFIG_SEED_PROVIDER_NAME, "com.netflix.priam.cassandra.extensions.NFSeedProvider"); } public double getMemtableCleanupThreshold() { @@ -754,8 +806,7 @@ public String getYamlLocation() { } @Override - public String getJVMOptionsFileLocation() - { + public String getJVMOptionsFileLocation() { return config.get(PRIAM_PRE + ".jvm.options.location", getCassHome() + "/conf/jvm.options"); } @@ -779,7 +830,6 @@ public String getInternodeCompression() { @Override public void setRestorePrefix(String prefix) { config.set(CONFIG_RESTORE_PREFIX, prefix); - } @Override @@ -789,7 +839,8 @@ public boolean isBackingUpCommitLogs() { @Override public String getCommitLogBackupPropsFile() { - return config.get(CONFIG_COMMITLOG_PROPS_FILE, getCassHome() + DEFAULT_COMMITLOG_PROPS_FILE); + return config.get( + CONFIG_COMMITLOG_PROPS_FILE, getCassHome() + DEFAULT_COMMITLOG_PROPS_FILE); } @Override @@ -892,14 +943,17 @@ public Map getExtraEnvParams() { String priamKey = pair[0]; String cassKey = pair[1]; String cassVal = config.get(priamKey); - logger.info("getExtraEnvParams: Start-up/ env params: Priamkey[{}], CassStartupKey[{}], Val[{}]", priamKey, cassKey, cassVal); + logger.info( + "getExtraEnvParams: Start-up/ env params: Priamkey[{}], CassStartupKey[{}], Val[{}]", + priamKey, + cassKey, + cassVal); if (!StringUtils.isBlank(cassKey) && !StringUtils.isBlank(cassVal)) { extraEnvParamsMap.put(cassKey, cassVal); } } } return extraEnvParamsMap; - } public String getCassYamlVal(String priamKey) { @@ -915,7 +969,6 @@ public boolean isCreateNewTokenEnable() { return config.get(CONFIG_CREATE_NEW_TOKEN_ENABLE, true); } - @Override public String getPrivateKeyLocation() { return config.get(CONFIG_PRIKEY_LOC); @@ -958,7 +1011,8 @@ public String getGcsServiceAccountId() { @Override public String getGcsServiceAccountPrivateKeyLoc() { - return config.get(CONFIG_GCS_SERVICE_ACCT_PRIVATE_KEY_LOC, "/apps/tomcat/conf/gcsentryptedkey.p12"); + return config.get( + CONFIG_GCS_SERVICE_ACCT_PRIVATE_KEY_LOC, "/apps/tomcat/conf/gcsentryptedkey.p12"); } @Override @@ -1020,7 +1074,8 @@ public int getTombstoneFailureThreshold() { @Override public int getStreamingSocketTimeoutInMS() { - return config.get(CONFIG_STREAMING_SOCKET_TIMEOUT_IN_MS, DEFAULT_STREAMING_SOCKET_TIMEOUT_IN_MS); + return config.get( + CONFIG_STREAMING_SOCKET_TIMEOUT_IN_MS, DEFAULT_STREAMING_SOCKET_TIMEOUT_IN_MS); } @Override @@ -1035,7 +1090,9 @@ public String getFlushInterval() { @Override public String getBackupStatusFileLoc() { - return config.get(CONFIG_BACKUP_STATUS_FILE_LOCATION, getDataFileLocation() + File.separator + "backup.status"); + return config.get( + CONFIG_BACKUP_STATUS_FILE_LOCATION, + getDataFileLocation() + File.separator + "backup.status"); } @Override @@ -1060,12 +1117,16 @@ public String getPostRestoreHook() { @Override public String getPostRestoreHookHeartbeatFileName() { - return config.get(CONFIG_POST_RESTORE_HOOK_HEARTBEAT_FILENAME, getDataFileLocation() + File.separator + "postrestorehook_heartbeat"); + return config.get( + CONFIG_POST_RESTORE_HOOK_HEARTBEAT_FILENAME, + getDataFileLocation() + File.separator + "postrestorehook_heartbeat"); } @Override public String getPostRestoreHookDoneFileName() { - return config.get(CONFIG_POST_RESTORE_HOOK_DONE_FILENAME, getDataFileLocation() + File.separator + "postrestorehook_done"); + return config.get( + CONFIG_POST_RESTORE_HOOK_DONE_FILENAME, + getDataFileLocation() + File.separator + "postrestorehook_done"); } @Override @@ -1084,8 +1145,7 @@ public int getPostRestoreHookHeartbeatCheckFrequencyInMs() { } @Override - public String getProperty(String key, String defaultValue) - { + public String getProperty(String key, String defaultValue) { return config.get(key, defaultValue); } diff --git a/priam/src/main/java/com/netflix/priam/config/PriamConfigurationPersister.java b/priam/src/main/java/com/netflix/priam/config/PriamConfigurationPersister.java index 7c4882567..a22d9a54b 100644 --- a/priam/src/main/java/com/netflix/priam/config/PriamConfigurationPersister.java +++ b/priam/src/main/java/com/netflix/priam/config/PriamConfigurationPersister.java @@ -16,6 +16,12 @@ */ package com.netflix.priam.config; +import com.fasterxml.jackson.core.util.MinimalPrettyPrinter; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.ObjectWriter; +import com.netflix.priam.scheduler.CronTimer; +import com.netflix.priam.scheduler.Task; +import com.netflix.priam.scheduler.TaskTimer; import java.io.File; import java.io.IOException; import java.nio.file.Files; @@ -25,23 +31,12 @@ import java.util.Map; import javax.inject.Inject; import javax.inject.Singleton; - import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.fasterxml.jackson.core.util.MinimalPrettyPrinter; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.ObjectWriter; -import com.netflix.priam.scheduler.CronTimer; -import com.netflix.priam.scheduler.Task; -import com.netflix.priam.scheduler.TaskTimer; - -/** - * Task that persists structured and merged priam configuration to disk. - */ +/** Task that persists structured and merged priam configuration to disk. */ @Singleton -public class PriamConfigurationPersister extends Task -{ +public class PriamConfigurationPersister extends Task { public static final String NAME = "PriamConfigurationPersister"; private static final Logger logger = LoggerFactory.getLogger(PriamConfigurationPersister.class); @@ -57,28 +52,32 @@ public PriamConfigurationPersister(IConfiguration config) { structuredPath = Paths.get(config.getMergedConfigurationDirectory(), "structured.json"); } - private synchronized void ensurePaths() throws IOException - { + private synchronized void ensurePaths() throws IOException { File directory = mergedConfigDirectory.toFile(); if (directory.mkdirs()) { - Files.setPosixFilePermissions(mergedConfigDirectory, PosixFilePermissions.fromString("rwx------")); + Files.setPosixFilePermissions( + mergedConfigDirectory, PosixFilePermissions.fromString("rwx------")); logger.info("Set up PriamConfigurationPersister directory successfully"); } } - @Override - public void execute() throws Exception - { + public void execute() throws Exception { ensurePaths(); Path tempPath = null; try { - File output = File.createTempFile(structuredPath.getFileName().toString(), ".tmp", mergedConfigDirectory.toFile()); + File output = + File.createTempFile( + structuredPath.getFileName().toString(), + ".tmp", + mergedConfigDirectory.toFile()); tempPath = output.toPath(); - // The configuration might contain sensitive information, so ... don't let non Priam users read it - // Theoretically createTempFile creates the file with the right permissions, but I want to be explicit + // The configuration might contain sensitive information, so ... don't let non Priam + // users read it + // Theoretically createTempFile creates the file with the right permissions, but I want + // to be explicit Files.setPosixFilePermissions(tempPath, PosixFilePermissions.fromString("rw-------")); Map structuredConfiguration = config.getStructuredConfiguration("all"); @@ -91,14 +90,12 @@ public void execute() throws Exception if (!output.renameTo(structuredPath.toFile())) logger.error("Failed to persist structured Priam configuration"); } finally { - if (tempPath != null) - Files.deleteIfExists(tempPath); + if (tempPath != null) Files.deleteIfExists(tempPath); } } @Override - public String getName() - { + public String getName() { return NAME; } @@ -106,10 +103,10 @@ public String getName() * Timer to be used for configuration writing. * * @param config {@link IConfiguration} to get configuration details from priam. - * @return the timer to be used for Configuration Persisting from {@link IConfiguration#getMergedConfigurationCronExpression()} + * @return the timer to be used for Configuration Persisting from {@link + * IConfiguration#getMergedConfigurationCronExpression()} */ public static TaskTimer getTimer(IConfiguration config) { return CronTimer.getCronTimer(NAME, config.getMergedConfigurationCronExpression()); } - } diff --git a/priam/src/main/java/com/netflix/priam/configSource/AbstractConfigSource.java b/priam/src/main/java/com/netflix/priam/configSource/AbstractConfigSource.java index bc770afd7..a95f00fdd 100644 --- a/priam/src/main/java/com/netflix/priam/configSource/AbstractConfigSource.java +++ b/priam/src/main/java/com/netflix/priam/configSource/AbstractConfigSource.java @@ -16,17 +16,14 @@ */ package com.netflix.priam.configSource; +import static com.google.common.base.Preconditions.checkNotNull; + import com.google.common.collect.ImmutableList; import com.google.common.collect.Lists; -import org.apache.commons.lang3.StringUtils; - import java.util.List; +import org.apache.commons.lang3.StringUtils; -import static com.google.common.base.Preconditions.checkNotNull; - -/** - * Base implementations for most methods on {@link IConfigSource}. - */ +/** Base implementations for most methods on {@link IConfigSource}. */ public abstract class AbstractConfigSource implements IConfigSource { private String asgName; @@ -174,5 +171,4 @@ private List getTrimmedStringList(String[] strings) { } return list; } - } diff --git a/priam/src/main/java/com/netflix/priam/configSource/CompositeConfigSource.java b/priam/src/main/java/com/netflix/priam/configSource/CompositeConfigSource.java index 4229f9395..c9ee7e0d7 100644 --- a/priam/src/main/java/com/netflix/priam/configSource/CompositeConfigSource.java +++ b/priam/src/main/java/com/netflix/priam/configSource/CompositeConfigSource.java @@ -20,24 +20,27 @@ import com.google.common.collect.ImmutableCollection; import com.google.common.collect.ImmutableList; import com.google.common.collect.Iterables; - import java.util.Collection; /** - * A {@link IConfigSource} that delegates method calls to the underline sources. The order in which values are provided - * depend on the {@link IConfigSource}s provided. If user asks for key 'foo', and this composite has three sources, it - * will first check if the key is found in the first source, if not it will check the second and if not, the third, else - * return null or false if {@link #contains(String)} was called. + * A {@link IConfigSource} that delegates method calls to the underline sources. The order in which + * values are provided depend on the {@link IConfigSource}s provided. If user asks for key 'foo', + * and this composite has three sources, it will first check if the key is found in the first + * source, if not it will check the second and if not, the third, else return null or false if + * {@link #contains(String)} was called. * - * Implementation note: get methods with a default are implemented in {@link AbstractConfigSource}, if the underlying - * source overrides one of these methods, then that implementation will be ignored. + *

Implementation note: get methods with a default are implemented in {@link + * AbstractConfigSource}, if the underlying source overrides one of these methods, then that + * implementation will be ignored. */ public class CompositeConfigSource extends AbstractConfigSource { private final ImmutableCollection sources; public CompositeConfigSource(final ImmutableCollection sources) { - Preconditions.checkArgument(!sources.isEmpty(), "Can not create a composite config source without config sources!"); + Preconditions.checkArgument( + !sources.isEmpty(), + "Can not create a composite config source without config sources!"); this.sources = sources; } @@ -56,7 +59,7 @@ public CompositeConfigSource(final IConfigSource... sources) { @Override public void intialize(final String asgName, final String region) { for (final IConfigSource source : sources) { - //TODO should this catch any potential exceptions? + // TODO should this catch any potential exceptions? source.intialize(asgName, region); } } @@ -96,8 +99,10 @@ public String get(final String key) { public void set(final String key, final String value) { Preconditions.checkNotNull(value, "Value can not be null for configurations."); final IConfigSource firstSource = Iterables.getFirst(sources, null); - // firstSource shouldn't be null because the collection is immutable, and the collection is non empty. - Preconditions.checkState(firstSource != null, "There was no IConfigSource found at the first location?"); + // firstSource shouldn't be null because the collection is immutable, and the collection is + // non empty. + Preconditions.checkState( + firstSource != null, "There was no IConfigSource found at the first location?"); firstSource.set(key, value); } } diff --git a/priam/src/main/java/com/netflix/priam/configSource/IConfigSource.java b/priam/src/main/java/com/netflix/priam/configSource/IConfigSource.java index eb6ee1315..b86b80f4e 100644 --- a/priam/src/main/java/com/netflix/priam/configSource/IConfigSource.java +++ b/priam/src/main/java/com/netflix/priam/configSource/IConfigSource.java @@ -17,18 +17,16 @@ package com.netflix.priam.configSource; import com.google.inject.ImplementedBy; - import java.util.List; -/** - * Defines the configurations for an application. - */ +/** Defines the configurations for an application. */ @ImplementedBy(PriamConfigSource.class) public interface IConfigSource { /** - * Must be called before any other method. This method will allow implementations to do any setup that they require - * before being called. + * Must be called before any other method. This method will allow implementations to do any + * setup that they require before being called. + * * @param asgName: Name of the asg * @param region: Name of the region */ @@ -67,7 +65,7 @@ public interface IConfigSource { /** * Get a String associated with the given configuration key. * - * @param key to look up value. + * @param key to look up value. * @param defaultValue if value is not present. * @return value from config or defaultValue if not present. */ @@ -76,7 +74,7 @@ public interface IConfigSource { /** * Get a boolean associated with the given configuration key. * - * @param key to look up value. + * @param key to look up value. * @param defaultValue if value is not present. * @return value from config or defaultValue if not present. */ @@ -85,7 +83,7 @@ public interface IConfigSource { /** * Get a Class associated with the given configuration key. * - * @param key to look up value. + * @param key to look up value. * @param defaultValue if value is not present. * @return value from config or defaultValue if not present. */ @@ -94,9 +92,9 @@ public interface IConfigSource { /** * Get a Enum associated with the given configuration key. * - * @param key to look up value. + * @param key to look up value. * @param defaultValue if value is not present. - * @param enum type. + * @param enum type. * @return value from config or defaultValue if not present. */ > T get(String key, T defaultValue); @@ -104,7 +102,7 @@ public interface IConfigSource { /** * Get a int associated with the given configuration key. * - * @param key to look up value. + * @param key to look up value. * @param defaultValue if value is not present. * @return value from config or defaultValue if not present. */ @@ -113,7 +111,7 @@ public interface IConfigSource { /** * Get a long associated with the given configuration key. * - * @param key to look up value. + * @param key to look up value. * @param defaultValue if value is not present. * @return value from config or defaultValue if not present. */ @@ -122,7 +120,7 @@ public interface IConfigSource { /** * Get a float associated with the given configuration key. * - * @param key to look up value. + * @param key to look up value. * @param defaultValue if value is not present. * @return value from config or defaultValue if not present. */ @@ -131,7 +129,7 @@ public interface IConfigSource { /** * Get a double associated with the given configuration key. * - * @param key to look up value. + * @param key to look up value. * @param defaultValue if value is not present. * @return value from config or defaultValue if not present. */ @@ -140,7 +138,7 @@ public interface IConfigSource { /** * Get a list of strings associated with the given configuration key. * - * @param key to look up value. + * @param key to look up value. * @return value from config or an immutable list if not present. */ List getList(String key); @@ -148,7 +146,7 @@ public interface IConfigSource { /** * Get a list of strings associated with the given configuration key. * - * @param key to look up value. + * @param key to look up value. * @param defaultValue if value is not present. * @return value from config or defaultValue if not present. */ @@ -157,7 +155,7 @@ public interface IConfigSource { /** * Set the value for the given key. * - * @param key to set value for. + * @param key to set value for. * @param value to set. */ void set(String key, String value); diff --git a/priam/src/main/java/com/netflix/priam/configSource/MemoryConfigSource.java b/priam/src/main/java/com/netflix/priam/configSource/MemoryConfigSource.java index 1b6b5a0a2..25c4f80bd 100644 --- a/priam/src/main/java/com/netflix/priam/configSource/MemoryConfigSource.java +++ b/priam/src/main/java/com/netflix/priam/configSource/MemoryConfigSource.java @@ -17,7 +17,6 @@ package com.netflix.priam.configSource; import com.google.common.collect.Maps; - import java.util.Map; public final class MemoryConfigSource extends AbstractConfigSource { diff --git a/priam/src/main/java/com/netflix/priam/configSource/PriamConfigSource.java b/priam/src/main/java/com/netflix/priam/configSource/PriamConfigSource.java index a2d78c467..fcc6c6f35 100644 --- a/priam/src/main/java/com/netflix/priam/configSource/PriamConfigSource.java +++ b/priam/src/main/java/com/netflix/priam/configSource/PriamConfigSource.java @@ -16,27 +16,22 @@ */ package com.netflix.priam.configSource; -import com.netflix.priam.configSource.CompositeConfigSource; -import com.netflix.priam.configSource.PropertiesConfigSource; -import com.netflix.priam.configSource.SimpleDBConfigSource; -import com.netflix.priam.configSource.SystemPropertiesConfigSource; -import com.netflix.priam.configSource.IConfigSource; - import javax.inject.Inject; /** - * Default {@link IConfigSource} pulling in configs from SimpleDB, local Properties, and System Properties. + * Default {@link com.netflix.priam.configSource.IConfigSource} pulling in configs from SimpleDB, + * local Properties, and System Properties. */ public class PriamConfigSource extends CompositeConfigSource { @Inject - public PriamConfigSource(final SimpleDBConfigSource simpleDBConfigSource, - final PropertiesConfigSource propertiesConfigSource, - final SystemPropertiesConfigSource systemPropertiesConfigSource) { - // this order was based off PriamConfigurations loading. W/e loaded last could override, but with Composite, first + public PriamConfigSource( + final SimpleDBConfigSource simpleDBConfigSource, + final PropertiesConfigSource propertiesConfigSource, + final SystemPropertiesConfigSource systemPropertiesConfigSource) { + // this order was based off PriamConfigurations loading. W/e loaded last could override, + // but with Composite, first // has the highest priority. - super(simpleDBConfigSource, - propertiesConfigSource, - systemPropertiesConfigSource); + super(simpleDBConfigSource, propertiesConfigSource, systemPropertiesConfigSource); } } diff --git a/priam/src/main/java/com/netflix/priam/configSource/PropertiesConfigSource.java b/priam/src/main/java/com/netflix/priam/configSource/PropertiesConfigSource.java index 6385fbecd..1cee1393e 100644 --- a/priam/src/main/java/com/netflix/priam/configSource/PropertiesConfigSource.java +++ b/priam/src/main/java/com/netflix/priam/configSource/PropertiesConfigSource.java @@ -16,25 +16,23 @@ */ package com.netflix.priam.configSource; +import static com.google.common.base.Preconditions.checkNotNull; + import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; import com.google.common.base.Strings; import com.google.common.collect.Maps; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import java.io.IOException; import java.net.URL; import java.util.Map; import java.util.Properties; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; -import static com.google.common.base.Preconditions.checkNotNull; - -/** - * Loads the 'Priam.properties' file as a source. - */ +/** Loads the 'Priam.properties' file as a source. */ public class PropertiesConfigSource extends AbstractConfigSource { - private static final Logger logger = LoggerFactory.getLogger(PropertiesConfigSource.class.getName()); + private static final Logger logger = + LoggerFactory.getLogger(PropertiesConfigSource.class.getName()); private static final String DEFAULT_PRIAM_PROPERTIES = "Priam.properties"; @@ -84,7 +82,6 @@ public void set(final String key, final String value) { data.put(key, value); } - @Override public int size() { return data.size(); @@ -96,7 +93,7 @@ public boolean contains(final String prop) { } /** - * Clones all the values from the properties. If the value is null, it will be ignored. + * Clones all the values from the properties. If the value is null, it will be ignored. * * @param properties to clone */ diff --git a/priam/src/main/java/com/netflix/priam/configSource/SimpleDBConfigSource.java b/priam/src/main/java/com/netflix/priam/configSource/SimpleDBConfigSource.java index f2d4446dd..b1a7b7ae4 100644 --- a/priam/src/main/java/com/netflix/priam/configSource/SimpleDBConfigSource.java +++ b/priam/src/main/java/com/netflix/priam/configSource/SimpleDBConfigSource.java @@ -25,29 +25,33 @@ import com.google.common.base.Preconditions; import com.google.common.collect.Maps; import com.netflix.priam.cred.ICredential; +import java.util.Iterator; +import java.util.Map; +import javax.inject.Inject; import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import javax.inject.Inject; -import java.util.Iterator; -import java.util.Map; - /** - * Loads config data from SimpleDB. {@link #intialize(String, String)} will query the SimpleDB domain "PriamProperties" - * for any potential configurations. The domain is set up to support multiple different clusters; this is done by using - * amazon's auto scaling groups (ASG). + * Loads config data from SimpleDB. {@link #intialize(String, String)} will query the SimpleDB + * domain "PriamProperties" for any potential configurations. The domain is set up to support + * multiple different clusters; this is done by using amazon's auto scaling groups (ASG). + * + *

Schema * - * Schema

    - *
  • "appId" // ASG up to first instance of '-'. So ASG name priam-test will create appId priam, ASG priam_test - * will create appId priam_test.
    • - *
  • "property" // key to use for configs.
    • - *
  • "value" // value to set for the given property/key.
    • - *
  • "region" // region the config belongs to. If left empty, then applies to all regions.
    • - *
} + *
    + *
  • "appId" // ASG up to first instance of '-'. So ASG name priam-test will create appId priam, + * ASG priam_test will create appId priam_test. + *
  • "property" // key to use for configs. + *
  • "value" // value to set for the given property/key. + *
  • "region" // region the config belongs to. If left empty, then applies to all regions. + *
+ * + * } */ public final class SimpleDBConfigSource extends AbstractConfigSource { - private static final Logger logger = LoggerFactory.getLogger(SimpleDBConfigSource.class.getName()); + private static final Logger logger = + LoggerFactory.getLogger(SimpleDBConfigSource.class.getName()); private static final String DOMAIN = "PriamProperties"; @@ -64,10 +68,14 @@ public void intialize(final String asgName, final String region) { super.intialize(asgName, region); // End point is us-east-1 - AmazonSimpleDB simpleDBClient = AmazonSimpleDBClient.builder().withCredentials(provider.getAwsCredentialProvider()).build(); + AmazonSimpleDB simpleDBClient = + AmazonSimpleDBClient.builder() + .withCredentials(provider.getAwsCredentialProvider()) + .build(); String nextToken = null; - String appid = asgName.lastIndexOf('-') > 0 ? asgName.substring(0, asgName.indexOf('-')) : asgName; + String appid = + asgName.lastIndexOf('-') > 0 ? asgName.substring(0, asgName.indexOf('-')) : asgName; logger.info("appid used to fetch properties is: {}", appid); do { String ALL_QUERY = "select * from " + DOMAIN + " where " + Attributes.APP_ID + "='%s'"; @@ -77,15 +85,14 @@ public void intialize(final String asgName, final String region) { nextToken = result.getNextToken(); for (Item item : result.getItems()) addProperty(item); - } - while (nextToken != null); + } while (nextToken != null); } private static class Attributes { - public final static String APP_ID = "appId"; // ASG - public final static String PROPERTY = "property"; - public final static String PROPERTY_VALUE = "value"; - public final static String REGION = "region"; + public static final String APP_ID = "appId"; // ASG + public static final String PROPERTY = "property"; + public static final String PROPERTY_VALUE = "value"; + public static final String REGION = "region"; } private void addProperty(Item item) { @@ -95,19 +102,14 @@ private void addProperty(Item item) { String dc = ""; while (attrs.hasNext()) { Attribute att = attrs.next(); - if (att.getName().equals(Attributes.PROPERTY)) - prop = att.getValue(); - else if (att.getName().equals(Attributes.PROPERTY_VALUE)) - value = att.getValue(); - else if (att.getName().equals(Attributes.REGION)) - dc = att.getValue(); + if (att.getName().equals(Attributes.PROPERTY)) prop = att.getValue(); + else if (att.getName().equals(Attributes.PROPERTY_VALUE)) value = att.getValue(); + else if (att.getName().equals(Attributes.REGION)) dc = att.getValue(); } // Ignore, if not this region - if (StringUtils.isNotBlank(dc) && !dc.equals(getRegion())) - return; + if (StringUtils.isNotBlank(dc) && !dc.equals(getRegion())) return; // Override only if region is specified - if (data.containsKey(prop) && StringUtils.isBlank(dc)) - return; + if (data.containsKey(prop) && StringUtils.isBlank(dc)) return; data.put(prop, value); } diff --git a/priam/src/main/java/com/netflix/priam/configSource/SystemPropertiesConfigSource.java b/priam/src/main/java/com/netflix/priam/configSource/SystemPropertiesConfigSource.java index d1dd68bbc..73cc9ea45 100644 --- a/priam/src/main/java/com/netflix/priam/configSource/SystemPropertiesConfigSource.java +++ b/priam/src/main/java/com/netflix/priam/configSource/SystemPropertiesConfigSource.java @@ -19,15 +19,15 @@ import com.google.common.base.Preconditions; import com.google.common.collect.Maps; import com.netflix.priam.config.PriamConfiguration; - import java.util.Map; import java.util.Properties; /** * Loads {@link System#getProperties()} as a source. * - * Implementation note: {@link #set(String, String)} does not write to system properties, but will write to a new map. - * This means that setting values to this source has no effect on system properties or other instances of this class. + *

Implementation note: {@link #set(String, String)} does not write to system properties, but + * will write to a new map. This means that setting values to this source has no effect on system + * properties or other instances of this class. */ public final class SystemPropertiesConfigSource extends AbstractConfigSource { private static final String BLANK = ""; @@ -41,8 +41,7 @@ public void intialize(final String asgName, final String region) { Properties systemProps = System.getProperties(); for (final String key : systemProps.stringPropertyNames()) { - if (!key.startsWith(PriamConfiguration.PRIAM_PRE)) - continue; + if (!key.startsWith(PriamConfiguration.PRIAM_PRE)) continue; final String value = systemProps.getProperty(key); if (value != null && !BLANK.equals(value)) { data.put(key, value); diff --git a/priam/src/main/java/com/netflix/priam/cred/ClearCredential.java b/priam/src/main/java/com/netflix/priam/cred/ClearCredential.java index ccfec2fc9..3d3a0ea6a 100644 --- a/priam/src/main/java/com/netflix/priam/cred/ClearCredential.java +++ b/priam/src/main/java/com/netflix/priam/cred/ClearCredential.java @@ -19,20 +19,17 @@ import com.amazonaws.auth.AWSCredentials; import com.amazonaws.auth.AWSCredentialsProvider; import com.amazonaws.auth.BasicAWSCredentials; +import java.io.FileInputStream; +import java.util.Properties; import org.apache.cassandra.io.util.FileUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.FileInputStream; -import java.util.Properties; - /** - * This is a basic implementation of ICredentials. User should prefer to - * implement their own versions for more secured access. This class requires - * clear AWS key and access. - * - * Set the following properties in "conf/awscredntial.properties" + * This is a basic implementation of ICredentials. User should prefer to implement their own + * versions for more secured access. This class requires clear AWS key and access. * + *

Set the following properties in "conf/awscredntial.properties" */ public class ClearCredential implements ICredential { private static final Logger logger = LoggerFactory.getLogger(ClearCredential.class); @@ -46,7 +43,10 @@ public ClearCredential() { fis = new FileInputStream(CRED_FILE); final Properties props = new Properties(); props.load(fis); - AWS_ACCESS_ID = props.getProperty("AWSACCESSID") != null ? props.getProperty("AWSACCESSID").trim() : ""; + AWS_ACCESS_ID = + props.getProperty("AWSACCESSID") != null + ? props.getProperty("AWSACCESSID").trim() + : ""; AWS_KEY = props.getProperty("AWSKEY") != null ? props.getProperty("AWSKEY").trim() : ""; } catch (Exception e) { logger.error("Exception with credential file ", e); diff --git a/priam/src/main/java/com/netflix/priam/cred/ICredential.java b/priam/src/main/java/com/netflix/priam/cred/ICredential.java index 9cdb79878..127ef1c48 100644 --- a/priam/src/main/java/com/netflix/priam/cred/ICredential.java +++ b/priam/src/main/java/com/netflix/priam/cred/ICredential.java @@ -19,14 +19,9 @@ import com.amazonaws.auth.AWSCredentialsProvider; import com.google.inject.ImplementedBy; -/** - * Credential file interface for services supporting - * Access ID and key authentication - */ +/** Credential file interface for services supporting Access ID and key authentication */ @ImplementedBy(ClearCredential.class) public interface ICredential { - /** - * @return AWS Credential Provider object - */ + /** @return AWS Credential Provider object */ AWSCredentialsProvider getAwsCredentialProvider(); } diff --git a/priam/src/main/java/com/netflix/priam/cred/ICredentialGeneric.java b/priam/src/main/java/com/netflix/priam/cred/ICredentialGeneric.java index d4975ad48..59893385d 100755 --- a/priam/src/main/java/com/netflix/priam/cred/ICredentialGeneric.java +++ b/priam/src/main/java/com/netflix/priam/cred/ICredentialGeneric.java @@ -17,14 +17,16 @@ package com.netflix.priam.cred; /** - * Credential file interface for services supporting - * Access ID and key authentication for non-AWS + * Credential file interface for services supporting Access ID and key authentication for non-AWS */ public interface ICredentialGeneric extends ICredential { byte[] getValue(KEY key); enum KEY { - PGP_PUBLIC_KEY_LOC, PGP_PASSWORD, GCS_SERVICE_ID, GCS_PRIVATE_KEY_LOC + PGP_PUBLIC_KEY_LOC, + PGP_PASSWORD, + GCS_SERVICE_ID, + GCS_PRIVATE_KEY_LOC } } diff --git a/priam/src/main/java/com/netflix/priam/cryptography/IFileCryptography.java b/priam/src/main/java/com/netflix/priam/cryptography/IFileCryptography.java index 894e44ed7..3e8d0d58b 100755 --- a/priam/src/main/java/com/netflix/priam/cryptography/IFileCryptography.java +++ b/priam/src/main/java/com/netflix/priam/cryptography/IFileCryptography.java @@ -1,16 +1,14 @@ /** * Copyright 2017 Netflix, Inc. - *

- * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and + * + *

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file + * except in compliance with the License. You may obtain a copy of the License at + * + *

http://www.apache.org/licenses/LICENSE-2.0 + * + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing permissions and * limitations under the License. */ package com.netflix.priam.cryptography; @@ -23,7 +21,8 @@ public interface IFileCryptography { /** * @param in - a handle to the encrypted, compressed data stream * @param passwd - pass phrase used to extract the PGP private key from the encrypted content. - * @param objectName - name of the object we are decrypting, currently use for debugging purposes only. + * @param objectName - name of the object we are decrypting, currently use for debugging + * purposes only. * @return a handle to the decrypted, uncompress data stream. */ InputStream decryptStream(InputStream in, char[] passwd, String objectName) throws Exception; @@ -33,6 +32,4 @@ public interface IFileCryptography { * @return - an iterate of the ciphertext stream */ Iterator encryptStream(InputStream is, String fileName) throws Exception; - - -} \ No newline at end of file +} diff --git a/priam/src/main/java/com/netflix/priam/cryptography/pgp/PgpCredential.java b/priam/src/main/java/com/netflix/priam/cryptography/pgp/PgpCredential.java index acc2caac9..cc5675ec5 100755 --- a/priam/src/main/java/com/netflix/priam/cryptography/pgp/PgpCredential.java +++ b/priam/src/main/java/com/netflix/priam/cryptography/pgp/PgpCredential.java @@ -1,16 +1,14 @@ /** * Copyright 2017 Netflix, Inc. - *

- * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and + * + *

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file + * except in compliance with the License. You may obtain a copy of the License at + * + *

http://www.apache.org/licenses/LICENSE-2.0 + * + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing permissions and * limitations under the License. */ package com.netflix.priam.cryptography.pgp; @@ -22,7 +20,7 @@ /* * A generic implementation of fetch keys as plaintext. The key values are used within PGP cryptography algorithm. Users may - * want to provide an implementation where your key(s)' value is decrypted using AES encryption algorithm. + * want to provide an implementation where your key(s)' value is decrypted using AES encryption algorithm. */ public class PgpCredential implements ICredentialGeneric { @@ -51,8 +49,5 @@ public byte[] getValue(KEY key) { } else { throw new IllegalArgumentException("Key value not supported."); } - } - - } diff --git a/priam/src/main/java/com/netflix/priam/cryptography/pgp/PgpCryptography.java b/priam/src/main/java/com/netflix/priam/cryptography/pgp/PgpCryptography.java index f3a4f063a..4f1289aed 100755 --- a/priam/src/main/java/com/netflix/priam/cryptography/pgp/PgpCryptography.java +++ b/priam/src/main/java/com/netflix/priam/cryptography/pgp/PgpCryptography.java @@ -1,16 +1,14 @@ /** * Copyright 2017 Netflix, Inc. - *

- * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and + * + *

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file + * except in compliance with the License. You may obtain a copy of the License at + * + *

http://www.apache.org/licenses/LICENSE-2.0 + * + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing permissions and * limitations under the License. */ package com.netflix.priam.cryptography.pgp; @@ -18,18 +16,17 @@ import com.google.inject.Inject; import com.netflix.priam.config.IConfiguration; import com.netflix.priam.cryptography.IFileCryptography; -import org.apache.commons.io.IOUtils; -import org.bouncycastle.jce.provider.BouncyCastleProvider; -import org.bouncycastle.openpgp.*; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import java.io.*; import java.security.NoSuchProviderException; import java.security.SecureRandom; import java.security.Security; import java.util.Date; import java.util.Iterator; +import org.apache.commons.io.IOUtils; +import org.bouncycastle.jce.provider.BouncyCastleProvider; +import org.bouncycastle.openpgp.*; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class PgpCryptography implements IFileCryptography { private static final Logger logger = LoggerFactory.getLogger(PgpCryptography.class); @@ -37,15 +34,14 @@ public class PgpCryptography implements IFileCryptography { private IConfiguration config; static { - Security.addProvider(new BouncyCastleProvider()); //tell the JVM the security provider is PGP + // tell the JVM the security provider is PGP + Security.addProvider(new BouncyCastleProvider()); } @Inject public PgpCryptography(IConfiguration config) { this.config = config; - - } private PGPSecretKeyRingCollection getPgpSecurityCollection() { @@ -54,7 +50,8 @@ private PGPSecretKeyRingCollection getPgpSecurityCollection() { try { keyIn = new BufferedInputStream(new FileInputStream(config.getPrivateKeyLocation())); } catch (FileNotFoundException e) { - throw new IllegalStateException("PGP private key file not found. file: " + config.getPrivateKeyLocation()); + throw new IllegalStateException( + "PGP private key file not found. file: " + config.getPrivateKeyLocation()); } try { @@ -62,10 +59,11 @@ private PGPSecretKeyRingCollection getPgpSecurityCollection() { return new PGPSecretKeyRingCollection(PGPUtil.getDecoderStream(keyIn)); } catch (Exception e) { - logger.error("Exception in reading PGP security collection ring. Msg: {}", e.getLocalizedMessage()); + logger.error( + "Exception in reading PGP security collection ring. Msg: {}", + e.getLocalizedMessage()); throw new IllegalStateException("Exception in reading PGP security collection ring", e); } - } private PGPPublicKey getPubKey() { @@ -74,7 +72,9 @@ private PGPPublicKey getPubKey() { pubKeyIS = new BufferedInputStream(new FileInputStream(config.getPgpPublicKeyLoc())); } catch (FileNotFoundException e) { - logger.error("Exception in reading PGP security collection ring. Msg: {}", e.getLocalizedMessage()); + logger.error( + "Exception in reading PGP security collection ring. Msg: {}", + e.getLocalizedMessage()); throw new RuntimeException("Exception in reading PGP public key", e); } @@ -94,59 +94,84 @@ private PGPPublicKey getPubKey() { * @return a handle to the decrypted, uncompress data stream. */ @Override - public InputStream decryptStream(InputStream in, char[] passwd, String objectName) throws Exception { + public InputStream decryptStream(InputStream in, char[] passwd, String objectName) + throws Exception { logger.info("Start to decrypt object: {}", objectName); in = PGPUtil.getDecoderStream(in); - PGPObjectFactory inPgpReader = new PGPObjectFactory(in); //general class for reading a stream of data. + // general class for reading a stream of data. + PGPObjectFactory inPgpReader = new PGPObjectFactory(in); Object o = inPgpReader.nextObject(); PGPEncryptedDataList encryptedDataList; // the first object might be a PGP marker packet. - if (o instanceof PGPEncryptedDataList) - encryptedDataList = (PGPEncryptedDataList) o; + if (o instanceof PGPEncryptedDataList) encryptedDataList = (PGPEncryptedDataList) o; else - encryptedDataList = (PGPEncryptedDataList) inPgpReader.nextObject(); //first object was a marker, the real data is the next one. + // first object was a marker, the real data is the next one. + encryptedDataList = (PGPEncryptedDataList) inPgpReader.nextObject(); - Iterator encryptedDataIterator = encryptedDataList.getEncryptedDataObjects(); //get the iterator so we can iterate through all the encrypted data. + // get the iterator so we can iterate through all the encrypted data. + Iterator encryptedDataIterator = encryptedDataList.getEncryptedDataObjects(); - PGPPrivateKey privateKey = null; //to be use for decryption - PGPPublicKeyEncryptedData encryptedDataStreamHandle = null; //a handle to the encrypted data stream + // to be use for decryption + PGPPrivateKey privateKey = null; + // a handle to the encrypted data stream + PGPPublicKeyEncryptedData encryptedDataStreamHandle = null; while (privateKey == null && encryptedDataIterator.hasNext()) { - encryptedDataStreamHandle = (PGPPublicKeyEncryptedData) encryptedDataIterator.next(); //a handle to the encrypted data stream + // a handle to the encrypted data stream + encryptedDataStreamHandle = (PGPPublicKeyEncryptedData) encryptedDataIterator.next(); try { - privateKey = findSecretKey(getPgpSecurityCollection(), encryptedDataStreamHandle.getKeyID(), passwd); + privateKey = + findSecretKey( + getPgpSecurityCollection(), + encryptedDataStreamHandle.getKeyID(), + passwd); } catch (Exception ex) { - throw new IllegalStateException("decryption exception: object: " + objectName + ", Exception when fetching private key using key: " + encryptedDataStreamHandle.getKeyID(), ex); + throw new IllegalStateException( + "decryption exception: object: " + + objectName + + ", Exception when fetching private key using key: " + + encryptedDataStreamHandle.getKeyID(), + ex); } - } if (privateKey == null) - throw new IllegalStateException("decryption exception: object: " + objectName + ", Private key for message not found."); + throw new IllegalStateException( + "decryption exception: object: " + + objectName + + ", Private key for message not found."); - //finally, lets decrypt the object + // finally, lets decrypt the object InputStream decryptInputStream = encryptedDataStreamHandle.getDataStream(privateKey, "BC"); PGPObjectFactory decryptedDataReader = new PGPObjectFactory(decryptInputStream); - //the decrypted data object is compressed, lets decompress it. - PGPCompressedData compressedDataReader = (PGPCompressedData) decryptedDataReader.nextObject(); //get a handle to the decrypted, compress data stream - InputStream compressedStream = new BufferedInputStream(compressedDataReader.getDataStream()); + // the decrypted data object is compressed, lets decompress it. + // get a handle to the decrypted, compress data stream + PGPCompressedData compressedDataReader = + (PGPCompressedData) decryptedDataReader.nextObject(); + InputStream compressedStream = + new BufferedInputStream(compressedDataReader.getDataStream()); PGPObjectFactory compressedStreamReader = new PGPObjectFactory(compressedStream); Object data = compressedStreamReader.nextObject(); if (data instanceof PGPLiteralData) { PGPLiteralData dataPgpReader = (PGPLiteralData) data; - return dataPgpReader.getInputStream(); //a handle to the decrypted, uncompress data stream + // a handle to the decrypted, uncompress data stream + return dataPgpReader.getInputStream(); } else if (data instanceof PGPOnePassSignatureList) { - throw new PGPException("decryption exception: object: " + objectName + ", encrypted data contains a signed message - not literal data."); + throw new PGPException( + "decryption exception: object: " + + objectName + + ", encrypted data contains a signed message - not literal data."); } else { - throw new PGPException("decryption exception: object: " + objectName + ", data is not a simple encrypted file - type unknown."); + throw new PGPException( + "decryption exception: object: " + + objectName + + ", data is not a simple encrypted file - type unknown."); } - - } /* @@ -158,7 +183,9 @@ public InputStream decryptStream(InputStream in, char[] passwd, String objectNam * @param pass - pass phrase used to extract the PGP private key from the encrypted content. * @return PGP private key, null if not found. */ - private static PGPPrivateKey findSecretKey(PGPSecretKeyRingCollection securityCollection, long keyID, char[] pass) throws PGPException, NoSuchProviderException { + private static PGPPrivateKey findSecretKey( + PGPSecretKeyRingCollection securityCollection, long keyID, char[] pass) + throws PGPException, NoSuchProviderException { PGPSecretKey privateKey = securityCollection.getSecretKey(keyID); if (privateKey == null) { @@ -166,7 +193,6 @@ private static PGPPrivateKey findSecretKey(PGPSecretKeyRingCollection securityCo } return privateKey.extractPrivateKey(pass, "BC"); - } @Override @@ -196,7 +222,6 @@ public ChunkEncryptorStream(InputStream is, String fileName, PGPPublicKey pubKey @Override public boolean hasNext() { return this.hasnext; - } /* @@ -212,13 +237,16 @@ public byte[] next() { int count; while ((count = encryptedSrc.read(buffer, 0, buffer.length)) != -1) { pgout.write(buffer, 0, count); - if (bos.size() >= MAX_CHUNK) - return returnSafe(); + if (bos.size() >= MAX_CHUNK) return returnSafe(); } - return done(); //flush remaining data in buffer and close resources. + // flush remaining data in buffer and close resources. + return done(); } catch (Exception e) { - throw new RuntimeException("Error encountered returning next chunk of ciphertext. Msg: " + e.getLocalizedMessage(), e); + throw new RuntimeException( + "Error encountered returning next chunk of ciphertext. Msg: " + + e.getLocalizedMessage(), + e); } } @@ -241,67 +269,80 @@ private byte[] returnSafe() { * flush remaining data in buffer and close resources. */ private byte[] done() throws IOException { - pgout.flush(); //flush whatever is in the buffer to the output stream + pgout.flush(); // flush whatever is in the buffer to the output stream - this.hasnext = false; //tell clients that there is no more data + this.hasnext = false; // tell clients that there is no more data byte[] returnData = this.bos.toByteArray(); - IOUtils.closeQuietly(pgout); //close the handle to the buffered output - IOUtils.closeQuietly(bos); //close the handle to the actual output + IOUtils.closeQuietly(pgout); // close the handle to the buffered output + IOUtils.closeQuietly(bos); // close the handle to the actual output return returnData; - } - } public class EncryptedInputStream extends InputStream { - private final InputStream srcHandle; //handle to the source stream - private ByteArrayOutputStream bos = null; //Handle to encrypted stream - private int bosOff = 0; //current position within encrypted stream - private OutputStream pgpBosWrapper; //wrapper around the buffer which will contain the encrypted data. - private OutputStream encryptedOsWrapper; //handle to the encrypted data - private PGPCompressedDataGenerator compressedDataGenerator; //a means to compress data using PGP - private String fileName; //TODO: eliminate once debugging is completed. + private final InputStream srcHandle; // handle to the source stream + private ByteArrayOutputStream bos = null; // Handle to encrypted stream + private int bosOff = 0; // current position within encrypted stream + private OutputStream + pgpBosWrapper; // wrapper around the buffer which will contain the encrypted data. + private OutputStream encryptedOsWrapper; // handle to the encrypted data + private PGPCompressedDataGenerator + compressedDataGenerator; // a means to compress data using PGP + private String fileName; // TODO: eliminate once debugging is completed. public EncryptedInputStream(InputStream is, String fileName, PGPPublicKey pubKey) { this.srcHandle = is; this.bos = new ByteArrayOutputStream(); - //creates a cipher stream which will have an integrity packet associated with it - PGPEncryptedDataGenerator encryptedDataGenerator = new PGPEncryptedDataGenerator(PGPEncryptedData.CAST5, true, new SecureRandom(), "BC"); + // creates a cipher stream which will have an integrity packet associated with it + PGPEncryptedDataGenerator encryptedDataGenerator = + new PGPEncryptedDataGenerator( + PGPEncryptedData.CAST5, true, new SecureRandom(), "BC"); try { - encryptedDataGenerator.addMethod(pubKey); //Add a key encryption method to be used to encrypt the session data associated with this encrypted data - pgpBosWrapper = encryptedDataGenerator.open(bos, new byte[1 << 15]); //wrapper around the buffer which will contain the encrypted data. + // Add a key encryption method to be used to encrypt the session data associated + // with this encrypted data + encryptedDataGenerator.addMethod(pubKey); + // wrapper around the buffer which will contain the encrypted data. + pgpBosWrapper = encryptedDataGenerator.open(bos, new byte[1 << 15]); } catch (Exception e) { - throw new RuntimeException("Exception when wrapping PGP around our output stream", e); + throw new RuntimeException( + "Exception when wrapping PGP around our output stream", e); } - //a means to compress data using PGP - this.compressedDataGenerator = new PGPCompressedDataGenerator(PGPCompressedData.UNCOMPRESSED); + // a means to compress data using PGP + this.compressedDataGenerator = + new PGPCompressedDataGenerator(PGPCompressedData.UNCOMPRESSED); - /* + /* * Open a literal data packet, returning a stream to store the data inside the packet as an indefinite stream. - * A "literal data packet" in PGP world is the body of a message; data that is not to be further interpreted. - * - * The stream is written out as a series of partial packets with a chunk size determine by the size of the passed in buffer. - * @param outputstream - the stream we want the packet in - * @param format - the format we are using. - * @param filename - * @param the time of last modification we want stored. - * @param the buffer to use for collecting data to put into chunks. - */ + * A "literal data packet" in PGP world is the body of a message; data that is not to be further interpreted. + * + * The stream is written out as a series of partial packets with a chunk size determine by the size of the passed in buffer. + * @param outputstream - the stream we want the packet in + * @param format - the format we are using. + * @param filename + * @param the time of last modification we want stored. + * @param the buffer to use for collecting data to put into chunks. + */ try { PGPLiteralDataGenerator literalDataGenerator = new PGPLiteralDataGenerator(); - this.encryptedOsWrapper = literalDataGenerator.open(compressedDataGenerator.open(pgpBosWrapper), PGPLiteralData.BINARY, fileName, new Date(), new byte[1 << 15]); + this.encryptedOsWrapper = + literalDataGenerator.open( + compressedDataGenerator.open(pgpBosWrapper), + PGPLiteralData.BINARY, + fileName, + new Date(), + new byte[1 << 15]); } catch (Exception e) { - throw new RuntimeException("Exception when creating the PGP encrypted wrapper around the output stream.", e); + throw new RuntimeException( + "Exception when creating the PGP encrypted wrapper around the output stream.", + e); } - this.fileName = fileName; //TODO: eliminate once debugging is completed. - - + this.fileName = fileName; // TODO: eliminate once debugging is completed. } /* @@ -315,50 +356,45 @@ public EncryptedInputStream(InputStream is, String fileName, PGPPublicKey pubKey @Override public synchronized int read(byte b[], int off, int len) throws IOException { if (this.bosOff < this.bos.size()) { - //if here, you still have data in the encrypted stream, lets give it to the client + // if here, you still have data in the encrypted stream, lets give it to the client return copyToBuff(b, off, len); } - //If here, it's time to read the next chunk from the input and do the encryption. + // If here, it's time to read the next chunk from the input and do the encryption. this.bos.reset(); this.bosOff = 0; - //== read up to "len" or end of file from input stream and encrypt it. + // == read up to "len" or end of file from input stream and encrypt it. byte[] buff = new byte[1 << 16]; - int bytesRead = 0; //num of bytes read from the source input stream - - while (this.bos.size() < len && (bytesRead = this.srcHandle.read(buff, 0, len)) > 0) { //lets process each chunk from input until we fill our output stream or we reach end of input - - /* TODO: msg was only for debug purposes - * - logger.info("Reading input file: " + this.fileName + ", number of bytes read from input stream: " + bytesRead - + ", size of buffer: " - + buff.length - ); - - */ + int bytesRead = 0; // num of bytes read from the source input stream + while (this.bos.size() < len && (bytesRead = this.srcHandle.read(buff, 0, len)) > 0) { + // lets process each chunk from input until we fill our output + // stream or we reach end of input this.encryptedOsWrapper.write(buff, 0, bytesRead); } - if (bytesRead < 0) { //we have read everything from the source input, lets perform cleanup on any resources. + if (bytesRead < 0) { + // we have read everything from the source input, lets perform cleanup on + // any resources. this.encryptedOsWrapper.close(); this.compressedDataGenerator.close(); this.pgpBosWrapper.close(); } if (bytesRead < 0 && this.bos.size() == 0) { - //if here, read all the bytes from the input and there is nothing in the encrypted stream. + // if here, read all the bytes from the input and there is nothing in the encrypted + // stream. return bytesRead; } - - /* - * If here, one of the following occurred: - * 1. you read data from the input and encrypted it. - * 2. there was no more data in the input but you still had some data in the encrypted stream. - * - */ + + /* + * If here, one of the following occurred: + * 1. you read data from the input and encrypted it. + * 2. there was no more data in the input but you still had some data in the encrypted stream. + * + */ return copyToBuff(b, off, len); } @@ -372,14 +408,17 @@ public synchronized int read(byte b[], int off, int len) throws IOException { * @return number of bytes copied from the encrypted stream to the output buffer */ private int copyToBuff(byte[] buff, int off, int len) { - /* - * num of bytes to copy within encrypted stream = (current size of bytes within encrypted stream - current position within encrypted stream) < size of output buffer, - * then copy what is in the encrypted stream; otherwise, copy up to the max size of the output buffer. - */ - int wlen = (this.bos.size() - this.bosOff) < len ? (this.bos.size() - this.bosOff) : len; - System.arraycopy(this.bos.toByteArray(), this.bosOff, buff, off, wlen); //copy data within encrypted stream to the output buffer - - this.bosOff = this.bosOff + wlen; //now update the current position within the encrypted stream + /* + * num of bytes to copy within encrypted stream = (current size of bytes within encrypted stream - current position within encrypted stream) < size of output buffer, + * then copy what is in the encrypted stream; otherwise, copy up to the max size of the output buffer. + */ + int wlen = + (this.bos.size() - this.bosOff) < len ? (this.bos.size() - this.bosOff) : len; + // copy data within encrypted stream to the output buffer + System.arraycopy(this.bos.toByteArray(), this.bosOff, buff, off, wlen); + + // now update the current position within the encrypted stream + this.bosOff = this.bosOff + wlen; return wlen; } @@ -392,8 +431,8 @@ public void close() throws IOException { @Override public int read() throws IOException { - throw new UnsupportedOperationException("Not supported, invoke read(byte[] bytes, int off, int len) instead."); + throw new UnsupportedOperationException( + "Not supported, invoke read(byte[] bytes, int off, int len) instead."); } - } -} \ No newline at end of file +} diff --git a/priam/src/main/java/com/netflix/priam/cryptography/pgp/PgpUtil.java b/priam/src/main/java/com/netflix/priam/cryptography/pgp/PgpUtil.java index 633d2f7b4..651c7c81c 100755 --- a/priam/src/main/java/com/netflix/priam/cryptography/pgp/PgpUtil.java +++ b/priam/src/main/java/com/netflix/priam/cryptography/pgp/PgpUtil.java @@ -1,44 +1,42 @@ /** * Copyright 2017 Netflix, Inc. - *

- * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and + * + *

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file + * except in compliance with the License. You may obtain a copy of the License at + * + *

http://www.apache.org/licenses/LICENSE-2.0 + * + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing permissions and * limitations under the License. */ package com.netflix.priam.cryptography.pgp; -import org.bouncycastle.openpgp.*; - import java.io.BufferedInputStream; import java.io.FileInputStream; import java.io.IOException; import java.io.InputStream; import java.security.NoSuchProviderException; import java.util.Iterator; - +import org.bouncycastle.openpgp.*; public class PgpUtil { /** - * Search a secret key ring collection for a secret key corresponding to keyID if it - * exists. + * Search a secret key ring collection for a secret key corresponding to keyID if it exists. * * @param pgpSec a secret key ring collection. - * @param keyID keyID we want. - * @param pass passphrase to decrypt secret key with. + * @param keyID keyID we want. + * @param pass passphrase to decrypt secret key with. * @return secret or private key corresponding to the keyID. - * @throws PGPException if there is any exception in getting the PGP key corresponding to the ID provided. + * @throws PGPException if there is any exception in getting the PGP key corresponding to the ID + * provided. * @throws NoSuchProviderException If PGP Provider is not available. */ - public static PGPPrivateKey findSecretKey(PGPSecretKeyRingCollection pgpSec, long keyID, char[] pass) throws PGPException, NoSuchProviderException { + public static PGPPrivateKey findSecretKey( + PGPSecretKeyRingCollection pgpSec, long keyID, char[] pass) + throws PGPException, NoSuchProviderException { PGPSecretKey pgpSecKey = pgpSec.getSecretKey(keyID); @@ -57,8 +55,8 @@ public static PGPPublicKey readPublicKey(String fileName) throws IOException, PG } /** - * A simple routine that opens a key ring file and loads the first available key - * suitable for encryption. + * A simple routine that opens a key ring file and loads the first available key suitable for + * encryption. * * @param input inputstream to the pgp file key ring. * @return PGP key from the key ring. @@ -68,10 +66,12 @@ public static PGPPublicKey readPublicKey(String fileName) throws IOException, PG @SuppressWarnings("rawtypes") public static PGPPublicKey readPublicKey(InputStream input) throws IOException, PGPException { - PGPPublicKeyRingCollection pgpPub = new PGPPublicKeyRingCollection(PGPUtil.getDecoderStream(input)); + PGPPublicKeyRingCollection pgpPub = + new PGPPublicKeyRingCollection(PGPUtil.getDecoderStream(input)); // - // we just loop through the collection till we find a key suitable for encryption, in the real + // we just loop through the collection till we find a key suitable for encryption, in the + // real // world you would probably want to be a bit smarter about this. // @@ -91,6 +91,4 @@ public static PGPPublicKey readPublicKey(InputStream input) throws IOException, throw new IllegalArgumentException("Can't find encryption key in key ring."); } - - -} \ No newline at end of file +} diff --git a/priam/src/main/java/com/netflix/priam/defaultimpl/CassandraOperations.java b/priam/src/main/java/com/netflix/priam/defaultimpl/CassandraOperations.java index fec12705a..f13ff383b 100644 --- a/priam/src/main/java/com/netflix/priam/defaultimpl/CassandraOperations.java +++ b/priam/src/main/java/com/netflix/priam/defaultimpl/CassandraOperations.java @@ -1,16 +1,14 @@ /** * Copyright 2018 Netflix, Inc. - *

- * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and + * + *

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file + * except in compliance with the License. You may obtain a copy of the License at + * + *

http://www.apache.org/licenses/LICENSE-2.0 + * + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing permissions and * limitations under the License. */ package com.netflix.priam.defaultimpl; @@ -18,44 +16,45 @@ import com.netflix.priam.config.IConfiguration; import com.netflix.priam.utils.JMXNodeTool; import com.netflix.priam.utils.RetryableCallable; +import java.util.*; +import javax.inject.Inject; import org.apache.cassandra.db.ColumnFamilyStoreMBean; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import javax.inject.Inject; -import java.util.*; - -/** - * This class encapsulates interactions with Cassandra. - * Created by aagrawal on 6/19/18. - */ +/** This class encapsulates interactions with Cassandra. Created by aagrawal on 6/19/18. */ public class CassandraOperations { private static final Logger logger = LoggerFactory.getLogger(CassandraOperations.class); private final IConfiguration configuration; @Inject - CassandraOperations(IConfiguration configuration) - { + CassandraOperations(IConfiguration configuration) { this.configuration = configuration; } /** - * This method neds to be synchronized. Context: During the transition phase to backup version 2.0, we might be executing - * multiple snapshots at the same time. To avoid, unknown behavior by Cassanddra, it is wise to keep this method sync. - * Also, with backups being on CRON, we don't know how often operator is taking snapshot. - * @param snapshotName Name of the snapshot on disk. This snapshotName should be UNIQUE among all the snapshots. - * Try to append UUID to snapshotName to ensure uniqueness. - * This is to ensure a) Snapshot fails if name are not unique. - * b) You might take snapshots which are not "part" of same snapshot. e.g. Any leftovers from previous operation. - * c) Once snapshot fails, this will clean the failed snapshot. + * This method neds to be synchronized. Context: During the transition phase to backup version + * 2.0, we might be executing multiple snapshots at the same time. To avoid, unknown behavior by + * Cassanddra, it is wise to keep this method sync. Also, with backups being on CRON, we don't + * know how often operator is taking snapshot. + * + * @param snapshotName Name of the snapshot on disk. This snapshotName should be UNIQUE among + * all the snapshots. Try to append UUID to snapshotName to ensure uniqueness. This is to + * ensure a) Snapshot fails if name are not unique. b) You might take snapshots which are + * not "part" of same snapshot. e.g. Any leftovers from previous operation. c) Once snapshot + * fails, this will clean the failed snapshot. * @throws Exception in case of error while taking a snapshot by Cassandra. */ public synchronized void takeSnapshot(final String snapshotName) throws Exception { - //Retry max of 6 times with 10 second in between (for one minute). This is to ensure that we overcome any temporary glitch. - //Note that operation MAY fail if cassandra successfully took the snapshot of certain columnfamily(ies) and we try to create snapshot with - //same name. It is a good practice to call clearSnapshot after this operation fails, to ensure we don't leave - //any left overs. - //Example scenario: Change of file permissions by manual intervention and C* unable to take snapshot of one CF. + // Retry max of 6 times with 10 second in between (for one minute). This is to ensure that + // we overcome any temporary glitch. + // Note that operation MAY fail if cassandra successfully took the snapshot of certain + // columnfamily(ies) and we try to create snapshot with + // same name. It is a good practice to call clearSnapshot after this operation fails, to + // ensure we don't leave + // any left overs. + // Example scenario: Change of file permissions by manual intervention and C* unable to take + // snapshot of one CF. try { new RetryableCallable(6, 10000) { public Void retriableCall() throws Exception { @@ -64,8 +63,10 @@ public Void retriableCall() throws Exception { return null; } }.call(); - }catch (Exception e){ - logger.error("Error while taking snapshot {}. Asking Cassandra to clear snapshot to avoid accumulation of snapshots.", snapshotName); + } catch (Exception e) { + logger.error( + "Error while taking snapshot {}. Asking Cassandra to clear snapshot to avoid accumulation of snapshots.", + snapshotName); clearSnapshot(snapshotName); throw e; } @@ -73,6 +74,7 @@ public Void retriableCall() throws Exception { /** * Clear the snapshot tag from disk. + * * @param snapshotTag Name of the snapshot to be removed. * @throws Exception in case of error while clearing a snapshot. */ @@ -86,49 +88,53 @@ public Void retriableCall() throws Exception { }.call(); } - - public List getKeyspaces() throws Exception{ - return new RetryableCallable>(){ - public List retriableCall() throws Exception{ - try(JMXNodeTool nodeTool = JMXNodeTool.instance(configuration)) { + public List getKeyspaces() throws Exception { + return new RetryableCallable>() { + public List retriableCall() throws Exception { + try (JMXNodeTool nodeTool = JMXNodeTool.instance(configuration)) { return nodeTool.getKeyspaces(); } } }.call(); } - public Map> getColumnfamilies() throws Exception{ - return new RetryableCallable>>(){ - public Map> retriableCall() throws Exception{ - try(JMXNodeTool nodeTool = JMXNodeTool.instance(configuration)) { + public Map> getColumnfamilies() throws Exception { + return new RetryableCallable>>() { + public Map> retriableCall() throws Exception { + try (JMXNodeTool nodeTool = JMXNodeTool.instance(configuration)) { final Map> columnfamilies = new HashMap<>(); - Iterator> columnfamilyStoreMBean = nodeTool.getColumnFamilyStoreMBeanProxies(); - columnfamilyStoreMBean.forEachRemaining(entry -> { - columnfamilies.putIfAbsent(entry.getKey(), new ArrayList<>()); - columnfamilies.get(entry.getKey()).add(entry.getValue().getColumnFamilyName()); - }); + Iterator> columnfamilyStoreMBean = + nodeTool.getColumnFamilyStoreMBeanProxies(); + columnfamilyStoreMBean.forEachRemaining( + entry -> { + columnfamilies.putIfAbsent(entry.getKey(), new ArrayList<>()); + columnfamilies + .get(entry.getKey()) + .add(entry.getValue().getColumnFamilyName()); + }); return columnfamilies; } } }.call(); } - public void forceKeyspaceCompaction(String keyspaceName, String... columnfamilies) throws Exception{ - new RetryableCallable(){ - public Void retriableCall() throws Exception{ - try(JMXNodeTool nodeTool = JMXNodeTool.instance(configuration)) { - nodeTool.forceKeyspaceCompaction(false, keyspaceName, columnfamilies); - return null; + public void forceKeyspaceCompaction(String keyspaceName, String... columnfamilies) + throws Exception { + new RetryableCallable() { + public Void retriableCall() throws Exception { + try (JMXNodeTool nodeTool = JMXNodeTool.instance(configuration)) { + nodeTool.forceKeyspaceCompaction(false, keyspaceName, columnfamilies); + return null; } } }.call(); } - public void forceKeyspaceFlush(String keyspaceName) throws Exception{ - new RetryableCallable(){ - public Void retriableCall() throws Exception{ - try(JMXNodeTool nodeTool = JMXNodeTool.instance(configuration)) { - nodeTool.forceKeyspaceFlush(keyspaceName); + public void forceKeyspaceFlush(String keyspaceName) throws Exception { + new RetryableCallable() { + public Void retriableCall() throws Exception { + try (JMXNodeTool nodeTool = JMXNodeTool.instance(configuration)) { + nodeTool.forceKeyspaceFlush(keyspaceName); return null; } } diff --git a/priam/src/main/java/com/netflix/priam/defaultimpl/CassandraProcessManager.java b/priam/src/main/java/com/netflix/priam/defaultimpl/CassandraProcessManager.java index 6dbaeaf52..86bc7e49c 100644 --- a/priam/src/main/java/com/netflix/priam/defaultimpl/CassandraProcessManager.java +++ b/priam/src/main/java/com/netflix/priam/defaultimpl/CassandraProcessManager.java @@ -1,16 +1,14 @@ /** * Copyright 2017 Netflix, Inc. - *

- * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and + * + *

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file + * except in compliance with the License. You may obtain a copy of the License at + * + *

http://www.apache.org/licenses/LICENSE-2.0 + * + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing permissions and * limitations under the License. */ package com.netflix.priam.defaultimpl; @@ -21,10 +19,6 @@ import com.netflix.priam.health.InstanceState; import com.netflix.priam.merics.CassMonitorMetrics; import com.netflix.priam.utils.JMXNodeTool; -import org.apache.commons.lang3.StringUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import java.io.ByteArrayOutputStream; import java.io.File; import java.io.IOException; @@ -33,6 +27,9 @@ import java.util.List; import java.util.Map; import java.util.concurrent.*; +import org.apache.commons.lang3.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class CassandraProcessManager implements ICassandraProcess { private static final Logger logger = LoggerFactory.getLogger(CassandraProcessManager.class); @@ -43,7 +40,10 @@ public class CassandraProcessManager implements ICassandraProcess { private final CassMonitorMetrics cassMonitorMetrics; @Inject - public CassandraProcessManager(IConfiguration config, InstanceState instanceState, CassMonitorMetrics cassMonitorMetrics) { + public CassandraProcessManager( + IConfiguration config, + InstanceState instanceState, + CassMonitorMetrics cassMonitorMetrics) { this.config = config; this.instanceState = instanceState; this.cassMonitorMetrics = cassMonitorMetrics; @@ -67,14 +67,13 @@ public void start(boolean join_ring) throws IOException { instanceState.setShouldCassandraBeAlive(true); List command = Lists.newArrayList(); - if(config.useSudo()) { + if (config.useSudo()) { logger.info("Configured to use sudo to start C*"); if (!"root".equals(System.getProperty("user.name"))) { command.add(SUDO_STRING); command.add("-n"); command.add("-E"); } - } command.addAll(getStartCommand()); @@ -93,14 +92,12 @@ public void start(boolean join_ring) throws IOException { logger.info("Starting cassandra server ...."); try { - int code = starter.waitFor(); + int code = starter.waitFor(); if (code == 0) { logger.info("Cassandra server has been started"); instanceState.setCassandraProcessAlive(true); this.cassMonitorMetrics.incCassStart(); - } - else - logger.error("Unable to start cassandra server. Error code: {}", code); + } else logger.error("Unable to start cassandra server. Error code: {}", code); logProcessOutput(starter); } catch (Exception e) { @@ -111,8 +108,7 @@ public void start(boolean join_ring) throws IOException { protected List getStartCommand() { List startCmd = new LinkedList<>(); for (String param : config.getCassStartupScript().split(" ")) { - if (StringUtils.isNotBlank(param)) - startCmd.add(param); + if (StringUtils.isNotBlank(param)) startCmd.add(param); } return startCmd; } @@ -132,16 +128,14 @@ String readProcessStream(InputStream inputStream) throws IOException { final byte[] buffer = new byte[512]; final ByteArrayOutputStream baos = new ByteArrayOutputStream(buffer.length); int cnt; - while ((cnt = inputStream.read(buffer)) != -1) - baos.write(buffer, 0, cnt); + while ((cnt = inputStream.read(buffer)) != -1) baos.write(buffer, 0, cnt); return baos.toString(); } - public void stop(boolean force) throws IOException { logger.info("Stopping cassandra server ...."); List command = Lists.newArrayList(); - if(config.useSudo()) { + if (config.useSudo()) { logger.info("Configured to use sudo to stop C*"); if (!"root".equals(System.getProperty("user.name"))) { @@ -151,8 +145,7 @@ public void stop(boolean force) throws IOException { } } for (String param : config.getCassStopScript().split(" ")) { - if (StringUtils.isNotBlank(param)) - command.add(param); + if (StringUtils.isNotBlank(param)) command.add(param); } ProcessBuilder stopCass = new ProcessBuilder(command); stopCass.directory(new File("/")); @@ -161,31 +154,42 @@ public void stop(boolean force) throws IOException { instanceState.setShouldCassandraBeAlive(false); if (!force && config.getGracefulDrainHealthWaitSeconds() >= 0) { ExecutorService executor = Executors.newSingleThreadExecutor(); - Future drainFuture = executor.submit(() -> { - // As the node has been marked as shutting down above in setShouldCassandraBeAlive, we wait this - // duration to allow external healthcheck systems time to pick up the state change. - try { - Thread.sleep(config.getGracefulDrainHealthWaitSeconds() * 1000); - } catch (InterruptedException e) { - return; - } - - try { - JMXNodeTool nodetool = JMXNodeTool.instance(config); - nodetool.drain(); - } catch (InterruptedException | IOException | ExecutionException e) { - logger.error("Exception draining Cassandra, could not drain. Proceeding with shutdown.", e); - } - // Once Cassandra is drained the thrift/native servers are shutdown and there is no need to wait to - // stop Cassandra. Just stop it now. - }); - - // In case drain hangs, timeout the future and continue stopping anyways. Give drain 30s always + Future drainFuture = + executor.submit( + () -> { + // As the node has been marked as shutting down above in + // setShouldCassandraBeAlive, we wait this + // duration to allow external healthcheck systems time to pick up + // the state change. + try { + Thread.sleep(config.getGracefulDrainHealthWaitSeconds() * 1000); + } catch (InterruptedException e) { + return; + } + + try { + JMXNodeTool nodetool = JMXNodeTool.instance(config); + nodetool.drain(); + } catch (InterruptedException + | IOException + | ExecutionException e) { + logger.error( + "Exception draining Cassandra, could not drain. Proceeding with shutdown.", + e); + } + // Once Cassandra is drained the thrift/native servers are shutdown + // and there is no need to wait to + // stop Cassandra. Just stop it now. + }); + + // In case drain hangs, timeout the future and continue stopping anyways. Give drain 30s + // always // In production we freqently see servers that do not want to drain try { drainFuture.get(config.getGracefulDrainHealthWaitSeconds() + 30, TimeUnit.SECONDS); } catch (ExecutionException | TimeoutException | InterruptedException e) { - logger.error("Waited 30s for drain but it did not complete, continuing to shutdown", e); + logger.error( + "Waited 30s for drain but it did not complete, continuing to shutdown", e); } } Process stopper = stopCass.start(); @@ -195,8 +199,7 @@ public void stop(boolean force) throws IOException { logger.info("Cassandra server has been stopped"); this.cassMonitorMetrics.incCassStop(); instanceState.setCassandraProcessAlive(false); - } - else { + } else { logger.error("Unable to stop cassandra server. Error code: {}", code); logProcessOutput(stopper); } @@ -204,4 +207,4 @@ public void stop(boolean force) throws IOException { logger.warn("couldn't shut down cassandra correctly", e); } } -} \ No newline at end of file +} diff --git a/priam/src/main/java/com/netflix/priam/defaultimpl/ICassandraProcess.java b/priam/src/main/java/com/netflix/priam/defaultimpl/ICassandraProcess.java index fd33b86d7..8f0599649 100644 --- a/priam/src/main/java/com/netflix/priam/defaultimpl/ICassandraProcess.java +++ b/priam/src/main/java/com/netflix/priam/defaultimpl/ICassandraProcess.java @@ -17,8 +17,6 @@ package com.netflix.priam.defaultimpl; import com.google.inject.ImplementedBy; -import com.netflix.priam.defaultimpl.CassandraProcessManager; - import java.io.IOException; /** diff --git a/priam/src/main/java/com/netflix/priam/defaultimpl/InjectedWebListener.java b/priam/src/main/java/com/netflix/priam/defaultimpl/InjectedWebListener.java index a596ac1f3..b1c3ab2cd 100644 --- a/priam/src/main/java/com/netflix/priam/defaultimpl/InjectedWebListener.java +++ b/priam/src/main/java/com/netflix/priam/defaultimpl/InjectedWebListener.java @@ -22,25 +22,25 @@ import com.google.inject.Module; import com.google.inject.servlet.GuiceServletContextListener; import com.google.inject.servlet.ServletModule; -import com.netflix.priam.config.IConfiguration; import com.netflix.priam.PriamServer; +import com.netflix.priam.config.IConfiguration; import com.sun.jersey.api.core.PackagesResourceConfig; import com.sun.jersey.guice.spi.container.servlet.GuiceContainer; import com.sun.jersey.spi.container.servlet.ServletContainer; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import javax.servlet.ServletContextEvent; import org.quartz.Scheduler; import org.quartz.SchedulerException; import org.quartz.SchedulerFactory; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import javax.servlet.ServletContextEvent; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - public class InjectedWebListener extends GuiceServletContextListener { protected static final Logger logger = LoggerFactory.getLogger(InjectedWebListener.class); private Injector injector; + @Override protected Injector getInjector() { List moduleList = Lists.newArrayList(); @@ -59,14 +59,12 @@ protected Injector getInjector() { @Override public void contextDestroyed(ServletContextEvent servletContextEvent) { - try - { - for (Scheduler scheduler : injector.getInstance(SchedulerFactory.class).getAllSchedulers()){ + try { + for (Scheduler scheduler : + injector.getInstance(SchedulerFactory.class).getAllSchedulers()) { scheduler.shutdown(); } - } - catch (SchedulerException e) - { + } catch (SchedulerException e) { throw new RuntimeException(e); } super.contextDestroyed(servletContextEvent); @@ -82,5 +80,4 @@ protected void configureServlets() { serve("/REST/*").with(GuiceContainer.class, params); } } - -} \ No newline at end of file +} diff --git a/priam/src/main/java/com/netflix/priam/defaultimpl/PriamGuiceModule.java b/priam/src/main/java/com/netflix/priam/defaultimpl/PriamGuiceModule.java index 10e8092a6..a351ed42c 100644 --- a/priam/src/main/java/com/netflix/priam/defaultimpl/PriamGuiceModule.java +++ b/priam/src/main/java/com/netflix/priam/defaultimpl/PriamGuiceModule.java @@ -18,8 +18,6 @@ import com.google.inject.AbstractModule; import com.google.inject.name.Names; -import com.netflix.priam.cred.ICredential; -import com.netflix.priam.cred.ICredentialGeneric; import com.netflix.priam.aws.S3CrossAccountFileSystem; import com.netflix.priam.aws.S3EncryptedFileSystem; import com.netflix.priam.aws.S3FileSystem; @@ -27,6 +25,8 @@ import com.netflix.priam.aws.auth.IS3Credential; import com.netflix.priam.aws.auth.S3RoleAssumptionCredential; import com.netflix.priam.backup.IBackupFileSystem; +import com.netflix.priam.cred.ICredential; +import com.netflix.priam.cred.ICredentialGeneric; import com.netflix.priam.cryptography.IFileCryptography; import com.netflix.priam.cryptography.pgp.PgpCredential; import com.netflix.priam.cryptography.pgp.PgpCryptography; @@ -37,23 +37,36 @@ import org.quartz.SchedulerFactory; import org.quartz.impl.StdSchedulerFactory; - public class PriamGuiceModule extends AbstractModule { @Override protected void configure() { bind(SchedulerFactory.class).to(StdSchedulerFactory.class).asEagerSingleton(); bind(IBackupFileSystem.class).annotatedWith(Names.named("backup")).to(S3FileSystem.class); - bind(IBackupFileSystem.class).annotatedWith(Names.named("encryptedbackup")).to(S3EncryptedFileSystem.class); + bind(IBackupFileSystem.class) + .annotatedWith(Names.named("encryptedbackup")) + .to(S3EncryptedFileSystem.class); bind(S3CrossAccountFileSystem.class); - bind(IBackupFileSystem.class).annotatedWith(Names.named("gcsencryptedbackup")).to(GoogleEncryptedFileSystem.class); - bind(IS3Credential.class).annotatedWith(Names.named("awss3roleassumption")).to(S3RoleAssumptionCredential.class); - bind(ICredential.class).annotatedWith(Names.named("awsec2roleassumption")).to(EC2RoleAssumptionCredential.class); - bind(IFileCryptography.class).annotatedWith(Names.named("filecryptoalgorithm")).to(PgpCryptography.class); - bind(ICredentialGeneric.class).annotatedWith(Names.named("gcscredential")).to(GcsCredential.class); - bind(ICredentialGeneric.class).annotatedWith(Names.named("pgpcredential")).to(PgpCredential.class); + bind(IBackupFileSystem.class) + .annotatedWith(Names.named("gcsencryptedbackup")) + .to(GoogleEncryptedFileSystem.class); + bind(IS3Credential.class) + .annotatedWith(Names.named("awss3roleassumption")) + .to(S3RoleAssumptionCredential.class); + bind(ICredential.class) + .annotatedWith(Names.named("awsec2roleassumption")) + .to(EC2RoleAssumptionCredential.class); + bind(IFileCryptography.class) + .annotatedWith(Names.named("filecryptoalgorithm")) + .to(PgpCryptography.class); + bind(ICredentialGeneric.class) + .annotatedWith(Names.named("gcscredential")) + .to(GcsCredential.class); + bind(ICredentialGeneric.class) + .annotatedWith(Names.named("pgpcredential")) + .to(PgpCredential.class); bind(Registry.class).toInstance(new NoopRegistry()); } } diff --git a/priam/src/main/java/com/netflix/priam/google/GcsCredential.java b/priam/src/main/java/com/netflix/priam/google/GcsCredential.java index 7416e2804..21cfa4023 100755 --- a/priam/src/main/java/com/netflix/priam/google/GcsCredential.java +++ b/priam/src/main/java/com/netflix/priam/google/GcsCredential.java @@ -1,16 +1,14 @@ /** * Copyright 2017 Netflix, Inc. - *

- * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and + * + *

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file + * except in compliance with the License. You may obtain a copy of the License at + * + *

http://www.apache.org/licenses/LICENSE-2.0 + * + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing permissions and * limitations under the License. */ package com.netflix.priam.google; @@ -22,7 +20,7 @@ /* * A generic implementation of fetch keys as plaintext. The key values are used with Google Cloud Storage. Users may - * want to provide an implementation where your key(s)' value is decrypted using AES encryption algorithm. + * want to provide an implementation where your key(s)' value is decrypted using AES encryption algorithm. */ public class GcsCredential implements ICredentialGeneric { @@ -53,5 +51,4 @@ public byte[] getValue(KEY key) { throw new IllegalArgumentException("Key value not supported."); } } - } diff --git a/priam/src/main/java/com/netflix/priam/google/GoogleEncryptedFileSystem.java b/priam/src/main/java/com/netflix/priam/google/GoogleEncryptedFileSystem.java index f2dfdeb5d..4dd53c38c 100755 --- a/priam/src/main/java/com/netflix/priam/google/GoogleEncryptedFileSystem.java +++ b/priam/src/main/java/com/netflix/priam/google/GoogleEncryptedFileSystem.java @@ -1,16 +1,14 @@ /** * Copyright 2017 Netflix, Inc. - *

- * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and + * + *

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file + * except in compliance with the License. You may obtain a copy of the License at + * + *

http://www.apache.org/licenses/LICENSE-2.0 + * + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing permissions and * limitations under the License. */ package com.netflix.priam.google; @@ -26,26 +24,25 @@ import com.google.inject.Inject; import com.google.inject.Provider; import com.google.inject.name.Named; -import com.netflix.priam.config.IConfiguration; -import com.netflix.priam.cred.ICredentialGeneric; -import com.netflix.priam.cred.ICredentialGeneric.KEY; import com.netflix.priam.aws.S3BackupPath; import com.netflix.priam.backup.AbstractBackupPath; import com.netflix.priam.backup.AbstractFileSystem; import com.netflix.priam.backup.BackupRestoreException; +import com.netflix.priam.config.IConfiguration; +import com.netflix.priam.cred.ICredentialGeneric; +import com.netflix.priam.cred.ICredentialGeneric.KEY; import com.netflix.priam.merics.BackupMetrics; import com.netflix.priam.notification.BackupNotificationMgr; -import org.apache.commons.io.IOUtils; -import org.apache.commons.lang3.StringUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import java.io.*; import java.nio.file.Path; import java.util.ArrayList; import java.util.Collection; import java.util.Date; import java.util.Iterator; +import org.apache.commons.io.IOUtils; +import org.apache.commons.lang3.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class GoogleEncryptedFileSystem extends AbstractFileSystem { @@ -55,7 +52,8 @@ public class GoogleEncryptedFileSystem extends AbstractFileSystem { private static final JsonFactory JSON_FACTORY = JacksonFactory.getDefaultInstance(); private HttpTransport httpTransport; - private Credential credential; //represents our "service account" credentials we will use to access GCS + // represents our "service account" credentials we will use to access GCS + private Credential credential; private Storage gcsStorageHandle; private Storage.Objects objectsResoruceHandle = null; @@ -67,8 +65,12 @@ public class GoogleEncryptedFileSystem extends AbstractFileSystem { private final BackupMetrics backupMetrics; @Inject - public GoogleEncryptedFileSystem(Provider pathProvider, final IConfiguration config - , @Named("gcscredential") ICredentialGeneric credential, BackupMetrics backupMetrics, BackupNotificationMgr backupNotificationManager) { + public GoogleEncryptedFileSystem( + Provider pathProvider, + final IConfiguration config, + @Named("gcscredential") ICredentialGeneric credential, + BackupMetrics backupMetrics, + BackupNotificationMgr backupNotificationManager) { super(config, backupMetrics, backupNotificationManager); this.backupMetrics = backupMetrics; this.pathProvider = pathProvider; @@ -78,15 +80,16 @@ public GoogleEncryptedFileSystem(Provider pathProvider, fina try { this.httpTransport = GoogleNetHttpTransport.newTrustedTransport(); } catch (Exception e) { - throw new IllegalStateException("Unable to create a handle to the Google Http tranport", e); + throw new IllegalStateException( + "Unable to create a handle to the Google Http tranport", e); } this.srcBucketName = getSourcebucket(getPathPrefix()); } /* - * @param pathprefix - the absolute path (including bucket name) to the object. - */ + * @param pathprefix - the absolute path (including bucket name) to the object. + */ private String getSourcebucket(String pathPrefix) { String[] paths = pathPrefix.split(String.valueOf(S3BackupPath.PATH_SEP)); return paths[0]; @@ -105,7 +108,7 @@ private Storage.Objects constructObjectResourceHandle() { /* * Get a handle to the GCS api to manage our data within their storage. Code derive from * https://code.google.com/p/google-api-java-client/source/browse/storage-cmdline-sample/src/main/java/com/google/api/services/samples/storage/cmdline/StorageSample.java?repo=samples - * + * * Note: GCS storage will use our credential to do auto-refresh of expired tokens */ private Storage constructGcsStorageHandle() { @@ -119,12 +122,17 @@ private Storage constructGcsStorageHandle() { throw new IllegalStateException("Exception during GCS authorization", e); } - this.gcsStorageHandle = new Storage.Builder(this.httpTransport, JSON_FACTORY, this.credential).setApplicationName(APPLICATION_NAME).build(); + this.gcsStorageHandle = + new Storage.Builder(this.httpTransport, JSON_FACTORY, this.credential) + .setApplicationName(APPLICATION_NAME) + .build(); return this.gcsStorageHandle; } - /** Authorizes the installed application to access user's protected data, code from https://developers.google.com/maps-engine/documentation/oauth/serviceaccount - * and http://javadoc.google-api-java-client.googlecode.com/hg/1.8.0-beta/com/google/api/client/googleapis/auth/oauth2/GoogleCredential.html + /** + * Authorizes the installed application to access user's protected data, code from + * https://developers.google.com/maps-engine/documentation/oauth/serviceaccount and + * http://javadoc.google-api-java-client.googlecode.com/hg/1.8.0-beta/com/google/api/client/googleapis/auth/oauth2/GoogleCredential.html */ private Credential constructGcsCredential() throws Exception { @@ -133,36 +141,46 @@ private Credential constructGcsCredential() throws Exception { } synchronized (this) { - if (this.credential == null) { - String service_acct_email = new String(this.gcsCredential.getValue(KEY.GCS_SERVICE_ID)); + String service_acct_email = + new String(this.gcsCredential.getValue(KEY.GCS_SERVICE_ID)); - if (this.config.getGcsServiceAccountPrivateKeyLoc() == null || this.config.getGcsServiceAccountPrivateKeyLoc().isEmpty()) { - throw new NullPointerException("Fast property for the the GCS private key file is null/empty."); + if (this.config.getGcsServiceAccountPrivateKeyLoc() == null + || this.config.getGcsServiceAccountPrivateKeyLoc().isEmpty()) { + throw new NullPointerException( + "Fast property for the the GCS private key file is null/empty."); } - //Take the encrypted private key, decrypted into an in-transit file which is passed to GCS - File gcsPrivateKeyHandle = new File(this.config.getGcsServiceAccountPrivateKeyLoc() + ".output"); + // Take the encrypted private key, decrypted into an in-transit file which is passed + // to GCS + File gcsPrivateKeyHandle = + new File(this.config.getGcsServiceAccountPrivateKeyLoc() + ".output"); ByteArrayOutputStream byteos = new ByteArrayOutputStream(); - byte[] gcsPrivateKeyPlainText = this.gcsCredential.getValue(KEY.GCS_PRIVATE_KEY_LOC); - try(BufferedOutputStream bos = new BufferedOutputStream(new FileOutputStream(gcsPrivateKeyHandle))) { + byte[] gcsPrivateKeyPlainText = + this.gcsCredential.getValue(KEY.GCS_PRIVATE_KEY_LOC); + try (BufferedOutputStream bos = + new BufferedOutputStream(new FileOutputStream(gcsPrivateKeyHandle))) { byteos.write(gcsPrivateKeyPlainText); byteos.writeTo(bos); } catch (IOException e) { - throw new IOException("Exception when writing decrypted gcs private key value to disk.", e); + throw new IOException( + "Exception when writing decrypted gcs private key value to disk.", e); } Collection scopes = new ArrayList<>(1); scopes.add(StorageScopes.DEVSTORAGE_READ_ONLY); - this.credential = new GoogleCredential.Builder().setTransport(this.httpTransport) - .setJsonFactory(JSON_FACTORY) - .setServiceAccountId(service_acct_email) - .setServiceAccountScopes(scopes) - .setServiceAccountPrivateKeyFromP12File(gcsPrivateKeyHandle) //Cryptex decrypted service account key derive from the GCS console - .build(); + // Cryptex decrypted service account key derive from the GCS console + this.credential = + new GoogleCredential.Builder() + .setTransport(this.httpTransport) + .setJsonFactory(JSON_FACTORY) + .setServiceAccountId(service_acct_email) + .setServiceAccountScopes(scopes) + .setServiceAccountPrivateKeyFromP12File(gcsPrivateKeyHandle) + .build(); } } @@ -170,24 +188,37 @@ private Credential constructGcsCredential() throws Exception { } @Override - protected void downloadFileImpl(Path remotePath, Path localPath) throws BackupRestoreException{ + protected void downloadFileImpl(Path remotePath, Path localPath) throws BackupRestoreException { String objectName = parseObjectname(getPathPrefix()); com.google.api.services.storage.Storage.Objects.Get get; try { get = constructObjectResourceHandle().get(this.srcBucketName, remotePath.toString()); } catch (IOException e) { - throw new BackupRestoreException("IO error retrieving metadata for: " + objectName + " from bucket: " + this.srcBucketName, e); + throw new BackupRestoreException( + "IO error retrieving metadata for: " + + objectName + + " from bucket: " + + this.srcBucketName, + e); } - get.getMediaHttpDownloader().setDirectDownloadEnabled(true); // If you're not using GCS' AppEngine, download the whole thing (instead of chunks) in one request, if possible. - try(OutputStream os = new FileOutputStream(localPath.toFile()); - InputStream is = get.executeMediaAsInputStream()) { + // If you're not using GCS' AppEngine, download the whole thing (instead of chunks) in one + // request, if possible. + get.getMediaHttpDownloader().setDirectDownloadEnabled(true); + try (OutputStream os = new FileOutputStream(localPath.toFile()); + InputStream is = get.executeMediaAsInputStream()) { IOUtils.copyLarge(is, os); } catch (IOException e) { - throw new BackupRestoreException("IO error during streaming of object: " + objectName + " from bucket: " + this.srcBucketName, e); + throw new BackupRestoreException( + "IO error during streaming of object: " + + objectName + + " from bucket: " + + this.srcBucketName, + e); } catch (Exception ex) { - throw new BackupRestoreException("Exception encountered when copying bytes from input to output", ex); + throw new BackupRestoreException( + "Exception encountered when copying bytes from input to output", ex); } backupMetrics.recordDownloadRate(get.getLastResponseHeaders().getContentLength()); @@ -224,15 +255,11 @@ public long getFileSize(Path remotePath) throws BackupRestoreException { return 0; } - /** - * Get restore prefix which will be used to locate GVS files - */ + /** Get restore prefix which will be used to locate GVS files */ private String getPathPrefix() { String prefix; - if (StringUtils.isNotBlank(config.getRestorePrefix())) - prefix = config.getRestorePrefix(); - else - prefix = config.getBackupPrefix(); + if (StringUtils.isNotBlank(config.getRestorePrefix())) prefix = config.getRestorePrefix(); + else prefix = config.getBackupPrefix(); return prefix; } @@ -245,4 +272,4 @@ static String parseObjectname(String pathPrefix) { int offset = pathPrefix.lastIndexOf(0x2f); return pathPrefix.substring(offset + 1); } -} \ No newline at end of file +} diff --git a/priam/src/main/java/com/netflix/priam/google/GoogleFileIterator.java b/priam/src/main/java/com/netflix/priam/google/GoogleFileIterator.java index 1d608c0c2..fe185cf59 100755 --- a/priam/src/main/java/com/netflix/priam/google/GoogleFileIterator.java +++ b/priam/src/main/java/com/netflix/priam/google/GoogleFileIterator.java @@ -1,16 +1,14 @@ /** * Copyright 2017 Netflix, Inc. - *

- * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and + * + *

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file + * except in compliance with the License. You may obtain a copy of the License at + * + *

http://www.apache.org/licenses/LICENSE-2.0 + * + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing permissions and * limitations under the License. */ package com.netflix.priam.google; @@ -21,13 +19,12 @@ import com.google.inject.Provider; import com.netflix.priam.aws.S3BackupPath; import com.netflix.priam.backup.AbstractBackupPath; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import java.io.IOException; import java.util.Date; import java.util.Iterator; import java.util.List; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /* * Represents a list of objects within Google Cloud Storage (GCS) @@ -56,7 +53,12 @@ public class GoogleFileIterator implements Iterator { * @param start - timeframe of object to restore * @param till - timeframe of object to restore */ - public GoogleFileIterator(Provider pathProvider, Storage gcsStorageHandle, String path, Date start, Date till) { + public GoogleFileIterator( + Provider pathProvider, + Storage gcsStorageHandle, + String path, + Date start, + Date till) { this.start = start; this.till = till; @@ -76,24 +78,28 @@ public GoogleFileIterator(Provider pathProvider, Storage gcs this.bucketName = paths[0]; this.pathWithinBucket = pathProvider.get().remotePrefix(start, till, path); - logger.info("Listing objects from GCS: {}, prefix: {}", this.bucketName, this.pathWithinBucket); + logger.info( + "Listing objects from GCS: {}, prefix: {}", this.bucketName, this.pathWithinBucket); try { - this.listObjectsSrvcHandle = objectsResoruceHandle.list(bucketName); //== list objects within bucket + this.listObjectsSrvcHandle = + objectsResoruceHandle.list(bucketName); // == list objects within bucket } catch (IOException e) { throw new RuntimeException("Unable to get gcslist handle to bucket: " + bucketName, e); } - this.listObjectsSrvcHandle.setPrefix(this.pathWithinBucket); //fetch elements within bucket that matches this prefix + // fetch elements within bucket that matches this prefix + this.listObjectsSrvcHandle.setPrefix(this.pathWithinBucket); - try { //== Get the initial page of results + try { // == Get the initial page of results this.iterator = createIterator(); } catch (Exception e) { - throw new RuntimeException("Exception encountered fetching elements, msg: ." + e.getLocalizedMessage(), e); + throw new RuntimeException( + "Exception encountered fetching elements, msg: ." + e.getLocalizedMessage(), e); } } @@ -101,18 +107,34 @@ public GoogleFileIterator(Provider pathProvider, Storage gcs * Fetch a page of results */ private Iterator createIterator() throws Exception { - List temp = Lists.newArrayList(); //a container of results + List temp = Lists.newArrayList(); // a container of results - this.objectsContainerHandle = listObjectsSrvcHandle.execute(); //Sends the metadata request to the server and returns the parsed metadata response. + // Sends the metadata request to the server and returns the parsed metadata response. + this.objectsContainerHandle = listObjectsSrvcHandle.execute(); - for (StorageObject object : this.objectsContainerHandle.getItems()) { //processing a page of results + for (StorageObject object : + this.objectsContainerHandle.getItems()) { // processing a page of results String fileName = GoogleEncryptedFileSystem.parseObjectname(object.getName()); - logger.debug("id: {}, parse file name: {}, name: {}", object.getId(), fileName, object.getName()); + logger.debug( + "id: {}, parse file name: {}, name: {}", + object.getId(), + fileName, + object.getName()); + + // e.g. of objectname: + // prod_backup/us-east-1/cass_account/113427455640312821154458202479064646083/201408250801/META/meta.json AbstractBackupPath path = pathProvider.get(); - path.parseRemote(object.getName()); //e.g. of objectname: prod_backup/us-east-1/cass_account/113427455640312821154458202479064646083/201408250801/META/meta.json - logger.debug("New key {} path = {} start: {} end: {} my {}", object.getName(), path.getRemotePath(), start, till, path.getTime()); - if ((path.getTime().after(start) && path.getTime().before(till)) || path.getTime().equals(start)) { + path.parseRemote(object.getName()); + logger.debug( + "New key {} path = {} start: {} end: {} my {}", + object.getName(), + path.getRemotePath(), + start, + till, + path.getTime()); + if ((path.getTime().after(start) && path.getTime().before(till)) + || path.getTime().equals(start)) { temp.add(path); logger.debug("Added key {}", object.getName()); } @@ -125,28 +147,30 @@ private Iterator createIterator() throws Exception { @Override public boolean hasNext() { - if (this.iterator == null) - return false; + if (this.iterator == null) return false; if (this.iterator.hasNext()) { return true; } - if (this.nextPageToken == null) { //there is no additional results + if (this.nextPageToken == null) { // there is no additional results return false; } - try {//if here, you have iterated through all elements of the previous page, now, get the next page of results + try { // if here, you have iterated through all elements of the previous page, now, get the + // next page of results - this.listObjectsSrvcHandle.setPageToken(this.nextPageToken); //get the next page of results + this.listObjectsSrvcHandle.setPageToken(this.nextPageToken); + // get the next page of results this.iterator = createIterator(); } catch (Exception e) { - throw new RuntimeException("Exception encountered fetching elements, see previous messages for details.", e); + throw new RuntimeException( + "Exception encountered fetching elements, see previous messages for details.", + e); } return this.iterator.hasNext(); - } @Override @@ -159,5 +183,4 @@ public void remove() { // TODO Auto-generated method stub } - -} \ No newline at end of file +} diff --git a/priam/src/main/java/com/netflix/priam/health/InstanceState.java b/priam/src/main/java/com/netflix/priam/health/InstanceState.java index 24a26bf19..96f702371 100644 --- a/priam/src/main/java/com/netflix/priam/health/InstanceState.java +++ b/priam/src/main/java/com/netflix/priam/health/InstanceState.java @@ -21,34 +21,36 @@ import com.netflix.priam.backup.BackupMetadata; import com.netflix.priam.backup.Status; import com.netflix.priam.utils.GsonJsonSerializer; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import java.time.LocalDateTime; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLong; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** - * Contains the state of the health of processed managed by Priam, and - * maintains the isHealthy flag used for reporting discovery health check. - *

- * Created by aagrawal on 9/19/17. + * Contains the state of the health of processed managed by Priam, and maintains the isHealthy flag + * used for reporting discovery health check. + * + *

Created by aagrawal on 9/19/17. */ @Singleton public class InstanceState { private static final Logger logger = LoggerFactory.getLogger(InstanceState.class); public enum NODE_STATE { - JOIN, //This state to be used when Priam is joining the ring for the first time or was already assigned this token. - REPLACE //This state to be used when Priam replaces an instance from the token range. + // This state to be used when Priam is joining the ring for the first time or was + // already assigned this token. + JOIN, + // This state to be used when Priam replaces an instance from the token range. + REPLACE } - //Bootstrap status + // Bootstrap status private final AtomicBoolean isBootstrapping = new AtomicBoolean(false); private NODE_STATE nodeState; private LocalDateTime bootstrapTime; - //Cassandra process status + // Cassandra process status private final AtomicBoolean isCassandraProcessAlive = new AtomicBoolean(false); private final AtomicBoolean shouldCassandraBeAlive = new AtomicBoolean(false); private final AtomicLong lastAttemptedStartTime = new AtomicLong(Long.MAX_VALUE); @@ -60,14 +62,14 @@ public enum NODE_STATE { private final AtomicBoolean isHealthy = new AtomicBoolean(false); private final AtomicBoolean isHealthyOverride = new AtomicBoolean(true); - //Backup status + // Backup status private BackupMetadata backupStatus; - //Restore status + // Restore status private final RestoreStatus restoreStatus; @Inject - InstanceState(RestoreStatus restoreStatus){ + InstanceState(RestoreStatus restoreStatus) { this.restoreStatus = restoreStatus; } @@ -185,8 +187,9 @@ public RestoreStatus getRestoreStatus() { return restoreStatus; } - // A dirty way to set restore status. This is required as setting restore status implies health could change. - public void setRestoreStatus(Status status){ + // A dirty way to set restore status. This is required as setting restore status implies health + // could change. + public void setRestoreStatus(Status status) { restoreStatus.status = status; setHealthy(); } @@ -195,20 +198,21 @@ public boolean isHealthy() { return isHealthy.get(); } - private boolean isRestoring(){ - return restoreStatus != null && restoreStatus.getStatus() != null && restoreStatus.getStatus() == Status.STARTED; + private boolean isRestoring() { + return restoreStatus != null + && restoreStatus.getStatus() != null + && restoreStatus.getStatus() == Status.STARTED; } + private void setHealthy() { this.isHealthy.set( - isRestoring() || - (isCassandraProcessAlive() && - isRequiredDirectoriesExist() && - isGossipActive() && - isYmlWritten() && - isHealthyOverride() && - (isThriftActive() || isNativeTransportActive()) - ) - ); + isRestoring() + || (isCassandraProcessAlive() + && isRequiredDirectoriesExist() + && isGossipActive() + && isYmlWritten() + && isHealthyOverride() + && (isThriftActive() || isNativeTransportActive()))); } public boolean isYmlWritten() { @@ -220,12 +224,14 @@ public void setYmlWritten(boolean yml) { } public static class RestoreStatus { - private LocalDateTime startDateRange, endDateRange; //Date range to restore from - private LocalDateTime executionStartTime, executionEndTime; //Start-end time of the actual restore execution - private String snapshotMetaFile; //Location of the snapshot meta file selected for restore. - private Status status; //the state of a restore. Note: this is different than the "status" of a Task. - - public void resetStatus(){ + private LocalDateTime startDateRange, endDateRange; // Date range to restore from + private LocalDateTime executionStartTime, + executionEndTime; // Start-end time of the actual restore execution + private String snapshotMetaFile; // Location of the snapshot meta file selected for restore. + // the state of a restore. Note: this is different than the "status" of a Task. + private Status status; + + public void resetStatus() { this.snapshotMetaFile = null; this.status = null; this.startDateRange = endDateRange = null; @@ -281,4 +287,4 @@ public void setSnapshotMetaFile(String snapshotMetaFile) { this.snapshotMetaFile = snapshotMetaFile; } } -} \ No newline at end of file +} diff --git a/priam/src/main/java/com/netflix/priam/identity/AwsInstanceEnvIdentity.java b/priam/src/main/java/com/netflix/priam/identity/AwsInstanceEnvIdentity.java index e0d12b3d3..4d47037c9 100644 --- a/priam/src/main/java/com/netflix/priam/identity/AwsInstanceEnvIdentity.java +++ b/priam/src/main/java/com/netflix/priam/identity/AwsInstanceEnvIdentity.java @@ -1,16 +1,14 @@ /** * Copyright 2017 Netflix, Inc. - *

- * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and + * + *

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file + * except in compliance with the License. You may obtain a copy of the License at + * + *

http://www.apache.org/licenses/LICENSE-2.0 + * + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing permissions and * limitations under the License. */ package com.netflix.priam.identity; @@ -32,7 +30,8 @@ public AwsInstanceEnvIdentity() { if (vpcId == null || vpcId.isEmpty()) { this.isClassic = true; } else { - this.isNonDefaultVpc = true; //our instances run under a non default ("persistence_*") AWS acct + this.isNonDefaultVpc = + true; // our instances run under a non default ("persistence_*") AWS acct } } @@ -58,5 +57,4 @@ public Boolean isDefaultVpc() { public Boolean isNonDefaultVpc() { return this.isNonDefaultVpc; } - } diff --git a/priam/src/main/java/com/netflix/priam/identity/DoubleRing.java b/priam/src/main/java/com/netflix/priam/identity/DoubleRing.java index 2de127c5e..5319d9a86 100644 --- a/priam/src/main/java/com/netflix/priam/identity/DoubleRing.java +++ b/priam/src/main/java/com/netflix/priam/identity/DoubleRing.java @@ -20,15 +20,12 @@ import com.google.inject.Inject; import com.netflix.priam.config.IConfiguration; import com.netflix.priam.utils.ITokenManager; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import java.io.*; import java.util.List; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; -/** - * Class providing functionality for doubling the ring - */ +/** Class providing functionality for doubling the ring */ public class DoubleRing { private static final Logger logger = LoggerFactory.getLogger(DoubleRing.class); private static File TMP_BACKUP_FILE; @@ -37,59 +34,75 @@ public class DoubleRing { private final ITokenManager tokenManager; @Inject - public DoubleRing(IConfiguration config, IPriamInstanceFactory factory, ITokenManager tokenManager) { + public DoubleRing( + IConfiguration config, IPriamInstanceFactory factory, ITokenManager tokenManager) { this.config = config; this.factory = factory; this.tokenManager = tokenManager; } /** - * Doubling is done by pre-calculating all slots of a double ring and - * registering them. When new nodes come up, they will get the unsed token - * assigned per token logic. + * Doubling is done by pre-calculating all slots of a double ring and registering them. When new + * nodes come up, they will get the unsed token assigned per token logic. */ public void doubleSlots() { List local = filteredRemote(factory.getAllIds(config.getAppName())); // delete all - for (PriamInstance data : local) - factory.delete(data); + for (PriamInstance data : local) factory.delete(data); int hash = tokenManager.regionOffset(config.getDC()); // move existing slots. for (PriamInstance data : local) { int slot = (data.getId() - hash) * 2; - factory.create(data.getApp(), hash + slot, data.getInstanceId(), data.getHostName(), data.getHostIP(), data.getRac(), data.getVolumes(), data.getToken()); + factory.create( + data.getApp(), + hash + slot, + data.getInstanceId(), + data.getHostName(), + data.getHostIP(), + data.getRac(), + data.getVolumes(), + data.getToken()); } int new_ring_size = local.size() * 2; for (PriamInstance data : filteredRemote(factory.getAllIds(config.getAppName()))) { // if max then rotate. int currentSlot = data.getId() - hash; - int new_slot = currentSlot + 3 > new_ring_size ? (currentSlot + 3) - new_ring_size : currentSlot + 3; + int new_slot = + currentSlot + 3 > new_ring_size + ? (currentSlot + 3) - new_ring_size + : currentSlot + 3; String token = tokenManager.createToken(new_slot, new_ring_size, config.getDC()); - factory.create(data.getApp(), new_slot + hash, InstanceIdentity.DUMMY_INSTANCE_ID, config.getHostname(), config.getHostIP(), data.getRac(), null, token); + factory.create( + data.getApp(), + new_slot + hash, + InstanceIdentity.DUMMY_INSTANCE_ID, + config.getHostname(), + config.getHostIP(), + data.getRac(), + null, + token); } } // filter other DC's private List filteredRemote(List lst) { List local = Lists.newArrayList(); - for (PriamInstance data : lst) - if (data.getDC().equals(config.getDC())) - local.add(data); + for (PriamInstance data : lst) if (data.getDC().equals(config.getDC())) local.add(data); return local; } - /** - * Backup the current state in case of failure - */ + /** Backup the current state in case of failure */ public void backup() throws IOException { // writing to the backup file. TMP_BACKUP_FILE = File.createTempFile("Backup-instance-data", ".dat"); - try (ObjectOutputStream stream = new ObjectOutputStream(new FileOutputStream(TMP_BACKUP_FILE))) { + try (ObjectOutputStream stream = + new ObjectOutputStream(new FileOutputStream(TMP_BACKUP_FILE))) { stream.writeObject(filteredRemote(factory.getAllIds(config.getAppName()))); - logger.info("Wrote the backup of the instances to: {}", TMP_BACKUP_FILE.getAbsolutePath()); + logger.info( + "Wrote the backup of the instances to: {}", TMP_BACKUP_FILE.getAbsolutePath()); } } @@ -104,13 +117,23 @@ public void restore() throws IOException, ClassNotFoundException { factory.delete(data); // read from the file. - try (ObjectInputStream stream = new ObjectInputStream(new FileInputStream(TMP_BACKUP_FILE))) { + try (ObjectInputStream stream = + new ObjectInputStream(new FileInputStream(TMP_BACKUP_FILE))) { @SuppressWarnings("unchecked") List allInstances = (List) stream.readObject(); for (PriamInstance data : allInstances) - factory.create(data.getApp(), data.getId(), data.getInstanceId(), data.getHostName(), data.getHostIP(), data.getRac(), data.getVolumes(), data.getToken()); - logger.info("Successfully restored the Instances from the backup: {}", TMP_BACKUP_FILE.getAbsolutePath()); + factory.create( + data.getApp(), + data.getId(), + data.getInstanceId(), + data.getHostName(), + data.getHostIP(), + data.getRac(), + data.getVolumes(), + data.getToken()); + logger.info( + "Successfully restored the Instances from the backup: {}", + TMP_BACKUP_FILE.getAbsolutePath()); } } - } diff --git a/priam/src/main/java/com/netflix/priam/identity/IMembership.java b/priam/src/main/java/com/netflix/priam/identity/IMembership.java index 6cbac34be..03e30ed6a 100644 --- a/priam/src/main/java/com/netflix/priam/identity/IMembership.java +++ b/priam/src/main/java/com/netflix/priam/identity/IMembership.java @@ -18,13 +18,12 @@ import com.google.inject.ImplementedBy; import com.netflix.priam.aws.AWSMembership; - import java.util.Collection; import java.util.List; /** - * Interface to manage membership meta information such as size of RAC, list of - * nodes in RAC etc. Also perform ACL updates used in multi-regional clusters + * Interface to manage membership meta information such as size of RAC, list of nodes in RAC etc. + * Also perform ACL updates used in multi-regional clusters */ @ImplementedBy(AWSMembership.class) public interface IMembership { @@ -35,9 +34,7 @@ public interface IMembership { */ List getRacMembership(); - /** - * @return Size of current RAC - */ + /** @return Size of current RAC */ int getRacMembershipSize(); /** @@ -85,4 +82,4 @@ public interface IMembership { * @param count */ void expandRacMembership(int count); -} \ No newline at end of file +} diff --git a/priam/src/main/java/com/netflix/priam/identity/IPriamInstanceFactory.java b/priam/src/main/java/com/netflix/priam/identity/IPriamInstanceFactory.java index fc11a603c..79f51f78e 100644 --- a/priam/src/main/java/com/netflix/priam/identity/IPriamInstanceFactory.java +++ b/priam/src/main/java/com/netflix/priam/identity/IPriamInstanceFactory.java @@ -18,18 +18,18 @@ import com.google.inject.ImplementedBy; import com.netflix.priam.aws.SDBInstanceFactory; - import java.util.List; import java.util.Map; /** - * Interface for managing Cassandra instance data. Provides functionality - * to register, update, delete or list instances from the registry + * Interface for managing Cassandra instance data. Provides functionality to register, update, + * delete or list instances from the registry */ @ImplementedBy(SDBInstanceFactory.class) public interface IPriamInstanceFactory { /** * Return a list of all Cassandra server nodes registered. + * * @param appName the cluster name * @return a list of all nodes in {@code appName} */ @@ -37,6 +37,7 @@ public interface IPriamInstanceFactory { /** * Return the Cassandra server node with the given {@code id}. + * * @param appName the cluster name * @param id the node id * @return the node with the given {@code id}, or {@code null} if none found @@ -45,6 +46,7 @@ public interface IPriamInstanceFactory { /** * Create/Register an instance of the server with its info. + * * @param app * @param id * @param instanceID @@ -55,31 +57,43 @@ public interface IPriamInstanceFactory { * @param token * @return the new node */ - PriamInstance create(String app, int id, String instanceID, String hostname, String ip, String rac, Map volumes, String token); + PriamInstance create( + String app, + int id, + String instanceID, + String hostname, + String ip, + String rac, + Map volumes, + String token); /** * Delete the server node from the registry + * * @param inst the node to delete */ void delete(PriamInstance inst); /** * Update the details of the server node in registry + * * @param inst the node to update */ void update(PriamInstance inst); /** * Sort the list by instance ID + * * @param return_ the list of nodes to sort */ void sort(List return_); /** * Attach volumes if required + * * @param instance * @param mountPath * @param device */ void attachVolumes(PriamInstance instance, String mountPath, String device); -} \ No newline at end of file +} diff --git a/priam/src/main/java/com/netflix/priam/identity/InstanceEnvIdentity.java b/priam/src/main/java/com/netflix/priam/identity/InstanceEnvIdentity.java index e0441fe14..2c583c990 100644 --- a/priam/src/main/java/com/netflix/priam/identity/InstanceEnvIdentity.java +++ b/priam/src/main/java/com/netflix/priam/identity/InstanceEnvIdentity.java @@ -1,16 +1,14 @@ /** * Copyright 2017 Netflix, Inc. - *

- * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and + * + *

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file + * except in compliance with the License. You may obtain a copy of the License at + * + *

http://www.apache.org/licenses/LICENSE-2.0 + * + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing permissions and * limitations under the License. */ package com.netflix.priam.identity; @@ -38,7 +36,8 @@ public interface InstanceEnvIdentity { Boolean isNonDefaultVpc(); enum InstanceEnvironent { - CLASSIC, DEFAULT_VPC, NONDEFAULT_VPC + CLASSIC, + DEFAULT_VPC, + NONDEFAULT_VPC } - } diff --git a/priam/src/main/java/com/netflix/priam/identity/InstanceIdentity.java b/priam/src/main/java/com/netflix/priam/identity/InstanceIdentity.java index 1ec9e67ac..056d1bb69 100644 --- a/priam/src/main/java/com/netflix/priam/identity/InstanceIdentity.java +++ b/priam/src/main/java/com/netflix/priam/identity/InstanceIdentity.java @@ -30,53 +30,61 @@ import com.netflix.priam.utils.ITokenManager; import com.netflix.priam.utils.RetryableCallable; import com.netflix.priam.utils.Sleeper; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import java.net.UnknownHostException; import java.util.Collection; import java.util.HashMap; import java.util.LinkedList; import java.util.List; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** - * This class provides the central place to create and consume the identity of - * the instance - token, seeds etc. - * + * This class provides the central place to create and consume the identity of the instance - token, + * seeds etc. */ @Singleton public class InstanceIdentity { private static final Logger logger = LoggerFactory.getLogger(InstanceIdentity.class); public static final String DUMMY_INSTANCE_ID = "new_slot"; - private final ListMultimap locMap = Multimaps.newListMultimap(new HashMap>(), Lists::newArrayList); + private final ListMultimap locMap = + Multimaps.newListMultimap( + new HashMap>(), Lists::newArrayList); private final IPriamInstanceFactory factory; private final IMembership membership; private final IConfiguration config; private final Sleeper sleeper; private final ITokenManager tokenManager; - private final Predicate differentHostPredicate = new Predicate() { - @Override - public boolean apply(PriamInstance instance) { - if (config.getAutoBoostrap()) { - // auto_bootstrap = true indicates that the cluster is up and running normally, in such a case - // we cannot provide the local instance as a seed otherwise we can bootstrap nodes with no data - return (!instance.getInstanceId().equalsIgnoreCase(DUMMY_INSTANCE_ID) && !instance.getHostName().equals(myInstance.getHostName())); - } else { - // auto_bootstrap = false indicates a freshly provisioned cluster. Some nodes in such a cluster must - // provide itself as a seed due to the changes in CASSANDRA-10134 which made it so the cluster would - // not start up when auto_bootstrap was false. This is because in 3.11 failing the shadow round - // (which will happen on bootup by definition) is acceptable for seeds, but not for non seeds - return (!instance.getInstanceId().equalsIgnoreCase(DUMMY_INSTANCE_ID)); - } - } + private final Predicate differentHostPredicate = + new Predicate() { + @Override + public boolean apply(PriamInstance instance) { + if (config.getAutoBoostrap()) { + // auto_bootstrap = true indicates that the cluster is up and running + // normally, in such a case + // we cannot provide the local instance as a seed otherwise we can bootstrap + // nodes with no data + return (!instance.getInstanceId().equalsIgnoreCase(DUMMY_INSTANCE_ID) + && !instance.getHostName().equals(myInstance.getHostName())); + } else { + // auto_bootstrap = false indicates a freshly provisioned cluster. Some + // nodes in such a cluster must + // provide itself as a seed due to the changes in CASSANDRA-10134 which made + // it so the cluster would + // not start up when auto_bootstrap was false. This is because in 3.11 + // failing the shadow round + // (which will happen on bootup by definition) is acceptable for seeds, but + // not for non seeds + return (!instance.getInstanceId().equalsIgnoreCase(DUMMY_INSTANCE_ID)); + } + } - @Override - public boolean test(PriamInstance input) { - return apply(input); - } - }; + @Override + public boolean test(PriamInstance input) { + return apply(input); + } + }; private PriamInstance myInstance; private boolean isReplace = false; @@ -87,13 +95,18 @@ public boolean test(PriamInstance input) { private final INewTokenRetriever newTokenRetriever; @Inject - //Note: do not parameterized the generic type variable to an implementation as it confuses Guice in the binding. - public InstanceIdentity(IPriamInstanceFactory factory, IMembership membership, IConfiguration config, - Sleeper sleeper, ITokenManager tokenManager - , IDeadTokenRetriever deadTokenRetriever - , IPreGeneratedTokenRetriever preGeneratedTokenRetriever - , INewTokenRetriever newTokenRetriever - ) throws Exception { + // Note: do not parameterized the generic type variable to an implementation as it confuses + // Guice in the binding. + public InstanceIdentity( + IPriamInstanceFactory factory, + IMembership membership, + IConfiguration config, + Sleeper sleeper, + ITokenManager tokenManager, + IDeadTokenRetriever deadTokenRetriever, + IPreGeneratedTokenRetriever preGeneratedTokenRetriever, + INewTokenRetriever newTokenRetriever) + throws Exception { this.factory = factory; this.membership = membership; this.config = config; @@ -111,135 +124,150 @@ public PriamInstance getInstance() { public void init() throws Exception { // try to grab the token which was already assigned - myInstance = new RetryableCallable() { - @Override - public PriamInstance retriableCall() throws Exception { - // Check if this node is decomissioned - List deadInstances = factory.getAllIds(config.getAppName() + "-dead"); - for (PriamInstance ins : deadInstances) { - logger.info("[Dead] Iterating though the hosts: {}", ins.getInstanceId()); - if (ins.getInstanceId().equals(config.getInstanceName())) { - ins.setOutOfService(true); - logger.info("[Dead] found that this node is dead." - + " application: {}" - + ", id: {}" - + ", instance: {}" - + ", region: {}" - + ", host ip: {}" - + ", host name: {}" - + ", token: {}", - ins.getApp(), ins.getId(), ins.getInstanceId(), - ins.getDC(), ins.getHostIP(), ins.getHostName(), - ins.getToken()); - return ins; - } - } - List aliveInstances = factory.getAllIds(config.getAppName()); - for (PriamInstance ins : aliveInstances) { - logger.info("[Alive] Iterating though the hosts: {} My id = [{}]", ins.getInstanceId(), ins.getId()); - if (ins.getInstanceId().equals(config.getInstanceName())) { - logger.info("[Alive] found that this node is alive." - + " application: {}" - + ", id: {}" - + ", instance: {}" - + ", region: {}" - + ", host ip: {}" - + ", host name: {}" - + ", token: {}", - ins.getApp(), ins.getId(), ins.getInstanceId(), - ins.getDC(), ins.getHostIP(), ins.getHostName(), - ins.getToken()); - return ins; + myInstance = + new RetryableCallable() { + @Override + public PriamInstance retriableCall() throws Exception { + // Check if this node is decomissioned + List deadInstances = + factory.getAllIds(config.getAppName() + "-dead"); + for (PriamInstance ins : deadInstances) { + logger.info( + "[Dead] Iterating though the hosts: {}", ins.getInstanceId()); + if (ins.getInstanceId().equals(config.getInstanceName())) { + ins.setOutOfService(true); + logger.info( + "[Dead] found that this node is dead." + + " application: {}" + + ", id: {}" + + ", instance: {}" + + ", region: {}" + + ", host ip: {}" + + ", host name: {}" + + ", token: {}", + ins.getApp(), + ins.getId(), + ins.getInstanceId(), + ins.getDC(), + ins.getHostIP(), + ins.getHostName(), + ins.getToken()); + return ins; + } + } + List aliveInstances = factory.getAllIds(config.getAppName()); + for (PriamInstance ins : aliveInstances) { + logger.info( + "[Alive] Iterating though the hosts: {} My id = [{}]", + ins.getInstanceId(), + ins.getId()); + if (ins.getInstanceId().equals(config.getInstanceName())) { + logger.info( + "[Alive] found that this node is alive." + + " application: {}" + + ", id: {}" + + ", instance: {}" + + ", region: {}" + + ", host ip: {}" + + ", host name: {}" + + ", token: {}", + ins.getApp(), + ins.getId(), + ins.getInstanceId(), + ins.getDC(), + ins.getHostIP(), + ins.getHostName(), + ins.getToken()); + return ins; + } + } + return null; } - - } - return null; - } - }.call(); + }.call(); // Grab a dead token if (null == myInstance) { - myInstance = new RetryableCallable() { - - @Override - public PriamInstance retriableCall() throws Exception { - PriamInstance result; - result = deadTokenRetriever.get(); - if (result != null) { - - isReplace = true; //indicate that we are acquiring a dead instance's token - - if (deadTokenRetriever.getReplaceIp() != null) { //The IP address of the dead instance to which we will acquire its token - replacedIp = deadTokenRetriever.getReplaceIp(); + myInstance = + new RetryableCallable() { + + @Override + public PriamInstance retriableCall() throws Exception { + PriamInstance result; + result = deadTokenRetriever.get(); + if (result != null) { + + isReplace = + true; // indicate that we are acquiring a dead instance's + // token + + if (deadTokenRetriever.getReplaceIp() + != null) { // The IP address of the dead instance to which + // we will acquire its token + replacedIp = deadTokenRetriever.getReplaceIp(); + } + } + + return result; } - } - - return result; - } - - @Override - public void forEachExecution() { - populateRacMap(); - deadTokenRetriever.setLocMap(locMap); - } - - }.call(); + @Override + public void forEachExecution() { + populateRacMap(); + deadTokenRetriever.setLocMap(locMap); + } + }.call(); } - // Grab a pre-generated token if there is such one if (null == myInstance) { - myInstance = new RetryableCallable() { - - @Override - public PriamInstance retriableCall() throws Exception { - PriamInstance result; - result = preGeneratedTokenRetriever.get(); - if (result != null) { - isTokenPregenerated = true; - } - return result; - } - - @Override - public void forEachExecution() { - populateRacMap(); - preGeneratedTokenRetriever.setLocMap(locMap); - } - - }.call(); + myInstance = + new RetryableCallable() { + + @Override + public PriamInstance retriableCall() throws Exception { + PriamInstance result; + result = preGeneratedTokenRetriever.get(); + if (result != null) { + isTokenPregenerated = true; + } + return result; + } + @Override + public void forEachExecution() { + populateRacMap(); + preGeneratedTokenRetriever.setLocMap(locMap); + } + }.call(); } - // Grab a new token if (null == myInstance) { if (this.config.isCreateNewTokenEnable()) { - myInstance = new RetryableCallable() { - - @Override - public PriamInstance retriableCall() throws Exception { - super.set(100, 100); - newTokenRetriever.setLocMap(locMap); - return newTokenRetriever.get(); - } + myInstance = + new RetryableCallable() { - @Override - public void forEachExecution() { - populateRacMap(); - newTokenRetriever.setLocMap(locMap); - } + @Override + public PriamInstance retriableCall() throws Exception { + super.set(100, 100); + newTokenRetriever.setLocMap(locMap); + return newTokenRetriever.get(); + } - }.call(); + @Override + public void forEachExecution() { + populateRacMap(); + newTokenRetriever.setLocMap(locMap); + } + }.call(); } else { - throw new IllegalStateException("Node attempted to erroneously create a new token when we should be grabbing an existing token."); + throw new IllegalStateException( + "Node attempted to erroneously create a new token when we should be grabbing an existing token."); } - } logger.info("My token: {}", myInstance.getToken()); @@ -262,23 +290,24 @@ public List getSeeds() throws UnknownHostException { if (membership.getRacMembershipSize() != locMap.get(myInstance.getRac()).size()) return seeds; // If seed node, return the next node in the list - if (locMap.get(myInstance.getRac()).size() > 1 && locMap.get(myInstance.getRac()).get(0).getHostIP().equals(myInstance.getHostIP())) { + if (locMap.get(myInstance.getRac()).size() > 1 + && locMap.get(myInstance.getRac()) + .get(0) + .getHostIP() + .equals(myInstance.getHostIP())) { PriamInstance instance = locMap.get(myInstance.getRac()).get(1); if (instance != null && !isInstanceDummy(instance)) { - if (config.isMultiDC()) - seeds.add(instance.getHostIP()); - else - seeds.add(instance.getHostName()); + if (config.isMultiDC()) seeds.add(instance.getHostIP()); + else seeds.add(instance.getHostName()); } } } for (String loc : locMap.keySet()) { - PriamInstance instance = Iterables.tryFind(locMap.get(loc), differentHostPredicate).orNull(); + PriamInstance instance = + Iterables.tryFind(locMap.get(loc), differentHostPredicate).orNull(); if (instance != null && !isInstanceDummy(instance)) { - if (config.isMultiDC()) - seeds.add(instance.getHostIP()); - else - seeds.add(instance.getHostName()); + if (config.isMultiDC()) seeds.add(instance.getHostIP()); + else seeds.add(instance.getHostName()); } } return seeds; @@ -304,11 +333,10 @@ public String getReplacedIp() { public void setReplacedIp(String replacedIp) { this.replacedIp = replacedIp; - if (!replacedIp.isEmpty()) - this.isReplace = true; + if (!replacedIp.isEmpty()) this.isReplace = true; } private static boolean isInstanceDummy(PriamInstance instance) { return instance.getInstanceId().equals(DUMMY_INSTANCE_ID); } -} \ No newline at end of file +} diff --git a/priam/src/main/java/com/netflix/priam/identity/PriamInstance.java b/priam/src/main/java/com/netflix/priam/identity/PriamInstance.java index 90d8d2287..19d1e50ea 100644 --- a/priam/src/main/java/com/netflix/priam/identity/PriamInstance.java +++ b/priam/src/main/java/com/netflix/priam/identity/PriamInstance.java @@ -16,11 +16,10 @@ */ package com.netflix.priam.identity; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import java.io.Serializable; import java.util.Map; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class PriamInstance implements Serializable { private static final long serialVersionUID = 5606412386974488659L; @@ -36,7 +35,7 @@ public class PriamInstance implements Serializable { private String publicip; private String location; private String token; - //Handles Storage objects + // Handles Storage objects private Map volumes; public String getApp() { @@ -110,8 +109,9 @@ public void setVolumes(Map volumes) { @Override public String toString() { - return String.format("Hostname: %s, InstanceId: %s, APP_NAME: %s, RAC : %s Location %s, Id: %s: Token: %s", getHostName(), getInstanceId(), getApp(), getRac(), getDC(), getId(), - getToken()); + return String.format( + "Hostname: %s, InstanceId: %s, APP_NAME: %s, RAC : %s Location %s, Id: %s: Token: %s", + getHostName(), getInstanceId(), getApp(), getRac(), getDC(), getId(), getToken()); } public String getDC() { @@ -137,6 +137,4 @@ public boolean isOutOfService() { public void setOutOfService(boolean outOfService) { this.outOfService = outOfService; } - - -} \ No newline at end of file +} diff --git a/priam/src/main/java/com/netflix/priam/identity/config/AWSVpcInstanceDataRetriever.java b/priam/src/main/java/com/netflix/priam/identity/config/AWSVpcInstanceDataRetriever.java index f4255b217..7e0564b4b 100644 --- a/priam/src/main/java/com/netflix/priam/identity/config/AWSVpcInstanceDataRetriever.java +++ b/priam/src/main/java/com/netflix/priam/identity/config/AWSVpcInstanceDataRetriever.java @@ -1,16 +1,14 @@ /** * Copyright 2017 Netflix, Inc. - *

- * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and + * + *

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file + * except in compliance with the License. You may obtain a copy of the License at + * + *

http://www.apache.org/licenses/LICENSE-2.0 + * + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing permissions and * limitations under the License. */ package com.netflix.priam.identity.config; @@ -21,24 +19,29 @@ /** * Calls AWS metadata to get info on the location of the running instance within vpc environment. - * */ -public class AWSVpcInstanceDataRetriever extends InstanceDataRetrieverBase{ +public class AWSVpcInstanceDataRetriever extends InstanceDataRetrieverBase { private static final Logger logger = LoggerFactory.getLogger(AWSVpcInstanceDataRetriever.class); + @Override public String getVpcId() { String nacId = getMac(); - if (nacId == null || nacId.isEmpty()) - return null; + if (nacId == null || nacId.isEmpty()) return null; String vpcId = null; try { - vpcId = SystemUtils.getDataFromUrl("http://169.254.169.254/latest/meta-data/network/interfaces/macs/" + nacId + "vpc-id").trim(); + vpcId = + SystemUtils.getDataFromUrl( + "http://169.254.169.254/latest/meta-data/network/interfaces/macs/" + + nacId + + "vpc-id") + .trim(); } catch (Exception e) { - logger.info("Vpc id does not exist for running instance, not fatal as running instance maybe not be in vpc. Msg: {}", e.getLocalizedMessage()); + logger.info( + "Vpc id does not exist for running instance, not fatal as running instance maybe not be in vpc. Msg: {}", + e.getLocalizedMessage()); } return vpcId; } - -} \ No newline at end of file +} diff --git a/priam/src/main/java/com/netflix/priam/identity/config/AwsClassicInstanceDataRetriever.java b/priam/src/main/java/com/netflix/priam/identity/config/AwsClassicInstanceDataRetriever.java index e975c2830..27b986772 100644 --- a/priam/src/main/java/com/netflix/priam/identity/config/AwsClassicInstanceDataRetriever.java +++ b/priam/src/main/java/com/netflix/priam/identity/config/AwsClassicInstanceDataRetriever.java @@ -1,29 +1,26 @@ /** * Copyright 2017 Netflix, Inc. - *

- * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and + * + *

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file + * except in compliance with the License. You may obtain a copy of the License at + * + *

http://www.apache.org/licenses/LICENSE-2.0 + * + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing permissions and * limitations under the License. */ package com.netflix.priam.identity.config; /** - * Calls AWS metadata to get info on the location of the running instance within classic environment. - * + * Calls AWS metadata to get info on the location of the running instance within classic + * environment. */ - public class AwsClassicInstanceDataRetriever extends InstanceDataRetrieverBase { @Override public String getVpcId() { - throw new UnsupportedOperationException("Not applicable as running instance is in classic environment"); + throw new UnsupportedOperationException( + "Not applicable as running instance is in classic environment"); } - -} \ No newline at end of file +} diff --git a/priam/src/main/java/com/netflix/priam/identity/config/InstanceDataRetriever.java b/priam/src/main/java/com/netflix/priam/identity/config/InstanceDataRetriever.java index 6f2f68204..0f402492f 100644 --- a/priam/src/main/java/com/netflix/priam/identity/config/InstanceDataRetriever.java +++ b/priam/src/main/java/com/netflix/priam/identity/config/InstanceDataRetriever.java @@ -1,76 +1,82 @@ /** * Copyright 2017 Netflix, Inc. - *

- * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and + * + *

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file + * except in compliance with the License. You may obtain a copy of the License at + * + *

http://www.apache.org/licenses/LICENSE-2.0 + * + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing permissions and * limitations under the License. */ package com.netflix.priam.identity.config; import org.codehaus.jettison.json.JSONException; -/** - * A means to fetch meta data of running instance - */ +/** A means to fetch meta data of running instance */ public interface InstanceDataRetriever { /** * Get the availability zone of the running instance. + * * @return the availability zone of the running instance. e.g. us-east-1c */ String getRac(); /** * Get the public hostname for the running instance. - * @return the public hostname for the running instance. e.g.: ec2-12-34-56-78.compute-1.amazonaws.com + * + * @return the public hostname for the running instance. e.g.: + * ec2-12-34-56-78.compute-1.amazonaws.com */ String getPublicHostname(); /** * Get public ip address for running instance. Can be null. + * * @return private ip address for running instance. */ String getPublicIP(); /** * Get private ip address for running instance. + * * @return private ip address for running instance. */ String getPrivateIP(); /** * Get the instance id of the running instance. + * * @return the instance id of the running instance. e.g. i-07a88a49ff155353 */ String getInstanceId(); /** * Get the instance type of the running instance. + * * @return the instance type of the running instance. e.g. i3.2xlarge */ String getInstanceType(); /** * Get the id of the network interface for running instance. + * * @return id of the network interface for running instance - */ + */ String getMac(); /** * Get the id of the vpc account for running instance. + * * @return the id of the vpc account for running instance, null if does not exist. */ - String getVpcId(); //the id of the vpc for running instance + String getVpcId(); // the id of the vpc for running instance /** * AWS Account ID of running instance. + * * @return the id (e.g. 12345) of the AWS account of running instance, could be null /empty. * @throws JSONException */ @@ -78,13 +84,16 @@ public interface InstanceDataRetriever { /** * Get the region of the AWS account of running instance - * @return the region (e.g. us-east-1) of the AWS account of running instance, could be null /empty. + * + * @return the region (e.g. us-east-1) of the AWS account of running instance, could be null + * /empty. * @throws JSONException */ String getRegion() throws JSONException; /** * Get the availability zone of the running instance. + * * @return the availability zone of the running instance. e.g. us-east-1c * @throws JSONException */ diff --git a/priam/src/main/java/com/netflix/priam/identity/config/InstanceDataRetrieverBase.java b/priam/src/main/java/com/netflix/priam/identity/config/InstanceDataRetrieverBase.java index 2b9598a40..bb32a3b67 100644 --- a/priam/src/main/java/com/netflix/priam/identity/config/InstanceDataRetrieverBase.java +++ b/priam/src/main/java/com/netflix/priam/identity/config/InstanceDataRetrieverBase.java @@ -1,16 +1,14 @@ /** * Copyright 2017 Netflix, Inc. - *

- * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and + * + *

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file + * except in compliance with the License. You may obtain a copy of the License at + * + *

http://www.apache.org/licenses/LICENSE-2.0 + * + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing permissions and * limitations under the License. */ package com.netflix.priam.identity.config; @@ -19,19 +17,21 @@ import org.codehaus.jettison.json.JSONException; import org.codehaus.jettison.json.JSONObject; -public abstract class InstanceDataRetrieverBase implements InstanceDataRetriever{ +public abstract class InstanceDataRetrieverBase implements InstanceDataRetriever { protected JSONObject identityDocument = null; - public String getPrivateIP(){ + public String getPrivateIP() { return SystemUtils.getDataFromUrl("http://169.254.169.254/latest/meta-data/local-ipv4"); } public String getRac() { - return SystemUtils.getDataFromUrl("http://169.254.169.254/latest/meta-data/placement/availability-zone"); + return SystemUtils.getDataFromUrl( + "http://169.254.169.254/latest/meta-data/placement/availability-zone"); } public String getPublicHostname() { - return SystemUtils.getDataFromUrl("http://169.254.169.254/latest/meta-data/public-hostname"); + return SystemUtils.getDataFromUrl( + "http://169.254.169.254/latest/meta-data/public-hostname"); } public String getPublicIP() { @@ -47,12 +47,16 @@ public String getInstanceType() { } public String getMac() { - return SystemUtils.getDataFromUrl("http://169.254.169.254/latest/meta-data/network/interfaces/macs/").trim(); + return SystemUtils.getDataFromUrl( + "http://169.254.169.254/latest/meta-data/network/interfaces/macs/") + .trim(); } public String getAWSAccountId() throws JSONException { if (this.identityDocument == null) { - String jsonStr = SystemUtils.getDataFromUrl("http://169.254.169.254/latest/dynamic/instance-identity/document"); + String jsonStr = + SystemUtils.getDataFromUrl( + "http://169.254.169.254/latest/dynamic/instance-identity/document"); this.identityDocument = new JSONObject(jsonStr); } return this.identityDocument.getString("accountId"); @@ -60,13 +64,16 @@ public String getAWSAccountId() throws JSONException { public String getRegion() throws JSONException { if (this.identityDocument == null) { - String jsonStr = SystemUtils.getDataFromUrl("http://169.254.169.254/latest/dynamic/instance-identity/document"); + String jsonStr = + SystemUtils.getDataFromUrl( + "http://169.254.169.254/latest/dynamic/instance-identity/document"); this.identityDocument = new JSONObject(jsonStr); } return this.identityDocument.getString("region"); } public String getAvailabilityZone() throws JSONException { - return SystemUtils.getDataFromUrl("http://169.254.169.254/latest/meta-data/placement/availability-zone"); + return SystemUtils.getDataFromUrl( + "http://169.254.169.254/latest/meta-data/placement/availability-zone"); } } diff --git a/priam/src/main/java/com/netflix/priam/identity/config/LocalInstanceDataRetriever.java b/priam/src/main/java/com/netflix/priam/identity/config/LocalInstanceDataRetriever.java index 028e655a7..f12d19ff0 100644 --- a/priam/src/main/java/com/netflix/priam/identity/config/LocalInstanceDataRetriever.java +++ b/priam/src/main/java/com/netflix/priam/identity/config/LocalInstanceDataRetriever.java @@ -1,16 +1,14 @@ /** * Copyright 2017 Netflix, Inc. - *

- * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and + * + *

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file + * except in compliance with the License. You may obtain a copy of the License at + * + *

http://www.apache.org/licenses/LICENSE-2.0 + * + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing permissions and * limitations under the License. */ package com.netflix.priam.identity.config; @@ -18,8 +16,8 @@ import org.codehaus.jettison.json.JSONException; /** - * Looks at local (system) properties for metadata about the running 'instance'. - * Typically, this is used for locally-deployed testing. + * Looks at local (system) properties for metadata about the running 'instance'. Typically, this is + * used for locally-deployed testing. */ public class LocalInstanceDataRetriever implements InstanceDataRetriever { private static final String PREFIX = "Priam.localInstance."; @@ -73,4 +71,4 @@ public String getAvailabilityZone() throws JSONException { public String getRegion() throws JSONException { return System.getProperty(PREFIX + "region", ""); } -} \ No newline at end of file +} diff --git a/priam/src/main/java/com/netflix/priam/identity/token/DeadTokenRetriever.java b/priam/src/main/java/com/netflix/priam/identity/token/DeadTokenRetriever.java index cac643d81..8e59975b2 100755 --- a/priam/src/main/java/com/netflix/priam/identity/token/DeadTokenRetriever.java +++ b/priam/src/main/java/com/netflix/priam/identity/token/DeadTokenRetriever.java @@ -1,16 +1,14 @@ /** * Copyright 2017 Netflix, Inc. - *

- * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and + * + *

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file + * except in compliance with the License. You may obtain a copy of the License at + * + *

http://www.apache.org/licenses/LICENSE-2.0 + * + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing permissions and * limitations under the License. */ package com.netflix.priam.identity.token; @@ -28,6 +26,10 @@ import com.sun.jersey.api.client.WebResource; import com.sun.jersey.api.client.config.ClientConfig; import com.sun.jersey.api.client.config.DefaultClientConfig; +import java.util.Arrays; +import java.util.List; +import java.util.Random; +import javax.ws.rs.core.MediaType; import org.apache.commons.lang3.StringUtils; import org.json.simple.JSONObject; import org.json.simple.parser.JSONParser; @@ -35,25 +37,24 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import javax.ws.rs.core.MediaType; -import java.util.Arrays; -import java.util.List; -import java.util.Random; - public class DeadTokenRetriever extends TokenRetrieverBase implements IDeadTokenRetriever { private static final Logger logger = LoggerFactory.getLogger(DeadTokenRetriever.class); private final IPriamInstanceFactory factory; private final IMembership membership; private final IConfiguration config; private final Sleeper sleeper; - private String replacedIp; //The IP address of the dead instance to which we will acquire its token + private String + replacedIp; // The IP address of the dead instance to which we will acquire its token private ListMultimap locMap; private final InstanceEnvIdentity insEnvIdentity; - @Inject - public DeadTokenRetriever(IPriamInstanceFactory factory, IMembership membership, IConfiguration config, - Sleeper sleeper, InstanceEnvIdentity insEnvIdentity) { + public DeadTokenRetriever( + IPriamInstanceFactory factory, + IMembership membership, + IConfiguration config, + Sleeper sleeper, + InstanceEnvIdentity insEnvIdentity) { this.factory = factory; this.membership = membership; this.config = config; @@ -68,11 +69,17 @@ private List getDualAccountRacMembership(List asgInstances) { if (logger.isInfoEnabled()) { if (insEnvIdentity.isClassic()) { - logger.info("EC2 classic instances (local ASG): " + Arrays.toString(asgInstances.toArray())); - logger.info("VPC Account (cross-account ASG): " + Arrays.toString(crossAccountAsgInstances.toArray())); + logger.info( + "EC2 classic instances (local ASG): " + + Arrays.toString(asgInstances.toArray())); + logger.info( + "VPC Account (cross-account ASG): " + + Arrays.toString(crossAccountAsgInstances.toArray())); } else { logger.info("VPC Account (local ASG): " + Arrays.toString(asgInstances.toArray())); - logger.info("EC2 classic instances (cross-account ASG): " + Arrays.toString(crossAccountAsgInstances.toArray())); + logger.info( + "EC2 classic instances (cross-account ASG): " + + Arrays.toString(crossAccountAsgInstances.toArray())); } } @@ -102,23 +109,41 @@ public PriamInstance get() throws Exception { sleeper.sleep(new Random().nextInt(5000) + 10000); for (PriamInstance dead : allIds) { // test same zone and is it is alive. - if (!dead.getRac().equals(config.getRac()) || asgInstances.contains(dead.getInstanceId()) || super.isInstanceDummy(dead)) - continue; + if (!dead.getRac().equals(config.getRac()) + || asgInstances.contains(dead.getInstanceId()) + || super.isInstanceDummy(dead)) continue; logger.info("Found dead instances: {}", dead.getInstanceId()); - PriamInstance markAsDead = factory.create(dead.getApp() + "-dead", dead.getId(), dead.getInstanceId(), dead.getHostName(), dead.getHostIP(), dead.getRac(), dead.getVolumes(), - dead.getToken()); + PriamInstance markAsDead = + factory.create( + dead.getApp() + "-dead", + dead.getId(), + dead.getInstanceId(), + dead.getHostName(), + dead.getHostIP(), + dead.getRac(), + dead.getVolumes(), + dead.getToken()); // remove it as we marked it down... factory.delete(dead); - //find the replaced IP + // find the replaced IP this.replacedIp = findReplaceIp(allIds, markAsDead.getToken(), markAsDead.getDC()); - if (this.replacedIp == null) - this.replacedIp = markAsDead.getHostIP(); + if (this.replacedIp == null) this.replacedIp = markAsDead.getHostIP(); String payLoad = markAsDead.getToken(); - logger.info("Trying to grab slot {} with availability zone {}", markAsDead.getId(), markAsDead.getRac()); - return factory.create(config.getAppName(), markAsDead.getId(), config.getInstanceName(), config.getHostname(), config.getHostIP(), config.getRac(), markAsDead.getVolumes(), payLoad); - + logger.info( + "Trying to grab slot {} with availability zone {}", + markAsDead.getId(), + markAsDead.getRac()); + return factory.create( + config.getAppName(), + markAsDead.getId(), + config.getInstanceName(), + config.getHostname(), + config.getHostIP(), + config.getRac(), + markAsDead.getVolumes(), + payLoad); } return null; @@ -133,7 +158,8 @@ private String findReplaceIp(List allIds, String token, String lo String ip; for (PriamInstance ins : allIds) { logger.info("Calling getIp on hostname[{}] and token[{}]", ins.getHostName(), token); - if (ins.getToken().equals(token) || !ins.getDC().equals(location)) { //avoid using dead instance and other regions' instances + if (ins.getToken().equals(token) || !ins.getDC().equals(location)) { + // avoid using dead instance and other regions instances continue; } @@ -162,17 +188,22 @@ private String getIp(String host, String token) throws ParseException { String textEntity; try { - clientResp = service.path("Priam/REST/v1/cassadmin/gossipinfo").accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); + clientResp = + service.path("Priam/REST/v1/cassadmin/gossipinfo") + .accept(MediaType.APPLICATION_JSON) + .get(ClientResponse.class); - if (clientResp.getStatus() != 200) - return null; + if (clientResp.getStatus() != 200) return null; textEntity = clientResp.getEntity(String.class); - logger.info("Respond from calling gossipinfo on host[{}] and token[{}] : {}", host, token, textEntity); + logger.info( + "Respond from calling gossipinfo on host[{}] and token[{}] : {}", + host, + token, + textEntity); - if (StringUtils.isEmpty(textEntity)) - return null; + if (StringUtils.isEmpty(textEntity)) return null; } catch (Exception e) { logger.info("Error in reaching out to host: {}", baseURI); return null; @@ -191,10 +222,13 @@ private String getIp(String host, String token) throws ParseException { String tokenVal = (String) msg.get("Token"); if (token.equals(tokenVal)) { - logger.info("Using gossipinfo from host[{}] and token[{}], the replaced address is : {}", host, token, key); + logger.info( + "Using gossipinfo from host[{}] and token[{}], the replaced address is : {}", + host, + token, + key); return (String) key; } - } return null; } @@ -206,7 +240,5 @@ private String getBaseURI(String host) { @Override public void setLocMap(ListMultimap locMap) { this.locMap = locMap; - } - -} \ No newline at end of file +} diff --git a/priam/src/main/java/com/netflix/priam/identity/token/IDeadTokenRetriever.java b/priam/src/main/java/com/netflix/priam/identity/token/IDeadTokenRetriever.java index 9061f5bfd..d14b82d72 100755 --- a/priam/src/main/java/com/netflix/priam/identity/token/IDeadTokenRetriever.java +++ b/priam/src/main/java/com/netflix/priam/identity/token/IDeadTokenRetriever.java @@ -1,16 +1,14 @@ /** * Copyright 2017 Netflix, Inc. - *

- * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and + * + *

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file + * except in compliance with the License. You may obtain a copy of the License at + * + *

http://www.apache.org/licenses/LICENSE-2.0 + * + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing permissions and * limitations under the License. */ package com.netflix.priam.identity.token; diff --git a/priam/src/main/java/com/netflix/priam/identity/token/INewTokenRetriever.java b/priam/src/main/java/com/netflix/priam/identity/token/INewTokenRetriever.java index ae3dcf1a9..ccf320008 100755 --- a/priam/src/main/java/com/netflix/priam/identity/token/INewTokenRetriever.java +++ b/priam/src/main/java/com/netflix/priam/identity/token/INewTokenRetriever.java @@ -1,16 +1,14 @@ /** * Copyright 2017 Netflix, Inc. - *

- * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and + * + *

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file + * except in compliance with the License. You may obtain a copy of the License at + * + *

http://www.apache.org/licenses/LICENSE-2.0 + * + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing permissions and * limitations under the License. */ package com.netflix.priam.identity.token; diff --git a/priam/src/main/java/com/netflix/priam/identity/token/IPreGeneratedTokenRetriever.java b/priam/src/main/java/com/netflix/priam/identity/token/IPreGeneratedTokenRetriever.java index a98e2da5c..74aed283f 100755 --- a/priam/src/main/java/com/netflix/priam/identity/token/IPreGeneratedTokenRetriever.java +++ b/priam/src/main/java/com/netflix/priam/identity/token/IPreGeneratedTokenRetriever.java @@ -1,16 +1,14 @@ /** * Copyright 2017 Netflix, Inc. - *

- * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and + * + *

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file + * except in compliance with the License. You may obtain a copy of the License at + * + *

http://www.apache.org/licenses/LICENSE-2.0 + * + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing permissions and * limitations under the License. */ package com.netflix.priam.identity.token; diff --git a/priam/src/main/java/com/netflix/priam/identity/token/NewTokenRetriever.java b/priam/src/main/java/com/netflix/priam/identity/token/NewTokenRetriever.java index 6fde4e53e..9f072a8a2 100755 --- a/priam/src/main/java/com/netflix/priam/identity/token/NewTokenRetriever.java +++ b/priam/src/main/java/com/netflix/priam/identity/token/NewTokenRetriever.java @@ -1,16 +1,14 @@ /** * Copyright 2017 Netflix, Inc. - *

- * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and + * + *

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file + * except in compliance with the License. You may obtain a copy of the License at + * + *

http://www.apache.org/licenses/LICENSE-2.0 + * + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing permissions and * limitations under the License. */ package com.netflix.priam.identity.token; @@ -23,11 +21,10 @@ import com.netflix.priam.identity.PriamInstance; import com.netflix.priam.utils.ITokenManager; import com.netflix.priam.utils.Sleeper; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import java.util.List; import java.util.Random; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class NewTokenRetriever extends TokenRetrieverBase implements INewTokenRetriever { @@ -40,19 +37,25 @@ public class NewTokenRetriever extends TokenRetrieverBase implements INewTokenRe private ListMultimap locMap; @Inject - //Note: do not parameterized the generic type variable to an implementation as it confuses Guice in the binding. - public NewTokenRetriever(IPriamInstanceFactory factory, IMembership membership, IConfiguration config, Sleeper sleeper, ITokenManager tokenManager) { - this.factory = factory; - this.membership = membership; - this.config = config; - this.sleeper = sleeper; - this.tokenManager = tokenManager; - } - - @Override - public PriamInstance get() throws Exception { - - logger.info("Generating my own and new token"); + // Note: do not parameterized the generic type variable to an implementation as it confuses + // Guice in the binding. + public NewTokenRetriever( + IPriamInstanceFactory factory, + IMembership membership, + IConfiguration config, + Sleeper sleeper, + ITokenManager tokenManager) { + this.factory = factory; + this.membership = membership; + this.config = config; + this.sleeper = sleeper; + this.tokenManager = tokenManager; + } + + @Override + public PriamInstance get() throws Exception { + + logger.info("Generating my own and new token"); // Sleep random interval - upto 15 sec sleeper.sleep(new Random().nextInt(15000)); int hash = tokenManager.regionOffset(config.getDC()); @@ -62,23 +65,43 @@ public PriamInstance get() throws Exception { int max = hash; List allInstances = factory.getAllIds(config.getAppName()); for (PriamInstance data : allInstances) - max = (data.getRac().equals(config.getRac()) && (data.getId() > max)) ? data.getId() : max; + max = + (data.getRac().equals(config.getRac()) && (data.getId() > max)) + ? data.getId() + : max; int maxSlot = max - hash; int my_slot; if (hash == max && locMap.get(config.getRac()).size() == 0) { int idx = config.getRacs().indexOf(config.getRac()); if (idx < 0) - throw new Exception(String.format("Rac %s is not in Racs %s", config.getRac(), config.getRacs())); + throw new Exception( + String.format( + "Rac %s is not in Racs %s", config.getRac(), config.getRacs())); my_slot = idx + maxSlot; - } else - my_slot = config.getRacs().size() + maxSlot; - - logger.info("Trying to createToken with slot {} with rac count {} with rac membership size {} with dc {}", - my_slot, membership.getRacCount(), membership.getRacMembershipSize(), config.getDC()); - String payload = tokenManager.createToken(my_slot, membership.getRacCount(), membership.getRacMembershipSize(), config.getDC()); - return factory.create(config.getAppName(), my_slot + hash, config.getInstanceName(), config.getHostname(), config.getHostIP(), config.getRac(), null, payload); + } else my_slot = config.getRacs().size() + maxSlot; + logger.info( + "Trying to createToken with slot {} with rac count {} with rac membership size {} with dc {}", + my_slot, + membership.getRacCount(), + membership.getRacMembershipSize(), + config.getDC()); + String payload = + tokenManager.createToken( + my_slot, + membership.getRacCount(), + membership.getRacMembershipSize(), + config.getDC()); + return factory.create( + config.getAppName(), + my_slot + hash, + config.getInstanceName(), + config.getHostname(), + config.getHostIP(), + config.getRac(), + null, + payload); } /* @@ -88,5 +111,4 @@ public PriamInstance get() throws Exception { public void setLocMap(ListMultimap locMap) { this.locMap = locMap; } - } diff --git a/priam/src/main/java/com/netflix/priam/identity/token/PreGeneratedTokenRetriever.java b/priam/src/main/java/com/netflix/priam/identity/token/PreGeneratedTokenRetriever.java index a65c19682..a4c7a940b 100755 --- a/priam/src/main/java/com/netflix/priam/identity/token/PreGeneratedTokenRetriever.java +++ b/priam/src/main/java/com/netflix/priam/identity/token/PreGeneratedTokenRetriever.java @@ -1,16 +1,14 @@ /** * Copyright 2017 Netflix, Inc. - *

- * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and + * + *

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file + * except in compliance with the License. You may obtain a copy of the License at + * + *

http://www.apache.org/licenses/LICENSE-2.0 + * + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing permissions and * limitations under the License. */ package com.netflix.priam.identity.token; @@ -22,13 +20,13 @@ import com.netflix.priam.identity.IPriamInstanceFactory; import com.netflix.priam.identity.PriamInstance; import com.netflix.priam.utils.Sleeper; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import java.util.List; import java.util.Random; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; -public class PreGeneratedTokenRetriever extends TokenRetrieverBase implements IPreGeneratedTokenRetriever { +public class PreGeneratedTokenRetriever extends TokenRetrieverBase + implements IPreGeneratedTokenRetriever { private static final Logger logger = LoggerFactory.getLogger(PreGeneratedTokenRetriever.class); private final IPriamInstanceFactory factory; @@ -38,7 +36,11 @@ public class PreGeneratedTokenRetriever extends TokenRetrieverBase implements IP private ListMultimap locMap; @Inject - public PreGeneratedTokenRetriever(IPriamInstanceFactory factory, IMembership membership, IConfiguration config, Sleeper sleeper) { + public PreGeneratedTokenRetriever( + IPriamInstanceFactory factory, + IMembership membership, + IConfiguration config, + Sleeper sleeper) { this.factory = factory; this.membership = membership; this.config = config; @@ -55,18 +57,37 @@ public PriamInstance get() throws Exception { sleeper.sleep(new Random().nextInt(5000) + 10000); for (PriamInstance dead : allIds) { // test same zone and is it is alive. - if (!dead.getRac().equals(config.getRac()) || asgInstances.contains(dead.getInstanceId()) || !isInstanceDummy(dead)) - continue; + if (!dead.getRac().equals(config.getRac()) + || asgInstances.contains(dead.getInstanceId()) + || !isInstanceDummy(dead)) continue; logger.info("Found pre-generated token: {}", dead.getToken()); - PriamInstance markAsDead = factory.create(dead.getApp() + "-dead", dead.getId(), dead.getInstanceId(), dead.getHostName(), dead.getHostIP(), dead.getRac(), dead.getVolumes(), - dead.getToken()); + PriamInstance markAsDead = + factory.create( + dead.getApp() + "-dead", + dead.getId(), + dead.getInstanceId(), + dead.getHostName(), + dead.getHostIP(), + dead.getRac(), + dead.getVolumes(), + dead.getToken()); // remove it as we marked it down... factory.delete(dead); - String payLoad = markAsDead.getToken(); - logger.info("Trying to grab slot {} with availability zone {}", markAsDead.getId(), markAsDead.getRac()); - return factory.create(config.getAppName(), markAsDead.getId(), config.getInstanceName(), config.getHostname(), config.getHostIP(), config.getRac(), markAsDead.getVolumes(), payLoad); + logger.info( + "Trying to grab slot {} with availability zone {}", + markAsDead.getId(), + markAsDead.getRac()); + return factory.create( + config.getAppName(), + markAsDead.getId(), + config.getInstanceName(), + config.getHostname(), + config.getHostIP(), + config.getRac(), + markAsDead.getVolumes(), + payLoad); } return null; } @@ -74,7 +95,5 @@ public PriamInstance get() throws Exception { @Override public void setLocMap(ListMultimap locMap) { this.locMap = locMap; - } - } diff --git a/priam/src/main/java/com/netflix/priam/identity/token/TokenRetrieverBase.java b/priam/src/main/java/com/netflix/priam/identity/token/TokenRetrieverBase.java index 3a929f903..04353f450 100755 --- a/priam/src/main/java/com/netflix/priam/identity/token/TokenRetrieverBase.java +++ b/priam/src/main/java/com/netflix/priam/identity/token/TokenRetrieverBase.java @@ -1,28 +1,25 @@ /** * Copyright 2017 Netflix, Inc. - *

- * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and + * + *

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file + * except in compliance with the License. You may obtain a copy of the License at + * + *

http://www.apache.org/licenses/LICENSE-2.0 + * + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing permissions and * limitations under the License. */ package com.netflix.priam.identity.token; import com.netflix.priam.identity.PriamInstance; - import java.util.Random; public class TokenRetrieverBase { public static final String DUMMY_INSTANCE_ID = "new_slot"; - private static final int MAX_VALUE_IN_MILISECS = 300000; //sleep up to 5 minutes + private static final int MAX_VALUE_IN_MILISECS = 300000; // sleep up to 5 minutes protected final Random randomizer; public TokenRetrieverBase() { @@ -34,8 +31,8 @@ protected boolean isInstanceDummy(PriamInstance instance) { } /* - * Return a random time for a thread to sleep. - * + * Return a random time for a thread to sleep. + * * @return time in millisecs */ protected long getSleepTime() { diff --git a/priam/src/main/java/com/netflix/priam/merics/BackupMetrics.java b/priam/src/main/java/com/netflix/priam/merics/BackupMetrics.java index 27a96c910..7f86b273e 100644 --- a/priam/src/main/java/com/netflix/priam/merics/BackupMetrics.java +++ b/priam/src/main/java/com/netflix/priam/merics/BackupMetrics.java @@ -1,16 +1,14 @@ /** * Copyright 2017 Netflix, Inc. - *

- * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and + * + *

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file + * except in compliance with the License. You may obtain a copy of the License at + * + *

http://www.apache.org/licenses/LICENSE-2.0 + * + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing permissions and * limitations under the License. */ package com.netflix.priam.merics; @@ -21,17 +19,23 @@ import com.netflix.spectator.api.DistributionSummary; import com.netflix.spectator.api.Registry; -/** - * Created by vinhn on 2/13/17. - */ +/** Created by vinhn on 2/13/17. */ @Singleton public class BackupMetrics { private final Registry registry; /** - * Distribution summary will provide the metric like count (how many uploads were made), max no. of bytes uploaded and total amount of bytes uploaded. + * Distribution summary will provide the metric like count (how many uploads were made), max no. + * of bytes uploaded and total amount of bytes uploaded. */ private final DistributionSummary uploadRate, downloadRate; - private final Counter validUploads, validDownloads, invalidUploads, invalidDownloads, snsNotificationSuccess, snsNotificationFailure, forgottenFiles; + + private final Counter validUploads, + validDownloads, + invalidUploads, + invalidDownloads, + snsNotificationSuccess, + snsNotificationFailure, + forgottenFiles; public static final String uploadQueueSize = Metrics.METRIC_PREFIX + "upload.queue.size"; public static final String downloadQueueSize = Metrics.METRIC_PREFIX + "download.queue.size"; @@ -44,8 +48,10 @@ public BackupMetrics(Registry registry) { invalidUploads = registry.counter(Metrics.METRIC_PREFIX + "upload.invalid"); uploadRate = registry.distributionSummary(Metrics.METRIC_PREFIX + "upload.rate"); downloadRate = registry.distributionSummary(Metrics.METRIC_PREFIX + "download.rate"); - snsNotificationSuccess = registry.counter(Metrics.METRIC_PREFIX + "sns.notification.success"); - snsNotificationFailure = registry.counter(Metrics.METRIC_PREFIX + "sns.notification.failure"); + snsNotificationSuccess = + registry.counter(Metrics.METRIC_PREFIX + "sns.notification.success"); + snsNotificationFailure = + registry.counter(Metrics.METRIC_PREFIX + "sns.notification.failure"); forgottenFiles = registry.counter(Metrics.METRIC_PREFIX + "forgotten.files"); } diff --git a/priam/src/main/java/com/netflix/priam/merics/CassMonitorMetrics.java b/priam/src/main/java/com/netflix/priam/merics/CassMonitorMetrics.java index 5cafa5c12..a4338964a 100644 --- a/priam/src/main/java/com/netflix/priam/merics/CassMonitorMetrics.java +++ b/priam/src/main/java/com/netflix/priam/merics/CassMonitorMetrics.java @@ -1,16 +1,14 @@ /** * Copyright 2018 Netflix, Inc. - *

- * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and + * + *

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file + * except in compliance with the License. You may obtain a copy of the License at + * + *

http://www.apache.org/licenses/LICENSE-2.0 + * + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing permissions and * limitations under the License. */ package com.netflix.priam.merics; @@ -20,9 +18,7 @@ import com.netflix.spectator.api.Gauge; import com.netflix.spectator.api.Registry; -/** - * @author vchella - */ +/** @author vchella */ @Singleton public class CassMonitorMetrics { private final Gauge cassStop, cassAutoStart, cassStart; @@ -45,5 +41,4 @@ public void incCassAutoStart() { public void incCassStart() { cassStart.set(cassStart.value() + 1); } - } diff --git a/priam/src/main/java/com/netflix/priam/merics/CompactionMeasurement.java b/priam/src/main/java/com/netflix/priam/merics/CompactionMeasurement.java index fbd1dad1e..b3ff3a1cf 100644 --- a/priam/src/main/java/com/netflix/priam/merics/CompactionMeasurement.java +++ b/priam/src/main/java/com/netflix/priam/merics/CompactionMeasurement.java @@ -18,14 +18,10 @@ import com.netflix.spectator.api.Counter; import com.netflix.spectator.api.Registry; - import javax.inject.Inject; import javax.inject.Singleton; -/** - * Measurement class for scheduled compactions - * Created by aagrawal on 2/28/18. - */ +/** Measurement class for scheduled compactions Created by aagrawal on 2/28/18. */ @Singleton public class CompactionMeasurement implements IMeasurement { private final Counter failure, success; @@ -36,7 +32,6 @@ public CompactionMeasurement(Registry registry) { success = registry.counter(Metrics.METRIC_PREFIX + "compaction.success"); } - public void incrementFailure() { this.failure.increment(); } diff --git a/priam/src/main/java/com/netflix/priam/merics/IMeasurement.java b/priam/src/main/java/com/netflix/priam/merics/IMeasurement.java index 0a83d356a..e00a5e95d 100644 --- a/priam/src/main/java/com/netflix/priam/merics/IMeasurement.java +++ b/priam/src/main/java/com/netflix/priam/merics/IMeasurement.java @@ -1,25 +1,22 @@ /** * Copyright 2017 Netflix, Inc. - *

- * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and + * + *

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file + * except in compliance with the License. You may obtain a copy of the License at + * + *

http://www.apache.org/licenses/LICENSE-2.0 + * + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing permissions and * limitations under the License. */ package com.netflix.priam.merics; /** - * * Represents a specific measurement for publishing to a metric system * - * Created by vinhn on 10/14/16. + *

Created by vinhn on 10/14/16. */ public interface IMeasurement { void incrementFailure(); diff --git a/priam/src/main/java/com/netflix/priam/merics/Metrics.java b/priam/src/main/java/com/netflix/priam/merics/Metrics.java index 1c9f1168c..41cb61867 100644 --- a/priam/src/main/java/com/netflix/priam/merics/Metrics.java +++ b/priam/src/main/java/com/netflix/priam/merics/Metrics.java @@ -17,9 +17,7 @@ package com.netflix.priam.merics; -/** - * Created by aagrawal on 8/15/18. - */ +/** Created by aagrawal on 8/15/18. */ public interface Metrics { String METRIC_PREFIX = "priam."; } diff --git a/priam/src/main/java/com/netflix/priam/merics/NodeToolFlushMeasurement.java b/priam/src/main/java/com/netflix/priam/merics/NodeToolFlushMeasurement.java index 3670feb5f..89b16a745 100644 --- a/priam/src/main/java/com/netflix/priam/merics/NodeToolFlushMeasurement.java +++ b/priam/src/main/java/com/netflix/priam/merics/NodeToolFlushMeasurement.java @@ -1,31 +1,27 @@ /** * Copyright 2017 Netflix, Inc. - *

- * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and + * + *

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file + * except in compliance with the License. You may obtain a copy of the License at + * + *

http://www.apache.org/licenses/LICENSE-2.0 + * + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing permissions and * limitations under the License. */ package com.netflix.priam.merics; import com.netflix.spectator.api.Counter; import com.netflix.spectator.api.Registry; - import javax.inject.Inject; import javax.inject.Singleton; /** - * * Represents the value to be publish to a telemetry endpoint * - * Created by vinhn on 10/14/16. + *

Created by vinhn on 10/14/16. */ @Singleton public class NodeToolFlushMeasurement implements IMeasurement { @@ -45,4 +41,3 @@ public void incrementSuccess() { success.increment(); } } - diff --git a/priam/src/main/java/com/netflix/priam/notification/AWSSnsNotificationService.java b/priam/src/main/java/com/netflix/priam/notification/AWSSnsNotificationService.java index 624121976..daab229c7 100644 --- a/priam/src/main/java/com/netflix/priam/notification/AWSSnsNotificationService.java +++ b/priam/src/main/java/com/netflix/priam/notification/AWSSnsNotificationService.java @@ -23,74 +23,82 @@ import com.amazonaws.services.sns.model.PublishResult; import com.google.inject.Inject; import com.google.inject.Singleton; -import com.netflix.priam.config.IConfiguration; import com.netflix.priam.aws.IAMCredential; +import com.netflix.priam.config.IConfiguration; import com.netflix.priam.merics.BackupMetrics; import com.netflix.priam.utils.BoundedExponentialRetryCallable; +import java.util.Map; import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.Map; - /* * A single, persisted, connection to Amazon SNS. */ @Singleton public class AWSSnsNotificationService implements INotificationService { - private static final Logger logger = LoggerFactory.getLogger(AWSSnsNotificationService.class); + private static final Logger logger = LoggerFactory.getLogger(AWSSnsNotificationService.class); + + private final IConfiguration configuration; + private final AmazonSNS snsClient; + private final BackupMetrics backupMetrics; + + @Inject + public AWSSnsNotificationService( + IConfiguration config, IAMCredential iamCredential, BackupMetrics backupMetrics) { + this.configuration = config; + this.backupMetrics = backupMetrics; + String ec2_region = this.configuration.getDC(); + snsClient = + AmazonSNSClient.builder() + .withCredentials(iamCredential.getAwsCredentialProvider()) + .withRegion(ec2_region) + .build(); + } + + @Override + public void notify( + final String msg, final Map messageAttributes) { + // e.g. arn:aws:sns:eu-west-1:1234:eu-west-1-cass-sample-backup + final String topic_arn = this.configuration.getBackupNotificationTopicArn(); + if (StringUtils.isEmpty(topic_arn)) { + return; + } - private final IConfiguration configuration; - private final AmazonSNS snsClient; - private final BackupMetrics backupMetrics; - - @Inject - public AWSSnsNotificationService(IConfiguration config, IAMCredential iamCredential - , BackupMetrics backupMetrics) { - this.configuration = config; - this.backupMetrics = backupMetrics; - String ec2_region = this.configuration.getDC(); - snsClient = AmazonSNSClient.builder() - .withCredentials(iamCredential.getAwsCredentialProvider()) - .withRegion(ec2_region).build(); - } - - @Override - public void notify(final String msg, final Map messageAttributes) { - final String topic_arn = this.configuration.getBackupNotificationTopicArn(); //e.g. arn:aws:sns:eu-west-1:1234:eu-west-1-cass-sample-backup - if (StringUtils.isEmpty(topic_arn)) { - return; - } - - PublishResult publishResult; - try { - publishResult = new BoundedExponentialRetryCallable() { - @Override - public PublishResult retriableCall() throws Exception { - PublishRequest publishRequest = new PublishRequest(topic_arn, msg).withMessageAttributes(messageAttributes); - PublishResult result = snsClient.publish(publishRequest); - return result; - } - }.call(); - - } catch (Exception e) { - logger.error(String.format("Exhausted retries. Publishing notification metric for failure and moving on. Failed msg to publish: {}", msg), e); - backupMetrics.incrementSnsNotificationFailure(); - return; - } + PublishResult publishResult; + try { + publishResult = + new BoundedExponentialRetryCallable() { + @Override + public PublishResult retriableCall() throws Exception { + PublishRequest publishRequest = + new PublishRequest(topic_arn, msg) + .withMessageAttributes(messageAttributes); + PublishResult result = snsClient.publish(publishRequest); + return result; + } + }.call(); - //If here, message was published. As a extra validation, ensure we have a msg id - String publishedMsgId = publishResult.getMessageId(); - if (publishedMsgId == null || publishedMsgId.isEmpty() ) { - backupMetrics.incrementSnsNotificationFailure(); - return; - } + } catch (Exception e) { + logger.error( + String.format( + "Exhausted retries. Publishing notification metric for failure and moving on. Failed msg to publish: {}", + msg), + e); + backupMetrics.incrementSnsNotificationFailure(); + return; + } - backupMetrics.incrementSnsNotificationSuccess(); - if (logger.isTraceEnabled()) { - logger.trace("Published msg: {} aws sns messageId - {}", msg, publishedMsgId); - } - } + // If here, message was published. As a extra validation, ensure we have a msg id + String publishedMsgId = publishResult.getMessageId(); + if (publishedMsgId == null || publishedMsgId.isEmpty()) { + backupMetrics.incrementSnsNotificationFailure(); + return; + } - -} \ No newline at end of file + backupMetrics.incrementSnsNotificationSuccess(); + if (logger.isTraceEnabled()) { + logger.trace("Published msg: {} aws sns messageId - {}", msg, publishedMsgId); + } + } +} diff --git a/priam/src/main/java/com/netflix/priam/notification/BackupEvent.java b/priam/src/main/java/com/netflix/priam/notification/BackupEvent.java index 4107f60e2..9f156858c 100644 --- a/priam/src/main/java/com/netflix/priam/notification/BackupEvent.java +++ b/priam/src/main/java/com/netflix/priam/notification/BackupEvent.java @@ -20,10 +20,9 @@ import com.netflix.priam.backup.AbstractBackupPath; /** - * POJO to encapsulate the details of backup of a file. - * Use this class to notify of the backup and its status. This will allow us to add more details about the backup event - * without modifying AbstractBackupPath. - * Created by aagrawal on 8/11/17. + * POJO to encapsulate the details of backup of a file. Use this class to notify of the backup and + * its status. This will allow us to add more details about the backup event without modifying + * AbstractBackupPath. Created by aagrawal on 8/11/17. */ public class BackupEvent { private AbstractBackupPath abstractBackupPath; diff --git a/priam/src/main/java/com/netflix/priam/notification/BackupNotificationMgr.java b/priam/src/main/java/com/netflix/priam/notification/BackupNotificationMgr.java index c2c4d9f7e..da1414575 100644 --- a/priam/src/main/java/com/netflix/priam/notification/BackupNotificationMgr.java +++ b/priam/src/main/java/com/netflix/priam/notification/BackupNotificationMgr.java @@ -1,36 +1,33 @@ /** * Copyright 2017 Netflix, Inc. - *

- * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and + * + *

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file + * except in compliance with the License. You may obtain a copy of the License at + * + *

http://www.apache.org/licenses/LICENSE-2.0 + * + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing permissions and * limitations under the License. */ package com.netflix.priam.notification; import com.amazonaws.services.sns.model.MessageAttributeValue; import com.google.inject.Inject; -import com.netflix.priam.config.IConfiguration; import com.netflix.priam.backup.AbstractBackupPath; +import com.netflix.priam.config.IConfiguration; +import java.util.HashMap; +import java.util.Map; import org.codehaus.jettison.json.JSONException; import org.codehaus.jettison.json.JSONObject; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.HashMap; -import java.util.Map; - /** * A means to nofity interested party(ies) of an uploaded file, success or failed. * - * Created by vinhn on 10/30/16. + *

Created by vinhn on 10/30/16. */ public class BackupNotificationMgr implements EventObserver { @@ -64,14 +61,26 @@ private void notify(AbstractBackupPath abp, String uploadStatus) { jsonObject.put("backuptype", abp.getType().name()); jsonObject.put("uploadstatus", uploadStatus); - //SNS Attributes for filtering messages. Cluster name and backup file type. + // SNS Attributes for filtering messages. Cluster name and backup file type. Map messageAttributes = new HashMap<>(); - messageAttributes.putIfAbsent("s3clustername", new MessageAttributeValue().withDataType("String").withStringValue(abp.getClusterName())); - messageAttributes.putIfAbsent("backuptype", new MessageAttributeValue().withDataType("String").withStringValue(abp.getType().name())); + messageAttributes.putIfAbsent( + "s3clustername", + new MessageAttributeValue() + .withDataType("String") + .withStringValue(abp.getClusterName())); + messageAttributes.putIfAbsent( + "backuptype", + new MessageAttributeValue() + .withDataType("String") + .withStringValue(abp.getType().name())); this.notificationService.notify(jsonObject.toString(), messageAttributes); } catch (JSONException exception) { - logger.error("JSON exception during generation of notification for upload {}. Local file {}. Ignoring to continue with rest of backup. Msg: {}", uploadStatus, abp.getFileName(), exception.getLocalizedMessage()); + logger.error( + "JSON exception during generation of notification for upload {}. Local file {}. Ignoring to continue with rest of backup. Msg: {}", + uploadStatus, + abp.getFileName(), + exception.getLocalizedMessage()); } } @@ -94,5 +103,4 @@ public void updateEventSuccess(BackupEvent event) { public void updateEventStop(BackupEvent event) { // Do nothing. } - } diff --git a/priam/src/main/java/com/netflix/priam/notification/EventGenerator.java b/priam/src/main/java/com/netflix/priam/notification/EventGenerator.java index 068f46503..99b28f02b 100644 --- a/priam/src/main/java/com/netflix/priam/notification/EventGenerator.java +++ b/priam/src/main/java/com/netflix/priam/notification/EventGenerator.java @@ -18,24 +18,24 @@ package com.netflix.priam.notification; /** - * Generic interface which an event generator class should implement so all the observers could be notified of the events. - * Any class interested in state change for the event can subscribe by registering themselves. - * Created by aagrawal on 8/11/17. + * Generic interface which an event generator class should implement so all the observers could be + * notified of the events. Any class interested in state change for the event can subscribe by + * registering themselves. Created by aagrawal on 8/11/17. */ public interface EventGenerator { /** * Subscribes {@code observer} to receive generated events. * - * @param observer {@link EventObserver} interested in receiving updates from - * this event generator. May not be null. + * @param observer {@link EventObserver} interested in receiving updates from this event + * generator. May not be null. */ void addObserver(EventObserver observer); /** * Removes {@code observer} from receiving any further events from this generator. * - * @param observer {@link EventObserver} that is to stop receiving updates - * from this event generator. May not be null. + * @param observer {@link EventObserver} that is to stop receiving updates from this event + * generator. May not be null. */ void removeObserver(EventObserver observer); diff --git a/priam/src/main/java/com/netflix/priam/notification/EventObserver.java b/priam/src/main/java/com/netflix/priam/notification/EventObserver.java index 21cb02bd1..3d9326828 100644 --- a/priam/src/main/java/com/netflix/priam/notification/EventObserver.java +++ b/priam/src/main/java/com/netflix/priam/notification/EventObserver.java @@ -18,8 +18,8 @@ package com.netflix.priam.notification; /** - * Subscriber who wishes to receive event notifications from the {@link EventGenerator} - * Created by aagrawal on 8/11/17. + * Subscriber who wishes to receive event notifications from the {@link EventGenerator} Created by + * aagrawal on 8/11/17. */ public interface EventObserver { /** diff --git a/priam/src/main/java/com/netflix/priam/notification/INotificationService.java b/priam/src/main/java/com/netflix/priam/notification/INotificationService.java index 62725eba5..7fa036f6e 100644 --- a/priam/src/main/java/com/netflix/priam/notification/INotificationService.java +++ b/priam/src/main/java/com/netflix/priam/notification/INotificationService.java @@ -1,36 +1,30 @@ /** * Copyright 2017 Netflix, Inc. - *

- * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and + * + *

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file + * except in compliance with the License. You may obtain a copy of the License at + * + *

http://www.apache.org/licenses/LICENSE-2.0 + * + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing permissions and * limitations under the License. */ package com.netflix.priam.notification; import com.amazonaws.services.sns.model.MessageAttributeValue; import com.google.inject.ImplementedBy; - import java.util.Map; -/** - * Service to notify of a message. - * Created by vinhn on 11/3/16. - */ +/** Service to notify of a message. Created by vinhn on 11/3/16. */ @ImplementedBy(AWSSnsNotificationService.class) interface INotificationService { /** * Notify the message. + * * @param msg Message that needs to be notified * @param messageAttributes Message attributes to be used while sending the message. */ void notify(String msg, Map messageAttributes); - } diff --git a/priam/src/main/java/com/netflix/priam/resources/BackupServlet.java b/priam/src/main/java/com/netflix/priam/resources/BackupServlet.java index 52d9864f2..f2eb9f711 100644 --- a/priam/src/main/java/com/netflix/priam/resources/BackupServlet.java +++ b/priam/src/main/java/com/netflix/priam/resources/BackupServlet.java @@ -20,29 +20,17 @@ import com.google.inject.Inject; import com.google.inject.Provider; import com.google.inject.name.Named; -import com.netflix.priam.defaultimpl.ICassandraProcess; -import com.netflix.priam.config.IConfiguration; import com.netflix.priam.PriamServer; import com.netflix.priam.backup.*; import com.netflix.priam.backup.AbstractBackupPath.BackupFileType; +import com.netflix.priam.config.IConfiguration; +import com.netflix.priam.defaultimpl.ICassandraProcess; import com.netflix.priam.identity.IPriamInstanceFactory; import com.netflix.priam.identity.PriamInstance; import com.netflix.priam.restore.Restore; -import com.netflix.priam.tuner.ICassandraTuner; import com.netflix.priam.scheduler.PriamScheduler; +import com.netflix.priam.tuner.ICassandraTuner; import com.netflix.priam.utils.*; -import org.apache.commons.lang3.StringUtils; -import org.apache.commons.lang3.exception.ExceptionUtils; -import org.codehaus.jettison.json.JSONArray; -import org.codehaus.jettison.json.JSONException; -import org.codehaus.jettison.json.JSONObject; -import org.joda.time.DateTime; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import javax.ws.rs.*; -import javax.ws.rs.core.MediaType; -import javax.ws.rs.core.Response; import java.io.File; import java.io.IOException; import java.math.BigInteger; @@ -52,6 +40,17 @@ import java.util.List; import java.util.concurrent.*; import java.util.stream.Collectors; +import javax.ws.rs.*; +import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.exception.ExceptionUtils; +import org.codehaus.jettison.json.JSONArray; +import org.codehaus.jettison.json.JSONException; +import org.codehaus.jettison.json.JSONObject; +import org.joda.time.DateTime; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; @Path("/v1/backup") @Produces(MediaType.APPLICATION_JSON) @@ -84,17 +83,25 @@ public class BackupServlet { private final ITokenManager tokenManager; private final ICassandraProcess cassProcess; private final BackupVerification backupVerification; - @Inject - private PriamScheduler scheduler; - @Inject - private MetaData metaData; + @Inject private PriamScheduler scheduler; + @Inject private MetaData metaData; private final IBackupStatusMgr completedBkups; @Inject - public BackupServlet(PriamServer priamServer, IConfiguration config, @Named("backup") IBackupFileSystem backupFs, Restore restoreObj, Provider pathProvider, ICassandraTuner tuner, - SnapshotBackup snapshotBackup, IPriamInstanceFactory factory, ITokenManager tokenManager, ICassandraProcess cassProcess - , IBackupStatusMgr completedBkups, BackupVerification backupVerification) { + public BackupServlet( + PriamServer priamServer, + IConfiguration config, + @Named("backup") IBackupFileSystem backupFs, + Restore restoreObj, + Provider pathProvider, + ICassandraTuner tuner, + SnapshotBackup snapshotBackup, + IPriamInstanceFactory factory, + ITokenManager tokenManager, + ICassandraProcess cassProcess, + IBackupStatusMgr completedBkups, + BackupVerification backupVerification) { this.priamServer = priamServer; this.config = config; this.backupFs = backupFs; @@ -119,21 +126,24 @@ public Response backup() throws Exception { @GET @Path("/incremental_backup") public Response backupIncrementals() throws Exception { - scheduler.addTask("IncrementalBackup", IncrementalBackup.class, IncrementalBackup.getTimer()); + scheduler.addTask( + "IncrementalBackup", IncrementalBackup.class, IncrementalBackup.getTimer()); return Response.ok(REST_SUCCESS, MediaType.APPLICATION_JSON).build(); } - @GET @Path("/list") /* * Fetch the list of files for the requested date range. - * + * * @param date range * @param filter. The type of data files fetched. E.g. META will only fetch the daily snapshot meta data file (meta.json). * @return the list of files in json format as part of the Http response body. */ - public Response list(@QueryParam(REST_HEADER_RANGE) String daterange, @QueryParam(REST_HEADER_FILTER) @DefaultValue("") String filter) throws Exception { + public Response list( + @QueryParam(REST_HEADER_RANGE) String daterange, + @QueryParam(REST_HEADER_FILTER) @DefaultValue("") String filter) + throws Exception { Date startTime; Date endTime; @@ -147,22 +157,29 @@ public Response list(@QueryParam(REST_HEADER_RANGE) String daterange, @QueryPara endTime = path.parseDate(restore[1]); } - logger.info("Parameters: {backupPrefix: [{}], daterange: [{}], filter: [{}]}", - config.getBackupPrefix(), daterange, filter); + logger.info( + "Parameters: {backupPrefix: [{}], daterange: [{}], filter: [{}]}", + config.getBackupPrefix(), + daterange, + filter); - Iterator it = backupFs.list(config.getBackupPrefix(), startTime, endTime); + Iterator it = + backupFs.list(config.getBackupPrefix(), startTime, endTime); JSONObject object = new JSONObject(); object = constructJsonResponse(object, it, filter); return Response.ok(object.toString(2), MediaType.APPLICATION_JSON).build(); } - @GET @Path("/status") @Produces(MediaType.APPLICATION_JSON) public Response status() throws Exception { - int restoreQueueSize = restoreObj.getDownloadTasksQueued(); //Items left to restore from the filesystem. - logger.info("Thread counts for restore is: {}. Items in queue: {}", config.getRestoreThreads(), restoreQueueSize); + int restoreQueueSize = + restoreObj.getDownloadTasksQueued(); // Items left to restore from the filesystem. + logger.info( + "Thread counts for restore is: {}. Items in queue: {}", + config.getRestoreThreads(), + restoreQueueSize); JSONObject object = new JSONObject(); object.put("SnapshotStatus", snapshotBackup.state().toString()); return Response.ok(object.toString(), MediaType.APPLICATION_JSON).build(); @@ -198,14 +215,17 @@ public Response statusByDate(@PathParam("date") String date) throws Exception { } if (bkupMetadata.getCompleted() != null) { - object.put("completetime", DateUtil.formatyyyyMMddHHmm(bkupMetadata.getCompleted())); + object.put( + "completetime", DateUtil.formatyyyyMMddHHmm(bkupMetadata.getCompleted())); } else { object.put("completetime", "not_available"); } - } else { //Backup do not exist for that date. + } else { // Backup do not exist for that date. object.put("Snapshotstatus", false); - String token = SystemUtils.getDataFromUrl("http://localhost:8080/Priam/REST/v1/cassconfig/get_token"); + String token = + SystemUtils.getDataFromUrl( + "http://localhost:8080/Priam/REST/v1/cassconfig/get_token"); if (token != null && !token.isEmpty()) { object.put("token", token); } else { @@ -231,7 +251,12 @@ public Response snapshotsByDate(@PathParam("date") String date) throws Exception List snapshots = new ArrayList<>(); if (metadata != null && !metadata.isEmpty()) - snapshots.addAll(metadata.stream().map(backupMetadata -> DateUtil.formatyyyyMMddHHmm(backupMetadata.getStart())).collect(Collectors.toList())); + snapshots.addAll( + metadata.stream() + .map( + backupMetadata -> + DateUtil.formatyyyyMMddHHmm(backupMetadata.getStart())) + .collect(Collectors.toList())); object.put("Snapshots", snapshots); return Response.ok(object.toString(), MediaType.APPLICATION_JSON).build(); @@ -239,21 +264,24 @@ public Response snapshotsByDate(@PathParam("date") String date) throws Exception private List getLatestBackupMetadata(Date startTime, Date endTime) { List backupMetadata = this.completedBkups.locate(endTime); - if (backupMetadata != null && !backupMetadata.isEmpty()) - return backupMetadata; + if (backupMetadata != null && !backupMetadata.isEmpty()) return backupMetadata; if (DateUtil.formatyyyyMMdd(startTime).equals(DateUtil.formatyyyyMMdd(endTime))) { - logger.info("Start & end date are same. No SNAPSHOT found for date: {}", DateUtil.formatyyyyMMdd(endTime)); + logger.info( + "Start & end date are same. No SNAPSHOT found for date: {}", + DateUtil.formatyyyyMMdd(endTime)); return null; } else { Date previousDay = new Date(endTime.getTime()); do { - //We need to find the latest backupmetadata in this date range. + // We need to find the latest backupmetadata in this date range. previousDay = new DateTime(previousDay.getTime()).minusDays(1).toDate(); - logger.info("Will try to find snapshot for previous day: {}", DateUtil.formatyyyyMMdd(previousDay)); + logger.info( + "Will try to find snapshot for previous day: {}", + DateUtil.formatyyyyMMdd(previousDay)); backupMetadata = completedBkups.locate(previousDay); - if (backupMetadata != null && !backupMetadata.isEmpty()) - return backupMetadata; - } while (!DateUtil.formatyyyyMMdd(startTime).equals(DateUtil.formatyyyyMMdd(previousDay))); + if (backupMetadata != null && !backupMetadata.isEmpty()) return backupMetadata; + } while (!DateUtil.formatyyyyMMdd(startTime) + .equals(DateUtil.formatyyyyMMdd(previousDay))); } return null; } @@ -267,7 +295,8 @@ private List getLatestBackupMetadata(Date startTime, Date endTim @GET @Path("/validate/snapshot/{daterange}") @Produces(MediaType.APPLICATION_JSON) - public Response validateSnapshotByDate(@PathParam("daterange") String daterange) throws Exception { + public Response validateSnapshotByDate(@PathParam("daterange") String daterange) + throws Exception { Date startTime; Date endTime; @@ -284,7 +313,10 @@ public Response validateSnapshotByDate(@PathParam("daterange") String daterange) JSONObject jsonReply = new JSONObject(); jsonReply.put("inputStartDate", DateUtil.formatyyyyMMddHHmm(startTime)); jsonReply.put("inputEndDate", DateUtil.formatyyyyMMddHHmm(endTime)); - logger.info("Will try to validate latest backup during startTime: {}, and endTime: {}", DateUtil.formatyyyyMMddHHmm(startTime), DateUtil.formatyyyyMMddHHmm(endTime)); + logger.info( + "Will try to validate latest backup during startTime: {}, and endTime: {}", + DateUtil.formatyyyyMMddHHmm(startTime), + DateUtil.formatyyyyMMddHHmm(endTime)); List metadata = getLatestBackupMetadata(startTime, endTime); BackupVerificationResult result = backupVerification.verifyBackup(metadata, startTime); @@ -301,31 +333,26 @@ public Response validateSnapshotByDate(@PathParam("daterange") String daterange) } /** + * Life_Of_C*Row : With this REST call, mutations/existence of a rowkey can be found. It uses + * SSTable2Json utility which will convert SSTables on disk to JSON format and Search for the + * desired rowkey. + * + *

Steps include: 1. Restoring data for given data range and other params 2. Searching + * provided rowkey in SSTables and writing search result to JSON 3. Delete all the files under + * Keyspace Directory. Deletion is done for efficient space usage, so that same node can be + * reused for subsequent runs. + * *

- * Life_Of_C*Row : With this REST call, mutations/existence of a rowkey can be found. - * It uses SSTable2Json utility which will convert SSTables on disk to JSON format and - * Search for the desired rowkey. - *

- * Steps include: - * 1. Restoring data for given data range and other params - * 2. Searching provided rowkey in SSTables and writing search result to JSON - * 3. Delete all the files under Keyspace Directory. - * Deletion is done for efficient space usage, so that same node can be reused for - * subsequent runs. - *

- *

- * Similar to Restore call and few additional params. - *

- * daterange : Can not be Null or Default. Comma separated Start and End date eg. 201311250000,201311260000 - * rowkey : rowkey to search (In Hex format) - * ks : keyspace of mentioned rowkey - * cf : column family of mentioned rowkey - * fileExtension : Part of SSTable Data file names - * eg. if file name = KS1-CF1-hf-100-Data.db - * then fileExtension = KS1-CF1-hf * - * @return Creates JSON file based on the passed date at hardcoded dir location : /tmp/priam_sstables - * If rowkey is not found in the SSTable, JSON file will be empty. + *

Similar to Restore call and few additional params. + * + *

daterange : Can not be Null or Default. Comma separated Start and End date eg. + * 201311250000,201311260000 rowkey : rowkey to search (In Hex format) ks : keyspace of + * mentioned rowkey cf : column family of mentioned rowkey fileExtension : Part of SSTable Data + * file names eg. if file name = KS1-CF1-hf-100-Data.db then fileExtension = KS1-CF1-hf + * + * @return Creates JSON file based on the passed date at hardcoded dir location : + * /tmp/priam_sstables If rowkey is not found in the SSTable, JSON file will be empty. */ @GET @Path("/life_of_crow") @@ -339,19 +366,21 @@ public Response restore_verify_key( @QueryParam(REST_LOCR_ROWKEY) String rowkey, @QueryParam(REST_LOCR_KEYSPACE) String ks, @QueryParam(REST_LOCR_COLUMNFAMILY) String cf, - @QueryParam(REST_LOCR_FILEEXTENSION) String fileExtension) throws Exception { + @QueryParam(REST_LOCR_FILEEXTENSION) String fileExtension) + throws Exception { Date startTime; Date endTime; - //Creating Dir for Json storage + // Creating Dir for Json storage SystemUtils.createDirs(SSTABLE2JSON_DIR_LOCATION); String JSON_FILE_PATH; try { - if (StringUtils.isBlank(daterange) - || daterange.equalsIgnoreCase("default")) { - return Response.ok("\n[\"daterange can't be blank or default.eg.201311250000,201311260000\"]\n", MediaType.APPLICATION_JSON) + if (StringUtils.isBlank(daterange) || daterange.equalsIgnoreCase("default")) { + return Response.ok( + "\n[\"daterange can't be blank or default.eg.201311250000,201311260000\"]\n", + MediaType.APPLICATION_JSON) .build(); } @@ -365,20 +394,18 @@ public Response restore_verify_key( config.setRestorePrefix(restorePrefix); } - restore(token, region, startTime, endTime, keyspaces); // Since this call is probably never called in parallel, config is // multi-thread safe to be edited config.setRestorePrefix(origRestorePrefix); - while (!CassandraMonitor.hasCassadraStarted()) - Thread.sleep(1000L); + while (!CassandraMonitor.hasCassadraStarted()) Thread.sleep(1000L); // initialize json file name JSON_FILE_PATH = daterange.split(",")[0].substring(0, 8) + ".json"; - //Convert SSTable2Json and search for given rowkey + // Convert SSTable2Json and search for given rowkey checkSSTablesForKey(rowkey, ks, cf, fileExtension, JSON_FILE_PATH); } catch (Exception e) { @@ -387,33 +414,41 @@ public Response restore_verify_key( removeAllDataFiles(ks); } - return Response.ok(REST_SUCCESS, MediaType.APPLICATION_JSON) - .build(); + return Response.ok(REST_SUCCESS, MediaType.APPLICATION_JSON).build(); } /** * Restore with the specified start and end time. * - * @param token Overrides the current token with this one, if specified - * @param region Override the region for searching backup + * @param token Overrides the current token with this one, if specified + * @param region Override the region for searching backup * @param startTime Start time - * @param endTime End time upto which the restore should fetch data + * @param endTime End time upto which the restore should fetch data * @param keyspaces Comma seperated list of keyspaces to restore * @throws Exception if restore is not successful */ - private void restore(String token, String region, Date startTime, Date endTime, String keyspaces) throws Exception { + private void restore( + String token, String region, Date startTime, Date endTime, String keyspaces) + throws Exception { String origRegion = config.getDC(); String origToken = priamServer.getId().getInstance().getToken(); - if (StringUtils.isNotBlank(token)) - priamServer.getId().getInstance().setToken(token); + if (StringUtils.isNotBlank(token)) priamServer.getId().getInstance().setToken(token); if (config.isRestoreClosestToken()) - priamServer.getId().getInstance().setToken(closestToken(priamServer.getId().getInstance().getToken(), config.getDC())); + priamServer + .getId() + .getInstance() + .setToken( + closestToken( + priamServer.getId().getInstance().getToken(), config.getDC())); if (StringUtils.isNotBlank(region)) { config.setDC(region); logger.info("Restoring from region {}", region); - priamServer.getId().getInstance().setToken(closestToken(priamServer.getId().getInstance().getToken(), region)); + priamServer + .getId() + .getInstance() + .setToken(closestToken(priamServer.getId().getInstance().getToken(), region)); logger.info("Restore will use token {}", priamServer.getId().getInstance().getToken()); } @@ -429,30 +464,28 @@ private void restore(String token, String region, Date startTime, Date endTime, cassProcess.start(true); } - /** - * Find closest token in the specified region - */ + /** Find closest token in the specified region */ private String closestToken(String token, String region) { List plist = factory.getAllIds(config.getAppName()); List tokenList = Lists.newArrayList(); for (PriamInstance ins : plist) { - if (ins.getDC().equalsIgnoreCase(region)) - tokenList.add(new BigInteger(ins.getToken())); + if (ins.getDC().equalsIgnoreCase(region)) tokenList.add(new BigInteger(ins.getToken())); } return tokenManager.findClosestToken(new BigInteger(token), tokenList).toString(); } /* - * A list of files for requested filter. Currently, the only supported filter is META, all others will be ignore. + * A list of files for requested filter. Currently, the only supported filter is META, all others will be ignore. * For filter of META, ONLY the daily snapshot meta file (meta.json) are accounted for, not the incremental meta file. * In addition, we do ONLY list the name of the meta data file, not the list of data files within it. - * + * * @param handle to the json response * @param a list of all files (data (*.db), and meta data file (*.json)) from S3 for requested dates. * @param backup meta data file filter. Currently, the only supported filter is META, all others will be ignore. * @return a list of files in Json format. */ - private JSONObject constructJsonResponse(JSONObject object, Iterator it, String filter) throws Exception { + private JSONObject constructJsonResponse( + JSONObject object, Iterator it, String filter) throws Exception { int fileCnt = 0; filter = filter.contains("?") ? filter.substring(0, filter.indexOf("?")) : filter; @@ -460,8 +493,7 @@ private JSONObject constructJsonResponse(JSONObject object, Iterator callable = p::waitFor; @@ -516,10 +547,8 @@ public void checkSSTablesForKey(String rowkey, String keyspace, String cf, Strin try { Future future = exeService.submit(callable); int returnVal = future.get(TIMEOUT_PERIOD, TimeUnit.MINUTES); - if (returnVal == 0) - logger.info("Finished SSTable2Json conversion and search."); - else - logger.error("Error occurred during SSTable2Json conversion and search."); + if (returnVal == 0) logger.info("Finished SSTable2Json conversion and search."); + else logger.error("Error occurred during SSTable2Json conversion and search."); } catch (TimeoutException e) { logger.error(ExceptionUtils.getStackTrace(e)); throw e; @@ -533,10 +562,22 @@ public void checkSSTablesForKey(String rowkey, String keyspace, String cf, Strin } } - public String formulateCommandToRun(String rowkey, String keyspace, String cf, String fileExtension, String jsonFilePath) { + public String formulateCommandToRun( + String rowkey, String keyspace, String cf, String fileExtension, String jsonFilePath) { StringBuffer sbuff = new StringBuffer(); - sbuff.append("for i in $(ls ").append(config.getDataFileLocation()).append(File.separator).append(keyspace).append(File.separator).append(cf).append(File.separator).append(fileExtension).append("-*-Data.db); do ").append(config.getCassHome()).append(SSTABLE2JSON_COMMAND_FROM_CASSHOME).append(" $i -k "); + sbuff.append("for i in $(ls ") + .append(config.getDataFileLocation()) + .append(File.separator) + .append(keyspace) + .append(File.separator) + .append(cf) + .append(File.separator) + .append(fileExtension) + .append("-*-Data.db); do ") + .append(config.getCassHome()) + .append(SSTABLE2JSON_COMMAND_FROM_CASSHOME) + .append(" $i -k "); sbuff.append(rowkey); sbuff.append(" | grep "); sbuff.append(rowkey); @@ -544,7 +585,10 @@ public String formulateCommandToRun(String rowkey, String keyspace, String cf, S sbuff.append(SSTABLE2JSON_DIR_LOCATION).append(File.separator).append(jsonFilePath); sbuff.append(" ; done"); - logger.info("SSTable2JSON location <" + SSTABLE2JSON_DIR_LOCATION + "{}{}>", File.separator, jsonFilePath); + logger.info( + "SSTable2JSON location <" + SSTABLE2JSON_DIR_LOCATION + "{}{}>", + File.separator, + jsonFilePath); logger.info("Running Command = {}", sbuff); return sbuff.toString(); } @@ -555,5 +599,4 @@ public void removeAllDataFiles(String ks) throws Exception { SystemUtils.cleanupDir(cleanupDirPath, null); logger.info("*** Done cleaning all the files inside <{}>", cleanupDirPath); } - -} \ No newline at end of file +} diff --git a/priam/src/main/java/com/netflix/priam/resources/CassandraAdmin.java b/priam/src/main/java/com/netflix/priam/resources/CassandraAdmin.java index 4b7dc0b99..2d364a24e 100644 --- a/priam/src/main/java/com/netflix/priam/resources/CassandraAdmin.java +++ b/priam/src/main/java/com/netflix/priam/resources/CassandraAdmin.java @@ -18,13 +18,20 @@ import com.google.common.collect.Lists; import com.google.inject.Inject; -import com.netflix.priam.defaultimpl.ICassandraProcess; -import com.netflix.priam.config.IConfiguration; import com.netflix.priam.cluster.management.Compaction; import com.netflix.priam.cluster.management.Flush; import com.netflix.priam.compress.SnappyCompression; +import com.netflix.priam.config.IConfiguration; +import com.netflix.priam.defaultimpl.ICassandraProcess; import com.netflix.priam.utils.JMXConnectionException; import com.netflix.priam.utils.JMXNodeTool; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.util.concurrent.ExecutionException; +import javax.ws.rs.*; +import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response; import org.apache.cassandra.exceptions.ConfigurationException; import org.apache.commons.lang3.StringUtils; import org.codehaus.jettison.json.JSONArray; @@ -33,17 +40,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import javax.ws.rs.*; -import javax.ws.rs.core.MediaType; -import javax.ws.rs.core.Response; -import java.io.FileInputStream; -import java.io.FileOutputStream; -import java.io.IOException; -import java.util.concurrent.ExecutionException; - -/** - * Do general operations. Start/Stop and some JMX node tool commands - */ +/** Do general operations. Start/Stop and some JMX node tool commands */ @SuppressWarnings("deprecation") @Path("/v1/cassadmin") @Produces(MediaType.APPLICATION_JSON) @@ -59,7 +56,11 @@ public class CassandraAdmin { private final Compaction compaction; @Inject - public CassandraAdmin(IConfiguration config, ICassandraProcess cassProcess, Flush flush, Compaction compaction) { + public CassandraAdmin( + IConfiguration config, + ICassandraProcess cassProcess, + Flush flush, + Compaction compaction) { this.config = config; this.cassProcess = cassProcess; this.flush = flush; @@ -75,14 +76,16 @@ public Response cassStart() throws IOException, InterruptedException, JSONExcept @GET @Path("/stop") - public Response cassStop(@DefaultValue("false") @QueryParam("force") boolean force) throws IOException, InterruptedException, JSONException { + public Response cassStop(@DefaultValue("false") @QueryParam("force") boolean force) + throws IOException, InterruptedException, JSONException { cassProcess.stop(force); return Response.ok(REST_SUCCESS, MediaType.APPLICATION_JSON).build(); } @GET @Path("/refresh") - public Response cassRefresh(@QueryParam(REST_HEADER_KEYSPACES) String keyspaces) throws IOException, ExecutionException, InterruptedException, JSONException { + public Response cassRefresh(@QueryParam(REST_HEADER_KEYSPACES) String keyspaces) + throws IOException, ExecutionException, InterruptedException, JSONException { logger.debug("node tool refresh is being called"); if (StringUtils.isBlank(keyspaces)) return Response.status(400).entity("Missing keyspace in request").build(); @@ -91,9 +94,9 @@ public Response cassRefresh(@QueryParam(REST_HEADER_KEYSPACES) String keyspaces) try { nodeTool = JMXNodeTool.instance(config); } catch (JMXConnectionException e) { - logger.error("Exception in fetching c* jmx tool . Msgl: {}", e.getLocalizedMessage(), e); - return Response.status(503).entity("JMXConnectionException") - .build(); + logger.error( + "Exception in fetching c* jmx tool . Msgl: {}", e.getLocalizedMessage(), e); + return Response.status(503).entity("JMXConnectionException").build(); } nodeTool.refresh(Lists.newArrayList(keyspaces.split(","))); return Response.ok(REST_SUCCESS, MediaType.APPLICATION_JSON).build(); @@ -106,9 +109,9 @@ public Response cassInfo() throws IOException, InterruptedException, JSONExcepti try { nodeTool = JMXNodeTool.instance(config); } catch (JMXConnectionException e) { - logger.error("Exception in fetching c* jmx tool . Msgl: {}", e.getLocalizedMessage(), e); - return Response.status(503).entity("JMXConnectionException") - .build(); + logger.error( + "Exception in fetching c* jmx tool . Msgl: {}", e.getLocalizedMessage(), e); + return Response.status(503).entity("JMXConnectionException").build(); } logger.debug("node tool info being called"); return Response.ok(nodeTool.info(), MediaType.APPLICATION_JSON).build(); @@ -121,9 +124,9 @@ public Response cassPartitioner() throws IOException, InterruptedException, JSON try { nodeTool = JMXNodeTool.instance(config); } catch (JMXConnectionException e) { - logger.error("Exception in fetching c* jmx tool . Msgl: {}", e.getLocalizedMessage(), e); - return Response.status(503).entity("JMXConnectionException") - .build(); + logger.error( + "Exception in fetching c* jmx tool . Msgl: {}", e.getLocalizedMessage(), e); + return Response.status(503).entity("JMXConnectionException").build(); } logger.debug("node tool getPartitioner being called"); return Response.ok(nodeTool.getPartitioner(), MediaType.APPLICATION_JSON).build(); @@ -131,14 +134,15 @@ public Response cassPartitioner() throws IOException, InterruptedException, JSON @GET @Path("/ring/{id}") - public Response cassRing(@PathParam("id") String keyspace) throws IOException, InterruptedException, JSONException { + public Response cassRing(@PathParam("id") String keyspace) + throws IOException, InterruptedException, JSONException { JMXNodeTool nodeTool; try { nodeTool = JMXNodeTool.instance(config); } catch (JMXConnectionException e) { - logger.error("Exception in fetching c* jmx tool . Msgl: {}", e.getLocalizedMessage(), e); - return Response.status(503).entity("JMXConnectionException") - .build(); + logger.error( + "Exception in fetching c* jmx tool . Msgl: {}", e.getLocalizedMessage(), e); + return Response.status(503).entity("JMXConnectionException").build(); } logger.debug("node tool ring being called"); return Response.ok(nodeTool.ring(keyspace), MediaType.APPLICATION_JSON).build(); @@ -158,11 +162,9 @@ public Response cassFlush() throws IOException, InterruptedException, ExecutionE rootObj.put("status", "ERRROR"); rootObj.put("desc", e.getLocalizedMessage()); } catch (Exception e1) { - return Response.status(503).entity("FlushError") - .build(); + return Response.status(503).entity("FlushError").build(); } - return Response.status(503).entity(rootObj) - .build(); + return Response.status(503).entity(rootObj).build(); } } @@ -180,11 +182,9 @@ public Response cassCompact() throws IOException, ExecutionException, Interrupte rootObj.put("status", "ERRROR"); rootObj.put("desc", e.getLocalizedMessage()); } catch (Exception e1) { - return Response.status(503).entity("CompactionError") - .build(); + return Response.status(503).entity("CompactionError").build(); } - return Response.status(503).entity(rootObj) - .build(); + return Response.status(503).entity(rootObj).build(); } } @@ -195,9 +195,9 @@ public Response cassCleanup() throws IOException, ExecutionException, Interrupte try { nodeTool = JMXNodeTool.instance(config); } catch (JMXConnectionException e) { - logger.error("Exception in fetching c* jmx tool . Msgl: {}", e.getLocalizedMessage(), e); - return Response.status(503).entity("JMXConnectionException") - .build(); + logger.error( + "Exception in fetching c* jmx tool . Msgl: {}", e.getLocalizedMessage(), e); + return Response.status(503).entity("JMXConnectionException").build(); } logger.debug("node tool cleanup being called"); nodeTool.cleanup(); @@ -206,14 +206,18 @@ public Response cassCleanup() throws IOException, ExecutionException, Interrupte @GET @Path("/repair") - public Response cassRepair(@QueryParam("sequential") boolean isSequential, @QueryParam("localDC") boolean localDCOnly, @DefaultValue("false") @QueryParam("primaryRange") boolean primaryRange) throws IOException, ExecutionException, InterruptedException { + public Response cassRepair( + @QueryParam("sequential") boolean isSequential, + @QueryParam("localDC") boolean localDCOnly, + @DefaultValue("false") @QueryParam("primaryRange") boolean primaryRange) + throws IOException, ExecutionException, InterruptedException { JMXNodeTool nodeTool; try { nodeTool = JMXNodeTool.instance(config); } catch (JMXConnectionException e) { - logger.error("Exception in fetching c* jmx tool . Msgl: {}", e.getLocalizedMessage(), e); - return Response.status(503).entity("JMXConnectionException") - .build(); + logger.error( + "Exception in fetching c* jmx tool . Msgl: {}", e.getLocalizedMessage(), e); + return Response.status(503).entity("JMXConnectionException").build(); } logger.debug("node tool repair being called"); nodeTool.repair(isSequential, localDCOnly, primaryRange); @@ -227,11 +231,14 @@ public Response version() throws IOException, ExecutionException, InterruptedExc try { nodeTool = JMXNodeTool.instance(config); } catch (JMXConnectionException e) { - logger.error("Exception in fetching c* jmx tool . Msgl: {}", e.getLocalizedMessage(), e); - return Response.status(503).entity("JMXConnectionException") - .build(); + logger.error( + "Exception in fetching c* jmx tool . Msgl: {}", e.getLocalizedMessage(), e); + return Response.status(503).entity("JMXConnectionException").build(); } - return Response.ok(new JSONArray().put(nodeTool.getReleaseVersion()), MediaType.APPLICATION_JSON).build(); + return Response.ok( + new JSONArray().put(nodeTool.getReleaseVersion()), + MediaType.APPLICATION_JSON) + .build(); } @GET @@ -241,9 +248,9 @@ public Response disablegossip() throws IOException, ExecutionException, Interrup try { nodeTool = JMXNodeTool.instance(config); } catch (JMXConnectionException e) { - logger.error("Exception in fetching c* jmx tool . Msgl: {}", e.getLocalizedMessage(), e); - return Response.status(503).entity("JMXConnectionException") - .build(); + logger.error( + "Exception in fetching c* jmx tool . Msgl: {}", e.getLocalizedMessage(), e); + return Response.status(503).entity("JMXConnectionException").build(); } nodeTool.stopGossiping(); return Response.ok(REST_SUCCESS, MediaType.APPLICATION_JSON).build(); @@ -256,9 +263,9 @@ public Response enablegossip() throws IOException, ExecutionException, Interrupt try { nodeTool = JMXNodeTool.instance(config); } catch (JMXConnectionException e) { - logger.error("Exception in fetching c* jmx tool . Msgl: {}", e.getLocalizedMessage(), e); - return Response.status(503).entity("JMXConnectionException") - .build(); + logger.error( + "Exception in fetching c* jmx tool . Msgl: {}", e.getLocalizedMessage(), e); + return Response.status(503).entity("JMXConnectionException").build(); } nodeTool.startGossiping(); return Response.ok(REST_SUCCESS, MediaType.APPLICATION_JSON).build(); @@ -271,9 +278,9 @@ public Response disablethrift() throws IOException, ExecutionException, Interrup try { nodeTool = JMXNodeTool.instance(config); } catch (JMXConnectionException e) { - logger.error("Exception in fetching c* jmx tool . Msgl: {}", e.getLocalizedMessage(), e); - return Response.status(503).entity("JMXConnectionException") - .build(); + logger.error( + "Exception in fetching c* jmx tool . Msgl: {}", e.getLocalizedMessage(), e); + return Response.status(503).entity("JMXConnectionException").build(); } nodeTool.stopThriftServer(); return Response.ok(REST_SUCCESS, MediaType.APPLICATION_JSON).build(); @@ -286,9 +293,9 @@ public Response enablethrift() throws IOException, ExecutionException, Interrupt try { nodeTool = JMXNodeTool.instance(config); } catch (JMXConnectionException e) { - logger.error("Exception in fetching c* jmx tool . Msgl: {}", e.getLocalizedMessage(), e); - return Response.status(503).entity("JMXConnectionException") - .build(); + logger.error( + "Exception in fetching c* jmx tool . Msgl: {}", e.getLocalizedMessage(), e); + return Response.status(503).entity("JMXConnectionException").build(); } nodeTool.startThriftServer(); return Response.ok(REST_SUCCESS, MediaType.APPLICATION_JSON).build(); @@ -296,34 +303,43 @@ public Response enablethrift() throws IOException, ExecutionException, Interrupt @GET @Path("/statusthrift") - public Response statusthrift() throws IOException, ExecutionException, InterruptedException, JSONException { + public Response statusthrift() + throws IOException, ExecutionException, InterruptedException, JSONException { JMXNodeTool nodeTool; try { nodeTool = JMXNodeTool.instance(config); } catch (JMXConnectionException e) { - logger.error("Exception in fetching c* jmx tool . Msgl: {}", e.getLocalizedMessage(), e); - return Response.status(503).entity("JMXConnectionException") - .build(); + logger.error( + "Exception in fetching c* jmx tool . Msgl: {}", e.getLocalizedMessage(), e); + return Response.status(503).entity("JMXConnectionException").build(); } - return Response.ok(new JSONObject().put("status", (nodeTool.isThriftServerRunning() ? "running" : "not running")), MediaType.APPLICATION_JSON).build(); + return Response.ok( + new JSONObject() + .put( + "status", + (nodeTool.isThriftServerRunning() + ? "running" + : "not running")), + MediaType.APPLICATION_JSON) + .build(); } @GET @Path("/gossipinfo") - public Response gossipinfo() throws IOException, ExecutionException, InterruptedException, JSONException { + public Response gossipinfo() + throws IOException, ExecutionException, InterruptedException, JSONException { JMXNodeTool nodeTool; try { nodeTool = JMXNodeTool.instance(config); } catch (JMXConnectionException e) { - logger.error("Exception in fetching c* jmx tool . Msgl: {}", e.getLocalizedMessage(), e); - return Response.status(503).entity("JMXConnectionException") - .build(); + logger.error( + "Exception in fetching c* jmx tool . Msgl: {}", e.getLocalizedMessage(), e); + return Response.status(503).entity("JMXConnectionException").build(); } JSONObject rootObj = parseGossipInfo(nodeTool.getGossipInfo()); return Response.ok(rootObj, MediaType.APPLICATION_JSON).build(); } - // helper method for parsing, to be tested easily private static JSONObject parseGossipInfo(String gossipinfo) throws JSONException { String[] ginfo = gossipinfo.split("\n"); @@ -350,29 +366,26 @@ private static JSONObject parseGossipInfo(String gossipinfo) throws JSONExceptio } } } - if (StringUtils.isNotBlank(key)) - rootObj.put(key, obj); + if (StringUtils.isNotBlank(key)) rootObj.put(key, obj); return rootObj; } - @GET @Path("/move") - public Response moveToken(@QueryParam(REST_HEADER_TOKEN) String newToken) throws IOException, ExecutionException, InterruptedException, ConfigurationException { + public Response moveToken(@QueryParam(REST_HEADER_TOKEN) String newToken) + throws IOException, ExecutionException, InterruptedException, ConfigurationException { JMXNodeTool nodeTool; try { nodeTool = JMXNodeTool.instance(config); } catch (JMXConnectionException e) { - logger.error("Exception in fetching c* jmx tool . Msgl: {}", e.getLocalizedMessage(), e); - return Response.status(503).entity("JMXConnectionException") - .build(); + logger.error( + "Exception in fetching c* jmx tool . Msgl: {}", e.getLocalizedMessage(), e); + return Response.status(503).entity("JMXConnectionException").build(); } nodeTool.move(newToken); return Response.ok(REST_SUCCESS, MediaType.APPLICATION_JSON).build(); } - - @GET @Path("/drain") public Response cassDrain() throws IOException, ExecutionException, InterruptedException { @@ -380,9 +393,9 @@ public Response cassDrain() throws IOException, ExecutionException, InterruptedE try { nodeTool = JMXNodeTool.instance(config); } catch (JMXConnectionException e) { - logger.error("Exception in fetching c* jmx tool . Msgl: {}", e.getLocalizedMessage(), e); - return Response.status(503).entity("JMXConnectionException") - .build(); + logger.error( + "Exception in fetching c* jmx tool . Msgl: {}", e.getLocalizedMessage(), e); + return Response.status(503).entity("JMXConnectionException").build(); } logger.debug("node tool drain being called"); nodeTool.drain(); @@ -396,7 +409,8 @@ public Response cassDrain() throws IOException, ExecutionException, InterruptedE */ @GET @Path("/decompress") - public Response decompress(@QueryParam("in") String in, @QueryParam("out") String out) throws Exception { + public Response decompress(@QueryParam("in") String in, @QueryParam("out") String out) + throws Exception { SnappyCompression compress = new SnappyCompression(); compress.decompressAndClose(new FileInputStream(in), new FileOutputStream(out)); JSONObject object = new JSONObject(); @@ -404,5 +418,4 @@ public Response decompress(@QueryParam("in") String in, @QueryParam("out") Strin object.put("Output decompress file", out); return Response.ok(object.toString(), MediaType.APPLICATION_JSON).build(); } - } diff --git a/priam/src/main/java/com/netflix/priam/resources/CassandraConfig.java b/priam/src/main/java/com/netflix/priam/resources/CassandraConfig.java index 292269d15..9634bbd6e 100644 --- a/priam/src/main/java/com/netflix/priam/resources/CassandraConfig.java +++ b/priam/src/main/java/com/netflix/priam/resources/CassandraConfig.java @@ -19,11 +19,10 @@ import com.google.inject.Inject; import com.netflix.priam.PriamServer; import com.netflix.priam.identity.DoubleRing; -import org.apache.commons.lang3.StringUtils; -import org.json.simple.JSONValue; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - +import java.io.IOException; +import java.util.HashMap; +import java.util.List; +import java.util.Map; import javax.ws.rs.GET; import javax.ws.rs.POST; import javax.ws.rs.Path; @@ -31,14 +30,13 @@ import javax.ws.rs.QueryParam; import javax.ws.rs.core.MediaType; import javax.ws.rs.core.Response; -import java.io.IOException; -import java.util.HashMap; -import java.util.List; -import java.util.Map; +import org.apache.commons.lang3.StringUtils; +import org.json.simple.JSONValue; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** - * This servlet will provide the configuration API service as and when Cassandra - * requests for it. + * This servlet will provide the configuration API service as and when Cassandra requests for it. */ @Path("/v1/cassconfig") @Produces(MediaType.TEXT_PLAIN) @@ -58,8 +56,7 @@ public CassandraConfig(PriamServer server, DoubleRing doubleRing) { public Response getSeeds() { try { final List seeds = priamServer.getId().getSeeds(); - if (!seeds.isEmpty()) - return Response.ok(StringUtils.join(seeds, ',')).build(); + if (!seeds.isEmpty()) return Response.ok(StringUtils.join(seeds, ',')).build(); logger.error("Cannot find the Seeds"); } catch (Exception e) { logger.error("Error while executing get_seeds", e); @@ -99,7 +96,6 @@ public Response isReplaceToken() { } } - @GET @Path("/get_replaced_ip") public Response getReplacedIp() { @@ -140,7 +136,6 @@ public Response getExtraEnvParams() { } } - @GET @Path("/double_ring") public Response doubleRing() throws IOException, ClassNotFoundException { @@ -155,4 +150,4 @@ public Response doubleRing() throws IOException, ClassNotFoundException { } return Response.status(200).build(); } -} \ No newline at end of file +} diff --git a/priam/src/main/java/com/netflix/priam/resources/PriamConfig.java b/priam/src/main/java/com/netflix/priam/resources/PriamConfig.java index 103a54041..49f8e683f 100644 --- a/priam/src/main/java/com/netflix/priam/resources/PriamConfig.java +++ b/priam/src/main/java/com/netflix/priam/resources/PriamConfig.java @@ -16,6 +16,8 @@ */ package com.netflix.priam.resources; +import com.google.inject.Inject; +import com.netflix.priam.PriamServer; import java.util.HashMap; import java.util.Map; import javax.ws.rs.GET; @@ -25,20 +27,15 @@ import javax.ws.rs.QueryParam; import javax.ws.rs.core.MediaType; import javax.ws.rs.core.Response; - import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.google.inject.Inject; -import com.netflix.priam.PriamServer; - /** * This servlet will provide the configuration API service for use by external scripts and tooling */ @Path("/v1/config") @Produces(MediaType.APPLICATION_JSON) -public class PriamConfig -{ +public class PriamConfig { private static final Logger logger = LoggerFactory.getLogger(PriamConfig.class); private final PriamServer priamServer; @@ -50,7 +47,8 @@ public PriamConfig(PriamServer server) { private Response doGetPriamConfig(String group, String name) { try { final Map result = new HashMap<>(); - final Map value = priamServer.getConfiguration().getStructuredConfiguration(group); + final Map value = + priamServer.getConfiguration().getStructuredConfiguration(group); if (name != null && value.containsKey(name)) { result.put(name, value.get(name)); return Response.ok(result, MediaType.APPLICATION_JSON).build(); @@ -68,22 +66,24 @@ private Response doGetPriamConfig(String group, String name) { } } - @GET @Path("/structured/{group}") - public Response getPriamConfig(@PathParam("group") String group, @PathParam("name") String name) { + public Response getPriamConfig( + @PathParam("group") String group, @PathParam("name") String name) { return doGetPriamConfig(group, null); } @GET @Path("/structured/{group}/{name}") - public Response getPriamConfigByName(@PathParam("group") String group, @PathParam("name") String name) { + public Response getPriamConfigByName( + @PathParam("group") String group, @PathParam("name") String name) { return doGetPriamConfig(group, name); } @GET @Path("/unstructured/{name}") - public Response getProperty(@PathParam("name") String name, @QueryParam("default") String defaultValue) { + public Response getProperty( + @PathParam("name") String name, @QueryParam("default") String defaultValue) { Map result = new HashMap<>(); try { String value = priamServer.getConfiguration().getProperty(name, defaultValue); @@ -100,4 +100,4 @@ public Response getProperty(@PathParam("name") String name, @QueryParam("default return Response.serverError().build(); } } -} \ No newline at end of file +} diff --git a/priam/src/main/java/com/netflix/priam/resources/PriamInstanceResource.java b/priam/src/main/java/com/netflix/priam/resources/PriamInstanceResource.java index 3085e697e..828f4294a 100644 --- a/priam/src/main/java/com/netflix/priam/resources/PriamInstanceResource.java +++ b/priam/src/main/java/com/netflix/priam/resources/PriamInstanceResource.java @@ -20,19 +20,16 @@ import com.netflix.priam.config.IConfiguration; import com.netflix.priam.identity.IPriamInstanceFactory; import com.netflix.priam.identity.PriamInstance; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - +import java.net.URI; +import java.util.List; import javax.ws.rs.*; import javax.ws.rs.core.MediaType; import javax.ws.rs.core.Response; import javax.ws.rs.core.UriBuilder; -import java.net.URI; -import java.util.List; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; -/** - * Resource for manipulating priam instances. - */ +/** Resource for manipulating priam instances. */ @Path("/v1/instances") @Produces(MediaType.TEXT_PLAIN) public class PriamInstanceResource { @@ -42,7 +39,8 @@ public class PriamInstanceResource { private final IPriamInstanceFactory factory; @Inject - //Note: do not parameterized the generic type variable to an implementation as it confuses Guice in the binding. + // Note: do not parameterized the generic type variable to an implementation as it confuses + // Guice in the binding. public PriamInstanceResource(IConfiguration config, IPriamInstanceFactory factory) { this.config = config; this.factory = factory; @@ -50,6 +48,7 @@ public PriamInstanceResource(IConfiguration config, IPriamInstanceFactory factor /** * Get the list of all priam instances + * * @return the list of all priam instances */ @GET @@ -84,12 +83,23 @@ public String getInstance(@PathParam("id") int id) { */ @POST public Response createInstance( - @QueryParam("id") int id, @QueryParam("instanceID") String instanceID, - @QueryParam("hostname") String hostname, @QueryParam("ip") String ip, - @QueryParam("rack") String rack, @QueryParam("token") String token) { - log.info("Creating instance [id={}, instanceId={}, hostname={}, ip={}, rack={}, token={}", - id, instanceID, hostname, ip, rack, token); - PriamInstance instance = factory.create(config.getAppName(), id, instanceID, hostname, ip, rack, null, token); + @QueryParam("id") int id, + @QueryParam("instanceID") String instanceID, + @QueryParam("hostname") String hostname, + @QueryParam("ip") String ip, + @QueryParam("rack") String rack, + @QueryParam("token") String token) { + log.info( + "Creating instance [id={}, instanceId={}, hostname={}, ip={}, rack={}, token={}", + id, + instanceID, + hostname, + ip, + rack, + token); + PriamInstance instance = + factory.create( + config.getAppName(), id, instanceID, hostname, ip, rack, null, token); URI uri = UriBuilder.fromPath("/{id}").build(instance.getId()); return Response.created(uri).build(); } @@ -109,8 +119,8 @@ public Response deleteInstance(@PathParam("id") int id) { } /** - * Returns the PriamInstance with the given {@code id}, or - * throws a WebApplicationException(400) if none found. + * Returns the PriamInstance with the given {@code id}, or throws a WebApplicationException(400) + * if none found. * * @param id the node id * @return PriamInstance with the given {@code id} @@ -124,6 +134,7 @@ private PriamInstance getByIdIfFound(int id) { } private static WebApplicationException notFound(String message) { - return new WebApplicationException(Response.status(Response.Status.NOT_FOUND).entity(message).build()); + return new WebApplicationException( + Response.status(Response.Status.NOT_FOUND).entity(message).build()); } } diff --git a/priam/src/main/java/com/netflix/priam/resources/RestoreServlet.java b/priam/src/main/java/com/netflix/priam/resources/RestoreServlet.java index 403638438..545211b12 100644 --- a/priam/src/main/java/com/netflix/priam/resources/RestoreServlet.java +++ b/priam/src/main/java/com/netflix/priam/resources/RestoreServlet.java @@ -1,16 +1,14 @@ /** * Copyright 2017 Netflix, Inc. - *

- * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and + * + *

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file + * except in compliance with the License. You may obtain a copy of the License at + * + *

http://www.apache.org/licenses/LICENSE-2.0 + * + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing permissions and * limitations under the License. */ package com.netflix.priam.resources; @@ -18,30 +16,29 @@ import com.google.common.collect.Lists; import com.google.inject.Inject; import com.google.inject.Provider; -import com.netflix.priam.defaultimpl.ICassandraProcess; -import com.netflix.priam.config.IConfiguration; import com.netflix.priam.PriamServer; import com.netflix.priam.backup.AbstractBackupPath; +import com.netflix.priam.config.IConfiguration; +import com.netflix.priam.defaultimpl.ICassandraProcess; import com.netflix.priam.health.InstanceState; import com.netflix.priam.identity.IPriamInstanceFactory; import com.netflix.priam.identity.PriamInstance; import com.netflix.priam.restore.Restore; import com.netflix.priam.tuner.ICassandraTuner; import com.netflix.priam.utils.ITokenManager; -import org.apache.commons.lang3.StringUtils; -import org.joda.time.DateTime; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - +import java.math.BigInteger; +import java.util.Date; +import java.util.List; import javax.ws.rs.GET; import javax.ws.rs.Path; import javax.ws.rs.Produces; import javax.ws.rs.QueryParam; import javax.ws.rs.core.MediaType; import javax.ws.rs.core.Response; -import java.math.BigInteger; -import java.util.Date; -import java.util.List; +import org.apache.commons.lang3.StringUtils; +import org.joda.time.DateTime; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; @Path("/v1") @Produces(MediaType.APPLICATION_JSON) @@ -66,8 +63,16 @@ public class RestoreServlet { private final InstanceState instanceState; @Inject - public RestoreServlet(IConfiguration config, Restore restoreObj, Provider pathProvider, PriamServer priamServer - , IPriamInstanceFactory factory, ICassandraTuner tuner, ICassandraProcess cassProcess, ITokenManager tokenManager, InstanceState instanceState) { + public RestoreServlet( + IConfiguration config, + Restore restoreObj, + Provider pathProvider, + PriamServer priamServer, + IPriamInstanceFactory factory, + ICassandraTuner tuner, + ICassandraProcess cassProcess, + ITokenManager tokenManager, + InstanceState instanceState) { this.config = config; this.restoreObj = restoreObj; this.pathProvider = pathProvider; @@ -79,7 +84,6 @@ public RestoreServlet(IConfiguration config, Restore restoreObj, Provider plist = factory.getAllIds(config.getAppName()); List tokenList = Lists.newArrayList(); for (PriamInstance ins : plist) { - if (ins.getDC().equalsIgnoreCase(region)) - tokenList.add(new BigInteger(ins.getToken())); + if (ins.getDC().equalsIgnoreCase(region)) tokenList.add(new BigInteger(ins.getToken())); } return tokenManager.findClosestToken(new BigInteger(token), tokenList).toString(); } - -} \ No newline at end of file +} diff --git a/priam/src/main/java/com/netflix/priam/resources/SecurityGroupAdmin.java b/priam/src/main/java/com/netflix/priam/resources/SecurityGroupAdmin.java index 2dd494115..6b1f10a7b 100644 --- a/priam/src/main/java/com/netflix/priam/resources/SecurityGroupAdmin.java +++ b/priam/src/main/java/com/netflix/priam/resources/SecurityGroupAdmin.java @@ -1,33 +1,30 @@ /** * Copyright 2017 Netflix, Inc. - *

- * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and + * + *

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file + * except in compliance with the License. You may obtain a copy of the License at + * + *

http://www.apache.org/licenses/LICENSE-2.0 + * + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing permissions and * limitations under the License. */ package com.netflix.priam.resources; import com.google.inject.Inject; import com.netflix.priam.identity.IMembership; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - +import java.util.Collections; import javax.ws.rs.*; import javax.ws.rs.core.MediaType; import javax.ws.rs.core.Response; -import java.util.Collections; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** - * This http endpoint allows direct updates (adding/removing) (CIDR) IP addresses and port - * ranges to the security group for this app. + * This http endpoint allows direct updates (adding/removing) (CIDR) IP addresses and port ranges to + * the security group for this app. */ @Path("/v1/secgroup") @Produces(MediaType.TEXT_PLAIN) @@ -42,9 +39,11 @@ public SecurityGroupAdmin(IMembership membership) { } @POST - public Response addACL(@QueryParam("ip") String ipAddr, @QueryParam("fromPort") int fromPort, @QueryParam("toPort") int toPort) { - if (!ipAddr.endsWith(CIDR_TAG)) - ipAddr += CIDR_TAG; + public Response addACL( + @QueryParam("ip") String ipAddr, + @QueryParam("fromPort") int fromPort, + @QueryParam("toPort") int toPort) { + if (!ipAddr.endsWith(CIDR_TAG)) ipAddr += CIDR_TAG; try { membership.addACL(Collections.singletonList(ipAddr), fromPort, toPort); } catch (Exception e) { @@ -55,9 +54,11 @@ public Response addACL(@QueryParam("ip") String ipAddr, @QueryParam("fromPort") } @DELETE - public Response removeACL(@QueryParam("ip") String ipAddr, @QueryParam("fromPort") int fromPort, @QueryParam("toPort") int toPort) { - if (!ipAddr.endsWith(CIDR_TAG)) - ipAddr += CIDR_TAG; + public Response removeACL( + @QueryParam("ip") String ipAddr, + @QueryParam("fromPort") int fromPort, + @QueryParam("toPort") int toPort) { + if (!ipAddr.endsWith(CIDR_TAG)) ipAddr += CIDR_TAG; try { membership.removeACL(Collections.singletonList(ipAddr), fromPort, toPort); } catch (Exception e) { @@ -67,4 +68,3 @@ public Response removeACL(@QueryParam("ip") String ipAddr, @QueryParam("fromPort return Response.ok().build(); } } - diff --git a/priam/src/main/java/com/netflix/priam/restore/AbstractRestore.java b/priam/src/main/java/com/netflix/priam/restore/AbstractRestore.java index 14e1ca83d..ef86982ea 100644 --- a/priam/src/main/java/com/netflix/priam/restore/AbstractRestore.java +++ b/priam/src/main/java/com/netflix/priam/restore/AbstractRestore.java @@ -19,20 +19,14 @@ import com.google.common.collect.Iterators; import com.google.common.collect.Lists; import com.google.inject.Provider; -import com.netflix.priam.defaultimpl.ICassandraProcess; -import com.netflix.priam.config.IConfiguration; import com.netflix.priam.backup.*; import com.netflix.priam.backup.AbstractBackupPath.BackupFileType; +import com.netflix.priam.config.IConfiguration; +import com.netflix.priam.defaultimpl.ICassandraProcess; import com.netflix.priam.health.InstanceState; import com.netflix.priam.identity.InstanceIdentity; import com.netflix.priam.scheduler.Task; import com.netflix.priam.utils.*; -import org.apache.commons.collections4.CollectionUtils; -import org.apache.commons.io.FileUtils; -import org.apache.commons.lang3.StringUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import java.io.File; import java.io.IOException; import java.math.BigInteger; @@ -40,12 +34,17 @@ import java.time.LocalDateTime; import java.util.*; import java.util.concurrent.Future; +import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.io.FileUtils; +import org.apache.commons.lang3.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** - * A means to perform a restore. This class contains the following characteristics: - * - It is agnostic to the source type of the restore, this is determine by the injected IBackupFileSystem. - * - This class can be scheduled, i.e. it is a "Task". - * - When this class is executed, it uses its own thread pool to execute the restores. + * A means to perform a restore. This class contains the following characteristics: - It is agnostic + * to the source type of the restore, this is determine by the injected IBackupFileSystem. - This + * class can be scheduled, i.e. it is a "Task". - When this class is executed, it uses its own + * thread pool to execute the restores. */ public abstract class AbstractRestore extends Task implements IRestoreStrategy { // keeps track of the last few download which was executed. @@ -66,10 +65,18 @@ public abstract class AbstractRestore extends Task implements IRestoreStrategy { private final MetaData metaData; private final IPostRestoreHook postRestoreHook; - AbstractRestore(IConfiguration config, IBackupFileSystem fs, String name, Sleeper sleeper, - Provider pathProvider, - InstanceIdentity instanceIdentity, RestoreTokenSelector tokenSelector, - ICassandraProcess cassProcess, MetaData metaData, InstanceState instanceState, IPostRestoreHook postRestoreHook) { + AbstractRestore( + IConfiguration config, + IBackupFileSystem fs, + String name, + Sleeper sleeper, + Provider pathProvider, + InstanceIdentity instanceIdentity, + RestoreTokenSelector tokenSelector, + ICassandraProcess cassProcess, + MetaData metaData, + InstanceState instanceState, + IPostRestoreHook postRestoreHook) { super(config); this.fs = fs; this.sleeper = sleeper; @@ -79,13 +86,17 @@ public abstract class AbstractRestore extends Task implements IRestoreStrategy { this.cassProcess = cassProcess; this.metaData = metaData; this.instanceState = instanceState; - backupRestoreUtil = new BackupRestoreUtil(config.getRestoreIncludeCFList(), config.getRestoreExcludeCFList()); + backupRestoreUtil = + new BackupRestoreUtil( + config.getRestoreIncludeCFList(), config.getRestoreExcludeCFList()); this.postRestoreHook = postRestoreHook; } public static final boolean isRestoreEnabled(IConfiguration conf) { boolean isRestoreMode = StringUtils.isNotBlank(conf.getRestoreSnapshot()); - boolean isBackedupRac = (CollectionUtils.isEmpty(conf.getBackupRacs()) || conf.getBackupRacs().contains(conf.getRac())); + boolean isBackedupRac = + (CollectionUtils.isEmpty(conf.getBackupRacs()) + || conf.getBackupRacs().contains(conf.getRac())); return (isRestoreMode && isBackedupRac); } @@ -93,48 +104,60 @@ public void setRestoreConfiguration(String restoreIncludeCFList, String restoreE backupRestoreUtil.setFilters(restoreIncludeCFList, restoreExcludeCFList); } - private List> download(Iterator fsIterator, BackupFileType bkupFileType, boolean waitForCompletion) throws Exception { + private List> download( + Iterator fsIterator, + BackupFileType bkupFileType, + boolean waitForCompletion) + throws Exception { List> futureList = new ArrayList<>(); while (fsIterator.hasNext()) { AbstractBackupPath temp = fsIterator.next(); - if (temp.getType() == BackupFileType.SST && tracker.contains(temp)) - continue; - - if (backupRestoreUtil.isFiltered(temp.getKeyspace(), temp.getColumnFamily())) { //is filtered? - logger.info("Bypassing restoring file \"{}\" as it is part of the keyspace.columnfamily filter list. Its keyspace:cf is: {}:{}", - temp.newRestoreFile(), temp.getKeyspace(), temp.getColumnFamily()); + if (temp.getType() == BackupFileType.SST && tracker.contains(temp)) continue; + + if (backupRestoreUtil.isFiltered( + temp.getKeyspace(), temp.getColumnFamily())) { // is filtered? + logger.info( + "Bypassing restoring file \"{}\" as it is part of the keyspace.columnfamily filter list. Its keyspace:cf is: {}:{}", + temp.newRestoreFile(), + temp.getKeyspace(), + temp.getColumnFamily()); continue; } if (temp.getType() == bkupFileType) { File localFileHandler = temp.newRestoreFile(); if (logger.isDebugEnabled()) - logger.debug("Created local file name: " + localFileHandler.getAbsolutePath() + File.pathSeparator + localFileHandler.getName()); + logger.debug( + "Created local file name: " + + localFileHandler.getAbsolutePath() + + File.pathSeparator + + localFileHandler.getName()); futureList.add(downloadFile(temp, localFileHandler)); } } - //Wait for all download to finish that were started from this method. - if (waitForCompletion) - waitForCompletion(futureList); + // Wait for all download to finish that were started from this method. + if (waitForCompletion) waitForCompletion(futureList); return futureList; } private void waitForCompletion(List> futureList) throws Exception { - for (Future future : futureList) - future.get(); + for (Future future : futureList) future.get(); } - private List> downloadCommitLogs(Iterator fsIterator, BackupFileType filter, int lastN, boolean waitForCompletion) throws Exception { - if (fsIterator == null) - return null; + private List> downloadCommitLogs( + Iterator fsIterator, + BackupFileType filter, + int lastN, + boolean waitForCompletion) + throws Exception { + if (fsIterator == null) return null; BoundedList bl = new BoundedList(lastN); while (fsIterator.hasNext()) { AbstractBackupPath temp = fsIterator.next(); - if (temp.getType() == BackupFileType.SST && tracker.contains(temp)) - continue; + if (temp.getType() == BackupFileType.SST && tracker.contains(temp)) continue; if (temp.getType() == filter) { bl.add(temp); @@ -151,10 +174,8 @@ private void stopCassProcess() throws IOException { private String getRestorePrefix() { String prefix; - if (StringUtils.isNotBlank(config.getRestorePrefix())) - prefix = config.getRestorePrefix(); - else - prefix = config.getBackupPrefix(); + if (StringUtils.isNotBlank(config.getRestorePrefix())) prefix = config.getRestorePrefix(); + else prefix = config.getBackupPrefix(); return prefix; } @@ -162,18 +183,22 @@ private String getRestorePrefix() { /* * Fetches meta.json used to store snapshots metadata. */ - private void fetchSnapshotMetaFile(String restorePrefix, List out, Date startTime, Date endTime) throws IllegalStateException { + private void fetchSnapshotMetaFile( + String restorePrefix, List out, Date startTime, Date endTime) + throws IllegalStateException { logger.debug("Looking for snapshot meta file within restore prefix: {}", restorePrefix); Iterator backupfiles = fs.list(restorePrefix, startTime, endTime); if (!backupfiles.hasNext()) { - throw new IllegalStateException("meta.json not found, restore prefix: " + restorePrefix); + throw new IllegalStateException( + "meta.json not found, restore prefix: " + restorePrefix); } while (backupfiles.hasNext()) { AbstractBackupPath path = backupfiles.next(); if (path.getType() == BackupFileType.META) - //Since there are now meta file for incrementals as well as snapshot, we need to find the correct one (i.e. the snapshot meta file (meta.json)) + // Since there are now meta file for incrementals as well as snapshot, we need to + // find the correct one (i.e. the snapshot meta file (meta.json)) if (path.getFileName().equalsIgnoreCase("meta.json")) { out.add(path); } @@ -182,8 +207,7 @@ private void fetchSnapshotMetaFile(String restorePrefix, List metas = Lists.newArrayList(); @@ -249,13 +271,13 @@ public void restore(Date startTime, Date endTime) throws Exception { logger.info("Snapshot Meta file for restore {}", meta.getRemotePath()); instanceState.getRestoreStatus().setSnapshotMetaFile(meta.getRemotePath()); - //Download the meta.json file. + // Download the meta.json file. ArrayList metaFile = new ArrayList<>(); metaFile.add(meta); download(metaFile.iterator(), BackupFileType.META, true); List> futureList = new ArrayList<>(); - //Parse meta.json file to find the files required to download from this snapshot. + // Parse meta.json file to find the files required to download from this snapshot. List snapshots = metaData.toJson(meta.newRestoreFile()); // Download snapshot which is listed in the meta file. @@ -266,19 +288,27 @@ public void restore(Date startTime, Date endTime) throws Exception { Iterator incrementals = fs.list(prefix, meta.getTime(), endTime); futureList.addAll(download(incrementals, BackupFileType.SST, false)); - //Downloading CommitLogs + // Downloading CommitLogs if (config.isBackingUpCommitLogs()) { - logger.info("Delete all backuped commitlog files in {}", config.getBackupCommitLogLocation()); + logger.info( + "Delete all backuped commitlog files in {}", + config.getBackupCommitLogLocation()); SystemUtils.cleanupDir(config.getBackupCommitLogLocation(), null); logger.info("Delete all commitlog files in {}", config.getCommitLogLocation()); SystemUtils.cleanupDir(config.getCommitLogLocation(), null); - Iterator commitLogPathIterator = fs.list(prefix, meta.getTime(), endTime); - futureList.addAll(downloadCommitLogs(commitLogPathIterator, BackupFileType.CL, config.maxCommitLogsRestore(), false)); + Iterator commitLogPathIterator = + fs.list(prefix, meta.getTime(), endTime); + futureList.addAll( + downloadCommitLogs( + commitLogPathIterator, + BackupFileType.CL, + config.maxCommitLogsRestore(), + false)); } - //Wait for all the futures to finish. + // Wait for all the futures to finish. waitForCompletion(futureList); // Given that files are restored now, kick off post restore hook @@ -290,11 +320,11 @@ public void restore(Date startTime, Date endTime) throws Exception { instanceState.getRestoreStatus().setExecutionEndTime(LocalDateTime.now()); instanceState.setRestoreStatus(Status.FINISHED); - //Start cassandra if restore is successful. - if (!config.doesCassandraStartManually()) - cassProcess.start(true); + // Start cassandra if restore is successful. + if (!config.doesCassandraStartManually()) cassProcess.start(true); else - logger.info("config.doesCassandraStartManually() is set to True, hence Cassandra needs to be started manually ..."); + logger.info( + "config.doesCassandraStartManually() is set to True, hence Cassandra needs to be started manually ..."); } catch (Exception e) { instanceState.setRestoreStatus(Status.FAILED); instanceState.getRestoreStatus().setExecutionEndTime(LocalDateTime.now()); @@ -306,14 +336,17 @@ public void restore(Date startTime, Date endTime) throws Exception { } /** - * Download file to the location specified. After downloading the file will be decrypted(optionally) and decompressed before saving to final location. + * Download file to the location specified. After downloading the file will be + * decrypted(optionally) and decompressed before saving to final location. * - * @param path - path of object to download from source S3/GCS. - * @param restoreLocation - path to the final location of the decompressed and/or decrypted file. + * @param path - path of object to download from source S3/GCS. + * @param restoreLocation - path to the final location of the decompressed and/or decrypted + * file. * @return Future of the job to track the progress of the job. * @throws Exception If there is any error in downloading file from the remote file system. */ - protected abstract Future downloadFile(final AbstractBackupPath path, final File restoreLocation) throws Exception; + protected abstract Future downloadFile( + final AbstractBackupPath path, final File restoreLocation) throws Exception; final class BoundedList extends LinkedList { diff --git a/priam/src/main/java/com/netflix/priam/restore/AwsCrossAccountCryptographyRestoreStrategy.java b/priam/src/main/java/com/netflix/priam/restore/AwsCrossAccountCryptographyRestoreStrategy.java index 8b3366f64..010648b78 100755 --- a/priam/src/main/java/com/netflix/priam/restore/AwsCrossAccountCryptographyRestoreStrategy.java +++ b/priam/src/main/java/com/netflix/priam/restore/AwsCrossAccountCryptographyRestoreStrategy.java @@ -1,16 +1,14 @@ /** * Copyright 2017 Netflix, Inc. - *

- * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and + * + *

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file + * except in compliance with the License. You may obtain a copy of the License at + * + *

http://www.apache.org/licenses/LICENSE-2.0 + * + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing permissions and * limitations under the License. */ package com.netflix.priam.restore; @@ -19,14 +17,14 @@ import com.google.inject.Provider; import com.google.inject.Singleton; import com.google.inject.name.Named; -import com.netflix.priam.defaultimpl.ICassandraProcess; -import com.netflix.priam.config.IConfiguration; -import com.netflix.priam.cred.ICredentialGeneric; import com.netflix.priam.aws.S3CrossAccountFileSystem; import com.netflix.priam.backup.AbstractBackupPath; import com.netflix.priam.backup.MetaData; import com.netflix.priam.compress.ICompression; +import com.netflix.priam.config.IConfiguration; +import com.netflix.priam.cred.ICredentialGeneric; import com.netflix.priam.cryptography.IFileCryptography; +import com.netflix.priam.defaultimpl.ICassandraProcess; import com.netflix.priam.health.InstanceState; import com.netflix.priam.identity.InstanceIdentity; import com.netflix.priam.scheduler.SimpleTimer; @@ -42,26 +40,47 @@ @Singleton public class AwsCrossAccountCryptographyRestoreStrategy extends EncryptedRestoreBase { - private static final Logger logger = LoggerFactory.getLogger(AwsCrossAccountCryptographyRestoreStrategy.class); + private static final Logger logger = + LoggerFactory.getLogger(AwsCrossAccountCryptographyRestoreStrategy.class); public static final String JOBNAME = "AWS_CROSS_ACCT_CRYPTOGRAPHY_RESTORE_JOB"; - //Note: see javadoc for S3CrossAccountFileSystem for reason why we inject a concrete class (S3CrossAccountFileSystem) instead of the inteface IBackupFileSystem + // Note: see javadoc for S3CrossAccountFileSystem for reason why we inject a concrete class + // (S3CrossAccountFileSystem) instead of the inteface IBackupFileSystem @Inject - public AwsCrossAccountCryptographyRestoreStrategy(final IConfiguration config, ICassandraProcess cassProcess - , S3CrossAccountFileSystem crossAcctfs - , Sleeper sleeper - , @Named("filecryptoalgorithm") IFileCryptography fileCryptography - , @Named("pgpcredential") ICredentialGeneric credential - , ICompression compress, Provider pathProvider, - InstanceIdentity id, RestoreTokenSelector tokenSelector, MetaData metaData, InstanceState instanceState, IPostRestoreHook postRestoreHook) { + public AwsCrossAccountCryptographyRestoreStrategy( + final IConfiguration config, + ICassandraProcess cassProcess, + S3CrossAccountFileSystem crossAcctfs, + Sleeper sleeper, + @Named("filecryptoalgorithm") IFileCryptography fileCryptography, + @Named("pgpcredential") ICredentialGeneric credential, + ICompression compress, + Provider pathProvider, + InstanceIdentity id, + RestoreTokenSelector tokenSelector, + MetaData metaData, + InstanceState instanceState, + IPostRestoreHook postRestoreHook) { - super(config, crossAcctfs.getBackupFileSystem(), JOBNAME, sleeper, cassProcess, pathProvider, id, tokenSelector, credential, fileCryptography, compress, metaData, instanceState, postRestoreHook); + super( + config, + crossAcctfs.getBackupFileSystem(), + JOBNAME, + sleeper, + cassProcess, + pathProvider, + id, + tokenSelector, + credential, + fileCryptography, + compress, + metaData, + instanceState, + postRestoreHook); } - /** - * @return a timer used by the scheduler to determine when "this" should be run. - */ + /** @return a timer used by the scheduler to determine when "this" should be run. */ public static TaskTimer getTimer() { return new SimpleTimer(JOBNAME); } -} \ No newline at end of file +} diff --git a/priam/src/main/java/com/netflix/priam/restore/EncryptedRestoreBase.java b/priam/src/main/java/com/netflix/priam/restore/EncryptedRestoreBase.java index 86c8f8b6e..300ad9e45 100755 --- a/priam/src/main/java/com/netflix/priam/restore/EncryptedRestoreBase.java +++ b/priam/src/main/java/com/netflix/priam/restore/EncryptedRestoreBase.java @@ -1,46 +1,41 @@ /** * Copyright 2017 Netflix, Inc. - *

- * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and + * + *

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file + * except in compliance with the License. You may obtain a copy of the License at + * + *

http://www.apache.org/licenses/LICENSE-2.0 + * + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing permissions and * limitations under the License. */ package com.netflix.priam.restore; import com.google.inject.Provider; -import com.netflix.priam.defaultimpl.ICassandraProcess; -import com.netflix.priam.config.IConfiguration; -import com.netflix.priam.cred.ICredentialGeneric; import com.netflix.priam.backup.*; import com.netflix.priam.compress.ICompression; +import com.netflix.priam.config.IConfiguration; +import com.netflix.priam.cred.ICredentialGeneric; import com.netflix.priam.cryptography.IFileCryptography; +import com.netflix.priam.defaultimpl.ICassandraProcess; import com.netflix.priam.health.InstanceState; import com.netflix.priam.identity.InstanceIdentity; import com.netflix.priam.scheduler.NamedThreadPoolExecutor; import com.netflix.priam.utils.RetryableCallable; import com.netflix.priam.utils.Sleeper; -import org.bouncycastle.util.io.Streams; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import java.io.*; import java.nio.file.Path; import java.nio.file.Paths; import java.util.concurrent.Future; import java.util.concurrent.ThreadPoolExecutor; +import org.bouncycastle.util.io.Streams; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; -/** - * Provides common functionality applicable to all restore strategies - */ -public abstract class EncryptedRestoreBase extends AbstractRestore{ +/** Provides common functionality applicable to all restore strategies */ +public abstract class EncryptedRestoreBase extends AbstractRestore { private static final Logger logger = LoggerFactory.getLogger(EncryptedRestoreBase.class); private final String jobName; @@ -49,11 +44,33 @@ public abstract class EncryptedRestoreBase extends AbstractRestore{ private final ICompression compress; private final ThreadPoolExecutor executor; - protected EncryptedRestoreBase(IConfiguration config, IBackupFileSystem fs, String jobName, Sleeper sleeper, - ICassandraProcess cassProcess, Provider pathProvider, - InstanceIdentity instanceIdentity, RestoreTokenSelector tokenSelector, ICredentialGeneric pgpCredential, - IFileCryptography fileCryptography, ICompression compress, MetaData metaData, InstanceState instanceState, IPostRestoreHook postRestoreHook) { - super(config, fs, jobName, sleeper, pathProvider, instanceIdentity, tokenSelector, cassProcess, metaData, instanceState, postRestoreHook); + protected EncryptedRestoreBase( + IConfiguration config, + IBackupFileSystem fs, + String jobName, + Sleeper sleeper, + ICassandraProcess cassProcess, + Provider pathProvider, + InstanceIdentity instanceIdentity, + RestoreTokenSelector tokenSelector, + ICredentialGeneric pgpCredential, + IFileCryptography fileCryptography, + ICompression compress, + MetaData metaData, + InstanceState instanceState, + IPostRestoreHook postRestoreHook) { + super( + config, + fs, + jobName, + sleeper, + pathProvider, + instanceIdentity, + tokenSelector, + cassProcess, + metaData, + instanceState, + postRestoreHook); this.jobName = jobName; this.pgpCredential = pgpCredential; @@ -61,82 +78,126 @@ protected EncryptedRestoreBase(IConfiguration config, IBackupFileSystem fs, Stri this.compress = compress; executor = new NamedThreadPoolExecutor(config.getRestoreThreads(), jobName); executor.allowCoreThreadTimeOut(true); - logger.info("Trying to restore cassandra cluster with filesystem: {}, RestoreStrategy: {}, Encryption: ON, Compression: {}", - fs.getClass(), jobName, compress.getClass()); + logger.info( + "Trying to restore cassandra cluster with filesystem: {}, RestoreStrategy: {}, Encryption: ON, Compression: {}", + fs.getClass(), + jobName, + compress.getClass()); } @Override - protected final Future downloadFile(final AbstractBackupPath path, final File restoreLocation) throws Exception{ - final char[] passPhrase = new String(this.pgpCredential.getValue(ICredentialGeneric.KEY.PGP_PASSWORD)).toCharArray(); + protected final Future downloadFile( + final AbstractBackupPath path, final File restoreLocation) throws Exception { + final char[] passPhrase = + new String(this.pgpCredential.getValue(ICredentialGeneric.KEY.PGP_PASSWORD)) + .toCharArray(); File tempFile = new File(restoreLocation.getAbsolutePath() + ".tmp"); - return executor.submit(new RetryableCallable() { - - @Override - public Path retriableCall() throws Exception { - - //== download object from source bucket - try { - //Not retrying to download file here as it is already in RetryCallable. - fs.downloadFile(Paths.get(path.getRemotePath()), Paths.get(tempFile.getAbsolutePath()), 0); - tracker.adjustAndAdd(path); - } catch (Exception ex) { - //This behavior is retryable; therefore, lets get to a clean state before each retry. - if (tempFile.exists()) { - tempFile.createNewFile(); + return executor.submit( + new RetryableCallable() { + + @Override + public Path retriableCall() throws Exception { + + // == download object from source bucket + try { + // Not retrying to download file here as it is already in RetryCallable. + fs.downloadFile( + Paths.get(path.getRemotePath()), + Paths.get(tempFile.getAbsolutePath()), + 0); + tracker.adjustAndAdd(path); + } catch (Exception ex) { + // This behavior is retryable; therefore, lets get to a clean state + // before each retry. + if (tempFile.exists()) { + tempFile.createNewFile(); + } + + throw new Exception( + "Exception downloading file from: " + + path.getRemotePath() + + " to: " + + tempFile.getAbsolutePath(), + ex); } - throw new Exception("Exception downloading file from: " + path.getRemotePath() + " to: " + tempFile.getAbsolutePath(), ex); - } - - //== object downloaded successfully from source, decrypt it. - File decryptedFile = new File(tempFile.getAbsolutePath() + ".decrypted"); - try(OutputStream fOut = new BufferedOutputStream(new FileOutputStream(decryptedFile)); //destination file after decryption) - InputStream in = new BufferedInputStream(new FileInputStream(tempFile.getAbsolutePath()))) { - InputStream encryptedDataInputStream = fileCryptography.decryptStream(in, passPhrase, tempFile.getAbsolutePath()); - Streams.pipeAll(encryptedDataInputStream, fOut); - logger.info("Completed decrypting file: {} to final file dest: {}", tempFile.getAbsolutePath(), decryptedFile.getAbsolutePath()); - - } catch (Exception ex) { - //This behavior is retryable; therefore, lets get to a clean state before each retry. - if (tempFile.exists()) { - tempFile.createNewFile(); + // == object downloaded successfully from source, decrypt it. + File decryptedFile = new File(tempFile.getAbsolutePath() + ".decrypted"); + try (OutputStream fOut = + new BufferedOutputStream( + new FileOutputStream( + decryptedFile)); // destination file after + // decryption) + InputStream in = + new BufferedInputStream( + new FileInputStream(tempFile.getAbsolutePath()))) { + InputStream encryptedDataInputStream = + fileCryptography.decryptStream( + in, passPhrase, tempFile.getAbsolutePath()); + Streams.pipeAll(encryptedDataInputStream, fOut); + logger.info( + "Completed decrypting file: {} to final file dest: {}", + tempFile.getAbsolutePath(), + decryptedFile.getAbsolutePath()); + + } catch (Exception ex) { + // This behavior is retryable; therefore, lets get to a clean state + // before each retry. + if (tempFile.exists()) { + tempFile.createNewFile(); + } + + if (decryptedFile.exists()) { + decryptedFile.createNewFile(); + } + + throw new Exception( + "Exception during decryption file: " + + decryptedFile.getAbsolutePath(), + ex); } - if (decryptedFile.exists()) { - decryptedFile.createNewFile(); + // == object downloaded and decrypted successfully, now uncompress it + logger.info( + "Start uncompressing file: {} to the FINAL destination stream", + decryptedFile.getAbsolutePath()); + + try (InputStream is = + new BufferedInputStream( + new FileInputStream(decryptedFile)); + BufferedOutputStream finalDestination = + new BufferedOutputStream( + new FileOutputStream(restoreLocation))) { + compress.decompressAndClose(is, finalDestination); + } catch (Exception ex) { + throw new Exception( + "Exception uncompressing file: " + + decryptedFile.getAbsolutePath() + + " to the FINAL destination stream", + ex); } - throw new Exception("Exception during decryption file: " + decryptedFile.getAbsolutePath(), ex); - } - - //== object downloaded and decrypted successfully, now uncompress it - logger.info("Start uncompressing file: {} to the FINAL destination stream", decryptedFile.getAbsolutePath()); + logger.info( + "Completed uncompressing file: {} to the FINAL destination stream " + + " current worker: {}", + decryptedFile.getAbsolutePath(), + Thread.currentThread().getName()); + // if here, everything was successful for this object, lets remove unneeded + // file(s) + if (tempFile.exists()) tempFile.delete(); - try(InputStream is = new BufferedInputStream(new FileInputStream(decryptedFile)); - BufferedOutputStream finalDestination = new BufferedOutputStream(new FileOutputStream(restoreLocation))) { - compress.decompressAndClose(is, finalDestination); - } catch (Exception ex) { - throw new Exception("Exception uncompressing file: " + decryptedFile.getAbsolutePath() + " to the FINAL destination stream", ex); - } - - logger.info("Completed uncompressing file: {} to the FINAL destination stream " - + " current worker: {}", decryptedFile.getAbsolutePath(), Thread.currentThread().getName()); - //if here, everything was successful for this object, lets remove unneeded file(s) - if (tempFile.exists()) - tempFile.delete(); + if (decryptedFile.exists()) { + decryptedFile.delete(); + } - if (decryptedFile.exists()) { - decryptedFile.delete(); + return Paths.get(path.getRemotePath()); } - - return Paths.get(path.getRemotePath()); - } - - }); } + }); + } @Override public String getName() { return this.jobName; } -} \ No newline at end of file +} diff --git a/priam/src/main/java/com/netflix/priam/restore/EncryptedRestoreStrategy.java b/priam/src/main/java/com/netflix/priam/restore/EncryptedRestoreStrategy.java index 40391e4b8..fdef4f4ad 100755 --- a/priam/src/main/java/com/netflix/priam/restore/EncryptedRestoreStrategy.java +++ b/priam/src/main/java/com/netflix/priam/restore/EncryptedRestoreStrategy.java @@ -1,16 +1,14 @@ /** * Copyright 2017 Netflix, Inc. - *

- * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and + * + *

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file + * except in compliance with the License. You may obtain a copy of the License at + * + *

http://www.apache.org/licenses/LICENSE-2.0 + * + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing permissions and * limitations under the License. */ package com.netflix.priam.restore; @@ -19,14 +17,14 @@ import com.google.inject.Provider; import com.google.inject.Singleton; import com.google.inject.name.Named; -import com.netflix.priam.defaultimpl.ICassandraProcess; -import com.netflix.priam.config.IConfiguration; -import com.netflix.priam.cred.ICredentialGeneric; import com.netflix.priam.backup.AbstractBackupPath; import com.netflix.priam.backup.IBackupFileSystem; import com.netflix.priam.backup.MetaData; import com.netflix.priam.compress.ICompression; +import com.netflix.priam.config.IConfiguration; +import com.netflix.priam.cred.ICredentialGeneric; import com.netflix.priam.cryptography.IFileCryptography; +import com.netflix.priam.defaultimpl.ICassandraProcess; import com.netflix.priam.health.InstanceState; import com.netflix.priam.identity.InstanceIdentity; import com.netflix.priam.scheduler.SimpleTimer; @@ -44,15 +42,36 @@ public class EncryptedRestoreStrategy extends EncryptedRestoreBase { public static final String JOBNAME = "CRYPTOGRAPHY_RESTORE_JOB"; @Inject - public EncryptedRestoreStrategy(final IConfiguration config, ICassandraProcess cassProcess, - @Named("encryptedbackup") IBackupFileSystem fs, Sleeper sleeper - , @Named("filecryptoalgorithm") IFileCryptography fileCryptography - , @Named("pgpcredential") ICredentialGeneric credential - , ICompression compress, Provider pathProvider, - InstanceIdentity id, RestoreTokenSelector tokenSelector, MetaData metaData, InstanceState instanceState, IPostRestoreHook postRestoreHook - ) { + public EncryptedRestoreStrategy( + final IConfiguration config, + ICassandraProcess cassProcess, + @Named("encryptedbackup") IBackupFileSystem fs, + Sleeper sleeper, + @Named("filecryptoalgorithm") IFileCryptography fileCryptography, + @Named("pgpcredential") ICredentialGeneric credential, + ICompression compress, + Provider pathProvider, + InstanceIdentity id, + RestoreTokenSelector tokenSelector, + MetaData metaData, + InstanceState instanceState, + IPostRestoreHook postRestoreHook) { - super(config, fs, JOBNAME, sleeper, cassProcess, pathProvider, id, tokenSelector, credential, fileCryptography, compress, metaData, instanceState, postRestoreHook); + super( + config, + fs, + JOBNAME, + sleeper, + cassProcess, + pathProvider, + id, + tokenSelector, + credential, + fileCryptography, + compress, + metaData, + instanceState, + postRestoreHook); } /* @@ -61,5 +80,4 @@ public EncryptedRestoreStrategy(final IConfiguration config, ICassandraProcess c public static TaskTimer getTimer() { return new SimpleTimer(JOBNAME); } - -} \ No newline at end of file +} diff --git a/priam/src/main/java/com/netflix/priam/restore/GoogleCryptographyRestoreStrategy.java b/priam/src/main/java/com/netflix/priam/restore/GoogleCryptographyRestoreStrategy.java index 5dfdcfef7..1c0fc9d79 100755 --- a/priam/src/main/java/com/netflix/priam/restore/GoogleCryptographyRestoreStrategy.java +++ b/priam/src/main/java/com/netflix/priam/restore/GoogleCryptographyRestoreStrategy.java @@ -1,16 +1,14 @@ /** * Copyright 2017 Netflix, Inc. - *

- * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and + * + *

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file + * except in compliance with the License. You may obtain a copy of the License at + * + *

http://www.apache.org/licenses/LICENSE-2.0 + * + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing permissions and * limitations under the License. */ package com.netflix.priam.restore; @@ -19,14 +17,14 @@ import com.google.inject.Provider; import com.google.inject.Singleton; import com.google.inject.name.Named; -import com.netflix.priam.defaultimpl.ICassandraProcess; -import com.netflix.priam.config.IConfiguration; -import com.netflix.priam.cred.ICredentialGeneric; import com.netflix.priam.backup.AbstractBackupPath; import com.netflix.priam.backup.IBackupFileSystem; import com.netflix.priam.backup.MetaData; import com.netflix.priam.compress.ICompression; +import com.netflix.priam.config.IConfiguration; +import com.netflix.priam.cred.ICredentialGeneric; import com.netflix.priam.cryptography.IFileCryptography; +import com.netflix.priam.defaultimpl.ICassandraProcess; import com.netflix.priam.health.InstanceState; import com.netflix.priam.identity.InstanceIdentity; import com.netflix.priam.scheduler.SimpleTimer; @@ -37,28 +35,44 @@ @Singleton public class GoogleCryptographyRestoreStrategy extends EncryptedRestoreBase { - private static final Logger logger = LoggerFactory.getLogger(GoogleCryptographyRestoreStrategy.class); + private static final Logger logger = + LoggerFactory.getLogger(GoogleCryptographyRestoreStrategy.class); public static final String JOBNAME = "GOOGLECLOUDSTORAGE_RESTORE_JOB"; @Inject - public GoogleCryptographyRestoreStrategy(final IConfiguration config, ICassandraProcess cassProcess, @Named("gcsencryptedbackup") IBackupFileSystem fs, Sleeper sleeper - , @Named("filecryptoalgorithm") IFileCryptography fileCryptography - , @Named("pgpcredential") ICredentialGeneric credential - , ICompression compress, Provider pathProvider, - InstanceIdentity id, RestoreTokenSelector tokenSelector, MetaData metaData, InstanceState instanceState, IPostRestoreHook postRestoreHook - ) { - super(config, fs, JOBNAME, sleeper, cassProcess, pathProvider, id, tokenSelector, credential, fileCryptography, compress, metaData, instanceState, postRestoreHook); + public GoogleCryptographyRestoreStrategy( + final IConfiguration config, + ICassandraProcess cassProcess, + @Named("gcsencryptedbackup") IBackupFileSystem fs, + Sleeper sleeper, + @Named("filecryptoalgorithm") IFileCryptography fileCryptography, + @Named("pgpcredential") ICredentialGeneric credential, + ICompression compress, + Provider pathProvider, + InstanceIdentity id, + RestoreTokenSelector tokenSelector, + MetaData metaData, + InstanceState instanceState, + IPostRestoreHook postRestoreHook) { + super( + config, + fs, + JOBNAME, + sleeper, + cassProcess, + pathProvider, + id, + tokenSelector, + credential, + fileCryptography, + compress, + metaData, + instanceState, + postRestoreHook); } - - /** - * @return a timer used by the scheduler to determine when "this" should be run. - */ + /** @return a timer used by the scheduler to determine when "this" should be run. */ public static TaskTimer getTimer() { return new SimpleTimer(JOBNAME); } - - - - -} \ No newline at end of file +} diff --git a/priam/src/main/java/com/netflix/priam/restore/IPostRestoreHook.java b/priam/src/main/java/com/netflix/priam/restore/IPostRestoreHook.java index f743fbabf..7f7e3335a 100644 --- a/priam/src/main/java/com/netflix/priam/restore/IPostRestoreHook.java +++ b/priam/src/main/java/com/netflix/priam/restore/IPostRestoreHook.java @@ -18,11 +18,10 @@ import com.google.inject.ImplementedBy; -/** - * Interface for post restore hook - */ +/** Interface for post restore hook */ @ImplementedBy(PostRestoreHook.class) public interface IPostRestoreHook { boolean hasValidParameters(); + void execute() throws Exception; } diff --git a/priam/src/main/java/com/netflix/priam/restore/IRestoreStrategy.java b/priam/src/main/java/com/netflix/priam/restore/IRestoreStrategy.java index 5869e397b..1da3a7677 100755 --- a/priam/src/main/java/com/netflix/priam/restore/IRestoreStrategy.java +++ b/priam/src/main/java/com/netflix/priam/restore/IRestoreStrategy.java @@ -1,16 +1,14 @@ /** * Copyright 2017 Netflix, Inc. - *

- * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and + * + *

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file + * except in compliance with the License. You may obtain a copy of the License at + * + *

http://www.apache.org/licenses/LICENSE-2.0 + * + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing permissions and * limitations under the License. */ package com.netflix.priam.restore; @@ -19,5 +17,5 @@ * A means to restore C* files from various source types (e.g. Google, AWS bucket whose objects are not owned by the current IAM role), and encrypted / non-encrypted data. */ public interface IRestoreStrategy { - //public void restore(Date startTime, Date endTime) throws Exception; + // public void restore(Date startTime, Date endTime) throws Exception; } diff --git a/priam/src/main/java/com/netflix/priam/restore/PostRestoreHook.java b/priam/src/main/java/com/netflix/priam/restore/PostRestoreHook.java index a9b12b81e..18f40b01f 100644 --- a/priam/src/main/java/com/netflix/priam/restore/PostRestoreHook.java +++ b/priam/src/main/java/com/netflix/priam/restore/PostRestoreHook.java @@ -20,11 +20,6 @@ import com.netflix.priam.scheduler.NamedThreadPoolExecutor; import com.netflix.priam.utils.RetryableCallable; import com.netflix.priam.utils.Sleeper; -import org.apache.commons.lang3.StringUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import javax.inject.Inject; import java.io.File; import java.io.IOException; import java.io.RandomAccessFile; @@ -32,10 +27,15 @@ import java.nio.channels.FileLock; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; +import javax.inject.Inject; +import org.apache.commons.lang3.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** - * An implementation of IPostRestoreHook. Kicks off a child process for post restore hook using ProcessBuilder; uses heart beat monitor to monitor progress of the sub process - * and uses a file lock to pass the active state to the sub process + * An implementation of IPostRestoreHook. Kicks off a child process for post restore hook using + * ProcessBuilder; uses heart beat monitor to monitor progress of the sub process and uses a file + * lock to pass the active state to the sub process */ public class PostRestoreHook implements IPostRestoreHook { private static final Logger logger = LoggerFactory.getLogger(PostRestoreHook.class); @@ -54,11 +54,12 @@ public PostRestoreHook(IConfiguration config, Sleeper sleeper) { /** * Checks parameters to make sure none are blank + * * @return if all parameters are valid */ public boolean hasValidParameters() { - if(config.isPostRestoreHookEnabled()) { - if(StringUtils.isBlank(config.getPostRestoreHook()) + if (config.isPostRestoreHookEnabled()) { + if (StringUtils.isBlank(config.getPostRestoreHook()) || StringUtils.isBlank(config.getPostRestoreHookHeartbeatFileName()) || StringUtils.isBlank(config.getPostRestoreHookDoneFileName())) { return false; @@ -68,16 +69,21 @@ public boolean hasValidParameters() { } /** - * Executes a sub process as part of post restore hook, and waits for the completion of the process. In case of lack of heart beat from the sub process, existing sub process is terminated - * and new sub process is kicked off + * Executes a sub process as part of post restore hook, and waits for the completion of the + * process. In case of lack of heart beat from the sub process, existing sub process is + * terminated and new sub process is kicked off + * * @throws Exception */ public void execute() throws Exception { - if(config.isPostRestoreHookEnabled()) { + if (config.isPostRestoreHookEnabled()) { logger.debug("Started PostRestoreHook execution"); - //create a temp file to be used to indicate state of the current process, to the sub-process - File tempLockFile = File.createTempFile(PriamPostRestoreHookFilePrefix, PriamPostRestoreHookFileSuffix); + // create a temp file to be used to indicate state of the current process, to the + // sub-process + File tempLockFile = + File.createTempFile( + PriamPostRestoreHookFilePrefix, PriamPostRestoreHookFileSuffix); RandomAccessFile raf = new RandomAccessFile(tempLockFile.getPath(), "rw"); FileChannel fileChannel = raf.getChannel(); FileLock lock = fileChannel.lock(); @@ -88,26 +94,38 @@ public void execute() throws Exception { int countOfProcessStarts = 0; while (true) { if (doneFileExists()) { - logger.info("Not starting PostRestoreHook since DONE file already exists."); + logger.info( + "Not starting PostRestoreHook since DONE file already exists."); break; } String postRestoreHook = config.getPostRestoreHook(); - //add temp file path as parameter to the jar file - postRestoreHook = postRestoreHook + PostRestoreHookCommandDelimiter + PriamPostRestoreHookFileOptionName + tempLockFile.getAbsolutePath(); - String[] processCommandArguments = postRestoreHook.split(PostRestoreHookCommandDelimiter); + // add temp file path as parameter to the jar file + postRestoreHook = + postRestoreHook + + PostRestoreHookCommandDelimiter + + PriamPostRestoreHookFileOptionName + + tempLockFile.getAbsolutePath(); + String[] processCommandArguments = + postRestoreHook.split(PostRestoreHookCommandDelimiter); ProcessBuilder processBuilder = new ProcessBuilder(processCommandArguments); - //start sub-process + // start sub-process Process process = processBuilder.inheritIO().start(); - logger.info("Started PostRestoreHook: {} - Attempt#{}", postRestoreHook, ++countOfProcessStarts); + logger.info( + "Started PostRestoreHook: {} - Attempt#{}", + postRestoreHook, + ++countOfProcessStarts); - //monitor progress of sub-process + // monitor progress of sub-process monitorPostRestoreHookHeartBeat(process); - //block until sub-process completes or until the timeout - if (!process.waitFor(config.getPostRestoreHookTimeOutInDays(), TimeUnit.DAYS)) { - logger.info("PostRestoreHook process did not complete within {} days. Forcefully terminating the process.", config.getPostRestoreHookTimeOutInDays()); + // block until sub-process completes or until the timeout + if (!process.waitFor( + config.getPostRestoreHookTimeOutInDays(), TimeUnit.DAYS)) { + logger.info( + "PostRestoreHook process did not complete within {} days. Forcefully terminating the process.", + config.getPostRestoreHookTimeOutInDays()); process.destroyForcibly(); } @@ -119,10 +137,13 @@ public void execute() throws Exception { } logger.debug("Completed PostRestoreHook execution"); } else { - throw new PostRestoreHookException(String.format("Could not acquire lock on a temp file necessary for PostRestoreHook to execute. Path to temp file: %s", tempLockFile.getAbsolutePath())); + throw new PostRestoreHookException( + String.format( + "Could not acquire lock on a temp file necessary for PostRestoreHook to execute. Path to temp file: %s", + tempLockFile.getAbsolutePath())); } } finally { - //close and delete temp file + // close and delete temp file lock.release(); fileChannel.close(); raf.close(); @@ -131,37 +152,44 @@ public void execute() throws Exception { } } - /** * Monitors heart beat of the process + * * @param process Process to be monitored * @throws InterruptedException * @throws IOException */ - private void monitorPostRestoreHookHeartBeat(Process process) throws InterruptedException, IOException { + private void monitorPostRestoreHookHeartBeat(Process process) + throws InterruptedException, IOException { File heartBeatFile = new File(config.getPostRestoreHookHeartbeatFileName()); - ThreadPoolExecutor heartBeatPoolExecutor = new NamedThreadPoolExecutor(1, "PostRestoreHook_HeartBeatThreadPool"); + ThreadPoolExecutor heartBeatPoolExecutor = + new NamedThreadPoolExecutor(1, "PostRestoreHook_HeartBeatThreadPool"); heartBeatPoolExecutor.allowCoreThreadTimeOut(true); - heartBeatPoolExecutor.submit(new RetryableCallable() { - @Override - public Integer retriableCall() throws Exception { - while (true) { - sleeper.sleep(config.getPostRestoreHookHeartbeatCheckFrequencyInMs()); - if(System.currentTimeMillis() - heartBeatFile.lastModified() > config.getPostRestoreHookHeartBeatTimeoutInMs()) { - //kick off post restore hook process, since there is no heartbeat - logger.info("No heartbeat for the last {} ms, killing the existing process.", config.getPostRestoreHookHeartBeatTimeoutInMs()); - if(process.isAlive()) { - process.destroyForcibly(); + heartBeatPoolExecutor.submit( + new RetryableCallable() { + @Override + public Integer retriableCall() throws Exception { + while (true) { + sleeper.sleep(config.getPostRestoreHookHeartbeatCheckFrequencyInMs()); + if (System.currentTimeMillis() - heartBeatFile.lastModified() + > config.getPostRestoreHookHeartBeatTimeoutInMs()) { + // kick off post restore hook process, since there is no heartbeat + logger.info( + "No heartbeat for the last {} ms, killing the existing process.", + config.getPostRestoreHookHeartBeatTimeoutInMs()); + if (process.isAlive()) { + process.destroyForcibly(); + } + return 0; + } } - return 0; } - } - } - }); + }); } /** * Checks for presence of DONE file + * * @return if done file exists */ private boolean doneFileExists() { diff --git a/priam/src/main/java/com/netflix/priam/restore/PostRestoreHookException.java b/priam/src/main/java/com/netflix/priam/restore/PostRestoreHookException.java index e6daf7d78..8b09f6441 100644 --- a/priam/src/main/java/com/netflix/priam/restore/PostRestoreHookException.java +++ b/priam/src/main/java/com/netflix/priam/restore/PostRestoreHookException.java @@ -16,9 +16,7 @@ package com.netflix.priam.restore; -/** - * Exception raised by PostRestoreHook - */ +/** Exception raised by PostRestoreHook */ public class PostRestoreHookException extends Exception { public PostRestoreHookException(String message) { @@ -28,5 +26,4 @@ public PostRestoreHookException(String message) { public PostRestoreHookException(String message, Exception e) { super(message, e); } - -} \ No newline at end of file +} diff --git a/priam/src/main/java/com/netflix/priam/restore/Restore.java b/priam/src/main/java/com/netflix/priam/restore/Restore.java index 41baf7b80..a7a5d2d6c 100644 --- a/priam/src/main/java/com/netflix/priam/restore/Restore.java +++ b/priam/src/main/java/com/netflix/priam/restore/Restore.java @@ -20,43 +20,61 @@ import com.google.inject.Provider; import com.google.inject.Singleton; import com.google.inject.name.Named; -import com.netflix.priam.defaultimpl.ICassandraProcess; -import com.netflix.priam.config.IConfiguration; import com.netflix.priam.backup.AbstractBackupPath; import com.netflix.priam.backup.IBackupFileSystem; import com.netflix.priam.backup.MetaData; +import com.netflix.priam.config.IConfiguration; +import com.netflix.priam.defaultimpl.ICassandraProcess; import com.netflix.priam.health.InstanceState; import com.netflix.priam.identity.InstanceIdentity; import com.netflix.priam.scheduler.SimpleTimer; import com.netflix.priam.scheduler.TaskTimer; import com.netflix.priam.utils.Sleeper; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import java.io.File; import java.nio.file.Path; import java.nio.file.Paths; import java.util.concurrent.Future; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; -/** - * Main class for restoring data from backup. Backup restored using this way are not encrypted. - */ +/** Main class for restoring data from backup. Backup restored using this way are not encrypted. */ @Singleton public class Restore extends AbstractRestore { public static final String JOBNAME = "AUTO_RESTORE_JOB"; private static final Logger logger = LoggerFactory.getLogger(Restore.class); @Inject - public Restore(IConfiguration config, @Named("backup") IBackupFileSystem fs, Sleeper sleeper, ICassandraProcess cassProcess, - Provider pathProvider, - InstanceIdentity instanceIdentity, RestoreTokenSelector tokenSelector, MetaData metaData, InstanceState instanceState, IPostRestoreHook postRestoreHook) { - super(config, fs, JOBNAME, sleeper, pathProvider, instanceIdentity, tokenSelector, cassProcess, metaData, instanceState, postRestoreHook); + public Restore( + IConfiguration config, + @Named("backup") IBackupFileSystem fs, + Sleeper sleeper, + ICassandraProcess cassProcess, + Provider pathProvider, + InstanceIdentity instanceIdentity, + RestoreTokenSelector tokenSelector, + MetaData metaData, + InstanceState instanceState, + IPostRestoreHook postRestoreHook) { + super( + config, + fs, + JOBNAME, + sleeper, + pathProvider, + instanceIdentity, + tokenSelector, + cassProcess, + metaData, + instanceState, + postRestoreHook); } @Override - protected final Future downloadFile(final AbstractBackupPath path, final File restoreLocation) throws Exception { + protected final Future downloadFile( + final AbstractBackupPath path, final File restoreLocation) throws Exception { tracker.adjustAndAdd(path); - return fs.asyncDownloadFile(Paths.get(path.getRemotePath()), Paths.get(restoreLocation.getAbsolutePath()), 5); + return fs.asyncDownloadFile( + Paths.get(path.getRemotePath()), Paths.get(restoreLocation.getAbsolutePath()), 5); } public static TaskTimer getTimer() { @@ -67,4 +85,4 @@ public static TaskTimer getTimer() { public String getName() { return JOBNAME; } -} \ No newline at end of file +} diff --git a/priam/src/main/java/com/netflix/priam/restore/RestoreContext.java b/priam/src/main/java/com/netflix/priam/restore/RestoreContext.java index 284a1cebf..ba54a819d 100755 --- a/priam/src/main/java/com/netflix/priam/restore/RestoreContext.java +++ b/priam/src/main/java/com/netflix/priam/restore/RestoreContext.java @@ -1,16 +1,14 @@ /** * Copyright 2017 Netflix, Inc. - *

- * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and + * + *

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file + * except in compliance with the License. You may obtain a copy of the License at + * + *

http://www.apache.org/licenses/LICENSE-2.0 + * + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing permissions and * limitations under the License. */ package com.netflix.priam.restore; @@ -37,37 +35,48 @@ public RestoreContext(IConfiguration config, PriamScheduler scheduler) { this.scheduler = scheduler; } - public boolean isRestoreEnabled(){ + public boolean isRestoreEnabled() { return !StringUtils.isEmpty(config.getRestoreSnapshot()); } public void restore() throws Exception { - if (!isRestoreEnabled()) - return; + if (!isRestoreEnabled()) return; - //Restore is required. + // Restore is required. if (StringUtils.isEmpty(config.getRestoreSourceType()) && !config.isRestoreEncrypted()) { - //Restore is needed and it will be done from the primary AWS account - scheduler.addTask(Restore.JOBNAME, Restore.class, Restore.getTimer());//restore from the AWS primary acct + // Restore is needed and it will be done from the primary AWS account + scheduler.addTask( + Restore.JOBNAME, + Restore.class, + Restore.getTimer()); // restore from the AWS primary acct logger.info("Scheduled task " + Restore.JOBNAME); } else if (config.isRestoreEncrypted()) { SourceType sourceType = SourceType.lookup(config.getRestoreSourceType(), true, false); - if (sourceType == null) - { - scheduler.addTask(EncryptedRestoreStrategy.JOBNAME, EncryptedRestoreStrategy.class, EncryptedRestoreStrategy.getTimer()); + if (sourceType == null) { + scheduler.addTask( + EncryptedRestoreStrategy.JOBNAME, + EncryptedRestoreStrategy.class, + EncryptedRestoreStrategy.getTimer()); logger.info("Scheduled task " + Restore.JOBNAME); return; } switch (sourceType) { case AWSCROSSACCT: - scheduler.addTask(AwsCrossAccountCryptographyRestoreStrategy.JOBNAME, AwsCrossAccountCryptographyRestoreStrategy.class, AwsCrossAccountCryptographyRestoreStrategy.getTimer()); - logger.info("Scheduled task " + AwsCrossAccountCryptographyRestoreStrategy.JOBNAME); + scheduler.addTask( + AwsCrossAccountCryptographyRestoreStrategy.JOBNAME, + AwsCrossAccountCryptographyRestoreStrategy.class, + AwsCrossAccountCryptographyRestoreStrategy.getTimer()); + logger.info( + "Scheduled task " + AwsCrossAccountCryptographyRestoreStrategy.JOBNAME); break; case GOOGLE: - scheduler.addTask(GoogleCryptographyRestoreStrategy.JOBNAME, GoogleCryptographyRestoreStrategy.class, GoogleCryptographyRestoreStrategy.getTimer()); + scheduler.addTask( + GoogleCryptographyRestoreStrategy.JOBNAME, + GoogleCryptographyRestoreStrategy.class, + GoogleCryptographyRestoreStrategy.getTimer()); logger.info("Scheduled task " + GoogleCryptographyRestoreStrategy.JOBNAME); break; } @@ -75,7 +84,8 @@ public void restore() throws Exception { } enum SourceType { - AWSCROSSACCT("AWSCROSSACCT"), GOOGLE("GOOGLE"); + AWSCROSSACCT("AWSCROSSACCT"), + GOOGLE("GOOGLE"); private static final Logger logger = LoggerFactory.getLogger(SourceType.class); @@ -85,12 +95,16 @@ enum SourceType { this.sourceType = sourceType.toUpperCase(); } - public static SourceType lookup(String sourceType, boolean acceptNullOrEmpty, boolean acceptIllegalValue) throws UnsupportedTypeException { + public static SourceType lookup( + String sourceType, boolean acceptNullOrEmpty, boolean acceptIllegalValue) + throws UnsupportedTypeException { if (StringUtils.isEmpty(sourceType)) - if (acceptNullOrEmpty) - return null; + if (acceptNullOrEmpty) return null; else { - String message = String.format("%s is not a supported SourceType. Supported values are %s", sourceType, getSupportedValues()); + String message = + String.format( + "%s is not a supported SourceType. Supported values are %s", + sourceType, getSupportedValues()); logger.error(message); throw new UnsupportedTypeException(message); } @@ -98,10 +112,15 @@ public static SourceType lookup(String sourceType, boolean acceptNullOrEmpty, bo try { return SourceType.valueOf(sourceType.toUpperCase()); } catch (IllegalArgumentException ex) { - String message = String.format("%s is not a supported SourceType. Supported values are %s", sourceType, getSupportedValues()); + String message = + String.format( + "%s is not a supported SourceType. Supported values are %s", + sourceType, getSupportedValues()); if (acceptIllegalValue) { - message = message + ". Since acceptIllegalValue is set to True, returning NULL instead."; + message = + message + + ". Since acceptIllegalValue is set to True, returning NULL instead."; logger.error(message); return null; } @@ -111,13 +130,11 @@ public static SourceType lookup(String sourceType, boolean acceptNullOrEmpty, bo } } - private static String getSupportedValues() { StringBuilder supportedValues = new StringBuilder(); boolean first = true; for (SourceType type : SourceType.values()) { - if (!first) - supportedValues.append(","); + if (!first) supportedValues.append(","); supportedValues.append(type); first = false; } @@ -133,4 +150,4 @@ public String getSourceType() { return sourceType; } } -} \ No newline at end of file +} diff --git a/priam/src/main/java/com/netflix/priam/restore/RestoreTokenSelector.java b/priam/src/main/java/com/netflix/priam/restore/RestoreTokenSelector.java index 9d80bb472..7f0563cf8 100644 --- a/priam/src/main/java/com/netflix/priam/restore/RestoreTokenSelector.java +++ b/priam/src/main/java/com/netflix/priam/restore/RestoreTokenSelector.java @@ -21,44 +21,35 @@ import com.netflix.priam.backup.AbstractBackupPath; import com.netflix.priam.backup.IBackupFileSystem; import com.netflix.priam.utils.ITokenManager; - import java.math.BigInteger; import java.util.ArrayList; import java.util.Date; import java.util.Iterator; import java.util.List; -/** - * Runs algorithms as finding closest token from a list of token (in a backup) - */ +/** Runs algorithms as finding closest token from a list of token (in a backup) */ public class RestoreTokenSelector { private final ITokenManager tokenManager; private final IBackupFileSystem fs; @Inject + public RestoreTokenSelector(ITokenManager tokenManager, @Named("backup") IBackupFileSystem fs) { - public RestoreTokenSelector(ITokenManager tokenManager, @Named("backup") IBackupFileSystem fs) - - { this.tokenManager = tokenManager; this.fs = fs; } /** - * Get the closest token to current token from the list of tokens available - * in the backup + * Get the closest token to current token from the list of tokens available in the backup * - * @param tokenToSearch - * Token to search for - * @param startDate - * Date for which the backups are available + * @param tokenToSearch Token to search for + * @param startDate Date for which the backups are available * @return Token as BigInteger */ public BigInteger getClosestToken(BigInteger tokenToSearch, Date startDate) { List tokenList = new ArrayList<>(); Iterator iter = fs.listPrefixes(startDate); - while (iter.hasNext()) - tokenList.add(new BigInteger(iter.next().getToken())); + while (iter.hasNext()) tokenList.add(new BigInteger(iter.next().getToken())); return tokenManager.findClosestToken(tokenToSearch, tokenList); } } diff --git a/priam/src/main/java/com/netflix/priam/scheduler/BlockingSubmitThreadPoolExecutor.java b/priam/src/main/java/com/netflix/priam/scheduler/BlockingSubmitThreadPoolExecutor.java index 9f8a4cad4..3a047a7a4 100644 --- a/priam/src/main/java/com/netflix/priam/scheduler/BlockingSubmitThreadPoolExecutor.java +++ b/priam/src/main/java/com/netflix/priam/scheduler/BlockingSubmitThreadPoolExecutor.java @@ -16,25 +16,26 @@ */ package com.netflix.priam.scheduler; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import java.util.concurrent.*; import java.util.concurrent.atomic.AtomicInteger; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** - * {@link ThreadPoolExecutor} that will block in the {@code submit()} method - * until the task can be successfully added to the queue. + * {@link ThreadPoolExecutor} that will block in the {@code submit()} method until the task can be + * successfully added to the queue. */ public class BlockingSubmitThreadPoolExecutor extends ThreadPoolExecutor { private static final long DEFAULT_SLEEP = 100; private static final long DEFAULT_KEEP_ALIVE = 100; - private static final Logger logger = LoggerFactory.getLogger(BlockingSubmitThreadPoolExecutor.class); + private static final Logger logger = + LoggerFactory.getLogger(BlockingSubmitThreadPoolExecutor.class); private final BlockingQueue queue; private final long giveupTime; private final AtomicInteger active; - public BlockingSubmitThreadPoolExecutor(int maximumPoolSize, BlockingQueue workQueue, long timeoutAdding) { + public BlockingSubmitThreadPoolExecutor( + int maximumPoolSize, BlockingQueue workQueue, long timeoutAdding) { super(maximumPoolSize, maximumPoolSize, DEFAULT_KEEP_ALIVE, TimeUnit.SECONDS, workQueue); this.queue = workQueue; this.giveupTime = timeoutAdding; @@ -42,9 +43,8 @@ public BlockingSubmitThreadPoolExecutor(int maximumPoolSize, BlockingQueue Future submit(Callable task) { @@ -73,9 +73,7 @@ protected void afterExecute(Runnable r, Throwable t) { active.decrementAndGet(); } - /** - * blocking call to test if the threads are done or not. - */ + /** blocking call to test if the threads are done or not. */ public void sleepTillEmpty() { long timeout = 0; @@ -84,7 +82,8 @@ public void sleepTillEmpty() { if (timeout <= giveupTime) { Thread.sleep(DEFAULT_SLEEP); timeout += DEFAULT_SLEEP; - logger.debug("After Sleeping for empty: {}, Count: {}", +queue.size(), active.get()); + logger.debug( + "After Sleeping for empty: {}, Count: {}", +queue.size(), active.get()); } else { throw new RuntimeException("Timed out because TPE is too busy..."); } @@ -92,6 +91,5 @@ public void sleepTillEmpty() { throw new RuntimeException(e); } } - } } diff --git a/priam/src/main/java/com/netflix/priam/scheduler/CronTimer.java b/priam/src/main/java/com/netflix/priam/scheduler/CronTimer.java index d5997d0b6..b190b6198 100644 --- a/priam/src/main/java/com/netflix/priam/scheduler/CronTimer.java +++ b/priam/src/main/java/com/netflix/priam/scheduler/CronTimer.java @@ -16,23 +16,26 @@ */ package com.netflix.priam.scheduler; +import java.text.ParseException; import org.apache.commons.lang3.StringUtils; import org.quartz.*; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.text.ParseException; - -/** - * Runs jobs at the specified absolute time and frequency - */ +/** Runs jobs at the specified absolute time and frequency */ public class CronTimer implements TaskTimer { private static final Logger logger = LoggerFactory.getLogger(CronTimer.class); private final String cronExpression; private String name; public enum DayOfWeek { - SUN, MON, TUE, WED, THU, FRI, SAT + SUN, + MON, + TUE, + WED, + THU, + FRI, + SAT } /* @@ -43,33 +46,25 @@ public CronTimer(String name, int min) { cronExpression = "*" + " " + "0/" + min + " " + "* * * ?"; } - /** - * Hourly cron. - */ + /** Hourly cron. */ public CronTimer(String name, int minute, int sec) { this.name = name; cronExpression = sec + " " + minute + " 0/1 * * ?"; } - /** - * Daily Cron - */ + /** Daily Cron */ public CronTimer(String name, int hour, int minute, int sec) { this.name = name; cronExpression = sec + " " + minute + " " + hour + " * * ?"; } - /** - * Weekly cron jobs - */ + /** Weekly cron jobs */ public CronTimer(String name, DayOfWeek dayofweek, int hour, int minute, int sec) { this.name = name; cronExpression = sec + " " + minute + " " + hour + " * * " + dayofweek; } - /** - * Cron Expression. - */ + /** Cron Expression. */ public CronTimer(String expression) { this.cronExpression = expression; } @@ -80,7 +75,10 @@ public CronTimer(String name, String expression) { } public Trigger getTrigger() throws ParseException { - return TriggerBuilder.newTrigger().withIdentity(name, Scheduler.DEFAULT_GROUP).withSchedule(CronScheduleBuilder.cronSchedule(cronExpression)).build(); + return TriggerBuilder.newTrigger() + .withIdentity(name, Scheduler.DEFAULT_GROUP) + .withSchedule(CronScheduleBuilder.cronSchedule(cronExpression)) + .build(); } @Override @@ -88,18 +86,26 @@ public String getCronExpression() { return this.cronExpression; } - public static CronTimer getCronTimer(final String jobName, final String cronExpression) throws IllegalArgumentException { + public static CronTimer getCronTimer(final String jobName, final String cronExpression) + throws IllegalArgumentException { CronTimer cronTimer = null; if (!StringUtils.isEmpty(cronExpression) && cronExpression.equalsIgnoreCase("-1")) { - logger.info("Skipping {} as it is disabled via setting {} cron to -1.", jobName, jobName); + logger.info( + "Skipping {} as it is disabled via setting {} cron to -1.", jobName, jobName); } else { - if (StringUtils.isEmpty(cronExpression) || !CronExpression.isValidExpression(cronExpression)) - throw new IllegalArgumentException("Invalid CRON expression: " + cronExpression + - ". Please use -1, if you wish to disable " + jobName + " else fix the CRON expression and try again!"); + if (StringUtils.isEmpty(cronExpression) + || !CronExpression.isValidExpression(cronExpression)) + throw new IllegalArgumentException( + "Invalid CRON expression: " + + cronExpression + + ". Please use -1, if you wish to disable " + + jobName + + " else fix the CRON expression and try again!"); cronTimer = new CronTimer(jobName, cronExpression); - logger.info("Starting {} with CRON expression {}", jobName, cronTimer.getCronExpression()); + logger.info( + "Starting {} with CRON expression {}", jobName, cronTimer.getCronExpression()); } return cronTimer; } diff --git a/priam/src/main/java/com/netflix/priam/scheduler/NamedThreadPoolExecutor.java b/priam/src/main/java/com/netflix/priam/scheduler/NamedThreadPoolExecutor.java index 2a6f282df..8649f1ef8 100644 --- a/priam/src/main/java/com/netflix/priam/scheduler/NamedThreadPoolExecutor.java +++ b/priam/src/main/java/com/netflix/priam/scheduler/NamedThreadPoolExecutor.java @@ -17,7 +17,6 @@ package com.netflix.priam.scheduler; import com.google.common.util.concurrent.ThreadFactoryBuilder; - import java.util.concurrent.*; public class NamedThreadPoolExecutor extends ThreadPoolExecutor { @@ -26,7 +25,12 @@ public NamedThreadPoolExecutor(int poolSize, String poolName) { } public NamedThreadPoolExecutor(int poolSize, String poolName, BlockingQueue queue) { - super(poolSize, poolSize, 1000, TimeUnit.MILLISECONDS, queue, + super( + poolSize, + poolSize, + 1000, + TimeUnit.MILLISECONDS, + queue, new ThreadFactoryBuilder().setDaemon(true).setNameFormat(poolName + "-%d").build(), new LocalRejectedExecutionHandler(queue)); } @@ -44,10 +48,9 @@ public void rejectedExecution(Runnable task, ThreadPoolExecutor executor) { throw new RejectedExecutionException("ThreadPoolExecutor has shut down"); try { - if (queue.offer(task, 1000, TimeUnit.MILLISECONDS)) - break; + if (queue.offer(task, 1000, TimeUnit.MILLISECONDS)) break; } catch (InterruptedException e) { - //NOP + // NOP } } } diff --git a/priam/src/main/java/com/netflix/priam/scheduler/PriamScheduler.java b/priam/src/main/java/com/netflix/priam/scheduler/PriamScheduler.java index 08339c07f..5c61120f3 100644 --- a/priam/src/main/java/com/netflix/priam/scheduler/PriamScheduler.java +++ b/priam/src/main/java/com/netflix/priam/scheduler/PriamScheduler.java @@ -19,15 +19,12 @@ import com.google.inject.Inject; import com.google.inject.Singleton; import com.netflix.priam.utils.Sleeper; +import java.text.ParseException; import org.quartz.*; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.text.ParseException; - -/** - * Scheduling class to schedule Priam tasks. Uses Quartz scheduler - */ +/** Scheduling class to schedule Priam tasks. Uses Quartz scheduler */ @Singleton public class PriamScheduler { private static final Logger logger = LoggerFactory.getLogger(PriamScheduler.class); @@ -47,15 +44,20 @@ public PriamScheduler(SchedulerFactory factory, GuiceJobFactory jobFactory, Slee this.sleeper = sleeper; } - /** - * Add a task to the scheduler - */ - public void addTask(String name, Class taskclass, TaskTimer timer) throws SchedulerException, ParseException { + /** Add a task to the scheduler */ + public void addTask(String name, Class taskclass, TaskTimer timer) + throws SchedulerException, ParseException { assert timer != null : "Cannot add scheduler task " + name + " as no timer is set"; - JobDetail job = JobBuilder.newJob().withIdentity(name, Scheduler.DEFAULT_GROUP).ofType(taskclass).build();//new JobDetail(name, Scheduler.DEFAULT_GROUP, taskclass); + JobDetail job = + JobBuilder.newJob() + .withIdentity(name, Scheduler.DEFAULT_GROUP) + .ofType(taskclass) + .build(); // new JobDetail(name, Scheduler.DEFAULT_GROUP, taskclass); if (timer.getCronExpression() != null && !timer.getCronExpression().isEmpty()) { - logger.info("Scheduled task metadata. Task name: {}" - + ", cron expression: {}", taskclass.getName(), timer.getCronExpression()); + logger.info( + "Scheduled task metadata. Task name: {}" + ", cron expression: {}", + taskclass.getName(), + timer.getCronExpression()); } else { logger.info("Scheduled task metadata. Task name: {}", taskclass.getName()); @@ -63,24 +65,39 @@ public void addTask(String name, Class taskclass, TaskTimer time scheduler.scheduleJob(job, timer.getTrigger()); } - /** - * Add a delayed task to the scheduler - */ - public void addTaskWithDelay(final String name, Class taskclass, final TaskTimer timer, final int delayInSeconds) throws SchedulerException, ParseException { + /** Add a delayed task to the scheduler */ + public void addTaskWithDelay( + final String name, + Class taskclass, + final TaskTimer timer, + final int delayInSeconds) + throws SchedulerException, ParseException { assert timer != null : "Cannot add scheduler task " + name + " as no timer is set"; - final JobDetail job = JobBuilder.newJob().withIdentity(name, Scheduler.DEFAULT_GROUP).ofType(taskclass).build();//new JobDetail(name, Scheduler.DEFAULT_GROUP, taskclass); + final JobDetail job = + JobBuilder.newJob() + .withIdentity(name, Scheduler.DEFAULT_GROUP) + .ofType(taskclass) + .build(); // new JobDetail(name, Scheduler.DEFAULT_GROUP, taskclass); - //we know Priam doesn't do too many new tasks, so this is probably easy/safe/simple - new Thread(() -> { - try { - sleeper.sleepQuietly(delayInSeconds * 1000L); - scheduler.scheduleJob(job, timer.getTrigger()); - } catch (SchedulerException e) { - logger.warn("problem occurred while scheduling a job with name {}", name, e); - } catch (ParseException e) { - logger.warn("problem occurred while parsing a job with name {}", name, e); - } - }).start(); + // we know Priam doesn't do too many new tasks, so this is probably easy/safe/simple + new Thread( + () -> { + try { + sleeper.sleepQuietly(delayInSeconds * 1000L); + scheduler.scheduleJob(job, timer.getTrigger()); + } catch (SchedulerException e) { + logger.warn( + "problem occurred while scheduling a job with name {}", + name, + e); + } catch (ParseException e) { + logger.warn( + "problem occurred while parsing a job with name {}", + name, + e); + } + }) + .start(); } public void runTaskNow(Class taskclass) throws Exception { diff --git a/priam/src/main/java/com/netflix/priam/scheduler/SchedulerType.java b/priam/src/main/java/com/netflix/priam/scheduler/SchedulerType.java index 6f3221e9e..0663f5455 100644 --- a/priam/src/main/java/com/netflix/priam/scheduler/SchedulerType.java +++ b/priam/src/main/java/com/netflix/priam/scheduler/SchedulerType.java @@ -20,12 +20,10 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; - -/** - * Created by aagrawal on 3/8/17. - */ +/** Created by aagrawal on 3/8/17. */ public enum SchedulerType { - HOUR("HOUR"), CRON("CRON"); + HOUR("HOUR"), + CRON("CRON"); private static final Logger logger = LoggerFactory.getLogger(SchedulerType.class); private final String schedulerType; @@ -48,12 +46,16 @@ public enum SchedulerType { * Illegal value |NA |False |UnsupportedTypeException */ - public static SchedulerType lookup(String schedulerType, boolean acceptNullOrEmpty, boolean acceptIllegalValue) throws UnsupportedTypeException { + public static SchedulerType lookup( + String schedulerType, boolean acceptNullOrEmpty, boolean acceptIllegalValue) + throws UnsupportedTypeException { if (StringUtils.isEmpty(schedulerType)) - if (acceptNullOrEmpty) - return null; + if (acceptNullOrEmpty) return null; else { - String message = String.format("%s is not a supported SchedulerType. Supported values are %s", schedulerType, getSupportedValues()); + String message = + String.format( + "%s is not a supported SchedulerType. Supported values are %s", + schedulerType, getSupportedValues()); logger.error(message); throw new UnsupportedTypeException(message); } @@ -61,10 +63,15 @@ public static SchedulerType lookup(String schedulerType, boolean acceptNullOrEmp try { return SchedulerType.valueOf(schedulerType.toUpperCase()); } catch (IllegalArgumentException ex) { - String message = String.format("%s is not a supported SchedulerType. Supported values are %s", schedulerType, getSupportedValues()); + String message = + String.format( + "%s is not a supported SchedulerType. Supported values are %s", + schedulerType, getSupportedValues()); if (acceptIllegalValue) { - message = message + ". Since acceptIllegalValue is set to True, returning NULL instead."; + message = + message + + ". Since acceptIllegalValue is set to True, returning NULL instead."; logger.error(message); return null; } @@ -78,8 +85,7 @@ private static String getSupportedValues() { StringBuilder supportedValues = new StringBuilder(); boolean first = true; for (SchedulerType type : SchedulerType.values()) { - if (!first) - supportedValues.append(","); + if (!first) supportedValues.append(","); supportedValues.append(type); first = false; } @@ -94,5 +100,4 @@ public static SchedulerType lookup(String schedulerType) throws UnsupportedTypeE public String getSchedulerType() { return schedulerType; } - } diff --git a/priam/src/main/java/com/netflix/priam/scheduler/SimpleTimer.java b/priam/src/main/java/com/netflix/priam/scheduler/SimpleTimer.java index eaf8a416d..4d5416a72 100644 --- a/priam/src/main/java/com/netflix/priam/scheduler/SimpleTimer.java +++ b/priam/src/main/java/com/netflix/priam/scheduler/SimpleTimer.java @@ -16,52 +16,57 @@ */ package com.netflix.priam.scheduler; +import java.text.ParseException; +import java.util.Date; import org.quartz.Scheduler; import org.quartz.SimpleScheduleBuilder; import org.quartz.Trigger; import org.quartz.TriggerBuilder; -import java.text.ParseException; -import java.util.Date; - /** - * SimpleTimer allows jobs to run starting from specified time occurring at - * regular frequency's. Frequency of the execution timestamp since epoch. + * SimpleTimer allows jobs to run starting from specified time occurring at regular frequency's. + * Frequency of the execution timestamp since epoch. */ public class SimpleTimer implements TaskTimer { private final Trigger trigger; public SimpleTimer(String name, long interval) { - this.trigger = TriggerBuilder.newTrigger() - .withIdentity(name) - .withSchedule(SimpleScheduleBuilder.simpleSchedule().withIntervalInMilliseconds(interval) - .repeatForever().withMisfireHandlingInstructionFireNow()) - .build(); - //.new SimpleTrigger(name, SimpleTrigger.REPEAT_INDEFINITELY, interval); + this.trigger = + TriggerBuilder.newTrigger() + .withIdentity(name) + .withSchedule( + SimpleScheduleBuilder.simpleSchedule() + .withIntervalInMilliseconds(interval) + .repeatForever() + .withMisfireHandlingInstructionFireNow()) + .build(); + // .new SimpleTrigger(name, SimpleTrigger.REPEAT_INDEFINITELY, interval); } - /** - * Run once at given time... - */ + /** Run once at given time... */ public SimpleTimer(String name, String group, long startTime) { - this.trigger = TriggerBuilder.newTrigger() - .withIdentity(name, group) - .withSchedule(SimpleScheduleBuilder.simpleSchedule().withMisfireHandlingInstructionFireNow()) - .startAt(new Date(startTime)) - .build(); - //new SimpleTrigger(name, group, new Date(startTime)); + this.trigger = + TriggerBuilder.newTrigger() + .withIdentity(name, group) + .withSchedule( + SimpleScheduleBuilder.simpleSchedule() + .withMisfireHandlingInstructionFireNow()) + .startAt(new Date(startTime)) + .build(); + // new SimpleTrigger(name, group, new Date(startTime)); } - /** - * Run immediatly and dont do that again. - */ + /** Run immediatly and dont do that again. */ public SimpleTimer(String name) { - this.trigger = TriggerBuilder.newTrigger() - .withIdentity(name, Scheduler.DEFAULT_GROUP) - .withSchedule(SimpleScheduleBuilder.simpleSchedule().withMisfireHandlingInstructionFireNow()) - .startNow() - .build(); - //new SimpleTrigger(name, Scheduler.DEFAULT_GROUP); + this.trigger = + TriggerBuilder.newTrigger() + .withIdentity(name, Scheduler.DEFAULT_GROUP) + .withSchedule( + SimpleScheduleBuilder.simpleSchedule() + .withMisfireHandlingInstructionFireNow()) + .startNow() + .build(); + // new SimpleTrigger(name, Scheduler.DEFAULT_GROUP); } public Trigger getTrigger() throws ParseException { diff --git a/priam/src/main/java/com/netflix/priam/scheduler/Task.java b/priam/src/main/java/com/netflix/priam/scheduler/Task.java index 16c5130da..bc9592a1a 100644 --- a/priam/src/main/java/com/netflix/priam/scheduler/Task.java +++ b/priam/src/main/java/com/netflix/priam/scheduler/Task.java @@ -17,28 +17,31 @@ package com.netflix.priam.scheduler; import com.netflix.priam.config.IConfiguration; +import java.lang.management.ManagementFactory; +import java.util.concurrent.atomic.AtomicInteger; +import javax.management.MBeanServer; +import javax.management.ObjectName; import org.quartz.Job; import org.quartz.JobExecutionContext; import org.quartz.JobExecutionException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import javax.management.MBeanServer; -import javax.management.ObjectName; -import java.lang.management.ManagementFactory; -import java.util.concurrent.atomic.AtomicInteger; - /** - * Task class that should be implemented by all cron tasks. Jobconf will contain - * any instance specific data + * Task class that should be implemented by all cron tasks. Jobconf will contain any instance + * specific data * - * NOTE: Constructor must not throw any exception. This will cause Quartz to set the job to failure + *

NOTE: Constructor must not throw any exception. This will cause Quartz to set the job to + * failure */ public abstract class Task implements Job, TaskMBean { public STATE status = STATE.DONE; public enum STATE { - ERROR, RUNNING, DONE, NOT_APPLICABLE + ERROR, + RUNNING, + DONE, + NOT_APPLICABLE } protected final IConfiguration config; @@ -63,24 +66,18 @@ protected Task(IConfiguration config, MBeanServer mBeanServer) { } } - - /** - * This method has to be implemented and cannot thow any exception. - */ + /** This method has to be implemented and cannot thow any exception. */ public void initialize() throws ExecutionException { // nothing to initialize } public abstract void execute() throws Exception; - /** - * Main method to execute a task - */ + /** Main method to execute a task */ public void execute(JobExecutionContext context) throws JobExecutionException { executions.incrementAndGet(); try { - if (status == STATE.RUNNING) - return; + if (status == STATE.RUNNING) return; status = STATE.RUNNING; execute(); @@ -89,8 +86,7 @@ public void execute(JobExecutionContext context) throws JobExecutionException { logger.error("Couldnt execute the task because of {}", e.getMessage(), e); errors.incrementAndGet(); } - if (status != STATE.ERROR) - status = STATE.DONE; + if (status != STATE.ERROR) status = STATE.DONE; } public STATE state() { @@ -106,5 +102,4 @@ public int getExecutionCount() { } public abstract String getName(); - -} \ No newline at end of file +} diff --git a/priam/src/main/java/com/netflix/priam/scheduler/TaskMBean.java b/priam/src/main/java/com/netflix/priam/scheduler/TaskMBean.java index e25666867..471e88051 100644 --- a/priam/src/main/java/com/netflix/priam/scheduler/TaskMBean.java +++ b/priam/src/main/java/com/netflix/priam/scheduler/TaskMBean.java @@ -16,10 +16,7 @@ */ package com.netflix.priam.scheduler; -/** - * MBean to monitor Task executions. - * - */ +/** MBean to monitor Task executions. */ public interface TaskMBean { int getErrorCount(); diff --git a/priam/src/main/java/com/netflix/priam/scheduler/TaskTimer.java b/priam/src/main/java/com/netflix/priam/scheduler/TaskTimer.java index 052a112ce..5d790293f 100644 --- a/priam/src/main/java/com/netflix/priam/scheduler/TaskTimer.java +++ b/priam/src/main/java/com/netflix/priam/scheduler/TaskTimer.java @@ -16,13 +16,10 @@ */ package com.netflix.priam.scheduler; -import org.quartz.Trigger; - import java.text.ParseException; +import org.quartz.Trigger; -/** - * Interface to represent time/interval - */ +/** Interface to represent time/interval */ public interface TaskTimer { Trigger getTrigger() throws ParseException; diff --git a/priam/src/main/java/com/netflix/priam/scheduler/UnsupportedTypeException.java b/priam/src/main/java/com/netflix/priam/scheduler/UnsupportedTypeException.java index 06737e162..36385e4c1 100644 --- a/priam/src/main/java/com/netflix/priam/scheduler/UnsupportedTypeException.java +++ b/priam/src/main/java/com/netflix/priam/scheduler/UnsupportedTypeException.java @@ -1,24 +1,19 @@ /** * Copyright 2017 Netflix, Inc. - *

- * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and + * + *

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file + * except in compliance with the License. You may obtain a copy of the License at + * + *

http://www.apache.org/licenses/LICENSE-2.0 + * + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing permissions and * limitations under the License. */ - package com.netflix.priam.scheduler; -/** - * Created by aagrawal on 3/14/17. - */ +/** Created by aagrawal on 3/14/17. */ public class UnsupportedTypeException extends Exception { public UnsupportedTypeException(String msg, Throwable th) { super(msg, th); diff --git a/priam/src/main/java/com/netflix/priam/services/SnapshotMetaService.java b/priam/src/main/java/com/netflix/priam/services/SnapshotMetaService.java index e97570ed9..568e798ae 100644 --- a/priam/src/main/java/com/netflix/priam/services/SnapshotMetaService.java +++ b/priam/src/main/java/com/netflix/priam/services/SnapshotMetaService.java @@ -1,33 +1,36 @@ /** * Copyright 2018 Netflix, Inc. - *

- * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and + * + *

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file + * except in compliance with the License. You may obtain a copy of the License at + * + *

http://www.apache.org/licenses/LICENSE-2.0 + * + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing permissions and * limitations under the License. */ package com.netflix.priam.services; import com.google.inject.Provider; -import com.netflix.priam.config.IConfiguration; import com.netflix.priam.backup.AbstractBackup; import com.netflix.priam.backup.AbstractBackupPath; import com.netflix.priam.backup.BackupRestoreUtil; import com.netflix.priam.backup.IFileSystemContext; import com.netflix.priam.backupv2.*; import com.netflix.priam.config.IBackupRestoreConfig; +import com.netflix.priam.config.IConfiguration; import com.netflix.priam.defaultimpl.CassandraOperations; import com.netflix.priam.scheduler.CronTimer; import com.netflix.priam.scheduler.TaskTimer; import com.netflix.priam.utils.CassandraMonitor; import com.netflix.priam.utils.DateUtil; +import java.io.File; +import java.time.Instant; +import java.util.*; +import javax.inject.Inject; +import javax.inject.Singleton; import org.apache.commons.io.FileUtils; import org.apache.commons.io.filefilter.FileFilterUtils; import org.apache.commons.lang3.StringUtils; @@ -35,25 +38,19 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import javax.inject.Inject; -import javax.inject.Singleton; -import java.io.File; -import java.time.Instant; -import java.util.*; - /** - * This service will run on CRON as specified by {@link IBackupRestoreConfig#getSnapshotMetaServiceCronExpression()} - * The intent of this service is to run a full snapshot on Cassandra, get the list of the SSTables on disk - * and then create a manifest.json file which will encapsulate the list of the files i.e. capture filesystem at a moment - * in time. - * This manifest.json file will ensure the true filesystem status is exposed (for external entities) and will be - * used in future for Priam Backup Version 2 where a file is not uploaded to backup file system unless SSTable has - * been modified. This will lead to huge reduction in storage costs and provide bandwidth back to Cassandra instead - * of creating/uploading snapshots. - * Note that this component will "try" to enqueue the files to upload, but no guarantee is provided. If the enqueue fails - * for any reason, it is considered "OK" as there will be another service pushing all the files in the queue for upload - * (think of this like a cleanup thread and will help us in "resuming" any failed backup for any reason). - * Created by aagrawal on 6/18/18. + * This service will run on CRON as specified by {@link + * IBackupRestoreConfig#getSnapshotMetaServiceCronExpression()} The intent of this service is to run + * a full snapshot on Cassandra, get the list of the SSTables on disk and then create a + * manifest.json file which will encapsulate the list of the files i.e. capture filesystem at a + * moment in time. This manifest.json file will ensure the true filesystem status is exposed (for + * external entities) and will be used in future for Priam Backup Version 2 where a file is not + * uploaded to backup file system unless SSTable has been modified. This will lead to huge reduction + * in storage costs and provide bandwidth back to Cassandra instead of creating/uploading snapshots. + * Note that this component will "try" to enqueue the files to upload, but no guarantee is provided. + * If the enqueue fails for any reason, it is considered "OK" as there will be another service + * pushing all the files in the queue for upload (think of this like a cleanup thread and will help + * us in "resuming" any failed backup for any reason). Created by aagrawal on 6/18/18. */ @Singleton public class SnapshotMetaService extends AbstractBackup { @@ -70,35 +67,52 @@ public class SnapshotMetaService extends AbstractBackup { private String snapshotName = null; @Inject - SnapshotMetaService(IConfiguration config, IFileSystemContext backupFileSystemCtx, Provider pathFactory, - MetaFileWriterBuilder metaFileWriter, MetaFileManager metaFileManager, CassandraOperations cassandraOperations) { + SnapshotMetaService( + IConfiguration config, + IFileSystemContext backupFileSystemCtx, + Provider pathFactory, + MetaFileWriterBuilder metaFileWriter, + MetaFileManager metaFileManager, + CassandraOperations cassandraOperations) { super(config, backupFileSystemCtx, pathFactory); this.cassandraOperations = cassandraOperations; - backupRestoreUtil = new BackupRestoreUtil(config.getSnapshotIncludeCFList(), config.getSnapshotExcludeCFList()); + backupRestoreUtil = + new BackupRestoreUtil( + config.getSnapshotIncludeCFList(), config.getSnapshotExcludeCFList()); this.metaFileWriter = metaFileWriter; this.metaFileManager = metaFileManager; } /** - * Interval between generating snapshot meta file using {@link com.netflix.priam.services.SnapshotMetaService}. + * Interval between generating snapshot meta file using {@link + * com.netflix.priam.services.SnapshotMetaService}. * - * @param backupRestoreConfig {@link IBackupRestoreConfig#getSnapshotMetaServiceCronExpression()} to get configuration details from priam. Use "-1" to disable the service. + * @param backupRestoreConfig {@link + * IBackupRestoreConfig#getSnapshotMetaServiceCronExpression()} to get configuration details + * from priam. Use "-1" to disable the service. * @return the timer to be used for snapshot meta service. - * @throws Exception if the configuration is not set correctly or are not valid. This is to ensure we fail-fast. - **/ + * @throws Exception if the configuration is not set correctly or are not valid. This is to + * ensure we fail-fast. + */ public static TaskTimer getTimer(IBackupRestoreConfig backupRestoreConfig) throws Exception { CronTimer cronTimer = null; String cronExpression = backupRestoreConfig.getSnapshotMetaServiceCronExpression(); if (!StringUtils.isEmpty(cronExpression) && cronExpression.equalsIgnoreCase("-1")) { - logger.info("Skipping SnapshotMetaService as SnapshotMetaService cron is disabled via -1."); + logger.info( + "Skipping SnapshotMetaService as SnapshotMetaService cron is disabled via -1."); } else { - if (StringUtils.isEmpty(cronExpression) || !CronExpression.isValidExpression(cronExpression)) - throw new Exception("Invalid CRON expression: " + cronExpression + - ". Please use -1, if you wish to disable SnapshotMetaService else fix the CRON expression and try again!"); + if (StringUtils.isEmpty(cronExpression) + || !CronExpression.isValidExpression(cronExpression)) + throw new Exception( + "Invalid CRON expression: " + + cronExpression + + ". Please use -1, if you wish to disable SnapshotMetaService else fix the CRON expression and try again!"); cronTimer = new CronTimer(JOBNAME, cronExpression); - logger.info("Starting SnapshotMetaService with CRON expression {}", cronTimer.getCronExpression()); + logger.info( + "Starting SnapshotMetaService with CRON expression {}", + cronTimer.getCronExpression()); } return cronTimer; } @@ -119,26 +133,27 @@ public void execute() throws Exception { snapshotName = generateSnapshotName(snapshotInstant); logger.info("Initializing SnapshotMetaService for taking a snapshot {}", snapshotName); - //Perform a cleanup of old snapshot meta_v2.json files, if any, as we don't want our disk to be filled by them. - //These files may be leftover + // Perform a cleanup of old snapshot meta_v2.json files, if any, as we don't want our + // disk to be filled by them. + // These files may be leftover // 1) when Priam shutdown in middle of this service and may not be full JSON // 2) No permission to upload to backup file system. metaFileManager.cleanupOldMetaFiles(); - //TODO: enqueue all the old backup folder for upload/delete, if any, as we don't want our disk to be filled by them. - //processOldSnapshotV2Folders(); + // TODO: enqueue all the old backup folder for upload/delete, if any, as we don't want + // our disk to be filled by them. + // processOldSnapshotV2Folders(); - //Take a new snapshot + // Take a new snapshot cassandraOperations.takeSnapshot(snapshotName); - //Process the snapshot and upload the meta file. + // Process the snapshot and upload the meta file. processSnapshot(snapshotInstant).uploadMetaFile(true); logger.info("Finished processing snapshot meta service"); } catch (Exception e) { logger.error("Error while executing SnapshotMetaService", e); } - } MetaFileWriterBuilder.UploadStep processSnapshot(Instant snapshotInstant) throws Exception { @@ -149,8 +164,9 @@ MetaFileWriterBuilder.UploadStep processSnapshot(Instant snapshotInstant) throws private File getValidSnapshot(File snapshotDir, String snapshotName) { for (File fileName : snapshotDir.listFiles()) - if (fileName.exists() && fileName.isDirectory() && fileName.getName().matches(snapshotName)) - return fileName; + if (fileName.exists() + && fileName.isDirectory() + && fileName.getName().matches(snapshotName)) return fileName; return null; } @@ -159,20 +175,26 @@ public String getName() { return JOBNAME; } - - private ColumnfamilyResult convertToColumnFamilyResult(String keyspace, String columnFamilyName, Map> filePrefixToFileMap) { + private ColumnfamilyResult convertToColumnFamilyResult( + String keyspace, + String columnFamilyName, + Map> filePrefixToFileMap) { ColumnfamilyResult columnfamilyResult = new ColumnfamilyResult(keyspace, columnFamilyName); - filePrefixToFileMap.forEach((key, value) -> { - ColumnfamilyResult.SSTableResult ssTableResult = new ColumnfamilyResult.SSTableResult(); - ssTableResult.setPrefix(key); - ssTableResult.setSstableComponents(value); - columnfamilyResult.addSstable(ssTableResult); - }); + filePrefixToFileMap.forEach( + (key, value) -> { + ColumnfamilyResult.SSTableResult ssTableResult = + new ColumnfamilyResult.SSTableResult(); + ssTableResult.setPrefix(key); + ssTableResult.setSstableComponents(value); + columnfamilyResult.addSstable(ssTableResult); + }); return columnfamilyResult; } @Override - protected void processColumnFamily(final String keyspace, final String columnFamily, final File backupDir) throws Exception { + protected void processColumnFamily( + final String keyspace, final String columnFamily, final File backupDir) + throws Exception { File snapshotDir = getValidSnapshot(backupDir, snapshotName); // Process this snapshot folder for the given columnFamily if (snapshotDir == null) { @@ -183,11 +205,11 @@ protected void processColumnFamily(final String keyspace, final String columnFam logger.debug("Scanning for all SSTables in: {}", snapshotDir.getAbsolutePath()); Map> filePrefixToFileMap = new HashMap<>(); - Collection files = FileUtils.listFiles(snapshotDir, FileFilterUtils.fileFileFilter(), null); + Collection files = + FileUtils.listFiles(snapshotDir, FileFilterUtils.fileFileFilter(), null); - for (File file: files){ - if (!file.exists()) - continue; + for (File file : files) { + if (!file.exists()) continue; try { String prefix = PrefixGenerator.getSSTFileBase(file.getName()); @@ -196,53 +218,69 @@ protected void processColumnFamily(final String keyspace, final String columnFam prefix = "manifest"; if (prefix == null) { - logger.error("Unknown file type with no SSTFileBase found: ", file.getAbsolutePath()); + logger.error( + "Unknown file type with no SSTFileBase found: ", + file.getAbsolutePath()); return; } - FileUploadResult fileUploadResult = FileUploadResult.getFileUploadResult(keyspace, columnFamily, file); + FileUploadResult fileUploadResult = + FileUploadResult.getFileUploadResult(keyspace, columnFamily, file); filePrefixToFileMap.putIfAbsent(prefix, new ArrayList<>()); filePrefixToFileMap.get(prefix).add(fileUploadResult); } catch (Exception e) { - /* If you are here it means either of the issues. In that case, do not upload the meta file. - * @throws UnsupportedOperationException - * if an attributes of the given type are not supported - * @throws IOException - * if an I/O error occurs - * @throws SecurityException - * In the case of the default provider, a security manager is - * installed, its {@link SecurityManager#checkRead(String) checkRead} - * method is invoked to check read access to the file. If this - * method is invoked to read security sensitive attributes then the - * security manager may be invoke to check for additional permissions. - */ - logger.error("Internal error while trying to generate FileUploadResult and/or reading FileAttributes for file: " + file.getAbsolutePath(), e); + /* If you are here it means either of the issues. In that case, do not upload the meta file. + * @throws UnsupportedOperationException + * if an attributes of the given type are not supported + * @throws IOException + * if an I/O error occurs + * @throws SecurityException + * In the case of the default provider, a security manager is + * installed, its {@link SecurityManager#checkRead(String) checkRead} + * method is invoked to check read access to the file. If this + * method is invoked to read security sensitive attributes then the + * security manager may be invoke to check for additional permissions. + */ + logger.error( + "Internal error while trying to generate FileUploadResult and/or reading FileAttributes for file: " + + file.getAbsolutePath(), + e); throw e; } } - ColumnfamilyResult columnfamilyResult = convertToColumnFamilyResult(keyspace, columnFamily, filePrefixToFileMap); - filePrefixToFileMap.clear(); //Release the resources. - - logger.debug("Starting the processing of KS: {}, CF: {}, No.of SSTables: {}", columnfamilyResult.getKeyspaceName(), columnfamilyResult.getColumnfamilyName(), columnfamilyResult.getSstables().size()); - - //TODO: Future - Ensure that all the files are en-queued for Upload. Use BackupCacheService (BCS) to find the - //location where files are uploaded and BackupUploadDownloadService(BUDS) to enque if they are not. - //Note that BUDS will be responsible for actually deleting the files after they are processed as they really should not be deleted unless they are successfully uploaded. + ColumnfamilyResult columnfamilyResult = + convertToColumnFamilyResult(keyspace, columnFamily, filePrefixToFileMap); + filePrefixToFileMap.clear(); // Release the resources. + + logger.debug( + "Starting the processing of KS: {}, CF: {}, No.of SSTables: {}", + columnfamilyResult.getKeyspaceName(), + columnfamilyResult.getColumnfamilyName(), + columnfamilyResult.getSstables().size()); + + // TODO: Future - Ensure that all the files are en-queued for Upload. Use BackupCacheService + // (BCS) to find the + // location where files are uploaded and BackupUploadDownloadService(BUDS) to enque if they + // are not. + // Note that BUDS will be responsible for actually deleting the files after they are + // processed as they really should not be deleted unless they are successfully uploaded. FileUtils.cleanDirectory(snapshotDir); FileUtils.deleteDirectory(snapshotDir); dataStep.addColumnfamilyResult(columnfamilyResult); - logger.debug("Finished processing KS: {}, CF: {}", columnfamilyResult.getKeyspaceName(), columnfamilyResult.getColumnfamilyName()); - + logger.debug( + "Finished processing KS: {}, CF: {}", + columnfamilyResult.getKeyspaceName(), + columnfamilyResult.getColumnfamilyName()); } @Override protected void addToRemotePath(String remotePath) { - //Do nothing + // Do nothing } - //For testing purposes only. + // For testing purposes only. void setSnapshotName(String snapshotName) { this.snapshotName = snapshotName; } diff --git a/priam/src/main/java/com/netflix/priam/tuner/GCTuner.java b/priam/src/main/java/com/netflix/priam/tuner/GCTuner.java index 9f8812cf2..c3f174b13 100644 --- a/priam/src/main/java/com/netflix/priam/tuner/GCTuner.java +++ b/priam/src/main/java/com/netflix/priam/tuner/GCTuner.java @@ -17,89 +17,88 @@ package com.netflix.priam.tuner; -import javax.inject.Singleton; import java.util.Arrays; import java.util.HashSet; import java.util.Set; +import javax.inject.Singleton; /** - * List of Garbage collection parameters for CMS/G1GC. This list is used to automatically enable/disable configurations, if found in jvm.options. - * Created by aagrawal on 8/23/17. + * List of Garbage collection parameters for CMS/G1GC. This list is used to automatically + * enable/disable configurations, if found in jvm.options. Created by aagrawal on 8/23/17. */ @Singleton public class GCTuner { -// private Set commonOptions = new HashSet<>(Arrays.asList( -// "-XX:ParallelGCThreads", -// "-XX:ConcGCThreads", -// "-XX:+ParallelRefProcEnabled", -// "-XX:+AlwaysPreTouch", -// "-XX:+UseTLAB", -// "-XX:+ResizeTLAB", -// "-XX:-UseBiasedLocking" -// )); + // private Set commonOptions = new HashSet<>(Arrays.asList( + // "-XX:ParallelGCThreads", + // "-XX:ConcGCThreads", + // "-XX:+ParallelRefProcEnabled", + // "-XX:+AlwaysPreTouch", + // "-XX:+UseTLAB", + // "-XX:+ResizeTLAB", + // "-XX:-UseBiasedLocking" + // )); - private static final Set cmsOptions = new HashSet<>(Arrays.asList( - "-XX:+UseConcMarkSweepGC", - "-XX:+UseParNewGC", - "-XX:+UseParallelGC", - "-XX:+CMSConcurrentMTEnabled", - "-XX:CMSInitiatingOccupancyFraction", - "-XX:+UseCMSInitiatingOccupancyOnly", - "-XX:+CMSClassUnloadingEnabled", - "-XX:+CMSIncrementalMode", - "-XX:+CMSPermGenSweepingEnabled", - "-XX:+ExplicitGCInvokesConcurrent", - "-XX:+ExplicitGCInvokesConcurrentAndUnloadsClasses", - "-XX:+DisableExplicitGC", - "-XX:+CMSParallelRemarkEnabled", - "-XX:SurvivorRatio", - "-XX:MaxTenuringThreshold", - "-XX:CMSWaitDuration", - "-XX:+CMSParallelInitialMarkEnabled", - "-XX:+CMSEdenChunksRecordAlways" - )); + private static final Set cmsOptions = + new HashSet<>( + Arrays.asList( + "-XX:+UseConcMarkSweepGC", + "-XX:+UseParNewGC", + "-XX:+UseParallelGC", + "-XX:+CMSConcurrentMTEnabled", + "-XX:CMSInitiatingOccupancyFraction", + "-XX:+UseCMSInitiatingOccupancyOnly", + "-XX:+CMSClassUnloadingEnabled", + "-XX:+CMSIncrementalMode", + "-XX:+CMSPermGenSweepingEnabled", + "-XX:+ExplicitGCInvokesConcurrent", + "-XX:+ExplicitGCInvokesConcurrentAndUnloadsClasses", + "-XX:+DisableExplicitGC", + "-XX:+CMSParallelRemarkEnabled", + "-XX:SurvivorRatio", + "-XX:MaxTenuringThreshold", + "-XX:CMSWaitDuration", + "-XX:+CMSParallelInitialMarkEnabled", + "-XX:+CMSEdenChunksRecordAlways")); - private static final Set g1gcOptions = new HashSet<>(Arrays.asList( - "-XX:+UseG1GC", - "-XX:G1HeapRegionSize", - "-XX:MaxGCPauseMillis", - "-XX:G1NewSizePercent", - "-XX:G1MaxNewSizePercent", - "-XX:-ResizePLAB", - "-XX:InitiatingHeapOccupancyPercent", - "-XX:G1MixedGCLiveThresholdPercent", - "-XX:G1HeapWastePercent", - "-XX:G1MixedGCCountTarget", - "-XX:G1OldCSetRegionThresholdPercent", - "-XX:G1ReservePercent", - "-XX:SoftRefLRUPolicyMSPerMB", - "-XX:G1ConcRefinementThreads", - "-XX:MaxGCPauseMillis", - "-XX:+UnlockExperimentalVMOptions", - "-XX:NewRatio", - "-XX:G1RSetUpdatingPauseTimePercent" - )); + private static final Set g1gcOptions = + new HashSet<>( + Arrays.asList( + "-XX:+UseG1GC", + "-XX:G1HeapRegionSize", + "-XX:MaxGCPauseMillis", + "-XX:G1NewSizePercent", + "-XX:G1MaxNewSizePercent", + "-XX:-ResizePLAB", + "-XX:InitiatingHeapOccupancyPercent", + "-XX:G1MixedGCLiveThresholdPercent", + "-XX:G1HeapWastePercent", + "-XX:G1MixedGCCountTarget", + "-XX:G1OldCSetRegionThresholdPercent", + "-XX:G1ReservePercent", + "-XX:SoftRefLRUPolicyMSPerMB", + "-XX:G1ConcRefinementThreads", + "-XX:MaxGCPauseMillis", + "-XX:+UnlockExperimentalVMOptions", + "-XX:NewRatio", + "-XX:G1RSetUpdatingPauseTimePercent")); - final static GCType getGCType(String option) { - if (cmsOptions.contains(option)) - return GCType.CMS; + static final GCType getGCType(String option) { + if (cmsOptions.contains(option)) return GCType.CMS; - if (g1gcOptions.contains(option)) - return GCType.G1GC; + if (g1gcOptions.contains(option)) return GCType.G1GC; return null; } - final static GCType getGCType(JVMOption jvmOption) - { + static final GCType getGCType(JVMOption jvmOption) { return getGCType(jvmOption.getJvmOption()); } - public Set getCmsOptions(){ + public Set getCmsOptions() { return cmsOptions; } - public Set getG1gcOptions(){ + public Set getG1gcOptions() { return g1gcOptions; } } diff --git a/priam/src/main/java/com/netflix/priam/tuner/GCType.java b/priam/src/main/java/com/netflix/priam/tuner/GCType.java index 112072e84..bccd687e7 100644 --- a/priam/src/main/java/com/netflix/priam/tuner/GCType.java +++ b/priam/src/main/java/com/netflix/priam/tuner/GCType.java @@ -23,17 +23,17 @@ import org.slf4j.LoggerFactory; /** - * Garbage collection types supported by Priam for Cassandra (CMS/G1GC). - * Created by aagrawal on 8/24/17. + * Garbage collection types supported by Priam for Cassandra (CMS/G1GC). Created by aagrawal on + * 8/24/17. */ public enum GCType { - CMS("CMS"),G1GC("G1GC"); + CMS("CMS"), + G1GC("G1GC"); private static final Logger logger = LoggerFactory.getLogger(GCType.class); private final String gcType; - GCType(String gcType) - { + GCType(String gcType) { this.gcType = gcType.toUpperCase(); } @@ -51,25 +51,32 @@ public enum GCType { * Illegal value |NA |False |UnsupportedTypeException */ - public static GCType lookup(String gcType, boolean acceptNullOrEmpty, boolean acceptIllegalValue) throws UnsupportedTypeException { + public static GCType lookup( + String gcType, boolean acceptNullOrEmpty, boolean acceptIllegalValue) + throws UnsupportedTypeException { if (StringUtils.isEmpty(gcType)) - if(acceptNullOrEmpty) - return null; - else - { - String message = String.format("%s is not a supported GC Type. Supported values are %s", gcType, getSupportedValues()); + if (acceptNullOrEmpty) return null; + else { + String message = + String.format( + "%s is not a supported GC Type. Supported values are %s", + gcType, getSupportedValues()); logger.error(message); throw new UnsupportedTypeException(message); } - try{ + try { return GCType.valueOf(gcType.toUpperCase()); - }catch (IllegalArgumentException ex) - { - String message = String.format("%s is not a supported GCType. Supported values are %s", gcType, getSupportedValues()); + } catch (IllegalArgumentException ex) { + String message = + String.format( + "%s is not a supported GCType. Supported values are %s", + gcType, getSupportedValues()); if (acceptIllegalValue) { - message = message + ". Since acceptIllegalValue is set to True, returning NULL instead."; + message = + message + + ". Since acceptIllegalValue is set to True, returning NULL instead."; logger.error(message); return null; } @@ -79,13 +86,11 @@ public static GCType lookup(String gcType, boolean acceptNullOrEmpty, boolean ac } } - private static String getSupportedValues() - { + private static String getSupportedValues() { StringBuilder supportedValues = new StringBuilder(); boolean first = true; for (GCType type : GCType.values()) { - if (!first) - supportedValues.append(","); + if (!first) supportedValues.append(","); supportedValues.append(type); first = false; } @@ -93,14 +98,11 @@ private static String getSupportedValues() return supportedValues.toString(); } - public static GCType lookup(String gcType) throws UnsupportedTypeException - { + public static GCType lookup(String gcType) throws UnsupportedTypeException { return lookup(gcType, false, false); } - public String getGcType() - { + public String getGcType() { return gcType; } - } diff --git a/priam/src/main/java/com/netflix/priam/tuner/ICassandraTuner.java b/priam/src/main/java/com/netflix/priam/tuner/ICassandraTuner.java index 69a88d522..9af05c409 100644 --- a/priam/src/main/java/com/netflix/priam/tuner/ICassandraTuner.java +++ b/priam/src/main/java/com/netflix/priam/tuner/ICassandraTuner.java @@ -1,28 +1,25 @@ /** * Copyright 2017 Netflix, Inc. - *

- * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and + * + *

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file + * except in compliance with the License. You may obtain a copy of the License at + * + *

http://www.apache.org/licenses/LICENSE-2.0 + * + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing permissions and * limitations under the License. */ package com.netflix.priam.tuner; import com.google.inject.ImplementedBy; - import java.io.IOException; @ImplementedBy(StandardTuner.class) -public interface ICassandraTuner -{ - void writeAllProperties(String yamlLocation, String hostname, String seedProvider) throws Exception; +public interface ICassandraTuner { + void writeAllProperties(String yamlLocation, String hostname, String seedProvider) + throws Exception; void updateAutoBootstrap(String yamlLocation, boolean autobootstrap) throws IOException; diff --git a/priam/src/main/java/com/netflix/priam/tuner/JVMOption.java b/priam/src/main/java/com/netflix/priam/tuner/JVMOption.java index 01273e951..a8c578d33 100644 --- a/priam/src/main/java/com/netflix/priam/tuner/JVMOption.java +++ b/priam/src/main/java/com/netflix/priam/tuner/JVMOption.java @@ -16,24 +16,22 @@ */ package com.netflix.priam.tuner; -import org.apache.commons.lang3.StringUtils; - import java.util.Objects; import java.util.regex.Matcher; import java.util.regex.Pattern; +import org.apache.commons.lang3.StringUtils; -/** - * POJO to parse and store the JVM option from jvm.options file. - * Created by aagrawal on 8/28/17. - */ +/** POJO to parse and store the JVM option from jvm.options file. Created by aagrawal on 8/28/17. */ public class JVMOption { private String jvmOption; private String value; private boolean isCommented; private boolean isHeapJVMOption; private static final Pattern pattern = Pattern.compile("(#)*(-[^=]+)=?(.*)?"); - //A new pattern is required because heap do not separate JVM key,value with "=". - private static final Pattern heapPattern = Pattern.compile("(#)*(-Xm[x|s|n])([0-9]+[K|M|G])?"); //Pattern.compile("(#)*-(Xm[x|s|n])([0-9]+)(K|M|G)?"); + // A new pattern is required because heap do not separate JVM key,value with "=". + private static final Pattern heapPattern = + Pattern.compile( + "(#)*(-Xm[x|s|n])([0-9]+[K|M|G])?"); // Pattern.compile("(#)*-(Xm[x|s|n])([0-9]+)(K|M|G)?"); public JVMOption(String jvmOption) { this.jvmOption = jvmOption; @@ -48,12 +46,10 @@ public JVMOption(String jvmOption, String value, boolean isCommented, boolean is public String toJVMOptionString() { final StringBuilder sb = new StringBuilder(); - if (isCommented) - sb.append("#"); + if (isCommented) sb.append("#"); sb.append(jvmOption); if (value != null) { - if (!isHeapJVMOption) - sb.append("="); + if (!isHeapJVMOption) sb.append("="); sb.append(value); } return sb.toString(); @@ -64,10 +60,10 @@ public boolean equals(Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; JVMOption jvmOption1 = (JVMOption) o; - return isCommented == jvmOption1.isCommented && - isHeapJVMOption == jvmOption1.isHeapJVMOption && - Objects.equals(jvmOption, jvmOption1.jvmOption) && - Objects.equals(value, jvmOption1.value); + return isCommented == jvmOption1.isCommented + && isHeapJVMOption == jvmOption1.isHeapJVMOption + && Objects.equals(jvmOption, jvmOption1.jvmOption) + && Objects.equals(value, jvmOption1.value); } @Override @@ -90,8 +86,7 @@ public String getValue() { } public JVMOption setValue(String value) { - if (!StringUtils.isEmpty(value)) - this.value = value; + if (!StringUtils.isEmpty(value)) this.value = value; return this; } @@ -116,18 +111,23 @@ public JVMOption setHeapJVMOption(boolean heapJVMOption) { public static JVMOption parse(String line) { JVMOption result = null; - //See if it is heap JVM option. + // See if it is heap JVM option. Matcher matcher = heapPattern.matcher(line); if (matcher.matches()) { boolean isCommented = (matcher.group(1) != null); - return new JVMOption(matcher.group(2)).setCommented(isCommented).setValue(matcher.group(3)).setHeapJVMOption(true); + return new JVMOption(matcher.group(2)) + .setCommented(isCommented) + .setValue(matcher.group(3)) + .setHeapJVMOption(true); } - //See if other heap option. + // See if other heap option. matcher = pattern.matcher(line); if (matcher.matches()) { boolean isCommented = (matcher.group(1) != null); - return new JVMOption(matcher.group(2)).setCommented(isCommented).setValue(matcher.group(3)); + return new JVMOption(matcher.group(2)) + .setCommented(isCommented) + .setValue(matcher.group(3)); } return result; diff --git a/priam/src/main/java/com/netflix/priam/tuner/JVMOptionsTuner.java b/priam/src/main/java/com/netflix/priam/tuner/JVMOptionsTuner.java index a7ea57a7f..cb9e34e76 100644 --- a/priam/src/main/java/com/netflix/priam/tuner/JVMOptionsTuner.java +++ b/priam/src/main/java/com/netflix/priam/tuner/JVMOptionsTuner.java @@ -19,21 +19,20 @@ import com.google.inject.Inject; import com.netflix.priam.config.IConfiguration; -import org.apache.commons.lang3.StringUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import java.io.File; import java.nio.file.Files; import java.util.*; import java.util.stream.Collectors; +import org.apache.commons.lang3.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** - * This is to tune the jvm.options file introduced in Cassandra 3.x to pass JVM parameters to Cassandra. - * It supports configuring GC type (CMS/G1GC) where it automatically activates default properties as provided in - * jvm.options file. Note that this will not "add" any GC options. - *

- * Created by aagrawal on 8/23/17. + * This is to tune the jvm.options file introduced in Cassandra 3.x to pass JVM parameters to + * Cassandra. It supports configuring GC type (CMS/G1GC) where it automatically activates default + * properties as provided in jvm.options file. Note that this will not "add" any GC options. + * + *

Created by aagrawal on 8/23/17. */ public class JVMOptionsTuner { private static final Logger logger = LoggerFactory.getLogger(JVMOptionsTuner.class); @@ -50,7 +49,8 @@ public JVMOptionsTuner(IConfiguration config) { * configuring GC {@link IConfiguration#getGCType()}etc. * * @param outputFile File name with which this configured JVM options should be written. - * @throws Exception when encountered with invalid configured GC type. {@link IConfiguration#getGCType()} + * @throws Exception when encountered with invalid configured GC type. {@link + * IConfiguration#getGCType()} */ public void updateAndSaveJVMOptions(final String outputFile) throws Exception { List configuredJVMOptions = updateJVMOptions(); @@ -61,23 +61,24 @@ public void updateAndSaveJVMOptions(final String outputFile) throws Exception { logger.info("Updating jvm.options with following values: " + buffer.toString()); } - //Verify we can write to output file and it is not directory. + // Verify we can write to output file and it is not directory. File file = new File(outputFile); if (file.exists() && !file.canWrite()) { throw new Exception("Not enough permissions to write to file: " + outputFile); } - //Write jvm.options back to override defaults. + // Write jvm.options back to override defaults. Files.write(new File(outputFile).toPath(), configuredJVMOptions); } /** - * Update the JVM options file for cassandra by updating/removing JVM options - * {@link IConfiguration#getJVMExcludeSet()} and {@link IConfiguration#getJVMUpsertSet()}, - * configuring GC {@link IConfiguration#getGCType()}etc. + * Update the JVM options file for cassandra by updating/removing JVM options {@link + * IConfiguration#getJVMExcludeSet()} and {@link IConfiguration#getJVMUpsertSet()}, configuring + * GC {@link IConfiguration#getGCType()}etc. * * @return List of Configuration as String after reading the configuration from jvm.options - * @throws Exception when encountered with invalid configured GC type. {@link IConfiguration#getGCType()} + * @throws Exception when encountered with invalid configured GC type. {@link + * IConfiguration#getGCType()} */ protected List updateJVMOptions() throws Exception { File jvmOptionsFile = new File(config.getJVMOptionsFileLocation()); @@ -86,26 +87,33 @@ protected List updateJVMOptions() throws Exception { final Map excludeSet = config.getJVMExcludeSet(); - //Make a copy of upsertSet, so we can delete the entries as we process them. + // Make a copy of upsertSet, so we can delete the entries as we process them. Map upsertSet = config.getJVMUpsertSet(); - //Don't use streams for processing as upsertSet jvm options needs to be removed if we find them - //already in jvm.options file. - List optionsFromFile = Files.lines(jvmOptionsFile.toPath()).collect(Collectors.toList()); + // Don't use streams for processing as upsertSet jvm options needs to be removed if we find + // them + // already in jvm.options file. + List optionsFromFile = + Files.lines(jvmOptionsFile.toPath()).collect(Collectors.toList()); List configuredOptions = new LinkedList<>(); for (String line : optionsFromFile) { - configuredOptions.add(updateConfigurationValue(line, configuredGC, upsertSet, excludeSet)); + configuredOptions.add( + updateConfigurationValue(line, configuredGC, upsertSet, excludeSet)); } - //Add all the upserts(inserts only left) from config. + // Add all the upserts(inserts only left) from config. if (upsertSet != null && !upsertSet.isEmpty()) { configuredOptions.add("#################"); configuredOptions.add("# USER PROVIDED CUSTOM JVM CONFIGURATIONS #"); configuredOptions.add("#################"); - configuredOptions.addAll(upsertSet.values().stream() - .map(JVMOption::toJVMOptionString).collect(Collectors.toList())); + configuredOptions.addAll( + upsertSet + .values() + .stream() + .map(JVMOption::toJVMOptionString) + .collect(Collectors.toList())); } return configuredOptions; @@ -117,27 +125,31 @@ private void setHeapSetting(String configuredValue, JVMOption option) { } /** - * @param line a line as read from jvm.options file. + * @param line a line as read from jvm.options file. * @param configuredGC GCType configured by user for Cassandra. - * @param upsertSet configured upsert set of JVM properties as provided by user for Cassandra. - * @param excludeSet configured exclude set of JVM properties as provided by user for Cassandra. - * @return the "comment" as is, if not a valid JVM option. Else, a string representation of JVM option + * @param upsertSet configured upsert set of JVM properties as provided by user for Cassandra. + * @param excludeSet configured exclude set of JVM properties as provided by user for Cassandra. + * @return the "comment" as is, if not a valid JVM option. Else, a string representation of JVM + * option */ - private String updateConfigurationValue(final String line, GCType configuredGC, Map upsertSet, Map excludeSet) { + private String updateConfigurationValue( + final String line, + GCType configuredGC, + Map upsertSet, + Map excludeSet) { JVMOption option = JVMOption.parse(line); - if (option == null) - return line; + if (option == null) return line; - //Is parameter for heap setting. + // Is parameter for heap setting. if (option.isHeapJVMOption()) { String configuredValue; switch (option.getJvmOption()) { - //Special handling for heap new size ("Xmn") + // Special handling for heap new size ("Xmn") case "-Xmn": configuredValue = config.getHeapNewSize(); break; - //Set min and max heap size to same value + // Set min and max heap size to same value default: configuredValue = config.getHeapSize(); break; @@ -172,28 +184,32 @@ private String updateConfigurationValue(final String line, GCType configuredGC, private void validate(File jvmOptionsFile) throws Exception { if (!jvmOptionsFile.exists()) - throw new Exception("JVM Option File does not exist: " + jvmOptionsFile.getAbsolutePath()); + throw new Exception( + "JVM Option File does not exist: " + jvmOptionsFile.getAbsolutePath()); if (jvmOptionsFile.isDirectory()) - throw new Exception("JVM Option File is a directory: " + jvmOptionsFile.getAbsolutePath()); + throw new Exception( + "JVM Option File is a directory: " + jvmOptionsFile.getAbsolutePath()); if (!jvmOptionsFile.canRead() || !jvmOptionsFile.canWrite()) - throw new Exception("JVM Option File does not have right permission: " + jvmOptionsFile.getAbsolutePath()); - + throw new Exception( + "JVM Option File does not have right permission: " + + jvmOptionsFile.getAbsolutePath()); } /** - * Util function to parse comma separated list of jvm options to a Map (jvmOptionName, JVMOption). - * It will ignore anything which is not a valid JVM option. + * Util function to parse comma separated list of jvm options to a Map (jvmOptionName, + * JVMOption). It will ignore anything which is not a valid JVM option. * * @param property comma separated list of JVM options. * @return Map of (jvmOptionName, JVMOption). */ public static final Map parseJVMOptions(String property) { - if (StringUtils.isEmpty(property)) - return null; - return new HashSet<>(Arrays.asList(property.split(","))).stream() - .map(JVMOption::parse).filter(Objects::nonNull).collect(Collectors.toMap(JVMOption::getJvmOption, jvmOption -> jvmOption)); + if (StringUtils.isEmpty(property)) return null; + return new HashSet<>(Arrays.asList(property.split(","))) + .stream() + .map(JVMOption::parse) + .filter(Objects::nonNull) + .collect(Collectors.toMap(JVMOption::getJvmOption, jvmOption -> jvmOption)); } - } diff --git a/priam/src/main/java/com/netflix/priam/tuner/StandardTuner.java b/priam/src/main/java/com/netflix/priam/tuner/StandardTuner.java index 294abc945..63ae4c9ca 100644 --- a/priam/src/main/java/com/netflix/priam/tuner/StandardTuner.java +++ b/priam/src/main/java/com/netflix/priam/tuner/StandardTuner.java @@ -1,38 +1,36 @@ /** * Copyright 2017 Netflix, Inc. - *

- * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and + * + *

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file + * except in compliance with the License. You may obtain a copy of the License at + * + *

http://www.apache.org/licenses/LICENSE-2.0 + * + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing permissions and * limitations under the License. */ package com.netflix.priam.tuner; import com.google.common.collect.Lists; import com.google.inject.Inject; -import com.netflix.priam.config.IConfiguration; import com.netflix.priam.backup.SnapshotBackup; +import com.netflix.priam.config.IConfiguration; import com.netflix.priam.restore.Restore; +import java.io.*; +import java.util.List; +import java.util.Map; +import java.util.Properties; import org.apache.commons.collections4.CollectionUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.yaml.snakeyaml.DumperOptions; import org.yaml.snakeyaml.Yaml; -import java.io.*; -import java.util.List; -import java.util.Map; -import java.util.Properties; - /** - * Tune the standard cassandra parameters/configurations. eg. cassandra.yaml, jvm.options, bootstrap etc. + * Tune the standard cassandra parameters/configurations. eg. cassandra.yaml, jvm.options, bootstrap + * etc. */ public class StandardTuner implements ICassandraTuner { private static final Logger logger = LoggerFactory.getLogger(StandardTuner.class); @@ -43,7 +41,8 @@ public StandardTuner(IConfiguration config) { this.config = config; } - public void writeAllProperties(String yamlLocation, String hostname, String seedProvider) throws Exception { + public void writeAllProperties(String yamlLocation, String hostname, String seedProvider) + throws Exception { DumperOptions options = new DumperOptions(); options.setDefaultFlowStyle(DumperOptions.FlowStyle.BLOCK); Yaml yaml = new Yaml(options); @@ -58,7 +57,7 @@ public void writeAllProperties(String yamlLocation, String hostname, String seed map.put("native_transport_port", config.getNativeTransportPort()); map.put("listen_address", hostname); map.put("rpc_address", hostname); - //Dont bootstrap in restore mode + // Dont bootstrap in restore mode if (!Restore.isRestoreEnabled(config)) { map.put("auto_bootstrap", config.getAutoBoostrap()); } else { @@ -70,7 +69,10 @@ public void writeAllProperties(String yamlLocation, String hostname, String seed map.put("hints_directory", config.getHintsLocation()); map.put("data_file_directories", Lists.newArrayList(config.getDataFileLocation())); - boolean enableIncremental = (SnapshotBackup.isBackupEnabled(config) && config.isIncrBackup()) && (CollectionUtils.isEmpty(config.getBackupRacs()) || config.getBackupRacs().contains(config.getRac())); + boolean enableIncremental = + (SnapshotBackup.isBackupEnabled(config) && config.isIncrBackup()) + && (CollectionUtils.isEmpty(config.getBackupRacs()) + || config.getBackupRacs().contains(config.getRac())); map.put("incremental_backups", enableIncremental); map.put("endpoint_snitch", config.getSnitch()); @@ -78,7 +80,9 @@ public void writeAllProperties(String yamlLocation, String hostname, String seed map.remove("in_memory_compaction_limit_in_mb"); } map.put("compaction_throughput_mb_per_sec", config.getCompactionThroughput()); - map.put("partitioner", derivePartitioner(map.get("partitioner").toString(), config.getPartitioner())); + map.put( + "partitioner", + derivePartitioner(map.get("partitioner").toString(), config.getPartitioner())); if (map.containsKey("memtable_total_space_in_mb")) { map.remove("memtable_total_space_in_mb"); @@ -103,17 +107,19 @@ public void writeAllProperties(String yamlLocation, String hostname, String seed map.put("rpc_server_type", config.getRpcServerType()); map.put("rpc_min_threads", config.getRpcMinThreads()); map.put("rpc_max_threads", config.getRpcMaxThreads()); - // Add private ip address as broadcast_rpc_address. This will ensure that COPY function works correctly. + // Add private ip address as broadcast_rpc_address. This will ensure that COPY function + // works correctly. map.put("broadcast_rpc_address", config.getInstanceDataRetriever().getPrivateIP()); - //map.put("index_interval", config.getIndexInterval()); - + // map.put("index_interval", config.getIndexInterval()); map.put("tombstone_warn_threshold", config.getTombstoneWarnThreshold()); map.put("tombstone_failure_threshold", config.getTombstoneFailureThreshold()); map.put("streaming_socket_timeout_in_ms", config.getStreamingSocketTimeoutInMS()); map.put("memtable_cleanup_threshold", config.getMemtableCleanupThreshold()); - map.put("compaction_large_partition_warning_threshold_mb", config.getCompactionLargePartitionWarnThresholdInMB()); + map.put( + "compaction_large_partition_warning_threshold_mb", + config.getCompactionLargePartitionWarnThresholdInMB()); List seedp = (List) map.get("seed_provider"); Map m = (Map) seedp.get(0); @@ -121,13 +127,12 @@ public void writeAllProperties(String yamlLocation, String hostname, String seed configfureSecurity(map); configureGlobalCaches(config, map); - //force to 1 until vnodes are properly supported + // force to 1 until vnodes are properly supported map.put("num_tokens", 1); - addExtraCassParams(map); - //remove troublesome properties + // remove troublesome properties map.remove("flush_largest_memtables_at"); map.remove("reduce_cache_capacity_to"); @@ -146,17 +151,14 @@ protected String getSnitch() { return config.getSnitch(); } - /** - * Setup the cassandra 1.1 global cache values - */ + /** Setup the cassandra 1.1 global cache values */ private void configureGlobalCaches(IConfiguration config, Map yaml) { final String keyCacheSize = config.getKeyCacheSizeInMB(); if (keyCacheSize != null) { yaml.put("key_cache_size_in_mb", Integer.valueOf(keyCacheSize)); final String keyCount = config.getKeyCacheKeysToSave(); - if (keyCount != null) - yaml.put("key_cache_keys_to_save", Integer.valueOf(keyCount)); + if (keyCount != null) yaml.put("key_cache_keys_to_save", Integer.valueOf(keyCount)); } final String rowCacheSize = config.getRowCacheSizeInMB(); @@ -164,43 +166,41 @@ private void configureGlobalCaches(IConfiguration config, Map yaml) { yaml.put("row_cache_size_in_mb", Integer.valueOf(rowCacheSize)); final String rowCount = config.getRowCacheKeysToSave(); - if (rowCount != null) - yaml.put("row_cache_keys_to_save", Integer.valueOf(rowCount)); + if (rowCount != null) yaml.put("row_cache_keys_to_save", Integer.valueOf(rowCount)); } } String derivePartitioner(String fromYaml, String fromConfig) { - if (fromYaml == null || fromYaml.isEmpty()) - return fromConfig; - //this check is to prevent against overwriting an existing yaml file that has + if (fromYaml == null || fromYaml.isEmpty()) return fromConfig; + // this check is to prevent against overwriting an existing yaml file that has // a partitioner not RandomPartitioner or (as of cass 1.2) Murmur3Partitioner. - //basically we don't want to hose existing deployments by changing the partitioner unexpectedly on them + // basically we don't want to hose existing deployments by changing the partitioner + // unexpectedly on them final String lowerCase = fromYaml.toLowerCase(); - if (lowerCase.contains("randomparti") || lowerCase.contains("murmur")) - return fromConfig; + if (lowerCase.contains("randomparti") || lowerCase.contains("murmur")) return fromConfig; return fromYaml; } protected void configfureSecurity(Map map) { - //the client-side ssl settings + // the client-side ssl settings Map clientEnc = (Map) map.get("client_encryption_options"); clientEnc.put("enabled", config.isClientSslEnabled()); - //the server-side (internode) ssl settings + // the server-side (internode) ssl settings Map serverEnc = (Map) map.get("server_encryption_options"); serverEnc.put("internode_encryption", config.getInternodeEncryption()); } protected void configureCommitLogBackups() throws IOException { - if (!config.isBackingUpCommitLogs()) - return; + if (!config.isBackingUpCommitLogs()) return; Properties props = new Properties(); props.put("archive_command", config.getCommitLogBackupArchiveCmd()); props.put("restore_command", config.getCommitLogBackupRestoreCmd()); props.put("restore_directories", config.getCommitLogBackupRestoreFromDirs()); props.put("restore_point_in_time", config.getCommitLogBackupRestorePointInTime()); - try(FileOutputStream fos = new FileOutputStream(new File(config.getCommitLogBackupPropsFile()))) { + try (FileOutputStream fos = + new FileOutputStream(new File(config.getCommitLogBackupPropsFile()))) { props.store(fos, "cassandra commit log archive props, as written by priam"); } } @@ -211,7 +211,7 @@ public void updateAutoBootstrap(String yamlFile, boolean autobootstrap) throws I Yaml yaml = new Yaml(options); @SuppressWarnings("rawtypes") Map map = yaml.load(new FileInputStream(yamlFile)); - //Dont bootstrap in restore mode + // Dont bootstrap in restore mode map.put("auto_bootstrap", autobootstrap); if (logger.isInfoEnabled()) { logger.info("Updating yaml: " + yaml.dump(map)); @@ -222,7 +222,7 @@ public void updateAutoBootstrap(String yamlFile, boolean autobootstrap) throws I @Override public final void updateJVMOptions() throws Exception { JVMOptionsTuner tuner = new JVMOptionsTuner(config); - //Overwrite default jvm.options file. + // Overwrite default jvm.options file. tuner.updateAndSaveJVMOptions(config.getJVMOptionsFileLocation()); } @@ -240,7 +240,11 @@ public void addExtraCassParams(Map map) { String priamKey = pair[0]; String cassKey = pair[1]; String cassVal = config.getCassYamlVal(priamKey); - logger.info("Updating yaml: Priamkey[{}], CassKey[{}], Val[{}]", priamKey, cassKey, cassVal); + logger.info( + "Updating yaml: Priamkey[{}], CassKey[{}], Val[{}]", + priamKey, + cassKey, + cassVal); map.put(cassKey, cassVal); } } diff --git a/priam/src/main/java/com/netflix/priam/tuner/TuneCassandra.java b/priam/src/main/java/com/netflix/priam/tuner/TuneCassandra.java index aa0b9257d..a225cdfde 100644 --- a/priam/src/main/java/com/netflix/priam/tuner/TuneCassandra.java +++ b/priam/src/main/java/com/netflix/priam/tuner/TuneCassandra.java @@ -22,13 +22,13 @@ import com.netflix.priam.scheduler.SimpleTimer; import com.netflix.priam.scheduler.Task; import com.netflix.priam.scheduler.TaskTimer; +import java.io.IOException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.IOException; - /** - * Tune Cassandra (Open source or DSE) via updating various configuration files (dse.yaml, cassandra.yaml, jvm.options etc) + * Tune Cassandra (Open source or DSE) via updating various configuration files (dse.yaml, + * cassandra.yaml, jvm.options etc) */ @Singleton public class TuneCassandra extends Task { @@ -38,7 +38,8 @@ public class TuneCassandra extends Task { private final InstanceState instanceState; @Inject - public TuneCassandra(IConfiguration config, ICassandraTuner tuner, InstanceState instanceState) { + public TuneCassandra( + IConfiguration config, ICassandraTuner tuner, InstanceState instanceState) { super(config); this.tuner = tuner; this.instanceState = instanceState; @@ -50,7 +51,8 @@ public void execute() throws Exception { while (!isDone) { try { - tuner.writeAllProperties(config.getYamlLocation(), null, config.getSeedProviderName()); + tuner.writeAllProperties( + config.getYamlLocation(), null, config.getSeedProviderName()); tuner.updateJVMOptions(); isDone = true; instanceState.setYmlWritten(true); @@ -58,7 +60,6 @@ public void execute() throws Exception { LOGGER.error("Fail wrting cassandra.yml file. Retry again!", e); } } - } @Override diff --git a/priam/src/main/java/com/netflix/priam/tuner/dse/AuditLogTunerLog4J.java b/priam/src/main/java/com/netflix/priam/tuner/dse/AuditLogTunerLog4J.java index 60f248de0..1d0e820b1 100644 --- a/priam/src/main/java/com/netflix/priam/tuner/dse/AuditLogTunerLog4J.java +++ b/priam/src/main/java/com/netflix/priam/tuner/dse/AuditLogTunerLog4J.java @@ -21,19 +21,17 @@ import com.google.common.io.Files; import com.google.inject.Inject; import com.netflix.priam.config.IConfiguration; -import org.apache.cassandra.io.util.FileUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import java.io.BufferedWriter; import java.io.File; import java.nio.charset.Charset; import java.util.List; +import org.apache.cassandra.io.util.FileUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** - * Dse tuner for audit log via log4j. - * Use this instead of AuditLogTunerYaml if you are on DSE version 3.x. - * Created by aagrawal on 8/8/17. + * Dse tuner for audit log via log4j. Use this instead of AuditLogTunerYaml if you are on DSE + * version 3.x. Created by aagrawal on 8/8/17. */ public class AuditLogTunerLog4J implements IAuditLogTuner { @@ -51,18 +49,22 @@ public AuditLogTunerLog4J(IConfiguration config, IDseConfiguration dseConfig) { } /** - * Note: supporting the direct hacking of a log4j props file is far from elegant, - * but seems less odious than other solutions I've come up with. - * Operates under the assumption that the only people mucking with the audit log - * entries in the value are DataStax themselves and this program, and that the original - * property names are somehow still preserved. Otherwise, YMMV. + * Note: supporting the direct hacking of a log4j props file is far from elegant, but seems less + * odious than other solutions I've come up with. Operates under the assumption that the only + * people mucking with the audit log entries in the value are DataStax themselves and this + * program, and that the original property names are somehow still preserved. Otherwise, YMMV. */ public void tuneAuditLog() { BufferedWriter writer = null; try { final File srcFile = new File(config.getCassHome() + AUDIT_LOG_FILE); final List lines = Files.readLines(srcFile, Charset.defaultCharset()); - final File backupFile = new File(config.getCassHome() + AUDIT_LOG_FILE + "." + System.currentTimeMillis()); + final File backupFile = + new File( + config.getCassHome() + + AUDIT_LOG_FILE + + "." + + System.currentTimeMillis()); Files.move(srcFile, backupFile); writer = Files.newWriter(srcFile, Charset.defaultCharset()); @@ -70,30 +72,37 @@ public void tuneAuditLog() { try { loggerPrefix += findAuditLoggerName(lines); } catch (IllegalStateException ise) { - logger.warn("cannot locate " + PRIMARY_AUDIT_LOG_ENTRY + " property, will ignore any audit log updating"); + logger.warn( + "cannot locate " + + PRIMARY_AUDIT_LOG_ENTRY + + " property, will ignore any audit log updating"); return; } for (String line : lines) { - if (line.contains(loggerPrefix) || line.contains(PRIMARY_AUDIT_LOG_ENTRY) || line.contains(AUDIT_LOG_ADDITIVE_ENTRY)) { + if (line.contains(loggerPrefix) + || line.contains(PRIMARY_AUDIT_LOG_ENTRY) + || line.contains(AUDIT_LOG_ADDITIVE_ENTRY)) { if (dseConfig.isAuditLogEnabled()) { - //first, check to see if we need to uncomment the line + // first, check to see if we need to uncomment the line while (line.startsWith("#")) { line = line.substring(1); } - //next, check if we need to change the prop's value + // next, check if we need to change the prop's value if (line.contains("ActiveCategories")) { - final String cats = Joiner.on(",").join(dseConfig.getAuditLogCategories()); + final String cats = + Joiner.on(",").join(dseConfig.getAuditLogCategories()); line = line.substring(0, line.indexOf("=") + 1).concat(cats); } else if (line.contains("ExemptKeyspaces")) { - line = line.substring(0, line.indexOf("=") + 1).concat(dseConfig.getAuditLogExemptKeyspaces()); + line = + line.substring(0, line.indexOf("=") + 1) + .concat(dseConfig.getAuditLogExemptKeyspaces()); } } else { if (line.startsWith("#")) { - //make sure there's only one # at the beginning of the line - while (line.charAt(1) == '#') - line = line.substring(1); + // make sure there's only one # at the beginning of the line + while (line.charAt(1) == '#') line = line.substring(1); } else { line = "#" + line; } @@ -111,7 +120,6 @@ public void tuneAuditLog() { } } - private String findAuditLoggerName(List lines) throws IllegalStateException { for (final String l : lines) { if (l.contains(PRIMARY_AUDIT_LOG_ENTRY)) { diff --git a/priam/src/main/java/com/netflix/priam/tuner/dse/AuditLogTunerYaml.java b/priam/src/main/java/com/netflix/priam/tuner/dse/AuditLogTunerYaml.java index 9e775e0b8..efb2b1b71 100644 --- a/priam/src/main/java/com/netflix/priam/tuner/dse/AuditLogTunerYaml.java +++ b/priam/src/main/java/com/netflix/priam/tuner/dse/AuditLogTunerYaml.java @@ -18,21 +18,17 @@ package com.netflix.priam.tuner.dse; import com.google.inject.Inject; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.yaml.snakeyaml.DumperOptions; -import org.yaml.snakeyaml.Yaml; - import java.io.FileInputStream; import java.io.FileNotFoundException; import java.io.FileWriter; import java.io.IOException; import java.util.Map; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.yaml.snakeyaml.DumperOptions; +import org.yaml.snakeyaml.Yaml; -/** - * Dse tuner for audit log via YAML. Use this for DSE version 4.x - * Created by aagrawal on 8/8/17. - */ +/** Dse tuner for audit log via YAML. Use this for DSE version 4.x Created by aagrawal on 8/8/17. */ public class AuditLogTunerYaml implements IAuditLogTuner { private final IDseConfiguration dseConfig; @@ -53,9 +49,12 @@ public void tuneAuditLog() { Map map = yaml.load(new FileInputStream(dseYaml)); if (map.containsKey(AUDIT_LOG_DSE_ENTRY)) { - Boolean isEnabled = (Boolean) ((Map) map.get(AUDIT_LOG_DSE_ENTRY)).get("enabled"); + Boolean isEnabled = + (Boolean) + ((Map) map.get(AUDIT_LOG_DSE_ENTRY)).get("enabled"); - // Enable/disable audit logging (need this in addition to log4j-server.properties settings) + // Enable/disable audit logging (need this in addition to log4j-server.properties + // settings) if (dseConfig.isAuditLogEnabled()) { if (!isEnabled) { ((Map) map.get(AUDIT_LOG_DSE_ENTRY)).put("enabled", true); @@ -70,9 +69,12 @@ public void tuneAuditLog() { } yaml.dump(map, new FileWriter(dseYaml)); } catch (FileNotFoundException fileNotFound) { - logger.error("FileNotFound while trying to read yaml audit log for tuning: {}", dseYaml); + logger.error( + "FileNotFound while trying to read yaml audit log for tuning: {}", dseYaml); } catch (IOException e) { - logger.error("IOException while trying to write yaml file for audit log tuning: {}", dseYaml); + logger.error( + "IOException while trying to write yaml file for audit log tuning: {}", + dseYaml); } } } diff --git a/priam/src/main/java/com/netflix/priam/tuner/dse/DseProcessManager.java b/priam/src/main/java/com/netflix/priam/tuner/dse/DseProcessManager.java index 2ee07d7b9..9560a3925 100644 --- a/priam/src/main/java/com/netflix/priam/tuner/dse/DseProcessManager.java +++ b/priam/src/main/java/com/netflix/priam/tuner/dse/DseProcessManager.java @@ -1,16 +1,14 @@ /** * Copyright 2017 Netflix, Inc. - *

- * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and + * + *

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file + * except in compliance with the License. You may obtain a copy of the License at + * + *

http://www.apache.org/licenses/LICENSE-2.0 + * + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing permissions and * limitations under the License. */ package com.netflix.priam.tuner.dse; @@ -21,14 +19,17 @@ import com.netflix.priam.health.InstanceState; import com.netflix.priam.merics.CassMonitorMetrics; import com.netflix.priam.tuner.dse.IDseConfiguration.NodeType; - import java.util.Map; public class DseProcessManager extends CassandraProcessManager { private final IDseConfiguration dseConfig; @Inject - public DseProcessManager(IConfiguration config, IDseConfiguration dseConfig, InstanceState instanceState, CassMonitorMetrics cassMonitorMetrics) { + public DseProcessManager( + IConfiguration config, + IDseConfiguration dseConfig, + InstanceState instanceState, + CassMonitorMetrics cassMonitorMetrics) { super(config, instanceState, cassMonitorMetrics); this.dseConfig = dseConfig; } @@ -37,14 +38,9 @@ protected void setEnv(Map env) { super.setEnv(env); NodeType nodeType = dseConfig.getNodeType(); - if (nodeType == NodeType.ANALYTIC_HADOOP) - env.put("CLUSTER_TYPE", "-t"); - else if (nodeType == NodeType.ANALYTIC_SPARK) - env.put("CLUSTER_TYPE", "-k"); - else if (nodeType == NodeType.ANALYTIC_HADOOP_SPARK) - env.put("CLUSTER_TYPE", "-k -t"); - else if (nodeType == NodeType.SEARCH) - env.put("CLUSTER_TYPE", "-s"); + if (nodeType == NodeType.ANALYTIC_HADOOP) env.put("CLUSTER_TYPE", "-t"); + else if (nodeType == NodeType.ANALYTIC_SPARK) env.put("CLUSTER_TYPE", "-k"); + else if (nodeType == NodeType.ANALYTIC_HADOOP_SPARK) env.put("CLUSTER_TYPE", "-k -t"); + else if (nodeType == NodeType.SEARCH) env.put("CLUSTER_TYPE", "-s"); } - } diff --git a/priam/src/main/java/com/netflix/priam/tuner/dse/DseTuner.java b/priam/src/main/java/com/netflix/priam/tuner/dse/DseTuner.java index df07c7c22..81f5751bc 100644 --- a/priam/src/main/java/com/netflix/priam/tuner/dse/DseTuner.java +++ b/priam/src/main/java/com/netflix/priam/tuner/dse/DseTuner.java @@ -1,34 +1,31 @@ /** * Copyright 2017 Netflix, Inc. - *

- * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and + * + *

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file + * except in compliance with the License. You may obtain a copy of the License at + * + *

http://www.apache.org/licenses/LICENSE-2.0 + * + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing permissions and * limitations under the License. */ package com.netflix.priam.tuner.dse; +import static com.netflix.priam.tuner.dse.IDseConfiguration.NodeType; +import static org.apache.cassandra.locator.SnitchProperties.RACKDC_PROPERTY_FILENAME; + import com.google.inject.Inject; import com.netflix.priam.config.IConfiguration; import com.netflix.priam.tuner.StandardTuner; -import org.apache.cassandra.io.util.FileUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import java.io.FileReader; import java.io.FileWriter; import java.io.Reader; import java.util.Properties; - -import static com.netflix.priam.tuner.dse.IDseConfiguration.NodeType; -import static org.apache.cassandra.locator.SnitchProperties.RACKDC_PROPERTY_FILENAME; +import org.apache.cassandra.io.util.FileUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * Makes Datastax Enterprise-specific changes to the c* yaml and dse-yaml. @@ -42,13 +39,15 @@ public class DseTuner extends StandardTuner { private final IAuditLogTuner auditLogTuner; @Inject - public DseTuner(IConfiguration config, IDseConfiguration dseConfig, IAuditLogTuner auditLogTuner) { + public DseTuner( + IConfiguration config, IDseConfiguration dseConfig, IAuditLogTuner auditLogTuner) { super(config); this.dseConfig = dseConfig; this.auditLogTuner = auditLogTuner; } - public void writeAllProperties(String yamlLocation, String hostname, String seedProvider) throws Exception { + public void writeAllProperties(String yamlLocation, String hostname, String seedProvider) + throws Exception { super.writeAllProperties(yamlLocation, hostname, seedProvider); writeCassandraSnitchProperties(); auditLogTuner.tuneAuditLog(); @@ -56,8 +55,7 @@ public void writeAllProperties(String yamlLocation, String hostname, String seed private void writeCassandraSnitchProperties() { final NodeType nodeType = dseConfig.getNodeType(); - if (nodeType == NodeType.REAL_TIME_QUERY) - return; + if (nodeType == NodeType.REAL_TIME_QUERY) return; Reader reader = null; try { @@ -66,14 +64,10 @@ private void writeCassandraSnitchProperties() { Properties properties = new Properties(); properties.load(reader); String suffix = ""; - if (nodeType == NodeType.SEARCH) - suffix = "_solr"; - if (nodeType == NodeType.ANALYTIC_HADOOP) - suffix = "_hadoop"; - if (nodeType == NodeType.ANALYTIC_HADOOP_SPARK) - suffix = "_hadoop_spark"; - if (nodeType == NodeType.ANALYTIC_SPARK) - suffix = "_spark"; + if (nodeType == NodeType.SEARCH) suffix = "_solr"; + if (nodeType == NodeType.ANALYTIC_HADOOP) suffix = "_hadoop"; + if (nodeType == NodeType.ANALYTIC_HADOOP_SPARK) suffix = "_hadoop_spark"; + if (nodeType == NodeType.ANALYTIC_SPARK) suffix = "_spark"; properties.put("dc_suffix", suffix); properties.store(new FileWriter(filePath), ""); diff --git a/priam/src/main/java/com/netflix/priam/tuner/dse/IAuditLogTuner.java b/priam/src/main/java/com/netflix/priam/tuner/dse/IAuditLogTuner.java index f8d959b6e..909238273 100644 --- a/priam/src/main/java/com/netflix/priam/tuner/dse/IAuditLogTuner.java +++ b/priam/src/main/java/com/netflix/priam/tuner/dse/IAuditLogTuner.java @@ -20,9 +20,8 @@ import com.google.inject.ImplementedBy; /** - * This is intended for tuning audit log settings. - * Audit log settings file change between cassandra version from log4j to yaml. - * Created by aagrawal on 8/8/17. + * This is intended for tuning audit log settings. Audit log settings file change between cassandra + * version from log4j to yaml. Created by aagrawal on 8/8/17. */ @ImplementedBy(AuditLogTunerYaml.class) interface IAuditLogTuner { diff --git a/priam/src/main/java/com/netflix/priam/tuner/dse/IDseConfiguration.java b/priam/src/main/java/com/netflix/priam/tuner/dse/IDseConfiguration.java index 471fcb058..3c6de287f 100644 --- a/priam/src/main/java/com/netflix/priam/tuner/dse/IDseConfiguration.java +++ b/priam/src/main/java/com/netflix/priam/tuner/dse/IDseConfiguration.java @@ -1,16 +1,14 @@ /** * Copyright 2017 Netflix, Inc. - *

- * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and + * + *

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file + * except in compliance with the License. You may obtain a copy of the License at + * + *

http://www.apache.org/licenses/LICENSE-2.0 + * + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing permissions and * limitations under the License. */ package com.netflix.priam.tuner.dse; @@ -23,9 +21,7 @@ * @author jason brown */ public interface IDseConfiguration { - /** - * Using Datastax's terms here for the different types of nodes. - */ + /** Using Datastax's terms here for the different types of nodes. */ enum NodeType { /** vanilla Cassandra node */ REAL_TIME_QUERY("cassandra"), @@ -50,8 +46,7 @@ enum NodeType { public static NodeType getByAltName(String altName) { for (NodeType nt : NodeType.values()) { - if (nt.altName.toLowerCase().equals(altName)) - return nt; + if (nt.altName.toLowerCase().equals(altName)) return nt; } throw new IllegalArgumentException("Unknown node type: " + altName); } @@ -67,7 +62,7 @@ public static NodeType getByAltName(String altName) { boolean isAuditLogEnabled(); - /** @return comma-delimited list of keyspace names */ + /** @return comma-delimited list of keyspace names */ String getAuditLogExemptKeyspaces(); /** @@ -75,7 +70,13 @@ public static NodeType getByAltName(String altName) { * http://www.datastax.com/docs/datastax_enterprise3.1/security/data_auditing#data-auditing */ enum AuditLogCategory { - ADMIN, ALL, AUTH, DML, DDL, DCL, QUERY + ADMIN, + ALL, + AUTH, + DML, + DDL, + DCL, + QUERY } Set getAuditLogCategories(); diff --git a/priam/src/main/java/com/netflix/priam/utils/BoundedExponentialRetryCallable.java b/priam/src/main/java/com/netflix/priam/utils/BoundedExponentialRetryCallable.java index ec38df3a4..413ee01e5 100644 --- a/priam/src/main/java/com/netflix/priam/utils/BoundedExponentialRetryCallable.java +++ b/priam/src/main/java/com/netflix/priam/utils/BoundedExponentialRetryCallable.java @@ -16,18 +16,18 @@ */ package com.netflix.priam.utils; +import java.util.concurrent.CancellationException; import org.apache.commons.lang3.exception.ExceptionUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.concurrent.CancellationException; - public abstract class BoundedExponentialRetryCallable extends RetryableCallable { - protected final static long MAX_SLEEP = 10000; - protected final static long MIN_SLEEP = 1000; - protected final static int MAX_RETRIES = 10; + protected static final long MAX_SLEEP = 10000; + protected static final long MIN_SLEEP = 1000; + protected static final int MAX_RETRIES = 10; - private static final Logger logger = LoggerFactory.getLogger(BoundedExponentialRetryCallable.class); + private static final Logger logger = + LoggerFactory.getLogger(BoundedExponentialRetryCallable.class); private final long max; private final long min; private final int maxRetries; @@ -46,7 +46,7 @@ public BoundedExponentialRetryCallable(long minSleep, long maxSleep, int maxNumR } public T call() throws Exception { - long delay = min;// ms + long delay = min; // ms int retry = 0; int logCounter = 0; while (true) { @@ -65,7 +65,10 @@ public T call() throws Exception { sleeper.sleep(delay); } else if (delay >= max && retry <= maxRetries) { if (logger.isErrorEnabled()) { - logger.error(String.format("Retry #%d for: %s", retry, ExceptionUtils.getStackTrace(e))); + logger.error( + String.format( + "Retry #%d for: %s", + retry, ExceptionUtils.getStackTrace(e))); } sleeper.sleep(max); } else { @@ -76,5 +79,4 @@ public T call() throws Exception { } } } - } diff --git a/priam/src/main/java/com/netflix/priam/utils/CassandraMonitor.java b/priam/src/main/java/com/netflix/priam/utils/CassandraMonitor.java index ab1ddecba..6ed146c7b 100644 --- a/priam/src/main/java/com/netflix/priam/utils/CassandraMonitor.java +++ b/priam/src/main/java/com/netflix/priam/utils/CassandraMonitor.java @@ -18,23 +18,22 @@ import com.google.inject.Inject; import com.google.inject.Singleton; -import com.netflix.priam.defaultimpl.ICassandraProcess; import com.netflix.priam.config.IConfiguration; +import com.netflix.priam.defaultimpl.ICassandraProcess; import com.netflix.priam.health.InstanceState; import com.netflix.priam.merics.CassMonitorMetrics; import com.netflix.priam.scheduler.SimpleTimer; import com.netflix.priam.scheduler.Task; import com.netflix.priam.scheduler.TaskTimer; -import org.apache.cassandra.tools.NodeProbe; -import org.apache.commons.io.IOUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import java.io.BufferedReader; import java.io.File; import java.io.IOException; import java.io.InputStreamReader; import java.util.concurrent.atomic.AtomicBoolean; +import org.apache.cassandra.tools.NodeProbe; +import org.apache.commons.io.IOUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /* * This task checks if the Cassandra process is running. @@ -50,7 +49,11 @@ public class CassandraMonitor extends Task { private final CassMonitorMetrics cassMonitorMetrics; @Inject - protected CassandraMonitor(IConfiguration config, InstanceState instanceState, ICassandraProcess cassProcess, CassMonitorMetrics cassMonitorMetrics) { + protected CassandraMonitor( + IConfiguration config, + InstanceState instanceState, + ICassandraProcess cassProcess, + CassMonitorMetrics cassMonitorMetrics) { super(config); this.instanceState = instanceState; this.cassProcess = cassProcess; @@ -59,11 +62,10 @@ protected CassandraMonitor(IConfiguration config, InstanceState instanceState, I @Override public void execute() throws Exception { - try{ + try { checkRequiredDirectories(); instanceState.setIsRequiredDirectoriesExist(true); - }catch (IllegalStateException e) - { + } catch (IllegalStateException e) { instanceState.setIsRequiredDirectoriesExist(false); } @@ -71,14 +73,20 @@ public void execute() throws Exception { BufferedReader input = null; try { // This returns pid for the Cassandra process - // This needs to be sent as command list as "pipe" of results is not allowed. Also, do not try to change - // with pgrep as it has limitation of 4K command list (cassandra command can go upto 5-6 KB as cassandra lists all the libraries in command. - final String[] cmd = { "/bin/sh", "-c", "ps -ef |grep -v -P \"\\sgrep\\s\" | grep " + config.getCassProcessName()}; + // This needs to be sent as command list as "pipe" of results is not allowed. Also, do + // not try to change + // with pgrep as it has limitation of 4K command list (cassandra command can go upto 5-6 + // KB as cassandra lists all the libraries in command. + final String[] cmd = { + "/bin/sh", + "-c", + "ps -ef |grep -v -P \"\\sgrep\\s\" | grep " + config.getCassProcessName() + }; process = Runtime.getRuntime().exec(cmd); input = new BufferedReader(new InputStreamReader(process.getInputStream())); String line = input.readLine(); if (line != null) { - //Setting cassandra flag to true + // Setting cassandra flag to true instanceState.setCassandraProcessAlive(true); isCassandraStarted.set(true); NodeProbe bean = JMXNodeTool.instance(this.config); @@ -86,7 +94,7 @@ public void execute() throws Exception { instanceState.setIsNativeTransportActive(bean.isNativeTransportRunning()); instanceState.setIsThriftActive(bean.isThriftServerRunning()); } else { - //Setting cassandra flag to false + // Setting cassandra flag to false instanceState.setCassandraProcessAlive(false); isCassandraStarted.set(false); } @@ -101,16 +109,18 @@ public void execute() throws Exception { IOUtils.closeQuietly(process.getErrorStream()); } - if (input != null) - IOUtils.closeQuietly(input); + if (input != null) IOUtils.closeQuietly(input); } try { int rate = config.getRemediateDeadCassandraRate(); if (rate >= 0 && !config.doesCassandraStartManually()) { - if (instanceState.shouldCassandraBeAlive() && !instanceState.isCassandraProcessAlive()) { + if (instanceState.shouldCassandraBeAlive() + && !instanceState.isCassandraProcessAlive()) { long msNow = System.currentTimeMillis(); - if (rate == 0 || ((instanceState.getLastAttemptedStartTime() + rate * 1000) < msNow)) { + if (rate == 0 + || ((instanceState.getLastAttemptedStartTime() + rate * 1000) + < msNow)) { cassMonitorMetrics.incCassAutoStart(); cassProcess.start(true); instanceState.markLastAttemptedStartTime(); @@ -137,10 +147,13 @@ private void checkDirectory(String directory) { private void checkDirectory(File directory) { if (!directory.exists()) - throw new IllegalStateException(String.format("Directory: {} does not exist", directory)); + throw new IllegalStateException( + String.format("Directory: {} does not exist", directory)); if (!directory.canRead() || !directory.canWrite()) - throw new IllegalStateException(String.format("Directory: {} does not have read/write permissions.", directory)); + throw new IllegalStateException( + String.format( + "Directory: {} does not have read/write permissions.", directory)); } public static TaskTimer getTimer() { @@ -156,10 +169,9 @@ public static Boolean hasCassadraStarted() { return isCassandraStarted.get(); } - //Added for testing only + // Added for testing only public static void setIsCassadraStarted() { - //Setting cassandra flag to true + // Setting cassandra flag to true isCassandraStarted.set(true); } - } diff --git a/priam/src/main/java/com/netflix/priam/utils/DateUtil.java b/priam/src/main/java/com/netflix/priam/utils/DateUtil.java index 353947bda..a5a67bc30 100644 --- a/priam/src/main/java/com/netflix/priam/utils/DateUtil.java +++ b/priam/src/main/java/com/netflix/priam/utils/DateUtil.java @@ -17,31 +17,26 @@ package com.netflix.priam.utils; - -import org.apache.commons.lang3.StringUtils; -import org.apache.http.client.utils.DateUtils; - -import javax.inject.Singleton; import java.time.Instant; import java.time.LocalDateTime; import java.time.ZoneId; import java.time.format.DateTimeFormatter; import java.time.format.DateTimeParseException; import java.util.Date; +import javax.inject.Singleton; +import org.apache.commons.lang3.StringUtils; +import org.apache.http.client.utils.DateUtils; -/** - * Utility functions for date. - * Created by aagrawal on 7/10/17. - */ +/** Utility functions for date. Created by aagrawal on 7/10/17. */ @Singleton public class DateUtil { - public final static String yyyyMMdd = "yyyyMMdd"; - public final static String yyyyMMddHHmm = "yyyyMMddHHmm"; - public final static String ddMMyyyyHHmm = "ddMMyyyyHHmm"; - private final static String[] patterns = {yyyyMMddHHmm, yyyyMMdd, ddMMyyyyHHmm}; - private final static ZoneId defaultZoneId = ZoneId.systemDefault(); - private final static ZoneId utcZoneId = ZoneId.of("UTC"); + public static final String yyyyMMdd = "yyyyMMdd"; + public static final String yyyyMMddHHmm = "yyyyMMddHHmm"; + public static final String ddMMyyyyHHmm = "ddMMyyyyHHmm"; + private static final String[] patterns = {yyyyMMddHHmm, yyyyMMdd, ddMMyyyyHHmm}; + private static final ZoneId defaultZoneId = ZoneId.systemDefault(); + private static final ZoneId utcZoneId = ZoneId.of("UTC"); /** * Format the given date in format yyyyMMdd @@ -50,8 +45,7 @@ public class DateUtil { * @return date formatted in yyyyMMdd */ public static String formatyyyyMMdd(Date date) { - if (date == null) - return null; + if (date == null) return null; return DateUtils.formatDate(date, yyyyMMdd); } @@ -62,15 +56,14 @@ public static String formatyyyyMMdd(Date date) { * @return date formatted in yyyyMMddHHmm */ public static String formatyyyyMMddHHmm(Date date) { - if (date == null) - return null; + if (date == null) return null; return DateUtils.formatDate(date, yyyyMMddHHmm); } /** * Format the given date in given format * - * @param date to format + * @param date to format * @param pattern e.g. yyyyMMddHHmm * @return formatted date */ @@ -85,17 +78,17 @@ public static String formatDate(Date date, String pattern) { * @return the parsed date or null if input could not be parsed */ public static Date getDate(String date) { - if (StringUtils.isEmpty(date)) - return null; + if (StringUtils.isEmpty(date)) return null; return DateUtils.parseDate(date, patterns); } /** * Convert date to LocalDateTime using system default zone. + * * @param date Date to be transformed * @return converted date to LocalDateTime */ - public static LocalDateTime convert(Date date){ + public static LocalDateTime convert(Date date) { if (date == null) return null; return date.toInstant().atZone(defaultZoneId).toLocalDateTime(); } @@ -106,7 +99,7 @@ public static LocalDateTime convert(Date date){ * @param date to format * @return date formatted in yyyyMMdd */ - public static String formatyyyyMMdd(LocalDateTime date){ + public static String formatyyyyMMdd(LocalDateTime date) { if (date == null) return null; return date.format(DateTimeFormatter.ofPattern(yyyyMMdd)); } @@ -128,35 +121,36 @@ public static String formatyyyyMMddHHmm(LocalDateTime date) { * @param date to parse. Accepted formats are yyyyMMddHHmm and yyyyMMdd * @return the parsed LocalDateTime or null if input could not be parsed */ - public static LocalDateTime getLocalDateTime(String date){ - if (StringUtils.isEmpty(date)) - return null; - - for (String pattern : patterns){ - LocalDateTime localDateTime = LocalDateTime.parse(date, DateTimeFormatter.ofPattern(pattern)); - if (localDateTime != null) - return localDateTime; + public static LocalDateTime getLocalDateTime(String date) { + if (StringUtils.isEmpty(date)) return null; + + for (String pattern : patterns) { + LocalDateTime localDateTime = + LocalDateTime.parse(date, DateTimeFormatter.ofPattern(pattern)); + if (localDateTime != null) return localDateTime; } return null; } /** * Return the current instant + * * @return the instant */ - public static Instant getInstant(){ + public static Instant getInstant() { return Instant.now(); } /** - * Format the instant based on the pattern passed. If instant or pattern is null, null is returned. + * Format the instant based on the pattern passed. If instant or pattern is null, null is + * returned. + * * @param pattern Pattern that should * @param instant Instant in time * @return The formatted instant based on the pattern. Null, if pattern or instant is null. */ - public static String formatInstant(String pattern, Instant instant){ - if (instant == null || StringUtils.isEmpty(pattern)) - return null; + public static String formatInstant(String pattern, Instant instant) { + if (instant == null || StringUtils.isEmpty(pattern)) return null; DateTimeFormatter formatter = DateTimeFormatter.ofPattern(pattern).withZone(utcZoneId); return formatter.format(instant); @@ -164,24 +158,24 @@ public static String formatInstant(String pattern, Instant instant){ /** * Parse the dateTime string to Instant based on the predefined set of patterns. + * * @param dateTime DateTime string that needs to be parsed. * @return Instant object depicting the date/time. */ - public static final Instant parseInstant(String dateTime){ - if (StringUtils.isEmpty(dateTime)) - return null; + public static final Instant parseInstant(String dateTime) { + if (StringUtils.isEmpty(dateTime)) return null; - for (String pattern : patterns){ + for (String pattern : patterns) { try { - Instant instant = DateTimeFormatter.ofPattern(pattern).withZone(utcZoneId).parse(dateTime, Instant::from); - if (instant != null) - return instant; - }catch (DateTimeParseException e) - { - //Do nothing. + Instant instant = + DateTimeFormatter.ofPattern(pattern) + .withZone(utcZoneId) + .parse(dateTime, Instant::from); + if (instant != null) return instant; + } catch (DateTimeParseException e) { + // Do nothing. } } return null; } - } diff --git a/priam/src/main/java/com/netflix/priam/utils/ExponentialRetryCallable.java b/priam/src/main/java/com/netflix/priam/utils/ExponentialRetryCallable.java index 02f4e8691..e3b11498b 100644 --- a/priam/src/main/java/com/netflix/priam/utils/ExponentialRetryCallable.java +++ b/priam/src/main/java/com/netflix/priam/utils/ExponentialRetryCallable.java @@ -16,14 +16,13 @@ */ package com.netflix.priam.utils; +import java.util.concurrent.CancellationException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.concurrent.CancellationException; - public abstract class ExponentialRetryCallable extends RetryableCallable { - public final static long MAX_SLEEP = 240000; - public final static long MIN_SLEEP = 200; + public static final long MAX_SLEEP = 240000; + public static final long MIN_SLEEP = 200; private static final Logger logger = LoggerFactory.getLogger(ExponentialRetryCallable.class); private final long max; @@ -40,7 +39,7 @@ public ExponentialRetryCallable(long minSleep, long maxSleep) { } public T call() throws Exception { - long delay = min;// ms + long delay = min; // ms while (true) { try { return retriableCall(); @@ -58,5 +57,4 @@ public T call() throws Exception { } } } - } diff --git a/priam/src/main/java/com/netflix/priam/utils/FifoQueue.java b/priam/src/main/java/com/netflix/priam/utils/FifoQueue.java index bb1417375..4cb9e60b7 100644 --- a/priam/src/main/java/com/netflix/priam/utils/FifoQueue.java +++ b/priam/src/main/java/com/netflix/priam/utils/FifoQueue.java @@ -35,7 +35,6 @@ public FifoQueue(int capacity, Comparator comparator) { public synchronized void adjustAndAdd(E e) { add(e); - if (capacity < size()) - pollFirst(); + if (capacity < size()) pollFirst(); } } diff --git a/priam/src/main/java/com/netflix/priam/utils/GsonJsonSerializer.java b/priam/src/main/java/com/netflix/priam/utils/GsonJsonSerializer.java index 08a37e866..b4775b329 100644 --- a/priam/src/main/java/com/netflix/priam/utils/GsonJsonSerializer.java +++ b/priam/src/main/java/com/netflix/priam/utils/GsonJsonSerializer.java @@ -20,7 +20,6 @@ import com.google.gson.stream.JsonReader; import com.google.gson.stream.JsonToken; import com.google.gson.stream.JsonWriter; - import java.io.IOException; import java.lang.annotation.Retention; import java.lang.annotation.RetentionPolicy; @@ -30,23 +29,22 @@ import java.time.LocalDateTime; import java.util.Date; -/** - * Created by aagrawal on 10/12/17. - */ +/** Created by aagrawal on 10/12/17. */ public class GsonJsonSerializer { - private static final Gson gson = new GsonBuilder() - //.serializeNulls() - .serializeSpecialFloatingPointValues() - .setPrettyPrinting() - .disableHtmlEscaping() - .registerTypeAdapter(Date.class, new DateTypeAdapter()) - .registerTypeAdapter(LocalDateTime.class, new LocalDateTimeTypeAdapter()) - .registerTypeAdapter(Instant.class, new InstantTypeAdapter()) - .registerTypeAdapter(Path.class, new PathTypeAdapter()) - .setExclusionStrategies(new PriamAnnotationExclusionStrategy()) - .create(); - - public static Gson getGson(){ + private static final Gson gson = + new GsonBuilder() + // .serializeNulls() + .serializeSpecialFloatingPointValues() + .setPrettyPrinting() + .disableHtmlEscaping() + .registerTypeAdapter(Date.class, new DateTypeAdapter()) + .registerTypeAdapter(LocalDateTime.class, new LocalDateTimeTypeAdapter()) + .registerTypeAdapter(Instant.class, new InstantTypeAdapter()) + .registerTypeAdapter(Path.class, new PathTypeAdapter()) + .setExclusionStrategies(new PriamAnnotationExclusionStrategy()) + .create(); + + public static Gson getGson() { return gson; } @@ -63,7 +61,7 @@ public boolean shouldSkipField(FieldAttributes f) { public static class PriamAnnotation { @Retention(RetentionPolicy.RUNTIME) -// @Target({ElementType.FIELD,ElementType.METHOD}) + // @Target({ElementType.FIELD,ElementType.METHOD}) public @interface GsonIgnore { // Field tag only annotation } @@ -71,8 +69,7 @@ public static class PriamAnnotation { static class DateTypeAdapter extends TypeAdapter { @Override - public void write(JsonWriter out, Date value) - throws IOException { + public void write(JsonWriter out, Date value) throws IOException { out.value(DateUtil.formatyyyyMMddHHmm(value)); } @@ -87,14 +84,12 @@ public Date read(JsonReader in) throws IOException { return null; } return DateUtil.getDate(result); - } } static class LocalDateTimeTypeAdapter extends TypeAdapter { @Override - public void write(JsonWriter out, LocalDateTime value) - throws IOException { + public void write(JsonWriter out, LocalDateTime value) throws IOException { out.value(DateUtil.formatyyyyMMddHHmm(value)); } @@ -109,19 +104,17 @@ public LocalDateTime read(JsonReader in) throws IOException { return null; } return DateUtil.getLocalDateTime(result); - } } static class InstantTypeAdapter extends TypeAdapter { @Override - public void write(JsonWriter out, Instant value) - throws IOException { - out.value(getEpoch(value)); + public void write(JsonWriter out, Instant value) throws IOException { + out.value(getEpoch(value)); } - private long getEpoch(Instant value){ - return (value == null)? null: value.toEpochMilli(); + private long getEpoch(Instant value) { + return (value == null) ? null : value.toEpochMilli(); } @Override @@ -135,15 +128,13 @@ public Instant read(JsonReader in) throws IOException { return null; } return Instant.ofEpochMilli(Long.parseLong(result)); - } } static class PathTypeAdapter extends TypeAdapter { @Override - public void write(JsonWriter out, Path value) - throws IOException { - String fileName = (value != null)? value.toFile().getName():null; + public void write(JsonWriter out, Path value) throws IOException { + String fileName = (value != null) ? value.toFile().getName() : null; out.value(fileName); } @@ -158,7 +149,6 @@ public Path read(JsonReader in) throws IOException { return null; } return Paths.get(result); - } } } diff --git a/priam/src/main/java/com/netflix/priam/utils/INodeToolObservable.java b/priam/src/main/java/com/netflix/priam/utils/INodeToolObservable.java index 5e70530f4..5f6505347 100644 --- a/priam/src/main/java/com/netflix/priam/utils/INodeToolObservable.java +++ b/priam/src/main/java/com/netflix/priam/utils/INodeToolObservable.java @@ -1,16 +1,14 @@ /** * Copyright 2017 Netflix, Inc. - *

- * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and + * + *

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file + * except in compliance with the License. You may obtain a copy of the License at + * + *

http://www.apache.org/licenses/LICENSE-2.0 + * + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing permissions and * limitations under the License. */ package com.netflix.priam.utils; diff --git a/priam/src/main/java/com/netflix/priam/utils/INodeToolObserver.java b/priam/src/main/java/com/netflix/priam/utils/INodeToolObserver.java index 154b051ce..7d094e5aa 100644 --- a/priam/src/main/java/com/netflix/priam/utils/INodeToolObserver.java +++ b/priam/src/main/java/com/netflix/priam/utils/INodeToolObserver.java @@ -1,16 +1,14 @@ /** * Copyright 2017 Netflix, Inc. - *

- * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and + * + *

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file + * except in compliance with the License. You may obtain a copy of the License at + * + *

http://www.apache.org/licenses/LICENSE-2.0 + * + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing permissions and * limitations under the License. */ package com.netflix.priam.utils; diff --git a/priam/src/main/java/com/netflix/priam/utils/ITokenManager.java b/priam/src/main/java/com/netflix/priam/utils/ITokenManager.java index 506ec2043..7ac64288a 100644 --- a/priam/src/main/java/com/netflix/priam/utils/ITokenManager.java +++ b/priam/src/main/java/com/netflix/priam/utils/ITokenManager.java @@ -17,7 +17,6 @@ package com.netflix.priam.utils; import com.google.inject.ImplementedBy; - import java.math.BigInteger; import java.util.List; diff --git a/priam/src/main/java/com/netflix/priam/utils/JMXConnectionException.java b/priam/src/main/java/com/netflix/priam/utils/JMXConnectionException.java index e20faab8e..20269da1a 100644 --- a/priam/src/main/java/com/netflix/priam/utils/JMXConnectionException.java +++ b/priam/src/main/java/com/netflix/priam/utils/JMXConnectionException.java @@ -1,16 +1,14 @@ /** * Copyright 2017 Netflix, Inc. - *

- * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and + * + *

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file + * except in compliance with the License. You may obtain a copy of the License at + * + *

http://www.apache.org/licenses/LICENSE-2.0 + * + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing permissions and * limitations under the License. */ package com.netflix.priam.utils; @@ -28,5 +26,4 @@ public JMXConnectionException(String message) { public JMXConnectionException(String message, Exception e) { super(message, e); } - } diff --git a/priam/src/main/java/com/netflix/priam/utils/JMXConnectorMgr.java b/priam/src/main/java/com/netflix/priam/utils/JMXConnectorMgr.java index 5cea1d2ae..309c0da10 100644 --- a/priam/src/main/java/com/netflix/priam/utils/JMXConnectorMgr.java +++ b/priam/src/main/java/com/netflix/priam/utils/JMXConnectorMgr.java @@ -1,32 +1,29 @@ /** * Copyright 2017 Netflix, Inc. - *

- * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and + * + *

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file + * except in compliance with the License. You may obtain a copy of the License at + * + *

http://www.apache.org/licenses/LICENSE-2.0 + * + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing permissions and * limitations under the License. */ package com.netflix.priam.utils; import com.netflix.priam.config.IConfiguration; +import java.io.IOException; import org.apache.cassandra.tools.NodeProbe; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.IOException; - /** - * Represents a connection to remote JMX mbean server. This object differs from JMXNodeTool as it is meant for short - * lived connection to remote mbean server. + * Represents a connection to remote JMX mbean server. This object differs from JMXNodeTool as it is + * meant for short lived connection to remote mbean server. * - * Created by vinhn on 10/11/16. + *

Created by vinhn on 10/11/16. */ public class JMXConnectorMgr extends NodeProbe { private static final Logger logger = LoggerFactory.getLogger(JMXConnectorMgr.class); @@ -44,7 +41,6 @@ public JMXConnectorMgr(IConfiguration config) throws IOException, InterruptedExc close the connection to remote mbean server */ public void close() throws IOException { - super.close(); //close the connection to remote mbean server + super.close(); // close the connection to remote mbean server } - } diff --git a/priam/src/main/java/com/netflix/priam/utils/JMXNodeTool.java b/priam/src/main/java/com/netflix/priam/utils/JMXNodeTool.java index 54d723d30..b87b97849 100644 --- a/priam/src/main/java/com/netflix/priam/utils/JMXNodeTool.java +++ b/priam/src/main/java/com/netflix/priam/utils/JMXNodeTool.java @@ -19,19 +19,6 @@ import com.google.inject.Inject; import com.google.inject.Singleton; import com.netflix.priam.config.IConfiguration; -import org.apache.cassandra.db.ColumnFamilyStoreMBean; -import org.apache.cassandra.repair.messages.RepairOption; -import org.apache.cassandra.tools.NodeProbe; -import org.codehaus.jettison.json.JSONArray; -import org.codehaus.jettison.json.JSONException; -import org.codehaus.jettison.json.JSONObject; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import javax.management.JMX; -import javax.management.MBeanServerConnection; -import javax.management.MalformedObjectNameException; -import javax.management.ObjectName; import java.io.IOException; import java.io.PrintStream; import java.lang.management.ManagementFactory; @@ -43,10 +30,20 @@ import java.util.*; import java.util.Map.Entry; import java.util.concurrent.ExecutionException; +import javax.management.JMX; +import javax.management.MBeanServerConnection; +import javax.management.MalformedObjectNameException; +import javax.management.ObjectName; +import org.apache.cassandra.db.ColumnFamilyStoreMBean; +import org.apache.cassandra.repair.messages.RepairOption; +import org.apache.cassandra.tools.NodeProbe; +import org.codehaus.jettison.json.JSONArray; +import org.codehaus.jettison.json.JSONException; +import org.codehaus.jettison.json.JSONObject; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; -/** - * Class to get data out of Cassandra JMX - */ +/** Class to get data out of Cassandra JMX */ @Singleton public class JMXNodeTool extends NodeProbe implements INodeToolObservable { private static final Logger logger = LoggerFactory.getLogger(JMXNodeTool.class); @@ -56,22 +53,22 @@ public class JMXNodeTool extends NodeProbe implements INodeToolObservable { private static final Set observers = new HashSet<>(); /** - * Hostname and Port to talk to will be same server for now optionally we - * might want the ip to poll. + * Hostname and Port to talk to will be same server for now optionally we might want the ip to + * poll. * - * NOTE: This class shouldn't be a singleton and this shouldn't be cached. + *

NOTE: This class shouldn't be a singleton and this shouldn't be cached. * - * This will work only if cassandra runs. + *

This will work only if cassandra runs. */ public JMXNodeTool(String host, int port) throws IOException, InterruptedException { super(host, port); } - public JMXNodeTool(String host, int port, String username, String password) throws IOException, InterruptedException { + public JMXNodeTool(String host, int port, String username, String password) + throws IOException, InterruptedException { super(host, port, username, password); } - @Inject public JMXNodeTool(IConfiguration config) throws IOException, InterruptedException { super("localhost", config.getJmxPort()); @@ -79,46 +76,54 @@ public JMXNodeTool(IConfiguration config) throws IOException, InterruptedExcepti /** * try to create if it is null. + * * @throws JMXConnectionException */ public static JMXNodeTool instance(IConfiguration config) throws JMXConnectionException { - if (!testConnection()) - tool = connect(config); + if (!testConnection()) tool = connect(config); return tool; } - public static T getRemoteBean(Class clazz, String mbeanName, IConfiguration config, boolean mxbean) throws IOException, MalformedObjectNameException { + public static T getRemoteBean( + Class clazz, String mbeanName, IConfiguration config, boolean mxbean) + throws IOException, MalformedObjectNameException { if (mxbean) - return ManagementFactory.newPlatformMXBeanProxy(JMXNodeTool.instance(config).mbeanServerConn, mbeanName, clazz); + return ManagementFactory.newPlatformMXBeanProxy( + JMXNodeTool.instance(config).mbeanServerConn, mbeanName, clazz); else - return JMX.newMBeanProxy(JMXNodeTool.instance(config).mbeanServerConn, new ObjectName(mbeanName), clazz); - + return JMX.newMBeanProxy( + JMXNodeTool.instance(config).mbeanServerConn, new ObjectName(mbeanName), clazz); } /** - * This method will test if you can connect and query something before handing over the connection, - * This is required for our retry logic. + * This method will test if you can connect and query something before handing over the + * connection, This is required for our retry logic. + * * @return */ private static boolean testConnection() { // connecting first time hence return false. - if (tool == null) - return false; + if (tool == null) return false; try { MBeanServerConnection serverConn = tool.mbeanServerConn; if (serverConn == null) { - logger.info("Test connection to remove MBean server failed as there is no connection."); + logger.info( + "Test connection to remove MBean server failed as there is no connection."); return false; } - if (serverConn.getMBeanCount() < 1) { //If C* is up, it should have at multiple MBeans registered. - logger.info("Test connection to remove MBean server failed as there is no registered MBeans."); + if (serverConn.getMBeanCount() + < 1) { // If C* is up, it should have at multiple MBeans registered. + logger.info( + "Test connection to remove MBean server failed as there is no registered MBeans."); return false; } } catch (Throwable ex) { SystemUtils.closeQuietly(tool); - logger.error("Exception while checking JMX connection to C*, msg: {}", ex.getLocalizedMessage()); + logger.error( + "Exception while checking JMX connection to C*, msg: {}", + ex.getLocalizedMessage()); return false; } return true; @@ -126,73 +131,94 @@ private static boolean testConnection() { /** * A means to clean up existing and recreate the JMX connection to the Cassandra process. + * * @return the new connection. */ - public static synchronized JMXNodeTool createNewConnection(final IConfiguration config) throws JMXConnectionException { + public static synchronized JMXNodeTool createNewConnection(final IConfiguration config) + throws JMXConnectionException { return createConnection(config); } - public static synchronized JMXNodeTool connect(final IConfiguration config) throws JMXConnectionException { - //lets make sure some other monitor didn't sneak in the recreated the connection already + public static synchronized JMXNodeTool connect(final IConfiguration config) + throws JMXConnectionException { + // lets make sure some other monitor didn't sneak in the recreated the connection already if (!testConnection()) { if (tool != null) { try { - tool.close(); //Ensure we properly close any existing (even if it's corrupted) connection to the remote jmx agent + tool.close(); // Ensure we properly close any existing (even if it's + // corrupted) connection to the remote jmx agent } catch (IOException e) { - logger.warn("Exception performing house cleaning -- closing current connection to jmx remote agent. Msg: {}", e.getLocalizedMessage(), e); + logger.warn( + "Exception performing house cleaning -- closing current connection to jmx remote agent. Msg: {}", + e.getLocalizedMessage(), + e); } } } else { - //Someone beat you and already created the connection, nothing you need to do.. + // Someone beat you and already created the connection, nothing you need to do.. return tool; } return createConnection(config); } - private static JMXNodeTool createConnection(final IConfiguration config) throws JMXConnectionException { + private static JMXNodeTool createConnection(final IConfiguration config) + throws JMXConnectionException { // If Cassandra is started then only start the monitoring if (!CassandraMonitor.hasCassadraStarted()) { - String exceptionMsg = "Cannot perform connection to remove jmx agent as Cassandra has not yet started, check back again later"; + String exceptionMsg = + "Cannot perform connection to remove jmx agent as Cassandra has not yet started, check back again later"; logger.debug(exceptionMsg); throw new JMXConnectionException(exceptionMsg); } - if (tool != null) { //lets make sure we properly close any existing (even if it's corrupted) connection to the remote jmx agent + if (tool + != null) { // lets make sure we properly close any existing (even if it's corrupted) + // connection to the remote jmx agent try { tool.close(); } catch (IOException e) { - logger.warn("Exception performing house cleaning -- closing current connection to jmx remote agent. Msg: {}", e.getLocalizedMessage(), e); + logger.warn( + "Exception performing house cleaning -- closing current connection to jmx remote agent. Msg: {}", + e.getLocalizedMessage(), + e); } } try { - tool = new BoundedExponentialRetryCallable() { - @Override - public JMXNodeTool retriableCall() throws Exception { - JMXNodeTool nodetool; - if ((config.getJmxUsername() == null || config.getJmxUsername().isEmpty()) && - (config.getJmxPassword() == null || config.getJmxPassword().isEmpty())) { - nodetool = new JMXNodeTool("localhost", config.getJmxPort()); - } - else { - nodetool = new JMXNodeTool("localhost", config.getJmxPort(), config.getJmxUsername(), config.getJmxPassword()); - } - - Field fields[] = NodeProbe.class.getDeclaredFields(); - for (Field field : fields) { - if (!field.getName().equals("mbeanServerConn")) - continue; - field.setAccessible(true); - nodetool.mbeanServerConn = (MBeanServerConnection) field.get(nodetool); - } - - return nodetool; - } - }.call(); + tool = + new BoundedExponentialRetryCallable() { + @Override + public JMXNodeTool retriableCall() throws Exception { + JMXNodeTool nodetool; + if ((config.getJmxUsername() == null + || config.getJmxUsername().isEmpty()) + && (config.getJmxPassword() == null + || config.getJmxPassword().isEmpty())) { + nodetool = new JMXNodeTool("localhost", config.getJmxPort()); + } else { + nodetool = + new JMXNodeTool( + "localhost", + config.getJmxPort(), + config.getJmxUsername(), + config.getJmxPassword()); + } + + Field fields[] = NodeProbe.class.getDeclaredFields(); + for (Field field : fields) { + if (!field.getName().equals("mbeanServerConn")) continue; + field.setAccessible(true); + nodetool.mbeanServerConn = + (MBeanServerConnection) field.get(nodetool); + } + + return nodetool; + } + }.call(); } catch (Exception e) { logger.error(e.getMessage(), e); @@ -208,12 +234,13 @@ public JMXNodeTool retriableCall() throws Exception { } /** - * You must do the compaction before running this to remove the duplicate - * tokens out of the server. TODO code it. + * You must do the compaction before running this to remove the duplicate tokens out of the + * server. TODO code it. */ @SuppressWarnings("unchecked") public JSONObject estimateKeys() throws JSONException { - Iterator> it = super.getColumnFamilyStoreMBeanProxies(); + Iterator> it = + super.getColumnFamilyStoreMBeanProxies(); JSONObject object = new JSONObject(); while (it.hasNext()) { Entry entry = it.next(); @@ -279,29 +306,38 @@ public JSONArray ring(String keyspace) throws JSONException { } catch (UnknownHostException e) { rack = "Unknown"; } - String status = liveNodes.contains(primaryEndpoint) - ? "Up" - : deadNodes.contains(primaryEndpoint) - ? "Down" - : "?"; + String status = + liveNodes.contains(primaryEndpoint) + ? "Up" + : deadNodes.contains(primaryEndpoint) ? "Down" : "?"; String state = "Normal"; - if (joiningNodes.contains(primaryEndpoint)) - state = "Joining"; - else if (leavingNodes.contains(primaryEndpoint)) - state = "Leaving"; - else if (movingNodes.contains(primaryEndpoint)) - state = "Moving"; + if (joiningNodes.contains(primaryEndpoint)) state = "Joining"; + else if (leavingNodes.contains(primaryEndpoint)) state = "Leaving"; + else if (movingNodes.contains(primaryEndpoint)) state = "Moving"; String load = loadMap.getOrDefault(primaryEndpoint, "?"); - String owns = new DecimalFormat("##0.00%").format(ownerships.get(token) == null ? 0.0F : ownerships.get(token)); - ring.put(createJson(primaryEndpoint, dataCenter, rack, status, state, load, owns, token)); + String owns = + new DecimalFormat("##0.00%") + .format(ownerships.get(token) == null ? 0.0F : ownerships.get(token)); + ring.put( + createJson( + primaryEndpoint, dataCenter, rack, status, state, load, owns, token)); } return ring; } - private JSONObject createJson(String primaryEndpoint, String dataCenter, String rack, String status, String state, String load, String owns, String token) throws JSONException { + private JSONObject createJson( + String primaryEndpoint, + String dataCenter, + String rack, + String status, + String state, + String load, + String owns, + String token) + throws JSONException { JSONObject object = new JSONObject(); object.put("endpoint", primaryEndpoint); object.put("dc", dataCenter); @@ -314,34 +350,36 @@ private JSONObject createJson(String primaryEndpoint, String dataCenter, String return object; } - public void repair(boolean isSequential, boolean localDataCenterOnly) throws IOException, ExecutionException, InterruptedException { + public void repair(boolean isSequential, boolean localDataCenterOnly) + throws IOException, ExecutionException, InterruptedException { repair(isSequential, localDataCenterOnly, false); } - public void repair(boolean isSequential, boolean localDataCenterOnly, boolean primaryRange) throws IOException, ExecutionException, InterruptedException { - Map repairOptions = new HashMap<>(); - repairOptions.put(RepairOption.PARALLELISM_KEY, Boolean.toString(!isSequential)); - repairOptions.put(RepairOption.PRIMARY_RANGE_KEY, Boolean.toString(primaryRange)); - if (localDataCenterOnly) - repairOptions.put(RepairOption.DATACENTERS_KEY, getDataCenter()); + public void repair(boolean isSequential, boolean localDataCenterOnly, boolean primaryRange) + throws IOException, ExecutionException, InterruptedException { + Map repairOptions = new HashMap<>(); + repairOptions.put(RepairOption.PARALLELISM_KEY, Boolean.toString(!isSequential)); + repairOptions.put(RepairOption.PRIMARY_RANGE_KEY, Boolean.toString(primaryRange)); + if (localDataCenterOnly) repairOptions.put(RepairOption.DATACENTERS_KEY, getDataCenter()); PrintStream printStream = new PrintStream("repair.log"); - for (String keyspace : getKeyspaces()) - repairAsync(printStream, keyspace, repairOptions); + for (String keyspace : getKeyspaces()) repairAsync(printStream, keyspace, repairOptions); } public void cleanup() throws IOException, ExecutionException, InterruptedException { - for (String keyspace : getKeyspaces()) - forceKeyspaceCleanup(0, keyspace); + for (String keyspace : getKeyspaces()) forceKeyspaceCleanup(0, keyspace); } - - public void refresh(List keyspaces) throws IOException, ExecutionException, InterruptedException { - Iterator> it = super.getColumnFamilyStoreMBeanProxies(); + + public void refresh(List keyspaces) + throws IOException, ExecutionException, InterruptedException { + Iterator> it = + super.getColumnFamilyStoreMBeanProxies(); while (it.hasNext()) { Entry entry = it.next(); if (keyspaces.contains(entry.getKey())) { - logger.info("Refreshing {} {}", entry.getKey(), entry.getValue().getColumnFamilyName()); + logger.info( + "Refreshing {} {}", entry.getKey(), entry.getValue().getColumnFamilyName()); loadNewSSTables(entry.getKey(), entry.getValue().getColumnFamilyName()); } } @@ -355,26 +393,20 @@ public void close() throws IOException { } } - /** - * @param observer to add to list of internal observers. This behavior is thread-safe. - */ + /** @param observer to add to list of internal observers. This behavior is thread-safe. */ @Override public void addObserver(INodeToolObserver observer) { - if (observer == null) - throw new NullPointerException("Cannot not observer."); + if (observer == null) throw new NullPointerException("Cannot not observer."); synchronized (observers) { - observers.add(observer); //if observer exist, it's a noop + observers.add(observer); // if observer exist, it's a noop } - } - /** - * @param observer to be removed; behavior is thread-safe. - */ + /** @param observer to be removed; behavior is thread-safe. */ @Override public void deleteObserver(INodeToolObserver observer) { synchronized (observers) { observers.remove(observer); } } -} \ No newline at end of file +} diff --git a/priam/src/main/java/com/netflix/priam/utils/MaxSizeHashMap.java b/priam/src/main/java/com/netflix/priam/utils/MaxSizeHashMap.java index 8cf81f206..0da65266a 100644 --- a/priam/src/main/java/com/netflix/priam/utils/MaxSizeHashMap.java +++ b/priam/src/main/java/com/netflix/priam/utils/MaxSizeHashMap.java @@ -20,12 +20,10 @@ import java.util.LinkedHashMap; import java.util.Map; -/** - * Created by aagrawal on 7/11/17. - */ +/** Created by aagrawal on 7/11/17. */ /* - Limit the size of the hashmap using FIFO algorithm. - */ +Limit the size of the hashmap using FIFO algorithm. +*/ public class MaxSizeHashMap extends LinkedHashMap { private final int maxSize; @@ -38,4 +36,3 @@ protected boolean removeEldestEntry(Map.Entry eldest) { return size() > maxSize; } } - diff --git a/priam/src/main/java/com/netflix/priam/utils/RetryableCallable.java b/priam/src/main/java/com/netflix/priam/utils/RetryableCallable.java index 105345c9d..813bc1bd4 100644 --- a/priam/src/main/java/com/netflix/priam/utils/RetryableCallable.java +++ b/priam/src/main/java/com/netflix/priam/utils/RetryableCallable.java @@ -16,13 +16,12 @@ */ package com.netflix.priam.utils; +import java.util.concurrent.Callable; +import java.util.concurrent.CancellationException; import org.apache.commons.lang3.exception.ExceptionUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.concurrent.Callable; -import java.util.concurrent.CancellationException; - public abstract class RetryableCallable implements Callable { private static final Logger logger = LoggerFactory.getLogger(RetryableCallable.class); private static final int DEFAULT_NUMBER_OF_RETRIES = 15; @@ -72,4 +71,4 @@ public T call() throws Exception { protected void forEachExecution() { // do nothing by default. } -} \ No newline at end of file +} diff --git a/priam/src/main/java/com/netflix/priam/utils/Sleeper.java b/priam/src/main/java/com/netflix/priam/utils/Sleeper.java index b94e3ad42..4d864bdf2 100644 --- a/priam/src/main/java/com/netflix/priam/utils/Sleeper.java +++ b/priam/src/main/java/com/netflix/priam/utils/Sleeper.java @@ -18,9 +18,7 @@ import com.google.inject.ImplementedBy; -/** - * An abstraction to {@link Thread#sleep(long)} so we can mock it in tests. - */ +/** An abstraction to {@link Thread#sleep(long)} so we can mock it in tests. */ @ImplementedBy(ThreadSleeper.class) public interface Sleeper { void sleep(long waitTimeMs) throws InterruptedException; diff --git a/priam/src/main/java/com/netflix/priam/utils/SystemUtils.java b/priam/src/main/java/com/netflix/priam/utils/SystemUtils.java index a1a5a81a6..05b59b5d3 100644 --- a/priam/src/main/java/com/netflix/priam/utils/SystemUtils.java +++ b/priam/src/main/java/com/netflix/priam/utils/SystemUtils.java @@ -20,18 +20,16 @@ import com.google.common.hash.HashCode; import com.google.common.hash.Hashing; import com.google.common.io.Files; -import org.apache.commons.codec.binary.Base64; -import org.apache.commons.io.FileUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import javax.management.remote.JMXConnector; import java.io.*; import java.net.HttpURLConnection; import java.net.URL; import java.security.MessageDigest; import java.util.List; - +import javax.management.remote.JMXConnector; +import org.apache.commons.codec.binary.Base64; +import org.apache.commons.io.FileUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class SystemUtils { private static final Logger logger = LoggerFactory.getLogger(SystemUtils.class); @@ -49,8 +47,7 @@ public static String getDataFromUrl(String url) { ByteArrayOutputStream bos = new ByteArrayOutputStream(); DataInputStream d = new DataInputStream((FilterInputStream) conn.getContent()); int c; - while ((c = d.read(b, 0, b.length)) != -1) - bos.write(b, 0, c); + while ((c = d.read(b, 0, b.length)) != -1) bos.write(b, 0, c); String return_ = new String(bos.toByteArray(), Charsets.UTF_8); logger.info("Calling URL API: {} returns: {}", url, return_); conn.disconnect(); @@ -58,22 +55,19 @@ public static String getDataFromUrl(String url) { } catch (Exception ex) { throw new RuntimeException(ex); } - } /** * delete all the files/dirs in the given Directory but dont delete the dir itself. * - * @param dirPath The directory path where all the child directories exist. + * @param dirPath The directory path where all the child directories exist. * @param childdirs List of child directories to be cleaned up in the dirPath * @throws IOException If there is any error encountered during cleanup. */ public static void cleanupDir(String dirPath, List childdirs) throws IOException { - if (childdirs == null || childdirs.size() == 0) - FileUtils.cleanDirectory(new File(dirPath)); + if (childdirs == null || childdirs.size() == 0) FileUtils.cleanDirectory(new File(dirPath)); else { - for (String cdir : childdirs) - FileUtils.cleanDirectory(new File(dirPath + "/" + cdir)); + for (String cdir : childdirs) FileUtils.cleanDirectory(new File(dirPath + "/" + cdir)); } } @@ -82,8 +76,7 @@ public static void createDirs(String location) { if (dirFile.exists() && dirFile.isFile()) { dirFile.delete(); dirFile.mkdirs(); - } else if (!dirFile.exists()) - dirFile.mkdirs(); + } else if (!dirFile.exists()) dirFile.mkdirs(); } public static byte[] md5(byte[] buf) { @@ -136,7 +129,6 @@ public static void closeQuietly(JMXNodeTool tool) { } catch (Exception e) { logger.warn("failed to close jxm node tool", e); } - } public static void closeQuietly(JMXConnector jmc) { diff --git a/priam/src/main/java/com/netflix/priam/utils/ThreadSleeper.java b/priam/src/main/java/com/netflix/priam/utils/ThreadSleeper.java index 412179817..57f4f1b71 100644 --- a/priam/src/main/java/com/netflix/priam/utils/ThreadSleeper.java +++ b/priam/src/main/java/com/netflix/priam/utils/ThreadSleeper.java @@ -16,9 +16,7 @@ */ package com.netflix.priam.utils; -/** - * Sleeper impl that delegates to Thread.sleep - */ +/** Sleeper impl that delegates to Thread.sleep */ public class ThreadSleeper implements Sleeper { @Override public void sleep(long waitTimeMs) throws InterruptedException { @@ -29,10 +27,7 @@ public void sleepQuietly(long waitTimeMs) { try { sleep(waitTimeMs); } catch (InterruptedException e) { - //no-op + // no-op } - } - - -} \ No newline at end of file +} diff --git a/priam/src/main/java/com/netflix/priam/utils/TokenManager.java b/priam/src/main/java/com/netflix/priam/utils/TokenManager.java index e7f1bdb9f..cad5906f7 100644 --- a/priam/src/main/java/com/netflix/priam/utils/TokenManager.java +++ b/priam/src/main/java/com/netflix/priam/utils/TokenManager.java @@ -21,7 +21,6 @@ import com.google.common.collect.Ordering; import com.google.inject.Inject; import com.netflix.priam.config.IConfiguration; - import java.math.BigInteger; import java.util.Collections; import java.util.List; @@ -32,7 +31,6 @@ public class TokenManager implements ITokenManager { public static final BigInteger MINIMUM_TOKEN_MURMUR3 = new BigInteger("-2").pow(63); public static final BigInteger MAXIMUM_TOKEN_MURMUR3 = new BigInteger("2").pow(63); - private final BigInteger minimumToken; private final BigInteger maximumToken; private final BigInteger tokenRangeSize; @@ -54,13 +52,14 @@ public TokenManager(IConfiguration config) { } /** - * Calculate a token for the given position, evenly spaced from other size-1 nodes. See + * Calculate a token for the given position, evenly spaced from other size-1 nodes. See * http://wiki.apache.org/cassandra/Operations. * * @param size number of slots by which the token space will be divided * @param position slot number, multiplier * @param offset added to token - * @return MAXIMUM_TOKEN / size * position + offset, if <= MAXIMUM_TOKEN, otherwise wrap around the MINIMUM_TOKEN + * @return MAXIMUM_TOKEN / size * position + offset, if <= MAXIMUM_TOKEN, otherwise wrap around + * the MINIMUM_TOKEN */ @VisibleForTesting BigInteger initialToken(int size, int position, int offset) { @@ -71,7 +70,8 @@ BigInteger initialToken(int size, int position, int offset) { * unit test failures. */ Preconditions.checkArgument(position >= 0, "position must be >= 0"); - return tokenRangeSize.divide(BigInteger.valueOf(size)) + return tokenRangeSize + .divide(BigInteger.valueOf(size)) .multiply(BigInteger.valueOf(position)) .add(BigInteger.valueOf(offset)) .add(minimumToken); @@ -80,14 +80,10 @@ BigInteger initialToken(int size, int position, int offset) { /** * Creates a token given the following parameter * - * @param my_slot - * -- Slot where this instance has to be. - * @param rac_count - * -- Rac count is the numeber of RAC's - * @param rac_size - * -- number of memberships in the rac - * @param region - * -- name of the DC where it this token is created. + * @param my_slot -- Slot where this instance has to be. + * @param rac_count -- Rac count is the numeber of RAC's + * @param rac_size -- number of memberships in the rac + * @param region -- name of the DC where it this token is created. */ @Override public String createToken(int my_slot, int rac_count, int rac_size, String region) { @@ -107,17 +103,20 @@ public BigInteger findClosestToken(BigInteger tokenToSearch, List to int index = Collections.binarySearch(sortedTokens, tokenToSearch, Ordering.natural()); if (index < 0) { int i = Math.abs(index) - 1; - if ((i >= sortedTokens.size()) || (i > 0 && sortedTokens.get(i).subtract(tokenToSearch) - .compareTo(tokenToSearch.subtract(sortedTokens.get(i - 1))) > 0)) - --i; + if ((i >= sortedTokens.size()) + || (i > 0 + && sortedTokens + .get(i) + .subtract(tokenToSearch) + .compareTo( + tokenToSearch.subtract(sortedTokens.get(i - 1))) + > 0)) --i; return sortedTokens.get(i); } return sortedTokens.get(index); } - /** - * Create an offset to add to token values by hashing the region name. - */ + /** Create an offset to add to token values by hashing the region name. */ @Override public int regionOffset(String region) { return Math.abs(region.hashCode()); diff --git a/priam/src/test/java/com/netflix/priam/TestModule.java b/priam/src/test/java/com/netflix/priam/TestModule.java index 390411303..5da711e30 100644 --- a/priam/src/test/java/com/netflix/priam/TestModule.java +++ b/priam/src/test/java/com/netflix/priam/TestModule.java @@ -40,21 +40,23 @@ import org.quartz.SchedulerFactory; import org.quartz.impl.StdSchedulerFactory; - @Ignore -public class TestModule extends AbstractModule -{ +public class TestModule extends AbstractModule { @Override - protected void configure() - { - bind(IConfiguration.class).toInstance( - new FakeConfiguration(FakeConfiguration.FAKE_REGION, "fake-app", "az1", "fakeInstance1")); + protected void configure() { + bind(IConfiguration.class) + .toInstance( + new FakeConfiguration( + FakeConfiguration.FAKE_REGION, "fake-app", "az1", "fakeInstance1")); bind(IBackupRestoreConfig.class).to(FakeBackupRestoreConfig.class); bind(IPriamInstanceFactory.class).to(FakePriamInstanceFactory.class); bind(SchedulerFactory.class).to(StdSchedulerFactory.class).in(Scopes.SINGLETON); - bind(IMembership.class).toInstance(new FakeMembership( - ImmutableList.of("fakeInstance1", "fakeInstance2", "fakeInstance3"))); + bind(IMembership.class) + .toInstance( + new FakeMembership( + ImmutableList.of( + "fakeInstance1", "fakeInstance2", "fakeInstance3"))); bind(ICredential.class).to(FakeCredentials.class).in(Scopes.SINGLETON); bind(IBackupFileSystem.class).to(NullBackupFileSystem.class); bind(Sleeper.class).to(FakeSleeper.class); diff --git a/priam/src/test/java/com/netflix/priam/backup/BRTestModule.java b/priam/src/test/java/com/netflix/priam/backup/BRTestModule.java index e48e36e12..4aac0528b 100644 --- a/priam/src/test/java/com/netflix/priam/backup/BRTestModule.java +++ b/priam/src/test/java/com/netflix/priam/backup/BRTestModule.java @@ -38,31 +38,41 @@ import com.netflix.priam.utils.Sleeper; import com.netflix.spectator.api.DefaultRegistry; import com.netflix.spectator.api.Registry; +import java.util.Arrays; import org.junit.Ignore; import org.quartz.SchedulerFactory; import org.quartz.impl.StdSchedulerFactory; -import java.util.Arrays; @Ignore -public class BRTestModule extends AbstractModule -{ - +public class BRTestModule extends AbstractModule { + @Override - protected void configure() - { - bind(IConfiguration.class).toInstance(new FakeConfiguration(FakeConfiguration.FAKE_REGION, "fake-app", "az1", "fakeInstance1")); + protected void configure() { + bind(IConfiguration.class) + .toInstance( + new FakeConfiguration( + FakeConfiguration.FAKE_REGION, "fake-app", "az1", "fakeInstance1")); bind(IBackupRestoreConfig.class).to(FakeBackupRestoreConfig.class); bind(IPriamInstanceFactory.class).to(FakePriamInstanceFactory.class); bind(SchedulerFactory.class).to(StdSchedulerFactory.class).in(Scopes.SINGLETON); bind(IMembership.class).toInstance(new FakeMembership(Arrays.asList("fakeInstance1"))); bind(ICredential.class).to(FakeNullCredential.class).in(Scopes.SINGLETON); - bind(IBackupFileSystem.class).annotatedWith(Names.named("backup")).to(FakeBackupFileSystem.class).in(Scopes.SINGLETON); + bind(IBackupFileSystem.class) + .annotatedWith(Names.named("backup")) + .to(FakeBackupFileSystem.class) + .in(Scopes.SINGLETON); bind(Sleeper.class).to(FakeSleeper.class); - bind(IS3Credential.class).annotatedWith(Names.named("awss3roleassumption")).to(S3RoleAssumptionCredential.class); + bind(IS3Credential.class) + .annotatedWith(Names.named("awss3roleassumption")) + .to(S3RoleAssumptionCredential.class); - bind(IBackupFileSystem.class).annotatedWith(Names.named("encryptedbackup")).to(NullBackupFileSystem.class); - bind(IFileCryptography.class).annotatedWith(Names.named("filecryptoalgorithm")).to(PgpCryptography.class); + bind(IBackupFileSystem.class) + .annotatedWith(Names.named("encryptedbackup")) + .to(NullBackupFileSystem.class); + bind(IFileCryptography.class) + .annotatedWith(Names.named("filecryptoalgorithm")) + .to(PgpCryptography.class); bind(IIncrementalBackup.class).to(IncrementalBackup.class); bind(InstanceEnvIdentity.class).to(FakeInstanceEnvIdentity.class); bind(ICassandraProcess.class).to(FakeCassandraProcess.class); diff --git a/priam/src/test/java/com/netflix/priam/backup/FakeBackupFileSystem.java b/priam/src/test/java/com/netflix/priam/backup/FakeBackupFileSystem.java index b5adcbe88..6331352e4 100644 --- a/priam/src/test/java/com/netflix/priam/backup/FakeBackupFileSystem.java +++ b/priam/src/test/java/com/netflix/priam/backup/FakeBackupFileSystem.java @@ -25,12 +25,11 @@ import com.netflix.priam.config.IConfiguration; import com.netflix.priam.merics.BackupMetrics; import com.netflix.priam.notification.BackupNotificationMgr; -import org.json.simple.JSONArray; - import java.io.FileWriter; import java.io.IOException; import java.nio.file.Path; import java.util.*; +import org.json.simple.JSONArray; @Singleton public class FakeBackupFileSystem extends AbstractFileSystem { @@ -39,20 +38,20 @@ public class FakeBackupFileSystem extends AbstractFileSystem { public Set uploadedFiles; public String baseDir, region, clusterName; - @Inject - Provider pathProvider; + @Inject Provider pathProvider; @Inject - public FakeBackupFileSystem(IConfiguration configuration, BackupMetrics backupMetrics, - BackupNotificationMgr backupNotificationMgr){ + public FakeBackupFileSystem( + IConfiguration configuration, + BackupMetrics backupMetrics, + BackupNotificationMgr backupNotificationMgr) { super(configuration, backupMetrics, backupNotificationMgr); } public void setupTest(List files) { clearTest(); flist = new ArrayList(); - for (String file : files) - { + for (String file : files) { S3BackupPath path = pathProvider.get(); path.parseRemote(file); flist.add(path); @@ -61,24 +60,19 @@ public void setupTest(List files) { uploadedFiles = new HashSet(); } - public void setupTest() - { + public void setupTest() { clearTest(); flist = new ArrayList(); downloadedFiles = new HashSet(); uploadedFiles = new HashSet(); } - public void clearTest() - { - if (flist != null) - flist.clear(); - if (downloadedFiles != null) - downloadedFiles.clear(); + public void clearTest() { + if (flist != null) flist.clear(); + if (downloadedFiles != null) downloadedFiles.clear(); } - public void addFile(String file) - { + public void addFile(String file) { S3BackupPath path = pathProvider.get(); path.parseRemote(file); flist.add(path); @@ -88,29 +82,29 @@ public void addFile(String file) @Override public Iterator list(String bucket, Date start, Date till) { String[] paths = bucket.split(String.valueOf(S3BackupPath.PATH_SEP)); - - if( paths.length > 1){ + + if (paths.length > 1) { baseDir = paths[1]; region = paths[2]; clusterName = paths[3]; } - - List tmpList = new ArrayList(); - for (AbstractBackupPath path : flist) - { - if ((path.time.after(start) && path.time.before(till)) || path.time.equals(start) - && path.baseDir.equals(baseDir) && path.clusterName.equals(clusterName) && path.region.equals(region)) - { - tmpList.add(path); + List tmpList = new ArrayList(); + for (AbstractBackupPath path : flist) { + + if ((path.time.after(start) && path.time.before(till)) + || path.time.equals(start) + && path.baseDir.equals(baseDir) + && path.clusterName.equals(clusterName) + && path.region.equals(region)) { + tmpList.add(path); } } return tmpList.iterator(); } - public void shutdown() { - //nop + // nop } @Override @@ -119,28 +113,25 @@ public long getFileSize(Path remotePath) throws BackupRestoreException { } @Override - public Iterator listPrefixes(Date date) - { + public Iterator listPrefixes(Date date) { // TODO Auto-generated method stub return null; } @Override - public void cleanup() - { + public void cleanup() { // TODO Auto-generated method stub } @Override protected void downloadFileImpl(Path remotePath, Path localPath) throws BackupRestoreException { - //if (path.type == BackupFileType.META) + // if (path.type == BackupFileType.META) { // List all files and generate the file try (FileWriter fr = new FileWriter(localPath.toFile())) { JSONArray jsonObj = new JSONArray(); for (AbstractBackupPath filePath : flist) { - if (filePath.type == BackupFileType.SNAP) - jsonObj.add(filePath.getRemotePath()); + if (filePath.type == BackupFileType.SNAP) jsonObj.add(filePath.getRemotePath()); } fr.write(jsonObj.toJSONString()); fr.flush(); diff --git a/priam/src/test/java/com/netflix/priam/backup/FakeCredentials.java b/priam/src/test/java/com/netflix/priam/backup/FakeCredentials.java index f82c097a4..b73fe6e10 100644 --- a/priam/src/test/java/com/netflix/priam/backup/FakeCredentials.java +++ b/priam/src/test/java/com/netflix/priam/backup/FakeCredentials.java @@ -20,10 +20,9 @@ import com.amazonaws.auth.AWSCredentialsProvider; import com.netflix.priam.cred.ICredential; -public class FakeCredentials implements ICredential -{ - public AWSCredentialsProvider getAwsCredentialProvider() { - // TODO Auto-generated method stub - return null; - } +public class FakeCredentials implements ICredential { + public AWSCredentialsProvider getAwsCredentialProvider() { + // TODO Auto-generated method stub + return null; + } } diff --git a/priam/src/test/java/com/netflix/priam/backup/FakeNullCredential.java b/priam/src/test/java/com/netflix/priam/backup/FakeNullCredential.java index fa0419e2d..42baeb807 100644 --- a/priam/src/test/java/com/netflix/priam/backup/FakeNullCredential.java +++ b/priam/src/test/java/com/netflix/priam/backup/FakeNullCredential.java @@ -20,10 +20,9 @@ import com.amazonaws.auth.AWSCredentialsProvider; import com.netflix.priam.cred.ICredential; -public class FakeNullCredential implements ICredential -{ - public AWSCredentialsProvider getAwsCredentialProvider() { - // TODO Auto-generated method stub - return null; - } +public class FakeNullCredential implements ICredential { + public AWSCredentialsProvider getAwsCredentialProvider() { + // TODO Auto-generated method stub + return null; + } } diff --git a/priam/src/test/java/com/netflix/priam/backup/FakePostRestoreHook.java b/priam/src/test/java/com/netflix/priam/backup/FakePostRestoreHook.java index ef23915ba..94907a051 100644 --- a/priam/src/test/java/com/netflix/priam/backup/FakePostRestoreHook.java +++ b/priam/src/test/java/com/netflix/priam/backup/FakePostRestoreHook.java @@ -24,6 +24,6 @@ public boolean hasValidParameters() { } public void execute() { - //no op + // no op } } diff --git a/priam/src/test/java/com/netflix/priam/backup/NullBackupFileSystem.java b/priam/src/test/java/com/netflix/priam/backup/NullBackupFileSystem.java index 7ba439ec6..af74dab37 100644 --- a/priam/src/test/java/com/netflix/priam/backup/NullBackupFileSystem.java +++ b/priam/src/test/java/com/netflix/priam/backup/NullBackupFileSystem.java @@ -21,17 +21,17 @@ import com.netflix.priam.config.IConfiguration; import com.netflix.priam.merics.BackupMetrics; import com.netflix.priam.notification.BackupNotificationMgr; - import java.nio.file.Path; import java.util.Date; import java.util.Iterator; - public class NullBackupFileSystem extends AbstractFileSystem { @Inject - public NullBackupFileSystem(IConfiguration configuration, BackupMetrics backupMetrics, - BackupNotificationMgr backupNotificationMgr){ + public NullBackupFileSystem( + IConfiguration configuration, + BackupMetrics backupMetrics, + BackupNotificationMgr backupNotificationMgr) { super(configuration, backupMetrics, backupNotificationMgr); } @@ -40,9 +40,8 @@ public Iterator list(String bucket, Date start, Date till) { return null; } - public void shutdown() - { - //NOP + public void shutdown() { + // NOP } @Override @@ -51,24 +50,21 @@ public long getFileSize(Path remotePath) throws BackupRestoreException { } @Override - public Iterator listPrefixes(Date date) { return null; } @Override - public void cleanup() - { + public void cleanup() { // TODO Auto-generated method stub } @Override - protected void downloadFileImpl(Path remotePath, Path localPath) throws BackupRestoreException { - - } + protected void downloadFileImpl(Path remotePath, Path localPath) + throws BackupRestoreException {} @Override protected long uploadFileImpl(Path localPath, Path remotePath) throws BackupRestoreException { return 0; } -} \ No newline at end of file +} diff --git a/priam/src/test/java/com/netflix/priam/backup/TestAbstractFileSystem.java b/priam/src/test/java/com/netflix/priam/backup/TestAbstractFileSystem.java index f66403c7c..41d34485d 100644 --- a/priam/src/test/java/com/netflix/priam/backup/TestAbstractFileSystem.java +++ b/priam/src/test/java/com/netflix/priam/backup/TestAbstractFileSystem.java @@ -24,13 +24,6 @@ import com.netflix.priam.merics.BackupMetrics; import com.netflix.priam.notification.BackupNotificationMgr; import com.netflix.priam.utils.BackupFileUtils; -import org.apache.commons.io.FileUtils; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import java.io.File; import java.nio.file.Path; import java.nio.file.Paths; @@ -40,14 +33,21 @@ import java.util.List; import java.util.Random; import java.util.concurrent.*; +import org.apache.commons.io.FileUtils; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** - * The goal of this class is to test common functionality which are encapsulated in AbstractFileSystem. - * The actual upload/download of a file to remote file system is beyond the scope of this class. - * Created by aagrawal on 9/22/18. + * The goal of this class is to test common functionality which are encapsulated in + * AbstractFileSystem. The actual upload/download of a file to remote file system is beyond the + * scope of this class. Created by aagrawal on 9/22/18. */ public class TestAbstractFileSystem { - private static final Logger logger = LoggerFactory.getLogger(TestAbstractFileSystem.class.getName()); + private static final Logger logger = + LoggerFactory.getLogger(TestAbstractFileSystem.class.getName()); private Injector injector; private IConfiguration configuration; private BackupMetrics backupMetrics; @@ -57,11 +57,9 @@ public class TestAbstractFileSystem { @Before public void setBackupMetrics() { - if (injector == null) - injector = Guice.createInjector(new BRTestModule()); + if (injector == null) injector = Guice.createInjector(new BRTestModule()); - if (configuration == null) - configuration = injector.getInstance(IConfiguration.class); + if (configuration == null) configuration = injector.getInstance(IConfiguration.class); if (backupNotificationMgr == null) backupNotificationMgr = injector.getInstance(BackupNotificationMgr.class); @@ -69,7 +67,8 @@ public void setBackupMetrics() { backupMetrics = injector.getInstance(BackupMetrics.class); if (failureFileSystem == null) - failureFileSystem = new FailureFileSystem(configuration, backupMetrics, backupNotificationMgr); + failureFileSystem = + new FailureFileSystem(configuration, backupMetrics, backupNotificationMgr); if (myFileSystem == null) myFileSystem = new MyFileSystem(configuration, backupMetrics, backupNotificationMgr); @@ -77,16 +76,16 @@ public void setBackupMetrics() { BackupFileUtils.cleanupDir(Paths.get(configuration.getDataFileLocation())); } - @Test public void testFailedRetriesUpload() throws Exception { try { Collection files = generateFiles(1, 1, 1); for (File file : files) { - failureFileSystem.uploadFile(file.toPath(), file.toPath(), getDummyPath(file.toPath()), 2, true); + failureFileSystem.uploadFile( + file.toPath(), file.toPath(), getDummyPath(file.toPath()), 2, true); } } catch (BackupRestoreException e) { - //Verify the failure metric for upload is incremented. + // Verify the failure metric for upload is incremented. Assert.assertEquals(1, (int) backupMetrics.getInvalidUploads().count()); } } @@ -97,9 +96,11 @@ private AbstractBackupPath getDummyPath(Path localPath) throws ParseException { return path; } - private Collection generateFiles(int noOfKeyspaces, int noOfCf, int noOfSstables) throws Exception { + private Collection generateFiles(int noOfKeyspaces, int noOfCf, int noOfSstables) + throws Exception { Path dataDir = Paths.get(configuration.getDataFileLocation()); - BackupFileUtils.generateDummyFiles(dataDir, noOfKeyspaces, noOfCf, noOfSstables, "snapshot", "201812310000"); + BackupFileUtils.generateDummyFiles( + dataDir, noOfKeyspaces, noOfCf, noOfSstables, "snapshot", "201812310000"); String[] ext = {"db"}; return FileUtils.listFiles(dataDir.toFile(), ext, true); } @@ -109,7 +110,7 @@ public void testFailedRetriesDownload() { try { failureFileSystem.downloadFile(Paths.get(""), null, 2); } catch (BackupRestoreException e) { - //Verify the failure metric for download is incremented. + // Verify the failure metric for download is incremented. Assert.assertEquals(1, (int) backupMetrics.getInvalidDownloads().count()); } } @@ -117,13 +118,14 @@ public void testFailedRetriesDownload() { @Test public void testUpload() throws Exception { Collection files = generateFiles(1, 1, 1); - //Dummy upload with compressed size. + // Dummy upload with compressed size. for (File file : files) { - myFileSystem.uploadFile(file.toPath(), file.toPath(), getDummyPath(file.toPath()), 2, true); - //Verify the success metric for upload is incremented. + myFileSystem.uploadFile( + file.toPath(), file.toPath(), getDummyPath(file.toPath()), 2, true); + // Verify the success metric for upload is incremented. Assert.assertEquals(1, (int) backupMetrics.getValidUploads().actualCount()); - //Verify delete of the original file if flag provided. + // Verify delete of the original file if flag provided. Assert.assertFalse(file.exists()); break; } @@ -131,21 +133,24 @@ public void testUpload() throws Exception { @Test public void testDownload() throws Exception { - //Dummy download + // Dummy download myFileSystem.downloadFile(Paths.get(""), null, 2); - //Verify the success metric for download is incremented. + // Verify the success metric for download is incremented. Assert.assertEquals(1, (int) backupMetrics.getValidDownloads().actualCount()); } @Test public void testAsyncUpload() throws Exception { - //Testing single async upload. + // Testing single async upload. Collection files = generateFiles(1, 1, 1); for (File file : files) { - myFileSystem.asyncUploadFile(file.toPath(), file.toPath(), getDummyPath(file.toPath()), 2, true).get(); - //1. Verify the success metric for upload is incremented. + myFileSystem + .asyncUploadFile( + file.toPath(), file.toPath(), getDummyPath(file.toPath()), 2, true) + .get(); + // 1. Verify the success metric for upload is incremented. Assert.assertEquals(1, (int) backupMetrics.getValidUploads().actualCount()); - //2. The task queue is empty after upload is finished. + // 2. The task queue is empty after upload is finished. Assert.assertEquals(0, myFileSystem.getUploadTasksQueued()); break; } @@ -153,40 +158,44 @@ public void testAsyncUpload() throws Exception { @Test public void testAsyncUploadBulk() throws Exception { - //Testing the queue feature works. - //1. Give 1000 dummy files to upload. File upload takes some random time to upload + // Testing the queue feature works. + // 1. Give 1000 dummy files to upload. File upload takes some random time to upload Collection files = generateFiles(1, 1, 20); List> futures = new ArrayList<>(); for (File file : files) { - futures.add(myFileSystem.asyncUploadFile(file.toPath(), file.toPath(), getDummyPath(file.toPath()), 2, true)); + futures.add( + myFileSystem.asyncUploadFile( + file.toPath(), file.toPath(), getDummyPath(file.toPath()), 2, true)); } - //Verify all the work is finished. + // Verify all the work is finished. for (Future future : futures) { future.get(); } - //2. Success metric is incremented correctly + // 2. Success metric is incremented correctly Assert.assertEquals(files.size(), (int) backupMetrics.getValidUploads().actualCount()); - //3. The task queue is empty after upload is finished. + // 3. The task queue is empty after upload is finished. Assert.assertEquals(0, myFileSystem.getUploadTasksQueued()); } @Test public void testUploadDedup() throws Exception { - //Testing the de-duping works. + // Testing the de-duping works. Collection files = generateFiles(1, 1, 1); File file = files.iterator().next(); AbstractBackupPath abstractBackupPath = getDummyPath(file.toPath()); - //1. Give same file to upload x times. Only one request will be entertained. + // 1. Give same file to upload x times. Only one request will be entertained. int size = 10; ExecutorService threads = Executors.newFixedThreadPool(size); List> torun = new ArrayList<>(size); for (int i = 0; i < size; i++) { - torun.add(() -> { - myFileSystem.uploadFile(file.toPath(), file.toPath(), abstractBackupPath, 2, true); - return Boolean.TRUE; - }); + torun.add( + () -> { + myFileSystem.uploadFile( + file.toPath(), file.toPath(), abstractBackupPath, 2, true); + return Boolean.TRUE; + }); } // all tasks executed in different threads, at 'once'. @@ -197,29 +206,32 @@ public void testUploadDedup() throws Exception { for (Future future : futures) { try { future.get(); - }catch (InterruptedException | ExecutionException e){ - //Do nothing. + } catch (InterruptedException | ExecutionException e) { + // Do nothing. } } - //2. Verify the success metric for upload is not same as size, i.e. some amount of de-duping happened. + // 2. Verify the success metric for upload is not same as size, i.e. some amount of + // de-duping happened. Assert.assertNotEquals(size, (int) backupMetrics.getValidUploads().actualCount()); } @Test public void testAsyncUploadFailure() throws Exception { - //Testing single async upload. + // Testing single async upload. Collection files = generateFiles(1, 1, 1); for (File file : files) { - Future future = failureFileSystem.asyncUploadFile(file.toPath(), file.toPath(), getDummyPath(file.toPath()), 2, true); + Future future = + failureFileSystem.asyncUploadFile( + file.toPath(), file.toPath(), getDummyPath(file.toPath()), 2, true); try { future.get(); } catch (Exception e) { - //1. Future get returns error message. + // 1. Future get returns error message. - //2. Verify the failure metric for upload is incremented. + // 2. Verify the failure metric for upload is incremented. Assert.assertEquals(1, (int) backupMetrics.getInvalidUploads().count()); - //3. The task queue is empty after upload is finished. + // 3. The task queue is empty after upload is finished. Assert.assertEquals(0, failureFileSystem.getUploadTasksQueued()); break; } @@ -228,33 +240,33 @@ public void testAsyncUploadFailure() throws Exception { @Test public void testAsyncDownload() throws Exception { - //Testing single async download. + // Testing single async download. Future future = myFileSystem.asyncDownloadFile(Paths.get(""), null, 2); future.get(); - //1. Verify the success metric for download is incremented. + // 1. Verify the success metric for download is incremented. Assert.assertEquals(1, (int) backupMetrics.getValidDownloads().actualCount()); - //2. Verify the queue size is '0' after success. + // 2. Verify the queue size is '0' after success. Assert.assertEquals(0, myFileSystem.getDownloadTasksQueued()); } @Test public void testAsyncDownloadBulk() throws Exception { - //Testing the queue feature works. - //1. Give 1000 dummy files to download. File download takes some random time to download. + // Testing the queue feature works. + // 1. Give 1000 dummy files to download. File download takes some random time to download. int totalFiles = 1000; List> futureList = new ArrayList<>(); for (int i = 0; i < totalFiles; i++) futureList.add(myFileSystem.asyncDownloadFile(Paths.get("" + i), null, 2)); - //Ensure processing is finished. + // Ensure processing is finished. for (Future future1 : futureList) { future1.get(); } - //2. Success metric is incremented correctly -> exactly 1000 times. + // 2. Success metric is incremented correctly -> exactly 1000 times. Assert.assertEquals(totalFiles, (int) backupMetrics.getValidDownloads().actualCount()); - //3. The task queue is empty after download is finished. + // 3. The task queue is empty after download is finished. Assert.assertEquals(0, myFileSystem.getDownloadTasksQueued()); } @@ -264,7 +276,7 @@ public void testAsyncDownloadFailure() throws Exception { try { future.get(); } catch (Exception e) { - //Verify the failure metric for upload is incremented. + // Verify the failure metric for upload is incremented. Assert.assertEquals(1, (int) backupMetrics.getInvalidDownloads().count()); } } @@ -272,18 +284,27 @@ public void testAsyncDownloadFailure() throws Exception { class FailureFileSystem extends NullBackupFileSystem { @Inject - public FailureFileSystem(IConfiguration configuration, BackupMetrics backupMetrics, BackupNotificationMgr backupNotificationMgr) { + public FailureFileSystem( + IConfiguration configuration, + BackupMetrics backupMetrics, + BackupNotificationMgr backupNotificationMgr) { super(configuration, backupMetrics, backupNotificationMgr); } @Override - protected void downloadFileImpl(Path remotePath, Path localPath) throws BackupRestoreException { - throw new BackupRestoreException("User injected failure file system error for testing download. Remote path: " + remotePath); + protected void downloadFileImpl(Path remotePath, Path localPath) + throws BackupRestoreException { + throw new BackupRestoreException( + "User injected failure file system error for testing download. Remote path: " + + remotePath); } @Override - protected long uploadFileImpl(Path localPath, Path remotePath) throws BackupRestoreException { - throw new BackupRestoreException("User injected failure file system error for testing upload. Local path: " + localPath); + protected long uploadFileImpl(Path localPath, Path remotePath) + throws BackupRestoreException { + throw new BackupRestoreException( + "User injected failure file system error for testing upload. Local path: " + + localPath); } } @@ -292,12 +313,16 @@ class MyFileSystem extends NullBackupFileSystem { private Random random = new Random(); @Inject - public MyFileSystem(IConfiguration configuration, BackupMetrics backupMetrics, BackupNotificationMgr backupNotificationMgr) { + public MyFileSystem( + IConfiguration configuration, + BackupMetrics backupMetrics, + BackupNotificationMgr backupNotificationMgr) { super(configuration, backupMetrics, backupNotificationMgr); } @Override - protected void downloadFileImpl(Path remotePath, Path localPath) throws BackupRestoreException { + protected void downloadFileImpl(Path remotePath, Path localPath) + throws BackupRestoreException { try { Thread.sleep(random.nextInt(20)); } catch (InterruptedException e) { @@ -306,7 +331,8 @@ protected void downloadFileImpl(Path remotePath, Path localPath) throws BackupRe } @Override - protected long uploadFileImpl(Path localPath, Path remotePath) throws BackupRestoreException { + protected long uploadFileImpl(Path localPath, Path remotePath) + throws BackupRestoreException { try { Thread.sleep(random.nextInt(20)); } catch (InterruptedException e) { diff --git a/priam/src/test/java/com/netflix/priam/backup/TestBackup.java b/priam/src/test/java/com/netflix/priam/backup/TestBackup.java index e5e65a90a..746c7421f 100644 --- a/priam/src/test/java/com/netflix/priam/backup/TestBackup.java +++ b/priam/src/test/java/com/netflix/priam/backup/TestBackup.java @@ -17,83 +17,77 @@ package com.netflix.priam.backup; +import com.google.inject.Guice; +import com.google.inject.Injector; +import com.google.inject.Key; +import com.google.inject.name.Names; import java.io.BufferedOutputStream; import java.io.File; import java.io.FileOutputStream; import java.io.IOException; import java.util.HashSet; import java.util.Set; - import mockit.Mock; import mockit.MockUp; - import org.apache.cassandra.tools.NodeProbe; import org.apache.commons.io.FileUtils; import org.junit.*; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.google.inject.Guice; -import com.google.inject.Injector; -import com.google.inject.Key; -import com.google.inject.name.Names; - /** * Unit test case to test a snapshot backup and incremental backup - * + * * @author Praveen Sadhu - * */ -public class TestBackup -{ +public class TestBackup { private static Injector injector; private static FakeBackupFileSystem filesystem; private static final Logger logger = LoggerFactory.getLogger(TestBackup.class); private static Set expectedFiles = new HashSet(); @BeforeClass - public static void setup() throws InterruptedException, IOException - { - new MockNodeProbe(); + public static void setup() throws InterruptedException, IOException { + new MockNodeProbe(); injector = Guice.createInjector(new BRTestModule()); - filesystem = (FakeBackupFileSystem) injector.getInstance(Key.get(IBackupFileSystem.class,Names.named("backup"))); + filesystem = + (FakeBackupFileSystem) + injector.getInstance( + Key.get(IBackupFileSystem.class, Names.named("backup"))); } - + @AfterClass - public static void cleanup() throws IOException - { + public static void cleanup() throws IOException { File file = new File("target/data"); FileUtils.deleteQuietly(file); } @Test - public void testSnapshotBackup() throws Exception - { - filesystem.setupTest(); + public void testSnapshotBackup() throws Exception { + filesystem.setupTest(); SnapshotBackup backup = injector.getInstance(SnapshotBackup.class); -// -// backup.execute(); -// Assert.assertEquals(3, filesystem.uploadedFiles.size()); -// System.out.println("***** "+filesystem.uploadedFiles.size()); -// boolean metafile = false; -// for (String filePath : expectedFiles) -// Assert.assertTrue(filesystem.uploadedFiles.contains(filePath)); -// -// for(String filepath : filesystem.uploadedFiles){ -// if( filepath.endsWith("meta.json")){ -// metafile = true; -// break; -// } -// } -// Assert.assertTrue(metafile); + // + // backup.execute(); + // Assert.assertEquals(3, filesystem.uploadedFiles.size()); + // System.out.println("***** "+filesystem.uploadedFiles.size()); + // boolean metafile = false; + // for (String filePath : expectedFiles) + // Assert.assertTrue(filesystem.uploadedFiles.contains(filePath)); + // + // for(String filepath : filesystem.uploadedFiles){ + // if( filepath.endsWith("meta.json")){ + // metafile = true; + // break; + // } + // } + // Assert.assertTrue(metafile); } @Test - public void testIncrementalBackup() throws Exception - { - filesystem.setupTest(); + public void testIncrementalBackup() throws Exception { + filesystem.setupTest(); generateIncrementalFiles(); IncrementalBackup backup = injector.getInstance(IncrementalBackup.class); backup.execute(); @@ -103,43 +97,47 @@ public void testIncrementalBackup() throws Exception } @Test - public void testClusterSpecificColumnFamiliesSkippedBefore21() throws Exception - { - String[] columnFamilyDirs = {"schema_columns","local", "peers", "LocationInfo"}; + public void testClusterSpecificColumnFamiliesSkippedBefore21() throws Exception { + String[] columnFamilyDirs = {"schema_columns", "local", "peers", "LocationInfo"}; testClusterSpecificColumnFamiliesSkipped(columnFamilyDirs); } @Test - public void testClusterSpecificColumnFamiliesSkippedFrom21() throws Exception - { - String[] columnFamilyDirs = {"schema_columns-296e9c049bec30c5828dc17d3df2132a", - "local-7ad54392bcdd45d684174c047860b347", - "peers-37c71aca7ac2383ba74672528af04d4f", - "LocationInfo-9f5c6374d48633299a0a5094bf9ad1e4"}; + public void testClusterSpecificColumnFamiliesSkippedFrom21() throws Exception { + String[] columnFamilyDirs = { + "schema_columns-296e9c049bec30c5828dc17d3df2132a", + "local-7ad54392bcdd45d684174c047860b347", + "peers-37c71aca7ac2383ba74672528af04d4f", + "LocationInfo-9f5c6374d48633299a0a5094bf9ad1e4" + }; testClusterSpecificColumnFamiliesSkipped(columnFamilyDirs); } - private void testClusterSpecificColumnFamiliesSkipped(String[] columnFamilyDirs) throws Exception - { + private void testClusterSpecificColumnFamiliesSkipped(String[] columnFamilyDirs) + throws Exception { filesystem.setupTest(); File tmp = new File("target/data/"); - if (tmp.exists()) - cleanup(tmp); + if (tmp.exists()) cleanup(tmp); // Generate "data" generateIncrementalFiles(); Set systemfiles = new HashSet(); // Generate system files - for (String columnFamilyDir: columnFamilyDirs) { + for (String columnFamilyDir : columnFamilyDirs) { String columnFamily = columnFamilyDir.split("-")[0]; - systemfiles.add(String.format("target/data/system/%s/backups/system-%s-ka-1-Data.db", columnFamilyDir, columnFamily)); - systemfiles.add(String.format("target/data/system/%s/backups/system-%s-ka-1-Index.db", columnFamilyDir, columnFamily)); + systemfiles.add( + String.format( + "target/data/system/%s/backups/system-%s-ka-1-Data.db", + columnFamilyDir, columnFamily)); + systemfiles.add( + String.format( + "target/data/system/%s/backups/system-%s-ka-1-Index.db", + columnFamilyDir, columnFamily)); } - for (String systemFilePath: systemfiles) - { + for (String systemFilePath : systemfiles) { File file = new File(systemFilePath); genTestFile(file); - //Not cluster specific columns should be backed up - if(systemFilePath.contains("schema_columns")) + // Not cluster specific columns should be backed up + if (systemFilePath.contains("schema_columns")) expectedFiles.add(file.getAbsolutePath()); } IncrementalBackup backup = injector.getInstance(IncrementalBackup.class); @@ -149,11 +147,9 @@ private void testClusterSpecificColumnFamiliesSkipped(String[] columnFamilyDirs) Assert.assertTrue(filesystem.uploadedFiles.contains(filePath)); } - private static void generateIncrementalFiles() - { + private static void generateIncrementalFiles() { File tmp = new File("target/data/"); - if (tmp.exists()) - cleanup(tmp); + if (tmp.exists()) cleanup(tmp); // Setup Set files = new HashSet(); files.add("target/data/Keyspace1/Standard1/backups/Keyspace1-Standard1-ia-1-Data.db"); @@ -162,79 +158,69 @@ private static void generateIncrementalFiles() files.add("target/data/Keyspace1/Standard1/backups/Keyspace1-Standard1-ia-3-Data.db"); expectedFiles.clear(); - for (String filePath : files) - { + for (String filePath : files) { File file = new File(filePath); genTestFile(file); expectedFiles.add(file.getAbsolutePath()); } } - private static void genTestFile(File file) - { - try - { + private static void genTestFile(File file) { + try { File parent = file.getParentFile(); - if (!parent.exists()) - parent.mkdirs(); + if (!parent.exists()) parent.mkdirs(); BufferedOutputStream bos1 = new BufferedOutputStream(new FileOutputStream(file)); - for (long i = 0; i < (5L * 1024); i++) - bos1.write((byte) 8); + for (long i = 0; i < (5L * 1024); i++) bos1.write((byte) 8); bos1.flush(); bos1.close(); - } - catch (Exception e) - { + } catch (Exception e) { logger.error(e.getMessage()); } } - private static void cleanup(File dir) - { + private static void cleanup(File dir) { FileUtils.deleteQuietly(dir); } // Mock Nodeprobe class @Ignore - static class MockNodeProbe extends MockUp - { + static class MockNodeProbe extends MockUp { @Mock - public void $init(String host, int port) throws IOException, InterruptedException - { - } + public void $init(String host, int port) throws IOException, InterruptedException {} @Mock - public void takeSnapshot(String snapshotName, String columnFamily, String... keyspaces) throws IOException - { + public void takeSnapshot(String snapshotName, String columnFamily, String... keyspaces) + throws IOException { File tmp = new File("target/data/"); - if (tmp.exists()) - cleanup(tmp); + if (tmp.exists()) cleanup(tmp); // Setup Set files = new HashSet(); - files.add("target/data/Keyspace1/Standard1/snapshots/" + snapshotName + "/Keyspace1-Standard1-ia-5-Data.db"); - files.add("target/data/Keyspace1/Standard1/snapshots/201101081230/Keyspace1-Standard1-ia-6-Data.db"); - files.add("target/data/Keyspace1/Standard1/snapshots/" + snapshotName + "/Keyspace1-Standard1-ia-7-Data.db"); + files.add( + "target/data/Keyspace1/Standard1/snapshots/" + + snapshotName + + "/Keyspace1-Standard1-ia-5-Data.db"); + files.add( + "target/data/Keyspace1/Standard1/snapshots/201101081230/Keyspace1-Standard1-ia-6-Data.db"); + files.add( + "target/data/Keyspace1/Standard1/snapshots/" + + snapshotName + + "/Keyspace1-Standard1-ia-7-Data.db"); expectedFiles.clear(); - for (String filePath : files) - { + for (String filePath : files) { File file = new File(filePath); genTestFile(file); - if (filePath.indexOf("Keyspace1-Standard1-ia-6-Data.db") == -1)// skip - expectedFiles.add(file.getAbsolutePath()); + if (filePath.indexOf("Keyspace1-Standard1-ia-6-Data.db") == -1) // skip + expectedFiles.add(file.getAbsolutePath()); } } @Mock - public void close() throws IOException - { - } + public void close() throws IOException {} @Mock - public void clearSnapshot(String tag, String... keyspaces) throws IOException - { + public void clearSnapshot(String tag, String... keyspaces) throws IOException { cleanup(new File("target/data")); } } - -} \ No newline at end of file +} diff --git a/priam/src/test/java/com/netflix/priam/backup/TestBackupFile.java b/priam/src/test/java/com/netflix/priam/backup/TestBackupFile.java index 2fc5f1213..6c66e259c 100644 --- a/priam/src/test/java/com/netflix/priam/backup/TestBackupFile.java +++ b/priam/src/test/java/com/netflix/priam/backup/TestBackupFile.java @@ -17,67 +17,60 @@ package com.netflix.priam.backup; +import com.google.inject.Guice; +import com.google.inject.Injector; +import com.netflix.priam.aws.S3BackupPath; +import com.netflix.priam.backup.AbstractBackupPath.BackupFileType; +import com.netflix.priam.config.FakeConfiguration; +import com.netflix.priam.identity.InstanceIdentity; import java.io.BufferedOutputStream; import java.io.File; import java.io.FileOutputStream; import java.io.IOException; import java.sql.Date; import java.text.ParseException; - import org.apache.commons.io.FileUtils; import org.junit.AfterClass; +import org.junit.Assert; import org.junit.BeforeClass; import org.junit.Test; -import org.junit.Assert; - -import com.google.inject.Guice; -import com.google.inject.Injector; -import com.netflix.priam.config.FakeConfiguration; -import com.netflix.priam.aws.S3BackupPath; -import com.netflix.priam.backup.AbstractBackupPath.BackupFileType; -import com.netflix.priam.identity.InstanceIdentity; -public class TestBackupFile -{ +public class TestBackupFile { private static Injector injector; @BeforeClass - public static void setup() throws IOException - { + public static void setup() throws IOException { injector = Guice.createInjector(new BRTestModule()); - File file = new File("target/data/Keyspace1/Standard1/", "Keyspace1-Standard1-ia-5-Data.db"); - if (!file.exists()) - { + File file = + new File("target/data/Keyspace1/Standard1/", "Keyspace1-Standard1-ia-5-Data.db"); + if (!file.exists()) { File dir1 = new File("target/data/Keyspace1/Standard1/"); - if (!dir1.exists()) - dir1.mkdirs(); + if (!dir1.exists()) dir1.mkdirs(); byte b = 8; long oneKB = (1024L); System.out.println(oneKB); BufferedOutputStream bos1 = new BufferedOutputStream(new FileOutputStream(file)); - for (long i = 0; i < oneKB; i++) - { + for (long i = 0; i < oneKB; i++) { bos1.write(b); } bos1.flush(); bos1.close(); } InstanceIdentity factory = injector.getInstance(InstanceIdentity.class); - factory.getInstance().setToken("1234567");//Token + factory.getInstance().setToken("1234567"); // Token } @AfterClass - public static void cleanup() throws IOException - { + public static void cleanup() throws IOException { File file = new File("Keyspace1-Standard1-ia-5-Data.db"); FileUtils.deleteQuietly(file); } @Test - public void testBackupFileCreation() throws ParseException - { + public void testBackupFileCreation() throws ParseException { // Test snapshot file - String snapshotfile = "target/data/Keyspace1/Standard1/snapshots/201108082320/Keyspace1-Standard1-ia-5-Data.db"; + String snapshotfile = + "target/data/Keyspace1/Standard1/snapshots/201108082320/Keyspace1-Standard1-ia-5-Data.db"; S3BackupPath backupfile = injector.getInstance(S3BackupPath.class); backupfile.parseLocal(new File(snapshotfile), BackupFileType.SNAP); Assert.assertEquals(BackupFileType.SNAP, backupfile.type); @@ -87,13 +80,16 @@ public void testBackupFileCreation() throws ParseException Assert.assertEquals("fake-app", backupfile.clusterName); Assert.assertEquals(FakeConfiguration.FAKE_REGION, backupfile.region); Assert.assertEquals("casstestbackup", backupfile.baseDir); - Assert.assertEquals("casstestbackup/"+FakeConfiguration.FAKE_REGION+"/fake-app/1234567/201108082320/SNAP/Keyspace1/Standard1/Keyspace1-Standard1-ia-5-Data.db", backupfile.getRemotePath()); + Assert.assertEquals( + "casstestbackup/" + + FakeConfiguration.FAKE_REGION + + "/fake-app/1234567/201108082320/SNAP/Keyspace1/Standard1/Keyspace1-Standard1-ia-5-Data.db", + backupfile.getRemotePath()); } @Test - public void testIncBackupFileCreation() throws ParseException - { - // Test incremental file + public void testIncBackupFileCreation() throws ParseException { + // Test incremental file File bfile = new File("target/data/Keyspace1/Standard1/Keyspace1-Standard1-ia-5-Data.db"); S3BackupPath backupfile = injector.getInstance(S3BackupPath.class); backupfile.parseLocal(bfile, BackupFileType.SST); @@ -105,12 +101,17 @@ public void testIncBackupFileCreation() throws ParseException Assert.assertEquals(FakeConfiguration.FAKE_REGION, backupfile.region); Assert.assertEquals("casstestbackup", backupfile.baseDir); String datestr = AbstractBackupPath.formatDate(new Date(bfile.lastModified())); - Assert.assertEquals("casstestbackup/"+FakeConfiguration.FAKE_REGION+"/fake-app/1234567/" + datestr + "/SST/Keyspace1/Standard1/Keyspace1-Standard1-ia-5-Data.db", backupfile.getRemotePath()); + Assert.assertEquals( + "casstestbackup/" + + FakeConfiguration.FAKE_REGION + + "/fake-app/1234567/" + + datestr + + "/SST/Keyspace1/Standard1/Keyspace1-Standard1-ia-5-Data.db", + backupfile.getRemotePath()); } @Test - public void testMetaFileCreation() throws ParseException - { + public void testMetaFileCreation() throws ParseException { // Test snapshot file String filestr = "cass/data/1234567.meta"; File bfile = new File(filestr); @@ -122,6 +123,10 @@ public void testMetaFileCreation() throws ParseException Assert.assertEquals("fake-app", backupfile.clusterName); Assert.assertEquals(FakeConfiguration.FAKE_REGION, backupfile.region); Assert.assertEquals("casstestbackup", backupfile.baseDir); - Assert.assertEquals("casstestbackup/"+FakeConfiguration.FAKE_REGION+"/fake-app/1234567/201108082320/META/1234567.meta", backupfile.getRemotePath()); + Assert.assertEquals( + "casstestbackup/" + + FakeConfiguration.FAKE_REGION + + "/fake-app/1234567/201108082320/META/1234567.meta", + backupfile.getRemotePath()); } } diff --git a/priam/src/test/java/com/netflix/priam/backup/TestCompression.java b/priam/src/test/java/com/netflix/priam/backup/TestCompression.java index a899d45a6..429d3dea6 100644 --- a/priam/src/test/java/com/netflix/priam/backup/TestCompression.java +++ b/priam/src/test/java/com/netflix/priam/backup/TestCompression.java @@ -17,8 +17,17 @@ package com.netflix.priam.backup; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + import com.netflix.priam.compress.SnappyCompression; import com.netflix.priam.utils.SystemUtils; +import java.io.*; +import java.util.Enumeration; +import java.util.Iterator; +import java.util.zip.ZipEntry; +import java.util.zip.ZipFile; +import java.util.zip.ZipOutputStream; import org.apache.commons.io.IOUtils; import org.junit.After; import org.junit.Before; @@ -27,52 +36,42 @@ import org.xerial.snappy.SnappyInputStream; import org.xerial.snappy.SnappyOutputStream; -import java.io.*; -import java.util.Enumeration; -import java.util.Iterator; -import java.util.zip.ZipEntry; -import java.util.zip.ZipFile; -import java.util.zip.ZipOutputStream; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; - @Ignore("does this test really have to generate smoke to verify correct behavior?") -public class TestCompression -{ +public class TestCompression { @Before - public void setup() throws IOException - { + public void setup() throws IOException { File f = new File("/tmp/compress-test.txt"); - try(FileOutputStream stream = new FileOutputStream(f)) { + try (FileOutputStream stream = new FileOutputStream(f)) { for (int i = 0; i < (1000 * 1000); i++) { - stream.write("This is a test... Random things happen... and you are responsible for it...\n".getBytes("UTF-8")); - stream.write("The quick brown fox jumps over the lazy dog.The quick brown fox jumps over the lazy dog.The quick brown fox jumps over the lazy dog.\n".getBytes("UTF-8")); + stream.write( + "This is a test... Random things happen... and you are responsible for it...\n" + .getBytes("UTF-8")); + stream.write( + "The quick brown fox jumps over the lazy dog.The quick brown fox jumps over the lazy dog.The quick brown fox jumps over the lazy dog.\n" + .getBytes("UTF-8")); } } } @After - public void done() - { + public void done() { File f = new File("/tmp/compress-test.txt"); - if (f.exists()) - f.delete(); + if (f.exists()) f.delete(); } - void validateCompression(String uncompress, String compress) - { + void validateCompression(String uncompress, String compress) { File uncompressed = new File(uncompress); File compressed = new File(compress); assertTrue(uncompressed.length() > compressed.length()); } @Test - public void zip() throws IOException - { + public void zip() throws IOException { BufferedInputStream source = null; - try(ZipOutputStream out = new ZipOutputStream(new BufferedOutputStream(new FileOutputStream("/tmp/compressed.zip")))) { + try (ZipOutputStream out = + new ZipOutputStream( + new BufferedOutputStream(new FileOutputStream("/tmp/compressed.zip")))) { byte data[] = new byte[2048]; File file = new File("/tmp/compress-test.txt"); FileInputStream fi = new FileInputStream(file); @@ -91,12 +90,12 @@ public void zip() throws IOException public void unzip() throws IOException { ZipFile zipfile = new ZipFile("/tmp/compressed.zip"); Enumeration e = zipfile.entries(); - while (e.hasMoreElements()) - { + while (e.hasMoreElements()) { ZipEntry entry = (ZipEntry) e.nextElement(); - try(BufferedInputStream is = new BufferedInputStream(zipfile.getInputStream(entry)); - BufferedOutputStream dest1 = new BufferedOutputStream(new FileOutputStream("/tmp/compress-test-out-0.txt"), 2048)) { - ; + try (BufferedInputStream is = new BufferedInputStream(zipfile.getInputStream(entry)); + BufferedOutputStream dest1 = + new BufferedOutputStream( + new FileOutputStream("/tmp/compress-test-out-0.txt"), 2048)) {; int c; byte d[] = new byte[2048]; @@ -111,15 +110,15 @@ public void unzip() throws IOException { } @Test - public void snappyCompress() throws IOException - { + public void snappyCompress() throws IOException { FileInputStream fi = new FileInputStream("/tmp/compress-test.txt"); - SnappyOutputStream out = new SnappyOutputStream(new BufferedOutputStream(new FileOutputStream("/tmp/test0.snp"))); + SnappyOutputStream out = + new SnappyOutputStream( + new BufferedOutputStream(new FileOutputStream("/tmp/test0.snp"))); BufferedInputStream origin = new BufferedInputStream(fi, 1024); byte data[] = new byte[1024]; int count; - while ((count = origin.read(data, 0, 1024)) != -1) - { + while ((count = origin.read(data, 0, 1024)) != -1) { out.write(data, 0, count); } IOUtils.closeQuietly(origin); @@ -130,16 +129,16 @@ public void snappyCompress() throws IOException } @Test - public void snappyDecompress() throws IOException - { + public void snappyDecompress() throws IOException { // decompress normally. - SnappyInputStream is = new SnappyInputStream(new BufferedInputStream(new FileInputStream("/tmp/test0.snp"))); + SnappyInputStream is = + new SnappyInputStream( + new BufferedInputStream(new FileInputStream("/tmp/test0.snp"))); byte d[] = new byte[1024]; FileOutputStream fos = new FileOutputStream("/tmp/compress-test-out-1.txt"); BufferedOutputStream dest1 = new BufferedOutputStream(fos, 1024); int c; - while ((c = is.read(d, 0, 1024)) != -1) - { + while ((c = is.read(d, 0, 1024)) != -1) { dest1.write(d, 0, c); } IOUtils.closeQuietly(dest1); @@ -151,15 +150,16 @@ public void snappyDecompress() throws IOException } @Test - public void compress() throws IOException - { + public void compress() throws IOException { SnappyCompression compress = new SnappyCompression(); File file = new File(new File("/tmp/compress-test.txt"), "r"); - long chunkSize = 5L*1024*1024; - Iterator it = compress.compress(new AbstractBackupPath.RafInputStream(new RandomAccessFile(file, "r")), chunkSize); + long chunkSize = 5L * 1024 * 1024; + Iterator it = + compress.compress( + new AbstractBackupPath.RafInputStream(new RandomAccessFile(file, "r")), + chunkSize); FileOutputStream ostream = new FileOutputStream("/tmp/test1.snp"); - while (it.hasNext()) - { + while (it.hasNext()) { byte[] chunk = it.next(); ostream.write(chunk); } @@ -168,10 +168,11 @@ public void compress() throws IOException } @Test - public void decompress() throws IOException - { + public void decompress() throws IOException { SnappyCompression compress = new SnappyCompression(); - compress.decompressAndClose(new FileInputStream("/tmp/test1.snp"), new FileOutputStream("/tmp/compress-test-out-2.txt")); + compress.decompressAndClose( + new FileInputStream("/tmp/test1.snp"), + new FileOutputStream("/tmp/compress-test-out-2.txt")); String md1 = SystemUtils.md5(new File("/tmp/compress-test.txt")); String md2 = SystemUtils.md5(new File("/tmp/compress-test-out-2.txt")); assertEquals(md1, md2); diff --git a/priam/src/test/java/com/netflix/priam/backup/TestCustomizedTPE.java b/priam/src/test/java/com/netflix/priam/backup/TestCustomizedTPE.java index b84cc1beb..2bdce5483 100644 --- a/priam/src/test/java/com/netflix/priam/backup/TestCustomizedTPE.java +++ b/priam/src/test/java/com/netflix/priam/backup/TestCustomizedTPE.java @@ -17,71 +17,60 @@ package com.netflix.priam.backup; -import org.junit.Assert; +import com.netflix.priam.scheduler.BlockingSubmitThreadPoolExecutor; import java.util.concurrent.Callable; import java.util.concurrent.LinkedBlockingDeque; import java.util.concurrent.atomic.AtomicInteger; - +import org.junit.Assert; import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.netflix.priam.scheduler.BlockingSubmitThreadPoolExecutor; - -public class TestCustomizedTPE -{ +public class TestCustomizedTPE { private static final Logger logger = LoggerFactory.getLogger(TestCustomizedTPE.class); private static final int MAX_THREADS = 10; // timeout 1 sec private static final int TIME_OUT = 10 * 1000; - private BlockingSubmitThreadPoolExecutor startTest = new BlockingSubmitThreadPoolExecutor(MAX_THREADS, new LinkedBlockingDeque(MAX_THREADS), TIME_OUT); + private BlockingSubmitThreadPoolExecutor startTest = + new BlockingSubmitThreadPoolExecutor( + MAX_THREADS, new LinkedBlockingDeque(MAX_THREADS), TIME_OUT); @Test - public void testExecutor() throws InterruptedException - { + public void testExecutor() throws InterruptedException { final AtomicInteger count = new AtomicInteger(); - for (int i = 0; i < 100; i++) - { - startTest.submit(new Callable() - { - @Override - public Void call() throws Exception - { - Thread.sleep(100); - logger.info("Count:{}", count.incrementAndGet()); - return null; - } - }); + for (int i = 0; i < 100; i++) { + startTest.submit( + new Callable() { + @Override + public Void call() throws Exception { + Thread.sleep(100); + logger.info("Count:{}", count.incrementAndGet()); + return null; + } + }); } startTest.sleepTillEmpty(); Assert.assertEquals(100, count.get()); } @Test - public void testException() - { + public void testException() { boolean success = false; - try - { - for (int i = 0; i < 100; i++) - { - startTest.submit(new Callable() - { - @Override - public Void call() throws Exception - { - logger.info("Sleeping for 2 * timeout."); - Thread.sleep(TIME_OUT * 2); - return null; - } - }); + try { + for (int i = 0; i < 100; i++) { + startTest.submit( + new Callable() { + @Override + public Void call() throws Exception { + logger.info("Sleeping for 2 * timeout."); + Thread.sleep(TIME_OUT * 2); + return null; + } + }); } - } - catch (RuntimeException ex) - { + } catch (RuntimeException ex) { success = true; } Assert.assertTrue("Failure to timeout...", success); } - } diff --git a/priam/src/test/java/com/netflix/priam/backup/TestFileIterator.java b/priam/src/test/java/com/netflix/priam/backup/TestFileIterator.java index 7d4212dc7..701174824 100644 --- a/priam/src/test/java/com/netflix/priam/backup/TestFileIterator.java +++ b/priam/src/test/java/com/netflix/priam/backup/TestFileIterator.java @@ -24,10 +24,12 @@ import com.amazonaws.services.s3.model.S3ObjectSummary; import com.google.inject.Guice; import com.google.inject.Injector; +import com.netflix.priam.aws.S3FileIterator; import com.netflix.priam.config.FakeConfiguration; import com.netflix.priam.config.IConfiguration; -import com.netflix.priam.aws.S3FileIterator; import com.netflix.priam.identity.InstanceIdentity; +import java.io.IOException; +import java.util.*; import mockit.Mock; import mockit.MockUp; import org.junit.Assert; @@ -35,30 +37,25 @@ import org.junit.Ignore; import org.junit.Test; -import java.io.IOException; -import java.util.*; - /** * Unit test for backup file iterator - * - * @author Praveen Sadhu * + * @author Praveen Sadhu */ -public class TestFileIterator -{ +public class TestFileIterator { private static Injector injector; private static Date startTime, endTime; private static Calendar cal; private static AmazonS3Client s3client; - + private static IConfiguration conf; private static InstanceIdentity factory; + @BeforeClass - public static void setup() throws InterruptedException, IOException - { - s3client = new MockAmazonS3Client().getMockInstance(); - new MockObjectListing(); + public static void setup() throws InterruptedException, IOException { + s3client = new MockAmazonS3Client().getMockInstance(); + new MockObjectListing(); injector = Guice.createInjector(new BRTestModule()); conf = injector.getInstance(IConfiguration.class); @@ -74,21 +71,22 @@ public static void setup() throws InterruptedException, IOException // MockAmazonS3Client class @Ignore - public static class MockAmazonS3Client extends MockUp - { + public static class MockAmazonS3Client extends MockUp { public static String bucketName = ""; public static String prefix = ""; - + @Mock - public ObjectListing listObjects(ListObjectsRequest listObjectsRequest) throws AmazonClientException { + public ObjectListing listObjects(ListObjectsRequest listObjectsRequest) + throws AmazonClientException { ObjectListing listing = new ObjectListing(); listing.setBucketName(listObjectsRequest.getBucketName()); listing.setPrefix(listObjectsRequest.getPrefix()); return listing; } - @Mock - public ObjectListing listNextBatchOfObjects(ObjectListing previousObjectListing) throws AmazonClientException { + @Mock + public ObjectListing listNextBatchOfObjects(ObjectListing previousObjectListing) + throws AmazonClientException { ObjectListing listing = new ObjectListing(); listing.setBucketName(previousObjectListing.getBucketName()); listing.setPrefix(previousObjectListing.getPrefix()); @@ -98,198 +96,353 @@ public ObjectListing listNextBatchOfObjects(ObjectListing previousObjectListing) // MockObjectListing class @Ignore - public static class MockObjectListing extends MockUp - { + public static class MockObjectListing extends MockUp { public static boolean truncated = true; public static boolean firstcall = true; - public static boolean simfilter = false;//Simulate filtering + public static boolean simfilter = false; // Simulate filtering @Mock - public List getObjectSummaries() - { - if (firstcall) - { + public List getObjectSummaries() { + if (firstcall) { firstcall = false; - if( simfilter ) - return getObjectSummaryEmpty(); + if (simfilter) return getObjectSummaryEmpty(); return getObjectSummary(); - } - else - { - if( simfilter ){ - simfilter = false;//reset + } else { + if (simfilter) { + simfilter = false; // reset return getObjectSummaryEmpty(); - } - else - truncated = false; + } else truncated = false; return getNextObjectSummary(); } } @Mock - public boolean isTruncated() - { + public boolean isTruncated() { return truncated; } } @Test - public void testIteratorEmptySet() - { + public void testIteratorEmptySet() { cal.set(2011, 7, 11, 6, 1, 0); cal.set(Calendar.MILLISECOND, 0); Date stime = cal.getTime(); cal.add(Calendar.HOUR, 5); Date etime = cal.getTime(); MockAmazonS3Client.bucketName = "TESTBUCKET"; - MockAmazonS3Client.prefix = conf.getBackupLocation() + "/" + conf.getDC() + "/" + conf.getAppName() + "/" + factory.getInstance().getToken(); + MockAmazonS3Client.prefix = + conf.getBackupLocation() + + "/" + + conf.getDC() + + "/" + + conf.getAppName() + + "/" + + factory.getInstance().getToken(); MockAmazonS3Client.prefix += "/20110811"; - - S3FileIterator fileIterator = new S3FileIterator(injector.getProvider(AbstractBackupPath.class), s3client, "TESTBUCKET", stime, etime); + + S3FileIterator fileIterator = + new S3FileIterator( + injector.getProvider(AbstractBackupPath.class), + s3client, + "TESTBUCKET", + stime, + etime); Set files = new HashSet(); - while (fileIterator.hasNext()) - files.add(fileIterator.next().getRemotePath()); + while (fileIterator.hasNext()) files.add(fileIterator.next().getRemotePath()); Assert.assertEquals(0, files.size()); } @Test - public void testIterator() - { + public void testIterator() { MockObjectListing.truncated = false; MockObjectListing.firstcall = true; MockObjectListing.simfilter = false; MockAmazonS3Client.bucketName = "TESTBUCKET"; - MockAmazonS3Client.prefix = conf.getBackupLocation() + "/" + conf.getDC() + "/" + conf.getAppName() + "/" + factory.getInstance().getToken(); + MockAmazonS3Client.prefix = + conf.getBackupLocation() + + "/" + + conf.getDC() + + "/" + + conf.getAppName() + + "/" + + factory.getInstance().getToken(); MockAmazonS3Client.prefix += "/20110811"; - - S3FileIterator fileIterator = new S3FileIterator(injector.getProvider(AbstractBackupPath.class), s3client, "TESTBUCKET", startTime, endTime); + + S3FileIterator fileIterator = + new S3FileIterator( + injector.getProvider(AbstractBackupPath.class), + s3client, + "TESTBUCKET", + startTime, + endTime); Set files = new HashSet(); - while (fileIterator.hasNext()) - files.add(fileIterator.next().getRemotePath()); + while (fileIterator.hasNext()) files.add(fileIterator.next().getRemotePath()); Assert.assertEquals(3, files.size()); - Assert.assertTrue(files.contains("test_backup/"+FakeConfiguration.FAKE_REGION+"/fakecluster/123456/201108110030/SNAP/ks1/cf1/f1.db")); - Assert.assertTrue(files.contains("test_backup/"+FakeConfiguration.FAKE_REGION+"/fakecluster/123456/201108110430/SST/ks1/cf1/f2.db")); - Assert.assertTrue(files.contains("test_backup/"+FakeConfiguration.FAKE_REGION+"/fakecluster/123456/201108110030/META/meta.json")); - Assert.assertFalse(files.contains("test_backup/"+FakeConfiguration.FAKE_REGION+"/fakecluster/123456/201108110600/SST/ks1/cf1/f3.db")); + Assert.assertTrue( + files.contains( + "test_backup/" + + FakeConfiguration.FAKE_REGION + + "/fakecluster/123456/201108110030/SNAP/ks1/cf1/f1.db")); + Assert.assertTrue( + files.contains( + "test_backup/" + + FakeConfiguration.FAKE_REGION + + "/fakecluster/123456/201108110430/SST/ks1/cf1/f2.db")); + Assert.assertTrue( + files.contains( + "test_backup/" + + FakeConfiguration.FAKE_REGION + + "/fakecluster/123456/201108110030/META/meta.json")); + Assert.assertFalse( + files.contains( + "test_backup/" + + FakeConfiguration.FAKE_REGION + + "/fakecluster/123456/201108110600/SST/ks1/cf1/f3.db")); } @Test - public void testIteratorTruncated() - { + public void testIteratorTruncated() { MockObjectListing.truncated = true; MockObjectListing.firstcall = true; MockObjectListing.simfilter = false; MockAmazonS3Client.bucketName = "TESTBUCKET"; - MockAmazonS3Client.prefix = conf.getBackupLocation() + "/" + conf.getDC() + "/" + conf.getAppName() + "/" + factory.getInstance().getToken(); + MockAmazonS3Client.prefix = + conf.getBackupLocation() + + "/" + + conf.getDC() + + "/" + + conf.getAppName() + + "/" + + factory.getInstance().getToken(); MockAmazonS3Client.prefix += "/20110811"; - S3FileIterator fileIterator = new S3FileIterator(injector.getProvider(AbstractBackupPath.class), s3client, "TESTBUCKET", startTime, endTime); + S3FileIterator fileIterator = + new S3FileIterator( + injector.getProvider(AbstractBackupPath.class), + s3client, + "TESTBUCKET", + startTime, + endTime); Set files = new HashSet(); - while (fileIterator.hasNext()) - files.add(fileIterator.next().getRemotePath()); + while (fileIterator.hasNext()) files.add(fileIterator.next().getRemotePath()); Assert.assertEquals(5, files.size()); - Assert.assertTrue(files.contains("test_backup/"+FakeConfiguration.FAKE_REGION+"/fakecluster/123456/201108110030/SNAP/ks1/cf1/f1.db")); - Assert.assertTrue(files.contains("test_backup/"+FakeConfiguration.FAKE_REGION+"/fakecluster/123456/201108110430/SST/ks1/cf1/f2.db")); - Assert.assertTrue(files.contains("test_backup/"+FakeConfiguration.FAKE_REGION+"/fakecluster/123456/201108110030/META/meta.json")); - Assert.assertFalse(files.contains("test_backup/"+FakeConfiguration.FAKE_REGION+"/fakecluster/123456/201108110600/SST/ks1/cf1/f3.db")); - - Assert.assertTrue(files.contains("test_backup/"+FakeConfiguration.FAKE_REGION+"/fakecluster/123456/201108110030/SNAP/ks2/cf1/f1.db")); - Assert.assertTrue(files.contains("test_backup/"+FakeConfiguration.FAKE_REGION+"/fakecluster/123456/201108110430/SST/ks2/cf1/f2.db")); - Assert.assertFalse(files.contains("test_backup/"+FakeConfiguration.FAKE_REGION+"/fakecluster/123456/201108110600/SST/ks2/cf1/f3.db")); + Assert.assertTrue( + files.contains( + "test_backup/" + + FakeConfiguration.FAKE_REGION + + "/fakecluster/123456/201108110030/SNAP/ks1/cf1/f1.db")); + Assert.assertTrue( + files.contains( + "test_backup/" + + FakeConfiguration.FAKE_REGION + + "/fakecluster/123456/201108110430/SST/ks1/cf1/f2.db")); + Assert.assertTrue( + files.contains( + "test_backup/" + + FakeConfiguration.FAKE_REGION + + "/fakecluster/123456/201108110030/META/meta.json")); + Assert.assertFalse( + files.contains( + "test_backup/" + + FakeConfiguration.FAKE_REGION + + "/fakecluster/123456/201108110600/SST/ks1/cf1/f3.db")); + Assert.assertTrue( + files.contains( + "test_backup/" + + FakeConfiguration.FAKE_REGION + + "/fakecluster/123456/201108110030/SNAP/ks2/cf1/f1.db")); + Assert.assertTrue( + files.contains( + "test_backup/" + + FakeConfiguration.FAKE_REGION + + "/fakecluster/123456/201108110430/SST/ks2/cf1/f2.db")); + Assert.assertFalse( + files.contains( + "test_backup/" + + FakeConfiguration.FAKE_REGION + + "/fakecluster/123456/201108110600/SST/ks2/cf1/f3.db")); } @Test - public void testIteratorTruncatedOOR() - { + public void testIteratorTruncatedOOR() { MockObjectListing.truncated = true; MockObjectListing.firstcall = true; MockObjectListing.simfilter = true; MockAmazonS3Client.bucketName = "TESTBUCKET"; - MockAmazonS3Client.prefix = conf.getBackupLocation() + "/" + conf.getDC() + "/" + conf.getAppName() + "/" + factory.getInstance().getToken(); + MockAmazonS3Client.prefix = + conf.getBackupLocation() + + "/" + + conf.getDC() + + "/" + + conf.getAppName() + + "/" + + factory.getInstance().getToken(); MockAmazonS3Client.prefix += "/20110811"; - S3FileIterator fileIterator = new S3FileIterator(injector.getProvider(AbstractBackupPath.class), s3client, "TESTBUCKET", startTime, endTime); + S3FileIterator fileIterator = + new S3FileIterator( + injector.getProvider(AbstractBackupPath.class), + s3client, + "TESTBUCKET", + startTime, + endTime); Set files = new HashSet(); - while (fileIterator.hasNext()) - files.add(fileIterator.next().getRemotePath()); + while (fileIterator.hasNext()) files.add(fileIterator.next().getRemotePath()); Assert.assertEquals(2, files.size()); - Assert.assertFalse(files.contains("test_backup/"+FakeConfiguration.FAKE_REGION+"/fakecluster/123456/201107110030/SNAP/ks1/cf1/f1.db")); - Assert.assertFalse(files.contains("test_backup/"+FakeConfiguration.FAKE_REGION+"/fakecluster/123456/201107110430/SST/ks1/cf1/f2.db")); - Assert.assertFalse(files.contains("test_backup/"+FakeConfiguration.FAKE_REGION+"/fakecluster/123456/201107110030/META/meta.json")); - Assert.assertFalse(files.contains("test_backup/"+FakeConfiguration.FAKE_REGION+"/fakecluster/123456/201107110600/SST/ks1/cf1/f3.db")); + Assert.assertFalse( + files.contains( + "test_backup/" + + FakeConfiguration.FAKE_REGION + + "/fakecluster/123456/201107110030/SNAP/ks1/cf1/f1.db")); + Assert.assertFalse( + files.contains( + "test_backup/" + + FakeConfiguration.FAKE_REGION + + "/fakecluster/123456/201107110430/SST/ks1/cf1/f2.db")); + Assert.assertFalse( + files.contains( + "test_backup/" + + FakeConfiguration.FAKE_REGION + + "/fakecluster/123456/201107110030/META/meta.json")); + Assert.assertFalse( + files.contains( + "test_backup/" + + FakeConfiguration.FAKE_REGION + + "/fakecluster/123456/201107110600/SST/ks1/cf1/f3.db")); - Assert.assertTrue(files.contains("test_backup/"+FakeConfiguration.FAKE_REGION+"/fakecluster/123456/201108110030/SNAP/ks2/cf1/f1.db")); - Assert.assertTrue(files.contains("test_backup/"+FakeConfiguration.FAKE_REGION+"/fakecluster/123456/201108110430/SST/ks2/cf1/f2.db")); - Assert.assertFalse(files.contains("test_backup/"+FakeConfiguration.FAKE_REGION+"/fakecluster/123456/201108110600/SST/ks2/cf1/f3.db")); - + Assert.assertTrue( + files.contains( + "test_backup/" + + FakeConfiguration.FAKE_REGION + + "/fakecluster/123456/201108110030/SNAP/ks2/cf1/f1.db")); + Assert.assertTrue( + files.contains( + "test_backup/" + + FakeConfiguration.FAKE_REGION + + "/fakecluster/123456/201108110430/SST/ks2/cf1/f2.db")); + Assert.assertFalse( + files.contains( + "test_backup/" + + FakeConfiguration.FAKE_REGION + + "/fakecluster/123456/201108110600/SST/ks2/cf1/f3.db")); } @Test - public void testRestorePathIteration() - { + public void testRestorePathIteration() { MockObjectListing.truncated = true; MockObjectListing.firstcall = true; MockObjectListing.simfilter = false; MockAmazonS3Client.bucketName = "RESTOREBUCKET"; - MockAmazonS3Client.prefix = "test_restore_backup/fake-restore-region/fakerestorecluster" + "/" + factory.getInstance().getToken(); + MockAmazonS3Client.prefix = + "test_restore_backup/fake-restore-region/fakerestorecluster" + + "/" + + factory.getInstance().getToken(); MockAmazonS3Client.prefix += "/20110811"; - S3FileIterator fileIterator = new S3FileIterator(injector.getProvider(AbstractBackupPath.class), s3client, "RESTOREBUCKET/test_restore_backup/fake-restore-region/fakerestorecluster", startTime, endTime); + S3FileIterator fileIterator = + new S3FileIterator( + injector.getProvider(AbstractBackupPath.class), + s3client, + "RESTOREBUCKET/test_restore_backup/fake-restore-region/fakerestorecluster", + startTime, + endTime); Set files = new HashSet(); - while (fileIterator.hasNext()) - files.add(fileIterator.next().getRemotePath()); - while (fileIterator.hasNext()) - files.add(fileIterator.next().getRemotePath()); + while (fileIterator.hasNext()) files.add(fileIterator.next().getRemotePath()); + while (fileIterator.hasNext()) files.add(fileIterator.next().getRemotePath()); Assert.assertEquals(5, files.size()); - Assert.assertTrue(files.contains("test_backup/"+FakeConfiguration.FAKE_REGION+"/fakecluster/123456/201108110030/SNAP/ks1/cf1/f1.db")); - Assert.assertTrue(files.contains("test_backup/"+FakeConfiguration.FAKE_REGION+"/fakecluster/123456/201108110430/SST/ks1/cf1/f2.db")); - Assert.assertTrue(files.contains("test_backup/"+FakeConfiguration.FAKE_REGION+"/fakecluster/123456/201108110030/META/meta.json")); - Assert.assertFalse(files.contains("test_backup/"+FakeConfiguration.FAKE_REGION+"/fakecluster/123456/201108110600/SST/ks1/cf1/f3.db")); - - Assert.assertTrue(files.contains("test_backup/"+FakeConfiguration.FAKE_REGION+"/fakecluster/123456/201108110030/SNAP/ks2/cf1/f1.db")); - Assert.assertTrue(files.contains("test_backup/"+FakeConfiguration.FAKE_REGION+"/fakecluster/123456/201108110430/SST/ks2/cf1/f2.db")); - Assert.assertFalse(files.contains("test_backup/"+FakeConfiguration.FAKE_REGION+"/fakecluster/123456/201108110600/SST/ks2/cf1/f3.db")); + Assert.assertTrue( + files.contains( + "test_backup/" + + FakeConfiguration.FAKE_REGION + + "/fakecluster/123456/201108110030/SNAP/ks1/cf1/f1.db")); + Assert.assertTrue( + files.contains( + "test_backup/" + + FakeConfiguration.FAKE_REGION + + "/fakecluster/123456/201108110430/SST/ks1/cf1/f2.db")); + Assert.assertTrue( + files.contains( + "test_backup/" + + FakeConfiguration.FAKE_REGION + + "/fakecluster/123456/201108110030/META/meta.json")); + Assert.assertFalse( + files.contains( + "test_backup/" + + FakeConfiguration.FAKE_REGION + + "/fakecluster/123456/201108110600/SST/ks1/cf1/f3.db")); + Assert.assertTrue( + files.contains( + "test_backup/" + + FakeConfiguration.FAKE_REGION + + "/fakecluster/123456/201108110030/SNAP/ks2/cf1/f1.db")); + Assert.assertTrue( + files.contains( + "test_backup/" + + FakeConfiguration.FAKE_REGION + + "/fakecluster/123456/201108110430/SST/ks2/cf1/f2.db")); + Assert.assertFalse( + files.contains( + "test_backup/" + + FakeConfiguration.FAKE_REGION + + "/fakecluster/123456/201108110600/SST/ks2/cf1/f3.db")); } - public static List getObjectSummary() - { + public static List getObjectSummary() { List list = new ArrayList(); S3ObjectSummary summary = new S3ObjectSummary(); - summary.setKey("test_backup/"+FakeConfiguration.FAKE_REGION+"/fakecluster/123456/201108110030/SNAP/ks1/cf1/f1.db"); + summary.setKey( + "test_backup/" + + FakeConfiguration.FAKE_REGION + + "/fakecluster/123456/201108110030/SNAP/ks1/cf1/f1.db"); list.add(summary); summary = new S3ObjectSummary(); - summary.setKey("test_backup/"+FakeConfiguration.FAKE_REGION+"/fakecluster/123456/201108110430/SST/ks1/cf1/f2.db"); + summary.setKey( + "test_backup/" + + FakeConfiguration.FAKE_REGION + + "/fakecluster/123456/201108110430/SST/ks1/cf1/f2.db"); list.add(summary); summary = new S3ObjectSummary(); - summary.setKey("test_backup/"+FakeConfiguration.FAKE_REGION+"/fakecluster/123456/201108110600/SST/ks1/cf1/f3.db"); + summary.setKey( + "test_backup/" + + FakeConfiguration.FAKE_REGION + + "/fakecluster/123456/201108110600/SST/ks1/cf1/f3.db"); list.add(summary); summary = new S3ObjectSummary(); - summary.setKey("test_backup/"+FakeConfiguration.FAKE_REGION+"/fakecluster/123456/201108110030/META/meta.json"); + summary.setKey( + "test_backup/" + + FakeConfiguration.FAKE_REGION + + "/fakecluster/123456/201108110030/META/meta.json"); list.add(summary); return list; } - public static List getObjectSummaryEmpty() - { + public static List getObjectSummaryEmpty() { return new ArrayList(); } - public static List getNextObjectSummary() - { + public static List getNextObjectSummary() { List list = new ArrayList(); S3ObjectSummary summary = new S3ObjectSummary(); - summary.setKey("test_backup/"+FakeConfiguration.FAKE_REGION+"/fakecluster/123456/201108110030/SNAP/ks2/cf1/f1.db"); + summary.setKey( + "test_backup/" + + FakeConfiguration.FAKE_REGION + + "/fakecluster/123456/201108110030/SNAP/ks2/cf1/f1.db"); list.add(summary); summary = new S3ObjectSummary(); - summary.setKey("test_backup/"+FakeConfiguration.FAKE_REGION+"/fakecluster/123456/201108110430/SST/ks2/cf1/f2.db"); + summary.setKey( + "test_backup/" + + FakeConfiguration.FAKE_REGION + + "/fakecluster/123456/201108110430/SST/ks2/cf1/f2.db"); list.add(summary); summary = new S3ObjectSummary(); - summary.setKey("test_backup/"+FakeConfiguration.FAKE_REGION+"/fakecluster/123456/201108110600/SST/ks2/cf1/f3.db"); + summary.setKey( + "test_backup/" + + FakeConfiguration.FAKE_REGION + + "/fakecluster/123456/201108110600/SST/ks2/cf1/f3.db"); list.add(summary); return list; } - -} \ No newline at end of file +} diff --git a/priam/src/test/java/com/netflix/priam/backup/TestRestore.java b/priam/src/test/java/com/netflix/priam/backup/TestRestore.java index 4caf5a17f..83f56984f 100644 --- a/priam/src/test/java/com/netflix/priam/backup/TestRestore.java +++ b/priam/src/test/java/com/netflix/priam/backup/TestRestore.java @@ -24,31 +24,31 @@ import com.netflix.priam.config.FakeConfiguration; import com.netflix.priam.config.IConfiguration; import com.netflix.priam.restore.Restore; -import org.apache.commons.io.FileUtils; -import org.junit.AfterClass; -import org.junit.Assert; -import org.junit.BeforeClass; -import org.junit.Test; - import java.io.File; import java.io.IOException; import java.util.ArrayList; import java.util.Calendar; import java.util.Date; +import org.apache.commons.io.FileUtils; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; -public class TestRestore -{ +public class TestRestore { private static Injector injector; private static FakeBackupFileSystem filesystem; private static ArrayList fileList; - private static Calendar cal; + private static Calendar cal; private static IConfiguration conf; @BeforeClass - public static void setup() throws InterruptedException, IOException - { + public static void setup() throws InterruptedException, IOException { injector = Guice.createInjector(new BRTestModule()); - filesystem = (FakeBackupFileSystem) injector.getInstance(Key.get(IBackupFileSystem.class, Names.named("backup"))); + filesystem = + (FakeBackupFileSystem) + injector.getInstance( + Key.get(IBackupFileSystem.class, Names.named("backup"))); conf = injector.getInstance(IConfiguration.class); fileList = new ArrayList(); File cassdir = new File(conf.getDataFileLocation()); @@ -57,20 +57,43 @@ public static void setup() throws InterruptedException, IOException } @AfterClass - public static void cleanup() throws IOException - { + public static void cleanup() throws IOException { File file = new File("cass"); FileUtils.deleteQuietly(file); } - private static void populateBackupFileSystem(String baseDir){ + private static void populateBackupFileSystem(String baseDir) { fileList.clear(); - fileList.add(baseDir + "/"+FakeConfiguration.FAKE_REGION+"/fakecluster/123456/201108110030/META/meta.json"); - fileList.add(baseDir + "/"+FakeConfiguration.FAKE_REGION+"/fakecluster/123456/201108110030/SNAP/ks1/cf1/f1.db"); - fileList.add(baseDir + "/"+FakeConfiguration.FAKE_REGION+"/fakecluster/123456/201108110030/SNAP/ks1/cf1/f2.db"); - fileList.add(baseDir + "/"+FakeConfiguration.FAKE_REGION+"/fakecluster/123456/201108110030/SNAP/ks2/cf1/f2.db"); - fileList.add(baseDir + "/"+FakeConfiguration.FAKE_REGION+"/fakecluster/123456/201108110530/SST/ks2/cf1/f3.db"); - fileList.add(baseDir + "/"+FakeConfiguration.FAKE_REGION+"/fakecluster/123456/201108110600/SST/ks2/cf1/f4.db"); + fileList.add( + baseDir + + "/" + + FakeConfiguration.FAKE_REGION + + "/fakecluster/123456/201108110030/META/meta.json"); + fileList.add( + baseDir + + "/" + + FakeConfiguration.FAKE_REGION + + "/fakecluster/123456/201108110030/SNAP/ks1/cf1/f1.db"); + fileList.add( + baseDir + + "/" + + FakeConfiguration.FAKE_REGION + + "/fakecluster/123456/201108110030/SNAP/ks1/cf1/f2.db"); + fileList.add( + baseDir + + "/" + + FakeConfiguration.FAKE_REGION + + "/fakecluster/123456/201108110030/SNAP/ks2/cf1/f2.db"); + fileList.add( + baseDir + + "/" + + FakeConfiguration.FAKE_REGION + + "/fakecluster/123456/201108110530/SST/ks2/cf1/f3.db"); + fileList.add( + baseDir + + "/" + + FakeConfiguration.FAKE_REGION + + "/fakecluster/123456/201108110600/SST/ks2/cf1/f4.db"); filesystem.baseDir = baseDir; filesystem.region = FakeConfiguration.FAKE_REGION; filesystem.clusterName = "fakecluster"; @@ -78,8 +101,7 @@ private static void populateBackupFileSystem(String baseDir){ } @Test - public void testRestore() throws Exception - { + public void testRestore() throws Exception { populateBackupFileSystem("test_backup"); File tmpdir = new File(conf.getDataFileLocation() + "/test"); tmpdir.mkdir(); @@ -100,12 +122,14 @@ public void testRestore() throws Exception Assert.assertFalse(tmpdir.exists()); } - //Pick latest file + // Pick latest file @Test - public void testRestoreLatest() throws Exception - { + public void testRestoreLatest() throws Exception { populateBackupFileSystem("test_backup"); - String metafile = "test_backup/"+FakeConfiguration.FAKE_REGION+"/fakecluster/123456/201108110130/META/meta.json"; + String metafile = + "test_backup/" + + FakeConfiguration.FAKE_REGION + + "/fakecluster/123456/201108110130/META/meta.json"; filesystem.addFile(metafile); Restore restore = injector.getInstance(Restore.class); cal.set(2011, Calendar.AUGUST, 11, 0, 30, 0); @@ -123,8 +147,7 @@ public void testRestoreLatest() throws Exception } @Test - public void testNoSnapshots() throws Exception - { + public void testNoSnapshots() throws Exception { try { filesystem.setupTest(new ArrayList()); Restore restore = injector.getInstance(Restore.class); @@ -132,22 +155,20 @@ public void testNoSnapshots() throws Exception Date startTime = cal.getTime(); cal.add(Calendar.HOUR, 5); restore.restore(startTime, cal.getTime()); - Assert.assertFalse(true);//No exception thrown + Assert.assertFalse(true); // No exception thrown } catch (IllegalStateException e) { - //We are ok. No snapshot found. + // We are ok. No snapshot found. } catch (Exception e) { throw e; } - } - - + @Test - public void testRestoreFromDiffCluster() throws Exception - { + public void testRestoreFromDiffCluster() throws Exception { populateBackupFileSystem("test_backup_new"); - FakeConfiguration conf = (FakeConfiguration)injector.getInstance(IConfiguration.class); - conf.setRestorePrefix("RESTOREBUCKET/test_backup_new/"+FakeConfiguration.FAKE_REGION+"/fakecluster"); + FakeConfiguration conf = (FakeConfiguration) injector.getInstance(IConfiguration.class); + conf.setRestorePrefix( + "RESTOREBUCKET/test_backup_new/" + FakeConfiguration.FAKE_REGION + "/fakecluster"); Restore restore = injector.getInstance(Restore.class); cal.set(2011, Calendar.AUGUST, 11, 0, 30, 0); cal.set(Calendar.MILLISECOND, 0); diff --git a/priam/src/test/java/com/netflix/priam/backup/TestS3FileSystem.java b/priam/src/test/java/com/netflix/priam/backup/TestS3FileSystem.java index ae3fa327a..5bae592da 100644 --- a/priam/src/test/java/com/netflix/priam/backup/TestS3FileSystem.java +++ b/priam/src/test/java/com/netflix/priam/backup/TestS3FileSystem.java @@ -33,6 +33,12 @@ import com.netflix.priam.backup.AbstractBackupPath.BackupFileType; import com.netflix.priam.config.FakeConfiguration; import com.netflix.priam.merics.BackupMetrics; +import java.io.BufferedOutputStream; +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.nio.file.Paths; +import java.util.List; import mockit.Mock; import mockit.MockUp; import org.junit.AfterClass; @@ -42,25 +48,17 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.BufferedOutputStream; -import java.io.File; -import java.io.FileOutputStream; -import java.io.IOException; -import java.nio.file.Paths; -import java.util.List; - public class TestS3FileSystem { private static Injector injector; private static final Logger logger = LoggerFactory.getLogger(TestS3FileSystem.class); - private static String FILE_PATH = "target/data/Keyspace1/Standard1/backups/201108082320/Keyspace1-Standard1-ia-1-Data.db"; + private static String FILE_PATH = + "target/data/Keyspace1/Standard1/backups/201108082320/Keyspace1-Standard1-ia-1-Data.db"; private static BackupMetrics backupMetrics; public TestS3FileSystem() { - if (injector == null) - injector = Guice.createInjector(new BRTestModule()); + if (injector == null) injector = Guice.createInjector(new BRTestModule()); - if (backupMetrics == null) - backupMetrics = injector.getInstance(BackupMetrics.class); + if (backupMetrics == null) backupMetrics = injector.getInstance(BackupMetrics.class); } @BeforeClass @@ -69,8 +67,7 @@ public static void setup() throws InterruptedException, IOException { new MockAmazonS3Client(); File dir1 = new File("target/data/Keyspace1/Standard1/backups/201108082320"); - if (!dir1.exists()) - dir1.mkdirs(); + if (!dir1.exists()) dir1.mkdirs(); File file = new File(FILE_PATH); long fiveKB = (5L * 1024); byte b = 8; @@ -94,7 +91,12 @@ public void testFileUpload() throws Exception { S3BackupPath backupfile = injector.getInstance(S3BackupPath.class); backupfile.parseLocal(new File(FILE_PATH), BackupFileType.SNAP); long noOfFilesUploaded = backupMetrics.getUploadRate().count(); - fs.uploadFile(Paths.get(backupfile.getBackupFile().getAbsolutePath()), Paths.get(backupfile.getRemotePath()), backupfile, 0, false); + fs.uploadFile( + Paths.get(backupfile.getBackupFile().getAbsolutePath()), + Paths.get(backupfile.getRemotePath()), + backupfile, + 0, + false); Assert.assertEquals(1, backupMetrics.getUploadRate().count() - noOfFilesUploaded); } @@ -104,15 +106,22 @@ public void testFileUploadFailures() throws Exception { MockS3PartUploader.partFailure = true; long noOfFailures = backupMetrics.getInvalidUploads().count(); S3FileSystem fs = injector.getInstance(S3FileSystem.class); - String snapshotfile = "target/data/Keyspace1/Standard1/backups/201108082320/Keyspace1-Standard1-ia-1-Data.db"; + String snapshotfile = + "target/data/Keyspace1/Standard1/backups/201108082320/Keyspace1-Standard1-ia-1-Data.db"; S3BackupPath backupfile = injector.getInstance(S3BackupPath.class); backupfile.parseLocal(new File(snapshotfile), BackupFileType.SNAP); try { - fs.uploadFile(Paths.get(backupfile.getBackupFile().getAbsolutePath()), Paths.get(backupfile.getRemotePath()), backupfile, 0, false); + fs.uploadFile( + Paths.get(backupfile.getBackupFile().getAbsolutePath()), + Paths.get(backupfile.getRemotePath()), + backupfile, + 0, + false); } catch (BackupRestoreException e) { // ignore } - //Assert.assertEquals(RetryableCallable.DEFAULT_NUMBER_OF_RETRIES, MockS3PartUploader.partAttempts); + // Assert.assertEquals(RetryableCallable.DEFAULT_NUMBER_OF_RETRIES, + // MockS3PartUploader.partAttempts); Assert.assertEquals(0, MockS3PartUploader.compattempts); Assert.assertEquals(1, backupMetrics.getInvalidUploads().count() - noOfFailures); } @@ -122,17 +131,23 @@ public void testFileUploadCompleteFailure() throws Exception { MockS3PartUploader.setup(); MockS3PartUploader.completionFailure = true; S3FileSystem fs = injector.getInstance(S3FileSystem.class); - String snapshotfile = "target/data/Keyspace1/Standard1/backups/201108082320/Keyspace1-Standard1-ia-1-Data.db"; + String snapshotfile = + "target/data/Keyspace1/Standard1/backups/201108082320/Keyspace1-Standard1-ia-1-Data.db"; S3BackupPath backupfile = injector.getInstance(S3BackupPath.class); backupfile.parseLocal(new File(snapshotfile), BackupFileType.SNAP); try { - fs.uploadFile(Paths.get(backupfile.getBackupFile().getAbsolutePath()), Paths.get(backupfile.getRemotePath()), backupfile, 0, false); + fs.uploadFile( + Paths.get(backupfile.getBackupFile().getAbsolutePath()), + Paths.get(backupfile.getRemotePath()), + backupfile, + 0, + false); } catch (BackupRestoreException e) { // ignore } - //Assert.assertEquals(1, MockS3PartUploader.partAttempts); + // Assert.assertEquals(1, MockS3PartUploader.partAttempts); // No retries with the new logic - //Assert.assertEquals(1, MockS3PartUploader.compattempts); + // Assert.assertEquals(1, MockS3PartUploader.compattempts); } @Test @@ -143,7 +158,8 @@ public void testCleanupAdd() throws Exception { Assert.assertEquals(1, MockAmazonS3Client.bconf.getRules().size()); BucketLifecycleConfiguration.Rule rule = MockAmazonS3Client.bconf.getRules().get(0); logger.info(rule.getPrefix()); - Assert.assertEquals("casstestbackup/" + FakeConfiguration.FAKE_REGION + "/fake-app/", rule.getPrefix()); + Assert.assertEquals( + "casstestbackup/" + FakeConfiguration.FAKE_REGION + "/fake-app/", rule.getPrefix()); Assert.assertEquals(5, rule.getExpirationInDays()); } @@ -155,11 +171,11 @@ public void testCleanupIgnore() throws Exception { Assert.assertEquals(1, MockAmazonS3Client.bconf.getRules().size()); BucketLifecycleConfiguration.Rule rule = MockAmazonS3Client.bconf.getRules().get(0); logger.info(rule.getPrefix()); - Assert.assertEquals("casstestbackup/" + FakeConfiguration.FAKE_REGION + "/fake-app/", rule.getPrefix()); + Assert.assertEquals( + "casstestbackup/" + FakeConfiguration.FAKE_REGION + "/fake-app/", rule.getPrefix()); Assert.assertEquals(5, rule.getExpirationInDays()); } - // Mock Nodeprobe class static class MockS3PartUploader extends MockUp { static int compattempts = 0; @@ -176,8 +192,7 @@ static class MockS3PartUploader extends MockUp { @Mock private Void uploadPart() throws AmazonClientException, BackupRestoreException { ++partAttempts; - if (partFailure) - throw new BackupRestoreException("Test exception"); + if (partFailure) throw new BackupRestoreException("Test exception"); partETags.add(new PartETag(0, null)); return null; } @@ -185,15 +200,13 @@ private Void uploadPart() throws AmazonClientException, BackupRestoreException { @Mock public CompleteMultipartUploadResult completeUpload() throws BackupRestoreException { ++compattempts; - if (completionFailure) - throw new BackupRestoreException("Test exception"); + if (completionFailure) throw new BackupRestoreException("Test exception"); return null; } @Mock - public void abortUpload() { - } + public void abortUpload() {} @Mock public Void retriableCall() throws AmazonClientException, BackupRestoreException { @@ -214,15 +227,17 @@ static class MockAmazonS3Client extends MockUp { static BucketLifecycleConfiguration bconf = new BucketLifecycleConfiguration(); @Mock - public void $init() { - } + public void $init() {} @Mock - public InitiateMultipartUploadResult initiateMultipartUpload(InitiateMultipartUploadRequest initiateMultipartUploadRequest) throws AmazonClientException { + public InitiateMultipartUploadResult initiateMultipartUpload( + InitiateMultipartUploadRequest initiateMultipartUploadRequest) + throws AmazonClientException { return new InitiateMultipartUploadResult(); } - public PutObjectResult putObject(PutObjectRequest putObjectRequest) throws SdkClientException, AmazonServiceException { + public PutObjectResult putObject(PutObjectRequest putObjectRequest) + throws SdkClientException, AmazonServiceException { PutObjectResult result = new PutObjectResult(); result.setETag("ad"); return result; @@ -232,21 +247,24 @@ public PutObjectResult putObject(PutObjectRequest putObjectRequest) throws SdkCl public BucketLifecycleConfiguration getBucketLifecycleConfiguration(String bucketName) { List rules = Lists.newArrayList(); if (ruleAvailable) { - String clusterPath = "casstestbackup/" + FakeConfiguration.FAKE_REGION + "/fake-app/"; - BucketLifecycleConfiguration.Rule rule = new BucketLifecycleConfiguration.Rule().withExpirationInDays(5).withPrefix(clusterPath); + String clusterPath = + "casstestbackup/" + FakeConfiguration.FAKE_REGION + "/fake-app/"; + BucketLifecycleConfiguration.Rule rule = + new BucketLifecycleConfiguration.Rule() + .withExpirationInDays(5) + .withPrefix(clusterPath); rule.setStatus(BucketLifecycleConfiguration.ENABLED); rule.setId(clusterPath); rules.add(rule); - } bconf.setRules(rules); return bconf; } @Mock - public void setBucketLifecycleConfiguration(String bucketName, BucketLifecycleConfiguration bucketLifecycleConfiguration) { + public void setBucketLifecycleConfiguration( + String bucketName, BucketLifecycleConfiguration bucketLifecycleConfiguration) { bconf = bucketLifecycleConfiguration; } - } -} \ No newline at end of file +} diff --git a/priam/src/test/java/com/netflix/priam/backup/TestSnapshotStatusMgr.java b/priam/src/test/java/com/netflix/priam/backup/TestSnapshotStatusMgr.java index 7d6a26cd5..e70ae6858 100644 --- a/priam/src/test/java/com/netflix/priam/backup/TestSnapshotStatusMgr.java +++ b/priam/src/test/java/com/netflix/priam/backup/TestSnapshotStatusMgr.java @@ -21,6 +21,9 @@ import com.google.inject.Injector; import com.netflix.priam.config.IConfiguration; import com.netflix.priam.utils.DateUtil; +import java.io.File; +import java.util.Date; +import java.util.List; import org.joda.time.DateTime; import org.junit.Assert; import org.junit.BeforeClass; @@ -28,33 +31,25 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.File; -import java.util.Date; -import java.util.List; - -/** - * Created by aagrawal on 7/11/17. - */ +/** Created by aagrawal on 7/11/17. */ public class TestSnapshotStatusMgr { private static final Logger logger = LoggerFactory.getLogger(TestSnapshotStatusMgr.class); private static IBackupStatusMgr backupStatusMgr; @BeforeClass - public static void setup() { + public static void setup() { Injector injector = Guice.createInjector(new BRTestModule()); - //cleanup old saved file, if any + // cleanup old saved file, if any IConfiguration configuration = injector.getInstance(IConfiguration.class); File f = new File(configuration.getBackupStatusFileLoc()); - if (f.exists()) - f.delete(); + if (f.exists()) f.delete(); backupStatusMgr = injector.getInstance(IBackupStatusMgr.class); } @Test - public void testSnapshotStatusAddFinish() throws Exception - { + public void testSnapshotStatusAddFinish() throws Exception { Date startTime = DateUtil.getDate("198407110720"); BackupMetadata backupMetadata = new BackupMetadata("123", startTime); @@ -77,8 +72,7 @@ public void testSnapshotStatusAddFinish() throws Exception } @Test - public void testSnapshotStatusAddFailed() throws Exception - { + public void testSnapshotStatusAddFailed() throws Exception { Date startTime = DateUtil.getDate("198407120720"); BackupMetadata backupMetadata = new BackupMetadata("123", startTime); @@ -101,13 +95,11 @@ public void testSnapshotStatusAddFailed() throws Exception } @Test - public void testSnapshotStatusMultiAddFinishInADay() throws Exception - { + public void testSnapshotStatusMultiAddFinishInADay() throws Exception { final int noOfEntries = 10; Date startTime = DateUtil.getDate("19840101"); - for (int i = 0 ; i < noOfEntries; i++) - { + for (int i = 0; i < noOfEntries; i++) { Date time = new DateTime(startTime.getTime()).plusHours(i).toDate(); BackupMetadata backupMetadata = new BackupMetadata("123", time); backupStatusMgr.start(backupMetadata); @@ -118,13 +110,11 @@ public void testSnapshotStatusMultiAddFinishInADay() throws Exception Assert.assertEquals(noOfEntries, metadataList.size()); logger.info(metadataList.toString()); - //Ensure that list is always maintained from latest to eldest + // Ensure that list is always maintained from latest to eldest Date latest = null; - for (BackupMetadata backupMetadata: metadataList) { - if (latest == null) - latest = backupMetadata.getStart(); - else - { + for (BackupMetadata backupMetadata : metadataList) { + if (latest == null) latest = backupMetadata.getStart(); + else { Assert.assertTrue(backupMetadata.getStart().before(latest)); latest = backupMetadata.getStart(); } @@ -132,13 +122,11 @@ public void testSnapshotStatusMultiAddFinishInADay() throws Exception } @Test - public void testSnapshotStatusSize() throws Exception - { + public void testSnapshotStatusSize() throws Exception { final int noOfEntries = backupStatusMgr.getCapacity() + 1; Date startTime = DateUtil.getDate("19850101"); - for (int i = 0 ; i < noOfEntries; i++) - { + for (int i = 0; i < noOfEntries; i++) { Date time = new DateTime(startTime.getTime()).plusDays(i).toDate(); BackupMetadata backupMetadata = new BackupMetadata("123", time); backupStatusMgr.start(backupMetadata); @@ -146,7 +134,7 @@ public void testSnapshotStatusSize() throws Exception } // Verify there is only capacity entries - Assert.assertEquals(backupStatusMgr.getCapacity(), backupStatusMgr.getAllSnapshotStatus().size()); + Assert.assertEquals( + backupStatusMgr.getCapacity(), backupStatusMgr.getAllSnapshotStatus().size()); } - } diff --git a/priam/src/test/java/com/netflix/priam/backup/identity/DoubleRingTest.java b/priam/src/test/java/com/netflix/priam/backup/identity/DoubleRingTest.java index 8366a454b..5cbd60c84 100644 --- a/priam/src/test/java/com/netflix/priam/backup/identity/DoubleRingTest.java +++ b/priam/src/test/java/com/netflix/priam/backup/identity/DoubleRingTest.java @@ -17,23 +17,19 @@ package com.netflix.priam.backup.identity; -import java.util.List; - -import org.junit.Test; +import static org.junit.Assert.assertEquals; import com.google.common.collect.Lists; import com.netflix.priam.identity.DoubleRing; import com.netflix.priam.identity.InstanceIdentity; import com.netflix.priam.identity.PriamInstance; +import java.util.List; +import org.junit.Test; -import static org.junit.Assert.assertEquals; - -public class DoubleRingTest extends InstanceTestUtils -{ +public class DoubleRingTest extends InstanceTestUtils { @Test - public void testDouble() throws Exception - { + public void testDouble() throws Exception { createInstances(); int originalSize = factory.getAllIds(config.getAppName()).size(); new DoubleRing(config, factory, tokenManager).doubleSlots(); @@ -44,29 +40,23 @@ public void testDouble() throws Exception validate(doubled); } - private void validate(List doubled) - { + private void validate(List doubled) { List validator = Lists.newArrayList(); - for (int i = 0; i < doubled.size(); i++) - { + for (int i = 0; i < doubled.size(); i++) { validator.add(tokenManager.createToken(i, doubled.size(), config.getDC())); - } - - for (int i = 0; i < doubled.size(); i++) - { + + for (int i = 0; i < doubled.size(); i++) { PriamInstance ins = doubled.get(i); assertEquals(validator.get(i), ins.getToken()); int id = ins.getId() - tokenManager.regionOffset(config.getDC()); System.out.println(ins); - if (0 != id % 2) - assertEquals(ins.getInstanceId(), InstanceIdentity.DUMMY_INSTANCE_ID); + if (0 != id % 2) assertEquals(ins.getInstanceId(), InstanceIdentity.DUMMY_INSTANCE_ID); } } @Test - public void testBR() throws Exception - { + public void testBR() throws Exception { createInstances(); int intialSize = factory.getAllIds(config.getAppName()).size(); DoubleRing ring = new DoubleRing(config, factory, tokenManager); diff --git a/priam/src/test/java/com/netflix/priam/backup/identity/FakeInstanceEnvIdentity.java b/priam/src/test/java/com/netflix/priam/backup/identity/FakeInstanceEnvIdentity.java index a9ef419c1..1d23b4e56 100644 --- a/priam/src/test/java/com/netflix/priam/backup/identity/FakeInstanceEnvIdentity.java +++ b/priam/src/test/java/com/netflix/priam/backup/identity/FakeInstanceEnvIdentity.java @@ -21,19 +21,18 @@ public class FakeInstanceEnvIdentity implements InstanceEnvIdentity { - @Override - public Boolean isClassic() { - return null; - } + @Override + public Boolean isClassic() { + return null; + } - @Override - public Boolean isDefaultVpc() { - return null; - } + @Override + public Boolean isDefaultVpc() { + return null; + } - @Override - public Boolean isNonDefaultVpc() { - return null; - } - -} \ No newline at end of file + @Override + public Boolean isNonDefaultVpc() { + return null; + } +} diff --git a/priam/src/test/java/com/netflix/priam/backup/identity/InstanceIdentityTest.java b/priam/src/test/java/com/netflix/priam/backup/identity/InstanceIdentityTest.java index eb3bbfdb2..2759e5679 100644 --- a/priam/src/test/java/com/netflix/priam/backup/identity/InstanceIdentityTest.java +++ b/priam/src/test/java/com/netflix/priam/backup/identity/InstanceIdentityTest.java @@ -17,24 +17,20 @@ package com.netflix.priam.backup.identity; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + import com.netflix.priam.identity.DoubleRing; import com.netflix.priam.identity.InstanceIdentity; import com.netflix.priam.identity.PriamInstance; - -import org.junit.Test; - import java.util.List; +import org.junit.Test; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.assertFalse; - -public class InstanceIdentityTest extends InstanceTestUtils -{ +public class InstanceIdentityTest extends InstanceTestUtils { @Test - public void testCreateToken() throws Exception - { + public void testCreateToken() throws Exception { identity = createInstanceIdentity("az1", "fakeinstance1"); int hash = tokenManager.regionOffset(config.getDC()); @@ -66,10 +62,9 @@ public void testCreateToken() throws Exception identity = createInstanceIdentity("az3", "fakeinstance9"); assertEquals(8, identity.getInstance().getId() - hash); } - + @Test - public void testGetSeedsAutobootstrapTrue() throws Exception - { + public void testGetSeedsAutobootstrapTrue() throws Exception { boolean previous = (Boolean) config.getFakeConfig("auto_bootstrap"); try { config.setFakeConfig("auto_bootstrap", true); @@ -80,12 +75,10 @@ public void testGetSeedsAutobootstrapTrue() throws Exception } finally { config.setFakeConfig("auto_bootstrap", previous); } - } @Test - public void testGetSeedsAutobootstrapFalse() throws Exception - { + public void testGetSeedsAutobootstrapFalse() throws Exception { boolean previous = (Boolean) config.getFakeConfig("auto_bootstrap"); try { config.setFakeConfig("auto_bootstrap", false); @@ -99,27 +92,23 @@ public void testGetSeedsAutobootstrapFalse() throws Exception } @Test - public void testDoubleSlots() throws Exception - { + public void testDoubleSlots() throws Exception { createInstances(); int before = factory.getAllIds("fake-app").size(); new DoubleRing(config, factory, tokenManager).doubleSlots(); List lst = factory.getAllIds(config.getAppName()); // sort it so it will look good if you want to print it. factory.sort(lst); - for (int i = 0; i < lst.size(); i++) - { + for (int i = 0; i < lst.size(); i++) { System.out.println(lst.get(i)); - if (0 == i % 2) - continue; + if (0 == i % 2) continue; assertEquals(InstanceIdentity.DUMMY_INSTANCE_ID, lst.get(i).getInstanceId()); } assertEquals(before * 2, lst.size()); } @Test - public void testDoubleGrap() throws Exception - { + public void testDoubleGrap() throws Exception { createInstances(); new DoubleRing(config, factory, tokenManager).doubleSlots(); config.zone = "az1"; @@ -129,11 +118,8 @@ public void testDoubleGrap() throws Exception printInstance(identity.getInstance(), hash); } - public void printInstance(PriamInstance ins, int hash) - { + public void printInstance(PriamInstance ins, int hash) { System.out.println("ID: " + (ins.getId() - hash)); System.out.println("PayLoad: " + ins.getToken()); - } - } diff --git a/priam/src/test/java/com/netflix/priam/backup/identity/InstanceTestUtils.java b/priam/src/test/java/com/netflix/priam/backup/identity/InstanceTestUtils.java index ed9b14677..587948f26 100644 --- a/priam/src/test/java/com/netflix/priam/backup/identity/InstanceTestUtils.java +++ b/priam/src/test/java/com/netflix/priam/backup/identity/InstanceTestUtils.java @@ -31,16 +31,13 @@ import com.netflix.priam.utils.ITokenManager; import com.netflix.priam.utils.Sleeper; import com.netflix.priam.utils.TokenManager; - -import org.junit.Before; -import org.junit.Ignore; - import java.util.ArrayList; import java.util.List; +import org.junit.Before; +import org.junit.Ignore; @Ignore -public abstract class InstanceTestUtils -{ +public abstract class InstanceTestUtils { List instances = new ArrayList(); IMembership membership; @@ -50,13 +47,12 @@ public abstract class InstanceTestUtils Sleeper sleeper; DeadTokenRetriever deadTokenRetriever; PreGeneratedTokenRetriever preGeneratedTokenRetriever; - NewTokenRetriever newTokenRetriever; - ITokenManager tokenManager; - InstanceEnvIdentity insEnvIdentity; + NewTokenRetriever newTokenRetriever; + ITokenManager tokenManager; + InstanceEnvIdentity insEnvIdentity; @Before - public void setup() - { + public void setup() { instances.add("fakeinstance1"); instances.add("fakeinstance2"); instances.add("fakeinstance3"); @@ -72,13 +68,15 @@ public void setup() tokenManager = new TokenManager(config); factory = new FakePriamInstanceFactory(config); sleeper = new FakeSleeper(); - this.deadTokenRetriever = new DeadTokenRetriever(factory, membership, config, sleeper, insEnvIdentity); - this.preGeneratedTokenRetriever = new PreGeneratedTokenRetriever(factory, membership, config, sleeper); - this.newTokenRetriever = new NewTokenRetriever(factory, membership, config, sleeper, tokenManager); + this.deadTokenRetriever = + new DeadTokenRetriever(factory, membership, config, sleeper, insEnvIdentity); + this.preGeneratedTokenRetriever = + new PreGeneratedTokenRetriever(factory, membership, config, sleeper); + this.newTokenRetriever = + new NewTokenRetriever(factory, membership, config, sleeper, tokenManager); } - public void createInstances() throws Exception - { + public void createInstances() throws Exception { createInstanceIdentity("az1", "fakeinstance1"); createInstanceIdentity("az1", "fakeinstance2"); createInstanceIdentity("az1", "fakeinstance3"); @@ -92,14 +90,18 @@ public void createInstances() throws Exception createInstanceIdentity("az3", "fakeinstance9"); } - protected InstanceIdentity createInstanceIdentity(String zone, String instanceId) throws Exception - { + protected InstanceIdentity createInstanceIdentity(String zone, String instanceId) + throws Exception { config.zone = zone; config.instance_id = instanceId; - return new InstanceIdentity(factory, membership, config, sleeper, new TokenManager(config) - , this.deadTokenRetriever - , this.preGeneratedTokenRetriever - , this.newTokenRetriever - ); + return new InstanceIdentity( + factory, + membership, + config, + sleeper, + new TokenManager(config), + this.deadTokenRetriever, + this.preGeneratedTokenRetriever, + this.newTokenRetriever); } } diff --git a/priam/src/test/java/com/netflix/priam/backup/identity/token/FakeDeadTokenRetriever.java b/priam/src/test/java/com/netflix/priam/backup/identity/token/FakeDeadTokenRetriever.java index 9c213e6b4..8f9b825da 100755 --- a/priam/src/test/java/com/netflix/priam/backup/identity/token/FakeDeadTokenRetriever.java +++ b/priam/src/test/java/com/netflix/priam/backup/identity/token/FakeDeadTokenRetriever.java @@ -23,22 +23,21 @@ public class FakeDeadTokenRetriever implements IDeadTokenRetriever { - @Override - public PriamInstance get() throws Exception { - // TODO Auto-generated method stub - return null; - } + @Override + public PriamInstance get() throws Exception { + // TODO Auto-generated method stub + return null; + } - @Override - public String getReplaceIp() { - // TODO Auto-generated method stub - return null; - } + @Override + public String getReplaceIp() { + // TODO Auto-generated method stub + return null; + } - @Override - public void setLocMap(ListMultimap locMap) { - // TODO Auto-generated method stub - - } + @Override + public void setLocMap(ListMultimap locMap) { + // TODO Auto-generated method stub + } } diff --git a/priam/src/test/java/com/netflix/priam/backup/identity/token/FakeNewTokenRetriever.java b/priam/src/test/java/com/netflix/priam/backup/identity/token/FakeNewTokenRetriever.java index 7f9f300d5..7dc3c1487 100755 --- a/priam/src/test/java/com/netflix/priam/backup/identity/token/FakeNewTokenRetriever.java +++ b/priam/src/test/java/com/netflix/priam/backup/identity/token/FakeNewTokenRetriever.java @@ -23,16 +23,15 @@ public class FakeNewTokenRetriever implements INewTokenRetriever { - @Override - public PriamInstance get() throws Exception { - // TODO Auto-generated method stub - return null; - } + @Override + public PriamInstance get() throws Exception { + // TODO Auto-generated method stub + return null; + } - @Override - public void setLocMap(ListMultimap locMap) { - // TODO Auto-generated method stub - - } + @Override + public void setLocMap(ListMultimap locMap) { + // TODO Auto-generated method stub + } } diff --git a/priam/src/test/java/com/netflix/priam/backup/identity/token/FakePreGeneratedTokenRetriever.java b/priam/src/test/java/com/netflix/priam/backup/identity/token/FakePreGeneratedTokenRetriever.java index 48dc8450d..32b81ebad 100755 --- a/priam/src/test/java/com/netflix/priam/backup/identity/token/FakePreGeneratedTokenRetriever.java +++ b/priam/src/test/java/com/netflix/priam/backup/identity/token/FakePreGeneratedTokenRetriever.java @@ -21,19 +21,17 @@ import com.netflix.priam.identity.PriamInstance; import com.netflix.priam.identity.token.IPreGeneratedTokenRetriever; -public class FakePreGeneratedTokenRetriever implements - IPreGeneratedTokenRetriever { +public class FakePreGeneratedTokenRetriever implements IPreGeneratedTokenRetriever { - @Override - public PriamInstance get() throws Exception { - // TODO Auto-generated method stub - return null; - } + @Override + public PriamInstance get() throws Exception { + // TODO Auto-generated method stub + return null; + } - @Override - public void setLocMap(ListMultimap locMap) { - // TODO Auto-generated method stub - - } + @Override + public void setLocMap(ListMultimap locMap) { + // TODO Auto-generated method stub + } } diff --git a/priam/src/test/java/com/netflix/priam/backupv2/TestLocalDBReaderWriter.java b/priam/src/test/java/com/netflix/priam/backupv2/TestLocalDBReaderWriter.java index 8585828b0..2a63551bf 100644 --- a/priam/src/test/java/com/netflix/priam/backupv2/TestLocalDBReaderWriter.java +++ b/priam/src/test/java/com/netflix/priam/backupv2/TestLocalDBReaderWriter.java @@ -22,15 +22,6 @@ import com.netflix.priam.backup.BRTestModule; import com.netflix.priam.config.IConfiguration; import com.netflix.priam.utils.DateUtil; -import org.apache.cassandra.io.sstable.Component; -import org.apache.commons.io.FileUtils; -import org.junit.After; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import java.io.IOException; import java.nio.file.Path; import java.nio.file.Paths; @@ -42,12 +33,19 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.Future; +import org.apache.cassandra.io.sstable.Component; +import org.apache.commons.io.FileUtils; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; -/** - * Created by aagrawal on 9/4/18. - */ +/** Created by aagrawal on 9/4/18. */ public class TestLocalDBReaderWriter { - private static final Logger logger = LoggerFactory.getLogger(TestLocalDBReaderWriter.class.getName()); + private static final Logger logger = + LoggerFactory.getLogger(TestLocalDBReaderWriter.class.getName()); private static IConfiguration configuration; private static LocalDBReaderWriter localDBReaderWriter; private static Path dummyDataDirectoryLocation; @@ -56,8 +54,7 @@ public class TestLocalDBReaderWriter { public void setUp() { Injector injector = Guice.createInjector(new BRTestModule()); - if (configuration == null) - configuration = injector.getInstance(IConfiguration.class); + if (configuration == null) configuration = injector.getInstance(IConfiguration.class); if (localDBReaderWriter == null) localDBReaderWriter = injector.getInstance(LocalDBReaderWriter.class); @@ -67,7 +64,7 @@ public void setUp() { } @After - public void destroy(){ + public void destroy() { cleanupDir(dummyDataDirectoryLocation); } @@ -85,79 +82,110 @@ public void readWriteLocalDB() throws Exception { int noOfKeyspaces = 2; int noOfCf = 1; int noOfSstables = 2; - Path localDbPath = Paths.get(dummyDataDirectoryLocation.toString(), LocalDBReaderWriter.LOCAL_DB); - List localDBList = generateDummyLocalDB(noOfKeyspaces, noOfCf, noOfSstables); - - localDBList.stream().forEach(localDB -> { - FileUploadResult fileUploadResult = localDB.getLocalDBEntries().get(0).getFileUploadResult(); - final Path localDBPath = localDBReaderWriter.getLocalDBPath(fileUploadResult); - try { - localDBReaderWriter.writeLocalDB(localDBPath, localDB); - } catch (Exception e) { - logger.error("Error while writing to local DB: " + e.getMessage(), e); - } - }); - - //Verify the write succeeded for each KS/CF/SStable. + Path localDbPath = + Paths.get(dummyDataDirectoryLocation.toString(), LocalDBReaderWriter.LOCAL_DB); + List localDBList = + generateDummyLocalDB(noOfKeyspaces, noOfCf, noOfSstables); + + localDBList + .stream() + .forEach( + localDB -> { + FileUploadResult fileUploadResult = + localDB.getLocalDBEntries().get(0).getFileUploadResult(); + final Path localDBPath = + localDBReaderWriter.getLocalDBPath(fileUploadResult); + try { + localDBReaderWriter.writeLocalDB(localDBPath, localDB); + } catch (Exception e) { + logger.error( + "Error while writing to local DB: " + e.getMessage(), e); + } + }); + + // Verify the write succeeded for each KS/CF/SStable. Assert.assertEquals(localDbPath.toFile().listFiles().length, noOfKeyspaces); Path cfLocalDBPath = localDbPath.toFile().listFiles()[0].listFiles()[0].toPath(); Assert.assertEquals(noOfSstables, cfLocalDBPath.toFile().listFiles().length); - //Read the database. - LocalDBReaderWriter.LocalDB localDB = localDBReaderWriter.readLocalDB(cfLocalDBPath.toFile().listFiles()[0].toPath()); - Assert.assertEquals(EnumSet.allOf(Component.Type.class).size(), localDB.getLocalDBEntries().size()); + // Read the database. + LocalDBReaderWriter.LocalDB localDB = + localDBReaderWriter.readLocalDB(cfLocalDBPath.toFile().listFiles()[0].toPath()); + Assert.assertEquals( + EnumSet.allOf(Component.Type.class).size(), localDB.getLocalDBEntries().size()); } @Test - public void upsertLocalDB() throws Exception{ + public void upsertLocalDB() throws Exception { LocalDBReaderWriter.LocalDB localDB = generateDummyLocalDB(1, 1, 1).get(0); - //Lets do write with each LocalDBEntry first. - localDB.getLocalDBEntries().stream().forEach(localDBEntry -> { - try { - localDBReaderWriter.upsertLocalDBEntry(localDBEntry); - } catch (Exception e) { - e.printStackTrace(); - } - }); - - //Verify the write has happened. - LocalDBReaderWriter.LocalDBEntry localDBEntry = localDBReaderWriter.getLocalDBEntry(localDB.getLocalDBEntries().get(0).getFileUploadResult()); + // Lets do write with each LocalDBEntry first. + localDB.getLocalDBEntries() + .stream() + .forEach( + localDBEntry -> { + try { + localDBReaderWriter.upsertLocalDBEntry(localDBEntry); + } catch (Exception e) { + e.printStackTrace(); + } + }); + + // Verify the write has happened. + LocalDBReaderWriter.LocalDBEntry localDBEntry = + localDBReaderWriter.getLocalDBEntry( + localDB.getLocalDBEntries().get(0).getFileUploadResult()); Assert.assertNotNull(localDBEntry); - //Now lets see if we can write the same entry again?? - LocalDBReaderWriter.LocalDB localDBUpsert = localDBReaderWriter.upsertLocalDBEntry(localDBEntry); - Assert.assertEquals(localDB.getLocalDBEntries().size(), localDBUpsert.getLocalDBEntries().size()); + // Now lets see if we can write the same entry again?? + LocalDBReaderWriter.LocalDB localDBUpsert = + localDBReaderWriter.upsertLocalDBEntry(localDBEntry); + Assert.assertEquals( + localDB.getLocalDBEntries().size(), localDBUpsert.getLocalDBEntries().size()); - //Now lets change the localDBEntry and see if upsert succeeds. + // Now lets change the localDBEntry and see if upsert succeeds. localDBEntry.setTimeLastReferenced(DateUtil.getInstant()); localDBUpsert = localDBReaderWriter.upsertLocalDBEntry(localDBEntry); - LocalDBReaderWriter.LocalDBEntry localDBEntryUpsert = localDBReaderWriter.getLocalDBEntry(localDBEntry.getFileUploadResult()); - Assert.assertEquals(localDBEntryUpsert.getTimeLastReferenced(),localDBEntry.getTimeLastReferenced()); - Assert.assertEquals(localDB.getLocalDBEntries().size(), localDBUpsert.getLocalDBEntries().size()); - - //Now change the file modification time. This should end up creating a new DB Entry. + LocalDBReaderWriter.LocalDBEntry localDBEntryUpsert = + localDBReaderWriter.getLocalDBEntry(localDBEntry.getFileUploadResult()); + Assert.assertEquals( + localDBEntryUpsert.getTimeLastReferenced(), localDBEntry.getTimeLastReferenced()); + Assert.assertEquals( + localDB.getLocalDBEntries().size(), localDBUpsert.getLocalDBEntries().size()); + + // Now change the file modification time. This should end up creating a new DB Entry. localDBEntry.getFileUploadResult().setLastModifiedTime(DateUtil.getInstant()); localDBUpsert = localDBReaderWriter.upsertLocalDBEntry(localDBEntry); - localDBEntryUpsert = localDBReaderWriter.getLocalDBEntry(localDBEntry.getFileUploadResult()); - Assert.assertEquals(localDB.getLocalDBEntries().size() + 1, localDBUpsert.getLocalDBEntries().size()); - Assert.assertEquals(localDBEntry.getFileUploadResult().getLastModifiedTime(), localDBEntryUpsert.getFileUploadResult().getLastModifiedTime()); + localDBEntryUpsert = + localDBReaderWriter.getLocalDBEntry(localDBEntry.getFileUploadResult()); + Assert.assertEquals( + localDB.getLocalDBEntries().size() + 1, localDBUpsert.getLocalDBEntries().size()); + Assert.assertEquals( + localDBEntry.getFileUploadResult().getLastModifiedTime(), + localDBEntryUpsert.getFileUploadResult().getLastModifiedTime()); } @Test - public void readConcurrentLocalDB() throws Exception{ + public void readConcurrentLocalDB() throws Exception { List localDBList = generateDummyLocalDB(1, 1, 1); - localDBList.stream().forEach(localDB -> { - FileUploadResult fileUploadResult = localDB.getLocalDBEntries().get(0).getFileUploadResult(); - final Path localDBPath = localDBReaderWriter.getLocalDBPath(fileUploadResult); - try { - localDBReaderWriter.writeLocalDB(localDBPath, localDB); - } catch (Exception e) { - logger.error("Error while writing to local DB: " + e.getMessage(), e); - } - }); - - FileUploadResult sample = localDBList.get(0).getLocalDBEntries().get(0).getFileUploadResult(); + localDBList + .stream() + .forEach( + localDB -> { + FileUploadResult fileUploadResult = + localDB.getLocalDBEntries().get(0).getFileUploadResult(); + final Path localDBPath = + localDBReaderWriter.getLocalDBPath(fileUploadResult); + try { + localDBReaderWriter.writeLocalDB(localDBPath, localDB); + } catch (Exception e) { + logger.error( + "Error while writing to local DB: " + e.getMessage(), e); + } + }); + + FileUploadResult sample = + localDBList.get(0).getLocalDBEntries().get(0).getFileUploadResult(); int size = 5; ExecutorService threads = Executors.newFixedThreadPool(size); @@ -174,21 +202,24 @@ public void readConcurrentLocalDB() throws Exception{ // check the results of the tasks. int noOfBadRun = 0; for (Future fut : futures) { - if (!fut.get()) noOfBadRun ++; + if (!fut.get()) noOfBadRun++; } Assert.assertEquals(0, noOfBadRun); } @Test - public void writeConcurrentLocalDB() throws Exception{ + public void writeConcurrentLocalDB() throws Exception { LocalDBReaderWriter.LocalDB localDB = generateDummyLocalDB(1, 1, 1).get(0); int size = localDB.getLocalDBEntries().size(); ExecutorService threads = Executors.newFixedThreadPool(size); List> torun = new ArrayList<>(size); for (int i = 0; i < size; i++) { int finalI = i; - torun.add(() -> localDBReaderWriter.upsertLocalDBEntry(localDB.getLocalDBEntries().get(finalI))); + torun.add( + () -> + localDBReaderWriter.upsertLocalDBEntry( + localDB.getLocalDBEntries().get(finalI))); } // all tasks executed in different threads, at 'once'. List> futures = threads.invokeAll(torun); @@ -198,22 +229,27 @@ public void writeConcurrentLocalDB() throws Exception{ // check the results of the tasks. int noOfBadRun = 0; for (Future fut : futures) { - //We expect exception here. - try{ + // We expect exception here. + try { fut.get(); - }catch(Exception e){ + } catch (Exception e) { noOfBadRun++; } } Assert.assertEquals(0, noOfBadRun); - LocalDBReaderWriter.LocalDB localDBRead = localDBReaderWriter.readLocalDB(localDBReaderWriter.getLocalDBPath(localDB.getLocalDBEntries().get(0).getFileUploadResult())); - Assert.assertEquals(localDB.getLocalDBEntries().size(), localDBRead.getLocalDBEntries().size()); + LocalDBReaderWriter.LocalDB localDBRead = + localDBReaderWriter.readLocalDB( + localDBReaderWriter.getLocalDBPath( + localDB.getLocalDBEntries().get(0).getFileUploadResult())); + Assert.assertEquals( + localDB.getLocalDBEntries().size(), localDBRead.getLocalDBEntries().size()); } - private List generateDummyLocalDB(int noOfKeyspaces, int noOfCf, int noOfSstables) throws Exception { + private List generateDummyLocalDB( + int noOfKeyspaces, int noOfCf, int noOfSstables) throws Exception { - //Clean the dummy directory + // Clean the dummy directory cleanupDir(dummyDataDirectoryLocation); List localDBList = new ArrayList<>(); @@ -227,12 +263,29 @@ private List generateDummyLocalDB(int noOfKeyspaces for (int k = 1; k <= noOfSstables; k++) { String prefixName = "mc-" + k + "-big"; - LocalDBReaderWriter.LocalDB localDB = new LocalDBReaderWriter.LocalDB(new ArrayList<>()); + LocalDBReaderWriter.LocalDB localDB = + new LocalDBReaderWriter.LocalDB(new ArrayList<>()); localDBList.add(localDB); for (Component.Type type : EnumSet.allOf(Component.Type.class)) { - Path componentPath = Paths.get(dummyDataDirectoryLocation.toFile().getAbsolutePath(), keyspaceName, columnfamilyname, prefixName + "-" + type.name() + ".db"); - FileUploadResult fileUploadResult = new FileUploadResult(componentPath, keyspaceName, columnfamilyname, DateUtil.getInstant(), DateUtil.getInstant(), random.nextLong()); - LocalDBReaderWriter.LocalDBEntry localDBEntry = new LocalDBReaderWriter.LocalDBEntry(fileUploadResult, DateUtil.getInstant(), DateUtil.getInstant()); + Path componentPath = + Paths.get( + dummyDataDirectoryLocation.toFile().getAbsolutePath(), + keyspaceName, + columnfamilyname, + prefixName + "-" + type.name() + ".db"); + FileUploadResult fileUploadResult = + new FileUploadResult( + componentPath, + keyspaceName, + columnfamilyname, + DateUtil.getInstant(), + DateUtil.getInstant(), + random.nextLong()); + LocalDBReaderWriter.LocalDBEntry localDBEntry = + new LocalDBReaderWriter.LocalDBEntry( + fileUploadResult, + DateUtil.getInstant(), + DateUtil.getInstant()); localDB.getLocalDBEntries().add(localDBEntry); } } diff --git a/priam/src/test/java/com/netflix/priam/cassandra/token/TestDoublingLogic.java b/priam/src/test/java/com/netflix/priam/cassandra/token/TestDoublingLogic.java index ddd7970ea..b7ebd9d51 100644 --- a/priam/src/test/java/com/netflix/priam/cassandra/token/TestDoublingLogic.java +++ b/priam/src/test/java/com/netflix/priam/cassandra/token/TestDoublingLogic.java @@ -23,101 +23,82 @@ import java.util.HashMap; import java.util.List; import java.util.Map; - import org.apache.commons.lang3.StringUtils; import org.junit.Test; -/** - * Seems like skip 3 is the magic number.... this test will make sure to test the same. - * - */ -public class TestDoublingLogic -{ +/** Seems like skip 3 is the magic number.... this test will make sure to test the same. */ +public class TestDoublingLogic { private static final int RACS = 2; private static final int NODES_PER_RACS = 2; @Test - public void testSkip() - { + public void testSkip() { List nodes = new ArrayList(); for (int i = 0; i < NODES_PER_RACS; i++) - for (int j = 0; j < RACS; j++) - nodes.add("RAC-" + j); - //printNodes(nodes); - + for (int j = 0; j < RACS; j++) nodes.add("RAC-" + j); + // printNodes(nodes); + List newNodes = nodes; - for (int i = 0; i < 15; i++) - { + for (int i = 0; i < 15; i++) { int count = newNodes.size(); newNodes = doubleNodes(newNodes); - assertEquals(newNodes.size(), count *2); + assertEquals(newNodes.size(), count * 2); // printNodes(newNodes); validate(newNodes, nodes); } } - public void printNodes(List nodes) - { + public void printNodes(List nodes) { System.out.println("=== Printing - Array of Size :" + nodes.size()); System.out.println(StringUtils.join(nodes, "\n")); - System.out.println("=====================Completed doubling===============================" + nodes.size()); + System.out.println( + "=====================Completed doubling===============================" + + nodes.size()); } - private void validate(List newNodes, List nodes) - { + private void validate(List newNodes, List nodes) { String temp = ""; int count = 0; - for (String node : newNodes) - { - if (temp.equals(node)) - count++; - else - count = 0; - - if (count == 2) - { + for (String node : newNodes) { + if (temp.equals(node)) count++; + else count = 0; + + if (count == 2) { System.out.println("Found an issue....."); throw new RuntimeException(); } - temp = node; + temp = node; } - + // compare if they are the same set... boolean test = true; - for (int i = 0; i < nodes.size(); i++) - { - if (!newNodes.get(i).equals(nodes.get(i))) - test = false; + for (int i = 0; i < nodes.size(); i++) { + if (!newNodes.get(i).equals(nodes.get(i))) test = false; } if (test) - throw new RuntimeException("Awesome we are back to the natural order... No need to test more"); + throw new RuntimeException( + "Awesome we are back to the natural order... No need to test more"); } - private List doubleNodes(List nodes) - { + private List doubleNodes(List nodes) { List lst = new ArrayList(); Map return_ = new HashMap(); - for (int i = 0; i < nodes.size(); i++) - { + for (int i = 0; i < nodes.size(); i++) { return_.put(i * 2, nodes.get(i)); } - for (int i = 0; i < nodes.size() * 2; i++) - { - if (0 == i % 2) - { - - //rotate - if (i + 3 >= (nodes.size() * 2)) - { - int delta = (i+3) - (nodes.size() *2); - return_.put(delta, return_.get(i)); + for (int i = 0; i < nodes.size() * 2; i++) { + if (0 == i % 2) { + + // rotate + if (i + 3 >= (nodes.size() * 2)) { + int delta = (i + 3) - (nodes.size() * 2); + return_.put(delta, return_.get(i)); } return_.put(i + 3, return_.get(i)); } } - for (int i = 0; i < nodes.size() * 2; i++) - lst.add(return_.get(i)); + for (int i = 0; i < nodes.size() * 2; i++) lst.add(return_.get(i)); return lst; } diff --git a/priam/src/test/java/com/netflix/priam/config/FakeBackupRestoreConfig.java b/priam/src/test/java/com/netflix/priam/config/FakeBackupRestoreConfig.java index 8a8410926..0194044c2 100644 --- a/priam/src/test/java/com/netflix/priam/config/FakeBackupRestoreConfig.java +++ b/priam/src/test/java/com/netflix/priam/config/FakeBackupRestoreConfig.java @@ -1,26 +1,22 @@ /** * Copyright 2018 Netflix, Inc. - *

- * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and + * + *

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file + * except in compliance with the License. You may obtain a copy of the License at + * + *

http://www.apache.org/licenses/LICENSE-2.0 + * + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing permissions and * limitations under the License. */ package com.netflix.priam.config; -/** - * Created by aagrawal on 6/26/18. - */ -public class FakeBackupRestoreConfig implements IBackupRestoreConfig{ +/** Created by aagrawal on 6/26/18. */ +public class FakeBackupRestoreConfig implements IBackupRestoreConfig { @Override public String getSnapshotMetaServiceCronExpression() { - return "0 0/2 * 1/1 * ? *"; //Every 2 minutes for testing purposes + return "0 0/2 * 1/1 * ? *"; // Every 2 minutes for testing purposes } } diff --git a/priam/src/test/java/com/netflix/priam/config/FakeConfiguration.java b/priam/src/test/java/com/netflix/priam/config/FakeConfiguration.java index 32b2eb1b4..33560b2d6 100644 --- a/priam/src/test/java/com/netflix/priam/config/FakeConfiguration.java +++ b/priam/src/test/java/com/netflix/priam/config/FakeConfiguration.java @@ -19,25 +19,22 @@ import com.google.common.collect.Lists; import com.google.inject.Singleton; -import com.netflix.priam.tuner.JVMOption; -import com.netflix.priam.tuner.GCType; import com.netflix.priam.identity.config.InstanceDataRetriever; import com.netflix.priam.identity.config.LocalInstanceDataRetriever; import com.netflix.priam.scheduler.SchedulerType; import com.netflix.priam.scheduler.UnsupportedTypeException; - +import com.netflix.priam.tuner.GCType; +import com.netflix.priam.tuner.JVMOption; import java.io.File; import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.stream.Collectors; @Singleton -public class FakeConfiguration implements IConfiguration -{ +public class FakeConfiguration implements IConfiguration { - public static final String FAKE_REGION = "us-east-1"; + public static final String FAKE_REGION = "us-east-1"; public String region; public String appName; @@ -46,19 +43,17 @@ public class FakeConfiguration implements IConfiguration public String restorePrefix; public Map fakeConfig; public Map fakeProperties = new HashMap<>(); - - public FakeConfiguration() - { + + public FakeConfiguration() { this(FAKE_REGION, "my_fake_cluster", "my_zone", "i-01234567"); } - public FakeConfiguration(String region, String appName, String zone, String ins_id) - { + public FakeConfiguration(String region, String appName, String zone, String ins_id) { this.region = region; this.appName = appName; this.zone = zone; this.instance_id = ins_id; - this.restorePrefix = ""; + this.restorePrefix = ""; fakeConfig = new HashMap<>(); fakeConfig.put("auto_bootstrap", false); } @@ -78,29 +73,25 @@ public void initialize() { } @Override - public String getBackupLocation() - { + public String getBackupLocation() { // TODO Auto-generated method stub return "casstestbackup"; } @Override - public String getBackupPrefix() - { + public String getBackupPrefix() { // TODO Auto-generated method stub return "TEST-netflix.platform.S3"; } @Override - public String getCommitLogLocation() - { + public String getCommitLogLocation() { // TODO Auto-generated method stub return "cass/commitlog"; } @Override - public String getDataFileLocation() - { + public String getDataFileLocation() { // TODO Auto-generated method stub return "target/data"; } @@ -116,15 +107,13 @@ public String getHintsLocation() { } @Override - public String getCacheLocation() - { + public String getCacheLocation() { // TODO Auto-generated method stub return "cass/caches"; } @Override - public List getRacs() - { + public List getRacs() { return Arrays.asList("az1", "az2", "az3"); } @@ -134,53 +123,45 @@ public int getThriftPort() { } @Override - public int getNativeTransportPort() - { + public int getNativeTransportPort() { return 9042; } @Override - public String getSnitch() - { + public String getSnitch() { return "org.apache.cassandra.locator.SimpleSnitch"; } @Override - public String getRac() - { + public String getRac() { return this.zone; } @Override - public String getHostname() - { + public String getHostname() { // TODO Auto-generated method stub return instance_id; } @Override - public String getInstanceName() - { + public String getInstanceName() { return instance_id; } @Override - public String getHeapSize() - { + public String getHeapSize() { // TODO Auto-generated method stub return null; } @Override - public String getHeapNewSize() - { + public String getHeapNewSize() { // TODO Auto-generated method stub return null; } @Override - public int getBackupHour() - { + public int getBackupHour() { // TODO Auto-generated method stub return 12; } @@ -196,8 +177,7 @@ public SchedulerType getBackupSchedulerType() { } @Override - public String getRestoreSnapshot() - { + public String getRestoreSnapshot() { // TODO Auto-generated method stub return null; } @@ -208,8 +188,7 @@ public String getAppName() { } @Override - public String getACLGroupName() - { + public String getACLGroupName() { return this.getAppName(); } @@ -229,41 +208,37 @@ public int getRestoreThreads() { return 2; } - public void setRestorePrefix(String prefix) - { + public void setRestorePrefix(String prefix) { // TODO Auto-generated method stub restorePrefix = prefix; } @Override - public String getRestorePrefix() - { + public String getRestorePrefix() { // TODO Auto-generated method stub return restorePrefix; } @Override - public String getBackupCommitLogLocation() - { + public String getBackupCommitLogLocation() { return "cass/backup/cl/"; } @Override - public boolean isMultiDC() - { + public boolean isMultiDC() { // TODO Auto-generated method stub return false; } @Override - public String getASGName() - { + public String getASGName() { // TODO Auto-generated method stub return null; } /** - * Amazon specific setting to query Additional/ Sibling ASG Memberships in csv format to consider while calculating RAC membership + * Amazon specific setting to query Additional/ Sibling ASG Memberships in csv format to + * consider while calculating RAC membership */ @Override public String getSiblingASGNames() { @@ -271,28 +246,25 @@ public String getSiblingASGNames() { } @Override - public boolean isIncrBackup() - { + public boolean isIncrBackup() { return true; } @Override - public String getHostIP() - { + public String getHostIP() { // TODO Auto-generated method stub return null; } - @Override - public int getUploadThrottle() - { + public int getUploadThrottle() { // TODO Auto-generated method stub return 0; } @Override - public InstanceDataRetriever getInstanceDataRetriever() throws InstantiationException, IllegalAccessException, ClassNotFoundException { + public InstanceDataRetriever getInstanceDataRetriever() + throws InstantiationException, IllegalAccessException, ClassNotFoundException { return new LocalInstanceDataRetriever(); } @@ -309,27 +281,23 @@ public int getCompactionThroughput() { } @Override - public String getMaxDirectMemory() - { + public String getMaxDirectMemory() { // TODO Auto-generated method stub return null; } @Override - public String getBootClusterName() - { + public String getBootClusterName() { return "cass_bootstrap"; } @Override - public String getCassHome() - { + public String getCassHome() { return "/tmp/priam"; } @Override - public String getCassStartupScript() - { + public String getCassStartupScript() { return "/usr/bin/false"; } @@ -339,22 +307,19 @@ public long getBackupChunkSize() { } @Override - public void setDC(String region) - { + public void setDC(String region) { // TODO Auto-generated method stub - + } @Override - public boolean isRestoreClosestToken() - { + public boolean isRestoreClosestToken() { // TODO Auto-generated method stub return false; } @Override - public String getCassStopScript() - { + public String getCassStopScript() { return "true"; } @@ -369,104 +334,84 @@ public int getRemediateDeadCassandraRate() { } @Override - public int getStoragePort() - { + public int getStoragePort() { return 7101; } @Override - public String getSeedProviderName() - { + public String getSeedProviderName() { return "org.apache.cassandra.locator.SimpleSeedProvider"; } @Override - public int getBackupRetentionDays() - { + public int getBackupRetentionDays() { return 5; } @Override - public List getBackupRacs() - { + public List getBackupRacs() { return Lists.newArrayList(); } - - public int getMaxHintWindowInMS() - { + + public int getMaxHintWindowInMS() { return 36000; } - public int getHintedHandoffThrottleKb() - { + public int getHintedHandoffThrottleKb() { return 1024; } - public int getMaxHintThreads() - { + public int getMaxHintThreads() { return 1; } - - - /** - * @return memtable_cleanup_threshold in C* yaml - */ + /** @return memtable_cleanup_threshold in C* yaml */ @Override public double getMemtableCleanupThreshold() { return 0.11; } @Override - public int getStreamingThroughputMB() - { + public int getStreamingThroughputMB() { return 400; } - public String getPartitioner() { return "org.apache.cassandra.dht.RandomPartitioner"; } @Override - public int getSSLStoragePort() - { + public int getSSLStoragePort() { // TODO Auto-generated method stub return 7103; } - public String getKeyCacheSizeInMB() - { + public String getKeyCacheSizeInMB() { return "16"; } - public String getKeyCacheKeysToSave() - { + public String getKeyCacheKeysToSave() { return "32"; } - public String getRowCacheSizeInMB() - { + public String getRowCacheSizeInMB() { return "4"; } - public String getRowCacheKeysToSave() - { + public String getRowCacheKeysToSave() { return "4"; } - @Override - public String getCassProcessName() { - return "CassandraDaemon"; - } + @Override + public String getCassProcessName() { + return "CassandraDaemon"; + } - public int getNumTokens() - { + public int getNumTokens() { return 1; } - public String getYamlLocation() - { + public String getYamlLocation() { return "conf/cassandra.yaml"; } @@ -490,14 +435,14 @@ public Map getJVMUpsertSet() { return null; } - public String getAuthenticator() - { + public String getAuthenticator() { return PriamConfiguration.DEFAULT_AUTHENTICATOR; } public String getAuthorizer() { return PriamConfiguration.DEFAULT_AUTHORIZER; } + public boolean doesCassandraStartManually() { return false; } @@ -508,81 +453,66 @@ public boolean isVpcRing() { } @Override - public String getCommitLogBackupPropsFile() - { + public String getCommitLogBackupPropsFile() { return getCassHome() + PriamConfiguration.DEFAULT_COMMITLOG_PROPS_FILE; } @Override - public String getCommitLogBackupArchiveCmd() - { + public String getCommitLogBackupArchiveCmd() { return null; } @Override - public String getCommitLogBackupRestoreCmd() - { + public String getCommitLogBackupRestoreCmd() { return null; } @Override - public String getCommitLogBackupRestoreFromDirs() - { + public String getCommitLogBackupRestoreFromDirs() { return null; } @Override - public String getCommitLogBackupRestorePointInTime() - { + public String getCommitLogBackupRestorePointInTime() { return null; } - public void setRestoreKeySpaces(List keyspaces) { - - } + public void setRestoreKeySpaces(List keyspaces) {} @Override - public int maxCommitLogsRestore() { - return 0; + public int maxCommitLogsRestore() { + return 0; } - public boolean isClientSslEnabled() - { + public boolean isClientSslEnabled() { return true; } - public String getInternodeEncryption() - { + public String getInternodeEncryption() { return "all"; } - public boolean isDynamicSnitchEnabled() - { + public boolean isDynamicSnitchEnabled() { return true; } - public boolean isThriftEnabled() - { + public boolean isThriftEnabled() { return true; } - public boolean isNativeTransportEnabled() - { + public boolean isNativeTransportEnabled() { return false; } - public int getConcurrentReadsCnt() - { + public int getConcurrentReadsCnt() { return 8; } - public int getConcurrentWritesCnt() - { + public int getConcurrentWritesCnt() { return 8; } - public int getConcurrentCompactorsCnt() - { + public int getConcurrentCompactorsCnt() { return 1; } @@ -592,11 +522,11 @@ public int getCompactionLargePartitionWarnThresholdInMB() { } public String getExtraConfigParams() { - return null; - } - + return null; + } + public String getCassYamlVal(String priamKey) { - return ""; + return ""; } @Override @@ -605,61 +535,59 @@ public boolean getAutoBoostrap() { } @Override - public boolean isCreateNewTokenEnable() { - return true; //allow Junit test to create new tokens + return true; // allow Junit test to create new tokens } - @Override - public String getPrivateKeyLocation() { - return null; - } - - @Override - public String getRestoreSourceType() { - return null; - } + @Override + public String getPrivateKeyLocation() { + return null; + } - @Override - public boolean isEncryptBackupEnabled() { - return false; - } + @Override + public String getRestoreSourceType() { + return null; + } @Override + public boolean isEncryptBackupEnabled() { + return false; + } + @Override public String getAWSRoleAssumptionArn() { return null; } @Override - public String getClassicEC2RoleAssumptionArn() { - return null; - } + public String getClassicEC2RoleAssumptionArn() { + return null; + } @Override - public String getVpcEC2RoleAssumptionArn() { - return null; - } + public String getVpcEC2RoleAssumptionArn() { + return null; + } - @Override - public String getGcsServiceAccountId() { - return null; - } + @Override + public String getGcsServiceAccountId() { + return null; + } - @Override - public String getGcsServiceAccountPrivateKeyLoc() { - return null; - } + @Override + public String getGcsServiceAccountPrivateKeyLoc() { + return null; + } - @Override - public String getPgpPasswordPhrase() { - return null; - } + @Override + public String getPgpPasswordPhrase() { + return null; + } - @Override - public String getPgpPublicKeyLoc() { - return null; - } + @Override + public String getPgpPublicKeyLoc() { + return null; + } /** * Use this method for adding extra/ dynamic cassandra startup options or env properties @@ -673,7 +601,7 @@ public Map getExtraEnvParams() { @Override public String getVpcId() { - return ""; + return ""; } @Override @@ -736,14 +664,12 @@ public int getPostRestoreHookTimeOutInDays() { } @Override - public String getProperty(String key, String defaultValue) - { + public String getProperty(String key, String defaultValue) { return fakeProperties.getOrDefault(key, defaultValue); } @Override - public String getMergedConfigurationDirectory() - { + public String getMergedConfigurationDirectory() { return fakeProperties.getOrDefault("priam_test_config", "/tmp/priam_test_config"); } } diff --git a/priam/src/test/java/com/netflix/priam/config/PriamConfigurationPersisterTest.java b/priam/src/test/java/com/netflix/priam/config/PriamConfigurationPersisterTest.java index 6388d4a6f..515c70d0a 100644 --- a/priam/src/test/java/com/netflix/priam/config/PriamConfigurationPersisterTest.java +++ b/priam/src/test/java/com/netflix/priam/config/PriamConfigurationPersisterTest.java @@ -1,31 +1,27 @@ package com.netflix.priam.config; +import static junit.framework.TestCase.assertTrue; +import static org.junit.Assert.assertEquals; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.inject.Guice; +import com.google.inject.Injector; +import com.netflix.priam.TestModule; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; import java.util.HashMap; import java.util.Map; - import org.junit.After; import org.junit.Before; import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.google.inject.Guice; -import com.google.inject.Injector; -import com.netflix.priam.TestModule; - -import static junit.framework.TestCase.assertTrue; -import static org.junit.Assert.assertEquals; - -public class PriamConfigurationPersisterTest -{ +public class PriamConfigurationPersisterTest { private static PriamConfigurationPersister persister; - @Rule - public TemporaryFolder folder = new TemporaryFolder(); + @Rule public TemporaryFolder folder = new TemporaryFolder(); private FakeConfiguration fakeConfiguration; @@ -35,8 +31,7 @@ public void setUp() { fakeConfiguration = (FakeConfiguration) injector.getInstance(IConfiguration.class); fakeConfiguration.fakeProperties.put("priam_test_config", folder.getRoot().getPath()); - if (persister == null) - persister = injector.getInstance(PriamConfigurationPersister.class); + if (persister == null) persister = injector.getInstance(PriamConfigurationPersister.class); } @After @@ -46,21 +41,22 @@ public void cleanUp() { @Test @SuppressWarnings("unchecked") - public void execute() throws Exception - { + public void execute() throws Exception { Path structuredJson = Paths.get(folder.getRoot().getPath(), "structured.json"); persister.execute(); assertTrue(structuredJson.toFile().exists()); ObjectMapper objectMapper = new ObjectMapper(); - Map myMap = objectMapper.readValue(Files.readAllBytes(structuredJson), HashMap.class); + Map myMap = + objectMapper.readValue(Files.readAllBytes(structuredJson), HashMap.class); assertEquals(myMap.get("backupLocation"), fakeConfiguration.getBackupLocation()); } @Test - public void getTimer() - { - assertEquals("0 * * * * ? *", PriamConfigurationPersister.getTimer(fakeConfiguration).getCronExpression()); + public void getTimer() { + assertEquals( + "0 * * * * ? *", + PriamConfigurationPersister.getTimer(fakeConfiguration).getCronExpression()); } -} \ No newline at end of file +} diff --git a/priam/src/test/java/com/netflix/priam/configSource/AbstractConfigSourceTest.java b/priam/src/test/java/com/netflix/priam/configSource/AbstractConfigSourceTest.java index 536f8c703..6175a284d 100644 --- a/priam/src/test/java/com/netflix/priam/configSource/AbstractConfigSourceTest.java +++ b/priam/src/test/java/com/netflix/priam/configSource/AbstractConfigSourceTest.java @@ -18,20 +18,18 @@ package com.netflix.priam.configSource; import com.google.common.collect.ImmutableList; +import java.util.List; import org.junit.Assert; import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.List; - -public final class AbstractConfigSourceTest -{ - private static final Logger LOGGER = LoggerFactory.getLogger(AbstractConfigSourceTest.class.getName()); +public final class AbstractConfigSourceTest { + private static final Logger LOGGER = + LoggerFactory.getLogger(AbstractConfigSourceTest.class.getName()); @Test - public void lists() - { + public void lists() { AbstractConfigSource source = new MemoryConfigSource(); source.set("foo", "bar,baz, qux "); final List values = source.getList("foo"); @@ -40,8 +38,7 @@ public void lists() } @Test - public void oneItem() - { + public void oneItem() { AbstractConfigSource source = new MemoryConfigSource(); source.set("foo", "bar"); final List values = source.getList("foo"); @@ -50,8 +47,7 @@ public void oneItem() } @Test - public void oneItemWithSpace() - { + public void oneItemWithSpace() { AbstractConfigSource source = new MemoryConfigSource(); source.set("foo", "\tbar "); final List values = source.getList("foo"); diff --git a/priam/src/test/java/com/netflix/priam/configSource/CompositeConfigSourceTest.java b/priam/src/test/java/com/netflix/priam/configSource/CompositeConfigSourceTest.java index b6ec25be9..31b6304f3 100644 --- a/priam/src/test/java/com/netflix/priam/configSource/CompositeConfigSourceTest.java +++ b/priam/src/test/java/com/netflix/priam/configSource/CompositeConfigSourceTest.java @@ -22,13 +22,12 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -public final class CompositeConfigSourceTest -{ - private static final Logger LOGGER = LoggerFactory.getLogger(CompositeConfigSourceTest.class.getName()); +public final class CompositeConfigSourceTest { + private static final Logger LOGGER = + LoggerFactory.getLogger(CompositeConfigSourceTest.class.getName()); @Test - public void read() - { + public void read() { MemoryConfigSource memoryConfigSource = new MemoryConfigSource(); IConfigSource configSource = new CompositeConfigSource(memoryConfigSource); configSource.intialize("foo", "bar"); @@ -44,8 +43,7 @@ public void read() } @Test - public void readMultiple() - { + public void readMultiple() { MemoryConfigSource m1 = new MemoryConfigSource(); m1.set("foo", "foo"); MemoryConfigSource m2 = new MemoryConfigSource(); diff --git a/priam/src/test/java/com/netflix/priam/configSource/PropertiesConfigSourceTest.java b/priam/src/test/java/com/netflix/priam/configSource/PropertiesConfigSourceTest.java index 91ab4a759..5e1b33125 100644 --- a/priam/src/test/java/com/netflix/priam/configSource/PropertiesConfigSourceTest.java +++ b/priam/src/test/java/com/netflix/priam/configSource/PropertiesConfigSourceTest.java @@ -22,30 +22,30 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; - -public final class PropertiesConfigSourceTest -{ - private static final Logger LOGGER = LoggerFactory.getLogger(PropertiesConfigSourceTest.class.getName()); +public final class PropertiesConfigSourceTest { + private static final Logger LOGGER = + LoggerFactory.getLogger(PropertiesConfigSourceTest.class.getName()); @Test - public void readFile() - { + public void readFile() { PropertiesConfigSource configSource = new PropertiesConfigSource("conf/Priam.properties"); configSource.intialize("asgName", "region"); - Assert.assertEquals("\"/tmp/commitlog\"", configSource.get("Priam.backup.commitlog.location")); + Assert.assertEquals( + "\"/tmp/commitlog\"", configSource.get("Priam.backup.commitlog.location")); Assert.assertEquals(7102, configSource.get("Priam.thrift.port", 0)); - // File has 13 lines, but line 6 is "Priam.jmx.port7501", so it gets filtered out with empty string check. + // File has 13 lines, but line 6 is "Priam.jmx.port7501", so it gets filtered out with empty + // string check. Assert.assertEquals(12, configSource.size()); } @Test - public void updateKey() - { + public void updateKey() { PropertiesConfigSource configSource = new PropertiesConfigSource("conf/Priam.properties"); configSource.intialize("asgName", "region"); - // File has 13 lines, but line 6 is "Priam.jmx.port7501", so it gets filtered out with empty string check. + // File has 13 lines, but line 6 is "Priam.jmx.port7501", so it gets filtered out with empty + // string check. Assert.assertEquals(12, configSource.size()); configSource.set("foo", "bar"); diff --git a/priam/src/test/java/com/netflix/priam/configSource/SystemPropertiesConfigSourceTest.java b/priam/src/test/java/com/netflix/priam/configSource/SystemPropertiesConfigSourceTest.java index 77584e437..8af563c4e 100644 --- a/priam/src/test/java/com/netflix/priam/configSource/SystemPropertiesConfigSourceTest.java +++ b/priam/src/test/java/com/netflix/priam/configSource/SystemPropertiesConfigSourceTest.java @@ -22,13 +22,12 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -public final class SystemPropertiesConfigSourceTest -{ - private static final Logger LOGGER = LoggerFactory.getLogger(SystemPropertiesConfigSourceTest.class.getName()); +public final class SystemPropertiesConfigSourceTest { + private static final Logger LOGGER = + LoggerFactory.getLogger(SystemPropertiesConfigSourceTest.class.getName()); @Test - public void read() - { + public void read() { final String key = "java.version"; SystemPropertiesConfigSource configSource = new SystemPropertiesConfigSource(); configSource.intialize("asgName", "region"); diff --git a/priam/src/test/java/com/netflix/priam/defaultimpl/CassandraProcessManagerTest.java b/priam/src/test/java/com/netflix/priam/defaultimpl/CassandraProcessManagerTest.java index a34e5c760..47d10d6f9 100644 --- a/priam/src/test/java/com/netflix/priam/defaultimpl/CassandraProcessManagerTest.java +++ b/priam/src/test/java/com/netflix/priam/defaultimpl/CassandraProcessManagerTest.java @@ -18,54 +18,46 @@ package com.netflix.priam.defaultimpl; import com.google.inject.Guice; +import com.netflix.priam.backup.BRTestModule; import com.netflix.priam.config.FakeConfiguration; import com.netflix.priam.config.IConfiguration; -import com.netflix.priam.backup.BRTestModule; import com.netflix.priam.health.InstanceState; import com.netflix.priam.merics.CassMonitorMetrics; +import java.io.IOException; import org.junit.Assert; import org.junit.Before; import org.junit.Test; -import java.io.IOException; - -public class CassandraProcessManagerTest -{ +public class CassandraProcessManagerTest { CassandraProcessManager cpm; @Before - public void setup() - { - IConfiguration config = new FakeConfiguration("us-east-1", "test_cluster", "us-east-1a", "i-2378afd3"); - InstanceState instanceState = Guice.createInjector(new BRTestModule()).getInstance(InstanceState.class); - CassMonitorMetrics cassMonitorMetrics = Guice.createInjector(new BRTestModule()).getInstance(CassMonitorMetrics.class); + public void setup() { + IConfiguration config = + new FakeConfiguration("us-east-1", "test_cluster", "us-east-1a", "i-2378afd3"); + InstanceState instanceState = + Guice.createInjector(new BRTestModule()).getInstance(InstanceState.class); + CassMonitorMetrics cassMonitorMetrics = + Guice.createInjector(new BRTestModule()).getInstance(CassMonitorMetrics.class); cpm = new CassandraProcessManager(config, instanceState, cassMonitorMetrics); } @Test - public void logProcessOutput_BadApp() throws IOException, InterruptedException - { + public void logProcessOutput_BadApp() throws IOException, InterruptedException { Process p = null; - try - { + try { p = new ProcessBuilder("ls", "/tmppppp").start(); int exitValue = p.waitFor(); Assert.assertTrue(0 != exitValue); cpm.logProcessOutput(p); - } - catch(IOException ioe) - { - if(p!=null) - cpm.logProcessOutput(p); + } catch (IOException ioe) { + if (p != null) cpm.logProcessOutput(p); } } - /** - * note: this will succeed on a *nix machine, unclear about anything else... - */ + /** note: this will succeed on a *nix machine, unclear about anything else... */ @Test - public void logProcessOutput_GoodApp() throws IOException, InterruptedException - { + public void logProcessOutput_GoodApp() throws IOException, InterruptedException { Process p = new ProcessBuilder("true").start(); int exitValue = p.waitFor(); Assert.assertEquals(0, exitValue); diff --git a/priam/src/test/java/com/netflix/priam/defaultimpl/FakeCassandraProcess.java b/priam/src/test/java/com/netflix/priam/defaultimpl/FakeCassandraProcess.java index 12dd386ff..2905a0a8d 100644 --- a/priam/src/test/java/com/netflix/priam/defaultimpl/FakeCassandraProcess.java +++ b/priam/src/test/java/com/netflix/priam/defaultimpl/FakeCassandraProcess.java @@ -2,18 +2,16 @@ import java.io.IOException; -/** - * Created by aagrawal on 10/3/17. - */ +/** Created by aagrawal on 10/3/17. */ public class FakeCassandraProcess implements ICassandraProcess { @Override public void start(boolean join_ring) throws IOException { - //do nothing + // do nothing } @Override public void stop(boolean force) throws IOException { - //do nothing + // do nothing } -} \ No newline at end of file +} diff --git a/priam/src/test/java/com/netflix/priam/health/TestInstanceStatus.java b/priam/src/test/java/com/netflix/priam/health/TestInstanceStatus.java index af9580092..7cac3f82e 100644 --- a/priam/src/test/java/com/netflix/priam/health/TestInstanceStatus.java +++ b/priam/src/test/java/com/netflix/priam/health/TestInstanceStatus.java @@ -8,10 +8,7 @@ import org.junit.Before; import org.junit.Test; -/** - * Test InstanceState - * Created by aagrawal on 9/22/17. - */ +/** Test InstanceState Created by aagrawal on 9/22/17. */ public class TestInstanceStatus { private TestInstanceState testInstanceState; @@ -23,35 +20,84 @@ public void setUp() { } @Test - public void testHealth(){ - //Verify good health. - Assert.assertTrue(testInstanceState.setParams(false, true, true, true, true, true, true, true).isHealthy()); - Assert.assertTrue(testInstanceState.setParams(false,true, true, true, true, false, true, true).isHealthy()); - Assert.assertTrue(testInstanceState.setParams(false,true, true, true, false, true, true, true).isHealthy()); - Assert.assertTrue(testInstanceState.setParams(true,false, true, true, false, true, true, true).isHealthy()); - Assert.assertTrue(testInstanceState.setParams(true,true, false, true, true, true, true, true).isHealthy()); - Assert.assertTrue(testInstanceState.setParams(true,true, true, false, true, true, true, true).isHealthy()); - Assert.assertTrue(testInstanceState.setParams(true,true, true, true, true, true, false, true).isHealthy()); - Assert.assertTrue(testInstanceState.setParams(true,true, true, true, false, false, true, true).isHealthy()); - - //Negative health case scenarios. - Assert.assertFalse(testInstanceState.setParams(false,false, true, true, false, true, true, true).isHealthy()); - Assert.assertFalse(testInstanceState.setParams(false,true, false, true, true, true, true, true).isHealthy()); - Assert.assertFalse(testInstanceState.setParams(false,true, true, false, true, true, true, true).isHealthy()); - Assert.assertFalse(testInstanceState.setParams(false,true, true, true, true, true, false, true).isHealthy()); - Assert.assertFalse(testInstanceState.setParams(false,true, true, true, false, false, true, true).isHealthy()); - Assert.assertFalse(testInstanceState.setParams(false,true, true, true, false, false, true, false).isHealthy()); + public void testHealth() { + // Verify good health. + Assert.assertTrue( + testInstanceState + .setParams(false, true, true, true, true, true, true, true) + .isHealthy()); + Assert.assertTrue( + testInstanceState + .setParams(false, true, true, true, true, false, true, true) + .isHealthy()); + Assert.assertTrue( + testInstanceState + .setParams(false, true, true, true, false, true, true, true) + .isHealthy()); + Assert.assertTrue( + testInstanceState + .setParams(true, false, true, true, false, true, true, true) + .isHealthy()); + Assert.assertTrue( + testInstanceState + .setParams(true, true, false, true, true, true, true, true) + .isHealthy()); + Assert.assertTrue( + testInstanceState + .setParams(true, true, true, false, true, true, true, true) + .isHealthy()); + Assert.assertTrue( + testInstanceState + .setParams(true, true, true, true, true, true, false, true) + .isHealthy()); + Assert.assertTrue( + testInstanceState + .setParams(true, true, true, true, false, false, true, true) + .isHealthy()); + // Negative health case scenarios. + Assert.assertFalse( + testInstanceState + .setParams(false, false, true, true, false, true, true, true) + .isHealthy()); + Assert.assertFalse( + testInstanceState + .setParams(false, true, false, true, true, true, true, true) + .isHealthy()); + Assert.assertFalse( + testInstanceState + .setParams(false, true, true, false, true, true, true, true) + .isHealthy()); + Assert.assertFalse( + testInstanceState + .setParams(false, true, true, true, true, true, false, true) + .isHealthy()); + Assert.assertFalse( + testInstanceState + .setParams(false, true, true, true, false, false, true, true) + .isHealthy()); + Assert.assertFalse( + testInstanceState + .setParams(false, true, true, true, false, false, true, false) + .isHealthy()); } - private class TestInstanceState{ + private class TestInstanceState { private InstanceState instanceState; - TestInstanceState(InstanceState instanceState1){ + TestInstanceState(InstanceState instanceState1) { this.instanceState = instanceState1; } - InstanceState setParams(boolean isRestoring, boolean isYmlWritten, boolean isCassandraProcessAlive, boolean isGossipEnabled, boolean isThriftEnabled, boolean isNativeEnabled, boolean isRequiredDirectoriesExist, boolean shouldCassandraBeAlive){ + InstanceState setParams( + boolean isRestoring, + boolean isYmlWritten, + boolean isCassandraProcessAlive, + boolean isGossipEnabled, + boolean isThriftEnabled, + boolean isNativeEnabled, + boolean isRequiredDirectoriesExist, + boolean shouldCassandraBeAlive) { instanceState.setYmlWritten(isYmlWritten); instanceState.setCassandraProcessAlive(isCassandraProcessAlive); instanceState.setIsNativeTransportActive(isNativeEnabled); @@ -60,10 +106,8 @@ InstanceState setParams(boolean isRestoring, boolean isYmlWritten, boolean isCas instanceState.setIsRequiredDirectoriesExist(isRequiredDirectoriesExist); instanceState.setShouldCassandraBeAlive(shouldCassandraBeAlive); - if (isRestoring) - instanceState.setRestoreStatus(Status.STARTED); - else - instanceState.setRestoreStatus(Status.FINISHED); + if (isRestoring) instanceState.setRestoreStatus(Status.STARTED); + else instanceState.setRestoreStatus(Status.FINISHED); return instanceState; } diff --git a/priam/src/test/java/com/netflix/priam/identity/FakeMembership.java b/priam/src/test/java/com/netflix/priam/identity/FakeMembership.java index bcdd948fc..e3185c914 100644 --- a/priam/src/test/java/com/netflix/priam/identity/FakeMembership.java +++ b/priam/src/test/java/com/netflix/priam/identity/FakeMembership.java @@ -20,73 +20,59 @@ import java.util.Collection; import java.util.List; -import com.netflix.priam.identity.IMembership; - -public class FakeMembership implements IMembership -{ +public class FakeMembership implements IMembership { private List instances; - public FakeMembership(List priamInstances) - { + public FakeMembership(List priamInstances) { this.instances = priamInstances; } - - public void setInstances( List priamInstances) - { + + public void setInstances(List priamInstances) { this.instances = priamInstances; } @Override - public List getRacMembership() - { + public List getRacMembership() { return instances; } - + @Override - public List getCrossAccountRacMembership() - { - return null; + public List getCrossAccountRacMembership() { + return null; } - @Override - public int getRacMembershipSize() - { + public int getRacMembershipSize() { return 3; } @Override - public int getRacCount() - { + public int getRacCount() { return 3; } @Override - public void addACL(Collection listIPs, int from, int to) - { + public void addACL(Collection listIPs, int from, int to) { // TODO Auto-generated method stub - + } @Override - public void removeACL(Collection listIPs, int from, int to) - { + public void removeACL(Collection listIPs, int from, int to) { // TODO Auto-generated method stub - + } @Override - public List listACL(int from, int to) - { + public List listACL(int from, int to) { // TODO Auto-generated method stub return null; } @Override - public void expandRacMembership(int count) - { + public void expandRacMembership(int count) { // TODO Auto-generated method stub - + } } diff --git a/priam/src/test/java/com/netflix/priam/identity/FakePriamInstanceFactory.java b/priam/src/test/java/com/netflix/priam/identity/FakePriamInstanceFactory.java index 0a94a3f2e..e791b6a4a 100644 --- a/priam/src/test/java/com/netflix/priam/identity/FakePriamInstanceFactory.java +++ b/priam/src/test/java/com/netflix/priam/identity/FakePriamInstanceFactory.java @@ -20,38 +20,39 @@ import com.google.common.collect.Maps; import com.google.inject.Inject; import com.netflix.priam.config.IConfiguration; -import com.netflix.priam.identity.IPriamInstanceFactory; -import com.netflix.priam.identity.PriamInstance; - import java.util.*; -public class FakePriamInstanceFactory implements IPriamInstanceFactory -{ - private final Map instances = Maps.newHashMap(); +public class FakePriamInstanceFactory implements IPriamInstanceFactory { + private final Map instances = Maps.newHashMap(); private final IConfiguration config; @Inject - public FakePriamInstanceFactory(IConfiguration config) - { + public FakePriamInstanceFactory(IConfiguration config) { this.config = config; } @Override - public List getAllIds(String appName) - { + public List getAllIds(String appName) { List result = new ArrayList<>(instances.values()); sort(result); return result; } - + @Override public PriamInstance getInstance(String appName, String dc, int id) { - return instances.get(id); + return instances.get(id); } @Override - public PriamInstance create(String app, int id, String instanceID, String hostname, String ip, String rac, Map volumes, String payload) - { + public PriamInstance create( + String app, + int id, + String instanceID, + String hostname, + String ip, + String rac, + Map volumes, + String payload) { PriamInstance ins = new PriamInstance(); ins.setApp(app); ins.setRac(rac); @@ -66,40 +67,32 @@ public PriamInstance create(String app, int id, String instanceID, String hostna } @Override - public void delete(PriamInstance inst) - { + public void delete(PriamInstance inst) { instances.remove(inst.getId()); } @Override - public void update(PriamInstance inst) - { + public void update(PriamInstance inst) { instances.put(inst.getId(), inst); } - @Override - public void sort(List return_) - { - Comparator comparator = new Comparator() - { + public void sort(List return_) { + Comparator comparator = + new Comparator() { - @Override - public int compare(PriamInstance o1, PriamInstance o2) - { - Integer c1 = o1.getId(); - Integer c2 = o2.getId(); - return c1.compareTo(c2); - } - }; + @Override + public int compare(PriamInstance o1, PriamInstance o2) { + Integer c1 = o1.getId(); + Integer c2 = o2.getId(); + return c1.compareTo(c2); + } + }; Collections.sort(return_, comparator); } @Override - public void attachVolumes(PriamInstance instance, String mountPath, String device) - { + public void attachVolumes(PriamInstance instance, String mountPath, String device) { // TODO Auto-generated method stub } - - } diff --git a/priam/src/test/java/com/netflix/priam/resources/BackupServletTest.java b/priam/src/test/java/com/netflix/priam/resources/BackupServletTest.java index 6277929a3..54be67f9b 100644 --- a/priam/src/test/java/com/netflix/priam/resources/BackupServletTest.java +++ b/priam/src/test/java/com/netflix/priam/resources/BackupServletTest.java @@ -17,15 +17,17 @@ package com.netflix.priam.resources; +import static org.junit.Assert.assertEquals; + import com.google.common.collect.ImmutableList; import com.google.common.collect.Lists; import com.google.inject.Guice; import com.google.inject.Injector; import com.google.inject.Provider; -import com.netflix.priam.defaultimpl.ICassandraProcess; -import com.netflix.priam.config.IConfiguration; import com.netflix.priam.PriamServer; import com.netflix.priam.backup.*; +import com.netflix.priam.config.IConfiguration; +import com.netflix.priam.defaultimpl.ICassandraProcess; import com.netflix.priam.health.InstanceState; import com.netflix.priam.identity.IPriamInstanceFactory; import com.netflix.priam.identity.InstanceIdentity; @@ -34,6 +36,10 @@ import com.netflix.priam.tuner.ICassandraTuner; import com.netflix.priam.utils.ITokenManager; import com.netflix.priam.utils.TokenManager; +import java.util.Date; +import java.util.List; +import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response; import mockit.Expectations; import mockit.Mocked; import mockit.integration.junit4.JMockit; @@ -42,30 +48,14 @@ import org.junit.Test; import org.junit.runner.RunWith; -import javax.ws.rs.core.MediaType; -import javax.ws.rs.core.Response; -import java.util.Date; -import java.util.List; - -import static org.junit.Assert.assertEquals; - @RunWith(JMockit.class) -public class BackupServletTest -{ +public class BackupServletTest { private @Mocked PriamServer priamServer; private IConfiguration config; - private - @Mocked - IBackupFileSystem bkpFs; - private - @Mocked - Restore restoreObj; - private - @Mocked - Provider pathProvider; - private - @Mocked - ICassandraTuner tuner; + private @Mocked IBackupFileSystem bkpFs; + private @Mocked Restore restoreObj; + private @Mocked Provider pathProvider; + private @Mocked ICassandraTuner tuner; private @Mocked SnapshotBackup snapshotBackup; private @Mocked IPriamInstanceFactory factory; private @Mocked ICassandraProcess cassProcess; @@ -75,35 +65,57 @@ public class BackupServletTest private BackupVerification backupVerification; @Before - public void setUp() - { + public void setUp() { Injector injector = Guice.createInjector(new BRTestModule()); config = injector.getInstance(IConfiguration.class); InstanceState instanceState = injector.getInstance(InstanceState.class); ITokenManager tokenManager = new TokenManager(config); - resource = new BackupServlet(priamServer, config, bkpFs, restoreObj, pathProvider, - tuner, snapshotBackup, factory, tokenManager, cassProcess, bkupStatusMgr, backupVerification); - restoreResource = new RestoreServlet(config, restoreObj, pathProvider,priamServer, factory, tuner, cassProcess - , tokenManager, instanceState); + resource = + new BackupServlet( + priamServer, + config, + bkpFs, + restoreObj, + pathProvider, + tuner, + snapshotBackup, + factory, + tokenManager, + cassProcess, + bkupStatusMgr, + backupVerification); + restoreResource = + new RestoreServlet( + config, + restoreObj, + pathProvider, + priamServer, + factory, + tuner, + cassProcess, + tokenManager, + instanceState); } @Test - public void backup() throws Exception - { - new Expectations() {{ - snapshotBackup.execute(); - }}; + public void backup() throws Exception { + new Expectations() { + { + snapshotBackup.execute(); + } + }; Response response = resource.backup(); assertEquals(200, response.getStatus()); assertEquals("[\"ok\"]", response.getEntity()); - assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getMetadata().get("Content-Type").get(0)); + assertEquals( + MediaType.APPLICATION_JSON_TYPE, response.getMetadata().get("Content-Type").get(0)); } @Test - public void restore_minimal(@Mocked final InstanceIdentity identity, - @Mocked final PriamInstance instance) throws Exception - { + public void restore_minimal( + @Mocked final InstanceIdentity identity, @Mocked final PriamInstance instance) + throws Exception { final String dateRange = null; final String newRegion = null; final String newToken = null; @@ -114,18 +126,25 @@ public void restore_minimal(@Mocked final InstanceIdentity identity, new Expectations() { { - priamServer.getId(); result = identity; times = 2; + priamServer.getId(); + result = identity; + times = 2; } }; new Expectations() { { - config.getDC(); result = oldRegion; - identity.getInstance(); result = instance; times = 2; - instance.getToken(); result = oldToken; + config.getDC(); + result = oldRegion; + identity.getInstance(); + result = instance; + times = 2; + instance.getToken(); + result = oldToken; - config.isRestoreClosestToken(); result = false; + config.isRestoreClosestToken(); + result = false; restoreObj.restore((Date) any, (Date) any); // TODO: test default value @@ -137,16 +156,20 @@ public void restore_minimal(@Mocked final InstanceIdentity identity, expectCassandraStartup(); - Response response = restoreResource.restore(dateRange, newRegion, newToken, keyspaces, null); + Response response = + restoreResource.restore(dateRange, newRegion, newToken, keyspaces, null); assertEquals(200, response.getStatus()); assertEquals("[\"ok\"]", response.getEntity()); - assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getMetadata().get("Content-Type").get(0)); + assertEquals( + MediaType.APPLICATION_JSON_TYPE, response.getMetadata().get("Content-Type").get(0)); } @Test - public void restore_withDateRange(@Mocked final InstanceIdentity identity, - @Mocked final PriamInstance instance, @Mocked final AbstractBackupPath backupPath) throws Exception - { + public void restore_withDateRange( + @Mocked final InstanceIdentity identity, + @Mocked final PriamInstance instance, + @Mocked final AbstractBackupPath backupPath) + throws Exception { final String dateRange = "201101010000,20111231259"; final String newRegion = null; final String newToken = null; @@ -157,27 +180,37 @@ public void restore_withDateRange(@Mocked final InstanceIdentity identity, new Expectations() { { - priamServer.getId(); result = identity; times = 2; + priamServer.getId(); + result = identity; + times = 2; } }; new Expectations() { { - pathProvider.get(); result = backupPath; - backupPath.parseDate(dateRange.split(",")[0]); result = new DateTime(2011, 01, 01, 00, 00).toDate(); times = 1; - backupPath.parseDate(dateRange.split(",")[1]); result = new DateTime(2011, 12, 31, 23, 59).toDate(); times = 1; - -// config.getDC(); result = oldRegion; - identity.getInstance(); result = instance; times = 2; - instance.getToken(); result = oldToken; + pathProvider.get(); + result = backupPath; + backupPath.parseDate(dateRange.split(",")[0]); + result = new DateTime(2011, 01, 01, 00, 00).toDate(); + times = 1; + backupPath.parseDate(dateRange.split(",")[1]); + result = new DateTime(2011, 12, 31, 23, 59).toDate(); + times = 1; + + // config.getDC(); result = oldRegion; + identity.getInstance(); + result = instance; + times = 2; + instance.getToken(); + result = oldToken; - // config.isRestoreClosestToken(); result = false; + // config.isRestoreClosestToken(); result = false; restoreObj.restore( - new DateTime(2011, 01, 01, 00, 00).toDate(), - new DateTime(2011, 12, 31, 23, 59).toDate()); + new DateTime(2011, 01, 01, 00, 00).toDate(), + new DateTime(2011, 12, 31, 23, 59).toDate()); - // config.setDC(oldRegion); + // config.setDC(oldRegion); instance.setToken(oldToken); tuner.updateAutoBootstrap(config.getYamlLocation(), false); } @@ -185,71 +218,75 @@ public void restore_withDateRange(@Mocked final InstanceIdentity identity, expectCassandraStartup(); - Response response = restoreResource.restore(dateRange, newRegion, newToken, keyspaces, null); + Response response = + restoreResource.restore(dateRange, newRegion, newToken, keyspaces, null); assertEquals(200, response.getStatus()); assertEquals("[\"ok\"]", response.getEntity()); - assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getMetadata().get("Content-Type").get(0)); + assertEquals( + MediaType.APPLICATION_JSON_TYPE, response.getMetadata().get("Content-Type").get(0)); } -// @Test -// public void restore_withRegion() throws Exception -// { -// final String dateRange = null; -// final String newRegion = "us-west-1"; -// final String newToken = null; -// final String keyspaces = null; -// -// final String oldRegion = "us-east-1"; -// final String oldToken = "1234"; -// final String appName = "myApp"; -// -// new Expectations() { -// @NonStrict InstanceIdentity identity; -// PriamInstance instance; -// @NonStrict PriamInstance instance1, instance2, instance3; -// -// { -// config.getDC(); result = oldRegion; -// priamServer.getId(); result = identity; times = 3; -// identity.getInstance(); result = instance; times = 3; -// instance.getToken(); result = oldToken; -// -// config.isRestoreClosestToken(); result = false; -// -// config.setDC(newRegion); -// instance.getToken(); result = oldToken; -// config.getAppName(); result = appName; -// factory.getAllIds(appName); result = ImmutableList.of(instance, instance1, instance2, instance3); -// instance.getDC(); result = oldRegion; -// instance.getToken(); result = oldToken; -// instance1.getDC(); result = oldRegion; -// instance2.getDC(); result = oldRegion; -// instance3.getDC(); result = oldRegion; -// instance1.getToken(); result = "1234"; -// instance2.getToken(); result = "5678"; -// instance3.getToken(); result = "9000"; -// instance.setToken((String) any); // TODO: test mocked closest token -// -// restoreObj.restore((Date) any, (Date) any); // TODO: test default value -// -// config.setDC(oldRegion); -// instance.setToken(oldToken); -// tuneCassandra.writeAllProperties(false); -// } -// }; -// -// expectCassandraStartup(); -// -// Response response = resource.restore(dateRange, newRegion, newToken, keyspaces); -// assertEquals(200, response.getStatus()); -// assertEquals("[\"ok\"]", response.getEntity()); -// assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getMetadata().get("Content-Type").get(0)); -// } + // @Test + // public void restore_withRegion() throws Exception + // { + // final String dateRange = null; + // final String newRegion = "us-west-1"; + // final String newToken = null; + // final String keyspaces = null; + // + // final String oldRegion = "us-east-1"; + // final String oldToken = "1234"; + // final String appName = "myApp"; + // + // new Expectations() { + // @NonStrict InstanceIdentity identity; + // PriamInstance instance; + // @NonStrict PriamInstance instance1, instance2, instance3; + // + // { + // config.getDC(); result = oldRegion; + // priamServer.getId(); result = identity; times = 3; + // identity.getInstance(); result = instance; times = 3; + // instance.getToken(); result = oldToken; + // + // config.isRestoreClosestToken(); result = false; + // + // config.setDC(newRegion); + // instance.getToken(); result = oldToken; + // config.getAppName(); result = appName; + // factory.getAllIds(appName); result = ImmutableList.of(instance, instance1, + // instance2, instance3); + // instance.getDC(); result = oldRegion; + // instance.getToken(); result = oldToken; + // instance1.getDC(); result = oldRegion; + // instance2.getDC(); result = oldRegion; + // instance3.getDC(); result = oldRegion; + // instance1.getToken(); result = "1234"; + // instance2.getToken(); result = "5678"; + // instance3.getToken(); result = "9000"; + // instance.setToken((String) any); // TODO: test mocked closest token + // + // restoreObj.restore((Date) any, (Date) any); // TODO: test default value + // + // config.setDC(oldRegion); + // instance.setToken(oldToken); + // tuneCassandra.writeAllProperties(false); + // } + // }; + // + // expectCassandraStartup(); + // + // Response response = resource.restore(dateRange, newRegion, newToken, keyspaces); + // assertEquals(200, response.getStatus()); + // assertEquals("[\"ok\"]", response.getEntity()); + // assertEquals(MediaType.APPLICATION_JSON_TYPE, + // response.getMetadata().get("Content-Type").get(0)); + // } @Test - public void restore_withToken(@Mocked final InstanceIdentity identity, - @Mocked final PriamInstance instance) throws Exception - { + public void restore_withToken( + @Mocked final InstanceIdentity identity, @Mocked final PriamInstance instance) + throws Exception { final String dateRange = null; final String newRegion = null; final String newToken = "myNewToken"; @@ -260,18 +297,24 @@ public void restore_withToken(@Mocked final InstanceIdentity identity, new Expectations() { { - priamServer.getId(); result = identity; times = 3; + priamServer.getId(); + result = identity; + times = 3; } }; new Expectations() { { - config.getDC(); result = oldRegion; - identity.getInstance(); result = instance; times = 3; - instance.getToken(); result = oldToken; + config.getDC(); + result = oldRegion; + identity.getInstance(); + result = instance; + times = 3; + instance.getToken(); + result = oldToken; instance.setToken(newToken); - //config.isRestoreClosestToken(); result = false; + // config.isRestoreClosestToken(); result = false; restoreObj.restore((Date) any, (Date) any); // TODO: test default value @@ -283,16 +326,18 @@ public void restore_withToken(@Mocked final InstanceIdentity identity, expectCassandraStartup(); - Response response = restoreResource.restore(dateRange, newRegion, newToken, keyspaces, null); + Response response = + restoreResource.restore(dateRange, newRegion, newToken, keyspaces, null); assertEquals(200, response.getStatus()); assertEquals("[\"ok\"]", response.getEntity()); - assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getMetadata().get("Content-Type").get(0)); + assertEquals( + MediaType.APPLICATION_JSON_TYPE, response.getMetadata().get("Content-Type").get(0)); } @Test - public void restore_withKeyspaces(@Mocked final InstanceIdentity identity, - @Mocked final PriamInstance instance) throws Exception - { + public void restore_withKeyspaces( + @Mocked final InstanceIdentity identity, @Mocked final PriamInstance instance) + throws Exception { final String dateRange = null; final String newRegion = null; final String newToken = null; @@ -301,14 +346,16 @@ public void restore_withKeyspaces(@Mocked final InstanceIdentity identity, final String oldRegion = "us-east-1"; final String oldToken = "1234"; - new Expectations() { + new Expectations() { { - config.getDC(); result = oldRegion; - config.isRestoreClosestToken(); result = false; + config.getDC(); + result = oldRegion; + config.isRestoreClosestToken(); + result = false; - List restoreKeyspaces = Lists.newArrayList(); - restoreKeyspaces.clear(); - restoreKeyspaces.addAll(ImmutableList.of("keyspace1", "keyspace2")); + List restoreKeyspaces = Lists.newArrayList(); + restoreKeyspaces.clear(); + restoreKeyspaces.addAll(ImmutableList.of("keyspace1", "keyspace2")); result = restoreKeyspaces; config.setDC(oldRegion); @@ -320,8 +367,11 @@ public void restore_withKeyspaces(@Mocked final InstanceIdentity identity, new Expectations() { { - identity.getInstance(); result = instance; times = 2; - instance.getToken(); result = oldToken; + identity.getInstance(); + result = instance; + times = 2; + instance.getToken(); + result = oldToken; restoreObj.restore((Date) any, (Date) any); // TODO: test default value @@ -332,19 +382,24 @@ public void restore_withKeyspaces(@Mocked final InstanceIdentity identity, expectCassandraStartup(); - Response response = restoreResource.restore(dateRange, newRegion, newToken, keyspaces, null); + Response response = + restoreResource.restore(dateRange, newRegion, newToken, keyspaces, null); assertEquals(200, response.getStatus()); assertEquals("[\"ok\"]", response.getEntity()); - assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getMetadata().get("Content-Type").get(0)); + assertEquals( + MediaType.APPLICATION_JSON_TYPE, response.getMetadata().get("Content-Type").get(0)); } // TODO: this should also set/test newRegion and keyspaces @Test - public void restore_maximal(@Mocked final InstanceIdentity identity, - @Mocked final PriamInstance instance, @Mocked final PriamInstance instance1, - @Mocked final PriamInstance instance2, @Mocked final PriamInstance instance3, - @Mocked final AbstractBackupPath backupPath) throws Exception - { + public void restore_maximal( + @Mocked final InstanceIdentity identity, + @Mocked final PriamInstance instance, + @Mocked final PriamInstance instance1, + @Mocked final PriamInstance instance2, + @Mocked final PriamInstance instance3, + @Mocked final AbstractBackupPath backupPath) + throws Exception { final String dateRange = "201101010000,20111231259"; final String newRegion = null; final String newToken = "5678"; @@ -366,55 +421,75 @@ public void restore_maximal(@Mocked final InstanceIdentity identity, new Expectations() { { - pathProvider.get(); result = backupPath; - backupPath.parseDate(dateRange.split(",")[0]); result = new DateTime(2011, 01, 01, 00, 00).toDate(); times = 1; - backupPath.parseDate(dateRange.split(",")[1]); result = new DateTime(2011, 12, 31, 23, 59).toDate(); times = 1; - -// identity.getInstance(); result = instance; times = 5; -// instance.getToken(); result = oldToken; -// instance.setToken(newToken); -// -// instance.getToken(); result = oldToken; -// factory.getAllIds(appName); result = ImmutableList.of(instance, instance1, instance2, instance3); -// instance.getDC(); result = oldRegion; -// instance.getToken(); result = oldToken; -// instance1.getDC(); result = oldRegion; -// instance2.getDC(); result = oldRegion; -// instance3.getDC(); result = oldRegion; -// instance1.getToken(); result = "1234"; -// instance2.getToken(); result = "5678"; -// instance3.getToken(); result = "9000"; -// instance.setToken((String) any); // TODO: test mocked closest token + pathProvider.get(); + result = backupPath; + backupPath.parseDate(dateRange.split(",")[0]); + result = new DateTime(2011, 01, 01, 00, 00).toDate(); + times = 1; + backupPath.parseDate(dateRange.split(",")[1]); + result = new DateTime(2011, 12, 31, 23, 59).toDate(); + times = 1; + + // identity.getInstance(); result = instance; times = 5; + // instance.getToken(); result = oldToken; + // instance.setToken(newToken); + // + // instance.getToken(); result = oldToken; + // factory.getAllIds(appName); result = ImmutableList.of(instance, + // instance1, instance2, instance3); + // instance.getDC(); result = oldRegion; + // instance.getToken(); result = oldToken; + // instance1.getDC(); result = oldRegion; + // instance2.getDC(); result = oldRegion; + // instance3.getDC(); result = oldRegion; + // instance1.getToken(); result = "1234"; + // instance2.getToken(); result = "5678"; + // instance3.getToken(); result = "9000"; + // instance.setToken((String) any); // TODO: test mocked closest + // token restoreObj.restore( - new DateTime(2011, 01, 01, 00, 00).toDate(), - new DateTime(2011, 12, 31, 23, 59).toDate()); + new DateTime(2011, 01, 01, 00, 00).toDate(), + new DateTime(2011, 12, 31, 23, 59).toDate()); - // instance.setToken(oldToken); + // instance.setToken(oldToken); tuner.updateAutoBootstrap(config.getYamlLocation(), false); } }; expectCassandraStartup(); - Response response = restoreResource.restore(dateRange, newRegion, newToken, keyspaces, null); + Response response = + restoreResource.restore(dateRange, newRegion, newToken, keyspaces, null); assertEquals(200, response.getStatus()); assertEquals("[\"ok\"]", response.getEntity()); - assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getMetadata().get("Content-Type").get(0)); + assertEquals( + MediaType.APPLICATION_JSON_TYPE, response.getMetadata().get("Content-Type").get(0)); } // TODO: create CassandraController interface and inject, instead of static util method private Expectations expectCassandraStartup() { - return new Expectations() {{ - config.getCassStartupScript(); result = "/usr/bin/false"; - config.getHeapNewSize(); result = "2G"; - config.getHeapSize(); result = "8G"; - config.getDataFileLocation(); result = "/var/lib/cassandra/data"; - config.getCommitLogLocation(); result = "/var/lib/cassandra/commitlog"; - config.getBackupLocation(); result = "backup"; - config.getCacheLocation(); result = "/var/lib/cassandra/saved_caches"; - config.getJmxPort(); result = 7199; - config.getMaxDirectMemory(); result = "50G"; - }}; + return new Expectations() { + { + config.getCassStartupScript(); + result = "/usr/bin/false"; + config.getHeapNewSize(); + result = "2G"; + config.getHeapSize(); + result = "8G"; + config.getDataFileLocation(); + result = "/var/lib/cassandra/data"; + config.getCommitLogLocation(); + result = "/var/lib/cassandra/commitlog"; + config.getBackupLocation(); + result = "backup"; + config.getCacheLocation(); + result = "/var/lib/cassandra/saved_caches"; + config.getJmxPort(); + result = 7199; + config.getMaxDirectMemory(); + result = "50G"; + } + }; } } diff --git a/priam/src/test/java/com/netflix/priam/resources/CassandraConfigTest.java b/priam/src/test/java/com/netflix/priam/resources/CassandraConfigTest.java index e0521018c..aa9b262ea 100644 --- a/priam/src/test/java/com/netflix/priam/resources/CassandraConfigTest.java +++ b/priam/src/test/java/com/netflix/priam/resources/CassandraConfigTest.java @@ -17,11 +17,18 @@ package com.netflix.priam.resources; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; + import com.google.common.collect.ImmutableList; import com.netflix.priam.PriamServer; import com.netflix.priam.identity.DoubleRing; import com.netflix.priam.identity.InstanceIdentity; import com.netflix.priam.identity.PriamInstance; +import java.io.IOException; +import java.net.UnknownHostException; +import java.util.List; +import javax.ws.rs.core.Response; import mockit.Expectations; import mockit.Mocked; import mockit.integration.junit4.JMockit; @@ -29,35 +36,28 @@ import org.junit.Test; import org.junit.runner.RunWith; -import javax.ws.rs.core.Response; -import java.io.IOException; -import java.net.UnknownHostException; -import java.util.List; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.fail; - @RunWith(JMockit.class) -public class CassandraConfigTest -{ +public class CassandraConfigTest { private @Mocked PriamServer priamServer; private @Mocked DoubleRing doubleRing; private CassandraConfig resource; @Before - public void setUp() - { + public void setUp() { resource = new CassandraConfig(priamServer, doubleRing); } @Test - public void getSeeds(@Mocked final InstanceIdentity identity) throws Exception - { + public void getSeeds(@Mocked final InstanceIdentity identity) throws Exception { final List seeds = ImmutableList.of("seed1", "seed2", "seed3"); new Expectations() { { - priamServer.getId(); result = identity; times = 1; - identity.getSeeds(); result = seeds; times = 1; + priamServer.getId(); + result = identity; + times = 1; + identity.getSeeds(); + result = seeds; + times = 1; } }; @@ -67,13 +67,16 @@ public void getSeeds(@Mocked final InstanceIdentity identity) throws Exception } @Test - public void getSeeds_notFound(@Mocked final InstanceIdentity identity) throws Exception - { + public void getSeeds_notFound(@Mocked final InstanceIdentity identity) throws Exception { final List seeds = ImmutableList.of(); new Expectations() { { - priamServer.getId(); result = identity; times = 1; - identity.getSeeds(); result = seeds; times = 1; + priamServer.getId(); + result = identity; + times = 1; + identity.getSeeds(); + result = seeds; + times = 1; } }; @@ -82,12 +85,14 @@ public void getSeeds_notFound(@Mocked final InstanceIdentity identity) throws Ex } @Test - public void getSeeds_handlesUnknownHostException(@Mocked final InstanceIdentity identity) throws Exception - { + public void getSeeds_handlesUnknownHostException(@Mocked final InstanceIdentity identity) + throws Exception { new Expectations() { { - priamServer.getId(); result = identity; - identity.getSeeds(); result = new UnknownHostException(); + priamServer.getId(); + result = identity; + identity.getSeeds(); + result = new UnknownHostException(); } }; @@ -96,14 +101,20 @@ public void getSeeds_handlesUnknownHostException(@Mocked final InstanceIdentity } @Test - public void getToken(@Mocked final InstanceIdentity identity, @Mocked final PriamInstance instance) - { + public void getToken( + @Mocked final InstanceIdentity identity, @Mocked final PriamInstance instance) { final String token = "myToken"; new Expectations() { { - priamServer.getId(); result = identity; times = 2; - identity.getInstance(); result = instance; times = 2; - instance.getToken(); result = token; times = 2; + priamServer.getId(); + result = identity; + times = 2; + identity.getInstance(); + result = instance; + times = 2; + instance.getToken(); + result = token; + times = 2; } }; @@ -113,14 +124,17 @@ public void getToken(@Mocked final InstanceIdentity identity, @Mocked final Pria } @Test - public void getToken_notFound(@Mocked final InstanceIdentity identity, @Mocked final PriamInstance instance) - { + public void getToken_notFound( + @Mocked final InstanceIdentity identity, @Mocked final PriamInstance instance) { final String token = ""; new Expectations() { { - priamServer.getId(); result = identity; - identity.getInstance(); result = instance; - instance.getToken(); result = token; + priamServer.getId(); + result = identity; + identity.getInstance(); + result = instance; + instance.getToken(); + result = token; } }; @@ -129,13 +143,16 @@ public void getToken_notFound(@Mocked final InstanceIdentity identity, @Mocked f } @Test - public void getToken_handlesException(@Mocked final InstanceIdentity identity, @Mocked final PriamInstance instance) - { + public void getToken_handlesException( + @Mocked final InstanceIdentity identity, @Mocked final PriamInstance instance) { new Expectations() { { - priamServer.getId(); result = identity; - identity.getInstance(); result = instance; - instance.getToken(); result = new RuntimeException(); + priamServer.getId(); + result = identity; + identity.getInstance(); + result = instance; + instance.getToken(); + result = new RuntimeException(); } }; @@ -144,12 +161,13 @@ public void getToken_handlesException(@Mocked final InstanceIdentity identity, @ } @Test - public void isReplaceToken(@Mocked final InstanceIdentity identity) - { + public void isReplaceToken(@Mocked final InstanceIdentity identity) { new Expectations() { { - priamServer.getId(); result = identity; - identity.isReplace(); result = true; + priamServer.getId(); + result = identity; + identity.isReplace(); + result = true; } }; @@ -159,12 +177,13 @@ public void isReplaceToken(@Mocked final InstanceIdentity identity) } @Test - public void isReplaceToken_handlesException(@Mocked final InstanceIdentity identity) - { + public void isReplaceToken_handlesException(@Mocked final InstanceIdentity identity) { new Expectations() { { - priamServer.getId(); result = identity; - identity.isReplace(); result = new RuntimeException(); + priamServer.getId(); + result = identity; + identity.isReplace(); + result = new RuntimeException(); } }; @@ -173,13 +192,14 @@ public void isReplaceToken_handlesException(@Mocked final InstanceIdentity ident } @Test - public void getReplacedAddress(@Mocked final InstanceIdentity identity) - { - final String replacedIp = "127.0.0.1"; + public void getReplacedAddress(@Mocked final InstanceIdentity identity) { + final String replacedIp = "127.0.0.1"; new Expectations() { { - priamServer.getId(); result = identity; - identity.getReplacedIp(); result = replacedIp; + priamServer.getId(); + result = identity; + identity.getReplacedIp(); + result = replacedIp; } }; @@ -187,58 +207,64 @@ public void getReplacedAddress(@Mocked final InstanceIdentity identity) assertEquals(200, response.getStatus()); assertEquals(replacedIp, response.getEntity()); } - + @Test - public void doubleRing() throws Exception - { - new Expectations() {{ - doubleRing.backup(); - doubleRing.doubleSlots(); - }}; + public void doubleRing() throws Exception { + new Expectations() { + { + doubleRing.backup(); + doubleRing.doubleSlots(); + } + }; Response response = resource.doubleRing(); assertEquals(200, response.getStatus()); } @Test - public void doubleRing_ioExceptionInBackup() throws Exception - { + public void doubleRing_ioExceptionInBackup() throws Exception { final IOException exception = new IOException(); - new Expectations() {{ - doubleRing.backup(); result = exception; - doubleRing.restore(); - }}; - - try - { - resource.doubleRing(); - fail("Excepted RuntimeException"); - } - catch (RuntimeException e) - { - assertEquals(exception, e.getCause()); + new Expectations() { + { + doubleRing.backup(); + result = exception; + doubleRing.restore(); + } + }; + + try { + resource.doubleRing(); + fail("Excepted RuntimeException"); + } catch (RuntimeException e) { + assertEquals(exception, e.getCause()); } } - @Test(expected=IOException.class) - public void doubleRing_ioExceptionInRestore() throws Exception - { - new Expectations() {{ - doubleRing.backup(); result = new IOException(); - doubleRing.restore(); result = new IOException(); - }}; + @Test(expected = IOException.class) + public void doubleRing_ioExceptionInRestore() throws Exception { + new Expectations() { + { + doubleRing.backup(); + result = new IOException(); + doubleRing.restore(); + result = new IOException(); + } + }; resource.doubleRing(); } - @Test(expected=ClassNotFoundException.class) - public void doubleRing_classNotFoundExceptionInRestore() throws Exception - { - new Expectations() {{ - doubleRing.backup(); result = new IOException(); - doubleRing.restore(); result = new ClassNotFoundException(); - }}; + @Test(expected = ClassNotFoundException.class) + public void doubleRing_classNotFoundExceptionInRestore() throws Exception { + new Expectations() { + { + doubleRing.backup(); + result = new IOException(); + doubleRing.restore(); + result = new ClassNotFoundException(); + } + }; resource.doubleRing(); } -} \ No newline at end of file +} diff --git a/priam/src/test/java/com/netflix/priam/resources/PriamConfigTest.java b/priam/src/test/java/com/netflix/priam/resources/PriamConfigTest.java index 9c5d76cb1..e10cf9f54 100644 --- a/priam/src/test/java/com/netflix/priam/resources/PriamConfigTest.java +++ b/priam/src/test/java/com/netflix/priam/resources/PriamConfigTest.java @@ -1,26 +1,21 @@ package com.netflix.priam.resources; -import java.util.HashMap; -import java.util.Map; -import javax.ws.rs.core.Response; - -import org.junit.Before; -import org.junit.Test; -import org.junit.runner.RunWith; +import static org.junit.Assert.*; -import com.google.inject.Inject; import com.netflix.priam.PriamServer; import com.netflix.priam.config.FakeConfiguration; -import com.netflix.priam.config.IConfiguration; +import java.util.HashMap; +import java.util.Map; +import javax.ws.rs.core.Response; import mockit.Expectations; import mockit.Mocked; import mockit.integration.junit4.JMockit; - -import static org.junit.Assert.*; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; @RunWith(JMockit.class) -public class PriamConfigTest -{ +public class PriamConfigTest { private @Mocked PriamServer priamServer; private PriamConfig resource; @@ -34,10 +29,8 @@ public void setUp() { fakeConfiguration.fakeProperties.put("test.prop", "test_value"); } - @Test - public void getPriamConfig() - { + public void getPriamConfig() { final Map expected = new HashMap<>(); expected.put("backupLocation", "casstestbackup"); new Expectations() { @@ -57,8 +50,7 @@ public void getPriamConfig() } @Test - public void getProperty() - { + public void getProperty() { final Map expected = new HashMap<>(); expected.put("test.prop", "test_value"); new Expectations() { @@ -82,4 +74,4 @@ public void getProperty() Response badResponse = resource.getProperty("not.a.property", null); assertEquals(404, badResponse.getStatus()); } -} \ No newline at end of file +} diff --git a/priam/src/test/java/com/netflix/priam/resources/PriamInstanceResourceTest.java b/priam/src/test/java/com/netflix/priam/resources/PriamInstanceResourceTest.java index d0d9ad59b..fc227da61 100644 --- a/priam/src/test/java/com/netflix/priam/resources/PriamInstanceResourceTest.java +++ b/priam/src/test/java/com/netflix/priam/resources/PriamInstanceResourceTest.java @@ -17,30 +17,25 @@ package com.netflix.priam.resources; -import java.util.List; - -import javax.ws.rs.WebApplicationException; -import javax.ws.rs.core.Response; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; import com.google.common.collect.ImmutableList; import com.netflix.priam.config.IConfiguration; import com.netflix.priam.identity.IPriamInstanceFactory; import com.netflix.priam.identity.PriamInstance; - +import java.util.List; +import javax.ws.rs.WebApplicationException; +import javax.ws.rs.core.Response; import mockit.Expectations; import mockit.Mocked; import mockit.integration.junit4.JMockit; - import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.fail; - @RunWith(JMockit.class) -public class PriamInstanceResourceTest -{ +public class PriamInstanceResourceTest { private static final String APP_NAME = "myApp"; private static final int NODE_ID = 3; @@ -49,23 +44,29 @@ public class PriamInstanceResourceTest private PriamInstanceResource resource; @Before - public void setUp() - { + public void setUp() { resource = new PriamInstanceResource(config, factory); } @Test - public void getInstances(@Mocked final PriamInstance instance1, @Mocked final PriamInstance instance2, @Mocked final PriamInstance instance3) - { + public void getInstances( + @Mocked final PriamInstance instance1, + @Mocked final PriamInstance instance2, + @Mocked final PriamInstance instance3) { new Expectations() { List instances = ImmutableList.of(instance1, instance2, instance3); { - config.getAppName(); result = APP_NAME; - factory.getAllIds(APP_NAME); result = instances; - instance1.toString(); result = "instance1"; - instance2.toString(); result = "instance2"; - instance3.toString(); result = "instance3"; + config.getAppName(); + result = APP_NAME; + factory.getAllIds(APP_NAME); + result = instances; + instance1.toString(); + result = "instance1"; + instance2.toString(); + result = "instance2"; + instance3.toString(); + result = "instance3"; } }; @@ -73,14 +74,16 @@ public void getInstances(@Mocked final PriamInstance instance1, @Mocked final Pr } @Test - public void getInstance(@Mocked final PriamInstance instance) - { + public void getInstance(@Mocked final PriamInstance instance) { final String expected = "plain text describing the instance"; new Expectations() { { - config.getAppName(); result = APP_NAME; - factory.getInstance(APP_NAME, config.getDC(), NODE_ID); result = instance; - instance.toString(); result = expected; + config.getAppName(); + result = APP_NAME; + factory.getInstance(APP_NAME, config.getDC(), NODE_ID); + result = instance; + instance.toString(); + result = expected; } }; @@ -88,27 +91,28 @@ public void getInstance(@Mocked final PriamInstance instance) } @Test - public void getInstance_notFound() - { - new Expectations() {{ - config.getAppName(); result = APP_NAME; - factory.getInstance(APP_NAME, config.getDC(), NODE_ID); result = null; - }}; - - try - { + public void getInstance_notFound() { + new Expectations() { + { + config.getAppName(); + result = APP_NAME; + factory.getInstance(APP_NAME, config.getDC(), NODE_ID); + result = null; + } + }; + + try { resource.getInstance(NODE_ID); fail("Expected WebApplicationException thrown"); - } catch(WebApplicationException e) - { + } catch (WebApplicationException e) { assertEquals(404, e.getResponse().getStatus()); - assertEquals("No priam instance with id " + NODE_ID + " found", e.getResponse().getEntity()); + assertEquals( + "No priam instance with id " + NODE_ID + " found", e.getResponse().getEntity()); } } @Test - public void createInstance(@Mocked final PriamInstance instance) - { + public void createInstance(@Mocked final PriamInstance instance) { final String instanceID = "i-abc123"; final String hostname = "dom.com"; final String ip = "123.123.123.123"; @@ -116,27 +120,31 @@ public void createInstance(@Mocked final PriamInstance instance) final String token = "1234567890"; new Expectations() { - { - config.getAppName(); result = APP_NAME; - factory.create(APP_NAME, NODE_ID, instanceID, hostname, ip, rack, null, token); result = instance; - instance.getId(); result = NODE_ID; - } + { + config.getAppName(); + result = APP_NAME; + factory.create(APP_NAME, NODE_ID, instanceID, hostname, ip, rack, null, token); + result = instance; + instance.getId(); + result = NODE_ID; + } }; Response response = resource.createInstance(NODE_ID, instanceID, hostname, ip, rack, token); assertEquals(201, response.getStatus()); - assertEquals("/"+NODE_ID, response.getMetadata().getFirst("location").toString()); + assertEquals("/" + NODE_ID, response.getMetadata().getFirst("location").toString()); } @Test - public void deleteInstance(@Mocked final PriamInstance instance) - { + public void deleteInstance(@Mocked final PriamInstance instance) { new Expectations() { - { - config.getAppName(); result = APP_NAME; - factory.getInstance(APP_NAME, config.getDC(), NODE_ID); result = instance; - factory.delete(instance); - } + { + config.getAppName(); + result = APP_NAME; + factory.getInstance(APP_NAME, config.getDC(), NODE_ID); + result = instance; + factory.delete(instance); + } }; Response response = resource.deleteInstance(NODE_ID); @@ -144,21 +152,23 @@ public void deleteInstance(@Mocked final PriamInstance instance) } @Test - public void deleteInstance_notFound() - { - new Expectations() {{ - config.getAppName(); result = APP_NAME; - factory.getInstance(APP_NAME, config.getDC(), NODE_ID); result = null; - }}; - - try - { + public void deleteInstance_notFound() { + new Expectations() { + { + config.getAppName(); + result = APP_NAME; + factory.getInstance(APP_NAME, config.getDC(), NODE_ID); + result = null; + } + }; + + try { resource.getInstance(NODE_ID); fail("Expected WebApplicationException thrown"); - } catch(WebApplicationException e) - { + } catch (WebApplicationException e) { assertEquals(404, e.getResponse().getStatus()); - assertEquals("No priam instance with id " + NODE_ID + " found", e.getResponse().getEntity()); + assertEquals( + "No priam instance with id " + NODE_ID + " found", e.getResponse().getEntity()); } } -} \ No newline at end of file +} diff --git a/priam/src/test/java/com/netflix/priam/restore/TestPostRestoreHook.java b/priam/src/test/java/com/netflix/priam/restore/TestPostRestoreHook.java index 5eb7fadc6..3360b7594 100644 --- a/priam/src/test/java/com/netflix/priam/restore/TestPostRestoreHook.java +++ b/priam/src/test/java/com/netflix/priam/restore/TestPostRestoreHook.java @@ -18,38 +18,39 @@ import com.google.inject.Guice; import com.google.inject.Injector; -import com.netflix.priam.config.IConfiguration; import com.netflix.priam.TestModule; +import com.netflix.priam.config.IConfiguration; +import java.io.File; import org.apache.commons.io.FileUtils; import org.junit.After; import org.junit.Assert; import org.junit.Before; import org.junit.Test; -import java.io.File; - public class TestPostRestoreHook { - @Before @After + @Before + @After public void setup() { Injector inject = Guice.createInjector(new TestModule()); IConfiguration configuration = inject.getInstance(IConfiguration.class); - //ensure heartbeat and done files are not present + // ensure heartbeat and done files are not present File heartBeatFile = new File(configuration.getPostRestoreHookHeartbeatFileName()); - if(heartBeatFile.exists()) { + if (heartBeatFile.exists()) { heartBeatFile.delete(); } File doneFile = new File(configuration.getPostRestoreHookDoneFileName()); - if(doneFile.exists()) { + if (doneFile.exists()) { doneFile.delete(); } } @Test /** - * Test to validate hasValidParameters. Expected to pass since none of the parameters in FakeConfiguration are blank + * Test to validate hasValidParameters. Expected to pass since none of the parameters in + * FakeConfiguration are blank */ public void testPostRestoreHookValidParameters() { Injector inject = Guice.createInjector(new TestModule()); @@ -59,59 +60,71 @@ public void testPostRestoreHookValidParameters() { @Test /** - * Test to validate execute method. This is a happy path since heart beat file is emited as soon as test case starts, and postrestorehook completes execution once the child process completes execution. - * Test fails in case of any exception. + * Test to validate execute method. This is a happy path since heart beat file is emited as soon + * as test case starts, and postrestorehook completes execution once the child process completes + * execution. Test fails in case of any exception. */ public void testPostRestoreHookExecuteHappyPath() throws Exception { Injector inject = Guice.createInjector(new TestModule()); IPostRestoreHook postRestoreHook = inject.getInstance(IPostRestoreHook.class); IConfiguration configuration = inject.getInstance(IConfiguration.class); - startHeartBeatThreadWithDelay(0, configuration.getPostRestoreHookHeartbeatFileName(), configuration.getPostRestoreHookDoneFileName()); + startHeartBeatThreadWithDelay( + 0, + configuration.getPostRestoreHookHeartbeatFileName(), + configuration.getPostRestoreHookDoneFileName()); postRestoreHook.execute(); } @Test /** - * Test to validate execute method. This is a variant of above method, where heartbeat is produced after an initial delay. This delay causes PostRestoreHook to terminate the child process since there is - * no heartbeat multiple times, and eventually once the heartbeat starts, PostRestoreHook waits for the child process to complete execution. - * Test fails in case of any exception. + * Test to validate execute method. This is a variant of above method, where heartbeat is + * produced after an initial delay. This delay causes PostRestoreHook to terminate the child + * process since there is no heartbeat multiple times, and eventually once the heartbeat starts, + * PostRestoreHook waits for the child process to complete execution. Test fails in case of any + * exception. */ public void testPostRestoreHookExecuteHeartBeatDelay() throws Exception { Injector inject = Guice.createInjector(new TestModule()); IPostRestoreHook postRestoreHook = inject.getInstance(IPostRestoreHook.class); IConfiguration configuration = inject.getInstance(IConfiguration.class); - startHeartBeatThreadWithDelay(1000, configuration.getPostRestoreHookHeartbeatFileName(), configuration.getPostRestoreHookDoneFileName()); + startHeartBeatThreadWithDelay( + 1000, + configuration.getPostRestoreHookHeartbeatFileName(), + configuration.getPostRestoreHookDoneFileName()); postRestoreHook.execute(); } /** * Starts a thread to emit heartbeat and finish with a done file. + * * @param delayInMs any start up delay if needed * @param heartBeatfileName name of the heart beat file * @param doneFileName name of the done file */ - private void startHeartBeatThreadWithDelay(long delayInMs, String heartBeatfileName, String doneFileName) { - Thread heartBeatEmitThread = new Thread() { - public void run() { - File heartBeatFile = new File(heartBeatfileName); - try { - //add a delay to heartbeat - Thread.sleep(delayInMs); - if (!heartBeatFile.exists() && !heartBeatFile.createNewFile()) { - Assert.fail("Unable to create heartbeat file"); - } - for(int i = 0; i < 10; i++) { - FileUtils.touch(heartBeatFile); - Thread.sleep(1000); - } + private void startHeartBeatThreadWithDelay( + long delayInMs, String heartBeatfileName, String doneFileName) { + Thread heartBeatEmitThread = + new Thread() { + public void run() { + File heartBeatFile = new File(heartBeatfileName); + try { + // add a delay to heartbeat + Thread.sleep(delayInMs); + if (!heartBeatFile.exists() && !heartBeatFile.createNewFile()) { + Assert.fail("Unable to create heartbeat file"); + } + for (int i = 0; i < 10; i++) { + FileUtils.touch(heartBeatFile); + Thread.sleep(1000); + } - File doneFile = new File(doneFileName); - doneFile.createNewFile(); - } catch (Exception ex) { - Assert.fail(ex.getMessage()); - } - } - }; + File doneFile = new File(doneFileName); + doneFile.createNewFile(); + } catch (Exception ex) { + Assert.fail(ex.getMessage()); + } + } + }; heartBeatEmitThread.start(); } diff --git a/priam/src/test/java/com/netflix/priam/scheduler/TestGuiceSingleton.java b/priam/src/test/java/com/netflix/priam/scheduler/TestGuiceSingleton.java index d6fb5b74a..85448849e 100644 --- a/priam/src/test/java/com/netflix/priam/scheduler/TestGuiceSingleton.java +++ b/priam/src/test/java/com/netflix/priam/scheduler/TestGuiceSingleton.java @@ -17,18 +17,15 @@ package com.netflix.priam.scheduler; -import org.junit.Test; - import com.google.inject.AbstractModule; import com.google.inject.Guice; import com.google.inject.Injector; import com.google.inject.Singleton; +import org.junit.Test; -public class TestGuiceSingleton -{ +public class TestGuiceSingleton { @Test - public void testSingleton() - { + public void testSingleton() { Injector injector = Guice.createInjector(new GModules()); injector.getInstance(EmptryInterface.class).print(); injector.getInstance(EmptryInterface.class).print(); @@ -38,36 +35,29 @@ public void testSingleton() printInjected(); printInjected(); } - - public void printInjected() - { + + public void printInjected() { Injector injector = Guice.createInjector(new GModules()); injector.getInstance(EmptryInterface.class).print(); } - public interface EmptryInterface - { + public interface EmptryInterface { String print(); } @Singleton - public static class GuiceSingleton implements EmptryInterface - { + public static class GuiceSingleton implements EmptryInterface { - public String print() - { + public String print() { System.out.println(this.toString()); return this.toString(); } } - public static class GModules extends AbstractModule - { + public static class GModules extends AbstractModule { @Override - protected void configure() - { + protected void configure() { bind(EmptryInterface.class).to(GuiceSingleton.class).asEagerSingleton(); } - } } diff --git a/priam/src/test/java/com/netflix/priam/scheduler/TestScheduler.java b/priam/src/test/java/com/netflix/priam/scheduler/TestScheduler.java index 6d0ed0eba..305f052f2 100644 --- a/priam/src/test/java/com/netflix/priam/scheduler/TestScheduler.java +++ b/priam/src/test/java/com/netflix/priam/scheduler/TestScheduler.java @@ -21,24 +21,21 @@ import com.google.inject.Inject; import com.google.inject.Injector; import com.google.inject.Singleton; -import com.netflix.priam.config.IConfiguration; import com.netflix.priam.TestModule; +import com.netflix.priam.config.IConfiguration; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import javax.management.MBeanServerFactory; import org.junit.Assert; import org.junit.Ignore; import org.junit.Test; -import javax.management.MBeanServerFactory; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.TimeUnit; - -public class TestScheduler -{ +public class TestScheduler { // yuck, but marginally better than using Thread.sleep private static CountDownLatch latch; @Test - public void testSchedule() throws Exception - { + public void testSchedule() throws Exception { latch = new CountDownLatch(1); Injector inject = Guice.createInjector(new TestModule()); PriamScheduler scheduler = inject.getInstance(PriamScheduler.class); @@ -50,9 +47,9 @@ public void testSchedule() throws Exception } @Test - @Ignore("not sure what this test really does, except test countdown latch and thread context switching") - public void testSingleInstanceSchedule() throws Exception - { + @Ignore( + "not sure what this test really does, except test countdown latch and thread context switching") + public void testSingleInstanceSchedule() throws Exception { latch = new CountDownLatch(3); Injector inject = Guice.createInjector(new TestModule()); PriamScheduler scheduler = inject.getInstance(PriamScheduler.class); @@ -65,65 +62,54 @@ public void testSingleInstanceSchedule() throws Exception } @Ignore - public static class TestTask extends Task - { + public static class TestTask extends Task { @Inject - public TestTask(IConfiguration config) - { - // todo: mock the MBeanServer instead, but this will prevent exceptions due to duplicate registrations + public TestTask(IConfiguration config) { + // todo: mock the MBeanServer instead, but this will prevent exceptions due to duplicate + // registrations super(config, MBeanServerFactory.newMBeanServer()); } @Override - public void execute() - { + public void execute() { latch.countDown(); } @Override - public String getName() - { + public String getName() { return "test"; } - } @Ignore @Singleton - public static class SingleTestTask extends Task - { + public static class SingleTestTask extends Task { @Inject - public SingleTestTask(IConfiguration config) - { + public SingleTestTask(IConfiguration config) { super(config, MBeanServerFactory.newMBeanServer()); } - public static int count =0; + public static int count = 0; + @Override - public void execute() - { + public void execute() { ++count; latch.countDown(); - try - { + try { // todo : why is this sleep important? - Thread.sleep(55);//5sec - } - catch (InterruptedException e) - { + Thread.sleep(55); // 5sec + } catch (InterruptedException e) { // TODO Auto-generated catch block e.printStackTrace(); } } @Override - public String getName() - { + public String getName() { return "test2"; } - public static TaskTimer getTimer() - { + public static TaskTimer getTimer() { return new SimpleTimer("test2", 11L); } } diff --git a/priam/src/test/java/com/netflix/priam/services/TestSnapshotMetaService.java b/priam/src/test/java/com/netflix/priam/services/TestSnapshotMetaService.java index 89a0da984..d397549ad 100644 --- a/priam/src/test/java/com/netflix/priam/services/TestSnapshotMetaService.java +++ b/priam/src/test/java/com/netflix/priam/services/TestSnapshotMetaService.java @@ -18,29 +18,27 @@ import com.google.inject.Guice; import com.google.inject.Injector; -import com.netflix.priam.config.IConfiguration; import com.netflix.priam.backup.AbstractBackup; import com.netflix.priam.backup.BRTestModule; import com.netflix.priam.backupv2.*; import com.netflix.priam.config.IBackupRestoreConfig; +import com.netflix.priam.config.IConfiguration; import com.netflix.priam.scheduler.TaskTimer; import com.netflix.priam.utils.BackupFileUtils; import com.netflix.priam.utils.DateUtil; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.time.Instant; import org.junit.Assert; import org.junit.Before; import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.time.Instant; - -/** - * Created by aagrawal on 6/20/18. - */ +/** Created by aagrawal on 6/20/18. */ public class TestSnapshotMetaService { - private static final Logger logger = LoggerFactory.getLogger(TestSnapshotMetaService.class.getName()); + private static final Logger logger = + LoggerFactory.getLogger(TestSnapshotMetaService.class.getName()); private static Path dummyDataDirectoryLocation; private static IConfiguration configuration; private static IBackupRestoreConfig backupRestoreConfig; @@ -52,8 +50,7 @@ public class TestSnapshotMetaService { public void setUp() { Injector injector = Guice.createInjector(new BRTestModule()); - if (configuration == null) - configuration = injector.getInstance(IConfiguration.class); + if (configuration == null) configuration = injector.getInstance(IConfiguration.class); if (backupRestoreConfig == null) backupRestoreConfig = injector.getInstance(IBackupRestoreConfig.class); @@ -61,15 +58,12 @@ public void setUp() { if (snapshotMetaService == null) snapshotMetaService = injector.getInstance(SnapshotMetaService.class); - if (metaFileReader == null) - metaFileReader = new TestMetaFileReader(); + if (metaFileReader == null) metaFileReader = new TestMetaFileReader(); - if (prefixGenerator == null) - prefixGenerator = injector.getInstance(PrefixGenerator.class); + if (prefixGenerator == null) prefixGenerator = injector.getInstance(PrefixGenerator.class); dummyDataDirectoryLocation = Paths.get(configuration.getDataFileLocation()); BackupFileUtils.cleanupDir(dummyDataDirectoryLocation); - } @Test @@ -79,7 +73,7 @@ public void testSnapshotMetaServiceEnabled() throws Exception { } @Test - public void testPrefix() throws Exception{ + public void testPrefix() throws Exception { Assert.assertTrue(prefixGenerator.getPrefix().endsWith("ppa-ekaf/1808575600")); Assert.assertTrue(prefixGenerator.getMetaPrefix().endsWith("ppa-ekaf/1808575600/META")); } @@ -93,17 +87,24 @@ public void testMetaFileName() throws Exception { Assert.assertFalse(metaFileReader.isValidMetaFile(path)); } - private void test(int noOfSstables, int noOfKeyspaces, int noOfCf) throws Exception{ + private void test(int noOfSstables, int noOfKeyspaces, int noOfCf) throws Exception { Instant snapshotInstant = DateUtil.getInstant(); String snapshotName = snapshotMetaService.generateSnapshotName(snapshotInstant); - BackupFileUtils.generateDummyFiles(dummyDataDirectoryLocation, noOfKeyspaces, noOfCf, noOfSstables, AbstractBackup.SNAPSHOT_FOLDER, snapshotName); + BackupFileUtils.generateDummyFiles( + dummyDataDirectoryLocation, + noOfKeyspaces, + noOfCf, + noOfSstables, + AbstractBackup.SNAPSHOT_FOLDER, + snapshotName); snapshotMetaService.setSnapshotName(snapshotName); - Path metaFileLocation = snapshotMetaService.processSnapshot(snapshotInstant).getMetaFilePath(); + Path metaFileLocation = + snapshotMetaService.processSnapshot(snapshotInstant).getMetaFilePath(); Assert.assertNotNull(metaFileLocation); Assert.assertTrue(metaFileLocation.toFile().exists()); Assert.assertTrue(metaFileLocation.toFile().isFile()); - //Try reading meta file. + // Try reading meta file. metaFileReader.setNoOfSstables(noOfSstables + 1); metaFileReader.readMeta(metaFileLocation); @@ -113,22 +114,21 @@ private void test(int noOfSstables, int noOfKeyspaces, int noOfCf) throws Except Assert.assertEquals(configuration.getRac(), metaFileInfo.getRack()); Assert.assertEquals(configuration.getDC(), metaFileInfo.getRegion()); - //Cleanup + // Cleanup metaFileLocation.toFile().delete(); BackupFileUtils.cleanupDir(dummyDataDirectoryLocation); } @Test public void testMetaFile() throws Exception { - test(5, 1,1); + test(5, 1, 1); } @Test public void testSize() throws Exception { - test (1000, 2,2); + test(1000, 2, 2); } - public static class TestMetaFileReader extends MetaFileReader { private int noOfSstables; @@ -142,6 +142,4 @@ public void process(ColumnfamilyResult columnfamilyResult) { Assert.assertEquals(noOfSstables, columnfamilyResult.getSstables().size()); } } - - } diff --git a/priam/src/test/java/com/netflix/priam/stream/StreamingTest.java b/priam/src/test/java/com/netflix/priam/stream/StreamingTest.java index 7efa32ef9..5c57421ba 100644 --- a/priam/src/test/java/com/netflix/priam/stream/StreamingTest.java +++ b/priam/src/test/java/com/netflix/priam/stream/StreamingTest.java @@ -17,88 +17,120 @@ package com.netflix.priam.stream; -import java.io.IOException; - -import org.junit.Assert; - -import org.junit.Test; - import com.google.inject.Guice; import com.google.inject.Injector; -import com.netflix.priam.config.FakeConfiguration; -import com.netflix.priam.config.IConfiguration; import com.netflix.priam.aws.S3BackupPath; import com.netflix.priam.backup.AbstractBackupPath; import com.netflix.priam.backup.BRTestModule; +import com.netflix.priam.config.FakeConfiguration; +import com.netflix.priam.config.IConfiguration; import com.netflix.priam.identity.InstanceIdentity; import com.netflix.priam.utils.FifoQueue; +import java.io.IOException; +import org.junit.Assert; +import org.junit.Test; -public class StreamingTest -{ - public void teststream() throws IOException, InterruptedException - { +public class StreamingTest { + public void teststream() throws IOException, InterruptedException { IConfiguration config = new FakeConfiguration("test", "cass_upg107_ccs", "test", "ins_id"); } @Test - public void testFifoAddAndRemove() - { + public void testFifoAddAndRemove() { FifoQueue queue = new FifoQueue(10); - for (long i = 0; i < 100; i++) - queue.adjustAndAdd(i); + for (long i = 0; i < 100; i++) queue.adjustAndAdd(i); Assert.assertEquals(10, queue.size()); Assert.assertEquals(new Long(90), queue.first()); } @Test - public void testAbstractPath() - { + public void testAbstractPath() { Injector injector = Guice.createInjector(new BRTestModule()); IConfiguration conf = injector.getInstance(IConfiguration.class); InstanceIdentity factory = injector.getInstance(InstanceIdentity.class); FifoQueue queue = new FifoQueue(10); - for (int i = 10; i < 30; i++) - { + for (int i = 10; i < 30; i++) { S3BackupPath path = new S3BackupPath(conf, factory); - path.parseRemote("test_backup/"+FakeConfiguration.FAKE_REGION+"/fakecluster/123456/201108" + i + "0000" + "/SNAP/ks1/cf2/f1" + i + ".db"); + path.parseRemote( + "test_backup/" + + FakeConfiguration.FAKE_REGION + + "/fakecluster/123456/201108" + + i + + "0000" + + "/SNAP/ks1/cf2/f1" + + i + + ".db"); queue.adjustAndAdd(path); } - for (int i = 10; i < 30; i++) - { + for (int i = 10; i < 30; i++) { S3BackupPath path = new S3BackupPath(conf, factory); - path.parseRemote("test_backup/"+FakeConfiguration.FAKE_REGION+"/fakecluster/123456/201108" + i + "0000" + "/SNAP/ks1/cf2/f2" + i + ".db"); + path.parseRemote( + "test_backup/" + + FakeConfiguration.FAKE_REGION + + "/fakecluster/123456/201108" + + i + + "0000" + + "/SNAP/ks1/cf2/f2" + + i + + ".db"); queue.adjustAndAdd(path); } - for (int i = 10; i < 30; i++) - { + for (int i = 10; i < 30; i++) { S3BackupPath path = new S3BackupPath(conf, factory); - path.parseRemote("test_backup/"+FakeConfiguration.FAKE_REGION+"/fakecluster/123456/201108" + i + "0000" + "/SNAP/ks1/cf2/f3" + i + ".db"); + path.parseRemote( + "test_backup/" + + FakeConfiguration.FAKE_REGION + + "/fakecluster/123456/201108" + + i + + "0000" + + "/SNAP/ks1/cf2/f3" + + i + + ".db"); queue.adjustAndAdd(path); } S3BackupPath path = new S3BackupPath(conf, factory); - path.parseRemote("test_backup/"+FakeConfiguration.FAKE_REGION+"/fakecluster/123456/201108290000" + "/SNAP/ks1/cf2/f129.db"); + path.parseRemote( + "test_backup/" + + FakeConfiguration.FAKE_REGION + + "/fakecluster/123456/201108290000" + + "/SNAP/ks1/cf2/f129.db"); Assert.assertTrue(queue.contains(path)); - path.parseRemote("test_backup/"+FakeConfiguration.FAKE_REGION+"/fakecluster/123456/201108290000" + "/SNAP/ks1/cf2/f229.db"); + path.parseRemote( + "test_backup/" + + FakeConfiguration.FAKE_REGION + + "/fakecluster/123456/201108290000" + + "/SNAP/ks1/cf2/f229.db"); Assert.assertTrue(queue.contains(path)); - path.parseRemote("test_backup/"+FakeConfiguration.FAKE_REGION+"/fakecluster/123456/201108290000" + "/SNAP/ks1/cf2/f329.db"); + path.parseRemote( + "test_backup/" + + FakeConfiguration.FAKE_REGION + + "/fakecluster/123456/201108290000" + + "/SNAP/ks1/cf2/f329.db"); Assert.assertTrue(queue.contains(path)); - path.parseRemote("test_backup/"+FakeConfiguration.FAKE_REGION+"/fakecluster/123456/201108260000/SNAP/ks1/cf2/f326.db To: cass/data/ks1/cf2/f326.db"); + path.parseRemote( + "test_backup/" + + FakeConfiguration.FAKE_REGION + + "/fakecluster/123456/201108260000/SNAP/ks1/cf2/f326.db To: cass/data/ks1/cf2/f326.db"); Assert.assertEquals(path, queue.first()); } @Test - public void testIgnoreIndexFiles() - { - String[] testInputs = new String[] { "User_Authentication_Audit.User_Authentication_Audit_appkey_idx-hc-93-Digest.sha1", - "User_Authentication_Audit.User_Authentication_Audit_appkey_idx-hc-93-Filter.db", "User_Authentication_Audit.User_Authentication_Audit_appkey_idx-hc-93-Data.db", - "User_Authentication_Audit.User_Authentication_Audit_appkey_idx-hc-93-Statistics.db", "CS_Agents.CS_Agents_supervisorEmpSk_idx-hc-1-Filter.db", - "CS_Agents.CS_Agents_supervisorEmpSk_idx-hc-1-Digest.sha1", "CS_Agents.CS_Agents_supervisorEmpSk_idx-hc-1-Statistics.db", "CS_Agents.CS_Agents_supervisorEmpSk_idx-hc-1-Data.db" }; - + public void testIgnoreIndexFiles() { + String[] testInputs = + new String[] { + "User_Authentication_Audit.User_Authentication_Audit_appkey_idx-hc-93-Digest.sha1", + "User_Authentication_Audit.User_Authentication_Audit_appkey_idx-hc-93-Filter.db", + "User_Authentication_Audit.User_Authentication_Audit_appkey_idx-hc-93-Data.db", + "User_Authentication_Audit.User_Authentication_Audit_appkey_idx-hc-93-Statistics.db", + "CS_Agents.CS_Agents_supervisorEmpSk_idx-hc-1-Filter.db", + "CS_Agents.CS_Agents_supervisorEmpSk_idx-hc-1-Digest.sha1", + "CS_Agents.CS_Agents_supervisorEmpSk_idx-hc-1-Statistics.db", + "CS_Agents.CS_Agents_supervisorEmpSk_idx-hc-1-Data.db" + }; } - } diff --git a/priam/src/test/java/com/netflix/priam/tuner/JVMOptionTunerTest.java b/priam/src/test/java/com/netflix/priam/tuner/JVMOptionTunerTest.java index 684be39ce..9628dca23 100644 --- a/priam/src/test/java/com/netflix/priam/tuner/JVMOptionTunerTest.java +++ b/priam/src/test/java/com/netflix/priam/tuner/JVMOptionTunerTest.java @@ -17,65 +17,67 @@ package com.netflix.priam.tuner; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + import com.netflix.priam.config.FakeConfiguration; import com.netflix.priam.config.IConfiguration; import com.netflix.priam.scheduler.UnsupportedTypeException; -import org.junit.Test; - import java.util.*; import java.util.stream.Collectors; +import org.junit.Test; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; - -/** - * Created by aagrawal on 8/29/17. - */ +/** Created by aagrawal on 8/29/17. */ public class JVMOptionTunerTest { private IConfiguration config; JVMOptionsTuner tuner; @Test - public void testCMS() throws Exception - { + public void testCMS() throws Exception { config = new GCConfiguration(GCType.CMS, null, null, null, null); List jvmOptionMap = getConfiguredJVMOptions(config); - //Validate that all CMS options should be uncommented. - long failedVerification = jvmOptionMap.stream().map(jvmOption -> { - GCType gcType = GCTuner.getGCType(jvmOption); - if (gcType != null && gcType != GCType.CMS) - { - return 1; - } - return 0; - }).filter(returncode -> (returncode != 0)).count(); - - if (failedVerification > 0) - throw new Exception ("Failed validation for CMS"); + // Validate that all CMS options should be uncommented. + long failedVerification = + jvmOptionMap + .stream() + .map( + jvmOption -> { + GCType gcType = GCTuner.getGCType(jvmOption); + if (gcType != null && gcType != GCType.CMS) { + return 1; + } + return 0; + }) + .filter(returncode -> (returncode != 0)) + .count(); + + if (failedVerification > 0) throw new Exception("Failed validation for CMS"); } @Test - public void testG1GC() throws Exception - { + public void testG1GC() throws Exception { config = new GCConfiguration(GCType.G1GC, null, null, null, null); List jvmOptionMap = getConfiguredJVMOptions(config); - //Validate that all G1GC options should be uncommented. - long failedVerification = jvmOptionMap.stream().map(jvmOption -> { - GCType gcType = GCTuner.getGCType(jvmOption); - if (gcType != null && gcType != GCType.G1GC) - { - return 1; - } - return 0; - }).filter(returncode -> (returncode != 0)).count(); - - if (failedVerification > 0) - throw new Exception ("Failed validation for G1GC"); + // Validate that all G1GC options should be uncommented. + long failedVerification = + jvmOptionMap + .stream() + .map( + jvmOption -> { + GCType gcType = GCTuner.getGCType(jvmOption); + if (gcType != null && gcType != GCType.G1GC) { + return 1; + } + return 0; + }) + .filter(returncode -> (returncode != 0)) + .count(); + + if (failedVerification > 0) throw new Exception("Failed validation for G1GC"); } @Test - public void testCMSUpsert() throws Exception - { + public void testCMSUpsert() throws Exception { JVMOption option1 = new JVMOption("-Dsample"); JVMOption option2 = new JVMOption("-Dsample2", "10", false, false); JVMOption option3 = new JVMOption("-XX:NumberOfGCLogFiles", "20", false, false); @@ -84,24 +86,35 @@ public void testCMSUpsert() throws Exception JVMOption xmxOption = new JVMOption("-Xmx", "20G", false, true); JVMOption xmsOption = new JVMOption("-Xms", "20G", false, true); - StringBuffer buffer = new StringBuffer(option1.toJVMOptionString() + "," + option2.toJVMOptionString() + "," + option3.toJVMOptionString()); - config = new GCConfiguration(GCType.CMS, null, buffer.toString(), xmnOption.getValue(), xmxOption.getValue()); + StringBuffer buffer = + new StringBuffer( + option1.toJVMOptionString() + + "," + + option2.toJVMOptionString() + + "," + + option3.toJVMOptionString()); + config = + new GCConfiguration( + GCType.CMS, + null, + buffer.toString(), + xmnOption.getValue(), + xmxOption.getValue()); List jvmOptions = getConfiguredJVMOptions(config); - //Verify all the options do exist. + // Verify all the options do exist. assertTrue(jvmOptions.contains(option3)); assertTrue(jvmOptions.contains(option2)); assertTrue(jvmOptions.contains(option1)); - //Verify heap options exist with the value provided. + // Verify heap options exist with the value provided. assertTrue(jvmOptions.contains(xmnOption)); assertTrue(jvmOptions.contains(xmxOption)); assertTrue(jvmOptions.contains(xmsOption)); } @Test - public void testCMSExclude() throws Exception - { + public void testCMSExclude() throws Exception { JVMOption youngHeap = new JVMOption("-Xmn", "3G", false, true); JVMOption maxHeap = new JVMOption("-Xmx", "12G", false, true); @@ -109,11 +122,17 @@ public void testCMSExclude() throws Exception JVMOption option2 = new JVMOption("-XX:NumberOfGCLogFiles", "20", false, false); JVMOption option3 = new JVMOption("-XX:+UseG1GC", null, false, false); - StringBuffer buffer = new StringBuffer(option1.toJVMOptionString() + "," + option2.toJVMOptionString() + "," + option3.toJVMOptionString()); + StringBuffer buffer = + new StringBuffer( + option1.toJVMOptionString() + + "," + + option2.toJVMOptionString() + + "," + + option3.toJVMOptionString()); config = new GCConfiguration(GCType.CMS, buffer.toString(), null, "3G", "12G"); List jvmOptions = getConfiguredJVMOptions(config); - //Verify all the options do not exist. + // Verify all the options do not exist. assertFalse(jvmOptions.contains(option3)); assertFalse(jvmOptions.contains(option2)); assertFalse(jvmOptions.contains(option1)); @@ -124,28 +143,37 @@ public void testCMSExclude() throws Exception } @Test - public void testG1GCUpsertExclude() throws Exception - { + public void testG1GCUpsertExclude() throws Exception { JVMOption youngHeap = new JVMOption("-Xmn", "3G", true, true); JVMOption maxHeap = new JVMOption("-Xmx", "12G", false, true); JVMOption option1 = new JVMOption("-Dsample"); JVMOption option2 = new JVMOption("-Dsample2", "10", false, false); JVMOption option3 = new JVMOption("-XX:NumberOfGCLogFiles", "20", false, false); - StringBuffer upsert = new StringBuffer(option1.toJVMOptionString() + "," + option2.toJVMOptionString() + "," + option3.toJVMOptionString()); + StringBuffer upsert = + new StringBuffer( + option1.toJVMOptionString() + + "," + + option2.toJVMOptionString() + + "," + + option3.toJVMOptionString()); JVMOption option4 = new JVMOption("-XX:NumberOfGCLogFiles", null, false, false); JVMOption option5 = new JVMOption("-XX:+UseG1GC", null, false, false); - StringBuffer exclude = new StringBuffer(option4.toJVMOptionString() + "," + option5.toJVMOptionString()); + StringBuffer exclude = + new StringBuffer(option4.toJVMOptionString() + "," + option5.toJVMOptionString()); - config = new GCConfiguration(GCType.G1GC, exclude.toString(), upsert.toString(), "3G", "12G"); + config = + new GCConfiguration( + GCType.G1GC, exclude.toString(), upsert.toString(), "3G", "12G"); List jvmOptions = getConfiguredJVMOptions(config); // Verify upserts exist assertTrue(jvmOptions.contains(option1)); assertTrue(jvmOptions.contains(option2)); - // Verify exclude exist. This is to prove that if an element is in EXCLUDE, it will always be excluded. + // Verify exclude exist. This is to prove that if an element is in EXCLUDE, it will always + // be excluded. assertFalse(jvmOptions.contains(option3)); assertFalse(jvmOptions.contains(option4)); assertFalse(jvmOptions.contains(option5)); @@ -158,37 +186,39 @@ public void testG1GCUpsertExclude() throws Exception assertTrue(allJVMOptions.contains(youngHeap)); } - private List getConfiguredJVMOptions(IConfiguration config) throws Exception { return getConfiguredJVMOptions(config, true); } - private List getConfiguredJVMOptions(IConfiguration config, boolean filter) throws Exception{ + private List getConfiguredJVMOptions(IConfiguration config, boolean filter) + throws Exception { tuner = new JVMOptionsTuner(config); List configuredJVMOptions = tuner.updateJVMOptions(); if (filter) { - return configuredJVMOptions.stream() - .map(JVMOption::parse) - .filter(jvmOption -> (jvmOption != null)) - .filter(jvmOption -> !jvmOption.isCommented()) - .collect(Collectors.toList()); + return configuredJVMOptions + .stream() + .map(JVMOption::parse) + .filter(jvmOption -> (jvmOption != null)) + .filter(jvmOption -> !jvmOption.isCommented()) + .collect(Collectors.toList()); } else { - return configuredJVMOptions.stream() - .map(JVMOption::parse) - .collect(Collectors.toList()); + return configuredJVMOptions.stream().map(JVMOption::parse).collect(Collectors.toList()); } } - - private class GCConfiguration extends FakeConfiguration{ + private class GCConfiguration extends FakeConfiguration { private GCType gcType; private String configuredJVMExclude; private String configuredJVMUpsert; private String configuredHeapNewSize; private String configuredHeapSize; - GCConfiguration(GCType gcType, String configuredJVMExclude,String configuredJVMUpsert, String configuredHeapNewSize, String configuredHeapSize) - { + GCConfiguration( + GCType gcType, + String configuredJVMExclude, + String configuredJVMUpsert, + String configuredHeapNewSize, + String configuredHeapSize) { this.gcType = gcType; this.configuredJVMExclude = configuredJVMExclude; this.configuredJVMUpsert = configuredJVMUpsert; @@ -197,29 +227,28 @@ private class GCConfiguration extends FakeConfiguration{ } @Override - public GCType getGCType() throws UnsupportedTypeException{ + public GCType getGCType() throws UnsupportedTypeException { return gcType; } @Override - public Map getJVMExcludeSet(){ + public Map getJVMExcludeSet() { return JVMOptionsTuner.parseJVMOptions(configuredJVMExclude); } @Override - public String getHeapSize(){ + public String getHeapSize() { return configuredHeapSize; } @Override - public String getHeapNewSize(){ + public String getHeapNewSize() { return configuredHeapNewSize; } @Override - public Map getJVMUpsertSet(){ + public Map getJVMUpsertSet() { return JVMOptionsTuner.parseJVMOptions(configuredJVMUpsert); } } - } diff --git a/priam/src/test/java/com/netflix/priam/tuner/StandardTunerTest.java b/priam/src/test/java/com/netflix/priam/tuner/StandardTunerTest.java index 1aec70d9e..986b5d046 100644 --- a/priam/src/test/java/com/netflix/priam/tuner/StandardTunerTest.java +++ b/priam/src/test/java/com/netflix/priam/tuner/StandardTunerTest.java @@ -16,17 +16,16 @@ package com.netflix.priam.tuner; -import java.io.File; +import static org.junit.Assert.assertEquals; import com.google.common.io.Files; import com.netflix.priam.config.FakeConfiguration; import com.netflix.priam.config.IConfiguration; +import java.io.File; import org.junit.Before; import org.junit.Test; -import static org.junit.Assert.assertEquals; -public class StandardTunerTest -{ +public class StandardTunerTest { /* note: these are, more or less, arbitrary paritioner class names. as long as the tests exercise the code, all is good */ private static final String A_PARTITIONER = "com.netflix.priam.utils.NonexistentPartitioner"; private static final String RANDOM_PARTITIONER = "org.apache.cassandra.dht.RandomPartitioner"; @@ -36,51 +35,44 @@ public class StandardTunerTest private StandardTuner tuner; @Before - public void setup() - { + public void setup() { IConfiguration config = new FakeConfiguration(); tuner = new StandardTuner(config); } @Test - public void derivePartitioner_NullYamlEntry() - { + public void derivePartitioner_NullYamlEntry() { String partitioner = tuner.derivePartitioner(null, A_PARTITIONER); assertEquals(A_PARTITIONER, partitioner); } @Test - public void derivePartitioner_EmptyYamlEntry() - { + public void derivePartitioner_EmptyYamlEntry() { String partitioner = tuner.derivePartitioner("", A_PARTITIONER); assertEquals(A_PARTITIONER, partitioner); } @Test - public void derivePartitioner_RandomPartitioner() - { + public void derivePartitioner_RandomPartitioner() { String partitioner = tuner.derivePartitioner(RANDOM_PARTITIONER, RANDOM_PARTITIONER); assertEquals(RANDOM_PARTITIONER, partitioner); } @Test - public void derivePartitioner_MurmurPartitioner() - { + public void derivePartitioner_MurmurPartitioner() { String partitioner = tuner.derivePartitioner(MURMUR_PARTITIONER, MURMUR_PARTITIONER); assertEquals(MURMUR_PARTITIONER, partitioner); } @Test - public void derivePartitioner_BOPPartitionerInYaml() - { + public void derivePartitioner_BOPPartitionerInYaml() { String partitioner = tuner.derivePartitioner(BOP_PARTITIONER, MURMUR_PARTITIONER); assertEquals(BOP_PARTITIONER, partitioner); } @Test - public void derivePartitioner_BOPPartitionerInConfig() - { + public void derivePartitioner_BOPPartitionerInConfig() { String partitioner = tuner.derivePartitioner(RANDOM_PARTITIONER, BOP_PARTITIONER); assertEquals(BOP_PARTITIONER, partitioner); } @@ -88,7 +80,9 @@ public void derivePartitioner_BOPPartitionerInConfig() @Test public void dump() throws Exception { String target = "/tmp/priam_test.yaml"; - Files.copy(new File("src/main/resources/incr-restore-cassandra.yaml"), new File("/tmp/priam_test.yaml")); + Files.copy( + new File("src/main/resources/incr-restore-cassandra.yaml"), + new File("/tmp/priam_test.yaml")); tuner.writeAllProperties(target, "your_host", "YourSeedProvider"); } } diff --git a/priam/src/test/java/com/netflix/priam/tuner/dse/DseConfigStub.java b/priam/src/test/java/com/netflix/priam/tuner/dse/DseConfigStub.java index dbc44e16b..999c6f557 100644 --- a/priam/src/test/java/com/netflix/priam/tuner/dse/DseConfigStub.java +++ b/priam/src/test/java/com/netflix/priam/tuner/dse/DseConfigStub.java @@ -17,46 +17,41 @@ */ import com.netflix.priam.config.FakeConfiguration; - import java.util.HashSet; import java.util.Set; -public class DseConfigStub implements IDseConfiguration -{ +public class DseConfigStub implements IDseConfiguration { boolean auditLogEnabled; - public String getDseYamlLocation() - { + public String getDseYamlLocation() { return new FakeConfiguration().getCassHome() + "/resources/dse/conf/dse.yaml"; } - public String getDseDelegatingSnitch() - { + public String getDseDelegatingSnitch() { return null; } - public NodeType getNodeType() - { + public NodeType getNodeType() { return null; } - public boolean isAuditLogEnabled() - { + public boolean isAuditLogEnabled() { return auditLogEnabled; } - public void setAuditLogEnabled(boolean b) - { + public void setAuditLogEnabled(boolean b) { auditLogEnabled = b; } - public String getAuditLogExemptKeyspaces() - { + public String getAuditLogExemptKeyspaces() { return "YourSwellKeyspace"; } - public Set getAuditLogCategories() - { - return new HashSet(){{ this.add(AuditLogCategory.ALL); }}; + public Set getAuditLogCategories() { + return new HashSet() { + { + this.add(AuditLogCategory.ALL); + } + }; } } diff --git a/priam/src/test/java/com/netflix/priam/tuner/dse/DseTunerTest.java b/priam/src/test/java/com/netflix/priam/tuner/dse/DseTunerTest.java index f7698878d..e7a695650 100644 --- a/priam/src/test/java/com/netflix/priam/tuner/dse/DseTunerTest.java +++ b/priam/src/test/java/com/netflix/priam/tuner/dse/DseTunerTest.java @@ -16,21 +16,18 @@ * */ +import com.google.common.io.Files; +import com.netflix.priam.config.FakeConfiguration; +import com.netflix.priam.config.IConfiguration; import java.io.File; import java.io.FileReader; import java.io.IOException; import java.util.Properties; - -import com.google.common.io.Files; -import com.netflix.priam.config.FakeConfiguration; -import com.netflix.priam.config.IConfiguration; - import org.junit.Assert; import org.junit.Before; import org.junit.Test; -public class DseTunerTest -{ +public class DseTunerTest { IConfiguration config; DseConfigStub dseConfig; DseTuner dseTunerYaml; @@ -41,8 +38,7 @@ public class DseTunerTest File targetDseYamlFile; @Before - public void setup() throws IOException - { + public void setup() throws IOException { config = new FakeConfiguration(); dseConfig = new DseConfigStub(); auditLogTunerYaml = new AuditLogTunerYaml(dseConfig); @@ -51,16 +47,14 @@ public void setup() throws IOException dseTunerLog4j = new DseTuner(config, dseConfig, auditLogTunerLog4j); File targetDir = new File(config.getCassHome() + "/conf"); - if(!targetDir.exists()) - targetDir.mkdirs(); + if (!targetDir.exists()) targetDir.mkdirs(); targetFile = new File(config.getCassHome() + AuditLogTunerLog4J.AUDIT_LOG_FILE); Files.copy(new File("src/test/resources/" + AuditLogTunerLog4J.AUDIT_LOG_FILE), targetFile); } @Test - public void auditLogProperties_Enabled() throws IOException - { + public void auditLogProperties_Enabled() throws IOException { dseConfig.setAuditLogEnabled(true); auditLogTunerLog4j.tuneAuditLog(); @@ -70,8 +64,7 @@ public void auditLogProperties_Enabled() throws IOException } @Test - public void auditLogProperties_Disabled() throws IOException - { + public void auditLogProperties_Disabled() throws IOException { dseConfig.setAuditLogEnabled(false); auditLogTunerLog4j.tuneAuditLog(); @@ -81,14 +74,13 @@ public void auditLogProperties_Disabled() throws IOException } /** - * This is different because we test the disabled step using the already used enabled file - * (not a clean copy over of the original props file from the resources dir), and vice versa + * This is different because we test the disabled step using the already used enabled file (not + * a clean copy over of the original props file from the resources dir), and vice versa * * @throws IOException when file is not found or permissions. */ @Test - public void auditLogProperties_ThereAndBackAgain() throws IOException - { + public void auditLogProperties_ThereAndBackAgain() throws IOException { auditLogProperties_Enabled(); auditLogProperties_Disabled(); auditLogProperties_Enabled(); @@ -114,31 +106,38 @@ public void auditLogProperties_ThereAndBackAgain() throws IOException @Test public void auditLogYamlProperties_Enabled() throws IOException { File targetDseDir = new File(config.getCassHome() + "/resources/dse/conf/"); - if(!targetDseDir.exists()) { + if (!targetDseDir.exists()) { targetDseDir.mkdirs(); } int index = dseConfig.getDseYamlLocation().lastIndexOf('/') + 1; - targetDseYamlFile = new File(targetDseDir + dseConfig.getDseYamlLocation().substring(index - 1)); - Files.copy(new File("src/test/resources/conf/" + dseConfig.getDseYamlLocation().substring(index)), targetDseYamlFile); - + targetDseYamlFile = + new File(targetDseDir + dseConfig.getDseYamlLocation().substring(index - 1)); + Files.copy( + new File( + "src/test/resources/conf/" + + dseConfig.getDseYamlLocation().substring(index)), + targetDseYamlFile); dseConfig.setAuditLogEnabled(true); auditLogTunerYaml.tuneAuditLog(); - } @Test public void auditLogYamlProperties_Disabled() throws IOException { File targetDseDir = new File(config.getCassHome() + "/resources/dse/conf/"); - if(!targetDseDir.exists()) { + if (!targetDseDir.exists()) { targetDseDir.mkdirs(); } int index = dseConfig.getDseYamlLocation().lastIndexOf('/') + 1; - targetDseYamlFile = new File(targetDseDir + dseConfig.getDseYamlLocation().substring(index - 1)); - Files.copy(new File("src/test/resources/conf/" + dseConfig.getDseYamlLocation().substring(index)), targetDseYamlFile); - + targetDseYamlFile = + new File(targetDseDir + dseConfig.getDseYamlLocation().substring(index - 1)); + Files.copy( + new File( + "src/test/resources/conf/" + + dseConfig.getDseYamlLocation().substring(index)), + targetDseYamlFile); dseConfig.setAuditLogEnabled(false); auditLogTunerYaml.tuneAuditLog(); diff --git a/priam/src/test/java/com/netflix/priam/utils/BackupFileUtils.java b/priam/src/test/java/com/netflix/priam/utils/BackupFileUtils.java index 3f7fdcb4d..59e840227 100644 --- a/priam/src/test/java/com/netflix/priam/utils/BackupFileUtils.java +++ b/priam/src/test/java/com/netflix/priam/utils/BackupFileUtils.java @@ -17,20 +17,17 @@ package com.netflix.priam.utils; -import org.apache.cassandra.io.sstable.Component; -import org.apache.commons.io.FileUtils; - import java.io.FileWriter; import java.io.IOException; import java.nio.file.Path; import java.nio.file.Paths; import java.util.EnumSet; +import org.apache.cassandra.io.sstable.Component; +import org.apache.commons.io.FileUtils; -/** - * Created by aagrawal on 9/23/18. - */ +/** Created by aagrawal on 9/23/18. */ public class BackupFileUtils { - public static void cleanupDir(Path dir){ + public static void cleanupDir(Path dir) { if (dir.toFile().exists()) try { FileUtils.cleanDirectory(dir.toFile()); @@ -39,8 +36,15 @@ public static void cleanupDir(Path dir){ } } - public static void generateDummyFiles(Path dummyDir, int noOfKeyspaces, int noOfCf, int noOfSstables, String backupDir, String snapshotName) throws Exception { - //Clean the dummy directory + public static void generateDummyFiles( + Path dummyDir, + int noOfKeyspaces, + int noOfCf, + int noOfSstables, + String backupDir, + String snapshotName) + throws Exception { + // Clean the dummy directory cleanupDir(dummyDir); for (int i = 1; i <= noOfKeyspaces; i++) { @@ -53,17 +57,30 @@ public static void generateDummyFiles(Path dummyDir, int noOfKeyspaces, int noOf String prefixName = "mc-" + k + "-big"; for (Component.Type type : EnumSet.allOf(Component.Type.class)) { - Path componentPath = Paths.get(dummyDir.toFile().getAbsolutePath(), keyspaceName, columnfamilyname, backupDir, snapshotName, prefixName + "-" + type.name() + ".db"); + Path componentPath = + Paths.get( + dummyDir.toFile().getAbsolutePath(), + keyspaceName, + columnfamilyname, + backupDir, + snapshotName, + prefixName + "-" + type.name() + ".db"); componentPath.getParent().toFile().mkdirs(); try (FileWriter fileWriter = new FileWriter(componentPath.toFile())) { fileWriter.write(""); } - } } - Path componentPath = Paths.get(dummyDir.toFile().getAbsolutePath(), keyspaceName, columnfamilyname, backupDir, snapshotName, "manifest.json"); - try(FileWriter fileWriter = new FileWriter(componentPath.toFile())){ + Path componentPath = + Paths.get( + dummyDir.toFile().getAbsolutePath(), + keyspaceName, + columnfamilyname, + backupDir, + snapshotName, + "manifest.json"); + try (FileWriter fileWriter = new FileWriter(componentPath.toFile())) { fileWriter.write(""); } } diff --git a/priam/src/test/java/com/netflix/priam/utils/FakeSleeper.java b/priam/src/test/java/com/netflix/priam/utils/FakeSleeper.java index a0d2b460e..231a5066e 100644 --- a/priam/src/test/java/com/netflix/priam/utils/FakeSleeper.java +++ b/priam/src/test/java/com/netflix/priam/utils/FakeSleeper.java @@ -17,19 +17,14 @@ package com.netflix.priam.utils; -/** - * TODO: Replace with a mock object - */ -public class FakeSleeper implements Sleeper -{ +/** TODO: Replace with a mock object */ +public class FakeSleeper implements Sleeper { @Override - public void sleep(long waitTimeMs) throws InterruptedException - { + public void sleep(long waitTimeMs) throws InterruptedException { // no-op } - public void sleepQuietly(long waitTimeMs) - { - //no-op + public void sleepQuietly(long waitTimeMs) { + // no-op } -} \ No newline at end of file +} diff --git a/priam/src/test/java/com/netflix/priam/utils/Murmur3TokenManagerTest.java b/priam/src/test/java/com/netflix/priam/utils/Murmur3TokenManagerTest.java index cf8ab6a20..8a946eebd 100644 --- a/priam/src/test/java/com/netflix/priam/utils/Murmur3TokenManagerTest.java +++ b/priam/src/test/java/com/netflix/priam/utils/Murmur3TokenManagerTest.java @@ -17,20 +17,19 @@ package com.netflix.priam.utils; +import static com.netflix.priam.utils.TokenManager.MAXIMUM_TOKEN_MURMUR3; +import static com.netflix.priam.utils.TokenManager.MINIMUM_TOKEN_MURMUR3; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; + import com.google.common.collect.ImmutableList; import com.netflix.priam.config.FakeConfiguration; import com.netflix.priam.config.IConfiguration; -import org.junit.Before; -import org.junit.Test; - import java.math.BigInteger; import java.util.Collections; import java.util.List; - -import static com.netflix.priam.utils.TokenManager.MAXIMUM_TOKEN_MURMUR3; -import static com.netflix.priam.utils.TokenManager.MINIMUM_TOKEN_MURMUR3; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; +import org.junit.Before; +import org.junit.Test; public class Murmur3TokenManagerTest { private TokenManager tokenManager; @@ -65,9 +64,14 @@ public void initialToken_positionZero() { @Test public void initialToken_offsets_zeroPosition() { - assertEquals(MINIMUM_TOKEN_MURMUR3.add(BigInteger.valueOf(7)), tokenManager.initialToken(1, 0, 7)); - assertEquals(MINIMUM_TOKEN_MURMUR3.add(BigInteger.valueOf(11)), tokenManager.initialToken(2, 0, 11)); - assertEquals(MINIMUM_TOKEN_MURMUR3.add(BigInteger.valueOf(Integer.MAX_VALUE)), + assertEquals( + MINIMUM_TOKEN_MURMUR3.add(BigInteger.valueOf(7)), + tokenManager.initialToken(1, 0, 7)); + assertEquals( + MINIMUM_TOKEN_MURMUR3.add(BigInteger.valueOf(11)), + tokenManager.initialToken(2, 0, 11)); + assertEquals( + MINIMUM_TOKEN_MURMUR3.add(BigInteger.valueOf(Integer.MAX_VALUE)), tokenManager.initialToken(256, 0, Integer.MAX_VALUE)); } @@ -76,12 +80,18 @@ public void initialToken_cannotExceedMaximumToken() { final int maxRingSize = Integer.MAX_VALUE; final int maxPosition = maxRingSize - 1; final int maxOffset = Integer.MAX_VALUE; - assertEquals(1, MAXIMUM_TOKEN_MURMUR3.compareTo(tokenManager.initialToken(maxRingSize, maxPosition, maxOffset))); + assertEquals( + 1, + MAXIMUM_TOKEN_MURMUR3.compareTo( + tokenManager.initialToken(maxRingSize, maxPosition, maxOffset))); } @Test public void createToken() { - assertEquals(MAXIMUM_TOKEN_MURMUR3.subtract(MINIMUM_TOKEN_MURMUR3).divide(BigInteger.valueOf(8 * 32)) + assertEquals( + MAXIMUM_TOKEN_MURMUR3 + .subtract(MINIMUM_TOKEN_MURMUR3) + .divide(BigInteger.valueOf(8 * 32)) .multiply(BigInteger.TEN) .add(BigInteger.valueOf(tokenManager.regionOffset("region"))) .add(MINIMUM_TOKEN_MURMUR3) @@ -97,33 +107,45 @@ public void findClosestToken_emptyTokenList() { @Test public void findClosestToken_singleTokenList() { final BigInteger onlyToken = BigInteger.valueOf(100); - assertEquals(onlyToken, tokenManager.findClosestToken(BigInteger.TEN, ImmutableList.of(onlyToken))); + assertEquals( + onlyToken, + tokenManager.findClosestToken(BigInteger.TEN, ImmutableList.of(onlyToken))); } @Test public void findClosestToken_multipleTokenList() { - List tokenList = ImmutableList.of(BigInteger.ONE, BigInteger.TEN, BigInteger.valueOf(100)); + List tokenList = + ImmutableList.of(BigInteger.ONE, BigInteger.TEN, BigInteger.valueOf(100)); assertEquals(BigInteger.ONE, tokenManager.findClosestToken(BigInteger.ONE, tokenList)); - assertEquals(BigInteger.TEN, tokenManager.findClosestToken(BigInteger.valueOf(9), tokenList)); + assertEquals( + BigInteger.TEN, tokenManager.findClosestToken(BigInteger.valueOf(9), tokenList)); assertEquals(BigInteger.TEN, tokenManager.findClosestToken(BigInteger.TEN, tokenList)); - assertEquals(BigInteger.TEN, tokenManager.findClosestToken(BigInteger.valueOf(12), tokenList)); - assertEquals(BigInteger.TEN, tokenManager.findClosestToken(BigInteger.valueOf(51), tokenList)); - assertEquals(BigInteger.valueOf(100), tokenManager.findClosestToken(BigInteger.valueOf(56), tokenList)); - assertEquals(BigInteger.valueOf(100), tokenManager.findClosestToken(BigInteger.valueOf(100), tokenList)); + assertEquals( + BigInteger.TEN, tokenManager.findClosestToken(BigInteger.valueOf(12), tokenList)); + assertEquals( + BigInteger.TEN, tokenManager.findClosestToken(BigInteger.valueOf(51), tokenList)); + assertEquals( + BigInteger.valueOf(100), + tokenManager.findClosestToken(BigInteger.valueOf(56), tokenList)); + assertEquals( + BigInteger.valueOf(100), + tokenManager.findClosestToken(BigInteger.valueOf(100), tokenList)); } @Test public void findClosestToken_tieGoesToLargerToken() { - assertEquals(BigInteger.TEN, tokenManager.findClosestToken(BigInteger.valueOf(5), - ImmutableList.of(BigInteger.ZERO, BigInteger.TEN))); + assertEquals( + BigInteger.TEN, + tokenManager.findClosestToken( + BigInteger.valueOf(5), ImmutableList.of(BigInteger.ZERO, BigInteger.TEN))); } @Test public void test4Splits() { // example tokens from http://wiki.apache.org/cassandra/Operations - final String expectedTokens = "-9223372036854775808,-4611686018427387904," - + "0,4611686018427387904"; + final String expectedTokens = + "-9223372036854775808,-4611686018427387904," + "0,4611686018427387904"; String[] tokens = expectedTokens.split(","); int splits = tokens.length; for (int i = 0; i < splits; i++) @@ -132,14 +154,15 @@ public void test4Splits() { @Test public void test16Splits() { - final String expectedTokens = "-9223372036854775808,-8070450532247928832," - + "-6917529027641081856,-5764607523034234880," - + "-4611686018427387904,-3458764513820540928," - + "-2305843009213693952,-1152921504606846976," - + "0,1152921504606846976," - + "2305843009213693952,3458764513820540928," - + "4611686018427387904,5764607523034234880," - + "6917529027641081856,8070450532247928832"; + final String expectedTokens = + "-9223372036854775808,-8070450532247928832," + + "-6917529027641081856,-5764607523034234880," + + "-4611686018427387904,-3458764513820540928," + + "-2305843009213693952,-1152921504606846976," + + "0,1152921504606846976," + + "2305843009213693952,3458764513820540928," + + "4611686018427387904,5764607523034234880," + + "6917529027641081856,8070450532247928832"; String[] tokens = expectedTokens.split(","); int splits = tokens.length; for (int i = 0; i < splits; i++) @@ -152,10 +175,13 @@ public void regionOffset() { for (String region1 : allRegions.split(",")) for (String region2 : allRegions.split(",")) { - if (region1.equals(region2)) - continue; - assertFalse("Diffrence seems to be low", - Math.abs(tokenManager.regionOffset(region1) - tokenManager.regionOffset(region2)) < 100); + if (region1.equals(region2)) continue; + assertFalse( + "Diffrence seems to be low", + Math.abs( + tokenManager.regionOffset(region1) + - tokenManager.regionOffset(region2)) + < 100); } } diff --git a/priam/src/test/java/com/netflix/priam/utils/RandomTokenManagerTest.java b/priam/src/test/java/com/netflix/priam/utils/RandomTokenManagerTest.java index 46a6ac8e7..6a4aeda46 100644 --- a/priam/src/test/java/com/netflix/priam/utils/RandomTokenManagerTest.java +++ b/priam/src/test/java/com/netflix/priam/utils/RandomTokenManagerTest.java @@ -17,64 +17,61 @@ package com.netflix.priam.utils; -import java.math.BigInteger; -import java.util.Collections; -import java.util.List; - -import org.junit.Test; -import org.junit.Before; - -import com.google.common.collect.ImmutableList; import static com.netflix.priam.utils.TokenManager.MAXIMUM_TOKEN_RANDOM; import static com.netflix.priam.utils.TokenManager.MINIMUM_TOKEN_RANDOM; -import com.netflix.priam.config.FakeConfiguration; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; -public class RandomTokenManagerTest -{ +import com.google.common.collect.ImmutableList; +import com.netflix.priam.config.FakeConfiguration; +import java.math.BigInteger; +import java.util.Collections; +import java.util.List; +import org.junit.Before; +import org.junit.Test; + +public class RandomTokenManagerTest { private FakeConfiguration config; private TokenManager tokenManager; @Before - public void setUp() - { + public void setUp() { this.config = new FakeConfiguration(); this.tokenManager = new TokenManager(config); } @Test(expected = IllegalArgumentException.class) - public void initialToken_zeroSize() - { + public void initialToken_zeroSize() { tokenManager.initialToken(0, 0, 1); } @Test(expected = IllegalArgumentException.class) - public void initialToken_negativePosition() - { + public void initialToken_negativePosition() { tokenManager.initialToken(1, -1, 1); } @Test(expected = IllegalArgumentException.class) - public void initialToken_negativeOffset() - { + public void initialToken_negativeOffset() { tokenManager.initialToken(1, 0, -1); } @Test - public void initialToken_positionZero() - { + public void initialToken_positionZero() { assertEquals(MINIMUM_TOKEN_RANDOM, tokenManager.initialToken(1, 0, 0)); assertEquals(MINIMUM_TOKEN_RANDOM, tokenManager.initialToken(10, 0, 0)); assertEquals(MINIMUM_TOKEN_RANDOM, tokenManager.initialToken(133, 0, 0)); } @Test - public void initialToken_offsets_zeroPosition() - { - assertEquals(MINIMUM_TOKEN_RANDOM.add(BigInteger.valueOf(7)), tokenManager.initialToken(1, 0, 7)); - assertEquals(MINIMUM_TOKEN_RANDOM.add(BigInteger.valueOf(11)), tokenManager.initialToken(2, 0, 11)); - assertEquals(MINIMUM_TOKEN_RANDOM.add(BigInteger.valueOf(Integer.MAX_VALUE)), + public void initialToken_offsets_zeroPosition() { + assertEquals( + MINIMUM_TOKEN_RANDOM.add(BigInteger.valueOf(7)), + tokenManager.initialToken(1, 0, 7)); + assertEquals( + MINIMUM_TOKEN_RANDOM.add(BigInteger.valueOf(11)), + tokenManager.initialToken(2, 0, 11)); + assertEquals( + MINIMUM_TOKEN_RANDOM.add(BigInteger.valueOf(Integer.MAX_VALUE)), tokenManager.initialToken(256, 0, Integer.MAX_VALUE)); } @@ -83,59 +80,71 @@ public void initialToken_cannotExceedMaximumToken() { final int maxRingSize = Integer.MAX_VALUE; final int maxPosition = maxRingSize - 1; final int maxOffset = Integer.MAX_VALUE; - assertEquals(1, MAXIMUM_TOKEN_RANDOM.compareTo(tokenManager.initialToken(maxRingSize, maxPosition, maxOffset))); + assertEquals( + 1, + MAXIMUM_TOKEN_RANDOM.compareTo( + tokenManager.initialToken(maxRingSize, maxPosition, maxOffset))); } @Test - public void createToken() - { - assertEquals(MAXIMUM_TOKEN_RANDOM.divide(BigInteger.valueOf(8 * 32)) - .multiply(BigInteger.TEN) - .add(BigInteger.valueOf(tokenManager.regionOffset("region"))) - .toString(), + public void createToken() { + assertEquals( + MAXIMUM_TOKEN_RANDOM + .divide(BigInteger.valueOf(8 * 32)) + .multiply(BigInteger.TEN) + .add(BigInteger.valueOf(tokenManager.regionOffset("region"))) + .toString(), tokenManager.createToken(10, 8, 32, "region")); } @Test(expected = IllegalArgumentException.class) - public void findClosestToken_emptyTokenList() - { + public void findClosestToken_emptyTokenList() { tokenManager.findClosestToken(BigInteger.ZERO, Collections.emptyList()); } @Test - public void findClosestToken_singleTokenList() - { + public void findClosestToken_singleTokenList() { final BigInteger onlyToken = BigInteger.valueOf(100); - assertEquals(onlyToken, tokenManager.findClosestToken(BigInteger.TEN, ImmutableList.of(onlyToken))); + assertEquals( + onlyToken, + tokenManager.findClosestToken(BigInteger.TEN, ImmutableList.of(onlyToken))); } @Test - public void findClosestToken_multipleTokenList() - { - List tokenList = ImmutableList.of(BigInteger.ONE, BigInteger.TEN, BigInteger.valueOf(100)); + public void findClosestToken_multipleTokenList() { + List tokenList = + ImmutableList.of(BigInteger.ONE, BigInteger.TEN, BigInteger.valueOf(100)); assertEquals(BigInteger.ONE, tokenManager.findClosestToken(BigInteger.ONE, tokenList)); - assertEquals(BigInteger.TEN, tokenManager.findClosestToken(BigInteger.valueOf(9), tokenList)); + assertEquals( + BigInteger.TEN, tokenManager.findClosestToken(BigInteger.valueOf(9), tokenList)); assertEquals(BigInteger.TEN, tokenManager.findClosestToken(BigInteger.TEN, tokenList)); - assertEquals(BigInteger.TEN, tokenManager.findClosestToken(BigInteger.valueOf(12), tokenList)); - assertEquals(BigInteger.TEN, tokenManager.findClosestToken(BigInteger.valueOf(51), tokenList)); - assertEquals(BigInteger.valueOf(100), tokenManager.findClosestToken(BigInteger.valueOf(56), tokenList)); - assertEquals(BigInteger.valueOf(100), tokenManager.findClosestToken(BigInteger.valueOf(100), tokenList)); + assertEquals( + BigInteger.TEN, tokenManager.findClosestToken(BigInteger.valueOf(12), tokenList)); + assertEquals( + BigInteger.TEN, tokenManager.findClosestToken(BigInteger.valueOf(51), tokenList)); + assertEquals( + BigInteger.valueOf(100), + tokenManager.findClosestToken(BigInteger.valueOf(56), tokenList)); + assertEquals( + BigInteger.valueOf(100), + tokenManager.findClosestToken(BigInteger.valueOf(100), tokenList)); } @Test - public void findClosestToken_tieGoesToLargerToken() - { - assertEquals(BigInteger.TEN, tokenManager.findClosestToken(BigInteger.valueOf(5), - ImmutableList.of(BigInteger.ZERO, BigInteger.TEN))); + public void findClosestToken_tieGoesToLargerToken() { + assertEquals( + BigInteger.TEN, + tokenManager.findClosestToken( + BigInteger.valueOf(5), ImmutableList.of(BigInteger.ZERO, BigInteger.TEN))); } @Test - public void test4Splits() - { + public void test4Splits() { // example tokens from http://wiki.apache.org/cassandra/Operations - final String expectedTokens = "0,42535295865117307932921825928971026432," - + "85070591730234615865843651857942052864,127605887595351923798765477786913079296"; + final String expectedTokens = + "0,42535295865117307932921825928971026432," + + "85070591730234615865843651857942052864,127605887595351923798765477786913079296"; String[] tokens = expectedTokens.split(","); int splits = tokens.length; for (int i = 0; i < splits; i++) @@ -143,16 +152,16 @@ public void test4Splits() } @Test - public void test16Splits() - { - final String expectedTokens = "0,10633823966279326983230456482242756608," - + "21267647932558653966460912964485513216,31901471898837980949691369446728269824," - + "42535295865117307932921825928971026432,53169119831396634916152282411213783040," - + "63802943797675961899382738893456539648,74436767763955288882613195375699296256," - + "85070591730234615865843651857942052864,95704415696513942849074108340184809472," - + "106338239662793269832304564822427566080,116972063629072596815535021304670322688," - + "127605887595351923798765477786913079296,138239711561631250781995934269155835904," - + "148873535527910577765226390751398592512,159507359494189904748456847233641349120"; + public void test16Splits() { + final String expectedTokens = + "0,10633823966279326983230456482242756608," + + "21267647932558653966460912964485513216,31901471898837980949691369446728269824," + + "42535295865117307932921825928971026432,53169119831396634916152282411213783040," + + "63802943797675961899382738893456539648,74436767763955288882613195375699296256," + + "85070591730234615865843651857942052864,95704415696513942849074108340184809472," + + "106338239662793269832304564822427566080,116972063629072596815535021304670322688," + + "127605887595351923798765477786913079296,138239711561631250781995934269155835904," + + "148873535527910577765226390751398592512,159507359494189904748456847233641349120"; String[] tokens = expectedTokens.split(","); int splits = tokens.length; for (int i = 0; i < splits; i++) @@ -160,23 +169,23 @@ public void test16Splits() } @Test - public void regionOffset() - { + public void regionOffset() { String allRegions = "us-west-2,us-east,us-west,eu-east,eu-west,ap-northeast,ap-southeast"; for (String region1 : allRegions.split(",")) - for (String region2 : allRegions.split(",")) - { - if (region1.equals(region2)) - continue; - assertFalse("Diffrence seems to be low", - Math.abs(tokenManager.regionOffset(region1) - tokenManager.regionOffset(region2)) < 100); + for (String region2 : allRegions.split(",")) { + if (region1.equals(region2)) continue; + assertFalse( + "Diffrence seems to be low", + Math.abs( + tokenManager.regionOffset(region1) + - tokenManager.regionOffset(region2)) + < 100); } } @Test - public void testMultiToken() - { + public void testMultiToken() { int h1 = tokenManager.regionOffset("vijay"); int h2 = tokenManager.regionOffset("vijay2"); BigInteger t1 = tokenManager.initialToken(100, 10, h1); diff --git a/priam/src/test/java/com/netflix/priam/utils/TestCassandraMonitor.java b/priam/src/test/java/com/netflix/priam/utils/TestCassandraMonitor.java index 4539bd35e..d38b4ca80 100644 --- a/priam/src/test/java/com/netflix/priam/utils/TestCassandraMonitor.java +++ b/priam/src/test/java/com/netflix/priam/utils/TestCassandraMonitor.java @@ -19,42 +19,35 @@ import com.google.inject.Guice; import com.google.inject.Injector; -import com.netflix.priam.defaultimpl.ICassandraProcess; -import com.netflix.priam.config.IConfiguration; import com.netflix.priam.backup.BRTestModule; +import com.netflix.priam.config.IConfiguration; +import com.netflix.priam.defaultimpl.ICassandraProcess; import com.netflix.priam.health.InstanceState; import com.netflix.priam.merics.CassMonitorMetrics; -import org.junit.Assert; +import java.io.ByteArrayInputStream; +import java.io.InputStream; import mockit.*; import org.apache.cassandra.tools.NodeProbe; +import org.junit.Assert; import org.junit.Before; import org.junit.Test; -import java.io.ByteArrayInputStream; -import java.io.InputStream; - -/** - * Created by aagrawal on 7/18/17. - */ +/** Created by aagrawal on 7/18/17. */ public class TestCassandraMonitor { private static CassandraMonitor monitor; private static InstanceState instanceState; private static CassMonitorMetrics cassMonitorMetrics; private IConfiguration config; - @Mocked - private Process mockProcess; - @Mocked - private NodeProbe nodeProbe; - @Mocked - private ICassandraProcess cassProcess; + @Mocked private Process mockProcess; + @Mocked private NodeProbe nodeProbe; + @Mocked private ICassandraProcess cassProcess; @Before public void setUp() { Injector injector = Guice.createInjector(new BRTestModule()); config = injector.getInstance(IConfiguration.class); - if (instanceState == null) - instanceState = injector.getInstance(InstanceState.class); + if (instanceState == null) instanceState = injector.getInstance(InstanceState.class); if (cassMonitorMetrics == null) cassMonitorMetrics = injector.getInstance(CassMonitorMetrics.class); @@ -78,26 +71,36 @@ public void testCassandraMonitor() throws Exception { @Test public void testNoAutoRemediation() throws Exception { - new MockUp() - { + new MockUp() { @Mock NodeProbe instance(IConfiguration config) { return nodeProbe; } }; final InputStream mockOutput = new ByteArrayInputStream("a process".getBytes()); - new Expectations() {{ - mockProcess.getInputStream(); result= mockOutput; - nodeProbe.isGossipRunning(); result=true; - nodeProbe.isNativeTransportRunning(); result=true; - nodeProbe.isThriftServerRunning(); result=true; - }}; + new Expectations() { + { + mockProcess.getInputStream(); + result = mockOutput; + nodeProbe.isGossipRunning(); + result = true; + nodeProbe.isNativeTransportRunning(); + result = true; + nodeProbe.isThriftServerRunning(); + result = true; + } + }; // Mock out the ps call final Runtime r = Runtime.getRuntime(); - String[] cmd = { "/bin/sh", "-c", "ps -ef |grep -v -P \"\\sgrep\\s\" | grep " + config.getCassProcessName()}; + String[] cmd = { + "/bin/sh", + "-c", + "ps -ef |grep -v -P \"\\sgrep\\s\" | grep " + config.getCassProcessName() + }; new Expectations(r) { { - r.exec(cmd); result=mockProcess; + r.exec(cmd); + result = mockProcess; } }; instanceState.setShouldCassandraBeAlive(false); @@ -108,7 +111,10 @@ NodeProbe instance(IConfiguration config) { Assert.assertTrue(!instanceState.shouldCassandraBeAlive()); Assert.assertTrue(instanceState.isCassandraProcessAlive()); new Verifications() { - { cassProcess.start(anyBoolean); times=0; } + { + cassProcess.start(anyBoolean); + times = 0; + } }; } @@ -117,17 +123,27 @@ public void testAutoRemediationRateLimit() throws Exception { final InputStream mockOutput = new ByteArrayInputStream("".getBytes()); instanceState.setShouldCassandraBeAlive(true); instanceState.markLastAttemptedStartTime(); - new Expectations() {{ - // 6 calls to execute should = 12 calls to getInputStream(); - mockProcess.getInputStream(); result=mockOutput; times=12; - cassProcess.start(true); times=2; - }}; + new Expectations() { + { + // 6 calls to execute should = 12 calls to getInputStream(); + mockProcess.getInputStream(); + result = mockOutput; + times = 12; + cassProcess.start(true); + times = 2; + } + }; // Mock out the ps call final Runtime r = Runtime.getRuntime(); - String[] cmd = { "/bin/sh", "-c", "ps -ef |grep -v -P \"\\sgrep\\s\" | grep " + config.getCassProcessName()}; + String[] cmd = { + "/bin/sh", + "-c", + "ps -ef |grep -v -P \"\\sgrep\\s\" | grep " + config.getCassProcessName() + }; new Expectations(r) { { - r.exec(cmd); result=mockProcess; + r.exec(cmd); + result = mockProcess; } }; // Sleep ahead to ensure we have permits in the rate limiter diff --git a/priam/src/test/java/com/netflix/priam/utils/TestGsonJsonSerializer.java b/priam/src/test/java/com/netflix/priam/utils/TestGsonJsonSerializer.java index 809d5b420..a4fb8a09a 100644 --- a/priam/src/test/java/com/netflix/priam/utils/TestGsonJsonSerializer.java +++ b/priam/src/test/java/com/netflix/priam/utils/TestGsonJsonSerializer.java @@ -18,17 +18,14 @@ import com.netflix.priam.backup.BackupMetadata; import com.netflix.priam.health.InstanceState; +import java.time.LocalDateTime; +import java.util.Calendar; import org.junit.Assert; import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.time.LocalDateTime; -import java.util.Calendar; - -/** - * Created by aagrawal on 10/12/17. - */ +/** Created by aagrawal on 10/12/17. */ public class TestGsonJsonSerializer { private static final Logger LOG = LoggerFactory.getLogger(TestGsonJsonSerializer.class); @@ -38,8 +35,9 @@ public void testBackupMetaData() throws Exception { BackupMetadata metadata = new BackupMetadata("123", Calendar.getInstance().getTime()); String json = metadata.toString(); LOG.info(json); - //Deserialize it. - BackupMetadata metadata1 = GsonJsonSerializer.getGson().fromJson(json, BackupMetadata.class); + // Deserialize it. + BackupMetadata metadata1 = + GsonJsonSerializer.getGson().fromJson(json, BackupMetadata.class); LOG.info(metadata1.toString()); Assert.assertEquals(metadata.getSnapshotDate(), metadata1.getSnapshotDate()); Assert.assertEquals(metadata.getToken(), metadata1.getToken()); @@ -53,10 +51,13 @@ public void testRestoreStatus() throws Exception { restoreStatus.setExecutionStartTime(LocalDateTime.now().withSecond(0).withNano(0)); LOG.info(restoreStatus.toString()); - InstanceState.RestoreStatus restoreStatus1 = GsonJsonSerializer.getGson().fromJson(restoreStatus.toString(), InstanceState.RestoreStatus.class); + InstanceState.RestoreStatus restoreStatus1 = + GsonJsonSerializer.getGson() + .fromJson(restoreStatus.toString(), InstanceState.RestoreStatus.class); LOG.info(restoreStatus1.toString()); - Assert.assertEquals(restoreStatus.getExecutionStartTime(), restoreStatus1.getExecutionStartTime()); + Assert.assertEquals( + restoreStatus.getExecutionStartTime(), restoreStatus1.getExecutionStartTime()); Assert.assertEquals(restoreStatus.getStartDateRange(), restoreStatus1.getStartDateRange()); Assert.assertEquals(restoreStatus.getEndDateRange(), restoreStatus1.getEndDateRange()); } From 1b9248009184340b98e4e2c8f490a4c527c739bb Mon Sep 17 00:00:00 2001 From: Arun Agrawal Date: Wed, 17 Oct 2018 10:30:53 -0700 Subject: [PATCH 021/228] Fix NPE while traversing the file system in AbstractBackup. --- .../netflix/priam/backup/AbstractBackup.java | 30 +++++++++----- .../priam/backup/AbstractBackupPath.java | 18 ++++---- .../netflix/priam/backup/BackupStatusMgr.java | 14 +++---- .../netflix/priam/backup/CommitLogBackup.java | 9 ++-- .../priam/backup/IIncrementalBackup.java | 27 ------------ .../priam/backup/IncrementalBackup.java | 14 +------ .../priam/backup/RangeReadInputStream.java | 41 +++++++++---------- .../netflix/priam/backup/SnapshotBackup.java | 23 +++++++++-- .../priam/backupv2/FileUploadResult.java | 2 +- .../defaultimpl/CassandraProcessManager.java | 2 +- .../AWSSnsNotificationService.java | 3 +- .../priam/restore/AbstractRestore.java | 2 +- .../com/netflix/priam/scheduler/Task.java | 5 ++- .../priam/services/SnapshotMetaService.java | 12 ++++++ .../priam/utils/RetryableCallable.java | 2 +- .../netflix/priam/backup/BRTestModule.java | 1 - 16 files changed, 98 insertions(+), 107 deletions(-) delete mode 100644 priam/src/main/java/com/netflix/priam/backup/IIncrementalBackup.java diff --git a/priam/src/main/java/com/netflix/priam/backup/AbstractBackup.java b/priam/src/main/java/com/netflix/priam/backup/AbstractBackup.java index f0ecebc3a..7bc76668f 100644 --- a/priam/src/main/java/com/netflix/priam/backup/AbstractBackup.java +++ b/priam/src/main/java/com/netflix/priam/backup/AbstractBackup.java @@ -35,12 +35,12 @@ /** Abstract Backup class for uploading files to backup location */ public abstract class AbstractBackup extends Task { private static final Logger logger = LoggerFactory.getLogger(AbstractBackup.class); - public static final String INCREMENTAL_BACKUP_FOLDER = "backups"; + static final String INCREMENTAL_BACKUP_FOLDER = "backups"; public static final String SNAPSHOT_FOLDER = "snapshots"; - protected final Provider pathFactory; + final Provider pathFactory; - protected IBackupFileSystem fs; + private IBackupFileSystem fs; @Inject public AbstractBackup( @@ -74,12 +74,15 @@ private AbstractBackupPath getAbstractBackupPath(final File file, final BackupFi * @return List of files that are successfully uploaded as part of backup * @throws Exception when there is failure in uploading files. */ - protected List upload( - final File parent, final BackupFileType type, boolean async) throws Exception { + List upload(final File parent, final BackupFileType type, boolean async) + throws Exception { final List bps = Lists.newArrayList(); final List> futures = Lists.newArrayList(); - for (File file : parent.listFiles()) { + File[] files = parent.listFiles(); + if (files == null) return bps; + + for (File file : files) { if (file.isFile() && file.exists()) { AbstractBackupPath bp = getAbstractBackupPath(file, type); @@ -117,18 +120,23 @@ protected final void initiateBackup( String monitoringFolder, BackupRestoreUtil backupRestoreUtil) throws Exception { File dataDir = new File(config.getDataFileLocation()); - if (!dataDir.exists()) { + if (!dataDir.exists() || !dataDir.isDirectory()) { throw new IllegalArgumentException( - "The configured 'data file location' does not exist: " + "The configured 'data file location' does not exist or is not a directory: " + config.getDataFileLocation()); } logger.debug("Scanning for backup in: {}", dataDir.getAbsolutePath()); - for (File keyspaceDir : dataDir.listFiles()) { + File[] keyspaceDirectories = dataDir.listFiles(); + if (keyspaceDirectories == null) return; + + for (File keyspaceDir : keyspaceDirectories) { if (keyspaceDir.isFile()) continue; logger.debug("Entering {} keyspace..", keyspaceDir.getName()); + File[] columnFamilyDirectories = keyspaceDir.listFiles(); + if (columnFamilyDirectories == null) continue; - for (File columnFamilyDir : keyspaceDir.listFiles()) { + for (File columnFamilyDir : columnFamilyDirectories) { File backupDir = new File(columnFamilyDir, monitoringFolder); if (!isValidBackupDir(keyspaceDir, backupDir)) { @@ -161,7 +169,7 @@ protected abstract void processColumnFamily( /** Filters unwanted keyspaces */ private boolean isValidBackupDir(File keyspaceDir, File backupDir) { - if (!backupDir.isDirectory() && !backupDir.exists()) return false; + if (backupDir == null || !backupDir.isDirectory() || !backupDir.exists()) return false; String keyspaceName = keyspaceDir.getName(); if (BackupRestoreUtil.FILTER_KEYSPACE.contains(keyspaceName)) { logger.debug( diff --git a/priam/src/main/java/com/netflix/priam/backup/AbstractBackupPath.java b/priam/src/main/java/com/netflix/priam/backup/AbstractBackupPath.java index b1dd6f852..2348180f6 100644 --- a/priam/src/main/java/com/netflix/priam/backup/AbstractBackupPath.java +++ b/priam/src/main/java/com/netflix/priam/backup/AbstractBackupPath.java @@ -49,11 +49,9 @@ public enum BackupFileType { META_V2; public static boolean isDataFile(BackupFileType type) { - if (type != BackupFileType.META + return type != BackupFileType.META && type != BackupFileType.META_V2 - && type != BackupFileType.CL) return true; - - return false; + && type != BackupFileType.CL; } } @@ -66,13 +64,13 @@ public static boolean isDataFile(BackupFileType type) { protected String token; protected String region; protected Date time; - protected long size; // uncompressed file size - protected long compressedFileSize = 0; + private long size; // uncompressed file size + private long compressedFileSize = 0; protected final InstanceIdentity factory; protected final IConfiguration config; - protected File backupFile; - protected long lastModified = 0; - protected Date uploadedTs; + private File backupFile; + private long lastModified = 0; + private Date uploadedTs; public AbstractBackupPath(IConfiguration config, InstanceIdentity factory) { this.factory = factory; @@ -138,7 +136,7 @@ public void parseLocal(File file, BackupFileType type) throws ParseException { } /** Given a date range, find a common string prefix Eg: 20120212, 20120213 = 2012021 */ - public String match(Date start, Date end) { + protected String match(Date start, Date end) { String sString = formatDate(start); String eString = formatDate(end); int diff = StringUtils.indexOfDifference(sString, eString); diff --git a/priam/src/main/java/com/netflix/priam/backup/BackupStatusMgr.java b/priam/src/main/java/com/netflix/priam/backup/BackupStatusMgr.java index d600a3f4b..9a2441413 100644 --- a/priam/src/main/java/com/netflix/priam/backup/BackupStatusMgr.java +++ b/priam/src/main/java/com/netflix/priam/backup/BackupStatusMgr.java @@ -176,13 +176,11 @@ public void failed(BackupMetadata backupMetadata) { @Override public String toString() { - String sb = - "BackupStatusMgr{" - + "backupMetadataMap=" - + backupMetadataMap - + ", capacity=" - + capacity - + '}'; - return sb; + return "BackupStatusMgr{" + + "backupMetadataMap=" + + backupMetadataMap + + ", capacity=" + + capacity + + '}'; } } diff --git a/priam/src/main/java/com/netflix/priam/backup/CommitLogBackup.java b/priam/src/main/java/com/netflix/priam/backup/CommitLogBackup.java index 6e691501f..04fff0880 100644 --- a/priam/src/main/java/com/netflix/priam/backup/CommitLogBackup.java +++ b/priam/src/main/java/com/netflix/priam/backup/CommitLogBackup.java @@ -20,7 +20,6 @@ import com.netflix.priam.backup.AbstractBackupPath.BackupFileType; import java.io.File; import java.nio.file.Paths; -import java.util.ArrayList; import java.util.List; import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; @@ -30,8 +29,8 @@ public class CommitLogBackup { private static final Logger logger = LoggerFactory.getLogger(CommitLogBackup.class); private final Provider pathFactory; - static final List observers = new ArrayList(); - private final List clRemotePaths = new ArrayList(); + private static final List observers = Lists.newArrayList(); + private final List clRemotePaths = Lists.newArrayList(); private final IBackupFileSystem fs; @Inject @@ -58,7 +57,7 @@ public List upload(String archivedDir, final String snapshot if (logger.isDebugEnabled()) { logger.debug("Scanning for backup in: {}", archivedCommitLogDir.getAbsolutePath()); } - List bps = Lists.newArrayList(); + List bps = Lists.newArrayList(); for (final File file : archivedCommitLogDir.listFiles()) { logger.debug("Uploading commit log {} for backup", file.getCanonicalFile()); try { @@ -103,7 +102,7 @@ public void notifyObservers() { } } - protected void addToRemotePath(String remotePath) { + private void addToRemotePath(String remotePath) { this.clRemotePaths.add(remotePath); } } diff --git a/priam/src/main/java/com/netflix/priam/backup/IIncrementalBackup.java b/priam/src/main/java/com/netflix/priam/backup/IIncrementalBackup.java deleted file mode 100644 index 62c1d5e09..000000000 --- a/priam/src/main/java/com/netflix/priam/backup/IIncrementalBackup.java +++ /dev/null @@ -1,27 +0,0 @@ -/** - * Copyright 2017 Netflix, Inc. - * - *

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file - * except in compliance with the License. You may obtain a copy of the License at - * - *

http://www.apache.org/licenses/LICENSE-2.0 - * - *

Unless required by applicable law or agreed to in writing, software distributed under the - * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.netflix.priam.backup; - -public interface IIncrementalBackup { - - long INCREMENTAL_INTERVAL_IN_MILLISECS = 10L * 1000; - - /* - * @return the number of files pending to be uploaded. The semantic depends on whether the implementation - * is synchronous or asynchronous. - */ - long getNumPendingFiles(); - - String getJobName(); -} diff --git a/priam/src/main/java/com/netflix/priam/backup/IncrementalBackup.java b/priam/src/main/java/com/netflix/priam/backup/IncrementalBackup.java index 09a139da9..6395c5fc4 100644 --- a/priam/src/main/java/com/netflix/priam/backup/IncrementalBackup.java +++ b/priam/src/main/java/com/netflix/priam/backup/IncrementalBackup.java @@ -34,7 +34,7 @@ * Incremental/SSTable backup */ @Singleton -public class IncrementalBackup extends AbstractBackup implements IIncrementalBackup { +public class IncrementalBackup extends AbstractBackup { private static final Logger logger = LoggerFactory.getLogger(IncrementalBackup.class); public static final String JOBNAME = "IncrementalBackup"; private final List incrementalRemotePaths = new ArrayList<>(); @@ -104,7 +104,7 @@ protected void processColumnFamily(String keyspace, String columnFamily, File ba // format of yyyymmddhhmm (e.g. 201505060901) String incrementalUploadTime = AbstractBackupPath.formatDate(uploadedFiles.get(0).getTime()); - String metaFileName = "meta_" + backupDir.getParent() + "_" + incrementalUploadTime; + String metaFileName = "meta_" + columnFamily + "_" + incrementalUploadTime; logger.info("Uploading meta file for incremental backup: {}", metaFileName); this.metaData.setMetaFileName(metaFileName); this.metaData.set(uploadedFiles, incrementalUploadTime); @@ -116,14 +116,4 @@ protected void processColumnFamily(String keyspace, String columnFamily, File ba protected void addToRemotePath(String remotePath) { incrementalRemotePaths.add(remotePath); } - - @Override - public long getNumPendingFiles() { - throw new UnsupportedOperationException(); - } - - @Override - public String getJobName() { - return JOBNAME; - } } diff --git a/priam/src/main/java/com/netflix/priam/backup/RangeReadInputStream.java b/priam/src/main/java/com/netflix/priam/backup/RangeReadInputStream.java index aa1fafa03..bb771c151 100644 --- a/priam/src/main/java/com/netflix/priam/backup/RangeReadInputStream.java +++ b/priam/src/main/java/com/netflix/priam/backup/RangeReadInputStream.java @@ -57,29 +57,26 @@ public int read(final byte b[], final int off, final int len) throws IOException // meaning if you want to download the first 10 bytes of a file, request bytes 0..9 final long endByte = curEndByte - 1; try { - Integer cnt = - new RetryableCallable() { - public Integer retriableCall() throws IOException { - GetObjectRequest req = new GetObjectRequest(bucketName, remotePath); - req.setRange(firstByte, endByte); - try (S3ObjectInputStream is = - s3Client.getObject(req).getObjectContent()) { - byte[] readBuf = new byte[4092]; - int rCnt; - int readTotal = 0; - int incomingOffet = off; - while ((rCnt = is.read(readBuf, 0, readBuf.length)) >= 0) { - System.arraycopy(readBuf, 0, b, incomingOffet, rCnt); - readTotal += rCnt; - incomingOffet += rCnt; - } - if (readTotal == 0 && rCnt == -1) return -1; - offset += readTotal; - return readTotal; - } + return new RetryableCallable() { + public Integer retriableCall() throws IOException { + GetObjectRequest req = new GetObjectRequest(bucketName, remotePath); + req.setRange(firstByte, endByte); + try (S3ObjectInputStream is = s3Client.getObject(req).getObjectContent()) { + byte[] readBuf = new byte[4092]; + int rCnt; + int readTotal = 0; + int incomingOffet = off; + while ((rCnt = is.read(readBuf, 0, readBuf.length)) >= 0) { + System.arraycopy(readBuf, 0, b, incomingOffet, rCnt); + readTotal += rCnt; + incomingOffet += rCnt; } - }.call(); - return cnt; + if (readTotal == 0 && rCnt == -1) return -1; + offset += readTotal; + return readTotal; + } + } + }.call(); } catch (Exception e) { String msg = String.format( diff --git a/priam/src/main/java/com/netflix/priam/backup/SnapshotBackup.java b/priam/src/main/java/com/netflix/priam/backup/SnapshotBackup.java index e25bc9c78..e0e1ab103 100644 --- a/priam/src/main/java/com/netflix/priam/backup/SnapshotBackup.java +++ b/priam/src/main/java/com/netflix/priam/backup/SnapshotBackup.java @@ -31,6 +31,8 @@ import com.netflix.priam.utils.ThreadSleeper; import java.io.File; import java.util.*; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReentrantLock; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -50,6 +52,7 @@ public class SnapshotBackup extends AbstractBackup { private String snapshotName = null; private List abstractBackupPaths = null; private final CassandraOperations cassandraOperations; + private static final Lock lock = new ReentrantLock(); @Inject public SnapshotBackup( @@ -81,6 +84,21 @@ public void execute() throws Exception { sleeper.sleep(WAIT_TIME_MS); } + // Do not allow more than one snapshot to run at the same time. This is possible as this + // happens on CRON. + if (!lock.tryLock()) { + logger.warn("Snapshot Operation is already running! Try again later."); + throw new Exception("Snapshot Operation already running"); + } + + try { + executeSnapshot(); + } finally { + lock.unlock(); + } + } + + private void executeSnapshot() throws Exception { Date startTime = Calendar.getInstance(TimeZone.getTimeZone("GMT")).getTime(); snapshotName = pathFactory.get().formatDate(startTime); String token = instanceIdentity.getInstance().getToken(); @@ -103,9 +121,8 @@ public void execute() throws Exception { // All the files are uploaded successfully as part of snapshot. // pre condition notify of meta.json upload - File tmpMetaFile = - metaData.createTmpMetaFile(); // Note: no need to remove this temp as it is done - // within createTmpMetaFile() + File tmpMetaFile = metaData.createTmpMetaFile(); + // Note: no need to remove this temp as it is done within createTmpMetaFile() AbstractBackupPath metaJsonAbp = metaData.decorateMetaJson(tmpMetaFile, snapshotName); // Upload meta file diff --git a/priam/src/main/java/com/netflix/priam/backupv2/FileUploadResult.java b/priam/src/main/java/com/netflix/priam/backupv2/FileUploadResult.java index b356f0229..672cc50c6 100644 --- a/priam/src/main/java/com/netflix/priam/backupv2/FileUploadResult.java +++ b/priam/src/main/java/com/netflix/priam/backupv2/FileUploadResult.java @@ -55,7 +55,7 @@ public FileUploadResult( this.fileSizeOnDisk = fileSizeOnDisk; } - public static FileUploadResult getFileUploadResult( + private static FileUploadResult getFileUploadResult( String keyspaceName, String columnFamilyName, Path file) throws Exception { BasicFileAttributes fileAttributes = Files.readAttributes(file, BasicFileAttributes.class); return new FileUploadResult( diff --git a/priam/src/main/java/com/netflix/priam/defaultimpl/CassandraProcessManager.java b/priam/src/main/java/com/netflix/priam/defaultimpl/CassandraProcessManager.java index 86bc7e49c..0d08ae4f9 100644 --- a/priam/src/main/java/com/netflix/priam/defaultimpl/CassandraProcessManager.java +++ b/priam/src/main/java/com/netflix/priam/defaultimpl/CassandraProcessManager.java @@ -184,7 +184,7 @@ public void stop(boolean force) throws IOException { // In case drain hangs, timeout the future and continue stopping anyways. Give drain 30s // always - // In production we freqently see servers that do not want to drain + // In production we frequently see servers that do not want to drain try { drainFuture.get(config.getGracefulDrainHealthWaitSeconds() + 30, TimeUnit.SECONDS); } catch (ExecutionException | TimeoutException | InterruptedException e) { diff --git a/priam/src/main/java/com/netflix/priam/notification/AWSSnsNotificationService.java b/priam/src/main/java/com/netflix/priam/notification/AWSSnsNotificationService.java index daab229c7..4fcc8ef41 100644 --- a/priam/src/main/java/com/netflix/priam/notification/AWSSnsNotificationService.java +++ b/priam/src/main/java/com/netflix/priam/notification/AWSSnsNotificationService.java @@ -74,8 +74,7 @@ public PublishResult retriableCall() throws Exception { PublishRequest publishRequest = new PublishRequest(topic_arn, msg) .withMessageAttributes(messageAttributes); - PublishResult result = snsClient.publish(publishRequest); - return result; + return snsClient.publish(publishRequest); } }.call(); diff --git a/priam/src/main/java/com/netflix/priam/restore/AbstractRestore.java b/priam/src/main/java/com/netflix/priam/restore/AbstractRestore.java index ef86982ea..ddde21a91 100644 --- a/priam/src/main/java/com/netflix/priam/restore/AbstractRestore.java +++ b/priam/src/main/java/com/netflix/priam/restore/AbstractRestore.java @@ -154,7 +154,7 @@ private List> downloadCommitLogs( throws Exception { if (fsIterator == null) return null; - BoundedList bl = new BoundedList(lastN); + BoundedList bl = new BoundedList(lastN); while (fsIterator.hasNext()) { AbstractBackupPath temp = fsIterator.next(); if (temp.getType() == BackupFileType.SST && tracker.contains(temp)) continue; diff --git a/priam/src/main/java/com/netflix/priam/scheduler/Task.java b/priam/src/main/java/com/netflix/priam/scheduler/Task.java index bc9592a1a..d371240e6 100644 --- a/priam/src/main/java/com/netflix/priam/scheduler/Task.java +++ b/priam/src/main/java/com/netflix/priam/scheduler/Task.java @@ -66,7 +66,7 @@ protected Task(IConfiguration config, MBeanServer mBeanServer) { } } - /** This method has to be implemented and cannot thow any exception. */ + /** This method has to be implemented and cannot throw any exception. */ public void initialize() throws ExecutionException { // nothing to initialize } @@ -83,7 +83,8 @@ public void execute(JobExecutionContext context) throws JobExecutionException { } catch (Throwable e) { status = STATE.ERROR; - logger.error("Couldnt execute the task because of {}", e.getMessage(), e); + logger.error("Could not execute the task: {} because of {}", getName(), e.getMessage()); + e.printStackTrace(); errors.incrementAndGet(); } if (status != STATE.ERROR) status = STATE.DONE; diff --git a/priam/src/main/java/com/netflix/priam/services/SnapshotMetaService.java b/priam/src/main/java/com/netflix/priam/services/SnapshotMetaService.java index 568e798ae..8b71b3f0a 100644 --- a/priam/src/main/java/com/netflix/priam/services/SnapshotMetaService.java +++ b/priam/src/main/java/com/netflix/priam/services/SnapshotMetaService.java @@ -29,6 +29,8 @@ import java.io.File; import java.time.Instant; import java.util.*; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReentrantLock; import javax.inject.Inject; import javax.inject.Singleton; import org.apache.commons.io.FileUtils; @@ -65,6 +67,7 @@ public class SnapshotMetaService extends AbstractBackup { private final MetaFileManager metaFileManager; private final CassandraOperations cassandraOperations; private String snapshotName = null; + private static final Lock lock = new ReentrantLock(); @Inject SnapshotMetaService( @@ -128,6 +131,13 @@ public void execute() throws Exception { return; } + // Do not allow more than one snapshotMetaService to run at the same time. This is possible + // as this happens on CRON. + if (!lock.tryLock()) { + logger.warn("SnapshotMetaService is already running! Try again later."); + throw new Exception("SnapshotMetaService already running"); + } + try { Instant snapshotInstant = DateUtil.getInstant(); snapshotName = generateSnapshotName(snapshotInstant); @@ -153,6 +163,8 @@ public void execute() throws Exception { logger.info("Finished processing snapshot meta service"); } catch (Exception e) { logger.error("Error while executing SnapshotMetaService", e); + } finally { + lock.unlock(); } } diff --git a/priam/src/main/java/com/netflix/priam/utils/RetryableCallable.java b/priam/src/main/java/com/netflix/priam/utils/RetryableCallable.java index 813bc1bd4..f53f3472d 100644 --- a/priam/src/main/java/com/netflix/priam/utils/RetryableCallable.java +++ b/priam/src/main/java/com/netflix/priam/utils/RetryableCallable.java @@ -25,7 +25,7 @@ public abstract class RetryableCallable implements Callable { private static final Logger logger = LoggerFactory.getLogger(RetryableCallable.class); private static final int DEFAULT_NUMBER_OF_RETRIES = 15; - public static final long DEFAULT_WAIT_TIME = 100; + private static final long DEFAULT_WAIT_TIME = 100; private int retrys; private long waitTime; diff --git a/priam/src/test/java/com/netflix/priam/backup/BRTestModule.java b/priam/src/test/java/com/netflix/priam/backup/BRTestModule.java index 4aac0528b..ec29a9cb0 100644 --- a/priam/src/test/java/com/netflix/priam/backup/BRTestModule.java +++ b/priam/src/test/java/com/netflix/priam/backup/BRTestModule.java @@ -73,7 +73,6 @@ protected void configure() { bind(IFileCryptography.class) .annotatedWith(Names.named("filecryptoalgorithm")) .to(PgpCryptography.class); - bind(IIncrementalBackup.class).to(IncrementalBackup.class); bind(InstanceEnvIdentity.class).to(FakeInstanceEnvIdentity.class); bind(ICassandraProcess.class).to(FakeCassandraProcess.class); bind(IPostRestoreHook.class).to(FakePostRestoreHook.class); From 50da709a99a90c6c0ec6c8abc7b45e6729a72958 Mon Sep 17 00:00:00 2001 From: Arun Agrawal Date: Wed, 17 Oct 2018 13:23:06 -0700 Subject: [PATCH 022/228] Update README and add logo for Priam. --- README.md | 57 ++++++++++++++++++++++++++++++++++++++++++----- images/logo.jpg | Bin 0 -> 733089 bytes images/priam.png | Bin 0 -> 79546 bytes 3 files changed, 51 insertions(+), 6 deletions(-) create mode 100644 images/logo.jpg create mode 100644 images/priam.png diff --git a/README.md b/README.md index 475036d64..f0905d41f 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,34 @@ -# Priam +

+ Priam Logo +

-[![Build Status](https://travis-ci.org/Netflix/Priam.svg?branch=3.x)](https://travis-ci.org/Netflix/Priam) +
-### Priam 3.11 branch supports Cassandra 3.x +[Releases][release]   |   [Wiki Documentation][wiki]   |    + +[![Build Status][img-travis-ci]][travis-ci] + +
+ +## Important Notice +* Priam 3.11 branch supports Cassandra 3.x. Netflix internally uses Apache Cassandra 3.0.17. + +## Table of Contents +[**TL;DR**](#tldr) + +[**Features**](#features) + +[**Compatibility**](#compatibility) + +[**Installation**](#installation) + +**Additional Info** + * [**Cluster Management**](#clustermanagement) + * [**Changelog**](#changelog) + + +## TL;DR Priam is a process/tool that runs alongside Apache Cassandra to automate the following: - Backup and recovery (Complete and incremental) - Token management @@ -17,7 +42,7 @@ The name 'Priam' refers to the King of Troy in Greek mythology, who was the fath Priam is actively developed and used at Netflix. -Features: +## Features - Token management using SimpleDB - Support multi-region Cassandra deployment in AWS via public IP. - Automated security group update in multi-region environment. @@ -28,5 +53,25 @@ Features: - APIs to list and restore backup data. - REST APIs for backup/restore and other operations -Compatibility: -Please see https://github.com/Netflix/Priam/wiki/Compatibility for details. +## Compatibility +See [Compatibility](https://github.com/Netflix/Priam/wiki/Compatibility) for details. + + +## Installation +See [Setup](https://github.com/Netflix/Priam/wiki/Setup) for details. + + +## Cluster Management +Basic configuration/REST API's to manage cassandra cluster. See [Cluster Management](https://github.com/Netflix/Priam/wiki/Cluster-Management) for details. +## Changelog +See [CHANGELOG.md](CHANGELOG.md) + + +[release]:https://github.com/Netflix/Priam/releases/latest "Latest Release (external link) ➶" +[wiki]:https://github.com/Netflix/Priam/wiki +[repo]:https://github.com/Netflix/Priam +[img-travis-ci]:https://travis-ci.org/Netflix/Priam.svg?branch=3.11 +[travis-ci]:https://travis-ci.org/Netflix/Priam + diff --git a/images/logo.jpg b/images/logo.jpg new file mode 100644 index 0000000000000000000000000000000000000000..30ee04d4fa0cba7823d4d5368b72925023395b83 GIT binary patch literal 733089 zcmeF&XH*mI+bH?~N|mM}Hbg}cM5$5*1VyEag(_8P(t9Ai_udOh2oMMmdhZ~Jh}e7Y zk6p3%vgdibV0z)v3{X1g-|6o|v&!9-*_j|vL{{7+q zZtmZ8|Ho$kclhPnf3JIa>;HNA|5yBvIfm&OrD?$DU%;y*4frE~T^0ecLq%E9zp)I6 zmj=4-bOz!M5U5rch+Y3j>;eMS4g5QQr)!`yNNe{5NZYFih+RNB$g=+(yMh+>+x;77 z9my{P>CN8(f!Lh?yMZ?SJogG8F`xbpwH# z{^Vzt0mlvkEzmIlQt;nP|DX5&!)IXDzjY;&)&{MI+C4hF1r-Yub%XRA^;a6`8GJGP z$LR8+L1U!}*|gX!!F=~(rzNYF>MVU@ackME<-qd#6}*+$RmH21SRGswWbLumex22N za~oZopBvt7d}@1V(`CEU_LG~39eTI4Z*6w0cdBugx^P{oZn$j-_X>|<&wQ_J?+l-` z?Ww*geo6juJE8*)1V-+R*cBEOwmWoB*xvBq$dH4f(P6RS@ev98k|U4q&pMEQu=QXRbKm9b6<8N~ z79A+gEg_cHmW`F)0)MNtfH*_LV3}|XLXI3nT}6Mun&2Gp!GsJVmQ+n1p;WEE!VHQ#4kOC@sZ{mxocN^_w=TXvc{S97NC?A*Br=RaN4y|m);<|{r|L$1YMPrp%k6LJfCn|gf`%q&-M~k?|q( z*D?LLKOz zn$LAhR4(=V8w%8n#+IhJ=2tC-t()3{+EY8=o#L*Z?h8Gy`}F%a3sT=+TlOY@uZC*`m6f1Upr(41Db_6F@YI>QU-3zK!-^bGVK z>JJ)l46}^978x$OW87|nHH|UbZ1%_e{Nmarl}is;*e}ywcH6RdIcr7s%Ai$iSN&Rj z)2e@sz`AVh!F4X{&DX!TxwN5YqtF%toc}<3+s#Is-#T2}(zmtB5$%-aywAnM)!NO# z?c=ul?sFb}p0!>~Z@5p+_ITeAKW~4B9aaHmfeQk^?|i-MVbHbRXZD=fI}+R*(jM9r zRv%syQN2$dS+!q$Ky*-eNEjuI7R5+nWpUNIp(>m^X&3n3;YU$i=v8COY%!$Wt4I;SY6Ryc?xn9`U>_3VUDyxxuf@D z60n6hJYGWRAkL8Pk-t*SXq)Llj1(q}C1kgAPH`XcehDmvuA&HWwuB(9ts0ZvmVd8a zrrcI@pti7%p=z$5X?WZSYO-qf*2J`wv@+Y8+owAocK+(N>~Zal=*#NI4#)=khAs@h z7}XlH9N#*z>v;TR;S}M7Y`W{@shRtyKA&DRXLH8q?7?%{=P?(=7dtPVzWn6MpKB|x zd)zp5v*;G>cJ-Z}yQl9xxc}whqDSi=dp-&KC;4g7Gt6_|i;Et^XYQrT8oFTl@DDKhFO=^K0n0_)qR%&;L6Ahd>ir;aab>`8t6*?-nQ*9s*i` z5xrvlP5O5Y1cnht3ysDXRT{e)zcFbtJz}=X?1s5$aqJSyB{!E!Es~b4UG~_rae3Yf z*A?GZj;|uF-ei(KW>yov8>-5)MS+BIo->`k7{>JOJ^_$9pw!p&v>E?b1%9g~f zPLA4+*PJxYXqRYL2RCiEo7+0v$sR{MeY}=^TT&gKKGN zz1FVX;ngj3MIp~?` zbLQt$F0@_zd-=c>)zxp;!*A5w{Bk?&j^ggS`+*PG5AQy9eA4~Y=h?)I-7jZfhrc;D zzwh1o5Bon}{T%aU?wjj(@{ie{Z-34IzVJu2Hzq!s8>=xqPvm@$m^iUj;k{Kj8g#C^K>Q2=6|`gi6kaw$dL8iU+V zhH^DXHHlTBK&>WTP8dav6A$@)LSH8qt?0$wra1lZgxgBO@4taZ2}`Cqa0zct69)gq z;j*72z$`h~5oyMFcDMt1jgI$ALL1RRD|N6x8OpD}q30Xc+?|4%*Xm7)VA+a*rgGR5 z=|xrs93{G0_8s9aaEQN#wBWw>eU3`u%&**o_2m?No`;<4UVQrobb0H<@mo-I<0bVs zn4v0`X#yKi29~~q6J-tY)(D71=}SUh5|^z+U}T&ZpH@PaOl-a74oMnRP9#IlcC#8< zp_y$)^gpn*jlW9`U_Vubab);!<*e^^WJ)z`r3Pck)&Ib$B%VdwxLbK{S~I>85;z8{ z7eSu%f1<~TFBv{B*$Cv;{ZlHIRMeqAT`fKg5sW zOi~hZVJ)0ki*%y|6Bi>fg)~AZ@=eqOf)s`Id_)XD6BhrWf^iiueMtMMn=Tm=^T`8) z`Gjr~Tgf48A~us?_>+X+`9JU+0{V~!uO#gB^d=+_I+u_ryU9Aw^$Fd=gXgc|Kl9r9 zqwt40QHoyNJ60~?F|L(yGOrycpr;<}z>U+QJUZ|UTF8>WWH0){r^fiwTISjFxQmL3 z-aK4rRjlk2_N{myz8HH#FqBKePV$NlJiz|ts66)IZgOIlW|0mu=AMM(&~10mSYlr` zedxZ3Ev<)DQL!6p;5b*Tj{HFm7rRtCe!vHtBo24~jN2(*y>yhwVd_2_#hxDyI^B%* z?G5fiV&>ZCq(?BM<{Q}Un8Jp`*(WiC+O+)xm|N9Q_Yv&0BGqC6;RDm-VIDSo=G-g} z(>F2MS%BF)6e7vT81>{~%Fw^s*Je9nRyO1JM`McAaCa^?wJzVnk5Ix)d~g8s?h5ym z5EFf-yhDh7HoZiwMz@Y-pwFV~2l9?}pwD#U_UmFcv{$je@<6Z{?GCp`m^oNp5}`eremmy_=qqGgUi=@?Yw9r}q>DRbyh$2&M3@JYepu z-Z@rCAFLWz_s|l>t?YBuZ9;9xG$o8TeKeRXC+6i({^@x)-R`gZ`r|wke{f}SIm$>b=Ok|NO8*25N(p4BF1SOi7mTnnn8U`&A#xC z?lp-V>Y`4MSgRZ;$i5IpEjh1KzhW7Q)smR%N_^Rf2yP<^8*-eMldP+6m};mARPp)m zv?u2fgHNfsGv0MY6q^Y-{R?^VkOqt;dG}VPOb`{F!r)`XJ*}BeAmZuj8Pi9UdFtqS z8(R0Ri-UnwmrH%Mx5-cEF!VgqmFe|hL*lRT>nX{^qeHiXEs5qGV~&@J{OTq%6N;32 zdsZ!rfF17HD`|n{Do%;qU}B0X|2yn`$p_v7cxj>|Clz5Cl*A4}Jh5+P51?%HYIqkh zw`bte3q-H38L>5CoxEHyjgKVF@Op5^iw8L`aCr$U*_pV|U3gX&uFyW5HH**DPvMdY z*y&aYm9w?eK=_HJTXm1$$gm~GaocFei=gapRO#X6EHY(y=UFD75@K(_d`a~)(B)(h zSDuIyN6O;cX@W-*J83enM;MF$>iUDU4+==IX_JFgw|WI|CUrESTLR<}!Apfq^FM;TpRkMv8Esp)sq9 zcxKX4IMYRH-OaCPrHSpi@lE>J&+PJgqx>4?Tn#h!F(X$o6nKn|m80!y>9f*LhQ-Xc zB-7)z!h@qtEk%6o{yU;%&Oco|>~^+J>w|nVAH*nIk*hVH<1bR*R(J3c+Us>tv- zGmW%oB1*X6)Jx42uWd3$IL={>WMUdvt^GE6c&1@jf2;6C#NuutkUB_c`F%bM~z}S=;i&I00|ANX|+qC^;Cr$ zxiMrU-oQjd2(-OktM)Qu4K{nQTQm#@`&tQiAcEKL5WYd; zbhJcCm~H*qsu%drbq{M|aqrls3TK=vY_2LDJDjyj`~r(PFef~Xg>TOhBx8@SHx@j{ zebKohG{K5`d3D{aWT$S{xjUtp8VPl$@!Xf)ycxAjC|Q4eikH0BIf#L*od6i-2Hn5>sUcPU|tV%WZg&p z0@j0tTX`>W=x#+#rz%hBs6Z+k=!I1jc~~Vvd_$U?epr|)dLB8zPZTcoaplenO4jS} zocKcvyLroSx4PzPOxqq*r^_EThte9P?;5sLoD}b<6{U>}E?3hc{dp(k5#CR^(XvbH zeshV!pStV0nfQ%eCQA9>A;qw~v`34!P>O4JuW%7v)+|g56lAN(`|j~#>i2lJac?MzcfPhD4RmD`S9qGF}n1}edQB5e1aqn8CMIs*0)dB&}3ZwLq9 zaA(~QjzIWbcaoEZ=XOjgUCzCsC5gTZ(Ee7F~TJiR?R zVGZr;?sB4p|LFeUnBoT-D?6&7VBxNoXlOQWwlM;VEqbX^K(8hdYWBktLJlbia9y`6 zvQhZil?cT$#8VJRdt@g5%G~y6gFSsFE+L zt*(jU15&Fr`v50j;}t>3d( zqS0)2br)%BKCDU-UE*SN*76gvR<)|;o>?>YT!VV@JF=+u{^sYk+=1ELl0D#b{0S*GfG?ap!`F-nij*% z0vzk7U~LYPx~p*DCZ!<_;RgZL5(m_1g+NbUBpJ9)zeoJseS&6KKGnXILQl7DzE36})Ki<0 zuJ7=wyG%kmtg4wMU07mVha>})M4>;RR`p*NdJ==W?(>?;{ zc>g!GFBxYY7AsFN!Ty@pBxksvq;`mi(jq%P0ku5>Tmqag76nyiTm@YT3q?cQu7aI~rRQiv3?y(=v#E zbxl?k-(hq02I<HC%=O_-zTM(c*K{nu67Rg3)j z$}6=K4nYc(JaDO~+L!SfP><0d!KvQDq0jg)9pc_IrM)c+J5fi2)FW+M_ABbHv{?D) zDB+EH4lar{@@-286&*|+&~o8rw3#5KH)85LzNsUBEVdM`xiqoE-CY7a7>#ObYTmcy#v62e15q;BQt$4l86N1mdfvs-*3Xj32 zBYZ2Ha}7Uuea1QJQOaUK#R7_a+`PdZA|EcT6=kRQt*A!FeeYZoq<&|rVfyV^S*_ZmUU z4;y^zbHyl=-Sz)a8UUpMROp0xzXb$6RMM4_o!D#9#)9ImNodf@R<&8*+F zAxHH@Tx}w$_odti)GQiA{yF|?JP;B!pdTGA`rKeK zE0sO7FyTU@ZV4g^1mYM1YT&|VTy?>Q*-7k<6wp)`CMz;?%nYNo!*AdR`jz97o_{cA zD{pr1#;_KScb>q^0IC#F|D3x*J6B+Q3Ql>Ge0H*kjEwXexk6IzSld5AvUgn5EhpKp za_O8PYw3bJ%<(CJI*$e+jLx?3Eei~15;)z-9mk_tXCpTZH!>4;xb!g?Y{%HHIEHjp zPlq3atmoE7!ruddxK@C2Jkui!D6l!1Dv3?LG;v7yIWl;tM&P(3ueX@L#Idu}k)ODF zPuo4Nx!#jjS3(S+QUK*Q_etea5Hu}T&ZKCMqvby$DTC8hZ+7(b(4{Y(Ogmhq>sHUU zj`6_yu$I+?5kLt6csyXhI*j}obwQ_B_ z#T(WCtnnk70BRCYTTcJ!E-%PGL2X~25<8ZpIT`t7fY)>(U~_kr+S^Ic_FB2c3fAJq zn>X0s%q0R(q_|Ik9R5B%Jv3L4ej=dHB&A?Xx9i0I)dTMBmje!VebAISjkMvEd#vg- zQ+z`MTJvHe&_?mJ(4hALFdD@EhCl2UWMte9?hM^m{{}9CKBuQ4EMUx%2t)^bXY5{N zF+$J#3rdRkYN3baq112BLaYc^ul|HI;D3(%g>J*|t$PSPha0BiVMttbQ9C>YXB_<( z;fa0ay$y-S*;|xh4>(&a1HEqAs7PILA&NdRa+eX4U+K?4{3;A41pA;tOM7=G2u@ zf8ZM^%e?9l405-{WwZ}P`Epx1xq8dRPH=qH`9Y%!Tk)7OqEc7zi3Em#c*KHjP;<_q zC?{9|tHR3@KFX9Vvq6P2YM#559n#!8f1})8eW?EkctxG9;z5OBbv)6%a%I)^d{4-3 z@$exOlqy{5c^ke&ptmd&iD7MgT3DJhAUzjQ7Sr{v_ig#!)^@oR9Nu`3U|*4=%FT&?g^D2B!1R%$KUi)ICFGwF5~qLU~UY2fqvasC9d; zs?x5p@L)ECq?+>F2eYo}U3MA~#X9!5zr_7g%G~YJji<@oBV`+>_EyQt14aZmC>YXL zo~u_e*J*p;H^im&kmpk9T)qFYH}ID%^rMdAd-w6Df0WE!>*#u3dhT4c^jg{5nQhov zu;TufFyzu~~@M;)*So0eHCT~nz($*95wmcQZ@Z!ytlkp|=mf@<){STv`dYyNdNXwl zX^IY}a5KK5S!8j9CB~HW(B%)-iuBm@F~Of~dD8>6wCdz!8cJI{rHMwq78>%`AV2f? z@Ntwgr!xHwip`RTC!#koH@Vbdb}=59Mc{YP7GAGJ%&YHBj3MUhPB&deyj3pZo<%N@ zsbDf>h=iPOf}9jy4?BrE#vgG>K{NOavj$u?E%53*!lL`@csgQr+oMJy!nWCt(}mbq ze+k-!P}JN>JBVDZhz{F_yij$@#R2tGdd6%X+fOUKQU@=ZSUna2uN*Q^^WfB;BkXDT zX!~Z!Tg0m7yrZLtnucAW^T_x*w#xz(vHFU+1J;gKb(seHc_wN!7p^rO(l7yEK6aY* z2Yzs14iX3-=uSQAjtFUc9V$UC(7>D@ptja5GCz#zqYYn@!#Zwsj&#F1FHAI;!_Lg| zSnNkV~PYEQ?LjCqylTk4LE$b+|;VA}jba!OqnParKnEHM@>Hx;RCX-T! zK1GWpPoU{#PsoeVh@^QE9nITgN=ipxaS)K5Fa(3&)CbtNr+L&h47=VV6muG>dXj8G zHKUv%9V9<3r4uJepAs(-kCHrh?;^$#k30C1>`69;EJ_ct`_yA{kGQ_Oku)RtsjwzK z({Jcu(&nCvv+ zn|uSwg%B#WE>6Q=5~U|>z>5W|gO=mfJnZHP0+M&esGM|*G=9>Ec)gw3X-w48=vDO- zd>iJ7+woPkZ;Ax?&DBMRF}U}#l3m?+eW`e}0{=kbZghsUiS*;dE27`9L5DTrSg%5Q z3*Xp5Clun%H2)O7!ZoRf4uf#zs$IL%an#z9%_Mw^?D(R^L?PLJ>NDZuNkMxu{@X;D z)E@6U1jV1iHT7;PjKu|aRK!zpYg)eUT7`3I+P%3D_ea*bD3oYJj+|OUU|kAp8^d?Z zIZ6b$&nHgeeQ+7$3kq)IbceRaZ^Yj1y|VKZ_G9a&&4IWP8Fx`RVV0aXN))lm4>S=4 zSzr^vE?ydV0^QCLRSfl<^`M=_ zSD%9B%GjY~U*sC*DN=Cu4hD`$Jam?hCH(eFqx~ct-{40-K}0XSwD8#LvT1^4O? zvZr|%jxTEsCkFnCA!C&u+eI&6JU?ViL($c~?`aEZ&o^wL&Cu`Z=F{c)xc+M#m~v8O z%Q`JbvNtfDr3lyy`f<^@tO6QKPfH{MQ^>U?+rLp3DPPK_5yNYqod6n1tfOU*QK-(@>_js z63V8e#}!%SC8Y0Fb>J0cXC)iKw~wq6zOR@J7YS@C$2>0aS3`!a4hY~-1!xaXgm}`j zxjGm5T=G>$LGnmel26FE(o3QyWN7LVK|L}rOvU?&?DH_@8K7*eEciQ7Kea5lzUblR z7xHJ6jpCN7&7_Bfm*NKEbjh%=nD8NG9UnzF9QuI!9RI_e#QB4NW3`aCn5eHc$1%cq zHVw&oc-f+C=?YE=eo$1$$}H{^xU0f0v3=1CK3*`K)XC-WG$FS+X53Bg25dV{jg=oqhkZr+C2KS0 zyZW>APNT74Qk<;XkFyqnl(&llcvt18lGbtFR@sIWveP6U+fK78#kZ`2*gJT$I`t|dG^Wd4U?aNGI)55$UDEl) z$iueOKa^ZPN#om#_D&qfDEY64CkoDTr~AbT*Vy-Z3WBp)2ilKqvtfqS1zK%m-r|if zaA25WFUpA;co~IWraoT28fKshD(}qRTw@777SpTPU9o3ZOx4%QZH`3gBxJFLr`#LD z0LJS)MqQUZY^p_?(sk;;AO@h7wRH&WvEu3m1UmY;>@T8yr$l-lsk8NyM1?H2@U1$H z0!GSkKxk0ar$%?u5KUXvM5IENY-Mr@o{gzh-N-n1c7Q$dRTK z>;Sa_P?l;v4w!0OCu6}YCCa0W$;?agLi*k)q4XBbIZ!3;rwp=*-;{E$Hc`I&#GC1znR-)RlI+q)=PeHtpgK8x7dHT<_m?Kjmxw}KbT}d z1)@Qcx%wd0Bl5YLZ#5+q8H$UFgBinBkE=w7rX)Kh@qrse7NXBva|P!`qGcf>LzX$9 z_5&(FHPUvJoKaKQyc7IW5z@fO*jJTM7k?;2Jg&SE@L0&LZr*xO;3Jz}Rwz8kasb7$ zPocMqM^rieGi05b72Sqlh5S?7Kk4tQ7HaYj8HN2cgmgNPi zWgkKph<#MMkFO=Ely8Rbfp^PK_T$sV(yKkr2ZzKiotpxpgdh!S%LhJ7k+!T< znnHukletDjZ9PN-9PNw}e%%IBg}-ueoAk%6tDOewf)nxe3vrUU9X zpe)s<9pa>;a99zd-Ffmn;t zAli7_*KR!AviV;7hTQ3fc?~MTu5No{LuhFAvxbjer{tUK&f2c3LP$=SK9to{fx9-A z4WLrnu8e?Smzw+fJ#z^S)7_}U&+3djeuplq?rLrJ;>%t&1a5p+otQ405|1@rpD_#6lk7Vv^{?u*Ze=;1Bfp?m^2DiZP>SmS3Wd~4&e_`!VD;6vTdspkfE} z`|XNGdstnwN3z>*wRaqt)EsH83%sa~Yc_J(tXfp#Z@seCS)gZFR#QU)S{nvX?J1$1 zNiGMa&zzK${fto`UsJv{c;Sdqxx!1)F9ADmy3vylu3x&hryiWA71@3hQV0T3UD2RY zhtml7Y)0CNDfq@%^YH}Otze&_RM;)A-aZiAaMRDOGw_{DTe|w;JG2d3uOt2f3J<7= z*@gIz897stxS`mkqxY~^gLe%)#wK~4>p@{1?7Ta@u-+E_9cQr(+GkqyF;4(>7Ennu zeGHF`oXLf>+*s2Q0d+xeOh1YI!yD9XMJ}}ywvUk!7Vq2skW+N%&E?ok5Qw%2Q1Fui zL3jpn;uZH}tn)CGV;T(a%VcT1cXd8zUbg$$R>@koOwn4vShZk#(|Rm0a+DSTsH*86 zxkX0%gqhSoHg2d|^dY#hmn01JhIjlFKdXx3p|!X%?JrG{ph+URoZYx~KJ2 zoQ%uk*)<`tu)(j2p5W6xcVyqZ>pMbZ2kf~mO_E2JRT@{;mxVjjZa6xiSb!Qk!PfL- zyczq{Xc{XR#MY+-Kk7-Td*D6VK3ns_zO%(f=4^Sl8P8t7klR4P0j($vsF;;@p9t>) zXM7rKZxhB=5Bjv22EXW8+-&GQ-Co=fZa=OWllfS_Yc^&(EgWlDfO`a}2{h<&ZiR74 z^xXA|(Na~9ZDmf`eOVC125f@Yfqbm+$=wLOTbXg-0PGQj>8^!vh1Qw1qlRHg56#N_ zQHy5B%Ds?}yIjBFX)r96d zkRmJuqqT~CdOTnL(b(*-M9CbB6q4;K91;Va0p<14SF8d_2j@S$qOAffsRPGf=EiQr< z(JQV;=hctKO*}1d?8i25C~W95-~|?MY?+7qlnga`rJpQ2Quiag1#G4qccoNJDfAcD zLT=I}*R1mv&Nhu-%jZnKZrV_2J|f~CD5~mffMu5WcWy~vRr;}|Fx<1escDgGL`7n) z_Ttl!RC>?V6}jT8H^;u^+nh^jj3`jgq;oJuUK8h_&Bc#~rqd9mir$Z5z2ylVS}wmU z7S)_t`~sp)pS{FFeJs`=QJ|idlK4>B?qD zXUt>BoRJA$23vnYjM#-(IW&(bMQl=CMaU7`n8k?ei0F!8q$@(03PH9b=7XbANyvn) zwP+ghm606hiC%H85@ta$9C!sQB|WcO06RgvNxuQNCHw)$!|MqVDTfdt_=&v*$QAhH zt(#DB_>x7dFu(DzGf+r8|7E`}yqQ)YiRv@#t^zx z@s?}I1Jomn$mn^p@9F-Ezsk|xoJx@#TYVd{TneEshbTpwvPIB?f+a~tW_-hNgXK6)MW3-~Ka*HZ~{I~M7vc7a3gd+DzoPicdU3PcC zDn-Ayq{3%K>x>;y2gomG9)leQ{JQ>tuXfwZ>ncjxBFIZBt(!(m;wtCsS&0sido?e1 zheBUhQ?|In{bW|gxyV7v;*;mf1yi58R)cqrX~`~vpA8%)?W<698x}WJLfSkNWRUXa z-9fLQB(>ibBiJSRjByn*km5G|u-y5)sx!U(?y2W88!&0Iny9MSJu*>jR%zQ$POyS( z>P7`&p?*G3z zI^gm^JaJLQ_$Ck1n#$|C`s6P#-H8lhJ#2SNAkhE@6IBz6VNKWu{8!l1{QdZP`0m&g zJPtk{0LFJALToz-ClMcY+ejP~VoZrA5}TUY_!Wc`!euxa{vV7tE)egUXM|gdYl(@) zt;8MLF@wv-+1r-ku{f+=05JxaKk@+kg!8rW3RcRt;G?lF%zdbrn1}SnoZp!9w7t<7 z%xmhp9k$qgltZ?TxEzXHZu#7b0RgYZ}_Dy#Oz zp?;LA`li1Q+FB94@iN9q!qY#2TS9o#{}7cn#!&U5Tn2Zthme1Iyx?WXNA0j<#>lUl zy@z~I{*88iy{N(Z1sjLaV-lSHFKiWYNxv@2bXKBTg}gaMW^Y0^kG_XJLh=VrWHFE} zJqZV|B0qH~{f?pvo6s8>=v+y+`Cn#=!Sk<7u_<_u4VEOytLQXXZLQQ%0sg!QG=|)4FgDVVa0ouCt0F;(WYBE}GTDK&z=x$FuhkUDW> z1DQyXhF>G=le;~cq#tBQ>k!ID^1Kd_Vud~00;Yva{zzU@_lRgDLyDVVYw2UM5BFzk z8VSYG3*SXr!a{m(B^fYlti8y#%od$rWFKsoW*;?EMHPRiWYioWI+2qVTS`}xN~?TR zMu=nL>aZDNq!8=zm>4Fgv34QF^HLWikxvUg7({yoXR!>jle@o0;A zaywzPksA7eaJb=s$1y@^EyUWE=qWH=@RC@FJJdKz{x)?(w1V_%EE8W!d_P!JTu5Br z^E)|^P}+$Ktsf2$UOvFw>kF6RQ+c)2C-7nlb0MGwOhYEY8a%G2y^A`D~&GFX6tB;>x~ z^+NjMmvJ{iJ$A#{AED=*8+_hC}3la2Bj26C_Mu+|4_ELSbKe7K( zbYja`eDaE*?My5w*XbH_C26zeSJp6T7*N~LAjLv~m&lGj%FE_|g{|j^xp%U?Sg+a7 zVqOB{uzh#kW5AdOPJ;|fhHAMJlTRN7B?3JnfB6c2Qq3y59+#tVfmX6lR;3=3u&l+i zF-I7~Le?%My@&tC2~UsUcP|fQw6HTlC#jdwL|FuHxcMV3pYv873wg!1uX}RLoJmy< zN55ojQ7qYIM_(^H?374rmtI&NL;t`o*IG^WMNd@8c#%CF)Oik~BO6lA8fq!Xk}<6s zZ$%^NlMPxs=V;?9PbYg?Ky}mdWO@c0s&#~N0{vF{oo6w2kc#B^4i-ZUSdiW|Sx$`G zo!QZ=>F~C;ok&_~^D)O4)cNXx<$LHl>})L|B?GgBx4iyG0RsEA_H|)P2~4S1G<1Y4 zPc6O{IbUT|y2$Th0*K#nv)DIyZHZ z)D>;LuT%^?EA-VD;?cTxTLj)-GX~x`q6GNcas=u9?pAC>jKnyK!wSxJDicdRR%2)1yN#5PT2jb z_E8CZo0sxLDP=8DSzNY3f2KCLtO@w{tyDlU+H5PLYQM&jvaM{tx}U6-{!*1d{1k0fvy-?as9m8T zpxvTnQiA*15Sa;~)NrBfJw6stU1$(?rgAP|;2P($A%k`Uhfs`o>z0FeIwD8p=N!v`cn^JMR`C zZRSR;t(UyvWEwt@GD(L4bq!GZjp#a6X;A%UWp&!yn#c0jQC^DkRrNuuWviuzZeEga zl9;vEC2IvUM*fmck`z!+&>&1_<0{ST(&eg|#?-Xl8bp1{p%04W+RwXgR@K$8T>nVO zieGC_h!w(3M#m%zNqvBN3#b_NeD}+en<`2BpR|&i$d)&U)Cz~@^SiiJnT<}a(-QyM zsI?OD4xyn@kr?rB%SQSF^rO|oP6Zanrurrez8yw&u?la5b+mOBUGQD3Ib3|jVO5iP zNsA@7iCi+S{XuP9ZVUnuuxL<)-kjkUo7NINvbr(>DeFh}*v9iuNV*265tzCqYuz;r5_^$1J4FmYS4$D;; z_)W{>Q~`MAg3+3M9H59=fO6}(!v-Dm>u6&J#XoA1(NBb}X}m~dZy%~>QM()h>fES= z&`f(yw zSk(+NKP)_<_>FrCC~q{V{7Ls-<>q7N?auP-cwn?vm09S0^%05Rc9F_dyvrf8hAi5< zB13tNvqg84VjbQdQ2PLNsjE+Y@0e*@ld2^17+-XLLH-er$a)GzRX~S zK)IC@t_zcw;*SC<8BjG{_;!P1T5SjoA-=L1*0ea3)u2#wx5w0NQ`I@_RaVJ&tsqx- zadzo)Wq0r;fXYRKj(s`VQLsGW>+zm~$$gTMPlcF()&XHrg3FDbm&MNOQo7C*YfN;z zUli{Ffz);t+dv@PGc+iB-~_AuMq=-{u>5i4hM@=GHG!Z$AsFjw-1Qy&XENIIiq-$McEn5+_Fr@qZ&j`|I#)17o`Vad1~*I|t{wp4hqp=VAJx zm55se@>1PK+X9LKsD~44nX3|S4Li~cBD4CkXo~|8o!_a8T_3i2QNZiJYI-T1X7!r8 zq*7Z3s<6{7Lh=CMfNV z*3B9^CIC?H0p&h=qV7&o@E}O}JF>ecLh(GXtNo^Y%5970tE|k1+q6||Y<{_#!gnYl?-AtjuezdW356$%im&N zINq*+VV)YdKtr%g&Yj4KWPcmX$=kxxuRE20oe{w}T?nTQRvamIr5;VqD)~rp4(64$ zk{ul};Ju_S69dQ=V&<7k$AohGeq|0{x}nx4FHMw4PcB%?zYKOM{Kzd&0T*9qAKm+| zw2@iw=vE%em^X>5^rdvqt?M#*wy%&w| zJzR1_kiGR~8G_$rBB>~++?(yq+|j$H=iITx4$o?2&VCJp@-EL$ZChqj;8SOwbfaiT zb>^O1B`&hMtu3Y3rTb0Jg0E7wPvJ8V6Sul2vlPR&3PiT7x10PsSI~he4au+7bSBvp z4yfsSl8U=jjav&!qZIunU%}DTH8be+&u9F)o@QB2zmPX%uN?n}Y@fSoXmtrC-?`^R zBDyf51GD=|@jgxF))ghM6-ZOd@(ms7ruf@ zA%5}0upFq)PBHv2^pssYq6Su^?}mTm zW94haX))H|G{MaPor*rb!6pnuk9X8y4jw?P9$s2l-x0577QJX$&m)!SHEN^WO6^oj zbEIYI%F1YDdA~enhpfV+Dtc3VWr8@$&;V9LoE_3CcsWdNXfJZ_BXawSlRKM``6bj= z?d)fzR~u)do|SvnAKu{s=GI==w6)@z6k+HCvmw45Tvm{M5>@}baA4vLSHI}_a6DqN z#H!CeJG`{8lN6OvcCF3a{}kAxQEL0LqFB1$@CbB*w4hH1ua|wY_8e{?_arM0mzqz5 zF>uQY{j&VA^F^NzT4R5dG;Y6!iz@S4zYec1571deG^m*90b`bzeXl9Pn3bPpd_}(p z0hV9{)I;Abm0Z?qg@)L0_{{ifjooyL@!5P zL1`+dkWW$N>1xy()RD+%C;}?i#|~|aqOHrr6rpY{2*T|{S9Uyt_tN$#Oo8gw3hF$< zhWw@ig9s&^Ns}Y$iH{dNYx?2zBdR&{l+d3nsDdIM4R|FlV8kObrwAw^7<) z=ee7~S#Vvp*U>?EKJ$N3b=PrKCI932L0UvaK@coTR1^>e1t|du3F&Swm)qUl-Nn6h zr-0q=+8w)gtlhEdy4IS%60%%2?Mt zYyK@J)cC>f1p5j9>BvnvOk1ZnSZ6O5{|Ka z!qLK4tOD;Kv6MArRVALnKHFU;N|YSc928y>Utv8EhKsHu_6p3xyJfb5EaCi=Rza-5 z6doYZ2y(pFh^7htuzDhV$7$A)0zJ~&xGL1i5 zp0ey9pDY{n4j0^%4OmwSojET$Zt}<5h^j68summORNh@P6F!xPG)*qqzzfyiT-(W8 zq4QbBP>1TnEQNmBmE|qGjtdFgzMKIQ{2z_)H6T% zG}pB&a+w_$(LT+4DNmu@JSm+Y##OYfH|3Q4>b zU#=V_9SL5b+)U<7hiPDBKA_TBV53aCiJ=ijYRVY~LY>-*A*gp(9%2Y{*C-A!hT}RF zK8&Y}dlkXVh11ch4rUpkLOE832#vFd&cCa=EZl)#rQ`@+)P*RL1%5edc_H5%H(&NE zKW1^Q>^y(|^k(H0K9Jul^x|0Q7pQa^JnxwjuX>HERD>z(Yl(6z`TZO>*;H9`>}jb= zx@U2^v|mb_eqBB;Apy#cW2IZ6Cu>%;0{y6Wz&HR{v7@Jz+a>SmJPz~~@NI8vilwD3Y3pZ7*cPW)fAPmA zA!uK{%nQ10^U+un2b$o6mg8&e`RkEK?#B;4U|*#+Xk!S)!Q*uO7$ zkyh-}>NCSbqE$Qw6wp7E{MnfRfl~Y19>W4*PAzTlq0+Y&26A!w!lnV#j+N1-228L& z+Qh*6PX5RI1sekNvAvn=CCau|>^3T>1%@*~O_p4Ierb9WnBbFEYdlQ&7|AyD6My%+ zV>m`CcJ?%#BXt7Gons|dw+zv}DIklUz8JEjNlXtZ5g3Q*sj0gR-gHjHSG^1Ugr7yf zkp9uR)NqAi0hBApN+fBXCg74Q%%1$qjpt0R{H4X$4B@` zesmt!FW>^di$9rTB^)pxP@X2~P1h9UMnA)Cd0=sx-bVU5<(`%*c^tveuq4@jY7JWg znKD_sPgDpfpa(1X+|=7NiR5BDVRQx$=&$R&i>~V&wV5eu4PSjGqFzl?-Srz#lT{a{ zRB4(d6@an3JL@&sy)0Il$In8L!{-uSMnVf!ap@qQNYWJ~e zjI$e1`5Y_Csh)-MM%eyNrfhjde%ooOOBSgmRpJu+#y!St*cZx0czZ-o7 zz_ZK+0iy?>yAoOpVN2S_o2Qf?X`SBmGV{5`Vwx4JX%ZMtEzLIB8wz|@n{2fK4xxq> zf*XK>0jj%me{Vd@wmrIwS*~tr>F~-Nw#;ao8CzibyXE;(cO%F0-e<89rmJ$O)RP3C z0EGfndM9V(0Q5xLx51(E%$Az|@XSi{#on4&7gK*XZpkfUKzo9Zqv4G%)d8WqBD4b( zj$?%k+_tjOQe(TR3F@538S{aGlcB>8;0GcggVD&T0T=pzLs_{k8SF>?YGu_4$6l}k z?m|GxHt$9MR)Zf8!szpMjvT_~CW8kz;(Q{K`xoLK2JG%VipRMX^n>tTR*voekbrm8 zziQ>i2b9zr@6iHEMILSFF@>HS-G70yBOivf4hx1@JH&qZMoFw%I zLo_ur_eB3Z)s&>)x<4syMuc{jD5eJ%w{KK<&G6``m%f~|*1U}?1(XZN3gJKU&3vSK z<=}SH-rVs%kl}68Xm_Gs65-a-raK&1({@|8U;@nEu}9eJPV>i(;=HY_n!9N8f2#VuEYiDz zfSqxq@2L%2my-RW{@JJ@gM+(9caVpDG#d=$5vNNVYAL|^mN(3wcT`RG?xXC<1v9ow zSi)5JW+W>xI(%$5^G@)Wk*mzpvxA03%)OIM!|Pb{043sB{VXegSEA@{LBKY5;q_Gh zCTpQhbm{0?LG6;9;Vk~A*#m<&1k;=o237n3K*j#25Id)+N(xf9d{o>^Z5#hvJ`#=C z5G6aZggO)~%b8<0AeZGj9~%f3CjrX(KP3b<+kq4yw)p7{sawVu>*hrhN3hztOQeHk znul|u`pxRbDIxtcBr(9+&kO&lmhGgLuLZKrI?MgkePey*s%Y%+lP1v;?!XDt!8y@= zU-XbE)qNQfU`)x}_Mht7?$K>uVA-6~aWr-NSbaMpx_G#~ZRZl`0J0^1&aytSe(#j3 z-km>M6EpDsoT!as7UT`kmMu%sQH_XpBt{5*B8|abMy$ls@NpTpG@ zRf}%px3DmiW5_314-d=-B};dYCWAX9Vv9RumzXNR!48X<=mf+y;dm(w^+NDHaTIe= zfLJ8QMGC?uw~^ZUWxp=33paL*FhM)?4NYk97i~U&5;Q;!LwUfTq){V+z&qX#4AgpW3Dy5rZywXI`@jyvv*C*%yuL3*3t;cM zDG4;Bf9L##JQSw+A7=x;SA22Tlgix(`})t+VD}~JzSO!^ZO!aFLiP3+xE<&ZJ+JZCuU@=eb0TwO^!73ruk$n_|u>Ix+feN(v%WWrX+ z`*Qqof6Hp4j}YKenBM`SNJ_A0(TW71aWT;1q1%Gs=H~OVO5_6bDslvBz__En7Za_A ztVd#RYU`pi@euVFKP`c*HrSu0N(JVzWQcyStaT;~-}_oxgK+PXkkXJN?J;#1(Zww* zvrl53%HuhV4vR@+ZRnuaJSW1oH;fAb6n|*1`V#tS`H1R{&rY&a)HxM=B>9tZ+ z;K-V+jp)~dHBr;BlAb%h-*8aV7Y7jeAHnyb*Q`J5Wlc}mZeTBA3wti48uurA8B9_= z%&td_XYkkuP=_MDIaL_T>_^=5SRm1gk0V?gY@~BgEhYo~AB?pijgg9t#p)O>ICNDJ zb25H(T>-P5@G@cntCr+6o66oxYPO!r{fpY&Z=keLhYXLX>uGkp^VBo6G)yvWElpV& zNxMaRkq)P~)2mmA7##Yk+1bovj2YIQoOb5*-Xo-Jfmk0-4~t|K4kSEFqx6}*EL zi>Y&Y>(flsFFdQ|FKB;pf1B;an8&l8G?&fbqI;GRDC%hKQ6f=!m19Q=RWu?WlCH|F z%G1ds(k-bEDT9*D%PG_miG$B)+CItpNfIC++1_;pZ)@47=_hzJnc3EaN5%@oN8*@% zX&IZO)rO_!kxgpha&JnX%INce+OFI*={EBh?q8idu{V2Xs`ul*c7?N+;qSIbAtDL; zTJlRT5OKgBEn zKjHUd%Lcsui;qFh+ca+R+o{K9gVxz+Dx zxk_&5g|OC$&%_^OZ4)~$@n_eG2HekcE{lMOp3?%v=6FV}DN6ul>@x`Pj~Lf<&2{e? zuQc0pCCsI2hxmMEk8(@!Jyx+Y&ixv@Q@#dJBp^1Y(la`z@ki)e+nVvQ^xG}=btMc3 zGbiT@qr{XE_ndLb&>uX`7Ye>&`r@iVY;kM8h^sKMFOfTtN zlOv{YYa5E=GU8h22V)pdENb`1%%>_pK*a;G`5Wqo&4+kK8e-f97fTx+!PnN)?hiWT z1kw}x(&8fNr@KD|hcmEk5$@-hLF$l3%FvTHq`C@DbV>5&*$1`j*zm}*sHgZ{s{eBy_^i1{Yw3TtZ@iclPK3F zeitxyJe&Cq`9^!5dAGVoQ_kFyQ=7Wg$C;g_V$GhX5Sy0Xkr z$y*z=TEC6w;lAJSgy%o$x(2~T{ZtvvB*oL(RC9;?aQ+EXkF0a;DZ_rLaphcnqU3l` zhK?>ldD!b!;zKs8)#<#@359%ZsW5!0ozi5{m*p#sW7;!o)%r^sXyo6zDm5%9O)FIg zc@*oAigX*3s-B1Zsf^~$t)iNHO%Bb}{37E#^YGd{{qiQa$OGC@lTVPF=8@s3$2#pM zRfEk@AExh-SF+POM6!#ZiKXT1HGKNrCr}q4Cn1VnW#o!UkfWD zFdoGLj{2@)8*FcJ=|C7RVV$TKi#s1XsoM(Q7^3S8#=o7*?ieGinBw1ggfJV}C&7vq z@l*K^deiKRuJ)DE?AHZ!8)!~3cRM|3&LQUZL|Wk7U)n}!3#N3pn`z0wo(!2R+X+>7 zrXRrzDpK_v;dZ4z>2l-x#q4fZakhmFwr=3m&VAqNz)5xSX#I@?Jo{Rp9sjAceV)?S zMUZZVi-ETql-ol{%{|J+^TI596&GAiG_R3*0A&UJr&4;N&5cEwop(*6>8kchV{%MJ z%TvROkV?xt{e^iRmKoa3E}tzBDRAHZSWEcx3f;SJx1TIZ?CfnlmtNoQ*FuVMYf(1$ zg~VDg7TtIWR8ipW^EKJ)PVOkLbBw#Tbx!@3@Y$Qv8&U#1 z#<5_*4Ccmo@HU%^8$LjQ*j$@UQ2bP%cHD)_feDcH0S4My-xZPzn(q607@{X;1?|4;3@1$kHbY`_85Tf1Jr z8xaeFu&*PpHzvZf(NgHU!l&3cgfJe1JB+#(_z=GnL$_Z`>BZLV_zp6`2z}AuLx_Bx z733hwhZP0eiarZHiP(rG7U)rfc(=G>ObtO2n1NkKRNJ2_xH zH@%k^RYw@MP)ch5(MdtqK|6q{&5_1()wb1os76^4U=3TYTQz+c! z(Y&wZ@>Z&(uKdYHSE8xv;mGMaXzl%hN7=ogeZ6<0=Qd96I_00#h-kXvco-WZ{Bz_4 zHm*`_d5kNmIVB#&7uWA5d?b{B>9wsy3^X9CoYV{-UHOQ-1Nmz1V@f|d&H62K5q4-e z5b+sWW)`62@KjMQ`Zm%U-+^6(8mcM5i7l| zGnmA2Fmyw7?904|uo89;W+!4Q8(jGZ@;W zsY_AQ*_oypsrPGZV<$D_H9tYZ8xiK@av!MDcq7dh-ljjlVkI(N*FOh`lxsR|(G)J< zs*_ty8b+yquX7rh%%Xs@dioHfU`l6E845DiR*@=%{odld{5J&G(l;j?u|xlxZ8s&D z@7K{*)xLF#dVX!fMqB2&dhLek2&aYdQft}tw=eQs|^}V(y z$xryPZ4ao;m1)Wuv;{Ra^tUu!eITrs_Pyb7Ng!PeDNTOGh=837t6(}Ho_Pvb^~gg% z-iP?s)*7M$vRrjhQC)bBG@k@Y{6LN*Cx$$uB$K~+dQihDD}N}4 zqj@Vji@8%$j@i#VOdi2rV(td3ao3n#1qTSW%%Q~j#MR7GAz7pn=F6G)$lcZS2z^_mXh31U3peLK!mSZJT7$#42+EfJK{zCEn0bb16s)&e&2Hr^ zG@gNn>57Ht5KlE%i8ZKn^=c3YtyXsB&%!)XEKPWfTPORnl#ge~8fN}V$dzui(z6{n z^@gp`ovrHx&*9Q$4}unH({!+YKMG_d=b_M-_3(r`Y^wJ3(l{JR8!>YiK38RBb)BW* z==CQc&3%sq5inMdE&dt8xznva0Lf}wntKOr)7%$-8zVG7Tmsy;P2`y)IITM0dOB+v zXP4f(@wah%eh2j6hy-5=uN<6F*M_*+yDJxgf_L4D7oy*EdM#z&gYWFv%o3%>7tvE2+%YU z6Z)V)Dw+r9C1(gz#a3s?gjrc z_DQ~sug9HEBJer*-QlVHLj?1@G{G3r#qoo1koa$e$Uew2a<@|9m=f-3>fzcw+!mU3 zUOyK?3rzfli>1Mr9pxUOZJ+1J>!n?ETEah0KQK`nOgx*#dx6ej@8=n7F0(iC=HxzR ztGK4bLbjRfyG+1w;=G#Yz`4tLwd}F514F>^H~(*J2)8&=E! zhMVTHnKp(Frx&c_qU{r95%wG6D`FeN^h>H9l_u*PYSY)9)cjSiUyaoSHk?=t>=e|v z!82aF7lNC#U7Zg5cZ`UhNAT73HbjBXs1G-Kmz+_ZgB)15TKNK29DP%n4S%>eRGEp4 z@N`t&M_O#$72DCk+%)i1QgDAO@1hDCHYsPL(Zx>`t1;koy1W6K8=WM(fs0*qQ1%#a zGxNTp1V7UTCF2u;xlRsnV9(0AWDrO%yH7q>^ta5J;+yuHG=WkRbx3lOvTYGb@(0yw z=1tim%0rvuk}Y(X359teMRL;W_e-!kOVrN`ONACydn#7&RGGQzfq<`wT@)_R%g=bG3QaPL?M~rr z{<5FSRfsWvtV0Nkn_>z+2`(5-sqTVB`j4yh{KL9~3-9r->ehNL0`@Q;wY@C>@iQhA zN-wz68C^F+aHZX+pj}YW8lSR@zp{pVG9{bH2b)-EI4se}-Q*p&H@s zlgoR`UbP~s{Ik9_AFpXhfLT)N)XT1zSAtBwd~-V3Yidlh9J~&QQ$&cVKNK^lF`-!u znp^h9;@t2e>sphtQIs&xv;sP1nZURfM)l1xR>0X)tD6epWS~F)0z3DIVt)Y_H@!xj zESqZLA@5~{8EsJ4{4J2b15mg#w z$R|F?BIyT^)kYJ*3=iL*o%h0J? z1smewV*t1^j&1G?kk7%*s!;gyT7(>fdNs%AVra>SjfAMy?7aZ;cCA zu9S_0l_;vEpM9eg%cOs}nw3dno)tx!%E|qybm}(4rQ%;zxAfo)H|1X4;@Gzetkybg znOvYr_O+GQt0}HZMV?GxH7HSYa6eUq8rW;3n4!X2epz=!S!`~I?NZn^C51khJD7;` z|B#*0Z*rB%`7)l>31IDvWI~k{4z`zV$-dr?taMB~*_&S-7V&pCyEZ21Q5U%0##7Oq zSpV1_+f@(x3e@2^+VDfMM#@_H7S{c;eo4mkZb_;$25xBQNM_dotv(=b`qm z&`kShKrR{3gOVW#|0<4E+0Q9vm%bR4xns;p0>zQ$GmJ>y-{Br zz|A+%=YTa_Nb2H=D=g&z-qg=6w3<2)da;U%GHgQ!ag`m#(Ro_R=xK6h{Zz zAc^n);>wbHtpwh=EJgDnUc;I#=11I55t&VMxyGOm#)sTAudBwd+(nK91`FpQkd?aa zKQ-PmPx>VbZ!t*2)@Yi}NoW!CO|IhWL1zuu#R9MEhRfm-$JhF7!BarN{!`NCS(<%W zb>?w(%NmU7yK2{p55_err=X1nd*wZ^tp+E>8YgdEjtCez6i5H3vMg<;i&@#`N+Wtr zjA^SuyJD~5tidkGqOa8ldtK3os?R$`Y3oEWz}lJH|5KSaw^B=Ki8Y(Hly|K39*?N( zUwm;iv6?;m!v=HBg(>GpkJPROR>4dK0j<9z7r*959kM@U4R3r?yD-UNEVJ%c;HJF_ ziVF@Nde~r@10I?K7Q0Y}=YY3b$-9q2fw5?D1^H*2lBF8UfO;n08)=5kTWJ__fzJxA z@BajUKj%ij4T3jyasO2W)_Pq>2U-MNVHdr3;{VF39668qoVaypA(ppN(yzza2lw?t zaE)_wd&+PvQ*ZP-<3QG5+NKb7fbwBG{Z#ow28tqa>wq(*XQioEM%f#Drn`u8a_+va zMU>61u&(12<|Iw?7uqu*dvp8$Y9p(B;3#`6@l+q2-LSH!$AKLje4+Cu>;2q&9Xi%= z*P9)0SqE+GEG6trK-K-H8v1KQ6^ZwH!-WS|Zs>XjiF{u0?haeo^|@!-`ec_~|7!b9Qf$L&n#Kdh zqXc*VQ{}yvbY6+Cx<6{CtrT}&(CCBZ?d=-N+`+ba>U!6wZL?(uZFr^_UNxYE|5F?I z^{vIrTb4GomnTS*8>LmRNyAWLU4DHQ9N8dRZ;LF0D59ECPvFV(Z=r*cvuwtw$5C&G z$EvXPW=mzQ9eAFYTi*!b5^WlGzz)SB_s_Qw4fY5M_!R(Zn4q2cfy76Z0NIBgsPBF^VU3 ze}7|<7v0a8SxTVS@UK;H=o_#%sxkEY)zjLZ-1#hX>}=mq6=io+GRs}kfj=@)CGWD6sX*4Ik&=S{C~lPc`C zkgE9Cy9%-~Egv-p^KvX&_SV9uP0^@1r8S24u#reRbS9fjVP$IQQw|>zYedS zZ1;lrRbbuenfXV5ka}m%@7?cN2MahI_mRmZuB|W2mzM*n#A(o~EoPS$H)@`m9?acb ztI(I*dlSb6^E>P^HjVeIZmi!rvWNLkK5oznp(u9im6RFETDs~|pH}|XVZQ=WW83<1 zF0|&BL1Q0B#0jE0mcq9b&ruB{gUVhqccBN< z{8^g_{_ubMY-nSf3EW<{UzrK%Y)E3%z{(rtu=@xH=#G*-$bIlTYsb+DdPAZP0Sa4(z+)M%Xw$CRvSWBCx{Ls7XZQ zEDw}~G{@>3tD8D*;a81PzDsLr%c)q(v-*DOt;SCcmucZeF%VZ;TT&m)kM0@z9NtbB zc>RGu(@$9^GVd@SHXSP;;qMk7t1|F&$fQ~y-c9g$J&6Y|Olr8y{gC(v#FM){6a;PO z8of5d4sah@_b_9)KBn~2Se37cT9Ko8K}xB%laDl5YIQPHK>^5Jnvoa{21`mpJ~n<5 zpZ8Kj(c+iZe>1jmbB!*=Yt6AjL>b+bOw6myGbDj_*4XMY^5u0y8o`>@hG}Y-kWO%~ zD&Gqj=Tn@Rw1Sb!Wg6@Xf9ZNGFqFD>tR%=QtXpLuo2pZmS9yZC^cU))Z8U_r<5pZxnJ_5&vT!*b`|vU*76#(b_DzLof0B!`bWxWo61$HGCN}Pgt zxabPW4W)OxKz@e$cWjwlr5c8vM9|n6gckV$eF8NJy}p`<&cW!iWteg7-IzAqbo`=4 zW%xkCa<|JwFrns0S#+r4a$^qZ33D7;OO~Kkz`jzZS0y5t6kZk$!$LR>nf>rboVlxgka~{O!u6;D zj` zZe19?@MoV*@~9@rAfm<8{i1sV_W(W6rk10@x7F<#A&`rzh0&g{R;6>$e)yo`nA-qa zC;PVt%dwK!R*kg*d#BgdwV+_A`d-VzvMUYSO@FTIZ9HVOj?zI}^!tM_unygGw{Fx0 z`HP7w6em@7^e?0QRRis*f!?Y6wX?0P0Q8{k<+>>Fw-#WZE6B}ea!?@jQj^?`h02xx z%#alP!RBEo4O;lKEK(_bJ1CqHAAAqRda z>Ur*;cmeDceuOZH09Ni1CCGb)y`;71h_yQMJdAt=l;Vy(=WC*T#SKsXME{Nt{GpgI z&U(yz!eiWAtb#aL--c6=QVK@#4WzTlX9+NJ!}9w?DrJ&yIjMxw=o~`*M4dKK8?5)p z^(+PEG3q1BTqnc?vH1DJ*dSJIvIlN0D|vY~p2*6X|A~;us&kH}U|D}nDCPyYgZM7` zPsDX`a%~SPQY6Z|fd&Z=B!Mv&;mq)7*i(YG`A2X{f!Vo$9LfJV*T#4V4b?nC2g8HZ zgEfC6`c&)lqLFVE?-DPgQ|0HvIT)VYX?_QpNf`fH_QiN{4|Zud?HJHd_vJF;A34ZlndnVj)3Z?u#VOn zxoLefD%(vWX5*C~ijqhAgX3J|17&ck>)7Q~4ja^&d52>G zkH!)?Hz1=+4sx+D{H!ay?QoFo8~#P4?GHtNODto6p(2O}<28JC*?ML?f}e4ZNk?9f z>0#bPhb-}9Nin6f=;*!C;4$brktpC5Q%IrwZ|O~>J$zL3dvY!BW3Vq} zF>m9nP|9Q8dAo;<7rdXbId%$;qO1h1!vC&-6x}C`%B#|yi6yf9Xe!A=N?m-CbX#gS zD}YRp9I|((zZcA&C=1hooo58qZ^xPR&x>Gqg$|R}PDs;QMSBuG)b|!ch>z8sUY|(~ zN|b#hJw~v6LZQ!NR-mY-Bxwzt6-y;68teQ&NRB}6PJ1dl09^$1 z%$Fg7tZd+^JO;TYJPw&$Arx+hcCH^165&%4J%yo&+U3o{5afWrS$GK5HSM+-g6g!2 z7scWZPh4T+Fz*=Vad{>05%zrcAa4h*YfS@BjL%y>o!3RM_fO@oAjqcO6le%lR>1TL zC4EAHcX9lw(lRE8M7^1HkAtImCG6o8)7FL`(Y?SXG|{%KK>bVu6R7$pP4WA@V91O5XxRWOo=qBxwwz$-h zzC#n5DPXv$ed1+|xhj`sXBZ{QEWaVfAIdPdc;+|hC+n@8Ib0x~;s6JzhGu1sEE%MI zH;rWm((8=X@u75rA$XaXKA@-j2^h0<4sOet%j8|wr`byG&wsh{dQ);CH!sQ*S5lYa zV2mhtikfC9tW+(2ssB=a$$OL0wzkM|t1+)G6Npoi;r~Y2jH*hh-lg(#o|7)2X8PJ= z+Uh#5Ri`vvph=7OY04XRdhgJ#2OoEuu3htEqFr$Q#}~PQr!uR;pgs&P%$=*A+W2^_ zrz!?ouqso@f%PuNC|ARO^Bz)pAP7!n%8N)VU?q{6_;Y@_e1qa5>S#`cA`QJSxlqo- zw644+JBKYIIUMUV@?E=()k#CuoXs)Ox`=)4-@L|$S;Uyt9@|dtz@O4p} zaFt-#$5E&iSSAk(xAO}CQ75PViQ~HL_VyDz{zKBXE#u zydMb&vf#;`f^DKdf%AX;Po)YX^v3K`L5Hp~kt~2{6_N4$!&rSg!YZR6#)eC^wWl_#gQcF3#rlOmf- zYs-L|2cepNDCUFQq&8MXTFQ89U=@6IK=aj_vXCQ|i*@sT7g@~pD!1t^MIf6=w@t-h z8Bp_4c-ll+Iojqj5O(d~z=+wW=+UMvja4CPlNMs}jW9(*m2SwU&CrcDFx@xA6reU@ zD8PGw103t$G&La9Ytu}tkc#Lfh6gBo2wcAd9psy=FGY{I9X52Ml{R-Y>#@&(^9c-K zG~i#QHw+UhlV9jJ6YfXd(ea5zA#vIo;-D{7^O|^K`ZS#{QEaGf*ErHUY>f&zbK%7KKh+G)X5QB17EKTj z6Gc}8UlU#Wn<|}W=j*Aw$bC4yR;A@0x4k4!;wJxJm3>!TD_xn)R!@>NM`fwvB&(LT z0^?(U&i`9+Sp0H&u#zj(7tX#!({&tUF*RCGyA$Y;7yKQo#@6cAcJT! zvhc_GwAKlg^rrh8%s$$=D-3QQD(RrY{|bm{UxcuDhPH?OeX@UQ>7mwuLYyZMOFHrn5ELC_t_F?R^~0TE5r0+DBL zleJ)Ma;#C#Ct2C6?s9DZpR#s0&ybO#CtGr)mqN45OQjqA-YP6Ht?Ji>MeMb8QKhZOdmvV&sC*E7s5UYU33&+Gw892f(RkYD z6D$yV)|$g4AzrpUEzYmFrTkV_TjR{cR<5ddK!jGmZMaxAr>+OWOZ9H>gwF_n58i^T z^dW)2p)OByX2@}CT8`wNg0#!;7M8-ybX~~?1RcgMzko_A*-&*IGimMfS|fIIxO;s% zKHO&-D3`Egl8C;Se9*EfYcl?`^j5Ad;W>3%!7%Y2W9zE{PjG%hQt ziKm8m|5`VndUVn&`eX*L>1etO>!V~&Rw3JpGAlQSJ*81u=+1U4N-x>Wwn_S0evR!F zwzIO0o#4%?sbG)Uq|t~Rw#k0&A*n*tv~I0rFX=!wQ~U;8n8z0{D2y)36@ijcN-e?z zVJpfjglO-Ss<*gIb_JycMat$xP4P|z zEc8SvMSj$KZslb;+U7LX#MA1d5^lE11SxB`w#+2#U6*7&3{q!zn$-CNd3y}THBXB| z^ixBNitp>lv))#GR&TTUOikf!(#?!t4(za7m%Oki1OHFj#m@ZtM_HL|#(bY#c1!3Q zM!|OrE97-io4I_}){0tft?fLjBk!c%qXD0Dn*X#hyD*#(3AHVatiJ%;QSmO%jKI}M z6VgyK>z^z!qC>#v+^SfX$C^V<0&x7h4@ z9NB&`9%Le4nsE)foU-hPVwmJRa|H|y<5l4j#u?-vB~Yey)w*&pb4k|Ms%&OjjJ)P7 zQ@rR+?L+3-8M_))tbcnc?3c2b>|D_xy)!RTGzYPNL7h0SPch?2Tk<+BN#RpQU-t(G3@o~#>9ZLqJo8;yhI+X~b4hcmX7^y+3u zA1|-cykB^?B3-j-h5$rRTm`mFo6RW~&q;Ib4xl<^qB`)f@|@b%*X4l)Gni-%sjN!PX;To|2zH?{~xQ&$;G12&jnszLr!K>~TPFj=(dW{#Qu)yC4l8AoD zFGPud4z!At?y>}fC7u0I77N9`QRYfpg#EkHpVm}=ta=meb>ZCF_w?Acn?TFx%!o-~ z5Bd$iXJ9=e#AO-k3IkY|&H3SUMX&h3V$PMV;S=iiSGw@M3PP(5ylu%3YTxjxS19Wr z@)r5^fa-ZME>TDW@86aM=%kwK||3q1g;k*jtW^{Owv%#%)ZRs){HSdp#shSr_TdOv!TbB!JcB}sI z^R30H)Gq7cze|7iqv&sP9<*&kH5Y7d;ni#}{$Ro8RhGSKA|>Tj4jRudPpaN&oa=k5 zMyB_6iG=4${vE{vTEFE)XwFx}22x?cy{g@0*AiIH67v4?BMG}GvTD1~msC>impQI9 zIB2H*Bj$~Ue`mmnQz^I5?PXWs&DiOcM=Q7DqH3_~m*P|Et>aG;sv0<<{~udt0TpG? z#(O{oF%T(>kZz=r?(XjHC6{GkySoFHrMtUCL;)McKt;uFY{kOXyQts2-*?XCJjT^%ztK{(FQ}ZPb(A4+;fAaHo#Q@e>5VfZTSws>5Tf^SNVdh;e_BqnH)io zSWzG#!)dX&BHzFy8+f|ll(I}oWZ@YpUC2za<=WjqEQ7&W#f7Ok;W_n1J#lXU9>rGz zW%34q_ndYW=#)G+ep8qSdc0@3@J(r%R8HA}vVZTRQkAr+@^jgWnU0Y3*!gT-$lU<( zoD)zV#}q&cw97auZx*&=4>3Ov79#}*o5KG+Wsq2MALdS$Qd%BnKITCNAJgVr33QTD0oty3@9(%FX>EaZ)SYhday9t$%9aqkSk>^S8fiN*3pG% z=bOm)K?V73;$KlJML*UkR7b9$C^5e%F|zng!P$_*B@#uO-J46E6>C|gfF1+wwZ}>q zOEl!kWtHdl%Zut;Zc;awq4N(%1)kG z;)4p8H~|kw1OHN4hq5e5CUM7d1jN~(TtFDn-<6$rkZ{@JcK$B{SnF27G9g+XUwoV3 zB=H&c6DR(UlFtZZF2&hoJzzW!e3EmVA?>;c;6^vI0O!@x_*w<|NLsyobkRM^qjk!# z8QdSu0V9@)N&8)KEB0{a2p<_(nBC6{bQ#TclfCbFcagu5!g$rH&En_-l@De|Jx%~+e*UHlReEu4R zv`Ai$Ag5jruSMFVKl1KGHfA2Pdx3nFjWynbD$G?@L87w&4x5i)W&YNE;ODU&7y%C9T=R|E-)!*jXYD-y3osM1$vh z>XuUAUHgB59pDRwufT8MZ&l38>fqnESixiwq6Snn*7#Rb;vq#fxcQJkU_P$KL$t^m z2iVUCCF4vDsihh?DdpwTW4Ntbd>|u#q7aC7K` zq<9&cmQ0gZl?k9Aiq00x@-gD$zxMwqKt78D4K6G^%r0?XDBi;wv^iV+gC%M35;(&= ztejn9&UmxMxa=*({=cb+3ie1a-*<@}6Sm}(#rhC=%UqH*6*H*i#a52jmVeDPNx-i6 zX3ZptMn)o~Or#;K-IE9-8YFtomr5&%D|C{jxg;!^&C>LecWN2ZSg9Kov>9<};_K&W z?HPZ@HAQFEDAZKAD#<=Y%?C$HNYi!9CKf!u2jP~71>1Jh~Tbbg?w5U*EYt6)GVEr4&xA;5}a*X*Ga zf2mL)>Q-@v*Af~546(n7K3t+>>WIEta$BG!xPOE0^u>lb&??{6Flg@sbE1Fg`wyf2eF~Y*#$v2(SDjNLp^A-8kJB2?W%$$ z%#Ug+8IMdOh*T_$==z&_zpgO~+U=xU?G|cfQ6Qv8$m$}6AyKWn4p#fe@HRQun2Y2P zF%B{Lj~-H}vPK~tynj~wi2UNDT#*}tvk2s0kGrf}&aY2+vg>kpt#~ z4Bs{Tl0PK`{ma{*GoQHXn9PYzerG<6~3nZz#(*u9hVwyb=MHtAE@9kB)CP1vr#_WvjzK@3i|KT8n8 zPne|=W)K^6f(gS2+g;Czn-F1JEQq@iFU3Xi-%2CR11vgvb$S%BpzlqT|1wcqB~aeJy}{gzh*imcpIjcow|O#lwc~IEz-TA z`G0B(bnqWlBWz1IHp{PC$Y|EZRmo?*P#zadWSx`US_R0~Ti?g~n42L|U%vut*X)rA z|EQ{p^8m70dWBP-K$pfJ%qJ+n`&_?Dw2P5Sjim$ z0?e|xpb}+W99OyIu<|X=b&#g)7}p(Su%U&?0Q-wH+tctr?fn0xsBG`@t7bXuNQj9p zkgW$1D37r|L2k)1*@2J`8^-9zVIM@w;TP27zh3_N%3@X`C(R<6&ydr)KFnLl-^ygB z5al78$2^Xz*#M(q(cvOW@jqVvQF|G!cp0<3Omo~(oi9vl+%shp#v(pM)`roK*WUo8 zmf`Rsy)or~DLy9Zs-F|#LM$N|3QkGTitx(RPdXMe8nHb!GU0*ia(Ye5B@=kY;f&B7 za#{DX;A_6}O1V)%zoT?;hr`IRZ$ZGwJBirz+*tXP+hOHE$=U>9T;4Ia)vt)6C#^+CUrRca;|=;OmtXYnR8~GT|u{zAU>$5 zPRTLVzxcA)2iL8*4>aXhz&&4MikE{id5r%~lHBEHuv&%nP7S)^q$lJk$l=w6kJptGMgr`QuTYE#F zI{sUIntwZgOi?TW%d(INN62G&Wc#EnKPogTr6m{+xSlo@u|A%gaX!YyKQ3!2!PoX< zPGgF!?$%uYbkA+og|v)|YfrflL}W(wMxH3Y96K2c$W2WsO<0JtOWvHa#m_30kZxit zpU%(H)uCr-<@9aq%qz;hzea&XA!ULeq*Q}P!jb8w*|exrnSC+!u~&08`dm)v21MCB zPkNqD&~8c&DU93}ohw!Jrxk_hV{yI@^0YuRf#32kW}XP`Dnv$mM(Pwxcn`!x0#P;_ z<0eX0wAaTMmTGL11YB6K-VH>s?WF^D?yfM*2RJe;3TBG=yrzF}kUC$Sb4vlypIE2pid=;9D##&v8 z*n;lVvWeP*{;^FXD<5-a&6lYC*yT`4As5ZLvdCA{9KB-5juD!Ei==)}v!J6S3oF;q zeBu+$&k-RcnQhux*~CA+4pAQEZ1cHsg>RjXv0tR3+%K^D!wEhVCc~pWpq^1>IS_2g z*sNI__JI+!O*N~8`ltUOdLvoc?5<8Mw0CH#vQK&7dcU$bT+HhN|Eh<(|7IS+vLLXR z`%p6~OpKc(^E<1Pru8?6aKEf=eoKW0*-OFb6hdxs_)M4v;79aPx0CtQ_!@JCg2&0a zYUD!S)D>xMuzPyn8U@5aXOe`GHu)!0)nX1O8>WXOYz(=TDU!5ZYqA5~5 zF68QE+emAa6z3dUqsm;$A4R`OMgvyk#nZMX4kQj`V1t`e-TY!>wqOH zcT;^zVA5X25um$k6lzPUAm9baB6}$0R;gZGUxYtcHPAFVu56Rj9juh+0K5Oe$keA+M_h zB=AwT(uKv_&~bmMk`RxZ#H`GE-weXrnE5~`q0%20x{u)D7#|T%kTg+>QpLYl(T)2> z2$iN4-N*mAhoZX(cYVP;nDM}?mr)a=>i2^_;U|^}R)<1q=Qg(zh2G1t&$B6^Dh#`BVe*qcv^6mT=-u>0bf0CvDi_RLV-xURMNH zO;=d+RSGRs%0frhsh0G#4&y@h%6Nv^hX1)ix!`nw8C(vK^hD_YG=Sl14{ zp7TfFfE>(^PT7~_o??+UoN^(2Fk>;@)+;xwBa^aUJtri4RxdNRJa1L&T1>aEonw4DUxZVwN z9vb@h9_GY`yoO#$aF6JK*@o^;SE>5{n6^+JdvPU5ex ztb=~(v<-2mgKDU@K@(vPly*1Y$ZU#^<@4x=e=;YU~-Evuu#nPBbjFB)Vqa};2 zqV4>?13_^fPUa=-L?hE+P6;VT8chp!bG8wAzNQk95*4!Q{mF??7YU z#uZ7P$HynMtl4M(Df7^PsLFDyaPa_1NLZAX%P3SkcAF&w`Z)fwt_Ex_DP#u%u`cccfEUw$n`j7Pcv{o*VA21{m=HajLV$>N$l z-25*U7M%tx_3|FilmbgQJk9zJK5o{N11c-kPRWIoi|^QxZ&U6l^|G`Y@_5Z2^{1DD z5epqkT7r*xtEZlYzjjDS|Az23+m~65XwZ6*)r@$mbN~>H+_h00*a46x-&r% z$M>pBcECm1-$<>)#hXs1N8yUK&>2y<3MI3g@3`+$%O#>Q{eM#-_&5p5F|S>T`INo( zamla9%cg#*!{kLR%d`>l4<-5R*W?3I_ezF|qW(EXbp1{Joygk0OxH4Uc%Z5E88Rk> zVz5en7T&Bv+KC0i}ri!w^y=oJ;GgAOW(0JoMp$|QjMN*NNV z@b3`mzh3^PUY~MoApVXD05;^1`BJ_k)KzawK@xOYS+dX)CMS~)8iK(jX5a~k(f^@J zGD}d$9QtyMP&4L505SA>-G#iJ=%_s_`3mSJ8M%@~%tnbpcsVBRzbVfr=|qCS!6FMu z*k|@NCzs%)o1P0LAojEX+6k|>eg#Go_DM{@bFtU{n{v9tb@P5|y@4y?pJ}+tT^B^% zo5<}5^^}A2PlWH05S}z(`!jt~!jRo0jEMmavA+0)i-0BkTW>~#p4_zkGajy@( zF`6WIof#S{vEd25EuJYF>xrfRQRYkbu2E>5(@w(kXtw23VtVXrgP+8)1Vy!vB;CXp zav#ayWNFDr(o(9E$ntbf)0%zRzZnEa02-7WX{m{^PPH%f@ zA=!cZp8aPfK;gzU3a+)5icY8c2usgeuxNv$a~$-S5e~U5wQR&3piN#GxsfhMhfmJ45@X*4moK*Nwp8}y2U02%*wJdIsdkw=Czm(ENB1`@}DafBb z`G0DPgGEVhDcSsMsS8*`e?J%oo>!eJ+g}zdCxl3r@ujrj-H@&|UxtuDBLF)v zk*anjI+z7Hjwl|zhmYW;fs~Pi9A%>?O=P13M)0rUZQ8TzHqTM2* z@%Z7j6#8dY8Y5;FN7qZR(mO;iNRm}^q90H0R1h+1Q%0oG^nx@iF<)v_=9e{l;GZ`v z;^RMGzQpV4K)qTLFyo1u5qU$FmO>|aG3&kbbFxj2sn|FvBk$arWf|_T{XfbWn+kB! z!{Xxdg4DL+_44f%hy?Ncv(lgNUkZMTZz600{^>KUsT>&5^^e+soGupVrJ}b3y;YYn zTY>KtA~26iVx+HQ>7WL2Rh&zC_?jh31pGhB^f+9sEMLzQ(OEXF+KoI~ZmqBjg)Og; z?nECGwMg$Vzu@ElZU6VPX&lrXKB0RHt_kl@4M6z9KgiD@J|i-uaVQ9KMr;-R1@*6X z{4e|ax0u*}ehacnMQ?$A6>ln?*>P}Z`!4=&+#aBEs!Euu zhnn?XeRUO$?V6IB-?dh>mvwIIUeP9{lTR}&s2zD@d-B9p3?W|r=e5tCVvh0m_Z8O%KaxRJL~ zAXTVVWK$dp%qt;*T1uzEH_N_26rrXte|VN?zjHlm5`7i(9w&o0B=`_BNjP#Nh@(Sypmv>z;ShTz9doB8U#!}G@=uO-$+HLNg zs=Iaf8t*qhXnrVs#C(i-0$fgc8uZNex#o*aFW$bq{A%Pi=S|Vul$DTo4)3);Z2s`( zlQsiJceElu(kmzOH_K&IW5q zDaqSXqZ{ec8JnCo@7VlIR0bF_$=fVsH_G1GJ|Ksck5({M+^qOiX=aCTXUVRB-Fkc0 z?|Gm+tJ0{7R!iCIs;;H6LF1|B8LfV8u1=Y5qMnDok%6MYZ^IWx*NkUOI`%P5%gmC^ z{Vc34HLbQ=i(9|5d9?qU?OD4?`!0uSN4gW*8RU}dn&KAY9_A6~>Fed~?cw9-yWh{k z-!x!f;JzT!VDk{`Q2Q{~aG!|K$e5`3=;WA;SU_A!JS+j9$V#eCZciCaJ({+Vem~=R z=KHK4IU92o0P1=B@|_BT3X_X~#du&4nfV?&ap zg_N|^gN>8Y+)de=_ix#><@VM=8QiugSp(T$+b_zs$U_uD74?-Qly2=9-^t#UyW3;W z-aS8*Z>WeuQ>&Hk4O2JM*sk$T^Rm{Mwm=6edj78ZItH5!-WXmoI$&IDg4vgC8f0c| zu4W-^@xk(*)x7lqo0k17TZA3JKGwn4(bmb(S;a-xb-n9Xw^!~DJg#}3^E%;u*k{~# z(68ISJ)k+TF{nPcHbfX&6($H5LlK(6Iu`j9M*vew3PA8uDp*+7Sv~=ogWiC>g#SWrM`@z1 zFy7dBToIl?s3s1PPLuCaKGS6B1`H2ovgrI#0cVgq$9u^CQMp54UKJ?JttQsA)K1sk zs{h)k)MVKl+EUcYZ0l)1-SMbPtXrkWwl}J;q@Oj=Gk9|7!SIjK9b@L>fd{fDa0lxq z4;)&UdVWNFddH0MtozZ}V@1b_C+bd)oSHj*Z|>9CE%Tb^?9PW>$i0YN5MCO-eD=!Y z#b4KUU$WdC z$twfzuD<{NQR9=}D)2M^%iz}&-!6VX_v6q{;jg^kHh-RfxY(?CfcQ&^igm8*R@OId z2p6>ghoyiUbvNFTuG|#3dBf(JEs(7iTVKm`Z%dWkBfGS{N-kP{hx`qN8pTAVy-JUE zbnVRFWxngn?wLI#%SPofHtW$05_RH^oZ98YzYL9Y=bF^_%aC+~2$z{lu z<5nzc5iC3vJ->L}_MY`=_oe#*{bK`M1NDRCgMJ1-4_OR78a5DK7eSANN99E)#stT@ z$JxXiCuk<_PLdNT!s}ChrG8EOl>Q-OCG&08+w7H`cex({pYy&Kh!w6c+EgqHR4P#g z>6GpR+m^YP2SQ?@nJ^$6gOu>nXE4f{G*&%( zm~)bQllQh_L**V(x$qUHR>NxewF7l$>Yp@9G^sT^wIsEo+nU=?bUf>l>9*`i=*9Q- z^ahC}%=FBx*iqMGgyVB3x1NePT{pLU)_fjz z?&O8-7n2tTF8#g|y4ZI0%k_Yz<{PWG{BJkhS-IzWpY!0(Ba_FY%XUv^pL@PI^D5x= zg||T~7vG0`xcVt-b^eR_SIW0D-{1Ur`*Y!!@OQ{>Q2|)%xjfNBKSIyV2cfT{JBC4` z&mC1l%trJncU_)7g_$S~qeH9jQ;dAd@`Ck!peVVVBKJfTm2^n+SVr`=qJ;wBo84ne;T zJ9k}x)%JRD_QO-!apkuVzKs`?Zz1Dq-*&DeRzkB#7^I))E22JO6!# z3Y9&%x#J1+*x?FRFRXDasO$i|vG;Sb8scPYl=m#kzQ*6o9%IHCTR}l&9_wB2fP~!G zG(8PDe__7E5Q;x3&DsphoJuRRhUX7gCLKl8bb`FuD8;%5WsOwRRi$5tc zD9OXO$a*OGwv(iElm}y!cobDx+Cp?iuS@74JVWz5#tAntna0Y*SnQb>Lb5US!{rww zZwmI%J0g`F-twC8k(5sRO&}86KxPC>qGWsxL7G7Dupnp4tH$csDA=eIAdZtTV17Sd+S!(S*e2kqc+= zll2!Sw&6`{JQ_=J3sq7SbzFUgYVms+U-X!xTBMa z^AkAZQJpbc?BV{Jz1*9BqE4>>@&Kskppj09WR7cd3b1@|ueMkR0`8O7L z!Qf83)i|R0>@;x12(u9teLgiZm1Jbe1eZRyJrR&5&fT z4l>_2B+gu9q}H71K1cgf#pK_h9Abf4UD>q#O;x-)s#IMUQkEhne3pHZtX=67CR*LVZE{K`zG2=s+)cbn z{dswXkvGU3}3K|H;zjnjqKNmFNy;Fbed7`Jez>@?}7 zxQDtW)VuijQ#XY#IXxo_0(Ta)0l>e*AX8s)C+X|JDE2J1E7gJdkWv&F&rl=#+pf}x zBtP9tR9hnKB)e+7HhVa!(nhFS_l@_aQisyaIm08Dmav|1tWvb&n?sEQX>7*Ur|KtGUCi>Chxiu{q7el6%a)`mM>n^p`?mfGsVgVz=!u zpKRjoIOMvNaQDvh14i>^#?LYo7oy&M@mAO?yVL{59!4E+5j=?^QtGd zosyeOZJm>0xd?K24VQ>m~gYltTA&EkFZi(X>5(t6QHF&UYv`+2ZH;8g-BG(in;A1so&)xnLRR5&N}bJ5JNP zgd2|2uqtE?;o`N9F+y?ACahW}Sc`3k8Z{Ybxc6(X(&zY_V{7W7)eivv5GCxqu(%Nx8Z=Qz!^GJF-}oKS`{l()CifmIdL!^1I{ zUZdqLQW6C|p zt@c8(Em?G{9L5a2RQnMPONkr9H>va?n$Oi;>a1kFs~&0jjc^srG>qlm<-e_AN1x_8 z3yQqlS%W-Zs|!vWy0E zZq%;x+QIUyJY!mxjl3Civ@n zU{TLF@vX|9Q_Q(qfmH~-ph8TijT*=})I%LqhjmnM>FI#AlW`r9unXWPEp~9L4DSX{ z_+r?xnk+<4SgtHxq0TwN>Yu^nb&JXP!)jR00N4&LS8H^xk<+TXRG4kXUENmA!aotv~NX-=%}Uj#Fip#U?WjYA!P@C@Z|plV z-_F}9!_|}))exJ?M|I&YK%S4F$PC81!l7z~i{iw;XnWc1fAkRlYWvnhTpYaV_vlHW zPyNn*^%PKbLDtZ9G3yt>yO6{ z1D;oQ&WrZ+>Z7r4O^pI=-ygM)d0cxyl|6f-@nK#y9k2e1wVR?RTp6A1KS8-U(B0Jq z8R;Hvy_PfGcBj!LR;Nj==D6>N+6;laJwkAVd(!wRPoEj9Vahx}H4+L(OUI2U4FlnW z<`8&yY_~9n(uQoaiCJx&Zg}E*saC$4ZC@lPsMu%xg?o+lR3nR-LlYBPjD{XwM~NKJ z8+V0xbsG%4%t>nV?>Zf`&`4`(^Bt+VRR^|@5SR*_MU;?Zs=;9z(mI%jj?>|aXw@Um zu<7FD2^H9QDt%ZBE*W;J*BRd6vA;bUk!|hTScr_!UtbMF!BrsqQS=hC@|Y@K9j!36 zh!Zci8oz+`N%a`~f_V`J?J>ce^AK-~#$2cd~9Nk>GF~z^%nimv$yR(oz>3*rXpXp~!sl83RZ{Q#hqY%}ubAm}xjFsu6 zmS@N>2O}D;75*H_u9Z!3>f=?hLvMDRsW5YY(7c{=-+G|dky&p5uPmSy?zLi{B_kLY zkKE{yLOz?Xj)&WndNLW7^?Wnkll-6CsT?T z2aaS6O(5?~B=+tpTpb3s-%fGut#7&&`lS7S?K}5hO}bT^tzXup@-z%?SJX2L_fD}l zQ_UEFBMt}Jkz*6uLr#Tvhxhe}){yjuw+)0|YiBiXcmLG5SY2xUphkiJz~E^`6pOj{ zIqNL-ET-hbM#LtN#Hly%l+4nZS8%Q9=LbK-z zeyaWkcU(_d@Pa9;R><+BioZj`o1i{R-{9|{cV}@3e;Bw+5itQ*Vs{`N;HS!sk#`W} zq@$=7WQtcXdKi^$qKXYfFTPcTIuQ?EI{}?1+?mdWB@=Wzl;L7{0rMVw3a<+uL{#7| zBsL>?IGX1q>J~1?Bofny!@rs-_u(loR+s-~zdxJ~nP$Fi3xv`cwu~?skk(xagqKrI z6J`)U~^!3ZhbX%9>S{bERq=!?k((s;HF0@qKOMMd4+mTZ+enf^7FXZ`8S}?Q?C!Cm;N3%ZQN9*HE@{xsXV_c z7q|to)Uq)S2P>#WdblHkE86z0L$xzQA6u26E_^t59|S$sJ)U1mnVxPi1W!!JlWoek z45EsW5K`wtoHk6Samix`LaqwGPZ3qbntfzZqJOjV+y;=wC8M#%Qp>sB^#x$Sj1`Gn zcKU#S(JI8Z-!gU<`o8so`y=?18mWDnC=d3^ePu#xdGDEYf(j&aq#54~-C6q(?+m>` z)WH3K4HhQiZo=!LCvmqC?XG+9I>?6xzwz(T^gBOrsrZW19k_M4=R@zXlem}~11uB! z29L(#up{~3u~pd0sFT-p|K-R@-Q*X{J||4Q+l6}g3+T{ z;`U%qHCEaB119i^l}$T%xkK?;S5fXbbe_+@_R?$v8~9TEv3CTP~r`k${Cb$ z%~|v{R92NWKpXXde>CDU8qM)=Uc_WGx(#1r4^eJhJBZX7F`hkwH0`_F(}8s9T&NgE z7Pb6FeMg>Z2+W0}0;+2w@=?Dl9yt%7f3R1Kpv*w(raNguK(}5uuE#t#)$ep*wl6jLfh*7!HeR2>H3e>B|V!qqQxkkR2yG-U4DE#ad;+ z_&gxO@lXtM7h-HgI%hjFx?6NCBTA#`wsRobuIh?W0%nNSdeM(LUIsbj#)vF;Za1KB zDF4pcMqPr;!`@RCphq&Vke|ZNhrA$b!GAg^lA4jCb$=u-+T}ce-hr(>I8DpJ?rl9J zigkLF)kq1!5TF%g1O}ARK{|oKhe(r*F@yGF#0%KXdI`j<*x~tR>OID}i4BxtME5!MWQtxQ{^AD10TlwY`+Pg$pYyo;LlMHpHd;o9}FCRO>`K+ z)&mLUeGjR!`0Jhdr6N78MLJa%FI&eCyoP&LC1&>xzlm$EkHe$rCMQjZJ;z3d%?TH$ zZr9BbwvF$nl;FVwL!~8nsV>8mY25K%;S+gNi@@k9z!4rbuzLr1~OY;0+Jk>c9=RBhC!Be04Om>oR@ zyesHQ!8guhbSb`&t&WZcmbCmJimm}j{atkTmMS@E(oIa-5H2jw2+p7n9^RkEH=D0vGJI+?}!(OA@x z#I~;!<6ARn!lM`!`q#?w0!tc?cP37BmK1x{CyZjoRIwfii-u&(K8!E6UsM`>|5w<1IDbrKIs%J!=y2DW9k7OpIfg4yrfB zf|cvz>RrJ%=;bwmWtC7=6|uZD8&Po&f{VoS!l0AxPuLIOO6CG)HDW}AKyOC%4D4R)&fy2v<_iEst8gk*oDf?;_%y1l@a4y7t~YtVAcvc#=Mkq2>nx2nsyZPu4_e9dk$o=T|7AtjdcD_#@JGbea#!c2rL$Ah5d_JXw)UvHjAe~GuzL{fj@R(oe^ z&+@8kbgH*;QYo2&d{%r}GygrKJmWXFnRYFl$Iha9y7e(#$BKB;m z{@#8^$g4_jzCwCfajpI>*oT*2vp@X}M@LW-c7r84tH~{a{)L@p&Zf!J2eiPHWb*g! zz-q{#WYssp?VdfPsEU~Ou+mf9k4^pQSoUb$jxZ*ZT?KZt6`goCYc8PnF+XYbQZ`U+ zy4tJNCl;#^0_?~-;zR!LzPQq8Ze7P*x+*)l**eUH=~l~elcU>I&Y4$Hli07cu9A?UKbew^cCPnmKZO8u5>=PW)A~#% z(&d`OyFZtHVjt*`1lOVcTkOmH3(qvvmg^Kp~E&Dtur!O{}>R2BOf;w6EzBA>l6tI3D1R>@FxPMDg}6dM-5&E?ukhV+XmO9 zZq2a3!Rq_kYd8XCVe@wu0V&x~&S=Z;sNF?-pO95`pNa^`tr({4c9i0>NJmXdSs)@r zy_1e5xYd1b8>(((9BS4Sj3H9$d-z}Tchm%NfeB(&+gQH?_VHgch8=!!uF+CW>X_## z@){dxuZfFwg{==-Pz;x*)P_Uwr*-dZ)$+Ql8>$B52Lu@vF#!^M7p}C!PmVov+TnYNReIzu9n(nZFMylWDR=Dyd=SpHZi$zztgL_O&*dS7haDE!Kt9yu zE`z|b)0m~6r25pnX?b+Wm#)(oKeh>WtM<*nW-X!9R}YUi;ahlWkdf z#^f5qn?0zpOr_E`aSx1MD*cH&Gq?^+1kLm$lxb%)wcjZF8HsJqgUEUj>&2nL`?m?3 zVIu|>6<&xym3JI9lrk4HvK>jr4fls5%|Y0%%gAjRwry~vUgYK`3nbJ_qjnj&VzX3L zjlvtm^F7dRs`c!h7#+^Lp#vl{+@?N7;>MCEoi&8*=|iny`0R*=#+|rVo_A_)<2W{_ z1dTW^g9=_0Znv5R%M!PdlRL;^UBUu-r5QI%d^_ss%IUT(0aQ*zNP`N+&2zH)C3%z0 zrOHFZ+XjnVJYhy{SQJ}*lKo?#scJvwYmZ??8Sp{-Y3_@(2hA{cWyJ1!Z)T!rNA+I1 zm(6m;7pke@cFtw8*52KWt3*q7ZU3vr08Cdmw)QxX+pZzZPpfHKs`L(jR@ch2_Y?@r z*j6@QD&iP+hVC49s^{KfMmP!1PVD#Y+=hX4J!v@%Ol@N|)TaTOd}}(w&(!S}9Q9;Z zz2#lszn#CtS}_E%FVa=_j?-U|8SG7c9|lY@zFq8YS)gH?Wt&Z!apSW_R5-8pK+Rdt z%&IEE_WkSm7;dN`nO(-{-}{m-q!h3Q`!0+PV>WhW4csnXY1`Btllr0YQ0w#X#9C}a zzNceVg7BfuQ@$fV(vZS7XRfL%(qpJ<1kWQ+!P;fD2W`QWoEsy3Ws32e`|Zk>1ARL$ zlz(=fYKeu~n4hV802Au)suB5yOG< z9c1KoXI`@bQeb|f_9)6y=YhZ#^+ovtX9c}Ta6WW~Xi|nA_aJP{;SS!ytH-tURO46y z58LB#CeFB~E!bD)cWZ{QQ@VDQPS|!85c>zVm9TEIn(16-FlIoH%ds1nryh(0bYmz% z0cYC+$lA`##*HL-3)$*P;#S?tia5e9m0xTVyfgmx!J0~G@Xt|uURKWb{>PkkalT!3 z%sT-mThr-F&dm)*)K3;3!snEoy3hD^qySYQi%96ePfpyd%L6|iAyse7{?cz*`7X|~ z^9_Gfz)Z_1$IW@P9?m3M6bapE*Y%9}>nIMY7n$FPxA1in-mTH#tHa+KJ+hzn_0^Wd zsdr{qT?!a&*;nD~JXN>KnYG|m%`jc`GI%uVW3}B(I_Vakao}wCN${0neB00LM}2lp zxVYUNpX>Go3^X6Fdg^?*u7cle(O;Foj@9Gy4Cp>;QA|s6KR)6>(_kj}@NiW3yX@nA zQf-f7-*p^l{3w!;!8N~}`|Df<>nvKUHggsA8o2iud1?g43MI5Wd(Hwp1vqzf2K*$+ zaw@DWBD81hN%>CC|HIH(g+FyE~q!f@)I;FdFyJL2Cc4oVSU1I4@rR>J; z?m+$R?*6%+XXk1!zVFOA?;FxzNH+Dh?>I$CbUxH%Ol`Gvso739Hb_@qW;Bw19Y--{ zl&2k9KxbyjcaPE& z=}}JIS0dAA^^O`yQsWm6oD-#m9P0uFEbrGX_PjGLuj_VmO|7SB{9vKcN7)jN6*=$l zJ>8vh{oYzlUDkz>IAw3V`@n43zmWHxdnI@u+vY;y4_D8+X?(eLvbvfZZk#G*z+1_b z!`=-)$`9@NSGO|j-SC(eiI3?=sXaoMcG}8+`NTI}mYjD**W!eO*7fS;ym8}h=^Vs| z3>`YrMk>FtN748_d**Ot-Oc#?{;As6p_Uzc)GK_7o66Z?mD>u zHmu?YVc7M;jFWqD75pbV7vO7o!rBzV1ulluM>0l;Rm;c;L0*3gZh>rVMh z4m9VUF*>&zIeq|hTy<_R9JgE%t-FY?lKzFt2rtA>D-B3o;jQGwlvw`hz<63IcZZXV z@e`)qajW{#B02U8O>JoDAHqDV{jDY9QnhmSR{RHbV1++%STS!qi`*bN8n}XbOPJv_ z%0RfIw^)@g2MUf7QPl4Az8LhcHf6OhmfJK9+=|~&Coabm7iw2-pGsb?lm^_T7E8`M zU1QkrM{cgFT(funk@ZzOhF|pBpaTY`XkYr{gx1Y{aFR`c>ZChD#L>tdR$OLyL{B!FgZKYfEJubH&C{Y;MT%RawjF#MFp@a z$5lt2r_pOg&#pbISb1sFp~T7^r`C37qhb!5svyja-6qUW*hhnZOPle}I|mYvk=8V> z4cJOiRC_otqnAsJuT%h^FsTRT1JAJIo%O&i{3@j*aGy}X7?^BTA1-lVZ6UkGUuJbv zcKh?dR9czcCpN%9Uji8k499&X420p=F^4hCh?Z*^pBO~C7c-u5u;>-@GSe#V4M1k< z{jFGefUkWJxSzG?LM6?EcYY6!rr~zAZKb_M6tWfc6u6iAmj0OYU*QvmlCv?+iCGS5 z{LTRaXtq5UT*elkqft1jxLx(s1&TQ>ZIfZagrt`?U3`mjiZ&veUO0m;5ctOG7y=&A zPsO}~4B8)IftD7H)`}^V@0wYYNx(tn?1JD)8moD%8U31g zonJB2Utnkdi8UMfd^(uq(S2v+20Hlpp_D9ZyGrt+&>Nfbms6FssWC;g z8|sgKmW&cvtb-pACn!JhnQ-?IFm#)!+Ow!3krX~OD7Z#m-IqFf%URm~FHc3?*vOAr zP2=ir`^{pQDgJZV4lEIlpIAtso<1;SOY}GzsqZDe+xt$io-{tpCe9_F=>3{!Mfutm z5Pg^yRUhMfhyF%wA1US=_j4{&|JkgBjTq|>k0K;mVZR%Ejn-6?3VYISAZSiM-3+VZgwkbM*P)*b zvj_rok#TxsA@qRRWJBUa0z(J?a*7aAuLSx6OSBUZp3?)bftEmTF=gy4P+I0&_HOo* z@aOEqZ1zTfXe!&+<}6grUUndYeLHDVETA3P>Jik>|73G6aGzW4x+(Dv zp9`7_o4me)ZM^TcacmmmxtGCeZg1|yvoOtv)ov^o{bTSmYfA0Ps^`Gp>eP(oEN5lS zwtXy-^r6=TXfOO@3$d4QKkmr`UJkzQ&;k2;SExz=xWgOt0g9Rns-gfweNB2RFsMDS z4PdQRUiP}iIxZQu?E&}jt9E06;ysVrUjW`C3zg%*ya69p1M^E)X5~1rxP_MP1*Fu& z+X8`;)umqJEV_KT?HQ0L=-X8ftU8w8p2U2zU!nA7?j1eHTE*-gcvTt4Jk$9m?E$c+ z>HOAzfVhV1)xio?F0p+A1`2l$4a(o5Mw%em7L2vjSJH&7q)&(qaEHs7!kvW8DF%W< z;^WZSd~b5sdTZVmYP)3)x0C*Dr&tP6jT+^W1=I?Oo2ZjINZTvSqAe=jE?7saNnXWY zOOFT%J3}C?6UUT^1U$|563~S8RMq0`DPv;r2-G zd+xL8UL8DlmheW`FD$9?oJ*fX*cqi zyCE2lxWhj^4#GX0vsMZ?59;VI5WLaes}u6?Xo3W4-k5TL+{SH_yA|C;4oglX`5_BL zv|tAqD{%H0foF4XTixc|f|-4xf6tPIS}tkFDGQY$8ZxDXy#y)g_3uyFVUYS zTPQa+Jr|a+&bvzZCs{AdZ}T!htEMV-mnchiPuV7T%5s%!`E(R0ZRMF~Js00V{zM}p z3;4~ZQ31qx;o8Xi2Yt8L#O;ArHqum;>L1dd3YJn2qHFvq0_w8yGK#1GXCncqaV3USnH zfm{k~qVbJIMAsACll*Lp8%P5Zr@yB7(K%PLPrj_>A2U#D*YK|Lq&T7WNv2fTTKzbx zU9envb5l02S$fp3wU$Y{;sEyQ_^*o8Sp3{(#&qfqFj+lQxKX;O0NHi*;)Lc<_d03N_O>e zykhG|)i`mR_aFHUvV|igoksOA5sJ#_G<8>V067pLH5ili#Q*BnlK&M4=+;q?<#8ovNto$N=^%a(3@>=y$ z)|9Zfie%PI?_6mY%ha)5Je`$kIw0V)MwAl`TLs0O=DGs@BEl9OjeDl>m8KhM+}^4D z0e6S>$P+kcyyGN$p>K|tL@toGnK}O~TckA5dn-3XuWJkCYw&GazH~}qsd}$?=Jq7z ze4$YoQ%2z@dB=-CbLEczgbet)8H?}AaZ-SjBY-CmriQ0|i+8UcRsSoTuKJ|3+&*6s zAxjI(k?s}iywk-|f~QXQf>%5@v-7-uSS|ljH`F>0TA_Q_7>2u1ZB{2NXj5ftZzt`S zA5wdUxk|m|V((bdS@DEZkborcGhfIHL&oKVy4ao_?0q^xhcyn;9Bl3=h*$pA+b3b< zxw?+fkCKOKBX2JeM#gc95ts;nns4HMOl^Y34bvZQY6`ixGjaBO>yVQ0;2dWF3e z^Q!(UzBur0%~YbrdZuO->91{{VjXp%5laHm19`qZ-$*KIc*hUYn~IsOtI4UELyZyS zk1@~OSLTPX<(^i;uZAs(j}k^!nWD{-v1I z#6w+n*qvctTQ}l^eRCRoh%;QvYrIL1tvuCkly?SWvOM~HnsnqD@q5+D9dk+Bau4?U zlMW^hbuh@~VHcW*DN(*b^*5;7UD1=hP_mU+`JH~qkR{V- z1#ylg=Cq!IUWN5FT0jxLS+$ky>ebpQeA+j4*C}w(_>GNPAr7NksEtANFv7b9WCuVZ*dmK5?moM4wE0X7)q7gfg+#Kw(6?feJt5Ky1vH z?7bnQ#oW5y*^-hmpX zS-yVP1fwVU?2s`76FIvtlK}>R9YV(84HZqbOmj!aS{+kn{%5k0vdTa&y#X|!vi7^e z;(Xm0lT(s>X~#Z@73tXXlYJ^+SGx~8cEg^=Z197ltcD0SSkP23SZKIj@&)ukq4$0k z>GKCh;{|7uUk=vsJtD)q?{O~&ylb;SOTq@PE}eL{wzyGK|u~1m5LoX=TZB1 z@6po>mJY|%+)Z)nAF1X<5<35?A_8q&Lga>?dG(0+i<7YWpy00MF2!Qr45Kd58Mq#G zcz0#XngZkDCHlW9j{THckI1sl{nfNUt)lOB-Lf2Jj?U){rrVScSIl( za6J-xrTq1w%ed#2zk9agzoK5MyNSy&M}T*vSe&NJj66g@Cz7asq>N41=_@FC_7lt^ zTIW?eR1Z1#z@$->{H}|KrBPtjY}|dS9TQE+q24XINt#D%NT8DM(s7$oXj2&34ps~; z6L;xe_WQyk5GbO7Y7-A zE7WlAa_PFdklpU(Av)VuRHeJdSK5d&RMM!YFfOt)MbmK@$@(}g;hs?DFC^=EmmT)d zj>AvRIuyU@I6O8~wyH(lvarIaK`F_tvaTJa97D%fzbf>{_N$EJ3<>_SE&j=5Ly@<` zU0NFV&FPaxA9j8pC6?OuooaeozPWRkIKDEwzbIG_%(F} zen&}mDvYBEl_x4ceufrbK6mFhZM4!vKR{oEnj;XU}A7WOu$HHHiES|C2VCvM1*y?JKo2%8fCf z*6Y2SSx&!fo5FHu+8=pOxWwK%kU~5FUapmpBwz-&om>LW#vP>Ovo2)6p-x)JBFVJd ztQzlf29Z^7+YAs`BZq$DZi=h>5d25cEZqmfBLNqQBHrV#!M2lUb1!GFrKBPHhQ1yG^^J=O>P(S*w-c6-p^=L}-^AVRjH-OGdM@$t9xF2s(A4 zV3qeW`V{U?yOqo+*kk`G%*qyi_YQ1!!?)@J9Ivi~vlPEq`vQH9xLQrf+)wJ1|BYBf zDVIPS-_vM<$94gXdt8HkLbO%CVOIx+)}_>-v7>F%A$Q#0Mgn>*0ayDb(~Y#cIx+k> z`L&{X<5k*H@iDu62E^OH7erxpg?C;=U)p&=-GyD#7s4jvP#wJ}BK~Xh{ES{=e_eby zo-EgtY}`dXEQ_>bGW-N4d+Mvc9)8|whEClJtEXeSMizrNar65zRnPHt9Xm37iP?>3 z!;Q$%y0MKSDp!HGQ!!=;1IL=6=VfVaD^QrLn6QD${z&e#3WZu}aiiwoVnTS!QE(>FUT)2LO8m&gv96Mim5;OTlgCnBz}b{j zp})Wc>emg<-~)P$RUi8>xTKPC%_BqS>P|$ww+z{OM-`u9ELT&T@uU)=ati^FlgNMr3V=g5u4-|W*O`f z@((cPY+nB#YZ;Vn9S_E{IXkRrsp^ylN7}g3RlI^eSN5Cwiyk}awEjpxCF)O}&!7mp zLt2^9yteh1fI_6!TERNTfd{Toy&5F?)zqOnrpSagPq&?FN~_Q~m29H@P|n{jr05kN zo`e`P9Ar)FhnQ$lfc1OeBllw88S>!Jj5$Z(b3NTgc}G9367>X5K88=zGpNsfR$Id+Mi%U1I~ z1{krW+)Zv**!y9Lr59ug-D?hjgtZUlH=y-8H_(l}QwyUjvw$1ZNtu$17(`g^DkprB?^HUP$H!eWxaNloA8 zBVe(l)9osFNN~!sogIq2Z3Ng>eV=8u;FT^uYZ+MF`VmD0=Qes}uVCG+B}V^cJ=FAX zR)F*5SKTgw<)UYn=fEF4!-kXKsZkG^IY=0M2{eL>djQld)>zy9tTC3j(JFc$OI?$( zxqx*{g}5C79VDHWufTo$r8P0?j#81JQF*DHN4+oqR2f;mNM?dEN_UrJV2mO%L<6{R z-!DRALaoya{utTTbQ@1g{h+H?e8$-c*2t6bmQ-iyZv5Y}brKuGo3yQ>X40Z?h0t{p zm0rxhL^%g(saautCP%wGs&|=QVzcaFEoXQQl9hgKufo2ctdP~ z&qHn-dzI5mWDj`2Y!v0icP&6w3kK`we7r7Om<9`;AQ|P?iyj$B6xTD;B zp9{!o_?z>5WC}-ZJ|E77{M9a^=UNgnTew#M) z@)k~Jf8EAb4pbzvTB+MQw6{3&k$|-gEf^!=Xfa1zvRgwjIw6yIix`5 z=A(smqLa?HoUi;ma{-5ftW_Qo)^!=ea|F1yWrTx#*Txq`rQFYT&B;;7Q|(X=`bY!2h7?gTCa-xQ5RXiJ+c zob+tNN69owx!n)pd78EH2>%>oif~y?EG?aWRXd$Vt;$jN($v{El-acNabxV$*kp)j!1~1xsRl()XjbYXA-dw7(vJT%>xk?Pj~Ba3a)Ij* zG$FbHf7x(Q@Q!oJp20VQ2259Q@$61si>68)PVG?TE0i{h6Tw5B6`O>6xLm=rvUVENL6Wv|;BoS7Dn9h8mo3HOa}fuka(` z2HFndbbmW_28rqRpJF9twKZLGmHNo=n(z#L9~0B^gt{72)Hp^BE^w=7Qpww2)kIUT zY&%_TNz3s2t^7%!@Agb~oqoysh(+vC+hy9< ztUKFO>RT+W-y6j#7Gd2~=@C}G%>vOl3uW|<|BoePtZyTdUtL8iR#rDAJ z%gFFHjOrAu@p~b^$Z1$-A$bT5*~AI|unaJt}E0W&fBF+SItX0 zqlu6wZL?CwOFI3Y%kqRj*G&?g`B^qc1b30A#@~1`oOZgY{#4@vl&tnkU3VT|`&t*9 zq*Px}Kim3QaY;e-J0$%iUAit)>?jtXN;{;V znG~;9=zeY8rx2)b`qfC=wY;w6WuMc7jV81##x|9WCD6KTgg ztg)}7yPH4b&4b4DI|%|WlbTGDiBqU%D*2p6y1bRzW-vLJqyt!HUmy8Sk)}&cS)L|q ztDxjZBTZqHiJ-1}BdW~HQTrdQz=@>lqeojDl@S;j26seUW+m3J*PJz4w64>b<(U@T z@*TJsUDj|F;0ATo9tH?rk=3mL+i91Q2Q*vSNpZjf!(w3<%NBFJ`vT9m@LT&M?) z&1c~c(NX$-&X=GaHBFoqUKN^pDBbCcLd)J^2}&B+7KV2OAHi-+Th}S+*23Fu@5Q*( zZ%r?SebKJ<*Z6;fI(27wMP3~B8RU_(o%}Y8wLC5U$}uzY5oEBxVZhG6>Pv-Z+Psw+ zsgIgA$-YD{t9O;u1<7>|qIfTX+F7v3IbI&jy=ygFjE1ioA^fMDg_w%Y^x89pCt5qT zds447?ouC#`d!znI1|Lu_DU~%F;sg+51cDym-!#8e8n@l0Y*po2l-f8~$WX!-t4L8Fuh{4V-yiNMTDbR7sbPBf z=!LSeSm6$R1wQ0%Z%9>vZ%q4fG=JTaCQIy1TTv~Mu*u}P`Vi^F>NheJ)wk&M?s{}m z`nO>nCMa&k02!MVYTdmFH{c7lP7plSZEGkX-m`tEyG5=yC8#2))HMgBPV|tX8)Mf= zr!uDPJWHMxXVuqAMu*0AVkygg_q8}t{#&Q1zd*ff7pg6xJvIHOoXwcK79&~Bv?w|@ z>Iqn;f8Md4ITGj4yNHPkWwt+LCi(tu8fIeFy{^MC`|Ng9PoCDzSn@AGuE89!3vj0B z)bJ_9AYAt=;mF;Q2SmOkqGJ7Xq0a6TLQ5EQ4 z^kAq%#LY18za_XGm)LcKA0OJ&x`+GPFQGw)G`N@7kl_;hRCP8d+&oN{4MiEe5?*Fg zi@xqWrO3?)>|Z6bj4SN2k}L?_**b}7^2^e{7nr;Ab(eX8_HxyJgkUa_Uf|p@^b;mQ zVDa3Y&egJvj6Rj>LR?j6pTakEphYbm{x|V{%C;?AvSZ_8LbpYROqn*%FI7(#ubX^N zmSDGiuPTfeYkp9gjaVBp1Rpu6#Y=WvuA7-r(ifw>8duVZR$mLPZHZG{_Y2e8N>90S zbTdUq?R%83`8Uk3O6PDr3mMm?2fN?Hs6Ln%+Dp!%W;5Vb_ z`NxQ}FwbMCq%ZhF-%zR#@vrT7`g_XN!TabyT~tLT^aE5c#^Zu{932@yp(~Lnz6zq-MgXs;>&GR zbY0naqig2T%C|LVBPf_)^+xYn{97r}-k-K^A&NsPxb~UJ97tnXA*|&-# zdec$w%c9!8XL?s64V4iK(C@W_-g)>g`7!%ga;xC}9(Q7M!QKuzsj3*Sx=FSx{lcD3 zep``?T1dHGbtPjq^%-XSwlKOW?t|wrqk}kX?G02@_Uz)|EYL;mj`#%32_-;C$IWF8 z5X=DI-(dOw$OJi=ee#E!~%o&C&pCOl;@hAoSNfv~e4jVGQ zVCkIDatYoZs!rWZoWcGRmPKC5F87>5#e*+wEEx`9)y~TmbL8!fFDq|L&qyAih~oG3 zqnO1av$96)eL-@H5dVhP8~TA{$DO+2K4ll2V3ST)L$y27$`o~W4eb?7UAB04)kY1E zwg>f1Ib1r9c`AFJ{0_fRVjQXhq0`-OD@0_Dz}CP6Yfh>Hgu7DgfTXKv|Zfw14bp{!CCd^%W8V6M6)XB z?ZZ^NsMhD<4b=z> z%D?narwA%Pbb1#*K$|zmZpUIpbrB);1RIU@hC=cp>0_HmG$;Oe&u8Y>g5;V8z_54< z-v@9ktsor+0?N-7tpjjXA&ENRH2U_IR8|;H?4AlPCSomgY}-2k{nU2ZuO^rLU+93?-ZJgPoKX`qH7 zHB@J+mJmtZOZ`+Zg@&PF;;+$@=}Us_8AbGy?$ekD8C{lntO90l`*&hLCrx8Us)J6# zcI0r#0dGzI%jV`Uqr70xkGG^v1-k=B>7n5Eb&nZ0KqJeYKp*Qv8xH?QB2|qOeu`H? z--s84Je&!sf&V=(g51GF$4Mz~x$^^aXqj;Lx+{#eoOsL6z;gEQRvVl^i=TAV4rolE z>4XMl3pSXjl$+#MlSU;?aVsgyMfQOP)OP-!b;s!-H`vO7sp6bzS%$?nPgUH&b?VQu z1MvN|r5FWaU-i43f5cxZN^A!iCHomLL0u`vuRB1W%YS0Eo#~3CG&^7#dsZs!u@fEB z!Dif#=KYu{gthvF9DrD%n~3!zeODI+v{CwG7uM~h^@$v)uS6+_P(t^dQ#Etn_%hTJb| zW&DQ(mrkiX4tJHOWGZ2+$}3U1oPB7bzm`M9`MTJ1&_rJ|bIurrR(F--TxBDk4t+-r z(w{+Xm?af^pm;1bLk6wF&x_=;zY&i4{bT0M?*8z?)?^w22nd#xWhK(-PhU=MkW zRu3W+r*b`*LK#SJ0i&o%5m&$xnz^40Y^Tq1{>C|$UuDMW_5*yjfxrmZ1s}JU#(pa-tg??!)8Y{+SHt5VsGcRGUWRbLJ#$DlV-XEr;Ktw?^ zH}d^UZ!rV7&(dO;Or#ZT7ZNg@kMZk6*-@=x~gJE?Y^+`?iwMP z&{njqEIvRhZ}dwM(>~NjZtI};X%c)<423+_Wg2s*D9r)|eBzGCJY>fCXF$5-X;A|9 zpjc7TnEy@WTlOi*T=1b17n;o9kG9-6i`S0*=HS3RL|kBeooh--k>-i#RQj==h?Z2H z$1WC5L3`#q^50h0b4S@i;nV+0`HgwJX7Ep z>LWJ*;N-HAEMQZ@EVvJ_3qB4T1M9u)IDJ5>Ln5aVXf=)Gq_L(6N4d-RCJYQB<5{7s z;nUpNIrlif;rsF3ob8;m!3yXb^xn$`8ini~0H~2oGHro|!OMcH@G1Fm`UN;gwy>(2 z(kpp`=k-UhHQ#R?D>OT!oTm($mVc2n_08>a}M#VI3`VFv>VV#J*{#S z@~mB$&12VB)B=MpA{T}yUMR> zWwBOKih@(W%A}ZikyVo2*khZmMQ`vs+}XnQ#3MEuem!NWQ4VhtjRj^YFA#F^ZVETz z>Y~#!G;x1wl%$75j(8#ZNzU7JMwmsRx@YlkQg_%K;SuO&Mz6V{j8;~X{3p{CXD?$i z7ZkpeoMOC7*&|-gxE#S2!i<-j6oSP}JNE?MFq38TiAx8j8s{N+KnXBq<%kknAzc8E z7siS)oX08Nlk}?T5dne-Xu~EH?+?4iJ%xLmz0}qU8Jz6=AA`riaON$kfn)^pR(wtD zTJTU5Cln>$798f^5C6+g;stIh;w?iaKYVeg!HKpB$b8OJla;V1`y?}5Vyg+ms6^jX ziwlH8mV7js%l|124Ikq*h#qe8<5meE_cElM=VOb7dB_5jB+gOJT*g^3SRafD6lK(8 z7WfL@SEnU=^D(M*;RK$seEO!D$WQSD_Z0Y#;IJ(hw&0GN@HnOLaz={StVM$UESlCh zE&r{ct}bu;8@{nt9PY&(S2g-SK$>Ku?qP6?=(eqtbCK_2GRU!jUlJEL{mti=-fM^| zoXBjeqnB94Rq7OFV2HP-p>l<{v$7X;%lV+}81|sWU-2ix!?phj^C?RSKN~#DkCnct z>#R)5GSY`sb9nz^F3DBg_Fz+~Cz7@CjOZh5>tZ6f#aUvN&$|JwG@@|lus7mttNDtN zVw9RH+m{X~?ImL|df72i*OtE$s-St}DN!I#?-DB5h>TiIQl2W!ZZ4YxN~x_Y>iZ3 zUA#ioR*Owvp~%uk#W+b{sQzqe5zFPujj*s(;^wk}Um$p7b&@OOUNE+RU%>XIH+rh_ zeY0+Mye#A-Tx|_3KDYHsBfiYf|4E&sV#32kt3&Oww^4OsdrY6oju0-bi4?n#-<4T) z?kRhpJxQmnU?==*I#Fe`^_t!ub=UuT4H0w6W1VIc_su?5@tx>xmLv@)_pDtc{6Je- zdbE89_CnUBmS|jC!m)-^c>dO*x{U;-|9Nc-@q$N~dI8zXUMQzi_Lv=$OsAEs9T7ZW zEGdm^eMFUIWjCp)LkSrDO6q@Gbu|*2i~r^7wX|N33S}!j-u|M@kFnU?QS^cF$smaT zgL%DldP@ZOF3Y6R7_3OxTsO-4w3Vv!VM+Y&tFN*`Jj4n%%fVrh)SG2t4vUts{0&C= zzN~_hbItR37c&3oe{%;DOlo@(_0}|PAdL85R!wlk9zAj`)awu=sbK$SeqNZx4l-QD zYXtX{v^6b}T4%o2n~E_BvuYj*4{r6V1_g`#_o$qC6&{1K7syqID)AXO&SIh967<_J zk7vogRU&8%S3S*qQjb?O$A78`la*|>)V!8h`gbbz!rLB=vI>5k1794+Ew%^}tr9n6ZKvraE;O!_ z-ysWD_lObd(=6&>4@R4)?M=W^!}~if;#dI}TQ=awyj=BMqM7p#-C5F3Ypr?}CC$WH z#-f?8$r6>&FJz_lkCPuHGP@}h#i=uRDjNy1ZAH!8SI}CYwle zO%I5TAOnL*2QfP(i`DgAb~^EVyPxDv_`T+};yHn)^*e>@y~cH6{5+ResyAG{%|BT? z{M_`NsGU=8P{wzFJhQ+qq1q(zTH71Nc=*MpYcgTrjQS!;u-6XlYvDVWrz$?bW%5^B zxM;H(q6nB_P{+Fiz1W*wI*>KeWnE6mbJH|edKZ>J7f_2z(CGb`MdhWL5?o+aS$GSf z0mJZmNY22I+IZ90)ALcAsJd-S zu_-jZ*9O8>y58mtrJHemBq8fQJFSJ2`v{zn3k!Nd7LZa*2aPM9mr+?a(!k1YR{K^E zvzRq0bH`s}RoKoXzh;#Tkus=acGIQo5z!RspS*nn8RKu^KE8ALy^?p_)2SmBc?f6g zbo6vM!1D@zI%k@#Cpn6JaL2W@dhM3R^vqxCQe zWtalx#6RrWiA(1u*=CVWac&M0Q!SgD8mu#P4PIhF_J6f!XeaXhw3%hoijJw?r=*v` za#EN_6;u4q6Tw~qqkIlhTe21_*Cux%aG~dgtzJJ6h6JSlJkt zyrs0H=1^!=rB*%9(+z7V-C)~GJjti_?ZX_;qSo=SUve)AUGPf_o>SThe#MWAKM-YQ zkGDIJS5$rr-bFcrvG#yzr|^(vHPeM`(`!>XST?5jGh$R&*Xe z6VsWrneYs^H`s*qo$%5lnOa1SwEW6&ratdBFWXA6*Tq(Z5NGhhs+JJX6aS&^k(vve zFgM7G#2EYnO2ZZm5l21Zv5K;hzRYSXJ(9uibT6i}W>*8H2Y{PMZ~1U5>npH z%5+gi{5JFkL1@s1N#C!Z`#$0UWS!MdYCgxR<$ms8Jy$tau(P%f3MzKh%CUW=BWhBf zMFmcg8+RCmmWBkr#de4++`Eb1ykFL4l<)Ah=KDF%Iu0wIPwstxu%V)m#s!#vB~R+6 zw7DK_VQ>(PTx3+vk-U`k42 znm**tDfh2?9ecm3s``3hJm#vR+Z{#VPuj7PDcL-9V+Q4ZR*Ec=ipy29Y^iSxgs7)9 zbTK}gLtjybk1?b_sZ?)X!?=gO=~@6d;FHW(u+EZK>z@(+l}?c^BTXq^!u&$IT=A`H zgxp*8Agh$pg?=9GM!kw#vk6bPCzQH+GA@z6nsb1o)V=lQ*fE?~Ov2B>OBuBU69TzX zOmrpYWs*o}5-y5Censxyw2-Q%{Bw1nYiNA)r_4JHzuJ8CGR8&W6-*Ojf^LU>#F(fk z!&@_MWC#gej8Bp0N%_om{`HhNCg{4DreXfH*vMQ72y_#buee7AR8$A@h58L+fGj94 zz#_0o`dR!sj%Q>b(H0{6N0B4gMT^R_*t0< z6DnGmZirhc@Qb)eu;&r{7m;X_8~Hy}V|bOt=wwIL|A4C-5E5v4g_lP$Ic`iRG)!bf-$8-ujM>AM@s>P1`y!=+f9db`)cU@X( zJPObbr7gfn)pNst;~vSg{91^4;zpNG6nFj$%h_}bBwhWXbWzU|ZhEpScBuJn(u! zNKp;>k1QldFP~^-vnKrYr(q!8}v3 zp|FJcy~=)jHV}vo4kH3PuwrimRw(|L<95)J)NQ(vy@@K4WzizA)4|)cM>q$Z9X$f? zR{2?`VeX`5PnX7*Ow6##{o%nxz0Yz~v$eVY$SFBCz! zDa8BI8wu;kv&7ht2ug`?@y6M-DZF7vfc^-nG&{$<%?TDG4zWWh{<3wA-m}}vRJyCc&3rY}2Vo+9g1{^{vXI}L>r-}}cduYU z<_g~YqQ^0z+_2JTfjp$X(s+F=vI0%9)x+a>Ln8zpCB22?k-=gT=`oUBsw%UEC(5}Q z>2PKxH(JGcjv5L0#gSuIdHmy~;p=U8axzJ!#%np%lhTS$&~x+{u^Bp!`B-WSk#Wn? z8ITEnQ`BSjLqc@`lYNF{XL zfg7p!qGCX2T4sRh3Yc{LjL(4)~JiF!KW){a}a(Ty+v3eQrga)_+%p}3AqE2Qp?^LRY zSdcjLSmf%`|2zf0YLjSjjc93t-Kgbk;ovmq8*RmUg730K1=_QvJB_7`;~Qk#dlJ zUXdPgi?Ln`Z1!ec74>*zF&p?d>|%i?uBGWx)+lF?d7Nh3G#i^vJ6C_Ju!L@0!%iuu zr)oY&)Y8Y5ahq2#s-#06VazQeW4i<(jyGuP$*O|uDHz4`toaqc<(iy7IiSoie>~xj zL{!ueMiTui?ekqCWLLa*-7GLgr&ulJJ;%>6;B()Ru9F`~p`xLRW=U1a#hf+bb!8V4 zIKrnDkHe-5no#yW`}r75yQ?KH4IgF|!NrnX44sjqRAhAGL%K)JtF^$ zPvl;u&_nHzw^Rq8fXQasbk~KjnD)bJ4{XjbH}c>-VkQwD@b)C+-e(zP#kFz`h9;j|M?y0=j>h(oA^S1j6`rB7F6rf5QU1u-&+(BD$3B9n z(k&r=?AM~R-gnszf@0V8Y#wi^bpcxoPZ(#i_d%a<`#H?I6(t#*EL}`SC3I0kiB&_( zl_eq5*|2Q0_d&Lg*uvG7Z6z?UCa?pzVa6~U%eheLUOSk1_kZ@@GODev-yWsz-r{aW zi&Na)-2wzcNQk@3&Woo z+;-y@=`r4K^p)y|{5jBvS{(0B*0;)SytfIK>dU-`p$L^F@4fd;MF-#8d9~c1Uv2Fn zn+X`2%#+RlHla(jIg$vds%n9_HLFj95j~B+t{M=zg|1hw6w1Bd%3Hzj&VI5oP-~5q zrho}1C6aQ`0KK?so_Y~9Tk}9Unnh5<6x8@$hLovTG8AbE$Nn*vGn8 za#ZAI(kFf{OhWCeTwk*W>ZDn(4ao9WjaC-LQ`w`3hJ)0^}QEnCW%dk7`GpmP* zzGi>Tm60NzvA88BsF)5uk2f@ zrYNK4p!x&-y5Vdknsu$zyE7d7C^w|75ucLm+I*C7GSax=9I4RXyY@E4!EKZF2hG9$ zVWkTr&iuQs<94gzHbpAuPRYLZRaAM-u@*TkKIu~95xRcljJkXD5B`BQ#>|;+=c|yc zsI`vj^Xv^4FeQ$=&L}`$%G+K7Z?)i@$YC}ObDkx&H5ln`yB^eLaAW)vwVSz(ZXYXc zcxG!6szKgii$e;K4;t;1wEd3#PsL#nJZsi5|`v?rpiI$5&8-@0;Cyw%-E zT_hS=JF0XQezXjedkB+_AyPIt2EE;gsC3G4uNSI&l0s{}RQ3^k?FL1&|I$iZ`5gDz zs!LMA+S`h)l6jUi*&)%0aies#@F29W;X}>K9E*BC?enBHHE%2bM5Jp&)$jcNYBniP zyBnyI<=fW2QFusqSXRk$#czyHNk&DS>|;BsiWVi$+;S8;5oNk*6gCjJZbNJNSx;C; z49eW`W%CtmkF{4lh8SX!RBcApTk}wp!l=r@Z^gjZC#(9#5P4Cgo_7@;ftb!$7z59q z))TlhPLYio;#TWjwQx$W$>%B@eMEnc>N{&7Cuj3-OnowTV-aq36suc--yK-pUPt76 zK5N#J0jKWzYU+TEbM+?rH&d17CktkPR9@n2%yHQ?NFGa$?Qy1Tiz0SbP`3vjYgN%s zc)2w0W2|ucTljNs--+VGrsBqZ`zQfSjK;qGHYGnj0&zpvcHe_DC1c=G*l|8IjEHFjTCFlD16AegsZ7 z21@UE(Q11n!_Hc*yLgwaM*T>%!#q*(L3q=MBI5{sbDX+hl|VA1{jmCB6rn|;Di7S= z@KABX3tAf}4|P^m4NC`XtJHLfuQ^Pi6}>mwD039P*+GDB&kJflj<{A>uHISkt0aWu zj|nKdgJR&eAu+l2#C*)GnE4c6{BG|(bRN0Rb_?5n#Ia4*K+!ZBb z4Im$2epVQwPveO>6x>VVpJ;DVD5cN4kouP{x1DCKVr|<}2VoKyw=62&O~NUXN+-yt zn2zw5R7=F=3Lly(n~pioSQ;fKh?twbO({Fr47(VnHJ7njmHU8`(qshzIJL6s;^&;_ z^pmh+u3PyQ#C)zMs|od%H#hP(elf4o>lL}0A7&?JWC1HS?aBHob!?#K-INfd3ks{m zM`?tTXz>d87Pzknl6koTFFX?|#NHH!c%2~iffwu!&^Lj^jY~3mwUzZVat13Oi;om+ z&;(MWiyaQfFjKR7?$un#?CpXI z&*WZfyGvFSS~aJ`)|YIpKaxIO&aN?!K%s%ka<6j2b!CseB`sF^rJI-%wJp1PI%8Ac zPvN4RH$9t3_aW(>p`}}(-&#(l*T5e)ct@;6FRWI31rP-4OZGn0zw)r|Mr=)9a`kz< zLt!|0p0E@8mQ+PbC}WozPz;a@)6_IG^sX>CBLmNH*RWoZnby|a>+};{%aOxnJGDa8 zYebqrgLzspN-V^^#(0*@C)(o+rau!l2}uSK329_I}gG~Mr!sam0|2u@Zjr&XvG@0CzN67C7WZ+L~&Qci@fEX zP4WoHU|TIK47#?NiZ+4uH3S#DL!fJCB|bybt4;@3B7x2u#Cg^$fW#yjb%hE=(X z`c$&kg7KaqW?yl3=Ot8C>ATj`1-$ZejoTAzQ6Fnn!O8gas$#c6^0dmw<_Dus7F9o+ zO3K4Y!)UBRe+GnJ4n0zl!YC?p%Qs>Q5QpM#v2LKTfy+6i_(+#tUK|-}F$A#bBeive zo3K}s1tdgyCEcBTtYRM0nQDy&@|Mwdx<1XmK=HfD2;_Bi}{N#<6NFYj_CG5A7=5 zab*^G3jK{ahl0R9WcI<|;V-cQbD~L$Szlv^C^Ggn|Ks#v&LtNLD~mV7@;SGLpIX_3 zmt>^N*l9i{88782d!4tk9fPVVpV_ zW?pN}SHq{P^N0#$RTT<`McZr2vof&1RLB?<(NKZ&H=@HS2^z_%W;Edp`e7N8 zaUYh-`ocU9|A_TvRU#{jA?!rd^E3*_6kLC-06}#a>6A z$&n^t%pecZ3sA=>pBScvY?=+@da541gLx=?C$oSx;iJb+W&d%U!)@WB%`X6Rcprp6 z@YVbt+Cw6ezqjH9=?;H0q=^#EZ%C=2e&N@IhtYrXk9d!=-1vVT7jyOlSPLtD8t@i; zj^)U5sK4<-X)p2{(M>X0@Psraew}=svP--s>?`fIDBZi4887U2G-m%5T3Nt(hrsRN z0`z264K*J7SJ%2YAAe0PF32bLE2omn$omw|VVh}lWeV>~My=$Fqca;T7Fe|KB1O)E zbd;>oi*g8aqkb0R2JUjr_WT!wlUm>8*`!aJV_{iTv?|RT#aJ%C?dZ+=EtOl0av$q* zjP_J)=vYLFz$|HXLZES$O>6Vz1c$nBNk>TN>Y*@4%BM=eJC43nmEh>g+A3df@s!&l z!E>SF=)6wCY4N&3`!Z|M8z?geE9!ww#rKKuh?1bMqGGg-`&AJYzrrp~bb*{?QYbE` z@8MJk7M2bXRts|B5ExBBLR`xp5eQMR_@#m!Sj!+OIGeD>eFG>Ud)X0$QM7iGk0M`| z6MF`?A3csc$MeJ1mEw3c_^7OxynDo%aTWXw(z_si{x9k>cOr0so?%xlc+0$LN)x(q z*vx*`1Z4m#Wp`6|mgI44XkA&(oKCtd_BJ=0(H%I#i)KD`&*GP}6YK_oNzOCVM_?MS zfzihB=ayl#%oOehsEoCkJC-SC4RW8xs@V-Z)4)c~d7j8UkLSgoYxj(Qhd*c*B4_~U z^dmG+u@HTSE*38>e$60@*cm@{w-`5L4Oqp(gg`iZ2)yVX!%YVn_IiAKFw{%|OoJx$ zIg|#qEjp35LOETGqB|<~XGj=sa!AZRCR#Qx(1G<-a@0MX(<+wOoARnfWV2B~C=}2z zYSp#G@mMXh7&zklNEzv45-2ap0H9B>)o9=hB9}1cb=sr&Fl+cCE84V zL{c}SPy|X;!_3z-i`nwWYfOd#?g%@68>T-Y_@jUlBJG z6V>dzCiorI$->$BP~}5tO7dmJOW2NxDEUuBwC@eMHF~wnG8qcL*y^41DLK`MCT*qn z5qjmcQX@D)zN+j{-g_Ar5t&>ry;AWo{FyWeGw8cPasofa`vcPH>1O_97H7 z467Abve%T>3s7uPPP+icew4Uf5W|TJyCKNr9QHL7Y~jLP48XNKd+U$jJ^pGFrEm){ z5A%$_1Uyx`kY5j~a_srTU|?bf|GeN~SSK)7(BQicUtc; z-_S&MQ{5xzH%@EKRJJwOSbHo%7f@Zv5BD1agE;J@++6dMDJ9JrzfK$2yj2oV!a zRtx(?uK5}5_i~@56}EOjJYzkZeTqfFpBnmMbG^^i%`YE!eqLRMI%hLnIf;8|ny7}8 zpQdRtfa%8u)<1=x3jS1Ei|q1#Q0F_3O(j%-m69rq*dz2`gD|K%-vdFj6?7_?KfPS_w&k8!hlPrdLKE#R-hcG%{41i zI5U13jL2@Yr}AKRLj2ZLc+Cak{1}$@5{VqVyYc|}k@rXS2vz9LJHhQ-h6W-VNI9ymkzS8Jr%ICQ2U{w3OQOAp<(^`U%O2?mQHLEt@u z{JO5=M6}U>SPEXs^;M6klTy0WuFB8R^~x`bGeI8}fpWh0L0Ol~*yXA;UvkV&EkTL} z=HJ9aB9hTPvA3`(Woci0Ze|R;cXdH$2)1juc#&^Tdvxi(bHmaKI+ES(indoqI4}G zP*#^zMeJv$%M_tpp}_{3iuW@mra6VRBL>lE&bA9t)U~q*e4kg}4)93<)u z+%49I$`YQ?v`kLqQw--z?*nNmF^xw7w-{3WdB842R?7fRA^k7K2+%uga1@x?zjt6_}S|*04bC7Xz&WWMLs$HCa-IZ&%e0Nxxg6W|4Tpfv4P| ztASQ3uN8Tievm#ERvDg?GznL2aDtkqf2sLgI+Sxp6ba9Ugi}nAE1^tS2YOLiYsL&* z9CAGTG;s)P=y8d5)Q`GetI^=zBTwZP}?qRFWnlOP@yKFU^ai^YC4TH4I|$l~R2sEp>VM z9aeMbB>EwH%EOA7%#FA8rd{Q|Yypz=fmy1iv`JtIw<;SBnBme3<^uLb4~t&`)+t$V zQy@IF03`yb?tcjJz-3zu%|wvebUEHsv0CYnVl4OOq-0FXvamk6-=x^WFGbE0LGn>u z>$=vEpA~S?nEO3EO|;s!i7F6M8_Qz0)$EXeO?;?*#Ewm$U1f|(&xzE;6na38sffu= zrGTO_WIK{08*v}QjY53wWG`;M4$tYt70#|mW;sk0H;#U3^ z%s14Jgc^bkE-dg5SwYMOa0%$xK9zs?eb#UaTv6g2vMqz+_keSx~+iugL>Y zbBKfSo3XnovjYo>Z8WOu2I@zqlT{ZB$1%|M<^CY=7uG=bQtnZ)P)FK)ge7bj-7Hst z5HeEZ-l5x>8v|bBzq8l4DkujzBUW#iRlJ6(Sy^=MVQ^)hnx{uuQwZk;!@re;@#r~w z;V9mgxa^9T{CNR&xE?;ol}$bbm|D9srgZ($TGJb)alno2FOmWB%mNe1V)*RhG%+;$ z73`#FU+e>68MgVwAO-v1~%IDwByrFqYvdarqBg!HRhm<$6 zV@j!t)L0aPB|9Bpf;lLSbw!a9#Sg93^dq816*C##c#OX~!>gW7q~x5dwTEqiu(hLE zCrhxEr7@q%`&EDaZ(@uTwXRU&q%6pK2faYDMA?_LtTT*7M8(rgNID$vEm zWR*g<*PM%Rfrt1fhnB~lfh)@SHTq!CINyf9-McMoxK?n1&s5*uP#>PY?# zHO@4k?k)3-`bFP{q^>Vw-oo5>a^w(+t>#L81g%{@jM6L88&ny#5(U8si+}0b zt+uBp;E9C$k$ogQIo-E|x`q1E$&s1NIB$XBE@eNKM3wlH<}p3W^2xcFH3$<*N6~|d zC)6uxaO@1ad*loPim~6lY!xo#0;b`A#~?6FZsFE%_CeOW7o#OxRkmL?ob16kQa#R(vZB7oIL$ z4BsQnOI=xk0N+Qf!xF(}pK;<5u-M6pmJ7yOCbF)AF<@8jn0hw#J7i4xxFQhBR_ul# z%RJ4%)*6e5c~N!+`bDf=e=VPdCgkzj(>%ZnjiDr_RnBIBW_;6%z5 z*m&*`Ex7zv!Yld}ltG9eb1PQmX~o`4bhh8Z)2Eu6J^`AU!JG~(7O|AT#?vZt%e;sY zXnD>cc_wxw!Jeu|un!)f$B_j9P2w40a^e^Jw(jT~$ zxzzD+Oa9D2*fF=L>>^c9K#Bt{x2PbwR+|nao}R!91UvBp9UsD#?-`#l)8Flf8k; z5xZpzGe=OHq=vCSu{R{HL1n}oaiRx`x>&?qYsj1<95>f;CWQX<#Nr<9eDr>pZzTww zU5-${$;_%ysUl-LFz*x%K}!jpvQHi{6qYn)tsO%xo-{wqL5aAuy+ylqnhdAL|H1|y4H%}BW0T3;h2cUA=WT7 z6XMNQ!tzt=IVt7j$hX|}703Jtyc}$tYZJehFl~bugi>c2e*(WS?T8|(wtQtdiuMs% zQQ*WdKy6JKVXnjciOgit@u1%#&McC@D}_5uHL_{ppJPOrFa!_SJc2Xv6K))CN**K> z=P#q06JMmvr*X-+hzW)z)q4FfYXN<(YbnQ(x!C3uua2E&@(lRK?ZjK)cG45etO(~B zHhBRg72{2EDaD;R5Fw|nW+~TqF^bp?u6WiI=eW%@w~A+B8Vz{!rMPI!V!k^JjT7^8 zbDIeYJ|XE6iOCm4ETe4TcdjSVE%;YmOPDtRXImrAS)k9fnqMby$5vOom1s*ZV>U?^ zm~5UE>*i@uY1UdzWah=^x$>;VBFi;ZfsV@(LBL z#pMKG?pAJ2+=S0op9o(|nxUNZy-N8kzw7GHSRwsko6XLZ1e)3MprS)ap=3{HdXbH! zGk1N)MoA$=FK(%1Ve#W&vUsd)w&!p0HYCi^O}q#5%5sVLJMn~}OoE{GA|?daA=?Uj zLI2{58E#-<>HXM#kXXJr*a_^YAbQ>hzhhTACJI@EJC-<62<4NJyV!!cxV)YF489Vg z<~1M~>4p5c6_;X{@L`yUpi%x)+*8j1ppQ7>m?fyCoUj}MM;Y^tMntpNiEt?kgH0^p zu}QdJX>5)qVM`33bBY8BD&zK0f;~mN!?bkAJYY6MWcf`{sjDg(EL_FSD%(QOqdMdd zFa~LEsbkCobdTtFtZqg?kQ;l5nc<1&I%obU0IYL9{wVJRw?;z8djr4#IxYpo&(3ALLq$6$V#`N+nJ*Ysb@ifLVi3l$!a z85!^7_lq6l74n4A#4rOK{yIb|j|QRf@eRRC8M~K)+%bDSpN_D~J~^ zL?20y6h>jSaWO(?JSr5b>m3{KixAEw>wag1!PIWM{lYHBEOU-1f>m#LM)ZNZEPpF7 zi)@}Y0Mt^lV$T9+shZFkzysPv-$a2UW3B6M!7k>2-A%BFoo0SY7|zuzOmQM^)CXKV%UAv-)|J3om1%vYa(fYatW0U)_>`?-P~UYx}|ki<_isuuEqsJu(; zd;p(1t&0$5#oBU{0d)wCdlzW-`N*pT`dqK`D}Y1xk$?zzY!N3oC-5@50e%xm@~F%w zk~OKrY`C~8<~}D$Y#ox!c_`Z8^Nzbgr0+Vy+a|2F597ZQnpu~zP?SD0zzjmkpevo#8Z0j(vy zr}|aFiuhXX(W1!k(5kEwiQfy&WZ75u{pxt+CWp()+n8Fb-3l$C-6T;ird-ocl^tZ< zPG43TQrZzWt%-+whuu>fBCh!jE6@!I|BO**s{FCYCmmo&5?A%91@7d0dYS9i(to0pHKbLCyN_34kWLPX- z$$yl(L6pG_k3Ayl;kJif6@k2Yes&@Y9>smDaEkZDu}(P1FR@u6yvcuP<|6U{0K=^! zg#qz+DlKfCC_(J@6{Tq-bhPht=?M25OUx1z>Ya3n6v9QjpMi?MmYh)-K z1FyxL>=dTI3G;6IoqfjNp`|4MpvQy89Yt@Q_SZdx;cTAPEJ8Y(-Ka{$d^04gj}zan zPE>BD)yC$u&dXmO1~z>y%_%2eMwoGZRoPts6}9m4OP;H$msXr{&aOO$`C)rs-9;#|uu<+O-!yup zaHrGPT#_$fjmMm?oko_0-l<7JY5d-6%P|){gDbP}=FT1JLSm;~w6dO(Y0;=yKsPY1 zklkbc&_~JUa9Uy}v}3p*p;xP@gigONn!BWY&qDQbin;R>C6fBhu15Zu{>$Q>Y>>6w z_>%M@Cs6;HbRBP9%y^|QWjgexCX8n8_g?jlp68jNL@}D3Kg(Y*pW0oQb+IEXy`-Bt zyG?8)KY3mT$r3(a8FO6i&q9aZR5h^|`aMx9+2=fi6pox)=linD+(Ns@(te)3C0sJa z`(XmtotNG>=oEhhw#95xPVs$0CzKq1mfx77p5NzbEsx+&J5NfdfdsqblD|N+C0Y^) zJT|Ejj|n0S?uo^MmKdUPu`n-mlj0@#!>?Xp0N(R>Cp!dAI`5TMfIIAlB{)!JNf+-0 zF(w0I2v}^mLhK3_G@$d167R{Gg_h|t?77guoJ@=zObIcEd_-75ZzOf17r^_1kK&zC zO>Pe43H()SQ^s~Gv92KfQLaWR$)17$%)tEP&?J<$=mP9_K{L!Au`^MNIExkr&%*R{A%>G{G8%yTzf)G*&`w;2#F$5 z(CZEo%ISNp&(ez7k=pbaJzR}Yo)|%hrS3~(l9nQNWZx$5&+~@>RCN65QY1Yx@BvcI z^jlYhU&LnEETxjTFDnJ%>5RLAXR*tey_E2z%Pe|1D&qhoUM>DCAMQ}!79De7QFE#bxb zJvfju*Ey7YjZU*vGVOF33`Hqxu=Y$*##!7K%me!hyzK zpMu%U=yY}?3E2xRZ|d$GyG0}M3fddS{FFoVTj-L^3C5+O_Ph_wGim+BNY>s+5!{V^ zX8laG1LwQ50};zBwKAiV`7eYe(T4zc+U3L?AQF|6b{BvZ?#~ee+|<{F2%s_2237|g z@g1-D3`{%Q5;Ov|RTk~M;63Ocp_RR*-i(vV)>eQi6e+oIW|mMgl$ud+R-zZNuQXkp z;JX3&SJdfjhrcN@wi3`1gad-8u!U7$sVid=D)W$6lT_*(5JtvZ6(r?uzEp8NBDN$) zp6821#z-$aTjSFuXsaIT1F;i83$bflLcvCL)b$|a5-n@6kU8mRv;iraymrms2z_Xe zYS=djaY7+7?ewR%e4;}*U8;xe>hJWMchJ2dZ{T9SA4hZ5gY^aNA%{ImN%iw65CM`x+q^O zTpR)76-82$Ti5=g`_kRbQS1w>FYMhpUg#10Xh9yfANIc39p92W4`xOL5{@9CWMPN| z{gJxI^9A7qW6s(y)K{!d^HJ6#ZXPQ!eT+H-N6f*})|4JAI8XP?nFIAP#M==M!WQu7 z1z$h{e4=L^ZY}?l!(z$|pvNMW$r2zK?um26Uon+wI5DY2KYOcqMpje7FHv>ePu&fY zO)wpytK8|y#|G<8r%lM2LVF94@kO|Rj)`km)6tJpl2ooxe&#LZyezYPzWiQXJv3Xk zD|mG|QY!V#z*I}}9c)PF#YqH$WncaGmJp7oT?_klVzxI;;)SLMH6It&ejZj>O6>LOa|>L zbR+;^yn(m7g|dTCTWm$#SAz{*$Pu&GrQ5d>mX zF=mp~;2%lYL3y|C4CM?RXZxAy#qu_t;v%@q2@4A5kS*Y=idRz#^7Bg1Qae*Nl{09s zA}^y@j4;1DxSP!3bxmYAyTon|!-?x_=EUjX8{!w{pxN1F4h2ux7xTo>6&y(No3bFz z>&QrC4_E5fkA?8u*U3qdyf1d<^ep~UGa~yZ@DLZ6Nf7G6@OiP|aPHm0t6+0-aVZj{ zMob{qff;^Kj4$ZB4kSXswRZlr&AQlw&1@Qo!d9i5$iJ0d%L$Ut$<2hA$o!JtL4Qfn z5tQ;Jk~+WnXoz@ZT?t`G^wut(W+w_Un_@2(?!^2~B~~pe&CWWhVdvb;zo#}%sx1Dg z+!e7N9`8ya<&!A$XNwhS@x7SMbG1V zX{yqO(4~yRa+Y^Ji->A8J4$r(LWz@*Jzn+yk`p#;w2rYeV@LL^bjUY`(=NC zsF)JuIRG1U2QvXQdWnb&fHO`9sbzw>)-vWLL50aT?x^50Y%0G~BF~>$>>_@fih$*d zInjR*B=P*9O7sDd+RK-)UbM=omvT(lV%^P55r&&Oa3R9ku!Ot?Dyuw2;iTeo%GXkF z`LF1WY9fl(>pv0~}Llw~$yUk~0(SNViV~4TkTJ-Z$IDtkt1xt8^1!SZ*tvwxyE~Zz< zb>JKsO<@ZOH<+8&E6KZA$K0H06YN)O-B|iuH;d<7BCpYS9RTNl%6|xlal_JHBY$#h z;@mOTyg8w}@OC`T`e0HP@6);;)B}9d+90N%uHN8E&Pkxhn9RQ|NXUo6{DnSge1t9d zCUy+f2wn+=U~hv*eSZ++!S;3gD6c`@T31FrSZwLZ2?Zg>eS9K_%)481P*#+-0{%#< zh%K*BNJ=DJi1AYaJQ74)B)Y?AyZQ#uxc!!jvqiR6j*8DN3rB zkra7XGM-Z(7$d2LEc4thAwVxWB}wq`>()JzO7wh_?UFkLuD+`@k~*IGM*zt_p7>bs zD8D>X2)Y%m2wVXYN?v$w11HL7IthdjRDsP>(PG>+(@4>Ga<%?b@iT^9=4y_zs5;>X z7bvNTxW@A->kR1U-9SuwiuwI$Uneavgu7ysA$UuwH60Wd(c}hF(GV*uBbs4e?ww%E zR3jTB99jM7r2#4I!`N!iZ0>wQlv52)Lw2+g0KPOkvz6ddW{Tlr;UP{^`WA{MmK3j| zEyV|g_t5)@#{MsuHspn#fvjuP1y1#xak`JqX5JK&YE}YFvo9NR!RNe%>5hb9ifjBD zvOTpd+>h!`8}P5B8!#4n&SIJ~iB5;v8LVeEx4FAHShJ0QC2x`8H87B`PCJc#!lK3< zC+uUd341}dWMA=5qLy$vJ#Nqsa4VdSv0Qmxw)&it{5fV%`8fV3BTK<9;9?pCJ;nc~ zJ0V{P%m`~G9sr{KZ7CK&rN=p1FEHj*&nyo@Rmmx^_8&MNzND+rrEAww+R=RTl2W!H6tnT4&0aET4|pw zi;UMKrQZnrrY_C#_AXbE3&vb-DHYJ|cCCut@T(R^@-Ju$qZD~5p=I@T*-z@Z1P^&z z*1Je+d1+p6;0f9DLacY5thfZ@xzp-Tj|1y0~Y(!gO^DNiOHuSNO~AYq?$gZC2~}a3Iihtsoe#!1d*ZIVVRXyhQ-%y_Wo zwB$VdN$5#&W`)Rqn;3(h@Y*3x#ksk@6E7k3Ii!o;kR7eZMSrN5%)W>n8Em5lv5IY{ zA0__81wwa%Gw~e%7O;^R;?)X1BU`xM70#iqa>x_9(^uHc6e^hp=6<4?Y%`;mqCT#b z{#J1)?|x_vznJpCUjmfUEWE^m`E;r4DFKo3$01g5o5itN2IAOX%rQa(F5K8n_?~w` ze_FH?m=_A;qL^F#;k+>x%&UYyz|MB<2O>Ge4lckQZkhEb!3G}8oB?9^DaOS@7~pLX zED8a>hw5`2xEKAMx#xL$UXHw5JdrDx&*9rU7y#D%5$i9&FCf#LA$SJdFeZaP1z83J z;ZwmuJ-xqw{~zEGR1ZDfGi$b$p5BcA`t(1Z|G)1uX3m;DXYRcD3l=U~ykzOJVfkD9`p<&?>kx|hx zv2pPUiAl*RscGpMnOWI6xw?2mNMTViw4@YP1}{e-D^O?*7KbMgNn{F@MrSZtYz~*l z2Lzx{B$h~Ja)nZ*)>Kw$t7~fO>KhuHnp;}i+B-VCx;OOnZrs$jxqr*nZQFP39N4vc z&)$9e2M-J#JaqWT(PP8MM^2m^J$3rb*xB)O6O-pJT)cF7>dMt?*Kgdsb^FfUd-oqa zeDwIq(`V0LynOZg&D(eHKYaZ3`ODXD-+%o4^?UlyU%eSKX3m^7YqrjVx%1}DpT9uo z!=lBDm*}imwtV>tog1rG>-^9+&^cmctn)-?ip~|CEjnLx#{8Ev|FP!3z4`x`^B-?? z)+8mTre$Pi=jIg@7MH-_hzc~0K%&xFTs|n0DpZx)n)=4(*7nZso{fE*w`|+Kb71!# zoyh-a{QpS&Z;k&E`Clq^GIct2LUl@YQgvFNJk_av{z9ks)oY#Nx9{G)*NOi4>C%;$C zOaIP?f9J!$^Wp#8U;bSe|E`PwV_h8muXh-l$IlVnJIrZW(fctK@y*#v!OQWgs%Mim zjgK!jwB5hbyy4bV$L1>+dv~0l*u3}b*!DxCqq|3jM-H9&r}#SrnfY7C;kS;%j}yH3 zPiLfs@6Rgmug|H07ZV!I(~DL053bd=+`8G?Id!GGZ}Q^ConsT5_nkbu?clM~1ILC= z9ys|=@pq_D#{swc_b|=l`$>M>$1w@y-MAe4YC_3{we+*ES&jE$Kls;y2qE( zV9fh*N$#sj8S=>`IqmK(IseK%xpwkSb=%qNZM~yYU0a4PZrE|?{Kj2_x;e4?+_u3T zXLlX^r}z(tVf=qMd_E(LdV5Zs_56ag?7=lD;rcx(aN&Wx_U!GNj#Jm$Hx5sA_8-2~ zy>0M9@6J8v`*!b~*t&1)xq-odia$CZe(E^<9AS8T91}*oo)D)$xhyHZbw^xr>46j+ zf1s#4b*sK}M91OqmCnrvE_ZL;d#Pvpz=cf%+s<#<-9Ncw??1(#Lq#)w9w)5+af0db z{wx^!;=Cy7{&i8#)q5h$^r}G*FVMd;i4HoMhL6Ejk4FhJqHFpy(Emjc?V3p@K8h;dm@vZc%Z32cB{E_=z3ev z-mC4K2Cj5%-afTq%a+RE7Ho)!V;us&7lrY8@mZff6pv#q=TMn_NI zweF1@ul8)}xw5IR`%3@je~O>Spn9KAQCGg2;F{dI$uhd|kQ;pF1z33ewVZtLjRxHP zOrzTNsJeRd{rZND_nMn~?zFXb-|lGdy4BUuajU1R{pP0be~RzNOY}aBF_t~P%v^l= zKEv|tGj90E8&Tn*_X@)94+_qf*Gh5kbB&_=X^p1iaf7z~QBzIp!`9lC2krGu54su} z?{8@Qr}%QRRPW^^W6sq(`1$9aGVM>i;YA+)AkN?YO;pkURYKYDLB?)-s|1=~SBV;5 z)kqp%*30T%G|FqAw-MfO)n|wrCGWv?;aO5LDZ1)ddM*kl^ zwEH)RYW*%I)PIpvYCfwO)gP-^RUc~Dl^^Ojn)eM{^}A-C>Rl`UpW^vMzTUD^RLm8jSL16;|=18YllzipHEhjKAfx-ycucKJR5Fle0;RE z?cu@h?t6oqHs0C0wg1+xUE6Q$II!#L*5iApwoDA3+xWl7za0MCyMKN7*N6YPj{cnw z|IUYh=fnTGulybOzxS*Ix;^U;)IB-X-12a=wd>yT?oGFk_HDa|Cb z{crIf4uiVAdkAm$=LjR<+X*n~(`k9phchbNyRk~no3U#7%dy&;r)L|RAB{J4-aXT? z@%qW$Z5NMj-E(ee_t5FR2ak^IIx~ED$JrzQTm0Rn8;e2se>hAZWdwXZDNOu$Mqc=C zOoe@OR#(vNoL2H;qNeKcWL?9(i;ZnJCfa*0oaxy*KC*fDslz*mh6e|a9@&5D_<`ML zkN$7*KRNt9#`OO(DoXfpM*e@X_m)v{oo)a2PMUgq*HTYiQg7-_y^XYKlE&Q#aolAX zW8B?c7-nFA!G^)z9Rif#9^8Txxc*mi-%r=`ylcG)^wa&JAC9&1Z67(eaGk&F+GRWd2=l zJ0!z8qvA}R1s;%cnJ;m|BTB6l`OyB8VS$6+BS_^TVmq+iEMsYiw?=ohZnFul4O*ugFB zd`v@^m|oQ(krX#6^*ME#i1ezk*rc+^q`0DlwCIBD%&6SbjK2$r{~rg>x8`$EUrrX2 zwvUui5A|2k&%zAEwY8P)-O$O$R(A_nCGApK-Ziy8qfQs$fO$-OMMPqBXx>P3??RSG!n# z<-HtoVK+~Z-6B(^-s-<$4iS6R7-g+C!`?q8KCB|iZm7uo+kgakE>bT%vYg@d=2Q{x zlaX@b4}CSH-&&g~yX!mY&Xqlkz@k1jExU^^N^6m;lb}OvgVr1c`~Qeq^R3?BW~fSt z(^aL$|6Ts=aPg6wS>A6zs&4AVlIx6++M#C^ze+Qmoab9`SNE5*OtR};DkW+0!{ zbdmOzUZ=R^4Kgtq{X9lemp~NTCRIc=E48=!|8Us<8|zKCdw)$`Or+{+g7xnL?E8_W zj7yJ97X)k=s=&S8@hA3mn1O6A?;{>A7^J#q4zn?ieh%H%&Es1;#8PvcTp8A)))<;} zI^8v+LDOh9s{XW^6b-iUzYB2eM{Z_bd3d}eUND>*``-2TEkSb7K=vOZ2jASl>Nv;w#&mq6{v@s zoBSTE>PEd?G=SThJxV^2JV|x4-Jk}UN9lMy^w9M48OlB}Tiz?@%6gQ1Nw-EI?$QfI zonazjr&;`WSxP&A+$eNk*Inz2)O7kjR?;8xe$FU0@dPeN2n} zyG-ZpLz=GbM9O>I?#Ukxd?s}|-~-!|{}16HaF+(44od;{6dw?rIm^_G%$v;1j75$M zZGq=Tofo=O=EWZ5IjJXUPVPmTRsL;ysy>Iu--F~2x!#*P<@t#=%SPn22J9TBbV3{Nn0gr$VF z8?wXObj6ld?bV1HHT)TPbwk8z`N!ekHT$Oy|J31s-#hZ zBO7&fQ5BlHs4`8xb+rI9bg;o^^(}`auaB40Q9mwZh3=irp*qdvFg>QTxgisoLdsB@ zgnQkgl=dVTR9&_ReY-W*)Ebc-(Gs2$)f83{Q?0L!E!JI)Db&?PuNGkWk3;;W4Zz|3 z(NYHLhq)ZWp6NW==^OcM_sKk7@I;P~G@K>o^ry)rJr0ekGa+2xYKt*nvnE;VESa{N z@Y48FV|7BFp*A+ha5ZMN{2zy8?~fKUL%yHMCGNgaKtDB6$Z;Jn`&JdRT4jYPszYd-8q|e|ZB)@%eBe z3;o@6K55rP5#z*o5&Oz$G0%UfNQmnz64N>hq`c-Fwd`uDL0@6FSW4n-(S^~DxLj+F zJu9LzDaBk9pB!Ejw^~31JQued5}-q>&j<6_=x?VA$UDc27{^9S*q4S&xjy}+d~|o2 zh}>2xVb|v?WR)2@eQ|QQB|jlLCObAUJ|ia6o*G@@ut!!WBu3Q4ua^I>!(0yLo5@1T zjkiss*B{Zl5DL$KP}9Xm1K=cONft8j!jFl$Cf(c zqpK3*qN?Lp%Wa3a|I{JP_x;&i?BTjc^^g~1CEa(18j$2PH&#$e4i>a%DA2ukJ zN(=JTy6jA&B`q}~+K~jujuTUo;u4A-F|k#NF}CW2)$*4^;^hs?>AvsH}CW!o*b9@tDOG5@}A6N|&BvFgr5L(Fv(BakgZKJ<3t&uqIa7 zBNM6Fazjvn#@L!FUP__=0)AnAkW*qNoU^%z6v0duhnCS9GE+fBM zEXgQUYLg3e;R!k6QMSzJIBR-Rq9rXq$(&MUH#@2mR?BUN_*)KFHY{fPzjLE7^o!9_ z$}a4#cdSjYbC;9RNhWQP>Da2eTkB5_KUT$50$HN_N}tdaRvn>i;jAuK1? zZp^GoG^ST4td_qV>{nh~$PU;zQB3@7xSae$Up4jD_G|Rr*E)FnYrA-t%X)aIyiOi1 ztyv^Wtdps2)ta!#im-_A(#U8-aYC%FC?{T9SRJp;t64R`3eQEtsr!M$<%PMNz_-Rq z2%82fNt?Us$UiorHjWaD8oLq^3F{ zT3MZAQ&iR3lx3^uFNYMD=VtPQUmqzWyx&(%{IsQ+^jTdeeS2jO^HlM5wpY#|7n=h6 z&A2Xs(Ap+bgg2|ThDL)?(_jf#)x}20ucle0S4$z)s|G~DIwYRD52U$0bE7ci<-v;3 zw>ld_H#W4BK7|>`E|`Hh=MJ&_(qOL{f1Sg$b_)dI?J}96RjtxC>vgJYW`m+JI!xMN z50}*En8o!4mem5Hp@aSOeT$jyPmPsgpNBILFJ5cKZh&(jU&9P!f8G%NV)`gMAYp_R zYVBt;!g~Zf!>#_mT_sbs>XeEWlUmvwsTDWH>V?-*48m*as|LhC2gjNFW^=tB8!AUX z-qwhEw6+ubMp+->$NVA6;fyi*<-{BGfT&Rh-ZaRf>iYOBHJm|E!1)7Nr&cWKFiOSk zW|^=pO2KbUQ1V;ts?`Ewe?>rw(|wagJ`eQN1l?EH7Vxj~Ud+1%gM=-aW0a%zY4YW$ zX^O9DoQBqnFbRr59!1tKrc18NS)x8QThOcH@_WK~yl#tt(-kA+blFx7i2oJ2nc=i{ zsN8qWwPqKjq9<^D;UMS5&mJ+!` z3KC~fO=b`3smy^e8lyjg&ghS1tQKJZ6`9UEiF95)k5qMetScG_em;8)_lbQrWNXxN z$N?k3oKY=>UXm=3-37CBZ|*e5k3Gc?VBHW0F(#$K^a%xuHlYroj_c5ru`n!o%(QAi z@~_BX*={%kaSSQG?tOpuSn!6V*?>(};P;aeW)Mo4L5KnF6c-SkSbzIWRPJV^w(Vo2sQ);UHRk?c(wxVOR^aiz z0eEg!0N*V_5VVsG(0l11^bi%0kCFlXI1#W;!u;VB%pXqwPyT=fva=YVbVdW!s<~Qo zA5t*zC6Y052(iyOKWGIPp4Ed(Z_0qn$9&-V6%+XVKmoy9hyc4i6cBgd0A(i@(08E$ zb9V?}|A7MBJt)B68w`YdgMegTAdv1~EkN@f0>Tg610tNB0#Pn+gLvOBfE~3BB;gMN zJNevF0@HIgmWP_Oiiu+u4P(d@&K)pBi>~VurG1)IMXxGb)uSv@cgrg^o$_ni26>CN zM&7zA{&Dzsz5c1gKXv%u_s)OC^`QOF^C9x=Qy|*)Es*H@IdFt*T~5XySWYH8FC{TN z=i<4+H*6x}Xrz)cWH$2p!>r;yeY~t!o1*MdXKT8Z#o8`Kjjlz}q_0u5=qpwUxaDAm z&!}4tHrF?S-S4yIl#neq(?a(zrcs<0QW);D4o=XG1OZ_zM#>n9)baYw7D=BmR?(wR zR(EM~be-x_eTTBaSf^|w<>W1(NRbzOe>Y6D})fBcG zZaY|D9d0?qyS)KY{6AUD!2C3uMf_tXn|5+Kn{{P8lj}E}#>e(M#MGVyCATBSC~l6h zDjUKRbhXB`uxfpQxl&tgDb_SZ8QY1fTB z`tiv;_QjDL*n4LQ&^_rQa=SyxZi?57uSQu^l@^=6%;Yc?8?!BihO)?9UA;9!*I-H2 zHHNPSpoixo0@mS{L!#SjOKAZg&t>DjpUNZeoG4%(9V_6RA1dH`_T>qJJ99*#&1rI0 ztz9dwfVotOHOf$6Nes^o&xptlE4F4B>!MN&4G~HDM)PX;KMr>H*KVcTr?fgeui;lG(Er2RTt%sen$!am(s%DvoK#`kV2;^C^Z z#mv%Fl{i1qpvsQ57}BF{=9EZBWKu+KG|WC?;==2#vEdDt)$nhJgZF@24oU8>EMx|K zFqI#=d90YWZMcN7cc6@QqPK!`zP*a=RaeT!mFJ0=h3N`WwnL{%j}JE_+oH{h(e}vr zsO;!iYh_GKL|s(0r6FQ900X=pBLCyy@OWiDE9m{n0>W3LCA2L=WsKd|D_MuTYdNQz zuTnj$%URfxLLnnJOD;-J)vA({!VHP=R@LS&3BBRa}f9upZ|7Zqu3h+GYSImEy^ zBzwF(mmU1>L=owWky7eU0~PcgJvGdI?SC>(H2g{OsHkFM3rYlx%siPeIa94nOfed4 zNfF^uiE)t;a1_HFUlJ1@TNh=HsgGO@|I6V?kmC8$OfKr3v0~DvLuHij`l@MLI~o`} znp(-nYnsXKCAADpPC1XBS|kxB=BgC7OoQHaI@-X1L>ecWG3{;Io<`h9aNZA)D{^;kt4*{z_Fj?Sp&(vr%= zg4iO3+?uD;nzPMe#>^OtF2iBfq!&bK((0nL$@P(|;V*}{b5Gn%_j=(*0p^Y2GUEHa z)ud0_nkk<)bTEFd?qVD*?xMS9w=z)44ID~bjgV)pkjuhLwQ56=$)GKW3RmSPS>$?`SA?4X1;4k_xIfD>LP6YFa+{Pw7zq;A58h{DbAqF}mUwC9H-|z8(3hYzuf1)3+>j-_R zqKEQDQ9pfG&Jgo-$|%#rHp~b%4>Ae5ejZ)bE8)tzR6=p5Q6g-Qkn!5$6x`M{6}zQG z&1z{_6~GJ~Y=_n^rCoe*yg2aT-Wt?{4Q;`XRCeRvEWS?uE_ax=FLj*m95+Svv`kWi zbufoe-s=CaOBmu_B}dq!=kdG41>CME5xX-{!tBVDGCGP@1+YMexWjAbvo75`SRS;l z<(d~#-GzFxpj5E*@m%+`))t z+*$o1!9_Vs_7+dk0|XOX6n9*NVUNmi%n?;6V^~L|4~3DaLy=_4U@Vn9V5hAH5d9sp z3}-=xOHUy6EoYFjUf&1uheBRTpTK?^KOeF+0-*Nm0p_Fv;LnQ|NiMv3x(9of#h&-u7ktR$b#PLWpVLS%28bIuK$V|>}$hB)l-QX+roz#1QTWu zG(b2=0hA*oz&J(#>=SsvJBb6r(^w!mg8?!pm_3{g0a|AiFszF4{HNgY*CSb@zaS}7 zC)UK>JpZT}Tzo+Tu51(ow@-Wf7lK4hd+V= ze@_q)?S;9+z5t-u?+?@mRzpeIJxKca%Shtfk4Vh&{x#v?#6xOuX1xfUdzl3;y-fk` zAA|y*Pp}~H3p7A~g#!4^L4f#eAfSBb4`|=}0p<^Wfc>K{;Qr(T_&<9C(a&B$^2=%f z;?EGE`W*rKS|E|?Pb@(r!a^sMQMc|3`luw0IV{fQ{QJJR${!H4^ScBc zf+l!hTorW-R|=58d-Ilq0p6SD3on9bk9R4`B+2CsipvE9qDE1Vs9oG5>J-S0sR<*xaN3aC#u`v2^RCNsgng1S zagVrB(k5<~UX^xA%cY&NVrhqbr2r|c!z~BniAO->#pgks=i4AD@Ux|4%+EKI347)o zloL~lj7#Hj9Pg1RA!^W~BwP@Kxk(5}c3x5|oT&C;v#dTG0&Le{P*lD8}K zR>5rt?SCAs7oUSOb8ms+!UlD!v{sR?sFqhME9GsfB6*uSSJ9@*S_OYOguyyQLx)7KH;XIJScJH-C#yK_Aa$;=Y^CCjB;^O*=T0#dPY+V7qjs@cdd5 zCD{5HHLc1L#w{~h#l`vrd4V=vm8UM$=BR6R8LAdtimFxXP`9a9%3ls9=wQ3}44g-K zZ6Q7Q!?W1}0eFM3ylRY`?i)|TPp9Y5rTNSIMm0I=OLbFAj8y2g`(x<31 zba}coZKXa%(`>M7TC|CpR?SNJKMt{%o&m|;ug+(pKA6f4{c0kgx@EYK@yGQ-_R-D) zj&oBs+q))3h%HNy(+Xp>+?>d8Nk({#A~h^YlWfS=C+W)!iTWmEoUTO|t83M)6d;G^ zLJ!ZytvbYAdU`p<`<2-&)O(ZpgfB)5X+I4XGk@+G3~qlGRC&<3g+(Sa?;uA0=id877vq`BB5p^s<2vcP(vmYXfdqO%47`Ng35Mw}6dK%Mw!TX>v}i zLo1F-FexJ7$bWcjl0M9q14EJOFhg{c(P(YfuN0v86gupL`QXWifc^3l^O=6nj~C)z z8!98c(_2mcu>DW+=Z!6>ofY-C(*>0jkIZ6Lh$Bxxj?0pFFzJDLs3gl7Fp z`O6{d#u zshD9bQu4L=dWkaEqLAgps>Ru^k!!2})RPTseW;qu^UNzg+*H6af+wxaK^?jk*3)=T@S zpr5fNqn~*=v6pty+DZ2bYh$A|EkdIFnw&0f&~o_oWKw1$#J=Npgee8YoqU)+79$1@LL$K7WPxW$Qq{q<`|(Jiyow0H1*TG zHGQmLd5-`m>XwmsU0NEuE1b#fjN#BaleyH+LLRxZg->hhUNJxm9W2nnabnG6!Id>V z)!y(O1n;%wJ?JM2`w1JfhbiACPmp)VOcGCokCQHG#wecBQAU7ZgcrgYk>HraY69&0 z$<#qB?D-Sur2Y&B;d(if(ATzNfF3$T?nG{;9Yuyq&LWM?r;*Ao-+PPtG0)}<6Fx|u z4E@nIhx@}khdZI23q3ELCA;zA3<770>(87N!`TC62z5-4A&;7I#F6Mw!ib%KAI>J? zhDu2*1sI`&?Kfm5dmqwSyBn$MyZ~nq0v^m6#=Mkr1GCAt5b~25gzVM;%uy-8JMouE z7ubt*7sfo-oi;1-BG1TuNYiRR!jvH(^oAu6HyIO*op7Kq6PY0^1%!VBzrFAiGFbW@ zQq}PbJpN@QXC&a!NS#!a=|f8W6NY0<#bvz#e4*_-ibHewG5*=SjL*{Irs5!gWN&%6Zkg4nskm~m5;0(eJm;sz$>zHF62vD3x0nM3UU~mcq z7UuvEeQu=y`v#=C{9Yt)@GY2s>_F_Zr|*egzVKKWxV%9HT;CA^k5AaZ=W8mg2N8t) z5(@C!ae(|A2GD;G0jyoYfV(>g2>u8J;=KVtz7OUO`+b4_fDZ^ixKgS!*CM$!>k!BE zdr0if?MNgze2)Q~dPD)7pAmqIuP}kjMl$gD5D$Dm!GPc|LICFLU=aFEARvGHKly_% z;QZ(Vct3dq;qC0f11Pt+1I^Zzl3TO}No-t;Siw6;82ACvfZc1P;L!a%aN;odCwc*Kkx*^4?O_+BX>ai*bOi~b_MKBE`ayR6(Icd zGHhH{3ShjC!1x3Kk^}bv<(Vge;qoiMFVI6Kc=%It<$_5bS{SJr;`~t+Fe_4tm?3s_C9-lF> z&QIxtZWAh%@0i>aJR*s~42vA0gMw_*0I!tN&uyUga=K}4-0SpuUO&B(-_I!F53C5l zh7Ka=AU}8?eAGV zEszlO>0%=0huMVCos+SY!(&m5vqKi1Yk!!`w^yq}b*U`4c6l7BMUqNu66G@+1XZl7 zf;M)QppR1`?BnE%`ncIE1hC&jfEd={mV*g8L|=Ix#QVItY=;>{66U+9B*JfFiR6Pr zvGg<7qqtYP%@Ut>qb9gn8;);K#!zZy4o0;kn_VF;E*2i z!1<4Z5;~Zj{sn9<&w@ms*KZ~VeKeB-=l@d(+lC$Feb?>ulihKgi)~i1*R^mZs#b3d zt<+elWy%C*i9DTCC@tg{NNNRnk}hGEv`3gG?GYr)R>o}y39N$(I#`?@1+gyAfFz$+ z7gB@YpGwDmHJVQNWgv~RyC;QqwB61+-xwqCs)>-J%ENSoVx5^<0JE`Nn2lx2a|M~Q zN>RG3O`Iz47C9hxdAD#S04}V7``fSLV`D?@ngoJ{!&={%}2u zvZFJTcCaa#ajrIw=T#mhMHQJfggm2(mJPG942_+is>%{3D@(-=#Wjgt*(Hfrc8TMZ z-J+H7mxBg6L_&uIm-RPOeP5c%M7=$pgWoimNBX8GkGi!jhj^eqjdrdgk?U1#6QlAX zRfNoN11-%M$#&@C`F3r(C_!B$iBs1}W7VBfo4QL9qwW%~gxd~MSO+b1usS^q5?!8L zNb`GfDm&!O(frU4`U}ZlbQMv4Y|6*)ug;)4mpIs-`SGIQ%xDE6H9|*A3O92Sj5a~6 zAw?Xc%a=y!s%2JPr_8GDltya1Br5^%;JLWvpgXt@L^(YKl3bsd%kX=CG8g^Ya3SHH zz7oo&_R`QF>I<>^E3(PXg{dshY`Y*hHC7gy6s4iXN0`_#;n9L9Q<69$EJtcFR?5xB z4!Ic)qniy~l9h1VK?WW4&>_a@!DWZ*1=KW(<42ZAC#0J4UM;{X)#eoHXQm8m@SE7lQ~OfG?&Q@@P&M%sZ$mf)+JpDfDh|% z%fSF0Y)%g@rMUiUI>&$gXc6}L{&K?0oz=LFjkSK?Rg|Om78VknGPCLK$!Xl61cw+O zlc1zV#Om4MF%f)YRJ>ScO_yq|C320mMXrtLl?_`oOnq>wUj1sS5ccw}fyeHJ|2|n8gWzA4~-WDJlbE5eXOGn{a8bb z&zqG^zTXsH4c?tmfj^a0LUD=BXZc6w2(YFM8CjpEVXBi&T!kY>C{0Qhi|zSRk-bhT zPUwNy-$Zv;{>kx|)QLg_wMsmv#z10L zTdDLaJA+!4%_LVy{4oW^Jv}x z@s*5W%BP8g#4Xl-;(lW<>9neweo5NJ@f383{5kDP6r;_6rM5-l$!&H5u{E1SXssd< zTKY)Dx&PoOn(O!(=)gtfE+rQ zzK1!?He|SXD{{5<2e=1+9?2gFd>~^K{k(lDPDE-c;Kc?-k7NvpO7hsU+`3(f6!EIK;UF=;7S0>ci^?~6*80g71Gf3 zG*Z(0E9}89A{k>okJzXEH$*M^e-O45xLLInyj8r6-pyUc9bhbzk5ZQzr-;iuC;YP5 z8MCas5VEAdgt}?D9JCng61b4!8nBS>?mu6(B7k-y%){PA8q4oTihJKgvPX6yX%pw} zNt|_iENaQ~Wh3x@Uj=->6oG&rIRLet2CzHHfV77I=m)TXeFzN%M^Hd|EC{Gi1Ome; ze-Lrn55%7F0V!vFK#ucD0G1b##;kQnam~H(|Naq@G`$OnTR5{W5?p-D0IqCM0Jpb= z!0Tfc@cW7ag1#dF%+EML*op>}Ur~U$BM5MJ1_0qMe<1tA7pV960K;A{VA<~pYzI6* z(!rI`7>^)D1#6MihIL5%{3l4%@(#odj;_;zGmpx^`Dgjy@+%DBwvhz9KfnY3PtXAM zISSyu3IxQ>{($CwAeAu`F-4<66zSRXpZCeQ?=?Icmu?C4* zd=eh}V?+g<{%=l8R3Fo5jxnr zjtGuCfB~l-4Fcz$@B^3Edjt1pJ%RW0?!f;AR}j463P8Vf3E*D700^&~2jo}J0oto) z0rNE{zP1 z>D@uOJO4QRyGH+2bzs8lL3r#Tpn%td#`RU8h1Y{7_-mko*Mp3*3kW%f08f5$iETW4 zlNohkk#4`TK+SZWrxdx*k*hstNNrwI#OvM@#8ICi(wJ`_dBV4YI^o+wo%FjKAQWDc z6nIUtpo93>gFt=mDWG?K85n%t1BT!)ffn}*P?2}tl(G&jh{Px7`C8{$PQ;}dR=n#p zBh6!qp67LgTH!N6Y4RH*_xcZ$NBsLJWBwhqasO+yiNLG$y8{T&feIbi&_N0vwCA1# zVXiL$lh3=r6!gWi5&P4kjl+n8mnm02ZH(z!LP?VmRi_JXOsY6PA~L!R@vR;M+(e&#c833TW>H`-qb9hU(Shn<45FGCqaiiSF?1Po z3{%V+zbk*S^ecChnNZLCUkGqVlzC$k?@!y3hwu}5)*>@i#(XZ)_*cA!HC z-mbMkee`}{I`1D%$l{KM-VH#4b-3lg-?av4po96` zzu>z;&o0IKyfzyb^uc5t`l}HeeoMcV{6~+Ob+p4MbZ*foT>n%Ud}^hUfmNb}kP1OM zu8db0TEeX*7IC|X`MeQQHYA-tLQ3V25bwrq2L^NyLI>T^`#=Qz9;VIp>BR(}SEdt# z-W^Lse?E|a|FJig^m|7X<8V_r-?>gNbFETqeaqzLpkk>FQz%Zx=LvI(xq=FEHoujU zDHx=r35F;R;V>mhI8449fDG%vgbt#eYk>YJ?7f{I263)W&D(u8Oge(z9!^GY>T}?~ z?X(lNH`{23u10X2E5pRDB|4RFp(-pmS7F6wNfQYfl1y@%sDzp#YNREL`e=61AT0qd zJmXiuUk+mEV1y1)&JTeE*C%F^eV!jr4SF3eQ9tNO$9>rbe_OOZo_we>ihZ`kEOgB` zDtzImC{P(N8%tHh5goErid|YjOOVvk<0ajUIO!nWCLN^3tN=iPbzs3dNO!`X^C;}S zogZAXyFNCZ>htVKM$pUGv(OtmvoN1tOT%ofwv!H)+L&kaBL%Km;WFPeg9eqX4aeEl zF{F5O=4gMZOU1rGE@?_$H!g#t(qrN5eU!*I!JPx&nF3$`8r2u}zCo7qH zCMA)5CDA7Iu|>&)tr1$BIXs*cX0p)?VW~`=v5=)THn26uewJ3>&(!G$7tx=JU3mfdv-#A$6)hK@ zDfrXv-HcklZ|xN!J8dPQ$1H`^^Ts^3yCz%cr^u9}BU$$r&_eav7bO z+(BpA2Wgyyp}PVY(18yf4BOy0d-fvJ=|_?N;uA<~{eHOre12U?hsXNtR*#LzjlP>> zuLl1XS%p6wR!%vqDP_9Ii}>E+0$CtGUmL>CHRBjLu>@L9I*FQ7N~UDDlc|}*WLoOo z0Ziy1{2D$Rwjj$1yO8nhy+~K(@9-M{dy%qk*L(7N{GUkg@_!?t-T#ZImf&rsM(ln~ z9qFXJmT^H;&2!^bN_|)rngDt^{6X`wSPZEw4M!*|#S==~@Pv{vJTd?70M_g98Tlo= zFMmSjVgKJ>^b1ng@)R-wIA+N^vg?wV}3HeFaiQBDcCmj*BG0t#X zco&(?5?5-I#*=g{+?Q}I)*s)P9*Au$3&Q@{9)ztQ4Z>DWpza3Xyaw+}I0F&=4Kk7W zHPTx5JW|&69g^RF63H6!zBgqw@X7d*;MXIEQJ)wF(LX2#a63dWkAYi8$C!QGGt@rG zIa06sQfRNq1=kznhUrc925eH+_P3fHKZGLKqjFz>h0W zW5x|`s21x!?2_8;rM>_0ksSAY;YnBPFgGM_{$s_sQ{2H!za$A3qX zrkvKs&bd4iG4Jube!=@~#e&}_!iAu3*bC^NX$zs-Neh&np^L0N*hT*SkVV z{_ubwq;>~s`&>csepgU?;BHKs?nNq6)*#uXYmvmhhmh!{&k+mQ3Fi=wty2T1M&nvG6Q(LMFPI>;X%;H7=ZZ<1qffjcQC&82h4Ba?8A3nK>WQ2Q2giy12z|6`Ql1dRIq7i||ei4y^j}Shb#bCo)(7?fa3D5%r&cY1h;uHSBb-fSpde#g0Kkos8 zUvvZL4K4ux@?}7L4xb7B$Qc5)%4 z|I{36{PYZJ_Vh%^{OMuL!s+YS#WTIQJM)jjzw2|m4uSAopy9co!E?bo{V)(+dKyUJ zxey1u2LzbU0f+oOU~snqntUf98}}{~qYf?+>_=|mGmkCe3QsIxt4_^hT29ZR`_4?G z$DD4UXPibc^G*Y}1*dM@!r4yzg7cjKf$(|=f({tyz&NlDzIOI7kX~F5@P$P_i^#0|3J^aLr}X`EQ%JVvNHKN8w`VK8*)LVxIu3q6Fn zi*1CtOAW-i%UAEhZHHj!fQM@s&i;FV^wfhub@54{ao+$mejArms81K<`0r-L)NRvz z{_aVR;=lyUcyx?zJvBm0bQ-2)oEst+UKk`*U+gEgU+N(aT<#)HUT!DOUTGlBx>S+p zTr2JZ2!?e)K?eeK;6n!moG;d1cmn9%UR>7uzO|qW-ZZ1eeS1Sr-a0Pk>>d?J4-Rp4 z#|K!J)BTJ%=RR8Mg&u0&rEW^al}>VtOB?yRYcqM=wShe2UPYO4FQLqO6jSdEKtYEP z=s?;E*Hin}0u`Ju)?auGgtf=@O#!#|T# z9GJmw2ufr32B&Z)f*qWxphWIeP{Lif?SO@KU_uAQ9@zgMTeoaI_rOf7%VT5lUe639 z_`lMf5d3ay4Cd?l2;#3*Vax-iTEWRerTl!ZOy`m%F?*(qVtrEuDFMm+{NN;BHOkKG z3Q6FNhQ#wHQL+3RC>#Gq@SOk{SO*+*V8eY@itR!1!faHWumWupv|(a| z!>o2(gMwFp=VBY=n3KYmrP~%#uko zB;-y2EUW__)`1HhG`nu~|7+&roYsvwF5Tas?*4FRmgiHA86Iy{IRd{dj>m1yji&6& zh+v&aHVMur>gBF+TCI1q+U##t#s*vDspxQ7A=V_T$A!tRL&mXTvIz_XeJ21G_Bgj3 z_&>sV6qv*WqJzWK4vaySkJBk@@j7J>WE}o`0;5yh8GwThq0m799d5m+Wj%;Y zC7xO{lzx6~ckbnNO@-&4sLpkJy*R`Fvz%n~FKG$HJxMnD@i;5@T$DxZ5@Aw#nT$q% z_(cqqR%gekwK+JIwhFJ*c0tB*D$O`nrJlgtiQ5hYIOi?=0romOkfrDY$Yj!Sq(9RM z=_qu*r@q+f@$v%K*9x-zKFv%E`Ptzh`~h41W6?H_b3~NriYY?jr8n#SHKvGQWmp16 zZp^|-jTLx_u>&tPj^SkbajZ-?fw>a^59>gL4q~|9Z2T2jfWP@SmV5;1%{h*=lpa}I zQ+DRD;$oLqbMkyPrDmaiN=(D=j&V?qMkKJ#nqmc)^)WI}b(Gdm9%&AeM8u*+mUOJZ zT#6T%+aM!2p=kmm3cEA69mtzt-wVGzVA_Vv#qCCh(%|fU!Cs`H>W?)Qm8TvpD7*Y} zMv?c&NqIp(#Ae}kMy8Vwn^Kr(bV>Y6ssyRKEKcnsjx_}eVq#F-m{bfWx){rjZpN~s zhOpeo2@E%40(~byDC~KuAH!Vk8+c!CL8cOZL#}7-K$^>cK&oqZA;q;PAI`42v>~P3 z^Mm-}!0)UDm>uCc#Dn@w`YBa9_q;4s>?%rDdGV5rer$Vm5F;@eMNce5(-Irew1fdP zJ#I3DVY@Sc038?~!Dqu)$a3V5$e812ScA>T)!I*y(uN&K-k--GN~^o@d}58qd(oBt zo6TkDZTe!uUR5FOm^6=bR+uBY#LZT^F*6O`w2UY}N`@nllwJ@-Om7G#rVRv)cbv5s73v( zs>bhHS*6&Wh z?OFJ2cn4mOA0amqK0rE(oA4c-qK1VV-_94k#=kJa0a(yzg)AMy>yWb{tE9yIG zGk!b&8f6c=fpv&pFE~N2lRFWv>d)h9t(P#hjw>Ox1+F2rjjkcp1Fop58Mly%IrlpO zsIS1k*>51rmN$`s^rw-kvU}n2pF`5SzegN>hmnN-%j=^1Js%J6_kBrwJ@7qQU&t4N z9{i8&9?EulH)}V!OR%5NB|n1e)E~!mT2G-mlgQM@R+OF<38KB*L`N> zjsV5O$kk{BDa>4hq?WHmY$NLt%hYE`*xU|8w|I1|a_Q_tqUB3Za)8^5bl~+G5%_P! zgOCp}0KW+Z$e#rQ#+Uwp_q7j@eB%Yw-+2Jj4{pE)Gl-O*uYiIrmqEkU%b@*^TnmFU z7)fvj;?@kp)wPIe<|RZ6a}WibKahZfYx&^R159x4Ulefp=}_SQ0tWcJi~>Qg2LjAn z{vdRt51_p31qa1F01wVVNIt#-)Sp}eCYV9Qzzib!3ujR9r86kKBc(P3NzX!%*a|oU z(X<90`xTf0d;(`6enFVv4}=1azzpIv%plIg4C2ZY0l;IuFYtZV8w9}&0u3{W(3e~Q z8D4y+=QYNrX92IIMvT<}fU9KRotx2nxJ`_<;`* zFX-V0XH%}gEaW0MfSd=%);NRHYn{Nkd(MDM>rMgJbtizwy~lyieMdpSeTPBt{f7Yd zfdhc>zcFp>a?3(tU-hIRW=I#l9 zuzMsB?7kiZ{rc;rn|elK{^yf zKs_ECMN|+41O&DHjQK_) zE%$s2b3ED~*>Xv^dXG>RtZKzcm{Z@6@|pp1E?5-OCgoxz#JTxjv#ibbX`R;@Yp)o%ueo_i}kTSH||FUN>_$O$d!e}s}u*Ns}}|oXy^M^>E+T}4YKLasaf_aby&p3bY6~p{LnI;88x$+F3izv)JCF&f17wYmn&NbQHnqec< zkm7vteuCGHiWq1GXcCd2x9KV@2&VN@H0aKpR>up$5KUx*X7FW-6c93+6s*Mo3gEs)ucOJ zs7Us>UJ_52Du@k{%Z-Xs$%;tQNDs@>NewMANDiqtObqU%B?R{xC4>yq5<;d7<3pwm z{sb|_d9c75Jda}TJp*%zmtZvO7W5ZN!TSnXcwVKz{JO zk<3gExR;t3u96fVs}UcYsuL5FZx9t-O^b+XGYXG-YaAZ+4L^U;A|j`#e}b6dJXr6+ z?<#aLktcCIUV!g;m!P-&GQ6m}3XkjVEp5H8v97L?x}y@`WmH^fcQ!BAS}SH?kcUyXv}CTSrtQ-*(nm}B0w-HqQ> z_}v*T3=`>RV4(0kysJI~PZ};md!y`<#(H(G>iY)UN-N9`<(Jx=&MI`ioSNq;k(lEr z8=D!T6qO#O9+sA*9h{P_ADCQb=%3tdL{ENhL{I8RCTIZ(Q`A54Zw>aCdHwPHj6DWp znZod?^fgw`(y{&~cCI_-Btxu(uIf^9~yWff}^t~IIAEXeP z8=)GMouK8Pm8s{OSwi*BXfX83cuMoi_)POkA2;+#oud8;VuADEggy6x0(gBqgy-a8 zcwe;}x|;>y!Gnv?&?e1X^+081af>c*Zlm$O^jb^dr27scv6ZejBFnv{LQ4bW0*k|y z{fgq$y$jN{Jqn8S-16%TT=ToBu6du3F$4GPY5hMzEU|{)J#!)aFqOO)28wpTi^gTp z{%|)mJ~{{YAKzvw?NC~t->$tW;{k0?a*M^W*haf^5%n%tLTbGv0`B|Y^{oz5@T!be zajQsEcd01Uaw@OYaVYQ9btvo8btoOxbu6CM`4hw%v%e=^uOqg>cm^NzR&c=MmZi|t zu>-2RPC?nzTMPwH6*w}xH8&)8Qg_C6m>rI2w>=g7&{@R4)$_Vf3;ni7bI4uSrWkpr z##Ckdh5}XFhFUe7hE8>x`aX4=x>0qT8lv$hhz-`@i^TH4U>*l_-e-a4Hhd4_6Fw;U z=Q!lQx&c|Q<(Hm` z-BBQC*-@)t(b1)7@#vGHW&5at#lv}}KS7)};eE>n_?gTB-Nnq%(!dDSkC~z9^(M&e zISH8`Bp|g{nI*ngYeht_!N%YZraS2Gtq*whI39C*=YHDxt=|Rvw;`8o-$Y-tdXsw7 z;!T0L*_+zirf<3=O<(m%n!fxYY4Utl`cDuKZoDpV!m}(!cu>g*_whXlg-=-_t9KWq zeLfFK{n8LOpu!UQReO2xSL%Ad0h6ts16KU50}clq``wP(_4^20_XnM}?2kHU-k)^Q zv_DtWxWDR>(Lmc}TK~H%w9i9VXnnJPg!ps9>jZ!{+=o$92T;_(1Q|W6AnEG?i2E)I z(Ie6j_CuL9a8z@--7_fezGPNU}A?8of*ZN{7hEXO<#n2-4%F&PUxW;7Oe+;BYO zgu!_6DgE*KGkW8lXLZLvp40vD^N$RK19W5p)Kvl$w_*>%6DEis+60kfM<8@U6oP(9 zG0>+JSiPpzmbuR8tag~Ca@)+BZnT=S<~5&l_{(I@ZHLjEH^1S0;2wkd2m#&s#QoYN z8~Y&2a34nF5z;#XS1w*u62E;*?8|yYkzxaD+2DoCEh$B&6YC|-ZSrEMyW`wrJn3%7n5u5c?;>e>%+^{pk z8?8hjT8Rj>5((QVBx{EXDcz|=>USxUF8)8!lnYRPA0X=?c4a&ToCyxJA$-^su^$}B zX|N?%&<;p4n2~#EC6v)hXe>1#dhB|HwoHeZa%vH)6&l2Tr5bTrO(C9ZlnI?n5xXGd zNi6q0lDb}oeXK)= zNPs4}i)KQRQ4Ng*g=jG=6Fp`nV#uOEOj+g7M#!R#xJ#Vb?hsElDMDwzO+q;&NbIs( zB%Sk5NE2z0BKP1n zQ3MG>fm=iaZW0~1PN;B=m@r%+mJFAP9fKHgW)vl!j2B4&)1M%Uc;%KuD?!FQo4gYn~&2D8V18j$1T$QUw0CBlnAl&}VQtU(UbpAObwCW+tfScARR zGGb3#Lu{;gh_x#pv83-H<`H{{Y2rR&oOO^G6(1%>)kg`fNpPO_=-8a$bDt%J!!^X- zf``~TZ6(&eJBelJUSb}9VBRd_@SJJE(OHv8C}LNmr~!qY}iPW&=@e`136_2d|B z^yClP^vPiwj!Pq_zSGF*0V8r|F^DqOpolf7VDd7=)MIlSzugsBh^qz%aiOjzPUbws z!C@=0^V&6U6D%-m6?1ULBK63$dG4_(v$EsAOlyTFOQH-2||)VTljuVp4_BZ4!_FZ`zd6PnAGA+7HDH%>w%@k&{AZh*i=V6?i1b-M6a8S-EB4O% zo7iispO;@+&0TqJNv=JyBG(>Uk?V^=)UXB$)}W86#|Bf6J6b9~?EVhK-QPi4%O?U2 z*NxK6d53%)b`5%Z?(cUGJo?!s@I&fHeY^i({=TQ&HHOl zZNFaYwjI0v*k<--hYh*ahP2ue@x>tOc(<>LH5g**u}4$pg@!0Vnqe+Pk##auW7%k^ z0oV6n)6D|`wmUxgx$W!qrVI9Xgq?ir8hh@QbE?R5r@YHg9m}qEJJjFkvVU~*vE6I& zcKd$uhxVfqt@hKmo9&5YBU1MV{;feB6R0t!9!I=R`Jf>RmV~KrdFHW5HMZ}Ox~uxb zjd=P(the<9It#q^^A>#W9dzo6XVisGx5P^wu9?@`oeOV1bgC9_b!@ra;`mgm(ea~n zz2mTSt>e_48b@-c+KJq$_ya@(=Rp(GpBbhe7rai<#b7>E0w$vFF%HL4mJGydFYk+? zu6q}0vGql$gTRwOk7JMN{%6{K!bBc;#a?OgNWIbMmMc;3S|(NNQh%q$r9-CL`JHT~ z%b;wz%cN|X3z02Bit)>LF^DGCpo8hp0rYd+Gj#3aRuOVWIg_Xx~}pM;Tw^Y9se(fDodHF%mY!`z;uzM?tPU{h_X z+3w0jyQ8IXE~g8ly+v~)1FmI-g-N7`#@&c&Hj1@J1~*U=W$4 z!~_x*M0qj(t-)j$eox`Lq#uH*$m8%m=`?)IItMQc#GteAF7tzY)#VL2`Wve=O!>=F zZH^QsI-kmo^AyR7pTh`y5;mMj++nxhmGQcj5qZc&R2exV)_Jb+B7MuZSb zWH3=#45Evv%WNl}m$)VZ4!~rzAbd?d4n4W}B)&)l+DoLFnu{o$HTilx71<^`i!!Va z=cPKG%u4dOm=^DMH90m|B0f6uPHa@7TvSAsQbc$uB`myAEhOxjdPvw8WLzyIoKQl; zh{|FRJ*>e3?_=EXcmxT+c%L#_-2M5347sVHa|Qo*#qt6=b)+bHbYH?5=VKNHg{nWZF^3>#ewu3 zd*P%k*Yok|-dCbi18zm6gh_`a#oh}{OjGnvC{*!_uT}Mlf2{5k_YwJ_<{LLp@r@%Y zi$U~p9)5SQd2ffguf>8tU$gGgRJJU@NZDT$@{N_*X`W`M%^r zTcP-T=d;ndo|nS2={JHiLnH&zV`Ti&Qsli;^OZeQ?^8Td9;v#gd{A{y9#Qp3o>TEm zCW}H0@F~1KUMqdK!c-J5^k=Mx=Os(wVfA*XuRQ@(^*5lTUT$fAt;U+n`v$xzm8N^* z%WaNEl{%dcD|QzND)hbPpC2sllNTlJnVWRaJttSeCA&(=IlEoODeJw8bJno3bLK2U zloo@~Hsd*ga}lr^#^cuE^?(%~m*ZOz>-eCmQ3y(#uR=kKENfP?+RBs$y-o3T#=D|w ztPX`$I|v6=xt*t1_+0WX54_<~8Y$sgl6c3dBuCbvxI*5p_@RPbQ4cbtU{^S!U{^ro z7lW9h4RG6tPZ7AGH<=YW3(yEuqY-G@2qh1WKz^GTWVPR6PHk7=jDM)JKC0DdTWE`g zKwy)-pkJfwDX)6(3vP7*mz--Ot~=Buh}+g=OIp`d+_Ad<@UG?k9vQ3Zp}SU9GcuNy zWKoFK23!l=_&v!EPtuv7wFHeoJtpFZYayp&A7pf0fRrbajPXyD*rU3&xI#J&w*)*k z-{sR`yWiuHv!H9c*GZ=~|8w?j;UYE<6E0gm%)V~^pyH<4gNL_FAG{YgYaPC2);uk4 z+C(H4gE*|k>5!;V z_mmi|i(Fm|;=T-e!_Ww%;jReG-({HnvfCLU@$V%N^Nt@PKAeTnkKz#2r@-X*NrTO+ z&tRonpDDLfpY;~|KF4h~eIERleRKiyzR&|EeQ`&O`Z5J+ePu$1eXYXOzBj_u-tWTH z55I&B-jkDyF-vEFCvgD3^LGXAjL1PNk=Vrqk-gjy^7#M+3|s*Epagh-lV@`OuC~69jzY1)q^a;4yju+{SN#^Ur&X_7f_s z){|On7Lx{?roT*98vn9dLz{9~N1bxppf}~aQD-`Mv(|JBuf}xRR`r>pZE7=h+f`>f zcTi?N?xai)ElL}jq5Eit3TgmS&_RTE1NePo2lt;_!Ex#!*v*^;>$z)SiCNo>$TJ!d zRTe{{y~Kb}+4P7hhYqnq2jPGY!WA8a4?2ioE@cwER*|G}E0DtV@}y>i9BJpdM_zAS zjM{9pQn(J&&_P6^gYfwWz z >Sd@u5t0J*kB9EOA_lO6Z4Dn~bL&7=vp$?Mh9&V$1 z5J&fLi&!z-AP$Vzh&$sILT9>6!kENJJhLcCUz9BDj){E$K<~uyUjUd40FKRyzGW>a z<6OvLr^;QdLlV10#L>yzAR_2FuD})Sp1DMBgDBQ>5q-#c^dV=_hnyjraEj=|31SSw z#1f7Z2RKIDL6G<^3b~En**7s;h-3D@hj~-|Cf;A(!zYmHEX0V)Mrc;ciJ{9HV&KP3 z^usq2y@bs~H6mBItm46ZKhHXT-awFa94ognvXA(ILBs zb;#af?S&yySc5p`8)>|+SH`@li+R&n2A|5SG81z>HezPBf|xpT5o7NS#3*<(F^u7x zr>6cjYmmEr#-McPw0_O5Y5f-dDgDR0C-q+M{;BtQ&zRouo)NvtJwtkA&v!kt_nRIO z7}O>E7vtX=B(MeS4GP3+7t!n|oIjd??vnb>Kwp%+^*Z)Lr9#?p;v%AC$SX%@ci zr)m6-ag%iZG2{F_qekU>e;CyXjL;tL8>T(m|BcprV8HO(fiJY52R_l}4)z(6!ygUF zkq=aI6cJnqatCXW#2S<_ck5u@G(|6Fi%!Z>kqMpD(m6-Nfa==|1$?`2C@eCSMPKFd0AGV?2BGtuZ&V^xR*3?iEi(^%vM4;%XI^=@*SzV-2eZdVd(7SmzBcPW_QGsb=$YBH@Do#V zvdfg5>ih#l2JdHOF=J`sTo~cHw88V!<+l%#Wt{S&u#S7_E*o|?Uh~b>mZ#s@jqkG~ zedkB}ust7aDvda|g!#l$eM8a5sRD;i}JGR_l8`ocZ3l zdhdGe6eRG%A?naG`$WNR+sxygHiaiUY^qK_vTixsX4QS}fmQGM7VDw&P1e6IG+2|1 z_0~kB_79MIcu%c_dGmLdj0O52XIz(FS73rJ2_u2>OkV@lmVOG*U;dtM&i&fkp6_oD zcm8fS`u@i*VMp4XVvj#`Og-6ZpL?d+zVv*PUERe7+eadGwr|C1Y`=K-K6y|@3d8qctkRqnCpD%?^ombvDNmAI5$DRyqSR_N4qBj2g_W}frN&0Oc1n>o(p zMz#yNz8FLSYtX{;(+t;yBOVX0V=x|g1_r~$pclJDUdPHXK8aCde-uew+ZtxU+Yn^W zUlZVVu*%O@sN5&`RH;|g`6ACm(E^Xm%Xw}^*K*xzZf3ha63=pdC!XQ@T_W9WN<7_- z+)8sJHy48_VhuWYep=v~aNY-VK1bk3@Co>WZ$^9(oe8-705Cr&KeJMC(wXa0?J&nodWj|aC?JYGvCdwi8l@|e7x zsDn6Z9$@>$`UaNP^j^n)-ICI}y6PQuH?i_n!U#qcmmg{>)GcTG)<$>z#P z>s_T`P6rA@JdfoC(obavgj~p^M~kKVC0|YP$+?;2T`rO6-7FdJ{kK${cfWL;_fPyJ zw_|;X#9|O7tl@VDn++Zh_uViVAOM3A2jPAEQFxYo4j!f6hL#kirFBW#tE=LTHkZa& z?JSIRIFK9eek?20_f&dt(1p~%NU@}V#A^xktXpw@CAVYz8l2=7zLVFBk8L&C-4g5$5n1f}1K3M{%E5l|-`7SMSo)c@n%(16i9 zp#gJJp#ek^xxElX<+lsMxp2U_@ZJHVA={xZZW}yL-39F#r=U6OCe&uhvsPqit}ITa z^5i9%Z_i4w-Io^Yax^*G>tsTt|M}R6P_d}6m}?QCskcHy3M7Mr?@I>-ciasO{(y|! z2@IZ<4h$xei-AAxhxaR3$M0Pcn7xO?dEtE`4|HejfCt$ppgtGh2AF$~r8HZ8c|oSZ zhU_%cZRyE20?CO^N8{r?PQ=9eor{bK77L4tz7`Uhd@C>_@3wzQSlyWt;Y{{@YC$Kig#RVXizWiHHDO`Ao)Ui9CprGQXHurPdt#ZSYhsJEOTueu z*Mx7#FDch}BIy=SZZ8DU<-v0T>+o6+<6$e|c>)VO%3_B4qD@d)dKik!MIo<3nlTeQ zr&7wa*CdqCHb)nk?+h=n-5;FiEEtgMdD1W2|GZaLn5cVZ+*Q|%jGIpBB@&M5O}8D= zUrIWp4c>N0o4oCqN+cG8P}k#kCE~XR-b6CN;}j-n&SS*v#|_1m`ysdbJY?43uS4Ef zT#`_&xiY$n$`f8;wk^2KW{-cV(;=S{4&iaAja!(Ev$jDfFz`nHR!Jksx#R_ueI9gUE?6&e&6k|OSP|%V^#1eyUOTuHkGL&mK8;pEGinV zm{&Z%VqX6Bih0@O74uSZbuoy=YW$vJhgYEh?fCq?4)b^E@8{2TY>?W#9TFZKhnR;~ zA+k+|DYT8k7SN`<%J-qs2G0i;d~U7wJDplw1?*dV4%##a30gHp37a>io-%DJJZs$4 zaNfA-`32*K!SlxTzb+Wpk&BB#>^R_?FFLe_zXT7cZkG1+6)n$hasd} z1OlH(0{y8Hi}w?)W$sT5S37r`Z*b_g-D2DAvdyZ?i{GLvP{6b^>Y#CF$`M*;p&+%h zQHa|4@;LSJ;Bmu_2_b4b5k^id1aW1Ae}Xap;~Z4s+aU@t|E4wqMB}f820i5h`is5b z^ZG1!zP$zR@8lU>-l?-X_ULoi^_Z@-erLVb;+@k5vv;1Gjo$@qrM-*TPJNfWOTVXJ zw_Z=(UfrH&0=jPp1oYng+^hSV>|c!c-gpdf4r;P6f1`s)M-LN&4j>SJEySyb8{B*M zgA4w8tK;VzVE;uHZ2Boomi;X;sj{+RJny>Ux+opFa{TH|iZHGcZ7RQnmWnlh2V zr81GbR(Z0TTWRvqdc|Mw*DFr^*r53H=YllHV-E)IPRT(Jlz?xj2u2U%{tUowm;)@w zwt~sT0igXl2?kSQpf`OBv}f*u)~pg}&S@~H&*?F$&KogP=FM4@i5)r!S9B1*Y;q)& zU6v$p+$A~7?vP5(JEV2FGgdTbbBlH6n4EVYVMsec7coTabZjdPE zYb1^NDk)&ON@|ft)`ci2K?j5mA`Fj#CmsXq=KzKu0mtURohTb|+@0tk4xob&!oG_$ z*mEI*Jr`HeLx`h?kU|e3iylG&Jp=_rh!$KR25^p;qKmMF)5Hy31RY&OIJ$@=ByB;` z@jQ$~59E!`G^$mdf55V3J7W^Bz@Ne6KWA4K7_QPc&h-00=an9j5BIr9V;~3X* z3~_WF(&##5;V4mpLqrV@5M9`ZbF&vc$R6|{{KNxx5$^>dV)*>#Jm%d?=pQ67ci+X# ztAxEEYRW(~^q7dIIV;g{Vk7E4%ZOUYa-tf$lBlMwCaSq>2&I&ZQ0}iKl;(Bwl#X?C zlo#AHl#kp~D&M&$R3^B`DRbPT6vF-E|JX31LUFvucgpGivSYr`7&m z|4XfR{eo*`AT@w+P7v=HPH)^HK?)pg_!K5JIM+^v2a z@3j>%|7kH1Jrh=JUEGV;L3TesMT|tNnEpYW z#uJ{O8XtH@H3oS`G{$+pYs~U|(;yoMHOQu~>SW7+I^kUiavAUGMKSk_Blj@-t6=Wd z630FOdEB9(!9=LECB(pneO}*f`K%s&)wFIn*OX5Dx=EdM?g{O@4L`NZc*eDBH;!qy zZXDI_+Vn%KXVZ|@z@|a1(M9>lP#aM2;V0yvh|ZD`D-D_6|CVh<|`@8d5W0* z)i2_`K6)`4dNE@)CSqi;WY)-%W18l?V$#rO^-pTZ+Hr#z?os`e4L|g=H;(8PZyMIC z-aMq&wB@_*<1K@_Z+ZK5zwmz2`@#EBZB;=dJIkBY(XynA-l*fb4p%Px$}VC%d2OlRXPTu4C4` zh3AF><~&VYn+CWpP4V2Y!Z*L!C^1ghXt9o38?g^r*{t|x;l?#!PGA4UG;HH1li1B4 zO;UM#jdQoYH!9uMV^q8SE$!isH?-$F|DpBmdTKPZtD82t`!S8|?Jy+!+W(LJ`1wDO z8(4!R9uF0q3te0jMz|&{#b6eDb0!?*7=AdavwU-)vh~|puJ~l@#MNuMquyj=7tPe$EW96Sfy&((wlTaFKyQR~4quE_zFQoz0fNbF}ArY46VS zw=JFbiFMevPOI3R9hND(+bwbg+RRJ$KQOO7&}!CpsM+k*;Rdt*BX#EEM{3OH1n-*@ z!D^&x5y&m9;T~ozEj&k!(8X9CgBgdjFy?v%20bLaMk zB3xocgo_t}h~xN*n0a+^ZJMD6u{{El&L?2VLj*qi+<^CfvhdPZjkVjG%Gu#*vG$>x z!=@G&kG~q7>AUJ2L-*D=#2l!$Pd;2}mn~RfTOw3uQzu+v({Zxcrsq_l&G%CUHdCka zZOExSTXHIQ5r_oVpn`eR0FTG--2e^;VGMU?4EUUZ5A;j$GC&Ht11K!*bUn^iU$eE1 zUUr*mJzTd{yZP`}x&-bkcaAtz>J)#p*fCwG(4pW&zJ2wnJo|@da_nE9&9?t~F3W!M zY^FUqo8j;;XYliXAd;9f)iCGLuommxFpCai#Pbk*q92DhxKrgx@NIY$ti;?Bq_eES z-Rdvy5qA8sg7SR zra1k)km5wnCp-VkIsE(|$Q>MC1M{Xa)?&94rrdVJHy;7`5O@gw4mkrIVYi?)Oo6!} zREy(&FpaAsz;aV5-C7Q_yinD^9~nE@rpZ{=$U>t-lOnBoO`WE zjC+Szw0kcyA{ON_D;niNM39RMLGEGJ)WOVa{(sqf%zG<*3fv0+gztn$Q751|`Wn>5 z$TC$%tFf0x>aQ*gGv&z*vDumxybW6h9Wh&>DP9n2 zaK}zX(jA84M3tp^@j5HBVvW|PMO*SEN80a92zM2T4fQ@86%r^E9vpEhG$`R*aA4NO zz<@F_|A3aubpKaZ=>CIO=mArg=mA6w5nTwPjCq%a+1ri>raie~Ab=D8j#vT@~_RNy9h)^dL0Uj@IMwD7Irc)H14c_ zNXA9KkPGckxbd98IvluR(t8!W3TDCFhi^kj z+6dLD2cb0W0u*H2hMaUo){Hbw&g2y8+V~{1%`u5K+auzg_JqcI91M!_6ZDS`IpG%- zbJjaD?Sf}Sk*Iq_!zH(fXP4c=`;m!D?%_lX5nTwPj=9?uabAPDa|t{RV}`akCa6ta z4`mtqpfKwkOHMIgke^~sF+|ikhKMW#(dNSSxCVP5IN+5(BXmUK8xfN5 zjRYB7P?)m^a`H|?M*a;*DY(a!kgv)fou{`dJjZxLNVX+kK&JgpzYJFa?{x1&9%+Hc zTvH=YI;A9?aY)HKZD4YMb;485gljA{Xrv7l!Cz_P620?*}HlCSvxD#b?iH z=pS-8A*XOBq!kN8QVG5lq4X|ebg9bH@DlA6!Ns(7{)Oh7eG6>2dFDIwyXAT9cg_ts z?3fcFWS5h8(k46ij8%5^Im_(F=Pj~6owv*!J8zjm&MgME&Er~F07g@g)0h^Z2V$Z9DFtyW|Stk&e9S5a4cSDLQ(sIb}MQtq_fvCLz)eJOpP zO=;L6%aQ~^^OEf2rX^J;Oo}^BniTh)G$|T8X;L_U%A|lS2r*jp&G|W58~av1l&9Mz`6SfI6S=wcF)AY=D9prJy&P4_(y+<**_-i#{XDz z(*AK?MSbDLrT-#`TlYl_kM@hqOg8B%IC&4l+UlZR6c#@qI{fMqwY1BaQvuXbH!?(bi?MV{ewpjt+6kj!Z8| zEgpm799#qFAfnL&(Oa%$6Q{)rtLX_eT zp)6bjap-{p&;huh129Jipob1XW0DgnGh0A$ZV%j>KY|Y86gr5D*l%$adm+Se--k5r z`;bE~po|_u13iR3h!GR?1Gd-`;f{_V5c?wH&=X`KW#}ZD79qx~hc?0dVb4hJc`_5(AQmDU z#Y$w8mJr#jr9`$6sbHIztwSEL&&hVN&&a-IpOWoopOF2*J}x`OJ}OJtf7~PNBeG=K zuq@$R2yy~z5W<{!7X8B&%z3v&G5g{3?z@<~?`h(5Xd}dyk;uC<6ZrsEA|J7Y$R#YD zm&;(AlgnqDl`CVPm8)f+k!#_Ymh0sBCHID7LhcL4nA`}*54kChVR^zaBu|!omnWRx zFVfb5!v&=dj{1=QqV)oP$b)^Q#hBF`!6R z_A8QA3qej{&OD8|{}LjBSySc$?hm?-W8Xn1L_r^5z(CY2nda1-S!PsymQ1OJuuW2; z*(WH;96u>p%f=~%oMV(q&QVIk@*kA8 zYkE}(*FuofSi^bD+}APt-@!E@k6ug}v%i|uzfNjiONU`b%ar+-mILcgP0yucnt|-2 z8WGDzG!i(6HPV+4spqZure3yUP`!5LSM>)g2h^Xf`mEl&>Z8W@l^--FR`;mSbG=n3 z>)xml?pNrh7J{6?d?$)X;4zWIHKBs*QXSW&wgk-U%EOeN2E$K1Bj!;(8`fbxH@5G( zejI~3p_~Icu`Bv@QdWM}&R+FNyLfe zd#%xj9rulfc2pTn@2W5&yUL9Sf7v3COE?F=bFV7q?%(?`j1R&u3n3V`J_}#$uEKl! zJMhXumGP;)0qbKs^JVQe_NyLPyRU7w^xM#A5wfY?Jes%GJZWoyP zQMtF!qGeyc#ozn$EWYf|u^2y)Z9xuXS^mrZOiS`VkQ+F@JZ4R8ti^Z-%wP|~Fz&_Z zb3OvET+c$6`%P%~kY{Ld*J5pOqb;j(v0Po{?8sg2=)S4c!H=)lK4e>=ZS>B3+oauj zHrWC>)}{NitQ!tyT6Z1Fu>N>B&3g23s`cFAR2y0j{6 zW;=Yv-S|Cjd*O-aNoe=J1}#4Kpx#@Z<-Qk{qr$^{RjHdDcaf{>rUGYgzC5SE?KzI& z{8eC>&`|5F&c$=;&@U-F1^Kjaf?dGvH z)0Ms>-6eE)s&kA$vQx@|M8~|t@s3qT;~XCzi*5k;1I0 zhS}Q?>#*7gKb&}=*JA@b#a~8vNIwP*0b+1JP#VeuRG5qXb=V91j8^9OTCL0UcG#5e z<+e4|!*@rLd(iF#x5#~Qu1N=DT(XZuyObY`bZI#r;qpp2++`5?bv(?42!**4!G$1q zFl%aH9Yz~q-ex^~cUcSXJUQVpoe!D=k3db32vp#nu@ZDr1wmSDIRUhlnf?~*(){c; zCi}YZC3<`9i1P~A9pf3kFUlkSP=tHt(J=Q?p-}fG;b8Z_PXxOSoCtEC6b^DH$B|<} z9{&rGS&P>N?tgRdj2#zzb>)D6e3n2P?g6L|J_MDa7oa3e0t&(um~uij*)l_@D^i2Z z)+PnmY>fAJ;*0h3*dFCe-yPu-x-ZN-_E3m-x?qr3kx+nF{Rw}sCnxEiUry4!Ch*_n zI3lzVL=H2r4%T7L1-~3|7rZ+Qbo;SDOE3@I58DrA5$B)~-{_kgDaV)@p}sUVTz`2| zs0mkmh&4}4kRxwop!@dl0N>pq{=xf#=+TD){89z!z6Hm9eQQtn_;#K2_Wg{E@F;*88p61tB2oZ~qEACEzR4ylMwT%xMs-P2lrAU! zu=twjaLWx5VfI_XLS457hxqUZ1PAV;2Spz84NN}j9guh2GvNLSkATM~-TnKJu@fHt zMA##M97lxy3sGK$*Cxc89p1VyK$kB*`wzyw2a#M*60;lf;!Z*qzC|`I;Vwf`yb5c4 zoX)c7Sla4{Xp8lsQFfbyB3=ISkMP>%8y+Cw9TtAjGc@t2duWc3YeCGk@rRwFvIHF>D~{Vo zwh7xseh{{g_<_tFw~rt~$g%%IG&pcQE`>e^fG%%*BjfMf6@}S1fgQ3^{({srK}bj! zg;?}KQ5gzMVHp}rgVXhw2c(&B`KDTJ@J?~q;*spW%{9rF-zhOfz#$>_pj|@75u5lj zL96%&LRRrTLRN7j$m}ueSR%L(L}w|k1x9#{`M;fx{STPGOEG)p;ICAqW^94@>_ZTp za~>k_S7bx;?lA`DsO>B{Gj>9gG~BZ%KRJzBsrJ?(&b zddXq4wALf0Y447hr4A!AN6b>lZw~(#LS=^U)_~V#%)U4Wl`-fa&;g`k{))?650M1| z5K?p+0*kK!z2q+VmZ&h}5B@E6FE-?KDK=Z}SY*4-uF!d-O`#XBWkKLJ^Ma^dCi$s* zjPi>GX!*_iY58yW)AEM)({g9_({jjyU>t(CnE%`Su>Sz_cQNMQ%zwwn=g%dpAh3KV z_*Dvlca<1;R7-+ewE}~4wFZl0wLY6&wF#$nwe>2?D#x|vRUR8mD*ZPZRYvd`R;Fw- zs4Uu{SJ|{vuky_<{qmumdSx@a^hyc;LX2AA9C%<4NC>V&>;cHl!2FLHJRCiUUll8O z*7AaD{ULB_JO>UT>8!7+`7$4 zJlf3#o3xr6H)}S(*`n1nyhW>FW{YM$;aiAzxE3GcI;f7u{ErSIEf4cIj_;2d)D<(J z-Gg;t-L@Mn+J(Wa;}V#3+y>*v@<8iU2g6Q%CWB5B7QHU(B|2TsY}#Gk%QU+}maBIq ztW@jHUrp(5SVQT4%|+?@zDBk47Z>F*Sxf04>;6k8u7ls}umsmYCVHSa96t!hcgOK< z(1RFvEd}Z`UeJGj5OiOh0qvJpLF<(?XuehijW-&g{?-80-kLF}zO!RedFRHg+~d!x z)Dyi#;eF;(`S(?9@*ld``EPfNHw;S851=pZ&3 z4WhrYkZKT_dW3ugk?H>bp)eiS06K`MDs)ijfgI4o7^8>LdFHtpqS<9zjl=1aeLU z`yp;(29-r8g4zGFA!b2qAXnWGI*@B&n2+O-R0eWA2PtBhzfs9BccX!E_C_1y%#Ejv zQ#U>^PTu&&IDX?NhrA$AiYM92QTA0S9I+;eLdYFc# z2ABq=#+U}AXPNt@3CkC0!una7Ecvtm7z#&MZArZJgRrcs$3<{vV}%p)?@%)>HG%->}mvwV|z z%hE5~&+{!s8`8deBl8d8X6{-%(`GN_Qr@>QXLrC*_f^@~CS>t}^W zte+HKvwl$c!un3}$C5XS(`+vl2>Wvdvg~iSg$;b{j_26<|<98~Qa(82U8rn0htcn0qw+Sl?=dE_tmHv-G7#GTU?YZ1!jB zMI2AmtCw}Fw{mu>KVAMvbpzQubs{)AbmBQ5 z>7*@h)6QG@P`hkZt9JeB7VVBTP1^6))awj#)#^;Hy{|*KtC1=lvc7Tw$XWCs*D-g? z;#{cUoM`TcDP2Jr(LV$I2A84NPzv7ARN!x#9zz$+jJbnmzogyJjr{@DZ&?d9czLry z)XGM5LiPHYTy^?|YisnY*WK5Dz+I*PVts}Fmknk5<2)t$WMeT>vIanD~FCPROvo6P)`WOszNJI zu6!$h?p(`Io@|TQOIZiX|ohN8nG2PSaRmsJFLpF zb6cBb>$^U~CTL@t_5Y*mti!8F@~?k#?~Nx68a%iKcMB3+0)ar37zuHAcXxMp;z0rg zcXtMMaCdj7-^1?yb|JIxAD`!Dc%HeZyQ;cxpHtte&R4DUFYj03S3jW4Z|k5^ze7Wc z{VokH@_RJ2(C@?0LVsaMfxj^Le<3WV*EFW*wdJw6s^FzhXWa7dh_fMmaX5S&_C(Ib z_NbNE5@n1{QPz^?NEi8r2yf-uu)q%MLnFFWg~WBM2u|%)7M#a6VKT zN1}RQckD=Pjh~5)37Tk1FhE^`g?N3uy z$RCguQ873pylGfk_|D;};U`C?gx?`=Mka^<7@iyOiM7NySxL#Q? zseLn|a|fhFmkmjYYEVmx+CCyN>iEcns9U2FqF(Xy`-p@nVK`B1i=d~(@krn6D$8qu zbIzgMi7-lp-SM5VC3z^C`R3HR)J0gIwi*>_rlQhR8=0aMXT|(vuRn5=0yuCiM>Hyl5ZnoVuj&Etu2C~0>`5~KDvu> zB>*@{?|U$owFgNZurX~A>N6%`edc^rWNM={(^yoLX(gSX;i!<4?$ItY&975>YDm|V zl<1yG$%%auk~0UyB^3>cNvu(eO58FcBJuF3h{S87!V_Qc^XtfnL}55lYl~nk&+#aO zr}Ui{f`Oyaz@9{U?^H!JX7XlK!sn14+q<$F@k|GQzYHbnB zWSLEhanl<(9mY0@qxVjs@5_`$O>Q4lm8nx+AlOaZ(wkC<L z3kkls0GH|ik43Q!=)YT-0o3P+QBl|v#l<6$S26?HrAv@eri;`vQ&CcxwM;^(lR|8X zXS=8p|BeyGVJcxoF+GBdQd9#AbNc%imJjkRXddcauy>eO!Jleg`A^im^FIyq&J%tz zv@L=)@X(!Ok>2+pbC~Vv^j*2kADBZFm8u}OVi+Dz@5E5?$>e z7g6o57*^%`M{s3Gr@+dnuKpECJ$)*&RlUkf`+Jl(4sV= z>EK2exqt?5CBKHCKYSV@I(gM6sJPc>_HeB$?(JMx-^ZzLS3k$Pv;7=vAN6yr`9OsJ zj_d!H~D>Q~Z(=h6eFaF3L@4SYMr#?c6878A0034&U55U|Ar{#&d> zzFQony|;MCdT#MoaBB%uc4>)g@6?js$)Tl4#jd5Un{CUE9yTp!dfIG$*wbdyhn_YY zgqp#!2w>2TR8?Am;;s5gJ&~`iDwQFOdsmSwLxwN`oQHNYakBKfWu+# zMR{ZuZ25*|o1+%6K4uTAWA0Mxj`@o$j)zOu9#51pJDwwJa-vG!_{0_kqZ20-jE>(| zFh2H9!RY7@MWZ8vQd^G2a2-w-$00L_pE+1QJ#aGXal&`927o@?fjhTao$QQtrw76O z%mkR7oek4-OJKq^LPi&iu;zjV4F9x;;YAM^TnvQ%rD*8=l_sV8SBXgH@&@thD~H9a zF5eQb{_CZ9^`$T3)fWXxTlW0y17?9W)Gyjd|4R=Xxr;Rb%pshP0v6063@^$+_iA_O zTpxzjHzq;*rUq8tVhzMCZD`###PU1lSayedVBU4d(z^jzdM_GF?q^`}gK{i-(1Jw| zk7MD(dsuk?9TwjE{x`Pp8m#B|E9LoTQa_sdeyqp&KYfEgft7b=PNPixjr;rm}2$|Ys`Gx)!6p*Ibp?U4H5_htazLXi|U9U0~LCG4quU%shIs?qUFQ6g6fLW0-eLWCk&f8N_UU?|jx> zXfp3u#kvba^x|5S?pznsRq#U>A&R78f8D?Jmr#G!ZD7h9 zFv|foWvSVbdfjgP_XXVE}#RP%?_%e>}baB<2s3nLkW6CyvAi!jv%P1M&37=_C)r^iom{ zLA?P#)Hmb1`VM?kKY*|5C-7y)C45xBk9X=X@ml>8Ud|BkVy1xSv;H?iA0ERH`u`DR zJiX?W+5B$i59-Vx)Xj)J@nVfc2)%JENyU#@Iru)S7~f`B;_K{se3`uwpEX+XNn;;A zY8=N0jX&{L;~rjVyufpfk9ayqz>~QG9?ui-=zk#eXZ|sq`NJ4`|Ea78naTWNHuDD! zZF*tm4|8mZJM*DHe4iVIuk(`dd0rMi%`d>m`Q`X9zXtE;H{;!ct$4d&4_+@gidPFR z;KhPFc(UL*9xeQc2a5#UUo7Arxw}}9y3+<>2)*_w&iPNG_n*Z!nKzv|6!V9L%pVpR z5=(qzCiQu-A3iP)$NMGmc)KJWZ38R|KbCMzXfl6T&M~oE4_}s> zh!yd__>08qQmP$pZhNu%*7jnWq4+w zkB5fqaCeO}ZW;OFx=|>u7{}rt#;H<&8fS~n8yAVs8dr+X7}rZq8MjDI8t;}qZhTz& zn8{_C!={g94w$}^-Dmn;ZjY%TzuQz$*!4dUCUdS`gMN2;U)DjeE!Oaj^ClDU#&jm0 zm@dLSv(>n1W`@ga?eV9%C(fD&;j~4h=%jgq__%qx?_R%>wD+8QTpTyfOKPjtv8RD8fDMtYx3vdmuVEZN=G1@gPB%N2H5 z*DG$fZc*B1y;pgw^{I9();HU2vU%~x2AePKn{9;lO*TS@#{YpZlfHi`eYYNu#Y7cf zE%?@ZtD(4OGa6THnLpSs!3hU_9C5V30Y^vdb@USLb_|m2bc~SM?hr4x%^^);t3!@r zi+!>3X8Y=P8|^pzvB7>vhbH@@9UJYhcC5F5(y7k=W9J$Nq4WCx4}u0g_X=XjV=?c6 z57xc$*sd?G+YiB6r^z_xvJeMdS7Wc68Fsqaq1DYzyw%N5y2UkAc9Uz2{05gK#b%dG z<9XP z6CQ_U7reGp!Cl9$_|v5yj&r}*10Hj++jAwhdl_M?w>36UOq!iQBy%MQ(3<6uApM3f+b7q+3B7 zgoX5*x;zffIe)ilkLM0bxbCEYQy!`~=sgCzeP&~u?{c*G8DN9oIyCt?itGJ6WNLl= z<<|RzDpvW#w5#+^YG3Y^*|F5CuuF+oRo5cV=I(`_yL#q(p6r$9dAoP6*Q;JRUf+q( zlk{kdu!MfsfX88>h);I%xZxzhpKdZZ>fHl-{6?ZRU@15myOf}Q@MxPAd7pfV) z-+HI}3cb_)gkEhCmdW#ar0=zq!&7?+{&MA-e=oiT(XT6Z1gW8g^+C-ci%=iB8a1J& zs0y_aSA;lAmxg%B6$b|>76yg4%MXg}kQbQJIVUiuYgRx>_l$tLUg-hbR8s?v^hpW0 z);A^KdEexKFMW~&1l8m~p?6yZEjf+_Ip(6$xamm0PQ5j*3vIr|tNraC zCo%X?--O`H{o;e4lFxnPgM~gswJm}+J+GMz-rGpw3Vr7(AI>?m_IyW32W*NMgoen8 zSRXYHmC-9u7Hx>)=yjrkXnRRsl$&gJq>n;oM38cNL}Z85h=k55;pttI!V7vNgjMy9 z3)|EuHf(>tn6Qieqr;w%PyM39gg!*IEy8MPUXx-xv7_Ij?>t8DyO$Zj)^H^>MfXQd z%y?ABYM?Z3If~-+ksog^l^bU($%=K6&4~3@NQ((nPKgO`pA;R}IUy>wYg|-bkC>>6 z-cgYo`bI|X?H3Vwp?`SfBl3}d3w=qSwg`F>WmMBZkbL0ZLSNFSErOv4-|4-tu^&$QvF<*UbFRz)nqs9nme<5K*TVp2l>h)Ry?7?G5u5|)(JJtV2L z7hhG|Cm?ZWKmWut52vDIJspc~_jD?J)5EFodk?1q;cvXQ;`KoPf9Pi$B(M#r zUzx-FA)od4#mpbddLgP}1i~w)A*5;nf~r;`pxO|A)#mVBZzu9z?<(=E_Lgz44w84R zj#6^2PHyK|o!h~_s)wf?bbDI{KKO8Pik9-K+r5O?}|Dc@&&mro(Z|LO5()0lTdRu-Hrs4qz0Czy ztv*ugT0_JZt+A4|t?4pmttE0M+neN#x9=y{7X2 z*HFK>iWv-ZkZAf)zfJVN+qe(IP8C@18Vt+b6JW7tHq7~kDYL!WFy-5>P4=0?c)uNt z_Pb%t0Y4ZXh=9StWGVfF1tQ&pbz+@EyU8W7&VgrQo&8@VI{O6awj79N9-2Y_&-O2; z2hVO`2E!a8h(6S9JLll{a36+)vRHGZCk&2pFUsSSpmTgSR-agcRVTEeebNvsPnu)J zDLZJLa>Me|0a$h>8k%P^u=H#NmYmy)#ph0A@!3aMa^@YDoD%-Vc3y)`h0I{6pWnzn z;JJtL+&y-(2820`$zfpiNhxTalgE-jdtuSVVOaQ&30T0ol=+tzV%`-k%)O$AIake~ zam@y^ue)K^^+3$L5r-K!bD@5#mh8p!+t)Dt<_k=}@r8W*jg1`tHEjPP>ZfjI2Gq(7 zYB$@TKHT&O_36WxoM#4bMT}W@R50^if2iLdiD~yIV(NpLnDTHwCO=$;iH}xe!XqP$ ze{6-ZPh2qOX#hq)jm4;ExfuDp4kKRd!SELs$usimH|jY4N~xbg{V2A*5B2R2P@g%# zD(Wx3zzpCjF!iwrlV5hk#8B+i~Zmh#l;a;Vk_zszNtie!V z_8`l3C^A9_q?tKLxE@Br3`8O{{6-%26R00ZeJARh{nS4LEVu|vzYdJO0}Oix<}9G5 zh#8AKGnP)wJi1d)m3jm42WwOmxsR+Ivk(b$87V;nG_GVdF(v#l5cs1k@J9I0{3C^* z|3&CTuib;*yBFzC??0GcbLe8SnwSyh4?{djF#nDwNhAxxumVzsA8PCIU9Az{)LQUW zZ705{9mMD1r|@z3HM~=Mgx6|s@M8E^JRc$8*+>CTNBte4E4_AaW)FRb^Sj403t{cS zsD*_2!zg12BW(!thmip!5xgCFQZHFd2|&%jjqSXF`MvV%yzsVvmfuq zoWz?kSMhSpLp&Yx8jr_*#iMZo9*!6AfWcB*gr3yzPp>V4@(FURKPknmb5zGK4uuY~;W){RuYPu%T!;k6a z_%h7_AE$ZY-Lzo5nI4VT)06O0Jp(V)^YKi*6i?Nw@nl9L9?#flLof{(`?W1YBe$^e4H%T&Mf+ZlgxomED>;=9BYFxj9z;*=l!Ss%t4sNEMPvhXez!fo{RU3m*dqU13X={ z4i6VOF;DWwoy8%zxg;9bmn7lpl1yA)l8;ME%kbyY8k}Fc0p~Qg|H5{enl9m%DNEfvIS{yR5$9@wJ>@^L*Zqsn=G>w(oVVWXpHO&%jGb+EK_TnDZAdbP_Ejx zO1{drS+T-)r&77?3FT7To6042FO-YzzA6{l3Ce~417S8j?{Xf8o&vsc&i9F#9ImXD z!%3^&IAAvlJMCv;tHV-kcGN?&(^@n-*`nUbRjSs>N3`B4SW@j2DP8FpFI(Z5CRgT| zr%>uxrdaG)uUzEV+OE*?$RGKRSKH?~K5L)r^tpYGlkms?LYOPdnj9X7p)6jSN^sRe zjB{38cVOQgyPbw(tIITObX|l-=0SCCCa7_Mh1WWic{xaxc)5#% zo>6kS9*GJ$9vMnm9);~PJ*wNMdu-~M=CQ9+s>k`xDIO0yCwqM8oa`xdO7avsk`BKj zET-qx72%yRm;-Xon?JLMoushKts^#j4n(8(1l0J6$V#CI)}-%}aQzWq__Hx5<)vr!hHiQ)iV6a<(eFTff( z0ZyW<08dFqfWJ(df2drle~d!1e~NOVf6gBXeq|lw{2DvQ`t9rz<9Avm+V5W1DF3%A zQT{>~(wTJn6>inWEC{%5PS5$X4XD4%lYYlX9t{C~P#rWHWx+F06ucPu!K;xIVvMX1 zOJoE)h|+@HBq_nZ(n-O=a*07v3h_Zn%CSLN?PG#UIz|Q6ca99&t`ZS+vTJzI9r8vc zJm^Q4@F1acTlm<8uSTr7XFr@`8|EKn{sEJ7hVEI~OkG^2e)Xi>+o(Av(SoFonjJ>E4q^me!4 z(AWI@p%NS_bRnI8<>%wp8;*s)Y}pTNgWc3`q4#TGT})+IcN9kqLtf+*WHAp)kJ3VF zlmU{X*CH|6Mk+qqSsWYXDIFaZAR8GKt`Hs>rxX^M_D4u$LC2uT^_>GFx2pI@9_{KM zd9$m3#4GY$#XmymLOTD7pbOmRSU6ALd5CSWgZdj;hp?WHex;FJkk9wUWW`QIS{&d1 z5Vs78ak_|$H$_aml~i=Rqc}3oLpnUpPc}3zR6aN^Mkz2h^^btqybgY`Rh@idTe^71 z9#-*=z24P3=B0{v%(pJyF+%6IaA`Ph)Bm5Q|KGAz2+ny%vGzZ<1F{kZAT@Cu z5|d^jE{XduB(FwPvJoPZ*C8y~UKE<_CJ9dVkqJx=mh(@JQuIwqR`yQHY44R((a|Gm zQ)l<2gI(N`u6A)te9^@{@oQ)I1mP#0e&rj-Bkd86d2lQSF^8glO$>b}vzY8e1*E3% z9TaJ!5R*0?k?9K&p1u;H83qW>Sc||68>xT{XR%*~m&7L{P}VymLf$hYQOP|ctDS3l zS$pU74IQ1*_jhtizud_w?HT#f$thLn*p{az%pV+C|G+kArT=bprMr|YO3y1qQWs9QUx%%^-;Kw$I{C5tON$*STq(~Ipg&om*Kj`b?AYa9&grtz?BnhA^MMKEt(fwdd-VYa~xCL63_ywMp(8@;iH7nR|rcqxNT z*&_W-l_K5ETgAGYPl|On-52X_crVs%79?#s%xm#y7T3tOuPk62P&MX`vLWf%bCHfr~jt5Uo-vZX6kQa4Ztq0 zgV@^vTKoHA`GFD8JUAIk56;HoLyNKK&`K;ktd9kU%`pFnE#@6@!`!2Rm~%7^8r*DU z_OV*bI=%-pk6*>iqc1V*$QR5yEd0h6TF-j6e^~{u#afQTCi>q^tU+Ksg5?g@fb8Kq zh=aiVVY5Tp-spu9H-}*Otuau$Jr%=lGk>_V7(?!8WAI%g47z89f%o0e|9%Mi zJxD>{hvn$=Xd6@?ogoj=`@!4aSWo>z=5ML&_wX%T2f`fSX9l%~If&L#VBRTU+IeOG ze=!5N4h(rBk3mnnVgTQt)9?8Z^m#D~y0>SIf}lwH`XXu|NkFU$uJ| z1f_RLP<&qkg%6t`|KSkiKVJBa66$AcWE~FM-kWW2yO;WhsDBKY%N%6NpTOv=%pYz7 zeV+iTAI0eTSqa_0s6gdQFLe4k0PVjGgVOiWkpDgzGCyWQ%=IWV3dET3;%6-gH>W1K zgu$0U!C%Pcf5uYZZznS#w!JaiUW;u%hiyOk5-@`2KHxUc<37;!1DJV$Yg@!z+ak}* zqXRP!6=oj2_?`X9P%@fKCK_ZZ(I=LKHvxe^MqCTSmrDMIAjj-OiP?iP=l?s?`*)iT zq5A^D{Gl83hi=Rty1Eg6{vAQ$Nh)C-Mz=zI>sF4hU2E{ATN6HaYr&^(yYR8wVSMOz z7H_-V#>=kH@T}VhJn7CFjGq7EQ7_hF{DPoFuic4Ww=?~BPkMh!P4&x3;SaZnWA4@$thL1}n1CH+~UUwjC4~DZ(MlpXF%}i?aV*bs&I7gb`(@0yq z8{vjGBmD7lWEftIjAgczj3*;A@pxoD9*r!;gHh{oe^e9hjoOMkqxRv}s8hH;>IN>4 zd4h{$-{U-Upfk*YPLop;e@9Sd{?MQK!|=Y$f!H6Dm_JNr9yDnlK2OqO)?JW`2@~RxrS3y zALID+_c*Hl1BY23bch^mgV3Kf2Sez&$MJJ2vlw-@#mtHLG;aR_tDU06UkQ#rCCl zu}$+ewrGCG=4Aplk&SH-)Hvrqk=}n6YyRi=#E(V&@MiH)JmQ|UHy6#orNxVJW~nYt zXqw^ZGCLev=7#;t{jhg=2=*+G#xAWy?9@ufjum-mT~Uf{D{8Q19r?64` z7MfSRK;!DKs9!Be{iLo9!YD;%Vf6mZU3)^%8uthf?wuSBPf4Dm3Xgp+SF#RK5N&sak_; zQtJ(#imDAiiz*ET@lPt+AdHh?7AMcOaXgmg%6QFn2zOYM@uxQP2OZ`Q`m?dyU>UY^ z@1CuO=GeT(4jb3FVZ$0(I^u2MhU1hPD8D6o>Yx-nW);hUQ}hWRa|LuNL+4m zNm6S1SW;s8K~iigNQy{d8-&U9-Sc=H%VhCsr3~wwWN>*E^9TK2IAAmqJ5AKF)pRj7 znXX2&nF$)rtWdw!3AJlIQL{Dx)oa60Wga6{X`U=qZk{D7GcOjGny;4>n{Spbve++O zXmMU9-{OHx-nw@(x$6X(od1C^UCca=tPtVBD*7!w5zgpK|8qxTeyGlhII?j zxNaqCE!SYZr3I=i?NMRnhH@)klv)L)v%*eXF(Xq7H5uqu$`Sy#&BT5pievED74 zWqn32)8?LBhRs{KbQ|HHq}d3vzaq?%`sYu}d-~4ndi1^gsXJ=I+5>ae9#{^>2J4Ba zx0!?Ww#!jztB-P9GnCrdpxDkCg?3&junUmNw+k2L+Qo`<>{2CJcDXW{_GPl^_Kk9B z_B-TL?N2JC*xy!2c6hCjGm zpwY1p);o_zxr;iAnFkfPY9r5e4YFM=kmYKJ3^x~~xp_;ax&?}o-6AAOZVA$fZW*%i zZbkBOZZ!%qZd(Hd`nibUa8xGg%zBKu(ty$;WRzb)HGt(C8GE%f);BJ47l$h$H9Il=1|mc z^5T4_H|HDuIw6N^#nOYuA~|>_5`q^YE?67U!G?$oS&N7e8-#^8BQ(TI6dV#D4ho5o z4hTt*^$*FC_YEmk^a*KF@($V4&MV{sc}zYjdj$(hMDbU?b3ESU^>~8gV2>*^7;4w^ z5wy&o&-+2j$O!3+wSnj3`#{h-gr9kJzQ`7I9A5HR7SNTlfbeC=tb9dCSM&E9{3O90xl*f96p1 zTtR%^52o)7lS6W3FT`>0ps4662#=nNkQna25Tk>@7-RUytb<>yJ$z!_q`YH&MP9KX z5|7vz8MoLpS=ZP?dFR+VMW@&uN{+E-$O9$EnD<0bbd3I;N4m^`tawjg8*JtIH}cW9 zJe=!4NJ^9hv9aC2*OwtQVFH2@X2U;mF?(oNu;VF{-U2RLT9!#=}b$}TfP zY?GNNvChnqvCOQJU6;94&LZ=;oJGc6Ig9kSau#Vni0~WNcuzROHfRmtSd8FUjN>?< z1xQGwek$wlGZf&R)f?_P!{Nd##wk|=4!KKVpT{*Ad4{mbTMO%a8(8JLz%t(l>+(aT z%=6Djq`6w8|S^2HqQM<1esqs$Ln#g5A%mGuEU{zNeX>e z8uJI*5dUn>|L2L|RHy>`;z6(}84Iftb*w900E^P)Sj%-vW@W}OEn5eZGJ6=8d%&nX z5Npb#VNj6<{fZJPy^3a$ZpA^7PQ?w8ZrMwbZs}K%ZiyiNm7kg9cJ{%~9IA@vouA42 z`yA#E`OF`f!`L&Au&h+X+UnjgT|XSgHIuNWW;P6J7eT*v1@vn5pj&5()pb@_Rp$il zdT*?(55tQ3L})eSV|hazmNo2wX2W05tb2}SwV$!9M)-|=ybdESMfkw7}CH3=5*aqcngKD-x z9cv&OS%a~G`9}+D5L%_NU`JQX+c^Mpc8!9@?#YF+ zBlqE;woWT+Fm|y9Vjn&5A!$rF+6Ci|^~G4$rHnZ~9-~jFW7LUx7;!=q!%yl!?UX5o zowCKy(_R>SIs${vWMbgi^%!t=FZ!RofdOY;V8H1w7;x&p)Uof&*zVcuc`Y`u4|v|r z|IUDPcLGcI0<#ZN|0qD~gu&;;=zCECs(*Du@5}ws>&gi9yfP6zuFgdFYYWls`bu=Y zZUmJZw&-%x8=Y@Oqtop?bhzD&_IHk=-JLs7zWEmIexZ{3c{R*nc<#ZQI0m+G3~Xlx zwVV3;sec5Rc!C+g8DJo@h+bE~@0LR62g>O9up8Px>I>yZtbces7V=M~LiXugNIzW$ z@iRk6J+}s4@aF%5H=7q6PaA zoCee`0R1il-TA+DxD6;i1LWUHAoE@c;tySb550ho1ITbPp3EeR`Cy_;_+$9VlSC0- z?4Q`+pV+~_kikQXq`o)**@pdY$a7!HbDwz<790!V!XAHIGf zMP!H);j#QP3mM4Ikz@*)NBDyztR=1_gm3_|CIv#le-OazLyBG(^xtB7f9WawTILTj z%pWAoAH>Wb#LmQrgpe5idlJ4&GVo2Di?8A$d=XdRv$z(Y#Lf61+KP9gy?7%!hL=+R zz;oQkW4y#e(HA_Bu>L}t>p^6Q?0*p?)R(5$mC*YuG7o9b^)T(3KeS{1pv3$^(Hw$; z13$g+T`>q>6(jLkF&>{3Q}I!ebr^~Tc&AW`w+hvGqtJ*~3N3i4up7@6j^e5OMLd+h zhkFXIa7XDYZnb9(Mn~3QFi^Vw3xXW=75hTyzzm`*+obz=!u+8d^M`J%iRo$zR{P3nfXIcu7T*sHu`(yY-*7?lFhha4+xHcpaSB9kHuc5iPIJ6iShF0O+utuC2wiTz;_Tz-w85|vc z8;3@|z=6?Uuy2fjJ!JP!#{L(f7d?MJdTzC!IRx8c>=1k(HwN#Vh;__%G{5i%8=f(u$%-C?88XJp~!=NUclY(_uanXZPv)F%35Gn-_&G^ApfAKOLLr=V9Z5GHh5-ho*&F z(75mb8Wx>L?c)1bzvLaNmk6jLm2D7)NLZ63OMPY#vla1LqXX{E?uttq{c&>MIOaUF zv3Id1a~(ajE-}NFr8d~S)CC)tdSQcRAeuD8(Wn`VhGog9UzUa1Wksl2UXAry8&RdT z2bEf{!{$;uxnUMZlc4Z=vycTc7F*O1`-JPB)!q;Yw!JWej^jf0vav1_?H z^Bul(apfv(SZRbN?R99-c0irBJ8HCjv3^xBs#is#a#aE%By-{x{_43;!gWpZ|w2j<-zS-oMTV?ku49ULu7vOC&g?#r%Qq?%AR< z85?xxp+QdzwR-wkufG;m`ZlQ0cSgCsCrS+hP+|~X0POB5M9pwQS2`6fQdGYLknNhGpO6Od(^iA=L1sSL9ksWh`K zqSUpA#3^epi<8zq6(^d1CW1J@ToC;VOAVRF(eEvx=j6}JQ9aK0l5Gac*kIfbb*5ub zWi|t4YZsw-tu_kR8Y0i!9J%JU$ToLImboV~ECP^j5soyAIHayilS*EfFG^ZhB}%m1 zB#yV-FNw3fD2cUtEQzuDNCa_o8-Dh~M;^ls`p(nTK43(@WlXPOBG25ZH>xZ~pmg0- z6k76~2VAR_W2uKsD^p}xSs~5J0V!7QNVfJxl643Yt)mfdlPncylPij~sSrooZjeOT z?v;+TJue+$`%pTqIuc7BOqu=6BZIihSv!@;?vmS;* zn@PyEor5e}O{ClEAl23g$#xbY}#Cx1jZhf9Sy$BROpGsPh;C6Zv5 z2I)YToiYI~r)2|N?#cSQyp!>F5v1Gl&#}OMIAh3uFsI+L;d*mh?)OGY96B?18h}ja zu}F29fkc;uh<9Cq7*~Bnxtb!v)e_;Z_6T!zL#UfCLfk^7g4|<8f$r(z0QVw^pL?CO zug7*7ACHr=J|1^vy*=K_c)R}~g7mL^<5;-Kv3Syi{a{7!Z%^;ufHgw?zBcglb%w8>7kvBz;pG=4^7Km*DuX#>MYD5u|_R3CF@k_QN5LfmWV>lRN#62mKC7_m)MXZx6)y z4@HFk1cV06LQueB1O%*vU!XpG0!`r^Xa%o8M|cEzz&*$xZb1=JuEB{Sm*5hU&s2%ouKA7H|!-gG;z8oWp(L6dnqP@OY7ZM3&etqFiDV zu}Rt{;*hj;#C2)w@R!ooVPA|>l^7vl{Z{!iAiu~JsC86wNrQn5vB zv&2010J$PDk9jUJkN!f0-?++q!eL$mt?Yv)>X%bDhmYWKq12C{?~0a(cU(`n#}9)G zvlz#OS+Gx72)jfr*d*$~I+6QOCa!~Jk^?M~JYb#_fVIg{FiTE_X>yU2Npge8IC-zg zDEX4uDCw!lDDjiXC_(s*a~zBNI1XEUc~78j5p~le={cgAKg4nFIZ+Jf6csq64uoym z7+9rG!@Bgjut;BuwHd2mmSG4}*2b7**uW^$1#2>WVVD^PgUm$eXXQaJs}{OhyP%VG z9y*zip_B1ZN+(_TjpJ;C9qfZ>VVVJ)F2KmdOpRWVm0wd@Yti$R8d#oz(fOcUJRuslUt1t`8i>k1!Xd5((PD8Wc zAvE*fV_81)hrIvtGYe}8WE)VwB!S+WHYkSL0n~QMX8w^+|6e4AK}i?rmi5Q#vQf}3 zpNtjdv!TWPdzV*eVOfP9mR6WxNu?DQS2|-+r7sp#MPPna8s=4%Vs6zY%&9ttIhA)Y zr{XQbhVL|<9 z%x{>2xeXea)368{jVmy_k^5janqfwhE!25|Pj3ps)aC?CY0k&w=0;4~Z~zmVZ(>r@ zD@47O* z2VwHo(U`PtGA3-Bh4HNmF|Jh$V_WqwX1f_iF?1if!y6-ZL_lpv28Qjd!qA;NF=Xc@ z4BhbzL$`m%(AFRSp^^Pw!Sl{9VjGll98_`~*02Vlfi(cl+=pWmYcRI324XvFAa()6 z_sL=C{;n8&pdSVuRKvi7<1ygSbo4(o7kv+FqR(Mns2(vxFNUr?kNTkpYf`!$D@51h zo1k+16jY8qMAswl$(R37P5olFc?PwksqIT`2Wp#ATYD?lVNhFrH}&_?gCAxM#4(`T zDKR>oQACGx-SEe`eo#KI2Bq^8pm<>>6fP`+{GY2Jd(jLs7o8w|DF~8(q(J;v1*HDk zPR;`VctD>2hcdQ%Hv2n{Z646XIvi@7@w``2dm+zzI{%;$p{9}4f90{5I)6v&J2ekvalc%lz@Vh@fI9?QSU zW1lD1GlSW{45sB@bwI0kQhzV?sXdz7gQ(q$+MO;j`?yN&8=wxkOYTYYyOcQ@)s4ld zeSs%xz|)C*K-J*H$a3J7F&9QUvOqG3Bm-~R=x;f|-csz}q&Dz6;CS=kxm)twb=mHV z4zT}^Fn>D98W3uCr*^w*fb>mj-XZtN6Y`wA{5P*;sMVg|+XHw%kPnD#lh5kFH~uJo zHw1pz^TCxjV(yCyA(Q0&hks@tVD|7&INy$mg!u!QKR};Y5J%!k0`UzI_<}fmLJB@0 z6Yr3RHz>v{RPu1@@B*xX0Baz?8VH=iBV5A+{&f#;*=*m~e8NAu#n1oaU)uD#_{lIb zjxc`!^9SeAjw&#c6Eu@SzAtnh((&^xg= z-iU+nN*sX~;y64LvkpU?fk)yzJP?=QuBZxk#0|J5Zov)lZd?-|$7S&qTogaVd71Y( zBgYyHdDdSj2srr*f(-R#=>L_OKXhiFbYcF`nR!qr4Seme3?JI*;kEKwyim5sQ)TXj zq3p*Y7=nAstiw=d9foo$ZYpQvhH@dUDVO82avd%yZ^EDLcH>;T<2c>^3Qlx-jH49te(E!S=+68>mHp9&^*O3z@wt~e-t=6EXRHr;*xeZSx?AB^ zcPCu$?u9Ge1MpY(aQxXL2IqSu;cSm|oavd1Q$0&?qE|JJ_iDz`-rI4w_hB4Vy@-8% zA7FR?ci1^Vz;@Egnw)LFAatbX@212YgljMcF@G4`AD;&g$E$&p@Tk8A?)1~dwSIcI z)PF6`4Y0-Ofvz|?$Op$+pL29@1P%|5#i7B;I4~p=`-kLX@6a;r8Cr*3!&coYiu0#0~@$Lt9i75re6@c()age2Jtfo8PNsbM)bt1;REqt*eF~dIu#ex z7U0wfZ5$hEgu|oOG1GCtzR~W?bbPUEOfYtijl}k`acCWvg017Sv1MElHjl5yrturF zVZtsnPdJIjiMLQU`6X&6f5rMK0;<~}^ps*f3^RyPJdUx7cssTe9*pjetD^=o?-`Gy z6EtvOk|y>{<~uN^n4xv5H8UM2v`qEH=4t-eI4u+#rbnT9dIB1!r=vkV4|VG0sGZS( zni;LAo_Q3NGq0h1_A`{u{)`e00mW_jXNlQJ-Y&+Axb92BwPZ55#$!1>Srvz;kHVfA z)3JTlLTs6>%}i$vHfUI&Ny82eb6in3#~ZbC0#P$J4ApaEP&F?ZmGiPuKCc*M^J`JE zU<-;D9755;f1qIDW8^LRh}=cYAKLKG61VZZrA_87kn!%N>GWLFSpP7qJ2RhQ*gAhQ zHZGit#zo6fw@42)i%n6z*b0@49Z|8^9c4>c1F?iP5KAIav@`*QOVd%HS%5svD&#KP zgq&sjkhT0gGPNEeUF$v4v;?HK;h)hm^&d{-JST7Qr{?h)cP`(#uz>l)66Oz@<50J3 z7S=Cc%v@&`%Cy#?M9UmSTDB-$;f(whp2%C_kKC1E$X*$Ptd%Lq)Xqivs&b^QYDVhn z-AGw|21z>ik*M<)2|5De+aQbuzVQ5S&t~lbJ=YQ5-uEn__txavhZTKMt38T2&vcZn zUVx(2S}4%bL!OQ)a&)YarQ?81T{mRt`XEg=7^!-ZNYP6~l71Ev^-Gap(13V@9f&nN zg&4y-h&FtKD8nB_K;$oQD{eez`(Ir^pTVEGeJlB#Pc~@Fp;o6CD)rP*qCXi026K>W zpouI4?mJ*;gf!OYq#D{H+0YqDYdn##CIInk!VzZ_k65D&L>m_&%A^jFCas7tJ&thG z+Xyp#g;3M)L_o+d{EYY?&Z6Jr`5)1y*U{q~n=aQK=(6^}pc_iKMk?QU0;RZ*RS~y>)lb{|V6E{=PB3(J}kJKyv0@Ywxr7Uh}NAkLshE zR2SW;n&@q+jyb5R*oRaZ`?x_x?Ar$Av0sAUR2JL)f536%dXF*iyeF|g{1-#npr#YfZeNR>zyGHNjR*32tgg@KbFrIO@L zDoWXFP?&PYpdjVZUiqo7_sUQC9Q{RvWW^@bZcAo$=4yk}yG{XrLc$maM# zTAMgQtCQxbCD}v`$yTaMc2;$=w<^gIDo^I@!(^WQr=+SlHAh9MB`Qp-Q9)Xp^3yjc zFMW?eZpLYYoQzw&ax(tiD?8(}UfJnC8)T<<{~vt9{rHL(_ai>I1TV)3h)_k_;@Rsb zk$aFbRxN3>RgWH2!&%Oi>5eK-_f%;*wFl{8D$0me0d-^f8ClB9EK+V(m2$FLl%2g! zS=qY`GP6(i%E-9^Ug?#d{VDj#AU&)5|KLM=?nys#K=8psa34+PSz`)uFP|N0)IMa4 zQhnwORc9HiB73ZpuTPoirGk)M-TmZOC762+I-DXx6AVk@>P zrsA-oD;_q8s(223Xb@HYy`sw4A^r{jrXL=SMweyVdn|{sn8!HCC+8Ye7m;^e+F$wQ zW6^kKDzkEt(yEp#rOHl8Rc=bG_Emg!h~lcF6kDC7n3_yQ)f6hSrdkoTZ3?g5q_Elp z3gr-iklJU!e-%>mokFVr0NsD#g<$SUd~g=K4;C;M;q53U|Dpu_p`1N`75YQ%aHZBw zRT5fELW8N|8aM-@(MeI9ofFv@poqqBg*C=0v?)y?P5BCLs!&jKvjUsf%fETA{F={$ zC*|Apo_rg>kzeC)@@we+3r|MT5A^>@`1ioOv5fN{Dmee43jSL3hX$U#HV;-@>v%=C z%~nL)Vuh_*p^#N}3SRB1pw&JK;GOUO$?BKmCU>ONc(EIAM3am8W{(07qpT8&;kECVy}(F-&+RBW9vA% zZks9R?F;3&-AoQUtYp8#NwzyYWwRqd);l9)wUcu&cV^3CXPH)XHEUVdCYg5~l3CYP znRdM*(;c75Z2OO(`!AfQ|Bqnz&L(n!TDTXmyAk%Y2KlaH1FW|ns?xjzi<-Y?{DRPTutoTi4Wk;hdTxC$S%&o*h~C*fM@VW$U!*X zOAAj9mC>p3T5xKn=5ucC+|#C-bJ|j~&p2w<8BfhP6Qt>9V>I<_hNhe=)ueMRns{!T zCY(Q|ap#}VxO4Ao{Mm0b{>*QG;V}F?^;qjR-~+fzwv&UgiyVmkT9#$K7CF<0km^wmWgb#1vuT(i^g>+TwMJxD`t#OVPFK?mQg z*1(%p@yebncPp}OzMaq9i#boF{_0h=Hb7GP_@BG@zeKEtB(EECSN z^#B8(<-*V2V8i+%c<(RlgnJ$SZajzvgTGVZ4u{(dZd+_Ny-W_sb@+V?l zp#g|`6rn$8IdB9%AROHV{Xv;q7xahw=#O5!3j{srNAw_8IV^~PHxbTE_zJM94Et)}YGin}>1F!lCHmwAZPRlG zd-ROKVLfGVR!wSOJwSMGYfGcP+f1@^Y2A46imnZLN|%SerHjKq*Fz(J(K)o3v*66%(A~E$c|d3o zqtGA5qREV%pzlV_)Tbki_1>@*dSjTA{yE%N&y5VzQ={Vb#OO3VHYQh(j49U5u@$;8 zwocc^HS6lQc3mF7UJsAori&By>cWKMIzREE&Q83o)06(K6H`9Xv8g}mC_2pH{|~wc z68DcL_h4d=7BW~rP8y+4CQQ&fV`uB1f zx-hdu=Vw;x+^l+?nbo4xvpRKZ_9mT}-KFDm4(TXb%;CAWbcjT={qsN6-UZ)l&w_64 zMvK|?H*^oD4U@oh12iDsb2qg=xjjSm!qmxnf-{6~&RwA^^PO~Yfsf7`hU%(q)xI=XVD4p};BzoieF zPB5BIw02r0YKL{Ywpr(Dt2H?gHkI0BLk@%uIS{t%wGJ(2t=(aD+FwwI{o`tPcuT7s zzfhawuWEJdR?FYeJrj9(K2tv2&#!==<>smd@s0)hgY_gGwq2lo_RF-}!46%=9Zkny zTO7l+$uU+N9h0@*DO2m5^0n5fRGrQ>>Tqt>YL_)y<+4p}t_RfOc23Q1kEzN1bv3wu zrh50ERp;LQf6zUT>E4(=fd2_wVjTzgnc`2{p+7i{(_WW3+UaV7zGJ0LZZ6v3?xS_? z!CK=Ur4IK5wR@y#l}C=+Jc`unS*d2v1~qxLtI=z->b>`=j&qD_eQv47_g|{={X~_% zKY?yl^uXsXy79sLrYyrY#2L=SoNhd8W!XJKPRw3+at}Obp!F<5pGGMnV0#B+u@P^7b9KAG%SvKfL z(5>RY|KSHC`T^fR!!mo_gFUu4v5pt}J5TnQKEt%xZ>rY&8>%zFT&n_X)f(Wcra<1i z7#OVjz$n!QCa5MbO;tfTsthVtMR2vsgIiP~Lg?7zYid)lAa-;ZZ(eBTmqzt#s$R7dbUwS}0VD_N-_)LFHmUaAfa zP-SSiDnesb7MiTmuq>5?6{DvVg8g2)}pk36cps4L2idR{qEA1FKOd(f?H zw4gun3H|U2W8g7tzZ6KGb1*Sy5c&t-Hv|t*N7y*EhR;@G#A4M&Sg1O}L6wp2Dx+Sj zG%`fRkx?p&OjKc1h6+lVP(WU3;wJ0xbKu6`v>5u z$US_xoUuUv-^BKZLKz2P_yBAUVXq(FS8b7_)EGTOwJ{4-8M91fF}5m+aaB=_w+do{ zlou1B+}Jqf#HA`bE>~IcrOJ%2Q$|9&(i68TE%A_26E7+y>1pu3k`uoLzbiR`ry}?8 zz9rjCckTylKN~>~N+do2>-oMax|bSbhpQ%TD*Dm_l_r>?BUz~+!AW@up2|rGP*!4? zG81E!k(8qJq->=n7b`WnMky()l$^3zNvQ{vn0i48sZW9bC_eQY#i#rZi2alA;vMeC zCmDyA@xjSxbf_4{0qEp=GuyA)gdwU(nuxwMR|P3c(U4XuJHo%yb!Grk7DDLTFTFTCM|CK!Mo zhwVq<>|*=5fvs>GpLI#ZzSL4c<|B&R5`9g=e!vyN8$QhgAOaur@!rm(_Bg%++=NYO3@7oAp6(W441dP{+YUn;QR7X{{b|Aps#7z_B|G&b)| zCGLfHRR%F1D9)z#Kacv0f?-N2oQ$qBPjSUd6jNfMs1kccmbfXrl=onihA6Z&S|O!L z3M$J|U|ER*%If4_-XXv8o$@U|A)oSF@-BZ}-esT5yYy$!E$@=Mc#^Sri9R?EcULCy zP8Q=Jhj=3w{sQ!eV)7qK2cheXQ&iVglV_8wJeqvu))XSwrf9h| zr^vZESB}jUa%gUmJ%@_e5=7awT$WAqi?V6@ST>D60CvsyaFy|Q3jRHCuP-9zDq$SJ zoeOtT4g3wnzRkowt%K#ciuZ7=o+;PWMsjX9lViK39NHaZ-{B$K4u9Engv+`kUX~r1 zTG?4Fi_Qiu?_8&4oqJ{8c|m5K&&sUhLz%UIC$rVR12hP5fqp*<_x4hBsB-dP;4XnX zv!3^0z#Y;??AJjJ$lCj4zkV1x%0yXhoUN4`jb*XPT+26EY1t+xnQ!ux>E=M0Y>w2@ z&BelpoPLQA?PY0<7ZTDZ$tM!U>pxZ6tecROj`ZZFN< z9jw`VVl`_|hGy(3)%3m1n!0z3rtCeY$$KBsq&;tI^6oD+dDk!Cw>vlrcTX-*XCpck zc9+1N33nXa!Ek$QC;si?8T?+(!8||?#G!tgab&or9hs=9M`vm>c`1{QnrPxNON~F~ zsBtX1V~&St^zj6ZJf5rJCu%h8#99qGc|Z@GysG<8{7VlU|5Ohi`w{$n2m9gO*vg!= znsLyH57sjd2uFDYBA?ylAnYdx<1jf8$KXFHO+ptKb*`6&JaoSvxG+Y8FHF^-3v)H# z;$ro`xI%p|I;am1VR~N-Q?Ea@G=QsU05^KG(A^J4Gs#T`vpMj`7%XQqWY0qE z4I-F?x#@2+$Zuoh?Oh}Xo&}GAe}OmeU^6zi(ZtG~+za?S5u1aLqXC_w4uv0I{1E)` zPJnmV4K#p9r~!FgEHdD!UT7`@*#He^;Tw;xGM!DSAq%k?+K4R=hrHMrM3QsD#Pezu z8^$#NlGnKKYd6s_@CjH;(;D{S1GrP+4ujj{9CbJs;J-``$~AI8Zczhsn;eWD-e$Zs7D!A^j95e+n6B z!RKX1(O~d-B+c`J+Zyi0*Qr6m=CRm(|I_&XdAR-wUIn<2!1t~e@;;yc1s{TsY0Ibd z&*x*%Sf-{3!z_r#vead@gQ*(paXuP13`2LKM`~PK+OFe`op)x++V8+J)jN!qZ2)11F`v5qWdoNhy!R5$I%AP6Q^HC8|cZy zc!MT?LeszJ+aJX1oP_~iy@zg%;TzBVMec#9MY*FtnDJ>#)`lnCA^c`68bB(Xx$qUE zLsW9bI<$yp%wG-P8eFjnzU}bsf$uPUr;)5K!*d(=y^8kn0lLRGXc51oMUaC5p1Oy- z{Nx_o(II*?nI1i839YrnZWrwJhc^<=MC{AtN(Ii0eKjI^@kDs)PTqyUN3A6 z!M<4dQn4)?u0puVxpED*HDFsSc6GqF0lsbU?SbzoeCOc0p=xw~_9yU0ni@nb1p}7xO2&8P46k(H{oV zjzJIV*Z!mR<$Y81p22***4s=k^s&>^eLeMfzaTw2AWF9eChGd23|$?Rrz?Yt_3-@_ zx^#c7ETB&E{fG9A?$+MF zfg_0TmYEGV&>zG5kY+Vl?+>5`WWY2%^MJ8#4_~R9Bb{}1w689W3Dt$MF*-jsS!c&( z>h!ogof==FljAvyVtl=hO=#7T37tATakCCmr*vS_G3}dtNqZ(gsohgh%%*;!?bCkM zHguS+yPZEmxm6}lcUG$=7f2=JlRYSO|{k8Y3@2TJwV52 zgzM;xI31puqC+#YbYND#_RlKSzFAe;GrK{%XRp$(IqS7^&JJy#dq~^no!6Fmk7@J# z*R)~5XIj7DXRSknSqs+O!=Gh#A5LVtn|61)y*=eVbf1yBID3Z9%w448^H=J~0^U7o z=&gN5!P;XKsoh2i>ROnpoeQ(Iom!=B#%0=ST%*m4nzdxC8uYqf6NH|VF5AM{0DW)c(RTxUZ4X@&9%qG zR$b)9>@f91&k5BQvuJHHOVmcQbgehf)jD%>Ak4{uSk|D9WvjJ%`9`fmi)mePQY{wO z)U@&iHLgU}wESLmXfQpfU3nMX@P9OmWyP4d*A#x1oAZ30T#Ei+j{dNGj=EMX(KZW9 zZC>ewrsIXC6NsL}yB92D)oz)r)mE8mv&vVib*WmcYt>}asz#f2s<+*#I=f@4vAe8l z`)5^U|Gp~izg30(A1ddpnm_T=6qXq5e-7?jme>zYtYnY7qPO;NX3P$o>Dp|&Q0wiM zYmJ>9x{jMx+54*1p85y-C^b1GsL>%!^$t0zb1YJ=W0k6%npEYqMwQOnRqk?FWv-W0 z>iV=wT>qnD*Kbth`nw8UyZ^!$GZ=>~aZkc~)s8sBp8Q%q_u8<>bG8cQ03|dxJ_m+EwhiRfV1hRlq`+ z@Aaheyx&!>H#5H1Z_4rN{tKVXV<}w5IDq$}Gx~=M`%5SGcaFq6F617#jn``TIcnhy z<3>*l)p^>h#?xI@Uy(LmtHbd~R_g-`u00R(p+Bv(F6G`z}ON zT85TnqjEnNmH2t7*e_6pe&H(ci&dU~vU2^il@m~;?7(Ve1-2?PXrnTM_bM&;tWtv? zQ)?+?U)3fAkN~$@i83Vy>W}stTTjzBE@wp-a$_ zER-8+r|eKyWrlexJuFaZVG&BDW-BE;P00~?N{TF3Vq}vNqSh)tYM0`oPboI~mSUn` z1D`26>SsXge-Cf4&FRUBy6DGPgmY&Q@fM(@Kn=mf8=-?$8a@tvX_oRLjnR;nDI?NG zX;IEfjq+4-RDhDA!ju>ttAywj#mD3*j(V-w*apSKbt*b;r=sFdC^G(rBH~|FMBJwe zkNZjC=svCBq3pB6h}a#MI~e@GOQGqHc7-M%Q%Lf41t-6(;G|C! zocJT?{tGWSpb6sl({S#Eb5jKQ=8^OPpXJfSzOnZyJAQ;R5~nC7ah{TrmMA{SLb1tq zib-}=R5I_NNC{FzN~FS55)_)6p^(%<1(TZ-l(t%d>01?$enkH1SLK)flKj#>20zFz z^$$S)0eFUfxPlLk!`~G}>>tB8h-I(OXHh)Q`;y2zPZ@%)G(qv?=ES5MD=K4|A~LKM zp6R5}OizVm1}Hc)T!C3}3dl;6e^$Qyva95q-74?w&GO1QB+s17;CXpwe<;ta@4+AP z%qQ!*e8Y(2$L_v8g70B5+ z{`qe5%lDOUKJUTEkCs{4S@w z?mM`}I6Q~V`{3S)&8_fOrxNp}!OvDa9z7%~|2}k`;pjP&6hK~#Z^xDtzLr__F97`k zT)_W_;NAlFs%-M!a~TJ4r@x;9h_`7w#mu!{PR;ho7+7rj7W!eSpk6M#;2uik7aKt0ilUwP=mGjMrGn zXsweB*LrEe+91ta8>6{v(=~ftv1YDo)Qoi-HGSP-OD$aSZJU**Y;)3NCi;onLp6STg2rvn)tDXC8nvTSBX{i8h#i+SY{!clzWrm3*!I0f zZ2k2P_LeX&mNO?+qK9I4F5F3Qhr{jDiT<#j_fAkHf(OMWlm~f9Kf9pcgz~}hYj?>7UJ(60kjcA@Pt+&>3go4`KiV{0wpK84NnJ7g78IpHG9gzz27*zKJ=h6(6i- zK7cy`e}`hTXBRmL`#1{}Uc+PXpM?JmIT+{2!Qe6kF<=0T-Ec6DiEbL0$3|!gi||S& zUp6Aw13)|pvUKr!gJ>|=?1Ig_=2b=yk%RIu`qNdBS96c^aXvAD5Rz+y z7r;xs&?fN7E5p!4@X6~l&_j&atS{%GDeZVa7>$C-?}Jj%Mj*2lVCaV!3F`N7FTkBj z)573(gWCdM8$Ap^Hjli8{&O3iKY3r|`@g_z;7#xjco)0}{tG^&Egz4dP<{sFlMcl3t^eD>%McEF43l@R?x%>AR% z&>?csKZ=RqE72mT13?G*oLKiWbdXPoxj#k+`B0q6Nj=K@)Wq~e+V2nv{NK}#9{r)G9%c#LR>a?KXaE8HW)z%B@MWU4!leFYbUB`lJSak)a9$dbMZuQ|?4|=qip8lAO<}??ZsR1D$<$by1&p><` z$uLNOD;>LX8Incl9_3uU8mMrbc9iHo(!EwP|03GnfZ{v`4$iT2D%wFcX783f3{-){vS%u zjmO7x@hk5Eku~a4;o#|Df)4ghSs&|u4^lsPj`!?w4 zeLHlx&p{pRb5;lX-qgN+FKJJIM(n`v)isDZkiX$h8+y}*K1hn(=I_xjub|UC%{}s1 zZ$t8Emg(XC_PQ{T_ka!x(CNVuI{5(a0)3D=kRh2mIwVhrhZgJ5&`KQ~RpEu%lwrZGQgH$LY8(-$k)z^rP?vE zTH7ZzYU`wSZJxYIo2Km6hAGFjZt4}So%XCcr@ycE>ECMgjBc$0ZTHZ_|25q2j)8w7 z(=WJ6jn2g}Lv?226djvls6*4twQstucF%BA*9<@Hm>HsNGo!S1R)RLqO4X*>+1fC> zK>k9LqGz z(sz3{%g-G4m$TU4O&_4cv&U)QygAx6e~Gp)Sg9?%Pim8)r#2V`Xq^!?7z?SvSeU5x zh3Q&tOb&!`v04{Zsd-V8nihAeVaZn2FFB~%rRP;+^0=x@-clv?NfoBQsvI4r+~h94 zVOo868cPgI?=@ri!LfzpIWM?RdlsTUESjz@ON_K(=`yWFi|OFJnAN77zhD}q7Sjkd zo5iZpELja^nX0E&sg~TBnq_sWLW`*+FQ$COK9yOVRjI|JDqi`zidKHE!j(U(01c+V z;x0ZL&(uGUWsfEIqA78P8SxgMyO*-RGNJate3I5KpQrW}CTc^AX|`}wqlJg+S5p75 zlKO|0QL0**pi0X$RaoYz%&J(WRy8WNZdH-Z1{K=uR=({i<=Nd*uHCE3vHMinc0Vc0 zuKO>1h!0-E_FF7*XIX9!vGniebK44HZwt;nupFyZ*0a=Xvq%lL%T;S@t7=;pRoZ&1 z+%{09cHt_q)50m$F<6K>Qbij31v9nP`b;@N^|)H{HRpt z?!WLJ{r?PhU$sO7vLVi}CHCiYgEg_YO<%Rzk$d1U4UNZ8RgR`Acd}BclcS2A+*Ro0 zrvm3-ZB@Us9syM@sbgK?xq+ zf8lLIwjKE365K}}&_5iB{rOx2+8leS!FiZ!Tqdc)b*@U?mZ->Wg$ms5l;=+EfxEY| z+yj;A9Rdm2%MFl>r$iU|m5%_^30=@&kD?EVs zAG|_8+_Gb93GY5PVjU0q0JONX*Y_me@E)KdpE1hwovv&@LuL4xD9wMRQvB_e!Mg|mZga04h0BMZ}mWEQa$JV(k?i>#D$5 z>L22$e~62cZ#?gyh^PJ`f%=DpMtLTzlSkqnxhI~JTjCRPOL$kV317=K{uj`F2ahuL z&cVMA{*7#Z+d>(KVeIumRwVq{CZ!V{(_G;yAS5*I5VX}SD4OWBuNDW7Cd zc_#k@z5>6~>ja?~qmYNm*t;3f_`s))%tO{7IG>yaxkZrtgnp z`?d&TK5VXwWgNy6@4y|O%=r^(#J(BCJz0a$bjHX%cbZ)D=F2&6i5&Bn%OT%J_W91T zE%1^}L7=P(qGX8{v$8N(D+(*Mys%Bn3b)Fv@EEu$)56zfTJX8d@~OcS;iHcw2=BFql#-XtawF+<)6x^><1Z@qCbGs*xrTD zJK?TR;aI6+(oul!qt2C~?U1QpJX>|K}javPTMy>ilquag#zuv)~bjBjw z4LRtcaOc2{#Hld2eQKydY2f*PGx2{L`bP)nFs$vZ$?N~2iR;H`!iK3Dw_%RPZZOv9 zjpiD)(OM%mx@h=DKMma!sRuWuYVf8K4cyeM0h_j}-=-7lyYW%*mile@RQ)$l2LgI> zf!Yd~lZwzou{#a!XzUJz+pUG?@2iRbJBdHn^9+6yIS5VAo{z z+cjH#cP&((UFPb&%SHyf+*tg$*>=;(yD@Gz7u>xD?Bnk*foH+H41&+@U@bN`mN6$) zG7hji0q#(^Nk!4Grs z%NTN*U%vbZpdFxzzAkUY2mEt9+`(JugPmwV*vocQbKsqLk{l3zt>1ZgFM`Y9D!2xE za#C&#L>C#(0z8h5`7|~I^VuAlvN6FQJxPgAMS*N`MlkkiI`x_TfD1lz;|^*&@d1CG z3U@>oa{)HnA7Kn&@7y!+!#k8N>~jUa>);l63_K2=1W$u!z_awt^Mlb2Swa9uJHGbgOpWW4r7q^*+Jqj_1=m3|f>XHd zGFrrw#O!YpvwuoA{zMGTc^H4gUG1SK_V3X`MxhK$fqMa2Bj)hh@mn55>_ONTg{u>} zQaX0!5*HW2Q-(d&SXmEO3qyl5InhBLM-zGsUp#^@ZlDBQgY^nt=&2IAgjE+|xd>ZN zE=CW!>B76{@x@RSit%vI#=n2&V7S2P%N0VgD+aD)crxJ0g@^MvX~!+vaRVLXIywj^ z48XM!uAN-}AiZ&h+xR+q$Fq3lJyeme&?0_DDdIfNd-z`)deKw;>EjXjc`EiA!fVcN z+QI1#UjSU;TrCcsWOy>rSaPwa2(EItYS29zVQhnIExN~6xc1_S6FSUK9;8POFmU&w z@Nl9oUh4h}|7!!o7d_@bXhQ!&mwHCC=)cA2Ef#P(auqMQg5Zk4t~l&UfhQA&T)2wh zDo10fMPF%#ubt|ojd1OvTIwkOzlc{JMbkH&#dcaQmhq!6aP(SS&8ltXYQQ9#qUfYMKXzTDSZ5ffT zO(RORab&gDk8IYuQJq>ddaF7{A5i<4b6Pd_F}03;UCrY^SJU`k)HtDA4fpUn{2%p4 zdL7R6$F$FJ;X23nQx6Q%(c$BDVDudA8M{Y3DF%=$?cv#7ne zhp({v9eAIfM4T~|cnchz%wB(dAMKquQah(iCwIn3n`W46-Ao&;ndzdAS>9SbD^P8- z!qhrDM$NMm)iftv4Rdl;Ket4+bE{P|uSHe!*QsKDm&zBMP^sZHl^DLLVxx~#X!HX> zd$@;BhBCFo{|M9S`ML0eeV~g!Z<~ewFn6NX&Y!D}1xvKbaHU!d?bT$&`3puqsxt~w z?ZOCEFN{^yLe5__&Q!T^fl3!usAO@YiqK*Tmr(z(l==sg%gQx*PC2F@D9iLa@CWEt zCR)sY$0F~;$5HJ%Y0Ul>>;#*Qh<%NyeJ~!Qw#75mykwyo$cw2pu~xN-lPXO-(Q>GN zFbz?OX_Sgh6I5uHrUJ8E<(Zc%7cC}xS-Y~9Z&l`sLrS-}q%@0Xlxp$5QY^j&)E-!n zdjQ^mr{UZy;2YvouwDcjmZ znKph(w+T_2O_Wk?6O>|`p(MKkCE8ah!J$=g4jUEgxKA-o=N0Yrq@tYORix9`;8#UD zcHhCvY&UMee-hq3miWMmeLiRbb)eFEkVWV{6uzA{_`r_X-6pDcUtuQEoAcbWc`7y{OAccBGDa1Qb!9H0E@+ncEZ@mJ1*T~;*m;C%r%h&%=`S`ynA3vT*_)>e|OWgr@ zf^G15?B5UnW@p9$`cDI>aAA+<#vaRq=Z&5ZDA{|Q;=N~}A1zRn&r(JBtWcP*twMaA z6-<6kkY9iT{Ua2>`8j?88S*7J$0x8(-hmzR3fd{ppp)_ldPMF)Zvfu^5cm_|>_za{ zO6Dbd#sd5s;B1An29&t-ywwYSwt`811JIX7D=J_r`q4av1};``;4%dUS}TD396$1N zd^tbICpb*rA#w5yNs~utf!srDyXa%GjlK^4CEKV^WgGbe_ycgpIsJYF{%zRY z?#sR4e^&#_;xjo2{!pGfhxI~NdQkpRIuz2q1h zD2KR6*~KNvHav$*6D`IqVX|Bj=g29^SPn^MvP-g*ZL))GlHFyM>?g~V zFs($3S;4!lmZz3#S!$EaQa8#p^`J~rFM}6llJb#ElD`AL$rK&tEd0CiIj?!)OeOL! z!iak#i1{Lkx!?|rC+1D>7}gEX!HuS>7_s z3YJM$jFx7nX>oRu7G>AVID4IpviHd_`yzNwhFKrTF!Ng(W%4f2%s+G-pKlN5et^3M z-oj|cK`hVR;SPn{HwFH5o_%K#f9DL;iahj({K@Djvt?4SP)iC-wYYGl78TlSVWGQ> z3jJkR6ruS=$(mb~r#VH{npM=PnZ>&`qv*V*7d@@%h3{)d!8hO+%_J}80Jd+$?q;~l z;m(CS8SV(U1K{?6+aVYIqmcN&q_>RA9+VMhwHTI9)%=RNnp7{8^A(~nhr^!`Wnpjn?3012!u4;$IR-M+EswXt2@;@3|@fG-4>-w`(amC~%wC5dYUx1JTSm3~fAvU)@*3JBDg#$5=hsIaT*}at>$b zA`R?ZrU9L{>euP6K5K&1drh2rt;yl0r}1mnun_G91kKb2spCEHIrvJQY21&QXmQ!- zk#NTs-~;UTswDQUCI6t2_`iktdo}!>oCCU!vnV$b18?asgDt}llE#6l2&{9#BEaIa z)gCaxY~#k-hGVxC11_>{8-ITiJOW+;@AJDKsRiDOeC8zVPJufTyZzvHhI=J8FYX}z zhIb0Qqqa~7u^s+hVq)zL29n@59MFz^w4-MOwtq1Y)06&esWn`|KdhIh&i_~9M25B=pJ{AdFNN$4uH zVe z@Xrx&d(&<9*ldp57Qj0d-cg6pfA}>6+(;nB?*U99pkpHrF85(GIG9bqNOX~j1P8Ow zM2y*_(vC-103Quuqm)A6fU&pf)W^F3+QSoE;E8K0YsLp~$HE=Bi5w7Yw%EfM#NO%f zjyZ+>=is?;kMAm^YKBRKT!6Doi^1@7+ud?jaF+ej}TPX}i=d)+s%r#i}ufoyx2Ucm#i@W2>6 zFtEq+KU>n%iXQf-0fU{e_^vwgGM}%4*TI_xgbxD<Zy>K2^IrOYDipACj5vO!-?VJP$p7QM6!uv3s4D4 z$rz|Yp{OIqZ$b@OMfALe2zHa+Av5DGV$j!#8DAw{ewldkMgHb_V*KaOS)S$U&)h># zh00w;f)gRd$rS3T$e2b%ZiIHR9A&_P$k2=542CZXu0*)fiBxk@LyAyCdQQlwf~y{` zX1G?vwGOT=@a!h`K8o8Oq5~i0zgOtckLlPSP!76v{cq^TeRpwJk@%Smm7c7OFUdss zl<4v1*gGEH+3+sLHVf=>fXf3eKdu@APZV~=!;_3X8E|nr1=&-qm$ezMcxxH=53Xmzh=)zN@b4an4*0r~12SgQ6x zwOY+vmf8ldSIhlfYJQL&8gfPTL!MXdun$x-{5w?-|3g(HIGGZB1?Ri>5eW~1{{a>N zaFp+d`VJtoX0&zes*%xZ8T%muIsULJ#$Qs|gr`+H@jVq!`c_4g{s5c^2|np{H~H?y zXcmWYWIcnuOuk)AbK6HykuiFb){UF1&hbmMdV+=8CfciI5~o5;@>av7K-EtUQ|;s! z)l5lL)s%EqPUTd{sii8NR;QBbt5q~(iwb8PQ2xw^lsD^1<;;3l*|Wb^*6iN^SrOp9 z`oiio~=TYMx=FhM8unn`Nb%S&pij#oHif`>Jeq zkV#e==mAw4k4N#DiP~eN0#1<0TWYDCrS{5N z>Z**TR798rD%B)hDJHQ>Hsv%#GfqP!Qzp*5L9xr$C}w$=qNrGkTyaYgD_#SiDSQRh z2rI}&0MFz1hvzd6;N7w0t|HTBLiDke-9LXWUpfHgXSDLorYgsLzOu}hC}Y`jr7g2j z$}%SQHr%lQnW>uqE;3w(voTfE2PxI(ONDA?v-3bOfB zLDoM4&JG7pvi}AA=v{ zYySrzvk`xv!QMUaZDa$oY9&u!S?()95y-V98^P*9rCN_ulFbao+bmG5?NUYAE?0z| zjl%7yMzHfxh`qmp?ZXu05UW5(su7$Dj8T#<{%3*aO0y<0MyK z54rgI%bDsaN52F)_-D!9zf^VsO|m70+&b_8xFjnwb*utD0N=`r$}KB@o~+OZ`)rsK z;BSGm63zlPr5P?fg?5MEizvvO9kA~p`TC8Pm;V%b_|K7BfU#Tx%;ZG%lw+X19D>|r z7vwA3;858F$H_V*LzW@MS{c$Hi_mpi5xP&yLoaAq=riDbEerWZ%Y%Ofob+XdQi{*l z!ruUQnFr(06Ga9@`@m1&5)eQX5Y%7p!6W1nG7&{*rX0cyWf!(owqYw|9d0A5aA#RY zc*_EHW<^A_mPMw@JhD(`QFStnTBD^=d%$@uiF!&)BHsgFYe~c}S{lJ==kRZbza5)v zu(=3ck~|a(H^o@;4`#m|N;DeYTMm&A$}VcGtfQyNGG?wUVisvdjG30jTFN}uL1wWY zGKmY&(zr-1iBHy|_&gcMb1Fo9hYS@TE=Epw)-qnJ*ue2ccXBo!+!Aae$4cOj- zzboL)!{(GAqCmKV!mvMrC?pz1A{Ip=et=daj+A-QM42YdL|HM^lH{dYoV-HDDYjae z;v%CIUoA)t)BMx~%}veL93Jb>N?WCwX*)C{?UbgIB{MzsZOuse68xl@$-iqa{A;nj z9_|vjvqDg1;0}e`4{mq3?GuPblTifHdTVjUKeRA&4C=}hEy$Xqd07iJH``RRvsY?X zwu5HocxZY~kf!CtXbPwKPNKwQVs4AZ=Wfxs+~XRX`-sNoys2^7pMxJXKI^wTSyHR< zIYNf=!?_nCQDxu`ilYzUc7od~ji=FBL}R%`#0C8{vv3#+%Q#Ifnx-j5b2X`WktP;BT~8eH-(-Cz8P{!#Ru9xVFh zPKHt=c9+79u%tw|BjM(-synKX4cyD%UQ`Hw2@z?zP+10OboFqJtRAo7)zdVrW}b%B zEYd%i;0M>(YEX^42Gj>bg3r_hHwWHuG7-uY+>r#Te%S3? z#26^!WYj9EQflFE#C|R_bQP5en54CR!2@6vll(+5gG~<`fb~qg>zQ=dJ2UC{0|wNF z9Do5E)`0{3+hyYIqAXn3J$O4sMbF8qz2-K2sEI!lN6w)up zX-7}0>Ip9t6FU4P4nD=jPjSIh{PL*_;8E}lZv?{!aK~0LAH(fJQ&(d1qV;6OVDC71 z|FN5_h#t;E;21awFy)Rya*ogQz1W2GN3|Hj#&k4l$rLu}^Uw@v2PeqhDV4lIpWMLM zn~i`A-#h|n$F1wUnTvTDZVsxK7u+@*&>yy;iec{rc!wP3?X<`5ah~V%61W1cg6rTW zz^w#kY)om(?Ez>M!%;{klEEF*C~oi)~A?Ez68DGskuuJF%VEVQ`qCVWu|y0u3i`+BDQ8 zO`4?5{~W1)n{RyM9~m?E`s&_2*IFCSUi0j|j~2t-3pbZGmBF2Pj-5f*p-SPeGrn5= zmiIA(2Rs9w1uuY?z$@T2@CWcG@U|t&3+?p->gJPBZtJF!NR_fuq@5E)M%g*D0bK+; zzNC`BG_p=(H-@1<%mBuYj6g7bO2{wdcR!*(d{3_XclD4H521gIq6v(Xo2?{!T}O_% zNgr#gJ|xHdfc)-VGRb#nv~QB*y+MZfI$8KDWN$@B@K(>qg)-6@ywD`ro_*q!18 z{Lo$^Q7Tg5HNjf}XCr#cJh=McT7VWY2G26|h*j{c!~IQ!VH+CDZZv`e2&Y3$CR`?xagS&bvPnna}9j{~&kgbPQmu z&bVa5ci8Zm%;BDmuU=Rk3TFa-aWW@b%>AqmxraKri@g61+QvofxRrXj1v_r0ecXs0 zoHu|zFbUUAcn-jEf>v=(XZY5YlyZs)UO{c1L=WP*%l!4%DHt~VG{rxC5YKb6A*L?G z(jU%9JV=3$T_M!TO=vgA!2b4df`QMshiQISPzO{>G))Vs|=rN zDYXfn4tToZ>BkqI5K25Qqh5}pgK!Q5En){;2jMu0HP_N>?j$--Qd4izYCfkY{R8}h zwZ9S-x-l@i`-9*Pa5Me&I##G$!^u8p(PFNoPOeZUd8f~(^LmRE1(0ZGpr-sQq78h;(LjN&qxtb;b$+CQU6duLQ=w_Tlfv1@3@%noh0 z@6lHKL2a4ElQCzHYttM~rgYe>b&h+q*6Em5J72ApF1Kri%M+S#eOt@izRGCwr8dp2 z*T%UmT0eK5*17j;jr*`xdyHwN=L#+N+@J|Bc4c@U(GnkaW%%BvQQyZj;`fFY`F*a1 z{yzfr2k@31!#JNKe zec^ur|4?r@yCJ|U_iM>(Xu@9*; z_EmMp{g39weGmSvdFU|D;{RRnpNIEyE^`ODc{Iw{=GKLR;wah(RcjOb|SPm zIzgk+=~@(%r-d=a8jfK{1W&sjh;30{T&H^D`qdLZqV9wh>PpJPk7UysGke z8f4xJs>u5Yup1%wXTbG~%lq~Ce-gg^OgFZsp}&9?fYe1Jybh(?sz1Y3^D}+Xl0wyy z6|1(aWVK{xsVO^OjhtrFki%|-+-BA0b*d(BP}Qczs?1-bih}JbFF2~Q!fRAoc(+Ol zpI33=hu|9(7qAvs%s-mz%Wci-~O(3fVZxzHW`$X|6u z;i@iX&nkx}OJ%cC zC#&*(@RhPF{t157McTks#QPBZ+w$lK`NRM$g1f%}{vvXo67uiT8LBIDMMv^cMLAET zD34Z2MUsjuGF4b@@~di;hZd7t-J_hEh03m3uFTp=Ww3-iz3wWd)!nXCR_dhI zy{FXLFO^pFBVctWb#RK9?}L95PHyP48i~A@*KWAk2&Bde@~%p2^dv`>qQw-~1gfw$ zQu(zB%BxFLZe6aj>x-3DU!_d6n2d%lr8N#Km1HTUX`_-JRj27T#j{c;o|QTY zjbDHt0J(n#<1#Vd4gUtXCzyIJDrcSx+AGOBt62?D%Y3)aTtyA`%14XIYw}Z0bEvYK zW0ldItn`*FrM46(rL{uIt@TPoi%Do3P<-1`#kQ|kO#2>1x1Ul}`z?xWe_RpmZ!5Cx zZ;ERBhoV~l#naDd3-E8p=QZ#yfp@5iI)J;W4*wg-y_>k_*n{4*gM%2tN1KQwMO@!Cldc{?X2z*iLf4ZuE!wvlQPuSFyc* z=qRCz?2A!Ef3m{*vlQB2sE~n51r0PQaG+cMgNx)lxJo{Q+vPoYT%Lp1%Y#)i9)o|B z=fGd&+5erq`hVe+Zt4Nw*VoV%;T~?J9W)aIxU1nVfjhSw{h^mrDF?{Ehiw$PzzGe- zQ$Y&@6tE~uj9@`?PF;2!DyP!Fv4{Fxv zYnn6inH&~*X6{wS2!UmYj8TsD|c4e$K zm;KtAvRmgW+jZWuUKcE@by2cdmn8Fb*@$~I?sYiEjv=jI2v&ie;3T+(zda7#U?BJu zd^L4a(Q3Xr)`cF5-<@zb;J2Xx<&u`Az#TI|{!205YvJF(UHM6J;H@?U&;iV4VDSYZ zAO@rWI{G#m_;#Gz-Ur6PW^fc-%ikUZFM;>@{a-jCn;6VTi|e0W9n=bU6@KS0r41}6 z|3@|mS)aRMG(Z2XLyE;Mik-UDgV!t+5I`9XqvkXRgC z!$7zfT*do4!PDSRe3$!M_`GS3-AKVAQ5pWqe zh8@ST<2ZJlaAyMI$HfmjPR21I%V6*;0u2BsPZ5h#ThUC8fg5=L5O@K+J$*{g7%{-_ zX1L4Y&fQ2G#OE-o&VOeiH@WL@aK)$iE*0FY-h%VgPpsF|Wbz0!&TWg$B2ulVS1K8D5)H zaPcz|nrZC?H%NG<^^jY6y$xKnU}8RlCE1wfH2A1WlA!68BKPx3^CB@K zIRKA>$E?sqW^u!ox_KrLErfb`F&CYpmL<)-XcZIaB2?~c#{q@EcB_Vnc?aAa5GNDd z92}?+V(fl__ID#?8a^7l_X2~D@R5lAR||QL_s@eD!OP%P@H+UDEy){o^IiaJLgKlZ zp3e>KMt02%k=U+66WNI-auiL(&_FKf4~79USZ(}{iE;NY)NHLd@v!BMC3uqs=lea&LR`5Exqg)-6@ywUmGm=bT%2F7rdeHCc>KyZwbGwg|AiblVARseETi({x{GcUdN7CvEyaz zco93E$Bt*oE}td~e3H!2SZ(ka4d-F9c4Lps1LWNIQSzl7Dfhr=;J-F-m@InG-_e6U zM-O7XA^xSona}i5c$zgBT;<`5CU&X#lZPjz=m9n89?kgDfiFF9^rNpVz|&E9#^K>9 zA=q(_jQ$MT)Rm|Jr}*y*zI8cKKTgTVrc|>p(FlGbQvY3p@fFW=oG4Me|q&*BYWQ3D<6FDUG*;b+lK22odXc|=q~&H`Yr_0T@b4)N<|%mZ0@s@v z+Q;-I;RL$WQS{0~)X9GO>K=5{UG$S3w2f_M#oB6Ksm&I3nzZ1_7?vH{VA-Q}Rs&jV zHKNtl%e2adlPPVsYQpw_md&_AOYLsZ;+ZIUGhfoE{U;ik&6&uve+I^BLJYO$4E+pi z=j^%kap^h2>j671$yT=7ZRx7*HvZaT8=*@}((PH-9U*{D9B-Rku{t{%S&>h`}!UH;FjGvGtb3t(Sh zz%S|u;K|nT-^Q?hmP_3+AAI*?ssko{nRaDo8JGCmc;D9!)1rBYvF7 z;K$P-{mV7vU!#G5CiMq&s5h`z^Me+sJ7`>8!R&|#*{Sx>%hVQjjatI)QgisTY6^cJ zd<}jAoVDt8oTKql{OdW(z>o4mf$u+=u1oO-=@~&_@4TrmFAi__*kzwkIic?nu__m?kyG&Qk+gOg-yjY75M9pZdY0IQ{c}kP5uJ> zsInyTZsKjM1-cCWUD3?3W0~iJaWERgT$|4ZVyx90>!j{DZ#1M}wZ})Pm8V=cC#0(} zAy@SYMXF1zP;FAZs*~DPmE5b!ltn5}U7@nH%_>PdpyKqaRFr<33NxNiLB>1aZz@Rt z0q~S>>fj8%9)fEd(~Wfr#6J=Ld0hbdK~JKEI+JFjDY>H`1*kDOT=glOhLDn?>eMV% zrsk_6txV-C#iW+DE2XSoNo9+bSk8$I?3#+NxLk1+DDV}W$WZYI#g_jCd6h#LrE;0SNlx<*%5nZV&6$6XX3u|7b9z3O15X)u?EX;);h)6! z3H)9NcMm?dl8)5EUDC+<3%E1jPV9ici~N7SnSA^0!FPJkQv{88`+2I5b_mppBaF@diyBHr_x(4kuB3$3&R zex3t261ZaF4qZV0Il``x#oU7*XAQz~c7?2P=Rub<; z@b-2xCgOJ;+{JKb@w0?6?*HSnAEBJPnq4XDxd)FsOx?M-nA4bT!E7%0t_*TsAb<;B z1R`4kps^bjy_<^NO`*mXm_1v;ac~p=eiXa{-s6O7#-#bQ#Xe#{%*)}^Nith{s`P20P+$6gw`%j$?^vBRPP^cANkor{Lpj0Y5x( zHLv#oVsZR+jlerFL=5;!4crBAr>&w7;Isc`_-RTr;WgV2C)G*fFs+4L&g&K66gX{( zE;0j+g7$I7ok^J=6M}FwktAROG`RCUfS{hIzzh8N0&Vlc1K=5+2u2LxZsgCUaA(3D zi_d{Oi9i0@UE(!3PlI#d8gKzz2d)P<0E*Pj*mBEECWo|<+kHq@u;Z>|Cagt(MtAox z+6C?6UJAbNN^mn^_c4CA6Z0y#P57M%cPKu49KnBsm#7ZK1Kb4&Daj{V z3&5>uJ%svsjF>#`jTV9(Pp6}Ylyf7ugT>@H{QM+I-C;C~GvEfEa*h@VcQM@Qa7P|x z3^>Ue9R6&6o-!}-8k`2-gWzHCD0mz^0iFWSfM>z;;6?C?1?xh7$66CF63GbZ{VZ-M zSCiDy_&&z*Pqw0o5adsdC&gUSK#bEdjQub?A(cGy7jkayaF7d@kd+yGQCi8#y3iu} z(FGQet&O23EYtgBPVZ_h>jgHE*&8Q??%}@iA#&#vm~s}a;AYHwfZY8B%>9In^9OR; z|G#}GhAnnv!*k&cLjQ;%7f(lb$>*09a5ccwjuz2__Avm*LNfU=N?r!fNHJc#aaTQ-tk0s_8DO>q+wWcX{_U)yTR{@PD5UIvtDeWre@a_~{F01Uf_#dP@#` z#e}#Tf0`(%17CWmiUD%e1ysLrLdG~eE8$s3bxp!!oRD#VW^cv{wQI-^1O z;!ikSiRciScv65b<#feb{AkvB?6`(}`)b<9S@QlfoT_^jcAO?2r^vQXl7U}Nog7Cc zxQtjBXNDZcqk|}B2WVOQiN&Qg7(c-KDR>$D9^6YTE)t8ISc7prvABR1b2WMYS=#4S z=nto98?4xdgMA>hkK?q{W9Sc;VaE~dIE)Zb}uELJ@;CvDo z!;LW%7{efEaKk$jPB-}c;EN!t3Gk%DlLt>RJQeWN!qNm!8$F^6u0FUHz{S%!sgqTF zx{_ZlCrX@SLWJ&Mm)n#4^(~Z=ztWoj0e+$VO%HE||7XDoa0DEnkL+c<*hTx?jy|>( z-E@++u@P-*18sC2cC5vYHQ2EVJ62LJD`=+^;$%wdWGU}BCqrXqJ2hf{REx~dYJtT? z4O={+LCd!^VD(oGSbqn8rVUOHyVsbL>sWIswB^!h!{ykD`M#MAyiV|X!5N}8<}q4j zk*pP#S(>mkX_-}tmReV8iFKV8+q7uZW}ZfDd$n-JLM^Zx*U-$h8nEB4{#i%VJNuk^ z=G?Asho{u#@Mm>8exZ3zKY)MH2WSKEUCE{9Fhl%qXQl^WqZ4zyS!e*ZoCayj`Ug8d zjn53z68kuf%}&+moNO&}D9}Pj_Fy=&2g9jB1I}&gckWTI%aD3p7pvQCwL0f+Rma>z zYIDCztsb|j#p7``dGJgo&%c2m)#%ARclfV`>x2u_0(X4(WO@WxHmv3Rdzp*5mO9VT zn2U!Nxdv*1TO^uJf(G2v)aRa~UXMccc$BN#vqqhs&6?*mPwn3QYV{sfi_Z!*`EFK& z-+tBmpH^+aEvgAXZ4LO7ssjHC{sI22JMjM;{KoWSr!W5d!yf=QuZ#INhYo6?*9;AL zxvJmWSG_)=>ha|?M&BgO^UF}XU!L0hi`C*^spbIoKLoP>A#lFxf)=VKctX`7n^YOP zR~2Dbs4VOzm4-i}lJGx(zW}aA;s4St#Qd}eZP1Sx1WoHN6QN9dL*Nc(o*78q6=0?L z0gmbl^i)SspxS~X)Dj%8rjS%Mgs}f1v_Q3?WvXFqOjUTZD$!!fBZgHLIj)i@o(37c zM};x0u8O%)`LPen6#E+Z416b3%rCmZjoVN7yBGdVVN7|s%r1$9pVt8{{k?qF6=tT6 zu$gK_i)o7RReeN=Y9l!fAu>@_k?E?4%2ioZu}Y(>R1(vuBD9#oxIq=fFOi8SE9LQ2 zrJTeQ%1*pqSxFBnGwD_EKgvw}7W}MhsRIts(@uO}5BEe2@dtxoel+<;6!U#vo1$&e zl$_C$yi^$*sPfndmBz)XI4(s+@mcT{C_kZGro?*XqQ&GS_bWSjOqnTbl##ka>8Xrz zY1b(=?E$5vy#hW3-zp{bXI(=Kju7{4@UMY)N&K|_(whK3)AUx*$Y-?)ma0troyrpD zswBxzg-M~xPl{GvQlfG>SvWhzq^#6ZWunDoq_rwNy;rFjBTC6!rR2=*O3J!S30W5u zpLM_DvR($C@c!?L&-_VesPBWsd^7wj;TlaM2Fc|9fCEw02x?N8Z>O56B-KuZX)ZFQ zdn+eBNLlHT%FKvYdPb_!GINxQ7L$@yrKId;CFb-fA$O7Db5|-hZ>wVRjw;%8UQwp| z6lr=9e5|OvZ@@owni%XO-W%bcfNx^pOP(QIuke5*1yXuE=7OB1*~>UQ(}6lEsj+K?RpBQ(*a|0?H4{zx%|GViXoQfTdL1=qSMu+CQj zb;0thkCIP)qP*)f<;f{B9u1Y6%Ly`WY$9=K8k2L=Iyp7%l>=*H<}}@|*-cM__vFBu z7>9;`a5^@9l9(^S??Jdb;cdj{Dts=2I|uHxD*Ue@_o_#KXq5a~X2`qMNuI49a&PsQ zTU(f1+hXP1mMW+ATsgFtYIb|Q?Agp>M*?Ttu}U@_J7wK*xvV;FmSqQbi`xH>tlIxa z)@|Qt2mGt>eFW}ad~Socu8ekAK@8x|fE)Q$(M{w&E%3K<|CRkgPTe!*(Bq<6J>Htx z6C}HyNZHO$l=b{fS}pFJ4;tN|FZmF2L5%!cP8Vll`JGsq3o!H4O{ z#^SDF95EKQEa(AaU;{V+&VhUQ+wub@W;Le9T9qxFz8U7W} z3qLM74Z}Y|4mM`ZMa>?#P?4S>fR4pLvy@K0l-zD94azuvXek94k3bq<1$OcN6u1bU z;x})BPdObMQ{d(B9Zo%BJcPR#?ks+q2zNvu{6p|B#Q#z5!7m{PpAh@(YsbMBa17kY-yQ|8fe$nWZy&rJ)O91=<#6Z0 z&6ZR}!5xUt9*g0pN^Mray#}lY8^I<^fE|-4=y5k__?yPHv~r?KrW0 zE!YpP<~#QR>SNp6Jn4*P!B>zIRReb+-05(~z#X)dJs`{3gRvU@$Kc%rwt^jCCm^)i zV*zZr@cs_r-IYlIb{xbG<3_;YG*Ac#@?m0em;fH;$A?dXTfyUe?_~|Z+tou1;4Xzb z3+{NhL-5%Xv*&DL4~W5QaP9#I!C`O|Tm}g3)I|7lD<%PU=ps%`NIaPw2BL>x$Jq=} z3JCI9VsRE{&K?360B!T^zxEb6D44(VJ zgWzHCDEK{i0z3tt0ndRK!7DZ-akP;)gIE)i!YL)C=ph~4+#f>|*@Pwn*@u^t?OoD9 zj8|hn%v{dy4CGF{8PBi_gC?=k;Z^n(B5w>JEnJo5)~?!S|Be?fj{ zoDljy=no&F#k_<5@CNzRtK{1+VaE&D@f>;GGuZJIc056r^?U4il&t7sI?jV+qxX{$ z-$SFh3l-uH{&JDb{x0DPV;`8f4xh0{+e#h(@DTl@C&?u1MdK%DLoAFy zc@nc=xOg%M`one9$$9ertFhxOcATMJuB3gOCf~k7y>JYX;g7(J$0SX|CNsbgp{N9ZGmSPOE1yni2g|6b+6QH=c+aMaKX8sXt|OzhZ< z9h20{M%w8H>SP`NH!LxB$E+lBEAViF`k0_6Ekl18r(KNGK3|5@=SF4V?zcC~p1iQg@VzC7+hG(?F=ZjCFlo$(7GCY~^ znBXafrvkPbxEkSVhpPuJPNpOloQgs1Owx<@V%bSr@AaBL)uUdbHGd4g#_AvWWO^86 z)SS8%u+xIeFxX(md><`_Gs-B5r%|vzXB?ew32RcuXrrU_l||UG5IYuN$1v@5(5zVl zX7ki%-ltxRMe4DbP?zOKbz1FFhcybC4SO(b*n?ri9t>Na2r+{-5Ho(98h(u!TxN~` z4Dpk*m|}nxGs$_F8jM@oXo^P|q(<8fg6HxvC~ri=RReb975&~&2Fa}v>W($(RVt9GYCwQ^=l zvvaMQTw2uN+NC#b;qvzZ#e z$paR(!V7+;A&Yn&@+9Zs65Q)*sUDX(>U4Eihnv6J=7y1ctN>=AAxVcPdZQhkK+G! z_}2Oodw=)?$TdK(AM+bNn}Zj(j994j$Q4S9+M<+b&OV4fr=;k+lo|;NglhucMPbB0 z0{=lIS;4`HCH)WOi7rlio(5>A0DK<@CfBZ#3?%>RaueQ%7`jddUU1I&|*?z zx|AF{tfaUJCB#oEKK`KMI6gln;Z8*-JOke6{g;Y~|4}E2!EWNe4t^s&7>=6OU)rPL z=Y1WvR7Nt_Uajbm+b%3Kk zRKjZk)3MxmD`h3jQhEYU#7y*2Qev!v?1!uP^D5p;WIg90= zyH0+&d*zdNTHblL$t&+kdF8$%@7%x1C-)yyslX;;y$t^01oRL1TA2FPg7Q@S2ifW5 z-WklfGnsQ{St~l*L6JFg6`t#>(A*G(Xhh)*aphunsqw*+R zBlp7Hnp=2<+=_0IYvJQ^DSV69&*fI|z4p=u*Tpa`D3y#dqcCz{-WQNey1I@J&usj2!8j%-3n)I*0csxKw6NQ$9-qG z;|kC}iqIdJ8vB*o$g9FZ9+h+DUg;~h$`H9!Ma#J=S&miNnp0h@S#08&S=}MKnjzV; zKgp(MtE_4+lV#1dvZ#5G*H>g&{jsdjVzyBK6Yvk?a~IrA@K(WF407{{K_U7_F?k2v zA?5H_k$=`&$fe#+jt$P5-QX$v#sJwhhHC~Qq76Z`YRs2KW2MZSnkldsECCzAL2xy= z2Ru)}KLDR|GB)*q@BMJM!&?V$IX)M_jnv2bB680Pa<6Lihg$N#2G-zH^vpIJS+~!T zMTeWrJA6o*8PGZiY6lhCLFILnfO^nH1&@HWU@y3m&+Y(EgFl1MvA4({+e^8ZHizZTZux3dPIlN_want{Zg3UmgZfDYD6|MJX^YgeVRk(}R}k+)xanR>ggdf{{J$0c4)}42%d)Hn z#DH!|1VTN$2E1`55Jb|r=+Gm?g8e}n!T1q=G(vrhOoF3)=Q{8(c$MFMs1dk( z;BBcS_s8rqxO3q)G@xj>gXggarU(8$_;JT#A^f9Yu^E*}ESApXLgz&1o(p_vT-3>O z>STE;FaZi&jswg2;qtX$KR5^O;kz$@cQpv_yc(`a)O8iy1(3-O>SYJ@ zvXgq*O}*^Fj(wqMBJqGa*-z!}Cl>o~WIqM$I}9#>`@wVSgtw)g7{G1np$_1VSVaDh z&(6!ypDytloIAj7un!Q@sR{5QUMW&X&88=W$LyILyP${AK2BlBY1+nV6ClW^dI5!= zqL5Rk0kJsssOEJM1N<%~<{9`M4Yxl{*kv91li@F+o8mP%kAUNVrcKgAx!@{r2Al&| zgY(#Ott}H}ClUrPbPDV+&L_F07~t?NIC#rOfZ;cv2X{`L)La2Kmlh>pVi4SJ_&j4L z*V4yBpkRg%(2HJVHDkivXE`#`jncfatMZXcs5J zb*h8AkfxHlmT>^T*_y7IhR;U~$GzZw@E~{?JPLjfo&-;W=fI2R z=peH>iv&B~hyf+H&>y}>fB1@=`>)*B{U7p~kI)~Cy(oVquX}|&m*>^N(E&#f zF87l?4s*YL6rQEkb}6e|VUD>H+d8o}Gsu9pp#ds0w{#vcqtUz_ApL z6>u2mVQi(@>?7MgMpM0-rgR6@`V{%}hcxByvFz9Xg<-=#z-Qo1@VqREnj_Wh1y?Y> z#PaJ@cyi$=rWsaZdmZH)>oD4BqFpfb!ZAem7=dFc94l$U8)!~DsEtEdaf;@36W!_| z{`Cf3`U}9*N&W+a{}b>EFq)Ln)NUgdHxY~LXcy09swQ9=|PiEw1Vk%#@oM7e^}YpC@GI9kyo=D{%^jv+WkX%Q2& z&UI)Jm-c~NMc#fZmOaieZ#?_>Yw$1dD{XpuY1b{h8bgdR3>*W8h{b-i7-pjIguoF) zX-RPKWKitbj4rf^cFJjfM7;qmf+vI$k2S<&HSJ?18TxYACivz!7B9u4CA5?=yfu2# zDC*k?tzZNVcV%b3K~SI@-os`s-@!Scx4guwyxP zEW?g*>{v=Xme4-N=r5y`z6c8zV&O268p5*y>Z709?j!QXi95Z-q8AP1KKP9I8^h6o zDemdZk#T8Si!QZ_H5eDg`QbFHXxt@IHi24{dVBr`}oRMG^4?xKh<& zmZdhce6^aFsKug6O%@Gmv}{+sRiEmtM^t08QdPEFR5|02%4eKasofnav3p9zc7Im! z%+JAhQ&98=E!u5OWrWA{S5WJGnqEpSgYI00ZqqU?biP2Iict|F>10+ zRKtvP)z9FG5O#&CnOUxC`#M$5YE${_UX?j4Qi&t2#%YTRoDV7A<*f2t?oh7F)5>vq zS2?a6EMrr%aVOn^fXbqGFe76}mPn->qAwxeJu*KA~)nNo9E+P=@DKO82^`RIewM z;`I)%e*-`03gW%zcTA7qU*{S2JERri>Yy)ttz*D*uy z&y7-%dx8qwQ3$}q`IjjLEhagzU5P;hN(e^D3*qd8kiCiy6so(1stgc5@==69e9l=AOFY{?5@z8h+;j8HqJhB_-b%u@+r{)!6=Rcv^S zVj_|i6`7^T$U;R#RVh5WS)noXqS#Rd#ja6c+)nw&T`s@)8|54SsC?r7Am6ypls`9V{gM{RCux2R&4M>2o%q8WkwxyAgZ_|*{!m~hmm*s^lsIa3iM#Ae{A5=WD%+A6*_5Qnsx(&? zXfft!F+4g4jDmGwFSrui4xS>|Z-dXYhWa0YzXzY2;jMzR2!C@yI@}yhB$k0G)I{!6 zfc{WS{!_*tgi0&fRLw$5ag_xFh*>p_x|&8?jU(0ku)2s!tpOci0ayukf|KAD@HpT3 z6JYHD{6qL|tVLoui^|~4&n5==O!J!3pu!5#A4nKadj!1B??2`VXIvZbxdmPx4mG6#mEMjk}XZ)rm7*1g)zcbb~Rl z5gg)g7r=x3=4HU?f%x7DcLSV;22==g;7*1+8t&j~^6xtGzef05xd%;Xxo}f?=Eene z`oh{rEc)jnruZ<(FrXO?dr zQh4)0I^2c^6ow1F`0Un3{dc0j^uXN@h5$usp#_~9J4Wo$MX+OxIx!YqE)4~AvZXk< zl!BMyz|vl@j1I6J&?c7L%6FgOH*azlHB~}ASHfFN^|AF(Ed5gi+X&bAsV-0q!^<%9_I7kEpc^$D>HxG=04d4jhxsLBX1YS}r zf3Ahs*binR<|%MT!5skiTzt02Uvqvs#k-u>)nF}H4=7HXz$Dmg$;EF5lOabg#MrUR zpGiRkNMVv(2pRwd?pgtMgDaUl-T|Ia6F%4Q=VG`s;EshmaDX*X_-w!UKkIJ{UW3!% z+6H!jU4WvP{7-8H2Y5Yb#>AAE9HpHeqkWvfj+4<$*fT&mpfQ~wkS8{Sqi71(gZtD- z*QtP;ZJ$cO%n-OemcWm>Oy;NkGaD^mvdX2`dyj}!%g1fR90t_Cia*F3(UhfAFf=9q(;0f?Fcn-Wo-Mmgb z{ul}}01fbu^8tJR|Td zg=Zx^>&axck-zUFY{%g_PabnO`P}n_(>M$C7jlZ}vp|it7?*7L9{#_8_sBC}MSpmn zyy_`p@i;l=qpUl45dGmk^6k6PAMU`8i`Zd2m*f^Qo}1JM#}G|pgeI~S4&ywGjWo|4 zG}A*g*DGmCw@|H*k<-3QF8>{t{rbQ7H@x3~55ddeaqxib&_3Mo#UGvsew_f1aTbP& z@=GYclJaY5CdN99HaI%r`0YB3ab3nha+EqbOy@la+kPUjk4WtyqPvOoZu0G2*kRyn zIR6Zu2KRy6!1c0%%aIJ#3qOM3h=L;tKQbxJMES*(UqSgi4}?teAerlaGQWLff_sR^ zZraCAvgGZwjcxpY3*VZ=;!Tvk5vC2OTt5N0q1Z6!+)bYpP%BMmSXqweiyOWfflosHA0)IlSwq8jkJvo*s&fv)+!GU zo({@-A+lO}L_It_A(VO8tSHP60aEyybBWCi*IE2Z%zj*?TbKNF5TE@6f6l%YdZ!BY=UD;!;v zJxGrk! zj-m-ILZ@3ue_enbL)bBh9sTr|KI|aNhO3%h(g;^O9KDp!Ih0u4N-Z^0A5GM9BkEoQ z{iB`{rjDLiM<2NsE@K!rEwRef@%#t#T`DlAS6N)IJc|dEYyOgQ%s&EOPF?c2f^oJAQ<51>i)J!aVp_z~WzDl= z+GxW)7E3ENo7q#c3w&Pi1)$@Esmd%y73PU5w@6c&MUF}=3sr1Yu0rd2`x8waT*FrHq*;ls5AQrP@EFl$oz6+5Thj)zl?;JN&ER8=HgwPVl=hU3F&q%ligC ztHY*RTWeKY&sL?4o62l>_JeJZifuW6VMeS9?2=`&%T(S>lXC1!l|8FYnRD8d?%1bP zCsyz{uT_%sE+x2}P`nFgAGkcE7?)Q8r!%>Hqr+zO1N>j^Kpk+I?PcoG!8FL2uGjLu ziq9&rrwlEoWR9(hW;?22j=M|_e#&(WQMO~0vYZl>;hd&4=Uk<_6f4=aT1jp#N|-xe zaqc6E^;oTFPtHE@~_>lNnpph7)g0iP($^Be8Aq8?_`2k_m{fV#Yx=X>KjsN{XA zCpizF7jXGCxmltg*(=MwxCN*BU?_1d?ljLO?pJEIv!a zo@9UKd;V66^|x2Fzl$OQJQWrYpwPfDg#^VaC^$udB&7i%CGrccl`jkAy~Bs(6)_=? zh|O}3I4rlwYvdYvuUsNt03XUVg8lx)V7UicApBi$H3buc5biev6UYLofF&P_15qJn z3J;l~&`?JOg}N&s%vb(lA@U24mQO^IywPGjql)AaRjs+voWBq=AeWeNImb@QG4_xg zVz1WkV(-zc*yq9f`d!Qy+C&V-se?iIjoaq+p~N7J`VS}mAPpdKp!YD}j3D=oB>#`H zmVfjt`9!EsJ%%WWS6vGwn=AX zlXQoylb(`I;yd88seN&aiE%Idt#H@CQvy$36#5HD262GxSPG71&KXDU7jG`l1fGGB z>3K$tgKh4#}~aosuehl4ZNpGR;VBlucT3d{uJlZjhD@5Ak z;7>A3{gh{V(H4Rj6W}*=s7kmBW5~BaDx3)*8s5+Z@{UCKlbHLbkpHJ!%OPX7W@Wf( zW~Ps3WCqC=qpdkJ#xjdDAXpb;mR(Ch^S}av%2gnTD^kwo{QD;GD0rR9{#eUtb3^cV z;As4c#K9d2cTh69XDa$bI{caBzd4q&HO-Wj$ypXAPnnqrrip?~6p)_= z@4Js6Ff2QSLS>*q@ zb5$8-esaD0|P6XV6+2lTX7rxLB8G&cMY7y#Cb{s;^)RXq$s$9;r1zj zzZm{9_$%SB5go`J!>vi$xwthEi)Od!0k)OK)rMoO6xzy2EBWr7 z;8}k6p626s6WkSW7QmSWQVC-$KM#Z3uY~-&9R6z70MwC#6Iw3b7!EotxwsOGPJ0Fy zCj#m|J<#=r($NU6A@Iz{`1uW>7mNei#QYOzD7WzaC&8PXMNMenEP*o*WK!3Oa7Vx$ zfD7*U>_8E=&Hv#gtV}8>AM}F(>SWM{K@K|>IMc~IriJfOI{9c4$m8EtpaYEX_YHtH zvG6+ZFu!?iO1i3qv#<~wKnmQ^a0kKdfzJ+v(T47&9$;_|frWtLnBbwMfF;;5jvdRe zV>x!Lq-_{`b5=(Kg1m}YOr4Q3!0Sq|2i@chxSQ`kqgptP=359d;Esbk1a7Z3_`A@5 z`rt5l7XgEB8CVV|YDy1T!z)c~N)Op+g+^h=q|6DO0y}mFF-fIPc2L3FD*?`IqmXS| z&`XYk8~N@ds^U+jAQ$dLxWnM~?w;O*Wwn5^$AH0Wa2i~jz-F)&Yy&&ME`V!HTUj4s zjut{2IYK)MR~#;_XDNt(H*U z3a}Pz0tVl1z)z>NkVCv40ha+>nz}i91+S;URp1=Y{4GP6Cz2`!X~OVsU}O z&r{%qW8l2X;LfI<#S!yB{C4HfHipmZ0U`Te9potQPk>Y4N^k~T4bFpW!S#U9lAQim z2f4?R4%sNv6KTh`9%T&oH=+I2%42zGCXs z#P=MpH0A$?=Waf`7d!wS0*`{n!BgNl@RAuC$Q;f{^+Wea0LADYbnMq@(64Rcz(wQk z|8F&r>HRQ2lhgdb-1%GbnQzGL{>EAmo&-m3UP8B99I19tD;a}hz@G*D;JjeazC%M=D7_oSW9PdrI82mO zf-jZm5p{4h!_f{$7wSSE8SM~y!6+QdXfkVPI$MasKAO=|e9AAO{ONTVmy`KiT8D8NgXaOD!uL4F*;yB&BuO`pDl31|% zl`iKCM;JcDlD!*eab{7ziSmmnzno!VdL712+Q<&t$TsZQO1*4mK-)yy*obPdo>;7< zwAFaEirQSs&Xbk2xE1glcnRM7zzu*AW2&1Q-TnZvU?;g-UVONn*Kc{zaE2c7uW!dA#y2T1cApKUkv@h5e|1a{NM<|mq-}m;7Nuj z1D+guX#qTC=pEH?HSr(kQ1YvKtgpqZ8tS8p-cd;luORvrtiUQ~y;eEx!Wee7x`{<6?QQ>Ty=1;1_EnK zshJW=E}|tB672#;iF`&B6aB-4K4&5p#^uJS1LM-fr2@aQZc6ri|se zgLczX!{*scfkBl$bM2YT^K6;#T3f2f!cGNdPBNLfqviN2+dN2F7U9aUh*i2}lG3a) zm11R5vUQmfZR(X^J5O;l*#BU+Oi?o@6=8o!;r8bgYJZnP?4MEa%y$%Q|JhVMtj5<- z_)WDS1YSlI)_CV3(ly zndyqPXaB=&_CL(2SA^p{g*go=#Ce&5ohKFOa!CFz=j7*dmwa5F0q=s(Iro=-h@T6% zymtM6?7eqbR9DwFz7@MBj}^tHFyP3$O5s)`^Wy~Fh0VHi3L zb!HeChAO=Y*n5q}oZo$h%SF+2HqP;WBtAMcWn23nI32aO@K22Ua%495E>hhY7~(2vRV zVN1zW>osJ`@Qq~B2w1Sqelp(X6d60xm5d&Fm5dt2BsQbs$%v6UGJIq`j(3P#PfSNq z#{-b@17vO*iFy7g)PFSOWHai zGIC=k*~}#qMlK@bkry*|^jBoenD5A_v3rRP5`;&Lb0*f~{m9Vq3^HVV92q<=p9~sT zha;Z3_r+8d`rnK4w?W>g7mkiL`g*7Qm3mNjWVL)W`d*KZ;3%kl6zC!p9X<(-GaAag_9(VNZI`@FKlt zVE=^~QeuU@YOJQ0k{;8qlXK9)VH@-Z)O8ceUpEo1@FcXsWVAgn2Y4T!rvsDm_qZvT z_fLiIGp#qVp8gLqboy{I80(Yn!Ou>mn90U{szZf>P6gViewjcWS{Fpy%%)e4`mKtl%tnLo9|M7DGPT2J8b)qd*S; zljDyBKq^p#bDGErw837q{dbU$T4g_Mn_& zg}6>%(sSNm(w#Qg`6y&QDli`>d^{gm3S(LaYyozoQW%97x&i_C%mpxG__z>gB0s|h zc0&Harg$<0sIU&0e%1*za;`joZmza;j$kfpN>PVhO8xYH6U*e z@IEjdmuz`!|bpa9n)_W<&@qP}0?vK7$rr;vvwU&uj1KLjSvMf~Ao*uX;gf2dkN zsJQ2Hf^xcnbWa#iUkv60p+p$g`cWtdmHGk|`T`ezu>?SSe6baPO?+Vsc;dV$oUa0O z7zi4$4>j%$ki8c5T#CyUK+him9{^J!8{M7QEQb7LkpCH;0YkmGX`w9OODhbjy`jYZ zxVa5R$AVpKM&oXQF1Dc1&A50oF4*!7j=O*p0PJ#87=+4z99&n6eW+0sE}sb5BbUSfTnYJW5r0_^xlk=83MdQM0$ptF(Q%XAjy9o#{Xg1Z(4ThGW6!Z2fqc-9tY3ioGX|lv4A98 zQ$)T(U00#mmY|(K!STc8XamR|gJv9xI``d#9DuEmL(8RQ?gmg4CU_ka9+(=YN{ETT zM(G6D6W~EV1h5^0n<3i77L9BRU7Xnf{De3H+T*kb&I!k)Q32$T&sV?}(auY7oCn!6 zAbT869|m3b-G;JpnMLk?9FG9M04N5N-Hx~f%*-+gw8zmAI1f8@fql4x$3@u2#ToFS zp==M>g*z(a2GVXg(bXU4@i2+S6Ktqy1+WmZu@x7Y2-()#A)l@@U0+;ckxR>j8aw3R z=mNL_sFEc%;fbRc05x~SLHZ)b0UlxF;Jd$%IK?7-MxzFy;6N1Sj|==G;K(40A9l6` zvgd5VScH1p;FNxJooW5j^*syF^`qro1TF)#9ADrn0Q0gqCLuV60g*s7!03kf1b7In z;k=>03Ss<0D4f3qF^fY03g!ACR>CBoLH2yu*$n7>H0u2hEt{^h9pDUH0K5UH4$j_x z$iZh8zy5PO95E2sM*0Fcw5W6nDw+%-iKtX!2;Q-dF?lz}fx{RB zQ19-vY+BEk03RR_fXXb65H0IZIWqjM1mXc&RuYg3WB^$}9$)~9fU-Vts%`LI-kFGj zEJX|i6)!!2#SZpJHlUyU|7Q%OC;1gV_cP3$A7TD}AHEEBU`Nzo1!*E{;kkbWZ*epH zR5yp%Q*X8GI@;@(v7`t>hEVuC>|m_{SY5rv2c9s8g&Vdyvn&EYk07ia)XKrVdl zOz0vNx=4agr9s|7Jp33XJO>#(3Nd&H!GjNvhX-$q1D}e8!Gj5JFcv(b;on8Vn8VPh zA?PeYkQRWm{ZXPHo^$&`%RZ3o4R``CAhYcc*_k-fO)DqeAg4dB8V++C4;j-@mk(hQ z3sC+Nn8#-*e=W-Y0%p1qb)hr*x1pPK?1$k49^U9wm*J_?85kGvi#yo4L0c}!a&bn~ z!x^$^H~}q?sRJ|sZPILj?pk5cMF@P|Ao!U6@Tssm8+Dii{tHn4VwC^yc^DqB5jXH~ z1rKNF!wCbLBW%MSKKD83!WJc-hW1XuU{AubPQdC;Kt7FJ$fLUv2cXTL?gn&sxC~uf zMEt=6@dr1wi7Vm{uCNVf@Nh!AIKV#a$#Rsx3R?dhGQNV0O_1>&Wb8m|?8neXuLL;< zHtz6a0}v-*VTez}FQu@ahX7?k9OTh;4*{+K^swg)H~{D1bK8=Cz`{nL{BbCM3d)}e z?sH&y^ga}yLdG)4SPAW8FHCs$2f>40d9fG$We>jDg{yWz7eAqN`fa-3Bhs}U7QP)Z zknOOV#<;@FnSiQu{*AAjlaGIipdh ziI6o7^}zcl!2=#YWG#X$>;Z|h7D4l$KwH?g6qa)wzdC`bH=?akh*rp9QMo_^aL;lZ zq;>EMum||QJEmzpFiiy3TVbBr4O3TqzY=}*GqlNa*v2y0=u-5RCE&3LJU&G{W+6&m zfPWw3mwC7XJ9MJ_tzfqcBizr3lsMq5%jn?|h%!n5Ek>PkfL^mk$KRj>$llQtJ|}K3 z8*r<`)`rOOz_h(L=DPTPu~m1->y5e%fV`oQWrMnngS^R*J_GV*A&NQ=|6gQ@%*^ce z1$hq-;HF#cB-45zqo{{HncNeR&Yt09LQgRn-y@xj>rqU`S+!W&;Uwz1uNS75{hM51#{k=Jm#VCwufEA6dOiKJ1QnKXe~OKCl{3W?D@p@Abg?hn{oD zv|bCz)ZWX<yMRH{T;}t0bayrKqMJ40ILB9WD;xS#0=}-f>uI1 zK<~RDf9pRmZ5|AlVJPYkd^!ZOfw}nZ!-2TRgU<*3dXV?~^d;~08BC`48BV759YZGf zokS+}!}^E*SpP5p>mS}(PR0yePeu*iLTrZYB*TaNOst1GkfF$l89XeK3>+pS?+n$F z0Ygj3fFZ5K4mvmj{r`kIZ-%_}nD%^zDaT@90q_w%&mIQ96PN*B(~uW4Wl&EtX<&ac zVc-xlZqP_FcJO#Idhj$dYRD{NgILT6#A2+iKPN+nZz4lR>>z`XApDMv9qB*Plk^=K zM*55tk={0`q?b(*&_ryZgM-lZ50L*gG2fFuV-JvCV-fWo z=RvH-1(ELK_@vv|WH@j}OyC>HDd=F|AjDuG|0~E_H3~WyjWz)2*vtFCG<-K17!O`! zfRVuPQ9a1eQP_V0>y!rK%Xh|3ApNmUso%KSq|f+;r1yjsq$grARui|9?vwVyaX$&T z0s#OANJNDUKt0}R1zUiQw?IDaF0L2@9gKwz#-aUjoB>P$CgA%qz)1XUjo${3?@r#C z(1-M!IEeI_ID+(=G?w(3G=*4AexG!kG7lFmfur&{DzF*Y1?Tw~;0$0QJe`H_H257H zrq+?8u(zL}-)~UgwJ3k-c<5jPbO6i*rUDc3-8f(rFdUzUOv2oEa(B{qYH!kW`a7ii zbQsHYRA9zLU`B_tJo8f&unLO&8u$sud=zj5Fd=`R0jTl29_QARL%95VT)qMFSD}15 zgfI^>kmAx2|G{xQfF)tXW-@%QDagN=hWY=D?xg3;-lW?HP~$iIILvOa>`8=Iybx@$N|r;$LdckdVrBwU@i)|K zi9wCT=V8+k|9B7b--rJ{yBkjIg)-5p=fR-n!=Pv*`WS>ip4s8R(+LEYSj=}Qct8HO z1ug?oI7fwRbU*{y3Hh5Ke+^_ULH*`K&MaUCFa?+ZjD~u#Z8aJ60px#(H5ebk2c1t) zRu7cj2LlNPjz!SL66j(H8f*zJT8a~wqCJ+b0bmnL_W-AW3-~=8hA#*5a9ut59`fmY zsLvqh6V&k|$a)``4ot@9v8eM1sCMui$cL%*`WW&*f&9h5((dTU(8Y?rxS?QxTZM*Q z16{0vLf3%s8nnln&wz~pY+^O+aM2c_Zv%BXqF=7jM7?8`tBw z9XJd)q0;_1myb9~2Cl0j8zFlYWG#Z~=Ay2%aKu(fmOP|UsPjJU4!E~ z;7ecwfGU!2&?cL~WpnTv5;3u@pPVm@0 z0D~Ud^3Km*ny{zKQ7mWyg&lGq_{2adadJ-|NT0B{I60vq^&i7eLE92pj=^ z24G^CWOq!$&fs_!um>D_VsictVji#!ceIK7hlm}Za2GV3^KRfY@;SVKSezS2mM(`a zKvi^TcPeCKTQbWUtnRdq>H4AxEp_|_$5VhUULaee z1;7(PH7pK^ACCS2inC0R!*E1XTOJs)tq=nN56LLRL1rLsfQIIylH6TzYM?kK2s0ur zc{sDs-V;G@80y{g0IsL&40T!R>WU*R6Q*pD6@+6b5CKF1P$yvm9DoO)iQ#y6?AxKn zF&R8E2O%ag4o9>}1{9tKgHC~6B%@+USZj=rQ-6dF>_rR;DzeCS#L*p~^-9;1u3I#~ z1h{|z5Cbwm3B&^$AQ?ylv_LLk08BtRc+?NU9w1Y&$;1N0LB2p-1j;T!;}+9T|2twJ zzvK7@ppP#we|~~F>qGducGvqBGA7F->&0btl>D)`*6}6UR&hS`;p=mzCaV^ z@fDcEn=yYkVeVdt8FN17?OL(~5rmcSlGZ`S2FTb18Q;N!`~h<4^(gzGo+D_E6KEDY zG>;p+TzWsyNT^YQ=A-ulZMDQc=y$RD0dx%FC1n07y8?gffDFxO5OrCKdaOi~d=6jvOVscicuwEKXZ#-Ku@iFkLC#T_=PASu9AQG2AS(z=I54SX zm~s`g^Bmt&Xl@GgdJ6HTj`&MHj){OApv{T{T`&=Uh=q?CgEonVKNSg2qT`(;OQ7M; zQ2rXkCBA?Mw-GWnqfYdG7&{d3!1gJU`5 zrT-zDgClL~AwU3h;fMHx58@BrXcI5^{a&yQPw=>icJY9HxZ&pI3ZIJ3!#Iz&aRLtq z@aWi&^8gqehl$&xdtby~!5Gr`V3r9jHKE0R11NaJ6yj$c`6sk&x|`F(3q1t50_Wj# zJ0bqyh&FM6Ph}53*A6_+qFv6w(>e{G>Lgi))?W!3pF_r1kg*9ewxjG_7*Y>IBd5_i zE*RE*@l7mRO9M@n08ao4{yhD@;*KH37e^0(?w0hBi(Z4WyP@2^7%qmOY#Wq49+(RL zGck11>pw%!Z74knst6 z#}ZiPO33*f4rJ6bFdzw+U?<2@W$3APraz*=w!hMbE4eG51S z(6V;{7`808J;>#&G_~wFg%Qyz&XSOF2SP1aJCf8u?Ga&<2{T{t|*_r zHSPz#2fhK;cfW6Nq5wzH{=X}oFTvn)MpH2O++u523hYz)*Q%LfWMYO z&RUfJHDVm!<5xP8b_S6O4~#Uy0M=k&M3{)tcp~DZ4k(|#MehQ>gc7rtqz_PhO60SA@t(Jax2=ae~oNplW^WLa`U(_9-;~pRNfh&aXu+#u?j_zbe z_nu^Wx4w`y5VD4$ZX+QJ>mTrGJQ$4w)3Mz?A)~u}Mn+kENo=gXBO`k3CDuLh{>h%M zWJpiEf3hc=4CwSoc{@hW7n}4DN^Z z5B>I$0sT*regoV{U*yE}Movtx0cz5#zmD|mSB>!vQ(Wl#Ta>>Z@>UE)+Yg5Bfmy%| zU@E2{6Y>2xa2hiJ>mK^|AS3(sCL{U|Al7|{kYRmo$k2Y{$dGBmd8-lMkZy>@5SyO?jKVfP07k|UahSY=!|*C{7&`a?dfo{6s~~HUHEeJ= z+5kX$1(^hl!*@tKL5v4J$FS~X$WY`y4DCt!59>|(Vx3ZN>!GCA@R6j)i1EbAW;*Fc zCkTxE1fVg5a#JLxv5C(42`O+&|- zj*84cC1#9AN16sjqCG6Jm^CPHGq4*t1zf=Q;W$SQ;>b2E@q=LAEGjNx3y&v)f}`3oj~vr zY=VBF%SSr^^ujrAKnTu};+h;Ny`F3Z^>w&xDaxOR@;{h_Y5^1RH$rV>6kv_N2SF~n z9_ck5^527eh#;0&Ob;jzx|lzpqeIaNT?=v1LY(x;JYX3BoA~5=;3&R##(9Al07bYa z8>qLe%Ug|0K81`qFsJu`DZm7L#um{8+anQ#KS_TS+3S7CpAFe_02Gg#2FmP?M(l}! z4Z5I%%PXLZ6{yq-5T@4$tyl!C1-4=UJ_tDAJU>*Lhifw6aMzG8QOD(w^D*SmF`((d zWc)oIMU2GX=p^-wh^|L_ ztltEM?*VLaL-0aehKXxZFd*QaVvx6J2Gj{zbPQ-3>N^=2gUhW^=K&uufM_!>Mm_eV!VN8btq-inHR3-Vi*0yt^QkN6!n@y!LC8-+nB9@inC z6tWheZ7ne%)R&F{A=FBSqif;yaai+(@@cuK3Z^Eg2SCf*2z&#;%a!1jeF5`rt3?~1$~Q5bT3Oz$UYno0;m!u z+8uuS2^>!WDBdzTb?l9p#~@6W(I&1l5T{s(N#p0hc1+%Xflu#_^TIHRlaU2b(>(nA z9&|o#IcyLr>qXa@)-g`EOfcy>9s+&_Xj!L$vw$4{Gq=PhTyS&)(A02*I${M^p^u;; z$eS34BN{pYcHxIg`RvErc0 zAwbKL11dlbBm%U|OduC90A`>XXzqtiCdMGJFdOlZmB(+nq)rO5ijAQVfaGKEqVuTIb;C8wmO zXJlq&=j7(=^af*LQL(wCw7jD7T6IlreM4hY%k|dw8#iy=xqI*a!$*&wK70P+*H^E9 z`l93&^;!= z`2V9f{8yhl^!0BF`wxx%S1RkQ&i;F)bx~*koyz_<8vCym_AmM(-MV$}Ze`U2Hq)zj z?>>F{!fyHx81N2E=N|(H4ubUz84CNc9u5n#83`ML5y6U}6W9?9iMAw*DRtP=e;AX+ zmS9R>enk(Z7)ihXamTJb`wkpB^2_m)XU^I?IlFmW^19;Z9~crA5f#H=ad;Rm>CrMS z9!6oYiT^N(4x6wT1#J~D3)lq=1C{~PfNiw3!#Zx>f_>b%3k$jb;K4%}$>S$ap2AL^ zzj*QT<*%@n*T4Pt=FRWFQ~$5m|Gs`DX*ZuF*UQfhZ>e-x(_HR+zNykPsIfXQrnZ8` zzgEJLmlX-s<^pMoNw3H#%8$z`%uUEOW+&+iveFFt%&fw{i?eZ}sr6ed(<| zytRk7_VB;nU*7hMxBcSZ`i0@2a~SL9N{7|W73a@4UAq+2P!kYSQ^Dj{mT={zMZ$z) z%wdW!hbhDy#)vsgL3UEUJ}cFbr_~z&F7DABN=}fL@^jWr*X&m}RXRB}R$UCLuL+2) zu3!qv!Jz~kOyE!m4#s>{mLXT2tItWy&(BWLXKOPH{w`<^Z8V39v%{LI>{m5jb3ETr zeId9G9IC*ftc0sDgF_KG7{S2+4*I~r-r8Yk=4o;ssSDTZVm!6%b`@6VX@EP=QoHSRS z9okTBx4ORC-nFjQEws8OkXZo^C1$R|1P%oSk`z5SU!6(%96)^ zxdv%!mR^>gp;Kg}>QvgKe05erUUK%|#jQf@zJR@I8mdlPL;m{ex-%E6>RrRjtAp7k z;7|w-dXqq%TPRA*f-W+^BNc6uq*rDn=;AZu^OLgvE^ZbrCoSMmUwvvgVHXsr)FZ zsXadOTK$=?%NuOH%j!L%%vE8WLU7QPu;tmsJXHpCkpdoxh2q2nqcl0LK#`&}#HGp% znzX-*R`U|lPaC|~(lkK+RCcA*by34VKst9gwMXWHh zj44SeWy=yUPQ;n{Dn+p{PG*wCOH4Ags7RS0C{q7z)Lokg9qjK`)_7v7x#jE*<8>z= zU7dGqUUeiVvoe~W3?Aw-rdV0Zk;+PWGD(S0Au@{<0<%oXGb>fx;`qOf%G&v)yy0Mv zl9m(i7q!~$*4=P($!_)z(bhz=QmUdj>dF|tqJklmma|2oa-LXFCXn#T#8OVFRLU+@ z$e5*Ze;cI@OG!!7;ohd!lXLYq9k%A&xo{%$j;}{z>(zj`hR{fPO%zjH#o!4l*?evV zSHP(d2w4>(A+ua6iY=FmV#-zGzl-ALHKh3Z(f$QDPtD4`>-2f%gG)OTpLm{BJh<#E zxP8Tw-5%h_xE>Z5-5ebf*~AEoXkV;9E^^)k2I{DwmuNm{mBi(Ltx5$3r zUDNr6_li6Y-ZuCJv>C!9TlCS)#vCTEK9eh|O%uqfQzWWuNeXp^CQehX)+Cq3rKFas zwCN=ZeMW)A@R#wHLx=9(>fx;({QWe2%m_PoS#Em1xRx6e%THaT&##30XzyDS5`! zY<+G*US5VupY>PqGIOp)54TNr|F{hfcfjG+pB#+QtVRQqUu)nOXmGE)s=ShX|yAAODb%@2}AP$n5ovKTPNhGG_=loUt$>CP9-MirMv4umS zcTjr~I21=S8_Wzr4W4Ns9;Bw@AO`rm`SAU7pbIT{q~#>%CFNx0Cus9?{wkhleMIh~ zEi4>LoIbu~c0F*b&^xfbC^)jEB$C-s&fr%d9`t7nA|L*44s@Z-S83D0BPlmACm}Z@ zH$GRF^H*WvP_T#Q@XpN=$N4men?+uM?WUl}mhvzL{cNuUK9;e=|4+_=->n6YH0UE) z7pG0gPsobPOU+j1L zQHTGpg~A+;3}~qK3@xh;VHf@3|7R6*Rq5c73?2!EVoiLZG*M}kC&`VfWJ!TK zMO2XVm+?4vHo0B62QkXC{aVW%=CqbM?`tV@_p7Y;h%{A&S^WQO`2XpMg(R7Is)S;G zT%1W5uP}+#vLac6q)3q&4J&Y4!=98lqFT8kE%*|3b|RJl$u2%+)}-<<)!JBjxH(I(0=wnSFSl}btlGEs?8E-aBK_=o{; z%}OQ59IyJTxLq)fw3h5aT<%1ln%YzIN}A6d)VH~LXV(VAW>iPAldGb+wEwUC!~YkT zbH&0kzJy;Ul=4dH7=TPh#{gI*aq_>4Hq&^-qjr<(+M@%?nocb;+;G~LecRI|qs{+n zazjX@8gU3^RSZ{J$>57BSpt3qSIDd23pwRN5xZP0LJUC6C|8JM%T$uTiu&@wq^9~O zQql0sVB-zj#W{Cff5^P=eIoIJx3lu5kC(XBKS0nF8p>^mjAYfvMl)+!v9YxrW=su_ z6;`FF4?i} zd$~s6@w*stGuSKaMz~K%TeM$rE8}WVD?1?YIyW%jx**WMMI7YcA`AYjC~lfe%+32r z{@oKpGauQ#m-xbNiQ+eh4FbyPN5&i1gW)eP9uIovea8QZzn$Ns5J#Vf5$CTwh<5S5 z&v5m+$9B7XkLT`rSLoq+SMrxZ#ZM!@rEerp^H21BT;TNHQ-k|AkMpjcy_Fpq)Rqwy zc|Da8+nmg1H){C2hIp~CPAQkv%Hw1;QjMZooT91{WyW0-)AGbI|h#R*-7 zg+m6-;bh-O7^|NYxNmv{4mWcmL)x;zK^x0x%3yIC(zt@UWTB)+BUMzXm2s7E2?-U7 zl*DqmHn~iy$73dAs*Ycjl*cz|x(b@ZD{y$M2M43mjK>9T;NS-i5ut54;E)r;Xv$)8 z8#1`UKjP-f%0#8QB0-&qn0ZR6QkzzyFk~3S1?joM!sIN0DY2`d68_|Hs?R?;==|(% z8-s3Myd=d6h});Z&c8f9iqekQ*M>mLC<>q63F~2CqJsBd*C7C@QrQb$Ny&sWeTU zW=>AlnlxECdR2aImfVn;CMirw6&EFS6*Pxm!QqkN4-Q7RO%HXxb~kgwLfdrUV2ENi z>KVK`9bz&0{Ej$Cwj$M>sm?T|C+8NXX6bU&d3ouGNhQk*Q<9`bNnHh{`IEzGiyj_z zaIn9b7aH28k39{JKJLDCEbN870Xkoieikxd(`P=yI|$ z3~6b(`H6}8EOopwEnZca(p6YEXg89F1*iKwD0G_sPY$}^(00V;uA8D5jpkTht(hgQ zz%y1lAEW@jzYczX4)QQE5Ccic%FRzm)n&(N3er_-V@g*+B~B(UEga7Dws0tP-Sj~3 zV}DZ@7}8!C5^>!e!DuXx;Z@=JrWvsqBk~}0$Ui_1gf;^_=*$C64zeLKb=k_4f^99bbaAO`_42w7SN@={XrX)>Y6PUSw?~4~(|G?-w~udsyhQ>4D*j!>t0};CA!XFyuH# zSHizB!PhjPE$AGCOvE5k@a+GO_(P^rm!g&Fa=2nA%nRo;1nA;lA;#g~Nj) zmv8PDc-hmjn3f9f;Hp~xSTp=QWZPD9!LPR7Eu;kbAcf$N5C30l<|tF) z`zM-&>i8nDMp-CJlo^$lXaB;2lw^KEPBPDE=rVpwnLwW9Y{WCxv%T(`9H-m^hZ|=1 zZq^#Y;_6wHWaSVNqNHzc5k5Elk&N z3bVV6*Qw*lliZEuPSM$(w~HMo-zj$fx~b~?-qJ<~FJrxDw5~dwlMTN+4Y?4SGL|F` zwy7xL$)#q?^M57c4=R4KGLBoUj$@mW<5{M(F5}nqvE*_7MsnL^+vBF$VPb2!)5?lQ zyZy%NE|>G_d}6b!qu6QJqIsIiSV0`}gyiLHv80SE5tRz0f>M!;S1OTlO5}2OiAv5a zNl-9KlDdrN+EL_zegnB#e5QMQsr~q>dfO#Mt&RutZ(Q)uHv5I7)kZKh)lnQ(RSZvt z{vobl34|3KA-|j_;+6|UoN|$vRVEcP%jAFf{_!|=4UcoI$=!mlkf(L3TYdFu>*Ch4 zEAnr-?A6}&a!9%5cR3C@2=b=jFmXe8l%OsumRlRkWY@4*tZELMSHKM?eLW%|vL!kwqL~pA-pmROYvP23HuA$m8if(T4U&kU23cfRajl>yX)0So z=Yn)M+&ncl>w)8^Nza@&C||k!B>c_oF#E-YlhNQ3_VCL2koy6yL3cwu0`7!g^uHZ_ z>FTXmPrqBN%f2@`UOqSZ-dApjuUxqy_30`~O+86v?Mh;7Khh)n;pvfyFHg@>ys=#- zq|SZCq#U-p9)7^(P1He`H?fDDe`6g!|C)Qm>9z2v<10y*K?z3@DsDOX zJ#|-)*V+?fe#^33_B#FIj$f0n+TBkIz0#f-6Vaw-MPFC(7%g%Us|j7PQKaHE2oePK zycAJACqrDv&JowK^pZMev9vO_TxyP~kaPuaIdtgjtsdU$;a~gB|JEK-{&NP7$4rG?dtI=>*Sc%*>=lbWnA3(GWDwcz0@$@_LS&| z)&K; zNvQbc&A;*5%ubG#2zDW0XxSJm4*Pa#?(V7w)(~`($HmUiXMwOUfFIR}_ zBx-4mC|O=D$W&JI@>Nw_Q(Ot#9A{*eDs{M#cLkIf9MsDZFW%kb*PK&hUgd&=_R_BB znc$Ea?$@3P4(Ty5Eh#KkQzDPspcV=1RB}m;JYHTcO-8I(8(%5ZtIPSt35AFi>oA$g zVU;Pmf`4+z*lqPP=k(}ba~+nw%<|mzOdH^EH!J*VdlookM#r?IGg*x(TwZ;mP*kg? zV=-~cYx1P{3aM68E;b~Y_(jQjo=KC#F~?SdFJ>u zZmFUxpd{dsu$;Wk+++0u9A4%*E_(qEPjUkt?&gMFZO;RTyr`Jw90t2V%NEpS@THcV zlq6L`nL05UxhUy{GDBvb*qEk8Oe&3Uj!)*5D!YPz4^}ic)FPC}bsZT>&NQ(8I6Vy;jfi&yIes2M68doe%XsXYU%W z`rI%EgteJNqM9orST*G=VJV(B8kp6vLFYfH@-nj(*?O%k&6p-lEJ~87i`88LB^ypCaCn)uxBFAwxlzvy&dVMfF8*}i zc=`0LA}`O@Qt!a}TA#4;>TtFR{-z#!$cF!)23=?j@>CVp9>{g6nNpoDL!50)6{Qs= z36qKwx&lfL4oTpUy|4Qdz1^s%;PA-k@%`PRi^tnbF1cfUkY7c;ca#aw-wp7+bMX8> zo%a8cgFv-BBFiX>`Dp+X}pOi1J#Qx(Yy>qCX%5YH60N!^A(a}F|rZlaePx;JjaxzW))?s znWh3Yqre>B6;P_7 z#Z5+JLS4ZSDUg*QE#*mt$VlLqNEIApBe2YBCBvMeiZy4eV$4O#u7HXgOkSrgB~S7X zk_UwjBak)qK~1gQC*~HXU50k|^Es{F{+ab*(J6>bsF9hVKtw=N!4e9~xgvgDP<_yT%W0;57Aq*o#?xbrH|#&lz2mks^Pcyq#QRrV6t{dXi`uUG@mqp}*iGS~ z%*LpQ*ak*qbUiCNs*W2IStp1MuN5)EYGsVj8Wl67M#Jg~ZY1|WglZ8YR7Xiw?cpAU zZD%It-E~}%@$lj{&9e)KWX~_2=0EauWZw7g$V#{scscB5sCVd%NT1;LXy2eVhF@SC z`)WWd*WbTY5ODRnB+&1=vP-B=??&$F7m#M`EN!~}bML&nwi7cRonNGW>GYNK&H3-S zl*_)D*X~EdUtBy9{LJf2z!Sf7S04x1`&u#+!kw->h&u26AjZY(KGXH`J&v2_J%Rfr zWF~Y4<=Sqf-87q+S~ipX+sFH)KRG!j{`JWZMbznKEb8p~D9V0w2zCAkf68r_FXg%K z%9|^Pyk1{D;`u7*=S#na9=q@|;<(3)=o9WQ7$@DIvroA`@@!WxGLz{i(z6ds8RoU8K$~a-*DjOb)v_TF+^UGI5$BD>yZg)tu7E8g@x1 z@s>k}uHNe5{~|ra3?&rb(l6kFpy|BB2X}!d1`hOCz;#G%HTCJ^7sw0 zMnOYNx!_uKmB1WT#V?4e=5-cS><~f;pa&H=X!Z}L62T$CVLg=y4oTpU803C80Umig zBcfHwiM}oqGMdFQR+Au}+sIAgH?T7W^~`)>J;Nlbj;#=v#8irm(bt5!=xTm`CqXg5 z0XrjFI3$8Y((zf8#&P}c$rpCMN&$zIAon{-;r{KK*ofA6c65tEz-*REJEFJzBw;-_ z6H!~8q?TDCEko3{Ft$RX1CN}TDt>1{F$Q;Vh=U%I4h*J}!6C_U{hQPaJAX~{J@q6F z98$p{8Brn)E2brm$83^|xt0i#Fj0)?khF$lkXIryWMY)b^^9_9ZfvDk8(SsF>?9~A zIEbK!xXm3LQotd_aUC7Sev#>W@<}E*WP}G;qD0Azn3e=Ct5GH5(Gj9ru|`@W&?>5U zMwJ&;=v&qdPq4ji%NA|_bT(^j^|n6kQMBHM+-e< zA}Wv;8)L~TOc3xb5h9sJQ6;9ML#BiRM27MZ8Pc-L9aY;e?pHZ14Ua3OEt&pfW*M#w%#D8!Y zM5Uda{hQWt&GWoVKRnL&IrdQJ?{Zg%$b>Eo(GhUa#c=BL*rMtzk*X{$E``ocp)*rb zB}K`JBC}d8ELFw{%B3n^C7yd!3OfUeHIPt}k2>^_ddR}zRkq{mr@BkqAL+0B{J`LM z{;naw=Vn1rXd4}wD2`;;7cs_sBvq}?LPR9xkBEp;Au5;21(l-Cf@1%p zgF^z^g65EZYWA6ndaJq=7@mso5{`oj-eQxbW=* zqvydpCa-fh%-$ESmtVPBUw<{Iyf!?x2$6|Ac;o5F&`@JdiZV~1g=j>oL}N-6$D8A^ zL|q}0RY*Duit|nf2hC>kd-~x)Z!%BMda8F@{?K?~^WCCLdvBCnKG{-v`Fu^o<;!J_ zSHcRbBiQ-KgwKHQttk{~6xfMCtkhey3;{L%QaSQn^R5y6+DQ&*!XuN*eFRv~jCaWr%lY$sPd>LCRGYb^7@6RpL!0*pVU=|junR(_o zRz|6km0TtVWSs%U>rW^-^pLa#(c7Z~pXnT?-YayS*H-Gbtft;|U1_uXL1WuR=e#!W ztJ?bDs8smh35WqGU=!j}zLZxYm9xz8@cYx?^Xpa7MWxD^{8B}1W|^$Bp!oeTub+c) zdJB1#_4B|-1?R`#EOwb$U*kNxtl4E_VVnEW{F|OG+BQGG)W(qTgqlbuy(&Ob$rRAp z0JQJVC{;>hN)zGlYh{t8Mp=ZZTpFbV@;e7g&W!x=ACfnCtJBl`ll^X(?cc#JAtOw!j-Tjmxo*w6>v=Tw zzK=uv9iNNRc3)rN^`Jm*Q+ODwAv!X)jujJC%VR{;h?rs3a#m=ynjKP=!U?L%;RaTj zIe}Frox(G54~z}7kyW`BE0)e!)mI(2DsDMDR)5QJarQm;Z&M$69*lqHc}DW^@_F8! zD;JoEiNv%A`$o1#_=jJQ2@Gvv1qV0tLIRt_VF8WuaQ{YigkM8ygl|J$q;Erhr*K=^ z4fZ$-ageQ8q4G0!hxnzp;pVvs+4r3mr#^A}I_{<0E-`$6&hrbWV;+0jhd)48!o2`D zOIE_|2xKJ0&>0EHM&KbEL3G9YhTO-iUG3}HmgeW#mfk7UDewlVw3*0I`I3~^Zzl!q zC;DaIx0{&q%zlCDwf*Np%HdlUb$(~m?`{V|UR^jE@Y3t})#tvae4Yi`dOr<0=k+AQ z&hv4M!=*<|$BPfSP8S{u&wD(OJG*G(jb8{0_s{bT)-pC2Eq zppMVxBO`%9*{%(z>^25cPFsB`*YCZli#sk;-n%bRetR$c93peW01{ccv>+_WFCMPOjiz5~Mtvwa~ogEdm?Ht9}ewN~X zZ%c`Pw524SLU9^4`Y!HHG%uzR1@##bQr}oHsqY4aQ9oD*P-&$MxB}G zOgSucqFfd`P#2ckQI}VorF=fKrLL|zO$Dt!MTMCqa72ALDZK$e5oz{J*gjtxKn#Zo~I6vx2H}_IZK_JafWhw{}kmm z`vi4q&N0gSqhBb$c}Jj`x zg>3If@%N9TI49>*Jjb;Z&*M9a>%IRCE8yfyMx_1IShnkfuj4ApQ^zn`FW! z`LIa=Y?60kJ|%QmO9|b#Q3CG+zw!J}J?DfwJ!UaI9x#Mn_hRI}ccRn*w<1%5ZiZ_^ zZiMBBwTBjlw}+HRTo0*>tPgF7tPE?6Ebb(zAaIBP2NpO8z(EKOLU54Sucf5mAn`i* zT6FcyGk&P^BQE3OeKz0c4pZiTD>gp(Ms#vmdt_!rTSR_TYj{yiYuL4zhVa_h%JBLa zb9h6Hu>;XK{_F7DgA7p%7R@Vh@-sj2sr*XdE$_EwkoofnG|z9 zCX>+;rDwK8ma^(1YFMQab<83_4=%aj(HT%7&_mQ89K_%t0SD#rxs=*&4VB>f9i{d> z_$uDl_Ng++^?@wH`?i=Ba6=#rYvn4Vud@;vEsRW7bF_io6jjNou*9Eg*@nnkR$gQs zGb^$_wlknY26u4aLJyK3`cYDFh&wTt(%7x0l3c(0J<0RnYmM)jr|Ll0`>HVS+cHLA zyOlIQzQ!v?49XBy!_AATWoJdzu`;6S8R?w>6$%c~&;#!e z4l;01pZJJMwqNyos_VDErF!m1Oz6zhqyX3Z38CJ%RMA0gGH&E`kuCL*X!#frQx6ET{E@!Wc@R#b-++rrp#NnUKF2r(djT5JtBIi{AC z7*ogS45%=~A7Y>fKJ*|12PHTpotphyy5ovpweFiRD5QmR3)etWXVbpf5~H&$ukiHN=57?iBZkfFlyOqMjfLw_zwR zUS-nR85h2On(MXeah~6?M|lAb4|9Sp-pN8dBt0U$If>1zQOQLm;*2;0uQ&l&Im#5a zCH^8%v#NPb$yR_kv?#fRO z^u9;$>;2E&(FeJ;>q4$H<%I`VWin!mu!1XJo}|`Fup&lKqEhqArE%O!p^{t8Q*dfH za&{fFGx#?S3UEj`Jo>i`+nLXET^Bvld4BQ0;Jy8BfzJWNAWq&W3UF*T2YFPRg8fZ- z(XqLyd~upOUae6Xl2uZ3yh2qQh8Kq&-2?b1{G)ET4*JdU1Qj696gi-}xRW9Mn zD}~5u;tNDIoX&uXL=1!pJ&3?T^;2Ie<>%qgavdh$FYuUi%XD!`YuTkW^>tofR5f4S zT5>($a6wz3Q+7+BPf}fIguIl&;}uEdOrtiQky{eSN-b5eHRV!He5Ht^KuWr-n#+~d z{Ewg_5r1HTgSdmkoAh6XJ=Qr*xLfExv#rEqUVZJwg_TX-Yt7eveky49JCSqC-!-*8 z&{x$M8X>HXWwXk~;;52Td8D~a9-Rw+KfPSaNU9Vw)z<_pRW*-=9KZhwR1`R{r(&E2 zhqxVmUu7N}{J`Kc>SnR)B;;UBt8BbD-+cY@hJtqQy}7r2Y}4=hd&IZ-`$-x@!;y{8 zh`lD{MU-nqVdZ*pn5jY>kyjy(%B&Q{q+AomCRFn~11cIZ5Dqvxu-n~5Ch?YgA5#!cJ_Rnf2!||Vu$|Kb?4qGX>p!r zY3qb}k~V^xH2MN@=d zVPj-xKrwq`Y?zEbv4*_R{y^G`f5EDRU#tq->_+6@cAk^@z+-LVGq-JuU)>J~UR*fA z#Hxgd2Una!?gqFA-VVRye>2w0_Xh9Em3FDGS6jl>%dHvyo~?#}OV?`yF4tc7|Nq!~ zueheNx9xw4Vi|iKdl`Ef``8_O?+Qroy>}845=bBR|58?x&wi8l_7_0Yf<$cSrdC=gA1&0t&y=^IPu$gZDMm7EeQ9IkYC* z+qY&puv-c_b}dgFZJVo6H9u8d8LKzk!mc;girJG)VRUCxZMurA z>0M80^sdUEstLco1*xk^%j3nQ;>!+F(s~yefZmMELBkaZV7NUTj1C2W*=bL(y5a`3 zTh3rBg*zi;9l%+E4Q@(yAW~(5ubRzxkOplmT+4biR+~DStYbBjt!p`4tZzA7@>6}l zFLJ%RH;Fv1nnLb(>>@dRH%aP<()3tRTO10S>wK~QKnR9^xPaL)N1&Xt2b*(tz`Dc) z$18MjzGe+>H>e=GWd%OBEg?|C9Ks~cAV$g*lBJCyN5&X(f2zMy@ax^h_~q``lSsR6g7!uh7;I;N*)BRz_ff&-4=Z3DvH;EzGjKX;3a-bE zLHMT;c%3i=zmxh9d|D5p&gekW+5e87+`|u%J;e{O^-LpSu$K74AH)-VxWRRCE;0xf z$j+mK@?r{TEw=!J)n;I}&IBkMjKF4-0oZNP1BY$8z}v13{NJ@exKk6ncd0|bZZ!zo zs|xY^R3PD}D!?zGCBDLs-NQ6wA4`Z6tS9!cm)If;p~IEQ6u2|Z4CH1Sfyx|x&|aVm zhKsbpe2FGdm#Tx!a#dihPyx;=C2(G?2ySZ>K(tm4{MN}r=z3|0+kipsrveT#5bkdT z+>Q|Nyg3ED6c>P(&Trsrx)*$DC&7n(4LqHtLFlOh?!nsN7Gp5#nqo5Il4CyPTxd1u zR7M@(Ra*CR|DyLg*4lJCHZt0{&5TA~3*)m>E8~Mx+fViXJp8*x|3B4%_Mg|H>wmfy zwg25~;fUFvhx0*z^T7+}gE!6xKkeTj&}1(JTAzXd)^+f8k_9iJ8i)dQ$AnSF!|q8I zgKk-r0e%6k-{qlApYu~@kJI0_UA)hB?M@A>MyDp$XXj?t2bUJsYkn*HX96yIa6%96 z=)oI3_$V%fK=crTc`uZ53PS8|Kpepsn;olMTYf6wVeWQbM?i=ke9(h0dI&)e;pid4 z>J)@CZ$PM{JOsLGj{A8V4*P^!40y#^_leS(J;FRzx1hwn)2))z!GFhX<=64*TFc*Flu=9*98?(Tp1q;UEv8E?Q%OBI6(V3AaN~>D2D_!KK0Nv&$#J7ycXfdVZCA1OJJz(e;tAk^eJ+ z6V|~6>+sEkA9@JF`4EX7Voi2KJmnI`}`xB-9E|oonE=z zuO7wDZNg`+ErM#dTKCU_e|dZnRC(09KS7m=>fK93^@5)XoH6_J*Aft+2Y>VsqBtL7 zbdbfE?1Ci92}ojGhXl49#PYO|jhPIF`dIe{h1+%cCvv{}WIMNc7Q3~0Ja?}bR(re? zefD_nStojeDnk`})rktdzIgmh|K-6GJp`bKaP$zbvl>!Nv2Tug9MTw9u}4UDJc+A0 z66b0%5Gk_m2@7U*1jTXN{W4u!yo-fRp07NsJ*&N5c-4A8@vilLhRat!#A`NAB5P9LjZ&?91k-}fBQZUs6_r&>h~$I(Ce9SzKgNgf-bBWER0U`IJqdUm{LueH$T#+s z8}KPGE1=pxGoZ#VJ)p)nEuhx>XZpv30PEm`bqK~f#4F8)483Ke_bfII-nZG;_kew> zyV&u1`vYg$#v;Dv=lgCZA96%CFH-`XDq^EV55v;}ib6_43xeK-X9s-<$q4!wlos?U zAT_AkFD0nPH#w-*=Vt;}WFY_PAp|`nD$Rl{{a=R*EZ6rvVC?QHbvXXD%;{3|Bfdo4 zBR9p5rOt*g^WANpWcoUlB!!6zVv+)LA_~GY!rw%uB6~>={TQ4S@+lw@8=w}La#^)2(Arw6%DbIu){l$ZYR%^RUZFjVnJ05Cz!aH66)cI<4C0F+KV@HFE z2X6F|TyI`}dax)nF)lDAHa9#WrV5Y3dlw!T@gXEG{8M0Dcy&NrShZhVXpQgB1a8P5 zzIpIN4`H_^LyGb*WA_Xe_7ziBbv$BkZmHnxX?W&%`1337nYXVQGS4a<^dFY-X$6HM zUS>|9C@C#EFg7_eJTj>wDl-0EL}cvy(5UE-K~Yhk0;40V{i7pl{C=i?JP7~8L&PmH zNK>9Zl5aS#`=Rx+wkPc08lTy3t$oA!{oMzf(=XoH+%12})-9}Xw$6SibWASv7slj9 z_=jbsg#~3kiVRMB7ZIBDJ}fN$LvVQP$DoLqPXQ58)&4&d2#|qzpa*}fL)1+%$WWd( zSZFl6qny5|snTwF?Q7Pmw;%1cysTlJtf;n?EPl(<&V9k7q*e$VV#|C4VGqK5{qLuQ zco#ej_qq2jEFc4q=1chy5}f!kC?x(L*Q2BWVOdGA zu=HcFXWqwPpRA8TercZq1Cp!#0~2ceex`psh|ohIdWgG0#&eY?cRe%~Z+vDw?ZaD& z_^TSb`OoScHhzM0#=Q%(BJHI4k_$_RYt$@9XfFL;&Y&z>IHbw3q|kb!uwL@tINQm>HyA{o+HiC^jXpou;Blo^j3ZCByfA@=5fbv&Qm zV=oog#a0jP;28OQb+U4A_h7oT2HJC*jnJ6 zp9?&Zf%xJM+wjB4g3go9hd0TGziyDKT7B}kfjXz8)pmVB2kSsapY6r?A*NK=pq;AU zpuL{3pJ&SN5mI=a{&ddQC|h=08r!b5*n!#d7svKX1Bd;-iR19Rnag?H!uy%Pdph0| z%W*#LCnI^s$d{^vq^jmRj&D4Au1BQYJu+&{gc)Gkn zcOy=}uNk{H%+j_yfy(I0rP*{ovY~Z+u%Wj!F_<-NHg-%wV0u&tmS>&8`m!U~-n0jfgdOmu8Q>vD13v{SgeqA#i=+=-BT3^5Q=>s`elTdQ z@&?^40x;ae1JgtHV0GLM7^fM)K1T!IMGClGu>{X+=HP$B6vA&Aq7 zxT_DJq<^L_;ef1!kx+fAU8{dxt|Xjt2m&u-WCkE(!u<9 zDp>Ea0_J{mus>)DPKS-b?WiGm9@7W^KXoDOqz=TL(t`BU8c=vv11iq_OfM6#2cw9P z^r~+?5VJ`jY``9fL&yOx5LdW`3_=$cj4@E;sOHT)$+Cp$KSPL!|JHUx?5V+h^z!6;n``{bEij@G{3~6BA zmjlKlMX-6U4D=7GK(AK=T8H`wtw*Du-mB5c=+|gv4rnym4rK|5tgS|8I5R z>#qeq*1`FIsl!dIgCy9c%K)=L{=e#A^S{=i&*obl`ZT}Qq2a$%2O3@r8s<$K^uWez z!MQsf_^Jy*fF9hoYTK*R!0OkjW%cWRWcTa+NI=KTOGgh3^k9!3c<4cZ9z=R;L4+PW z=!ZeTJ_|1VYvAN70o({#;3O%5eXbg?OEkc)QXA~v>5efQ^oE%o`aQOt2JLp;294|< zgU|N8h96LWIrJI*NMM5=Y|sM}Jvg8T7pdvsfgZfkgOAzo;7vOMBGx%@bGi;LUQ*x` zssP*sRp4Z6g2MwHU_aFxW4$#Pv1>FMu={GcGp;0mmZ!F^3AH z5&L&0L+mEAZg#tQi$jNbonxo@2gfdp*W514XWVYf3SQ3-1q`eM6Em+JdT>M!LaAxs zr!pUcbXP+#=H6iHVF<822fmIsLF6U_?tUuZ8l?r!Y5HTl`$nUVPt1lm?=1T5o2}Y8 zZImyLUnw7W9n@F6PHLr7r}bl}F6%PqZt9N&%mr8j^uR(7JoF%voCbmDAr!NBxaoEX zryhb(<~ay*xH<0YB0K8otvVtM(;0G0G#qfrHS2dOv+U*mMeTBIqBU~cXrG+g>93r> z+B|jcV3fObGD=;#Y##7`tbaV%p$9g4a6%8>l2ai>WiCWw_Kr5)3el8<5XCq%8g74M zD9A~+-_Jv>*DFZ3+auPbQ;=!-mH)uH-TApqv(p!5jdLsWl}np#1;5>{%=N2XF{;q5 z!?xfj0$Z#D3p1}hdT>Dxev;x4t}+K=beBWC>1Ifv>>r6^oEnH>U+<0J$#ez_)V})r z>9u-Cnl*`1s11Swn>yF0b~XIZtT(PL>d3){g{s z^Z)I^0X?|lZ+iifQy@xZHpJ^Lg=EtWqbZa<1Iacgx)WJfzQ#JCF493x~X{-y)$(%d&@qDG4C{NvYAwlM^{9|mY zyffGpqGE2j@U>H^N0W1bsL453)a;z)+3b|z*}_ZnY;{caY~}n&z`{D%W7cy-4?_Iy zKSW|O#3}y*sd@`XvdmWZm zmT$Qr!?)Hw)we;A?Azd)7xBKl=x%K{6rL>C`1#H=pY-gQ>%})LiExaEIm!RkYb2lG71pkTsNmt3xJ!#VZ5WkX9rGxofOZ-`l@vo0Qn(Vm$ zFN>2|FX*?Ep0ZV<%bg5EN`(}^2mW@V`%zrC{0tYD{Ku{?d5x}anT>oQem+5z*zD{T z*TVCTY2p4z;D{c0=)r9lX1-&X|1Xl>`?pEc6KV4H?QQbB#$d|hFP4jnKU25nezH8C z`i^!z_O+ct=yRT~f2F`&^u*W3tvrJ5RF>}OSXS=j_^^@3D`<9d$!_Mkrnhhf$*qpU z#8%Fa1YBew&Y1l@c3|J$Au^hAntUy}K&oGzB+sj*$&)W;VhCmryftM2)?o1xnqA4}e=2)e_|bUW6I$86T~Mz(!ht83&@9ndgEU z?e2Kja}@=3Zd%TteGECZ;pVK`bPBV!+?r8cZ_WJcE8XsC2ZLSK$#f{{WODLr(@A;#NmA0RLhiSlP0#73EKTX6ZjI@(J`~(ZKj+=SyeatV zAmh}|S9WOk(zI&})w5|!Hlno^n^4<6no%1%P3a%I%xzwETQVzptZd7ADL)eMk%0)1 zKZGtPecAKLb3CE4tYIZ7Y&%c#I#nlS^%~Af9yDDMJ!rlyWYFrM&!F`w!2sj3(}4YL z&VY-w-GE5JW*|s~IuNgJIhdzqKKNY6tgltqvb9H-TGyvz{h?o<_PXETM*;yd5YNS= zBUYSL<%^Mr&!*tmn@D!=MUpIC%D4#2)>2Ts>; z@53z$E;=kAO4yD4yF&G&Ap}>0*c@sm(!#qNAD<+Y|&*I1*wqqZ{ zX%Yl?#C$=1h7eTeyMWd*4(PA71(Qv5u-b+_5IZe^z1I|Y`;9tpmpMG{IuNI#@4M0p?;wa9ARb>_G-Z$Q}ZgOF;C>+mN>E29&J64lmaHNcVFH zNhu>F6l)+v2H^?K?ZS!sKFhg0y4-TlqM^H#uQ`_;xb@3 zRSL|eN&scrZLpbs6WB8_6#jAr0q`Yk2*iyfv(G{DoF6Iu0U_a)gm@r_v2P{#HxnFt z7LL1&sA5lu0?rL-*heJr+T6t6kn6}puEI5P1#Xi|aF<*Fd2$|9$XU=NXF!jf0uyo) zD3ksK#-wB5Aa)d7#SVkl*3-fFg$}+qO>RvBYsKln#67J{)8)XRZveXE z4xoDM1FPUeU=e>D%yLeGN$D9detr&&sxN?1^F=W1z6=JVSHKXifH7PhHifHwW^lF3 z0w)$i57z&9IQ(x9*(brYDmT;qu0yi6}aI5|Y0z=Ha2IxT#J?Ntc z3jU75lAjJ7^x$Z`5;)XNz-H|NTi1iY@I4N+h|@q#IS-WlOJG@k6)av~2lLN2!Myzz zm=4|^HygV%Xfb}L+Y0WqQQ=ObHAsA>fn?1O1V-q=a4n9%6aR}I=r<;TgB)@(&H3PD zv=VrfO~A3;4ff85z|QLgFhkD*J@FD)=UfG9=?$PfzXev+cfhhm5-j?qMl469dMRUf z+pWi?nrLvhjtbYa z3=STrfE9EBY-6tiBjYB}ix3G`NP_kIyJOTQnGs5_Y(HgKw!?ZhnNoumXe@o50Ox4>GBvz!jVUjz2QO z$Qxjna%apoUuu-`NM^+5wcH@RUZIcHrPxIqQfy|7D1K&+D8IKIRe5DMs#0k;rdsg> zfhl@0#k^^Z9;|RZ$+?Q}a+#^%sWun9^_PQ}`6dw2_JBM4=s4fy?5LCXm0@n^ts#d5 zsR4GjY(J}5q0hETxtm$5(!uzu+QJ-E`)oI){+2bY@q#t1S-~FBd}KeOS!O>vp}-9D zrWtxLK@Zl)acy}8*LBk3*p4^{{PmWBzu5-xwcb7AWp{K)$UEEb=5ejpCGbu+FGi-* zF8PSv9=;A?{9r0M%I#to!SDi0N;`LDZf7{>&pi-kh_YBEhwGfuCe6KSI9A zIZ35~dr#vF=aKejhqt<)9GdmsI`r#5w+bCCoZflaa;DgmB$O!x1KH`@*eubcNa+Y!9?Q-R#T1*5K(SRVNHlsBw)` z|LB~l^Ojd)@P_-!=modIxRTpzQpW2yEpi$#&376!&2=6$$#xzx&iaACV&4Doz{cOz zgcrYY5HSc>o-rD&v!FlLcx6|t<(9T+`u@gn*2%hH=PT8I9#ZeT0+e10qqLs8rRqO* zDKL5B{M7uBQ>{g*bGPMv=RV7OE`1i+{C%R zBL@kVm;_Nu(}v=;=X56;Ep1D(Sl^gnz3X$V?a>br+{MCq0} z!Uxt_f>-nm_ZE7pdp9jv*lnFC?4c$Idnxh4KFc^^zr{oXE6n~B%$pSSz&n8bAjlxX zZ;{~yg~{D%n$uge4CdD7m@WO3ZMOMMn)Ttz1oovz(auuEVM5h{AV0kvzX;Pz?_^4< zXFfey^o)@xYGB5Tx@_V^UG!K{w{?uDhZ-&FwTc$?SxzLN{D%kYZJ4+I!1;*zKk^0{ zOp=?_k*O};bWi`6+5)46Z}W^dRb^2Rm8aNUERN?%1_32=5;22(MlW%4#A31#>qQ>p;hx#ozz$>>YiL^rcFZ zwp>-QFNHedAB(jXyeu@__~f3|!IBK-g@R;9$?P~+m9!`?y~MB}v)JGmYGhz0BP^iI zE+n8D)nOar-^mE^>!OGFc3X$|^iqB#prQwB^uXAH_vK#Xa(|N1*vq6VU4k^_E0Ed} zRk44Ssmyy;tiQgj!19m69LD*abO(vFWEbVcc#&>Qbbx7iL?kshER7KmT4LuH@`2?S z(qZc#)X4}4?4kz-bkhR;d#Hhay_AXcj|Up&Zrjb+lY;p_Zjdj9lH|i< zsYx%MD9wFbrnl}vvH8Ax1@yBSIqcg>8BU5ZDegMq3BD#lvEdZo=wutOs3KcY)ZccZ z$PQbt@J@zLXcyf#xSQr1)JydX?4?X3u%3m#p`Zu$Mtnx&yEE(%=}SIEn(txuet40* zsk}DnX{F-q(kD7=3d&6PWEESVPA;&!8Jo+M56^Pb3QY4h@<|D`6eT6n1c?Pq*MwKL zuJK=)?lE099+BO2QFxEFXJ{|QE2P(IA^{CO(9r{D1KyW-Z$$3L>u>~XaD@C_xtBb9 zahW`NAwRRQN^4b4h4IeRN0bwBCCqCPMI4!+d{=d!TrYi(>|it3%y_DEMjoA)@q)oi zZ?|zy>ZbGKyRF?~dZ_M^y;klKeG>}kGjT3p_W#z;>bsc?$No-QbAHEbu$EN4*h$LY zTp$nrB{SpRYmH^;FO0S)JhMC+`Gj#P_>qI8Z>fv2N3lrTwJ^|#cR$wLp&*ySE~uii z^V_Hn**#QlS}(;Zsn5zKzR$uXu75&-4SHar2R^>H0@sq>q;;gOXc2i)h57$qTgihD z=SluY`KcKnG?pa3GuRyU#^ONmOZr*gD)vo}N@rPqg@-Elae$6}d90ybd9DfLQI#39 ztj(19px>O8*Kcm0HDJa`8!&ZDnOML?24aW&!5w{sEg^Lo)5-Hvaq{TpR8sh9C%IR9 zjbzoSPD!cLn;-MpcwK0X#csb(wBw@ptV^zMoo;jgC6r~q2~cLfj@6*QzNc;d`h_m- zWt*<`vq61&*^mLFXxPv;Z`jZ-ci3PefgLgs2RyabdlC5@Glf*;P9|mLVx-{jX(ap0 zL6Y8Zo1`>rOo?kYm>b?~x+$Qv0qY@`qgKP7C(MRJ&KM6To;Mu1 ze^Gzr^<}-muUGW?;fi59TrqBht0rII+C&1LIQbYr$de>O3hv|hn7^a{o=igeHjzL) zlhSYOCh-C}aRI2#b^@J64q&*#7R=YtfwqYXcH1m~yVDc|cn*W_A9@gpXD?(N(SkDM z5H-iuq347;zzOXEIHA)!vATqWR7Md}i05IXV-2D)e+LZCBc9_Mkv|;99*9f0A3}06 z2Nb5;f!b_3=q#Xs@e&KLTy6^VRYt&Gs}D}=bwIdL69P7?K@2j8oE?hr_;&@U-6apb zyX67)$o)VsgRlo96?-7C4{6DXHwY031Wk}p9+#fv6RCA#;=~XZ5p7tD3}P!$#9j;;twU~DuCjA;wOh_eg~g=;`RXg%o0 zZU&vqZJ=GW1GJy)1g*bu$hzI2*}exf`}Tqc?E5ci!M_1z4{5!=Bic^4So+XDur`#`_y577T~0Q8y; zfnN6^&>cGrx^Q?5HD~~bdko>oS0gyuYz)UH5~$$2;+qFK^dO5_Pa8d$NlpT)%2c4~ z%>iqRB|x=Z1y;@*z`|=Qn1${H(}X=>oV_269~=Oqr-#7s-C;1SKMDq&$3TDN*r+}n zA2NVry*Tby6Zo^)6i$3GgOd{pRMCSnGJAPk$0;7c>wp=@3Nwz4@>F2z&H<*`60osZ z4K(gXwDrh%Y#O??4^n+k_{|K0t9|M!u$Hz@R|2b;habnnL=;VMAoa{4(lV45Y zRI?eJ`C<-dKUqvHJQ|3A3a(!ikp-z@)-%S8L&x7ZS&CD^UV9GMn=S!6+8Sh1n~_QV z4m7`gK#e>IR>?=kE$Wd z63 ziEL0ES&%9+5F^YuHh6z>uVdDbodQDjnd2V%i-z6JSNFTpxAZtW?CInQ4t?eLo@lcV zKik4eywqfyeYJsEa-*K{;$|(Q?$$>}=k0ekBNDG{K&pZPcgvV?x7Zf$7TSW;L_!o# zKm*s38n~u5!)Jso-k;7_kpW0g1~1ha!#;Wo`n*k7c8T!ZId}H%W>=TPjm};tzi@*u z)N*34R|*hbP2i8|#PP?qWBKEnG1v?n%?GuK1bWDVbkT!3zPIdf zPPp#F>u`efhFvDDu@dAmkt&_@q?m^Ur)ox#RA zA=pDk{g09^JhP@L;X0{FlOq4hQ2gats_wG#MDr~VVr&lPN7$dq4sp4j7U*#|$)MHdH#A8e+L^!S$A_R?z1p3HfzImYFJA;FB zLb#I*1sx>q(I?55tI$bi!JA zZzA2{J4k)oPPK+5mSOfAmWT$Gn%v>`KtvM)K6c``1Mb0sp4FA);$DIX9K zsOA$Mts@G}G!TY7GI9^8HFgi_Hxveq=!*hIbw&PT+Ma%(Ig$SHV2F8_iO(tiS~BLh z4zI%oQj@Wcye!y3o)n!Wr4R0k7Zj;4$SyGaEhX1-cYK!3@yIm$OCib5xBU}5G;g$JnO0F zRmRhh+q8-#DN03=jMbBmvXt^38LLMiM=b(*S{}&LvnCc;%)n>UEYjyOnY_e32#+v( ze#_q(kHtvB>p3L){Z10`=>iF4+F+^Rlf`=HkMtd!5BB?6?_G}By!Sk9 z{VwE!)w|>?7VnC$nZ5mV-QutA>t?UumiaTdWAO+itcpQ$A^`;%h|?7E&X17Cv4q^m z{GEwC`0?0_5nlZ(39kEt1k_(9evNWso=qAv+?wq%Z`a7$X4B}jo7(7c zz_Kyuh$FiFoH4A0^Tu!CqDd88o=Cu)Lh1#W|HE((2G$@8 zYY>n75JIsR!>@5I@oGItMD3S}dxx|bzf*NOw@YU(tIK#XqswX~t&6$Ns@rjsdAHj( zlWyOghTTzn^m{V*>-9W2pxfPWP^V+`kZwC1)NO$ydUbGYA^{iA;lvu0VE)d*eGm!A zAVQxJ;`MGaaqnDDTzU@>UjGH+I4D8bLyF?eVT~EqBl>f!M$8tPkJ5fM8D%Xu9CKQ& zKjyJcXFO3WC2Q3OhI#+5g5)e0P|V8Ktl$>ny-PIv{ga0 zL!Gf%X!zKY5p}BT`-aE$038lYY>SH!V5VJ_ciuFd?LgQdoc{K2SN+`Qq(Z_ zE8)HrIqZLs#{LH><_3vQs!!1xHH$aL>96i8POi}vlIR7KtEfN z6>UO4>p_vM0}ZkoImim+Aj^P3enkc{k>D1tKW<5nk%sqOO zLEUN^sIzB+n%i7Z^;-xkk&8h&Whp4%TL#J{D?q7oB`CdL1xod+LFp@Ma1AKJ|Bw=_ z8B>NegDSADU-kQf+vwr?76O+r=UqNd;MQ4W05@@b1p=BnQ$WjNI%wI=0!{vW(C}G= ztHNJFEpa)hWv>9$2dhA}Vl}A#YYnK@tp(M#b)YhUg5SPr496YCafh(6rcWIxZc*D0(#~%K$kfibetD}w$~ES3Rwo4@hd?yb2Vt( zUjrKD>p=b0Z=hbg9@JabgIfQFaW&X5iW$E& z?W&ES^=Z?%R@3HD&ECz!nq!*>v|w|;Hf-+Gf~}p}u&q@Gwm0d*j_(O1a1C_}JzT{# zy(G>FMa;Zfm^Y1NFn_8~0W+gtz|49cm^v&5W5IGT@>>mt5$nJpX#?nIZvy=Xo5%H@ zZXMNoyKPvvVcURi_x4`h(e2%Ou)RwUc68{%&K5n`Rj&`bKO1~sAcbp}+n8@|d}Bd) zJ=FfhHST518+UQ7tvUs$2EQPanukp4SFqr(1T*ioW2PY+Moi*14;!a%9WpN1K4?__ z`+(7_oxO%%c6Awc?D}drysOO!cDES9-bO>%|Jexs_+Siw{A~;ez9+bg9wcx*eFyo2 z64pW+=Y;76T-Qk8xNG{({qYu@T1ipTl^tH8~D7E#-J%#(lbHqYJF zWmdYU)3j=DyJ_vdX4Cfljb?-U>P_K5oe3QJWCBP2GJ_+}&EaU}1OjQ~lalD+E@ppq ztc3ycAj@<38`o`IS13+K9yDWwZ8Cqrj<&Ry>9D5T#%*IK&3D^Z>#&_|)c8Ful+69j zRz(LIEuS8&xBPgh*0S}`C(HgL?=9fi-xl!a3kx{$)DliSvVs#2ClJWsT2ks8m&1EQ z8*{fQ@*o<{OBUW6j&fqdJdNpnT*G-?9IK^Y?OAJDSuUHJY&~~0FoO1cv5EfUGcEN{ zjdlK!Pt@|`@2T(p{EOOj;w82BWEBO@Jf*P4(>ZYvi$bHwGBV~hLL#u zrA_w9=k$^@&uA~sRnY6tm(#m1l+nj8meAo+Au7)XF6CkaVm1RVd`}>UY)}S0sNwZ6 z#_M5?ECxBqxYI?_FOVP|9!gVMJhW%l3r!c*xY1X=cd_5{hR5Iig5!1YnSJo_3RcYN z$98Gw%4`cRmNF}@6f>%?7BV}o6);DyEKJ zB8S9F_t`nu@>vgW+++RgRyM2Ub|!21b_NS>rLy5>G8=9r+Qan(d${&Jfda1S<Mi9peZ$Q8&IPP+W|R7PF>4 zii+2Ry^V~z?7 z;8L1P$csqXDHY*bb03D7tSkz&-je6X+L!IkJC-hTKbs=- zy`1P4dNa;7K_c2EM>^7_TsFeFS}x3`8wE0;5)+veSF)u z$J;>qgj-0n|1MG;c7VLbvr;Q#q$fX$)|m4k!e~WaC}ne2pzYo?KgVN9KCWluJ-x2P zcm&;ybdQk?6J*MSxRuHWxxH5ibnBE46o71?8%PJZfmDDiNTTk1PoRu^QVpM#=IFx~ zYk_Adb$hNS^>|k6+o<13Rood;o**Sw9H%xrKgM8Lc7)}ov{1&Lq#%c*asDo6qI^Y{ z!@UD;g?L6u1&Y#S{X8Bh_zM40^bvkl^zi^Wl#GuM?)tccq^~4D_}uUc3YGK-Oq3M_7oKvv)j((c|MK|>$hC8S?8kG#X)_m{C#NqNe8Qj~F&3qLQdPLKAEc z1jag^@Qrr8;2G(4-90?`j%!$)v~y^#JTJ6Tfg94KzzZ3Xa|#9-=RmmY;tx`MKal*M zKn*>7>t|))yl|OEYQ3kBmtj*#MdB1voUw-FWgjA0_pXvuWKs$EU8(3Sqt#&@-%}{o9MbECzrFiUA{9|$q%>s`$;(+mGV}M7l>3)RLZOU!^nLZYVfhBj1MgXE z^vR*`^2lNza?5Z!>6|XS;F#)vjhzyG%QiVvl9BxQE+eV_E+cVRikSeCcCjGAih?_A zJjn2S0!?ItmdM38)5!-9Jbxe<_a9*P%*Wi3QLvaKJ=jI!O3sp~hmvApr7AN6OZ1lb zJTP19QDnW%^}gLcrvly)PM-TI);+%qjNHg8wA_pv)SSn+sM+pN!a5PB)H-h@qel~&8tFt zzVM0ha(+2wJ?{~7yM3ACKHGx6&!zGKN;Y;R)aK$1Y zu2^Nm6-vhU1qM@*5lv?}wMN!61-O`bLVX;L|K!nguX z89#>8CM9sjv4OnNX-ApAd>Jm_z29Oew9_K= zKS>g9wTifXwcbqIYV!p))%2y*YWvle)%^8l)!v&;s>8P%Rj2PXsD8X#|5L*rgAapy z^xwmNy|-{s|0NvOfA)Q03?uJc2`R%G+)Kdxjr$NHF@O3Y0}x<8h*Ql9!uhh7*f*RZ zEIcQK*`y#wZ`PbjX*QZ^*+Q9X)?&NRxP`aWutm5+zcpx$ZfnAC+HFM}wAwyw)N1bA zsMQD?H5*}zdIN0NsDmBf(-#LqDsUbYL}UI&1`+)q`BQ)lz~Sv=V%xHY(A)PA>yDGe zs_PoD=$0mCJt|_xJ-XtCy{6Ok`>bc`^s#1Z^*hhg==WNvIuO2CWgz2MrNPI`6bBoY zEB1}8Q0#{P)JpvS|3fd4fjmGCl!@0MHlL6{i)2Kf)vh*|R0#=^P1Yo_hyI z=1Bm|yEFcMJwzUuornwy8AMPqj*r)X{eloHWB|saGl(8$ZcXfgP=)=-AdVAx>^YFZ zjC~hbh!pY%3G98ijr%Tcff;fLI_7lFB;5ZYb_v2JUqtqB0m{YCLoKRr>Un_gDmNDA zU>5F!K@KD=$MKN?SRw<^K?b0Xy%s59=pjl>NJ7#*N_FsU=G*B3}}Wq%vKB}ou`1L*Hn-Uoeq-m zGe9!q7mzHN36f>AK=L{2<7|*>M0L&q$q^L%qcJI%Jt_rrP;*D5VBYrxXK?-YCwe%9 z9!?w~aPAb2kGcE$T^wHn^N%U^sW2vk3{M=SMbkh!cm_zv{sPizvp_m;Hb|GCD(8Uo z`?(-pkLo}T&l{J9d85)WcUa~h%^#A61q0Hsa6sn&6r4p5Cw34xdXT_rO6_@gz{7PXPtTsUR<$4)OuNfPB;}kWZcs@;P%s{=qzuuR#4ZALQ#$Ul)wY z4J{a!hXsSE0aU*{Eb5no#Xa({g1xTZge9{${q*8sEs4b1)$7jgVM*e9cc>s^CM zph}$rs`k@B#qAeR@tpaeU$9agrg!KxNDSY59UYrZGAh->;Y+mOlOoVc|c$H#f0dNH<{7pl;-nex0PHeLC68dUcAH_vlot=+b__@~d{^$`h*Vt`gw~^11PNUFe9fol$+6~iJwHXwwZZUYYrrF@l+6IIAwVw^U*47w| zuK%PDo8Ifg=C^vV<+VO+eP-}|!4+hK=h4GW%>ME?FEnso=piFC#yMdjNk%9tVgnSt zsXY|SnVnX4^V%()7q?i5mNl6Nu52)iT2pVDyspkPcl~FR(hW5xFE)NKsoV6oNyp~b z#>3lQ7{iV#-1Jvz3_BmA9-4e#a22!7CCs{#nDbPy7P>eujPV||z~BC9w@4pDVNxeU zTfB{7HnYjbW_~@*ami0E z#oDLcs)w9(-97BM%drthUH%w($mPkXMwd^c_qm9%d&wTM(^ZVAa}%R~g;>NkEMa^D z-XE6K-447*+(zJw&jh>5z_V`jevbp=8r;u~-{XFN!cLF(6L)xsiFKsbLrkdg6cegE#dtEVIbt!j z<}%JpBVxnxaOsC1UW4&IU^JeEPRG5-g}5HQR&qJoT=8tAgT~1)uMS5;f_fYZit67O zkTkf#KWoGu-{LX5e5xnZd+(i8=Y4WYt=FxoHQukMRe6hPTfN29N*^(0i;tLG<|8Kl z3bBN3&|@w(a$aoa^UA$9{$d@(E1u2qAaVk3#OmO3!Yav`_>GFkW9>8!N4s}yjPUQd zH!OU>u8_E)^}%VQwg={q-xg3Yx!S*e+E%|KGb;S9&MNnNF{{k)`^-{*F{8v^Oe^*m zQ;Yn?RZAD~U63HhBe(CmSmqO0?0~7w^(> zcZ^TZ`lt}??Ge$#YQmDoYz@txSP@)0wJfN1W=Y_I*~NjE<`f1#nNtw-RVP14%*qcI zGxCDO^xPmZ^;d|cjK79{kR`88+S|7?-iCC;!|0y4nm7>WQm5fW`Z63!H&WP_YOTI2 z*{MTaf>+ONaY5QuF_FV7q7ufIMPyDY4lkNs7*;(iKXl)m+>mqgazY->%MSfAH!D== zWQB=Yyw1oB71PPI=KS3T{WkPQsd?SB@WWpVFT=HPC%ywUPd}W@n2f_&y4at!UVe9` zWy`uW2d$bEkDirD0ovsW;loSf;>H%nq)pC?%Ab)FS*ep1v3p)d#OVcT5%#B}IuDzd|f$8#ZY0ncSqkTj5ivDm9lJE~QZWXZOH?ym8o5umJUiYh||;n5u2f zvujtDEc*ht;$$l4rV6+8@+#pQTe=0FyQ^15Pg(P-3{%tdYKYU$RJ zjVk3uHkw5R&Rz5Jy!vJ51P)EliWr@e5kDy*J!5)YYLQNK%C>otDTfz^r`%W+p8R$} zM6#F{ks{_qCW+ZmiDKrj5PF=CW~zKHsNi#eJg!Dccpu2%ShfuI6?H^Czh8V?xelr- zR?3ubF;*@vvu>SN;@CN>$fHkMf&bv-{IF5+xv>*sa?++oW*5u~%c`Cml6hc3Q0CPI zK^br62WN=6WKKw$m>rfXW`?Ew7qPm@hf(HpK>-h_xh`_;`*`|4x@TuuJ8Y{Mg37Hk zP+q-UQe17Qn77qZBdgNBL)sR%o=IiC+HoZz!=sC1#)cQAOb#i`n-N&BRmU%X|6Jev z-{$$|y_)NnC+7I&irM}-Vpd?LnDHxwK67BJz_}oUYt;Xz6S?-?q&QxO9!J_U;?GHE3{HWz@)^ilp)WTXLrOlvm8~EN__Q zQFd9!qx98m&k`|<%=9V}GkkcE!LJZTjPD@DGhgPQN&BYJ|6m=&cKRS&wke@#M<3+v znt*KjK^c42NK^J~R8H8vsa4EQrw$SIo;^Z#1oR82ix}d&J#nO0ZT2|#+AWh@w(Xtj zyzSC7mzq~oovXz(=PEJPrBY0FD-%dOzle_Wn>+8`FKghKqYnW3*`6!2m zJ!3cTy*$o-&#STayTtg-J4wBm;7}{Z{|aFZy!K=cBIyA!2fOG&RBh!sjMRS_4b)$U z+9LMoAVeLXiii_S5PDKyCg`M@g8xZdHJ_6%tvycoXt|yU>Ed)euBZL+jJ|fq%d~Bd z?-{h|_~pSi$6gQKbVLlbJ|u>d5u5gteg8!`0yhI#1Hv5a&E*{67?e`~WmA78upT7r zq#}aO_Cf%?7~hL?;B!e2UYCtz+%H?nyIgiuak}i;(*AN#Yun2)?KfRX?`UAWJ`>luhrB^-7FN$7f=S3g0bE2Q=88P5jymX@n!yGhbQ2%obwrr#Rr5BY*A0&(# z*yp?~-2QA2m+OP!baN^kZY_fS?KRkZXCrLy*h)9ub&#JmOFI?5^UQ)}A_X{*k z?$@<8esH?YhI`L6H{AK&*66m-GQ2H18r~9}f5r7cp2I;OXlEhyH|Ie9F6vKe@Nllf zdsBlu{w9Up{f@ACI0#mcC&J>14osge#l~mrVDj7)#xHCo8(z3d4PW}p8oY{>*MF6! zxb}6K@|rgdDy!dIRbBn+gX-$%q6xKC&qT{#af;7_Jsg8-&VeG<?{U0hg2Xp8J z#?lA!J;C_&!c6GHtfUXK@C6tbtiMoXOilKs3*+^suQHIe7sKg!jA5*aj5VE~I|n`J zJ#=M_N+;$;i#|wO-gK>mAJkb3(o$qWjcrjA)&HSs4kqv(@aG(K;2bo$0_fcY<~{(X zeFfu!-Uk>{f&FXES`RJusT)0(-snc3suOEewAda^*2}b_7oy7CC~_Z@oLCMC+rpPb z!UurxA-n-a)PE2o`943Gx<;G2f9N=f5!C);=Ru68_MfnU{)jET0S|g3A=Jk4)TWsb zGmEGXw@Sq99b})3&^bmfkelSOjF|IQDs;X}#T+5?jeMOWq)q%aN67vPF^X*%!Zr*Z z&G>A`*va%`<}&VbdIE-wyNU7KsRx6oi(^@{l1^P#AQAH_WW>DfGGhK-shEF+oFmtz zV!=c5R`$n&Z?az(2?Vv24R|uI-LyJ0>t6GsZLSOvYWpxNGT&n9~z*qz>|> zHxVuo%M+zyc^18?VzO0Etk@y{W5s^??|LWXzv*38_@Z}D;iKLwg?D;i6yB^93a?fQ z#g}?QnW((b6RN*LjH9+1L5(?{{=;;(Wgf?4F?}&zYX4=csDCz6e{AOazn4U;36cI- z6DRv^O}gCIwFUBjt=*#Vd2OxYr?q<(KdwEh^kMCJrFU!ZD!pF&Lizc+&q`0%3FRm2 zgxaIEqQ%3tq9sv(@C(HF9vlN|{wdV|b9vR}c<6CnuHu|n!#UC92d%fkH$yj>FGhi~ zpNyj9KNzJbyf?~GdTUgo{KlwS<+af+m6t|`R9_gKReff3OYM=-Gqw90KB?U`66&{& zM5|jyqV)|!(dIh+p=-ZDOk^7-^yBz*Ocpo!AoPRQO<-L5K}NdxZmf?l8!hm`%t_|0 znXlYyvv7r%W(i8q%`#M;nii@)Hmzv!$aF``ho+4!@0*@hziWC!{ifMd^=oDyTK{e; zG%uNob{9;A*7=P>>)b}s;p{IElc>3;@ZOl)jUGf__M79ef#b1pI=-_u=ac1XytUql zmzy?ApKkJ$f3zuB>A|KLm3!8yE$&$7w!CFkrg6h+TkC694Vr&io@o1<<<)irqTPDPrbW}y4!?*w(5l+n+M>X({Ma- zort?0b8*dk6)yRhO3!+4Ry^tD(c+kAKx7m&!i7oOkxy z>3n3s4(H#rw>v)>xXt;?z-nhPsLDkQ-0C8cY3At+~Jwice_XF zfNk!zgR0#dhg7+pA6n`5VAvM-kHgB{#n3W$F{IR83@-5ygNi-Gz~+buZukwtg`lZ88KR4$VTLjXLah`VLY!Le4)SVS9~jhWyMJW&ZGMTp zt9SNaqU+TvX`wA^d&@KUc+Ba6N6jwX_GVIFO3Li{^b1&8&h z2#V`l9+0YC>Yq2H*l)}5Lcg7(@_mnw$@RTCHplnP*la&BnvBfy6C=p5Og}NSIbsg` zzLHvNqXs>}R`eq^@XWgfZUpem{jlCR8Z{n`F$=IKcCDm7)>NT3#;#>`lxy3{2;WZS z;UPUrLZkZ?g(MFw2+kgs8&oRJ(O_WR}Pm$6mTMn+CO#-_QcOcec~!fZGwqHRlH4$ia2M@ zvKa49#Zf^$3L+!>a1f?3)-DesNN4_*3$ILTrQ>OGcAX zvEgE5bHrlmZUaTm1qFO^k>VO(cg}^#;CPf2`x821XUZ_trp-iE`f^lc7|NBVTdEeO z+PBV2cJG*-v3a9oV=Bi*NAH~&8GUY2MD!E#Z9-(U7)QpC zQISz%WOIZr$G}LQb3qo5sdp~>bKN_f+B;5y-6`#{J$*2$GN)lnwk}Gu4P*oZkIw2%pj3Z-0 zv=l4$LL^z}L<{eE-QoV?1?NDs@k`8p`wgqNs2J@{1QBr)0HEMzM)P zYLRt|#6m~SxB`!k(fR&8!t=uVhUCT#3d~6#=9gVO$|tLCtXI~F@m^W?$9rXd9_N)Q z#&~Cn(cT$iq))mS(HyZx!gUhho(pv+b>Cs;U{4CQFV}y|^A%87(gQhVW01LJ4$><0 zkX&ITm$1cBC8pfIRb-i4`>;~qE4r)`ngo*4|J;BHpHRg=rD(h zyTcr|d>ZCZE`~XjiJ?t6mWUzE;c_)zIM9FKTs#~@{~?WIkVpT4`YVI$;ECHb5nHE? zsQO8W*tq~_B--=+wG|CXS3sQ zf14e52iR=?G{B}-XxnTfHNS@C@wm90S%Nbv)p!Zhs)GjqQPi_L~mmbhJ89)y3k#;cgZO z?sl`-|GAragXm$tSM)UBLoebNytm~Z5a!?@^>;nTppsr#0pq7qgGN$=`O*jQJfa1+ zV}szreNs**=EL#iO4y$=!sb&}GPb83rPl$FKjtecLW1=JJ)SNra!3jPO_B4$FJ&0oFKZEgO=z|382VCg`*q%{@ z_4)3wyvXmvxHJuBmlwkHw^i8qyD?1uu!ixU&XNsReWixiBIMRzOO@BZUaGk6MuXDY z8#k2KU3;gr_D`X@_IJ@@%@v_eG@5gP_riWY3u-wB%PQyrF@7B52eJQd^Z{(>Lzr-H z&iWf|VQ{NI*4-JyHTfA>eQz;V@q2Vv-roSd2i92ez!}RQ`eFH_Xe@i2DOviYN=EnT zajEW;r&8UALb~LEkR@_NzBx_va0ll=1wDv7j(;NK2mfz;`VhwSAy%^%K<`OAEPvJq zOP>$N;un*!@a1eQc(oYwUa!QQ*XyD4#sah6I$-8IZ_IcXiRtgNFzrJPrhPn(sh^%> z>N|m{Z^-NavYS3sRT<|1<0mtIDC4`c|C<=!i1Anc2`qjFEcn<4b3b*%Y@QoCr71iHXq}^FT>dHYccwVF?|6W`Vj8)A;NeTMH>BqD(dy)6TxM;=!~alL%{jP}_WT+`=-YJv}e4o^jG@OgV07d)`fKDzLXC19@?@-r8T_=HMT{Go{SvpWMr5V z33I~DfTWzz^lC;ne-yKi9?)UV|C98e*ni!t>^tL+dr0r&6WC8MCKywOK8q54kJjv) z7JZK{to7)@T93X^5Q89PZa}|^W(Ao~R*(&Z4*=GsusD*${Rh#P`mXaxh;F1OJ&0Zt zA^K7K4`A)V07GI;TnV-RKx+R%$@D66=vkD|GpK?Xwu9`Wc0WcgkXzKyPsj)S7$NYD z{5A3)zKjz16`~)tZkG{^O`X}7?HE9{(lht*@hm&Ao^2h4&-$h{h(1Z z`1ypW&^nvD*<y{9W=@6r>xLvHH| z`R0g0eK^)+4E>m?%!v-iV?O7^!dcXS^n(_y!?z{o_@e88k4wGrZdoYaERVzM6={-} zEAk{S^vYzO>D9^s}CoO874%$&6o@p+vN-s&&K3xoA|Y-oiC8=P_1*iUlHI9z(& zI9~Ruak|`Z#`y}DjkhRXG~TXs-gv+AS>sbGr;Tr@oG^K=der1|i^C>D{h*1^Xe9fM zg~q;LAVzQuCQg@T!UL?X1Hcyk1G~l(u)?s za_20f70y^BE1j~)Ryko_tai-2s>Mo&d9s@)_y)R1aY z#m$&oe}NcJ&tVp|*0MJLco4>&@Y%dOo?G?D-A%*sr`RR$w{u! z$y;fkW3cL8hbZ;k4oR(d+GlC*urF%AeREZZ+Rb}9)@(l6xoY#xE|r^KciFO8bSWog zn?>hR`@iYL>%S0_I0kd5dG$2tL9iWW?eNaJ10Hfe##Q?PIOjA8$6XiUu-iH`x|_@F zb+?z_q@2dlor; z>rv<=dK5T`?)lE58|l&K#B7}m3pfMzVmF37q+c%%bw@`x%9zt_whLBH6Q!D zS7W!&MwxmaTlwwYt|~QNJ}s*}L)uh&M7P`Gk=(J&J-bVZTS@mKx7wZst_OPOyI$&( z>-wZ`j@w^-vfV^4(lg6V^dQ}uBc{`XSxn8V&vuw8;ln0H+}SLT-?{hvq(=`N^cjr? zzd6_$poh8uW0`FMo8+tfom49PywuBm12s#1BD9LU6FL=mXLQT+D(so#S=A@ov!P$6 z=ehnFo)7z{dw%Yh<|+E7dWqho7wOR)F_Rw5QhAPp0&COcaNkCP-<+g4=O&LMK3&n^ zKLR_0W?_5qa#V*H%2b9}%9jT_sFVhIs22tLYZe5AY32FHcFy)o?VjnI*E`*}qF<`d z?g1%2r?iuN?hj1z`8XiaSM(!&NpI4#IYNhXVFfj>u`J$ON^p(Z_q?kFC%vR-4Cshm z!9!8Y`k?A?T~tIENXjD2<%`2Ns}zR0spp0IYG#K7cgPHm?vfsq+#@wGr%!S~dH;lf zdhPgt6NBRX?+l6!_%JXwKnx)LVgf`T(yKXQF2`Y&480%;9#VH+apIaYUpEf>a^FEP zwSV{^R7FljMbu)HMX!?-MVrbMMA<6mM!K}jityG<4-eEz4U6cK9GcK0AvCj3Tu4d( znBcmBQNc$CM+V;-91;9(P(-lMCjBEqL|@XoIbuG?VJ%;6%z#_eo#)-S?(WO?eZKS8 zhqXa<)BtRWnS_$K1t^MNgZy|C*_^mdN|~`vEz@E=+ayQ(Yb8d7b%~3N>k$)?-X|)e zsDF5P?ZB|`!-GS@Zwv_ue>*57TxgU2q;F`r=+hjrP|D{b@R7?-SE)Oja&Rz!wFi9d z+7_jbinzWg;rELdB+f%_(n@3}Z;;JMvQkVE0-}UA=}-ERKF#5>Abl9%krg$c3->-S2MruYkn7 zetrpCwS5xy5Asgf1h~K*C$T&X%4?WfX@Zs4qrRZdN2nZgIy8Sebk?& ziE_wG>x!(55lGLRffUv#C1&eO;KKljuIZMW3R-zB zPI+&qoIQOVb1wFC%zn|=F-tVj$0<|vCcTCWrFXnOs>&HLe$4|BjVsHOfbXZ&3HAPI%k zK4tVDDtaJr>qz)l&46$95_s3Fg=dY4jC+l>oNJAvqH~R>ibHi^%gxo%t!=B*+FDl^ zYgtwA>}XMau9Ice^G=qPqLW31==?WZcx}c9j>B)Rd>#Zd2b3RM8NV=}>+jTn(Oanh ztGEWgT?-y{1L0af5zag3!fEFUIPO{x``zY}&AaWSw!2;BH|_RQwAvk^V!k`Mh3W1> zjg7l^v^Lp&rnSkgXRS?kiZ;eOL|fxJ(e78=w_y&N<^jiG8^^zt@w4dzCD4ZmsiFR5 zJ%IflRoLz8iB0=Qz`Ai7EDtP%#le*@KV$^6LzXZ->>x2Y>?yP1NQkV_k$Cy_NAeU6 zj@ByaA3de4f8?pM{-N(G>kbOFbq7R?wT+@>bAD$I4sssuNT&X0{`1SIzqeBVQipp} zhuX6q!0ZsP;g}{2PxQt5lcS-3Y6jMwUWB!0R$|Q=Bdk7aiB;zuvGSY`^jK7~;zEXG z`Nc|^WfzaiEW7YfX4&~K(q(6b+_E!5e(7nU(3}%Z&m;PgRXMD|VE@yZ`zZP#zPnfh zLLF+s_y#9fgTR`UC70V_@o#;x@Q+bgc$GB}R~KOZH9gF`z8-ULSYXagN9f%0!|Yoz zn0-4Bv+nG~%)5VL=B@XbbyHy04T0I$|I5CZf4s<2#?Pvz{;gvT0DTCj{fvM3pZLqp z(F6ERf?0RfG2?zWOk-Wj)JJ15`O!2?dOQykpDe@pCp?4VsTsyTbHM25z8Li)79(F4 zV&tm^jCg$;BVK&Nh^OM0)TQ%T!1x$WMVzBY@Zvud zasH(;ei-AsF}`I}eEMLEE&|i(LyWl%40-_c{|d$gJr)IeEE@D!v>2}|H7J+p{ak0??=toy z#(fL+6ZCSxm|#pPV=6JG27QmV{LD_IJLyM;kTImGEepvSVnKKVvR3DxwluZp{~_8^ z+i6f&X;NouHeIRx+s`DrM4wndw4?vf&WD8Y?{N@XX(SJ#Ln*0(=vYtou_?#MMRJ=w z!;em%@Rg1K+(Z}Bm0m;>-TsYePtC76l%Glc-+>-PCu;w$)5#*ThM3YLu_s<6gny4A z)c)Oa*!5ykN&jLyp%2pY2suYTidUl^G32!H6;mxE%yq;8nSChBn#pDJ&pK=^er~HA3Qy$?S{h-_QfNs(Qx;|6jD*2QA z(G1ap{qIlBKe+QhKFB1F$JEjE0yrMi7U27|mH0f}81JXs;LQwIyqf8c7qi0gY*sv; z%ud6j*?D-VQ-%jRHMlot5AM!6hTC(l;M&}W_-+0>Tw3r0=NAf`T__}{$*F}xra7Vy zwSH6lk<^`2x>5hKO>=mU%$T zKasj~HZ}PCUi1M5;_H&pc(-IKo-bK|N4l$Vm;1$TF1N$g74EpA7l4Z^BXE9I0?w|= zz?s$gl2fa<$edVHCv$vFqx9&Sv(m$B???}x9B?vWx8e1!6Gcj-lpk z(w$9p5W4gumiNJXy&-s}Hy-y^&c=<^%W-9`5zeo(#u)==oLujNtMc{@LS-ANA=OP73FV*pI|ma~ z=tZe>KZ+(Fv0C`Nx+|Wq?TfoSKlo3BsW@-A7^irC&@mHp9NuV;1EwC5{iXrZ2Ga=H zJ*M&UyG_#-cADlZ)thcnt~1@CvfcEs>Ne9WYSm^>)V7*^X;EP&T5d5DEy_vRKdABg ze~3}^9HuBx|0`2>w&0_+H9e?yc(A@R{xs^3vnCU8+-yD$S**r>tBu%eZHwL3t};8V zePwr8hsbZYj#1oZovd7Am91K3RnlUs)wY%uR{J%!Se?@-vwF~~#QI~aB5TpAkQ7)8 z4Wgdk3^A7dpCQK_upO%v@zp>H&kR*?!$=+H%zELN)fhBxnuEQzde~{V0XsHZOKR<% zq&4QDy6)QJKs8ra+tCicOw=A_Q&?vFn+PcVgk7l9mskZsHciQIKy=$9eC)#G) z3C%1!(T21pt$s#KVE=WboCDO}>tylNkoD%qJnznw-*;f$4UKjqvDqQ z3a^DIqaRe_V^&FVmw4gq805S+LLxaBc{`P&;`D)qxPrnJWmhe*k;ySQ~T|7(?pHe z090_VR;k}S6#1`2LBIw{Zh)0+w!ecyroX##x}RT*RKHM-B;Od#1fSIQaX$GSW4x<6 zM||gDs@#!J8FQgItx9gM3;f1O{uw1x9Jc1SGeQ^3UxU;a|}u z%ztmUQ2%q?L;N0h5BC4sHP~NtA{~iVbGTjm@7Jz}>*>K*aqoo#&po8}toP*mKKG@R z2KGW>@Mz?Q&O%n$Qe=edBQ@MinjCH`pBUz>93SST784q%5fu`l84;4$J}fx9V{mYJ zm!P0s-2#HnboUQ>ME>gPA0#@Hj-&(8`We24@}2~JXpVy|HJ=Ohh8N%WeW^PGS$h!L z4LRW>kQp%@Y4n4Vqt+rZ+C(NkdXrpil%rBql!t0Wq`!JtL|B{Pi1>Da;h7!%!%MsP zhSzuV2|Lx@JM01Z($zambS53WLq!Lo^)p-&#CN{(-Q+l&Vh$QTsJZ;8JNfEY6e>kd zWGAFY4?#-I6!5FF5FfV+vGE%uQSp|t5pnhkVR3FMA+f$K17ky4`Nzbz^^Hl_@`^6* zv!{IW1= zj!3S1#?cx~!sk2kO=;#zDIuNbqG|k6{)c@x>7Y}j{)CV#L;mkn{ zbzdyCZ(=LN@XXqXv@r-xpM~HIT?A#UML?#B#4poY)+f_Z!7IZ<**zmb%{3!J!#O>v zjYE1~yUpn}T6XD2I@+e)>1dn!v4dTz(6UPr?f)iOw6pt(rnzvN<8acM+M8og%lKuC zotwz`$@Cx6=|5z3LtxHu_~lN8Pu>D}<*kHgz9HQ6Eo9vC?POi@T@{@2eUQ%Yzjy%VJf` z%d^x>%PU%LEI*)bQhrn2r1ZVINr})fDJDh#59E?r7QLb+2y;1-11#Qe)-OSbC&l)lOI`=Ko5rTvkR%e%UFX^N&Q*F z8iYF5Anaxh#6Csn9q5eZ2M1!=p|Q|CJOfJ(FT~;_dRTPS5DSl5V8Jm*%s=LXdB>wL z_e3`4oT$Z|lV>sK_$$mkCNTFXIU*!;5C5A({`5oRsQ;P&{33cVTQ~;P;U2YIgRf@| z2DRv_MrvSc;n}AYG2=`pOh3zWAkK}()bmp@<-$BnzOWpVE*fCsC7wZX*$LyX1Yq26 z2^jlF8OHp17-N2agt3>uV(cY>aTm#jf3u5opvea-V*IoU>R4(4E;BcvlzN_ts*-Ju~#b?}UC20@3$j z3i>?a{*TA!(EH&#^u909=iYy*;ruUQ{7lA=-bN3IeYa=dP4+YXVb*_~pa*c4H4vA9 zA=jwGA4}2iSsV0u-W9!G^h1xA!_e*3cyxI^1D#$kM29yk(eAAY+P-su=KDaj{*Z=N zAGbl{(?#+Q>hAI>$R#lMK)>5Sm;3Y|UV(KNVBH1iu_(}2 zQK#q8j-S_s^dl#>m_kpz+i z!W-Zxgp?XrZU}@t^}m!?3AMkJYu!@%4>I&0B=jFZ|3TtOf=Cp9p2TY=q4t-Qk!rG& z>?bG4WpWqa@fu(77e4Y6K1hiykt6KL&j?wzL5Vt7nfhCiK7jHFc54cm&p*?DP%`Gv z=|3pC@z4GwfPQ1Qh9Alo@KyN^J}JM%d*v^9qrw`D7OcPEAii!z zKZ1tR&j`iA^n$2!TleLaw58r{KatRXXh;7+(~wvZ`VX2u{Pz$NLy}1rzPByFx3=Z@ z+HM>EYPTC-+8xHH_UG`S-EF*S_X024f5x*80#E4&J?Ts@f}YT0`awSh^dYo4 zE(1sNI+ZM-&$9~O`y1m+e;a)4?}GOOeDOv*6tA^o@KQS&F9v4e*}y_P9aMqGgX-{T z@P0fPd{4Wrl*#GWa@9)))J_PF^MzBqzI3A-o z9wT+=^(@2t5e9fY!UE4nI^gL@PdpkGhzFx1aes6I?v6>v?J;?{HMR^l#%{y4aSixm z+(}#>e;wy1KE>%tpKx-Dz;SYPs=yI)xEZ25(?!%w0G+>nM)T z`VB{P9^>GgPuS18oQ8Pk%*UZcTX0};9riCifPG8OV~_5A z>|FXDJC+J;Un(TEWE-hzhUiDnVTc0zufo`^sDInz3(rbT!b@= z*W!e(DUL3+!=YuaII!Fo`&WdZK`#b-^^&n?Wj1!LEWyrIHIf~x8YFeAPD^T6-;t?a z{YGZ%8X>J9TS)op{~`vm|0AUI!Pt&D)Zh!$@n&IL`cCv87SVrLO8;TSJRDlN5{;`j zpka+QcCU5B&UId>*AGOUegw7~BuKUyq|4M86iBPqS4y|8-z8hQ{$*7H<4Q zDSzX0rQD6*lyWu-rRnY;sx~|w~ zI2^l-r=yPN=G2(3MV0wRRGQmhi-j}FEWBk(ErX=RmXUHrmI?9&7MY6q7R5@r7Tc7w zEgDs_EH0{MSUyorxBN>r)l#Ss<&-8YgwoH5QOv<~wqYUggz>Mg;(G5ouD9yTV*dv2 zJ1`xLT8qi3vRr@)>s2V*v;ieHRw%M@kQCZ_Nb_y|Wpi!9 zs-@aAv`Dcz*D}fGVar6D&n*&cgj&3fP$epHHbVJlxa2@j5csy3Inbm3p-NYY)LrKv7naw#r>ib*aJ$_dVi zs&UR)En=L@)T5p2TSYpbXdU5vr**jV`&MDjLY=fEEkd1y+Rt!1?%%H)52^XCtY;1w zf4>#=jy2caZPifb*b7C@qmbu16WR2GGThf9&BFw#9@dg%4@YUDhlhN;hreR1dzf;J zdz@O7dq&F$w_=Sjx9zP%+>W*hcDvOk$n9P0AUB~w)Je-gH__r}{(j|I#C->x2j`8c zxA@w#$Ci3$GuJyEcvv52G}6dtCF(C{?~4mU*8*tsO%}E28cgB3hCbKhyMFxW9(a z1bTvv9D_Q>ukfYj^5>c(UqRzS)Dacl3t^EX5fU{Wfzb=$AH5QOF^2GoF^6}Ios4IU zi>!N$kAiDVh>~+mtg2&7W=s3%a*fT=ds^8>UnDPD*+vNs+eo2KTG~a57C-Zy^YIq% zhZCE4{)6j3K2&KC*PW?B<3p*tA{7xF(-i@6L*N@f3El~F;hC@u9*OJVmbekFN!AkQ zBuA-JlBb+QQlP@-q-bT^q%_q{i6t$p5_h$oJ&WmOA zML8D_GY9p|eap87Ycp8AZyb8TBeA8D~{Z(x0lBq)#pbaLDcnJ9;rT+$XgucP^~*mccS_EiCd) zV4iOcvwSCPEbx|?6og7Q6eP+T7UavXFW9bNP;g4YApfy~LEbk7{am4_pF^_$%}qWN zj#7T?q$F$dp$g-veUs=vq|kpzr}oRH{^yyw7DXLkT08(IC1YS*Ivqx33t-6FnDylb zFeo>Lez`5yZE?lgEdf|v5i40$ku9^bqFP3;@|bi*#X}jr@~_erWkPmEDJl6k7dQ|0 zdol+c`!dGQO=Au+m;=Uk&!_$`X8lDuYXB-$v8JjkR#gwe%9?Rlv27-nZ(E3EwR%`u zyB@mR&9P*=Jr>t_VNqQe7S^R={*H3Y+i{TG#r*AGFuzt{{x(u0BtO&S2kzt?sATR7 z+3)mR`d|fs4`Vwrw)s}Bf!DAGUCOd7P$)?mQ73(!9E1_MuV|Hnyk;@{Lo(GzC=^B6y=l=`=lV^GcS z17U1K#$LXM@f%r#ahP=g)Wh0mCFprk5#6t}L$}|0qU#@n(dEx^=zMi1I$v9iPS@6< z<8=#kxZwt^n-OS#DKD?V$0fJ6b%}hU$}1P<}EMicc3n{@H5C zJvWE!3wKCg#z5v}DI_nC1210y&js-G-;}Wbnaq7O`|iWO+p+IP%=_{K^q-E>|2aty z;2hBBGQE#$tbe!#sN4sX-bo<;K^fAI)cha20H68-pNH}~fma=($CpDh;!a`-gM48- zK0hMg{zc9fK93pSzm6Ud`)FK6Fp^FJSVmNg)kfF4%?Eyiwf50HBZNZtbYLD(G{ z2SkmuBi%@UGKx$mONj|_B^>m>J&?V;UilY^oOeOYy%WcMVv0;N@F%^Ghj@>7{NH!H z<$v+!{|DYB{FlCj`XAK(7)q#jLH_~tA7Du6KY;!N=szHYM3Y34LGl>8l(Dxm_6~f; zKE^)5*jE_yAztzSU*IdA^S^B3*)I?h`X92r2=%{&dRLB~klZ-_o&JMVmp`Nb0CQqb zJV*e47LM-{)?r9e@l}#dzo(F~x8S3s7Vjl{+2^B-eUUNm;|T}vk<52Iqz`moLExU^ z|DeR{{~+Yqf0b^e6GRJMTXS4m)01j74MLs%gX${&n=$VzS=$9(RhQKZNB4LTh?E&u;_O!f#1lL zpAc&7za}+*8|wd#)ZAS;E?tJv7vXqxnuG5hmf?%mdVJ8bz}pTEc-6rJFFFL^S;ufZ z=@^U09aHeIQ#KxSD#E=^mAKQX9=AFl#Es79@kf_?xYYeM&h`9`)2z!mMNahjhvU5d zFQP5`uSM^n6E$~}573w6(U0TNcMSgOGo3X(i}0$~8a(S|f=9h=aKE<;?)LG)t-isy z(Kia$`X=Jfei`_!Up_ANFUN%e+i`Bdew@)hgOdYq&UWD`G*5LF66P%c6gQJt3ad?Us4o(R~ z1o(IJs-PgY{9M>J5WFKFm}wmg4)?nP_6R?TXh5~Nd?)`4AGO?pO0BF zLY6)V+c8m%^*(L!aC{eBAFquIlgH!aw7EDka|I5}HpD(13+$a^kKJ?KuydX->gR=^ zZhkbjFGxo1f^5_*EJ5|cT5Mg^h{{D5P`>yfN|$^>@e+X|Qb-CG{~OVVIT*w?j3ZNI z@P4Wso=lSE{Ob=a|FBWiVRux+U`s+W49YFQvEmq(yt zc|5kPNJrU|53W^(zNr+v_X)DF)aH1%F#1|3Ap_`ZrBp&}WXr zttFfX?E3*Dj=@IiUQ%kNfPAaY$l|#<>9&)QVmB9wo0lWrULUaz8xiBM3DJ&@h;;Oj z33v3DhB-#ag*YZE1UY6a20Ct0_IKQ)?B{q+#ngyRS=i?cx;N_97=;2YK?B-Fg;_7i))y3nXii^h=Wfym$&_l)9JkD>u{s;Hy^#b#RhPYs9+uMx@6`gn3Rwh}QxHdFvs-+W`JPrttHz zfse0~#M{?X>ggLO>+TyZ@9LAP=;BkP?Br9Y;^=cy#lh#kii7uOWqWU-WbY*uiNen` zwc*Y(`aqlq`>d$>7`x1g>y6G_^L1g}xrZtuyt^aBcNhZwCd1!vE`0rY{zAYycm){4 zGr$V&f%b3)+3P^7rK6C_ANNC+f>2nlg_cXtnQ zB7{J2cXy}ImQw1Swo})sdpn&@d;gzHN}=E1|C#6Qxnz>_?zQ$ld++t0wf3#*oKn)# zHK}A<_k@yTJ>pBAeju*o!ya+PzjTi)GIRr7;|lMf3)egNfwAxw{qPjy;4HpB4EK&q ze2~rAdpYF(7j#uA-`!JKGEw=ZbCpwOtE_S-WtMv=y&^zq6%k6UNKi^;hDlOofmvc@ zWvBScrY^CSTe?P99_b!cd82z|<@?=lcAD$xwxX2hd zn#EXzcS8{|M+y98#9gb2Jvlo#xo(&e8>T33%{;}fS)%AhCq*@SD6%m?;Y|?=Yf4aP zQ>H?ii%o)>YfS^2*PHn_@9*T-e7Tcv^V^+#n||oz+i2+Iy9PA;9q%$HJQal&M&BPS zU@XGhQcm0lcV0EIcRjIRV;4oW^jE~%u?lN7SI9am1+81AK+baxX!Dl;`e6C3kCE^C zRC#a6m)C~X^4!oWkM`Xr?(G*$+}htTaa;eLiCde&)NLJT{X1R`W^QI29>f28;BK$N z2Q}yqb;Q1n=pSo|ecFgW+aHzhrlIntM#+;ij6JqkYt>csT%5A=K-4-U7ZSiv6 zo++p8rCPDQQI0#d$zjK7+3$Es_S?RZ!&ZYFwt&rl$1^ebfH80o`**$6j_{@tGOqwHRq&*W}kGE`6++RJjM5ro=(?v zgq3M$S~T_SK214$LsQQDTT@Q`p{XYgntI~zFzQeT@cnjt-iFOpn-~Y%h<|sH1F(

N1VI#u*e(`fJ#eF&cV3 zTSIPCYw(S&8uZi!4ZQJz20!^<4Y_8}kgI>kVaDAq_&09G2iRPK&1w7TgF}+{QO@Cj z+ZyiK7oU|6S;dWm=_WW=?`ursQVz>DBWNDJNClA1)sNIcLjE5!W{*-?@9RKwuXBa+!Ns* z{tS5t&yoM|qSW;@auD9=%3{}>Np1)UsN>iKnX>>JC*Bn#0s4UiUI-MkL8Eg&!kCX= z1>f`8@HcFSf8B2Sfd0p|8zXbo29s1K4>>dbr&o{9BZ8+aI@`6vn=YZSkYw#^T`Mwt$4X(HTLIeCo#~6dy9)#^K_p|A1e?f5HF2@7=j~A21Tk05-q_#DILz0C?C9yZ%Cl?qSp&?&=RlO@w&& zj{e}rH4wxAqyCTw$^f|_YD9Nfhq|!|>K!Ny`)S58nsbh3JxTM(fzW&C5MR>VU+(B1 z-|$yo|B0`--bIK0L7Xe%Z;^X&M-!RN)dsi#U+{PR;diu{-&7A@3wP%%3T=gNH++ZC zTu#Ax1=Zmed~dW7QtBoUmf-}!?GT}E&5n{^dY*``}D`VXfbc2 z2fcZxB|qpTQ}Qp&40;|t=C%>!=G=v$3z|b$-u-t*e|U(`kI)|vPawC3{^(|{U%RXz zm&aS*bq>{6UE=h4*EIdRYo0#oTB47-Rq2Cn4SKg*tKRNLElBsBdZRnFAl<12>2X~z z^ms+L9>OD!{h}Ma$-O{}xdyKGG3d&DFg%F;51>6fh6d4}whTajcw)4E?>kLDKR%!Q zon`v0mzzHB<*)a9N9vu&lk~4Xnfhm+0=?3=Trc&j(F^^Xbi4n0-FjlHZVuR|X9k|o zQ)n^Q2fe5(Lq68UVL$2IaD&c_Fz7T|%qej4J{WqUIrJp%f1)co#6zT5_0=E4hv>(l zBV6odTw|unobIuPL7@$U8L({Ds^p4ovw~;(dBU)b#eSo zT^N5v=O$dx8S11?PWnK{CjX!#Qw%zU7IP3B_&+f8fx91%>TpveRz8n=NI#A3qt8YS z)w?4m>ebN}x;<{Go|)*T>y!O;b!r5fPJ%8>PuIB_c{)pO%;}j`I%U2_C&-OCK5MIv z&OV?cEO3YBJfj11-_c%+Z?$`#K|8??u-yoYdoc7j>5x^K+6v>v)K zd$2CepQ1CC3w6TUUPl*m_M)vnnoflF*~M$GU7B_;&DE}D#oDo~O55$%Xq)|dZE@J4 z%??Mk(eaAf9beJ<<)3Ta^53*}xj`+UnOdp)aYtI7$W%L>C5|cnRhFAu3yJ^dp+8um zKiH1f(WP^Bz3eOt9YAFJBu7gc#1RB42lK^6DGForhZ|BvT$<^bGJvE*K4`9H<=@QO#Ycja*H zaG$0v9+ukZ#djWfyJ?+|pIUvx)Z!bfX1^3Q`ekd4e~}veSF0|dQMCc>st(v=QWbdG zq%!bX(~7_kP0Ip*HYp7-n3Mz@Q@*e`57-m~UCC%ji(+2gtq_j29q)?M4F zliKVvSsVQ3Yn{KHS^`|u6zHurLBVPWic(#0qH2RPR1;F5>W~Vp4y{*Z=z5d#&|Riw zp(jmC!k#uO4tw9MF!V>$f>47ICix+#@Attt7QdRycF2y{-vuAA%%1Qf-r~B`tFt!y zJ)!l16SOvXu9`w@)e!2Wx-d`Gh6kuRJVI3w@v4kSQ$=L1$|Flv8d+;n61mQ_C~AjU zVbpQ6{HPn9@}l1DlpFPfSx%(EG&|B@k{w|%$+`~)#^Ps;!B_FY4R>NrANs(bc#AF2 z7JpN%59+Iy&@oyQK1+2G)~bngR8^F_Dx>{W9v!B#m{^s@q^Kk|TSc+ODvYf*$&YI> z&5PS+mJ@fhQ+C{woigLz?vxSxJusN1#~4h3NqY1>7`32J=!X}a=?Atvrvi-{5c_Pf zA((w;STC)K9I4vq8LEo4R7IS<%Hme4G~P$W@xdxeh*Cj9lJXNXm6uqk+{D!;*@;c2 znMqsBGLjB=N=v%hIW_5Dol_FO?Ua&eFiTD_m?p>HfeF_;_?@xv0sU}`F>nd{kB1U( zg`t0d)^PUPQRE)P4O3aak8T#bmwODJJvlPSF_#v*>h#=^dmQOrrn9SB!->=!ffU z#ZP1R0k(Zx;>iDC8(Ir0lDnult-ta!#wjOrma?)IDI?oK>Deoln&YFCoM0vAL@O~j zNeQ{xiq9)iY+kKNOx}94sJy*qk$LAkMdZEQDLnTpv+$fhfWb5@8)T6!{3kwJOkNga z@d`dT0skJh;_XTJAep-J6!yEESCgClxH5A`DLrqxQt}rnxxh||1I zq2)0ODNj{!MV^8xDiu)CEdR>wCcc%YOnfSDoA^|GX5v%!KNIg#gAwvBxrcX{6Q1H3 zIF0`gq%#)bZOX<6aA)W9-m{3fqpYia5C*0|!#9bA{zSYDYbv@-@KTN(2ljOZ-j=UNd$)nL;tH_OUZ}OH~Q?Oi{ zV&u}CCg!A4zrShjy4%MHKCs*OGTy1&Dy zj-8-K_Lg*H2h~^N6oWjWCd(~K#Td7ZR>4gWVLOYEVo-|;r7K^ zu)|UFcDT!8hrj0TjMSW+shYL3NanlNXy&dRnz5UcDR%L#q&t4njO_-^*alE(?&2Jy z@(?|-gC1B5Z&f{t1l&n*hp!_m1a7A-lFbfIrPxDd#J&eL{lF8NdSH~M9Gs%b2j^-M zr?^Z!)^Dh!dR7aJriYo#~^2XNPIP*$H~$ z9H(NQvsAzH_Ue1yU5{S~Qtu0i>UFV5Jufxu(Mt#PkIT38=*4gJ*m;9`o-?S|eK>&M zx5M4G9v@(H?q;$gwo{pc&8rSj8F84bkYhx}C#lRhOJ=|Y>G3P50#BN$^NmN<>8SxS zdup^ypPnicPP8>4Q$^3XQ-&4-aPqU|fEz!1nrYz;u0Qc}!{4wIpKsj6ECp}LE@p>) z^Z_<|Vzd2m_)lT~IZo!hgeq_qRp5ry?Iu|f&xzMSop=rM2-9AFwnQV4Unlc=YA&y5 zmXH?c#ukC)=q;w6w{i5_5WIaEyvK6!1Ni+fY=OT8e^=n|Oze(?+Y4?7n!VsWr-JZj zqn?zWc$&(PTV!QC5BJO9b?`6nwh0PJ50r|>Sq=xIR*Yk5HfM{l7}a1UibWX6V%x$a z_cca-jT66q3w-_;*74&iY|lAFW)430!EQ(FUT|5O0{3vZ`(pD0FJk*^fAYS|^#kw` z_!N8wzA%I5LDZ3cY}rRjKTSieuwwhnwD=qC_#f@~op$`O9bn)ePyL13{mc&79)E%{ zcm`GOBK+4-5tc-NE}Y*M9@dAPnxv6#1Sj-AtGaCvExtF}AIPz(g<~ zIDrt50T_UH6dIg==hTv&MB1Z)QHdDx=amRpWJLU>NcizN5+nm-Mnn;)Bs#A{9cV%k zX(LMCj7qQ*h2j88!3mmj36;W_nek85iBE{!e?&2$8U+|t5u+ON<$W;7sE8O92{IBy z1pJpGv53zrfENe_@plx7-;@tmDO^=>HNe$^3pSumY=dtPd`I9r3)gjcUc_A=;x;lf zXz9n?R~btn?ws}Qbo?5csQ5w3NpFPq@o z0p9^EKMCJuhRkz%4{hQde&b(w>UHk+DtHCFd>_c;;$LJ$=wZ+o82e=+=;)8%#J7{m z2hK3~;^0bS2Tl!#(?Q~&k`ToV}(1M0q{k_l?WG=A@s?c^vOTzlUL}Im$X(d zpc3827q`gRxk=X0(=c2oW9S;KzHCbVPA4)lI#V42&ULv5gBdD>i7CoV7t|Qki3jjY zw_*BEr-}O3WR5;FvC;b`PI}AKTd#Kt(aW7<^g@>u-R_#DTiptEvs<~I?p~v(dNk^K zkM(-;fo;0_;C@|x=(H~W~)Lr~;3p zNId?i{?n^J88;*KQI9EltH*r3{LnJp?&+qRy*U}PPq?1!8?USV(sa3hjxIh?qzeNo zbZ%gs&JJqP=|P)xa`0}Q7;;?4hFsN=p)c#;@Xxew#IM>j(xBa77ub0p3`}tx1cWjK z1~Q!t=&Y~%J**FU_tUGrN2B=6(v3ljb(PAgizB>rZe)T&F^M`cCPT-@=IQ9T z5*-;|r9f`! z03U!i2Xi9C6A$W{;X`$0%v7Bpzd)xZEz|KSZaOm6Plu+5>A;Lw?W1yO@60UiF)z?A z^K$K+RjVDd*J|7B&DuI=zc$Y~tBrG?)dq_X)MoLcT2W+LEIQE4^)Bud|1r(~hw1f0 zrux^I+HQfT#&*%ANmOJ^AFpHPb98Wyt@h1z(jKaocFhmcjs=n0wlG0k7p7^mWv(_^ z7Hgwbl{PGD)cQs3T8BE*YJEyAi=S50;&-*i=6luKphhf4jQ}UZ(!b}sLoP{DWZo>b3)Hf# zLe0zS)o9k(DBURAm4>ne5qTE(t^sK~{jLL;2-!7!X9bOKA#T(%e1y!Ep~kpYKn z*8tFpCvRgo%IRjJ&g zS*0GERpNP2MV^;b==qBBJ-<|*=kLn(FenFPuet|grqburP{u4718jlLEk_LiBvfk~ zSu*X;{j|3xSFkF)qg3vls4|}nmG~5>*tb$ezH3zA zw^8|i`<3f=K{^n(rK0Pa(+C?a5oGy5wSqH*`$ zYVsVVdhh9~@mZ+VzMO>U=b|!yFO~WSsyHA*MFH_D3`|o#89RAFWy%ezS9Z__Wd-k5 zM({bM2S2aW;Lns2^qZ0cS%E=Pz&&t)y*^+ZJhu!r0PbU}h&DI8UWPOH>@>q@rLC6@>Vs@`NcjG*&sGDasDbQC3)qGQw(=9^R(Z@ZCxYKcl4Z z+e!@oR0(1KReUHbFo+Aehwmma78rxi;DhsU9`a(3>x~)!nn9iagIXOlSY;s-RUA53 zg<*@8AMU8!a5v>d_$VtPSecPg%HV|M^r$SQMinU~s#;0WtxAmEsf6fLii^Ib*yv9b z6ZNa2BgqksFevIie9iOlCgbl0em?`}em|osL?3{9usWD;!wegsqKI+IkDR5PC@W<} z+bc81Rp~KaN{b0pYHWm(V-u7Vm!ZVC0>#IzR$P3GV&k_fnhjc1{7ps1f2@eO|0q1x zpzs(FeGi{A7ra70aIm6I!nr$;cq^Db0M%^2O2a#;FsiR|V@4}0ZiX`A7bq>>PALga zO6J?w6BGTFkQk=;#8|~9r79*VPti#gicD@&MDkXJCm&O2^3w`Q{!qb5zbH77oZSS2 zg5vMtQ|5u^S1=av!BKd(hcXVr@j(QqL4tx9BJ%iN%19ip)TAj&O14m9ijCq^92LvA zug9eNC^|J*QEAbNNK00DdXB=<%M_BnM!^}I6_jyA0U0;spZi%xHm^K4x-s>!(A9lHb5eZK*~c(N*k>B^a+a1n5F1UD@A55Q$&`F z!m>OSniZfBGIoN=*a^(ZR6tI#{B!E$o6|0z+(YupeNvt|@5wX!M|or!?E=Q(%5rnqDW+;NaT)I80t(&-h2h++bS@BxdIB@H^Kv0L^P7qzV7O)V??PRmN!rIvu=d${FEKhOt<;NF}{+?$RM zGKo9kPJlbSkhr6i*sG$u+*kKURT(Xps;P3Ww$O@dYb~#_mt&2q>}$NWtTseTYvZ&8 zY00*(R5o>uvaZ{%MSRtXRoyGHs`*A%)o2eWGgc@wckv|8!AX3-2b6M4>ZVnivMErLHpOTn zQrCpd6&km>O=Gtl(wHqbHD=T28nf{ajcq52*>DfW45fW=Z-BS1jO-xnPRH(OxC7vJ zgL_$d?~ek)lKJK^6;R>lF|%pXPrIQoDdJ=RZ;93QTSj!)Es$7kt*6N}XSgrmCgDxmAh zaCJVJu1+T_Wp-+#OirDW>8aOccJc>xI$==fBYQJHZP4d4tKznd!{*~nb54IPYMciWE!c8JQ9qwvE!4Bo!y(Eu)y5yGtq zUnqSvWwCpJN$znrKttFFj%5L!!NhCHs{z{aqA&VLJbDF#?B%sA3P*Tt@htd+&;R)g z8{ut)w*+6O;p;GLUd6B49f$uE{y&HQbP@gMDwQEd-e<2z`<-xc^f6T2f1GX~&Z3in*N zCtOB8sGD0cX(gn`X+cAybEwG0q&n#NLRFwUTh8rlaVr>ERnfnZrGzs(2oBk zgGw?uHUXUb>r;QB3fprIqQUTEADUlo4gMWXGS=n-$B5qyE`e!Zi8e8An_`xEbSHKId*;3P;9e~Ze9JNm;Uu5-Z>Y<1zU zebGO{;EKntG;GR+s~An7g1gti)d*KBT$|w9iRFjjIfM3b9WCNzw1`g`x<4_5$;<%H zgWLC^Lv!fCPl&l?Y|GvLn2D`c@H%3j2et*FvqW*XM227nHsvuiO0c7nVNwfM6I^X* z5nJKf3(s*LpUY?;&!a?(k( z3?;6b+?+=E+Thy;-vOO8IjiHQ&*-SxJ38F?8y)EShxT`)Dg^B9eh(e|w4pQTLOVKj zXMFXfKL^7-5l>PTVrr`!ot$*Fi`(#p$q&E@n}EDG{&R- zo>^zTYSvq~x{lEG2dC*$&xJbo_%fa9yHY3m`|9WuAv!W3T89QE>A;{2?H`<{eM3sL zXGoQH4{g-WVH>o4_%3Z5eq38dJgH41-qeOsU#o4@A8H+K&|0dLTJFQ0!n=Pkg)zN- z%<}UF(=ADgWcPH_mA;%vIdFnb44tDR!)#vxOR+-)%Niz+BP9e zTP74}^Tcv(oK&awNv&Exd7IiMA5rU+%W9eOikhZ=p@wP9DAUj&K<(6fxT6I<&a^!M z{b2-4!5EgaQRolDOm%8F6&YhkY5#;7+CAA)JEq!e>$H{HJl#hdX9TN#W~A1elNDi3 zR>Z7at({$>mf6*6n$xVtIh)lm_kil=UQn&Y3#zjCw<_oTR~6_l<)F-nxg9V(hSZGh zAB<)>n9R~Toj37QdFwWbJ?6wmv}f8-ZRcCAH_x7@_PI;6ex8%o&G*pS1yn>V2vgI- z7_G5PR)b}x>a7Y@YgMV5MQc=r7E@`xR~6Q0Rkrv!l`Q^5MT?o;78_JxO*VoNi#lMy z?_aX~zXkuz+1PKv9-F0RKhyl~S>3eNVu0Egj8~i0Y_%-1M%Qsb*IB80TOZZh2CHUC zq^g%BXtiCMD(&)AzO+nbOY2m^=Bs$wE*09JQoj97<=KCz9Q&V@jsB9gjJn9B_h8iH z{ymoE-5eh*B+g(9asce)^Ogmj(3kpZt<7jPE}5bFr3+Py7E|rutkn*ls&EWY`SLK8 zE{|2oiWC*E$X4NsVih>mD9@=?xom>6olhv!`DtZ1zppgspOi|qREiNR?%|g~EWa%G zx9719vLW7Eg8g6%pErOOJF*e%hpTq^R8_5*rwS)ql{qh0i3`;TuD&XC4N<|$DCMtA zRIXd5a@-1)x7l;FHizpewyvqUMFH5 z(7J+n2UNRuSB3ikm3WLpS29?HZ;D$&1O z@&0QR7qCe&0f!VFa8;23Zz;n6JB9lZ^Mf!np}Y8;@&5{ae~sNi!K-Z`uC`1WC(TWdBQe1GhVuDK*9bB)- zkak6c98g%uWrc>ksgU4r6&%D)An;EFa0Q=EV+*>NtueMAb|cnV#U9s#*dJ7aVjolG z`9H3#z!6Fhnu4xmq2!RoN(^;SLg-4xg?cMCEKo6Fk%|saP-J+fBEpLm7E!Cvi1iAN z*r%X~iwcPNrvk#ik$)JmKU$Dq$US^Co9(b2V*#5Fc+wBv#Qr|?0b8#UkQcySGx#y3 zh7Lhjny7?`*@}y>QcUDBbRB0!MR_P9%3tA8VG4_mQ%H2Wf@2C56jQB$n04}x*(2YW z^YV##Mc&b0%PWdELy_dGq6OW>yYtziJE8+&^B%Z2_%ROQtq9~@8^{bL{*HJ+2~h*l zl*TDKW~L%z7oh1ZQCQq^g~YijINldsB}4)7G4fAHksrabPr_<>C$5!e;x2h4o|Svz zOL9y2Qf~2o$Ssac1T>+$c+;A(z&JRG%{$<34PqRG-~+fb!^j4Rq9P-Pij4Ta3QriR z(8Q@|I&&46Y^{J~d-)~1$T!7PJ}H6nN{N(bYLYxsbL5^{F1NHMxu$KGbJ}S+r9H0| zsb6SC^6y%aWYBUW(1nb6i7oFXd~gKrEpRu%TN!~5;7*Pv_Kqj^O)^n%3i?CZ5c#3S z_+*&ND`TNNGnZ&p=5o1by2~w#s*EhEGP2_2l%1g!*(Gw!UL%L>ty-3SLQAuslU>$l zvdjFR?9gEB((mC}#^O2rzaQ=mk;J|5mc*g~!5x=K%#%v&n?d}Y)k7XR{pCicjBDN$ z^pv@B%3q`v`OD-;wv0o8r|b&?wX`5wc7>_3Ei9BxVXdr-HfmAPQCSw@D>-gX#e%}fATDVsypo1k5bEOdTq@zD%5%=VgkzQyj$Ksx{FBz<*rQ@`u zY`SdA=EA-w5WWgtjc|~up&$gs9c&?kt2)B)tXb;rrDJTWnOth<`o~yyzEz* zqs5q`#oWb3#sKLG+KkO>2!%?qI~#5i!4#H*{RQxs5POvqf3NPUh1HL1e$6mh)J)La z+8LTtJ72TvY-C=yTr=z3HN7rC)9Rx&wLV>w>q|AUp;;3e_Go;=RgJ5EU*l_k(fAtj z-Hkw#G2%4---pd@X*`ec7G~pvT=a*0`kk!zQ(lLXcUv{$W~8{SQnyU>k>6|UA_jl)oW1Ob`5B|peNei*1*=EG;l4u z_ZHCHfg{+y4c;br%i+x_M1O!g6mGB8Xh3i;UPJuZ!aMkNn zZ_8Nq-8xNuwpyt7RvY!&=A@q6y!GhzNIkqgLl17RRF56)>b~QIx@~_+J+^(T2eumY z;Ff#X2e+|8s}_G3m*Rs8szR{auZ})wB>rE^$r$Uge-k+vTZut;@E(3Qr&I3hqD}{T z%JjfMnH(6+1UCg(u;|$W9`=L&AdZa?h8@C)L;TU9Yv4nE{tGbh`yJQ;XG4sn<~fMa4HgHzo=FBZFjU=$M_?KsN}D*k4Qrz(pagzb_QyfY7w0|W0s z{`|31oXmL^?hD{DxB{+rWs-Z04bT(3h#J9*7#6{&=dc-|9nZ1=JsZYiz{C9<20q74 zpS#3ny!p(^;^0F}&v3I|h3P!21ZiW=7s8!PDR-xCL&5 z=grVu9$+)}IGV^1Hs|Bn9L{26LOb4DgIb0@IT?~pZ5{WoCnwMppv0-M}s-UT!h_raL?lZ9&;J}3!A$?!@ZvaF99R( zTi|`b&>|2eC;+&XKpZ~sBlx)!8cQ$oPlgdZO#?Q-3&epU&;;lg18o5v`hzjfHOBq- z>kpRv+W~lhAP^0Vxgc5OfE1z$lo6L!p?`E#sQd>Vd(&u4s6Y;K_g|7u}#NZs>UEhuxu{L>>_LY( zPLEt+I(vyvpOB&QGZ{PNUw}jRp@aKw8%!BzJ@9iM>>bG?FqOw}KAd*gFQg=P=&5h)Q;w7JZCr_ah1ol_6mJeYn33%$)e% z47(qq9|z*oaqyaB-y-hh0G}INzHo(LR}^+7=!i+04w>fYpjnX)m{n+Br+V$_+^XGO zwrXdWgWBHpyta0|t*gVY+U~>MHv9(vzkzYec?o^!8OG-|D#nt`1C1M2U?oi>;*<*I=sxv+M z>eyqWb@1_-+SkugyPn`1DF(V~+aND(862ohL&CLjXsk92OVRq_S!x?zsMZk`T062q z%_BFcanx=#j6S9M(a)%6%=@Yu`=ctyk^=$C$J~R#gsBw%5Bo9Y4`KNk%JMgun2YK5 zaKFd2ckocOoypod(n6a?+i1hs4G?5&LN#sCG&Q;xHav-ME zs(MPRR!`lo%4x?`KJA7|r@f=%>120JM}GkM)9%5jg?+(N^X71tqH!#{EH_8E?i)k= zJ+iyDjvb(l6UU+Jm}~7+D>Y4DrZqEMR6o;Gb>;!8F%MJqtQf7Hovg~)St_4Xq_R0x zDw*4?;<;N?XmMEi7T1(J?_bKE_l>g9Uot@k8q8h%w+GWccE31*rI@Mx_zd``6MIi( zk4e?i`e}W%cIGHG&Yq_Fx${+HVXLZnoQ5#pT@?#_RJM@Q5En+O*fK#ymgy?6%2&Qs zg>o0IQO=@G%CbJF4C~8Ew|-5j)?X{x`VS>7BJThsT6N$D`v3jWEc5W5W7$7AmskgE z@CD23tN7p=oJXzkfergh(7YIa zu-cZ~gQfjdWIsmvj??)*eb)xQR&W9BRG32#U)TlE)hy}Nl*eO3&*(@Dt2YH zqF1&ma^)^XxSdg$+w%%t`I$moe^an4E3pf3?!c$?!wa){7U4U%gzWRB#GK36-`T+r zO6@zTV0jVzQ;{C|itt#i zFweCL_1vLg&(jL>d`@4`+i5u|FtuCgx&W zn!f5GC3_80g7-LdrJ0KHU7#pGTSfReD%^ji!u-7z;vb~ofJk(eBn1R!%RjJ8zJZPM z4%{lQz~k}=ys1@zAIm-9KXRv9%MCrq&9?*pV(edM%XfSQ&m+8RR}%ZfTjGxWAl;Mq z&OY50=hq)yX|y5(rz$+iLZLy{3J$hc5U1$`27Agsgo=oeaQTKL$U8JsUZExO2yKvi z*k-wf9hGa?({c&@NX{X@$T`>`ryvm6fmi1-FW`g2@Nb5@iS1`4yajCk(m|ph@7n@9 zD$2-DZUJ<_Xhz!-L$XK~YrE4X*IWAFka*o=l z6;X%f7b2cgzfV#>|sP>|(jc+RH80RjzSfa)}F)b6k{G#HYwHK3@*;)mj$6UQ6Q- zYDxT)vWz|EI{I--G$mYiDT%U0i&>mXRYvMMS*7liW$G0z zNPR~Ol7G|!G?)eGFL&`I(>Oe@lQ$}xSg z?5SE>nlW8VGA(4AX|2WN##oaZvnb0;mRZ4Cm>sM6*_oP`U8cF&&6=IFTeGq+$~^mD zGSB*6=IAiy=rDJ2X%%5;Ai6Bvb>ZY+5E5m;9Un{nNdopK!=Fa%nZ^6}obFnb*B32i zxEAD3(ENg#vM89ZxrH{GQ|PE!g`7-L=%*RnWO`AmrWO@xN^yfG6>rys; zuKZd@GteJ$&>!HofqPy#@%L&@W~epM$ohvh zyuQDNHjL1ah6x(HW~K(NS)c)&;?jSOv-&mqsBdG09&b!nucmVKY--b^O-J-_({p;H z@k>3{@P~TV6G{Au{q%Jk{;q;IAKp}a9tF2wK79bU9o!45IGLfA(;?T8gV0P2*h&n( z{sBF_zONqKFjPG@j92#!Gt_Ov0(IG7r_SvwW!4@b({>DN=SJ-fU>CT|!}lKek;%~T z7m!NS2yYp@*?Eiu><%s=|Db|Cs3r%cp7_5B{i(T?M^rB=-P0DZN>I2Qlf6K^K|U@lxnI3($2xUQ95^Z(!t881&RJ7N}d` zBS1Sq1I;U@pHu1QaDK&W6Me9qF#zw>gXG{GJP@6oY}wzIFSS51z#|CiH0W$jtsPqTw>EA zcq%Zx2A(xU_pR`3f^j!d_A%n-i)bIu;j$0V3ce?H=UZCA>)@Z@wLfuBe=y}I4+5he z#HkeI9iXsdqdm5{!RLofVerJlkqk#BHtvR2k4Qr zXc0G=(%$1=-*T49?_{cwe*yOViF^E%jnSU3(4{`4M(0g@@dCcMNo~*#w3ushWNL7S z$CtZ?U{4e_#KV&cPZkyzpm>zQRSj1Yx<@-4yYa>eG>_|yR8FVBtD70oo8aDfAMS6% zx3uAX@G><*HyIb#7#A0*4LXZ1b&|Z8WAw=pG@(O0CkGS?52s_IgY4m1*+u=>PR1ya zHkl|5sDy2B?Vu$`wVqySV-&VBnp#Ymo6R`)w-aaYa5ds?8@~FJ-#GO=B{OV(6rT>l zit*St3r;Ke?6u3(RXfbQw7pY+wssEFmM$^c)HO*PyJe`od!E+!C{tUHTD3mVs+I@0 zsrjL!YJB*b8XkF5wU2$R>Rx}Ssu$-VgUUZ)Y{L)WqfSggJ(z~U*{&=lrd_bPxAu1) zuHD_IX#0Z}+VZH4HuZE=dv7Yw0bhyhKi z8Ms+h0}rZl&?S`*dPSv!zf|#%-&Ht-+6z$dCyZ_QvNO{<)9VeU`ZM4N%kh5xz2}i0 z+TMGBHuW2)4Fk;8I>=HjLpTv~sFT(VTc!HpeyST0qMDIWsvb!W#He&tjwT0Ubh*mL zG^k{3yNbr{Q^C0N${Y8Ba>jqAtnt4ob3EF^xO@1SHhkQZ>3;}I-w60e!p(KtFyf6t zYzY6}ISSg<4Sw2|_G8V0_i^A~s)u_rNUnw{x(6 zE_+|jG3bgx40ZUi~ZSUbDmib%Qdj`%u^Y8)7ebYkr zcU;Rs(L!<$7WGp0;$h0Lov73$vy^OSsl=sAl(1~M;+DB7*4|su_CboWk5r^XqQV`r z6zW*25XS}uk(eB?{IL9&KPg|wcje>wgS;I$fxw>l|4+O(im_nMwr@ULR7*6VMZ}!e z?C-c1bIo1Met*fMXiI~XY(Gv34l@+zIA1Z#Z4|x2L6Iw5(04o)<`ke%r*MTh$D^-g zD8QN80~cx!T-xRBa!_6_*W}^yj#fE;FL!cs?tr`;ckmYD@45xg1N>XzYq6yd?1+1p z5buD@rNlfA4=8c@6N+;ht!U?|igcN)aF<02b6u*Cl}-w#Rw~HNPXTVB@^_1suX~z& z+zaL9ULz0p^;)%RzuZ<`k*oV#a&h}kE-MXkHUdq^h}WjD1%>}0{F~rjvy3rdPuvT# zKnmN^c;~K)aqWw?G(utSlN3UoRPd^W3iMclzO!6@eB-yTr?8g0RbKgW z^Qw}ocbi<+|VIBZY$cPu|hl}v93i`l_eJGKzaL*k!QeEc?8Und!Uuv0`24)v_dXH?r1B1S`i#3M}lPsa&wl2l*=xp zMYbWkWD{~;)*-KGQSjGVga)%H5CnAKX{(NKWE=cVa96nD19($C;P<5N--p=OpZGhl zyH*AFl^a@&Yv@EdhnZ_d*nBMyw~<4*1G5wWs~%#bxF=~zWJ$})0? z7DS%WyvUa{FXAiB3;#p&&|l_-cHkO5IE>#nVsky*#qef&;{#vphubd*{UL8JeFwPZs3H%uTV^oD^5hO7W3-N|o*O{B#JXuEg#D?+ewoDI zImDm&T{MpRpfN=QHM(e&Mix!dh+=aMD_*Ff#daE8;;ca>J{nLGq5dVQ>Q`E#KBX<{ zUAkYrN^huF$;awl{Hq=>BAO}ez+SjpvAGgn4!}_oK97XkFOe|-HwU`Q67E?A@E1!X z%XkOBx{LZ%^;Dm#0eZZ8qY*KH6YXl8L<`a25c^dHv`@{><)(8BZEGG+X`+{R5huBb0DhVucZ#5 zff%^CtIV2v%A{oglPjB!mPvp`uZ0J##St*L*W%Q*>3~08y8#^I=eNP{&q5L75P#dPMuVy& z{%?dI-T~`5nRFvL7+c}r0d|8uv|}&r*w+i-kNqRq0ML$ubI?d^fGeQ052pZpahMw% zImz`!@Ne)l_(j#&TL^C|{toBYysGH~d~MlG{Exk(`LhA=KDrD2XD_@5z!7j9oB*f5 z>8{{WCft7LBE!%rCb9tFk4sCyD(P|r+6YEo;l@{YgDY&B-sJiv_*#|tI){Fat3`jn zT`sM}|Lu%Hc*nszXdnEC(0}O4JA7xkUIdrHRq!OZ4sMvBhjd2|!6&x{qfv}wb3{9g z$#5?-@w^&`Zor^^eI0rQM!d-%z4&|J5Iic`+m4oeJ+f zc$dKGj7?tXIYHPGferC+r06}`@eb{Hi*~$0tos`J!^`LoFA(Y8CIY@iy!#A;_bFJf z)52@q@hW$}0xsVN16_9yX88Ise9Je35bJ(H-odB%;v-_+chRNZq!#5>`s79QhugH{ z7VWr6JD#Q;#ypJcwBs6i7+27yE@?A5!G2=)v*-jjiQC`jQT|@XU_1IJjQJQw+~cRL ziuQc3?)bGIc8)~BnSx#O=usQ;K2~7QDmeV$2;nh_LSKnTUrE8^d`k#A$Vu99f?T0v zXj4as`1x)Qrjujz$~E+X*O=D6;3vN^#r-`GJLWsmi6@R zI@-{BAMUo{2lzh$uYjk~=&o>1%Q?oyN%EAAqQx8{N9X{3vKLKgH`>%r+OY#2WE<_+ ziVm^`J!TU+$VQ%(b~K^&gDk}`NQ^3`Zkt_IV3)tR=b z#%!0WI-Sz$PS2>K%Lgj!`lCv^aUx}RV7vlj8@>W>fg4P5XF3u4ni`jd&T#ia+Zh7y z1ht#aQd?(BwRYuH%5F|->fx@&2Yl4cHlsln(=w%L6!`ru=@Z!^7~@5wU8bibSHHvYY(Yd7?rzFPOt zNHzDIsx`eWRM&?SA^X~^s-KHi_vafSp5Ro-0U;_K7^RYdi7KK-sBmzB@&~V0?vQ3> z58bA$p~sXl?1s{Yy{DAnKPq`R`41q`h+zhvL8jbCn2!6i6tWH2GYD?5o~5{zskXV# zV`_L}h-wB+(CQ)PDj&8`rNcRYVZ?G3ja;dMQJly)IzYLj!<928R#{_Hl{q$7>Ep_k zI&O`U$8T2R_`^z=a7}R&-d4^Jd1{gpr)4W) zdWqtu*DGd5yP{?sP~?m&3ZL<<7xd5Of$Lg3)4(_~bGAVHnFlOWd9b#Qqbp9n^x= z{JU&?XBAKEqx>nu(RL;&bNVc$&#+YL%q2=OU#_HCd>g|oFU8LeRNU+c#mx-{Ncep~Lv1!Q8R;WJNCQz*|Gu(!hUc8rwm0e84hWHHZBj z*L;u-(&s*?l=%acxL~y6ET<|4EhcIaXCExGQ@HgCg)Med$YLJ_+XO4nCRzbD)E?MU zdtkd-UbfU8EZL=1OUN=?@{(5Cej!)e-{oq)CtU34>3@z&ZXxbzESHZU@jXFpt@8vkgaakrgPqvx*a&6R`@2;+bKy?&GsjVnU ztwovWI|XVgDN{p9t?El!Ra@GNwlb`$(lx3m+p4m%LnY>F2Gt^aNsg9CGYAy9sb6L0=%i~pF zo{qkgtD1@uRafv`*hM}WiDOk)9th1E-zkIHmDjn5O<*nN45Y<%2sH!Ff zeJ5MxwM8mptyD>E1Nusb3hM@x&)l5c`i;t|->0nl)5>7cc3S;&N@ZS7YVG$*Wlqi% zs(GG59o!86)$m^l=cYn@P)r?^kn@4ga`L_^#=AAH}5Q5Q(;vsG`OU z6|z>Ups5UPrA|4`ZOSHD&TL+x^p;7bw(M1M%QZ@BxlIWz&nTXGIq}TPiAR5lZhY9RPe*kyF ze+K@2*u4qvu{z=a?ryjn;I4wZsFU2YM@n5{s>FUv#Sb_ucEDTFgTaa#j8^1evci{U zDQs!6LYCGjXsA_zLw)ie8k6tPR(TH}k>~J@@)&wl?n7_OgE=uC1FT#&(9hgje1C%a z-i6)c4a`AlLWhF89qt;qOS`duG5iDMUPI)cBeNB<+(tnw7Aatbul!ep%6BwYKBKAf z8qJmG$}+jHY?SND9xWPMF6Xh0avWnHim~ftJNA%lR=z2_(Qjn8f+wrXr*S>DAI9#j z&BS6W`A$1)VBl_syBzNPe)68ByIea~F4f}Aehkpb7o8jkj^LCK~?=_?F3xGXv1AYV*o<=kTV@%`!LR!oQMEPJcJ?GPu2hoJUK=Ow1eX*a1VHmb3OszfuFPk z&h;yZMf}~%sf}w{gEGPV18k1jPX51}H9-5A0|2kZF?de^F2x`dJq4Vb#Z7-coxL56 z+!eu>a$HY2Zb$)ivNuxMH)7ZpyA*%7aa!dT>HwSL zuVfC&LF|Xu<|J!yPQ!N|Tn8=!F2#VK0VufDgbv?=!Jsv|h%+}J9|lE~#65LY)(Fc28F$zoNf#6>>J=QIBx}a zfP2CH;CJ9b@Gzj7^(Zm&I5G01BLgFE=7vy?7cv;ARWU!Lmj`$2&<+j&F8B&xdi7QC z*3{domhPF>V9K#M^J;X+b7(+>j|03Gw{h*e07akTGq|1vPlIQ{^Wa7BGT@R^TF9Tx znR{Z(Lt{?{?GYf01f~vsVvuz!82m9rpK$R{G2&BpFE~V7xN2I1DY$?JgP()3mzP>- zj$yB1tHJvOcm^1J|HH-SH^AF~>LodzPL@9A_&N3Sl_d$3JBcji_#u}K6^V_XM$kmI zp@~5H3x@t;tU@p}hgsnA`U^vU2mmo40~CWA(8?40UNY9DH1A*1^1BgKTQt#HhJvV$Th!E)PHLqf8evBL;QRH2SWoe=7Jb& zbG$(WNJe|fgR=s@M!1Z%C`;fOf@cL;^lEbSaq`5=`-1L;<1ma@)h#lW+`n2+%bZu>V3J;oY)-O+l2urZF(XTn*GP1V@bj4hpT^r3wW zp$)7+Us(;qIMuV6kk|poK6HX(RNEP}f{Vy@_n~_{PuLnWQ@*8Y|9|UHeq9Ep^04;e zD>>myU+jzEo5|=Oxp0-iQ%g9sz|n<=`>rdpMlZrAHo;+5Er-04bDJpibe-W z!JZs=O0b80DA35Rp&VCJj+6M~IOVts9por;r;ZS#hw#NgKJDl1E2)V+M8z%?o}Dml zhjAPH2DSpD3||1FnLh;{0JnonwAmYIBNxzO&Z5PfB0f)2C&y_g$IzyZ5Tl1^D+kqv zJ>6K{57#odR>3|A*LEWG;FL#<+S!C<8;Gb0+Q;~my7>zVCwox>?p)N;^i(v1-{}60 zuKB!7v2`JKx?rCloDp0x5xX+sD$t%8<=SOZqn$IGw8OMR+st~k)oiIY&l=Sx^Yz*= zd#fgRtI)W`)mk^_lGe<5NUP@jUMuH)s?h~MX~hCo-Wixbtv?t(cx)E-&*7Fgm%NkX z33G1ACg#+ijSiT)Yp;2bcFl>=j(I8Cwjf7a78YrfWra3c)oIeYMH4n%T5r>@b+#i~ zW4A`D?KW%7{z{EHoX~QIi(2N0is$&62Aw|DfD?+Z6LTQ`1*2Pi8P5Bxc>KWa_l!N< z4&*z0-pB9uE|{yGR!-V#>#I%n;o9KH+aR3Mwca^TYh6mUW>K|Pxi)Idjr|whdbPrR zNFyF&8ur+rAh}IvUEV*b(;Mx0=O~^ zp?g07c7yGU7y<#-xovcxxY_0HR{exe*mig6c$iGEP{kt>} zFra?+YgrOFuHK*>>JB=h&fp7b557lj!SpV{e^Ybt4{8ejMNMci2Hs>C^r$l$j0bh# z&G^=jyccW*8-2-n_}w}mORe%-gtp_aG4Mxx{{Z|q!e`8Z z*%yrcA>{sG9iLYRF}?_%tC3Jg4TXDaFfv5_QL$PQ&E5ww+3JZYR99?;I%Dh99@nb2 zxL&oO#WcmU_8?)4>JoSx1CwH_6Yo%E;p zyD5_K9bh`UR`Bz1l(`0DY}FU*q2BmFb)&^}CMK#QDMM{Z`D#rrRWs|9no^q8kkXC5 zGN{_rF;%B-Qf1mcm8YFjY1(ZnPJ3EKY5zxssjNLfgDFUv#`9jpLI6WU_zy*)0mU-T zkHdbB%fS%nk7w>dqLsRm7pWt~Pi?8;YDtSn-$_$rMvm$;idC1%8zC}zBSdD0Dzo}k zo;9k{>u-w*ee1nM9O z|AV1KaxRWNsf@SN=BqWsNzGZ_YRuw|nAtI^%}G{GZkDQY3sjj~uJXJ(mFBgfuPjkf z{t6Z3Pbe>ck8%o5Dy!gTWfVN2^!#^}&b*xTJf4m)Hz%FBIge2Xm+<#V?A-(BMux+y z;arx6{T#c~$@{X%yK*elkZZ5nd=FLU2dc6lQWb^lzfhR2(xN;S7qR}Ks73|FEy^qI zRc^_MvP;%0vt*ajOO7kGzjzz3O( zv*GRGb6XBMPagSqA#cPevQb5`tIA6JR8ktIqS83@omAzsZ%tl#k#blol~vKC%!+QM zvra0ta;=gpcPO#)D#ce`RBYuVim7~4F%|!<7&Mp|=H|pOC+9xu-~#?P<|A!|du=wk ze=c3I)YdDh zwo{3<>Lc* zdy23h?&>n0J6AB~uO|PlwNMsXOh&z%(i;4g+7Pbf#yBN4rYfN^7k#Bfu}!s#Zfa9x zbDzSS#}wAQMIp_H6x4j30-7I`U-O^k-}III(P0A6VFKz|TMPfy@b89y0&cc^Wlb9~ zSdRbUE{B^Xams4o*>966nvSIsTAdWv=B1dnAVs%FDY88gZ6#A-9fb<*s8ldoOkiiP z{5x03w{xSsI}gaS^Mc$v@0VNWALQQgh1}bj`_Kl^VQz%~7=GW5-D}_;hPS5@4F+ze zJu#PvysHWQp_Tlz^G73;5>OQwMSENlk1S577c~Tc{ol^!|8GuE|47)ifo5lWxZ@b zmdn;?!LnVNx9l3tS$2nJ55FLb;g2+D=m*VR%53UE=GJ2SZtPwU_cC~U8_@ybu4zGk zXeaOLBL9awvY#~=OUeI6*oRo!76{vTNSRks}f|fDof_8N@TXG zUNd>S(hT0Lq}5x%QE-XQ_ep~8|A22W`e(o%XZU<8+^gUngtx2Bn1@6hz@5`e?g@9) z5cx0Mo?OmpHP67;k%KZP#blBj-lQY&0CcvKbncVM+-wLgW0KVdF1CS!Zx{#r!L{H) z&iO0&f&lyxu*VtP6Zm^M{-z4mj?b&%E`U1??&#(4uY`XM`;c-e%MHxI*vz0~+f0Ip zfyWMO;0!#cA2#fap6+ON5#YP1)LmR`*BICdPJz4m`zzoRzVjXUS=&2_#UA=){N1wz z9}J*_!JP?r9A6E_X3t6ZH}MR98|x6b6c6RN!Gir}1k+pwJT`!G9HJbD>3EOO=^e=c zr2yw1;etmuf+OH&etrhL2fpTfRu5qB=pgfZD9rdz$#*3J^4Rh^V&-O4exw- zXW~utB^RXbC4v;{%BK)0#%@Ny~1H@Mx;GIYDXB;>9z!^ZbPH7=GaJ&)R1TNu|-_GX- zXopVW&fqkV8*)6FNIqylKfuttFzW7O;1)hV$MGGGpK0|PViAA0P0$}x&zaa9eI**; zA>t2v=NY^deTwrUM_f5I_`j3m-QZquA9w)#4m>m)t-zW=vMV|T<#?JK(KE$p6>VS? zU1S%U#aY0Yo__)GtdFAg;%`HPDWHxMu{jtYyJD{~2gby(kz!BjFZXbK5Ih1N15bjd z!L#5wz$HoP&?jaxPvZ(EZM&dNP%rPKpih*ePxPRNVDJYANGPrcl;gu^v~)K<;8zt# z@c}kRo+k$IuPyeP-NE$?J1_G-!DoZ(Mes6s9sCje1-uDxElKk9L+g(@el`ak#F_aZ z!6dWkB-FLc3mHHQ!SJ65te-BRhZwT}jC}uc?rp(mLx1oEhW?NO@<9b?L=)&h16e|r zG(>K+0&QS5n#4F6D=_9` z7+Si`44}$+J!wi02fUPdr>Bqd2D4qkSkfr8vg>rc6 zu%{J{ZaDhj7(yRdK?tmdWgMQ(aO@&X4xkmBAZ)HB%e;e7eS(s_Nwt1WER_p?hoA=wQ_9~{Ph z7%SjdP4k$5V+)!7ZkP|F6`UrVZ$kI@9sj>VGya%d_9tqI_mTVxm&@=I{9gfM4#r=Y zgYh!Hc$T?2PcZM`QRad?$Xt;7sgrxi`|l+0|MxtMC2%Z-V+ESTYB(m~;GH1E#(6q3 zcEg|~capb1O_aQYcm7W9&OAL}%*8NZl;LOa1$Y}6;r;~kQ66Lt#=ZFBPPCX?(PC~^ zDm=WM6#d~k%5i~moW~z$iH|eH=_$%_4dviX3-D~<+B?xA4w0{)=Kq_iorlTU|2P%3 zdx^3=FdNwYuQGfC@4LWgb`OEO@x`s={WmcO<9gc21+vwUK{@ z8gi6qy@1wn7c*6iSw!!mlKfp`aIgFqj2JMwm`8!ZPwzE#OE@VLc<00G2&WgEAzU#A zyHeoFhO3ZPP!3lO_B6rOP9*f=fu-=Rq(&y-+M%%-hc#+)PRmX1&@z*!HEjB>mYOls z%j_2o&SL)ujz)K8bg@Qvc%vz|zgdhg&A1iQ-R{#Y>dXpW7kufb%`?Kaab~19DD_#^!Wubnbvw%wrG8`Rg_Oo9$Y9#UTwWWaf_Lomy=9ta`29 zQ@8cE>au1IgaIo!iH z2dgBlw8_+{ZN8SsOp0zRU5oV)!c|HgDJv+yevXFwfxo+nk6)cVfVsA)+7SOMiyIV3g0xe960f&D8H_qs0NP>Iw2! zS8%vGLgLgGnyS{&Y&C}!sVTfd4dL~wi)dFZT1<81a#co-t2}a-N~4aeIO--9Mm?MP};zf2*5H+|#(-(u_4K0Ia%zz5uB*M`6khQRsmDMyPbja#PT_;o6b-=X~YW6DjqQP~L(DKq|0 z;7eto!(^brWW+LukYUD+*nbuLOx4z8IOF^X?2m+><6_Xo=eFp%YL0PGW1Oez(PC;7 zB2}G`fWDKailkhXB^9eQsanO!O)5<8R({Hma#PkQJ7t?PQ;#S;^?IeI{!S?=|E1)V zFO;0jlL-S!==jvZb@<$vqP-oy_0i;(-`k(kb7p4|K`k7L9T=H@;sE2AE2!K z2xS(;E4?69sf9U8DJ(%>sZl~vtKy56D7JW1(Zw4T!G!Iw;&TcuzE{D;Jdr5+RKbNm zDY$@<61bT)G$k?(%^p#3Ql{YD(yjNir%N0^FsUX%!1yr1oU&Y<>sdz~~?ir6 z!T5oT)WI?QZ_Gzp3->U*z3{feTbl=e0nh%6&>u?3{mRfEDi$h%c`2`nDX?Q&}#)S~9Ka&Flr$Cj&Q&zu*|j6GV+ z8R~y8{Jd&LBXAq@fLe?30o$qU1j*70w2I_Xiyn&C&C@oLhjQ}{@2Cx*Tt*>S+Y>}OB`j}=OOF909o}# zXklN17W8LoZhx^X`s-xQ#wuq0%Qdrqqh|CULSVT8JVL;|Nyqo4W-R`vOct}Y7uz>s z_b4{^!P^FJ9d?(&ozsB+0CzOp!EpQZvId}^{AVdS;4*WWk66iU#7U+jo-$#L5Q{?r z6>A84BV25xl^`Dk>%m@dj^Eu6eoqJR8Nv5GU`&V4*K_(Xyxs7!+)UNjUC>M&z#R*B z$YS{6UNi*%2y-At$-!5f;kViCKIPa;Ij*D}`zXi$FpvoHKrO%* z`?<&g>f^w*e0~_b!MR_6@3jif!5-=X-iAK^wcoeLZ^HZP#>pW1|NcNv;trE!`p_t z=+=}yMhswc;70tv4Ssmd3|_)|itlQUr@>ip9$Wy|f$OQ88_XGyS)zw90K3_n0V3tN zH67569jM8^=2UX8n&Ia{tWf!Qn#=4Av>fW3-jWav8MJ$evh=W`R?mI)QG%m!EzL z{B`QxRIL+ef&3|hI*!6-@59VN;xu!Em+G71y_w_f;9l?mcn~}S9s^GRE=OYhZ!P5I z+2|9*$e$_4U*pgx=z!j8L!+RQ-`xYwp;g=ko>cD^d;qth!6Y1|4zSsoKhMQoQ@W2~ zr@{L>U~n2-&w&>JRm$Xs>9sL`;`mqa7I+uD4?Z$O2eBoQ^<_bSA_;XlT1YQ@v93oC zp-#Ss=zC-J!)5)$I2!Xo4E@0a7<-b&gDg-2YRM8?&;`25)cVl|hRK#j(FfL$D^H*i zY-U}G@icosIm=aKbZ05XZz;}$6zgS*_YwK~&#Zu$elM({e;D&G3=QHx%V6ja3&A2V zy&mQ#rNWtyCQt!aJv?pbA3fxU{mjW2W(v$GJZtne<#>ZE@6Tj@f1u&KM)vy>!TLNo z-ZSLfPjSZMl>9NSZ16q|?gPf282?!YBL}x22X%nc3){lbA`-A53$7xrQbk_a07n~7 z&wJqLr%HyYnia6D!LA8-w!*Q8-1!J$a~hsY=mif^lGg~|FVG~W*PdM757d~8@#`{t zPrZC4Gik6+Q$glaX`HPBpsJr83k9MkhK&XNV6Cf~h=PUUKp31d~vRe0wpRvp30!|)sP zFfPxxhWWMCV?-aTWhxPZxVHk@6BH`-raoM?5t>2g5l26V6Y8(Zr0VcQ-J) zf*bI~1>)iiTFf=f5n@&@o=wJRcD#GR4*kK_n6t*MXco8P`S{jNrveO)E4pQ+gOXTXd=@EG=ATEOj@Tg#PB z43ieY?+Q1^5f`2@IWxv_K!0$w(qiXD>UQx}r)#L%Ss&Eqo~#yk)<1aUtI@Mm^`14V z^J-R&SC6W^hg9LSMrA(RRN`}3MZVXn!1q4o`M#oD-%pg|`;&6eVGKMthjxg)$KcxK zPVVo?_|ltUFBtS<2+q&F{JzV5zB)V|)#~l3W}iSc`bMhWFJ5*2X{zzhQFTDEDg!E2 z9@wZ-w3yiP=?|%p{S0P$V zK3Ys}M8C2lRw^@clhPyiD>d@0k|Xa@V&scTi2R!pB7Oj@R1HTvr{2%t`~C25f^$_6 zxKGaM#VOFXNcTstSw@M>}(RZR$6rG@g=yc`BwOo={BkUPUFJQbh9Y3QK-Yp~)X8H0gVVB{Kg2T*3#(@%2vZXWG0*BJn{q{*Qql z)bhC^mi#-O+%I8{@{;V7gBFvO?5m8FP&A%sG@c|Sr)4TBtw0HB6=*Asib?NQRK~C( zGT471W4A&wu2xXSZ3@VEM*ivVD)5GwuHG!ND zumy^WlgR&57;~r1QbxL!($Hd3GQH4uf|QsQsramT#b&1|COc12XfY8v^$N@BR7mbp z1?8?&K<+O2<(`yJ?k)1leOjJ5@5w9YJ9(kOcxSPm6dxR<-Z#O&67GIDJK=0h#{cjZ zrILH6^SmvS`3Ko1O3In9_*{F%=D8~--%nBbp^7MA4~T+fg%)I^uM{h=utxqxZSpM| zkT?6)coyxDd(m;Z7X4N(MNi7P=v}!Keyc?VjM!Kob%r{)68{_P!A2N%GL1p4Y19F{ zWf|mLS&X-H$h-2;9}38Si)N$kSSzf=MIohL3Mvg!U}>cMdAGJNU+^g}kQZ8v2a8wS z*g(UjVoXjITjjt4dOJ4pv#EGo*5&WWw(Re+MT4;eCyBY8)c0C!AHwD?cnuAxD%;SX z$UEUqfjhn!{h^fnyJDvNtLDqM+Fst(Zt|@5l}Am8+-hR9h;>RXwYhStEt5lSqwMN> zWnH&ImUWx;o4UiAS9hc4)IB1L+BYSI(WG`E{&Tnc5`xfKUf2H zfm7fvoc$8`h`{?6u)i37U&B|}>Xto@=!>wqyo{WuoZJ^~B2!ICT(125{P7Hy8z5!Etaazke2dfaAYGc=@L`Yn-MN&*V&*_#miZ7j+mV@c>T|fMS^@_yLT7J?>?>*H zD-!?#Wi0es*$Vi=%1LktT;#kb!Ml9#Yw+XLzCc58cGVDz@K#dvJh)kosi+R-;Pk-X z$3C2c%z;?OJ`^k9Uj^2HwPxI4=F+)YGth9Pak&Gd!=<53(!eIE*hwxpN#QrF=C}tC z6O#{d-W%Wx@ICK4!yRz8HBt{v!~xtHaI;;z0^xSY1iR(vPh;pWYvCOS8^LC<1#C6t zW`{p^Skb`nhjH`W9YE(#2fK$lF~p`l_+k$R?B2@pB*(kKYn=ND_*TPk_OwwC_`9-m zdLB+Z{tg*tA5^N5>A9M7nU_m2xSXDgawW(8-~c#if+k{4hkFHsL3;+Il;e0H1G89= z190*Qd~tj|$AjRve13-W-UDB0pqqNY=6X&k9H0&;ZrDoZAFP9a1N^_{W0f%C-3wbae^whWNnm>)tpZb|@!0Ap`j4lv>(C*Sk{c%9FGQy;voOKA%u z^vC!+nxgt{BL23+znAM?<~2A^gKNPJ;3B}Bslo8A9B&7A0^F>7XQ5G8kt{7DX$h7d zP5~8w2KC4|ILtu$5_lZ^K|REnu^&wV+=*~A6<99&h(Au7ve&TD;Jp#t3=E!ozTB}iOGu`Uk0y%KY%}hH^4jKeels0tc!8y zfqFCvHFffJJ6Z@v|D87S_tR)1#@jf4t$~>Evn4R}hs*0>()e5eD#?r*$Q0YjPkUL- zF@OfLj67{6S?pRer%5uGt>n#n$lDJ=e>GX%4d@^DkXb!T@jj%OKTzE1_rV(a$3wt) zC&_=7!6?rh;0V0YBEp%ikO*fs8Da@~L^Zi-6S-MCS$r>=!~o_FldF%yvxcYolVnlb z;Mt2VaTMmWWSF<27d*+eMOI7IG#nr3?|`TOI1dWYux4{GU`DV*eZ;#Xz(3jX)N@4+)NhtC%6!wp*k z&{!hzTM~XV_QEKjiIkxzRAa;RJdAVX{l+|u)0E>H{Be>@_&6P@p*$Sr|HG8%ASFM5 zg0zpwxe{Ijm-oXk{O~1s3p@uN0(Z#-z6IFh2$vVO1Y=7KJjrlm!BGH58Jb5m+CT#w zEpT+gu>_7GuD_DV97pq*UI((BxY&xfj2Sz|Dyog>K^rN9F$crIcW@fb$Y|C^^D~;_ zb>J+%xEh_1Q6RSX!4n2g9JQPRM>ZSzZh*^2_a z3Ev~Yu>UML2KIwpU^~9p!W@i^%!`?zt*)m|)}lYGp}nqB8az2L6~a}9-cbWr6I>l| zF$aR*^mFzSO5RIs_E4K$)N&^^-a+rtP7l)tzX79*GrC^G|5t%40X-?a+}iQQI&`Tu z%$FLYPDZJdv&y4_pAX>h<4+szqbfVJFq#`t+uX7FACuLGRu57bGYNrV>9 zj8m^^in`6R)HN$#9kWW+ZeF9-+0AO1-K{2zrD~Y7N_BHLt7h&2Rn0rAig|acY~J%K zng6~@7JRE>)&v#LH~P!z+tF3%`-Aa|&ZBmK-J{8%YQNH~h z%C&z^IS%iGe*p9cd)_lh8P3Ch2)=FH_SZSU?+7>O<8!YaLtz{CUa*?0b{l)O+PSOA z-d~Ll;i`9xRjniIADl8(N6Dn~E1LQK9P!<-1NQ$8E2&+)gRO?KY*k zJ)=~&_mtxH9boSRaJw1xumBwju8qzN6&I1~xRLjQZdb3#ncaM^NuU4Y(EsFPj zN^!pL0#-%(F!uo5Jcl8X4ebD)ad=mFGUWD#AGCpHes1s}_w$^oYR`qL@ODDq@l=U# zfQtOWRp1w^eE$^X24pEapio(XmC8ViNek*$O3<(pgVrlPc$Z>>PbfP0W<>@+p@`sj z6cO|dV3sDhNF5x*@7v*D>qQLsPU|o20pvUWjBorH@AJEIK9>f}RZ*aw3WD5}7woH? zkPu~sL@P5iQR$%>N)5|bN?5s)&|(t8I~5nPRMG4e6B)5X;q2KP8gWU%5sxb<;w=S* z|F?qCV1mP#Lx>L!xiCbAe-(U#fs8MM8RLSw5OOY#WkKZMe9jLwS5BCD;g3XDA_|Ja-48~dnyV&9Zc z%-8ZogYk?0Md$Fres|gfwyuD`53cqw{2vZKCJbwoczLXOh zj59b%yzQi}*TBCtf_i|nF_v5d&f<9NPeA)fWW1Y9?wLyNm1eH+3@e3ZI4L-j_kd>l zD@gIsfZXi^J`_G@m@d07E}T#HIA0&9_oKR z{Kk5)PHe7+*U*6SGw}i3NpMHyk^8~zR|0=IbKt9H$fm|Z3u~>lpw1b6#S?ucP_yeI zHLEU3W_8&zsVk#lHG;)p3~UD{z^#A;r1pI#JbeRx(suZb`AB_mx58Tuub~0ufHb&c z;SPn{A8wCI_-n~O8h8fVY$nsz`I^yc#~0~PTYUhP+R6o6jf?Tc)*8?SM!+U;1Y817 z68P_euW|fO+CZ$0V0U*m?J*Y@;qQV1d;k)Qi2=+Bs3P})+X-&#W^%xG=3sQqp9&)|#ZbZYZyP;_*Lz(4E>0%_zl>}52f zWn6T)j$+o=vH3v_t&u%U7I- z(4Vl^)Zpdongp1_0C!po!LrsnXa&T_VJCD7%3&-@J(j{}9DTG6aIqsh z!8z~{=MfW!SVe%ZoB4AEb)D5iJHX!o!{q-fv41VGHwnjO-o1Q201ktr;3|M?|3eQs zi%-rkL@yvtulFW*j|5qK#>v-X*mWB?9^-f?c!~2qP$#^|sZ(pAlJIxP7#bipTW*4% zCe7e}n%CgG8k_;=!L{H9a3i=0{1)5-ZUc7`CwI?f&kzUn0$&gZ3eiO{_yLT&|4Nbx zzWl($;5D^U*X3|$z#R#<*JkEG;V#pCT)zks*EyWj)L@iFo7Nhotdve84DNG_M7hwMNP z!LeT(nYf`j{2IpoAcp?n1B~@BNgxN5pn=q&VlWTUXWGU|vEN^4PU&%B7NY?(U{tr3)KPd9_`j>||8XAPL4&*=ijk3%Fw!i}oB$$jW zmYg&Vu6%gP$kJ+2RGQ)Fz|3CE9e`&UJS%a-I(Rn1vx7|L0Is@*>bZ#P9zc`$FEYI^ zr+AIIAIAEV-vUF2_}^vtk=*L<6fOckBs-D;hl_8C$ZkqY&+*oU7K*1RCLJgQFXcK5V-DUXu0H!dCLvE8#ea zH!kx3!!+Z+QbXU8(=u-P7mRrrh97;Dwf*sjh zzmN>JjAmM`Q{>xMlVhGF!#qxgdyH}%B|Z+*i5{d*cq=L;IY(D^7kTsxWZ|FkTi!$R zEBpwrVf(AVXj(>-Gn&B-_~JZqaYkfva?`(1GK zas6fF&1>k|Ht}ySI>$*(V#PQN>!;>md{2%2dk)6u@VyQm0!9~e9XJh+%M`9Fu*DH= z#gi)q!V`%t@o=QUkqt)yT0|*PSOrf#mbb#u&GiQ;!3y+&aXhkv+sqO2=IglC-B0OW z$5USd)>MxY| zbG-KxFf`L?9~|XKSFMSeyIic4C7W?eL{wTNZrz z*j0wcQllD^7FC<{s&eKsm7A_psp(D?n_i_tvx~|%dsw-qe^$<{&%lqs&}*k}h5O;$ zOxM1e+ugGH4AbT@{F}pYkKc8fnyJI&H)=C+P>U)0gPETiW`(NGJX*E06IE@It}2UM zRm>??`P^!i&TB?r=~3Z=Wy-%|y>hPDsjP*^m9g-W(iT3dl!b37#gdhPmj49g{*>W_ z3AsOf;|uXUL!kj%hMS;+&#hLBFRoy$H`iPZbFEc3-&r*ay;Nlxs0ynHm08EB)Fws6 zHmrZJW&ML4>mTeIm2KapOx_=q?yy#=JPb*4IHm;0i;8o6STT;g`_b`Bz?;g^Vs4}i zN3nIAB||$qhMSIzFP+JIK@+#TdVa3u_FH3Xg1$3f<@WX}b#Pa)qn`?$LX_{s`Uhv; zKk1ULESEfGEGkp_qB^CxwkyeXKnZTE6z8^8(QZc+>CWB%@ik&W`)T$JPPt!(!|WqL#?-7`*U zo~cUl%2AS6u@by%6zAQl7@s~x`HU&tXNy984=dQ0y$^hUCx4$mfX@}+^RoiHdFF1x za1ftQVDobL7JD$x_riW}_(2&c=6402jCZ}vlDgUk|1D`YYKlREhpEiuX@a zTtKE`&|;zjs}vE~tgyhv3TAy$V9+M{2Qk|$=vsLN-7k-z*X0@XnY;r3DX#$5`{RRM z47rTCKt_7d?Sl{e@I5GF_)`e-_?!jO{Y{h_Fi%NnF$qDgiVOBZ<6-}WkVr*^Bq$;@ zU16d53JI%FP*|e^!h7T!zFgkn8{`?jUvA;&wJ7{PIfuV0m+()uDC{S>hO&+lA8g0Z ztKlE;B@W?at4~z~G0p~g!Q@;VQ#mGu@cch)wqnAq6~!8*$Oun`M+PV~GE5=t850zh zqJZcew3Sl%MAyqJrc3TI%j6m}Dd(7da*R1A`_?zIYgtG|FoJjIscoU+@y<^FH;>rILW}xrPk#Dk%yxF_gGsROLDFJdz4cDU7 zcsZw~qp=joK8-yf(%WQ}KB$H1>ohNYx8|f@quJ@VYgXEGGEaM7v)QZ00v%>A@wT42 z9)iCUZqj$=|4;`pjJ4rSW>_8rcUU61XEM1@8u@p|EV*W0A(t$BIc2%ZAzX@BJbc;P1!o zR=BI-ErK^2UL+>P!yO5CFx)K9lSdZ$SMChi6@GKB%H%+rcoHiA=J#I;^xqZ z=**4Iw4M&Fo=P=D#ReSMK%sfBH7_#){a`KF3(j-i@4%n{6O!fGRNw+SHQVXVy~7h@XxJSDatFqA@Z}D^ z(9r`{f*s%#=iUQe=X;+5-mjKI-wbzsHtiA4Tx?DU#yq4@xP8i51A$9z>zD)4NDkNv ze+QtN)nm%d#e&XtAr0FOW9ZZkv3qJx3dRj!NPh*#Hn0qA=I5)x?VS4(_?YkgLj!QP z=Fu<1SyV_pfF!t?0w}*~_z5eU=4t)Mqzm4~U;r!yLtq(o!k)ESeg*mjYTyzr`+C0i}7dVe@@-TRd z^S@RbbzFn%)5@ylVB6r0=9weU?k@)Xh0-Gz!XbT@awLUyHLZ5J=~7;Q4+0{YLNv z4-?;36TIa#;Y_%r@we|X<^bVq3#x9)UP5__m!?gEGp&Q1;&>LE2iJn@0hgT8CvN6= z8@O{84-Bo)3*68=!ayc^MJ*3<2f-FVf$zD8LH#TLqroJ?9R#<_IQl22O!?NZ_b4#z zI|FV2m%uGxiiabY`5!IhAwE9}9tTf@=ct>PD95Xw=o7J^i1`}$;w6IU#X}^PTGpTZR`i^iN7t4dNpjm3Y-QP0AW0p1l`N=LGTE83_Jy%1DAp`M)GZM}uxU!gQQiRS@i5^jhi7i~UlWcx5JcDQjBWM$=$mGV+ z3$~yaT#2iWlgD0%E^!~4!K>tRUrgyPhW=r!KQVL&LxcF=W%wSye5E-DUZBSO3qy;zT!tUu{sO!Q{vcDhtnix)w)j&0VT3^(9BJ5*gAa{8K`RK)S~Q6! znnW8-qnjquPg5CzV>Pv~f&6h7dciS5>pDXF0cz(DXa>gHe5UsRHRfUL`Bxdfh4(}7 z8h8TS5AMJhx1hybBGuyC8+JgSDiE?b9924jd z>R-&$j7S}>K3_K2((Y;;+jBar|m<04>STO@G3%IO_cxQM#xn&2y z6NWu8@FdbQ(`Xgh)IcFEwG6IWcv|4;q16mgqP2KuJFVylk#hs3dK;7zO#+^ zY(XDurcN4ZuMPAYb;KyyEqp2PWx-bfUm3Mh2VWa}eQ+47c8nQ3`#Iw*J?`Cj?ge_A z4*+u@D8ofKjN7462Ww3k7MU}=n$7qQwDFn8cWN@33Flm{Yy+n=e4g+HsC-74N@ql? z#Dw=h%uH9IX|D23OO$I?qwHC&%AB=C>E@$KHQ%V@+543^`>f(;->ul$FDPdA2a2}% zJ4bSVxQ#Zj39glM@cnP#w`8~q8Wu9XnNQ9!m$BY#GgX+)SJ^Ckm6*G#Xts|EEP|9j zCqlV%Ls{1T%CHGl znl0-eY+3(cm#IYi0>#_&{z-=>MLYB=!g0C698nb=_bJfvto$ACmapTB@^Snd$L|5p z+$qCO3+e%mrEqpRlIu8QJ17H1AfMmmQl@NsQ}mq$=sWgGb#zs-laCUegOuPLsW_K- z#kiy?YEhmdUCR{a+Mp1(ZUwn5lfT=9eBJiS%l))G-0qZ{+Y8`B@DI7Wa?hX+wk{wR z;qP}M1{RTaxMIH>{2&*QQYyoRyobLfyUtOftBvB_c>jdEr=mRq6y*`72+vrBd8R1D zD_cQcCFm=4^7HPLw+}7aXT98f_GppMDLMPxAxEF*_OheG1hCAqP-U?(#JvJzHVqdz6$mWR*+wm0{j!@=btH` zfFkr2_Fo8WmuujlTmsk1F=&_UgP4IAc)P3up9SyBI^bK`pu_CP2NU>t8T?)FG{95t zL%!vU@BQEhi69n4`IC19%v5OLJOu~ZqVc%MKiEsY!2$9K371z$ygbli+(HY`SgPa{ z)+&dve%Xevk#+b^Eet=Y`C+$eZW#M7guSDAq2FjeI?Nt?u#P$(#Qt_~+5uc8aODP& zZv~R?fH;Q6ksvgLF=r_GFItRGxRtyj9OV(=F84@3xkiS{B`QWvXfY1Vi?NF?mvwZr zEMt~vLCh-6joGHzF~>A3=8{aK9|Ld7Eb0r*isYRO4)n>?c^~{O@K?cE2wzq(V|_T| zK@_~0DgQ`v&uH@h7*n~%&XY@=t(@Xq3CrLYbu1P?;TEjGi}rlFtXg zwcvLI_#eTi82a7R_Lalf--+F|@D{_F1!qbWbpRsa4v8V}fZGFZmt^vvG@b!xnrcq= zJegyc0YI>d;-~bT1)I1djhp! zb0uHThcg|{L~M=$;c@sNfpI^VrTNP~6aH*+p!}JdSxCoPWC^IyA~%3BMHo^`IMTdXYeo!~vIYX17t}HbqY+NbVQ9iN098CS$mr)dXv)ooatu=r z-leSN?liVQKwvHB;>$5$gfEVaf?b?*j&mPI&v*y?H+w=;C3*CVR9zNGfI9?kuPXSl zd44l<5NKXgyqGejzbxap9B|30LC-3VYi82f5g*1wkV(q1(E|`{8wsin_+kUGIXOtD zya}A(yt~m(UgtZXtByJ@!`B5MmAa0A+XvSQV7@eIS>mB8Sg00!3%up8_J`@lhP7!YbxNdYEK?bUjE9!U!2xZs0ML7kkV zg3mUhAq=CFaFJ7IIPZ7hkE({ZfIlU{9Rjy2+zYXJhT&g>cQY_}4NimWDnRp^8iaD0 z|Ik4$^7AEd3!wS^&s>oE{n02ANg}A^dmy=MJsJrYyMq(&cuM7Mj1Bm;u^+J)cH68* z|C`jbZ})IC>^ufefeYXUVDQ`wZU=XQd%%4Fv-B`{91!+Xb3u&F<9_eN!{iY3iA;15 zjD88C7cuNbjClT@srN!9@T06%*#1 zbIv&j6axkXBuJ8+a|Q(zR7A`<&X}`e&RM69#TT-R~D?&%la?tWIS zT2)=OsP2aUTMk66!7v0fs0%Hi0~D?W`Oc{NGo$PebP(^DLA_?C^Mc0mjM-fQGqHyR z>pe#4w;0P`rP3}k`#*zoPVjftBOG~%GpY3-sSpP$2O%-@f1}o;n6L(<27i@q#4MmS zISFfY)N@Julam-q{$n)xj|t>IrlK>GS>$|DLQ9yts%LU+M9Ya8m3kgWI8~KNm7QTO z^9pwsuKkE$8w~0`kpI;Gh)s+vD3cMh!fN6UbIcq3ZTT|u+%wEjj$_AR?8w88{miOz z=(zSU3(82et4-asMKW+Gg_ES*R zcm|F#r%~@@-Osw19M&D|rA~G;zf8f7MC^zc{Rq#&XpA6K$5Pr!n!oG^V@DwM;?GRk zmpbt#>*2|_JMe}Zb37Na4Jz;91P-tTlnrm;4ye#lp{+ta317q!7ctDMB3TO(&fG4P z`8IFL!+%}Kee|F#ebE?%#&9&opfM4R8FU}>$rmg~V?7$qc*Kv|N#ZB<{S=AlU$4!`3Dc`#GN^&8}t6{*x`gfwlUA$N}O(HR=J5e z=0;|n>xq$d*s+%Q;4Pr!30BjUZ=p-~z%!9_wQ3E<4Zisdh0nlH>oBgPbpTY8SK6w( za|hKGIxy$nOpeZ;x$XvGfks_48lz!_MmsdD$q#g=#J$lOKqL%9V+`k4_kql2*1eSf zuczf~$2&n>a?*(4)9{i@AMXJ9gCl6ALI`+*BXCvGToP=_Kdi(T%ZbmW)X5U&x{I0X zE+YT106XSk#~kdKjU6+wV+Q`1M!igBc0QR;6WMD5z8Htc#!x!-G?!7t=15|5B&}{F zHYik=qq;wO3r&~2o-JJ^dWAWFdH7-u_bAOK|1gt2V+OG?ofw@;TbYa}4oQ%Xqe(R2>)Z2B`lXkDcTPS=hmFoTiL3PU;fbJ6O3;$d( zgY8tIjaCVC4AD{f2Xk6MZL}Itre5{BX!XY%!}$L=B6bE=EhY!Cj&D>J zEtnj{E>?S;foDYacaWMG6qbJ>+nF#KCV(0;jm8%vMJcp+_9Jz|UF@{VI%qYb zY|SZ8YxFu&8{N?Bjox7NM)T8DG#22cHGJzxZ|jfOQs`-pv4Z42N4*F2+#Z#GD;y3e z|IgAvUl(l{0YkMJei9dh$i)mGKKs#U^rcR?qaM9_=ruvFC3@{BTW9|DL~{U|Bk|5; z^yZ1yTC0VX);7^X%MZ^bQH%K;b&KeJMJc{(+MtRqMNpb=%Q0wbk=PkI_WhN9rbwsLox1u(6M8j-q}^e^Iw&lBiuWU(_mbOVlj+625BM;%01Ej{a(kfdu#J<+9jDPdirqUcz%s%UReL$obbPqZn`{SRfh|Di1RKQQ@dR=%HT zTz&+(mPw+%(L7Pd$X3)eauk+E0m8y4MVJ|pg*Lj&_6>a33=cNo^Tp^-M{jHed~eJc zw-V!Q=*nm7ip)9qeFto6XINacH7qY$8<`5L3g)6k#oD4-MectvZXp^Qw-XI3brJPV z`iQzFL&>#F5H(Ea2n*9yqME6LsAB3TOihzSCDX&g*z^v(6_reW*TjMy{$7Co6tqT~ zFur8S(W?sj&7UtE)iK^zaL}l|8qLTSmQPJF27@8-Fa_0Hs7xP=9Y_&I{T(uvshK;CFljkqgnbKB8o;I8nT2uF$V}RTQuBT$HHs zRXE^-75I1-k^`pXVvM;*sY1Q}!m!>bQMUecVNidGC{cf-C|2J?=+=)C+VygTR=tZtyY5q= zQUx5)yb-DDjna)WT3N$=eFJcH;b z@PT(<(I->ilhGfJc3>js3iyAHxUDy>WA?>+$Fh zLAxh<*68slUe*|4F1q!hRzo4o(XP^r{6h=={%55l4BC_w#oHK(Vr_U1W}E6lyA21n zZ3=Cn8w?oVS{o!c$;c%4$QblP2jiEl&K--MdRzlmb9r_)a19zqq@1~E@r3oM^ z)TS~W3W3}MNA|#3J=#ML96JOi!6JUPhn?)30w*~Z8@hiK^Uzo8kp`o!4pljzR?rL@ zLM^n-(Kg|fhMic0&;|YOf^$M|8nz}EQ;trZz#c$gtA%9)IN^X+z|LwiJ765l1LCs3 zCnVAV94Gj&q3;S!CB+P`>;L3h6IQ}rlJmA<*ViIK?+L}IrFtWza zY8_BZKC2v%r8RAU&?--vOHj31gcz4)jt5T5<(W&z&_P@%7 z@{JgVULQ>D3LU%9T)Wc_@OM?J(Gc^Cjo|!$>MBiK%3ziA!9rLJOJO;vMX;+&kk`PD zbyXOY)MksL?Kt_`q5K|WZEe}dg8@@A`=6$hekum`AiOAVSGH;$P6M>fhj0%RWiB?J z^G*S!I}el|?qr}_n3Gt`mN3%{0`1vuhOJr*jEgfkG$KD?2GofwmFq%fZXd^hos&3H zACBP+U~}33j^orKt6{+7(_x0#6uFKf?5l**NC+smTX&)r@-MEICmG$OvU)r zDA*@@6WiSG$-BIX0qWU6X;Wnk#|sDhu53La0F+iV#6cpYKssc=9>{?Ma0rgVNjRs= zJuelQ%+)40(ViR#g+4or+y_oQ;m3NBw7+sNmQ<6WD!0;BepY2x6RxPbWN1;>uTKB6x2YsAplQ9E5r%=btwG zjz7M^J1Be}M=8o{_fyzN_17(BKeh-h}5*&krLLUtyG|b54)TS(r zDN8HL(Sh@KB{X{xCjHSEg2qU~W*lXkLbIGj?qebI*HwhFdLF17p*x62$7w#w*cY0ZU;CJVsA7p)62MI}~U! znr>hRapA)JZaa12#2nL+*l@s(&Ddd2d~9H5x1O1v9W&dt)J@^@ILC7SDRifEIsY=c zxQ)y>-RRoY8jLKW_XZYz1xfcSxzwm<))baKimflWfohtYVFSKcNB&_g@yQe7>9T52 zj(TV`H|<2nCy{w?6LW{XAJXhD8NZ!+JUqSjsk&)|fy zpt=>+y{WE1b%m|Ebo!t( z2%V9fdlIc?HlLPbl|9kxLF7bp=|2b$*y{&K4XUeCT`oPR=5nx7qg!}soXO{D_+l#g zhsoq0CQ&C7iH!-YjTuM&VJvox#*R_gF#7siubxui`LXfCt6JhT2}|^qdh%9d)jGxYzRh!9urFoFU?AS zB5)a@Tq^g#kWFZF*5b58m47hC<5khJq+E5-;{B8KVcp1PbS3M+T^Q)8jL1;VKOPUU z27>x%MJcUlA1!Fz&B?MhBct4m-cQ|?sN!%7C?Cv$$uJ6rXfeKmzI^TtJ@JJNZM8df z(v>`N7xIOjiH%OgXh-^v_T)j@@vkl4w8lCsywn0OHYaMD5?PID9}S6#`gpz`JzQP- z?z+TgU3!0Y+1~(Mvcz<4e9z@&XfcMZ`shM;K6mAJowchS(aZQehreL+;ydIBFc zvA7of`RGqZQ!U#bV89r+6xz@U+VgWehG=c|bwnGz5~7uEd10knNwm;26V3E$h^EE3 z|3RPoAM~w6gW}x(zyQ2X3GRO=Ia*j6Ocxdm2+R!ZMHK^2VPX(1j16)`#gZ39g^~sE zUegz>!iL%Ck4I~0ImVZUOkbcA@H$%28d{cT-t!CNO#@xg%)mf2F{mIKmE!per7cAL zGId1VvW-OTaxF#8@*RYwp^dOG96*j`w5Vb?L33!z?;7!4LnG#1M*5p>k8#{H(c!qU`0SX8bc z%&SxuW>u<-Dpl(W)2dBGC9~F|qFHBQRILv=mf@n5`D9Vje7?{(Un_LYw+n6a5GLrS74n8RJ}N0Ck}jR0s2FjQh-si^>+|g^5Ka zVO+hMs9;%37+N+I{%ChY zr(Jd853R;f4{FuGT&RY&X>G>-b(sH=iz(HxgecyyyeQVtMCdj&7ut>Zl4r(nvknJa zLmwCc(}1f_V|t>-zI@LWq7gPU#D)g%#8P5xD*hjies8p`(Q1uW(^`BDb)W`VKozu& z>!II}dk`9H3;m}0Lc4hxf`))?PUSV{V3jy&(HuHZnLS}BOoByV&+mO8iG7dZ=!fv0 z_ihmj=ntm6J<#FNx1uFljqBnAsKsYXwpG!tgtj3=<5Ep|4rU82p>0(RgAJIZQkiWz zQCljoEhlK#1lmA17zpEGKCI_^FG%DVY-s%mK5A~67=u2q+!0;TYD+PiK|`nmHK7{X zCTJTrL%#+3xLT{Np!3otcuLTb&~aE3)Lp31E}X1O1851>&=*F-Y*<6UyF(oNAK^Is zzOcgMLT!J407G*KYX2rX(FZjk_0t z-A7d_1?r8ncvg$((}V4Bm<}u1XFEi({~;WIpJP9X36y;h+TE4=IPA|HP(6OOq*^Om zp^sY)IGt`Mr3*ID3wlFe9h`_A0}ZH9>SQRL$WQ`nD83kq!-jIe&_O^<3|RsW>=%mY zevY}zh2bO5gyuA8b;if7DXV535T8{Jh-G-90-?}}Z}r_9=Y{?HxrhbGml8Qc(RVj@h2sW4rSTnKi|tw6584C?Wj!pxxnb1-=J z3?L?EdT?<_VgGXs)Lv?y$=nrFHS2KjcSE$R<33}`Y|xLo$Bmk@D%}Y%MQOoIm;>`* z0W1PsEtV6f>W17k70G*8uqCL}gNRpivXv8IDf>8+$B1UYdYEG#@!l`$0ovp5rfAnh zo6C(bP-Pwks;o*^Y0d?u#c4D-iB)WEVJ)nOjj$OUv{@icoVpqVLGRL>!5Sxb8qQoE z!?&{ImH_spKJ2-_n6T}FzgrPwb^Fr~;j+@I%o9PCSLrT>6@VKxTJ~(WfCD(fc5sCq z-~~G&06RkPM}#Rk5CT7xiVh|&0w?huC-Ze@zj*$jdQ|k}*X_}6M7hl=cR6;^Qe{^D zT?VS0s(eaMX>q6~AK}B+4+0?=!XOf2AOWauO|D>XDe@9k$$?NOSvY(52<~B-PwoSU zrEqX!mL><&0_~dAbp=93S81y<+kx_{6R7ejJ*5>6ROMfJkW@bJf=t*8`ymgG!YMcp z*Ghv0H#E=yF7zP>GMUNUY9_WGz*{Y-@A}i|2j-CjQsq{4tNiQ&%C9sdChLV|OJl3F zvO#Gbf+KJoP6J{7R}SP3pC7_gcm?kaxv`@TpF1+aB`%)givkMyNUebQ&m4$0zf)^4 zjG;QzhnApLW7{x0>dS0#2&3|m%u>dYmzd07uV)}NkGbn&X7{T|!E9g+h%+kbQTBFm6S(*(Refewg zVDbbbDEj|AoAd|)e24~`OAzHSyH_h7b~9s7KC z@8C5Qz-{I<*O^ycW*%{txy)&12}hZ&+XK)8pwH}0Qe38N5m(!Wgq){iS z)JY<|hGhW+7 zX_f98P%z`V+rI#Ktc?8FW)?ASrQxDg*NT)>>U zi11VcGzJsOBheU-hPvxwKAu=bt*E!NdeYQlY3hgY&{Jxw@c9?>*v^4FXsM>8v{kcH zO^_C&X%IwGK@g~JU>j@%JD>&QFFpKKhI3bf zYLuZCXKsK-Q#4wk(H@P?XxO0I2aUm;e-y1{66c?TS60$`wrJL1jG{J2(rJt!!bhMz z9EL$UIw7FCzfGXJMRhq)UDaHeg)e3jpEJ1sf*BJU=9Hrr8V%5BN{Ow|Xh-X?CNgbM z@5}jza_+H2$uvB(h{#>XoZXEYi{RUXSo#>`U#Kombw#RMQsrI87h>elVbw^;wFzgtD9fPrBAa)GEAN{D8zI^J<9zF4e4JGN$nY$7-orzv+>cg6r zZB5I!#twz517}d(qtc%N6Ja#ao1yWq`G;R9PepX9pu_tom~;2Sjvm-yL%nn(KDzR+ zGv8Qqq{nLLO!eMd(!3mwQbv_q*a?Y1>mwZa!I@mzB}+Kl>WLf_el zXm3dSXh0M;pfxt2M{Ix%Zk%7mpYnn7KbJ4+kfA8pKo_uvPV^TY$Un5_{-Ac$Nn3JL zZHUp?9l`Z8{?^lc(OkAQJ3hggGw!WjGDwm4U+Db^xc;9ik8@* zF8`}Q`CvRSe9#P2yX&G2?RA(6XfxL2_g46#CHb)yYKf(WlJ;s;%=oe?4U4pR< z+ZO!HpSML5JuUJp#Y7`51JO{+5Iqw-U`83a|3Ry+sHxpV)X;7vs_S$T<~luvneJdw zMVHk?dUHf2z15lWWN?3_1CAt5h zWKZ%fLxhpR1W~@!JW;yT8d0*evnXDQw?UOk6M6>6gs#C|p$7k(UCh=V=nc#gDF`-Hy~`l_jaWhqPNBYsq2CLAZs`zKm5BkU2Q|S0%%Cz#wl^=#?!Kn8aX% z#XWc{77+Ursb?M~A#6-(3uvfRg3ABY21}?0ysk-@K*eh0A1s)ESdxFJSxo5Eq5!oi zKy41L!xw77O5G;V2D(v@fk2O6kEpA=4m{W=p8XHQeI}Lf#T;UAEcM)Kye0Ghnv8#IYYCmY#RwPzsR4o1kU})#1daHzQ9~SQ1=gIbFI#&4 z#VI)Q9&Eecu!#!VojSlH0VF}gvnY5dz#AicJU#d3w zhkDF`8wk#)iw42e3}3Y1L@hW`OLdaERA@8k0KH%Y%;5K{z=i#yaP}dNyHCgSNld2x zhfsDKblRcOtQMw0EvU}VW;Lh-J{#d;mZb@Oibz+)j?hw@PE3y=rb62h)E#hW2P(1y z2X<%-)JKQzKumO)0?TQ1PV5^&vgIJhVMFWBVjTMYEvW~}r|tt&IiR{wlg}3Xt}>sE z2(wa+&~M5dtOeSwfYWnnAdi9_*2Ophb<&N%vY}8m_`(K*ZCXNS7yuJ!_>0(Q3!Pvv z`|o3-dzWKBixHU2D~Ch}bXwqF9{;1s0oC9XRry?z;*_S$#W>wxx}Esk1-gL^^umrl z*wGI=22v*~$*9((48&mrF?3*WI-gM#b}nqB6YyjIECS#r$9@!p(CUVN+o07H)O~2& zekZDNxN&pzDYIT%bUK34#kHD?c^|g@fv{%~!^KXEL@#!XGAIo4u{d}vCm+KJ#&FQ+ zp?pV7j9NngdU9b%p+mU9ac@O$v^t?>1x=s<{UKJ%81gkI=op@Bp2_^z zIvo66t2ONa?Q&G3p7OIQuhJa`xKpDyku4#nx!BHRI~(S~0$2>o^%xM)=xu56s|l)A zwA1B1fDT9UAyNVd29D0DhQwopS$UjY zHGlfDV~ayI(-EcR4g{KJcVLq96{EVHac*F}W34ML{iP zjZYEn_*FxEZZ?`;VKT=kZB@(4zZ<{-l;&S$<8uVWKs+Qt8tjHWK&5M@ut(XRf{Spo zIMZrV-VoD>Czy66GlJL|3V4c>p9mJsaZQbh#K25?q=oFSv{jwrD$P_(l~a{Zl`RGS zQ|mClI}WGeJY0dBa1S2Cb9f7%OETT3PTtW3-%#+E_~O}Grv2)C8`~JidNI3> zAd9eEU@o0Tup|*2aaa@0ykBKxR7OO- znM7qJHp4gc-oXoa@&^ytUKKi2Wm(EnnWJl<(-4i8l%+k{iY{bCdQvU@&=`!u2$FMS zna@l{XC@_Ih}J4*b(^rjgG@ps;k%zOen@B)zL|9c+qIzX3Q<+1s_hnB235V;kO3*o za+0W%L^1&h%qn7-J4IkeDE_H#qbOQ<5VQ7pYgeK$PS(I}zNxrp|)sY!R z0BZr#2=VjO(r1mXT9L5`)LkJD(YgeOARS^M6jbx}722HJfO8vDnrfi#PHDvXTX6oi zoWB$2??!2R;raezJuza(+;t6htR@q%l9}gnzFA7?7gGZDUY!Mad@i~(VH!+@3ux^H zr5yks;0#;A9ySO)&RrJ!O(>N))aLvRIe&A`-x`$;bYER4Q%^MdGtX1cflyD!n2OGP zI(S)zumotuoT)fnr6HJt1tqMF zw;Q0*42@Q3v?n4v)2;SEqdzT03rCjjT{X1$^Mgp_j)9{|!D||O~f3~Wd z*$LZV4Jh4tKzD(!3cKrx%seMhCliQ`@x`-@xjK+>pwA&HPP1T(tL$PBB{uqQp z;gcyQ6D{+psWtr8ndpgNFpx*>sdqvbz9YLUTh+y??pJlki$Q5mg0Y}3*27>Z3?V)T zGYcQg?3Z~ZDi-L}re!ohhc{zjM^EgqVHVzqMsE@YPM;mgH zZRio&ps(=e{F`7I%z_Cp0tP`pP#5!F(37^>gFeHCS-1@`+8sN(GRIbPZfor5NWFBR zUfS`gE#J1LPFhilmef=WYP1=V)s&v02`#NLInc(`M`QFAE}Wkp6OR=RN|gQpP`cfq zGjxKE(1G}DPo1j@s14|gZO4FR?ed4xGRI0Fe4XQm7cUJ>+q`5cUQ#*)ecnbPl4gk7rH`w zP}ro>r@b?y4(?=Yc7m+Yl(`wjf9b& zl`z!nB+Bab7Nv>}7bS~L5&HUzg`U2>&?(0Iq4eT~mTo@Nx7$pi-e}sxH1tPM#@-Bh zR92`Jm-;59C_ku4Zgr>zRSZZdl++^YqAMzuC?<@H8;Ap=H#a0qP*2#aE!^RanO~j;nzYHk3caaQ+Tyg*TcuIiC6+h<>-yv;%Z_ z@CGYj7+--oQ~?vH$nT8|wMBVDUGa-yF;?1;k*HuG^!Y;1*hJ_UR})&5YB6PQ%mGXr zOu9gSoJWf@oeT7YCOh~(mcmdM74Kme*#P`72LJa%zjJv;{b)6+h!3DTR0UHo2BQjE zqI?A%QMQ7vFsR5=F)I}#`%^+_DJNAihN>K3SvYMpqcxjZLth-pfU(*vKCb~czK>yF z{80HmC+E5K_+SL}+>0`*iL_c3RKF7OZ^B_vna`DgMM1*QSX-0@gGyvX$PVdNA%M*& zfH@Utf$`N%!2;?M#LaLdJxq;WFal=4Dt_<6_gp0{u)+KRBA>;S62t=Kw?SVmtZ2e% z>QrVos0t?hYz!4lv_v_!2K=r##nt7mDPchYED5BV1hER9IuxLe8I6hhs7rmQM{3lw zfuS&k-z^7c_K9Hsg9JP_)chjG6MqBH@51@5st}8uU#$o-XRBOm!tW}gTb^wxe%7x> zld4Vy)&SLE)WweaG}=ZuvoXGC!p`c}swNExEGxEM*$!ko5tgveHueoeDEE)d%vUkG z0@o$AI3L8k{%R%rS$9~l;qOjo*ZZy)Y462Pzbui=S%IH>LTbeTK z*QPtE_lGWS=7Ldp5oyJ?Eq1ggHabx!oe8YY_@Xlhb#4UIN0**#N7DJsW}o#G*q7kT zqBFeBv0p@g^TOTL-C^p^u$pWwD67f}8S|ONaYCOmYjL{2bX)Ve9Z*DJ4Y*r$$Bv%G za4PZH59g{|@cI#(eQ6(k*`e=XI>BiKzb*TD5`bwObA=A!qv(lFdvwo6lpr5NK`; z4GMDvYuK&>d)Nxj*x_Q#mH=}^WJ@mwjyTJngVt|n-xvmQ$3$z&+p-PyfOZx9&84nz zFgKj7Dyu5zEKucBdMkji(r9gDOH zPmsaqeQ<=fl2?sY6jbzH;v$2>r7UNag&X_EXmT*M`q2)D(f_J)D{WO~cF_#@x3E>3 z99r>Z_(!5frr1kjvV5>PlH&Ssknhv7J!h6`{NZo@-(25;drNF5RzCfs__ zh@ZQYXrREaaq>%5iU0nemX5AovEn648k8>cOS$rfMiq@KnO3e+wVJs_^%^y6)u~&r ze#1tMn>1_Q!m3s4HtpJX=xE)!OSkSeJ$m)-)35)4frEz(9X4X*s4-*5jh{Ga@|0=Q zXUv*CXYTw33l}X}x@^VDRjb#mwOhYoqy6SBTOAyooLyYq+&w(Kym$Kg1q22KhlYhm zL`BEM#U~^tr=+Iu+MSWLXK(ht{Ra-_9m+p)^w{x}r%srB4O6<2(!QWOze<#AftBU`gG=RRU1q@P6U^s1Hv}yzs zR4bTDGnlE`!93Lv7Sj@ztEON}TUe(W!zR@lw$U86tM;%%H3%PCguiMMA*xM8(kNn8 zt4N|*q^Wk1sTxKOEhASojl-&KoS<=>QLW>mY97DQK5nQ6a+enJP&JXKs*Sv&k-Sr_ zzCGlw1)rN&;OtMOK53nYwPIf5;Mj0^@|rTLEIRWDpi`$ z`K4^xazu}z5%E*em%J*J|a2_o)90;o)aN2U%h%wki32O?mcnx@zbZzpT7_*-@bqU@#FX3 z<^S*HmofE(jBh7oBI~DijntC|CzbnpVor?@XP35nb8*d(m%nbB{rsAt&N%c>k+F>B68^9uYlQB%!;^k_ ze|~oL7uOcIeSCNI(EE=z%)Rr(Vg051F0Q9;diWpt%{SuU#lVE@v*BqOCu8@d9!=l> zPx;G+Wc;vagzlH4lYV(~X;$^8w-&d(|8Uii+fUcez4mPDy3-H0yBxW_!#DSuPiQvv zk#RmWDdlwZuJ{uvd;Tdhjwit?KPVfrN9cY!KB>&htFtT~-CNY=&XZL`uD@6}=faDv zYmYtN?v#7i-Fxp1@4(%^`9-8$3XY3EADJ9|CUMt4MPoxEHl$#~zTrCWPfjZR^v0}e z_Z}~7b>sQU!I$5xn|i}l-8VS?*PzJgOA&G5=i`(9DKZ{E zB=J}G)ZXI9f#EuD&P*!x==Q9tx1TPu`t{X{!RJ2M%{=;M^QwK%oHl1XbaP3$>**DL zYo}lI^?=~;-@?L!FULmzQ)E1Aj{dOWYu<3}=jSJuymNnMzFpAr$oD0E4@lc_yT5Iko%q3VY4jUc+whkj z>w};9Z1R8N@38Z6u#?xL2$z3~Ok(9hS}XB=Uw2V(Zh*Laf1=*W=d-Kif1KYUS1#(2 zC2dC}%T1G`zd6nc{p7MJ@coYEes8^3`Mma9i#iu)5s9HZG!*rF4J~?-!aqctJfTlFTV5LKLssx z{TROFpCZ$_CuC0x@$g&=applkarot=;@MxOo2JQG^^)ZL*3oix*ATg>kDuH&$Xjk7 z?jhYrxl6CHuF`k>b{RO)S%yvir^w7|;?3dO;@sWF;_!2xNc3@h@l-jvVyv7|JzUOh z6ew4=+9@}7@Q@B&+~jr}7wOj1S$g(%lso%wlY#vmWaxl@%D0@#;__vEarjwlk^NzS zNR^}XqviOr!E&nUPC47sLoTo9D%Ur5mRp){lTKC+(zVqVxueZy>D_jd^lNV~gFF0F zt{l}BM+!{D-uFC1R`wB5a)@rA9BtqwCzW@RGb?SAi>hvsYb@-geT|KBYpo5^sm^-o zQg@x)QO{0#*Iz6B8vIja2>nA?D@p&m%V$SY58G3g2-{q$bg}(dgho64< z*EaMs9)8Bd&v^LX*A*GcH7lHJRwUP~1g=?G;|=7&8I`{uS=9K;FgD9T%rmhkvyVic&icE^P{yTEr5?-8PCS|OchT4oL0v@Qhh%KX8CUY#(dnii zPA_cy=KRVYFE6hj|MaTE(#O}`w%ons?S1u3K=}D%Veu#PV$zP}B=5=1%-p{(!$6+NC_UpL|L?XAo1-F9)fan;l3!g>F& zlP5#tj~t0k%RQKwm7TMDf9Bpp2h%c6?EAYE+F(`ahg59XKd!{*lhaM!Tv*WP`Lz`` zkMGz`c<^BJvYYpuw_UpB;X@82?8w>RxPvDmQ?rl6XJs7RwJ+_!!Q7;r6Z`%y8XF?9 zA)2~KAFTCz?zj>kPED`$>dO3v1-F*lJb18n!kx#PmjCw9$>9uh|D)G-h90~W6q|E4 zA|>NwTn71%?8L*l`{VLX?EAaOaO&bu8+HxW`hIX+@posY89%>1zv09C%evn!STo_q zQ~PD-3mmr`yXWSad)+%Y`?7!ZZj}Q$Lk{F*%AT0xIXThCPiFsJWOyy=0YAjxhuwp< zz8)G|{PnqM6$@_9tAFqD(r&k&*^a;R!hXrAr;eNQ?z_5Y-}Le)2NF&WB=*l7NcgF& z%+OQ$e;XO`FE(Tj5}yu_)qinuT7`%A=hnOZbZOUXuWZMid%bb-kr&%G?tAF!vipvw zk0uB5TTlc!kl64G@rl9bcBclOJ&^Wy(by1;AL6<*=Nl;A9~)b&;OaEPJCEnqzV>2C z*GunKjXCjd!-Cvb4tALZE{-YpJ$A(3_VJCn5fBu1H7qRnN^F$>rL+0^Xzl)5*hFEM!=__YeTO*7z+u~D*l-6KJ#wo|81;9N zX;s9lBgW$N{p#Z2OB<2(d4x`q99Jq*PBspbGcA1OqI#ZkO;cC7$;w$ewsn*)9UP=b zr>)Y*dW#I`vRQ_7-6SKs+sinczsvW%<;0nDdgwP48LTBtl!LV+L5OOB)cEE2T7rd^8pY^;RT2y}8Q0TLt ze)#E!e{DNIN&0*1k@Ua(I@UK^IOyjpKB3CMgFS|k=T%e z4cSu+e?Kt4=GTKO+I&2?zW@8&ZIj>Z-?8Lnw*Q7_SrOX{b|rW}NJ z^@uYG=K?PzAN9WyoA38qbWxH1$`8$j48jjl1GHosHsnk({C;p=&CmJE+rB@%Zor#- zhp8_Qxi5d3>$~|;PMG_>j2Qo0smT%75;Eg1#T-aJA9W(_c<8zGgF%-P4g_6^D=N~T z@noQ~p$qf90nGPC6_fj?l>e4Lx5me#%iF#?W;gKFv8~gd9d%v#=Cjn3NrQDWRzRn+-A8unQYfSLg#~Zo+J1_1H3q8F%Cgyl*a^jK1%+$R2yxj+4&tzmpUP#G^ zxR_W}{<0w$Kg4pk`R-B0em^+5+~?zSYrH?bwC$_2YX&_%ziGz9^Uk)nPV8{LobR{u z%)XG&V_8u#`RR#CxhcES_9gA#y*Kf6=B}9YscF#{lZ%QBpe}-_3uQwbHe`;{`<^$s z+{cr1EZ>}6+UCVY+d&1t+Rwaq(Q(bSGw#mkj_&k1o*NvRpB)i%AR{g*J3TEeDcue-**u=~XmL2cflaZQ!EHfebTzY)cg_NT5mklBKA%S;7?is1~ zDhP znU`*FwLNv!#WDY!m-oIC{=r#CLZZ_1q7st#C#J^dWbBH`&d-R< z!Mp<@6dRKJ3Yjxf_XF!NUR;@F{`k(KR`(vS9B}R7x*2EhZCQQv=Jsv5mp#1pob?S# zKM@?6bTl$PE5BfB3hbtzc(i#{-d)yV+}Po<>+;TkKUvu9>Wk(5&OBQ?_3)ETD{>w> zZrOd;%{}?1w_ohlfY8Xx;ZdO%;^KnNrziTKJDBWy_F~e`Q2X^Z|E&P z<`36;a&cmb+Ye?|y7p{Av-7W)^*#1#&Ex|w?U!UebKH>f*v%>Sp0`KDEq~vT8=*k~ zSEIvxe@%(>zMLKHdHH0F$K?w}Mg~`BUcs|xVt69&o}R3M7%J{vpHS@AN3)I3y;#uf z#JeTE55Bh@zvr#}{M6TutK*)zZHjp8MMg$2_mAdD7_5OPIMYg;zt>wFdp1e`z`L0yncwF$NRtcN z$H`T_BBcG0K)G$~PU$+?>vzxTJHGqQa{C%I*X2w2eCJOw3mrcuFLwBlxpeD?y~~P< zj54EdsxRIhsw*zut}l*086Xb4omf2M+ce`OIkQ%boZmc5uIv~fH`;hhhkhQ?Ww4v{ z7`9#dj&zoRqa9_~*ljXq{8pJXaf{5DyjkukGBUQZ_&uwd`1PW`I9kwB?0>@(k-v`B zPhjmq1bL7kIj63#T+!51ZfNZ)w{~!r&Yc~ldp8H^W3yET^xPstdu^6ceKyI2zV>og z|BW)E$jGEh;>uw?aqN+i$a&jNq<`x#;^YXuFgdQYpPW+BQ_eDTl}l?lNxQlZa&yBi z(y_@V>C$YY^k}g``n232{jJu^kXGwtR2w^)*w#)a7n!Sjw8XK~x+426Pig+%PQ=Kb zB3KU6@gWc5CMTA0l+z72$pw`*$W>L=$qi<9a*O#|=~&%Xx>(vu_Zq9EXU$b|XRVbo zu=WZWR%eBbC^E+nX^EW6IwIwJZPtKTi2(8-p5#HCSr4*Bj?rH)CmXDmv&yWLi_0#T ztI98x>kOC3O%;~NtrZqa$BK*PcH>3T&3K{osI*Y}m@JSxi;VQ3kMg8{@aiO_AJ@$Y z){-VpDkD?pR*|X88~#XH-|<_rWB)G+p5s2p1wb?~XJ zF9R<*KKH+~{ejnA*K4l#-7ju`=vEYd+ED1PpMLo1hktE1|4+n&C;fvL{ew6CgFj>1 zs1dp{d14uvKBvm>yOuZnwrgFdFKOEbeoEdk@k3(3-1l)&%iqSPta}r^ck`?8JjdrD z$6cNVUT`n)zv+3~>$dj=x4S;4T_1Q91*x%t@q@A<5F28K>&mo=rDf*qD&Mn~HvE!h zXZyUT5+@`!q^wT61jiJ4};J8-}S#5bjj;R$Vrde z{zrG*^(hL{3qN>cgAXMd9CU2*HNJO{+)a*fzY!bn_iJQ&$i?vNh;t!_ zqt67Ni#h6dCH8>t^{||sw*rfT^kOXMgCBO{hfvmeCt^e9gi_xQ%ryO!x2VzE{MB7w z=G%{Wn&&j<(E$(JJ6V2?*HS~gE+)hToQ+8iJsFu9c|1Hf=1AD7xV)gt@!0{_A~OAN z1{VeC{Vz6zVM7u&unyyE-VD~%Uh zGT?Z8LfGM$T~UW3_r>N$9E;xQZ8Q*l*D(A=kr-g2slO)P*lLL|{YeQ0?Dy#+Uqjgf$Q+7c_ivW<}S6bL&RkJGEv0 zwWBU}=W@NAPV5cv%-V!iz$o4UDUm z4UyQ8HdOn^zVRhL9h+Y1&FKXVo}XXQ<rgZdFM|%uRVUm!!a+{*E45ta6nc@MA)vh_~?|B^thzteTfOl zr;=jgFULj2UX3gY8XJ7E!4E$~W5cc?THjcQ@$S?#;}@6b)q8wnS?9Yq*Npo0w@vd- zpLJS$__+JF{fB%!vi1l1@7fz4nw$|Em9Q%{E|#Sj(dj3WB2zBMhbLZ*EDF-coO<}f zhFI1}WenE(k~gmSn={iYJiR`*?t^3HS->Jg`JZO^p0*wM(apu-7~{`na(zWMoaI}crn_1=Fqq9{mT{NUe+ zJP7xhCwFJ=JwUuXHCp%njVWbrJf3TD`Pt%jC!ekwlJ{ucj6L@^uSmP)WS?+t$M&ep zJH0|L1o;J?jR^KTlMv=}dUu59sY6j7r+$mtapZbfQINjX$X8=S8299)brqiu_7er? zM`_=uZM0we>12kNIz@{Y{_R#hat7fw(qQTzoLi;MB8u)sMVc z*k=FR<^6ZRUOOS_<)(Sj&z)9;K6PIg_+;m1-+~}VuZIyXJ08TlyWUIpaK5|G%kj=> z?`?N(776KZMl4`MoR#>V)1LW%S8@675OJztda3-^bF1(BFuzsC$7Q{eKdc=Y^M2FR zuy;=L0^jXey7TSMRi3Yc?A%{OY;b)ZXYc$pb&KPZy$%iqN49M#IOkXtWFSwU3~4}4 zu9fbu!zPh`nBwqz z^0X~KGN*6;esHG!w_~%4f($cd%urptI&3V?+^s4OJ?|>A-;dNz`#zymoSbSBE@#yU zl8YMel(wxr<))6Va$8qt>Dt4Q`#>C|U*D}Vr2iHfIdHQ~7_>?59N4{4Kk#|dKuMmolNXxCo?*)l{sCC!jGhq;_M+Ek#|K;WPap{t+FdQ zfc{!Ra(FRs)`PgpspYrH*_F1)WmWB^o%sg2$#R`^sA(s+*Ip~#>#UJpb#0|@z11?P z{wf*XV5N*}v_hs7iSt=JC;23Atompy5@ZwBU~~|EvWM6~9>htGEVfxrEV)ijFJmhg zlv^cN7_E?Y6_-nUnDCG0Jt&EF2 z8cXv27+GI2w_HxrUm|A~UnCclTp*Vj%$KW6&6Reg=g1AP zxy)?oP-d2N`el}M`DLcuQFey(DiYF}zIr?TA zKZiz7`V^8j>tk@v{EvZ0mVEF(yZpWH^;PeD9or({YYDA`6YFAi%%(=y1!4{KJ0Cr_mtPs!Si26#IATAnzrs~P|oHj{`uP;`<~hU z$mfRJZI8Pi7hUdqoN{@*{iw?`hyNDo!Z^y6F^saohjDe-U@e)zdbRXvhTnECu>7=p zRm=CgHuiXv>NN6YlIM(P@qtSUqND5{g(YpdAC$5EuK)fWw{{-$zTth*_p;}0zf{;3JP1c5~Dw1I$sY?^|%s{?Q_vD-|xKdS^qOTZv`A750mS0-y_H4vCDspbj1&D z_`!{FwI4P_6|M#OzGq7LPdW1}-|bsr^(tpwuV;H5#y-w;n|Ck8XU)yHpsiP;qFgS7 zC3&6+&fIx2FxUUM|H;5(epiDIdfyDp_P*zx<@LzzzvV9*+_AyGJM+DP%)Lh!`<6YW z{Ko@xtG~%zZuKJ9u6MzKE#vR+aanLP-OKKBQsCBeF%hmOBjUY|g{J!+4$cWU6nHfF zVBm$29N!xunLF?9+_m$;j{g>o4et0s*${{gu>+X*jxP3f-{f-d^X686mA}mD>EX3~ z9_DSHa689&(XW}FcIQ(3x1NX(bv+Ut<8>$^#qU7qp1^$}heEQ0&xB?MTn|h4zvGwU zci-c`MY>TJf7%e#jqx=$>>j1}`M~6|?+(wgcyVlL%Yx%;`rJEgKk<65)57z6cC0;_ z?z{C!Qn2g6xG3*^QAvJ#BQk=r!g511LQjUJhg=I!4!rH37dHbj&(q7<0}jO3pcFn$SQ)XP_HsQj>GeIZBX>sHkH`L4tr}$*}*8 z&vU)kfAESNeh2TtI^|x}T~)o;x>xPuVQtUHy4n@TApkjG24W)p-ps*5#l=14nm>0` z8SQVbp{9W>5Lp`#hVua}tDFG>Ac|l2`w;4xi%<=~1O+)!r)Vm2u5DjCnbJ7b_&;fL-jjSX#&4E8lXvbgxB+D`6qDNC=S zfM;Et<-y2H^>xWg2=+*ejS?nBCj}%#<^;z@l!nAaJP(Zwed`|{{K0d&06vbxBnNlo z5QH4!uaL3qyEFP9U>}HD_4TdK^!I*vYJTD0T3gwga;9Esk+W4nu7IAE;p38)5-3Pa zh!DobB?LsrW(G&bl!Qja)P;sdybTBr``|TQ{0|2K_8ka0iCX6h8OfCp>nl@O*zrVd zUDF%A-S1wSoqzI#DpOHK*DZSBWSLXoMo-J}c23L)5X7d0`9vkf`G+T_2ZbgSh6E=( z3Jr{T6BrQn-g~yjoA8o`ZKO+U z9d+_6xRi_%SI49RFQ?cXfA`4DQ14JY=Ny!p8WfO{AL5r%6CzA{6DW*(?>$|BE53_K z4qm8vC-VoXm_I1IBl@{UZtlmos;k~M>g;^--t=t6Yip^(ItQ(+8n#7B1&D+68g-}>{jKX^?S;D#Io$iWxSqQo2{qu9f@tKtUv@Jf2di;wC{9<}Oh zD{nDASQNQp;Z~XG&8x^3I#*QqI+fQ8 zxkU{g(*?M%#kqhS0x-Xr@HZJMI7M1(FO!$=r9^6*HI`Pi>;7KYZgf1W)#6rS6ID5? z!OI5hmgvTPmLp(4t@L2leeht`d~}^IK!6-Px8a(= zJWA>z?16BMynS_wJo+djTHdCytguVx_v~(iBgvo5uSU01WkNqWYWTNu485COE!>;D zsJx~?n$yP!2B$HR#ca&xFdEC992?&|(Ox%krwj1-34gP<;#}B;?_m#VuH8c(e>g+R zKgo&}cBwDR`l7WZrC;w*OrP2L&|a!|Ko4EkyT?gY(Cwzp@A5Wq?g}*FbVZu8x{@f2 zt{f}Jt}+{1=PRmxTeIzS0bZ#6ees-F!dB8#vYFJq+d|4)FOtG8*_l~iRF)-;X#5&8 zqPr(_#PqoTF!h4>F#V?cF!wHR$W_K^$V-tk6rjQyicn(=C1}!zva}tBN_6ap>h$ae z8}+9P@L7%TVIvvKT1%cko=wVI{v-umS4if7^vuL@#U+uTwjO&?Z1V$?eV$;0y(s9X zU4V0e3;ZiA@Vw5z0+bG+w=w@9f%%7f=s!r=LX9*P-pEcD;I|TgBUX}U#WP98i`gW< z>mW%Vyh7q9?#>JcxrNw^Vx>1|{^AZsTX|rOy(k>^v4C}mj>Vq?xF4f|&p-AUbhCws z(^QB*V*?rItfBb471UmwE+BXrd69`3h$=CX*S(gc4F84aFfNc#5Et`B9|U_*EOr64 zm2A*o#{i389Ke1v_M-UR7C76fSb$lBV5b!b|Fnd_-4qDlV*zn{%^_pI8QecGUA)2m z+ZBbFfp{WDl14UR2IC-l0H=u;`XGF`Gm`@{a~(l>ku7K~wFcuAmSDBY0_@kA1LG$% z;H)zRm-Qy#w!s*UZ)5QN#R!5o8scFm14!C5T~wh5l2;)@5ru!YJuxAE$~>T z3Bu(X5VT^tC_oP+=`pqlhlS|B{7hWX2cg4$LWSdmg73rx@cw7GE2;x>Vp^awLj$yC zs)PP4RWP2V0_L-o!E&||*vwG``#B2WI9CB!bLGKlt{k||n=Zfsf3qC$H_HL*2PWzZ zKGyc$QnSEcbrA#@t_FYW&EUt_Ga+<6GVbGdcFa5S%BW|`tr3sByTgJq>0!58`61WW zN`w55DqmbatMxkfX>_prHJj=E+8-PTG^T_90DbScAMg06P(}1r>pdO+}G@JFW2pMdt%tde{b@c_sOiq>9fUqW+&w( zqsO$4*>5l%03A8dv4){zP3w#r$LlKU4CD~0xDcXr){I7(Zyk!TKQs`^IolsBxYZlr zFVpQCsoLq2r1RM`$EZV4V&3LjWBG~yn%eBrZ1PO{rq{n@$IK{Eh(Q zzZ`g|FMNL zsTMsLOs6nQoICCBJNH;k2VXfbaU3Q&xS_`Jzl{3-&Wwo!`31wNnydR$jkk5DSRd&~ zqF-r^bC&%W>7o5T*w6fpf0*qHVZ39VR|e~mMd8=Q7VS{sWIX*CIiqTb^69OO{(ZBi*&remeNdfrT|K3pI`h zatJ~WvA4xWQ)TCW$x&O?ooBGEJ(qH{Iotlm`xKVa%XnA4CsE#%n$SSIiogiQ1HVLW zkuZ~Y-@Di~-|L}!u4lb_rl5(RD)_`s7JPP?4*rJ&2RV46&I-ARb(Og2aJuZgzI@e{ zp9}T3wicQnZOF5|@j8R4SeMMxtB&=glt=j67l(u~?+3;@<@u-avwaKPGKH0b44+pX zDPE24iC(STc+U>{bO0=@?I*{9i#p5u_32Os2s8Hn+uTOtEFGIP30lvjMJ(D|*g-1Nxn5}WIf^XUqe8BV$-$!-?; zaX$9hQGv|#@CfIWkObGn;4DFWV2NjJ;A5|7{|}y#e$6fszHO}O0N6MVlN_8;>j{sc z&NxrTuop#t9)9~TQ(oTkPQ*Tgw8l6Hgi-j|xy zUa7RS>50~+x3$K9J*l+1T2V}sE6QW*Q&vmCpWqQ-Y(gL|bDG_}Cq(paNVz#GGLYbFW!YePYxR0)$(XHI+063^OCpoy| znViY|0rp_%&$~j}Diewkq|-Wl<3p6QtaL0XBYTl#ZPx0H{rZi%hj=>WLM!3jBdpzaRC8#z-je^7Xl zwA6}|H}7TU)HSHBtEkuCSNO{8d{&*!z2sU4wU{clQCJzzGO)yh=3D5;^ePB-a?g+D zx#p$2@$!n@o%8Ah&e=_T=k!+2bO27s!TERG8!)F8c>w3aQS!O$40-?Z3aM+5n_bY#-tJuVgHm5xNyI&;sf0gf=?6Y^F&9<~sGgNVN4N42mP=U-*Qqqkg;QF@W0gMP zvx-}sSp{vZ=>R{|;+jAXe)t^)`#|*P9U?6ckC12emoS5%IP*c9=BoTw-JR)8CMV+? zEN?~B+sOyMVQ2|oIvaaDcc<{{gtlm&JFp+eFqn_hS@cIm9LGm>9EXRW*bWsPj?)3S zAO{|5|Df$;B6%n2e6W+ezq0Q)ML@VV=P_wP1PZM25 z(8$%{HMkmaKX{un-v?Pa)<;um^=Wo?^@TLs`o}ct+g2L&c?Wem0N!f+z1)oZ;}$ZQ zwTU!5UWI)a_K}jdnA2uNJ?aw3f^#weJ^_;;}^0YzGPyYzC^7tOx3qt-gF#nGQhkBd*ESq$OiEc~~_QpTC}D_Z}xHLpMq6nDp$h z3B@J;6KZR{Cv>;CPnhj=nXui@nPB|Q1nzO{g?Z8eyw2D|ziK3nKfizwSv-X zmhkij1)6S72jGRbp~lW44-3Rd@#7gJvu`U&9Q}tx!8Obw+?|QNC>9DpWrYi9ujPQr zMkZKeFA9h4_Q2k03og50m5)CV7;O z8Hh4blJRl|i68%iguyZ5kJ$q+xFv?YDCTfLVWA_aFSQ4QAE{um#v1G&o2F`pXa9N-P?h6#bXQ2WFOba=McpY0U zUI+IMuWN&4gasS$+1rQ(>?20_PIS?a(S)l+4Q>-@qDhs+|GGHhs z4JKkzU?C<6Rx|E_?TowNFyjueX5InkX#p&;uBKr9V1f0d4eBfgY8;n)VjxhO3m*DQ zKw$L~xYIX-8-FM8{q|4rBL5zDN%?2YIrq$%Q|b9p?!!wXoL5(d*^So*nH@L2F#B%x zF-F9@=@Ylx?cq+VJ&1n~|Ht7!WAy*bI9UF7KG>q(WZ`_^-xUK-#kt_6yA(VrYr(_u zH*k0PbKK46(3or3@lk%_nGu)l3&YOESB9LbuMcsa-x_3pxbub8ez%v|d#{@@EY;yS zCf!1tko-skiSGfdu$Hqz4iub|G}N11)SK=SV&Ee`7ksssg3x^Zgpd8!Q7`Vk5f9I! zL+-(62Hj#W5Af4(d~qqb)9+j^*~fh>)61!s?`E|ser9$nw=o7)njD5y-aCxRzjd6D z`5wUfzZ@))gClA^7u0&5cSONYZY~6BE}aN4T|eq)yM0K=J~-g*ezM=w|5BfOz4zHYUcQ~5q1xtL zpw-H)&}(5oHT=kGF!{h}H+#$IF@DDQqF2it)~IHVseBJ$gX3U>9ITN83pu!5Ko4Rv zD<(5%JVI^hP=w*nU&5?*_6F0Bb_KXx?C|xv^T{hnu0;@~-o#JVZFJ5y{=h9UuV>d< zzG2r}yBg-fB;Q|D@%C)_vGhnyDMDo(de z8K>X8m^)-}pF67aJpdJJS}Kl%9cmmW)Oud%7YAYA!APmuLkY@D`V#bh?u@tivn|&C zL~{iDMnjOR{98Y7&6nQ626Z0MW{=!cENfhIZK`->c9kwq?aQ1SX(djR9!#%guJeF# zw)3$5_W-uY!4_*;d*tAPT5mE75^{r##YoN?NKsncovO2;J;m(L=0w|5AEFuJZ^C#A z&w@O)9{B|rR(nU7S9l~?m%3-#mbewsid-K$-sjiT^LXu!+33M!@cPZtc!P%D!~bw_ zK%K=uf^z}=!LVy&B>vvazI4SOIPQTo^5`1JQdVF?>7xY-g3i{2y2e89;G0A~}947Mz0jJ1B^c6CYa(6~o zj{L&bB8{~T1xCAG=UAS6oJPA}ArQE+O{0KyH3>srs7w zVuPK}@+l{3GVO1cCbQ)3$MH0CqCAW;!h9(y!6DSdz!--(|5QedUjaMHx0;KcTe#?b zFvEmh)G(i3%J+burvAf$jT}7Do5TJGBk`9=Z{}^$wj$YiA1l;Wy(!iI<8h(IKNUH4 zH;U7ka(Ri)ni;X~Mk$d(N+ae|^fEp1V?hqQ5zzhz}<^+b6aRWmBb@C7S#P$p7wD$|_vHl)_hT|~F z!3j0*Wd0!H7|w-r)j*e6;En(w^f#z9V;rdxt5dbD3g}KQBO?a8^k1f zT13S8Q$wP|90H@_n0}F&Y+*zR*C(RR$vdKj;~n;y<{jE?^F4g!;D8)lcHz3jJtFET z&V@6iz33Wge0Wdn)l7-n?T5KjyKO)t`JS5rICLke% zCXA0^c*mx*Jz|Tvg4jn~_n0P*AnG$s5ZP_>J%9u1%}EY?)ZLgr7{~irda_QEmag@#4B|z6Wqb4vt&?lSjc`6eEcT@jV4UTb6%a{q4bB83kj9B5|L$erI2E-w*P%MWAA)#iXbY5!%bID4kbF=c9oU9ra zJM#mRliq31N$s)z9)OM<=*YnX>zJ^eWFT!1X??Jtym@($JbHhflz)(0bpM^k#>`g+ z`;(rTpN)P@6%VO#l=H9RXn2?N_1#N7O?f2&mQKYHRCZC4J) zZ@bkqWt`u+D6`)RH0f{rbZM`{4QyX08QZ+hH?@9UV`lZT!OZI4UK8sl{f6HIuvX$+ z!0$<6>qu|bQu3r~7CwIiDQG`IvOY_YlrDvNv7M@`!#?Z$=HFqw->cp7AGbF93;3Dj z2K$qX1ijTmn%3$kZ`&HKWYe0cYSo&nZrNI?L20Siv}o$lv}hR6_#S|R{)6{wG7`6t zJSmt#%CQ$^KAywK=sr!7`tFEC_sh-=`J%kSuV3>Q&wiuruKkw#ocrznX7{m9(ED7@ zI`ny5wCnS`O6?23Zrzu7%d#)$Hl?pj!lLiRUGu*7duBZY-vu}?M-OB^d73Ih%1T8^ z-t$=`wI9!644xs8!?#3&Mx|!^j4Cd6A5&lLGG_1#XUyU^=9t|dj$@47_G3=_ZO7dX z*^K-AZ8aWzlrkQ3++sZag!y>kDbtCX(TfU^QP8>=tMMeW5yV7O4Smkt%rnpaQ;&l_6w_62vW0gw&J3YJva+97;Dh~*n%01-RMF5O^o3T(MLZ<8^nnQNQtO{qNoa}i7JD(m?9X= zPymw|azL3W3pO)lfHq4Sn6sq7X_h3o&b|xY2>&_XL;8I}B5@47U*NU%6QaZ(eGm%z zAcp9JXv1FgK8_G2%*V)Mj}RHmgh+t|xrbiJT~Hu*K!w}}b#e=|$xYBFH^7)&2Xm3D zU?p+|Y(*{uUE~t5MZX8oM~$P0wY@g}ZX2QAv_`$@a7P4K3NwMDGaopVrNE}I1{VKk zVES$WM#OetB<})x&YlU!lKtZjH3!FOFAk5=8vY)!Z#yz%*LQ5dZuoe=E&S6_fj*rfMad>;7kJfUj$G-b=r~Tm74!enKZFX>_#SYFl+rzo30ERdY z2Kc+Jha42|2gDdqkxHJ{O2ss!c2Lr4n zZBX+v&S7nN6ZMYtOz>7+FyUqRy4nai7^B{_Ma|1b z&Fgv{^|#~<@Ks(g>Z`wUP-wZa-^+1luZPRwE;p}JpLs!-+MS|re&VFuZDr-iG&3G3 zG&$BPH`3mz)!Tp4cw^tI_R3*I;TavIAEDX&kOAUT;VTDYtZ7YA>(Nm2PG(}Rz z-{?GZXx4q~(5>@`F|1b2oKUD>f>aq3?oI_T!Eu=6V1YVoGMD3mn%5J(Isdz2!=VcE z`@^(Xc88j6?g+Bm+v>+Y(d5Oy`oZ1%-aCGf!Yk)!)#uz4%{oq=&SO@&UM=gXK{ex} zVFkU@u$(od{eV5LTEqsqLN-WG1u#VpCdk1OIeg8d;9lv2{q}rIMSZKj2s z0sVaLm}U+a6tkT`eky$BV2T{9QR}f$>$&3|;fMY9LT`)q$H>m@Oi*9Z8gIC{G0N(| zyD*2d&jYzPANjgVS9=MSDg+@KrEW30#r#ymLYF*~0_O^|Jg1lDIouZWY^OfsOqUUz zbQe%fa{;BP0A^U*PjaB5)|<>gc;FrpaDfa(h>P?j$joU^RbAecY_O?5o^s&dDB77v zq3oMg0etCFp_g*8SD@y7j|jayw?w0C*DTXa{sW5)-V;iyOQU59uiGqzKdhg~A6HN0 zgX&ZObL3!#`oa#io)hj7UdPBp(0MWteUo%1OV4i2#B=}YdcVF(HsAj^*6wsw1oK8| zu#0q|zlU<3&`&GdJ5)czBhENYkZzvrR*1e!ja7ncy>-0nXG*;Lpi!**m{yG2xcXH1 z%E25t&`|45<_~<1lCjXUq%ZC|=}42D*_5la_+7T{#;0j!`)U%Xr^})lHwwd@rSgK^ zm9zYXT4}z)`pMqWCJA0C7I7YVmN9}#>uA9nn@B;MRg~v|NrcC!PK3vp=2QR+9EV8` z4#dX=aB%cD%wy%oter;6houjfT^r82`@l~aSf zwUPn?4B~tvOk#wI7LnfBR^eV{Hep`>QbRmjZ9=`jn1y(c=!STWYEOl)94wFn9nU7Y z;U3}t7a58=NxD)lla@RQvG>LD3!mNB`nfvCWY2>%>yrhE4%f0`*^;S|e5J%NPtCYs zKmDk{Fq3fqI7+BrrggAy2{lkyXB*(tWE<$)Zyq2V)(;Sl=u8Empw61)z}SIn0@r8Y zAuMmR2MX#<)^=PIxHf_hkiPh1 z|72#^T}e)1+>1+aR*Z^u*9eR9(GQ9UGVu?K zvJi%*SbK*QP(4E)+Ij?ku=Nb>rFaGp8hQo|>rDl)#Bs1h4wIP}?2j@YwvY5+560&F z6Xb2xMe?LtYFTO*kyDimT(6diTsm%PTx-&O8z4M z)}A3XwfE*es8L;$SE08fy~ONDBA%Oz&ULsQmc^C}Oy{Zjrh4jlCHol)62r{+2?-RJ z_#A7exC$F?+*=!`xK4_5?4Xfz%&`7c04vnIw!dQj0Oy4NRx%K~owVlsLEcpVK^{Fh zMJnnfW*0tIUYS*+yEVDO^l(h6)w%E@+O5C>mb7o4i?V07K+`SLSD%*=YRpZKGiRq| zQJ87vR*cj)h|d=(y-oC0Bhtx!|(oXI2OU1@jYPvpkzI%!@DRdpC2d1FK^Gx z{a0~CYMu6$xLV@_5!IHbgDUK<2}>DwJxiSAU5nh+obL;@IRzp5%=}nm$NWrFT3(sC zeco$x``ivwhn!)3hpZ8usQ^^uz}SGll|Pg5i1nl`>qqjeYBs5Qv4Ir7IY#p8#l_O! zDJ)5Nt+_Gsh2id?rxwS3AKPB^sHNZH*Knnrs@xP#z!KgO1cuZp|fGui&E@}+lHDn-uDS28jlT_7WABK0olAMNkg=NWw>j zg;5O}>w-TR>=4$QANF`_bDIC!;WGCXN1XY>Rnp%J`-&?7t7 zyGLc2dymdKmu{0S>~5<+=-srvv~Knx+iw0*n{KavEV~0wTXe^qHS12lVA@@B(WJZX zvT=9IWuwnSmyJ3mrUW?6Cm#a{sYxQF2*1B))QXY#&b1_>?*IwB^R5$5k!BhK3?BZ8ggBYwM0N5c1-j3nL{f6Or3M5fukp(VO5X zdILCO*TGfnD)=D$r$jPlVL~hM|Hqyn^tS|kA2G~-EJOceJyC_<@tM2vd4J(^PN3g$ z4*iZR=ylvA*WeDh0+Qqs^0^3#Cnvy$`~wc;I54IJsGvWp zh`*U~SW~NGJ!^oP*IXRGJq4%?=}{}UWkaU7KXI}Vzt zyNxbj4gtr(MphK;HD>|Md_LGaECD;0AHmk=C!mI{2WrAbu*uvsVSRtgxOK(v<5o|$ zjaj|hK4RJO#}K7w=Ky7BS3d>*?6rVDdk|d|_+u)7DrRz&k%J=E^}49LO;P(>qwcns z5e2&1Y@nMg07v^};NY|xXrAjQ?1O(Dw~O5}W}CKcl$y6=gj%+9*rxW+A?r7L2CbX- z^;>oA@3kB}&}BJsu#*CZIxJy-yA|x73ZRA@RL}!a#+q6mb+^S?tS4@u_LLF@mg;O^ z8!ZGDb@>E?{nHp-uyNEO;J0B~)b=6!(56& zTMZm*w;VtI$r6sWTESnrQmqG@Nd?Za>><)qk$p za`ar2HJoa+f#V-+;jgIx8puHn>o`rUsZCLLQ%_@^kGh+CR}^@PvnTj^KaB7!R}DHl z{_=(EvbB%xy}O4Qe6WiibEMNDl?wNMG2j|J2?Hy9*$21&_Ew%l7kLvUdmw{1JvDY^e9~G$38K_&$KuuTx_OgT>EHu|5hWl>dpt7mv`UVG~at=-F5Fdb?DYpJGlCU z1{WSX!s)31n#e&NHLpI_^;Wn?&~cA&!ac(6w#b;5?5qJFjURfwP1baJQn$3bvG%mO z@Q*ZceaSwtcyTNY2P4$HlUWcJu1_AW4bNL-$X8}oUx3;Vo&JVDeG*##-sI`H|AVX3$#*Uu z7hiGxZ#-j1+@^xJbJHk9&=nKmkkox9JoFezH-n) z4(6ygCvzCif1}(n#lwDyBPuRH`$GBW`^Qpo6+w8#z!= zZ_;sX@NjK-pCki8mq}-&gjj2w{QM6w+N)nhnr?j@LOodR&p1`)<9wyqQy^a8?kkn& z8X}*~i&4&SPE$*BD$q>jR%<77-svQA+O-p%hE$SxAe+F0J5vF4CpnZFEL;6B5lJ=O}A|I3F=DtnPTvZoi{Cjn{^?|Zr$CE|=+{^hsuD7#2y`?e)feNW^ zkt)gjB#lH~j&{6Dxo)h>OTB2PR=pVBfO?E8C`7x$y{Yh(gD&bD8{DtixHbd_$cXBPG* zn^LZkcbQVN>eAI#R3#Z~c@Rh0Ul3_`B0H3MIW5pxJjq`m6)zMj#CV6OMta6*hI^#x zh6)PxgWVq)1i3dF1_^q#gFPoyf;~ZIDtzT&h#V&KD9$)9z4wwYA%{tO+zHZBYNITa%8G`kb@mX}-kF zs$7+2B^i3Z=B1eJ$xN_0o*d(NF+PHGD>{@f86NDV5E2-m8t5OP>F1ZIEA-7Z@D^4Y zc?sVkI`w?~#?*cMKyE64K58se)Vz*3FI~5iaqPj^6}5+aOgltg6`UZC3UALSFHl-q zn4`NfJKc15YO?jQgm{Mw(J}0s;gLMa;BXK5fKWdb-{3G!@1S^HkHBmL_keOkw}3ZB zZUG(og1`|Ck3f*03SfX7%zwt;Nt~C0-^fVNHqsII2YH{pi##nlMyg80#Y&147w6?` z|D2v}vMVv&>PSqA{kiZ&=8fPu=ez#V0y$x%P{lJMM8iESR>w6gQ=b=FYUmvL($G1i zO`jJwticZjg{c6BsCg-|YJU5Yn(9NU;_5Z8FjpX5jBc!bCrs(}r`9)bpTI-Vw zjQ)ttq5Kt*X?G?#jd9IC*-64D(M`r9-do8nHb~tiCR*DmI$e(wU82v9dS<|iYSrgN z4{10>fx=V(BjjL>b&L~^h45$68?~M^p#FdIU>&KbI82JFu8HJS$}UJN(^wl{Vz4dp zzWKqBJnAX`9LLMvnH+IJI$w&P<|Xf(5}?9Kj?!QzrE1fYigX>5>hx#{ExPo?L3L&V zC?Mp&1u#YqlUXs|S~3x^hIGU)Ay4yWk*bPSq^M>;$;G}L>9x}H5^K~}MOEtk7E)%m z&%ea_xOWlly!(CjbzVMCf}7_l&C2mtpl3&_(6Uq1?Xn6rZL{jMsF^LAwwZ$}v<#3# z$bJi8irRlNmm|O)kf967vy_>nx^M<5u31KMp8QGDpPna4&+g5Ru2Wtf`dDY9f33+b z?`o^R+$-%*xl}MOa>`t8FiJh{I6Ux|vMY&@vnfeYv??i5wk&?COetzrwk#Y}uqgzo zser8%agRnGJ`2f+~Z^GM3e?Iixq84~?YVn*m&rN#cQwbpsPG}`L+ zFJ-UuGrJ?~I>ssb6Xy%|j|Eq$kNj>~J&L$PsZG6SURx|>R{KKQ^kJK{S+r$xu-$v^aqIW)rz!Qm=gsRQE}7P+Ts5vQx@J`W{JPQGw(CZ32Cf;uhD%cc z9OjV$fe3jVj`}-|kep)7Ak@wvQ6GOIA*~09f7?0Y({Wow&>=gA*P*tU)1kMD(P6rQ z)?u^Rw!?9|bq8k`rNeEnd57?zNk`aUMjc5<3_A*r8FbVh*Y9ZhN5AdMKl&|jd@2A} zl)Ui63`RU5xsw?L)ZY=$W{|*+Ux`omK_ciqOL+Y^3AbNrCjE=jJo_(NKTrpZmRk*2 zuA&UsuQeZFZ7?0+Z8RSA+H5!&^qc-*>{h+OtnE5OWjl0+Uj3mx)V5Rm%lMRd?~Wc& z1R?pDK}bRU9fA7Ww`Cp?eA!G~hW8Wp=t)8!yGrcFB}AQkY$oWYP%GavG%XO;odA%xheiOAg5ALH6l#2R0 z67{zbJqZ5DQo_cb7!K%z*kDf#3f_@oin`wj?uqDuJockdo1q1|Gc~af1)c+$tqOK? zRDd~88C>Qog2w^{2wW%+F^l9NW05Qr{~!ZT7t6r2Z=x7Iko0Wq1&TQgUmOEx%z`+= zVtn?`=t1m258@ztA1Ck}$OX(o+`w}n66iz75(Q8ucn(BF7W72$42Y;CSc=^PJF&aK z#Pc66Gj4;|OmPUBc@tu1-hk{`Qy~*QkXZDA{4s~&f@5IUK+t=ci@wVmq6M4LgV=@7 zJcPc-KSUD!hr8%c-N8(VINZbBg)C++6w!-NLoY%by$A#JBFxZ>utqP!ft&>ndJ%5q z6!;C5 z-mV0L=2hbcovTI-23HRoOspQn!Th2RYx?zJb)P=0`W8U)zZ}F-W67enP&N!#-U5VIDQ!zr>_8`f*--CY$X^yS_MXLR*xGstr<1! z{At*5aP6Ss#M&>0u&&Pl*7X{|Plz?&0!SeT2^tDy(N(#R@s)~Vy z@hq^gn+N9HMPTl^1k8e#gIUZZorFPT~KL+TZdn*856gV5>g|Y^@gp zmAQ1x#&zYWweL^ER$&{4EaNv0QZhCTSlr+8#iIQ8e)A{W`pn+%=r(KnEYJUo9f9f5qa}~tEQD@GC17*=D z&2hz$oy(d5TkoI0*aUCtvyS<_*D7Uuk7e%8Zpwo{yDT2==`?@0ug$#mK#N)L!6wt; zLyhLJuL1q2_vlB}Tf(+)0c4Sb6zUr#)E0UN@g4kw|37MfnmB%2krNwdY0e#HnJpe* z*stt!WEM6XOG;ccb-mL57JM*Dq zZz%BBD@!==!W#BGw}xHc0?45+CygA`a9$dr?zY19X^-oZag$7NWW$XocpWQ8XAqShNF-ID$Q%-!a%02Vmvh3VD%ewQgDGe8%TXbBgqYRvR zY&G#utqmM`NQHwnwy^J80C}t>Wl&>j;+&Ywf>3d7(9fU;aGi|sB*nhCDbMY8HCWo= zVzsuF>$s(f<@{#@-Q#dQ&Hwlt`-oGo>=MsCx6Qinlv;f0iB0X5TI>3&)mClSDy{l2 zms7{jmf69HQhPX3N`u4S0w^E{In-D>I435vAoe&fS*OUj3!Xi2zbo49r8uYEQ+H{z zyT$qjp8d9WPV9Xz*=~P7WeQI`riYw;=n!+UnwEN{(mwBcxn0H0GTVP|l~9|-i>SS~ z3hl-&->1RZ0!KJeK!>B>0w|&bz7vvPi}UzP;1Et zFVpp}1=MX%`OJN_&b*`59M4k~tbp@njPNTZ^n~k0j+wXaI~3n3pw&v`+JCs4ZP$4> z%VFe379B2SGT=-m3r>6sUpXkFzM0GhQPB@#;$FqW%suw1?ex7#T7zU}H3Vube(h(p z{)xBswiTOx1nl3)CC!= ztMRkkR_;yPUo2oBD{$kV$>n)n%5o04p23X}Pva!sO<`wCC9z6n5}40q;~33yajgD( zaU8fA=LDDIoZ;NJ0LsWg9rvlP83>#cPJ74*_JQp7J3(4PFOm0Acg3DZsw{dKs=u~8 z&|+(`uid_UZ|2c#50^6;Zi36HeBT?%E}^#*oMZ3BIi*X-aEs)kIgjKc*^LSjoL-qo zr}5j7E^sZ12N%DEuN*Xy!{oPrI<8UPE;8tKfOG^NBaM;g$m_Vf+vNmzpBr(mL3g70QIe6o6xnc>eECr48pRN&_e#OeU2-A3 zvAdzzpgq(LE`JN4vIh4@+^4K?PB3v!xc))`C%0@s?;!yyAk=Xq`3p zBTRnF39;Ur9^h~^*_U%B-iLoF#?$jgq=)~VaQ6tQ5Vu6xAlDp)Kz^l?KmV<=AMdk5 zfZK>P{nc3$34Opf5W+($e8DEq!aajL(ER{JmoJ^lXjhyrpnDLNYY%H z6=$?LHHxx3A>8iom=MOPh(PCyq5keS0)2h%_zQ!jgx)c7US1iBo?a!&0?+3v?w+kG z9$teoo<4BfO9p%M{Bcf11vB|ZJ3CkO9B-q#5sle37;Wwda2FAo~)@&z7E(nW?rs zIaPmST$0(&$atGWAu$do0;1UGeIxkSyuv(h3qt%QU4tWJU4oJooC5DFaRVQzZ~~f? zxq)9~or6Jw7YsMQ1yILvn9OO>k%#+Fq&ILSX^5Lo9%Zg1WjTAu{rn3gJ702EYOc!C z_$=KGQR$}JG4^sGAdz-l7|%NA73*@vJz8*!ALVz~DKbo&6Oky-3eQ($gg;cGhkaCH zg!Rd?!a;%)0XM$|&{&1_3WAOBT!DMSTvC@fi&W+;BZURKNKWBdl3pY+GqF%;58|HnC-dzPK$dYO^obHgD$%ZNh})*Z^=|?6m*9Lv{qH)Dyb2H@HL;?k1bac|U_Z*#ip?ax>L`h>xgio(Ejur; zN_~Z}LjPxva?A{y@n4SxP3@qfLC2;ZI~p7nRd z-0GF)Ilt3h#(rz`ljB>;ul8^3w%WXA?y`K%+h_jD>yYWI;3Gz_5{?_b%0FT7>hVea z7cD3CpZA|Icm~G}p2E>@0cfJ+og3=^kpJlKbku$^{*<3J`ip*1 z?|-&Y?Mv&gYQ4k1s&&K0Z{a0+Af;iLfk?#+1fGKkK@Y+MGXPFqO9-QH3$Y*APi%%x z63XypVm5Mz7>&w`>W`|<&>hp6sWWCWTWic}uEscRzWO+Of!c)YB9#f>ACxB|@$5(X zQUxekCJ#@S%flx`|2I*A9!PEiW*{(!5n4uwCwdTU%t6?XEF)GEn}``^4~*~(h936B z(8lvQ>X_I+%BA=a@Q`rF; zWE-B3`5k?bEy!mR(7p+{iZ$E?tm!XcO@H%m)W4|x<*sABFNOL~9ksZr2&mG;K+S0; zsCmr-wUF7M7CQ&jQs;tN?mSQ{nGb3;h!+b$tzp50dfS3AwJ!@t)W#MLslmbl#1}-r z8Z1D}|K9*tv4*>d94=!`e;f6U3~GNx)cz`xSZ}M~H(+B?(6pTan%r5S=`ja11LuNf z^gPf^UI3ceh@yp{S+xi>pDh~K{P4r5X4~Rn&HlxMnxl*RHDPg|1}y2-fWWXXVKiow>NFKGn3Tl6id*}%%W36c*3OY73L6y= zO>($_I!_wkhcaq^4b=YH5~#x!vCh*I0|TpBV8EOU2Cnlb^nDkN>xV5K(~n;=s-M1W zL@$5&uwL1UA-zXG4(h#K`9-&NRj+Q(>Mq@p)t_}>O@}V5YSV=uKk35qZvk!~hbyT2 zZ{z!r$M>O*+Fu8?zrk(PUh;SzKwAtQOu76hAFGQ z80M_*H!S(7&!A>)ufgl}-TKWNI`q4LZqpzBxfK_8vp%eALO<%G0j&BK;3n47SCPYA z)L6=>{k2j18=S{yqc>+RD*_bF8RM3wb4Dok3kEH?OTL(UtmrfIU)^gOv9`w~VSSfL z=FgqRg&RK`Rc`7q`ghAG!;il;8+LAOG#K3a!4NjTM?dPFG5q}21lGMVfwlh|;1+VY zj=EnOHI_PRe|^;crnojJxHhb%$e69_j3Ha2xnHO@KlEC&mUmgXuK7$6u5Y&p+1O?t zv!&H6b!&@h?v7@Y(w!fTpZwWi^kL6C!}h(ejlTT(!enB{GgJ7z&I~p`F^7$h&Ec2- z4Iqv+y*O%adDK|i_`Ta0*9HaWB^B>pu)j+N9hGMEIqJ{rqFFBKuyb7XiR!$e+1hjS zN6Ub14U~vo@68kTyfe?-_tvcVz-zOIhyFEvceu{v)8R)ZeTN>JjqRrhiNJ$+d&yfd9+R&`2F|<~7CG;HNWzk{K^ZH%0`k{%`j@Zk z@N3x6?%S@`>eIcs#pj;-aql-<8hw6jJm$yN9)kxu8pxJ_#iPLv;23nlaQ}fj)`kB9 z8VXxZ-(pngZQ>Sso~FU^AoU3U&14&i%L#6iyJP)ko{0*Z-x(getRplz10pKdLoBY)wrFTMm|u20MU!Wq`bChq3U) zx(J#}zoM4Vr-aqio2o{SGWX(MQ=h*l-D>RlWamj;2|hDBVuR;vpn$WuCl;SJIaE8Zz>OE>&nB}s`7BQax~aM9K#Xh zO$V$^AIwMSEcy|&~(m+m-G(=|qax%$CH!`N!iTmK{Z( zQe9N)x|+xwwaW04E#+Z#+e<^+c9n!)*i#hpaBo4#`#ptWzqS-bFx8?+wx%eGtsV`g zg<~*A&U40m_~A1%T#>%vyGdT9FQSL|KFsR{J8-|N!+R#zOuQq@VPZ?V$Bf1l|M~Sv zq04IGV^&wjCao`z$x^3=u5*7%+%=7)*q55gao=~P zCNTB1M7AER8x5w7+E&QB9+;1iiPRrIk=|s;()}VCx{O|ovlW}@WW@pQmhvMa$4acF zYYUyHRpfciDaj64QkWU8oR=QEE;}t*EdzTxr6m{dOirrXo0xcFe?nr9W?aH^&G^J` zyAzVx)}$1+0jwVlb{O+uBZbdKj77LKeNU94m)Yq3FC9nSmGkLT^+w!xYjPd0G8C>W zx00$Xagr}B@|<0m@4q-VH*{52cC<=*R-#%;M&{PU^unF-X*GLeQ``1Or(V*GN_nOk zmHKs0OxmBVvFU7MTn1Ae4W@@Nut)Cp#c@Q9qc16V_MV6R2g}FO>6*FJUZ+Y;_4_#+ z>hy)Gt1ZTtRXE5MmbuT$E%9BHRTR83tsru3a$ds5gxvHku{i}hqOz;@L}az>56`-w z5t{h~eBK+LHMA`D+X!kej-zg{Y$*-$Qv+g^uDo zduxvA3Y0gPNfy=GP0p=$Rm`mPS&&*Dv^=RaVoiK;oN9DY>gI^TyzOBH6}y7-oAw6f zpWh#t|7d?;-lsi5`9s@63fRWb0=8i^m@#s{EAoG^2z`j>p&MD8Scmu?$~r!3X`Dd~ z&1fp4 z%Z~5zEjzczr}W_-@6u1ZeM^6D^)F)^0?L@`XfSgjtViVka6Y<+{C_zg{Rfrs563vE zp>--%x38qKj%`%bp~aJT!e~rpo7IGr7N@BR%^pfI$Nd&WHij+_JsQ0_s3Ccse|@%^ zPkouXS6$;akGiuv-0L3haI5*a-J|yRX0KYd9;_P;W-WlVz(;SQh^`^~o<$Er`|$Jc z7Cis$l%AeemGLL&Z@{lw@_H-+$%zO#c*1TPF|k6Py2mZa>} zmZjp}TB_>WdQ8o^_3UQnmPebNnm=uFZXQx~ZDMP|n$f^xb9^ok-Hpc{47u0?p%Tx( zho3*=8GPYcNy@n}mohG{M-JRWiI;VGV=tMGiM(h#KJ6iH_>DqLPzp;v9Zf&Ee+ghAqw~z1#-L?|-zvVQ}`k`fz6HMi>z;4SYmbk*%GU3UzS*19a?04g)IQ{M}zqj-HyW?U<_K( zhfxOyQH*Es85cMx?gkGQzia6n~n zP{a8bbC3lG5Qlv^Lmu!_K<{+&K~IY3+YRLQZWlSf*CNOFhMe{vEO~4{IPzJ4@DQ~8 z7$9u^5#K}lF;U#)Q;vkur^<0hKDUiG{Cq{q@XM?2C zNPve4x`rNz`$Ucdl4LVDjV%5wfP+wm3)q0(i*3l{``{rC;T;eIcnDMUMA*O$IKxGF z!w-aT>|t@7yI3~oPF8_;KbpaLuI=m*_jdMtR9fIbYT)AX;UE&x2N4VplJK$utVNZgsz4h!IVyGVP(^TY$#4)Mm;*O>7|Vx5h69AXHw54yq)<~H^=6~bV=oDvh zkvrzV0&{Q}{U{oYaIPFUM`6^I#(7P}IVi%d%tNgusI?N#V-3wmugn}A&nz6zOdQX2 z9M3e&lN@?NCc}fs!VyWs5lJyW8V?UL8f+c1&1z)4b;!G$ko|WeW9>!u*Fg5yG)4}! zNB-~u;T-Hx5+^&914_A=b}eY)X4)O#9Jt2KbRL3Uo*|vjJcHT;JpDS1=Lh(%&A9vi ze_-qXdkm`Z4_k*_%zD(`4j+T;uLb|0ZHN5fg&vVGPIfo}WN|UwVs55e4H|iv?g?;~ zm+4;N{iAoE_qX0l-e0<(d4KBm^L^K2eBX2#-&bA6`{h6I;Lm@+RB=DQ2IHWHY_kJn zp@HnLwH0+WP}cz2!v_-w!gJ{mB7!1v*Qz&79*)*|Z;JE=X$d0J~xR~>ct zA$REFzTXmF#*LF12XQgu7#?Pv$~$D7$M@T~od1_`y}*ERvtYmRDWQJji$Xt*ZwY@l zej)PJ_@l@tF@e!w8<8_rk#)BtYieRVbl1RzZNmNj z9%N7*+;f}Zp4XZ4kA*+aZ;MF2UlvIM{TA7RKg^4TznfQ!^qDt~`DWfB_SO8n*cbEb z;-Ac(jeT$SUgC|}4~bW1eF&;;d{mnMO z1MEcp)55)`F~@IP2d)7-Z{8oaVFG=&@xotiGRAziE)f4@T_N$&xk>XBK_3k1c;oKeS@w?^`j+d)9319cv~&8f+7?)fUv(z}nQuc$i{c zT4P<>ZA13bq(LVGj-Spp+~1r%_&+-Z34d^m7JKKABKgKXN9vVbiA=9;&BPbBO_QJ7 zbWVM0b4mV*^@C{-tY0bIvHm*krp@4#>$Ys-RXZl#W5=W}+p}?_!PJp;cVH~EfDz`y z3Twh1Yr?>$@u-njdT_PT~kK6gowe(IDl`LScc)Q650)9yJO zoqosR#LQdv-HJEtZ_m17_j1-H+t0Hv*#DZ*4HtCIkxe-Z4|K+fNsk8Gf@9c&jHin~ z&5`}>uqIrTX~<(E{qV;39{3%iw|=HPy}ph@Pkp?`KJp5gaL*%V(k=HC`Ri^u)33Ug zDE7G2%(~>FohHFK<#d9@@TNFsJ|Z` z#84h%fcbF3^?|)=2YlAimw;{bHdvEhgcxx@3bqxx8|W@^-9JF)if@G6CGP}vC^c*y+}`_@wLkr6*kPENgRlwXDUpZ%H#;P?HB!I_}A)H+nMp(O}z9 z`vCHuF^Ztp3<$CuotLi<67T0;Rd38Qasn(ax z7!9@^$8ZQ)%WMjA2Ij+4iGBqvpiiMIs5f#wJ&M^$w_*=-^~9J9bw@f$oDTPr?Fe3ZSX<%qYej`0TU6oC z=2iqSrO{wJQTs5mmLb)|lH*BAS~SzqkmuUs6! zmKF!G1;xQ^&SQVr@j!yy~#T#TB6$%gaLYl}myv{wfM;Tvr&_rJ5gjV?%DhtBtvV zKi1|2vlV$EY;j&Fn>QM4A9D9_-m}AeU=QaZJRG(V28jj*he= z0xc=l;*E*UGWGG^@-?x6ij~n3^U5OQ7MDb%E-wntRn8A9U7Hu$us$c`wx+2mZ4uEGV~7L!|)K_1$iZXKAp?lK%H6p zIa)If_>ZSsir1$&NmnO%POV7rS1gSSn_Cncv$!BSd3kPB*6QrYVwKE@TGjOM_Kj)b zmo}w@J=>fT{&hoYm!v6mkGUrlP z&U$Lk-Ahe5dVCF8=3+G&4ihR;-Q`M>{bm*>hRn@Nh+32rm$*DLHe+>qOrc6@bhT=7 zbcCnQIFN*qdsp;j2`+cF_x`Jie-yOgB?U|a|z^L97lizeTzi)PUJ=I6Qi>^ zv#29ag_^N<@X>r--r8KVF%{W%QYD$LQwr04X5^&?&B;!VSd@_zzdSWDZFO=&o=ReT z<%amUCbihO^P6L0AF4;ke%urj_j_$@0$UlEz?O~%JA~Sn$ldP9-$5euGMbleCga_M zY#}<8uSl(hYpAh!C)E|{@KhF>h?M5rjxWr0nVg&LJv}Qka87!9_`;O5*ky^SDa!H5 zxhgTq<*HFhjcSpJXE%o@-d7Jx{IDq^X;39HnXQUSVM|AY9Y%d?5jc8b`VqoMcjGvz zCyk5Fv)gDpOtAcB(2n#9dZuEL>P@Ety;BG$||JQz0$ae^zo%=z@f-=%ull zNvonWve!nWm#T)P9aRfSJH0tL?at<))OYyvmr7_lQx3~u%SMCgBm3J4;&XwYo+Ec& zPvoHPEDq!zd z@Ar`ZFQwwycOJ55DW1JnPom1&B~)6si3;mAICAUs`7>+H#Zs&7CnQ$6PKm4VnGsbU zG$*_)Vqr*W{IbB3j8*={g@5@J)vfa?I;rYebaR78;hPO!g#&B7i`Yt^BDQoimbQyNW05{}tO#vFB)jjZ>Q z5337M45|&E>t7SM$hSIenOAkeD)*||HEvZMYh9~usJK+VQE{ypSnXEHmb+K7#iPM2 zIk6Uq?uR4$!o!@z7#v6Ts;WcwX%eTL)>)L%_7|nL@1Ud;hq&U}jRd0Gti;0G9LI;W zdQ1vv@ssy$4xQ=M6g$V`cU zMuXWCz465u#Nc}{Fa|An52Cgf`5*Z&3-1P{oSH@nr&mzy8Fh+2t4R@O^?5?hm8FV7@SJVX=3coZTIdB~X_3R@5E7}~sSB!bQ zu2>7YUvVDe+T$hR)Dt9S-xDQk+mk%mx+hQGvZqGDyr*-9dC%RMW|uz9FuV9`y7>jB z0OUub*BcH7W6+8G-H17;ME=Y}{!2y%kAMg9zb1s|;M2+T&I)qBx0zh;?#>yAZKtbG{Xbcz(EwYq6YyUCK4Ik4<5wjJ|8(fQy}~2OUd@7Dp~dJAdB9E zWd6#4%wAh?n7(%4GJfsBZS*FP*YHggzy8})LA|#{Lb~sc2_JsfEu!=Oxrol2?;<*{ z*#G20JbEzlaUIs+Js5bPbnHtK1rOubgC2<6=mU6*y&yhJC$mqB$@ueHGW@cc^uO#T z-LG17_?rP~e>0;)eRibP=S~N|2XJV9kK)w$k;b+EX9@SdpU1iP_Fv@L)Bl`j&(Ci> zd%m+#>A)NeUx#H_gPHI^F>nBWm+&42`ccf`0E~Y~l0m;b9sV_s4h^oP1HZ8s#gIDf z9okKM{%F#!KZoH3jNl=L-vMz%kAgSe&j^K!NJ3sOfRAVZr{N0jfH$Mk2oF^O50wK4 z5epCG4+r1`2Vi=SNbeKDeTc#XOoD?@M32Wp)LMx;D)10$sId+G5PL}F-|v7NM&E@I z+<+CWfCpKMtiKrj7mHXrD5Zt05p<18b;EzwU;@@)zy;JtFNi5T%%LYl`vwVh`QRYN z;k>4x-b}ca`KYxN&O;dP>@RQG)lFk6MdS zYdM_9YSdCe9W|0cUx*aOMiTupV(?YM=#>$`9P#0UkOwk$G}uz)ngz&KE0(~8EJyxD z_TPl;uMYpP#SqxSOL&41^l8L_4CLbiPyrf{(OSVNZ~@$eLwQQr{gQs~_)Y^mhUn)` zM&Eb*hraEMMuRQGF)T*5S_S{G7RRy)Id3a^54OQSY&XEqwZ_kN2SIo?9}CiukMlt} z@>v~d1}8x`xIsgEAJgF8w;Tg|`Z#{<9iqOyjK1z+^ktacj6RP7TRw}}(uMFK$X6RM z7F+*9z0Ihx4?j~EUI_kSuPdI*2Oy6{gB0ZBTn?sD3Tin1XdLGl(mcWWTk{;}pypN1 zUz(3Ne`>zv?9=?l`9xuN2W$n7VI}$>RFHSoF&4X4qn;Y-?1qnk ze>edDaL@^v*bhW-{5h1!F?1-4bMR0R*RMlWTm#xix%#!+xPNM&;r^lB!~I?RA@^79 z*E}D!zw*4-9^`oo5A_fko{|9U(a{p@hhmAOvofr!ZcrmRFa1rogI`CqLjgW`! zX-LnTV^A-Yb3i|i>!*G?_jmn#o<99@o^SefykGTO_&)2O;`^k3iT|Vi1O9jVuLNG{ ze-(JC|4ZPxKI4C;&-k9`Gv3GgjOWoPuvIt)6`+o+xew!U2I9yMM@GGMWBE3d;#ymHADf-0d zv)DtUfid@tnDD>gf^Hczfg8q*|N1Df)yUc#;X#J`gAQVS>SIk9t$_={nlRf-1LlXR z&(eavSUPiku=L}3YZ=b}+9E-)*CIpYg?WMKbMtcXr)CXfpP02tJ~BHu?t$5jad*s~ zkH2C1ar{-Ye#stlCVt7BiC%;U>NaP>Kxiav4UR#5D#kz_=a2C)#`-j01rM-+25ffH zH(M?GXm7&t#@>Ou*WQconO(5Z6WbWkhc+o=@7d%?-mxwjf6KaN!gcE=>1$S}WUg5C zOuS_Ecw)EZyGiG)e#oA&W>Q_~2|8uN#&+5;F(5h;rh+lpj?8&**g;@E%&|7HSM8uZ z_Fr(?LhqgTQ?IKbJ#)3;e(2&RaK|}d%nhfAu~!`v#`QR)Pq<{CFLS}ZeA0Ql`pM_) z+T_mIouAredt1KKws+bI+i&u1c7GGfVqkNmdLE&oGw#ov_Yf}exnSsxFvF7JSGC%qzM+CAbYx4NfI zZFb95XmTr^(db&IbkwzNc7sdzoH~~Wb8DR6&8v3lSFCbl)2iIrluCCdTj{}MM#9vQ zyETw`jiixvFc!}8^ut4u-ulj`Cjl$y7WNMA2{}OL;e$@2Kj>tjn|PbQ@AziF5ZUAC z5jy6bG_Aoab4HzKp;C=U)toAirg@d_rx#SX-&|Pg_G)2?`}eseo@{1`7n@e>%_f6M zBVpSx4_e5ZCKwC5VFxjpzWT`1OYFsPH)Ij@gl(kr;TqHxZpeKi%u28ϟ!u=n_* zfq@h20wUyU{Noj>eA5&wd~#=(`IOBo^**|w*!$$7BCjh;3cOw{&G-7YFyDvG&VvWa z^J7zi+(_6C<;l#|iY?Q-(hH%hHpO$#gw@9(BV9bw%%`6VZCy%~2MD$08iX z>cc$7*M#~{tPBpFS{@WLy)-acsW>2OPN9GC{CvOKMR|U0OLKfLEz9zKvOLrG^O7w8 zp}AQBOff5vDFFGAuwAIFD~8;Qv2Ym&A2*KPVBfp@c=!EM^c*@Jt3oH@c2i5-VXkAb zW&(B5_F~nMZsRK=d?%KM1R}$?lTNo86mmd*6JtsU)DJv{>ZboR{!nBZzB`G1tmnVmu zU6~ksZ&gC@`<00ygNqVF*_@;>HZv)l%@_&WhuTMkuoi^jC57mXKQG-5Ql5htCNlR%ag3f6el>1D~R)y$&K}w%Z?6J$cTzjN{dXMn-Y<|Fe##R zX+rqX6>;HRt75}%D@TXDRgMWCSR5O{=EX%a#keRob0kayV_+h2 zMosvxn1=K%RFi&yqdeV^uO!t{q#)T*GB?RXCM&^jN_u>V0(@e8KLMJXl4CeBTW$y^v0UA#0Vs(wXOROhP5s2j@Rk*}2_qWYIaMzi@* zF-!?4j)Wb=5t;GhD&?hjJ{)ul3FJ}?-bP5`rq*;hI-0qRYO*#_MYaZ&Wa;x3WSR@- zrrS$orn*U|rT9!vNe-Hpm=viPml!`cCLv>CRD9vm@VMF)VR0R+LgKC|2gklr4vG7@ zG&G*g4~u8B!V;L$NZ27hd|L6s4sqcV3Z3u~MCXt~I#M~PF>4alKO{w~q?) z^muY}&4ex%*XX6d`)07&KxriSLPb11KsSIF(m2e_+2JIXjwF96!v7P0jg&8m)!>k2#6?9`hXMbu>W6y&-(EYeRy( zQ+?(Phx$?_yZWZtw)Ge1*wnq4V^jNmwrwq&X}uGwk)T1H9((_13_8-#5r9No zf%V|xBi%B-omC@G_xsS@<={A65R-fl}UsVz#P;d zkrttTdLwpbfSCyD6ehpZ)5-hna`HT%zR$ohP+ zsO9;XvF7K~$C;ilkuvE%E^XX>Nyg~>D;cA6KV^*0vI$0Kn3U1!fAR!tu^ZPyTLz2( z%up%nXCalxp?)w@p*wa>b?T8Jhbwc*?%Eo%xweI@uWOLi4L!2BX-4KZ?Kn+uxpSM` z^5-?W6)9kNJ4MLgc9DqQzZ%Er-nk^Id%IUu_vUv|-5YGo;p^bqKe>i=FuV?rVGUGN zqbmaEpV*GB0Mz$HDz&@9iB!x_#t-Gm=+R;_e7uefo@^)mr<$bqRF8C@nc=CuJ?T96 zAng}H99l18xemU}Me5%kfv>drGo#I){z+>V%v=S`9M)eN>PMiy*YNpY z#+w;8VFex%9r!wy_I;P5-9P5gj-ShDTmN6AKA=XM2ey;iuYI)P*CARzXh151=JeNZ zM_N7PL&`&uwCYbftU(1*{0VRk_TVk}{156&u?AozGGL~nI?)vaGc;WP2+YjE2SmGJ zMz*6XLLIwxY{YII>)BjbiKQ?Et6@Oa<7aQiTOm7OK=#2DXv0h#fhn+pnQ(GO@wio z2GcSNogWKeL6)MHGP*HT@N?B@Doo83yrD7?T^Z7_G~;pMOE6Cohb~g4WT0z?6_W6=5XiA%!o5ky!Q@lDRrD z<^4b(SOYf@2%?dgVFOm@fD#ylYS0MU!5Pp4?t_;!wB|GYS~EcXYZ(0i-$CCntN#g` zHILYAB))}6;Y)EW%1EDop&McCUZi6f2^9%Ax7APXNz7iY-uruqsz$N;# z{to?CeL(~3Khux(1Jnn-^c8#opNCoZPuOf6!+fw5sc#jIMMW8QWFwOCZlD9q(5>JE zd_g!!fW658FhrZm>Cfgm8rs}Uzc+W%;N}bTOZ_JORDVu=>L2NgdOv+qXY>KQ-^}RU z|A5Uw?L|o7E0Oxvq8mXCMnYW`hGqxQf+aEr_P`5CHH7|bkA;0orC&R8VW3K=e`gIW zR3rV^*+JiTo~OQD*XhfyXY_H`2YR>bC%xIl=oRRNsd@=si~^erBeD!>d<_z?8jff9 zjip^GutQs6hctiz(xEl|-s?sK`vU3bz9{;>KZ*MGXVF)ULi(an$?;j^D90y_HjWRP zXF1+$UgLbL`INI)^F8Nt%^#dkVTK;V3Oxk(!9BD=jf5?P0Z@jGSdTQk4P&tvYeEBS z;sDmffju;I@GuP=G^g)c&h+(=FMZYyqmSD09PhQ$Ip1pMalO$g<9el2$K9*b%>7cQ zi|4t{6`rR$k9i;JyyLyA^PTsW&L5uZI*j|O4&%B4dbAnmNZ3-G|5~Kq&2l*BnXm#_ zo7$L95 z92dB2&?)#YgG+)p4ITH}-Z2U1 zjqx~)xiMHue~zrBA4Xf~v(bKfYhpkzO|9vfnH%RLGk@;;rV+gVGELyWX__v0-85h5 zs!5q}k4gQQOD3(N7fsHJb(`E2J8SY%yvyW^Sf|PFF&(B%_=G7FY&T^BZKjNGBy2U# ze-l#ZURfkjjE5oChRFizhn@Lkv4LJ&?xJT_hv|W}Io-B#;<{$-&D&!g!hgvsTBzG9 zN#wj`mgrf_Lh;j8zh2I#{Em|_r1gMQPY=p6}kfWcr)9D@F)_Ll*jBdNAQID$zb-NkT88>V0 zPFENH6RtkOtuDc$&CXF{k2@ufJLZ@nb=0vyrrx1qVx2?dq=V$yM6Fm|AZC zZE~3dlPPs%<4c{GM2Qm<9|==~iP(b#sEaggioc^X;;Z8ndg-D-cik3HkH>mChh3FV zdF#;$Zwu~LFGv0+Pfy`v9syzv?%|SkZn07|t|>BAt~nDcTuP>tJJ(Gub#9k0alWKb zmmd%+o$DVxG21^`F4Hf2 zTDot^j8xwS#T4JpSxLUv=Op;Nnv>}Jb4DVpP@+GZ2&6~CcB1xSKG-2X>UZF#hi)8n z-G_@V2Jlm7s5~`?FQ=mso2e#JgDNBSxk@9<`HI5rg$u&m#BxJ@B(p<;Cu9UiPD~3* zkV_5Bn3fz^G&3=vRw+K<#GJT*D|2J~U(SsQ_^ucWD-;{VCIi`#u-&My$Ah)NP4AI4 z?s(!&0Dn$88_Yv(5mV?G-khq9-bj@(`=~TVm$N9^j5j~ZRwyUZMJzMITQWU7a6)QW z_{8MUIJtz7v}ti6`7>jJt7k9`luZE>N5b}^z9AQ` z2M&7TjLheQyn+0EDvXnwq9;;a>_V!HQ>D`Q-Bc8>!Ix#*nMlLoYWm;5t?#zhr%30xIEptM{F3t-Hdj`JE3c)sdp%F|DOd1Kk*bwWH zgZgcedvUvU*&n$x3{Rqwe~-pVQ&r+TDos+M!la#)mwbpLJIRPABhgAAHNkOAQoOrF zLY%KuY;1^ZbWHS=$e5&QVbM7=L!!%P1w}Q@35@EV7ZCMio`2NW*#XgPdSEo03MP#N z4+^m!k$dflZh2u0ko!(XB6Gwd_a%*^vXt3Wn7W2?)3#Ao+CfTBGvrQ5wct-ou@{L; zb`y_D@);kQ6eJU#7&$p4AyGakK5Irme5sOOeB&J7_;Yi;;~vlRj{7p(C;rcL-*`3^ zOdiRUG8bT7%!^Z@CgwD3s*sqs_&QZp31Q;HQmQ;yE^NI5gdJ>}sXx8zT=+>?iz&u>UIyi~!c^vht((p7qOMtR-u`6WWGD^-@qlEmu6jyMVBRb!h zCnDckAT-Z$OmMEp*nnLB@qRg>GTzy-lRdN3`SLR&vUEFv}_PlcICx z&@|^vCg+mLCXED<1izb*y^;UVhQmL=LDZvuX*P04J{P4H;oC7w7E)~KI*KaYNfBjQ z6k2A;6RsY1;aMCar#|D*{P^tszYI&f?Z+PblbxFGi(b! zPPZ!V+xbm;(7W zE+enw8_47ME^===M6OLoIGmd-xg47ucJI?fYjg(1K z=LF-X2NR4NKTj||_Gi5DQ8o@pjO4iku1AbPM;xxj%wZ1(2ZIcpREzwL9PHK1L2m7` z0P!Z-5xhO+!Mg5b0wPV(3K1xtt%D0 z2d{SU9=Lj+??BHd-UF9^^B%awcr`EnlM9G{ZJ2|)e5^y9eOGQYYA!IT}K*sx6=N5`)S|(!?gE-3GI1+Zzg@{PP-ok(=Kd#vh#61 z?Re5i+n@B%_Q&sN`@;d+{(#Z82aIDRCx_<%YjD_u<-h?XG{ZriAaX@_f)zZ30X)co zn{WV6g=yytIokem9&PPKSH!FJwE5Lm+VpxKslC>wjc<%d^^G;Hf9po;-UX4$`vhA1 zzKH(%&`f`Qyg_R|e4;gP8LfHqPmX8dv!D#u0nR@O^+QnK?Z5T)QC|b~w>?A-|0F_c zUnY_2S4CR)Z4s^QTSaU7)|2vgbz1p-7cKvBfR_Hyr^P?bX<@$;&F>GOc>{4Y_g4YU z9c-dGgO_Rc?^iT?@W(%>!Sz=J2a$^L4?}&A;riX^3PJt-sK4a_x+31f1N`PEC#rA)Ti`nOpgTef9^weRfGxa$7dj*&;UY3YH5|c7aP1!q zd$OP0i4HD)KG@= zSP$p11&(7631XKHK79CK8wKKI0dNtRbnFs>uk8K@!yYgO>(2-EZBT!B{P&>#M$}*V zoM=AAU^XMvB-G@<`3d0sBycX$=>C`lzamdU=vv{!4kFyxK?LC$tOHxXKA;Qm3kjVn zXk7&n|A5IXBsK;4P63&FIy{IXvj1!~VsqdhX2UBxyEKpoNXXDtBxfcw(kQKz1!#}Kme^>$ku-q8f0Z$MNVn7Q1TQ>b!QA9&4 zD(Ux%1{z%1Lcdm?rh%20>F3J()VK0AeOdX9KCT?1_mEoez+3QU6xcLm+}X&O3ueGW z;8<2KMGjX1TY)Apgd4I3ZXke$)<)3a+5{R<$)J9feEO+UMn6<)sZXVezOCz|uj?+- zmv#5(!@5`WX5CkMwf;A~gadjG2lNa)#owO}|M@qVq7qyX^7c~rhc)=SD%OPB8sy;3 z0Bb@`m;R`k(;$3M|0W;$zA2QxZHlF@n^WoY<{bL8xfrgeir%Xqqj%~j=*^aHdcEZi zz1Y%APq%!bM_UHz{uV}e!EJDJ6xbY`{}MTP5aitr`1=;j&9;^BJ{yojchcYvE&92` zn7-|>qt81%=)=xHIG!jto#l=?*fI?s2=lQQ^RaI&{o1#M`u6RoPy6)g zt%eo#YPisIO+R{iAdDU#h@%Gw)9Bv8T#h>jOF3>GtmV9+)y#QK>onIDt*cy@w4QNy zYklHAt2MxNT8nX>0-XmLN5?@%BVkK${%eqRHz9xTo`UT89~U!tXcc`sw1M7h?V?_7 z9eS!`Mh^}<(4E6xbVE0oX}@}AW@!FxvUB43x@ z1O5)ZxBTsTKlxhq7*Dev<8IPpT*vJw_fn_6{7jIY*+@zxkIo{=zRjKM}^&OJB|E%+F{$<%MC zKp%|e&~u|@bk}4fT{YcD7tQtPoP{NISvYZ?wD96SVG+dBhBvrc%o7Bf&C>;so9781 zGcOl8YJOC-9&d2fnfHj-m^~G*GW#r6VLl{MZq5YDutgN$D;)t_i|oHe0AnDA+=;&* znLuAnCQ+~1G`eR#pRQP~qx06gsLMu&I&4j;&CZUq+18!sxUC=GG22jq2Ade6I-4Ys zTI(#)YU?7gO6!`j71k}1<<{rNm0CR*S8VluT#@xJ@j`1hroe^?<=ZfUk+Aimm9OSmx@t3v&e^Y_PKWK(=BPzYPDXUh$(pOd$(g6t(VM^8F;KA5 zAws0WAzrl1K25yDK2M_9zG7UVU87Wi-RTMWcDJN+?cPY|*!PdiwrAp54r~k%210*_ zZGd~&%?lrf<1pfbsB9|Dk0_Wth`Oewna-B+~vYqOsvz$)KW;$J)nC{p+G0o}wgfwR+nd;2M zQeD^>Ao6$ECgkpY-2cwrzf8I5nI#9^w&kK8M?pI6I*r;qmQkap8r6C2qiQcbs_-)7 zEcLSEDfV>bFZA#c%J&Ewlj|NSp6#9>k>!>#KEth0I?c64Hr2IlQnKsi$w{uyCnvgo zlTCDE;}hK3SRgtQrjGiW$h`)f^wA7?3wh_N6DM`M@KT586guv`nCg5tP!+yqtK3hA zO8iVX3w>>P@_n88b9_97vb+PvWO#*%r+dXprh2AIC41({Bzjg(Oz>=(9OrRiO035d zxfqWxlVUvoOo;Jfl0Y1Y{T;Ry^|kOT-3XZzzvge^UjL#i?zKI*sKsX@HDCvss(^J= z7Pym&0}oL_pb=+ofE7=6fFpl~zq?SHpYNCyzYy^x-)PB1-(;zHpB$N3pYlo3KF6m- z`J9^?>GNo6g!iW@5k5oG5x#637z@Pz4mNLt7bNO8CVFU%4sc|yv$)^y@J8kekfs{! zAX65+mWo2QQ+~)n$_X*#$PBUIP7k){O9^rnObYZCNeB!Qj|+^Hj15SXiVnz_%Y+25@jwEM{X2Lu_}{lWA57q2>@f!J@E<(Nx?x4a)Ciz@&Q42<^6*`ObrMglnn@GQa}<&{2ko-!Os#sH%IPuLjQ#q-oy{U z7zE*dJ8Uc!N6w_YsFjozy_wRZ_fcx}VM>ZN;Yx_I=8cVV5{QoU5Q&WN7YmOFl?V-w zl?o0|lL-tfoa7%?C+8d1In^iZw!C-fJ9(e5!HGWMY=Un%lN<@#k7GE3=kAyXL?!CN zEnf>fNL?5*Pn0m_$0$%%+%igw+epdqK?(8N6c=yA853v86B*~gA0F!_92)B@8XOZM z5f~Fa-Y+^u+9x`HqE~dy6wm06sUFccrn*PH1-~YGM6n4T(M)nA>>yDe*5f_Q!v%~% z2hP7S9N9Y>&wbL96uoI9M7a&;+f<~a0^bo$p1I6 z7S9Zi0nWb;4xkhsB0GtbQc`6pK7Ag=Wc)>u8CxkVLxVyxbtx#*gexG!n%6JGNx&z= zQ`jp#K-4`wVytUg!g%MjY-z`|3R#D=wn=trS0~w}z6Sj=cByQxFmExA4O9eCaI z+yz|n{DhtI!bBZ%BTm!ftS;z%rdV!{fADX@4N`C&(6@6z?;S-PFv%QVTYOpjd3OgWs& zY`GlETzKuveE4n4LWHbKW5-yOW{8=WmX0+oZI(1C?U6JteI;pJ(l23B%*0KKm?#+Y zcY1L>Tp1pNP>ex5)&c70qjoAhKm>B2Z#frvRLhbpcCc}(RVIhpO=MrUhivN(kxktZ zvaYuz%X$Y+i+T@kv-&_@llo|Wqxv+#BlX3?2K7xM`t_H`=-0g-qgUH6qF2L&^{ScB zNFL&I@fd|GX<_zt!X@E&Yoya${A z$(7;HM8vP7DR65C9yStua*M)A5UERGLyUzXO*m;(5>^$>N z&R`ujVGUGc{0mV(tp@oQ9>NDX)BzsG1pNRyXNmS+Z50B*oP+(!<7D?rQNOViR13bgp+Tw3^PDb4@1hUR|W zNV7lhAf+!FH1n$-P5)|1)4sWpd|wz%{hmQ`KWb^pkMrOqP5wSWllzAL50%BZ4siZS zs2_~6cfr`3ogvb`fSoa}zyaJMQhq?RXxIbvbJLuGu{7(KEY19_KnlNSll;&^nliM4 zCjC)?@7RntLUzLm975lP35j5b4k7e{2x5m)0eA;~^rY~!Tj0U}p$PR;Q9lgz-B90R zxc)`F6?2tH^n z9;Cs9`NU+9{U;)8%ECX$!avBE;lDWo_y_4w{BtZw1v#J?RDuT30#4!oxdiUeAL$n~ zIN=NRPxwXOaqxZ8@E`x>EB^dDOcKsP78!FA{KF)8fT_#iMAqZ9;2-4SAEp`;le34b zfq#$-2GJl1|2`AI#Y}~ZnOX;$z)AWe*G<3WZqk68%pn$1Fy#!7I=UAM|(FWH<+f>G=7`+;i~v zh42rHSA)%PJNxj@y6__wz=?(y_|V{j5E@tzL;VYr>Bqt>`o6G``WBYc*M;@)JuUD( zr|Bbn(ECMC=+&b4^m6e}dbSwO13X^B=;0_Z`58DTJnx^4{JjLnvJz`T8U8^TYhu+- zfE_|tn!xec(T|lL^lfDzeOVPjpH?N%hgE6xepN2LRW6}7$~E*_xrus}JL%=>9(uO= zF+E!Sj_$4bNw?7tbp0iBt55qh8ZdpY?w``=(>bv21bf{O| zl%8#|r^k4M?7`MRy1OlcZf#4T8{5lK~a z)khuZ$!P~|=*MXVEhAv_;Xam$<8nskR)>$-fw9;%i+=9Hu9Umh(yN_Y>FKTmbbq%Y z-P~hMSNFQm<$b<%abE~^;|;EJ=+8N;kx8e~pVOsTNu8RF937gcI8Ge6!r6M@8E5l> z&zy}1hB%IDGHL+zpbmXGe}^p|hdDs@SHp4az*y{^M8Ebc&?k+#)VqHny z!!!I9hPMUE4BrTp82;odHe@`7hKw6<6&Nzkzr$8x3^w3-^Dh2>_aF>vV+X#&67*PS z0$tNnqVtBzbjoNewHs?vGkS9xO)aUx)RF2;JveGj{kf`5!?-I=V|gk}Q~1hEbNEY5 zO9e_y8w86?PYM;9UKP$aeJPx4+9#M}%J{QQ8E=*;;{n`%hppkoI^g_I{y)S+pAYlU zV_hD)g*PO+ji*zm*>Y+%-$acT`>5Vhmujp`smjWZDy&>N%dC93ORa)5rMF^+>{q8*tK5CHst zhiyRpy~zHDkU5bxuA%;UYrFw&gWdxN8LD%hOBF6_sKj+E6}W0ro|_(JyO~j@n=NO$ zn+tcUs~2yIYaoA;YlL8eOT0+DONMByOObfAOT9#t%c*gZF1N;oySxRz#KK*f2oMZ& zWdeT(EvR@)ljtMn;hqVe_oDu3WY0GIx^Hk9PnB-7sKi5=3Ov;*$8#TLc^;;8PZLV@ zwB}6qbmUI-^x%#6@Z*p32o;R+h!u(QNE3^6FBlu{UMm^q-Z4JJ{rdP|_t#Rv?)~Dy z9&8K{3i4orf9Kz~Jh;txiCdnV7=vy*-0LHIHo2hpz+Hk$y=GE@_X^7KQKJl>J(T9F zP07ATDAC7~Gv3FZJJ!dIC)&q{Kgv5;Fv2@pB+NTSEW|r+Y_NB=WT1EZ_yF&#QvP1A zr2M^qj`jCuqCgl3{hfc`>R>HA#XMZa7@ToJ=5xasc;I=Xw+I#bDo~F9Qp)gGrIi1N zt+$Sk>iWLF*F*^cTHM{;EjS4z#NFN9-QC^YNl1tw2`<51OZ#mp6!%h~P$;x;pEXcE z1b)vS@7D}Tn03!S=iYnvy8E1o(?UU!aNiuu0dX+26B>&keOtS^dx6dYLb^^N@9R?QewDlLSkIcxWv@n zF$sC!MI}`9i%e)95T3AmP*}p*!J!Fv$zT0L`L&9qH|fi%L!I z6_J|rU06z4-;k6|1A9W zR+4;3R#xwz%+l`zG8_8&W$qZ@n|W%WPv-4`J{d3j`(y~;k={hUJK|~YOXZz!;GQ)`%!@Mp%KDL`XrPRB%Cr zY+ylxoPU0Xyl;MSAMgD7?>+N7`g!D^=KA*)siD=*-|NO-r6Q@)_PdRwB@>t>E;(Qrpi zVUEKc?EjWR#-M^RsAcZeKo5qwxbJ4>pKa1`*g6b$+or*0`(jvaUki&4b(nV;z^uap zrkxHj>GXhcXAq2b#K4drX0WpqdOJ6Zbax&kzlwA^pNn)l1d+~mvh8b5@mjQ(eXxb` zZ=?rMQO|ME#QeL3H4yF0|F&~?$epZ#*dq_aeWRhje8XmnXX zy~`15UEWYV6p9Up5}|T9AIgU}V*QalSa;+K)*X6=bzL8@?x28m2fn6@ebB+N*Ua|U zu}vsDFSp5J!OZCwf8Y)MzN4o{lxA7i0CAHCT0qJ7b>J!-})! zSbokC%g*^g;d}&^o=?Y;3l&&=aXS`WJdZ^eo?y|rcUW{*z~VDsvy)@6mHl5&{Svl4 zeJj_2baEZWZmxsa&vhV&xI4sg=HRCRg^LoH_oE!<{5%Y^uTH`&exuKtC~_jD0};@>Y5< zY@?z-9UmKhp!aC5g!o`eWk5Q5gSd3dVAc&X~tbG3v=$ zjCi7sVZR$<$WvPk`okLo|A@wbXSwMAd=vV;IE=n8ZsYrBZ}8ny;VYUp(u1LXF8e)h z7d@bT+?j-JH#x@mv+b)d0E?~wGk&H2aFag3b716Otbuqv2t(eC#K6BNqTk;$(EIHI z^nAAhvhUVI>K`45{sb4_-M8-alv3EL5{S$!N zSzy&A)_<_=Q-7oOZTb(7fPwG8^(LUl(u1ByU#>S9!WxfJoP@;JGO4-(v^L-Yi1M1SEW zKH#MUeGeY|3#tDXp(k_PzVsgYG0*J9+`s>P2m_e!_E+Q2joCzd{@jbuf9Mqp&Im~k z-G#y)-H;}FK3tD7 zWG7w^>B6far}5|DtN3H^13Vu55)X%d!2MxdhccY&P)4vG;~PHekD&iBax^_49?PWJ zg#N?Cb?jPg{>+>njT1hM^~Jwq!|--&9Nvse!|QRmcr~sVFUM8l#kh@lKCT7N#&zK7 zxczuC?j#kLgMHeMTl8&&bE48D)4dqZapPG~w>dt++jN z4{psojvF(t;OZ>aVa$Gki*w%NJZo{zlGEhWHwa@#u+8)zrjz;f9~RS>SwjC|$#VR= zNExpe>fyx#OZ>jT1&K@A##u$_y%F(5PDzVQ=hd53iM)Dj$<9pbk^N0#LJay z@cRl4++S{tTPy5weT64}Ss8>YtDwd@Hb?>lioq(OBQ;gEr2vey&o4Nne-t#4t+v*Al@G7!)}Hj;Y%uMy@k_g4_H7DSppNDut0)eC=U^~WvEVfcZb&}qG8IBKu~ z2Mi6d$H*Exja|@b;*0GjA=qjXjV-2$Xfw?~t64r;%*xShwo%k%-Y(H-zE7gT{ETG1 z`Ax}M^XC%P=I=#S<^n3s1XO&3ut4I|T588b8uOo$=3+ zbcO9T=`!1=(xtZlNS4?L62&%xsK`b@cRt$N;>)@Q@JyHfgCU=F8cX1;u^bLtjKLo3 zdFZrRgDtk2Xt6Uvv%M`E?On0a!58%oA*gkTLXAVBsLC-zqSCQIvcj=Ks?2edbg5&9 zOtIq;*&@fEdK5T5md$s3E1l;kNai{U5;=~7s5^oJ^;KAFuFduK2CTU^XRcw%XaDqL z_Sud=hr=wiJFY~tlNuVG4N>o6g<2OURJ(Yg(lrp}uHm9G*Eoq%*Hp=3*Bq%L*HW1R z*LvA}*DXDAT@T9TxLxX*<@%s!rrR6Y3^zeK!&Q(>cNHYM!!pp1x7p{60jEK@z+2pL zJLcYY-1op~2->-uZ?l^MHoB{z)?E)(9_A?ba6p-dJ4!tLP~;gZD)fw&$oEW=%Ja;U z&hadg$@Z-2k?Gkgm*KguXS&yU`Bcw)y;3~?>XG6l$Rv9TQbe*lA8+-VeBR6${A9&v z-VV$)c+20z9C#~hbDBLTV53Grbah@AOXaebqDGSCA#ralV37cUTI`9gNry zCd@f)83PyQz3$BUNUPWPXz-nY8ovc7=eLcQ&<`r~H$tAjHF5%+kR9NO%zyw9f*p_+?2go6 zUr};!h(uCIv{Yh9vP^tPwrpHTsa#A*gM4&IXYa_66MZ5=ZuAKcdC@C84}F8OV-OKsx=PlqhW^MVlZY z+6M8_&WMfnLQHg^L{xNyWMp)_ba-@zY*=)WTu5}Cd{Fe(-ht6a`UFH@>*F8&te1cE zdpZATLDoM?km-(iEZjBY`M_~_*oPhr_1nW)a~;X~KFN<6gsj*pNQ>jQV#X;VAzl-4 z@kWS=w?b6BBO>EH5E1V$3QGu+3{8lY4o*mu4NNGI3rMKx>6g&n%QxXrZ=Zx;dV9zJ z(aSsjUpeo1LDoA?km=4lUKeigd^p1~cp#8D7xgzY|E!6l{}IcaBcUHslO`ZBc|PKj zS0g%E4Ux(E2v0FbSc)A&Qd|+7;wuVD36Tg$iI(z9O_A|U$&>X?sgm6w)z<&l*m2^mNSpN7gYzkRj6DdBE$E*cUy-80?7VSfp-U8uLDSFsT{zAF`wnkxTy} zZyEyg7Q>%jjBma=yz>pld;Ydq={5_ZrSjB;JL7meXxz~Z=!Bh4#z=0*B_9uLgrq@{oqqN z9-gIh;a;`^u4T$_F4KlnxiK8et>IAa4EqW%*j5C?rXm*B73m_&%3?{2%0?;k%H2|C zl^3NUhWej%jUKN)0da$TBgZV}~m~C`}$wogIZ;XIZLoy5-3SiJs552~n zBE80QBHf0EBHjA8(5VxkQ~NbyPjWxgpKXkP6Wd)y?Se{?TdJ^O>o%xt zJq_h8_o3YOHXlDM|!u)RwYXG*3puS@u zRCkWYhMlvayh{P=cPV1sZgnW`*2mi27Fe^#5v%w3VAbAmtlXQ5<$Fu9Y+oxB_8o)5 z-n&rP^9BmL1SstMn!O41VA%hS)URZ_bE%y`A0)VqH2_-ys}9y+>|_nZKIWiZl2AB2 z5KE4X!{VbevFO-hEIhUb3y!H`-f=z5J#K+HC!8?*gg<7TjK$28xtM;c0n<($BsVer z#9x?xT)^~WU$Z@n9t`!x>#+)(=)q7sjM^U5w%NtqIrp*#j&L7{lUxUJP687z z%45Q%;TU&$62|^O|KW#a7=2|OM*XOTkw2PY#81u`{&OIP{gQ;CSIaQu*A5K&^$G_6 z{0u{WdXFJL3SZI6{;#Kg3EQ6D&UGNn!@byUTWT8|p#EX%pP&bDhII%RS^sba{9F+R z-s*_~cZQ(f-SOyqm;S@OMd)>Z4SL>JM~??4kbUR`nMc8pdXx&uM>UXmv=^d>*Lh=j z#fgIO6^+y{r+yaO9^LuL1G3vs{X>AtF|I>7O%LEaF!={y)GxrG-+*4X=>t5MgzSr6 zkot2lL@!74LDUp7mn!h=zEL)!(Bk)8GwJudl8W(eaTQVmP{v$>9Z(uV#3K8 zcPZs|)g+nlL>D-~g`;0lK>bAO2l5adySNt#+pfa4D_j6({>0c{W80}Me~;P^smmtd z9bq`XMuz|Xp72;c`XF=ox`JpBOX5vpNH(ed3IX&!M9lfYJQMUEK>q=&2>l1pe*pal z(0@QEq5lB-51{{mVp2)!Ni*3>r(+kx)5SwP4L;jL5!{+6o8YpG`ZCAAGNrS{;3BCp2YwxO5WkFK9meS2aAEB4I6LldoMLg+ z338kqBS$BGK^Q^(QGMvc^kw}Gy_gyFA7-*YW-MlnjUxV>qJ^iEP4RHDJ?>5R#I4DJ z_-%3|u1!h6FH_R-6YFz+m|BcWQ>$=cS|iR)--0vK_u$lw6F4#BXB?gN2wk(^;J|Fw zV9a6t1=;-#!We3gXYM_n9>842Vgc(y7EHpM`E$9JYB?UxQNgX*`nWd73O{nK)}?tq zI6prGXXi)b)Pf|OT$q953-fSvQ7Mirs>Pwjn{jY)C-yHngndgcV)xR!*rD(e9SW?! zAY1;Uov)t}CeU}7!rXhFEd3wGW7#0QRT#~6Ra04u|!&b#JXji<6R;6cXR(g*n zB>|gAqawYCZU|GP=)>@mv6RpMSIXkU>Rx!hY9Q{e7>S=(Ovm}v%Wy(*0}d-0;J`X7 z>|O7SUCQ3rp%R1+l}K#g5Ra`JQn6)24%$>p(5hO4&1##`thN)I)Q_W4{VM9!e@CtQ zJJhHPsQQmeHC|o*L*S*A%MPh8zPyU?Tv3W^y7)~9tNYLb({_}Y9ze0_1yP~tJyE{tU&u2RkZZ!L z>^HFF@M8?#Q2&-1^|hI6@K$q7oBo6TaC8{WM7zmyG@EWfqnRG+&CF3_ZjUN+cT}4D zqTC_`WfoB=u}DO*WhRO&3sGoQEy}lQ5#?F$k;t_^Bav-=TQbx7r6|K%K)RKHG%=Q6 z_}C9?xz?Sx+h28<_Zm{)nE4K0J52|p&3r1FESI3(Y8`5g$gtljneK2>D%IhJREqs`$t3#^q9l6(iFRM} zaml9=^F7|)u2A=+IrCoL=60EJj>p=ZCYy<ivZ_4Rr_Z)OZGm~x(P&Aiu+wFfrL`Rsb3-fIWRFUtZj~o|sWVzTO z!_^h(uHHy>4MK`rB$C|{L`iO$5{d4`lJV|!QgQBErDHt~%S3zpA`|8QR65fAAIV5} zK_bFk5Jk8N_>8cMdq1!rt{HF~u>D>3%r%^t_i_r<=+pz%uA@-uHVcLB%aQA$j4Tfw zWO$e$&BF#M9!^N|@I<1gKN38{5$_cziuFpDi18|vjP|aPiu7)mj_~f13HSL?Hq85p zOsMxesZcLLBE(A&bw^y2^H7~WkO`l8bIP;NnK5u<3`i~Oa>_i1qtI(Qa=e!!lYUT| zj~0@BjF9MKiFh9e#QC@(*2fnyz9ES6jS)rqrAkEjLJIw$ zB!6|p`x_wE-yG5Yc8CgaL1ch8A_9XD78r%lz+{P#pj^qIpbF{0pv^J?LHlI=gD=YZ z1wEAY4f9&!#{a2*?dio z>_NY0BqC#GAS`AnLSmH=7^{JRSOfUQn!`8N4&HID@QU+=XIuz8;$ubb@#zw-@x_uZ z@eNW=2|J}76V6CG#NU&4hQwG2(_z{=s%?J z**{;y_{z=1;gdcNo*8rCp0NzB+&9NLQxi^^T!WFxH5gg;u*-6XZI(Z5vLaxWodnD5 zJXmDch|F`gNtouGkTA`@En$-NN@S8Lz$Ak+_HO*bYtk{E1G_@G{(<@%sacl6I+!fx z-o&4Hjp;JKbA^&T{ew^3~+s?Q*Q@=W!xoO)Yez7Ca2?t@vutf8F#LpM(FT-?Pm(3Z>?wC2N`5;?6>IQon19yE zKxg9+Xf;fN2KQ=FYg`7^MkT0hQit*;?nAN30_!$8K&ii~)61sTonnbub%QgVD(Rvx(2ZnU^ZI z%V9NZQdVu9gcaN7V)-@&EZe4trQ6l9WV-}>*OZ~dB`@w_Hczn<|fV;Y!M|DOj#U9ksa*2Wr)t<>*i4a6?y;Cp*u z*8U-wd0+yjADD${2bW;#!8Mq2kl#$&rH_eSR+wCh!7*fC+0zdrL338YJeog-U zANABPVcgRh^N4Nq;izrRcI#4GiP}r3J%ifgs6CY0eSQL@e+4zjEpn#^CqM&u;~Pb9 zWeRV?^ZAubt2rUi-~`ozUICBe4_@^CDC7e(2Kd=g!Vh@HWBHGdd-Al?gJIl#cG82| zNBu79^G_F@r2j+h(U)2O@iV=bYk=scSeM)6`GRf6oqo&w=&6>wn}@Kdyrw4BIZQ1LAZXN(|Yev#dd3yGK&H?{&6a z+~#}a0eMWGlUIacrP;uOT=JI3@=tGm_ApMACzC~FJuxD@DGT8wo$%xrs{e-o=D1+~ z53&Cs_CZ*CfaPQZF(kIcg9MRi^4Wi2EeJ~KJ#ZZe8t6T=;4QZEQ1g6MbxaI zLLEJqCTecMpV&#wE_yC!=zd(K9_ujpTe0u)fZY3!yL|nOAfmpA`9J7Cd^`^N51{{m zwM3Jc5(gsogMz6Y&GsfyH-nma)GT2etEjh;R#gj!M+b+=e(Ig%kh#L4!~HP$S2y@S zuk#arC0D;ekY)Wv53V?xDcVOyAE%A>)@5VIiB})#P2=5@JK!g59A|pPd)*6ZxDv| z_+%-HeUK^iAEplBny0aNIcWwSPgsmQ;}mgioEEN(H^qgC_Bb=i1E(hYk{!}uT zEaYWpfixb^?TPEN2IAtpNjR}+5e_X;!afCU>{@1u&gFL4w!#%#R{EfQWiVP-MWSU@ zJepUhp=ot48d)3Du%;gMYuiz`c0Xzq&!I~34k{F1p^V;3DJdbv-S}wh&Za&uX-}8% zHn5aA*HQ^wSS*iY%g18>s=3&?b``cOs-c};Ov^e;G_U6xjP)L9Q1(Z?awzIlqEV}o zgld&cRBb3k<%TMht8PY_+Afr+okEe?4HT%oKt6q!T$1w*A1|p3*nVo?V(xi`x4pCE z_-YyKR~n8^29-l)(FM447NO10upqMeE&?OYUU zm!m+Z5qY|u$kjcDY`tHRsrv^qbpJ(~E`0}*s`CYI`$R8{w}F*>=FLm{SvBT-HY^_g?*)MD9myya!YSw2FH#al#M(Di0*PImvK{q-H~GDha{^IBwEEFf%Q4@)_I7tsYHxz z3!-iJAnq5c7LdO)0dwDPvU(TueRmLpJX zJp=hROOb7(gbW)Eq}m!F+13JywsuIcbwRwHH)8FA5Mv*SX!|5YIpiSHp&Sv8O$c+` zEedryCkk=ACkl3ajUWd;0r&>-G2G`kxXc(FqwXGSKJ&KavrfKNTeJ4Sb|~`gry`4c z=A=2SL9&Ay5*_sr?`VoxM{7hoIw8u@1CdSvh;Rx=xN`!+Trv^jQi@>LMp2OKPEnxS z8HoVbI}-k`ui)n*z*h|YpwGNp#<9TZ&Ur)ToK_qM_RKXL83PCA8hl^iI1pLR6OraJ zA4x7N5$~dc7#D3sxfmnD#gcmV2z7Nsh?_5h-9i!M9*02pbOd-5!{1}0$j`G=!q@YZ zgpcQK32%>=A}<2s69ytH!!EoAB%=g)@eUa)u4vFq_5bLoF zQ6B3M;h~8z4+Dg-MkvVB4uPI72=MZTzgIB)ykp?&odzGDLU{YsiM)KbOL+R8knr%m zDdFz(r^wA)fU6j=f*H-fnag{1I(OppZqo`hMU8M^*PS0&vA_Kg+oLb>?0Fk7nu#)$O_m*wZSUtkjOIXSCM7pGgw5t zhefym^KRT^Kb*Fq|G>C!^=ChXvL7PY50UgA!stJ6D(oNG7d}yA;Tb&(ZZS*Y60;Ug zF{*Hk)rEbm3G8C6VH@iVn^^8c85a!8xL8=kXTm(b45sm$VUlnV#tA>eIPMP^#r{h; zTaNj{FUCA48G~KEye3e;K8jvg9LHijbBW-uK|i$^dZ|s&P1^^Zv@6g~`5oHH{}2J%Nng0g z^YNeu{RhUrDVo=W1olG;^R6`JU8&5yiDQ}wHW>q8nK=&TnX_P;wFJgl+y^sj0}Qft zpr36F-E3>W$`<#EE2G?@C%1H_Qmx` z&D5{S2^Tp!2NGz($z=HY;%&*^udG)6;ukHcn)x5>LYT+yPgz$W1{5Mj+ zqJU$Onu*j4sbv0LO%JG+`DZ@U~V|?2^OlbWZ6IhEfVe=QZbNn??zlLou zrgj>2BkP#|Ze;%1$Y=0P+>5h?H4tsA!Pv?ixRZ4NJ9}Z=uHhK7Ya&MPo{dqvxd+AW zH5ktQQikm@#Lzvq7`)dDgZ4&Y;JzI6-`9YC`@7J0-);0`Jxc%G0tW2*LQ4kEg(8lF zDta)~j%9oOsq08x6Y8o{cXbE%g4jv@J*)#bz;zghxc=gp4CIavK#$|2A$wvfWKJxA z^of;_I-v^5lZFtT@>{IO2Q~aP)XZW*6Z&J9l{CLA)MwKjPu~dO_m00e;V#1dE=eTLfpB$=q~4k zRE3iP-UNSkU=dmnZw5TMuCkF=ck$ns`1+8%`X4nl^k5kCl+E;D+Ib#O+k6MdAhlQT zXZ*Xk2InX}fRprJ&VpVJwSOQ#ldI&q6emEv>8T9mgmfIeiJ9~WmU42qo_>e{Cnt{d z3V7l_<_Y(NgX;;8;|V|L@y-8H!f}?)xJNKlo@}=z|3r)JUfsnS7-~;C1q?d}e8FYUTBiZTDf@ZP?#B{M8!9d>-39k=jEpGxonw<2n)R-X#wRyY-_7^Ng=A$V>7U z`CFEsJ&-=gSWcE_lT}2A*pWbzL<$Hy;6MKV{RahR=gNd@KwwQ=i9d-TV*eqHxp)r! zk3wejTn7TK17Xhn7VY#Sb}&EhqBn4w-oVd1vJT z`cgMoBXpg8autcxN~2Z|+f_upN}Q)Bbe6F=h21zII)-BsKjE(QgO#_dklg1Fm4_poi!f@;9~(WevtK z)?WNa8(+l;z3Dsfv=)YmSd+qdjOvHKhmXJ?Lnh(gfO)vye-(ZhsD|@{`CTAGt#Evp zGmZ@R!l4m?I5;vK`$xrM@2F(#8J&q;qYJQOOa(f}G+_JKt=Ky50NTf$N9%a51DW^= znaJ!KRrlXq>eh8)u(F-Rv8vp7R`) zbGiP5l#{Y<2t)qUpTbKGxlOK3l*A9?zQ?I4<8WxkJnWsd3OnYgVcR?dw9m6Z>wE`n zUf_-}Z|su!1|YDpt1mUf_g=~0v_{DNYIrzqt5r2J(9 z@<^^2g-?83=11L^)2Tm~w;Nt!&&}hlZIpAp!(Xa=?|Q9!H0dNeQ7M&oi5)UU8c z?FuJUuk=I}Yhx-_hN65`G|E;bp@g+D#jA@^xTX#TYqub8Z5MJCFC$y=5i%9uBAxzA z+JB_-Rg5s6aiIPK>Rwf#|FDvC+!f5Z=*8?+Ddx&J{)Ly-~0}5c$gC$Wx9(j!GJ`Rq~NZFD7F{3({2gAyxG}lGW}bQSCJnRC%TS zk2t=H5&JMNd0D?j{Yz`6SU;X#&s_RDD^aGZf?_ot`Z^}aQ|I~z zey3Enx;rv8e379Mf;7!&q-rK3MJoqMTIER8ZbE|gF2w1aL9EUlMC-gll#YN%ZC>@e zA@*UOF#gxpFy~ZZuA{+uKG~wm`PBxleNZ1ozh@?jwG@!At%MvM4P@yUAVbHTo{k+- zbX}0F>xD$UKqTlzAYMNKvHF>aF(^T_K?9-;I}u@Y65&QS5o-7+LbyMTm|%l1h>zg` zWAG!}eq58ejxOil%A3P!~ytTzg<-jW{Dm z#2UFH#>fv*#-WHbjzxq?I>Jp05o%h85VP$FHamtO^WP9)_8k6Zy!)Bbf$zrKIh+RY z_Igo`9t^d28L9Yd`IbKST{IkwT;oZ|Fqw~3)0IdxRYtt27Gli|5p8CHNHaS` zn7JU#+#8|h!3eR4MzBQ+f-Lh9XjKD$t1a-eJ_29sU*TiTtG(5Gc(ER*J1^#Q3Z%rb zsKs1|`km%{=54_km@(htvE-WeM~3-$BwNg(x3ml~mg^8@seuS9eS}$=BE-rX!B$QP zvi3xvbpZTrBH?G71Yg@+_}Ep!+pY~>_J`nU{|h|q|A4#Qzi_h^zT%H1919!=CmDl% z)ZJ#yabUwaCSMD9EE$%)k!(Gh{?bf(OG^-8yB49gstC5#K_K^1_P4c!pPd7I?cL#H z?+0&(aCkW+z|%1s9!?c-=L1SN=Yw!{{s}Hlzr)$_UpP4maH1ddna9g{O;G3gXiP86 ziv3{E+5^W=9!w68CB;?_arPtVF-@hnv=G6Ls}Sg@41Y&0_&OTG+sPbWPImBgc7=zt z58Pcs;pP$tSJzCqxRt@lZ8IF*_rt;c2iUtkfxYWLu%j2$od-N0E^+)FW(;<4y4B*$ zesJZpJ{QJ-#}e-(jY#Jq2z8l+K-YQjb6pM}w{`Gx(|{-U?{RlCg`2w#T-}}F;_d|} zk6<`@#=yZd9rj)&u=8qyt@mEocwd6G*JD_Dz9W1(L{I25cX&RWTGp?)2wz9k{d zxkxnM2M~`S`VS$)=<7^|9laPE?wey3rUc6{HTo&KFbg+5y3e> zaf+b-5JmqXdMwOiX2LXfF^pqZ(N9r^L97<^V-2AjX91l!2WZE6LX%#MMm)a@G@ffP z;;XSCfom`lPC+^TE|lY5LzzB|a?BS_S<-)CpSK4v_oiNXEc+pW{zD?i0bf0c4KYdV z3B%;!&`X{Sos_xIN>PAj>RPCC<5-9jK<7Kqb`%%BikcpXLuGHbOBi18dUDv6_ns zR;3@u%Cy^9nfe!2rqFw!7xS4T9Q&QVyf(A_HF3!VOYhZ4y4ZMnycPRpw)AWg{k6 z9>k=I>zG{r5>v_qOey_BC)2>q)URdROQ@Yqt+*1#pp<)omNEY>XZ~BseJH9~15nEv zgnAiF-#8Fc8^&Nt!!%54oR5i(%Q1mA&xhWFEo3b#psTM<; z_F!<+FBsDJ0z(^^MQr>+dkkYh{R+0ZfV#;QJRhq#4r=H_QB$v;`+ziX9YPapFg7y> zZDS3@R!I!r)(?ZWkHCQKlhA+r9Q5m0itjrV(Yr$v@*Sq=+2Mp9oq>?;OoDW0DI`0$ zLZb5wBs(5KYWrJAZxjAU6ZLBt?-I5*oo$V(`{coh>tGC;=|Q#718V1Yp>CsoC-rx7 zFNi%{hp}H0q66Raq#8yhaN;qG@aEH{Nb_Bbm=g~6LmX&_GYK1Wcpv$NKYL2v@o(RM zq>g=E&RAzt`?Ck*KwTrYS9L4(JE%|HiPRl_0Qmk8>ky8D9tt@rMT4~u8OjOJINp3` z@S?kj4*}NFAK`Ia6FR;~^$)q6fo2ab}xM z*k)C>dBtw(^Ux=Bu?B?ywHI|IPE-3lxkP>-KN5bAF|H2a4R|E|kjb13%;N-k6(=T| z^a`wqFMSc7;P*=DS8O3{#QmQ*dE&8K~H8|t==YvkN z{^cySE|M$cD!EQ>klW-gxi3L4q$jI!hV^47p)N_E$Woob+KkDA(>qkw#J|w@B zKM2Fh#F$z;XVC_f9XH`!#a$2^dH{Re_&QgPht;qoNUaEJaW4#{P%DdC`Sd?Zs8>x-qJes?3`-}& z(?yLl4Aph|5l@)2|NTGz<0e1x2KntjKKT#K|IwRpFNlx+1J~qWIZ>guK6S0A=|Vjp zY6Vj(itS3E_mN7iEGiVxVc>okti!lRPv{QoFm5pxH|R86;{f`Fzxk2>Q|x_QBS|Hblx?%8mOHil>3xK9NJLXBky>fg%4TZ|IaJHJ z8mW$2o2k`Ftu9950=PUK7-}U_D~-NK4qdP!Ixm$R35_^{t<>6wgZ$|}Mtu)k zv5OJi!B6ib9o-N_Vto!zDf$pS7!R3&_#hg^KAy(DT*MYA(%aEwM2*=Z8)~_*UEXY0 zAhp7AL==NVq9hy?WnjNV9`;F;Vvl4Uc1gBkhtzI#NS(wsnd{in;}5j<{1=<$S%cB* z6HUDZh!LczFH62B{dgST_ry!NL3k)V2G=BK;RmT@tf5iHi5|K*(vy2}_Oio)KCamN zy$^Qx4Z^N|;n>kX79IVQv3)=mwhk;p`@kx+4QfWq;2mfld>osGT*by=Pf<7gJ!(b> zs3uk2;MPO*e#j7BF36LCyqxv!fvY_R;C!EnI6iOzYiQPB?=W@j9ASWtk(Sst$^lzO zyP<8g4_d|qVe^;>G>whJrg5og7?*>M<4aLDz8*CbwxW7s7b+)SM%koCD4F~gMNv(PzV8MaPZkJc&LXr5|}P1CH= zFwGJ5)7?=!-4``8f>AXi3Y9YxP(CvQWwQ!VGP@c@vs+L&XD{;SoLl7s9e<;YsP2^k7Ik*078smpF5dD%-O(tk-H z@!b$_J^gGv{Re7a;w|quZ@&i?bDlk)^Q(FN(6neg>X*z#wZd|gFH=UzGA$G?H$wgj zOXRMwM-FRavQ~N{b7dgXS4ALgRRU60XCj$iOyZh)B&^+z__fCnt4KFd@j0UCyF`)* zF>5~|jOJ}-I&VYNJ-w1sfi=`$P5l*|XDiTuSV8|`^$ZlRSwe4TEqxs|WGU()L&+3r zO4dkGazyevcO}Y^tBP9Z-gLy z3k2%h!Qa3Yeg;19H4K4|VJy6jGT>!g0uSR&a5vclHz|(pS+^tpMYOMtqYeP8On8V4& z4vw~Nu($Pton1I=?UG<)pAT#MT39)Bz|!FqEFA8@-2N|^*$XhU`@$`rho?CP_p!}e zso!YNe9M9TK$3VY5j>V4n||=M8v`%9>2SAS1Xuf&aJF9$M|%x8IOxO9!4$R*Hn4GY zfwiL#teiq&$=V!q=UkY%)WFnb8%$hIz}V$BjGSK))*U!~;Tq4umF<`{Wo!_cD&2A*4>=Xs29{e#C#=&~k9$L$M0vLB9c9CWhHo2g&x!SjJBK?V_f zF#g2dO$JW`Keh+BC3K#|`K|eqpdVzY-2{eUvpe?ilU7gIE#8H5d{6E>PCQtO(~ijQ=>o7<9Vud|-QPs8<}xdY~Bkaj{%~&sUddu0J5U z(UQ=J835JT(NKw-Mn7df*2OP_QoJJ8#;amYye?M7n_^{x9hN6}U|B*imL?`(Nn$=0 zB{pDT;yx@${1FQh{=kCx_gE0e+53O&=a}2#&GUi!mDDOo;5nJhv5>;C$X9z}Ow@^D zN)N0`8$v&2JeH@=fI|90EKOg5#To0cC_@7aGYl|4!xHl{Trek-`%thkGc$8AJ+lte zvUXu==4DLHc#3K1|6*DipQQfBPL6>V#=4GeE~QRR26N9Wo)5%5o4FU!A?tFO|L5@; zc)mR577WGg!tt0{I0G{Z7h+oB3QR3rk10i(m{eqpiA8o8U*v_c#o-uJoQ_dk8#JQ0 z1H+3iU_{Xqj41pEBMStK%>TsJK*oUjm27t*wbH2*Rlpb&axG91^KYVB%o+@)tMkjb z4xp0jAgbjtx_Ssk)r`Z4n&}u`vj9VDR$xf&dJL-7#=u%L^sjYBKbm~s)g_`=T?u;D zwL`A%6y$2}L%!y3$X5%HulmGh#=ka|G05duq+UV^uSsPb2h=mCo@N!F!Pl?`qK^A; zY-A3)2^ieW9K5v$=f}qLr*e@7r4=63Qw?kWcjB_$Xdv6VBcn7Z?jKs^FQDI^&|Q9JP-KeAZj~P+n8-uVVjroaHsF*J|taShjNrPASbCM z))i}BAwLs-1k*TvKDkZq@>uTo=Old;C#lmu6>LA_aXdHW!&47>A~EzR82DEde6@pg zeI$i_9m<%yv8&A3ZdJBhfrmMru^!1b_c_fqIOqBKmx);S8o5OnN+!Qw>OubC`{(3O z@>dV~AcN_HOr#I8kO_=3F(aOY$06hr4m@GgM?U!v;+$Vx10l|Lr;vqYEz$b_*m?`_ zuCDC+d*Rv^51Ir*gb;UkcXxM(B!mPAfndQUxVzJ0EtFDdq3)ed+nMUry`8E1^#8de z{Mvb+_vy3x6~4J^?{oG&_w04|ITz#rV+}|HmtH_HtkehEo1KEitu@8me zC>p`t=n|L6S05&4evX{|UzqkK8SgLX1U>a1x3!N?`E1}%{egS_GHjU+oIwDH0U4ke zKAy>{Cb;IowFsW&=mKlt;@Pa|4_}M@FmwQ}6L6g)U%$%npCD&vPf)Vw_xbfZ9O<_9 z@fPpj?8P54xDEZm$o-945OZJ$yzw;x-efrQI6^so)x*^S7tduS@27Gco;5IRq+7P4 z53nBw`olZu4{wvAV;D%jfarS&sY+cp5y_iyrNRd|Ly+2w?PwH7&;) z5C!359Gsc(6~R@7PqXo9EWrpoibE!9ag-2fk0> zFRbv@6(55*LM+jhhF|$`m7%fJ(#nl!EUmhR9apjA3R>O$Xmyu}Q7R5l=Z?a4j;fUF ztX6ssi;NXR-=dmPgVBc?jJ|*3|7_@o4d6%Y_z*kZ)D--kN#A)fEQP_D0AHpa=u@c6 zeR&>fKb{BDzfl+Zx9Z%0g}P_p65Ty;l}-=Zs8fS>=)~YdIyU4U9U1zd4h?-y2Zn#3 zeIvfto>A0ajAkte*x8F)HVooa9>UPVY5fj(tzSRA&}X!s=r>Id4|382L;ZE>j!0b? znXI#;vvv2FBAp&vp_60lbbMTsj*V;65t9xb8ox{j$FI@;30t&p!anVpcuKn_-LD;! zp3=4{?`n(bx9T?gLmSO|uz^BJIglVjfnx*QHd9DsjE!MtimD)S4Ub`)twaa3jc1&NS?bExo)pCP2TkTZ0)lqG< zzNqyS{jIfmQ>$(Nt(CSYtYF1o@W-7DZ}7i2l_B4n%Z?3~Bwp|5^98dpI%8o$9gVXN z+4yPS44yw>o1mRDGql|0OoWc4FdsFRm zvePbSPi=GI*$=KU+US<7^=?^O>t3KW?qyo#QKOX}v$evrRm(jWX{lG2mY~Hf_TH;S zKBu*i!N1-2Ma}p7RBe90sMQbs0Uc&;FTSJQKfw1F;eN!MX#khqbH4CRC+O7qcT<^}Ci zYw!up4Y{J0kmuAK`jMJKe^Mj*%k014KW1EB9l0dJe?5pA<4|&)5aySG%=58jtM3?X zWdGnbL5^A(%yS<@LbN0#MvFs|(R4D^9-6NOq3nGaR;{+M*_s>Prk3zd&52l}rpWDT zj6AB@QJ2&Z{j}<%KTuut52}r#_5jrM;tMOz11@ck!}&l2Z4k};671*wHf-4tdTIwAwr9vPweQSoYvO4Hn^9LHOQ=!c`CyCHLS%re4p}1`px;y>OpOCD%!#-aVQ5T>^95xB*%e zZ=(5$)76^fh^FMF#-t$4N{&>0GHVZ#(^Qk3qpFl*Ri;*{B6YUP)8?xzeVIzrH>x;e zpNcZisxb3W6=c4x{LHVFmqAsg0qSxLyh}ekLkv6^Pwo%*@l58pS=?*QAm_xM#mRj% zKV`I9Qq9qp>@+LQU3KZ~y_nA42kAV2GCf)48CfdJC{$@?g-Ws-RFu`G!t5m~$X>7f zoW07+y<0iCHH%UM5Wo$Dxp57C?`XO)aMlBmMK54PI-BAm7Bj<+4<{~Rj^x` zg=dse_^{FoUsqb;zm-}*RVM4SZsVT;OpEFJOK_cm|4|dh|&KID^!MnE_9jKP`0W7Gb4X~$z?~BSuDzDO1 zIh8hOInK(c@=;oKuu@qom0X>uq?!yR)D$Sbrb2OOF)^%@imvNYWZhOp)E`rL{R0ZC ze@UTrpDVQXSMY~IYkKe`v44d&xZBWyYRGqH5etpP0%&6zTwjm=P{)1Ox*_N~6VP*} zE4jg5iL*QuKPy0Sv%(cKJ5JHFS%WbY<2 z$gTBffc}6MW8fNbc%C*t20tYW>hK&G1_HF;CACU)m)LU*N1x?421dznUW-lkETPiw^HM|D^C`x?3N2aVdm zUBq4(`!gSge|Hz>101V1(gvH@4~2BB7^H5d4}i~B?tyQk24Dv@2s`^}%)=)JDfs>t3=L*G+^g}k{Qz+YXCa(P$Ed?O!8(-F@ZSypIch*Iq5)ij`wDmvTm#n! za?u?D#xvkfMH`vPgv^}_0PXQu5-0+VOi-2q8uoDx^!O9tE%4bb7#hs_1DuO=m$4S9 z1m2WW=z#FL@pGodnh5Xk2hacv&WFLH;BoLIplS6?U-XnAOajKBPfTHQY=cJO%H#+; zUQb4+C`B`%Pu^S&4sg@vKENNp^~Npy|7kE`w6i-uv%a4i6#O0fFgyk?=9ApP1^xzJ z2LAxBgEzt3{m>?cqfJaeo0yI^;X;iDc6^yiokj&W5a%Nmr~OM_!9Q zunDDOyZ(hH^cLCm>*U<8kdePc(7wPo&!U<5^kJhk}Kh^oQ@MUtOLU=0B1M1*uf~OT-fI1B9 zxPcuHp+8*HX1I1UgdCwa&Z0#;h!*h-t@18fgi*2cJ4z8X7=OY4+F(d~gWw)V|4paQ zobb<&BT$2(B)GEisR*75cWZHH?KTwQRjhie-Vu^*n3MCc`Y=5Z`~ok;l- zh2~dUi5d*BuNQx{;cMFDL-3}Car#fD|7Osh?)Vo(8^*ww3RezMUyO&9_*4g16I`vZ zEu;!&DeB8A_%^XZX%}2a;kiJh-qfzXuWCpC&$Mj-XVJhvw1xFSn|pEFhA-iMm*MV} zK86I`7t2S%ZH}+@@OtZ9-%y?DAE%Q8({y}Lu8s~a*5SdGIyj_W2ZlCj-_SPg9oC`U z!&_N`-3|f_Qo>& zj$=6HGIVwD0Npo`H6X(*b!w!Gj*a%$p|KI#KQ2*wSs%1}e4ciWFVXf1RoXVOURx$M zt9#OXZJN}n4U<=C-IUE*J7u3%vv2N7vj^2>{-Tym{X|Qq{i4okj4&2|;=k~J2IuRO z84jm$>9HX1WOzHzrSSCFJE)~G)j^)Ay~oT`JI#Z&ZCbQ8PfOM&i%e~pp09P&OSRUr zTB|K*X_Zxrx~$r@+Wc`ild(H?JqT>>QfP`itokw8Pp;TV}XuqphFT%?#ISyEv`1Pt^*0)?nBd zYNhwYWa1Id1Q&$?ZEex}!g!!`#MK zwEbI_T>c!n6uUDG;PQWp*TXK%E0_*!wHvLCj#IVP*TD%r&4q8l;_hvQv9MCME^J?&YRP}yutJd!u)u6-Fputr4 z;xjI_ui*Ei9<+fUIcES<8-M25KJfG1de`A-JyW#YYlb?}VmhdcY4;7&JYV)c@MZl2 zwJ~%2^408Dszz#KX8W`EA?uXt(PHWXH>xIRpQ?lJQB}wdRfN2u@{s?iEQBgRP#WBW z54ps?fX@%ZZ%ltqhtmgP zI86;9IjRdOQf&x(ABN6SRal!U!QoxtqLS#vDvDw4LF`Ty#GO=L+%@IKzpNbg4bF-CP1$i& zMaK5vEn@Hq+TcnAT3j5teRqbCjuON6$c0a#2mRkE&vVRS^@V zvKZDL#HOh@HdjS)B`S=sQ9*pO@)J6go486jiQAQ(bX=LqSCx_cH>D?kuJoi|0qYJD zd+-`lE@SF<8NM@2{|+(b*_p;1JB9nr*fKws+&gxt8sa9RD_N>C-cjWVo+?QQP*Fm- z3KC+KpO~V&q-^CT6)7jVO4%um%1UWhM(RqXr)^bQ+A*c3Ur}=Ui{MiwQJ0gH#wyv= z9{k;lX(%y3X|_(meIScDZZ`dp$(nyYv)zRnk_M?d*+k{Z)6kRbRFvYTf>b}{rG_ep z`kZX)bF$Jhl#yPb^z;g)v6pdb=6t1Ou253e7A0mKQ9|}*#brMaK2dDe&w!odGkfqn zad$nMSitY&aPP|_=geoG&1(mrn=`oIn%)n6X|#$m%+Ql&C^yqZ*_rJBm=&b-tVpF% zpOc!Mu9R$^L6K9Y#GHC1sCzuAw}h1Qe^(Kipb|~MBYz;UDI-V@HG8! zl{PTyfsVqxyO>xg>8V4RUqHTvE#*1n-Z^(DH+Pb?52fS>C@DW&i3M?r zFGx{bL5^YzOB7vLtEi&6iY#8D@ZyaMD>yN-Hu^3R+BJ$xOwUxGJucJt#{<6kQsv$g(6w zlw~TstVp5d)e0$}qo9h#3anT!|H^&xtGpoJ$|vPh$#XC&evnT&&%o%#&3Iyg7&wXl z`{CVIODxor^8up{g)NCyh)t;#=noa66k9n(QB~H8sCHC%wWmU>0~Asdp`e<01=gf1 zpf+EAwH5NIZIpN2LV4D$kw^U=xz(SQYyIQkJ-OC>FIRLJS9BNy*Jy)twD~dk_rklm zkyx0676qD`j5-wVd(PtCdjq*|{ZNH8j91XCX$qJ<6CK4>zO#Mg(-AxsChj>oetCAK>2w@1{0l zVF7s`m^H7b4ke?N+;=YZ2Xp($Ywk#Sv`$7(v6O3@y`0rN|KbvkHzr-v+-1ZwJ%NSQB5lIaqj z1+uh4la@AX!qO!&S-M$cmmSlXW!E)&>6;p}oLpeJD0jgV0S4!WMGSZG$v$+ZgrVHbwomS*g!9M;g)_gn|S>CvU3+ zbmBIQ-?kH+1y6D@dY?h)TktFRua3aKX9Zuc<9y)wi#C&g!%?~w9STHjqXu9*&*RuZ z4azQRF!oS`v5y@5KwpHsVPG6E1J;1t@{kWOF1`$UhYCRhpfe9~@I%MIb$<66_yYVu z1N?kzU*H}1$aXSX23HH_R>GCFn>gHy4hEmY0r(HWe+2(IOrMiXLQjJ;{lRd=P3*Ya z6kx|a*m2euL~_EV0|NeB3!oFvalmu;f@cA3arWQf+glpUR(xCu=X`!ryPtCr&Nw&& z;BfM%v=$OT7OOe5M`vyarzv9_ZCzjuHd>ED&C&Gx$#zj=g~Xbf3Wou7Zca4e$tf z6g&=|piQ2nO`aZ)9x{!Ikv$p(cDxkJO`3cr?X#G$Ekiq?q5pw1|GWwQhEDRDHXPvk zc$o1B?vfMK;ovV*FFBvbfBYQ_?|=v4FnA44gYRiTbCS&VXd!>+^;JMO>&=1OAREoi zO8Vx*ndl(c@mV|t>_zAkbHFMz5uE;#2K|cm`05cYgS!Rp+jTIZ`0R?$7FVc&xyBwW zhQGIYpW{7Ec#H3KUf%}q0=o8Ag7z`5pMfv?ptsN_-%X+(#1|{lB0;{7i25 z6FKG&WOd(@RenbXXzWXBtho4!iUea%Ql65FCefpPe9Y@dz}TDPUC^UH7_|?E{$SK& zOaU{27YGMQ`Y$;qt8n3|fu|9UR`iI4#X)E{V;TUv}!k79+duJ{-TZ!DbY9H9W8D)7fxi_(l| zZ8ZNvx?u@CUGS_!>DY`eu@jl>0LMQ?&VGs9^--*N7473Qy7Xs0{XtleAn&LMe<}&RdVFcdhc?1;Av{aq=|YoOhep7D z80Zf-$-W;ZJHLkhaFt(NLCLsG58YmaauL=Gz4(*A$Chu=V!ogrElXy$ZjgJ&U3OV9^aa{8{v`>hNiM%I6l;p9Fvg2x!rUgOgj z=n=o75wITyINFQ9+Q0}-JNL)mF|_AYe6;5@^yUbm@WoTnkq%EDJSFf{!BY=U6FjYW z+z!_gxK_cn5uTlJ9Ol2Xw<=Ti`|Brw%#27=*W9kF<2>Ndde%iw8&XEG~Es9=Ff-4g( zCm+8`;HreHUOW0UX?x!`ZSA{AoBMUCyZ=UQ?7vIv2OQVhftR&<&{JAD_+70S`kj^! zXAKBg+Kb=e{2aVFn4IrUhG1X}{}=lXfP0Ki^qHz7{T*~*khk^@3DK^hG1@+ieJF)dY)ypZqyvD8NEQO#w^v!F>AGg^+C(V9nlh#d$ri)2`w7`juuY* zP75ZXwDw{iuSOeuz>xML!~7!*uUCMff1Kh{cx2dM?Z0yZ^)uGmHqJ$x$NOpH1ooh0 zebAaoDOxotOI?!-v|@6pmQAVFQd6EmY1*nqW{cEezEbVxn>Byx0nM96#8^C{7K=AE zXZqJ_vSi;!(AbNw2GIr+xZGHf_u6nfVg~%y%rE)8XEN(ACXLZ1)2Ui#Zm-qTJk@0p zsAbb5wZt-Bi!E7$VU?|gR)tz%U9S1owQ93zRxA7Gw#-kj4j{aahS<7eG zXo;1uJvRkI`eFFICfw$m)la%xq*bEoQD)~d#3r>b2~ zs?zO2Rk*#Ra(DJVaQ{uEXfe0(KJES-zF&7G2E52Kmrtj^k2+O(&QS#u;Bv23DrFK@;&V*J zzE@P}`+^F5KUKc(F8~!B_x6x|#Fw-1Q zTQz#Qs=?b=bv_}g@rhEk56@okNmGSyj>>$CRq9u*61142fOZuIbSXb@i}HewC@1)` zvV)&fR`AEl4E`B#M zcT`n?rz!#iR2CSn(!f|12PLa0C{qQ&1Nq@4?^c_ZzgoQHygbihhVC_Xq2EzZ6^Mg)qkr8mzkD@v087Ku@w$ zX{eiuL)rTvG(`DftUU-zP)>NdvcvO~6;Y;)$aY;d{UeRqApKJVWeVr47!)cQl#vF^$|momfmI-@}$B?5c|Cqmr1BDu|t|+*m7R z#qs>hI1i=8`6(qnRLRumBqby%F(FIwiA9P_s#a`Lvtp7LDJo^1B2)G%JoT)?sLKgW zWi@8%cM7E@=N72Td4m4FA7Af=?=ajuvx$WqYH_mY2R_eAq3)k)SbpMAbfpQ(NU~6B z@=PTsyDBl+NAW4aic4YrLu!JeQ(6Cz#`=e}N`&C?FR3T!$^i8RQ#j1JRVmDkoR_-A znW<5mW8mR%Vu3z5f&cs9-df6Bzl>PmwYdl#i0=xpDX3Mv*RNh7P@+z7qkK!)56>pbo@hQ2KJS^vuH|1RXwVaFo z3%Gk%NNp=|a27w0;`eU2yQ|1|YS5xUV3-j%R$&tjpJo`YKu1&yiLA5>0RD*0hGBn%ZzxW(}{%y#6aq zt@|DDWC8S;i}b+>+I>Gf+u&W-LM)I1%xfbS=Ar|UmZUb*22Io-@b6BIgJs`1Rx_JS zHKWN&Hq8#QYW9$2bAT-7L}=QaM48XYmf4&#O=+2}NiB;sp=EZPhCnIETIj61Me-sgk^m+a``X~UtywQ zE6g=yg^dQUaMr*TJ{qthRQ*;YsLzTV1{ngXi-t9dTe?W@&B8 ze=ojn<=E?1^3@v7NpPzUWi@dC0)PwOS*@i8WF0jS>&Zbk4#4OUU;>y5Y!MK+_--Pv zqb5b0800q5$eU(?MPL)ZIR>tQe{xa&9Q?rKfc4h2{Z7p6hGW$R#>S20-~7A~B!iG{ z`d~AAac*H9!d7ZPaA)97aS|CA3`T>AbmlZL6S#pu5DOSw_Rto439!B7y2fIUJ@@hZ z=fQ{I8!nJMy4c+s8|MN%={x8HxV&}|1Mr#dg?~T(AB6h|z$G0cw>^#> z$43F$;{@$-g7!G+&Y%|r5&(nCsRlHY#Q;Z6(I2NC0dIlN08g*L?{0XP!`TX7)gHz~ z_@Xh{=Kys;hv0{E)NyJ+PQrT{oCW8>gH#K=m%(%39q{QbscIFx3*fBfpV<;#v4jXTzANrBk<2g>jD;&5fAXmNC?y=KY2jIf;~e~$M|gI?vlO0{@N9r*8$A07 z<5OhZmpPSqCJ8NK>>7I>mb2iVUKonlUv2o2Hu*PulYGqjq<5*oc!Re1JGD74koU72 z7)Q#4rvNRY4326}&jwD@X7q?Qm=>ZBEamiF1_7+Eh85PaqCGZfK4(ZLT&t+y+5*phEV_r*c?hNG?^@jVb1mxki#qzV z;;k3o!S^9}sXzQf7={OP8R*CS4qfUv`spE=(W16+y5XC@w)6?troM67&@V;n`m+vX zK)zNFEY-?^Rq7hlpyh*Fv~=)7bq-mf#X~o$W7uA`4?m;%cU;%JJ6=)is4vtq`ZvuP z%|4W%85nKwCfrXkq&>)`?;KI$5h2v_pU|gf-naoq$_$8VJvN`<#*L_^tPWf<)$&WSw>%E!R@LjtTb8P!zYIK~adZ+2Cp)RJ% z#a$JyekyYfQHg7mirf-Z=$5X0YGd-;OO=BblTBSrmRF}Tyw)q7^+~BdXO-gfn385@PCrD_4`5{&a~o`#6Ttk+r=QAwMxiN9K~J($o}YuV{oIx5 z@27PC5T*G?DK#Kb$$=S43Mx=yP^IF78x}^#f7CQHY`uk;pK{on5D>w1qzQ`sj#T+3W++Y;AkrEqF)F9R#4P` z!5<2WoP@ZYk@4O!F7896ZD*p^C`DQxEJJVfW znSS!f3YB|StlY9w<$@ODoKq>soMzeQE|y*HM%m^a(u}QMMTjwx{vd&PL?jzG_uDEItna?Q7uOM$bT z3cTc45G4D;DA^Sz$+jp*Gm6S&T|8Ts#T}Ynyk65v4#>RZKADv~FSFuLWnTQVrWSD* z5gq1Y60v}v`$~v~D(<`2_SB)Y)uTV~zPN^a?|dJHO@7$qQp$5k%f`yCY>H-SOrf}gwL>26?rZbFNiL+;&z7RCF#M*3kE_4nB1Sx4?w*GJRqcn(ON33`gT zOzW*RrQT7K8ay?zAyDHRqGZyLqOr5`HF{Q!M$T^29kW+z*z8>zGW)Cs&wfIKXT7hX zv%c5R2JWiXqd(C92jSl_n{i<-u`r)_TtMy%%I0xCw2^!Ay+3fqE-Ns(r9XPgof^ z(a}Tv+k=4qIPx+0M%(ajBYaEYngdrkTcTxC(Gh*NKsbXp?pkc=o{v<4k zOqz?)D_WR1t^qXSGaTTVrvPiu@wFY^x&y=jys<}#L3nNPbs~NaImfXt0)x}wyAE!G zN5SLZN$@my20RB|051(h3mJ_*VU9L|9q$C8hon-gQHdtfPBKBKe{>9R(2pM1B21cn zm}?_nC7xg$IL$ajQ%%C(A%?$yN*^pE*uuZzg$m+1#`s_)6>z9p~v8qx4SWP$89Nmh3S{o_e8-FGqTTk`impU3hB z?~VG4=Rgm)!EeCOA54I;CdV20!x;r%3S7Bp0;OcEHE_&^qlMa!`Q)jKaCsTI`YL!f zz_X3Kc`sVQade3bWSBS53tl0Q{ha3e?H1=V9NEx59tTD(#{b&TkI$p{r8z#@!Rdux zq434yPX;^%aFoMQ3&(6YTF^e`Qy;O2OmZ1Ks|bgU=n^~NFxH}+=Kj606UH-SoA04} zd{6i?UjSFY?e!>swc&T#;s^TUKbi=qHGaA9m;Q8hB;B4w+Zp?T7Q#_ZCSME3Y&crb z2j=1JB6ya;vl^amICc@nN6VPzc#ycbLf(HLdH)6Ses*ai z+H&A225&Rxj zeve|q8jxQ68$M%rF^0P9{Tbr>^$g3$(4`LQF8rK~f7Wn1!RJj22g4NwR|0;q4um*e zh8;_=qmzDFM4T?<=nMKR*1W!})!J{XTKXN3DDU?T<}%x7Aij=<*G#MX%+QK{E?PRkN1X#hv}jP277k8O`;b)4 zADXSUp#^FkR;HHWwQ9bjS&etLYxc-4HH_M9TRr%zNDrId@2?BhP>0>G|`~zXRwG zy?Biw{n060k}T{&9QkJ?Q}s%^p)&6#AarpZp4HN{i)rU9xm z3sa3*jH=9&RAHW>@~L?$om!@nX>}^HXi>rR#mb+)PPvvm`_bxdWm(@;hV`4uu=!f) zHfRsM_}h5t;pto&?U>^`(FQI|Z+KnDdG^p(5)z6wl{?c;XQsdQ=nvKd(0InEVuq%qbaDZYJnS=1Oz7QHqD7k~}<= z;OVD0&k)6WMJvWDNm1Teiu5T~gm10F{CNJP-!cXJZ&nZm!T|wPy#>AqK9hgIuYfBv zxIw>Pr2S98xi5lgZZy+x(8>E&?5Pi;&X)Gc_8zD-?=edDnW6+AOU3!vDaO}Tk-pxF z@C#D7U!=nP6BHVdp^(5r1qD_sFsMcT!As;9vPnK6hvXf4S)QRUfKS0M@(Q8$02nn% z=kfO#oW?E5&GE!S0(0y*=D66xvIZ4~kaGs~Rbs$hiVK{m=)h@;2((pLkds1!JQW=5 zub|*C1%|}QKQv8#q51L+tCV+mv%De}%Ohfg+#?UlHS#{WL_G&SkxL}cz(j*_r8eg( zeSQxdM`Ac96UqHk$TOJwwI`zkVM`@;6dGox;4o_ihC9eV++BVV zzVe9-k#}T_ywGAiqH^UPT`t#{M!Cdx$SHQ69OL%OKJH%G#XSo?20zK3wNVD{Cl1a; zk{ZCjBaK`qlYB3We2>>zd@jeP9R7{%ffW`#5G`r6{9`7|H)guLV|fNftcyJ2yyYGj zB-i*zxx^>RDIr^q>{(+^vTT>sF59HFvPs@6>*NcvOnw@CD66C&WsL@7oyh*?w82S^ zzaPGB*~DWW@mN4S<`WA!Xi?p?+_JBz7 zmmQ5Y6D?*2Ym{u#*aITHUenXr10sVxAToB#Ec2X9GuZol->kyYiP;W8dmbVh8O>#)9H*0 z@b9P~7U~$AW>Npp$k+rbc%Mx=5LZh-V3Qj**Q}Q^eQLTn$&P{t0kf6rFThXgQ-%IOzwd>AYXf<2 zGuMb#w9$EtO>OkUT+Yck-2dl$?%0Zo(CzsW*9xdDQ_ zi6C#HgPU-!3CEh+acUi}2f$_U0w>+a;CqOE(_#3HJz=`xTfKnm)+m=6L>f<0j9tTH~=p|ptm#7@cbaP({SxL*1i(##=#5VX->Qk zz&8N>0e<88&>P@fxsMWcTOe$sB*eM=|jWEUUTa6SN|4kddvV-oNM z_Q0I)DAU!T_3&>5n+)^?*s%#aHetu6sbB_;>IwotETApA3G{9nubV&aKEiLVaSUQ& z<7ePIZO6AwaNMp#VXNg^d(tMr5eS^RiGj^%P`EH?8@xLJO?pcU*~9Bz1B1YwTyW@@ zeP#?c*s-73*dI*5CxIf~Hvt-XKYx7S07bGe6K!!GC(rZ8 z=dbe}ZE@}+ZGv+dd~@NefREu!;rQkTr{w|GLBTnYQ|cDqX-qdPvp1fU* zCNdYDg1~zBAUC!-=zBNRzMty?AM^1y4u5@4Fb?6b3H}Z+{55zV00!?(@C0}U+~VQ& zCGZM(4ZI272JeCoz$b&bi9UgKF~r9={_M$`f)+xb{IHO{LgL5*hBb|4e@O9rY+*bC3Hg>JC1oEvQI_BM0rHh~rnFN7NxlHNr6$X=(wv`eJyN z!?Tv|GS;FPyFMJ#!vxYbK3&DK`)Ms>?+s%u%I$S1Mm@$~ZTN+L`G)@Z96jhm>JHvv zebO7W#VcqrFOv5^L%#han$Y9yLGcK7+&~9;n0~pA4)P$`>{aZzLI!$?T=!lw@C)?G zIkNV<`B&pfF-Gqkr#FwmYOFyy(u>`|)}&M>3w^Ydp=zSbMS^ zYy(C;hGD}`;B)XV@FL(21OAPN)2P9)=Lnu~1;G;qM(JN9A6 z9_(P(2Y6P&(M`U-m;aw3TCb5izeG=cPW1lDzxLE%tmM^b|1W@XN*Sk{ar!+#Tii=q z+(TQOBJV$j7IT<2N(adM_YoU=uwxf?>>xh25vM%WfC#9Di~TsUV?FV)7L{W)ebR-; z%W3(gc+-i(vWTea;A~n5cY80shSNBm4gQCLG2|IT(kTtX&#|!GEj+!Mgca-$qN$C5q-Sh%|F{H4*)d*kIsO z_+AE&0Asi{hT>!30J`aJbg6Bcz}aSjM|Sw-&QbmFD-^$?@hbtYG`Ls?f*lKJk9ovL zD-10#H)ByFaWRXQYoI^s7+hjhGC(q8y>BiJ9q}gUFoVEk*o5N zB`OL|O-?G?gk5&WzoLu@UxEKBeIgpf1lApJX}J#P`Kerv zExG(#F}Iw~+>+1Bu&EteTJ9XG#*yRIFnXG5$J(lz^+A;;-sm}jDjgrLk_oXYnvkr5 ziJ8iuSfJcV70RAGTUk@)E5o!)X{KA1Vs=c)<_{>*{BKH_`neLO{RUY3FqQqA$8jFO zcMq49BlgTK9q0pISK2bi$DUSm@~w&J4--bKV$u|qPPS6<6bBWWx+&kxSGi`v$}x{r zmU+A~rlu-=TCP&3l`6%ePD#^Sm0-C{@mAf6wLYw9>&uF=d0vq=pDJ?3FN&B!Wg=S4 zV^in{F0ZE@xjp2{+|r$Uy-b0YIWzUdo+fOgq#TWBxC*CDP@cs!lHvvjGya8`MNzVAGZ&|Pk_A- zT)B492IpNkAK~2-Kr99^w*-s0+_zv)Ez_V9{yoc~zf$c-D#>A@;vFp%<2XZ6PELw& z@=&<5pF*8^{(}q8e{kjb4`?v~?&b1#ZKyl=vmYHTT>ebQ)?c-LWyaWzq-o0-DgtQ6vAuOK&9 z1-kpl-y>Llo>B5ei}CTwmX~*_Jbh-#-M3wCerx37w@1$Y=j7=B1b82O52!t0znUxb z`x!sZNjP^z5eqR){bNZzct1OWIVRud22$@$n?!jHR+!fq1$$3cfcJFydE3g@$64M! zp7Qbykf&dS+|gp({8|4HP%P)bdN~EnmqYL>*#+;?%#eFDBjhpgF5pRJ!K{4%mpmAk z@%KO^xqlpU>_oDU#g=SrNetn>dq95$_>Yipzyx^*OqFM#jobqrG>_+LGcs5+BBNy!l`5;~d`*w8mIWI}OpRSGv)FAijXSN$aW^$7?rlwp{RU7$%=)Nv zw82r@e>Z&H8N_2Y@t8|I<`4^+-0$PN4D5;F-vi>wIbsINA!d~9VkgQrcB*XRY-Anh zD9bocO^*-IG_)9V)+m|rJf|s%>;aK9M-!5kYCJVD4VBMOw&`wXWOi#< z<}nS;d{9F&U*+{n4WmkCI4hP;;q3vu-wxOM5+byMv!s%<1XPrBC$@~}z@`vv^T0-1 z>@m+7pouvnG(Km7#^so6Y_7FN=Q?U+uBYzI4bt$u7!A!!)8PCf4a{#)|NMpO%k~m| z3J&5F*X#Tj!6)jI_k;SfbIdVXelPr6;96Tvgw}JG%;GF*pemz|=%}G*s!)co&57Sw zmU2fJj3^z9(lQ#AWs(M!T4+G&O!Y5wRo^mByD}QLjP5Msz~vmEd>&X0_5w1UvS+}D zoC4p1Uv(JYcH!?P_*OL%p>sG(=8_4upj0+j$V${T=+! z1egIU;0QcHFo*+L0OO5=)h`D-z&+gRejL2ZY49Jwt`zvTonvo+y9;v{wG*KWQ7RWO zTJkx%jq03M;sCoWfC(51)Y=ye1)~9%;?`+^%S|i8Y%6Wi%IVJ&g*2CrY@N+(C)f;5 zLU0qjg)I9O_))v@Zwtp<3%603(!P|DVhL@)`=rI3p^MlVp@WPROyvD=(1-7t!hr!` z7#K?fP6ifWCU6HqARZ7vo%Ba1PIayahe`V%#MxKD=iqzoz|Tz_cQsA5WF=;Sa*)x5 zG6g(WP?Z3t@cu5o@4E`_HDGNo)?vq5>{y2#>#$?pbYKs>KscZ+)|CPp+K^t>?g96K z=Q++tfC_B9{N39Rx+z)u~w5`){| zhp*2rIB|jLK9?InlQI?W!BJif92>y&><)(F@u(%!xGWH(rvjL+B!D77Tb$zjIDL@I z*8Sjl-oM|oOD$Z@aFxQ9x|0}y&v`ei!#I^j?uXytH298#Q{WzO7Mug;!3A&;+zT#& z%e2Xrai|yAam^WoFojG*#i$~sAmDG{e8vYr+8N3Fk!Se)o3OoyF&RZ$tOT4}e{t4a$ z?|={bqKu5DLWKDEigDxLaa4*FlWKFae@hU2N5_75hP>yms)$h~G8Rk&4!{Q(J9H$X zh-8r|7Lu)2poY|;78p;Aor_|y0KH=|xzq}B8KcI(8?|C5`om%F>YpR4zD`#2H?qsm zFs)~2Qe%e>qcY4LGU%SCrM%8b`zj~T-^qTB{Qo%`@o75eNq+el9vHkgfWdtk+zbA{HvGzN5Z{yW z{f7#bFXTja=nGdkJPD{1=^Q_ge7c0=S91J1{Aq+^E*uL8g-*Jvi_loFhdF_-aRObX zL+>YOE|K+Jq_vEdDMnq6p@tZ}U{q)P)rKGFm#@h9KH{$TJKXhtjY@=Ujuptvl6rxTu)aBM_9*o{hY93Eqb(5E@IjP*l5qaO6E zO4-lrzFr&t3-`C+6Tm1=?+t@y z?lJ5*iXDd;$Pbb;@25}p@c&&{vjfYGXZmiTg}dS11UB}?c0~+9iPKz zoYDrrF$5UH-brvo1Nh5P_%{VkD~{v{mnS>{@Pxq=15YA8rQuIDTm^8I!BqoSBV4Qu z;oG@bH;1Db`?=00ivAya?*Sgg(d>Jda>k&XbIv&nx_*P&Nyf9{q+igW>)w;-*@i4&pG!i4uCg zE3$eZi#tQmV=DHTLcdIc`vg9XL!&X&ZAR0Ik$B2*`ePU#-iYNJi29+(9|Hb`EH(6< zv|&iL=W=9+d~0Ui99vDLiuc4J*9&gJ$csi+GTbwf)frhOw5diLFVDtSl$l^Y!wKy*WtUi(c);~!|s ztnNfMZp$3o+=_N~!k#|J3`HJq#57NqQ5IP;(lTF$Tb9T$tFF>$-BX6z@Ky?&K~iTs zOa}2J+gkhS(%)gR)Htq}K2H0kSI3jmv(t0Z!}&Am?m|77JA@R@(e4L4uz^1|2=M;$(Eb_G%Khg_X~1eJ4%{V0LDW5io|gRJi;^Gm6I-6$4Ic5PAHwK| zSf(h6tOVe4zk=<8cx({O+!9@y0!asaY@|O=Zt3gmE4}?frKf+4boWn^s(=is4CIc8 zz!Iqls+6)|?uei|Qyel(3PUGILD(GW9KKR=BX&qm#4*W=d{QzaKak8Q)(BFW$)qxK z8ve&<`>o0NVmkehNk3%Z2dT_0(PwlVS!g74>=1M5734%!$y2(81WK3C2q_PZm(s9Q zDGAG#qVNJKjHrP&7_gXEA5WW~&qjM(Lp9=Bam`&CY|<^#jtY~Tlt%hVaa5=jL`6$}bfV-% zXGm^Lo@B?CN*2|bOjgRI$Ja|*!WcRRtHdWCk+_scB_`!PiB0*AeO4nR zQw@Ok5!$>bmuf%(zEH&c8q8pSJi0WY%fK||`Pig9&Ps}?&g92=Np5_QWXDG!FHSNN zQYAexM^clDB!%isa&m7;OsSKE)KLdtMCoB--u7) zU*c21Y6NU>02wL`Sl1nYti~UEG0$c@lFx(Drw`}4Vw-&EQt~Y&u5$;8&i9bW`~Zo_ z50|ilSP3mik>J7{2`nm>0ID;7#r?#mq*1&}CW>e2d~q*ZCvIg2#HH*(=~VVA+pom6 z>`!qkWiCVe_R-#LJ@Cgqp|pAcV^bY5F_^KDZ4*=RLAB)G=v2b_OmvAw*Ptp(@u_kW zuPP7msPY%L?&0FvJx-jvr%A`|dE(fkTndQ#0s}rC=rY1_PYm&vXCR@yFN=Q~{SU(!je*~BYmV)iz7R27|K2mppkNP%0{0!QiEXh^r+{qfO<=@sCOhem{z{rzOelkoatp@vWwgFHxZ9Z7#BH~3c`UGu;&;PNCpwGyai_L+khJY zdMrne(?a1)JDjg%$e_fPUGG9_kTko6|rE4%Zvt*jwn}Eil@AD>wr!KC$Tw<5OzY z%9PpgRx4BLKsS(s`^0SE?kwavC^=ieHgF5r4fcS&tw1*$C&T;cn*%OP2JnxA_~^k* zPyq&VQNR|5Fz6we9J&Xb2Or29Vs=su%NCGP;Z4IBY? zfTIAjl8oqTB9>Y3cd~8o5OTT=P#Lw(y)I$c5SQFgeE@#Ea4r(Im-iX#U5Hmh^0{$Qxq=Q20 zAC+W9eaK4G-6;*MSQtsRHjYewGI{!RvZguYYYV9pEF+s-Lk6>1KB5CZq|@GKeZo6b z2;L;4eGM)z!|_F+R%M(4>dp|Y4oV+;;6?o-oQg#fGIPn4%aGH9yu1cZb?|S5|45oS zj@rs(vgzsMN^_902pP+faWke-Z%R2p+2{oF)O%9iA&39oC|BK`qGaC<)cP2`4!@FT z^1Nwe1|c(+`bP$BDWn};$P}x|iTcC84*resA4!*tMaCp#Orx9T;1-K;j}_#u>yfjQ zdco~<pPQdJMA)fopsiw=L#C*K1q`n1srnZDFoBCui-va*p;Ogk#! z-wXcz;a`VqHjtx_K*m^FJBhl-G~~=d&SGS&!j(58XCHEo^Z(<7^9AyEt`A66PY6{| zt22}izk_c8qn5F>cAixkkCW@3!4{{;?d~J*zni@OE^0#DzYhOi@b5>M4kB!ZlA{l& zCNTy%6Ol6wIdiEK+(bQMEi!IFlfw*ar?AY+^wQUOjDcL%fR@cr?f)9Q51t3g zdjTl<_X0H(F`5%u=Csp^%Yhd%Ly#APtYq4hg`5Jom(r%Lw5bPe>4U5R$f`$H6IK{Y zec(o9%|)B#)FL(!LATOd_fs2q9#8%ZFZzuhY<@>JR}JHEb3fZV!2!UKMVt9uMXag0 zxX?adWQHLx7Fj8@Fbg^Pw5t?Jyd4xhCZPw< z185yyKZpnzhz_+t*?=pUoUq_hU`?)Lg?w{V4*d5@J?ejuMZK{i&3STQKu)PUr%)4 zDlQM$a{1{*&gsUb5v+6O^3G5{-;(*QSYT5}Ei>Ga=v@M&q<?c;<}Fqf%k}u9Vw-E9Lfo0_wIlTn=2h zEcxIAT=q8sb<^-7Z>BjMpNuYJtauxySx0Fw^OAatAQ@yCDYaG!)N|6L#wJ_(+7w7{ z+cK%P>n1(ydrOtWKFLfy-l(@A{r}cKZ(e4H&a& z^KmY12Sb<|gfp*;V5-A*KA)!rFsJb1n(D&5!ofxc*tFQD>6|NOh?%GpI-TF(h`%o$L7%K&yGo`cFGRgJcD%m`>CDZ2-$?$nwGJL-U z1|v>+;)}@J&vav398;xu=2UEFbJ?GaKBIz}S74X^o@P?*<{;hOJfzayU%GgNNx5gN zlzJvhiC3l+d3Tlq?^4P4sggY3zLM)#FWLU1B{N`}qz5jM)WA)W62vojg3m}o@EejC z@{J@0{{_hXY3mW>?@7QHQ<&GK(hqEBa;cqwKEu(aE{wTFAir_>+DN&ti|lO-x zl!Q1>iH-A@n0TJZ5Faa%3EUBpm@Q${VnUKDB{;dS1g7vrhSc%mmo`^?($Nli1SmSitc zDQ*&x;wxdP!4i@hCBbQl5}1}D{^5%go`@eFGxdcA2r;w>T@>h2w*Xqu=(1TQrZBr$+DD>&YxpHii2VD}nglE}E za8@S?%<>ZdY@SM)9WFjO@#2+}CZ4%@;+|J7u6fns+__FV=8qPq{F&lVutMw#c8E>k z39&9bC)NcQ#kSxFV36D3xvPw^sfw|wnwaRr*w~kP7~4iZ*Y?C8(WeZ3vawA(x`Y&3 zP{VN$&jL5`DD)M#!VqyOj27pjWa&_pBaX!-Vqel-Y)fjzx^#qCmQEA%vZZ2H#st3X zPVfY{zz><qoSyu=v`EunjlBOwOG1v@_01|+1?{MyG z@Qdt+-!}NKL*BAFVqysOum);!U?}_j>ZwJcle#-6v4-D1v5z3+8~e-~f0K1HTHs zU;yU53ABGBs;)xbVn}DBvbsBGI0LK%d+6l*G4e~`67?DC56E9nn^z!n(T&8!bYcSXhN;+q zZY)NhG|okiC;tTkD2tzO!{O7 zeKLbSnGp@}iy7ErI)Of&{+N#9(-(rxTnLWhbWeki==kqt1MOAoP?mCf?h=}Y`wm$| zJo33<0evu^Iv8+c--=^Q>L?oNVZkx%u>^Z8L60TDAP!{F!R4SI7y(dx@y+nSFBd(+ zB<(%GUFh&v>rfW2KrNuwq4Wi%%NP?u7;xvaHRqUIqA-BXRjgzGDs0&A2)yWE^jMz& zIs^P-eLcr9_<9(w-^Kp@;6?ac(yT)ny@oyjT>uXTlSshNKw`5VIU50HBxz}`gV3$U z$?wi9TiB9#Fd4%pyB!#OJ*bg{fMie%h|_%|!CU~dee}oPC&By1btrR@F@6K(LQYiy zO6(GiJg+Ut-v(bLPx0RaZUvaqSPMDK_I7Xt+;Ii4HYO-{*)mZkUex5^o-`7N3P51r zi!JV34)%fvz;ni@)J{fL1MRCuUjBA!Ft<>LqQdUsG?WEStn2luiL z2)TAGa#g$T0!prusbri5Mj33M0_VVa@EmvnybN9k?|=)I)IsQ%Ph$8{yMUy+hI+_2 z>LH6+`cK2ZR?mcJT?1*UKX`LI93;zM@IgEg6l zI~jT~^^Z6*kxXP2k)2h-zZaYaus&fh{D;AR6u;GvquG;@F&!Cm$y*mAVT`FjG?q=IMT)-XA*US8OT{cw!MN{ z!6t^Y{aEE5>IBbl_-7$!G5^+*H}4>4zk^of9t~i!Idp3x)TPr`q2l{}B6o07Dhuv_q~t zG6RqoL7U=fQyOw|;9iKFGUQa!o}Sbq`XZ~ANF73Nj6l`|WX(XvVxs3}j_k%t>KU_7 z<3S%m^)qOGKh%rJQp4KaKn;ZqQ}ku?C26Ht&`u}X=S2-72zimnNdV10X>GGM?LnaL&89$4&ZD* zzR?%I=#3S5zYSX6h8I0V559u+zTg|bQT5~v2*|nHic1iex?P>HKUn3+<=uv93i?c8 zycmyvj;3xp9N%apUWeeLgVAFUdeovvfApxKUs(Boyk5piNDnltLSx<~i<~8R&qg%8 zl`7{0^x_LrV0KCJ%~=Cs!JP=mKZe``p4h+#8-Qg@L*}_NpI|C5&X#=3+=A$HppEXd z&yV(nV#8>pCDL-93V|L~W`$B|UM^iMx=Xobjg(r|OR@DRDYTv{`8JCr&vw1!*d358 z`v)Y${zXZ5_)O9re*uiy9k?7Kb8jGi5W*BCl=)0BHV7c!L!VLTG6b3dw%n0n=_EZZ zJ*3LYUn;G6BBV`>l-njrDR&JO+vQ22eX-;_@J0y7-jeGyNU}SQkjzd~B;9$Tq`It= zWY_(Y==y-fyS*s!?w7!?lHkto{>V6t{9WPnLo`z*E^iCiPLE(tfj&*>!b1?9>lUQ*OCPzpMQOK0a;$#qVaY?my_axIVyw{l5$?mFm?U~mlz6Xs z66?KIVtn>Wl+Q_t@OfS$d_M!fN+h+I6Mjq&kiR3ADOMuBm_$FYy)mAtFZwi~%K+@s z6S{Iwb187Qmppej$@cJ-Opjnm_l%M>&jd;F;*JRKTuJmPkp$l=iSz9zF@6ma4qMf?t@#vo=3uuH4m_{X`6Z@jPg#D|DiLbP}!CW||@7+3CEa!#(2jw}>#N*yi^ zsW*yU+7huz+agx!cZfy$qhguQ1AsbKgm9LZbkmup5*?0$@lsZm;Fga!H^ns;8;~}$kB%f^hgA(ed#Ob zQgg8>u@{RHS1~JLFe?cL3@jydRw+(h3Q-viDH}{sj|FqUTCksET&2ri2A81wNp@rJ z&B$Mkyd?w4{rQS|pv5Rq&wk%Q_~QUfhA#2w5)7>e`q+0j6SFEi6m$k&fWWOH$f^jU zDjck;7$_scwEHM96RhIc9&i%8K)`X+d$h_B5zRxaoNNeJ%aiJsAIne$BNJ= z6MLo8t z4*(G)jY$AU8(a?XiNSnv@MN$AY^QO@xd1!^J|_5C|1bn!K+aNR&WCKqRIU?1-HX$6 zA}+-7WV$nK4EZlQ*#R!(IIV&i`_{mL&T|F$g(|X-iouZSpa@`Sb(58GJ<0+GzD?i` z8u=6h&_(Ju)F0@oRg>_?>BPirVqy*?v#`Mo+z0dVfNqJp5kEvH9$tpS0**1EYTiyo zkE!T!qa$zw^vRS^kN|Q)7eIeZp`mJ#%jBCe^dWGT!T5a!!^^S`zAHGjcme*nh_PuQ zrkan-%%cu9mp(uzE;-T(nDLp^fk6a7XWXiNBj}g;*ke9=%=ZKI$@~;h2#Ak)4Qy$^ zycJ*%0eA|$#>MF?<5OyvFM;JUR9wo~2zuRwA95@S1OaDY&T*z>&6B)UY*)7hbB@~q z7l0mWLF(#EPyq%2I(0S5uilFB@rhN>5)7Z1tV8Js|1yvULV-J zItXk?X0*WFj1%(h*3?Px(;eucCIvh5KzD#GcF@T?VYurc+q2+p;}cTUI+U?s5a#KKfMOP^l9FyK^uo5Q{4+xumd05MJ*VY zvD(k~RU1{Cjshi9$!p>NF#C^!$G{WdDR2&)2hV{Q!K>gcaG?|T#)R>sel|%p{`m

fx5$uQFFch^mW89 z!M<%-hO1xh?lJs84=>TwUxnGmLJ)JfkxNUDNn^&Gb3BleAy4K`5l(Dyv9v4unUDkU zd)THrOIb4su*oI~rYTk*<1GEsQSk9amD8u4)#m+PB59ozC3}(|H3I@E8BE9)Np4iX zo?lFBlm7K01$%6MwF3>IuG(#XY!Pm9^m%G1S5k{XaRqe2UNjWgBxHJs*z08-*(IY{ zRiV4eR?O|akZObuz^Phg^|R0|4}a;#|MCyr z_!FcJ4A%Ij^_V=|1T7AHwp4v_4U51FACw$803BJK+qXE~Zzk};)D-F=SU(8ritaMK zKPngJvk}bv7k-wU1?MP$!8Y`An`+&DmjZvSq|fuMgyJz9G-;exy_QhE#{CD!k`%T6 zIb`)Fu9|AmrLLcNUe8UtQv;I_OUL%$q=0578twwO-J9@C;KgD6Ee4VZCY|s%OJKno z(j+(`Xkvj1!4-gNYU~hl?Qy8WW#OVQ#hElIU;>kI!Eq@5pTeQ!6fwvr1oKrO6k|#S zj)Nh5I`@cJ)vrPxto|nCfk|%WwUCjR{K@y3od1!6x+OW>S#HoX>%wmBG-FJAs$0_g z7^x*E!+WjgkO{QEEBD|?=d_<`HR|0WzSmuA%E+C7Fdmh1wlG*Kiy zEgU;;2MHA8SOEccmwiZV^2<$f(JhQ@$|2_&Y>S1 zFHt{5-Ms$2qHeg!kH4pG zJYz(?C2~PDy>k!~0X1Yk{ovq38|QUFg;fpW0?9KbmSLKK3Xif!xpEYGEwQzLJSl4{ zx!TN!{8lQm148+$Jx~5c@v|%cGF<*}mrG0Hwemkt)gG*F2{7E)>!>0N+IrFY`C~!} z5qPYs2vhI^s9`7!25hG>>Bf;ck^SHj0O0YdsOlI*+0qaZhz#^^W{Y9gui%#x;m^4F z|8=kendY+0#^MGasvqso4yoy4TmFY;A*#RET8J@pw=F4e zb@;*21rc}i^)gpaMD?*u03t#OFoM7Y zFv6-FOQ~!pS?Sft5I?d?$G(x3Yz#EavI{n@6E2RC>a8Y===0je3G>?f&F0mLHA2S? zWxm=F(EIwX@_~cfW6iTmb5VKy_+VMd(Y5;O7yZb%OwFoK8Efi3p=8WUD{kO|anZcT zy>v+q!fM-nWBk_&Dp@b$g^0&{{Ugg0@$}s5x?9QE%$AxN16lTFE(X1wmgqi^>&-z> zdMmooY+tusX^6tARzXWY?&D|D=?19|{muAUUGzDnggC0oY0~RA2aj+E?0i_d?6Okl z?i-4Hw@=kFUUb}~g}=tTZ2$#W9Kyq-%=l?y2zkan$N31cdsuup7r1D`#$a7l<`vu7$JRl^K=+d}}8BiP42qjFLmD2QcRtoaNiJuaj zSwS12Czf$rm(1-ydH!m|1e`?IV!Dd!k}xQTJ;eI!gD{68`)GP|o<4XY_xl%_(eRz< zqt|!)f*h~+rf)Wd+V+o3LoXG@H>VF&4;OKho?M}De=g6d2Ur^6c*t?kxRp-hQo0zy zgQn~cp+#`kau!;&LtBA?WX`So4PQ0(aI^%h#h)Ycr*rk}%S-7$=5@xz>5Qqs979x( z@NEk2xg#IsEng_zW-flXzXbohQgBnOyR*egc*Frw)FTqi{zjh~=@`0oFU>y?=TTH+ zJ$I);XLc0I3Uk8d=m(0_t-;XZl-*$pXN*$r-?p{t{A|lC8LM+Vj)8kVw`Pq6738=* zZ;Q9_F0=US`9~f8?&c1Pr^h#WQL@y*{x+9OgL?YgGCg$Wky?9Nvp;q9%I)^_9@NAv zH7WbE!!EGarKtzvZl2x6#)okdn+Q9sYu9(}E5i>)D{R-RJVY7@Zx_&Z_PGR!ugB*z z1`rsCrRW${MJG-sCb8ByM~yG3B0?D^xQxq4k6BjZ9LhYv;>w>KdkA<`$8L!?q6K10 z!35Pt|6DNO-0xK|fPcDz0rwDULhnFhq8eSr>Gt5mX_r}R{>3uuacHkDxBin_s%qy7 zT_gIS->buNX=$bVPL)a5a;iT(T!uI6z8EZE8+JKTE~(Rzqoq*M#@t7K$L^5_PK)oH z9h51?i`On5q(C2cDMlanQ2(#Rbm2V)hmLhVT=yMi(<~<+{e!~+7NC}tNL@-;k+c!t zEA}?&ZfooPJ6ll^_(ut5s-s#ZYtl+DwWXx|)$3v|#$$cLQ6?0|*O-tu218z= zz+pBqO0Qq*q1Ae!-D;Pdig!JE`-9`muEg>iefkMdMpny1*1DxmU7$$%0HYMPP>~dE ztIy5)qb|^0fFhMH|3^W}s=4&fZ|bie5bnJw9ZmkhF_*LqkEbuEN)QPdT^6)E=T?`@ zZ&DlelXRjB#OB8e*V0uCl{z`f@gO_O5eWWo z>$G(OHzO{rTr05p3CB92GHJuB^F6j`KjDxw3Qmd2#tFm+@_$Y!*lY5z*?A@Ut>@^g z(RzjKR2!KrjjDZnh~1Sp4lD9umc-RpG41nkeSeRLStlVrs{k>B-Y3x9lD-KnlNP ztW+~1Dgf+1@l6)KzsummN%`m;0jxIU~ctkoa|?n6mFnZ*yy6oozH zQz$d_&HMK?qoC)o(rTG<^rh1GBUeU`EZ<{qF1Ogxb`a7Z+pvOE<#Gp|`~}|~{SS%X zQJyznX;dl7N!EToNbx40kLJA8J+_j)z38|>OD8?~Qun>+ju30Bp=M~^9bdalos(~75|oaY%ZJMGH11>lOT(51KC3u|(a#+|dxJ~R z4yVw`=&@KU%Sqg|%pp0S|I-@(@K~2L1^TErK_NQ{AI@kMu$@$Q=vn_93k#VA(s_oN zrqD3&0c9g6G_@q@Y@zob*I>Fwusax~{+|c)_yk5cWYnfdMD+*eqZT zgnNK**wlqnSh@(&RqzH3QCs2+6U)CvIa_e2!+9K}};(w~o zD}_<-ZdQluXlB5@?G{f)Ed1Z8Xauip^(hX6f-E-N-t?^|0ojj?8(ST>WD?Kk_tiKm z)lhd1hRNo>|iWC|ukJ|2z+Z~s@6dEc&@iToN zdhNjhzXOI>#UB0D;%5gPGyEQp`NbIS7Lcd_OW2;c7wrl)Ia$#F%8_!k$*a{@^tM2Y zCA7Fx`?CN#O?Ju8B1=Y^(Dmxw9}VXHnd62cSPaUFH|HnV4}Tz3#+~Zt_Qy7J`tL7Z zu|Se2U%HkeEDB-*B2MF=R6#v;TVtwt#tD&f#n;|d@-mwQY`LR0x0ILZa}h7xXo%jV zO*U6|{VM9!JLlXz@7}ftX#K!gMLxV9qC40aoxG!!i$P&qB1{Oq`=DFA-!$Fg+wQZg z{lc(JY^Ka!&zxj`P30ekU;AfHfv1?nPi&pVQd0;`5z+#?J}n%(h`mgA4;_!8q)%7g zPw4kbv@DrYi{|J;cC@J&uO>!`{?)Eq%V{Z2>DnLPJ|5{zi%t5fMCH2b6jGK}K=YD4 zJ}0C>jj%{%CI7o)Jt7A*Bp0|{RVo~AD;6#DkSObF9|LZ<)TN%6lIN0AgCy*2KI*Gz zuMZzy6WM9!5_JE}w%#!7+0UOyyi*Cg?Cx9>1ZgIiY|M(_JD=l;o_n?0%`af?Vc)vp zOXvsEj3E18V;^T=7*+1NhSYlN-Sn9#{+iYiW|`eC$N5xn{@O_EKqgw*@L@m(7!3dsQ~$nx zp>_Sz*Xj$<2E{7hdN1$wrN8g2cN1<(-Q9RC9Sb_qrdJXsVsY_jZA&)ZyLARqmdm4X zVZxy>gXC$zy6N!W(mWUM+_TG7-Zb~U6!hj4;&t3Pv<50YogvyogWV97my%AEQ3l-M zdK^WDY3YTW#W=`*OI|-{GRE@oFu&OFesp0FVn4Um&0h#bU0Ixiyx@4; zEGrt3aTGbk3wz8B#vS%ptTUC>*bk%1-1~(`Vd7citYwex?xW-Nr{^oa-<+=?Tyb2& z7?Vx)(uj3&bDlrXYpvODCw#nQ=!8&ubl)tSsN+P8dyjSMeC+z(nKxJ5^W{EfQL6!M zp-x|itjK%BYgsqT4KQ zs<-l#qZK$7WNH&I_dqb>_Aw*ho z-mOUVq{QR022Zt>fL7$crHoxucQuK z@kU=Ycu#k_Uu?7Dech`5kIzZyg&jq;TYSH%51+kdmmI`d9MKIm=?a8rk++4m2kF(l zJ9>;DcH3ghZ8CDb z%i&0l!<#sI^m{$RJ;I+c`pI(4U;F&|uNZUk_ajoxZPfdcTC^POwDmejtXhZ&M|m<# z$%%}pY{a}Aa&)3uImv{C!^kn@TQ&o-blp@9$qHs>aXgdT8Ik~;kExGI*BCB&$J5gV zfqML2F8NDST=MH~!7A6d(Et>=$3)-wLN`KKwyA!M;^qQvZxT}w5 z1r=Kgc?uP6dyv}T)Y6+UHiNv8yARmTuC}w+=&&{Z-(_0>wbO@>f=5h^Md%R8Qj~@Mdwl7&qhOP)1E>V z*{8DW8r4xB%7PAfUGQD9>ATFA)wn*Hqk|nim{Vk~_ws_oH8>Jxk#XVs&m0dL>9#L^ za5(0Rcf1$69_&#Y^8$IG|G}q33&96BaLgJryb3U=f|*G##SWaZuBWs?G2>t;%<{bo zskDrUT1F>sM#SGZz@{4e;K*f>Y!cy^7#;KC0!85wCy7|NE_@5{oJ4Q;Nxnl%Ym<9p;cCqgm(4F_MI7e`S(5T znv+py$W5u7jSoL)@KX0a!l=fN%0Eix4nKACuFv7r8l>^@pv;Xlfp#Zep0BaA;V+S%vdaN_EyGJJTgNlX1e zx20q`Z7<5HkKACtoI+^CbQg!TI~{t~d{VS*+YHyGw|z|ohV!>L?GT0nJY`K@B3=f! z`9u{Gz@QYqu2pb%A79Gmj9JFKo5PRoKOFh=3df?G9_vad2nzn4Pw+y?K^lwZ-%Jr1 zb%4iHZ-GQp<3SkY0wq7ux@5TfCpl5r_7F=Q|$S>M~QYAcZz2sB`VLnF;8ZmL>RY)I7X8D z(%jXgoa|FKJtzhTiY`j)^2QZKizK``bbEQTcu7#`Xpm`7-$Jsz3i0bh;lQWHB+^pN zL2tf~8WD2ZqfY|XMwLc@M*M%;_y)|g2_x%w{f$xD$}&pNA}`NK#)Q|Nc-Qf@tVv*; zY6?sR_f#;muL-Dy8;Bw1D%^D9GDOb8{Ls`+b{Cve$wt)=8`J^g1p=MhI&2np)v7IZLvoAg~$cAY@t zoo6YJl>Vyk#Vdj%;0$Ji1B5kP>T~EpH&PcGUb2jiIJI^Qg!a8KE_4bLuLMVcS53ZT zX{LY8r-3h`OKpYjZ2hywzbx%yYVzB(2OoW&AiPb~9p z3_rI>&8H1!6SH`=_UXAyM%6@w{o{SDT!Qcqj+XGwzj1;mfD>HV2-H7wg9m4(@?}lF z`%jpj%RGCk?LTQg23HuLz`}J|cVO$F-3t32G=g6!Yl!H7FRWQ*Uk$R8X@+;k6BGKv%he>YDk+Z!58nJo6lc4a~~<9XOOYMYQNuyuGNR0-7jd_BsW(si}y%z_@sMJe2#y(pLU20*-~Qr z7*R7Q+O$zK@~lOFcr%tSpWz4_2Rb4=5kyX%WdI+rWh)DyOy*$mh%&+=suN+39G`j? zoC0seATHR-gkRXor~fN$ktG-LN_;LrRplkKR81c(kVVmKMpjpLf z<&aCV+w%Qi(dFL)%+D$0QU--`pw2Ny(BV|+Q2v%`rP>$T!t)N$2Ayl&YMHUijKnG< z0*_4;#4gv-xRftOY$i|+hS0)LSL~&}K}^|aKb6%jK|_t6YMVExi2G?k5o4Ae%`TQW zjg1Hc%H9|rAU*6&r&j zI=sHh_>HBW2|tgn?#C@fs_A-BwH>lefKVrUHgKRse@KfNCj-8Zj0lGOv2t#OYzt>(;mu9kG zC%sAc&h14yD%x?VOz3*_F#;QHvK7-=?q65;*=luh(pB@!Ld6r2$0i6gWS|(dhrs~y zkE>t+|I~q+3uhfUe&dLO;LY7fqIge#QhmkfOlcIHs`S2onYG!Uc?o41(23IPbA6hp z{pQj7eIp*ju=*Bn?k>9-T2Ve#iRqAVS=2-KUre$ZrhDa&?uC#P+07slGD%pY!oDS^ zIr9aYl%dn&(HzR)AU>MP)r?<6ZaXw;=sL74X7Ba$x2Uf_I0|u#b`_8y-O}Wp2|V_{ zv*G>eU^L_&{u{qZ6!=ZX$sq<-4x@Q5n?)wgIN6VYknq>>V91Fq^JmgL6nJMAuowIX zlt(tuzi>Xw@&~T4{~fqunaiwLjzubR5d!yl`v#qW~whXVqM z&6c>DXTxWP)11HaH9|o$qdso3!Pi&Bw?2BM57+Chpe+~eZSJG-`;b6B-m-a|`7ZJH zC3<00ZAvD}Z*wLDfxQd4LG5bs5whvZ)ZbZ>R@tO}K-VvHu47QzP+>M zTbYywn#1sdoStNsvWf@5h{7n08w4;ka58Iz##RpaOw|d=#Zscl#l7d>Bzd-;yY1nt z6|bjzxL|0R`JjFE9f=C>BSwND+?iKN;Ii&qxOHZ|EcNCX+!O=Qn+}+S6D{;DLImFOPp>L-L>Q{6m&NhygSiTap1<{KOT2Y&L~p2NVHA z{PQ6H!5&Z?&R}@C5%R)CVH@{SiZOrB`Wc($nC!4NRD^jp@Sj8_+K@24OQ!t4BO)?J z-tC4%q%QufgzilH4@J9 zxzVIA#WP(F}JiLdE#>NNW&yaUO`Dw%+Tmq4k(F87=C~abD zJuak9cF&U(rElIE!w;WPhK6Ip-aE~UoQ(J58wEI2|& z-xK%;y{>@*x!PfOqj_U}(%sR`qlA4U0hIbiOb>r!J)iA%`PBo1l6tS6=D;#`5sq|< zR@`0x2-ZD`3LMUN?aSY-TktdjYOo5mBS875pJ^Io$Z`JH^8zMD$7~d~Ze+~iS%wZm zHE@kjymP4Zn828H*3{(V&A;Rggto{fS(6fbRQE6)uQzR+l^yMWd7KsB!zR>XV<+L) zGCHX>SNkD{NPI(E;W0~7Tz{BnJZg5wg&!P@Slr9sW3|7jD0jQ>G#Etbti^NnCw^S0 zcI>;PgFHU$WYB_2+NckHKQj{72_kU=>fL3#xH(q`tof_*sHO7gv5?LkKJ=OX^m3E6 zc*$$VLV=Z!Dq3Q9uBMbJ*+wMn{^?*qBgewk2mkRCMcG3M!fWy+?nDm`E}! zrz*!{$k8nY@gFnhs($O#q<1vRN9T`9YwAZC5W_w!y?e`SL(TS`e#TE?sqUz|_^0u> z%HhV3xQnrAx6D`1oMRgG=;UMK0$wK$O~vyzZVmZv_(p@!NAY={4$n{J3dnb6qkZ}E zZSqLxiI<{HMQv@L1R?41(yrD(b*<3W7k8MgafgqXXjVc3f}ponih%8 zqz7OfSf2UV z@8j#0esJj4rJaXXq)K~BUNBmbGSoG+A`Rho5Uyi13RQi^P*<$&9&=9&aSL3fbCMs% z*c=X>CtiJ}u&nIr>b0R7s>MwLDUa}aP?{wl0DJ4L} zTG0G{x?594>4q5D$KB_n#Xsc8SyZ|e_h&iYAm%moxumW9;M?&Kd%kq@!{T1|lS(ml zU**dC|L~BA#_yddVE%T01l^oZy}$66p=lJPl@ube7%3jiDk1QO$XkL1$^rlgc|gS7 z!Z9i^iNMV7VbcQ+DH}1L22P$RwSH|5PcpdR4kW3{?==17X-hB+QU2GKU|27a{MUw_ z(6`9}J63w0Z}Ghu>0@c~<~i2*h4vTQ2t5U3Mn_^&|=g5)t`2McIPdv*noA>yxFJnp^ z#t+$$v56t1W4Wyfd|TAaxht;>*d{+&9EyFtE0$u>D^ZljPhH-xk;(6EzuWwkQ46)o zrRe_N-Q7Z`Xgh{PVA7VtPaNJ)jC}ckNP9tDbg7L4R(7sj(Hv~6DX&0t(HD|+8wV}& z4b4bBr@0>=es{QNX8q}Q=Z^dcX-z#BJ9SiOjJ+NQT)N-X)dCb9CRQqZo}5r zk8kTTYHwE$9_@t5Y;f9>F`oL^FXMUz_vc9(PCHKJJ!*BJ@Yc?kJ5hOMNq9jrDx+>k zu~T-F`Yws*93xham1pF`n|%k{?V&eZL)uQz4efR3%Um%>UkP^49B!8@aqJ6kt4dRW z2=^VPbb;@9>V|XWSMKL6E5#NC3%AQd??DLfI0u;{rt40ue}8k)T)C(ZLMMgpmqe{g zpLH5meC}0^4I$GIV{i{xA8IR7j%@Ia6OH)v`pIHPg|LhbcMtldi)OFibneRPExrXe zcR_rY4pHW#nUdN-kE}92ywZDCGtG|MDWBXhT3&!<%scc9_YPc?H5Lr``eajZ2dGbj znxNIx%Z~Y~Yw@_LPg9+} z8W#y~ZSS-_Is{!-@Bf1sMZN*IY~0`brbrU(c{nmE6Jl`YGQVMv0VED!8v!=UNMd2^ z=cO^08yikWj9KPS8Jsam)J<;^YsJ#@q3BlqG&Q&I|4VCb>DxDBJnZCeeACtDi7=Vz z?=WYi5l**<`H+K7Iwtx zy%qiJzv8Sr1bOq11XTum3}a?FtSTDBZtT)q7oqR%@EUFD*6uXioSgv4{CJXifrCs- zJvla{eA{s_xFCu!)A8PS@zS?HI4(dYZRv5|^&F9N7=hixZwn0+mEjAH$PkVh6`G2Y zl5DW>9^SL^<`!;~1BX8yzrNCQv3;SUgTl&;S3)!JpGNka)Kl7{%r+L8=#%Kg-;CsM zeBgDXL{)=Ex1-IVJuE!hVCZ_)^Ai88QCvI}ZLcnmD4ZarG!-p^WXM*}$kF;IgGbi{ zEAe0EScW5IAE4R}^L@keI&qP8oBmL-urj9wzV!Y0%;f2A6vZf25K=io_O|7(aF8pP zUDG30f;N7YpLdNz4BfCV5{DPQRi0h{Cw&na{LntzX!z&lLUTzwA*-||g{%K{U2|T1|GB+&`ZovoXw}%XRISb}axRYQFeHKTDFauaeAHse z!1NDZs!7q&YM|qQeToYzXo%r_URu(OLYk(Wg-r>S#R;BGBNAy=`3Ge#;vZrO2y!!~s zAi?S#=;arFmv!4WrIl!&60>E}k3~DS8wdtyz3GbGk4M8H5%@t<)4p3!ROYkO&FBst zvtB%Qq4z^$xz?8Kibh6=yPrTnE96UuD7#H!z0ggcrFFasI=I0I@cwLq!VC*Pvdkf=hXCAzx zQFxJ2lnF6O$&wHvhH0~;Yzd_@g+gNNJ0nX-CA-Kj`!Zu4V`lunvslyfzR&ai-v00N zxzD)Gz4N{Id%owK@7a%VmHr|5cLy7tnaQ%3E2Sr94~!r)EJf{UTbhN9v#b&N-j>G& ze)a^924de0BbqtuQbnDyt1-$A%jzYvN4gU}*Sv{&F1mnluG{`2G##i(h=t_nK(a?+ zLWb^JBjr55{SLjy=!J|1rbBoN1dW=fUf__R}3^NhP)v*jk~aaKb>!EVZ9_6 zZ#rw?>Bh#kQPQpFirDG;A8M0wZn^RK(!mouQp>M5CET{QD1;r6^zMeka&-iYF@kS0 zf|`Wo>avP$#UcuEvZ{r!i=c`s3AvG~o0bl&JL0?=H37>DZ(lJTk(Z^Zlw_y@Aw}89 z7XLDYyn0>fZuhhGpXMcbVq*@i@^DZoafKJO`75^Gy6V;9^R?Cb(h1lZ70uC#-d4N9 zXV*nqAKN`RpZ54u?z4hUM#)U>PfpXsys03nr-wf7JR;D*1P9E~GR)P=gkrYj6PxE}2lF2hL;VAE*Wd3V|ua;yKevYbSy z*(c~#C+N{LyACtzpYEO5R(OWcAF!i4Y(-1Zb9Nmz`CSQK9<_6&yW{NyI|(*HH_3_tF^OgE<0HYwc9V2auo_SieM1IT&XailF>4f zniet7&%`WGwRD}wR{c>=Y!pqN(c-R>&O^OLLf)Z1hZn4f4HQ||ohV|RLm2#eN`b=X zaRk(UxJR^h8&p}Fxx&wKVg3d?4F?T1Vz2Z^S$lh8eUq~jVso|Ax4ye|?|OYdHZPHK zAc&HnR{4Rh9pQ;A+MW41D&L$(eL`bdSSo?|9FL~@ZL+2(*I?US+5^+*jO0I{rA-nt zvyKFq8X)coZpk>Ayv{JaOj(y1%wpnDVx9rLWlF-N(# z8_HGV*NDr) za$}CZ)KZA;7yDM_eLDHm(TrF8k@kKFGEiy=3o#PKnEgM$s`XU64h%ngf(fBHwi9$z z(t#Z?w!zwwx@c>v0UnZ=3iyNnC&I@BLNf62J#?*&yd3Eq-Y2{oo*%SC5)+d*Xxa`MEqjsg5xcT=5Yhco zvY^1(t|oX=hGG+F1x1r~XIMaQ+q5-`#5RsY_O&T559`Q&SuEP82kQ~Uh88sf-SRA` zj&z~{R3;shg+{g)`ijnXpl=_CT;E#;?Gv$A0CJtF4xGsXf3Z@6sRbtnXc?9m8|rKb zI{a9nDtzbA0?V-Xy^*^k1CwI{wCtWfmAaAhbJOB>89rdH=>}3prS+Ac)i&=Aq$Tn# zOUhi0AIlE1w?S%BB!K@(BGfSB6t1}nts1vpHT3AKQ}<}`4F^ZoBm019eb!ozRUg^G@~%b&LQ5FbK0QWb;9At#jOv#Z_GY0t+Ye9DhQik;6PayVpp9OB!j z!!a=9G`X{L9I|G}B;>IAN{5Uys7zeE)I-5M*vbW)d$g_`92%3dRJ||{50U}^;FTuK z9eA0ovxF_ivP|3q&SqOOX2=@=c>up*qcQjc@Q2NbCgp9`Xu)C=cs6+#7>K7E9ozT6 zE0Um~%_Gk)y_*58>Y!^^y40FRI&ZDDXn8%-TmNcUe8?Ik z7UL9)(_S9STb&nB}Y}UtnGW%;JpH7fq5I3zk9a_GB`iODe*4c zVB6hlp++b1rNFds?npa?Z=Ni7Cu_@?K%7YontlRrI)tr2&aN6jsV*WY;;S&`fj;>nKMPDhtnzK_nq-)a8_=61EW%3p44@q|c5WfW9>d zOtx)kXE4PXstarH1|@?djcgb8*S3#5uKp_dL~75V?l%uTEuA%_Q;Chjs#dTSo%^e5 zSG+widck&Iz@3XfN)w9bn5kPUqVkx8FVfSdiD}+uijH%SiFs%9XWRdD8ErzbtUw zAmDZn&hhQ;!L6=4hnN(3PWq}^eDPrF%VVdX$QoY5$@zhmhE*-7{BZV#iCvh#cJkM6D$B@R&@6&D)ev9Q zgj;J={qtLmFs-mC_E6Q5Q511KBi!8NQZXax0IU|<{+hm&3Ah4p#dH||njWd6#Qe2qG2lbU|4lxmzRnec3bxU&|6DF| zH(w5zPlK@x^XZ+w3(h&BE_EA;%RO*Sn`F-1)GpS@9VikV3+w1LlLmsL=tms4mA&2P zgebvNrA55kOzRE;J3Zl5UNd?NC>i>SdZHO7nJ2IRZBYHw(*|t45}_ym5Cm+mW{3Ge zO8<~eJhRZ!3|wZhBcAPPtABj;tU2EQxy@+nPJO+0NSE4o%?oJlT;nSX64&jEQfPXx zEERWzQvmRg%DWjWe$j-*9=ynS0ZO*Un<9|4~0jWee3=$vX=B_o_fmk`&-e!OMg6GUM%WB|VoC648y1^qA>o{*;hx1Ya>#^i5uf4OS&F zDpqCm>e6QOA-!^sWbjPw2&3s}@mJ^g2S6q`4*-hx0YMyiZ(1|8J@A=oF%xfCxJkh` z5UAeL>*g-FAOG+6X}V03mlEYB1$!N=&hK0rHMn^wG%}(wHSuv>=lZt0=XR0uU;BLM zTr9OpmZwGM5Lr|Y+p0~zBpaP9^&BOT)kgC!$EptEUuNub_*|_M(PqTI&_^u1+4sP{ z$0jYv?tqh52N=%ttoYhBXTF*Gm!e{QE6l$E)5^Pm`*yCQD!C5pD$=&U%Xrx&ENQpw zP0Q}^Y>#ug*-$fuK-VwtH79%-xicw@r;NNm&$jKpKF6u#f!zg#)ylqZ>1N3LV~-rf z>!@Pnh$au`%B5n)r6Mo1_8E>GveByKc!#sf%K&Q^dyHJ)SeTC*Q-?=LVlbH@{bH7f<%Cw_#x#9qOi5p+9m>uD{fg%cMZny5SNzlD&S7UqrO!wF!yQF&@Bs z+_PbjKjhmA;4pBrDQ)J8z2%#+?ol@E*sWdEHd=YlkklA(Is*;su!U8d;Wz~P>OjFU>T@zcbU!`D`9sq(Sw~)Jqk%@xdtd9;(P8wNB`1``pVm zA=#P|u7e?^X~#me>Z>J)N@-z3>Wn7nKv&@cW?|z((S3bcCE`DmTB`Gp_+V?Lc?Us( zT(zc9&lV9_Y@J0;tSz5u z8krI*zzyK)REY&Zc5KQ6pnozGOBiIzq+j;uFQUeM)^bh$jJ1|BlWzl_l_Gd2bVF?|&tF&pqcP@1JgnkmTVF;?)>QvSE{{%enxaI~PBeYh6o>y|VdX zXKhhT3n*7+mv}d0y#bxSzgMKpNYbV4i;u%`WH)~35Rfhg8qt|BF0yDrdYPA1x}jN} zeHL^VDB|_jydT*hFPN0ejk4D%_-mS+`zjA<_uLK{)*b@y#NC7SD~q9j)2gB3-)Yq_ z0~-Tf;m=4Ocg0lJ9+3;Vwzh?9dl`CnIC+FWZ)+0v0c=^CC}~ld+l`#>r#u$Svz-=w~vdauD0*T+sq@hPFf&q+VJCm7AB^k45hHVT2|Corb5SzS}Wo4di z4cN*gHvq5M^8R#1^4}yvut;oHALURq4w*Z=BC@iqj77qK;`4&@qm0c4b?c$rG4_#E z%lP|uWkBjvh(A8ReJ;iP2I)4*SLgxgpB$TtN;Nr+^}?(I8#|zuu-xgLEuL zTe(H!>d*N5o;adNw*%oe4{0O^L>=+(B;SMQhSu79D}Gf|iQj%{xW2Zil3|!mKBi;E zGcd>kOCJ3hUSV-khOz7`TQ`wy47aHf%Bo11*rzC#6Xm8w!1x4{T`FMDT+R&}i zlh^CMt%+Xw8qk(Uc~Mm6h%P8Jt6o8@&h*rh<=gY|K(Kk}In{OuKKR=19?P+V5xa#O z6XG3j=5YzFT4@V8INx=bQov09JDnH*&^x?hW**F-XGYG<8i9m5&yW=CF%vatr79CNpxRprpU7}eoVgK$PL)AH{}s5!X^*+ zGsdzJ`v&lFKH7i-NBh`uCBY@;urF(9 zyp>fAW2k)WN;)t3dFHwd6_;xpguf@`-hvh_AoLRSy3OoE1Cy(pLm6;ki>6Evu@&h_ zK(co}MwNh9#S`Kti+~jjm~=g*DYJ;eiTT?pm2dH{^vQbW)GaI=Q)2!BFT=XP`jYWJ zK#m=U{8o_t^k)QFI6OyZb1CMq-f)EaPu*i#qUsjyL1B>=c?j`u7-;Y&`5eCv<0bj- z$aH}(v@+EKo8l2!U%*d-EaoUO(!rXZaTB&DgeowvE5 z_}Wm|N5qZpEi__2f_PXoCg%$OQspZUEfAn6-pG|(6}$TLDMdj6|B0m0V=RscApV32 zb;*Nu8fy=Fy{u|HC#Q(Geb)2X_M8H)RcSH#A5>SJ*uUl*L=w0aRf*^WjmCwLd>3*o8iCgUr%gcP}o!lzQmx2R+k|J@-Vn0HYzJ4+vZ-(cmx1jWd!LNhG{5Y4Xrt5Ww@{B@di)HgM*uiAiLK%54mu2(trq6P$q+Nh5onpMhj}h0zDbz#9I-A;lh4VUjBZ1Ejys&l8v3mr31xm4hO|^i) z(&sbjaB5d&lN-yF2mmsN2_8TTkTzg?zZ6Dg0TAFCi}V?LH|v<>9Ww;#Vd(#Firuwr zEb0C3>mt2c*H+4vBoCh6XA}?{re#2p?G16DYrlCRYQEogYnUEosSw^rN~>uZVRe@p zx^8bapDe%4gTo+#Udo@^NcL+sIgwWfPpz=h;9Zex(opM@=mPUuM+s;H{ioE0Z^X^C zB{IZO7C}OobrC~%X9@`fKtMK7hBsNe;RJShN3o7hR*dm}kv9De{WA20gNP6csurYO zw`DDZ%{RDN{Wjv7w@qN@p8ovi9p6^b!=x9(Y(@@k5(-SM9#)!w`Xu=O{t5oUm0yij z&soMQ*7~E*Vt}puv0)$$n6-+F(66wuP^xDU7nWJses!_z&G=5SzfVzM@M*_+ojt^i z=S}H5yWDdH^cVFQTw?j%`6=!eQR@~;M z(*DsE-nxrAI>FdkD9Ppm04o55v|-udVi}~54;Q@RmRQ=t?`-9{+A)~8$1{ZdfE(T6F9Bjy zHV#oZ$M%4bH!fn{SR#y~D$J;I5cD5CorT^!14TVMukh(?E5WaX4AgLAtgC*&sCI*R zcdW_FeZ1I>29l4A)@r4$bv=n)=SyAF=bYC75;AUPm@{F@0_aqklSwq4hM<#p1;~E} z=)vo=aB`B?*jye6WiyZy$PbgZfD^`DPgG^s&l1(9ytX>V&LK8i$lT0A>7@evQoh2k(XVCm3k!93F>k z$iBj{NNe$e>+*){d$;x1kXtjh(W0f~SKvt^^(W)aI)XJ_u54~wom`hq)V|jR#7bd2 zCsPvu*yve2fmi_M+$?s`cL%{=M-GvDOs_@WjEy_u{iyLQXs$Bk;fP)=j@(4PVWR#O z`QuHY*4x zFwB!+u}MjW@uLY@h;0`fhTevC4i(LjUlI8y@+&TweiR)QlqAri!S-9Ab8bu(tp5^{ z1TAG8(sA~zDtbREb0(d$K>=PL0E>b8L-mgJYqpk}=oV9tcMAuNn)F8FcHdO*yHhpX zQ5sT;jIGQov|n~Aftcs^(<TC*qO19f0CM~z6m;HcGOh{h1GNCtp0y>vtk$?<;d@i>vEmT$GRmKxhKMtyZXQBI{Zb=`-Q}x` zPTqgA;8tuD?FYOwk76ZWL6smbN+xk=dz|{#^Yr85G1;1XKeg~9Psbr!%7#IupZe^L z+wsm~aAlE9*pAu*z0%q#zKE8V%68P9w7t4u*bd5yCn4PxUOjd7wsKOjiOB*u#R^9Ob(MhXrltp8n)Tk`55vaDgpR6K9S`X`zVV?Ofv>Szj{o78h*WxwMU&zH4aIN1NjlyX&&gFe4X+6n9c)x(<1w_&~0 z-~YtB0!e?i1N*7$q_Vx8TNNk;8-P+^9AwW>3h;9%1wQ|YQgET@5ym{w=G1B{xM$_@ zE5esX-8Zy6gbd!P#B3~hcK85FV#K`Wd`+Rb`IapC)xP?O%ya2*<@9oVV{}Q6An177KH_SMwKE#UZehoE5NGaqW09=X`^;5HE*Kt z!ckXv&0xA@ky4C%sd9pA#dk!vxsd7*WvsGHa;@9j_KlY{9UkhL>Rnlxb!`4Z)xJEn zP8Cpsv>XkLewu_`6EZ?DpfIpNEc^qXY`VHMfd+tl8PnQ3xy!L`Whgb^crx*0*3YKO zZLogsSv*1HtVgiBCFl@{oWB(ZQ~rQK4(>)~7+~9eFAmzW#KB*xNQ32NO^q!X0Oc!% zcr_U71!|4AViSa$nLbEguzBaWyvMK&Hc1^%} z%Dgh1?s+;0Saxev-pL49J}|uYVey;m7bv^j&yPWX_uk2-xmo(>2xwksQ~l`&@p>@i zQ#Q@37^^mje*!#&rJBCsZHDv(j)R5Ud?W@S*9BnRD~>`PRL0^|b!5%O64yvou9k{P z{_(Y2<0<8_ZROWQ>-}{X3N5U~Vt***w)m=pUfyREK`-yncT5PdBhDv*8Js6|=C)!c zaPQlq?_{j%+BgpJvwGKCGT5BIPqgnctQ&6{Ms>4eL@_q!1c>yT#y3#bx>sE+6{H*w zpO#iZAgg8BN;pPfUh)^~-Pkjb_+>X}PBAU5g7bH26+M$_6=*o>r#6G?*aF*>VS(}G z1SL+^9e>@+lx?mHr z2p@;kz-W9vKhg0pswZv@wgyl4kpFhKN>nj+!2 z4)HV(?tE)_5YH9FD|a7hpW0lvA1HOTLQ{)1obKme1ijw$fe~KaN#`OHtPFovlYI}Q z+7YwH4)Mx3ZoI#AK|8WWhr`4C9-Ra{5N)Qy4pjPNPlgL1yy76)?76l)Gb$uhWxD%vPpEs zyIz@ND>jm30O3_J6F4BP)tIgr>Bg{o%cVQqp?^m9ACuZk@5!S-0g1lV|DtU;0~ux> z`)&m$I<^KJcV*-|fVvT9RFqtMTbCU+1i4GD%WR&2+@%laARu=M6r5>sI7t3-Rj*ak z=8|}NG9Pc)yj7xOLE#Sfy3mp!BJge87x>pw>XM~KenGoTe=gAai2csD0ZL%()s6nG z@u6LbU#!Em3$2*>4z7~(y5T>OJT^E@lKwl2hfDm1zTTSKC5elrdE}Q$-v-=OGCFj^ z#KD9Sy9qD=2mBrJ>g^%I+D|KRrav{>pwLE#^_cx+l_n1twQ$3=4s!my ztBuwjQb1L5hU2ajWq=6byF_HRFf^;~LMt+{^F&egRn>mn@VY?SEyU0;XI9ajjJ3q| zu04VDii%3ko=OFT9B@h+=CfLT)S|*?LJxyS`&drwx~xl+))p4(3gu4j%B<@E!?!cn zfVJsPjyI*!g3FAYj$iL$+yZ_MuCZcN;FTHk?7vvdnU41-S{kExN%!cAU-9FRYuLdX zC?AJ1Z2wV=GRTo2P|opiqrUJE7c!Y||>J^ky%~u;rv@I3YE0_`TtmdlF z`ar<3LgB}Cbu~}o2^xzRJUQZ)dlSYOeyL)&d7+eN<^$Dq)ReeeDX_n5zZH z`hcrMrrWr)4}=aNyn}8@I=gT8P$#U{y}~aWca;{|4~ygk0T4s^m)zGs7E;x7TU360 z{ZeTN{&hryYxUxW1lNPQZbuoR1g9k-oDfx6NftV?36;rWMs*m%H4FW~pI|ZO3WJ$U zr`8#|Is*aNwj(eQYkva2XC0GrCSyCB?OI?V&y$HKfKV_7LI2hj5dSA!0p^7D$d;Ez zfY;bu-zt6P>uRDtxVLWiQfbHpf#@EO99DhTaZV;S zN4(5ayWaBjdAlvX6X7$^mn-_+PpMp83%us1lyiN%ujvca{)DCrUSFBS^H{M$sJwuR?RiO@qVq3Xq-fxxprT+iYr}+Y-biEpSIsHH+Bck!p||? zOfqcbf?X?6GSW7Xc;!eU`5Wljp#g}Z{r1iDY)jJ+&T7!*RfANRMVDHoDMk1$S##0e zwd13JDMg>kNinMfF{BZ;>HnHRvG5R;`!7@V!W@=k|1XTu@1Z>8RC!{2;`4CrUCf+> zH{R`pvIe`#Eu>{l5>9CxdTnIq(`R4$S*m+s1`6q&wstR|wnF#?q`X{{t2z8v5(2bN zmU{4RHLX)XR%6@uk}LCZpiC=h{F(PPy7+sXG*8UigRqej{Md~ZN07A_R+w4H^0e_Z z#j)nf_+-1W0fhoCuuIEeN6O3139gE7J$n?I&wS4NHm@QiCN7DP>RR!!PV4!z^8)5a z9cznTae&s@=~$3vyD&16no8i#ukhrT4L-hsm)6=&kntYG2W5a3@_rOXWD`ZdwCB`% zL*kzOz+7*Vyja>vW$Z1eS`w$dO{IAEnIgqF*NR%K)80q8Gq&EnxA5;U`y0Q&vM3j9 zt(^zQ?3Rg*LwwC5C1C2H>&;_O+@Z6A%~mG)3;SgBqAN9gg88s{j@SfWna;VucT;i( zz>KL1sH|U2j?dnx_s+VXMfU)Dtb?V8@O0Hf=&ir;554;{27wQNOLeLL)<5LWHVClz zGVl!X429eubQu&~QDsOHQsX*y3zh?#jX&5yIFYf@Mjy&aUME*9+`QN%qVnrz`%#6@ zj~w1BbF0T&ma%7k*2-;1wnnV6_L&O!Qz$X1-Cm{>n)Kd-+s0Uqr0z?xcCqZT8@)`f4Px$)o$TtaTcNxi~!Cp#^ zn-+uY43JxK@9cOmn*fQVpuq zOc28tH1@Ej^VTp= ze|@e>_086aJ>|9hUQfB|^k@RPR z7@j=LpFKY#gKSRB%Z5RF<{KQ?T(_^smx7V%x7Ugqz5q26ihP{Pe4$CMFzW>GYuq0> zv$z0*GfW~Lm<${D7}J{IPoBlhb$>D;Ma<2gW{Xq|cl7jQJJ_I8iO}B~BL4{AaVP&9 ze1{DJfWX0t;}A4#EPJ%90!nc;tD^&sL^zffR= zy5n@Ln0-E%P@@by>|mbk_y+VDDlafgnf%_jVf7cj4Taw5pudxO9jV;qy* z7zN~&!2e9cF!LLs$cI^ooHIwUY6MAXK^#=Oqn0$dy_$7ipVDtdS2l!f#O^L{|(fa-x} zG(|+y1r@fGvY0tJ#x1Prda_V>@`Mxg#hZ@lfzb{2bpn*_PWtxS^J0Ru(-*2&p5fT?h$>1A+UfyGrzp(Whf)myv2fEG7eD% zM$iqQ#1Ycw>PH8w;V0p*yOe$RYlZY)F&%Dh0C}{x%?>1Oj9@rfzga)z+VSHc zMZ{l%B7u+ofcKZi`}YE6?w%i8XQUhXpKg@TeN&gBc0E-`H$0ji&%egEeKF zw=^^ONh`t{yNcorC2kza>k~a|-u}r(yqGNbF=I3F)LYRsi7;iuE!#dn+En~WdJ}Xx zHSZd;907fIM@pifN#uf3yb8ZAGTCAXX-#Ub%ZLN=i$c53Hc&}aGrHY2NcZ?pLh%)3 zAHN<&0@kA{YkMimiYs`Zm#0kr*7RNpjcq75pW@qBK-VY!hGJP^NkA%d9Pp;(xibV< zc*9U6GJ?qxKmx6iW7ZwenXB$Vch~Rg4kXosO8Ui8yZJX<8PjLDgO}p}_Q1 zhE~)Q0cv1xY1U*Ln)0oUmgidK7RG19bJi9ji7Gj;_GY#H3w7VM#2U@4R6Qr|#V*o}dJ#Or zDlFjti%*{ZFAnF5**9&Uu;Tn;zj-nti?HrwNhDKNOMny%n%g={Gz0b=a6d~2Vw{3e zz#j&inLLK-Z-Z*BMemVg(*MIeh4sDw3qopu6C(%R2W?aPj?HU<4HaWwa-*tzD9>4d znd6g(r#wyKX5Ydx)i28)N!<{-qT9P1Rd&rp9KQ4Y5U;`Yh#lH7$M$i`^WWzODv&o& zfflS8_0w7JDI=(62*)_nDmoF)+kT+F0R3Gbexm5`X);!_`qT|(s660=CntmDKMrXh z7*R~K(^jh1GIj*P^2k-4(z%$}$yj;(`H5Kh$4pC2zL@ISFZvTp%U-8RhgoFehdM+B zMx=04TD@U3=W~q*@NvjIX%rFE!49U1k_(z}&P9A{QON5h^G4&w0yXy>?v_3c%1;Bb ztEKWb_weQ}js76|9iQJ?afxFMy`a6`8IhIofs7{kW!0}-eNb4$;luk?1BN3fc|gU- zi-YEaZDmV^A`{YgP2tBRqQIL9=HM}BEtx3AKrU6ZDwXeV3o7U7+esqcjj1YeMk9uc zx3xUF(T`0D@}`DABUQSm9d0m@=m`)gyOh8M^;tan<|$?C3YAJ`?#g{&L~ad#KkAzz zWU?=ZmAeu(Mvxix9~A_N*PEK4Yep;h*R3|tJaOfuOMS-~4{Ml3dm1Pp0umw$9_Vj> zAkeSar?c|&*W4Grmu@H?0`X4Uw=*X)%cQW|JS}Fo`8KIXX77Roa7T72Z1HWn?6kN} zVhlUq9Z z#UCrfR1WGIJe4>D#gJgl-rayLl^*$KSp@yQOnK{9xB=+8J^F$YYxS?vS5Ez{^py+^ z{HK)3wPc(Ce!-AO7EhO4z_NQ5Pp}CVC|G`+U#c-qA}NcNOchd_5`!jb7Tki@CNE+5 zc7X;STUlNoRYdUs@jB^OsUg4`?*AtYFz#f+UmIZF%?QvP9*2-fOhssTnj*Bx;Vl5q z6voeSuHRL?_BmeTxrt`UVV7~p{iy6ML+YK-k|@lT1{&u_Ta75vj^_)z`MP)SA;Hd? zf5N>6l^svYd<5s$GO6|b#+n9&yk|J8 z1Nk$}1sm_ti-6p>$7`d+WiJiwQ%j>haDJ;SDwP#jou;-;e%I>v?t9LSH>@dT-sPT<;4deoJ=<(|uMB>t)k;al@g3>4fZzEhSW0!(q1_ zJ%Xwur9rPj9duVpAFatf-dKTi4% z_kY=$V;*)BahuEEX}A>*hd&H=TE?|1E;27Bv;gTD=t@p^Zxt+SCG4(EJiNEp0$^?R zlwAVe(RVjCxPFCY?lIp7Z{+x$KkHFl@EL?Fc1gVsiw$=2&L8eG=e}2*He%ZH2@684 zn$nG?cpk{ksWCy6oOs^l64BGV!BF7hJLHRtx zra;7YZ)k(ua9(2vp`8akx;4$)diIBnI*%ha@S4kLnsaFomTIqW;YR_akBN z44QIp6aRrwZrzi3x&hhz-MNyOd#f#{@4n7jvg5LXuxfk1b0627j-Pjh|9R@cVdNd4 zVDdDJI#(Ato%l?m;Z-(?=x%^u<0SF@BX|$`PRtsT%I%F>&o0tSi_f;8tHZX))5LVb z$SrpMLXnbbDU7%IC9Vh9v6+qEL`u3;l6X~oiLh;MjQkp&v zZ+qg;xhuQIJqVP|0VO!eqRmT@e#T8thb3fgEhS>>RZBaOyC}|}(4aIos53}fF>ehq z=xl<@ld*#J9Yg4`7N9x}`P8Nl#aSP)07;Qt@4cd~AYJfp#UhjnQBh7~CsObBf7Jyx zAhY)DDp#SfceMXKRod8s*s5s>3{Ve1DpjrG}q^JHVTHqgkd3vI&Woj=lG&ctYv%dmqR9H;6_5?xXKlS@ zc?P%Jvod~GLj+#TJGSzb@8)?0f}kwqb5JwouEZjc zwu^Sgz}wD~E%Z~vBhNmq2dx0v#?I2qXM}}wW%p<|vhoCWJi&KxS&MZh=-hT`_)@uVhrPvMDIn73}@qqX1 zQESNrFcFIZe6Jp&u)Y(!1$}`Er@c%&+^Di+EJsVcR@W?33D;f>8$L~I?yWz)kQ)_z z8~x~NFi(TzlcD=rr2oTk=>CL83vHtO+(1N%^r61G}q1e!vZ0|sTPVq*j<+fPR> z+^G06_3;Xjj#7KwxuYUOfmDB+D%+qf-yI-O8l3P$!PN>h)T`w)b9gBai~-SIeu1uY z7$CtLM&Os*;84Wc4}OZNHa_k;J4S=B?*fqg@#W-YO`H!fOEA}^B*GN0Ke7-`#SjbW z{rvm;K5TLWNckOO_VUvFvE#0ac5xC5=TVp9+H`t?=@p4yHM=_LYZ3#@4n3|O%n1gi z-|}-DELAl$#~~k;^(ku~o`}Dvtlp5%`^D3K2?Y8n8Du&+lp%JO_XH;pX^wB5-1q2n zUEjg+-DN2v)uIDP)cceCwH~ziH_7{IEV*iZN#jAUVwcX6TUO@w*5c*e{XZ^zW`>{k z=7kCLD|4Sc$K17NxK^!oR+Djf8`}j5S5J%ytUn*QPlGcss>ObVtZiSmhew7eQTx3* zR6z5i-}-Q$p><~_1Jd8lVhAwdpcjS>TMCc+kSxmDFIyOyP&>OZWF0IqN6 zgXf#ifXX2_O+ujIM)%aPVmVF}L`%aI*J$x~?@0}3-a#T!0Y0tKV=(}(`n~!S` zmhJM$M3rjF`RN525~~akm}z{^I*`Y;conz(LhiFeH-5V#GhhF&uEKw&OgYG>@+(m;K|=O?$STioQ{87O<5c%{6ja$>lnw`_?Nq~HA~+{ zm3KF!$2`?-e8~~+z)1j|7CgvBzzswx#4l6qz1C_rx}vr4KmXb}`#07`y1tpd{L8zpH^T2Z(K zy#Tn@RhHYBDiLdO-v=KazYik_qP%OL_G5kQO&Ux&Z0s^Ard3pl5Lu&O=svxH6MKiO zrEnoKRZE2;TssUV-}4`yWN5Js7{Ou_`~3C^d;nO%Fmbbu#4Kp-WunSdsNdwhtY6JR z0n^cjZN&g#3M`Jv(}kPo(3&{pJPx^v4=3p2hWPi7Lu`viRX`aao+0tc(f?Z~s7)k0 zx?&w@-0_=0Rv!?^I+Y%SGaZM(R)Gj|#ZvkMDD?!k zG3u^zG1Gl*3c_LM-EW4{`k_{I(NL-*#*3sjY(auLrh&LUu4m%4TW;j8!j@cu^=uw% zuElNv(NJ8A1ye9xInWvK=+2sWnwM$Br5%z6e7mlP5@$PU%e+aFTy6Q z;)H=iksi*!^Wsr#ro1}xl(Yg~o~%o>&PA@gtoUBoAvd^fd8qdM=;n+M$!5 zaEE6xQ!ch`%R~>^*3YF-|B)PIY;GS@p9TzS&K%?e>>Ok^MFiiYVX(7(2I7C=Gx(P@ z!jlL51jBAk7Y7t!-^DpfT35~!q`fXC;L>k7=ZfA9SWBW5T8P2Er<8I%96LAfDX6z8 z+dd9)hk`f;M*XvO8pQ0_+Uzz^|19PHYMh4^5}hp$%AfV!N9-iq!wbVS?ZcJ60_}WZ zAmVxIYoMKrsa|+b7#f_0cq0RJBLN2ow;?9P&wHzbq@3nXEiF79!W*6^600$WJl zk0wRF-HnlW`{kBU-qX&#e8P1qQt$(9q2K4(SWN!d)$lke#jRpPf%2u6QrOLf4G!t@ho(_%M zklT>zt8*p|8f=_B{Ea0hMOzqkpY7o-fSS}ai8rGtS6$w3k4;teS^g=XP$$Q?X9P4PiFcOezBIwo z*hsLL!N?{;_9kiRRWTD6l&>0_+gWT%f3Z(R%6SQ&k;c=pdFpXr5Cdm3oC=lW13*pQ z8ooVStS#ChjHJDB$lZ+Pg`$2|P0zR#%QUj;_kvI&XO&mfTzg?4PU$8kF_UYb+a&3} z)jGIO0`yK>*Pe`Ma)p0qCfCNEP?oHQ8(RVoHfQAzkXviJ$oZ{}-N}cizI8KNr;SA) zxg%$<5;i0V3_?c0b;Wq~|GOhYhp~;!U?5XM#B?459t!SaP{Lx?A8@9`e2>WugZZzC zFc@}e6Bd1e4Q47`h)Hx4X@WY5L|&TC+;Uat(4Ua2U?gk0Vk<+gBFvc}6wgi&TE%25 zJOZ$;W>D%ED;ZEoM}i=T$cI!=^(hlyR?V9JfAkrN&Eww?Pdeo+{-QImQO5X|ORGc= zng71615YmI0&x+J1kG9}sok`-F7VmzNQKc?etf{;YB{L;!|t|>EGEpH%eQiM_P!^+ z2ZTCpL33RESGyXG_ojkzDUWi^7r1e|)Si3Na1V6I+_mej>r~noo8^HSg0m^%fMH>O z1Drv~g;aR?P@hPf>hiZ8Z7)ki`}nXi=@}tIcq6a_NR$0O@+%L=!HyO`=i;}dQG;kM zMMtn{RYQcDMfVQSoKm%|7%%gGexM4=8IA}r?i1$Nz`MzonOMl|Sv>Qh;I8enqPqa~ zG1T+v#4)+WOpV3f8K%)HBQ+3SLFf3blCASclx(^PDAytdv~0gQwd@pzAA$`FFv~gj ziKYMPJ~2roqgqWD4A*&x$9wQ9f6((#yNf+zJP^|SW0e+Fk7RTNat!m;oYp$;`OprK zz?EwzHerUHg%_-~1Pi4tcpt+98q%oJaHcV3e9~nE(2!;Xku6?_5htH(@^F#h7~7bi z8{(AXH*!c}4M@3!T`3sKWwap?ZA@@$jPOpePgT`%DBeeO1}&x#X*gR7Xis%D6TghS zrXbCKOL&*@z2U-OMi(6Y7mkz{iNGpgt`j>GX=c6MfA&wF{-GK1LEur06l=Er${KsN zB|vOK&zW)8lr@N(`Lp%2aDTRui8VGV0>e4xKbNE*{zqJA*xdS2!M_k}Bp8zZBtX=F zeTEgkJk4!CMElondlpFk`cu7Jv}f>(WYciJ%%-%(+Eo;2mj-=H#qKoZ>w2T=_gTE~ zgZ_pWhB=}@Pnk^x^IhRB_C|DNHB>s`scx_SErr2?ugIm;ZzT0t370hGcU=08hPLS=D zoAS=EZc}LpA?3PKPTJ-neMV;_VD&TxMOBv_hKY3LxF;rZcwKwe|70v6f&v2GG=O}f zUowajJWVPKdsol3C)&l!6hvhh5HBM=hs9KN!njmFtd`{hGW&sdn=={3jRQ?g8sie( z>nhea+}+M>js#&fN22%wX7OJ<;6FPV*3rplC^64GizoQ_cIL{lYYzZyU>ynu6TFPg zj>VW4z|GTT{a<43W(W3z$EKS$ASvmlT~K-v8s(6U?3B7nAHwwqxWdcBJB=n^>%Vmp ziEQeldVkTPf5*1B;CInN$005E$_y+bws(@}z$#~`}XW+d`N6K4)`xy{n-9{#Omt9NQx@Vv)@-km7{0c{**AToDd z4h&XDj?PWl5?{J1CW_;6TkN5}!FFj%g{V)k6ZRZbH;+=)C-legYxZg zM-KA00oU>_cC|Gy!`WWBGcsI>YXtVl^?zzJ3S@Wb(bQ70v*L+=vHHgar4QR&1wa9g zBM?x41FufJnzbyv2v9;KBO@;E@&+whGqci15NMM7ImJB$#Aiu88?4#yw05QEbv|su zkNf;Ta3dhLE3Um-*6HEpgcRTq`37y2DL@1d3I3=ocUViTaBI{S7VCo=R)EqX)K#Lp z?KM)OM}VOWBP_1MDqlbPGVlwc1K8XTUpsmFjxU`jJ(8}J=l zJzzay_7VUI9`;)azWjedf-h%D@GPF7)&L`T&>E?ocA{kJJ$i~LDBDb-5Y$+6XHD^M ze!lC}bbR~FU_{d~M}4@XyyY4cbai-7v(qY-LcTo_+GSlad_dK{Bg8CdB3p}}rsS2}W%*8#jpI)B&iLmJ7yLxBn8V(ekZ}Ia3 zUafr>bd&`K@^6$V9sFtb=|cwe1bg!F&9jV zfg@mb!D%XP5k|a1O8)1MlW((6UB`_2wVKnjhLE7=7RWKcHy}Y-s+Kx!IZKmO#Cqf8 zV;QoHo)SbY&@bi-H!6j!@*xH3*vN4zVY!(ce=FKwj%o+hy@N(mCCJ*0YFMufZf@yw z?RiXZ2Gy{r^;yz>U^fhmY+}5{pE=+^DU1zcFH50etv&bztQ|am3J{OLcYsW|Cn$wQ zs0s!XfSSE;fa}w@4z`(yC4aH?V=%tb|H6jR_E#H57uYZ+wU%8cutTI@eFZv||IAl# zX8*38SGSzydVC7P{e9Cmw_q=yRWkbe2O%qh0T&hrxNmI(!4~rM59TojpCss){347kR+DJ`DVgJvMr>dIxUG6!;PNfgX^7vfn0n5ui*bBxGg+XLe zYNP3rPP$-H|6uB+OPoMEjSd0YX+dOOV_P5kFO4{2?^_D(7w z{3h9Ef1{Jvt6SFVUKvT=Zo`C+#Z7C-(oJi`EgF@mj)_c;5Lti1443?XQ1z(gGWJ%C z9Y8nhq3*%S*ZD&y=sg>!e9O}n*BMox8L+uDi`%n^72xv>eO7FOTmUVwAE3I8l~uMOBg6; zcS?~QFk&p;6jPd9N{|+5(i>spn_?1|D6zbF+XcGdH#`4QV(-0oWbd=U@c3Q?R300P zKKk5UTIHhjS6~nrR#}AEd>0Btr~V}^ox)p9J+W!ED&Nd|jXTjGdV(h+1cXlStR8qS+-s#}QN4neGj=uA_2=7pq{8*|S<5YAi!KtEz z2iQ|z9hYf`G^om+P-ng7pZ(0xWm-YE&WZfsRRI3GGD(2JGIJ-;dJfD7C_g(~6%fuO zqXomT_1RKVhk*vB+Ke1|KMr{^sHzk3CrYZ^fUQHe1auOH z3qi@QH*-`}sWYD3FBlVeT>q;(tup>^xzl)jnqVsp1O8uoUji4?+CM(TjZnH8WowX> zM#{YSt{*=ZbfJnS7fx1BuSf+N`s_J(ne_+k@hBSqh-F#RIh{GvdCr;VdA`f@-5)|?@46T}|LET6;;J{s(YI}%uv8e`CzK#e zJjlSOV1oKq(~8AW1yc`G{E_`ie3&`!Zb#f~O)l`TD%o?kn)NxSlt~eGYkUuJVu@Z4 z-jP*l*{2tJUt4DQES424+if{$yOQmHYD1bwtA$X@s}(Ea_v{FK^ggIzZ}=rNVg!Y; ze*s4xP*t*;W1zga=^!3Q^ph&vK)DeIvjFRGWb?wx7>&8HH>&GDb;tF@&?H1f#9!&c9WLw zk1W%)CA3Zb=>;eaL7j28TxUU!z{3!eJB-RpQ_jt{-8I8WFDCVaAE;1PM*S#T{Rhs* ze=1{EQ;>8}Y{u}IO&CBM{3OCfb97RX`|Jawg`srDjk3UohKf^|KL-BioRdXOjqQip zd~X^XT3_$!*E|Ek(Fv?g;D1RPiMoJKUAIDBlgmx03#iaP^UJz`o=9Cl-qbKCabKty zUH`8tMp*j&l=x(q)435R#$lNUo0ZLV^0W$TGl~yXCnWrSAkgaVd|6}%n_Sl&ZHG3w z#d%3pR!n&xlQn=lq(_eKi>XC6@98+?gd`t~^LrL*@IhT4w1~Lk%vSIsz29=Sv*emX zw@Npt$fQcASuz8-;%2f!W5|nZar+D!W)?R;?re%uP--#tOIay+b+(Y~%_J&^E`Y^p za5Z~LLuA`DUxqI_K43%1e(yAJr|EM(Pd z84)_3J7?05w}6OTm@O$-kgn*{<^Q?#9?kr>F%;Tezo?p!UO~EFHer&h`YGwc{p!Ux zYea6C<6cqCANae7JOYD2_yb`sO8qkz$Q~1*UGTX0IgZ2wbHhDQdE$%RQ8DeD+YX+E zNgcXw4D~Va$N?H#Brs(9Es}-}M`}nK38i~pn=MPbC5G)b+xKN%y%#@V`3ySjbu(Ii z09X2DD!9{EQ^C1&xVrkt>bYaJ!NzOhRzo<|V{y9Htp1YyEcz4COgrz2`>Ueq$ol<&z{^1v33y#!xTn1gcL@=qIv;~q8@%3 ze$n{s-?-Ik`%G(AO3{`ExR5R$JrkQq{+Sxrt(n?yeT;drZ4ad2Z`)NIr`E4^dq-jO zopzP$^a8>AJIXh<}&2C~EzVj3+RsMROmzfx)sR`K&V0wEoH5-DDK5T@6@Xs6Ro zD`^KOi4PZjwbdg_^NpviKcW`2RqNApsp(F0PX9cuD9|}X_B!OiW57h=iGWo${Mf{~ zV*;JW=-f0taYOo4mH>E)!yQ-94@BLvNi9yXD$%^G*x%^4i$y@{dq+Wi@BByViy)Rx zXXndAFWUG3(yLvBrm$X*icBe}8S6fqQG0=2PL|S1(z>heBm!RiR+UX1^<*>Vv@G{I zj3uF6da!$2c?)j+3fDs2mKG<{1`kNo*wR_V!xApK{F4{1IlMqU=o-%}foTW-j!fV_ ziuls#2m3s9b^fB#E4M8u6LG1rZo?`Q*G5K}fSjK|x34M_!H6=kovUZP97dUF8Ah3i zxEDg)YhiurM_K<}p|9m`bS@|oFLfxH9#}!%x@Nz%P-~#o<@s|v7l1Rw_<|$t-SY>@ zy|%*Zmh7%O5Gj~P?$$JdLTs_FbO^({9ADViOv+M>?SJ_!be%m^L1|JLpPvfqg(FyD zS+UWT4c%w$qqpWBT0xn1x!QTmOA$*lk$LISmP{XIZDqpLed6NXekIzMmP`7m%68#E zf7l!}NPpnPf`K?-tVdTEXDzwV-bKsxk{bQ?^>H%v3ctd($Idb|0ul`CA*X%4tp9?} z3&O^<@!KWuyQFpzPJuXB1GX~UUe?Tsa++QD+e&J;O%y~5_fsFZ9S6OG3fm9IZ+$G- zbtN$Z^p4eXl=_ekJ(|qh0uB4(1}-*yOBdv&SaB(YDgehZSqVk=b3>E_Hj5dgfIur~ zG7kNKQSlMSSEVf-&f)_M3uGv9R@|21OEl<=M2QA}W)hTW=-mvZg~&b1U+$syQ4TB7 z@R&V02GMd4y*D0SRFHbMxU_W0kTOez|h2h+eV0 zgz|&t^rNEw6we|>5K4^XI(fi17OfFES+?PL#-!ci-8Fu43qNm!Iw2Jq9y0Y7E88~J zeDqSA_R-WtwQ)1?Tn+Rmz=MBV*>I}<+Imo-yYt6!t{A#=P_vu-QAQ+bp>*_D*guH4(%Yqq5B)>T@* zLV=5jLlzR)ttZ~6S)A0&Fk5=(j+)aUl@~Sr*I6C{jpTW*hRoh~l$oDPwzwpp3SVa2 z)b+wO*O(Qe{gBjhMr3P&#=*{-b9^pYfGK8HmrhD7HJ&*BmVxfimlq$3-o2w~ylLUB z1D(XzHXV&tw{O?(kEaQVA5}jl3=tkT9ZB>Bg-vN~i@hXzPHSEINNQEFz54O3@hb`_ z<-3V7`7#@s$5(7wb&#f;a&VVry_7E!$6XDS_rt{~(|V&*)=_2}wqrZAo{*+BxIK4U zPp=TvVN_n29vWt1_S#gld%NedGd)2?)FV)yIo2qY$9$m2-`ZjPXYla#I-sjxQQ^5` z0CIv>T6@{54IiO#o4^IMf){;dnX`yzD16nt6+9B=EvYux!=&r$<}KyHF(oL<;WNXT zG>d{AqEM@)SV1- zX(K!q*Ozz^@nKQ*AKN#RuKYw2sw~djD{`_YIAK*;_nQDOiL&yXSp?=(a~fm54>9nH z)q#ahdjdWLzc8&h{WIaP^`8m{S`=j3JZIUj9zva(I(!Wh7>6mWgLW< z?m17u?GZ;)pZ>~K#Wvx$;F@kkcj#K_o1)gx4{7LU$DDa07Ll(LOa*&DR5n9R~BX8=Y*`qcW^h}qE zVV>Wp)O$F&n%*M)c?UTswQI)hkMzv!Rx#JrX##8F7j2(qbDy1CGVLj_0Sm?UBF%qgto_$l0_$1}f{fZEuwEFHcBtGerc*9}0 z$704-7WW5F_cwiwZ(oGRd~1LbfK=$Lp?${CX*-*0LE^xiZaQ2#9t4WW`Ge(1xUq(R z+(@lRKFJ7u2_PKx$XA4Z)sYbT1DFZ_TIg5h6~-StaBM7(*+_;`6y8PaRb0~P$4Q0q zk@q0uQb-tw21u*P(_{BMc7_0H$->(TE#=Jl7Gll`l434_pbXz}Tti8} zWHxWUGU&OTwlCeR8#2L-yD;=RY~$j$P*;ax>G}JD({`lhl=1l~0{wIid_99LDz6K8 zUhaxLOnmdD49ewiM>_rQ&}lJakl>4wX4vbT+|_zh97R<^w-ZJ{JYF>kq;dM z3JJs%42LvFDBwxo=va9^01T7a&BcC4Ke%U}=M;T8V6JgYACxyX{)(L@bVTeleNg-& zkn&YK%{9bM0|W~b0O6H<{&3#xs7geJZ1xqiT5!ZuW5$A>F$QJj&yPZO;VsB6yoHxK zyZ+~(m$+Ym-xUJCyWY~hEB~Z2FECm7+y+Klnwj)b3X9wDnX|2xqQeue*YlW;cC$ zOs>g=3+lIWV5ePh0Xsux?>CKp!%<4c9(EK0I^}?KF!Ne440N*5(eN<^xR0&0+c~Te zeeD;LTS!{96LRW_Hl5F}oL(9AnS)d6CQm^SUHd zwgz~mUZ@Sol%%~jhT<$!-0@5@z4H$1@!cOA#vhXIe7Hb2xKEw+{r^5p z!${-v6@&+R3_$}Vx)DVfKrlEE97GXDhy8(B4IXouBD)#l$ANadMSKF6XP}Rv@9;0+ zfKM7soN)}5uc|?tM@9|W!O~`ZT@AX<%NE2)Pl+=*W)`y;FQVTO{S)T8>M(d*t^^9JUw_W#nX`A&&Wpc-nT1jzta>3=;MVZjI>{W;P1@Y>A;xtG$ zZFvbBFJu<6f;+3gN76{M5P>~V?}W76y|=ABs%5@ym()rvR{NeJsJLBGngr8_z~sO6 za4o<#Yvl8aAb7)_S9EB6CUh9?Q0WduNd8#GbNU(*!y+Aqv5=lxobf~d^F704ITeja zmXqeYoW7ubViIb+3;&I0I&aEG-UUjti;xYhpG*zE~~Ydf8qT)lSCyy?riW@ zRB86F@Ck-!t8?rR=eVq@)LX*`{RNMN{P(<1E6F8)+`fW!6D_9(2^2^y?Vq0B+T~R! zUCf&7c(bHpy;gGGq2om-BTe<{ynKkY*~In?4q5_F55|X{2tU95Aa2Hj!gpss1W28l z@%}J=^Ez#rtAK_DYEq1}0i3~`L$TBEB&vOO>~HsrVzAVDjWeL~s;k$ATj{e(x5^(6BB2)b&YP)0lr8e&J@)soY6Dkv`+9Bk{NC z>az+h6HeCrdGO~2QN?dRw+Aov^U2TaWhOvLV7f`pL9DlIV&zUJxQZ$DN>jfC_0q;P15xk(pDk{?Uv`FQyJkr-hEWjxv2n`j{PY}^pC_mE2|{3J zt#y-#7eUZfp3$Qel3m)Ic89Pm%uYc2O0OVR9;-g4pzgasP!5@8SYcJVkY!7WBN}1I zC5j*uzlwq47~mh=LRc)k2RI2PTJg^ew*C!G$3y}lxW@zI>Z5N<+=q2ets4pF6!gU= zJ~BS>aCo|n;xpP@zhC86{4p;rC2-Oo5g`-BVshqt842E)%K+76-i+-5bz^J5$u(ZJ(aTwEue*t?Q&T%kS3&lk)Y5`IL zl6}E_mvUy$_-3DQZ%F79OM9~ONH@+8Weu0)`WhCAflcms6a$)sFwOp>OjDin? z9T&pza=?yZj34k`BB$~?m)AA_Yk-ha_%L(3Cr?d5+8edlm=-bsSco+GBiNAt}b z0mt6p!G4$5#9G=Q5=QwInz_h@^cOx^F2qrW%bgcY1Ve8R6|07}Lj4l>2I@HV+Jxa# z!0;nr9RA)pw3zYpBrFI@n!Oew#D~HHg%kQ^D73zCE^*f}^eve3H&_7Y0vIkZOhBA9 z(S1|DB14Zv)dY0)_>RxyWdVGN2#BmZEL3<*8DrJ+F*F$D%^8drL%rgBVmkcDIX2}u zQy&t(W<_!v0sAHwEx*?be7ilO0uK$Z=9O?pp^4Ho@M9k3p_~l^8$b&sT$B_r^4-ox z0p3$fxdJ)*Fg!I1V00?}wlfA`6n{||0d)Q)83UOD@BYGB!3F*>WDKu|cOzg2kPk+g zI0L5iS0re?Q3;x}RH8Hbe>!Iz;2BThb*RXM$kBY#(qNFFt&rQ`rNSr`ML#6u2xt<; z6*xo04%#Q)yWmBRpi#1_b9hf?*1M}vRJE{NM3)c#40p`Ao$6YaKqd-D- z=^A_h4Fk0J)E;<+)?SWsgf3Q55o`)t6)>mJ6TcSR!bzFUftmA`6_5EFc&OoeK zUs5QH6rnp54Crk{RT>QBNF~m?i{S)nBk?cbr(?d=9bSN73iq`rhxQxJ#xcsF`DT0Y z%>OU0GY+!rX2BElIs6%Y?H5XrG>12!&_p@0fhbpMI>-JG%9VMj!9@uSN7!8(v;uT0 zoFE@D8P>+p(J*S`_|HL1{KzRBi11qg$6#7UN(}?s1&Csz@^%gqV2djH`>44BX^H`v z{S+ot^PvM92aq~S0B4L@1B5A~5a!asUw?r?7^ZwEoKP_FDzoUYq0bYMhJp@-E=uS; zZxZ?xn{Y9(!UvWR18*?C(KoF(Bxe{$VbrLR1Eq)m=uG})kfV?%1asLj&@W2D@PSY< z0P)|%2!7#~fJX+u4n+&hrSU+sI_hg8*0NC%>n|YZ&(HWAg^@}eXbdJ#47l(C%#}~F z2J#%-31IKY?Z^K{dqUw2L{2PH7jk>PW0-)^LhBJIi_h2h!Q=Q|uH!B@@@l)Iu<|e9 zETn^}PJ;_EQSNdKgef*i(?ubT30FfMOT79i=$vxAgd3T}ZslPmXx&j8$6tVc8AtfN zglKdv&@LuYp~4XNhcMb5&bq+{HozM>0P-kuV9a>Pf7xx=J90VuJv5v@#kB3y`I7=I z0+f?+w=ym7ncKKpE)~pklHW)$7zaC5*Zt~4eAU<>$ImLh7ZT&n?Fd|;33gn*g6gHvy*VxQc4470|}nF1V|> zcI}Ma5Tq;+4cq%t$JCt-W4g;Olt?}IIYDII%1j)tr#5pn1RYn6&8;SQFH2T=DsFiH zh?ApglLKyCey^Mu!L>lcPPyKY(-pMaqv!wq43K|80`9rq5C`kLNW zDO$;7264Mts=L;_0i07mS_D7VQSae0)oJpNqMD7fD7P#<^}01X8$06S)kXRyzH}Y4 zU`sG;lxJ<-usk(j>1@HwbD@5{nlgnm9*Y^9T0BHq(CGO;ngM+K^)I%5S;}2H&ap8` znk^W7J@N~|1^83qUolW517FKvrev`fTTD2PkX#LJ#0YprPv(Dg*Gj(*RTW zlz+4doH5k`aP8p&+)ui4(YuANo|nN1U-uy&Fgpe1XvR{y?8}(q`h|NqsVnr zJjn^?GvE0w9Kb#PL03e}M)7QE|3dvH?-mh{F+*kPgUj4&t@xV85kq%@{0SNE%|nKVDDOhM4}{cI1`)8@YZ#wL;{ zbzGl9^?09#<&?zEYco{3btbME!2MPl?&bv5+03P#JqHbi-OC}J9{u`%m;vJ##Yj*U zVv2$T;0{0^Jh^-t%kcaLWhQ_YUkR^6_BW}GJ}loPUumVcG2+-tZK$1>oGwG(&)p<3 z12#!~qHaj2vteCOm$@uVaSwsC}P>Y+qrksXP6s6hD)R?>vFQp-LQ*afoPB#2J z)XwJ{Nm!B#pU|r_h^-H=4h?232p+Ln%pkuKydiX^+FiMIw_0+>814*e zk~6b*x_w(JCCNuSD&WKdC?QU!_Gml znNHOt$35#9*UyC<^-io?EH4g)YD{_(Bu#n)b;UN{e2ucg(evLk1D`7qQ>uRf8##xn zU0j?(vFnS}3GRLL0&RuLJj82^DvzW&wl??OpYNu8Nc4c+J9i#sS#&mJ3b7_QM&vvp zW)gdeYY*slz9K;Nb*1XeQcoz0*hv;~+_T|j1YzfCw|_8nD#AfE3(A)7DQuyycug4NDB>Zb>AZo=pE zElyckLxtFf)<3Qh>R2l=Hsfl6-C5F_Ld(Jq7orPM*6T-5!u_(2b~o$|zaCS>?mxRz z?D33#{TpY1tcDp0F+MUh0E*|uM>{ci9Rp5-c-|3?k3T(_NlQdH62Bdk)8p3 z`*U6%_*c9HTyGMJF+&%$*!_aEG_RB;i|LP6(XYFUQUGaIn`hD}UXn6Q4~J>L#uInP zCY#lN^4EmzDOplYsemF8^voaXd(_|TBqv!2yS@<=qP(%FvPkL@$&NcNDZx+(aLaOp?lOzmy#%NHByY@1w+3LFKv)5MeD_gmqiSmWxJP; zg#tL~DvTZphx}N@kPYGENOOCx4B+w_$%cKPH!Z%U7OI)kF5mf-w&aq4s1TuNZ~xZb z$s!J7`sPhbH7Bp1EgSzNLg=OT=hq&P!%nK}Kw01D`5&GE@=A6NC!f(6?b11p zp2TxP+_u_cUK?}h{p(eqG7qaNq^=xqCpnqn{-9>%MiWQd4Iy3bDtFH;SOeO11n4yn zWN23e664F4-7spYK9WGVPo^<53C}VTnVWl;)m@QsyIMddvpyQJ^%AFV4QD{RbB^Vf$o4@ zgM4rfKq_QuVulA=87;;XBo3GSwa=Y0}>_Gn7>rY_A{OnNhi^?+EKfp-;bBl)^sDo<2h z=Jnb(X=_-O+yJg{Ia9uKx#Y)dV#e3&oX2~ep1znaX(eGry)D+`3Akf36x3-*)j+`iw_iN8I$5) zbv?HmU-9ze5x)SvJFKXVxHxybd;XPKjH5wrw|*~GzDAoUqxbgLs-|_$0sXbPEwJlM zQE6hqH!vYQW;@y$z8P@s#WG{H_72KtjES~XX1LW%-}}*PqUuVF z#Z&HHUZ6N{w>opqgB~H-+XFaS<(SRo>WSrk`nEG_SmUZ*y4!bx_2rG%^7K&6ceArK z>0TFX25_=nfvZdFGP?`vHSm>Y5Algae7ed5?^mexy`pW1Nds#Jj%2|eAx@d0Z`iOU(85u@kM?G}0@ z&>R3IozeLO?K zJAKOaem|PCozcT%?Yl`z2?XOKxjlAsj)nVVnk#>6b2!?cVP?P>Uz0gm+r6r6rt^VR zfghi!-kUR_n`PflZT^K}(fD{80q(U#S6|uf zF6OpU@74VQoEK$cpMIMV-ApUtlhVuAK^s%Y&m(UXF;>r`x`LOB?A=GSjPUeVC|ly) zm+WsxYVxhhuLN(oF+($(HOA!j@1(MjvQM@x{&T!p7o*npe5&lwH|uP$8(o9`n`U5Z zil4lE`G)q831=$QvOOHslim>$S*IfN=z7gF?DE8mRPeo<%)&0<4yx2F@7o)GZvZ#0 zeD$H5;cwT7#oUj$YS~koKY&Z{4Y=Nrp>kr*CcT6Q-hYB%eq`*^%pd^`e524!A3Sce v0F(tLwmhBhJHbF!N-KO7USCY^#vGi$)o`CJZo`npf72BH|Fv}E2j2V-eUv45 literal 0 HcmV?d00001 diff --git a/images/priam.png b/images/priam.png new file mode 100644 index 0000000000000000000000000000000000000000..b23dc5f0928081e81771dcd487d3eb2ef2162de2 GIT binary patch literal 79546 zcmZs?1yodR*FKJcfq_bi0wW?_0@5%Dh;(;LcbCKriXZ|~(hM+2cX!OFbc1xxLk^A9 z05c5y4$u4jzgNEXKWkC5IOptppS`br?Q7pBL`6yZ9?=sb0s?}2vNCVf2ncQ;5D;7? zx^)Bi#L2=y82EABN?cK#fS^3W52qRf{Qb;KMop1`Aefkd;4AQj2TuCn0N9MLWpSSnU^72tfGiu6e$L(m+7hcUl@Io2H!ATf@^;r_*KR0~@hP8vF2g~sU zH{O4^l0)!PcP$?wzWwv{3r>CtpN7ucr(ttbc6EGDjVsd9~jxhr46 zh>sx5>Wq!j#bGVuxqh;qzTT5$$|R_2-g(0@KmGRB^Jb#s8J;g=N5NaAr)uj2?|MUR zD>sPsd!YnEHCLZ`ofE@WXbE0OLF|ybZQj-k-XRpX)EvHs^YHM5*HOhs*QFyj{Dn^M zL3Tyyc0G{^!Q0&-7DO@E-CbsNm}QwIz@nmYT?>Qv&P7_!1^$%o>pbZUS^gs^qWUJ! zy1TAPK(*1^zT^Y!SEifyd49Ogn*#SZ zKJ@$}RCjy(!_9ZszkWi$yU}!I|G{+=TJAR=muNL*?r?wTeRji~xLzEr@{IJpl%&f8 z;zttogXDR)Sb|u->*YPfk_!V}Fw z=ld6KxRBI&H<23Mar$)BQn@>M{K!ufamW9fSOlC=hQ$4rXW%=Dt0p|(0|q7P-n^H2 zFZJTlXvkPA!SjT=5FK$5r6vg~dIVip)PXEVmdv-Psu=e`_cZs&gHQzqyBO-Ohvrwj z8SA6{qE8b~wE~`-yqKr|89%|KNr`}N z*<*dW>AMBUBuBsdsFLJKjX34uM<{hECoL+zSLjw~R#=%!Ti;TItw_md#0@G9ezx98 zVB&Pq`=wi9CfoDvC%v0%sB45PRlP*LR=u$6&hpxx<6g*KBIN z)5=uH)Hi#pt~spDqg|=Hk03V+`%YF`tz4*Wr=%|}tl$gt48~+dq(merMJQGFOBz;` zd~!^7Ox<=6{YoRrsk-xFP7$Jmk`OaO%AJ3{V*k$O2d}`pmt~~QBRU#gcxFj*_m8o|L|({ym<&yZ&#v z7DIWbrY3zd4{bKQ_~QAl!btd#6HlfXruZh6IL3ARb&ZgIy-KqYHrjUlrpmREHDcys z3+XFA-1FStFXYX4dfGduIRAE&a~4`T_%+#@v@^3Jf;L^Om|JdQU09#jTb=s)}kE;OgJq#tJ41Xq{?{Z&zxcY}*Qfg?fGAjD)W=IJ4Gsl8BJlK3aK%e-!zG<3$jC zHN)b!%oyce^YK8*y7M8ASnHQVs`IYA|bVi63@>aWT03Wv8ng7kwj zDKV6qlukU7Jj#}`R=XA0io;4TbLFKvv|sc6>WJzRVQqFVx9wG*+{5ap>G!yvHt^fp z_o4(4U1q~WeSv);F#k@TF4jKZW_o*hZG`Jt&S`lujXvzo3!-DfmTCLiQG00zwBruS z1Yvd_p&YdN%T9_xYFP>ts@pb>;Kmm$6b7Z13*+S3Dc3n={mRX7L#~_|;>0xCwde9(OKgj^3g%Qry*@zR z{YhW^8UJKCIV1TfxGs2m%x9!#+-5uu)K1kea9RbfLLAw(kB8qQx+irH6!|COE7d)b zsbTBUiHWQcjo)|q@4#{+q-YvX!Z-L+**~*a7gvpCEdEpX=eW`-6G<&J56Qs2STyhP z(nD$1s!kD2f5&WMk|K^HqWk~!e^h5x^6;AUtcExp;G>6aa|`=B)0hS21z`T3^9KF( zll%GTT+UX*#~TsrP-mCXwrSBTqWFX+PQ&_5=!t2m`c6Hor?!%Db-mB}lSRD_kDiS! z2sy+-Oy_iP%xMw@ZCQVdR@{IYUhBEpe_!aXQQfl7#-2mz^!QFu6VFRhZg(#M*Y`^t zxb3CpEMLwYs~Na|*%~d@WS%ZxdO0$VW*nb)WVZ2Txid+1d-#=LkfM@%cK)>S_8jY#^*I>H5JZIm|UFXLa zwD(FQnv5_FzGMEFUDy);zFUyfGKvP$@qQc!-o&^1t=!xU4vq8yHG;kUed?l`%Kkh( z3f#x>UuUKPXy$Ea8C^F50uH*1uPg7f#XNxzY2BrC+$CKtOx$gp9G_{}I9L#HzvL1S z;(95>#q;bXH}GAE`;|YJ9t8oxA&u-CagFyzo3nTPs5B>!cik-c*T|+`H0Rh{pBTGHeugR*6|njAq;)?wi9E|TTe zp1`O(?(fr8OdHRz#bs@M76VmheX|#g`@hHD9YP4x<4Aq2O$ew^Kw4(k17fYryJjRO z5R8*x_B7PRjO2L65dQZB0)h`suf;(`zv4-(s5yc>8l(2oH8Rj;@s>7qC)9FzPf-(N zp8CsoFxwgzO#Z*e+sA~EFIC};v7iUF3;b&XDONA=A(2`g+qnwsb!RDKXyJ5r z)XtyMIeF~~3Y`&jRV~YR#8g%DR$7a3S)(HPrSm>h5kfkB^_7yr!{6FagTaMv76=5H zH#m4FJrrxuS0iI2bl6pc{yz(o3=IG;6_3#smE#oi>lFA)M~q1zg$I6n2x(7Sskc%P z2{_F_Xcl%M)@r=y74xM#K7IsP%IT3Ng1+8dC*|L#NI$paRfk=IL~f6S_J(4sn&tAQ zYqusnAfs+DW1h>)>|`dwMrrePoVwf>{V*& z*0-(3=lsJRrWtNty1AgZIA~7-7gE{N-oE6~;9YVSn)^-s^u9j4X-9Rg^7�xC7-N z?7cgZJAL-f^wPPHUlCznW*CY53guVI7ciD_30KJ&$>Le`TD>s-AdQUD|9Yt;Txea| zHwe44mpfL%`+tmoLI`=xmCC5?CnLL|ul6NSmXK`Fdw+V45i|j@waCi=D{g7X%~kov z)MYOa7dI+Ax^#bu%yi-w$Nd0GO|FMXO%SxBAxQ<~B^-sqT)`ig$Q9qxA4&LK5D00j z${iQAoc6>PU-|p|bP!;^y-;LQ19HNtR})*-s1t0a)|J-c6eR_@kDNYUg6ur6eqNXX zwt}IUj5^JZ~_v*~#QYZR6WxPu|zC z>KoLqq3n>UI;K1S$9;d^h=V@A!R=OBMN+dOzQc0YOFwG*K!Xigj8%XnB#)~?3M?m8 zi}7+@viUBM>aGP2A9)>{csE>UPrsc#fy-fcg%8wD0$A7GYR)YZRuT5?>Kn<5m$ThR zWiU9Uj7{EA$a}1Vo!N2BBpcTE0X3h{EuEe>@X%4g&vMq-mSlC_4GEi&!t4hzes?A-Se*9KF6r^8Gbh5YsDS%JH zXe(TD!w)7&agfBMYj;s*ablAqNQjqVc~AIj1-tV`DI>h)dBYiy^|QHJOYS0#hvN^Q3b}r5o|XA$_FbA@lll0Y&LrX%e<@ zqAUBrUaYh5CT12fOTON{Hc-W0I2AFWTfwX@uGdEysH0)*X4c zJ+20}T3A2Cl4hjt5;C{RA{vVK#dVz|rtr9bCv7ZY^R-oPyc`}(uSpL~KCtHw2#GK( zZ#8&F!CVTv4JgSE1yAo6mjxTV6O&XQ$yFC)^0g0FAAD>*++nE{*%sqJK-9OHjPn8#)}CtJmZ4zWQvPeZ}o~WV1^H{Kg{LcnfYkr#uD4hBMor z-#_AmshW&lYGMxfRn4C{-3{N6=Kx-IQ)!SODoq6gD;DKUMZ9Jo!6f zUI)4aAyf+%t8{}q6PqN7c9mlG(Ef1(>~DgSQQyfi5AIIQIVsFL7o&z0pgvw|)wzH@>&( z3iX~6g@c2;zK|UX0`aJ!YOF+rovS)ABipGk9i#S9aN=Rs?gv#`%Y`mw52rceG+D0x z_j&{b519RywwcP;AKUeg@vxSBi3}+noX-usTfYOb#rTLp`CHasA*YZJct46>#qB*- z<~xeMrVe++wba6GeH$NjxgC`8B_BKxu~Y?-l%s3?iuRKFK!rtefQIQh1? zFxS40e5>PV-$yAU6EEZdra4&kv!<>15iFCd-aB$PYey_sdv$uez$|B5%)7U)Fe^4; zwQQo#RuIz5>>I1%Wxp}?|68rSzXl+#)~xj=g-dZ`*lF8v>LCYG?~-;W!R|ozed!p1 z?eORPa~52bzN4rgk;Hx5e$apJMRa5Z+3C5N=lY3Jnl{k?z>58sL#w>q4)el4gYLu&Fz9d83yLFLXgFvzR6LUs3pnb5^q{-b|Xx?dF()!>*j}tZ^Vt?c%%t> z_?wU!(`UPb3p~EUMk^*$p|!WC6E1>8=W`IG{z)WV87C?O@sNGhcX)*ao$_vS59X72 zGg-s(*8plKR>p{Xd9B7j`SIWO{Jxo7~RQuL_V=w`_5vOcw8yu8^&;wNkd*QMB3rN4IGCx&x><5K4VK0SBGfKo zacKn?F1_4AhQ*A!oL0Hgx6d+V3#bPtDZcY`t2Hb|I1VGnp~B^@b9SzOU!LIe6++0v zLGP6jqKB*qSvi-2`^{nXO+>>~CSIi|jNm$>e;Ag^Qo&mXh{o`a0YNs7K=l#tL|MWf z=(b*LF$ZjKGZ~(A**gz1(>Wun*~{9)L1O!^%qfv$WF#~8A4W=gGc&-!0}eAS9LizX zy}I-P6W#ck_XcfA@v;2tWix#@9{0{LAG*w)Bfdd?$~-m?m^D@HS6^ zEN#iC3Dy)WT>$RSlw*A}YY>XUnBZKFXWnANN0m$3V{70K%%0HJI27h97kdxR)#Sb< zKDv=QwNcR z>2Q>4J}rCssYcO}9xR81Tg54}hW3=o()SOU^!9DjZ3I+{U(`^?n8QTulXAdWM-E0T zco%m7d`DbnHtF6d%$1-khO(OO`J9cIMw2GQ;?4V<6fS|w>-#E${J)YZblP|rdrzC9 z7Hp}Ng48l9O{_sV!t-6Ug>Xsit>@nV0al>46 zMQ_&4?yl2(#zxH}id?bbvrySf&*JfQBJA72F*i%z&ggEKphQ58<-Nhpda`*hG+h^^?pl+m&+nyU+PL;+ex5W_(zRmyB2I{7#J6(5A z5iRDkY;zhO_KzjOZ3oPd8q}-RGXDUY$n6f&lK=$2g8)=3L#7>>4FkFI;Flb?rw;wk zMGqMrh$m~PF=LlZ1-hQksvDEpF^sxq$c$XP1T}fRuJ%8UorXhk@#jG6;01nLlvn!h zUl*Tlngzw8M%;u3vV4|3x6fu$H4+rg*OHN0&o5n+GBe%jvB(9x@<%42WoQv4E%}aP zu$}Ern;}gaplJFz@&rgh2(<^%){a%7xdVlDZ*F_XrH$ehr{W6h-I$s;Vhxy%zOeQ{ zPiz}67YCV70SMZft(3K3)`yLTa^6w0*}NivSD|gLx<`w&(vWZ!Rg`?T7)#TIC(c&C zQfjI09{L!G@Y5G-0 z?Ja5iVt1C~<$0I{__x1fWc(KCA(^{43MLc*_xKW34qB~uc*Q*a4|?Em1*&dZUA)7|t00kx~DL0&w zE?%CwWGo-i<`yeMZb`|7tYMZgVQ->Swb2cAWKHZ|zb{nsR*dp6J{D$gG z>rMB|%0uVnZ{4ifBr@=h?h(xmUu=}WG%cF-n50**!EwmEjHG!J| zH#b{nEQE>x={g7mfI)cv2WI*+n`Fqc?e>lBT5kF7#wQrq{$5Tb)gST80P~iY2wPG1 zFaL7uDP;=PXAO+C+K4?D$zlJ}-IO;_SvJs>-5hy0yJny=0~|Sh&`&4@sn5v$KK<%u z_vp-_J<19>+F`nI!rq|?t~|->X8AwYrzgTntc!TH-&VI&bc&al$!fvR*gvtklfliY zu+O&c7TAZAb%09)`30X=8A*^-8%c_ z_RuAB=rgwU zjl?sw-DO^az|T8Vkd9|EZH)?-)G4O-YWWX>R5Clmq;+=B^U_=pjz*bCIrXmed?Mj+ z=ydL0lJ?t^ZA6*3nCP_cSLO`_wo6s4*)4kh^du6U{Toy)I`W3lZZTwec%fi5*ry>v ziQcseRI$-Nr;G&NFX`-Gx<@sZ&lU zUqI7oJ84a+L3o!Hj*9HCSpzFGPkw-3R4KIXPtm22Gl572$IKQ*XX2wEVt9(T{+;RecSacCT!Q^q$m-K>%7|7t788$5ir0UZD16kZO7>4j%(p3BLIDZeie~ zgj>vEibA5u{5!(gHaNEKbFMcdp_d8k_!{lArm z@12~l*4?4AyVH?$9#13jW2Kzb2!Itd5U(2@P`F%;8VKjIr z7E{z&i(e~8HQ{vgHW5c0mIb*;1kPQ48`yR6Ca8Rc+Fgkih=XX(bZaeG!erfpijLe{_RxpKPFd358-sUt~E=?I1f z*INqGKQ(aoGSJ`kp3KGq-It@_nz+>nx>j7PZLGa?tCirya7u(HS$#kfvCx52F z8$kEog*@kIuH2a_-#;wXL=4vVng1O~ADS$aEX1e$=DvQaNfX^dLl}sa6dsz< z==7rh!0uF1{m_lpONzu*q7qMWoWF%}6(J-T|5lsflLG7fMXjSbh?l!H1SW+ajyNR)uSzqS!mlr)i z{o|KoP3)^10#LnrwkK|xH2X#7zjM$_Wde01b(*u9PexgcHlmz=Uruz2zM(X6ci5x&a&#<(2QUNC z@$(>)&1RM%Hjg(S%(T7{mVq#$k%D+@503i;@_Hhf$r9F9;;?q6EJ-0z`?eXfrfZHn&LFQmJjCeD7w%i8z1e$>C1N?3r@SBX}a#eWO6bya;bC+jTr~cAg1S zQgu{ip?oo_O)dpdD$jUO>+ZGH?2YA|LOHAd?Y$qJVSWI15f0lfaC^cWg|vw`miNmf z#Abdhq>HcsWt3|84=I45;mjKsJfIW`7^^ur?!A{na(>0PJmxfdCtl!k6J55{lFoa}e6lO=kDdb3D?#5tZ_F90D+zng(jUkbE zXG{nPWt}I9*s0t+PJmwWu_sPGrr4c6OpJJ$ZzSl`%Yd?-UJtcVF7;)L`BigKt!d13 z`$rFb1{!D0!$&D>Iix#{1I zd-vu|N&3z%q1Gl|JqK>4yI?1jvSL}?8|J09SCTgnJ7` zvc=fX@l-#nghVc_FqdsR!b4`oF;{7>suD5_)VDH7X(Bp%u~krP*zC!B!sdUt@XecD zW0DHObbaAkyH&NlXHC!p?G5K|u)_zQ5jR#G@E%BZPh;Lg#;1IBm*l~B?{zIHZ%=ci zoTU*fz_9#1YDep3|LQTwm6!zRG){Kf-6hZ@SmV#J$L@X0 zgi3%Y1eg3$9MH^U6yfZTSPx|S$IZUxYGt*@4%e$=QFM)snht<$S$apm7d@D%o7|Sj z-N21PJd)pRR!?(mF81}(u0}NOtZDo<99ABnI(!;~dVgaj75%F+`|oAvTXnoXov$Gyl*2z5dE9xY_K;Qo-ia{AxQz zzE_aiM&FK+$brOz_(nyZsx_oFN&};?;Vt(fm6h?VzYjC)hNSF|mNv@8Q2cJlZ%uBl zTFksjW%v8<82C}vxeSZ!4{|y@8&jkRN7`d4KIvRxvjD zx0DkCAN@X|JAcEHgekYsDH|Jq`+d`G+ox%3OC98d z`_y&MXdU5JCdTvXLJCrXcy9T|RZy&!jRGyZbwpG9QUXPV-1(70hcpWu<=o5Wh5Xw} z0!cQ`J6Hhar#X^6r;ZnFdV)x`H%eVB<8J0UGqzl1g!^epK|UVMtNydZSO$Oa8xu2` zm{Na>ZF7Gi;V5Qm&P+u2aFG@U{WWJ9bus_)2SS3sttcZK+keyOHpiA(&!0pJ)6%mB z-KO%whJ*4NkO81L!ur3DK+x~}EoH>6^8$0F$n7b%_1H(_tGGEEt@;$)Jz)k^)9Jft z&4bd2NGfOPY1O!Er75@mwtPutj_kn0;MG=CF&vKdWiAT#5SZ?>wF&#g&E~wV_+po)v7q~fK$feK!j1t=XE2@+r9N3K*?n*@ zp(e6Yvg43e9cRr1aqmlzgHN9(cijsg!LPjY|E#hbJZ7k#X4C9AK%-e0>+>cTJLf%+ zh~0e}bFQ=KGb^>Y;40N2%{p+=RypqdZSQNPWkqq+%D_Nk`PKsEQCs#Q1?_U9LPKd) zn>ZyoiJ&*!gHUw@?Ey>kTQR#BvPgf%^+i@f`a5EMVahOk{~hI|Wh&Isn)7FvBHgM> z_Sn4{xLl|u?Iv*oxM8J^IkH&S=C@?2RneyzIBndy-_{iKK#|}qd6(D5pB=}sKS^{b zF2cxa1t({(mO#Pt3HRR3BiJRSGm3}KXs~wO)UFp3W@(^GMZ&K3rP8CDTHTY7KUtbf zwG-iLJc3qLS)yMOxRy<=ZuwjtfY#AOS3fQHv{%&``&E;~9f8MU)HL5lsR+R}Y4y_u zFpAm|N8!AKAz;q!`2O%K&rKXpR=#pAetvqO1k#8=ySdj5_1QwudD{P^+_3$sTYf$c zkchO2gQ06H$(PJ~X-%g$Si6;B)1joPo8NeTFMcmuf~K+6K&jD z2WAs9-hWV=9F>+bC-;$7K^YcP2_~jv#?$hW1L@|X@J_@?~)mY9(lrF1?+r~-BL{UqMpN7zm7DqXh zf%so|z2^jQfV;Ic(ef=x26hjp57v zq#AYxetW;*og$gy-qXiD*#&4mB;WEHud!NSS$pc^dShRV4@v(H&LQ5SHbw_HGxIE9t#FafsRcE(v(J*LFa!3W%>?`(vYtENb6pB3^J6WSt7rOv}Oz6^!;ad*-0h2d9(!jV{Y^z>YiyXh*U0Vg5 zbFIJB1STX~%*eO3s$+~($>Qx79x7CDl-|RI#DQ|=ff8Loz{!jDai?YXVZJw~oD`tM z&;I;871CN6)od0ykU^dWehnkV;H)}=D`^{%%#3KNz??K0@E zfD8T_(&(YHVtEv8@?9aa=#HaEs8`eRk|fZaUfrR4^HIIC94j}iykge9ZVR~(h}vW3 zq5Cnj?@iklm)dfuS3$wRuEl{w4_y}np|%3_x<;=d$+Jc-L6Y6{GOzLYv-h+2;y5Q7 zXAf@edawL}+{!2OED%(?e+HWckeAnMTeHhnH0fxSi(WMHWD_%h$O4NgJ+;&Asiji) z_Ly#D=}5XHraiFk1W1&j3b)sE_AvYb*T%Szy#oDBW2VLK2Ke2umXxLrAynZo;q3U40Jw+9uP4}Xvh12@T5 z>2$e4w8^^PO^>SY9M$m!B)gn4yuhz;o6>&N@l=0mtu}bFzHg~@g~yVLlw^0~&^m%I zp9W|TWAKw4fso}`sSS#$nyuR9FRRDLDH{iLr*bTRD0R=Ppd}~Uv5aHT{y=ZD{f?EC zcg!b>%>(ye#?o2~p{-p{2yQL*MFJaD9pNTQ~oHJRX!w>g|DfR_VZBx06KCaK(?|3{#`VmLk zQX#h>d-eEZL+Q_C68#t(T(`D~;V0o3rZBij~iXa-%HSG$Z+Qspz9;zrX@k zzRafBbGI<_p-^^dkn@_@{pwJ)jlU6&z(Iu&LQSE_ih`~k+%)}-jk!&cJ1qlg68?j?}(6KmS+($X_l~W*gd#U0-%#clOB$$p&gl zV0-o(x?PUARJjmja@F~@&Xte#wMlI3F4HM%ZsblUom9ChmvFbUB)#;|-fq**IPq-x zy+O?_X7G3QW(#)F;;(m-ngqR(9?$^n9$AJ!FC%<712NKY7v2AGMEmi-Q1(#@VzWah znZG5u$uA~IJ@C5Upl<)6fETsT7NPuDsc_vvbTyK%e$|zxbii+E$u+&je9F!4Ok^w9 zZq9_&CI0@jx-7DT{7)hzY3ndUi)h1&x`XPELB~*~-3;5`zI$6Z$jdj)3GI6Fn92ufRS6Exf?uCjVFz-duBp;ysA6Kb7wPV~aj_CPn@gT>l4d z6~(}{yUgnwWA(nCJLJ3;#HG#XKNR-V(h5J!?u+_+ch)%Y9!_e5s*F*FnMb4dgsK^< zpKC{YqqedI^mNMGs6qKnhjGMlirl(?8~K%+lH$OjvC97J!kGcFVc`x#wpOrz}sfpa}H-k0lb21@XUzsJX^ ziXv<89GLARel7eHk-*=q*}fbGi_ABun!i@bqIm!B=R(+93CMXC!;uBa0v_e03tF^o z<}v*;S)tgA%@uZ?T+21Unm=VAQX-fKLa|NlBQeB3op68f0-wAGYmUZZ!ByLz3(%XQ z+o4zE3+sE_f|2jM5=4i621=R>^n$(COM$J85DT{%@0s>*drKMRQayYiw;$C{BiZ2Z zE_Gx~uOUQOA&yk7{V9rEQHsje0e77Z%j&2g=Cv@at4bpI=Sqkc#QQ6cxfq?^NJ35N zT|!LE{QD|e&ULqAV3&;XC(tlFt^%dHr!uW;5ovGBUqAHIDdIQuhu49|X;XFO=A<^~ zcKpIK$PB(a%Qr}Ln`*9>(n{9FgH5Z+Z&)=hS2sT5Z`Y_etI_ju(wMZ>`nzE7Nx9U& zk@0r7RrIIrk^W8Lbk7X{_d>*8Q66VC$Nuw%GP6UniAbKb-uBo_1DNYa!zHmog`4vj zuXymtnRd)m$gh5nRz?4n(VPk^eX%U!#>Ip3Bc~WoLAU5kGiA6KSv);x&V3{WR7->*kzlWMQmF~uFSf^X}_ejCmJmm*I0#7}F8H#;)lv73tl(l+6o z^k+Bptxd+jD!e%#gB6Dey5)Z+H%I?Ucz^oEGF#Iceg}_H z-T6{@EV)tQBvy;Dhs@kNqk9jj;YatAYEt2h3jQ8~eF%7`tCj{c`KeYKCB>|?8)CH^ zA0xlKU;eW1KUr&*x?^ztc0;5etTD=LFE#AnF$uAWD*Q~1#oGdyY|`)x5@mslgaga= zsa?h=l2sC5Wfu$0M?a#y?J#-(6|>wU?*Y?~l{z!T-)kp|cMAd+8-G++8IqZ3iad~l&=L(d1yU)? zX+eUR?(cV@t1fb&rQYrOA=z7iKwPfNG2CuFl*8yG4Aam~K%fcQkyOXuppCRyrY?Gg zUwX2adjD@RS|v(B9B%d}$i^g00mw27oka2rehT=*?vGmnFmIH$HM|IFK`rC|RM`%x zDO{(DE$#u(j!%Lu+mK5&Z)2eQ6rS5e2GOAq@d4GDKn|;jrj;&EP~gemLk1=R9O;4J z4dX=en$8!wfRmr7djfT`x9mq3BV1F}_#cS5fLFv1sD#Xg_U*dz9A7>Z2^7=+8Rm%i zZZKM=CWK0VN)zn*g3L_Nk(Y19geSsM2{Hk>YX0pk)IN#qWA>7@ z!sJnm$;TLFa9LEJ@;{KJ#Tg55&Sn^P#K|F>;{l6zR@rF2T)wZz-aW5bu-dfnVwW@W zB)Xq{$*VR^*X2BC*h-0i^4FN~xTTH2N!h;kGrdTr)ESqV6k&FGj^8G1aSDAw9b7y7 z(kAHsX5ht!{~n|SkpW=oFIRJ^$g;1KS7h(e5OP6XU*&A#x`5Slmi|x8*<`$K{U0D>PX3k4FzPe$&IseYzSzkpRZ@cyXL& zZHfJ-On!t|AjCdJ-Wke|ZSa)c`0RV%|L3-ng`YdL;B-r^Q`mi<+Od6gO{~QX{Kf4l zz19@>Nn6Fuaj^}CJJKD@WaDdsr=hnE^Sy}#06E_gkV(a?G6I0niQoIB zxIu7zfD=hfKa!h!nso>!x%wfm815hNZ9$#ky2^*FNBtYK&D#l6#ep--*%JlEoKC}~ zuY?@3`8*4^$Cmn2i>FFP!U>)&*ls-KR98a`1fEFcpWE5!xyFP5ZxPpDdla(woUAZh zx8fIh!OSDm&rqu}_&3`o4cDS?&mH-wU9otX1O#$RE#E<+d+V3h)-N@O*)H>GJ2gZ_ ze*K9@B_JmgT5ZqN$1EN4@7f427<|~F zCeXe%PN!Pkqu5%%#y)I~h~Q-)TTiH_<|v3MZ1Y-N>~MBjcdXjtfty?TBh%7Y*MtX{ zjT&ePxvX8~x|*GKsiU6Lv}eMaX=~S(wu0tb*jp@F`gI4fFTY|TQaQkm@ri^E9mxor z)=(@JZO1xm8KCzqa#73*oirO_`U`t@P-7;pHfu53j$pSH%6NLj_ZC-vJD{p>bQGnH zFCwbk=Nl|zzK9F}Q9C$8rX!AOU+gCS+H=f*xG4HFi+kA3-~h&mTS$K1LGn!FzHy;e@l@i}Rf6XkHPAB&Jm30Vk`)Si z8pFA&_aA{L@R&6Bed$ESS&!PDSWv9fc)kywm${nzeC_C&ZSmlaaG-{~z5c07?)n*B z+Z}=rc?HGAsf7z|px-Fe{R9brC-RKnhB4Im@CrquU8~vCuvM*dr?$`>|H+!vmD-O4 zAKY13SlS~|$faxX^u*%>H9O?ngh}mnc04Pj`*?jZT~2k0W_$TV{%YSNkan?N^{k<2 z(+sfmDJK$IqLg1Nkn4c_Akw}>FTE(+RZ#kKfcj#i%!)*avDaJ3wI^RFNs497 zN=EwT%osKnGf*NYha`hygb)HO@czZ?&srN4kNm5Gj$4 zpd$xRYSVxC$FqCmr_ohQ8o67#3-mAXuaDUUF2tO@M6OoyBP-9VY5vvGXx^17T% zO#Ze<#o1iS%eC*nOJd&}FpReJBygTwyiLIUk3mqWQ1o%xqdj}PA0$>O+-rPELk*dI z68vBfEvtWOZUG3~3ee6&@&m!Ep>GZTnrSmA5`3Er3jwA^YN+$qA+|B-0k zn&~;Px_GZaHqu`0raOofp8hapM&|Dzj%@9qZm;S(n%c-QyvWJQ`MDVKNo;Fmj)piq z*7$a<*r=C0g;Z_&z{$_4_C=Qds=y}IU`XluYA3dC)$1$a;bM720GI%;I(YNhQ>?{X zn8t$ltRKR6px^lGh;ho555U_e1O!109`s;JTp;~Y?!i%=Z4Avlp+L2;*fNTmSGjw(ri!|GWhV|b4#xZ~jFlrezvvmrHV z(yPucENlGIpL~1O*-U2fXI#tYs{)y)5C#gvPt)+|^4(*>UizS zq09VkQz$IMek-d3?=c&a=C>2N;YUY4H&9!|Lo~tPT0>cBlfcwsm1q~;`V|=~- z_C;h))mcrd$QvLD8&`tqpA~ zZf{74_Yv^lC%?XlkxH3qX^sz^S6S$~#M~sX&S%|QLw79fl4$Fz%@o5cgBueExRm9* zQ@>Ck)%3;uhniYXB^AKvCL|>xd3Yd`bGNFhTt9$R03pEO-;?gV-|Z51i`F!OCuo2{ zXvZIkv5(e&W~~+^o#H+CHO_v_IqV5)d-~&?pv`XhpEYU}tIqN}R2Frp)4MZJ1hm52 z1YioH9 z-t53GB2rc!Y+&2QKf?D4tfFpwPJ`x*>+`wr7p*4a3bMAwSh`$;(6Xgmf zs@*)p_NI{|opq>9P&t-%>J`Rj6rvR4Gb5TkQ^YbTBMHf3eqTVVh znq>e1qs%6M`ZjpSqjJ?M^sIpa?0Jf`TSptK>go#iD_$dr@{J$W*(CiIR`*^v801>% zoe!po;T#*|pEEQv0{>hhc&Fh~(v23ek~hIi5k>E3)XzkL#ujnDijs@?{gpnL<+o2O zM&CILGalw1zi7FdTZBDY(eI4!4rY4NJ(^gt-C?#Zfr-OqAnu|$mjob!)3~E*r7ugm z*YB5&7gl85W6>!aSQ zD=I1mbzQh*wd17C*Fkd8p+e?No@n=B%-b?<<~GGe46JJ89jpxGAwsoKVAIGa)Y# z`bm+6J1Z(A@hd*N;rq`M2rmI?(>fOWsY7oUxwbOz$iOXk5QDkKdoMh4AXV6_qRS#1 zs1wm1Ltmsn#!xWA4Z>ks6;PhYIES!Q_3Zh|(wUMyB(+8s}V zo}muw@?SLpnf$Jwfjjv~-u9?WtwJ?u(=)nzcqmsoxCDS_&jyOTvTMBhf8K>_1F`W| zDHm0e^v5!D@jdH$T_+H3KGeE9i8`F3b7Y^W$4kr+drlQ^6Xq8dsZxe!wZ0Y>PE-xP;IObYvtTt%3550|<6qwq`r?H}W=K$JPd z%=emO&TH|D!ys$_wyb?M)Bi`-SBFK}cHfSeV4;AhCFH<6EBR+$7A!HJ6bF$@cGE2~-V~~7cB&m=iG#P1 zdBt`OofSnNMVl`|9{KDbGV4nI9M!FQ=o!-<=kyNVoclsZO?kK=;HwPM; ze|)n;QJWdVFbp}<_$Q)s-^OELfzA+jej{0yM%&du)#+iza%cXv&DcWx?E)rrWudd^{zz|`h@Z-D&|Z*`=-;Iqtu`Aj*>2?!#%6V5D1g2N5~wZs-<0> z{$fk+RgskN*fh*;l9lt$Iez2aT_8tO6j@IIgW=zPg2Rp|j1bpWG7cQ5oYewJ(stx> z!Pe1{{^QYf)j^4br8Ex&>@2YAqCk#E+EGzNheymy;p#gk7&WAA^Yh`}L6%@W5uj8s zLlMoYS!6QH6|&v^^!9gTVarJZ#xtNE@--v!hf&l|%AAY)qRNVZ8U(NvXWzD}(lFop z|KN7q${@7HW@|=YAA`Wo3R*^{It>kQFP?;*K6QVOclWd5AeF5GbpX8s8Btfq>=-3$ z;cs(e$$cpk77hcXF6{mUNm0+5kYyh?S2r&E1{tVk=a>6`xUoaudMUf*xvS{6y)w`r zg3s){H2JOlB-r3u!<=zR8<1&odMts+T{!C@3VZ8`m;W~$U$RGr`Mr8?kJ3H7(?szY zgHqWppFe%ZANhMe^c>`)s@9Smk5ExEjVrgGl&pQ>j?_iQ3qI1xW53?VMC6A&qL|aD zHExQUy{;R$0sXblh&&qy(pDxNtf=x^73e!u1bNQI?mD=a@%4#tZb}`BJ($d*;|o_& zB^TFz_%ULSL;d8dV{)fNlA~W$)a5A}Pvd>ui|iQ3+NIQdirVSRh_x@wx68e4!7ywR z8q5->|H_^A3}}tlc9i#l2Dn++U`le4QMr0R`-|tZwAq>h?H>NihEUK4$8U%F_RXwo zs95y7B+UaxxaML1Sjou#YN*}UG%hT6(+-e^kxB>8w|8i(*<@@-p9$>9Lx(lI(f6q% zR8PWh9Lk{n=Cf4HAxn!>q2@rZbt$i;@U+B$+^sy>lz8nYI8K`7uR$W!HTX||C6rN* z^&0W3MeOQI+3PAaANMB;6!&yF9M1$_YLvLHLK>}cdj8`7Lu_m(2$oJAELQ!a>2Y$?Vm*T)9`!E zb@g$xvbzEC$YxG}&UiZ?QAyR)!ku{v^N{U%&6U3-_>pBQpRLy`I+1@I6LMlcj8;oD z$1sg~9T1k+V(RR|9XYM-86c1!f1linJ2nLSei@_^{Jjf=V@0~KsOW)K zYM!^4w7tKK;awTpC*pby+PYI8A^;`%E3-QNk1XwCu7dVTo2;R*rivDSYYS+a*Can~C)Mpt{c|@Bn9^l#o*ElH! zqab49(9D|L&L__YSKltVs#J7P=#^Bj6flHuLm!d4X=Zueu#%nZyr}>yz&z4$63AiCfDbo7KGH#{`rsiIx2?Ui{(R)wau6S!u61nxk}9xz4^OwB$>C&!%RdlXMCJN^1J zh!xgZr>jzr8!u+zMD32MYmAy{8C zmnGgHz|QfTf4_id!0flC`RDS%irK@hay7-m7L!9ZNC&NJpNA8GGQ)|Q2=rV=uWHJl z6oKr$xUW6>te=teUfv_O1Rit6ghyKax3uUkDT zL=w3SH;B*a4PEk)X1>qAt?OnVNEP;EevWHf91u26WbPGW-hw+*A~vFF7rACCz;}a? zYpP8eow=P=-Cw{FZtuTtE)W7tJ7%4kq5~99Y>!|d($1O;W)AXJ+r-bz?wyaFR?Cd} zuG0_w-qiN~=y3dleMA0EPjK>!{**do=XH2lyY`nVMECu%!|e&JaJHyEsTc5^_IY7C zxcv*uehpz(w=ajbs5H@psZuMM9Hf%CCLt&cUayMXLEa*qB3$V*5W z`ZVm8818S5aah(qgHjLJaXtwyKIktfC}2z^hrA^poXwrD&=$!`&fGitNIuGDmtR`S zLkn4`niQYUR}Jn@de4*ekVbIx93J1jFXpisoHiG6?a&4eD)EH8!|#FBdq52|e@gi3 zCH94-i~~v_2nZTK?89i)3=MBK=A#+^eZ|Asuu~pOi|niXNKM?!srV?A{&ly_G~U&o z3mX9+gPn0CENzuefPCGnI>&Sx@t10i)WIvFdT}18fkX?95=A;nNAhR-HE$*f!krs$ zzA9Cg#lOgJ`Kgv>fjFyLn)P=4eaH;p-Qx$;RNR}{lap;V$%GEb4y%zM|HAB@fR+v^ zXAl>>t#vee2vaZV4L^xG5%EkQmMX*ti?6$Yd2i6r7xu0$TM%Y^(7F&%h(>ep{e&P| zMGQk;r)(f>=i|3-ZD%w-2xF=DO0nOgvg+^xlFO?N;lm$XNU}>{##8=vQSMVTRW^dn z#1B1lgevV#JMlK{EV>@rO8Vbqaw)&x6zcs|xLLoli$M~iqCR81?w|G;M_mzv?NK%+ z&EwH1hsYa<)47&=N*(2HSz2qExE2}tGDcDPQh~0~qLTHVH`Q#-eL*Q$Vye>h9<1-7fs~_{E z)W>EfJNHYFC17WFFGNbNHdQZE)>NGaq`l0YbSVTPJqyvJEv4n1Ht$*3Qc*xn8lP!( za##9SuIuRp_lY$<&Lac7`YFFBbc<{CEp7X|!P2$Ny4}l@Y$4(~N(gh=~)=kd)r`L@hA69&QZ+#8IAe9degpj&6?)ad%BJBJO*0#r+V4 zD-c+ALCK#7h!04rS5$Ts-SzVI;UC97JIqA+d+MTVJl~4a0|}o_$I^o8ArTvTM~A;7 zqcL7x72%F?ceLl_Mfi_5`Pa8&KQ3L=+>X2WjU%Leo^|(OO5qm$v=p<6h^8*f_IdaI8FU5H zs7EIHxqEo$&70;zuQ|O5AKC?jy2;si&I@~p6PlGUorD9st#~}#*)b#eRa$f$gSWQE zh(J47w?tY~zy6-1Z5?3rEp#zRJ&3YA|8gcTKR@xEjD0lmz?w83Z~bHc8LRk5xw0ah zeej?dc@B(NP_efrf-N@ug9@~xv569!f4#b*zjl5mEFjjyu9-&u&S&M5mg+B2mnYd* zDx&R1{#>=?dwJ(*;x30@gNxX9kEyN_hv-CYOn#^Nt3AAg72|v)o$VW)GYb#%Kujx6 zW=#{d@iKjE^h=%{Fc38oBuX$m(%gq<+maph5iSAVbEuxJ)E_f8@~Si_kS z+^+Clg#;EseCv94IhRAa7nJMor#ldzrp%(!#5;Sm*~gd$|4Yjm&ma-}CLTS-ZYpTW z@%Y;q+zGGHCRNhzMfSwbIyjK}5D?IR!w^ezMk4~ZW_E-%QCM~5Kly!ssv5=PU^>}^ zm6N5ma7W%oiz9Tkk}HuBN44=qX zA0&X}``G9@_1WXd~MxN4Ijni^(ELj_JCUm04Z#CVGj zgSEGVWx&>i&}ZZ{o;(!m3Xu=I5~v2(u&Su8Q862HqjNQ`D$J6d7o2*FjL z!CZq{S6-gd{)x3#d;#2}AIA~PK7E`dV;mSGQnuMSHaWQ#tE#oyIcqpLQ}M!J=J71q zx0cx#Ks~qBM%JC1X!535O4}IwUBtSnG35wbXO}wEK3+fORhiXdGDz@5aajHotZORI zKKe(+R#Og3GncSGRF7vzO*x}Vf!I!XW-riYidOOD!H)a2|BS+Fya){owdi*4;^HEi zJfVZ(t|s7wo|>*7DGv2Ma(D`x|LE*|Iq3>S*(augomb(5ySE0NeVyg`AA$M#`S-{| zI^CV3y8<4mdi<pD#qSno+E?G4(wYQc@2LXyuM&QKpHMJ8(=Sjv4p>j2 zn;G_tL~YThw%Vo+cP|k{fO0f?uAfne8p*B6%g$Y=FMH4*R&PbCrs)dxy(!0r>csPM zlm#oyjCpZ)A+wIs-ig`G>Zc#k3`J83;02 zQxYum2}itc^%RW z6uXK4CUY}pWV_-f3*&04_gc>!rj?+dtnA$TqZd`XGaMX(bv|zdkcBL`kK`gt^;IQU9n|&-v@<2M-=!j z=`1=`Ou}Uz&K|Y(>1)N$B8!_BBA8M>)v=lP2&u`!GH-N)E_WRoj`Iy~nOZ%ca4wtU>~~LiRVLeul4U*Iy10w?6GD;Vrskbikzza<{dTL1EDT-@Rd3i9)?CPe zE(SsoU!rfSbVuJFj^_xMjUx<``R`bq>e_LEmC#vPoXp7ko!c6vx@v>-pzs>U*8~pH zRWi+3k?)&7H&G~Fv3x40cQy59G_Q`HZ*UE@-_n#9dKzL9y^{?$r*R5COriKUUN)NY zUP$SdxQ_37X2jcheTP5<`H5OLxSea(J((SWs~)R&RgV|0UUEMKeRC`;Tt~$KnJ{zW zoL7QIoUyK8tiW;bYy_NiXv(kTI}~x}+c)@wSU*8to=eo#(ki4R-!WBuLW}IqeG9p3 ztKodzd?oi$@L^M;YN~X@m}M6G>nUBz+uEccCUO_VS;R^bdhEkoM*GeW?*L`O`D&El?zR;&s9EiF|FXI)JFz22qr2nR#UxmhsONMbY!(0oO|f5pNgvjtagtudprYVI ztEQ&L-Bz*ikWt*QHlw=QrNr_)gsy~HbJuW4+#k@KXy1luUkK*{*y3`)^oy5PtXvh2 z9^a|OAb9D4LZJ2uBca~j#KZCGuyTJu>nHG=n4Vf!mes5BTyDe3%+7F@F<*n(0rsgh zMyPC4T+osxy}1?8Ca*o#v3IbhA4C@334Qs|=+-u5G&Pq%@Eg+K)1GpO4k@u-A_nj4 zqpdoloNOU9;tK};rDA6W7nD?0J(<+cv?!QTt%w+MQH%uZH6$*erJ=>InaBsZmTynC z*z3kM5=J_BIpx$1I58}Yh92Dcc%2-GItUGzEP3#vt<2IhP0fgTk?g5^nY(L+83wgx zmv{V8_^+|LF*o8LqTJ7sF0q# z$NXzM8B=dS&)o~kJLMA*Q`L!NU zZOaCGi^_JXNhl5Pf+stT&$j}%p1$XEX-xOhu6~*Rvg+!h`tVJi#RhB$ZZR9vMP5kwItTLh|c=8%$-VA02?e z5D3_Qh40wc@vm!Sm|<%8g$NS4cdX(%6U{c=ax@l#Z)oD%SIQV-dlv1N9g}Y{qpe1y zu%8BLN`*bTjDde(?Evl0bAa}lUt*8|+uHIFlB|eWP-fs#4X&3DRCy_TID2_?DH*z?cN+q#Z-?p1Wx zN5#;TdJZd4_&nGEW9T#R5=mayQ=1e_On_&woMAPbTW2e$ZA0sbGqJGy4T>Q%Crwwuh%>tEyZI(+0cb$O5bnS^JW zpQj>wEX^NaNO$Mzl!y@~l}aha$!?jIF+}BQdB=<{=6BuH9!dQhVyv)d!{gq9kCzM>zP&Aa{jFH1~@`+r+|h=n4@o5 z$un9>G8PfO()2&fzzZ-#yg(<%?`OFYmwl-z?;WtzFJ%Lz2N6PGFpSeeRehIj#qj`) zXK4!gz+qqr?#PyHZw49NzCX-bAy?8}5Sv{jtJJ&u2D$ zn#4z#`|<;ASfPRZd-_m5E<@BGkA(^_=(lM7n0Xbh8F)I=VOi`ToXi>;8qvFCyEXEG z50r67C_*_vO@2;Ic`iqo#Oats`55C>v6(xY(6l$dFf=0tg{cmYWdm?r?5>1e5ageB z9Hg5%MO*psE&i$Yj$_k1HAr8-C{~~uy~(;z3!1IAo=17Az#A7`bPJSbt1M84YTm

zjYRQ+tZ1PYWJPk=jF~V@pCn!*1DUah*gjm8qx2pJJ3EHJWM`F~FAYgK%Fy0jE_CMTE?t4m%gYF($90thX^=e zLLO-qtWlj#MPVe#B*WAi_BYt;k0QPK`D?!CN2;l)aCse{FwjKA9fx`i*@r4hY$5_~ zql4nI<-Wh6hlmrB%|*A!Qi($?76z9tX0P?E+qvldDp9^$5R;aQq7Tj)e;825H~Go# zA)x~eyMJpQ-)?X9#m00lQjOGz4Q>_3Da|MdYB1uTD2VONvl7CL&=Y7AVDU~reiHQ> z`=b^%1XUJQwH`z1HQF{WT?#&okYfn9*3qqdk6OhYxYgW9Wtuji?Z4<)bmMml{U$8x zPwgY|qZ%3;sL5nz!0ZE9575i~Xw)pxtF@HanVT)Kc;nFza^K~r(7Nx2;j(>$9Bh56 zV#4|S6RWm5U$X=d)#*QB!dJoe_)t)iCJIL837KTb?VB4j+izmMQoR*K$akJkPENW2 z-6O`jd$5&81CS)GgY&;LdRc+C=dVi2pePtn~4rGb=K>_+!! z&GF18La>ktRr@D7NMi53fe1~ebZ1kw==7oTk)do5Xyg3Z`;NBXj~qT_qBoq|0xcQn z&tSZ-f4sz;j0YbgS*4YfG|FN3=J;**AOn-_R7TF4>mB&u>TJE$)i3HzbOy+Xrx?UJ=L<@>~cN>axYw^r!eKn3d zPon$@x2?O(l*a+3B{8Z%dw>0L)?`DHIJ2;a6^v95P;B&Uh7E1OY}B5dzebxISmI)l$Kcwgqg&H;f`qwL>>SS(dICBqw=>^m{jwX2Be z&B?onZVVkpmeSU%7<^=WorLS>8}6cW&le8CkzSV}f-ozw{9+*N82`r+@X0`|$H#w) zFkPJ+s$bu9!>i7#q;~yQkQLX))L}4~Jp3a&AgXTTD0Zb3 z?%s1z)&Dg)S=a3g>qq|kI53NiLf(|*6&z3%wY9a? z0g}yE&sac0ZF4-7m1&FfVlajC+klUI*;5!DzamjgCE?FjKVoh(waI61;xvhJMYl@w z2Zshd6KOk?HUo7=8#TwBHPxUHg}V%4JAg~)1`z-48@h6MR1*wA!zTOmO}TX^hdep) z&x@WnaxGd4-?wm-)>?+Y;DK*=?toa#)coo0nbw1%BCmh+R58)%$o~{n6Qy#VkY~Oo z5YPLK>A?7zEnmFD5j4-tIGX%8B&8yYch!!g*h{19-v5Z}9mq=rhSQwGn(VsA0kOSu zvn5a)z4Yb|BxR(kfdPOCNJ4_f8T?#JKU&qJ)i*L7n(Xa;h+**NHOvx_xF(j*k;wTX zOQ064Ry@ez-zvxPvw5j{OPvORwI~b2p0QY@5FB%C`C1mKN__=(PnZstZz&~-17>iC z6kiS&P`f@vslNz;2Q)Ay{4W^6hOT-uq00Z*4_(6a?J`aBvrL`%+SXmRIr|!)jK*-A z=gy;es_+NGG>J>{e#<=AqPv_@U=E)D(L2lpi%{3vkfy-bZ6YhPK)xj;^FZqt_`?n^ z$C0WaTk?Ak1vN0;IS*x6<9LOceQ}?O>09bT!tY;zEXc-Ja14j=PcuJj3lE||*-V1^ zXIxeVy2{GR+J43hWGbWIS*eO|EeFf$*H-7}H;YFCgWlVyxt+R1C+?YMX6U`kkgUO>dM*gJRl)B!nG+3UVPp#BO8)i}~0%v!0^ zDW#MF-&IVU)|O}cCzFEcTM=S#yOF?~goaYYs@Fwk(99R30R^OiAu#OIoXrqPH;}`( z-X|r-@D8mkKA58IzYP8^5Ls>owYJI;nz~&HUS;wnUk?@YPalA#zR&}X=UKIC0ZD$S zEB7t>#q2m!KhIde(E1Q@atyo=G8nE^F?noLfAlHuq^Ywz?O~~YF=aAXD|>3&8M+|E z_`DjJ9j*6MhC3$#rIuM+duTn#UPFyGY3jA1{D!`H^>534%_qUlcJ6SG z{~yPk_<6n*Us&+~j!pp7=GZyf4vcGe$Ele^_rhPNA}iTQMQA>43TEe1s2sHrbPl|1 zy&-U|rjxI>E_I30`N>*Z`{zJ#0ClcE$4xRMf5@XGLR9xltZh%~gGibROB?Gn#?f+& zp}-H0WhCfuizAp4gJ!v#PNP`t9E2 zxywegU3`E0PYpt8My9^dL0Nnm`2MAAe;~i_);S`;8;m1`)xMzdsNMnvq^Il2);e>e z=L@>yt)H8Y4lq)BdN~@#fwfLgi+$a0Ywv68@;z}Sf^N@p2!O307Qon5<5YCuI-T** zCcnWc%@8{?YaGAuS4wZE%#MgSa#yg7`GM*D=Rt(~n1);VOc-5r)U!SPMjE0_EgMh7 z4zUPRt%}oQKhFXG>vAp$oKcqyZa$In6Ug^?Ybe_Y(5wS+_yd}RjJ%tXlzT$k^WOkI znbR50?Yz%wCOihituMd0(^JCLX|V%rtq1^+7>LAAv<5d%%4}^Nxo^?7+J9SH>jpF> zq3D!q;Q{x${{06y9)0n%`Gpe{k3jC^|luK51GF$5rj&bMzz5)*>mo9;^eouF}5LDS>8RAY|+c&X%JglRfV7 zjimMazC_@t882|k%BQG1gn1DO{P0!TmW-E`EdSsJ>8^Rv(iu?vfEeBVIr+;=Q1G^f z(zZWA$gzH0=@=kDKj2#?;nv-~MVuw@RQ<$Q=VQ!T!0lO%E)sy~*fsrykIoEmg>bn{`Uw*&_t*o5m)A;lS^ zN$|Fl)b-u9?=}=cfF-uSRX+u`8<9qkcxJ@2Pg={BrH8I+YK6pfy=?SX3vLkW^4_kX z;ooRb+SoU67HA^_(hT?IGN5pyVSZWrf-+@8*mj;ATkGd(=~d0ps?JjE9v;~5W(mAE zZtzbWFY_viIR5!714xW(wE(VAuC{_eL9&p3OaT_;OT$8dO|vPr>Q$VWX*YEmJvzG? zpIY=uk+2#jWs$cN)Su62eTd8~?ODA_ZST}3 zb{vcaFD(hm3u15vlzpWy9$Sw{cX+uC0gv~14JbMGT>2tL?{BRFxcsB!^HW27_0Ddv z1eIbEMHuw=lEVp@ZvS7wOWDwY(oO?T!^UWdrWXVh4GK!?4J*lR6>4*zv^wXmC=Tt!Kl*yZ^*Ur zda?f8QsAx5#>S zkcD>2m9!Cyr#kkyQ|}jOct6yyC@Zd8NOv}m5gOjhkMdi)vAVhGRcLU?>YD83(FcDo z;c1joxiGRlIF11YBE@e{S&{v+$brTEymR0X-g)QfSnu_G4@S21{P4^98v`Yl0C3n9oRi~yB|9bjZ!kGaOI06-04Uo3_`0$c;O#)7EPt04 z<5*P{X2*Zdhlq?y@PrSMur3){o^Cbvg;k;cZ364=qLvE0{)uT3pS ziOeq3y9$#SIcUhwof$?nEaeE#bmOeIlJ}#XMQN9q*Nz~V=^r=OZkT<6O;w8uZ~Vbh zp*nOY=-1#IE7o?udO220$5cD{4^sqf3E~^LcXoCB77)71wOdjEb82v9z!P}*aT>=l zL77eewiMl{&f!w4Lj7$aK%M}uFaGus&(K45)VKNG%Gi7}GTqQ}Ayd_AElN>0utQ(s zHHp2Zv(bs**_33FD95}94QVW@9iI!xKr(;jE2IZVvWsFyg&9MmmgD(34kIlo$-^sB zXbIkU>)i@cS$Q1RP%tSSj)vieEOGI|!zWZ^7Yp3_X4ERmHkLLKDBc&xA5SXOSRaV& zCxfnT@;I-bFqfyAl^I~CuxdUu4zt%$LKPpt*p6$_eo@9kfX-a|lh^Gf&WKX5PXRl#F0IfHm}whaNBz4Cnd z#_eP)FKe)-Fydn+{EhjPx+rGR)p&KgM1H#&9#H>>JZb{j$IY?QEk#~K6J}lgnZd!q z{b(6U7G1f5&HgpQcKr@B)ea;UkchTI_{!=C7ij`Po<>oUR$SqA>L8PKA z%#VQe?VE!sI-PYhI^&5hQ?%I- zyR_!tDbxe}MAA(A??3Xc@uy{7eZ$d8j8xG zRH-NFsVgp+Ivl=rpj~CXdULlPn8*MwgvHoZC+RFigP3s)S4NAFN&1t|9TVEwl1N{Q z%Lh^bfocvoYOP`P6VA1kl$DKk&xk@^`d9y#Adf;Zgx&^z2kM@RE$0<4e8&S7#NNQouwm z!K}d%TDcPg@HV8bda%LM&AYg1+94BoP8_ zM?TEad*5 zkOtZQsP!6Z!jqCc0UO`T)bEwsc|**<-w>=77-5j)+y+)952Z+~zMM@2(NdpH3T^iH zQC}5@DYR)f?%3n@&rv>J=PSCN69iy#uzS`-Nl1<+21TG8F7+xV@|D4UjuNI2SOHKk z%gWxX9h;I3va7CqYRLzZMFWj4q&9=Xt4`88zzHswKNZk}eDk22`Iz}Vk-r3bybo^N zX!55HjvNI4Ad7SUjhZ?gts9^e63}8acltsp75{!&MvYZ9(OQ~|c7B-?48IV#`c0MN zq(2N3?$s&*Di=WIMjjqrv(kEEOHT2w*D6)F(U!=!%m_BLk3Gis`l=YVK6S3)C5JB& zLGH$Cvg_BWvQV_L{-EXug+$|=>yP?7=vF<3H?_e8j?}wa(>QMxyUl7}pa{DF0xQ zfLYck-{&TCJ_82mR(5vY&AwNDj*8d37crV};Z~uYHP9G)UkyOp-OAYR76rHRp0JGO z+oSqJw}bHVR~{)1^YEmwXCL^0%w4hI>Kk2M*Tnb3D@XBSuO#R{wmt@g4Z$zaL=>?~ z3M?UD1oyzuc4PUMwCLy6!f8~J9X$b4)r2+W6)#I&)I9rLM|gNx`*kQc{!sLlx4z&^ zQk8W#%Qqz#wN+ku*3<6>F*!uQ=McpY=7Ks(_N19-N2OMWu(`tlbue~1Tb!(N)H%Mx z)+?9BX=7qYAT%k&Q07e?Wm(_LD=G%?&%ws}Y%(Rc_ayc>#dfKmL!bLA^t(n(*Z3It zpqmfg|2W5>s$By@cZ_zW(L@w9!9snzakpkxcm79B4{fkgQzQ1q|AYoNEV#Vr=we33 z@_1kASo{#R0rLAodNqBFG86hQ1=AQvy#Hu(3!bvFvD|S^dv;Xwk5&`1(Np&NN{-QG zbi#(_(;$yPvfaTwI5TCEbePve0QwQ_pv2Gq>ujqFqn^OhEsaTC9@is_C{sb+t=6VM z?lJrfbXXEXm%7-r9YCO%ot@i3HHlY#T%qE&KebzFbpoqw!(AMsIz;VUbnU8|+r>N4 zAIMvPN$zEW4adPEFaN`gM{WO_ZBxK9OEDI7_Ha)LpfB1@cUEc~ok>#`)6Z^xc$RLL z<5Fvc!zUu<{w6RO0QcrS#MxDsChpeHo%f`Ii^gf39DnmGA!~Q)?Ff6P!s8I*>{i|! zhEWVUn;@lLz(If{`>a4?zgCcwL&4bcsFF>$T7vk`@iE_Xz5vh#Oo32fHzRkZ=Mtw_V4ZzjEj z2+4Q=b0D!GaH{Wo|Ha2+!P7>`JAZykk#|Ria2F{sC_=rRa<_6i_!?uEmO$Nq&mg5e zcEZtTM+~c1n0FA{r)GZIFdsAF>Xf2t4+S2#-B|S#@zUgb)8X9IX&@PV`M8+3+e3|q zEpoh{0KFpug4;Wju|`o;5|V3FQ-Y=ZHVZ$xmaafV2UH6Z9h|3-MjSPI2Au>1_UHNBX5>C`0IZwNG)u`g062bK6J`qM?eCKh0 ztjOR=d<9L_Y3^)TYc0=)viKNv2kT5u+~G=RhQyEWw9}?ws&4`EZzf)$(_baY;P#}1~Bcd z=>s;-1isGKPgn|*vSaieY?qrZuFhl9`L@57mu|u!0vhpx0cxOX$wB>a;x2-Oug^BJ!;a zTZ3|=&L?DVuT)l6T35#&(10{eBu!L{&@u*!Mt6Bes+J`Y?@5m6T5n_e?s4B|N+SJ1 zS}4PlmIV!^276PQE1bVNp%VFZM|552A{H>65$(J6#SQqM8LACQU{!W7 z?Vy*Ca#vtEAS_moFccSq8)yS*O+Yp=K`7_cY;iYqo)(l?1x}Y=hjc($lj4shy#@6w znyy4rj`Hv{j9h^D%MK*0VNVwUH7{7_S{Ncv^#%tlTY&Df5_#ud6T!)^)-o99^Goti zx#u9Apw8a=BVoISJ;^H7ImJ|x#<@^#%G#t{qBJeAOTZIq8)w>T%w=C=D~Zm1D;u<| z$-@B~^}TzEHHq?6g?QVCfSZ!f8l(surW#-u@+cpISYyw!fWTEjLGzG9QGxAAI!+B|O@NTd0w$=Yaz)TqJJx{c4f z-0d6+lhBg1^CTY>!MO<@PB#sv+kG4_p-zcs8h-st0T+vytRLAidNg}&oNvsHMd&9(}DdcbIK z#(`JrBB4>6@Z8lWC>yUD>YFz)*XC!NZX*?_nk_~OG^qT_O8E?(q+zc#=msq42pE|F z1+a4x{K2=^)~c$iz5#46wKT%3$DAcF4idwwrrv!(vP{-oW?lPZ{E zZ|qxSVSDn$;J7=Cj-I+^4wO$?l|TdBb3;PqFh6{BdJT{Y-V_{j@AO{Y7#SW`muAV> z4^WjwO$`q3nv)sQOT-MUqC+y$|Jw6qrLH{&;Y&O0p`U2Tw*~^-CbccsOq|AHtf}f zwU!&2gD3Gx{wgq)J5jn)^^1TioE-rba{|O&Z-3RfAnXw*Y2UlR@3X=$Y=9-bm|@W~0?MX8 zzSe5p(Cm}il2lmq%2q4kwSnj)aAn$OFf640;~3da$}O8ek09Unvi#}w>z9@0}s{Zh6QBdp-V!R%8d4r-Uom0z=D93{mzp_4Q)2+Vkp z{7}P_v6l;cOvQe|Re36rr!^KPZ+}j90Kv9D7>`>3u`8(1{0_)y{AfL;iQCRaR#tIK z^#H&`cqDIG^?bKmkqDxa@Ov{-wHBr;57NU5SpLpHuNHrk(GkFc|z+SGh<2t(~2Fer*dZxt59kv{;l zbZxRLmX)6vu=APWcJ5tG{&|oy#1&g;9;d5caw|+xlb{D8 zT}PAuJzf5hM09I{&2B|m=jm+XJ5HsB_%Jf*;lXNUI? zcb5q_to>(vH1wNcs(}FRJaO67K*cBVVaBOr8}v8~0LPrbTPyF;3i8{#JX5s8u)2Fy zpAuz^P1bJ=T_XL#;#HOP{U@O?`r#|kYDtjMg?)s-o>Aj@<}2a(YRy9e(BUS~m+AL= zB|KBjW03l?hrDD-wO{iH+%|YJwFtbZTyQCvTU%|^^l^bDeK=xpxYB?bXt*+8cgZag zI<*PGNVnX{GTLAoq(_+TNgU2tp2{bji_rX-umEzX#W0ijoy!6pzksq;N{Qen1nBIX z3o}w)6xiBA|M&7QTf(X{8rvzg%fAeD_s5Gher*<^iT(w!nV})g|D0#ckDiGvQMswi zN#06tApsdqGNagqBJEM9cmtq@QBl=1p+H%JFtpH8GFBr5jAEqJ8#T|JU*g%l&(VvK}ydDmx?xDi~lvPYNy@dBMM-@|A3u zk~26oE~Alfd-XBCb&*3S&bh|x7nyur_)KlbD;1i40%ZuEz`!IN&K3AYTLR=`<7^K? zAvGBt5m?ECM;l_`r^d{E28U|&nY5~^x_YxB8Vz0s5LfaE>KF~ObK_E`ri%isj9Yw$ zF%91j&;lmyAy`4A;Z;L6Jd=8g8qVTc@SSlL;|tiZaE(HNE+OY;k6)|!#P z;9c1KdAew6-(vtY+YB%BdtVeP{r7Ur*Fb8K75E(KL~`OOo~9xuTziI_NpDqTz&h=A zPmapL`SWIO6oYZ-7rk>C@`$R>DpA@?DF!nLxST`A3vWZG>;|=VjY>e}(qMp1#C*zS z=^~?a=lXT00t=Jh(~{tFLmMdQ>xoX@6F3puJ1;JYzU%VkuqVy6SQ`JG9T6rKr!?+n z)>_rdRj8xX+R2ED?}u#)xz_c=6MOZ8m7sKH1XTifUQ#E;TL@@L+m{h%&V?=5+UHx+ ze=>0$qr}cR^9PeWH$NR@(Mkj{U(S+)^UTZon1n-&^?zsRQE$36gDRP#f^X7ih?hCO~dg5+Jl}U}++!EPG)= zX(@qwy<~$$Gf?gBx3DVQeW=<%kQxY)oy$y8%X~|GsE>N;z&-F4?tX%BZ?_stCZ+W& zwAf=2xd+zRkE6f;g)y7_IeKB2|B=Hm?YA%bnZ4cH?=5;)8abrjV4b%1KThLTJtTE( zcLdA?;h{6i(Ad3>V0O+7R5tQMqw9Lb1IvJ{%x)|sPyf1!yxtptq<}an9m(XS#Bs79 zIRMlJ+dpu)M8*q}#RYZ#j=T4^p&E96gI;HRxd#o`%p^k{dnm=r$kL_7%z)qwQuma&BPa=q*eaYefk`Zfy zxt*AJg3XjZ?(`}+_%Y3(5iOh_*ib!l?8SeYM@7n z@c*ZKHGpNyp%C~K5DVRP%nYE^3NC6+Rtl0W^ApUhb#7)M-T0R_3Y8HjLLt)bcPjJOWbbTd>|d;Uq^g`zMy!7fFi$A+JRz& zx?+O5LpEseiXaoLwnJz#Zx;Y;6Wr;}?9^_d4tle_Xj|`r>D3M5K!P)T#d)WxO=n0TntviN_D;V)-(%7okhd}JCp zb=5!he^|EJI$>!@O}`$&}Yp0SX+mmXz`G(H7r(is-{1Gfq}@LXPL4<1Lmv|$K&4I#DAsz;Fis$RWB$VG9f9LFAj`E z&vDt;a1Tg!Yd(wsW4hvZL}>r0w^WgiAn`Slk|}z)U*ST0Ji~Rx`>vTXlM9%G44HrL z=`$!rf55Dp39@)?fI2z~joK?QeoRbElmmSTj|HI((!ab*!a9|?1HDcOes3gt$l_&7D$ zf8#ThBA7Y;_?tAkXtSuE#tbBQ*qx?8dSo+p_ZcO(*1fxl0JpVrZY?1pVfNTZqQ)jB z1QDc4>fE^ukfdOe#kFJc-Gs03Ow%rkViIH&umJmCWc8tgH zo0Hz#-BvkkLBf{DwhQ6%j|Z8AutGo|Yl`M2f`gUuVM$4pGEL%>k8UxX zJY)SPObYIO4`eCUp(sPMmt)_vgMeNbt(5rrjG){&Q9#}$Lf6&zi@Z<}+3y%`5B(V6 zKU{A|dr{%Xd z3{uRDN?$Dbp;4F&S=)cV{MnrT&<_PYnoT=k04v*-DZD$iNCZWkTU$BA3HH+3{>Jra zN~GXAL+BG2Syaa1?&h}86ZH%XqD3#$`H(!YjUkM-X@K8oc+nowZm$u{vL#5VU#oa- zF)`y>qc!>B*=83dx1-D^dM?@*3kyMAxOwmK;vcN<7-BRdE1mF}u3~Cs!_i4RYS2iI zX8sNP!~Ev<_Q;VMc0JJNrd(Z+uOX{Bsd~q+d9lmy{S^v5^CbIV$}L)$r_u2ka7^SE zV1xXB{!oPG%YC7}aQc~iuJ?34os?c6soY5lI|)vz1+(}Q-A)i44WVjp-?^nIuQ_(^-S zh5Q%h0v*xDf&wn7&A8XZ%(1atZKr2Knwyhyjh?Fo$IVYbhoOC@+4%^}V#Y$>PfQlr z{?_u}{M%smdFPN3_|w@!(9mK)>H9;Ct5=-eaRyLf$M;sR${T>@s8L7ndr^)H1NtQZ z16^AL+zqu-h1Ht1DVg?|;&XlOPunEHSDlrmP#ySiO=yAU$!iBg=@8)=!~66#$#=fM z0FDc$`(&9BKoI%4bA#?Es?5gNbtS)BD^u9=?>%z5Ybm9L@k`^3+7rq!wS$HpO*XSF z#ql%qS`I71quH|)DM zB&VMKJh;f^cZz@WZU$%tS!P{_jK^qGdflI9#NnBL$AliQL00Nrt-mYDR7PHGwzefM z5}(`zkNq&@vmz^m_i-{R4VJE$`&L7%Zdw-*-gSxnm|?*qQf;t21`&LCVBRmbcwcL@ z5bB<|%YiV;ZfI~4y?hhATw}At*j0|{vb8xw^|&;plnExnnbPecFG}yo*;@cN-+OU7W;)6? zL2E_Ub{JJ*bl~iJ6ZGqu$Z(b$FFqcUB3{c4S|LhQCZ+v7M)$+@AnJU$)$n%D6fSTr zlv0zD;bDw3qF^iY+Cl^b1pL&`hf$amrSKjAE9tU8GhQBiK6aoWe$HsgNuua#n(VMm zJ_G8fA3X-TAP9cu2|)eHBp(1Zj%x(pva*5x=?5i`lq%@e(F&lwq z7MUKS$sF~d|E^g^M7{mahqP@C>O!_^qUR6-i$0#Fa@^pC>1nr&{i|0UpyE03L6xIS zN_C?`ahfWKVL2;AX6tA~Icz9h;X@cNH!W9^@&d>Q)_^vscdlz?3Y|l0zsOoUXFRxY zUIzJfALRY#e@MLxotDJ<_Lb!J5Yy_7OlAyNHCJ%&j$?TJoOhEtk>oOu=T=(A3xyCYLhlCccnm?-SjEsr8(4$3xm_o$ZG(KYt&0>`dTg0 z=6m!;(81;V9~v89squU#(a!}!`Rw0TI6uQdDKE`_<#OlSD%kIFs=YR!zvtod7ng-R zW%@D=8rKHq6$UUDFiW31e(ZD`>txY8)>LW<`8^|pnP$Jibe11lhi;bu$UXCSm@8w% z;Z*sjboM%F5OFIq{iiL$eM`i|2BoB9;OyC5FE_VH?Paj2Fb`0+YmK$I#!5w}k-_Wp zg+Av;@MR7(gwkmEv?8{supAy5yCn;hXLNt?_`dd03A|{N3{LVm-%E z@(O!}qu(Ef{?{8f_`rv@wej0G+x^Y`@q(h=rt8oI8ALlH$d$jSd93XLK{s?y?E;9a97|5UJGsVd6g0HB9Y6;^sa^gDbXK6+ZwqLQCa&}Ov}G{Sk*QF-OZGSfhZ z@Kx`?U_i8)(hc+ovHxrPD4KX$`eP)TwUxZ3`yC9ZaEubOG@p#HojQ4#<8{-Bmpc+6Kw6#$Ky95Jr}q@K)^ccp<8 z(at!k9U$B{xhbR(-YOJC=M`a*euPNFfboc??L-nu4YgZzLphXw<`AHc-%*dFrTIS7 zaj!zmD6?s6ylA{gOt_0rDK`9~C6IZ0q@-WaHLE3v&gUn;3&HU|(^3-;f1JG*a}76H6|u znOnEq^{EXY`_|Z<^2vQ-;^R%eZZ~^JUlGd#KH>!Zj3Wxy#lZ6u%=H z*2KOPH_U~uGifFMQHSC zSE+Efsc@I|Q`~bkr&OhA9Pj3ZD0TmGWuHe%8}J$56>(viigRB1?Sr-T%zTnqEjnf# z^$BJ$rNx2ShI8S#Z+^fG}%Y%1O zJf+z&P+Un_nT3_wv+kk6a#)PK)xJGstX1F8!P$P?o>x6*8vTr#ChF0p_%?aXi`mTu zZZxoelwgV<(=9G+ozj(5FxP25*d@gX;TSk8Ri^^#n=NbLjhCejsu<;4TtevM-X*&*zgS*I+-w4cC=po6a}c+NRNPJSS; z%FLM4I3_`jtuvqW{(cU^@3t0?$sl)=^xU)%j~qmWKpdy&UPIWM&(&p4&2cmGkg+)x z*1};-nmE_!q9|9}6|Qht`mgNfrZ=$BM{AgkzoH!4h!IJ}%Ue5p4*TTAfO56nA5^0q zo6kvZG=wjFoY?w}3RavEe%DDRz~u&s;>;PvaTf)Br!%t4wF_G(C)bW^+vxNOgnViq zjCh_LqOK0~Gp{QQ{Ej~yxNYX~g=Y=G9{L_p&i6r#Te0Y1Px?YXa~U%BFalm+_tLwn zFp#H8o69LtspNj10I4ld65}zMu5>LdD_)F_!7%wwpUb_NYjPunmk2(OdwFm!fgFq@ zE4SR!zm~I~L+}ZXyDZj!xxnVPqdH(4Q#TCgrLT9=#Gr?7Eqd|AN}n=EE37*&Q~0l& zfwPyXah18#@L1Dl2)Zc;XGao}C#cD$?{N;0A@^U8zRmu#9IFdCNktcPh+V;>F{BW& zPuhL0#fIgXHm{!brzpfVd_@d{`! z+EN=DVrY0QY}8Cyn8`C%17|zhWp1dNT=GWye_5k{OVIc5V5U?ER=jj}RwyvDnQ}>F z)EE8R+t{J^^^+?L_w@HRrpfB%m`}ml8J)E3O+$>y9NcEEUPd=Nm<^NL8sX{IQR(>x zvETx9o^${$m#*F|8)>;Z3*bCiVd0If2#ma6UnK_jdyQx!SmptYsW5fXEBmQiAtj-t;H%h_zBmS$!Ssj3;S4pOZ`mRV+4H1(FRW}DSJjn_DKs)zHqDt*NHFfcA}w}~12 zJ%lhWUw}pPp8(Ry6TdmCXqg`uXko0<4kfLB&B{!D-gRq?z`f`DcOS#KGTt278_x5d zq?k&oGaLE_;@*CsSaOJ@V{#z<&ND$1E>Qy`KBiH+Cj#a`Os~{yY-B zyAc4kj>b&mVXF-OL65zLalwRj^uGOZ2?-OZP!+nYn`y1)ch{wBXKg$tl_ullnRmWJY6R9@#pg$OXn<1<&-T0#{QdMr4V0jIdVo6Gg-Yw z?d6-kK4hxW1d`~$lXH@q;o5x}NdD(PutKwdod(4s-nIfCT6~^JFa*R140`F7Y8*My z-1Ib!7s$G-cF>2YiPhy|`|giEOQrZ8Kho|;m^nJ=W+O=w%M!kPiPWH0Po~f_YPY8B z)$H%yDqCBAg|$;t7vYv=xfk@U`gi9wc|)N=37(`sm*xPPLh7++`FQgZ?J3&zR%5Ja zCAFWxdxEd_jD%WUdGT&c&zSD9Po0Kf(acN=28gUlFB$yZQ%mjSz_m=c8 zb3*|(QOyAoC~S2Q^>l;NnEs5?WbbC^pEJ2G8jh`JfbCM`2OoPPuYbW;1KoK&a)C=t z-e&A;uu+9oyV)%UEW7Hca`@3?s>7%iFC&!#g5mlzp>w7GD-zP-79M08D;dglf>VBhJxPi3mA1A{z^4PG0tUdV>6qz z^e{e6!}uqD^G*UF#RQ*86U@t-J>?4zmg;&VOC$2_(2zmFA z|2)CxUQC7L;5nl|D$d>MnMVQvVsC1lZFg`i4`G=VP{UKcbz1eE`^C%wFFI#$MC?UA z86>_dAOZ^zmg1#a6K6c5Gs(a0M`Kd1P9*;XH<3!XrPs$sVyRshUD;73=hrTPyrf=- z-^?{!X+MFke7yHwYDSZS#fhInb(Z~#TmaXz-(~Lh(9n>m#-%o7WsPSkVv8PP@{rmE z$~$y$4w9pI+Du&ZhS0}&Yhi&bZMB-t(}d8j3AlT5!Oe#-orTkkfy{=!D^xJZ!7X7mzk~Yf?ytC!0j&Y9r4=VIGSJt-c<>1E_)O4 zS+MDoeoudj#S`|hsN94*Y7#3*uQzuh3lvy?7xT+D3k2i};iX?BKYybH%dG3@o|t;0 zoyEPJ9>_BY!&Zvo<^5`TVAzTCh;6W;;I0$Iz;_W+bI&EFDvX6l~uwk zM_(nO<~2iu`oGE^9ux$B9QYBf#r9%yH(QxD^QMY!h@JaKjOdHxJK$En66SO$xqFXD z{43>O7EcchEr?z|V-Bm5N>-(CpM;(+c`MxHBlv02k)*q121igwqvkQmJNQ=0=iYsm zKRA+gCuCb}GJTrJC~No+9@a{^4X@u6rIKP6+wH+IpE|YvRk>{C@$U3(I$G4!rA0ZM z5;$iXpP~TNxEGfumE!b`MbrEvxn<_zWaH&KY+VN~7lbv74J8DVS}Oj=Or7J|WM>r+ zIJo5>TYo z%q~9zOx&ieJtii%O^6pgzmxhZ_4SYbKQQo}j32M?8T4lGu;S7Vt^~wf35Zk6%LJMd zAY%efzKkwf&eojm)8NC5uIC^wQ62fQ@OSfnR0pME*=IH7tdU!q9M}o zZ!_HZ2i-jyNmtBnHNOOyAC=^TrF0+g0PleZSk`Q54s3*^ULUF|xPF=?drTMYLT}fvZR_%;_3hHr zI{yOaJ%>(YEGj)Cki-yN-tsZOq7-N2VtEBM&Nxv>; zZf-7zJdITM^V|0{EHnl7jofhbQXfVx)}RJP*=%G@t8vDxe2Hjey*cq{$sBpnLc|1Hd-uh~@|}DU$pl!1 z6CfU>QXJ^?(nUNE*wmXL8Yxj6f_gxo-d9BYDHw5?T-ge|U-T2%Me?u2LW-9ju0dw; z+>!^L?#V5A6VR%)fhQ>aN2;M`DfTU|hTR3jf5P0%Mp%W#Xa*M*vut~XHN`KtaO*te zv7DU~8(yrZUv70LG`eNzP~>3MC;h8L44M%cE%u0a#HXF!vg`>9@d!3!*FEA?|I4&ED@5786FMrhfV2+gaa_ z*wKFF2Fx`dBs)+l=vGDn`5Kn^>vnTepM5%i(|)IvLE^{W!BA3Q*AQ&F(yOW9xp5U% zLSm_M0ylJWXQ;{IHn4*mS*WE28T7(Ld|A&y`-VGXbu@ATBr{ZyXG29DA(ifby*|a_ zM9dAk%%pDu_jh~%rYK*Jkl2=no(Qn9Rbr!{vl*Zn?4S0)m)b{wrPB>FH5eWi6bE9E z3UAUHEp-T!0aMkS(ue zW^J>spbWA(%!xLKsfN6vbAqxTJDG)4fZ#HYTJcN-nYghN%w%FIm zZIwyM(MvY3S?70~DLp6{a6%VtF-aCz`^I`S?%ra*uR*aZc~uLJgON5M5P_-alcWdE z(qI&`n9U^DnTL%eF+BChah--=66?& zJ=uh>&`kW{^y20mOj;9pK(>6do;4zw`}SVNyMz{}Q(U7#qRa+(vpRoB`0V|at+jyi z#7=qHlx%sjSvjBToq$QV^$3AR`AVz8rA+y=BdpmV=n$4Fx_3oHj`vKCpDRzFM4lLkLKOza8ZZdOZ>o8^=DwDbDMBF#+u_0c%`JMsO?Iq7R`4^) zATKotSC^G_EN2T%_qLTxY^lK2X@xoGo1}p{6QgIsKNnIqfM@)lCD&O1leGh7VT6m^ zzfev~qgLzh|3QuWzP#aNhJpiv5gN%L~nPn6*4a&9$?i}DMfp2CT)D)4^>zW-^s zV6Fo8cec{sXSOMo)Pa#~*^3Zd*hhJ~CX5YYu~>T$CO!9B~@n^T;YPd}0 zt$9B_V_hGG=UX^z^Vv)0Jd*!3#r}@m4B*?%<;9olV1^~Ro7cIl#BulBz}d$sX?OP! z7W_MW_ACsiU>4t*n67yQW*Qaen6x~d4yM5-A8oS~ zH?@fCtX*^spW1KQM9a^@quyUL;!DzXG9mZR|CqED@(W!`V{?U4R`Ef9WCWdO^}%8I zoHx6Kx%jwOo!?Y0CAFsdabUIO5nyv^o|p1)?I5Sl*zfPGsf2VNJU}^G80bTw7pKmF zmf`9ux|5~BYLq;dG0nmVivEuH`Vln48pw-f7thmh60t?RQ?5L0*LW8DW2I*pN4<>s zd}lDprrQUmpV#s>cQ1$uV;6h=d=8UP_LYcSgRA&TO?yri@_ z2brbgG!!H_wKqWK0EMXeV-P#9=NwOi8M!WQlYixT2J0K_#)ZU_daqF9dKU@SiT_Uwyi088>+^^sryAK}*&0Oz{|IZ2pJYj7qet%l#Z{+bi z4(t!@YY?yQ|K5dWRWv`oS3B=-M%;O%s_KT3;uCk2akC6r;tTjS474O)@7fQS7@G`y z_JkCw%fctF62i)~(?D*8pXJDyWaB=&|(3WmpBoAnapKkV*K>i7q{Ek zmv3BF>^e4?wtni`@Z?^y$~hY@*}-$~-_A-_F688~r|VbT&x4LiESHTQ3UHsFBvuBu za;&%dpdKq_-Cp==QrLWyH=kD}-GGr_S}o6407mh(kl zfLaoE{^MhI2@`c{uZM<)BQ6wss_vKI;Sr_O?SZU*r#lm$k&)4;PHl8M;nk~weH;w@ zka32=etEdL7XjY)->@fs@J|(;_z zFZ$vmhT;FUsagfJp1-|*^(T2i>+t!y($N0-ZgpFW{mfcS432PZPi%%77M=bOh9xI0 z3VkxgnpGS~$Sv^8Oyb8lmq|TK=3r>Ix)*S4Hu{$?9cd5z+*P2Su3mXtv+Y4-IU1hH z4kon@Ki|55q`0}ACWVuNE^2K3Kj=X!_rrADgzdiLSLw-Df4ETIedQ(7cRqP5qMbJ& zwp5BGnRbqo3%o8VJdV1i`rPHhj_TIYu581R#^9V$p_2i9-cA+iwB7ml2POU*-JX-n zzkH9#&ia~TFjT2x;mv%H->?w83m69k?H=s)P|0j;Xw(kF$BG13L11bKFx?jyq-Hn zN?y$%m5Gv77)3Y_D{{XOn33Qnf@8B*R+02dqeOLZJ7-7o2c8r59$fLzem}exhznK1 z#%)|1kGK%~9FjhCvtW4PrL?O?@jyK8o8L^*jiW>n-T-Vge0Oz~r8WMcDEY1eq| zdt=a(kKh6*qjm{n|NS3qWE>! z2i`045bmASOGx~Q6DUI0mZ4Cvn8Fi`DXE~0WN?b+@GZ}UNP4L17x4^BUaGrS!CNNM z@Cb?L3H%ct)I6rgODHKkQ9LwQ0;{-t8U0J6N6@d7P3_Bl=rbDduStHbo@A6|0{B04 zlk!Z`94{WK7cG-IZsZ|uP|&Rj_)BNIy*M5b4LymHNsbW7aYW4`!>O4;y`sV&gXgu) zM|Ma0KTHYReA_$)%M9C0{CU&#r(-t85qjGTj6-w6oQ)vi2Cl6|nk z@m7&NaGFLIa6JqySJtnAOc5%=8OZVIBP~@eS6J1TnY`&ZG-VFNHvI?t>^5)Ai^alNbdL2 zLi7ExM@k1q61UCw+>3bfoWSjD{)RZOvl!Hz%_^iM-Z7g2PSiD3XJ54Pj=|P-?#Tau zLe`DdcZFMawD^s@|HRD`rNZfPUtiqqUFh;-?^BA%RV%m@>WM2W^?tp;oJqUwV<11x z#s~jXkJ)LMxzaF^2}aX$^KD-r~i)8%irXv?7WWLkC+m6d!QB)HYJpy6(rq?W|(gUZ^Q{25}- zhZ)Eyj{KM)H3sTw^H-xq!dU^v9y{zZuEjjh>ZfI(Y@zeYmzR6&`ITVFMp)w&UqR!5 z!vh~t8F!x#SZL)Dl^_o09a}33AN4r*ikI!TM?5ojyHUlI^wZ6pBjYfP=WUk6M<|b% zdHdgmR1GP$Nd~8>H!7Kp`Gzco{ayijG4bLOmn3$j7Fd@DaImf3e%<`s{naamp3MU( zcBFjlp9OIxZ3VDQe9cn=v5Zn2NcGZwd0D=IIe$Mv4xv`=WnMdlQ2e#gH#bY;62;@V zl?ot>>?r_o(ZxqUAD~U3%aX`bo-@W8kE;i9)F5jPbx|+ze13Ng%T|_`rJdA_`mS9G z>$-iP9$p~SlU^AviRmTJvhY1k4%23zdHrXG(^;O-{uG|?6%z{D$juDTt_eNZS7+PL z#J+{E&}=qr?Ry2h{Pxx_OQbfK7w%0Y5@z3NR%Pw`A6CEY$xQD{uUwse+sMZ@>eCna z8d~;g*Jao3eE{p(>vDbmNYh0BHKnkgfmk$i^rGpzvGs>xUamh@Q* z@GVKI)4#pD#>#7IYXUIUK{*2{4tpxM2FrmLP{CG!Y8@H~m;;%1d+Cr5Z&eO8ZdJY0 z#?)z-ZuyEa8}iCt>w~VZtx-(`{c_fLyvCFmXG9>w)v)lZ*TW?S7p{rqH+6P#lvMy% zh+02SxCLz-5ipu`>#Bq^+}oULxTxtL182wP5httG{azFKm?qHMwn!mc-TGGc_;LRL zfWQXhGw34Ov4nl)G%wTxYT`xCcJTo@v%gXuSC+T+B7zTmF{6CVGUj(tg;-$Drf*_}C7O zp-;B__@LpC%c@E53+X)C<#rZ;OllqJOoKpDxeYz%qg99Ye=#L?uEj&2sM#NG zyJpmcTM;mJsPxfS&Zo{9Qr3@kY4+F#SOw9rIxQX#Lz7R%-MDeW?%9za+o9(wWmZ;1 zZuR=;e@nPW73O5K>nyepE0!|R>VA+c-76GJJT2-~D1!RDweKknvg={v_Jh4xV z3x>nPWnz>>cj?I&-=s|O zAnoftY$rFCnRriSzD0VaKkgm!!+w;K(vh$DMKM~%-gEV7dQfAp0hUMtt5rX6T0n z2=uCyyWXp)-%!qJXlPJ}YZ2@~2!0IN`wZ&c<#-q*uk}}3_UTQonOnXu3xviUACTWx zo_24^yd7~&Iyjrttv(c=WEKke(atQi2PNM-P%;0<{Os*2^z+e$x2M;j56P;A;k5}< z=-^n;#D2#Li|(bP||r)<13Z{q&lOi>-{u-2`kM5Q8e4=)L~i!am`ULHL~~*<(c*pH zoD{j-dMEoq4M1itD?QAfeSUdCGNtowz0#e}(lHdK}zl=+t)jd_qbl^{+BAi!;z0ZTw`6_zWDC(EIyg#Zb zkj)#4h1}i}n^G3~Tqb?q{C!2ri^cIKXgxx^aC+MOQnp74BNB>2?Uzn?CD^%1v0dc6+ZLxRk} z;f_Av0Dy>(=a+6b2oxrA;o^YL1n{#ysPkc*;Q;5t+gw~=4AKI`Nd?+b3aP66Fwmhr{ zLSo%VT_LXfL)stir>Ya%s9_5me^X{xL|;&ztjD*%3)oUKg1EWwiSDl>mLElG-*UCP zszh`SXMH(HnzR+j`g!WUGdk?x5a_qG&6=CVMl1V{iH(yg-_PN52Fo3?rT1f4D{%DR z6npzSW6blI&jGH3lRCxws|!nj^d+jWjO&Xib)lX81hFQ_SBK#vBTw~zwDgC@n5A{G z&na0lh17sbT#3+gvQ6q{NwlSIdn9X?)vrC3Y#kL#On>|xkX>KHwV^KK^`LboI8mn$ znoG|3Vfjh-7#_UXy*i;wZbsBV@Nuef1^Aymu>lFGj)~B!CR_EmGp3#a`A^rR_3wJ2 z1RGBcn}7bi_L_sof{UR~%TK;y=IK7hqOu>ac^(Tob9}u|m~74sJ;FBLNdR~?n72Jx zPWHPsM@bIy<~YHW&=Uj4gcxUVRFbOSLwyZYz|je+n-;)G`I+4+1MKtlk-=f;tFg~v z5Z?8XNFR_iw>qI=wstD+^-#fk6j$^7-v9E{r{B zvPXmD_Hm&WD0d3^$x&`|?7GH?-bCX{jeblcWc$=-5rtZQ+_m_*>K{(<_P^#tc8E$b zo}X>lO=Ew%%SAJL{MpsQqy3M!)gW?Nl1m+HwvtsEaCf!7`=uq`7;r}iB1wl856>OG zgYJOf;e8E76%#uHK~{&6FgwFY=hTzQI{*@7>yLbat_-`4>uaI~u~Aju^PXAXO@?DE zwu2HAiE#335B=P5BFha;CHG%1)0Oaazb|L4##k`{@}q3wU#~Ae7A-n!G_jp0R1B=%2zRR^9-&f{F%C7HjZ~1w_o=>;>Ws0F7w!Ij> zPO8b%Qg+)xs0(a9P8m-k2k>U_Kg15US?9k1CfuyQlPMS{zVm3Gb zJx3{#Z_l<@O8UhIaZ6oWfoaqqQG{?3(DLR2}M%8@yd5zo8 zo%eMap=-=)Yuer9?{BnlKJ(x=a{ZFPMawL=$B82gcZ$97^~ea7Ouu-(AwQ2jy*@V@ z(!@!U#|ZCI7*Jw*J80|9XjT;v8-TUz z8A=SR@;F@VnB3Dd9fu$bfgLW&&CAp0@u&s4Kn=*d zZIi`mjM}a0b=ZF#09a_c`-Dds@!T!s;}6?_$`fO`=Y6ypN;>(L2V{Hv)C3zgQNjuC z_ga|x$ZlGTdUuFjp+mpSmW(V4FH}}J&w(rd4Rf0;Inrpu*&)8+nu_aZJ1a7Gx9BXl)5?` zk`FIZJyT*)@ZT`FrI$|;W1&EW&phD!iVE%Un%!<7dVC}orzUpP&6s@lonZZqD?Bz)d+rG%kU0&gOVg2K0e*o9JY((VQ%GhwS`aZNMYJ3sAeq(g50gSH` zyG2^=!A-&@?tTn}Z$Cg7jiXE@u&Qz#ud(hw@BUF zmY;6zIU{I~p&96k;{vKnh~O7hu&=y(*k6M1bsLpy#G2L=Hum7JWFE-ZzNuKOYV_j& zy)~g?_Uq^s0MPQ5{z>XwC~+=|1q1+)2*ym>meg7Q!JRrX!Yu)A9ESv0d2!k0)rACI~$-Kw= zAfo^J`_-%AVKp8{V0u2N<Rr}G=)HH!*clk3vC zg+w9EB@MS+tbElE4P)>9kuVs4OF%aJ-ek}TV!KV6XSvY$AZp5eoV#Vg?0ceY#rDRV z9~koyiI2-3dtzt_MYF6bucMz8l}(|*sQg|-N3-e*+5d}+frt(H$UA9!C zh)r##Sd{Br;m?HqdPX2$Y9e9W`3xJa}CIu{9a&j zH;w&VQHeTURQ{Cya%%MR9;oB^umg_03S2bxoT7JUam-rII}}{Ud}3(mO2CziRJlOV zVVgiF0U&7Ih+4f7vE~s9?a?uGb|T2lqiPC3&FQ+vTOaMLBTCHbU8Cb;|J5KoSJ8!a zBa+{}Q*lsgy>s~@R-rQxLL~%vfKgHsa`dx5da+5Mw|s_pwII}~Jeo#3Z(em_08 zEb^zzc}gK?T47&MwYQ^+{WdF>c3PbqtW1d?ij~x&v5lvs{kty!ovffieKV)DiwPI^ z+A+X}b}7uypu|YzcwtD<6o+Az6e5UabR)bB#O@zt@Q9rGtIyvA~=;L2%z+i7J%>EiO>CCW01OyaBt{@9nnZLaUOX2XJ zA*itM9cn`prEC~nGE??D#hAFvFhftL2yY-vwTt8xO;mhB%K9zK@q6xm(KKdcbs{Qf zbsc00J{vzZ2MiA)9J1ik3LaiUzYivSYtct~sBpA(L+FKZ+nlRz={ozAVL;P5IS&i* zH^xj;`zABg+3umhA$|Y0JT`Xy)_gn2?lTJYih8O~faw$cTM8t6A*HT8&nZJ5y_5ELAR=U@*;jppXccmz1XEMM8wK7~ z%jiFcw|WPJwV>JSw2bH8O^Pf7V5FE(Acg(-G=>7B7FM2biJ+HYm8#1}u9i(gJfubQe7X)Xc)a#mPoEZnjX}S`Uk_fFazoY{Cse<=jv;|3 z6N2L=X)h8AhmoR(zsDMW;K536=FIopSdeNNeNIu>HjidH9BM&MhII{#6YNk=_7ddi zTk35=wOkcNVdaqd$`RJ5KVxz%&3mP9KIsdwI?4H>cc(eaWYKV#!Ur|w$QUzjCGPvw z%v?mO>lX;&r8s(d+|&7IuAlSS`<0d+ms!jNSH`WyEb2FZcc%?^^eeI@Sm2Rcjh

    P7wmUn3I1}D%` zoS{I&Y#3bzf}jhKC|PaGDT=>Qab(mQ;2{D9NY+!U=rA1sxYl^{>^}TiPrLxUqfE_4OZl(XK(6Y zE&Jx3mOxL)ufyB)^eoWHH|iBF9U&?=SKc42EH`uzQ%{Pdir8j)Rp#?ja3A8qzk&C? z15teY6|OG7bpGt-Fza^q<(_-_4jg@=H7>F7A(70@kDPm4A^(JX`Rx#K%+S7Sq^Ps` zXLj0G>l5TlsKuRPhPFl=*go|{uIZu+=c3uoe_{La6+4NEQ)vJ$rfT~HdI!OPZw5wE zEu3($MV;);%Bq36o983~5OF%VzBziQoPr^i{NsTmf1*fx{GO;_4S~w=g<9L!^tA(C z;2`K{dVo-2N|aOh$HGmuPb9;nI8c^V-)9*+*@@Mx{3)&BhGV5ZIaaQAi zdz63Py>uH%Ff7qLYF^2}O3fSIZu_!sGy-rFE4e&Goj_@S6u7?3`EfsJJt;D+TFP5| z%x6%N%*R^Mg<#S-8pV#`H6T>U&rHYLP`O7Cz9$Vq3o9R&qH9%l?Z8EncIS=rwVKAWd#3sClG_ z2dY{LcJ1gNqd3d|qBy{<#+@Rs5N19?jc#! zQH5iAU(ex^WWm45XMgcx9eJTHwJ=C0J^SaAL~W>3P~5EtM2tO)v%K+N#s~k6yU!Dh zxh)fK?-t86uPr^PcR9N9v<1^)u)1aD)n2eLp$tb7%QcLN6%H-K=D$gh^9tK|37LuB%f(I1k*?AOwS#m~+ZsY+wxVpMJVMD2D4CK)T z0uVXhO#^TS^226RLqi@D2vGZAYrCBl{Ms3)GqEUS_B5q}OtC~dt}B%AVY!?l_~=dp zb-zcwh_H{~uRW9^Vu0e8v|p&@Rr5tGh5kEGchvgdb&0dHQH_~PR=vjPe$>QuK070ryElM#*1O!tkHc$j^G%^Yu8tBfwi`*rxB; z*G>&m#p(=v-f>0MjhE9z2JnrS^)E9ZPtzpEZv`!&V|t@jCf zPY0%w#a;ifUMMYOKqo1UdCufHo+IK)rz)&N9c@hor9sCexB%=a0@KDHr|^ zTVEay^}GJBw30+cg|ty3wAi;&S|o{tFry^dvu7EjqAZ~@_H4!2*X(;DvhT9Z6j=w8 zb(k^ZcfWPc`JD5;e)HG4&eg~D-k#^V@7H$Uj*+p)J4d{u&=gMArw9E|#<<}$d(XI9 z_L>^@vXz|lG)sTWjJe-XS_Y`p)<)Y8@4=>{)Qv7jo0VAFCAaeF4uq&FTA4GQV%b7v z+V^RSnqR0Ylqr#TCU6N)BgL9qxHWV!;WZqG`U_RBx!2(&ixs;&9*rE_wn^>mW=y@x zt;?SXySW%&3OLCim7$lWh`)+_mESvAdj>k5(Nt%ELDoBQ;mY2JC+wT-t_#|~7dXy& z)c-xl-!JAQf^fx${cIL=a7R`N?pBSowavohLL(N8(E3*Ewxl^J1 z8R^NJrMn`_A9CDL6qiPu7#WMln%I_4U)^JxN)F^;y}wt?QXSV7wr>m6IkVDaAA7WE zJ5yCnJKA*n?~7v>=#nU9OUACG-~4>dU9h4~T?|=T($@5AguP#G=fWoMPYQjDhG0I6 z=n(9+GrPo8b!D-l3j7S1dg>NA_BMn>JnfdnZfm z6}E{jLrSr>4o+Pe)jSuTy3S?)1?k6s$p%FL?iE;^1$@Z>w^MW zgV@>JSdvt5E6Hv#G570ogeB1L2~TuR(_8$mIhWbDn>rUpp`c#70sD^IQtxbRWXEZ0 z_Q*1Kcege`O-xK%6F}HXoT?pIYdEHA07!w!h+g{hk3D^TYbTEykVMhK-tIxLGb3)% zw7}5NEYe8tIuMr$Wk$CQhi=~T7FncdVY9e<_%0WzTEX%qTB{nV{R(?<2qG``OtHGE zDH@xw)?DE=1^IzXkoQjRY_?Y=#>5y4;AXIYrV`hkel~a~g)6$55M+*R?ewdz2of`F z#}IDBI)8YNo|xD6u<>l8*XlceA>9^g|AZ?^zd3Db@Mj|PG8Xw3zLA~n} za%f_yrNg;CEB)PdX#xF}VrwVFBa=N(6B6*~?dx%^O!55SwVf*eG9I^MQ__+6O<_CW zu;+Z`E3J--uV@?h5l?Ef-IL}sP(@m{iF`Y=dUz6dg54m0u}6VhdO|dgNd+n%Wt?%AMH|e6@Bn4e_+a7rL3?Qir2xCjPhL@%-D;xP2 zKHJbhTG96-Lc)ui!u2bD-Y?f31$9VI)NCb8{@!;4a15=?>^V;B?vVV5jha@DZ23gh z7r3tAFa&$6%CmjZ(YIjTi%5LgbP~EAe^R2moe)Z26 zLMxNpmvisYQ(VJD!ns#sI8ID3UeX!V&UHpIx>zg?>5fK&_LL=?s(q=c=IJ<1lr?mCiL=jAGiHReHlNa%`Hm+~)#`u^f=cTlK?kA} ziNK{+Ru(Q7H&}KUDR3Um|2i!*3mbwPw$!@+Z^Zv%iyiBtg4o`RzI&dsoO@g)aP1|9 z|7?=a$g@q+ZANIx_*RYweBOaCf2a)Z_MmuKweF|#bhlCftQ)6g8O*D8n#MMbPiK4* zA$@=%DQ@Dvomiry84Mv=AsI(wDoyJ_d0B8W#ap#aL5C zz~eT*D<)!R((eVR#Uc^8fL2Y!QU137Y$ri{#f>;7mcc4p#|RWJ6go}80_V&B^&-=s zm<*fH*NjkY+M-S@nzzjS`#8lg&%t{#zwzhz2NMOo$930)x%i5H zCcJ*+yHf6P`tDm`tQ?@{s=Qe1K~Kq@-$R?`G~_MvF0)$B>S*h z2>GaW%L25t+@xB6#Mk#!Y<7BD-bnSxdo|a5K<3Ui3lTJ8Uu&Q*Ava@fqp#k!j3Z#@ zzb1ZGp?ygUH7K1?GYthL^2=EJ(BAm18xJ9NuwmKC+=dUrY+&Bm~gY293{>E)X4=o$nG)s3Lh;1h&7NwEdtMmyq5Y+vD7SdA~($4 zO>-~(l<2*ecEsjo(x)b{eE?K^y^a1UaOB{J&K(?ek+~ZDf;!d%pV3(PcrV%8rYr2Q_~m_V#`a9z`NN$SL0?*B8cErnGnGcG`nNOC%UDfm z%4F|4oZ|*n1&ai+V`W&z)x_>t9ZSM$ym!!S&a5W~D6v{f7c&n`P(;fIKoFXkaN?+9 zrR$P8jqwy&e|Gkb+y=)Jr2B9M8(U0G-X`4|$0!<375zjts_S02MEVBhln*NH6Cv4i zq$XBjF8?SsxBI6<0-Yy+6$2+{{4mZY8sa$)I$0!A8l)-zpbroS#Hon9x8?3Co*zF(7Z%U%+w5-s;wZfM= zXI5il8o4AMyRxV~d3&Z4y%JwhK7K~u9!wEvkatF#CAE;ds}-|62darm-Re3;rbC4; zy~5mTBd;UBEG<(MV=;F4mWL-lbZTXRcLe+dOXdk<)r@pm~#S5 zjd>t2ye(FNG+a4&{Md$CHCR zH)ezFtpRR6uI{(QNrd6>v9I1zU%^&#mt4g&PHaCvmA6QU&}*N1iB_+(pPL&gm{?YfZoeXqy>K(ZD@m$8x)`{lEnM;45LQ#-+($b0dv0ft z&_*-XeDh;l<{llK)fB;|A^o1UT~NqS#W;SGPWP$A3NTFu@Z>3cvL278gRK zdaX6&rYIFqIL=PAxaH0ztFCJ@+eHnW9)CR69^X3|QeyE6+qiJ7Op%!yi{^Gsuvw&} zz4Ch>(y^Yn+c?jOoQ}6_`)d3WR2t2ZQqDt3a zc9qE}rKM#|saRFdIn)V8=%s=?4?8;jNTux`yhBh`K8~YPJ9yM;Lh%4y{g^BgbLghGQVnPd~1vU=y`EF!P6> zGSj$vx0IFq<8%4ivI4)VFr_s_qq4t2e@%Obkfd@D&goTwqFxvgb03$7#>q6@teQZ0+Dx7SKvZD*9rUe z9nZs$Kws|mfy3TUHt=e4t0uC08E(kYLOySWmA;xp{y{lSuU!{0oNcnHtbe#jCpnBe z??&bSCT$nf*D#~9DF5!QKQU2ZG{?$rkhgY|+B|#G^^lmvka3ex5ai6zLhO3i-SFISw>bzkN!4R&wJQY9|{t4ZCXPc9`s}0#SOq>1E|( z_+Q6nog-*Lu5aAKP6=7b!26V=WT7=5%O)B8v^YW9T5+;v!{$v28=5$ws_8m}(Qpok z(VK9ppQko~E@^1}&$~}`sizMU9M>m_&fC>M1|Gf*Y}f=ns?%zSb^^|~?zFIQ<1IuT z68^V&GMLm^)=(WJAnm6rlgh{P{B9caB6dmGO6 zzJ|w5p=Dy5m;{@u=8U0pwnw(m`ji#ENHn_)cLBxmB^Q z@rWu7G|HG`Dyra^_4wTB(u~ry>E$7^wkq9QG>T5x`)t8iHffrI&a8Z@RnBTNi0!+uFc*i6BxFM85U5^ zBB)6v>8m&PT%z{kgSKmFQ-_Eq$Y%W}*x6Db@mZ2k#xH!WH!wGSS(2NpT>rwu)3ni| zef+5qGZYOFmX9_fd(bH&LG7=-z%9D0ZmH$2YWfW8sYnz{ZyzAzTkQ?Q+SnhCUSTUz zGXsz3S(oONJ(n@z`?Rw88puBulq%Op37w1fS)Fc#iSxxO(OOmXM4Z@7oV1b*97bY} z=!z9@yj#PwtasmcEdRBht27o)n9}l5Qdv?%eOsGNn_M0f&hI8+v*eyW(!s30l=~d( z0`-;J&AK4ovg=|_a2oMmvvxs1Nu+K(ay#u_Mr1}%#;4_tXMGb_bRz8j|!U*a)?zJWxq_$QY#!qp0* zHv0r%Usd>Q3e&v|-KT^yYDDc$ve>Y_5ijRsnT~yyFFPyAyKzGWvih^GYNF8+#_F#<0zW(I zv~lr^J{#J4M>$1o7w=6S9r-_lR%wTwgAiQb^G#B$M~VIp%3ge4^L+8KxopVhfg45$ zoL=Lj-zbJrbi}NYAwvS~{l(qUF%>eMQG3ENlQVE@VqV^7G`h6KT{+}ZH%?Sct ziTl_iI|TJN*ps~Hx@_e|$F~_A+wM?CGd+BsvB$8hc-x^5w?pF|Yo6 zy>l|+ySEmxaSc`!=C3Ue**JzgDGm%S#lGhPzC7e+Qxom3Z0XL4aRPdf!GIe$GID z&#+-+MPHW4u~Dm&z1TC5SJ!@9B+^v6= zJvBKymiA}I%^d7fx^a5;2R%UH8bxxVoP4^R{cw}R)#%%8^w(JHtgzhQz1y>NOH6u0 zOm5yndvxvmG$z}-mp}M5v#DtMMDep#{ClsF=n(SE^G`1HJF#Xf^dM3(p=aE0%5S@~ z#}~&T5u1B(r+xW_g`^P~$58A3XS;He9Rk^~{FKJsqMgoVBUENVEkKNHt8ieHQ|3TbOjBzQgAlFvs7fuC(90 z#S^}#FDr?U%V+0=n4bn(&Hh-I&QF%7a(4;i%b~;yS|*B8BHR~yOE!_R;V$>p>4GzQ!OnmPErQcn~{LR^klzTJ3yG&DqtH*c?ol&rZtR=hKg8yLSQ>isz-*V5LWGU`u!K()hD$h}cx zZ_kdWphmnG)ZPWhEH$cQ8~p?-Z4*Wk=fhUY^?onQyf))%z}SPsg;2T}KP!qTPu>>B zUp5S0Cu-Fzb8ifP2@%$5IqSXu9waxx=(r)A@R7e?94lRvOf){ zODT^oMQeJj&yu4Q;WdKJ9Pn68GqRdoZu#x}7&!EQ?u92Kz8eX1QPx|j6Vv=4U+Dhj*N0WALp~;bbu5P+WsGkv?iX zSuWu^CN)`n_`KApV0A@ODFJm_rSpq$n6)e@|Et6zhr-TeIww36(eeu5e;PH*SY#~S z+DHZ5ollvmBaD~K<=XoxWcL-!?IN>Egqc%B z4C`E~LpG{u6J%q?$J~Nn@*9V(e}r$xZBngeW$uOviAaOG$_FKdDMs-w3R{4z`aD@2 zVMD{rt*G4PE17`hsJh%hqB5d#xCg~ zK@EsGzhTY?F-OQcgZ~XfINhO+|t40qlIL`&)o?YA^Xs0hl+O&P_7Dh^M^IO ze;*hl9A?mEKl{ zJN@U2#nQ*2tm)zFwmOKB96o4BcLHVxo5)Q@&Km(yON5IO@HCs9d}R09!}!M&Rk!PN z?Nd~0RWPg{acgD_fnV;L+^?1{VUC?SeeG(7Z|`N?m(Z7;Itu6aK>RFbnNRA=eX2LF zvBqPsdf+O?&H_d0nC@fGb^%1z-c`^O9-97o;O)Ri|3-AST}SCceBvT=@nnzWFbp;}B9t;F^Ya1jbmn#sZ)3w9M)4Fpb( zNRc~AC1W_AWuCvh20po+T2P-{geaONk)z(uPKpEW`3ilHh9a&qsigxtw%v;piRU_Q zGJL55y_HD~clUI+fS2261lmjM>5}+hociK|1**@{^rH80BjwEJ`Gj9GFKmLjz#3NQJ4 zXL~=D6|-wWNbCMph6@b062QYM_5}CjY>4I~k4%%`n-Rw1O;>8eAayvggf~R^5#@g7Ug#-2a z4vo2H-;{mGz934&C+*Q#f{PA#f@3G)3k!Eht+aWm=#?|`W_xRCF-*GNunb4V|mH>`l3%VhM zvVrFzU=6Dlp`bc;Z3S9GLvU{>ApVEWJDfN0_?OYDWd{n2GFI6kz4b_!)OY1xXD>_z z)0q-uXSY7SQewotSvkkPS<0&&_qp%%-s!ScuH$cZ3wVTyeujx)+_87Px|4F5b#`E( zL?xC$*GEoLjXec|!GyW*Gae?f8NW*=z_i(u}TZFLfzm6q=7^-KX=}Oi z4m~r7pyixYuq1GBsp~EqC@L-?)FH2FoO!=VK1R$ULeuS8++=wSO(-S zT&Tq*r&_pw+Bg3d4nV~Nt_hzrCVQc>i(cG&w!hNyEpDr&WET zv{Jf0|)M|_%4gh3+55U|H76&;<#T?fk5L;>~hp((P#8I-zTkO|!0 z?DO4IsPOV41!W2mdk0+GUv0B4`>7sy6zI*Xjl%Vk(n%a21NTEovyB{UT(vV;ICI7_ zWxkZMUNIdwl*HPra5-#XjleOFx~8|hnrs}f%KLSa@+R8Vwg*F*!PLHRTV^zoflJB0 z3HmHijL4F(R{xk^D)=^{fHuF_9(DBQY`4D*La^Vp1AS5_+{X27u0g1HZUw;YYCn(r z>Vx126!{JpoKFi_gR%xyj?bLki&rRCeQc1pzLY8QWWP=kcVS^6{{;bYfmcaS>vjzS z4c%xL#XaTEJKeb9ljzFXQ(08-xwc`ppBD0x?q=bw$>ZQsLsq!-Lzczx_QtOKn$&&U zvA4=o-_WBqvreig(}D_%Iz71fO}po!D0tA&4z8Sm*WImQwcev^BE=Rz!fJ^otnfg+ zUp%}S)6{39%H)&q`Llf(CnyeI7$(SnGO4mXRsdjX{O*Mj}8JT#4VIM7AjUYgaD%| z-%I81D%aDHyp%2_=Mzy_BUPsQ#-2xwUtQ<0z!a0&+YUqK@}d-wmcJV}I3Tw$Li1T^ zE%DQ;<34U`Zki{&MhwlJLqWxwd|*N!`FI*UqNmR?OK}jQ;IffUCtM3P^61ynIOGd` z41Bp#TO56+0OU3UjpQHnOGAr_SJxA*f4H*jdl1`Nu9OnwN4W3Qn&}*7H8pzy$vam0 zbUdO=hyNPA33soZSZb)DUAWAKfY_&GAx3YMN(2Oc*eENiXJMP?++2%DtU4>1yPv~D zNS7($S$AZKEcwj}KuPpg0X+RN!@!pFa!jw|ZH zeI3mYECT~k?}O`=haXgu{FW-$=`q}4L#5+x2p#PCam3s#UJ)O4jSh(d8On)nsKZjJDDNIHN8^_(wIT!E_m!|*xeloVK3ZWuqlU332+-{(j z>EH-Tzn^bjAav4;BJqQxMNq3hA^&*98qSPaOH{gzQr0G}9$Iz(txtT65a1-oS|l{a z7+cFs@n7eQymtK#5Kgt?T_Y!N}7b{&gIKG!wHW>x<-4W;{PA zm|=>VS>I^`#}J{JW3}H|O&)5Bbt7%jK~j$oL`t&q6XZWXdd|k zkWcL}R+byJv1}DU0e}0{s5jN}v+MHOJdc{2a#N(#x_Y_plunNZvzcQispM1E^)LPY zOiYq7H?CZ|YrRA5T@L8w;VzzgQ*)cmvXx%M(Z(eLrDA!R)x!_iTD2a#3lj1qN((v4 zF0ZWD5bQL)WX!2Kj1F5_f_rDCqwwL0zOZ6=#zv77*Ygz<*nM6EeP@ZlnRdf)?XSoZ z=t)bpa*ujDYy*jviFc{WWKK9FJasLHN?-uz>xee=+U+euH7*ho0ZEA5;D=*gzQ-m0 zSL;=v(@IZ`k0<0A=i{AbES7YBeMYpEhsl-}jFc457jN|;CXDOb!GnW4_sX_Bf&M-w z4r*QDVFPA1LaYW6=V#%@bu^_$y|YP_>=# zwe^;p(uDMHimnR(FPT`q{8Zcc6l6crhK{t;_V?dY*&GR#f>Ze(pH)Uw$ev26#50^9 zugFhKNI615zekvCT0*y`$`4I1xuu`|%3EF z>5l)hGK=i*=0&$J(IQ7nES2jHHnT2`I`1TiEFghy^M_B~gcIN-d_a~7_bu;#nMXN; zx&AA4q+}N3Hq}5TEaa=fDO`1-2e^kbxRNGZf!QZ|0 zLy^26Jwev)HgP7GSapxg+9I7Y6Fjom?$GerWfWhNbTNwmfxTZ=H zZO$}uXw@|7q?BzFf^-{G6sP)E8zbZ4iHQZ-Nh2?96#5QbNn5CA4gLD z%%NmsN;~n`*KO%xCTz>=i2ECcB86+1$5G77e~@*0bW!K*mp4LWoPbLx&rB=3N$%&` ztfz6#qD8Rta9D(|CfV&hoW%hs^84?E*59|-_Tl1h&R%6Z=e^bST%pR3@vq++j#X;J zf>vn$u<4d8rL3H1=y1(T15E|OB|%>S%vkf@*OIhAeEqqeP| zMfiA+_%AOn&nTz(-!h95@IClh3aJ7-B~nQ}721$P1Dq;bLibkib3?p_AdcbT7{od3 z3Wa}z5xe#!{Ay2h->eWH zA9IXtX{&-@mJAR%nhyp(`|uSxJgEI*5lT4DIX_UstJ*N0&u)6Q3u_=`t50q(@7^EV zzp%Aq1|DTGs4lT6v5H?hq3?!m(jZFo3$k%+%>vp8j~m{E0DMc{Ed^t-ut5j{@YVRq zbxt1u`y0xk@)Oc67uEvyFd5}0Gb$RF9>yYA%J?tsw0q5xCMO6cGkoMN%FBKI&XB~I z@+i(IK#^Nd@pW!|{9%wu{N*g3EgT)ckoeJB>L!vQYWt@@(X>%dV1{a6am86+O<#Ra zuf-*wEMEsK225#L`txyaKQRgnV}{?fSlWO`6rPX7$SMC6{gE7nHklkONXN zmU&pmYao8yPbF=);pgAJUJi@TU&?la@yCXrkcQSIRnQg*oekuEI;In%$2@JXgDBS@ z<=&l@zEA*ZTep#co)<5jju3#{YJbrHe(kpQF z4U}o1J;HvTi37>d6%U#>-N|sWNx`9cIKWpk=@wR%N*x|w8g1ksIL(%5Bz_b z_04&^zIr30?xlgQffaaJ(j~gKT@z)__$lG9D4pc+KovXuHw07>tNo0&=0jHu!srUk z4L#3uwS~sKH%y^x$f^Xs-aVgYtgHwTZ>`E4o|`h7u1tkw4} z7B^-0%#G;9EuKQ%kS(9?VM@&xvbr<1!)%DA#8f3puL~SZt-R_4L9(DsUZUKnJV3i_ zV14*uFxv8#pptR_#VVbgX)yRy3;pgnneJ3nq?$jwy<1Z(JYMK?* zW54t!kewi)NkLmbMOlY@Bv{O>DI%s*KBd2Y zz4QPZf;TS(8mh7FE1NdGv4eH#%GqOEEZHe4DCjh!J^oN@ zgT7w@JOB8~q0SlBl@G5=K2I-PBHc)Rsp^{+gE64;-z_fLA<)|aXY(a)ZS=_^SMab$4 z-!n-tCPaLp(Dvi&3oZ}QBqJq_0)8SnVk;MLn;Q&&=z4QaHA*IZVeEJ@%kRirW8{$B zx@<&zP6{nE?ORj1ajiklQHBVDv*l9Hfwfsr(4{DWsc=8S-E5#=tDsY6GCG}bU%xUj zsP(WE5c~0?!f8`enc_gWRN-Jc&|`(=2=8)U4(yZ zm4OiZLp>S#&a8=7&EXe+!-R|i{=ns)*s|{j{|Sc=n^~41%V`5;uHEvd$FXWT z?`&7Eee=g}mwUxlZLaxYJ4ekcoxMOd=Gng`^=cS@YLNwb!zH_cXn6|Zx4DY7 zuLE`vYOAORVfsD$;2we-V0leo`9c1Tb1f@R91R0|$BVAMp7|oq>H0MWppbyIY%bR3R^Lo%<#dU4pcqo= zsJ((h;w3eF26VP_(pFcG{W5gzrJ2aW!~K0FRvDH`Gmy(}eCd|f(`uS6b=-7Ua4Bh!E(YEdh*is;r>t23oiAM*MX%n{f?!$|YY zne{WceoNJ~>ql=JY6)9!y!wP>7{Mi!SO#V%jq38*h@Kt{Kprm&BqG)_7%0%nW*p^n zY4zJLJoc}~BQI~NR@!;$w* z>(6E0Oi*@g&qB|KcA`rEu$N3m&to3oJNK?vP-uB}MDg`y1=G8OTll%LyZw)OnudxDi1zS<;R$y-DX^> zu$Shuo@OT${U&vA5W9m@rG~!N=8%PQ^mjV8I5UdKqx%cVw46!6k36?J%~~p^-=!Lt zOi~r@2?^csY0W~YTBIcsRD;<=Q6TAq?q?9$Lk+7sx|oc-HzXK+E-8O*3X@0>l@E;^ zYKf?_ug%ICFkI+YaN^#s3Zuox6i)VCt+U;2Q2D2+W8Qm4 zvq5;t{qf_Wx?JBP{>eL)LLGFAM(ZJaz1kVt?r`Kt(w+M8>t*khN-0`zd>A0>US#st z#c7~%4AQxw@d+reV;3SHU}sAIU6RSk9C3b+p3}Kat6>r#1weKMH6O;OVb1c2IpG@1 z*AwI=(RFs+*6%VG_Q4#ke+QD_m6U&;2Ap@EWoc`(P;8EPes#h432BF~!@UNs^lG$h z;6k`)7_Gduac(K1(<9-%YI0%7P|0`}C}fREV(!S1q&%j{#z=Iwk|*UdV!Bo#Br0G5 z2}0Tvg^ahme$@Ixq6B1i5ehO~Dup&xse-1BAGe1c()ct7OYKG*WHyc@B#>mOx86d{ z_ds!Vc~N;eJx*v-TEzLq(F2EA&S%E8?^Ldv8PcFN0JOG{xQ$)Mggvwt?($!5a7eJg7Tc6ww3-5db=<%m628KZLi ziQBg)G?SL;mpoiUy0Z57RF!^hcMRioqg{K3*}ddu`o!h;QyX;mnm%O^+yA(bNQN%* z;#o~PhG+3(cb3TU(c)0+2Yo8HUI(4@zODAJ6v;okXwI8on&kxT1@_-oU5jh# zYfZg{oBA8Lz{*N@Mzy5S@2iF_kTP^OR;pT2u$IzUtzgACQfo4&O|CBN6zY%TrO${W@ zrtsX=d=Th7zN;KxBVD9Aw6+nMwZ*U6D6Z8Q9RK$E|EchJ5QnAFL$x_4>8Er^;ltR? zu`hWnfp*V(HYnbnZ((T_NAjbzRJycA30wE|KBB@Eir<12j7{S%q=h)5!oww^(BVj9 zkd|tpo3*iuhG0-#2^1W0Pl__rAaBKS!xOLU=X@8QaU*n-YCd1Lmq@V)#7GLvclP)7 zlzfL&+p{8V<3jZs=n13sUOPAHv<-)-Mi^gn2hOO?nEktNDjw=@emWa^<~ zUptZ=X|Y2(Iv%8$tO($`nbaOIqzOEhhO~pI!3iBM+ivaK zN^}PxbmEd71cRHEQ?>&fT=y;(g$RT}FR-U{W}uLNf5*}ba(&RD>uJE?jCRzXlKr$9 zV&OE1(l5#|wo zPF4~tHEX@bMJOBBL(BtA@F7W(pDhYj=CG~>IT%~3AknSDK(~*xhlR2?n zUwgx6+h4@UQ5eygq!`E6Gc>f!`dGP=6oMXBiNoucTcRAVluQH{8TJXrlyr%7n=g|s+1bUw=8;TCsXGOXGu-ax;h~FMc!*9IKl=!)|r_u z#sKJ5FtPRI;`e5oLc!oz39GnM0S7legNHlt-rGa%q{gJ7RqOl4GSv%|@~KB|k4etF z=Px%Ci%I;<-TTG=6#3PZuJN; z#}to{E9FIBveG|4d8Hno-0WBxfr655t?Y|Q}n zE%*pdQ)UeCa=onA7;eh-WYVZHRnexC&QM@3fN37y>*1T|Dia=#rwp|VY>8i!utdK4 zMbIuYjb!I1nxyTS?Z)Tb()%v53IdBeLAl^OdDmIDJrM>*cHC+V+}p*B%v%#;GP#cY^_1&_7RdP&o?&HP&Zzj6g!jUG{Se*es#Krobk=Z}p`Fxlko+rU%*ZbM z)*&J($Ugnu!Y+hQ$nmC%7lR`G;--Pl0kLDVJVYLi;jY{Ow_f1Q2sX&E1S5Y>1hY>W z_wa{zNZMqkW+)M%MrS&=IQPZd^$x(mKjNu0n5f1iUe30Ij45js6FFK-yw@%1dJ=TY zM%6{u_C$_RMgZq&vl4viWxI=XM;K&wSc_T32C7is1CI`|2-p7GaQyPEJOi<#fZx{nMxmk<25)_M-VMr)Xl?C)5X9HoIViNW zbaHa!C1Crn?7H^@EWI7n5uNuDkcoIOLvAE`q@D(<2vejP+PUFKXeUy_N{XAw5d~D7 zYNb?&Ipbx@bF&A}khZOUpFZ>%i{Z(}&~7%LuiFDa-!sy&EF=E-ij@7fa<5r_#B8F~ zUefVNbDAL%#j#&2Y{Ee=2)FGZ5PX|e$Ix1Jpvm#EPwT%Vp5fmfH`ENd-Wk&=&`|mj zX-+oYd7XoDz3)WD=x z6nK|px;QuGtY&s_wI6Z@);hL;JR=iX5alZn`0`R`@s{-Mc_ zmuv_;p?TUnhWMDMfquP{HYYr6O7Q+)sRK#Qg@uxsPPn|`Ew4g8z6#Gv2f(W04U9uPG$PyXIrlbq-?4G$W1eaHpFz4Z$4xdzJ zl!8Z);bQ#qEr_RFLe^~O8;E{}mC$x8&EsP#t|$vqNxN~7V+s>!D^t0N0LnC-0mKw( zKlZ{~b{UVSyP=b!2)#{{Mgtd1*7NzkKfbvx&UZj*eHZ4zQM`{5R}Qk{oT&rr$?yj_ zEyd4%w0`X~AOjR9ff3SIS5T^oNuCZK!lDY|X;LV_lfhG6c2i&;wSbuV`dVMcbfhI`R#r49!Cco z9rc=YHmy08P<{x@4)jN=v8mleJVDjh&TfrhIR}HFKr;1_+fnYyV9WCaX}nj-Wc@F? zgbB|wCgP?!TuYor!u=?vENoY1UTP+Ti3%iP=q7h9l1AlCF@M){@84D&090f@q|7h$C1=Cgl7*gc(obR z>MvA(Vc!(bJG9?np9aw>>6kV|OkSQmaCF$|u!?Wiu0`MeRjFBG_OvL)2WhXFSgsQ9 zI0mAo7lcU$I@)M#B;2~Enb9{(mqHW{q7fMx4pA;NwG%7EA2LwomoadjLSOP-S5tR^2TQOPr&ESZHssle`ElHh=zIn zHPCbZC49%MO-Crz`;%HrZHWi>OOXd2Z){*MI=j2ew6Hu5&!LN+@~Ih5LpFdzh|>#c zR(A?1J^~hFDUoKS(6ia|#<-!X#wl90t)?H(A_XrD`$tv6Q$uHXN{cBD1E-8`o^>=650w5aSbnM@Y6f}uL5>+emjL+ zie$TU6k(>6%cW;^Lf;O*K?+)A3h0eq0Gg5@%O?~vCQsA57o$r^H>saa+K+VEduvp3 zsTw*B@~wOZT$|{0BFf{5HS=2)WdG%Q&Cg7uWX1?F zN$9-$p%=7kY&VnnY0@0;(H!Hyv;5!i*^AA(Y{c$Z8!n1Y`ed2<`3wlYAz0r?9K6|R zJ;4Z^+{_er1R}>VLGOo+@gf&mVMbfna8D)_gteDN?Mkbfviz2R8=j|+-2$6QzcTrT! z@U0y?gel-Pa1wAzv>Cj-@*#NV630Xo(^q7_DZ*Tx=ougd>lZpJ*&z_aA}o}DepN*Q znD0aHUf5Z2%v;C2C|k3h&@E0ZKX3+&wtNUV?MD9bLZvbvs-&!wWxE$;ft5?4!WG&DwsLKJEso`=P?~YZsal&r99=~ z*9t*Iu(M}?Pnd1AkB{PPRXUy zvIeBu45i2=bo6=A7BpPyPpJb3K0wWW{BDuF)B zv`rkp`Us#1bn{yCAuyoQ2{#6MU`ly{`XTt;QWC*c#ilMzZVGCZU$q8ku_q zj@&1v7LbzupPygPAK)!syfDkzMzrI$`1`dW6{QinYaBT*|M~3`Fr7B)-*JceN&LvT zD9qGTu|MR-tNg#O-g6KXAgpZo{r8ev*_ZdXeE4?wwZ{LxN+V0CB-q&SF7bIBU-FVS z$u57f>>W^w%2uPRSs-^9WdHvH&bcb)|qyA2?V=l}epi`fu;)f&F35Hd>SjTNt1 z|Eo0l-#>NH2|1RtHSQM}^&4TgJZRq2z=XX{hLjbWm(v&d~0~`6m|6F60!&uEBwZ;RcuxiHaf8${iyKTV#zS>i#83(EJ zJMkvxf5gI+Pn!Mj$EUiaTH+q4Eqt#ZKl{($g8$|DQPSc2kTkP(!~n~9(=(61`RYaA z{`b3ra`Cz1ZN>P!xVlGF{|iXnag6xC-+@LRvP(R8o)4)*QXmJ>x{y?lOsoH|AAEAv zFU=wt)htV^Un;62w0_h6_qqB7B5H@-coS+llVRxj5fu(d0*4atf0if{%l9@Ixhoqk z`Gfy*vQ5)?{jD$FNCEa=zs?>FIM;Q=t(t7V9Ypp$o&Wn`D~roRb0ZwiC8<7E01=te zvBzI*;DfIofJK_?zy21KCHIlxHeOphKki`7HKDTpeNIRxgqQi>OT)9D6gTEFJ$1%G zWaAN>{GY#5wfI)%GTP{y$fdyg`B#%`%apvH$%GhVgc8;@Of0|F5iT zk7s)QIplU&N40JocP%sbTg6mz%RP!xCyC5lCTWR= z(?TMoV{VPijLqgeU;R$D{hq(RukG_Z&-ZzMKbP{|^H~HSbk8=BXIVm`Q>+|w3e;ALsA%_<2Ej>^Ts0qSp%FnqKtNs_ za;u)A@))c&7n=Ub79-%6DZ8WEfI-60?^{`8HpuJN`Ms7z)Jzr zNTR&>StlU2u07mU7HBNpxBDwam1BjW?jCmn{$b8>02zuctf$4!@28u3T z6&wsB&yuykf$PV_?L1~0}eVfk`R1Xsk9h0ClCge_Y`SH09 zQEmBUHxMBySk)@<5SSNEd0OZW8|X%Jo*Jtaqa;K@-3zJA7m=5sCtKq(Vjz3nkZJ(- z3e$&=K?SM=noepjcr}Bqt#aNtc8zD~huk!FZO6Ga0u+!hwRO$P=VCC{8E-}7`%by4 zV!)s-WyKaC+e(G9^{;#mv7M4hK-QWXZd{Uu$ZR&VJ%o0p308C(a{n<7CgoXd>(>`^ak%USg zhACqN8F0?P*sO|9HK7kUKwA$2&B`8x_v9=@p1-@Kn_}Ze#Nt3z*D2GLFJ5K#+T=b& zncQKZ-1c*`8eO3fH-SbDP$Pu;dZ+HXlUjSA3iUdDNah47W*rLQ|K{eKYrZ(%qIRL5 zc2+23us_pXpdYw21BW=tXqR1{Nk}7e`MFK@`|p1vEM{#S8^h97<_pHFZI22Ylug8r zv|eM|)LZY*NU$WO6xIDdj^JDeK&a~GCZGhuT2 zV!%5=yYX}aP{aF;_Je7C-=1&v1+QT}bNwooY>$JBy`Eo1@Y((lFmcY45DOSFQMRCb zJB=V*o?|O#LDY={OuqnC+Eb#)GgiIRxjWHUrEvM zHAe(CGL+*K*D!^jc0n?Qe?DoNc|q3B$8;N6sXQVIoNmlziNz5vnt<&Skiln>&+%YI zk2sucts9t{g*=@Fo9rCWCN^lT_{HLmNKVxMq|%AL&ri8z@i>WL!K9i@d%hE%_0Z`g z=gcgZ=r67|v)X2AI-aedx>EBvm6J?Xe73Xz87d~1@5FvIsf)d80 zzf(heD-FJDg1^#V8u$7o74ax#&Z>IyHH+C6Tq~mYR)B;qp!C{Avu?Ni$#ANiQD;gV z2VamKk1T6Y5}DkOeq{Np@oxwljm}{t<}A`D_SB&3K6L6NpDWJIZ|OYnoAO73Q?t87brwSZpH30jj+YcPpJ5b!euJFAD6NUdm&kz=r75C!9n)~uDxu2^~i-#TW&fY#Y zXIjH07~UTgW7Hk(XJ?W40W+JU=Y=PV0VP$y7t1u(bCwQ17Zj%KbWwy+D)1I&O=NK6jSIFZ+pdD)1 z3KZ=ZxvCbG0$6IB|7qkagJaDosS=PEr=!t>6(T?jE;SA&Ij@sXX<>32y|3MT1yfd! z9N@y=l@m)a1D0AMC?CpiflyBLU0q|V$CI_g+EUi*P4LGq*7C-trwQ=THvz(<+q6UG zuppM9$Ftu!cKlvU$hTgC5-(>|SkB1OAGC=-+tFHv&T?w^xhEym6b})C4rv5BKb;&Y z5rtC!%~u()r*TEC7C>?f+=Kj!_s>K38vD%j5HUp3(w3FLZFDAVfxgK4*3Q!L)wNkU zhf6$mBeXMlgN^li#sYNU(y1lkW_e5e6>M(jxO!v(*uG(^K_v9J`?Vazo9(_g8p3DF667eJk%a$+@qB2#vDE z{a*SOeoYtW$nus`_%H8s+u046ELs~L&XmQbb*6raF^Poejtz0gHe%31>(AJc+2G}RyaBURJO!$jyXI1vLic$ z`G9xE(|g|UdKse>^<$Xs&Ih+iQ6|~wzsoicoH=;(;0zN7D*8o2r|rh0GWP_gXa)}@w|)xOt9`EDaNr4^jlDV= zSJdCrFTHGB$@ewYXN-57umc~K-QpbM?eb7B@;UWWMc4T9oYVNI&&4 zuxr$MoWy$kZob+#0Bb6MnEugJ4==oIP;}T;Cd2)q4$iK>ann%7t4noG0*I((WafDF z!4OIa>YrC0nHiS+8R+8S?p9xSIUBxV*+hjux=};}Y~zvH0}ry!ZGJl8W6RKbtd1z` qFTXvpFoPS%v(^5Cas?U|J*5&CxZ_}AUC1-&*Iz8G%^#bgBmV_9=R*Sk literal 0 HcmV?d00001 From f15a577a7c1637c0f422e497d94c725de3e0596b Mon Sep 17 00:00:00 2001 From: Arun Agrawal Date: Wed, 17 Oct 2018 14:49:45 -0700 Subject: [PATCH 023/228] add release notes for 3.11.34 --- CHANGELOG.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index aa4077bdb..90d42db40 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,9 @@ # Changelog +##2018/10/17 3.11.34 +* (#739) BugFix: Null pointer exception while traversing filesystem. +* (#737) Google java format validator addition. Use ./gradlew goJF to fix the formatting before sending PR. +* (#740) Last but not least, a new logo for Priam. + ## 2018/10/08 3.11.33 ***WARNING*** THIS IS A BREAKING RELEASE ### New Feature From 28e24a499df1231ee1de62bb49f6de52bf0542ae Mon Sep 17 00:00:00 2001 From: Arun Agrawal Date: Wed, 17 Oct 2018 14:50:03 -0700 Subject: [PATCH 024/228] Update CHANGELOG.md --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 90d42db40..7993a534b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,5 @@ # Changelog -##2018/10/17 3.11.34 +## 2018/10/17 3.11.34 * (#739) BugFix: Null pointer exception while traversing filesystem. * (#737) Google java format validator addition. Use ./gradlew goJF to fix the formatting before sending PR. * (#740) Last but not least, a new logo for Priam. From 17fb71a39d41805447d9793ea5d32eb85580e76a Mon Sep 17 00:00:00 2001 From: Arun Agrawal Date: Sat, 13 Oct 2018 10:33:27 -0700 Subject: [PATCH 025/228] Aggregate Instance Data --- .../java/com/netflix/priam/PriamServer.java | 20 +- .../com/netflix/priam/aws/AWSMembership.java | 52 +- .../com/netflix/priam/aws/S3BackupPath.java | 4 +- .../priam/aws/S3CrossAccountFileSystem.java | 8 +- .../priam/aws/S3EncryptedFileSystem.java | 6 +- .../com/netflix/priam/aws/S3FileSystem.java | 6 +- .../netflix/priam/aws/S3PrefixIterator.java | 4 +- .../netflix/priam/aws/SDBInstanceFactory.java | 14 +- .../aws/auth/EC2RoleAssumptionCredential.java | 12 +- .../priam/backup/AbstractBackupPath.java | 12 +- .../priam/backupv2/MetaFileWriterBuilder.java | 4 +- .../netflix/priam/config/IConfiguration.java | 144 +++-- .../priam/config/PriamConfiguration.java | 597 ++++-------------- .../configSource/AbstractConfigSource.java | 2 +- .../configSource/CompositeConfigSource.java | 4 +- .../priam/configSource/IConfigSource.java | 2 +- .../configSource/PropertiesConfigSource.java | 4 +- .../configSource/SimpleDBConfigSource.java | 6 +- .../SystemPropertiesConfigSource.java | 4 +- .../identity/AwsInstanceEnvIdentity.java | 60 -- .../netflix/priam/identity/DoubleRing.java | 23 +- .../priam/identity/InstanceEnvIdentity.java | 43 -- .../priam/identity/InstanceIdentity.java | 17 +- .../identity/config/AWSInstanceInfo.java | 214 +++++++ .../config/AWSVpcInstanceDataRetriever.java | 47 -- .../AwsClassicInstanceDataRetriever.java | 26 - .../config/InstanceDataRetrieverBase.java | 79 --- ...ceDataRetriever.java => InstanceInfo.java} | 54 +- ...aRetriever.java => LocalInstanceInfo.java} | 32 +- .../identity/token/DeadTokenRetriever.java | 20 +- .../identity/token/NewTokenRetriever.java | 29 +- .../token/PreGeneratedTokenRetriever.java | 16 +- .../AWSSnsNotificationService.java | 8 +- .../notification/BackupNotificationMgr.java | 10 +- .../priam/resources/BackupServlet.java | 282 +-------- .../priam/resources/CassandraConfig.java | 15 +- .../resources/PriamInstanceResource.java | 11 +- .../priam/resources/RestoreServlet.java | 152 +---- .../priam/restore/AbstractRestore.java | 22 +- .../netflix/priam/tuner/StandardTuner.java | 14 +- .../com/netflix/priam/tuner/dse/DseTuner.java | 8 +- priam/src/main/resources/Priam.properties | 1 + .../java/com/netflix/priam/TestModule.java | 10 +- .../netflix/priam/backup/BRTestModule.java | 12 +- .../priam/backup/FakeBackupFileSystem.java | 14 +- .../netflix/priam/backup/TestBackupFile.java | 17 +- .../priam/backup/TestFileIterator.java | 90 ++- .../com/netflix/priam/backup/TestRestore.java | 75 +-- .../priam/backup/TestS3FileSystem.java | 19 +- .../priam/backup/identity/DoubleRingTest.java | 8 +- .../identity/FakeInstanceEnvIdentity.java | 38 -- .../backup/identity/InstanceIdentityTest.java | 14 +- .../backup/identity/InstanceTestUtils.java | 62 +- .../priam/config/FakeConfiguration.java | 248 +------- .../CompositeConfigSourceTest.java | 2 +- .../PropertiesConfigSourceTest.java | 4 +- .../SystemPropertiesConfigSourceTest.java | 2 +- .../CassandraProcessManagerTest.java | 3 +- .../identity/FakePriamInstanceFactory.java | 7 +- .../identity/config/FakeInstanceInfo.java | 97 +++ .../priam/resources/BackupServletTest.java | 363 +---------- .../priam/resources/CassandraConfigTest.java | 18 +- .../priam/resources/PriamConfigTest.java | 2 +- .../resources/PriamInstanceResourceTest.java | 12 +- .../services/TestSnapshotMetaService.java | 14 +- .../netflix/priam/stream/StreamingTest.java | 20 +- .../priam/tuner/StandardTunerTest.java | 14 +- .../netflix/priam/tuner/dse/DseTunerTest.java | 2 - 68 files changed, 1011 insertions(+), 2244 deletions(-) delete mode 100644 priam/src/main/java/com/netflix/priam/identity/AwsInstanceEnvIdentity.java delete mode 100644 priam/src/main/java/com/netflix/priam/identity/InstanceEnvIdentity.java create mode 100644 priam/src/main/java/com/netflix/priam/identity/config/AWSInstanceInfo.java delete mode 100644 priam/src/main/java/com/netflix/priam/identity/config/AWSVpcInstanceDataRetriever.java delete mode 100644 priam/src/main/java/com/netflix/priam/identity/config/AwsClassicInstanceDataRetriever.java delete mode 100644 priam/src/main/java/com/netflix/priam/identity/config/InstanceDataRetrieverBase.java rename priam/src/main/java/com/netflix/priam/identity/config/{InstanceDataRetriever.java => InstanceInfo.java} (59%) rename priam/src/main/java/com/netflix/priam/identity/config/{LocalInstanceDataRetriever.java => LocalInstanceInfo.java} (67%) delete mode 100644 priam/src/test/java/com/netflix/priam/backup/identity/FakeInstanceEnvIdentity.java create mode 100644 priam/src/test/java/com/netflix/priam/identity/config/FakeInstanceInfo.java diff --git a/priam/src/main/java/com/netflix/priam/PriamServer.java b/priam/src/main/java/com/netflix/priam/PriamServer.java index ec795168c..d9b598d7b 100644 --- a/priam/src/main/java/com/netflix/priam/PriamServer.java +++ b/priam/src/main/java/com/netflix/priam/PriamServer.java @@ -48,7 +48,7 @@ public class PriamServer { private final PriamScheduler scheduler; private final IConfiguration config; private final IBackupRestoreConfig backupRestoreConfig; - private final InstanceIdentity id; + private final InstanceIdentity instanceIdentity; private final Sleeper sleeper; private final ICassandraProcess cassProcess; private final RestoreContext restoreContext; @@ -67,14 +67,14 @@ public PriamServer( this.config = config; this.backupRestoreConfig = backupRestoreConfig; this.scheduler = scheduler; - this.id = id; + this.instanceIdentity = id; this.sleeper = sleeper; this.cassProcess = cassProcess; this.restoreContext = restoreContext; } public void initialize() throws Exception { - if (id.getInstance().isOutOfService()) return; + if (instanceIdentity.getInstance().isOutOfService()) return; // start to schedule jobs scheduler.start(); @@ -84,13 +84,14 @@ public void initialize() throws Exception { scheduler.runTaskNow(UpdateSecuritySettings.class); // sleep for 150 sec if this is a new node with new IP for SG to be updated by other // seed nodes - if (id.isReplace() || id.isTokenPregenerated()) sleeper.sleep(150 * 1000); + if (instanceIdentity.isReplace() || instanceIdentity.isTokenPregenerated()) + sleeper.sleep(150 * 1000); else if (UpdateSecuritySettings.firstTimeUpdated) sleeper.sleep(60 * 1000); scheduler.addTask( UpdateSecuritySettings.JOBNAME, UpdateSecuritySettings.class, - UpdateSecuritySettings.getTimer(id)); + UpdateSecuritySettings.getTimer(instanceIdentity)); } // Run the task to tune Cassandra @@ -100,12 +101,13 @@ public void initialize() throws Exception { // set it off, set backup hour to -1) or set backup cron to "-1" if (SnapshotBackup.getTimer(config) != null && (CollectionUtils.isEmpty(config.getBackupRacs()) - || config.getBackupRacs().contains(config.getRac()))) { + || config.getBackupRacs() + .contains(instanceIdentity.getInstanceInfo().getRac()))) { scheduler.addTask( SnapshotBackup.JOBNAME, SnapshotBackup.class, SnapshotBackup.getTimer(config)); // Start the Incremental backup schedule if enabled - if (config.isIncrBackup()) { + if (config.isIncrementalBackupEnabled()) { scheduler.addTask( IncrementalBackup.JOBNAME, IncrementalBackup.class, @@ -193,8 +195,8 @@ private void setUpSnapshotService() throws Exception { } } - public InstanceIdentity getId() { - return id; + public InstanceIdentity getInstanceIdentity() { + return instanceIdentity; } public PriamScheduler getScheduler() { diff --git a/priam/src/main/java/com/netflix/priam/aws/AWSMembership.java b/priam/src/main/java/com/netflix/priam/aws/AWSMembership.java index 0e9726d9b..482580dc1 100644 --- a/priam/src/main/java/com/netflix/priam/aws/AWSMembership.java +++ b/priam/src/main/java/com/netflix/priam/aws/AWSMembership.java @@ -30,7 +30,8 @@ import com.netflix.priam.config.IConfiguration; import com.netflix.priam.cred.ICredential; import com.netflix.priam.identity.IMembership; -import com.netflix.priam.identity.InstanceEnvIdentity; +import com.netflix.priam.identity.InstanceIdentity; +import com.netflix.priam.identity.config.InstanceInfo; import java.util.*; import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; @@ -44,7 +45,7 @@ public class AWSMembership implements IMembership { private static final Logger logger = LoggerFactory.getLogger(AWSMembership.class); private final IConfiguration config; private final ICredential provider; - private final InstanceEnvIdentity insEnvIdentity; + private final InstanceIdentity instanceIdentity; private final ICredential crossAccountProvider; @Inject @@ -52,10 +53,10 @@ public AWSMembership( IConfiguration config, ICredential provider, @Named("awsec2roleassumption") ICredential crossAccountProvider, - InstanceEnvIdentity insEnvIdentity) { + InstanceIdentity instanceIdentity) { this.config = config; this.provider = provider; - this.insEnvIdentity = insEnvIdentity; + this.instanceIdentity = instanceIdentity; this.crossAccountProvider = crossAccountProvider; } @@ -64,7 +65,7 @@ public List getRacMembership() { AmazonAutoScaling client = null; try { List asgNames = new ArrayList<>(); - asgNames.add(config.getASGName()); + asgNames.add(instanceIdentity.getInstanceInfo().getAutoScalingGroup()); asgNames.addAll(Arrays.asList(config.getSiblingASGNames().split("\\s*,\\s*"))); client = getAutoScalingClient(); DescribeAutoScalingGroupsRequest asgReq = @@ -85,7 +86,7 @@ public List getRacMembership() { logger.info( String.format( "Querying Amazon returned following instance in the RAC: %s, ASGs: %s --> %s", - config.getRac(), + instanceIdentity.getInstanceInfo().getRac(), StringUtils.join(asgNames, ","), StringUtils.join(instanceIds, ","))); } @@ -103,7 +104,8 @@ public int getRacMembershipSize() { client = getAutoScalingClient(); DescribeAutoScalingGroupsRequest asgReq = new DescribeAutoScalingGroupsRequest() - .withAutoScalingGroupNames(config.getASGName()); + .withAutoScalingGroupNames( + instanceIdentity.getInstanceInfo().getAutoScalingGroup()); DescribeAutoScalingGroupsResult res = client.describeAutoScalingGroups(asgReq); int size = 0; for (AutoScalingGroup asg : res.getAutoScalingGroups()) { @@ -121,7 +123,7 @@ public List getCrossAccountRacMembership() { AmazonAutoScaling client = null; try { List asgNames = new ArrayList<>(); - asgNames.add(config.getASGName()); + asgNames.add(instanceIdentity.getInstanceInfo().getAutoScalingGroup()); asgNames.addAll(Arrays.asList(config.getSiblingASGNames().split("\\s*,\\s*"))); client = getCrossAccountAutoScalingClient(); DescribeAutoScalingGroupsRequest asgReq = @@ -142,7 +144,8 @@ public List getCrossAccountRacMembership() { logger.info( String.format( "Querying Amazon returned following instance in the cross-account ASG: %s --> %s", - config.getRac(), StringUtils.join(instanceIds, ","))); + instanceIdentity.getInstanceInfo().getRac(), + StringUtils.join(instanceIds, ","))); } return instanceIds; } finally { @@ -155,6 +158,11 @@ public int getRacCount() { return config.getRacs().size(); } + private boolean isClassic() { + return instanceIdentity.getInstanceInfo().getInstanceEnvironment() + == InstanceInfo.InstanceEnvironment.CLASSIC; + } + /** * Adding peers' IPs as ingress to the running instance SG. The running instance could be in * "classic" or "vpc" @@ -171,7 +179,7 @@ public void addACL(Collection listIPs, int from, int to) { .withIpRanges(listIPs) .withToPort(to)); - if (this.insEnvIdentity.isClassic()) { + if (isClassic()) { client.authorizeSecurityGroupIngress( new AuthorizeSecurityGroupIngressRequest( config.getACLGroupName(), ipPermissions)); @@ -206,7 +214,10 @@ protected String getVpcGoupId() { client = getEc2Client(); Filter nameFilter = new Filter().withName("group-name").withValues(config.getACLGroupName()); // SG - Filter vpcFilter = new Filter().withName("vpc-id").withValues(config.getVpcId()); + Filter vpcFilter = + new Filter() + .withName("vpc-id") + .withValues(instanceIdentity.getInstanceInfo().getVpcId()); DescribeSecurityGroupsRequest req = new DescribeSecurityGroupsRequest().withFilters(nameFilter, vpcFilter); @@ -216,13 +227,13 @@ protected String getVpcGoupId() { "got group-id:{} for group-name:{},vpc-id:{}", group.getGroupId(), config.getACLGroupName(), - config.getVpcId()); + instanceIdentity.getInstanceInfo().getVpcId()); return group.getGroupId(); } logger.error( "unable to get group-id for group-name={} vpc-id={}", config.getACLGroupName(), - config.getVpcId()); + instanceIdentity.getInstanceInfo().getVpcId()); return ""; } finally { if (client != null) client.shutdown(); @@ -242,7 +253,7 @@ public void removeACL(Collection listIPs, int from, int to) { .withIpRanges(listIPs) .withToPort(to)); - if (this.insEnvIdentity.isClassic()) { + if (isClassic()) { client.revokeSecurityGroupIngress( new RevokeSecurityGroupIngressRequest( config.getACLGroupName(), ipPermissions)); @@ -276,7 +287,7 @@ public List listACL(int from, int to) { client = getEc2Client(); List ipPermissions = new ArrayList<>(); - if (this.insEnvIdentity.isClassic()) { + if (isClassic()) { DescribeSecurityGroupsRequest req = new DescribeSecurityGroupsRequest() @@ -293,7 +304,7 @@ public List listACL(int from, int to) { Filter nameFilter = new Filter().withName("group-name").withValues(config.getACLGroupName()); - String vpcid = config.getVpcId(); + String vpcid = instanceIdentity.getInstanceInfo().getVpcId(); if (vpcid == null || vpcid.isEmpty()) { throw new IllegalStateException( "vpcid is null even though instance is running in vpc."); @@ -325,7 +336,8 @@ public void expandRacMembership(int count) { client = getAutoScalingClient(); DescribeAutoScalingGroupsRequest asgReq = new DescribeAutoScalingGroupsRequest() - .withAutoScalingGroupNames(config.getASGName()); + .withAutoScalingGroupNames( + instanceIdentity.getInstanceInfo().getAutoScalingGroup()); DescribeAutoScalingGroupsResult res = client.describeAutoScalingGroups(asgReq); AutoScalingGroup asg = res.getAutoScalingGroups().get(0); UpdateAutoScalingGroupRequest ureq = new UpdateAutoScalingGroupRequest(); @@ -342,21 +354,21 @@ public void expandRacMembership(int count) { protected AmazonAutoScaling getAutoScalingClient() { return AmazonAutoScalingClientBuilder.standard() .withCredentials(provider.getAwsCredentialProvider()) - .withRegion(config.getDC()) + .withRegion(instanceIdentity.getInstanceInfo().getRegion()) .build(); } protected AmazonAutoScaling getCrossAccountAutoScalingClient() { return AmazonAutoScalingClientBuilder.standard() .withCredentials(crossAccountProvider.getAwsCredentialProvider()) - .withRegion(config.getDC()) + .withRegion(instanceIdentity.getInstanceInfo().getRegion()) .build(); } protected AmazonEC2 getEc2Client() { return AmazonEC2ClientBuilder.standard() .withCredentials(provider.getAwsCredentialProvider()) - .withRegion(config.getDC()) + .withRegion(instanceIdentity.getInstanceInfo().getRegion()) .build(); } } diff --git a/priam/src/main/java/com/netflix/priam/aws/S3BackupPath.java b/priam/src/main/java/com/netflix/priam/aws/S3BackupPath.java index 6000e09cc..37e4f9216 100644 --- a/priam/src/main/java/com/netflix/priam/aws/S3BackupPath.java +++ b/priam/src/main/java/com/netflix/priam/aws/S3BackupPath.java @@ -97,7 +97,7 @@ public void parsePartialPrefix(String remoteFilePath) { @Override public String remotePrefix(Date start, Date end, String location) { StringBuilder buff = new StringBuilder(clusterPrefix(location)); - token = factory.getInstance().getToken(); + token = instanceIdentity.getInstance().getToken(); buff.append(token).append(S3BackupPath.PATH_SEP); // match the common characters to prefix. buff.append(match(start, end)); @@ -110,7 +110,7 @@ public String clusterPrefix(String location) { String[] elements = location.split(String.valueOf(S3BackupPath.PATH_SEP)); if (elements.length <= 1) { baseDir = config.getBackupLocation(); - region = config.getDC(); + region = instanceIdentity.getInstanceInfo().getRegion(); clusterName = config.getAppName(); } else { assert elements.length >= 4 : "Too few elements in path " + location; diff --git a/priam/src/main/java/com/netflix/priam/aws/S3CrossAccountFileSystem.java b/priam/src/main/java/com/netflix/priam/aws/S3CrossAccountFileSystem.java index 0f0befead..0676b859a 100755 --- a/priam/src/main/java/com/netflix/priam/aws/S3CrossAccountFileSystem.java +++ b/priam/src/main/java/com/netflix/priam/aws/S3CrossAccountFileSystem.java @@ -21,6 +21,7 @@ import com.netflix.priam.aws.auth.IS3Credential; import com.netflix.priam.backup.IBackupFileSystem; import com.netflix.priam.config.IConfiguration; +import com.netflix.priam.identity.config.InstanceInfo; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -41,16 +42,19 @@ public class S3CrossAccountFileSystem { private final S3FileSystem s3fs; private final IConfiguration config; private final IS3Credential s3Credential; + private final InstanceInfo instanceInfo; @Inject public S3CrossAccountFileSystem( @Named("backup") IBackupFileSystem fs, @Named("awss3roleassumption") IS3Credential s3Credential, - IConfiguration config) { + IConfiguration config, + InstanceInfo instanceInfo) { this.s3fs = (S3FileSystem) fs; this.config = config; this.s3Credential = s3Credential; + this.instanceInfo = instanceInfo; } public IBackupFileSystem getBackupFileSystem() { @@ -68,7 +72,7 @@ public AmazonS3 getCrossAcctS3Client() { this.s3Client = AmazonS3Client.builder() .withCredentials(s3Credential.getAwsCredentialProvider()) - .withRegion(config.getDC()) + .withRegion(instanceInfo.getRegion()) .build(); } catch (Exception e) { diff --git a/priam/src/main/java/com/netflix/priam/aws/S3EncryptedFileSystem.java b/priam/src/main/java/com/netflix/priam/aws/S3EncryptedFileSystem.java index 91645b7ef..7794e9007 100755 --- a/priam/src/main/java/com/netflix/priam/aws/S3EncryptedFileSystem.java +++ b/priam/src/main/java/com/netflix/priam/aws/S3EncryptedFileSystem.java @@ -30,6 +30,7 @@ import com.netflix.priam.config.IConfiguration; import com.netflix.priam.cred.ICredential; import com.netflix.priam.cryptography.IFileCryptography; +import com.netflix.priam.identity.config.InstanceInfo; import com.netflix.priam.merics.BackupMetrics; import com.netflix.priam.notification.BackupNotificationMgr; import java.io.*; @@ -55,14 +56,15 @@ public S3EncryptedFileSystem( ICredential cred, @Named("filecryptoalgorithm") IFileCryptography fileCryptography, BackupMetrics backupMetrics, - BackupNotificationMgr backupNotificationMgr) { + BackupNotificationMgr backupNotificationMgr, + InstanceInfo instanceInfo) { super(pathProvider, compress, config, backupMetrics, backupNotificationMgr); this.encryptor = fileCryptography; super.s3Client = AmazonS3Client.builder() .withCredentials(cred.getAwsCredentialProvider()) - .withRegion(config.getDC()) + .withRegion(instanceInfo.getRegion()) .build(); } diff --git a/priam/src/main/java/com/netflix/priam/aws/S3FileSystem.java b/priam/src/main/java/com/netflix/priam/aws/S3FileSystem.java index 51df86653..5622d857f 100644 --- a/priam/src/main/java/com/netflix/priam/aws/S3FileSystem.java +++ b/priam/src/main/java/com/netflix/priam/aws/S3FileSystem.java @@ -29,6 +29,7 @@ import com.netflix.priam.backup.RangeReadInputStream; import com.netflix.priam.compress.ICompression; import com.netflix.priam.config.IConfiguration; +import com.netflix.priam.identity.config.InstanceInfo; import com.netflix.priam.merics.BackupMetrics; import com.netflix.priam.notification.BackupNotificationMgr; import com.netflix.priam.utils.BoundedExponentialRetryCallable; @@ -55,12 +56,13 @@ public S3FileSystem( ICompression compress, final IConfiguration config, BackupMetrics backupMetrics, - BackupNotificationMgr backupNotificationMgr) { + BackupNotificationMgr backupNotificationMgr, + InstanceInfo instanceInfo) { super(pathProvider, compress, config, backupMetrics, backupNotificationMgr); s3Client = AmazonS3Client.builder() .withCredentials(cred.getAwsCredentialProvider()) - .withRegion(config.getDC()) + .withRegion(instanceInfo.getRegion()) .build(); } diff --git a/priam/src/main/java/com/netflix/priam/aws/S3PrefixIterator.java b/priam/src/main/java/com/netflix/priam/aws/S3PrefixIterator.java index 2c6d55254..5d4583847 100644 --- a/priam/src/main/java/com/netflix/priam/aws/S3PrefixIterator.java +++ b/priam/src/main/java/com/netflix/priam/aws/S3PrefixIterator.java @@ -24,6 +24,7 @@ import com.google.inject.Provider; import com.netflix.priam.backup.AbstractBackupPath; import com.netflix.priam.config.IConfiguration; +import com.netflix.priam.identity.config.InstanceInfo; import java.text.SimpleDateFormat; import java.util.Date; import java.util.Iterator; @@ -48,6 +49,7 @@ public class S3PrefixIterator implements Iterator { private final SimpleDateFormat datefmt = new SimpleDateFormat("yyyyMMdd"); private ObjectListing objectListing = null; final Date date; + @Inject private InstanceInfo instanceInfo; @Inject public S3PrefixIterator( @@ -119,7 +121,7 @@ private String remotePrefix(String location) { String[] elements = location.split(String.valueOf(S3BackupPath.PATH_SEP)); if (elements.length <= 1) { buff.append(config.getBackupLocation()).append(S3BackupPath.PATH_SEP); - buff.append(config.getDC()).append(S3BackupPath.PATH_SEP); + buff.append(instanceInfo.getRegion()).append(S3BackupPath.PATH_SEP); buff.append(config.getAppName()).append(S3BackupPath.PATH_SEP); } else { assert elements.length >= 4 : "Too few elements in path " + location; diff --git a/priam/src/main/java/com/netflix/priam/aws/SDBInstanceFactory.java b/priam/src/main/java/com/netflix/priam/aws/SDBInstanceFactory.java index bf4795068..53f05a281 100644 --- a/priam/src/main/java/com/netflix/priam/aws/SDBInstanceFactory.java +++ b/priam/src/main/java/com/netflix/priam/aws/SDBInstanceFactory.java @@ -22,22 +22,28 @@ import com.netflix.priam.config.IConfiguration; import com.netflix.priam.identity.IPriamInstanceFactory; import com.netflix.priam.identity.PriamInstance; +import com.netflix.priam.identity.config.InstanceInfo; import java.util.*; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -/** SimpleDB based instance factory. Requires 'InstanceIdentity' domain to be created ahead */ +/** + * SimpleDB based instance instanceIdentity. Requires 'InstanceIdentity' domain to be created ahead + */ @Singleton public class SDBInstanceFactory implements IPriamInstanceFactory { private static final Logger logger = LoggerFactory.getLogger(SDBInstanceFactory.class); private final IConfiguration config; private final SDBInstanceData dao; + private final InstanceInfo instanceInfo; @Inject - public SDBInstanceFactory(IConfiguration config, SDBInstanceData dao) { + public SDBInstanceFactory( + IConfiguration config, SDBInstanceData dao, InstanceInfo instanceInfo) { this.config = config; this.dao = dao; + this.instanceInfo = instanceInfo; } @Override @@ -69,7 +75,7 @@ public PriamInstance create( // remove old data node which are dead. if (app.endsWith("-dead")) { try { - PriamInstance oldData = dao.getInstance(app, config.getDC(), id); + PriamInstance oldData = dao.getInstance(app, instanceInfo.getRegion(), id); // clean up a very old data... if (null != oldData && oldData.getUpdatetime() @@ -140,7 +146,7 @@ private PriamInstance makePriamInstance( ins.setHostIP(ip); ins.setId(id); ins.setInstanceId(instanceID); - ins.setDC(config.getDC()); + ins.setDC(instanceInfo.getRegion()); ins.setToken(token); ins.setVolumes(v); return ins; diff --git a/priam/src/main/java/com/netflix/priam/aws/auth/EC2RoleAssumptionCredential.java b/priam/src/main/java/com/netflix/priam/aws/auth/EC2RoleAssumptionCredential.java index 744e0903c..45131f622 100644 --- a/priam/src/main/java/com/netflix/priam/aws/auth/EC2RoleAssumptionCredential.java +++ b/priam/src/main/java/com/netflix/priam/aws/auth/EC2RoleAssumptionCredential.java @@ -18,21 +18,22 @@ import com.google.inject.Inject; import com.netflix.priam.config.IConfiguration; import com.netflix.priam.cred.ICredential; -import com.netflix.priam.identity.InstanceEnvIdentity; +import com.netflix.priam.identity.InstanceIdentity; +import com.netflix.priam.identity.config.InstanceInfo; public class EC2RoleAssumptionCredential implements ICredential { private static final String AWS_ROLE_ASSUMPTION_SESSION_NAME = "AwsRoleAssumptionSession"; private final ICredential cred; private final IConfiguration config; - private final InstanceEnvIdentity insEnvIdentity; + private final InstanceIdentity instanceIdentity; private AWSCredentialsProvider stsSessionCredentialsProvider; @Inject public EC2RoleAssumptionCredential( - ICredential cred, IConfiguration config, InstanceEnvIdentity insEnvIdentity) { + ICredential cred, IConfiguration config, InstanceIdentity instanceIdentity) { this.cred = cred; this.config = config; - this.insEnvIdentity = insEnvIdentity; + this.instanceIdentity = instanceIdentity; } @Override @@ -47,7 +48,8 @@ public AWSCredentialsProvider getAwsCredentialProvider() { * current environment is VPC, then the assumed role is for EC2 classic, and * vice versa. */ - if (this.insEnvIdentity.isClassic()) { + if (instanceIdentity.getInstanceInfo().getInstanceEnvironment() + == InstanceInfo.InstanceEnvironment.CLASSIC) { roleArn = this.config.getClassicEC2RoleAssumptionArn(); // Env is EC2 classic --> IAM assumed role for VPC created } else { diff --git a/priam/src/main/java/com/netflix/priam/backup/AbstractBackupPath.java b/priam/src/main/java/com/netflix/priam/backup/AbstractBackupPath.java index 2348180f6..1e6160ac3 100644 --- a/priam/src/main/java/com/netflix/priam/backup/AbstractBackupPath.java +++ b/priam/src/main/java/com/netflix/priam/backup/AbstractBackupPath.java @@ -66,14 +66,14 @@ public static boolean isDataFile(BackupFileType type) { protected Date time; private long size; // uncompressed file size private long compressedFileSize = 0; - protected final InstanceIdentity factory; + protected final InstanceIdentity instanceIdentity; protected final IConfiguration config; private File backupFile; private long lastModified = 0; private Date uploadedTs; - public AbstractBackupPath(IConfiguration config, InstanceIdentity factory) { - this.factory = factory; + public AbstractBackupPath(IConfiguration config, InstanceIdentity instanceIdentity) { + this.instanceIdentity = instanceIdentity; this.config = config; } @@ -121,8 +121,8 @@ public void parseLocal(File file, BackupFileType type) throws ParseException { String[] elements = rpath.split("" + PATH_SEP); this.clusterName = config.getAppName(); this.baseDir = config.getBackupLocation(); - this.region = config.getDC(); - this.token = factory.getInstance().getToken(); + this.region = instanceIdentity.getInstanceInfo().getRegion(); + this.token = instanceIdentity.getInstance().getToken(); this.type = type; if (BackupFileType.isDataFile(type)) { this.keyspace = elements[0]; @@ -259,7 +259,7 @@ public void setFileName(String fileName) { } public InstanceIdentity getInstanceIdentity() { - return this.factory; + return this.instanceIdentity; } public void setUploadedTs(Date uploadedTs) { diff --git a/priam/src/main/java/com/netflix/priam/backupv2/MetaFileWriterBuilder.java b/priam/src/main/java/com/netflix/priam/backupv2/MetaFileWriterBuilder.java index 00ec5d83e..750f00029 100644 --- a/priam/src/main/java/com/netflix/priam/backupv2/MetaFileWriterBuilder.java +++ b/priam/src/main/java/com/netflix/priam/backupv2/MetaFileWriterBuilder.java @@ -95,8 +95,8 @@ private MetaFileWriter( metaFileInfo = new MetaFileInfo( configuration.getAppName(), - configuration.getDC(), - configuration.getRac(), + instanceIdentity.getInstanceInfo().getRegion(), + instanceIdentity.getInstanceInfo().getRac(), backupIdentifier); } diff --git a/priam/src/main/java/com/netflix/priam/config/IConfiguration.java b/priam/src/main/java/com/netflix/priam/config/IConfiguration.java index 031b484f4..3f38a7610 100644 --- a/priam/src/main/java/com/netflix/priam/config/IConfiguration.java +++ b/priam/src/main/java/com/netflix/priam/config/IConfiguration.java @@ -19,13 +19,14 @@ import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.inject.ImplementedBy; -import com.netflix.priam.identity.config.InstanceDataRetriever; import com.netflix.priam.scheduler.SchedulerType; import com.netflix.priam.scheduler.UnsupportedTypeException; import com.netflix.priam.tuner.GCType; import com.netflix.priam.tuner.JVMOption; +import java.io.File; import java.util.List; import java.util.Map; +import org.apache.commons.lang3.StringUtils; /** Interface for Priam's configuration */ @ImplementedBy(PriamConfiguration.class) @@ -67,7 +68,9 @@ public interface IConfiguration { * seconds before gracefully draining cassandra (nodetool drain) and stopping cassandra with * the stop script. */ - int getGracefulDrainHealthWaitSeconds(); + default int getGracefulDrainHealthWaitSeconds() { + return -1; + } /** * @return int representing how often (in seconds) Priam should auto-remediate Cassandra process @@ -76,7 +79,9 @@ public interface IConfiguration { * seconds. For example a value of 60 means that Priam will only restart Cassandra once per * 60 seconds If a negative number, Priam will not restart Cassandra due to crash at all */ - int getRemediateDeadCassandraRate(); + default int getRemediateDeadCassandraRate() { + return 3600; + } /** * Eg: 'my_backup' will result in all files stored under this dir/prefix @@ -104,9 +109,6 @@ public interface IConfiguration { */ String getRestorePrefix(); - /** @param prefix Set the current restore prefix */ - void setRestorePrefix(String prefix); - /** @return Location of the local data dir */ String getDataFileLocation(); @@ -153,15 +155,23 @@ default boolean enableRemoteJMX() { } /** @return Cassandra storage/cluster communication port */ - int getStoragePort(); + default int getStoragePort() { + return 7000; + } - int getSSLStoragePort(); + default int getSSLStoragePort() { + return 7001; + } /** @return Cassandra's thrift port */ - int getThriftPort(); + default int getThriftPort() { + return 9160; + } /** @return Port for CQL binary transport. */ - int getNativeTransportPort(); + default int getNativeTransportPort() { + return 9042; + } /** @return Snitch to be used in cassandra.yaml */ String getSnitch(); @@ -169,18 +179,9 @@ default boolean enableRemoteJMX() { /** @return Cluster name */ String getAppName(); - /** @return RAC (or zone for AWS) */ - String getRac(); - /** @return List of all RAC used for the cluster */ List getRacs(); - /** @return Local hostmame */ - String getHostname(); - - /** @return Get instance name (for AWS) */ - String getInstanceName(); - /** @return Max heap size be used for Cassandra */ String getHeapSize(); @@ -233,10 +234,13 @@ default String getCompactionExcludeCFList() { /** * @return Backup hour for snapshot backups (0 - 23) - * @deprecated Use the {{@link #getBackupCronExpression()}} instead. + * @deprecated Use the {{@link #getBackupCronExpression()}} instead. Scheduled for deletion in + * Dec 2018. */ @Deprecated - int getBackupHour(); + default int getBackupHour() { + return 12; + } /** * Cron expression to be used for snapshot backups. @@ -246,7 +250,9 @@ default String getCompactionExcludeCFList() { * href="http://www.quartz-scheduler.org/documentation/quartz-2.x/tutorials/crontrigger.html">quartz-scheduler * @see http://www.cronmaker.com To build new cron timer */ - String getBackupCronExpression(); + default String getBackupCronExpression() { + return "0 0 12 1/1 * ? *"; + } /** * Backup scheduler type to use for backup. @@ -256,7 +262,9 @@ default String getCompactionExcludeCFList() { * #getBackupCronExpression()}. * @throws UnsupportedTypeException if the scheduler type is not CRON/HOUR. */ - SchedulerType getBackupSchedulerType() throws UnsupportedTypeException; + default SchedulerType getBackupSchedulerType() throws UnsupportedTypeException { + return SchedulerType.HOUR; + } /** * Column Family(ies), comma delimited, to include during snapshot backup. Note 1: The expected @@ -359,14 +367,10 @@ default String getRestoreExcludeCFList() { /** @return Get the region to connect to SDB for instance identity */ String getSDBInstanceIdentityRegion(); - /** @return Get the Data Center name (or region for AWS) */ - String getDC(); - - /** @param region Set the current data center */ - void setDC(String region); - /** @return true if it is a multi regional cluster */ - boolean isMultiDC(); + default boolean isMultiDC() { + return false; + } /** @return Number of backup threads for uploading files when using async feature */ default int getBackupThreads() { @@ -379,10 +383,9 @@ default int getRestoreThreads() { } /** @return true if restore should search for nearest token if current token is not found */ - boolean isRestoreClosestToken(); - - /** Amazon specific setting to query ASG Membership */ - String getASGName(); + default boolean isRestoreClosestToken() { + return false; + } /** * Amazon specific setting to query Additional/ Sibling ASG Memberships in csv format to @@ -394,35 +397,37 @@ default int getRestoreThreads() { String getACLGroupName(); /** @return true if incremental backups are enabled */ - boolean isIncrBackup(); - - /** @return Get host IP */ - String getHostIP(); + default boolean isIncrementalBackupEnabled() { + return true; + } /** @return Bytes per second to throttle for backups */ - int getUploadThrottle(); - - /** - * @return InstanceDataRetriever which encapsulates meta-data about the running instance like - * region, RAC, name, ip address etc. - */ - InstanceDataRetriever getInstanceDataRetriever() - throws InstantiationException, IllegalAccessException, ClassNotFoundException; + default int getUploadThrottle() { + return -1; + } /** @return true if Priam should local config file for tokens and seeds */ boolean isLocalBootstrapEnabled(); /** @return Compaction throughput */ - int getCompactionThroughput(); + default int getCompactionThroughput() { + return 8; + } /** @return compaction_throughput_mb_per_sec */ - int getMaxHintWindowInMS(); + default int getMaxHintWindowInMS() { + return 10800000; + } /** @return hinted_handoff_throttle_in_kb */ - int getHintedHandoffThrottleKb(); + default int getHintedHandoffThrottleKb() { + return 1024; + } /** @return Size of Cassandra max direct memory */ - String getMaxDirectMemory(); + default String getMaxDirectMemory() { + return "50G"; + } /** @return Bootstrap cluster name (depends on another cass cluster) */ String getBootClusterName(); @@ -440,7 +445,9 @@ default double getMemtableCleanupThreshold() { } /** @return stream_throughput_outbound_megabits_per_sec in yaml */ - int getStreamingThroughputMB(); + default int getStreamingThroughputMB() { + return 400; + } /** * Get the paritioner for this cassandra cluster/node. @@ -465,10 +472,14 @@ default double getMemtableCleanupThreshold() { String getCassProcessName(); /** Defaults to 'allow all'. */ - String getAuthenticator(); + default String getAuthenticator() { + return "org.apache.cassandra.auth.AllowAllAuthenticator"; + } /** Defaults to 'allow all'. */ - String getAuthorizer(); + default String getAuthorizer() { + return "org.apache.cassandra.auth.AllowAllAuthorizer"; + } /** @return true/false, if Cassandra needs to be started manually */ boolean doesCassandraStartManually(); @@ -500,9 +511,6 @@ default boolean isBackingUpCommitLogs() { int maxCommitLogsRestore(); - /** @return true/false, if Cassandra is running in a VPC environment */ - boolean isVpcRing(); - boolean isClientSslEnabled(); String getInternodeEncryption(); @@ -630,9 +638,6 @@ default boolean isRestoreEncrypted() { */ Map getExtraEnvParams(); - /** @return the vpc id of the running instance. */ - String getVpcId(); - /* * @return the Amazon Resource Name (ARN) for EC2 classic. */ @@ -744,10 +749,13 @@ default int getStreamingSocketTimeoutInMS() { * * @return the interval to run the flush task. Format is name=value where “name” is an enum of * hour, daily, value is ... - * @deprecated Use the {{@link #getFlushCronExpression()} instead. + * @deprecated Use the {{@link #getFlushCronExpression()} instead. This is set for deletion in + * Dec 2018. */ @Deprecated - String getFlushInterval(); + default String getFlushInterval() { + return null; + } /** * Scheduler type to use for flush. Default: HOUR. @@ -757,7 +765,9 @@ default int getStreamingSocketTimeoutInMS() { * #getFlushCronExpression()}. * @throws UnsupportedTypeException if the scheduler type is not HOUR/CRON. */ - SchedulerType getFlushSchedulerType() throws UnsupportedTypeException; + default SchedulerType getFlushSchedulerType() throws UnsupportedTypeException { + return SchedulerType.HOUR; + } /** * Cron expression to be used for flush. Use "-1" to disable the CRON. Default: -1 @@ -772,7 +782,9 @@ default String getFlushCronExpression() { } /** @return the absolute path to store the backup status on disk */ - String getBackupStatusFileLoc(); + default String getBackupStatusFileLoc() { + return getDataFileLocation() + File.separator + "backup.status"; + } /** @return Decides whether to use sudo to start C* or not */ default boolean useSudo() { @@ -787,7 +799,9 @@ default boolean useSudo() { * * @return SNS Topic ARN to be used to send notification. */ - String getBackupNotificationTopicArn(); + default String getBackupNotificationTopicArn() { + return StringUtils.EMPTY; + } /** * Post restore hook enabled state. If enabled, jar represented by getPostRepairHook is called @@ -805,7 +819,7 @@ default boolean isPostRestoreHookEnabled() { * @return post restore hook to be executed once restore is complete */ default String getPostRestoreHook() { - return ""; + return StringUtils.EMPTY; } /** diff --git a/priam/src/main/java/com/netflix/priam/config/PriamConfiguration.java b/priam/src/main/java/com/netflix/priam/config/PriamConfiguration.java index b0d4d0e4d..89bf5d34e 100644 --- a/priam/src/main/java/com/netflix/priam/config/PriamConfiguration.java +++ b/priam/src/main/java/com/netflix/priam/config/PriamConfiguration.java @@ -18,7 +18,8 @@ import com.amazonaws.services.ec2.AmazonEC2; import com.amazonaws.services.ec2.AmazonEC2ClientBuilder; -import com.amazonaws.services.ec2.model.*; +import com.amazonaws.services.ec2.model.AvailabilityZone; +import com.amazonaws.services.ec2.model.DescribeAvailabilityZonesResult; import com.fasterxml.jackson.annotation.JsonIgnore; import com.google.common.collect.ImmutableList; import com.google.common.collect.Lists; @@ -26,14 +27,12 @@ import com.google.inject.Singleton; import com.netflix.priam.configSource.IConfigSource; import com.netflix.priam.cred.ICredential; -import com.netflix.priam.identity.InstanceEnvIdentity; -import com.netflix.priam.identity.config.InstanceDataRetriever; +import com.netflix.priam.identity.config.InstanceInfo; import com.netflix.priam.scheduler.SchedulerType; import com.netflix.priam.scheduler.UnsupportedTypeException; import com.netflix.priam.tuner.GCType; import com.netflix.priam.tuner.JVMOption; import com.netflix.priam.tuner.JVMOptionsTuner; -import com.netflix.priam.utils.RetryableCallable; import com.netflix.priam.utils.SystemUtils; import java.io.File; import java.util.HashMap; @@ -47,253 +46,29 @@ public class PriamConfiguration implements IConfiguration { public static final String PRIAM_PRE = "priam"; - private static final String CONFIG_CASS_HOME_DIR = PRIAM_PRE + ".cass.home"; - private static final String CONFIG_CASS_START_SCRIPT = PRIAM_PRE + ".cass.startscript"; - private static final String CONFIG_CASS_STOP_SCRIPT = PRIAM_PRE + ".cass.stopscript"; - private static final String CONFIG_CASS_USE_SUDO = PRIAM_PRE + ".cass.usesudo"; - private static final String CONFIG_CLUSTER_NAME = PRIAM_PRE + ".clustername"; - private static final String CONFIG_SEED_PROVIDER_NAME = PRIAM_PRE + ".seed.provider"; - private static final String CONFIG_LOAD_LOCAL_PROPERTIES = PRIAM_PRE + ".localbootstrap.enable"; - private static final String CONFIG_MAX_HEAP_SIZE = PRIAM_PRE + ".heap.size."; - private static final String CONFIG_DATA_LOCATION = PRIAM_PRE + ".data.location"; - private static final String CONFIG_LOGS_LOCATION = PRIAM_PRE + ".logs.location"; - private static final String CONFIG_MR_ENABLE = PRIAM_PRE + ".multiregion.enable"; - private static final String CONFIG_CL_LOCATION = PRIAM_PRE + ".commitlog.location"; - private static final String CONFIG_JMX_LISTERN_PORT_NAME = PRIAM_PRE + ".jmx.port"; - private static final String CONFIG_JMX_USERNAME = PRIAM_PRE + ".jmx.username"; - private static final String CONFIG_JMX_PASSWORD = PRIAM_PRE + ".jmx.password"; - private static final String CONFIG_JMX_ENABLE_REMOTE = PRIAM_PRE + ".jmx.remote.enable"; - private static final String CONFIG_AVAILABILITY_ZONES = PRIAM_PRE + ".zones.available"; - private static final String CONFIG_SAVE_CACHE_LOCATION = PRIAM_PRE + ".cache.location"; - private static final String CONFIG_NEW_MAX_HEAP_SIZE = PRIAM_PRE + ".heap.newgen.size."; - private static final String CONFIG_DIRECT_MAX_HEAP_SIZE = PRIAM_PRE + ".direct.memory.size."; - private static final String CONFIG_THRIFT_LISTEN_PORT_NAME = PRIAM_PRE + ".thrift.port"; - private static final String CONFIG_THRIFT_ENABLED = PRIAM_PRE + ".thrift.enabled"; - private static final String CONFIG_NATIVE_PROTOCOL_PORT = PRIAM_PRE + ".nativeTransport.port"; - private static final String CONFIG_NATIVE_PROTOCOL_ENABLED = - PRIAM_PRE + ".nativeTransport.enabled"; - private static final String CONFIG_STORAGE_LISTERN_PORT_NAME = PRIAM_PRE + ".storage.port"; - private static final String CONFIG_SSL_STORAGE_LISTERN_PORT_NAME = - PRIAM_PRE + ".ssl.storage.port"; - private static final String CONFIG_CL_BK_LOCATION = PRIAM_PRE + ".backup.commitlog.location"; - private static final String CONFIG_THROTTLE_UPLOAD_PER_SECOND = PRIAM_PRE + ".upload.throttle"; - private static final String CONFIG_COMPACTION_THROUHPUT = PRIAM_PRE + ".compaction.throughput"; - private static final String CONFIG_MAX_HINT_WINDOW_IN_MS = PRIAM_PRE + ".hint.window"; - private static final String CONFIG_BOOTCLUSTER_NAME = PRIAM_PRE + ".bootcluster"; - private static final String CONFIG_ENDPOINT_SNITCH = PRIAM_PRE + ".endpoint_snitch"; - private static final String CONFIG_MEMTABLE_CLEANUP_THRESHOLD = - PRIAM_PRE + ".memtable.cleanup.threshold"; - private static final String CONFIG_CASS_PROCESS_NAME = PRIAM_PRE + ".cass.process"; - private static final String CONFIG_VNODE_NUM_TOKENS = PRIAM_PRE + ".vnodes.numTokens"; - private static final String CONFIG_YAML_LOCATION = PRIAM_PRE + ".yamlLocation"; - private static final String CONFIG_AUTHENTICATOR = PRIAM_PRE + ".authenticator"; - private static final String CONFIG_AUTHORIZER = PRIAM_PRE + ".authorizer"; - private static final String CONFIG_CASS_MANUAL_START_ENABLE = - PRIAM_PRE + ".cass.manual.start.enable"; - private static final String CONFIG_REMEDIATE_DEAD_CASSANDRA_RATE_S = - PRIAM_PRE + ".remediate.dead.cassandra.rate"; - private static final String CONFIG_CREATE_NEW_TOKEN_ENABLE = - PRIAM_PRE + ".create.new.token.enable"; - - // Backup and Restore - private static final String CONFIG_BACKUP_THREADS = PRIAM_PRE + ".backup.threads"; private static final String CONFIG_RESTORE_PREFIX = PRIAM_PRE + ".restore.prefix"; - private static final String CONFIG_INCR_BK_ENABLE = PRIAM_PRE + ".backup.incremental.enable"; - - private static final String CONFIG_AUTO_RESTORE_SNAPSHOTNAME = PRIAM_PRE + ".restore.snapshot"; - private static final String CONFIG_BUCKET_NAME = PRIAM_PRE + ".s3.bucket"; - private static final String CONFIG_BACKUP_SCHEDULE_TYPE = PRIAM_PRE + ".backup.schedule.type"; - private static final String CONFIG_BACKUP_HOUR = PRIAM_PRE + ".backup.hour"; - private static final String CONFIG_BACKUP_CRON_EXPRESSION = PRIAM_PRE + ".backup.cron"; - private static final String CONFIG_S3_BASE_DIR = PRIAM_PRE + ".s3.base_dir"; - private static final String CONFIG_RESTORE_THREADS = PRIAM_PRE + ".restore.threads"; - private static final String CONFIG_RESTORE_CLOSEST_TOKEN = PRIAM_PRE + ".restore.closesttoken"; - private static final String CONFIG_BACKUP_CHUNK_SIZE = PRIAM_PRE + ".backup.chunksizemb"; - private static final String CONFIG_BACKUP_RETENTION = PRIAM_PRE + ".backup.retention"; - private static final String CONFIG_BACKUP_RACS = PRIAM_PRE + ".backup.racs"; - private static final String CONFIG_BACKUP_STATUS_FILE_LOCATION = - PRIAM_PRE + ".backup.status.location"; - private static final String CONFIG_STREAMING_THROUGHPUT_MB = - PRIAM_PRE + ".streaming.throughput.mb"; - private static final String CONFIG_STREAMING_SOCKET_TIMEOUT_IN_MS = - PRIAM_PRE + ".streaming.socket.timeout.ms"; - private static final String CONFIG_TOMBSTONE_FAILURE_THRESHOLD = - PRIAM_PRE + ".tombstone.failure.threshold"; - private static final String CONFIG_TOMBSTONE_WARNING_THRESHOLD = - PRIAM_PRE + ".tombstone.warning.threshold"; - - private static final String CONFIG_PARTITIONER = PRIAM_PRE + ".partitioner"; - private static final String CONFIG_KEYCACHE_SIZE = PRIAM_PRE + ".keyCache.size"; - private static final String CONFIG_KEYCACHE_COUNT = PRIAM_PRE + ".keyCache.count"; - private static final String CONFIG_ROWCACHE_SIZE = PRIAM_PRE + ".rowCache.size"; - private static final String CONFIG_ROWCACHE_COUNT = PRIAM_PRE + ".rowCache.count"; - - private static final String CONFIG_MAX_HINT_THREADS = PRIAM_PRE + ".hints.maxThreads"; - private static final String CONFIG_HINTS_THROTTLE_KB = PRIAM_PRE + ".hints.throttleKb"; - private static final String CONFIG_INTERNODE_COMPRESSION = PRIAM_PRE + ".internodeCompression"; - - private static final String CONFIG_COMMITLOG_BKUP_ENABLED = PRIAM_PRE + ".clbackup.enabled"; - private static final String CONFIG_COMMITLOG_PROPS_FILE = PRIAM_PRE + ".clbackup.propsfile"; - private static final String CONFIG_COMMITLOG_ARCHIVE_CMD = PRIAM_PRE + ".clbackup.archiveCmd"; - private static final String CONFIG_COMMITLOG_RESTORE_CMD = PRIAM_PRE + ".clbackup.restoreCmd"; - private static final String CONFIG_COMMITLOG_RESTORE_DIRS = PRIAM_PRE + ".clbackup.restoreDirs"; - private static final String CONFIG_COMMITLOG_RESTORE_POINT_IN_TIME = - PRIAM_PRE + ".clbackup.restoreTime"; - private static final String CONFIG_COMMITLOG_RESTORE_MAX = PRIAM_PRE + ".clrestore.max"; - private static final String CONFIG_CLIENT_SSL_ENABLED = PRIAM_PRE + ".client.sslEnabled"; - private static final String CONFIG_INTERNODE_ENCRYPTION = PRIAM_PRE + ".internodeEncryption"; - private static final String CONFIG_DSNITCH_ENABLED = PRIAM_PRE + ".dsnitchEnabled"; - - private static final String CONFIG_CONCURRENT_READS = PRIAM_PRE + ".concurrentReads"; - private static final String CONFIG_CONCURRENT_WRITES = PRIAM_PRE + ".concurrentWrites"; - private static final String CONFIG_CONCURRENT_COMPACTORS = PRIAM_PRE + ".concurrentCompactors"; - - private static final String CONFIG_RPC_SERVER_TYPE = PRIAM_PRE + ".rpc.server.type"; - private static final String CONFIG_RPC_MIN_THREADS = PRIAM_PRE + ".rpc.min.threads"; - private static final String CONFIG_RPC_MAX_THREADS = PRIAM_PRE + ".rpc.max.threads"; - private static final String CONFIG_EXTRA_PARAMS = PRIAM_PRE + ".extra.params"; - private static final String CONFIG_AUTO_BOOTSTRAP = PRIAM_PRE + ".auto.bootstrap"; - private static final String CONFIG_EXTRA_ENV_PARAMS = PRIAM_PRE + ".extra.env.params"; - - private static final String CONFIG_RESTORE_SOURCE_TYPE = - PRIAM_PRE + ".restore.source.type"; // the type of source for the restore. Valid values - // are: AWSCROSSACCT or GOOGLE. - private static final String CONFIG_ENCRYPTED_BACKUP_ENABLED = - PRIAM_PRE + ".encrypted.backup.enabled"; // enable encryption of backup (snapshots, - // incrementals, commit logs). - - // Backup and restore cryptography - private static final String CONFIG_PRIKEY_LOC = - PRIAM_PRE + ".private.key.location"; // the location on disk of the private key used by - // the cryptography algorithm - private static final String CONFIG_PGP_PASSWORD_PHRASE = - PRIAM_PRE + ".pgp.password.phrase"; // pass phrase used by the cryptography algorithm - private static final String CONFIG_PGP_PUB_KEY_LOC = PRIAM_PRE + ".pgp.pubkey.file.location"; - - // Restore from Google Cloud Storage - private static final String CONFIG_GCS_SERVICE_ACCT_ID = - PRIAM_PRE + ".gcs.service.acct.id"; // Google Cloud Storage service account id - private static final String CONFIG_GCS_SERVICE_ACCT_PRIVATE_KEY_LOC = - PRIAM_PRE + ".gcs.service.acct.private.key"; // the absolute path on disk for the Google - // Cloud Storage PFX file (i.e. the combined - // format of the private key and - // certificate). - - // Amazon specific - private static final String CONFIG_ASG_NAME = PRIAM_PRE + ".az.asgname"; - private static final String CONFIG_SIBLING_ASG_NAMES = PRIAM_PRE + ".az.sibling.asgnames"; - private static final String CONFIG_REGION_NAME = PRIAM_PRE + ".az.region"; - private static final String SDB_INSTANCE_INDENTITY_REGION_NAME = - PRIAM_PRE + ".sdb.instanceIdentity.region"; - private static final String CONFIG_ACL_GROUP_NAME = PRIAM_PRE + ".acl.groupname"; - private static String ASG_NAME = System.getenv("ASG_NAME"); - private static String REGION = System.getenv("EC2_REGION"); - private static final String CONFIG_VPC_RING = PRIAM_PRE + ".vpc"; - private static final String CONFIG_S3_ROLE_ASSUMPTION_ARN = - PRIAM_PRE - + ".roleassumption.arn"; // Restore from AWS. This is applicable when restoring - // from an AWS account which requires cross account - // assumption. - private static final String CONFIG_EC2_ROLE_ASSUMPTION_ARN = - PRIAM_PRE + ".ec2.roleassumption.arn"; - private static final String CONFIG_VPC_ROLE_ASSUMPTION_ARN = - PRIAM_PRE + ".vpc.roleassumption.arn"; - private static final String CONFIG_DUAL_ACCOUNT = PRIAM_PRE + ".roleassumption.dualaccount"; - - // Post Restore Hook - private static final String CONFIG_POST_RESTORE_HOOK_ENABLED = - PRIAM_PRE + ".postrestorehook.enabled"; - private static final String CONFIG_POST_RESTORE_HOOK = PRIAM_PRE + ".postrestorehook"; - private static final String CONFIG_POST_RESTORE_HOOK_HEARTBEAT_FILENAME = - PRIAM_PRE + ".postrestorehook.heartbeat.filename"; - private static final String CONFIG_POST_RESTORE_HOOK_DONE_FILENAME = - PRIAM_PRE + ".postrestorehook.done.filename"; - private static final String CONFIG_POST_RESTORE_HOOK_TIMEOUT_IN_DAYS = - PRIAM_PRE + ".postrestorehook.timeout.in.days"; - private static final String CONFIG_POST_RESTORE_HOOK_HEARTBEAT_TIMEOUT_MS = - PRIAM_PRE + ".postrestorehook.heartbeat.timeout"; - private static final String CONFIG_POST_RESTORE_HOOK_HEARTBEAT_CHECK_FREQUENCY_MS = - PRIAM_PRE + ".postrestorehook.heartbeat.check.frequency"; - - // Running instance meta data - private String RAC; - private String INSTANCE_ID; - - // == vpc specific - private String NETWORK_VPC; // Fetch the vpc id of running instance - private final String CASS_BASE_DATA_DIR = "/var/lib/cassandra"; - public static final String DEFAULT_AUTHENTICATOR = - "org.apache.cassandra.auth.AllowAllAuthenticator"; - public static final String DEFAULT_AUTHORIZER = "org.apache.cassandra.auth.AllowAllAuthorizer"; - public static final String DEFAULT_COMMITLOG_PROPS_FILE = - "/conf/commitlog_archiving.properties"; - // private String DEFAULT_AVAILABILITY_ZONES = ""; private List DEFAULT_AVAILABILITY_ZONES = ImmutableList.of(); - private final int DEFAULT_HINTS_MAX_THREADS = 2; // default value from 1.2 yaml - - private static final String DEFAULT_RPC_SERVER_TYPE = "hsha"; - private static final int DEFAULT_RPC_MIN_THREADS = 16; - private static final int DEFAULT_RPC_MAX_THREADS = 2048; - private static final int DEFAULT_STREAMING_SOCKET_TIMEOUT_IN_MS = 86400000; // 24 Hours - private static final int DEFAULT_TOMBSTONE_WARNING_THRESHOLD = 1000; // C* defaults - private static final int DEFAULT_TOMBSTONE_FAILURE_THRESHOLD = 100000; // C* defaults - - // AWS EC2 Dual Account - private static final boolean DEFAULT_DUAL_ACCOUNT = false; - private final IConfigSource config; private static final Logger logger = LoggerFactory.getLogger(PriamConfiguration.class); private final ICredential provider; - @JsonIgnore private final InstanceEnvIdentity insEnvIdentity; - @JsonIgnore private InstanceDataRetriever instanceDataRetriever; + @JsonIgnore private InstanceInfo instanceInfo; @Inject public PriamConfiguration( - ICredential provider, IConfigSource config, InstanceEnvIdentity insEnvIdentity) { + ICredential provider, IConfigSource config, InstanceInfo instanceInfo) { this.provider = provider; this.config = config; - this.insEnvIdentity = insEnvIdentity; + this.instanceInfo = instanceInfo; } @Override public void initialize() { - try { - if (this.insEnvIdentity.isClassic()) { - this.instanceDataRetriever = - (InstanceDataRetriever) - Class.forName( - "com.netflix.priam.identity.config.AwsClassicInstanceDataRetriever") - .newInstance(); - - } else if (this.insEnvIdentity.isNonDefaultVpc()) { - this.instanceDataRetriever = - (InstanceDataRetriever) - Class.forName( - "com.netflix.priam.identity.config.AWSVpcInstanceDataRetriever") - .newInstance(); - } else { - throw new IllegalStateException( - "Unable to determine environemt (vpc, classic) for running instance."); - } - } catch (Exception e) { - throw new IllegalStateException( - "Exception when instantiating the instance data retriever. Msg: " - + e.getLocalizedMessage()); - } - - RAC = instanceDataRetriever.getRac(); - INSTANCE_ID = instanceDataRetriever.getInstanceId(); - - NETWORK_VPC = instanceDataRetriever.getVpcId(); - - setupEnvVars(); - this.config.intialize(ASG_NAME, REGION); - setDefaultRACList(REGION); - populateProps(); + this.config.initialize(instanceInfo.getAutoScalingGroup(), instanceInfo.getRegion()); + setDefaultRACList(instanceInfo.getRegion()); SystemUtils.createDirs(getBackupCommitLogLocation()); SystemUtils.createDirs(getCommitLogLocation()); SystemUtils.createDirs(getCacheLocation()); @@ -302,72 +77,8 @@ public void initialize() { SystemUtils.createDirs(getLogDirLocation()); } - public InstanceDataRetriever getInstanceDataRetriever() { - return instanceDataRetriever; - } - - private void setupEnvVars() { - // Search in java opt properties - REGION = StringUtils.isBlank(REGION) ? System.getProperty("EC2_REGION") : REGION; - // Infer from zone - if (StringUtils.isBlank(REGION)) REGION = RAC.substring(0, RAC.length() - 1); - ASG_NAME = StringUtils.isBlank(ASG_NAME) ? System.getProperty("ASG_NAME") : ASG_NAME; - if (StringUtils.isBlank(ASG_NAME)) - ASG_NAME = populateASGName(REGION, getInstanceDataRetriever().getInstanceId()); - logger.info("REGION set to {}, ASG Name set to {}", REGION, ASG_NAME); - } - - /** Query amazon to get ASG name. Currently not available as part of instance info api. */ - private String populateASGName(String region, String instanceId) { - GetASGName getASGName = new GetASGName(region, instanceId); - - try { - return getASGName.call(); - } catch (Exception e) { - logger.error("Failed to determine ASG name.", e); - return null; - } - } - - private class GetASGName extends RetryableCallable { - private static final int NUMBER_OF_RETRIES = 15; - private static final long WAIT_TIME = 30000; - private final String region; - private final String instanceId; - private final AmazonEC2 client; - - public GetASGName(String region, String instanceId) { - super(NUMBER_OF_RETRIES, WAIT_TIME); - this.region = region; - this.instanceId = instanceId; - client = - AmazonEC2ClientBuilder.standard() - .withCredentials(provider.getAwsCredentialProvider()) - .withRegion(region) - .build(); - } - - @Override - public String retriableCall() throws IllegalStateException { - DescribeInstancesRequest desc = - new DescribeInstancesRequest().withInstanceIds(instanceId); - DescribeInstancesResult res = client.describeInstances(desc); - - for (Reservation resr : res.getReservations()) { - for (Instance ins : resr.getInstances()) { - for (com.amazonaws.services.ec2.model.Tag tag : ins.getTags()) { - if (tag.getKey().equals("aws:autoscaling:groupName")) return tag.getValue(); - } - } - } - - logger.warn("Couldn't determine ASG name"); - throw new IllegalStateException("Couldn't determine ASG name"); - } - } - /** Get the fist 3 available zones in the region */ - public void setDefaultRACList(String region) { + private void setDefaultRACList(String region) { AmazonEC2 client = AmazonEC2ClientBuilder.standard() .withCredentials(provider.getAwsCredentialProvider()) @@ -382,23 +93,14 @@ public void setDefaultRACList(String region) { DEFAULT_AVAILABILITY_ZONES = ImmutableList.copyOf(zone); } - private void populateProps() { - config.set(CONFIG_ASG_NAME, ASG_NAME); - config.set(CONFIG_REGION_NAME, REGION); - } - - public String getInstanceName() { - return INSTANCE_ID; - } - @Override public String getCassStartupScript() { - return config.get(CONFIG_CASS_START_SCRIPT, "/etc/init.d/cassandra start"); + return config.get(PRIAM_PRE + ".cass.startscript", "/etc/init.d/cassandra start"); } @Override public String getCassStopScript() { - return config.get(CONFIG_CASS_STOP_SCRIPT, "/etc/init.d/cassandra stop"); + return config.get(PRIAM_PRE + ".cass.stopscript", "/etc/init.d/cassandra stop"); } @Override @@ -408,32 +110,33 @@ public int getGracefulDrainHealthWaitSeconds() { @Override public int getRemediateDeadCassandraRate() { - return config.get(CONFIG_REMEDIATE_DEAD_CASSANDRA_RATE_S, 3600); // Default to once per hour + return config.get( + PRIAM_PRE + ".remediate.dead.cassandra.rate", 3600); // Default to once per hour } @Override public String getCassHome() { - return config.get(CONFIG_CASS_HOME_DIR, "/etc/cassandra"); + return config.get(PRIAM_PRE + ".cass.home", "/etc/cassandra"); } @Override public String getBackupLocation() { - return config.get(CONFIG_S3_BASE_DIR, "backup"); + return config.get(PRIAM_PRE + ".s3.base_dir", "backup"); } @Override public String getBackupPrefix() { - return config.get(CONFIG_BUCKET_NAME, "cassandra-archive"); + return config.get(PRIAM_PRE + ".s3.bucket", "cassandra-archive"); } @Override public int getBackupRetentionDays() { - return config.get(CONFIG_BACKUP_RETENTION, 0); + return config.get(PRIAM_PRE + ".backup.retention", 0); } @Override public List getBackupRacs() { - return config.getList(CONFIG_BACKUP_RACS); + return config.getList(PRIAM_PRE + ".backup.racs"); } @Override @@ -443,12 +146,12 @@ public String getRestorePrefix() { @Override public String getDataFileLocation() { - return config.get(CONFIG_DATA_LOCATION, CASS_BASE_DATA_DIR + "/data"); + return config.get(PRIAM_PRE + ".data.location", CASS_BASE_DATA_DIR + "/data"); } @Override public String getLogDirLocation() { - return config.get(CONFIG_LOGS_LOCATION, CASS_BASE_DATA_DIR + "/logs"); + return config.get(PRIAM_PRE + ".logs.location", CASS_BASE_DATA_DIR + "/logs"); } @Override @@ -458,124 +161,112 @@ public String getHintsLocation() { @Override public String getCacheLocation() { - return config.get(CONFIG_SAVE_CACHE_LOCATION, CASS_BASE_DATA_DIR + "/saved_caches"); + return config.get(PRIAM_PRE + ".cache.location", CASS_BASE_DATA_DIR + "/saved_caches"); } @Override public String getCommitLogLocation() { - return config.get(CONFIG_CL_LOCATION, CASS_BASE_DATA_DIR + "/commitlog"); + return config.get(PRIAM_PRE + ".commitlog.location", CASS_BASE_DATA_DIR + "/commitlog"); } @Override public String getBackupCommitLogLocation() { - return config.get(CONFIG_CL_BK_LOCATION, ""); + return config.get(PRIAM_PRE + ".backup.commitlog.location", ""); } @Override public long getBackupChunkSize() { - long size = config.get(CONFIG_BACKUP_CHUNK_SIZE, 10); + long size = config.get(PRIAM_PRE + ".backup.chunksizemb", 10); return size * 1024 * 1024L; } @Override public int getJmxPort() { - return config.get(CONFIG_JMX_LISTERN_PORT_NAME, 7199); + return config.get(PRIAM_PRE + ".jmx.port", 7199); } @Override public String getJmxUsername() { - return config.get(CONFIG_JMX_USERNAME, ""); + return config.get(PRIAM_PRE + ".jmx.username", ""); } @Override public String getJmxPassword() { - return config.get(CONFIG_JMX_PASSWORD, ""); + return config.get(PRIAM_PRE + ".jmx.password", ""); } /** @return Enables Remote JMX connections n C* */ @Override public boolean enableRemoteJMX() { - return config.get(CONFIG_JMX_ENABLE_REMOTE, false); + return config.get(PRIAM_PRE + ".jmx.remote.enable", false); } public int getNativeTransportPort() { - return config.get(CONFIG_NATIVE_PROTOCOL_PORT, 9042); + return config.get(PRIAM_PRE + ".nativeTransport.port", 9042); } @Override public int getThriftPort() { - return config.get(CONFIG_THRIFT_LISTEN_PORT_NAME, 9160); + return config.get(PRIAM_PRE + ".thrift.port", 9160); } @Override public int getStoragePort() { - return config.get(CONFIG_STORAGE_LISTERN_PORT_NAME, 7000); + return config.get(PRIAM_PRE + ".storage.port", 7000); } @Override public int getSSLStoragePort() { - return config.get(CONFIG_SSL_STORAGE_LISTERN_PORT_NAME, 7001); + return config.get(PRIAM_PRE + ".ssl.storage.port", 7001); } @Override public String getSnitch() { - return config.get(CONFIG_ENDPOINT_SNITCH, "org.apache.cassandra.locator.Ec2Snitch"); + return config.get(PRIAM_PRE + ".endpoint_snitch", "org.apache.cassandra.locator.Ec2Snitch"); } @Override public String getAppName() { - return config.get(CONFIG_CLUSTER_NAME, "cass_cluster"); - } - - @Override - public String getRac() { - return RAC; + return config.get(PRIAM_PRE + ".clustername", "cass_cluster"); } @Override public List getRacs() { - return config.getList(CONFIG_AVAILABILITY_ZONES, DEFAULT_AVAILABILITY_ZONES); - } - - @JsonIgnore - @Override - public String getHostname() { - if (this.isVpcRing()) return getInstanceDataRetriever().getPrivateIP(); - else return getInstanceDataRetriever().getPublicHostname(); + return config.getList(PRIAM_PRE + ".zones.available", DEFAULT_AVAILABILITY_ZONES); } @Override public String getHeapSize() { - return config.get( - CONFIG_MAX_HEAP_SIZE + getInstanceDataRetriever().getInstanceType(), "8G"); + return config.get((PRIAM_PRE + ".heap.size.") + instanceInfo.getInstanceType(), "8G"); } @Override public String getHeapNewSize() { return config.get( - CONFIG_NEW_MAX_HEAP_SIZE + getInstanceDataRetriever().getInstanceType(), "2G"); + (PRIAM_PRE + ".heap.newgen.size.") + instanceInfo.getInstanceType(), "2G"); } @Override public String getMaxDirectMemory() { return config.get( - CONFIG_DIRECT_MAX_HEAP_SIZE + getInstanceDataRetriever().getInstanceType(), "50G"); + (PRIAM_PRE + ".direct.memory.size.") + instanceInfo.getInstanceType(), "50G"); } @Override public int getBackupHour() { - return config.get(CONFIG_BACKUP_HOUR, 12); + return config.get(PRIAM_PRE + ".backup.hour", 12); } @Override public String getBackupCronExpression() { - return config.get(CONFIG_BACKUP_CRON_EXPRESSION, "0 0 12 1/1 * ? *"); // Backup daily at 12 + return config.get(PRIAM_PRE + ".backup.cron", "0 0 12 1/1 * ? *"); // Backup daily at 12 } @Override public SchedulerType getBackupSchedulerType() throws UnsupportedTypeException { String schedulerType = - config.get(CONFIG_BACKUP_SCHEDULE_TYPE, SchedulerType.HOUR.getSchedulerType()); + config.get( + PRIAM_PRE + ".backup.schedule.type", SchedulerType.HOUR.getSchedulerType()); return SchedulerType.lookup(schedulerType); } @@ -655,7 +346,7 @@ public String getRestoreExcludeCFList() { @Override public String getRestoreSnapshot() { - return config.get(CONFIG_AUTO_RESTORE_SNAPSHOTNAME, ""); + return config.get(PRIAM_PRE + ".restore.snapshot", ""); } @Override @@ -665,144 +356,116 @@ public boolean isRestoreEncrypted() { @Override public String getSDBInstanceIdentityRegion() { - return config.get(SDB_INSTANCE_INDENTITY_REGION_NAME, "us-east-1"); - } - - @Override - public String getDC() { - return config.get(CONFIG_REGION_NAME, ""); - } - - @Override - public void setDC(String region) { - config.set(CONFIG_REGION_NAME, region); + return config.get(PRIAM_PRE + ".sdb.instanceIdentity.region", "us-east-1"); } @Override public boolean isMultiDC() { - return config.get(CONFIG_MR_ENABLE, false); + return config.get(PRIAM_PRE + ".multiregion.enable", false); } @Override public int getBackupThreads() { - return config.get(CONFIG_BACKUP_THREADS, 2); + return config.get(PRIAM_PRE + ".backup.threads", 2); } @Override public int getRestoreThreads() { - return config.get(CONFIG_RESTORE_THREADS, 8); + return config.get(PRIAM_PRE + ".restore.threads", 8); } @Override public boolean isRestoreClosestToken() { - return config.get(CONFIG_RESTORE_CLOSEST_TOKEN, false); - } - - @Override - public String getASGName() { - return config.get(CONFIG_ASG_NAME, ""); + return config.get(PRIAM_PRE + ".restore.closesttoken", false); } - /** - * Amazon specific setting to query Additional/ Sibling ASG Memberships in csv format to - * consider while calculating RAC membership - */ @Override public String getSiblingASGNames() { - return config.get(CONFIG_SIBLING_ASG_NAMES, ","); + return config.get(PRIAM_PRE + ".az.sibling.asgnames", ","); } @Override public String getACLGroupName() { - return config.get(CONFIG_ACL_GROUP_NAME, this.getAppName()); + return config.get(PRIAM_PRE + ".acl.groupname", this.getAppName()); } @Override - public boolean isIncrBackup() { - return config.get(CONFIG_INCR_BK_ENABLE, true); - } - - @Override - public String getHostIP() { - if (this.isVpcRing()) return getInstanceDataRetriever().getPrivateIP(); - else return getInstanceDataRetriever().getPublicIP(); + public boolean isIncrementalBackupEnabled() { + return config.get(PRIAM_PRE + ".backup.incremental.enable", true); } @Override public int getUploadThrottle() { - return config.get(CONFIG_THROTTLE_UPLOAD_PER_SECOND, Integer.MAX_VALUE); + return config.get(PRIAM_PRE + ".upload.throttle", -1); } @Override public boolean isLocalBootstrapEnabled() { - return config.get(CONFIG_LOAD_LOCAL_PROPERTIES, false); + return config.get(PRIAM_PRE + ".localbootstrap.enable", false); } @Override public int getCompactionThroughput() { - return config.get(CONFIG_COMPACTION_THROUHPUT, 8); + return config.get(PRIAM_PRE + ".compaction.throughput", 8); } @Override public int getMaxHintWindowInMS() { - return config.get(CONFIG_MAX_HINT_WINDOW_IN_MS, 10800000); + return config.get(PRIAM_PRE + ".hint.window", 10800000); } public int getHintedHandoffThrottleKb() { - return config.get(CONFIG_HINTS_THROTTLE_KB, 1024); + return config.get(PRIAM_PRE + ".hints.throttleKb", 1024); } @Override public String getBootClusterName() { - return config.get(CONFIG_BOOTCLUSTER_NAME, ""); + return config.get(PRIAM_PRE + ".bootcluster", ""); } @Override public String getSeedProviderName() { return config.get( - CONFIG_SEED_PROVIDER_NAME, "com.netflix.priam.cassandra.extensions.NFSeedProvider"); + PRIAM_PRE + ".seed.provider", + "com.netflix.priam.cassandra.extensions.NFSeedProvider"); } public double getMemtableCleanupThreshold() { - return config.get(CONFIG_MEMTABLE_CLEANUP_THRESHOLD, 0.11); + return config.get(PRIAM_PRE + ".memtable.cleanup.threshold", 0.11); } @Override public int getStreamingThroughputMB() { - return config.get(CONFIG_STREAMING_THROUGHPUT_MB, 400); + return config.get(PRIAM_PRE + ".streaming.throughput.mb", 400); } public String getPartitioner() { - return config.get(CONFIG_PARTITIONER, "org.apache.cassandra.dht.RandomPartitioner"); + return config.get(PRIAM_PRE + ".partitioner", "org.apache.cassandra.dht.RandomPartitioner"); } public String getKeyCacheSizeInMB() { - return config.get(CONFIG_KEYCACHE_SIZE); + return config.get(PRIAM_PRE + ".keyCache.size"); } public String getKeyCacheKeysToSave() { - return config.get(CONFIG_KEYCACHE_COUNT); + return config.get(PRIAM_PRE + ".keyCache.count"); } public String getRowCacheSizeInMB() { - return config.get(CONFIG_ROWCACHE_SIZE); + return config.get(PRIAM_PRE + ".rowCache.size"); } public String getRowCacheKeysToSave() { - return config.get(CONFIG_ROWCACHE_COUNT); + return config.get(PRIAM_PRE + ".rowCache.count"); } @Override public String getCassProcessName() { - return config.get(CONFIG_CASS_PROCESS_NAME, "CassandraDaemon"); - } - - public int getNumTokens() { - return config.get(CONFIG_VNODE_NUM_TOKENS, 1); + return config.get(PRIAM_PRE + ".cass.process", "CassandraDaemon"); } public String getYamlLocation() { - return config.get(CONFIG_YAML_LOCATION, getCassHome() + "/conf/cassandra.yaml"); + return config.get(PRIAM_PRE + ".yamlLocation", getCassHome() + "/conf/cassandra.yaml"); } @Override @@ -811,111 +474,105 @@ public String getJVMOptionsFileLocation() { } public String getAuthenticator() { - return config.get(CONFIG_AUTHENTICATOR, DEFAULT_AUTHENTICATOR); + return config.get( + PRIAM_PRE + ".authenticator", "org.apache.cassandra.auth.AllowAllAuthenticator"); } public String getAuthorizer() { - return config.get(CONFIG_AUTHORIZER, DEFAULT_AUTHORIZER); + return config.get( + PRIAM_PRE + ".authorizer", "org.apache.cassandra.auth.AllowAllAuthorizer"); } @Override public boolean doesCassandraStartManually() { - return config.get(CONFIG_CASS_MANUAL_START_ENABLE, false); + return config.get(PRIAM_PRE + ".cass.manual.start.enable", false); } public String getInternodeCompression() { - return config.get(CONFIG_INTERNODE_COMPRESSION, "all"); - } - - @Override - public void setRestorePrefix(String prefix) { - config.set(CONFIG_RESTORE_PREFIX, prefix); + return config.get(PRIAM_PRE + ".internodeCompression", "all"); } @Override public boolean isBackingUpCommitLogs() { - return config.get(CONFIG_COMMITLOG_BKUP_ENABLED, false); + return config.get(PRIAM_PRE + ".clbackup.enabled", false); } @Override public String getCommitLogBackupPropsFile() { return config.get( - CONFIG_COMMITLOG_PROPS_FILE, getCassHome() + DEFAULT_COMMITLOG_PROPS_FILE); + PRIAM_PRE + ".clbackup.propsfile", + getCassHome() + "/conf/commitlog_archiving.properties"); } @Override public String getCommitLogBackupArchiveCmd() { - return config.get(CONFIG_COMMITLOG_ARCHIVE_CMD, "/bin/ln %path /mnt/data/backup/%name"); + return config.get( + PRIAM_PRE + ".clbackup.archiveCmd", "/bin/ln %path /mnt/data/backup/%name"); } @Override public String getCommitLogBackupRestoreCmd() { - return config.get(CONFIG_COMMITLOG_RESTORE_CMD, "/bin/mv %from %to"); + return config.get(PRIAM_PRE + ".clbackup.restoreCmd", "/bin/mv %from %to"); } @Override public String getCommitLogBackupRestoreFromDirs() { - return config.get(CONFIG_COMMITLOG_RESTORE_DIRS, "/mnt/data/backup/commitlog/"); + return config.get(PRIAM_PRE + ".clbackup.restoreDirs", "/mnt/data/backup/commitlog/"); } @Override public String getCommitLogBackupRestorePointInTime() { - return config.get(CONFIG_COMMITLOG_RESTORE_POINT_IN_TIME, ""); + return config.get(PRIAM_PRE + ".clbackup.restoreTime", ""); } @Override public int maxCommitLogsRestore() { - return config.get(CONFIG_COMMITLOG_RESTORE_MAX, 10); - } - - @Override - public boolean isVpcRing() { - return config.get(CONFIG_VPC_RING, false); + return config.get(PRIAM_PRE + ".clrestore.max", 10); } public boolean isClientSslEnabled() { - return config.get(CONFIG_CLIENT_SSL_ENABLED, false); + return config.get(PRIAM_PRE + ".client.sslEnabled", false); } public String getInternodeEncryption() { - return config.get(CONFIG_INTERNODE_ENCRYPTION, "none"); + return config.get(PRIAM_PRE + ".internodeEncryption", "none"); } public boolean isDynamicSnitchEnabled() { - return config.get(CONFIG_DSNITCH_ENABLED, true); + return config.get(PRIAM_PRE + ".dsnitchEnabled", true); } public boolean isThriftEnabled() { - return config.get(CONFIG_THRIFT_ENABLED, true); + return config.get(PRIAM_PRE + ".thrift.enabled", true); } public boolean isNativeTransportEnabled() { - return config.get(CONFIG_NATIVE_PROTOCOL_ENABLED, false); + return config.get(PRIAM_PRE + ".nativeTransport.enabled", false); } public int getConcurrentReadsCnt() { - return config.get(CONFIG_CONCURRENT_READS, 32); + return config.get(PRIAM_PRE + ".concurrentReads", 32); } public int getConcurrentWritesCnt() { - return config.get(CONFIG_CONCURRENT_WRITES, 32); + return config.get(PRIAM_PRE + ".concurrentWrites", 32); } public int getConcurrentCompactorsCnt() { int cpus = Runtime.getRuntime().availableProcessors(); - return config.get(CONFIG_CONCURRENT_COMPACTORS, cpus); + return config.get(PRIAM_PRE + ".concurrentCompactors", cpus); } public String getRpcServerType() { - return config.get(CONFIG_RPC_SERVER_TYPE, DEFAULT_RPC_SERVER_TYPE); + return config.get(PRIAM_PRE + ".rpc.server.type", "hsha"); } public int getRpcMinThreads() { - return config.get(CONFIG_RPC_MIN_THREADS, DEFAULT_RPC_MIN_THREADS); + return config.get(PRIAM_PRE + ".rpc.min.threads", 16); } public int getRpcMaxThreads() { - return config.get(CONFIG_RPC_MAX_THREADS, DEFAULT_RPC_MAX_THREADS); + return config.get(PRIAM_PRE + ".rpc.max.threads", 2048); } @Override @@ -924,12 +581,12 @@ public int getCompactionLargePartitionWarnThresholdInMB() { } public String getExtraConfigParams() { - return config.get(CONFIG_EXTRA_PARAMS); + return config.get(PRIAM_PRE + ".extra.params"); } public Map getExtraEnvParams() { - String envParams = config.get(CONFIG_EXTRA_ENV_PARAMS); + String envParams = config.get(PRIAM_PRE + ".extra.env.params"); if (envParams == null) { logger.info("getExtraEnvParams: No extra env params"); return null; @@ -961,76 +618,69 @@ public String getCassYamlVal(String priamKey) { } public boolean getAutoBoostrap() { - return config.get(CONFIG_AUTO_BOOTSTRAP, true); + return config.get(PRIAM_PRE + ".auto.bootstrap", true); } @Override public boolean isCreateNewTokenEnable() { - return config.get(CONFIG_CREATE_NEW_TOKEN_ENABLE, true); + return config.get(PRIAM_PRE + ".create.new.token.enable", true); } @Override public String getPrivateKeyLocation() { - return config.get(CONFIG_PRIKEY_LOC); + return config.get(PRIAM_PRE + ".private.key.location"); } @Override public String getRestoreSourceType() { - return config.get(CONFIG_RESTORE_SOURCE_TYPE); + return config.get(PRIAM_PRE + ".restore.source.type"); } @Override public boolean isEncryptBackupEnabled() { - return config.get(CONFIG_ENCRYPTED_BACKUP_ENABLED, false); + return config.get(PRIAM_PRE + ".encrypted.backup.enabled", false); } @Override public String getAWSRoleAssumptionArn() { - return config.get(CONFIG_S3_ROLE_ASSUMPTION_ARN); + return config.get(PRIAM_PRE + ".roleassumption.arn"); } @Override public String getClassicEC2RoleAssumptionArn() { - return config.get(CONFIG_EC2_ROLE_ASSUMPTION_ARN); + return config.get(PRIAM_PRE + ".ec2.roleassumption.arn"); } @Override public String getVpcEC2RoleAssumptionArn() { - return config.get(CONFIG_VPC_ROLE_ASSUMPTION_ARN); + return config.get(PRIAM_PRE + ".vpc.roleassumption.arn"); } @Override public boolean isDualAccount() { - return config.get(CONFIG_DUAL_ACCOUNT, DEFAULT_DUAL_ACCOUNT); + return config.get(PRIAM_PRE + ".roleassumption.dualaccount", false); } @Override public String getGcsServiceAccountId() { - return config.get(CONFIG_GCS_SERVICE_ACCT_ID); + return config.get(PRIAM_PRE + ".gcs.service.acct.id"); } @Override public String getGcsServiceAccountPrivateKeyLoc() { return config.get( - CONFIG_GCS_SERVICE_ACCT_PRIVATE_KEY_LOC, "/apps/tomcat/conf/gcsentryptedkey.p12"); + PRIAM_PRE + ".gcs.service.acct.private.key", + "/apps/tomcat/conf/gcsentryptedkey.p12"); } @Override public String getPgpPasswordPhrase() { - return config.get(CONFIG_PGP_PASSWORD_PHRASE); + return config.get(PRIAM_PRE + ".pgp.password.phrase"); } @Override public String getPgpPublicKeyLoc() { - return config.get(CONFIG_PGP_PUB_KEY_LOC); - } - - @Override - /* - * @return the vpc id of the running instance. - */ - public String getVpcId() { - return NETWORK_VPC; + return config.get(PRIAM_PRE + ".pgp.pubkey.file.location"); } @Override @@ -1064,18 +714,17 @@ public long getDownloadTimeout() { @Override public int getTombstoneWarnThreshold() { - return config.get(CONFIG_TOMBSTONE_WARNING_THRESHOLD, DEFAULT_TOMBSTONE_WARNING_THRESHOLD); + return config.get(PRIAM_PRE + ".tombstone.warning.threshold", 1000); } @Override public int getTombstoneFailureThreshold() { - return config.get(CONFIG_TOMBSTONE_FAILURE_THRESHOLD, DEFAULT_TOMBSTONE_FAILURE_THRESHOLD); + return config.get(PRIAM_PRE + ".tombstone.failure.threshold", 100000); } @Override public int getStreamingSocketTimeoutInMS() { - return config.get( - CONFIG_STREAMING_SOCKET_TIMEOUT_IN_MS, DEFAULT_STREAMING_SOCKET_TIMEOUT_IN_MS); + return config.get(PRIAM_PRE + ".streaming.socket.timeout.ms", 86400000); } @Override @@ -1091,13 +740,13 @@ public String getFlushInterval() { @Override public String getBackupStatusFileLoc() { return config.get( - CONFIG_BACKUP_STATUS_FILE_LOCATION, + PRIAM_PRE + ".backup.status.location", getDataFileLocation() + File.separator + "backup.status"); } @Override public boolean useSudo() { - return config.get(CONFIG_CASS_USE_SUDO, true); + return config.get(PRIAM_PRE + ".cass.usesudo", true); } @Override @@ -1107,41 +756,41 @@ public String getBackupNotificationTopicArn() { @Override public boolean isPostRestoreHookEnabled() { - return config.get(CONFIG_POST_RESTORE_HOOK_ENABLED, false); + return config.get(PRIAM_PRE + ".postrestorehook.enabled", false); } @Override public String getPostRestoreHook() { - return config.get(CONFIG_POST_RESTORE_HOOK); + return config.get(PRIAM_PRE + ".postrestorehook"); } @Override public String getPostRestoreHookHeartbeatFileName() { return config.get( - CONFIG_POST_RESTORE_HOOK_HEARTBEAT_FILENAME, + PRIAM_PRE + ".postrestorehook.heartbeat.filename", getDataFileLocation() + File.separator + "postrestorehook_heartbeat"); } @Override public String getPostRestoreHookDoneFileName() { return config.get( - CONFIG_POST_RESTORE_HOOK_DONE_FILENAME, + PRIAM_PRE + ".postrestorehook.done.filename", getDataFileLocation() + File.separator + "postrestorehook_done"); } @Override public int getPostRestoreHookTimeOutInDays() { - return config.get(CONFIG_POST_RESTORE_HOOK_TIMEOUT_IN_DAYS, 2); + return config.get(PRIAM_PRE + ".postrestorehook.timeout.in.days", 2); } @Override public int getPostRestoreHookHeartBeatTimeoutInMs() { - return config.get(CONFIG_POST_RESTORE_HOOK_HEARTBEAT_TIMEOUT_MS, 120000); + return config.get(PRIAM_PRE + ".postrestorehook.heartbeat.timeout", 120000); } @Override public int getPostRestoreHookHeartbeatCheckFrequencyInMs() { - return config.get(CONFIG_POST_RESTORE_HOOK_HEARTBEAT_CHECK_FREQUENCY_MS, 120000); + return config.get(PRIAM_PRE + ".postrestorehook.heartbeat.check.frequency", 120000); } @Override diff --git a/priam/src/main/java/com/netflix/priam/configSource/AbstractConfigSource.java b/priam/src/main/java/com/netflix/priam/configSource/AbstractConfigSource.java index a95f00fdd..ee4485559 100644 --- a/priam/src/main/java/com/netflix/priam/configSource/AbstractConfigSource.java +++ b/priam/src/main/java/com/netflix/priam/configSource/AbstractConfigSource.java @@ -30,7 +30,7 @@ public abstract class AbstractConfigSource implements IConfigSource { private String region; @Override - public void intialize(final String asgName, final String region) { + public void initialize(final String asgName, final String region) { this.asgName = checkNotNull(asgName, "ASG name is not defined"); this.region = checkNotNull(region, "Region is not defined"); } diff --git a/priam/src/main/java/com/netflix/priam/configSource/CompositeConfigSource.java b/priam/src/main/java/com/netflix/priam/configSource/CompositeConfigSource.java index c9ee7e0d7..97703c24e 100644 --- a/priam/src/main/java/com/netflix/priam/configSource/CompositeConfigSource.java +++ b/priam/src/main/java/com/netflix/priam/configSource/CompositeConfigSource.java @@ -57,10 +57,10 @@ public CompositeConfigSource(final IConfigSource... sources) { } @Override - public void intialize(final String asgName, final String region) { + public void initialize(final String asgName, final String region) { for (final IConfigSource source : sources) { // TODO should this catch any potential exceptions? - source.intialize(asgName, region); + source.initialize(asgName, region); } } diff --git a/priam/src/main/java/com/netflix/priam/configSource/IConfigSource.java b/priam/src/main/java/com/netflix/priam/configSource/IConfigSource.java index b86b80f4e..08b336e0b 100644 --- a/priam/src/main/java/com/netflix/priam/configSource/IConfigSource.java +++ b/priam/src/main/java/com/netflix/priam/configSource/IConfigSource.java @@ -30,7 +30,7 @@ public interface IConfigSource { * @param asgName: Name of the asg * @param region: Name of the region */ - void intialize(String asgName, String region); + void initialize(String asgName, String region); /** * A non-negative integer indicating a count of elements. diff --git a/priam/src/main/java/com/netflix/priam/configSource/PropertiesConfigSource.java b/priam/src/main/java/com/netflix/priam/configSource/PropertiesConfigSource.java index 1cee1393e..40ce0246f 100644 --- a/priam/src/main/java/com/netflix/priam/configSource/PropertiesConfigSource.java +++ b/priam/src/main/java/com/netflix/priam/configSource/PropertiesConfigSource.java @@ -55,8 +55,8 @@ public PropertiesConfigSource(final Properties properties) { } @Override - public void intialize(final String asgName, final String region) { - super.intialize(asgName, region); + public void initialize(final String asgName, final String region) { + super.initialize(asgName, region); Properties properties = new Properties(); URL url = PropertiesConfigSource.class.getClassLoader().getResource(priamFile); if (url != null) { diff --git a/priam/src/main/java/com/netflix/priam/configSource/SimpleDBConfigSource.java b/priam/src/main/java/com/netflix/priam/configSource/SimpleDBConfigSource.java index b1a7b7ae4..e836d7f02 100644 --- a/priam/src/main/java/com/netflix/priam/configSource/SimpleDBConfigSource.java +++ b/priam/src/main/java/com/netflix/priam/configSource/SimpleDBConfigSource.java @@ -33,7 +33,7 @@ import org.slf4j.LoggerFactory; /** - * Loads config data from SimpleDB. {@link #intialize(String, String)} will query the SimpleDB + * Loads config data from SimpleDB. {@link #initialize(String, String)} will query the SimpleDB * domain "PriamProperties" for any potential configurations. The domain is set up to support * multiple different clusters; this is done by using amazon's auto scaling groups (ASG). * @@ -64,8 +64,8 @@ public SimpleDBConfigSource(final ICredential provider) { } @Override - public void intialize(final String asgName, final String region) { - super.intialize(asgName, region); + public void initialize(final String asgName, final String region) { + super.initialize(asgName, region); // End point is us-east-1 AmazonSimpleDB simpleDBClient = diff --git a/priam/src/main/java/com/netflix/priam/configSource/SystemPropertiesConfigSource.java b/priam/src/main/java/com/netflix/priam/configSource/SystemPropertiesConfigSource.java index 73cc9ea45..d975bce42 100644 --- a/priam/src/main/java/com/netflix/priam/configSource/SystemPropertiesConfigSource.java +++ b/priam/src/main/java/com/netflix/priam/configSource/SystemPropertiesConfigSource.java @@ -35,8 +35,8 @@ public final class SystemPropertiesConfigSource extends AbstractConfigSource { private final Map data = Maps.newConcurrentMap(); @Override - public void intialize(final String asgName, final String region) { - super.intialize(asgName, region); + public void initialize(final String asgName, final String region) { + super.initialize(asgName, region); Properties systemProps = System.getProperties(); diff --git a/priam/src/main/java/com/netflix/priam/identity/AwsInstanceEnvIdentity.java b/priam/src/main/java/com/netflix/priam/identity/AwsInstanceEnvIdentity.java deleted file mode 100644 index 4d47037c9..000000000 --- a/priam/src/main/java/com/netflix/priam/identity/AwsInstanceEnvIdentity.java +++ /dev/null @@ -1,60 +0,0 @@ -/** - * Copyright 2017 Netflix, Inc. - * - *

    Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file - * except in compliance with the License. You may obtain a copy of the License at - * - *

    http://www.apache.org/licenses/LICENSE-2.0 - * - *

    Unless required by applicable law or agreed to in writing, software distributed under the - * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.netflix.priam.identity; - -import com.google.inject.Singleton; -import com.netflix.priam.identity.config.AWSVpcInstanceDataRetriever; -import com.netflix.priam.identity.config.InstanceDataRetriever; - -/* - * A means to determine if running instance is within classic, default vpc account, or non-default vpc account - */ -@Singleton -public class AwsInstanceEnvIdentity implements InstanceEnvIdentity { - - private Boolean isClassic = false, isDefaultVpc = false, isNonDefaultVpc = false; - - public AwsInstanceEnvIdentity() { - String vpcId = getVpcId(); - if (vpcId == null || vpcId.isEmpty()) { - this.isClassic = true; - } else { - this.isNonDefaultVpc = - true; // our instances run under a non default ("persistence_*") AWS acct - } - } - - /* - * @return the vpc id of the running instance, null if instance is not running within vpc. - */ - private String getVpcId() { - InstanceDataRetriever insDataRetriever = new AWSVpcInstanceDataRetriever(); - return insDataRetriever.getVpcId(); - } - - @Override - public Boolean isClassic() { - return this.isClassic; - } - - @Override - public Boolean isDefaultVpc() { - return this.isDefaultVpc; - } - - @Override - public Boolean isNonDefaultVpc() { - return this.isNonDefaultVpc; - } -} diff --git a/priam/src/main/java/com/netflix/priam/identity/DoubleRing.java b/priam/src/main/java/com/netflix/priam/identity/DoubleRing.java index 5319d9a86..e0bb5898d 100644 --- a/priam/src/main/java/com/netflix/priam/identity/DoubleRing.java +++ b/priam/src/main/java/com/netflix/priam/identity/DoubleRing.java @@ -32,13 +32,18 @@ public class DoubleRing { private final IConfiguration config; private final IPriamInstanceFactory factory; private final ITokenManager tokenManager; + private final InstanceIdentity instanceIdentity; @Inject public DoubleRing( - IConfiguration config, IPriamInstanceFactory factory, ITokenManager tokenManager) { + IConfiguration config, + IPriamInstanceFactory factory, + ITokenManager tokenManager, + InstanceIdentity instanceIdentity) { this.config = config; this.factory = factory; this.tokenManager = tokenManager; + this.instanceIdentity = instanceIdentity; } /** @@ -51,7 +56,7 @@ public void doubleSlots() { // delete all for (PriamInstance data : local) factory.delete(data); - int hash = tokenManager.regionOffset(config.getDC()); + int hash = tokenManager.regionOffset(instanceIdentity.getInstanceInfo().getRegion()); // move existing slots. for (PriamInstance data : local) { int slot = (data.getId() - hash) * 2; @@ -74,13 +79,17 @@ public void doubleSlots() { currentSlot + 3 > new_ring_size ? (currentSlot + 3) - new_ring_size : currentSlot + 3; - String token = tokenManager.createToken(new_slot, new_ring_size, config.getDC()); + String token = + tokenManager.createToken( + new_slot, + new_ring_size, + instanceIdentity.getInstanceInfo().getRegion()); factory.create( data.getApp(), new_slot + hash, InstanceIdentity.DUMMY_INSTANCE_ID, - config.getHostname(), - config.getHostIP(), + instanceIdentity.getInstanceInfo().getHostname(), + instanceIdentity.getInstanceInfo().getHostIP(), data.getRac(), null, token); @@ -90,7 +99,9 @@ public void doubleSlots() { // filter other DC's private List filteredRemote(List lst) { List local = Lists.newArrayList(); - for (PriamInstance data : lst) if (data.getDC().equals(config.getDC())) local.add(data); + for (PriamInstance data : lst) + if (data.getDC().equals(instanceIdentity.getInstanceInfo().getRegion())) + local.add(data); return local; } diff --git a/priam/src/main/java/com/netflix/priam/identity/InstanceEnvIdentity.java b/priam/src/main/java/com/netflix/priam/identity/InstanceEnvIdentity.java deleted file mode 100644 index 2c583c990..000000000 --- a/priam/src/main/java/com/netflix/priam/identity/InstanceEnvIdentity.java +++ /dev/null @@ -1,43 +0,0 @@ -/** - * Copyright 2017 Netflix, Inc. - * - *

    Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file - * except in compliance with the License. You may obtain a copy of the License at - * - *

    http://www.apache.org/licenses/LICENSE-2.0 - * - *

    Unless required by applicable law or agreed to in writing, software distributed under the - * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.netflix.priam.identity; - -import com.google.inject.ImplementedBy; - -/* - * A means to determine the environment for the running instance - */ -@ImplementedBy(AwsInstanceEnvIdentity.class) -public interface InstanceEnvIdentity { - /* - * @return true if running instance is in "classic", false otherwise. - */ - Boolean isClassic(); - - /* - * @return true if running instance is in vpc, under your default AWS account, false otherwise. - */ - Boolean isDefaultVpc(); - - /* - * @return true if running instance is in vpc, under a specific AWS account, false otherwise. - */ - Boolean isNonDefaultVpc(); - - enum InstanceEnvironent { - CLASSIC, - DEFAULT_VPC, - NONDEFAULT_VPC - } -} diff --git a/priam/src/main/java/com/netflix/priam/identity/InstanceIdentity.java b/priam/src/main/java/com/netflix/priam/identity/InstanceIdentity.java index 056d1bb69..6825ab9f5 100644 --- a/priam/src/main/java/com/netflix/priam/identity/InstanceIdentity.java +++ b/priam/src/main/java/com/netflix/priam/identity/InstanceIdentity.java @@ -24,6 +24,7 @@ import com.google.inject.Inject; import com.google.inject.Singleton; import com.netflix.priam.config.IConfiguration; +import com.netflix.priam.identity.config.InstanceInfo; import com.netflix.priam.identity.token.IDeadTokenRetriever; import com.netflix.priam.identity.token.INewTokenRetriever; import com.netflix.priam.identity.token.IPreGeneratedTokenRetriever; @@ -87,6 +88,8 @@ public boolean test(PriamInstance input) { }; private PriamInstance myInstance; + // Instance information contains other information like ASG/vpc-id etc. + private InstanceInfo myInstanceInfo; private boolean isReplace = false; private boolean isTokenPregenerated = false; private String replacedIp = ""; @@ -105,7 +108,8 @@ public InstanceIdentity( ITokenManager tokenManager, IDeadTokenRetriever deadTokenRetriever, IPreGeneratedTokenRetriever preGeneratedTokenRetriever, - INewTokenRetriever newTokenRetriever) + INewTokenRetriever newTokenRetriever, + InstanceInfo instanceInfo) throws Exception { this.factory = factory; this.membership = membership; @@ -115,6 +119,7 @@ public InstanceIdentity( this.deadTokenRetriever = deadTokenRetriever; this.preGeneratedTokenRetriever = preGeneratedTokenRetriever; this.newTokenRetriever = newTokenRetriever; + this.myInstanceInfo = instanceInfo; init(); } @@ -122,19 +127,23 @@ public PriamInstance getInstance() { return myInstance; } + public InstanceInfo getInstanceInfo() { + return myInstanceInfo; + } + public void init() throws Exception { // try to grab the token which was already assigned myInstance = new RetryableCallable() { @Override public PriamInstance retriableCall() throws Exception { - // Check if this node is decomissioned + // Check if this node is decommissioned List deadInstances = factory.getAllIds(config.getAppName() + "-dead"); for (PriamInstance ins : deadInstances) { logger.info( "[Dead] Iterating though the hosts: {}", ins.getInstanceId()); - if (ins.getInstanceId().equals(config.getInstanceName())) { + if (ins.getInstanceId().equals(myInstanceInfo.getInstanceId())) { ins.setOutOfService(true); logger.info( "[Dead] found that this node is dead." @@ -161,7 +170,7 @@ public PriamInstance retriableCall() throws Exception { "[Alive] Iterating though the hosts: {} My id = [{}]", ins.getInstanceId(), ins.getId()); - if (ins.getInstanceId().equals(config.getInstanceName())) { + if (ins.getInstanceId().equals(myInstanceInfo.getInstanceId())) { logger.info( "[Alive] found that this node is alive." + " application: {}" diff --git a/priam/src/main/java/com/netflix/priam/identity/config/AWSInstanceInfo.java b/priam/src/main/java/com/netflix/priam/identity/config/AWSInstanceInfo.java new file mode 100644 index 000000000..ac742109b --- /dev/null +++ b/priam/src/main/java/com/netflix/priam/identity/config/AWSInstanceInfo.java @@ -0,0 +1,214 @@ +/** + * Copyright 2017 Netflix, Inc. + * + *

    Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file + * except in compliance with the License. You may obtain a copy of the License at + * + *

    http://www.apache.org/licenses/LICENSE-2.0 + * + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.netflix.priam.identity.config; + +import com.amazonaws.services.ec2.AmazonEC2; +import com.amazonaws.services.ec2.AmazonEC2ClientBuilder; +import com.amazonaws.services.ec2.model.DescribeInstancesRequest; +import com.amazonaws.services.ec2.model.DescribeInstancesResult; +import com.amazonaws.services.ec2.model.Instance; +import com.amazonaws.services.ec2.model.Reservation; +import com.google.inject.Inject; +import com.google.inject.Singleton; +import com.netflix.priam.cred.ICredential; +import com.netflix.priam.utils.RetryableCallable; +import com.netflix.priam.utils.SystemUtils; +import org.apache.commons.lang3.StringUtils; +import org.codehaus.jettison.json.JSONException; +import org.codehaus.jettison.json.JSONObject; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +@Singleton +public class AWSInstanceInfo implements InstanceInfo { + private static final Logger logger = LoggerFactory.getLogger(AWSInstanceInfo.class); + private JSONObject identityDocument = null; + private String privateIp; + private String publicIp; + private String rac; + private String publicHostName; + private String instanceId; + private String instanceType; + private String mac; + private String availabilityZone; + private ICredential credential; + private String vpcId; + private InstanceEnvironment instanceEnvironment; + + @Inject + public AWSInstanceInfo(ICredential credential) { + this.credential = credential; + } + + @Override + public String getPrivateIP() { + if (privateIp == null) { + privateIp = + SystemUtils.getDataFromUrl( + "http://169.254.169.254/latest/meta-data/local-ipv4"); + } + return privateIp; + } + + @Override + public String getRac() { + if (rac == null) { + rac = + SystemUtils.getDataFromUrl( + "http://169.254.169.254/latest/meta-data/placement/availability-zone"); + } + return rac; + } + + public String getPublicHostname() { + if (publicHostName == null) { + publicHostName = + SystemUtils.getDataFromUrl( + "http://169.254.169.254/latest/meta-data/public-hostname"); + } + return publicHostName; + } + + public String getPublicIP() { + if (publicIp == null) { + publicIp = + SystemUtils.getDataFromUrl( + "http://169.254.169.254/latest/meta-data/public-ipv4"); + } + return publicIp; + } + + @Override + public String getInstanceId() { + if (instanceId == null) { + instanceId = + SystemUtils.getDataFromUrl( + "http://169.254.169.254/latest/meta-data/instance-id"); + } + return instanceId; + } + + @Override + public String getInstanceType() { + if (instanceType == null) { + instanceType = + SystemUtils.getDataFromUrl( + "http://169.254.169.254/latest/meta-data/instance-type"); + } + return instanceType; + } + + private String getMac() { + if (mac == null) { + mac = + SystemUtils.getDataFromUrl( + "http://169.254.169.254/latest/meta-data/network/interfaces/macs/") + .trim(); + } + return mac; + } + + @Override + public String getRegion() { + try { + getIdentityDocument(); + return this.identityDocument.getString("region"); + } catch (JSONException e) { + // If there is any issue in getting region, use AZ as backup. + return getRac().substring(0, getRac().length() - 1); + } + } + + private void getIdentityDocument() throws JSONException { + if (this.identityDocument == null) { + String jsonStr = + SystemUtils.getDataFromUrl( + "http://169.254.169.254/latest/dynamic/instance-identity/document"); + this.identityDocument = new JSONObject(jsonStr); + } + } + + @Override + public String getVpcId() { + String nacId = getMac(); + if (StringUtils.isEmpty(nacId)) return null; + + if (vpcId == null) + try { + vpcId = + SystemUtils.getDataFromUrl( + "http://169.254.169.254/latest/meta-data/network/interfaces/macs/" + + nacId + + "vpc-id") + .trim(); + } catch (Exception e) { + logger.info( + "Vpc id does not exist for running instance, not fatal as running instance maybe not be in vpc. Msg: {}", + e.getLocalizedMessage()); + } + + return vpcId; + } + + @Override + public String getAutoScalingGroup() { + final AmazonEC2 client = + AmazonEC2ClientBuilder.standard() + .withCredentials(credential.getAwsCredentialProvider()) + .withRegion(getRegion()) + .build(); + try { + return new RetryableCallable(15, 30000) { + public String retriableCall() throws IllegalStateException { + DescribeInstancesRequest desc = + new DescribeInstancesRequest().withInstanceIds(getInstanceId()); + DescribeInstancesResult res = client.describeInstances(desc); + + for (Reservation resr : res.getReservations()) { + for (Instance ins : resr.getInstances()) { + for (com.amazonaws.services.ec2.model.Tag tag : ins.getTags()) { + if (tag.getKey().equals("aws:autoscaling:groupName")) + return tag.getValue(); + } + } + } + + throw new IllegalStateException("Couldn't determine ASG name"); + } + }.call(); + } catch (Exception e) { + logger.error("Failed to determine ASG name.", e); + return null; + } + } + + @Override + public InstanceEnvironment getInstanceEnvironment() { + if (instanceEnvironment == null) { + instanceEnvironment = + (getVpcId() == null) ? InstanceEnvironment.CLASSIC : InstanceEnvironment.VPC; + } + return instanceEnvironment; + } + + @Override + public String getHostname() { + return (getPublicHostname() == null) ? getPrivateIP() : getPublicHostname(); + } + + @Override + public String getHostIP() { + return (getPublicIP() == null) ? getPrivateIP() : getPublicIP(); + } +} diff --git a/priam/src/main/java/com/netflix/priam/identity/config/AWSVpcInstanceDataRetriever.java b/priam/src/main/java/com/netflix/priam/identity/config/AWSVpcInstanceDataRetriever.java deleted file mode 100644 index 7e0564b4b..000000000 --- a/priam/src/main/java/com/netflix/priam/identity/config/AWSVpcInstanceDataRetriever.java +++ /dev/null @@ -1,47 +0,0 @@ -/** - * Copyright 2017 Netflix, Inc. - * - *

    Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file - * except in compliance with the License. You may obtain a copy of the License at - * - *

    http://www.apache.org/licenses/LICENSE-2.0 - * - *

    Unless required by applicable law or agreed to in writing, software distributed under the - * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.netflix.priam.identity.config; - -import com.netflix.priam.utils.SystemUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * Calls AWS metadata to get info on the location of the running instance within vpc environment. - */ -public class AWSVpcInstanceDataRetriever extends InstanceDataRetrieverBase { - private static final Logger logger = LoggerFactory.getLogger(AWSVpcInstanceDataRetriever.class); - - @Override - public String getVpcId() { - String nacId = getMac(); - if (nacId == null || nacId.isEmpty()) return null; - - String vpcId = null; - try { - vpcId = - SystemUtils.getDataFromUrl( - "http://169.254.169.254/latest/meta-data/network/interfaces/macs/" - + nacId - + "vpc-id") - .trim(); - } catch (Exception e) { - logger.info( - "Vpc id does not exist for running instance, not fatal as running instance maybe not be in vpc. Msg: {}", - e.getLocalizedMessage()); - } - - return vpcId; - } -} diff --git a/priam/src/main/java/com/netflix/priam/identity/config/AwsClassicInstanceDataRetriever.java b/priam/src/main/java/com/netflix/priam/identity/config/AwsClassicInstanceDataRetriever.java deleted file mode 100644 index 27b986772..000000000 --- a/priam/src/main/java/com/netflix/priam/identity/config/AwsClassicInstanceDataRetriever.java +++ /dev/null @@ -1,26 +0,0 @@ -/** - * Copyright 2017 Netflix, Inc. - * - *

    Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file - * except in compliance with the License. You may obtain a copy of the License at - * - *

    http://www.apache.org/licenses/LICENSE-2.0 - * - *

    Unless required by applicable law or agreed to in writing, software distributed under the - * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.netflix.priam.identity.config; - -/** - * Calls AWS metadata to get info on the location of the running instance within classic - * environment. - */ -public class AwsClassicInstanceDataRetriever extends InstanceDataRetrieverBase { - @Override - public String getVpcId() { - throw new UnsupportedOperationException( - "Not applicable as running instance is in classic environment"); - } -} diff --git a/priam/src/main/java/com/netflix/priam/identity/config/InstanceDataRetrieverBase.java b/priam/src/main/java/com/netflix/priam/identity/config/InstanceDataRetrieverBase.java deleted file mode 100644 index bb32a3b67..000000000 --- a/priam/src/main/java/com/netflix/priam/identity/config/InstanceDataRetrieverBase.java +++ /dev/null @@ -1,79 +0,0 @@ -/** - * Copyright 2017 Netflix, Inc. - * - *

    Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file - * except in compliance with the License. You may obtain a copy of the License at - * - *

    http://www.apache.org/licenses/LICENSE-2.0 - * - *

    Unless required by applicable law or agreed to in writing, software distributed under the - * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.netflix.priam.identity.config; - -import com.netflix.priam.utils.SystemUtils; -import org.codehaus.jettison.json.JSONException; -import org.codehaus.jettison.json.JSONObject; - -public abstract class InstanceDataRetrieverBase implements InstanceDataRetriever { - protected JSONObject identityDocument = null; - - public String getPrivateIP() { - return SystemUtils.getDataFromUrl("http://169.254.169.254/latest/meta-data/local-ipv4"); - } - - public String getRac() { - return SystemUtils.getDataFromUrl( - "http://169.254.169.254/latest/meta-data/placement/availability-zone"); - } - - public String getPublicHostname() { - return SystemUtils.getDataFromUrl( - "http://169.254.169.254/latest/meta-data/public-hostname"); - } - - public String getPublicIP() { - return SystemUtils.getDataFromUrl("http://169.254.169.254/latest/meta-data/public-ipv4"); - } - - public String getInstanceId() { - return SystemUtils.getDataFromUrl("http://169.254.169.254/latest/meta-data/instance-id"); - } - - public String getInstanceType() { - return SystemUtils.getDataFromUrl("http://169.254.169.254/latest/meta-data/instance-type"); - } - - public String getMac() { - return SystemUtils.getDataFromUrl( - "http://169.254.169.254/latest/meta-data/network/interfaces/macs/") - .trim(); - } - - public String getAWSAccountId() throws JSONException { - if (this.identityDocument == null) { - String jsonStr = - SystemUtils.getDataFromUrl( - "http://169.254.169.254/latest/dynamic/instance-identity/document"); - this.identityDocument = new JSONObject(jsonStr); - } - return this.identityDocument.getString("accountId"); - } - - public String getRegion() throws JSONException { - if (this.identityDocument == null) { - String jsonStr = - SystemUtils.getDataFromUrl( - "http://169.254.169.254/latest/dynamic/instance-identity/document"); - this.identityDocument = new JSONObject(jsonStr); - } - return this.identityDocument.getString("region"); - } - - public String getAvailabilityZone() throws JSONException { - return SystemUtils.getDataFromUrl( - "http://169.254.169.254/latest/meta-data/placement/availability-zone"); - } -} diff --git a/priam/src/main/java/com/netflix/priam/identity/config/InstanceDataRetriever.java b/priam/src/main/java/com/netflix/priam/identity/config/InstanceInfo.java similarity index 59% rename from priam/src/main/java/com/netflix/priam/identity/config/InstanceDataRetriever.java rename to priam/src/main/java/com/netflix/priam/identity/config/InstanceInfo.java index 0f402492f..92c2cee1d 100644 --- a/priam/src/main/java/com/netflix/priam/identity/config/InstanceDataRetriever.java +++ b/priam/src/main/java/com/netflix/priam/identity/config/InstanceInfo.java @@ -13,10 +13,11 @@ */ package com.netflix.priam.identity.config; -import org.codehaus.jettison.json.JSONException; +import com.google.inject.ImplementedBy; /** A means to fetch meta data of running instance */ -public interface InstanceDataRetriever { +@ImplementedBy(AWSInstanceInfo.class) +public interface InstanceInfo { /** * Get the availability zone of the running instance. * @@ -25,19 +26,20 @@ public interface InstanceDataRetriever { String getRac(); /** - * Get the public hostname for the running instance. + * Get the hostname for the running instance. Cannot be null. * * @return the public hostname for the running instance. e.g.: - * ec2-12-34-56-78.compute-1.amazonaws.com + * ec2-12-34-56-78.compute-1.amazonaws.com, if available. Else return private ip address for + * running instance. */ - String getPublicHostname(); + String getHostname(); /** - * Get public ip address for running instance. Can be null. + * Get ip address for running instance. Cannot be null. * - * @return private ip address for running instance. + * @return public ip if one is provided or private ip address for running instance. */ - String getPublicIP(); + String getHostIP(); /** * Get private ip address for running instance. @@ -60,13 +62,6 @@ public interface InstanceDataRetriever { */ String getInstanceType(); - /** - * Get the id of the network interface for running instance. - * - * @return id of the network interface for running instance - */ - String getMac(); - /** * Get the id of the vpc account for running instance. * @@ -75,27 +70,30 @@ public interface InstanceDataRetriever { String getVpcId(); // the id of the vpc for running instance /** - * AWS Account ID of running instance. + * Get the region/data center of the running instance * - * @return the id (e.g. 12345) of the AWS account of running instance, could be null /empty. - * @throws JSONException + * @return the region of the running instance, could be null/empty (e.g. us-east-1) */ - String getAWSAccountId() throws JSONException; + String getRegion(); /** - * Get the region of the AWS account of running instance + * Get the ASG in which this instance is deployed. Note that Priam requires instances to be + * under an ASG. * - * @return the region (e.g. us-east-1) of the AWS account of running instance, could be null - * /empty. - * @throws JSONException + * @return the ASG of the instance. ex: cassandra_app--useast1e */ - String getRegion() throws JSONException; + String getAutoScalingGroup(); /** - * Get the availability zone of the running instance. + * Environment of the current running instance. AWS only allows VPC environment (default). + * Classic is deprecated environment by AWS. * - * @return the availability zone of the running instance. e.g. us-east-1c - * @throws JSONException + * @return Environment of the current running instance. */ - String getAvailabilityZone() throws JSONException; + InstanceEnvironment getInstanceEnvironment(); + + enum InstanceEnvironment { + CLASSIC, + VPC + } } diff --git a/priam/src/main/java/com/netflix/priam/identity/config/LocalInstanceDataRetriever.java b/priam/src/main/java/com/netflix/priam/identity/config/LocalInstanceInfo.java similarity index 67% rename from priam/src/main/java/com/netflix/priam/identity/config/LocalInstanceDataRetriever.java rename to priam/src/main/java/com/netflix/priam/identity/config/LocalInstanceInfo.java index f12d19ff0..be89bb918 100644 --- a/priam/src/main/java/com/netflix/priam/identity/config/LocalInstanceDataRetriever.java +++ b/priam/src/main/java/com/netflix/priam/identity/config/LocalInstanceInfo.java @@ -13,25 +13,26 @@ */ package com.netflix.priam.identity.config; -import org.codehaus.jettison.json.JSONException; - /** * Looks at local (system) properties for metadata about the running 'instance'. Typically, this is * used for locally-deployed testing. */ -public class LocalInstanceDataRetriever implements InstanceDataRetriever { +public class LocalInstanceInfo implements InstanceInfo { private static final String PREFIX = "Priam.localInstance."; + @Override public String getRac() { return System.getProperty(PREFIX + "availabilityZone", ""); } - public String getPublicHostname() { - return System.getProperty(PREFIX + "publicHostname", ""); + @Override + public String getHostname() { + return System.getProperty(PREFIX + "privateIp", ""); } - public String getPublicIP() { - return System.getProperty(PREFIX + "publicIp", ""); + @Override + public String getHostIP() { + return System.getProperty(PREFIX + "privateIp", ""); } @Override @@ -39,36 +40,33 @@ public String getPrivateIP() { return System.getProperty(PREFIX + "privateIp", ""); } + @Override public String getInstanceId() { return System.getProperty(PREFIX + "instanceId", ""); } + @Override public String getInstanceType() { return System.getProperty(PREFIX + "instanceType", ""); } - @Override - public String getMac() { - return System.getProperty(PREFIX + "networkinterface", ""); - } - @Override public String getVpcId() { return System.getProperty(PREFIX + "vpcid", ""); } @Override - public String getAWSAccountId() throws JSONException { - return System.getProperty(PREFIX + "awsacctid", ""); + public String getAutoScalingGroup() { + return System.getProperty(PREFIX + "asg", ""); } @Override - public String getAvailabilityZone() throws JSONException { - return System.getProperty(PREFIX + "availabilityzone", ""); + public InstanceEnvironment getInstanceEnvironment() { + return (getVpcId() == null) ? InstanceEnvironment.CLASSIC : InstanceEnvironment.VPC; } @Override - public String getRegion() throws JSONException { + public String getRegion() { return System.getProperty(PREFIX + "region", ""); } } diff --git a/priam/src/main/java/com/netflix/priam/identity/token/DeadTokenRetriever.java b/priam/src/main/java/com/netflix/priam/identity/token/DeadTokenRetriever.java index 8e59975b2..0e66fbf59 100755 --- a/priam/src/main/java/com/netflix/priam/identity/token/DeadTokenRetriever.java +++ b/priam/src/main/java/com/netflix/priam/identity/token/DeadTokenRetriever.java @@ -18,8 +18,8 @@ import com.netflix.priam.config.IConfiguration; import com.netflix.priam.identity.IMembership; import com.netflix.priam.identity.IPriamInstanceFactory; -import com.netflix.priam.identity.InstanceEnvIdentity; import com.netflix.priam.identity.PriamInstance; +import com.netflix.priam.identity.config.InstanceInfo; import com.netflix.priam.utils.Sleeper; import com.sun.jersey.api.client.Client; import com.sun.jersey.api.client.ClientResponse; @@ -46,7 +46,7 @@ public class DeadTokenRetriever extends TokenRetrieverBase implements IDeadToken private String replacedIp; // The IP address of the dead instance to which we will acquire its token private ListMultimap locMap; - private final InstanceEnvIdentity insEnvIdentity; + private final InstanceInfo instanceInfo; @Inject public DeadTokenRetriever( @@ -54,12 +54,12 @@ public DeadTokenRetriever( IMembership membership, IConfiguration config, Sleeper sleeper, - InstanceEnvIdentity insEnvIdentity) { + InstanceInfo instanceInfo) { this.factory = factory; this.membership = membership; this.config = config; this.sleeper = sleeper; - this.insEnvIdentity = insEnvIdentity; + this.instanceInfo = instanceInfo; } private List getDualAccountRacMembership(List asgInstances) { @@ -68,7 +68,7 @@ private List getDualAccountRacMembership(List asgInstances) { List crossAccountAsgInstances = membership.getCrossAccountRacMembership(); if (logger.isInfoEnabled()) { - if (insEnvIdentity.isClassic()) { + if (instanceInfo.getInstanceEnvironment() == InstanceInfo.InstanceEnvironment.CLASSIC) { logger.info( "EC2 classic instances (local ASG): " + Arrays.toString(asgInstances.toArray())); @@ -109,7 +109,7 @@ public PriamInstance get() throws Exception { sleeper.sleep(new Random().nextInt(5000) + 10000); for (PriamInstance dead : allIds) { // test same zone and is it is alive. - if (!dead.getRac().equals(config.getRac()) + if (!dead.getRac().equals(instanceInfo.getRac()) || asgInstances.contains(dead.getInstanceId()) || super.isInstanceDummy(dead)) continue; logger.info("Found dead instances: {}", dead.getInstanceId()); @@ -138,10 +138,10 @@ public PriamInstance get() throws Exception { return factory.create( config.getAppName(), markAsDead.getId(), - config.getInstanceName(), - config.getHostname(), - config.getHostIP(), - config.getRac(), + instanceInfo.getInstanceId(), + instanceInfo.getHostname(), + instanceInfo.getHostIP(), + instanceInfo.getRac(), markAsDead.getVolumes(), payLoad); } diff --git a/priam/src/main/java/com/netflix/priam/identity/token/NewTokenRetriever.java b/priam/src/main/java/com/netflix/priam/identity/token/NewTokenRetriever.java index 9f072a8a2..c94471b48 100755 --- a/priam/src/main/java/com/netflix/priam/identity/token/NewTokenRetriever.java +++ b/priam/src/main/java/com/netflix/priam/identity/token/NewTokenRetriever.java @@ -19,6 +19,7 @@ import com.netflix.priam.identity.IMembership; import com.netflix.priam.identity.IPriamInstanceFactory; import com.netflix.priam.identity.PriamInstance; +import com.netflix.priam.identity.config.InstanceInfo; import com.netflix.priam.utils.ITokenManager; import com.netflix.priam.utils.Sleeper; import java.util.List; @@ -35,6 +36,7 @@ public class NewTokenRetriever extends TokenRetrieverBase implements INewTokenRe private final Sleeper sleeper; private final ITokenManager tokenManager; private ListMultimap locMap; + private InstanceInfo instanceInfo; @Inject // Note: do not parameterized the generic type variable to an implementation as it confuses @@ -44,12 +46,14 @@ public NewTokenRetriever( IMembership membership, IConfiguration config, Sleeper sleeper, - ITokenManager tokenManager) { + ITokenManager tokenManager, + InstanceInfo instanceInfo) { this.factory = factory; this.membership = membership; this.config = config; this.sleeper = sleeper; this.tokenManager = tokenManager; + this.instanceInfo = instanceInfo; } @Override @@ -58,7 +62,7 @@ public PriamInstance get() throws Exception { logger.info("Generating my own and new token"); // Sleep random interval - upto 15 sec sleeper.sleep(new Random().nextInt(15000)); - int hash = tokenManager.regionOffset(config.getDC()); + int hash = tokenManager.regionOffset(instanceInfo.getRegion()); // use this hash so that the nodes are spred far away from the other // regions. @@ -66,18 +70,19 @@ public PriamInstance get() throws Exception { List allInstances = factory.getAllIds(config.getAppName()); for (PriamInstance data : allInstances) max = - (data.getRac().equals(config.getRac()) && (data.getId() > max)) + (data.getRac().equals(instanceInfo.getRac()) && (data.getId() > max)) ? data.getId() : max; int maxSlot = max - hash; int my_slot; - if (hash == max && locMap.get(config.getRac()).size() == 0) { - int idx = config.getRacs().indexOf(config.getRac()); + if (hash == max && locMap.get(instanceInfo.getRac()).size() == 0) { + int idx = config.getRacs().indexOf(instanceInfo.getRac()); if (idx < 0) throw new Exception( String.format( - "Rac %s is not in Racs %s", config.getRac(), config.getRacs())); + "Rac %s is not in Racs %s", + instanceInfo.getRac(), config.getRacs())); my_slot = idx + maxSlot; } else my_slot = config.getRacs().size() + maxSlot; @@ -86,20 +91,20 @@ public PriamInstance get() throws Exception { my_slot, membership.getRacCount(), membership.getRacMembershipSize(), - config.getDC()); + instanceInfo.getRegion()); String payload = tokenManager.createToken( my_slot, membership.getRacCount(), membership.getRacMembershipSize(), - config.getDC()); + instanceInfo.getRegion()); return factory.create( config.getAppName(), my_slot + hash, - config.getInstanceName(), - config.getHostname(), - config.getHostIP(), - config.getRac(), + instanceInfo.getInstanceId(), + instanceInfo.getHostname(), + instanceInfo.getHostIP(), + instanceInfo.getRac(), null, payload); } diff --git a/priam/src/main/java/com/netflix/priam/identity/token/PreGeneratedTokenRetriever.java b/priam/src/main/java/com/netflix/priam/identity/token/PreGeneratedTokenRetriever.java index a4c7a940b..dda193459 100755 --- a/priam/src/main/java/com/netflix/priam/identity/token/PreGeneratedTokenRetriever.java +++ b/priam/src/main/java/com/netflix/priam/identity/token/PreGeneratedTokenRetriever.java @@ -19,6 +19,7 @@ import com.netflix.priam.identity.IMembership; import com.netflix.priam.identity.IPriamInstanceFactory; import com.netflix.priam.identity.PriamInstance; +import com.netflix.priam.identity.config.InstanceInfo; import com.netflix.priam.utils.Sleeper; import java.util.List; import java.util.Random; @@ -34,17 +35,20 @@ public class PreGeneratedTokenRetriever extends TokenRetrieverBase private final IConfiguration config; private final Sleeper sleeper; private ListMultimap locMap; + private InstanceInfo instanceInfo; @Inject public PreGeneratedTokenRetriever( IPriamInstanceFactory factory, IMembership membership, IConfiguration config, - Sleeper sleeper) { + Sleeper sleeper, + InstanceInfo instanceInfo) { this.factory = factory; this.membership = membership; this.config = config; this.sleeper = sleeper; + this.instanceInfo = instanceInfo; } @Override @@ -57,7 +61,7 @@ public PriamInstance get() throws Exception { sleeper.sleep(new Random().nextInt(5000) + 10000); for (PriamInstance dead : allIds) { // test same zone and is it is alive. - if (!dead.getRac().equals(config.getRac()) + if (!dead.getRac().equals(instanceInfo.getRac()) || asgInstances.contains(dead.getInstanceId()) || !isInstanceDummy(dead)) continue; logger.info("Found pre-generated token: {}", dead.getToken()); @@ -82,10 +86,10 @@ public PriamInstance get() throws Exception { return factory.create( config.getAppName(), markAsDead.getId(), - config.getInstanceName(), - config.getHostname(), - config.getHostIP(), - config.getRac(), + instanceInfo.getInstanceId(), + instanceInfo.getHostname(), + instanceInfo.getHostIP(), + instanceInfo.getRac(), markAsDead.getVolumes(), payLoad); } diff --git a/priam/src/main/java/com/netflix/priam/notification/AWSSnsNotificationService.java b/priam/src/main/java/com/netflix/priam/notification/AWSSnsNotificationService.java index 4fcc8ef41..8774ac881 100644 --- a/priam/src/main/java/com/netflix/priam/notification/AWSSnsNotificationService.java +++ b/priam/src/main/java/com/netflix/priam/notification/AWSSnsNotificationService.java @@ -25,6 +25,7 @@ import com.google.inject.Singleton; import com.netflix.priam.aws.IAMCredential; import com.netflix.priam.config.IConfiguration; +import com.netflix.priam.identity.config.InstanceInfo; import com.netflix.priam.merics.BackupMetrics; import com.netflix.priam.utils.BoundedExponentialRetryCallable; import java.util.Map; @@ -45,10 +46,13 @@ public class AWSSnsNotificationService implements INotificationService { @Inject public AWSSnsNotificationService( - IConfiguration config, IAMCredential iamCredential, BackupMetrics backupMetrics) { + IConfiguration config, + IAMCredential iamCredential, + BackupMetrics backupMetrics, + InstanceInfo instanceInfo) { this.configuration = config; this.backupMetrics = backupMetrics; - String ec2_region = this.configuration.getDC(); + String ec2_region = instanceInfo.getRegion(); snsClient = AmazonSNSClient.builder() .withCredentials(iamCredential.getAwsCredentialProvider()) diff --git a/priam/src/main/java/com/netflix/priam/notification/BackupNotificationMgr.java b/priam/src/main/java/com/netflix/priam/notification/BackupNotificationMgr.java index da1414575..158b40216 100644 --- a/priam/src/main/java/com/netflix/priam/notification/BackupNotificationMgr.java +++ b/priam/src/main/java/com/netflix/priam/notification/BackupNotificationMgr.java @@ -17,6 +17,7 @@ import com.google.inject.Inject; import com.netflix.priam.backup.AbstractBackupPath; import com.netflix.priam.config.IConfiguration; +import com.netflix.priam.identity.config.InstanceInfo; import java.util.HashMap; import java.util.Map; import org.codehaus.jettison.json.JSONException; @@ -37,11 +38,16 @@ public class BackupNotificationMgr implements EventObserver { private static final Logger logger = LoggerFactory.getLogger(BackupNotificationMgr.class); private final IConfiguration config; private final INotificationService notificationService; + private final InstanceInfo instanceInfo; @Inject - public BackupNotificationMgr(IConfiguration config, INotificationService notificationService) { + public BackupNotificationMgr( + IConfiguration config, + INotificationService notificationService, + InstanceInfo instanceInfo) { this.config = config; this.notificationService = notificationService; + this.instanceInfo = instanceInfo; } private void notify(AbstractBackupPath abp, String uploadStatus) { @@ -53,7 +59,7 @@ private void notify(AbstractBackupPath abp, String uploadStatus) { jsonObject.put("keyspace", abp.getKeyspace()); jsonObject.put("cf", abp.getColumnFamily()); jsonObject.put("region", abp.getRegion()); - jsonObject.put("rack", this.config.getRac()); + jsonObject.put("rack", instanceInfo.getRac()); jsonObject.put("token", abp.getToken()); jsonObject.put("filename", abp.getFileName()); jsonObject.put("uncompressfilesize", abp.getSize()); diff --git a/priam/src/main/java/com/netflix/priam/resources/BackupServlet.java b/priam/src/main/java/com/netflix/priam/resources/BackupServlet.java index f2eb9f711..0b180152c 100644 --- a/priam/src/main/java/com/netflix/priam/resources/BackupServlet.java +++ b/priam/src/main/java/com/netflix/priam/resources/BackupServlet.java @@ -16,35 +16,23 @@ */ package com.netflix.priam.resources; -import com.google.common.collect.Lists; import com.google.inject.Inject; -import com.google.inject.Provider; import com.google.inject.name.Named; -import com.netflix.priam.PriamServer; import com.netflix.priam.backup.*; import com.netflix.priam.backup.AbstractBackupPath.BackupFileType; import com.netflix.priam.config.IConfiguration; -import com.netflix.priam.defaultimpl.ICassandraProcess; -import com.netflix.priam.identity.IPriamInstanceFactory; -import com.netflix.priam.identity.PriamInstance; -import com.netflix.priam.restore.Restore; import com.netflix.priam.scheduler.PriamScheduler; -import com.netflix.priam.tuner.ICassandraTuner; -import com.netflix.priam.utils.*; -import java.io.File; -import java.io.IOException; -import java.math.BigInteger; +import com.netflix.priam.utils.DateUtil; +import com.netflix.priam.utils.SystemUtils; import java.util.ArrayList; import java.util.Date; import java.util.Iterator; import java.util.List; -import java.util.concurrent.*; import java.util.stream.Collectors; import javax.ws.rs.*; import javax.ws.rs.core.MediaType; import javax.ws.rs.core.Response; import org.apache.commons.lang3.StringUtils; -import org.apache.commons.lang3.exception.ExceptionUtils; import org.codehaus.jettison.json.JSONArray; import org.codehaus.jettison.json.JSONException; import org.codehaus.jettison.json.JSONObject; @@ -60,58 +48,24 @@ public class BackupServlet { private static final String REST_SUCCESS = "[\"ok\"]"; private static final String REST_HEADER_RANGE = "daterange"; private static final String REST_HEADER_FILTER = "filter"; - private static final String REST_HEADER_TOKEN = "token"; - private static final String REST_HEADER_REGION = "region"; - private static final String REST_KEYSPACES = "keyspaces"; - private static final String REST_RESTORE_PREFIX = "restoreprefix"; - private static final String FMT = "yyyyMMddHHmm"; - private static final String REST_LOCR_ROWKEY = "verifyrowkey"; - private static final String REST_LOCR_KEYSPACE = "verifyks"; - private static final String REST_LOCR_COLUMNFAMILY = "verifycf"; - private static final String REST_LOCR_FILEEXTENSION = "verifyfileextension"; - private static final String SSTABLE2JSON_DIR_LOCATION = "/tmp/priam_sstables"; - private static final String SSTABLE2JSON_COMMAND_FROM_CASSHOME = "/bin/sstable2json"; - - private final PriamServer priamServer; private final IConfiguration config; private final IBackupFileSystem backupFs; - private final Restore restoreObj; - private final Provider pathProvider; - private final ICassandraTuner tuner; private final SnapshotBackup snapshotBackup; - private final IPriamInstanceFactory factory; - private final ITokenManager tokenManager; - private final ICassandraProcess cassProcess; private final BackupVerification backupVerification; @Inject private PriamScheduler scheduler; - @Inject private MetaData metaData; - private final IBackupStatusMgr completedBkups; + @Inject private MetaData metaData; @Inject public BackupServlet( - PriamServer priamServer, IConfiguration config, @Named("backup") IBackupFileSystem backupFs, - Restore restoreObj, - Provider pathProvider, - ICassandraTuner tuner, SnapshotBackup snapshotBackup, - IPriamInstanceFactory factory, - ITokenManager tokenManager, - ICassandraProcess cassProcess, IBackupStatusMgr completedBkups, BackupVerification backupVerification) { - this.priamServer = priamServer; this.config = config; this.backupFs = backupFs; - this.restoreObj = restoreObj; - this.pathProvider = pathProvider; - this.tuner = tuner; this.snapshotBackup = snapshotBackup; - this.factory = factory; - this.tokenManager = tokenManager; - this.cassProcess = cassProcess; this.completedBkups = completedBkups; this.backupVerification = backupVerification; } @@ -152,9 +106,8 @@ public Response list( endTime = new DateTime().toDate(); } else { String[] restore = daterange.split(","); - AbstractBackupPath path = pathProvider.get(); - startTime = path.parseDate(restore[0]); - endTime = path.parseDate(restore[1]); + startTime = DateUtil.getDate(restore[0]); + endTime = DateUtil.getDate(restore[1]); } logger.info( @@ -174,12 +127,6 @@ public Response list( @Path("/status") @Produces(MediaType.APPLICATION_JSON) public Response status() throws Exception { - int restoreQueueSize = - restoreObj.getDownloadTasksQueued(); // Items left to restore from the filesystem. - logger.info( - "Thread counts for restore is: {}. Items in queue: {}", - config.getRestoreThreads(), - restoreQueueSize); JSONObject object = new JSONObject(); object.put("SnapshotStatus", snapshotBackup.state().toString()); return Response.ok(object.toString(), MediaType.APPLICATION_JSON).build(); @@ -332,148 +279,6 @@ public Response validateSnapshotByDate(@PathParam("daterange") String daterange) return Response.ok(jsonReply.toString()).build(); } - /** - * Life_Of_C*Row : With this REST call, mutations/existence of a rowkey can be found. It uses - * SSTable2Json utility which will convert SSTables on disk to JSON format and Search for the - * desired rowkey. - * - *

    Steps include: 1. Restoring data for given data range and other params 2. Searching - * provided rowkey in SSTables and writing search result to JSON 3. Delete all the files under - * Keyspace Directory. Deletion is done for efficient space usage, so that same node can be - * reused for subsequent runs. - * - *

    - * - *

    Similar to Restore call and few additional params. - * - *

    daterange : Can not be Null or Default. Comma separated Start and End date eg. - * 201311250000,201311260000 rowkey : rowkey to search (In Hex format) ks : keyspace of - * mentioned rowkey cf : column family of mentioned rowkey fileExtension : Part of SSTable Data - * file names eg. if file name = KS1-CF1-hf-100-Data.db then fileExtension = KS1-CF1-hf - * - * @return Creates JSON file based on the passed date at hardcoded dir location : - * /tmp/priam_sstables If rowkey is not found in the SSTable, JSON file will be empty. - */ - @GET - @Path("/life_of_crow") - @Produces(MediaType.APPLICATION_JSON) - public Response restore_verify_key( - @QueryParam(REST_HEADER_RANGE) String daterange, - @QueryParam(REST_HEADER_REGION) String region, - @QueryParam(REST_HEADER_TOKEN) String token, - @QueryParam(REST_KEYSPACES) String keyspaces, - @QueryParam(REST_RESTORE_PREFIX) String restorePrefix, - @QueryParam(REST_LOCR_ROWKEY) String rowkey, - @QueryParam(REST_LOCR_KEYSPACE) String ks, - @QueryParam(REST_LOCR_COLUMNFAMILY) String cf, - @QueryParam(REST_LOCR_FILEEXTENSION) String fileExtension) - throws Exception { - - Date startTime; - Date endTime; - // Creating Dir for Json storage - SystemUtils.createDirs(SSTABLE2JSON_DIR_LOCATION); - String JSON_FILE_PATH; - - try { - - if (StringUtils.isBlank(daterange) || daterange.equalsIgnoreCase("default")) { - return Response.ok( - "\n[\"daterange can't be blank or default.eg.201311250000,201311260000\"]\n", - MediaType.APPLICATION_JSON) - .build(); - } - - String[] restore = daterange.split(","); - AbstractBackupPath path = pathProvider.get(); - startTime = path.parseDate(restore[0]); - endTime = path.parseDate(restore[1]); - - String origRestorePrefix = config.getRestorePrefix(); - if (StringUtils.isNotBlank(restorePrefix)) { - config.setRestorePrefix(restorePrefix); - } - - restore(token, region, startTime, endTime, keyspaces); - - // Since this call is probably never called in parallel, config is - // multi-thread safe to be edited - config.setRestorePrefix(origRestorePrefix); - - while (!CassandraMonitor.hasCassadraStarted()) Thread.sleep(1000L); - - // initialize json file name - JSON_FILE_PATH = daterange.split(",")[0].substring(0, 8) + ".json"; - - // Convert SSTable2Json and search for given rowkey - checkSSTablesForKey(rowkey, ks, cf, fileExtension, JSON_FILE_PATH); - - } catch (Exception e) { - logger.info(ExceptionUtils.getStackTrace(e)); - } finally { - removeAllDataFiles(ks); - } - - return Response.ok(REST_SUCCESS, MediaType.APPLICATION_JSON).build(); - } - - /** - * Restore with the specified start and end time. - * - * @param token Overrides the current token with this one, if specified - * @param region Override the region for searching backup - * @param startTime Start time - * @param endTime End time upto which the restore should fetch data - * @param keyspaces Comma seperated list of keyspaces to restore - * @throws Exception if restore is not successful - */ - private void restore( - String token, String region, Date startTime, Date endTime, String keyspaces) - throws Exception { - String origRegion = config.getDC(); - String origToken = priamServer.getId().getInstance().getToken(); - if (StringUtils.isNotBlank(token)) priamServer.getId().getInstance().setToken(token); - - if (config.isRestoreClosestToken()) - priamServer - .getId() - .getInstance() - .setToken( - closestToken( - priamServer.getId().getInstance().getToken(), config.getDC())); - - if (StringUtils.isNotBlank(region)) { - config.setDC(region); - logger.info("Restoring from region {}", region); - priamServer - .getId() - .getInstance() - .setToken(closestToken(priamServer.getId().getInstance().getToken(), region)); - logger.info("Restore will use token {}", priamServer.getId().getInstance().getToken()); - } - - restoreObj.setRestoreConfiguration(keyspaces, null); - - try { - restoreObj.restore(startTime, endTime); - } finally { - config.setDC(origRegion); - priamServer.getId().getInstance().setToken(origToken); - } - tuner.updateAutoBootstrap(config.getYamlLocation(), false); - cassProcess.start(true); - } - - /** Find closest token in the specified region */ - private String closestToken(String token, String region) { - List plist = factory.getAllIds(config.getAppName()); - List tokenList = Lists.newArrayList(); - for (PriamInstance ins : plist) { - if (ins.getDC().equalsIgnoreCase(region)) tokenList.add(new BigInteger(ins.getToken())); - } - return tokenManager.findClosestToken(new BigInteger(token), tokenList).toString(); - } - /* * A list of files for requested filter. Currently, the only supported filter is META, all others will be ignore. * For filter of META, ONLY the daily snapshot meta file (meta.json) are accounted for, not the incremental meta file. @@ -500,10 +305,10 @@ private JSONObject constructJsonResponse( backupJSON.put("app", p.getClusterName()); backupJSON.put("region", p.getRegion()); backupJSON.put("token", p.getToken()); - backupJSON.put("ts", new DateTime(p.getTime()).toString(FMT)); + backupJSON.put("ts", DateUtil.formatyyyyMMddHHmm(p.getTime())); backupJSON.put( "instance_id", p.getInstanceIdentity().getInstance().getInstanceId()); - backupJSON.put("uploaded_ts", new DateTime(p.getUploadedTs()).toString(FMT)); + backupJSON.put("uploaded_ts", DateUtil.formatyyyyMMddHHmm(p.getUploadedTs())); if ("meta".equalsIgnoreCase(filter)) { // only check for existence of meta file p.setFileName( "meta.json"); // ignore incremental meta files, we are only interested @@ -526,77 +331,4 @@ private JSONObject constructJsonResponse( } return object; } - - /** Convert SSTable2Json and search for given key */ - public void checkSSTablesForKey( - String rowkey, String keyspace, String cf, String fileExtension, String jsonFilePath) - throws Exception { - try { - logger.info("Starting SSTable2Json conversion ..."); - // Setting timeout to 10 Mins - long TIMEOUT_PERIOD = 10L; - String unixCmd = - formulateCommandToRun(rowkey, keyspace, cf, fileExtension, jsonFilePath); - - String[] cmd = {"/bin/sh", "-c", unixCmd}; - final Process p = Runtime.getRuntime().exec(cmd); - - Callable callable = p::waitFor; - - ExecutorService exeService = Executors.newSingleThreadExecutor(); - try { - Future future = exeService.submit(callable); - int returnVal = future.get(TIMEOUT_PERIOD, TimeUnit.MINUTES); - if (returnVal == 0) logger.info("Finished SSTable2Json conversion and search."); - else logger.error("Error occurred during SSTable2Json conversion and search."); - } catch (TimeoutException e) { - logger.error(ExceptionUtils.getStackTrace(e)); - throw e; - } finally { - p.destroy(); - exeService.shutdown(); - } - - } catch (IOException e) { - logger.error(ExceptionUtils.getStackTrace(e)); - } - } - - public String formulateCommandToRun( - String rowkey, String keyspace, String cf, String fileExtension, String jsonFilePath) { - StringBuffer sbuff = new StringBuffer(); - - sbuff.append("for i in $(ls ") - .append(config.getDataFileLocation()) - .append(File.separator) - .append(keyspace) - .append(File.separator) - .append(cf) - .append(File.separator) - .append(fileExtension) - .append("-*-Data.db); do ") - .append(config.getCassHome()) - .append(SSTABLE2JSON_COMMAND_FROM_CASSHOME) - .append(" $i -k "); - sbuff.append(rowkey); - sbuff.append(" | grep "); - sbuff.append(rowkey); - sbuff.append(" >> "); - sbuff.append(SSTABLE2JSON_DIR_LOCATION).append(File.separator).append(jsonFilePath); - sbuff.append(" ; done"); - - logger.info( - "SSTable2JSON location <" + SSTABLE2JSON_DIR_LOCATION + "{}{}>", - File.separator, - jsonFilePath); - logger.info("Running Command = {}", sbuff); - return sbuff.toString(); - } - - public void removeAllDataFiles(String ks) throws Exception { - String cleanupDirPath = config.getDataFileLocation() + File.separator + ks; - logger.info("Starting to clean all the files inside <{}>", cleanupDirPath); - SystemUtils.cleanupDir(cleanupDirPath, null); - logger.info("*** Done cleaning all the files inside <{}>", cleanupDirPath); - } } diff --git a/priam/src/main/java/com/netflix/priam/resources/CassandraConfig.java b/priam/src/main/java/com/netflix/priam/resources/CassandraConfig.java index 9634bbd6e..c19a969e9 100644 --- a/priam/src/main/java/com/netflix/priam/resources/CassandraConfig.java +++ b/priam/src/main/java/com/netflix/priam/resources/CassandraConfig.java @@ -55,7 +55,7 @@ public CassandraConfig(PriamServer server, DoubleRing doubleRing) { @Path("/get_seeds") public Response getSeeds() { try { - final List seeds = priamServer.getId().getSeeds(); + final List seeds = priamServer.getInstanceIdentity().getSeeds(); if (!seeds.isEmpty()) return Response.ok(StringUtils.join(seeds, ',')).build(); logger.error("Cannot find the Seeds"); } catch (Exception e) { @@ -69,10 +69,11 @@ public Response getSeeds() { @Path("/get_token") public Response getToken() { try { - String token = priamServer.getId().getInstance().getToken(); + String token = priamServer.getInstanceIdentity().getInstance().getToken(); if (StringUtils.isNotBlank(token)) { logger.info("Returning token value \"{}\" for this instance to caller.", token); - return Response.ok(priamServer.getId().getInstance().getToken()).build(); + return Response.ok(priamServer.getInstanceIdentity().getInstance().getToken()) + .build(); } logger.error("Cannot find token for this instance."); @@ -88,7 +89,8 @@ public Response getToken() { @Path("/is_replace_token") public Response isReplaceToken() { try { - return Response.ok(String.valueOf(priamServer.getId().isReplace())).build(); + return Response.ok(String.valueOf(priamServer.getInstanceIdentity().isReplace())) + .build(); } catch (Exception e) { // TODO: can this ever happen? if so, what conditions would cause an exception here? logger.error("Error while executing is_replace_token", e); @@ -100,7 +102,8 @@ public Response isReplaceToken() { @Path("/get_replaced_ip") public Response getReplacedIp() { try { - return Response.ok(String.valueOf(priamServer.getId().getReplacedIp())).build(); + return Response.ok(String.valueOf(priamServer.getInstanceIdentity().getReplacedIp())) + .build(); } catch (Exception e) { logger.error("Error while executing get_replaced_ip", e); return Response.serverError().build(); @@ -111,7 +114,7 @@ public Response getReplacedIp() { @Path("/set_replaced_ip") public Response setReplacedIp(@QueryParam("ip") String ip) { try { - priamServer.getId().setReplacedIp(ip); + priamServer.getInstanceIdentity().setReplacedIp(ip); return Response.ok().build(); } catch (Exception e) { logger.error("Error while overriding replacement ip", e); diff --git a/priam/src/main/java/com/netflix/priam/resources/PriamInstanceResource.java b/priam/src/main/java/com/netflix/priam/resources/PriamInstanceResource.java index 828f4294a..883f06fba 100644 --- a/priam/src/main/java/com/netflix/priam/resources/PriamInstanceResource.java +++ b/priam/src/main/java/com/netflix/priam/resources/PriamInstanceResource.java @@ -20,6 +20,7 @@ import com.netflix.priam.config.IConfiguration; import com.netflix.priam.identity.IPriamInstanceFactory; import com.netflix.priam.identity.PriamInstance; +import com.netflix.priam.identity.config.InstanceInfo; import java.net.URI; import java.util.List; import javax.ws.rs.*; @@ -37,13 +38,16 @@ public class PriamInstanceResource { private final IConfiguration config; private final IPriamInstanceFactory factory; + private final InstanceInfo instanceInfo; @Inject - // Note: do not parameterized the generic type variable to an implementation as it confuses + // Note: do not parameterize the generic type variable to an implementation as it confuses // Guice in the binding. - public PriamInstanceResource(IConfiguration config, IPriamInstanceFactory factory) { + public PriamInstanceResource( + IConfiguration config, IPriamInstanceFactory factory, InstanceInfo instanceInfo) { this.config = config; this.factory = factory; + this.instanceInfo = instanceInfo; } /** @@ -126,7 +130,8 @@ public Response deleteInstance(@PathParam("id") int id) { * @return PriamInstance with the given {@code id} */ private PriamInstance getByIdIfFound(int id) { - PriamInstance instance = factory.getInstance(config.getAppName(), config.getDC(), id); + PriamInstance instance = + factory.getInstance(config.getAppName(), instanceInfo.getRegion(), id); if (instance == null) { throw notFound(String.format("No priam instance with id %s found", id)); } diff --git a/priam/src/main/java/com/netflix/priam/resources/RestoreServlet.java b/priam/src/main/java/com/netflix/priam/resources/RestoreServlet.java index 545211b12..c4c32e098 100644 --- a/priam/src/main/java/com/netflix/priam/resources/RestoreServlet.java +++ b/priam/src/main/java/com/netflix/priam/resources/RestoreServlet.java @@ -13,22 +13,11 @@ */ package com.netflix.priam.resources; -import com.google.common.collect.Lists; import com.google.inject.Inject; -import com.google.inject.Provider; -import com.netflix.priam.PriamServer; -import com.netflix.priam.backup.AbstractBackupPath; -import com.netflix.priam.config.IConfiguration; -import com.netflix.priam.defaultimpl.ICassandraProcess; import com.netflix.priam.health.InstanceState; -import com.netflix.priam.identity.IPriamInstanceFactory; -import com.netflix.priam.identity.PriamInstance; import com.netflix.priam.restore.Restore; -import com.netflix.priam.tuner.ICassandraTuner; -import com.netflix.priam.utils.ITokenManager; -import java.math.BigInteger; +import com.netflix.priam.utils.DateUtil; import java.util.Date; -import java.util.List; import javax.ws.rs.GET; import javax.ws.rs.Path; import javax.ws.rs.Produces; @@ -45,51 +34,25 @@ public class RestoreServlet { private static final Logger logger = LoggerFactory.getLogger(RestoreServlet.class); - private static final String REST_HEADER_RANGE = "daterange"; - private static final String REST_HEADER_REGION = "region"; - private static final String REST_HEADER_TOKEN = "token"; - private static final String REST_KEYSPACES = "keyspaces"; - private static final String REST_RESTORE_PREFIX = "restoreprefix"; - private static final String REST_SUCCESS = "[\"ok\"]"; - - private final IConfiguration config; private final Restore restoreObj; - private final Provider pathProvider; - private final PriamServer priamServer; - private final IPriamInstanceFactory factory; - private final ICassandraTuner tuner; - private final ICassandraProcess cassProcess; - private final ITokenManager tokenManager; private final InstanceState instanceState; @Inject - public RestoreServlet( - IConfiguration config, - Restore restoreObj, - Provider pathProvider, - PriamServer priamServer, - IPriamInstanceFactory factory, - ICassandraTuner tuner, - ICassandraProcess cassProcess, - ITokenManager tokenManager, - InstanceState instanceState) { - this.config = config; + public RestoreServlet(Restore restoreObj, InstanceState instanceState) { this.restoreObj = restoreObj; - this.pathProvider = pathProvider; - this.priamServer = priamServer; - this.factory = factory; - this.tuner = tuner; - this.cassProcess = cassProcess; - this.tokenManager = tokenManager; this.instanceState = instanceState; } /* * @return metadata of current restore. If no restore in progress, returns the metadata of most recent restore attempt. - * status:[not_started|running|success|failure] - * daterange:[startdaterange,enddatarange] - * starttime:[yyyymmddhhmm] - * endtime:[yyyymmddmm] + * restoreStatus: { + * startDateRange: "[yyyymmddhhmm]", + * endDateRange: "[yyyymmddhhmm]", + * executionStartTime: "[yyyymmddhhmm]", + * executionEndTime: "[yyyymmddhhmm]", + * snapshotMetaFile: " used for full snapshot", + * status: "STARTED|FINISHED|FAILED" + * } */ @GET @Path("/restore/status") @@ -99,13 +62,7 @@ public Response status() throws Exception { @GET @Path("/restore") - public Response restore( - @QueryParam(REST_HEADER_RANGE) String daterange, - @QueryParam(REST_HEADER_REGION) String region, - @QueryParam(REST_HEADER_TOKEN) String token, - @QueryParam(REST_KEYSPACES) String keyspaces, - @QueryParam(REST_RESTORE_PREFIX) String restorePrefix) - throws Exception { + public Response restore(@QueryParam("daterange") String daterange) throws Exception { Date startTime; Date endTime; @@ -114,89 +71,12 @@ public Response restore( endTime = new DateTime().toDate(); } else { String[] restore = daterange.split(","); - AbstractBackupPath path = pathProvider.get(); - startTime = path.parseDate(restore[0]); - endTime = path.parseDate(restore[1]); - } - - String origRestorePrefix = config.getRestorePrefix(); - if (StringUtils.isNotBlank(restorePrefix)) { - config.setRestorePrefix(restorePrefix); - } - - logger.info( - "Parameters: { token: [{}], region: [{}], startTime: [{}], endTime: [{}], keyspaces: [{}], restorePrefix: [{}]}", - token, - region, - startTime, - endTime, - keyspaces, - restorePrefix); - - restore(token, region, startTime, endTime, keyspaces); - - // Since this call is probably never called in parallel, config is multi-thread safe to be - // edited - if (origRestorePrefix != null) config.setRestorePrefix(origRestorePrefix); - else config.setRestorePrefix(""); - - return Response.ok(REST_SUCCESS, MediaType.APPLICATION_JSON).build(); - } - - /** - * Restore with the specified start and end time. - * - * @param token Overrides the current token with this one, if specified - * @param region Override the region for searching backup - * @param startTime Start time - * @param endTime End time upto which the restore should fetch data - * @param keyspaces Comma seperated list of keyspaces to restore - * @throws Exception - */ - private void restore( - String token, String region, Date startTime, Date endTime, String keyspaces) - throws Exception { - String origRegion = config.getDC(); - String origToken = priamServer.getId().getInstance().getToken(); - if (StringUtils.isNotBlank(token)) priamServer.getId().getInstance().setToken(token); - - if (config.isRestoreClosestToken()) - priamServer - .getId() - .getInstance() - .setToken( - closestToken( - priamServer.getId().getInstance().getToken(), config.getDC())); - - if (StringUtils.isNotBlank(region)) { - config.setDC(region); - logger.info("Restoring from region {}", region); - priamServer - .getId() - .getInstance() - .setToken(closestToken(priamServer.getId().getInstance().getToken(), region)); - logger.info("Restore will use token {}", priamServer.getId().getInstance().getToken()); + startTime = DateUtil.getDate(restore[0]); + endTime = DateUtil.getDate(restore[1]); } - restoreObj.setRestoreConfiguration(keyspaces, null); - - try { - restoreObj.restore(startTime, endTime); - } finally { - config.setDC(origRegion); - priamServer.getId().getInstance().setToken(origToken); - } - tuner.updateAutoBootstrap(config.getYamlLocation(), false); - cassProcess.start(true); - } - - /** Find closest token in the specified region */ - private String closestToken(String token, String region) { - List plist = factory.getAllIds(config.getAppName()); - List tokenList = Lists.newArrayList(); - for (PriamInstance ins : plist) { - if (ins.getDC().equalsIgnoreCase(region)) tokenList.add(new BigInteger(ins.getToken())); - } - return tokenManager.findClosestToken(new BigInteger(token), tokenList).toString(); + logger.info("Parameters: {startTime: [{}], endTime: [{}]}", startTime, endTime); + restoreObj.restore(startTime, endTime); + return Response.ok("[\"ok\"]", MediaType.APPLICATION_JSON).build(); } } diff --git a/priam/src/main/java/com/netflix/priam/restore/AbstractRestore.java b/priam/src/main/java/com/netflix/priam/restore/AbstractRestore.java index ddde21a91..a5b965b4c 100644 --- a/priam/src/main/java/com/netflix/priam/restore/AbstractRestore.java +++ b/priam/src/main/java/com/netflix/priam/restore/AbstractRestore.java @@ -25,6 +25,7 @@ import com.netflix.priam.defaultimpl.ICassandraProcess; import com.netflix.priam.health.InstanceState; import com.netflix.priam.identity.InstanceIdentity; +import com.netflix.priam.identity.config.InstanceInfo; import com.netflix.priam.scheduler.Task; import com.netflix.priam.utils.*; import java.io.File; @@ -58,7 +59,7 @@ public abstract class AbstractRestore extends Task implements IRestoreStrategy { final Sleeper sleeper; private final BackupRestoreUtil backupRestoreUtil; private final Provider pathProvider; - private final InstanceIdentity id; + private final InstanceIdentity instanceIdentity; private final RestoreTokenSelector tokenSelector; private final ICassandraProcess cassProcess; private final InstanceState instanceState; @@ -81,7 +82,7 @@ public abstract class AbstractRestore extends Task implements IRestoreStrategy { this.fs = fs; this.sleeper = sleeper; this.pathProvider = pathProvider; - this.id = instanceIdentity; + this.instanceIdentity = instanceIdentity; this.tokenSelector = tokenSelector; this.cassProcess = cassProcess; this.metaData = metaData; @@ -92,11 +93,11 @@ public abstract class AbstractRestore extends Task implements IRestoreStrategy { this.postRestoreHook = postRestoreHook; } - public static final boolean isRestoreEnabled(IConfiguration conf) { + public static final boolean isRestoreEnabled(IConfiguration conf, InstanceInfo instanceInfo) { boolean isRestoreMode = StringUtils.isNotBlank(conf.getRestoreSnapshot()); boolean isBackedupRac = (CollectionUtils.isEmpty(conf.getBackupRacs()) - || conf.getBackupRacs().contains(conf.getRac())); + || conf.getBackupRacs().contains(instanceInfo.getRac())); return (isRestoreMode && isBackedupRac); } @@ -207,13 +208,12 @@ private void fetchSnapshotMetaFile( @Override public void execute() throws Exception { - if (!isRestoreEnabled(config)) return; + if (!isRestoreEnabled(config, instanceIdentity.getInstanceInfo())) return; logger.info("Starting restore for {}", config.getRestoreSnapshot()); String[] restore = config.getRestoreSnapshot().split(","); - AbstractBackupPath path = pathProvider.get(); - final Date startTime = path.parseDate(restore[0]); - final Date endTime = path.parseDate(restore[1]); + final Date startTime = DateUtil.getDate(restore[0]); + final Date endTime = DateUtil.getDate(restore[1]); new RetryableCallable() { public Void retriableCall() throws Exception { logger.info("Attempting restore"); @@ -239,12 +239,12 @@ public void restore(Date startTime, Date endTime) throws Exception { instanceState.getRestoreStatus().setEndDateRange(DateUtil.convert(endTime)); instanceState.getRestoreStatus().setExecutionStartTime(LocalDateTime.now()); instanceState.setRestoreStatus(Status.STARTED); - String origToken = id.getInstance().getToken(); + String origToken = instanceIdentity.getInstance().getToken(); try { if (config.isRestoreClosestToken()) { restoreToken = tokenSelector.getClosestToken(new BigInteger(origToken), startTime); - id.getInstance().setToken(restoreToken.toString()); + instanceIdentity.getInstance().setToken(restoreToken.toString()); } // Stop cassandra if its running @@ -331,7 +331,7 @@ public void restore(Date startTime, Date endTime) throws Exception { logger.error("Error while trying to restore: {}", e.getMessage(), e); throw e; } finally { - id.getInstance().setToken(origToken); + instanceIdentity.getInstance().setToken(origToken); } } diff --git a/priam/src/main/java/com/netflix/priam/tuner/StandardTuner.java b/priam/src/main/java/com/netflix/priam/tuner/StandardTuner.java index 63ae4c9ca..f48b82002 100644 --- a/priam/src/main/java/com/netflix/priam/tuner/StandardTuner.java +++ b/priam/src/main/java/com/netflix/priam/tuner/StandardTuner.java @@ -17,6 +17,7 @@ import com.google.inject.Inject; import com.netflix.priam.backup.SnapshotBackup; import com.netflix.priam.config.IConfiguration; +import com.netflix.priam.identity.config.InstanceInfo; import com.netflix.priam.restore.Restore; import java.io.*; import java.util.List; @@ -35,10 +36,12 @@ public class StandardTuner implements ICassandraTuner { private static final Logger logger = LoggerFactory.getLogger(StandardTuner.class); protected final IConfiguration config; + private final InstanceInfo instanceInfo; @Inject - public StandardTuner(IConfiguration config) { + public StandardTuner(IConfiguration config, InstanceInfo instanceInfo) { this.config = config; + this.instanceInfo = instanceInfo; } public void writeAllProperties(String yamlLocation, String hostname, String seedProvider) @@ -58,7 +61,7 @@ public void writeAllProperties(String yamlLocation, String hostname, String seed map.put("listen_address", hostname); map.put("rpc_address", hostname); // Dont bootstrap in restore mode - if (!Restore.isRestoreEnabled(config)) { + if (!Restore.isRestoreEnabled(config, instanceInfo)) { map.put("auto_bootstrap", config.getAutoBoostrap()); } else { map.put("auto_bootstrap", false); @@ -70,9 +73,9 @@ public void writeAllProperties(String yamlLocation, String hostname, String seed map.put("data_file_directories", Lists.newArrayList(config.getDataFileLocation())); boolean enableIncremental = - (SnapshotBackup.isBackupEnabled(config) && config.isIncrBackup()) + (SnapshotBackup.isBackupEnabled(config) && config.isIncrementalBackupEnabled()) && (CollectionUtils.isEmpty(config.getBackupRacs()) - || config.getBackupRacs().contains(config.getRac())); + || config.getBackupRacs().contains(instanceInfo.getRac())); map.put("incremental_backups", enableIncremental); map.put("endpoint_snitch", config.getSnitch()); @@ -109,8 +112,7 @@ public void writeAllProperties(String yamlLocation, String hostname, String seed map.put("rpc_max_threads", config.getRpcMaxThreads()); // Add private ip address as broadcast_rpc_address. This will ensure that COPY function // works correctly. - map.put("broadcast_rpc_address", config.getInstanceDataRetriever().getPrivateIP()); - // map.put("index_interval", config.getIndexInterval()); + map.put("broadcast_rpc_address", instanceInfo.getPrivateIP()); map.put("tombstone_warn_threshold", config.getTombstoneWarnThreshold()); map.put("tombstone_failure_threshold", config.getTombstoneFailureThreshold()); diff --git a/priam/src/main/java/com/netflix/priam/tuner/dse/DseTuner.java b/priam/src/main/java/com/netflix/priam/tuner/dse/DseTuner.java index 81f5751bc..d5e11cf24 100644 --- a/priam/src/main/java/com/netflix/priam/tuner/dse/DseTuner.java +++ b/priam/src/main/java/com/netflix/priam/tuner/dse/DseTuner.java @@ -18,6 +18,7 @@ import com.google.inject.Inject; import com.netflix.priam.config.IConfiguration; +import com.netflix.priam.identity.config.InstanceInfo; import com.netflix.priam.tuner.StandardTuner; import java.io.FileReader; import java.io.FileWriter; @@ -40,8 +41,11 @@ public class DseTuner extends StandardTuner { @Inject public DseTuner( - IConfiguration config, IDseConfiguration dseConfig, IAuditLogTuner auditLogTuner) { - super(config); + IConfiguration config, + IDseConfiguration dseConfig, + IAuditLogTuner auditLogTuner, + InstanceInfo instanceInfo) { + super(config, instanceInfo); this.dseConfig = dseConfig; this.auditLogTuner = auditLogTuner; } diff --git a/priam/src/main/resources/Priam.properties b/priam/src/main/resources/Priam.properties index 8c1ccd42e..b392c2fde 100644 --- a/priam/src/main/resources/Priam.properties +++ b/priam/src/main/resources/Priam.properties @@ -1,3 +1,4 @@ +priam.clustername=cass_cluster priam.direct.memory.size.m1.large=1G priam.heap.newgen.size.m1.large=2G priam.heap.size.m1.large=4G diff --git a/priam/src/test/java/com/netflix/priam/TestModule.java b/priam/src/test/java/com/netflix/priam/TestModule.java index 5da711e30..15529c61b 100644 --- a/priam/src/test/java/com/netflix/priam/TestModule.java +++ b/priam/src/test/java/com/netflix/priam/TestModule.java @@ -32,6 +32,8 @@ import com.netflix.priam.identity.FakePriamInstanceFactory; import com.netflix.priam.identity.IMembership; import com.netflix.priam.identity.IPriamInstanceFactory; +import com.netflix.priam.identity.config.FakeInstanceInfo; +import com.netflix.priam.identity.config.InstanceInfo; import com.netflix.priam.utils.FakeSleeper; import com.netflix.priam.utils.Sleeper; import com.netflix.spectator.api.DefaultRegistry; @@ -45,11 +47,11 @@ public class TestModule extends AbstractModule { @Override protected void configure() { - bind(IConfiguration.class) - .toInstance( - new FakeConfiguration( - FakeConfiguration.FAKE_REGION, "fake-app", "az1", "fakeInstance1")); + bind(IConfiguration.class).toInstance(new FakeConfiguration("fake-app")); bind(IBackupRestoreConfig.class).to(FakeBackupRestoreConfig.class); + bind(InstanceInfo.class) + .toInstance(new FakeInstanceInfo("fakeInstance1", "az1", "us-east-1")); + bind(IPriamInstanceFactory.class).to(FakePriamInstanceFactory.class); bind(SchedulerFactory.class).to(StdSchedulerFactory.class).in(Scopes.SINGLETON); bind(IMembership.class) diff --git a/priam/src/test/java/com/netflix/priam/backup/BRTestModule.java b/priam/src/test/java/com/netflix/priam/backup/BRTestModule.java index ec29a9cb0..934d29f5a 100644 --- a/priam/src/test/java/com/netflix/priam/backup/BRTestModule.java +++ b/priam/src/test/java/com/netflix/priam/backup/BRTestModule.java @@ -22,7 +22,6 @@ import com.google.inject.name.Names; import com.netflix.priam.aws.auth.IS3Credential; import com.netflix.priam.aws.auth.S3RoleAssumptionCredential; -import com.netflix.priam.backup.identity.FakeInstanceEnvIdentity; import com.netflix.priam.config.FakeBackupRestoreConfig; import com.netflix.priam.config.FakeConfiguration; import com.netflix.priam.config.IBackupRestoreConfig; @@ -33,6 +32,8 @@ import com.netflix.priam.defaultimpl.FakeCassandraProcess; import com.netflix.priam.defaultimpl.ICassandraProcess; import com.netflix.priam.identity.*; +import com.netflix.priam.identity.config.FakeInstanceInfo; +import com.netflix.priam.identity.config.InstanceInfo; import com.netflix.priam.restore.IPostRestoreHook; import com.netflix.priam.utils.FakeSleeper; import com.netflix.priam.utils.Sleeper; @@ -48,11 +49,11 @@ public class BRTestModule extends AbstractModule { @Override protected void configure() { - bind(IConfiguration.class) - .toInstance( - new FakeConfiguration( - FakeConfiguration.FAKE_REGION, "fake-app", "az1", "fakeInstance1")); + bind(IConfiguration.class).toInstance(new FakeConfiguration("fake-app")); bind(IBackupRestoreConfig.class).to(FakeBackupRestoreConfig.class); + bind(InstanceInfo.class) + .toInstance(new FakeInstanceInfo("fakeInstance1", "az1", "us-east-1")); + bind(IPriamInstanceFactory.class).to(FakePriamInstanceFactory.class); bind(SchedulerFactory.class).to(StdSchedulerFactory.class).in(Scopes.SINGLETON); bind(IMembership.class).toInstance(new FakeMembership(Arrays.asList("fakeInstance1"))); @@ -73,7 +74,6 @@ protected void configure() { bind(IFileCryptography.class) .annotatedWith(Names.named("filecryptoalgorithm")) .to(PgpCryptography.class); - bind(InstanceEnvIdentity.class).to(FakeInstanceEnvIdentity.class); bind(ICassandraProcess.class).to(FakeCassandraProcess.class); bind(IPostRestoreHook.class).to(FakePostRestoreHook.class); bind(Registry.class).toInstance(new DefaultRegistry()); diff --git a/priam/src/test/java/com/netflix/priam/backup/FakeBackupFileSystem.java b/priam/src/test/java/com/netflix/priam/backup/FakeBackupFileSystem.java index 6331352e4..294047fc6 100644 --- a/priam/src/test/java/com/netflix/priam/backup/FakeBackupFileSystem.java +++ b/priam/src/test/java/com/netflix/priam/backup/FakeBackupFileSystem.java @@ -50,21 +50,21 @@ public FakeBackupFileSystem( public void setupTest(List files) { clearTest(); - flist = new ArrayList(); + flist = new ArrayList<>(); for (String file : files) { S3BackupPath path = pathProvider.get(); path.parseRemote(file); flist.add(path); } - downloadedFiles = new HashSet(); - uploadedFiles = new HashSet(); + downloadedFiles = new HashSet<>(); + uploadedFiles = new HashSet<>(); } public void setupTest() { clearTest(); - flist = new ArrayList(); - downloadedFiles = new HashSet(); - uploadedFiles = new HashSet(); + flist = new ArrayList<>(); + downloadedFiles = new HashSet<>(); + uploadedFiles = new HashSet<>(); } public void clearTest() { @@ -89,7 +89,7 @@ public Iterator list(String bucket, Date start, Date till) { clusterName = paths[3]; } - List tmpList = new ArrayList(); + List tmpList = new ArrayList<>(); for (AbstractBackupPath path : flist) { if ((path.time.after(start) && path.time.before(till)) diff --git a/priam/src/test/java/com/netflix/priam/backup/TestBackupFile.java b/priam/src/test/java/com/netflix/priam/backup/TestBackupFile.java index 6c66e259c..631c23ef6 100644 --- a/priam/src/test/java/com/netflix/priam/backup/TestBackupFile.java +++ b/priam/src/test/java/com/netflix/priam/backup/TestBackupFile.java @@ -21,7 +21,6 @@ import com.google.inject.Injector; import com.netflix.priam.aws.S3BackupPath; import com.netflix.priam.backup.AbstractBackupPath.BackupFileType; -import com.netflix.priam.config.FakeConfiguration; import com.netflix.priam.identity.InstanceIdentity; import java.io.BufferedOutputStream; import java.io.File; @@ -37,6 +36,7 @@ public class TestBackupFile { private static Injector injector; + private static String region; @BeforeClass public static void setup() throws IOException { @@ -58,6 +58,7 @@ public static void setup() throws IOException { } InstanceIdentity factory = injector.getInstance(InstanceIdentity.class); factory.getInstance().setToken("1234567"); // Token + region = factory.getInstanceInfo().getRegion(); } @AfterClass @@ -78,11 +79,11 @@ public void testBackupFileCreation() throws ParseException { Assert.assertEquals("Standard1", backupfile.columnFamily); Assert.assertEquals("1234567", backupfile.token); Assert.assertEquals("fake-app", backupfile.clusterName); - Assert.assertEquals(FakeConfiguration.FAKE_REGION, backupfile.region); + Assert.assertEquals(region, backupfile.region); Assert.assertEquals("casstestbackup", backupfile.baseDir); Assert.assertEquals( "casstestbackup/" - + FakeConfiguration.FAKE_REGION + + region + "/fake-app/1234567/201108082320/SNAP/Keyspace1/Standard1/Keyspace1-Standard1-ia-5-Data.db", backupfile.getRemotePath()); } @@ -98,12 +99,12 @@ public void testIncBackupFileCreation() throws ParseException { Assert.assertEquals("Standard1", backupfile.columnFamily); Assert.assertEquals("1234567", backupfile.token); Assert.assertEquals("fake-app", backupfile.clusterName); - Assert.assertEquals(FakeConfiguration.FAKE_REGION, backupfile.region); + Assert.assertEquals(region, backupfile.region); Assert.assertEquals("casstestbackup", backupfile.baseDir); String datestr = AbstractBackupPath.formatDate(new Date(bfile.lastModified())); Assert.assertEquals( "casstestbackup/" - + FakeConfiguration.FAKE_REGION + + region + "/fake-app/1234567/" + datestr + "/SST/Keyspace1/Standard1/Keyspace1-Standard1-ia-5-Data.db", @@ -121,12 +122,10 @@ public void testMetaFileCreation() throws ParseException { Assert.assertEquals(BackupFileType.META, backupfile.type); Assert.assertEquals("1234567", backupfile.token); Assert.assertEquals("fake-app", backupfile.clusterName); - Assert.assertEquals(FakeConfiguration.FAKE_REGION, backupfile.region); + Assert.assertEquals(region, backupfile.region); Assert.assertEquals("casstestbackup", backupfile.baseDir); Assert.assertEquals( - "casstestbackup/" - + FakeConfiguration.FAKE_REGION - + "/fake-app/1234567/201108082320/META/1234567.meta", + "casstestbackup/" + region + "/fake-app/1234567/201108082320/META/1234567.meta", backupfile.getRemotePath()); } } diff --git a/priam/src/test/java/com/netflix/priam/backup/TestFileIterator.java b/priam/src/test/java/com/netflix/priam/backup/TestFileIterator.java index 701174824..406b68294 100644 --- a/priam/src/test/java/com/netflix/priam/backup/TestFileIterator.java +++ b/priam/src/test/java/com/netflix/priam/backup/TestFileIterator.java @@ -25,7 +25,6 @@ import com.google.inject.Guice; import com.google.inject.Injector; import com.netflix.priam.aws.S3FileIterator; -import com.netflix.priam.config.FakeConfiguration; import com.netflix.priam.config.IConfiguration; import com.netflix.priam.identity.InstanceIdentity; import java.io.IOException; @@ -51,6 +50,7 @@ public class TestFileIterator { private static IConfiguration conf; private static InstanceIdentity factory; + private static String region; @BeforeClass public static void setup() throws InterruptedException, IOException { @@ -60,6 +60,7 @@ public static void setup() throws InterruptedException, IOException { injector = Guice.createInjector(new BRTestModule()); conf = injector.getInstance(IConfiguration.class); factory = injector.getInstance(InstanceIdentity.class); + region = factory.getInstanceInfo().getRegion(); cal = Calendar.getInstance(); cal.set(2011, 7, 11, 0, 30, 0); @@ -133,7 +134,7 @@ public void testIteratorEmptySet() { MockAmazonS3Client.prefix = conf.getBackupLocation() + "/" - + conf.getDC() + + region + "/" + conf.getAppName() + "/" @@ -161,7 +162,7 @@ public void testIterator() { MockAmazonS3Client.prefix = conf.getBackupLocation() + "/" - + conf.getDC() + + region + "/" + conf.getAppName() + "/" @@ -181,22 +182,22 @@ public void testIterator() { Assert.assertTrue( files.contains( "test_backup/" - + FakeConfiguration.FAKE_REGION + + region + "/fakecluster/123456/201108110030/SNAP/ks1/cf1/f1.db")); Assert.assertTrue( files.contains( "test_backup/" - + FakeConfiguration.FAKE_REGION + + region + "/fakecluster/123456/201108110430/SST/ks1/cf1/f2.db")); Assert.assertTrue( files.contains( "test_backup/" - + FakeConfiguration.FAKE_REGION + + region + "/fakecluster/123456/201108110030/META/meta.json")); Assert.assertFalse( files.contains( "test_backup/" - + FakeConfiguration.FAKE_REGION + + region + "/fakecluster/123456/201108110600/SST/ks1/cf1/f3.db")); } @@ -209,7 +210,7 @@ public void testIteratorTruncated() { MockAmazonS3Client.prefix = conf.getBackupLocation() + "/" - + conf.getDC() + + region + "/" + conf.getAppName() + "/" @@ -229,38 +230,38 @@ public void testIteratorTruncated() { Assert.assertTrue( files.contains( "test_backup/" - + FakeConfiguration.FAKE_REGION + + region + "/fakecluster/123456/201108110030/SNAP/ks1/cf1/f1.db")); Assert.assertTrue( files.contains( "test_backup/" - + FakeConfiguration.FAKE_REGION + + region + "/fakecluster/123456/201108110430/SST/ks1/cf1/f2.db")); Assert.assertTrue( files.contains( "test_backup/" - + FakeConfiguration.FAKE_REGION + + region + "/fakecluster/123456/201108110030/META/meta.json")); Assert.assertFalse( files.contains( "test_backup/" - + FakeConfiguration.FAKE_REGION + + region + "/fakecluster/123456/201108110600/SST/ks1/cf1/f3.db")); Assert.assertTrue( files.contains( "test_backup/" - + FakeConfiguration.FAKE_REGION + + region + "/fakecluster/123456/201108110030/SNAP/ks2/cf1/f1.db")); Assert.assertTrue( files.contains( "test_backup/" - + FakeConfiguration.FAKE_REGION + + region + "/fakecluster/123456/201108110430/SST/ks2/cf1/f2.db")); Assert.assertFalse( files.contains( "test_backup/" - + FakeConfiguration.FAKE_REGION + + region + "/fakecluster/123456/201108110600/SST/ks2/cf1/f3.db")); } @@ -273,7 +274,7 @@ public void testIteratorTruncatedOOR() { MockAmazonS3Client.prefix = conf.getBackupLocation() + "/" - + conf.getDC() + + region + "/" + conf.getAppName() + "/" @@ -293,38 +294,38 @@ public void testIteratorTruncatedOOR() { Assert.assertFalse( files.contains( "test_backup/" - + FakeConfiguration.FAKE_REGION + + region + "/fakecluster/123456/201107110030/SNAP/ks1/cf1/f1.db")); Assert.assertFalse( files.contains( "test_backup/" - + FakeConfiguration.FAKE_REGION + + region + "/fakecluster/123456/201107110430/SST/ks1/cf1/f2.db")); Assert.assertFalse( files.contains( "test_backup/" - + FakeConfiguration.FAKE_REGION + + region + "/fakecluster/123456/201107110030/META/meta.json")); Assert.assertFalse( files.contains( "test_backup/" - + FakeConfiguration.FAKE_REGION + + region + "/fakecluster/123456/201107110600/SST/ks1/cf1/f3.db")); Assert.assertTrue( files.contains( "test_backup/" - + FakeConfiguration.FAKE_REGION + + region + "/fakecluster/123456/201108110030/SNAP/ks2/cf1/f1.db")); Assert.assertTrue( files.contains( "test_backup/" - + FakeConfiguration.FAKE_REGION + + region + "/fakecluster/123456/201108110430/SST/ks2/cf1/f2.db")); Assert.assertFalse( files.contains( "test_backup/" - + FakeConfiguration.FAKE_REGION + + region + "/fakecluster/123456/201108110600/SST/ks2/cf1/f3.db")); } @@ -355,38 +356,38 @@ public void testRestorePathIteration() { Assert.assertTrue( files.contains( "test_backup/" - + FakeConfiguration.FAKE_REGION + + region + "/fakecluster/123456/201108110030/SNAP/ks1/cf1/f1.db")); Assert.assertTrue( files.contains( "test_backup/" - + FakeConfiguration.FAKE_REGION + + region + "/fakecluster/123456/201108110430/SST/ks1/cf1/f2.db")); Assert.assertTrue( files.contains( "test_backup/" - + FakeConfiguration.FAKE_REGION + + region + "/fakecluster/123456/201108110030/META/meta.json")); Assert.assertFalse( files.contains( "test_backup/" - + FakeConfiguration.FAKE_REGION + + region + "/fakecluster/123456/201108110600/SST/ks1/cf1/f3.db")); Assert.assertTrue( files.contains( "test_backup/" - + FakeConfiguration.FAKE_REGION + + region + "/fakecluster/123456/201108110030/SNAP/ks2/cf1/f1.db")); Assert.assertTrue( files.contains( "test_backup/" - + FakeConfiguration.FAKE_REGION + + region + "/fakecluster/123456/201108110430/SST/ks2/cf1/f2.db")); Assert.assertFalse( files.contains( "test_backup/" - + FakeConfiguration.FAKE_REGION + + region + "/fakecluster/123456/201108110600/SST/ks2/cf1/f3.db")); } @@ -394,27 +395,18 @@ public static List getObjectSummary() { List list = new ArrayList(); S3ObjectSummary summary = new S3ObjectSummary(); summary.setKey( - "test_backup/" - + FakeConfiguration.FAKE_REGION - + "/fakecluster/123456/201108110030/SNAP/ks1/cf1/f1.db"); + "test_backup/" + region + "/fakecluster/123456/201108110030/SNAP/ks1/cf1/f1.db"); list.add(summary); summary = new S3ObjectSummary(); summary.setKey( - "test_backup/" - + FakeConfiguration.FAKE_REGION - + "/fakecluster/123456/201108110430/SST/ks1/cf1/f2.db"); + "test_backup/" + region + "/fakecluster/123456/201108110430/SST/ks1/cf1/f2.db"); list.add(summary); summary = new S3ObjectSummary(); summary.setKey( - "test_backup/" - + FakeConfiguration.FAKE_REGION - + "/fakecluster/123456/201108110600/SST/ks1/cf1/f3.db"); + "test_backup/" + region + "/fakecluster/123456/201108110600/SST/ks1/cf1/f3.db"); list.add(summary); summary = new S3ObjectSummary(); - summary.setKey( - "test_backup/" - + FakeConfiguration.FAKE_REGION - + "/fakecluster/123456/201108110030/META/meta.json"); + summary.setKey("test_backup/" + region + "/fakecluster/123456/201108110030/META/meta.json"); list.add(summary); return list; } @@ -427,21 +419,15 @@ public static List getNextObjectSummary() { List list = new ArrayList(); S3ObjectSummary summary = new S3ObjectSummary(); summary.setKey( - "test_backup/" - + FakeConfiguration.FAKE_REGION - + "/fakecluster/123456/201108110030/SNAP/ks2/cf1/f1.db"); + "test_backup/" + region + "/fakecluster/123456/201108110030/SNAP/ks2/cf1/f1.db"); list.add(summary); summary = new S3ObjectSummary(); summary.setKey( - "test_backup/" - + FakeConfiguration.FAKE_REGION - + "/fakecluster/123456/201108110430/SST/ks2/cf1/f2.db"); + "test_backup/" + region + "/fakecluster/123456/201108110430/SST/ks2/cf1/f2.db"); list.add(summary); summary = new S3ObjectSummary(); summary.setKey( - "test_backup/" - + FakeConfiguration.FAKE_REGION - + "/fakecluster/123456/201108110600/SST/ks2/cf1/f3.db"); + "test_backup/" + region + "/fakecluster/123456/201108110600/SST/ks2/cf1/f3.db"); list.add(summary); return list; } diff --git a/priam/src/test/java/com/netflix/priam/backup/TestRestore.java b/priam/src/test/java/com/netflix/priam/backup/TestRestore.java index 83f56984f..446913154 100644 --- a/priam/src/test/java/com/netflix/priam/backup/TestRestore.java +++ b/priam/src/test/java/com/netflix/priam/backup/TestRestore.java @@ -23,14 +23,13 @@ import com.google.inject.name.Names; import com.netflix.priam.config.FakeConfiguration; import com.netflix.priam.config.IConfiguration; +import com.netflix.priam.identity.config.InstanceInfo; import com.netflix.priam.restore.Restore; import java.io.File; import java.io.IOException; import java.util.ArrayList; import java.util.Calendar; import java.util.Date; -import org.apache.commons.io.FileUtils; -import org.junit.AfterClass; import org.junit.Assert; import org.junit.BeforeClass; import org.junit.Test; @@ -40,7 +39,9 @@ public class TestRestore { private static FakeBackupFileSystem filesystem; private static ArrayList fileList; private static Calendar cal; - private static IConfiguration conf; + private static FakeConfiguration conf; + private static String region; + private static Restore restore; @BeforeClass public static void setup() throws InterruptedException, IOException { @@ -49,55 +50,26 @@ public static void setup() throws InterruptedException, IOException { (FakeBackupFileSystem) injector.getInstance( Key.get(IBackupFileSystem.class, Names.named("backup"))); - conf = injector.getInstance(IConfiguration.class); - fileList = new ArrayList(); - File cassdir = new File(conf.getDataFileLocation()); - cassdir.mkdirs(); + conf = (FakeConfiguration) injector.getInstance(IConfiguration.class); + region = injector.getInstance(InstanceInfo.class).getRegion(); + restore = injector.getInstance(Restore.class); + fileList = new ArrayList<>(); cal = Calendar.getInstance(); } - @AfterClass - public static void cleanup() throws IOException { - File file = new File("cass"); - FileUtils.deleteQuietly(file); - } - private static void populateBackupFileSystem(String baseDir) { fileList.clear(); + fileList.add(baseDir + "/" + region + "/fakecluster/123456/201108110030/META/meta.json"); fileList.add( - baseDir - + "/" - + FakeConfiguration.FAKE_REGION - + "/fakecluster/123456/201108110030/META/meta.json"); - fileList.add( - baseDir - + "/" - + FakeConfiguration.FAKE_REGION - + "/fakecluster/123456/201108110030/SNAP/ks1/cf1/f1.db"); - fileList.add( - baseDir - + "/" - + FakeConfiguration.FAKE_REGION - + "/fakecluster/123456/201108110030/SNAP/ks1/cf1/f2.db"); - fileList.add( - baseDir - + "/" - + FakeConfiguration.FAKE_REGION - + "/fakecluster/123456/201108110030/SNAP/ks2/cf1/f2.db"); + baseDir + "/" + region + "/fakecluster/123456/201108110030/SNAP/ks1/cf1/f1.db"); fileList.add( - baseDir - + "/" - + FakeConfiguration.FAKE_REGION - + "/fakecluster/123456/201108110530/SST/ks2/cf1/f3.db"); + baseDir + "/" + region + "/fakecluster/123456/201108110030/SNAP/ks1/cf1/f2.db"); fileList.add( - baseDir - + "/" - + FakeConfiguration.FAKE_REGION - + "/fakecluster/123456/201108110600/SST/ks2/cf1/f4.db"); - filesystem.baseDir = baseDir; - filesystem.region = FakeConfiguration.FAKE_REGION; - filesystem.clusterName = "fakecluster"; + baseDir + "/" + region + "/fakecluster/123456/201108110030/SNAP/ks2/cf1/f2.db"); + fileList.add(baseDir + "/" + region + "/fakecluster/123456/201108110530/SST/ks2/cf1/f3.db"); + fileList.add(baseDir + "/" + region + "/fakecluster/123456/201108110600/SST/ks2/cf1/f4.db"); filesystem.setupTest(fileList); + conf.setRestorePrefix("RESTOREBUCKET/" + baseDir + "/" + region + "/fakecluster"); } @Test @@ -106,7 +78,6 @@ public void testRestore() throws Exception { File tmpdir = new File(conf.getDataFileLocation() + "/test"); tmpdir.mkdir(); Assert.assertTrue(tmpdir.exists()); - Restore restore = injector.getInstance(Restore.class); cal.set(2011, Calendar.AUGUST, 11, 0, 30, 0); cal.set(Calendar.MILLISECOND, 0); Date startTime = cal.getTime(); @@ -127,11 +98,8 @@ public void testRestore() throws Exception { public void testRestoreLatest() throws Exception { populateBackupFileSystem("test_backup"); String metafile = - "test_backup/" - + FakeConfiguration.FAKE_REGION - + "/fakecluster/123456/201108110130/META/meta.json"; + "test_backup/" + region + "/fakecluster/123456/201108110130/META/meta.json"; filesystem.addFile(metafile); - Restore restore = injector.getInstance(Restore.class); cal.set(2011, Calendar.AUGUST, 11, 0, 30, 0); cal.set(Calendar.MILLISECOND, 0); Date startTime = cal.getTime(); @@ -149,8 +117,7 @@ public void testRestoreLatest() throws Exception { @Test public void testNoSnapshots() throws Exception { try { - filesystem.setupTest(new ArrayList()); - Restore restore = injector.getInstance(Restore.class); + filesystem.setupTest(fileList); cal.set(2011, Calendar.SEPTEMBER, 11, 0, 30); Date startTime = cal.getTime(); cal.add(Calendar.HOUR, 5); @@ -158,18 +125,13 @@ public void testNoSnapshots() throws Exception { Assert.assertFalse(true); // No exception thrown } catch (IllegalStateException e) { // We are ok. No snapshot found. - } catch (Exception e) { - throw e; + Assert.assertTrue(true); } } @Test public void testRestoreFromDiffCluster() throws Exception { populateBackupFileSystem("test_backup_new"); - FakeConfiguration conf = (FakeConfiguration) injector.getInstance(IConfiguration.class); - conf.setRestorePrefix( - "RESTOREBUCKET/test_backup_new/" + FakeConfiguration.FAKE_REGION + "/fakecluster"); - Restore restore = injector.getInstance(Restore.class); cal.set(2011, Calendar.AUGUST, 11, 0, 30, 0); cal.set(Calendar.MILLISECOND, 0); Date startTime = cal.getTime(); @@ -181,6 +143,5 @@ public void testRestoreFromDiffCluster() throws Exception { Assert.assertTrue(filesystem.downloadedFiles.contains(fileList.get(3))); Assert.assertFalse(filesystem.downloadedFiles.contains(fileList.get(4))); Assert.assertFalse(filesystem.downloadedFiles.contains(fileList.get(5))); - conf.setRestorePrefix(""); } } diff --git a/priam/src/test/java/com/netflix/priam/backup/TestS3FileSystem.java b/priam/src/test/java/com/netflix/priam/backup/TestS3FileSystem.java index 5bae592da..476d167e3 100644 --- a/priam/src/test/java/com/netflix/priam/backup/TestS3FileSystem.java +++ b/priam/src/test/java/com/netflix/priam/backup/TestS3FileSystem.java @@ -31,7 +31,7 @@ import com.netflix.priam.aws.S3FileSystem; import com.netflix.priam.aws.S3PartUploader; import com.netflix.priam.backup.AbstractBackupPath.BackupFileType; -import com.netflix.priam.config.FakeConfiguration; +import com.netflix.priam.identity.config.InstanceInfo; import com.netflix.priam.merics.BackupMetrics; import java.io.BufferedOutputStream; import java.io.File; @@ -54,11 +54,14 @@ public class TestS3FileSystem { private static String FILE_PATH = "target/data/Keyspace1/Standard1/backups/201108082320/Keyspace1-Standard1-ia-1-Data.db"; private static BackupMetrics backupMetrics; + private static String region; public TestS3FileSystem() { if (injector == null) injector = Guice.createInjector(new BRTestModule()); if (backupMetrics == null) backupMetrics = injector.getInstance(BackupMetrics.class); + InstanceInfo instanceInfo = injector.getInstance(InstanceInfo.class); + region = instanceInfo.getRegion(); } @BeforeClass @@ -120,8 +123,6 @@ public void testFileUploadFailures() throws Exception { } catch (BackupRestoreException e) { // ignore } - // Assert.assertEquals(RetryableCallable.DEFAULT_NUMBER_OF_RETRIES, - // MockS3PartUploader.partAttempts); Assert.assertEquals(0, MockS3PartUploader.compattempts); Assert.assertEquals(1, backupMetrics.getInvalidUploads().count() - noOfFailures); } @@ -145,9 +146,6 @@ public void testFileUploadCompleteFailure() throws Exception { } catch (BackupRestoreException e) { // ignore } - // Assert.assertEquals(1, MockS3PartUploader.partAttempts); - // No retries with the new logic - // Assert.assertEquals(1, MockS3PartUploader.compattempts); } @Test @@ -158,8 +156,7 @@ public void testCleanupAdd() throws Exception { Assert.assertEquals(1, MockAmazonS3Client.bconf.getRules().size()); BucketLifecycleConfiguration.Rule rule = MockAmazonS3Client.bconf.getRules().get(0); logger.info(rule.getPrefix()); - Assert.assertEquals( - "casstestbackup/" + FakeConfiguration.FAKE_REGION + "/fake-app/", rule.getPrefix()); + Assert.assertEquals("casstestbackup/" + region + "/fake-app/", rule.getPrefix()); Assert.assertEquals(5, rule.getExpirationInDays()); } @@ -171,8 +168,7 @@ public void testCleanupIgnore() throws Exception { Assert.assertEquals(1, MockAmazonS3Client.bconf.getRules().size()); BucketLifecycleConfiguration.Rule rule = MockAmazonS3Client.bconf.getRules().get(0); logger.info(rule.getPrefix()); - Assert.assertEquals( - "casstestbackup/" + FakeConfiguration.FAKE_REGION + "/fake-app/", rule.getPrefix()); + Assert.assertEquals("casstestbackup/" + region + "/fake-app/", rule.getPrefix()); Assert.assertEquals(5, rule.getExpirationInDays()); } @@ -247,8 +243,7 @@ public PutObjectResult putObject(PutObjectRequest putObjectRequest) public BucketLifecycleConfiguration getBucketLifecycleConfiguration(String bucketName) { List rules = Lists.newArrayList(); if (ruleAvailable) { - String clusterPath = - "casstestbackup/" + FakeConfiguration.FAKE_REGION + "/fake-app/"; + String clusterPath = "casstestbackup/" + region + "/fake-app/"; BucketLifecycleConfiguration.Rule rule = new BucketLifecycleConfiguration.Rule() .withExpirationInDays(5) diff --git a/priam/src/test/java/com/netflix/priam/backup/identity/DoubleRingTest.java b/priam/src/test/java/com/netflix/priam/backup/identity/DoubleRingTest.java index 5cbd60c84..f340a03dc 100644 --- a/priam/src/test/java/com/netflix/priam/backup/identity/DoubleRingTest.java +++ b/priam/src/test/java/com/netflix/priam/backup/identity/DoubleRingTest.java @@ -32,7 +32,7 @@ public class DoubleRingTest extends InstanceTestUtils { public void testDouble() throws Exception { createInstances(); int originalSize = factory.getAllIds(config.getAppName()).size(); - new DoubleRing(config, factory, tokenManager).doubleSlots(); + new DoubleRing(config, factory, tokenManager, identity).doubleSlots(); List doubled = factory.getAllIds(config.getAppName()); factory.sort(doubled); @@ -43,13 +43,13 @@ public void testDouble() throws Exception { private void validate(List doubled) { List validator = Lists.newArrayList(); for (int i = 0; i < doubled.size(); i++) { - validator.add(tokenManager.createToken(i, doubled.size(), config.getDC())); + validator.add(tokenManager.createToken(i, doubled.size(), instanceInfo.getRegion())); } for (int i = 0; i < doubled.size(); i++) { PriamInstance ins = doubled.get(i); assertEquals(validator.get(i), ins.getToken()); - int id = ins.getId() - tokenManager.regionOffset(config.getDC()); + int id = ins.getId() - tokenManager.regionOffset(instanceInfo.getRegion()); System.out.println(ins); if (0 != id % 2) assertEquals(ins.getInstanceId(), InstanceIdentity.DUMMY_INSTANCE_ID); } @@ -59,7 +59,7 @@ private void validate(List doubled) { public void testBR() throws Exception { createInstances(); int intialSize = factory.getAllIds(config.getAppName()).size(); - DoubleRing ring = new DoubleRing(config, factory, tokenManager); + DoubleRing ring = new DoubleRing(config, factory, tokenManager, identity); ring.backup(); ring.doubleSlots(); assertEquals(intialSize * 2, factory.getAllIds(config.getAppName()).size()); diff --git a/priam/src/test/java/com/netflix/priam/backup/identity/FakeInstanceEnvIdentity.java b/priam/src/test/java/com/netflix/priam/backup/identity/FakeInstanceEnvIdentity.java deleted file mode 100644 index 1d23b4e56..000000000 --- a/priam/src/test/java/com/netflix/priam/backup/identity/FakeInstanceEnvIdentity.java +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Copyright 2017 Netflix, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.netflix.priam.backup.identity; - -import com.netflix.priam.identity.InstanceEnvIdentity; - -public class FakeInstanceEnvIdentity implements InstanceEnvIdentity { - - @Override - public Boolean isClassic() { - return null; - } - - @Override - public Boolean isDefaultVpc() { - return null; - } - - @Override - public Boolean isNonDefaultVpc() { - return null; - } -} diff --git a/priam/src/test/java/com/netflix/priam/backup/identity/InstanceIdentityTest.java b/priam/src/test/java/com/netflix/priam/backup/identity/InstanceIdentityTest.java index 2759e5679..4a72ce972 100644 --- a/priam/src/test/java/com/netflix/priam/backup/identity/InstanceIdentityTest.java +++ b/priam/src/test/java/com/netflix/priam/backup/identity/InstanceIdentityTest.java @@ -33,7 +33,7 @@ public class InstanceIdentityTest extends InstanceTestUtils { public void testCreateToken() throws Exception { identity = createInstanceIdentity("az1", "fakeinstance1"); - int hash = tokenManager.regionOffset(config.getDC()); + int hash = tokenManager.regionOffset(instanceInfo.getRegion()); assertEquals(0, identity.getInstance().getId() - hash); identity = createInstanceIdentity("az1", "fakeinstance2"); @@ -42,7 +42,7 @@ public void testCreateToken() throws Exception { identity = createInstanceIdentity("az1", "fakeinstance3"); assertEquals(6, identity.getInstance().getId() - hash); - // try next region + // try next zone identity = createInstanceIdentity("az2", "fakeinstance4"); assertEquals(1, identity.getInstance().getId() - hash); @@ -94,8 +94,8 @@ public void testGetSeedsAutobootstrapFalse() throws Exception { @Test public void testDoubleSlots() throws Exception { createInstances(); - int before = factory.getAllIds("fake-app").size(); - new DoubleRing(config, factory, tokenManager).doubleSlots(); + int before = factory.getAllIds(config.getAppName()).size(); + new DoubleRing(config, factory, tokenManager, identity).doubleSlots(); List lst = factory.getAllIds(config.getAppName()); // sort it so it will look good if you want to print it. factory.sort(lst); @@ -110,10 +110,8 @@ public void testDoubleSlots() throws Exception { @Test public void testDoubleGrap() throws Exception { createInstances(); - new DoubleRing(config, factory, tokenManager).doubleSlots(); - config.zone = "az1"; - config.instance_id = "fakeinstancex"; - int hash = tokenManager.regionOffset(config.getDC()); + new DoubleRing(config, factory, tokenManager, identity).doubleSlots(); + int hash = tokenManager.regionOffset(instanceInfo.getRegion()); identity = createInstanceIdentity("az1", "fakeinstancex"); printInstance(identity.getInstance(), hash); } diff --git a/priam/src/test/java/com/netflix/priam/backup/identity/InstanceTestUtils.java b/priam/src/test/java/com/netflix/priam/backup/identity/InstanceTestUtils.java index 587948f26..f291d8525 100644 --- a/priam/src/test/java/com/netflix/priam/backup/identity/InstanceTestUtils.java +++ b/priam/src/test/java/com/netflix/priam/backup/identity/InstanceTestUtils.java @@ -18,15 +18,10 @@ package com.netflix.priam.backup.identity; import com.netflix.priam.config.FakeConfiguration; -import com.netflix.priam.identity.FakeMembership; -import com.netflix.priam.identity.FakePriamInstanceFactory; -import com.netflix.priam.identity.IMembership; -import com.netflix.priam.identity.IPriamInstanceFactory; -import com.netflix.priam.identity.InstanceEnvIdentity; -import com.netflix.priam.identity.InstanceIdentity; -import com.netflix.priam.identity.token.DeadTokenRetriever; -import com.netflix.priam.identity.token.NewTokenRetriever; -import com.netflix.priam.identity.token.PreGeneratedTokenRetriever; +import com.netflix.priam.identity.*; +import com.netflix.priam.identity.config.FakeInstanceInfo; +import com.netflix.priam.identity.config.InstanceInfo; +import com.netflix.priam.identity.token.*; import com.netflix.priam.utils.FakeSleeper; import com.netflix.priam.utils.ITokenManager; import com.netflix.priam.utils.Sleeper; @@ -44,15 +39,13 @@ public abstract class InstanceTestUtils { FakeConfiguration config; IPriamInstanceFactory factory; InstanceIdentity identity; - Sleeper sleeper; - DeadTokenRetriever deadTokenRetriever; - PreGeneratedTokenRetriever preGeneratedTokenRetriever; - NewTokenRetriever newTokenRetriever; + private Sleeper sleeper; ITokenManager tokenManager; - InstanceEnvIdentity insEnvIdentity; + InstanceInfo instanceInfo; + private final String region = "us-east-1"; @Before - public void setup() { + public void setup() throws Exception { instances.add("fakeinstance1"); instances.add("fakeinstance2"); instances.add("fakeinstance3"); @@ -64,44 +57,47 @@ public void setup() { instances.add("fakeinstance9"); membership = new FakeMembership(instances); - config = new FakeConfiguration("fake", "fake-app", "az1", "fakeinstance1"); + config = new FakeConfiguration("fake-app"); + instanceInfo = new FakeInstanceInfo("fakeinstance1", "az1", region); tokenManager = new TokenManager(config); - factory = new FakePriamInstanceFactory(config); + factory = new FakePriamInstanceFactory(config, instanceInfo); sleeper = new FakeSleeper(); - this.deadTokenRetriever = - new DeadTokenRetriever(factory, membership, config, sleeper, insEnvIdentity); - this.preGeneratedTokenRetriever = - new PreGeneratedTokenRetriever(factory, membership, config, sleeper); - this.newTokenRetriever = - new NewTokenRetriever(factory, membership, config, sleeper, tokenManager); + identity = createInstanceIdentity(instanceInfo.getRac(), instanceInfo.getInstanceId()); } public void createInstances() throws Exception { createInstanceIdentity("az1", "fakeinstance1"); createInstanceIdentity("az1", "fakeinstance2"); createInstanceIdentity("az1", "fakeinstance3"); - // try next region + // try next zone createInstanceIdentity("az2", "fakeinstance4"); createInstanceIdentity("az2", "fakeinstance5"); createInstanceIdentity("az2", "fakeinstance6"); - // next region + // next zone createInstanceIdentity("az3", "fakeinstance7"); createInstanceIdentity("az3", "fakeinstance8"); createInstanceIdentity("az3", "fakeinstance9"); } - protected InstanceIdentity createInstanceIdentity(String zone, String instanceId) - throws Exception { - config.zone = zone; - config.instance_id = instanceId; + InstanceIdentity createInstanceIdentity(String zone, String instanceId) throws Exception { + InstanceInfo newInstanceInfo = new FakeInstanceInfo(instanceId, zone, region); + IDeadTokenRetriever deadTokenRetriever = + new DeadTokenRetriever(factory, membership, config, sleeper, newInstanceInfo); + IPreGeneratedTokenRetriever preGeneratedTokenRetriever = + new PreGeneratedTokenRetriever( + factory, membership, config, sleeper, newInstanceInfo); + INewTokenRetriever newTokenRetriever = + new NewTokenRetriever( + factory, membership, config, sleeper, tokenManager, newInstanceInfo); return new InstanceIdentity( factory, membership, config, sleeper, - new TokenManager(config), - this.deadTokenRetriever, - this.preGeneratedTokenRetriever, - this.newTokenRetriever); + this.tokenManager, + deadTokenRetriever, + preGeneratedTokenRetriever, + newTokenRetriever, + newInstanceInfo); } } diff --git a/priam/src/test/java/com/netflix/priam/config/FakeConfiguration.java b/priam/src/test/java/com/netflix/priam/config/FakeConfiguration.java index 33560b2d6..ae42b5290 100644 --- a/priam/src/test/java/com/netflix/priam/config/FakeConfiguration.java +++ b/priam/src/test/java/com/netflix/priam/config/FakeConfiguration.java @@ -19,9 +19,6 @@ import com.google.common.collect.Lists; import com.google.inject.Singleton; -import com.netflix.priam.identity.config.InstanceDataRetriever; -import com.netflix.priam.identity.config.LocalInstanceDataRetriever; -import com.netflix.priam.scheduler.SchedulerType; import com.netflix.priam.scheduler.UnsupportedTypeException; import com.netflix.priam.tuner.GCType; import com.netflix.priam.tuner.JVMOption; @@ -34,26 +31,18 @@ @Singleton public class FakeConfiguration implements IConfiguration { - public static final String FAKE_REGION = "us-east-1"; - - public String region; - public String appName; - public String zone; - public String instance_id; - public String restorePrefix; + private String appName; + private String restorePrefix = ""; public Map fakeConfig; + public Map fakeProperties = new HashMap<>(); public FakeConfiguration() { - this(FAKE_REGION, "my_fake_cluster", "my_zone", "i-01234567"); + this("my_fake_cluster"); } - public FakeConfiguration(String region, String appName, String zone, String ins_id) { - this.region = region; + public FakeConfiguration(String appName) { this.appName = appName; - this.zone = zone; - this.instance_id = ins_id; - this.restorePrefix = ""; fakeConfig = new HashMap<>(); fakeConfig.put("auto_bootstrap", false); } @@ -66,6 +55,11 @@ public void setFakeConfig(String key, Object value) { fakeConfig.put(key, value); } + @Override + public boolean getAutoBoostrap() { + return (Boolean) fakeConfig.getOrDefault("auto_bootstrap", false); + } + @Override public void initialize() { // TODO Auto-generated method stub @@ -117,37 +111,11 @@ public List getRacs() { return Arrays.asList("az1", "az2", "az3"); } - @Override - public int getThriftPort() { - return 9160; - } - - @Override - public int getNativeTransportPort() { - return 9042; - } - @Override public String getSnitch() { return "org.apache.cassandra.locator.SimpleSnitch"; } - @Override - public String getRac() { - return this.zone; - } - - @Override - public String getHostname() { - // TODO Auto-generated method stub - return instance_id; - } - - @Override - public String getInstanceName() { - return instance_id; - } - @Override public String getHeapSize() { // TODO Auto-generated method stub @@ -160,22 +128,6 @@ public String getHeapNewSize() { return null; } - @Override - public int getBackupHour() { - // TODO Auto-generated method stub - return 12; - } - - @Override - public String getBackupCronExpression() { - return null; - } - - @Override - public SchedulerType getBackupSchedulerType() { - return SchedulerType.HOUR; - } - @Override public String getRestoreSnapshot() { // TODO Auto-generated method stub @@ -198,94 +150,37 @@ public String getSDBInstanceIdentityRegion() { return null; } - @Override - public String getDC() { - return this.region; - } - @Override public int getRestoreThreads() { return 2; } - public void setRestorePrefix(String prefix) { - // TODO Auto-generated method stub - restorePrefix = prefix; - } - @Override public String getRestorePrefix() { - // TODO Auto-generated method stub - return restorePrefix; - } - - @Override - public String getBackupCommitLogLocation() { - return "cass/backup/cl/"; + return this.restorePrefix; } - @Override - public boolean isMultiDC() { - // TODO Auto-generated method stub - return false; + // For testing purposes only. + public void setRestorePrefix(String restorePrefix) { + this.restorePrefix = restorePrefix; } @Override - public String getASGName() { - // TODO Auto-generated method stub - return null; + public String getBackupCommitLogLocation() { + return "cass/backup/cl/"; } - /** - * Amazon specific setting to query Additional/ Sibling ASG Memberships in csv format to - * consider while calculating RAC membership - */ @Override public String getSiblingASGNames() { return null; } - @Override - public boolean isIncrBackup() { - return true; - } - - @Override - public String getHostIP() { - // TODO Auto-generated method stub - return null; - } - - @Override - public int getUploadThrottle() { - // TODO Auto-generated method stub - return 0; - } - - @Override - public InstanceDataRetriever getInstanceDataRetriever() - throws InstantiationException, IllegalAccessException, ClassNotFoundException { - return new LocalInstanceDataRetriever(); - } - @Override public boolean isLocalBootstrapEnabled() { // TODO Auto-generated method stub return false; } - @Override - public int getCompactionThroughput() { - // TODO Auto-generated method stub - return 0; - } - - @Override - public String getMaxDirectMemory() { - // TODO Auto-generated method stub - return null; - } - @Override public String getBootClusterName() { return "cass_bootstrap"; @@ -306,38 +201,16 @@ public long getBackupChunkSize() { return 5L * 1024 * 1024; } - @Override - public void setDC(String region) { - // TODO Auto-generated method stub - - } - - @Override - public boolean isRestoreClosestToken() { - // TODO Auto-generated method stub - return false; - } - @Override public String getCassStopScript() { return "true"; } - @Override - public int getGracefulDrainHealthWaitSeconds() { - return -1; - } - @Override public int getRemediateDeadCassandraRate() { return 1; } - @Override - public int getStoragePort() { - return 7101; - } - @Override public String getSeedProviderName() { return "org.apache.cassandra.locator.SimpleSeedProvider"; @@ -353,39 +226,10 @@ public List getBackupRacs() { return Lists.newArrayList(); } - public int getMaxHintWindowInMS() { - return 36000; - } - - public int getHintedHandoffThrottleKb() { - return 1024; - } - - public int getMaxHintThreads() { - return 1; - } - - /** @return memtable_cleanup_threshold in C* yaml */ - @Override - public double getMemtableCleanupThreshold() { - return 0.11; - } - - @Override - public int getStreamingThroughputMB() { - return 400; - } - public String getPartitioner() { return "org.apache.cassandra.dht.RandomPartitioner"; } - @Override - public int getSSLStoragePort() { - // TODO Auto-generated method stub - return 7103; - } - public String getKeyCacheSizeInMB() { return "16"; } @@ -407,10 +251,6 @@ public String getCassProcessName() { return "CassandraDaemon"; } - public int getNumTokens() { - return 1; - } - public String getYamlLocation() { return "conf/cassandra.yaml"; } @@ -435,26 +275,13 @@ public Map getJVMUpsertSet() { return null; } - public String getAuthenticator() { - return PriamConfiguration.DEFAULT_AUTHENTICATOR; - } - - public String getAuthorizer() { - return PriamConfiguration.DEFAULT_AUTHORIZER; - } - public boolean doesCassandraStartManually() { return false; } - @Override - public boolean isVpcRing() { - return false; - } - @Override public String getCommitLogBackupPropsFile() { - return getCassHome() + PriamConfiguration.DEFAULT_COMMITLOG_PROPS_FILE; + return getCassHome() + "/conf/commitlog_archiving.properties"; } @Override @@ -477,8 +304,6 @@ public String getCommitLogBackupRestorePointInTime() { return null; } - public void setRestoreKeySpaces(List keyspaces) {} - @Override public int maxCommitLogsRestore() { return 0; @@ -529,11 +354,6 @@ public String getCassYamlVal(String priamKey) { return ""; } - @Override - public boolean getAutoBoostrap() { - return (Boolean) fakeConfig.getOrDefault("auto_bootstrap", false); - } - @Override public boolean isCreateNewTokenEnable() { return true; // allow Junit test to create new tokens @@ -599,11 +419,6 @@ public Map getExtraEnvParams() { return null; } - @Override - public String getVpcId() { - return ""; - } - @Override public int getBackupQueueSize() { return 100; @@ -614,31 +429,6 @@ public String getFlushKeyspaces() { return ""; } - @Override - public String getFlushInterval() { - return null; - } - - @Override - public String getBackupStatusFileLoc() { - return "backupstatus.ser"; - } - - @Override - public String getBackupNotificationTopicArn() { - return null; - } - - @Override - public SchedulerType getFlushSchedulerType() throws UnsupportedTypeException { - return SchedulerType.HOUR; - } - - @Override - public String getFlushCronExpression() { - return null; - } - @Override public boolean isPostRestoreHookEnabled() { return true; @@ -659,10 +449,6 @@ public String getPostRestoreHookDoneFileName() { return System.getProperty("java.io.tmpdir") + File.separator + "postrestorehook.done"; } - public int getPostRestoreHookTimeOutInDays() { - return 2; - } - @Override public String getProperty(String key, String defaultValue) { return fakeProperties.getOrDefault(key, defaultValue); diff --git a/priam/src/test/java/com/netflix/priam/configSource/CompositeConfigSourceTest.java b/priam/src/test/java/com/netflix/priam/configSource/CompositeConfigSourceTest.java index 31b6304f3..40759df61 100644 --- a/priam/src/test/java/com/netflix/priam/configSource/CompositeConfigSourceTest.java +++ b/priam/src/test/java/com/netflix/priam/configSource/CompositeConfigSourceTest.java @@ -30,7 +30,7 @@ public final class CompositeConfigSourceTest { public void read() { MemoryConfigSource memoryConfigSource = new MemoryConfigSource(); IConfigSource configSource = new CompositeConfigSource(memoryConfigSource); - configSource.intialize("foo", "bar"); + configSource.initialize("foo", "bar"); Assert.assertEquals(0, configSource.size()); configSource.set("foo", "bar"); diff --git a/priam/src/test/java/com/netflix/priam/configSource/PropertiesConfigSourceTest.java b/priam/src/test/java/com/netflix/priam/configSource/PropertiesConfigSourceTest.java index 5e1b33125..194fc841e 100644 --- a/priam/src/test/java/com/netflix/priam/configSource/PropertiesConfigSourceTest.java +++ b/priam/src/test/java/com/netflix/priam/configSource/PropertiesConfigSourceTest.java @@ -29,7 +29,7 @@ public final class PropertiesConfigSourceTest { @Test public void readFile() { PropertiesConfigSource configSource = new PropertiesConfigSource("conf/Priam.properties"); - configSource.intialize("asgName", "region"); + configSource.initialize("asgName", "region"); Assert.assertEquals( "\"/tmp/commitlog\"", configSource.get("Priam.backup.commitlog.location")); @@ -42,7 +42,7 @@ public void readFile() { @Test public void updateKey() { PropertiesConfigSource configSource = new PropertiesConfigSource("conf/Priam.properties"); - configSource.intialize("asgName", "region"); + configSource.initialize("asgName", "region"); // File has 13 lines, but line 6 is "Priam.jmx.port7501", so it gets filtered out with empty // string check. diff --git a/priam/src/test/java/com/netflix/priam/configSource/SystemPropertiesConfigSourceTest.java b/priam/src/test/java/com/netflix/priam/configSource/SystemPropertiesConfigSourceTest.java index 8af563c4e..7def39eef 100644 --- a/priam/src/test/java/com/netflix/priam/configSource/SystemPropertiesConfigSourceTest.java +++ b/priam/src/test/java/com/netflix/priam/configSource/SystemPropertiesConfigSourceTest.java @@ -30,7 +30,7 @@ public final class SystemPropertiesConfigSourceTest { public void read() { final String key = "java.version"; SystemPropertiesConfigSource configSource = new SystemPropertiesConfigSource(); - configSource.intialize("asgName", "region"); + configSource.initialize("asgName", "region"); // sys props are filtered to starting with priam, so this should be missing. Assert.assertEquals(null, configSource.get(key)); diff --git a/priam/src/test/java/com/netflix/priam/defaultimpl/CassandraProcessManagerTest.java b/priam/src/test/java/com/netflix/priam/defaultimpl/CassandraProcessManagerTest.java index 47d10d6f9..dc7116923 100644 --- a/priam/src/test/java/com/netflix/priam/defaultimpl/CassandraProcessManagerTest.java +++ b/priam/src/test/java/com/netflix/priam/defaultimpl/CassandraProcessManagerTest.java @@ -33,8 +33,7 @@ public class CassandraProcessManagerTest { @Before public void setup() { - IConfiguration config = - new FakeConfiguration("us-east-1", "test_cluster", "us-east-1a", "i-2378afd3"); + IConfiguration config = new FakeConfiguration("test_cluster"); InstanceState instanceState = Guice.createInjector(new BRTestModule()).getInstance(InstanceState.class); CassMonitorMetrics cassMonitorMetrics = diff --git a/priam/src/test/java/com/netflix/priam/identity/FakePriamInstanceFactory.java b/priam/src/test/java/com/netflix/priam/identity/FakePriamInstanceFactory.java index e791b6a4a..442d1ac51 100644 --- a/priam/src/test/java/com/netflix/priam/identity/FakePriamInstanceFactory.java +++ b/priam/src/test/java/com/netflix/priam/identity/FakePriamInstanceFactory.java @@ -20,15 +20,18 @@ import com.google.common.collect.Maps; import com.google.inject.Inject; import com.netflix.priam.config.IConfiguration; +import com.netflix.priam.identity.config.InstanceInfo; import java.util.*; public class FakePriamInstanceFactory implements IPriamInstanceFactory { private final Map instances = Maps.newHashMap(); private final IConfiguration config; + private final InstanceInfo instanceInfo; @Inject - public FakePriamInstanceFactory(IConfiguration config) { + public FakePriamInstanceFactory(IConfiguration config, InstanceInfo instanceInfo) { this.config = config; + this.instanceInfo = instanceInfo; } @Override @@ -61,7 +64,7 @@ public PriamInstance create( ins.setInstanceId(instanceID); ins.setToken(payload); ins.setVolumes(volumes); - ins.setDC(config.getDC()); + ins.setDC(instanceInfo.getRegion()); instances.put(id, ins); return ins; } diff --git a/priam/src/test/java/com/netflix/priam/identity/config/FakeInstanceInfo.java b/priam/src/test/java/com/netflix/priam/identity/config/FakeInstanceInfo.java new file mode 100644 index 000000000..e9f064821 --- /dev/null +++ b/priam/src/test/java/com/netflix/priam/identity/config/FakeInstanceInfo.java @@ -0,0 +1,97 @@ +/* + * Copyright 2018 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.netflix.priam.identity.config; + +/** Created by aagrawal on 10/17/18. */ +public class FakeInstanceInfo implements InstanceInfo { + private String instanceId; + private String availabilityZone; + private String region; + private String instanceType; + private String asg; + private String vpcId; + + public FakeInstanceInfo(String instanceId, String availabilityZone, String region) { + this(instanceId, availabilityZone, region, "i2.xlarge", availabilityZone, ""); + } + + public FakeInstanceInfo( + String instanceId, + String availabilityZone, + String region, + String instanceType, + String asg, + String vpcId) { + this.instanceId = instanceId; + this.availabilityZone = availabilityZone; + this.region = region; + this.instanceType = instanceType; + this.asg = asg; + this.vpcId = vpcId; + } + + @Override + public String getRac() { + return availabilityZone; + } + + @Override + public String getHostname() { + return instanceId; + } + + @Override + public String getHostIP() { + return instanceId; + } + + @Override + public String getPrivateIP() { + return instanceId; + } + + @Override + public String getInstanceId() { + return instanceId; + } + + @Override + public String getInstanceType() { + return instanceType; + } + + @Override + public String getVpcId() { + return vpcId; + } + + @Override + public String getRegion() { + return region; + } + + @Override + public String getAutoScalingGroup() { + return asg; + } + + @Override + public InstanceEnvironment getInstanceEnvironment() { + return InstanceEnvironment.VPC; + } +} diff --git a/priam/src/test/java/com/netflix/priam/resources/BackupServletTest.java b/priam/src/test/java/com/netflix/priam/resources/BackupServletTest.java index 54be67f9b..20c9078e4 100644 --- a/priam/src/test/java/com/netflix/priam/resources/BackupServletTest.java +++ b/priam/src/test/java/com/netflix/priam/resources/BackupServletTest.java @@ -19,8 +19,6 @@ import static org.junit.Assert.assertEquals; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.Lists; import com.google.inject.Guice; import com.google.inject.Injector; import com.google.inject.Provider; @@ -30,14 +28,11 @@ import com.netflix.priam.defaultimpl.ICassandraProcess; import com.netflix.priam.health.InstanceState; import com.netflix.priam.identity.IPriamInstanceFactory; -import com.netflix.priam.identity.InstanceIdentity; -import com.netflix.priam.identity.PriamInstance; +import com.netflix.priam.identity.config.InstanceInfo; import com.netflix.priam.restore.Restore; import com.netflix.priam.tuner.ICassandraTuner; -import com.netflix.priam.utils.ITokenManager; -import com.netflix.priam.utils.TokenManager; +import com.netflix.priam.utils.DateUtil; import java.util.Date; -import java.util.List; import javax.ws.rs.core.MediaType; import javax.ws.rs.core.Response; import mockit.Expectations; @@ -63,38 +58,17 @@ public class BackupServletTest { private BackupServlet resource; private RestoreServlet restoreResource; private BackupVerification backupVerification; + private InstanceInfo instanceInfo; @Before public void setUp() { Injector injector = Guice.createInjector(new BRTestModule()); config = injector.getInstance(IConfiguration.class); InstanceState instanceState = injector.getInstance(InstanceState.class); - ITokenManager tokenManager = new TokenManager(config); + instanceInfo = injector.getInstance(InstanceInfo.class); resource = - new BackupServlet( - priamServer, - config, - bkpFs, - restoreObj, - pathProvider, - tuner, - snapshotBackup, - factory, - tokenManager, - cassProcess, - bkupStatusMgr, - backupVerification); - restoreResource = - new RestoreServlet( - config, - restoreObj, - pathProvider, - priamServer, - factory, - tuner, - cassProcess, - tokenManager, - instanceState); + new BackupServlet(config, bkpFs, snapshotBackup, bkupStatusMgr, backupVerification); + restoreResource = new RestoreServlet(restoreObj, instanceState); } @Test @@ -113,51 +87,21 @@ public void backup() throws Exception { } @Test - public void restore_minimal( - @Mocked final InstanceIdentity identity, @Mocked final PriamInstance instance) - throws Exception { + public void restore_minimal() throws Exception { final String dateRange = null; - final String newRegion = null; - final String newToken = null; - final String keyspaces = null; - final String oldRegion = "us-east-1"; - final String oldToken = "1234"; - - new Expectations() { - { - priamServer.getId(); - result = identity; - times = 2; - } - }; - new Expectations() { - { - config.getDC(); + instanceInfo.getRegion(); result = oldRegion; - identity.getInstance(); - result = instance; - times = 2; - instance.getToken(); - result = oldToken; - - config.isRestoreClosestToken(); - result = false; restoreObj.restore((Date) any, (Date) any); // TODO: test default value - - config.setDC(oldRegion); - instance.setToken(oldToken); - tuner.updateAutoBootstrap(config.getYamlLocation(), false); } }; expectCassandraStartup(); - Response response = - restoreResource.restore(dateRange, newRegion, newToken, keyspaces, null); + Response response = restoreResource.restore(dateRange); assertEquals(200, response.getStatus()); assertEquals("[\"ok\"]", response.getEntity()); assertEquals( @@ -165,302 +109,27 @@ public void restore_minimal( } @Test - public void restore_withDateRange( - @Mocked final InstanceIdentity identity, - @Mocked final PriamInstance instance, - @Mocked final AbstractBackupPath backupPath) - throws Exception { - final String dateRange = "201101010000,20111231259"; - final String newRegion = null; - final String newToken = null; - final String keyspaces = null; - - final String oldRegion = "us-east-1"; - final String oldToken = "1234"; + public void restore_withDateRange() throws Exception { + final String dateRange = "201101010000,201112312359"; - new Expectations() { - { - priamServer.getId(); - result = identity; - times = 2; - } - }; new Expectations() { { - pathProvider.get(); - result = backupPath; - backupPath.parseDate(dateRange.split(",")[0]); + DateUtil.getDate(dateRange.split(",")[0]); result = new DateTime(2011, 01, 01, 00, 00).toDate(); times = 1; - backupPath.parseDate(dateRange.split(",")[1]); + DateUtil.getDate(dateRange.split(",")[1]); result = new DateTime(2011, 12, 31, 23, 59).toDate(); times = 1; - - // config.getDC(); result = oldRegion; - identity.getInstance(); - result = instance; - times = 2; - instance.getToken(); - result = oldToken; - - // config.isRestoreClosestToken(); result = false; - restoreObj.restore( - new DateTime(2011, 01, 01, 00, 00).toDate(), - new DateTime(2011, 12, 31, 23, 59).toDate()); - - // config.setDC(oldRegion); - instance.setToken(oldToken); - tuner.updateAutoBootstrap(config.getYamlLocation(), false); - } - }; - - expectCassandraStartup(); - - Response response = - restoreResource.restore(dateRange, newRegion, newToken, keyspaces, null); - assertEquals(200, response.getStatus()); - assertEquals("[\"ok\"]", response.getEntity()); - assertEquals( - MediaType.APPLICATION_JSON_TYPE, response.getMetadata().get("Content-Type").get(0)); - } - - // @Test - // public void restore_withRegion() throws Exception - // { - // final String dateRange = null; - // final String newRegion = "us-west-1"; - // final String newToken = null; - // final String keyspaces = null; - // - // final String oldRegion = "us-east-1"; - // final String oldToken = "1234"; - // final String appName = "myApp"; - // - // new Expectations() { - // @NonStrict InstanceIdentity identity; - // PriamInstance instance; - // @NonStrict PriamInstance instance1, instance2, instance3; - // - // { - // config.getDC(); result = oldRegion; - // priamServer.getId(); result = identity; times = 3; - // identity.getInstance(); result = instance; times = 3; - // instance.getToken(); result = oldToken; - // - // config.isRestoreClosestToken(); result = false; - // - // config.setDC(newRegion); - // instance.getToken(); result = oldToken; - // config.getAppName(); result = appName; - // factory.getAllIds(appName); result = ImmutableList.of(instance, instance1, - // instance2, instance3); - // instance.getDC(); result = oldRegion; - // instance.getToken(); result = oldToken; - // instance1.getDC(); result = oldRegion; - // instance2.getDC(); result = oldRegion; - // instance3.getDC(); result = oldRegion; - // instance1.getToken(); result = "1234"; - // instance2.getToken(); result = "5678"; - // instance3.getToken(); result = "9000"; - // instance.setToken((String) any); // TODO: test mocked closest token - // - // restoreObj.restore((Date) any, (Date) any); // TODO: test default value - // - // config.setDC(oldRegion); - // instance.setToken(oldToken); - // tuneCassandra.writeAllProperties(false); - // } - // }; - // - // expectCassandraStartup(); - // - // Response response = resource.restore(dateRange, newRegion, newToken, keyspaces); - // assertEquals(200, response.getStatus()); - // assertEquals("[\"ok\"]", response.getEntity()); - // assertEquals(MediaType.APPLICATION_JSON_TYPE, - // response.getMetadata().get("Content-Type").get(0)); - // } - - @Test - public void restore_withToken( - @Mocked final InstanceIdentity identity, @Mocked final PriamInstance instance) - throws Exception { - final String dateRange = null; - final String newRegion = null; - final String newToken = "myNewToken"; - final String keyspaces = null; - - final String oldRegion = "us-east-1"; - final String oldToken = "1234"; - - new Expectations() { - { - priamServer.getId(); - result = identity; - times = 3; - } - }; - new Expectations() { - - { - config.getDC(); - result = oldRegion; - identity.getInstance(); - result = instance; - times = 3; - instance.getToken(); - result = oldToken; - instance.setToken(newToken); - - // config.isRestoreClosestToken(); result = false; - - restoreObj.restore((Date) any, (Date) any); // TODO: test default value - - config.setDC(oldRegion); - instance.setToken(oldToken); - tuner.updateAutoBootstrap(config.getYamlLocation(), false); - } - }; - - expectCassandraStartup(); - - Response response = - restoreResource.restore(dateRange, newRegion, newToken, keyspaces, null); - assertEquals(200, response.getStatus()); - assertEquals("[\"ok\"]", response.getEntity()); - assertEquals( - MediaType.APPLICATION_JSON_TYPE, response.getMetadata().get("Content-Type").get(0)); - } - - @Test - public void restore_withKeyspaces( - @Mocked final InstanceIdentity identity, @Mocked final PriamInstance instance) - throws Exception { - final String dateRange = null; - final String newRegion = null; - final String newToken = null; - final String keyspaces = "keyspace1,keyspace2"; - - final String oldRegion = "us-east-1"; - final String oldToken = "1234"; - - new Expectations() { - { - config.getDC(); - result = oldRegion; - config.isRestoreClosestToken(); - result = false; - - List restoreKeyspaces = Lists.newArrayList(); - restoreKeyspaces.clear(); - restoreKeyspaces.addAll(ImmutableList.of("keyspace1", "keyspace2")); - - result = restoreKeyspaces; - config.setDC(oldRegion); - priamServer.getId(); - result = identity; - times = 2; - } - }; - new Expectations() { - - { - identity.getInstance(); - result = instance; - times = 2; - instance.getToken(); - result = oldToken; - - restoreObj.restore((Date) any, (Date) any); // TODO: test default value - - instance.setToken(oldToken); - tuner.updateAutoBootstrap(config.getYamlLocation(), false); - } - }; - - expectCassandraStartup(); - - Response response = - restoreResource.restore(dateRange, newRegion, newToken, keyspaces, null); - assertEquals(200, response.getStatus()); - assertEquals("[\"ok\"]", response.getEntity()); - assertEquals( - MediaType.APPLICATION_JSON_TYPE, response.getMetadata().get("Content-Type").get(0)); - } - - // TODO: this should also set/test newRegion and keyspaces - @Test - public void restore_maximal( - @Mocked final InstanceIdentity identity, - @Mocked final PriamInstance instance, - @Mocked final PriamInstance instance1, - @Mocked final PriamInstance instance2, - @Mocked final PriamInstance instance3, - @Mocked final AbstractBackupPath backupPath) - throws Exception { - final String dateRange = "201101010000,20111231259"; - final String newRegion = null; - final String newToken = "5678"; - final String keyspaces = null; - - final String oldRegion = "us-east-1"; - final String oldToken = "1234"; - final String appName = "myApp"; - - instance.setDC(oldRegion); - instance1.setDC(oldRegion); - instance2.setDC(oldRegion); - instance3.setDC(oldRegion); - instance.setToken(oldToken); - instance1.setToken("1234"); - instance2.setToken("5678"); - instance3.setToken("9000"); - - new Expectations() { - - { - pathProvider.get(); - result = backupPath; - backupPath.parseDate(dateRange.split(",")[0]); - result = new DateTime(2011, 01, 01, 00, 00).toDate(); - times = 1; - backupPath.parseDate(dateRange.split(",")[1]); - result = new DateTime(2011, 12, 31, 23, 59).toDate(); - times = 1; - - // identity.getInstance(); result = instance; times = 5; - // instance.getToken(); result = oldToken; - // instance.setToken(newToken); - // - // instance.getToken(); result = oldToken; - // factory.getAllIds(appName); result = ImmutableList.of(instance, - // instance1, instance2, instance3); - // instance.getDC(); result = oldRegion; - // instance.getToken(); result = oldToken; - // instance1.getDC(); result = oldRegion; - // instance2.getDC(); result = oldRegion; - // instance3.getDC(); result = oldRegion; - // instance1.getToken(); result = "1234"; - // instance2.getToken(); result = "5678"; - // instance3.getToken(); result = "9000"; - // instance.setToken((String) any); // TODO: test mocked closest - // token - - restoreObj.restore( - new DateTime(2011, 01, 01, 00, 00).toDate(), - new DateTime(2011, 12, 31, 23, 59).toDate()); - - // instance.setToken(oldToken); - tuner.updateAutoBootstrap(config.getYamlLocation(), false); + DateUtil.getDate(dateRange.split(",")[0]), + DateUtil.getDate(dateRange.split(",")[1])); } }; expectCassandraStartup(); - Response response = - restoreResource.restore(dateRange, newRegion, newToken, keyspaces, null); + Response response = restoreResource.restore(dateRange); assertEquals(200, response.getStatus()); assertEquals("[\"ok\"]", response.getEntity()); assertEquals( diff --git a/priam/src/test/java/com/netflix/priam/resources/CassandraConfigTest.java b/priam/src/test/java/com/netflix/priam/resources/CassandraConfigTest.java index aa9b262ea..ae0b74939 100644 --- a/priam/src/test/java/com/netflix/priam/resources/CassandraConfigTest.java +++ b/priam/src/test/java/com/netflix/priam/resources/CassandraConfigTest.java @@ -52,7 +52,7 @@ public void getSeeds(@Mocked final InstanceIdentity identity) throws Exception { final List seeds = ImmutableList.of("seed1", "seed2", "seed3"); new Expectations() { { - priamServer.getId(); + priamServer.getInstanceIdentity(); result = identity; times = 1; identity.getSeeds(); @@ -71,7 +71,7 @@ public void getSeeds_notFound(@Mocked final InstanceIdentity identity) throws Ex final List seeds = ImmutableList.of(); new Expectations() { { - priamServer.getId(); + priamServer.getInstanceIdentity(); result = identity; times = 1; identity.getSeeds(); @@ -89,7 +89,7 @@ public void getSeeds_handlesUnknownHostException(@Mocked final InstanceIdentity throws Exception { new Expectations() { { - priamServer.getId(); + priamServer.getInstanceIdentity(); result = identity; identity.getSeeds(); result = new UnknownHostException(); @@ -106,7 +106,7 @@ public void getToken( final String token = "myToken"; new Expectations() { { - priamServer.getId(); + priamServer.getInstanceIdentity(); result = identity; times = 2; identity.getInstance(); @@ -129,7 +129,7 @@ public void getToken_notFound( final String token = ""; new Expectations() { { - priamServer.getId(); + priamServer.getInstanceIdentity(); result = identity; identity.getInstance(); result = instance; @@ -147,7 +147,7 @@ public void getToken_handlesException( @Mocked final InstanceIdentity identity, @Mocked final PriamInstance instance) { new Expectations() { { - priamServer.getId(); + priamServer.getInstanceIdentity(); result = identity; identity.getInstance(); result = instance; @@ -164,7 +164,7 @@ public void getToken_handlesException( public void isReplaceToken(@Mocked final InstanceIdentity identity) { new Expectations() { { - priamServer.getId(); + priamServer.getInstanceIdentity(); result = identity; identity.isReplace(); result = true; @@ -180,7 +180,7 @@ public void isReplaceToken(@Mocked final InstanceIdentity identity) { public void isReplaceToken_handlesException(@Mocked final InstanceIdentity identity) { new Expectations() { { - priamServer.getId(); + priamServer.getInstanceIdentity(); result = identity; identity.isReplace(); result = new RuntimeException(); @@ -196,7 +196,7 @@ public void getReplacedAddress(@Mocked final InstanceIdentity identity) { final String replacedIp = "127.0.0.1"; new Expectations() { { - priamServer.getId(); + priamServer.getInstanceIdentity(); result = identity; identity.getReplacedIp(); result = replacedIp; diff --git a/priam/src/test/java/com/netflix/priam/resources/PriamConfigTest.java b/priam/src/test/java/com/netflix/priam/resources/PriamConfigTest.java index e10cf9f54..048cd1334 100644 --- a/priam/src/test/java/com/netflix/priam/resources/PriamConfigTest.java +++ b/priam/src/test/java/com/netflix/priam/resources/PriamConfigTest.java @@ -25,7 +25,7 @@ public class PriamConfigTest { @Before public void setUp() { resource = new PriamConfig(priamServer); - fakeConfiguration = new FakeConfiguration("test", "cass_test", "test", "12345"); + fakeConfiguration = new FakeConfiguration("cass_test"); fakeConfiguration.fakeProperties.put("test.prop", "test_value"); } diff --git a/priam/src/test/java/com/netflix/priam/resources/PriamInstanceResourceTest.java b/priam/src/test/java/com/netflix/priam/resources/PriamInstanceResourceTest.java index fc227da61..cbd2bf65f 100644 --- a/priam/src/test/java/com/netflix/priam/resources/PriamInstanceResourceTest.java +++ b/priam/src/test/java/com/netflix/priam/resources/PriamInstanceResourceTest.java @@ -24,6 +24,7 @@ import com.netflix.priam.config.IConfiguration; import com.netflix.priam.identity.IPriamInstanceFactory; import com.netflix.priam.identity.PriamInstance; +import com.netflix.priam.identity.config.InstanceInfo; import java.util.List; import javax.ws.rs.WebApplicationException; import javax.ws.rs.core.Response; @@ -41,11 +42,12 @@ public class PriamInstanceResourceTest { private @Mocked IConfiguration config; private @Mocked IPriamInstanceFactory factory; + private @Mocked InstanceInfo instanceInfo; private PriamInstanceResource resource; @Before public void setUp() { - resource = new PriamInstanceResource(config, factory); + resource = new PriamInstanceResource(config, factory, instanceInfo); } @Test @@ -80,7 +82,7 @@ public void getInstance(@Mocked final PriamInstance instance) { { config.getAppName(); result = APP_NAME; - factory.getInstance(APP_NAME, config.getDC(), NODE_ID); + factory.getInstance(APP_NAME, instanceInfo.getRegion(), NODE_ID); result = instance; instance.toString(); result = expected; @@ -96,7 +98,7 @@ public void getInstance_notFound() { { config.getAppName(); result = APP_NAME; - factory.getInstance(APP_NAME, config.getDC(), NODE_ID); + factory.getInstance(APP_NAME, instanceInfo.getRegion(), NODE_ID); result = null; } }; @@ -141,7 +143,7 @@ public void deleteInstance(@Mocked final PriamInstance instance) { { config.getAppName(); result = APP_NAME; - factory.getInstance(APP_NAME, config.getDC(), NODE_ID); + factory.getInstance(APP_NAME, instanceInfo.getRegion(), NODE_ID); result = instance; factory.delete(instance); } @@ -157,7 +159,7 @@ public void deleteInstance_notFound() { { config.getAppName(); result = APP_NAME; - factory.getInstance(APP_NAME, config.getDC(), NODE_ID); + factory.getInstance(APP_NAME, instanceInfo.getRegion(), NODE_ID); result = null; } }; diff --git a/priam/src/test/java/com/netflix/priam/services/TestSnapshotMetaService.java b/priam/src/test/java/com/netflix/priam/services/TestSnapshotMetaService.java index d397549ad..0a30b5b68 100644 --- a/priam/src/test/java/com/netflix/priam/services/TestSnapshotMetaService.java +++ b/priam/src/test/java/com/netflix/priam/services/TestSnapshotMetaService.java @@ -23,6 +23,7 @@ import com.netflix.priam.backupv2.*; import com.netflix.priam.config.IBackupRestoreConfig; import com.netflix.priam.config.IConfiguration; +import com.netflix.priam.identity.config.InstanceInfo; import com.netflix.priam.scheduler.TaskTimer; import com.netflix.priam.utils.BackupFileUtils; import com.netflix.priam.utils.DateUtil; @@ -45,9 +46,9 @@ public class TestSnapshotMetaService { private static SnapshotMetaService snapshotMetaService; private static TestMetaFileReader metaFileReader; private static PrefixGenerator prefixGenerator; + private static InstanceInfo instanceInfo; - @Before - public void setUp() { + public TestSnapshotMetaService() { Injector injector = Guice.createInjector(new BRTestModule()); if (configuration == null) configuration = injector.getInstance(IConfiguration.class); @@ -62,6 +63,11 @@ public void setUp() { if (prefixGenerator == null) prefixGenerator = injector.getInstance(PrefixGenerator.class); + if (instanceInfo == null) instanceInfo = injector.getInstance(InstanceInfo.class); + } + + @Before + public void setUp() { dummyDataDirectoryLocation = Paths.get(configuration.getDataFileLocation()); BackupFileUtils.cleanupDir(dummyDataDirectoryLocation); } @@ -111,8 +117,8 @@ private void test(int noOfSstables, int noOfKeyspaces, int noOfCf) throws Except MetaFileInfo metaFileInfo = metaFileReader.getMetaFileInfo(); Assert.assertEquals(1, metaFileInfo.getVersion()); Assert.assertEquals(configuration.getAppName(), metaFileInfo.getAppName()); - Assert.assertEquals(configuration.getRac(), metaFileInfo.getRack()); - Assert.assertEquals(configuration.getDC(), metaFileInfo.getRegion()); + Assert.assertEquals(instanceInfo.getRac(), metaFileInfo.getRack()); + Assert.assertEquals(instanceInfo.getRegion(), metaFileInfo.getRegion()); // Cleanup metaFileLocation.toFile().delete(); diff --git a/priam/src/test/java/com/netflix/priam/stream/StreamingTest.java b/priam/src/test/java/com/netflix/priam/stream/StreamingTest.java index 5c57421ba..39625ad8a 100644 --- a/priam/src/test/java/com/netflix/priam/stream/StreamingTest.java +++ b/priam/src/test/java/com/netflix/priam/stream/StreamingTest.java @@ -22,18 +22,13 @@ import com.netflix.priam.aws.S3BackupPath; import com.netflix.priam.backup.AbstractBackupPath; import com.netflix.priam.backup.BRTestModule; -import com.netflix.priam.config.FakeConfiguration; import com.netflix.priam.config.IConfiguration; import com.netflix.priam.identity.InstanceIdentity; import com.netflix.priam.utils.FifoQueue; -import java.io.IOException; import org.junit.Assert; import org.junit.Test; public class StreamingTest { - public void teststream() throws IOException, InterruptedException { - IConfiguration config = new FakeConfiguration("test", "cass_upg107_ccs", "test", "ins_id"); - } @Test public void testFifoAddAndRemove() { @@ -48,13 +43,14 @@ public void testAbstractPath() { Injector injector = Guice.createInjector(new BRTestModule()); IConfiguration conf = injector.getInstance(IConfiguration.class); InstanceIdentity factory = injector.getInstance(InstanceIdentity.class); + String region = factory.getInstanceInfo().getRegion(); FifoQueue queue = new FifoQueue(10); for (int i = 10; i < 30; i++) { S3BackupPath path = new S3BackupPath(conf, factory); path.parseRemote( "test_backup/" - + FakeConfiguration.FAKE_REGION + + region + "/fakecluster/123456/201108" + i + "0000" @@ -68,7 +64,7 @@ public void testAbstractPath() { S3BackupPath path = new S3BackupPath(conf, factory); path.parseRemote( "test_backup/" - + FakeConfiguration.FAKE_REGION + + region + "/fakecluster/123456/201108" + i + "0000" @@ -82,7 +78,7 @@ public void testAbstractPath() { S3BackupPath path = new S3BackupPath(conf, factory); path.parseRemote( "test_backup/" - + FakeConfiguration.FAKE_REGION + + region + "/fakecluster/123456/201108" + i + "0000" @@ -95,26 +91,26 @@ public void testAbstractPath() { S3BackupPath path = new S3BackupPath(conf, factory); path.parseRemote( "test_backup/" - + FakeConfiguration.FAKE_REGION + + region + "/fakecluster/123456/201108290000" + "/SNAP/ks1/cf2/f129.db"); Assert.assertTrue(queue.contains(path)); path.parseRemote( "test_backup/" - + FakeConfiguration.FAKE_REGION + + region + "/fakecluster/123456/201108290000" + "/SNAP/ks1/cf2/f229.db"); Assert.assertTrue(queue.contains(path)); path.parseRemote( "test_backup/" - + FakeConfiguration.FAKE_REGION + + region + "/fakecluster/123456/201108290000" + "/SNAP/ks1/cf2/f329.db"); Assert.assertTrue(queue.contains(path)); path.parseRemote( "test_backup/" - + FakeConfiguration.FAKE_REGION + + region + "/fakecluster/123456/201108260000/SNAP/ks1/cf2/f326.db To: cass/data/ks1/cf2/f326.db"); Assert.assertEquals(path, queue.first()); } diff --git a/priam/src/test/java/com/netflix/priam/tuner/StandardTunerTest.java b/priam/src/test/java/com/netflix/priam/tuner/StandardTunerTest.java index 986b5d046..4ecb0a479 100644 --- a/priam/src/test/java/com/netflix/priam/tuner/StandardTunerTest.java +++ b/priam/src/test/java/com/netflix/priam/tuner/StandardTunerTest.java @@ -19,14 +19,13 @@ import static org.junit.Assert.assertEquals; import com.google.common.io.Files; -import com.netflix.priam.config.FakeConfiguration; -import com.netflix.priam.config.IConfiguration; +import com.google.inject.Guice; +import com.netflix.priam.backup.BRTestModule; import java.io.File; -import org.junit.Before; import org.junit.Test; public class StandardTunerTest { - /* note: these are, more or less, arbitrary paritioner class names. as long as the tests exercise the code, all is good */ + /* note: these are, more or less, arbitrary partitioner class names. as long as the tests exercise the code, all is good */ private static final String A_PARTITIONER = "com.netflix.priam.utils.NonexistentPartitioner"; private static final String RANDOM_PARTITIONER = "org.apache.cassandra.dht.RandomPartitioner"; private static final String MURMUR_PARTITIONER = "org.apache.cassandra.dht.Murmur3Partitioner"; @@ -34,11 +33,8 @@ public class StandardTunerTest { private StandardTuner tuner; - @Before - public void setup() { - - IConfiguration config = new FakeConfiguration(); - tuner = new StandardTuner(config); + public StandardTunerTest() { + this.tuner = Guice.createInjector(new BRTestModule()).getInstance(StandardTuner.class); } @Test diff --git a/priam/src/test/java/com/netflix/priam/tuner/dse/DseTunerTest.java b/priam/src/test/java/com/netflix/priam/tuner/dse/DseTunerTest.java index e7a695650..948a0d693 100644 --- a/priam/src/test/java/com/netflix/priam/tuner/dse/DseTunerTest.java +++ b/priam/src/test/java/com/netflix/priam/tuner/dse/DseTunerTest.java @@ -43,8 +43,6 @@ public void setup() throws IOException { dseConfig = new DseConfigStub(); auditLogTunerYaml = new AuditLogTunerYaml(dseConfig); auditLogTunerLog4j = new AuditLogTunerLog4J(config, dseConfig); - dseTunerYaml = new DseTuner(config, dseConfig, auditLogTunerYaml); - dseTunerLog4j = new DseTuner(config, dseConfig, auditLogTunerLog4j); File targetDir = new File(config.getCassHome() + "/conf"); if (!targetDir.exists()) targetDir.mkdirs(); From 2cd38fc53e3aa5d40cc7f1d70deb30d3c3800b44 Mon Sep 17 00:00:00 2001 From: Arun Agrawal Date: Wed, 24 Oct 2018 17:14:26 -0700 Subject: [PATCH 026/228] 3.11 (#747) * fix test cases to use lambda * fix test --- .../TestFlushTask.groovy | 2 +- .../TestSchedulerType.groovy | 2 +- .../backup/TestBackupScheduler.groovy | 2 +- .../cluser/management/TestCompaction.groovy | 7 +- .../netflix/priam/backup/BRTestModule.java | 5 +- .../priam/backup/FakeBackupFileSystem.java | 8 +- .../priam/backup/FakeNullCredential.java | 2 +- .../priam/backup/FakePostRestoreHook.java | 2 +- .../priam/backup/TestAbstractFileSystem.java | 6 +- .../com/netflix/priam/backup/TestBackup.java | 20 ++--- .../netflix/priam/backup/TestCompression.java | 80 +++++++++---------- .../priam/backup/TestCustomizedTPE.java | 32 ++++---- .../priam/backup/TestFileIterator.java | 22 ++--- .../com/netflix/priam/backup/TestRestore.java | 3 +- .../priam/backup/TestS3FileSystem.java | 11 +-- .../priam/backup/TestSnapshotStatusMgr.java | 2 + .../backup/identity/InstanceTestUtils.java | 6 +- .../token/FakeDeadTokenRetriever.java | 2 +- .../identity/token/FakeNewTokenRetriever.java | 2 +- .../token/FakePreGeneratedTokenRetriever.java | 2 +- .../backupv2/TestLocalDBReaderWriter.java | 55 ++++++------- .../cassandra/token/TestDoublingLogic.java | 6 +- .../priam/config/FakeConfiguration.java | 4 +- .../priam/health/TestInstanceStatus.java | 2 +- .../identity/FakePriamInstanceFactory.java | 22 ++--- .../priam/resources/BackupServletTest.java | 6 +- .../resources/PriamInstanceResourceTest.java | 2 +- .../priam/restore/TestPostRestoreHook.java | 71 ++++++++-------- .../priam/scheduler/TestGuiceSingleton.java | 8 +- .../services/TestSnapshotMetaService.java | 4 +- .../netflix/priam/stream/StreamingTest.java | 4 +- .../priam/tuner/StandardTunerTest.java | 2 +- 32 files changed, 186 insertions(+), 218 deletions(-) diff --git a/priam/src/test/groovy/com.netflix.priam.scheduler/TestFlushTask.groovy b/priam/src/test/groovy/com.netflix.priam.scheduler/TestFlushTask.groovy index ab8630706..e03a147a9 100644 --- a/priam/src/test/groovy/com.netflix.priam.scheduler/TestFlushTask.groovy +++ b/priam/src/test/groovy/com.netflix.priam.scheduler/TestFlushTask.groovy @@ -16,7 +16,7 @@ class TestFlushTask extends Specification { Flush.getTimer(new FlushConfiguration(flushSchedulerType, flushCronExpression, flushInterval)) then: - def error = thrown(expectedException) + thrown(expectedException) where: flushSchedulerType | flushCronExpression | flushInterval || expectedException diff --git a/priam/src/test/groovy/com.netflix.priam.scheduler/TestSchedulerType.groovy b/priam/src/test/groovy/com.netflix.priam.scheduler/TestSchedulerType.groovy index 1e48dbe82..2c8429045 100644 --- a/priam/src/test/groovy/com.netflix.priam.scheduler/TestSchedulerType.groovy +++ b/priam/src/test/groovy/com.netflix.priam.scheduler/TestSchedulerType.groovy @@ -14,7 +14,7 @@ class TestSchedulerType extends Specification{ SchedulerType.lookup(schedulerType, acceptNullorEmpty, acceptIllegalValue) then: - def error = thrown(expectedException) + thrown(expectedException) where: schedulerType | acceptNullorEmpty | acceptIllegalValue || expectedException diff --git a/priam/src/test/groovy/com.netflix.priam/backup/TestBackupScheduler.groovy b/priam/src/test/groovy/com.netflix.priam/backup/TestBackupScheduler.groovy index bcd49664d..45c8f89e0 100644 --- a/priam/src/test/groovy/com.netflix.priam/backup/TestBackupScheduler.groovy +++ b/priam/src/test/groovy/com.netflix.priam/backup/TestBackupScheduler.groovy @@ -30,7 +30,7 @@ class TestBackupScheduler extends Specification { SnapshotBackup.isBackupEnabled(new BackupConfiguration("cron", configCRON, 1)) then: - def error = thrown(expectedException) + thrown(expectedException) where: configCRON || expectedException diff --git a/priam/src/test/groovy/com/netflix/priam/cluser/management/TestCompaction.groovy b/priam/src/test/groovy/com/netflix/priam/cluser/management/TestCompaction.groovy index 273889ed0..cb4b1ca28 100644 --- a/priam/src/test/groovy/com/netflix/priam/cluser/management/TestCompaction.groovy +++ b/priam/src/test/groovy/com/netflix/priam/cluser/management/TestCompaction.groovy @@ -132,8 +132,9 @@ class TestCompaction extends Specification { 1 || 0 } - private int concurrentRuns(int size) { - CassandraMonitor.setIsCassadraStarted(); + + private static int concurrentRuns(int size) { + CassandraMonitor.setIsCassadraStarted() ExecutorService threads = Executors.newFixedThreadPool(size) List> torun = new ArrayList<>(size) for (int i = 0; i < size; i++) { @@ -156,7 +157,7 @@ class TestCompaction extends Specification { //We expect exception here. try{ fut.get() - }catch(Exception e){ + }catch(Exception ignored){ noOfBadRun++ } } diff --git a/priam/src/test/java/com/netflix/priam/backup/BRTestModule.java b/priam/src/test/java/com/netflix/priam/backup/BRTestModule.java index 934d29f5a..176b610d5 100644 --- a/priam/src/test/java/com/netflix/priam/backup/BRTestModule.java +++ b/priam/src/test/java/com/netflix/priam/backup/BRTestModule.java @@ -39,7 +39,7 @@ import com.netflix.priam.utils.Sleeper; import com.netflix.spectator.api.DefaultRegistry; import com.netflix.spectator.api.Registry; -import java.util.Arrays; +import java.util.Collections; import org.junit.Ignore; import org.quartz.SchedulerFactory; import org.quartz.impl.StdSchedulerFactory; @@ -56,7 +56,8 @@ protected void configure() { bind(IPriamInstanceFactory.class).to(FakePriamInstanceFactory.class); bind(SchedulerFactory.class).to(StdSchedulerFactory.class).in(Scopes.SINGLETON); - bind(IMembership.class).toInstance(new FakeMembership(Arrays.asList("fakeInstance1"))); + bind(IMembership.class) + .toInstance(new FakeMembership(Collections.singletonList("fakeInstance1"))); bind(ICredential.class).to(FakeNullCredential.class).in(Scopes.SINGLETON); bind(IBackupFileSystem.class) .annotatedWith(Names.named("backup")) diff --git a/priam/src/test/java/com/netflix/priam/backup/FakeBackupFileSystem.java b/priam/src/test/java/com/netflix/priam/backup/FakeBackupFileSystem.java index 294047fc6..1429d66eb 100644 --- a/priam/src/test/java/com/netflix/priam/backup/FakeBackupFileSystem.java +++ b/priam/src/test/java/com/netflix/priam/backup/FakeBackupFileSystem.java @@ -36,7 +36,9 @@ public class FakeBackupFileSystem extends AbstractFileSystem { private List flist; public Set downloadedFiles; public Set uploadedFiles; - public String baseDir, region, clusterName; + private String baseDir; + private String region; + private String clusterName; @Inject Provider pathProvider; @@ -131,7 +133,9 @@ protected void downloadFileImpl(Path remotePath, Path localPath) throws BackupRe try (FileWriter fr = new FileWriter(localPath.toFile())) { JSONArray jsonObj = new JSONArray(); for (AbstractBackupPath filePath : flist) { - if (filePath.type == BackupFileType.SNAP) jsonObj.add(filePath.getRemotePath()); + if (filePath.type == BackupFileType.SNAP) { + jsonObj.add(filePath.getRemotePath()); + } } fr.write(jsonObj.toJSONString()); fr.flush(); diff --git a/priam/src/test/java/com/netflix/priam/backup/FakeNullCredential.java b/priam/src/test/java/com/netflix/priam/backup/FakeNullCredential.java index 42baeb807..5e99fbd46 100644 --- a/priam/src/test/java/com/netflix/priam/backup/FakeNullCredential.java +++ b/priam/src/test/java/com/netflix/priam/backup/FakeNullCredential.java @@ -20,7 +20,7 @@ import com.amazonaws.auth.AWSCredentialsProvider; import com.netflix.priam.cred.ICredential; -public class FakeNullCredential implements ICredential { +class FakeNullCredential implements ICredential { public AWSCredentialsProvider getAwsCredentialProvider() { // TODO Auto-generated method stub return null; diff --git a/priam/src/test/java/com/netflix/priam/backup/FakePostRestoreHook.java b/priam/src/test/java/com/netflix/priam/backup/FakePostRestoreHook.java index 94907a051..df2350392 100644 --- a/priam/src/test/java/com/netflix/priam/backup/FakePostRestoreHook.java +++ b/priam/src/test/java/com/netflix/priam/backup/FakePostRestoreHook.java @@ -18,7 +18,7 @@ import com.netflix.priam.restore.IPostRestoreHook; -public class FakePostRestoreHook implements IPostRestoreHook { +class FakePostRestoreHook implements IPostRestoreHook { public boolean hasValidParameters() { return true; } diff --git a/priam/src/test/java/com/netflix/priam/backup/TestAbstractFileSystem.java b/priam/src/test/java/com/netflix/priam/backup/TestAbstractFileSystem.java index 41d34485d..b970643b2 100644 --- a/priam/src/test/java/com/netflix/priam/backup/TestAbstractFileSystem.java +++ b/priam/src/test/java/com/netflix/priam/backup/TestAbstractFileSystem.java @@ -37,8 +37,6 @@ import org.junit.Assert; import org.junit.Before; import org.junit.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; /** * The goal of this class is to test common functionality which are encapsulated in @@ -46,8 +44,6 @@ * scope of this class. Created by aagrawal on 9/22/18. */ public class TestAbstractFileSystem { - private static final Logger logger = - LoggerFactory.getLogger(TestAbstractFileSystem.class.getName()); private Injector injector; private IConfiguration configuration; private BackupMetrics backupMetrics; @@ -310,7 +306,7 @@ protected long uploadFileImpl(Path localPath, Path remotePath) class MyFileSystem extends NullBackupFileSystem { - private Random random = new Random(); + private final Random random = new Random(); @Inject public MyFileSystem( diff --git a/priam/src/test/java/com/netflix/priam/backup/TestBackup.java b/priam/src/test/java/com/netflix/priam/backup/TestBackup.java index 746c7421f..1cde032b7 100644 --- a/priam/src/test/java/com/netflix/priam/backup/TestBackup.java +++ b/priam/src/test/java/com/netflix/priam/backup/TestBackup.java @@ -44,7 +44,7 @@ public class TestBackup { private static Injector injector; private static FakeBackupFileSystem filesystem; private static final Logger logger = LoggerFactory.getLogger(TestBackup.class); - private static Set expectedFiles = new HashSet(); + private static final Set expectedFiles = new HashSet<>(); @BeforeClass public static void setup() throws InterruptedException, IOException { @@ -120,7 +120,7 @@ private void testClusterSpecificColumnFamiliesSkipped(String[] columnFamilyDirs) if (tmp.exists()) cleanup(tmp); // Generate "data" generateIncrementalFiles(); - Set systemfiles = new HashSet(); + Set systemfiles = new HashSet<>(); // Generate system files for (String columnFamilyDir : columnFamilyDirs) { String columnFamily = columnFamilyDir.split("-")[0]; @@ -151,7 +151,7 @@ private static void generateIncrementalFiles() { File tmp = new File("target/data/"); if (tmp.exists()) cleanup(tmp); // Setup - Set files = new HashSet(); + Set files = new HashSet<>(); files.add("target/data/Keyspace1/Standard1/backups/Keyspace1-Standard1-ia-1-Data.db"); files.add("target/data/Keyspace1/Standard1/backups/Keyspace1-Standard1-ia-1-Index.db"); files.add("target/data/Keyspace1/Standard1/backups/Keyspace1-Standard1-ia-2-Data.db"); @@ -185,16 +185,13 @@ private static void cleanup(File dir) { // Mock Nodeprobe class @Ignore static class MockNodeProbe extends MockUp { - @Mock - public void $init(String host, int port) throws IOException, InterruptedException {} @Mock - public void takeSnapshot(String snapshotName, String columnFamily, String... keyspaces) - throws IOException { + public void takeSnapshot(String snapshotName, String columnFamily, String... keyspaces) { File tmp = new File("target/data/"); if (tmp.exists()) cleanup(tmp); // Setup - Set files = new HashSet(); + Set files = new HashSet<>(); files.add( "target/data/Keyspace1/Standard1/snapshots/" + snapshotName @@ -210,16 +207,13 @@ public void takeSnapshot(String snapshotName, String columnFamily, String... key for (String filePath : files) { File file = new File(filePath); genTestFile(file); - if (filePath.indexOf("Keyspace1-Standard1-ia-6-Data.db") == -1) // skip + if (!filePath.contains("Keyspace1-Standard1-ia-6-Data.db")) // skip expectedFiles.add(file.getAbsolutePath()); } } @Mock - public void close() throws IOException {} - - @Mock - public void clearSnapshot(String tag, String... keyspaces) throws IOException { + public void clearSnapshot(String tag, String... keyspaces) { cleanup(new File("target/data")); } } diff --git a/priam/src/test/java/com/netflix/priam/backup/TestCompression.java b/priam/src/test/java/com/netflix/priam/backup/TestCompression.java index 429d3dea6..eee1b1b03 100644 --- a/priam/src/test/java/com/netflix/priam/backup/TestCompression.java +++ b/priam/src/test/java/com/netflix/priam/backup/TestCompression.java @@ -28,7 +28,6 @@ import java.util.zip.ZipEntry; import java.util.zip.ZipFile; import java.util.zip.ZipOutputStream; -import org.apache.commons.io.IOUtils; import org.junit.After; import org.junit.Before; import org.junit.Ignore; @@ -60,15 +59,15 @@ public void done() { if (f.exists()) f.delete(); } - void validateCompression(String uncompress, String compress) { - File uncompressed = new File(uncompress); + private void validateCompression(String compress) { + File uncompressed = new File("/tmp/compress-test.txt"); File compressed = new File(compress); assertTrue(uncompressed.length() > compressed.length()); } @Test public void zip() throws IOException { - BufferedInputStream source = null; + BufferedInputStream source; try (ZipOutputStream out = new ZipOutputStream( new BufferedOutputStream(new FileOutputStream("/tmp/compressed.zip")))) { @@ -83,7 +82,7 @@ public void zip() throws IOException { out.write(data, 0, count); } } - validateCompression("/tmp/compress-test.txt", "/tmp/compressed.zip"); + validateCompression("/tmp/compressed.zip"); } @Test @@ -95,7 +94,7 @@ public void unzip() throws IOException { try (BufferedInputStream is = new BufferedInputStream(zipfile.getInputStream(entry)); BufferedOutputStream dest1 = new BufferedOutputStream( - new FileOutputStream("/tmp/compress-test-out-0.txt"), 2048)) {; + new FileOutputStream("/tmp/compress-test-out-0.txt"), 2048)) { int c; byte d[] = new byte[2048]; @@ -111,42 +110,41 @@ public void unzip() throws IOException { @Test public void snappyCompress() throws IOException { - FileInputStream fi = new FileInputStream("/tmp/compress-test.txt"); - SnappyOutputStream out = - new SnappyOutputStream( - new BufferedOutputStream(new FileOutputStream("/tmp/test0.snp"))); - BufferedInputStream origin = new BufferedInputStream(fi, 1024); - byte data[] = new byte[1024]; - int count; - while ((count = origin.read(data, 0, 1024)) != -1) { - out.write(data, 0, count); + try (BufferedInputStream origin = + new BufferedInputStream( + new FileInputStream("/tmp/compress-test.txt"), 1024); + SnappyOutputStream out = + new SnappyOutputStream( + new BufferedOutputStream(new FileOutputStream("/tmp/test0.snp")))) { + byte data[] = new byte[1024]; + int count; + while ((count = origin.read(data, 0, 1024)) != -1) { + out.write(data, 0, count); + } + validateCompression("/tmp/test0.snp"); } - IOUtils.closeQuietly(origin); - IOUtils.closeQuietly(fi); - IOUtils.closeQuietly(out); - - validateCompression("/tmp/compress-test.txt", "/tmp/test0.snp"); } @Test public void snappyDecompress() throws IOException { // decompress normally. - SnappyInputStream is = - new SnappyInputStream( - new BufferedInputStream(new FileInputStream("/tmp/test0.snp"))); - byte d[] = new byte[1024]; - FileOutputStream fos = new FileOutputStream("/tmp/compress-test-out-1.txt"); - BufferedOutputStream dest1 = new BufferedOutputStream(fos, 1024); - int c; - while ((c = is.read(d, 0, 1024)) != -1) { - dest1.write(d, 0, c); - } - IOUtils.closeQuietly(dest1); - IOUtils.closeQuietly(is); + try (SnappyInputStream is = + new SnappyInputStream( + new BufferedInputStream(new FileInputStream("/tmp/test0.snp"))); + BufferedOutputStream dest1 = + new BufferedOutputStream( + new FileOutputStream("/tmp/compress-test-out-1.txt"), 1024)) { + + byte d[] = new byte[1024]; + int c; + while ((c = is.read(d, 0, 1024)) != -1) { + dest1.write(d, 0, c); + } - String md1 = SystemUtils.md5(new File("/tmp/compress-test-out-1.txt")); - String md2 = SystemUtils.md5(new File("/tmp/compress-test.txt")); - assertEquals(md1, md2); + String md1 = SystemUtils.md5(new File("/tmp/compress-test-out-1.txt")); + String md2 = SystemUtils.md5(new File("/tmp/compress-test.txt")); + assertEquals(md1, md2); + } } @Test @@ -158,13 +156,13 @@ public void compress() throws IOException { compress.compress( new AbstractBackupPath.RafInputStream(new RandomAccessFile(file, "r")), chunkSize); - FileOutputStream ostream = new FileOutputStream("/tmp/test1.snp"); - while (it.hasNext()) { - byte[] chunk = it.next(); - ostream.write(chunk); + try (FileOutputStream ostream = new FileOutputStream("/tmp/test1.snp")) { + while (it.hasNext()) { + byte[] chunk = it.next(); + ostream.write(chunk); + } } - IOUtils.closeQuietly(ostream); - validateCompression("/tmp/compress-test.txt", "/tmp/test1.snp"); + validateCompression("/tmp/test1.snp"); } @Test diff --git a/priam/src/test/java/com/netflix/priam/backup/TestCustomizedTPE.java b/priam/src/test/java/com/netflix/priam/backup/TestCustomizedTPE.java index 2bdce5483..0f02ddf39 100644 --- a/priam/src/test/java/com/netflix/priam/backup/TestCustomizedTPE.java +++ b/priam/src/test/java/com/netflix/priam/backup/TestCustomizedTPE.java @@ -31,23 +31,21 @@ public class TestCustomizedTPE { private static final int MAX_THREADS = 10; // timeout 1 sec private static final int TIME_OUT = 10 * 1000; - private BlockingSubmitThreadPoolExecutor startTest = + private final BlockingSubmitThreadPoolExecutor startTest = new BlockingSubmitThreadPoolExecutor( - MAX_THREADS, new LinkedBlockingDeque(MAX_THREADS), TIME_OUT); + MAX_THREADS, new LinkedBlockingDeque<>(MAX_THREADS), TIME_OUT); @Test public void testExecutor() throws InterruptedException { final AtomicInteger count = new AtomicInteger(); for (int i = 0; i < 100; i++) { startTest.submit( - new Callable() { - @Override - public Void call() throws Exception { - Thread.sleep(100); - logger.info("Count:{}", count.incrementAndGet()); - return null; - } - }); + (Callable) + () -> { + Thread.sleep(100); + logger.info("Count:{}", count.incrementAndGet()); + return null; + }); } startTest.sleepTillEmpty(); Assert.assertEquals(100, count.get()); @@ -59,14 +57,12 @@ public void testException() { try { for (int i = 0; i < 100; i++) { startTest.submit( - new Callable() { - @Override - public Void call() throws Exception { - logger.info("Sleeping for 2 * timeout."); - Thread.sleep(TIME_OUT * 2); - return null; - } - }); + (Callable) + () -> { + logger.info("Sleeping for 2 * timeout."); + Thread.sleep(TIME_OUT * 2); + return null; + }); } } catch (RuntimeException ex) { success = true; diff --git a/priam/src/test/java/com/netflix/priam/backup/TestFileIterator.java b/priam/src/test/java/com/netflix/priam/backup/TestFileIterator.java index 406b68294..44b8f4e8b 100644 --- a/priam/src/test/java/com/netflix/priam/backup/TestFileIterator.java +++ b/priam/src/test/java/com/netflix/priam/backup/TestFileIterator.java @@ -148,7 +148,7 @@ public void testIteratorEmptySet() { "TESTBUCKET", stime, etime); - Set files = new HashSet(); + Set files = new HashSet<>(); while (fileIterator.hasNext()) files.add(fileIterator.next().getRemotePath()); Assert.assertEquals(0, files.size()); } @@ -176,7 +176,7 @@ public void testIterator() { "TESTBUCKET", startTime, endTime); - Set files = new HashSet(); + Set files = new HashSet<>(); while (fileIterator.hasNext()) files.add(fileIterator.next().getRemotePath()); Assert.assertEquals(3, files.size()); Assert.assertTrue( @@ -224,7 +224,7 @@ public void testIteratorTruncated() { "TESTBUCKET", startTime, endTime); - Set files = new HashSet(); + Set files = new HashSet<>(); while (fileIterator.hasNext()) files.add(fileIterator.next().getRemotePath()); Assert.assertEquals(5, files.size()); Assert.assertTrue( @@ -288,7 +288,7 @@ public void testIteratorTruncatedOOR() { "TESTBUCKET", startTime, endTime); - Set files = new HashSet(); + Set files = new HashSet<>(); while (fileIterator.hasNext()) files.add(fileIterator.next().getRemotePath()); Assert.assertEquals(2, files.size()); Assert.assertFalse( @@ -348,7 +348,7 @@ public void testRestorePathIteration() { "RESTOREBUCKET/test_restore_backup/fake-restore-region/fakerestorecluster", startTime, endTime); - Set files = new HashSet(); + Set files = new HashSet<>(); while (fileIterator.hasNext()) files.add(fileIterator.next().getRemotePath()); while (fileIterator.hasNext()) files.add(fileIterator.next().getRemotePath()); @@ -391,8 +391,8 @@ public void testRestorePathIteration() { + "/fakecluster/123456/201108110600/SST/ks2/cf1/f3.db")); } - public static List getObjectSummary() { - List list = new ArrayList(); + private static List getObjectSummary() { + List list = new ArrayList<>(); S3ObjectSummary summary = new S3ObjectSummary(); summary.setKey( "test_backup/" + region + "/fakecluster/123456/201108110030/SNAP/ks1/cf1/f1.db"); @@ -411,12 +411,12 @@ public static List getObjectSummary() { return list; } - public static List getObjectSummaryEmpty() { - return new ArrayList(); + private static List getObjectSummaryEmpty() { + return new ArrayList<>(); } - public static List getNextObjectSummary() { - List list = new ArrayList(); + private static List getNextObjectSummary() { + List list = new ArrayList<>(); S3ObjectSummary summary = new S3ObjectSummary(); summary.setKey( "test_backup/" + region + "/fakecluster/123456/201108110030/SNAP/ks2/cf1/f1.db"); diff --git a/priam/src/test/java/com/netflix/priam/backup/TestRestore.java b/priam/src/test/java/com/netflix/priam/backup/TestRestore.java index 446913154..b603f3dd0 100644 --- a/priam/src/test/java/com/netflix/priam/backup/TestRestore.java +++ b/priam/src/test/java/com/netflix/priam/backup/TestRestore.java @@ -35,7 +35,6 @@ import org.junit.Test; public class TestRestore { - private static Injector injector; private static FakeBackupFileSystem filesystem; private static ArrayList fileList; private static Calendar cal; @@ -45,7 +44,7 @@ public class TestRestore { @BeforeClass public static void setup() throws InterruptedException, IOException { - injector = Guice.createInjector(new BRTestModule()); + Injector injector = Guice.createInjector(new BRTestModule()); filesystem = (FakeBackupFileSystem) injector.getInstance( diff --git a/priam/src/test/java/com/netflix/priam/backup/TestS3FileSystem.java b/priam/src/test/java/com/netflix/priam/backup/TestS3FileSystem.java index 476d167e3..3ef9c5db9 100644 --- a/priam/src/test/java/com/netflix/priam/backup/TestS3FileSystem.java +++ b/priam/src/test/java/com/netflix/priam/backup/TestS3FileSystem.java @@ -18,7 +18,6 @@ package com.netflix.priam.backup; import com.amazonaws.AmazonClientException; -import com.amazonaws.AmazonServiceException; import com.amazonaws.SdkClientException; import com.amazonaws.services.s3.AmazonS3; import com.amazonaws.services.s3.AmazonS3Client; @@ -51,7 +50,7 @@ public class TestS3FileSystem { private static Injector injector; private static final Logger logger = LoggerFactory.getLogger(TestS3FileSystem.class); - private static String FILE_PATH = + private static final String FILE_PATH = "target/data/Keyspace1/Standard1/backups/201108082320/Keyspace1-Standard1-ia-1-Data.db"; private static BackupMetrics backupMetrics; private static String region; @@ -201,9 +200,6 @@ public CompleteMultipartUploadResult completeUpload() throws BackupRestoreExcept return null; } - @Mock - public void abortUpload() {} - @Mock public Void retriableCall() throws AmazonClientException, BackupRestoreException { logger.info("MOCK UPLOADING..."); @@ -222,9 +218,6 @@ static class MockAmazonS3Client extends MockUp { static boolean ruleAvailable = false; static BucketLifecycleConfiguration bconf = new BucketLifecycleConfiguration(); - @Mock - public void $init() {} - @Mock public InitiateMultipartUploadResult initiateMultipartUpload( InitiateMultipartUploadRequest initiateMultipartUploadRequest) @@ -233,7 +226,7 @@ public InitiateMultipartUploadResult initiateMultipartUpload( } public PutObjectResult putObject(PutObjectRequest putObjectRequest) - throws SdkClientException, AmazonServiceException { + throws SdkClientException { PutObjectResult result = new PutObjectResult(); result.setETag("ad"); return result; diff --git a/priam/src/test/java/com/netflix/priam/backup/TestSnapshotStatusMgr.java b/priam/src/test/java/com/netflix/priam/backup/TestSnapshotStatusMgr.java index e70ae6858..6eb493601 100644 --- a/priam/src/test/java/com/netflix/priam/backup/TestSnapshotStatusMgr.java +++ b/priam/src/test/java/com/netflix/priam/backup/TestSnapshotStatusMgr.java @@ -100,6 +100,7 @@ public void testSnapshotStatusMultiAddFinishInADay() throws Exception { Date startTime = DateUtil.getDate("19840101"); for (int i = 0; i < noOfEntries; i++) { + assert startTime != null; Date time = new DateTime(startTime.getTime()).plusHours(i).toDate(); BackupMetadata backupMetadata = new BackupMetadata("123", time); backupStatusMgr.start(backupMetadata); @@ -127,6 +128,7 @@ public void testSnapshotStatusSize() throws Exception { Date startTime = DateUtil.getDate("19850101"); for (int i = 0; i < noOfEntries; i++) { + assert startTime != null; Date time = new DateTime(startTime.getTime()).plusDays(i).toDate(); BackupMetadata backupMetadata = new BackupMetadata("123", time); backupStatusMgr.start(backupMetadata); diff --git a/priam/src/test/java/com/netflix/priam/backup/identity/InstanceTestUtils.java b/priam/src/test/java/com/netflix/priam/backup/identity/InstanceTestUtils.java index f291d8525..67d9433a5 100644 --- a/priam/src/test/java/com/netflix/priam/backup/identity/InstanceTestUtils.java +++ b/priam/src/test/java/com/netflix/priam/backup/identity/InstanceTestUtils.java @@ -34,8 +34,8 @@ @Ignore public abstract class InstanceTestUtils { - List instances = new ArrayList(); - IMembership membership; + private final List instances = new ArrayList<>(); + private IMembership membership; FakeConfiguration config; IPriamInstanceFactory factory; InstanceIdentity identity; @@ -60,7 +60,7 @@ public void setup() throws Exception { config = new FakeConfiguration("fake-app"); instanceInfo = new FakeInstanceInfo("fakeinstance1", "az1", region); tokenManager = new TokenManager(config); - factory = new FakePriamInstanceFactory(config, instanceInfo); + factory = new FakePriamInstanceFactory(instanceInfo); sleeper = new FakeSleeper(); identity = createInstanceIdentity(instanceInfo.getRac(), instanceInfo.getInstanceId()); } diff --git a/priam/src/test/java/com/netflix/priam/backup/identity/token/FakeDeadTokenRetriever.java b/priam/src/test/java/com/netflix/priam/backup/identity/token/FakeDeadTokenRetriever.java index 8f9b825da..ff0d9f041 100755 --- a/priam/src/test/java/com/netflix/priam/backup/identity/token/FakeDeadTokenRetriever.java +++ b/priam/src/test/java/com/netflix/priam/backup/identity/token/FakeDeadTokenRetriever.java @@ -21,7 +21,7 @@ import com.netflix.priam.identity.PriamInstance; import com.netflix.priam.identity.token.IDeadTokenRetriever; -public class FakeDeadTokenRetriever implements IDeadTokenRetriever { +class FakeDeadTokenRetriever implements IDeadTokenRetriever { @Override public PriamInstance get() throws Exception { diff --git a/priam/src/test/java/com/netflix/priam/backup/identity/token/FakeNewTokenRetriever.java b/priam/src/test/java/com/netflix/priam/backup/identity/token/FakeNewTokenRetriever.java index 7dc3c1487..14a357e51 100755 --- a/priam/src/test/java/com/netflix/priam/backup/identity/token/FakeNewTokenRetriever.java +++ b/priam/src/test/java/com/netflix/priam/backup/identity/token/FakeNewTokenRetriever.java @@ -21,7 +21,7 @@ import com.netflix.priam.identity.PriamInstance; import com.netflix.priam.identity.token.INewTokenRetriever; -public class FakeNewTokenRetriever implements INewTokenRetriever { +class FakeNewTokenRetriever implements INewTokenRetriever { @Override public PriamInstance get() throws Exception { diff --git a/priam/src/test/java/com/netflix/priam/backup/identity/token/FakePreGeneratedTokenRetriever.java b/priam/src/test/java/com/netflix/priam/backup/identity/token/FakePreGeneratedTokenRetriever.java index 32b81ebad..b4f827daa 100755 --- a/priam/src/test/java/com/netflix/priam/backup/identity/token/FakePreGeneratedTokenRetriever.java +++ b/priam/src/test/java/com/netflix/priam/backup/identity/token/FakePreGeneratedTokenRetriever.java @@ -21,7 +21,7 @@ import com.netflix.priam.identity.PriamInstance; import com.netflix.priam.identity.token.IPreGeneratedTokenRetriever; -public class FakePreGeneratedTokenRetriever implements IPreGeneratedTokenRetriever { +class FakePreGeneratedTokenRetriever implements IPreGeneratedTokenRetriever { @Override public PriamInstance get() throws Exception { diff --git a/priam/src/test/java/com/netflix/priam/backupv2/TestLocalDBReaderWriter.java b/priam/src/test/java/com/netflix/priam/backupv2/TestLocalDBReaderWriter.java index 2a63551bf..1b9a0124d 100644 --- a/priam/src/test/java/com/netflix/priam/backupv2/TestLocalDBReaderWriter.java +++ b/priam/src/test/java/com/netflix/priam/backupv2/TestLocalDBReaderWriter.java @@ -87,21 +87,17 @@ public void readWriteLocalDB() throws Exception { List localDBList = generateDummyLocalDB(noOfKeyspaces, noOfCf, noOfSstables); - localDBList - .stream() - .forEach( - localDB -> { - FileUploadResult fileUploadResult = - localDB.getLocalDBEntries().get(0).getFileUploadResult(); - final Path localDBPath = - localDBReaderWriter.getLocalDBPath(fileUploadResult); - try { - localDBReaderWriter.writeLocalDB(localDBPath, localDB); - } catch (Exception e) { - logger.error( - "Error while writing to local DB: " + e.getMessage(), e); - } - }); + localDBList.forEach( + localDB -> { + FileUploadResult fileUploadResult = + localDB.getLocalDBEntries().get(0).getFileUploadResult(); + final Path localDBPath = localDBReaderWriter.getLocalDBPath(fileUploadResult); + try { + localDBReaderWriter.writeLocalDB(localDBPath, localDB); + } catch (Exception e) { + logger.error("Error while writing to local DB: " + e.getMessage(), e); + } + }); // Verify the write succeeded for each KS/CF/SStable. Assert.assertEquals(localDbPath.toFile().listFiles().length, noOfKeyspaces); @@ -121,7 +117,6 @@ public void upsertLocalDB() throws Exception { // Lets do write with each LocalDBEntry first. localDB.getLocalDBEntries() - .stream() .forEach( localDBEntry -> { try { @@ -168,21 +163,17 @@ public void upsertLocalDB() throws Exception { @Test public void readConcurrentLocalDB() throws Exception { List localDBList = generateDummyLocalDB(1, 1, 1); - localDBList - .stream() - .forEach( - localDB -> { - FileUploadResult fileUploadResult = - localDB.getLocalDBEntries().get(0).getFileUploadResult(); - final Path localDBPath = - localDBReaderWriter.getLocalDBPath(fileUploadResult); - try { - localDBReaderWriter.writeLocalDB(localDBPath, localDB); - } catch (Exception e) { - logger.error( - "Error while writing to local DB: " + e.getMessage(), e); - } - }); + localDBList.forEach( + localDB -> { + FileUploadResult fileUploadResult = + localDB.getLocalDBEntries().get(0).getFileUploadResult(); + final Path localDBPath = localDBReaderWriter.getLocalDBPath(fileUploadResult); + try { + localDBReaderWriter.writeLocalDB(localDBPath, localDB); + } catch (Exception e) { + logger.error("Error while writing to local DB: " + e.getMessage(), e); + } + }); FileUploadResult sample = localDBList.get(0).getLocalDBEntries().get(0).getFileUploadResult(); @@ -247,7 +238,7 @@ public void writeConcurrentLocalDB() throws Exception { } private List generateDummyLocalDB( - int noOfKeyspaces, int noOfCf, int noOfSstables) throws Exception { + int noOfKeyspaces, int noOfCf, int noOfSstables) { // Clean the dummy directory cleanupDir(dummyDataDirectoryLocation); diff --git a/priam/src/test/java/com/netflix/priam/cassandra/token/TestDoublingLogic.java b/priam/src/test/java/com/netflix/priam/cassandra/token/TestDoublingLogic.java index b7ebd9d51..295e127e3 100644 --- a/priam/src/test/java/com/netflix/priam/cassandra/token/TestDoublingLogic.java +++ b/priam/src/test/java/com/netflix/priam/cassandra/token/TestDoublingLogic.java @@ -33,7 +33,7 @@ public class TestDoublingLogic { @Test public void testSkip() { - List nodes = new ArrayList(); + List nodes = new ArrayList<>(); for (int i = 0; i < NODES_PER_RACS; i++) for (int j = 0; j < RACS; j++) nodes.add("RAC-" + j); // printNodes(nodes); @@ -81,8 +81,8 @@ private void validate(List newNodes, List nodes) { } private List doubleNodes(List nodes) { - List lst = new ArrayList(); - Map return_ = new HashMap(); + List lst = new ArrayList<>(); + Map return_ = new HashMap<>(); for (int i = 0; i < nodes.size(); i++) { return_.put(i * 2, nodes.get(i)); } diff --git a/priam/src/test/java/com/netflix/priam/config/FakeConfiguration.java b/priam/src/test/java/com/netflix/priam/config/FakeConfiguration.java index ae42b5290..b9288baf4 100644 --- a/priam/src/test/java/com/netflix/priam/config/FakeConfiguration.java +++ b/priam/src/test/java/com/netflix/priam/config/FakeConfiguration.java @@ -31,11 +31,11 @@ @Singleton public class FakeConfiguration implements IConfiguration { - private String appName; + private final String appName; private String restorePrefix = ""; public Map fakeConfig; - public Map fakeProperties = new HashMap<>(); + public final Map fakeProperties = new HashMap<>(); public FakeConfiguration() { this("my_fake_cluster"); diff --git a/priam/src/test/java/com/netflix/priam/health/TestInstanceStatus.java b/priam/src/test/java/com/netflix/priam/health/TestInstanceStatus.java index 7cac3f82e..edba35aab 100644 --- a/priam/src/test/java/com/netflix/priam/health/TestInstanceStatus.java +++ b/priam/src/test/java/com/netflix/priam/health/TestInstanceStatus.java @@ -83,7 +83,7 @@ public void testHealth() { } private class TestInstanceState { - private InstanceState instanceState; + private final InstanceState instanceState; TestInstanceState(InstanceState instanceState1) { this.instanceState = instanceState1; diff --git a/priam/src/test/java/com/netflix/priam/identity/FakePriamInstanceFactory.java b/priam/src/test/java/com/netflix/priam/identity/FakePriamInstanceFactory.java index 442d1ac51..bedaae527 100644 --- a/priam/src/test/java/com/netflix/priam/identity/FakePriamInstanceFactory.java +++ b/priam/src/test/java/com/netflix/priam/identity/FakePriamInstanceFactory.java @@ -19,18 +19,15 @@ import com.google.common.collect.Maps; import com.google.inject.Inject; -import com.netflix.priam.config.IConfiguration; import com.netflix.priam.identity.config.InstanceInfo; import java.util.*; public class FakePriamInstanceFactory implements IPriamInstanceFactory { private final Map instances = Maps.newHashMap(); - private final IConfiguration config; private final InstanceInfo instanceInfo; @Inject - public FakePriamInstanceFactory(IConfiguration config, InstanceInfo instanceInfo) { - this.config = config; + public FakePriamInstanceFactory(InstanceInfo instanceInfo) { this.instanceInfo = instanceInfo; } @@ -82,16 +79,13 @@ public void update(PriamInstance inst) { @Override public void sort(List return_) { Comparator comparator = - new Comparator() { - - @Override - public int compare(PriamInstance o1, PriamInstance o2) { - Integer c1 = o1.getId(); - Integer c2 = o2.getId(); - return c1.compareTo(c2); - } - }; - Collections.sort(return_, comparator); + (Comparator) + (o1, o2) -> { + Integer c1 = o1.getId(); + Integer c2 = o2.getId(); + return c1.compareTo(c2); + }; + return_.sort(comparator); } @Override diff --git a/priam/src/test/java/com/netflix/priam/resources/BackupServletTest.java b/priam/src/test/java/com/netflix/priam/resources/BackupServletTest.java index 20c9078e4..793292ffb 100644 --- a/priam/src/test/java/com/netflix/priam/resources/BackupServletTest.java +++ b/priam/src/test/java/com/netflix/priam/resources/BackupServletTest.java @@ -116,7 +116,7 @@ public void restore_withDateRange() throws Exception { { DateUtil.getDate(dateRange.split(",")[0]); - result = new DateTime(2011, 01, 01, 00, 00).toDate(); + result = new DateTime(2011, 1, 1, 0, 0).toDate(); times = 1; DateUtil.getDate(dateRange.split(",")[1]); result = new DateTime(2011, 12, 31, 23, 59).toDate(); @@ -137,8 +137,8 @@ public void restore_withDateRange() throws Exception { } // TODO: create CassandraController interface and inject, instead of static util method - private Expectations expectCassandraStartup() { - return new Expectations() { + private void expectCassandraStartup() { + new Expectations() { { config.getCassStartupScript(); result = "/usr/bin/false"; diff --git a/priam/src/test/java/com/netflix/priam/resources/PriamInstanceResourceTest.java b/priam/src/test/java/com/netflix/priam/resources/PriamInstanceResourceTest.java index cbd2bf65f..d7470945b 100644 --- a/priam/src/test/java/com/netflix/priam/resources/PriamInstanceResourceTest.java +++ b/priam/src/test/java/com/netflix/priam/resources/PriamInstanceResourceTest.java @@ -56,7 +56,7 @@ public void getInstances( @Mocked final PriamInstance instance2, @Mocked final PriamInstance instance3) { new Expectations() { - List instances = ImmutableList.of(instance1, instance2, instance3); + final List instances = ImmutableList.of(instance1, instance2, instance3); { config.getAppName(); diff --git a/priam/src/test/java/com/netflix/priam/restore/TestPostRestoreHook.java b/priam/src/test/java/com/netflix/priam/restore/TestPostRestoreHook.java index 3360b7594..68afaaa17 100644 --- a/priam/src/test/java/com/netflix/priam/restore/TestPostRestoreHook.java +++ b/priam/src/test/java/com/netflix/priam/restore/TestPostRestoreHook.java @@ -48,10 +48,10 @@ public void setup() { } @Test - /** - * Test to validate hasValidParameters. Expected to pass since none of the parameters in - * FakeConfiguration are blank - */ + /* + Test to validate hasValidParameters. Expected to pass since none of the parameters in + FakeConfiguration are blank + */ public void testPostRestoreHookValidParameters() { Injector inject = Guice.createInjector(new TestModule()); IPostRestoreHook postRestoreHook = inject.getInstance(IPostRestoreHook.class); @@ -59,11 +59,11 @@ public void testPostRestoreHookValidParameters() { } @Test - /** - * Test to validate execute method. This is a happy path since heart beat file is emited as soon - * as test case starts, and postrestorehook completes execution once the child process completes - * execution. Test fails in case of any exception. - */ + /* + Test to validate execute method. This is a happy path since heart beat file is emited as soon + as test case starts, and postrestorehook completes execution once the child process completes + execution. Test fails in case of any exception. + */ public void testPostRestoreHookExecuteHappyPath() throws Exception { Injector inject = Guice.createInjector(new TestModule()); IPostRestoreHook postRestoreHook = inject.getInstance(IPostRestoreHook.class); @@ -76,13 +76,13 @@ public void testPostRestoreHookExecuteHappyPath() throws Exception { } @Test - /** - * Test to validate execute method. This is a variant of above method, where heartbeat is - * produced after an initial delay. This delay causes PostRestoreHook to terminate the child - * process since there is no heartbeat multiple times, and eventually once the heartbeat starts, - * PostRestoreHook waits for the child process to complete execution. Test fails in case of any - * exception. - */ + /* + Test to validate execute method. This is a variant of above method, where heartbeat is + produced after an initial delay. This delay causes PostRestoreHook to terminate the child + process since there is no heartbeat multiple times, and eventually once the heartbeat starts, + PostRestoreHook waits for the child process to complete execution. Test fails in case of any + exception. + */ public void testPostRestoreHookExecuteHeartBeatDelay() throws Exception { Injector inject = Guice.createInjector(new TestModule()); IPostRestoreHook postRestoreHook = inject.getInstance(IPostRestoreHook.class); @@ -104,27 +104,26 @@ public void testPostRestoreHookExecuteHeartBeatDelay() throws Exception { private void startHeartBeatThreadWithDelay( long delayInMs, String heartBeatfileName, String doneFileName) { Thread heartBeatEmitThread = - new Thread() { - public void run() { - File heartBeatFile = new File(heartBeatfileName); - try { - // add a delay to heartbeat - Thread.sleep(delayInMs); - if (!heartBeatFile.exists() && !heartBeatFile.createNewFile()) { - Assert.fail("Unable to create heartbeat file"); - } - for (int i = 0; i < 10; i++) { - FileUtils.touch(heartBeatFile); - Thread.sleep(1000); - } + new Thread( + () -> { + File heartBeatFile = new File(heartBeatfileName); + try { + // add a delay to heartbeat + Thread.sleep(delayInMs); + if (!heartBeatFile.exists() && !heartBeatFile.createNewFile()) { + Assert.fail("Unable to create heartbeat file"); + } + for (int i = 0; i < 10; i++) { + FileUtils.touch(heartBeatFile); + Thread.sleep(1000); + } - File doneFile = new File(doneFileName); - doneFile.createNewFile(); - } catch (Exception ex) { - Assert.fail(ex.getMessage()); - } - } - }; + File doneFile = new File(doneFileName); + doneFile.createNewFile(); + } catch (Exception ex) { + Assert.fail(ex.getMessage()); + } + }); heartBeatEmitThread.start(); } diff --git a/priam/src/test/java/com/netflix/priam/scheduler/TestGuiceSingleton.java b/priam/src/test/java/com/netflix/priam/scheduler/TestGuiceSingleton.java index 85448849e..e0eff0417 100644 --- a/priam/src/test/java/com/netflix/priam/scheduler/TestGuiceSingleton.java +++ b/priam/src/test/java/com/netflix/priam/scheduler/TestGuiceSingleton.java @@ -41,16 +41,16 @@ public void printInjected() { injector.getInstance(EmptryInterface.class).print(); } - public interface EmptryInterface { - String print(); + interface EmptryInterface { + void print(); } @Singleton public static class GuiceSingleton implements EmptryInterface { - public String print() { + public void print() { System.out.println(this.toString()); - return this.toString(); + this.toString(); } } diff --git a/priam/src/test/java/com/netflix/priam/services/TestSnapshotMetaService.java b/priam/src/test/java/com/netflix/priam/services/TestSnapshotMetaService.java index 0a30b5b68..6a41e869b 100644 --- a/priam/src/test/java/com/netflix/priam/services/TestSnapshotMetaService.java +++ b/priam/src/test/java/com/netflix/priam/services/TestSnapshotMetaService.java @@ -135,11 +135,11 @@ public void testSize() throws Exception { test(1000, 2, 2); } - public static class TestMetaFileReader extends MetaFileReader { + static class TestMetaFileReader extends MetaFileReader { private int noOfSstables; - public void setNoOfSstables(int noOfSstables) { + void setNoOfSstables(int noOfSstables) { this.noOfSstables = noOfSstables; } diff --git a/priam/src/test/java/com/netflix/priam/stream/StreamingTest.java b/priam/src/test/java/com/netflix/priam/stream/StreamingTest.java index 39625ad8a..8ac833be8 100644 --- a/priam/src/test/java/com/netflix/priam/stream/StreamingTest.java +++ b/priam/src/test/java/com/netflix/priam/stream/StreamingTest.java @@ -32,7 +32,7 @@ public class StreamingTest { @Test public void testFifoAddAndRemove() { - FifoQueue queue = new FifoQueue(10); + FifoQueue queue = new FifoQueue<>(10); for (long i = 0; i < 100; i++) queue.adjustAndAdd(i); Assert.assertEquals(10, queue.size()); Assert.assertEquals(new Long(90), queue.first()); @@ -45,7 +45,7 @@ public void testAbstractPath() { InstanceIdentity factory = injector.getInstance(InstanceIdentity.class); String region = factory.getInstanceInfo().getRegion(); - FifoQueue queue = new FifoQueue(10); + FifoQueue queue = new FifoQueue<>(10); for (int i = 10; i < 30; i++) { S3BackupPath path = new S3BackupPath(conf, factory); path.parseRemote( diff --git a/priam/src/test/java/com/netflix/priam/tuner/StandardTunerTest.java b/priam/src/test/java/com/netflix/priam/tuner/StandardTunerTest.java index 4ecb0a479..fd9f107ec 100644 --- a/priam/src/test/java/com/netflix/priam/tuner/StandardTunerTest.java +++ b/priam/src/test/java/com/netflix/priam/tuner/StandardTunerTest.java @@ -31,7 +31,7 @@ public class StandardTunerTest { private static final String MURMUR_PARTITIONER = "org.apache.cassandra.dht.Murmur3Partitioner"; private static final String BOP_PARTITIONER = "org.apache.cassandra.dht.ByteOrderedPartitioner"; - private StandardTuner tuner; + private final StandardTuner tuner; public StandardTunerTest() { this.tuner = Guice.createInjector(new BRTestModule()).getInstance(StandardTuner.class); From aec76f31598f9917eab6c8d45bb0570415ee7fc2 Mon Sep 17 00:00:00 2001 From: Arun Agrawal Date: Thu, 25 Oct 2018 16:31:54 -0700 Subject: [PATCH 027/228] Reduce scope of InstanceIdentity by using InstanceInfo --- .../com/netflix/priam/aws/AWSMembership.java | 42 ++++++++----------- .../aws/auth/EC2RoleAssumptionCredential.java | 9 ++-- 2 files changed, 21 insertions(+), 30 deletions(-) diff --git a/priam/src/main/java/com/netflix/priam/aws/AWSMembership.java b/priam/src/main/java/com/netflix/priam/aws/AWSMembership.java index 482580dc1..ec05f304d 100644 --- a/priam/src/main/java/com/netflix/priam/aws/AWSMembership.java +++ b/priam/src/main/java/com/netflix/priam/aws/AWSMembership.java @@ -30,7 +30,6 @@ import com.netflix.priam.config.IConfiguration; import com.netflix.priam.cred.ICredential; import com.netflix.priam.identity.IMembership; -import com.netflix.priam.identity.InstanceIdentity; import com.netflix.priam.identity.config.InstanceInfo; import java.util.*; import org.apache.commons.lang3.StringUtils; @@ -45,7 +44,7 @@ public class AWSMembership implements IMembership { private static final Logger logger = LoggerFactory.getLogger(AWSMembership.class); private final IConfiguration config; private final ICredential provider; - private final InstanceIdentity instanceIdentity; + private final InstanceInfo instanceInfo; private final ICredential crossAccountProvider; @Inject @@ -53,10 +52,10 @@ public AWSMembership( IConfiguration config, ICredential provider, @Named("awsec2roleassumption") ICredential crossAccountProvider, - InstanceIdentity instanceIdentity) { + InstanceInfo instanceInfo) { this.config = config; this.provider = provider; - this.instanceIdentity = instanceIdentity; + this.instanceInfo = instanceInfo; this.crossAccountProvider = crossAccountProvider; } @@ -65,7 +64,7 @@ public List getRacMembership() { AmazonAutoScaling client = null; try { List asgNames = new ArrayList<>(); - asgNames.add(instanceIdentity.getInstanceInfo().getAutoScalingGroup()); + asgNames.add(instanceInfo.getAutoScalingGroup()); asgNames.addAll(Arrays.asList(config.getSiblingASGNames().split("\\s*,\\s*"))); client = getAutoScalingClient(); DescribeAutoScalingGroupsRequest asgReq = @@ -86,7 +85,7 @@ public List getRacMembership() { logger.info( String.format( "Querying Amazon returned following instance in the RAC: %s, ASGs: %s --> %s", - instanceIdentity.getInstanceInfo().getRac(), + instanceInfo.getRac(), StringUtils.join(asgNames, ","), StringUtils.join(instanceIds, ","))); } @@ -104,8 +103,7 @@ public int getRacMembershipSize() { client = getAutoScalingClient(); DescribeAutoScalingGroupsRequest asgReq = new DescribeAutoScalingGroupsRequest() - .withAutoScalingGroupNames( - instanceIdentity.getInstanceInfo().getAutoScalingGroup()); + .withAutoScalingGroupNames(instanceInfo.getAutoScalingGroup()); DescribeAutoScalingGroupsResult res = client.describeAutoScalingGroups(asgReq); int size = 0; for (AutoScalingGroup asg : res.getAutoScalingGroups()) { @@ -123,7 +121,7 @@ public List getCrossAccountRacMembership() { AmazonAutoScaling client = null; try { List asgNames = new ArrayList<>(); - asgNames.add(instanceIdentity.getInstanceInfo().getAutoScalingGroup()); + asgNames.add(instanceInfo.getAutoScalingGroup()); asgNames.addAll(Arrays.asList(config.getSiblingASGNames().split("\\s*,\\s*"))); client = getCrossAccountAutoScalingClient(); DescribeAutoScalingGroupsRequest asgReq = @@ -144,8 +142,7 @@ public List getCrossAccountRacMembership() { logger.info( String.format( "Querying Amazon returned following instance in the cross-account ASG: %s --> %s", - instanceIdentity.getInstanceInfo().getRac(), - StringUtils.join(instanceIds, ","))); + instanceInfo.getRac(), StringUtils.join(instanceIds, ","))); } return instanceIds; } finally { @@ -159,8 +156,7 @@ public int getRacCount() { } private boolean isClassic() { - return instanceIdentity.getInstanceInfo().getInstanceEnvironment() - == InstanceInfo.InstanceEnvironment.CLASSIC; + return instanceInfo.getInstanceEnvironment() == InstanceInfo.InstanceEnvironment.CLASSIC; } /** @@ -214,10 +210,7 @@ protected String getVpcGoupId() { client = getEc2Client(); Filter nameFilter = new Filter().withName("group-name").withValues(config.getACLGroupName()); // SG - Filter vpcFilter = - new Filter() - .withName("vpc-id") - .withValues(instanceIdentity.getInstanceInfo().getVpcId()); + Filter vpcFilter = new Filter().withName("vpc-id").withValues(instanceInfo.getVpcId()); DescribeSecurityGroupsRequest req = new DescribeSecurityGroupsRequest().withFilters(nameFilter, vpcFilter); @@ -227,13 +220,13 @@ protected String getVpcGoupId() { "got group-id:{} for group-name:{},vpc-id:{}", group.getGroupId(), config.getACLGroupName(), - instanceIdentity.getInstanceInfo().getVpcId()); + instanceInfo.getVpcId()); return group.getGroupId(); } logger.error( "unable to get group-id for group-name={} vpc-id={}", config.getACLGroupName(), - instanceIdentity.getInstanceInfo().getVpcId()); + instanceInfo.getVpcId()); return ""; } finally { if (client != null) client.shutdown(); @@ -304,7 +297,7 @@ public List listACL(int from, int to) { Filter nameFilter = new Filter().withName("group-name").withValues(config.getACLGroupName()); - String vpcid = instanceIdentity.getInstanceInfo().getVpcId(); + String vpcid = instanceInfo.getVpcId(); if (vpcid == null || vpcid.isEmpty()) { throw new IllegalStateException( "vpcid is null even though instance is running in vpc."); @@ -336,8 +329,7 @@ public void expandRacMembership(int count) { client = getAutoScalingClient(); DescribeAutoScalingGroupsRequest asgReq = new DescribeAutoScalingGroupsRequest() - .withAutoScalingGroupNames( - instanceIdentity.getInstanceInfo().getAutoScalingGroup()); + .withAutoScalingGroupNames(instanceInfo.getAutoScalingGroup()); DescribeAutoScalingGroupsResult res = client.describeAutoScalingGroups(asgReq); AutoScalingGroup asg = res.getAutoScalingGroups().get(0); UpdateAutoScalingGroupRequest ureq = new UpdateAutoScalingGroupRequest(); @@ -354,21 +346,21 @@ public void expandRacMembership(int count) { protected AmazonAutoScaling getAutoScalingClient() { return AmazonAutoScalingClientBuilder.standard() .withCredentials(provider.getAwsCredentialProvider()) - .withRegion(instanceIdentity.getInstanceInfo().getRegion()) + .withRegion(instanceInfo.getRegion()) .build(); } protected AmazonAutoScaling getCrossAccountAutoScalingClient() { return AmazonAutoScalingClientBuilder.standard() .withCredentials(crossAccountProvider.getAwsCredentialProvider()) - .withRegion(instanceIdentity.getInstanceInfo().getRegion()) + .withRegion(instanceInfo.getRegion()) .build(); } protected AmazonEC2 getEc2Client() { return AmazonEC2ClientBuilder.standard() .withCredentials(provider.getAwsCredentialProvider()) - .withRegion(instanceIdentity.getInstanceInfo().getRegion()) + .withRegion(instanceInfo.getRegion()) .build(); } } diff --git a/priam/src/main/java/com/netflix/priam/aws/auth/EC2RoleAssumptionCredential.java b/priam/src/main/java/com/netflix/priam/aws/auth/EC2RoleAssumptionCredential.java index 45131f622..c01c92442 100644 --- a/priam/src/main/java/com/netflix/priam/aws/auth/EC2RoleAssumptionCredential.java +++ b/priam/src/main/java/com/netflix/priam/aws/auth/EC2RoleAssumptionCredential.java @@ -18,22 +18,21 @@ import com.google.inject.Inject; import com.netflix.priam.config.IConfiguration; import com.netflix.priam.cred.ICredential; -import com.netflix.priam.identity.InstanceIdentity; import com.netflix.priam.identity.config.InstanceInfo; public class EC2RoleAssumptionCredential implements ICredential { private static final String AWS_ROLE_ASSUMPTION_SESSION_NAME = "AwsRoleAssumptionSession"; private final ICredential cred; private final IConfiguration config; - private final InstanceIdentity instanceIdentity; + private final InstanceInfo instanceInfo; private AWSCredentialsProvider stsSessionCredentialsProvider; @Inject public EC2RoleAssumptionCredential( - ICredential cred, IConfiguration config, InstanceIdentity instanceIdentity) { + ICredential cred, IConfiguration config, InstanceInfo instanceInfo) { this.cred = cred; this.config = config; - this.instanceIdentity = instanceIdentity; + this.instanceInfo = instanceInfo; } @Override @@ -48,7 +47,7 @@ public AWSCredentialsProvider getAwsCredentialProvider() { * current environment is VPC, then the assumed role is for EC2 classic, and * vice versa. */ - if (instanceIdentity.getInstanceInfo().getInstanceEnvironment() + if (instanceInfo.getInstanceEnvironment() == InstanceInfo.InstanceEnvironment.CLASSIC) { roleArn = this.config.getClassicEC2RoleAssumptionArn(); // Env is EC2 classic --> IAM assumed role for VPC created From 253932078c7d187b5d364b49ef544427b4bc290f Mon Sep 17 00:00:00 2001 From: Arun Agrawal Date: Fri, 26 Oct 2018 09:38:06 -0700 Subject: [PATCH 028/228] Reduce scope of InstanceIdentity by using InstanceInfo (#748) --- .../com/netflix/priam/aws/AWSMembership.java | 42 ++++++++----------- .../aws/auth/EC2RoleAssumptionCredential.java | 9 ++-- 2 files changed, 21 insertions(+), 30 deletions(-) diff --git a/priam/src/main/java/com/netflix/priam/aws/AWSMembership.java b/priam/src/main/java/com/netflix/priam/aws/AWSMembership.java index 482580dc1..ec05f304d 100644 --- a/priam/src/main/java/com/netflix/priam/aws/AWSMembership.java +++ b/priam/src/main/java/com/netflix/priam/aws/AWSMembership.java @@ -30,7 +30,6 @@ import com.netflix.priam.config.IConfiguration; import com.netflix.priam.cred.ICredential; import com.netflix.priam.identity.IMembership; -import com.netflix.priam.identity.InstanceIdentity; import com.netflix.priam.identity.config.InstanceInfo; import java.util.*; import org.apache.commons.lang3.StringUtils; @@ -45,7 +44,7 @@ public class AWSMembership implements IMembership { private static final Logger logger = LoggerFactory.getLogger(AWSMembership.class); private final IConfiguration config; private final ICredential provider; - private final InstanceIdentity instanceIdentity; + private final InstanceInfo instanceInfo; private final ICredential crossAccountProvider; @Inject @@ -53,10 +52,10 @@ public AWSMembership( IConfiguration config, ICredential provider, @Named("awsec2roleassumption") ICredential crossAccountProvider, - InstanceIdentity instanceIdentity) { + InstanceInfo instanceInfo) { this.config = config; this.provider = provider; - this.instanceIdentity = instanceIdentity; + this.instanceInfo = instanceInfo; this.crossAccountProvider = crossAccountProvider; } @@ -65,7 +64,7 @@ public List getRacMembership() { AmazonAutoScaling client = null; try { List asgNames = new ArrayList<>(); - asgNames.add(instanceIdentity.getInstanceInfo().getAutoScalingGroup()); + asgNames.add(instanceInfo.getAutoScalingGroup()); asgNames.addAll(Arrays.asList(config.getSiblingASGNames().split("\\s*,\\s*"))); client = getAutoScalingClient(); DescribeAutoScalingGroupsRequest asgReq = @@ -86,7 +85,7 @@ public List getRacMembership() { logger.info( String.format( "Querying Amazon returned following instance in the RAC: %s, ASGs: %s --> %s", - instanceIdentity.getInstanceInfo().getRac(), + instanceInfo.getRac(), StringUtils.join(asgNames, ","), StringUtils.join(instanceIds, ","))); } @@ -104,8 +103,7 @@ public int getRacMembershipSize() { client = getAutoScalingClient(); DescribeAutoScalingGroupsRequest asgReq = new DescribeAutoScalingGroupsRequest() - .withAutoScalingGroupNames( - instanceIdentity.getInstanceInfo().getAutoScalingGroup()); + .withAutoScalingGroupNames(instanceInfo.getAutoScalingGroup()); DescribeAutoScalingGroupsResult res = client.describeAutoScalingGroups(asgReq); int size = 0; for (AutoScalingGroup asg : res.getAutoScalingGroups()) { @@ -123,7 +121,7 @@ public List getCrossAccountRacMembership() { AmazonAutoScaling client = null; try { List asgNames = new ArrayList<>(); - asgNames.add(instanceIdentity.getInstanceInfo().getAutoScalingGroup()); + asgNames.add(instanceInfo.getAutoScalingGroup()); asgNames.addAll(Arrays.asList(config.getSiblingASGNames().split("\\s*,\\s*"))); client = getCrossAccountAutoScalingClient(); DescribeAutoScalingGroupsRequest asgReq = @@ -144,8 +142,7 @@ public List getCrossAccountRacMembership() { logger.info( String.format( "Querying Amazon returned following instance in the cross-account ASG: %s --> %s", - instanceIdentity.getInstanceInfo().getRac(), - StringUtils.join(instanceIds, ","))); + instanceInfo.getRac(), StringUtils.join(instanceIds, ","))); } return instanceIds; } finally { @@ -159,8 +156,7 @@ public int getRacCount() { } private boolean isClassic() { - return instanceIdentity.getInstanceInfo().getInstanceEnvironment() - == InstanceInfo.InstanceEnvironment.CLASSIC; + return instanceInfo.getInstanceEnvironment() == InstanceInfo.InstanceEnvironment.CLASSIC; } /** @@ -214,10 +210,7 @@ protected String getVpcGoupId() { client = getEc2Client(); Filter nameFilter = new Filter().withName("group-name").withValues(config.getACLGroupName()); // SG - Filter vpcFilter = - new Filter() - .withName("vpc-id") - .withValues(instanceIdentity.getInstanceInfo().getVpcId()); + Filter vpcFilter = new Filter().withName("vpc-id").withValues(instanceInfo.getVpcId()); DescribeSecurityGroupsRequest req = new DescribeSecurityGroupsRequest().withFilters(nameFilter, vpcFilter); @@ -227,13 +220,13 @@ protected String getVpcGoupId() { "got group-id:{} for group-name:{},vpc-id:{}", group.getGroupId(), config.getACLGroupName(), - instanceIdentity.getInstanceInfo().getVpcId()); + instanceInfo.getVpcId()); return group.getGroupId(); } logger.error( "unable to get group-id for group-name={} vpc-id={}", config.getACLGroupName(), - instanceIdentity.getInstanceInfo().getVpcId()); + instanceInfo.getVpcId()); return ""; } finally { if (client != null) client.shutdown(); @@ -304,7 +297,7 @@ public List listACL(int from, int to) { Filter nameFilter = new Filter().withName("group-name").withValues(config.getACLGroupName()); - String vpcid = instanceIdentity.getInstanceInfo().getVpcId(); + String vpcid = instanceInfo.getVpcId(); if (vpcid == null || vpcid.isEmpty()) { throw new IllegalStateException( "vpcid is null even though instance is running in vpc."); @@ -336,8 +329,7 @@ public void expandRacMembership(int count) { client = getAutoScalingClient(); DescribeAutoScalingGroupsRequest asgReq = new DescribeAutoScalingGroupsRequest() - .withAutoScalingGroupNames( - instanceIdentity.getInstanceInfo().getAutoScalingGroup()); + .withAutoScalingGroupNames(instanceInfo.getAutoScalingGroup()); DescribeAutoScalingGroupsResult res = client.describeAutoScalingGroups(asgReq); AutoScalingGroup asg = res.getAutoScalingGroups().get(0); UpdateAutoScalingGroupRequest ureq = new UpdateAutoScalingGroupRequest(); @@ -354,21 +346,21 @@ public void expandRacMembership(int count) { protected AmazonAutoScaling getAutoScalingClient() { return AmazonAutoScalingClientBuilder.standard() .withCredentials(provider.getAwsCredentialProvider()) - .withRegion(instanceIdentity.getInstanceInfo().getRegion()) + .withRegion(instanceInfo.getRegion()) .build(); } protected AmazonAutoScaling getCrossAccountAutoScalingClient() { return AmazonAutoScalingClientBuilder.standard() .withCredentials(crossAccountProvider.getAwsCredentialProvider()) - .withRegion(instanceIdentity.getInstanceInfo().getRegion()) + .withRegion(instanceInfo.getRegion()) .build(); } protected AmazonEC2 getEc2Client() { return AmazonEC2ClientBuilder.standard() .withCredentials(provider.getAwsCredentialProvider()) - .withRegion(instanceIdentity.getInstanceInfo().getRegion()) + .withRegion(instanceInfo.getRegion()) .build(); } } diff --git a/priam/src/main/java/com/netflix/priam/aws/auth/EC2RoleAssumptionCredential.java b/priam/src/main/java/com/netflix/priam/aws/auth/EC2RoleAssumptionCredential.java index 45131f622..c01c92442 100644 --- a/priam/src/main/java/com/netflix/priam/aws/auth/EC2RoleAssumptionCredential.java +++ b/priam/src/main/java/com/netflix/priam/aws/auth/EC2RoleAssumptionCredential.java @@ -18,22 +18,21 @@ import com.google.inject.Inject; import com.netflix.priam.config.IConfiguration; import com.netflix.priam.cred.ICredential; -import com.netflix.priam.identity.InstanceIdentity; import com.netflix.priam.identity.config.InstanceInfo; public class EC2RoleAssumptionCredential implements ICredential { private static final String AWS_ROLE_ASSUMPTION_SESSION_NAME = "AwsRoleAssumptionSession"; private final ICredential cred; private final IConfiguration config; - private final InstanceIdentity instanceIdentity; + private final InstanceInfo instanceInfo; private AWSCredentialsProvider stsSessionCredentialsProvider; @Inject public EC2RoleAssumptionCredential( - ICredential cred, IConfiguration config, InstanceIdentity instanceIdentity) { + ICredential cred, IConfiguration config, InstanceInfo instanceInfo) { this.cred = cred; this.config = config; - this.instanceIdentity = instanceIdentity; + this.instanceInfo = instanceInfo; } @Override @@ -48,7 +47,7 @@ public AWSCredentialsProvider getAwsCredentialProvider() { * current environment is VPC, then the assumed role is for EC2 classic, and * vice versa. */ - if (instanceIdentity.getInstanceInfo().getInstanceEnvironment() + if (instanceInfo.getInstanceEnvironment() == InstanceInfo.InstanceEnvironment.CLASSIC) { roleArn = this.config.getClassicEC2RoleAssumptionArn(); // Env is EC2 classic --> IAM assumed role for VPC created From 34e822372072f436a89a8313a989f33d338c8948 Mon Sep 17 00:00:00 2001 From: Arun Agrawal Date: Fri, 26 Oct 2018 09:44:39 -0700 Subject: [PATCH 029/228] Release notes for 3.11.36 --- CHANGELOG.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7993a534b..6e0ddc867 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,8 @@ # Changelog -## 2018/10/17 3.11.34 +## 2018/10/26 3.11.36 +* (#747) Aggregate InstanceData in InstanceInfo and pull information about running instance + +## 2018/10/17 3.11.35 * (#739) BugFix: Null pointer exception while traversing filesystem. * (#737) Google java format validator addition. Use ./gradlew goJF to fix the formatting before sending PR. * (#740) Last but not least, a new logo for Priam. From c94f7e2661e6345db8e1f10a974fec7d831f310c Mon Sep 17 00:00:00 2001 From: Arun Agrawal Date: Fri, 26 Oct 2018 16:34:04 -0700 Subject: [PATCH 030/228] Add "schema.cql" file to known list of files. --- .../priam/services/SnapshotMetaService.java | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/priam/src/main/java/com/netflix/priam/services/SnapshotMetaService.java b/priam/src/main/java/com/netflix/priam/services/SnapshotMetaService.java index 8b71b3f0a..030444275 100644 --- a/priam/src/main/java/com/netflix/priam/services/SnapshotMetaService.java +++ b/priam/src/main/java/com/netflix/priam/services/SnapshotMetaService.java @@ -61,6 +61,7 @@ public class SnapshotMetaService extends AbstractBackup { private static final Logger logger = LoggerFactory.getLogger(SnapshotMetaService.class); private static final String SNAPSHOT_PREFIX = "snap_v2_"; private static final String CASSANDRA_MANIFEST_FILE = "manifest.json"; + private static final String CASSANDRA_SCHEMA_FILE = "schema.cql"; private final BackupRestoreUtil backupRestoreUtil; private final MetaFileWriterBuilder metaFileWriter; private MetaFileWriterBuilder.DataStep dataStep; @@ -164,6 +165,14 @@ public void execute() throws Exception { } catch (Exception e) { logger.error("Error while executing SnapshotMetaService", e); } finally { + // Make sure you clean up snapshots whatever be the case. + try { + cassandraOperations.clearSnapshot(snapshotName); + } catch (Exception e) { + logger.error(e.getMessage(), e); + } + + // Release the lock. lock.unlock(); } } @@ -229,11 +238,14 @@ protected void processColumnFamily( if (prefix == null && file.getName().equalsIgnoreCase(CASSANDRA_MANIFEST_FILE)) prefix = "manifest"; + if (prefix == null && file.getName().equalsIgnoreCase(CASSANDRA_SCHEMA_FILE)) + prefix = "schema"; + if (prefix == null) { logger.error( - "Unknown file type with no SSTFileBase found: ", + "Unknown file type with no SSTFileBase found: {}", file.getAbsolutePath()); - return; + continue; } FileUploadResult fileUploadResult = From 0c3970b9a275570b43af91af02c3b7034fa6b8cb Mon Sep 17 00:00:00 2001 From: Arun Agrawal Date: Fri, 26 Oct 2018 22:23:13 -0700 Subject: [PATCH 031/228] Add compression test --- .../netflix/priam/backup/TestCompression.java | 183 +++++++----------- 1 file changed, 72 insertions(+), 111 deletions(-) diff --git a/priam/src/test/java/com/netflix/priam/backup/TestCompression.java b/priam/src/test/java/com/netflix/priam/backup/TestCompression.java index eee1b1b03..38c42398c 100644 --- a/priam/src/test/java/com/netflix/priam/backup/TestCompression.java +++ b/priam/src/test/java/com/netflix/priam/backup/TestCompression.java @@ -28,21 +28,19 @@ import java.util.zip.ZipEntry; import java.util.zip.ZipFile; import java.util.zip.ZipOutputStream; +import org.apache.commons.io.FileUtils; import org.junit.After; import org.junit.Before; -import org.junit.Ignore; import org.junit.Test; -import org.xerial.snappy.SnappyInputStream; -import org.xerial.snappy.SnappyOutputStream; -@Ignore("does this test really have to generate smoke to verify correct behavior?") public class TestCompression { + private final File randomContentFile = new File("/tmp/content.txt"); + @Before public void setup() throws IOException { - File f = new File("/tmp/compress-test.txt"); - try (FileOutputStream stream = new FileOutputStream(f)) { - for (int i = 0; i < (1000 * 1000); i++) { + try (FileOutputStream stream = new FileOutputStream(randomContentFile)) { + for (int i = 0; i < (5 * 5); i++) { stream.write( "This is a test... Random things happen... and you are responsible for it...\n" .getBytes("UTF-8")); @@ -55,124 +53,87 @@ public void setup() throws IOException { @After public void done() { - File f = new File("/tmp/compress-test.txt"); - if (f.exists()) f.delete(); - } - - private void validateCompression(String compress) { - File uncompressed = new File("/tmp/compress-test.txt"); - File compressed = new File(compress); - assertTrue(uncompressed.length() > compressed.length()); - } - - @Test - public void zip() throws IOException { - BufferedInputStream source; - try (ZipOutputStream out = - new ZipOutputStream( - new BufferedOutputStream(new FileOutputStream("/tmp/compressed.zip")))) { - byte data[] = new byte[2048]; - File file = new File("/tmp/compress-test.txt"); - FileInputStream fi = new FileInputStream(file); - source = new BufferedInputStream(fi, 2048); - ZipEntry entry = new ZipEntry(file.getName()); - out.putNextEntry(entry); - int count; - while ((count = source.read(data, 0, 2048)) != -1) { - out.write(data, 0, count); - } - } - validateCompression("/tmp/compressed.zip"); + FileUtils.deleteQuietly(randomContentFile); } @Test - public void unzip() throws IOException { - ZipFile zipfile = new ZipFile("/tmp/compressed.zip"); - Enumeration e = zipfile.entries(); - while (e.hasMoreElements()) { - ZipEntry entry = (ZipEntry) e.nextElement(); - try (BufferedInputStream is = new BufferedInputStream(zipfile.getInputStream(entry)); - BufferedOutputStream dest1 = - new BufferedOutputStream( - new FileOutputStream("/tmp/compress-test-out-0.txt"), 2048)) { - int c; - byte d[] = new byte[2048]; - - while ((c = is.read(d, 0, 2048)) != -1) { - dest1.write(d, 0, c); + public void zipTest() throws IOException { + String zipFileName = "/tmp/compressed.zip"; + File decompressedTempOutput = new File("/tmp/compress-test-out.txt"); + + try { + try (ZipOutputStream out = + new ZipOutputStream( + new BufferedOutputStream(new FileOutputStream(zipFileName))); + BufferedInputStream source = + new BufferedInputStream( + new FileInputStream(randomContentFile), 2048); ) { + byte data[] = new byte[2048]; + ZipEntry entry = new ZipEntry(randomContentFile.getName()); + out.putNextEntry(entry); + int count; + while ((count = source.read(data, 0, 2048)) != -1) { + out.write(data, 0, count); } } - } - String md1 = SystemUtils.md5(new File("/tmp/compress-test.txt")); - String md2 = SystemUtils.md5(new File("/tmp/compress-test-out-0.txt")); - assertEquals(md1, md2); - } - - @Test - public void snappyCompress() throws IOException { - try (BufferedInputStream origin = - new BufferedInputStream( - new FileInputStream("/tmp/compress-test.txt"), 1024); - SnappyOutputStream out = - new SnappyOutputStream( - new BufferedOutputStream(new FileOutputStream("/tmp/test0.snp")))) { - byte data[] = new byte[1024]; - int count; - while ((count = origin.read(data, 0, 1024)) != -1) { - out.write(data, 0, count); - } - validateCompression("/tmp/test0.snp"); - } - } - - @Test - public void snappyDecompress() throws IOException { - // decompress normally. - try (SnappyInputStream is = - new SnappyInputStream( - new BufferedInputStream(new FileInputStream("/tmp/test0.snp"))); - BufferedOutputStream dest1 = - new BufferedOutputStream( - new FileOutputStream("/tmp/compress-test-out-1.txt"), 1024)) { - - byte d[] = new byte[1024]; - int c; - while ((c = is.read(d, 0, 1024)) != -1) { - dest1.write(d, 0, c); + assertTrue(randomContentFile.length() > new File(zipFileName).length()); + + ZipFile zipfile = new ZipFile(zipFileName); + Enumeration e = zipfile.entries(); + while (e.hasMoreElements()) { + ZipEntry entry = (ZipEntry) e.nextElement(); + try (BufferedInputStream is = + new BufferedInputStream(zipfile.getInputStream(entry)); + BufferedOutputStream dest1 = + new BufferedOutputStream( + new FileOutputStream(decompressedTempOutput), 2048)) { + int c; + byte d[] = new byte[2048]; + + while ((c = is.read(d, 0, 2048)) != -1) { + dest1.write(d, 0, c); + } + } } - - String md1 = SystemUtils.md5(new File("/tmp/compress-test-out-1.txt")); - String md2 = SystemUtils.md5(new File("/tmp/compress-test.txt")); + String md1 = SystemUtils.md5(randomContentFile); + String md2 = SystemUtils.md5(decompressedTempOutput); assertEquals(md1, md2); + } finally { + FileUtils.deleteQuietly(new File(zipFileName)); + FileUtils.deleteQuietly(decompressedTempOutput); } } @Test - public void compress() throws IOException { + public void snappyTest() throws IOException { SnappyCompression compress = new SnappyCompression(); - File file = new File(new File("/tmp/compress-test.txt"), "r"); + File compressedOutputFile = new File("/tmp/test1.snp"); + File decompressedTempOutput = new File("/tmp/compress-test-out.txt"); long chunkSize = 5L * 1024 * 1024; - Iterator it = - compress.compress( - new AbstractBackupPath.RafInputStream(new RandomAccessFile(file, "r")), - chunkSize); - try (FileOutputStream ostream = new FileOutputStream("/tmp/test1.snp")) { - while (it.hasNext()) { - byte[] chunk = it.next(); - ostream.write(chunk); + try { + Iterator it = + compress.compress( + new AbstractBackupPath.RafInputStream( + new RandomAccessFile(randomContentFile, "r")), + chunkSize); + try (FileOutputStream ostream = new FileOutputStream(compressedOutputFile)) { + while (it.hasNext()) { + byte[] chunk = it.next(); + ostream.write(chunk); + } } - } - validateCompression("/tmp/test1.snp"); - } - @Test - public void decompress() throws IOException { - SnappyCompression compress = new SnappyCompression(); - compress.decompressAndClose( - new FileInputStream("/tmp/test1.snp"), - new FileOutputStream("/tmp/compress-test-out-2.txt")); - String md1 = SystemUtils.md5(new File("/tmp/compress-test.txt")); - String md2 = SystemUtils.md5(new File("/tmp/compress-test-out-2.txt")); - assertEquals(md1, md2); + assertTrue(randomContentFile.length() > compressedOutputFile.length()); + + compress.decompressAndClose( + new FileInputStream(compressedOutputFile), + new FileOutputStream(decompressedTempOutput)); + String md1 = SystemUtils.md5(randomContentFile); + String md2 = SystemUtils.md5(decompressedTempOutput); + assertEquals(md1, md2); + } finally { + FileUtils.deleteQuietly(compressedOutputFile); + FileUtils.deleteQuietly(decompressedTempOutput); + } } } From 1b3575d92518f6e082f51bbdc6e0460405274ce3 Mon Sep 17 00:00:00 2001 From: Arun Agrawal Date: Fri, 26 Oct 2018 23:03:20 -0700 Subject: [PATCH 032/228] Defaults in IConfiguration and move creation of directories to PriamServer. --- .../java/com/netflix/priam/PriamServer.java | 14 + .../netflix/priam/config/IConfiguration.java | 289 +++++++++++++---- .../priam/config/PriamConfiguration.java | 36 +-- .../netflix/priam/tuner/JVMOptionsTuner.java | 6 +- .../netflix/priam/tuner/StandardTuner.java | 13 +- .../priam/config/FakeConfiguration.java | 297 +----------------- .../priam/tuner/JVMOptionTunerTest.java | 8 +- 7 files changed, 270 insertions(+), 393 deletions(-) diff --git a/priam/src/main/java/com/netflix/priam/PriamServer.java b/priam/src/main/java/com/netflix/priam/PriamServer.java index d9b598d7b..650ef5e7f 100644 --- a/priam/src/main/java/com/netflix/priam/PriamServer.java +++ b/priam/src/main/java/com/netflix/priam/PriamServer.java @@ -38,6 +38,7 @@ import com.netflix.priam.tuner.TuneCassandra; import com.netflix.priam.utils.CassandraMonitor; import com.netflix.priam.utils.Sleeper; +import com.netflix.priam.utils.SystemUtils; import org.apache.commons.collections4.CollectionUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -73,7 +74,20 @@ public PriamServer( this.restoreContext = restoreContext; } + private void createDirectories() { + SystemUtils.createDirs(config.getBackupCommitLogLocation()); + SystemUtils.createDirs(config.getCommitLogLocation()); + SystemUtils.createDirs(config.getCacheLocation()); + SystemUtils.createDirs(config.getDataFileLocation()); + SystemUtils.createDirs(config.getLogDirLocation()); + SystemUtils.createDirs(config.getHintsLocation()); + } + public void initialize() throws Exception { + // Create all the required directories for priam and Cassandra. + createDirectories(); + + // Do not start Priam if you are out of service. if (instanceIdentity.getInstance().isOutOfService()) return; // start to schedule jobs diff --git a/priam/src/main/java/com/netflix/priam/config/IConfiguration.java b/priam/src/main/java/com/netflix/priam/config/IConfiguration.java index 3f38a7610..d473ac3c5 100644 --- a/priam/src/main/java/com/netflix/priam/config/IConfiguration.java +++ b/priam/src/main/java/com/netflix/priam/config/IConfiguration.java @@ -22,8 +22,8 @@ import com.netflix.priam.scheduler.SchedulerType; import com.netflix.priam.scheduler.UnsupportedTypeException; import com.netflix.priam.tuner.GCType; -import com.netflix.priam.tuner.JVMOption; import java.io.File; +import java.util.Collections; import java.util.List; import java.util.Map; import org.apache.commons.lang3.StringUtils; @@ -35,30 +35,47 @@ public interface IConfiguration { void initialize(); /** @return Path to the home dir of Cassandra */ - String getCassHome(); + default String getCassHome() { + return "/etc/cassandra"; + } - String getYamlLocation(); + /** @return Location to `cassandra.yaml`. */ + default String getYamlLocation() { + return getCassHome() + "/conf/cassandra.yaml"; + } /** @return Path to jvm.options file. This is used to pass JVM options to Cassandra. */ - String getJVMOptionsFileLocation(); + default String getJVMOptionsFileLocation() { + return getCassHome() + "/conf/jvm.options"; + } /** * @return Type of garbage collection mechanism to use for Cassandra. Supported values are * CMS,G1GC */ - GCType getGCType() throws UnsupportedTypeException; + default GCType getGCType() throws UnsupportedTypeException { + return GCType.CMS; + } /** @return Set of JVM options to exclude/comment. */ - Map getJVMExcludeSet(); + default String getJVMExcludeSet() { + return StringUtils.EMPTY; + } /** @return Set of JMV options to add/upsert */ - Map getJVMUpsertSet(); + default String getJVMUpsertSet() { + return StringUtils.EMPTY; + } /** @return Path to Cassandra startup script */ - String getCassStartupScript(); + default String getCassStartupScript() { + return "/etc/init.d/cassandra start"; + } /** @return Path to Cassandra stop sript */ - String getCassStopScript(); + default String getCassStopScript() { + return "/etc/init.d/cassandra stop"; + } /** * @return int representing how many seconds Priam should fail healthchecks for before @@ -88,29 +105,56 @@ default int getRemediateDeadCassandraRate() { * * @return Prefix that will be added to remote backup location */ - String getBackupLocation(); + default String getBackupLocation() { + return "backup"; + } /** @return Get Backup retention in days */ - int getBackupRetentionDays(); + default int getBackupRetentionDays() { + return 0; + } /** @return Get list of racs to backup. Backup all racs if empty */ - List getBackupRacs(); + default List getBackupRacs() { + return Collections.EMPTY_LIST; + } /** - * Bucket name in case of AWS + * Backup location i.e. remote file system to upload backups. e.g. for S3 it will be s3 bucket + * name * * @return Bucket name used for backups */ - String getBackupPrefix(); + default String getBackupPrefix() { + return "cassandra-archive"; + } /** * @return Location containing backup files. Typically bucket name followed by path to the * clusters backup */ - String getRestorePrefix(); + default String getRestorePrefix() { + return StringUtils.EMPTY; + } + + /** + * This is the location of the data/logs/hints for the cassandra. Priam will by default, create + * all the sub-directories required. This dir should have permission to be altered by both + * cassandra and Priam. If this is configured correctly, there is no need to configure {@link + * #getDataFileLocation()}, {@link #getLogDirLocation()}, {@link #getCacheLocation()} and {@link + * #getCommitLogLocation()}. Alternatively all the other directories should be set explicitly by + * user. Set this location to a drive with fast read/writes performance and sizable disk space. + * + * @return Location where all the data/logs/hints for the cassandra will sit. + */ + default String getCassandraBaseDirectory() { + return "/var/lib/cassandra"; + } /** @return Location of the local data dir */ - String getDataFileLocation(); + default String getDataFileLocation() { + return getCassandraBaseDirectory() + "/data"; + } /** * Path where cassandra logs should be stored. This is passed to Cassandra as where to store @@ -118,21 +162,34 @@ default int getRemediateDeadCassandraRate() { * * @return Path to cassandra logs. */ - String getLogDirLocation(); + default String getLogDirLocation() { + return getCassandraBaseDirectory() + "/logs"; + } /** @return Location of the hints data directory */ - String getHintsLocation(); + default String getHintsLocation() { + return getCassandraBaseDirectory() + "/hints"; + } + /** @return Location of local cache */ - String getCacheLocation(); + default String getCacheLocation() { + return getCassandraBaseDirectory() + "/saved_caches"; + } /** @return Location of local commit log dir */ - String getCommitLogLocation(); + default String getCommitLogLocation() { + return getCassandraBaseDirectory() + "/commitlog"; + } /** @return Remote commit log location for backups */ - String getBackupCommitLogLocation(); + default String getBackupCommitLogLocation() { + return StringUtils.EMPTY; + } /** @return Preferred data part size for multi part uploads */ - long getBackupChunkSize(); + default long getBackupChunkSize() { + return 10 * 1024 * 1024L; + } /** @return Cassandra's JMX port */ default int getJmxPort() { @@ -174,19 +231,27 @@ default int getNativeTransportPort() { } /** @return Snitch to be used in cassandra.yaml */ - String getSnitch(); + default String getSnitch() { + return "org.apache.cassandra.locator.Ec2Snitch"; + } /** @return Cluster name */ - String getAppName(); + default String getAppName() { + return "cass_cluster"; + } /** @return List of all RAC used for the cluster */ List getRacs(); /** @return Max heap size be used for Cassandra */ - String getHeapSize(); + default String getHeapSize() { + return "8G"; + } /** @return New heap size for Cassandra */ - String getHeapNewSize(); + default String getHeapNewSize() { + return "2G"; + } /** * Cron expression to be used to schedule regular compactions. Use "-1" to disable the CRON. @@ -362,10 +427,14 @@ default String getRestoreExcludeCFList() { * * @return Snapshot to be searched and restored */ - String getRestoreSnapshot(); + default String getRestoreSnapshot() { + return StringUtils.EMPTY; + } /** @return Get the region to connect to SDB for instance identity */ - String getSDBInstanceIdentityRegion(); + default String getSDBInstanceIdentityRegion() { + return "us-east-1"; + } /** @return true if it is a multi regional cluster */ default boolean isMultiDC() { @@ -391,10 +460,14 @@ default boolean isRestoreClosestToken() { * Amazon specific setting to query Additional/ Sibling ASG Memberships in csv format to * consider while calculating RAC membership */ - String getSiblingASGNames(); + default String getSiblingASGNames() { + return ","; + } /** Get the security group associated with nodes in this cluster */ - String getACLGroupName(); + default String getACLGroupName() { + return getAppName(); + } /** @return true if incremental backups are enabled */ default boolean isIncrementalBackupEnabled() { @@ -407,7 +480,9 @@ default int getUploadThrottle() { } /** @return true if Priam should local config file for tokens and seeds */ - boolean isLocalBootstrapEnabled(); + default boolean isLocalBootstrapEnabled() { + return false; + } /** @return Compaction throughput */ default int getCompactionThroughput() { @@ -430,10 +505,14 @@ default String getMaxDirectMemory() { } /** @return Bootstrap cluster name (depends on another cass cluster) */ - String getBootClusterName(); + default String getBootClusterName() { + return StringUtils.EMPTY; + } /** @return Get the name of seed provider */ - String getSeedProviderName(); + default String getSeedProviderName() { + return "com.netflix.priam.cassandra.extensions.NFSeedProvider"; + } /** * memtable_cleanup_threshold defaults to 1 / (memtable_flush_writers + 1) = 0.11 @@ -454,22 +533,34 @@ default int getStreamingThroughputMB() { * * @return the fully-qualified name of the partitioner class */ - String getPartitioner(); + default String getPartitioner() { + return "org.apache.cassandra.dht.RandomPartitioner"; + } /** Support for c* 1.1 global key cache size */ - String getKeyCacheSizeInMB(); + default String getKeyCacheSizeInMB() { + return StringUtils.EMPTY; + } /** Support for limiting the total number of keys in c* 1.1 global key cache. */ - String getKeyCacheKeysToSave(); + default String getKeyCacheKeysToSave() { + return StringUtils.EMPTY; + } /** Support for c* 1.1 global row cache size */ - String getRowCacheSizeInMB(); + default String getRowCacheSizeInMB() { + return StringUtils.EMPTY; + } /** Support for limiting the total number of rows in c* 1.1 global row cache. */ - String getRowCacheKeysToSave(); + default String getRowCacheKeysToSave() { + return StringUtils.EMPTY; + } /** @return C* Process Name */ - String getCassProcessName(); + default String getCassProcessName() { + return "CassandraDaemon"; + } /** Defaults to 'allow all'. */ default String getAuthenticator() { @@ -482,7 +573,9 @@ default String getAuthorizer() { } /** @return true/false, if Cassandra needs to be started manually */ - boolean doesCassandraStartManually(); + default boolean doesCassandraStartManually() { + return false; + } /** @return possible values: all, dc, none */ default String getInternodeCompression() { @@ -499,33 +592,61 @@ default boolean isBackingUpCommitLogs() { return false; } - String getCommitLogBackupPropsFile(); + default String getCommitLogBackupPropsFile() { + return getCassHome() + "/conf/commitlog_archiving.properties"; + } - String getCommitLogBackupArchiveCmd(); + default String getCommitLogBackupArchiveCmd() { + return "/bin/ln %path /mnt/data/backup/%name"; + } - String getCommitLogBackupRestoreCmd(); + default String getCommitLogBackupRestoreCmd() { + return "/bin/mv %from %to"; + } - String getCommitLogBackupRestoreFromDirs(); + default String getCommitLogBackupRestoreFromDirs() { + return "/mnt/data/backup/commitlog/"; + } - String getCommitLogBackupRestorePointInTime(); + default String getCommitLogBackupRestorePointInTime() { + return StringUtils.EMPTY; + } - int maxCommitLogsRestore(); + default int maxCommitLogsRestore() { + return 10; + } - boolean isClientSslEnabled(); + default boolean isClientSslEnabled() { + return false; + } - String getInternodeEncryption(); + default String getInternodeEncryption() { + return "none"; + } - boolean isDynamicSnitchEnabled(); + default boolean isDynamicSnitchEnabled() { + return true; + } - boolean isThriftEnabled(); + default boolean isThriftEnabled() { + return true; + } - boolean isNativeTransportEnabled(); + default boolean isNativeTransportEnabled() { + return true; + } - int getConcurrentReadsCnt(); + default int getConcurrentReadsCnt() { + return 32; + } - int getConcurrentWritesCnt(); + default int getConcurrentWritesCnt() { + return 32; + } - int getConcurrentCompactorsCnt(); + default int getConcurrentCompactorsCnt() { + return Runtime.getRuntime().availableProcessors(); + } default String getRpcServerType() { return "hsha"; @@ -543,20 +664,30 @@ default int getRpcMaxThreads() { * @return the warning threshold in MB's for large partitions encountered during compaction. * Default value of 100 is used (default from cassandra.yaml) */ - int getCompactionLargePartitionWarnThresholdInMB(); + default int getCompactionLargePartitionWarnThresholdInMB() { + return 100; + } - String getExtraConfigParams(); + default String getExtraConfigParams() { + return StringUtils.EMPTY; + } String getCassYamlVal(String priamKey); - boolean getAutoBoostrap(); + default boolean getAutoBoostrap() { + return true; + } - boolean isCreateNewTokenEnable(); + default boolean isCreateNewTokenEnable() { + return true; + } /* * @return the location on disk of the private key used by the cryptography algorithm */ - String getPrivateKeyLocation(); + default String getPrivateKeyLocation() { + return StringUtils.EMPTY; + } /** * @return the type of source for the restore. Valid values are: AWSCROSSACCT or GOOGLE. Note: @@ -567,7 +698,9 @@ default int getRpcMaxThreads() { * a different account. *

    GOOGLE - You are restoring from Google Cloud Storage */ - String getRestoreSourceType(); + default String getRestoreSourceType() { + return StringUtils.EMPTY; + } /** * Should backups be encrypted. If this is on, then all the files uploaded will be compressed @@ -597,14 +730,18 @@ default boolean isRestoreEncrypted() { * should be optional. Specifically, if it does not exist, it should not cause an adverse * impact on current functionality. */ - String getAWSRoleAssumptionArn(); + default String getAWSRoleAssumptionArn() { + return StringUtils.EMPTY; + } /** * @return Google Cloud Storage service account id to be use within the restore functionality. * Note: for backward compatibility, this property should be optional. Specifically, if it * does not exist, it should not cause an adverse impact on current functionality. */ - String getGcsServiceAccountId(); + default String getGcsServiceAccountId() { + return StringUtils.EMPTY; + } /** * @return the absolute path on disk for the Google Cloud Storage PFX file (i.e. the combined @@ -613,7 +750,9 @@ default boolean isRestoreEncrypted() { * optional. Specifically, if it does not exist, it should not cause an adverse impact on * current functionality. */ - String getGcsServiceAccountPrivateKeyLoc(); + default String getGcsServiceAccountPrivateKeyLoc() { + return StringUtils.EMPTY; + } /** * @return the pass phrase use by PGP cryptography. This information is to be use within the @@ -621,7 +760,9 @@ default boolean isRestoreEncrypted() { * compatibility, this property should be optional. Specifically, if it does not exist, it * should not cause an adverse impact on current functionality. */ - String getPgpPasswordPhrase(); + default String getPgpPasswordPhrase() { + return StringUtils.EMPTY; + } /** * @return public key use by PGP cryptography. This information is to be use within the restore @@ -629,24 +770,32 @@ default boolean isRestoreEncrypted() { * this property should be optional. Specifically, if it does not exist, it should not cause * an adverse impact on current functionality. */ - String getPgpPublicKeyLoc(); + default String getPgpPublicKeyLoc() { + return StringUtils.EMPTY; + } /** * Use this method for adding extra/ dynamic cassandra startup options or env properties * * @return A map of extra paramaters. */ - Map getExtraEnvParams(); + default Map getExtraEnvParams() { + return Collections.EMPTY_MAP; + } /* * @return the Amazon Resource Name (ARN) for EC2 classic. */ - String getClassicEC2RoleAssumptionArn(); + default String getClassicEC2RoleAssumptionArn() { + return StringUtils.EMPTY; + } /* * @return the Amazon Resource Name (ARN) for VPC. */ - String getVpcEC2RoleAssumptionArn(); + default String getVpcEC2RoleAssumptionArn() { + return StringUtils.EMPTY; + } /** * Is cassandra cluster spanning more than one account. This may be true if you are migrating @@ -742,7 +891,9 @@ default int getStreamingSocketTimeoutInMS() { * * @return a comma delimited list of keyspaces to flush */ - String getFlushKeyspaces(); + default String getFlushKeyspaces() { + return StringUtils.EMPTY; + } /** * Interval to be used for flush. diff --git a/priam/src/main/java/com/netflix/priam/config/PriamConfiguration.java b/priam/src/main/java/com/netflix/priam/config/PriamConfiguration.java index 89bf5d34e..de956f10f 100644 --- a/priam/src/main/java/com/netflix/priam/config/PriamConfiguration.java +++ b/priam/src/main/java/com/netflix/priam/config/PriamConfiguration.java @@ -31,9 +31,6 @@ import com.netflix.priam.scheduler.SchedulerType; import com.netflix.priam.scheduler.UnsupportedTypeException; import com.netflix.priam.tuner.GCType; -import com.netflix.priam.tuner.JVMOption; -import com.netflix.priam.tuner.JVMOptionsTuner; -import com.netflix.priam.utils.SystemUtils; import java.io.File; import java.util.HashMap; import java.util.List; @@ -45,10 +42,6 @@ @Singleton public class PriamConfiguration implements IConfiguration { public static final String PRIAM_PRE = "priam"; - - private static final String CONFIG_RESTORE_PREFIX = PRIAM_PRE + ".restore.prefix"; - private final String CASS_BASE_DATA_DIR = "/var/lib/cassandra"; - private List DEFAULT_AVAILABILITY_ZONES = ImmutableList.of(); private final IConfigSource config; @@ -69,12 +62,6 @@ public PriamConfiguration( public void initialize() { this.config.initialize(instanceInfo.getAutoScalingGroup(), instanceInfo.getRegion()); setDefaultRACList(instanceInfo.getRegion()); - SystemUtils.createDirs(getBackupCommitLogLocation()); - SystemUtils.createDirs(getCommitLogLocation()); - SystemUtils.createDirs(getCacheLocation()); - SystemUtils.createDirs(getDataFileLocation()); - SystemUtils.createDirs(getHintsLocation()); - SystemUtils.createDirs(getLogDirLocation()); } /** Get the fist 3 available zones in the region */ @@ -141,32 +128,34 @@ public List getBackupRacs() { @Override public String getRestorePrefix() { - return config.get(CONFIG_RESTORE_PREFIX); + return config.get(PRIAM_PRE + ".restore.prefix"); } @Override public String getDataFileLocation() { - return config.get(PRIAM_PRE + ".data.location", CASS_BASE_DATA_DIR + "/data"); + return config.get(PRIAM_PRE + ".data.location", getCassandraBaseDirectory() + "/data"); } @Override public String getLogDirLocation() { - return config.get(PRIAM_PRE + ".logs.location", CASS_BASE_DATA_DIR + "/logs"); + return config.get(PRIAM_PRE + ".logs.location", getCassandraBaseDirectory() + "/logs"); } @Override public String getHintsLocation() { - return config.get(PRIAM_PRE + ".hints.location", CASS_BASE_DATA_DIR + "/hints"); + return config.get(PRIAM_PRE + ".hints.location", getCassandraBaseDirectory() + "/hints"); } @Override public String getCacheLocation() { - return config.get(PRIAM_PRE + ".cache.location", CASS_BASE_DATA_DIR + "/saved_caches"); + return config.get( + PRIAM_PRE + ".cache.location", getCassandraBaseDirectory() + "/saved_caches"); } @Override public String getCommitLogLocation() { - return config.get(PRIAM_PRE + ".commitlog.location", CASS_BASE_DATA_DIR + "/commitlog"); + return config.get( + PRIAM_PRE + ".commitlog.location", getCassandraBaseDirectory() + "/commitlog"); } @Override @@ -277,13 +266,13 @@ public GCType getGCType() throws UnsupportedTypeException { } @Override - public Map getJVMExcludeSet() { - return JVMOptionsTuner.parseJVMOptions(config.get(PRIAM_PRE + ".jvm.options.exclude")); + public String getJVMExcludeSet() { + return config.get(PRIAM_PRE + ".jvm.options.exclude"); } @Override - public Map getJVMUpsertSet() { - return JVMOptionsTuner.parseJVMOptions(config.get(PRIAM_PRE + ".jvm.options.upsert")); + public String getJVMUpsertSet() { + return config.get(PRIAM_PRE + ".jvm.options.upsert"); } @Override @@ -584,6 +573,7 @@ public String getExtraConfigParams() { return config.get(PRIAM_PRE + ".extra.params"); } + @Override public Map getExtraEnvParams() { String envParams = config.get(PRIAM_PRE + ".extra.env.params"); diff --git a/priam/src/main/java/com/netflix/priam/tuner/JVMOptionsTuner.java b/priam/src/main/java/com/netflix/priam/tuner/JVMOptionsTuner.java index cb9e34e76..9f771ad57 100644 --- a/priam/src/main/java/com/netflix/priam/tuner/JVMOptionsTuner.java +++ b/priam/src/main/java/com/netflix/priam/tuner/JVMOptionsTuner.java @@ -85,10 +85,12 @@ protected List updateJVMOptions() throws Exception { validate(jvmOptionsFile); final GCType configuredGC = config.getGCType(); - final Map excludeSet = config.getJVMExcludeSet(); + final Map excludeSet = + JVMOptionsTuner.parseJVMOptions(config.getJVMExcludeSet()); // Make a copy of upsertSet, so we can delete the entries as we process them. - Map upsertSet = config.getJVMUpsertSet(); + Map upsertSet = + JVMOptionsTuner.parseJVMOptions(config.getJVMUpsertSet()); // Don't use streams for processing as upsertSet jvm options needs to be removed if we find // them diff --git a/priam/src/main/java/com/netflix/priam/tuner/StandardTuner.java b/priam/src/main/java/com/netflix/priam/tuner/StandardTuner.java index f48b82002..edeb7d239 100644 --- a/priam/src/main/java/com/netflix/priam/tuner/StandardTuner.java +++ b/priam/src/main/java/com/netflix/priam/tuner/StandardTuner.java @@ -24,6 +24,7 @@ import java.util.Map; import java.util.Properties; import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.yaml.snakeyaml.DumperOptions; @@ -156,19 +157,21 @@ protected String getSnitch() { /** Setup the cassandra 1.1 global cache values */ private void configureGlobalCaches(IConfiguration config, Map yaml) { final String keyCacheSize = config.getKeyCacheSizeInMB(); - if (keyCacheSize != null) { + if (!StringUtils.isEmpty(keyCacheSize)) { yaml.put("key_cache_size_in_mb", Integer.valueOf(keyCacheSize)); final String keyCount = config.getKeyCacheKeysToSave(); - if (keyCount != null) yaml.put("key_cache_keys_to_save", Integer.valueOf(keyCount)); + if (!StringUtils.isEmpty(keyCount)) + yaml.put("key_cache_keys_to_save", Integer.valueOf(keyCount)); } final String rowCacheSize = config.getRowCacheSizeInMB(); - if (rowCacheSize != null) { + if (!StringUtils.isEmpty(rowCacheSize)) { yaml.put("row_cache_size_in_mb", Integer.valueOf(rowCacheSize)); final String rowCount = config.getRowCacheKeysToSave(); - if (rowCount != null) yaml.put("row_cache_keys_to_save", Integer.valueOf(rowCount)); + if (!StringUtils.isEmpty(rowCount)) + yaml.put("row_cache_keys_to_save", Integer.valueOf(rowCount)); } } @@ -230,7 +233,7 @@ public final void updateJVMOptions() throws Exception { public void addExtraCassParams(Map map) { String params = config.getExtraConfigParams(); - if (params == null) { + if (StringUtils.isEmpty(params)) { logger.info("Updating yaml: no extra cass params"); return; } diff --git a/priam/src/test/java/com/netflix/priam/config/FakeConfiguration.java b/priam/src/test/java/com/netflix/priam/config/FakeConfiguration.java index b9288baf4..3b34b2e8b 100644 --- a/priam/src/test/java/com/netflix/priam/config/FakeConfiguration.java +++ b/priam/src/test/java/com/netflix/priam/config/FakeConfiguration.java @@ -17,11 +17,7 @@ package com.netflix.priam.config; -import com.google.common.collect.Lists; import com.google.inject.Singleton; -import com.netflix.priam.scheduler.UnsupportedTypeException; -import com.netflix.priam.tuner.GCType; -import com.netflix.priam.tuner.JVMOption; import java.io.File; import java.util.Arrays; import java.util.HashMap; @@ -61,49 +57,26 @@ public boolean getAutoBoostrap() { } @Override - public void initialize() { - // TODO Auto-generated method stub - + public String getCassHome() { + return "/tmp/priam"; } + @Override + public void initialize() {} + @Override public String getBackupLocation() { - // TODO Auto-generated method stub return "casstestbackup"; } @Override public String getBackupPrefix() { - // TODO Auto-generated method stub return "TEST-netflix.platform.S3"; } @Override - public String getCommitLogLocation() { - // TODO Auto-generated method stub - return "cass/commitlog"; - } - - @Override - public String getDataFileLocation() { - // TODO Auto-generated method stub - return "target/data"; - } - - @Override - public String getLogDirLocation() { - return null; - } - - @Override - public String getHintsLocation() { - return "target/hints"; - } - - @Override - public String getCacheLocation() { - // TODO Auto-generated method stub - return "cass/caches"; + public String getCassandraBaseDirectory() { + return "target"; } @Override @@ -116,45 +89,11 @@ public String getSnitch() { return "org.apache.cassandra.locator.SimpleSnitch"; } - @Override - public String getHeapSize() { - // TODO Auto-generated method stub - return null; - } - - @Override - public String getHeapNewSize() { - // TODO Auto-generated method stub - return null; - } - - @Override - public String getRestoreSnapshot() { - // TODO Auto-generated method stub - return null; - } - @Override public String getAppName() { return appName; } - @Override - public String getACLGroupName() { - return this.getAppName(); - } - - @Override - public String getSDBInstanceIdentityRegion() { - // TODO Auto-generated method stub - return null; - } - - @Override - public int getRestoreThreads() { - return 2; - } - @Override public String getRestorePrefix() { return this.restorePrefix; @@ -170,42 +109,11 @@ public String getBackupCommitLogLocation() { return "cass/backup/cl/"; } - @Override - public String getSiblingASGNames() { - return null; - } - - @Override - public boolean isLocalBootstrapEnabled() { - // TODO Auto-generated method stub - return false; - } - - @Override - public String getBootClusterName() { - return "cass_bootstrap"; - } - - @Override - public String getCassHome() { - return "/tmp/priam"; - } - @Override public String getCassStartupScript() { return "/usr/bin/false"; } - @Override - public long getBackupChunkSize() { - return 5L * 1024 * 1024; - } - - @Override - public String getCassStopScript() { - return "true"; - } - @Override public int getRemediateDeadCassandraRate() { return 1; @@ -221,36 +129,6 @@ public int getBackupRetentionDays() { return 5; } - @Override - public List getBackupRacs() { - return Lists.newArrayList(); - } - - public String getPartitioner() { - return "org.apache.cassandra.dht.RandomPartitioner"; - } - - public String getKeyCacheSizeInMB() { - return "16"; - } - - public String getKeyCacheKeysToSave() { - return "32"; - } - - public String getRowCacheSizeInMB() { - return "4"; - } - - public String getRowCacheKeysToSave() { - return "4"; - } - - @Override - public String getCassProcessName() { - return "CassandraDaemon"; - } - public String getYamlLocation() { return "conf/cassandra.yaml"; } @@ -260,175 +138,14 @@ public String getJVMOptionsFileLocation() { return "src/test/resources/conf/jvm.options"; } - @Override - public GCType getGCType() throws UnsupportedTypeException { - return GCType.CMS; - } - - @Override - public Map getJVMExcludeSet() { - return null; - } - - @Override - public Map getJVMUpsertSet() { - return null; - } - - public boolean doesCassandraStartManually() { - return false; - } - - @Override public String getCommitLogBackupPropsFile() { return getCassHome() + "/conf/commitlog_archiving.properties"; } - @Override - public String getCommitLogBackupArchiveCmd() { - return null; - } - - @Override - public String getCommitLogBackupRestoreCmd() { - return null; - } - - @Override - public String getCommitLogBackupRestoreFromDirs() { - return null; - } - - @Override - public String getCommitLogBackupRestorePointInTime() { - return null; - } - - @Override - public int maxCommitLogsRestore() { - return 0; - } - - public boolean isClientSslEnabled() { - return true; - } - - public String getInternodeEncryption() { - return "all"; - } - - public boolean isDynamicSnitchEnabled() { - return true; - } - - public boolean isThriftEnabled() { - return true; - } - - public boolean isNativeTransportEnabled() { - return false; - } - - public int getConcurrentReadsCnt() { - return 8; - } - - public int getConcurrentWritesCnt() { - return 8; - } - - public int getConcurrentCompactorsCnt() { - return 1; - } - - @Override - public int getCompactionLargePartitionWarnThresholdInMB() { - return 100; - } - - public String getExtraConfigParams() { - return null; - } - public String getCassYamlVal(String priamKey) { return ""; } - @Override - public boolean isCreateNewTokenEnable() { - return true; // allow Junit test to create new tokens - } - - @Override - public String getPrivateKeyLocation() { - return null; - } - - @Override - public String getRestoreSourceType() { - return null; - } - - @Override - public boolean isEncryptBackupEnabled() { - return false; - } - - @Override - public String getAWSRoleAssumptionArn() { - return null; - } - - @Override - public String getClassicEC2RoleAssumptionArn() { - return null; - } - - @Override - public String getVpcEC2RoleAssumptionArn() { - return null; - } - - @Override - public String getGcsServiceAccountId() { - return null; - } - - @Override - public String getGcsServiceAccountPrivateKeyLoc() { - return null; - } - - @Override - public String getPgpPasswordPhrase() { - return null; - } - - @Override - public String getPgpPublicKeyLoc() { - return null; - } - - /** - * Use this method for adding extra/ dynamic cassandra startup options or env properties - * - * @return - */ - @Override - public Map getExtraEnvParams() { - return null; - } - - @Override - public int getBackupQueueSize() { - return 100; - } - - @Override - public String getFlushKeyspaces() { - return ""; - } - @Override public boolean isPostRestoreHookEnabled() { return true; diff --git a/priam/src/test/java/com/netflix/priam/tuner/JVMOptionTunerTest.java b/priam/src/test/java/com/netflix/priam/tuner/JVMOptionTunerTest.java index 9628dca23..1942704f5 100644 --- a/priam/src/test/java/com/netflix/priam/tuner/JVMOptionTunerTest.java +++ b/priam/src/test/java/com/netflix/priam/tuner/JVMOptionTunerTest.java @@ -232,8 +232,8 @@ public GCType getGCType() throws UnsupportedTypeException { } @Override - public Map getJVMExcludeSet() { - return JVMOptionsTuner.parseJVMOptions(configuredJVMExclude); + public String getJVMExcludeSet() { + return configuredJVMExclude; } @Override @@ -247,8 +247,8 @@ public String getHeapNewSize() { } @Override - public Map getJVMUpsertSet() { - return JVMOptionsTuner.parseJVMOptions(configuredJVMUpsert); + public String getJVMUpsertSet() { + return configuredJVMUpsert; } } } From 45fa0c0b878baa4fc9c5efa7710a33c9034f3e2e Mon Sep 17 00:00:00 2001 From: Arun Agrawal Date: Sun, 28 Oct 2018 11:20:32 -0700 Subject: [PATCH 033/228] Move logic to get `defaultRacks` to AWSInstanceInfo --- .../priam/config/PriamConfiguration.java | 32 ++----------------- .../identity/config/AWSInstanceInfo.java | 25 ++++++++++++--- .../priam/identity/config/InstanceInfo.java | 13 ++++++++ 3 files changed, 36 insertions(+), 34 deletions(-) diff --git a/priam/src/main/java/com/netflix/priam/config/PriamConfiguration.java b/priam/src/main/java/com/netflix/priam/config/PriamConfiguration.java index de956f10f..24a356e01 100644 --- a/priam/src/main/java/com/netflix/priam/config/PriamConfiguration.java +++ b/priam/src/main/java/com/netflix/priam/config/PriamConfiguration.java @@ -16,17 +16,10 @@ */ package com.netflix.priam.config; -import com.amazonaws.services.ec2.AmazonEC2; -import com.amazonaws.services.ec2.AmazonEC2ClientBuilder; -import com.amazonaws.services.ec2.model.AvailabilityZone; -import com.amazonaws.services.ec2.model.DescribeAvailabilityZonesResult; import com.fasterxml.jackson.annotation.JsonIgnore; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.Lists; import com.google.inject.Inject; import com.google.inject.Singleton; import com.netflix.priam.configSource.IConfigSource; -import com.netflix.priam.cred.ICredential; import com.netflix.priam.identity.config.InstanceInfo; import com.netflix.priam.scheduler.SchedulerType; import com.netflix.priam.scheduler.UnsupportedTypeException; @@ -42,18 +35,14 @@ @Singleton public class PriamConfiguration implements IConfiguration { public static final String PRIAM_PRE = "priam"; - private List DEFAULT_AVAILABILITY_ZONES = ImmutableList.of(); private final IConfigSource config; private static final Logger logger = LoggerFactory.getLogger(PriamConfiguration.class); - private final ICredential provider; @JsonIgnore private InstanceInfo instanceInfo; @Inject - public PriamConfiguration( - ICredential provider, IConfigSource config, InstanceInfo instanceInfo) { - this.provider = provider; + public PriamConfiguration(IConfigSource config, InstanceInfo instanceInfo) { this.config = config; this.instanceInfo = instanceInfo; } @@ -61,23 +50,6 @@ public PriamConfiguration( @Override public void initialize() { this.config.initialize(instanceInfo.getAutoScalingGroup(), instanceInfo.getRegion()); - setDefaultRACList(instanceInfo.getRegion()); - } - - /** Get the fist 3 available zones in the region */ - private void setDefaultRACList(String region) { - AmazonEC2 client = - AmazonEC2ClientBuilder.standard() - .withCredentials(provider.getAwsCredentialProvider()) - .withRegion(region) - .build(); - DescribeAvailabilityZonesResult res = client.describeAvailabilityZones(); - List zone = Lists.newArrayList(); - for (AvailabilityZone reg : res.getAvailabilityZones()) { - if (reg.getState().equals("available")) zone.add(reg.getZoneName()); - if (zone.size() == 3) break; - } - DEFAULT_AVAILABILITY_ZONES = ImmutableList.copyOf(zone); } @Override @@ -221,7 +193,7 @@ public String getAppName() { @Override public List getRacs() { - return config.getList(PRIAM_PRE + ".zones.available", DEFAULT_AVAILABILITY_ZONES); + return config.getList(PRIAM_PRE + ".zones.available", instanceInfo.getDefaultRacks()); } @Override diff --git a/priam/src/main/java/com/netflix/priam/identity/config/AWSInstanceInfo.java b/priam/src/main/java/com/netflix/priam/identity/config/AWSInstanceInfo.java index ac742109b..b2c104daf 100644 --- a/priam/src/main/java/com/netflix/priam/identity/config/AWSInstanceInfo.java +++ b/priam/src/main/java/com/netflix/priam/identity/config/AWSInstanceInfo.java @@ -15,15 +15,15 @@ import com.amazonaws.services.ec2.AmazonEC2; import com.amazonaws.services.ec2.AmazonEC2ClientBuilder; -import com.amazonaws.services.ec2.model.DescribeInstancesRequest; -import com.amazonaws.services.ec2.model.DescribeInstancesResult; -import com.amazonaws.services.ec2.model.Instance; -import com.amazonaws.services.ec2.model.Reservation; +import com.amazonaws.services.ec2.model.*; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Lists; import com.google.inject.Inject; import com.google.inject.Singleton; import com.netflix.priam.cred.ICredential; import com.netflix.priam.utils.RetryableCallable; import com.netflix.priam.utils.SystemUtils; +import java.util.List; import org.apache.commons.lang3.StringUtils; import org.codehaus.jettison.json.JSONException; import org.codehaus.jettison.json.JSONObject; @@ -71,6 +71,23 @@ public String getRac() { return rac; } + @Override + public List getDefaultRacks() { + // Get the fist 3 available zones in the region + AmazonEC2 client = + AmazonEC2ClientBuilder.standard() + .withCredentials(credential.getAwsCredentialProvider()) + .withRegion(getRegion()) + .build(); + DescribeAvailabilityZonesResult res = client.describeAvailabilityZones(); + List zone = Lists.newArrayList(); + for (AvailabilityZone reg : res.getAvailabilityZones()) { + if (reg.getState().equals("available")) zone.add(reg.getZoneName()); + if (zone.size() == 3) break; + } + return ImmutableList.copyOf(zone); + } + public String getPublicHostname() { if (publicHostName == null) { publicHostName = diff --git a/priam/src/main/java/com/netflix/priam/identity/config/InstanceInfo.java b/priam/src/main/java/com/netflix/priam/identity/config/InstanceInfo.java index 92c2cee1d..f368e6c4b 100644 --- a/priam/src/main/java/com/netflix/priam/identity/config/InstanceInfo.java +++ b/priam/src/main/java/com/netflix/priam/identity/config/InstanceInfo.java @@ -13,7 +13,10 @@ */ package com.netflix.priam.identity.config; +import com.google.common.collect.ImmutableList; import com.google.inject.ImplementedBy; +import com.netflix.priam.config.IConfiguration; +import java.util.List; /** A means to fetch meta data of running instance */ @ImplementedBy(AWSInstanceInfo.class) @@ -25,6 +28,16 @@ public interface InstanceInfo { */ String getRac(); + /** + * Get the list of default racks available for this DC. This is used if no value is configured + * for {@link IConfiguration#getRacs()} + * + * @return list of default racks. + */ + default List getDefaultRacks() { + return ImmutableList.of(getRac()); + } + /** * Get the hostname for the running instance. Cannot be null. * From 7f0a981bf78f873e2e66fe8e2fc3f34c3c922b3a Mon Sep 17 00:00:00 2001 From: Arun Agrawal Date: Mon, 29 Oct 2018 13:41:48 -0700 Subject: [PATCH 034/228] update to changelog --- CHANGELOG.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6e0ddc867..7d9629c46 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,9 @@ # Changelog +## 2018/10/29 3.11.37 +* Bug Fix: SnapshotMetaService can leave snapshots if there is any error. +* Bug Fix: SnapshotMetaService should continue building snapshot even if an unexpected file is found in snapshot. +* More cleanup of IConfiguration and moving code to appropriate places. + ## 2018/10/26 3.11.36 * (#747) Aggregate InstanceData in InstanceInfo and pull information about running instance From 420dd2ded3903626fa087d9aa4f574c1157c979a Mon Sep 17 00:00:00 2001 From: Arun Agrawal Date: Thu, 29 Nov 2018 13:43:50 -0800 Subject: [PATCH 035/228] Enable uploads from SnapshotMetaService in backup 2.0 format. --- .../java/com/netflix/priam/PriamServer.java | 9 +- .../com/netflix/priam/aws/S3BackupPath.java | 155 ++++++++++--- .../netflix/priam/aws/S3FileSystemBase.java | 24 +++ .../netflix/priam/backup/AbstractBackup.java | 11 +- .../priam/backup/AbstractBackupPath.java | 90 +++----- .../priam/backup/AbstractFileSystem.java | 32 ++- .../priam/backup/IBackupFileSystem.java | 12 ++ .../priam/backup/IncrementalBackup.java | 2 +- .../com/netflix/priam/backup/MetaData.java | 2 +- .../netflix/priam/backup/SnapshotBackup.java | 2 +- .../priam/backupv2/FileUploadResult.java | 6 +- .../priam/backupv2/PrefixGenerator.java | 65 +----- .../priam/config/BackupRestoreConfig.java | 6 + .../priam/config/IBackupRestoreConfig.java | 10 + .../priam/services/SnapshotMetaService.java | 127 ++++++++--- .../netflix/priam/aws/TestS3BackupPath.java | 203 ++++++++++++++++++ .../priam/backup/TestAbstractFileSystem.java | 18 +- .../netflix/priam/backup/TestBackupFile.java | 2 +- .../netflix/priam/backup/TestCompression.java | 5 +- .../priam/backupv2/TestMetaFileManager.java | 98 +++++++++ .../services/TestSnapshotMetaService.java | 6 - 21 files changed, 670 insertions(+), 215 deletions(-) create mode 100644 priam/src/test/java/com/netflix/priam/aws/TestS3BackupPath.java create mode 100644 priam/src/test/java/com/netflix/priam/backupv2/TestMetaFileManager.java diff --git a/priam/src/main/java/com/netflix/priam/PriamServer.java b/priam/src/main/java/com/netflix/priam/PriamServer.java index 650ef5e7f..b883e40e9 100644 --- a/priam/src/main/java/com/netflix/priam/PriamServer.java +++ b/priam/src/main/java/com/netflix/priam/PriamServer.java @@ -53,6 +53,7 @@ public class PriamServer { private final Sleeper sleeper; private final ICassandraProcess cassProcess; private final RestoreContext restoreContext; + private final SnapshotMetaService snapshotMetaService; private static final int CASSANDRA_MONITORING_INITIAL_DELAY = 10; private static final Logger logger = LoggerFactory.getLogger(PriamServer.class); @@ -64,7 +65,8 @@ public PriamServer( InstanceIdentity id, Sleeper sleeper, ICassandraProcess cassProcess, - RestoreContext restoreContext) { + RestoreContext restoreContext, + SnapshotMetaService snapshotMetaService) { this.config = config; this.backupRestoreConfig = backupRestoreConfig; this.scheduler = scheduler; @@ -72,6 +74,7 @@ public PriamServer( this.sleeper = sleeper; this.cassProcess = cassProcess; this.restoreContext = restoreContext; + this.snapshotMetaService = snapshotMetaService; } private void createDirectories() { @@ -206,6 +209,10 @@ private void setUpSnapshotService() throws Exception { SnapshotMetaService.class, snapshotMetaServiceTimer); logger.info("Added SnapshotMetaService Task."); + + // Try to upload previous snapshots, if any which might have been interrupted by Priam + // restart. + snapshotMetaService.uploadFiles(); } } diff --git a/priam/src/main/java/com/netflix/priam/aws/S3BackupPath.java b/priam/src/main/java/com/netflix/priam/aws/S3BackupPath.java index 37e4f9216..c392cd16e 100644 --- a/priam/src/main/java/com/netflix/priam/aws/S3BackupPath.java +++ b/priam/src/main/java/com/netflix/priam/aws/S3BackupPath.java @@ -19,8 +19,12 @@ import com.google.common.collect.Lists; import com.google.inject.Inject; import com.netflix.priam.backup.AbstractBackupPath; +import com.netflix.priam.compress.ICompression; import com.netflix.priam.config.IConfiguration; import com.netflix.priam.identity.InstanceIdentity; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.time.Instant; import java.util.Date; import java.util.List; @@ -32,54 +36,140 @@ public S3BackupPath(IConfiguration config, InstanceIdentity factory) { super(config, factory); } + private Path getV2Prefix() { + return Paths.get(baseDir, getAppNameWithHash(), instanceIdentity.getInstance().getToken()); + } + + private void parseV2Prefix(Path remoteFilePath) { + if (remoteFilePath.getNameCount() < 3) + throw new RuntimeException( + "Not enough no. of elements to parseV2Prefix : " + remoteFilePath); + baseDir = remoteFilePath.getName(0).toString(); + clusterName = parseAndValidateAppNameWithHash(remoteFilePath.getName(1).toString()); + token = remoteFilePath.getName(2).toString(); + } + + /* This will ensure that there is some randomness in the path at the start so that remote file systems + can hash the contents better when we have lot of clusters backing up at the same remote location. + */ + private String getAppNameWithHash() { + return String.format("%d_%s", config.getAppName().hashCode() % 10000, config.getAppName()); + } + + private String parseAndValidateAppNameWithHash(String appNameWithHash) { + int hash = Integer.parseInt(appNameWithHash.substring(0, appNameWithHash.indexOf("_"))); + String appName = appNameWithHash.substring(appNameWithHash.indexOf("_") + 1); + // Validate the hash + int calculatedHash = appName.hashCode() % 10000; + assert calculatedHash == hash; + return appName; + } + + /* + * This method will generate the location for the V2 backups. + * Note that we use epochMillis to sort the directory instead of traditional YYYYMMddHHmm. This will allow greater + * flexibility when doing restores as the s3 list calls with common prefix will have greater chance of match instead + * of traditional way (where it takes lot of s3 list calls when month or year changes). + * Another major difference w.r.t. V1 is having no distinction between SNAP and SST files as we upload SSTables only + * once to remote file system. + */ + private Path getV2Location() { + Path prefix = + Paths.get( + getV2Prefix().toString(), + type.toString(), + getLastModified().toEpochMilli() + ""); + + if (type == BackupFileType.SST_V2) { + prefix = Paths.get(prefix.toString(), keyspace, columnFamily); + } + + return Paths.get(prefix.toString(), fileName + "." + getCompression().toString()); + } + + private void parseV2Location(String remoteFile) { + Path remoteFilePath = Paths.get(remoteFile); + parseV2Prefix(remoteFilePath); + assert remoteFilePath.getNameCount() >= 6; + type = BackupFileType.valueOf(remoteFilePath.getName(3).toString()); + setLastModified(Instant.ofEpochMilli(Long.parseLong(remoteFilePath.getName(4).toString()))); + + if (type == BackupFileType.SST_V2) { + keyspace = remoteFilePath.getName(5).toString(); + columnFamily = remoteFilePath.getName(6).toString(); + } + + String fileNameCompression = + remoteFilePath.getName(remoteFilePath.getNameCount() - 1).toString(); + fileName = fileNameCompression.substring(0, fileNameCompression.lastIndexOf(".")); + setCompression( + ICompression.CompressionAlgorithm.valueOf( + fileNameCompression.substring(fileNameCompression.lastIndexOf(".") + 1))); + } + /** - * Format of backup path: - * BASE/REGION/CLUSTER/TOKEN/[SNAPSHOTTIME]/[SST|SNP|META]/KEYSPACE/COLUMNFAMILY/FILE + * Format of backup path: 1. For old style backups: + * BASE/REGION/CLUSTER/TOKEN/[SNAPSHOTTIME]/[SST|SNAP|META]/KEYSPACE/COLUMNFAMILY/FILE + * + *

    2. For new style backups (SnapshotMetaService) + * BASE/[cluster_name_hash]_cluster/TOKEN//[META_V2|SST_V2]/KEYSPACE/COLUMNFAMILY/[last_modified_time_ms]/FILE.compression */ @Override public String getRemotePath() { StringBuilder buff = new StringBuilder(); - buff.append(baseDir).append(S3BackupPath.PATH_SEP); // Base dir - buff.append(region).append(S3BackupPath.PATH_SEP); - buff.append(clusterName).append(S3BackupPath.PATH_SEP); // Cluster name - buff.append(token).append(S3BackupPath.PATH_SEP); - buff.append(formatDate(time)).append(S3BackupPath.PATH_SEP); - buff.append(type).append(S3BackupPath.PATH_SEP); - if (BackupFileType.isDataFile(type)) - buff.append(keyspace) - .append(S3BackupPath.PATH_SEP) - .append(columnFamily) - .append(S3BackupPath.PATH_SEP); - buff.append(fileName); + if (type == BackupFileType.SST_V2 || type == BackupFileType.META_V2) { + buff.append(getV2Location()); + } else { + buff.append(baseDir).append(S3BackupPath.PATH_SEP); // Base dir + buff.append(region).append(S3BackupPath.PATH_SEP); + buff.append(clusterName).append(S3BackupPath.PATH_SEP); // Cluster name + buff.append(token).append(S3BackupPath.PATH_SEP); + buff.append(formatDate(time)).append(S3BackupPath.PATH_SEP); + buff.append(type).append(S3BackupPath.PATH_SEP); + if (BackupFileType.isDataFile(type)) + buff.append(keyspace) + .append(S3BackupPath.PATH_SEP) + .append(columnFamily) + .append(S3BackupPath.PATH_SEP); + buff.append(fileName); + } return buff.toString(); } @Override public void parseRemote(String remoteFilePath) { - String[] elements = remoteFilePath.split(String.valueOf(S3BackupPath.PATH_SEP)); - // parse out things which are empty - List pieces = Lists.newArrayList(); - for (String ele : elements) { - if (ele.equals("")) continue; - pieces.add(ele); + // Check for all backup file types to ensure how we parse + // TODO: We should clean this hack to get backupFileType for parsing when we delete V1 of + // backups. + for (BackupFileType fileType : BackupFileType.values()) { + if (remoteFilePath.contains(PATH_SEP + fileType.toString() + PATH_SEP)) { + type = fileType; + break; + } } - assert pieces.size() >= 7 : "Too few elements in path " + remoteFilePath; - baseDir = pieces.get(0); - region = pieces.get(1); - clusterName = pieces.get(2); - token = pieces.get(3); - time = parseDate(pieces.get(4)); - type = BackupFileType.valueOf(pieces.get(5)); - if (BackupFileType.isDataFile(type)) { - keyspace = pieces.get(6); - columnFamily = pieces.get(7); + + if (type == BackupFileType.SST_V2 || type == BackupFileType.META_V2) { + parseV2Location(remoteFilePath); + } else { + List pieces = parsePrefix(remoteFilePath); + assert pieces.size() >= 7 : "Too few elements in path " + remoteFilePath; + time = parseDate(pieces.get(4)); + type = BackupFileType.valueOf(pieces.get(5)); + if (BackupFileType.isDataFile(type)) { + keyspace = pieces.get(6); + columnFamily = pieces.get(7); + } + // append the rest + fileName = pieces.get(pieces.size() - 1); } - // append the rest - fileName = pieces.get(pieces.size() - 1); } @Override public void parsePartialPrefix(String remoteFilePath) { + parsePrefix(remoteFilePath); + } + + private List parsePrefix(String remoteFilePath) { String[] elements = remoteFilePath.split(String.valueOf(S3BackupPath.PATH_SEP)); // parse out things which are empty List pieces = Lists.newArrayList(); @@ -92,6 +182,7 @@ public void parsePartialPrefix(String remoteFilePath) { region = pieces.get(1); clusterName = pieces.get(2); token = pieces.get(3); + return pieces; } @Override diff --git a/priam/src/main/java/com/netflix/priam/aws/S3FileSystemBase.java b/priam/src/main/java/com/netflix/priam/aws/S3FileSystemBase.java index eb2533c17..eb43e7fb2 100755 --- a/priam/src/main/java/com/netflix/priam/aws/S3FileSystemBase.java +++ b/priam/src/main/java/com/netflix/priam/aws/S3FileSystemBase.java @@ -13,6 +13,8 @@ */ package com.netflix.priam.aws; +import com.amazonaws.AmazonClientException; +import com.amazonaws.AmazonServiceException; import com.amazonaws.services.s3.AmazonS3; import com.amazonaws.services.s3.model.BucketLifecycleConfiguration; import com.amazonaws.services.s3.model.BucketLifecycleConfiguration.Rule; @@ -174,6 +176,28 @@ public long getFileSize(Path remotePath) throws BackupRestoreException { .getContentLength(); } + @Override + public boolean doesRemoteFileExist(Path remotePath) throws BackupRestoreException { + boolean exists = false; + try { + exists = s3Client.doesObjectExist(getPrefix(config), remotePath.toString()); + } catch (AmazonServiceException ase) { + // Amazon S3 rejected request + throw new BackupRestoreException( + "AmazonServiceException while checking existence of object: " + + remotePath + + ". Error: " + + ase.getMessage()); + } catch (AmazonClientException ace) { + // No point throwing this exception up. + logger.error( + "AmazonClientException: client encountered a serious internal problem while trying to communicate with S3", + ace.getMessage()); + } + + return exists; + } + @Override public void shutdown() { if (executor != null) executor.shutdown(); diff --git a/priam/src/main/java/com/netflix/priam/backup/AbstractBackup.java b/priam/src/main/java/com/netflix/priam/backup/AbstractBackup.java index 7bc76668f..cec5c53c9 100644 --- a/priam/src/main/java/com/netflix/priam/backup/AbstractBackup.java +++ b/priam/src/main/java/com/netflix/priam/backup/AbstractBackup.java @@ -38,9 +38,9 @@ public abstract class AbstractBackup extends Task { static final String INCREMENTAL_BACKUP_FOLDER = "backups"; public static final String SNAPSHOT_FOLDER = "snapshots"; - final Provider pathFactory; + protected final Provider pathFactory; - private IBackupFileSystem fs; + protected IBackupFileSystem fs; @Inject public AbstractBackup( @@ -71,10 +71,13 @@ private AbstractBackupPath getAbstractBackupPath(final File file, final BackupFi * @param parent Parent dir * @param type Type of file (META, SST, SNAP etc) * @param async Upload the file(s) in async fashion if enabled. + * @param waitForCompletion wait for completion for all files to upload if using async API. If + * `false` it will queue the files and return with no guarantee to upload. * @return List of files that are successfully uploaded as part of backup * @throws Exception when there is failure in uploading files. */ - List upload(final File parent, final BackupFileType type, boolean async) + protected List upload( + final File parent, final BackupFileType type, boolean async, boolean waitForCompletion) throws Exception { final List bps = Lists.newArrayList(); final List> futures = Lists.newArrayList(); @@ -108,7 +111,7 @@ List upload(final File parent, final BackupFileType type, bo } // Wait for all files to be uploaded. - if (async) { + if (async && waitForCompletion) { for (Future future : futures) future.get(); // This might throw exception if there is any error } diff --git a/priam/src/main/java/com/netflix/priam/backup/AbstractBackupPath.java b/priam/src/main/java/com/netflix/priam/backup/AbstractBackupPath.java index 1e6160ac3..cee82da76 100644 --- a/priam/src/main/java/com/netflix/priam/backup/AbstractBackupPath.java +++ b/priam/src/main/java/com/netflix/priam/backup/AbstractBackupPath.java @@ -18,15 +18,13 @@ import com.google.inject.ImplementedBy; import com.netflix.priam.aws.S3BackupPath; +import com.netflix.priam.compress.ICompression; import com.netflix.priam.config.IConfiguration; import com.netflix.priam.identity.InstanceIdentity; import java.io.File; -import java.io.IOException; -import java.io.InputStream; -import java.io.RandomAccessFile; import java.text.ParseException; +import java.time.Instant; import java.util.Date; -import org.apache.commons.io.IOUtils; import org.apache.commons.lang3.StringUtils; import org.joda.time.DateTime; import org.joda.time.format.DateTimeFormat; @@ -46,7 +44,8 @@ public enum BackupFileType { SST, CL, META, - META_V2; + META_V2, + SST_V2; public static boolean isDataFile(BackupFileType type) { return type != BackupFileType.META @@ -69,51 +68,29 @@ public static boolean isDataFile(BackupFileType type) { protected final InstanceIdentity instanceIdentity; protected final IConfiguration config; private File backupFile; - private long lastModified = 0; + private Instant lastModified; private Date uploadedTs; + private ICompression.CompressionAlgorithm compression = + ICompression.CompressionAlgorithm.SNAPPY; public AbstractBackupPath(IConfiguration config, InstanceIdentity instanceIdentity) { this.instanceIdentity = instanceIdentity; this.config = config; } + // TODO: This is so wrong as it completely depends on the timezone where application is running. + // Hopefully everyone running Priam has their clocks set to UTC. public static String formatDate(Date d) { return new DateTime(d).toString(FMT); } + // TODO: This is so wrong as it completely depends on the timezone where application is running. + // Hopefully everyone running Priam has their clocks set to UTC. public Date parseDate(String s) { return DATE_FORMAT.parseDateTime(s).toDate(); } - public InputStream localReader() throws IOException { - assert backupFile != null; - - InputStream ret = null; - - while (true) { - if (ret != null) { - ret.close(); - } - - lastModified = backupFile.lastModified(); - ret = new RafInputStream(new RandomAccessFile(backupFile, "r")); - - // Verify that the file hasn't changed since we opened it. - // We could avoid this flow by using the fstat() system call, - // but I see no way to do that (easily) from the JVM. - // The JVM returns the last modified time in milliseconds, - // but on Linux systems tested, it appears to be using the - // stat.st_mtime result, which is accurate only to seconds. - if (backupFile.lastModified() == lastModified) { - break; - } - } - - return ret; - } - public void parseLocal(File file, BackupFileType type) throws ParseException { - // TODO cleanup. this.backupFile = file; String rpath = @@ -128,9 +105,17 @@ public void parseLocal(File file, BackupFileType type) throws ParseException { this.keyspace = elements[0]; this.columnFamily = elements[1]; } + + time = new Date(file.lastModified()); + + /* + 1. For old style snapshots, make this value to time at which backup was executed. + 2. This is to ensure that all the files from the snapshot are uploaded under single directory in remote file system. + 3. For META file we always override the time field via @link{Metadata#decorateMetaJson} + */ if (type == BackupFileType.SNAP) time = parseDate(elements[3]); - if (type == BackupFileType.SST || type == BackupFileType.CL) - time = new Date(file.lastModified()); + + this.lastModified = Instant.ofEpochMilli(file.lastModified()); this.fileName = file.getName(); this.size = file.length(); } @@ -231,6 +216,10 @@ public Date getTime() { return time; } + public void setTime(Date time) { + this.time = time; + } + /* @return original, uncompressed file size */ @@ -270,31 +259,20 @@ public Date getUploadedTs() { return this.uploadedTs; } - public long getLastModified() { + public Instant getLastModified() { return lastModified; } - public static class RafInputStream extends InputStream implements AutoCloseable { - private final RandomAccessFile raf; - - public RafInputStream(RandomAccessFile raf) { - this.raf = raf; - } - - @Override - public synchronized int read(byte[] bytes, int off, int len) throws IOException { - return raf.read(bytes, off, len); - } + public void setLastModified(Instant instant) { + this.lastModified = instant; + } - @Override - public void close() { - IOUtils.closeQuietly(raf); - } + public ICompression.CompressionAlgorithm getCompression() { + return compression; + } - @Override - public int read() throws IOException { - return 0; - } + public void setCompression(ICompression.CompressionAlgorithm compression) { + this.compression = compression; } @Override diff --git a/priam/src/main/java/com/netflix/priam/backup/AbstractFileSystem.java b/priam/src/main/java/com/netflix/priam/backup/AbstractFileSystem.java index ff38c5a86..07ed2fc4d 100644 --- a/priam/src/main/java/com/netflix/priam/backup/AbstractFileSystem.java +++ b/priam/src/main/java/com/netflix/priam/backup/AbstractFileSystem.java @@ -166,15 +166,29 @@ public void uploadFile( logger.info("Uploading file: {} to location: {}", localPath, remotePath); try { notifyEventStart(new BackupEvent(path)); - long uploadedFileSize = - new BoundedExponentialRetryCallable(500, 10000, retry) { - @Override - public Long retriableCall() throws Exception { - return uploadFileImpl(localPath, remotePath); - } - }.call(); - backupMetrics.recordUploadRate(uploadedFileSize); - backupMetrics.incrementValidUploads(); + long uploadedFileSize; + + // Upload file if it not present at remote location. + if (!doesRemoteFileExist(remotePath)) { + uploadedFileSize = + new BoundedExponentialRetryCallable(500, 10000, retry) { + @Override + public Long retriableCall() throws Exception { + return uploadFileImpl(localPath, remotePath); + } + }.call(); + backupMetrics.recordUploadRate(uploadedFileSize); + backupMetrics.incrementValidUploads(); + } else { + // file is already uploaded to remote file system. get the compressed file size + // to update it for notification message. + uploadedFileSize = getFileSize(remotePath); + logger.info( + "File: {} already present on remoteFileSystem with file size: {}", + remotePath, + uploadedFileSize); + } + path.setCompressedFileSize(uploadedFileSize); notifyEventSuccess(new BackupEvent(path)); logger.info( diff --git a/priam/src/main/java/com/netflix/priam/backup/IBackupFileSystem.java b/priam/src/main/java/com/netflix/priam/backup/IBackupFileSystem.java index cff60e6fe..68acf3850 100644 --- a/priam/src/main/java/com/netflix/priam/backup/IBackupFileSystem.java +++ b/priam/src/main/java/com/netflix/priam/backup/IBackupFileSystem.java @@ -144,6 +144,18 @@ Future asyncUploadFile( */ long getFileSize(Path remotePath) throws BackupRestoreException; + /** + * Checks if the file denoted by remotePath exists on the remote file system. It does not need + * check if object was completely uploaded to remote file system. + * + * @param remotePath location on the remote file system. + * @return boolean value indicating presence of the file on remote file system. + * @throws BackupRestoreException + */ + default boolean doesRemoteFileExist(Path remotePath) throws BackupRestoreException { + return false; + } + /** * Get the number of tasks en-queue in the filesystem for upload. * diff --git a/priam/src/main/java/com/netflix/priam/backup/IncrementalBackup.java b/priam/src/main/java/com/netflix/priam/backup/IncrementalBackup.java index 6395c5fc4..0dafbd699 100644 --- a/priam/src/main/java/com/netflix/priam/backup/IncrementalBackup.java +++ b/priam/src/main/java/com/netflix/priam/backup/IncrementalBackup.java @@ -98,7 +98,7 @@ private void notifyObservers() { protected void processColumnFamily(String keyspace, String columnFamily, File backupDir) throws Exception { List uploadedFiles = - upload(backupDir, BackupFileType.SST, config.enableAsyncIncremental()); + upload(backupDir, BackupFileType.SST, config.enableAsyncIncremental(), true); if (!uploadedFiles.isEmpty()) { // format of yyyymmddhhmm (e.g. 201505060901) diff --git a/priam/src/main/java/com/netflix/priam/backup/MetaData.java b/priam/src/main/java/com/netflix/priam/backup/MetaData.java index 210913212..a5cbb8737 100644 --- a/priam/src/main/java/com/netflix/priam/backup/MetaData.java +++ b/priam/src/main/java/com/netflix/priam/backup/MetaData.java @@ -87,7 +87,7 @@ public AbstractBackupPath decorateMetaJson(File metafile, String snapshotName) throws ParseException { AbstractBackupPath backupfile = pathFactory.get(); backupfile.parseLocal(metafile, BackupFileType.META); - backupfile.time = backupfile.parseDate(snapshotName); + backupfile.setTime(backupfile.parseDate(snapshotName)); return backupfile; } diff --git a/priam/src/main/java/com/netflix/priam/backup/SnapshotBackup.java b/priam/src/main/java/com/netflix/priam/backup/SnapshotBackup.java index e0e1ab103..3afc226b6 100644 --- a/priam/src/main/java/com/netflix/priam/backup/SnapshotBackup.java +++ b/priam/src/main/java/com/netflix/priam/backup/SnapshotBackup.java @@ -221,7 +221,7 @@ protected void processColumnFamily(String keyspace, String columnFamily, File ba // Add files to this dir abstractBackupPaths.addAll( - upload(snapshotDir, BackupFileType.SNAP, config.enableAsyncSnapshot())); + upload(snapshotDir, BackupFileType.SNAP, config.enableAsyncSnapshot(), true)); } // private void findForgottenFiles(File snapshotDir) { diff --git a/priam/src/main/java/com/netflix/priam/backupv2/FileUploadResult.java b/priam/src/main/java/com/netflix/priam/backupv2/FileUploadResult.java index 672cc50c6..8aebe76f6 100644 --- a/priam/src/main/java/com/netflix/priam/backupv2/FileUploadResult.java +++ b/priam/src/main/java/com/netflix/priam/backupv2/FileUploadResult.java @@ -38,7 +38,7 @@ public class FileUploadResult { // Valid compression technique for now is SNAPPY only. Future we need to support LZ4 and NONE private ICompression.CompressionAlgorithm compression = ICompression.CompressionAlgorithm.SNAPPY; - private Path backupPath; + private String backupPath; public FileUploadResult( Path fileName, @@ -112,11 +112,11 @@ public void setCompression(ICompression.CompressionAlgorithm compression) { this.compression = compression; } - public Path getBackupPath() { + public String getBackupPath() { return backupPath; } - public void setBackupPath(Path backupPath) { + public void setBackupPath(String backupPath) { this.backupPath = backupPath; } diff --git a/priam/src/main/java/com/netflix/priam/backupv2/PrefixGenerator.java b/priam/src/main/java/com/netflix/priam/backupv2/PrefixGenerator.java index 881beb5e5..8cede4311 100644 --- a/priam/src/main/java/com/netflix/priam/backupv2/PrefixGenerator.java +++ b/priam/src/main/java/com/netflix/priam/backupv2/PrefixGenerator.java @@ -16,74 +16,11 @@ */ package com.netflix.priam.backupv2; -import com.netflix.priam.config.IConfiguration; -import com.netflix.priam.identity.InstanceIdentity; -import com.netflix.priam.utils.DateUtil; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.time.Instant; -import javax.inject.Inject; - /** * This is a utility class to get the backup location of the SSTables/Meta files with backup version - * 2.0. TODO: All this functionality will be used when we have BackupUploadDownloadService. Created - * by aagrawal on 6/5/18. + * 2.0. */ public class PrefixGenerator { - - private final IConfiguration configuration; - private final InstanceIdentity instanceIdentity; - - @Inject - PrefixGenerator(IConfiguration configuration, InstanceIdentity instanceIdentity) { - this.configuration = configuration; - this.instanceIdentity = instanceIdentity; - } - - public Path getPrefix() { - return Paths.get( - configuration.getBackupLocation(), - configuration.getBackupPrefix(), - getAppNameReverse(), - instanceIdentity.getInstance().getToken()); - } - - public Path getSSTPrefix() { - return getPrefix(); - } - - public Path getSSTLocation( - Instant instant, - String keyspaceName, - String columnfamilyName, - String prefix, - String fileName) { - return Paths.get( - getPrefix().toString(), - DateUtil.formatInstant(DateUtil.ddMMyyyyHHmm, instant), - keyspaceName, - columnfamilyName, - prefix, - fileName); - } - - public Path getMetaPrefix() { - return Paths.get(getPrefix().toString(), "META"); - } - - public Path getMetaLocation(Instant instant, String metaFileName) { - return Paths.get( - getMetaPrefix().toString(), - DateUtil.formatInstant(DateUtil.ddMMyyyyHHmm, instant), - metaFileName); - } - - private String getAppNameReverse() { - return new StringBuilder(configuration.getAppName()).reverse().toString(); - } - - // e.g. mc-3-big-Data.db or sample_cf-ka-7213-Index.db - /** * Gives the prefix (common name) of the sstable components. Returns null if it is not sstable * component e.g. mc-3-big-Data.db or ks-cf-ka-7213-Index.db will return mc-3-big or diff --git a/priam/src/main/java/com/netflix/priam/config/BackupRestoreConfig.java b/priam/src/main/java/com/netflix/priam/config/BackupRestoreConfig.java index a7675594a..a0e0957ac 100644 --- a/priam/src/main/java/com/netflix/priam/config/BackupRestoreConfig.java +++ b/priam/src/main/java/com/netflix/priam/config/BackupRestoreConfig.java @@ -26,7 +26,13 @@ public BackupRestoreConfig(IConfigSource config) { this.config = config; } + @Override public String getSnapshotMetaServiceCronExpression() { return config.get("priam.snapshot.meta.cron", "-1"); } + + @Override + public boolean enableV2Backups() { + return config.get("priam.enableV2Backups", false); + } } diff --git a/priam/src/main/java/com/netflix/priam/config/IBackupRestoreConfig.java b/priam/src/main/java/com/netflix/priam/config/IBackupRestoreConfig.java index 391ec4898..9d5873d8d 100644 --- a/priam/src/main/java/com/netflix/priam/config/IBackupRestoreConfig.java +++ b/priam/src/main/java/com/netflix/priam/config/IBackupRestoreConfig.java @@ -33,4 +33,14 @@ public interface IBackupRestoreConfig { default String getSnapshotMetaServiceCronExpression() { return "-1"; } + + /** + * Enable the backup version 2.0 in new format. This will start uploading of backups in new + * format. This is to be used for migration from backup version 1.0. + * + * @return boolean value indicating if backups in version 2.0 should be started. + */ + default boolean enableV2Backups() { + return false; + } } diff --git a/priam/src/main/java/com/netflix/priam/services/SnapshotMetaService.java b/priam/src/main/java/com/netflix/priam/services/SnapshotMetaService.java index 030444275..43daf9d7e 100644 --- a/priam/src/main/java/com/netflix/priam/services/SnapshotMetaService.java +++ b/priam/src/main/java/com/netflix/priam/services/SnapshotMetaService.java @@ -14,10 +14,7 @@ package com.netflix.priam.services; import com.google.inject.Provider; -import com.netflix.priam.backup.AbstractBackup; -import com.netflix.priam.backup.AbstractBackupPath; -import com.netflix.priam.backup.BackupRestoreUtil; -import com.netflix.priam.backup.IFileSystemContext; +import com.netflix.priam.backup.*; import com.netflix.priam.backupv2.*; import com.netflix.priam.config.IBackupRestoreConfig; import com.netflix.priam.config.IConfiguration; @@ -27,6 +24,7 @@ import com.netflix.priam.utils.CassandraMonitor; import com.netflix.priam.utils.DateUtil; import java.io.File; +import java.nio.file.Paths; import java.time.Instant; import java.util.*; import java.util.concurrent.locks.Lock; @@ -62,6 +60,7 @@ public class SnapshotMetaService extends AbstractBackup { private static final String SNAPSHOT_PREFIX = "snap_v2_"; private static final String CASSANDRA_MANIFEST_FILE = "manifest.json"; private static final String CASSANDRA_SCHEMA_FILE = "schema.cql"; + private final IBackupRestoreConfig backupRestoreConfig; private final BackupRestoreUtil backupRestoreUtil; private final MetaFileWriterBuilder metaFileWriter; private MetaFileWriterBuilder.DataStep dataStep; @@ -70,15 +69,24 @@ public class SnapshotMetaService extends AbstractBackup { private String snapshotName = null; private static final Lock lock = new ReentrantLock(); + private enum MetaStep { + META_GENERATION, + UPLOAD_FILES + } + + private MetaStep metaStep = MetaStep.META_GENERATION; + @Inject SnapshotMetaService( IConfiguration config, + IBackupRestoreConfig backupRestoreConfig, IFileSystemContext backupFileSystemCtx, Provider pathFactory, MetaFileWriterBuilder metaFileWriter, MetaFileManager metaFileManager, CassandraOperations cassandraOperations) { super(config, backupFileSystemCtx, pathFactory); + this.backupRestoreConfig = backupRestoreConfig; this.cassandraOperations = cassandraOperations; backupRestoreUtil = new BackupRestoreUtil( @@ -125,6 +133,26 @@ String generateSnapshotName(Instant snapshotInstant) { return SNAPSHOT_PREFIX + DateUtil.formatInstant(DateUtil.yyyyMMddHHmm, snapshotInstant); } + /** + * Enqueue all the files for upload in the snapshot directory. This will only enqueue the files + * and do not give guarantee as when they will be uploaded. It will only try to upload files + * which matches backup version 2.0 naming conventions. + */ + public void uploadFiles() { + try { + // enqueue all the old snapshot folder for upload/delete, if any, as we don't want + // our disk to be filled by them. + metaStep = MetaStep.UPLOAD_FILES; + initiateBackup(SNAPSHOT_FOLDER, backupRestoreUtil); + logger.info("Finished queuing the files for upload"); + } catch (Exception e) { + logger.error("Error while trying to upload all the files", e); + e.printStackTrace(); + } finally { + metaStep = MetaStep.META_GENERATION; + } + } + @Override public void execute() throws Exception { if (!CassandraMonitor.hasCassadraStarted()) { @@ -151,10 +179,6 @@ public void execute() throws Exception { // 2) No permission to upload to backup file system. metaFileManager.cleanupOldMetaFiles(); - // TODO: enqueue all the old backup folder for upload/delete, if any, as we don't want - // our disk to be filled by them. - // processOldSnapshotV2Folders(); - // Take a new snapshot cassandraOperations.takeSnapshot(snapshotName); @@ -162,17 +186,12 @@ public void execute() throws Exception { processSnapshot(snapshotInstant).uploadMetaFile(true); logger.info("Finished processing snapshot meta service"); + + // Upload all the files from snapshot + uploadFiles(); } catch (Exception e) { logger.error("Error while executing SnapshotMetaService", e); } finally { - // Make sure you clean up snapshots whatever be the case. - try { - cassandraOperations.clearSnapshot(snapshotName); - } catch (Exception e) { - logger.error(e.getMessage(), e); - } - - // Release the lock. lock.unlock(); } } @@ -184,10 +203,12 @@ MetaFileWriterBuilder.UploadStep processSnapshot(Instant snapshotInstant) throws } private File getValidSnapshot(File snapshotDir, String snapshotName) { - for (File fileName : snapshotDir.listFiles()) - if (fileName.exists() - && fileName.isDirectory() - && fileName.getName().matches(snapshotName)) return fileName; + File[] snapshotDirectories = snapshotDir.listFiles(); + if (snapshotDirectories != null) + for (File fileName : snapshotDirectories) + if (fileName.exists() + && fileName.isDirectory() + && fileName.getName().matches(snapshotName)) return fileName; return null; } @@ -212,10 +233,53 @@ private ColumnfamilyResult convertToColumnFamilyResult( return columnfamilyResult; } + private void uploadAllFiles( + final String keyspace, final String columnFamily, final File backupDir) + throws Exception { + // Process all the snapshots with SNAPSHOT_PREFIX. This will ensure that we "resume" the + // uploads of previous snapshot leftover as Priam restarted or any failure for any reason + // (like we exhausted the wait time for upload) + File[] snapshotDirectories = backupDir.listFiles(); + if (snapshotDirectories != null) { + for (File snapshotDirectory : snapshotDirectories) { + // Is it a valid SNAPSHOT_PREFIX + if (!snapshotDirectory.getName().startsWith(SNAPSHOT_PREFIX) + || !snapshotDirectory.isDirectory()) continue; + + if (snapshotDirectory.list().length == 0 + || !backupRestoreConfig.enableV2Backups()) { + FileUtils.cleanDirectory(snapshotDirectory); + FileUtils.deleteDirectory(snapshotDirectory); + continue; + } + + // Process each snapshot of SNAPSHOT_PREFIX + // We do not want to wait for completion and we just want to add them to queue. This + // is to ensure that next run happens on time. + upload(snapshotDirectory, AbstractBackupPath.BackupFileType.SST_V2, true, false); + } + } + } + @Override protected void processColumnFamily( final String keyspace, final String columnFamily, final File backupDir) throws Exception { + switch (metaStep) { + case META_GENERATION: + generateMetaFile(keyspace, columnFamily, backupDir); + break; + case UPLOAD_FILES: + uploadAllFiles(keyspace, columnFamily, backupDir); + break; + default: + throw new Exception("Unknown meta file type: " + metaStep); + } + } + + private void generateMetaFile( + final String keyspace, final String columnFamily, final File backupDir) + throws Exception { File snapshotDir = getValidSnapshot(backupDir, snapshotName); // Process this snapshot folder for the given columnFamily if (snapshotDir == null) { @@ -250,6 +314,20 @@ protected void processColumnFamily( FileUploadResult fileUploadResult = FileUploadResult.getFileUploadResult(keyspace, columnFamily, file); + // Add isUploaded and remotePath here. + try { + AbstractBackupPath abstractBackupPath = pathFactory.get(); + abstractBackupPath.parseLocal(file, AbstractBackupPath.BackupFileType.SST_V2); + fileUploadResult.setBackupPath(abstractBackupPath.getRemotePath()); + fileUploadResult.setUploaded( + fs.doesRemoteFileExist(Paths.get(fileUploadResult.getBackupPath()))); + } catch (Exception e) { + logger.error( + "Error while setting the remoteLocation or checking if file exists. Ignoring them as they are not fatal.", + e.getMessage()); + e.printStackTrace(); + } + filePrefixToFileMap.putIfAbsent(prefix, new ArrayList<>()); filePrefixToFileMap.get(prefix).add(fileUploadResult); } catch (Exception e) { @@ -283,15 +361,6 @@ protected void processColumnFamily( columnfamilyResult.getColumnfamilyName(), columnfamilyResult.getSstables().size()); - // TODO: Future - Ensure that all the files are en-queued for Upload. Use BackupCacheService - // (BCS) to find the - // location where files are uploaded and BackupUploadDownloadService(BUDS) to enque if they - // are not. - // Note that BUDS will be responsible for actually deleting the files after they are - // processed as they really should not be deleted unless they are successfully uploaded. - FileUtils.cleanDirectory(snapshotDir); - FileUtils.deleteDirectory(snapshotDir); - dataStep.addColumnfamilyResult(columnfamilyResult); logger.debug( "Finished processing KS: {}, CF: {}", diff --git a/priam/src/test/java/com/netflix/priam/aws/TestS3BackupPath.java b/priam/src/test/java/com/netflix/priam/aws/TestS3BackupPath.java new file mode 100644 index 000000000..2141f14d0 --- /dev/null +++ b/priam/src/test/java/com/netflix/priam/aws/TestS3BackupPath.java @@ -0,0 +1,203 @@ +/* + * Copyright 2018 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.netflix.priam.aws; + +import com.google.inject.Guice; +import com.google.inject.Injector; +import com.google.inject.Provider; +import com.netflix.priam.backup.AbstractBackupPath; +import com.netflix.priam.backup.AbstractBackupPath.BackupFileType; +import com.netflix.priam.backup.BRTestModule; +import com.netflix.priam.config.IConfiguration; +import com.netflix.priam.utils.DateUtil; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.text.ParseException; +import java.time.Instant; +import org.junit.Assert; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** Created by aagrawal on 11/23/18. */ +public class TestS3BackupPath { + private static final Logger logger = LoggerFactory.getLogger(TestS3BackupPath.class); + private Provider pathFactory; + private IConfiguration configuration; + + public TestS3BackupPath() { + Injector injector = Guice.createInjector(new BRTestModule()); + pathFactory = injector.getProvider(AbstractBackupPath.class); + configuration = injector.getInstance(IConfiguration.class); + } + + @Test + public void testV1BackupPathsSST() throws ParseException { + Path path = + Paths.get( + configuration.getDataFileLocation(), + "keyspace1", + "columnfamily1", + "backup", + "mc-1234-Data.db"); + AbstractBackupPath abstractBackupPath = pathFactory.get(); + abstractBackupPath.parseLocal(path.toFile(), BackupFileType.SST); + + // Verify parse local + Assert.assertEquals( + 0, abstractBackupPath.getLastModified().toEpochMilli()); // File do not exist. + Assert.assertEquals("keyspace1", abstractBackupPath.getKeyspace()); + Assert.assertEquals("columnfamily1", abstractBackupPath.getColumnFamily()); + Assert.assertEquals(BackupFileType.SST, abstractBackupPath.getType()); + Assert.assertEquals(path.toFile(), abstractBackupPath.getBackupFile()); + Assert.assertEquals( + 0, + abstractBackupPath + .getTime() + .toInstant() + .toEpochMilli()); // Since file do not exist. + + // Verify toRemote and parseRemote. + String remotePath = abstractBackupPath.getRemotePath(); + logger.info(remotePath); + AbstractBackupPath abstractBackupPath2 = pathFactory.get(); + abstractBackupPath2.parseRemote(remotePath); + Assert.assertEquals(abstractBackupPath, abstractBackupPath2); + } + + @Test + public void testV1BackupPathsSnap() throws ParseException { + Path path = + Paths.get( + configuration.getDataFileLocation(), + "keyspace1", + "columnfamily1", + "snapshot", + "201801011201", + "mc-1234-Data.db"); + AbstractBackupPath abstractBackupPath = pathFactory.get(); + abstractBackupPath.parseLocal(path.toFile(), BackupFileType.SNAP); + + // Verify parse local + Assert.assertEquals( + 0, abstractBackupPath.getLastModified().toEpochMilli()); // File do not exist. + Assert.assertEquals("keyspace1", abstractBackupPath.getKeyspace()); + Assert.assertEquals("columnfamily1", abstractBackupPath.getColumnFamily()); + Assert.assertEquals(BackupFileType.SNAP, abstractBackupPath.getType()); + Assert.assertEquals(path.toFile(), abstractBackupPath.getBackupFile()); + Assert.assertEquals( + "201801011201", AbstractBackupPath.formatDate(abstractBackupPath.getTime())); + + // Verify toRemote and parseRemote. + String remotePath = abstractBackupPath.getRemotePath(); + logger.info(remotePath); + + AbstractBackupPath abstractBackupPath2 = pathFactory.get(); + abstractBackupPath2.parseRemote(remotePath); + Assert.assertEquals(abstractBackupPath, abstractBackupPath2); + } + + @Test + public void testV1BackupPathsMeta() throws ParseException { + Path path = Paths.get(configuration.getDataFileLocation(), "meta.json"); + AbstractBackupPath abstractBackupPath = pathFactory.get(); + abstractBackupPath.parseLocal(path.toFile(), BackupFileType.META); + + // Verify parse local + Assert.assertEquals( + 0, abstractBackupPath.getLastModified().toEpochMilli()); // File do not exist. + Assert.assertEquals(null, abstractBackupPath.getKeyspace()); + Assert.assertEquals(null, abstractBackupPath.getColumnFamily()); + Assert.assertEquals(BackupFileType.META, abstractBackupPath.getType()); + Assert.assertEquals(path.toFile(), abstractBackupPath.getBackupFile()); + + // Verify toRemote and parseRemote. + String remotePath = abstractBackupPath.getRemotePath(); + logger.info(remotePath); + + AbstractBackupPath abstractBackupPath2 = pathFactory.get(); + abstractBackupPath2.parseRemote(remotePath); + Assert.assertEquals(abstractBackupPath, abstractBackupPath2); + } + + @Test + public void testV2BackupPathSST() throws ParseException { + Path path = + Paths.get( + configuration.getDataFileLocation(), + "keyspace1", + "columnfamily1", + "backup", + "mc-1234-Data.db"); + AbstractBackupPath abstractBackupPath = pathFactory.get(); + abstractBackupPath.parseLocal(path.toFile(), BackupFileType.SST_V2); + + // Verify parse local + Assert.assertEquals( + 0, abstractBackupPath.getLastModified().toEpochMilli()); // File do not exist. + Assert.assertEquals("keyspace1", abstractBackupPath.getKeyspace()); + Assert.assertEquals("columnfamily1", abstractBackupPath.getColumnFamily()); + Assert.assertEquals(BackupFileType.SST_V2, abstractBackupPath.getType()); + Assert.assertEquals(path.toFile(), abstractBackupPath.getBackupFile()); + + // Verify toRemote and parseRemote. + Instant now = DateUtil.getInstant(); + abstractBackupPath.setLastModified(now); + String remotePath = abstractBackupPath.getRemotePath(); + logger.info(remotePath); + + Assert.assertTrue(remotePath.endsWith(abstractBackupPath.getCompression().toString())); + + AbstractBackupPath abstractBackupPath2 = pathFactory.get(); + abstractBackupPath2.parseRemote(remotePath); + Assert.assertEquals(now, abstractBackupPath2.getLastModified()); + Assert.assertEquals("mc-1234-Data.db", abstractBackupPath2.getFileName()); + Assert.assertEquals("keyspace1", abstractBackupPath.getKeyspace()); + Assert.assertEquals("columnfamily1", abstractBackupPath.getColumnFamily()); + Assert.assertEquals(BackupFileType.SST_V2, abstractBackupPath.getType()); + } + + @Test + public void testV2BackupPathMeta() throws ParseException { + Path path = Paths.get(configuration.getDataFileLocation(), "meta_v2_201801011201.json"); + AbstractBackupPath abstractBackupPath = pathFactory.get(); + abstractBackupPath.parseLocal(path.toFile(), BackupFileType.META_V2); + + // Verify parse local + Assert.assertEquals( + 0, abstractBackupPath.getLastModified().toEpochMilli()); // File do not exist. + Assert.assertEquals(null, abstractBackupPath.getKeyspace()); + Assert.assertEquals(null, abstractBackupPath.getColumnFamily()); + Assert.assertEquals(BackupFileType.META_V2, abstractBackupPath.getType()); + Assert.assertEquals(path.toFile(), abstractBackupPath.getBackupFile()); + + // Verify toRemote and parseRemote. + Instant now = DateUtil.getInstant(); + abstractBackupPath.setLastModified(now); + String remotePath = abstractBackupPath.getRemotePath(); + logger.info(remotePath); + + Assert.assertTrue(remotePath.endsWith(abstractBackupPath.getCompression().toString())); + + AbstractBackupPath abstractBackupPath2 = pathFactory.get(); + abstractBackupPath2.parseRemote(remotePath); + Assert.assertEquals(now, abstractBackupPath2.getLastModified()); + Assert.assertEquals("meta_v2_201801011201.json", abstractBackupPath2.getFileName()); + Assert.assertEquals(BackupFileType.META_V2, abstractBackupPath.getType()); + } +} diff --git a/priam/src/test/java/com/netflix/priam/backup/TestAbstractFileSystem.java b/priam/src/test/java/com/netflix/priam/backup/TestAbstractFileSystem.java index b970643b2..73fc7d984 100644 --- a/priam/src/test/java/com/netflix/priam/backup/TestAbstractFileSystem.java +++ b/priam/src/test/java/com/netflix/priam/backup/TestAbstractFileSystem.java @@ -117,7 +117,11 @@ public void testUpload() throws Exception { // Dummy upload with compressed size. for (File file : files) { myFileSystem.uploadFile( - file.toPath(), file.toPath(), getDummyPath(file.toPath()), 2, true); + file.toPath(), + Paths.get(file.toString() + ".tmp"), + getDummyPath(file.toPath()), + 2, + true); // Verify the success metric for upload is incremented. Assert.assertEquals(1, (int) backupMetrics.getValidUploads().actualCount()); @@ -142,7 +146,11 @@ public void testAsyncUpload() throws Exception { for (File file : files) { myFileSystem .asyncUploadFile( - file.toPath(), file.toPath(), getDummyPath(file.toPath()), 2, true) + file.toPath(), + Paths.get(file.toString() + ".tmp"), + getDummyPath(file.toPath()), + 2, + true) .get(); // 1. Verify the success metric for upload is incremented. Assert.assertEquals(1, (int) backupMetrics.getValidUploads().actualCount()); @@ -161,7 +169,11 @@ public void testAsyncUploadBulk() throws Exception { for (File file : files) { futures.add( myFileSystem.asyncUploadFile( - file.toPath(), file.toPath(), getDummyPath(file.toPath()), 2, true)); + file.toPath(), + Paths.get(file.toString() + ".tmp"), + getDummyPath(file.toPath()), + 2, + true)); } // Verify all the work is finished. diff --git a/priam/src/test/java/com/netflix/priam/backup/TestBackupFile.java b/priam/src/test/java/com/netflix/priam/backup/TestBackupFile.java index 631c23ef6..7fff9d830 100644 --- a/priam/src/test/java/com/netflix/priam/backup/TestBackupFile.java +++ b/priam/src/test/java/com/netflix/priam/backup/TestBackupFile.java @@ -117,8 +117,8 @@ public void testMetaFileCreation() throws ParseException { String filestr = "cass/data/1234567.meta"; File bfile = new File(filestr); S3BackupPath backupfile = injector.getInstance(S3BackupPath.class); - backupfile.time = backupfile.parseDate("201108082320"); backupfile.parseLocal(bfile, BackupFileType.META); + backupfile.setTime(backupfile.parseDate("201108082320")); Assert.assertEquals(BackupFileType.META, backupfile.type); Assert.assertEquals("1234567", backupfile.token); Assert.assertEquals("fake-app", backupfile.clusterName); diff --git a/priam/src/test/java/com/netflix/priam/backup/TestCompression.java b/priam/src/test/java/com/netflix/priam/backup/TestCompression.java index 38c42398c..4de6c36f3 100644 --- a/priam/src/test/java/com/netflix/priam/backup/TestCompression.java +++ b/priam/src/test/java/com/netflix/priam/backup/TestCompression.java @@ -112,10 +112,7 @@ public void snappyTest() throws IOException { long chunkSize = 5L * 1024 * 1024; try { Iterator it = - compress.compress( - new AbstractBackupPath.RafInputStream( - new RandomAccessFile(randomContentFile, "r")), - chunkSize); + compress.compress(new FileInputStream(randomContentFile), chunkSize); try (FileOutputStream ostream = new FileOutputStream(compressedOutputFile)) { while (it.hasNext()) { byte[] chunk = it.next(); diff --git a/priam/src/test/java/com/netflix/priam/backupv2/TestMetaFileManager.java b/priam/src/test/java/com/netflix/priam/backupv2/TestMetaFileManager.java new file mode 100644 index 000000000..a1dd5482a --- /dev/null +++ b/priam/src/test/java/com/netflix/priam/backupv2/TestMetaFileManager.java @@ -0,0 +1,98 @@ +/* + * Copyright 2018 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.netflix.priam.backupv2; + +import com.google.inject.Guice; +import com.google.inject.Injector; +import com.netflix.priam.backup.BRTestModule; +import com.netflix.priam.config.IConfiguration; +import com.netflix.priam.utils.DateUtil; +import java.io.File; +import java.io.IOException; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.time.temporal.ChronoUnit; +import org.apache.commons.io.FileUtils; +import org.junit.After; +import org.junit.Assert; +import org.junit.Test; + +/** Created by aagrawal on 11/28/18. */ +public class TestMetaFileManager { + + private MetaFileManager metaFileManager; + private IConfiguration configuration; + + public TestMetaFileManager() { + Injector injector = Guice.createInjector(new BRTestModule()); + metaFileManager = injector.getInstance(MetaFileManager.class); + configuration = injector.getInstance(IConfiguration.class); + } + + @After + public void cleanup() throws IOException { + FileUtils.cleanDirectory(new File(configuration.getDataFileLocation())); + } + + @Test + public void testCleanupOldMetaFiles() throws IOException { + generateDummyMetaFiles(); + Path dataDir = Paths.get(configuration.getDataFileLocation()); + Assert.assertEquals(4, dataDir.toFile().listFiles().length); + + // clean the directory + metaFileManager.cleanupOldMetaFiles(); + + Assert.assertEquals(1, dataDir.toFile().listFiles().length); + Path dummy = Paths.get(dataDir.toString(), "dummy.tmp"); + Assert.assertTrue(dummy.toFile().exists()); + } + + private void generateDummyMetaFiles() throws IOException { + Path dataDir = Paths.get(configuration.getDataFileLocation()); + FileUtils.write( + Paths.get( + configuration.getDataFileLocation(), + MetaFileInfo.getMetaFileName(DateUtil.getInstant())) + .toFile(), + "dummy", + "UTF-8"); + + FileUtils.write( + Paths.get( + configuration.getDataFileLocation(), + MetaFileInfo.getMetaFileName( + DateUtil.getInstant().minus(10, ChronoUnit.MINUTES))) + .toFile(), + "dummy", + "UTF-8"); + + FileUtils.write( + Paths.get( + configuration.getDataFileLocation(), + MetaFileInfo.getMetaFileName(DateUtil.getInstant()) + ".tmp") + .toFile(), + "dummy", + "UTF-8"); + + FileUtils.write( + Paths.get(configuration.getDataFileLocation(), "dummy.tmp").toFile(), + "dummy", + "UTF-8"); + } +} diff --git a/priam/src/test/java/com/netflix/priam/services/TestSnapshotMetaService.java b/priam/src/test/java/com/netflix/priam/services/TestSnapshotMetaService.java index 6a41e869b..ce0bc81b8 100644 --- a/priam/src/test/java/com/netflix/priam/services/TestSnapshotMetaService.java +++ b/priam/src/test/java/com/netflix/priam/services/TestSnapshotMetaService.java @@ -78,12 +78,6 @@ public void testSnapshotMetaServiceEnabled() throws Exception { Assert.assertNotNull(taskTimer); } - @Test - public void testPrefix() throws Exception { - Assert.assertTrue(prefixGenerator.getPrefix().endsWith("ppa-ekaf/1808575600")); - Assert.assertTrue(prefixGenerator.getMetaPrefix().endsWith("ppa-ekaf/1808575600/META")); - } - @Test public void testMetaFileName() throws Exception { String fileName = MetaFileInfo.getMetaFileName(DateUtil.getInstant()); From d6e7fc5075cb9901947a58264ea6292525824e9a Mon Sep 17 00:00:00 2001 From: Arun Agrawal Date: Fri, 30 Nov 2018 09:28:16 -0800 Subject: [PATCH 036/228] * Migrate Backup 1.0 paths * Daterange for easier input of daterange. --- .../com/netflix/priam/aws/S3BackupPath.java | 83 +++++++++---------- .../priam/aws/S3EncryptedFileSystem.java | 4 +- .../com/netflix/priam/aws/S3FileIterator.java | 18 ++-- .../com/netflix/priam/aws/S3FileSystem.java | 7 +- .../netflix/priam/aws/S3FileSystemBase.java | 20 +---- .../netflix/priam/aws/S3PrefixIterator.java | 27 ++---- .../priam/backup/AbstractFileSystem.java | 15 ++++ .../priam/backup/IBackupFileSystem.java | 10 +++ .../priam/resources/BackupServlet.java | 52 +++++------- .../priam/resources/RestoreServlet.java | 22 ++--- .../priam/restore/AbstractRestore.java | 17 ++-- .../com/netflix/priam/utils/DateUtil.java | 25 ++++++ .../com/netflix/priam/utils/TokenManager.java | 2 +- .../netflix/priam/aws/TestS3BackupPath.java | 24 ++++-- .../priam/backup/TestFileIterator.java | 18 ++-- .../netflix/priam/utils/TestDateUtils.java | 75 +++++++++++++++++ 16 files changed, 255 insertions(+), 164 deletions(-) create mode 100644 priam/src/test/java/com/netflix/priam/utils/TestDateUtils.java diff --git a/priam/src/main/java/com/netflix/priam/aws/S3BackupPath.java b/priam/src/main/java/com/netflix/priam/aws/S3BackupPath.java index c392cd16e..832d77f73 100644 --- a/priam/src/main/java/com/netflix/priam/aws/S3BackupPath.java +++ b/priam/src/main/java/com/netflix/priam/aws/S3BackupPath.java @@ -16,7 +16,6 @@ */ package com.netflix.priam.aws; -import com.google.common.collect.Lists; import com.google.inject.Inject; import com.netflix.priam.backup.AbstractBackupPath; import com.netflix.priam.compress.ICompression; @@ -26,7 +25,6 @@ import java.nio.file.Paths; import java.time.Instant; import java.util.Date; -import java.util.List; /** Represents an S3 object key */ public class S3BackupPath extends AbstractBackupPath { @@ -107,6 +105,41 @@ private void parseV2Location(String remoteFile) { fileNameCompression.substring(fileNameCompression.lastIndexOf(".") + 1))); } + private Path getV1Location() { + Path path = Paths.get(getV1Prefix().toString(), formatDate(time), type.toString()); + if (BackupFileType.isDataFile(type)) + path = Paths.get(path.toString(), keyspace, columnFamily); + return Paths.get(path.toString(), fileName); + } + + private void parseV1Location(Path remoteFilePath) { + parseV1Prefix(remoteFilePath); + assert remoteFilePath.getNameCount() >= 7 : "Too few elements in path " + remoteFilePath; + + time = parseDate(remoteFilePath.getName(4).toString()); + type = BackupFileType.valueOf(remoteFilePath.getName(5).toString()); + if (BackupFileType.isDataFile(type)) { + keyspace = remoteFilePath.getName(6).toString(); + columnFamily = remoteFilePath.getName(7).toString(); + } + // append the rest + fileName = remoteFilePath.getName(remoteFilePath.getNameCount() - 1).toString(); + } + + private Path getV1Prefix() { + return Paths.get(baseDir, region, clusterName, token); + } + + private void parseV1Prefix(Path remoteFilePath) { + if (remoteFilePath.getNameCount() < 4) + throw new RuntimeException( + "Not enough no. of elements to parseV1Prefix : " + remoteFilePath); + baseDir = remoteFilePath.getName(0).toString(); + region = remoteFilePath.getName(1).toString(); + clusterName = remoteFilePath.getName(2).toString(); + token = remoteFilePath.getName(3).toString(); + } + /** * Format of backup path: 1. For old style backups: * BASE/REGION/CLUSTER/TOKEN/[SNAPSHOTTIME]/[SST|SNAP|META]/KEYSPACE/COLUMNFAMILY/FILE @@ -116,24 +149,11 @@ private void parseV2Location(String remoteFile) { */ @Override public String getRemotePath() { - StringBuilder buff = new StringBuilder(); if (type == BackupFileType.SST_V2 || type == BackupFileType.META_V2) { - buff.append(getV2Location()); + return getV2Location().toString(); } else { - buff.append(baseDir).append(S3BackupPath.PATH_SEP); // Base dir - buff.append(region).append(S3BackupPath.PATH_SEP); - buff.append(clusterName).append(S3BackupPath.PATH_SEP); // Cluster name - buff.append(token).append(S3BackupPath.PATH_SEP); - buff.append(formatDate(time)).append(S3BackupPath.PATH_SEP); - buff.append(type).append(S3BackupPath.PATH_SEP); - if (BackupFileType.isDataFile(type)) - buff.append(keyspace) - .append(S3BackupPath.PATH_SEP) - .append(columnFamily) - .append(S3BackupPath.PATH_SEP); - buff.append(fileName); + return getV1Location().toString(); } - return buff.toString(); } @Override @@ -151,38 +171,13 @@ public void parseRemote(String remoteFilePath) { if (type == BackupFileType.SST_V2 || type == BackupFileType.META_V2) { parseV2Location(remoteFilePath); } else { - List pieces = parsePrefix(remoteFilePath); - assert pieces.size() >= 7 : "Too few elements in path " + remoteFilePath; - time = parseDate(pieces.get(4)); - type = BackupFileType.valueOf(pieces.get(5)); - if (BackupFileType.isDataFile(type)) { - keyspace = pieces.get(6); - columnFamily = pieces.get(7); - } - // append the rest - fileName = pieces.get(pieces.size() - 1); + parseV1Location(Paths.get(remoteFilePath)); } } @Override public void parsePartialPrefix(String remoteFilePath) { - parsePrefix(remoteFilePath); - } - - private List parsePrefix(String remoteFilePath) { - String[] elements = remoteFilePath.split(String.valueOf(S3BackupPath.PATH_SEP)); - // parse out things which are empty - List pieces = Lists.newArrayList(); - for (String ele : elements) { - if (ele.equals("")) continue; - pieces.add(ele); - } - assert pieces.size() >= 4 : "Too few elements in path " + remoteFilePath; - baseDir = pieces.get(0); - region = pieces.get(1); - clusterName = pieces.get(2); - token = pieces.get(3); - return pieces; + parseV1Prefix(Paths.get(remoteFilePath)); } @Override diff --git a/priam/src/main/java/com/netflix/priam/aws/S3EncryptedFileSystem.java b/priam/src/main/java/com/netflix/priam/aws/S3EncryptedFileSystem.java index 7794e9007..41164f71c 100755 --- a/priam/src/main/java/com/netflix/priam/aws/S3EncryptedFileSystem.java +++ b/priam/src/main/java/com/netflix/priam/aws/S3EncryptedFileSystem.java @@ -74,7 +74,7 @@ protected void downloadFileImpl(Path remotePath, Path localPath) throws BackupRe RangeReadInputStream rris = new RangeReadInputStream( s3Client, - getPrefix(config), + getBucket(), super.getFileSize(remotePath), remotePath.toString())) { /* @@ -87,7 +87,7 @@ protected void downloadFileImpl(Path remotePath, Path localPath) throws BackupRe "Exception encountered downloading " + remotePath + " from S3 bucket " - + getPrefix(config) + + getBucket() + ", Msg: " + e.getMessage(), e); diff --git a/priam/src/main/java/com/netflix/priam/aws/S3FileIterator.java b/priam/src/main/java/com/netflix/priam/aws/S3FileIterator.java index a3c237645..18cc1e499 100644 --- a/priam/src/main/java/com/netflix/priam/aws/S3FileIterator.java +++ b/priam/src/main/java/com/netflix/priam/aws/S3FileIterator.java @@ -38,22 +38,22 @@ public class S3FileIterator implements Iterator { private final Date till; private Iterator iterator; private ObjectListing objectListing; + private String bucket; + private String prefix; public S3FileIterator( Provider pathProvider, AmazonS3 s3Client, + String bucket, String path, Date start, Date till) { this.start = start; this.till = till; this.pathProvider = pathProvider; - ListObjectsRequest listReq = new ListObjectsRequest(); - String[] paths = path.split(String.valueOf(S3BackupPath.PATH_SEP)); - listReq.setBucketName(paths[0]); - listReq.setPrefix(pathProvider.get().remotePrefix(start, till, path)); this.s3Client = s3Client; - objectListing = s3Client.listObjects(listReq); + this.bucket = bucket; + this.prefix = path; iterator = createIterator(); } @@ -70,7 +70,15 @@ public boolean hasNext() { return iterator.hasNext(); } + private void initListing(String bucket, String path) { + ListObjectsRequest listReq = new ListObjectsRequest(); + listReq.setBucketName(bucket); + listReq.setPrefix(pathProvider.get().remotePrefix(start, till, path)); + objectListing = s3Client.listObjects(listReq); + } + private Iterator createIterator() { + if (objectListing == null) initListing(bucket, prefix); List temp = Lists.newArrayList(); for (S3ObjectSummary summary : objectListing.getObjectSummaries()) { AbstractBackupPath path = pathProvider.get(); diff --git a/priam/src/main/java/com/netflix/priam/aws/S3FileSystem.java b/priam/src/main/java/com/netflix/priam/aws/S3FileSystem.java index 5622d857f..b26bf8539 100644 --- a/priam/src/main/java/com/netflix/priam/aws/S3FileSystem.java +++ b/priam/src/main/java/com/netflix/priam/aws/S3FileSystem.java @@ -72,10 +72,7 @@ protected void downloadFileImpl(Path remotePath, Path localPath) throws BackupRe long remoteFileSize = super.getFileSize(remotePath); RangeReadInputStream rris = new RangeReadInputStream( - s3Client, - getPrefix(this.config), - remoteFileSize, - remotePath.toString()); + s3Client, getBucket(), remoteFileSize, remotePath.toString()); final long bufSize = MAX_BUFFERED_IN_STREAM_SIZE > remoteFileSize ? remoteFileSize @@ -88,7 +85,7 @@ protected void downloadFileImpl(Path remotePath, Path localPath) throws BackupRe "Exception encountered downloading " + remotePath + " from S3 bucket " - + getPrefix(config) + + getBucket() + ", Msg: " + e.getMessage(), e); diff --git a/priam/src/main/java/com/netflix/priam/aws/S3FileSystemBase.java b/priam/src/main/java/com/netflix/priam/aws/S3FileSystemBase.java index eb43e7fb2..204456433 100755 --- a/priam/src/main/java/com/netflix/priam/aws/S3FileSystemBase.java +++ b/priam/src/main/java/com/netflix/priam/aws/S3FileSystemBase.java @@ -35,7 +35,6 @@ import java.util.Iterator; import java.util.List; import java.util.concurrent.LinkedBlockingQueue; -import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -82,16 +81,6 @@ public void setS3Client(AmazonS3 client) { s3Client = client; } - /** Get S3 prefix which will be used to locate S3 files */ - String getPrefix(IConfiguration config) { - String prefix; - if (StringUtils.isNotBlank(config.getRestorePrefix())) prefix = config.getRestorePrefix(); - else prefix = config.getBackupPrefix(); - - String[] paths = prefix.split(String.valueOf(S3BackupPath.PATH_SEP)); - return paths[0]; - } - @Override public void cleanup() { @@ -172,15 +161,14 @@ void checkSuccessfulUpload( @Override public long getFileSize(Path remotePath) throws BackupRestoreException { - return s3Client.getObjectMetadata(getPrefix(config), remotePath.toString()) - .getContentLength(); + return s3Client.getObjectMetadata(getBucket(), remotePath.toString()).getContentLength(); } @Override public boolean doesRemoteFileExist(Path remotePath) throws BackupRestoreException { boolean exists = false; try { - exists = s3Client.doesObjectExist(getPrefix(config), remotePath.toString()); + exists = s3Client.doesObjectExist(getBucket(), remotePath.toString()); } catch (AmazonServiceException ase) { // Amazon S3 rejected request throw new BackupRestoreException( @@ -205,12 +193,12 @@ public void shutdown() { @Override public Iterator listPrefixes(Date date) { - return new S3PrefixIterator(config, pathProvider, s3Client, date); + return new S3PrefixIterator(config, pathProvider, s3Client, date, getBucket()); } @Override public Iterator list(String path, Date start, Date till) { - return new S3FileIterator(pathProvider, s3Client, path, start, till); + return new S3FileIterator(pathProvider, s3Client, getBucket(), path, start, till); } final long getChunkSize(Path localPath) throws BackupRestoreException { diff --git a/priam/src/main/java/com/netflix/priam/aws/S3PrefixIterator.java b/priam/src/main/java/com/netflix/priam/aws/S3PrefixIterator.java index 5d4583847..cb7e65851 100644 --- a/priam/src/main/java/com/netflix/priam/aws/S3PrefixIterator.java +++ b/priam/src/main/java/com/netflix/priam/aws/S3PrefixIterator.java @@ -49,14 +49,14 @@ public class S3PrefixIterator implements Iterator { private final SimpleDateFormat datefmt = new SimpleDateFormat("yyyyMMdd"); private ObjectListing objectListing = null; final Date date; - @Inject private InstanceInfo instanceInfo; @Inject public S3PrefixIterator( IConfiguration config, Provider pathProvider, AmazonS3 s3Client, - Date date) { + Date date, + String bucket) { this.config = config; this.pathProvider = pathProvider; this.s3Client = s3Client; @@ -65,9 +65,9 @@ public S3PrefixIterator( if (StringUtils.isNotBlank(config.getRestorePrefix())) path = config.getRestorePrefix(); else path = config.getBackupPrefix(); - String[] paths = path.split(String.valueOf(S3BackupPath.PATH_SEP)); - bucket = paths[0]; - this.clusterPath = remotePrefix(path); + this.bucket = bucket; + AbstractBackupPath abstractBackupPath = pathProvider.get(); + this.clusterPath = abstractBackupPath.clusterPrefix(path); iterator = createIterator(); } @@ -115,23 +115,6 @@ public AbstractBackupPath next() { @Override public void remove() {} - /** Get remote prefix upto the token */ - private String remotePrefix(String location) { - StringBuilder buff = new StringBuilder(); - String[] elements = location.split(String.valueOf(S3BackupPath.PATH_SEP)); - if (elements.length <= 1) { - buff.append(config.getBackupLocation()).append(S3BackupPath.PATH_SEP); - buff.append(instanceInfo.getRegion()).append(S3BackupPath.PATH_SEP); - buff.append(config.getAppName()).append(S3BackupPath.PATH_SEP); - } else { - assert elements.length >= 4 : "Too few elements in path " + location; - buff.append(elements[1]).append(S3BackupPath.PATH_SEP); - buff.append(elements[2]).append(S3BackupPath.PATH_SEP); - buff.append(elements[3]).append(S3BackupPath.PATH_SEP); - } - return buff.toString(); - } - /** Check to see if the path exists for the date */ private boolean pathExistsForDate(String tprefix, String datestr) { ListObjectsRequest listReq = new ListObjectsRequest(); diff --git a/priam/src/main/java/com/netflix/priam/backup/AbstractFileSystem.java b/priam/src/main/java/com/netflix/priam/backup/AbstractFileSystem.java index 07ed2fc4d..35241d87d 100644 --- a/priam/src/main/java/com/netflix/priam/backup/AbstractFileSystem.java +++ b/priam/src/main/java/com/netflix/priam/backup/AbstractFileSystem.java @@ -29,9 +29,11 @@ import com.netflix.spectator.api.patterns.PolledMeter; import java.io.FileNotFoundException; import java.nio.file.Path; +import java.nio.file.Paths; import java.util.Set; import java.util.concurrent.*; import org.apache.commons.io.FileUtils; +import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -45,6 +47,7 @@ public abstract class AbstractFileSystem implements IBackupFileSystem, EventGene private static final Logger logger = LoggerFactory.getLogger(AbstractFileSystem.class); private final CopyOnWriteArrayList> observers = new CopyOnWriteArrayList<>(); + private final IConfiguration configuration; protected final BackupMetrics backupMetrics; private final Set tasksQueued; private final ThreadPoolExecutor fileUploadExecutor; @@ -55,6 +58,7 @@ public AbstractFileSystem( IConfiguration configuration, BackupMetrics backupMetrics, BackupNotificationMgr backupNotificationMgr) { + this.configuration = configuration; this.backupMetrics = backupMetrics; // Add notifications. this.addObserver(backupNotificationMgr); @@ -216,6 +220,17 @@ public Long retriableCall() throws Exception { protected abstract long uploadFileImpl(final Path localPath, final Path remotePath) throws BackupRestoreException; + @Override + public String getBucket() { + Path prefix = Paths.get(configuration.getBackupPrefix()); + + if (StringUtils.isNotBlank(configuration.getRestorePrefix())) { + prefix = Paths.get(configuration.getRestorePrefix()); + } + + return prefix.getName(0).toString(); + } + @Override public final void addObserver(EventObserver observer) { if (observer == null) throw new NullPointerException("observer must not be null."); diff --git a/priam/src/main/java/com/netflix/priam/backup/IBackupFileSystem.java b/priam/src/main/java/com/netflix/priam/backup/IBackupFileSystem.java index 68acf3850..712fccc52 100644 --- a/priam/src/main/java/com/netflix/priam/backup/IBackupFileSystem.java +++ b/priam/src/main/java/com/netflix/priam/backup/IBackupFileSystem.java @@ -114,6 +114,16 @@ Future asyncUploadFile( final boolean deleteAfterSuccessfulUpload) throws FileNotFoundException, RejectedExecutionException, BackupRestoreException; + /** + * Get the bucket where this object should be stored. For local file system this should be empty + * or null. + * + * @return the location of the bucket. + */ + default String getBucket() { + return ""; + } + /** * List all files in the backup location for the specified time range. * diff --git a/priam/src/main/java/com/netflix/priam/resources/BackupServlet.java b/priam/src/main/java/com/netflix/priam/resources/BackupServlet.java index 0b180152c..ecda87563 100644 --- a/priam/src/main/java/com/netflix/priam/resources/BackupServlet.java +++ b/priam/src/main/java/com/netflix/priam/resources/BackupServlet.java @@ -32,7 +32,6 @@ import javax.ws.rs.*; import javax.ws.rs.core.MediaType; import javax.ws.rs.core.Response; -import org.apache.commons.lang3.StringUtils; import org.codehaus.jettison.json.JSONArray; import org.codehaus.jettison.json.JSONException; import org.codehaus.jettison.json.JSONObject; @@ -98,17 +97,6 @@ public Response list( @QueryParam(REST_HEADER_RANGE) String daterange, @QueryParam(REST_HEADER_FILTER) @DefaultValue("") String filter) throws Exception { - Date startTime; - Date endTime; - - if (StringUtils.isBlank(daterange) || daterange.equalsIgnoreCase("default")) { - startTime = new DateTime().minusDays(1).toDate(); - endTime = new DateTime().toDate(); - } else { - String[] restore = daterange.split(","); - startTime = DateUtil.getDate(restore[0]); - endTime = DateUtil.getDate(restore[1]); - } logger.info( "Parameters: {backupPrefix: [{}], daterange: [{}], filter: [{}]}", @@ -116,8 +104,13 @@ public Response list( daterange, filter); + DateUtil.DateRange dateRange = new DateUtil.DateRange(daterange); + Iterator it = - backupFs.list(config.getBackupPrefix(), startTime, endTime); + backupFs.list( + config.getBackupPrefix(), + Date.from(dateRange.getStartTime()), + Date.from(dateRange.getEndTime())); JSONObject object = new JSONObject(); object = constructJsonResponse(object, it, filter); return Response.ok(object.toString(2), MediaType.APPLICATION_JSON).build(); @@ -209,7 +202,9 @@ public Response snapshotsByDate(@PathParam("date") String date) throws Exception return Response.ok(object.toString(), MediaType.APPLICATION_JSON).build(); } - private List getLatestBackupMetadata(Date startTime, Date endTime) { + private List getLatestBackupMetadata(DateUtil.DateRange dateRange) { + Date startTime = Date.from(dateRange.getStartTime()); + Date endTime = Date.from(dateRange.getEndTime()); List backupMetadata = this.completedBkups.locate(endTime); if (backupMetadata != null && !backupMetadata.isEmpty()) return backupMetadata; if (DateUtil.formatyyyyMMdd(startTime).equals(DateUtil.formatyyyyMMdd(endTime))) { @@ -245,28 +240,23 @@ private List getLatestBackupMetadata(Date startTime, Date endTim public Response validateSnapshotByDate(@PathParam("daterange") String daterange) throws Exception { - Date startTime; - Date endTime; - - if (StringUtils.isBlank(daterange) || daterange.equalsIgnoreCase("default")) { - startTime = new DateTime().minusDays(1).toDate(); - endTime = new DateTime().toDate(); - } else { - String[] dates = daterange.split(","); - startTime = DateUtil.getDate(dates[0]); - endTime = DateUtil.getDate(dates[1]); - } + DateUtil.DateRange dateRange = new DateUtil.DateRange(daterange); JSONObject jsonReply = new JSONObject(); - jsonReply.put("inputStartDate", DateUtil.formatyyyyMMddHHmm(startTime)); - jsonReply.put("inputEndDate", DateUtil.formatyyyyMMddHHmm(endTime)); + jsonReply.put( + "inputStartDate", + DateUtil.formatInstant(DateUtil.yyyyMMddHHmm, dateRange.getStartTime())); + jsonReply.put( + "inputEndDate", + DateUtil.formatInstant(DateUtil.yyyyMMddHHmm, dateRange.getEndTime())); logger.info( "Will try to validate latest backup during startTime: {}, and endTime: {}", - DateUtil.formatyyyyMMddHHmm(startTime), - DateUtil.formatyyyyMMddHHmm(endTime)); + dateRange.getStartTime(), + dateRange.getEndTime()); - List metadata = getLatestBackupMetadata(startTime, endTime); - BackupVerificationResult result = backupVerification.verifyBackup(metadata, startTime); + List metadata = getLatestBackupMetadata(dateRange); + BackupVerificationResult result = + backupVerification.verifyBackup(metadata, Date.from(dateRange.getStartTime())); jsonReply.put("snapshotAvailable", result.snapshotAvailable); jsonReply.put("valid", result.valid); jsonReply.put("backupFileListAvailable", result.backupFileListAvail); diff --git a/priam/src/main/java/com/netflix/priam/resources/RestoreServlet.java b/priam/src/main/java/com/netflix/priam/resources/RestoreServlet.java index c4c32e098..6b987798c 100644 --- a/priam/src/main/java/com/netflix/priam/resources/RestoreServlet.java +++ b/priam/src/main/java/com/netflix/priam/resources/RestoreServlet.java @@ -24,8 +24,6 @@ import javax.ws.rs.QueryParam; import javax.ws.rs.core.MediaType; import javax.ws.rs.core.Response; -import org.apache.commons.lang3.StringUtils; -import org.joda.time.DateTime; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -63,20 +61,12 @@ public Response status() throws Exception { @GET @Path("/restore") public Response restore(@QueryParam("daterange") String daterange) throws Exception { - Date startTime; - Date endTime; - - if (StringUtils.isBlank(daterange) || daterange.equalsIgnoreCase("default")) { - startTime = new DateTime().minusDays(1).toDate(); - endTime = new DateTime().toDate(); - } else { - String[] restore = daterange.split(","); - startTime = DateUtil.getDate(restore[0]); - endTime = DateUtil.getDate(restore[1]); - } - - logger.info("Parameters: {startTime: [{}], endTime: [{}]}", startTime, endTime); - restoreObj.restore(startTime, endTime); + DateUtil.DateRange dateRange = new DateUtil.DateRange(daterange); + logger.info( + "Parameters: {startTime: [{}], endTime: [{}]}", + dateRange.getStartTime().toString(), + dateRange.getEndTime().toString()); + restoreObj.restore(Date.from(dateRange.getStartTime()), Date.from(dateRange.getEndTime())); return Response.ok("[\"ok\"]", MediaType.APPLICATION_JSON).build(); } } diff --git a/priam/src/main/java/com/netflix/priam/restore/AbstractRestore.java b/priam/src/main/java/com/netflix/priam/restore/AbstractRestore.java index a5b965b4c..47225d070 100644 --- a/priam/src/main/java/com/netflix/priam/restore/AbstractRestore.java +++ b/priam/src/main/java/com/netflix/priam/restore/AbstractRestore.java @@ -184,10 +184,10 @@ private String getRestorePrefix() { /* * Fetches meta.json used to store snapshots metadata. */ - private void fetchSnapshotMetaFile( - String restorePrefix, List out, Date startTime, Date endTime) - throws IllegalStateException { + private List fetchSnapshotMetaFile( + String restorePrefix, Date startTime, Date endTime) throws IllegalStateException { logger.debug("Looking for snapshot meta file within restore prefix: {}", restorePrefix); + List metas = Lists.newArrayList(); Iterator backupfiles = fs.list(restorePrefix, startTime, endTime); if (!backupfiles.hasNext()) { @@ -201,9 +201,14 @@ private void fetchSnapshotMetaFile( // Since there are now meta file for incrementals as well as snapshot, we need to // find the correct one (i.e. the snapshot meta file (meta.json)) if (path.getFileName().equalsIgnoreCase("meta.json")) { - out.add(path); + metas.add(path); } } + + // Sort the meta files in ascending order. + Collections.sort(metas); + + return metas; } @Override @@ -255,9 +260,8 @@ public void restore(Date startTime, Date endTime) throws Exception { if (dataDir.exists() && dataDir.isDirectory()) FileUtils.cleanDirectory(dataDir); // Try and read the Meta file. - List metas = Lists.newArrayList(); String prefix = getRestorePrefix(); - fetchSnapshotMetaFile(prefix, metas, startTime, endTime); + List metas = fetchSnapshotMetaFile(prefix, startTime, endTime); if (metas.size() == 0) { logger.info("[cass_backup] No snapshot meta file found, Restore Failed."); @@ -266,7 +270,6 @@ public void restore(Date startTime, Date endTime) throws Exception { return; } - Collections.sort(metas); AbstractBackupPath meta = Iterators.getLast(metas.iterator()); logger.info("Snapshot Meta file for restore {}", meta.getRemotePath()); instanceState.getRestoreStatus().setSnapshotMetaFile(meta.getRemotePath()); diff --git a/priam/src/main/java/com/netflix/priam/utils/DateUtil.java b/priam/src/main/java/com/netflix/priam/utils/DateUtil.java index a5a67bc30..7129a6f56 100644 --- a/priam/src/main/java/com/netflix/priam/utils/DateUtil.java +++ b/priam/src/main/java/com/netflix/priam/utils/DateUtil.java @@ -22,6 +22,7 @@ import java.time.ZoneId; import java.time.format.DateTimeFormatter; import java.time.format.DateTimeParseException; +import java.time.temporal.ChronoUnit; import java.util.Date; import javax.inject.Singleton; import org.apache.commons.lang3.StringUtils; @@ -178,4 +179,28 @@ public static final Instant parseInstant(String dateTime) { } return null; } + + public static class DateRange { + Instant startTime; + Instant endTime; + + public DateRange(String daterange) { + if (StringUtils.isBlank(daterange) || daterange.equalsIgnoreCase("default")) { + endTime = getInstant(); + startTime = endTime.minus(1, ChronoUnit.DAYS); + } else { + String[] dates = daterange.split(","); + startTime = parseInstant(dates[0]); + endTime = parseInstant(dates[1]); + } + } + + public Instant getStartTime() { + return startTime; + } + + public Instant getEndTime() { + return endTime; + } + } } diff --git a/priam/src/main/java/com/netflix/priam/utils/TokenManager.java b/priam/src/main/java/com/netflix/priam/utils/TokenManager.java index cad5906f7..1d26077f2 100644 --- a/priam/src/main/java/com/netflix/priam/utils/TokenManager.java +++ b/priam/src/main/java/com/netflix/priam/utils/TokenManager.java @@ -81,7 +81,7 @@ BigInteger initialToken(int size, int position, int offset) { * Creates a token given the following parameter * * @param my_slot -- Slot where this instance has to be. - * @param rac_count -- Rac count is the numeber of RAC's + * @param rac_count -- Rac count is the number of RAC's * @param rac_size -- number of memberships in the rac * @param region -- name of the DC where it this token is created. */ diff --git a/priam/src/test/java/com/netflix/priam/aws/TestS3BackupPath.java b/priam/src/test/java/com/netflix/priam/aws/TestS3BackupPath.java index 2141f14d0..e3774409b 100644 --- a/priam/src/test/java/com/netflix/priam/aws/TestS3BackupPath.java +++ b/priam/src/test/java/com/netflix/priam/aws/TestS3BackupPath.java @@ -77,7 +77,15 @@ public void testV1BackupPathsSST() throws ParseException { logger.info(remotePath); AbstractBackupPath abstractBackupPath2 = pathFactory.get(); abstractBackupPath2.parseRemote(remotePath); - Assert.assertEquals(abstractBackupPath, abstractBackupPath2); + validateAbstractBackupPath(abstractBackupPath, abstractBackupPath2); + Assert.assertEquals(abstractBackupPath.getTime(), abstractBackupPath2.getTime()); + } + + private void validateAbstractBackupPath(AbstractBackupPath abp1, AbstractBackupPath abp2) { + Assert.assertEquals(abp1.getKeyspace(), abp2.getKeyspace()); + Assert.assertEquals(abp1.getColumnFamily(), abp2.getColumnFamily()); + Assert.assertEquals(abp1.getFileName(), abp2.getFileName()); + Assert.assertEquals(abp1.getType(), abp2.getType()); } @Test @@ -109,7 +117,8 @@ public void testV1BackupPathsSnap() throws ParseException { AbstractBackupPath abstractBackupPath2 = pathFactory.get(); abstractBackupPath2.parseRemote(remotePath); - Assert.assertEquals(abstractBackupPath, abstractBackupPath2); + validateAbstractBackupPath(abstractBackupPath, abstractBackupPath2); + Assert.assertEquals(abstractBackupPath.getTime(), abstractBackupPath2.getTime()); } @Test @@ -132,7 +141,8 @@ public void testV1BackupPathsMeta() throws ParseException { AbstractBackupPath abstractBackupPath2 = pathFactory.get(); abstractBackupPath2.parseRemote(remotePath); - Assert.assertEquals(abstractBackupPath, abstractBackupPath2); + validateAbstractBackupPath(abstractBackupPath, abstractBackupPath2); + Assert.assertEquals(abstractBackupPath.getTime(), abstractBackupPath2.getTime()); } @Test @@ -166,10 +176,7 @@ public void testV2BackupPathSST() throws ParseException { AbstractBackupPath abstractBackupPath2 = pathFactory.get(); abstractBackupPath2.parseRemote(remotePath); Assert.assertEquals(now, abstractBackupPath2.getLastModified()); - Assert.assertEquals("mc-1234-Data.db", abstractBackupPath2.getFileName()); - Assert.assertEquals("keyspace1", abstractBackupPath.getKeyspace()); - Assert.assertEquals("columnfamily1", abstractBackupPath.getColumnFamily()); - Assert.assertEquals(BackupFileType.SST_V2, abstractBackupPath.getType()); + validateAbstractBackupPath(abstractBackupPath, abstractBackupPath2); } @Test @@ -197,7 +204,6 @@ public void testV2BackupPathMeta() throws ParseException { AbstractBackupPath abstractBackupPath2 = pathFactory.get(); abstractBackupPath2.parseRemote(remotePath); Assert.assertEquals(now, abstractBackupPath2.getLastModified()); - Assert.assertEquals("meta_v2_201801011201.json", abstractBackupPath2.getFileName()); - Assert.assertEquals(BackupFileType.META_V2, abstractBackupPath.getType()); + validateAbstractBackupPath(abstractBackupPath, abstractBackupPath2); } } diff --git a/priam/src/test/java/com/netflix/priam/backup/TestFileIterator.java b/priam/src/test/java/com/netflix/priam/backup/TestFileIterator.java index 44b8f4e8b..fad6ee2d2 100644 --- a/priam/src/test/java/com/netflix/priam/backup/TestFileIterator.java +++ b/priam/src/test/java/com/netflix/priam/backup/TestFileIterator.java @@ -51,6 +51,7 @@ public class TestFileIterator { private static IConfiguration conf; private static InstanceIdentity factory; private static String region; + private static String bucket = "TESTBUCKET"; @BeforeClass public static void setup() throws InterruptedException, IOException { @@ -130,7 +131,7 @@ public void testIteratorEmptySet() { Date stime = cal.getTime(); cal.add(Calendar.HOUR, 5); Date etime = cal.getTime(); - MockAmazonS3Client.bucketName = "TESTBUCKET"; + MockAmazonS3Client.bucketName = bucket; MockAmazonS3Client.prefix = conf.getBackupLocation() + "/" @@ -145,7 +146,8 @@ public void testIteratorEmptySet() { new S3FileIterator( injector.getProvider(AbstractBackupPath.class), s3client, - "TESTBUCKET", + bucket, + bucket, stime, etime); Set files = new HashSet<>(); @@ -173,7 +175,8 @@ public void testIterator() { new S3FileIterator( injector.getProvider(AbstractBackupPath.class), s3client, - "TESTBUCKET", + bucket, + bucket, startTime, endTime); Set files = new HashSet<>(); @@ -221,7 +224,8 @@ public void testIteratorTruncated() { new S3FileIterator( injector.getProvider(AbstractBackupPath.class), s3client, - "TESTBUCKET", + bucket, + bucket, startTime, endTime); Set files = new HashSet<>(); @@ -270,7 +274,7 @@ public void testIteratorTruncatedOOR() { MockObjectListing.truncated = true; MockObjectListing.firstcall = true; MockObjectListing.simfilter = true; - MockAmazonS3Client.bucketName = "TESTBUCKET"; + MockAmazonS3Client.bucketName = bucket; MockAmazonS3Client.prefix = conf.getBackupLocation() + "/" @@ -285,7 +289,8 @@ public void testIteratorTruncatedOOR() { new S3FileIterator( injector.getProvider(AbstractBackupPath.class), s3client, - "TESTBUCKET", + bucket, + bucket, startTime, endTime); Set files = new HashSet<>(); @@ -345,6 +350,7 @@ public void testRestorePathIteration() { new S3FileIterator( injector.getProvider(AbstractBackupPath.class), s3client, + "RESTOREBUCKET", "RESTOREBUCKET/test_restore_backup/fake-restore-region/fakerestorecluster", startTime, endTime); diff --git a/priam/src/test/java/com/netflix/priam/utils/TestDateUtils.java b/priam/src/test/java/com/netflix/priam/utils/TestDateUtils.java new file mode 100644 index 000000000..e871dcecc --- /dev/null +++ b/priam/src/test/java/com/netflix/priam/utils/TestDateUtils.java @@ -0,0 +1,75 @@ +/* + * Copyright 2018 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.netflix.priam.utils; + +import java.time.Duration; +import java.time.Instant; +import java.time.temporal.ChronoUnit; +import org.junit.Assert; +import org.junit.Test; + +/** Created by aagrawal on 11/29/18. */ +public class TestDateUtils { + + @Test + public void testDateRangeDefault() { + String input = "default"; + Instant now = DateUtil.getInstant(); + DateUtil.DateRange dateRange = new DateUtil.DateRange(input); + + // Start and end should be a day apart. + Assert.assertEquals( + dateRange.getEndTime(), dateRange.getStartTime().plus(1, ChronoUnit.DAYS)); + if (Duration.between(dateRange.getEndTime(), now).getSeconds() > 5) + throw new AssertionError( + String.format( + "End date: %s and now: %s should be almost same", + dateRange.getEndTime(), now)); + } + + @Test + public void testDateRangeEmpty() { + Instant now = DateUtil.getInstant(); + DateUtil.DateRange dateRange = new DateUtil.DateRange(" "); + + // Start and end should be a day apart. + Assert.assertEquals( + dateRange.getEndTime(), dateRange.getStartTime().plus(1, ChronoUnit.DAYS)); + if (Duration.between(dateRange.getEndTime(), now).getSeconds() > 5) + throw new AssertionError( + String.format( + "End date: %s and now: %s should be almost same", + dateRange.getEndTime(), now)); + } + + @Test + public void testDateRangeValues() { + String start = "201801011201"; + String end = "201801051201"; + DateUtil.DateRange dateRange = new DateUtil.DateRange(start + "," + end); + Assert.assertEquals(DateUtil.getDate(start).toInstant(), dateRange.getStartTime()); + Assert.assertEquals(DateUtil.getDate(end).toInstant(), dateRange.getEndTime()); + } + + @Test + public void testDateRangeRandom() { + DateUtil.DateRange dateRange = new DateUtil.DateRange("some,random,values"); + Assert.assertEquals(null, dateRange.getStartTime()); + Assert.assertEquals(null, dateRange.getEndTime()); + } +} From aeecadbafcb88b3e72acb39b89375fe70876478e Mon Sep 17 00:00:00 2001 From: Arun Agrawal Date: Sat, 1 Dec 2018 15:04:32 -0800 Subject: [PATCH 037/228] Refactor S3 and GCS code so individual file system only need to implement generic file iterator --- .../com/netflix/priam/aws/S3FileIterator.java | 111 ------------- .../netflix/priam/aws/S3FileSystemBase.java | 14 +- .../com/netflix/priam/aws/S3Iterator.java | 83 ++++++++++ .../netflix/priam/aws/S3PrefixIterator.java | 128 --------------- .../priam/backup/AbstractFileSystem.java | 55 ++++++- .../netflix/priam/backup/BackupMetadata.java | 5 +- .../priam/backup/IBackupFileSystem.java | 9 ++ .../google/GoogleEncryptedFileSystem.java | 39 +---- .../priam/google/GoogleFileIterator.java | 149 ++++-------------- .../AWSSnsNotificationService.java | 2 +- .../netflix/priam/utils/CassandraMonitor.java | 4 +- .../priam/backup/FakeBackupFileSystem.java | 22 +-- .../priam/backup/NullBackupFileSystem.java | 17 +- .../priam/backup/TestAbstractFileSystem.java | 21 ++- .../priam/backup/TestFileIterator.java | 103 ++++-------- 15 files changed, 250 insertions(+), 512 deletions(-) delete mode 100644 priam/src/main/java/com/netflix/priam/aws/S3FileIterator.java create mode 100644 priam/src/main/java/com/netflix/priam/aws/S3Iterator.java delete mode 100644 priam/src/main/java/com/netflix/priam/aws/S3PrefixIterator.java diff --git a/priam/src/main/java/com/netflix/priam/aws/S3FileIterator.java b/priam/src/main/java/com/netflix/priam/aws/S3FileIterator.java deleted file mode 100644 index 18cc1e499..000000000 --- a/priam/src/main/java/com/netflix/priam/aws/S3FileIterator.java +++ /dev/null @@ -1,111 +0,0 @@ -/* - * Copyright 2013 Netflix, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ -package com.netflix.priam.aws; - -import com.amazonaws.services.s3.AmazonS3; -import com.amazonaws.services.s3.model.ListObjectsRequest; -import com.amazonaws.services.s3.model.ObjectListing; -import com.amazonaws.services.s3.model.S3ObjectSummary; -import com.google.common.collect.Lists; -import com.google.inject.Provider; -import com.netflix.priam.backup.AbstractBackupPath; -import java.util.Date; -import java.util.Iterator; -import java.util.List; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** Iterator representing list of backup files available on S3 */ -public class S3FileIterator implements Iterator { - private static final Logger logger = LoggerFactory.getLogger(S3FileIterator.class); - private final Provider pathProvider; - private final AmazonS3 s3Client; - private final Date start; - private final Date till; - private Iterator iterator; - private ObjectListing objectListing; - private String bucket; - private String prefix; - - public S3FileIterator( - Provider pathProvider, - AmazonS3 s3Client, - String bucket, - String path, - Date start, - Date till) { - this.start = start; - this.till = till; - this.pathProvider = pathProvider; - this.s3Client = s3Client; - this.bucket = bucket; - this.prefix = path; - iterator = createIterator(); - } - - @Override - public boolean hasNext() { - if (iterator.hasNext()) { - return true; - } else { - while (objectListing.isTruncated() && !iterator.hasNext()) { - objectListing = s3Client.listNextBatchOfObjects(objectListing); - iterator = createIterator(); - } - } - return iterator.hasNext(); - } - - private void initListing(String bucket, String path) { - ListObjectsRequest listReq = new ListObjectsRequest(); - listReq.setBucketName(bucket); - listReq.setPrefix(pathProvider.get().remotePrefix(start, till, path)); - objectListing = s3Client.listObjects(listReq); - } - - private Iterator createIterator() { - if (objectListing == null) initListing(bucket, prefix); - List temp = Lists.newArrayList(); - for (S3ObjectSummary summary : objectListing.getObjectSummaries()) { - AbstractBackupPath path = pathProvider.get(); - path.parseRemote(summary.getKey()); - logger.debug( - "New key {} path = {} start: {} end: {} my {}", - summary.getKey(), - path.getRemotePath(), - start, - till, - path.getTime()); - if ((path.getTime().after(start) && path.getTime().before(till)) - || path.getTime().equals(start)) { - temp.add(path); - logger.debug("Added key {}", summary.getKey()); - } - } - return temp.iterator(); - } - - @Override - public AbstractBackupPath next() { - return iterator.next(); - } - - @Override - public void remove() { - throw new IllegalStateException(); - } -} diff --git a/priam/src/main/java/com/netflix/priam/aws/S3FileSystemBase.java b/priam/src/main/java/com/netflix/priam/aws/S3FileSystemBase.java index 204456433..2314c6c86 100755 --- a/priam/src/main/java/com/netflix/priam/aws/S3FileSystemBase.java +++ b/priam/src/main/java/com/netflix/priam/aws/S3FileSystemBase.java @@ -31,7 +31,6 @@ import com.netflix.priam.notification.BackupNotificationMgr; import com.netflix.priam.scheduler.BlockingSubmitThreadPoolExecutor; import java.nio.file.Path; -import java.util.Date; import java.util.Iterator; import java.util.List; import java.util.concurrent.LinkedBlockingQueue; @@ -44,7 +43,6 @@ public abstract class S3FileSystemBase extends AbstractFileSystem { private static final Logger logger = LoggerFactory.getLogger(S3FileSystemBase.class); AmazonS3 s3Client; final IConfiguration config; - private final Provider pathProvider; final ICompression compress; final BlockingSubmitThreadPoolExecutor executor; // a throttling mechanism, we can limit the amount of bytes uploaded to endpoint per second. @@ -56,8 +54,7 @@ public abstract class S3FileSystemBase extends AbstractFileSystem { final IConfiguration config, BackupMetrics backupMetrics, BackupNotificationMgr backupNotificationMgr) { - super(config, backupMetrics, backupNotificationMgr); - this.pathProvider = pathProvider; + super(config, backupMetrics, backupNotificationMgr, pathProvider); this.compress = compress; this.config = config; @@ -192,13 +189,8 @@ public void shutdown() { } @Override - public Iterator listPrefixes(Date date) { - return new S3PrefixIterator(config, pathProvider, s3Client, date, getBucket()); - } - - @Override - public Iterator list(String path, Date start, Date till) { - return new S3FileIterator(pathProvider, s3Client, getBucket(), path, start, till); + public Iterator list(String prefix, String delimiter) { + return new S3Iterator(s3Client, getBucket(), prefix, delimiter); } final long getChunkSize(Path localPath) throws BackupRestoreException { diff --git a/priam/src/main/java/com/netflix/priam/aws/S3Iterator.java b/priam/src/main/java/com/netflix/priam/aws/S3Iterator.java new file mode 100644 index 000000000..4a22ab61e --- /dev/null +++ b/priam/src/main/java/com/netflix/priam/aws/S3Iterator.java @@ -0,0 +1,83 @@ +/* + * Copyright 2018 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.netflix.priam.aws; + +import com.amazonaws.services.s3.AmazonS3; +import com.amazonaws.services.s3.model.ListObjectsRequest; +import com.amazonaws.services.s3.model.ObjectListing; +import com.amazonaws.services.s3.model.S3ObjectSummary; +import com.google.common.collect.Lists; +import java.util.Iterator; +import java.util.List; +import org.apache.commons.lang3.StringUtils; + +/** + * Iterate over the s3 file system. This is really required to find the manifest file for restore + * and downloading incrementals. Created by aagrawal on 11/30/18. + */ +public class S3Iterator implements Iterator { + private Iterator iterator; + private ObjectListing objectListing; + private final AmazonS3 s3Client; + private final String bucket; + private final String prefix; + private final String delimiter; + + public S3Iterator(AmazonS3 s3Client, String bucket, String prefix, String delimiter) { + this.s3Client = s3Client; + this.bucket = bucket; + this.prefix = prefix; + this.delimiter = delimiter; + iterator = createIterator(); + } + + private void initListing() { + ListObjectsRequest listReq = new ListObjectsRequest(); + listReq.setBucketName(bucket); + listReq.setPrefix(prefix); + if (StringUtils.isNotBlank(delimiter)) listReq.setDelimiter(delimiter); + objectListing = s3Client.listObjects(listReq); + } + + private Iterator createIterator() { + if (objectListing == null) initListing(); + List temp = Lists.newArrayList(); + for (S3ObjectSummary summary : objectListing.getObjectSummaries()) { + temp.add(summary.getKey()); + } + return temp.iterator(); + } + + @Override + public boolean hasNext() { + if (iterator.hasNext()) { + return true; + } else { + while (objectListing.isTruncated() && !iterator.hasNext()) { + objectListing = s3Client.listNextBatchOfObjects(objectListing); + iterator = createIterator(); + } + } + return iterator.hasNext(); + } + + @Override + public String next() { + return iterator.next(); + } +} diff --git a/priam/src/main/java/com/netflix/priam/aws/S3PrefixIterator.java b/priam/src/main/java/com/netflix/priam/aws/S3PrefixIterator.java deleted file mode 100644 index cb7e65851..000000000 --- a/priam/src/main/java/com/netflix/priam/aws/S3PrefixIterator.java +++ /dev/null @@ -1,128 +0,0 @@ -/* - * Copyright 2013 Netflix, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ -package com.netflix.priam.aws; - -import com.amazonaws.services.s3.AmazonS3; -import com.amazonaws.services.s3.model.ListObjectsRequest; -import com.amazonaws.services.s3.model.ObjectListing; -import com.google.common.collect.Lists; -import com.google.inject.Inject; -import com.google.inject.Provider; -import com.netflix.priam.backup.AbstractBackupPath; -import com.netflix.priam.config.IConfiguration; -import com.netflix.priam.identity.config.InstanceInfo; -import java.text.SimpleDateFormat; -import java.util.Date; -import java.util.Iterator; -import java.util.List; -import org.apache.commons.lang3.StringUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * Class to iterate over prefixes (S3 Common prefixes) upto the token element in the path. The - * abstract path generated by this class is partial (does not have all data). - */ -public class S3PrefixIterator implements Iterator { - private static final Logger logger = LoggerFactory.getLogger(S3PrefixIterator.class); - private final IConfiguration config; - private final AmazonS3 s3Client; - private final Provider pathProvider; - private Iterator iterator; - - private String bucket = ""; - private String clusterPath = ""; - private final SimpleDateFormat datefmt = new SimpleDateFormat("yyyyMMdd"); - private ObjectListing objectListing = null; - final Date date; - - @Inject - public S3PrefixIterator( - IConfiguration config, - Provider pathProvider, - AmazonS3 s3Client, - Date date, - String bucket) { - this.config = config; - this.pathProvider = pathProvider; - this.s3Client = s3Client; - this.date = date; - String path; - if (StringUtils.isNotBlank(config.getRestorePrefix())) path = config.getRestorePrefix(); - else path = config.getBackupPrefix(); - - this.bucket = bucket; - AbstractBackupPath abstractBackupPath = pathProvider.get(); - this.clusterPath = abstractBackupPath.clusterPrefix(path); - iterator = createIterator(); - } - - private void initListing() { - ListObjectsRequest listReq = new ListObjectsRequest(); - // Get list of tokens - listReq.setBucketName(bucket); - listReq.setPrefix(clusterPath); - listReq.setDelimiter(String.valueOf(AbstractBackupPath.PATH_SEP)); - logger.info("Using cluster prefix for searching tokens: {}", clusterPath); - objectListing = s3Client.listObjects(listReq); - } - - private Iterator createIterator() { - if (objectListing == null) initListing(); - List temp = Lists.newArrayList(); - for (String summary : objectListing.getCommonPrefixes()) { - if (pathExistsForDate(summary, datefmt.format(date))) { - AbstractBackupPath path = pathProvider.get(); - path.parsePartialPrefix(summary); - temp.add(path); - } - } - return temp.iterator(); - } - - @Override - public boolean hasNext() { - if (iterator.hasNext()) { - return true; - } else { - while (objectListing.isTruncated() && !iterator.hasNext()) { - objectListing = s3Client.listNextBatchOfObjects(objectListing); - iterator = createIterator(); - } - } - return iterator.hasNext(); - } - - @Override - public AbstractBackupPath next() { - return iterator.next(); - } - - @Override - public void remove() {} - - /** Check to see if the path exists for the date */ - private boolean pathExistsForDate(String tprefix, String datestr) { - ListObjectsRequest listReq = new ListObjectsRequest(); - // Get list of tokens - listReq.setBucketName(bucket); - listReq.setPrefix(tprefix + datestr); - ObjectListing listing; - listing = s3Client.listObjects(listReq); - return listing.getObjectSummaries().size() > 0; - } -} diff --git a/priam/src/main/java/com/netflix/priam/backup/AbstractFileSystem.java b/priam/src/main/java/com/netflix/priam/backup/AbstractFileSystem.java index 35241d87d..233565a45 100644 --- a/priam/src/main/java/com/netflix/priam/backup/AbstractFileSystem.java +++ b/priam/src/main/java/com/netflix/priam/backup/AbstractFileSystem.java @@ -18,6 +18,7 @@ package com.netflix.priam.backup; import com.google.inject.Inject; +import com.google.inject.Provider; import com.netflix.priam.config.IConfiguration; import com.netflix.priam.merics.BackupMetrics; import com.netflix.priam.notification.BackupEvent; @@ -27,11 +28,16 @@ import com.netflix.priam.scheduler.BlockingSubmitThreadPoolExecutor; import com.netflix.priam.utils.BoundedExponentialRetryCallable; import com.netflix.spectator.api.patterns.PolledMeter; +import java.io.File; import java.io.FileNotFoundException; import java.nio.file.Path; import java.nio.file.Paths; +import java.util.Date; +import java.util.Iterator; import java.util.Set; import java.util.concurrent.*; +import org.apache.commons.collections4.iterators.FilterIterator; +import org.apache.commons.collections4.iterators.TransformIterator; import org.apache.commons.io.FileUtils; import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; @@ -45,6 +51,7 @@ */ public abstract class AbstractFileSystem implements IBackupFileSystem, EventGenerator { private static final Logger logger = LoggerFactory.getLogger(AbstractFileSystem.class); + protected final Provider pathProvider; private final CopyOnWriteArrayList> observers = new CopyOnWriteArrayList<>(); private final IConfiguration configuration; @@ -57,9 +64,11 @@ public abstract class AbstractFileSystem implements IBackupFileSystem, EventGene public AbstractFileSystem( IConfiguration configuration, BackupMetrics backupMetrics, - BackupNotificationMgr backupNotificationMgr) { + BackupNotificationMgr backupNotificationMgr, + Provider pathProvider) { this.configuration = configuration; this.backupMetrics = backupMetrics; + this.pathProvider = pathProvider; // Add notifications. this.addObserver(backupNotificationMgr); tasksQueued = new ConcurrentHashMap<>().newKeySet(); @@ -222,13 +231,55 @@ protected abstract long uploadFileImpl(final Path localPath, final Path remotePa @Override public String getBucket() { + return getPrefix().getName(0).toString(); + } + + protected Path getPrefix() { Path prefix = Paths.get(configuration.getBackupPrefix()); if (StringUtils.isNotBlank(configuration.getRestorePrefix())) { prefix = Paths.get(configuration.getRestorePrefix()); } - return prefix.getName(0).toString(); + return prefix; + } + + @Override + public Iterator listPrefixes(Date date) { + String prefix = pathProvider.get().clusterPrefix(getPrefix().toString()); + Iterator fileIterator = list(prefix, File.pathSeparator); + + //noinspection unchecked + return new TransformIterator( + fileIterator, + remotePath -> { + AbstractBackupPath abstractBackupPath = pathProvider.get(); + abstractBackupPath.parsePartialPrefix(remotePath.toString()); + return abstractBackupPath; + }); + } + + @Override + public Iterator list(String path, Date start, Date till) { + String prefix = pathProvider.get().remotePrefix(start, till, path); + Iterator fileIterator = list(prefix, null); + + @SuppressWarnings("unchecked") + TransformIterator transformIterator = + new TransformIterator( + fileIterator, + remotePath -> { + AbstractBackupPath abstractBackupPath = pathProvider.get(); + abstractBackupPath.parseRemote(remotePath.toString()); + return abstractBackupPath; + }); + + return new FilterIterator<>( + transformIterator, + abstractBackupPath -> + (abstractBackupPath.getTime().after(start) + && abstractBackupPath.getTime().before(till)) + || abstractBackupPath.getTime().equals(start)); } @Override diff --git a/priam/src/main/java/com/netflix/priam/backup/BackupMetadata.java b/priam/src/main/java/com/netflix/priam/backup/BackupMetadata.java index 74cf5c587..ff81804a4 100644 --- a/priam/src/main/java/com/netflix/priam/backup/BackupMetadata.java +++ b/priam/src/main/java/com/netflix/priam/backup/BackupMetadata.java @@ -35,9 +35,8 @@ public BackupMetadata(String token, Date start) throws Exception { if (start == null || token == null || StringUtils.isEmpty(token)) throw new Exception( String.format( - "Invalid Input: Token: {} or start date:{} is null or empty.", - token, - start)); + "Invalid Input: Token: %s or start date: %s is null or empty.", + token, start)); this.snapshotDate = DateUtil.formatyyyyMMdd(start); this.token = token; diff --git a/priam/src/main/java/com/netflix/priam/backup/IBackupFileSystem.java b/priam/src/main/java/com/netflix/priam/backup/IBackupFileSystem.java index 712fccc52..eca2029b0 100644 --- a/priam/src/main/java/com/netflix/priam/backup/IBackupFileSystem.java +++ b/priam/src/main/java/com/netflix/priam/backup/IBackupFileSystem.java @@ -138,6 +138,15 @@ default String getBucket() { /** Get a list of prefixes for the cluster available in backup for the specified date */ Iterator listPrefixes(Date date); + /** + * List all the files with the given prefix and delimiter. This should never return null. + * + * @param prefix Common prefix of the elements to search in the backup file system. + * @param delimiter All the object will end with this delimiter. + * @return the iterator on the backup file system containing path of the files. + */ + Iterator list(String prefix, String delimiter); + /** Runs cleanup or set retention */ void cleanup(); diff --git a/priam/src/main/java/com/netflix/priam/google/GoogleEncryptedFileSystem.java b/priam/src/main/java/com/netflix/priam/google/GoogleEncryptedFileSystem.java index 4dd53c38c..cdf561dde 100755 --- a/priam/src/main/java/com/netflix/priam/google/GoogleEncryptedFileSystem.java +++ b/priam/src/main/java/com/netflix/priam/google/GoogleEncryptedFileSystem.java @@ -24,7 +24,6 @@ import com.google.inject.Inject; import com.google.inject.Provider; import com.google.inject.name.Named; -import com.netflix.priam.aws.S3BackupPath; import com.netflix.priam.backup.AbstractBackupPath; import com.netflix.priam.backup.AbstractFileSystem; import com.netflix.priam.backup.BackupRestoreException; @@ -37,10 +36,8 @@ import java.nio.file.Path; import java.util.ArrayList; import java.util.Collection; -import java.util.Date; import java.util.Iterator; import org.apache.commons.io.IOUtils; -import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -56,8 +53,6 @@ public class GoogleEncryptedFileSystem extends AbstractFileSystem { private Credential credential; private Storage gcsStorageHandle; private Storage.Objects objectsResoruceHandle = null; - - private final Provider pathProvider; private String srcBucketName; private final IConfiguration config; @@ -71,9 +66,8 @@ public GoogleEncryptedFileSystem( @Named("gcscredential") ICredentialGeneric credential, BackupMetrics backupMetrics, BackupNotificationMgr backupNotificationManager) { - super(config, backupMetrics, backupNotificationManager); + super(config, backupMetrics, backupNotificationManager, pathProvider); this.backupMetrics = backupMetrics; - this.pathProvider = pathProvider; this.config = config; this.gcsCredential = credential; @@ -84,15 +78,7 @@ public GoogleEncryptedFileSystem( "Unable to create a handle to the Google Http tranport", e); } - this.srcBucketName = getSourcebucket(getPathPrefix()); - } - - /* - * @param pathprefix - the absolute path (including bucket name) to the object. - */ - private String getSourcebucket(String pathPrefix) { - String[] paths = pathPrefix.split(String.valueOf(S3BackupPath.PATH_SEP)); - return paths[0]; + this.srcBucketName = getBucket(); } private Storage.Objects constructObjectResourceHandle() { @@ -189,7 +175,7 @@ private Credential constructGcsCredential() throws Exception { @Override protected void downloadFileImpl(Path remotePath, Path localPath) throws BackupRestoreException { - String objectName = parseObjectname(getPathPrefix()); + String objectName = parseObjectname(getPrefix().toString()); com.google.api.services.storage.Storage.Objects.Get get; try { @@ -225,14 +211,8 @@ protected void downloadFileImpl(Path remotePath, Path localPath) throws BackupRe } @Override - public Iterator list(String path, Date start, Date till) { - return new GoogleFileIterator(pathProvider, constructGcsStorageHandle(), path, start, till); - } - - @Override - public Iterator listPrefixes(Date date) { - // TODO Auto-generated method stub - return null; + public Iterator list(String prefix, String delimiter) { + return new GoogleFileIterator(constructGcsStorageHandle(), prefix, null); } @Override @@ -255,15 +235,6 @@ public long getFileSize(Path remotePath) throws BackupRestoreException { return 0; } - /** Get restore prefix which will be used to locate GVS files */ - private String getPathPrefix() { - String prefix; - if (StringUtils.isNotBlank(config.getRestorePrefix())) prefix = config.getRestorePrefix(); - else prefix = config.getBackupPrefix(); - - return prefix; - } - /* * @param pathPrefix * @return objectName diff --git a/priam/src/main/java/com/netflix/priam/google/GoogleFileIterator.java b/priam/src/main/java/com/netflix/priam/google/GoogleFileIterator.java index fe185cf59..ac06e5592 100755 --- a/priam/src/main/java/com/netflix/priam/google/GoogleFileIterator.java +++ b/priam/src/main/java/com/netflix/priam/google/GoogleFileIterator.java @@ -16,171 +16,86 @@ import com.google.api.services.storage.Storage; import com.google.api.services.storage.model.StorageObject; import com.google.common.collect.Lists; -import com.google.inject.Provider; -import com.netflix.priam.aws.S3BackupPath; -import com.netflix.priam.backup.AbstractBackupPath; import java.io.IOException; -import java.util.Date; import java.util.Iterator; import java.util.List; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; /* * Represents a list of objects within Google Cloud Storage (GCS) */ -public class GoogleFileIterator implements Iterator { - private static final Logger logger = LoggerFactory.getLogger(GoogleFileIterator.class); - - private final Date start; - private final Date till; - private Iterator iterator; - private final Provider pathProvider; +public class GoogleFileIterator implements Iterator { + private Iterator iterator; private String bucketName; - + private String prefix; private Storage.Objects objectsResoruceHandle = null; private Storage.Objects.List listObjectsSrvcHandle = null; private com.google.api.services.storage.model.Objects objectsContainerHandle = null; - private String pathWithinBucket; - - private String nextPageToken; - - /* - * @param pathProvider - * @param gcsStorageHandle - a means to perform operations within the destination storage. - * @param path - metadata about where the object exist within the destination. - * @param start - timeframe of object to restore - * @param till - timeframe of object to restore - */ - public GoogleFileIterator( - Provider pathProvider, - Storage gcsStorageHandle, - String path, - Date start, - Date till) { - - this.start = start; - this.till = till; - this.pathProvider = pathProvider; + public GoogleFileIterator(Storage gcsStorageHandle, String bucket, String prefix) { this.objectsResoruceHandle = gcsStorageHandle.objects(); + this.bucketName = bucket; + this.prefix = prefix; - if (path == null) { - throw new NullPointerException("Path of object to fetch is null"); - } - - String[] paths = path.split(String.valueOf(S3BackupPath.PATH_SEP)); - if (paths.length < 1) { - throw new IllegalStateException("Path of object to fetch is invalid. Path: " + path); + try { // == Get the initial page of results + this.iterator = createIterator(); + } catch (Exception e) { + throw new RuntimeException( + "Exception encountered fetching elements, msg: ." + e.getLocalizedMessage(), e); } + } - this.bucketName = paths[0]; - this.pathWithinBucket = pathProvider.get().remotePrefix(start, till, path); - - logger.info( - "Listing objects from GCS: {}, prefix: {}", this.bucketName, this.pathWithinBucket); - + private void initListing() { try { this.listObjectsSrvcHandle = objectsResoruceHandle.list(bucketName); // == list objects within bucket - + // fetch elements within bucket that matches this prefix + this.listObjectsSrvcHandle.setPrefix(this.prefix); } catch (IOException e) { throw new RuntimeException("Unable to get gcslist handle to bucket: " + bucketName, e); } - - // fetch elements within bucket that matches this prefix - this.listObjectsSrvcHandle.setPrefix(this.pathWithinBucket); - - try { // == Get the initial page of results - - this.iterator = createIterator(); - - } catch (Exception e) { - throw new RuntimeException( - "Exception encountered fetching elements, msg: ." + e.getLocalizedMessage(), e); - } } /* * Fetch a page of results */ - private Iterator createIterator() throws Exception { - List temp = Lists.newArrayList(); // a container of results + private Iterator createIterator() throws Exception { + if (listObjectsSrvcHandle == null) initListing(); + List temp = Lists.newArrayList(); // a container of results // Sends the metadata request to the server and returns the parsed metadata response. this.objectsContainerHandle = listObjectsSrvcHandle.execute(); - for (StorageObject object : - this.objectsContainerHandle.getItems()) { // processing a page of results - String fileName = GoogleEncryptedFileSystem.parseObjectname(object.getName()); - logger.debug( - "id: {}, parse file name: {}, name: {}", - object.getId(), - fileName, - object.getName()); - - // e.g. of objectname: - // prod_backup/us-east-1/cass_account/113427455640312821154458202479064646083/201408250801/META/meta.json - - AbstractBackupPath path = pathProvider.get(); - path.parseRemote(object.getName()); - logger.debug( - "New key {} path = {} start: {} end: {} my {}", - object.getName(), - path.getRemotePath(), - start, - till, - path.getTime()); - if ((path.getTime().after(start) && path.getTime().before(till)) - || path.getTime().equals(start)) { - temp.add(path); - logger.debug("Added key {}", object.getName()); - } + for (StorageObject object : this.objectsContainerHandle.getItems()) { + // processing a page of results + temp.add(object.getName()); } - - this.nextPageToken = this.objectsContainerHandle.getNextPageToken(); - return temp.iterator(); } @Override public boolean hasNext() { - if (this.iterator == null) return false; - if (this.iterator.hasNext()) { return true; } - if (this.nextPageToken == null) { // there is no additional results - return false; - } - - try { // if here, you have iterated through all elements of the previous page, now, get the - // next page of results - - this.listObjectsSrvcHandle.setPageToken(this.nextPageToken); - // get the next page of results - this.iterator = createIterator(); - - } catch (Exception e) { - throw new RuntimeException( - "Exception encountered fetching elements, see previous messages for details.", - e); - } + while (this.objectsContainerHandle.getNextPageToken() != null && !iterator.hasNext()) + try { // if here, you have iterated through all elements of the previous page, now, get + // the next page of results + this.listObjectsSrvcHandle.setPageToken(objectsContainerHandle.getNextPageToken()); + this.iterator = createIterator(); + } catch (Exception e) { + throw new RuntimeException( + "Exception encountered fetching elements, see previous messages for details.", + e); + } return this.iterator.hasNext(); } @Override - public AbstractBackupPath next() { + public String next() { return iterator.next(); } - - @Override - public void remove() { - // TODO Auto-generated method stub - - } } diff --git a/priam/src/main/java/com/netflix/priam/notification/AWSSnsNotificationService.java b/priam/src/main/java/com/netflix/priam/notification/AWSSnsNotificationService.java index 8774ac881..44cdf7c41 100644 --- a/priam/src/main/java/com/netflix/priam/notification/AWSSnsNotificationService.java +++ b/priam/src/main/java/com/netflix/priam/notification/AWSSnsNotificationService.java @@ -85,7 +85,7 @@ public PublishResult retriableCall() throws Exception { } catch (Exception e) { logger.error( String.format( - "Exhausted retries. Publishing notification metric for failure and moving on. Failed msg to publish: {}", + "Exhausted retries. Publishing notification metric for failure and moving on. Failed msg to publish: %s", msg), e); backupMetrics.incrementSnsNotificationFailure(); diff --git a/priam/src/main/java/com/netflix/priam/utils/CassandraMonitor.java b/priam/src/main/java/com/netflix/priam/utils/CassandraMonitor.java index 6ed146c7b..b4772489d 100644 --- a/priam/src/main/java/com/netflix/priam/utils/CassandraMonitor.java +++ b/priam/src/main/java/com/netflix/priam/utils/CassandraMonitor.java @@ -148,12 +148,12 @@ private void checkDirectory(String directory) { private void checkDirectory(File directory) { if (!directory.exists()) throw new IllegalStateException( - String.format("Directory: {} does not exist", directory)); + String.format("Directory: %s does not exist", directory)); if (!directory.canRead() || !directory.canWrite()) throw new IllegalStateException( String.format( - "Directory: {} does not have read/write permissions.", directory)); + "Directory: %s does not have read/write permissions.", directory)); } public static TaskTimer getTimer() { diff --git a/priam/src/test/java/com/netflix/priam/backup/FakeBackupFileSystem.java b/priam/src/test/java/com/netflix/priam/backup/FakeBackupFileSystem.java index 1429d66eb..1acdec042 100644 --- a/priam/src/test/java/com/netflix/priam/backup/FakeBackupFileSystem.java +++ b/priam/src/test/java/com/netflix/priam/backup/FakeBackupFileSystem.java @@ -29,6 +29,7 @@ import java.io.IOException; import java.nio.file.Path; import java.util.*; +import org.apache.commons.collections4.iterators.TransformIterator; import org.json.simple.JSONArray; @Singleton @@ -46,15 +47,16 @@ public class FakeBackupFileSystem extends AbstractFileSystem { public FakeBackupFileSystem( IConfiguration configuration, BackupMetrics backupMetrics, - BackupNotificationMgr backupNotificationMgr) { - super(configuration, backupMetrics, backupNotificationMgr); + BackupNotificationMgr backupNotificationMgr, + Provider pathProvider) { + super(configuration, backupMetrics, backupNotificationMgr, pathProvider); } public void setupTest(List files) { clearTest(); flist = new ArrayList<>(); for (String file : files) { - S3BackupPath path = pathProvider.get(); + AbstractBackupPath path = pathProvider.get(); path.parseRemote(file); flist.add(path); } @@ -75,7 +77,7 @@ public void clearTest() { } public void addFile(String file) { - S3BackupPath path = pathProvider.get(); + AbstractBackupPath path = pathProvider.get(); path.parseRemote(file); flist.add(path); } @@ -105,6 +107,11 @@ public Iterator list(String bucket, Date start, Date till) { return tmpList.iterator(); } + @Override + public Iterator list(String prefix, String delimiter) { + return new TransformIterator<>(flist.iterator(), AbstractBackupPath::getRemotePath); + } + public void shutdown() { // nop } @@ -114,12 +121,6 @@ public long getFileSize(Path remotePath) throws BackupRestoreException { return 0; } - @Override - public Iterator listPrefixes(Date date) { - // TODO Auto-generated method stub - return null; - } - @Override public void cleanup() { // TODO Auto-generated method stub @@ -127,7 +128,6 @@ public void cleanup() { @Override protected void downloadFileImpl(Path remotePath, Path localPath) throws BackupRestoreException { - // if (path.type == BackupFileType.META) { // List all files and generate the file try (FileWriter fr = new FileWriter(localPath.toFile())) { diff --git a/priam/src/test/java/com/netflix/priam/backup/NullBackupFileSystem.java b/priam/src/test/java/com/netflix/priam/backup/NullBackupFileSystem.java index af74dab37..b248b4f38 100644 --- a/priam/src/test/java/com/netflix/priam/backup/NullBackupFileSystem.java +++ b/priam/src/test/java/com/netflix/priam/backup/NullBackupFileSystem.java @@ -18,11 +18,12 @@ package com.netflix.priam.backup; import com.google.inject.Inject; +import com.google.inject.Provider; import com.netflix.priam.config.IConfiguration; import com.netflix.priam.merics.BackupMetrics; import com.netflix.priam.notification.BackupNotificationMgr; import java.nio.file.Path; -import java.util.Date; +import java.util.Collections; import java.util.Iterator; public class NullBackupFileSystem extends AbstractFileSystem { @@ -31,13 +32,9 @@ public class NullBackupFileSystem extends AbstractFileSystem { public NullBackupFileSystem( IConfiguration configuration, BackupMetrics backupMetrics, - BackupNotificationMgr backupNotificationMgr) { - super(configuration, backupMetrics, backupNotificationMgr); - } - - @Override - public Iterator list(String bucket, Date start, Date till) { - return null; + BackupNotificationMgr backupNotificationMgr, + Provider pathProvider) { + super(configuration, backupMetrics, backupNotificationMgr, pathProvider); } public void shutdown() { @@ -50,8 +47,8 @@ public long getFileSize(Path remotePath) throws BackupRestoreException { } @Override - public Iterator listPrefixes(Date date) { - return null; + public Iterator list(String prefix, String delimiter) { + return Collections.emptyIterator(); } @Override diff --git a/priam/src/test/java/com/netflix/priam/backup/TestAbstractFileSystem.java b/priam/src/test/java/com/netflix/priam/backup/TestAbstractFileSystem.java index 73fc7d984..b05fe0d78 100644 --- a/priam/src/test/java/com/netflix/priam/backup/TestAbstractFileSystem.java +++ b/priam/src/test/java/com/netflix/priam/backup/TestAbstractFileSystem.java @@ -20,6 +20,7 @@ import com.google.inject.Guice; import com.google.inject.Inject; import com.google.inject.Injector; +import com.google.inject.Provider; import com.netflix.priam.config.IConfiguration; import com.netflix.priam.merics.BackupMetrics; import com.netflix.priam.notification.BackupNotificationMgr; @@ -48,7 +49,7 @@ public class TestAbstractFileSystem { private IConfiguration configuration; private BackupMetrics backupMetrics; private BackupNotificationMgr backupNotificationMgr; - private AbstractFileSystem failureFileSystem; + private FailureFileSystem failureFileSystem; private MyFileSystem myFileSystem; @Before @@ -61,13 +62,17 @@ public void setBackupMetrics() { backupNotificationMgr = injector.getInstance(BackupNotificationMgr.class); backupMetrics = injector.getInstance(BackupMetrics.class); + Provider pathProvider = injector.getProvider(AbstractBackupPath.class); if (failureFileSystem == null) failureFileSystem = - new FailureFileSystem(configuration, backupMetrics, backupNotificationMgr); + new FailureFileSystem( + configuration, backupMetrics, backupNotificationMgr, pathProvider); if (myFileSystem == null) - myFileSystem = new MyFileSystem(configuration, backupMetrics, backupNotificationMgr); + myFileSystem = + new MyFileSystem( + configuration, backupMetrics, backupNotificationMgr, pathProvider); BackupFileUtils.cleanupDir(Paths.get(configuration.getDataFileLocation())); } @@ -295,8 +300,9 @@ class FailureFileSystem extends NullBackupFileSystem { public FailureFileSystem( IConfiguration configuration, BackupMetrics backupMetrics, - BackupNotificationMgr backupNotificationMgr) { - super(configuration, backupMetrics, backupNotificationMgr); + BackupNotificationMgr backupNotificationMgr, + Provider pathProvider) { + super(configuration, backupMetrics, backupNotificationMgr, pathProvider); } @Override @@ -324,8 +330,9 @@ class MyFileSystem extends NullBackupFileSystem { public MyFileSystem( IConfiguration configuration, BackupMetrics backupMetrics, - BackupNotificationMgr backupNotificationMgr) { - super(configuration, backupMetrics, backupNotificationMgr); + BackupNotificationMgr backupNotificationMgr, + Provider pathProvider) { + super(configuration, backupMetrics, backupNotificationMgr, pathProvider); } @Override diff --git a/priam/src/test/java/com/netflix/priam/backup/TestFileIterator.java b/priam/src/test/java/com/netflix/priam/backup/TestFileIterator.java index fad6ee2d2..f66d44168 100644 --- a/priam/src/test/java/com/netflix/priam/backup/TestFileIterator.java +++ b/priam/src/test/java/com/netflix/priam/backup/TestFileIterator.java @@ -24,10 +24,12 @@ import com.amazonaws.services.s3.model.S3ObjectSummary; import com.google.inject.Guice; import com.google.inject.Injector; -import com.netflix.priam.aws.S3FileIterator; +import com.netflix.priam.aws.S3FileSystem; import com.netflix.priam.config.IConfiguration; import com.netflix.priam.identity.InstanceIdentity; import java.io.IOException; +import java.nio.file.Path; +import java.nio.file.Paths; import java.util.*; import mockit.Mock; import mockit.MockUp; @@ -47,7 +49,7 @@ public class TestFileIterator { private static Calendar cal; private static AmazonS3Client s3client; - + private static S3FileSystem s3FileSystem; private static IConfiguration conf; private static InstanceIdentity factory; private static String region; @@ -62,6 +64,8 @@ public static void setup() throws InterruptedException, IOException { conf = injector.getInstance(IConfiguration.class); factory = injector.getInstance(InstanceIdentity.class); region = factory.getInstanceInfo().getRegion(); + s3FileSystem = injector.getInstance(S3FileSystem.class); + s3FileSystem.setS3Client(s3client); cal = Calendar.getInstance(); cal.set(2011, 7, 11, 0, 30, 0); @@ -124,6 +128,14 @@ public boolean isTruncated() { } } + private Path getClusterPrefix() { + return Paths.get( + conf.getBackupLocation(), + factory.getInstanceInfo().getRegion(), + conf.getAppName(), + factory.getInstance().getToken()); + } + @Test public void testIteratorEmptySet() { cal.set(2011, 7, 11, 6, 1, 0); @@ -132,24 +144,9 @@ public void testIteratorEmptySet() { cal.add(Calendar.HOUR, 5); Date etime = cal.getTime(); MockAmazonS3Client.bucketName = bucket; - MockAmazonS3Client.prefix = - conf.getBackupLocation() - + "/" - + region - + "/" - + conf.getAppName() - + "/" - + factory.getInstance().getToken(); - MockAmazonS3Client.prefix += "/20110811"; + MockAmazonS3Client.prefix = getClusterPrefix() + "/20110811"; - S3FileIterator fileIterator = - new S3FileIterator( - injector.getProvider(AbstractBackupPath.class), - s3client, - bucket, - bucket, - stime, - etime); + Iterator fileIterator = s3FileSystem.list(bucket, stime, etime); Set files = new HashSet<>(); while (fileIterator.hasNext()) files.add(fileIterator.next().getRemotePath()); Assert.assertEquals(0, files.size()); @@ -161,24 +158,10 @@ public void testIterator() { MockObjectListing.firstcall = true; MockObjectListing.simfilter = false; MockAmazonS3Client.bucketName = "TESTBUCKET"; - MockAmazonS3Client.prefix = - conf.getBackupLocation() - + "/" - + region - + "/" - + conf.getAppName() - + "/" - + factory.getInstance().getToken(); - MockAmazonS3Client.prefix += "/20110811"; + MockAmazonS3Client.prefix = getClusterPrefix() + "/20110811"; + + Iterator fileIterator = s3FileSystem.list(bucket, startTime, endTime); - S3FileIterator fileIterator = - new S3FileIterator( - injector.getProvider(AbstractBackupPath.class), - s3client, - bucket, - bucket, - startTime, - endTime); Set files = new HashSet<>(); while (fileIterator.hasNext()) files.add(fileIterator.next().getRemotePath()); Assert.assertEquals(3, files.size()); @@ -210,24 +193,10 @@ public void testIteratorTruncated() { MockObjectListing.firstcall = true; MockObjectListing.simfilter = false; MockAmazonS3Client.bucketName = "TESTBUCKET"; - MockAmazonS3Client.prefix = - conf.getBackupLocation() - + "/" - + region - + "/" - + conf.getAppName() - + "/" - + factory.getInstance().getToken(); - MockAmazonS3Client.prefix += "/20110811"; + MockAmazonS3Client.prefix = getClusterPrefix() + "/20110811"; + + Iterator fileIterator = s3FileSystem.list(bucket, startTime, endTime); - S3FileIterator fileIterator = - new S3FileIterator( - injector.getProvider(AbstractBackupPath.class), - s3client, - bucket, - bucket, - startTime, - endTime); Set files = new HashSet<>(); while (fileIterator.hasNext()) files.add(fileIterator.next().getRemotePath()); Assert.assertEquals(5, files.size()); @@ -275,24 +244,10 @@ public void testIteratorTruncatedOOR() { MockObjectListing.firstcall = true; MockObjectListing.simfilter = true; MockAmazonS3Client.bucketName = bucket; - MockAmazonS3Client.prefix = - conf.getBackupLocation() - + "/" - + region - + "/" - + conf.getAppName() - + "/" - + factory.getInstance().getToken(); - MockAmazonS3Client.prefix += "/20110811"; + MockAmazonS3Client.prefix = getClusterPrefix() + "/20110811"; + + Iterator fileIterator = s3FileSystem.list(bucket, startTime, endTime); - S3FileIterator fileIterator = - new S3FileIterator( - injector.getProvider(AbstractBackupPath.class), - s3client, - bucket, - bucket, - startTime, - endTime); Set files = new HashSet<>(); while (fileIterator.hasNext()) files.add(fileIterator.next().getRemotePath()); Assert.assertEquals(2, files.size()); @@ -346,14 +301,12 @@ public void testRestorePathIteration() { + factory.getInstance().getToken(); MockAmazonS3Client.prefix += "/20110811"; - S3FileIterator fileIterator = - new S3FileIterator( - injector.getProvider(AbstractBackupPath.class), - s3client, - "RESTOREBUCKET", + Iterator fileIterator = + s3FileSystem.list( "RESTOREBUCKET/test_restore_backup/fake-restore-region/fakerestorecluster", startTime, endTime); + Set files = new HashSet<>(); while (fileIterator.hasNext()) files.add(fileIterator.next().getRemotePath()); while (fileIterator.hasNext()) files.add(fileIterator.next().getRemotePath()); From 220ac71b30e2d068ea2f39d0ebc08606d7a1aca8 Mon Sep 17 00:00:00 2001 From: Arun Agrawal Date: Sat, 1 Dec 2018 22:10:36 -0800 Subject: [PATCH 038/228] Some common code refactoring --- .../java/com/netflix/priam/PriamServer.java | 3 +- ...3BackupPath.java => RemoteBackupPath.java} | 20 +++++--- .../priam/backup/AbstractBackupPath.java | 4 +- .../com/netflix/priam/utils/SystemUtils.java | 49 ++----------------- .../priam/backup/FakeBackupFileSystem.java | 4 +- .../netflix/priam/backup/TestBackupFile.java | 8 +-- .../priam/backup/TestS3FileSystem.java | 8 +-- .../netflix/priam/stream/StreamingTest.java | 10 ++-- .../netflix/priam/utils/TestSystemUtils.java | 32 ++++++++++++ 9 files changed, 66 insertions(+), 72 deletions(-) rename priam/src/main/java/com/netflix/priam/aws/{S3BackupPath.java => RemoteBackupPath.java} (92%) create mode 100644 priam/src/test/java/com/netflix/priam/utils/TestSystemUtils.java diff --git a/priam/src/main/java/com/netflix/priam/PriamServer.java b/priam/src/main/java/com/netflix/priam/PriamServer.java index b883e40e9..299b809a6 100644 --- a/priam/src/main/java/com/netflix/priam/PriamServer.java +++ b/priam/src/main/java/com/netflix/priam/PriamServer.java @@ -39,6 +39,7 @@ import com.netflix.priam.utils.CassandraMonitor; import com.netflix.priam.utils.Sleeper; import com.netflix.priam.utils.SystemUtils; +import java.io.IOException; import org.apache.commons.collections4.CollectionUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -77,7 +78,7 @@ public PriamServer( this.snapshotMetaService = snapshotMetaService; } - private void createDirectories() { + private void createDirectories() throws IOException { SystemUtils.createDirs(config.getBackupCommitLogLocation()); SystemUtils.createDirs(config.getCommitLogLocation()); SystemUtils.createDirs(config.getCacheLocation()); diff --git a/priam/src/main/java/com/netflix/priam/aws/S3BackupPath.java b/priam/src/main/java/com/netflix/priam/aws/RemoteBackupPath.java similarity index 92% rename from priam/src/main/java/com/netflix/priam/aws/S3BackupPath.java rename to priam/src/main/java/com/netflix/priam/aws/RemoteBackupPath.java index 832d77f73..f696d6e36 100644 --- a/priam/src/main/java/com/netflix/priam/aws/S3BackupPath.java +++ b/priam/src/main/java/com/netflix/priam/aws/RemoteBackupPath.java @@ -26,11 +26,15 @@ import java.time.Instant; import java.util.Date; -/** Represents an S3 object key */ -public class S3BackupPath extends AbstractBackupPath { +/** + * Represents location of an object on the remote file system. All the objects will be keyed with a + * common prefix (based on configuration, typically environment), name of the cluster and token of + * this instance. + */ +public class RemoteBackupPath extends AbstractBackupPath { @Inject - public S3BackupPath(IConfiguration config, InstanceIdentity factory) { + public RemoteBackupPath(IConfiguration config, InstanceIdentity factory) { super(config, factory); } @@ -184,7 +188,7 @@ public void parsePartialPrefix(String remoteFilePath) { public String remotePrefix(Date start, Date end, String location) { StringBuilder buff = new StringBuilder(clusterPrefix(location)); token = instanceIdentity.getInstance().getToken(); - buff.append(token).append(S3BackupPath.PATH_SEP); + buff.append(token).append(RemoteBackupPath.PATH_SEP); // match the common characters to prefix. buff.append(match(start, end)); return buff.toString(); @@ -193,7 +197,7 @@ public String remotePrefix(Date start, Date end, String location) { @Override public String clusterPrefix(String location) { StringBuilder buff = new StringBuilder(); - String[] elements = location.split(String.valueOf(S3BackupPath.PATH_SEP)); + String[] elements = location.split(String.valueOf(RemoteBackupPath.PATH_SEP)); if (elements.length <= 1) { baseDir = config.getBackupLocation(); region = instanceIdentity.getInstanceInfo().getRegion(); @@ -204,9 +208,9 @@ public String clusterPrefix(String location) { region = elements[2]; clusterName = elements[3]; } - buff.append(baseDir).append(S3BackupPath.PATH_SEP); - buff.append(region).append(S3BackupPath.PATH_SEP); - buff.append(clusterName).append(S3BackupPath.PATH_SEP); + buff.append(baseDir).append(RemoteBackupPath.PATH_SEP); + buff.append(region).append(RemoteBackupPath.PATH_SEP); + buff.append(clusterName).append(RemoteBackupPath.PATH_SEP); return buff.toString(); } diff --git a/priam/src/main/java/com/netflix/priam/backup/AbstractBackupPath.java b/priam/src/main/java/com/netflix/priam/backup/AbstractBackupPath.java index cee82da76..92ee67c01 100644 --- a/priam/src/main/java/com/netflix/priam/backup/AbstractBackupPath.java +++ b/priam/src/main/java/com/netflix/priam/backup/AbstractBackupPath.java @@ -17,7 +17,7 @@ package com.netflix.priam.backup; import com.google.inject.ImplementedBy; -import com.netflix.priam.aws.S3BackupPath; +import com.netflix.priam.aws.RemoteBackupPath; import com.netflix.priam.compress.ICompression; import com.netflix.priam.config.IConfiguration; import com.netflix.priam.identity.InstanceIdentity; @@ -32,7 +32,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -@ImplementedBy(S3BackupPath.class) +@ImplementedBy(RemoteBackupPath.class) public abstract class AbstractBackupPath implements Comparable { private static final Logger logger = LoggerFactory.getLogger(AbstractBackupPath.class); private static final String FMT = "yyyyMMddHHmm"; diff --git a/priam/src/main/java/com/netflix/priam/utils/SystemUtils.java b/priam/src/main/java/com/netflix/priam/utils/SystemUtils.java index 05b59b5d3..50b1f7416 100644 --- a/priam/src/main/java/com/netflix/priam/utils/SystemUtils.java +++ b/priam/src/main/java/com/netflix/priam/utils/SystemUtils.java @@ -58,7 +58,7 @@ public static String getDataFromUrl(String url) { } /** - * delete all the files/dirs in the given Directory but dont delete the dir itself. + * delete all the files/dirs in the given Directory but do not delete the dir itself. * * @param dirPath The directory path where all the child directories exist. * @param childdirs List of child directories to be cleaned up in the dirPath @@ -71,12 +71,8 @@ public static void cleanupDir(String dirPath, List childdirs) throws IOE } } - public static void createDirs(String location) { - File dirFile = new File(location); - if (dirFile.exists() && dirFile.isFile()) { - dirFile.delete(); - dirFile.mkdirs(); - } else if (!dirFile.exists()) dirFile.mkdirs(); + public static void createDirs(String location) throws IOException { + FileUtils.forceMkdir(new File(location)); } public static byte[] md5(byte[] buf) { @@ -138,43 +134,4 @@ public static void closeQuietly(JMXConnector jmc) { logger.warn("failed to close JMXConnectorMgr", e); } } - - /* - @param absolute path to input file - @return handle to input file - */ - public static BufferedReader readFile(String absPathToFile) throws IOException { - InputStream is = new FileInputStream(absPathToFile); - InputStreamReader isr = new InputStreamReader(is); - return new BufferedReader(isr); - } - - /* - Write the "line" to the file. If file does not exist, it's created. if file exists, its content will be overwritten with the input. - @param absolute path to file - @param input line - */ - public static void writeToFile(String filename, String line) { - File f = new File(filename); - PrintWriter pw = null; - FileWriter fw; - try { - if (!f.exists()) { - f.createNewFile(); - logger.info("File created, absolute path: {}", f.getAbsolutePath()); - } - - fw = new FileWriter(f, false); - pw = new PrintWriter(fw); - pw.print(line); - - } catch (IOException e) { - throw new IllegalStateException("Exception processing file: " + filename, e); - } finally { - if (pw != null) { - pw.flush(); - pw.close(); - } - } - } } diff --git a/priam/src/test/java/com/netflix/priam/backup/FakeBackupFileSystem.java b/priam/src/test/java/com/netflix/priam/backup/FakeBackupFileSystem.java index 1acdec042..c2144e081 100644 --- a/priam/src/test/java/com/netflix/priam/backup/FakeBackupFileSystem.java +++ b/priam/src/test/java/com/netflix/priam/backup/FakeBackupFileSystem.java @@ -20,7 +20,7 @@ import com.google.inject.Inject; import com.google.inject.Provider; import com.google.inject.Singleton; -import com.netflix.priam.aws.S3BackupPath; +import com.netflix.priam.aws.RemoteBackupPath; import com.netflix.priam.backup.AbstractBackupPath.BackupFileType; import com.netflix.priam.config.IConfiguration; import com.netflix.priam.merics.BackupMetrics; @@ -85,7 +85,7 @@ public void addFile(String file) { @SuppressWarnings("unchecked") @Override public Iterator list(String bucket, Date start, Date till) { - String[] paths = bucket.split(String.valueOf(S3BackupPath.PATH_SEP)); + String[] paths = bucket.split(String.valueOf(RemoteBackupPath.PATH_SEP)); if (paths.length > 1) { baseDir = paths[1]; diff --git a/priam/src/test/java/com/netflix/priam/backup/TestBackupFile.java b/priam/src/test/java/com/netflix/priam/backup/TestBackupFile.java index 7fff9d830..365c38110 100644 --- a/priam/src/test/java/com/netflix/priam/backup/TestBackupFile.java +++ b/priam/src/test/java/com/netflix/priam/backup/TestBackupFile.java @@ -19,7 +19,7 @@ import com.google.inject.Guice; import com.google.inject.Injector; -import com.netflix.priam.aws.S3BackupPath; +import com.netflix.priam.aws.RemoteBackupPath; import com.netflix.priam.backup.AbstractBackupPath.BackupFileType; import com.netflix.priam.identity.InstanceIdentity; import java.io.BufferedOutputStream; @@ -72,7 +72,7 @@ public void testBackupFileCreation() throws ParseException { // Test snapshot file String snapshotfile = "target/data/Keyspace1/Standard1/snapshots/201108082320/Keyspace1-Standard1-ia-5-Data.db"; - S3BackupPath backupfile = injector.getInstance(S3BackupPath.class); + RemoteBackupPath backupfile = injector.getInstance(RemoteBackupPath.class); backupfile.parseLocal(new File(snapshotfile), BackupFileType.SNAP); Assert.assertEquals(BackupFileType.SNAP, backupfile.type); Assert.assertEquals("Keyspace1", backupfile.keyspace); @@ -92,7 +92,7 @@ public void testBackupFileCreation() throws ParseException { public void testIncBackupFileCreation() throws ParseException { // Test incremental file File bfile = new File("target/data/Keyspace1/Standard1/Keyspace1-Standard1-ia-5-Data.db"); - S3BackupPath backupfile = injector.getInstance(S3BackupPath.class); + RemoteBackupPath backupfile = injector.getInstance(RemoteBackupPath.class); backupfile.parseLocal(bfile, BackupFileType.SST); Assert.assertEquals(BackupFileType.SST, backupfile.type); Assert.assertEquals("Keyspace1", backupfile.keyspace); @@ -116,7 +116,7 @@ public void testMetaFileCreation() throws ParseException { // Test snapshot file String filestr = "cass/data/1234567.meta"; File bfile = new File(filestr); - S3BackupPath backupfile = injector.getInstance(S3BackupPath.class); + RemoteBackupPath backupfile = injector.getInstance(RemoteBackupPath.class); backupfile.parseLocal(bfile, BackupFileType.META); backupfile.setTime(backupfile.parseDate("201108082320")); Assert.assertEquals(BackupFileType.META, backupfile.type); diff --git a/priam/src/test/java/com/netflix/priam/backup/TestS3FileSystem.java b/priam/src/test/java/com/netflix/priam/backup/TestS3FileSystem.java index 3ef9c5db9..0ce7be25a 100644 --- a/priam/src/test/java/com/netflix/priam/backup/TestS3FileSystem.java +++ b/priam/src/test/java/com/netflix/priam/backup/TestS3FileSystem.java @@ -26,7 +26,7 @@ import com.google.inject.Guice; import com.google.inject.Injector; import com.netflix.priam.aws.DataPart; -import com.netflix.priam.aws.S3BackupPath; +import com.netflix.priam.aws.RemoteBackupPath; import com.netflix.priam.aws.S3FileSystem; import com.netflix.priam.aws.S3PartUploader; import com.netflix.priam.backup.AbstractBackupPath.BackupFileType; @@ -90,7 +90,7 @@ public static void cleanup() { public void testFileUpload() throws Exception { MockS3PartUploader.setup(); IBackupFileSystem fs = injector.getInstance(NullBackupFileSystem.class); - S3BackupPath backupfile = injector.getInstance(S3BackupPath.class); + RemoteBackupPath backupfile = injector.getInstance(RemoteBackupPath.class); backupfile.parseLocal(new File(FILE_PATH), BackupFileType.SNAP); long noOfFilesUploaded = backupMetrics.getUploadRate().count(); fs.uploadFile( @@ -110,7 +110,7 @@ public void testFileUploadFailures() throws Exception { S3FileSystem fs = injector.getInstance(S3FileSystem.class); String snapshotfile = "target/data/Keyspace1/Standard1/backups/201108082320/Keyspace1-Standard1-ia-1-Data.db"; - S3BackupPath backupfile = injector.getInstance(S3BackupPath.class); + RemoteBackupPath backupfile = injector.getInstance(RemoteBackupPath.class); backupfile.parseLocal(new File(snapshotfile), BackupFileType.SNAP); try { fs.uploadFile( @@ -133,7 +133,7 @@ public void testFileUploadCompleteFailure() throws Exception { S3FileSystem fs = injector.getInstance(S3FileSystem.class); String snapshotfile = "target/data/Keyspace1/Standard1/backups/201108082320/Keyspace1-Standard1-ia-1-Data.db"; - S3BackupPath backupfile = injector.getInstance(S3BackupPath.class); + RemoteBackupPath backupfile = injector.getInstance(RemoteBackupPath.class); backupfile.parseLocal(new File(snapshotfile), BackupFileType.SNAP); try { fs.uploadFile( diff --git a/priam/src/test/java/com/netflix/priam/stream/StreamingTest.java b/priam/src/test/java/com/netflix/priam/stream/StreamingTest.java index 8ac833be8..c88953c2c 100644 --- a/priam/src/test/java/com/netflix/priam/stream/StreamingTest.java +++ b/priam/src/test/java/com/netflix/priam/stream/StreamingTest.java @@ -19,7 +19,7 @@ import com.google.inject.Guice; import com.google.inject.Injector; -import com.netflix.priam.aws.S3BackupPath; +import com.netflix.priam.aws.RemoteBackupPath; import com.netflix.priam.backup.AbstractBackupPath; import com.netflix.priam.backup.BRTestModule; import com.netflix.priam.config.IConfiguration; @@ -47,7 +47,7 @@ public void testAbstractPath() { FifoQueue queue = new FifoQueue<>(10); for (int i = 10; i < 30; i++) { - S3BackupPath path = new S3BackupPath(conf, factory); + RemoteBackupPath path = new RemoteBackupPath(conf, factory); path.parseRemote( "test_backup/" + region @@ -61,7 +61,7 @@ public void testAbstractPath() { } for (int i = 10; i < 30; i++) { - S3BackupPath path = new S3BackupPath(conf, factory); + RemoteBackupPath path = new RemoteBackupPath(conf, factory); path.parseRemote( "test_backup/" + region @@ -75,7 +75,7 @@ public void testAbstractPath() { } for (int i = 10; i < 30; i++) { - S3BackupPath path = new S3BackupPath(conf, factory); + RemoteBackupPath path = new RemoteBackupPath(conf, factory); path.parseRemote( "test_backup/" + region @@ -88,7 +88,7 @@ public void testAbstractPath() { queue.adjustAndAdd(path); } - S3BackupPath path = new S3BackupPath(conf, factory); + RemoteBackupPath path = new RemoteBackupPath(conf, factory); path.parseRemote( "test_backup/" + region diff --git a/priam/src/test/java/com/netflix/priam/utils/TestSystemUtils.java b/priam/src/test/java/com/netflix/priam/utils/TestSystemUtils.java new file mode 100644 index 000000000..60a596c0f --- /dev/null +++ b/priam/src/test/java/com/netflix/priam/utils/TestSystemUtils.java @@ -0,0 +1,32 @@ +/* + * Copyright 2018 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.netflix.priam.utils; + +import org.junit.Assert; +import org.junit.Test; + +/** Created by aagrawal on 12/1/18. */ +public class TestSystemUtils { + + @Test + public void testGetDataFromUrl() { + String dummyurl = "https://jsonplaceholder.typicode.com/todos/1"; + String response = SystemUtils.getDataFromUrl(dummyurl); + Assert.assertNotNull(response); + } +} From 74e217b33e6b4a4b5059a9db64ce650a90c4982c Mon Sep 17 00:00:00 2001 From: Arun Agrawal Date: Tue, 11 Dec 2018 09:11:41 -0800 Subject: [PATCH 039/228] fix test --- .../java/com/netflix/priam/backup/FakeBackupFileSystem.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/priam/src/test/java/com/netflix/priam/backup/FakeBackupFileSystem.java b/priam/src/test/java/com/netflix/priam/backup/FakeBackupFileSystem.java index c2144e081..81c25e104 100644 --- a/priam/src/test/java/com/netflix/priam/backup/FakeBackupFileSystem.java +++ b/priam/src/test/java/com/netflix/priam/backup/FakeBackupFileSystem.java @@ -41,8 +41,6 @@ public class FakeBackupFileSystem extends AbstractFileSystem { private String region; private String clusterName; - @Inject Provider pathProvider; - @Inject public FakeBackupFileSystem( IConfiguration configuration, From dbfa781b3093c353daf24833aeef93c8f597cd02 Mon Sep 17 00:00:00 2001 From: Arun Agrawal Date: Wed, 5 Dec 2018 15:44:34 -0800 Subject: [PATCH 040/228] Backup Validation of META_V2. --- .../netflix/priam/aws/RemoteBackupPath.java | 25 +- .../priam/aws/S3EncryptedFileSystem.java | 4 +- .../com/netflix/priam/aws/S3FileSystem.java | 4 +- .../netflix/priam/aws/S3FileSystemBase.java | 8 +- .../com/netflix/priam/aws/S3Iterator.java | 6 +- .../priam/backup/AbstractBackupPath.java | 3 + .../priam/backup/AbstractFileSystem.java | 9 +- .../priam/backup/IBackupFileSystem.java | 22 +- .../priam/backupv2/BackupValidator.java | 191 +++++++++++++++ .../google/GoogleEncryptedFileSystem.java | 4 +- .../priam/resources/BackupServlet.java | 30 +++ .../com/netflix/priam/utils/DateUtil.java | 18 ++ .../netflix/priam/aws/MockAmazonS3Client.java | 46 ++++ ...kupPath.java => TestRemoteBackupPath.java} | 20 +- .../priam/backup/FakeBackupFileSystem.java | 21 +- .../priam/backup/NullBackupFileSystem.java | 2 +- .../priam/backup/TestFileIterator.java | 52 +--- .../priam/backupv2/TestBackupValidator.java | 223 ++++++++++++++++++ .../netflix/priam/utils/TestDateUtils.java | 22 ++ 19 files changed, 627 insertions(+), 83 deletions(-) create mode 100644 priam/src/main/java/com/netflix/priam/backupv2/BackupValidator.java create mode 100644 priam/src/test/java/com/netflix/priam/aws/MockAmazonS3Client.java rename priam/src/test/java/com/netflix/priam/aws/{TestS3BackupPath.java => TestRemoteBackupPath.java} (92%) create mode 100644 priam/src/test/java/com/netflix/priam/backupv2/TestBackupValidator.java diff --git a/priam/src/main/java/com/netflix/priam/aws/RemoteBackupPath.java b/priam/src/main/java/com/netflix/priam/aws/RemoteBackupPath.java index f696d6e36..1538c68b3 100644 --- a/priam/src/main/java/com/netflix/priam/aws/RemoteBackupPath.java +++ b/priam/src/main/java/com/netflix/priam/aws/RemoteBackupPath.java @@ -39,7 +39,7 @@ public RemoteBackupPath(IConfiguration config, InstanceIdentity factory) { } private Path getV2Prefix() { - return Paths.get(baseDir, getAppNameWithHash(), instanceIdentity.getInstance().getToken()); + return Paths.get(baseDir, getAppNameWithHash(clusterName), token); } private void parseV2Prefix(Path remoteFilePath) { @@ -54,8 +54,8 @@ private void parseV2Prefix(Path remoteFilePath) { /* This will ensure that there is some randomness in the path at the start so that remote file systems can hash the contents better when we have lot of clusters backing up at the same remote location. */ - private String getAppNameWithHash() { - return String.format("%d_%s", config.getAppName().hashCode() % 10000, config.getAppName()); + private String getAppNameWithHash(String appName) { + return String.format("%d_%s", appName.hashCode() % 10000, appName); } private String parseAndValidateAppNameWithHash(String appNameWithHash) { @@ -63,7 +63,11 @@ private String parseAndValidateAppNameWithHash(String appNameWithHash) { String appName = appNameWithHash.substring(appNameWithHash.indexOf("_") + 1); // Validate the hash int calculatedHash = appName.hashCode() % 10000; - assert calculatedHash == hash; + if (calculatedHash != hash) + throw new RuntimeException( + String.format( + "Hash for the app name: %s was calculated to be: %d but provided was: %d", + appName, calculatedHash, hash)); return appName; } @@ -194,6 +198,19 @@ public String remotePrefix(Date start, Date end, String location) { return buff.toString(); } + @Override + public Path remoteV2Prefix(Path location, BackupFileType fileType) { + if (location.getNameCount() <= 1) { + baseDir = config.getBackupLocation(); + clusterName = config.getAppName(); + } else if (location.getNameCount() >= 3) { + baseDir = location.getName(1).toString(); + clusterName = parseAndValidateAppNameWithHash(location.getName(2).toString()); + } + token = instanceIdentity.getInstance().getToken(); + return Paths.get(getV2Prefix().toString(), fileType.toString()); + } + @Override public String clusterPrefix(String location) { StringBuilder buff = new StringBuilder(); diff --git a/priam/src/main/java/com/netflix/priam/aws/S3EncryptedFileSystem.java b/priam/src/main/java/com/netflix/priam/aws/S3EncryptedFileSystem.java index 41164f71c..60dc33f57 100755 --- a/priam/src/main/java/com/netflix/priam/aws/S3EncryptedFileSystem.java +++ b/priam/src/main/java/com/netflix/priam/aws/S3EncryptedFileSystem.java @@ -74,7 +74,7 @@ protected void downloadFileImpl(Path remotePath, Path localPath) throws BackupRe RangeReadInputStream rris = new RangeReadInputStream( s3Client, - getBucket(), + getShard(), super.getFileSize(remotePath), remotePath.toString())) { /* @@ -87,7 +87,7 @@ protected void downloadFileImpl(Path remotePath, Path localPath) throws BackupRe "Exception encountered downloading " + remotePath + " from S3 bucket " - + getBucket() + + getShard() + ", Msg: " + e.getMessage(), e); diff --git a/priam/src/main/java/com/netflix/priam/aws/S3FileSystem.java b/priam/src/main/java/com/netflix/priam/aws/S3FileSystem.java index b26bf8539..a51794d61 100644 --- a/priam/src/main/java/com/netflix/priam/aws/S3FileSystem.java +++ b/priam/src/main/java/com/netflix/priam/aws/S3FileSystem.java @@ -72,7 +72,7 @@ protected void downloadFileImpl(Path remotePath, Path localPath) throws BackupRe long remoteFileSize = super.getFileSize(remotePath); RangeReadInputStream rris = new RangeReadInputStream( - s3Client, getBucket(), remoteFileSize, remotePath.toString()); + s3Client, getShard(), remoteFileSize, remotePath.toString()); final long bufSize = MAX_BUFFERED_IN_STREAM_SIZE > remoteFileSize ? remoteFileSize @@ -85,7 +85,7 @@ protected void downloadFileImpl(Path remotePath, Path localPath) throws BackupRe "Exception encountered downloading " + remotePath + " from S3 bucket " - + getBucket() + + getShard() + ", Msg: " + e.getMessage(), e); diff --git a/priam/src/main/java/com/netflix/priam/aws/S3FileSystemBase.java b/priam/src/main/java/com/netflix/priam/aws/S3FileSystemBase.java index 2314c6c86..69df53fc1 100755 --- a/priam/src/main/java/com/netflix/priam/aws/S3FileSystemBase.java +++ b/priam/src/main/java/com/netflix/priam/aws/S3FileSystemBase.java @@ -158,14 +158,14 @@ void checkSuccessfulUpload( @Override public long getFileSize(Path remotePath) throws BackupRestoreException { - return s3Client.getObjectMetadata(getBucket(), remotePath.toString()).getContentLength(); + return s3Client.getObjectMetadata(getShard(), remotePath.toString()).getContentLength(); } @Override public boolean doesRemoteFileExist(Path remotePath) throws BackupRestoreException { boolean exists = false; try { - exists = s3Client.doesObjectExist(getBucket(), remotePath.toString()); + exists = s3Client.doesObjectExist(getShard(), remotePath.toString()); } catch (AmazonServiceException ase) { // Amazon S3 rejected request throw new BackupRestoreException( @@ -189,8 +189,8 @@ public void shutdown() { } @Override - public Iterator list(String prefix, String delimiter) { - return new S3Iterator(s3Client, getBucket(), prefix, delimiter); + public Iterator listFileSystem(String prefix, String delimiter, String marker) { + return new S3Iterator(s3Client, getShard(), prefix, delimiter, marker); } final long getChunkSize(Path localPath) throws BackupRestoreException { diff --git a/priam/src/main/java/com/netflix/priam/aws/S3Iterator.java b/priam/src/main/java/com/netflix/priam/aws/S3Iterator.java index 4a22ab61e..0eaf4052a 100644 --- a/priam/src/main/java/com/netflix/priam/aws/S3Iterator.java +++ b/priam/src/main/java/com/netflix/priam/aws/S3Iterator.java @@ -37,12 +37,15 @@ public class S3Iterator implements Iterator { private final String bucket; private final String prefix; private final String delimiter; + private final String marker; - public S3Iterator(AmazonS3 s3Client, String bucket, String prefix, String delimiter) { + public S3Iterator( + AmazonS3 s3Client, String bucket, String prefix, String delimiter, String marker) { this.s3Client = s3Client; this.bucket = bucket; this.prefix = prefix; this.delimiter = delimiter; + this.marker = marker; iterator = createIterator(); } @@ -51,6 +54,7 @@ private void initListing() { listReq.setBucketName(bucket); listReq.setPrefix(prefix); if (StringUtils.isNotBlank(delimiter)) listReq.setDelimiter(delimiter); + if (StringUtils.isNotBlank(marker)) listReq.setMarker(marker); objectListing = s3Client.listObjects(listReq); } diff --git a/priam/src/main/java/com/netflix/priam/backup/AbstractBackupPath.java b/priam/src/main/java/com/netflix/priam/backup/AbstractBackupPath.java index 92ee67c01..b5ac4ad80 100644 --- a/priam/src/main/java/com/netflix/priam/backup/AbstractBackupPath.java +++ b/priam/src/main/java/com/netflix/priam/backup/AbstractBackupPath.java @@ -22,6 +22,7 @@ import com.netflix.priam.config.IConfiguration; import com.netflix.priam.identity.InstanceIdentity; import java.io.File; +import java.nio.file.Path; import java.text.ParseException; import java.time.Instant; import java.util.Date; @@ -173,6 +174,8 @@ public boolean equals(Object obj) { */ public abstract String remotePrefix(Date start, Date end, String location); + public abstract Path remoteV2Prefix(Path location, BackupFileType fileType); + /** Provides the cluster prefix */ public abstract String clusterPrefix(String location); diff --git a/priam/src/main/java/com/netflix/priam/backup/AbstractFileSystem.java b/priam/src/main/java/com/netflix/priam/backup/AbstractFileSystem.java index 233565a45..e2b79298a 100644 --- a/priam/src/main/java/com/netflix/priam/backup/AbstractFileSystem.java +++ b/priam/src/main/java/com/netflix/priam/backup/AbstractFileSystem.java @@ -230,11 +230,12 @@ protected abstract long uploadFileImpl(final Path localPath, final Path remotePa throws BackupRestoreException; @Override - public String getBucket() { + public String getShard() { return getPrefix().getName(0).toString(); } - protected Path getPrefix() { + @Override + public Path getPrefix() { Path prefix = Paths.get(configuration.getBackupPrefix()); if (StringUtils.isNotBlank(configuration.getRestorePrefix())) { @@ -247,7 +248,7 @@ protected Path getPrefix() { @Override public Iterator listPrefixes(Date date) { String prefix = pathProvider.get().clusterPrefix(getPrefix().toString()); - Iterator fileIterator = list(prefix, File.pathSeparator); + Iterator fileIterator = listFileSystem(prefix, File.pathSeparator, null); //noinspection unchecked return new TransformIterator( @@ -262,7 +263,7 @@ public Iterator listPrefixes(Date date) { @Override public Iterator list(String path, Date start, Date till) { String prefix = pathProvider.get().remotePrefix(start, till, path); - Iterator fileIterator = list(prefix, null); + Iterator fileIterator = listFileSystem(prefix, null, null); @SuppressWarnings("unchecked") TransformIterator transformIterator = diff --git a/priam/src/main/java/com/netflix/priam/backup/IBackupFileSystem.java b/priam/src/main/java/com/netflix/priam/backup/IBackupFileSystem.java index eca2029b0..ceff7c0cf 100644 --- a/priam/src/main/java/com/netflix/priam/backup/IBackupFileSystem.java +++ b/priam/src/main/java/com/netflix/priam/backup/IBackupFileSystem.java @@ -115,15 +115,23 @@ Future asyncUploadFile( throws FileNotFoundException, RejectedExecutionException, BackupRestoreException; /** - * Get the bucket where this object should be stored. For local file system this should be empty - * or null. + * Get the shard where this object should be stored. For local file system this should be empty + * or null. For S3, it would be the location of the bucket. * - * @return the location of the bucket. + * @return the location of the shard. */ - default String getBucket() { + default String getShard() { return ""; } + /** + * Get the prefix path for the backup file system. This will be either the location of the + * remote file system for backup or the location from where we should restore. + * + * @return prefix path to the backup file system. + */ + Path getPrefix(); + /** * List all files in the backup location for the specified time range. * @@ -139,13 +147,15 @@ default String getBucket() { Iterator listPrefixes(Date date); /** - * List all the files with the given prefix and delimiter. This should never return null. + * List all the files with the given prefix, delimiter, and marker. This should never return + * null. * * @param prefix Common prefix of the elements to search in the backup file system. * @param delimiter All the object will end with this delimiter. + * @param marker Start the fetch with this as the first object. * @return the iterator on the backup file system containing path of the files. */ - Iterator list(String prefix, String delimiter); + Iterator listFileSystem(String prefix, String delimiter, String marker); /** Runs cleanup or set retention */ void cleanup(); diff --git a/priam/src/main/java/com/netflix/priam/backupv2/BackupValidator.java b/priam/src/main/java/com/netflix/priam/backupv2/BackupValidator.java new file mode 100644 index 000000000..eb7308f32 --- /dev/null +++ b/priam/src/main/java/com/netflix/priam/backupv2/BackupValidator.java @@ -0,0 +1,191 @@ +/* + * Copyright 2018 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.netflix.priam.backupv2; + +import com.google.inject.Provider; +import com.netflix.priam.backup.*; +import com.netflix.priam.config.IConfiguration; +import com.netflix.priam.utils.DateUtil; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Iterator; +import java.util.List; +import javax.inject.Inject; +import org.apache.commons.io.FileUtils; +import org.apache.commons.lang3.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * This class takes a meta file and verifies that all the components listed in that meta file are + * successfully uploaded to remote file system. Created by aagrawal on 11/28/18. + */ +public class BackupValidator { + private static final Logger logger = LoggerFactory.getLogger(BackupVerification.class); + private final IBackupFileSystem fs; + private final Provider abstractBackupPathProvider; + private boolean isBackupValid; + + @Inject + public BackupValidator( + IConfiguration configuration, + IFileSystemContext backupFileSystemCtx, + Provider abstractBackupPathProvider) { + fs = backupFileSystemCtx.getFileStrategy(configuration); + this.abstractBackupPathProvider = abstractBackupPathProvider; + } + + /** + * Fetch the list of all META_V2 files on the remote file system for the provided valid + * daterange. + * + * @param dateRange the time period to scan in the remote file system for meta files. + * @return List of all the META_V2 files from the remote file system. + */ + public List findMetaFiles(DateUtil.DateRange dateRange) { + ArrayList metas = new ArrayList<>(); + String prefix = getMetaPrefix(dateRange); + String marker = getMetaPrefix(new DateUtil.DateRange(dateRange.getStartTime(), null)); + logger.info( + "Listing filesystem with prefix: {}, marker: {}, daterange: {}", + prefix, + marker, + dateRange); + Iterator iterator = fs.listFileSystem(prefix, null, marker); + + while (iterator.hasNext()) { + AbstractBackupPath abstractBackupPath = abstractBackupPathProvider.get(); + abstractBackupPath.parseRemote(iterator.next()); + logger.debug("Meta file found: {}", abstractBackupPath); + if (abstractBackupPath.getLastModified().toEpochMilli() + >= dateRange.getStartTime().toEpochMilli() + && abstractBackupPath.getLastModified().toEpochMilli() + <= dateRange.getEndTime().toEpochMilli()) { + metas.add(abstractBackupPath); + } + } + + Collections.sort(metas, Collections.reverseOrder()); + + if (metas.size() == 0) { + logger.info( + "No meta file found on remote file system for the time period: {}", dateRange); + } + + return metas; + } + + /** + * Find the latest valid meta file in a given valid time range. It will return the handle to the + * file via AbstractBackupPath object. + * + * @param dateRange the time period to scan in the remote file system for meta files. + * @return the AbstractBackupPath denoting the "local" file which is valid or null. Caller needs + * to delete the file. + * @throws BackupRestoreException if there is issue contacting remote file system, fetching the + * file etc. + */ + public AbstractBackupPath findLatestValidMetaFile(DateUtil.DateRange dateRange) + throws BackupRestoreException { + List metas = findMetaFiles(dateRange); + logger.info("Meta files found: {}", metas); + + for (AbstractBackupPath meta : metas) { + Path localFile = Paths.get(meta.newRestoreFile().getAbsolutePath()); + fs.downloadFile(Paths.get(meta.getRemotePath()), localFile, 10); + boolean isValid = isMetaFileValid(localFile); + logger.info("Meta: {}, isValid: {}", meta, isValid); + if (!isValid) FileUtils.deleteQuietly(localFile.toFile()); + else return meta; + } + + return null; + } + + /** + * Get the prefix for the META_V2 file. This will depend on the configuration, if restore prefix + * is set. + * + * @param dateRange date range for which we are trying to find META_V2 files. + * @return prefix for the META_V2 files. + */ + public String getMetaPrefix(DateUtil.DateRange dateRange) { + Path location = fs.getPrefix(); + AbstractBackupPath abstractBackupPath = abstractBackupPathProvider.get(); + String match = StringUtils.EMPTY; + if (dateRange != null) match = dateRange.match(); + return Paths.get( + abstractBackupPath + .remoteV2Prefix(location, AbstractBackupPath.BackupFileType.META_V2) + .toString(), + match) + .toString(); + } + + /** + * Validate that all the files mentioned in the meta file actually exists on remote file system. + * + * @param metaFile Path to the local uncompressed/unencrypted meta file + * @return true if all the files mentioned in meta file are present on remote file system. It + * will return false in case of any error. + */ + public boolean isMetaFileValid(Path metaFile) { + try { + isBackupValid = true; + new MetaFileBackupValidator().readMeta(metaFile); + } catch (FileNotFoundException fne) { + isBackupValid = false; + logger.error(fne.getLocalizedMessage()); + } catch (IOException ioe) { + isBackupValid = false; + logger.error( + "IO Error while processing meta file: " + metaFile, ioe.getLocalizedMessage()); + ioe.printStackTrace(); + } + return isBackupValid; + } + + private class MetaFileBackupValidator extends MetaFileReader { + @Override + public void process(ColumnfamilyResult columnfamilyResult) { + for (ColumnfamilyResult.SSTableResult ssTableResult : + columnfamilyResult.getSstables()) { + for (FileUploadResult fileUploadResult : ssTableResult.getSstableComponents()) { + if (!isBackupValid) { + break; + } + + try { + isBackupValid = + isBackupValid + && fs.doesRemoteFileExist( + Paths.get(fileUploadResult.getBackupPath())); + } catch (BackupRestoreException e) { + // For any error, mark that file is not available. + isBackupValid = false; + break; + } + } + } + } + } +} diff --git a/priam/src/main/java/com/netflix/priam/google/GoogleEncryptedFileSystem.java b/priam/src/main/java/com/netflix/priam/google/GoogleEncryptedFileSystem.java index cdf561dde..0d26f51d9 100755 --- a/priam/src/main/java/com/netflix/priam/google/GoogleEncryptedFileSystem.java +++ b/priam/src/main/java/com/netflix/priam/google/GoogleEncryptedFileSystem.java @@ -78,7 +78,7 @@ public GoogleEncryptedFileSystem( "Unable to create a handle to the Google Http tranport", e); } - this.srcBucketName = getBucket(); + this.srcBucketName = getShard(); } private Storage.Objects constructObjectResourceHandle() { @@ -211,7 +211,7 @@ protected void downloadFileImpl(Path remotePath, Path localPath) throws BackupRe } @Override - public Iterator list(String prefix, String delimiter) { + public Iterator listFileSystem(String prefix, String delimiter, String marker) { return new GoogleFileIterator(constructGcsStorageHandle(), prefix, null); } diff --git a/priam/src/main/java/com/netflix/priam/resources/BackupServlet.java b/priam/src/main/java/com/netflix/priam/resources/BackupServlet.java index ecda87563..e4a6b2b0f 100644 --- a/priam/src/main/java/com/netflix/priam/resources/BackupServlet.java +++ b/priam/src/main/java/com/netflix/priam/resources/BackupServlet.java @@ -20,6 +20,7 @@ import com.google.inject.name.Named; import com.netflix.priam.backup.*; import com.netflix.priam.backup.AbstractBackupPath.BackupFileType; +import com.netflix.priam.backupv2.BackupValidator; import com.netflix.priam.config.IConfiguration; import com.netflix.priam.scheduler.PriamScheduler; import com.netflix.priam.utils.DateUtil; @@ -32,6 +33,7 @@ import javax.ws.rs.*; import javax.ws.rs.core.MediaType; import javax.ws.rs.core.Response; +import org.apache.commons.io.FileUtils; import org.codehaus.jettison.json.JSONArray; import org.codehaus.jettison.json.JSONException; import org.codehaus.jettison.json.JSONObject; @@ -54,6 +56,7 @@ public class BackupServlet { @Inject private PriamScheduler scheduler; private final IBackupStatusMgr completedBkups; @Inject private MetaData metaData; + @Inject private BackupValidator backupValidator; @Inject public BackupServlet( @@ -269,6 +272,33 @@ public Response validateSnapshotByDate(@PathParam("daterange") String daterange) return Response.ok(jsonReply.toString()).build(); } + @GET + @Path("/validate/snapshot/v2/{daterange}") + public Response validateV2SnapshotByDate(@PathParam("daterange") String daterange) + throws Exception { + try { + DateUtil.DateRange dateRange = new DateUtil.DateRange(daterange); + AbstractBackupPath validMeta = backupValidator.findLatestValidMetaFile(dateRange); + if (validMeta == null) { + return Response.noContent() + .entity("No valid meta found for provided timerange") + .build(); + } + + // Delete the file as we don't need it anymore. + FileUtils.deleteQuietly(validMeta.newRestoreFile()); + JSONObject jsonObject = new JSONObject(); + jsonObject.put("daterange", daterange); + jsonObject.put("remoteMetaPath", validMeta.getRemotePath()); + jsonObject.put("valid", true); + return Response.ok(jsonObject.toString()).build(); + } catch (BackupRestoreException ex) { + logger.error( + "Error while trying to fetch the latest valid meta file: {}", ex.getMessage()); + return Response.serverError().build(); + } + } + /* * A list of files for requested filter. Currently, the only supported filter is META, all others will be ignore. * For filter of META, ONLY the daily snapshot meta file (meta.json) are accounted for, not the incremental meta file. diff --git a/priam/src/main/java/com/netflix/priam/utils/DateUtil.java b/priam/src/main/java/com/netflix/priam/utils/DateUtil.java index 7129a6f56..4897a4f70 100644 --- a/priam/src/main/java/com/netflix/priam/utils/DateUtil.java +++ b/priam/src/main/java/com/netflix/priam/utils/DateUtil.java @@ -184,6 +184,11 @@ public static class DateRange { Instant startTime; Instant endTime; + public DateRange(Instant startTime, Instant endTime) { + this.startTime = startTime; + this.endTime = endTime; + } + public DateRange(String daterange) { if (StringUtils.isBlank(daterange) || daterange.equalsIgnoreCase("default")) { endTime = getInstant(); @@ -195,6 +200,15 @@ public DateRange(String daterange) { } } + public String match() { + if (startTime == null || endTime == null) return StringUtils.EMPTY; + String sString = startTime.toEpochMilli() + ""; + String eString = endTime.toEpochMilli() + ""; + int diff = StringUtils.indexOfDifference(sString, eString); + if (diff < 0) return sString; + return sString.substring(0, diff); + } + public Instant getStartTime() { return startTime; } @@ -202,5 +216,9 @@ public Instant getStartTime() { public Instant getEndTime() { return endTime; } + + public String toString() { + return GsonJsonSerializer.getGson().toJson(this); + } } } diff --git a/priam/src/test/java/com/netflix/priam/aws/MockAmazonS3Client.java b/priam/src/test/java/com/netflix/priam/aws/MockAmazonS3Client.java new file mode 100644 index 000000000..d80501a51 --- /dev/null +++ b/priam/src/test/java/com/netflix/priam/aws/MockAmazonS3Client.java @@ -0,0 +1,46 @@ +/* + * Copyright 2018 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.netflix.priam.aws; + +import com.amazonaws.AmazonClientException; +import com.amazonaws.services.s3.AmazonS3Client; +import com.amazonaws.services.s3.model.ListObjectsRequest; +import com.amazonaws.services.s3.model.ObjectListing; +import mockit.Mock; +import mockit.MockUp; + +/** Created by aagrawal on 12/6/18. */ +public class MockAmazonS3Client extends MockUp { + @Mock + public ObjectListing listObjects(ListObjectsRequest listObjectsRequest) + throws AmazonClientException { + ObjectListing listing = new ObjectListing(); + listing.setBucketName(listObjectsRequest.getBucketName()); + listing.setPrefix(listObjectsRequest.getPrefix()); + return listing; + } + + @Mock + public ObjectListing listNextBatchOfObjects(ObjectListing previousObjectListing) + throws AmazonClientException { + ObjectListing listing = new ObjectListing(); + listing.setBucketName(previousObjectListing.getBucketName()); + listing.setPrefix(previousObjectListing.getPrefix()); + return new ObjectListing(); + } +} diff --git a/priam/src/test/java/com/netflix/priam/aws/TestS3BackupPath.java b/priam/src/test/java/com/netflix/priam/aws/TestRemoteBackupPath.java similarity index 92% rename from priam/src/test/java/com/netflix/priam/aws/TestS3BackupPath.java rename to priam/src/test/java/com/netflix/priam/aws/TestRemoteBackupPath.java index e3774409b..e8bb8b874 100644 --- a/priam/src/test/java/com/netflix/priam/aws/TestS3BackupPath.java +++ b/priam/src/test/java/com/netflix/priam/aws/TestRemoteBackupPath.java @@ -35,12 +35,12 @@ import org.slf4j.LoggerFactory; /** Created by aagrawal on 11/23/18. */ -public class TestS3BackupPath { - private static final Logger logger = LoggerFactory.getLogger(TestS3BackupPath.class); +public class TestRemoteBackupPath { + private static final Logger logger = LoggerFactory.getLogger(TestRemoteBackupPath.class); private Provider pathFactory; private IConfiguration configuration; - public TestS3BackupPath() { + public TestRemoteBackupPath() { Injector injector = Guice.createInjector(new BRTestModule()); pathFactory = injector.getProvider(AbstractBackupPath.class); configuration = injector.getInstance(IConfiguration.class); @@ -206,4 +206,18 @@ public void testV2BackupPathMeta() throws ParseException { Assert.assertEquals(now, abstractBackupPath2.getLastModified()); validateAbstractBackupPath(abstractBackupPath, abstractBackupPath2); } + + @Test + public void testRemoteV2Prefix() throws ParseException { + Path path = Paths.get("test_backup"); + AbstractBackupPath abstractBackupPath = pathFactory.get(); + Assert.assertEquals( + "casstestbackup/1049_fake-app/1808575600/META_V2", + abstractBackupPath.remoteV2Prefix(path, BackupFileType.META_V2).toString()); + + path = Paths.get("s3-bucket-name", "fake_base_dir", "-6717_random_fake_app"); + Assert.assertEquals( + "fake_base_dir/-6717_random_fake_app/1808575600/META_V2", + abstractBackupPath.remoteV2Prefix(path, BackupFileType.META_V2).toString()); + } } diff --git a/priam/src/test/java/com/netflix/priam/backup/FakeBackupFileSystem.java b/priam/src/test/java/com/netflix/priam/backup/FakeBackupFileSystem.java index 81c25e104..7c2558d97 100644 --- a/priam/src/test/java/com/netflix/priam/backup/FakeBackupFileSystem.java +++ b/priam/src/test/java/com/netflix/priam/backup/FakeBackupFileSystem.java @@ -29,7 +29,6 @@ import java.io.IOException; import java.nio.file.Path; import java.util.*; -import org.apache.commons.collections4.iterators.TransformIterator; import org.json.simple.JSONArray; @Singleton @@ -106,8 +105,15 @@ public Iterator list(String bucket, Date start, Date till) { } @Override - public Iterator list(String prefix, String delimiter) { - return new TransformIterator<>(flist.iterator(), AbstractBackupPath::getRemotePath); + public Iterator listFileSystem(String prefix, String delimiter, String marker) { + ArrayList items = new ArrayList<>(); + flist.stream() + .forEach( + abstractBackupPath -> { + if (abstractBackupPath.getRemotePath().startsWith(prefix)) + items.add(abstractBackupPath.getRemotePath()); + }); + return items.iterator(); } public void shutdown() { @@ -119,6 +125,15 @@ public long getFileSize(Path remotePath) throws BackupRestoreException { return 0; } + @Override + public boolean doesRemoteFileExist(Path remotePath) throws BackupRestoreException { + for (AbstractBackupPath abstractBackupPath : flist) { + if (abstractBackupPath.getRemotePath().equalsIgnoreCase(remotePath.toString())) + return true; + } + return false; + } + @Override public void cleanup() { // TODO Auto-generated method stub diff --git a/priam/src/test/java/com/netflix/priam/backup/NullBackupFileSystem.java b/priam/src/test/java/com/netflix/priam/backup/NullBackupFileSystem.java index b248b4f38..326537beb 100644 --- a/priam/src/test/java/com/netflix/priam/backup/NullBackupFileSystem.java +++ b/priam/src/test/java/com/netflix/priam/backup/NullBackupFileSystem.java @@ -47,7 +47,7 @@ public long getFileSize(Path remotePath) throws BackupRestoreException { } @Override - public Iterator list(String prefix, String delimiter) { + public Iterator listFileSystem(String prefix, String delimiter, String marker) { return Collections.emptyIterator(); } diff --git a/priam/src/test/java/com/netflix/priam/backup/TestFileIterator.java b/priam/src/test/java/com/netflix/priam/backup/TestFileIterator.java index f66d44168..bb63946ee 100644 --- a/priam/src/test/java/com/netflix/priam/backup/TestFileIterator.java +++ b/priam/src/test/java/com/netflix/priam/backup/TestFileIterator.java @@ -17,19 +17,16 @@ package com.netflix.priam.backup; -import com.amazonaws.AmazonClientException; import com.amazonaws.services.s3.AmazonS3Client; -import com.amazonaws.services.s3.model.ListObjectsRequest; import com.amazonaws.services.s3.model.ObjectListing; import com.amazonaws.services.s3.model.S3ObjectSummary; import com.google.inject.Guice; import com.google.inject.Injector; +import com.netflix.priam.aws.MockAmazonS3Client; import com.netflix.priam.aws.S3FileSystem; import com.netflix.priam.config.IConfiguration; import com.netflix.priam.identity.InstanceIdentity; import java.io.IOException; -import java.nio.file.Path; -import java.nio.file.Paths; import java.util.*; import mockit.Mock; import mockit.MockUp; @@ -75,31 +72,6 @@ public static void setup() throws InterruptedException, IOException { endTime = cal.getTime(); } - // MockAmazonS3Client class - @Ignore - public static class MockAmazonS3Client extends MockUp { - public static String bucketName = ""; - public static String prefix = ""; - - @Mock - public ObjectListing listObjects(ListObjectsRequest listObjectsRequest) - throws AmazonClientException { - ObjectListing listing = new ObjectListing(); - listing.setBucketName(listObjectsRequest.getBucketName()); - listing.setPrefix(listObjectsRequest.getPrefix()); - return listing; - } - - @Mock - public ObjectListing listNextBatchOfObjects(ObjectListing previousObjectListing) - throws AmazonClientException { - ObjectListing listing = new ObjectListing(); - listing.setBucketName(previousObjectListing.getBucketName()); - listing.setPrefix(previousObjectListing.getPrefix()); - return new ObjectListing(); - } - } - // MockObjectListing class @Ignore public static class MockObjectListing extends MockUp { @@ -128,14 +100,6 @@ public boolean isTruncated() { } } - private Path getClusterPrefix() { - return Paths.get( - conf.getBackupLocation(), - factory.getInstanceInfo().getRegion(), - conf.getAppName(), - factory.getInstance().getToken()); - } - @Test public void testIteratorEmptySet() { cal.set(2011, 7, 11, 6, 1, 0); @@ -143,8 +107,6 @@ public void testIteratorEmptySet() { Date stime = cal.getTime(); cal.add(Calendar.HOUR, 5); Date etime = cal.getTime(); - MockAmazonS3Client.bucketName = bucket; - MockAmazonS3Client.prefix = getClusterPrefix() + "/20110811"; Iterator fileIterator = s3FileSystem.list(bucket, stime, etime); Set files = new HashSet<>(); @@ -157,8 +119,6 @@ public void testIterator() { MockObjectListing.truncated = false; MockObjectListing.firstcall = true; MockObjectListing.simfilter = false; - MockAmazonS3Client.bucketName = "TESTBUCKET"; - MockAmazonS3Client.prefix = getClusterPrefix() + "/20110811"; Iterator fileIterator = s3FileSystem.list(bucket, startTime, endTime); @@ -192,8 +152,6 @@ public void testIteratorTruncated() { MockObjectListing.truncated = true; MockObjectListing.firstcall = true; MockObjectListing.simfilter = false; - MockAmazonS3Client.bucketName = "TESTBUCKET"; - MockAmazonS3Client.prefix = getClusterPrefix() + "/20110811"; Iterator fileIterator = s3FileSystem.list(bucket, startTime, endTime); @@ -243,8 +201,6 @@ public void testIteratorTruncatedOOR() { MockObjectListing.truncated = true; MockObjectListing.firstcall = true; MockObjectListing.simfilter = true; - MockAmazonS3Client.bucketName = bucket; - MockAmazonS3Client.prefix = getClusterPrefix() + "/20110811"; Iterator fileIterator = s3FileSystem.list(bucket, startTime, endTime); @@ -294,12 +250,6 @@ public void testRestorePathIteration() { MockObjectListing.truncated = true; MockObjectListing.firstcall = true; MockObjectListing.simfilter = false; - MockAmazonS3Client.bucketName = "RESTOREBUCKET"; - MockAmazonS3Client.prefix = - "test_restore_backup/fake-restore-region/fakerestorecluster" - + "/" - + factory.getInstance().getToken(); - MockAmazonS3Client.prefix += "/20110811"; Iterator fileIterator = s3FileSystem.list( diff --git a/priam/src/test/java/com/netflix/priam/backupv2/TestBackupValidator.java b/priam/src/test/java/com/netflix/priam/backupv2/TestBackupValidator.java new file mode 100644 index 000000000..afdd7f2de --- /dev/null +++ b/priam/src/test/java/com/netflix/priam/backupv2/TestBackupValidator.java @@ -0,0 +1,223 @@ +/* + * Copyright 2018 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.netflix.priam.backupv2; + +import com.google.inject.Guice; +import com.google.inject.Injector; +import com.google.inject.Provider; +import com.netflix.priam.backup.AbstractBackupPath; +import com.netflix.priam.backup.BRTestModule; +import com.netflix.priam.backup.BackupRestoreException; +import com.netflix.priam.backup.FakeBackupFileSystem; +import com.netflix.priam.config.IConfiguration; +import com.netflix.priam.utils.DateUtil; +import java.io.IOException; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.time.Instant; +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; +import org.apache.commons.io.FileUtils; +import org.junit.Assert; +import org.junit.Test; + +/** Created by aagrawal on 12/5/18. */ +public class TestBackupValidator { + private FakeBackupFileSystem fs; + private Provider pathProvider; + private IConfiguration configuration; + private BackupValidator backupValidator; + private MetaFileWriterBuilder metaFileWriterBuilder; + + public TestBackupValidator() { + Injector injector = Guice.createInjector(new BRTestModule()); + pathProvider = injector.getProvider(AbstractBackupPath.class); + configuration = injector.getInstance(IConfiguration.class); + fs = injector.getInstance(FakeBackupFileSystem.class); + fs.setupTest(getFakeFiles()); + backupValidator = injector.getInstance(BackupValidator.class); + metaFileWriterBuilder = injector.getInstance(MetaFileWriterBuilder.class); + } + + @Test + public void testMetaPrefix() { + // Null date range + Assert.assertEquals(getPrefix() + "/META_V2", backupValidator.getMetaPrefix(null)); + // No end date. + Assert.assertEquals( + getPrefix() + "/META_V2", + backupValidator.getMetaPrefix(new DateUtil.DateRange(Instant.now(), null))); + // No start date + Assert.assertEquals( + getPrefix() + "/META_V2", + backupValidator.getMetaPrefix(new DateUtil.DateRange(null, Instant.now()))); + long start = 1834567890L; + long end = 1834877776L; + Assert.assertEquals( + getPrefix() + "/META_V2/1834", + backupValidator.getMetaPrefix( + new DateUtil.DateRange( + Instant.ofEpochSecond(start), Instant.ofEpochSecond(end)))); + } + + @Test + public void testIsMetaFileValid() throws Exception { + Path metaPath = createMeta(getFakeFiles()); + Assert.assertTrue(backupValidator.isMetaFileValid(metaPath)); + FileUtils.deleteQuietly(metaPath.toFile()); + + List fileToAdd = getFakeFiles(); + fileToAdd.add( + Paths.get( + getPrefix(), + AbstractBackupPath.BackupFileType.SST_V2.toString(), + "1859817645000", + "keyspace1", + "columnfamily1", + "file9.Data.db.SNAPPY") + .toString()); + + metaPath = createMeta(fileToAdd); + Assert.assertFalse(backupValidator.isMetaFileValid(metaPath)); + FileUtils.deleteQuietly(metaPath.toFile()); + + metaPath = Paths.get(configuration.getDataFileLocation(), "meta_v2_201901010000.json"); + Assert.assertFalse(backupValidator.isMetaFileValid(metaPath)); + } + + @Test + public void testFindMetaFiles() throws BackupRestoreException { + List metas = + backupValidator.findMetaFiles( + new DateUtil.DateRange( + Instant.ofEpochMilli(1859824860000L), + Instant.ofEpochMilli(1859828420000L))); + Assert.assertEquals(1, metas.size()); + Assert.assertEquals("meta_v2_202812071801.json", metas.get(0).getFileName()); + Assert.assertTrue(fs.doesRemoteFileExist(Paths.get(metas.get(0).getRemotePath()))); + + metas = + backupValidator.findMetaFiles( + new DateUtil.DateRange( + Instant.ofEpochMilli(1859824860000L), + Instant.ofEpochMilli(1859828460000L))); + Assert.assertEquals(2, metas.size()); + } + + @Test + public void testFindLatestValidMetaFile() {} + + private String getPrefix() { + return "casstestbackup/1049_fake-app/1808575600"; + } + + private Path createMeta(List filesToAdd) throws IOException { + MetaFileWriterBuilder.DataStep dataStep = + metaFileWriterBuilder + .newBuilder() + .startMetaFileGeneration(Instant.ofEpochMilli(1859824860000L)); + ColumnfamilyResult columnfamilyResult = + new ColumnfamilyResult("keyspace1", "columnfamily1"); + for (String file : filesToAdd) { + AbstractBackupPath path = pathProvider.get(); + path.parseRemote(file); + if (path.getType() == AbstractBackupPath.BackupFileType.SST_V2) { + ColumnfamilyResult.SSTableResult ssTableResult = + new ColumnfamilyResult.SSTableResult(); + + ssTableResult.setSstableComponents(new ArrayList<>()); + ssTableResult.getSstableComponents().add(getFileUploadResult(path)); + columnfamilyResult.addSstable(ssTableResult); + } + } + dataStep.addColumnfamilyResult(columnfamilyResult); + return dataStep.endMetaFileGeneration().getMetaFilePath(); + } + + private FileUploadResult getFileUploadResult(AbstractBackupPath path) { + FileUploadResult fileUploadResult = + new FileUploadResult( + Paths.get(path.getFileName()), + "keyspace1", + "columnfamily1", + path.getLastModified(), + path.getLastModified(), + path.getSize()); + fileUploadResult.setBackupPath(path.getRemotePath()); + return fileUploadResult; + } + + private List getFakeFiles() { + List files = new ArrayList<>(); + files.add( + Paths.get( + getPrefix(), + AbstractBackupPath.BackupFileType.SST_V2.toString(), + "1859817645000", + "keyspace1", + "columnfamily1", + "file1.Data.db.SNAPPY")); + files.add( + Paths.get( + getPrefix(), + AbstractBackupPath.BackupFileType.SST_V2.toString(), + "1859818845000", + "keyspace1", + "columnfamily1", + "file2.Data.db.SNAPPY")); + + files.add( + Paths.get( + getPrefix(), + AbstractBackupPath.BackupFileType.META_V2.toString(), + "1859824860000", + "meta_v2_202812071801.json.SNAPPY")); + files.add( + Paths.get( + getPrefix(), + AbstractBackupPath.BackupFileType.SST_V2.toString(), + "1859826045000", + "keyspace1", + "columnfamily1", + "manifest.SNAPPY")); + files.add( + Paths.get( + getPrefix(), + AbstractBackupPath.BackupFileType.SST_V2.toString(), + "1859828410000", + "keyspace1", + "columnfamily1", + "file3.Data.db.SNAPPY")); + files.add( + Paths.get( + getPrefix(), + AbstractBackupPath.BackupFileType.SST_V2.toString(), + "1859828420000", + "keyspace1", + "columnfamily1", + "file4.Data.db.SNAPPY")); + files.add( + Paths.get( + getPrefix(), + AbstractBackupPath.BackupFileType.META_V2.toString(), + "1859828460000", + "meta_v2_202812071901.json.SNAPPY")); + return files.stream().map(Path::toString).collect(Collectors.toList()); + } +} diff --git a/priam/src/test/java/com/netflix/priam/utils/TestDateUtils.java b/priam/src/test/java/com/netflix/priam/utils/TestDateUtils.java index e871dcecc..8820d8a42 100644 --- a/priam/src/test/java/com/netflix/priam/utils/TestDateUtils.java +++ b/priam/src/test/java/com/netflix/priam/utils/TestDateUtils.java @@ -20,6 +20,7 @@ import java.time.Duration; import java.time.Instant; import java.time.temporal.ChronoUnit; +import org.apache.commons.lang3.StringUtils; import org.junit.Assert; import org.junit.Test; @@ -72,4 +73,25 @@ public void testDateRangeRandom() { Assert.assertEquals(null, dateRange.getStartTime()); Assert.assertEquals(null, dateRange.getEndTime()); } + + @Test + public void testDateRangeMatch() { + Instant dateStart = Instant.ofEpochMilli(1543632497000L); + Instant dateEnd = Instant.ofEpochMilli(1543819697000L); + DateUtil.DateRange dateRange = new DateUtil.DateRange(dateStart, dateEnd); + Assert.assertEquals("1543", dateRange.match()); + + dateRange = new DateUtil.DateRange(dateStart, null); + Assert.assertEquals(StringUtils.EMPTY, dateRange.match()); + } + + @Test + public void testFutureDateRangeValues() { + String start = "202801011201"; + String end = "202801051201"; + DateUtil.DateRange dateRange = new DateUtil.DateRange(start + "," + end); + Assert.assertEquals(DateUtil.getDate(start).toInstant(), dateRange.getStartTime()); + Assert.assertEquals(DateUtil.getDate(end).toInstant(), dateRange.getEndTime()); + Assert.assertEquals("1830", dateRange.match()); + } } From 7250b0d7104b52517b46751b020178c91b214419 Mon Sep 17 00:00:00 2001 From: Arun Agrawal Date: Tue, 18 Dec 2018 17:01:50 -0800 Subject: [PATCH 041/228] Backup TTL Service. Add encyprtion information in backup file messages, META_V2. Fix dateutils --- build.gradle | 18 +- .../java/com/netflix/priam/PriamServer.java | 25 +- .../netflix/priam/aws/RemoteBackupPath.java | 44 +++- .../netflix/priam/aws/S3FileSystemBase.java | 24 ++ .../priam/backup/AbstractBackupPath.java | 11 + .../priam/backup/IBackupFileSystem.java | 16 +- .../priam/backupv2/BackupValidator.java | 16 +- .../priam/backupv2/FileUploadResult.java | 12 + .../priam/config/IBackupRestoreConfig.java | 16 ++ .../priam/cryptography/IFileCryptography.java | 5 + .../google/GoogleEncryptedFileSystem.java | 6 + .../priam/identity/InstanceIdentity.java | 5 - .../notification/BackupNotificationMgr.java | 2 + .../priam/resources/BackupServlet.java | 2 +- .../priam/services/BackupTTLService.java | 234 ++++++++++++++++++ .../com/netflix/priam/utils/DateUtil.java | 34 ++- .../priam/aws/TestRemoteBackupPath.java | 11 +- .../priam/backup/FakeBackupFileSystem.java | 50 ++-- .../priam/backup/NullBackupFileSystem.java | 6 + .../com/netflix/priam/backup/TestBackup.java | 6 +- .../priam/backup/TestS3FileSystem.java | 32 +++ .../priam/backupv2/TestBackupUtils.java | 96 +++++++ .../priam/backupv2/TestBackupValidator.java | 86 +++---- .../priam/config/FakeBackupRestoreConfig.java | 5 + .../priam/services/TestBackupTTLService.java | 197 +++++++++++++++ .../netflix/priam/utils/TestDateUtils.java | 14 +- 26 files changed, 835 insertions(+), 138 deletions(-) create mode 100644 priam/src/main/java/com/netflix/priam/services/BackupTTLService.java create mode 100644 priam/src/test/java/com/netflix/priam/backupv2/TestBackupUtils.java create mode 100644 priam/src/test/java/com/netflix/priam/services/TestBackupTTLService.java diff --git a/build.gradle b/build.gradle index e94f09413..a9dd2bf9b 100644 --- a/build.gradle +++ b/build.gradle @@ -42,23 +42,23 @@ allprojects { compile 'com.sun.jersey.contribs:jersey-guice:1.19.4' compile 'com.google.guava:guava:21.0' compile 'com.google.code.findbugs:jsr305:3.0.2' - compile 'com.amazonaws:aws-java-sdk:1.11.386' - compile 'com.google.inject:guice:4.1.0' - compile 'com.google.inject.extensions:guice-servlet:4.1.0' + compile 'com.amazonaws:aws-java-sdk:1.11.467' + compile 'com.google.inject:guice:4.2.2' + compile 'com.google.inject.extensions:guice-servlet:4.2.2' compile 'org.quartz-scheduler:quartz:2.3.0' compile 'com.googlecode.json-simple:json-simple:1.1.1' - compile 'org.xerial.snappy:snappy-java:1.1.2.6' - compile 'org.yaml:snakeyaml:1.19' + compile 'org.xerial.snappy:snappy-java:1.1.7.2' + compile 'org.yaml:snakeyaml:1.23' compile 'org.apache.cassandra:cassandra-all:3.0.17' compile 'javax.ws.rs:jsr311-api:1.1.1' - compile 'joda-time:joda-time:2.9.9' + compile 'joda-time:joda-time:2.10.1' compile 'org.apache.commons:commons-configuration2:2.1.1' - compile 'xerces:xercesImpl:2.11.0' + compile 'xerces:xercesImpl:2.12.0' compile 'net.java.dev.jna:jna:4.4.0' compile 'org.apache.httpcomponents:httpclient:4.5.3' compile 'org.apache.httpcomponents:httpcore:4.4.6' compile 'com.ning:compress-lzf:1.0.4' - compile 'com.google.code.gson:gson:2.8.2' + compile 'com.google.code.gson:gson:2.8.5' compile 'org.slf4j:slf4j-api:1.7.25' compile 'org.slf4j:slf4j-log4j12:1.7.25' compile 'org.bouncycastle:bcprov-jdk16:1.46' @@ -68,7 +68,7 @@ allprojects { } compile 'com.google.apis:google-api-services-storage:v1-rev100-1.22.0' compile 'com.google.http-client:google-http-client-jackson2:1.22.0' - compile 'com.netflix.spectator:spectator-api:0.74.2' + compile 'com.netflix.spectator:spectator-api:0.81.2' compileOnly 'javax.servlet:javax.servlet-api:3.1.0' testCompile 'org.jmockit:jmockit:1.31' testCompile "org.spockframework:spock-core:1.1-groovy-2.4" diff --git a/priam/src/main/java/com/netflix/priam/PriamServer.java b/priam/src/main/java/com/netflix/priam/PriamServer.java index 299b809a6..6e04dffaf 100644 --- a/priam/src/main/java/com/netflix/priam/PriamServer.java +++ b/priam/src/main/java/com/netflix/priam/PriamServer.java @@ -34,6 +34,7 @@ import com.netflix.priam.restore.RestoreContext; import com.netflix.priam.scheduler.PriamScheduler; import com.netflix.priam.scheduler.TaskTimer; +import com.netflix.priam.services.BackupTTLService; import com.netflix.priam.services.SnapshotMetaService; import com.netflix.priam.tuner.TuneCassandra; import com.netflix.priam.utils.CassandraMonitor; @@ -173,7 +174,9 @@ public void initialize() throws Exception { TaskTimer flushTaskTimer = Flush.getTimer(config); if (flushTaskTimer != null) { scheduler.addTask(IClusterManagement.Task.FLUSH.name(), Flush.class, flushTaskTimer); - logger.info("Added nodetool flush task."); + logger.info( + "Added nodetool flush task with schedule: [{}]", + flushTaskTimer.getCronExpression()); } // Set up compaction task @@ -181,7 +184,9 @@ public void initialize() throws Exception { if (compactionTimer != null) { scheduler.addTask( IClusterManagement.Task.COMPACTION.name(), Compaction.class, compactionTimer); - logger.info("Added compaction task."); + logger.info( + "Added compaction task with schedule: [{}]", + compactionTimer.getCronExpression()); } // Set up the background configuration dumping thread @@ -192,7 +197,7 @@ public void initialize() throws Exception { PriamConfigurationPersister.class, configurationPersisterTimer); logger.info( - "Added configuration persister task with schedule [{}]", + "Added configuration persister task with schedule: [{}]", configurationPersisterTimer.getCronExpression()); } else { logger.warn("Priam configuration persister disabled!"); @@ -209,11 +214,23 @@ private void setUpSnapshotService() throws Exception { SnapshotMetaService.JOBNAME, SnapshotMetaService.class, snapshotMetaServiceTimer); - logger.info("Added SnapshotMetaService Task."); + logger.info( + "Added SnapshotMetaService Task with schedule: [{}]", + snapshotMetaServiceTimer.getCronExpression()); // Try to upload previous snapshots, if any which might have been interrupted by Priam // restart. snapshotMetaService.uploadFiles(); + + // Schedule the TTL service + TaskTimer backupTTLTimer = BackupTTLService.getTimer(backupRestoreConfig); + if (backupTTLTimer != null) { + scheduler.addTask(BackupTTLService.JOBNAME, BackupTTLService.class, backupTTLTimer); + logger.info( + "Added {} Task with schedule: [{}]", + BackupTTLService.JOBNAME, + backupTTLTimer.getCronExpression()); + } } } diff --git a/priam/src/main/java/com/netflix/priam/aws/RemoteBackupPath.java b/priam/src/main/java/com/netflix/priam/aws/RemoteBackupPath.java index 1538c68b3..43140cc64 100644 --- a/priam/src/main/java/com/netflix/priam/aws/RemoteBackupPath.java +++ b/priam/src/main/java/com/netflix/priam/aws/RemoteBackupPath.java @@ -20,6 +20,7 @@ import com.netflix.priam.backup.AbstractBackupPath; import com.netflix.priam.compress.ICompression; import com.netflix.priam.config.IConfiguration; +import com.netflix.priam.cryptography.IFileCryptography; import com.netflix.priam.identity.InstanceIdentity; import java.nio.file.Path; import java.nio.file.Paths; @@ -90,27 +91,40 @@ private Path getV2Location() { prefix = Paths.get(prefix.toString(), keyspace, columnFamily); } - return Paths.get(prefix.toString(), fileName + "." + getCompression().toString()); + return Paths.get( + prefix.toString(), + getCompression().toString(), + getEncryption().toString(), + fileName); } private void parseV2Location(String remoteFile) { Path remoteFilePath = Paths.get(remoteFile); parseV2Prefix(remoteFilePath); - assert remoteFilePath.getNameCount() >= 6; - type = BackupFileType.valueOf(remoteFilePath.getName(3).toString()); - setLastModified(Instant.ofEpochMilli(Long.parseLong(remoteFilePath.getName(4).toString()))); + if (remoteFilePath.getNameCount() < 8) + throw new IndexOutOfBoundsException( + String.format( + "Too few elements (expected: [%d]) in path: %s", 8, remoteFilePath)); + int name_count_idx = 3; + + type = BackupFileType.valueOf(remoteFilePath.getName(name_count_idx++).toString()); + setLastModified( + Instant.ofEpochMilli( + Long.parseLong(remoteFilePath.getName(name_count_idx++).toString()))); if (type == BackupFileType.SST_V2) { - keyspace = remoteFilePath.getName(5).toString(); - columnFamily = remoteFilePath.getName(6).toString(); + keyspace = remoteFilePath.getName(name_count_idx++).toString(); + columnFamily = remoteFilePath.getName(name_count_idx++).toString(); } - String fileNameCompression = - remoteFilePath.getName(remoteFilePath.getNameCount() - 1).toString(); - fileName = fileNameCompression.substring(0, fileNameCompression.lastIndexOf(".")); setCompression( ICompression.CompressionAlgorithm.valueOf( - fileNameCompression.substring(fileNameCompression.lastIndexOf(".") + 1))); + remoteFilePath.getName(name_count_idx++).toString())); + + setEncryption( + IFileCryptography.CryptographyAlgorithm.valueOf( + remoteFilePath.getName(name_count_idx++).toString())); + fileName = remoteFilePath.getName(name_count_idx).toString(); } private Path getV1Location() { @@ -122,7 +136,10 @@ private Path getV1Location() { private void parseV1Location(Path remoteFilePath) { parseV1Prefix(remoteFilePath); - assert remoteFilePath.getNameCount() >= 7 : "Too few elements in path " + remoteFilePath; + if (remoteFilePath.getNameCount() < 7) + throw new IndexOutOfBoundsException( + String.format( + "Too few elements (expected: [%d]) in path: %s", 7, remoteFilePath)); time = parseDate(remoteFilePath.getName(4).toString()); type = BackupFileType.valueOf(remoteFilePath.getName(5).toString()); @@ -220,7 +237,10 @@ public String clusterPrefix(String location) { region = instanceIdentity.getInstanceInfo().getRegion(); clusterName = config.getAppName(); } else { - assert elements.length >= 4 : "Too few elements in path " + location; + if (elements.length < 4) + throw new IndexOutOfBoundsException( + String.format( + "Too few elements (expected: [%d]) in path: %s", 4, location)); baseDir = elements[1]; region = elements[2]; clusterName = elements[3]; diff --git a/priam/src/main/java/com/netflix/priam/aws/S3FileSystemBase.java b/priam/src/main/java/com/netflix/priam/aws/S3FileSystemBase.java index 69df53fc1..0bdb4ee4a 100755 --- a/priam/src/main/java/com/netflix/priam/aws/S3FileSystemBase.java +++ b/priam/src/main/java/com/netflix/priam/aws/S3FileSystemBase.java @@ -19,6 +19,7 @@ import com.amazonaws.services.s3.model.BucketLifecycleConfiguration; import com.amazonaws.services.s3.model.BucketLifecycleConfiguration.Rule; import com.amazonaws.services.s3.model.CompleteMultipartUploadResult; +import com.amazonaws.services.s3.model.DeleteObjectsRequest; import com.google.common.collect.Lists; import com.google.common.util.concurrent.RateLimiter; import com.google.inject.Provider; @@ -34,6 +35,7 @@ import java.util.Iterator; import java.util.List; import java.util.concurrent.LinkedBlockingQueue; +import java.util.stream.Collectors; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -193,6 +195,28 @@ public Iterator listFileSystem(String prefix, String delimiter, String m return new S3Iterator(s3Client, getShard(), prefix, delimiter, marker); } + @Override + public void deleteRemoteFiles(List remotePaths) throws BackupRestoreException { + if (remotePaths.isEmpty()) return; + + try { + List keys = + remotePaths + .stream() + .map( + remotePath -> + new DeleteObjectsRequest.KeyVersion( + remotePath.toString())) + .collect(Collectors.toList()); + s3Client.deleteObjects( + new DeleteObjectsRequest(getShard()).withKeys(keys).withQuiet(true)); + logger.info("Deleted {} objects from S3", remotePaths.size()); + } catch (Exception e) { + logger.error("Error while trying to delete the objects from S3: {}", e.getMessage()); + throw new BackupRestoreException(e + " while trying to delete the objects"); + } + } + final long getChunkSize(Path localPath) throws BackupRestoreException { long chunkSize = config.getBackupChunkSize(); long fileSize = localPath.toFile().length(); diff --git a/priam/src/main/java/com/netflix/priam/backup/AbstractBackupPath.java b/priam/src/main/java/com/netflix/priam/backup/AbstractBackupPath.java index b5ac4ad80..ad0cf036e 100644 --- a/priam/src/main/java/com/netflix/priam/backup/AbstractBackupPath.java +++ b/priam/src/main/java/com/netflix/priam/backup/AbstractBackupPath.java @@ -20,6 +20,7 @@ import com.netflix.priam.aws.RemoteBackupPath; import com.netflix.priam.compress.ICompression; import com.netflix.priam.config.IConfiguration; +import com.netflix.priam.cryptography.IFileCryptography; import com.netflix.priam.identity.InstanceIdentity; import java.io.File; import java.nio.file.Path; @@ -73,6 +74,8 @@ public static boolean isDataFile(BackupFileType type) { private Date uploadedTs; private ICompression.CompressionAlgorithm compression = ICompression.CompressionAlgorithm.SNAPPY; + private IFileCryptography.CryptographyAlgorithm encryption = + IFileCryptography.CryptographyAlgorithm.PLAINTEXT; public AbstractBackupPath(IConfiguration config, InstanceIdentity instanceIdentity) { this.instanceIdentity = instanceIdentity; @@ -278,6 +281,14 @@ public void setCompression(ICompression.CompressionAlgorithm compression) { this.compression = compression; } + public IFileCryptography.CryptographyAlgorithm getEncryption() { + return encryption; + } + + public void setEncryption(IFileCryptography.CryptographyAlgorithm encryption) { + this.encryption = encryption; + } + @Override public String toString() { return "From: " + getRemotePath() + " To: " + newRestoreFile().getPath(); diff --git a/priam/src/main/java/com/netflix/priam/backup/IBackupFileSystem.java b/priam/src/main/java/com/netflix/priam/backup/IBackupFileSystem.java index ceff7c0cf..a8ef84209 100644 --- a/priam/src/main/java/com/netflix/priam/backup/IBackupFileSystem.java +++ b/priam/src/main/java/com/netflix/priam/backup/IBackupFileSystem.java @@ -20,6 +20,7 @@ import java.nio.file.Path; import java.util.Date; import java.util.Iterator; +import java.util.List; import java.util.concurrent.Future; import java.util.concurrent.RejectedExecutionException; @@ -179,12 +180,25 @@ default String getShard() { * * @param remotePath location on the remote file system. * @return boolean value indicating presence of the file on remote file system. - * @throws BackupRestoreException + * @throws BackupRestoreException in case of failure to identify if object exists on the remote + * file system. */ default boolean doesRemoteFileExist(Path remotePath) throws BackupRestoreException { return false; } + /** + * Delete list of remote files from the remote file system. It should throw exception if there + * is anything wrong in processing the request. If the remotePath passed do not exist, then it + * should just keep quiet. + * + * @param remotePaths list of files on remote file system to be deleted. This path may or may + * not exist. + * @throws BackupRestoreException in case of remote file system not able to process the request + * or unable to reach. + */ + void deleteRemoteFiles(List remotePaths) throws BackupRestoreException; + /** * Get the number of tasks en-queue in the filesystem for upload. * diff --git a/priam/src/main/java/com/netflix/priam/backupv2/BackupValidator.java b/priam/src/main/java/com/netflix/priam/backupv2/BackupValidator.java index eb7308f32..583bbad41 100644 --- a/priam/src/main/java/com/netflix/priam/backupv2/BackupValidator.java +++ b/priam/src/main/java/com/netflix/priam/backupv2/BackupValidator.java @@ -110,8 +110,7 @@ public AbstractBackupPath findLatestValidMetaFile(DateUtil.DateRange dateRange) logger.info("Meta files found: {}", metas); for (AbstractBackupPath meta : metas) { - Path localFile = Paths.get(meta.newRestoreFile().getAbsolutePath()); - fs.downloadFile(Paths.get(meta.getRemotePath()), localFile, 10); + Path localFile = downloadMetaFile(meta); boolean isValid = isMetaFileValid(localFile); logger.info("Meta: {}, isValid: {}", meta, isValid); if (!isValid) FileUtils.deleteQuietly(localFile.toFile()); @@ -121,6 +120,19 @@ public AbstractBackupPath findLatestValidMetaFile(DateUtil.DateRange dateRange) return null; } + /** + * Download the meta file to disk. + * + * @param meta AbstractBackupPath denoting the meta file on remote file system. + * @return the location of the meta file on disk after downloading from remote file system. + * @throws BackupRestoreException if unable to download for any reason. + */ + public Path downloadMetaFile(AbstractBackupPath meta) throws BackupRestoreException { + Path localFile = Paths.get(meta.newRestoreFile().getAbsolutePath()); + fs.downloadFile(Paths.get(meta.getRemotePath()), localFile, 10); + return localFile; + } + /** * Get the prefix for the META_V2 file. This will depend on the configuration, if restore prefix * is set. diff --git a/priam/src/main/java/com/netflix/priam/backupv2/FileUploadResult.java b/priam/src/main/java/com/netflix/priam/backupv2/FileUploadResult.java index 8aebe76f6..d0228cb4f 100644 --- a/priam/src/main/java/com/netflix/priam/backupv2/FileUploadResult.java +++ b/priam/src/main/java/com/netflix/priam/backupv2/FileUploadResult.java @@ -17,6 +17,7 @@ package com.netflix.priam.backupv2; import com.netflix.priam.compress.ICompression; +import com.netflix.priam.cryptography.IFileCryptography; import com.netflix.priam.utils.GsonJsonSerializer; import java.io.File; import java.nio.file.Files; @@ -38,6 +39,9 @@ public class FileUploadResult { // Valid compression technique for now is SNAPPY only. Future we need to support LZ4 and NONE private ICompression.CompressionAlgorithm compression = ICompression.CompressionAlgorithm.SNAPPY; + // Valid encryption technique for now is PLAINTEXT only. In future we will support pgp and more. + private IFileCryptography.CryptographyAlgorithm encryption = + IFileCryptography.CryptographyAlgorithm.PLAINTEXT; private String backupPath; public FileUploadResult( @@ -112,6 +116,14 @@ public void setCompression(ICompression.CompressionAlgorithm compression) { this.compression = compression; } + public IFileCryptography.CryptographyAlgorithm getEncryption() { + return encryption; + } + + public void setEncryption(IFileCryptography.CryptographyAlgorithm encryption) { + this.encryption = encryption; + } + public String getBackupPath() { return backupPath; } diff --git a/priam/src/main/java/com/netflix/priam/config/IBackupRestoreConfig.java b/priam/src/main/java/com/netflix/priam/config/IBackupRestoreConfig.java index 9d5873d8d..86d366a0e 100644 --- a/priam/src/main/java/com/netflix/priam/config/IBackupRestoreConfig.java +++ b/priam/src/main/java/com/netflix/priam/config/IBackupRestoreConfig.java @@ -43,4 +43,20 @@ default String getSnapshotMetaServiceCronExpression() { default boolean enableV2Backups() { return false; } + + /** + * Cron expression to be used for the service which does TTL of the backups. This service will + * run only if v2 backups are enabled. The idea is to run this service at least once a day to + * ensure we are marking backup files for TTL as configured via {@link + * IConfiguration#getBackupRetentionDays()} + * + * @return Backup TTL Service cron expression for trying to delete backups. Note that this CRON + * is only the job trying to delete backups and is not the TTL of the backups. + * @see quartz-scheduler + * @see http://www.cronmaker.com + */ + default String getBackupTTLCronExpression() { + return "0 0 0/6 1/1 * ? *"; + } } diff --git a/priam/src/main/java/com/netflix/priam/cryptography/IFileCryptography.java b/priam/src/main/java/com/netflix/priam/cryptography/IFileCryptography.java index 3e8d0d58b..7cf26ad5c 100755 --- a/priam/src/main/java/com/netflix/priam/cryptography/IFileCryptography.java +++ b/priam/src/main/java/com/netflix/priam/cryptography/IFileCryptography.java @@ -18,6 +18,11 @@ public interface IFileCryptography { + enum CryptographyAlgorithm { + PLAINTEXT, + PGP + } + /** * @param in - a handle to the encrypted, compressed data stream * @param passwd - pass phrase used to extract the PGP private key from the encrypted content. diff --git a/priam/src/main/java/com/netflix/priam/google/GoogleEncryptedFileSystem.java b/priam/src/main/java/com/netflix/priam/google/GoogleEncryptedFileSystem.java index 0d26f51d9..cc49b7679 100755 --- a/priam/src/main/java/com/netflix/priam/google/GoogleEncryptedFileSystem.java +++ b/priam/src/main/java/com/netflix/priam/google/GoogleEncryptedFileSystem.java @@ -37,6 +37,7 @@ import java.util.ArrayList; import java.util.Collection; import java.util.Iterator; +import java.util.List; import org.apache.commons.io.IOUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -235,6 +236,11 @@ public long getFileSize(Path remotePath) throws BackupRestoreException { return 0; } + @Override + public void deleteRemoteFiles(List remotePaths) throws BackupRestoreException { + // TODO: Delete implementation + } + /* * @param pathPrefix * @return objectName diff --git a/priam/src/main/java/com/netflix/priam/identity/InstanceIdentity.java b/priam/src/main/java/com/netflix/priam/identity/InstanceIdentity.java index 6825ab9f5..31e0d8bbd 100644 --- a/priam/src/main/java/com/netflix/priam/identity/InstanceIdentity.java +++ b/priam/src/main/java/com/netflix/priam/identity/InstanceIdentity.java @@ -80,11 +80,6 @@ public boolean apply(PriamInstance instance) { return (!instance.getInstanceId().equalsIgnoreCase(DUMMY_INSTANCE_ID)); } } - - @Override - public boolean test(PriamInstance input) { - return apply(input); - } }; private PriamInstance myInstance; diff --git a/priam/src/main/java/com/netflix/priam/notification/BackupNotificationMgr.java b/priam/src/main/java/com/netflix/priam/notification/BackupNotificationMgr.java index 158b40216..ba775dce6 100644 --- a/priam/src/main/java/com/netflix/priam/notification/BackupNotificationMgr.java +++ b/priam/src/main/java/com/netflix/priam/notification/BackupNotificationMgr.java @@ -66,6 +66,8 @@ private void notify(AbstractBackupPath abp, String uploadStatus) { jsonObject.put("compressfilesize", abp.getCompressedFileSize()); jsonObject.put("backuptype", abp.getType().name()); jsonObject.put("uploadstatus", uploadStatus); + jsonObject.put("compression", abp.getCompression().name()); + jsonObject.put("encryption", abp.getEncryption().name()); // SNS Attributes for filtering messages. Cluster name and backup file type. Map messageAttributes = new HashMap<>(); diff --git a/priam/src/main/java/com/netflix/priam/resources/BackupServlet.java b/priam/src/main/java/com/netflix/priam/resources/BackupServlet.java index e4a6b2b0f..5d14c0eec 100644 --- a/priam/src/main/java/com/netflix/priam/resources/BackupServlet.java +++ b/priam/src/main/java/com/netflix/priam/resources/BackupServlet.java @@ -281,7 +281,7 @@ public Response validateV2SnapshotByDate(@PathParam("daterange") String daterang AbstractBackupPath validMeta = backupValidator.findLatestValidMetaFile(dateRange); if (validMeta == null) { return Response.noContent() - .entity("No valid meta found for provided timerange") + .entity("No valid meta found for provided time range") .build(); } diff --git a/priam/src/main/java/com/netflix/priam/services/BackupTTLService.java b/priam/src/main/java/com/netflix/priam/services/BackupTTLService.java new file mode 100644 index 000000000..a4f770d70 --- /dev/null +++ b/priam/src/main/java/com/netflix/priam/services/BackupTTLService.java @@ -0,0 +1,234 @@ +/* + * Copyright 2018 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.netflix.priam.services; + +import com.google.inject.Inject; +import com.google.inject.Provider; +import com.google.inject.Singleton; +import com.netflix.priam.backup.AbstractBackupPath; +import com.netflix.priam.backup.BackupRestoreException; +import com.netflix.priam.backup.IBackupFileSystem; +import com.netflix.priam.backup.IFileSystemContext; +import com.netflix.priam.backupv2.BackupValidator; +import com.netflix.priam.backupv2.ColumnfamilyResult; +import com.netflix.priam.backupv2.MetaFileReader; +import com.netflix.priam.config.IBackupRestoreConfig; +import com.netflix.priam.config.IConfiguration; +import com.netflix.priam.scheduler.CronTimer; +import com.netflix.priam.scheduler.Task; +import com.netflix.priam.scheduler.TaskTimer; +import com.netflix.priam.utils.DateUtil; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.time.Instant; +import java.time.temporal.ChronoUnit; +import java.util.*; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReentrantLock; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * This class is used to TTL or delete the SSTable components from the backups after they are not + * referenced in the backups for more than {@link IConfiguration#getBackupRetentionDays()}. This + * operation is executed on CRON and is configured via {@link + * IBackupRestoreConfig#getBackupTTLCronExpression()}. + * + *

    To TTL the SSTable components we refer to the first manifest file on the remote file system + * after the TTL period. Any sstable components referenced in that manifest file should not be + * deleted. Any other sstable components (files) on remote file system before the TTL period can be + * safely deleted. Created by aagrawal on 11/26/18. + */ +@Singleton +public class BackupTTLService extends Task { + private static final Logger logger = LoggerFactory.getLogger(BackupTTLService.class); + private IBackupRestoreConfig backupRestoreConfig; + private BackupValidator backupValidator; + private IBackupFileSystem fileSystem; + private Provider abstractBackupPathProvider; + public static final String JOBNAME = "BackupTTLService"; + private Map filesInMeta = new HashMap<>(); + private List filesToDelete = new ArrayList<>(); + private static final Lock lock = new ReentrantLock(); + private final int BATCH_SIZE = 1000; + private final Instant start_of_feature = DateUtil.parseInstant("201801010000"); + + @Inject + public BackupTTLService( + IConfiguration configuration, + IBackupRestoreConfig backupRestoreConfig, + BackupValidator backupValidator, + IFileSystemContext backupFileSystemCtx, + Provider abstractBackupPathProvider) { + super(configuration); + this.backupRestoreConfig = backupRestoreConfig; + this.backupValidator = backupValidator; + this.fileSystem = backupFileSystemCtx.getFileStrategy(configuration); + this.abstractBackupPathProvider = abstractBackupPathProvider; + } + + @Override + public void execute() throws Exception { + // Ensure that backup version 2.0 is actually enabled. + if (!backupRestoreConfig.enableV2Backups()) { + logger.info("Not executing the TTL Service for backups as V2 backups are not enabled."); + return; + } + + // Do not allow more than one backupTTLService to run at the same time. This is possible + // as this happens on CRON. + if (!lock.tryLock()) { + logger.warn("{} is already running! Try again later.", JOBNAME); + throw new Exception(JOBNAME + " already running"); + } + + try { + filesInMeta.clear(); + filesToDelete.clear(); + + Instant dateToTtl = + DateUtil.getInstant().minus(config.getBackupRetentionDays(), ChronoUnit.DAYS); + logger.info( + "Will try to delete(TTL) files which are before this time: {}. TTL in Days: {}", + dateToTtl.toEpochMilli(), + config.getBackupRetentionDays()); + + // Find the snapshot just after this date. + List metas = + backupValidator.findMetaFiles( + new DateUtil.DateRange(dateToTtl, DateUtil.getInstant())); + + if (metas.size() == 0) { + logger.info("No meta file found and thus cannot run TTL Service"); + return; + } + + // Get the first file after the TTL time as we get files which are sorted latest to + // oldest. + AbstractBackupPath metaFile = metas.get(metas.size() - 1); + + // Download the meta file to local file system. + Path localFile = backupValidator.downloadMetaFile(metaFile); + + // Walk over the file system iterator and if not in map, it is eligible for delete. + new MetaFileWalker().readMeta(localFile); + if (logger.isDebugEnabled()) + logger.debug("Files in meta file: {}", filesInMeta.keySet().toString()); + + Iterator remoteFileLocations = + fileSystem.listFileSystem(getSSTPrefix(), null, null); + + while (remoteFileLocations.hasNext()) { + AbstractBackupPath abstractBackupPath = abstractBackupPathProvider.get(); + abstractBackupPath.parseRemote(remoteFileLocations.next()); + // If lastModifiedTime is after the dateToTTL, we should get out of this loop as + // remote file systems always give locations which are sorted. + if (abstractBackupPath.getLastModified().isAfter(dateToTtl)) { + logger.info( + "Breaking from TTL. Got a key which is after the TTL time: {}", + abstractBackupPath.getRemotePath()); + break; + } + + if (!filesInMeta.containsKey(abstractBackupPath.getRemotePath())) { + deleteFile(abstractBackupPath, false); + } else { + if (logger.isDebugEnabled()) + logger.debug( + "Not deleting this key as it is referenced in backups: {}", + abstractBackupPath.getRemotePath()); + } + } + + // Delete the old META files. We are giving start date which is so back in past to get + // all the META files. + // This feature did not exist in Jan 2018. + metas = + backupValidator.findMetaFiles( + new DateUtil.DateRange( + start_of_feature, dateToTtl.minus(1, ChronoUnit.MINUTES))); + + for (AbstractBackupPath meta : metas) { + deleteFile(meta, false); + } + + // Delete remaining files. + deleteFile(null, true); + + logger.info("Finished processing files for TTL service"); + } finally { + lock.unlock(); + } + } + + private void deleteFile(AbstractBackupPath path, boolean forceClear) + throws BackupRestoreException { + if (path != null) filesToDelete.add(Paths.get(path.getRemotePath())); + + if (forceClear || filesToDelete.size() >= BATCH_SIZE) { + fileSystem.deleteRemoteFiles(filesToDelete); + filesToDelete.clear(); + } + } + + private String getSSTPrefix() { + Path location = fileSystem.getPrefix(); + AbstractBackupPath abstractBackupPath = abstractBackupPathProvider.get(); + return abstractBackupPath + .remoteV2Prefix(location, AbstractBackupPath.BackupFileType.SST_V2) + .toString(); + } + + @Override + public String getName() { + return JOBNAME; + } + + /** + * Interval between trying to TTL data on Remote file system. + * + * @param backupRestoreConfig {@link + * IBackupRestoreConfig#getSnapshotMetaServiceCronExpression()} to get configuration details + * from priam. Use "-1" to disable the service. + * @return the timer to be used for snapshot meta service. + * @throws Exception if the configuration is not set correctly or are not valid. This is to + * ensure we fail-fast. + */ + public static TaskTimer getTimer(IBackupRestoreConfig backupRestoreConfig) throws Exception { + String cronExpression = backupRestoreConfig.getBackupTTLCronExpression(); + return CronTimer.getCronTimer(JOBNAME, cronExpression); + } + + private class MetaFileWalker extends MetaFileReader { + @Override + public void process(ColumnfamilyResult columnfamilyResult) { + columnfamilyResult + .getSstables() + .forEach( + ssTableResult -> + ssTableResult + .getSstableComponents() + .forEach( + fileUploadResult -> + filesInMeta.put( + fileUploadResult + .getBackupPath(), + null))); + } + } +} diff --git a/priam/src/main/java/com/netflix/priam/utils/DateUtil.java b/priam/src/main/java/com/netflix/priam/utils/DateUtil.java index 4897a4f70..edcee23af 100644 --- a/priam/src/main/java/com/netflix/priam/utils/DateUtil.java +++ b/priam/src/main/java/com/netflix/priam/utils/DateUtil.java @@ -18,6 +18,7 @@ package com.netflix.priam.utils; import java.time.Instant; +import java.time.LocalDate; import java.time.LocalDateTime; import java.time.ZoneId; import java.time.format.DateTimeFormatter; @@ -34,8 +35,7 @@ public class DateUtil { public static final String yyyyMMdd = "yyyyMMdd"; public static final String yyyyMMddHHmm = "yyyyMMddHHmm"; - public static final String ddMMyyyyHHmm = "ddMMyyyyHHmm"; - private static final String[] patterns = {yyyyMMddHHmm, yyyyMMdd, ddMMyyyyHHmm}; + private static final String[] patterns = {yyyyMMddHHmm, yyyyMMdd}; private static final ZoneId defaultZoneId = ZoneId.systemDefault(); private static final ZoneId utcZoneId = ZoneId.of("UTC"); @@ -125,11 +125,20 @@ public static String formatyyyyMMddHHmm(LocalDateTime date) { public static LocalDateTime getLocalDateTime(String date) { if (StringUtils.isEmpty(date)) return null; - for (String pattern : patterns) { + try { LocalDateTime localDateTime = - LocalDateTime.parse(date, DateTimeFormatter.ofPattern(pattern)); + LocalDateTime.parse(date, DateTimeFormatter.ofPattern(yyyyMMddHHmm)); if (localDateTime != null) return localDateTime; + } catch (DateTimeParseException e) { + // Try the date only. + try { + LocalDate localDate = LocalDate.parse(date, DateTimeFormatter.ofPattern(yyyyMMdd)); + return localDate.atTime(0, 0); + } catch (DateTimeParseException ex) { + return null; + } } + return null; } @@ -164,20 +173,9 @@ public static String formatInstant(String pattern, Instant instant) { * @return Instant object depicting the date/time. */ public static final Instant parseInstant(String dateTime) { - if (StringUtils.isEmpty(dateTime)) return null; - - for (String pattern : patterns) { - try { - Instant instant = - DateTimeFormatter.ofPattern(pattern) - .withZone(utcZoneId) - .parse(dateTime, Instant::from); - if (instant != null) return instant; - } catch (DateTimeParseException e) { - // Do nothing. - } - } - return null; + LocalDateTime localDateTime = getLocalDateTime(dateTime); + if (localDateTime == null) return null; + return localDateTime.atZone(utcZoneId).toInstant(); } public static class DateRange { diff --git a/priam/src/test/java/com/netflix/priam/aws/TestRemoteBackupPath.java b/priam/src/test/java/com/netflix/priam/aws/TestRemoteBackupPath.java index e8bb8b874..90fca2b3a 100644 --- a/priam/src/test/java/com/netflix/priam/aws/TestRemoteBackupPath.java +++ b/priam/src/test/java/com/netflix/priam/aws/TestRemoteBackupPath.java @@ -24,6 +24,7 @@ import com.netflix.priam.backup.AbstractBackupPath.BackupFileType; import com.netflix.priam.backup.BRTestModule; import com.netflix.priam.config.IConfiguration; +import com.netflix.priam.cryptography.IFileCryptography; import com.netflix.priam.utils.DateUtil; import java.nio.file.Path; import java.nio.file.Paths; @@ -86,6 +87,8 @@ private void validateAbstractBackupPath(AbstractBackupPath abp1, AbstractBackupP Assert.assertEquals(abp1.getColumnFamily(), abp2.getColumnFamily()); Assert.assertEquals(abp1.getFileName(), abp2.getFileName()); Assert.assertEquals(abp1.getType(), abp2.getType()); + Assert.assertEquals(abp1.getCompression(), abp2.getCompression()); + Assert.assertEquals(abp1.getEncryption(), abp2.getEncryption()); } @Test @@ -162,6 +165,7 @@ public void testV2BackupPathSST() throws ParseException { 0, abstractBackupPath.getLastModified().toEpochMilli()); // File do not exist. Assert.assertEquals("keyspace1", abstractBackupPath.getKeyspace()); Assert.assertEquals("columnfamily1", abstractBackupPath.getColumnFamily()); + Assert.assertEquals("SNAPPY", abstractBackupPath.getCompression().name()); Assert.assertEquals(BackupFileType.SST_V2, abstractBackupPath.getType()); Assert.assertEquals(path.toFile(), abstractBackupPath.getBackupFile()); @@ -171,8 +175,6 @@ public void testV2BackupPathSST() throws ParseException { String remotePath = abstractBackupPath.getRemotePath(); logger.info(remotePath); - Assert.assertTrue(remotePath.endsWith(abstractBackupPath.getCompression().toString())); - AbstractBackupPath abstractBackupPath2 = pathFactory.get(); abstractBackupPath2.parseRemote(remotePath); Assert.assertEquals(now, abstractBackupPath2.getLastModified()); @@ -192,6 +194,9 @@ public void testV2BackupPathMeta() throws ParseException { Assert.assertEquals(null, abstractBackupPath.getColumnFamily()); Assert.assertEquals(BackupFileType.META_V2, abstractBackupPath.getType()); Assert.assertEquals(path.toFile(), abstractBackupPath.getBackupFile()); + Assert.assertEquals( + IFileCryptography.CryptographyAlgorithm.PLAINTEXT, + abstractBackupPath.getEncryption()); // Verify toRemote and parseRemote. Instant now = DateUtil.getInstant(); @@ -199,7 +204,7 @@ public void testV2BackupPathMeta() throws ParseException { String remotePath = abstractBackupPath.getRemotePath(); logger.info(remotePath); - Assert.assertTrue(remotePath.endsWith(abstractBackupPath.getCompression().toString())); + Assert.assertEquals("SNAPPY", abstractBackupPath.getCompression().name()); AbstractBackupPath abstractBackupPath2 = pathFactory.get(); abstractBackupPath2.parseRemote(remotePath); diff --git a/priam/src/test/java/com/netflix/priam/backup/FakeBackupFileSystem.java b/priam/src/test/java/com/netflix/priam/backup/FakeBackupFileSystem.java index 7c2558d97..91dd7b101 100644 --- a/priam/src/test/java/com/netflix/priam/backup/FakeBackupFileSystem.java +++ b/priam/src/test/java/com/netflix/priam/backup/FakeBackupFileSystem.java @@ -21,7 +21,6 @@ import com.google.inject.Provider; import com.google.inject.Singleton; import com.netflix.priam.aws.RemoteBackupPath; -import com.netflix.priam.backup.AbstractBackupPath.BackupFileType; import com.netflix.priam.config.IConfiguration; import com.netflix.priam.merics.BackupMetrics; import com.netflix.priam.notification.BackupNotificationMgr; @@ -33,9 +32,9 @@ @Singleton public class FakeBackupFileSystem extends AbstractFileSystem { - private List flist; - public Set downloadedFiles; - public Set uploadedFiles; + private List flist = new ArrayList<>(); + public Set downloadedFiles = new HashSet<>(); + public Set uploadedFiles = new HashSet<>(); private String baseDir; private String region; private String clusterName; @@ -50,27 +49,19 @@ public FakeBackupFileSystem( } public void setupTest(List files) { + System.out.println("Setting up: " + files); clearTest(); - flist = new ArrayList<>(); for (String file : files) { AbstractBackupPath path = pathProvider.get(); path.parseRemote(file); flist.add(path); } - downloadedFiles = new HashSet<>(); - uploadedFiles = new HashSet<>(); } - public void setupTest() { - clearTest(); - flist = new ArrayList<>(); - downloadedFiles = new HashSet<>(); - uploadedFiles = new HashSet<>(); - } - - public void clearTest() { - if (flist != null) flist.clear(); - if (downloadedFiles != null) downloadedFiles.clear(); + private void clearTest() { + flist.clear(); + downloadedFiles.clear(); + uploadedFiles.clear(); } public void addFile(String file) { @@ -113,6 +104,8 @@ public Iterator listFileSystem(String prefix, String delimiter, String m if (abstractBackupPath.getRemotePath().startsWith(prefix)) items.add(abstractBackupPath.getRemotePath()); }); + System.out.println(flist); + System.out.println(items); return items.iterator(); } @@ -134,19 +127,34 @@ public boolean doesRemoteFileExist(Path remotePath) throws BackupRestoreExceptio return false; } + @Override + public void deleteRemoteFiles(List remotePaths) throws BackupRestoreException { + remotePaths + .stream() + .forEach( + remotePath -> { + AbstractBackupPath path = pathProvider.get(); + path.parseRemote(remotePath.toString()); + flist.remove(path); + }); + } + @Override public void cleanup() { - // TODO Auto-generated method stub + clearTest(); } @Override protected void downloadFileImpl(Path remotePath, Path localPath) throws BackupRestoreException { - { + AbstractBackupPath path = pathProvider.get(); + path.parseRemote(remotePath.toString()); + + if (path.getType() == AbstractBackupPath.BackupFileType.META) { // List all files and generate the file try (FileWriter fr = new FileWriter(localPath.toFile())) { JSONArray jsonObj = new JSONArray(); for (AbstractBackupPath filePath : flist) { - if (filePath.type == BackupFileType.SNAP) { + if (filePath.type == AbstractBackupPath.BackupFileType.SNAP) { jsonObj.add(filePath.getRemotePath()); } } @@ -157,12 +165,12 @@ protected void downloadFileImpl(Path remotePath, Path localPath) throws BackupRe } } downloadedFiles.add(remotePath.toString()); - System.out.println("Downloading " + remotePath.toString()); } @Override protected long uploadFileImpl(Path localPath, Path remotePath) throws BackupRestoreException { uploadedFiles.add(localPath.toFile().getAbsolutePath()); + addFile(remotePath.toString()); return localPath.toFile().length(); } } diff --git a/priam/src/test/java/com/netflix/priam/backup/NullBackupFileSystem.java b/priam/src/test/java/com/netflix/priam/backup/NullBackupFileSystem.java index 326537beb..626b693ab 100644 --- a/priam/src/test/java/com/netflix/priam/backup/NullBackupFileSystem.java +++ b/priam/src/test/java/com/netflix/priam/backup/NullBackupFileSystem.java @@ -25,6 +25,7 @@ import java.nio.file.Path; import java.util.Collections; import java.util.Iterator; +import java.util.List; public class NullBackupFileSystem extends AbstractFileSystem { @@ -46,6 +47,11 @@ public long getFileSize(Path remotePath) throws BackupRestoreException { return 0; } + @Override + public void deleteRemoteFiles(List remotePaths) throws BackupRestoreException { + // Do nothing. + } + @Override public Iterator listFileSystem(String prefix, String delimiter, String marker) { return Collections.emptyIterator(); diff --git a/priam/src/test/java/com/netflix/priam/backup/TestBackup.java b/priam/src/test/java/com/netflix/priam/backup/TestBackup.java index 1cde032b7..12f2e3862 100644 --- a/priam/src/test/java/com/netflix/priam/backup/TestBackup.java +++ b/priam/src/test/java/com/netflix/priam/backup/TestBackup.java @@ -64,7 +64,7 @@ public static void cleanup() throws IOException { @Test public void testSnapshotBackup() throws Exception { - filesystem.setupTest(); + filesystem.cleanup(); SnapshotBackup backup = injector.getInstance(SnapshotBackup.class); // @@ -87,7 +87,7 @@ public void testSnapshotBackup() throws Exception { @Test public void testIncrementalBackup() throws Exception { - filesystem.setupTest(); + filesystem.cleanup(); generateIncrementalFiles(); IncrementalBackup backup = injector.getInstance(IncrementalBackup.class); backup.execute(); @@ -115,7 +115,7 @@ public void testClusterSpecificColumnFamiliesSkippedFrom21() throws Exception { private void testClusterSpecificColumnFamiliesSkipped(String[] columnFamilyDirs) throws Exception { - filesystem.setupTest(); + filesystem.cleanup(); File tmp = new File("target/data/"); if (tmp.exists()) cleanup(tmp); // Generate "data" diff --git a/priam/src/test/java/com/netflix/priam/backup/TestS3FileSystem.java b/priam/src/test/java/com/netflix/priam/backup/TestS3FileSystem.java index 0ce7be25a..c78e818fa 100644 --- a/priam/src/test/java/com/netflix/priam/backup/TestS3FileSystem.java +++ b/priam/src/test/java/com/netflix/priam/backup/TestS3FileSystem.java @@ -18,6 +18,7 @@ package com.netflix.priam.backup; import com.amazonaws.AmazonClientException; +import com.amazonaws.AmazonServiceException; import com.amazonaws.SdkClientException; import com.amazonaws.services.s3.AmazonS3; import com.amazonaws.services.s3.AmazonS3Client; @@ -36,7 +37,9 @@ import java.io.File; import java.io.FileOutputStream; import java.io.IOException; +import java.nio.file.Path; import java.nio.file.Paths; +import java.util.ArrayList; import java.util.List; import mockit.Mock; import mockit.MockUp; @@ -171,6 +174,27 @@ public void testCleanupIgnore() throws Exception { Assert.assertEquals(5, rule.getExpirationInDays()); } + @Test + public void testDeleteObjects() throws Exception { + S3FileSystem fs = injector.getInstance(S3FileSystem.class); + List filesToDelete = new ArrayList<>(); + // Empty files + fs.deleteRemoteFiles(filesToDelete); + + // Lets add some random files now. + filesToDelete.add(Paths.get("a.txt")); + fs.deleteRemoteFiles(filesToDelete); + + // Emulate error now. + try { + MockAmazonS3Client.emulateError = true; + fs.deleteRemoteFiles(filesToDelete); + Assert.assertTrue(false); + } catch (BackupRestoreException e) { + Assert.assertTrue(true); + } + } + // Mock Nodeprobe class static class MockS3PartUploader extends MockUp { static int compattempts = 0; @@ -217,6 +241,7 @@ public static void setup() { static class MockAmazonS3Client extends MockUp { static boolean ruleAvailable = false; static BucketLifecycleConfiguration bconf = new BucketLifecycleConfiguration(); + static boolean emulateError = false; @Mock public InitiateMultipartUploadResult initiateMultipartUpload( @@ -254,5 +279,12 @@ public void setBucketLifecycleConfiguration( String bucketName, BucketLifecycleConfiguration bucketLifecycleConfiguration) { bconf = bucketLifecycleConfiguration; } + + @Mock + public DeleteObjectsResult deleteObjects(DeleteObjectsRequest var1) + throws SdkClientException, AmazonServiceException { + if (emulateError) throw new AmazonServiceException("Unable to reach AWS"); + return null; + } } } diff --git a/priam/src/test/java/com/netflix/priam/backupv2/TestBackupUtils.java b/priam/src/test/java/com/netflix/priam/backupv2/TestBackupUtils.java new file mode 100644 index 000000000..1cd63c960 --- /dev/null +++ b/priam/src/test/java/com/netflix/priam/backupv2/TestBackupUtils.java @@ -0,0 +1,96 @@ +/* + * Copyright 2018 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.netflix.priam.backupv2; + +import com.google.inject.Guice; +import com.google.inject.Inject; +import com.google.inject.Injector; +import com.google.inject.Provider; +import com.netflix.priam.backup.AbstractBackupPath; +import com.netflix.priam.backup.BRTestModule; +import com.netflix.priam.config.IConfiguration; +import java.io.FileWriter; +import java.io.IOException; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.time.Instant; +import java.util.ArrayList; +import java.util.List; +import org.apache.commons.io.FileUtils; + +/** Created by aagrawal on 12/17/18. */ +public class TestBackupUtils { + private MetaFileWriterBuilder metaFileWriterBuilder; + private Provider pathProvider; + protected final String keyspace = "keyspace1"; + protected final String columnfamily = "columnfamily1"; + private final String dataDir; + + @Inject + public TestBackupUtils() { + Injector injector = Guice.createInjector(new BRTestModule()); + pathProvider = injector.getProvider(AbstractBackupPath.class); + metaFileWriterBuilder = injector.getInstance(MetaFileWriterBuilder.class); + dataDir = injector.getInstance(IConfiguration.class).getDataFileLocation(); + } + + public Path createMeta(List filesToAdd, Instant snapshotTime) throws IOException { + MetaFileWriterBuilder.DataStep dataStep = + metaFileWriterBuilder.newBuilder().startMetaFileGeneration(snapshotTime); + ColumnfamilyResult columnfamilyResult = new ColumnfamilyResult(keyspace, columnfamily); + for (String file : filesToAdd) { + AbstractBackupPath path = pathProvider.get(); + path.parseRemote(file); + if (path.getType() == AbstractBackupPath.BackupFileType.SST_V2) { + ColumnfamilyResult.SSTableResult ssTableResult = + new ColumnfamilyResult.SSTableResult(); + + ssTableResult.setSstableComponents(new ArrayList<>()); + ssTableResult.getSstableComponents().add(getFileUploadResult(path)); + columnfamilyResult.addSstable(ssTableResult); + } + } + dataStep.addColumnfamilyResult(columnfamilyResult); + Path metaPath = dataStep.endMetaFileGeneration().getMetaFilePath(); + metaPath.toFile().setLastModified(snapshotTime.toEpochMilli()); + return metaPath; + } + + public String createFile(String fileName, Instant lastModifiedTime) throws Exception { + Path path = Paths.get(dataDir, keyspace, columnfamily, fileName); + FileUtils.forceMkdirParent(path.toFile()); + try (FileWriter fileWriter = new FileWriter(path.toFile())) { + fileWriter.write(""); + } + path.toFile().setLastModified(lastModifiedTime.toEpochMilli()); + return path.toString(); + } + + private FileUploadResult getFileUploadResult(AbstractBackupPath path) { + FileUploadResult fileUploadResult = + new FileUploadResult( + Paths.get(path.getFileName()), + path.getKeyspace(), + path.getColumnFamily(), + path.getLastModified(), + path.getLastModified(), + path.getSize()); + fileUploadResult.setBackupPath(path.getRemotePath()); + return fileUploadResult; + } +} diff --git a/priam/src/test/java/com/netflix/priam/backupv2/TestBackupValidator.java b/priam/src/test/java/com/netflix/priam/backupv2/TestBackupValidator.java index afdd7f2de..0a52afc2d 100644 --- a/priam/src/test/java/com/netflix/priam/backupv2/TestBackupValidator.java +++ b/priam/src/test/java/com/netflix/priam/backupv2/TestBackupValidator.java @@ -19,14 +19,12 @@ import com.google.inject.Guice; import com.google.inject.Injector; -import com.google.inject.Provider; import com.netflix.priam.backup.AbstractBackupPath; import com.netflix.priam.backup.BRTestModule; import com.netflix.priam.backup.BackupRestoreException; import com.netflix.priam.backup.FakeBackupFileSystem; import com.netflix.priam.config.IConfiguration; import com.netflix.priam.utils.DateUtil; -import java.io.IOException; import java.nio.file.Path; import java.nio.file.Paths; import java.time.Instant; @@ -40,19 +38,17 @@ /** Created by aagrawal on 12/5/18. */ public class TestBackupValidator { private FakeBackupFileSystem fs; - private Provider pathProvider; private IConfiguration configuration; private BackupValidator backupValidator; - private MetaFileWriterBuilder metaFileWriterBuilder; + private TestBackupUtils backupUtils; public TestBackupValidator() { Injector injector = Guice.createInjector(new BRTestModule()); - pathProvider = injector.getProvider(AbstractBackupPath.class); configuration = injector.getInstance(IConfiguration.class); fs = injector.getInstance(FakeBackupFileSystem.class); - fs.setupTest(getFakeFiles()); + fs.setupTest(getRemoteFakeFiles()); backupValidator = injector.getInstance(BackupValidator.class); - metaFileWriterBuilder = injector.getInstance(MetaFileWriterBuilder.class); + backupUtils = new TestBackupUtils(); } @Test @@ -78,11 +74,11 @@ public void testMetaPrefix() { @Test public void testIsMetaFileValid() throws Exception { - Path metaPath = createMeta(getFakeFiles()); + Path metaPath = backupUtils.createMeta(getRemoteFakeFiles(), DateUtil.getInstant()); Assert.assertTrue(backupValidator.isMetaFileValid(metaPath)); FileUtils.deleteQuietly(metaPath.toFile()); - List fileToAdd = getFakeFiles(); + List fileToAdd = getRemoteFakeFiles(); fileToAdd.add( Paths.get( getPrefix(), @@ -90,10 +86,12 @@ public void testIsMetaFileValid() throws Exception { "1859817645000", "keyspace1", "columnfamily1", - "file9.Data.db.SNAPPY") + "SNAPPY", + "PLAINTEXT", + "file9.Data.db") .toString()); - metaPath = createMeta(fileToAdd); + metaPath = backupUtils.createMeta(fileToAdd, DateUtil.getInstant()); Assert.assertFalse(backupValidator.isMetaFileValid(metaPath)); FileUtils.deleteQuietly(metaPath.toFile()); @@ -127,43 +125,7 @@ private String getPrefix() { return "casstestbackup/1049_fake-app/1808575600"; } - private Path createMeta(List filesToAdd) throws IOException { - MetaFileWriterBuilder.DataStep dataStep = - metaFileWriterBuilder - .newBuilder() - .startMetaFileGeneration(Instant.ofEpochMilli(1859824860000L)); - ColumnfamilyResult columnfamilyResult = - new ColumnfamilyResult("keyspace1", "columnfamily1"); - for (String file : filesToAdd) { - AbstractBackupPath path = pathProvider.get(); - path.parseRemote(file); - if (path.getType() == AbstractBackupPath.BackupFileType.SST_V2) { - ColumnfamilyResult.SSTableResult ssTableResult = - new ColumnfamilyResult.SSTableResult(); - - ssTableResult.setSstableComponents(new ArrayList<>()); - ssTableResult.getSstableComponents().add(getFileUploadResult(path)); - columnfamilyResult.addSstable(ssTableResult); - } - } - dataStep.addColumnfamilyResult(columnfamilyResult); - return dataStep.endMetaFileGeneration().getMetaFilePath(); - } - - private FileUploadResult getFileUploadResult(AbstractBackupPath path) { - FileUploadResult fileUploadResult = - new FileUploadResult( - Paths.get(path.getFileName()), - "keyspace1", - "columnfamily1", - path.getLastModified(), - path.getLastModified(), - path.getSize()); - fileUploadResult.setBackupPath(path.getRemotePath()); - return fileUploadResult; - } - - private List getFakeFiles() { + private List getRemoteFakeFiles() { List files = new ArrayList<>(); files.add( Paths.get( @@ -172,7 +134,9 @@ private List getFakeFiles() { "1859817645000", "keyspace1", "columnfamily1", - "file1.Data.db.SNAPPY")); + "SNAPPY", + "PLAINTEXT", + "file1.Data.db")); files.add( Paths.get( getPrefix(), @@ -180,14 +144,18 @@ private List getFakeFiles() { "1859818845000", "keyspace1", "columnfamily1", - "file2.Data.db.SNAPPY")); + "SNAPPY", + "PLAINTEXT", + "file2.Data.db")); files.add( Paths.get( getPrefix(), AbstractBackupPath.BackupFileType.META_V2.toString(), "1859824860000", - "meta_v2_202812071801.json.SNAPPY")); + "SNAPPY", + "PLAINTEXT", + "meta_v2_202812071801.json")); files.add( Paths.get( getPrefix(), @@ -195,7 +163,9 @@ private List getFakeFiles() { "1859826045000", "keyspace1", "columnfamily1", - "manifest.SNAPPY")); + "SNAPPY", + "PLAINTEXT", + "manifest")); files.add( Paths.get( getPrefix(), @@ -203,7 +173,9 @@ private List getFakeFiles() { "1859828410000", "keyspace1", "columnfamily1", - "file3.Data.db.SNAPPY")); + "SNAPPY", + "PLAINTEXT", + "file3.Data.db")); files.add( Paths.get( getPrefix(), @@ -211,13 +183,17 @@ private List getFakeFiles() { "1859828420000", "keyspace1", "columnfamily1", - "file4.Data.db.SNAPPY")); + "SNAPPY", + "PLAINTEXT", + "file4.Data.db")); files.add( Paths.get( getPrefix(), AbstractBackupPath.BackupFileType.META_V2.toString(), "1859828460000", - "meta_v2_202812071901.json.SNAPPY")); + "SNAPPY", + "PLAINTEXT", + "meta_v2_202812071901.json")); return files.stream().map(Path::toString).collect(Collectors.toList()); } } diff --git a/priam/src/test/java/com/netflix/priam/config/FakeBackupRestoreConfig.java b/priam/src/test/java/com/netflix/priam/config/FakeBackupRestoreConfig.java index 0194044c2..7339d5b04 100644 --- a/priam/src/test/java/com/netflix/priam/config/FakeBackupRestoreConfig.java +++ b/priam/src/test/java/com/netflix/priam/config/FakeBackupRestoreConfig.java @@ -19,4 +19,9 @@ public class FakeBackupRestoreConfig implements IBackupRestoreConfig { public String getSnapshotMetaServiceCronExpression() { return "0 0/2 * 1/1 * ? *"; // Every 2 minutes for testing purposes } + + @Override + public boolean enableV2Backups() { + return true; + } } diff --git a/priam/src/test/java/com/netflix/priam/services/TestBackupTTLService.java b/priam/src/test/java/com/netflix/priam/services/TestBackupTTLService.java new file mode 100644 index 000000000..d6e2746df --- /dev/null +++ b/priam/src/test/java/com/netflix/priam/services/TestBackupTTLService.java @@ -0,0 +1,197 @@ +/* + * Copyright 2018 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.netflix.priam.services; + +import com.google.inject.Guice; +import com.google.inject.Injector; +import com.google.inject.Provider; +import com.netflix.priam.backup.AbstractBackupPath; +import com.netflix.priam.backup.BRTestModule; +import com.netflix.priam.backup.FakeBackupFileSystem; +import com.netflix.priam.backupv2.BackupValidator; +import com.netflix.priam.backupv2.TestBackupUtils; +import com.netflix.priam.config.IConfiguration; +import com.netflix.priam.utils.BackupFileUtils; +import com.netflix.priam.utils.DateUtil; +import java.io.File; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.text.ParseException; +import java.time.Instant; +import java.time.temporal.ChronoUnit; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import org.junit.After; +import org.junit.Assert; +import org.junit.Test; + +/** Created by aagrawal on 12/17/18. */ +public class TestBackupTTLService { + + private TestBackupUtils testBackupUtils = new TestBackupUtils(); + private IConfiguration configuration; + private static BackupTTLService backupTTLService; + private static FakeBackupFileSystem backupFileSystem; + private Provider pathProvider; + private static BackupValidator backupValidator; + private Path[] metas; + private Map allFilesMap = new HashMap<>(); + + public TestBackupTTLService() { + Injector injector = Guice.createInjector(new BRTestModule()); + configuration = injector.getInstance(IConfiguration.class); + if (backupTTLService == null) + backupTTLService = injector.getInstance(BackupTTLService.class); + if (backupFileSystem == null) + backupFileSystem = injector.getInstance(FakeBackupFileSystem.class); + pathProvider = injector.getProvider(AbstractBackupPath.class); + if (backupValidator == null) backupValidator = injector.getInstance(BackupValidator.class); + } + + public void prepTest(int daysForSnapshot) throws Exception { + BackupFileUtils.cleanupDir(Paths.get(configuration.getDataFileLocation())); + Instant current = DateUtil.getInstant(); + + List list = new ArrayList<>(); + List allFiles = new ArrayList<>(); + metas = new Path[3]; + + Instant time = current.minus(daysForSnapshot + 1, ChronoUnit.DAYS); + String file1 = testBackupUtils.createFile("mc-1-Data.db", time); + String file2 = + testBackupUtils.createFile("mc-2-Data.db", time.plus(10, ChronoUnit.MINUTES)); + list.clear(); + list.add(getRemoteFromLocal(file1)); + list.add(getRemoteFromLocal(file2)); + metas[0] = testBackupUtils.createMeta(list, time.plus(20, ChronoUnit.MINUTES)); + allFiles.add(getRemoteFromLocal(file1)); + allFiles.add(getRemoteFromLocal(file2)); + + time = current.minus(daysForSnapshot, ChronoUnit.DAYS); + String file3 = testBackupUtils.createFile("mc-3-Data.db", time); + String file4 = + testBackupUtils.createFile("mc-4-Data.db", time.plus(10, ChronoUnit.MINUTES)); + list.clear(); + list.add(getRemoteFromLocal(file1)); + list.add(getRemoteFromLocal(file4)); + metas[1] = testBackupUtils.createMeta(list, time.plus(20, ChronoUnit.MINUTES)); + allFiles.add(getRemoteFromLocal(file3)); + allFiles.add(getRemoteFromLocal(file4)); + + time = current.minus(daysForSnapshot - 1, ChronoUnit.DAYS); + String file5 = testBackupUtils.createFile("mc-5-Data.db", time); + String file6 = + testBackupUtils.createFile("mc-6-Data.db", time.plus(10, ChronoUnit.MINUTES)); + String file7 = + testBackupUtils.createFile("mc-7-Data.db", time.plus(20, ChronoUnit.MINUTES)); + list.clear(); + list.add(getRemoteFromLocal(file4)); + // list.add(getRemoteFromLocal(file6)); + list.add(getRemoteFromLocal(file7)); + metas[2] = testBackupUtils.createMeta(list, time.plus(40, ChronoUnit.MINUTES)); + allFiles.add(getRemoteFromLocal(file5)); + allFiles.add(getRemoteFromLocal(file6)); + allFiles.add(getRemoteFromLocal(file7)); + + allFiles.stream() + .forEach( + file -> { + Path path = Paths.get(file); + allFilesMap.put(path.toFile().getName(), file); + }); + + for (int i = 0; i < metas.length; i++) { + AbstractBackupPath path = pathProvider.get(); + path.parseLocal(metas[i].toFile(), AbstractBackupPath.BackupFileType.META_V2); + allFiles.add(path.getRemotePath()); + allFilesMap.put("META" + i, path.getRemotePath()); + } + + backupFileSystem.setupTest(allFiles); + } + + private String getRemoteFromLocal(String localPath) throws ParseException { + AbstractBackupPath path = pathProvider.get(); + path.parseLocal(new File(localPath), AbstractBackupPath.BackupFileType.SST_V2); + return path.getRemotePath(); + } + + @After + public void cleanup() { + BackupFileUtils.cleanupDir(Paths.get(configuration.getDataFileLocation())); + backupFileSystem.cleanup(); + } + + private List getAllFiles() { + List remoteFiles = new ArrayList<>(); + backupFileSystem + .listFileSystem("", null, null) + .forEachRemaining(file -> remoteFiles.add(file)); + return remoteFiles; + } + + @Test + public void testTTL() throws Exception { + int daysForSnapshot = configuration.getBackupRetentionDays(); + prepTest(daysForSnapshot); + // Run ttl till 2nd meta file. + backupTTLService.execute(); + + List remoteFiles = getAllFiles(); + + // Confirm the files. + Assert.assertEquals(7, remoteFiles.size()); + Assert.assertTrue(remoteFiles.contains(allFilesMap.get("mc-4-Data.db"))); + Assert.assertTrue(remoteFiles.contains(allFilesMap.get("mc-5-Data.db"))); + Assert.assertTrue(remoteFiles.contains(allFilesMap.get("mc-6-Data.db"))); + Assert.assertTrue(remoteFiles.contains(allFilesMap.get("mc-7-Data.db"))); + Assert.assertTrue(remoteFiles.contains(allFilesMap.get("mc-1-Data.db"))); + Assert.assertTrue(remoteFiles.contains(allFilesMap.get("META1"))); + Assert.assertTrue(remoteFiles.contains(allFilesMap.get("META2"))); + + Assert.assertFalse(remoteFiles.contains(allFilesMap.get("mc-2-Data.db"))); + Assert.assertFalse(remoteFiles.contains(allFilesMap.get("mc-3-Data.db"))); + Assert.assertFalse(remoteFiles.contains(allFilesMap.get("META0"))); + } + + @Test + public void testTTLNext() throws Exception { + int daysForSnapshot = configuration.getBackupRetentionDays() + 1; + prepTest(daysForSnapshot); + // Run ttl till 3rd meta file. + backupTTLService.execute(); + + List remoteFiles = getAllFiles(); + System.out.println(remoteFiles); + // Confirm the files. + Assert.assertEquals(4, remoteFiles.size()); + Assert.assertTrue(remoteFiles.contains(allFilesMap.get("mc-4-Data.db"))); + Assert.assertTrue(remoteFiles.contains(allFilesMap.get("mc-6-Data.db"))); + Assert.assertTrue(remoteFiles.contains(allFilesMap.get("mc-7-Data.db"))); + Assert.assertTrue(remoteFiles.contains(allFilesMap.get("META2"))); + + Assert.assertFalse(remoteFiles.contains(allFilesMap.get("mc-1-Data.db"))); + Assert.assertFalse(remoteFiles.contains(allFilesMap.get("mc-2-Data.db"))); + Assert.assertFalse(remoteFiles.contains(allFilesMap.get("mc-3-Data.db"))); + Assert.assertFalse(remoteFiles.contains(allFilesMap.get("mc-5-Data.db"))); + Assert.assertFalse(remoteFiles.contains(allFilesMap.get("META0"))); + Assert.assertFalse(remoteFiles.contains(allFilesMap.get("META1"))); + } +} diff --git a/priam/src/test/java/com/netflix/priam/utils/TestDateUtils.java b/priam/src/test/java/com/netflix/priam/utils/TestDateUtils.java index 8820d8a42..15cc254b3 100644 --- a/priam/src/test/java/com/netflix/priam/utils/TestDateUtils.java +++ b/priam/src/test/java/com/netflix/priam/utils/TestDateUtils.java @@ -63,8 +63,14 @@ public void testDateRangeValues() { String start = "201801011201"; String end = "201801051201"; DateUtil.DateRange dateRange = new DateUtil.DateRange(start + "," + end); - Assert.assertEquals(DateUtil.getDate(start).toInstant(), dateRange.getStartTime()); - Assert.assertEquals(DateUtil.getDate(end).toInstant(), dateRange.getEndTime()); + Assert.assertEquals(Instant.ofEpochSecond(1514808060), dateRange.getStartTime()); + Assert.assertEquals(Instant.ofEpochSecond(1515153660), dateRange.getEndTime()); + + start = "20180101"; + end = "20180105"; + dateRange = new DateUtil.DateRange(start + "," + end); + Assert.assertEquals(Instant.ofEpochSecond(1514764800), dateRange.getStartTime()); + Assert.assertEquals(Instant.ofEpochSecond(1515110400), dateRange.getEndTime()); } @Test @@ -90,8 +96,8 @@ public void testFutureDateRangeValues() { String start = "202801011201"; String end = "202801051201"; DateUtil.DateRange dateRange = new DateUtil.DateRange(start + "," + end); - Assert.assertEquals(DateUtil.getDate(start).toInstant(), dateRange.getStartTime()); - Assert.assertEquals(DateUtil.getDate(end).toInstant(), dateRange.getEndTime()); + Assert.assertEquals(Instant.ofEpochSecond(1830340860), dateRange.getStartTime()); + Assert.assertEquals(Instant.ofEpochSecond(1830686460), dateRange.getEndTime()); Assert.assertEquals("1830", dateRange.match()); } } From 0c44ccc5542862562d849729458c05a5db7ff38e Mon Sep 17 00:00:00 2001 From: Arun Agrawal Date: Thu, 20 Dec 2018 16:23:13 -0800 Subject: [PATCH 042/228] Introduction of MetaProxy layers for common functions. Update the last modified time of manifest file to snapshot time. --- .../netflix/priam/aws/RemoteBackupPath.java | 9 +- .../priam/backup/AbstractBackupPath.java | 24 +- .../priam/backup/BackupVerification.java | 108 +++---- .../backup/BackupVerificationResult.java | 6 + .../netflix/priam/backup/CommitLogBackup.java | 3 +- .../priam/backup/IncrementalBackup.java | 3 +- .../com/netflix/priam/backup/MetaData.java | 3 +- .../netflix/priam/backup/SnapshotBackup.java | 3 +- .../priam/backupv2/BackupValidator.java | 89 +----- .../netflix/priam/backupv2/IMetaProxy.java | 75 +++++ .../priam/backupv2/LocalDBReaderWriter.java | 302 ------------------ .../priam/backupv2/MetaFileManager.java | 67 ---- .../priam/backupv2/MetaFileWriterBuilder.java | 20 +- .../netflix/priam/backupv2/MetaV1Proxy.java | 112 +++++++ .../netflix/priam/backupv2/MetaV2Proxy.java | 142 ++++++++ .../java/com/netflix/priam/cli/Restorer.java | 15 +- .../priam/defaultimpl/PriamGuiceModule.java | 5 + .../priam/resources/BackupServlet.java | 22 +- .../priam/resources/RestoreServlet.java | 3 +- .../priam/restore/AbstractRestore.java | 99 ++---- .../priam/restore/EncryptedRestoreBase.java | 1 - .../com/netflix/priam/restore/Restore.java | 1 - .../priam/services/BackupTTLService.java | 15 +- .../priam/services/SnapshotMetaService.java | 9 +- .../com/netflix/priam/utils/DateUtil.java | 7 + .../netflix/priam/aws/MockAmazonS3Client.java | 46 --- .../priam/aws/TestRemoteBackupPath.java | 2 +- .../netflix/priam/backup/BRTestModule.java | 5 + .../priam/backup/FakeBackupFileSystem.java | 3 - .../netflix/priam/backup/TestBackupFile.java | 5 +- .../priam/backup/TestBackupVerification.java | 86 +++++ .../priam/backup/TestFileIterator.java | 54 ++-- .../com/netflix/priam/backup/TestRestore.java | 69 ++-- .../priam/backupv2/TestBackupValidator.java | 14 +- .../backupv2/TestLocalDBReaderWriter.java | 288 ----------------- .../priam/backupv2/TestMetaFileManager.java | 6 +- .../priam/resources/BackupServletTest.java | 26 +- .../services/TestSnapshotMetaService.java | 3 + 38 files changed, 658 insertions(+), 1092 deletions(-) create mode 100644 priam/src/main/java/com/netflix/priam/backupv2/IMetaProxy.java delete mode 100644 priam/src/main/java/com/netflix/priam/backupv2/LocalDBReaderWriter.java delete mode 100644 priam/src/main/java/com/netflix/priam/backupv2/MetaFileManager.java create mode 100644 priam/src/main/java/com/netflix/priam/backupv2/MetaV1Proxy.java create mode 100644 priam/src/main/java/com/netflix/priam/backupv2/MetaV2Proxy.java delete mode 100644 priam/src/test/java/com/netflix/priam/aws/MockAmazonS3Client.java create mode 100644 priam/src/test/java/com/netflix/priam/backup/TestBackupVerification.java delete mode 100644 priam/src/test/java/com/netflix/priam/backupv2/TestLocalDBReaderWriter.java diff --git a/priam/src/main/java/com/netflix/priam/aws/RemoteBackupPath.java b/priam/src/main/java/com/netflix/priam/aws/RemoteBackupPath.java index 43140cc64..b1e59e3d5 100644 --- a/priam/src/main/java/com/netflix/priam/aws/RemoteBackupPath.java +++ b/priam/src/main/java/com/netflix/priam/aws/RemoteBackupPath.java @@ -22,6 +22,7 @@ import com.netflix.priam.config.IConfiguration; import com.netflix.priam.cryptography.IFileCryptography; import com.netflix.priam.identity.InstanceIdentity; +import com.netflix.priam.utils.DateUtil; import java.nio.file.Path; import java.nio.file.Paths; import java.time.Instant; @@ -128,7 +129,11 @@ private void parseV2Location(String remoteFile) { } private Path getV1Location() { - Path path = Paths.get(getV1Prefix().toString(), formatDate(time), type.toString()); + Path path = + Paths.get( + getV1Prefix().toString(), + DateUtil.formatyyyyMMddHHmm(time), + type.toString()); if (BackupFileType.isDataFile(type)) path = Paths.get(path.toString(), keyspace, columnFamily); return Paths.get(path.toString(), fileName); @@ -141,7 +146,7 @@ private void parseV1Location(Path remoteFilePath) { String.format( "Too few elements (expected: [%d]) in path: %s", 7, remoteFilePath)); - time = parseDate(remoteFilePath.getName(4).toString()); + time = DateUtil.getDate(remoteFilePath.getName(4).toString()); type = BackupFileType.valueOf(remoteFilePath.getName(5).toString()); if (BackupFileType.isDataFile(type)) { keyspace = remoteFilePath.getName(6).toString(); diff --git a/priam/src/main/java/com/netflix/priam/backup/AbstractBackupPath.java b/priam/src/main/java/com/netflix/priam/backup/AbstractBackupPath.java index ad0cf036e..c3dd2afe2 100644 --- a/priam/src/main/java/com/netflix/priam/backup/AbstractBackupPath.java +++ b/priam/src/main/java/com/netflix/priam/backup/AbstractBackupPath.java @@ -22,23 +22,19 @@ import com.netflix.priam.config.IConfiguration; import com.netflix.priam.cryptography.IFileCryptography; import com.netflix.priam.identity.InstanceIdentity; +import com.netflix.priam.utils.DateUtil; import java.io.File; import java.nio.file.Path; import java.text.ParseException; import java.time.Instant; import java.util.Date; import org.apache.commons.lang3.StringUtils; -import org.joda.time.DateTime; -import org.joda.time.format.DateTimeFormat; -import org.joda.time.format.DateTimeFormatter; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @ImplementedBy(RemoteBackupPath.class) public abstract class AbstractBackupPath implements Comparable { private static final Logger logger = LoggerFactory.getLogger(AbstractBackupPath.class); - private static final String FMT = "yyyyMMddHHmm"; - private static final DateTimeFormatter DATE_FORMAT = DateTimeFormat.forPattern(FMT); public static final char PATH_SEP = File.separatorChar; public enum BackupFileType { @@ -82,18 +78,6 @@ public AbstractBackupPath(IConfiguration config, InstanceIdentity instanceIdenti this.config = config; } - // TODO: This is so wrong as it completely depends on the timezone where application is running. - // Hopefully everyone running Priam has their clocks set to UTC. - public static String formatDate(Date d) { - return new DateTime(d).toString(FMT); - } - - // TODO: This is so wrong as it completely depends on the timezone where application is running. - // Hopefully everyone running Priam has their clocks set to UTC. - public Date parseDate(String s) { - return DATE_FORMAT.parseDateTime(s).toDate(); - } - public void parseLocal(File file, BackupFileType type) throws ParseException { this.backupFile = file; @@ -117,7 +101,7 @@ public void parseLocal(File file, BackupFileType type) throws ParseException { 2. This is to ensure that all the files from the snapshot are uploaded under single directory in remote file system. 3. For META file we always override the time field via @link{Metadata#decorateMetaJson} */ - if (type == BackupFileType.SNAP) time = parseDate(elements[3]); + if (type == BackupFileType.SNAP) time = DateUtil.getDate(elements[3]); this.lastModified = Instant.ofEpochMilli(file.lastModified()); this.fileName = file.getName(); @@ -126,8 +110,8 @@ public void parseLocal(File file, BackupFileType type) throws ParseException { /** Given a date range, find a common string prefix Eg: 20120212, 20120213 = 2012021 */ protected String match(Date start, Date end) { - String sString = formatDate(start); - String eString = formatDate(end); + String sString = DateUtil.formatyyyyMMddHHmm(start); // formatDate(start); + String eString = DateUtil.formatyyyyMMddHHmm(end); // formatDate(end); int diff = StringUtils.indexOfDifference(sString, eString); if (diff < 0) return sString; return sString.substring(0, diff); diff --git a/priam/src/main/java/com/netflix/priam/backup/BackupVerification.java b/priam/src/main/java/com/netflix/priam/backup/BackupVerification.java index 0c7dbf45d..196edd228 100644 --- a/priam/src/main/java/com/netflix/priam/backup/BackupVerification.java +++ b/priam/src/main/java/com/netflix/priam/backup/BackupVerification.java @@ -14,27 +14,24 @@ package com.netflix.priam.backup; import com.google.inject.Inject; +import com.google.inject.Provider; import com.google.inject.Singleton; import com.google.inject.name.Named; +import com.netflix.priam.backupv2.IMetaProxy; import com.netflix.priam.config.IConfiguration; import com.netflix.priam.utils.DateUtil; -import java.io.FileReader; -import java.nio.file.FileSystems; import java.nio.file.Path; -import java.nio.file.Paths; import java.util.*; import java.util.stream.Collectors; import org.apache.commons.collections4.CollectionUtils; -import org.json.simple.parser.JSONParser; +import org.apache.commons.io.FileUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** * Created by aagrawal on 2/16/17. This class validates the backup by doing listing of files in the * backup destination and comparing with meta.json by downloading from the location. Input: - * BackupMetadata that needs to be verified. Since one backupmetadata can have multiple start time, - * provide one startTime if interested in verifying one particular backup. Leave startTime as null - * to get the latest snapshot for the provided BackupMetadata. + * BackupMetadata that needs to be verified. */ @Singleton public class BackupVerification { @@ -42,14 +39,31 @@ public class BackupVerification { private static final Logger logger = LoggerFactory.getLogger(BackupVerification.class); private final IBackupFileSystem bkpStatusFs; private final IConfiguration config; + private final IMetaProxy metaProxy; + private final Provider abstractBackupPathProvider; @Inject - BackupVerification(@Named("backup") IBackupFileSystem bkpStatusFs, IConfiguration config) { + BackupVerification( + @Named("backup") IBackupFileSystem bkpStatusFs, + IConfiguration config, + @Named("v1") IMetaProxy metaProxy, + Provider abstractBackupPathProvider) { this.bkpStatusFs = bkpStatusFs; this.config = config; + this.metaProxy = metaProxy; + this.abstractBackupPathProvider = abstractBackupPathProvider; } - public BackupVerificationResult verifyBackup(List metadata, Date startTime) { + public Optional getLatestBackupMetaData(List metadata) { + metadata = + metadata.stream() + .filter(backupMetadata -> backupMetadata.getStatus() == Status.FINISHED) + .collect(Collectors.toList()); + metadata.sort((o1, o2) -> o2.getStart().compareTo(o1.getStart())); + return metadata.stream().findFirst(); + } + + public BackupVerificationResult verifyBackup(List metadata) throws Exception { BackupVerificationResult result = new BackupVerificationResult(); if (metadata == null || metadata.isEmpty()) return result; @@ -58,29 +72,14 @@ public BackupVerificationResult verifyBackup(List metadata, Date // All the dates should be same. result.selectedDate = metadata.get(0).getSnapshotDate(); - List backups = - metadata.stream() - .map( - backupMetadata -> - DateUtil.formatyyyyMMddHHmm(backupMetadata.getStart())) - .collect(Collectors.toList()); - logger.info("Snapshots found for {} : [{}]", result.selectedDate, backups); - - // find the latest date (default) or verify if one provided - Date latestDate = null; - for (BackupMetadata backupMetadata : metadata) { - if (latestDate == null || latestDate.before(backupMetadata.getStart())) - latestDate = backupMetadata.getStart(); - - if (startTime != null - && DateUtil.formatyyyyMMddHHmm(backupMetadata.getStart()) - .equals(DateUtil.formatyyyyMMddHHmm(startTime))) { - latestDate = startTime; - break; - } + Optional latestBackupMetaData = getLatestBackupMetaData(metadata); + + if (!latestBackupMetaData.isPresent()) { + logger.error("No backup found which finished during the time provided."); + return result; } - result.snapshotTime = DateUtil.formatyyyyMMddHHmm(latestDate); + result.snapshotTime = DateUtil.formatyyyyMMddHHmm(latestBackupMetaData.get().getStart()); logger.info( "Latest/Requested snapshot date found: {}, for selected/provided date: {}", result.snapshotTime, @@ -88,11 +87,10 @@ public BackupVerificationResult verifyBackup(List metadata, Date // Get Backup File Iterator String prefix = config.getBackupPrefix(); - logger.info("Looking for meta file in the location: {}", prefix); - Date strippedMsSnapshotTime = DateUtil.getDate(result.snapshotTime); Iterator backupfiles = bkpStatusFs.list(prefix, strippedMsSnapshotTime, strippedMsSnapshotTime); + // Return validation fail if backup filesystem listing failed. if (!backupfiles.hasNext()) { logger.warn( @@ -100,46 +98,31 @@ public BackupVerificationResult verifyBackup(List metadata, Date return result; } + // Do remote file listing result.backupFileListAvail = true; - List metas = new LinkedList<>(); - List s3Listing = new ArrayList<>(); + List remoteListing = new ArrayList<>(); while (backupfiles.hasNext()) { AbstractBackupPath path = backupfiles.next(); - if (path.getFileName().equalsIgnoreCase("meta.json")) metas.add(path); - else s3Listing.add(path.getRemotePath()); + if (path.getType() == AbstractBackupPath.BackupFileType.META) metas.add(path); + else remoteListing.add(path.getRemotePath()); } if (metas.size() == 0) { logger.error( - "No meta found for snapshotdate: {}", DateUtil.formatyyyyMMddHHmm(latestDate)); + "Manifest file not found on remote file system for: {}", result.snapshotTime); return result; } result.metaFileFound = true; - // Download meta.json from backup location and uncompress it. - List metaFileList = new ArrayList<>(); - try { - Path metaFileLocation = - FileSystems.getDefault().getPath(config.getDataFileLocation(), "tmp_meta.json"); - bkpStatusFs.downloadFile(Paths.get(metas.get(0).getRemotePath()), metaFileLocation, 5); - logger.info( - "Meta file successfully downloaded to localhost: {}", - metaFileLocation.toString()); - JSONParser jsonParser = new JSONParser(); - org.json.simple.JSONArray fileList = - (org.json.simple.JSONArray) - jsonParser.parse(new FileReader(metaFileLocation.toFile())); - for (Object aFileList : fileList) metaFileList.add(aFileList.toString()); + // Download meta.json from backup location. + Path localMetaPath = metaProxy.downloadMetaFile(metas.get(0)); + List metaFileList = metaProxy.getSSTFilesFromMeta(localMetaPath); + FileUtils.deleteQuietly(localMetaPath.toFile()); - } catch (Exception e) { - logger.error("Error while fetching meta.json from path: {}", metas.get(0), e); - return result; - } - - if (metaFileList.isEmpty() && s3Listing.isEmpty()) { + if (metaFileList.isEmpty() && remoteListing.isEmpty()) { logger.info( "Uncommon Scenario: Both meta file and backup filesystem listing is empty. Considering this as success"); result.valid = true; @@ -147,12 +130,13 @@ public BackupVerificationResult verifyBackup(List metadata, Date } // Atleast meta file or s3 listing contains some file. - result.filesInS3Only = new ArrayList<>(s3Listing); - result.filesInS3Only.removeAll(metaFileList); - result.filesInMetaOnly = new ArrayList<>(metaFileList); - result.filesInMetaOnly.removeAll(s3Listing); + result.filesMatched = - (ArrayList) CollectionUtils.intersection(metaFileList, s3Listing); + (ArrayList) CollectionUtils.intersection(metaFileList, remoteListing); + result.filesInS3Only = remoteListing; + result.filesInS3Only.removeAll(result.filesMatched); + result.filesInMetaOnly = metaFileList; + result.filesInMetaOnly.removeAll(result.filesMatched); // There could be a scenario that backupfilesystem has more files than meta file. e.g. some // leftover objects diff --git a/priam/src/main/java/com/netflix/priam/backup/BackupVerificationResult.java b/priam/src/main/java/com/netflix/priam/backup/BackupVerificationResult.java index 2bb0dd948..26933bf1b 100644 --- a/priam/src/main/java/com/netflix/priam/backup/BackupVerificationResult.java +++ b/priam/src/main/java/com/netflix/priam/backup/BackupVerificationResult.java @@ -13,6 +13,7 @@ */ package com.netflix.priam.backup; +import com.netflix.priam.utils.GsonJsonSerializer; import java.util.List; /** @@ -29,4 +30,9 @@ public class BackupVerificationResult { public List filesInMetaOnly = null; public List filesInS3Only = null; public List filesMatched = null; + + @Override + public String toString() { + return GsonJsonSerializer.getGson().toJson(this); + } } diff --git a/priam/src/main/java/com/netflix/priam/backup/CommitLogBackup.java b/priam/src/main/java/com/netflix/priam/backup/CommitLogBackup.java index 04fff0880..f7b1e761a 100644 --- a/priam/src/main/java/com/netflix/priam/backup/CommitLogBackup.java +++ b/priam/src/main/java/com/netflix/priam/backup/CommitLogBackup.java @@ -18,6 +18,7 @@ import com.google.inject.Provider; import com.google.inject.name.Named; import com.netflix.priam.backup.AbstractBackupPath.BackupFileType; +import com.netflix.priam.utils.DateUtil; import java.io.File; import java.nio.file.Paths; import java.util.List; @@ -64,7 +65,7 @@ public List upload(String archivedDir, final String snapshot AbstractBackupPath bp = pathFactory.get(); bp.parseLocal(file, BackupFileType.CL); - if (snapshotName != null) bp.time = bp.parseDate(snapshotName); + if (snapshotName != null) bp.time = DateUtil.getDate(snapshotName); fs.uploadFile( Paths.get(bp.getBackupFile().getAbsolutePath()), diff --git a/priam/src/main/java/com/netflix/priam/backup/IncrementalBackup.java b/priam/src/main/java/com/netflix/priam/backup/IncrementalBackup.java index 0dafbd699..7c5aa2865 100644 --- a/priam/src/main/java/com/netflix/priam/backup/IncrementalBackup.java +++ b/priam/src/main/java/com/netflix/priam/backup/IncrementalBackup.java @@ -24,6 +24,7 @@ import com.netflix.priam.config.IConfiguration; import com.netflix.priam.scheduler.SimpleTimer; import com.netflix.priam.scheduler.TaskTimer; +import com.netflix.priam.utils.DateUtil; import java.io.File; import java.util.ArrayList; import java.util.List; @@ -103,7 +104,7 @@ protected void processColumnFamily(String keyspace, String columnFamily, File ba if (!uploadedFiles.isEmpty()) { // format of yyyymmddhhmm (e.g. 201505060901) String incrementalUploadTime = - AbstractBackupPath.formatDate(uploadedFiles.get(0).getTime()); + DateUtil.formatyyyyMMddHHmm(uploadedFiles.get(0).getTime()); String metaFileName = "meta_" + columnFamily + "_" + incrementalUploadTime; logger.info("Uploading meta file for incremental backup: {}", metaFileName); this.metaData.setMetaFileName(metaFileName); diff --git a/priam/src/main/java/com/netflix/priam/backup/MetaData.java b/priam/src/main/java/com/netflix/priam/backup/MetaData.java index a5cbb8737..258088d9f 100644 --- a/priam/src/main/java/com/netflix/priam/backup/MetaData.java +++ b/priam/src/main/java/com/netflix/priam/backup/MetaData.java @@ -22,6 +22,7 @@ import com.netflix.priam.backup.AbstractBackupPath.BackupFileType; import com.netflix.priam.backup.IMessageObserver.BACKUP_MESSAGE_TYPE; import com.netflix.priam.config.IConfiguration; +import com.netflix.priam.utils.DateUtil; import java.io.File; import java.io.FileReader; import java.io.FileWriter; @@ -87,7 +88,7 @@ public AbstractBackupPath decorateMetaJson(File metafile, String snapshotName) throws ParseException { AbstractBackupPath backupfile = pathFactory.get(); backupfile.parseLocal(metafile, BackupFileType.META); - backupfile.setTime(backupfile.parseDate(snapshotName)); + backupfile.setTime(DateUtil.getDate(snapshotName)); return backupfile; } diff --git a/priam/src/main/java/com/netflix/priam/backup/SnapshotBackup.java b/priam/src/main/java/com/netflix/priam/backup/SnapshotBackup.java index 3afc226b6..0cb046b47 100644 --- a/priam/src/main/java/com/netflix/priam/backup/SnapshotBackup.java +++ b/priam/src/main/java/com/netflix/priam/backup/SnapshotBackup.java @@ -28,6 +28,7 @@ import com.netflix.priam.scheduler.CronTimer; import com.netflix.priam.scheduler.TaskTimer; import com.netflix.priam.utils.CassandraMonitor; +import com.netflix.priam.utils.DateUtil; import com.netflix.priam.utils.ThreadSleeper; import java.io.File; import java.util.*; @@ -100,7 +101,7 @@ public void execute() throws Exception { private void executeSnapshot() throws Exception { Date startTime = Calendar.getInstance(TimeZone.getTimeZone("GMT")).getTime(); - snapshotName = pathFactory.get().formatDate(startTime); + snapshotName = DateUtil.formatyyyyMMddHHmm(startTime); String token = instanceIdentity.getInstance().getToken(); // Save start snapshot status diff --git a/priam/src/main/java/com/netflix/priam/backupv2/BackupValidator.java b/priam/src/main/java/com/netflix/priam/backupv2/BackupValidator.java index 583bbad41..1653ff571 100644 --- a/priam/src/main/java/com/netflix/priam/backupv2/BackupValidator.java +++ b/priam/src/main/java/com/netflix/priam/backupv2/BackupValidator.java @@ -17,7 +17,6 @@ package com.netflix.priam.backupv2; -import com.google.inject.Provider; import com.netflix.priam.backup.*; import com.netflix.priam.config.IConfiguration; import com.netflix.priam.utils.DateUtil; @@ -25,13 +24,10 @@ import java.io.IOException; import java.nio.file.Path; import java.nio.file.Paths; -import java.util.ArrayList; -import java.util.Collections; -import java.util.Iterator; import java.util.List; import javax.inject.Inject; +import javax.inject.Named; import org.apache.commons.io.FileUtils; -import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -42,56 +38,16 @@ public class BackupValidator { private static final Logger logger = LoggerFactory.getLogger(BackupVerification.class); private final IBackupFileSystem fs; - private final Provider abstractBackupPathProvider; + private IMetaProxy metaProxy; private boolean isBackupValid; @Inject public BackupValidator( IConfiguration configuration, IFileSystemContext backupFileSystemCtx, - Provider abstractBackupPathProvider) { + @Named("v2") IMetaProxy metaProxy) { fs = backupFileSystemCtx.getFileStrategy(configuration); - this.abstractBackupPathProvider = abstractBackupPathProvider; - } - - /** - * Fetch the list of all META_V2 files on the remote file system for the provided valid - * daterange. - * - * @param dateRange the time period to scan in the remote file system for meta files. - * @return List of all the META_V2 files from the remote file system. - */ - public List findMetaFiles(DateUtil.DateRange dateRange) { - ArrayList metas = new ArrayList<>(); - String prefix = getMetaPrefix(dateRange); - String marker = getMetaPrefix(new DateUtil.DateRange(dateRange.getStartTime(), null)); - logger.info( - "Listing filesystem with prefix: {}, marker: {}, daterange: {}", - prefix, - marker, - dateRange); - Iterator iterator = fs.listFileSystem(prefix, null, marker); - - while (iterator.hasNext()) { - AbstractBackupPath abstractBackupPath = abstractBackupPathProvider.get(); - abstractBackupPath.parseRemote(iterator.next()); - logger.debug("Meta file found: {}", abstractBackupPath); - if (abstractBackupPath.getLastModified().toEpochMilli() - >= dateRange.getStartTime().toEpochMilli() - && abstractBackupPath.getLastModified().toEpochMilli() - <= dateRange.getEndTime().toEpochMilli()) { - metas.add(abstractBackupPath); - } - } - - Collections.sort(metas, Collections.reverseOrder()); - - if (metas.size() == 0) { - logger.info( - "No meta file found on remote file system for the time period: {}", dateRange); - } - - return metas; + this.metaProxy = metaProxy; } /** @@ -106,11 +62,11 @@ public List findMetaFiles(DateUtil.DateRange dateRange) { */ public AbstractBackupPath findLatestValidMetaFile(DateUtil.DateRange dateRange) throws BackupRestoreException { - List metas = findMetaFiles(dateRange); + List metas = metaProxy.findMetaFiles(dateRange); logger.info("Meta files found: {}", metas); for (AbstractBackupPath meta : metas) { - Path localFile = downloadMetaFile(meta); + Path localFile = metaProxy.downloadMetaFile(meta); boolean isValid = isMetaFileValid(localFile); logger.info("Meta: {}, isValid: {}", meta, isValid); if (!isValid) FileUtils.deleteQuietly(localFile.toFile()); @@ -120,39 +76,6 @@ public AbstractBackupPath findLatestValidMetaFile(DateUtil.DateRange dateRange) return null; } - /** - * Download the meta file to disk. - * - * @param meta AbstractBackupPath denoting the meta file on remote file system. - * @return the location of the meta file on disk after downloading from remote file system. - * @throws BackupRestoreException if unable to download for any reason. - */ - public Path downloadMetaFile(AbstractBackupPath meta) throws BackupRestoreException { - Path localFile = Paths.get(meta.newRestoreFile().getAbsolutePath()); - fs.downloadFile(Paths.get(meta.getRemotePath()), localFile, 10); - return localFile; - } - - /** - * Get the prefix for the META_V2 file. This will depend on the configuration, if restore prefix - * is set. - * - * @param dateRange date range for which we are trying to find META_V2 files. - * @return prefix for the META_V2 files. - */ - public String getMetaPrefix(DateUtil.DateRange dateRange) { - Path location = fs.getPrefix(); - AbstractBackupPath abstractBackupPath = abstractBackupPathProvider.get(); - String match = StringUtils.EMPTY; - if (dateRange != null) match = dateRange.match(); - return Paths.get( - abstractBackupPath - .remoteV2Prefix(location, AbstractBackupPath.BackupFileType.META_V2) - .toString(), - match) - .toString(); - } - /** * Validate that all the files mentioned in the meta file actually exists on remote file system. * diff --git a/priam/src/main/java/com/netflix/priam/backupv2/IMetaProxy.java b/priam/src/main/java/com/netflix/priam/backupv2/IMetaProxy.java new file mode 100644 index 000000000..b9a1ad255 --- /dev/null +++ b/priam/src/main/java/com/netflix/priam/backupv2/IMetaProxy.java @@ -0,0 +1,75 @@ +/* + * Copyright 2018 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.netflix.priam.backupv2; + +import com.netflix.priam.backup.AbstractBackupPath; +import com.netflix.priam.backup.BackupRestoreException; +import com.netflix.priam.utils.DateUtil; +import java.nio.file.Path; +import java.util.List; + +/** Proxy to do management tasks for meta files. Created by aagrawal on 12/18/18. */ +public interface IMetaProxy { + + /** + * Path on the local file system where meta file should be stored for processing. + * + * @return location on local file system. + */ + Path getLocalMetaFileDirectory(); + + /** + * Get the prefix for the manifest file. This will depend on the configuration, if restore + * prefix is set. + * + * @param dateRange date range for which we are trying to find manifest files. + * @return prefix for the manifest files. + */ + String getMetaPrefix(DateUtil.DateRange dateRange); + + /** + * Fetch the list of all manifest files on the remote file system for the provided valid + * daterange. + * + * @param dateRange the time period to scan in the remote file system for meta files. + * @return List of all the manifest files from the remote file system. + */ + List findMetaFiles(DateUtil.DateRange dateRange); + + /** + * Download the meta file to disk. + * + * @param meta AbstractBackupPath denoting the meta file on remote file system. + * @return the location of the meta file on disk after downloading from remote file system. + * @throws BackupRestoreException if unable to download for any reason. + */ + Path downloadMetaFile(AbstractBackupPath meta) throws BackupRestoreException; + + /** + * Read the manifest file and give the contents of the file (all the sstable components) as + * list. + * + * @param localMetaPath location of the manifest file on disk. + * @return list containing all the remote locations of sstable components. + * @throws Exception if file is not found on local system or is corrupt. + */ + List getSSTFilesFromMeta(Path localMetaPath) throws Exception; + + /** Delete the old meta files, if any present in the metaFileDirectory */ + void cleanupOldMetaFiles(); +} diff --git a/priam/src/main/java/com/netflix/priam/backupv2/LocalDBReaderWriter.java b/priam/src/main/java/com/netflix/priam/backupv2/LocalDBReaderWriter.java deleted file mode 100644 index 154c25af1..000000000 --- a/priam/src/main/java/com/netflix/priam/backupv2/LocalDBReaderWriter.java +++ /dev/null @@ -1,302 +0,0 @@ -/* - * Copyright 2018 Netflix, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.netflix.priam.backupv2; - -import com.google.inject.Inject; -import com.netflix.priam.config.IConfiguration; -import com.netflix.priam.utils.GsonJsonSerializer; -import java.io.File; -import java.io.FileReader; -import java.io.FileWriter; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.time.Instant; -import java.util.ArrayList; -import java.util.List; -import java.util.stream.Collectors; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * This class is used to create a local DB entry for SSTable components. This will be used to - * identify when a version of a SSTable component was last uploaded or last referenced. This db - * entry will be used to enforce the TTL of a version of a SSTable component as we will not be - * relying on backup file system level TTL. Local DB should be copied over to new instance replacing - * this token in case of instance replacement. If no local DB is found then Priam will try to - * re-create the local DB using meta files uploaded to backup file system. The check operation for - * local DB is done at every start of Priam and when operator requests to re-build the local DB. - * Created by aagrawal on 8/31/18. - */ -public class LocalDBReaderWriter { - private static final Logger logger = LoggerFactory.getLogger(LocalDBReaderWriter.class); - private final IConfiguration configuration; - public static final String LOCAL_DB = "localdb"; - - @Inject - public LocalDBReaderWriter(IConfiguration configuration) { - this.configuration = configuration; - } - - public synchronized LocalDB upsertLocalDBEntry(final LocalDBEntry localDBEntry) - throws Exception { - // validate the localDBEntry first - if (localDBEntry.getTimeLastReferenced() == null) - throw new Exception("Time last referenced in localDB can never be null"); - - if (localDBEntry.getBackupTime() == null) - throw new Exception("Backup time for localDB can never be null"); - - final Path localDBFile = getLocalDBPath(localDBEntry.getFileUploadResult()); - - localDBFile.getParent().toFile().mkdirs(); - - LocalDB localDB = readAndGetLocalDB(localDBEntry.getFileUploadResult()); - - if (localDB == null) localDB = new LocalDB(new ArrayList<>()); - - // Verify again if someone beat you to write the entry. - LocalDBEntry entry = getLocalDBEntry(localDBEntry.getFileUploadResult(), localDB); - // Write entry as it might be either - - // 1. new component entry - // 2. new version of file (change in compression type or file is modified e.g. stats file) - if (entry == null) { - localDB.getLocalDBEntries().add(localDBEntry); - writeLocalDB(localDBFile, localDB); - } else { - // An entry already exists. Maybe last referenced time or backup time changed. We want - // to write the last time referenced. - entry.setBackupTime(localDBEntry.getBackupTime()); - entry.setTimeLastReferenced(localDBEntry.getTimeLastReferenced()); - writeLocalDB(localDBFile, localDB); - } - - return localDB; - } - - private LocalDB readAndGetLocalDB(final FileUploadResult fileUploadResult) throws Exception { - final Path localDbPath = getLocalDBPath(fileUploadResult); - return readLocalDB(localDbPath); - } - - /** - * Get the local database entry for a given file upload result. - * - * @param fileUploadResult File upload result for which local db is required. - * @return LocalDBEntry if one exists. - * @throws Exception if there is any error in getting local db file. - */ - public LocalDBEntry getLocalDBEntry(final FileUploadResult fileUploadResult) throws Exception { - LocalDB localDB = readAndGetLocalDB(fileUploadResult); - return getLocalDBEntry(fileUploadResult, localDB); - } - - private LocalDBEntry getLocalDBEntry( - final FileUploadResult fileUploadResult, final LocalDB localDB) throws Exception { - if (localDB == null - || localDB.getLocalDBEntries() == null - || localDB.getLocalDBEntries().isEmpty()) return null; - - // Get local db entry for same file and version. - List localDBEntries = - localDB.getLocalDBEntries() - .stream() - .filter( - localDBEntry -> - // Name of the file should be same. - (localDBEntry - .getFileUploadResult() - .getFileName() - .toFile() - .getName() - .toLowerCase() - .equals( - fileUploadResult - .getFileName() - .toFile() - .getName() - .toLowerCase()))) - // Should be same version (same last modified time) - .filter( - localDBEntry -> - (localDBEntry - .getFileUploadResult() - .getLastModifiedTime() - .equals(fileUploadResult.getLastModifiedTime()))) - // Same compression as before. If we switch compression technique we can - // upload again. - .filter( - localDBEntry -> - (localDBEntry - .getFileUploadResult() - .getCompression() - .equals(fileUploadResult.getCompression()))) - .collect(Collectors.toList()); - - if (localDBEntries.isEmpty()) return null; - - if (localDBEntries.size() == 1) { - if (logger.isDebugEnabled()) - logger.debug("Local entry found: {}", localDBEntries.get(0)); - - return localDBEntries.get(0); - } - - throw new Exception( - "Unexpected behavior: More than one entry found in local database for the same file. FileUploadResult: " - + fileUploadResult); - } - - /** - * Gets the local db path on the local file system. - * - * @param fileUploadResult This contains the SSTable component for which local db path is - * required. - * @return the local db path on local file system. - */ - public Path getLocalDBPath(final FileUploadResult fileUploadResult) { - return Paths.get( - configuration.getDataFileLocation(), - LOCAL_DB, - fileUploadResult.getKeyspaceName(), - fileUploadResult.getColumnFamilyName(), - PrefixGenerator.getSSTFileBase(fileUploadResult.getFileName().toFile().getName()) - + ".localdb"); - } - - /** - * Writes the local database to the local db file. This will do a complete replace of existing - * local database if any. - * - * @param localDBFile path to the local database file. - * @param localDB local database containing all the entries to the local database. - * @throws Exception If the path denoted is directory, write permission issues or any other - * exceptions. - */ - public void writeLocalDB(final Path localDBFile, final LocalDB localDB) throws Exception { - if (localDB == null || localDBFile == null || localDBFile.toFile().isDirectory()) - throw new Exception( - "Invalid Arguments: localDbFile: " + localDBFile + ", localDB: " + localDB); - - if (!localDBFile.getParent().toFile().exists()) localDBFile.getParent().toFile().mkdirs(); - - File tmpFile = - File.createTempFile( - localDBFile.toFile().getName(), ".tmp", localDBFile.getParent().toFile()); - try (FileWriter writer = new FileWriter(tmpFile)) { - writer.write(localDB.toString()); - - // Atomically swap out the new local db file for the old local db file. - if (!tmpFile.renameTo(localDBFile.toFile())) - logger.error("Failed to persist local db: {}", localDB); - } finally { - if (tmpFile != null) Files.deleteIfExists(tmpFile.toPath()); - } - } - - /** - * Reads the local database file stored on local file system. - * - * @param localDBFile path the local database. - * @return local database if file exists or empty local database. - * @throws Exception If there is any error in de-serializing the object or any other file system - * errors. - */ - public LocalDB readLocalDB(final Path localDBFile) throws Exception { - // Verify file exists - if (!localDBFile.toFile().exists()) return new LocalDB(new ArrayList<>()); - - try (FileReader reader = new FileReader(localDBFile.toFile())) { - LocalDB localDB = GsonJsonSerializer.getGson().fromJson(reader, LocalDB.class); - String columnfamilyName = localDBFile.getParent().toFile().getName(); - String keyspaceName = localDBFile.getParent().getParent().toFile().getName(); - localDB.getLocalDBEntries() - .forEach( - localDBEntry -> { - localDBEntry - .getFileUploadResult() - .setColumnFamilyName(columnfamilyName); - localDBEntry.getFileUploadResult().setKeyspaceName(keyspaceName); - }); - - if (logger.isDebugEnabled()) logger.debug("Local DB: {}", localDB); - return localDB; - } - } - - static class LocalDB { - private final List localDBEntries; - - public LocalDB(List localDBEntries) { - this.localDBEntries = localDBEntries; - } - - public List getLocalDBEntries() { - - return localDBEntries; - } - - @Override - public String toString() { - return GsonJsonSerializer.getGson().toJson(this); - } - } - - static class LocalDBEntry { - private final FileUploadResult fileUploadResult; - private Instant timeLastReferenced; - private Instant backupTime; - - public LocalDBEntry(FileUploadResult fileUploadResult) { - this.fileUploadResult = fileUploadResult; - } - - public FileUploadResult getFileUploadResult() { - return fileUploadResult; - } - - public Instant getTimeLastReferenced() { - return timeLastReferenced; - } - - public void setTimeLastReferenced(Instant timeLastReferenced) { - this.timeLastReferenced = timeLastReferenced; - } - - public Instant getBackupTime() { - return backupTime; - } - - public void setBackupTime(Instant backupTime) { - this.backupTime = backupTime; - } - - public LocalDBEntry( - FileUploadResult fileUploadResult, Instant timeLastReferenced, Instant backupTime) { - - this.fileUploadResult = fileUploadResult; - this.timeLastReferenced = timeLastReferenced; - this.backupTime = backupTime; - } - - @Override - public String toString() { - return GsonJsonSerializer.getGson().toJson(this); - } - } -} diff --git a/priam/src/main/java/com/netflix/priam/backupv2/MetaFileManager.java b/priam/src/main/java/com/netflix/priam/backupv2/MetaFileManager.java deleted file mode 100644 index 415dd5e97..000000000 --- a/priam/src/main/java/com/netflix/priam/backupv2/MetaFileManager.java +++ /dev/null @@ -1,67 +0,0 @@ -/* - * Copyright 2018 Netflix, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.netflix.priam.backupv2; - -import com.netflix.priam.config.IConfiguration; -import java.io.File; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.Collection; -import javax.inject.Inject; -import org.apache.commons.io.FileUtils; -import org.apache.commons.io.filefilter.FileFilterUtils; -import org.apache.commons.io.filefilter.IOFileFilter; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** Do any management task for meta files. Created by aagrawal on 8/2/18. */ -public class MetaFileManager { - private static final Logger logger = LoggerFactory.getLogger(MetaFileManager.class); - private final Path metaFileDirectory; - - @Inject - MetaFileManager(IConfiguration configuration) { - metaFileDirectory = Paths.get(configuration.getDataFileLocation()); - } - - public Path getMetaFileDirectory() { - return metaFileDirectory; - } - - /** Delete the old meta files, if any present in the metaFileDirectory */ - public void cleanupOldMetaFiles() { - logger.info("Deleting any old META_V2 files if any"); - IOFileFilter fileNameFilter = - FileFilterUtils.and( - FileFilterUtils.prefixFileFilter(MetaFileInfo.META_FILE_PREFIX), - FileFilterUtils.or( - FileFilterUtils.suffixFileFilter(MetaFileInfo.META_FILE_SUFFIX), - FileFilterUtils.suffixFileFilter( - MetaFileInfo.META_FILE_SUFFIX + ".tmp"))); - Collection files = - FileUtils.listFiles(metaFileDirectory.toFile(), fileNameFilter, null); - files.stream() - .filter(File::isFile) - .forEach( - file -> { - logger.debug( - "Deleting old META_V2 file found: {}", file.getAbsolutePath()); - file.delete(); - }); - } -} diff --git a/priam/src/main/java/com/netflix/priam/backupv2/MetaFileWriterBuilder.java b/priam/src/main/java/com/netflix/priam/backupv2/MetaFileWriterBuilder.java index 750f00029..4e0c259c9 100644 --- a/priam/src/main/java/com/netflix/priam/backupv2/MetaFileWriterBuilder.java +++ b/priam/src/main/java/com/netflix/priam/backupv2/MetaFileWriterBuilder.java @@ -31,6 +31,7 @@ import java.util.ArrayList; import java.util.List; import javax.inject.Inject; +import javax.inject.Named; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -76,8 +77,9 @@ public static class MetaFileWriter implements StartStep, DataStep, UploadStep { private final IBackupFileSystem backupFileSystem; private final MetaFileInfo metaFileInfo; - private final MetaFileManager metaFileManager; + private final IMetaProxy metaProxy; private JsonWriter jsonWriter; + private Instant snapshotInstant; private Path metaFilePath; @Inject @@ -86,10 +88,10 @@ private MetaFileWriter( InstanceIdentity instanceIdentity, Provider pathFactory, IFileSystemContext backupFileSystemCtx, - MetaFileManager metaFileManager) { + @Named("v2") IMetaProxy metaProxy) { this.pathFactory = pathFactory; this.backupFileSystem = backupFileSystemCtx.getFileStrategy(configuration); - this.metaFileManager = metaFileManager; + this.metaProxy = metaProxy; List backupIdentifier = new ArrayList<>(); backupIdentifier.add(instanceIdentity.getInstance().getToken()); metaFileInfo = @@ -107,10 +109,11 @@ private MetaFileWriter( */ public DataStep startMetaFileGeneration(Instant snapshotInstant) throws IOException { // Compute meta file name. + this.snapshotInstant = snapshotInstant; String fileName = MetaFileInfo.getMetaFileName(snapshotInstant); - metaFilePath = Paths.get(metaFileManager.getMetaFileDirectory().toString(), fileName); + metaFilePath = Paths.get(metaProxy.getLocalMetaFileDirectory().toString(), fileName); Path tempMetaFilePath = - Paths.get(metaFileManager.getMetaFileDirectory().toString(), fileName + ".tmp"); + Paths.get(metaProxy.getLocalMetaFileDirectory().toString(), fileName + ".tmp"); logger.info("Starting to write a new meta file: {}", metaFilePath); @@ -159,11 +162,16 @@ public MetaFileWriterBuilder.UploadStep endMetaFileGeneration() throws IOExcepti Path tempMetaFilePath = Paths.get( - metaFileManager.getMetaFileDirectory().toString(), + metaProxy.getLocalMetaFileDirectory().toString(), metaFilePath.toFile().getName() + ".tmp"); // Rename the tmp file. tempMetaFilePath.toFile().renameTo(metaFilePath.toFile()); + + // Set the last modified time to snapshot time as generating manifest file may take some + // time. + metaFilePath.toFile().setLastModified(snapshotInstant.toEpochMilli()); + logger.info("Finished writing to meta file: {}", metaFilePath); return this; diff --git a/priam/src/main/java/com/netflix/priam/backupv2/MetaV1Proxy.java b/priam/src/main/java/com/netflix/priam/backupv2/MetaV1Proxy.java new file mode 100644 index 000000000..38827a235 --- /dev/null +++ b/priam/src/main/java/com/netflix/priam/backupv2/MetaV1Proxy.java @@ -0,0 +1,112 @@ +/* + * Copyright 2018 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.netflix.priam.backupv2; + +import com.google.common.collect.Lists; +import com.google.inject.Inject; +import com.netflix.priam.backup.AbstractBackupPath; +import com.netflix.priam.backup.BackupRestoreException; +import com.netflix.priam.backup.IBackupFileSystem; +import com.netflix.priam.backup.IFileSystemContext; +import com.netflix.priam.config.IConfiguration; +import com.netflix.priam.utils.DateUtil; +import java.io.FileReader; +import java.nio.file.InvalidPathException; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.*; +import org.json.simple.parser.JSONParser; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** Created by aagrawal on 12/18/18. */ +public class MetaV1Proxy implements IMetaProxy { + private static final Logger logger = LoggerFactory.getLogger(MetaV1Proxy.class); + private final IBackupFileSystem fs; + + @Inject + MetaV1Proxy(IConfiguration configuration, IFileSystemContext backupFileSystemCtx) { + fs = backupFileSystemCtx.getFileStrategy(configuration); + } + + @Override + public Path getLocalMetaFileDirectory() { + return null; + } + + @Override + public String getMetaPrefix(DateUtil.DateRange dateRange) { + return null; + } + + @Override + public List findMetaFiles(DateUtil.DateRange dateRange) { + Date startTime = new Date(dateRange.getStartTime().toEpochMilli()); + Date endTime = new Date(dateRange.getEndTime().toEpochMilli()); + String restorePrefix = fs.getPrefix().toString(); + logger.debug("Looking for snapshot meta file within restore prefix: {}", restorePrefix); + List metas = Lists.newArrayList(); + + Iterator backupfiles = fs.list(restorePrefix, startTime, endTime); + + while (backupfiles.hasNext()) { + AbstractBackupPath path = backupfiles.next(); + if (path.getType() == AbstractBackupPath.BackupFileType.META) + // Since there are now meta file for incrementals as well as snapshot, we need to + // find the correct one (i.e. the snapshot meta file (meta.json)) + if (path.getFileName().equalsIgnoreCase("meta.json")) { + metas.add(path); + } + } + + Collections.sort(metas, Collections.reverseOrder()); + + if (metas.size() == 0) { + logger.info( + "No meta v1 file found on remote file system for the time period: {}", + dateRange); + } + + return metas; + } + + @Override + public Path downloadMetaFile(AbstractBackupPath meta) throws BackupRestoreException { + Path localFile = Paths.get(meta.newRestoreFile().getAbsolutePath() + ".download"); + fs.downloadFile(Paths.get(meta.getRemotePath()), localFile, 10); + return localFile; + } + + @Override + public List getSSTFilesFromMeta(Path localMetaPath) throws Exception { + if (localMetaPath.toFile().isDirectory() || !localMetaPath.toFile().exists()) + throw new InvalidPathException( + localMetaPath.toString(), "Input path is either directory or do not exist"); + + List result = new ArrayList<>(); + JSONParser jsonParser = new JSONParser(); + org.json.simple.JSONArray fileList = + (org.json.simple.JSONArray) + jsonParser.parse(new FileReader(localMetaPath.toFile())); + fileList.forEach(entry -> result.add(entry.toString())); + return result; + } + + @Override + public void cleanupOldMetaFiles() {} +} diff --git a/priam/src/main/java/com/netflix/priam/backupv2/MetaV2Proxy.java b/priam/src/main/java/com/netflix/priam/backupv2/MetaV2Proxy.java new file mode 100644 index 000000000..7ad048c39 --- /dev/null +++ b/priam/src/main/java/com/netflix/priam/backupv2/MetaV2Proxy.java @@ -0,0 +1,142 @@ +/* + * Copyright 2018 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.netflix.priam.backupv2; + +import com.google.inject.Provider; +import com.netflix.priam.backup.AbstractBackupPath; +import com.netflix.priam.backup.BackupRestoreException; +import com.netflix.priam.backup.IBackupFileSystem; +import com.netflix.priam.backup.IFileSystemContext; +import com.netflix.priam.config.IConfiguration; +import com.netflix.priam.utils.DateUtil; +import java.io.File; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.*; +import javax.inject.Inject; +import org.apache.commons.io.FileUtils; +import org.apache.commons.io.filefilter.FileFilterUtils; +import org.apache.commons.io.filefilter.IOFileFilter; +import org.apache.commons.lang3.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** Do any management task for meta files. Created by aagrawal on 8/2/18. */ +public class MetaV2Proxy implements IMetaProxy { + private static final Logger logger = LoggerFactory.getLogger(MetaV2Proxy.class); + private final Path metaFileDirectory; + private final IBackupFileSystem fs; + private final Provider abstractBackupPathProvider; + + @Inject + MetaV2Proxy( + IConfiguration configuration, + IFileSystemContext backupFileSystemCtx, + Provider abstractBackupPathProvider) { + fs = backupFileSystemCtx.getFileStrategy(configuration); + this.abstractBackupPathProvider = abstractBackupPathProvider; + metaFileDirectory = Paths.get(configuration.getDataFileLocation()); + } + + @Override + public Path getLocalMetaFileDirectory() { + return metaFileDirectory; + } + + @Override + public String getMetaPrefix(DateUtil.DateRange dateRange) { + Path location = fs.getPrefix(); + AbstractBackupPath abstractBackupPath = abstractBackupPathProvider.get(); + String match = StringUtils.EMPTY; + if (dateRange != null) match = dateRange.match(); + return Paths.get( + abstractBackupPath + .remoteV2Prefix(location, AbstractBackupPath.BackupFileType.META_V2) + .toString(), + match) + .toString(); + } + + @Override + public List findMetaFiles(DateUtil.DateRange dateRange) { + ArrayList metas = new ArrayList<>(); + String prefix = getMetaPrefix(dateRange); + String marker = getMetaPrefix(new DateUtil.DateRange(dateRange.getStartTime(), null)); + logger.info( + "Listing filesystem with prefix: {}, marker: {}, daterange: {}", + prefix, + marker, + dateRange); + Iterator iterator = fs.listFileSystem(prefix, null, marker); + + while (iterator.hasNext()) { + AbstractBackupPath abstractBackupPath = abstractBackupPathProvider.get(); + abstractBackupPath.parseRemote(iterator.next()); + logger.debug("Meta file found: {}", abstractBackupPath); + if (abstractBackupPath.getLastModified().toEpochMilli() + >= dateRange.getStartTime().toEpochMilli() + && abstractBackupPath.getLastModified().toEpochMilli() + <= dateRange.getEndTime().toEpochMilli()) { + metas.add(abstractBackupPath); + } + } + + Collections.sort(metas, Collections.reverseOrder()); + + if (metas.size() == 0) { + logger.info( + "No meta file found on remote file system for the time period: {}", dateRange); + } + + return metas; + } + + @Override + public Path downloadMetaFile(AbstractBackupPath meta) throws BackupRestoreException { + Path localFile = Paths.get(meta.newRestoreFile().getAbsolutePath()); + fs.downloadFile(Paths.get(meta.getRemotePath()), localFile, 10); + return localFile; + } + + @Override + public void cleanupOldMetaFiles() { + logger.info("Deleting any old META_V2 files if any"); + IOFileFilter fileNameFilter = + FileFilterUtils.and( + FileFilterUtils.prefixFileFilter(MetaFileInfo.META_FILE_PREFIX), + FileFilterUtils.or( + FileFilterUtils.suffixFileFilter(MetaFileInfo.META_FILE_SUFFIX), + FileFilterUtils.suffixFileFilter( + MetaFileInfo.META_FILE_SUFFIX + ".tmp"))); + Collection files = + FileUtils.listFiles(metaFileDirectory.toFile(), fileNameFilter, null); + files.stream() + .filter(File::isFile) + .forEach( + file -> { + logger.debug( + "Deleting old META_V2 file found: {}", file.getAbsolutePath()); + file.delete(); + }); + } + + @Override + public List getSSTFilesFromMeta(Path localMetaPath) throws Exception { + return null; + } +} diff --git a/priam/src/main/java/com/netflix/priam/cli/Restorer.java b/priam/src/main/java/com/netflix/priam/cli/Restorer.java index 77cb6113e..742fb9166 100644 --- a/priam/src/main/java/com/netflix/priam/cli/Restorer.java +++ b/priam/src/main/java/com/netflix/priam/cli/Restorer.java @@ -16,9 +16,9 @@ */ package com.netflix.priam.cli; -import com.netflix.priam.backup.AbstractBackupPath; import com.netflix.priam.restore.Restore; -import java.util.Date; +import com.netflix.priam.utils.DateUtil; +import java.time.Instant; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -32,20 +32,17 @@ static void displayHelp() { public static void main(String[] args) { try { Application.initialize(); - - Date startTime, endTime; + Instant startTime, endTime; if (args.length < 2) { displayHelp(); return; } - AbstractBackupPath path = - Application.getInjector().getInstance(AbstractBackupPath.class); - startTime = path.parseDate(args[0]); - endTime = path.parseDate(args[1]); + startTime = DateUtil.parseInstant(args[0]); + endTime = DateUtil.parseInstant(args[1]); Restore restorer = Application.getInjector().getInstance(Restore.class); try { - restorer.restore(startTime, endTime); + restorer.restore(new DateUtil.DateRange(startTime, endTime)); } catch (Exception e) { logger.error("Unable to restore: ", e); } diff --git a/priam/src/main/java/com/netflix/priam/defaultimpl/PriamGuiceModule.java b/priam/src/main/java/com/netflix/priam/defaultimpl/PriamGuiceModule.java index a351ed42c..0f68962ac 100644 --- a/priam/src/main/java/com/netflix/priam/defaultimpl/PriamGuiceModule.java +++ b/priam/src/main/java/com/netflix/priam/defaultimpl/PriamGuiceModule.java @@ -25,6 +25,9 @@ import com.netflix.priam.aws.auth.IS3Credential; import com.netflix.priam.aws.auth.S3RoleAssumptionCredential; import com.netflix.priam.backup.IBackupFileSystem; +import com.netflix.priam.backupv2.IMetaProxy; +import com.netflix.priam.backupv2.MetaV1Proxy; +import com.netflix.priam.backupv2.MetaV2Proxy; import com.netflix.priam.cred.ICredential; import com.netflix.priam.cred.ICredentialGeneric; import com.netflix.priam.cryptography.IFileCryptography; @@ -67,6 +70,8 @@ protected void configure() { bind(ICredentialGeneric.class) .annotatedWith(Names.named("pgpcredential")) .to(PgpCredential.class); + bind(IMetaProxy.class).annotatedWith(Names.named("v1")).to(MetaV1Proxy.class); + bind(IMetaProxy.class).annotatedWith(Names.named("v2")).to(MetaV2Proxy.class); bind(Registry.class).toInstance(new NoopRegistry()); } } diff --git a/priam/src/main/java/com/netflix/priam/resources/BackupServlet.java b/priam/src/main/java/com/netflix/priam/resources/BackupServlet.java index 5d14c0eec..68aa2f9b5 100644 --- a/priam/src/main/java/com/netflix/priam/resources/BackupServlet.java +++ b/priam/src/main/java/com/netflix/priam/resources/BackupServlet.java @@ -244,32 +244,14 @@ public Response validateSnapshotByDate(@PathParam("daterange") String daterange) throws Exception { DateUtil.DateRange dateRange = new DateUtil.DateRange(daterange); - - JSONObject jsonReply = new JSONObject(); - jsonReply.put( - "inputStartDate", - DateUtil.formatInstant(DateUtil.yyyyMMddHHmm, dateRange.getStartTime())); - jsonReply.put( - "inputEndDate", - DateUtil.formatInstant(DateUtil.yyyyMMddHHmm, dateRange.getEndTime())); logger.info( "Will try to validate latest backup during startTime: {}, and endTime: {}", dateRange.getStartTime(), dateRange.getEndTime()); List metadata = getLatestBackupMetadata(dateRange); - BackupVerificationResult result = - backupVerification.verifyBackup(metadata, Date.from(dateRange.getStartTime())); - jsonReply.put("snapshotAvailable", result.snapshotAvailable); - jsonReply.put("valid", result.valid); - jsonReply.put("backupFileListAvailable", result.backupFileListAvail); - jsonReply.put("metaFileFound", result.metaFileFound); - jsonReply.put("selectedDate", result.selectedDate); - jsonReply.put("snapshotTime", result.snapshotTime); - jsonReply.put("filesInMetaOnly", result.filesInMetaOnly); - jsonReply.put("filesInS3Only", result.filesInS3Only); - jsonReply.put("filesMatched", result.filesMatched); - return Response.ok(jsonReply.toString()).build(); + BackupVerificationResult result = backupVerification.verifyBackup(metadata); + return Response.ok(result.toString()).build(); } @GET diff --git a/priam/src/main/java/com/netflix/priam/resources/RestoreServlet.java b/priam/src/main/java/com/netflix/priam/resources/RestoreServlet.java index 6b987798c..9451eb33e 100644 --- a/priam/src/main/java/com/netflix/priam/resources/RestoreServlet.java +++ b/priam/src/main/java/com/netflix/priam/resources/RestoreServlet.java @@ -17,7 +17,6 @@ import com.netflix.priam.health.InstanceState; import com.netflix.priam.restore.Restore; import com.netflix.priam.utils.DateUtil; -import java.util.Date; import javax.ws.rs.GET; import javax.ws.rs.Path; import javax.ws.rs.Produces; @@ -66,7 +65,7 @@ public Response restore(@QueryParam("daterange") String daterange) throws Except "Parameters: {startTime: [{}], endTime: [{}]}", dateRange.getStartTime().toString(), dateRange.getEndTime().toString()); - restoreObj.restore(Date.from(dateRange.getStartTime()), Date.from(dateRange.getEndTime())); + restoreObj.restore(dateRange); return Response.ok("[\"ok\"]", MediaType.APPLICATION_JSON).build(); } } diff --git a/priam/src/main/java/com/netflix/priam/restore/AbstractRestore.java b/priam/src/main/java/com/netflix/priam/restore/AbstractRestore.java index 47225d070..3850d7432 100644 --- a/priam/src/main/java/com/netflix/priam/restore/AbstractRestore.java +++ b/priam/src/main/java/com/netflix/priam/restore/AbstractRestore.java @@ -16,11 +16,11 @@ */ package com.netflix.priam.restore; -import com.google.common.collect.Iterators; -import com.google.common.collect.Lists; +import com.google.inject.Inject; import com.google.inject.Provider; import com.netflix.priam.backup.*; import com.netflix.priam.backup.AbstractBackupPath.BackupFileType; +import com.netflix.priam.backupv2.IMetaProxy; import com.netflix.priam.config.IConfiguration; import com.netflix.priam.defaultimpl.ICassandraProcess; import com.netflix.priam.health.InstanceState; @@ -33,8 +33,10 @@ import java.math.BigInteger; import java.nio.file.Path; import java.time.LocalDateTime; +import java.time.ZoneId; import java.util.*; import java.util.concurrent.Future; +import javax.inject.Named; import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.io.FileUtils; import org.apache.commons.lang3.StringUtils; @@ -48,9 +50,6 @@ * thread pool to execute the restores. */ public abstract class AbstractRestore extends Task implements IRestoreStrategy { - // keeps track of the last few download which was executed. - // TODO fix the magic number of 1000 => the idea of 80% of 1000 files limit per s3 query - protected static final FifoQueue tracker = new FifoQueue<>(800); private static final Logger logger = LoggerFactory.getLogger(AbstractRestore.class); private static final String JOBNAME = "AbstractRestore"; private static final String SYSTEM_KEYSPACE = "system"; @@ -66,7 +65,15 @@ public abstract class AbstractRestore extends Task implements IRestoreStrategy { private final MetaData metaData; private final IPostRestoreHook postRestoreHook; - AbstractRestore( + @Inject + @Named("v1") + IMetaProxy metaV1Proxy; + + @Inject + @Named("v2") + IMetaProxy metaV2Proxy; + + public AbstractRestore( IConfiguration config, IBackupFileSystem fs, String name, @@ -113,8 +120,6 @@ private List> download( List> futureList = new ArrayList<>(); while (fsIterator.hasNext()) { AbstractBackupPath temp = fsIterator.next(); - if (temp.getType() == BackupFileType.SST && tracker.contains(temp)) continue; - if (backupRestoreUtil.isFiltered( temp.getKeyspace(), temp.getColumnFamily())) { // is filtered? logger.info( @@ -158,7 +163,7 @@ private List> downloadCommitLogs( BoundedList bl = new BoundedList(lastN); while (fsIterator.hasNext()) { AbstractBackupPath temp = fsIterator.next(); - if (temp.getType() == BackupFileType.SST && tracker.contains(temp)) continue; + if (temp.getType() == BackupFileType.SST) continue; if (temp.getType() == filter) { bl.add(temp); @@ -172,57 +177,16 @@ private void stopCassProcess() throws IOException { cassProcess.stop(true); } - private String getRestorePrefix() { - String prefix; - - if (StringUtils.isNotBlank(config.getRestorePrefix())) prefix = config.getRestorePrefix(); - else prefix = config.getBackupPrefix(); - - return prefix; - } - - /* - * Fetches meta.json used to store snapshots metadata. - */ - private List fetchSnapshotMetaFile( - String restorePrefix, Date startTime, Date endTime) throws IllegalStateException { - logger.debug("Looking for snapshot meta file within restore prefix: {}", restorePrefix); - List metas = Lists.newArrayList(); - - Iterator backupfiles = fs.list(restorePrefix, startTime, endTime); - if (!backupfiles.hasNext()) { - throw new IllegalStateException( - "meta.json not found, restore prefix: " + restorePrefix); - } - - while (backupfiles.hasNext()) { - AbstractBackupPath path = backupfiles.next(); - if (path.getType() == BackupFileType.META) - // Since there are now meta file for incrementals as well as snapshot, we need to - // find the correct one (i.e. the snapshot meta file (meta.json)) - if (path.getFileName().equalsIgnoreCase("meta.json")) { - metas.add(path); - } - } - - // Sort the meta files in ascending order. - Collections.sort(metas); - - return metas; - } - @Override public void execute() throws Exception { if (!isRestoreEnabled(config, instanceIdentity.getInstanceInfo())) return; logger.info("Starting restore for {}", config.getRestoreSnapshot()); - String[] restore = config.getRestoreSnapshot().split(","); - final Date startTime = DateUtil.getDate(restore[0]); - final Date endTime = DateUtil.getDate(restore[1]); + final DateUtil.DateRange dateRange = new DateUtil.DateRange(config.getRestoreSnapshot()); new RetryableCallable() { public Void retriableCall() throws Exception { logger.info("Attempting restore"); - restore(startTime, endTime); + restore(dateRange); logger.info("Restore completed"); // Wait for other server init to complete @@ -232,15 +196,20 @@ public Void retriableCall() throws Exception { }.call(); } - public void restore(Date startTime, Date endTime) throws Exception { + public void restore(DateUtil.DateRange dateRange) throws Exception { // fail early if post restore hook has invalid parameters if (!postRestoreHook.hasValidParameters()) { throw new PostRestoreHookException("Invalid PostRestoreHook parameters"); } + Date endTime = new Date(dateRange.getEndTime().toEpochMilli()); + // Set the restore status. instanceState.getRestoreStatus().resetStatus(); - instanceState.getRestoreStatus().setStartDateRange(DateUtil.convert(startTime)); + instanceState + .getRestoreStatus() + .setStartDateRange( + LocalDateTime.ofInstant(dateRange.getStartTime(), ZoneId.of("UTC"))); instanceState.getRestoreStatus().setEndDateRange(DateUtil.convert(endTime)); instanceState.getRestoreStatus().setExecutionStartTime(LocalDateTime.now()); instanceState.setRestoreStatus(Status.STARTED); @@ -248,7 +217,10 @@ public void restore(Date startTime, Date endTime) throws Exception { try { if (config.isRestoreClosestToken()) { - restoreToken = tokenSelector.getClosestToken(new BigInteger(origToken), startTime); + restoreToken = + tokenSelector.getClosestToken( + new BigInteger(origToken), + new Date(dateRange.getStartTime().toEpochMilli())); instanceIdentity.getInstance().setToken(restoreToken.toString()); } @@ -260,34 +232,33 @@ public void restore(Date startTime, Date endTime) throws Exception { if (dataDir.exists() && dataDir.isDirectory()) FileUtils.cleanDirectory(dataDir); // Try and read the Meta file. - String prefix = getRestorePrefix(); - List metas = fetchSnapshotMetaFile(prefix, startTime, endTime); + List metas = metaV1Proxy.findMetaFiles(dateRange); if (metas.size() == 0) { - logger.info("[cass_backup] No snapshot meta file found, Restore Failed."); + logger.info("No snapshot meta file found, Restore Failed."); instanceState.getRestoreStatus().setExecutionEndTime(LocalDateTime.now()); - instanceState.setRestoreStatus(Status.FINISHED); + instanceState.setRestoreStatus(Status.FAILED); return; } - AbstractBackupPath meta = Iterators.getLast(metas.iterator()); + AbstractBackupPath meta = metas.get(0); logger.info("Snapshot Meta file for restore {}", meta.getRemotePath()); instanceState.getRestoreStatus().setSnapshotMetaFile(meta.getRemotePath()); + // TODO: Validate that manifest file is valid. // Download the meta.json file. - ArrayList metaFile = new ArrayList<>(); - metaFile.add(meta); - download(metaFile.iterator(), BackupFileType.META, true); + Path metaFile = metaV1Proxy.downloadMetaFile(meta); List> futureList = new ArrayList<>(); // Parse meta.json file to find the files required to download from this snapshot. - List snapshots = metaData.toJson(meta.newRestoreFile()); + List snapshots = metaData.toJson(metaFile.toFile()); // Download snapshot which is listed in the meta file. futureList.addAll(download(snapshots.iterator(), BackupFileType.SNAP, false)); logger.info("Downloading incrementals"); // Download incrementals (SST) after the snapshot meta file. + String prefix = fs.getPrefix().toString(); Iterator incrementals = fs.list(prefix, meta.getTime(), endTime); futureList.addAll(download(incrementals, BackupFileType.SST, false)); diff --git a/priam/src/main/java/com/netflix/priam/restore/EncryptedRestoreBase.java b/priam/src/main/java/com/netflix/priam/restore/EncryptedRestoreBase.java index 300ad9e45..ee9ebfc6a 100755 --- a/priam/src/main/java/com/netflix/priam/restore/EncryptedRestoreBase.java +++ b/priam/src/main/java/com/netflix/priam/restore/EncryptedRestoreBase.java @@ -106,7 +106,6 @@ public Path retriableCall() throws Exception { Paths.get(path.getRemotePath()), Paths.get(tempFile.getAbsolutePath()), 0); - tracker.adjustAndAdd(path); } catch (Exception ex) { // This behavior is retryable; therefore, lets get to a clean state // before each retry. diff --git a/priam/src/main/java/com/netflix/priam/restore/Restore.java b/priam/src/main/java/com/netflix/priam/restore/Restore.java index a7a5d2d6c..ac0a27f66 100644 --- a/priam/src/main/java/com/netflix/priam/restore/Restore.java +++ b/priam/src/main/java/com/netflix/priam/restore/Restore.java @@ -72,7 +72,6 @@ public Restore( @Override protected final Future downloadFile( final AbstractBackupPath path, final File restoreLocation) throws Exception { - tracker.adjustAndAdd(path); return fs.asyncDownloadFile( Paths.get(path.getRemotePath()), Paths.get(restoreLocation.getAbsolutePath()), 5); } diff --git a/priam/src/main/java/com/netflix/priam/services/BackupTTLService.java b/priam/src/main/java/com/netflix/priam/services/BackupTTLService.java index a4f770d70..74dfca4a0 100644 --- a/priam/src/main/java/com/netflix/priam/services/BackupTTLService.java +++ b/priam/src/main/java/com/netflix/priam/services/BackupTTLService.java @@ -24,8 +24,8 @@ import com.netflix.priam.backup.BackupRestoreException; import com.netflix.priam.backup.IBackupFileSystem; import com.netflix.priam.backup.IFileSystemContext; -import com.netflix.priam.backupv2.BackupValidator; import com.netflix.priam.backupv2.ColumnfamilyResult; +import com.netflix.priam.backupv2.IMetaProxy; import com.netflix.priam.backupv2.MetaFileReader; import com.netflix.priam.config.IBackupRestoreConfig; import com.netflix.priam.config.IConfiguration; @@ -40,6 +40,7 @@ import java.util.*; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; +import javax.inject.Named; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -58,7 +59,7 @@ public class BackupTTLService extends Task { private static final Logger logger = LoggerFactory.getLogger(BackupTTLService.class); private IBackupRestoreConfig backupRestoreConfig; - private BackupValidator backupValidator; + private IMetaProxy metaProxy; private IBackupFileSystem fileSystem; private Provider abstractBackupPathProvider; public static final String JOBNAME = "BackupTTLService"; @@ -72,12 +73,12 @@ public class BackupTTLService extends Task { public BackupTTLService( IConfiguration configuration, IBackupRestoreConfig backupRestoreConfig, - BackupValidator backupValidator, + @Named("v2") IMetaProxy metaProxy, IFileSystemContext backupFileSystemCtx, Provider abstractBackupPathProvider) { super(configuration); this.backupRestoreConfig = backupRestoreConfig; - this.backupValidator = backupValidator; + this.metaProxy = metaProxy; this.fileSystem = backupFileSystemCtx.getFileStrategy(configuration); this.abstractBackupPathProvider = abstractBackupPathProvider; } @@ -110,7 +111,7 @@ public void execute() throws Exception { // Find the snapshot just after this date. List metas = - backupValidator.findMetaFiles( + metaProxy.findMetaFiles( new DateUtil.DateRange(dateToTtl, DateUtil.getInstant())); if (metas.size() == 0) { @@ -123,7 +124,7 @@ public void execute() throws Exception { AbstractBackupPath metaFile = metas.get(metas.size() - 1); // Download the meta file to local file system. - Path localFile = backupValidator.downloadMetaFile(metaFile); + Path localFile = metaProxy.downloadMetaFile(metaFile); // Walk over the file system iterator and if not in map, it is eligible for delete. new MetaFileWalker().readMeta(localFile); @@ -159,7 +160,7 @@ public void execute() throws Exception { // all the META files. // This feature did not exist in Jan 2018. metas = - backupValidator.findMetaFiles( + metaProxy.findMetaFiles( new DateUtil.DateRange( start_of_feature, dateToTtl.minus(1, ChronoUnit.MINUTES))); diff --git a/priam/src/main/java/com/netflix/priam/services/SnapshotMetaService.java b/priam/src/main/java/com/netflix/priam/services/SnapshotMetaService.java index 43daf9d7e..63d695d9d 100644 --- a/priam/src/main/java/com/netflix/priam/services/SnapshotMetaService.java +++ b/priam/src/main/java/com/netflix/priam/services/SnapshotMetaService.java @@ -30,6 +30,7 @@ import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; import javax.inject.Inject; +import javax.inject.Named; import javax.inject.Singleton; import org.apache.commons.io.FileUtils; import org.apache.commons.io.filefilter.FileFilterUtils; @@ -64,7 +65,7 @@ public class SnapshotMetaService extends AbstractBackup { private final BackupRestoreUtil backupRestoreUtil; private final MetaFileWriterBuilder metaFileWriter; private MetaFileWriterBuilder.DataStep dataStep; - private final MetaFileManager metaFileManager; + private final IMetaProxy metaProxy; private final CassandraOperations cassandraOperations; private String snapshotName = null; private static final Lock lock = new ReentrantLock(); @@ -83,7 +84,7 @@ private enum MetaStep { IFileSystemContext backupFileSystemCtx, Provider pathFactory, MetaFileWriterBuilder metaFileWriter, - MetaFileManager metaFileManager, + @Named("v2") IMetaProxy metaProxy, CassandraOperations cassandraOperations) { super(config, backupFileSystemCtx, pathFactory); this.backupRestoreConfig = backupRestoreConfig; @@ -92,7 +93,7 @@ private enum MetaStep { new BackupRestoreUtil( config.getSnapshotIncludeCFList(), config.getSnapshotExcludeCFList()); this.metaFileWriter = metaFileWriter; - this.metaFileManager = metaFileManager; + this.metaProxy = metaProxy; } /** @@ -177,7 +178,7 @@ public void execute() throws Exception { // These files may be leftover // 1) when Priam shutdown in middle of this service and may not be full JSON // 2) No permission to upload to backup file system. - metaFileManager.cleanupOldMetaFiles(); + metaProxy.cleanupOldMetaFiles(); // Take a new snapshot cassandraOperations.takeSnapshot(snapshotName); diff --git a/priam/src/main/java/com/netflix/priam/utils/DateUtil.java b/priam/src/main/java/com/netflix/priam/utils/DateUtil.java index edcee23af..ff28cf96f 100644 --- a/priam/src/main/java/com/netflix/priam/utils/DateUtil.java +++ b/priam/src/main/java/com/netflix/priam/utils/DateUtil.java @@ -218,5 +218,12 @@ public Instant getEndTime() { public String toString() { return GsonJsonSerializer.getGson().toJson(this); } + + @Override + public boolean equals(Object obj) { + return obj.getClass().equals(this.getClass()) + && (startTime.toEpochMilli() == ((DateRange) obj).startTime.toEpochMilli()) + && (endTime.toEpochMilli() == ((DateRange) obj).endTime.toEpochMilli()); + } } } diff --git a/priam/src/test/java/com/netflix/priam/aws/MockAmazonS3Client.java b/priam/src/test/java/com/netflix/priam/aws/MockAmazonS3Client.java deleted file mode 100644 index d80501a51..000000000 --- a/priam/src/test/java/com/netflix/priam/aws/MockAmazonS3Client.java +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Copyright 2018 Netflix, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.netflix.priam.aws; - -import com.amazonaws.AmazonClientException; -import com.amazonaws.services.s3.AmazonS3Client; -import com.amazonaws.services.s3.model.ListObjectsRequest; -import com.amazonaws.services.s3.model.ObjectListing; -import mockit.Mock; -import mockit.MockUp; - -/** Created by aagrawal on 12/6/18. */ -public class MockAmazonS3Client extends MockUp { - @Mock - public ObjectListing listObjects(ListObjectsRequest listObjectsRequest) - throws AmazonClientException { - ObjectListing listing = new ObjectListing(); - listing.setBucketName(listObjectsRequest.getBucketName()); - listing.setPrefix(listObjectsRequest.getPrefix()); - return listing; - } - - @Mock - public ObjectListing listNextBatchOfObjects(ObjectListing previousObjectListing) - throws AmazonClientException { - ObjectListing listing = new ObjectListing(); - listing.setBucketName(previousObjectListing.getBucketName()); - listing.setPrefix(previousObjectListing.getPrefix()); - return new ObjectListing(); - } -} diff --git a/priam/src/test/java/com/netflix/priam/aws/TestRemoteBackupPath.java b/priam/src/test/java/com/netflix/priam/aws/TestRemoteBackupPath.java index 90fca2b3a..67e1d0dc7 100644 --- a/priam/src/test/java/com/netflix/priam/aws/TestRemoteBackupPath.java +++ b/priam/src/test/java/com/netflix/priam/aws/TestRemoteBackupPath.java @@ -112,7 +112,7 @@ public void testV1BackupPathsSnap() throws ParseException { Assert.assertEquals(BackupFileType.SNAP, abstractBackupPath.getType()); Assert.assertEquals(path.toFile(), abstractBackupPath.getBackupFile()); Assert.assertEquals( - "201801011201", AbstractBackupPath.formatDate(abstractBackupPath.getTime())); + "201801011201", DateUtil.formatyyyyMMddHHmm(abstractBackupPath.getTime())); // Verify toRemote and parseRemote. String remotePath = abstractBackupPath.getRemotePath(); diff --git a/priam/src/test/java/com/netflix/priam/backup/BRTestModule.java b/priam/src/test/java/com/netflix/priam/backup/BRTestModule.java index 176b610d5..29c87b051 100644 --- a/priam/src/test/java/com/netflix/priam/backup/BRTestModule.java +++ b/priam/src/test/java/com/netflix/priam/backup/BRTestModule.java @@ -22,6 +22,9 @@ import com.google.inject.name.Names; import com.netflix.priam.aws.auth.IS3Credential; import com.netflix.priam.aws.auth.S3RoleAssumptionCredential; +import com.netflix.priam.backupv2.IMetaProxy; +import com.netflix.priam.backupv2.MetaV1Proxy; +import com.netflix.priam.backupv2.MetaV2Proxy; import com.netflix.priam.config.FakeBackupRestoreConfig; import com.netflix.priam.config.FakeConfiguration; import com.netflix.priam.config.IBackupRestoreConfig; @@ -78,5 +81,7 @@ protected void configure() { bind(ICassandraProcess.class).to(FakeCassandraProcess.class); bind(IPostRestoreHook.class).to(FakePostRestoreHook.class); bind(Registry.class).toInstance(new DefaultRegistry()); + bind(IMetaProxy.class).annotatedWith(Names.named("v1")).to(MetaV1Proxy.class); + bind(IMetaProxy.class).annotatedWith(Names.named("v2")).to(MetaV2Proxy.class); } } diff --git a/priam/src/test/java/com/netflix/priam/backup/FakeBackupFileSystem.java b/priam/src/test/java/com/netflix/priam/backup/FakeBackupFileSystem.java index 91dd7b101..b65589cd6 100644 --- a/priam/src/test/java/com/netflix/priam/backup/FakeBackupFileSystem.java +++ b/priam/src/test/java/com/netflix/priam/backup/FakeBackupFileSystem.java @@ -49,7 +49,6 @@ public FakeBackupFileSystem( } public void setupTest(List files) { - System.out.println("Setting up: " + files); clearTest(); for (String file : files) { AbstractBackupPath path = pathProvider.get(); @@ -104,8 +103,6 @@ public Iterator listFileSystem(String prefix, String delimiter, String m if (abstractBackupPath.getRemotePath().startsWith(prefix)) items.add(abstractBackupPath.getRemotePath()); }); - System.out.println(flist); - System.out.println(items); return items.iterator(); } diff --git a/priam/src/test/java/com/netflix/priam/backup/TestBackupFile.java b/priam/src/test/java/com/netflix/priam/backup/TestBackupFile.java index 365c38110..52478f3f6 100644 --- a/priam/src/test/java/com/netflix/priam/backup/TestBackupFile.java +++ b/priam/src/test/java/com/netflix/priam/backup/TestBackupFile.java @@ -22,6 +22,7 @@ import com.netflix.priam.aws.RemoteBackupPath; import com.netflix.priam.backup.AbstractBackupPath.BackupFileType; import com.netflix.priam.identity.InstanceIdentity; +import com.netflix.priam.utils.DateUtil; import java.io.BufferedOutputStream; import java.io.File; import java.io.FileOutputStream; @@ -101,7 +102,7 @@ public void testIncBackupFileCreation() throws ParseException { Assert.assertEquals("fake-app", backupfile.clusterName); Assert.assertEquals(region, backupfile.region); Assert.assertEquals("casstestbackup", backupfile.baseDir); - String datestr = AbstractBackupPath.formatDate(new Date(bfile.lastModified())); + String datestr = DateUtil.formatyyyyMMddHHmm(new Date(bfile.lastModified())); Assert.assertEquals( "casstestbackup/" + region @@ -118,7 +119,7 @@ public void testMetaFileCreation() throws ParseException { File bfile = new File(filestr); RemoteBackupPath backupfile = injector.getInstance(RemoteBackupPath.class); backupfile.parseLocal(bfile, BackupFileType.META); - backupfile.setTime(backupfile.parseDate("201108082320")); + backupfile.setTime(DateUtil.getDate("201108082320")); Assert.assertEquals(BackupFileType.META, backupfile.type); Assert.assertEquals("1234567", backupfile.token); Assert.assertEquals("fake-app", backupfile.clusterName); diff --git a/priam/src/test/java/com/netflix/priam/backup/TestBackupVerification.java b/priam/src/test/java/com/netflix/priam/backup/TestBackupVerification.java new file mode 100644 index 000000000..3b6c0d512 --- /dev/null +++ b/priam/src/test/java/com/netflix/priam/backup/TestBackupVerification.java @@ -0,0 +1,86 @@ +/* + * Copyright 2018 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.netflix.priam.backup; + +import com.google.inject.Guice; +import com.google.inject.Injector; +import com.netflix.priam.utils.DateUtil; +import java.time.Instant; +import java.time.temporal.ChronoUnit; +import java.util.ArrayList; +import java.util.Date; +import java.util.List; +import java.util.Optional; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +/** Created by aagrawal on 12/21/18. */ +public class TestBackupVerification { + + private static BackupVerification backupVerification; + private final String backupDate = "201812011000"; + private List backupMetadataList = new ArrayList<>(); + + public TestBackupVerification() { + Injector injector = Guice.createInjector(new BRTestModule()); + backupVerification = injector.getInstance(BackupVerification.class); + } + + @Before + public void prepare() throws Exception { + backupMetadataList.clear(); + Instant start = DateUtil.parseInstant(backupDate); + backupMetadataList.add(getBackupMetaData(start, Status.FINISHED)); + backupMetadataList.add(getBackupMetaData(start.plus(2, ChronoUnit.HOURS), Status.FAILED)); + backupMetadataList.add(getBackupMetaData(start.plus(4, ChronoUnit.HOURS), Status.FINISHED)); + backupMetadataList.add(getBackupMetaData(start.plus(6, ChronoUnit.HOURS), Status.FAILED)); + backupMetadataList.add(getBackupMetaData(start.plus(8, ChronoUnit.HOURS), Status.FAILED)); + } + + @Test + public void getLatestBackup() { + Optional backupMetadata = + backupVerification.getLatestBackupMetaData(backupMetadataList); + Instant start = DateUtil.parseInstant(backupDate); + Assert.assertEquals( + start.plus(4, ChronoUnit.HOURS), backupMetadata.get().getStart().toInstant()); + } + + @Test + public void getLatestBackupFailure() throws Exception { + Optional backupMetadata = + backupVerification.getLatestBackupMetaData(new ArrayList<>()); + Assert.assertFalse(backupMetadata.isPresent()); + + List failList = new ArrayList<>(); + failList.add(getBackupMetaData(DateUtil.getInstant(), Status.FAILED)); + backupMetadata = backupVerification.getLatestBackupMetaData(failList); + Assert.assertFalse(backupMetadata.isPresent()); + } + + private BackupMetadata getBackupMetaData(Instant startTime, Status status) throws Exception { + BackupMetadata backupMetadata = + new BackupMetadata("123", new Date(startTime.toEpochMilli())); + backupMetadata.setCompleted( + new Date(startTime.plus(30, ChronoUnit.MINUTES).toEpochMilli())); + backupMetadata.setStatus(status); + backupMetadata.setSnapshotLocation("file.txt"); + return backupMetadata; + } +} diff --git a/priam/src/test/java/com/netflix/priam/backup/TestFileIterator.java b/priam/src/test/java/com/netflix/priam/backup/TestFileIterator.java index bb63946ee..e79b45f51 100644 --- a/priam/src/test/java/com/netflix/priam/backup/TestFileIterator.java +++ b/priam/src/test/java/com/netflix/priam/backup/TestFileIterator.java @@ -17,15 +17,16 @@ package com.netflix.priam.backup; +import com.amazonaws.AmazonClientException; import com.amazonaws.services.s3.AmazonS3Client; +import com.amazonaws.services.s3.model.ListObjectsRequest; import com.amazonaws.services.s3.model.ObjectListing; import com.amazonaws.services.s3.model.S3ObjectSummary; import com.google.inject.Guice; import com.google.inject.Injector; -import com.netflix.priam.aws.MockAmazonS3Client; import com.netflix.priam.aws.S3FileSystem; -import com.netflix.priam.config.IConfiguration; import com.netflix.priam.identity.InstanceIdentity; +import com.netflix.priam.utils.DateUtil; import java.io.IOException; import java.util.*; import mockit.Mock; @@ -41,35 +42,46 @@ * @author Praveen Sadhu */ public class TestFileIterator { - private static Injector injector; private static Date startTime, endTime; - private static Calendar cal; - private static AmazonS3Client s3client; private static S3FileSystem s3FileSystem; - private static IConfiguration conf; - private static InstanceIdentity factory; private static String region; private static String bucket = "TESTBUCKET"; @BeforeClass public static void setup() throws InterruptedException, IOException { - s3client = new MockAmazonS3Client().getMockInstance(); + AmazonS3Client s3client = new MockAmazonS3Client().getMockInstance(); new MockObjectListing(); - injector = Guice.createInjector(new BRTestModule()); - conf = injector.getInstance(IConfiguration.class); - factory = injector.getInstance(InstanceIdentity.class); + Injector injector = Guice.createInjector(new BRTestModule()); + InstanceIdentity factory = injector.getInstance(InstanceIdentity.class); region = factory.getInstanceInfo().getRegion(); s3FileSystem = injector.getInstance(S3FileSystem.class); s3FileSystem.setS3Client(s3client); - cal = Calendar.getInstance(); - cal.set(2011, 7, 11, 0, 30, 0); - cal.set(Calendar.MILLISECOND, 0); - startTime = cal.getTime(); - cal.add(Calendar.HOUR, 5); - endTime = cal.getTime(); + DateUtil.DateRange dateRange = new DateUtil.DateRange("201108110030,201108110530"); + startTime = new Date(dateRange.getStartTime().toEpochMilli()); + endTime = new Date(dateRange.getEndTime().toEpochMilli()); + } + + static class MockAmazonS3Client extends MockUp { + @Mock + public ObjectListing listObjects(ListObjectsRequest listObjectsRequest) + throws AmazonClientException { + ObjectListing listing = new ObjectListing(); + listing.setBucketName(listObjectsRequest.getBucketName()); + listing.setPrefix(listObjectsRequest.getPrefix()); + return listing; + } + + @Mock + public ObjectListing listNextBatchOfObjects(ObjectListing previousObjectListing) + throws AmazonClientException { + ObjectListing listing = new ObjectListing(); + listing.setBucketName(previousObjectListing.getBucketName()); + listing.setPrefix(previousObjectListing.getPrefix()); + return new ObjectListing(); + } } // MockObjectListing class @@ -102,11 +114,9 @@ public boolean isTruncated() { @Test public void testIteratorEmptySet() { - cal.set(2011, 7, 11, 6, 1, 0); - cal.set(Calendar.MILLISECOND, 0); - Date stime = cal.getTime(); - cal.add(Calendar.HOUR, 5); - Date etime = cal.getTime(); + DateUtil.DateRange dateRange = new DateUtil.DateRange("201107110601,201107111101"); + Date stime = new Date(dateRange.getStartTime().toEpochMilli()); + Date etime = new Date(dateRange.getEndTime().toEpochMilli()); Iterator fileIterator = s3FileSystem.list(bucket, stime, etime); Set files = new HashSet<>(); diff --git a/priam/src/test/java/com/netflix/priam/backup/TestRestore.java b/priam/src/test/java/com/netflix/priam/backup/TestRestore.java index b603f3dd0..3a931f6bc 100644 --- a/priam/src/test/java/com/netflix/priam/backup/TestRestore.java +++ b/priam/src/test/java/com/netflix/priam/backup/TestRestore.java @@ -19,41 +19,34 @@ import com.google.inject.Guice; import com.google.inject.Injector; -import com.google.inject.Key; -import com.google.inject.name.Names; import com.netflix.priam.config.FakeConfiguration; import com.netflix.priam.config.IConfiguration; +import com.netflix.priam.health.InstanceState; import com.netflix.priam.identity.config.InstanceInfo; import com.netflix.priam.restore.Restore; -import java.io.File; +import com.netflix.priam.utils.DateUtil; import java.io.IOException; import java.util.ArrayList; -import java.util.Calendar; -import java.util.Date; import org.junit.Assert; import org.junit.BeforeClass; import org.junit.Test; public class TestRestore { private static FakeBackupFileSystem filesystem; - private static ArrayList fileList; - private static Calendar cal; + private static ArrayList fileList = new ArrayList<>(); private static FakeConfiguration conf; private static String region; private static Restore restore; + private static InstanceState instanceState; @BeforeClass public static void setup() throws InterruptedException, IOException { Injector injector = Guice.createInjector(new BRTestModule()); - filesystem = - (FakeBackupFileSystem) - injector.getInstance( - Key.get(IBackupFileSystem.class, Names.named("backup"))); - conf = (FakeConfiguration) injector.getInstance(IConfiguration.class); + if (filesystem == null) filesystem = injector.getInstance(FakeBackupFileSystem.class); + if (conf == null) conf = (FakeConfiguration) injector.getInstance(IConfiguration.class); region = injector.getInstance(InstanceInfo.class).getRegion(); - restore = injector.getInstance(Restore.class); - fileList = new ArrayList<>(); - cal = Calendar.getInstance(); + if (restore == null) restore = injector.getInstance(Restore.class); + if (instanceState == null) instanceState = injector.getInstance(InstanceState.class); } private static void populateBackupFileSystem(String baseDir) { @@ -74,22 +67,15 @@ private static void populateBackupFileSystem(String baseDir) { @Test public void testRestore() throws Exception { populateBackupFileSystem("test_backup"); - File tmpdir = new File(conf.getDataFileLocation() + "/test"); - tmpdir.mkdir(); - Assert.assertTrue(tmpdir.exists()); - cal.set(2011, Calendar.AUGUST, 11, 0, 30, 0); - cal.set(Calendar.MILLISECOND, 0); - Date startTime = cal.getTime(); - cal.add(Calendar.HOUR, 5); - restore.restore(startTime, cal.getTime()); + String dateRange = "201108110030,201108110530"; + restore.restore(new DateUtil.DateRange(dateRange)); Assert.assertTrue(filesystem.downloadedFiles.contains(fileList.get(0))); Assert.assertTrue(filesystem.downloadedFiles.contains(fileList.get(1))); Assert.assertTrue(filesystem.downloadedFiles.contains(fileList.get(2))); Assert.assertTrue(filesystem.downloadedFiles.contains(fileList.get(3))); Assert.assertFalse(filesystem.downloadedFiles.contains(fileList.get(4))); Assert.assertFalse(filesystem.downloadedFiles.contains(fileList.get(5))); - tmpdir = new File(conf.getDataFileLocation() + "/test"); - Assert.assertFalse(tmpdir.exists()); + Assert.assertEquals(Status.FINISHED, instanceState.getRestoreStatus().getStatus()); } // Pick latest file @@ -99,11 +85,8 @@ public void testRestoreLatest() throws Exception { String metafile = "test_backup/" + region + "/fakecluster/123456/201108110130/META/meta.json"; filesystem.addFile(metafile); - cal.set(2011, Calendar.AUGUST, 11, 0, 30, 0); - cal.set(Calendar.MILLISECOND, 0); - Date startTime = cal.getTime(); - cal.add(Calendar.HOUR, 5); - restore.restore(startTime, cal.getTime()); + String dateRange = "201108110030,201108110530"; + restore.restore(new DateUtil.DateRange(dateRange)); Assert.assertFalse(filesystem.downloadedFiles.contains(fileList.get(0))); Assert.assertTrue(filesystem.downloadedFiles.contains(metafile)); Assert.assertTrue(filesystem.downloadedFiles.contains(fileList.get(1))); @@ -111,36 +94,30 @@ public void testRestoreLatest() throws Exception { Assert.assertTrue(filesystem.downloadedFiles.contains(fileList.get(3))); Assert.assertFalse(filesystem.downloadedFiles.contains(fileList.get(4))); Assert.assertFalse(filesystem.downloadedFiles.contains(fileList.get(5))); + Assert.assertEquals(Status.FINISHED, instanceState.getRestoreStatus().getStatus()); + Assert.assertEquals(metafile, instanceState.getRestoreStatus().getSnapshotMetaFile()); } @Test public void testNoSnapshots() throws Exception { - try { - filesystem.setupTest(fileList); - cal.set(2011, Calendar.SEPTEMBER, 11, 0, 30); - Date startTime = cal.getTime(); - cal.add(Calendar.HOUR, 5); - restore.restore(startTime, cal.getTime()); - Assert.assertFalse(true); // No exception thrown - } catch (IllegalStateException e) { - // We are ok. No snapshot found. - Assert.assertTrue(true); - } + populateBackupFileSystem("test_backup"); + filesystem.setupTest(fileList); + String dateRange = "201109110030,201109110530"; + restore.restore(new DateUtil.DateRange(dateRange)); + Assert.assertEquals(Status.FAILED, instanceState.getRestoreStatus().getStatus()); } @Test public void testRestoreFromDiffCluster() throws Exception { populateBackupFileSystem("test_backup_new"); - cal.set(2011, Calendar.AUGUST, 11, 0, 30, 0); - cal.set(Calendar.MILLISECOND, 0); - Date startTime = cal.getTime(); - cal.add(Calendar.HOUR, 5); - restore.restore(startTime, cal.getTime()); + String dateRange = "201108110030,201108110530"; + restore.restore(new DateUtil.DateRange(dateRange)); Assert.assertTrue(filesystem.downloadedFiles.contains(fileList.get(0))); Assert.assertTrue(filesystem.downloadedFiles.contains(fileList.get(1))); Assert.assertTrue(filesystem.downloadedFiles.contains(fileList.get(2))); Assert.assertTrue(filesystem.downloadedFiles.contains(fileList.get(3))); Assert.assertFalse(filesystem.downloadedFiles.contains(fileList.get(4))); Assert.assertFalse(filesystem.downloadedFiles.contains(fileList.get(5))); + Assert.assertEquals(Status.FINISHED, instanceState.getRestoreStatus().getStatus()); } } diff --git a/priam/src/test/java/com/netflix/priam/backupv2/TestBackupValidator.java b/priam/src/test/java/com/netflix/priam/backupv2/TestBackupValidator.java index 0a52afc2d..3c9c23237 100644 --- a/priam/src/test/java/com/netflix/priam/backupv2/TestBackupValidator.java +++ b/priam/src/test/java/com/netflix/priam/backupv2/TestBackupValidator.java @@ -41,6 +41,7 @@ public class TestBackupValidator { private IConfiguration configuration; private BackupValidator backupValidator; private TestBackupUtils backupUtils; + private IMetaProxy metaProxy; public TestBackupValidator() { Injector injector = Guice.createInjector(new BRTestModule()); @@ -49,25 +50,26 @@ public TestBackupValidator() { fs.setupTest(getRemoteFakeFiles()); backupValidator = injector.getInstance(BackupValidator.class); backupUtils = new TestBackupUtils(); + metaProxy = injector.getInstance(MetaV2Proxy.class); } @Test public void testMetaPrefix() { // Null date range - Assert.assertEquals(getPrefix() + "/META_V2", backupValidator.getMetaPrefix(null)); + Assert.assertEquals(getPrefix() + "/META_V2", metaProxy.getMetaPrefix(null)); // No end date. Assert.assertEquals( getPrefix() + "/META_V2", - backupValidator.getMetaPrefix(new DateUtil.DateRange(Instant.now(), null))); + metaProxy.getMetaPrefix(new DateUtil.DateRange(Instant.now(), null))); // No start date Assert.assertEquals( getPrefix() + "/META_V2", - backupValidator.getMetaPrefix(new DateUtil.DateRange(null, Instant.now()))); + metaProxy.getMetaPrefix(new DateUtil.DateRange(null, Instant.now()))); long start = 1834567890L; long end = 1834877776L; Assert.assertEquals( getPrefix() + "/META_V2/1834", - backupValidator.getMetaPrefix( + metaProxy.getMetaPrefix( new DateUtil.DateRange( Instant.ofEpochSecond(start), Instant.ofEpochSecond(end)))); } @@ -102,7 +104,7 @@ public void testIsMetaFileValid() throws Exception { @Test public void testFindMetaFiles() throws BackupRestoreException { List metas = - backupValidator.findMetaFiles( + metaProxy.findMetaFiles( new DateUtil.DateRange( Instant.ofEpochMilli(1859824860000L), Instant.ofEpochMilli(1859828420000L))); @@ -111,7 +113,7 @@ public void testFindMetaFiles() throws BackupRestoreException { Assert.assertTrue(fs.doesRemoteFileExist(Paths.get(metas.get(0).getRemotePath()))); metas = - backupValidator.findMetaFiles( + metaProxy.findMetaFiles( new DateUtil.DateRange( Instant.ofEpochMilli(1859824860000L), Instant.ofEpochMilli(1859828460000L))); diff --git a/priam/src/test/java/com/netflix/priam/backupv2/TestLocalDBReaderWriter.java b/priam/src/test/java/com/netflix/priam/backupv2/TestLocalDBReaderWriter.java deleted file mode 100644 index 1b9a0124d..000000000 --- a/priam/src/test/java/com/netflix/priam/backupv2/TestLocalDBReaderWriter.java +++ /dev/null @@ -1,288 +0,0 @@ -/* - * Copyright 2018 Netflix, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.netflix.priam.backupv2; - -import com.google.inject.Guice; -import com.google.inject.Injector; -import com.netflix.priam.backup.BRTestModule; -import com.netflix.priam.config.IConfiguration; -import com.netflix.priam.utils.DateUtil; -import java.io.IOException; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.ArrayList; -import java.util.EnumSet; -import java.util.List; -import java.util.Random; -import java.util.concurrent.Callable; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.Future; -import org.apache.cassandra.io.sstable.Component; -import org.apache.commons.io.FileUtils; -import org.junit.After; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** Created by aagrawal on 9/4/18. */ -public class TestLocalDBReaderWriter { - private static final Logger logger = - LoggerFactory.getLogger(TestLocalDBReaderWriter.class.getName()); - private static IConfiguration configuration; - private static LocalDBReaderWriter localDBReaderWriter; - private static Path dummyDataDirectoryLocation; - - @Before - public void setUp() { - Injector injector = Guice.createInjector(new BRTestModule()); - - if (configuration == null) configuration = injector.getInstance(IConfiguration.class); - - if (localDBReaderWriter == null) - localDBReaderWriter = injector.getInstance(LocalDBReaderWriter.class); - - dummyDataDirectoryLocation = Paths.get(configuration.getDataFileLocation()); - cleanupDir(dummyDataDirectoryLocation); - } - - @After - public void destroy() { - cleanupDir(dummyDataDirectoryLocation); - } - - private void cleanupDir(Path dir) { - if (dir.toFile().exists()) - try { - FileUtils.cleanDirectory(dir.toFile()); - } catch (IOException e) { - e.printStackTrace(); - } - } - - @Test - public void readWriteLocalDB() throws Exception { - int noOfKeyspaces = 2; - int noOfCf = 1; - int noOfSstables = 2; - Path localDbPath = - Paths.get(dummyDataDirectoryLocation.toString(), LocalDBReaderWriter.LOCAL_DB); - List localDBList = - generateDummyLocalDB(noOfKeyspaces, noOfCf, noOfSstables); - - localDBList.forEach( - localDB -> { - FileUploadResult fileUploadResult = - localDB.getLocalDBEntries().get(0).getFileUploadResult(); - final Path localDBPath = localDBReaderWriter.getLocalDBPath(fileUploadResult); - try { - localDBReaderWriter.writeLocalDB(localDBPath, localDB); - } catch (Exception e) { - logger.error("Error while writing to local DB: " + e.getMessage(), e); - } - }); - - // Verify the write succeeded for each KS/CF/SStable. - Assert.assertEquals(localDbPath.toFile().listFiles().length, noOfKeyspaces); - Path cfLocalDBPath = localDbPath.toFile().listFiles()[0].listFiles()[0].toPath(); - Assert.assertEquals(noOfSstables, cfLocalDBPath.toFile().listFiles().length); - - // Read the database. - LocalDBReaderWriter.LocalDB localDB = - localDBReaderWriter.readLocalDB(cfLocalDBPath.toFile().listFiles()[0].toPath()); - Assert.assertEquals( - EnumSet.allOf(Component.Type.class).size(), localDB.getLocalDBEntries().size()); - } - - @Test - public void upsertLocalDB() throws Exception { - LocalDBReaderWriter.LocalDB localDB = generateDummyLocalDB(1, 1, 1).get(0); - - // Lets do write with each LocalDBEntry first. - localDB.getLocalDBEntries() - .forEach( - localDBEntry -> { - try { - localDBReaderWriter.upsertLocalDBEntry(localDBEntry); - } catch (Exception e) { - e.printStackTrace(); - } - }); - - // Verify the write has happened. - LocalDBReaderWriter.LocalDBEntry localDBEntry = - localDBReaderWriter.getLocalDBEntry( - localDB.getLocalDBEntries().get(0).getFileUploadResult()); - Assert.assertNotNull(localDBEntry); - - // Now lets see if we can write the same entry again?? - LocalDBReaderWriter.LocalDB localDBUpsert = - localDBReaderWriter.upsertLocalDBEntry(localDBEntry); - Assert.assertEquals( - localDB.getLocalDBEntries().size(), localDBUpsert.getLocalDBEntries().size()); - - // Now lets change the localDBEntry and see if upsert succeeds. - localDBEntry.setTimeLastReferenced(DateUtil.getInstant()); - localDBUpsert = localDBReaderWriter.upsertLocalDBEntry(localDBEntry); - LocalDBReaderWriter.LocalDBEntry localDBEntryUpsert = - localDBReaderWriter.getLocalDBEntry(localDBEntry.getFileUploadResult()); - Assert.assertEquals( - localDBEntryUpsert.getTimeLastReferenced(), localDBEntry.getTimeLastReferenced()); - Assert.assertEquals( - localDB.getLocalDBEntries().size(), localDBUpsert.getLocalDBEntries().size()); - - // Now change the file modification time. This should end up creating a new DB Entry. - localDBEntry.getFileUploadResult().setLastModifiedTime(DateUtil.getInstant()); - localDBUpsert = localDBReaderWriter.upsertLocalDBEntry(localDBEntry); - localDBEntryUpsert = - localDBReaderWriter.getLocalDBEntry(localDBEntry.getFileUploadResult()); - Assert.assertEquals( - localDB.getLocalDBEntries().size() + 1, localDBUpsert.getLocalDBEntries().size()); - Assert.assertEquals( - localDBEntry.getFileUploadResult().getLastModifiedTime(), - localDBEntryUpsert.getFileUploadResult().getLastModifiedTime()); - } - - @Test - public void readConcurrentLocalDB() throws Exception { - List localDBList = generateDummyLocalDB(1, 1, 1); - localDBList.forEach( - localDB -> { - FileUploadResult fileUploadResult = - localDB.getLocalDBEntries().get(0).getFileUploadResult(); - final Path localDBPath = localDBReaderWriter.getLocalDBPath(fileUploadResult); - try { - localDBReaderWriter.writeLocalDB(localDBPath, localDB); - } catch (Exception e) { - logger.error("Error while writing to local DB: " + e.getMessage(), e); - } - }); - - FileUploadResult sample = - localDBList.get(0).getLocalDBEntries().get(0).getFileUploadResult(); - int size = 5; - - ExecutorService threads = Executors.newFixedThreadPool(size); - List> torun = new ArrayList<>(size); - for (int i = 0; i < size; i++) { - torun.add(() -> localDBReaderWriter.getLocalDBEntry(sample) != null); - } - - // all tasks executed in different threads, at 'once'. - List> futures = threads.invokeAll(torun); - - // no more need for the threadpool - threads.shutdown(); - // check the results of the tasks. - int noOfBadRun = 0; - for (Future fut : futures) { - if (!fut.get()) noOfBadRun++; - } - - Assert.assertEquals(0, noOfBadRun); - } - - @Test - public void writeConcurrentLocalDB() throws Exception { - LocalDBReaderWriter.LocalDB localDB = generateDummyLocalDB(1, 1, 1).get(0); - int size = localDB.getLocalDBEntries().size(); - ExecutorService threads = Executors.newFixedThreadPool(size); - List> torun = new ArrayList<>(size); - for (int i = 0; i < size; i++) { - int finalI = i; - torun.add( - () -> - localDBReaderWriter.upsertLocalDBEntry( - localDB.getLocalDBEntries().get(finalI))); - } - // all tasks executed in different threads, at 'once'. - List> futures = threads.invokeAll(torun); - - // no more need for the threadpool - threads.shutdown(); - // check the results of the tasks. - int noOfBadRun = 0; - for (Future fut : futures) { - // We expect exception here. - try { - fut.get(); - } catch (Exception e) { - noOfBadRun++; - } - } - - Assert.assertEquals(0, noOfBadRun); - LocalDBReaderWriter.LocalDB localDBRead = - localDBReaderWriter.readLocalDB( - localDBReaderWriter.getLocalDBPath( - localDB.getLocalDBEntries().get(0).getFileUploadResult())); - Assert.assertEquals( - localDB.getLocalDBEntries().size(), localDBRead.getLocalDBEntries().size()); - } - - private List generateDummyLocalDB( - int noOfKeyspaces, int noOfCf, int noOfSstables) { - - // Clean the dummy directory - cleanupDir(dummyDataDirectoryLocation); - List localDBList = new ArrayList<>(); - - Random random = new Random(); - - for (int i = 1; i <= noOfKeyspaces; i++) { - String keyspaceName = "sample" + i; - - for (int j = 1; j <= noOfCf; j++) { - String columnfamilyname = "cf" + j; - - for (int k = 1; k <= noOfSstables; k++) { - String prefixName = "mc-" + k + "-big"; - LocalDBReaderWriter.LocalDB localDB = - new LocalDBReaderWriter.LocalDB(new ArrayList<>()); - localDBList.add(localDB); - for (Component.Type type : EnumSet.allOf(Component.Type.class)) { - Path componentPath = - Paths.get( - dummyDataDirectoryLocation.toFile().getAbsolutePath(), - keyspaceName, - columnfamilyname, - prefixName + "-" + type.name() + ".db"); - FileUploadResult fileUploadResult = - new FileUploadResult( - componentPath, - keyspaceName, - columnfamilyname, - DateUtil.getInstant(), - DateUtil.getInstant(), - random.nextLong()); - LocalDBReaderWriter.LocalDBEntry localDBEntry = - new LocalDBReaderWriter.LocalDBEntry( - fileUploadResult, - DateUtil.getInstant(), - DateUtil.getInstant()); - localDB.getLocalDBEntries().add(localDBEntry); - } - } - } - } - - return localDBList; - } -} diff --git a/priam/src/test/java/com/netflix/priam/backupv2/TestMetaFileManager.java b/priam/src/test/java/com/netflix/priam/backupv2/TestMetaFileManager.java index a1dd5482a..d816b3ac8 100644 --- a/priam/src/test/java/com/netflix/priam/backupv2/TestMetaFileManager.java +++ b/priam/src/test/java/com/netflix/priam/backupv2/TestMetaFileManager.java @@ -35,12 +35,12 @@ /** Created by aagrawal on 11/28/18. */ public class TestMetaFileManager { - private MetaFileManager metaFileManager; + private MetaV2Proxy metaV2Proxy; private IConfiguration configuration; public TestMetaFileManager() { Injector injector = Guice.createInjector(new BRTestModule()); - metaFileManager = injector.getInstance(MetaFileManager.class); + metaV2Proxy = injector.getInstance(MetaV2Proxy.class); configuration = injector.getInstance(IConfiguration.class); } @@ -56,7 +56,7 @@ public void testCleanupOldMetaFiles() throws IOException { Assert.assertEquals(4, dataDir.toFile().listFiles().length); // clean the directory - metaFileManager.cleanupOldMetaFiles(); + metaV2Proxy.cleanupOldMetaFiles(); Assert.assertEquals(1, dataDir.toFile().listFiles().length); Path dummy = Paths.get(dataDir.toString(), "dummy.tmp"); diff --git a/priam/src/test/java/com/netflix/priam/resources/BackupServletTest.java b/priam/src/test/java/com/netflix/priam/resources/BackupServletTest.java index 793292ffb..ce1ae0fb5 100644 --- a/priam/src/test/java/com/netflix/priam/resources/BackupServletTest.java +++ b/priam/src/test/java/com/netflix/priam/resources/BackupServletTest.java @@ -21,18 +21,13 @@ import com.google.inject.Guice; import com.google.inject.Injector; -import com.google.inject.Provider; -import com.netflix.priam.PriamServer; import com.netflix.priam.backup.*; import com.netflix.priam.config.IConfiguration; -import com.netflix.priam.defaultimpl.ICassandraProcess; import com.netflix.priam.health.InstanceState; -import com.netflix.priam.identity.IPriamInstanceFactory; import com.netflix.priam.identity.config.InstanceInfo; import com.netflix.priam.restore.Restore; -import com.netflix.priam.tuner.ICassandraTuner; import com.netflix.priam.utils.DateUtil; -import java.util.Date; +import java.time.Instant; import javax.ws.rs.core.MediaType; import javax.ws.rs.core.Response; import mockit.Expectations; @@ -45,19 +40,11 @@ @RunWith(JMockit.class) public class BackupServletTest { - private @Mocked PriamServer priamServer; private IConfiguration config; - private @Mocked IBackupFileSystem bkpFs; private @Mocked Restore restoreObj; - private @Mocked Provider pathProvider; - private @Mocked ICassandraTuner tuner; private @Mocked SnapshotBackup snapshotBackup; - private @Mocked IPriamInstanceFactory factory; - private @Mocked ICassandraProcess cassProcess; - private @Mocked BackupStatusMgr bkupStatusMgr; private BackupServlet resource; private RestoreServlet restoreResource; - private BackupVerification backupVerification; private InstanceInfo instanceInfo; @Before @@ -66,9 +53,8 @@ public void setUp() { config = injector.getInstance(IConfiguration.class); InstanceState instanceState = injector.getInstance(InstanceState.class); instanceInfo = injector.getInstance(InstanceInfo.class); - resource = - new BackupServlet(config, bkpFs, snapshotBackup, bkupStatusMgr, backupVerification); - restoreResource = new RestoreServlet(restoreObj, instanceState); + resource = injector.getInstance(BackupServlet.class); + restoreResource = injector.getInstance(RestoreServlet.class); } @Test @@ -95,7 +81,7 @@ public void restore_minimal() throws Exception { instanceInfo.getRegion(); result = oldRegion; - restoreObj.restore((Date) any, (Date) any); // TODO: test default value + restoreObj.restore(new DateUtil.DateRange((Instant) any, (Instant) any)); } }; @@ -121,9 +107,7 @@ public void restore_withDateRange() throws Exception { DateUtil.getDate(dateRange.split(",")[1]); result = new DateTime(2011, 12, 31, 23, 59).toDate(); times = 1; - restoreObj.restore( - DateUtil.getDate(dateRange.split(",")[0]), - DateUtil.getDate(dateRange.split(",")[1])); + restoreObj.restore(new DateUtil.DateRange(dateRange)); } }; diff --git a/priam/src/test/java/com/netflix/priam/services/TestSnapshotMetaService.java b/priam/src/test/java/com/netflix/priam/services/TestSnapshotMetaService.java index ce0bc81b8..b65480f17 100644 --- a/priam/src/test/java/com/netflix/priam/services/TestSnapshotMetaService.java +++ b/priam/src/test/java/com/netflix/priam/services/TestSnapshotMetaService.java @@ -103,6 +103,9 @@ private void test(int noOfSstables, int noOfKeyspaces, int noOfCf) throws Except Assert.assertNotNull(metaFileLocation); Assert.assertTrue(metaFileLocation.toFile().exists()); Assert.assertTrue(metaFileLocation.toFile().isFile()); + Assert.assertEquals( + snapshotInstant.getEpochSecond(), + (metaFileLocation.toFile().lastModified() / 1000)); // Try reading meta file. metaFileReader.setNoOfSstables(noOfSstables + 1); From 060dc2e1fdb5e83e8ca82b8ea74d4cb5acbd54f3 Mon Sep 17 00:00:00 2001 From: Arun Agrawal Date: Wed, 26 Dec 2018 12:46:16 -0800 Subject: [PATCH 043/228] Remove deprecated code --- build.gradle | 18 +-- .../netflix/priam/backup/CommitLogBackup.java | 19 ---- .../priam/backup/CommitLogBackupTask.java | 19 ---- .../priam/backup/IMessageObserver.java | 55 ---------- .../priam/backup/IncrementalBackup.java | 22 ---- .../com/netflix/priam/backup/MetaData.java | 23 ---- .../netflix/priam/backup/SnapshotBackup.java | 45 +------- .../priam/cluster/management/Flush.java | 53 +-------- .../netflix/priam/config/IConfiguration.java | 48 -------- .../priam/config/PriamConfiguration.java | 27 ----- .../priam/scheduler/SchedulerType.java | 103 ------------------ .../TestFlushTask.groovy | 82 -------------- .../TestSchedulerType.groovy | 52 --------- .../backup/TestBackupScheduler.groovy | 44 ++------ .../cluser/management/TestFlushTask.groovy | 80 ++++++++++++++ .../netflix/priam/backup/TestCompression.java | 11 +- 16 files changed, 115 insertions(+), 586 deletions(-) delete mode 100644 priam/src/main/java/com/netflix/priam/backup/IMessageObserver.java delete mode 100644 priam/src/main/java/com/netflix/priam/scheduler/SchedulerType.java delete mode 100644 priam/src/test/groovy/com.netflix.priam.scheduler/TestFlushTask.groovy delete mode 100644 priam/src/test/groovy/com.netflix.priam.scheduler/TestSchedulerType.groovy create mode 100644 priam/src/test/groovy/com/netflix/priam/cluser/management/TestFlushTask.groovy diff --git a/build.gradle b/build.gradle index a9dd2bf9b..f7554e4f6 100644 --- a/build.gradle +++ b/build.gradle @@ -30,9 +30,9 @@ allprojects { } dependencies { - compile 'org.apache.commons:commons-lang3:3.5' + compile 'org.apache.commons:commons-lang3:3.8.1' compile 'commons-logging:commons-logging:1.2' - compile 'org.apache.commons:commons-collections4:4.1' + compile 'org.apache.commons:commons-collections4:4.2' compile 'commons-io:commons-io:2.6' compile 'commons-cli:commons-cli:1.4' compile 'commons-httpclient:commons-httpclient:3.1' @@ -42,7 +42,7 @@ allprojects { compile 'com.sun.jersey.contribs:jersey-guice:1.19.4' compile 'com.google.guava:guava:21.0' compile 'com.google.code.findbugs:jsr305:3.0.2' - compile 'com.amazonaws:aws-java-sdk:1.11.467' + compile 'com.amazonaws:aws-java-sdk:1.11.475' compile 'com.google.inject:guice:4.2.2' compile 'com.google.inject.extensions:guice-servlet:4.2.2' compile 'org.quartz-scheduler:quartz:2.3.0' @@ -52,11 +52,11 @@ allprojects { compile 'org.apache.cassandra:cassandra-all:3.0.17' compile 'javax.ws.rs:jsr311-api:1.1.1' compile 'joda-time:joda-time:2.10.1' - compile 'org.apache.commons:commons-configuration2:2.1.1' + compile 'org.apache.commons:commons-configuration2:2.4' compile 'xerces:xercesImpl:2.12.0' - compile 'net.java.dev.jna:jna:4.4.0' - compile 'org.apache.httpcomponents:httpclient:4.5.3' - compile 'org.apache.httpcomponents:httpcore:4.4.6' + compile 'net.java.dev.jna:jna:5.2.0' + compile 'org.apache.httpcomponents:httpclient:4.5.6' + compile 'org.apache.httpcomponents:httpcore:4.4.10' compile 'com.ning:compress-lzf:1.0.4' compile 'com.google.code.gson:gson:2.8.5' compile 'org.slf4j:slf4j-api:1.7.25' @@ -66,9 +66,9 @@ allprojects { compile ('com.google.appengine.tools:appengine-gcs-client:0.7') { exclude module: 'guava' } - compile 'com.google.apis:google-api-services-storage:v1-rev100-1.22.0' + compile 'com.google.apis:google-api-services-storage:v1-rev141-1.25.0' compile 'com.google.http-client:google-http-client-jackson2:1.22.0' - compile 'com.netflix.spectator:spectator-api:0.81.2' + compile 'com.netflix.spectator:spectator-api:0.82.0' compileOnly 'javax.servlet:javax.servlet-api:3.1.0' testCompile 'org.jmockit:jmockit:1.31' testCompile "org.spockframework:spock-core:1.1-groovy-2.4" diff --git a/priam/src/main/java/com/netflix/priam/backup/CommitLogBackup.java b/priam/src/main/java/com/netflix/priam/backup/CommitLogBackup.java index f7b1e761a..c667d3a54 100644 --- a/priam/src/main/java/com/netflix/priam/backup/CommitLogBackup.java +++ b/priam/src/main/java/com/netflix/priam/backup/CommitLogBackup.java @@ -30,7 +30,6 @@ public class CommitLogBackup { private static final Logger logger = LoggerFactory.getLogger(CommitLogBackup.class); private final Provider pathFactory; - private static final List observers = Lists.newArrayList(); private final List clRemotePaths = Lists.newArrayList(); private final IBackupFileSystem fs; @@ -85,24 +84,6 @@ public List upload(String archivedDir, final String snapshot return bps; } - public static void addObserver(IMessageObserver observer) { - observers.add(observer); - } - - public static void removeObserver(IMessageObserver observer) { - observers.remove(observer); - } - - public void notifyObservers() { - for (IMessageObserver observer : observers) - if (observer != null) { - logger.debug("Updating CommitLog observers now ..."); - observer.update(IMessageObserver.BACKUP_MESSAGE_TYPE.COMMITLOG, this.clRemotePaths); - } else { - logger.debug("Observer is Null, hence can not notify ..."); - } - } - private void addToRemotePath(String remotePath) { this.clRemotePaths.add(remotePath); } diff --git a/priam/src/main/java/com/netflix/priam/backup/CommitLogBackupTask.java b/priam/src/main/java/com/netflix/priam/backup/CommitLogBackupTask.java index 1f7ba4766..894b8da0b 100644 --- a/priam/src/main/java/com/netflix/priam/backup/CommitLogBackupTask.java +++ b/priam/src/main/java/com/netflix/priam/backup/CommitLogBackupTask.java @@ -16,7 +16,6 @@ import com.google.inject.Inject; import com.google.inject.Provider; import com.google.inject.Singleton; -import com.netflix.priam.backup.IMessageObserver.BACKUP_MESSAGE_TYPE; import com.netflix.priam.config.IConfiguration; import com.netflix.priam.scheduler.SimpleTimer; import com.netflix.priam.scheduler.TaskTimer; @@ -33,7 +32,6 @@ public class CommitLogBackupTask extends AbstractBackup { private static final Logger logger = LoggerFactory.getLogger(CommitLogBackupTask.class); private final List clRemotePaths = new ArrayList<>(); - private static final List observers = new ArrayList<>(); private final CommitLogBackup clBackup; @Inject @@ -67,23 +65,6 @@ public static TaskTimer getTimer(IConfiguration config) { return new SimpleTimer(JOBNAME, 60L * 1000); // every 1 min } - public static void addObserver(IMessageObserver observer) { - observers.add(observer); - } - - public static void removeObserver(IMessageObserver observer) { - observers.remove(observer); - } - - public void notifyObservers() { - for (IMessageObserver observer : observers) { - if (observer != null) { - logger.debug("Updating CL observers now ..."); - observer.update(BACKUP_MESSAGE_TYPE.COMMITLOG, clRemotePaths); - } else logger.info("Observer is Null, hence can not notify ..."); - } - } - @Override protected void processColumnFamily(String keyspace, String columnFamily, File backupDir) throws Exception { diff --git a/priam/src/main/java/com/netflix/priam/backup/IMessageObserver.java b/priam/src/main/java/com/netflix/priam/backup/IMessageObserver.java deleted file mode 100644 index 58af87dca..000000000 --- a/priam/src/main/java/com/netflix/priam/backup/IMessageObserver.java +++ /dev/null @@ -1,55 +0,0 @@ -/* - * Copyright 2013 Netflix, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ -package com.netflix.priam.backup; - -import java.util.List; - -public interface IMessageObserver { - - enum BACKUP_MESSAGE_TYPE { - SNAPSHOT, - INCREMENTAL, - COMMITLOG, - META - } - - enum RESTORE_MESSAGE_TYPE { - SNAPSHOT, - INCREMENTAL, - COMMITLOG, - META - } - - enum RESTORE_MESSAGE_STATUS { - UPLOADED, - DOWNLOADED, - STREAMED - } - - void update(BACKUP_MESSAGE_TYPE bkpMsgType, List remotePathNames); - - void update( - RESTORE_MESSAGE_TYPE rstMsgType, - List remotePathNames, - RESTORE_MESSAGE_STATUS rstMsgStatus); - - void update( - RESTORE_MESSAGE_TYPE rstMsgType, - String remotePath, - String fileDiskPath, - RESTORE_MESSAGE_STATUS rstMsgStatus); -} diff --git a/priam/src/main/java/com/netflix/priam/backup/IncrementalBackup.java b/priam/src/main/java/com/netflix/priam/backup/IncrementalBackup.java index 7c5aa2865..610d7f894 100644 --- a/priam/src/main/java/com/netflix/priam/backup/IncrementalBackup.java +++ b/priam/src/main/java/com/netflix/priam/backup/IncrementalBackup.java @@ -20,7 +20,6 @@ import com.google.inject.Provider; import com.google.inject.Singleton; import com.netflix.priam.backup.AbstractBackupPath.BackupFileType; -import com.netflix.priam.backup.IMessageObserver.BACKUP_MESSAGE_TYPE; import com.netflix.priam.config.IConfiguration; import com.netflix.priam.scheduler.SimpleTimer; import com.netflix.priam.scheduler.TaskTimer; @@ -41,7 +40,6 @@ public class IncrementalBackup extends AbstractBackup { private final List incrementalRemotePaths = new ArrayList<>(); private final IncrementalMetaData metaData; private final BackupRestoreUtil backupRestoreUtil; - private static final List observers = new ArrayList<>(); @Inject public IncrementalBackup( @@ -63,9 +61,6 @@ public void execute() throws Exception { // Clearing remotePath List incrementalRemotePaths.clear(); initiateBackup(INCREMENTAL_BACKUP_FOLDER, backupRestoreUtil); - if (incrementalRemotePaths.size() > 0) { - notifyObservers(); - } } /** Run every 10 Sec */ @@ -78,23 +73,6 @@ public String getName() { return JOBNAME; } - public static void addObserver(IMessageObserver observer) { - observers.add(observer); - } - - public static void removeObserver(IMessageObserver observer) { - observers.remove(observer); - } - - private void notifyObservers() { - for (IMessageObserver observer : observers) { - if (observer != null) { - logger.debug("Updating incremental observers now ..."); - observer.update(BACKUP_MESSAGE_TYPE.INCREMENTAL, incrementalRemotePaths); - } else logger.info("Observer is Null, hence can not notify ..."); - } - } - @Override protected void processColumnFamily(String keyspace, String columnFamily, File backupDir) throws Exception { diff --git a/priam/src/main/java/com/netflix/priam/backup/MetaData.java b/priam/src/main/java/com/netflix/priam/backup/MetaData.java index 258088d9f..67be8ae4c 100644 --- a/priam/src/main/java/com/netflix/priam/backup/MetaData.java +++ b/priam/src/main/java/com/netflix/priam/backup/MetaData.java @@ -20,7 +20,6 @@ import com.google.inject.Inject; import com.google.inject.Provider; import com.netflix.priam.backup.AbstractBackupPath.BackupFileType; -import com.netflix.priam.backup.IMessageObserver.BACKUP_MESSAGE_TYPE; import com.netflix.priam.config.IConfiguration; import com.netflix.priam.utils.DateUtil; import java.io.File; @@ -44,7 +43,6 @@ public class MetaData { private static final Logger logger = LoggerFactory.getLogger(MetaData.class); private final Provider pathFactory; - private static final List observers = new ArrayList<>(); private final List metaRemotePaths = new ArrayList<>(); private final IBackupFileSystem fs; @@ -74,10 +72,6 @@ public AbstractBackupPath set(List bps, String snapshotName) 10, true); addToRemotePath(backupfile.getRemotePath()); - if (metaRemotePaths.size() > 0) { - notifyObservers(); - } - return backupfile; } @@ -120,23 +114,6 @@ public File createTmpMetaFile() throws IOException { return destFile; } - public static void addObserver(IMessageObserver observer) { - observers.add(observer); - } - - public static void removeObserver(IMessageObserver observer) { - observers.remove(observer); - } - - private void notifyObservers() { - for (IMessageObserver observer : observers) { - if (observer != null) { - logger.debug("Updating snapshot observers now ..."); - observer.update(BACKUP_MESSAGE_TYPE.META, metaRemotePaths); - } else logger.info("Observer is Null, hence can not notify ..."); - } - } - private void addToRemotePath(String remotePath) { metaRemotePaths.add(remotePath); } diff --git a/priam/src/main/java/com/netflix/priam/backup/SnapshotBackup.java b/priam/src/main/java/com/netflix/priam/backup/SnapshotBackup.java index 0cb046b47..785079a15 100644 --- a/priam/src/main/java/com/netflix/priam/backup/SnapshotBackup.java +++ b/priam/src/main/java/com/netflix/priam/backup/SnapshotBackup.java @@ -21,7 +21,6 @@ import com.google.inject.Provider; import com.google.inject.Singleton; import com.netflix.priam.backup.AbstractBackupPath.BackupFileType; -import com.netflix.priam.backup.IMessageObserver.BACKUP_MESSAGE_TYPE; import com.netflix.priam.config.IConfiguration; import com.netflix.priam.defaultimpl.CassandraOperations; import com.netflix.priam.identity.InstanceIdentity; @@ -44,7 +43,6 @@ public class SnapshotBackup extends AbstractBackup { public static final String JOBNAME = "SnapshotBackup"; private final MetaData metaData; private final List snapshotRemotePaths = new ArrayList<>(); - private static final List observers = new ArrayList<>(); private final ThreadSleeper sleeper = new ThreadSleeper(); private static final long WAIT_TIME_MS = 60 * 1000 * 10; private final InstanceIdentity instanceIdentity; @@ -133,11 +131,6 @@ private void executeSnapshot() throws Exception { backupMetadata.setSnapshotLocation( config.getBackupPrefix() + File.separator + metaJson.getRemotePath()); snapshotStatusMgr.finish(backupMetadata); - - if (snapshotRemotePaths.size() > 0) { - notifyObservers(); - } - } catch (Exception e) { logger.error( "Exception occurred while taking snapshot: {}. Exception: {}", @@ -170,43 +163,7 @@ public static boolean isBackupEnabled(IConfiguration config) throws Exception { } public static TaskTimer getTimer(IConfiguration config) throws Exception { - CronTimer cronTimer = null; - switch (config.getBackupSchedulerType()) { - case HOUR: - if (config.getBackupHour() < 0) - logger.info( - "Skipping {} as it is disabled via backup hour: {}", - JOBNAME, - config.getBackupHour()); - else { - cronTimer = new CronTimer(JOBNAME, config.getBackupHour(), 1, 0); - logger.info( - "Starting snapshot backup with backup hour: {}", - config.getBackupHour()); - } - break; - case CRON: - cronTimer = CronTimer.getCronTimer(JOBNAME, config.getBackupCronExpression()); - break; - } - return cronTimer; - } - - public static void addObserver(IMessageObserver observer) { - observers.add(observer); - } - - public static void removeObserver(IMessageObserver observer) { - observers.remove(observer); - } - - private void notifyObservers() { - for (IMessageObserver observer : observers) { - if (observer != null) { - logger.debug("Updating snapshot observers now ..."); - observer.update(BACKUP_MESSAGE_TYPE.SNAPSHOT, snapshotRemotePaths); - } else logger.info("Observer is Null, hence can not notify ..."); - } + return CronTimer.getCronTimer(JOBNAME, config.getBackupCronExpression()); } @Override diff --git a/priam/src/main/java/com/netflix/priam/cluster/management/Flush.java b/priam/src/main/java/com/netflix/priam/cluster/management/Flush.java index 3379772de..3f53cec6c 100644 --- a/priam/src/main/java/com/netflix/priam/cluster/management/Flush.java +++ b/priam/src/main/java/com/netflix/priam/cluster/management/Flush.java @@ -18,7 +18,6 @@ import com.netflix.priam.merics.NodeToolFlushMeasurement; import com.netflix.priam.scheduler.CronTimer; import com.netflix.priam.scheduler.TaskTimer; -import com.netflix.priam.scheduler.UnsupportedTypeException; import java.io.IOException; import java.util.ArrayList; import java.util.List; @@ -105,56 +104,12 @@ private void deriveKeyspaces() throws Exception { * Timer to be used for flush interval. * * @param config {@link IConfiguration} to get configuration details from priam. - * @return the timer to be used for flush interval. - *

    If {@link IConfiguration#getFlushSchedulerType()} is {@link - * com.netflix.priam.scheduler.SchedulerType#HOUR} then it expects {@link - * IConfiguration#getFlushInterval()} in the format of hour=x or daily=x - *

    If {@link IConfiguration#getFlushSchedulerType()} is {@link - * com.netflix.priam.scheduler.SchedulerType#CRON} then it expects a valid CRON expression - * from {@link IConfiguration#getFlushCronExpression()} - * @throws Exception if the configurations are wrong. .e.g invalid cron expression. + * @return the timer to be used for compaction interval from {@link + * IConfiguration#getFlushCronExpression()} + * @throws Exception If the cron expression is invalid. */ public static TaskTimer getTimer(IConfiguration config) throws Exception { - CronTimer cronTimer = null; - switch (config.getFlushSchedulerType()) { - case HOUR: - String timerVal = config.getFlushInterval(); // e.g. hour=0 or daily=10 - if (timerVal == null) return null; - String s[] = timerVal.split("="); - if (s.length != 2) { - throw new IllegalArgumentException( - "Flush interval format is invalid. Expecting name=value, received: " - + timerVal); - } - String name = s[0].toUpperCase(); - Integer time = new Integer(s[1]); - switch (name) { - case "HOUR": - cronTimer = - new CronTimer( - Task.FLUSH.name(), time, 0); // minute, sec after each hour - break; - case "DAILY": - cronTimer = - new CronTimer( - Task.FLUSH.name(), - time, - 0, - 0); // hour, minute, sec to run on a daily basis - break; - default: - throw new UnsupportedTypeException( - "Flush interval type is invalid. Expecting \"hour, daily\", received: " - + name); - } - - break; - case CRON: - cronTimer = - CronTimer.getCronTimer(Task.FLUSH.name(), config.getFlushCronExpression()); - break; - } - return cronTimer; + return CronTimer.getCronTimer(Task.FLUSH.name(), config.getFlushCronExpression()); } } diff --git a/priam/src/main/java/com/netflix/priam/config/IConfiguration.java b/priam/src/main/java/com/netflix/priam/config/IConfiguration.java index d473ac3c5..5fdd0be8f 100644 --- a/priam/src/main/java/com/netflix/priam/config/IConfiguration.java +++ b/priam/src/main/java/com/netflix/priam/config/IConfiguration.java @@ -19,7 +19,6 @@ import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.inject.ImplementedBy; -import com.netflix.priam.scheduler.SchedulerType; import com.netflix.priam.scheduler.UnsupportedTypeException; import com.netflix.priam.tuner.GCType; import java.io.File; @@ -297,16 +296,6 @@ default String getCompactionExcludeCFList() { return null; } - /** - * @return Backup hour for snapshot backups (0 - 23) - * @deprecated Use the {{@link #getBackupCronExpression()}} instead. Scheduled for deletion in - * Dec 2018. - */ - @Deprecated - default int getBackupHour() { - return 12; - } - /** * Cron expression to be used for snapshot backups. * @@ -319,18 +308,6 @@ default String getBackupCronExpression() { return "0 0 12 1/1 * ? *"; } - /** - * Backup scheduler type to use for backup. - * - * @return Type of scheduler to use for backup. Note the default is TIMER based i.e. to use - * {@link #getBackupHour()}. If value of "CRON" is provided it starts using {@link - * #getBackupCronExpression()}. - * @throws UnsupportedTypeException if the scheduler type is not CRON/HOUR. - */ - default SchedulerType getBackupSchedulerType() throws UnsupportedTypeException { - return SchedulerType.HOUR; - } - /** * Column Family(ies), comma delimited, to include during snapshot backup. Note 1: The expected * format is keyspace.cfname. If no value is provided then snapshot contains all KS,CF(s) Note @@ -895,31 +872,6 @@ default String getFlushKeyspaces() { return StringUtils.EMPTY; } - /** - * Interval to be used for flush. - * - * @return the interval to run the flush task. Format is name=value where “name” is an enum of - * hour, daily, value is ... - * @deprecated Use the {{@link #getFlushCronExpression()} instead. This is set for deletion in - * Dec 2018. - */ - @Deprecated - default String getFlushInterval() { - return null; - } - - /** - * Scheduler type to use for flush. Default: HOUR. - * - * @return Type of scheduler to use for flush. Note the default is TIMER based i.e. to use - * {@link #getFlushInterval()}. If value of "CRON" is provided it starts using {@link - * #getFlushCronExpression()}. - * @throws UnsupportedTypeException if the scheduler type is not HOUR/CRON. - */ - default SchedulerType getFlushSchedulerType() throws UnsupportedTypeException { - return SchedulerType.HOUR; - } - /** * Cron expression to be used for flush. Use "-1" to disable the CRON. Default: -1 * diff --git a/priam/src/main/java/com/netflix/priam/config/PriamConfiguration.java b/priam/src/main/java/com/netflix/priam/config/PriamConfiguration.java index 24a356e01..242dc0861 100644 --- a/priam/src/main/java/com/netflix/priam/config/PriamConfiguration.java +++ b/priam/src/main/java/com/netflix/priam/config/PriamConfiguration.java @@ -21,7 +21,6 @@ import com.google.inject.Singleton; import com.netflix.priam.configSource.IConfigSource; import com.netflix.priam.identity.config.InstanceInfo; -import com.netflix.priam.scheduler.SchedulerType; import com.netflix.priam.scheduler.UnsupportedTypeException; import com.netflix.priam.tuner.GCType; import java.io.File; @@ -213,24 +212,11 @@ public String getMaxDirectMemory() { (PRIAM_PRE + ".direct.memory.size.") + instanceInfo.getInstanceType(), "50G"); } - @Override - public int getBackupHour() { - return config.get(PRIAM_PRE + ".backup.hour", 12); - } - @Override public String getBackupCronExpression() { return config.get(PRIAM_PRE + ".backup.cron", "0 0 12 1/1 * ? *"); // Backup daily at 12 } - @Override - public SchedulerType getBackupSchedulerType() throws UnsupportedTypeException { - String schedulerType = - config.get( - PRIAM_PRE + ".backup.schedule.type", SchedulerType.HOUR.getSchedulerType()); - return SchedulerType.lookup(schedulerType); - } - @Override public GCType getGCType() throws UnsupportedTypeException { String gcType = config.get(PRIAM_PRE + ".gc.type", GCType.CMS.getGcType()); @@ -247,14 +233,6 @@ public String getJVMUpsertSet() { return config.get(PRIAM_PRE + ".jvm.options.upsert"); } - @Override - public SchedulerType getFlushSchedulerType() throws UnsupportedTypeException { - String schedulerType = - config.get( - PRIAM_PRE + ".flush.schedule.type", SchedulerType.HOUR.getSchedulerType()); - return SchedulerType.lookup(schedulerType); - } - @Override public String getFlushCronExpression() { return config.get(PRIAM_PRE + ".flush.cron", "-1"); @@ -694,11 +672,6 @@ public String getFlushKeyspaces() { return config.get(PRIAM_PRE + ".flush.keyspaces"); } - @Override - public String getFlushInterval() { - return config.get(PRIAM_PRE + ".flush.interval"); - } - @Override public String getBackupStatusFileLoc() { return config.get( diff --git a/priam/src/main/java/com/netflix/priam/scheduler/SchedulerType.java b/priam/src/main/java/com/netflix/priam/scheduler/SchedulerType.java deleted file mode 100644 index 0663f5455..000000000 --- a/priam/src/main/java/com/netflix/priam/scheduler/SchedulerType.java +++ /dev/null @@ -1,103 +0,0 @@ -/* - * Copyright 2013 Netflix, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ -package com.netflix.priam.scheduler; - -import org.apache.commons.lang3.StringUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** Created by aagrawal on 3/8/17. */ -public enum SchedulerType { - HOUR("HOUR"), - CRON("CRON"); - - private static final Logger logger = LoggerFactory.getLogger(SchedulerType.class); - private final String schedulerType; - - SchedulerType(String schedulerType) { - this.schedulerType = schedulerType.toUpperCase(); - } - - /* - * Helper method to find the scheduler type - case insensitive as user may put value which are not right case. - * This returns the ScheulerType if one is found. Refer to table below to understand the use-case. - * - * SchedulerTypeValue|acceptNullorEmpty|acceptIllegalValue|Result - * Valid value |NA |NA |SchedulerType - * Empty string |True |NA |NULL - * NULL |True |NA |NULL - * Empty string |False |NA |UnsupportedTypeException - * NULL |False |NA |UnsupportedTypeException - * Illegal value |NA |True |NULL - * Illegal value |NA |False |UnsupportedTypeException - */ - - public static SchedulerType lookup( - String schedulerType, boolean acceptNullOrEmpty, boolean acceptIllegalValue) - throws UnsupportedTypeException { - if (StringUtils.isEmpty(schedulerType)) - if (acceptNullOrEmpty) return null; - else { - String message = - String.format( - "%s is not a supported SchedulerType. Supported values are %s", - schedulerType, getSupportedValues()); - logger.error(message); - throw new UnsupportedTypeException(message); - } - - try { - return SchedulerType.valueOf(schedulerType.toUpperCase()); - } catch (IllegalArgumentException ex) { - String message = - String.format( - "%s is not a supported SchedulerType. Supported values are %s", - schedulerType, getSupportedValues()); - - if (acceptIllegalValue) { - message = - message - + ". Since acceptIllegalValue is set to True, returning NULL instead."; - logger.error(message); - return null; - } - - logger.error(message); - throw new UnsupportedTypeException(message, ex); - } - } - - private static String getSupportedValues() { - StringBuilder supportedValues = new StringBuilder(); - boolean first = true; - for (SchedulerType type : SchedulerType.values()) { - if (!first) supportedValues.append(","); - supportedValues.append(type); - first = false; - } - - return supportedValues.toString(); - } - - public static SchedulerType lookup(String schedulerType) throws UnsupportedTypeException { - return lookup(schedulerType, false, false); - } - - public String getSchedulerType() { - return schedulerType; - } -} diff --git a/priam/src/test/groovy/com.netflix.priam.scheduler/TestFlushTask.groovy b/priam/src/test/groovy/com.netflix.priam.scheduler/TestFlushTask.groovy deleted file mode 100644 index e03a147a9..000000000 --- a/priam/src/test/groovy/com.netflix.priam.scheduler/TestFlushTask.groovy +++ /dev/null @@ -1,82 +0,0 @@ -package com.netflix.priam.scheduler - -import com.netflix.priam.config.FakeConfiguration -import com.netflix.priam.cluster.management.Flush -import spock.lang.Specification -import spock.lang.Unroll - -/** - Created by aagrawal on 7/15/17. - */ -@Unroll -class TestFlushTask extends Specification { - - def "Exception for value #flushSchedulerType, #flushCronExpression, #flushInterval"() { - when: - Flush.getTimer(new FlushConfiguration(flushSchedulerType, flushCronExpression, flushInterval)) - - then: - thrown(expectedException) - - where: - flushSchedulerType | flushCronExpression | flushInterval || expectedException - "sdf" | null | null || UnsupportedTypeException - "hour" | null | "2" || IllegalArgumentException - "hour" | "0 0 2 * * ?" | "2" || IllegalArgumentException - "cron" | "abc" | null || IllegalArgumentException - "cron" | "abc" | "daily=2" || IllegalArgumentException - "cron" | null | "daily=2" || IllegalArgumentException - "hour" | null | "hour=2,daily=2" || IllegalArgumentException - } - - def "SchedulerType for value #flushSchedulerType, #flushCronExpression, #flushInterval is null"() { - expect: - Flush.getTimer(new FlushConfiguration(flushSchedulerType, flushCronExpression, flushInterval)) == result - - where: - flushSchedulerType | flushCronExpression | flushInterval || result - "hour" | null | null || null - "cron" | "-1" | null || null - "hour" | "abc" | null || null - "cron" | "-1" | "abc" || null - } - - def "SchedulerType for value #flushSchedulerType, #flushCronExpression, #flushInterval is #result"() { - expect: - Flush.getTimer(new FlushConfiguration(flushSchedulerType, flushCronExpression, flushInterval)).getCronExpression() == result - - where: - flushSchedulerType | flushCronExpression | flushInterval || result - "hour" | null | "daily=2" || "0 0 2 * * ?" - "hour" | null | "hour=2" || "0 2 0/1 * * ?" - "cron" | "0 0 0/1 1/1 * ? *" | null || "0 0 0/1 1/1 * ? *" - "cron" | "0 0 0/1 1/1 * ? *" | "daily=2" || "0 0 0/1 1/1 * ? *" - } - - - private class FlushConfiguration extends FakeConfiguration { - private String flushSchedulerType, flushCronExpression, flushInterval - - FlushConfiguration(String flushSchedulerType, String flushCronExpression, String flushInterval) { - this.flushCronExpression = flushCronExpression - this.flushSchedulerType = flushSchedulerType - this.flushInterval = flushInterval - } - - @Override - SchedulerType getFlushSchedulerType() throws UnsupportedTypeException { - return SchedulerType.lookup(flushSchedulerType) - } - - @Override - String getFlushCronExpression() { - return flushCronExpression - } - - @Override - String getFlushInterval() { - return flushInterval - } - } - -} diff --git a/priam/src/test/groovy/com.netflix.priam.scheduler/TestSchedulerType.groovy b/priam/src/test/groovy/com.netflix.priam.scheduler/TestSchedulerType.groovy deleted file mode 100644 index 2c8429045..000000000 --- a/priam/src/test/groovy/com.netflix.priam.scheduler/TestSchedulerType.groovy +++ /dev/null @@ -1,52 +0,0 @@ -package com.netflix.priam.scheduler - -/** - * Created by aagrawal on 3/16/17. - * This is used to test SchedulerType with all the values you might get. - */ -import spock.lang.* - -@Unroll -class TestSchedulerType extends Specification{ - - def "Exception for value #schedulerType , #acceptNullorEmpty , #acceptIllegalValue"() { - when: - SchedulerType.lookup(schedulerType, acceptNullorEmpty, acceptIllegalValue) - - then: - thrown(expectedException) - - where: - schedulerType | acceptNullorEmpty | acceptIllegalValue || expectedException - "sdf" | true | false || UnsupportedTypeException - "" | false | true || UnsupportedTypeException - null | false | true || UnsupportedTypeException - - } - - def "SchedulerType for value #schedulerType , #acceptNullorEmpty , #acceptIllegalValue is #result"() { - expect: - SchedulerType.lookup(schedulerType, acceptNullorEmpty, acceptIllegalValue) == result - - where: - schedulerType | acceptNullorEmpty | acceptIllegalValue || result - "hour" | true | true || SchedulerType.HOUR - "Hour" | true | true || SchedulerType.HOUR - "HOUR" | true | true || SchedulerType.HOUR - "hour" | true | false || SchedulerType.HOUR - "Hour" | true | false || SchedulerType.HOUR - "HOUR" | true | false || SchedulerType.HOUR - "hour" | false | false || SchedulerType.HOUR - "Hour" | false | false || SchedulerType.HOUR - "HOUR" | false | false || SchedulerType.HOUR - "hour" | false | true || SchedulerType.HOUR - "Hour" | false | true || SchedulerType.HOUR - "HOUR" | false | true || SchedulerType.HOUR - "" | true | false || null - null | true | false || null - "sdf" | false | true || null - "sdf" | true | true || null - } - - -} \ No newline at end of file diff --git a/priam/src/test/groovy/com.netflix.priam/backup/TestBackupScheduler.groovy b/priam/src/test/groovy/com.netflix.priam/backup/TestBackupScheduler.groovy index 45c8f89e0..22a391a68 100644 --- a/priam/src/test/groovy/com.netflix.priam/backup/TestBackupScheduler.groovy +++ b/priam/src/test/groovy/com.netflix.priam/backup/TestBackupScheduler.groovy @@ -1,8 +1,6 @@ package com.netflix.priam.backup import com.netflix.priam.config.FakeConfiguration -import com.netflix.priam.scheduler.SchedulerType -import com.netflix.priam.scheduler.UnsupportedTypeException import spock.lang.Specification import spock.lang.Unroll @@ -11,23 +9,19 @@ import spock.lang.Unroll */ @Unroll class TestBackupScheduler extends Specification { - def "IsBackupEnabled for SchedulerType #schedulerType with hour #configHour and CRON #configCRON is #result"() { + def "IsBackupEnabled CRON #configCRON is #result"() { expect: - SnapshotBackup.isBackupEnabled(new BackupConfiguration(schedulerType, configCRON, configHour)) == result + SnapshotBackup.isBackupEnabled(new BackupConfiguration(configCRON)) == result where: - schedulerType | configCRON | configHour || result - "hour" | null | -1 || false - "hour" | "0 0 9 1/1 * ? *" | -1 || false - "hour" | null | 1 || true - "cron" | "-1" | 1 || false - "cron" | "-1" | -1 || false - "cron" | "0 0 9 1/1 * ? *" | -1 || true + configCRON || result + "-1" || false + "0 0 9 1/1 * ? *" || true } def "Exception for illegal value of Snapshot CRON expression , #configCRON"() { when: - SnapshotBackup.isBackupEnabled(new BackupConfiguration("cron", configCRON, 1)) + SnapshotBackup.isBackupEnabled(new BackupConfiguration(configCRON)) then: thrown(expectedException) @@ -38,40 +32,26 @@ class TestBackupScheduler extends Specification { "0 9 1/1 * ? *"|| Exception } - def "Validate CRON for backup for SchedulerType #schedulerType with hour #configHour and CRON #configCRON is #result"() { + def "Validate CRON for backup CRON #configCRON is #result"() { expect: - SnapshotBackup.getTimer(new BackupConfiguration(schedulerType, configCRON, configHour)).cronExpression == result + SnapshotBackup.getTimer(new BackupConfiguration(configCRON)).cronExpression == result where: - schedulerType | configCRON | configHour || result - "hour" | null | 1 || "0 1 1 * * ?" - "cron" | "0 0 9 1/1 * ? *" | -1 || "0 0 9 1/1 * ? *" + configCRON || result + "0 0 9 1/1 * ? *" || "0 0 9 1/1 * ? *" } private class BackupConfiguration extends FakeConfiguration { - private String backupSchedulerType, backupCronExpression - private int backupHour + private String backupCronExpression - BackupConfiguration(String backupSchedulerType, String backupCronExpression, int backupHour) { + BackupConfiguration(String backupCronExpression) { this.backupCronExpression = backupCronExpression - this.backupSchedulerType = backupSchedulerType - this.backupHour = backupHour - } - - @Override - SchedulerType getBackupSchedulerType() throws UnsupportedTypeException { - return SchedulerType.lookup(backupSchedulerType) } @Override String getBackupCronExpression() { return backupCronExpression } - - @Override - int getBackupHour() { - return backupHour - } } } diff --git a/priam/src/test/groovy/com/netflix/priam/cluser/management/TestFlushTask.groovy b/priam/src/test/groovy/com/netflix/priam/cluser/management/TestFlushTask.groovy new file mode 100644 index 000000000..85a6ec12d --- /dev/null +++ b/priam/src/test/groovy/com/netflix/priam/cluser/management/TestFlushTask.groovy @@ -0,0 +1,80 @@ +/* + * Copyright 2018 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.netflix.priam.cluser.management + +import com.google.inject.Guice +import com.netflix.priam.backup.BRTestModule +import com.netflix.priam.cluster.management.Compaction +import com.netflix.priam.cluster.management.Flush +import com.netflix.priam.config.FakeConfiguration +import com.netflix.priam.defaultimpl.CassandraOperations +import mockit.Mock +import mockit.MockUp +import spock.lang.Shared +import spock.lang.Specification +import spock.lang.Unroll + +/** + Created by aagrawal on 7/15/17. + */ +@Unroll +class TestFlushTask extends Specification { + def "Exception for value #flushCronExpression"() { + when: + Flush.getTimer(new FlushConfiguration(flushCronExpression)) + + then: + thrown(expectedException) + + where: + flushCronExpression || expectedException + "abc" || IllegalArgumentException + null || IllegalArgumentException + } + + def "SchedulerType for value #flushSchedulerType, #flushCronExpression, #flushInterval is null"() { + expect: + Flush.getTimer(new FlushConfiguration(flushCronExpression)) == result + + where: + flushCronExpression || result + "-1" || null + } + + def "SchedulerType for value #flushCronExpression is #result"() { + expect: + Flush.getTimer(new FlushConfiguration(flushCronExpression)).getCronExpression() == result + + where: + flushCronExpression || result + "0 0 0/1 1/1 * ? *" || "0 0 0/1 1/1 * ? *" + } + + private class FlushConfiguration extends FakeConfiguration { + private String flushCronExpression + + FlushConfiguration(String flushCronExpression) { + this.flushCronExpression = flushCronExpression + } + + @Override + String getFlushCronExpression() { + return flushCronExpression + } + } +} \ No newline at end of file diff --git a/priam/src/test/java/com/netflix/priam/backup/TestCompression.java b/priam/src/test/java/com/netflix/priam/backup/TestCompression.java index 4de6c36f3..63fdb0dbd 100644 --- a/priam/src/test/java/com/netflix/priam/backup/TestCompression.java +++ b/priam/src/test/java/com/netflix/priam/backup/TestCompression.java @@ -20,6 +20,7 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; +import com.netflix.priam.compress.ICompression; import com.netflix.priam.compress.SnappyCompression; import com.netflix.priam.utils.SystemUtils; import java.io.*; @@ -106,11 +107,16 @@ public void zipTest() throws IOException { @Test public void snappyTest() throws IOException { - SnappyCompression compress = new SnappyCompression(); - File compressedOutputFile = new File("/tmp/test1.snp"); + ICompression compress = new SnappyCompression(); + testCompressor(compress); + } + + private void testCompressor(ICompression compress) throws IOException { + File compressedOutputFile = new File("/tmp/test1.compress"); File decompressedTempOutput = new File("/tmp/compress-test-out.txt"); long chunkSize = 5L * 1024 * 1024; try { + Iterator it = compress.compress(new FileInputStream(randomContentFile), chunkSize); try (FileOutputStream ostream = new FileOutputStream(compressedOutputFile)) { @@ -118,6 +124,7 @@ public void snappyTest() throws IOException { byte[] chunk = it.next(); ostream.write(chunk); } + ostream.flush(); } assertTrue(randomContentFile.length() > compressedOutputFile.length()); From bb140989e42e947616d1bb833a3d4ffee339491f Mon Sep 17 00:00:00 2001 From: Arun Agrawal Date: Fri, 28 Dec 2018 09:59:37 -0800 Subject: [PATCH 044/228] Backup Verification refactor --- .../priam/backup/BackupVerification.java | 92 ++----------------- .../backup/BackupVerificationResult.java | 15 ++- .../priam/backupv2/BackupValidator.java | 71 ++------------ .../netflix/priam/backupv2/IMetaProxy.java | 11 +++ .../netflix/priam/backupv2/MetaV1Proxy.java | 73 ++++++++++++++- .../netflix/priam/backupv2/MetaV2Proxy.java | 60 +++++++++++- .../priam/resources/BackupServlet.java | 50 ++++------ .../priam/restore/AbstractRestore.java | 4 +- .../priam/services/BackupTTLService.java | 5 + ...kupValidator.java => TestMetaV2Proxy.java} | 25 +++-- 10 files changed, 199 insertions(+), 207 deletions(-) rename priam/src/test/java/com/netflix/priam/backupv2/{TestBackupValidator.java => TestMetaV2Proxy.java} (89%) diff --git a/priam/src/main/java/com/netflix/priam/backup/BackupVerification.java b/priam/src/main/java/com/netflix/priam/backup/BackupVerification.java index 196edd228..8e7bd771e 100644 --- a/priam/src/main/java/com/netflix/priam/backup/BackupVerification.java +++ b/priam/src/main/java/com/netflix/priam/backup/BackupVerification.java @@ -18,13 +18,10 @@ import com.google.inject.Singleton; import com.google.inject.name.Named; import com.netflix.priam.backupv2.IMetaProxy; -import com.netflix.priam.config.IConfiguration; -import com.netflix.priam.utils.DateUtil; import java.nio.file.Path; +import java.nio.file.Paths; import java.util.*; import java.util.stream.Collectors; -import org.apache.commons.collections4.CollectionUtils; -import org.apache.commons.io.FileUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -37,19 +34,13 @@ public class BackupVerification { private static final Logger logger = LoggerFactory.getLogger(BackupVerification.class); - private final IBackupFileSystem bkpStatusFs; - private final IConfiguration config; private final IMetaProxy metaProxy; private final Provider abstractBackupPathProvider; @Inject BackupVerification( - @Named("backup") IBackupFileSystem bkpStatusFs, - IConfiguration config, @Named("v1") IMetaProxy metaProxy, Provider abstractBackupPathProvider) { - this.bkpStatusFs = bkpStatusFs; - this.config = config; this.metaProxy = metaProxy; this.abstractBackupPathProvider = abstractBackupPathProvider; } @@ -63,85 +54,20 @@ public Optional getLatestBackupMetaData(List met return metadata.stream().findFirst(); } - public BackupVerificationResult verifyBackup(List metadata) throws Exception { - BackupVerificationResult result = new BackupVerificationResult(); - - if (metadata == null || metadata.isEmpty()) return result; - - result.snapshotAvailable = true; - // All the dates should be same. - result.selectedDate = metadata.get(0).getSnapshotDate(); + public Optional verifyBackup(List metadata) { + if (metadata == null || metadata.isEmpty()) return null; Optional latestBackupMetaData = getLatestBackupMetaData(metadata); if (!latestBackupMetaData.isPresent()) { logger.error("No backup found which finished during the time provided."); - return result; - } - - result.snapshotTime = DateUtil.formatyyyyMMddHHmm(latestBackupMetaData.get().getStart()); - logger.info( - "Latest/Requested snapshot date found: {}, for selected/provided date: {}", - result.snapshotTime, - result.selectedDate); - - // Get Backup File Iterator - String prefix = config.getBackupPrefix(); - Date strippedMsSnapshotTime = DateUtil.getDate(result.snapshotTime); - Iterator backupfiles = - bkpStatusFs.list(prefix, strippedMsSnapshotTime, strippedMsSnapshotTime); - - // Return validation fail if backup filesystem listing failed. - if (!backupfiles.hasNext()) { - logger.warn( - "ERROR: No files available while doing backup filesystem listing. Declaring the verification failed."); - return result; - } - - // Do remote file listing - result.backupFileListAvail = true; - List metas = new LinkedList<>(); - List remoteListing = new ArrayList<>(); - - while (backupfiles.hasNext()) { - AbstractBackupPath path = backupfiles.next(); - if (path.getType() == AbstractBackupPath.BackupFileType.META) metas.add(path); - else remoteListing.add(path.getRemotePath()); + return null; } - if (metas.size() == 0) { - logger.error( - "Manifest file not found on remote file system for: {}", result.snapshotTime); - return result; - } - - result.metaFileFound = true; - - // Download meta.json from backup location. - Path localMetaPath = metaProxy.downloadMetaFile(metas.get(0)); - List metaFileList = metaProxy.getSSTFilesFromMeta(localMetaPath); - FileUtils.deleteQuietly(localMetaPath.toFile()); - - if (metaFileList.isEmpty() && remoteListing.isEmpty()) { - logger.info( - "Uncommon Scenario: Both meta file and backup filesystem listing is empty. Considering this as success"); - result.valid = true; - return result; - } - - // Atleast meta file or s3 listing contains some file. - - result.filesMatched = - (ArrayList) CollectionUtils.intersection(metaFileList, remoteListing); - result.filesInS3Only = remoteListing; - result.filesInS3Only.removeAll(result.filesMatched); - result.filesInMetaOnly = metaFileList; - result.filesInMetaOnly.removeAll(result.filesMatched); - - // There could be a scenario that backupfilesystem has more files than meta file. e.g. some - // leftover objects - if (result.filesInMetaOnly.size() == 0) result.valid = true; - - return result; + Path metadataLocation = Paths.get(latestBackupMetaData.get().getSnapshotLocation()); + metadataLocation = metadataLocation.subpath(1, metadataLocation.getNameCount()); + AbstractBackupPath abstractBackupPath = abstractBackupPathProvider.get(); + abstractBackupPath.parseRemote(metadataLocation.toString()); + return Optional.of((metaProxy.isMetaFileValid(abstractBackupPath))); } } diff --git a/priam/src/main/java/com/netflix/priam/backup/BackupVerificationResult.java b/priam/src/main/java/com/netflix/priam/backup/BackupVerificationResult.java index 26933bf1b..1076eb66a 100644 --- a/priam/src/main/java/com/netflix/priam/backup/BackupVerificationResult.java +++ b/priam/src/main/java/com/netflix/priam/backup/BackupVerificationResult.java @@ -14,6 +14,8 @@ package com.netflix.priam.backup; import com.netflix.priam.utils.GsonJsonSerializer; +import java.time.Instant; +import java.util.ArrayList; import java.util.List; /** @@ -21,15 +23,12 @@ * are all null and false. */ public class BackupVerificationResult { - public boolean snapshotAvailable = false; public boolean valid = false; - public boolean metaFileFound = false; - public boolean backupFileListAvail = false; - public String selectedDate = null; - public String snapshotTime = null; - public List filesInMetaOnly = null; - public List filesInS3Only = null; - public List filesMatched = null; + public String remotePath = null; + public Instant snapshotInstant = null; + public boolean manifestAvailable = false; + public List filesInMetaOnly = new ArrayList<>(); + public int filesMatched = 0; @Override public String toString() { diff --git a/priam/src/main/java/com/netflix/priam/backupv2/BackupValidator.java b/priam/src/main/java/com/netflix/priam/backupv2/BackupValidator.java index 1653ff571..bd6b332cf 100644 --- a/priam/src/main/java/com/netflix/priam/backupv2/BackupValidator.java +++ b/priam/src/main/java/com/netflix/priam/backupv2/BackupValidator.java @@ -20,14 +20,10 @@ import com.netflix.priam.backup.*; import com.netflix.priam.config.IConfiguration; import com.netflix.priam.utils.DateUtil; -import java.io.FileNotFoundException; -import java.io.IOException; -import java.nio.file.Path; -import java.nio.file.Paths; import java.util.List; +import java.util.Optional; import javax.inject.Inject; import javax.inject.Named; -import org.apache.commons.io.FileUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -39,7 +35,6 @@ public class BackupValidator { private static final Logger logger = LoggerFactory.getLogger(BackupVerification.class); private final IBackupFileSystem fs; private IMetaProxy metaProxy; - private boolean isBackupValid; @Inject public BackupValidator( @@ -55,72 +50,22 @@ public BackupValidator( * file via AbstractBackupPath object. * * @param dateRange the time period to scan in the remote file system for meta files. - * @return the AbstractBackupPath denoting the "local" file which is valid or null. Caller needs - * to delete the file. + * @return the BackupVerificationResult containing the details of the valid meta file. If none + * is found, null is returned. * @throws BackupRestoreException if there is issue contacting remote file system, fetching the * file etc. */ - public AbstractBackupPath findLatestValidMetaFile(DateUtil.DateRange dateRange) + public Optional findLatestValidMetaFile(DateUtil.DateRange dateRange) throws BackupRestoreException { List metas = metaProxy.findMetaFiles(dateRange); logger.info("Meta files found: {}", metas); for (AbstractBackupPath meta : metas) { - Path localFile = metaProxy.downloadMetaFile(meta); - boolean isValid = isMetaFileValid(localFile); - logger.info("Meta: {}, isValid: {}", meta, isValid); - if (!isValid) FileUtils.deleteQuietly(localFile.toFile()); - else return meta; + BackupVerificationResult result = metaProxy.isMetaFileValid(meta); + logger.info("BackupVerificationResult: {}", result); + if (result.valid) return Optional.of(result); } - return null; - } - - /** - * Validate that all the files mentioned in the meta file actually exists on remote file system. - * - * @param metaFile Path to the local uncompressed/unencrypted meta file - * @return true if all the files mentioned in meta file are present on remote file system. It - * will return false in case of any error. - */ - public boolean isMetaFileValid(Path metaFile) { - try { - isBackupValid = true; - new MetaFileBackupValidator().readMeta(metaFile); - } catch (FileNotFoundException fne) { - isBackupValid = false; - logger.error(fne.getLocalizedMessage()); - } catch (IOException ioe) { - isBackupValid = false; - logger.error( - "IO Error while processing meta file: " + metaFile, ioe.getLocalizedMessage()); - ioe.printStackTrace(); - } - return isBackupValid; - } - - private class MetaFileBackupValidator extends MetaFileReader { - @Override - public void process(ColumnfamilyResult columnfamilyResult) { - for (ColumnfamilyResult.SSTableResult ssTableResult : - columnfamilyResult.getSstables()) { - for (FileUploadResult fileUploadResult : ssTableResult.getSstableComponents()) { - if (!isBackupValid) { - break; - } - - try { - isBackupValid = - isBackupValid - && fs.doesRemoteFileExist( - Paths.get(fileUploadResult.getBackupPath())); - } catch (BackupRestoreException e) { - // For any error, mark that file is not available. - isBackupValid = false; - break; - } - } - } - } + return Optional.empty(); } } diff --git a/priam/src/main/java/com/netflix/priam/backupv2/IMetaProxy.java b/priam/src/main/java/com/netflix/priam/backupv2/IMetaProxy.java index b9a1ad255..4c173b599 100644 --- a/priam/src/main/java/com/netflix/priam/backupv2/IMetaProxy.java +++ b/priam/src/main/java/com/netflix/priam/backupv2/IMetaProxy.java @@ -19,6 +19,7 @@ import com.netflix.priam.backup.AbstractBackupPath; import com.netflix.priam.backup.BackupRestoreException; +import com.netflix.priam.backup.BackupVerificationResult; import com.netflix.priam.utils.DateUtil; import java.nio.file.Path; import java.util.List; @@ -70,6 +71,16 @@ public interface IMetaProxy { */ List getSSTFilesFromMeta(Path localMetaPath) throws Exception; + /** + * Validate that all the files mentioned in the meta file actually exists on remote file system. + * + * @param metaBackupPath Path to the remote meta file. + * @return backupVerificationResult containing the information like valid - if all the files + * mentioned in meta file are present on remote file system. It will return false in case of + * any error. + */ + BackupVerificationResult isMetaFileValid(AbstractBackupPath metaBackupPath); + /** Delete the old meta files, if any present in the metaFileDirectory */ void cleanupOldMetaFiles(); } diff --git a/priam/src/main/java/com/netflix/priam/backupv2/MetaV1Proxy.java b/priam/src/main/java/com/netflix/priam/backupv2/MetaV1Proxy.java index 38827a235..3c71101c6 100644 --- a/priam/src/main/java/com/netflix/priam/backupv2/MetaV1Proxy.java +++ b/priam/src/main/java/com/netflix/priam/backupv2/MetaV1Proxy.java @@ -19,17 +19,17 @@ import com.google.common.collect.Lists; import com.google.inject.Inject; -import com.netflix.priam.backup.AbstractBackupPath; -import com.netflix.priam.backup.BackupRestoreException; -import com.netflix.priam.backup.IBackupFileSystem; -import com.netflix.priam.backup.IFileSystemContext; +import com.netflix.priam.backup.*; import com.netflix.priam.config.IConfiguration; import com.netflix.priam.utils.DateUtil; import java.io.FileReader; import java.nio.file.InvalidPathException; import java.nio.file.Path; import java.nio.file.Paths; +import java.time.temporal.ChronoUnit; import java.util.*; +import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.io.FileUtils; import org.json.simple.parser.JSONParser; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -38,9 +38,11 @@ public class MetaV1Proxy implements IMetaProxy { private static final Logger logger = LoggerFactory.getLogger(MetaV1Proxy.class); private final IBackupFileSystem fs; + private final IConfiguration configuration; @Inject MetaV1Proxy(IConfiguration configuration, IFileSystemContext backupFileSystemCtx) { + this.configuration = configuration; fs = backupFileSystemCtx.getFileStrategy(configuration); } @@ -69,7 +71,7 @@ public List findMetaFiles(DateUtil.DateRange dateRange) { if (path.getType() == AbstractBackupPath.BackupFileType.META) // Since there are now meta file for incrementals as well as snapshot, we need to // find the correct one (i.e. the snapshot meta file (meta.json)) - if (path.getFileName().equalsIgnoreCase("meta.json")) { + if (path.getType() == AbstractBackupPath.BackupFileType.META) { metas.add(path); } } @@ -85,6 +87,67 @@ public List findMetaFiles(DateUtil.DateRange dateRange) { return metas; } + @Override + public BackupVerificationResult isMetaFileValid(AbstractBackupPath metaBackupPath) { + BackupVerificationResult result = new BackupVerificationResult(); + result.remotePath = metaBackupPath.getRemotePath(); + result.snapshotInstant = metaBackupPath.getTime().toInstant(); + + try { + // Download the meta file. + Path metaFile = downloadMetaFile(metaBackupPath); + // Read the local meta file. + List metaFileList = getSSTFilesFromMeta(metaFile); + FileUtils.deleteQuietly(metaFile.toFile()); + result.manifestAvailable = true; + + // List the remote file system to validate the backup. + String prefix = configuration.getBackupPrefix(); + Date strippedMsSnapshotTime = + new Date(result.snapshotInstant.truncatedTo(ChronoUnit.MINUTES).toEpochMilli()); + Iterator backupfiles = + fs.list(prefix, strippedMsSnapshotTime, strippedMsSnapshotTime); + + // Return validation fail if backup filesystem listing failed. + if (!backupfiles.hasNext()) { + logger.warn( + "ERROR: No files available while doing backup filesystem listing. Declaring the verification failed."); + return result; + } + + // Convert the remote listing to String. + List remoteListing = new ArrayList<>(); + while (backupfiles.hasNext()) { + AbstractBackupPath path = backupfiles.next(); + if (path.getType() == AbstractBackupPath.BackupFileType.SNAP) + remoteListing.add(path.getRemotePath()); + } + + if (metaFileList.isEmpty() && remoteListing.isEmpty()) { + logger.info( + "Uncommon Scenario: Both meta file and backup filesystem listing is empty. Considering this as success"); + result.valid = true; + return result; + } + + ArrayList filesMatched = + (ArrayList) CollectionUtils.intersection(metaFileList, remoteListing); + result.filesMatched = filesMatched.size(); + result.filesInMetaOnly = metaFileList; + result.filesInMetaOnly.removeAll(filesMatched); + + // There could be a scenario that backupfilesystem has more files than meta file. e.g. + // some leftover objects + result.valid = (result.filesInMetaOnly.isEmpty()); + } catch (Exception e) { + logger.error( + "Error while processing meta file: " + metaBackupPath, e.getLocalizedMessage()); + e.printStackTrace(); + } + + return result; + } + @Override public Path downloadMetaFile(AbstractBackupPath meta) throws BackupRestoreException { Path localFile = Paths.get(meta.newRestoreFile().getAbsolutePath() + ".download"); diff --git a/priam/src/main/java/com/netflix/priam/backupv2/MetaV2Proxy.java b/priam/src/main/java/com/netflix/priam/backupv2/MetaV2Proxy.java index 7ad048c39..fa9df765d 100644 --- a/priam/src/main/java/com/netflix/priam/backupv2/MetaV2Proxy.java +++ b/priam/src/main/java/com/netflix/priam/backupv2/MetaV2Proxy.java @@ -18,13 +18,12 @@ package com.netflix.priam.backupv2; import com.google.inject.Provider; -import com.netflix.priam.backup.AbstractBackupPath; -import com.netflix.priam.backup.BackupRestoreException; -import com.netflix.priam.backup.IBackupFileSystem; -import com.netflix.priam.backup.IFileSystemContext; +import com.netflix.priam.backup.*; import com.netflix.priam.config.IConfiguration; import com.netflix.priam.utils.DateUtil; import java.io.File; +import java.io.FileNotFoundException; +import java.io.IOException; import java.nio.file.Path; import java.nio.file.Paths; import java.util.*; @@ -139,4 +138,57 @@ public void cleanupOldMetaFiles() { public List getSSTFilesFromMeta(Path localMetaPath) throws Exception { return null; } + + @Override + public BackupVerificationResult isMetaFileValid(AbstractBackupPath metaBackupPath) { + MetaFileBackupValidator metaFileBackupValidator = new MetaFileBackupValidator(); + BackupVerificationResult result = metaFileBackupValidator.verificationResult; + result.remotePath = metaBackupPath.getRemotePath(); + result.snapshotInstant = metaBackupPath.getLastModified(); + + Path metaFile = null; + try { + metaFile = downloadMetaFile(metaBackupPath); + result.manifestAvailable = true; + + metaFileBackupValidator.readMeta(metaFile); + result.valid = (result.filesInMetaOnly.isEmpty()); + } catch (FileNotFoundException fne) { + logger.error(fne.getLocalizedMessage()); + } catch (IOException ioe) { + logger.error( + "IO Error while processing meta file: " + metaFile, ioe.getLocalizedMessage()); + ioe.printStackTrace(); + } catch (BackupRestoreException bre) { + logger.error("Error while trying to download the manifest file: {}", metaBackupPath); + } finally { + if (metaFile != null) FileUtils.deleteQuietly(metaFile.toFile()); + } + return result; + } + + private class MetaFileBackupValidator extends MetaFileReader { + private BackupVerificationResult verificationResult = new BackupVerificationResult(); + + @Override + public void process(ColumnfamilyResult columnfamilyResult) { + for (ColumnfamilyResult.SSTableResult ssTableResult : + columnfamilyResult.getSstables()) { + for (FileUploadResult fileUploadResult : ssTableResult.getSstableComponents()) { + try { + if (fs.doesRemoteFileExist(Paths.get(fileUploadResult.getBackupPath()))) { + verificationResult.filesMatched++; + } else { + verificationResult.filesInMetaOnly.add( + fileUploadResult.getBackupPath()); + } + } catch (BackupRestoreException e) { + // For any error, mark that file is not available. + verificationResult.valid = false; + break; + } + } + } + } + } } diff --git a/priam/src/main/java/com/netflix/priam/resources/BackupServlet.java b/priam/src/main/java/com/netflix/priam/resources/BackupServlet.java index 68aa2f9b5..37b012a96 100644 --- a/priam/src/main/java/com/netflix/priam/resources/BackupServlet.java +++ b/priam/src/main/java/com/netflix/priam/resources/BackupServlet.java @@ -25,15 +25,11 @@ import com.netflix.priam.scheduler.PriamScheduler; import com.netflix.priam.utils.DateUtil; import com.netflix.priam.utils.SystemUtils; -import java.util.ArrayList; -import java.util.Date; -import java.util.Iterator; -import java.util.List; +import java.util.*; import java.util.stream.Collectors; import javax.ws.rs.*; import javax.ws.rs.core.MediaType; import javax.ws.rs.core.Response; -import org.apache.commons.io.FileUtils; import org.codehaus.jettison.json.JSONArray; import org.codehaus.jettison.json.JSONException; import org.codehaus.jettison.json.JSONObject; @@ -244,41 +240,31 @@ public Response validateSnapshotByDate(@PathParam("daterange") String daterange) throws Exception { DateUtil.DateRange dateRange = new DateUtil.DateRange(daterange); - logger.info( - "Will try to validate latest backup during startTime: {}, and endTime: {}", - dateRange.getStartTime(), - dateRange.getEndTime()); - List metadata = getLatestBackupMetadata(dateRange); - BackupVerificationResult result = backupVerification.verifyBackup(metadata); - return Response.ok(result.toString()).build(); + Optional result = backupVerification.verifyBackup(metadata); + if (!result.isPresent()) { + return Response.noContent() + .entity("No valid meta found for provided time range") + .build(); + } + + return Response.ok(result.get().toString()).build(); } @GET @Path("/validate/snapshot/v2/{daterange}") public Response validateV2SnapshotByDate(@PathParam("daterange") String daterange) throws Exception { - try { - DateUtil.DateRange dateRange = new DateUtil.DateRange(daterange); - AbstractBackupPath validMeta = backupValidator.findLatestValidMetaFile(dateRange); - if (validMeta == null) { - return Response.noContent() - .entity("No valid meta found for provided time range") - .build(); - } - - // Delete the file as we don't need it anymore. - FileUtils.deleteQuietly(validMeta.newRestoreFile()); - JSONObject jsonObject = new JSONObject(); - jsonObject.put("daterange", daterange); - jsonObject.put("remoteMetaPath", validMeta.getRemotePath()); - jsonObject.put("valid", true); - return Response.ok(jsonObject.toString()).build(); - } catch (BackupRestoreException ex) { - logger.error( - "Error while trying to fetch the latest valid meta file: {}", ex.getMessage()); - return Response.serverError().build(); + DateUtil.DateRange dateRange = new DateUtil.DateRange(daterange); + Optional result = + backupValidator.findLatestValidMetaFile(dateRange); + if (!result.isPresent()) { + return Response.noContent() + .entity("No valid meta found for provided time range") + .build(); } + + return Response.ok(result.get().toString()).build(); } /* diff --git a/priam/src/main/java/com/netflix/priam/restore/AbstractRestore.java b/priam/src/main/java/com/netflix/priam/restore/AbstractRestore.java index 3850d7432..8e6d3a338 100644 --- a/priam/src/main/java/com/netflix/priam/restore/AbstractRestore.java +++ b/priam/src/main/java/com/netflix/priam/restore/AbstractRestore.java @@ -248,12 +248,12 @@ public void restore(DateUtil.DateRange dateRange) throws Exception { // TODO: Validate that manifest file is valid. // Download the meta.json file. Path metaFile = metaV1Proxy.downloadMetaFile(meta); - - List> futureList = new ArrayList<>(); // Parse meta.json file to find the files required to download from this snapshot. List snapshots = metaData.toJson(metaFile.toFile()); + FileUtils.deleteQuietly(metaFile.toFile()); // Download snapshot which is listed in the meta file. + List> futureList = new ArrayList<>(); futureList.addAll(download(snapshots.iterator(), BackupFileType.SNAP, false)); logger.info("Downloading incrementals"); diff --git a/priam/src/main/java/com/netflix/priam/services/BackupTTLService.java b/priam/src/main/java/com/netflix/priam/services/BackupTTLService.java index 74dfca4a0..a3170255e 100644 --- a/priam/src/main/java/com/netflix/priam/services/BackupTTLService.java +++ b/priam/src/main/java/com/netflix/priam/services/BackupTTLService.java @@ -41,6 +41,7 @@ import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; import javax.inject.Named; +import org.apache.commons.io.FileUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -128,9 +129,13 @@ public void execute() throws Exception { // Walk over the file system iterator and if not in map, it is eligible for delete. new MetaFileWalker().readMeta(localFile); + if (logger.isDebugEnabled()) logger.debug("Files in meta file: {}", filesInMeta.keySet().toString()); + // Delete the meta file downloaded locally + FileUtils.deleteQuietly(localFile.toFile()); + Iterator remoteFileLocations = fileSystem.listFileSystem(getSSTPrefix(), null, null); diff --git a/priam/src/test/java/com/netflix/priam/backupv2/TestBackupValidator.java b/priam/src/test/java/com/netflix/priam/backupv2/TestMetaV2Proxy.java similarity index 89% rename from priam/src/test/java/com/netflix/priam/backupv2/TestBackupValidator.java rename to priam/src/test/java/com/netflix/priam/backupv2/TestMetaV2Proxy.java index 3c9c23237..5fe5ba14b 100644 --- a/priam/src/test/java/com/netflix/priam/backupv2/TestBackupValidator.java +++ b/priam/src/test/java/com/netflix/priam/backupv2/TestMetaV2Proxy.java @@ -19,6 +19,7 @@ import com.google.inject.Guice; import com.google.inject.Injector; +import com.google.inject.Provider; import com.netflix.priam.backup.AbstractBackupPath; import com.netflix.priam.backup.BRTestModule; import com.netflix.priam.backup.BackupRestoreException; @@ -36,21 +37,21 @@ import org.junit.Test; /** Created by aagrawal on 12/5/18. */ -public class TestBackupValidator { +public class TestMetaV2Proxy { private FakeBackupFileSystem fs; private IConfiguration configuration; - private BackupValidator backupValidator; private TestBackupUtils backupUtils; private IMetaProxy metaProxy; + private Provider abstractBackupPathProvider; - public TestBackupValidator() { + public TestMetaV2Proxy() { Injector injector = Guice.createInjector(new BRTestModule()); configuration = injector.getInstance(IConfiguration.class); fs = injector.getInstance(FakeBackupFileSystem.class); fs.setupTest(getRemoteFakeFiles()); - backupValidator = injector.getInstance(BackupValidator.class); backupUtils = new TestBackupUtils(); metaProxy = injector.getInstance(MetaV2Proxy.class); + abstractBackupPathProvider = injector.getProvider(AbstractBackupPath.class); } @Test @@ -76,8 +77,12 @@ public void testMetaPrefix() { @Test public void testIsMetaFileValid() throws Exception { - Path metaPath = backupUtils.createMeta(getRemoteFakeFiles(), DateUtil.getInstant()); - Assert.assertTrue(backupValidator.isMetaFileValid(metaPath)); + Instant snapshotInstant = DateUtil.getInstant(); + Path metaPath = backupUtils.createMeta(getRemoteFakeFiles(), snapshotInstant); + AbstractBackupPath abstractBackupPath = abstractBackupPathProvider.get(); + abstractBackupPath.parseLocal(metaPath.toFile(), AbstractBackupPath.BackupFileType.META_V2); + + Assert.assertTrue(metaProxy.isMetaFileValid(abstractBackupPath).valid); FileUtils.deleteQuietly(metaPath.toFile()); List fileToAdd = getRemoteFakeFiles(); @@ -93,12 +98,12 @@ public void testIsMetaFileValid() throws Exception { "file9.Data.db") .toString()); - metaPath = backupUtils.createMeta(fileToAdd, DateUtil.getInstant()); - Assert.assertFalse(backupValidator.isMetaFileValid(metaPath)); + metaPath = backupUtils.createMeta(fileToAdd, snapshotInstant); + Assert.assertFalse(metaProxy.isMetaFileValid(abstractBackupPath).valid); FileUtils.deleteQuietly(metaPath.toFile()); - metaPath = Paths.get(configuration.getDataFileLocation(), "meta_v2_201901010000.json"); - Assert.assertFalse(backupValidator.isMetaFileValid(metaPath)); + metaPath = Paths.get(configuration.getDataFileLocation(), "meta_v2_201801010000.json"); + Assert.assertFalse(metaProxy.isMetaFileValid(abstractBackupPath).valid); } @Test From 43c7f1c77ba906637f4929efccdfc934fbca25e8 Mon Sep 17 00:00:00 2001 From: Arun Agrawal Date: Fri, 28 Dec 2018 14:23:10 -0800 Subject: [PATCH 045/228] Backup 2.0 Restore --- CHANGELOG.md | 10 ++ .../netflix/priam/backup/AbstractBackup.java | 4 - .../priam/backup/AbstractFileSystem.java | 4 +- .../priam/backup/BackupVerification.java | 16 +- .../priam/backup/CommitLogBackupTask.java | 8 - .../priam/backup/IncrementalBackup.java | 17 +- .../com/netflix/priam/backup/MetaData.java | 29 --- .../netflix/priam/backup/SnapshotBackup.java | 79 +------- .../priam/backupv2/BackupValidator.java | 8 +- .../priam/backupv2/ForgottenFilesManager.java | 147 +++++++++++++++ .../netflix/priam/backupv2/IMetaProxy.java | 11 ++ .../netflix/priam/backupv2/MetaV1Proxy.java | 24 ++- .../netflix/priam/backupv2/MetaV2Proxy.java | 71 +++++++- .../priam/config/BackupRestoreConfig.java | 10 ++ .../priam/config/IBackupRestoreConfig.java | 14 +- .../netflix/priam/config/IConfiguration.java | 13 ++ .../priam/config/PriamConfiguration.java | 10 ++ .../priam/restore/AbstractRestore.java | 116 +++++++----- .../priam/services/SnapshotMetaService.java | 11 +- .../priam/backup/FakeBackupFileSystem.java | 3 +- .../priam/backup/TestAbstractFileSystem.java | 10 +- .../priam/backup/TestS3FileSystem.java | 1 + .../backupv2/TestForgottenFileManager.java | 169 ++++++++++++++++++ .../priam/backupv2/TestMetaFileManager.java | 98 ---------- .../priam/backupv2/TestMetaV2Proxy.java | 85 ++++++++- .../priam/config/FakeBackupRestoreConfig.java | 5 + .../{backup => restore}/TestRestore.java | 52 +++++- 27 files changed, 712 insertions(+), 313 deletions(-) create mode 100644 priam/src/main/java/com/netflix/priam/backupv2/ForgottenFilesManager.java create mode 100644 priam/src/test/java/com/netflix/priam/backupv2/TestForgottenFileManager.java delete mode 100644 priam/src/test/java/com/netflix/priam/backupv2/TestMetaFileManager.java rename priam/src/test/java/com/netflix/priam/{backup => restore}/TestRestore.java (71%) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7d9629c46..c1926cfce 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,14 @@ # Changelog + +## 2018/01/11 3.11.38 +(#761) Add new file format (SST_V2) and methods to get/parse remote locations. +(#761) Upload files from SnapshotMetaService in backup version 2.0, if enabled. +(#761) Process older SNAPSHOT_V2 at the restart of Priam. +(#767) Backup Verification for Backup 2.0. +(#767) Restore for Backup 2.0 +(#767) Some API changes for Snapshot Verification +(#767) Remove deprecated code like flush hour or snapshot hour. + ## 2018/10/29 3.11.37 * Bug Fix: SnapshotMetaService can leave snapshots if there is any error. * Bug Fix: SnapshotMetaService should continue building snapshot even if an unexpected file is found in snapshot. diff --git a/priam/src/main/java/com/netflix/priam/backup/AbstractBackup.java b/priam/src/main/java/com/netflix/priam/backup/AbstractBackup.java index cec5c53c9..3f17facd8 100644 --- a/priam/src/main/java/com/netflix/priam/backup/AbstractBackup.java +++ b/priam/src/main/java/com/netflix/priam/backup/AbstractBackup.java @@ -106,7 +106,6 @@ protected List upload( true); bps.add(bp); - addToRemotePath(bp.getRemotePath()); } } @@ -183,7 +182,4 @@ private boolean isValidBackupDir(File keyspaceDir, File backupDir) { return true; } - - /** Adds Remote path to the list of Remote Paths */ - protected abstract void addToRemotePath(String remotePath); } diff --git a/priam/src/main/java/com/netflix/priam/backup/AbstractFileSystem.java b/priam/src/main/java/com/netflix/priam/backup/AbstractFileSystem.java index e2b79298a..c8126daff 100644 --- a/priam/src/main/java/com/netflix/priam/backup/AbstractFileSystem.java +++ b/priam/src/main/java/com/netflix/priam/backup/AbstractFileSystem.java @@ -115,8 +115,8 @@ public Future asyncDownloadFile( public void downloadFile(final Path remotePath, final Path localPath, final int retry) throws BackupRestoreException { // TODO: Should we download the file if localPath already exists? - if (remotePath == null) return; - + if (remotePath == null || localPath == null) return; + localPath.toFile().getParentFile().mkdirs(); logger.info("Downloading file: {} to location: {}", remotePath, localPath); try { new BoundedExponentialRetryCallable(500, 10000, retry) { diff --git a/priam/src/main/java/com/netflix/priam/backup/BackupVerification.java b/priam/src/main/java/com/netflix/priam/backup/BackupVerification.java index 8e7bd771e..13e8d881c 100644 --- a/priam/src/main/java/com/netflix/priam/backup/BackupVerification.java +++ b/priam/src/main/java/com/netflix/priam/backup/BackupVerification.java @@ -21,7 +21,6 @@ import java.nio.file.Path; import java.nio.file.Paths; import java.util.*; -import java.util.stream.Collectors; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -46,22 +45,21 @@ public class BackupVerification { } public Optional getLatestBackupMetaData(List metadata) { - metadata = - metadata.stream() - .filter(backupMetadata -> backupMetadata.getStatus() == Status.FINISHED) - .collect(Collectors.toList()); - metadata.sort((o1, o2) -> o2.getStart().compareTo(o1.getStart())); - return metadata.stream().findFirst(); + return metadata.stream() + .filter(backupMetadata -> backupMetadata != null) + .filter(backupMetadata -> backupMetadata.getStatus() == Status.FINISHED) + .sorted(Comparator.comparing(BackupMetadata::getStart).reversed()) + .findFirst(); } public Optional verifyBackup(List metadata) { - if (metadata == null || metadata.isEmpty()) return null; + if (metadata == null || metadata.isEmpty()) return Optional.empty(); Optional latestBackupMetaData = getLatestBackupMetaData(metadata); if (!latestBackupMetaData.isPresent()) { logger.error("No backup found which finished during the time provided."); - return null; + return Optional.empty(); } Path metadataLocation = Paths.get(latestBackupMetaData.get().getSnapshotLocation()); diff --git a/priam/src/main/java/com/netflix/priam/backup/CommitLogBackupTask.java b/priam/src/main/java/com/netflix/priam/backup/CommitLogBackupTask.java index 894b8da0b..18ca2566c 100644 --- a/priam/src/main/java/com/netflix/priam/backup/CommitLogBackupTask.java +++ b/priam/src/main/java/com/netflix/priam/backup/CommitLogBackupTask.java @@ -20,8 +20,6 @@ import com.netflix.priam.scheduler.SimpleTimer; import com.netflix.priam.scheduler.TaskTimer; import java.io.File; -import java.util.ArrayList; -import java.util.List; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -31,7 +29,6 @@ public class CommitLogBackupTask extends AbstractBackup { public static final String JOBNAME = "CommitLogBackup"; private static final Logger logger = LoggerFactory.getLogger(CommitLogBackupTask.class); - private final List clRemotePaths = new ArrayList<>(); private final CommitLogBackup clBackup; @Inject @@ -70,9 +67,4 @@ protected void processColumnFamily(String keyspace, String columnFamily, File ba throws Exception { // Do nothing. } - - @Override - protected void addToRemotePath(String remotePath) { - clRemotePaths.add(remotePath); - } } diff --git a/priam/src/main/java/com/netflix/priam/backup/IncrementalBackup.java b/priam/src/main/java/com/netflix/priam/backup/IncrementalBackup.java index 610d7f894..627c303e5 100644 --- a/priam/src/main/java/com/netflix/priam/backup/IncrementalBackup.java +++ b/priam/src/main/java/com/netflix/priam/backup/IncrementalBackup.java @@ -20,12 +20,12 @@ import com.google.inject.Provider; import com.google.inject.Singleton; import com.netflix.priam.backup.AbstractBackupPath.BackupFileType; +import com.netflix.priam.config.IBackupRestoreConfig; import com.netflix.priam.config.IConfiguration; import com.netflix.priam.scheduler.SimpleTimer; import com.netflix.priam.scheduler.TaskTimer; import com.netflix.priam.utils.DateUtil; import java.io.File; -import java.util.ArrayList; import java.util.List; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -37,13 +37,14 @@ public class IncrementalBackup extends AbstractBackup { private static final Logger logger = LoggerFactory.getLogger(IncrementalBackup.class); public static final String JOBNAME = "IncrementalBackup"; - private final List incrementalRemotePaths = new ArrayList<>(); private final IncrementalMetaData metaData; private final BackupRestoreUtil backupRestoreUtil; + private final IBackupRestoreConfig backupRestoreConfig; @Inject public IncrementalBackup( IConfiguration config, + IBackupRestoreConfig backupRestoreConfig, Provider pathFactory, IFileSystemContext backupFileSystemCtx, IncrementalMetaData metaData) { @@ -51,6 +52,7 @@ public IncrementalBackup( // a means to upload audit trail (via meta_cf_yyyymmddhhmm.json) of files successfully // uploaded) this.metaData = metaData; + this.backupRestoreConfig = backupRestoreConfig; backupRestoreUtil = new BackupRestoreUtil( config.getIncrementalIncludeCFList(), config.getIncrementalExcludeCFList()); @@ -59,7 +61,6 @@ public IncrementalBackup( @Override public void execute() throws Exception { // Clearing remotePath List - incrementalRemotePaths.clear(); initiateBackup(INCREMENTAL_BACKUP_FOLDER, backupRestoreUtil); } @@ -76,8 +77,11 @@ public String getName() { @Override protected void processColumnFamily(String keyspace, String columnFamily, File backupDir) throws Exception { + BackupFileType fileType = BackupFileType.SST; + if (backupRestoreConfig.enableV2Backups()) fileType = BackupFileType.SST_V2; + List uploadedFiles = - upload(backupDir, BackupFileType.SST, config.enableAsyncIncremental(), true); + upload(backupDir, fileType, config.enableAsyncIncremental(), true); if (!uploadedFiles.isEmpty()) { // format of yyyymmddhhmm (e.g. 201505060901) @@ -90,9 +94,4 @@ protected void processColumnFamily(String keyspace, String columnFamily, File ba logger.info("Uploaded meta file for incremental backup: {}", metaFileName); } } - - @Override - protected void addToRemotePath(String remotePath) { - incrementalRemotePaths.add(remotePath); - } } diff --git a/priam/src/main/java/com/netflix/priam/backup/MetaData.java b/priam/src/main/java/com/netflix/priam/backup/MetaData.java index 67be8ae4c..be7a0bdab 100644 --- a/priam/src/main/java/com/netflix/priam/backup/MetaData.java +++ b/priam/src/main/java/com/netflix/priam/backup/MetaData.java @@ -16,14 +16,12 @@ */ package com.netflix.priam.backup; -import com.google.common.collect.Lists; import com.google.inject.Inject; import com.google.inject.Provider; import com.netflix.priam.backup.AbstractBackupPath.BackupFileType; import com.netflix.priam.config.IConfiguration; import com.netflix.priam.utils.DateUtil; import java.io.File; -import java.io.FileReader; import java.io.FileWriter; import java.io.IOException; import java.nio.file.Paths; @@ -32,7 +30,6 @@ import java.util.List; import org.apache.commons.io.FileUtils; import org.json.simple.JSONArray; -import org.json.simple.parser.JSONParser; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -117,30 +114,4 @@ public File createTmpMetaFile() throws IOException { private void addToRemotePath(String remotePath) { metaRemotePaths.add(remotePath); } - - public List toJson(File input) { - List files = Lists.newArrayList(); - try { - JSONArray jsonObj = (JSONArray) new JSONParser().parse(new FileReader(input)); - for (Object aJsonObj : jsonObj) { - AbstractBackupPath p = pathFactory.get(); - p.parseRemote((String) aJsonObj); - files.add(p); - } - - } catch (Exception ex) { - throw new RuntimeException( - "Error transforming file " - + input.getAbsolutePath() - + " to JSON format. Msg:" - + ex.getLocalizedMessage(), - ex); - } - - logger.debug( - "Transformed file {} to JSON. Number of JSON elements: {}", - input.getAbsolutePath(), - files.size()); - return files; - } } diff --git a/priam/src/main/java/com/netflix/priam/backup/SnapshotBackup.java b/priam/src/main/java/com/netflix/priam/backup/SnapshotBackup.java index 785079a15..ff5efd488 100644 --- a/priam/src/main/java/com/netflix/priam/backup/SnapshotBackup.java +++ b/priam/src/main/java/com/netflix/priam/backup/SnapshotBackup.java @@ -21,6 +21,7 @@ import com.google.inject.Provider; import com.google.inject.Singleton; import com.netflix.priam.backup.AbstractBackupPath.BackupFileType; +import com.netflix.priam.backupv2.ForgottenFilesManager; import com.netflix.priam.config.IConfiguration; import com.netflix.priam.defaultimpl.CassandraOperations; import com.netflix.priam.identity.InstanceIdentity; @@ -30,6 +31,7 @@ import com.netflix.priam.utils.DateUtil; import com.netflix.priam.utils.ThreadSleeper; import java.io.File; +import java.time.Instant; import java.util.*; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; @@ -42,13 +44,14 @@ public class SnapshotBackup extends AbstractBackup { private static final Logger logger = LoggerFactory.getLogger(SnapshotBackup.class); public static final String JOBNAME = "SnapshotBackup"; private final MetaData metaData; - private final List snapshotRemotePaths = new ArrayList<>(); private final ThreadSleeper sleeper = new ThreadSleeper(); private static final long WAIT_TIME_MS = 60 * 1000 * 10; private final InstanceIdentity instanceIdentity; private final IBackupStatusMgr snapshotStatusMgr; private final BackupRestoreUtil backupRestoreUtil; + private final ForgottenFilesManager forgottenFilesManager; private String snapshotName = null; + private Instant snapshotInstant = DateUtil.getInstant(); private List abstractBackupPaths = null; private final CassandraOperations cassandraOperations; private static final Lock lock = new ReentrantLock(); @@ -61,7 +64,8 @@ public SnapshotBackup( IFileSystemContext backupFileSystemCtx, IBackupStatusMgr snapshotStatusMgr, InstanceIdentity instanceIdentity, - CassandraOperations cassandraOperations) { + CassandraOperations cassandraOperations, + ForgottenFilesManager forgottenFilesManager) { super(config, backupFileSystemCtx, pathFactory); this.metaData = metaData; this.snapshotStatusMgr = snapshotStatusMgr; @@ -70,6 +74,7 @@ public SnapshotBackup( backupRestoreUtil = new BackupRestoreUtil( config.getSnapshotIncludeCFList(), config.getSnapshotExcludeCFList()); + this.forgottenFilesManager = forgottenFilesManager; } @Override @@ -100,6 +105,7 @@ public void execute() throws Exception { private void executeSnapshot() throws Exception { Date startTime = Calendar.getInstance(TimeZone.getTimeZone("GMT")).getTime(); snapshotName = DateUtil.formatyyyyMMddHHmm(startTime); + snapshotInstant = DateUtil.getInstant(); String token = instanceIdentity.getInstance().getToken(); // Save start snapshot status @@ -108,8 +114,6 @@ private void executeSnapshot() throws Exception { try { logger.info("Starting snapshot {}", snapshotName); - // Clearing remotePath List - snapshotRemotePaths.clear(); cassandraOperations.takeSnapshot(snapshotName); // Collect all snapshot dir's under keyspace dir's @@ -177,74 +181,9 @@ protected void processColumnFamily(String keyspace, String columnFamily, File ba return; } + forgottenFilesManager.findAndMoveForgottenFiles(snapshotInstant, snapshotDir); // Add files to this dir abstractBackupPaths.addAll( upload(snapshotDir, BackupFileType.SNAP, config.enableAsyncSnapshot(), true)); } - - // private void findForgottenFiles(File snapshotDir) { - // try { - // Collection snapshotFiles = FileUtils.listFiles(snapshotDir, - // FileFilterUtils.fileFileFilter(), null); - // File columnfamilyDir = snapshotDir.getParentFile().getParentFile(); - // - // //Find all the files in columnfamily folder which is : - // // 1. Not a temp file. - // // 2. Is a file. (we don't care about directories) - // // 3. Is older than snapshot time, as new files keep getting created after taking - // a snapshot. - // IOFileFilter tmpFileFilter1 = FileFilterUtils.suffixFileFilter(TMP_EXT); - // IOFileFilter tmpFileFilter2 = FileFilterUtils.asFileFilter(pathname -> - // tmpFilePattern.matcher(pathname.getName()).matches()); - // IOFileFilter tmpFileFilter = FileFilterUtils.or(tmpFileFilter1, tmpFileFilter2); - // // Here we are allowing files which were more than - // @link{IConfiguration#getForgottenFileGracePeriodDays}. We do this to allow cassandra to - // // clean up any files which were generated as part of repair/compaction and - // cleanup thread has not already deleted. - // // Refer to https://issues.apache.org/jira/browse/CASSANDRA-6756 and - // https://issues.apache.org/jira/browse/CASSANDRA-7066 - // // for more information. - // IOFileFilter ageFilter = - // FileFilterUtils.ageFileFilter(snapshotInstant.minus(config.getForgottenFileGracePeriodDays(), - // ChronoUnit.DAYS).toEpochMilli()); - // IOFileFilter fileFilter = - // FileFilterUtils.and(FileFilterUtils.notFileFilter(tmpFileFilter), - // FileFilterUtils.fileFileFilter(), ageFilter); - // - // Collection columnfamilyFiles = FileUtils.listFiles(columnfamilyDir, - // fileFilter, null); - // - // //Remove the SSTable(s) which are part of snapshot from the CF file list. - // //This cannot be a simple removeAll as snapshot files have "different" file folder - // prefix. - // for (File file : snapshotFiles) { - // //Get its parent directory file based on this file. - // File originalFile = new File(columnfamilyDir, file.getName()); - // columnfamilyFiles.remove(originalFile); - // } - // - // //If there are no "extra" SSTables in CF data folder, we are done. - // if (columnfamilyFiles.size() == 0) - // return; - // - // columnfamilyFiles.parallelStream().forEach(file -> logger.info("Forgotten file: {} - // found for CF: {}", file.getAbsolutePath(), columnfamilyDir.getName())); - // - // //TODO: The eventual plan is to move the forgotten files to a lost+found directory - // and clean the directory after 'x' amount of time. This behavior should be configurable. - // backupMetrics.incrementForgottenFiles(columnfamilyFiles.size()); - // logger.warn("# of forgotten files: {} found for CF: {}", columnfamilyFiles.size(), - // columnfamilyDir.getName()); - // } catch (Exception e) { - // //Eat the exception, if there, for any reason. This should not stop the snapshot - // for any reason. - // logger.error("Exception occurred while trying to find forgottenFile. Ignoring the - // error and continuing with remaining backup", e); - // } - // } - - @Override - protected void addToRemotePath(String remotePath) { - snapshotRemotePaths.add(remotePath); - } } diff --git a/priam/src/main/java/com/netflix/priam/backupv2/BackupValidator.java b/priam/src/main/java/com/netflix/priam/backupv2/BackupValidator.java index bd6b332cf..2b7a7d3e4 100644 --- a/priam/src/main/java/com/netflix/priam/backupv2/BackupValidator.java +++ b/priam/src/main/java/com/netflix/priam/backupv2/BackupValidator.java @@ -18,7 +18,6 @@ package com.netflix.priam.backupv2; import com.netflix.priam.backup.*; -import com.netflix.priam.config.IConfiguration; import com.netflix.priam.utils.DateUtil; import java.util.List; import java.util.Optional; @@ -33,15 +32,10 @@ */ public class BackupValidator { private static final Logger logger = LoggerFactory.getLogger(BackupVerification.class); - private final IBackupFileSystem fs; private IMetaProxy metaProxy; @Inject - public BackupValidator( - IConfiguration configuration, - IFileSystemContext backupFileSystemCtx, - @Named("v2") IMetaProxy metaProxy) { - fs = backupFileSystemCtx.getFileStrategy(configuration); + public BackupValidator(@Named("v2") IMetaProxy metaProxy) { this.metaProxy = metaProxy; } diff --git a/priam/src/main/java/com/netflix/priam/backupv2/ForgottenFilesManager.java b/priam/src/main/java/com/netflix/priam/backupv2/ForgottenFilesManager.java new file mode 100644 index 000000000..4c8884008 --- /dev/null +++ b/priam/src/main/java/com/netflix/priam/backupv2/ForgottenFilesManager.java @@ -0,0 +1,147 @@ +/* + * Copyright 2019 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.netflix.priam.backupv2; + +import com.google.inject.Inject; +import com.netflix.priam.config.IConfiguration; +import com.netflix.priam.merics.BackupMetrics; +import java.io.File; +import java.io.IOException; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.time.Instant; +import java.time.temporal.ChronoUnit; +import java.util.Collection; +import java.util.regex.Pattern; +import org.apache.commons.io.FileUtils; +import org.apache.commons.io.filefilter.FileFilterUtils; +import org.apache.commons.io.filefilter.IOFileFilter; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** Created by aagrawal on 1/1/19. */ +public class ForgottenFilesManager { + private static final Logger logger = LoggerFactory.getLogger(ForgottenFilesManager.class); + + private BackupMetrics backupMetrics; + private IConfiguration config; + private static final String TMP_EXT = ".tmp"; + + @SuppressWarnings("Annotator") + private static final Pattern tmpFilePattern = + Pattern.compile("^((.*)\\-(.*)\\-)?tmp(link)?\\-((?:l|k).)\\-(\\d)*\\-(.*)$"); + + protected static final String LOST_FOUND = "lost+found"; + + @Inject + public ForgottenFilesManager(IConfiguration configuration, BackupMetrics backupMetrics) { + this.config = configuration; + this.backupMetrics = backupMetrics; + } + + public void findAndMoveForgottenFiles(Instant snapshotInstant, File snapshotDir) { + try { + Collection snapshotFiles = + FileUtils.listFiles(snapshotDir, FileFilterUtils.fileFileFilter(), null); + File columnfamilyDir = snapshotDir.getParentFile().getParentFile(); + Collection columnfamilyFiles = + getColumnfamilyFiles(snapshotInstant, columnfamilyDir); + + // Remove the SSTable(s) which are part of snapshot from the CF file list. + // This cannot be a simple removeAll as snapshot files have "different" file folder + // prefix. + for (File file : snapshotFiles) { + // Get its parent directory file based on this file. + File originalFile = new File(columnfamilyDir, file.getName()); + columnfamilyFiles.remove(originalFile); + } + + // If there are no "extra" SSTables in CF data folder, we are done. + if (columnfamilyFiles.size() == 0) return; + + logger.warn( + "# of forgotten files: {} found for CF: {}", + columnfamilyFiles.size(), + columnfamilyDir.getName()); + backupMetrics.incrementForgottenFiles(columnfamilyFiles.size()); + + // Move the files to lost_found directory if configured. + moveForgottenFiles(columnfamilyDir, columnfamilyFiles); + + } catch (Exception e) { + // Eat the exception, if there, for any reason. This should not stop the snapshot for + // any reason. + logger.error( + "Exception occurred while trying to find forgottenFile. Ignoring the error and continuing with remaining backup", + e); + e.printStackTrace(); + } + } + + protected Collection getColumnfamilyFiles(Instant snapshotInstant, File columnfamilyDir) { + // Find all the files in columnfamily folder which is : + // 1. Not a temp file. + // 2. Is a file. (we don't care about directories) + // 3. Is older than snapshot time, as new files keep getting created after taking a + // snapshot. + IOFileFilter tmpFileFilter1 = FileFilterUtils.suffixFileFilter(TMP_EXT); + IOFileFilter tmpFileFilter2 = + FileFilterUtils.asFileFilter( + pathname -> tmpFilePattern.matcher(pathname.getName()).matches()); + IOFileFilter tmpFileFilter = FileFilterUtils.or(tmpFileFilter1, tmpFileFilter2); + // Here we are allowing files which were more than + // @link{IConfiguration#getForgottenFileGracePeriodDays}. We do this to allow cassandra + // to clean up any files which were generated as part of repair/compaction and cleanup + // thread has not already deleted. + // Refer to https://issues.apache.org/jira/browse/CASSANDRA-6756 and + // https://issues.apache.org/jira/browse/CASSANDRA-7066 + // for more information. + IOFileFilter ageFilter = + FileFilterUtils.ageFileFilter( + snapshotInstant + .minus(config.getForgottenFileGracePeriodDays(), ChronoUnit.DAYS) + .toEpochMilli()); + IOFileFilter fileFilter = + FileFilterUtils.and( + FileFilterUtils.notFileFilter(tmpFileFilter), + FileFilterUtils.fileFileFilter(), + ageFilter); + + return FileUtils.listFiles(columnfamilyDir, fileFilter, null); + } + + protected void moveForgottenFiles(File columnfamilyDir, Collection columnfamilyFiles) { + final Path destDir = Paths.get(columnfamilyDir.getAbsolutePath(), LOST_FOUND); + for (File file : columnfamilyFiles) { + logger.warn( + "Forgotten file: {} found for CF: {}", + file.getAbsolutePath(), + columnfamilyDir.getName()); + if (config.isForgottenFileMoveEnabled()) { + try { + FileUtils.moveFileToDirectory(file, destDir.toFile(), true); + } catch (IOException e) { + logger.error( + "Exception occurred while trying to move forgottenFile: {}. Ignoring the error and continuing with remaining backup/forgotten files.", + file); + e.printStackTrace(); + } + } + } + } +} diff --git a/priam/src/main/java/com/netflix/priam/backupv2/IMetaProxy.java b/priam/src/main/java/com/netflix/priam/backupv2/IMetaProxy.java index 4c173b599..99c52abb2 100644 --- a/priam/src/main/java/com/netflix/priam/backupv2/IMetaProxy.java +++ b/priam/src/main/java/com/netflix/priam/backupv2/IMetaProxy.java @@ -22,6 +22,7 @@ import com.netflix.priam.backup.BackupVerificationResult; import com.netflix.priam.utils.DateUtil; import java.nio.file.Path; +import java.util.Iterator; import java.util.List; /** Proxy to do management tasks for meta files. Created by aagrawal on 12/18/18. */ @@ -71,6 +72,16 @@ public interface IMetaProxy { */ List getSSTFilesFromMeta(Path localMetaPath) throws Exception; + /** + * Get the list of incremental files given the daterange. + * + * @param dateRange the time period to scan in the remote file system for incremental files. + * @return iterator containing the list of path on the remote file system satisfying criteria. + * @throws BackupRestoreException if there is an issue contacting remote file system. + */ + Iterator getIncrementals(DateUtil.DateRange dateRange) + throws BackupRestoreException; + /** * Validate that all the files mentioned in the meta file actually exists on remote file system. * diff --git a/priam/src/main/java/com/netflix/priam/backupv2/MetaV1Proxy.java b/priam/src/main/java/com/netflix/priam/backupv2/MetaV1Proxy.java index 3c71101c6..cfce6ae81 100644 --- a/priam/src/main/java/com/netflix/priam/backupv2/MetaV1Proxy.java +++ b/priam/src/main/java/com/netflix/priam/backupv2/MetaV1Proxy.java @@ -29,6 +29,7 @@ import java.time.temporal.ChronoUnit; import java.util.*; import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.collections4.iterators.FilterIterator; import org.apache.commons.io.FileUtils; import org.json.simple.parser.JSONParser; import org.slf4j.Logger; @@ -38,11 +39,9 @@ public class MetaV1Proxy implements IMetaProxy { private static final Logger logger = LoggerFactory.getLogger(MetaV1Proxy.class); private final IBackupFileSystem fs; - private final IConfiguration configuration; @Inject MetaV1Proxy(IConfiguration configuration, IFileSystemContext backupFileSystemCtx) { - this.configuration = configuration; fs = backupFileSystemCtx.getFileStrategy(configuration); } @@ -71,12 +70,12 @@ public List findMetaFiles(DateUtil.DateRange dateRange) { if (path.getType() == AbstractBackupPath.BackupFileType.META) // Since there are now meta file for incrementals as well as snapshot, we need to // find the correct one (i.e. the snapshot meta file (meta.json)) - if (path.getType() == AbstractBackupPath.BackupFileType.META) { + if (path.getFileName().equalsIgnoreCase("meta.json")) { metas.add(path); } } - Collections.sort(metas, Collections.reverseOrder()); + metas.sort(Collections.reverseOrder()); if (metas.size() == 0) { logger.info( @@ -102,7 +101,7 @@ public BackupVerificationResult isMetaFileValid(AbstractBackupPath metaBackupPat result.manifestAvailable = true; // List the remote file system to validate the backup. - String prefix = configuration.getBackupPrefix(); + String prefix = fs.getPrefix().toString(); Date strippedMsSnapshotTime = new Date(result.snapshotInstant.truncatedTo(ChronoUnit.MINUTES).toEpochMilli()); Iterator backupfiles = @@ -170,6 +169,21 @@ public List getSSTFilesFromMeta(Path localMetaPath) throws Exception { return result; } + @Override + public Iterator getIncrementals(DateUtil.DateRange dateRange) + throws BackupRestoreException { + String prefix = fs.getPrefix().toString(); + Iterator iterator = + fs.list( + prefix, + new Date(dateRange.getStartTime().toEpochMilli()), + new Date(dateRange.getEndTime().toEpochMilli())); + return new FilterIterator<>( + iterator, + abstractBackupPath -> + abstractBackupPath.getType() == AbstractBackupPath.BackupFileType.SST); + } + @Override public void cleanupOldMetaFiles() {} } diff --git a/priam/src/main/java/com/netflix/priam/backupv2/MetaV2Proxy.java b/priam/src/main/java/com/netflix/priam/backupv2/MetaV2Proxy.java index fa9df765d..2a4167b1f 100644 --- a/priam/src/main/java/com/netflix/priam/backupv2/MetaV2Proxy.java +++ b/priam/src/main/java/com/netflix/priam/backupv2/MetaV2Proxy.java @@ -28,6 +28,8 @@ import java.nio.file.Paths; import java.util.*; import javax.inject.Inject; +import org.apache.commons.collections4.iterators.FilterIterator; +import org.apache.commons.collections4.iterators.TransformIterator; import org.apache.commons.io.FileUtils; import org.apache.commons.io.filefilter.FileFilterUtils; import org.apache.commons.io.filefilter.IOFileFilter; @@ -59,18 +61,61 @@ public Path getLocalMetaFileDirectory() { @Override public String getMetaPrefix(DateUtil.DateRange dateRange) { + return getMatch(dateRange, AbstractBackupPath.BackupFileType.META_V2); + } + + private String getMatch( + DateUtil.DateRange dateRange, AbstractBackupPath.BackupFileType backupFileType) { Path location = fs.getPrefix(); AbstractBackupPath abstractBackupPath = abstractBackupPathProvider.get(); String match = StringUtils.EMPTY; if (dateRange != null) match = dateRange.match(); + if (dateRange != null && dateRange.getEndTime() == null) + match = dateRange.getStartTime().toEpochMilli() + ""; return Paths.get( - abstractBackupPath - .remoteV2Prefix(location, AbstractBackupPath.BackupFileType.META_V2) - .toString(), + abstractBackupPath.remoteV2Prefix(location, backupFileType).toString(), match) .toString(); } + @Override + public Iterator getIncrementals(DateUtil.DateRange dateRange) + throws BackupRestoreException { + String incrementalPrefix = getMatch(dateRange, AbstractBackupPath.BackupFileType.SST_V2); + String marker = + getMatch( + new DateUtil.DateRange(dateRange.getStartTime(), null), + AbstractBackupPath.BackupFileType.SST_V2); + logger.info( + "Listing filesystem with prefix: {}, marker: {}, daterange: {}", + incrementalPrefix, + marker, + dateRange); + Iterator iterator = fs.listFileSystem(incrementalPrefix, null, marker); + Iterator transformIterator = + new TransformIterator<>( + iterator, + s -> { + AbstractBackupPath path = abstractBackupPathProvider.get(); + path.parseRemote(s); + return path; + }); + + return new FilterIterator<>( + transformIterator, + abstractBackupPath -> + (abstractBackupPath.getLastModified().isAfter(dateRange.getStartTime()) + && abstractBackupPath + .getLastModified() + .isBefore(dateRange.getEndTime())) + || abstractBackupPath + .getLastModified() + .equals(dateRange.getStartTime()) + || abstractBackupPath + .getLastModified() + .equals(dateRange.getEndTime())); + } + @Override public List findMetaFiles(DateUtil.DateRange dateRange) { ArrayList metas = new ArrayList<>(); @@ -95,7 +140,7 @@ public List findMetaFiles(DateUtil.DateRange dateRange) { } } - Collections.sort(metas, Collections.reverseOrder()); + metas.sort(Collections.reverseOrder()); if (metas.size() == 0) { logger.info( @@ -136,7 +181,9 @@ public void cleanupOldMetaFiles() { @Override public List getSSTFilesFromMeta(Path localMetaPath) throws Exception { - return null; + MetaFileBackupWalker metaFileBackupWalker = new MetaFileBackupWalker(); + metaFileBackupWalker.readMeta(localMetaPath); + return metaFileBackupWalker.backupRemotePaths; } @Override @@ -191,4 +238,18 @@ public void process(ColumnfamilyResult columnfamilyResult) { } } } + + private class MetaFileBackupWalker extends MetaFileReader { + private List backupRemotePaths = new ArrayList<>(); + + @Override + public void process(ColumnfamilyResult columnfamilyResult) { + for (ColumnfamilyResult.SSTableResult ssTableResult : + columnfamilyResult.getSstables()) { + for (FileUploadResult fileUploadResult : ssTableResult.getSstableComponents()) { + backupRemotePaths.add(fileUploadResult.getBackupPath()); + } + } + } + } } diff --git a/priam/src/main/java/com/netflix/priam/config/BackupRestoreConfig.java b/priam/src/main/java/com/netflix/priam/config/BackupRestoreConfig.java index a0e0957ac..aae6b6739 100644 --- a/priam/src/main/java/com/netflix/priam/config/BackupRestoreConfig.java +++ b/priam/src/main/java/com/netflix/priam/config/BackupRestoreConfig.java @@ -35,4 +35,14 @@ public String getSnapshotMetaServiceCronExpression() { public boolean enableV2Backups() { return config.get("priam.enableV2Backups", false); } + + @Override + public boolean enableV2Restore() { + return config.get("priam.enableV2Restore", false); + } + + @Override + public String getBackupTTLCronExpression() { + return config.get("priam.backupTTLCronExpression", "0 0 0/6 1/1 * ? *"); + } } diff --git a/priam/src/main/java/com/netflix/priam/config/IBackupRestoreConfig.java b/priam/src/main/java/com/netflix/priam/config/IBackupRestoreConfig.java index 86d366a0e..bfdfbca4f 100644 --- a/priam/src/main/java/com/netflix/priam/config/IBackupRestoreConfig.java +++ b/priam/src/main/java/com/netflix/priam/config/IBackupRestoreConfig.java @@ -35,8 +35,8 @@ default String getSnapshotMetaServiceCronExpression() { } /** - * Enable the backup version 2.0 in new format. This will start uploading of backups in new - * format. This is to be used for migration from backup version 1.0. + * Enable the backup version 2.0 in new format. This will start uploads of "incremental" backups + * in new format. This is to be used for migration from backup version 1.0. * * @return boolean value indicating if backups in version 2.0 should be started. */ @@ -59,4 +59,14 @@ default boolean enableV2Backups() { default String getBackupTTLCronExpression() { return "0 0 0/6 1/1 * ? *"; } + + /** + * If restore is enabled and if this flag is enabled, we will try to restore using Backup V2.0. + * + * @return if restore should be using backup version 2.0. If this is false we will use backup + * version 1.0. + */ + default boolean enableV2Restore() { + return false; + } } diff --git a/priam/src/main/java/com/netflix/priam/config/IConfiguration.java b/priam/src/main/java/com/netflix/priam/config/IConfiguration.java index 5fdd0be8f..b5def0c87 100644 --- a/priam/src/main/java/com/netflix/priam/config/IConfiguration.java +++ b/priam/src/main/java/com/netflix/priam/config/IConfiguration.java @@ -980,6 +980,19 @@ default int getForgottenFileGracePeriodDays() { return 1; } + /** + * If any forgotten file is found in Cassandra, it is usually good practice to move/delete them + * so when cassandra restarts, it does not load old data which should be removed else you may + * run into data resurrection issues. This behavior is fixed in 3.x. This configuration will + * allow Priam to move the forgotten files to a "lost_found" directory for user to review at + * later time at the same time ensuring that Cassandra does not resurrect data. + * + * @return true if Priam should move forgotten file to "lost_found" directory of that CF. + */ + default boolean isForgottenFileMoveEnabled() { + return false; + } + /** * A method for allowing access to outside programs to Priam configuration when paired with the * Priam configuration HTTP endpoint at /v1/config/structured/all/property diff --git a/priam/src/main/java/com/netflix/priam/config/PriamConfiguration.java b/priam/src/main/java/com/netflix/priam/config/PriamConfiguration.java index 242dc0861..09704ec6d 100644 --- a/priam/src/main/java/com/netflix/priam/config/PriamConfiguration.java +++ b/priam/src/main/java/com/netflix/priam/config/PriamConfiguration.java @@ -738,4 +738,14 @@ public String getMergedConfigurationCronExpression() { // Every minute on the top of the minute. return config.get(PRIAM_PRE + ".configMerge.cron", "0 * * * * ? *"); } + + @Override + public int getForgottenFileGracePeriodDays() { + return config.get(PRIAM_PRE + ".forgottenFileGracePeriodDays", 1); + } + + @Override + public boolean isForgottenFileMoveEnabled() { + return config.get(PRIAM_PRE + ".forgottenFileMoveEnabled", false); + } } diff --git a/priam/src/main/java/com/netflix/priam/restore/AbstractRestore.java b/priam/src/main/java/com/netflix/priam/restore/AbstractRestore.java index 8e6d3a338..574005351 100644 --- a/priam/src/main/java/com/netflix/priam/restore/AbstractRestore.java +++ b/priam/src/main/java/com/netflix/priam/restore/AbstractRestore.java @@ -21,6 +21,7 @@ import com.netflix.priam.backup.*; import com.netflix.priam.backup.AbstractBackupPath.BackupFileType; import com.netflix.priam.backupv2.IMetaProxy; +import com.netflix.priam.config.IBackupRestoreConfig; import com.netflix.priam.config.IConfiguration; import com.netflix.priam.defaultimpl.ICassandraProcess; import com.netflix.priam.health.InstanceState; @@ -32,10 +33,12 @@ import java.io.IOException; import java.math.BigInteger; import java.nio.file.Path; +import java.time.Instant; import java.time.LocalDateTime; import java.time.ZoneId; import java.util.*; import java.util.concurrent.Future; +import java.util.stream.Collectors; import javax.inject.Named; import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.io.FileUtils; @@ -73,6 +76,8 @@ public abstract class AbstractRestore extends Task implements IRestoreStrategy { @Named("v2") IMetaProxy metaV2Proxy; + @Inject IBackupRestoreConfig backupRestoreConfig; + public AbstractRestore( IConfiguration config, IBackupFileSystem fs, @@ -113,10 +118,7 @@ public void setRestoreConfiguration(String restoreIncludeCFList, String restoreE } private List> download( - Iterator fsIterator, - BackupFileType bkupFileType, - boolean waitForCompletion) - throws Exception { + Iterator fsIterator, boolean waitForCompletion) throws Exception { List> futureList = new ArrayList<>(); while (fsIterator.hasNext()) { AbstractBackupPath temp = fsIterator.next(); @@ -130,16 +132,14 @@ private List> download( continue; } - if (temp.getType() == bkupFileType) { - File localFileHandler = temp.newRestoreFile(); - if (logger.isDebugEnabled()) - logger.debug( - "Created local file name: " - + localFileHandler.getAbsolutePath() - + File.pathSeparator - + localFileHandler.getName()); - futureList.add(downloadFile(temp, localFileHandler)); - } + File localFileHandler = temp.newRestoreFile(); + if (logger.isDebugEnabled()) + logger.debug( + "Created local file name: " + + localFileHandler.getAbsolutePath() + + File.pathSeparator + + localFileHandler.getName()); + futureList.add(downloadFile(temp, localFileHandler)); } // Wait for all download to finish that were started from this method. @@ -153,24 +153,19 @@ private void waitForCompletion(List> futureList) throws Exception { } private List> downloadCommitLogs( - Iterator fsIterator, - BackupFileType filter, - int lastN, - boolean waitForCompletion) + Iterator fsIterator, int lastN, boolean waitForCompletion) throws Exception { if (fsIterator == null) return null; BoundedList bl = new BoundedList(lastN); while (fsIterator.hasNext()) { AbstractBackupPath temp = fsIterator.next(); - if (temp.getType() == BackupFileType.SST) continue; - - if (temp.getType() == filter) { + if (temp.getType() == BackupFileType.CL) { bl.add(temp); } } - return download(bl.iterator(), filter, waitForCompletion); + return download(bl.iterator(), waitForCompletion); } private void stopCassProcess() throws IOException { @@ -196,6 +191,22 @@ public Void retriableCall() throws Exception { }.call(); } + private Optional getLatestValidMetaPath( + IMetaProxy metaProxy, DateUtil.DateRange dateRange) { + // Get a list of manifest files. + List metas = metaProxy.findMetaFiles(dateRange); + + // Find a valid manifest file. + for (AbstractBackupPath meta : metas) { + BackupVerificationResult result = metaProxy.isMetaFileValid(meta); + if (result.valid) { + return Optional.of(meta); + } + } + + return Optional.empty(); + } + public void restore(DateUtil.DateRange dateRange) throws Exception { // fail early if post restore hook has invalid parameters if (!postRestoreHook.hasValidParameters()) { @@ -203,6 +214,8 @@ public void restore(DateUtil.DateRange dateRange) throws Exception { } Date endTime = new Date(dateRange.getEndTime().toEpochMilli()); + IMetaProxy metaProxy = metaV1Proxy; + if (backupRestoreConfig.enableV2Restore()) metaProxy = metaV2Proxy; // Set the restore status. instanceState.getRestoreStatus().resetStatus(); @@ -231,38 +244,60 @@ public void restore(DateUtil.DateRange dateRange) throws Exception { File dataDir = new File(config.getDataFileLocation()); if (dataDir.exists() && dataDir.isDirectory()) FileUtils.cleanDirectory(dataDir); - // Try and read the Meta file. - List metas = metaV1Proxy.findMetaFiles(dateRange); + // Find latest valid meta file. + Optional latestValidMetaFile = + getLatestValidMetaPath(metaProxy, dateRange); - if (metas.size() == 0) { - logger.info("No snapshot meta file found, Restore Failed."); + if (!latestValidMetaFile.isPresent()) { + logger.info("No valid snapshot meta file found, Restore Failed."); instanceState.getRestoreStatus().setExecutionEndTime(LocalDateTime.now()); instanceState.setRestoreStatus(Status.FAILED); return; } - AbstractBackupPath meta = metas.get(0); - logger.info("Snapshot Meta file for restore {}", meta.getRemotePath()); - instanceState.getRestoreStatus().setSnapshotMetaFile(meta.getRemotePath()); + logger.info( + "Snapshot Meta file for restore {}", latestValidMetaFile.get().getRemotePath()); + instanceState + .getRestoreStatus() + .setSnapshotMetaFile(latestValidMetaFile.get().getRemotePath()); - // TODO: Validate that manifest file is valid. // Download the meta.json file. - Path metaFile = metaV1Proxy.downloadMetaFile(meta); + Path metaFile = metaProxy.downloadMetaFile(latestValidMetaFile.get()); // Parse meta.json file to find the files required to download from this snapshot. - List snapshots = metaData.toJson(metaFile.toFile()); + List snapshots = + metaProxy + .getSSTFilesFromMeta(metaFile) + .stream() + .map( + value -> { + AbstractBackupPath path = pathProvider.get(); + path.parseRemote(value); + return path; + }) + .collect(Collectors.toList()); + FileUtils.deleteQuietly(metaFile.toFile()); // Download snapshot which is listed in the meta file. List> futureList = new ArrayList<>(); - futureList.addAll(download(snapshots.iterator(), BackupFileType.SNAP, false)); + futureList.addAll(download(snapshots.iterator(), false)); logger.info("Downloading incrementals"); + // Download incrementals (SST) after the snapshot meta file. - String prefix = fs.getPrefix().toString(); - Iterator incrementals = fs.list(prefix, meta.getTime(), endTime); - futureList.addAll(download(incrementals, BackupFileType.SST, false)); + Instant snapshotTime; + if (backupRestoreConfig.enableV2Restore()) + snapshotTime = latestValidMetaFile.get().getLastModified(); + else snapshotTime = latestValidMetaFile.get().getTime().toInstant(); + + DateUtil.DateRange incrementalDateRange = + new DateUtil.DateRange(snapshotTime, dateRange.getEndTime()); + Iterator incrementals = + metaProxy.getIncrementals(incrementalDateRange); + futureList.addAll(download(incrementals, false)); // Downloading CommitLogs + // Note for Backup V2.0 we do not backup commit logs, as saving them is cost-expensive. if (config.isBackingUpCommitLogs()) { logger.info( "Delete all backuped commitlog files in {}", @@ -271,15 +306,12 @@ public void restore(DateUtil.DateRange dateRange) throws Exception { logger.info("Delete all commitlog files in {}", config.getCommitLogLocation()); SystemUtils.cleanupDir(config.getCommitLogLocation(), null); - + String prefix = fs.getPrefix().toString(); Iterator commitLogPathIterator = - fs.list(prefix, meta.getTime(), endTime); + fs.list(prefix, latestValidMetaFile.get().getTime(), endTime); futureList.addAll( downloadCommitLogs( - commitLogPathIterator, - BackupFileType.CL, - config.maxCommitLogsRestore(), - false)); + commitLogPathIterator, config.maxCommitLogsRestore(), false)); } // Wait for all the futures to finish. diff --git a/priam/src/main/java/com/netflix/priam/services/SnapshotMetaService.java b/priam/src/main/java/com/netflix/priam/services/SnapshotMetaService.java index 63d695d9d..1cf62e885 100644 --- a/priam/src/main/java/com/netflix/priam/services/SnapshotMetaService.java +++ b/priam/src/main/java/com/netflix/priam/services/SnapshotMetaService.java @@ -61,7 +61,6 @@ public class SnapshotMetaService extends AbstractBackup { private static final String SNAPSHOT_PREFIX = "snap_v2_"; private static final String CASSANDRA_MANIFEST_FILE = "manifest.json"; private static final String CASSANDRA_SCHEMA_FILE = "schema.cql"; - private final IBackupRestoreConfig backupRestoreConfig; private final BackupRestoreUtil backupRestoreUtil; private final MetaFileWriterBuilder metaFileWriter; private MetaFileWriterBuilder.DataStep dataStep; @@ -80,14 +79,12 @@ private enum MetaStep { @Inject SnapshotMetaService( IConfiguration config, - IBackupRestoreConfig backupRestoreConfig, IFileSystemContext backupFileSystemCtx, Provider pathFactory, MetaFileWriterBuilder metaFileWriter, @Named("v2") IMetaProxy metaProxy, CassandraOperations cassandraOperations) { super(config, backupFileSystemCtx, pathFactory); - this.backupRestoreConfig = backupRestoreConfig; this.cassandraOperations = cassandraOperations; backupRestoreUtil = new BackupRestoreUtil( @@ -247,8 +244,7 @@ private void uploadAllFiles( if (!snapshotDirectory.getName().startsWith(SNAPSHOT_PREFIX) || !snapshotDirectory.isDirectory()) continue; - if (snapshotDirectory.list().length == 0 - || !backupRestoreConfig.enableV2Backups()) { + if (snapshotDirectory.list().length == 0) { FileUtils.cleanDirectory(snapshotDirectory); FileUtils.deleteDirectory(snapshotDirectory); continue; @@ -369,11 +365,6 @@ private void generateMetaFile( columnfamilyResult.getColumnfamilyName()); } - @Override - protected void addToRemotePath(String remotePath) { - // Do nothing - } - // For testing purposes only. void setSnapshotName(String snapshotName) { this.snapshotName = snapshotName; diff --git a/priam/src/test/java/com/netflix/priam/backup/FakeBackupFileSystem.java b/priam/src/test/java/com/netflix/priam/backup/FakeBackupFileSystem.java index b65589cd6..b55d385c5 100644 --- a/priam/src/test/java/com/netflix/priam/backup/FakeBackupFileSystem.java +++ b/priam/src/test/java/com/netflix/priam/backup/FakeBackupFileSystem.java @@ -151,7 +151,8 @@ protected void downloadFileImpl(Path remotePath, Path localPath) throws BackupRe try (FileWriter fr = new FileWriter(localPath.toFile())) { JSONArray jsonObj = new JSONArray(); for (AbstractBackupPath filePath : flist) { - if (filePath.type == AbstractBackupPath.BackupFileType.SNAP) { + if (filePath.type == AbstractBackupPath.BackupFileType.SNAP + && filePath.time.equals(path.time)) { jsonObj.add(filePath.getRemotePath()); } } diff --git a/priam/src/test/java/com/netflix/priam/backup/TestAbstractFileSystem.java b/priam/src/test/java/com/netflix/priam/backup/TestAbstractFileSystem.java index b05fe0d78..7bfeab6df 100644 --- a/priam/src/test/java/com/netflix/priam/backup/TestAbstractFileSystem.java +++ b/priam/src/test/java/com/netflix/priam/backup/TestAbstractFileSystem.java @@ -139,7 +139,7 @@ public void testUpload() throws Exception { @Test public void testDownload() throws Exception { // Dummy download - myFileSystem.downloadFile(Paths.get(""), null, 2); + myFileSystem.downloadFile(Paths.get(""), Paths.get(configuration.getDataFileLocation()), 2); // Verify the success metric for download is incremented. Assert.assertEquals(1, (int) backupMetrics.getValidDownloads().actualCount()); } @@ -254,7 +254,9 @@ public void testAsyncUploadFailure() throws Exception { @Test public void testAsyncDownload() throws Exception { // Testing single async download. - Future future = myFileSystem.asyncDownloadFile(Paths.get(""), null, 2); + Future future = + myFileSystem.asyncDownloadFile( + Paths.get(""), Paths.get(configuration.getDataFileLocation()), 2); future.get(); // 1. Verify the success metric for download is incremented. Assert.assertEquals(1, (int) backupMetrics.getValidDownloads().actualCount()); @@ -269,7 +271,9 @@ public void testAsyncDownloadBulk() throws Exception { int totalFiles = 1000; List> futureList = new ArrayList<>(); for (int i = 0; i < totalFiles; i++) - futureList.add(myFileSystem.asyncDownloadFile(Paths.get("" + i), null, 2)); + futureList.add( + myFileSystem.asyncDownloadFile( + Paths.get("" + i), Paths.get(configuration.getDataFileLocation()), 2)); // Ensure processing is finished. for (Future future1 : futureList) { diff --git a/priam/src/test/java/com/netflix/priam/backup/TestS3FileSystem.java b/priam/src/test/java/com/netflix/priam/backup/TestS3FileSystem.java index c78e818fa..e8f2f92e2 100644 --- a/priam/src/test/java/com/netflix/priam/backup/TestS3FileSystem.java +++ b/priam/src/test/java/com/netflix/priam/backup/TestS3FileSystem.java @@ -134,6 +134,7 @@ public void testFileUploadCompleteFailure() throws Exception { MockS3PartUploader.setup(); MockS3PartUploader.completionFailure = true; S3FileSystem fs = injector.getInstance(S3FileSystem.class); + fs.setS3Client(new MockAmazonS3Client().getMockInstance()); String snapshotfile = "target/data/Keyspace1/Standard1/backups/201108082320/Keyspace1-Standard1-ia-1-Data.db"; RemoteBackupPath backupfile = injector.getInstance(RemoteBackupPath.class); diff --git a/priam/src/test/java/com/netflix/priam/backupv2/TestForgottenFileManager.java b/priam/src/test/java/com/netflix/priam/backupv2/TestForgottenFileManager.java new file mode 100644 index 000000000..2f566d328 --- /dev/null +++ b/priam/src/test/java/com/netflix/priam/backupv2/TestForgottenFileManager.java @@ -0,0 +1,169 @@ +/* + * Copyright 2019 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.netflix.priam.backupv2; + +import com.google.inject.Guice; +import com.google.inject.Injector; +import com.netflix.priam.backup.BRTestModule; +import com.netflix.priam.config.FakeConfiguration; +import com.netflix.priam.config.IConfiguration; +import com.netflix.priam.merics.BackupMetrics; +import com.netflix.priam.utils.DateUtil; +import java.io.File; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.time.Instant; +import java.time.temporal.ChronoUnit; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.stream.Collectors; +import org.apache.commons.io.FileUtils; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +/** Created by aagrawal on 1/1/19. */ +public class TestForgottenFileManager { + private ForgottenFilesManager forgottenFilesManager; + private TestBackupUtils testBackupUtils; + private IConfiguration configuration; + private List allFiles = new ArrayList<>(); + private Instant snapshotInstant; + private Path snapshotDir; + + public TestForgottenFileManager() { + Injector injector = Guice.createInjector(new BRTestModule()); + BackupMetrics backupMetrics = injector.getInstance(BackupMetrics.class); + configuration = new ForgottenFilesConfiguration(); + forgottenFilesManager = new ForgottenFilesManager(configuration, backupMetrics); + testBackupUtils = injector.getInstance(TestBackupUtils.class); + } + + @Before + public void prep() throws Exception { + cleanup(); + Instant now = DateUtil.getInstant(); + snapshotInstant = now; + Path file1 = Paths.get(testBackupUtils.createFile("file1", now.minus(5, ChronoUnit.DAYS))); + Path file2 = Paths.get(testBackupUtils.createFile("file2", now.minus(3, ChronoUnit.DAYS))); + Path file3 = Paths.get(testBackupUtils.createFile("file3", now.minus(2, ChronoUnit.DAYS))); + Path file4 = Paths.get(testBackupUtils.createFile("file4", now.minus(6, ChronoUnit.HOURS))); + Path file5 = + Paths.get(testBackupUtils.createFile("file5", now.minus(1, ChronoUnit.MINUTES))); + Path file6 = + Paths.get( + testBackupUtils.createFile( + "tmplink-lb-59516-big-Index.db", now.minus(3, ChronoUnit.DAYS))); + Path file7 = + Paths.get(testBackupUtils.createFile("file7.tmp", now.minus(3, ChronoUnit.DAYS))); + + allFiles.add(file1); + allFiles.add(file2); + allFiles.add(file3); + allFiles.add(file4); + allFiles.add(file5); + allFiles.add(file6); + allFiles.add(file7); + + // Create a snapshot with file2, file3, file4. + Path columnfamilyDir = file1.getParent(); + snapshotDir = + Paths.get( + columnfamilyDir.toString(), + "snapshot", + "snap_v2_" + DateUtil.formatInstant(DateUtil.yyyyMMddHHmm, now)); + snapshotDir.toFile().mkdirs(); + Files.createLink(Paths.get(snapshotDir.toString(), file2.getFileName().toString()), file2); + Files.createLink(Paths.get(snapshotDir.toString(), file3.getFileName().toString()), file3); + Files.createLink(Paths.get(snapshotDir.toString(), file4.getFileName().toString()), file4); + } + + @After + public void cleanup() throws Exception { + String dataDir = configuration.getDataFileLocation(); + org.apache.commons.io.FileUtils.cleanDirectory(new File(dataDir)); + } + + @Test + public void testMoveForgottenFiles() { + Collection files = allFiles.stream().map(Path::toFile).collect(Collectors.toList()); + forgottenFilesManager.moveForgottenFiles( + new File(configuration.getDataFileLocation()), files); + Path lostFoundDir = + Paths.get(configuration.getDataFileLocation(), forgottenFilesManager.LOST_FOUND); + Collection movedFiles = FileUtils.listFiles(lostFoundDir.toFile(), null, false); + Assert.assertEquals(allFiles.size(), movedFiles.size()); + for (Path file : allFiles) + Assert.assertTrue( + movedFiles.contains( + Paths.get(lostFoundDir.toString(), file.getFileName().toString()) + .toFile())); + } + + @Test + public void getColumnfamilyFiles() { + Path columnfamilyDir = allFiles.get(0).getParent(); + Collection columnfamilyFiles = + forgottenFilesManager.getColumnfamilyFiles( + snapshotInstant, columnfamilyDir.toFile()); + Assert.assertEquals(3, columnfamilyFiles.size()); + Assert.assertTrue(columnfamilyFiles.contains(allFiles.get(0).toFile())); + Assert.assertTrue(columnfamilyFiles.contains(allFiles.get(1).toFile())); + Assert.assertTrue(columnfamilyFiles.contains(allFiles.get(2).toFile())); + } + + @Test + public void findAndMoveForgottenFiles() { + Path lostFoundDir = + Paths.get(allFiles.get(0).getParent().toString(), forgottenFilesManager.LOST_FOUND); + forgottenFilesManager.findAndMoveForgottenFiles(snapshotInstant, snapshotDir.toFile()); + + // Only one forgotten file - file1. + Collection movedFiles = FileUtils.listFiles(lostFoundDir.toFile(), null, false); + Assert.assertEquals(1, movedFiles.size()); + Assert.assertTrue( + movedFiles + .iterator() + .next() + .getName() + .equals(allFiles.get(0).getFileName().toString())); + + // All other files still remain in columnfamily dir. + Collection cfFiles = + FileUtils.listFiles(new File(allFiles.get(0).getParent().toString()), null, false); + Assert.assertEquals(6, cfFiles.size()); + int temp_file_name = 1; + for (File file : cfFiles) { + file.getName().equals(allFiles.get(temp_file_name++).getFileName().toString()); + } + + // Snapshot is untouched. + Collection snapshotFiles = FileUtils.listFiles(snapshotDir.toFile(), null, false); + Assert.assertEquals(3, snapshotFiles.size()); + } + + private class ForgottenFilesConfiguration extends FakeConfiguration { + @Override + public boolean isForgottenFileMoveEnabled() { + return true; + } + } +} diff --git a/priam/src/test/java/com/netflix/priam/backupv2/TestMetaFileManager.java b/priam/src/test/java/com/netflix/priam/backupv2/TestMetaFileManager.java deleted file mode 100644 index d816b3ac8..000000000 --- a/priam/src/test/java/com/netflix/priam/backupv2/TestMetaFileManager.java +++ /dev/null @@ -1,98 +0,0 @@ -/* - * Copyright 2018 Netflix, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.netflix.priam.backupv2; - -import com.google.inject.Guice; -import com.google.inject.Injector; -import com.netflix.priam.backup.BRTestModule; -import com.netflix.priam.config.IConfiguration; -import com.netflix.priam.utils.DateUtil; -import java.io.File; -import java.io.IOException; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.time.temporal.ChronoUnit; -import org.apache.commons.io.FileUtils; -import org.junit.After; -import org.junit.Assert; -import org.junit.Test; - -/** Created by aagrawal on 11/28/18. */ -public class TestMetaFileManager { - - private MetaV2Proxy metaV2Proxy; - private IConfiguration configuration; - - public TestMetaFileManager() { - Injector injector = Guice.createInjector(new BRTestModule()); - metaV2Proxy = injector.getInstance(MetaV2Proxy.class); - configuration = injector.getInstance(IConfiguration.class); - } - - @After - public void cleanup() throws IOException { - FileUtils.cleanDirectory(new File(configuration.getDataFileLocation())); - } - - @Test - public void testCleanupOldMetaFiles() throws IOException { - generateDummyMetaFiles(); - Path dataDir = Paths.get(configuration.getDataFileLocation()); - Assert.assertEquals(4, dataDir.toFile().listFiles().length); - - // clean the directory - metaV2Proxy.cleanupOldMetaFiles(); - - Assert.assertEquals(1, dataDir.toFile().listFiles().length); - Path dummy = Paths.get(dataDir.toString(), "dummy.tmp"); - Assert.assertTrue(dummy.toFile().exists()); - } - - private void generateDummyMetaFiles() throws IOException { - Path dataDir = Paths.get(configuration.getDataFileLocation()); - FileUtils.write( - Paths.get( - configuration.getDataFileLocation(), - MetaFileInfo.getMetaFileName(DateUtil.getInstant())) - .toFile(), - "dummy", - "UTF-8"); - - FileUtils.write( - Paths.get( - configuration.getDataFileLocation(), - MetaFileInfo.getMetaFileName( - DateUtil.getInstant().minus(10, ChronoUnit.MINUTES))) - .toFile(), - "dummy", - "UTF-8"); - - FileUtils.write( - Paths.get( - configuration.getDataFileLocation(), - MetaFileInfo.getMetaFileName(DateUtil.getInstant()) + ".tmp") - .toFile(), - "dummy", - "UTF-8"); - - FileUtils.write( - Paths.get(configuration.getDataFileLocation(), "dummy.tmp").toFile(), - "dummy", - "UTF-8"); - } -} diff --git a/priam/src/test/java/com/netflix/priam/backupv2/TestMetaV2Proxy.java b/priam/src/test/java/com/netflix/priam/backupv2/TestMetaV2Proxy.java index 5fe5ba14b..f434204d7 100644 --- a/priam/src/test/java/com/netflix/priam/backupv2/TestMetaV2Proxy.java +++ b/priam/src/test/java/com/netflix/priam/backupv2/TestMetaV2Proxy.java @@ -26,13 +26,18 @@ import com.netflix.priam.backup.FakeBackupFileSystem; import com.netflix.priam.config.IConfiguration; import com.netflix.priam.utils.DateUtil; +import java.io.File; +import java.io.IOException; import java.nio.file.Path; import java.nio.file.Paths; import java.time.Instant; +import java.time.temporal.ChronoUnit; import java.util.ArrayList; +import java.util.Iterator; import java.util.List; import java.util.stream.Collectors; import org.apache.commons.io.FileUtils; +import org.junit.After; import org.junit.Assert; import org.junit.Test; @@ -58,10 +63,11 @@ public TestMetaV2Proxy() { public void testMetaPrefix() { // Null date range Assert.assertEquals(getPrefix() + "/META_V2", metaProxy.getMetaPrefix(null)); + Instant now = Instant.now(); // No end date. Assert.assertEquals( - getPrefix() + "/META_V2", - metaProxy.getMetaPrefix(new DateUtil.DateRange(Instant.now(), null))); + getPrefix() + "/META_V2/" + now.toEpochMilli(), + metaProxy.getMetaPrefix(new DateUtil.DateRange(now, null))); // No start date Assert.assertEquals( getPrefix() + "/META_V2", @@ -106,6 +112,28 @@ public void testIsMetaFileValid() throws Exception { Assert.assertFalse(metaProxy.isMetaFileValid(abstractBackupPath).valid); } + @Test + public void testGetSSTFilesFromMeta() throws Exception { + Instant snapshotInstant = DateUtil.getInstant(); + List remoteFiles = getRemoteFakeFiles(); + Path metaPath = backupUtils.createMeta(remoteFiles, snapshotInstant); + List filesFromMeta = metaProxy.getSSTFilesFromMeta(metaPath); + filesFromMeta.removeAll(remoteFiles); + Assert.assertTrue(filesFromMeta.isEmpty()); + } + + @Test + public void testGetIncrementalFiles() throws Exception { + DateUtil.DateRange dateRange = new DateUtil.DateRange("202812071820,20281229"); + Iterator incrementals = metaProxy.getIncrementals(dateRange); + int i = 0; + while (incrementals.hasNext()) { + System.out.println(incrementals.next()); + i++; + } + Assert.assertEquals(3, i); + } + @Test public void testFindMetaFiles() throws BackupRestoreException { List metas = @@ -203,4 +231,57 @@ private List getRemoteFakeFiles() { "meta_v2_202812071901.json")); return files.stream().map(Path::toString).collect(Collectors.toList()); } + + @After + public void cleanup() throws IOException { + FileUtils.cleanDirectory(new File(configuration.getDataFileLocation())); + } + + @Test + public void testCleanupOldMetaFiles() throws IOException { + generateDummyMetaFiles(); + Path dataDir = Paths.get(configuration.getDataFileLocation()); + Assert.assertEquals(4, dataDir.toFile().listFiles().length); + + // clean the directory + metaProxy.cleanupOldMetaFiles(); + + Assert.assertEquals(1, dataDir.toFile().listFiles().length); + Path dummy = Paths.get(dataDir.toString(), "dummy.tmp"); + Assert.assertTrue(dummy.toFile().exists()); + } + + private void generateDummyMetaFiles() throws IOException { + Path dataDir = Paths.get(configuration.getDataFileLocation()); + FileUtils.cleanDirectory(dataDir.toFile()); + FileUtils.write( + Paths.get( + configuration.getDataFileLocation(), + MetaFileInfo.getMetaFileName(DateUtil.getInstant())) + .toFile(), + "dummy", + "UTF-8"); + + FileUtils.write( + Paths.get( + configuration.getDataFileLocation(), + MetaFileInfo.getMetaFileName( + DateUtil.getInstant().minus(10, ChronoUnit.MINUTES))) + .toFile(), + "dummy", + "UTF-8"); + + FileUtils.write( + Paths.get( + configuration.getDataFileLocation(), + MetaFileInfo.getMetaFileName(DateUtil.getInstant()) + ".tmp") + .toFile(), + "dummy", + "UTF-8"); + + FileUtils.write( + Paths.get(configuration.getDataFileLocation(), "dummy.tmp").toFile(), + "dummy", + "UTF-8"); + } } diff --git a/priam/src/test/java/com/netflix/priam/config/FakeBackupRestoreConfig.java b/priam/src/test/java/com/netflix/priam/config/FakeBackupRestoreConfig.java index 7339d5b04..246a91a86 100644 --- a/priam/src/test/java/com/netflix/priam/config/FakeBackupRestoreConfig.java +++ b/priam/src/test/java/com/netflix/priam/config/FakeBackupRestoreConfig.java @@ -24,4 +24,9 @@ public String getSnapshotMetaServiceCronExpression() { public boolean enableV2Backups() { return true; } + + @Override + public boolean enableV2Restore() { + return false; + } } diff --git a/priam/src/test/java/com/netflix/priam/backup/TestRestore.java b/priam/src/test/java/com/netflix/priam/restore/TestRestore.java similarity index 71% rename from priam/src/test/java/com/netflix/priam/backup/TestRestore.java rename to priam/src/test/java/com/netflix/priam/restore/TestRestore.java index 3a931f6bc..06a21b1a6 100644 --- a/priam/src/test/java/com/netflix/priam/backup/TestRestore.java +++ b/priam/src/test/java/com/netflix/priam/restore/TestRestore.java @@ -1,5 +1,5 @@ /* - * Copyright 2017 Netflix, Inc. + * Copyright 2018 Netflix, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,15 +15,17 @@ * */ -package com.netflix.priam.backup; +package com.netflix.priam.restore; import com.google.inject.Guice; import com.google.inject.Injector; +import com.netflix.priam.backup.BRTestModule; +import com.netflix.priam.backup.FakeBackupFileSystem; +import com.netflix.priam.backup.Status; import com.netflix.priam.config.FakeConfiguration; import com.netflix.priam.config.IConfiguration; import com.netflix.priam.health.InstanceState; import com.netflix.priam.identity.config.InstanceInfo; -import com.netflix.priam.restore.Restore; import com.netflix.priam.utils.DateUtil; import java.io.IOException; import java.util.ArrayList; @@ -78,20 +80,56 @@ public void testRestore() throws Exception { Assert.assertEquals(Status.FINISHED, instanceState.getRestoreStatus().getStatus()); } - // Pick latest file + @Test + public void testRestoreWithIncremental() throws Exception { + populateBackupFileSystem("test_backup"); + String dateRange = "201108110030,201108110730"; + restore.restore(new DateUtil.DateRange(dateRange)); + Assert.assertTrue(filesystem.downloadedFiles.contains(fileList.get(0))); + Assert.assertTrue(filesystem.downloadedFiles.contains(fileList.get(1))); + Assert.assertTrue(filesystem.downloadedFiles.contains(fileList.get(2))); + Assert.assertTrue(filesystem.downloadedFiles.contains(fileList.get(3))); + Assert.assertTrue(filesystem.downloadedFiles.contains(fileList.get(4))); + Assert.assertTrue(filesystem.downloadedFiles.contains(fileList.get(5))); + Assert.assertEquals(Status.FINISHED, instanceState.getRestoreStatus().getStatus()); + } + + @Test + public void testRestoreLatestWithEmptyMeta() throws Exception { + populateBackupFileSystem("test_backup"); + String metafile = + "test_backup/" + region + "/fakecluster/123456/201108110130/META/meta.json"; + filesystem.addFile(metafile); + String dateRange = "201108110030,201108110530"; + restore.restore(new DateUtil.DateRange(dateRange)); + Assert.assertFalse(filesystem.downloadedFiles.contains(fileList.get(0))); + Assert.assertTrue(filesystem.downloadedFiles.contains(metafile)); + Assert.assertFalse(filesystem.downloadedFiles.contains(fileList.get(1))); + Assert.assertFalse(filesystem.downloadedFiles.contains(fileList.get(2))); + Assert.assertFalse(filesystem.downloadedFiles.contains(fileList.get(3))); + Assert.assertFalse(filesystem.downloadedFiles.contains(fileList.get(4))); + Assert.assertFalse(filesystem.downloadedFiles.contains(fileList.get(5))); + Assert.assertEquals(Status.FINISHED, instanceState.getRestoreStatus().getStatus()); + Assert.assertEquals(metafile, instanceState.getRestoreStatus().getSnapshotMetaFile()); + } + @Test public void testRestoreLatest() throws Exception { populateBackupFileSystem("test_backup"); String metafile = "test_backup/" + region + "/fakecluster/123456/201108110130/META/meta.json"; filesystem.addFile(metafile); + String snapFile = + "test_backup/" + region + "/fakecluster/123456/201108110130/SNAP/ks1/cf1/f9.db"; + filesystem.addFile(snapFile); String dateRange = "201108110030,201108110530"; restore.restore(new DateUtil.DateRange(dateRange)); Assert.assertFalse(filesystem.downloadedFiles.contains(fileList.get(0))); Assert.assertTrue(filesystem.downloadedFiles.contains(metafile)); - Assert.assertTrue(filesystem.downloadedFiles.contains(fileList.get(1))); - Assert.assertTrue(filesystem.downloadedFiles.contains(fileList.get(2))); - Assert.assertTrue(filesystem.downloadedFiles.contains(fileList.get(3))); + Assert.assertTrue(filesystem.downloadedFiles.contains(snapFile)); + Assert.assertFalse(filesystem.downloadedFiles.contains(fileList.get(1))); + Assert.assertFalse(filesystem.downloadedFiles.contains(fileList.get(2))); + Assert.assertFalse(filesystem.downloadedFiles.contains(fileList.get(3))); Assert.assertFalse(filesystem.downloadedFiles.contains(fileList.get(4))); Assert.assertFalse(filesystem.downloadedFiles.contains(fileList.get(5))); Assert.assertEquals(Status.FINISHED, instanceState.getRestoreStatus().getStatus()); From daf5bbba6f729ba34e6b7612211a31afdba55427 Mon Sep 17 00:00:00 2001 From: Vinay Chella Date: Fri, 11 Jan 2019 15:43:22 -0800 Subject: [PATCH 046/228] Add metric on CassandraConfig resource calls (#766) * Add metric on CassandraConfig resource calls --- .../priam/merics/CassMonitorMetrics.java | 37 +++++++++++++++++++ .../priam/resources/CassandraConfig.java | 14 +++++-- .../priam/resources/CassandraConfigTest.java | 7 +++- 3 files changed, 54 insertions(+), 4 deletions(-) diff --git a/priam/src/main/java/com/netflix/priam/merics/CassMonitorMetrics.java b/priam/src/main/java/com/netflix/priam/merics/CassMonitorMetrics.java index a4338964a..919789689 100644 --- a/priam/src/main/java/com/netflix/priam/merics/CassMonitorMetrics.java +++ b/priam/src/main/java/com/netflix/priam/merics/CassMonitorMetrics.java @@ -17,17 +17,38 @@ import com.google.inject.Singleton; import com.netflix.spectator.api.Gauge; import com.netflix.spectator.api.Registry; +import com.netflix.spectator.api.patterns.PolledMeter; +import java.util.concurrent.atomic.AtomicLong; /** @author vchella */ @Singleton public class CassMonitorMetrics { private final Gauge cassStop, cassAutoStart, cassStart; + private final AtomicLong getSeedsCnt, getTokenCnt, getReplacedIpCnt, doubleRingCnt; @Inject public CassMonitorMetrics(Registry registry) { cassStop = registry.gauge(Metrics.METRIC_PREFIX + "cass.stop"); cassStart = registry.gauge(Metrics.METRIC_PREFIX + "cass.start"); cassAutoStart = registry.gauge(Metrics.METRIC_PREFIX + "cass.auto.start"); + + getSeedsCnt = + PolledMeter.using(registry) + .withName(Metrics.METRIC_PREFIX + "cass.getSeedCnt") + .monitorMonotonicCounter(new AtomicLong(0)); + getTokenCnt = + PolledMeter.using(registry) + .withName(Metrics.METRIC_PREFIX + "cass.getTokenCnt") + .monitorMonotonicCounter(new AtomicLong(0)); + getReplacedIpCnt = + PolledMeter.using(registry) + .withName(Metrics.METRIC_PREFIX + "cass.getReplacedIpCnt") + .monitorMonotonicCounter(new AtomicLong(0)); + + doubleRingCnt = + PolledMeter.using(registry) + .withName(Metrics.METRIC_PREFIX + "cass.doubleRingCnt") + .monitorMonotonicCounter(new AtomicLong(0)); } public void incCassStop() { @@ -41,4 +62,20 @@ public void incCassAutoStart() { public void incCassStart() { cassStart.set(cassStart.value() + 1); } + + public void incGetSeeds() { + getSeedsCnt.incrementAndGet(); + } + + public void incGetToken() { + getTokenCnt.incrementAndGet(); + } + + public void incGetReplacedIp() { + getReplacedIpCnt.incrementAndGet(); + } + + public void incDoubleRing() { + doubleRingCnt.incrementAndGet(); + } } diff --git a/priam/src/main/java/com/netflix/priam/resources/CassandraConfig.java b/priam/src/main/java/com/netflix/priam/resources/CassandraConfig.java index c19a969e9..206cedbe5 100644 --- a/priam/src/main/java/com/netflix/priam/resources/CassandraConfig.java +++ b/priam/src/main/java/com/netflix/priam/resources/CassandraConfig.java @@ -19,6 +19,7 @@ import com.google.inject.Inject; import com.netflix.priam.PriamServer; import com.netflix.priam.identity.DoubleRing; +import com.netflix.priam.merics.CassMonitorMetrics; import java.io.IOException; import java.util.HashMap; import java.util.List; @@ -44,11 +45,13 @@ public class CassandraConfig { private static final Logger logger = LoggerFactory.getLogger(CassandraConfig.class); private final PriamServer priamServer; private final DoubleRing doubleRing; + private final CassMonitorMetrics metrics; @Inject - public CassandraConfig(PriamServer server, DoubleRing doubleRing) { + public CassandraConfig(PriamServer server, DoubleRing doubleRing, CassMonitorMetrics metrics) { this.priamServer = server; this.doubleRing = doubleRing; + this.metrics = metrics; } @GET @@ -56,7 +59,10 @@ public CassandraConfig(PriamServer server, DoubleRing doubleRing) { public Response getSeeds() { try { final List seeds = priamServer.getInstanceIdentity().getSeeds(); - if (!seeds.isEmpty()) return Response.ok(StringUtils.join(seeds, ',')).build(); + if (!seeds.isEmpty()) { + metrics.incGetSeeds(); + return Response.ok(StringUtils.join(seeds, ',')).build(); + } logger.error("Cannot find the Seeds"); } catch (Exception e) { logger.error("Error while executing get_seeds", e); @@ -72,6 +78,7 @@ public Response getToken() { String token = priamServer.getInstanceIdentity().getInstance().getToken(); if (StringUtils.isNotBlank(token)) { logger.info("Returning token value \"{}\" for this instance to caller.", token); + metrics.incGetToken(); return Response.ok(priamServer.getInstanceIdentity().getInstance().getToken()) .build(); } @@ -102,6 +109,7 @@ public Response isReplaceToken() { @Path("/get_replaced_ip") public Response getReplacedIp() { try { + metrics.incGetReplacedIp(); return Response.ok(String.valueOf(priamServer.getInstanceIdentity().getReplacedIp())) .build(); } catch (Exception e) { @@ -114,7 +122,6 @@ public Response getReplacedIp() { @Path("/set_replaced_ip") public Response setReplacedIp(@QueryParam("ip") String ip) { try { - priamServer.getInstanceIdentity().setReplacedIp(ip); return Response.ok().build(); } catch (Exception e) { logger.error("Error while overriding replacement ip", e); @@ -143,6 +150,7 @@ public Response getExtraEnvParams() { @Path("/double_ring") public Response doubleRing() throws IOException, ClassNotFoundException { try { + metrics.incDoubleRing(); doubleRing.backup(); doubleRing.doubleSlots(); } catch (Throwable th) { diff --git a/priam/src/test/java/com/netflix/priam/resources/CassandraConfigTest.java b/priam/src/test/java/com/netflix/priam/resources/CassandraConfigTest.java index ae0b74939..d01991219 100644 --- a/priam/src/test/java/com/netflix/priam/resources/CassandraConfigTest.java +++ b/priam/src/test/java/com/netflix/priam/resources/CassandraConfigTest.java @@ -21,10 +21,13 @@ import static org.junit.Assert.fail; import com.google.common.collect.ImmutableList; +import com.google.inject.Guice; import com.netflix.priam.PriamServer; +import com.netflix.priam.backup.BRTestModule; import com.netflix.priam.identity.DoubleRing; import com.netflix.priam.identity.InstanceIdentity; import com.netflix.priam.identity.PriamInstance; +import com.netflix.priam.merics.CassMonitorMetrics; import java.io.IOException; import java.net.UnknownHostException; import java.util.List; @@ -44,7 +47,9 @@ public class CassandraConfigTest { @Before public void setUp() { - resource = new CassandraConfig(priamServer, doubleRing); + CassMonitorMetrics cassMonitorMetrics = + Guice.createInjector(new BRTestModule()).getInstance(CassMonitorMetrics.class); + resource = new CassandraConfig(priamServer, doubleRing, cassMonitorMetrics); } @Test From 62557f87c2a8709df994e7bdf12a819a05c273e9 Mon Sep 17 00:00:00 2001 From: Arun Agrawal Date: Fri, 18 Jan 2019 21:12:58 -0800 Subject: [PATCH 047/228] Support configure/tune complex parameters in cassandra.yaml. --- .../netflix/priam/tuner/StandardTuner.java | 31 ++++-- .../priam/tuner/StandardTunerTest.java | 94 ++++++++++++++++++- 2 files changed, 114 insertions(+), 11 deletions(-) diff --git a/priam/src/main/java/com/netflix/priam/tuner/StandardTuner.java b/priam/src/main/java/com/netflix/priam/tuner/StandardTuner.java index edeb7d239..c0746ae78 100644 --- a/priam/src/main/java/com/netflix/priam/tuner/StandardTuner.java +++ b/priam/src/main/java/com/netflix/priam/tuner/StandardTuner.java @@ -20,6 +20,7 @@ import com.netflix.priam.identity.config.InstanceInfo; import com.netflix.priam.restore.Restore; import java.io.*; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Properties; @@ -245,12 +246,30 @@ public void addExtraCassParams(Map map) { String priamKey = pair[0]; String cassKey = pair[1]; String cassVal = config.getCassYamlVal(priamKey); - logger.info( - "Updating yaml: Priamkey[{}], CassKey[{}], Val[{}]", - priamKey, - cassKey, - cassVal); - map.put(cassKey, cassVal); + + if (!StringUtils.isBlank(cassKey) && !StringUtils.isBlank(cassVal)) { + if (!cassKey.contains(".")) { + logger.info( + "Updating yaml: PriamKey: [{}], Key: [{}], OldValue: [{}], NewValue: [{}]", + priamKey, + cassKey, + map.get(cassKey), + cassVal); + map.put(cassKey, cassVal); + } else { + // split the cassandra key. We will get the group and get the key name. + String[] cassKeySplit = cassKey.split("\\."); + Map cassKeyMap = ((Map) map.getOrDefault(cassKeySplit[0], new HashMap())); + map.putIfAbsent(cassKeySplit[0], cassKeyMap); + logger.info( + "Updating yaml: PriamKey: [{}], Key: [{}], OldValue: [{}], NewValue: [{}]", + priamKey, + cassKey, + cassKeyMap.get(cassKeySplit[1]), + cassVal); + cassKeyMap.put(cassKeySplit[1], cassVal); + } + } } } } diff --git a/priam/src/test/java/com/netflix/priam/tuner/StandardTunerTest.java b/priam/src/test/java/com/netflix/priam/tuner/StandardTunerTest.java index fd9f107ec..50c3d86ed 100644 --- a/priam/src/test/java/com/netflix/priam/tuner/StandardTunerTest.java +++ b/priam/src/test/java/com/netflix/priam/tuner/StandardTunerTest.java @@ -21,8 +21,19 @@ import com.google.common.io.Files; import com.google.inject.Guice; import com.netflix.priam.backup.BRTestModule; +import com.netflix.priam.config.FakeConfiguration; +import com.netflix.priam.identity.config.InstanceInfo; import java.io.File; +import java.io.FileInputStream; +import java.util.HashMap; +import java.util.Map; +import org.apache.commons.io.FileUtils; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; import org.junit.Test; +import org.yaml.snakeyaml.DumperOptions; +import org.yaml.snakeyaml.Yaml; public class StandardTunerTest { /* note: these are, more or less, arbitrary partitioner class names. as long as the tests exercise the code, all is good */ @@ -32,9 +43,13 @@ public class StandardTunerTest { private static final String BOP_PARTITIONER = "org.apache.cassandra.dht.ByteOrderedPartitioner"; private final StandardTuner tuner; + private final InstanceInfo instanceInfo; + private final File target = new File("/tmp/priam_test.yaml"); public StandardTunerTest() { this.tuner = Guice.createInjector(new BRTestModule()).getInstance(StandardTuner.class); + this.instanceInfo = + Guice.createInjector(new BRTestModule()).getInstance(InstanceInfo.class); } @Test @@ -73,12 +88,81 @@ public void derivePartitioner_BOPPartitionerInConfig() { assertEquals(BOP_PARTITIONER, partitioner); } + @Before + @After + public void cleanup() { + FileUtils.deleteQuietly(target); + } + @Test public void dump() throws Exception { - String target = "/tmp/priam_test.yaml"; - Files.copy( - new File("src/main/resources/incr-restore-cassandra.yaml"), - new File("/tmp/priam_test.yaml")); - tuner.writeAllProperties(target, "your_host", "YourSeedProvider"); + Files.copy(new File("src/main/resources/incr-restore-cassandra.yaml"), target); + tuner.writeAllProperties(target.getAbsolutePath(), "your_host", "YourSeedProvider"); + } + + @Test + public void addExtraParams() throws Exception { + String cassParamName1 = "client_encryption_options.optional"; + String priamKeyName1 = "Priam.client_encryption.optional"; + String cassParamName2 = "client_encryption_options.keystore_password"; + String priamKeyName2 = "Priam.client_encryption.keystore_password"; + String cassParamName3 = "randomKey"; + String priamKeyName3 = "Priam.randomKey"; + String cassParamName4 = "randomGroup.randomKey"; + String priamKeyName4 = "Priam.randomGroup.randomKey"; + + String extraConfigParam = + String.format( + "%s=%s,%s=%s,%s=%s,%s=%s", + priamKeyName1, + cassParamName1, + priamKeyName2, + cassParamName2, + priamKeyName3, + cassParamName3, + priamKeyName4, + cassParamName4); + Map extraParamValues = new HashMap(); + extraParamValues.put(priamKeyName1, true); + extraParamValues.put(priamKeyName2, "test"); + extraParamValues.put(priamKeyName3, "randomKeyValue"); + extraParamValues.put(priamKeyName4, "randomGroupValue"); + StandardTuner tuner = + new StandardTuner( + new TunerConfiguration(extraConfigParam, extraParamValues), instanceInfo); + Files.copy(new File("src/main/resources/incr-restore-cassandra.yaml"), target); + tuner.writeAllProperties(target.getAbsolutePath(), "your_host", "YourSeedProvider"); + + // Read the tuned file and verify + DumperOptions options = new DumperOptions(); + options.setDefaultFlowStyle(DumperOptions.FlowStyle.BLOCK); + Yaml yaml = new Yaml(options); + Map map = yaml.load(new FileInputStream(target)); + Assert.assertEquals("your_host", map.get("listen_address")); + Assert.assertEquals("true", ((Map) map.get("client_encryption_options")).get("optional")); + Assert.assertEquals( + "test", ((Map) map.get("client_encryption_options")).get("keystore_password")); + Assert.assertEquals("randomKeyValue", map.get("randomKey")); + Assert.assertEquals("randomGroupValue", ((Map) map.get("randomGroup")).get("randomKey")); + } + + private class TunerConfiguration extends FakeConfiguration { + String extraConfigParams; + Map extraParamValues; + + TunerConfiguration(String extraConfigParam, Map extraParamValues) { + this.extraConfigParams = extraConfigParam; + this.extraParamValues = extraParamValues; + } + + @Override + public String getCassYamlVal(String priamKey) { + return extraParamValues.getOrDefault(priamKey, "").toString(); + } + + @Override + public String getExtraConfigParams() { + return extraConfigParams; + } } } From 3c81e9004b2746b70710986ca24f804eb867bca8 Mon Sep 17 00:00:00 2001 From: Arun Agrawal Date: Fri, 18 Jan 2019 20:21:39 -0800 Subject: [PATCH 048/228] Add Cass SNAPSHOT JMX status, snapshot version, last validated timestamp. Changes to Servlet API. --- build.gradle | 10 +- .../netflix/priam/backup/BackupMetadata.java | 63 +++++- .../netflix/priam/backup/BackupStatusMgr.java | 87 ++++++-- .../priam/backup/BackupVerification.java | 71 +++++-- .../priam/backup/FileSnapshotStatusMgr.java | 4 +- .../priam/backup/IBackupStatusMgr.java | 19 ++ .../netflix/priam/backup/SnapshotBackup.java | 4 +- .../priam/backupv2/BackupValidator.java | 65 ------ .../priam/backupv2/MetaFileWriterBuilder.java | 11 +- .../netflix/priam/config/IConfiguration.java | 11 + .../priam/config/PriamConfiguration.java | 5 + .../AWSSnsNotificationService.java | 2 +- .../priam/resources/BackupServlet.java | 104 ++-------- .../priam/resources/BackupServletV2.java | 78 +++++++ .../priam/services/BackupTTLService.java | 5 +- .../priam/services/SnapshotMetaService.java | 24 ++- ...tatusMgr.java => TestBackupStatusMgr.java} | 118 ++++++++++- .../priam/backup/TestBackupVerification.java | 190 +++++++++++++++--- .../priam/config/FakeBackupRestoreConfig.java | 2 +- .../priam/services/TestBackupTTLService.java | 3 - 20 files changed, 644 insertions(+), 232 deletions(-) delete mode 100644 priam/src/main/java/com/netflix/priam/backupv2/BackupValidator.java create mode 100644 priam/src/main/java/com/netflix/priam/resources/BackupServletV2.java rename priam/src/test/java/com/netflix/priam/backup/{TestSnapshotStatusMgr.java => TestBackupStatusMgr.java} (51%) diff --git a/build.gradle b/build.gradle index f7554e4f6..970aac5e2 100644 --- a/build.gradle +++ b/build.gradle @@ -42,7 +42,7 @@ allprojects { compile 'com.sun.jersey.contribs:jersey-guice:1.19.4' compile 'com.google.guava:guava:21.0' compile 'com.google.code.findbugs:jsr305:3.0.2' - compile 'com.amazonaws:aws-java-sdk:1.11.475' + compile 'com.amazonaws:aws-java-sdk:1.11.487' compile 'com.google.inject:guice:4.2.2' compile 'com.google.inject.extensions:guice-servlet:4.2.2' compile 'org.quartz-scheduler:quartz:2.3.0' @@ -56,19 +56,19 @@ allprojects { compile 'xerces:xercesImpl:2.12.0' compile 'net.java.dev.jna:jna:5.2.0' compile 'org.apache.httpcomponents:httpclient:4.5.6' - compile 'org.apache.httpcomponents:httpcore:4.4.10' + compile 'org.apache.httpcomponents:httpcore:4.4.11' compile 'com.ning:compress-lzf:1.0.4' compile 'com.google.code.gson:gson:2.8.5' compile 'org.slf4j:slf4j-api:1.7.25' compile 'org.slf4j:slf4j-log4j12:1.7.25' compile 'org.bouncycastle:bcprov-jdk16:1.46' compile 'org.bouncycastle:bcpg-jdk16:1.46' - compile ('com.google.appengine.tools:appengine-gcs-client:0.7') { + compile ('com.google.appengine.tools:appengine-gcs-client:0.8') { exclude module: 'guava' } compile 'com.google.apis:google-api-services-storage:v1-rev141-1.25.0' - compile 'com.google.http-client:google-http-client-jackson2:1.22.0' - compile 'com.netflix.spectator:spectator-api:0.82.0' + compile 'com.google.http-client:google-http-client-jackson2:1.28.0' + compile 'com.netflix.spectator:spectator-api:0.83.0' compileOnly 'javax.servlet:javax.servlet-api:3.1.0' testCompile 'org.jmockit:jmockit:1.31' testCompile "org.spockframework:spock-core:1.1-groovy-2.4" diff --git a/priam/src/main/java/com/netflix/priam/backup/BackupMetadata.java b/priam/src/main/java/com/netflix/priam/backup/BackupMetadata.java index ff81804a4..d95d4989b 100644 --- a/priam/src/main/java/com/netflix/priam/backup/BackupMetadata.java +++ b/priam/src/main/java/com/netflix/priam/backup/BackupMetadata.java @@ -29,19 +29,27 @@ public final class BackupMetadata implements Serializable { private String token; private Date start, completed; private Status status; + private boolean cassandraSnapshotSuccess; + private Date lastValidated; + private int backupVersion; private String snapshotLocation; - public BackupMetadata(String token, Date start) throws Exception { + public BackupMetadata(int backupVersion, String token, Date start) throws Exception { if (start == null || token == null || StringUtils.isEmpty(token)) throw new Exception( String.format( "Invalid Input: Token: %s or start date: %s is null or empty.", token, start)); - + this.backupVersion = backupVersion; this.snapshotDate = DateUtil.formatyyyyMMdd(start); this.token = token; this.start = start; this.status = Status.STARTED; + this.cassandraSnapshotSuccess = false; + } + + public BackupMetadata(String token, Date start) throws Exception { + this(1, token, start); } @Override @@ -53,7 +61,8 @@ public boolean equals(Object o) { return this.snapshotDate.equals(that.snapshotDate) && this.token.equals(that.token) - && this.start.equals(that.start); + && this.start.equals(that.start) + && this.backupVersion == that.backupVersion; } @Override @@ -145,6 +154,54 @@ public void setSnapshotLocation(String snapshotLocation) { this.snapshotLocation = snapshotLocation; } + /** + * Find if cassandra snapshot was successful or not. This is a JMX operation and it is possible + * that this operation failed. + * + * @return cassandra snapshot status. + */ + public boolean isCassandraSnapshotSuccess() { + return cassandraSnapshotSuccess; + } + + /** + * Set the cassandra snapshot status to be either finished successfully or fail. + * + * @param cassandraSnapshotSuccess is set to success if JMX operation for snapshot is + * successful. + */ + public void setCassandraSnapshotSuccess(boolean cassandraSnapshotSuccess) { + this.cassandraSnapshotSuccess = cassandraSnapshotSuccess; + } + + /** + * Get the backup version for the snapshot. + * + * @return backup version of the snapshot. + */ + public int getBackupVersion() { + return backupVersion; + } + + /** + * Return the last validation timestamp of this backup metadata. Validation of backup implies + * finding if all the files are successfully stored in remote file system. + * + * @return date of last backup validation. + */ + public Date getLastValidated() { + return lastValidated; + } + + /** + * Set the last validation date of backup metadata. + * + * @param lastValidated date value of backup validation. + */ + public void setLastValidated(Date lastValidated) { + this.lastValidated = lastValidated; + } + @Override public String toString() { return GsonJsonSerializer.getGson().toJson(this); diff --git a/priam/src/main/java/com/netflix/priam/backup/BackupStatusMgr.java b/priam/src/main/java/com/netflix/priam/backup/BackupStatusMgr.java index 9a2441413..d29584ea8 100644 --- a/priam/src/main/java/com/netflix/priam/backup/BackupStatusMgr.java +++ b/priam/src/main/java/com/netflix/priam/backup/BackupStatusMgr.java @@ -18,7 +18,10 @@ import com.netflix.priam.health.InstanceState; import com.netflix.priam.utils.DateUtil; import com.netflix.priam.utils.MaxSizeHashMap; +import java.time.Instant; +import java.time.temporal.ChronoUnit; import java.util.*; +import java.util.stream.Collectors; import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -111,7 +114,11 @@ public void finish(BackupMetadata backupMetadata) { Calendar.getInstance(TimeZone.getTimeZone("GMT")).getTime()); instanceState.setBackupStatus(backupMetadata); + update(backupMetadata); + } + @Override + public void update(BackupMetadata backupMetadata) { // Retrieve the snapshot metadata and then update the finish date/status. retrieveAndUpdate(backupMetadata); @@ -123,19 +130,29 @@ private void retrieveAndUpdate(final BackupMetadata backupMetadata) { // Retrieve the snapshot metadata and then update the date/status. LinkedList metadataLinkedList = locate(backupMetadata.getSnapshotDate()); - if (metadataLinkedList == null || metadataLinkedList.isEmpty()) { + if (metadataLinkedList == null) { logger.error( "No previous backupMetaData found. This should not happen. Creating new to ensure app keeps running."); metadataLinkedList = new LinkedList<>(); - metadataLinkedList.addFirst(backupMetadata); + backupMetadataMap.put(backupMetadata.getSnapshotDate(), metadataLinkedList); } - metadataLinkedList.forEach( + Optional searchedData = + metadataLinkedList + .stream() + .filter(backupMetadata1 -> backupMetadata.equals(backupMetadata1)) + .findFirst(); + if (!searchedData.isPresent()) { + metadataLinkedList.addFirst(backupMetadata); + } + searchedData.ifPresent( backupMetadata1 -> { - if (backupMetadata1.equals(backupMetadata)) { - backupMetadata1.setCompleted(backupMetadata.getCompleted()); - backupMetadata1.setStatus(backupMetadata.getStatus()); - } + backupMetadata1.setCompleted(backupMetadata.getCompleted()); + backupMetadata1.setStatus(backupMetadata.getStatus()); + backupMetadata1.setCassandraSnapshotSuccess( + backupMetadata.isCassandraSnapshotSuccess()); + backupMetadata1.setSnapshotLocation(backupMetadata.getSnapshotLocation()); + backupMetadata1.setLastValidated(backupMetadata.getLastValidated()); }); } @@ -150,12 +167,7 @@ public void failed(BackupMetadata backupMetadata) { if (backupMetadata.getStatus() != Status.FAILED) backupMetadata.setStatus(Status.FAILED); instanceState.setBackupStatus(backupMetadata); - - // Retrieve the snapshot metadata and then update the failure date/status. - retrieveAndUpdate(backupMetadata); - - // Save the backupMetaDataMap - save(backupMetadata); + update(backupMetadata); } /** @@ -174,6 +186,55 @@ public void failed(BackupMetadata backupMetadata) { */ protected abstract LinkedList fetch(String snapshotDate); + public List getLatestBackupMetadata( + int snapshotVersion, DateUtil.DateRange dateRange) { + Instant startTime = dateRange.getStartTime().truncatedTo(ChronoUnit.DAYS); + Instant endTime = dateRange.getEndTime().truncatedTo(ChronoUnit.DAYS); + + List allBackups = new ArrayList<>(); + Instant previousDay = endTime; + do { + // We need to find the latest backupmetadata in this date range. + logger.info( + "Will try to find snapshot for : {}", + DateUtil.formatInstant(DateUtil.yyyyMMddHHmm, previousDay)); + List backupsForDate = locate(new Date(previousDay.toEpochMilli())); + if (backupsForDate != null) allBackups.addAll(backupsForDate); + previousDay = previousDay.minus(1, ChronoUnit.DAYS); + } while (!previousDay.isBefore(startTime)); + + // Return all the backups which are FINISHED and were "started" in the dateRange provided. + // Do not compare the end time of snapshot as it may take random amount of time to finish + // the snapshot. + return allBackups + .stream() + .filter(backupMetadata -> backupMetadata != null) + .filter(backupMetadata -> backupMetadata.getStatus() == Status.FINISHED) + .filter(backupMetadata -> backupMetadata.getBackupVersion() == snapshotVersion) + .filter( + backupMetadata -> + backupMetadata + .getStart() + .toInstant() + .isAfter(dateRange.getStartTime()) + || backupMetadata + .getStart() + .toInstant() + .equals(dateRange.getStartTime())) + .filter( + backupMetadata -> + backupMetadata + .getStart() + .toInstant() + .isBefore(dateRange.getEndTime()) + || backupMetadata + .getStart() + .toInstant() + .equals(dateRange.getEndTime())) + .sorted(Comparator.comparing(BackupMetadata::getStart).reversed()) + .collect(Collectors.toList()); + } + @Override public String toString() { return "BackupStatusMgr{" diff --git a/priam/src/main/java/com/netflix/priam/backup/BackupVerification.java b/priam/src/main/java/com/netflix/priam/backup/BackupVerification.java index 13e8d881c..1986c15a5 100644 --- a/priam/src/main/java/com/netflix/priam/backup/BackupVerification.java +++ b/priam/src/main/java/com/netflix/priam/backup/BackupVerification.java @@ -18,6 +18,10 @@ import com.google.inject.Singleton; import com.google.inject.name.Named; import com.netflix.priam.backupv2.IMetaProxy; +import com.netflix.priam.scheduler.UnsupportedTypeException; +import com.netflix.priam.services.SnapshotMetaService; +import com.netflix.priam.utils.DateUtil; +import com.netflix.priam.utils.DateUtil.DateRange; import java.nio.file.Path; import java.nio.file.Paths; import java.util.*; @@ -33,39 +37,72 @@ public class BackupVerification { private static final Logger logger = LoggerFactory.getLogger(BackupVerification.class); - private final IMetaProxy metaProxy; + private final IMetaProxy metaV1Proxy; + private final IMetaProxy metaV2Proxy; + private final IBackupStatusMgr backupStatusMgr; private final Provider abstractBackupPathProvider; @Inject BackupVerification( - @Named("v1") IMetaProxy metaProxy, + @Named("v1") IMetaProxy metaV1Proxy, + @Named("v2") IMetaProxy metaV2Proxy, + IBackupStatusMgr backupStatusMgr, Provider abstractBackupPathProvider) { - this.metaProxy = metaProxy; + this.metaV1Proxy = metaV1Proxy; + this.metaV2Proxy = metaV2Proxy; + this.backupStatusMgr = backupStatusMgr; this.abstractBackupPathProvider = abstractBackupPathProvider; } - public Optional getLatestBackupMetaData(List metadata) { - return metadata.stream() - .filter(backupMetadata -> backupMetadata != null) - .filter(backupMetadata -> backupMetadata.getStatus() == Status.FINISHED) - .sorted(Comparator.comparing(BackupMetadata::getStart).reversed()) - .findFirst(); + private IMetaProxy getMetaProxy(int backupVersion) { + switch (backupVersion) { + case SnapshotBackup.BACKUP_VERSION: + return metaV1Proxy; + case SnapshotMetaService.BACKUP_VERSION: + return metaV2Proxy; + } + + return null; } - public Optional verifyBackup(List metadata) { - if (metadata == null || metadata.isEmpty()) return Optional.empty(); + public Optional verifyBackup(int backupVersion, DateRange dateRange) + throws UnsupportedTypeException, IllegalArgumentException { + IMetaProxy metaProxy = getMetaProxy(backupVersion); + if (metaProxy == null) { + throw new UnsupportedTypeException( + "BackupVersion type: " + backupVersion + " is not supported"); + } - Optional latestBackupMetaData = getLatestBackupMetaData(metadata); + if (dateRange == null) { + throw new IllegalArgumentException("dateRange provided is null"); + } - if (!latestBackupMetaData.isPresent()) { - logger.error("No backup found which finished during the time provided."); - return Optional.empty(); + List metadata = + backupStatusMgr.getLatestBackupMetadata(backupVersion, dateRange); + if (metadata == null || metadata.isEmpty()) return Optional.empty(); + for (BackupMetadata backupMetadata : metadata) { + BackupVerificationResult backupVerificationResult = + verifyBackup(metaProxy, backupMetadata); + if (logger.isDebugEnabled()) + logger.debug( + "BackupVerification: metadata: {}, result: {}", + backupMetadata, + backupVerificationResult); + if (backupVerificationResult.valid) { + backupMetadata.setLastValidated(new Date(DateUtil.getInstant().toEpochMilli())); + backupStatusMgr.update(backupMetadata); + return Optional.of(backupVerificationResult); + } } + return Optional.empty(); + } - Path metadataLocation = Paths.get(latestBackupMetaData.get().getSnapshotLocation()); + private BackupVerificationResult verifyBackup( + IMetaProxy metaProxy, BackupMetadata latestBackupMetaData) { + Path metadataLocation = Paths.get(latestBackupMetaData.getSnapshotLocation()); metadataLocation = metadataLocation.subpath(1, metadataLocation.getNameCount()); AbstractBackupPath abstractBackupPath = abstractBackupPathProvider.get(); abstractBackupPath.parseRemote(metadataLocation.toString()); - return Optional.of((metaProxy.isMetaFileValid(abstractBackupPath))); + return metaProxy.isMetaFileValid(abstractBackupPath); } } diff --git a/priam/src/main/java/com/netflix/priam/backup/FileSnapshotStatusMgr.java b/priam/src/main/java/com/netflix/priam/backup/FileSnapshotStatusMgr.java index 0f6c94e4f..e59c5880d 100644 --- a/priam/src/main/java/com/netflix/priam/backup/FileSnapshotStatusMgr.java +++ b/priam/src/main/java/com/netflix/priam/backup/FileSnapshotStatusMgr.java @@ -58,8 +58,10 @@ private void init() { // Retrieve entire file and re-populate the list. File snapshotFile = new File(filename); if (!snapshotFile.exists()) { + snapshotFile.getParentFile().mkdirs(); logger.info( "Snapshot status file do not exist on system. Bypassing initilization phase."); + backupMetadataMap = new MaxSizeHashMap<>(capacity); return; } @@ -90,7 +92,7 @@ private void init() { @Override public void save(BackupMetadata backupMetadata) { File snapshotFile = new File(filename); - if (!snapshotFile.exists()) snapshotFile.mkdirs(); + if (!snapshotFile.exists()) snapshotFile.getParentFile().mkdirs(); // Will save entire list to file. try (final ObjectOutputStream out = diff --git a/priam/src/main/java/com/netflix/priam/backup/IBackupStatusMgr.java b/priam/src/main/java/com/netflix/priam/backup/IBackupStatusMgr.java index 11f016516..bf29b7008 100644 --- a/priam/src/main/java/com/netflix/priam/backup/IBackupStatusMgr.java +++ b/priam/src/main/java/com/netflix/priam/backup/IBackupStatusMgr.java @@ -14,6 +14,7 @@ package com.netflix.priam.backup; import com.google.inject.ImplementedBy; +import com.netflix.priam.utils.DateUtil; import java.util.Date; import java.util.LinkedList; import java.util.List; @@ -65,6 +66,13 @@ public interface IBackupStatusMgr { */ void failed(BackupMetadata backupMetadata); + /** + * Update the backup information of backupmetadata in-memory and other implementations, if any. + * + * @param backupMetadata backupmetadata to be updated. + */ + void update(BackupMetadata backupMetadata); + /** * Get the capacity of in-memory status map holding the snapshot status. * @@ -80,4 +88,15 @@ public interface IBackupStatusMgr { * snapshot start time. */ Map> getAllSnapshotStatus(); + + /** + * Get the list of backup metadata which are finished and have started in the daterange + * provided, in reverse chronological order of start date. + * + * @param snapshotVersion snapshot version of the backups to search. + * @param dateRange time period in which snapshot should have started. Finish time may be after + * the endTime in input. + * @return list of backup metadata which satisfies the input criteria + */ + List getLatestBackupMetadata(int snapshotVersion, DateUtil.DateRange dateRange); } diff --git a/priam/src/main/java/com/netflix/priam/backup/SnapshotBackup.java b/priam/src/main/java/com/netflix/priam/backup/SnapshotBackup.java index ff5efd488..c34085cb0 100644 --- a/priam/src/main/java/com/netflix/priam/backup/SnapshotBackup.java +++ b/priam/src/main/java/com/netflix/priam/backup/SnapshotBackup.java @@ -55,6 +55,7 @@ public class SnapshotBackup extends AbstractBackup { private List abstractBackupPaths = null; private final CassandraOperations cassandraOperations; private static final Lock lock = new ReentrantLock(); + public static final int BACKUP_VERSION = 1; @Inject public SnapshotBackup( @@ -109,12 +110,13 @@ private void executeSnapshot() throws Exception { String token = instanceIdentity.getInstance().getToken(); // Save start snapshot status - BackupMetadata backupMetadata = new BackupMetadata(token, startTime); + BackupMetadata backupMetadata = new BackupMetadata(BACKUP_VERSION, token, startTime); snapshotStatusMgr.start(backupMetadata); try { logger.info("Starting snapshot {}", snapshotName); cassandraOperations.takeSnapshot(snapshotName); + backupMetadata.setCassandraSnapshotSuccess(true); // Collect all snapshot dir's under keyspace dir's abstractBackupPaths = Lists.newArrayList(); diff --git a/priam/src/main/java/com/netflix/priam/backupv2/BackupValidator.java b/priam/src/main/java/com/netflix/priam/backupv2/BackupValidator.java deleted file mode 100644 index 2b7a7d3e4..000000000 --- a/priam/src/main/java/com/netflix/priam/backupv2/BackupValidator.java +++ /dev/null @@ -1,65 +0,0 @@ -/* - * Copyright 2018 Netflix, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.netflix.priam.backupv2; - -import com.netflix.priam.backup.*; -import com.netflix.priam.utils.DateUtil; -import java.util.List; -import java.util.Optional; -import javax.inject.Inject; -import javax.inject.Named; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * This class takes a meta file and verifies that all the components listed in that meta file are - * successfully uploaded to remote file system. Created by aagrawal on 11/28/18. - */ -public class BackupValidator { - private static final Logger logger = LoggerFactory.getLogger(BackupVerification.class); - private IMetaProxy metaProxy; - - @Inject - public BackupValidator(@Named("v2") IMetaProxy metaProxy) { - this.metaProxy = metaProxy; - } - - /** - * Find the latest valid meta file in a given valid time range. It will return the handle to the - * file via AbstractBackupPath object. - * - * @param dateRange the time period to scan in the remote file system for meta files. - * @return the BackupVerificationResult containing the details of the valid meta file. If none - * is found, null is returned. - * @throws BackupRestoreException if there is issue contacting remote file system, fetching the - * file etc. - */ - public Optional findLatestValidMetaFile(DateUtil.DateRange dateRange) - throws BackupRestoreException { - List metas = metaProxy.findMetaFiles(dateRange); - logger.info("Meta files found: {}", metas); - - for (AbstractBackupPath meta : metas) { - BackupVerificationResult result = metaProxy.isMetaFileValid(meta); - logger.info("BackupVerificationResult: {}", result); - if (result.valid) return Optional.of(result); - } - - return Optional.empty(); - } -} diff --git a/priam/src/main/java/com/netflix/priam/backupv2/MetaFileWriterBuilder.java b/priam/src/main/java/com/netflix/priam/backupv2/MetaFileWriterBuilder.java index 4e0c259c9..a1a4fb878 100644 --- a/priam/src/main/java/com/netflix/priam/backupv2/MetaFileWriterBuilder.java +++ b/priam/src/main/java/com/netflix/priam/backupv2/MetaFileWriterBuilder.java @@ -70,6 +70,8 @@ public interface UploadStep { void uploadMetaFile(boolean deleteOnSuccess) throws Exception; Path getMetaFilePath(); + + String getRemoteMetaFilePath() throws Exception; } public static class MetaFileWriter implements StartStep, DataStep, UploadStep { @@ -190,7 +192,7 @@ public void uploadMetaFile(boolean deleteOnSuccess) throws Exception { metaFilePath.toFile(), AbstractBackupPath.BackupFileType.META_V2); backupFileSystem.uploadFile( metaFilePath, - Paths.get(abstractBackupPath.getRemotePath()), + Paths.get(getRemoteMetaFilePath()), abstractBackupPath, 10, deleteOnSuccess); @@ -199,5 +201,12 @@ public void uploadMetaFile(boolean deleteOnSuccess) throws Exception { public Path getMetaFilePath() { return metaFilePath; } + + public String getRemoteMetaFilePath() throws Exception { + AbstractBackupPath abstractBackupPath = pathFactory.get(); + abstractBackupPath.parseLocal( + metaFilePath.toFile(), AbstractBackupPath.BackupFileType.META_V2); + return abstractBackupPath.getRemotePath(); + } } } diff --git a/priam/src/main/java/com/netflix/priam/config/IConfiguration.java b/priam/src/main/java/com/netflix/priam/config/IConfiguration.java index b5def0c87..63549e635 100644 --- a/priam/src/main/java/com/netflix/priam/config/IConfiguration.java +++ b/priam/src/main/java/com/netflix/priam/config/IConfiguration.java @@ -894,6 +894,17 @@ default boolean useSudo() { return true; } + /** + * This flag is an easy way to enable/disable notifications as notification topics may be + * different per region. A notification is only sent when this flag is enabled and {@link + * #getBackupNotificationTopicArn()} is not empty. + * + * @return true if backup notification is enabled, false otherwise. + */ + default boolean enableBackupNotification() { + return true; + } + /** * SNS Notification topic to be used for sending backup event notifications. One start event is * sent before uploading any file and one complete/failure event is sent after the file is diff --git a/priam/src/main/java/com/netflix/priam/config/PriamConfiguration.java b/priam/src/main/java/com/netflix/priam/config/PriamConfiguration.java index 09704ec6d..4f4ae42d8 100644 --- a/priam/src/main/java/com/netflix/priam/config/PriamConfiguration.java +++ b/priam/src/main/java/com/netflix/priam/config/PriamConfiguration.java @@ -684,6 +684,11 @@ public boolean useSudo() { return config.get(PRIAM_PRE + ".cass.usesudo", true); } + @Override + public boolean enableBackupNotification() { + return config.get(PRIAM_PRE + ".enableBackupNotification", true); + } + @Override public String getBackupNotificationTopicArn() { return config.get(PRIAM_PRE + ".backup.notification.topic.arn", ""); diff --git a/priam/src/main/java/com/netflix/priam/notification/AWSSnsNotificationService.java b/priam/src/main/java/com/netflix/priam/notification/AWSSnsNotificationService.java index 44cdf7c41..61bcd91f3 100644 --- a/priam/src/main/java/com/netflix/priam/notification/AWSSnsNotificationService.java +++ b/priam/src/main/java/com/netflix/priam/notification/AWSSnsNotificationService.java @@ -65,7 +65,7 @@ public void notify( final String msg, final Map messageAttributes) { // e.g. arn:aws:sns:eu-west-1:1234:eu-west-1-cass-sample-backup final String topic_arn = this.configuration.getBackupNotificationTopicArn(); - if (StringUtils.isEmpty(topic_arn)) { + if (!configuration.enableBackupNotification() || StringUtils.isEmpty(topic_arn)) { return; } diff --git a/priam/src/main/java/com/netflix/priam/resources/BackupServlet.java b/priam/src/main/java/com/netflix/priam/resources/BackupServlet.java index 37b012a96..fd3946bc6 100644 --- a/priam/src/main/java/com/netflix/priam/resources/BackupServlet.java +++ b/priam/src/main/java/com/netflix/priam/resources/BackupServlet.java @@ -20,11 +20,9 @@ import com.google.inject.name.Named; import com.netflix.priam.backup.*; import com.netflix.priam.backup.AbstractBackupPath.BackupFileType; -import com.netflix.priam.backupv2.BackupValidator; import com.netflix.priam.config.IConfiguration; import com.netflix.priam.scheduler.PriamScheduler; import com.netflix.priam.utils.DateUtil; -import com.netflix.priam.utils.SystemUtils; import java.util.*; import java.util.stream.Collectors; import javax.ws.rs.*; @@ -33,7 +31,6 @@ import org.codehaus.jettison.json.JSONArray; import org.codehaus.jettison.json.JSONException; import org.codehaus.jettison.json.JSONObject; -import org.joda.time.DateTime; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -52,7 +49,6 @@ public class BackupServlet { @Inject private PriamScheduler scheduler; private final IBackupStatusMgr completedBkups; @Inject private MetaData metaData; - @Inject private BackupValidator backupValidator; @Inject public BackupServlet( @@ -135,43 +131,24 @@ public Response status() throws Exception { @Produces(MediaType.APPLICATION_JSON) public Response statusByDate(@PathParam("date") String date) throws Exception { JSONObject object = new JSONObject(); - List metadataLinkedList = this.completedBkups.locate(date); - - if (metadataLinkedList != null && !metadataLinkedList.isEmpty()) { - // backup exist base on requested date, lets fetch more of its metadata - BackupMetadata bkupMetadata = metadataLinkedList.get(0); - object.put("Snapshotstatus", bkupMetadata.getStatus().equals(Status.FINISHED)); - String token = bkupMetadata.getToken(); - if (token != null && !token.isEmpty()) { - object.put("token", bkupMetadata.getToken()); - } else { - object.put("token", "not available"); - } - if (bkupMetadata.getStart() != null) { - object.put("starttime", DateUtil.formatyyyyMMddHHmm(bkupMetadata.getStart())); - } else { - object.put("starttime", "not available"); - } - - if (bkupMetadata.getCompleted() != null) { - object.put( - "completetime", DateUtil.formatyyyyMMddHHmm(bkupMetadata.getCompleted())); - } else { - object.put("completetime", "not_available"); - } - - } else { // Backup do not exist for that date. + Optional backupMetadataOptional = + this.completedBkups + .locate(date) + .stream() + .filter(backupMetadata -> backupMetadata != null) + .filter(backupMetadata -> backupMetadata.getStatus() == Status.FINISHED) + .filter( + backupMetadata -> + backupMetadata.getBackupVersion() + == SnapshotBackup.BACKUP_VERSION) + .findFirst(); + if (!backupMetadataOptional.isPresent()) { object.put("Snapshotstatus", false); - String token = - SystemUtils.getDataFromUrl( - "http://localhost:8080/Priam/REST/v1/cassconfig/get_token"); - if (token != null && !token.isEmpty()) { - object.put("token", token); - } else { - object.put("token", "not available"); - } - } + } else { + object.put("Snapshotstatus", true); + object.put("Details", backupMetadataOptional.get()); + } return Response.ok(object.toString(), MediaType.APPLICATION_JSON).build(); } @@ -192,6 +169,10 @@ public Response snapshotsByDate(@PathParam("date") String date) throws Exception if (metadata != null && !metadata.isEmpty()) snapshots.addAll( metadata.stream() + .filter( + backupMetadata -> + backupMetadata.getBackupVersion() + == SnapshotBackup.BACKUP_VERSION) .map( backupMetadata -> DateUtil.formatyyyyMMddHHmm(backupMetadata.getStart())) @@ -201,32 +182,6 @@ public Response snapshotsByDate(@PathParam("date") String date) throws Exception return Response.ok(object.toString(), MediaType.APPLICATION_JSON).build(); } - private List getLatestBackupMetadata(DateUtil.DateRange dateRange) { - Date startTime = Date.from(dateRange.getStartTime()); - Date endTime = Date.from(dateRange.getEndTime()); - List backupMetadata = this.completedBkups.locate(endTime); - if (backupMetadata != null && !backupMetadata.isEmpty()) return backupMetadata; - if (DateUtil.formatyyyyMMdd(startTime).equals(DateUtil.formatyyyyMMdd(endTime))) { - logger.info( - "Start & end date are same. No SNAPSHOT found for date: {}", - DateUtil.formatyyyyMMdd(endTime)); - return null; - } else { - Date previousDay = new Date(endTime.getTime()); - do { - // We need to find the latest backupmetadata in this date range. - previousDay = new DateTime(previousDay.getTime()).minusDays(1).toDate(); - logger.info( - "Will try to find snapshot for previous day: {}", - DateUtil.formatyyyyMMdd(previousDay)); - backupMetadata = completedBkups.locate(previousDay); - if (backupMetadata != null && !backupMetadata.isEmpty()) return backupMetadata; - } while (!DateUtil.formatyyyyMMdd(startTime) - .equals(DateUtil.formatyyyyMMdd(previousDay))); - } - return null; - } - /* * Determines the validity of the backup by i) Downloading meta.json file ii) Listing of the backup directory * iii) Find the missing or extra files in backup location. @@ -238,26 +193,9 @@ private List getLatestBackupMetadata(DateUtil.DateRange dateRang @Produces(MediaType.APPLICATION_JSON) public Response validateSnapshotByDate(@PathParam("daterange") String daterange) throws Exception { - - DateUtil.DateRange dateRange = new DateUtil.DateRange(daterange); - List metadata = getLatestBackupMetadata(dateRange); - Optional result = backupVerification.verifyBackup(metadata); - if (!result.isPresent()) { - return Response.noContent() - .entity("No valid meta found for provided time range") - .build(); - } - - return Response.ok(result.get().toString()).build(); - } - - @GET - @Path("/validate/snapshot/v2/{daterange}") - public Response validateV2SnapshotByDate(@PathParam("daterange") String daterange) - throws Exception { DateUtil.DateRange dateRange = new DateUtil.DateRange(daterange); Optional result = - backupValidator.findLatestValidMetaFile(dateRange); + backupVerification.verifyBackup(SnapshotBackup.BACKUP_VERSION, dateRange); if (!result.isPresent()) { return Response.noContent() .entity("No valid meta found for provided time range") diff --git a/priam/src/main/java/com/netflix/priam/resources/BackupServletV2.java b/priam/src/main/java/com/netflix/priam/resources/BackupServletV2.java new file mode 100644 index 000000000..673125ce1 --- /dev/null +++ b/priam/src/main/java/com/netflix/priam/resources/BackupServletV2.java @@ -0,0 +1,78 @@ +/* + * Copyright 2019 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.netflix.priam.resources; + +import com.google.inject.Inject; +import com.netflix.priam.backup.BackupVerification; +import com.netflix.priam.backup.BackupVerificationResult; +import com.netflix.priam.backup.IBackupStatusMgr; +import com.netflix.priam.services.SnapshotMetaService; +import com.netflix.priam.utils.DateUtil; +import java.util.Optional; +import javax.ws.rs.GET; +import javax.ws.rs.Path; +import javax.ws.rs.PathParam; +import javax.ws.rs.Produces; +import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** Created by aagrawal on 1/16/19. */ +@Path("/v2/backup") +@Produces(MediaType.APPLICATION_JSON) +public class BackupServletV2 { + private static final Logger logger = LoggerFactory.getLogger(BackupServletV2.class); + private final BackupVerification backupVerification; + private final IBackupStatusMgr backupStatusMgr; + private final SnapshotMetaService snapshotMetaService; + private static final String REST_SUCCESS = "[\"ok\"]"; + + @Inject + public BackupServletV2( + IBackupStatusMgr backupStatusMgr, + BackupVerification backupVerification, + SnapshotMetaService snapshotMetaService) { + this.backupStatusMgr = backupStatusMgr; + this.backupVerification = backupVerification; + this.snapshotMetaService = snapshotMetaService; + } + + @GET + @Path("/do_snapshot") + public Response backup() throws Exception { + snapshotMetaService.execute(); + return Response.ok(REST_SUCCESS, MediaType.APPLICATION_JSON).build(); + } + + @GET + @Path("/validate/{daterange}") + public Response validateV2SnapshotByDate(@PathParam("daterange") String daterange) + throws Exception { + DateUtil.DateRange dateRange = new DateUtil.DateRange(daterange); + Optional result = + backupVerification.verifyBackup(SnapshotMetaService.BACKUP_VERSION, dateRange); + if (!result.isPresent()) { + return Response.noContent() + .entity("No valid meta found for provided time range") + .build(); + } + + return Response.ok(result.get().toString()).build(); + } +} diff --git a/priam/src/main/java/com/netflix/priam/services/BackupTTLService.java b/priam/src/main/java/com/netflix/priam/services/BackupTTLService.java index a3170255e..370673826 100644 --- a/priam/src/main/java/com/netflix/priam/services/BackupTTLService.java +++ b/priam/src/main/java/com/netflix/priam/services/BackupTTLService.java @@ -87,7 +87,10 @@ public BackupTTLService( @Override public void execute() throws Exception { // Ensure that backup version 2.0 is actually enabled. - if (!backupRestoreConfig.enableV2Backups()) { + if (!backupRestoreConfig.enableV2Backups() + && backupRestoreConfig + .getSnapshotMetaServiceCronExpression() + .equalsIgnoreCase("-1")) { logger.info("Not executing the TTL Service for backups as V2 backups are not enabled."); return; } diff --git a/priam/src/main/java/com/netflix/priam/services/SnapshotMetaService.java b/priam/src/main/java/com/netflix/priam/services/SnapshotMetaService.java index 1cf62e885..017e56a0d 100644 --- a/priam/src/main/java/com/netflix/priam/services/SnapshotMetaService.java +++ b/priam/src/main/java/com/netflix/priam/services/SnapshotMetaService.java @@ -19,6 +19,7 @@ import com.netflix.priam.config.IBackupRestoreConfig; import com.netflix.priam.config.IConfiguration; import com.netflix.priam.defaultimpl.CassandraOperations; +import com.netflix.priam.identity.InstanceIdentity; import com.netflix.priam.scheduler.CronTimer; import com.netflix.priam.scheduler.TaskTimer; import com.netflix.priam.utils.CassandraMonitor; @@ -68,6 +69,9 @@ public class SnapshotMetaService extends AbstractBackup { private final CassandraOperations cassandraOperations; private String snapshotName = null; private static final Lock lock = new ReentrantLock(); + public static final int BACKUP_VERSION = 2; + private final IBackupStatusMgr snapshotStatusMgr; + private final InstanceIdentity instanceIdentity; private enum MetaStep { META_GENERATION, @@ -83,8 +87,12 @@ private enum MetaStep { Provider pathFactory, MetaFileWriterBuilder metaFileWriter, @Named("v2") IMetaProxy metaProxy, + InstanceIdentity instanceIdentity, + IBackupStatusMgr snapshotStatusMgr, CassandraOperations cassandraOperations) { super(config, backupFileSystemCtx, pathFactory); + this.instanceIdentity = instanceIdentity; + this.snapshotStatusMgr = snapshotStatusMgr; this.cassandraOperations = cassandraOperations; backupRestoreUtil = new BackupRestoreUtil( @@ -165,8 +173,14 @@ public void execute() throws Exception { throw new Exception("SnapshotMetaService already running"); } + // Save start snapshot status + Instant snapshotInstant = DateUtil.getInstant(); + String token = instanceIdentity.getInstance().getToken(); + BackupMetadata backupMetadata = + new BackupMetadata(BACKUP_VERSION, token, new Date(snapshotInstant.toEpochMilli())); + snapshotStatusMgr.start(backupMetadata); + try { - Instant snapshotInstant = DateUtil.getInstant(); snapshotName = generateSnapshotName(snapshotInstant); logger.info("Initializing SnapshotMetaService for taking a snapshot {}", snapshotName); @@ -179,16 +193,22 @@ public void execute() throws Exception { // Take a new snapshot cassandraOperations.takeSnapshot(snapshotName); + backupMetadata.setCassandraSnapshotSuccess(true); // Process the snapshot and upload the meta file. - processSnapshot(snapshotInstant).uploadMetaFile(true); + MetaFileWriterBuilder.UploadStep uploadStep = processSnapshot(snapshotInstant); + backupMetadata.setSnapshotLocation( + config.getBackupPrefix() + File.separator + uploadStep.getRemoteMetaFilePath()); + uploadStep.uploadMetaFile(true); logger.info("Finished processing snapshot meta service"); // Upload all the files from snapshot uploadFiles(); + snapshotStatusMgr.finish(backupMetadata); } catch (Exception e) { logger.error("Error while executing SnapshotMetaService", e); + snapshotStatusMgr.failed(backupMetadata); } finally { lock.unlock(); } diff --git a/priam/src/test/java/com/netflix/priam/backup/TestSnapshotStatusMgr.java b/priam/src/test/java/com/netflix/priam/backup/TestBackupStatusMgr.java similarity index 51% rename from priam/src/test/java/com/netflix/priam/backup/TestSnapshotStatusMgr.java rename to priam/src/test/java/com/netflix/priam/backup/TestBackupStatusMgr.java index 6eb493601..532b108f8 100644 --- a/priam/src/test/java/com/netflix/priam/backup/TestSnapshotStatusMgr.java +++ b/priam/src/test/java/com/netflix/priam/backup/TestBackupStatusMgr.java @@ -21,33 +21,85 @@ import com.google.inject.Injector; import com.netflix.priam.config.IConfiguration; import com.netflix.priam.utils.DateUtil; +import com.netflix.priam.utils.DateUtil.DateRange; import java.io.File; +import java.time.Instant; +import java.time.temporal.ChronoUnit; +import java.util.Calendar; import java.util.Date; import java.util.List; +import java.util.Optional; +import org.apache.commons.io.FileUtils; import org.joda.time.DateTime; +import org.junit.After; import org.junit.Assert; +import org.junit.Before; import org.junit.BeforeClass; import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** Created by aagrawal on 7/11/17. */ -public class TestSnapshotStatusMgr { - private static final Logger logger = LoggerFactory.getLogger(TestSnapshotStatusMgr.class); - +public class TestBackupStatusMgr { + private static final Logger logger = LoggerFactory.getLogger(TestBackupStatusMgr.class); + private static IConfiguration configuration; private static IBackupStatusMgr backupStatusMgr; + private final String backupDate = "201812011000"; @BeforeClass public static void setup() { Injector injector = Guice.createInjector(new BRTestModule()); // cleanup old saved file, if any - IConfiguration configuration = injector.getInstance(IConfiguration.class); - File f = new File(configuration.getBackupStatusFileLoc()); - if (f.exists()) f.delete(); - + configuration = injector.getInstance(IConfiguration.class); backupStatusMgr = injector.getInstance(IBackupStatusMgr.class); } + @Before + @After + public void cleanup() { + FileUtils.deleteQuietly(new File(configuration.getBackupStatusFileLoc())); + } + + private void prepare() throws Exception { + cleanup(); + Instant start = DateUtil.parseInstant(backupDate); + backupStatusMgr.finish(getBackupMetaData(start, Status.FINISHED)); + backupStatusMgr.failed(getBackupMetaData(start.plus(2, ChronoUnit.HOURS), Status.FAILED)); + backupStatusMgr.finish(getBackupMetaData(start.plus(4, ChronoUnit.HOURS), Status.FINISHED)); + backupStatusMgr.failed(getBackupMetaData(start.plus(6, ChronoUnit.HOURS), Status.FAILED)); + backupStatusMgr.failed(getBackupMetaData(start.plus(8, ChronoUnit.HOURS), Status.FAILED)); + backupStatusMgr.finish(getBackupMetaData(start.plus(1, ChronoUnit.DAYS), Status.FINISHED)); + backupStatusMgr.finish(getBackupMetaData(start.plus(2, ChronoUnit.DAYS), Status.FINISHED)); + } + + private BackupMetadata getBackupMetaData(Instant startTime, Status status) throws Exception { + BackupMetadata backupMetadata = + new BackupMetadata("123", new Date(startTime.toEpochMilli())); + backupMetadata.setCompleted( + new Date(startTime.plus(30, ChronoUnit.MINUTES).toEpochMilli())); + backupMetadata.setStatus(status); + backupMetadata.setSnapshotLocation("file.txt"); + return backupMetadata; + } + + @Test + public void testSnapshotUpdateMethod() throws Exception { + Date startTime = DateUtil.getDate("198407110720"); + BackupMetadata backupMetadata = new BackupMetadata("123", startTime); + backupStatusMgr.start(backupMetadata); + Optional backupMetadata1 = + backupStatusMgr.locate(startTime).stream().findFirst(); + Assert.assertNull(backupMetadata1.get().getLastValidated()); + backupMetadata.setLastValidated(Calendar.getInstance().getTime()); + backupMetadata.setCassandraSnapshotSuccess(true); + backupMetadata.setSnapshotLocation("random"); + backupStatusMgr.update(backupMetadata); + backupMetadata1 = backupStatusMgr.locate(startTime).stream().findFirst(); + Assert.assertNotNull(backupMetadata1.get().getLastValidated()); + Assert.assertTrue(backupMetadata1.get().isCassandraSnapshotSuccess()); + Assert.assertEquals("random", backupMetadata1.get().getSnapshotLocation()); + } + @Test public void testSnapshotStatusAddFinish() throws Exception { Date startTime = DateUtil.getDate("198407110720"); @@ -139,4 +191,56 @@ public void testSnapshotStatusSize() throws Exception { Assert.assertEquals( backupStatusMgr.getCapacity(), backupStatusMgr.getAllSnapshotStatus().size()); } + + @Test + public void getLatestBackup() throws Exception { + prepare(); + Instant start = DateUtil.parseInstant(backupDate); + List list = + backupStatusMgr.getLatestBackupMetadata( + SnapshotBackup.BACKUP_VERSION, + new DateRange( + backupDate + + "," + + DateUtil.formatInstant( + DateUtil.yyyyMMddHHmm, + start.plus(12, ChronoUnit.HOURS)))); + + Optional backupMetadata = list.stream().findFirst(); + Assert.assertEquals( + start.plus(4, ChronoUnit.HOURS), backupMetadata.get().getStart().toInstant()); + } + + @Test + public void getLatestBackupFailure() throws Exception { + Optional backupMetadata = + backupStatusMgr + .getLatestBackupMetadata( + SnapshotBackup.BACKUP_VERSION, + new DateRange(backupDate + "," + backupDate)) + .stream() + .findFirst(); + + Assert.assertFalse(backupMetadata.isPresent()); + + backupStatusMgr.failed(getBackupMetaData(DateUtil.parseInstant(backupDate), Status.FAILED)); + backupMetadata = + backupStatusMgr + .getLatestBackupMetadata( + SnapshotBackup.BACKUP_VERSION, + new DateRange(backupDate + "," + backupDate)) + .stream() + .findFirst(); + Assert.assertFalse(backupMetadata.isPresent()); + } + + @Test + public void getLatestBackupMetadata() throws Exception { + prepare(); + List list = + backupStatusMgr.getLatestBackupMetadata( + SnapshotBackup.BACKUP_VERSION, + new DateRange(backupDate + "," + "201812031000")); + list.forEach(backupMetadata -> System.out.println(backupMetadata)); + } } diff --git a/priam/src/test/java/com/netflix/priam/backup/TestBackupVerification.java b/priam/src/test/java/com/netflix/priam/backup/TestBackupVerification.java index 3b6c0d512..cb7b18276 100644 --- a/priam/src/test/java/com/netflix/priam/backup/TestBackupVerification.java +++ b/priam/src/test/java/com/netflix/priam/backup/TestBackupVerification.java @@ -1,5 +1,5 @@ /* - * Copyright 2018 Netflix, Inc. + * Copyright 2019 Netflix, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -19,68 +19,202 @@ import com.google.inject.Guice; import com.google.inject.Injector; +import com.netflix.priam.backup.AbstractBackupPath.BackupFileType; +import com.netflix.priam.backupv2.MetaV1Proxy; +import com.netflix.priam.backupv2.MetaV2Proxy; +import com.netflix.priam.config.IConfiguration; +import com.netflix.priam.scheduler.UnsupportedTypeException; +import com.netflix.priam.services.SnapshotMetaService; import com.netflix.priam.utils.DateUtil; +import com.netflix.priam.utils.DateUtil.DateRange; +import java.io.File; +import java.nio.file.Path; +import java.nio.file.Paths; import java.time.Instant; import java.time.temporal.ChronoUnit; -import java.util.ArrayList; import java.util.Date; -import java.util.List; import java.util.Optional; +import mockit.Mock; +import mockit.MockUp; +import org.apache.commons.io.FileUtils; +import org.junit.After; import org.junit.Assert; import org.junit.Before; import org.junit.Test; -/** Created by aagrawal on 12/21/18. */ +/** Created by aagrawal on 1/23/19. */ public class TestBackupVerification { - private static BackupVerification backupVerification; + private final BackupVerification backupVerification; + private final IConfiguration configuration; + private final IBackupStatusMgr backupStatusMgr; private final String backupDate = "201812011000"; - private List backupMetadataList = new ArrayList<>(); public TestBackupVerification() { Injector injector = Guice.createInjector(new BRTestModule()); + backupVerification = injector.getInstance(BackupVerification.class); + configuration = injector.getInstance(IConfiguration.class); + backupStatusMgr = injector.getInstance(IBackupStatusMgr.class); + } + + static class MockMetaV1Proxy extends MockUp { + @Mock + public BackupVerificationResult isMetaFileValid(AbstractBackupPath metaBackupPath) { + return getBackupVerificationResult(); + } + } + + static class MockMetaV2Proxy extends MockUp { + @Mock + public BackupVerificationResult isMetaFileValid(AbstractBackupPath metaBackupPath) { + return getBackupVerificationResult(); + } } @Before - public void prepare() throws Exception { - backupMetadataList.clear(); + @After + public void cleanup() { + new MockMetaV1Proxy(); + new MockMetaV2Proxy(); + FileUtils.deleteQuietly(new File(configuration.getBackupStatusFileLoc())); + } + + @Test + public void illegalBackupVersion() { + try { + backupVerification.verifyBackup(3, null); + Assert.assertTrue(false); + } catch (UnsupportedTypeException e) { + Assert.assertTrue(true); + } + } + + @Test + public void illegalDateRange() throws UnsupportedTypeException { + try { + backupVerification.verifyBackup(1, null); + Assert.assertTrue(false); + } catch (IllegalArgumentException e) { + Assert.assertTrue(true); + } + } + + @Test + public void noBackup() throws Exception { + Optional backupVerificationResultOptinal = + backupVerification.verifyBackup( + SnapshotBackup.BACKUP_VERSION, new DateRange(Instant.now(), Instant.now())); + Assert.assertFalse(backupVerificationResultOptinal.isPresent()); + + backupVerificationResultOptinal = + backupVerification.verifyBackup( + SnapshotMetaService.BACKUP_VERSION, + new DateRange(Instant.now(), Instant.now())); + Assert.assertFalse(backupVerificationResultOptinal.isPresent()); + } + + private void setUp() throws Exception { Instant start = DateUtil.parseInstant(backupDate); - backupMetadataList.add(getBackupMetaData(start, Status.FINISHED)); - backupMetadataList.add(getBackupMetaData(start.plus(2, ChronoUnit.HOURS), Status.FAILED)); - backupMetadataList.add(getBackupMetaData(start.plus(4, ChronoUnit.HOURS), Status.FINISHED)); - backupMetadataList.add(getBackupMetaData(start.plus(6, ChronoUnit.HOURS), Status.FAILED)); - backupMetadataList.add(getBackupMetaData(start.plus(8, ChronoUnit.HOURS), Status.FAILED)); + backupStatusMgr.finish( + getBackupMetaData(SnapshotBackup.BACKUP_VERSION, start, Status.FINISHED)); + backupStatusMgr.failed( + getBackupMetaData( + SnapshotBackup.BACKUP_VERSION, + start.plus(20, ChronoUnit.MINUTES), + Status.FAILED)); + backupStatusMgr.finish( + getBackupMetaData(SnapshotMetaService.BACKUP_VERSION, start, Status.FINISHED)); } @Test - public void getLatestBackup() { + public void verifyBackupVersion1() throws Exception { + setUp(); + // Verify for backup version 1.0 + Optional backupVerificationResultOptinal = + backupVerification.verifyBackup( + SnapshotBackup.BACKUP_VERSION, + new DateRange(backupDate + "," + backupDate)); + Assert.assertTrue(backupVerificationResultOptinal.isPresent()); + Assert.assertEquals(Instant.EPOCH, backupVerificationResultOptinal.get().snapshotInstant); Optional backupMetadata = - backupVerification.getLatestBackupMetaData(backupMetadataList); - Instant start = DateUtil.parseInstant(backupDate); - Assert.assertEquals( - start.plus(4, ChronoUnit.HOURS), backupMetadata.get().getStart().toInstant()); + backupStatusMgr + .getLatestBackupMetadata( + SnapshotBackup.BACKUP_VERSION, + new DateRange(backupDate + "," + backupDate)) + .stream() + .findFirst(); + Assert.assertTrue(backupMetadata.isPresent()); + Assert.assertNotNull(backupMetadata.get().getLastValidated()); + + backupMetadata = + backupStatusMgr + .getLatestBackupMetadata( + SnapshotMetaService.BACKUP_VERSION, + new DateRange(backupDate + "," + backupDate)) + .stream() + .findFirst(); + Assert.assertTrue(backupMetadata.isPresent()); + Assert.assertNull(backupMetadata.get().getLastValidated()); } @Test - public void getLatestBackupFailure() throws Exception { + public void verifyBackupVersion2() throws Exception { + setUp(); + // Verify for backup version 2.0 + Optional backupVerificationResultOptinal = + backupVerification.verifyBackup( + SnapshotMetaService.BACKUP_VERSION, + new DateRange(backupDate + "," + backupDate)); + Assert.assertTrue(backupVerificationResultOptinal.isPresent()); + Assert.assertEquals(Instant.EPOCH, backupVerificationResultOptinal.get().snapshotInstant); Optional backupMetadata = - backupVerification.getLatestBackupMetaData(new ArrayList<>()); - Assert.assertFalse(backupMetadata.isPresent()); + backupStatusMgr + .getLatestBackupMetadata( + SnapshotMetaService.BACKUP_VERSION, + new DateRange(backupDate + "," + backupDate)) + .stream() + .findFirst(); + Assert.assertTrue(backupMetadata.isPresent()); + Assert.assertNotNull(backupMetadata.get().getLastValidated()); - List failList = new ArrayList<>(); - failList.add(getBackupMetaData(DateUtil.getInstant(), Status.FAILED)); - backupMetadata = backupVerification.getLatestBackupMetaData(failList); - Assert.assertFalse(backupMetadata.isPresent()); + backupMetadata = + backupStatusMgr + .getLatestBackupMetadata( + SnapshotBackup.BACKUP_VERSION, + new DateRange(backupDate + "," + backupDate)) + .stream() + .findFirst(); + Assert.assertTrue(backupMetadata.isPresent()); + Assert.assertNull(backupMetadata.get().getLastValidated()); } - private BackupMetadata getBackupMetaData(Instant startTime, Status status) throws Exception { + private BackupMetadata getBackupMetaData(int backupVersion, Instant startTime, Status status) + throws Exception { BackupMetadata backupMetadata = - new BackupMetadata("123", new Date(startTime.toEpochMilli())); + new BackupMetadata(backupVersion, "123", new Date(startTime.toEpochMilli())); backupMetadata.setCompleted( new Date(startTime.plus(30, ChronoUnit.MINUTES).toEpochMilli())); backupMetadata.setStatus(status); - backupMetadata.setSnapshotLocation("file.txt"); + Path location = + Paths.get( + "some_bucket/casstestbackup/1049_fake-app/1808575600", + BackupFileType.META_V2.toString(), + "1859817645000", + "SNAPPY", + "PLAINTEXT", + "meta_v2_201812011000.json"); + backupMetadata.setSnapshotLocation(location.toString()); return backupMetadata; } + + private static BackupVerificationResult getBackupVerificationResult() { + BackupVerificationResult result = new BackupVerificationResult(); + result.valid = true; + result.manifestAvailable = true; + result.remotePath = "some_random"; + result.filesMatched = 123; + result.snapshotInstant = Instant.EPOCH; + return result; + } } diff --git a/priam/src/test/java/com/netflix/priam/config/FakeBackupRestoreConfig.java b/priam/src/test/java/com/netflix/priam/config/FakeBackupRestoreConfig.java index 246a91a86..266273025 100644 --- a/priam/src/test/java/com/netflix/priam/config/FakeBackupRestoreConfig.java +++ b/priam/src/test/java/com/netflix/priam/config/FakeBackupRestoreConfig.java @@ -22,7 +22,7 @@ public String getSnapshotMetaServiceCronExpression() { @Override public boolean enableV2Backups() { - return true; + return false; } @Override diff --git a/priam/src/test/java/com/netflix/priam/services/TestBackupTTLService.java b/priam/src/test/java/com/netflix/priam/services/TestBackupTTLService.java index d6e2746df..b642d8445 100644 --- a/priam/src/test/java/com/netflix/priam/services/TestBackupTTLService.java +++ b/priam/src/test/java/com/netflix/priam/services/TestBackupTTLService.java @@ -23,7 +23,6 @@ import com.netflix.priam.backup.AbstractBackupPath; import com.netflix.priam.backup.BRTestModule; import com.netflix.priam.backup.FakeBackupFileSystem; -import com.netflix.priam.backupv2.BackupValidator; import com.netflix.priam.backupv2.TestBackupUtils; import com.netflix.priam.config.IConfiguration; import com.netflix.priam.utils.BackupFileUtils; @@ -50,7 +49,6 @@ public class TestBackupTTLService { private static BackupTTLService backupTTLService; private static FakeBackupFileSystem backupFileSystem; private Provider pathProvider; - private static BackupValidator backupValidator; private Path[] metas; private Map allFilesMap = new HashMap<>(); @@ -62,7 +60,6 @@ public TestBackupTTLService() { if (backupFileSystem == null) backupFileSystem = injector.getInstance(FakeBackupFileSystem.class); pathProvider = injector.getProvider(AbstractBackupPath.class); - if (backupValidator == null) backupValidator = injector.getInstance(BackupValidator.class); } public void prepTest(int daysForSnapshot) throws Exception { From 90a1f1654a8ddf725a3c9e225514cf766cde110b Mon Sep 17 00:00:00 2001 From: Arun Agrawal Date: Fri, 25 Jan 2019 11:37:55 -0800 Subject: [PATCH 049/228] Add backup verification `force` field, ttl and info methods. --- .../priam/backup/BackupVerification.java | 14 +++++- .../priam/resources/BackupServlet.java | 8 ++-- .../priam/resources/BackupServletV2.java | 41 +++++++++++++++-- .../priam/backup/TestBackupVerification.java | 44 ++++++++++++++----- 4 files changed, 89 insertions(+), 18 deletions(-) diff --git a/priam/src/main/java/com/netflix/priam/backup/BackupVerification.java b/priam/src/main/java/com/netflix/priam/backup/BackupVerification.java index 1986c15a5..a55b9ea1f 100644 --- a/priam/src/main/java/com/netflix/priam/backup/BackupVerification.java +++ b/priam/src/main/java/com/netflix/priam/backup/BackupVerification.java @@ -65,7 +65,8 @@ private IMetaProxy getMetaProxy(int backupVersion) { return null; } - public Optional verifyBackup(int backupVersion, DateRange dateRange) + public Optional verifyBackup( + int backupVersion, boolean force, DateRange dateRange) throws UnsupportedTypeException, IllegalArgumentException { IMetaProxy metaProxy = getMetaProxy(backupVersion); if (metaProxy == null) { @@ -81,6 +82,17 @@ public Optional verifyBackup(int backupVersion, DateRa backupStatusMgr.getLatestBackupMetadata(backupVersion, dateRange); if (metadata == null || metadata.isEmpty()) return Optional.empty(); for (BackupMetadata backupMetadata : metadata) { + if (backupMetadata.getLastValidated() != null && !force) { + // Backup is already validated. Nothing to do. + BackupVerificationResult result = new BackupVerificationResult(); + result.valid = true; + result.manifestAvailable = true; + result.snapshotInstant = backupMetadata.getStart().toInstant(); + Path snapshotLocation = Paths.get(backupMetadata.getSnapshotLocation()); + result.remotePath = + snapshotLocation.subpath(1, snapshotLocation.getNameCount()).toString(); + return Optional.of(result); + } BackupVerificationResult backupVerificationResult = verifyBackup(metaProxy, backupMetadata); if (logger.isDebugEnabled()) diff --git a/priam/src/main/java/com/netflix/priam/resources/BackupServlet.java b/priam/src/main/java/com/netflix/priam/resources/BackupServlet.java index fd3946bc6..99bc4e6fb 100644 --- a/priam/src/main/java/com/netflix/priam/resources/BackupServlet.java +++ b/priam/src/main/java/com/netflix/priam/resources/BackupServlet.java @@ -147,7 +147,7 @@ public Response statusByDate(@PathParam("date") String date) throws Exception { } else { object.put("Snapshotstatus", true); - object.put("Details", backupMetadataOptional.get()); + object.put("Details", backupMetadataOptional.get().toString()); } return Response.ok(object.toString(), MediaType.APPLICATION_JSON).build(); } @@ -191,11 +191,13 @@ public Response snapshotsByDate(@PathParam("date") String date) throws Exception @GET @Path("/validate/snapshot/{daterange}") @Produces(MediaType.APPLICATION_JSON) - public Response validateSnapshotByDate(@PathParam("daterange") String daterange) + public Response validateSnapshotByDate( + @PathParam("daterange") String daterange, + @DefaultValue("false") @QueryParam("force") boolean force) throws Exception { DateUtil.DateRange dateRange = new DateUtil.DateRange(daterange); Optional result = - backupVerification.verifyBackup(SnapshotBackup.BACKUP_VERSION, dateRange); + backupVerification.verifyBackup(SnapshotBackup.BACKUP_VERSION, force, dateRange); if (!result.isPresent()) { return Response.noContent() .entity("No valid meta found for provided time range") diff --git a/priam/src/main/java/com/netflix/priam/resources/BackupServletV2.java b/priam/src/main/java/com/netflix/priam/resources/BackupServletV2.java index 673125ce1..ceba37ea8 100644 --- a/priam/src/main/java/com/netflix/priam/resources/BackupServletV2.java +++ b/priam/src/main/java/com/netflix/priam/resources/BackupServletV2.java @@ -18,16 +18,24 @@ package com.netflix.priam.resources; import com.google.inject.Inject; +import com.netflix.priam.backup.BackupMetadata; import com.netflix.priam.backup.BackupVerification; import com.netflix.priam.backup.BackupVerificationResult; import com.netflix.priam.backup.IBackupStatusMgr; +import com.netflix.priam.services.BackupTTLService; import com.netflix.priam.services.SnapshotMetaService; import com.netflix.priam.utils.DateUtil; +import com.netflix.priam.utils.DateUtil.DateRange; +import java.time.Instant; +import java.time.temporal.ChronoUnit; +import java.util.List; import java.util.Optional; +import javax.ws.rs.DefaultValue; import javax.ws.rs.GET; import javax.ws.rs.Path; import javax.ws.rs.PathParam; import javax.ws.rs.Produces; +import javax.ws.rs.QueryParam; import javax.ws.rs.core.MediaType; import javax.ws.rs.core.Response; import org.slf4j.Logger; @@ -41,16 +49,19 @@ public class BackupServletV2 { private final BackupVerification backupVerification; private final IBackupStatusMgr backupStatusMgr; private final SnapshotMetaService snapshotMetaService; + private final BackupTTLService backupTTLService; private static final String REST_SUCCESS = "[\"ok\"]"; @Inject public BackupServletV2( IBackupStatusMgr backupStatusMgr, BackupVerification backupVerification, - SnapshotMetaService snapshotMetaService) { + SnapshotMetaService snapshotMetaService, + BackupTTLService backupTTLService) { this.backupStatusMgr = backupStatusMgr; this.backupVerification = backupVerification; this.snapshotMetaService = snapshotMetaService; + this.backupTTLService = backupTTLService; } @GET @@ -60,13 +71,37 @@ public Response backup() throws Exception { return Response.ok(REST_SUCCESS, MediaType.APPLICATION_JSON).build(); } + @GET + @Path("/ttl") + public Response ttl() throws Exception { + backupTTLService.execute(); + return Response.ok(REST_SUCCESS, MediaType.APPLICATION_JSON).build(); + } + + @GET + @Path("/info/{date}") + @Produces(MediaType.APPLICATION_JSON) + public Response info(@PathParam("date") String date) throws Exception { + Instant instant = DateUtil.parseInstant(date); + List metadataList = + backupStatusMgr.getLatestBackupMetadata( + SnapshotMetaService.BACKUP_VERSION, + new DateRange( + instant, + instant.plus(1, ChronoUnit.DAYS).truncatedTo(ChronoUnit.DAYS))); + return Response.ok(metadataList.toString(), MediaType.APPLICATION_JSON).build(); + } + @GET @Path("/validate/{daterange}") - public Response validateV2SnapshotByDate(@PathParam("daterange") String daterange) + public Response validateV2SnapshotByDate( + @PathParam("daterange") String daterange, + @DefaultValue("false") @QueryParam("force") boolean force) throws Exception { DateUtil.DateRange dateRange = new DateUtil.DateRange(daterange); Optional result = - backupVerification.verifyBackup(SnapshotMetaService.BACKUP_VERSION, dateRange); + backupVerification.verifyBackup( + SnapshotMetaService.BACKUP_VERSION, force, dateRange); if (!result.isPresent()) { return Response.noContent() .entity("No valid meta found for provided time range") diff --git a/priam/src/test/java/com/netflix/priam/backup/TestBackupVerification.java b/priam/src/test/java/com/netflix/priam/backup/TestBackupVerification.java index cb7b18276..0a4ffbef8 100644 --- a/priam/src/test/java/com/netflix/priam/backup/TestBackupVerification.java +++ b/priam/src/test/java/com/netflix/priam/backup/TestBackupVerification.java @@ -49,6 +49,14 @@ public class TestBackupVerification { private final IConfiguration configuration; private final IBackupStatusMgr backupStatusMgr; private final String backupDate = "201812011000"; + private final Path location = + Paths.get( + "some_bucket/casstestbackup/1049_fake-app/1808575600", + BackupFileType.META_V2.toString(), + "1859817645000", + "SNAPPY", + "PLAINTEXT", + "meta_v2_201812011000.json"); public TestBackupVerification() { Injector injector = Guice.createInjector(new BRTestModule()); @@ -83,7 +91,7 @@ public void cleanup() { @Test public void illegalBackupVersion() { try { - backupVerification.verifyBackup(3, null); + backupVerification.verifyBackup(3, false, null); Assert.assertTrue(false); } catch (UnsupportedTypeException e) { Assert.assertTrue(true); @@ -93,7 +101,7 @@ public void illegalBackupVersion() { @Test public void illegalDateRange() throws UnsupportedTypeException { try { - backupVerification.verifyBackup(1, null); + backupVerification.verifyBackup(1, false, null); Assert.assertTrue(false); } catch (IllegalArgumentException e) { Assert.assertTrue(true); @@ -104,12 +112,15 @@ public void illegalDateRange() throws UnsupportedTypeException { public void noBackup() throws Exception { Optional backupVerificationResultOptinal = backupVerification.verifyBackup( - SnapshotBackup.BACKUP_VERSION, new DateRange(Instant.now(), Instant.now())); + SnapshotBackup.BACKUP_VERSION, + false, + new DateRange(Instant.now(), Instant.now())); Assert.assertFalse(backupVerificationResultOptinal.isPresent()); backupVerificationResultOptinal = backupVerification.verifyBackup( SnapshotMetaService.BACKUP_VERSION, + false, new DateRange(Instant.now(), Instant.now())); Assert.assertFalse(backupVerificationResultOptinal.isPresent()); } @@ -134,6 +145,7 @@ public void verifyBackupVersion1() throws Exception { Optional backupVerificationResultOptinal = backupVerification.verifyBackup( SnapshotBackup.BACKUP_VERSION, + false, new DateRange(backupDate + "," + backupDate)); Assert.assertTrue(backupVerificationResultOptinal.isPresent()); Assert.assertEquals(Instant.EPOCH, backupVerificationResultOptinal.get().snapshotInstant); @@ -165,9 +177,12 @@ public void verifyBackupVersion2() throws Exception { Optional backupVerificationResultOptinal = backupVerification.verifyBackup( SnapshotMetaService.BACKUP_VERSION, + false, new DateRange(backupDate + "," + backupDate)); Assert.assertTrue(backupVerificationResultOptinal.isPresent()); Assert.assertEquals(Instant.EPOCH, backupVerificationResultOptinal.get().snapshotInstant); + Assert.assertEquals("some_random", backupVerificationResultOptinal.get().remotePath); + Optional backupMetadata = backupStatusMgr .getLatestBackupMetadata( @@ -178,6 +193,21 @@ public void verifyBackupVersion2() throws Exception { Assert.assertTrue(backupMetadata.isPresent()); Assert.assertNotNull(backupMetadata.get().getLastValidated()); + // Retry the verification, it should not try and re-verify + backupVerificationResultOptinal = + backupVerification.verifyBackup( + SnapshotMetaService.BACKUP_VERSION, + false, + new DateRange(backupDate + "," + backupDate)); + Assert.assertTrue(backupVerificationResultOptinal.isPresent()); + Assert.assertEquals( + DateUtil.parseInstant(backupDate), + backupVerificationResultOptinal.get().snapshotInstant); + Assert.assertNotEquals("some_random", backupVerificationResultOptinal.get().remotePath); + Assert.assertEquals( + location.subpath(1, location.getNameCount()).toString(), + backupVerificationResultOptinal.get().remotePath); + backupMetadata = backupStatusMgr .getLatestBackupMetadata( @@ -196,14 +226,6 @@ private BackupMetadata getBackupMetaData(int backupVersion, Instant startTime, S backupMetadata.setCompleted( new Date(startTime.plus(30, ChronoUnit.MINUTES).toEpochMilli())); backupMetadata.setStatus(status); - Path location = - Paths.get( - "some_bucket/casstestbackup/1049_fake-app/1808575600", - BackupFileType.META_V2.toString(), - "1859817645000", - "SNAPPY", - "PLAINTEXT", - "meta_v2_201812011000.json"); backupMetadata.setSnapshotLocation(location.toString()); return backupMetadata; } From e8c231487f2b01aba109e2856396437a40457a45 Mon Sep 17 00:00:00 2001 From: Arun Agrawal Date: Tue, 29 Jan 2019 13:26:42 -0800 Subject: [PATCH 050/228] Add backup version as enum, minor pr feedback --- .../netflix/priam/backup/BackupMetadata.java | 13 +-- .../netflix/priam/backup/BackupStatusMgr.java | 40 ++++---- .../priam/backup/BackupVerification.java | 9 +- .../netflix/priam/backup/BackupVersion.java | 92 +++++++++++++++++++ .../priam/backup/IBackupStatusMgr.java | 5 +- .../netflix/priam/backup/SnapshotBackup.java | 4 +- .../priam/resources/BackupServlet.java | 15 +-- .../priam/resources/BackupServletV2.java | 12 +-- .../priam/services/SnapshotMetaService.java | 7 +- .../com/netflix/priam/utils/SystemUtils.java | 2 +- .../backup/TestBackupRestoreUtil.groovy | 16 ++++ .../backup/TestBackupScheduler.groovy | 16 ++++ .../cluser/management/TestCompaction.groovy | 16 ++++ .../priam/backup/TestBackupStatusMgr.java | 26 ++++-- .../priam/backup/TestBackupVerification.java | 41 +++------ .../PriamConfigurationPersisterTest.java | 16 ++++ .../defaultimpl/FakeCassandraProcess.java | 16 ++++ .../priam/health/TestInstanceStatus.java | 16 ++++ .../priam/resources/PriamConfigTest.java | 16 ++++ .../priam/utils/TestGsonJsonSerializer.java | 5 +- 20 files changed, 290 insertions(+), 93 deletions(-) create mode 100644 priam/src/main/java/com/netflix/priam/backup/BackupVersion.java diff --git a/priam/src/main/java/com/netflix/priam/backup/BackupMetadata.java b/priam/src/main/java/com/netflix/priam/backup/BackupMetadata.java index d95d4989b..295078721 100644 --- a/priam/src/main/java/com/netflix/priam/backup/BackupMetadata.java +++ b/priam/src/main/java/com/netflix/priam/backup/BackupMetadata.java @@ -31,10 +31,10 @@ public final class BackupMetadata implements Serializable { private Status status; private boolean cassandraSnapshotSuccess; private Date lastValidated; - private int backupVersion; + private BackupVersion backupVersion; private String snapshotLocation; - public BackupMetadata(int backupVersion, String token, Date start) throws Exception { + public BackupMetadata(BackupVersion backupVersion, String token, Date start) throws Exception { if (start == null || token == null || StringUtils.isEmpty(token)) throw new Exception( String.format( @@ -48,10 +48,6 @@ public BackupMetadata(int backupVersion, String token, Date start) throws Except this.cassandraSnapshotSuccess = false; } - public BackupMetadata(String token, Date start) throws Exception { - this(1, token, start); - } - @Override public boolean equals(Object o) { if (this == o) return true; @@ -62,7 +58,7 @@ public boolean equals(Object o) { return this.snapshotDate.equals(that.snapshotDate) && this.token.equals(that.token) && this.start.equals(that.start) - && this.backupVersion == that.backupVersion; + && this.backupVersion.equals(that.backupVersion); } @Override @@ -70,6 +66,7 @@ public int hashCode() { int result = this.snapshotDate.hashCode(); result = 31 * result + this.token.hashCode(); result = 31 * result + this.start.hashCode(); + result = 31 * result + this.backupVersion.hashCode(); return result; } @@ -179,7 +176,7 @@ public void setCassandraSnapshotSuccess(boolean cassandraSnapshotSuccess) { * * @return backup version of the snapshot. */ - public int getBackupVersion() { + public BackupVersion getBackupVersion() { return backupVersion; } diff --git a/priam/src/main/java/com/netflix/priam/backup/BackupStatusMgr.java b/priam/src/main/java/com/netflix/priam/backup/BackupStatusMgr.java index d29584ea8..36489e022 100644 --- a/priam/src/main/java/com/netflix/priam/backup/BackupStatusMgr.java +++ b/priam/src/main/java/com/netflix/priam/backup/BackupStatusMgr.java @@ -187,12 +187,12 @@ public void failed(BackupMetadata backupMetadata) { protected abstract LinkedList fetch(String snapshotDate); public List getLatestBackupMetadata( - int snapshotVersion, DateUtil.DateRange dateRange) { - Instant startTime = dateRange.getStartTime().truncatedTo(ChronoUnit.DAYS); - Instant endTime = dateRange.getEndTime().truncatedTo(ChronoUnit.DAYS); + BackupVersion backupVersion, DateUtil.DateRange dateRange) { + Instant startDay = dateRange.getStartTime().truncatedTo(ChronoUnit.DAYS); + Instant endDay = dateRange.getEndTime().truncatedTo(ChronoUnit.DAYS); List allBackups = new ArrayList<>(); - Instant previousDay = endTime; + Instant previousDay = endDay; do { // We need to find the latest backupmetadata in this date range. logger.info( @@ -201,36 +201,28 @@ public List getLatestBackupMetadata( List backupsForDate = locate(new Date(previousDay.toEpochMilli())); if (backupsForDate != null) allBackups.addAll(backupsForDate); previousDay = previousDay.minus(1, ChronoUnit.DAYS); - } while (!previousDay.isBefore(startTime)); + } while (!previousDay.isBefore(startDay)); // Return all the backups which are FINISHED and were "started" in the dateRange provided. // Do not compare the end time of snapshot as it may take random amount of time to finish // the snapshot. return allBackups .stream() - .filter(backupMetadata -> backupMetadata != null) + .filter(Objects::nonNull) .filter(backupMetadata -> backupMetadata.getStatus() == Status.FINISHED) - .filter(backupMetadata -> backupMetadata.getBackupVersion() == snapshotVersion) + .filter(backupMetadata -> backupMetadata.getBackupVersion().equals(backupVersion)) .filter( backupMetadata -> backupMetadata - .getStart() - .toInstant() - .isAfter(dateRange.getStartTime()) - || backupMetadata - .getStart() - .toInstant() - .equals(dateRange.getStartTime())) - .filter( - backupMetadata -> - backupMetadata - .getStart() - .toInstant() - .isBefore(dateRange.getEndTime()) - || backupMetadata - .getStart() - .toInstant() - .equals(dateRange.getEndTime())) + .getStart() + .toInstant() + .compareTo(dateRange.getStartTime()) + >= 0 + && backupMetadata + .getStart() + .toInstant() + .compareTo(dateRange.getEndTime()) + <= 0) .sorted(Comparator.comparing(BackupMetadata::getStart).reversed()) .collect(Collectors.toList()); } diff --git a/priam/src/main/java/com/netflix/priam/backup/BackupVerification.java b/priam/src/main/java/com/netflix/priam/backup/BackupVerification.java index a55b9ea1f..877c6e363 100644 --- a/priam/src/main/java/com/netflix/priam/backup/BackupVerification.java +++ b/priam/src/main/java/com/netflix/priam/backup/BackupVerification.java @@ -19,7 +19,6 @@ import com.google.inject.name.Named; import com.netflix.priam.backupv2.IMetaProxy; import com.netflix.priam.scheduler.UnsupportedTypeException; -import com.netflix.priam.services.SnapshotMetaService; import com.netflix.priam.utils.DateUtil; import com.netflix.priam.utils.DateUtil.DateRange; import java.nio.file.Path; @@ -54,11 +53,11 @@ public class BackupVerification { this.abstractBackupPathProvider = abstractBackupPathProvider; } - private IMetaProxy getMetaProxy(int backupVersion) { + private IMetaProxy getMetaProxy(BackupVersion backupVersion) { switch (backupVersion) { - case SnapshotBackup.BACKUP_VERSION: + case SNAPSHOT_BACKUP: return metaV1Proxy; - case SnapshotMetaService.BACKUP_VERSION: + case SNAPSHOT_META_SERVICE: return metaV2Proxy; } @@ -66,7 +65,7 @@ private IMetaProxy getMetaProxy(int backupVersion) { } public Optional verifyBackup( - int backupVersion, boolean force, DateRange dateRange) + BackupVersion backupVersion, boolean force, DateRange dateRange) throws UnsupportedTypeException, IllegalArgumentException { IMetaProxy metaProxy = getMetaProxy(backupVersion); if (metaProxy == null) { diff --git a/priam/src/main/java/com/netflix/priam/backup/BackupVersion.java b/priam/src/main/java/com/netflix/priam/backup/BackupVersion.java new file mode 100644 index 000000000..84ea3ad36 --- /dev/null +++ b/priam/src/main/java/com/netflix/priam/backup/BackupVersion.java @@ -0,0 +1,92 @@ +/* + * Copyright 2019 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.netflix.priam.backup; + +import com.netflix.priam.scheduler.UnsupportedTypeException; +import java.util.HashMap; +import java.util.Map; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Enum to capture backup versions. Possible version are V1 and V2. Created by aagrawal on 1/29/19. + */ +public enum BackupVersion { + SNAPSHOT_BACKUP(1), + SNAPSHOT_META_SERVICE(2); + + private static final Logger logger = LoggerFactory.getLogger(BackupVersion.class); + + private final int backupVersion; + private static Map map = new HashMap<>(); + + static { + for (BackupVersion backupVersion : BackupVersion.values()) { + map.put(backupVersion.getBackupVersion(), backupVersion); + } + } + + BackupVersion(int backupVersion) { + this.backupVersion = backupVersion; + } + + public static BackupVersion lookup(int backupVersion, boolean acceptIllegalValue) + throws UnsupportedTypeException { + BackupVersion backupVersionResolved = map.get(backupVersion); + if (backupVersionResolved == null) { + String message = + String.format( + "%s is not a supported BackupVersion. Supported values are %s", + backupVersion, getSupportedValues()); + + if (acceptIllegalValue) { + message = + message + + ". Since acceptIllegalValue is set to True, returning NULL instead."; + logger.error(message); + return null; + } + + logger.error(message); + throw new UnsupportedTypeException(message); + } + return backupVersionResolved; + } + + private static String getSupportedValues() { + StringBuilder supportedValues = new StringBuilder(); + boolean first = true; + for (BackupVersion type : BackupVersion.values()) { + if (!first) { + supportedValues.append(","); + } + supportedValues.append(type); + first = false; + } + + return supportedValues.toString(); + } + + public static BackupVersion lookup(int backupVersion) throws UnsupportedTypeException { + return lookup(backupVersion, false); + } + + public int getBackupVersion() { + return backupVersion; + } +} diff --git a/priam/src/main/java/com/netflix/priam/backup/IBackupStatusMgr.java b/priam/src/main/java/com/netflix/priam/backup/IBackupStatusMgr.java index bf29b7008..95b69b007 100644 --- a/priam/src/main/java/com/netflix/priam/backup/IBackupStatusMgr.java +++ b/priam/src/main/java/com/netflix/priam/backup/IBackupStatusMgr.java @@ -93,10 +93,11 @@ public interface IBackupStatusMgr { * Get the list of backup metadata which are finished and have started in the daterange * provided, in reverse chronological order of start date. * - * @param snapshotVersion snapshot version of the backups to search. + * @param backupVersion backup version of the backups to search. * @param dateRange time period in which snapshot should have started. Finish time may be after * the endTime in input. * @return list of backup metadata which satisfies the input criteria */ - List getLatestBackupMetadata(int snapshotVersion, DateUtil.DateRange dateRange); + List getLatestBackupMetadata( + BackupVersion backupVersion, DateUtil.DateRange dateRange); } diff --git a/priam/src/main/java/com/netflix/priam/backup/SnapshotBackup.java b/priam/src/main/java/com/netflix/priam/backup/SnapshotBackup.java index c34085cb0..476d4eb5d 100644 --- a/priam/src/main/java/com/netflix/priam/backup/SnapshotBackup.java +++ b/priam/src/main/java/com/netflix/priam/backup/SnapshotBackup.java @@ -55,7 +55,6 @@ public class SnapshotBackup extends AbstractBackup { private List abstractBackupPaths = null; private final CassandraOperations cassandraOperations; private static final Lock lock = new ReentrantLock(); - public static final int BACKUP_VERSION = 1; @Inject public SnapshotBackup( @@ -110,7 +109,8 @@ private void executeSnapshot() throws Exception { String token = instanceIdentity.getInstance().getToken(); // Save start snapshot status - BackupMetadata backupMetadata = new BackupMetadata(BACKUP_VERSION, token, startTime); + BackupMetadata backupMetadata = + new BackupMetadata(BackupVersion.SNAPSHOT_BACKUP, token, startTime); snapshotStatusMgr.start(backupMetadata); try { diff --git a/priam/src/main/java/com/netflix/priam/resources/BackupServlet.java b/priam/src/main/java/com/netflix/priam/resources/BackupServlet.java index 99bc4e6fb..1c1fad89f 100644 --- a/priam/src/main/java/com/netflix/priam/resources/BackupServlet.java +++ b/priam/src/main/java/com/netflix/priam/resources/BackupServlet.java @@ -20,6 +20,7 @@ import com.google.inject.name.Named; import com.netflix.priam.backup.*; import com.netflix.priam.backup.AbstractBackupPath.BackupFileType; +import com.netflix.priam.backup.BackupVersion; import com.netflix.priam.config.IConfiguration; import com.netflix.priam.scheduler.PriamScheduler; import com.netflix.priam.utils.DateUtil; @@ -135,12 +136,13 @@ public Response statusByDate(@PathParam("date") String date) throws Exception { this.completedBkups .locate(date) .stream() - .filter(backupMetadata -> backupMetadata != null) + .filter(Objects::nonNull) .filter(backupMetadata -> backupMetadata.getStatus() == Status.FINISHED) .filter( backupMetadata -> - backupMetadata.getBackupVersion() - == SnapshotBackup.BACKUP_VERSION) + backupMetadata + .getBackupVersion() + .equals(BackupVersion.SNAPSHOT_BACKUP)) .findFirst(); if (!backupMetadataOptional.isPresent()) { object.put("Snapshotstatus", false); @@ -171,8 +173,9 @@ public Response snapshotsByDate(@PathParam("date") String date) throws Exception metadata.stream() .filter( backupMetadata -> - backupMetadata.getBackupVersion() - == SnapshotBackup.BACKUP_VERSION) + backupMetadata + .getBackupVersion() + .equals(BackupVersion.SNAPSHOT_BACKUP)) .map( backupMetadata -> DateUtil.formatyyyyMMddHHmm(backupMetadata.getStart())) @@ -197,7 +200,7 @@ public Response validateSnapshotByDate( throws Exception { DateUtil.DateRange dateRange = new DateUtil.DateRange(daterange); Optional result = - backupVerification.verifyBackup(SnapshotBackup.BACKUP_VERSION, force, dateRange); + backupVerification.verifyBackup(BackupVersion.SNAPSHOT_BACKUP, force, dateRange); if (!result.isPresent()) { return Response.noContent() .entity("No valid meta found for provided time range") diff --git a/priam/src/main/java/com/netflix/priam/resources/BackupServletV2.java b/priam/src/main/java/com/netflix/priam/resources/BackupServletV2.java index ceba37ea8..57eb1e98d 100644 --- a/priam/src/main/java/com/netflix/priam/resources/BackupServletV2.java +++ b/priam/src/main/java/com/netflix/priam/resources/BackupServletV2.java @@ -21,6 +21,7 @@ import com.netflix.priam.backup.BackupMetadata; import com.netflix.priam.backup.BackupVerification; import com.netflix.priam.backup.BackupVerificationResult; +import com.netflix.priam.backup.BackupVersion; import com.netflix.priam.backup.IBackupStatusMgr; import com.netflix.priam.services.BackupTTLService; import com.netflix.priam.services.SnapshotMetaService; @@ -80,16 +81,15 @@ public Response ttl() throws Exception { @GET @Path("/info/{date}") - @Produces(MediaType.APPLICATION_JSON) - public Response info(@PathParam("date") String date) throws Exception { + public Response info(@PathParam("date") String date) { Instant instant = DateUtil.parseInstant(date); List metadataList = backupStatusMgr.getLatestBackupMetadata( - SnapshotMetaService.BACKUP_VERSION, + BackupVersion.SNAPSHOT_META_SERVICE, new DateRange( instant, instant.plus(1, ChronoUnit.DAYS).truncatedTo(ChronoUnit.DAYS))); - return Response.ok(metadataList.toString(), MediaType.APPLICATION_JSON).build(); + return Response.ok(metadataList).build(); } @GET @@ -101,13 +101,13 @@ public Response validateV2SnapshotByDate( DateUtil.DateRange dateRange = new DateUtil.DateRange(daterange); Optional result = backupVerification.verifyBackup( - SnapshotMetaService.BACKUP_VERSION, force, dateRange); + BackupVersion.SNAPSHOT_META_SERVICE, force, dateRange); if (!result.isPresent()) { return Response.noContent() .entity("No valid meta found for provided time range") .build(); } - return Response.ok(result.get().toString()).build(); + return Response.ok(result.get()).build(); } } diff --git a/priam/src/main/java/com/netflix/priam/services/SnapshotMetaService.java b/priam/src/main/java/com/netflix/priam/services/SnapshotMetaService.java index 017e56a0d..25449a86c 100644 --- a/priam/src/main/java/com/netflix/priam/services/SnapshotMetaService.java +++ b/priam/src/main/java/com/netflix/priam/services/SnapshotMetaService.java @@ -15,6 +15,7 @@ import com.google.inject.Provider; import com.netflix.priam.backup.*; +import com.netflix.priam.backup.BackupVersion; import com.netflix.priam.backupv2.*; import com.netflix.priam.config.IBackupRestoreConfig; import com.netflix.priam.config.IConfiguration; @@ -69,7 +70,6 @@ public class SnapshotMetaService extends AbstractBackup { private final CassandraOperations cassandraOperations; private String snapshotName = null; private static final Lock lock = new ReentrantLock(); - public static final int BACKUP_VERSION = 2; private final IBackupStatusMgr snapshotStatusMgr; private final InstanceIdentity instanceIdentity; @@ -177,7 +177,10 @@ public void execute() throws Exception { Instant snapshotInstant = DateUtil.getInstant(); String token = instanceIdentity.getInstance().getToken(); BackupMetadata backupMetadata = - new BackupMetadata(BACKUP_VERSION, token, new Date(snapshotInstant.toEpochMilli())); + new BackupMetadata( + BackupVersion.SNAPSHOT_META_SERVICE, + token, + new Date(snapshotInstant.toEpochMilli())); snapshotStatusMgr.start(backupMetadata); try { diff --git a/priam/src/main/java/com/netflix/priam/utils/SystemUtils.java b/priam/src/main/java/com/netflix/priam/utils/SystemUtils.java index 50b1f7416..9696ed399 100644 --- a/priam/src/main/java/com/netflix/priam/utils/SystemUtils.java +++ b/priam/src/main/java/com/netflix/priam/utils/SystemUtils.java @@ -123,7 +123,7 @@ public static void closeQuietly(JMXNodeTool tool) { try { tool.close(); } catch (Exception e) { - logger.warn("failed to close jxm node tool", e); + logger.warn("failed to close jmx node tool", e); } } diff --git a/priam/src/test/groovy/com.netflix.priam/backup/TestBackupRestoreUtil.groovy b/priam/src/test/groovy/com.netflix.priam/backup/TestBackupRestoreUtil.groovy index c4582e22b..a196a44ab 100644 --- a/priam/src/test/groovy/com.netflix.priam/backup/TestBackupRestoreUtil.groovy +++ b/priam/src/test/groovy/com.netflix.priam/backup/TestBackupRestoreUtil.groovy @@ -1,3 +1,19 @@ +/* + * Copyright 2017 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ package com.netflix.priam.backup import spock.lang.Specification diff --git a/priam/src/test/groovy/com.netflix.priam/backup/TestBackupScheduler.groovy b/priam/src/test/groovy/com.netflix.priam/backup/TestBackupScheduler.groovy index 22a391a68..d5baaf0b1 100644 --- a/priam/src/test/groovy/com.netflix.priam/backup/TestBackupScheduler.groovy +++ b/priam/src/test/groovy/com.netflix.priam/backup/TestBackupScheduler.groovy @@ -1,3 +1,19 @@ +/* + * Copyright 2017 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ package com.netflix.priam.backup import com.netflix.priam.config.FakeConfiguration diff --git a/priam/src/test/groovy/com/netflix/priam/cluser/management/TestCompaction.groovy b/priam/src/test/groovy/com/netflix/priam/cluser/management/TestCompaction.groovy index cb4b1ca28..4c6f3a0a6 100644 --- a/priam/src/test/groovy/com/netflix/priam/cluser/management/TestCompaction.groovy +++ b/priam/src/test/groovy/com/netflix/priam/cluser/management/TestCompaction.groovy @@ -1,3 +1,19 @@ +/* + * Copyright 2017 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ package com.netflix.priam.cluser.management import com.google.inject.Guice diff --git a/priam/src/test/java/com/netflix/priam/backup/TestBackupStatusMgr.java b/priam/src/test/java/com/netflix/priam/backup/TestBackupStatusMgr.java index 532b108f8..e92733e26 100644 --- a/priam/src/test/java/com/netflix/priam/backup/TestBackupStatusMgr.java +++ b/priam/src/test/java/com/netflix/priam/backup/TestBackupStatusMgr.java @@ -74,7 +74,8 @@ private void prepare() throws Exception { private BackupMetadata getBackupMetaData(Instant startTime, Status status) throws Exception { BackupMetadata backupMetadata = - new BackupMetadata("123", new Date(startTime.toEpochMilli())); + new BackupMetadata( + BackupVersion.SNAPSHOT_BACKUP, "123", new Date(startTime.toEpochMilli())); backupMetadata.setCompleted( new Date(startTime.plus(30, ChronoUnit.MINUTES).toEpochMilli())); backupMetadata.setStatus(status); @@ -85,7 +86,8 @@ private BackupMetadata getBackupMetaData(Instant startTime, Status status) throw @Test public void testSnapshotUpdateMethod() throws Exception { Date startTime = DateUtil.getDate("198407110720"); - BackupMetadata backupMetadata = new BackupMetadata("123", startTime); + BackupMetadata backupMetadata = + new BackupMetadata(BackupVersion.SNAPSHOT_BACKUP, "123", startTime); backupStatusMgr.start(backupMetadata); Optional backupMetadata1 = backupStatusMgr.locate(startTime).stream().findFirst(); @@ -104,7 +106,8 @@ public void testSnapshotUpdateMethod() throws Exception { public void testSnapshotStatusAddFinish() throws Exception { Date startTime = DateUtil.getDate("198407110720"); - BackupMetadata backupMetadata = new BackupMetadata("123", startTime); + BackupMetadata backupMetadata = + new BackupMetadata(BackupVersion.SNAPSHOT_BACKUP, "123", startTime); backupStatusMgr.start(backupMetadata); List metadataList = backupStatusMgr.locate(startTime); Assert.assertNotNull(metadataList); @@ -127,7 +130,8 @@ public void testSnapshotStatusAddFinish() throws Exception { public void testSnapshotStatusAddFailed() throws Exception { Date startTime = DateUtil.getDate("198407120720"); - BackupMetadata backupMetadata = new BackupMetadata("123", startTime); + BackupMetadata backupMetadata = + new BackupMetadata(BackupVersion.SNAPSHOT_BACKUP, "123", startTime); backupStatusMgr.start(backupMetadata); List metadataList = backupStatusMgr.locate(startTime); Assert.assertNotNull(metadataList); @@ -154,7 +158,8 @@ public void testSnapshotStatusMultiAddFinishInADay() throws Exception { for (int i = 0; i < noOfEntries; i++) { assert startTime != null; Date time = new DateTime(startTime.getTime()).plusHours(i).toDate(); - BackupMetadata backupMetadata = new BackupMetadata("123", time); + BackupMetadata backupMetadata = + new BackupMetadata(BackupVersion.SNAPSHOT_BACKUP, "123", time); backupStatusMgr.start(backupMetadata); backupStatusMgr.finish(backupMetadata); } @@ -182,7 +187,8 @@ public void testSnapshotStatusSize() throws Exception { for (int i = 0; i < noOfEntries; i++) { assert startTime != null; Date time = new DateTime(startTime.getTime()).plusDays(i).toDate(); - BackupMetadata backupMetadata = new BackupMetadata("123", time); + BackupMetadata backupMetadata = + new BackupMetadata(BackupVersion.SNAPSHOT_BACKUP, "123", time); backupStatusMgr.start(backupMetadata); backupStatusMgr.finish(backupMetadata); } @@ -198,7 +204,7 @@ public void getLatestBackup() throws Exception { Instant start = DateUtil.parseInstant(backupDate); List list = backupStatusMgr.getLatestBackupMetadata( - SnapshotBackup.BACKUP_VERSION, + BackupVersion.SNAPSHOT_BACKUP, new DateRange( backupDate + "," @@ -216,7 +222,7 @@ public void getLatestBackupFailure() throws Exception { Optional backupMetadata = backupStatusMgr .getLatestBackupMetadata( - SnapshotBackup.BACKUP_VERSION, + BackupVersion.SNAPSHOT_BACKUP, new DateRange(backupDate + "," + backupDate)) .stream() .findFirst(); @@ -227,7 +233,7 @@ public void getLatestBackupFailure() throws Exception { backupMetadata = backupStatusMgr .getLatestBackupMetadata( - SnapshotBackup.BACKUP_VERSION, + BackupVersion.SNAPSHOT_BACKUP, new DateRange(backupDate + "," + backupDate)) .stream() .findFirst(); @@ -239,7 +245,7 @@ public void getLatestBackupMetadata() throws Exception { prepare(); List list = backupStatusMgr.getLatestBackupMetadata( - SnapshotBackup.BACKUP_VERSION, + BackupVersion.SNAPSHOT_BACKUP, new DateRange(backupDate + "," + "201812031000")); list.forEach(backupMetadata -> System.out.println(backupMetadata)); } diff --git a/priam/src/test/java/com/netflix/priam/backup/TestBackupVerification.java b/priam/src/test/java/com/netflix/priam/backup/TestBackupVerification.java index 0a4ffbef8..cef662187 100644 --- a/priam/src/test/java/com/netflix/priam/backup/TestBackupVerification.java +++ b/priam/src/test/java/com/netflix/priam/backup/TestBackupVerification.java @@ -24,7 +24,6 @@ import com.netflix.priam.backupv2.MetaV2Proxy; import com.netflix.priam.config.IConfiguration; import com.netflix.priam.scheduler.UnsupportedTypeException; -import com.netflix.priam.services.SnapshotMetaService; import com.netflix.priam.utils.DateUtil; import com.netflix.priam.utils.DateUtil.DateRange; import java.io.File; @@ -88,20 +87,10 @@ public void cleanup() { FileUtils.deleteQuietly(new File(configuration.getBackupStatusFileLoc())); } - @Test - public void illegalBackupVersion() { - try { - backupVerification.verifyBackup(3, false, null); - Assert.assertTrue(false); - } catch (UnsupportedTypeException e) { - Assert.assertTrue(true); - } - } - @Test public void illegalDateRange() throws UnsupportedTypeException { try { - backupVerification.verifyBackup(1, false, null); + backupVerification.verifyBackup(BackupVersion.SNAPSHOT_BACKUP, false, null); Assert.assertTrue(false); } catch (IllegalArgumentException e) { Assert.assertTrue(true); @@ -112,14 +101,14 @@ public void illegalDateRange() throws UnsupportedTypeException { public void noBackup() throws Exception { Optional backupVerificationResultOptinal = backupVerification.verifyBackup( - SnapshotBackup.BACKUP_VERSION, + BackupVersion.SNAPSHOT_BACKUP, false, new DateRange(Instant.now(), Instant.now())); Assert.assertFalse(backupVerificationResultOptinal.isPresent()); backupVerificationResultOptinal = backupVerification.verifyBackup( - SnapshotMetaService.BACKUP_VERSION, + BackupVersion.SNAPSHOT_META_SERVICE, false, new DateRange(Instant.now(), Instant.now())); Assert.assertFalse(backupVerificationResultOptinal.isPresent()); @@ -128,14 +117,14 @@ public void noBackup() throws Exception { private void setUp() throws Exception { Instant start = DateUtil.parseInstant(backupDate); backupStatusMgr.finish( - getBackupMetaData(SnapshotBackup.BACKUP_VERSION, start, Status.FINISHED)); + getBackupMetaData(BackupVersion.SNAPSHOT_BACKUP, start, Status.FINISHED)); backupStatusMgr.failed( getBackupMetaData( - SnapshotBackup.BACKUP_VERSION, + BackupVersion.SNAPSHOT_BACKUP, start.plus(20, ChronoUnit.MINUTES), Status.FAILED)); backupStatusMgr.finish( - getBackupMetaData(SnapshotMetaService.BACKUP_VERSION, start, Status.FINISHED)); + getBackupMetaData(BackupVersion.SNAPSHOT_META_SERVICE, start, Status.FINISHED)); } @Test @@ -144,7 +133,7 @@ public void verifyBackupVersion1() throws Exception { // Verify for backup version 1.0 Optional backupVerificationResultOptinal = backupVerification.verifyBackup( - SnapshotBackup.BACKUP_VERSION, + BackupVersion.SNAPSHOT_BACKUP, false, new DateRange(backupDate + "," + backupDate)); Assert.assertTrue(backupVerificationResultOptinal.isPresent()); @@ -152,7 +141,7 @@ public void verifyBackupVersion1() throws Exception { Optional backupMetadata = backupStatusMgr .getLatestBackupMetadata( - SnapshotBackup.BACKUP_VERSION, + BackupVersion.SNAPSHOT_BACKUP, new DateRange(backupDate + "," + backupDate)) .stream() .findFirst(); @@ -162,7 +151,7 @@ public void verifyBackupVersion1() throws Exception { backupMetadata = backupStatusMgr .getLatestBackupMetadata( - SnapshotMetaService.BACKUP_VERSION, + BackupVersion.SNAPSHOT_META_SERVICE, new DateRange(backupDate + "," + backupDate)) .stream() .findFirst(); @@ -176,7 +165,7 @@ public void verifyBackupVersion2() throws Exception { // Verify for backup version 2.0 Optional backupVerificationResultOptinal = backupVerification.verifyBackup( - SnapshotMetaService.BACKUP_VERSION, + BackupVersion.SNAPSHOT_META_SERVICE, false, new DateRange(backupDate + "," + backupDate)); Assert.assertTrue(backupVerificationResultOptinal.isPresent()); @@ -186,7 +175,7 @@ public void verifyBackupVersion2() throws Exception { Optional backupMetadata = backupStatusMgr .getLatestBackupMetadata( - SnapshotMetaService.BACKUP_VERSION, + BackupVersion.SNAPSHOT_META_SERVICE, new DateRange(backupDate + "," + backupDate)) .stream() .findFirst(); @@ -196,7 +185,7 @@ public void verifyBackupVersion2() throws Exception { // Retry the verification, it should not try and re-verify backupVerificationResultOptinal = backupVerification.verifyBackup( - SnapshotMetaService.BACKUP_VERSION, + BackupVersion.SNAPSHOT_META_SERVICE, false, new DateRange(backupDate + "," + backupDate)); Assert.assertTrue(backupVerificationResultOptinal.isPresent()); @@ -211,7 +200,7 @@ public void verifyBackupVersion2() throws Exception { backupMetadata = backupStatusMgr .getLatestBackupMetadata( - SnapshotBackup.BACKUP_VERSION, + BackupVersion.SNAPSHOT_BACKUP, new DateRange(backupDate + "," + backupDate)) .stream() .findFirst(); @@ -219,8 +208,8 @@ public void verifyBackupVersion2() throws Exception { Assert.assertNull(backupMetadata.get().getLastValidated()); } - private BackupMetadata getBackupMetaData(int backupVersion, Instant startTime, Status status) - throws Exception { + private BackupMetadata getBackupMetaData( + BackupVersion backupVersion, Instant startTime, Status status) throws Exception { BackupMetadata backupMetadata = new BackupMetadata(backupVersion, "123", new Date(startTime.toEpochMilli())); backupMetadata.setCompleted( diff --git a/priam/src/test/java/com/netflix/priam/config/PriamConfigurationPersisterTest.java b/priam/src/test/java/com/netflix/priam/config/PriamConfigurationPersisterTest.java index 515c70d0a..c675d6e72 100644 --- a/priam/src/test/java/com/netflix/priam/config/PriamConfigurationPersisterTest.java +++ b/priam/src/test/java/com/netflix/priam/config/PriamConfigurationPersisterTest.java @@ -1,3 +1,19 @@ +/* + * Copyright 2017 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ package com.netflix.priam.config; import static junit.framework.TestCase.assertTrue; diff --git a/priam/src/test/java/com/netflix/priam/defaultimpl/FakeCassandraProcess.java b/priam/src/test/java/com/netflix/priam/defaultimpl/FakeCassandraProcess.java index 2905a0a8d..e123169d1 100644 --- a/priam/src/test/java/com/netflix/priam/defaultimpl/FakeCassandraProcess.java +++ b/priam/src/test/java/com/netflix/priam/defaultimpl/FakeCassandraProcess.java @@ -1,3 +1,19 @@ +/* + * Copyright 2017 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ package com.netflix.priam.defaultimpl; import java.io.IOException; diff --git a/priam/src/test/java/com/netflix/priam/health/TestInstanceStatus.java b/priam/src/test/java/com/netflix/priam/health/TestInstanceStatus.java index edba35aab..1005857cf 100644 --- a/priam/src/test/java/com/netflix/priam/health/TestInstanceStatus.java +++ b/priam/src/test/java/com/netflix/priam/health/TestInstanceStatus.java @@ -1,3 +1,19 @@ +/* + * Copyright 2017 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ package com.netflix.priam.health; import com.google.inject.Guice; diff --git a/priam/src/test/java/com/netflix/priam/resources/PriamConfigTest.java b/priam/src/test/java/com/netflix/priam/resources/PriamConfigTest.java index 048cd1334..163c17f46 100644 --- a/priam/src/test/java/com/netflix/priam/resources/PriamConfigTest.java +++ b/priam/src/test/java/com/netflix/priam/resources/PriamConfigTest.java @@ -1,3 +1,19 @@ +/* + * Copyright 2017 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ package com.netflix.priam.resources; import static org.junit.Assert.*; diff --git a/priam/src/test/java/com/netflix/priam/utils/TestGsonJsonSerializer.java b/priam/src/test/java/com/netflix/priam/utils/TestGsonJsonSerializer.java index a4fb8a09a..08ed259e2 100644 --- a/priam/src/test/java/com/netflix/priam/utils/TestGsonJsonSerializer.java +++ b/priam/src/test/java/com/netflix/priam/utils/TestGsonJsonSerializer.java @@ -17,6 +17,7 @@ package com.netflix.priam.utils; import com.netflix.priam.backup.BackupMetadata; +import com.netflix.priam.backup.BackupVersion; import com.netflix.priam.health.InstanceState; import java.time.LocalDateTime; import java.util.Calendar; @@ -32,7 +33,9 @@ public class TestGsonJsonSerializer { @Test public void testBackupMetaData() throws Exception { - BackupMetadata metadata = new BackupMetadata("123", Calendar.getInstance().getTime()); + BackupMetadata metadata = + new BackupMetadata( + BackupVersion.SNAPSHOT_BACKUP, "123", Calendar.getInstance().getTime()); String json = metadata.toString(); LOG.info(json); // Deserialize it. From 3465c7c11bc10dcf305b2d18e3af60a0b5b50dae Mon Sep 17 00:00:00 2001 From: Arun Agrawal Date: Wed, 30 Jan 2019 14:05:06 -0800 Subject: [PATCH 051/228] changelog --- CHANGELOG.md | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c1926cfce..cdba4026a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,11 @@ # Changelog -## 2018/01/11 3.11.38 +## 2019/01/30 3.11.39 +(#765) Add metrics on CassandraConfig resource calls +(#768) Support configure/tune complex parameters in cassandra.yaml +(#770) Add Cass SNAPSHOT JMX status, snapshot version, last validated timestamp. Changes to Servlet API and new APIs. + +## 2019/01/11 3.11.38 (#761) Add new file format (SST_V2) and methods to get/parse remote locations. (#761) Upload files from SnapshotMetaService in backup version 2.0, if enabled. (#761) Process older SNAPSHOT_V2 at the restart of Priam. From 281e1c08c53ea89ab752ae8324200941a6056cb9 Mon Sep 17 00:00:00 2001 From: Arun Agrawal Date: Thu, 7 Feb 2019 15:33:42 -0800 Subject: [PATCH 052/228] Do not throw NPE when no backup is found for the requested date. --- .../priam/resources/BackupServlet.java | 24 +++++++++++-------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/priam/src/main/java/com/netflix/priam/resources/BackupServlet.java b/priam/src/main/java/com/netflix/priam/resources/BackupServlet.java index 1c1fad89f..c33fa51f8 100644 --- a/priam/src/main/java/com/netflix/priam/resources/BackupServlet.java +++ b/priam/src/main/java/com/netflix/priam/resources/BackupServlet.java @@ -24,6 +24,9 @@ import com.netflix.priam.config.IConfiguration; import com.netflix.priam.scheduler.PriamScheduler; import com.netflix.priam.utils.DateUtil; +import com.netflix.priam.utils.DateUtil.DateRange; +import java.time.Instant; +import java.time.temporal.ChronoUnit; import java.util.*; import java.util.stream.Collectors; import javax.ws.rs.*; @@ -131,25 +134,26 @@ public Response status() throws Exception { @Path("/status/{date}") @Produces(MediaType.APPLICATION_JSON) public Response statusByDate(@PathParam("date") String date) throws Exception { - JSONObject object = new JSONObject(); + Instant startTime = DateUtil.parseInstant(date); Optional backupMetadataOptional = this.completedBkups - .locate(date) + .getLatestBackupMetadata( + BackupVersion.SNAPSHOT_BACKUP, + new DateRange( + startTime.truncatedTo(ChronoUnit.DAYS), + startTime + .plus(1, ChronoUnit.DAYS) + .truncatedTo(ChronoUnit.DAYS))) .stream() - .filter(Objects::nonNull) - .filter(backupMetadata -> backupMetadata.getStatus() == Status.FINISHED) - .filter( - backupMetadata -> - backupMetadata - .getBackupVersion() - .equals(BackupVersion.SNAPSHOT_BACKUP)) .findFirst(); + + JSONObject object = new JSONObject(); if (!backupMetadataOptional.isPresent()) { object.put("Snapshotstatus", false); } else { object.put("Snapshotstatus", true); - object.put("Details", backupMetadataOptional.get().toString()); + object.put("Details", backupMetadataOptional.get()); } return Response.ok(object.toString(), MediaType.APPLICATION_JSON).build(); } From 5d9de49b3a390b476127506cfd4ba94bfb7ad8f0 Mon Sep 17 00:00:00 2001 From: Arun Agrawal Date: Fri, 8 Feb 2019 11:08:02 -0800 Subject: [PATCH 053/228] update to netflixoss 7.0.0 --- build.gradle | 5 ++--- .../test/java/com/netflix/priam/tuner/dse/DseConfigStub.java | 2 +- .../test/java/com/netflix/priam/tuner/dse/DseTunerTest.java | 2 +- 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/build.gradle b/build.gradle index 970aac5e2..8adcd63cb 100644 --- a/build.gradle +++ b/build.gradle @@ -1,5 +1,5 @@ plugins { - id 'nebula.netflixoss' version '5.2.0' + id 'nebula.netflixoss' version '7.0.0' id 'com.github.sherter.google-java-format' version '0.7.1' } @@ -35,14 +35,13 @@ allprojects { compile 'org.apache.commons:commons-collections4:4.2' compile 'commons-io:commons-io:2.6' compile 'commons-cli:commons-cli:1.4' - compile 'commons-httpclient:commons-httpclient:3.1' compile 'com.sun.jersey.contribs:jersey-multipart:1.19.4' compile 'com.sun.jersey:jersey-json:1.19.4' compile 'com.sun.jersey:jersey-bundle:1.19.4' compile 'com.sun.jersey.contribs:jersey-guice:1.19.4' compile 'com.google.guava:guava:21.0' compile 'com.google.code.findbugs:jsr305:3.0.2' - compile 'com.amazonaws:aws-java-sdk:1.11.487' + compile 'com.amazonaws:aws-java-sdk:1.11.488' compile 'com.google.inject:guice:4.2.2' compile 'com.google.inject.extensions:guice-servlet:4.2.2' compile 'org.quartz-scheduler:quartz:2.3.0' diff --git a/priam/src/test/java/com/netflix/priam/tuner/dse/DseConfigStub.java b/priam/src/test/java/com/netflix/priam/tuner/dse/DseConfigStub.java index 999c6f557..ec008e2d2 100644 --- a/priam/src/test/java/com/netflix/priam/tuner/dse/DseConfigStub.java +++ b/priam/src/test/java/com/netflix/priam/tuner/dse/DseConfigStub.java @@ -1,4 +1,3 @@ -package com.netflix.priam.tuner.dse; /* * Copyright 2017 Netflix, Inc. * @@ -15,6 +14,7 @@ * limitations under the License. * */ +package com.netflix.priam.tuner.dse; import com.netflix.priam.config.FakeConfiguration; import java.util.HashSet; diff --git a/priam/src/test/java/com/netflix/priam/tuner/dse/DseTunerTest.java b/priam/src/test/java/com/netflix/priam/tuner/dse/DseTunerTest.java index 948a0d693..b65f88ae5 100644 --- a/priam/src/test/java/com/netflix/priam/tuner/dse/DseTunerTest.java +++ b/priam/src/test/java/com/netflix/priam/tuner/dse/DseTunerTest.java @@ -1,4 +1,3 @@ -package com.netflix.priam.tuner.dse; /* * Copyright 2017 Netflix, Inc. * @@ -15,6 +14,7 @@ * limitations under the License. * */ +package com.netflix.priam.tuner.dse; import com.google.common.io.Files; import com.netflix.priam.config.FakeConfiguration; From d7903475cc4f7db9fff03e3e30065e435a7ddd3b Mon Sep 17 00:00:00 2001 From: Arun Agrawal Date: Fri, 8 Feb 2019 15:20:53 -0800 Subject: [PATCH 054/228] Do not check existence of file if it is not SST_V2. Note: AmazonServiceException can be for either Amazon unable to process request or for slow down. In either case, it is better to try to upload. --- .../com/netflix/priam/aws/S3FileSystemBase.java | 16 ++++------------ .../netflix/priam/backup/AbstractFileSystem.java | 3 ++- 2 files changed, 6 insertions(+), 13 deletions(-) diff --git a/priam/src/main/java/com/netflix/priam/aws/S3FileSystemBase.java b/priam/src/main/java/com/netflix/priam/aws/S3FileSystemBase.java index 0bdb4ee4a..188d42f31 100755 --- a/priam/src/main/java/com/netflix/priam/aws/S3FileSystemBase.java +++ b/priam/src/main/java/com/netflix/priam/aws/S3FileSystemBase.java @@ -13,8 +13,6 @@ */ package com.netflix.priam.aws; -import com.amazonaws.AmazonClientException; -import com.amazonaws.AmazonServiceException; import com.amazonaws.services.s3.AmazonS3; import com.amazonaws.services.s3.model.BucketLifecycleConfiguration; import com.amazonaws.services.s3.model.BucketLifecycleConfiguration.Rule; @@ -168,18 +166,12 @@ public boolean doesRemoteFileExist(Path remotePath) throws BackupRestoreExceptio boolean exists = false; try { exists = s3Client.doesObjectExist(getShard(), remotePath.toString()); - } catch (AmazonServiceException ase) { - // Amazon S3 rejected request - throw new BackupRestoreException( - "AmazonServiceException while checking existence of object: " - + remotePath - + ". Error: " - + ase.getMessage()); - } catch (AmazonClientException ace) { + } catch (Exception ex) { // No point throwing this exception up. logger.error( - "AmazonClientException: client encountered a serious internal problem while trying to communicate with S3", - ace.getMessage()); + "Exception while checking existence of object: {}. Error: {}", + remotePath, + ex.getMessage()); } return exists; diff --git a/priam/src/main/java/com/netflix/priam/backup/AbstractFileSystem.java b/priam/src/main/java/com/netflix/priam/backup/AbstractFileSystem.java index c8126daff..3e659837c 100644 --- a/priam/src/main/java/com/netflix/priam/backup/AbstractFileSystem.java +++ b/priam/src/main/java/com/netflix/priam/backup/AbstractFileSystem.java @@ -19,6 +19,7 @@ import com.google.inject.Inject; import com.google.inject.Provider; +import com.netflix.priam.backup.AbstractBackupPath.BackupFileType; import com.netflix.priam.config.IConfiguration; import com.netflix.priam.merics.BackupMetrics; import com.netflix.priam.notification.BackupEvent; @@ -182,7 +183,7 @@ public void uploadFile( long uploadedFileSize; // Upload file if it not present at remote location. - if (!doesRemoteFileExist(remotePath)) { + if (path.getType() != BackupFileType.SST_V2 || !doesRemoteFileExist(remotePath)) { uploadedFileSize = new BoundedExponentialRetryCallable(500, 10000, retry) { @Override From bd1159d6097ebced342d6fab2f850e47d0b3e90f Mon Sep 17 00:00:00 2001 From: Arun Agrawal Date: Fri, 8 Feb 2019 15:35:24 -0800 Subject: [PATCH 055/228] Change the exception scope --- .../src/main/java/com/netflix/priam/aws/S3FileSystemBase.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/priam/src/main/java/com/netflix/priam/aws/S3FileSystemBase.java b/priam/src/main/java/com/netflix/priam/aws/S3FileSystemBase.java index 188d42f31..7287f61ee 100755 --- a/priam/src/main/java/com/netflix/priam/aws/S3FileSystemBase.java +++ b/priam/src/main/java/com/netflix/priam/aws/S3FileSystemBase.java @@ -13,6 +13,7 @@ */ package com.netflix.priam.aws; +import com.amazonaws.AmazonClientException; import com.amazonaws.services.s3.AmazonS3; import com.amazonaws.services.s3.model.BucketLifecycleConfiguration; import com.amazonaws.services.s3.model.BucketLifecycleConfiguration.Rule; @@ -166,7 +167,7 @@ public boolean doesRemoteFileExist(Path remotePath) throws BackupRestoreExceptio boolean exists = false; try { exists = s3Client.doesObjectExist(getShard(), remotePath.toString()); - } catch (Exception ex) { + } catch (AmazonClientException ex) { // No point throwing this exception up. logger.error( "Exception while checking existence of object: {}. Error: {}", From 8fe222026e3072d46655013f21ec3d549737e780 Mon Sep 17 00:00:00 2001 From: Vinay Chella Date: Wed, 13 Feb 2019 16:52:33 -0800 Subject: [PATCH 056/228] ninja fix from #766 ninja fix from https://github.com/Netflix/Priam/pull/766 --- .../main/java/com/netflix/priam/resources/CassandraConfig.java | 1 + 1 file changed, 1 insertion(+) diff --git a/priam/src/main/java/com/netflix/priam/resources/CassandraConfig.java b/priam/src/main/java/com/netflix/priam/resources/CassandraConfig.java index 206cedbe5..32ac810c8 100644 --- a/priam/src/main/java/com/netflix/priam/resources/CassandraConfig.java +++ b/priam/src/main/java/com/netflix/priam/resources/CassandraConfig.java @@ -122,6 +122,7 @@ public Response getReplacedIp() { @Path("/set_replaced_ip") public Response setReplacedIp(@QueryParam("ip") String ip) { try { + priamServer.getInstanceIdentity().setReplacedIp(ip); return Response.ok().build(); } catch (Exception e) { logger.error("Error while overriding replacement ip", e); From 2df9766bda412df3b5ca0167582c176d45d8b196 Mon Sep 17 00:00:00 2001 From: Arun Agrawal Date: Sat, 9 Feb 2019 00:25:43 -0800 Subject: [PATCH 057/228] Put a cache in front of remote file system call of objectExist. Add a rate limiter to s3 objectExist so we don't get slow down from S3. --- .../netflix/priam/aws/S3FileSystemBase.java | 26 ++++++++++++-- .../priam/backup/AbstractFileSystem.java | 35 ++++++++++++++++++- .../priam/backup/IBackupFileSystem.java | 4 +-- .../netflix/priam/backupv2/MetaV2Proxy.java | 15 +++----- .../netflix/priam/config/IConfiguration.java | 10 ++++++ .../priam/config/PriamConfiguration.java | 5 +++ .../google/GoogleEncryptedFileSystem.java | 6 ++++ .../priam/services/SnapshotMetaService.java | 2 +- .../priam/backup/FakeBackupFileSystem.java | 2 +- .../priam/backup/NullBackupFileSystem.java | 5 +++ 10 files changed, 90 insertions(+), 20 deletions(-) diff --git a/priam/src/main/java/com/netflix/priam/aws/S3FileSystemBase.java b/priam/src/main/java/com/netflix/priam/aws/S3FileSystemBase.java index 7287f61ee..902f71117 100755 --- a/priam/src/main/java/com/netflix/priam/aws/S3FileSystemBase.java +++ b/priam/src/main/java/com/netflix/priam/aws/S3FileSystemBase.java @@ -46,8 +46,8 @@ public abstract class S3FileSystemBase extends AbstractFileSystem { final IConfiguration config; final ICompression compress; final BlockingSubmitThreadPoolExecutor executor; - // a throttling mechanism, we can limit the amount of bytes uploaded to endpoint per second. final RateLimiter rateLimiter; + private final RateLimiter objectExistLimiter; S3FileSystemBase( Provider pathProvider, @@ -64,8 +64,27 @@ public abstract class S3FileSystemBase extends AbstractFileSystem { this.executor = new BlockingSubmitThreadPoolExecutor(threads, queue, config.getUploadTimeout()); + // a throttling mechanism, we can limit the amount of bytes uploaded to endpoint per second. + this.rateLimiter = RateLimiter.create(1); + // a throttling mechanism, we can limit the amount of S3 API calls endpoint per second. + this.objectExistLimiter = RateLimiter.create(1); + configChangeListener(); + } + + /* + Call this method to change the configuration in runtime via callback. + */ + public void configChangeListener() { + int objectExistLimit = config.getRemoteFileSystemObjectExistsThrottle(); + objectExistLimiter.setRate(objectExistLimit < 1 ? Double.MAX_VALUE : objectExistLimit); + double throttleLimit = config.getUploadThrottle(); - this.rateLimiter = RateLimiter.create(throttleLimit < 1 ? Double.MAX_VALUE : throttleLimit); + rateLimiter.setRate(throttleLimit < 1 ? Double.MAX_VALUE : throttleLimit); + + logger.info( + "Updating rateLimiters: s3UploadThrottle: {}, objectExistLimiter: {}", + rateLimiter.getRate(), + objectExistLimiter.getRate()); } private AmazonS3 getS3Client() { @@ -163,7 +182,8 @@ public long getFileSize(Path remotePath) throws BackupRestoreException { } @Override - public boolean doesRemoteFileExist(Path remotePath) throws BackupRestoreException { + protected boolean doesRemoteFileExist(Path remotePath) { + objectExistLimiter.acquire(); boolean exists = false; try { exists = s3Client.doesObjectExist(getShard(), remotePath.toString()); diff --git a/priam/src/main/java/com/netflix/priam/backup/AbstractFileSystem.java b/priam/src/main/java/com/netflix/priam/backup/AbstractFileSystem.java index 3e659837c..7f3bc6a34 100644 --- a/priam/src/main/java/com/netflix/priam/backup/AbstractFileSystem.java +++ b/priam/src/main/java/com/netflix/priam/backup/AbstractFileSystem.java @@ -17,6 +17,8 @@ package com.netflix.priam.backup; +import com.google.common.cache.Cache; +import com.google.common.cache.CacheBuilder; import com.google.inject.Inject; import com.google.inject.Provider; import com.netflix.priam.backup.AbstractBackupPath.BackupFileType; @@ -60,6 +62,7 @@ public abstract class AbstractFileSystem implements IBackupFileSystem, EventGene private final Set tasksQueued; private final ThreadPoolExecutor fileUploadExecutor; private final ThreadPoolExecutor fileDownloadExecutor; + private final Cache objectCache; @Inject public AbstractFileSystem( @@ -72,6 +75,8 @@ public AbstractFileSystem( this.pathProvider = pathProvider; // Add notifications. this.addObserver(backupNotificationMgr); + this.objectCache = + CacheBuilder.newBuilder().maximumSize(configuration.getBackupQueueSize()).build(); tasksQueued = new ConcurrentHashMap<>().newKeySet(); /* Note: We are using different queue for upload and download as with Backup V2.0 we might download all the meta @@ -183,7 +188,7 @@ public void uploadFile( long uploadedFileSize; // Upload file if it not present at remote location. - if (path.getType() != BackupFileType.SST_V2 || !doesRemoteFileExist(remotePath)) { + if (path.getType() != BackupFileType.SST_V2 || !checkObjectExists(remotePath)) { uploadedFileSize = new BoundedExponentialRetryCallable(500, 10000, retry) { @Override @@ -191,6 +196,12 @@ public Long retriableCall() throws Exception { return uploadFileImpl(localPath, remotePath); } }.call(); + + // Add to cache after successful upload. + // We only add SST_V2 as other file types are usually not checked, so no point + // evicting our SST_V2 results. + if (path.getType() == BackupFileType.SST_V2) addObjectCache(remotePath); + backupMetrics.recordUploadRate(uploadedFileSize); backupMetrics.incrementValidUploads(); } else { @@ -227,6 +238,28 @@ public Long retriableCall() throws Exception { } else logger.info("Already in queue, no-op. File: {}", localPath); } + private void addObjectCache(Path remotePath) { + objectCache.put(remotePath, Boolean.TRUE); + } + + @Override + public boolean checkObjectExists(Path remotePath) { + // Check in cache, if remote file exists. + Boolean cacheResult = objectCache.getIfPresent(remotePath); + + // Cache hit. Return the value. + if (cacheResult != null) return cacheResult; + + // Cache miss - Check remote file system if object exist. + boolean remoteFileExist = doesRemoteFileExist(remotePath); + + if (remoteFileExist) addObjectCache(remotePath); + + return remoteFileExist; + } + + protected abstract boolean doesRemoteFileExist(Path remotePath); + protected abstract long uploadFileImpl(final Path localPath, final Path remotePath) throws BackupRestoreException; diff --git a/priam/src/main/java/com/netflix/priam/backup/IBackupFileSystem.java b/priam/src/main/java/com/netflix/priam/backup/IBackupFileSystem.java index a8ef84209..75bbdcc60 100644 --- a/priam/src/main/java/com/netflix/priam/backup/IBackupFileSystem.java +++ b/priam/src/main/java/com/netflix/priam/backup/IBackupFileSystem.java @@ -180,10 +180,8 @@ default String getShard() { * * @param remotePath location on the remote file system. * @return boolean value indicating presence of the file on remote file system. - * @throws BackupRestoreException in case of failure to identify if object exists on the remote - * file system. */ - default boolean doesRemoteFileExist(Path remotePath) throws BackupRestoreException { + default boolean checkObjectExists(Path remotePath) { return false; } diff --git a/priam/src/main/java/com/netflix/priam/backupv2/MetaV2Proxy.java b/priam/src/main/java/com/netflix/priam/backupv2/MetaV2Proxy.java index 2a4167b1f..5dafd2472 100644 --- a/priam/src/main/java/com/netflix/priam/backupv2/MetaV2Proxy.java +++ b/priam/src/main/java/com/netflix/priam/backupv2/MetaV2Proxy.java @@ -222,17 +222,10 @@ public void process(ColumnfamilyResult columnfamilyResult) { for (ColumnfamilyResult.SSTableResult ssTableResult : columnfamilyResult.getSstables()) { for (FileUploadResult fileUploadResult : ssTableResult.getSstableComponents()) { - try { - if (fs.doesRemoteFileExist(Paths.get(fileUploadResult.getBackupPath()))) { - verificationResult.filesMatched++; - } else { - verificationResult.filesInMetaOnly.add( - fileUploadResult.getBackupPath()); - } - } catch (BackupRestoreException e) { - // For any error, mark that file is not available. - verificationResult.valid = false; - break; + if (fs.checkObjectExists(Paths.get(fileUploadResult.getBackupPath()))) { + verificationResult.filesMatched++; + } else { + verificationResult.filesInMetaOnly.add(fileUploadResult.getBackupPath()); } } } diff --git a/priam/src/main/java/com/netflix/priam/config/IConfiguration.java b/priam/src/main/java/com/netflix/priam/config/IConfiguration.java index 63549e635..d690a6934 100644 --- a/priam/src/main/java/com/netflix/priam/config/IConfiguration.java +++ b/priam/src/main/java/com/netflix/priam/config/IConfiguration.java @@ -456,6 +456,16 @@ default int getUploadThrottle() { return -1; } + /** + * Get the throttle limit for API call of remote file system - get object exist. Default: 10. + * Use value of -1 to disable this. + * + * @return throttle limit for get object exist API call. + */ + default int getRemoteFileSystemObjectExistsThrottle() { + return -1; + } + /** @return true if Priam should local config file for tokens and seeds */ default boolean isLocalBootstrapEnabled() { return false; diff --git a/priam/src/main/java/com/netflix/priam/config/PriamConfiguration.java b/priam/src/main/java/com/netflix/priam/config/PriamConfiguration.java index 4f4ae42d8..45f4365b5 100644 --- a/priam/src/main/java/com/netflix/priam/config/PriamConfiguration.java +++ b/priam/src/main/java/com/netflix/priam/config/PriamConfiguration.java @@ -338,6 +338,11 @@ public int getUploadThrottle() { return config.get(PRIAM_PRE + ".upload.throttle", -1); } + @Override + public int getRemoteFileSystemObjectExistsThrottle() { + return config.get(PRIAM_PRE + ".remoteFileSystemObjectExistThrottle", -1); + } + @Override public boolean isLocalBootstrapEnabled() { return config.get(PRIAM_PRE + ".localbootstrap.enable", false); diff --git a/priam/src/main/java/com/netflix/priam/google/GoogleEncryptedFileSystem.java b/priam/src/main/java/com/netflix/priam/google/GoogleEncryptedFileSystem.java index cc49b7679..a0e39f5a4 100755 --- a/priam/src/main/java/com/netflix/priam/google/GoogleEncryptedFileSystem.java +++ b/priam/src/main/java/com/netflix/priam/google/GoogleEncryptedFileSystem.java @@ -211,6 +211,12 @@ protected void downloadFileImpl(Path remotePath, Path localPath) throws BackupRe backupMetrics.recordDownloadRate(get.getLastResponseHeaders().getContentLength()); } + @Override + protected boolean doesRemoteFileExist(Path remotePath) { + // TODO: Implement based on GCS. Since this is only used for upload, leaving it empty + return false; + } + @Override public Iterator listFileSystem(String prefix, String delimiter, String marker) { return new GoogleFileIterator(constructGcsStorageHandle(), prefix, null); diff --git a/priam/src/main/java/com/netflix/priam/services/SnapshotMetaService.java b/priam/src/main/java/com/netflix/priam/services/SnapshotMetaService.java index 25449a86c..0566727b8 100644 --- a/priam/src/main/java/com/netflix/priam/services/SnapshotMetaService.java +++ b/priam/src/main/java/com/netflix/priam/services/SnapshotMetaService.java @@ -340,7 +340,7 @@ private void generateMetaFile( abstractBackupPath.parseLocal(file, AbstractBackupPath.BackupFileType.SST_V2); fileUploadResult.setBackupPath(abstractBackupPath.getRemotePath()); fileUploadResult.setUploaded( - fs.doesRemoteFileExist(Paths.get(fileUploadResult.getBackupPath()))); + fs.checkObjectExists(Paths.get(fileUploadResult.getBackupPath()))); } catch (Exception e) { logger.error( "Error while setting the remoteLocation or checking if file exists. Ignoring them as they are not fatal.", diff --git a/priam/src/test/java/com/netflix/priam/backup/FakeBackupFileSystem.java b/priam/src/test/java/com/netflix/priam/backup/FakeBackupFileSystem.java index b55d385c5..8591c43a8 100644 --- a/priam/src/test/java/com/netflix/priam/backup/FakeBackupFileSystem.java +++ b/priam/src/test/java/com/netflix/priam/backup/FakeBackupFileSystem.java @@ -116,7 +116,7 @@ public long getFileSize(Path remotePath) throws BackupRestoreException { } @Override - public boolean doesRemoteFileExist(Path remotePath) throws BackupRestoreException { + public boolean doesRemoteFileExist(Path remotePath) { for (AbstractBackupPath abstractBackupPath : flist) { if (abstractBackupPath.getRemotePath().equalsIgnoreCase(remotePath.toString())) return true; diff --git a/priam/src/test/java/com/netflix/priam/backup/NullBackupFileSystem.java b/priam/src/test/java/com/netflix/priam/backup/NullBackupFileSystem.java index 626b693ab..4f11276a8 100644 --- a/priam/src/test/java/com/netflix/priam/backup/NullBackupFileSystem.java +++ b/priam/src/test/java/com/netflix/priam/backup/NullBackupFileSystem.java @@ -66,6 +66,11 @@ public void cleanup() { protected void downloadFileImpl(Path remotePath, Path localPath) throws BackupRestoreException {} + @Override + protected boolean doesRemoteFileExist(Path remotePath) { + return false; + } + @Override protected long uploadFileImpl(Path localPath, Path remotePath) throws BackupRestoreException { return 0; From dd478c9ca882017e3977505b3bd5af398562c830 Mon Sep 17 00:00:00 2001 From: Arun Agrawal Date: Fri, 15 Feb 2019 20:33:51 -0800 Subject: [PATCH 058/228] Provide an overwrite method to force Priam to replace a particular ip (#783) This allows us to work around the A->B->C replacement problem where Priam gets confused about who to replace --- .../priam/resources/CassandraConfig.java | 2 ++ .../priam/resources/CassandraConfigTest.java | 22 +++++++++++++++++++ 2 files changed, 24 insertions(+) diff --git a/priam/src/main/java/com/netflix/priam/resources/CassandraConfig.java b/priam/src/main/java/com/netflix/priam/resources/CassandraConfig.java index 32ac810c8..aacb0fe21 100644 --- a/priam/src/main/java/com/netflix/priam/resources/CassandraConfig.java +++ b/priam/src/main/java/com/netflix/priam/resources/CassandraConfig.java @@ -31,6 +31,7 @@ import javax.ws.rs.QueryParam; import javax.ws.rs.core.MediaType; import javax.ws.rs.core.Response; +import javax.ws.rs.core.Response.Status; import org.apache.commons.lang3.StringUtils; import org.json.simple.JSONValue; import org.slf4j.Logger; @@ -121,6 +122,7 @@ public Response getReplacedIp() { @POST @Path("/set_replaced_ip") public Response setReplacedIp(@QueryParam("ip") String ip) { + if (StringUtils.isEmpty(ip)) return Response.status(Status.BAD_REQUEST).build(); try { priamServer.getInstanceIdentity().setReplacedIp(ip); return Response.ok().build(); diff --git a/priam/src/test/java/com/netflix/priam/resources/CassandraConfigTest.java b/priam/src/test/java/com/netflix/priam/resources/CassandraConfigTest.java index d01991219..d590c5f48 100644 --- a/priam/src/test/java/com/netflix/priam/resources/CassandraConfigTest.java +++ b/priam/src/test/java/com/netflix/priam/resources/CassandraConfigTest.java @@ -18,6 +18,7 @@ package com.netflix.priam.resources; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; import com.google.common.collect.ImmutableList; @@ -44,11 +45,14 @@ public class CassandraConfigTest { private @Mocked PriamServer priamServer; private @Mocked DoubleRing doubleRing; private CassandraConfig resource; + private InstanceIdentity instanceIdentity; @Before public void setUp() { CassMonitorMetrics cassMonitorMetrics = Guice.createInjector(new BRTestModule()).getInstance(CassMonitorMetrics.class); + instanceIdentity = + Guice.createInjector(new BRTestModule()).getInstance(InstanceIdentity.class); resource = new CassandraConfig(priamServer, doubleRing, cassMonitorMetrics); } @@ -213,6 +217,24 @@ public void getReplacedAddress(@Mocked final InstanceIdentity identity) { assertEquals(replacedIp, response.getEntity()); } + @Test + public void setReplacedIp() { + new Expectations() { + { + priamServer.getInstanceIdentity(); + result = instanceIdentity; + } + }; + + Response response = resource.setReplacedIp("127.0.0.1"); + assertEquals(200, response.getStatus()); + assertEquals("127.0.0.1", instanceIdentity.getReplacedIp()); + assertTrue(instanceIdentity.isReplace()); + + response = resource.setReplacedIp(null); + assertEquals(400, response.getStatus()); + } + @Test public void doubleRing() throws Exception { new Expectations() { From 263dd5646e53ef7d73e58c63db373222d591952b Mon Sep 17 00:00:00 2001 From: Arun Agrawal Date: Fri, 15 Feb 2019 21:20:26 -0800 Subject: [PATCH 059/228] BackupVerificationService (#784) --- .../java/com/netflix/priam/PriamServer.java | 13 +++ .../priam/config/BackupRestoreConfig.java | 10 ++ .../priam/config/IBackupRestoreConfig.java | 25 +++++ .../priam/services/BackupTTLService.java | 12 +-- .../services/BackupVerificationService.java | 101 ++++++++++++++++++ .../TestBackupVerificationService.java | 91 ++++++++++++++++ 6 files changed, 244 insertions(+), 8 deletions(-) create mode 100644 priam/src/main/java/com/netflix/priam/services/BackupVerificationService.java create mode 100644 priam/src/test/java/com/netflix/priam/services/TestBackupVerificationService.java diff --git a/priam/src/main/java/com/netflix/priam/PriamServer.java b/priam/src/main/java/com/netflix/priam/PriamServer.java index 6e04dffaf..2aaf2b49a 100644 --- a/priam/src/main/java/com/netflix/priam/PriamServer.java +++ b/priam/src/main/java/com/netflix/priam/PriamServer.java @@ -231,6 +231,19 @@ private void setUpSnapshotService() throws Exception { BackupTTLService.JOBNAME, backupTTLTimer.getCronExpression()); } + + // Start the Incremental backup schedule if enabled + if (config.isIncrementalBackupEnabled() && backupRestoreConfig.enableV2Backups()) { + // Delete the old task, if scheduled. This is required, as we stop taking backup + // 1.0, we still want to take incremental backups + // Once backup 1.0 is gone, we should not check for enableV2Backups.. + scheduler.deleteTask(IncrementalBackup.JOBNAME); + scheduler.addTask( + IncrementalBackup.JOBNAME, + IncrementalBackup.class, + IncrementalBackup.getTimer()); + logger.info("Added incremental backup job"); + } } } diff --git a/priam/src/main/java/com/netflix/priam/config/BackupRestoreConfig.java b/priam/src/main/java/com/netflix/priam/config/BackupRestoreConfig.java index aae6b6739..6685eb96c 100644 --- a/priam/src/main/java/com/netflix/priam/config/BackupRestoreConfig.java +++ b/priam/src/main/java/com/netflix/priam/config/BackupRestoreConfig.java @@ -45,4 +45,14 @@ public boolean enableV2Restore() { public String getBackupTTLCronExpression() { return config.get("priam.backupTTLCronExpression", "0 0 0/6 1/1 * ? *"); } + + @Override + public int getBackupVerificationSLOInHours() { + return config.get("priam.backupVerificationSLOInHours", 24); + } + + @Override + public String getBackupVerificationCronExpression() { + return config.get("priam.backupVerificationCronExpression", "0 30 0/1 1/1 * ? *"); + } } diff --git a/priam/src/main/java/com/netflix/priam/config/IBackupRestoreConfig.java b/priam/src/main/java/com/netflix/priam/config/IBackupRestoreConfig.java index bfdfbca4f..6c0b53e76 100644 --- a/priam/src/main/java/com/netflix/priam/config/IBackupRestoreConfig.java +++ b/priam/src/main/java/com/netflix/priam/config/IBackupRestoreConfig.java @@ -60,6 +60,31 @@ default String getBackupTTLCronExpression() { return "0 0 0/6 1/1 * ? *"; } + /** + * Cron expression to be used for the service which does verification of the backups. This + * service will run only if v2 backups are enabled. + * + * @return Backup Verification Service cron expression for trying to verify backups. Default: + * run every hour at 30 minutes. + * @see quartz-scheduler + * @see http://www.cronmaker.com + */ + default String getBackupVerificationCronExpression() { + return "0 30 0/1 1/1 * ? *"; + } + + /** + * The default backup SLO for any cluster. This will ensure that we upload and validate a backup + * in that SLO window. If no valid backup is found, we log ERROR message. This service will run + * only if v2 backups are enabled. + * + * @return the backup SLO in hours. Default: 24 hours. + */ + default int getBackupVerificationSLOInHours() { + return 24; + } + /** * If restore is enabled and if this flag is enabled, we will try to restore using Backup V2.0. * diff --git a/priam/src/main/java/com/netflix/priam/services/BackupTTLService.java b/priam/src/main/java/com/netflix/priam/services/BackupTTLService.java index 370673826..bbf98a43a 100644 --- a/priam/src/main/java/com/netflix/priam/services/BackupTTLService.java +++ b/priam/src/main/java/com/netflix/priam/services/BackupTTLService.java @@ -87,10 +87,7 @@ public BackupTTLService( @Override public void execute() throws Exception { // Ensure that backup version 2.0 is actually enabled. - if (!backupRestoreConfig.enableV2Backups() - && backupRestoreConfig - .getSnapshotMetaServiceCronExpression() - .equalsIgnoreCase("-1")) { + if (backupRestoreConfig.getSnapshotMetaServiceCronExpression().equalsIgnoreCase("-1")) { logger.info("Not executing the TTL Service for backups as V2 backups are not enabled."); return; } @@ -211,10 +208,9 @@ public String getName() { /** * Interval between trying to TTL data on Remote file system. * - * @param backupRestoreConfig {@link - * IBackupRestoreConfig#getSnapshotMetaServiceCronExpression()} to get configuration details - * from priam. Use "-1" to disable the service. - * @return the timer to be used for snapshot meta service. + * @param backupRestoreConfig {@link IBackupRestoreConfig#getBackupTTLCronExpression()} to get + * configuration details from priam. Use "-1" to disable the service. + * @return the timer to be used for backup ttl service. * @throws Exception if the configuration is not set correctly or are not valid. This is to * ensure we fail-fast. */ diff --git a/priam/src/main/java/com/netflix/priam/services/BackupVerificationService.java b/priam/src/main/java/com/netflix/priam/services/BackupVerificationService.java new file mode 100644 index 000000000..44e681d18 --- /dev/null +++ b/priam/src/main/java/com/netflix/priam/services/BackupVerificationService.java @@ -0,0 +1,101 @@ +/* + * Copyright 2019 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.netflix.priam.services; + +import com.google.inject.Inject; +import com.google.inject.Singleton; +import com.netflix.priam.backup.BackupVerification; +import com.netflix.priam.backup.BackupVerificationResult; +import com.netflix.priam.backup.BackupVersion; +import com.netflix.priam.config.IBackupRestoreConfig; +import com.netflix.priam.config.IConfiguration; +import com.netflix.priam.scheduler.CronTimer; +import com.netflix.priam.scheduler.Task; +import com.netflix.priam.scheduler.TaskTimer; +import com.netflix.priam.utils.DateUtil; +import com.netflix.priam.utils.DateUtil.DateRange; +import java.time.temporal.ChronoUnit; +import java.util.Optional; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** Created by aagrawal on 1/28/19. */ +@Singleton +public class BackupVerificationService extends Task { + private static final Logger logger = LoggerFactory.getLogger(BackupVerificationService.class); + public static final String JOBNAME = "BackupVerificationService"; + + private IBackupRestoreConfig backupRestoreConfig; + private BackupVerification backupVerification; + + @Inject + public BackupVerificationService( + IConfiguration configuration, + IBackupRestoreConfig backupRestoreConfig, + BackupVerification backupVerification) { + super(configuration); + this.backupRestoreConfig = backupRestoreConfig; + this.backupVerification = backupVerification; + } + + @Override + public void execute() throws Exception { + // Ensure that backup version 2.0 is actually enabled. + if (backupRestoreConfig.getSnapshotMetaServiceCronExpression().equalsIgnoreCase("-1")) { + logger.info( + "Not executing the Verification Service for backups as V2 backups are not enabled."); + return; + } + + // Validate the backup done in last x hours. + DateRange dateRange = + new DateRange( + DateUtil.getInstant() + .minus( + backupRestoreConfig.getBackupVerificationSLOInHours(), + ChronoUnit.HOURS), + DateUtil.getInstant()); + Optional verificationResult = + backupVerification.verifyBackup( + BackupVersion.SNAPSHOT_META_SERVICE, false, dateRange); + if (verificationResult.isPresent() && !verificationResult.get().valid) { + logger.error( + "Not able to find any snapshot which is valid in our SLO window: {} hours", + backupRestoreConfig.getBackupVerificationSLOInHours()); + } + } + + /** + * Interval between trying to verify data manifest file on Remote file system. + * + * @param backupRestoreConfig {@link IBackupRestoreConfig#getBackupVerificationCronExpression()} + * to get configuration details from priam. Use "-1" to disable the service. + * @return the timer to be used for snapshot verification service. + * @throws Exception if the configuration is not set correctly or are not valid. This is to + * ensure we fail-fast. + */ + public static TaskTimer getTimer(IBackupRestoreConfig backupRestoreConfig) throws Exception { + String cronExpression = backupRestoreConfig.getBackupVerificationCronExpression(); + return CronTimer.getCronTimer(JOBNAME, cronExpression); + } + + @Override + public String getName() { + return JOBNAME; + } +} diff --git a/priam/src/test/java/com/netflix/priam/services/TestBackupVerificationService.java b/priam/src/test/java/com/netflix/priam/services/TestBackupVerificationService.java new file mode 100644 index 000000000..06ad75d43 --- /dev/null +++ b/priam/src/test/java/com/netflix/priam/services/TestBackupVerificationService.java @@ -0,0 +1,91 @@ +/* + * Copyright 2019 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.netflix.priam.services; + +import com.google.inject.Guice; +import com.google.inject.Injector; +import com.netflix.priam.backup.BRTestModule; +import com.netflix.priam.backup.BackupVerification; +import com.netflix.priam.backup.BackupVerificationResult; +import com.netflix.priam.backup.BackupVersion; +import com.netflix.priam.config.IConfiguration; +import com.netflix.priam.scheduler.UnsupportedTypeException; +import com.netflix.priam.utils.DateUtil.DateRange; +import java.util.Optional; +import mockit.Mock; +import mockit.MockUp; +import org.junit.Assert; +import org.junit.Test; + +/** Created by aagrawal on 2/1/19. */ +public class TestBackupVerificationService { + private static BackupVerificationService backupVerificationService; + private static IConfiguration configuration; + + public TestBackupVerificationService() { + new MockBackupVerification(); + Injector injector = Guice.createInjector(new BRTestModule()); + if (configuration == null) configuration = injector.getInstance(IConfiguration.class); + if (backupVerificationService == null) + backupVerificationService = injector.getInstance(BackupVerificationService.class); + } + + static class MockBackupVerification extends MockUp { + public static boolean failCall = false; + public static boolean throwError = false; + + @Mock + public Optional verifyBackup( + BackupVersion backupVersion, boolean force, DateRange dateRange) + throws UnsupportedTypeException, IllegalArgumentException { + if (throwError) throw new IllegalArgumentException("DummyError"); + + if (failCall) return Optional.of(new BackupVerificationResult()); + + BackupVerificationResult result = new BackupVerificationResult(); + result.valid = true; + return Optional.of(result); + } + } + + @Test + public void throwError() throws Exception { + MockBackupVerification.throwError = true; + MockBackupVerification.failCall = false; + try { + backupVerificationService.execute(); + Assert.assertTrue(false); + } catch (IllegalArgumentException e) { + if (!e.getMessage().equalsIgnoreCase("DummyError")) Assert.assertTrue(false); + } + } + + @Test + public void failCalls() throws Exception { + MockBackupVerification.throwError = false; + MockBackupVerification.failCall = true; + backupVerificationService.execute(); + } + + @Test + public void normalOperation() throws Exception { + MockBackupVerification.throwError = false; + MockBackupVerification.failCall = false; + backupVerificationService.execute(); + } +} From a7a1c7fdcba970e0eb4499dcdd15cc09cf56a0b2 Mon Sep 17 00:00:00 2001 From: Arun Agrawal Date: Sat, 16 Feb 2019 16:08:06 -0800 Subject: [PATCH 060/228] move cassandra monitor to services --- .../java/com/netflix/priam/PriamServer.java | 2 +- .../netflix/priam/backup/SnapshotBackup.java | 2 +- .../management/IClusterManagement.java | 2 +- .../defaultimpl/CassandraOperations.java | 31 +++------- .../defaultimpl/ICassandraOperations.java | 60 +++++++++++++++++++ .../{utils => services}/CassandraMonitor.java | 5 +- .../priam/services/SnapshotMetaService.java | 1 - .../com/netflix/priam/utils/JMXNodeTool.java | 2 + .../cluser/management/TestCompaction.groovy | 2 +- .../TestCassandraMonitor.java | 6 +- 10 files changed, 82 insertions(+), 31 deletions(-) create mode 100644 priam/src/main/java/com/netflix/priam/defaultimpl/ICassandraOperations.java rename priam/src/main/java/com/netflix/priam/{utils => services}/CassandraMonitor.java (98%) rename priam/src/test/java/com/netflix/priam/{utils => services}/TestCassandraMonitor.java (96%) diff --git a/priam/src/main/java/com/netflix/priam/PriamServer.java b/priam/src/main/java/com/netflix/priam/PriamServer.java index 2aaf2b49a..39112ff60 100644 --- a/priam/src/main/java/com/netflix/priam/PriamServer.java +++ b/priam/src/main/java/com/netflix/priam/PriamServer.java @@ -37,7 +37,7 @@ import com.netflix.priam.services.BackupTTLService; import com.netflix.priam.services.SnapshotMetaService; import com.netflix.priam.tuner.TuneCassandra; -import com.netflix.priam.utils.CassandraMonitor; +import com.netflix.priam.services.CassandraMonitor; import com.netflix.priam.utils.Sleeper; import com.netflix.priam.utils.SystemUtils; import java.io.IOException; diff --git a/priam/src/main/java/com/netflix/priam/backup/SnapshotBackup.java b/priam/src/main/java/com/netflix/priam/backup/SnapshotBackup.java index 476d4eb5d..b8d24a384 100644 --- a/priam/src/main/java/com/netflix/priam/backup/SnapshotBackup.java +++ b/priam/src/main/java/com/netflix/priam/backup/SnapshotBackup.java @@ -27,7 +27,7 @@ import com.netflix.priam.identity.InstanceIdentity; import com.netflix.priam.scheduler.CronTimer; import com.netflix.priam.scheduler.TaskTimer; -import com.netflix.priam.utils.CassandraMonitor; +import com.netflix.priam.services.CassandraMonitor; import com.netflix.priam.utils.DateUtil; import com.netflix.priam.utils.ThreadSleeper; import java.io.File; diff --git a/priam/src/main/java/com/netflix/priam/cluster/management/IClusterManagement.java b/priam/src/main/java/com/netflix/priam/cluster/management/IClusterManagement.java index a6df274c2..64c13701a 100644 --- a/priam/src/main/java/com/netflix/priam/cluster/management/IClusterManagement.java +++ b/priam/src/main/java/com/netflix/priam/cluster/management/IClusterManagement.java @@ -16,7 +16,7 @@ import com.netflix.priam.config.IConfiguration; import com.netflix.priam.merics.IMeasurement; import com.netflix.priam.scheduler.Task; -import com.netflix.priam.utils.CassandraMonitor; +import com.netflix.priam.services.CassandraMonitor; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; import org.slf4j.Logger; diff --git a/priam/src/main/java/com/netflix/priam/defaultimpl/CassandraOperations.java b/priam/src/main/java/com/netflix/priam/defaultimpl/CassandraOperations.java index f13ff383b..5139d1e74 100644 --- a/priam/src/main/java/com/netflix/priam/defaultimpl/CassandraOperations.java +++ b/priam/src/main/java/com/netflix/priam/defaultimpl/CassandraOperations.java @@ -14,6 +14,7 @@ package com.netflix.priam.defaultimpl; import com.netflix.priam.config.IConfiguration; +import com.netflix.priam.utils.JMXConnectionException; import com.netflix.priam.utils.JMXNodeTool; import com.netflix.priam.utils.RetryableCallable; import java.util.*; @@ -23,7 +24,7 @@ import org.slf4j.LoggerFactory; /** This class encapsulates interactions with Cassandra. Created by aagrawal on 6/19/18. */ -public class CassandraOperations { +public class CassandraOperations implements ICassandraOperations { private static final Logger logger = LoggerFactory.getLogger(CassandraOperations.class); private final IConfiguration configuration; @@ -32,27 +33,14 @@ public class CassandraOperations { this.configuration = configuration; } - /** - * This method neds to be synchronized. Context: During the transition phase to backup version - * 2.0, we might be executing multiple snapshots at the same time. To avoid, unknown behavior by - * Cassanddra, it is wise to keep this method sync. Also, with backups being on CRON, we don't - * know how often operator is taking snapshot. - * - * @param snapshotName Name of the snapshot on disk. This snapshotName should be UNIQUE among - * all the snapshots. Try to append UUID to snapshotName to ensure uniqueness. This is to - * ensure a) Snapshot fails if name are not unique. b) You might take snapshots which are - * not "part" of same snapshot. e.g. Any leftovers from previous operation. c) Once snapshot - * fails, this will clean the failed snapshot. - * @throws Exception in case of error while taking a snapshot by Cassandra. - */ + @Override public synchronized void takeSnapshot(final String snapshotName) throws Exception { // Retry max of 6 times with 10 second in between (for one minute). This is to ensure that // we overcome any temporary glitch. // Note that operation MAY fail if cassandra successfully took the snapshot of certain // columnfamily(ies) and we try to create snapshot with // same name. It is a good practice to call clearSnapshot after this operation fails, to - // ensure we don't leave - // any left overs. + // ensure we don't leave any left overs. // Example scenario: Change of file permissions by manual intervention and C* unable to take // snapshot of one CF. try { @@ -72,12 +60,7 @@ public Void retriableCall() throws Exception { } } - /** - * Clear the snapshot tag from disk. - * - * @param snapshotTag Name of the snapshot to be removed. - * @throws Exception in case of error while clearing a snapshot. - */ + @Override public void clearSnapshot(final String snapshotTag) throws Exception { new RetryableCallable() { public Void retriableCall() throws Exception { @@ -88,6 +71,7 @@ public Void retriableCall() throws Exception { }.call(); } + @Override public List getKeyspaces() throws Exception { return new RetryableCallable>() { public List retriableCall() throws Exception { @@ -98,6 +82,7 @@ public List retriableCall() throws Exception { }.call(); } + @Override public Map> getColumnfamilies() throws Exception { return new RetryableCallable>>() { public Map> retriableCall() throws Exception { @@ -118,6 +103,7 @@ public Map> retriableCall() throws Exception { }.call(); } + @Override public void forceKeyspaceCompaction(String keyspaceName, String... columnfamilies) throws Exception { new RetryableCallable() { @@ -130,6 +116,7 @@ public Void retriableCall() throws Exception { }.call(); } + @Override public void forceKeyspaceFlush(String keyspaceName) throws Exception { new RetryableCallable() { public Void retriableCall() throws Exception { diff --git a/priam/src/main/java/com/netflix/priam/defaultimpl/ICassandraOperations.java b/priam/src/main/java/com/netflix/priam/defaultimpl/ICassandraOperations.java new file mode 100644 index 000000000..aba793164 --- /dev/null +++ b/priam/src/main/java/com/netflix/priam/defaultimpl/ICassandraOperations.java @@ -0,0 +1,60 @@ +/* + * Copyright 2019 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.netflix.priam.defaultimpl; + +import java.util.List; +import java.util.Map; + +/** + * Created by aagrawal on 2/16/19. + */ +public interface ICassandraOperations { + + /** + * This method neds to be synchronized. Context: During the transition phase to backup version + * 2.0, we might be executing multiple snapshots at the same time. To avoid, unknown behavior by + * Cassandra, it is wise to keep this method sync. Also, with backups being on CRON, we don't + * know how often operator is taking snapshot. + * + * @param snapshotName Name of the snapshot on disk. This snapshotName should be UNIQUE among + * all the snapshots. Try to append UUID to snapshotName to ensure uniqueness. This is to + * ensure a) Snapshot fails if name are not unique. b) You might take snapshots which are + * not "part" of same snapshot. e.g. Any leftovers from previous operation. c) Once snapshot + * fails, this will clean the failed snapshot. + * @throws Exception in case of error while taking a snapshot by Cassandra. + */ + void takeSnapshot(final String snapshotName) throws Exception; + + /** + * Clear the snapshot tag from disk. + * + * @param snapshotTag Name of the snapshot to be removed. + * @throws Exception in case of error while clearing a snapshot. + */ + void clearSnapshot(final String snapshotTag) throws Exception; + + /** + * Get all the keyspaces existing on this node. + * @return List of keyspace names. + * @throws Exception in case of reaching to JMX endpoint. + */ + List getKeyspaces() throws Exception; + Map> getColumnfamilies() throws Exception; + void forceKeyspaceCompaction(String keyspaceName, String... columnfamilies) throws Exception; + void forceKeyspaceFlush(String keyspaceName) throws Exception; +} diff --git a/priam/src/main/java/com/netflix/priam/utils/CassandraMonitor.java b/priam/src/main/java/com/netflix/priam/services/CassandraMonitor.java similarity index 98% rename from priam/src/main/java/com/netflix/priam/utils/CassandraMonitor.java rename to priam/src/main/java/com/netflix/priam/services/CassandraMonitor.java index b4772489d..bdb0cbaea 100644 --- a/priam/src/main/java/com/netflix/priam/utils/CassandraMonitor.java +++ b/priam/src/main/java/com/netflix/priam/services/CassandraMonitor.java @@ -1,5 +1,5 @@ /* - * Copyright 2013 Netflix, Inc. + * Copyright 2019 Netflix, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,7 +14,7 @@ * limitations under the License. * */ -package com.netflix.priam.utils; +package com.netflix.priam.services; import com.google.inject.Inject; import com.google.inject.Singleton; @@ -25,6 +25,7 @@ import com.netflix.priam.scheduler.SimpleTimer; import com.netflix.priam.scheduler.Task; import com.netflix.priam.scheduler.TaskTimer; +import com.netflix.priam.utils.JMXNodeTool; import java.io.BufferedReader; import java.io.File; import java.io.IOException; diff --git a/priam/src/main/java/com/netflix/priam/services/SnapshotMetaService.java b/priam/src/main/java/com/netflix/priam/services/SnapshotMetaService.java index 0566727b8..8c01cc889 100644 --- a/priam/src/main/java/com/netflix/priam/services/SnapshotMetaService.java +++ b/priam/src/main/java/com/netflix/priam/services/SnapshotMetaService.java @@ -23,7 +23,6 @@ import com.netflix.priam.identity.InstanceIdentity; import com.netflix.priam.scheduler.CronTimer; import com.netflix.priam.scheduler.TaskTimer; -import com.netflix.priam.utils.CassandraMonitor; import com.netflix.priam.utils.DateUtil; import java.io.File; import java.nio.file.Paths; diff --git a/priam/src/main/java/com/netflix/priam/utils/JMXNodeTool.java b/priam/src/main/java/com/netflix/priam/utils/JMXNodeTool.java index b87b97849..487da689b 100644 --- a/priam/src/main/java/com/netflix/priam/utils/JMXNodeTool.java +++ b/priam/src/main/java/com/netflix/priam/utils/JMXNodeTool.java @@ -19,6 +19,7 @@ import com.google.inject.Inject; import com.google.inject.Singleton; import com.netflix.priam.config.IConfiguration; +import com.netflix.priam.services.CassandraMonitor; import java.io.IOException; import java.io.PrintStream; import java.lang.management.ManagementFactory; @@ -256,6 +257,7 @@ public JSONObject info() throws JSONException { JSONObject object = new JSONObject(); object.put("gossip_active", isInitialized()); object.put("thrift_active", isThriftServerRunning()); + object.put("native_active", isNativeTransportRunning()); object.put("token", getTokens().toString()); object.put("load", getLoadString()); object.put("generation_no", getCurrentGenerationNumber()); diff --git a/priam/src/test/groovy/com/netflix/priam/cluser/management/TestCompaction.groovy b/priam/src/test/groovy/com/netflix/priam/cluser/management/TestCompaction.groovy index 4c6f3a0a6..5838d298b 100644 --- a/priam/src/test/groovy/com/netflix/priam/cluser/management/TestCompaction.groovy +++ b/priam/src/test/groovy/com/netflix/priam/cluser/management/TestCompaction.groovy @@ -21,7 +21,7 @@ import com.netflix.priam.config.FakeConfiguration import com.netflix.priam.backup.BRTestModule import com.netflix.priam.cluster.management.Compaction import com.netflix.priam.defaultimpl.CassandraOperations -import com.netflix.priam.utils.CassandraMonitor +import com.netflix.priam.services.CassandraMonitor import mockit.Mock import mockit.MockUp import spock.lang.Shared diff --git a/priam/src/test/java/com/netflix/priam/utils/TestCassandraMonitor.java b/priam/src/test/java/com/netflix/priam/services/TestCassandraMonitor.java similarity index 96% rename from priam/src/test/java/com/netflix/priam/utils/TestCassandraMonitor.java rename to priam/src/test/java/com/netflix/priam/services/TestCassandraMonitor.java index d38b4ca80..e31618c73 100644 --- a/priam/src/test/java/com/netflix/priam/utils/TestCassandraMonitor.java +++ b/priam/src/test/java/com/netflix/priam/services/TestCassandraMonitor.java @@ -1,5 +1,5 @@ /* - * Copyright 2017 Netflix, Inc. + * Copyright 2019 Netflix, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,7 +15,7 @@ * */ -package com.netflix.priam.utils; +package com.netflix.priam.services; import com.google.inject.Guice; import com.google.inject.Injector; @@ -24,6 +24,8 @@ import com.netflix.priam.defaultimpl.ICassandraProcess; import com.netflix.priam.health.InstanceState; import com.netflix.priam.merics.CassMonitorMetrics; +import com.netflix.priam.services.CassandraMonitor; +import com.netflix.priam.utils.JMXNodeTool; import java.io.ByteArrayInputStream; import java.io.InputStream; import mockit.*; From bcc16c3959a0b5444434c56424dc3ec4dc1ca38f Mon Sep 17 00:00:00 2001 From: Arun Agrawal Date: Sat, 16 Feb 2019 16:56:38 -0800 Subject: [PATCH 061/228] Move JMX to connection package. --- .../java/com/netflix/priam/PriamServer.java | 2 +- .../priam/connection/INodeToolObservable.java | 26 +++++++ .../priam/connection/INodeToolObserver.java | 27 +++++++ .../connection/JMXConnectionException.java | 32 +++++++++ .../{utils => connection}/JMXNodeTool.java | 15 +++- .../defaultimpl/CassandraOperations.java | 3 +- .../defaultimpl/CassandraProcessManager.java | 2 +- .../defaultimpl/ICassandraOperations.java | 72 ++++++++++--------- .../priam/resources/CassandraAdmin.java | 4 +- .../priam/services/CassandraMonitor.java | 2 +- .../priam/utils/INodeToolObservable.java | 23 ------ .../priam/utils/INodeToolObserver.java | 24 ------- .../priam/utils/JMXConnectionException.java | 29 -------- .../netflix/priam/utils/JMXConnectorMgr.java | 46 ------------ .../com/netflix/priam/utils/SystemUtils.java | 17 ----- .../priam/services/TestCassandraMonitor.java | 3 +- 16 files changed, 141 insertions(+), 186 deletions(-) create mode 100644 priam/src/main/java/com/netflix/priam/connection/INodeToolObservable.java create mode 100644 priam/src/main/java/com/netflix/priam/connection/INodeToolObserver.java create mode 100644 priam/src/main/java/com/netflix/priam/connection/JMXConnectionException.java rename priam/src/main/java/com/netflix/priam/{utils => connection}/JMXNodeTool.java (97%) delete mode 100644 priam/src/main/java/com/netflix/priam/utils/INodeToolObservable.java delete mode 100644 priam/src/main/java/com/netflix/priam/utils/INodeToolObserver.java delete mode 100644 priam/src/main/java/com/netflix/priam/utils/JMXConnectionException.java delete mode 100644 priam/src/main/java/com/netflix/priam/utils/JMXConnectorMgr.java diff --git a/priam/src/main/java/com/netflix/priam/PriamServer.java b/priam/src/main/java/com/netflix/priam/PriamServer.java index 39112ff60..b21ed7353 100644 --- a/priam/src/main/java/com/netflix/priam/PriamServer.java +++ b/priam/src/main/java/com/netflix/priam/PriamServer.java @@ -35,9 +35,9 @@ import com.netflix.priam.scheduler.PriamScheduler; import com.netflix.priam.scheduler.TaskTimer; import com.netflix.priam.services.BackupTTLService; +import com.netflix.priam.services.CassandraMonitor; import com.netflix.priam.services.SnapshotMetaService; import com.netflix.priam.tuner.TuneCassandra; -import com.netflix.priam.services.CassandraMonitor; import com.netflix.priam.utils.Sleeper; import com.netflix.priam.utils.SystemUtils; import java.io.IOException; diff --git a/priam/src/main/java/com/netflix/priam/connection/INodeToolObservable.java b/priam/src/main/java/com/netflix/priam/connection/INodeToolObservable.java new file mode 100644 index 000000000..f299af625 --- /dev/null +++ b/priam/src/main/java/com/netflix/priam/connection/INodeToolObservable.java @@ -0,0 +1,26 @@ +/* + * Copyright 2019 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package com.netflix.priam.connection; + +public interface INodeToolObservable { + /* + * @param observer to add to list of internal observers. + */ + void addObserver(INodeToolObserver observer); + + void deleteObserver(INodeToolObserver observer); +} diff --git a/priam/src/main/java/com/netflix/priam/connection/INodeToolObserver.java b/priam/src/main/java/com/netflix/priam/connection/INodeToolObserver.java new file mode 100644 index 000000000..bd20c0044 --- /dev/null +++ b/priam/src/main/java/com/netflix/priam/connection/INodeToolObserver.java @@ -0,0 +1,27 @@ +/* + * Copyright 2019 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package com.netflix.priam.connection; + +import org.apache.cassandra.tools.NodeProbe; + +/* + * Represents an entity interested in a change of state to the NodeTool + */ +public interface INodeToolObserver { + + void nodeToolHasChanged(NodeProbe nodeTool); +} diff --git a/priam/src/main/java/com/netflix/priam/connection/JMXConnectionException.java b/priam/src/main/java/com/netflix/priam/connection/JMXConnectionException.java new file mode 100644 index 000000000..f815eb722 --- /dev/null +++ b/priam/src/main/java/com/netflix/priam/connection/JMXConnectionException.java @@ -0,0 +1,32 @@ +/* + * Copyright 2019 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package com.netflix.priam.connection; + +import java.io.IOException; + +public class JMXConnectionException extends IOException { + + private static final long serialVersionUID = 444L; + + public JMXConnectionException(String message) { + super(message); + } + + public JMXConnectionException(String message, Exception e) { + super(message, e); + } +} diff --git a/priam/src/main/java/com/netflix/priam/utils/JMXNodeTool.java b/priam/src/main/java/com/netflix/priam/connection/JMXNodeTool.java similarity index 97% rename from priam/src/main/java/com/netflix/priam/utils/JMXNodeTool.java rename to priam/src/main/java/com/netflix/priam/connection/JMXNodeTool.java index 487da689b..e650e863e 100644 --- a/priam/src/main/java/com/netflix/priam/utils/JMXNodeTool.java +++ b/priam/src/main/java/com/netflix/priam/connection/JMXNodeTool.java @@ -1,5 +1,5 @@ /* - * Copyright 2013 Netflix, Inc. + * Copyright 2019 Netflix, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,12 +14,13 @@ * limitations under the License. * */ -package com.netflix.priam.utils; +package com.netflix.priam.connection; import com.google.inject.Inject; import com.google.inject.Singleton; import com.netflix.priam.config.IConfiguration; import com.netflix.priam.services.CassandraMonitor; +import com.netflix.priam.utils.BoundedExponentialRetryCallable; import java.io.IOException; import java.io.PrintStream; import java.lang.management.ManagementFactory; @@ -121,7 +122,7 @@ private static boolean testConnection() { return false; } } catch (Throwable ex) { - SystemUtils.closeQuietly(tool); + closeQuietly(tool); logger.error( "Exception while checking JMX connection to C*, msg: {}", ex.getLocalizedMessage()); @@ -130,6 +131,14 @@ private static boolean testConnection() { return true; } + private static void closeQuietly(JMXNodeTool tool) { + try { + tool.close(); + } catch (Exception e) { + logger.warn("failed to close jmx node tool", e); + } + } + /** * A means to clean up existing and recreate the JMX connection to the Cassandra process. * diff --git a/priam/src/main/java/com/netflix/priam/defaultimpl/CassandraOperations.java b/priam/src/main/java/com/netflix/priam/defaultimpl/CassandraOperations.java index 5139d1e74..20a268284 100644 --- a/priam/src/main/java/com/netflix/priam/defaultimpl/CassandraOperations.java +++ b/priam/src/main/java/com/netflix/priam/defaultimpl/CassandraOperations.java @@ -14,8 +14,7 @@ package com.netflix.priam.defaultimpl; import com.netflix.priam.config.IConfiguration; -import com.netflix.priam.utils.JMXConnectionException; -import com.netflix.priam.utils.JMXNodeTool; +import com.netflix.priam.connection.JMXNodeTool; import com.netflix.priam.utils.RetryableCallable; import java.util.*; import javax.inject.Inject; diff --git a/priam/src/main/java/com/netflix/priam/defaultimpl/CassandraProcessManager.java b/priam/src/main/java/com/netflix/priam/defaultimpl/CassandraProcessManager.java index 0d08ae4f9..3ffe3b507 100644 --- a/priam/src/main/java/com/netflix/priam/defaultimpl/CassandraProcessManager.java +++ b/priam/src/main/java/com/netflix/priam/defaultimpl/CassandraProcessManager.java @@ -16,9 +16,9 @@ import com.google.common.collect.Lists; import com.google.inject.Inject; import com.netflix.priam.config.IConfiguration; +import com.netflix.priam.connection.JMXNodeTool; import com.netflix.priam.health.InstanceState; import com.netflix.priam.merics.CassMonitorMetrics; -import com.netflix.priam.utils.JMXNodeTool; import java.io.ByteArrayOutputStream; import java.io.File; import java.io.IOException; diff --git a/priam/src/main/java/com/netflix/priam/defaultimpl/ICassandraOperations.java b/priam/src/main/java/com/netflix/priam/defaultimpl/ICassandraOperations.java index aba793164..c3e32de31 100644 --- a/priam/src/main/java/com/netflix/priam/defaultimpl/ICassandraOperations.java +++ b/priam/src/main/java/com/netflix/priam/defaultimpl/ICassandraOperations.java @@ -20,41 +20,43 @@ import java.util.List; import java.util.Map; -/** - * Created by aagrawal on 2/16/19. - */ +/** Created by aagrawal on 2/16/19. */ public interface ICassandraOperations { - /** - * This method neds to be synchronized. Context: During the transition phase to backup version - * 2.0, we might be executing multiple snapshots at the same time. To avoid, unknown behavior by - * Cassandra, it is wise to keep this method sync. Also, with backups being on CRON, we don't - * know how often operator is taking snapshot. - * - * @param snapshotName Name of the snapshot on disk. This snapshotName should be UNIQUE among - * all the snapshots. Try to append UUID to snapshotName to ensure uniqueness. This is to - * ensure a) Snapshot fails if name are not unique. b) You might take snapshots which are - * not "part" of same snapshot. e.g. Any leftovers from previous operation. c) Once snapshot - * fails, this will clean the failed snapshot. - * @throws Exception in case of error while taking a snapshot by Cassandra. - */ - void takeSnapshot(final String snapshotName) throws Exception; - - /** - * Clear the snapshot tag from disk. - * - * @param snapshotTag Name of the snapshot to be removed. - * @throws Exception in case of error while clearing a snapshot. - */ - void clearSnapshot(final String snapshotTag) throws Exception; - - /** - * Get all the keyspaces existing on this node. - * @return List of keyspace names. - * @throws Exception in case of reaching to JMX endpoint. - */ - List getKeyspaces() throws Exception; - Map> getColumnfamilies() throws Exception; - void forceKeyspaceCompaction(String keyspaceName, String... columnfamilies) throws Exception; - void forceKeyspaceFlush(String keyspaceName) throws Exception; + /** + * This method neds to be synchronized. Context: During the transition phase to backup version + * 2.0, we might be executing multiple snapshots at the same time. To avoid, unknown behavior by + * Cassandra, it is wise to keep this method sync. Also, with backups being on CRON, we don't + * know how often operator is taking snapshot. + * + * @param snapshotName Name of the snapshot on disk. This snapshotName should be UNIQUE among + * all the snapshots. Try to append UUID to snapshotName to ensure uniqueness. This is to + * ensure a) Snapshot fails if name are not unique. b) You might take snapshots which are + * not "part" of same snapshot. e.g. Any leftovers from previous operation. c) Once snapshot + * fails, this will clean the failed snapshot. + * @throws Exception in case of error while taking a snapshot by Cassandra. + */ + void takeSnapshot(final String snapshotName) throws Exception; + + /** + * Clear the snapshot tag from disk. + * + * @param snapshotTag Name of the snapshot to be removed. + * @throws Exception in case of error while clearing a snapshot. + */ + void clearSnapshot(final String snapshotTag) throws Exception; + + /** + * Get all the keyspaces existing on this node. + * + * @return List of keyspace names. + * @throws Exception in case of reaching to JMX endpoint. + */ + List getKeyspaces() throws Exception; + + Map> getColumnfamilies() throws Exception; + + void forceKeyspaceCompaction(String keyspaceName, String... columnfamilies) throws Exception; + + void forceKeyspaceFlush(String keyspaceName) throws Exception; } diff --git a/priam/src/main/java/com/netflix/priam/resources/CassandraAdmin.java b/priam/src/main/java/com/netflix/priam/resources/CassandraAdmin.java index 2d364a24e..6de44eb91 100644 --- a/priam/src/main/java/com/netflix/priam/resources/CassandraAdmin.java +++ b/priam/src/main/java/com/netflix/priam/resources/CassandraAdmin.java @@ -22,9 +22,9 @@ import com.netflix.priam.cluster.management.Flush; import com.netflix.priam.compress.SnappyCompression; import com.netflix.priam.config.IConfiguration; +import com.netflix.priam.connection.JMXConnectionException; +import com.netflix.priam.connection.JMXNodeTool; import com.netflix.priam.defaultimpl.ICassandraProcess; -import com.netflix.priam.utils.JMXConnectionException; -import com.netflix.priam.utils.JMXNodeTool; import java.io.FileInputStream; import java.io.FileOutputStream; import java.io.IOException; diff --git a/priam/src/main/java/com/netflix/priam/services/CassandraMonitor.java b/priam/src/main/java/com/netflix/priam/services/CassandraMonitor.java index bdb0cbaea..5a9ed0fa0 100644 --- a/priam/src/main/java/com/netflix/priam/services/CassandraMonitor.java +++ b/priam/src/main/java/com/netflix/priam/services/CassandraMonitor.java @@ -19,13 +19,13 @@ import com.google.inject.Inject; import com.google.inject.Singleton; import com.netflix.priam.config.IConfiguration; +import com.netflix.priam.connection.JMXNodeTool; import com.netflix.priam.defaultimpl.ICassandraProcess; import com.netflix.priam.health.InstanceState; import com.netflix.priam.merics.CassMonitorMetrics; import com.netflix.priam.scheduler.SimpleTimer; import com.netflix.priam.scheduler.Task; import com.netflix.priam.scheduler.TaskTimer; -import com.netflix.priam.utils.JMXNodeTool; import java.io.BufferedReader; import java.io.File; import java.io.IOException; diff --git a/priam/src/main/java/com/netflix/priam/utils/INodeToolObservable.java b/priam/src/main/java/com/netflix/priam/utils/INodeToolObservable.java deleted file mode 100644 index 5f6505347..000000000 --- a/priam/src/main/java/com/netflix/priam/utils/INodeToolObservable.java +++ /dev/null @@ -1,23 +0,0 @@ -/** - * Copyright 2017 Netflix, Inc. - * - *

    Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file - * except in compliance with the License. You may obtain a copy of the License at - * - *

    http://www.apache.org/licenses/LICENSE-2.0 - * - *

    Unless required by applicable law or agreed to in writing, software distributed under the - * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.netflix.priam.utils; - -public interface INodeToolObservable { - /* - * @param observer to add to list of internal observers. - */ - void addObserver(INodeToolObserver observer); - - void deleteObserver(INodeToolObserver observer); -} diff --git a/priam/src/main/java/com/netflix/priam/utils/INodeToolObserver.java b/priam/src/main/java/com/netflix/priam/utils/INodeToolObserver.java deleted file mode 100644 index 7d094e5aa..000000000 --- a/priam/src/main/java/com/netflix/priam/utils/INodeToolObserver.java +++ /dev/null @@ -1,24 +0,0 @@ -/** - * Copyright 2017 Netflix, Inc. - * - *

    Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file - * except in compliance with the License. You may obtain a copy of the License at - * - *

    http://www.apache.org/licenses/LICENSE-2.0 - * - *

    Unless required by applicable law or agreed to in writing, software distributed under the - * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.netflix.priam.utils; - -import org.apache.cassandra.tools.NodeProbe; - -/* - * Represents an entity interested in a change of state to the NodeTool - */ -public interface INodeToolObserver { - - void nodeToolHasChanged(NodeProbe nodeTool); -} diff --git a/priam/src/main/java/com/netflix/priam/utils/JMXConnectionException.java b/priam/src/main/java/com/netflix/priam/utils/JMXConnectionException.java deleted file mode 100644 index 20269da1a..000000000 --- a/priam/src/main/java/com/netflix/priam/utils/JMXConnectionException.java +++ /dev/null @@ -1,29 +0,0 @@ -/** - * Copyright 2017 Netflix, Inc. - * - *

    Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file - * except in compliance with the License. You may obtain a copy of the License at - * - *

    http://www.apache.org/licenses/LICENSE-2.0 - * - *

    Unless required by applicable law or agreed to in writing, software distributed under the - * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.netflix.priam.utils; - -import java.io.IOException; - -public class JMXConnectionException extends IOException { - - private static final long serialVersionUID = 444L; - - public JMXConnectionException(String message) { - super(message); - } - - public JMXConnectionException(String message, Exception e) { - super(message, e); - } -} diff --git a/priam/src/main/java/com/netflix/priam/utils/JMXConnectorMgr.java b/priam/src/main/java/com/netflix/priam/utils/JMXConnectorMgr.java deleted file mode 100644 index 309c0da10..000000000 --- a/priam/src/main/java/com/netflix/priam/utils/JMXConnectorMgr.java +++ /dev/null @@ -1,46 +0,0 @@ -/** - * Copyright 2017 Netflix, Inc. - * - *

    Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file - * except in compliance with the License. You may obtain a copy of the License at - * - *

    http://www.apache.org/licenses/LICENSE-2.0 - * - *

    Unless required by applicable law or agreed to in writing, software distributed under the - * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.netflix.priam.utils; - -import com.netflix.priam.config.IConfiguration; -import java.io.IOException; -import org.apache.cassandra.tools.NodeProbe; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * Represents a connection to remote JMX mbean server. This object differs from JMXNodeTool as it is - * meant for short lived connection to remote mbean server. - * - *

    Created by vinhn on 10/11/16. - */ -public class JMXConnectorMgr extends NodeProbe { - private static final Logger logger = LoggerFactory.getLogger(JMXConnectorMgr.class); - - /* - * create a connection to remote mbean server and get proxy to various mbeans - * @throws exception if unable to create the connection, e.g. Cassandra process not running. - */ - public JMXConnectorMgr(IConfiguration config) throws IOException, InterruptedException { - super("localhost", config.getJmxPort()); - } - - @Override - /* - close the connection to remote mbean server - */ - public void close() throws IOException { - super.close(); // close the connection to remote mbean server - } -} diff --git a/priam/src/main/java/com/netflix/priam/utils/SystemUtils.java b/priam/src/main/java/com/netflix/priam/utils/SystemUtils.java index 9696ed399..d64730203 100644 --- a/priam/src/main/java/com/netflix/priam/utils/SystemUtils.java +++ b/priam/src/main/java/com/netflix/priam/utils/SystemUtils.java @@ -25,7 +25,6 @@ import java.net.URL; import java.security.MessageDigest; import java.util.List; -import javax.management.remote.JMXConnector; import org.apache.commons.codec.binary.Base64; import org.apache.commons.io.FileUtils; import org.slf4j.Logger; @@ -118,20 +117,4 @@ public static String toBase64(byte[] md5) { byte encoded[] = Base64.encodeBase64(md5, false); return new String(encoded); } - - public static void closeQuietly(JMXNodeTool tool) { - try { - tool.close(); - } catch (Exception e) { - logger.warn("failed to close jmx node tool", e); - } - } - - public static void closeQuietly(JMXConnector jmc) { - try { - jmc.close(); - } catch (Exception e) { - logger.warn("failed to close JMXConnectorMgr", e); - } - } } diff --git a/priam/src/test/java/com/netflix/priam/services/TestCassandraMonitor.java b/priam/src/test/java/com/netflix/priam/services/TestCassandraMonitor.java index e31618c73..3648a461c 100644 --- a/priam/src/test/java/com/netflix/priam/services/TestCassandraMonitor.java +++ b/priam/src/test/java/com/netflix/priam/services/TestCassandraMonitor.java @@ -21,11 +21,10 @@ import com.google.inject.Injector; import com.netflix.priam.backup.BRTestModule; import com.netflix.priam.config.IConfiguration; +import com.netflix.priam.connection.JMXNodeTool; import com.netflix.priam.defaultimpl.ICassandraProcess; import com.netflix.priam.health.InstanceState; import com.netflix.priam.merics.CassMonitorMetrics; -import com.netflix.priam.services.CassandraMonitor; -import com.netflix.priam.utils.JMXNodeTool; import java.io.ByteArrayInputStream; import java.io.InputStream; import mockit.*; From df8fdeefc6d9fc9c35ffab4f66c0e42bdc72f997 Mon Sep 17 00:00:00 2001 From: Arun Agrawal Date: Sat, 16 Feb 2019 20:37:38 -0800 Subject: [PATCH 062/228] Moving files to packages. No new functionality (#787) * move cassandra monitor to services * Move JMX to connection package. --- .../java/com/netflix/priam/PriamServer.java | 2 +- .../netflix/priam/backup/SnapshotBackup.java | 2 +- .../management/IClusterManagement.java | 2 +- .../priam/connection/INodeToolObservable.java | 26 ++++++++ .../priam/connection/INodeToolObserver.java | 27 ++++++++ .../connection/JMXConnectionException.java | 32 ++++++++++ .../{utils => connection}/JMXNodeTool.java | 17 ++++- .../defaultimpl/CassandraOperations.java | 32 +++------- .../defaultimpl/CassandraProcessManager.java | 2 +- .../defaultimpl/ICassandraOperations.java | 62 +++++++++++++++++++ .../priam/resources/CassandraAdmin.java | 4 +- .../{utils => services}/CassandraMonitor.java | 5 +- .../priam/services/SnapshotMetaService.java | 1 - .../priam/utils/INodeToolObservable.java | 23 ------- .../priam/utils/INodeToolObserver.java | 24 ------- .../priam/utils/JMXConnectionException.java | 29 --------- .../netflix/priam/utils/JMXConnectorMgr.java | 46 -------------- .../com/netflix/priam/utils/SystemUtils.java | 17 ----- .../cluser/management/TestCompaction.groovy | 2 +- .../TestCassandraMonitor.java | 5 +- 20 files changed, 183 insertions(+), 177 deletions(-) create mode 100644 priam/src/main/java/com/netflix/priam/connection/INodeToolObservable.java create mode 100644 priam/src/main/java/com/netflix/priam/connection/INodeToolObserver.java create mode 100644 priam/src/main/java/com/netflix/priam/connection/JMXConnectionException.java rename priam/src/main/java/com/netflix/priam/{utils => connection}/JMXNodeTool.java (97%) create mode 100644 priam/src/main/java/com/netflix/priam/defaultimpl/ICassandraOperations.java rename priam/src/main/java/com/netflix/priam/{utils => services}/CassandraMonitor.java (98%) delete mode 100644 priam/src/main/java/com/netflix/priam/utils/INodeToolObservable.java delete mode 100644 priam/src/main/java/com/netflix/priam/utils/INodeToolObserver.java delete mode 100644 priam/src/main/java/com/netflix/priam/utils/JMXConnectionException.java delete mode 100644 priam/src/main/java/com/netflix/priam/utils/JMXConnectorMgr.java rename priam/src/test/java/com/netflix/priam/{utils => services}/TestCassandraMonitor.java (97%) diff --git a/priam/src/main/java/com/netflix/priam/PriamServer.java b/priam/src/main/java/com/netflix/priam/PriamServer.java index 2aaf2b49a..b21ed7353 100644 --- a/priam/src/main/java/com/netflix/priam/PriamServer.java +++ b/priam/src/main/java/com/netflix/priam/PriamServer.java @@ -35,9 +35,9 @@ import com.netflix.priam.scheduler.PriamScheduler; import com.netflix.priam.scheduler.TaskTimer; import com.netflix.priam.services.BackupTTLService; +import com.netflix.priam.services.CassandraMonitor; import com.netflix.priam.services.SnapshotMetaService; import com.netflix.priam.tuner.TuneCassandra; -import com.netflix.priam.utils.CassandraMonitor; import com.netflix.priam.utils.Sleeper; import com.netflix.priam.utils.SystemUtils; import java.io.IOException; diff --git a/priam/src/main/java/com/netflix/priam/backup/SnapshotBackup.java b/priam/src/main/java/com/netflix/priam/backup/SnapshotBackup.java index 476d4eb5d..b8d24a384 100644 --- a/priam/src/main/java/com/netflix/priam/backup/SnapshotBackup.java +++ b/priam/src/main/java/com/netflix/priam/backup/SnapshotBackup.java @@ -27,7 +27,7 @@ import com.netflix.priam.identity.InstanceIdentity; import com.netflix.priam.scheduler.CronTimer; import com.netflix.priam.scheduler.TaskTimer; -import com.netflix.priam.utils.CassandraMonitor; +import com.netflix.priam.services.CassandraMonitor; import com.netflix.priam.utils.DateUtil; import com.netflix.priam.utils.ThreadSleeper; import java.io.File; diff --git a/priam/src/main/java/com/netflix/priam/cluster/management/IClusterManagement.java b/priam/src/main/java/com/netflix/priam/cluster/management/IClusterManagement.java index a6df274c2..64c13701a 100644 --- a/priam/src/main/java/com/netflix/priam/cluster/management/IClusterManagement.java +++ b/priam/src/main/java/com/netflix/priam/cluster/management/IClusterManagement.java @@ -16,7 +16,7 @@ import com.netflix.priam.config.IConfiguration; import com.netflix.priam.merics.IMeasurement; import com.netflix.priam.scheduler.Task; -import com.netflix.priam.utils.CassandraMonitor; +import com.netflix.priam.services.CassandraMonitor; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; import org.slf4j.Logger; diff --git a/priam/src/main/java/com/netflix/priam/connection/INodeToolObservable.java b/priam/src/main/java/com/netflix/priam/connection/INodeToolObservable.java new file mode 100644 index 000000000..f299af625 --- /dev/null +++ b/priam/src/main/java/com/netflix/priam/connection/INodeToolObservable.java @@ -0,0 +1,26 @@ +/* + * Copyright 2019 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package com.netflix.priam.connection; + +public interface INodeToolObservable { + /* + * @param observer to add to list of internal observers. + */ + void addObserver(INodeToolObserver observer); + + void deleteObserver(INodeToolObserver observer); +} diff --git a/priam/src/main/java/com/netflix/priam/connection/INodeToolObserver.java b/priam/src/main/java/com/netflix/priam/connection/INodeToolObserver.java new file mode 100644 index 000000000..bd20c0044 --- /dev/null +++ b/priam/src/main/java/com/netflix/priam/connection/INodeToolObserver.java @@ -0,0 +1,27 @@ +/* + * Copyright 2019 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package com.netflix.priam.connection; + +import org.apache.cassandra.tools.NodeProbe; + +/* + * Represents an entity interested in a change of state to the NodeTool + */ +public interface INodeToolObserver { + + void nodeToolHasChanged(NodeProbe nodeTool); +} diff --git a/priam/src/main/java/com/netflix/priam/connection/JMXConnectionException.java b/priam/src/main/java/com/netflix/priam/connection/JMXConnectionException.java new file mode 100644 index 000000000..f815eb722 --- /dev/null +++ b/priam/src/main/java/com/netflix/priam/connection/JMXConnectionException.java @@ -0,0 +1,32 @@ +/* + * Copyright 2019 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package com.netflix.priam.connection; + +import java.io.IOException; + +public class JMXConnectionException extends IOException { + + private static final long serialVersionUID = 444L; + + public JMXConnectionException(String message) { + super(message); + } + + public JMXConnectionException(String message, Exception e) { + super(message, e); + } +} diff --git a/priam/src/main/java/com/netflix/priam/utils/JMXNodeTool.java b/priam/src/main/java/com/netflix/priam/connection/JMXNodeTool.java similarity index 97% rename from priam/src/main/java/com/netflix/priam/utils/JMXNodeTool.java rename to priam/src/main/java/com/netflix/priam/connection/JMXNodeTool.java index b87b97849..e650e863e 100644 --- a/priam/src/main/java/com/netflix/priam/utils/JMXNodeTool.java +++ b/priam/src/main/java/com/netflix/priam/connection/JMXNodeTool.java @@ -1,5 +1,5 @@ /* - * Copyright 2013 Netflix, Inc. + * Copyright 2019 Netflix, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,11 +14,13 @@ * limitations under the License. * */ -package com.netflix.priam.utils; +package com.netflix.priam.connection; import com.google.inject.Inject; import com.google.inject.Singleton; import com.netflix.priam.config.IConfiguration; +import com.netflix.priam.services.CassandraMonitor; +import com.netflix.priam.utils.BoundedExponentialRetryCallable; import java.io.IOException; import java.io.PrintStream; import java.lang.management.ManagementFactory; @@ -120,7 +122,7 @@ private static boolean testConnection() { return false; } } catch (Throwable ex) { - SystemUtils.closeQuietly(tool); + closeQuietly(tool); logger.error( "Exception while checking JMX connection to C*, msg: {}", ex.getLocalizedMessage()); @@ -129,6 +131,14 @@ private static boolean testConnection() { return true; } + private static void closeQuietly(JMXNodeTool tool) { + try { + tool.close(); + } catch (Exception e) { + logger.warn("failed to close jmx node tool", e); + } + } + /** * A means to clean up existing and recreate the JMX connection to the Cassandra process. * @@ -256,6 +266,7 @@ public JSONObject info() throws JSONException { JSONObject object = new JSONObject(); object.put("gossip_active", isInitialized()); object.put("thrift_active", isThriftServerRunning()); + object.put("native_active", isNativeTransportRunning()); object.put("token", getTokens().toString()); object.put("load", getLoadString()); object.put("generation_no", getCurrentGenerationNumber()); diff --git a/priam/src/main/java/com/netflix/priam/defaultimpl/CassandraOperations.java b/priam/src/main/java/com/netflix/priam/defaultimpl/CassandraOperations.java index f13ff383b..20a268284 100644 --- a/priam/src/main/java/com/netflix/priam/defaultimpl/CassandraOperations.java +++ b/priam/src/main/java/com/netflix/priam/defaultimpl/CassandraOperations.java @@ -14,7 +14,7 @@ package com.netflix.priam.defaultimpl; import com.netflix.priam.config.IConfiguration; -import com.netflix.priam.utils.JMXNodeTool; +import com.netflix.priam.connection.JMXNodeTool; import com.netflix.priam.utils.RetryableCallable; import java.util.*; import javax.inject.Inject; @@ -23,7 +23,7 @@ import org.slf4j.LoggerFactory; /** This class encapsulates interactions with Cassandra. Created by aagrawal on 6/19/18. */ -public class CassandraOperations { +public class CassandraOperations implements ICassandraOperations { private static final Logger logger = LoggerFactory.getLogger(CassandraOperations.class); private final IConfiguration configuration; @@ -32,27 +32,14 @@ public class CassandraOperations { this.configuration = configuration; } - /** - * This method neds to be synchronized. Context: During the transition phase to backup version - * 2.0, we might be executing multiple snapshots at the same time. To avoid, unknown behavior by - * Cassanddra, it is wise to keep this method sync. Also, with backups being on CRON, we don't - * know how often operator is taking snapshot. - * - * @param snapshotName Name of the snapshot on disk. This snapshotName should be UNIQUE among - * all the snapshots. Try to append UUID to snapshotName to ensure uniqueness. This is to - * ensure a) Snapshot fails if name are not unique. b) You might take snapshots which are - * not "part" of same snapshot. e.g. Any leftovers from previous operation. c) Once snapshot - * fails, this will clean the failed snapshot. - * @throws Exception in case of error while taking a snapshot by Cassandra. - */ + @Override public synchronized void takeSnapshot(final String snapshotName) throws Exception { // Retry max of 6 times with 10 second in between (for one minute). This is to ensure that // we overcome any temporary glitch. // Note that operation MAY fail if cassandra successfully took the snapshot of certain // columnfamily(ies) and we try to create snapshot with // same name. It is a good practice to call clearSnapshot after this operation fails, to - // ensure we don't leave - // any left overs. + // ensure we don't leave any left overs. // Example scenario: Change of file permissions by manual intervention and C* unable to take // snapshot of one CF. try { @@ -72,12 +59,7 @@ public Void retriableCall() throws Exception { } } - /** - * Clear the snapshot tag from disk. - * - * @param snapshotTag Name of the snapshot to be removed. - * @throws Exception in case of error while clearing a snapshot. - */ + @Override public void clearSnapshot(final String snapshotTag) throws Exception { new RetryableCallable() { public Void retriableCall() throws Exception { @@ -88,6 +70,7 @@ public Void retriableCall() throws Exception { }.call(); } + @Override public List getKeyspaces() throws Exception { return new RetryableCallable>() { public List retriableCall() throws Exception { @@ -98,6 +81,7 @@ public List retriableCall() throws Exception { }.call(); } + @Override public Map> getColumnfamilies() throws Exception { return new RetryableCallable>>() { public Map> retriableCall() throws Exception { @@ -118,6 +102,7 @@ public Map> retriableCall() throws Exception { }.call(); } + @Override public void forceKeyspaceCompaction(String keyspaceName, String... columnfamilies) throws Exception { new RetryableCallable() { @@ -130,6 +115,7 @@ public Void retriableCall() throws Exception { }.call(); } + @Override public void forceKeyspaceFlush(String keyspaceName) throws Exception { new RetryableCallable() { public Void retriableCall() throws Exception { diff --git a/priam/src/main/java/com/netflix/priam/defaultimpl/CassandraProcessManager.java b/priam/src/main/java/com/netflix/priam/defaultimpl/CassandraProcessManager.java index 0d08ae4f9..3ffe3b507 100644 --- a/priam/src/main/java/com/netflix/priam/defaultimpl/CassandraProcessManager.java +++ b/priam/src/main/java/com/netflix/priam/defaultimpl/CassandraProcessManager.java @@ -16,9 +16,9 @@ import com.google.common.collect.Lists; import com.google.inject.Inject; import com.netflix.priam.config.IConfiguration; +import com.netflix.priam.connection.JMXNodeTool; import com.netflix.priam.health.InstanceState; import com.netflix.priam.merics.CassMonitorMetrics; -import com.netflix.priam.utils.JMXNodeTool; import java.io.ByteArrayOutputStream; import java.io.File; import java.io.IOException; diff --git a/priam/src/main/java/com/netflix/priam/defaultimpl/ICassandraOperations.java b/priam/src/main/java/com/netflix/priam/defaultimpl/ICassandraOperations.java new file mode 100644 index 000000000..c3e32de31 --- /dev/null +++ b/priam/src/main/java/com/netflix/priam/defaultimpl/ICassandraOperations.java @@ -0,0 +1,62 @@ +/* + * Copyright 2019 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.netflix.priam.defaultimpl; + +import java.util.List; +import java.util.Map; + +/** Created by aagrawal on 2/16/19. */ +public interface ICassandraOperations { + + /** + * This method neds to be synchronized. Context: During the transition phase to backup version + * 2.0, we might be executing multiple snapshots at the same time. To avoid, unknown behavior by + * Cassandra, it is wise to keep this method sync. Also, with backups being on CRON, we don't + * know how often operator is taking snapshot. + * + * @param snapshotName Name of the snapshot on disk. This snapshotName should be UNIQUE among + * all the snapshots. Try to append UUID to snapshotName to ensure uniqueness. This is to + * ensure a) Snapshot fails if name are not unique. b) You might take snapshots which are + * not "part" of same snapshot. e.g. Any leftovers from previous operation. c) Once snapshot + * fails, this will clean the failed snapshot. + * @throws Exception in case of error while taking a snapshot by Cassandra. + */ + void takeSnapshot(final String snapshotName) throws Exception; + + /** + * Clear the snapshot tag from disk. + * + * @param snapshotTag Name of the snapshot to be removed. + * @throws Exception in case of error while clearing a snapshot. + */ + void clearSnapshot(final String snapshotTag) throws Exception; + + /** + * Get all the keyspaces existing on this node. + * + * @return List of keyspace names. + * @throws Exception in case of reaching to JMX endpoint. + */ + List getKeyspaces() throws Exception; + + Map> getColumnfamilies() throws Exception; + + void forceKeyspaceCompaction(String keyspaceName, String... columnfamilies) throws Exception; + + void forceKeyspaceFlush(String keyspaceName) throws Exception; +} diff --git a/priam/src/main/java/com/netflix/priam/resources/CassandraAdmin.java b/priam/src/main/java/com/netflix/priam/resources/CassandraAdmin.java index 2d364a24e..6de44eb91 100644 --- a/priam/src/main/java/com/netflix/priam/resources/CassandraAdmin.java +++ b/priam/src/main/java/com/netflix/priam/resources/CassandraAdmin.java @@ -22,9 +22,9 @@ import com.netflix.priam.cluster.management.Flush; import com.netflix.priam.compress.SnappyCompression; import com.netflix.priam.config.IConfiguration; +import com.netflix.priam.connection.JMXConnectionException; +import com.netflix.priam.connection.JMXNodeTool; import com.netflix.priam.defaultimpl.ICassandraProcess; -import com.netflix.priam.utils.JMXConnectionException; -import com.netflix.priam.utils.JMXNodeTool; import java.io.FileInputStream; import java.io.FileOutputStream; import java.io.IOException; diff --git a/priam/src/main/java/com/netflix/priam/utils/CassandraMonitor.java b/priam/src/main/java/com/netflix/priam/services/CassandraMonitor.java similarity index 98% rename from priam/src/main/java/com/netflix/priam/utils/CassandraMonitor.java rename to priam/src/main/java/com/netflix/priam/services/CassandraMonitor.java index b4772489d..5a9ed0fa0 100644 --- a/priam/src/main/java/com/netflix/priam/utils/CassandraMonitor.java +++ b/priam/src/main/java/com/netflix/priam/services/CassandraMonitor.java @@ -1,5 +1,5 @@ /* - * Copyright 2013 Netflix, Inc. + * Copyright 2019 Netflix, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,11 +14,12 @@ * limitations under the License. * */ -package com.netflix.priam.utils; +package com.netflix.priam.services; import com.google.inject.Inject; import com.google.inject.Singleton; import com.netflix.priam.config.IConfiguration; +import com.netflix.priam.connection.JMXNodeTool; import com.netflix.priam.defaultimpl.ICassandraProcess; import com.netflix.priam.health.InstanceState; import com.netflix.priam.merics.CassMonitorMetrics; diff --git a/priam/src/main/java/com/netflix/priam/services/SnapshotMetaService.java b/priam/src/main/java/com/netflix/priam/services/SnapshotMetaService.java index 0566727b8..8c01cc889 100644 --- a/priam/src/main/java/com/netflix/priam/services/SnapshotMetaService.java +++ b/priam/src/main/java/com/netflix/priam/services/SnapshotMetaService.java @@ -23,7 +23,6 @@ import com.netflix.priam.identity.InstanceIdentity; import com.netflix.priam.scheduler.CronTimer; import com.netflix.priam.scheduler.TaskTimer; -import com.netflix.priam.utils.CassandraMonitor; import com.netflix.priam.utils.DateUtil; import java.io.File; import java.nio.file.Paths; diff --git a/priam/src/main/java/com/netflix/priam/utils/INodeToolObservable.java b/priam/src/main/java/com/netflix/priam/utils/INodeToolObservable.java deleted file mode 100644 index 5f6505347..000000000 --- a/priam/src/main/java/com/netflix/priam/utils/INodeToolObservable.java +++ /dev/null @@ -1,23 +0,0 @@ -/** - * Copyright 2017 Netflix, Inc. - * - *

    Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file - * except in compliance with the License. You may obtain a copy of the License at - * - *

    http://www.apache.org/licenses/LICENSE-2.0 - * - *

    Unless required by applicable law or agreed to in writing, software distributed under the - * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.netflix.priam.utils; - -public interface INodeToolObservable { - /* - * @param observer to add to list of internal observers. - */ - void addObserver(INodeToolObserver observer); - - void deleteObserver(INodeToolObserver observer); -} diff --git a/priam/src/main/java/com/netflix/priam/utils/INodeToolObserver.java b/priam/src/main/java/com/netflix/priam/utils/INodeToolObserver.java deleted file mode 100644 index 7d094e5aa..000000000 --- a/priam/src/main/java/com/netflix/priam/utils/INodeToolObserver.java +++ /dev/null @@ -1,24 +0,0 @@ -/** - * Copyright 2017 Netflix, Inc. - * - *

    Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file - * except in compliance with the License. You may obtain a copy of the License at - * - *

    http://www.apache.org/licenses/LICENSE-2.0 - * - *

    Unless required by applicable law or agreed to in writing, software distributed under the - * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.netflix.priam.utils; - -import org.apache.cassandra.tools.NodeProbe; - -/* - * Represents an entity interested in a change of state to the NodeTool - */ -public interface INodeToolObserver { - - void nodeToolHasChanged(NodeProbe nodeTool); -} diff --git a/priam/src/main/java/com/netflix/priam/utils/JMXConnectionException.java b/priam/src/main/java/com/netflix/priam/utils/JMXConnectionException.java deleted file mode 100644 index 20269da1a..000000000 --- a/priam/src/main/java/com/netflix/priam/utils/JMXConnectionException.java +++ /dev/null @@ -1,29 +0,0 @@ -/** - * Copyright 2017 Netflix, Inc. - * - *

    Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file - * except in compliance with the License. You may obtain a copy of the License at - * - *

    http://www.apache.org/licenses/LICENSE-2.0 - * - *

    Unless required by applicable law or agreed to in writing, software distributed under the - * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.netflix.priam.utils; - -import java.io.IOException; - -public class JMXConnectionException extends IOException { - - private static final long serialVersionUID = 444L; - - public JMXConnectionException(String message) { - super(message); - } - - public JMXConnectionException(String message, Exception e) { - super(message, e); - } -} diff --git a/priam/src/main/java/com/netflix/priam/utils/JMXConnectorMgr.java b/priam/src/main/java/com/netflix/priam/utils/JMXConnectorMgr.java deleted file mode 100644 index 309c0da10..000000000 --- a/priam/src/main/java/com/netflix/priam/utils/JMXConnectorMgr.java +++ /dev/null @@ -1,46 +0,0 @@ -/** - * Copyright 2017 Netflix, Inc. - * - *

    Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file - * except in compliance with the License. You may obtain a copy of the License at - * - *

    http://www.apache.org/licenses/LICENSE-2.0 - * - *

    Unless required by applicable law or agreed to in writing, software distributed under the - * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.netflix.priam.utils; - -import com.netflix.priam.config.IConfiguration; -import java.io.IOException; -import org.apache.cassandra.tools.NodeProbe; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * Represents a connection to remote JMX mbean server. This object differs from JMXNodeTool as it is - * meant for short lived connection to remote mbean server. - * - *

    Created by vinhn on 10/11/16. - */ -public class JMXConnectorMgr extends NodeProbe { - private static final Logger logger = LoggerFactory.getLogger(JMXConnectorMgr.class); - - /* - * create a connection to remote mbean server and get proxy to various mbeans - * @throws exception if unable to create the connection, e.g. Cassandra process not running. - */ - public JMXConnectorMgr(IConfiguration config) throws IOException, InterruptedException { - super("localhost", config.getJmxPort()); - } - - @Override - /* - close the connection to remote mbean server - */ - public void close() throws IOException { - super.close(); // close the connection to remote mbean server - } -} diff --git a/priam/src/main/java/com/netflix/priam/utils/SystemUtils.java b/priam/src/main/java/com/netflix/priam/utils/SystemUtils.java index 9696ed399..d64730203 100644 --- a/priam/src/main/java/com/netflix/priam/utils/SystemUtils.java +++ b/priam/src/main/java/com/netflix/priam/utils/SystemUtils.java @@ -25,7 +25,6 @@ import java.net.URL; import java.security.MessageDigest; import java.util.List; -import javax.management.remote.JMXConnector; import org.apache.commons.codec.binary.Base64; import org.apache.commons.io.FileUtils; import org.slf4j.Logger; @@ -118,20 +117,4 @@ public static String toBase64(byte[] md5) { byte encoded[] = Base64.encodeBase64(md5, false); return new String(encoded); } - - public static void closeQuietly(JMXNodeTool tool) { - try { - tool.close(); - } catch (Exception e) { - logger.warn("failed to close jmx node tool", e); - } - } - - public static void closeQuietly(JMXConnector jmc) { - try { - jmc.close(); - } catch (Exception e) { - logger.warn("failed to close JMXConnectorMgr", e); - } - } } diff --git a/priam/src/test/groovy/com/netflix/priam/cluser/management/TestCompaction.groovy b/priam/src/test/groovy/com/netflix/priam/cluser/management/TestCompaction.groovy index 4c6f3a0a6..5838d298b 100644 --- a/priam/src/test/groovy/com/netflix/priam/cluser/management/TestCompaction.groovy +++ b/priam/src/test/groovy/com/netflix/priam/cluser/management/TestCompaction.groovy @@ -21,7 +21,7 @@ import com.netflix.priam.config.FakeConfiguration import com.netflix.priam.backup.BRTestModule import com.netflix.priam.cluster.management.Compaction import com.netflix.priam.defaultimpl.CassandraOperations -import com.netflix.priam.utils.CassandraMonitor +import com.netflix.priam.services.CassandraMonitor import mockit.Mock import mockit.MockUp import spock.lang.Shared diff --git a/priam/src/test/java/com/netflix/priam/utils/TestCassandraMonitor.java b/priam/src/test/java/com/netflix/priam/services/TestCassandraMonitor.java similarity index 97% rename from priam/src/test/java/com/netflix/priam/utils/TestCassandraMonitor.java rename to priam/src/test/java/com/netflix/priam/services/TestCassandraMonitor.java index d38b4ca80..3648a461c 100644 --- a/priam/src/test/java/com/netflix/priam/utils/TestCassandraMonitor.java +++ b/priam/src/test/java/com/netflix/priam/services/TestCassandraMonitor.java @@ -1,5 +1,5 @@ /* - * Copyright 2017 Netflix, Inc. + * Copyright 2019 Netflix, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,12 +15,13 @@ * */ -package com.netflix.priam.utils; +package com.netflix.priam.services; import com.google.inject.Guice; import com.google.inject.Injector; import com.netflix.priam.backup.BRTestModule; import com.netflix.priam.config.IConfiguration; +import com.netflix.priam.connection.JMXNodeTool; import com.netflix.priam.defaultimpl.ICassandraProcess; import com.netflix.priam.health.InstanceState; import com.netflix.priam.merics.CassMonitorMetrics; From 0452b26cb6af5333f1f69707529b0f7ad141b072 Mon Sep 17 00:00:00 2001 From: Arun Agrawal Date: Wed, 27 Feb 2019 09:10:25 -0800 Subject: [PATCH 063/228] Output from the servlet should be JSON --- .../main/java/com/netflix/priam/resources/BackupServlet.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/priam/src/main/java/com/netflix/priam/resources/BackupServlet.java b/priam/src/main/java/com/netflix/priam/resources/BackupServlet.java index c33fa51f8..fc1ca1798 100644 --- a/priam/src/main/java/com/netflix/priam/resources/BackupServlet.java +++ b/priam/src/main/java/com/netflix/priam/resources/BackupServlet.java @@ -153,7 +153,7 @@ public Response statusByDate(@PathParam("date") String date) throws Exception { } else { object.put("Snapshotstatus", true); - object.put("Details", backupMetadataOptional.get()); + object.put("Details", new JSONObject(backupMetadataOptional.get().toString())); } return Response.ok(object.toString(), MediaType.APPLICATION_JSON).build(); } From 7291613a7cca22349d385ce65f03e6621f48cb06 Mon Sep 17 00:00:00 2001 From: Arun Agrawal Date: Wed, 27 Feb 2019 14:34:50 -0800 Subject: [PATCH 064/228] S3 - BucketLifecycleConfiguration has `prefix` method removed from latest library. --- build.gradle | 2 +- .../java/com/netflix/priam/PriamServer.java | 15 +++ .../netflix/priam/aws/S3FileSystemBase.java | 103 +++++++++++++----- .../priam/backup/TestS3FileSystem.java | 96 ++++++++++++---- 4 files changed, 163 insertions(+), 53 deletions(-) diff --git a/build.gradle b/build.gradle index 8adcd63cb..c981b8811 100644 --- a/build.gradle +++ b/build.gradle @@ -41,7 +41,7 @@ allprojects { compile 'com.sun.jersey.contribs:jersey-guice:1.19.4' compile 'com.google.guava:guava:21.0' compile 'com.google.code.findbugs:jsr305:3.0.2' - compile 'com.amazonaws:aws-java-sdk:1.11.488' + compile 'com.amazonaws:aws-java-sdk:1.11.501' compile 'com.google.inject:guice:4.2.2' compile 'com.google.inject.extensions:guice-servlet:4.2.2' compile 'org.quartz-scheduler:quartz:2.3.0' diff --git a/priam/src/main/java/com/netflix/priam/PriamServer.java b/priam/src/main/java/com/netflix/priam/PriamServer.java index b21ed7353..cef77507a 100644 --- a/priam/src/main/java/com/netflix/priam/PriamServer.java +++ b/priam/src/main/java/com/netflix/priam/PriamServer.java @@ -35,6 +35,7 @@ import com.netflix.priam.scheduler.PriamScheduler; import com.netflix.priam.scheduler.TaskTimer; import com.netflix.priam.services.BackupTTLService; +import com.netflix.priam.services.BackupVerificationService; import com.netflix.priam.services.CassandraMonitor; import com.netflix.priam.services.SnapshotMetaService; import com.netflix.priam.tuner.TuneCassandra; @@ -232,6 +233,20 @@ private void setUpSnapshotService() throws Exception { backupTTLTimer.getCronExpression()); } + // Schedule the backup verification service + TaskTimer backupVerificationTimer = + BackupVerificationService.getTimer(backupRestoreConfig); + if (backupVerificationTimer != null) { + scheduler.addTask( + BackupVerificationService.JOBNAME, + BackupVerificationService.class, + backupVerificationTimer); + logger.info( + "Added {} Task with schedule: [{}]", + BackupVerificationService.JOBNAME, + backupVerificationTimer.getCronExpression()); + } + // Start the Incremental backup schedule if enabled if (config.isIncrementalBackupEnabled() && backupRestoreConfig.enableV2Backups()) { // Delete the old task, if scheduled. This is required, as we stop taking backup diff --git a/priam/src/main/java/com/netflix/priam/aws/S3FileSystemBase.java b/priam/src/main/java/com/netflix/priam/aws/S3FileSystemBase.java index 902f71117..b282a02cf 100755 --- a/priam/src/main/java/com/netflix/priam/aws/S3FileSystemBase.java +++ b/priam/src/main/java/com/netflix/priam/aws/S3FileSystemBase.java @@ -19,6 +19,11 @@ import com.amazonaws.services.s3.model.BucketLifecycleConfiguration.Rule; import com.amazonaws.services.s3.model.CompleteMultipartUploadResult; import com.amazonaws.services.s3.model.DeleteObjectsRequest; +import com.amazonaws.services.s3.model.lifecycle.LifecycleAndOperator; +import com.amazonaws.services.s3.model.lifecycle.LifecycleFilter; +import com.amazonaws.services.s3.model.lifecycle.LifecyclePredicateVisitor; +import com.amazonaws.services.s3.model.lifecycle.LifecyclePrefixPredicate; +import com.amazonaws.services.s3.model.lifecycle.LifecycleTagPredicate; import com.google.common.collect.Lists; import com.google.common.util.concurrent.RateLimiter; import com.google.inject.Provider; @@ -33,6 +38,7 @@ import java.nio.file.Path; import java.util.Iterator; import java.util.List; +import java.util.Optional; import java.util.concurrent.LinkedBlockingQueue; import java.util.stream.Collectors; import org.slf4j.Logger; @@ -112,7 +118,9 @@ public void cleanup() { List rules = Lists.newArrayList(); lifeConfig.setRules(rules); } + List rules = lifeConfig.getRules(); + if (updateLifecycleRule(config, rules, clusterPath)) { if (rules.size() > 0) { lifeConfig.setRules(rules); @@ -121,42 +129,77 @@ public void cleanup() { } } - private boolean updateLifecycleRule(IConfiguration config, List rules, String prefix) { - Rule rule = null; - for (BucketLifecycleConfiguration.Rule lcRule : rules) { - if (lcRule.getPrefix().equals(prefix)) { - rule = lcRule; - break; + // Dummy class to get Prefix. - Why oh why AWS you can't give the details!! + private class PrefixVisitor implements LifecyclePredicateVisitor { + String prefix; + + @Override + public void visit(LifecyclePrefixPredicate lifecyclePrefixPredicate) { + prefix = lifecyclePrefixPredicate.getPrefix(); + } + + @Override + public void visit(LifecycleTagPredicate lifecycleTagPredicate) {} + + @Override + public void visit(LifecycleAndOperator lifecycleAndOperator) {} + } + + private Optional getBucketLifecycleRule(List rules, String prefix) { + if (rules == null || rules.isEmpty()) return Optional.empty(); + + for (Rule rule : rules) { + PrefixVisitor prefixVisitor = new PrefixVisitor(); + rule.getFilter().getPredicate().accept(prefixVisitor); + String rulePrefix = prefixVisitor.prefix; + if (prefix.equalsIgnoreCase(rulePrefix)) { + return Optional.of(rule); } } - if (rule == null && config.getBackupRetentionDays() <= 0) return false; - if (rule != null && rule.getExpirationInDays() == config.getBackupRetentionDays()) { - logger.info("Cleanup rule already set"); - return false; + + return Optional.empty(); + } + + private boolean updateLifecycleRule(IConfiguration config, List rules, String prefix) { + Optional rule = getBucketLifecycleRule(rules, prefix); + // No need to update the rule as it never existed and retention is not set. + if (!rule.isPresent() && config.getBackupRetentionDays() <= 0) return false; + + // Rule not required as retention days is zero or negative. + if (rule.isPresent() && config.getBackupRetentionDays() <= 0) { + logger.warn( + "Removing the rule for backup retention on prefix: {} as retention is set to [{}] days. Only positive values are supported by S3!!", + prefix, + config.getBackupRetentionDays()); + rules.remove(rule.get()); + return true; } - if (rule == null) { - // Create a new rule - rule = - new BucketLifecycleConfiguration.Rule() - .withExpirationInDays(config.getBackupRetentionDays()) - .withPrefix(prefix); - rule.setStatus(BucketLifecycleConfiguration.ENABLED); - rule.setId(prefix); - rules.add(rule); - logger.info( - "Setting cleanup for {} to {} days", - rule.getPrefix(), - rule.getExpirationInDays()); - } else if (config.getBackupRetentionDays() > 0) { + + // Rule present and is current. + if (rule.isPresent() + && rule.get().getExpirationInDays() == config.getBackupRetentionDays() + && rule.get().getStatus().equalsIgnoreCase(BucketLifecycleConfiguration.ENABLED)) { logger.info( - "Setting cleanup for {} to {} days", - rule.getPrefix(), + "Cleanup rule already set on prefix: {} with retention period: [{}] days", + prefix, config.getBackupRetentionDays()); - rule.setExpirationInDays(config.getBackupRetentionDays()); - } else { - logger.info("Removing cleanup rule for {}", rule.getPrefix()); - rules.remove(rule); + return false; + } + + if (!rule.isPresent()) { + // Create a new rule + rule = Optional.of(new BucketLifecycleConfiguration.Rule()); + rules.add(rule.get()); } + + rule.get().setStatus(BucketLifecycleConfiguration.ENABLED); + rule.get().setExpirationInDays(config.getBackupRetentionDays()); + rule.get().setFilter(new LifecycleFilter(new LifecyclePrefixPredicate(prefix))); + rule.get().setId(prefix); + logger.info( + "Setting cleanup rule for prefix: {} with retention period: [{}] days", + prefix, + config.getBackupRetentionDays()); return true; } diff --git a/priam/src/test/java/com/netflix/priam/backup/TestS3FileSystem.java b/priam/src/test/java/com/netflix/priam/backup/TestS3FileSystem.java index e8f2f92e2..719a38d83 100644 --- a/priam/src/test/java/com/netflix/priam/backup/TestS3FileSystem.java +++ b/priam/src/test/java/com/netflix/priam/backup/TestS3FileSystem.java @@ -23,6 +23,8 @@ import com.amazonaws.services.s3.AmazonS3; import com.amazonaws.services.s3.AmazonS3Client; import com.amazonaws.services.s3.model.*; +import com.amazonaws.services.s3.model.lifecycle.LifecycleFilter; +import com.amazonaws.services.s3.model.lifecycle.LifecyclePrefixPredicate; import com.google.common.collect.Lists; import com.google.inject.Guice; import com.google.inject.Injector; @@ -31,6 +33,7 @@ import com.netflix.priam.aws.S3FileSystem; import com.netflix.priam.aws.S3PartUploader; import com.netflix.priam.backup.AbstractBackupPath.BackupFileType; +import com.netflix.priam.config.IConfiguration; import com.netflix.priam.identity.config.InstanceInfo; import com.netflix.priam.merics.BackupMetrics; import java.io.BufferedOutputStream; @@ -41,6 +44,8 @@ import java.nio.file.Paths; import java.util.ArrayList; import java.util.List; +import java.util.Optional; +import java.util.stream.Collectors; import mockit.Mock; import mockit.MockUp; import org.junit.AfterClass; @@ -57,11 +62,15 @@ public class TestS3FileSystem { "target/data/Keyspace1/Standard1/backups/201108082320/Keyspace1-Standard1-ia-1-Data.db"; private static BackupMetrics backupMetrics; private static String region; + private static IConfiguration configuration; public TestS3FileSystem() { if (injector == null) injector = Guice.createInjector(new BRTestModule()); if (backupMetrics == null) backupMetrics = injector.getInstance(BackupMetrics.class); + + if (configuration == null) configuration = injector.getInstance(IConfiguration.class); + InstanceInfo instanceInfo = injector.getInstance(InstanceInfo.class); region = instanceInfo.getRegion(); } @@ -153,26 +162,38 @@ public void testFileUploadCompleteFailure() throws Exception { @Test public void testCleanupAdd() throws Exception { - MockAmazonS3Client.ruleAvailable = false; + MockAmazonS3Client.setRuleAvailable(false); S3FileSystem fs = injector.getInstance(S3FileSystem.class); fs.cleanup(); Assert.assertEquals(1, MockAmazonS3Client.bconf.getRules().size()); BucketLifecycleConfiguration.Rule rule = MockAmazonS3Client.bconf.getRules().get(0); - logger.info(rule.getPrefix()); - Assert.assertEquals("casstestbackup/" + region + "/fake-app/", rule.getPrefix()); - Assert.assertEquals(5, rule.getExpirationInDays()); + Assert.assertEquals("casstestbackup/" + region + "/fake-app/", rule.getId()); + Assert.assertEquals(configuration.getBackupRetentionDays(), rule.getExpirationInDays()); } @Test public void testCleanupIgnore() throws Exception { - MockAmazonS3Client.ruleAvailable = true; + MockAmazonS3Client.setRuleAvailable(true); S3FileSystem fs = injector.getInstance(S3FileSystem.class); fs.cleanup(); Assert.assertEquals(1, MockAmazonS3Client.bconf.getRules().size()); BucketLifecycleConfiguration.Rule rule = MockAmazonS3Client.bconf.getRules().get(0); - logger.info(rule.getPrefix()); - Assert.assertEquals("casstestbackup/" + region + "/fake-app/", rule.getPrefix()); - Assert.assertEquals(5, rule.getExpirationInDays()); + Assert.assertEquals("casstestbackup/" + region + "/fake-app/", rule.getId()); + Assert.assertEquals(configuration.getBackupRetentionDays(), rule.getExpirationInDays()); + } + + @Test + public void testCleanupUpdate() throws Exception { + MockAmazonS3Client.setRuleAvailable(true); + S3FileSystem fs = injector.getInstance(S3FileSystem.class); + String clusterPrefix = "casstestbackup/" + region + "/fake-app/"; + MockAmazonS3Client.updateRule( + MockAmazonS3Client.getBucketLifecycleConfig(clusterPrefix, 2)); + fs.cleanup(); + Assert.assertEquals(1, MockAmazonS3Client.bconf.getRules().size()); + BucketLifecycleConfiguration.Rule rule = MockAmazonS3Client.bconf.getRules().get(0); + Assert.assertEquals("casstestbackup/" + region + "/fake-app/", rule.getId()); + Assert.assertEquals(configuration.getBackupRetentionDays(), rule.getExpirationInDays()); } @Test @@ -240,8 +261,8 @@ public static void setup() { } static class MockAmazonS3Client extends MockUp { - static boolean ruleAvailable = false; - static BucketLifecycleConfiguration bconf = new BucketLifecycleConfiguration(); + private boolean ruleAvailable = false; + static BucketLifecycleConfiguration bconf; static boolean emulateError = false; @Mock @@ -260,18 +281,6 @@ public PutObjectResult putObject(PutObjectRequest putObjectRequest) @Mock public BucketLifecycleConfiguration getBucketLifecycleConfiguration(String bucketName) { - List rules = Lists.newArrayList(); - if (ruleAvailable) { - String clusterPath = "casstestbackup/" + region + "/fake-app/"; - BucketLifecycleConfiguration.Rule rule = - new BucketLifecycleConfiguration.Rule() - .withExpirationInDays(5) - .withPrefix(clusterPath); - rule.setStatus(BucketLifecycleConfiguration.ENABLED); - rule.setId(clusterPath); - rules.add(rule); - } - bconf.setRules(rules); return bconf; } @@ -287,5 +296,48 @@ public DeleteObjectsResult deleteObjects(DeleteObjectsRequest var1) if (emulateError) throw new AmazonServiceException("Unable to reach AWS"); return null; } + + static BucketLifecycleConfiguration.Rule getBucketLifecycleConfig( + String prefix, int expirationDays) { + return new BucketLifecycleConfiguration.Rule() + .withExpirationInDays(expirationDays) + .withFilter(new LifecycleFilter(new LifecyclePrefixPredicate(prefix))) + .withStatus(BucketLifecycleConfiguration.ENABLED) + .withId(prefix); + } + + static void setRuleAvailable(boolean ruleAvailable) { + if (ruleAvailable) { + bconf = new BucketLifecycleConfiguration(); + if (bconf.getRules() == null) bconf.setRules(Lists.newArrayList()); + + List rules = bconf.getRules(); + String clusterPath = "casstestbackup/" + region + "/fake-app/"; + + List potentialRules = + rules.stream() + .filter(rule -> rule.getId().equalsIgnoreCase(clusterPath)) + .collect(Collectors.toList()); + if (potentialRules == null || potentialRules.isEmpty()) + rules.add( + getBucketLifecycleConfig( + clusterPath, configuration.getBackupRetentionDays())); + } + } + + static void updateRule(BucketLifecycleConfiguration.Rule updatedRule) { + List rules = bconf.getRules(); + Optional updateRule = + rules.stream() + .filter(rule -> rule.getId().equalsIgnoreCase(updatedRule.getId())) + .findFirst(); + if (updateRule.isPresent()) { + rules.remove(updateRule.get()); + rules.add(updatedRule); + } else { + rules.add(updatedRule); + } + bconf.setRules(rules); + } } } From b9b901d746fce74fa2b4ee692c35beec1fe50b96 Mon Sep 17 00:00:00 2001 From: Arun Agrawal Date: Thu, 28 Feb 2019 10:07:21 -0800 Subject: [PATCH 065/228] Fix for ForgottenFiles when there are long running compaction job or Cassandra still reading from the files. (#789) * Fix for ForgottenFiles when there are long running compaction job or Cassandra still reading from the files. --- .../priam/backupv2/ForgottenFilesManager.java | 116 ++++++++++++++---- .../netflix/priam/config/IConfiguration.java | 26 +++- .../priam/config/PriamConfiguration.java | 9 +- .../backupv2/TestForgottenFileManager.java | 104 +++++++++++++--- 4 files changed, 205 insertions(+), 50 deletions(-) diff --git a/priam/src/main/java/com/netflix/priam/backupv2/ForgottenFilesManager.java b/priam/src/main/java/com/netflix/priam/backupv2/ForgottenFilesManager.java index 4c8884008..dd5247f4b 100644 --- a/priam/src/main/java/com/netflix/priam/backupv2/ForgottenFilesManager.java +++ b/priam/src/main/java/com/netflix/priam/backupv2/ForgottenFilesManager.java @@ -20,14 +20,18 @@ import com.google.inject.Inject; import com.netflix.priam.config.IConfiguration; import com.netflix.priam.merics.BackupMetrics; +import com.netflix.priam.utils.DateUtil; import java.io.File; import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.LinkOption; import java.nio.file.Path; import java.nio.file.Paths; import java.time.Instant; import java.time.temporal.ChronoUnit; import java.util.Collection; import java.util.regex.Pattern; +import java.util.stream.Collectors; import org.apache.commons.io.FileUtils; import org.apache.commons.io.filefilter.FileFilterUtils; import org.apache.commons.io.filefilter.IOFileFilter; @@ -75,10 +79,9 @@ public void findAndMoveForgottenFiles(Instant snapshotInstant, File snapshotDir) if (columnfamilyFiles.size() == 0) return; logger.warn( - "# of forgotten files: {} found for CF: {}", + "# of potential forgotten files: {} found for CF: {}", columnfamilyFiles.size(), columnfamilyDir.getName()); - backupMetrics.incrementForgottenFiles(columnfamilyFiles.size()); // Move the files to lost_found directory if configured. moveForgottenFiles(columnfamilyDir, columnfamilyFiles); @@ -104,17 +107,20 @@ protected Collection getColumnfamilyFiles(Instant snapshotInstant, File co FileFilterUtils.asFileFilter( pathname -> tmpFilePattern.matcher(pathname.getName()).matches()); IOFileFilter tmpFileFilter = FileFilterUtils.or(tmpFileFilter1, tmpFileFilter2); - // Here we are allowing files which were more than - // @link{IConfiguration#getForgottenFileGracePeriodDays}. We do this to allow cassandra - // to clean up any files which were generated as part of repair/compaction and cleanup - // thread has not already deleted. - // Refer to https://issues.apache.org/jira/browse/CASSANDRA-6756 and - // https://issues.apache.org/jira/browse/CASSANDRA-7066 - // for more information. + /* + Here we are allowing files which were more than + @link{IConfiguration#getForgottenFileGracePeriodDaysForCompaction}. We do this to allow cassandra + to have files which were generated as part of long running compaction. + Refer to https://issues.apache.org/jira/browse/CASSANDRA-6756 and + https://issues.apache.org/jira/browse/CASSANDRA-7066 + for more information. + */ IOFileFilter ageFilter = FileFilterUtils.ageFileFilter( snapshotInstant - .minus(config.getForgottenFileGracePeriodDays(), ChronoUnit.DAYS) + .minus( + config.getForgottenFileGracePeriodDaysForCompaction(), + ChronoUnit.DAYS) .toEpochMilli()); IOFileFilter fileFilter = FileFilterUtils.and( @@ -125,21 +131,83 @@ protected Collection getColumnfamilyFiles(Instant snapshotInstant, File co return FileUtils.listFiles(columnfamilyDir, fileFilter, null); } - protected void moveForgottenFiles(File columnfamilyDir, Collection columnfamilyFiles) { + protected void moveForgottenFiles(File columnfamilyDir, Collection columnfamilyFiles) + throws IOException { + // This is a list of potential forgotten file(s). Note that C* might still be using + // files as part of read, so we really do not want to move them until we meet the + // @link{IConfiguration#getForgottenFileGracePeriodDaysForRead} window elapses. + final Path destDir = Paths.get(columnfamilyDir.getAbsolutePath(), LOST_FOUND); - for (File file : columnfamilyFiles) { - logger.warn( - "Forgotten file: {} found for CF: {}", - file.getAbsolutePath(), - columnfamilyDir.getName()); - if (config.isForgottenFileMoveEnabled()) { - try { - FileUtils.moveFileToDirectory(file, destDir.toFile(), true); - } catch (IOException e) { - logger.error( - "Exception occurred while trying to move forgottenFile: {}. Ignoring the error and continuing with remaining backup/forgotten files.", - file); - e.printStackTrace(); + FileUtils.forceMkdir(destDir.toFile()); + final Collection columnfamilyPaths = + columnfamilyFiles + .parallelStream() + .map(file -> Paths.get(file.getAbsolutePath())) + .collect(Collectors.toList()); + + for (Path file : columnfamilyPaths) { + try { + final Path symbolic_link = + Paths.get(destDir.toFile().getAbsolutePath(), file.toFile().getName()); + // Lets see if there is a symbolic link to this file already? + if (!Files.exists(symbolic_link)) { + // If not, lets create one and work on next file. + Files.createSymbolicLink(symbolic_link, file); + continue; + } else if (Files.isSymbolicLink(symbolic_link)) { + // Symbolic link exists, is it older than our timeframe? + Instant last_modified_time = + Files.getLastModifiedTime(symbolic_link, LinkOption.NOFOLLOW_LINKS) + .toInstant(); + if (DateUtil.getInstant() + .isAfter( + last_modified_time.plus( + config.getForgottenFileGracePeriodDaysForRead(), + ChronoUnit.DAYS))) { + // Eligible for move. + logger.info( + "Eligible for move: Forgotten file: {} found for CF: {}", + file, + columnfamilyDir.getName()); + backupMetrics.incrementForgottenFiles(1); + if (config.isForgottenFileMoveEnabled()) { + try { + // Remove our symbolic link. Note that deletion of symbolic link + // does not remove the original file. + Files.delete(symbolic_link); + FileUtils.moveFileToDirectory( + file.toFile(), destDir.toFile(), true); + logger.warn( + "Successfully moved forgotten file: {} found for CF: {}", + file, + columnfamilyDir.getName()); + } catch (IOException e) { + logger.error( + "Exception occurred while trying to move forgottenFile: {}. Ignoring the error and continuing with remaining backup/forgotten files.", + file); + e.printStackTrace(); + } + } + } + } + + } catch (IOException e) { + logger.error("Forgotten file: Error while trying to process the file: {}", file); + e.printStackTrace(); + } + } + + // Clean LOST_FOUND directory of any previous symbolic link files which are not considered + // lost any more. + for (File file : FileUtils.listFiles(destDir.toFile(), null, false)) { + Path filePath = Paths.get(file.getAbsolutePath()); + if (Files.isSymbolicLink(filePath)) { + Path originalFile = Files.readSymbolicLink(filePath); + if (!columnfamilyPaths.contains(originalFile)) { + Files.delete(filePath); + logger.info( + "Deleting the symbolic link as it is not considered as lost anymore. filePath: {}", + filePath); } } } diff --git a/priam/src/main/java/com/netflix/priam/config/IConfiguration.java b/priam/src/main/java/com/netflix/priam/config/IConfiguration.java index d690a6934..4fe300446 100644 --- a/priam/src/main/java/com/netflix/priam/config/IConfiguration.java +++ b/priam/src/main/java/com/netflix/priam/config/IConfiguration.java @@ -992,13 +992,29 @@ default int getPostRestoreHookHeartbeatCheckFrequencyInMs() { } /** - * Grace period for the file(that should have been deleted by cassandra) that are considered to - * be forgotten. Only required for cassandra 2.x. + * Grace period in days for the file that 'could' be considered to be forgotten by cassandra, + * but actually may be output of a long-running compaction job. Note that cassandra creates + * output of the compaction as non-tmp-link files (whole SSTable) but are still not part of the + * final "view" and thus not part of a snapshot. Only required for cassandra 2.x. Default: 5 * - * @return grace period for the forgotten files. + * @return grace period for the compaction output forgotten files. */ - default int getForgottenFileGracePeriodDays() { - return 1; + default int getForgottenFileGracePeriodDaysForCompaction() { + return 5; + } + + /** + * Grace period in days for which a file is not considered forgotten by cassandra (that would be + * deleted by cassandra) as file could be used in the read path of the cassandra. Note that read + * path could imply streaming to a joining neighbor or for repair. When cassandra is done with a + * compaction, the input files to compaction, are removed from the "view" and thus not part of + * snapshot, but these files may very well be used for streaming, repair etc and thus cannot be + * removed. + * + * @return grace period in days for read path forgotten files. + */ + default int getForgottenFileGracePeriodDaysForRead() { + return 3; } /** diff --git a/priam/src/main/java/com/netflix/priam/config/PriamConfiguration.java b/priam/src/main/java/com/netflix/priam/config/PriamConfiguration.java index 45f4365b5..660f3a36c 100644 --- a/priam/src/main/java/com/netflix/priam/config/PriamConfiguration.java +++ b/priam/src/main/java/com/netflix/priam/config/PriamConfiguration.java @@ -750,8 +750,13 @@ public String getMergedConfigurationCronExpression() { } @Override - public int getForgottenFileGracePeriodDays() { - return config.get(PRIAM_PRE + ".forgottenFileGracePeriodDays", 1); + public int getForgottenFileGracePeriodDaysForCompaction() { + return config.get(PRIAM_PRE + ".forgottenFileGracePeriodDaysForCompaction", 5); + } + + @Override + public int getForgottenFileGracePeriodDaysForRead() { + return config.get(PRIAM_PRE + ".forgottenFileGracePeriodDaysForRead", 3); } @Override diff --git a/priam/src/test/java/com/netflix/priam/backupv2/TestForgottenFileManager.java b/priam/src/test/java/com/netflix/priam/backupv2/TestForgottenFileManager.java index 2f566d328..869891d99 100644 --- a/priam/src/test/java/com/netflix/priam/backupv2/TestForgottenFileManager.java +++ b/priam/src/test/java/com/netflix/priam/backupv2/TestForgottenFileManager.java @@ -21,10 +21,10 @@ import com.google.inject.Injector; import com.netflix.priam.backup.BRTestModule; import com.netflix.priam.config.FakeConfiguration; -import com.netflix.priam.config.IConfiguration; import com.netflix.priam.merics.BackupMetrics; import com.netflix.priam.utils.DateUtil; import java.io.File; +import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; @@ -44,7 +44,7 @@ public class TestForgottenFileManager { private ForgottenFilesManager forgottenFilesManager; private TestBackupUtils testBackupUtils; - private IConfiguration configuration; + private ForgottenFilesConfiguration configuration; private List allFiles = new ArrayList<>(); private Instant snapshotInstant; private Path snapshotDir; @@ -62,12 +62,11 @@ public void prep() throws Exception { cleanup(); Instant now = DateUtil.getInstant(); snapshotInstant = now; - Path file1 = Paths.get(testBackupUtils.createFile("file1", now.minus(5, ChronoUnit.DAYS))); - Path file2 = Paths.get(testBackupUtils.createFile("file2", now.minus(3, ChronoUnit.DAYS))); - Path file3 = Paths.get(testBackupUtils.createFile("file3", now.minus(2, ChronoUnit.DAYS))); - Path file4 = Paths.get(testBackupUtils.createFile("file4", now.minus(6, ChronoUnit.HOURS))); - Path file5 = - Paths.get(testBackupUtils.createFile("file5", now.minus(1, ChronoUnit.MINUTES))); + Path file1 = Paths.get(testBackupUtils.createFile("file1", now.minus(10, ChronoUnit.DAYS))); + Path file2 = Paths.get(testBackupUtils.createFile("file2", now.minus(8, ChronoUnit.DAYS))); + Path file3 = Paths.get(testBackupUtils.createFile("file3", now.minus(6, ChronoUnit.DAYS))); + Path file4 = Paths.get(testBackupUtils.createFile("file4", now.minus(4, ChronoUnit.DAYS))); + Path file5 = Paths.get(testBackupUtils.createFile("file5", now.minus(1, ChronoUnit.DAYS))); Path file6 = Paths.get( testBackupUtils.createFile( @@ -103,23 +102,58 @@ public void cleanup() throws Exception { } @Test - public void testMoveForgottenFiles() { + public void testMoveForgottenFiles() throws IOException, InterruptedException { Collection files = allFiles.stream().map(Path::toFile).collect(Collectors.toList()); - forgottenFilesManager.moveForgottenFiles( - new File(configuration.getDataFileLocation()), files); Path lostFoundDir = Paths.get(configuration.getDataFileLocation(), forgottenFilesManager.LOST_FOUND); + + // Lets create some extra symlinks in the LOST_FOUND folder. They should not exist anymore + Path randomSymlink = Paths.get(lostFoundDir.toFile().getAbsolutePath(), "random"); + Files.createDirectory(lostFoundDir); + Files.createSymbolicLink(randomSymlink, lostFoundDir); + + forgottenFilesManager.moveForgottenFiles( + new File(configuration.getDataFileLocation()), files); + + // Extra symlinks are deleted. + Assert.assertFalse(Files.exists(randomSymlink)); + + // Symlinks are created for all the files. They are not moved yet. + Collection symlinkFiles = FileUtils.listFiles(lostFoundDir.toFile(), null, false); + Assert.assertEquals(allFiles.size(), symlinkFiles.size()); + for (Path file : allFiles) { + Path symlink = Paths.get(lostFoundDir.toString(), file.getFileName().toString()); + Assert.assertTrue(symlinkFiles.contains(symlink.toFile())); + Assert.assertTrue(Files.isSymbolicLink(symlink)); + Assert.assertTrue(Files.exists(file)); + } + + // Lets change the configuration and try again!! + configuration.setGracePeriodForgottenFileInDaysForRead(0); + forgottenFilesManager.moveForgottenFiles( + new File(configuration.getDataFileLocation()), files); Collection movedFiles = FileUtils.listFiles(lostFoundDir.toFile(), null, false); Assert.assertEquals(allFiles.size(), movedFiles.size()); - for (Path file : allFiles) - Assert.assertTrue( - movedFiles.contains( - Paths.get(lostFoundDir.toString(), file.getFileName().toString()) - .toFile())); + movedFiles + .stream() + .forEach( + file -> { + Assert.assertTrue( + Files.isRegularFile(Paths.get(file.getAbsolutePath()))); + }); + allFiles.stream() + .forEach( + file -> { + Assert.assertFalse(file.toFile().exists()); + }); + + configuration.setGracePeriodForgottenFileInDaysForRead( + ForgottenFilesConfiguration.DEFAULT_GRACE_PERIOD); } @Test public void getColumnfamilyFiles() { + Path columnfamilyDir = allFiles.get(0).getParent(); Collection columnfamilyFiles = forgottenFilesManager.getColumnfamilyFiles( @@ -136,7 +170,7 @@ public void findAndMoveForgottenFiles() { Paths.get(allFiles.get(0).getParent().toString(), forgottenFilesManager.LOST_FOUND); forgottenFilesManager.findAndMoveForgottenFiles(snapshotInstant, snapshotDir.toFile()); - // Only one forgotten file - file1. + // Only one potential forgotten file - file1. It will be symlink here. Collection movedFiles = FileUtils.listFiles(lostFoundDir.toFile(), null, false); Assert.assertEquals(1, movedFiles.size()); Assert.assertTrue( @@ -145,10 +179,29 @@ public void findAndMoveForgottenFiles() { .next() .getName() .equals(allFiles.get(0).getFileName().toString())); + Assert.assertTrue( + Files.isSymbolicLink(Paths.get(movedFiles.iterator().next().getAbsolutePath()))); - // All other files still remain in columnfamily dir. + // All files still remain in columnfamily dir. Collection cfFiles = FileUtils.listFiles(new File(allFiles.get(0).getParent().toString()), null, false); + Assert.assertEquals(allFiles.size(), cfFiles.size()); + + // Snapshot is untouched. + Collection snapshotFiles = FileUtils.listFiles(snapshotDir.toFile(), null, false); + Assert.assertEquals(3, snapshotFiles.size()); + + // Lets change the configuration and try again!! + configuration.setGracePeriodForgottenFileInDaysForRead(0); + forgottenFilesManager.findAndMoveForgottenFiles(snapshotInstant, snapshotDir.toFile()); + configuration.setGracePeriodForgottenFileInDaysForRead( + ForgottenFilesConfiguration.DEFAULT_GRACE_PERIOD); + movedFiles = FileUtils.listFiles(lostFoundDir.toFile(), null, false); + Assert.assertEquals(1, movedFiles.size()); + Assert.assertTrue( + Files.isRegularFile(Paths.get(movedFiles.iterator().next().getAbsolutePath()))); + cfFiles = + FileUtils.listFiles(new File(allFiles.get(0).getParent().toString()), null, false); Assert.assertEquals(6, cfFiles.size()); int temp_file_name = 1; for (File file : cfFiles) { @@ -156,14 +209,27 @@ public void findAndMoveForgottenFiles() { } // Snapshot is untouched. - Collection snapshotFiles = FileUtils.listFiles(snapshotDir.toFile(), null, false); + snapshotFiles = FileUtils.listFiles(snapshotDir.toFile(), null, false); Assert.assertEquals(3, snapshotFiles.size()); } private class ForgottenFilesConfiguration extends FakeConfiguration { + protected static final int DEFAULT_GRACE_PERIOD = 3; + private int gracePeriodForgottenFileInDaysForRead = DEFAULT_GRACE_PERIOD; + @Override public boolean isForgottenFileMoveEnabled() { return true; } + + @Override + public int getForgottenFileGracePeriodDaysForRead() { + return gracePeriodForgottenFileInDaysForRead; + } + + public void setGracePeriodForgottenFileInDaysForRead( + int gracePeriodForgottenFileInDaysForRead) { + this.gracePeriodForgottenFileInDaysForRead = gracePeriodForgottenFileInDaysForRead; + } } } From 67ca668dcf0678dfee98dbc32c336d5bdc02fcd6 Mon Sep 17 00:00:00 2001 From: Arun Agrawal Date: Mon, 4 Mar 2019 10:24:00 -0800 Subject: [PATCH 066/228] clenaup code --- .../identity/token/DeadTokenRetriever.java | 21 ++----------------- 1 file changed, 2 insertions(+), 19 deletions(-) diff --git a/priam/src/main/java/com/netflix/priam/identity/token/DeadTokenRetriever.java b/priam/src/main/java/com/netflix/priam/identity/token/DeadTokenRetriever.java index 0e66fbf59..fd4066511 100755 --- a/priam/src/main/java/com/netflix/priam/identity/token/DeadTokenRetriever.java +++ b/priam/src/main/java/com/netflix/priam/identity/token/DeadTokenRetriever.java @@ -26,7 +26,6 @@ import com.sun.jersey.api.client.WebResource; import com.sun.jersey.api.client.config.ClientConfig; import com.sun.jersey.api.client.config.DefaultClientConfig; -import java.util.Arrays; import java.util.List; import java.util.Random; import javax.ws.rs.core.MediaType; @@ -43,8 +42,8 @@ public class DeadTokenRetriever extends TokenRetrieverBase implements IDeadToken private final IMembership membership; private final IConfiguration config; private final Sleeper sleeper; - private String - replacedIp; // The IP address of the dead instance to which we will acquire its token + // The IP address of the dead instance to which we will acquire its token + private String replacedIp; private ListMultimap locMap; private final InstanceInfo instanceInfo; @@ -67,22 +66,6 @@ private List getDualAccountRacMembership(List asgInstances) { List crossAccountAsgInstances = membership.getCrossAccountRacMembership(); - if (logger.isInfoEnabled()) { - if (instanceInfo.getInstanceEnvironment() == InstanceInfo.InstanceEnvironment.CLASSIC) { - logger.info( - "EC2 classic instances (local ASG): " - + Arrays.toString(asgInstances.toArray())); - logger.info( - "VPC Account (cross-account ASG): " - + Arrays.toString(crossAccountAsgInstances.toArray())); - } else { - logger.info("VPC Account (local ASG): " + Arrays.toString(asgInstances.toArray())); - logger.info( - "EC2 classic instances (cross-account ASG): " - + Arrays.toString(crossAccountAsgInstances.toArray())); - } - } - // Remove duplicates (probably there are not) asgInstances.removeAll(crossAccountAsgInstances); From 47437c6726456e0f90167761dac21dbfd6322994 Mon Sep 17 00:00:00 2001 From: Arun Agrawal Date: Tue, 5 Mar 2019 14:17:52 -0800 Subject: [PATCH 067/228] Use older API for prefix filtering, if prefix is available. --- .../java/com/netflix/priam/aws/S3FileSystemBase.java | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/priam/src/main/java/com/netflix/priam/aws/S3FileSystemBase.java b/priam/src/main/java/com/netflix/priam/aws/S3FileSystemBase.java index b282a02cf..baf03ff44 100755 --- a/priam/src/main/java/com/netflix/priam/aws/S3FileSystemBase.java +++ b/priam/src/main/java/com/netflix/priam/aws/S3FileSystemBase.java @@ -149,9 +149,15 @@ private Optional getBucketLifecycleRule(List rules, String prefix) { if (rules == null || rules.isEmpty()) return Optional.empty(); for (Rule rule : rules) { - PrefixVisitor prefixVisitor = new PrefixVisitor(); - rule.getFilter().getPredicate().accept(prefixVisitor); - String rulePrefix = prefixVisitor.prefix; + String rulePrefix = ""; + if (rule.getFilter() != null) { + PrefixVisitor prefixVisitor = new PrefixVisitor(); + rule.getFilter().getPredicate().accept(prefixVisitor); + rulePrefix = prefixVisitor.prefix; + } else if (rule.getPrefix() != null) { + // Being backwards compatible, here. + rulePrefix = rule.getPrefix(); + } if (prefix.equalsIgnoreCase(rulePrefix)) { return Optional.of(rule); } From 20a1bbbcd4ef8e013c6e7ff169467594a44236e3 Mon Sep 17 00:00:00 2001 From: Arun Agrawal Date: Tue, 5 Mar 2019 18:04:31 -0800 Subject: [PATCH 068/228] Send notification only if we upload the file. --- .../priam/backup/AbstractFileSystem.java | 21 +++++++++---------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/priam/src/main/java/com/netflix/priam/backup/AbstractFileSystem.java b/priam/src/main/java/com/netflix/priam/backup/AbstractFileSystem.java index 7f3bc6a34..671f56314 100644 --- a/priam/src/main/java/com/netflix/priam/backup/AbstractFileSystem.java +++ b/priam/src/main/java/com/netflix/priam/backup/AbstractFileSystem.java @@ -184,11 +184,11 @@ public void uploadFile( if (tasksQueued.add(localPath)) { logger.info("Uploading file: {} to location: {}", localPath, remotePath); try { - notifyEventStart(new BackupEvent(path)); long uploadedFileSize; // Upload file if it not present at remote location. if (path.getType() != BackupFileType.SST_V2 || !checkObjectExists(remotePath)) { + notifyEventStart(new BackupEvent(path)); uploadedFileSize = new BoundedExponentialRetryCallable(500, 10000, retry) { @Override @@ -204,18 +204,13 @@ public Long retriableCall() throws Exception { backupMetrics.recordUploadRate(uploadedFileSize); backupMetrics.incrementValidUploads(); + path.setCompressedFileSize(uploadedFileSize); + notifyEventSuccess(new BackupEvent(path)); } else { - // file is already uploaded to remote file system. get the compressed file size - // to update it for notification message. - uploadedFileSize = getFileSize(remotePath); - logger.info( - "File: {} already present on remoteFileSystem with file size: {}", - remotePath, - uploadedFileSize); + // file is already uploaded to remote file system. + logger.info("File: {} already present on remoteFileSystem.", remotePath); } - path.setCompressedFileSize(uploadedFileSize); - notifyEventSuccess(new BackupEvent(path)); logger.info( "Successfully uploaded file: {} to location: {}", localPath, remotePath); @@ -229,7 +224,11 @@ public Long retriableCall() throws Exception { backupMetrics.incrementInvalidUploads(); notifyEventFailure(new BackupEvent(path)); logger.error( - "Error while uploading file: {} to location: {}", localPath, remotePath); + "Error while uploading file: {} to location: {}. Exception: Msg: [{}], Trace: {}", + localPath, + remotePath, + e.getMessage(), + e.getStackTrace()); throw new BackupRestoreException(e.getMessage()); } finally { // Remove the task from the list so if we try to upload file ever again, we can. From 4ba78e8b821ab2ec5abb34fee49b0028996d5973 Mon Sep 17 00:00:00 2001 From: Arun Agrawal Date: Sat, 9 Mar 2019 16:17:38 -0800 Subject: [PATCH 069/228] No new functionality: Moving files --- .../java/com/netflix/priam/PriamServer.java | 8 +++--- .../netflix/priam/backup/SnapshotBackup.java | 2 +- .../BackupTTLService.java | 7 ++--- .../BackupVerificationService.java | 2 +- .../SnapshotMetaService.java | 26 ++++++++++--------- .../management/IClusterManagement.java | 2 +- .../netflix/priam/connection/JMXNodeTool.java | 2 +- .../CassandraMonitor.java | 3 +-- .../priam/resources/BackupServletV2.java | 4 +-- .../cluser/management/TestCompaction.groovy | 2 +- .../TestBackupTTLService.java | 5 ++-- .../TestBackupVerificationService.java | 2 +- .../TestSnapshotMetaService.java | 5 ++-- .../TestCassandraMonitor.java | 3 +-- 14 files changed, 34 insertions(+), 39 deletions(-) rename priam/src/main/java/com/netflix/priam/{services => backupv2}/BackupTTLService.java (97%) rename priam/src/main/java/com/netflix/priam/{services => backupv2}/BackupVerificationService.java (99%) rename priam/src/main/java/com/netflix/priam/{services => backupv2}/SnapshotMetaService.java (96%) rename priam/src/main/java/com/netflix/priam/{services => health}/CassandraMonitor.java (98%) rename priam/src/test/java/com/netflix/priam/{services => backupv2}/TestBackupTTLService.java (98%) rename priam/src/test/java/com/netflix/priam/{services => backupv2}/TestBackupVerificationService.java (98%) rename priam/src/test/java/com/netflix/priam/{services => backupv2}/TestSnapshotMetaService.java (98%) rename priam/src/test/java/com/netflix/priam/{services => health}/TestCassandraMonitor.java (98%) diff --git a/priam/src/main/java/com/netflix/priam/PriamServer.java b/priam/src/main/java/com/netflix/priam/PriamServer.java index cef77507a..e3faa8a32 100644 --- a/priam/src/main/java/com/netflix/priam/PriamServer.java +++ b/priam/src/main/java/com/netflix/priam/PriamServer.java @@ -23,6 +23,9 @@ import com.netflix.priam.backup.CommitLogBackupTask; import com.netflix.priam.backup.IncrementalBackup; import com.netflix.priam.backup.SnapshotBackup; +import com.netflix.priam.backupv2.BackupTTLService; +import com.netflix.priam.backupv2.BackupVerificationService; +import com.netflix.priam.backupv2.SnapshotMetaService; import com.netflix.priam.cluster.management.Compaction; import com.netflix.priam.cluster.management.Flush; import com.netflix.priam.cluster.management.IClusterManagement; @@ -30,14 +33,11 @@ import com.netflix.priam.config.IConfiguration; import com.netflix.priam.config.PriamConfigurationPersister; import com.netflix.priam.defaultimpl.ICassandraProcess; +import com.netflix.priam.health.CassandraMonitor; import com.netflix.priam.identity.InstanceIdentity; import com.netflix.priam.restore.RestoreContext; import com.netflix.priam.scheduler.PriamScheduler; import com.netflix.priam.scheduler.TaskTimer; -import com.netflix.priam.services.BackupTTLService; -import com.netflix.priam.services.BackupVerificationService; -import com.netflix.priam.services.CassandraMonitor; -import com.netflix.priam.services.SnapshotMetaService; import com.netflix.priam.tuner.TuneCassandra; import com.netflix.priam.utils.Sleeper; import com.netflix.priam.utils.SystemUtils; diff --git a/priam/src/main/java/com/netflix/priam/backup/SnapshotBackup.java b/priam/src/main/java/com/netflix/priam/backup/SnapshotBackup.java index b8d24a384..7a547912e 100644 --- a/priam/src/main/java/com/netflix/priam/backup/SnapshotBackup.java +++ b/priam/src/main/java/com/netflix/priam/backup/SnapshotBackup.java @@ -24,10 +24,10 @@ import com.netflix.priam.backupv2.ForgottenFilesManager; import com.netflix.priam.config.IConfiguration; import com.netflix.priam.defaultimpl.CassandraOperations; +import com.netflix.priam.health.CassandraMonitor; import com.netflix.priam.identity.InstanceIdentity; import com.netflix.priam.scheduler.CronTimer; import com.netflix.priam.scheduler.TaskTimer; -import com.netflix.priam.services.CassandraMonitor; import com.netflix.priam.utils.DateUtil; import com.netflix.priam.utils.ThreadSleeper; import java.io.File; diff --git a/priam/src/main/java/com/netflix/priam/services/BackupTTLService.java b/priam/src/main/java/com/netflix/priam/backupv2/BackupTTLService.java similarity index 97% rename from priam/src/main/java/com/netflix/priam/services/BackupTTLService.java rename to priam/src/main/java/com/netflix/priam/backupv2/BackupTTLService.java index bbf98a43a..22f2c7b13 100644 --- a/priam/src/main/java/com/netflix/priam/services/BackupTTLService.java +++ b/priam/src/main/java/com/netflix/priam/backupv2/BackupTTLService.java @@ -1,5 +1,5 @@ /* - * Copyright 2018 Netflix, Inc. + * Copyright 2019 Netflix, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,7 +15,7 @@ * */ -package com.netflix.priam.services; +package com.netflix.priam.backupv2; import com.google.inject.Inject; import com.google.inject.Provider; @@ -24,9 +24,6 @@ import com.netflix.priam.backup.BackupRestoreException; import com.netflix.priam.backup.IBackupFileSystem; import com.netflix.priam.backup.IFileSystemContext; -import com.netflix.priam.backupv2.ColumnfamilyResult; -import com.netflix.priam.backupv2.IMetaProxy; -import com.netflix.priam.backupv2.MetaFileReader; import com.netflix.priam.config.IBackupRestoreConfig; import com.netflix.priam.config.IConfiguration; import com.netflix.priam.scheduler.CronTimer; diff --git a/priam/src/main/java/com/netflix/priam/services/BackupVerificationService.java b/priam/src/main/java/com/netflix/priam/backupv2/BackupVerificationService.java similarity index 99% rename from priam/src/main/java/com/netflix/priam/services/BackupVerificationService.java rename to priam/src/main/java/com/netflix/priam/backupv2/BackupVerificationService.java index 44e681d18..9065838e1 100644 --- a/priam/src/main/java/com/netflix/priam/services/BackupVerificationService.java +++ b/priam/src/main/java/com/netflix/priam/backupv2/BackupVerificationService.java @@ -15,7 +15,7 @@ * */ -package com.netflix.priam.services; +package com.netflix.priam.backupv2; import com.google.inject.Inject; import com.google.inject.Singleton; diff --git a/priam/src/main/java/com/netflix/priam/services/SnapshotMetaService.java b/priam/src/main/java/com/netflix/priam/backupv2/SnapshotMetaService.java similarity index 96% rename from priam/src/main/java/com/netflix/priam/services/SnapshotMetaService.java rename to priam/src/main/java/com/netflix/priam/backupv2/SnapshotMetaService.java index 8c01cc889..1d6cfadf7 100644 --- a/priam/src/main/java/com/netflix/priam/services/SnapshotMetaService.java +++ b/priam/src/main/java/com/netflix/priam/backupv2/SnapshotMetaService.java @@ -1,25 +1,28 @@ -/** - * Copyright 2018 Netflix, Inc. +/* + * Copyright 2019 Netflix, Inc. * - *

    Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file - * except in compliance with the License. You may obtain a copy of the License at + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - *

    http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * - *

    Unless required by applicable law or agreed to in writing, software distributed under the - * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing permissions and + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and * limitations under the License. + * */ -package com.netflix.priam.services; +package com.netflix.priam.backupv2; import com.google.inject.Provider; import com.netflix.priam.backup.*; import com.netflix.priam.backup.BackupVersion; -import com.netflix.priam.backupv2.*; import com.netflix.priam.config.IBackupRestoreConfig; import com.netflix.priam.config.IConfiguration; import com.netflix.priam.defaultimpl.CassandraOperations; +import com.netflix.priam.health.CassandraMonitor; import com.netflix.priam.identity.InstanceIdentity; import com.netflix.priam.scheduler.CronTimer; import com.netflix.priam.scheduler.TaskTimer; @@ -101,8 +104,7 @@ private enum MetaStep { } /** - * Interval between generating snapshot meta file using {@link - * com.netflix.priam.services.SnapshotMetaService}. + * Interval between generating snapshot meta file using {@link SnapshotMetaService}. * * @param backupRestoreConfig {@link * IBackupRestoreConfig#getSnapshotMetaServiceCronExpression()} to get configuration details diff --git a/priam/src/main/java/com/netflix/priam/cluster/management/IClusterManagement.java b/priam/src/main/java/com/netflix/priam/cluster/management/IClusterManagement.java index 64c13701a..8b80a1d1a 100644 --- a/priam/src/main/java/com/netflix/priam/cluster/management/IClusterManagement.java +++ b/priam/src/main/java/com/netflix/priam/cluster/management/IClusterManagement.java @@ -14,9 +14,9 @@ package com.netflix.priam.cluster.management; import com.netflix.priam.config.IConfiguration; +import com.netflix.priam.health.CassandraMonitor; import com.netflix.priam.merics.IMeasurement; import com.netflix.priam.scheduler.Task; -import com.netflix.priam.services.CassandraMonitor; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; import org.slf4j.Logger; diff --git a/priam/src/main/java/com/netflix/priam/connection/JMXNodeTool.java b/priam/src/main/java/com/netflix/priam/connection/JMXNodeTool.java index e650e863e..2683d0ec4 100644 --- a/priam/src/main/java/com/netflix/priam/connection/JMXNodeTool.java +++ b/priam/src/main/java/com/netflix/priam/connection/JMXNodeTool.java @@ -19,7 +19,7 @@ import com.google.inject.Inject; import com.google.inject.Singleton; import com.netflix.priam.config.IConfiguration; -import com.netflix.priam.services.CassandraMonitor; +import com.netflix.priam.health.CassandraMonitor; import com.netflix.priam.utils.BoundedExponentialRetryCallable; import java.io.IOException; import java.io.PrintStream; diff --git a/priam/src/main/java/com/netflix/priam/services/CassandraMonitor.java b/priam/src/main/java/com/netflix/priam/health/CassandraMonitor.java similarity index 98% rename from priam/src/main/java/com/netflix/priam/services/CassandraMonitor.java rename to priam/src/main/java/com/netflix/priam/health/CassandraMonitor.java index 5a9ed0fa0..f55cd506c 100644 --- a/priam/src/main/java/com/netflix/priam/services/CassandraMonitor.java +++ b/priam/src/main/java/com/netflix/priam/health/CassandraMonitor.java @@ -14,14 +14,13 @@ * limitations under the License. * */ -package com.netflix.priam.services; +package com.netflix.priam.health; import com.google.inject.Inject; import com.google.inject.Singleton; import com.netflix.priam.config.IConfiguration; import com.netflix.priam.connection.JMXNodeTool; import com.netflix.priam.defaultimpl.ICassandraProcess; -import com.netflix.priam.health.InstanceState; import com.netflix.priam.merics.CassMonitorMetrics; import com.netflix.priam.scheduler.SimpleTimer; import com.netflix.priam.scheduler.Task; diff --git a/priam/src/main/java/com/netflix/priam/resources/BackupServletV2.java b/priam/src/main/java/com/netflix/priam/resources/BackupServletV2.java index 57eb1e98d..a42b5408e 100644 --- a/priam/src/main/java/com/netflix/priam/resources/BackupServletV2.java +++ b/priam/src/main/java/com/netflix/priam/resources/BackupServletV2.java @@ -23,8 +23,8 @@ import com.netflix.priam.backup.BackupVerificationResult; import com.netflix.priam.backup.BackupVersion; import com.netflix.priam.backup.IBackupStatusMgr; -import com.netflix.priam.services.BackupTTLService; -import com.netflix.priam.services.SnapshotMetaService; +import com.netflix.priam.backupv2.BackupTTLService; +import com.netflix.priam.backupv2.SnapshotMetaService; import com.netflix.priam.utils.DateUtil; import com.netflix.priam.utils.DateUtil.DateRange; import java.time.Instant; diff --git a/priam/src/test/groovy/com/netflix/priam/cluser/management/TestCompaction.groovy b/priam/src/test/groovy/com/netflix/priam/cluser/management/TestCompaction.groovy index 5838d298b..2de951228 100644 --- a/priam/src/test/groovy/com/netflix/priam/cluser/management/TestCompaction.groovy +++ b/priam/src/test/groovy/com/netflix/priam/cluser/management/TestCompaction.groovy @@ -21,7 +21,7 @@ import com.netflix.priam.config.FakeConfiguration import com.netflix.priam.backup.BRTestModule import com.netflix.priam.cluster.management.Compaction import com.netflix.priam.defaultimpl.CassandraOperations -import com.netflix.priam.services.CassandraMonitor +import com.netflix.priam.health.CassandraMonitor import mockit.Mock import mockit.MockUp import spock.lang.Shared diff --git a/priam/src/test/java/com/netflix/priam/services/TestBackupTTLService.java b/priam/src/test/java/com/netflix/priam/backupv2/TestBackupTTLService.java similarity index 98% rename from priam/src/test/java/com/netflix/priam/services/TestBackupTTLService.java rename to priam/src/test/java/com/netflix/priam/backupv2/TestBackupTTLService.java index b642d8445..7b526ba75 100644 --- a/priam/src/test/java/com/netflix/priam/services/TestBackupTTLService.java +++ b/priam/src/test/java/com/netflix/priam/backupv2/TestBackupTTLService.java @@ -1,5 +1,5 @@ /* - * Copyright 2018 Netflix, Inc. + * Copyright 2019 Netflix, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,7 +15,7 @@ * */ -package com.netflix.priam.services; +package com.netflix.priam.backupv2; import com.google.inject.Guice; import com.google.inject.Injector; @@ -23,7 +23,6 @@ import com.netflix.priam.backup.AbstractBackupPath; import com.netflix.priam.backup.BRTestModule; import com.netflix.priam.backup.FakeBackupFileSystem; -import com.netflix.priam.backupv2.TestBackupUtils; import com.netflix.priam.config.IConfiguration; import com.netflix.priam.utils.BackupFileUtils; import com.netflix.priam.utils.DateUtil; diff --git a/priam/src/test/java/com/netflix/priam/services/TestBackupVerificationService.java b/priam/src/test/java/com/netflix/priam/backupv2/TestBackupVerificationService.java similarity index 98% rename from priam/src/test/java/com/netflix/priam/services/TestBackupVerificationService.java rename to priam/src/test/java/com/netflix/priam/backupv2/TestBackupVerificationService.java index 06ad75d43..385f133d0 100644 --- a/priam/src/test/java/com/netflix/priam/services/TestBackupVerificationService.java +++ b/priam/src/test/java/com/netflix/priam/backupv2/TestBackupVerificationService.java @@ -15,7 +15,7 @@ * */ -package com.netflix.priam.services; +package com.netflix.priam.backupv2; import com.google.inject.Guice; import com.google.inject.Injector; diff --git a/priam/src/test/java/com/netflix/priam/services/TestSnapshotMetaService.java b/priam/src/test/java/com/netflix/priam/backupv2/TestSnapshotMetaService.java similarity index 98% rename from priam/src/test/java/com/netflix/priam/services/TestSnapshotMetaService.java rename to priam/src/test/java/com/netflix/priam/backupv2/TestSnapshotMetaService.java index b65480f17..87dc92b37 100644 --- a/priam/src/test/java/com/netflix/priam/services/TestSnapshotMetaService.java +++ b/priam/src/test/java/com/netflix/priam/backupv2/TestSnapshotMetaService.java @@ -1,5 +1,5 @@ /* - * Copyright 2018 Netflix, Inc. + * Copyright 2019 Netflix, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,13 +14,12 @@ * limitations under the License. * */ -package com.netflix.priam.services; +package com.netflix.priam.backupv2; import com.google.inject.Guice; import com.google.inject.Injector; import com.netflix.priam.backup.AbstractBackup; import com.netflix.priam.backup.BRTestModule; -import com.netflix.priam.backupv2.*; import com.netflix.priam.config.IBackupRestoreConfig; import com.netflix.priam.config.IConfiguration; import com.netflix.priam.identity.config.InstanceInfo; diff --git a/priam/src/test/java/com/netflix/priam/services/TestCassandraMonitor.java b/priam/src/test/java/com/netflix/priam/health/TestCassandraMonitor.java similarity index 98% rename from priam/src/test/java/com/netflix/priam/services/TestCassandraMonitor.java rename to priam/src/test/java/com/netflix/priam/health/TestCassandraMonitor.java index 3648a461c..55fc76f64 100644 --- a/priam/src/test/java/com/netflix/priam/services/TestCassandraMonitor.java +++ b/priam/src/test/java/com/netflix/priam/health/TestCassandraMonitor.java @@ -15,7 +15,7 @@ * */ -package com.netflix.priam.services; +package com.netflix.priam.health; import com.google.inject.Guice; import com.google.inject.Injector; @@ -23,7 +23,6 @@ import com.netflix.priam.config.IConfiguration; import com.netflix.priam.connection.JMXNodeTool; import com.netflix.priam.defaultimpl.ICassandraProcess; -import com.netflix.priam.health.InstanceState; import com.netflix.priam.merics.CassMonitorMetrics; import java.io.ByteArrayInputStream; import java.io.InputStream; From df01efb8881d6765a4dca3d344e3906f31256977 Mon Sep 17 00:00:00 2001 From: Arun Agrawal Date: Sat, 9 Mar 2019 18:16:43 -0800 Subject: [PATCH 070/228] move cassandra operations to connection --- .../netflix/priam/backup/SnapshotBackup.java | 2 +- .../priam/backupv2/SnapshotMetaService.java | 2 +- .../priam/cluster/management/Compaction.java | 2 +- .../priam/cluster/management/Flush.java | 2 +- .../CassandraOperations.java | 22 ++++++++++--------- .../ICassandraOperations.java | 2 +- .../cluser/management/TestCompaction.groovy | 4 ++-- .../cluser/management/TestFlushTask.groovy | 7 ------ 8 files changed, 19 insertions(+), 24 deletions(-) rename priam/src/main/java/com/netflix/priam/{defaultimpl => connection}/CassandraOperations.java (88%) rename priam/src/main/java/com/netflix/priam/{defaultimpl => connection}/ICassandraOperations.java (98%) diff --git a/priam/src/main/java/com/netflix/priam/backup/SnapshotBackup.java b/priam/src/main/java/com/netflix/priam/backup/SnapshotBackup.java index 7a547912e..46b21ab19 100644 --- a/priam/src/main/java/com/netflix/priam/backup/SnapshotBackup.java +++ b/priam/src/main/java/com/netflix/priam/backup/SnapshotBackup.java @@ -23,7 +23,7 @@ import com.netflix.priam.backup.AbstractBackupPath.BackupFileType; import com.netflix.priam.backupv2.ForgottenFilesManager; import com.netflix.priam.config.IConfiguration; -import com.netflix.priam.defaultimpl.CassandraOperations; +import com.netflix.priam.connection.CassandraOperations; import com.netflix.priam.health.CassandraMonitor; import com.netflix.priam.identity.InstanceIdentity; import com.netflix.priam.scheduler.CronTimer; diff --git a/priam/src/main/java/com/netflix/priam/backupv2/SnapshotMetaService.java b/priam/src/main/java/com/netflix/priam/backupv2/SnapshotMetaService.java index 1d6cfadf7..0f2794474 100644 --- a/priam/src/main/java/com/netflix/priam/backupv2/SnapshotMetaService.java +++ b/priam/src/main/java/com/netflix/priam/backupv2/SnapshotMetaService.java @@ -21,7 +21,7 @@ import com.netflix.priam.backup.BackupVersion; import com.netflix.priam.config.IBackupRestoreConfig; import com.netflix.priam.config.IConfiguration; -import com.netflix.priam.defaultimpl.CassandraOperations; +import com.netflix.priam.connection.CassandraOperations; import com.netflix.priam.health.CassandraMonitor; import com.netflix.priam.identity.InstanceIdentity; import com.netflix.priam.scheduler.CronTimer; diff --git a/priam/src/main/java/com/netflix/priam/cluster/management/Compaction.java b/priam/src/main/java/com/netflix/priam/cluster/management/Compaction.java index 73ea3e257..f2ebb0bf3 100644 --- a/priam/src/main/java/com/netflix/priam/cluster/management/Compaction.java +++ b/priam/src/main/java/com/netflix/priam/cluster/management/Compaction.java @@ -15,7 +15,7 @@ import com.netflix.priam.backup.BackupRestoreUtil; import com.netflix.priam.config.IConfiguration; -import com.netflix.priam.defaultimpl.CassandraOperations; +import com.netflix.priam.connection.CassandraOperations; import com.netflix.priam.merics.CompactionMeasurement; import com.netflix.priam.scheduler.CronTimer; import com.netflix.priam.scheduler.TaskTimer; diff --git a/priam/src/main/java/com/netflix/priam/cluster/management/Flush.java b/priam/src/main/java/com/netflix/priam/cluster/management/Flush.java index 3f53cec6c..e167eb8eb 100644 --- a/priam/src/main/java/com/netflix/priam/cluster/management/Flush.java +++ b/priam/src/main/java/com/netflix/priam/cluster/management/Flush.java @@ -14,7 +14,7 @@ package com.netflix.priam.cluster.management; import com.netflix.priam.config.IConfiguration; -import com.netflix.priam.defaultimpl.CassandraOperations; +import com.netflix.priam.connection.CassandraOperations; import com.netflix.priam.merics.NodeToolFlushMeasurement; import com.netflix.priam.scheduler.CronTimer; import com.netflix.priam.scheduler.TaskTimer; diff --git a/priam/src/main/java/com/netflix/priam/defaultimpl/CassandraOperations.java b/priam/src/main/java/com/netflix/priam/connection/CassandraOperations.java similarity index 88% rename from priam/src/main/java/com/netflix/priam/defaultimpl/CassandraOperations.java rename to priam/src/main/java/com/netflix/priam/connection/CassandraOperations.java index 20a268284..d99129c79 100644 --- a/priam/src/main/java/com/netflix/priam/defaultimpl/CassandraOperations.java +++ b/priam/src/main/java/com/netflix/priam/connection/CassandraOperations.java @@ -1,20 +1,22 @@ -/** - * Copyright 2018 Netflix, Inc. +/* + * Copyright 2019 Netflix, Inc. * - *

    Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file - * except in compliance with the License. You may obtain a copy of the License at + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - *

    http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * - *

    Unless required by applicable law or agreed to in writing, software distributed under the - * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing permissions and + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and * limitations under the License. + * */ -package com.netflix.priam.defaultimpl; +package com.netflix.priam.connection; import com.netflix.priam.config.IConfiguration; -import com.netflix.priam.connection.JMXNodeTool; import com.netflix.priam.utils.RetryableCallable; import java.util.*; import javax.inject.Inject; diff --git a/priam/src/main/java/com/netflix/priam/defaultimpl/ICassandraOperations.java b/priam/src/main/java/com/netflix/priam/connection/ICassandraOperations.java similarity index 98% rename from priam/src/main/java/com/netflix/priam/defaultimpl/ICassandraOperations.java rename to priam/src/main/java/com/netflix/priam/connection/ICassandraOperations.java index c3e32de31..971e154ef 100644 --- a/priam/src/main/java/com/netflix/priam/defaultimpl/ICassandraOperations.java +++ b/priam/src/main/java/com/netflix/priam/connection/ICassandraOperations.java @@ -15,7 +15,7 @@ * */ -package com.netflix.priam.defaultimpl; +package com.netflix.priam.connection; import java.util.List; import java.util.Map; diff --git a/priam/src/test/groovy/com/netflix/priam/cluser/management/TestCompaction.groovy b/priam/src/test/groovy/com/netflix/priam/cluser/management/TestCompaction.groovy index 2de951228..90fe7db80 100644 --- a/priam/src/test/groovy/com/netflix/priam/cluser/management/TestCompaction.groovy +++ b/priam/src/test/groovy/com/netflix/priam/cluser/management/TestCompaction.groovy @@ -17,10 +17,10 @@ package com.netflix.priam.cluser.management import com.google.inject.Guice -import com.netflix.priam.config.FakeConfiguration import com.netflix.priam.backup.BRTestModule import com.netflix.priam.cluster.management.Compaction -import com.netflix.priam.defaultimpl.CassandraOperations +import com.netflix.priam.config.FakeConfiguration +import com.netflix.priam.connection.CassandraOperations import com.netflix.priam.health.CassandraMonitor import mockit.Mock import mockit.MockUp diff --git a/priam/src/test/groovy/com/netflix/priam/cluser/management/TestFlushTask.groovy b/priam/src/test/groovy/com/netflix/priam/cluser/management/TestFlushTask.groovy index 85a6ec12d..2fa8766fb 100644 --- a/priam/src/test/groovy/com/netflix/priam/cluser/management/TestFlushTask.groovy +++ b/priam/src/test/groovy/com/netflix/priam/cluser/management/TestFlushTask.groovy @@ -17,15 +17,8 @@ package com.netflix.priam.cluser.management -import com.google.inject.Guice -import com.netflix.priam.backup.BRTestModule -import com.netflix.priam.cluster.management.Compaction import com.netflix.priam.cluster.management.Flush import com.netflix.priam.config.FakeConfiguration -import com.netflix.priam.defaultimpl.CassandraOperations -import mockit.Mock -import mockit.MockUp -import spock.lang.Shared import spock.lang.Specification import spock.lang.Unroll From ea598908e1c0ffb5a0d2f40bddedc661239842da Mon Sep 17 00:00:00 2001 From: Arun Agrawal Date: Sun, 10 Mar 2019 17:23:19 -0700 Subject: [PATCH 071/228] Move to service module for Backups. --- .../java/com/netflix/priam/PriamServer.java | 115 ++--------------- .../netflix/priam/backup/BackupService.java | 81 ++++++++++++ ...ckupTTLService.java => BackupTTLTask.java} | 6 +- .../priam/backupv2/BackupV2Service.java | 104 ++++++++++++++++ ...rvice.java => BackupVerificationTask.java} | 6 +- ...MetaService.java => SnapshotMetaTask.java} | 8 +- .../IService.java} | 13 +- .../priam/resources/BackupServletV2.java | 12 +- .../com/netflix/priam/scheduler/Task.java | 17 +-- .../priam/backup/TestBackupService.java | 92 ++++++++++++++ ...TTLService.java => TestBackupTTLTask.java} | 9 +- .../priam/backupv2/TestBackupV2Service.java | 116 ++++++++++++++++++ ...e.java => TestBackupVerificationTask.java} | 8 +- ...Service.java => TestSnapshotMetaTask.java} | 41 +++---- .../priam/scheduler/TestScheduler.java | 5 +- 15 files changed, 455 insertions(+), 178 deletions(-) create mode 100644 priam/src/main/java/com/netflix/priam/backup/BackupService.java rename priam/src/main/java/com/netflix/priam/backupv2/{BackupTTLService.java => BackupTTLTask.java} (99%) create mode 100644 priam/src/main/java/com/netflix/priam/backupv2/BackupV2Service.java rename priam/src/main/java/com/netflix/priam/backupv2/{BackupVerificationService.java => BackupVerificationTask.java} (96%) rename priam/src/main/java/com/netflix/priam/backupv2/{SnapshotMetaService.java => SnapshotMetaTask.java} (99%) rename priam/src/main/java/com/netflix/priam/{scheduler/TaskMBean.java => defaultimpl/IService.java} (72%) create mode 100644 priam/src/test/java/com/netflix/priam/backup/TestBackupService.java rename priam/src/test/java/com/netflix/priam/backupv2/{TestBackupTTLService.java => TestBackupTTLTask.java} (97%) create mode 100644 priam/src/test/java/com/netflix/priam/backupv2/TestBackupV2Service.java rename priam/src/test/java/com/netflix/priam/backupv2/{TestBackupVerificationService.java => TestBackupVerificationTask.java} (94%) rename priam/src/test/java/com/netflix/priam/backupv2/{TestSnapshotMetaService.java => TestSnapshotMetaTask.java} (79%) diff --git a/priam/src/main/java/com/netflix/priam/PriamServer.java b/priam/src/main/java/com/netflix/priam/PriamServer.java index e3faa8a32..b8ef63aff 100644 --- a/priam/src/main/java/com/netflix/priam/PriamServer.java +++ b/priam/src/main/java/com/netflix/priam/PriamServer.java @@ -18,21 +18,16 @@ import com.google.inject.Inject; import com.google.inject.Singleton; -import com.netflix.priam.aws.UpdateCleanupPolicy; import com.netflix.priam.aws.UpdateSecuritySettings; -import com.netflix.priam.backup.CommitLogBackupTask; -import com.netflix.priam.backup.IncrementalBackup; -import com.netflix.priam.backup.SnapshotBackup; -import com.netflix.priam.backupv2.BackupTTLService; -import com.netflix.priam.backupv2.BackupVerificationService; -import com.netflix.priam.backupv2.SnapshotMetaService; +import com.netflix.priam.backup.BackupService; +import com.netflix.priam.backupv2.BackupV2Service; import com.netflix.priam.cluster.management.Compaction; import com.netflix.priam.cluster.management.Flush; import com.netflix.priam.cluster.management.IClusterManagement; -import com.netflix.priam.config.IBackupRestoreConfig; import com.netflix.priam.config.IConfiguration; import com.netflix.priam.config.PriamConfigurationPersister; import com.netflix.priam.defaultimpl.ICassandraProcess; +import com.netflix.priam.defaultimpl.IService; import com.netflix.priam.health.CassandraMonitor; import com.netflix.priam.identity.InstanceIdentity; import com.netflix.priam.restore.RestoreContext; @@ -42,7 +37,6 @@ import com.netflix.priam.utils.Sleeper; import com.netflix.priam.utils.SystemUtils; import java.io.IOException; -import org.apache.commons.collections4.CollectionUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -51,33 +45,33 @@ public class PriamServer { private final PriamScheduler scheduler; private final IConfiguration config; - private final IBackupRestoreConfig backupRestoreConfig; private final InstanceIdentity instanceIdentity; private final Sleeper sleeper; private final ICassandraProcess cassProcess; private final RestoreContext restoreContext; - private final SnapshotMetaService snapshotMetaService; + private final IService backupV2Service; + private final IService backupService; private static final int CASSANDRA_MONITORING_INITIAL_DELAY = 10; private static final Logger logger = LoggerFactory.getLogger(PriamServer.class); @Inject public PriamServer( IConfiguration config, - IBackupRestoreConfig backupRestoreConfig, PriamScheduler scheduler, InstanceIdentity id, Sleeper sleeper, ICassandraProcess cassProcess, RestoreContext restoreContext, - SnapshotMetaService snapshotMetaService) { + BackupService backupService, + BackupV2Service backupV2Service) { this.config = config; - this.backupRestoreConfig = backupRestoreConfig; this.scheduler = scheduler; this.instanceIdentity = id; this.sleeper = sleeper; this.cassProcess = cassProcess; this.restoreContext = restoreContext; - this.snapshotMetaService = snapshotMetaService; + this.backupService = backupService; + this.backupV2Service = backupV2Service; } private void createDirectories() throws IOException { @@ -117,32 +111,6 @@ public void initialize() throws Exception { // Run the task to tune Cassandra scheduler.runTaskNow(TuneCassandra.class); - // Start the snapshot backup schedule - Always run this. (If you want to - // set it off, set backup hour to -1) or set backup cron to "-1" - if (SnapshotBackup.getTimer(config) != null - && (CollectionUtils.isEmpty(config.getBackupRacs()) - || config.getBackupRacs() - .contains(instanceIdentity.getInstanceInfo().getRac()))) { - scheduler.addTask( - SnapshotBackup.JOBNAME, SnapshotBackup.class, SnapshotBackup.getTimer(config)); - - // Start the Incremental backup schedule if enabled - if (config.isIncrementalBackupEnabled()) { - scheduler.addTask( - IncrementalBackup.JOBNAME, - IncrementalBackup.class, - IncrementalBackup.getTimer()); - logger.info("Added incremental backup job"); - } - } - - if (config.isBackingUpCommitLogs()) { - scheduler.addTask( - CommitLogBackupTask.JOBNAME, - CommitLogBackupTask.class, - CommitLogBackupTask.getTimer(config)); - } - // Determine if we need to restore from backup else start cassandra. if (restoreContext.isRestoreEnabled()) { restoreContext.restore(); @@ -165,12 +133,6 @@ public void initialize() throws Exception { CassandraMonitor.getTimer(), CASSANDRA_MONITORING_INITIAL_DELAY); - // Set cleanup - scheduler.addTask( - UpdateCleanupPolicy.JOBNAME, - UpdateCleanupPolicy.class, - UpdateCleanupPolicy.getTimer()); - // Set up nodetool flush task TaskTimer flushTaskTimer = Flush.getTimer(config); if (flushTaskTimer != null) { @@ -204,62 +166,11 @@ public void initialize() throws Exception { logger.warn("Priam configuration persister disabled!"); } - // Set up the SnapshotService - setUpSnapshotService(); - } - - private void setUpSnapshotService() throws Exception { - TaskTimer snapshotMetaServiceTimer = SnapshotMetaService.getTimer(backupRestoreConfig); - if (snapshotMetaServiceTimer != null) { - scheduler.addTask( - SnapshotMetaService.JOBNAME, - SnapshotMetaService.class, - snapshotMetaServiceTimer); - logger.info( - "Added SnapshotMetaService Task with schedule: [{}]", - snapshotMetaServiceTimer.getCronExpression()); + // Set up V1 Snapshot Service + backupService.scheduleService(); - // Try to upload previous snapshots, if any which might have been interrupted by Priam - // restart. - snapshotMetaService.uploadFiles(); - - // Schedule the TTL service - TaskTimer backupTTLTimer = BackupTTLService.getTimer(backupRestoreConfig); - if (backupTTLTimer != null) { - scheduler.addTask(BackupTTLService.JOBNAME, BackupTTLService.class, backupTTLTimer); - logger.info( - "Added {} Task with schedule: [{}]", - BackupTTLService.JOBNAME, - backupTTLTimer.getCronExpression()); - } - - // Schedule the backup verification service - TaskTimer backupVerificationTimer = - BackupVerificationService.getTimer(backupRestoreConfig); - if (backupVerificationTimer != null) { - scheduler.addTask( - BackupVerificationService.JOBNAME, - BackupVerificationService.class, - backupVerificationTimer); - logger.info( - "Added {} Task with schedule: [{}]", - BackupVerificationService.JOBNAME, - backupVerificationTimer.getCronExpression()); - } - - // Start the Incremental backup schedule if enabled - if (config.isIncrementalBackupEnabled() && backupRestoreConfig.enableV2Backups()) { - // Delete the old task, if scheduled. This is required, as we stop taking backup - // 1.0, we still want to take incremental backups - // Once backup 1.0 is gone, we should not check for enableV2Backups.. - scheduler.deleteTask(IncrementalBackup.JOBNAME); - scheduler.addTask( - IncrementalBackup.JOBNAME, - IncrementalBackup.class, - IncrementalBackup.getTimer()); - logger.info("Added incremental backup job"); - } - } + // Set up V2 Snapshot Service + backupV2Service.scheduleService(); } public InstanceIdentity getInstanceIdentity() { diff --git a/priam/src/main/java/com/netflix/priam/backup/BackupService.java b/priam/src/main/java/com/netflix/priam/backup/BackupService.java new file mode 100644 index 000000000..25aba98c7 --- /dev/null +++ b/priam/src/main/java/com/netflix/priam/backup/BackupService.java @@ -0,0 +1,81 @@ +/* + * Copyright 2019 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.netflix.priam.backup; + +import com.google.inject.Inject; +import com.netflix.priam.aws.UpdateCleanupPolicy; +import com.netflix.priam.config.IConfiguration; +import com.netflix.priam.defaultimpl.IService; +import com.netflix.priam.identity.InstanceIdentity; +import com.netflix.priam.scheduler.PriamScheduler; +import org.apache.commons.collections4.CollectionUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** Created by aagrawal on 3/9/19. */ +public class BackupService implements IService { + private final PriamScheduler scheduler; + private final IConfiguration config; + private final InstanceIdentity instanceIdentity; + private static final Logger logger = LoggerFactory.getLogger(BackupService.class); + + @Inject + public BackupService( + IConfiguration config, + PriamScheduler priamScheduler, + InstanceIdentity instanceIdentity) { + this.config = config; + this.scheduler = priamScheduler; + this.instanceIdentity = instanceIdentity; + } + + @Override + public void scheduleService() throws Exception { + // Start the snapshot backup schedule - Always run this. (If you want to + // set it off, set backup hour to -1) or set backup cron to "-1" + if (SnapshotBackup.getTimer(config) != null + && (CollectionUtils.isEmpty(config.getBackupRacs()) + || config.getBackupRacs() + .contains(instanceIdentity.getInstanceInfo().getRac()))) { + scheduler.addTask( + SnapshotBackup.JOBNAME, SnapshotBackup.class, SnapshotBackup.getTimer(config)); + + // Start the Incremental backup schedule if enabled + if (config.isIncrementalBackupEnabled()) { + scheduler.addTask( + IncrementalBackup.JOBNAME, + IncrementalBackup.class, + IncrementalBackup.getTimer()); + logger.info("Added incremental backup job"); + } + } + + if (config.isBackingUpCommitLogs()) { + scheduler.addTask( + CommitLogBackupTask.JOBNAME, + CommitLogBackupTask.class, + CommitLogBackupTask.getTimer(config)); + } + + // Set cleanup + scheduler.addTask( + UpdateCleanupPolicy.JOBNAME, + UpdateCleanupPolicy.class, + UpdateCleanupPolicy.getTimer()); + } +} diff --git a/priam/src/main/java/com/netflix/priam/backupv2/BackupTTLService.java b/priam/src/main/java/com/netflix/priam/backupv2/BackupTTLTask.java similarity index 99% rename from priam/src/main/java/com/netflix/priam/backupv2/BackupTTLService.java rename to priam/src/main/java/com/netflix/priam/backupv2/BackupTTLTask.java index 22f2c7b13..a3fa7caca 100644 --- a/priam/src/main/java/com/netflix/priam/backupv2/BackupTTLService.java +++ b/priam/src/main/java/com/netflix/priam/backupv2/BackupTTLTask.java @@ -54,8 +54,8 @@ * safely deleted. Created by aagrawal on 11/26/18. */ @Singleton -public class BackupTTLService extends Task { - private static final Logger logger = LoggerFactory.getLogger(BackupTTLService.class); +public class BackupTTLTask extends Task { + private static final Logger logger = LoggerFactory.getLogger(BackupTTLTask.class); private IBackupRestoreConfig backupRestoreConfig; private IMetaProxy metaProxy; private IBackupFileSystem fileSystem; @@ -68,7 +68,7 @@ public class BackupTTLService extends Task { private final Instant start_of_feature = DateUtil.parseInstant("201801010000"); @Inject - public BackupTTLService( + public BackupTTLTask( IConfiguration configuration, IBackupRestoreConfig backupRestoreConfig, @Named("v2") IMetaProxy metaProxy, diff --git a/priam/src/main/java/com/netflix/priam/backupv2/BackupV2Service.java b/priam/src/main/java/com/netflix/priam/backupv2/BackupV2Service.java new file mode 100644 index 000000000..f2e8df1dc --- /dev/null +++ b/priam/src/main/java/com/netflix/priam/backupv2/BackupV2Service.java @@ -0,0 +1,104 @@ +/* + * Copyright 2019 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.netflix.priam.backupv2; + +import com.google.inject.Inject; +import com.netflix.priam.backup.IncrementalBackup; +import com.netflix.priam.config.IBackupRestoreConfig; +import com.netflix.priam.config.IConfiguration; +import com.netflix.priam.defaultimpl.IService; +import com.netflix.priam.scheduler.PriamScheduler; +import com.netflix.priam.scheduler.TaskTimer; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** Created by aagrawal on 3/9/19. */ +public class BackupV2Service implements IService { + private final PriamScheduler scheduler; + private final IConfiguration configuration; + private final IBackupRestoreConfig backupRestoreConfig; + private final SnapshotMetaTask snapshotMetaTask; + private static final Logger logger = LoggerFactory.getLogger(BackupV2Service.class); + + @Inject + public BackupV2Service( + IConfiguration configuration, + IBackupRestoreConfig backupRestoreConfig, + PriamScheduler scheduler, + SnapshotMetaTask snapshotMetaService) { + this.configuration = configuration; + this.backupRestoreConfig = backupRestoreConfig; + this.scheduler = scheduler; + this.snapshotMetaTask = snapshotMetaService; + } + + @Override + public void scheduleService() throws Exception { + TaskTimer snapshotMetaServiceTimer = SnapshotMetaTask.getTimer(backupRestoreConfig); + if (snapshotMetaServiceTimer != null) { + scheduler.addTask( + SnapshotMetaTask.JOBNAME, SnapshotMetaTask.class, snapshotMetaServiceTimer); + logger.info( + "Added {} Task with schedule: [{}]", + SnapshotMetaTask.JOBNAME, + snapshotMetaServiceTimer.getCronExpression()); + + // Try to upload previous snapshots, if any which might have been interrupted by Priam + // restart. + snapshotMetaTask.uploadFiles(); + + // Schedule the TTL service + TaskTimer backupTTLTimer = BackupTTLTask.getTimer(backupRestoreConfig); + if (backupTTLTimer != null) { + scheduler.addTask(BackupTTLTask.JOBNAME, BackupTTLTask.class, backupTTLTimer); + logger.info( + "Added {} Task with schedule: [{}]", + BackupTTLTask.JOBNAME, + backupTTLTimer.getCronExpression()); + } + + // Schedule the backup verification service + TaskTimer backupVerificationTimer = + BackupVerificationTask.getTimer(backupRestoreConfig); + if (backupVerificationTimer != null) { + scheduler.addTask( + BackupVerificationTask.JOBNAME, + BackupVerificationTask.class, + backupVerificationTimer); + logger.info( + "Added {} Task with schedule: [{}]", + BackupVerificationTask.JOBNAME, + backupVerificationTimer.getCronExpression()); + } + + // Start the Incremental backup schedule if enabled + if (configuration.isIncrementalBackupEnabled() + && backupRestoreConfig.enableV2Backups()) { + // Delete the old task, if scheduled. This is required, as we stop taking backup + // 1.0, we still want to take incremental backups + // Once backup 1.0 is gone, we should not check for enableV2Backups.. + scheduler.deleteTask(IncrementalBackup.JOBNAME); + scheduler.addTask( + IncrementalBackup.JOBNAME, + IncrementalBackup.class, + IncrementalBackup.getTimer()); + logger.info("Added incremental backup job"); + } + } + } +} diff --git a/priam/src/main/java/com/netflix/priam/backupv2/BackupVerificationService.java b/priam/src/main/java/com/netflix/priam/backupv2/BackupVerificationTask.java similarity index 96% rename from priam/src/main/java/com/netflix/priam/backupv2/BackupVerificationService.java rename to priam/src/main/java/com/netflix/priam/backupv2/BackupVerificationTask.java index 9065838e1..64a62d221 100644 --- a/priam/src/main/java/com/netflix/priam/backupv2/BackupVerificationService.java +++ b/priam/src/main/java/com/netflix/priam/backupv2/BackupVerificationTask.java @@ -36,15 +36,15 @@ /** Created by aagrawal on 1/28/19. */ @Singleton -public class BackupVerificationService extends Task { - private static final Logger logger = LoggerFactory.getLogger(BackupVerificationService.class); +public class BackupVerificationTask extends Task { + private static final Logger logger = LoggerFactory.getLogger(BackupVerificationTask.class); public static final String JOBNAME = "BackupVerificationService"; private IBackupRestoreConfig backupRestoreConfig; private BackupVerification backupVerification; @Inject - public BackupVerificationService( + public BackupVerificationTask( IConfiguration configuration, IBackupRestoreConfig backupRestoreConfig, BackupVerification backupVerification) { diff --git a/priam/src/main/java/com/netflix/priam/backupv2/SnapshotMetaService.java b/priam/src/main/java/com/netflix/priam/backupv2/SnapshotMetaTask.java similarity index 99% rename from priam/src/main/java/com/netflix/priam/backupv2/SnapshotMetaService.java rename to priam/src/main/java/com/netflix/priam/backupv2/SnapshotMetaTask.java index 0f2794474..758e72ce5 100644 --- a/priam/src/main/java/com/netflix/priam/backupv2/SnapshotMetaService.java +++ b/priam/src/main/java/com/netflix/priam/backupv2/SnapshotMetaTask.java @@ -58,10 +58,10 @@ * us in "resuming" any failed backup for any reason). Created by aagrawal on 6/18/18. */ @Singleton -public class SnapshotMetaService extends AbstractBackup { +public class SnapshotMetaTask extends AbstractBackup { public static final String JOBNAME = "SnapshotMetaService"; - private static final Logger logger = LoggerFactory.getLogger(SnapshotMetaService.class); + private static final Logger logger = LoggerFactory.getLogger(SnapshotMetaTask.class); private static final String SNAPSHOT_PREFIX = "snap_v2_"; private static final String CASSANDRA_MANIFEST_FILE = "manifest.json"; private static final String CASSANDRA_SCHEMA_FILE = "schema.cql"; @@ -83,7 +83,7 @@ private enum MetaStep { private MetaStep metaStep = MetaStep.META_GENERATION; @Inject - SnapshotMetaService( + SnapshotMetaTask( IConfiguration config, IFileSystemContext backupFileSystemCtx, Provider pathFactory, @@ -104,7 +104,7 @@ private enum MetaStep { } /** - * Interval between generating snapshot meta file using {@link SnapshotMetaService}. + * Interval between generating snapshot meta file using {@link SnapshotMetaTask}. * * @param backupRestoreConfig {@link * IBackupRestoreConfig#getSnapshotMetaServiceCronExpression()} to get configuration details diff --git a/priam/src/main/java/com/netflix/priam/scheduler/TaskMBean.java b/priam/src/main/java/com/netflix/priam/defaultimpl/IService.java similarity index 72% rename from priam/src/main/java/com/netflix/priam/scheduler/TaskMBean.java rename to priam/src/main/java/com/netflix/priam/defaultimpl/IService.java index 471e88051..6a9c9f4d8 100644 --- a/priam/src/main/java/com/netflix/priam/scheduler/TaskMBean.java +++ b/priam/src/main/java/com/netflix/priam/defaultimpl/IService.java @@ -1,5 +1,5 @@ /* - * Copyright 2013 Netflix, Inc. + * Copyright 2019 Netflix, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,13 +14,10 @@ * limitations under the License. * */ -package com.netflix.priam.scheduler; -/** MBean to monitor Task executions. */ -public interface TaskMBean { - int getErrorCount(); +package com.netflix.priam.defaultimpl; - int getExecutionCount(); - - String getName(); +/** Created by aagrawal on 3/9/19. */ +public interface IService { + void scheduleService() throws Exception; } diff --git a/priam/src/main/java/com/netflix/priam/resources/BackupServletV2.java b/priam/src/main/java/com/netflix/priam/resources/BackupServletV2.java index a42b5408e..e959a6892 100644 --- a/priam/src/main/java/com/netflix/priam/resources/BackupServletV2.java +++ b/priam/src/main/java/com/netflix/priam/resources/BackupServletV2.java @@ -23,8 +23,8 @@ import com.netflix.priam.backup.BackupVerificationResult; import com.netflix.priam.backup.BackupVersion; import com.netflix.priam.backup.IBackupStatusMgr; -import com.netflix.priam.backupv2.BackupTTLService; -import com.netflix.priam.backupv2.SnapshotMetaService; +import com.netflix.priam.backupv2.BackupTTLTask; +import com.netflix.priam.backupv2.SnapshotMetaTask; import com.netflix.priam.utils.DateUtil; import com.netflix.priam.utils.DateUtil.DateRange; import java.time.Instant; @@ -49,16 +49,16 @@ public class BackupServletV2 { private static final Logger logger = LoggerFactory.getLogger(BackupServletV2.class); private final BackupVerification backupVerification; private final IBackupStatusMgr backupStatusMgr; - private final SnapshotMetaService snapshotMetaService; - private final BackupTTLService backupTTLService; + private final SnapshotMetaTask snapshotMetaService; + private final BackupTTLTask backupTTLService; private static final String REST_SUCCESS = "[\"ok\"]"; @Inject public BackupServletV2( IBackupStatusMgr backupStatusMgr, BackupVerification backupVerification, - SnapshotMetaService snapshotMetaService, - BackupTTLService backupTTLService) { + SnapshotMetaTask snapshotMetaService, + BackupTTLTask backupTTLService) { this.backupStatusMgr = backupStatusMgr; this.backupVerification = backupVerification; this.snapshotMetaService = snapshotMetaService; diff --git a/priam/src/main/java/com/netflix/priam/scheduler/Task.java b/priam/src/main/java/com/netflix/priam/scheduler/Task.java index d371240e6..f986544f1 100644 --- a/priam/src/main/java/com/netflix/priam/scheduler/Task.java +++ b/priam/src/main/java/com/netflix/priam/scheduler/Task.java @@ -17,10 +17,7 @@ package com.netflix.priam.scheduler; import com.netflix.priam.config.IConfiguration; -import java.lang.management.ManagementFactory; import java.util.concurrent.atomic.AtomicInteger; -import javax.management.MBeanServer; -import javax.management.ObjectName; import org.quartz.Job; import org.quartz.JobExecutionContext; import org.quartz.JobExecutionException; @@ -34,7 +31,7 @@ *

    NOTE: Constructor must not throw any exception. This will cause Quartz to set the job to * failure */ -public abstract class Task implements Job, TaskMBean { +public abstract class Task implements Job { public STATE status = STATE.DONE; public enum STATE { @@ -51,19 +48,7 @@ public enum STATE { private final AtomicInteger executions = new AtomicInteger(); protected Task(IConfiguration config) { - this(config, ManagementFactory.getPlatformMBeanServer()); - } - - protected Task(IConfiguration config, MBeanServer mBeanServer) { this.config = config; - // TODO: don't do mbean registration here - String mbeanName = "com.priam.scheduler:type=" + this.getClass().getName(); - try { - mBeanServer.registerMBean(this, new ObjectName(mbeanName)); - initialize(); - } catch (Exception e) { - throw new RuntimeException(e); - } } /** This method has to be implemented and cannot throw any exception. */ diff --git a/priam/src/test/java/com/netflix/priam/backup/TestBackupService.java b/priam/src/test/java/com/netflix/priam/backup/TestBackupService.java new file mode 100644 index 000000000..a752c5205 --- /dev/null +++ b/priam/src/test/java/com/netflix/priam/backup/TestBackupService.java @@ -0,0 +1,92 @@ +/* + * Copyright 2019 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.netflix.priam.backup; + +import com.google.inject.Guice; +import com.google.inject.Injector; +import com.netflix.priam.config.IConfiguration; +import com.netflix.priam.defaultimpl.IService; +import com.netflix.priam.identity.InstanceIdentity; +import com.netflix.priam.scheduler.PriamScheduler; +import mockit.Expectations; +import mockit.Mocked; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.quartz.SchedulerException; + +/** Created by aagrawal on 3/10/19. */ +public class TestBackupService { + private final PriamScheduler scheduler; + private final InstanceIdentity instanceIdentity; + + public TestBackupService() { + Injector injector = Guice.createInjector(new BRTestModule()); + this.scheduler = injector.getInstance(PriamScheduler.class); + this.instanceIdentity = injector.getInstance(InstanceIdentity.class); + } + + @Before + public void cleanup() throws SchedulerException { + scheduler.getScheduler().clear(); + } + + @Test + public void testBackupDisabled(@Mocked IConfiguration configuration) throws Exception { + new Expectations() { + { + configuration.getBackupCronExpression(); + result = "-1"; + } + }; + IService backupService = new BackupService(configuration, scheduler, instanceIdentity); + backupService.scheduleService(); + Assert.assertEquals(1, scheduler.getScheduler().getJobKeys(null).size()); + } + + @Test + public void testBackupEnabled(@Mocked IConfiguration configuration) throws Exception { + new Expectations() { + { + configuration.getBackupCronExpression(); + result = "0 0/1 * 1/1 * ? *"; + configuration.isIncrementalBackupEnabled(); + result = false; + } + }; + IService backupService = new BackupService(configuration, scheduler, instanceIdentity); + backupService.scheduleService(); + Assert.assertEquals(2, scheduler.getScheduler().getJobKeys(null).size()); + } + + @Test + public void testBackupEnabledWithIncremental(@Mocked IConfiguration configuration) + throws Exception { + new Expectations() { + { + configuration.getBackupCronExpression(); + result = "0 0/1 * 1/1 * ? *"; + configuration.isIncrementalBackupEnabled(); + result = true; + } + }; + IService backupService = new BackupService(configuration, scheduler, instanceIdentity); + backupService.scheduleService(); + Assert.assertEquals(3, scheduler.getScheduler().getJobKeys(null).size()); + } +} diff --git a/priam/src/test/java/com/netflix/priam/backupv2/TestBackupTTLService.java b/priam/src/test/java/com/netflix/priam/backupv2/TestBackupTTLTask.java similarity index 97% rename from priam/src/test/java/com/netflix/priam/backupv2/TestBackupTTLService.java rename to priam/src/test/java/com/netflix/priam/backupv2/TestBackupTTLTask.java index 7b526ba75..a78d3636e 100644 --- a/priam/src/test/java/com/netflix/priam/backupv2/TestBackupTTLService.java +++ b/priam/src/test/java/com/netflix/priam/backupv2/TestBackupTTLTask.java @@ -41,21 +41,20 @@ import org.junit.Test; /** Created by aagrawal on 12/17/18. */ -public class TestBackupTTLService { +public class TestBackupTTLTask { private TestBackupUtils testBackupUtils = new TestBackupUtils(); private IConfiguration configuration; - private static BackupTTLService backupTTLService; + private static BackupTTLTask backupTTLService; private static FakeBackupFileSystem backupFileSystem; private Provider pathProvider; private Path[] metas; private Map allFilesMap = new HashMap<>(); - public TestBackupTTLService() { + public TestBackupTTLTask() { Injector injector = Guice.createInjector(new BRTestModule()); configuration = injector.getInstance(IConfiguration.class); - if (backupTTLService == null) - backupTTLService = injector.getInstance(BackupTTLService.class); + if (backupTTLService == null) backupTTLService = injector.getInstance(BackupTTLTask.class); if (backupFileSystem == null) backupFileSystem = injector.getInstance(FakeBackupFileSystem.class); pathProvider = injector.getProvider(AbstractBackupPath.class); diff --git a/priam/src/test/java/com/netflix/priam/backupv2/TestBackupV2Service.java b/priam/src/test/java/com/netflix/priam/backupv2/TestBackupV2Service.java new file mode 100644 index 000000000..416b2ca58 --- /dev/null +++ b/priam/src/test/java/com/netflix/priam/backupv2/TestBackupV2Service.java @@ -0,0 +1,116 @@ +/* + * Copyright 2019 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.netflix.priam.backupv2; + +import com.google.inject.Guice; +import com.google.inject.Injector; +import com.netflix.priam.backup.BRTestModule; +import com.netflix.priam.config.IBackupRestoreConfig; +import com.netflix.priam.config.IConfiguration; +import com.netflix.priam.defaultimpl.IService; +import com.netflix.priam.scheduler.PriamScheduler; +import mockit.Expectations; +import mockit.Mocked; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.quartz.SchedulerException; + +/** Created by aagrawal on 3/9/19. */ +public class TestBackupV2Service { + private final PriamScheduler scheduler; + private final SnapshotMetaTask snapshotMetaTask; + + public TestBackupV2Service() { + Injector injector = Guice.createInjector(new BRTestModule()); + scheduler = injector.getInstance(PriamScheduler.class); + snapshotMetaTask = injector.getInstance(SnapshotMetaTask.class); + } + + @Before + public void cleanup() throws SchedulerException { + scheduler.getScheduler().clear(); + } + + @Test + public void testBackupDisabled( + @Mocked IConfiguration configuration, @Mocked IBackupRestoreConfig backupRestoreConfig) + throws Exception { + new Expectations() { + { + backupRestoreConfig.getSnapshotMetaServiceCronExpression(); + result = "-1"; + } + }; + IService backupService = + new BackupV2Service( + configuration, backupRestoreConfig, scheduler, snapshotMetaTask); + backupService.scheduleService(); + Assert.assertTrue(scheduler.getScheduler().getJobGroupNames().isEmpty()); + } + + @Test + public void testBackupEnabled( + @Mocked IConfiguration configuration, @Mocked IBackupRestoreConfig backupRestoreConfig) + throws Exception { + new Expectations() { + { + backupRestoreConfig.getSnapshotMetaServiceCronExpression(); + result = "0 0 0/1 1/1 * ? *"; + backupRestoreConfig.getBackupTTLCronExpression(); + result = "0 0 0/1 1/1 * ? *"; + backupRestoreConfig.getBackupVerificationCronExpression(); + result = "0 0 0/1 1/1 * ? *"; + backupRestoreConfig.enableV2Backups(); + result = true; + configuration.isIncrementalBackupEnabled(); + result = true; + } + }; + IService backupService = + new BackupV2Service( + configuration, backupRestoreConfig, scheduler, snapshotMetaTask); + backupService.scheduleService(); + Assert.assertEquals(4, scheduler.getScheduler().getJobKeys(null).size()); + } + + @Test + public void testBackup( + @Mocked IConfiguration configuration, @Mocked IBackupRestoreConfig backupRestoreConfig) + throws Exception { + new Expectations() { + { + backupRestoreConfig.getSnapshotMetaServiceCronExpression(); + result = "0 0 0/1 1/1 * ? *"; + backupRestoreConfig.getBackupTTLCronExpression(); + result = "0 0 0/1 1/1 * ? *"; + backupRestoreConfig.getBackupVerificationCronExpression(); + result = "0 0 0/1 1/1 * ? *"; + backupRestoreConfig.enableV2Backups(); + result = false; + configuration.isIncrementalBackupEnabled(); + result = true; + } + }; + IService backupService = + new BackupV2Service( + configuration, backupRestoreConfig, scheduler, snapshotMetaTask); + backupService.scheduleService(); + Assert.assertEquals(3, scheduler.getScheduler().getJobKeys(null).size()); + } +} diff --git a/priam/src/test/java/com/netflix/priam/backupv2/TestBackupVerificationService.java b/priam/src/test/java/com/netflix/priam/backupv2/TestBackupVerificationTask.java similarity index 94% rename from priam/src/test/java/com/netflix/priam/backupv2/TestBackupVerificationService.java rename to priam/src/test/java/com/netflix/priam/backupv2/TestBackupVerificationTask.java index 385f133d0..ec3fea7a9 100644 --- a/priam/src/test/java/com/netflix/priam/backupv2/TestBackupVerificationService.java +++ b/priam/src/test/java/com/netflix/priam/backupv2/TestBackupVerificationTask.java @@ -33,16 +33,16 @@ import org.junit.Test; /** Created by aagrawal on 2/1/19. */ -public class TestBackupVerificationService { - private static BackupVerificationService backupVerificationService; +public class TestBackupVerificationTask { + private static BackupVerificationTask backupVerificationService; private static IConfiguration configuration; - public TestBackupVerificationService() { + public TestBackupVerificationTask() { new MockBackupVerification(); Injector injector = Guice.createInjector(new BRTestModule()); if (configuration == null) configuration = injector.getInstance(IConfiguration.class); if (backupVerificationService == null) - backupVerificationService = injector.getInstance(BackupVerificationService.class); + backupVerificationService = injector.getInstance(BackupVerificationTask.class); } static class MockBackupVerification extends MockUp { diff --git a/priam/src/test/java/com/netflix/priam/backupv2/TestSnapshotMetaService.java b/priam/src/test/java/com/netflix/priam/backupv2/TestSnapshotMetaTask.java similarity index 79% rename from priam/src/test/java/com/netflix/priam/backupv2/TestSnapshotMetaService.java rename to priam/src/test/java/com/netflix/priam/backupv2/TestSnapshotMetaTask.java index 87dc92b37..02f5f02d4 100644 --- a/priam/src/test/java/com/netflix/priam/backupv2/TestSnapshotMetaService.java +++ b/priam/src/test/java/com/netflix/priam/backupv2/TestSnapshotMetaTask.java @@ -36,33 +36,26 @@ import org.slf4j.LoggerFactory; /** Created by aagrawal on 6/20/18. */ -public class TestSnapshotMetaService { +public class TestSnapshotMetaTask { private static final Logger logger = - LoggerFactory.getLogger(TestSnapshotMetaService.class.getName()); + LoggerFactory.getLogger(TestSnapshotMetaTask.class.getName()); private static Path dummyDataDirectoryLocation; - private static IConfiguration configuration; - private static IBackupRestoreConfig backupRestoreConfig; - private static SnapshotMetaService snapshotMetaService; - private static TestMetaFileReader metaFileReader; - private static PrefixGenerator prefixGenerator; - private static InstanceInfo instanceInfo; - - public TestSnapshotMetaService() { + private final IConfiguration configuration; + private final IBackupRestoreConfig backupRestoreConfig; + private final SnapshotMetaTask snapshotMetaService; + private final TestMetaFileReader metaFileReader; + private final PrefixGenerator prefixGenerator; + private final InstanceInfo instanceInfo; + + public TestSnapshotMetaTask() { Injector injector = Guice.createInjector(new BRTestModule()); - if (configuration == null) configuration = injector.getInstance(IConfiguration.class); - - if (backupRestoreConfig == null) - backupRestoreConfig = injector.getInstance(IBackupRestoreConfig.class); - - if (snapshotMetaService == null) - snapshotMetaService = injector.getInstance(SnapshotMetaService.class); - - if (metaFileReader == null) metaFileReader = new TestMetaFileReader(); - - if (prefixGenerator == null) prefixGenerator = injector.getInstance(PrefixGenerator.class); - - if (instanceInfo == null) instanceInfo = injector.getInstance(InstanceInfo.class); + configuration = injector.getInstance(IConfiguration.class); + backupRestoreConfig = injector.getInstance(IBackupRestoreConfig.class); + snapshotMetaService = injector.getInstance(SnapshotMetaTask.class); + metaFileReader = new TestMetaFileReader(); + prefixGenerator = injector.getInstance(PrefixGenerator.class); + instanceInfo = injector.getInstance(InstanceInfo.class); } @Before @@ -73,7 +66,7 @@ public void setUp() { @Test public void testSnapshotMetaServiceEnabled() throws Exception { - TaskTimer taskTimer = SnapshotMetaService.getTimer(backupRestoreConfig); + TaskTimer taskTimer = SnapshotMetaTask.getTimer(backupRestoreConfig); Assert.assertNotNull(taskTimer); } diff --git a/priam/src/test/java/com/netflix/priam/scheduler/TestScheduler.java b/priam/src/test/java/com/netflix/priam/scheduler/TestScheduler.java index 305f052f2..e2da965eb 100644 --- a/priam/src/test/java/com/netflix/priam/scheduler/TestScheduler.java +++ b/priam/src/test/java/com/netflix/priam/scheduler/TestScheduler.java @@ -25,7 +25,6 @@ import com.netflix.priam.config.IConfiguration; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; -import javax.management.MBeanServerFactory; import org.junit.Assert; import org.junit.Ignore; import org.junit.Test; @@ -67,7 +66,7 @@ public static class TestTask extends Task { public TestTask(IConfiguration config) { // todo: mock the MBeanServer instead, but this will prevent exceptions due to duplicate // registrations - super(config, MBeanServerFactory.newMBeanServer()); + super(config); } @Override @@ -86,7 +85,7 @@ public String getName() { public static class SingleTestTask extends Task { @Inject public SingleTestTask(IConfiguration config) { - super(config, MBeanServerFactory.newMBeanServer()); + super(config); } public static int count = 0; From 5c362034ecb632d6e59763c476f6f84c55e3e421 Mon Sep 17 00:00:00 2001 From: Arun Agrawal Date: Sun, 10 Mar 2019 18:20:06 -0700 Subject: [PATCH 072/228] move scheduling recurring jobs to service layer --- .../java/com/netflix/priam/PriamServer.java | 40 ++++----------- .../netflix/priam/backup/BackupService.java | 26 +++------- .../priam/backup/CommitLogBackupTask.java | 4 +- .../priam/backup/IncrementalBackup.java | 5 +- .../priam/backupv2/BackupV2Service.java | 49 ++++++------------- .../netflix/priam/defaultimpl/IService.java | 14 ++++++ .../defaultimpl/InjectedWebListener.java | 2 +- .../priam/resources/BackupServlet.java | 2 +- .../priam/backupv2/TestBackupV2Service.java | 2 - 9 files changed, 51 insertions(+), 93 deletions(-) diff --git a/priam/src/main/java/com/netflix/priam/PriamServer.java b/priam/src/main/java/com/netflix/priam/PriamServer.java index b8ef63aff..29ff2d93f 100644 --- a/priam/src/main/java/com/netflix/priam/PriamServer.java +++ b/priam/src/main/java/com/netflix/priam/PriamServer.java @@ -23,7 +23,6 @@ import com.netflix.priam.backupv2.BackupV2Service; import com.netflix.priam.cluster.management.Compaction; import com.netflix.priam.cluster.management.Flush; -import com.netflix.priam.cluster.management.IClusterManagement; import com.netflix.priam.config.IConfiguration; import com.netflix.priam.config.PriamConfigurationPersister; import com.netflix.priam.defaultimpl.ICassandraProcess; @@ -32,7 +31,6 @@ import com.netflix.priam.identity.InstanceIdentity; import com.netflix.priam.restore.RestoreContext; import com.netflix.priam.scheduler.PriamScheduler; -import com.netflix.priam.scheduler.TaskTimer; import com.netflix.priam.tuner.TuneCassandra; import com.netflix.priam.utils.Sleeper; import com.netflix.priam.utils.SystemUtils; @@ -42,7 +40,7 @@ /** Start all tasks here - Property update task - Backup task - Restore task - Incremental backup */ @Singleton -public class PriamServer { +public class PriamServer implements IService { private final PriamScheduler scheduler; private final IConfiguration config; private final InstanceIdentity instanceIdentity; @@ -83,7 +81,8 @@ private void createDirectories() throws IOException { SystemUtils.createDirs(config.getHintsLocation()); } - public void initialize() throws Exception { + @Override + public void scheduleService() throws Exception { // Create all the required directories for priam and Cassandra. createDirectories(); @@ -134,37 +133,16 @@ public void initialize() throws Exception { CASSANDRA_MONITORING_INITIAL_DELAY); // Set up nodetool flush task - TaskTimer flushTaskTimer = Flush.getTimer(config); - if (flushTaskTimer != null) { - scheduler.addTask(IClusterManagement.Task.FLUSH.name(), Flush.class, flushTaskTimer); - logger.info( - "Added nodetool flush task with schedule: [{}]", - flushTaskTimer.getCronExpression()); - } + scheduleTask(scheduler, Flush.class, Flush.getTimer(config)); // Set up compaction task - TaskTimer compactionTimer = Compaction.getTimer(config); - if (compactionTimer != null) { - scheduler.addTask( - IClusterManagement.Task.COMPACTION.name(), Compaction.class, compactionTimer); - logger.info( - "Added compaction task with schedule: [{}]", - compactionTimer.getCronExpression()); - } + scheduleTask(scheduler, Compaction.class, Compaction.getTimer(config)); // Set up the background configuration dumping thread - TaskTimer configurationPersisterTimer = PriamConfigurationPersister.getTimer(config); - if (configurationPersisterTimer != null) { - scheduler.addTask( - PriamConfigurationPersister.NAME, - PriamConfigurationPersister.class, - configurationPersisterTimer); - logger.info( - "Added configuration persister task with schedule: [{}]", - configurationPersisterTimer.getCronExpression()); - } else { - logger.warn("Priam configuration persister disabled!"); - } + scheduleTask( + scheduler, + PriamConfigurationPersister.class, + PriamConfigurationPersister.getTimer(config)); // Set up V1 Snapshot Service backupService.scheduleService(); diff --git a/priam/src/main/java/com/netflix/priam/backup/BackupService.java b/priam/src/main/java/com/netflix/priam/backup/BackupService.java index 25aba98c7..983b537f7 100644 --- a/priam/src/main/java/com/netflix/priam/backup/BackupService.java +++ b/priam/src/main/java/com/netflix/priam/backup/BackupService.java @@ -52,30 +52,16 @@ public void scheduleService() throws Exception { && (CollectionUtils.isEmpty(config.getBackupRacs()) || config.getBackupRacs() .contains(instanceIdentity.getInstanceInfo().getRac()))) { - scheduler.addTask( - SnapshotBackup.JOBNAME, SnapshotBackup.class, SnapshotBackup.getTimer(config)); + scheduleTask(scheduler, SnapshotBackup.class, SnapshotBackup.getTimer(config)); - // Start the Incremental backup schedule if enabled - if (config.isIncrementalBackupEnabled()) { - scheduler.addTask( - IncrementalBackup.JOBNAME, - IncrementalBackup.class, - IncrementalBackup.getTimer()); - logger.info("Added incremental backup job"); - } + // Start the Incremental backup schedule + scheduleTask(scheduler, IncrementalBackup.class, IncrementalBackup.getTimer(config)); } - if (config.isBackingUpCommitLogs()) { - scheduler.addTask( - CommitLogBackupTask.JOBNAME, - CommitLogBackupTask.class, - CommitLogBackupTask.getTimer(config)); - } + // Schedule commit log task + scheduleTask(scheduler, CommitLogBackupTask.class, CommitLogBackupTask.getTimer(config)); // Set cleanup - scheduler.addTask( - UpdateCleanupPolicy.JOBNAME, - UpdateCleanupPolicy.class, - UpdateCleanupPolicy.getTimer()); + scheduleTask(scheduler, UpdateCleanupPolicy.class, UpdateCleanupPolicy.getTimer()); } } diff --git a/priam/src/main/java/com/netflix/priam/backup/CommitLogBackupTask.java b/priam/src/main/java/com/netflix/priam/backup/CommitLogBackupTask.java index 18ca2566c..016675a10 100644 --- a/priam/src/main/java/com/netflix/priam/backup/CommitLogBackupTask.java +++ b/priam/src/main/java/com/netflix/priam/backup/CommitLogBackupTask.java @@ -59,7 +59,9 @@ public String getName() { } public static TaskTimer getTimer(IConfiguration config) { - return new SimpleTimer(JOBNAME, 60L * 1000); // every 1 min + if (config.isBackingUpCommitLogs()) + return new SimpleTimer(JOBNAME, 60L * 1000); // every 1 min + else return null; } @Override diff --git a/priam/src/main/java/com/netflix/priam/backup/IncrementalBackup.java b/priam/src/main/java/com/netflix/priam/backup/IncrementalBackup.java index 627c303e5..f364372bb 100644 --- a/priam/src/main/java/com/netflix/priam/backup/IncrementalBackup.java +++ b/priam/src/main/java/com/netflix/priam/backup/IncrementalBackup.java @@ -65,8 +65,9 @@ public void execute() throws Exception { } /** Run every 10 Sec */ - public static TaskTimer getTimer() { - return new SimpleTimer(JOBNAME, 10L * 1000); + public static TaskTimer getTimer(IConfiguration config) { + if (config.isIncrementalBackupEnabled()) return new SimpleTimer(JOBNAME, 10L * 1000); + return null; } @Override diff --git a/priam/src/main/java/com/netflix/priam/backupv2/BackupV2Service.java b/priam/src/main/java/com/netflix/priam/backupv2/BackupV2Service.java index f2e8df1dc..7548ed6de 100644 --- a/priam/src/main/java/com/netflix/priam/backupv2/BackupV2Service.java +++ b/priam/src/main/java/com/netflix/priam/backupv2/BackupV2Service.java @@ -49,55 +49,34 @@ public BackupV2Service( @Override public void scheduleService() throws Exception { - TaskTimer snapshotMetaServiceTimer = SnapshotMetaTask.getTimer(backupRestoreConfig); - if (snapshotMetaServiceTimer != null) { - scheduler.addTask( - SnapshotMetaTask.JOBNAME, SnapshotMetaTask.class, snapshotMetaServiceTimer); - logger.info( - "Added {} Task with schedule: [{}]", - SnapshotMetaTask.JOBNAME, - snapshotMetaServiceTimer.getCronExpression()); + TaskTimer snapshotMetaTimer = SnapshotMetaTask.getTimer(backupRestoreConfig); + if (snapshotMetaTimer != null) { + scheduleTask(scheduler, SnapshotMetaTask.class, snapshotMetaTimer); // Try to upload previous snapshots, if any which might have been interrupted by Priam // restart. snapshotMetaTask.uploadFiles(); // Schedule the TTL service - TaskTimer backupTTLTimer = BackupTTLTask.getTimer(backupRestoreConfig); - if (backupTTLTimer != null) { - scheduler.addTask(BackupTTLTask.JOBNAME, BackupTTLTask.class, backupTTLTimer); - logger.info( - "Added {} Task with schedule: [{}]", - BackupTTLTask.JOBNAME, - backupTTLTimer.getCronExpression()); - } + scheduleTask( + scheduler, BackupTTLTask.class, BackupTTLTask.getTimer(backupRestoreConfig)); // Schedule the backup verification service - TaskTimer backupVerificationTimer = - BackupVerificationTask.getTimer(backupRestoreConfig); - if (backupVerificationTimer != null) { - scheduler.addTask( - BackupVerificationTask.JOBNAME, - BackupVerificationTask.class, - backupVerificationTimer); - logger.info( - "Added {} Task with schedule: [{}]", - BackupVerificationTask.JOBNAME, - backupVerificationTimer.getCronExpression()); - } + scheduleTask( + scheduler, + BackupVerificationTask.class, + BackupVerificationTask.getTimer(backupRestoreConfig)); // Start the Incremental backup schedule if enabled - if (configuration.isIncrementalBackupEnabled() - && backupRestoreConfig.enableV2Backups()) { + if (backupRestoreConfig.enableV2Backups()) { // Delete the old task, if scheduled. This is required, as we stop taking backup // 1.0, we still want to take incremental backups - // Once backup 1.0 is gone, we should not check for enableV2Backups.. + // Once backup 1.0 is gone, we should not check for enableV2Backups. scheduler.deleteTask(IncrementalBackup.JOBNAME); - scheduler.addTask( - IncrementalBackup.JOBNAME, + scheduleTask( + scheduler, IncrementalBackup.class, - IncrementalBackup.getTimer()); - logger.info("Added incremental backup job"); + IncrementalBackup.getTimer(configuration)); } } } diff --git a/priam/src/main/java/com/netflix/priam/defaultimpl/IService.java b/priam/src/main/java/com/netflix/priam/defaultimpl/IService.java index 6a9c9f4d8..c905879a8 100644 --- a/priam/src/main/java/com/netflix/priam/defaultimpl/IService.java +++ b/priam/src/main/java/com/netflix/priam/defaultimpl/IService.java @@ -17,7 +17,21 @@ package com.netflix.priam.defaultimpl; +import com.netflix.priam.scheduler.PriamScheduler; +import com.netflix.priam.scheduler.Task; +import com.netflix.priam.scheduler.TaskTimer; +import java.text.ParseException; +import org.quartz.SchedulerException; + /** Created by aagrawal on 3/9/19. */ public interface IService { void scheduleService() throws Exception; + + default void scheduleTask( + PriamScheduler priamScheduler, Class task, TaskTimer taskTimer) + throws SchedulerException, ParseException { + if (taskTimer == null) return; + + priamScheduler.addTask(task.getName(), task, taskTimer); + } } diff --git a/priam/src/main/java/com/netflix/priam/defaultimpl/InjectedWebListener.java b/priam/src/main/java/com/netflix/priam/defaultimpl/InjectedWebListener.java index b1c3ab2cd..1d7a1db57 100644 --- a/priam/src/main/java/com/netflix/priam/defaultimpl/InjectedWebListener.java +++ b/priam/src/main/java/com/netflix/priam/defaultimpl/InjectedWebListener.java @@ -49,7 +49,7 @@ protected Injector getInjector() { injector = Guice.createInjector(moduleList); try { injector.getInstance(IConfiguration.class).initialize(); - injector.getInstance(PriamServer.class).initialize(); + injector.getInstance(PriamServer.class).scheduleService(); } catch (Exception e) { logger.error(e.getMessage(), e); throw new RuntimeException(e.getMessage(), e); diff --git a/priam/src/main/java/com/netflix/priam/resources/BackupServlet.java b/priam/src/main/java/com/netflix/priam/resources/BackupServlet.java index fc1ca1798..a54296472 100644 --- a/priam/src/main/java/com/netflix/priam/resources/BackupServlet.java +++ b/priam/src/main/java/com/netflix/priam/resources/BackupServlet.java @@ -79,7 +79,7 @@ public Response backup() throws Exception { @Path("/incremental_backup") public Response backupIncrementals() throws Exception { scheduler.addTask( - "IncrementalBackup", IncrementalBackup.class, IncrementalBackup.getTimer()); + "IncrementalBackup", IncrementalBackup.class, IncrementalBackup.getTimer(config)); return Response.ok(REST_SUCCESS, MediaType.APPLICATION_JSON).build(); } diff --git a/priam/src/test/java/com/netflix/priam/backupv2/TestBackupV2Service.java b/priam/src/test/java/com/netflix/priam/backupv2/TestBackupV2Service.java index 416b2ca58..07ee874bb 100644 --- a/priam/src/test/java/com/netflix/priam/backupv2/TestBackupV2Service.java +++ b/priam/src/test/java/com/netflix/priam/backupv2/TestBackupV2Service.java @@ -103,8 +103,6 @@ public void testBackup( result = "0 0 0/1 1/1 * ? *"; backupRestoreConfig.enableV2Backups(); result = false; - configuration.isIncrementalBackupEnabled(); - result = true; } }; IService backupService = From 2d5cb230bdefe3e22bb34c93ae3c5d322a578250 Mon Sep 17 00:00:00 2001 From: Arun Agrawal Date: Fri, 8 Mar 2019 19:38:05 -0800 Subject: [PATCH 073/228] Hotfix: TTL for backup version 2. Cassandra can flush files on file system like Index.db first and other component files later (like 30 mins). If there is a snapshot in between, then this "single" component file would not be part of snapshot as SSTable is still not part of cassandra's "view". Only if Cassandra could provide strong guarantees on file system such that - 1. All component will be flushed to disk as real SSTables only if they are part of view. Until that happens all the files will be "tmp" files. 2. All component flushed will have same "last modified" file. i.e. on first flush. Stats.db can change over time and that is OK. Since, this is the not the case, the TTL may end up deleting this file even though file are part of next snapshot. To avoid, this we add grace period (based on how long a compaction can run) when we delete the files. --- .../netflix/priam/backupv2/BackupTTLTask.java | 50 ++++++++++++------- .../priam/backupv2/ForgottenFilesManager.java | 6 +-- .../netflix/priam/config/IConfiguration.java | 14 ++++-- .../priam/config/PriamConfiguration.java | 2 +- .../priam/backupv2/TestBackupTTLTask.java | 18 ++++--- 5 files changed, 55 insertions(+), 35 deletions(-) diff --git a/priam/src/main/java/com/netflix/priam/backupv2/BackupTTLTask.java b/priam/src/main/java/com/netflix/priam/backupv2/BackupTTLTask.java index a3fa7caca..4f3945220 100644 --- a/priam/src/main/java/com/netflix/priam/backupv2/BackupTTLTask.java +++ b/priam/src/main/java/com/netflix/priam/backupv2/BackupTTLTask.java @@ -102,10 +102,6 @@ public void execute() throws Exception { Instant dateToTtl = DateUtil.getInstant().minus(config.getBackupRetentionDays(), ChronoUnit.DAYS); - logger.info( - "Will try to delete(TTL) files which are before this time: {}. TTL in Days: {}", - dateToTtl.toEpochMilli(), - config.getBackupRetentionDays()); // Find the snapshot just after this date. List metas = @@ -127,15 +123,45 @@ public void execute() throws Exception { // Walk over the file system iterator and if not in map, it is eligible for delete. new MetaFileWalker().readMeta(localFile); - if (logger.isDebugEnabled()) - logger.debug("Files in meta file: {}", filesInMeta.keySet().toString()); + logger.info("No. of component files loaded from meta file: {}", filesInMeta.size()); // Delete the meta file downloaded locally FileUtils.deleteQuietly(localFile.toFile()); + // If there are no files listed in meta, do not delete. This could be a bug!! + if (filesInMeta.isEmpty()) { + logger.warn("Meta file was empty. This should not happen. Getting out!!"); + return; + } + + // Delete the old META files. We are giving start date which is so back in past to get + // all the META files. + // This feature did not exist in Jan 2018. + metas = + metaProxy.findMetaFiles( + new DateUtil.DateRange( + start_of_feature, dateToTtl.minus(1, ChronoUnit.HOURS))); + + if (metas != null) { + logger.info( + "Will delete(TTL) {} META files starting from: [{}]", + metas.size(), + metas.get(metas.size() - 1).getLastModified()); + for (AbstractBackupPath meta : metas) { + deleteFile(meta, false); + } + } + Iterator remoteFileLocations = fileSystem.listFileSystem(getSSTPrefix(), null, null); + dateToTtl = dateToTtl.minus(config.getGracePeriodDaysForCompaction(), ChronoUnit.DAYS); + logger.info( + "Will delete(TTL) SST_V2 files which are before this time: {}. Input: [TTL: {} days, Grace Period: {} days]", + dateToTtl, + config.getBackupRetentionDays(), + config.getGracePeriodDaysForCompaction()); + while (remoteFileLocations.hasNext()) { AbstractBackupPath abstractBackupPath = abstractBackupPathProvider.get(); abstractBackupPath.parseRemote(remoteFileLocations.next()); @@ -158,18 +184,6 @@ public void execute() throws Exception { } } - // Delete the old META files. We are giving start date which is so back in past to get - // all the META files. - // This feature did not exist in Jan 2018. - metas = - metaProxy.findMetaFiles( - new DateUtil.DateRange( - start_of_feature, dateToTtl.minus(1, ChronoUnit.MINUTES))); - - for (AbstractBackupPath meta : metas) { - deleteFile(meta, false); - } - // Delete remaining files. deleteFile(null, true); diff --git a/priam/src/main/java/com/netflix/priam/backupv2/ForgottenFilesManager.java b/priam/src/main/java/com/netflix/priam/backupv2/ForgottenFilesManager.java index dd5247f4b..d63ee32f9 100644 --- a/priam/src/main/java/com/netflix/priam/backupv2/ForgottenFilesManager.java +++ b/priam/src/main/java/com/netflix/priam/backupv2/ForgottenFilesManager.java @@ -109,7 +109,7 @@ protected Collection getColumnfamilyFiles(Instant snapshotInstant, File co IOFileFilter tmpFileFilter = FileFilterUtils.or(tmpFileFilter1, tmpFileFilter2); /* Here we are allowing files which were more than - @link{IConfiguration#getForgottenFileGracePeriodDaysForCompaction}. We do this to allow cassandra + @link{IConfiguration#getGracePeriodDaysForCompaction}. We do this to allow cassandra to have files which were generated as part of long running compaction. Refer to https://issues.apache.org/jira/browse/CASSANDRA-6756 and https://issues.apache.org/jira/browse/CASSANDRA-7066 @@ -118,9 +118,7 @@ protected Collection getColumnfamilyFiles(Instant snapshotInstant, File co IOFileFilter ageFilter = FileFilterUtils.ageFileFilter( snapshotInstant - .minus( - config.getForgottenFileGracePeriodDaysForCompaction(), - ChronoUnit.DAYS) + .minus(config.getGracePeriodDaysForCompaction(), ChronoUnit.DAYS) .toEpochMilli()); IOFileFilter fileFilter = FileFilterUtils.and( diff --git a/priam/src/main/java/com/netflix/priam/config/IConfiguration.java b/priam/src/main/java/com/netflix/priam/config/IConfiguration.java index 4fe300446..7c9819708 100644 --- a/priam/src/main/java/com/netflix/priam/config/IConfiguration.java +++ b/priam/src/main/java/com/netflix/priam/config/IConfiguration.java @@ -992,14 +992,18 @@ default int getPostRestoreHookHeartbeatCheckFrequencyInMs() { } /** - * Grace period in days for the file that 'could' be considered to be forgotten by cassandra, - * but actually may be output of a long-running compaction job. Note that cassandra creates - * output of the compaction as non-tmp-link files (whole SSTable) but are still not part of the - * final "view" and thus not part of a snapshot. Only required for cassandra 2.x. Default: 5 + * Grace period in days for the file that 'could' be output of a long-running compaction job. + * Note that cassandra creates output of the compaction as non-tmp-link files (whole SSTable) + * but are still not part of the final "view" and thus not part of a snapshot. Another common + * issue is "index.db" published "way" before other component files. Thus index file has + * modification time before other files . + * + *

    This value is used to TTL the backups and to consider file which are forgotten by + * Cassandra. Default: 5 * * @return grace period for the compaction output forgotten files. */ - default int getForgottenFileGracePeriodDaysForCompaction() { + default int getGracePeriodDaysForCompaction() { return 5; } diff --git a/priam/src/main/java/com/netflix/priam/config/PriamConfiguration.java b/priam/src/main/java/com/netflix/priam/config/PriamConfiguration.java index 660f3a36c..dbf0d4159 100644 --- a/priam/src/main/java/com/netflix/priam/config/PriamConfiguration.java +++ b/priam/src/main/java/com/netflix/priam/config/PriamConfiguration.java @@ -750,7 +750,7 @@ public String getMergedConfigurationCronExpression() { } @Override - public int getForgottenFileGracePeriodDaysForCompaction() { + public int getGracePeriodDaysForCompaction() { return config.get(PRIAM_PRE + ".forgottenFileGracePeriodDaysForCompaction", 5); } diff --git a/priam/src/test/java/com/netflix/priam/backupv2/TestBackupTTLTask.java b/priam/src/test/java/com/netflix/priam/backupv2/TestBackupTTLTask.java index a78d3636e..39070ed6f 100644 --- a/priam/src/test/java/com/netflix/priam/backupv2/TestBackupTTLTask.java +++ b/priam/src/test/java/com/netflix/priam/backupv2/TestBackupTTLTask.java @@ -68,7 +68,10 @@ public void prepTest(int daysForSnapshot) throws Exception { List allFiles = new ArrayList<>(); metas = new Path[3]; - Instant time = current.minus(daysForSnapshot + 1, ChronoUnit.DAYS); + Instant time = + current.minus( + daysForSnapshot + configuration.getGracePeriodDaysForCompaction() + 1, + ChronoUnit.DAYS); String file1 = testBackupUtils.createFile("mc-1-Data.db", time); String file2 = testBackupUtils.createFile("mc-2-Data.db", time.plus(10, ChronoUnit.MINUTES)); @@ -152,7 +155,7 @@ public void testTTL() throws Exception { List remoteFiles = getAllFiles(); // Confirm the files. - Assert.assertEquals(7, remoteFiles.size()); + Assert.assertEquals(8, remoteFiles.size()); Assert.assertTrue(remoteFiles.contains(allFilesMap.get("mc-4-Data.db"))); Assert.assertTrue(remoteFiles.contains(allFilesMap.get("mc-5-Data.db"))); Assert.assertTrue(remoteFiles.contains(allFilesMap.get("mc-6-Data.db"))); @@ -160,9 +163,10 @@ public void testTTL() throws Exception { Assert.assertTrue(remoteFiles.contains(allFilesMap.get("mc-1-Data.db"))); Assert.assertTrue(remoteFiles.contains(allFilesMap.get("META1"))); Assert.assertTrue(remoteFiles.contains(allFilesMap.get("META2"))); + // Remains because of GRACE PERIOD. + Assert.assertTrue(remoteFiles.contains(allFilesMap.get("mc-3-Data.db"))); Assert.assertFalse(remoteFiles.contains(allFilesMap.get("mc-2-Data.db"))); - Assert.assertFalse(remoteFiles.contains(allFilesMap.get("mc-3-Data.db"))); Assert.assertFalse(remoteFiles.contains(allFilesMap.get("META0"))); } @@ -174,18 +178,18 @@ public void testTTLNext() throws Exception { backupTTLService.execute(); List remoteFiles = getAllFiles(); - System.out.println(remoteFiles); // Confirm the files. - Assert.assertEquals(4, remoteFiles.size()); + Assert.assertEquals(6, remoteFiles.size()); Assert.assertTrue(remoteFiles.contains(allFilesMap.get("mc-4-Data.db"))); Assert.assertTrue(remoteFiles.contains(allFilesMap.get("mc-6-Data.db"))); Assert.assertTrue(remoteFiles.contains(allFilesMap.get("mc-7-Data.db"))); Assert.assertTrue(remoteFiles.contains(allFilesMap.get("META2"))); + // GRACE PERIOD files. + Assert.assertTrue(remoteFiles.contains(allFilesMap.get("mc-3-Data.db"))); + Assert.assertTrue(remoteFiles.contains(allFilesMap.get("mc-5-Data.db"))); Assert.assertFalse(remoteFiles.contains(allFilesMap.get("mc-1-Data.db"))); Assert.assertFalse(remoteFiles.contains(allFilesMap.get("mc-2-Data.db"))); - Assert.assertFalse(remoteFiles.contains(allFilesMap.get("mc-3-Data.db"))); - Assert.assertFalse(remoteFiles.contains(allFilesMap.get("mc-5-Data.db"))); Assert.assertFalse(remoteFiles.contains(allFilesMap.get("META0"))); Assert.assertFalse(remoteFiles.contains(allFilesMap.get("META1"))); } From 7706572bc244d7091022d743e7c62b32424b280d Mon Sep 17 00:00:00 2001 From: Arun Agrawal Date: Fri, 8 Mar 2019 20:41:43 -0800 Subject: [PATCH 074/228] delete the files from internal cache when doing delete operation. --- .../netflix/priam/aws/S3FileSystemBase.java | 7 +++++-- .../priam/backup/AbstractFileSystem.java | 19 ++++++++++++++++++ .../netflix/priam/backupv2/BackupTTLTask.java | 8 ++++++++ .../priam/config/PriamConfiguration.java | 2 +- .../google/GoogleEncryptedFileSystem.java | 2 +- .../priam/backup/FakeBackupFileSystem.java | 2 +- .../priam/backup/NullBackupFileSystem.java | 2 +- .../priam/backup/TestS3FileSystem.java | 20 +++++++++++++++++++ 8 files changed, 56 insertions(+), 6 deletions(-) diff --git a/priam/src/main/java/com/netflix/priam/aws/S3FileSystemBase.java b/priam/src/main/java/com/netflix/priam/aws/S3FileSystemBase.java index baf03ff44..302c8cda1 100755 --- a/priam/src/main/java/com/netflix/priam/aws/S3FileSystemBase.java +++ b/priam/src/main/java/com/netflix/priam/aws/S3FileSystemBase.java @@ -258,7 +258,7 @@ public Iterator listFileSystem(String prefix, String delimiter, String m } @Override - public void deleteRemoteFiles(List remotePaths) throws BackupRestoreException { + public void deleteFiles(List remotePaths) throws BackupRestoreException { if (remotePaths.isEmpty()) return; try { @@ -274,7 +274,10 @@ public void deleteRemoteFiles(List remotePaths) throws BackupRestoreExcept new DeleteObjectsRequest(getShard()).withKeys(keys).withQuiet(true)); logger.info("Deleted {} objects from S3", remotePaths.size()); } catch (Exception e) { - logger.error("Error while trying to delete the objects from S3: {}", e.getMessage()); + logger.error( + "Error while trying to delete [{}] the objects from S3: {}", + remotePaths.size(), + e.getMessage()); throw new BackupRestoreException(e + " while trying to delete the objects"); } } diff --git a/priam/src/main/java/com/netflix/priam/backup/AbstractFileSystem.java b/priam/src/main/java/com/netflix/priam/backup/AbstractFileSystem.java index 671f56314..8f6db6b19 100644 --- a/priam/src/main/java/com/netflix/priam/backup/AbstractFileSystem.java +++ b/priam/src/main/java/com/netflix/priam/backup/AbstractFileSystem.java @@ -37,6 +37,7 @@ import java.nio.file.Paths; import java.util.Date; import java.util.Iterator; +import java.util.List; import java.util.Set; import java.util.concurrent.*; import org.apache.commons.collections4.iterators.FilterIterator; @@ -62,6 +63,9 @@ public abstract class AbstractFileSystem implements IBackupFileSystem, EventGene private final Set tasksQueued; private final ThreadPoolExecutor fileUploadExecutor; private final ThreadPoolExecutor fileDownloadExecutor; + + // This is going to be a write-thru cache containing the most frequently used items from remote + // file system. This is to ensure that we don't make too many API calls to remote file system. private final Cache objectCache; @Inject @@ -257,6 +261,21 @@ public boolean checkObjectExists(Path remotePath) { return remoteFileExist; } + @Override + public void deleteRemoteFiles(List remotePaths) throws BackupRestoreException { + if (remotePaths == null) return; + + // Note that we are trying to implement write-thru cache here so it is good idea to + // invalidate the cache first. This is important so that if there is any issue (because file + // was deleted), it is caught by our snapshot job we can re-upload the file. This will also + // help in ensuring that our validation job fails if there are any error caused due to TTL + // of a file. + objectCache.invalidateAll(remotePaths); + deleteFiles(remotePaths); + } + + protected abstract void deleteFiles(List remotePaths) throws BackupRestoreException; + protected abstract boolean doesRemoteFileExist(Path remotePath); protected abstract long uploadFileImpl(final Path localPath, final Path remotePath) diff --git a/priam/src/main/java/com/netflix/priam/backupv2/BackupTTLTask.java b/priam/src/main/java/com/netflix/priam/backupv2/BackupTTLTask.java index 4f3945220..6c5eb6ce2 100644 --- a/priam/src/main/java/com/netflix/priam/backupv2/BackupTTLTask.java +++ b/priam/src/main/java/com/netflix/priam/backupv2/BackupTTLTask.java @@ -155,6 +155,14 @@ public void execute() throws Exception { Iterator remoteFileLocations = fileSystem.listFileSystem(getSSTPrefix(), null, null); + /* + We really cannot delete the files until the TTL period. + Cassandra can flush files on file system like Index.db first and other component files later (like 30 mins). If there is a snapshot in between, then this "single" component file would not be part of the snapshot as SSTable is still not part of Cassandra's "view". Only if Cassandra could provide strong guarantees on the file system such that - + + 1. All component will be flushed to disk as real SSTables only if they are part of the view. Until that happens all the files will be "tmp" files. + 2. All component flushed will have the same "last modified" file. i.e. on the first flush. Stats.db can change over time and that is OK. + Since this is not the case, the TTL may end up deleting this file even though the file is part of the next snapshot. To avoid, this we add grace period (based on how long compaction can run) when we delete the files. + */ dateToTtl = dateToTtl.minus(config.getGracePeriodDaysForCompaction(), ChronoUnit.DAYS); logger.info( "Will delete(TTL) SST_V2 files which are before this time: {}. Input: [TTL: {} days, Grace Period: {} days]", diff --git a/priam/src/main/java/com/netflix/priam/config/PriamConfiguration.java b/priam/src/main/java/com/netflix/priam/config/PriamConfiguration.java index dbf0d4159..83b48fd71 100644 --- a/priam/src/main/java/com/netflix/priam/config/PriamConfiguration.java +++ b/priam/src/main/java/com/netflix/priam/config/PriamConfiguration.java @@ -751,7 +751,7 @@ public String getMergedConfigurationCronExpression() { @Override public int getGracePeriodDaysForCompaction() { - return config.get(PRIAM_PRE + ".forgottenFileGracePeriodDaysForCompaction", 5); + return config.get(PRIAM_PRE + ".gracePeriodDaysForCompaction", 5); } @Override diff --git a/priam/src/main/java/com/netflix/priam/google/GoogleEncryptedFileSystem.java b/priam/src/main/java/com/netflix/priam/google/GoogleEncryptedFileSystem.java index a0e39f5a4..8a4f153e5 100755 --- a/priam/src/main/java/com/netflix/priam/google/GoogleEncryptedFileSystem.java +++ b/priam/src/main/java/com/netflix/priam/google/GoogleEncryptedFileSystem.java @@ -243,7 +243,7 @@ public long getFileSize(Path remotePath) throws BackupRestoreException { } @Override - public void deleteRemoteFiles(List remotePaths) throws BackupRestoreException { + public void deleteFiles(List remotePaths) throws BackupRestoreException { // TODO: Delete implementation } diff --git a/priam/src/test/java/com/netflix/priam/backup/FakeBackupFileSystem.java b/priam/src/test/java/com/netflix/priam/backup/FakeBackupFileSystem.java index 8591c43a8..4d9c64bc2 100644 --- a/priam/src/test/java/com/netflix/priam/backup/FakeBackupFileSystem.java +++ b/priam/src/test/java/com/netflix/priam/backup/FakeBackupFileSystem.java @@ -125,7 +125,7 @@ public boolean doesRemoteFileExist(Path remotePath) { } @Override - public void deleteRemoteFiles(List remotePaths) throws BackupRestoreException { + public void deleteFiles(List remotePaths) throws BackupRestoreException { remotePaths .stream() .forEach( diff --git a/priam/src/test/java/com/netflix/priam/backup/NullBackupFileSystem.java b/priam/src/test/java/com/netflix/priam/backup/NullBackupFileSystem.java index 4f11276a8..04377f301 100644 --- a/priam/src/test/java/com/netflix/priam/backup/NullBackupFileSystem.java +++ b/priam/src/test/java/com/netflix/priam/backup/NullBackupFileSystem.java @@ -48,7 +48,7 @@ public long getFileSize(Path remotePath) throws BackupRestoreException { } @Override - public void deleteRemoteFiles(List remotePaths) throws BackupRestoreException { + public void deleteFiles(List remotePaths) throws BackupRestoreException { // Do nothing. } diff --git a/priam/src/test/java/com/netflix/priam/backup/TestS3FileSystem.java b/priam/src/test/java/com/netflix/priam/backup/TestS3FileSystem.java index 719a38d83..9fdd4fead 100644 --- a/priam/src/test/java/com/netflix/priam/backup/TestS3FileSystem.java +++ b/priam/src/test/java/com/netflix/priam/backup/TestS3FileSystem.java @@ -114,6 +114,26 @@ public void testFileUpload() throws Exception { Assert.assertEquals(1, backupMetrics.getUploadRate().count() - noOfFilesUploaded); } + @Test + public void testFileUploadDeleteExists() throws Exception { + MockS3PartUploader.setup(); + IBackupFileSystem fs = injector.getInstance(NullBackupFileSystem.class); + RemoteBackupPath backupfile = injector.getInstance(RemoteBackupPath.class); + backupfile.parseLocal(new File(FILE_PATH), BackupFileType.SST_V2); + fs.uploadFile( + Paths.get(backupfile.getBackupFile().getAbsolutePath()), + Paths.get(backupfile.getRemotePath()), + backupfile, + 0, + false); + Assert.assertTrue(fs.checkObjectExists(Paths.get(backupfile.getRemotePath()))); + // Lets delete the file now. + List deleteFiles = Lists.newArrayList(); + deleteFiles.add(Paths.get(backupfile.getRemotePath())); + fs.deleteRemoteFiles(deleteFiles); + Assert.assertFalse(fs.checkObjectExists(Paths.get(backupfile.getRemotePath()))); + } + @Test public void testFileUploadFailures() throws Exception { MockS3PartUploader.setup(); From 7f4990352cc16ae1e551fac4789f55d2ea19f268 Mon Sep 17 00:00:00 2001 From: Arun Agrawal Date: Wed, 13 Mar 2019 13:53:02 -0700 Subject: [PATCH 075/228] increment backup verification failure metric --- .../netflix/priam/backupv2/BackupVerificationTask.java | 7 ++++++- .../java/com/netflix/priam/merics/BackupMetrics.java | 9 ++++++++- .../test/java/com/netflix/priam/restore/TestRestore.java | 1 + 3 files changed, 15 insertions(+), 2 deletions(-) diff --git a/priam/src/main/java/com/netflix/priam/backupv2/BackupVerificationTask.java b/priam/src/main/java/com/netflix/priam/backupv2/BackupVerificationTask.java index 64a62d221..8e741f704 100644 --- a/priam/src/main/java/com/netflix/priam/backupv2/BackupVerificationTask.java +++ b/priam/src/main/java/com/netflix/priam/backupv2/BackupVerificationTask.java @@ -24,6 +24,7 @@ import com.netflix.priam.backup.BackupVersion; import com.netflix.priam.config.IBackupRestoreConfig; import com.netflix.priam.config.IConfiguration; +import com.netflix.priam.merics.BackupMetrics; import com.netflix.priam.scheduler.CronTimer; import com.netflix.priam.scheduler.Task; import com.netflix.priam.scheduler.TaskTimer; @@ -42,15 +43,18 @@ public class BackupVerificationTask extends Task { private IBackupRestoreConfig backupRestoreConfig; private BackupVerification backupVerification; + private BackupMetrics backupMetrics; @Inject public BackupVerificationTask( IConfiguration configuration, IBackupRestoreConfig backupRestoreConfig, - BackupVerification backupVerification) { + BackupVerification backupVerification, + BackupMetrics backupMetrics) { super(configuration); this.backupRestoreConfig = backupRestoreConfig; this.backupVerification = backupVerification; + this.backupMetrics = backupMetrics; } @Override @@ -77,6 +81,7 @@ public void execute() throws Exception { logger.error( "Not able to find any snapshot which is valid in our SLO window: {} hours", backupRestoreConfig.getBackupVerificationSLOInHours()); + backupMetrics.incrementBackupVerificationFailure(); } } diff --git a/priam/src/main/java/com/netflix/priam/merics/BackupMetrics.java b/priam/src/main/java/com/netflix/priam/merics/BackupMetrics.java index 7f86b273e..e9b5654a3 100644 --- a/priam/src/main/java/com/netflix/priam/merics/BackupMetrics.java +++ b/priam/src/main/java/com/netflix/priam/merics/BackupMetrics.java @@ -35,7 +35,8 @@ public class BackupMetrics { invalidDownloads, snsNotificationSuccess, snsNotificationFailure, - forgottenFiles; + forgottenFiles, + backupVerificationFailure; public static final String uploadQueueSize = Metrics.METRIC_PREFIX + "upload.queue.size"; public static final String downloadQueueSize = Metrics.METRIC_PREFIX + "download.queue.size"; @@ -53,6 +54,8 @@ public BackupMetrics(Registry registry) { snsNotificationFailure = registry.counter(Metrics.METRIC_PREFIX + "sns.notification.failure"); forgottenFiles = registry.counter(Metrics.METRIC_PREFIX + "forgotten.files"); + backupVerificationFailure = + registry.counter(Metrics.METRIC_PREFIX + "backup.verification.failure"); } public DistributionSummary getUploadRate() { @@ -91,6 +94,10 @@ public void incrementSnsNotificationFailure() { snsNotificationFailure.increment(); } + public void incrementBackupVerificationFailure() { + backupVerificationFailure.increment(); + } + public void recordUploadRate(long sizeInBytes) { uploadRate.record(sizeInBytes); } diff --git a/priam/src/test/java/com/netflix/priam/restore/TestRestore.java b/priam/src/test/java/com/netflix/priam/restore/TestRestore.java index 06a21b1a6..aeb169911 100644 --- a/priam/src/test/java/com/netflix/priam/restore/TestRestore.java +++ b/priam/src/test/java/com/netflix/priam/restore/TestRestore.java @@ -150,6 +150,7 @@ public void testRestoreFromDiffCluster() throws Exception { populateBackupFileSystem("test_backup_new"); String dateRange = "201108110030,201108110530"; restore.restore(new DateUtil.DateRange(dateRange)); + System.out.println("Downloaded files: " + filesystem.downloadedFiles); Assert.assertTrue(filesystem.downloadedFiles.contains(fileList.get(0))); Assert.assertTrue(filesystem.downloadedFiles.contains(fileList.get(1))); Assert.assertTrue(filesystem.downloadedFiles.contains(fileList.get(2))); From ef05406eb20c8289c6903893ddbd0f10972cfec7 Mon Sep 17 00:00:00 2001 From: Arun Agrawal Date: Fri, 15 Mar 2019 11:14:05 -0700 Subject: [PATCH 076/228] Fix X->Y->Z issue. Replace nodes when gossip actually converges. --- .../priam/connection/CassandraOperations.java | 67 +++++ .../connection/ICassandraOperations.java | 2 + .../identity/token/DeadTokenRetriever.java | 245 +++++++++++------- .../priam/resources/CassandraAdmin.java | 53 +--- .../connection/TestCassandraOperations.java | 83 ++++++ .../token/TestDeadTokenRetriever.java | 212 +++++++++++++++ .../src/test/resources/gossipInfoSample_1.txt | 105 ++++++++ 7 files changed, 624 insertions(+), 143 deletions(-) create mode 100644 priam/src/test/java/com/netflix/priam/connection/TestCassandraOperations.java create mode 100644 priam/src/test/java/com/netflix/priam/identity/token/TestDeadTokenRetriever.java create mode 100644 priam/src/test/resources/gossipInfoSample_1.txt diff --git a/priam/src/main/java/com/netflix/priam/connection/CassandraOperations.java b/priam/src/main/java/com/netflix/priam/connection/CassandraOperations.java index d99129c79..b82fc13a3 100644 --- a/priam/src/main/java/com/netflix/priam/connection/CassandraOperations.java +++ b/priam/src/main/java/com/netflix/priam/connection/CassandraOperations.java @@ -128,4 +128,71 @@ public Void retriableCall() throws Exception { } }.call(); } + + @Override + public List> gossipInfo() throws Exception { + List> returnPublicIpSourceIpMap = new ArrayList(); + try { + JMXNodeTool nodeTool; + try { + nodeTool = JMXNodeTool.instance(configuration); + } catch (JMXConnectionException e) { + logger.error( + "Exception in fetching c* jmx tool . Msg: {}", e.getLocalizedMessage(), e); + throw e; + } + String gossipInfoLines[] = nodeTool.getGossipInfo().split("/"); + Arrays.stream(gossipInfoLines) + .forEach( + gossipInfoLine -> { + Map gossipMap = new HashMap<>(); + String gossipInfoSubLines[] = gossipInfoLine.split("\\r?\\n"); + if (gossipInfoSubLines.length + > 2) // Random check for existence of some lines + { + gossipMap.put("PUBLIC_IP", gossipInfoSubLines[0].trim()); + if (gossipMap.get("PUBLIC_IP") != null) { + returnPublicIpSourceIpMap.add(gossipMap); + } + + for (String gossipInfoSubLine : gossipInfoSubLines) { + String gossipLineEntry[] = gossipInfoSubLine.split(":"); + if (gossipLineEntry.length == 2) { + gossipMap.put( + gossipLineEntry[0].trim().toUpperCase(), + gossipLineEntry[1].trim()); + } else if (gossipLineEntry.length == 3) { + if (gossipLineEntry[0] + .trim() + .equalsIgnoreCase("STATUS")) { + // Special handling for STATUS as C* puts first + // token in STATUS or "true". + gossipMap.put( + gossipLineEntry[0].trim().toUpperCase(), + gossipLineEntry[2].split(",")[0].trim()); + } else if (gossipLineEntry[0] + .trim() + .equalsIgnoreCase("TOKENS")) { + // Special handling for tokens as it is always + // "hidden". + gossipMap.put( + gossipLineEntry[0].trim().toUpperCase(), + nodeTool.getTokens( + gossipMap.get("PUBLIC_IP")) + .toString()); + } else { + gossipMap.put( + gossipLineEntry[0].trim().toUpperCase(), + gossipLineEntry[2].trim()); + } + } + } + } + }); + + } catch (Exception e) { + logger.error("Unable to parse nodetool gossipinfo output from Cassandra.", e); + } + return returnPublicIpSourceIpMap; + } } diff --git a/priam/src/main/java/com/netflix/priam/connection/ICassandraOperations.java b/priam/src/main/java/com/netflix/priam/connection/ICassandraOperations.java index 971e154ef..ae0c97201 100644 --- a/priam/src/main/java/com/netflix/priam/connection/ICassandraOperations.java +++ b/priam/src/main/java/com/netflix/priam/connection/ICassandraOperations.java @@ -59,4 +59,6 @@ public interface ICassandraOperations { void forceKeyspaceCompaction(String keyspaceName, String... columnfamilies) throws Exception; void forceKeyspaceFlush(String keyspaceName) throws Exception; + + List> gossipInfo() throws Exception; } diff --git a/priam/src/main/java/com/netflix/priam/identity/token/DeadTokenRetriever.java b/priam/src/main/java/com/netflix/priam/identity/token/DeadTokenRetriever.java index fd4066511..733c663a6 100755 --- a/priam/src/main/java/com/netflix/priam/identity/token/DeadTokenRetriever.java +++ b/priam/src/main/java/com/netflix/priam/identity/token/DeadTokenRetriever.java @@ -21,18 +21,15 @@ import com.netflix.priam.identity.PriamInstance; import com.netflix.priam.identity.config.InstanceInfo; import com.netflix.priam.utils.Sleeper; -import com.sun.jersey.api.client.Client; -import com.sun.jersey.api.client.ClientResponse; -import com.sun.jersey.api.client.WebResource; -import com.sun.jersey.api.client.config.ClientConfig; -import com.sun.jersey.api.client.config.DefaultClientConfig; +import com.netflix.priam.utils.SystemUtils; +import java.util.Collections; import java.util.List; import java.util.Random; -import javax.ws.rs.core.MediaType; +import java.util.stream.Collectors; import org.apache.commons.lang3.StringUtils; +import org.json.simple.JSONArray; import org.json.simple.JSONObject; import org.json.simple.parser.JSONParser; -import org.json.simple.parser.ParseException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -80,7 +77,8 @@ private List getDualAccountRacMembership(List asgInstances) { public PriamInstance get() throws Exception { logger.info("Looking for a token from any dead node"); - final List allIds = factory.getAllIds(config.getAppName()); + final List allInstancesWithinCluster = + factory.getAllIds(config.getAppName()); List asgInstances = membership.getRacMembership(); if (config.isDualAccount()) { asgInstances = getDualAccountRacMembership(asgInstances); @@ -90,45 +88,84 @@ public PriamInstance get() throws Exception { // Sleep random interval - upto 15 sec sleeper.sleep(new Random().nextInt(5000) + 10000); - for (PriamInstance dead : allIds) { - // test same zone and is it is alive. - if (!dead.getRac().equals(instanceInfo.getRac()) - || asgInstances.contains(dead.getInstanceId()) - || super.isInstanceDummy(dead)) continue; - logger.info("Found dead instances: {}", dead.getInstanceId()); + logger.info( + "About to iterate through all instances within cluster " + + config.getAppName() + + " to find an available token to acquire."); + + for (PriamInstance priamInstance : allInstancesWithinCluster) { + // test same zone and is it alive. + if (!priamInstance.getRac().equals(instanceInfo.getRac()) + || asgInstances.contains(priamInstance.getInstanceId()) + || super.isInstanceDummy(priamInstance)) continue; + // TODO: If instance is in SHUTTING_DOWN mode, it might not show up in asg instances (if + // cloud control plane is having issues), thus, we should not try to replace the + // instance as it will lead to "Cannot replace a live node" issue. + + logger.info("Found dead instance: {}", priamInstance.toString()); + PriamInstance markAsDead = factory.create( - dead.getApp() + "-dead", - dead.getId(), - dead.getInstanceId(), - dead.getHostName(), - dead.getHostIP(), - dead.getRac(), - dead.getVolumes(), - dead.getToken()); + priamInstance.getApp() + "-dead", + priamInstance.getId(), + priamInstance.getInstanceId(), + priamInstance.getHostName(), + priamInstance.getHostIP(), + priamInstance.getRac(), + priamInstance.getVolumes(), + priamInstance.getToken()); // remove it as we marked it down... - factory.delete(dead); + factory.delete(priamInstance); // find the replaced IP - this.replacedIp = findReplaceIp(allIds, markAsDead.getToken(), markAsDead.getDC()); - if (this.replacedIp == null) this.replacedIp = markAsDead.getHostIP(); + this.replacedIp = + findReplaceIp( + allInstancesWithinCluster, + priamInstance.getToken(), + priamInstance.getDC()); + // Lets not replace the instance if gossip info is not merging!! + if (replacedIp == null) return null; - String payLoad = markAsDead.getToken(); logger.info( - "Trying to grab slot {} with availability zone {}", - markAsDead.getId(), - markAsDead.getRac()); - return factory.create( - config.getAppName(), - markAsDead.getId(), - instanceInfo.getInstanceId(), - instanceInfo.getHostname(), - instanceInfo.getHostIP(), - instanceInfo.getRac(), - markAsDead.getVolumes(), - payLoad); + "Will try to replace token: {} with replacedIp (from gossip info): {} instead of ip from Token database: {}", + priamInstance.getToken(), + replacedIp, + priamInstance.getHostIP()); + + PriamInstance result; + try { + result = + factory.create( + config.getAppName(), + markAsDead.getId(), + instanceInfo.getInstanceId(), + instanceInfo.getHostname(), + instanceInfo.getHostIP(), + instanceInfo.getRac(), + markAsDead.getVolumes(), + markAsDead.getToken()); + } catch (Exception ex) { + long sleepTime = super.getSleepTime(); + logger.warn( + "Exception when acquiring dead token: " + + priamInstance.getToken() + + " , will sleep for " + + sleepTime + + " millisecs before we retry."); + Thread.currentThread().sleep(sleepTime); + throw ex; + } + + logger.info( + "Acquired token: " + + priamInstance.getToken() + + " and we will replace with replacedIp: " + + replacedIp); + + return result; } + logger.info("This node was NOT able to acquire any dead token"); return null; } @@ -137,87 +174,95 @@ public String getReplaceIp() { return this.replacedIp; } - private String findReplaceIp(List allIds, String token, String location) { - String ip; - for (PriamInstance ins : allIds) { + private String findReplaceIp(List allIds, String token, String dc) + throws Exception { + // Avoid using dead instance who we are trying to replace (duh!!) + // Avoid other regions instances to avoid communication over public ip address. + List eligibleInstances = + allIds.parallelStream() + .filter(priamInstance -> !priamInstance.getToken().equalsIgnoreCase(token)) + .filter(priamInstance -> priamInstance.getDC().equalsIgnoreCase(dc)) + .collect(Collectors.toList()); + // We want to get IP from min 1, max 3 instances to ensure we are not relying on gossip of a + // single instance. + // Good idea to shuffle so we are not talking to same instances every time. + Collections.shuffle(eligibleInstances); + // Potential issue could be when you have about 50% of your cluster C* DOWN or trying to be + // replaced. + // Think of a major disaster hitting your cluster. In that scenario chances of instance + // hitting DOWN C* are much much higher. + // In such a case you should rely on @link{CassandraConfig#setReplacedIp}. + int noOfInstancesGossipShouldMatch = Math.max(1, Math.min(3, eligibleInstances.size())); + int noOfInstancesWithGossipMatch = 0; + String replace_ip = null, ip = null; + for (PriamInstance ins : eligibleInstances) { logger.info("Calling getIp on hostname[{}] and token[{}]", ins.getHostName(), token); - if (ins.getToken().equals(token) || !ins.getDC().equals(location)) { - // avoid using dead instance and other regions instances - continue; - } - - try { - ip = getIp(ins.getHostName(), token); - } catch (ParseException e) { - ip = null; - } - - if (ip != null) { - logger.info("Found the IP: {}", ip); - return ip; + ip = getIp(ins.getHostName(), token); + if (StringUtils.isEmpty(replace_ip)) replace_ip = ip; + if (!StringUtils.isEmpty(replace_ip) && !StringUtils.isEmpty(ip)) { + if (replace_ip.equalsIgnoreCase(ip)) { + noOfInstancesWithGossipMatch++; + if (noOfInstancesWithGossipMatch >= noOfInstancesGossipShouldMatch) { + logger.info( + "Using replace_ip: {} as # of required gossip info match: {}", + replace_ip, + noOfInstancesGossipShouldMatch); + return replace_ip; + } + } else + throw new Exception( + String.format( + "Unexpected Exception: Gossip info from hosts are not matching: found {} and {}", + replace_ip, + ip)); } } - + logger.warn( + "Return null: Unable to find enough instances where gossip match. Required: {}", + noOfInstancesGossipShouldMatch); return null; } - private String getIp(String host, String token) throws ParseException { - ClientConfig config = new DefaultClientConfig(); - Client client = Client.create(config); - String baseURI = getBaseURI(host); - WebResource service = client.resource(baseURI); - - ClientResponse clientResp; - String textEntity; - + private String getIp(String host, String token) { + String response = null; try { - clientResp = - service.path("Priam/REST/v1/cassadmin/gossipinfo") - .accept(MediaType.APPLICATION_JSON) - .get(ClientResponse.class); - - if (clientResp.getStatus() != 200) return null; + response = SystemUtils.getDataFromUrl(getGossipInfoURL(host)); - textEntity = clientResp.getEntity(String.class); + String inputToken = String.format("[%s]", token); + JSONParser parser = new JSONParser(); + JSONArray jsonObject = (JSONArray) parser.parse(response); - logger.info( - "Respond from calling gossipinfo on host[{}] and token[{}] : {}", - host, - token, - textEntity); - - if (StringUtils.isEmpty(textEntity)) return null; - } catch (Exception e) { - logger.info("Error in reaching out to host: {}", baseURI); - return null; - } + for (Object key : jsonObject) { + JSONObject msg = (JSONObject) key; - JSONParser parser = new JSONParser(); - Object obj = parser.parse(textEntity); + // Ensure that we are not trying to replace a NORMAL token and token of that + // instance matches what we want to replace. + if (msg.get("STATUS") == null + || msg.get("STATUS").toString().equalsIgnoreCase("NORMAL") + || msg.get("TOKENS") == null + || msg.get("PUBLIC_IP") == null + || !msg.get("TOKENS").toString().equals(inputToken)) { + continue; + } - JSONObject jsonObject = (JSONObject) obj; - - for (Object key : jsonObject.keySet()) { - JSONObject msg = (JSONObject) jsonObject.get(key); - if (msg.get("Token") == null) { - continue; - } - String tokenVal = (String) msg.get("Token"); - - if (token.equals(tokenVal)) { logger.info( - "Using gossipinfo from host[{}] and token[{}], the replaced address is : {}", + "Using gossip info from host[{}] and token[{}], the replaced address is : [{}]", host, token, - key); - return (String) key; + msg.get("PUBLIC_IP")); + return (String) msg.get("PUBLIC_IP"); } + } catch (Exception e) { + logger.info( + "Error in reaching out to host: [{}} or parsing response from host: {}", + host, + response); } return null; } - private String getBaseURI(String host) { - return "http://" + host + ":8080/"; + private String getGossipInfoURL(String host) { + return "http://" + host + ":8080/Priam/REST/v1/cassadmin/gossipinfo"; } @Override diff --git a/priam/src/main/java/com/netflix/priam/resources/CassandraAdmin.java b/priam/src/main/java/com/netflix/priam/resources/CassandraAdmin.java index 6de44eb91..0e64b3cdc 100644 --- a/priam/src/main/java/com/netflix/priam/resources/CassandraAdmin.java +++ b/priam/src/main/java/com/netflix/priam/resources/CassandraAdmin.java @@ -22,12 +22,15 @@ import com.netflix.priam.cluster.management.Flush; import com.netflix.priam.compress.SnappyCompression; import com.netflix.priam.config.IConfiguration; +import com.netflix.priam.connection.CassandraOperations; import com.netflix.priam.connection.JMXConnectionException; import com.netflix.priam.connection.JMXNodeTool; import com.netflix.priam.defaultimpl.ICassandraProcess; import java.io.FileInputStream; import java.io.FileOutputStream; import java.io.IOException; +import java.util.List; +import java.util.Map; import java.util.concurrent.ExecutionException; import javax.ws.rs.*; import javax.ws.rs.core.MediaType; @@ -54,17 +57,20 @@ public class CassandraAdmin { private final ICassandraProcess cassProcess; private final Flush flush; private final Compaction compaction; + private final CassandraOperations cassandraOperations; @Inject public CassandraAdmin( IConfiguration config, ICassandraProcess cassProcess, Flush flush, - Compaction compaction) { + Compaction compaction, + CassandraOperations cassandraOperations) { this.config = config; this.cassProcess = cassProcess; this.flush = flush; this.compaction = compaction; + this.cassandraOperations = cassandraOperations; } @GET @@ -326,48 +332,9 @@ public Response statusthrift() @GET @Path("/gossipinfo") - public Response gossipinfo() - throws IOException, ExecutionException, InterruptedException, JSONException { - JMXNodeTool nodeTool; - try { - nodeTool = JMXNodeTool.instance(config); - } catch (JMXConnectionException e) { - logger.error( - "Exception in fetching c* jmx tool . Msgl: {}", e.getLocalizedMessage(), e); - return Response.status(503).entity("JMXConnectionException").build(); - } - JSONObject rootObj = parseGossipInfo(nodeTool.getGossipInfo()); - return Response.ok(rootObj, MediaType.APPLICATION_JSON).build(); - } - - // helper method for parsing, to be tested easily - private static JSONObject parseGossipInfo(String gossipinfo) throws JSONException { - String[] ginfo = gossipinfo.split("\n"); - JSONObject rootObj = new JSONObject(); - JSONObject obj = new JSONObject(); - String key = ""; - for (String line : ginfo) { - if (line.matches("^.*/.*$")) { - String[] data = line.split("/"); - if (StringUtils.isNotBlank(key)) { - rootObj.put(key, obj); - obj = new JSONObject(); - } - key = data[1]; - } else if (line.matches("^ .*:.*$")) { - String[] kv = line.split(":"); - kv[0] = kv[0].trim(); - if (kv[0].equals("STATUS")) { - obj.put(kv[0], kv[1]); - String[] vv = kv[1].split(","); - obj.put("Token", vv[1]); - } else { - obj.put(kv[0], kv[1]); - } - } - } - if (StringUtils.isNotBlank(key)) rootObj.put(key, obj); - return rootObj; + public Response gossipinfo() throws Exception { + List> parsedInfo = cassandraOperations.gossipInfo(); + return Response.ok(parsedInfo, MediaType.APPLICATION_JSON).build(); } @GET diff --git a/priam/src/test/java/com/netflix/priam/connection/TestCassandraOperations.java b/priam/src/test/java/com/netflix/priam/connection/TestCassandraOperations.java new file mode 100644 index 000000000..a7aa32ee3 --- /dev/null +++ b/priam/src/test/java/com/netflix/priam/connection/TestCassandraOperations.java @@ -0,0 +1,83 @@ +/* + * Copyright 2019 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.netflix.priam.connection; + +import com.google.inject.Guice; +import com.google.inject.Injector; +import com.mchange.io.FileUtils; +import com.netflix.priam.backup.BRTestModule; +import com.netflix.priam.config.IConfiguration; +import java.io.File; +import java.util.List; +import java.util.Map; +import mockit.Expectations; +import mockit.Mock; +import mockit.MockUp; +import mockit.Mocked; +import org.apache.cassandra.tools.NodeProbe; +import org.junit.Assert; +import org.junit.Test; + +/** Created by aagrawal on 3/1/19. */ +public class TestCassandraOperations { + private final String gossipInfo1 = "src/test/resources/gossipInfoSample_1.txt"; + @Mocked private NodeProbe nodeProbe; + @Mocked private JMXNodeTool jmxNodeTool; + private static CassandraOperations cassandraOperations; + + public TestCassandraOperations() { + new MockUp() { + @Mock + NodeProbe instance(IConfiguration config) { + return nodeProbe; + } + }; + Injector injector = Guice.createInjector(new BRTestModule()); + if (cassandraOperations == null) + cassandraOperations = injector.getInstance(CassandraOperations.class); + } + + @Test + public void testGossipInfo() throws Exception { + + String gossipInfoFromNodetool = FileUtils.getContentsAsString(new File(gossipInfo1)); + new Expectations() { + { + nodeProbe.getGossipInfo(); + result = gossipInfoFromNodetool; + nodeProbe.getTokens("127.0.0.1"); + result = "123,234"; + } + }; + List> gossipInfoList = cassandraOperations.gossipInfo(); + System.out.println(gossipInfoList); + Assert.assertEquals(7, gossipInfoList.size()); + gossipInfoList + .stream() + .forEach( + gossipInfo -> { + Assert.assertEquals("us-east", gossipInfo.get("DC")); + Assert.assertNotNull(gossipInfo.get("PUBLIC_IP")); + Assert.assertEquals("1565153", gossipInfo.get("HEARTBEAT")); + if (gossipInfo.get("STATUS").equalsIgnoreCase("NORMAL")) + Assert.assertNotNull(gossipInfo.get("TOKENS")); + if (gossipInfo.get("PUBLIC_IP").equalsIgnoreCase("127.0.0.1")) + Assert.assertEquals("[123,234]", gossipInfo.get("TOKENS")); + }); + } +} diff --git a/priam/src/test/java/com/netflix/priam/identity/token/TestDeadTokenRetriever.java b/priam/src/test/java/com/netflix/priam/identity/token/TestDeadTokenRetriever.java new file mode 100644 index 000000000..f32799f92 --- /dev/null +++ b/priam/src/test/java/com/netflix/priam/identity/token/TestDeadTokenRetriever.java @@ -0,0 +1,212 @@ +/* + * Copyright 2019 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.netflix.priam.identity.token; + +import com.google.common.collect.Lists; +import com.google.inject.Guice; +import com.google.inject.Injector; +import com.netflix.priam.backup.BRTestModule; +import com.netflix.priam.config.IConfiguration; +import com.netflix.priam.identity.IMembership; +import com.netflix.priam.identity.IPriamInstanceFactory; +import com.netflix.priam.identity.PriamInstance; +import com.netflix.priam.identity.config.InstanceInfo; +import com.netflix.priam.utils.FakeSleeper; +import com.netflix.priam.utils.SystemUtils; +import java.util.List; +import mockit.Expectations; +import mockit.Mocked; +import mockit.Verifications; +import org.junit.Assert; +import org.junit.Test; + +/** Created by aagrawal on 3/1/19. */ +public class TestDeadTokenRetriever { + @Mocked private IPriamInstanceFactory factory; + @Mocked private IMembership membership; + private IDeadTokenRetriever deadTokenRetriever; + private InstanceInfo instanceInfo; + private IConfiguration configuration; + + public TestDeadTokenRetriever() { + Injector injector = Guice.createInjector(new BRTestModule()); + if (instanceInfo == null) instanceInfo = injector.getInstance(InstanceInfo.class); + if (configuration == null) configuration = injector.getInstance(IConfiguration.class); + } + + @Test + public void testNoReplacementNormalScenario() throws Exception { + new Expectations() { + { + factory.getAllIds(anyString); + result = Lists.newArrayList(); + membership.getRacMembership(); + result = Lists.newArrayList(); + } + }; + deadTokenRetriever = + new DeadTokenRetriever( + factory, membership, configuration, new FakeSleeper(), instanceInfo); + PriamInstance priamInstance = deadTokenRetriever.get(); + Assert.assertNull(priamInstance); + } + + @Test + // There is no slot available for replacement as per Token Database. + public void testNoReplacementNoSpotAvailable() throws Exception { + List allInstances = getInstances(1); + List racMembership = getRacMembership(1); + racMembership.add(instanceInfo.getInstanceId()); + + new Expectations() { + { + factory.getAllIds(anyString); + result = allInstances; + membership.getRacMembership(); + result = racMembership; + } + }; + deadTokenRetriever = + new DeadTokenRetriever( + factory, membership, configuration, new FakeSleeper(), instanceInfo); + PriamInstance priamInstance = deadTokenRetriever.get(); + Assert.assertNull(priamInstance); + new Verifications() { + { + factory.delete(withAny(priamInstance)); + times = 0; + } + }; + } + + @Test + // There is a potential slot for dead token but we are unable to replace. + public void testNoReplacementNoGossipMatch(@Mocked SystemUtils systemUtils) throws Exception { + List allInstances = getInstances(2); + List racMembership = getRacMembership(1); + racMembership.add(instanceInfo.getInstanceId()); + PriamInstance instance = null; + // gossip info returns null, thus unable to replace the instance. + new Expectations() { + { + factory.getAllIds(anyString); + result = allInstances; + membership.getRacMembership(); + result = racMembership; + systemUtils.getDataFromUrl(anyString); + result = null; + times = 1; + } + }; + deadTokenRetriever = + new DeadTokenRetriever( + factory, membership, configuration, new FakeSleeper(), instanceInfo); + PriamInstance priamInstance = deadTokenRetriever.get(); + Assert.assertNull(priamInstance); + new Verifications() { + { + factory.delete(withAny(instance)); + times = 1; + } + }; + + // gossip info from another instance returns everything is OK and thus unable to replace. + String gossipInfo = + "[{\"TOKENS\":\"[1]\",\"PUBLIC_IP\":\"127.0.0.1\",\"RACK\":\"az1\",\"STATUS\":\"NORMAL\",\"DC\":\"us-east-1\"},{\"TOKENS\":\"[2]\",\"PUBLIC_IP\":\"127.0.0.2\",\"RACK\":\"az1\",\"STATUS\":\"NORMAL\",\"DC\":\"us-east-1\"}]"; + new Expectations() { + { + factory.getAllIds(anyString); + result = allInstances; + membership.getRacMembership(); + result = racMembership; + systemUtils.getDataFromUrl(anyString); + result = gossipInfo; + times = 1; + } + }; + priamInstance = deadTokenRetriever.get(); + Assert.assertNull(priamInstance); + new Verifications() { + { + factory.delete(withAny(instance)); + times = 1; + } + }; + } + + @Test + public void testReplacementGossipMatch(@Mocked SystemUtils systemUtils) throws Exception { + List allInstances = getInstances(6); + List racMembership = getRacMembership(2); + racMembership.add(instanceInfo.getInstanceId()); + String gossipResponse = + "[{\"TOKENS\":\"[1]\",\"PUBLIC_IP\":\"127.0.0.1\",\"RACK\":\"az1\",\"STATUS\":\"NORMAL\",\"DC\":\"us-east-1\"},{\"TOKENS\":\"[2]\",\"PUBLIC_IP\":\"127.0.0.2\",\"RACK\":\"az1\",\"STATUS\":\"NORMAL\",\"DC\":\"us-east-1\"},{\"TOKENS\":\"[3]\",\"PUBLIC_IP\":\"127.0.0.3\",\"RACK\":\"az1\",\"STATUS\":\"shutdown\",\"DC\":\"us-east-1\"},{\"TOKENS\":\"[4]\",\"PUBLIC_IP\":\"127.0.0.4\",\"RACK\":\"az2\",\"STATUS\":\"NORMAL\",\"DC\":\"us-east-1\"},{\"TOKENS\":\"[5]\",\"PUBLIC_IP\":\"127.0.0.5\",\"RACK\":\"az2\",\"STATUS\":\"NORMAL\",\"DC\":\"us-east-1\"},{\"TOKENS\":\"[6]\",\"PUBLIC_IP\":\"127.0.0.6\",\"RACK\":\"az2\",\"STATUS\":\"NORMAL\",\"DC\":\"us-east-1\"}]"; + PriamInstance instance = null; + new Expectations() { + { + factory.getAllIds(anyString); + result = allInstances; + membership.getRacMembership(); + result = racMembership; + systemUtils.getDataFromUrl(anyString); + returns(gossipResponse, gossipResponse, null, gossipResponse); + } + }; + deadTokenRetriever = + new DeadTokenRetriever( + factory, membership, configuration, new FakeSleeper(), instanceInfo); + PriamInstance priamInstance = deadTokenRetriever.get(); + Assert.assertNotNull(priamInstance); + Assert.assertEquals("127.0.0.3", deadTokenRetriever.getReplaceIp()); + } + + private List getInstances(int noOfInstances) { + List allInstances = Lists.newArrayList(); + for (int i = 1; i <= noOfInstances; i++) + allInstances.add( + create( + i, + String.format("instance_id_%d", i), + String.format("hostname_%d", i), + String.format("127.0.0.%d", i), + instanceInfo.getRac(), + i + "")); + return allInstances; + } + + private List getRacMembership(int noOfInstances) { + List racMembership = Lists.newArrayList(); + for (int i = 1; i <= noOfInstances; i++) + racMembership.add(String.format("instance_id_%d", i)); + return racMembership; + } + + private PriamInstance create( + int id, String instanceID, String hostname, String ip, String rac, String payload) { + PriamInstance ins = new PriamInstance(); + ins.setApp(configuration.getAppName()); + ins.setRac(rac); + ins.setHost(hostname); + ins.setHostIP(ip); + ins.setId(id); + ins.setInstanceId(instanceID); + ins.setDC(instanceInfo.getRegion()); + ins.setToken(payload); + return ins; + } +} diff --git a/priam/src/test/resources/gossipInfoSample_1.txt b/priam/src/test/resources/gossipInfoSample_1.txt new file mode 100644 index 000000000..bc8a476dc --- /dev/null +++ b/priam/src/test/resources/gossipInfoSample_1.txt @@ -0,0 +1,105 @@ +/127.0.0.1 + generation:1550517654 + heartbeat:1565153 + STATUS:388:NORMAL,56713727820156410577229101240436610841 + LOAD:1564991:370740.0 + SCHEMA:577:b3839e65-1930-34f5-bb71-5f35ac844191 + DC:9:us-east + RACK:11:1d + RELEASE_VERSION:5:2.1.19.6 + INTERNAL_IP:7:127.0.0.1 + RPC_ADDRESS:4:127.0.0.1 + SEVERITY:1565152:0.0 + NET_VERSION:2:8 + HOST_ID:3:52661ae2-43e8-4183-ae3a-09e6a752ba8a + TOKENS:387: +/127.0.0.2 + generation:1544121750 + heartbeat:1565153 + STATUS:21:NORMAL,28356863910078205288614550621122593220 + LOAD:20962586:1.81589793594E11 + SCHEMA:19398066:b3839e65-1930-34f5-bb71-5f35ac844191 + DC:8:us-east + RACK:10:1c + RELEASE_VERSION:4:2.1.19.5 + INTERNAL_IP:6:127.0.0.2 + RPC_ADDRESS:3:127.0.0.2 + SEVERITY:20962630:0.0 + NET_VERSION:1:8 + HOST_ID:2:5e138f0e-8123-4ef7-9260-c19e48666a99 + TOKENS:20: +/127.0.0.3 + generation:1548877968 + heartbeat:1565153 + STATUS:18:NORMAL,113427455640312821154458202479064646083 + LOAD:6537819:1.81428734494E11 + SCHEMA:4973397:b3839e65-1930-34f5-bb71-5f35ac844191 + DC:8:us-east + RACK:10:1c + RELEASE_VERSION:4:2.1.19.5 + INTERNAL_IP:6:127.0.0.3 + RPC_ADDRESS:3:127.0.0.3 + SEVERITY:6537948:0.0 + NET_VERSION:1:8 + HOST_ID:2:7e5d32b9-5298-42a4-9091-b4d65ca43596 + TOKENS:17: +/127.0.0.4 + generation:1544122124 + heartbeat:1565153 + STATUS:18:NORMAL,1808575600 + LOAD:20961630:1.81437564192E11 + SCHEMA:19397072:b3839e65-1930-34f5-bb71-5f35ac844191 + DC:8:us-east + RACK:10:1e + RELEASE_VERSION:4:2.1.19.5 + INTERNAL_IP:6:127.0.0.4 + RPC_ADDRESS:3:127.0.0.4 + SEVERITY:20961636:16.88311767578125 + NET_VERSION:1:8 + HOST_ID:2:4e4d6165-4c7a-455b-9bc2-bfb7a118417f + TOKENS:17: +/127.0.0.5 + generation:1548451833 + heartbeat:1565153 + STATUS:16:NORMAL,141784319550391026443072753098378663704 + LOAD:7830271:1.80978101023E11 + SCHEMA:6265686:b3839e65-1930-34f5-bb71-5f35ac844191 + DC:8:us-east + RACK:10:1d + RELEASE_VERSION:4:2.1.19.5 + INTERNAL_IP:6:127.0.0.5 + RPC_ADDRESS:3:127.0.0.5 + SEVERITY:7830444:0.0 + NET_VERSION:1:8 + HOST_ID:2:56a88676-37cb-4d87-b30c-be8f75ff3773 + TOKENS:15: +/127.0.0.6 + generation:1547873811 + heartbeat:1565153 + STATUS:16:NORMAL,85070591730234615865843651859750628462 + LOAD:9583414:1.81759352129E11 + SCHEMA:8018940:b3839e65-1930-34f5-bb71-5f35ac844191 + DC:8:us-east + RACK:10:1e + RELEASE_VERSION:4:2.1.19.5 + INTERNAL_IP:6:127.0.0.6 + RPC_ADDRESS:3:127.0.0.6 + SEVERITY:9583509:0.0 + NET_VERSION:1:8 + HOST_ID:2:a8b6c167-2c6b-44be-94e6-c41fa53c7515 + TOKENS:15: +/127.0.0.8 + generation:1551117498 + heartbeat:1565153 + STATUS:279:BOOT_REPLACE,127.0.0.7 + LOAD:989:7.26418875E8 + SCHEMA:100:b3839e65-1930-34f5-bb71-5f35ac844191 + DC:9:us-east + RACK:11:1d + RELEASE_VERSION:5:2.1.19.6 + INTERNAL_IP:7:100.0.0.0 + RPC_ADDRESS:4:100.0.0.0 + SEVERITY:1117:0.0 + NET_VERSION:1:8 + HOST_ID:3:ff004440-b678-4c45-a642-dba905903b84 + TOKENS:278: \ No newline at end of file From 3d96bf87246c7155185fec0d5f38be89be96c0be Mon Sep 17 00:00:00 2001 From: Arun Agrawal Date: Thu, 21 Mar 2019 14:46:04 -0700 Subject: [PATCH 077/228] Run backup TTL task on simple timer instead of CRON to avoid bombarding S3 with delete calls at same time and S3 chocking on us. --- .../netflix/priam/backupv2/BackupTTLTask.java | 10 ++++---- .../priam/config/BackupRestoreConfig.java | 4 ++-- .../priam/config/IBackupRestoreConfig.java | 19 ++++++++------- .../netflix/priam/scheduler/SimpleTimer.java | 23 ++++++++++++++++--- .../priam/backupv2/TestBackupV2Service.java | 8 +++---- 5 files changed, 42 insertions(+), 22 deletions(-) diff --git a/priam/src/main/java/com/netflix/priam/backupv2/BackupTTLTask.java b/priam/src/main/java/com/netflix/priam/backupv2/BackupTTLTask.java index 6c5eb6ce2..86505d6c5 100644 --- a/priam/src/main/java/com/netflix/priam/backupv2/BackupTTLTask.java +++ b/priam/src/main/java/com/netflix/priam/backupv2/BackupTTLTask.java @@ -26,7 +26,7 @@ import com.netflix.priam.backup.IFileSystemContext; import com.netflix.priam.config.IBackupRestoreConfig; import com.netflix.priam.config.IConfiguration; -import com.netflix.priam.scheduler.CronTimer; +import com.netflix.priam.scheduler.SimpleTimer; import com.netflix.priam.scheduler.Task; import com.netflix.priam.scheduler.TaskTimer; import com.netflix.priam.utils.DateUtil; @@ -227,15 +227,15 @@ public String getName() { /** * Interval between trying to TTL data on Remote file system. * - * @param backupRestoreConfig {@link IBackupRestoreConfig#getBackupTTLCronExpression()} to get - * configuration details from priam. Use "-1" to disable the service. + * @param backupRestoreConfig {@link IBackupRestoreConfig#getBackupTTLMonitorPeriodInSec()} to + * get configuration details from priam. Use "-1" to disable the service. * @return the timer to be used for backup ttl service. * @throws Exception if the configuration is not set correctly or are not valid. This is to * ensure we fail-fast. */ public static TaskTimer getTimer(IBackupRestoreConfig backupRestoreConfig) throws Exception { - String cronExpression = backupRestoreConfig.getBackupTTLCronExpression(); - return CronTimer.getCronTimer(JOBNAME, cronExpression); + return SimpleTimer.getSimpleTimer( + JOBNAME, backupRestoreConfig.getBackupTTLMonitorPeriodInSec()); } private class MetaFileWalker extends MetaFileReader { diff --git a/priam/src/main/java/com/netflix/priam/config/BackupRestoreConfig.java b/priam/src/main/java/com/netflix/priam/config/BackupRestoreConfig.java index 6685eb96c..21b088414 100644 --- a/priam/src/main/java/com/netflix/priam/config/BackupRestoreConfig.java +++ b/priam/src/main/java/com/netflix/priam/config/BackupRestoreConfig.java @@ -42,8 +42,8 @@ public boolean enableV2Restore() { } @Override - public String getBackupTTLCronExpression() { - return config.get("priam.backupTTLCronExpression", "0 0 0/6 1/1 * ? *"); + public int getBackupTTLMonitorPeriodInSec() { + return config.get("priam.backupTTLMonitorPeriodInSec", 21600); } @Override diff --git a/priam/src/main/java/com/netflix/priam/config/IBackupRestoreConfig.java b/priam/src/main/java/com/netflix/priam/config/IBackupRestoreConfig.java index 6c0b53e76..ac39b6915 100644 --- a/priam/src/main/java/com/netflix/priam/config/IBackupRestoreConfig.java +++ b/priam/src/main/java/com/netflix/priam/config/IBackupRestoreConfig.java @@ -45,19 +45,22 @@ default boolean enableV2Backups() { } /** - * Cron expression to be used for the service which does TTL of the backups. This service will - * run only if v2 backups are enabled. The idea is to run this service at least once a day to - * ensure we are marking backup files for TTL as configured via {@link - * IConfiguration#getBackupRetentionDays()} + * Monitoring period for the service which does TTL of the backups. This service will run only + * if v2 backups are enabled. The idea is to run this service at least once a day to ensure we + * are marking backup files for TTL as configured via {@link + * IConfiguration#getBackupRetentionDays()}. Use -1 to disable this service. * - * @return Backup TTL Service cron expression for trying to delete backups. Note that this CRON - * is only the job trying to delete backups and is not the TTL of the backups. + *

    NOTE: This should be scheduled on interval rather than CRON as this results in entire + * fleet to start deletion of files at the same time and remote file system may get overwhelmed. + * + * @return Backup TTL Service execution duration for trying to delete backups. Note that this + * denotes duration of the job trying to delete backups and is not the TTL of the backups. * @see quartz-scheduler * @see http://www.cronmaker.com */ - default String getBackupTTLCronExpression() { - return "0 0 0/6 1/1 * ? *"; + default int getBackupTTLMonitorPeriodInSec() { + return 21600; } /** diff --git a/priam/src/main/java/com/netflix/priam/scheduler/SimpleTimer.java b/priam/src/main/java/com/netflix/priam/scheduler/SimpleTimer.java index 4d5416a72..8759e6ea0 100644 --- a/priam/src/main/java/com/netflix/priam/scheduler/SimpleTimer.java +++ b/priam/src/main/java/com/netflix/priam/scheduler/SimpleTimer.java @@ -22,12 +22,15 @@ import org.quartz.SimpleScheduleBuilder; import org.quartz.Trigger; import org.quartz.TriggerBuilder; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * SimpleTimer allows jobs to run starting from specified time occurring at regular frequency's. * Frequency of the execution timestamp since epoch. */ public class SimpleTimer implements TaskTimer { + private static final Logger logger = LoggerFactory.getLogger(SimpleTimer.class); private final Trigger trigger; public SimpleTimer(String name, long interval) { @@ -40,7 +43,6 @@ public SimpleTimer(String name, long interval) { .repeatForever() .withMisfireHandlingInstructionFireNow()) .build(); - // .new SimpleTrigger(name, SimpleTrigger.REPEAT_INDEFINITELY, interval); } /** Run once at given time... */ @@ -53,7 +55,6 @@ public SimpleTimer(String name, String group, long startTime) { .withMisfireHandlingInstructionFireNow()) .startAt(new Date(startTime)) .build(); - // new SimpleTrigger(name, group, new Date(startTime)); } /** Run immediatly and dont do that again. */ @@ -66,7 +67,23 @@ public SimpleTimer(String name) { .withMisfireHandlingInstructionFireNow()) .startNow() .build(); - // new SimpleTrigger(name, Scheduler.DEFAULT_GROUP); + } + + public static SimpleTimer getSimpleTimer(final String jobName, final long interval) + throws IllegalArgumentException { + SimpleTimer simpleTimer = null; + + if (interval <= 0) { + logger.info( + "Skipping {} as it is disabled via setting {} to {}.", + jobName, + jobName, + interval); + } else { + simpleTimer = new SimpleTimer(jobName, interval); + logger.info("Starting {} with interval of {}", jobName, interval); + } + return simpleTimer; } public Trigger getTrigger() throws ParseException { diff --git a/priam/src/test/java/com/netflix/priam/backupv2/TestBackupV2Service.java b/priam/src/test/java/com/netflix/priam/backupv2/TestBackupV2Service.java index 07ee874bb..af0322f96 100644 --- a/priam/src/test/java/com/netflix/priam/backupv2/TestBackupV2Service.java +++ b/priam/src/test/java/com/netflix/priam/backupv2/TestBackupV2Service.java @@ -72,8 +72,8 @@ public void testBackupEnabled( { backupRestoreConfig.getSnapshotMetaServiceCronExpression(); result = "0 0 0/1 1/1 * ? *"; - backupRestoreConfig.getBackupTTLCronExpression(); - result = "0 0 0/1 1/1 * ? *"; + backupRestoreConfig.getBackupTTLMonitorPeriodInSec(); + result = 600; backupRestoreConfig.getBackupVerificationCronExpression(); result = "0 0 0/1 1/1 * ? *"; backupRestoreConfig.enableV2Backups(); @@ -97,8 +97,8 @@ public void testBackup( { backupRestoreConfig.getSnapshotMetaServiceCronExpression(); result = "0 0 0/1 1/1 * ? *"; - backupRestoreConfig.getBackupTTLCronExpression(); - result = "0 0 0/1 1/1 * ? *"; + backupRestoreConfig.getBackupTTLMonitorPeriodInSec(); + result = 600; backupRestoreConfig.getBackupVerificationCronExpression(); result = "0 0 0/1 1/1 * ? *"; backupRestoreConfig.enableV2Backups(); From b31b508c320185b4444fbf93d53bb776d3b17729 Mon Sep 17 00:00:00 2001 From: Arun Agrawal Date: Thu, 21 Mar 2019 21:07:31 -0700 Subject: [PATCH 078/228] Run backup verification or TTL only when we are not doing restore. --- .../netflix/priam/backupv2/BackupTTLTask.java | 16 ++++++++++++++-- .../priam/backupv2/BackupVerificationTask.java | 15 ++++++++++++++- .../priam/backupv2/TestBackupTTLTask.java | 15 +++++++++++++++ .../backupv2/TestBackupVerificationTask.java | 18 ++++++++++++++++++ 4 files changed, 61 insertions(+), 3 deletions(-) diff --git a/priam/src/main/java/com/netflix/priam/backupv2/BackupTTLTask.java b/priam/src/main/java/com/netflix/priam/backupv2/BackupTTLTask.java index 86505d6c5..57494bbd4 100644 --- a/priam/src/main/java/com/netflix/priam/backupv2/BackupTTLTask.java +++ b/priam/src/main/java/com/netflix/priam/backupv2/BackupTTLTask.java @@ -24,8 +24,10 @@ import com.netflix.priam.backup.BackupRestoreException; import com.netflix.priam.backup.IBackupFileSystem; import com.netflix.priam.backup.IFileSystemContext; +import com.netflix.priam.backup.Status; import com.netflix.priam.config.IBackupRestoreConfig; import com.netflix.priam.config.IConfiguration; +import com.netflix.priam.health.InstanceState; import com.netflix.priam.scheduler.SimpleTimer; import com.netflix.priam.scheduler.Task; import com.netflix.priam.scheduler.TaskTimer; @@ -60,6 +62,7 @@ public class BackupTTLTask extends Task { private IMetaProxy metaProxy; private IBackupFileSystem fileSystem; private Provider abstractBackupPathProvider; + private InstanceState instanceState; public static final String JOBNAME = "BackupTTLService"; private Map filesInMeta = new HashMap<>(); private List filesToDelete = new ArrayList<>(); @@ -73,19 +76,28 @@ public BackupTTLTask( IBackupRestoreConfig backupRestoreConfig, @Named("v2") IMetaProxy metaProxy, IFileSystemContext backupFileSystemCtx, - Provider abstractBackupPathProvider) { + Provider abstractBackupPathProvider, + InstanceState instanceState) { super(configuration); this.backupRestoreConfig = backupRestoreConfig; this.metaProxy = metaProxy; this.fileSystem = backupFileSystemCtx.getFileStrategy(configuration); this.abstractBackupPathProvider = abstractBackupPathProvider; + this.instanceState = instanceState; } @Override public void execute() throws Exception { // Ensure that backup version 2.0 is actually enabled. if (backupRestoreConfig.getSnapshotMetaServiceCronExpression().equalsIgnoreCase("-1")) { - logger.info("Not executing the TTL Service for backups as V2 backups are not enabled."); + logger.info("Not executing the TTL Task for backups as V2 backups are not enabled."); + return; + } + + if (instanceState.getRestoreStatus() != null + && instanceState.getRestoreStatus().getStatus() != null + && instanceState.getRestoreStatus().getStatus() == Status.STARTED) { + logger.info("Not executing the TTL Task for backups as Priam is in restore mode."); return; } diff --git a/priam/src/main/java/com/netflix/priam/backupv2/BackupVerificationTask.java b/priam/src/main/java/com/netflix/priam/backupv2/BackupVerificationTask.java index 8e741f704..8cc9acdc5 100644 --- a/priam/src/main/java/com/netflix/priam/backupv2/BackupVerificationTask.java +++ b/priam/src/main/java/com/netflix/priam/backupv2/BackupVerificationTask.java @@ -22,8 +22,10 @@ import com.netflix.priam.backup.BackupVerification; import com.netflix.priam.backup.BackupVerificationResult; import com.netflix.priam.backup.BackupVersion; +import com.netflix.priam.backup.Status; import com.netflix.priam.config.IBackupRestoreConfig; import com.netflix.priam.config.IConfiguration; +import com.netflix.priam.health.InstanceState; import com.netflix.priam.merics.BackupMetrics; import com.netflix.priam.scheduler.CronTimer; import com.netflix.priam.scheduler.Task; @@ -44,17 +46,20 @@ public class BackupVerificationTask extends Task { private IBackupRestoreConfig backupRestoreConfig; private BackupVerification backupVerification; private BackupMetrics backupMetrics; + private InstanceState instanceState; @Inject public BackupVerificationTask( IConfiguration configuration, IBackupRestoreConfig backupRestoreConfig, BackupVerification backupVerification, - BackupMetrics backupMetrics) { + BackupMetrics backupMetrics, + InstanceState instanceState) { super(configuration); this.backupRestoreConfig = backupRestoreConfig; this.backupVerification = backupVerification; this.backupMetrics = backupMetrics; + this.instanceState = instanceState; } @Override @@ -66,6 +71,14 @@ public void execute() throws Exception { return; } + if (instanceState.getRestoreStatus() != null + && instanceState.getRestoreStatus().getStatus() != null + && instanceState.getRestoreStatus().getStatus() == Status.STARTED) { + logger.info( + "Not executing the Verification Service for backups as Priam is in restore mode."); + return; + } + // Validate the backup done in last x hours. DateRange dateRange = new DateRange( diff --git a/priam/src/test/java/com/netflix/priam/backupv2/TestBackupTTLTask.java b/priam/src/test/java/com/netflix/priam/backupv2/TestBackupTTLTask.java index 39070ed6f..13e33e30b 100644 --- a/priam/src/test/java/com/netflix/priam/backupv2/TestBackupTTLTask.java +++ b/priam/src/test/java/com/netflix/priam/backupv2/TestBackupTTLTask.java @@ -23,7 +23,9 @@ import com.netflix.priam.backup.AbstractBackupPath; import com.netflix.priam.backup.BRTestModule; import com.netflix.priam.backup.FakeBackupFileSystem; +import com.netflix.priam.backup.Status; import com.netflix.priam.config.IConfiguration; +import com.netflix.priam.health.InstanceState; import com.netflix.priam.utils.BackupFileUtils; import com.netflix.priam.utils.DateUtil; import java.io.File; @@ -36,6 +38,8 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import mockit.Expectations; +import mockit.Mocked; import org.junit.After; import org.junit.Assert; import org.junit.Test; @@ -193,4 +197,15 @@ public void testTTLNext() throws Exception { Assert.assertFalse(remoteFiles.contains(allFilesMap.get("META0"))); Assert.assertFalse(remoteFiles.contains(allFilesMap.get("META1"))); } + + @Test + public void testRestoreMode(@Mocked InstanceState state) throws Exception { + new Expectations() { + { + state.getRestoreStatus().getStatus(); + result = Status.STARTED; + } + }; + backupTTLService.execute(); + } } diff --git a/priam/src/test/java/com/netflix/priam/backupv2/TestBackupVerificationTask.java b/priam/src/test/java/com/netflix/priam/backupv2/TestBackupVerificationTask.java index ec3fea7a9..fc13079d1 100644 --- a/priam/src/test/java/com/netflix/priam/backupv2/TestBackupVerificationTask.java +++ b/priam/src/test/java/com/netflix/priam/backupv2/TestBackupVerificationTask.java @@ -23,12 +23,16 @@ import com.netflix.priam.backup.BackupVerification; import com.netflix.priam.backup.BackupVerificationResult; import com.netflix.priam.backup.BackupVersion; +import com.netflix.priam.backup.Status; import com.netflix.priam.config.IConfiguration; +import com.netflix.priam.health.InstanceState; import com.netflix.priam.scheduler.UnsupportedTypeException; import com.netflix.priam.utils.DateUtil.DateRange; import java.util.Optional; +import mockit.Expectations; import mockit.Mock; import mockit.MockUp; +import mockit.Mocked; import org.junit.Assert; import org.junit.Test; @@ -36,6 +40,7 @@ public class TestBackupVerificationTask { private static BackupVerificationTask backupVerificationService; private static IConfiguration configuration; + private static BackupVerification backupVerification; public TestBackupVerificationTask() { new MockBackupVerification(); @@ -43,6 +48,8 @@ public TestBackupVerificationTask() { if (configuration == null) configuration = injector.getInstance(IConfiguration.class); if (backupVerificationService == null) backupVerificationService = injector.getInstance(BackupVerificationTask.class); + if (backupVerification == null) + backupVerification = injector.getInstance(BackupVerification.class); } static class MockBackupVerification extends MockUp { @@ -88,4 +95,15 @@ public void normalOperation() throws Exception { MockBackupVerification.failCall = false; backupVerificationService.execute(); } + + @Test + public void testRestoreMode(@Mocked InstanceState state) throws Exception { + new Expectations() { + { + state.getRestoreStatus().getStatus(); + result = Status.STARTED; + } + }; + backupVerificationService.execute(); + } } From 487117fa2534373d4b7ffbdad4e12e33eb3f053b Mon Sep 17 00:00:00 2001 From: Arun Agrawal Date: Thu, 21 Mar 2019 21:18:48 -0700 Subject: [PATCH 079/228] Add API to clear the filesystem cache if required. --- build.gradle | 2 +- .../netflix/priam/backup/AbstractFileSystem.java | 5 +++++ .../netflix/priam/backup/IBackupFileSystem.java | 3 +++ .../netflix/priam/backupv2/BackupTTLTask.java | 6 ------ .../netflix/priam/resources/BackupServletV2.java | 16 +++++++++++++++- 5 files changed, 24 insertions(+), 8 deletions(-) diff --git a/build.gradle b/build.gradle index c981b8811..0a21f0f54 100644 --- a/build.gradle +++ b/build.gradle @@ -1,5 +1,5 @@ plugins { - id 'nebula.netflixoss' version '7.0.0' + id 'nebula.netflixoss' version '7.1.2' id 'com.github.sherter.google-java-format' version '0.7.1' } diff --git a/priam/src/main/java/com/netflix/priam/backup/AbstractFileSystem.java b/priam/src/main/java/com/netflix/priam/backup/AbstractFileSystem.java index 8f6db6b19..1646b06a5 100644 --- a/priam/src/main/java/com/netflix/priam/backup/AbstractFileSystem.java +++ b/priam/src/main/java/com/netflix/priam/backup/AbstractFileSystem.java @@ -378,4 +378,9 @@ public int getUploadTasksQueued() { public int getDownloadTasksQueued() { return fileDownloadExecutor.getQueue().size(); } + + @Override + public void clearCache() { + objectCache.invalidateAll(); + } } diff --git a/priam/src/main/java/com/netflix/priam/backup/IBackupFileSystem.java b/priam/src/main/java/com/netflix/priam/backup/IBackupFileSystem.java index 75bbdcc60..ef8935a05 100644 --- a/priam/src/main/java/com/netflix/priam/backup/IBackupFileSystem.java +++ b/priam/src/main/java/com/netflix/priam/backup/IBackupFileSystem.java @@ -210,4 +210,7 @@ default boolean checkObjectExists(Path remotePath) { * @return the total no. of tasks to be executed. */ int getDownloadTasksQueued(); + + /** Clear the cache for the backup file system, if any. */ + void clearCache(); } diff --git a/priam/src/main/java/com/netflix/priam/backupv2/BackupTTLTask.java b/priam/src/main/java/com/netflix/priam/backupv2/BackupTTLTask.java index 57494bbd4..a70a21290 100644 --- a/priam/src/main/java/com/netflix/priam/backupv2/BackupTTLTask.java +++ b/priam/src/main/java/com/netflix/priam/backupv2/BackupTTLTask.java @@ -88,12 +88,6 @@ public BackupTTLTask( @Override public void execute() throws Exception { - // Ensure that backup version 2.0 is actually enabled. - if (backupRestoreConfig.getSnapshotMetaServiceCronExpression().equalsIgnoreCase("-1")) { - logger.info("Not executing the TTL Task for backups as V2 backups are not enabled."); - return; - } - if (instanceState.getRestoreStatus() != null && instanceState.getRestoreStatus().getStatus() != null && instanceState.getRestoreStatus().getStatus() == Status.STARTED) { diff --git a/priam/src/main/java/com/netflix/priam/resources/BackupServletV2.java b/priam/src/main/java/com/netflix/priam/resources/BackupServletV2.java index e959a6892..d066cb117 100644 --- a/priam/src/main/java/com/netflix/priam/resources/BackupServletV2.java +++ b/priam/src/main/java/com/netflix/priam/resources/BackupServletV2.java @@ -22,9 +22,12 @@ import com.netflix.priam.backup.BackupVerification; import com.netflix.priam.backup.BackupVerificationResult; import com.netflix.priam.backup.BackupVersion; +import com.netflix.priam.backup.IBackupFileSystem; import com.netflix.priam.backup.IBackupStatusMgr; +import com.netflix.priam.backup.IFileSystemContext; import com.netflix.priam.backupv2.BackupTTLTask; import com.netflix.priam.backupv2.SnapshotMetaTask; +import com.netflix.priam.config.IConfiguration; import com.netflix.priam.utils.DateUtil; import com.netflix.priam.utils.DateUtil.DateRange; import java.time.Instant; @@ -51,6 +54,7 @@ public class BackupServletV2 { private final IBackupStatusMgr backupStatusMgr; private final SnapshotMetaTask snapshotMetaService; private final BackupTTLTask backupTTLService; + private final IBackupFileSystem fs; private static final String REST_SUCCESS = "[\"ok\"]"; @Inject @@ -58,11 +62,14 @@ public BackupServletV2( IBackupStatusMgr backupStatusMgr, BackupVerification backupVerification, SnapshotMetaTask snapshotMetaService, - BackupTTLTask backupTTLService) { + BackupTTLTask backupTTLService, + IConfiguration configuration, + IFileSystemContext backupFileSystemCtx) { this.backupStatusMgr = backupStatusMgr; this.backupVerification = backupVerification; this.snapshotMetaService = snapshotMetaService; this.backupTTLService = backupTTLService; + this.fs = backupFileSystemCtx.getFileStrategy(configuration); } @GET @@ -79,6 +86,13 @@ public Response ttl() throws Exception { return Response.ok(REST_SUCCESS, MediaType.APPLICATION_JSON).build(); } + @GET + @Path("/clearCache") + public Response clearCache() throws Exception { + fs.clearCache(); + return Response.ok(REST_SUCCESS, MediaType.APPLICATION_JSON).build(); + } + @GET @Path("/info/{date}") public Response info(@PathParam("date") String date) { From e617d47c8c19c64767d29b49fc56cc95db4018b5 Mon Sep 17 00:00:00 2001 From: Arun Agrawal Date: Fri, 5 Apr 2019 09:47:37 -0700 Subject: [PATCH 080/228] expose api to get list of files --- .../priam/backup/BackupRestoreUtil.java | 60 +++++++++++++++++++ .../netflix/priam/backupv2/BackupTTLTask.java | 6 +- .../backupv2/BackupVerificationTask.java | 2 +- .../priam/backupv2/ForgottenFilesManager.java | 1 - .../netflix/priam/connection/JMXNodeTool.java | 3 - .../priam/identity/InstanceIdentity.java | 4 +- .../priam/resources/BackupServletV2.java | 41 ++++++++++--- .../priam/restore/AbstractRestore.java | 55 ++--------------- .../priam/restore/PostRestoreHook.java | 8 +-- .../priam/scheduler/PriamScheduler.java | 11 ++-- .../priam/backup/TestBackupStatusMgr.java | 2 +- .../priam/backupv2/TestBackupTTLTask.java | 4 +- .../priam/backupv2/TestSnapshotMetaTask.java | 2 - 13 files changed, 113 insertions(+), 86 deletions(-) diff --git a/priam/src/main/java/com/netflix/priam/backup/BackupRestoreUtil.java b/priam/src/main/java/com/netflix/priam/backup/BackupRestoreUtil.java index ef21aff8d..2293a2694 100644 --- a/priam/src/main/java/com/netflix/priam/backup/BackupRestoreUtil.java +++ b/priam/src/main/java/com/netflix/priam/backup/BackupRestoreUtil.java @@ -19,8 +19,16 @@ import com.google.common.collect.ImmutableMap; import com.google.inject.Inject; +import com.google.inject.Provider; +import com.netflix.priam.backupv2.IMetaProxy; +import com.netflix.priam.backupv2.MetaV2Proxy; +import com.netflix.priam.utils.DateUtil; +import java.nio.file.Path; +import java.time.Instant; import java.util.*; import java.util.regex.Pattern; +import java.util.stream.Collectors; +import org.apache.commons.io.FileUtils; import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -52,6 +60,58 @@ public BackupRestoreUtil setFilters(String configIncludeFilter, String configExc return this; } + public static Optional getLatestValidMetaPath( + IMetaProxy metaProxy, DateUtil.DateRange dateRange) { + // Get a list of manifest files. + List metas = metaProxy.findMetaFiles(dateRange); + + // Find a valid manifest file. + for (AbstractBackupPath meta : metas) { + BackupVerificationResult result = metaProxy.isMetaFileValid(meta); + if (result.valid) { + return Optional.of(meta); + } + } + + return Optional.empty(); + } + + public static List getAllFiles( + AbstractBackupPath latestValidMetaFile, + DateUtil.DateRange dateRange, + IMetaProxy metaProxy, + Provider pathProvider) + throws Exception { + // Download the meta.json file. + Path metaFile = metaProxy.downloadMetaFile(latestValidMetaFile); + // Parse meta.json file to find the files required to download from this snapshot. + List allFiles = + metaProxy + .getSSTFilesFromMeta(metaFile) + .stream() + .map( + value -> { + AbstractBackupPath path = pathProvider.get(); + path.parseRemote(value); + return path; + }) + .collect(Collectors.toList()); + + FileUtils.deleteQuietly(metaFile.toFile()); + + // Download incremental SSTables after the snapshot meta file. + Instant snapshotTime; + if (metaProxy instanceof MetaV2Proxy) snapshotTime = latestValidMetaFile.getLastModified(); + else snapshotTime = latestValidMetaFile.getTime().toInstant(); + + DateUtil.DateRange incrementalDateRange = + new DateUtil.DateRange(snapshotTime, dateRange.getEndTime()); + Iterator incremental = metaProxy.getIncrementals(incrementalDateRange); + while (incremental.hasNext()) allFiles.add(incremental.next()); + + return allFiles; + } + public static final Map> getFilter(String inputFilter) throws IllegalArgumentException { if (StringUtils.isEmpty(inputFilter)) return null; diff --git a/priam/src/main/java/com/netflix/priam/backupv2/BackupTTLTask.java b/priam/src/main/java/com/netflix/priam/backupv2/BackupTTLTask.java index a70a21290..294d8a2be 100644 --- a/priam/src/main/java/com/netflix/priam/backupv2/BackupTTLTask.java +++ b/priam/src/main/java/com/netflix/priam/backupv2/BackupTTLTask.java @@ -48,7 +48,7 @@ * This class is used to TTL or delete the SSTable components from the backups after they are not * referenced in the backups for more than {@link IConfiguration#getBackupRetentionDays()}. This * operation is executed on CRON and is configured via {@link - * IBackupRestoreConfig#getBackupTTLCronExpression()}. + * IBackupRestoreConfig#getBackupTTLMonitorPeriodInSec()}. * *

    To TTL the SSTable components we refer to the first manifest file on the remote file system * after the TTL period. Any sstable components referenced in that manifest file should not be @@ -148,7 +148,7 @@ public void execute() throws Exception { new DateUtil.DateRange( start_of_feature, dateToTtl.minus(1, ChronoUnit.HOURS))); - if (metas != null) { + if (metas != null || metas.size() != 0) { logger.info( "Will delete(TTL) {} META files starting from: [{}]", metas.size(), @@ -241,7 +241,7 @@ public String getName() { */ public static TaskTimer getTimer(IBackupRestoreConfig backupRestoreConfig) throws Exception { return SimpleTimer.getSimpleTimer( - JOBNAME, backupRestoreConfig.getBackupTTLMonitorPeriodInSec()); + JOBNAME, backupRestoreConfig.getBackupTTLMonitorPeriodInSec() * 1000); } private class MetaFileWalker extends MetaFileReader { diff --git a/priam/src/main/java/com/netflix/priam/backupv2/BackupVerificationTask.java b/priam/src/main/java/com/netflix/priam/backupv2/BackupVerificationTask.java index 8cc9acdc5..6627a3261 100644 --- a/priam/src/main/java/com/netflix/priam/backupv2/BackupVerificationTask.java +++ b/priam/src/main/java/com/netflix/priam/backupv2/BackupVerificationTask.java @@ -90,7 +90,7 @@ public void execute() throws Exception { Optional verificationResult = backupVerification.verifyBackup( BackupVersion.SNAPSHOT_META_SERVICE, false, dateRange); - if (verificationResult.isPresent() && !verificationResult.get().valid) { + if (!verificationResult.isPresent() || !verificationResult.get().valid) { logger.error( "Not able to find any snapshot which is valid in our SLO window: {} hours", backupRestoreConfig.getBackupVerificationSLOInHours()); diff --git a/priam/src/main/java/com/netflix/priam/backupv2/ForgottenFilesManager.java b/priam/src/main/java/com/netflix/priam/backupv2/ForgottenFilesManager.java index d63ee32f9..d1ea10760 100644 --- a/priam/src/main/java/com/netflix/priam/backupv2/ForgottenFilesManager.java +++ b/priam/src/main/java/com/netflix/priam/backupv2/ForgottenFilesManager.java @@ -46,7 +46,6 @@ public class ForgottenFilesManager { private IConfiguration config; private static final String TMP_EXT = ".tmp"; - @SuppressWarnings("Annotator") private static final Pattern tmpFilePattern = Pattern.compile("^((.*)\\-(.*)\\-)?tmp(link)?\\-((?:l|k).)\\-(\\d)*\\-(.*)$"); diff --git a/priam/src/main/java/com/netflix/priam/connection/JMXNodeTool.java b/priam/src/main/java/com/netflix/priam/connection/JMXNodeTool.java index 2683d0ec4..96a546578 100644 --- a/priam/src/main/java/com/netflix/priam/connection/JMXNodeTool.java +++ b/priam/src/main/java/com/netflix/priam/connection/JMXNodeTool.java @@ -247,7 +247,6 @@ public JMXNodeTool retriableCall() throws Exception { * You must do the compaction before running this to remove the duplicate tokens out of the * server. TODO code it. */ - @SuppressWarnings("unchecked") public JSONObject estimateKeys() throws JSONException { Iterator> it = super.getColumnFamilyStoreMBeanProxies(); @@ -261,7 +260,6 @@ public JSONObject estimateKeys() throws JSONException { return object; } - @SuppressWarnings("unchecked") public JSONObject info() throws JSONException { JSONObject object = new JSONObject(); object.put("gossip_active", isInitialized()); @@ -280,7 +278,6 @@ public JSONObject info() throws JSONException { return object; } - @SuppressWarnings("unchecked") public JSONArray ring(String keyspace) throws JSONException { JSONArray ring = new JSONArray(); Map tokenToEndpoint = getTokenToEndpointMap(); diff --git a/priam/src/main/java/com/netflix/priam/identity/InstanceIdentity.java b/priam/src/main/java/com/netflix/priam/identity/InstanceIdentity.java index 31e0d8bbd..be89384e1 100644 --- a/priam/src/main/java/com/netflix/priam/identity/InstanceIdentity.java +++ b/priam/src/main/java/com/netflix/priam/identity/InstanceIdentity.java @@ -32,7 +32,6 @@ import com.netflix.priam.utils.RetryableCallable; import com.netflix.priam.utils.Sleeper; import java.net.UnknownHostException; -import java.util.Collection; import java.util.HashMap; import java.util.LinkedList; import java.util.List; @@ -49,8 +48,7 @@ public class InstanceIdentity { public static final String DUMMY_INSTANCE_ID = "new_slot"; private final ListMultimap locMap = - Multimaps.newListMultimap( - new HashMap>(), Lists::newArrayList); + Multimaps.newListMultimap(new HashMap<>(), Lists::newArrayList); private final IPriamInstanceFactory factory; private final IMembership membership; private final IConfiguration config; diff --git a/priam/src/main/java/com/netflix/priam/resources/BackupServletV2.java b/priam/src/main/java/com/netflix/priam/resources/BackupServletV2.java index d066cb117..032d001f7 100644 --- a/priam/src/main/java/com/netflix/priam/resources/BackupServletV2.java +++ b/priam/src/main/java/com/netflix/priam/resources/BackupServletV2.java @@ -18,14 +18,10 @@ package com.netflix.priam.resources; import com.google.inject.Inject; -import com.netflix.priam.backup.BackupMetadata; -import com.netflix.priam.backup.BackupVerification; -import com.netflix.priam.backup.BackupVerificationResult; -import com.netflix.priam.backup.BackupVersion; -import com.netflix.priam.backup.IBackupFileSystem; -import com.netflix.priam.backup.IBackupStatusMgr; -import com.netflix.priam.backup.IFileSystemContext; +import com.google.inject.Provider; +import com.netflix.priam.backup.*; import com.netflix.priam.backupv2.BackupTTLTask; +import com.netflix.priam.backupv2.IMetaProxy; import com.netflix.priam.backupv2.SnapshotMetaTask; import com.netflix.priam.config.IConfiguration; import com.netflix.priam.utils.DateUtil; @@ -34,6 +30,8 @@ import java.time.temporal.ChronoUnit; import java.util.List; import java.util.Optional; +import java.util.stream.Collectors; +import javax.inject.Named; import javax.ws.rs.DefaultValue; import javax.ws.rs.GET; import javax.ws.rs.Path; @@ -55,6 +53,8 @@ public class BackupServletV2 { private final SnapshotMetaTask snapshotMetaService; private final BackupTTLTask backupTTLService; private final IBackupFileSystem fs; + private final IMetaProxy metaProxy; + private final Provider pathProvider; private static final String REST_SUCCESS = "[\"ok\"]"; @Inject @@ -64,12 +64,16 @@ public BackupServletV2( SnapshotMetaTask snapshotMetaService, BackupTTLTask backupTTLService, IConfiguration configuration, - IFileSystemContext backupFileSystemCtx) { + IFileSystemContext backupFileSystemCtx, + @Named("v2") IMetaProxy metaV2Proxy, + Provider pathProvider) { this.backupStatusMgr = backupStatusMgr; this.backupVerification = backupVerification; this.snapshotMetaService = snapshotMetaService; this.backupTTLService = backupTTLService; this.fs = backupFileSystemCtx.getFileStrategy(configuration); + this.metaProxy = metaV2Proxy; + this.pathProvider = pathProvider; } @GET @@ -124,4 +128,25 @@ public Response validateV2SnapshotByDate( return Response.ok(result.get()).build(); } + + @GET + @Path("/list/{daterange}") + public Response list(@PathParam("daterange") String daterange) throws Exception { + DateUtil.DateRange dateRange = new DateUtil.DateRange(daterange); + // Find latest valid meta file. + Optional latestValidMetaFile = + BackupRestoreUtil.getLatestValidMetaPath(metaProxy, dateRange); + if (!latestValidMetaFile.isPresent()) { + return Response.ok("No valid meta found!").build(); + } + List allFiles = + BackupRestoreUtil.getAllFiles( + latestValidMetaFile.get(), dateRange, metaProxy, pathProvider); + + return Response.ok( + allFiles.stream() + .map(AbstractBackupPath::getRemotePath) + .collect(Collectors.toList())) + .build(); + } } diff --git a/priam/src/main/java/com/netflix/priam/restore/AbstractRestore.java b/priam/src/main/java/com/netflix/priam/restore/AbstractRestore.java index 574005351..d69352f87 100644 --- a/priam/src/main/java/com/netflix/priam/restore/AbstractRestore.java +++ b/priam/src/main/java/com/netflix/priam/restore/AbstractRestore.java @@ -33,12 +33,10 @@ import java.io.IOException; import java.math.BigInteger; import java.nio.file.Path; -import java.time.Instant; import java.time.LocalDateTime; import java.time.ZoneId; import java.util.*; import java.util.concurrent.Future; -import java.util.stream.Collectors; import javax.inject.Named; import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.io.FileUtils; @@ -191,22 +189,6 @@ public Void retriableCall() throws Exception { }.call(); } - private Optional getLatestValidMetaPath( - IMetaProxy metaProxy, DateUtil.DateRange dateRange) { - // Get a list of manifest files. - List metas = metaProxy.findMetaFiles(dateRange); - - // Find a valid manifest file. - for (AbstractBackupPath meta : metas) { - BackupVerificationResult result = metaProxy.isMetaFileValid(meta); - if (result.valid) { - return Optional.of(meta); - } - } - - return Optional.empty(); - } - public void restore(DateUtil.DateRange dateRange) throws Exception { // fail early if post restore hook has invalid parameters if (!postRestoreHook.hasValidParameters()) { @@ -246,7 +228,7 @@ public void restore(DateUtil.DateRange dateRange) throws Exception { // Find latest valid meta file. Optional latestValidMetaFile = - getLatestValidMetaPath(metaProxy, dateRange); + BackupRestoreUtil.getLatestValidMetaPath(metaProxy, dateRange); if (!latestValidMetaFile.isPresent()) { logger.info("No valid snapshot meta file found, Restore Failed."); @@ -261,40 +243,13 @@ public void restore(DateUtil.DateRange dateRange) throws Exception { .getRestoreStatus() .setSnapshotMetaFile(latestValidMetaFile.get().getRemotePath()); - // Download the meta.json file. - Path metaFile = metaProxy.downloadMetaFile(latestValidMetaFile.get()); - // Parse meta.json file to find the files required to download from this snapshot. - List snapshots = - metaProxy - .getSSTFilesFromMeta(metaFile) - .stream() - .map( - value -> { - AbstractBackupPath path = pathProvider.get(); - path.parseRemote(value); - return path; - }) - .collect(Collectors.toList()); - - FileUtils.deleteQuietly(metaFile.toFile()); + List allFiles = + BackupRestoreUtil.getAllFiles( + latestValidMetaFile.get(), dateRange, metaProxy, pathProvider); // Download snapshot which is listed in the meta file. List> futureList = new ArrayList<>(); - futureList.addAll(download(snapshots.iterator(), false)); - - logger.info("Downloading incrementals"); - - // Download incrementals (SST) after the snapshot meta file. - Instant snapshotTime; - if (backupRestoreConfig.enableV2Restore()) - snapshotTime = latestValidMetaFile.get().getLastModified(); - else snapshotTime = latestValidMetaFile.get().getTime().toInstant(); - - DateUtil.DateRange incrementalDateRange = - new DateUtil.DateRange(snapshotTime, dateRange.getEndTime()); - Iterator incrementals = - metaProxy.getIncrementals(incrementalDateRange); - futureList.addAll(download(incrementals, false)); + futureList.addAll(download(allFiles.iterator(), false)); // Downloading CommitLogs // Note for Backup V2.0 we do not backup commit logs, as saving them is cost-expensive. diff --git a/priam/src/main/java/com/netflix/priam/restore/PostRestoreHook.java b/priam/src/main/java/com/netflix/priam/restore/PostRestoreHook.java index 18f40b01f..1e1c2009c 100644 --- a/priam/src/main/java/com/netflix/priam/restore/PostRestoreHook.java +++ b/priam/src/main/java/com/netflix/priam/restore/PostRestoreHook.java @@ -59,11 +59,9 @@ public PostRestoreHook(IConfiguration config, Sleeper sleeper) { */ public boolean hasValidParameters() { if (config.isPostRestoreHookEnabled()) { - if (StringUtils.isBlank(config.getPostRestoreHook()) - || StringUtils.isBlank(config.getPostRestoreHookHeartbeatFileName()) - || StringUtils.isBlank(config.getPostRestoreHookDoneFileName())) { - return false; - } + return !StringUtils.isBlank(config.getPostRestoreHook()) + && !StringUtils.isBlank(config.getPostRestoreHookHeartbeatFileName()) + && !StringUtils.isBlank(config.getPostRestoreHookDoneFileName()); } return true; } diff --git a/priam/src/main/java/com/netflix/priam/scheduler/PriamScheduler.java b/priam/src/main/java/com/netflix/priam/scheduler/PriamScheduler.java index 5c61120f3..5aac652c7 100644 --- a/priam/src/main/java/com/netflix/priam/scheduler/PriamScheduler.java +++ b/priam/src/main/java/com/netflix/priam/scheduler/PriamScheduler.java @@ -52,7 +52,7 @@ public void addTask(String name, Class taskclass, TaskTimer time JobBuilder.newJob() .withIdentity(name, Scheduler.DEFAULT_GROUP) .ofType(taskclass) - .build(); // new JobDetail(name, Scheduler.DEFAULT_GROUP, taskclass); + .build(); if (timer.getCronExpression() != null && !timer.getCronExpression().isEmpty()) { logger.info( "Scheduled task metadata. Task name: {}" + ", cron expression: {}", @@ -70,14 +70,13 @@ public void addTaskWithDelay( final String name, Class taskclass, final TaskTimer timer, - final int delayInSeconds) - throws SchedulerException, ParseException { + final int delayInSeconds) { assert timer != null : "Cannot add scheduler task " + name + " as no timer is set"; final JobDetail job = JobBuilder.newJob() .withIdentity(name, Scheduler.DEFAULT_GROUP) .ofType(taskclass) - .build(); // new JobDetail(name, Scheduler.DEFAULT_GROUP, taskclass); + .build(); // we know Priam doesn't do too many new tasks, so this is probably easy/safe/simple new Thread( @@ -104,8 +103,8 @@ public void runTaskNow(Class taskclass) throws Exception { jobFactory.guice.getInstance(taskclass).execute(null); } - public void deleteTask(String name) throws SchedulerException, ParseException { - scheduler.deleteJob(new JobKey(name, Scheduler.DEFAULT_GROUP)); + public void deleteTask(String name) throws SchedulerException { + scheduler.unscheduleJob(new TriggerKey(name, Scheduler.DEFAULT_GROUP)); } public final Scheduler getScheduler() { diff --git a/priam/src/test/java/com/netflix/priam/backup/TestBackupStatusMgr.java b/priam/src/test/java/com/netflix/priam/backup/TestBackupStatusMgr.java index e92733e26..b7635a59a 100644 --- a/priam/src/test/java/com/netflix/priam/backup/TestBackupStatusMgr.java +++ b/priam/src/test/java/com/netflix/priam/backup/TestBackupStatusMgr.java @@ -247,6 +247,6 @@ public void getLatestBackupMetadata() throws Exception { backupStatusMgr.getLatestBackupMetadata( BackupVersion.SNAPSHOT_BACKUP, new DateRange(backupDate + "," + "201812031000")); - list.forEach(backupMetadata -> System.out.println(backupMetadata)); + list.forEach(System.out::println); } } diff --git a/priam/src/test/java/com/netflix/priam/backupv2/TestBackupTTLTask.java b/priam/src/test/java/com/netflix/priam/backupv2/TestBackupTTLTask.java index 13e33e30b..74577b11f 100644 --- a/priam/src/test/java/com/netflix/priam/backupv2/TestBackupTTLTask.java +++ b/priam/src/test/java/com/netflix/priam/backupv2/TestBackupTTLTask.java @@ -143,9 +143,7 @@ public void cleanup() { private List getAllFiles() { List remoteFiles = new ArrayList<>(); - backupFileSystem - .listFileSystem("", null, null) - .forEachRemaining(file -> remoteFiles.add(file)); + backupFileSystem.listFileSystem("", null, null).forEachRemaining(remoteFiles::add); return remoteFiles; } diff --git a/priam/src/test/java/com/netflix/priam/backupv2/TestSnapshotMetaTask.java b/priam/src/test/java/com/netflix/priam/backupv2/TestSnapshotMetaTask.java index 02f5f02d4..ffb253154 100644 --- a/priam/src/test/java/com/netflix/priam/backupv2/TestSnapshotMetaTask.java +++ b/priam/src/test/java/com/netflix/priam/backupv2/TestSnapshotMetaTask.java @@ -44,7 +44,6 @@ public class TestSnapshotMetaTask { private final IBackupRestoreConfig backupRestoreConfig; private final SnapshotMetaTask snapshotMetaService; private final TestMetaFileReader metaFileReader; - private final PrefixGenerator prefixGenerator; private final InstanceInfo instanceInfo; public TestSnapshotMetaTask() { @@ -54,7 +53,6 @@ public TestSnapshotMetaTask() { backupRestoreConfig = injector.getInstance(IBackupRestoreConfig.class); snapshotMetaService = injector.getInstance(SnapshotMetaTask.class); metaFileReader = new TestMetaFileReader(); - prefixGenerator = injector.getInstance(PrefixGenerator.class); instanceInfo = injector.getInstance(InstanceInfo.class); } From 8f7a2f9000b89300274ffd9da6f25cfd25b0fd42 Mon Sep 17 00:00:00 2001 From: Arun Agrawal Date: Wed, 27 Mar 2019 20:04:34 -0700 Subject: [PATCH 081/228] Disable backups --- .../java/com/netflix/priam/PriamServer.java | 17 ++- .../netflix/priam/backup/AbstractBackup.java | 38 ++++++ .../netflix/priam/backup/BackupService.java | 49 +++++--- .../priam/backup/IncrementalBackup.java | 43 ++++++- .../netflix/priam/backup/SnapshotBackup.java | 32 ++++- .../priam/backupv2/BackupV2Service.java | 47 +++++--- .../priam/backupv2/SnapshotMetaTask.java | 54 +++++---- .../netflix/priam/connection/JMXNodeTool.java | 8 ++ .../netflix/priam/defaultimpl/IService.java | 52 +++++++- .../identity/token/NewTokenRetriever.java | 2 +- .../priam/resources/BackupServlet.java | 20 +++- .../priam/resources/BackupServletV2.java | 13 +- .../priam/scheduler/PriamScheduler.java | 17 ++- .../priam/tuner/CassandraTunerService.java | 48 ++++++++ .../netflix/priam/tuner/StandardTuner.java | 16 +-- .../netflix/priam/tuner/TuneCassandra.java | 4 +- .../com/netflix/priam/tuner/dse/DseTuner.java | 4 +- .../priam/backup/TestAbstractFileSystem.java | 2 +- .../priam/backup/TestBackupService.java | 109 +++++++++++++++-- .../priam/backupv2/TestBackupV2Service.java | 112 +++++++++++++++++- .../priam/backupv2/TestSnapshotMetaTask.java | 5 +- .../priam/tuner/StandardTunerTest.java | 9 +- .../netflix/priam/utils/BackupFileUtils.java | 5 +- 23 files changed, 609 insertions(+), 97 deletions(-) create mode 100644 priam/src/main/java/com/netflix/priam/tuner/CassandraTunerService.java diff --git a/priam/src/main/java/com/netflix/priam/PriamServer.java b/priam/src/main/java/com/netflix/priam/PriamServer.java index 29ff2d93f..5145680bd 100644 --- a/priam/src/main/java/com/netflix/priam/PriamServer.java +++ b/priam/src/main/java/com/netflix/priam/PriamServer.java @@ -31,7 +31,7 @@ import com.netflix.priam.identity.InstanceIdentity; import com.netflix.priam.restore.RestoreContext; import com.netflix.priam.scheduler.PriamScheduler; -import com.netflix.priam.tuner.TuneCassandra; +import com.netflix.priam.tuner.CassandraTunerService; import com.netflix.priam.utils.Sleeper; import com.netflix.priam.utils.SystemUtils; import java.io.IOException; @@ -49,6 +49,7 @@ public class PriamServer implements IService { private final RestoreContext restoreContext; private final IService backupV2Service; private final IService backupService; + private final IService cassandraTunerService; private static final int CASSANDRA_MONITORING_INITIAL_DELAY = 10; private static final Logger logger = LoggerFactory.getLogger(PriamServer.class); @@ -61,7 +62,8 @@ public PriamServer( ICassandraProcess cassProcess, RestoreContext restoreContext, BackupService backupService, - BackupV2Service backupV2Service) { + BackupV2Service backupV2Service, + CassandraTunerService cassandraTunerService) { this.config = config; this.scheduler = scheduler; this.instanceIdentity = id; @@ -70,6 +72,7 @@ public PriamServer( this.restoreContext = restoreContext; this.backupService = backupService; this.backupV2Service = backupV2Service; + this.cassandraTunerService = cassandraTunerService; } private void createDirectories() throws IOException { @@ -107,8 +110,8 @@ public void scheduleService() throws Exception { UpdateSecuritySettings.getTimer(instanceIdentity)); } - // Run the task to tune Cassandra - scheduler.runTaskNow(TuneCassandra.class); + // Set up cassandra tuning. + cassandraTunerService.scheduleService(); // Determine if we need to restore from backup else start cassandra. if (restoreContext.isRestoreEnabled()) { @@ -151,6 +154,12 @@ public void scheduleService() throws Exception { backupV2Service.scheduleService(); } + @Override + public void updateServicePre() throws Exception {} + + @Override + public void updateServicePost() throws Exception {} + public InstanceIdentity getInstanceIdentity() { return instanceIdentity; } diff --git a/priam/src/main/java/com/netflix/priam/backup/AbstractBackup.java b/priam/src/main/java/com/netflix/priam/backup/AbstractBackup.java index 3f17facd8..c073c9a4c 100644 --- a/priam/src/main/java/com/netflix/priam/backup/AbstractBackup.java +++ b/priam/src/main/java/com/netflix/priam/backup/AbstractBackup.java @@ -24,10 +24,14 @@ import com.netflix.priam.scheduler.Task; import com.netflix.priam.utils.SystemUtils; import java.io.File; +import java.nio.file.DirectoryStream; +import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; import java.text.ParseException; +import java.util.HashSet; import java.util.List; +import java.util.Set; import java.util.concurrent.Future; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -169,6 +173,40 @@ protected final void initiateBackup( protected abstract void processColumnFamily( String keyspace, String columnFamily, File backupDir) throws Exception; + /** + * Get all the backup directories for Cassandra. + * + * @param config to get the location of the data folder. + * @param monitoringFolder folder where cassandra backup's are configured. + * @return Set of the path(s) containing the backup folder for each columnfamily. + * @throws Exception incase of IOException. + */ + public static Set getBackupDirectories(IConfiguration config, String monitoringFolder) + throws Exception { + HashSet backupPaths = new HashSet<>(); + if (config.getDataFileLocation() == null) return backupPaths; + Path dataPath = Paths.get(config.getDataFileLocation()); + if (Files.exists(dataPath) && Files.isDirectory(dataPath)) + try (DirectoryStream directoryStream = + Files.newDirectoryStream(dataPath, path -> Files.isDirectory(path))) { + for (Path keyspaceDirPath : directoryStream) { + try (DirectoryStream keyspaceStream = + Files.newDirectoryStream( + keyspaceDirPath, path -> Files.isDirectory(path))) { + for (Path columnfamilyDirPath : keyspaceStream) { + Path backupDirPath = + Paths.get(columnfamilyDirPath.toString(), monitoringFolder); + if (Files.exists(backupDirPath) && Files.isDirectory(backupDirPath)) { + logger.debug("Backup folder: {}", backupDirPath); + backupPaths.add(backupDirPath); + } + } + } + } + } + return backupPaths; + } + /** Filters unwanted keyspaces */ private boolean isValidBackupDir(File keyspaceDir, File backupDir) { if (backupDir == null || !backupDir.isDirectory() || !backupDir.exists()) return false; diff --git a/priam/src/main/java/com/netflix/priam/backup/BackupService.java b/priam/src/main/java/com/netflix/priam/backup/BackupService.java index 983b537f7..7a16d4cba 100644 --- a/priam/src/main/java/com/netflix/priam/backup/BackupService.java +++ b/priam/src/main/java/com/netflix/priam/backup/BackupService.java @@ -19,49 +19,68 @@ import com.google.inject.Inject; import com.netflix.priam.aws.UpdateCleanupPolicy; +import com.netflix.priam.config.IBackupRestoreConfig; import com.netflix.priam.config.IConfiguration; import com.netflix.priam.defaultimpl.IService; -import com.netflix.priam.identity.InstanceIdentity; import com.netflix.priam.scheduler.PriamScheduler; -import org.apache.commons.collections4.CollectionUtils; +import com.netflix.priam.scheduler.TaskTimer; +import com.netflix.priam.tuner.CassandraTunerService; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -/** Created by aagrawal on 3/9/19. */ +/** + * Encapsulate the backup service 1.0 - Execute all the tasks required to run backup service. + * + *

    Created by aagrawal on 3/9/19. + */ public class BackupService implements IService { private final PriamScheduler scheduler; private final IConfiguration config; - private final InstanceIdentity instanceIdentity; + private final IBackupRestoreConfig backupRestoreConfig; + private final CassandraTunerService cassandraTunerService; private static final Logger logger = LoggerFactory.getLogger(BackupService.class); @Inject public BackupService( IConfiguration config, + IBackupRestoreConfig backupRestoreConfig, PriamScheduler priamScheduler, - InstanceIdentity instanceIdentity) { + CassandraTunerService cassandraTunerService) { this.config = config; + this.backupRestoreConfig = backupRestoreConfig; this.scheduler = priamScheduler; - this.instanceIdentity = instanceIdentity; + this.cassandraTunerService = cassandraTunerService; } @Override public void scheduleService() throws Exception { // Start the snapshot backup schedule - Always run this. (If you want to // set it off, set backup hour to -1) or set backup cron to "-1" - if (SnapshotBackup.getTimer(config) != null - && (CollectionUtils.isEmpty(config.getBackupRacs()) - || config.getBackupRacs() - .contains(instanceIdentity.getInstanceInfo().getRac()))) { - scheduleTask(scheduler, SnapshotBackup.class, SnapshotBackup.getTimer(config)); + TaskTimer snapshotTimer = SnapshotBackup.getTimer(config); + scheduleTask(scheduler, SnapshotBackup.class, snapshotTimer); - // Start the Incremental backup schedule - scheduleTask(scheduler, IncrementalBackup.class, IncrementalBackup.getTimer(config)); + if (snapshotTimer != null) { + // Schedule commit log task + scheduleTask( + scheduler, CommitLogBackupTask.class, CommitLogBackupTask.getTimer(config)); } - // Schedule commit log task - scheduleTask(scheduler, CommitLogBackupTask.class, CommitLogBackupTask.getTimer(config)); + // Start the Incremental backup schedule if enabled + scheduleTask( + scheduler, + IncrementalBackup.class, + IncrementalBackup.getTimer(config, backupRestoreConfig)); // Set cleanup scheduleTask(scheduler, UpdateCleanupPolicy.class, UpdateCleanupPolicy.getTimer()); } + + @Override + public void updateServicePre() throws Exception { + // Run the task to tune Cassandra + cassandraTunerService.onChangeUpdateService(); + } + + @Override + public void updateServicePost() throws Exception {} } diff --git a/priam/src/main/java/com/netflix/priam/backup/IncrementalBackup.java b/priam/src/main/java/com/netflix/priam/backup/IncrementalBackup.java index f364372bb..a32fff28d 100644 --- a/priam/src/main/java/com/netflix/priam/backup/IncrementalBackup.java +++ b/priam/src/main/java/com/netflix/priam/backup/IncrementalBackup.java @@ -20,13 +20,17 @@ import com.google.inject.Provider; import com.google.inject.Singleton; import com.netflix.priam.backup.AbstractBackupPath.BackupFileType; +import com.netflix.priam.backupv2.SnapshotMetaTask; import com.netflix.priam.config.IBackupRestoreConfig; import com.netflix.priam.config.IConfiguration; import com.netflix.priam.scheduler.SimpleTimer; import com.netflix.priam.scheduler.TaskTimer; import com.netflix.priam.utils.DateUtil; import java.io.File; +import java.nio.file.Path; import java.util.List; +import java.util.Set; +import org.apache.commons.io.FileUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -65,11 +69,46 @@ public void execute() throws Exception { } /** Run every 10 Sec */ - public static TaskTimer getTimer(IConfiguration config) { - if (config.isIncrementalBackupEnabled()) return new SimpleTimer(JOBNAME, 10L * 1000); + public static TaskTimer getTimer( + IConfiguration config, IBackupRestoreConfig backupRestoreConfig) { + if (IncrementalBackup.isEnabled(config, backupRestoreConfig)) + return new SimpleTimer(JOBNAME, 10L * 1000); return null; } + private static void cleanOldBackups(IConfiguration configuration) throws Exception { + Set backupPaths = + AbstractBackup.getBackupDirectories(configuration, INCREMENTAL_BACKUP_FOLDER); + for (Path backupDirPath : backupPaths) { + FileUtils.cleanDirectory(backupDirPath.toFile()); + } + } + + public static boolean isEnabled( + IConfiguration configuration, IBackupRestoreConfig backupRestoreConfig) { + boolean enabled = false; + try { + // Once backup 1.0 is gone, we should not check for enableV2Backups. + enabled = + (configuration.isIncrementalBackupEnabled() + && (SnapshotBackup.isBackupEnabled(configuration) + || (backupRestoreConfig.enableV2Backups() + && SnapshotMetaTask.isBackupEnabled( + configuration, backupRestoreConfig)))); + logger.info("Incremental backups are enabled: {}", enabled); + + if (!enabled) { + // Clean up the incremental backup folder. + cleanOldBackups(configuration); + } + } catch (Exception e) { + logger.error( + "Error while trying to find if incremental backup is enabled: " + + e.getMessage()); + } + return enabled; + } + @Override public String getName() { return JOBNAME; diff --git a/priam/src/main/java/com/netflix/priam/backup/SnapshotBackup.java b/priam/src/main/java/com/netflix/priam/backup/SnapshotBackup.java index 46b21ab19..5024b102d 100644 --- a/priam/src/main/java/com/netflix/priam/backup/SnapshotBackup.java +++ b/priam/src/main/java/com/netflix/priam/backup/SnapshotBackup.java @@ -31,10 +31,14 @@ import com.netflix.priam.utils.DateUtil; import com.netflix.priam.utils.ThreadSleeper; import java.io.File; +import java.nio.file.DirectoryStream; +import java.nio.file.Files; +import java.nio.file.Path; import java.time.Instant; import java.util.*; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; +import org.apache.commons.io.FileUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -96,6 +100,8 @@ public void execute() throws Exception { } try { + // Clean up all the backup directories, if any. + cleanOldBackups(config); executeSnapshot(); } finally { lock.unlock(); @@ -169,7 +175,25 @@ public static boolean isBackupEnabled(IConfiguration config) throws Exception { } public static TaskTimer getTimer(IConfiguration config) throws Exception { - return CronTimer.getCronTimer(JOBNAME, config.getBackupCronExpression()); + TaskTimer timer = CronTimer.getCronTimer(JOBNAME, config.getBackupCronExpression()); + if (timer == null) { + // Clean up all the backup directories, if any. + cleanOldBackups(config); + } + return timer; + } + + private static void cleanOldBackups(IConfiguration configuration) throws Exception { + Set backupPaths = AbstractBackup.getBackupDirectories(configuration, SNAPSHOT_FOLDER); + for (Path backupDirPath : backupPaths) + try (DirectoryStream directoryStream = + Files.newDirectoryStream(backupDirPath, path -> Files.isDirectory(path))) { + for (Path backupDir : directoryStream) { + if (isValidBackupDir(backupDir)) { + FileUtils.deleteDirectory(backupDir.toFile()); + } + } + } } @Override @@ -188,4 +212,10 @@ protected void processColumnFamily(String keyspace, String columnFamily, File ba abstractBackupPaths.addAll( upload(snapshotDir, BackupFileType.SNAP, config.enableAsyncSnapshot(), true)); } + + private static boolean isValidBackupDir(Path backupDir) { + String backupDirName = backupDir.toFile().getName(); + // Check if it of format yyyyMMddHHmm + return (DateUtil.getDate(backupDirName) != null); + } } diff --git a/priam/src/main/java/com/netflix/priam/backupv2/BackupV2Service.java b/priam/src/main/java/com/netflix/priam/backupv2/BackupV2Service.java index 7548ed6de..9b74047cb 100644 --- a/priam/src/main/java/com/netflix/priam/backupv2/BackupV2Service.java +++ b/priam/src/main/java/com/netflix/priam/backupv2/BackupV2Service.java @@ -24,15 +24,20 @@ import com.netflix.priam.defaultimpl.IService; import com.netflix.priam.scheduler.PriamScheduler; import com.netflix.priam.scheduler.TaskTimer; +import com.netflix.priam.tuner.CassandraTunerService; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -/** Created by aagrawal on 3/9/19. */ +/** + * Encapsulate the backup service 2.0 - Execute all the tasks required to run backup service. + * Created by aagrawal on 3/9/19. + */ public class BackupV2Service implements IService { private final PriamScheduler scheduler; private final IConfiguration configuration; private final IBackupRestoreConfig backupRestoreConfig; private final SnapshotMetaTask snapshotMetaTask; + private final CassandraTunerService cassandraTunerService; private static final Logger logger = LoggerFactory.getLogger(BackupV2Service.class); @Inject @@ -40,19 +45,21 @@ public BackupV2Service( IConfiguration configuration, IBackupRestoreConfig backupRestoreConfig, PriamScheduler scheduler, - SnapshotMetaTask snapshotMetaService) { + SnapshotMetaTask snapshotMetaService, + CassandraTunerService cassandraTunerService) { this.configuration = configuration; this.backupRestoreConfig = backupRestoreConfig; this.scheduler = scheduler; this.snapshotMetaTask = snapshotMetaService; + this.cassandraTunerService = cassandraTunerService; } @Override public void scheduleService() throws Exception { - TaskTimer snapshotMetaTimer = SnapshotMetaTask.getTimer(backupRestoreConfig); - if (snapshotMetaTimer != null) { - scheduleTask(scheduler, SnapshotMetaTask.class, snapshotMetaTimer); + TaskTimer snapshotMetaTimer = SnapshotMetaTask.getTimer(configuration, backupRestoreConfig); + scheduleTask(scheduler, SnapshotMetaTask.class, snapshotMetaTimer); + if (snapshotMetaTimer != null) { // Try to upload previous snapshots, if any which might have been interrupted by Priam // restart. snapshotMetaTask.uploadFiles(); @@ -66,18 +73,24 @@ public void scheduleService() throws Exception { scheduler, BackupVerificationTask.class, BackupVerificationTask.getTimer(backupRestoreConfig)); - - // Start the Incremental backup schedule if enabled - if (backupRestoreConfig.enableV2Backups()) { - // Delete the old task, if scheduled. This is required, as we stop taking backup - // 1.0, we still want to take incremental backups - // Once backup 1.0 is gone, we should not check for enableV2Backups. - scheduler.deleteTask(IncrementalBackup.JOBNAME); - scheduleTask( - scheduler, - IncrementalBackup.class, - IncrementalBackup.getTimer(configuration)); - } + } else { + scheduler.deleteTask(BackupTTLTask.JOBNAME); + scheduler.deleteTask(BackupVerificationTask.JOBNAME); } + + // Start the Incremental backup schedule if enabled + scheduleTask( + scheduler, + IncrementalBackup.class, + IncrementalBackup.getTimer(configuration, backupRestoreConfig)); } + + @Override + public void updateServicePre() throws Exception { + // Update the cassandra to enable/disable new incremental files. + cassandraTunerService.onChangeUpdateService(); + } + + @Override + public void updateServicePost() throws Exception {} } diff --git a/priam/src/main/java/com/netflix/priam/backupv2/SnapshotMetaTask.java b/priam/src/main/java/com/netflix/priam/backupv2/SnapshotMetaTask.java index 758e72ce5..aea7b8238 100644 --- a/priam/src/main/java/com/netflix/priam/backupv2/SnapshotMetaTask.java +++ b/priam/src/main/java/com/netflix/priam/backupv2/SnapshotMetaTask.java @@ -28,6 +28,9 @@ import com.netflix.priam.scheduler.TaskTimer; import com.netflix.priam.utils.DateUtil; import java.io.File; +import java.nio.file.DirectoryStream; +import java.nio.file.Files; +import java.nio.file.Path; import java.nio.file.Paths; import java.time.Instant; import java.util.*; @@ -38,8 +41,6 @@ import javax.inject.Singleton; import org.apache.commons.io.FileUtils; import org.apache.commons.io.filefilter.FileFilterUtils; -import org.apache.commons.lang3.StringUtils; -import org.quartz.CronExpression; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -109,31 +110,40 @@ private enum MetaStep { * @param backupRestoreConfig {@link * IBackupRestoreConfig#getSnapshotMetaServiceCronExpression()} to get configuration details * from priam. Use "-1" to disable the service. + * @param config configuration to get the data folder. * @return the timer to be used for snapshot meta service. * @throws Exception if the configuration is not set correctly or are not valid. This is to * ensure we fail-fast. */ - public static TaskTimer getTimer(IBackupRestoreConfig backupRestoreConfig) throws Exception { - CronTimer cronTimer = null; - String cronExpression = backupRestoreConfig.getSnapshotMetaServiceCronExpression(); - - if (!StringUtils.isEmpty(cronExpression) && cronExpression.equalsIgnoreCase("-1")) { - logger.info( - "Skipping SnapshotMetaService as SnapshotMetaService cron is disabled via -1."); - } else { - if (StringUtils.isEmpty(cronExpression) - || !CronExpression.isValidExpression(cronExpression)) - throw new Exception( - "Invalid CRON expression: " - + cronExpression - + ". Please use -1, if you wish to disable SnapshotMetaService else fix the CRON expression and try again!"); - - cronTimer = new CronTimer(JOBNAME, cronExpression); - logger.info( - "Starting SnapshotMetaService with CRON expression {}", - cronTimer.getCronExpression()); + public static TaskTimer getTimer( + IConfiguration config, IBackupRestoreConfig backupRestoreConfig) throws Exception { + TaskTimer timer = + CronTimer.getCronTimer( + JOBNAME, backupRestoreConfig.getSnapshotMetaServiceCronExpression()); + if (timer == null) { + cleanOldBackups(config); } - return cronTimer; + return timer; + } + + private static void cleanOldBackups(IConfiguration config) throws Exception { + // Clean up all the backup directories, if any. + Set backupPaths = AbstractBackup.getBackupDirectories(config, SNAPSHOT_FOLDER); + for (Path backupDirPath : backupPaths) + try (DirectoryStream directoryStream = + Files.newDirectoryStream(backupDirPath, path -> Files.isDirectory(path))) { + for (Path backupDir : directoryStream) { + if (backupDir.toFile().getName().startsWith(SNAPSHOT_PREFIX)) { + FileUtils.deleteDirectory(backupDir.toFile()); + } + } + } + } + + public static boolean isBackupEnabled( + IConfiguration configuration, IBackupRestoreConfig backupRestoreConfig) + throws Exception { + return (getTimer(configuration, backupRestoreConfig) != null); } String generateSnapshotName(Instant snapshotInstant) { diff --git a/priam/src/main/java/com/netflix/priam/connection/JMXNodeTool.java b/priam/src/main/java/com/netflix/priam/connection/JMXNodeTool.java index 96a546578..bae91c90f 100644 --- a/priam/src/main/java/com/netflix/priam/connection/JMXNodeTool.java +++ b/priam/src/main/java/com/netflix/priam/connection/JMXNodeTool.java @@ -379,6 +379,14 @@ public void cleanup() throws IOException, ExecutionException, InterruptedExcepti for (String keyspace : getKeyspaces()) forceKeyspaceCleanup(0, keyspace); } + public void setIncrementalBackupsEnabled(boolean enabled) { + super.setIncrementalBackupsEnabled(enabled); + } + + public boolean isIncrementalBackupsEnabled() { + return super.isIncrementalBackupsEnabled(); + } + public void refresh(List keyspaces) throws IOException, ExecutionException, InterruptedException { Iterator> it = diff --git a/priam/src/main/java/com/netflix/priam/defaultimpl/IService.java b/priam/src/main/java/com/netflix/priam/defaultimpl/IService.java index c905879a8..dddf8e10f 100644 --- a/priam/src/main/java/com/netflix/priam/defaultimpl/IService.java +++ b/priam/src/main/java/com/netflix/priam/defaultimpl/IService.java @@ -23,15 +23,63 @@ import java.text.ParseException; import org.quartz.SchedulerException; -/** Created by aagrawal on 3/9/19. */ +/** + * This is how we create a new service in Priam. Any service we start, should implement this + * interface so we can update the service at run-time if required. + * + *

    Created by aagrawal on 3/9/19. + */ public interface IService { + /** + * This method is called to schedule the service when we initialize it for first time ONLY. + * + * @throws Exception if there is any error while trying to schedule the service. + */ void scheduleService() throws Exception; + /** + * This method is called before we try to update the service. Use this method to do any kind of + * maintenance operations before we change the scheduling of all the jobs in service. + * + * @throws Exception if there is any error in pre-hook method of service. + */ + void updateServicePre() throws Exception; + + /** + * This method is called after we re-schedule all the services in PriamScheduler. Use this + * method for post hook maintenance operations after changing the scehdule of all the jobs. + * + * @throws Exception if there is any error in post-hook method of service. + */ + void updateServicePost() throws Exception; + + /** + * Schedule a given task. It will safely delete that task from scheduler before scheduling. + * + * @param priamScheduler Scheduler in use by Priam. + * @param task Task that needs to be scheduled in priamScheduler + * @param taskTimer Timer for the task + * @throws SchedulerException If there is any error in deleting the task or scheduling a new + * job. + * @throws ParseException If there is any error in parsing the taskTimer while trying to add a + * new job to scheduler. + */ default void scheduleTask( PriamScheduler priamScheduler, Class task, TaskTimer taskTimer) throws SchedulerException, ParseException { + priamScheduler.deleteTask(task.getName()); if (taskTimer == null) return; - priamScheduler.addTask(task.getName(), task, taskTimer); } + + /** + * Update the service. This method will be called to update the service while Priam is running. + * + * @throws Exception if any issue while updating the service. + */ + default void onChangeUpdateService() throws Exception { + updateServicePre(); + scheduleService(); + updateServicePost(); + } } diff --git a/priam/src/main/java/com/netflix/priam/identity/token/NewTokenRetriever.java b/priam/src/main/java/com/netflix/priam/identity/token/NewTokenRetriever.java index c94471b48..e0e32127c 100755 --- a/priam/src/main/java/com/netflix/priam/identity/token/NewTokenRetriever.java +++ b/priam/src/main/java/com/netflix/priam/identity/token/NewTokenRetriever.java @@ -63,7 +63,7 @@ public PriamInstance get() throws Exception { // Sleep random interval - upto 15 sec sleeper.sleep(new Random().nextInt(15000)); int hash = tokenManager.regionOffset(instanceInfo.getRegion()); - // use this hash so that the nodes are spred far away from the other + // use this hash so that the nodes are spread far away from the other // regions. int max = hash; diff --git a/priam/src/main/java/com/netflix/priam/resources/BackupServlet.java b/priam/src/main/java/com/netflix/priam/resources/BackupServlet.java index a54296472..31c9efdcc 100644 --- a/priam/src/main/java/com/netflix/priam/resources/BackupServlet.java +++ b/priam/src/main/java/com/netflix/priam/resources/BackupServlet.java @@ -21,6 +21,7 @@ import com.netflix.priam.backup.*; import com.netflix.priam.backup.AbstractBackupPath.BackupFileType; import com.netflix.priam.backup.BackupVersion; +import com.netflix.priam.config.IBackupRestoreConfig; import com.netflix.priam.config.IConfiguration; import com.netflix.priam.scheduler.PriamScheduler; import com.netflix.priam.utils.DateUtil; @@ -47,25 +48,31 @@ public class BackupServlet { private static final String REST_HEADER_RANGE = "daterange"; private static final String REST_HEADER_FILTER = "filter"; private final IConfiguration config; + private final IBackupRestoreConfig backupRestoreConfig; private final IBackupFileSystem backupFs; private final SnapshotBackup snapshotBackup; private final BackupVerification backupVerification; @Inject private PriamScheduler scheduler; private final IBackupStatusMgr completedBkups; + private final BackupService backupService; @Inject private MetaData metaData; @Inject public BackupServlet( IConfiguration config, + IBackupRestoreConfig backupRestoreConfig, @Named("backup") IBackupFileSystem backupFs, SnapshotBackup snapshotBackup, IBackupStatusMgr completedBkups, - BackupVerification backupVerification) { + BackupVerification backupVerification, + BackupService backupService) { this.config = config; + this.backupRestoreConfig = backupRestoreConfig; this.backupFs = backupFs; this.snapshotBackup = snapshotBackup; this.completedBkups = completedBkups; this.backupVerification = backupVerification; + this.backupService = backupService; } @GET @@ -79,7 +86,16 @@ public Response backup() throws Exception { @Path("/incremental_backup") public Response backupIncrementals() throws Exception { scheduler.addTask( - "IncrementalBackup", IncrementalBackup.class, IncrementalBackup.getTimer(config)); + "IncrementalBackup", + IncrementalBackup.class, + IncrementalBackup.getTimer(config, backupRestoreConfig)); + return Response.ok(REST_SUCCESS, MediaType.APPLICATION_JSON).build(); + } + + @GET + @Path("/updateService") + public Response updateService() throws Exception { + backupService.onChangeUpdateService(); return Response.ok(REST_SUCCESS, MediaType.APPLICATION_JSON).build(); } diff --git a/priam/src/main/java/com/netflix/priam/resources/BackupServletV2.java b/priam/src/main/java/com/netflix/priam/resources/BackupServletV2.java index 032d001f7..64ccff9e7 100644 --- a/priam/src/main/java/com/netflix/priam/resources/BackupServletV2.java +++ b/priam/src/main/java/com/netflix/priam/resources/BackupServletV2.java @@ -21,6 +21,7 @@ import com.google.inject.Provider; import com.netflix.priam.backup.*; import com.netflix.priam.backupv2.BackupTTLTask; +import com.netflix.priam.backupv2.BackupV2Service; import com.netflix.priam.backupv2.IMetaProxy; import com.netflix.priam.backupv2.SnapshotMetaTask; import com.netflix.priam.config.IConfiguration; @@ -55,6 +56,7 @@ public class BackupServletV2 { private final IBackupFileSystem fs; private final IMetaProxy metaProxy; private final Provider pathProvider; + private final BackupV2Service backupService; private static final String REST_SUCCESS = "[\"ok\"]"; @Inject @@ -66,7 +68,8 @@ public BackupServletV2( IConfiguration configuration, IFileSystemContext backupFileSystemCtx, @Named("v2") IMetaProxy metaV2Proxy, - Provider pathProvider) { + Provider pathProvider, + BackupV2Service backupService) { this.backupStatusMgr = backupStatusMgr; this.backupVerification = backupVerification; this.snapshotMetaService = snapshotMetaService; @@ -74,6 +77,7 @@ public BackupServletV2( this.fs = backupFileSystemCtx.getFileStrategy(configuration); this.metaProxy = metaV2Proxy; this.pathProvider = pathProvider; + this.backupService = backupService; } @GET @@ -97,6 +101,13 @@ public Response clearCache() throws Exception { return Response.ok(REST_SUCCESS, MediaType.APPLICATION_JSON).build(); } + @GET + @Path("/updateService") + public Response updateService() throws Exception { + backupService.onChangeUpdateService(); + return Response.ok(REST_SUCCESS, MediaType.APPLICATION_JSON).build(); + } + @GET @Path("/info/{date}") public Response info(@PathParam("date") String date) { diff --git a/priam/src/main/java/com/netflix/priam/scheduler/PriamScheduler.java b/priam/src/main/java/com/netflix/priam/scheduler/PriamScheduler.java index 5aac652c7..20fe13082 100644 --- a/priam/src/main/java/com/netflix/priam/scheduler/PriamScheduler.java +++ b/priam/src/main/java/com/netflix/priam/scheduler/PriamScheduler.java @@ -104,7 +104,22 @@ public void runTaskNow(Class taskclass) throws Exception { } public void deleteTask(String name) throws SchedulerException { - scheduler.unscheduleJob(new TriggerKey(name, Scheduler.DEFAULT_GROUP)); + TriggerKey triggerKey = TriggerKey.triggerKey(name, Scheduler.DEFAULT_GROUP); + + // Check if trigger exists for the job. If there is a trigger, we want to remove those + // trigger. + if (scheduler.checkExists(triggerKey)) { + logger.info("Removing triggers for the job: {}", name); + scheduler.pauseTrigger(triggerKey); + scheduler.unscheduleJob(triggerKey); + } + + // Check if any job exists for the key provided. If yes, we want to delete the job. + JobKey jobKey = JobKey.jobKey(name, Scheduler.DEFAULT_GROUP); + if (scheduler.checkExists(jobKey)) { + logger.info("Removing job from scheduler: {}", name); + scheduler.deleteJob(jobKey); + } } public final Scheduler getScheduler() { diff --git a/priam/src/main/java/com/netflix/priam/tuner/CassandraTunerService.java b/priam/src/main/java/com/netflix/priam/tuner/CassandraTunerService.java new file mode 100644 index 000000000..d04ba7927 --- /dev/null +++ b/priam/src/main/java/com/netflix/priam/tuner/CassandraTunerService.java @@ -0,0 +1,48 @@ +package com.netflix.priam.tuner; + +import com.netflix.priam.backup.IncrementalBackup; +import com.netflix.priam.config.IBackupRestoreConfig; +import com.netflix.priam.config.IConfiguration; +import com.netflix.priam.connection.JMXNodeTool; +import com.netflix.priam.defaultimpl.IService; +import com.netflix.priam.scheduler.PriamScheduler; +import com.netflix.priam.utils.RetryableCallable; +import javax.inject.Inject; + +public class CassandraTunerService implements IService { + private final PriamScheduler scheduler; + private final IConfiguration configuration; + private final IBackupRestoreConfig backupRestoreConfig; + + @Inject + public CassandraTunerService( + PriamScheduler priamScheduler, + IConfiguration configuration, + IBackupRestoreConfig backupRestoreConfig) { + this.scheduler = priamScheduler; + this.configuration = configuration; + this.backupRestoreConfig = backupRestoreConfig; + } + + @Override + public void scheduleService() throws Exception { + // Run the task to tune Cassandra + scheduler.runTaskNow(TuneCassandra.class); + } + + @Override + public void updateServicePre() throws Exception {} + + @Override + public void updateServicePost() throws Exception { + // Update the cassandra to enable/disable new incremental files. + new RetryableCallable(6, 10000) { + public Void retriableCall() throws Exception { + JMXNodeTool nodetool = JMXNodeTool.instance(configuration); + nodetool.setIncrementalBackupsEnabled( + IncrementalBackup.isEnabled(configuration, backupRestoreConfig)); + return null; + } + }.call(); + } +} diff --git a/priam/src/main/java/com/netflix/priam/tuner/StandardTuner.java b/priam/src/main/java/com/netflix/priam/tuner/StandardTuner.java index c0746ae78..8c27f48ef 100644 --- a/priam/src/main/java/com/netflix/priam/tuner/StandardTuner.java +++ b/priam/src/main/java/com/netflix/priam/tuner/StandardTuner.java @@ -15,7 +15,8 @@ import com.google.common.collect.Lists; import com.google.inject.Inject; -import com.netflix.priam.backup.SnapshotBackup; +import com.netflix.priam.backup.IncrementalBackup; +import com.netflix.priam.config.IBackupRestoreConfig; import com.netflix.priam.config.IConfiguration; import com.netflix.priam.identity.config.InstanceInfo; import com.netflix.priam.restore.Restore; @@ -24,7 +25,6 @@ import java.util.List; import java.util.Map; import java.util.Properties; -import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -38,11 +38,16 @@ public class StandardTuner implements ICassandraTuner { private static final Logger logger = LoggerFactory.getLogger(StandardTuner.class); protected final IConfiguration config; + protected final IBackupRestoreConfig backupRestoreConfig; private final InstanceInfo instanceInfo; @Inject - public StandardTuner(IConfiguration config, InstanceInfo instanceInfo) { + public StandardTuner( + IConfiguration config, + IBackupRestoreConfig backupRestoreConfig, + InstanceInfo instanceInfo) { this.config = config; + this.backupRestoreConfig = backupRestoreConfig; this.instanceInfo = instanceInfo; } @@ -74,10 +79,7 @@ public void writeAllProperties(String yamlLocation, String hostname, String seed map.put("hints_directory", config.getHintsLocation()); map.put("data_file_directories", Lists.newArrayList(config.getDataFileLocation())); - boolean enableIncremental = - (SnapshotBackup.isBackupEnabled(config) && config.isIncrementalBackupEnabled()) - && (CollectionUtils.isEmpty(config.getBackupRacs()) - || config.getBackupRacs().contains(instanceInfo.getRac())); + boolean enableIncremental = IncrementalBackup.isEnabled(config, backupRestoreConfig); map.put("incremental_backups", enableIncremental); map.put("endpoint_snitch", config.getSnitch()); diff --git a/priam/src/main/java/com/netflix/priam/tuner/TuneCassandra.java b/priam/src/main/java/com/netflix/priam/tuner/TuneCassandra.java index a225cdfde..eacbd6a4f 100644 --- a/priam/src/main/java/com/netflix/priam/tuner/TuneCassandra.java +++ b/priam/src/main/java/com/netflix/priam/tuner/TuneCassandra.java @@ -57,14 +57,14 @@ public void execute() throws Exception { isDone = true; instanceState.setYmlWritten(true); } catch (IOException e) { - LOGGER.error("Fail wrting cassandra.yml file. Retry again!", e); + LOGGER.error("Fail writing cassandra.yml file. Retry again!", e); } } } @Override public String getName() { - return "Tune-Cassandra"; + return JOBNAME; } public static TaskTimer getTimer() { diff --git a/priam/src/main/java/com/netflix/priam/tuner/dse/DseTuner.java b/priam/src/main/java/com/netflix/priam/tuner/dse/DseTuner.java index d5e11cf24..047d06052 100644 --- a/priam/src/main/java/com/netflix/priam/tuner/dse/DseTuner.java +++ b/priam/src/main/java/com/netflix/priam/tuner/dse/DseTuner.java @@ -17,6 +17,7 @@ import static org.apache.cassandra.locator.SnitchProperties.RACKDC_PROPERTY_FILENAME; import com.google.inject.Inject; +import com.netflix.priam.config.IBackupRestoreConfig; import com.netflix.priam.config.IConfiguration; import com.netflix.priam.identity.config.InstanceInfo; import com.netflix.priam.tuner.StandardTuner; @@ -42,10 +43,11 @@ public class DseTuner extends StandardTuner { @Inject public DseTuner( IConfiguration config, + IBackupRestoreConfig backupRestoreConfig, IDseConfiguration dseConfig, IAuditLogTuner auditLogTuner, InstanceInfo instanceInfo) { - super(config, instanceInfo); + super(config, backupRestoreConfig, instanceInfo); this.dseConfig = dseConfig; this.auditLogTuner = auditLogTuner; } diff --git a/priam/src/test/java/com/netflix/priam/backup/TestAbstractFileSystem.java b/priam/src/test/java/com/netflix/priam/backup/TestAbstractFileSystem.java index 7bfeab6df..695caf80b 100644 --- a/priam/src/test/java/com/netflix/priam/backup/TestAbstractFileSystem.java +++ b/priam/src/test/java/com/netflix/priam/backup/TestAbstractFileSystem.java @@ -101,7 +101,7 @@ private Collection generateFiles(int noOfKeyspaces, int noOfCf, int noOfSs throws Exception { Path dataDir = Paths.get(configuration.getDataFileLocation()); BackupFileUtils.generateDummyFiles( - dataDir, noOfKeyspaces, noOfCf, noOfSstables, "snapshot", "201812310000"); + dataDir, noOfKeyspaces, noOfCf, noOfSstables, "snapshot", "201812310000", true); String[] ext = {"db"}; return FileUtils.listFiles(dataDir.toFile(), ext, true); } diff --git a/priam/src/test/java/com/netflix/priam/backup/TestBackupService.java b/priam/src/test/java/com/netflix/priam/backup/TestBackupService.java index a752c5205..2e6171eb4 100644 --- a/priam/src/test/java/com/netflix/priam/backup/TestBackupService.java +++ b/priam/src/test/java/com/netflix/priam/backup/TestBackupService.java @@ -19,10 +19,20 @@ import com.google.inject.Guice; import com.google.inject.Injector; +import com.netflix.priam.config.IBackupRestoreConfig; import com.netflix.priam.config.IConfiguration; +import com.netflix.priam.connection.JMXNodeTool; import com.netflix.priam.defaultimpl.IService; -import com.netflix.priam.identity.InstanceIdentity; import com.netflix.priam.scheduler.PriamScheduler; +import com.netflix.priam.tuner.CassandraTunerService; +import com.netflix.priam.tuner.TuneCassandra; +import com.netflix.priam.utils.BackupFileUtils; +import com.netflix.priam.utils.DateUtil; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.time.Instant; +import java.util.Set; import mockit.Expectations; import mockit.Mocked; import org.junit.Assert; @@ -33,12 +43,12 @@ /** Created by aagrawal on 3/10/19. */ public class TestBackupService { private final PriamScheduler scheduler; - private final InstanceIdentity instanceIdentity; + private final CassandraTunerService cassandraTunerService; public TestBackupService() { Injector injector = Guice.createInjector(new BRTestModule()); this.scheduler = injector.getInstance(PriamScheduler.class); - this.instanceIdentity = injector.getInstance(InstanceIdentity.class); + this.cassandraTunerService = injector.getInstance(CassandraTunerService.class); } @Before @@ -47,20 +57,62 @@ public void cleanup() throws SchedulerException { } @Test - public void testBackupDisabled(@Mocked IConfiguration configuration) throws Exception { + public void testBackupDisabled( + @Mocked IConfiguration configuration, @Mocked IBackupRestoreConfig backupRestoreConfig) + throws Exception { new Expectations() { { configuration.getBackupCronExpression(); result = "-1"; + configuration.getDataFileLocation(); + result = "target/data"; } }; - IService backupService = new BackupService(configuration, scheduler, instanceIdentity); + + Path dummyDataDirectoryLocation = Paths.get(configuration.getDataFileLocation()); + Instant snapshotInstant = DateUtil.getInstant(); + + // Create one V1 snapshot. + String snapshotV1Name = DateUtil.formatInstant(DateUtil.yyyyMMdd, snapshotInstant); + BackupFileUtils.generateDummyFiles( + dummyDataDirectoryLocation, + 2, + 3, + 3, + AbstractBackup.SNAPSHOT_FOLDER, + snapshotV1Name, + true); + + String snapshotName = "meta_v2_" + snapshotV1Name; + // Create one V2 snapshot. + BackupFileUtils.generateDummyFiles( + dummyDataDirectoryLocation, + 2, + 3, + 3, + AbstractBackup.SNAPSHOT_FOLDER, + snapshotName, + false); + + IService backupService = + new BackupService( + configuration, backupRestoreConfig, scheduler, cassandraTunerService); backupService.scheduleService(); Assert.assertEquals(1, scheduler.getScheduler().getJobKeys(null).size()); + + // snapshot V1 name should not be there. + Set backupPaths = + AbstractBackup.getBackupDirectories(configuration, AbstractBackup.SNAPSHOT_FOLDER); + for (Path backupPath : backupPaths) { + Assert.assertTrue(Files.exists(Paths.get(backupPath.toString(), snapshotName))); + Assert.assertFalse(Files.exists(Paths.get(backupPath.toString(), snapshotV1Name))); + } } @Test - public void testBackupEnabled(@Mocked IConfiguration configuration) throws Exception { + public void testBackupEnabled( + @Mocked IConfiguration configuration, @Mocked IBackupRestoreConfig backupRestoreConfig) + throws Exception { new Expectations() { { configuration.getBackupCronExpression(); @@ -69,24 +121,63 @@ public void testBackupEnabled(@Mocked IConfiguration configuration) throws Excep result = false; } }; - IService backupService = new BackupService(configuration, scheduler, instanceIdentity); + IService backupService = + new BackupService( + configuration, backupRestoreConfig, scheduler, cassandraTunerService); backupService.scheduleService(); Assert.assertEquals(2, scheduler.getScheduler().getJobKeys(null).size()); } @Test - public void testBackupEnabledWithIncremental(@Mocked IConfiguration configuration) + public void testBackupEnabledWithIncremental( + @Mocked IConfiguration configuration, @Mocked IBackupRestoreConfig backupRestoreConfig) + throws Exception { + new Expectations() { + { + configuration.getBackupCronExpression(); + result = "0 0/1 * 1/1 * ? *"; + configuration.isIncrementalBackupEnabled(); + result = true; + } + }; + IService backupService = + new BackupService( + configuration, backupRestoreConfig, scheduler, cassandraTunerService); + backupService.scheduleService(); + Assert.assertEquals(3, scheduler.getScheduler().getJobKeys(null).size()); + } + + @Test + public void updateService( + @Mocked IConfiguration configuration, + @Mocked IBackupRestoreConfig backupRestoreConfig, + @Mocked JMXNodeTool nodeTool, + @Mocked TuneCassandra tuneCassandra) throws Exception { new Expectations() { { configuration.getBackupCronExpression(); result = "0 0/1 * 1/1 * ? *"; + result = "0 0/1 * 1/1 * ? *"; + result = "-1"; + result = "-1"; configuration.isIncrementalBackupEnabled(); result = true; + backupRestoreConfig.enableV2Backups(); + result = true; + backupRestoreConfig.getSnapshotMetaServiceCronExpression(); + result = "0 0/1 * 1/1 * ? *"; } }; - IService backupService = new BackupService(configuration, scheduler, instanceIdentity); + IService backupService = + new BackupService( + configuration, backupRestoreConfig, scheduler, cassandraTunerService); backupService.scheduleService(); Assert.assertEquals(3, scheduler.getScheduler().getJobKeys(null).size()); + + System.out.println("After updated"); + backupService.onChangeUpdateService(); + System.out.println(scheduler.getScheduler().getJobKeys(null)); + Assert.assertEquals(2, scheduler.getScheduler().getJobKeys(null).size()); } } diff --git a/priam/src/test/java/com/netflix/priam/backupv2/TestBackupV2Service.java b/priam/src/test/java/com/netflix/priam/backupv2/TestBackupV2Service.java index af0322f96..d69e779c0 100644 --- a/priam/src/test/java/com/netflix/priam/backupv2/TestBackupV2Service.java +++ b/priam/src/test/java/com/netflix/priam/backupv2/TestBackupV2Service.java @@ -19,11 +19,22 @@ import com.google.inject.Guice; import com.google.inject.Injector; +import com.netflix.priam.backup.AbstractBackup; import com.netflix.priam.backup.BRTestModule; import com.netflix.priam.config.IBackupRestoreConfig; import com.netflix.priam.config.IConfiguration; +import com.netflix.priam.connection.JMXNodeTool; import com.netflix.priam.defaultimpl.IService; import com.netflix.priam.scheduler.PriamScheduler; +import com.netflix.priam.tuner.CassandraTunerService; +import com.netflix.priam.tuner.TuneCassandra; +import com.netflix.priam.utils.BackupFileUtils; +import com.netflix.priam.utils.DateUtil; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.time.Instant; +import java.util.Set; import mockit.Expectations; import mockit.Mocked; import org.junit.Assert; @@ -35,11 +46,13 @@ public class TestBackupV2Service { private final PriamScheduler scheduler; private final SnapshotMetaTask snapshotMetaTask; + private final CassandraTunerService cassandraTunerService; public TestBackupV2Service() { Injector injector = Guice.createInjector(new BRTestModule()); scheduler = injector.getInstance(PriamScheduler.class); snapshotMetaTask = injector.getInstance(SnapshotMetaTask.class); + cassandraTunerService = injector.getInstance(CassandraTunerService.class); } @Before @@ -51,17 +64,56 @@ public void cleanup() throws SchedulerException { public void testBackupDisabled( @Mocked IConfiguration configuration, @Mocked IBackupRestoreConfig backupRestoreConfig) throws Exception { + new Expectations() { { backupRestoreConfig.getSnapshotMetaServiceCronExpression(); result = "-1"; + configuration.getDataFileLocation(); + result = "target/data"; } }; + Path dummyDataDirectoryLocation = Paths.get(configuration.getDataFileLocation()); + Instant snapshotInstant = DateUtil.getInstant(); + String snapshotName = snapshotMetaTask.generateSnapshotName(snapshotInstant); + // Create one V2 snapshot. + BackupFileUtils.generateDummyFiles( + dummyDataDirectoryLocation, + 2, + 3, + 3, + AbstractBackup.SNAPSHOT_FOLDER, + snapshotName, + true); + + // Create one V1 snapshot. + String snapshotV1Name = DateUtil.formatInstant(DateUtil.yyyyMMdd, snapshotInstant); + BackupFileUtils.generateDummyFiles( + dummyDataDirectoryLocation, + 2, + 3, + 3, + AbstractBackup.SNAPSHOT_FOLDER, + snapshotV1Name, + false); + IService backupService = new BackupV2Service( - configuration, backupRestoreConfig, scheduler, snapshotMetaTask); + configuration, + backupRestoreConfig, + scheduler, + snapshotMetaTask, + cassandraTunerService); backupService.scheduleService(); Assert.assertTrue(scheduler.getScheduler().getJobGroupNames().isEmpty()); + + // snapshot V2 name should not be there. + Set backupPaths = + AbstractBackup.getBackupDirectories(configuration, AbstractBackup.SNAPSHOT_FOLDER); + for (Path backupPath : backupPaths) { + Assert.assertFalse(Files.exists(Paths.get(backupPath.toString(), snapshotName))); + Assert.assertTrue(Files.exists(Paths.get(backupPath.toString(), snapshotV1Name))); + } } @Test @@ -80,11 +132,17 @@ public void testBackupEnabled( result = true; configuration.isIncrementalBackupEnabled(); result = true; + configuration.getBackupCronExpression(); + result = "-1"; } }; IService backupService = new BackupV2Service( - configuration, backupRestoreConfig, scheduler, snapshotMetaTask); + configuration, + backupRestoreConfig, + scheduler, + snapshotMetaTask, + cassandraTunerService); backupService.scheduleService(); Assert.assertEquals(4, scheduler.getScheduler().getJobKeys(null).size()); } @@ -101,14 +159,60 @@ public void testBackup( result = 600; backupRestoreConfig.getBackupVerificationCronExpression(); result = "0 0 0/1 1/1 * ? *"; - backupRestoreConfig.enableV2Backups(); + configuration.isIncrementalBackupEnabled(); result = false; + configuration.getDataFileLocation(); + result = "target/data"; + } + }; + IService backupService = + new BackupV2Service( + configuration, + backupRestoreConfig, + scheduler, + snapshotMetaTask, + cassandraTunerService); + backupService.scheduleService(); + Assert.assertEquals(3, scheduler.getScheduler().getJobKeys(null).size()); + } + + @Test + public void updateService( + @Mocked IConfiguration configuration, + @Mocked IBackupRestoreConfig backupRestoreConfig, + @Mocked JMXNodeTool nodeTool, + @Mocked TuneCassandra tuneCassandra) + throws Exception { + new Expectations() { + { + backupRestoreConfig.getSnapshotMetaServiceCronExpression(); + result = "0 0 0/1 1/1 * ? *"; + result = "0 0 0/1 1/1 * ? *"; + result = "-1"; + result = "-1"; + configuration.isIncrementalBackupEnabled(); + result = true; + backupRestoreConfig.enableV2Backups(); + result = true; + backupRestoreConfig.getBackupVerificationCronExpression(); + result = "-1"; + backupRestoreConfig.getBackupTTLMonitorPeriodInSec(); + result = 600; + configuration.getBackupCronExpression(); + result = "-1"; } }; IService backupService = new BackupV2Service( - configuration, backupRestoreConfig, scheduler, snapshotMetaTask); + configuration, + backupRestoreConfig, + scheduler, + snapshotMetaTask, + cassandraTunerService); backupService.scheduleService(); Assert.assertEquals(3, scheduler.getScheduler().getJobKeys(null).size()); + + backupService.onChangeUpdateService(); + Assert.assertEquals(0, scheduler.getScheduler().getJobKeys(null).size()); } } diff --git a/priam/src/test/java/com/netflix/priam/backupv2/TestSnapshotMetaTask.java b/priam/src/test/java/com/netflix/priam/backupv2/TestSnapshotMetaTask.java index ffb253154..4ae4de2c1 100644 --- a/priam/src/test/java/com/netflix/priam/backupv2/TestSnapshotMetaTask.java +++ b/priam/src/test/java/com/netflix/priam/backupv2/TestSnapshotMetaTask.java @@ -64,7 +64,7 @@ public void setUp() { @Test public void testSnapshotMetaServiceEnabled() throws Exception { - TaskTimer taskTimer = SnapshotMetaTask.getTimer(backupRestoreConfig); + TaskTimer taskTimer = SnapshotMetaTask.getTimer(configuration, backupRestoreConfig); Assert.assertNotNull(taskTimer); } @@ -86,7 +86,8 @@ private void test(int noOfSstables, int noOfKeyspaces, int noOfCf) throws Except noOfCf, noOfSstables, AbstractBackup.SNAPSHOT_FOLDER, - snapshotName); + snapshotName, + true); snapshotMetaService.setSnapshotName(snapshotName); Path metaFileLocation = snapshotMetaService.processSnapshot(snapshotInstant).getMetaFilePath(); diff --git a/priam/src/test/java/com/netflix/priam/tuner/StandardTunerTest.java b/priam/src/test/java/com/netflix/priam/tuner/StandardTunerTest.java index 50c3d86ed..ec4d8efb8 100644 --- a/priam/src/test/java/com/netflix/priam/tuner/StandardTunerTest.java +++ b/priam/src/test/java/com/netflix/priam/tuner/StandardTunerTest.java @@ -21,7 +21,9 @@ import com.google.common.io.Files; import com.google.inject.Guice; import com.netflix.priam.backup.BRTestModule; +import com.netflix.priam.config.BackupRestoreConfig; import com.netflix.priam.config.FakeConfiguration; +import com.netflix.priam.config.IBackupRestoreConfig; import com.netflix.priam.identity.config.InstanceInfo; import java.io.File; import java.io.FileInputStream; @@ -44,12 +46,15 @@ public class StandardTunerTest { private final StandardTuner tuner; private final InstanceInfo instanceInfo; + private final IBackupRestoreConfig backupRestoreConfig; private final File target = new File("/tmp/priam_test.yaml"); public StandardTunerTest() { this.tuner = Guice.createInjector(new BRTestModule()).getInstance(StandardTuner.class); this.instanceInfo = Guice.createInjector(new BRTestModule()).getInstance(InstanceInfo.class); + this.backupRestoreConfig = + Guice.createInjector(new BRTestModule()).getInstance(BackupRestoreConfig.class); } @Test @@ -129,7 +134,9 @@ public void addExtraParams() throws Exception { extraParamValues.put(priamKeyName4, "randomGroupValue"); StandardTuner tuner = new StandardTuner( - new TunerConfiguration(extraConfigParam, extraParamValues), instanceInfo); + new TunerConfiguration(extraConfigParam, extraParamValues), + backupRestoreConfig, + instanceInfo); Files.copy(new File("src/main/resources/incr-restore-cassandra.yaml"), target); tuner.writeAllProperties(target.getAbsolutePath(), "your_host", "YourSeedProvider"); diff --git a/priam/src/test/java/com/netflix/priam/utils/BackupFileUtils.java b/priam/src/test/java/com/netflix/priam/utils/BackupFileUtils.java index 59e840227..24ae8be37 100644 --- a/priam/src/test/java/com/netflix/priam/utils/BackupFileUtils.java +++ b/priam/src/test/java/com/netflix/priam/utils/BackupFileUtils.java @@ -42,10 +42,11 @@ public static void generateDummyFiles( int noOfCf, int noOfSstables, String backupDir, - String snapshotName) + String snapshotName, + boolean cleanup) throws Exception { // Clean the dummy directory - cleanupDir(dummyDir); + if (cleanup) cleanupDir(dummyDir); for (int i = 1; i <= noOfKeyspaces; i++) { String keyspaceName = "sample" + i; From be58d241de37d5169988369ee60064c494dc16c4 Mon Sep 17 00:00:00 2001 From: Chandrasekhar Thumuluru Date: Fri, 26 Apr 2019 22:37:20 -0700 Subject: [PATCH 082/228] Priam will check Cassandra gossip information while grabbing pre-assigned token to decide if it should start Cassandra in bootstrap mode or in replace mode. Moved token owner inferring logic based on Cassandra gossip into a util class. Refactored InstanceIdentity.init() method. Fixed DeadTokenRetriever to use the IP from token database when minimum number of instances are not reachable. --- .../priam/identity/InstanceIdentity.java | 284 ++++++++++-------- .../identity/token/DeadTokenRetriever.java | 136 ++------- .../identity/token/TokenRetrieverUtils.java | 166 ++++++++++ .../token/AssignedTokenRetrieverTest.java | 264 ++++++++++++++++ ...iever.java => DeadTokenRetrieverTest.java} | 16 +- .../token/TokenRetrieverUtilsTest.java | 209 +++++++++++++ 6 files changed, 823 insertions(+), 252 deletions(-) create mode 100644 priam/src/main/java/com/netflix/priam/identity/token/TokenRetrieverUtils.java create mode 100644 priam/src/test/java/com/netflix/priam/identity/token/AssignedTokenRetrieverTest.java rename priam/src/test/java/com/netflix/priam/identity/token/{TestDeadTokenRetriever.java => DeadTokenRetrieverTest.java} (93%) create mode 100644 priam/src/test/java/com/netflix/priam/identity/token/TokenRetrieverUtilsTest.java diff --git a/priam/src/main/java/com/netflix/priam/identity/InstanceIdentity.java b/priam/src/main/java/com/netflix/priam/identity/InstanceIdentity.java index be89384e1..43abbf6d5 100644 --- a/priam/src/main/java/com/netflix/priam/identity/InstanceIdentity.java +++ b/priam/src/main/java/com/netflix/priam/identity/InstanceIdentity.java @@ -28,13 +28,18 @@ import com.netflix.priam.identity.token.IDeadTokenRetriever; import com.netflix.priam.identity.token.INewTokenRetriever; import com.netflix.priam.identity.token.IPreGeneratedTokenRetriever; +import com.netflix.priam.identity.token.TokenRetrieverUtils; +import com.netflix.priam.identity.token.TokenRetrieverUtils.GossipParseException; import com.netflix.priam.utils.ITokenManager; import com.netflix.priam.utils.RetryableCallable; import com.netflix.priam.utils.Sleeper; import java.net.UnknownHostException; +import java.util.Collections; import java.util.HashMap; import java.util.LinkedList; import java.util.List; +import java.util.Optional; +import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -90,8 +95,12 @@ public boolean apply(PriamInstance instance) { private final IPreGeneratedTokenRetriever preGeneratedTokenRetriever; private final INewTokenRetriever newTokenRetriever; + private final java.util.function.Predicate sameHostPredicate = + (i) -> i.getInstanceId().equals(myInstanceInfo.getInstanceId()); + @Inject - // Note: do not parameterized the generic type variable to an implementation as it confuses + // Note: do not parameterized the generic type variable to an implementation as + // it confuses // Guice in the binding. public InstanceIdentity( IPriamInstanceFactory factory, @@ -125,154 +134,167 @@ public InstanceInfo getInstanceInfo() { } public void init() throws Exception { - // try to grab the token which was already assigned - myInstance = - new RetryableCallable() { - @Override - public PriamInstance retriableCall() throws Exception { - // Check if this node is decommissioned - List deadInstances = - factory.getAllIds(config.getAppName() + "-dead"); - for (PriamInstance ins : deadInstances) { - logger.info( - "[Dead] Iterating though the hosts: {}", ins.getInstanceId()); - if (ins.getInstanceId().equals(myInstanceInfo.getInstanceId())) { - ins.setOutOfService(true); - logger.info( - "[Dead] found that this node is dead." - + " application: {}" - + ", id: {}" - + ", instance: {}" - + ", region: {}" - + ", host ip: {}" - + ", host name: {}" - + ", token: {}", - ins.getApp(), - ins.getId(), - ins.getInstanceId(), - ins.getDC(), - ins.getHostIP(), - ins.getHostName(), - ins.getToken()); - return ins; - } - } - List aliveInstances = factory.getAllIds(config.getAppName()); - for (PriamInstance ins : aliveInstances) { - logger.info( - "[Alive] Iterating though the hosts: {} My id = [{}]", - ins.getInstanceId(), - ins.getId()); - if (ins.getInstanceId().equals(myInstanceInfo.getInstanceId())) { - logger.info( - "[Alive] found that this node is alive." - + " application: {}" - + ", id: {}" - + ", instance: {}" - + ", region: {}" - + ", host ip: {}" - + ", host name: {}" - + ", token: {}", - ins.getApp(), - ins.getId(), - ins.getInstanceId(), - ins.getDC(), - ins.getHostIP(), - ins.getHostName(), - ins.getToken()); - return ins; - } - } - return null; - } - }.call(); - - // Grab a dead token - if (null == myInstance) { - myInstance = - new RetryableCallable() { - - @Override - public PriamInstance retriableCall() throws Exception { - PriamInstance result; - result = deadTokenRetriever.get(); - if (result != null) { - - isReplace = - true; // indicate that we are acquiring a dead instance's - // token - - if (deadTokenRetriever.getReplaceIp() - != null) { // The IP address of the dead instance to which - // we will acquire its token - replacedIp = deadTokenRetriever.getReplaceIp(); - } - } + // Grab the token which was preassigned. + logger.info("trying to grab preassigned token."); + myInstance = grabPreAssignedToken(); + + // Grab a dead token. + if (myInstance == null) { + logger.info("unable to grab preassigned token. trying to grab a dead token."); + myInstance = grabDeadToken(); + } - return result; - } + // Grab a pre-generated token if there is such one. + if (myInstance == null) { + logger.info("unable to grab a dead token. trying to grab a pregenerated token."); + myInstance = grabPreGeneratedToken(); + } - @Override - public void forEachExecution() { - populateRacMap(); - deadTokenRetriever.setLocMap(locMap); - } - }.call(); + // Grab a new token + if (myInstance == null) { + logger.info("unable to grab a pregenerated token. trying to grab a new token."); + myInstance = grabNewToken(); } - // Grab a pre-generated token if there is such one - if (null == myInstance) { + logger.info("My token: {}", myInstance.getToken()); + } - myInstance = - new RetryableCallable() { + private PriamInstance grabPreAssignedToken() throws Exception { + return new RetryableCallable() { + @Override + public PriamInstance retriableCall() throws Exception { + // Check if this node is decommissioned. + List deadInstances = + factory.getAllIds(config.getAppName() + "-dead"); + PriamInstance instance = + findInstance(deadInstances, sameHostPredicate).orElse(null); + if (instance != null) { + instance.setOutOfService(true); + } - @Override - public PriamInstance retriableCall() throws Exception { - PriamInstance result; - result = preGeneratedTokenRetriever.get(); - if (result != null) { - isTokenPregenerated = true; + if (instance == null) { + List aliveInstances = factory.getAllIds(config.getAppName()); + instance = findInstance(aliveInstances, sameHostPredicate).orElse(null); + + if (instance != null) { + instance.setOutOfService(false); + + // Priam might have crashed before bootstrapping Cassandra in replace mode. + // So, it is premature to use the assigned token without checking Cassandra + // gossip. + try { + String replaceIp = + TokenRetrieverUtils.inferTokenOwnerFromGossip( + aliveInstances, instance.getToken(), instance.getDC()); + if (!StringUtils.isEmpty(replaceIp) + && !replaceIp.equals(instance.getHostIP())) { + setReplacedIp(replaceIp); } - return result; + } catch (GossipParseException e) { } + } + } - @Override - public void forEachExecution() { - populateRacMap(); - preGeneratedTokenRetriever.setLocMap(locMap); - } - }.call(); - } + if (instance != null) { + logger.info( + "{} found that this node is {}." + + " application: {}," + + " id: {}," + + " instance: {}," + + " region: {}," + + " host ip: {}," + + " host name: {}," + + " token: {}", + instance.isOutOfService() ? "[Dead]" : "[Alive]", + instance.isOutOfService() ? "dead" : "alive", + instance.getApp(), + instance.getId(), + instance.getInstanceId(), + instance.getDC(), + instance.getHostIP(), + instance.getHostName(), + instance.getToken()); + } - // Grab a new token - if (null == myInstance) { + return instance; + } + }.call(); + } - if (this.config.isCreateNewTokenEnable()) { + private PriamInstance grabDeadToken() throws Exception { + return new RetryableCallable() { + @Override + public PriamInstance retriableCall() throws Exception { + PriamInstance result = deadTokenRetriever.get(); + if (result != null) { + isReplace = true; // indicate that we are acquiring a dead instance's token. + + if (deadTokenRetriever.getReplaceIp() + != null) { // The IP address of the dead instance to which + // we will acquire its token + replacedIp = deadTokenRetriever.getReplaceIp(); + } + } - myInstance = - new RetryableCallable() { + return result; + } - @Override - public PriamInstance retriableCall() throws Exception { - super.set(100, 100); - newTokenRetriever.setLocMap(locMap); - return newTokenRetriever.get(); - } + @Override + public void forEachExecution() { + populateRacMap(); + deadTokenRetriever.setLocMap(locMap); + } + }.call(); + } - @Override - public void forEachExecution() { - populateRacMap(); - newTokenRetriever.setLocMap(locMap); - } - }.call(); + private PriamInstance grabPreGeneratedToken() throws Exception { + return new RetryableCallable() { + @Override + public PriamInstance retriableCall() throws Exception { + PriamInstance result = preGeneratedTokenRetriever.get(); + if (result != null) { + isTokenPregenerated = true; + } + return result; + } - } else { - throw new IllegalStateException( - "Node attempted to erroneously create a new token when we should be grabbing an existing token."); + @Override + public void forEachExecution() { + populateRacMap(); + preGeneratedTokenRetriever.setLocMap(locMap); } + }.call(); + } + + private PriamInstance grabNewToken() throws Exception { + if (!this.config.isCreateNewTokenEnable()) { + throw new IllegalStateException( + "Node attempted to erroneously create a new token when we should be grabbing an existing token."); } - logger.info("My token: {}", myInstance.getToken()); + return new RetryableCallable() { + @Override + public PriamInstance retriableCall() throws Exception { + set(100, 100); + newTokenRetriever.setLocMap(locMap); + return newTokenRetriever.get(); + } + + @Override + public void forEachExecution() { + populateRacMap(); + newTokenRetriever.setLocMap(locMap); + } + }.call(); + } + + private Optional findInstance( + List instances, java.util.function.Predicate predicate) { + return Optional.ofNullable(instances) + .orElse(Collections.emptyList()) + .stream() + .filter(predicate) + .findFirst(); } private void populateRacMap() { diff --git a/priam/src/main/java/com/netflix/priam/identity/token/DeadTokenRetriever.java b/priam/src/main/java/com/netflix/priam/identity/token/DeadTokenRetriever.java index 733c663a6..c961fba1e 100755 --- a/priam/src/main/java/com/netflix/priam/identity/token/DeadTokenRetriever.java +++ b/priam/src/main/java/com/netflix/priam/identity/token/DeadTokenRetriever.java @@ -20,16 +20,10 @@ import com.netflix.priam.identity.IPriamInstanceFactory; import com.netflix.priam.identity.PriamInstance; import com.netflix.priam.identity.config.InstanceInfo; +import com.netflix.priam.identity.token.TokenRetrieverUtils.GossipParseException; import com.netflix.priam.utils.Sleeper; -import com.netflix.priam.utils.SystemUtils; -import java.util.Collections; import java.util.List; import java.util.Random; -import java.util.stream.Collectors; -import org.apache.commons.lang3.StringUtils; -import org.json.simple.JSONArray; -import org.json.simple.JSONObject; -import org.json.simple.parser.JSONParser; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -98,9 +92,10 @@ public PriamInstance get() throws Exception { if (!priamInstance.getRac().equals(instanceInfo.getRac()) || asgInstances.contains(priamInstance.getInstanceId()) || super.isInstanceDummy(priamInstance)) continue; - // TODO: If instance is in SHUTTING_DOWN mode, it might not show up in asg instances (if - // cloud control plane is having issues), thus, we should not try to replace the - // instance as it will lead to "Cannot replace a live node" issue. + // TODO: If instance is in SHUTTING_DOWN mode, it might not show up in asg + // instances (if cloud control plane is having issues), thus, we should not try + // to + // replace the instance as it will lead to "Cannot replace a live node" issue. logger.info("Found dead instance: {}", priamInstance.toString()); @@ -118,19 +113,25 @@ public PriamInstance get() throws Exception { factory.delete(priamInstance); // find the replaced IP - this.replacedIp = - findReplaceIp( - allInstancesWithinCluster, - priamInstance.getToken(), - priamInstance.getDC()); - // Lets not replace the instance if gossip info is not merging!! - if (replacedIp == null) return null; + try { + replacedIp = + TokenRetrieverUtils.inferTokenOwnerFromGossip( + allInstancesWithinCluster, + priamInstance.getToken(), + priamInstance.getDC()); - logger.info( - "Will try to replace token: {} with replacedIp (from gossip info): {} instead of ip from Token database: {}", - priamInstance.getToken(), - replacedIp, - priamInstance.getHostIP()); + // Lets not replace the instance if gossip info is not merging!! + if (replacedIp == null) return null; + + logger.info( + "Will try to replace token: {} with replacedIp (from gossip info): {} instead of ip from Token database: {}", + priamInstance.getToken(), + replacedIp, + priamInstance.getHostIP()); + } catch (GossipParseException e) { + // In case of gossip exception, fallback to IP in token database. + this.replacedIp = priamInstance.getHostIP(); + } PriamInstance result; try { @@ -174,97 +175,6 @@ public String getReplaceIp() { return this.replacedIp; } - private String findReplaceIp(List allIds, String token, String dc) - throws Exception { - // Avoid using dead instance who we are trying to replace (duh!!) - // Avoid other regions instances to avoid communication over public ip address. - List eligibleInstances = - allIds.parallelStream() - .filter(priamInstance -> !priamInstance.getToken().equalsIgnoreCase(token)) - .filter(priamInstance -> priamInstance.getDC().equalsIgnoreCase(dc)) - .collect(Collectors.toList()); - // We want to get IP from min 1, max 3 instances to ensure we are not relying on gossip of a - // single instance. - // Good idea to shuffle so we are not talking to same instances every time. - Collections.shuffle(eligibleInstances); - // Potential issue could be when you have about 50% of your cluster C* DOWN or trying to be - // replaced. - // Think of a major disaster hitting your cluster. In that scenario chances of instance - // hitting DOWN C* are much much higher. - // In such a case you should rely on @link{CassandraConfig#setReplacedIp}. - int noOfInstancesGossipShouldMatch = Math.max(1, Math.min(3, eligibleInstances.size())); - int noOfInstancesWithGossipMatch = 0; - String replace_ip = null, ip = null; - for (PriamInstance ins : eligibleInstances) { - logger.info("Calling getIp on hostname[{}] and token[{}]", ins.getHostName(), token); - ip = getIp(ins.getHostName(), token); - if (StringUtils.isEmpty(replace_ip)) replace_ip = ip; - if (!StringUtils.isEmpty(replace_ip) && !StringUtils.isEmpty(ip)) { - if (replace_ip.equalsIgnoreCase(ip)) { - noOfInstancesWithGossipMatch++; - if (noOfInstancesWithGossipMatch >= noOfInstancesGossipShouldMatch) { - logger.info( - "Using replace_ip: {} as # of required gossip info match: {}", - replace_ip, - noOfInstancesGossipShouldMatch); - return replace_ip; - } - } else - throw new Exception( - String.format( - "Unexpected Exception: Gossip info from hosts are not matching: found {} and {}", - replace_ip, - ip)); - } - } - logger.warn( - "Return null: Unable to find enough instances where gossip match. Required: {}", - noOfInstancesGossipShouldMatch); - return null; - } - - private String getIp(String host, String token) { - String response = null; - try { - response = SystemUtils.getDataFromUrl(getGossipInfoURL(host)); - - String inputToken = String.format("[%s]", token); - JSONParser parser = new JSONParser(); - JSONArray jsonObject = (JSONArray) parser.parse(response); - - for (Object key : jsonObject) { - JSONObject msg = (JSONObject) key; - - // Ensure that we are not trying to replace a NORMAL token and token of that - // instance matches what we want to replace. - if (msg.get("STATUS") == null - || msg.get("STATUS").toString().equalsIgnoreCase("NORMAL") - || msg.get("TOKENS") == null - || msg.get("PUBLIC_IP") == null - || !msg.get("TOKENS").toString().equals(inputToken)) { - continue; - } - - logger.info( - "Using gossip info from host[{}] and token[{}], the replaced address is : [{}]", - host, - token, - msg.get("PUBLIC_IP")); - return (String) msg.get("PUBLIC_IP"); - } - } catch (Exception e) { - logger.info( - "Error in reaching out to host: [{}} or parsing response from host: {}", - host, - response); - } - return null; - } - - private String getGossipInfoURL(String host) { - return "http://" + host + ":8080/Priam/REST/v1/cassadmin/gossipinfo"; - } - @Override public void setLocMap(ListMultimap locMap) { this.locMap = locMap; diff --git a/priam/src/main/java/com/netflix/priam/identity/token/TokenRetrieverUtils.java b/priam/src/main/java/com/netflix/priam/identity/token/TokenRetrieverUtils.java new file mode 100644 index 000000000..ff08d9d4e --- /dev/null +++ b/priam/src/main/java/com/netflix/priam/identity/token/TokenRetrieverUtils.java @@ -0,0 +1,166 @@ +package com.netflix.priam.identity.token; + +import com.netflix.priam.identity.PriamInstance; +import com.netflix.priam.utils.SystemUtils; +import java.util.Collections; +import java.util.List; +import java.util.stream.Collectors; +import org.apache.commons.lang3.StringUtils; +import org.json.simple.JSONArray; +import org.json.simple.JSONObject; +import org.json.simple.parser.JSONParser; +import org.json.simple.parser.ParseException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** Common utilities for token retrieval. */ +public class TokenRetrieverUtils { + private static final Logger logger = LoggerFactory.getLogger(TokenRetrieverBase.class); + private static final String GOSSIP_INFO_URL_FORMAT = + "http://%s:8080/Priam/REST/v1/cassadmin/gossipinfo"; + + /** + * Utility method to infer the IP of the owner of a token in a given datacenter. This method + * uses Cassandra gossip information to find the owner. While it is ideal to check all the nodes + * in the ring to see if they agree on the IP to be replaced, in large clusters it may affect + * the startup performance. This method picks at most 3 random hosts from the ring and see if + * they all agree on the IP to be replaced. If not, it returns null. + * + * @param allIds + * @param token + * @param dc + * @return IP of the token owner based on gossip information or null if gossip doesn't converge. + * @throws GossipParseException when required number of instances are not available to fetch the + * gossip info. + */ + public static String inferTokenOwnerFromGossip( + List allIds, String token, String dc) throws GossipParseException { + // Avoid using dead instance who we are trying to replace (duh!!) + // Avoid other regions instances to avoid communication over public ip address. + List eligibleInstances = + allIds.stream() + .filter(priamInstance -> !priamInstance.getToken().equalsIgnoreCase(token)) + .filter(priamInstance -> priamInstance.getDC().equalsIgnoreCase(dc)) + .collect(Collectors.toList()); + // We want to get IP from min 1, max 3 instances to ensure we are not relying on + // gossip of a single instance. + // Good idea to shuffle so we are not talking to same instances every time. + Collections.shuffle(eligibleInstances); + // Potential issue could be when you have about 50% of your cluster C* DOWN or + // trying to be replaced. + // Think of a major disaster hitting your cluster. In that scenario chances of + // instance hitting DOWN C* are much much higher. + // In such a case you should rely on @link{CassandraConfig#setReplacedIp}. + int noOfInstancesGossipShouldMatch = Math.max(1, Math.min(3, eligibleInstances.size())); + + // While it is ideal to check all the nodes in the ring to see if they agree on + // the IP to be replaced, in large clusters it may affect the startup + // performance. So we pick three random hosts from the ring and see if they all + // agree on the IP to be replaced. If not, we don't replace. + String replaceIp = null; + int matchedGossipInstances = 0, reachableInstances = 0; + for (PriamInstance instance : eligibleInstances) { + logger.info( + "Calling getIp on hostname[{}] and token[{}]", instance.getHostName(), token); + + try { + String ip = getIp(instance.getHostName(), token); + reachableInstances++; + + if (StringUtils.isEmpty(ip)) { + continue; + } + + if (replaceIp == null) { + replaceIp = ip; + } else if (!replaceIp.equals(ip)) { + break; + } + + matchedGossipInstances++; + if (matchedGossipInstances == noOfInstancesGossipShouldMatch) { + return replaceIp; + } + } catch (GossipParseException e) { + logger.warn(e.getMessage()); + } + } + + // Throw exception if we are not able to reach at least minimum required + // instances. + if (reachableInstances < noOfInstancesGossipShouldMatch) { + throw new GossipParseException( + "Unable to reach minimum required instances to fetch gossip information."); + } + + logger.warn( + "Return null: Unable to find enough instances where gossip match. Required: {}", + noOfInstancesGossipShouldMatch); + return null; + } + + // helper method to get the token owner IP from a Cassandra node. + private static String getIp(String host, String token) throws GossipParseException { + String response = null; + try { + response = SystemUtils.getDataFromUrl(String.format(GOSSIP_INFO_URL_FORMAT, host)); + + String inputToken = String.format("[%s]", token); + JSONParser parser = new JSONParser(); + JSONArray jsonObject = (JSONArray) parser.parse(response); + + for (Object key : jsonObject) { + JSONObject msg = (JSONObject) key; + + // Ensure that we are not trying to replace a NORMAL token and token of that + // instance matches what we want to replace. + if (msg.get("STATUS") == null + || msg.get("STATUS").toString().equalsIgnoreCase("NORMAL") + || msg.get("TOKENS") == null + || msg.get("PUBLIC_IP") == null + || !msg.get("TOKENS").toString().equals(inputToken)) { + continue; + } + + logger.info( + "Using gossip info from host[{}] and token[{}], the replaced address is : [{}]", + host, + token, + msg.get("PUBLIC_IP")); + return (String) msg.get("PUBLIC_IP"); + } + + return null; + } catch (RuntimeException e) { + throw new GossipParseException( + String.format("Error in reaching out to host: [{}]", host), e); + } catch (ParseException e) { + throw new GossipParseException( + String.format( + "Error in parsing gossip response [{}] from host: [{}]", + response, + host), + e); + } + } + + /** + * This exception is thrown either when instances are not available or when they return invalid + * response. + */ + public static class GossipParseException extends Exception { + private static final long serialVersionUID = 1462488371031437486L; + + public GossipParseException() { + super(); + } + + public GossipParseException(String message) { + super(message); + } + + public GossipParseException(String message, Throwable t) { + super(message, t); + } + } +} diff --git a/priam/src/test/java/com/netflix/priam/identity/token/AssignedTokenRetrieverTest.java b/priam/src/test/java/com/netflix/priam/identity/token/AssignedTokenRetrieverTest.java new file mode 100644 index 000000000..31d5ec1c5 --- /dev/null +++ b/priam/src/test/java/com/netflix/priam/identity/token/AssignedTokenRetrieverTest.java @@ -0,0 +1,264 @@ +package com.netflix.priam.identity.token; + +import com.netflix.priam.config.IConfiguration; +import com.netflix.priam.identity.IMembership; +import com.netflix.priam.identity.IPriamInstanceFactory; +import com.netflix.priam.identity.InstanceIdentity; +import com.netflix.priam.identity.PriamInstance; +import com.netflix.priam.identity.config.InstanceInfo; +import com.netflix.priam.utils.ITokenManager; +import com.netflix.priam.utils.Sleeper; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.IntStream; +import junit.framework.Assert; +import mockit.Expectations; +import mockit.Mocked; +import org.apache.commons.lang3.StringUtils; +import org.junit.Test; + +public class AssignedTokenRetrieverTest { + public static final String APP = "testapp"; + public static final String DEAD_APP = "testapp-dead"; + + @Test + public void grabAssignedTokenStartDbInBootstrapModeWhenGossipAgreesCurrentInstanceIsTokenOwner( + @Mocked IPriamInstanceFactory factory, + @Mocked IConfiguration config, + @Mocked IMembership membership, + @Mocked Sleeper sleeper, + @Mocked ITokenManager tokenManager, + @Mocked InstanceInfo instanceInfo, + @Mocked TokenRetrieverUtils retrievalUtils) + throws Exception { + List liveHosts = newPriamInstances(); + Collections.shuffle(liveHosts); + + new Expectations() { + { + config.getAppName(); + result = APP; + + factory.getAllIds(DEAD_APP); + result = Collections.emptyList(); + + factory.getAllIds(APP); + result = liveHosts; + + instanceInfo.getInstanceId(); + result = liveHosts.get(0).getInstanceId(); + + TokenRetrieverUtils.inferTokenOwnerFromGossip( + liveHosts, liveHosts.get(0).getToken(), liveHosts.get(0).getDC()); + result = liveHosts.get(0).getHostIP(); + } + }; + + IDeadTokenRetriever deadTokenRetriever = + new DeadTokenRetriever(factory, membership, config, sleeper, instanceInfo); + IPreGeneratedTokenRetriever preGeneratedTokenRetriever = + new PreGeneratedTokenRetriever(factory, membership, config, sleeper, instanceInfo); + INewTokenRetriever newTokenRetriever = + new NewTokenRetriever( + factory, membership, config, sleeper, tokenManager, instanceInfo); + InstanceIdentity instanceIdentity = + new InstanceIdentity( + factory, + membership, + config, + sleeper, + tokenManager, + deadTokenRetriever, + preGeneratedTokenRetriever, + newTokenRetriever, + instanceInfo); + + Assert.assertEquals(false, instanceIdentity.isReplace()); + } + + @Test + public void grabAssignedTokenStartDbInReplaceModeWhenGossipAgreesOnPreviousTokenOwner( + @Mocked IPriamInstanceFactory factory, + @Mocked IConfiguration config, + @Mocked IMembership membership, + @Mocked Sleeper sleeper, + @Mocked ITokenManager tokenManager, + @Mocked InstanceInfo instanceInfo, + @Mocked TokenRetrieverUtils retrievalUtils) + throws Exception { + List liveHosts = newPriamInstances(); + Collections.shuffle(liveHosts); + + PriamInstance deadInstance = liveHosts.remove(0); + PriamInstance newInstance = + newMockPriamInstance( + APP, + deadInstance.getDC(), + deadInstance.getRac(), + deadInstance.getId(), + String.format("new-fakeInstance-%d", deadInstance.getId()), + String.format("127.1.1.%d", deadInstance.getId() + 100), + String.format("new-fakeHost-%d", deadInstance.getId()), + deadInstance.getToken()); + + // the case we are trying to test is when Priam restarted after it acquired the + // token. new instance is already registered with token database. + liveHosts.add(newInstance); + + new Expectations() { + { + config.getAppName(); + result = APP; + + factory.getAllIds(DEAD_APP); + result = Collections.singletonList(deadInstance); + factory.getAllIds(APP); + result = liveHosts; + + instanceInfo.getInstanceId(); + result = newInstance.getInstanceId(); + + TokenRetrieverUtils.inferTokenOwnerFromGossip( + liveHosts, newInstance.getToken(), newInstance.getDC()); + result = deadInstance.getHostIP(); + } + }; + + IDeadTokenRetriever deadTokenRetriever = + new DeadTokenRetriever(factory, membership, config, sleeper, instanceInfo); + IPreGeneratedTokenRetriever preGeneratedTokenRetriever = + new PreGeneratedTokenRetriever(factory, membership, config, sleeper, instanceInfo); + INewTokenRetriever newTokenRetriever = + new NewTokenRetriever( + factory, membership, config, sleeper, tokenManager, instanceInfo); + InstanceIdentity instanceIdentity = + new InstanceIdentity( + factory, + membership, + config, + sleeper, + tokenManager, + deadTokenRetriever, + preGeneratedTokenRetriever, + newTokenRetriever, + instanceInfo); + + Assert.assertEquals(deadInstance.getHostIP(), instanceIdentity.getReplacedIp()); + Assert.assertEquals(true, instanceIdentity.isReplace()); + } + + @Test + public void grabAssignedTokenStartDbInBootstrapModeWhenGossipDisagreesOnPreviousTokenOwner( + @Mocked IPriamInstanceFactory factory, + @Mocked IConfiguration config, + @Mocked IMembership membership, + @Mocked Sleeper sleeper, + @Mocked ITokenManager tokenManager, + @Mocked InstanceInfo instanceInfo, + @Mocked TokenRetrieverUtils retrievalUtils) + throws Exception { + List liveHosts = newPriamInstances(); + Collections.shuffle(liveHosts); + + new Expectations() { + { + config.getAppName(); + result = APP; + + factory.getAllIds(DEAD_APP); + result = Collections.emptyList(); + factory.getAllIds(APP); + result = liveHosts; + + instanceInfo.getInstanceId(); + result = liveHosts.get(0).getInstanceId(); + + TokenRetrieverUtils.inferTokenOwnerFromGossip( + liveHosts, liveHosts.get(0).getToken(), liveHosts.get(0).getDC()); + result = null; + } + }; + + IDeadTokenRetriever deadTokenRetriever = + new DeadTokenRetriever(factory, membership, config, sleeper, instanceInfo); + IPreGeneratedTokenRetriever preGeneratedTokenRetriever = + new PreGeneratedTokenRetriever(factory, membership, config, sleeper, instanceInfo); + INewTokenRetriever newTokenRetriever = + new NewTokenRetriever( + factory, membership, config, sleeper, tokenManager, instanceInfo); + InstanceIdentity instanceIdentity = + new InstanceIdentity( + factory, + membership, + config, + sleeper, + tokenManager, + deadTokenRetriever, + preGeneratedTokenRetriever, + newTokenRetriever, + instanceInfo); + + Assert.assertTrue(StringUtils.isEmpty(instanceIdentity.getReplacedIp())); + Assert.assertEquals(false, instanceIdentity.isReplace()); + } + + private List newPriamInstances() { + List instances = new ArrayList<>(); + + instances.addAll(newPriamInstances("eu-west", "1a", 0, "127.3.1.%d")); + instances.addAll(newPriamInstances("eu-west", "1b", 3, "127.3.2.%d")); + instances.addAll(newPriamInstances("eu-west", "1c", 6, "127.3.3.%d")); + + instances.addAll(newPriamInstances("us-east", "1c", 1, "127.1.3.%d")); + instances.addAll(newPriamInstances("us-east", "1a", 4, "127.1.1.%d")); + instances.addAll(newPriamInstances("us-east", "1b", 7, "127.1.2.%d")); + + instances.addAll(newPriamInstances("us-west-2", "2a", 2, "127.2.1.%d")); + instances.addAll(newPriamInstances("us-west-2", "2b", 5, "127.2.2.%d")); + instances.addAll(newPriamInstances("us-west-2", "2c", 8, "127.2.3.%d")); + + return instances; + } + + private List newPriamInstances( + String dc, String rack, int seqNo, String ipRanges) { + return IntStream.range(0, 3) + .map(e -> seqNo + (e * 9)) + .mapToObj( + e -> + newMockPriamInstance( + APP, + dc, + rack, + e, + String.format("fakeInstance-%d", e), + String.format(ipRanges, e), + String.format("fakeHost-%d", e), + Integer.toString(e))) + .collect(Collectors.toList()); + } + + private PriamInstance newMockPriamInstance( + String app, + String dc, + String rack, + int id, + String instanceId, + String hostIp, + String hostName, + String token) { + PriamInstance priamInstance = new PriamInstance(); + priamInstance.setApp(app); + priamInstance.setDC(dc); + priamInstance.setRac(rack); + priamInstance.setId(id); + priamInstance.setInstanceId(instanceId); + priamInstance.setHost(hostName); + priamInstance.setHostIP(hostIp); + priamInstance.setToken(token); + + return priamInstance; + } +} diff --git a/priam/src/test/java/com/netflix/priam/identity/token/TestDeadTokenRetriever.java b/priam/src/test/java/com/netflix/priam/identity/token/DeadTokenRetrieverTest.java similarity index 93% rename from priam/src/test/java/com/netflix/priam/identity/token/TestDeadTokenRetriever.java rename to priam/src/test/java/com/netflix/priam/identity/token/DeadTokenRetrieverTest.java index f32799f92..cdcaea329 100644 --- a/priam/src/test/java/com/netflix/priam/identity/token/TestDeadTokenRetriever.java +++ b/priam/src/test/java/com/netflix/priam/identity/token/DeadTokenRetrieverTest.java @@ -36,14 +36,14 @@ import org.junit.Test; /** Created by aagrawal on 3/1/19. */ -public class TestDeadTokenRetriever { - @Mocked private IPriamInstanceFactory factory; +public class DeadTokenRetrieverTest { + @Mocked private IPriamInstanceFactory factory; @Mocked private IMembership membership; private IDeadTokenRetriever deadTokenRetriever; private InstanceInfo instanceInfo; private IConfiguration configuration; - public TestDeadTokenRetriever() { + public DeadTokenRetrieverTest() { Injector injector = Guice.createInjector(new BRTestModule()); if (instanceInfo == null) instanceInfo = injector.getInstance(InstanceInfo.class); if (configuration == null) configuration = injector.getInstance(IConfiguration.class); @@ -108,8 +108,9 @@ public void testNoReplacementNoGossipMatch(@Mocked SystemUtils systemUtils) thro result = allInstances; membership.getRacMembership(); result = racMembership; - systemUtils.getDataFromUrl(anyString); - result = null; + SystemUtils.getDataFromUrl(anyString); + result = + "[{\"TOKENS\":\"[1]\",\"PUBLIC_IP\":\"\",\"RACK\":\"az1\",\"STATUS\":\"NORMAL\",\"DC\":\"us-east-1\"},{\"TOKENS\":\"[2]\",\"PUBLIC_IP\":\"\",\"RACK\":\"az1\",\"STATUS\":\"NORMAL\",\"DC\":\"us-east-1\"}]"; times = 1; } }; @@ -134,7 +135,7 @@ public void testNoReplacementNoGossipMatch(@Mocked SystemUtils systemUtils) thro result = allInstances; membership.getRacMembership(); result = racMembership; - systemUtils.getDataFromUrl(anyString); + SystemUtils.getDataFromUrl(anyString); result = gossipInfo; times = 1; } @@ -156,14 +157,13 @@ public void testReplacementGossipMatch(@Mocked SystemUtils systemUtils) throws E racMembership.add(instanceInfo.getInstanceId()); String gossipResponse = "[{\"TOKENS\":\"[1]\",\"PUBLIC_IP\":\"127.0.0.1\",\"RACK\":\"az1\",\"STATUS\":\"NORMAL\",\"DC\":\"us-east-1\"},{\"TOKENS\":\"[2]\",\"PUBLIC_IP\":\"127.0.0.2\",\"RACK\":\"az1\",\"STATUS\":\"NORMAL\",\"DC\":\"us-east-1\"},{\"TOKENS\":\"[3]\",\"PUBLIC_IP\":\"127.0.0.3\",\"RACK\":\"az1\",\"STATUS\":\"shutdown\",\"DC\":\"us-east-1\"},{\"TOKENS\":\"[4]\",\"PUBLIC_IP\":\"127.0.0.4\",\"RACK\":\"az2\",\"STATUS\":\"NORMAL\",\"DC\":\"us-east-1\"},{\"TOKENS\":\"[5]\",\"PUBLIC_IP\":\"127.0.0.5\",\"RACK\":\"az2\",\"STATUS\":\"NORMAL\",\"DC\":\"us-east-1\"},{\"TOKENS\":\"[6]\",\"PUBLIC_IP\":\"127.0.0.6\",\"RACK\":\"az2\",\"STATUS\":\"NORMAL\",\"DC\":\"us-east-1\"}]"; - PriamInstance instance = null; new Expectations() { { factory.getAllIds(anyString); result = allInstances; membership.getRacMembership(); result = racMembership; - systemUtils.getDataFromUrl(anyString); + SystemUtils.getDataFromUrl(anyString); returns(gossipResponse, gossipResponse, null, gossipResponse); } }; diff --git a/priam/src/test/java/com/netflix/priam/identity/token/TokenRetrieverUtilsTest.java b/priam/src/test/java/com/netflix/priam/identity/token/TokenRetrieverUtilsTest.java new file mode 100644 index 000000000..17e9c500e --- /dev/null +++ b/priam/src/test/java/com/netflix/priam/identity/token/TokenRetrieverUtilsTest.java @@ -0,0 +1,209 @@ +package com.netflix.priam.identity.token; + +import static org.hamcrest.core.AllOf.allOf; +import static org.hamcrest.core.IsNot.not; + +import com.netflix.priam.identity.PriamInstance; +import com.netflix.priam.utils.SystemUtils; +import java.util.Arrays; +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.IntStream; +import junit.framework.Assert; +import mockit.Expectations; +import mockit.Mocked; +import org.junit.Test; + +public class TokenRetrieverUtilsTest { + private static final String APP = "testapp"; + private static final String GOSSIP_INFO_URL_FORMAT = + "http://%s:8080/Priam/REST/v1/cassadmin/gossipinfo"; + + private List instances = + IntStream.range(0, 6) + .mapToObj( + e -> + newMockPriamInstance( + APP, + "us-east", + (e < 3) ? "az1" : "az2", + e, + String.format("fakeInstance-%d", e), + String.format("127.0.0.%d", e), + String.format("fakeHost-%d", e), + String.valueOf(e))) + .collect(Collectors.toList()); + + private String[] gossipInfos = + IntStream.range(0, 6) + .mapToObj( + e -> + newGossipRecord( + e, + String.format("127.0.0.%d", e), + "us-east-1", + (e < 3) ? "az1" : "az2", + "NORMAL")) + .collect(Collectors.toList()) + .toArray(new String[0]); + + @Test + public void testRetrieveTokenOwnerWhenGossipAgrees(@Mocked SystemUtils systemUtils) + throws Exception { + // updates instances with new instance owning token 4 as per token database. + List newInstances = + instances.stream().filter(e -> e.getId() != 4).collect(Collectors.toList()); + newInstances.add( + newMockPriamInstance( + APP, + "us-east", + "az2", + 4, + "fakeHost-400", + "127.0.0.400", + "fakeHost-400", + "4")); + + // mark previous instance with tokenNumber 4 as down in gossip. + String[] newGossipInfos = Arrays.copyOf(gossipInfos, gossipInfos.length); + newGossipInfos[4] = newGossipRecord(4, "127.0.0.4", "us-east-1", "az2", "shutdown"); + + new Expectations() { + { + SystemUtils.getDataFromUrl(anyString); + result = Arrays.toString(newGossipInfos); + } + }; + + String replaceIp = TokenRetrieverUtils.inferTokenOwnerFromGossip(instances, "4", "us-east"); + Assert.assertEquals("127.0.0.4", replaceIp); + } + + @SuppressWarnings("unchecked") + @Test + public void testRetrieveTokenOwnerWhenGossipDisagrees(@Mocked SystemUtils systemUtils) + throws Exception { + // updates instances with new instance owning token 4 as per token database. + List newInstances = + instances.stream().filter(e -> e.getId() != 4).collect(Collectors.toList()); + + // mark previous instance as token owner for tokenNumber 4 in the gossip. + String[] gossipInfoSetOne = Arrays.copyOf(gossipInfos, gossipInfos.length); + gossipInfoSetOne[4] = newGossipRecord(4, "127.0.0.4", "us-east-1", "az2", "shutdown"); + + // mark empty as token owner for tokenNumber 4 in the gossip. + String[] gossipInfoSetTwo = Arrays.copyOf(gossipInfos, gossipInfos.length); + gossipInfoSetTwo[4] = newGossipRecord(4, "", "us-east-1", "az2", "shutdown"); + + new Expectations() { + { + SystemUtils.getDataFromUrl( + withArgThat( + allOf( + not(String.format(GOSSIP_INFO_URL_FORMAT, "fakeHost-0")), + not(String.format(GOSSIP_INFO_URL_FORMAT, "fakeHost-5"))))); + result = Arrays.toString(gossipInfoSetTwo); + + SystemUtils.getDataFromUrl(String.format(GOSSIP_INFO_URL_FORMAT, "fakeHost-0")); + result = Arrays.toString(gossipInfoSetOne); + minTimes = 0; + SystemUtils.getDataFromUrl(String.format(GOSSIP_INFO_URL_FORMAT, "fakeHost-5")); + result = Arrays.toString(gossipInfoSetOne); + minTimes = 0; + } + }; + + String replaceIp = + TokenRetrieverUtils.inferTokenOwnerFromGossip(newInstances, "4", "us-east"); + Assert.assertEquals(null, replaceIp); + } + + @Test + public void testRetrieveTokenOwnerWhenAllHostsInGossipReturnsNull( + @Mocked SystemUtils systemUtils) throws Exception { + // updates instances with new instance owning token 4 as per token database. + List newInstances = + instances.stream().filter(e -> e.getId() != 4).collect(Collectors.toList()); + newInstances.add( + newMockPriamInstance( + APP, + "us-east", + "az2", + 4, + "fakeInstance-400", + "127.0.0.400", + "fakeHost-400", + "4")); + + // mark empty as token owner for tokenNumber 4 in the gossip. + String[] gossipInfo = Arrays.copyOf(gossipInfos, gossipInfos.length); + gossipInfo[4] = newGossipRecord(4, "", "us-east-1", "az2", "shutdown"); + + new Expectations() { + { + SystemUtils.getDataFromUrl(anyString); + result = Arrays.toString(gossipInfo); + } + }; + + String replaceIp = TokenRetrieverUtils.inferTokenOwnerFromGossip(instances, "4", "us-east"); + Assert.assertNull(replaceIp); + } + + @Test(expected = TokenRetrieverUtils.GossipParseException.class) + public void testRetrieveTokenOwnerWhenAllInstancesThrowGossipParseException( + @Mocked SystemUtils systemUtils) throws TokenRetrieverUtils.GossipParseException { + // updates instances with new instance owning token 4 as per token database. + List newInstances = + instances.stream().filter(e -> e.getId() != 4).collect(Collectors.toList()); + newInstances.add( + newMockPriamInstance( + APP, + "us-east", + "az2", + 4, + "fakeInstance-400", + "127.0.0.400", + "fakeHost-400", + "4")); + + new Expectations() { + { + SystemUtils.getDataFromUrl(anyString); + result = new TokenRetrieverUtils.GossipParseException(); + } + }; + + String replaceIp = TokenRetrieverUtils.inferTokenOwnerFromGossip(instances, "4", "us-east"); + Assert.assertNull(replaceIp); + } + + private String newGossipRecord( + int tokenNumber, String ip, String dc, String rack, String status) { + return String.format( + "{\"TOKENS\":\"[%d]\",\"PUBLIC_IP\":\"%s\",\"RACK\":\"%s\",\"STATUS\":\"%s\",\"DC\":\"%s\"}", + tokenNumber, ip, dc, status, rack); + } + + private PriamInstance newMockPriamInstance( + String app, + String dc, + String rack, + int id, + String instanceId, + String hostIp, + String hostName, + String token) { + PriamInstance priamInstance = new PriamInstance(); + priamInstance.setApp(app); + priamInstance.setDC(dc); + priamInstance.setRac(rack); + priamInstance.setId(id); + priamInstance.setInstanceId(instanceId); + priamInstance.setHost(hostName); + priamInstance.setHostIP(hostIp); + priamInstance.setToken(token); + + return priamInstance; + } +} From 76936a1d7f154d329f6008df863d463fc5266f4b Mon Sep 17 00:00:00 2001 From: Chandrasekhar Thumuluru Date: Tue, 14 May 2019 17:31:20 -0700 Subject: [PATCH 083/228] Changing the list in TokenRetrievalUtils to use wildcards. --- .../priam/identity/token/DeadTokenRetriever.java | 12 ++++++++---- .../priam/identity/token/TokenRetrieverUtils.java | 5 +++-- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/priam/src/main/java/com/netflix/priam/identity/token/DeadTokenRetriever.java b/priam/src/main/java/com/netflix/priam/identity/token/DeadTokenRetriever.java index c961fba1e..1e05e1317 100755 --- a/priam/src/main/java/com/netflix/priam/identity/token/DeadTokenRetriever.java +++ b/priam/src/main/java/com/netflix/priam/identity/token/DeadTokenRetriever.java @@ -94,8 +94,8 @@ public PriamInstance get() throws Exception { || super.isInstanceDummy(priamInstance)) continue; // TODO: If instance is in SHUTTING_DOWN mode, it might not show up in asg // instances (if cloud control plane is having issues), thus, we should not try - // to - // replace the instance as it will lead to "Cannot replace a live node" issue. + // to replace the instance as it will lead to "Cannot replace a live node" + // issue. logger.info("Found dead instance: {}", priamInstance.toString()); @@ -122,7 +122,6 @@ public PriamInstance get() throws Exception { // Lets not replace the instance if gossip info is not merging!! if (replacedIp == null) return null; - logger.info( "Will try to replace token: {} with replacedIp (from gossip info): {} instead of ip from Token database: {}", priamInstance.getToken(), @@ -131,6 +130,10 @@ public PriamInstance get() throws Exception { } catch (GossipParseException e) { // In case of gossip exception, fallback to IP in token database. this.replacedIp = priamInstance.getHostIP(); + logger.info( + "Will try to replace token: {} with replacedIp from Token database: {}", + priamInstance.getToken(), + priamInstance.getHostIP()); } PriamInstance result; @@ -153,7 +156,8 @@ public PriamInstance get() throws Exception { + " , will sleep for " + sleepTime + " millisecs before we retry."); - Thread.currentThread().sleep(sleepTime); + Thread.sleep(sleepTime); + throw ex; } diff --git a/priam/src/main/java/com/netflix/priam/identity/token/TokenRetrieverUtils.java b/priam/src/main/java/com/netflix/priam/identity/token/TokenRetrieverUtils.java index ff08d9d4e..66f151a58 100644 --- a/priam/src/main/java/com/netflix/priam/identity/token/TokenRetrieverUtils.java +++ b/priam/src/main/java/com/netflix/priam/identity/token/TokenRetrieverUtils.java @@ -34,10 +34,11 @@ public class TokenRetrieverUtils { * gossip info. */ public static String inferTokenOwnerFromGossip( - List allIds, String token, String dc) throws GossipParseException { + List allIds, String token, String dc) + throws GossipParseException { // Avoid using dead instance who we are trying to replace (duh!!) // Avoid other regions instances to avoid communication over public ip address. - List eligibleInstances = + List eligibleInstances = allIds.stream() .filter(priamInstance -> !priamInstance.getToken().equalsIgnoreCase(token)) .filter(priamInstance -> priamInstance.getDC().equalsIgnoreCase(dc)) From 23b421806cf24dd7da4a988090f3452d861c3ec6 Mon Sep 17 00:00:00 2001 From: Chandrasekhar Thumuluru Date: Fri, 24 May 2019 20:38:19 -0700 Subject: [PATCH 084/228] replacing replace_address with replace_address_first_boot which ignores doesn't try to bootstrap the node in replace mode if it is already bootstrapped successfully. --- .../netflix/priam/cassandra/extensions/PriamStartupAgent.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/priam-cass-extensions/src/main/java/com/netflix/priam/cassandra/extensions/PriamStartupAgent.java b/priam-cass-extensions/src/main/java/com/netflix/priam/cassandra/extensions/PriamStartupAgent.java index 50e340a10..a875b5a79 100644 --- a/priam-cass-extensions/src/main/java/com/netflix/priam/cassandra/extensions/PriamStartupAgent.java +++ b/priam-cass-extensions/src/main/java/com/netflix/priam/cassandra/extensions/PriamStartupAgent.java @@ -87,7 +87,7 @@ private void setPriamProperties() { if (FBUtilities.getReleaseVersionString().compareTo(REPLACED_ADDRESS_MIN_VER) < 0) { System.setProperty("cassandra.replace_token", token); } else { - System.setProperty("cassandra.replace_address", replacedIp); + System.setProperty("cassandra.replace_address_first_boot", replacedIp); } } } From f6b072d35d7b9dc52af7d9994725739e702a73e4 Mon Sep 17 00:00:00 2001 From: Chandrasekhar Thumuluru Date: Thu, 6 Jun 2019 22:26:07 -0700 Subject: [PATCH 085/228] rolling back the changes to use gossip info while grabbing pre-assigned and dead tokens. --- .../identity/token/DeadTokenRetriever.java | 27 +++---------------- .../identity/token/TokenRetrieverUtils.java | 12 +++++++++ .../token/DeadTokenRetrieverTest.java | 4 +-- .../token/TokenRetrieverUtilsTest.java | 9 +++---- 4 files changed, 21 insertions(+), 31 deletions(-) diff --git a/priam/src/main/java/com/netflix/priam/identity/token/DeadTokenRetriever.java b/priam/src/main/java/com/netflix/priam/identity/token/DeadTokenRetriever.java index 1e05e1317..0fb036c74 100755 --- a/priam/src/main/java/com/netflix/priam/identity/token/DeadTokenRetriever.java +++ b/priam/src/main/java/com/netflix/priam/identity/token/DeadTokenRetriever.java @@ -20,7 +20,6 @@ import com.netflix.priam.identity.IPriamInstanceFactory; import com.netflix.priam.identity.PriamInstance; import com.netflix.priam.identity.config.InstanceInfo; -import com.netflix.priam.identity.token.TokenRetrieverUtils.GossipParseException; import com.netflix.priam.utils.Sleeper; import java.util.List; import java.util.Random; @@ -112,29 +111,9 @@ public PriamInstance get() throws Exception { // remove it as we marked it down... factory.delete(priamInstance); - // find the replaced IP - try { - replacedIp = - TokenRetrieverUtils.inferTokenOwnerFromGossip( - allInstancesWithinCluster, - priamInstance.getToken(), - priamInstance.getDC()); - - // Lets not replace the instance if gossip info is not merging!! - if (replacedIp == null) return null; - logger.info( - "Will try to replace token: {} with replacedIp (from gossip info): {} instead of ip from Token database: {}", - priamInstance.getToken(), - replacedIp, - priamInstance.getHostIP()); - } catch (GossipParseException e) { - // In case of gossip exception, fallback to IP in token database. - this.replacedIp = priamInstance.getHostIP(); - logger.info( - "Will try to replace token: {} with replacedIp from Token database: {}", - priamInstance.getToken(), - priamInstance.getHostIP()); - } + // use entry in the token database always. + // this can cause "can't replace live token errors. + this.replacedIp = priamInstance.getHostIP(); PriamInstance result; try { diff --git a/priam/src/main/java/com/netflix/priam/identity/token/TokenRetrieverUtils.java b/priam/src/main/java/com/netflix/priam/identity/token/TokenRetrieverUtils.java index 66f151a58..26007262a 100644 --- a/priam/src/main/java/com/netflix/priam/identity/token/TokenRetrieverUtils.java +++ b/priam/src/main/java/com/netflix/priam/identity/token/TokenRetrieverUtils.java @@ -36,6 +36,12 @@ public class TokenRetrieverUtils { public static String inferTokenOwnerFromGossip( List allIds, String token, String dc) throws GossipParseException { + // TODO: Gossip info in some cases doesn't reflect the real C* state. + // Not using gossip info for now. + if (!useGossipInfo()) { + return null; + } + // Avoid using dead instance who we are trying to replace (duh!!) // Avoid other regions instances to avoid communication over public ip address. List eligibleInstances = @@ -100,6 +106,12 @@ public static String inferTokenOwnerFromGossip( return null; } + // TODO: Gossip info in some cases doesn't reflect the real C* state. + // Not using gossip info for now. + private static boolean useGossipInfo() { + return false; + } + // helper method to get the token owner IP from a Cassandra node. private static String getIp(String host, String token) throws GossipParseException { String response = null; diff --git a/priam/src/test/java/com/netflix/priam/identity/token/DeadTokenRetrieverTest.java b/priam/src/test/java/com/netflix/priam/identity/token/DeadTokenRetrieverTest.java index cdcaea329..39ca92c70 100644 --- a/priam/src/test/java/com/netflix/priam/identity/token/DeadTokenRetrieverTest.java +++ b/priam/src/test/java/com/netflix/priam/identity/token/DeadTokenRetrieverTest.java @@ -94,7 +94,7 @@ public void testNoReplacementNoSpotAvailable() throws Exception { }; } - @Test + // @Test // There is a potential slot for dead token but we are unable to replace. public void testNoReplacementNoGossipMatch(@Mocked SystemUtils systemUtils) throws Exception { List allInstances = getInstances(2); @@ -150,7 +150,7 @@ public void testNoReplacementNoGossipMatch(@Mocked SystemUtils systemUtils) thro }; } - @Test + // @Test public void testReplacementGossipMatch(@Mocked SystemUtils systemUtils) throws Exception { List allInstances = getInstances(6); List racMembership = getRacMembership(2); diff --git a/priam/src/test/java/com/netflix/priam/identity/token/TokenRetrieverUtilsTest.java b/priam/src/test/java/com/netflix/priam/identity/token/TokenRetrieverUtilsTest.java index 17e9c500e..6ed7657a7 100644 --- a/priam/src/test/java/com/netflix/priam/identity/token/TokenRetrieverUtilsTest.java +++ b/priam/src/test/java/com/netflix/priam/identity/token/TokenRetrieverUtilsTest.java @@ -12,7 +12,6 @@ import junit.framework.Assert; import mockit.Expectations; import mockit.Mocked; -import org.junit.Test; public class TokenRetrieverUtilsTest { private static final String APP = "testapp"; @@ -47,7 +46,7 @@ public class TokenRetrieverUtilsTest { .collect(Collectors.toList()) .toArray(new String[0]); - @Test + // @Test public void testRetrieveTokenOwnerWhenGossipAgrees(@Mocked SystemUtils systemUtils) throws Exception { // updates instances with new instance owning token 4 as per token database. @@ -80,7 +79,7 @@ public void testRetrieveTokenOwnerWhenGossipAgrees(@Mocked SystemUtils systemUti } @SuppressWarnings("unchecked") - @Test + // @Test public void testRetrieveTokenOwnerWhenGossipDisagrees(@Mocked SystemUtils systemUtils) throws Exception { // updates instances with new instance owning token 4 as per token database. @@ -118,7 +117,7 @@ public void testRetrieveTokenOwnerWhenGossipDisagrees(@Mocked SystemUtils system Assert.assertEquals(null, replaceIp); } - @Test + // @Test public void testRetrieveTokenOwnerWhenAllHostsInGossipReturnsNull( @Mocked SystemUtils systemUtils) throws Exception { // updates instances with new instance owning token 4 as per token database. @@ -150,7 +149,7 @@ public void testRetrieveTokenOwnerWhenAllHostsInGossipReturnsNull( Assert.assertNull(replaceIp); } - @Test(expected = TokenRetrieverUtils.GossipParseException.class) + // @Test(expected = TokenRetrieverUtils.GossipParseException.class) public void testRetrieveTokenOwnerWhenAllInstancesThrowGossipParseException( @Mocked SystemUtils systemUtils) throws TokenRetrieverUtils.GossipParseException { // updates instances with new instance owning token 4 as per token database. From 385a9b5d11d45cfbbc0a32c7758330e0e80b1e5b Mon Sep 17 00:00:00 2001 From: Arun Agrawal Date: Thu, 25 Jul 2019 18:31:11 -0700 Subject: [PATCH 086/228] Change to gh-pages documentation instead of wiki --- README.md | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index f0905d41f..983e49607 100644 --- a/README.md +++ b/README.md @@ -4,15 +4,14 @@

    -[Releases][release]   |   [Wiki Documentation][wiki]   |    - +[Releases][release]   |   [Documentation][wiki]   |    [![Build Status][img-travis-ci]][travis-ci]
    ## Important Notice -* Priam 3.11 branch supports Cassandra 3.x. Netflix internally uses Apache Cassandra 3.0.17. +* Priam 3.11 branch supports Cassandra 3.x. Netflix internally uses Apache Cassandra 3.0.19. ## Table of Contents [**TL;DR**](#tldr) @@ -54,15 +53,15 @@ Priam is actively developed and used at Netflix. - REST APIs for backup/restore and other operations ## Compatibility -See [Compatibility](https://github.com/Netflix/Priam/wiki/Compatibility) for details. +See [Compatibility](http://netflix.github.io/Priam/#compatibility) for details. ## Installation -See [Setup](https://github.com/Netflix/Priam/wiki/Setup) for details. +See [Setup](http://netflix.github.io/Priam/latest/mgmt/installation/) for details. ## Cluster Management -Basic configuration/REST API's to manage cassandra cluster. See [Cluster Management](https://github.com/Netflix/Priam/wiki/Cluster-Management) for details. +Basic configuration/REST API's to manage cassandra cluster. See [Cluster Management](http://netflix.github.io/Priam/latest/management/) for details. ## Changelog See [CHANGELOG.md](CHANGELOG.md) @@ -70,8 +69,7 @@ See [CHANGELOG.md](CHANGELOG.md) References --> [release]:https://github.com/Netflix/Priam/releases/latest "Latest Release (external link) ➶" -[wiki]:https://github.com/Netflix/Priam/wiki +[wiki]:http://netflix.github.io/Priam/ [repo]:https://github.com/Netflix/Priam [img-travis-ci]:https://travis-ci.org/Netflix/Priam.svg?branch=3.11 [travis-ci]:https://travis-ci.org/Netflix/Priam - From 48ab05a3b7ace04ecca5146007ad6e112545b60e Mon Sep 17 00:00:00 2001 From: aagrawal Date: Thu, 20 Jun 2019 15:05:24 -0700 Subject: [PATCH 087/228] Bug fix: It should be AND so that we don't go and try to fetch meta which do not exist. (cherry picked from commit 015a9578dd495f6d3c119f3ef937a93e9cccf694) --- .../src/main/java/com/netflix/priam/backupv2/BackupTTLTask.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/priam/src/main/java/com/netflix/priam/backupv2/BackupTTLTask.java b/priam/src/main/java/com/netflix/priam/backupv2/BackupTTLTask.java index 294d8a2be..6f6e36419 100644 --- a/priam/src/main/java/com/netflix/priam/backupv2/BackupTTLTask.java +++ b/priam/src/main/java/com/netflix/priam/backupv2/BackupTTLTask.java @@ -148,7 +148,7 @@ public void execute() throws Exception { new DateUtil.DateRange( start_of_feature, dateToTtl.minus(1, ChronoUnit.HOURS))); - if (metas != null || metas.size() != 0) { + if (metas != null && metas.size() != 0) { logger.info( "Will delete(TTL) {} META files starting from: [{}]", metas.size(), From 6b4a8d77f2fd7bbffe1ee4e82901ea67a90ec1f9 Mon Sep 17 00:00:00 2001 From: aagrawal Date: Fri, 21 Jun 2019 09:48:09 -0700 Subject: [PATCH 088/228] Removing functionality of incremental manifest file as no one uses it. (cherry picked from commit 54d8b01147b1a3d6d49574bccd2222cfda790cbb) --- .../priam/backup/IncrementalBackup.java | 21 +----- .../priam/backup/IncrementalMetaData.java | 66 ------------------- .../priam/connection/CassandraOperations.java | 17 +++-- .../netflix/priam/connection/JMXNodeTool.java | 11 ++++ .../priam/identity/InstanceIdentity.java | 3 + .../identity/token/DeadTokenRetriever.java | 26 +++++++- .../identity/token/TokenRetrieverUtils.java | 53 ++++----------- .../priam/resources/CassandraAdmin.java | 19 +++++- .../com/netflix/priam/backup/TestBackup.java | 4 +- .../token/DeadTokenRetrieverTest.java | 4 +- .../token/TokenRetrieverUtilsTest.java | 20 +++--- 11 files changed, 94 insertions(+), 150 deletions(-) delete mode 100644 priam/src/main/java/com/netflix/priam/backup/IncrementalMetaData.java diff --git a/priam/src/main/java/com/netflix/priam/backup/IncrementalBackup.java b/priam/src/main/java/com/netflix/priam/backup/IncrementalBackup.java index a32fff28d..ef8d9740a 100644 --- a/priam/src/main/java/com/netflix/priam/backup/IncrementalBackup.java +++ b/priam/src/main/java/com/netflix/priam/backup/IncrementalBackup.java @@ -25,10 +25,8 @@ import com.netflix.priam.config.IConfiguration; import com.netflix.priam.scheduler.SimpleTimer; import com.netflix.priam.scheduler.TaskTimer; -import com.netflix.priam.utils.DateUtil; import java.io.File; import java.nio.file.Path; -import java.util.List; import java.util.Set; import org.apache.commons.io.FileUtils; import org.slf4j.Logger; @@ -41,7 +39,6 @@ public class IncrementalBackup extends AbstractBackup { private static final Logger logger = LoggerFactory.getLogger(IncrementalBackup.class); public static final String JOBNAME = "IncrementalBackup"; - private final IncrementalMetaData metaData; private final BackupRestoreUtil backupRestoreUtil; private final IBackupRestoreConfig backupRestoreConfig; @@ -50,12 +47,10 @@ public IncrementalBackup( IConfiguration config, IBackupRestoreConfig backupRestoreConfig, Provider pathFactory, - IFileSystemContext backupFileSystemCtx, - IncrementalMetaData metaData) { + IFileSystemContext backupFileSystemCtx) { super(config, backupFileSystemCtx, pathFactory); // a means to upload audit trail (via meta_cf_yyyymmddhhmm.json) of files successfully // uploaded) - this.metaData = metaData; this.backupRestoreConfig = backupRestoreConfig; backupRestoreUtil = new BackupRestoreUtil( @@ -120,18 +115,6 @@ protected void processColumnFamily(String keyspace, String columnFamily, File ba BackupFileType fileType = BackupFileType.SST; if (backupRestoreConfig.enableV2Backups()) fileType = BackupFileType.SST_V2; - List uploadedFiles = - upload(backupDir, fileType, config.enableAsyncIncremental(), true); - - if (!uploadedFiles.isEmpty()) { - // format of yyyymmddhhmm (e.g. 201505060901) - String incrementalUploadTime = - DateUtil.formatyyyyMMddHHmm(uploadedFiles.get(0).getTime()); - String metaFileName = "meta_" + columnFamily + "_" + incrementalUploadTime; - logger.info("Uploading meta file for incremental backup: {}", metaFileName); - this.metaData.setMetaFileName(metaFileName); - this.metaData.set(uploadedFiles, incrementalUploadTime); - logger.info("Uploaded meta file for incremental backup: {}", metaFileName); - } + upload(backupDir, fileType, config.enableAsyncIncremental(), true); } } diff --git a/priam/src/main/java/com/netflix/priam/backup/IncrementalMetaData.java b/priam/src/main/java/com/netflix/priam/backup/IncrementalMetaData.java deleted file mode 100644 index 99aaf31c2..000000000 --- a/priam/src/main/java/com/netflix/priam/backup/IncrementalMetaData.java +++ /dev/null @@ -1,66 +0,0 @@ -/** - * Copyright 2017 Netflix, Inc. - * - *

    Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file - * except in compliance with the License. You may obtain a copy of the License at - * - *

    http://www.apache.org/licenses/LICENSE-2.0 - * - *

    Unless required by applicable law or agreed to in writing, software distributed under the - * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.netflix.priam.backup; - -import com.google.inject.Inject; -import com.google.inject.Provider; -import com.netflix.priam.config.IConfiguration; -import java.io.File; -import java.io.IOException; -import org.apache.commons.io.FileUtils; - -public class IncrementalMetaData extends MetaData { - - private String metaFileName = null; // format meta_cf_time (e.g. - - @Inject - public IncrementalMetaData( - IConfiguration config, - Provider pathFactory, - IFileSystemContext backupFileSystemCtx) { - super(pathFactory, backupFileSystemCtx, config); - } - - public void setMetaFileName(String name) { - this.metaFileName = name; - } - - @Override - public File createTmpMetaFile() throws IOException { - File metafile, destFile; - - if (this.metaFileName == null) { - - metafile = File.createTempFile("incrementalMeta", ".json"); - destFile = new File(metafile.getParent(), "incrementalMeta.json"); - - } else { - metafile = File.createTempFile(this.metaFileName, ".json"); - destFile = new File(metafile.getParent(), this.metaFileName + ".json"); - } - - if (destFile.exists()) destFile.delete(); - - try { - - FileUtils.moveFile(metafile, destFile); - - } finally { - if (metafile != null && metafile.exists()) { // clean up resource - FileUtils.deleteQuietly(metafile); - } - } - return destFile; - } -} diff --git a/priam/src/main/java/com/netflix/priam/connection/CassandraOperations.java b/priam/src/main/java/com/netflix/priam/connection/CassandraOperations.java index b82fc13a3..f275f8212 100644 --- a/priam/src/main/java/com/netflix/priam/connection/CassandraOperations.java +++ b/priam/src/main/java/com/netflix/priam/connection/CassandraOperations.java @@ -175,11 +175,18 @@ public List> gossipInfo() throws Exception { .equalsIgnoreCase("TOKENS")) { // Special handling for tokens as it is always // "hidden". - gossipMap.put( - gossipLineEntry[0].trim().toUpperCase(), - nodeTool.getTokens( - gossipMap.get("PUBLIC_IP")) - .toString()); + try { + gossipMap.put( + gossipLineEntry[0].trim().toUpperCase(), + nodeTool.getTokens( + gossipMap.get( + "PUBLIC_IP")) + .toString()); + } catch (Exception e) { + logger.warn( + "Unable to find TOKEN(s) for the IP: {}", + gossipMap.get("PUBLIC_IP")); + } } else { gossipMap.put( gossipLineEntry[0].trim().toUpperCase(), diff --git a/priam/src/main/java/com/netflix/priam/connection/JMXNodeTool.java b/priam/src/main/java/com/netflix/priam/connection/JMXNodeTool.java index bae91c90f..0be900d83 100644 --- a/priam/src/main/java/com/netflix/priam/connection/JMXNodeTool.java +++ b/priam/src/main/java/com/netflix/priam/connection/JMXNodeTool.java @@ -278,6 +278,17 @@ public JSONObject info() throws JSONException { return object; } + public JSONObject statusInformation() throws JSONException { + JSONObject jsonObject = new JSONObject(); + jsonObject.put("live", getLiveNodes()); + jsonObject.put("unreachable", getUnreachableNodes()); + jsonObject.put("joining", getJoiningNodes()); + jsonObject.put("leaving", getLeavingNodes()); + jsonObject.put("moving", getMovingNodes()); + jsonObject.put("tokenToEndpointMap", getTokenToEndpointMap()); + return jsonObject; + } + public JSONArray ring(String keyspace) throws JSONException { JSONArray ring = new JSONArray(); Map tokenToEndpoint = getTokenToEndpointMap(); diff --git a/priam/src/main/java/com/netflix/priam/identity/InstanceIdentity.java b/priam/src/main/java/com/netflix/priam/identity/InstanceIdentity.java index 43abbf6d5..ab4ad880c 100644 --- a/priam/src/main/java/com/netflix/priam/identity/InstanceIdentity.java +++ b/priam/src/main/java/com/netflix/priam/identity/InstanceIdentity.java @@ -189,6 +189,9 @@ public PriamInstance retriableCall() throws Exception { if (!StringUtils.isEmpty(replaceIp) && !replaceIp.equals(instance.getHostIP())) { setReplacedIp(replaceIp); + logger.info( + "Priam found that the token is not alive according to Cassandra and we should start Cassandra in replace mode with replace ip: " + + replaceIp); } } catch (GossipParseException e) { } diff --git a/priam/src/main/java/com/netflix/priam/identity/token/DeadTokenRetriever.java b/priam/src/main/java/com/netflix/priam/identity/token/DeadTokenRetriever.java index 0fb036c74..784a46aec 100755 --- a/priam/src/main/java/com/netflix/priam/identity/token/DeadTokenRetriever.java +++ b/priam/src/main/java/com/netflix/priam/identity/token/DeadTokenRetriever.java @@ -111,9 +111,29 @@ public PriamInstance get() throws Exception { // remove it as we marked it down... factory.delete(priamInstance); - // use entry in the token database always. - // this can cause "can't replace live token errors. - this.replacedIp = priamInstance.getHostIP(); + // find the replaced IP + try { + replacedIp = + TokenRetrieverUtils.inferTokenOwnerFromGossip( + allInstancesWithinCluster, + priamInstance.getToken(), + priamInstance.getDC()); + + // Lets not replace the instance if gossip info is not merging!! + if (replacedIp == null) return null; + logger.info( + "Will try to replace token: {} with replacedIp (from gossip info): {} instead of ip from Token database: {}", + priamInstance.getToken(), + replacedIp, + priamInstance.getHostIP()); + } catch (TokenRetrieverUtils.GossipParseException e) { + // In case of gossip exception, fallback to IP in token database. + this.replacedIp = priamInstance.getHostIP(); + logger.info( + "Will try to replace token: {} with replacedIp from Token database: {}", + priamInstance.getToken(), + priamInstance.getHostIP()); + } PriamInstance result; try { diff --git a/priam/src/main/java/com/netflix/priam/identity/token/TokenRetrieverUtils.java b/priam/src/main/java/com/netflix/priam/identity/token/TokenRetrieverUtils.java index 26007262a..812c85ead 100644 --- a/priam/src/main/java/com/netflix/priam/identity/token/TokenRetrieverUtils.java +++ b/priam/src/main/java/com/netflix/priam/identity/token/TokenRetrieverUtils.java @@ -16,8 +16,7 @@ /** Common utilities for token retrieval. */ public class TokenRetrieverUtils { private static final Logger logger = LoggerFactory.getLogger(TokenRetrieverBase.class); - private static final String GOSSIP_INFO_URL_FORMAT = - "http://%s:8080/Priam/REST/v1/cassadmin/gossipinfo"; + private static final String STATUS_URL_FORMAT = "http://%s:8080/Priam/REST/v1/cassadmin/status"; /** * Utility method to infer the IP of the owner of a token in a given datacenter. This method @@ -36,11 +35,6 @@ public class TokenRetrieverUtils { public static String inferTokenOwnerFromGossip( List allIds, String token, String dc) throws GossipParseException { - // TODO: Gossip info in some cases doesn't reflect the real C* state. - // Not using gossip info for now. - if (!useGossipInfo()) { - return null; - } // Avoid using dead instance who we are trying to replace (duh!!) // Avoid other regions instances to avoid communication over public ip address. @@ -106,44 +100,21 @@ public static String inferTokenOwnerFromGossip( return null; } - // TODO: Gossip info in some cases doesn't reflect the real C* state. - // Not using gossip info for now. - private static boolean useGossipInfo() { - return false; - } - // helper method to get the token owner IP from a Cassandra node. private static String getIp(String host, String token) throws GossipParseException { String response = null; try { - response = SystemUtils.getDataFromUrl(String.format(GOSSIP_INFO_URL_FORMAT, host)); - - String inputToken = String.format("[%s]", token); - JSONParser parser = new JSONParser(); - JSONArray jsonObject = (JSONArray) parser.parse(response); - - for (Object key : jsonObject) { - JSONObject msg = (JSONObject) key; - - // Ensure that we are not trying to replace a NORMAL token and token of that - // instance matches what we want to replace. - if (msg.get("STATUS") == null - || msg.get("STATUS").toString().equalsIgnoreCase("NORMAL") - || msg.get("TOKENS") == null - || msg.get("PUBLIC_IP") == null - || !msg.get("TOKENS").toString().equals(inputToken)) { - continue; - } - - logger.info( - "Using gossip info from host[{}] and token[{}], the replaced address is : [{}]", - host, - token, - msg.get("PUBLIC_IP")); - return (String) msg.get("PUBLIC_IP"); - } - - return null; + response = SystemUtils.getDataFromUrl(String.format(STATUS_URL_FORMAT, host)); + JSONObject jsonObject = (JSONObject) new JSONParser().parse(response); + JSONArray liveNodes = (JSONArray) jsonObject.get("live"); + JSONObject tokenToEndpointMap = (JSONObject) jsonObject.get("tokenToEndpointMap"); + String endpointInfo = tokenToEndpointMap.get(token).toString(); + // We intentionally do not use the "unreachable" nodes as it may or may not be the best + // place to start. + // We just verify that the endpoint we provide is not "live". + if (liveNodes.contains(endpointInfo)) return null; + + return endpointInfo; } catch (RuntimeException e) { throw new GossipParseException( String.format("Error in reaching out to host: [{}]", host), e); diff --git a/priam/src/main/java/com/netflix/priam/resources/CassandraAdmin.java b/priam/src/main/java/com/netflix/priam/resources/CassandraAdmin.java index 0e64b3cdc..34ed43bf6 100644 --- a/priam/src/main/java/com/netflix/priam/resources/CassandraAdmin.java +++ b/priam/src/main/java/com/netflix/priam/resources/CassandraAdmin.java @@ -131,7 +131,7 @@ public Response cassPartitioner() throws IOException, InterruptedException, JSON nodeTool = JMXNodeTool.instance(config); } catch (JMXConnectionException e) { logger.error( - "Exception in fetching c* jmx tool . Msgl: {}", e.getLocalizedMessage(), e); + "Exception in fetching c* jmx tool . Msg: {}", e.getLocalizedMessage(), e); return Response.status(503).entity("JMXConnectionException").build(); } logger.debug("node tool getPartitioner being called"); @@ -147,13 +147,28 @@ public Response cassRing(@PathParam("id") String keyspace) nodeTool = JMXNodeTool.instance(config); } catch (JMXConnectionException e) { logger.error( - "Exception in fetching c* jmx tool . Msgl: {}", e.getLocalizedMessage(), e); + "Exception in fetching c* jmx tool . Msg: {}", e.getLocalizedMessage(), e); return Response.status(503).entity("JMXConnectionException").build(); } logger.debug("node tool ring being called"); return Response.ok(nodeTool.ring(keyspace), MediaType.APPLICATION_JSON).build(); } + @GET + @Path("/status") + public Response statusInfo() throws JSONException { + JMXNodeTool nodeTool; + try { + nodeTool = JMXNodeTool.instance(config); + } catch (JMXConnectionException e) { + logger.error( + "Exception in fetching c* jmx tool . Msg: {}", e.getLocalizedMessage(), e); + return Response.status(503).entity("JMXConnectionException").build(); + } + logger.debug("node tool status being called"); + return Response.ok(nodeTool.statusInformation(), MediaType.APPLICATION_JSON).build(); + } + @GET @Path("/flush") public Response cassFlush() throws IOException, InterruptedException, ExecutionException { diff --git a/priam/src/test/java/com/netflix/priam/backup/TestBackup.java b/priam/src/test/java/com/netflix/priam/backup/TestBackup.java index 12f2e3862..a79f63f06 100644 --- a/priam/src/test/java/com/netflix/priam/backup/TestBackup.java +++ b/priam/src/test/java/com/netflix/priam/backup/TestBackup.java @@ -91,7 +91,7 @@ public void testIncrementalBackup() throws Exception { generateIncrementalFiles(); IncrementalBackup backup = injector.getInstance(IncrementalBackup.class); backup.execute(); - Assert.assertEquals(5, filesystem.uploadedFiles.size()); + Assert.assertEquals(4, filesystem.uploadedFiles.size()); for (String filePath : expectedFiles) Assert.assertTrue(filesystem.uploadedFiles.contains(filePath)); } @@ -142,7 +142,7 @@ private void testClusterSpecificColumnFamiliesSkipped(String[] columnFamilyDirs) } IncrementalBackup backup = injector.getInstance(IncrementalBackup.class); backup.execute(); - Assert.assertEquals(8, filesystem.uploadedFiles.size()); + Assert.assertEquals(6, filesystem.uploadedFiles.size()); for (String filePath : expectedFiles) Assert.assertTrue(filesystem.uploadedFiles.contains(filePath)); } diff --git a/priam/src/test/java/com/netflix/priam/identity/token/DeadTokenRetrieverTest.java b/priam/src/test/java/com/netflix/priam/identity/token/DeadTokenRetrieverTest.java index 39ca92c70..cdcaea329 100644 --- a/priam/src/test/java/com/netflix/priam/identity/token/DeadTokenRetrieverTest.java +++ b/priam/src/test/java/com/netflix/priam/identity/token/DeadTokenRetrieverTest.java @@ -94,7 +94,7 @@ public void testNoReplacementNoSpotAvailable() throws Exception { }; } - // @Test + @Test // There is a potential slot for dead token but we are unable to replace. public void testNoReplacementNoGossipMatch(@Mocked SystemUtils systemUtils) throws Exception { List allInstances = getInstances(2); @@ -150,7 +150,7 @@ public void testNoReplacementNoGossipMatch(@Mocked SystemUtils systemUtils) thro }; } - // @Test + @Test public void testReplacementGossipMatch(@Mocked SystemUtils systemUtils) throws Exception { List allInstances = getInstances(6); List racMembership = getRacMembership(2); diff --git a/priam/src/test/java/com/netflix/priam/identity/token/TokenRetrieverUtilsTest.java b/priam/src/test/java/com/netflix/priam/identity/token/TokenRetrieverUtilsTest.java index 6ed7657a7..c8700611b 100644 --- a/priam/src/test/java/com/netflix/priam/identity/token/TokenRetrieverUtilsTest.java +++ b/priam/src/test/java/com/netflix/priam/identity/token/TokenRetrieverUtilsTest.java @@ -12,11 +12,11 @@ import junit.framework.Assert; import mockit.Expectations; import mockit.Mocked; +import org.junit.Test; public class TokenRetrieverUtilsTest { private static final String APP = "testapp"; - private static final String GOSSIP_INFO_URL_FORMAT = - "http://%s:8080/Priam/REST/v1/cassadmin/gossipinfo"; + private static final String STATUS_URL_FORMAT = "http://%s:8080/Priam/REST/v1/cassadmin/status"; private List instances = IntStream.range(0, 6) @@ -46,7 +46,7 @@ public class TokenRetrieverUtilsTest { .collect(Collectors.toList()) .toArray(new String[0]); - // @Test + @Test public void testRetrieveTokenOwnerWhenGossipAgrees(@Mocked SystemUtils systemUtils) throws Exception { // updates instances with new instance owning token 4 as per token database. @@ -79,7 +79,7 @@ public void testRetrieveTokenOwnerWhenGossipAgrees(@Mocked SystemUtils systemUti } @SuppressWarnings("unchecked") - // @Test + @Test public void testRetrieveTokenOwnerWhenGossipDisagrees(@Mocked SystemUtils systemUtils) throws Exception { // updates instances with new instance owning token 4 as per token database. @@ -99,14 +99,14 @@ public void testRetrieveTokenOwnerWhenGossipDisagrees(@Mocked SystemUtils system SystemUtils.getDataFromUrl( withArgThat( allOf( - not(String.format(GOSSIP_INFO_URL_FORMAT, "fakeHost-0")), - not(String.format(GOSSIP_INFO_URL_FORMAT, "fakeHost-5"))))); + not(String.format(STATUS_URL_FORMAT, "fakeHost-0")), + not(String.format(STATUS_URL_FORMAT, "fakeHost-5"))))); result = Arrays.toString(gossipInfoSetTwo); - SystemUtils.getDataFromUrl(String.format(GOSSIP_INFO_URL_FORMAT, "fakeHost-0")); + SystemUtils.getDataFromUrl(String.format(STATUS_URL_FORMAT, "fakeHost-0")); result = Arrays.toString(gossipInfoSetOne); minTimes = 0; - SystemUtils.getDataFromUrl(String.format(GOSSIP_INFO_URL_FORMAT, "fakeHost-5")); + SystemUtils.getDataFromUrl(String.format(STATUS_URL_FORMAT, "fakeHost-5")); result = Arrays.toString(gossipInfoSetOne); minTimes = 0; } @@ -117,7 +117,7 @@ public void testRetrieveTokenOwnerWhenGossipDisagrees(@Mocked SystemUtils system Assert.assertEquals(null, replaceIp); } - // @Test + @Test public void testRetrieveTokenOwnerWhenAllHostsInGossipReturnsNull( @Mocked SystemUtils systemUtils) throws Exception { // updates instances with new instance owning token 4 as per token database. @@ -149,7 +149,7 @@ public void testRetrieveTokenOwnerWhenAllHostsInGossipReturnsNull( Assert.assertNull(replaceIp); } - // @Test(expected = TokenRetrieverUtils.GossipParseException.class) + @Test(expected = TokenRetrieverUtils.GossipParseException.class) public void testRetrieveTokenOwnerWhenAllInstancesThrowGossipParseException( @Mocked SystemUtils systemUtils) throws TokenRetrieverUtils.GossipParseException { // updates instances with new instance owning token 4 as per token database. From 9e9f19c7650fd88635c16b5833037505af0f456d Mon Sep 17 00:00:00 2001 From: aagrawal Date: Fri, 26 Jul 2019 17:34:18 -0700 Subject: [PATCH 089/228] Fix the gossip status information. (cherry picked from commit 3c1a8d2ef0b986a4cd025f65182648decb9dd281) --- .../identity/token/TokenRetrieverUtils.java | 42 +++--- .../token/DeadTokenRetrieverTest.java | 62 +++++---- .../token/TokenRetrieverUtilsTest.java | 126 +++++++----------- 3 files changed, 107 insertions(+), 123 deletions(-) diff --git a/priam/src/main/java/com/netflix/priam/identity/token/TokenRetrieverUtils.java b/priam/src/main/java/com/netflix/priam/identity/token/TokenRetrieverUtils.java index 812c85ead..7011729c4 100644 --- a/priam/src/main/java/com/netflix/priam/identity/token/TokenRetrieverUtils.java +++ b/priam/src/main/java/com/netflix/priam/identity/token/TokenRetrieverUtils.java @@ -20,7 +20,7 @@ public class TokenRetrieverUtils { /** * Utility method to infer the IP of the owner of a token in a given datacenter. This method - * uses Cassandra gossip information to find the owner. While it is ideal to check all the nodes + * uses Cassandra status information to find the owner. While it is ideal to check all the nodes * in the ring to see if they agree on the IP to be replaced, in large clusters it may affect * the startup performance. This method picks at most 3 random hosts from the ring and see if * they all agree on the IP to be replaced. If not, it returns null. @@ -28,7 +28,8 @@ public class TokenRetrieverUtils { * @param allIds * @param token * @param dc - * @return IP of the token owner based on gossip information or null if gossip doesn't converge. + * @return IP of the token owner based on gossip information or null if C* status doesn't + * converge. * @throws GossipParseException when required number of instances are not available to fetch the * gossip info. */ @@ -68,14 +69,20 @@ public static String inferTokenOwnerFromGossip( String ip = getIp(instance.getHostName(), token); reachableInstances++; - if (StringUtils.isEmpty(ip)) { - continue; - } - if (replaceIp == null) { replaceIp = ip; - } else if (!replaceIp.equals(ip)) { - break; + } + + if (StringUtils.isEmpty(ip) || !replaceIp.equals(ip)) { + // If the IP address produced by getIp call is empty it means it was able to + // parse the status information and that token was still alive!! + // We do not want to do anything if token is considered as alive by Cassandra. + logger.info( + "Not producing anything in replaceIp as according to C* that token is still alive or " + + "there is a mismatch in status information per Cassandra. ip: [{}], replaceIp: [{}]", + ip, + replaceIp); + return null; } matchedGossipInstances++; @@ -91,12 +98,11 @@ public static String inferTokenOwnerFromGossip( // instances. if (reachableInstances < noOfInstancesGossipShouldMatch) { throw new GossipParseException( - "Unable to reach minimum required instances to fetch gossip information."); + String.format( + "Unable to find enough instances where gossip match. Required: [%d]", + noOfInstancesGossipShouldMatch)); } - logger.warn( - "Return null: Unable to find enough instances where gossip match. Required: {}", - noOfInstancesGossipShouldMatch); return null; } @@ -112,18 +118,20 @@ private static String getIp(String host, String token) throws GossipParseExcepti // We intentionally do not use the "unreachable" nodes as it may or may not be the best // place to start. // We just verify that the endpoint we provide is not "live". - if (liveNodes.contains(endpointInfo)) return null; + if (liveNodes.contains(endpointInfo)) { + logger.warn("The token [{}] is considered as alive by [{}].", token, host); + return null; + } return endpointInfo; } catch (RuntimeException e) { throw new GossipParseException( - String.format("Error in reaching out to host: [{}]", host), e); + String.format("Error in reaching out to host: [%s]", host), e); } catch (ParseException e) { throw new GossipParseException( String.format( - "Error in parsing gossip response [{}] from host: [{}]", - response, - host), + "Error in parsing gossip response [%s] from host: [%s]", + response, host), e); } } diff --git a/priam/src/test/java/com/netflix/priam/identity/token/DeadTokenRetrieverTest.java b/priam/src/test/java/com/netflix/priam/identity/token/DeadTokenRetrieverTest.java index cdcaea329..1c9ac7ab3 100644 --- a/priam/src/test/java/com/netflix/priam/identity/token/DeadTokenRetrieverTest.java +++ b/priam/src/test/java/com/netflix/priam/identity/token/DeadTokenRetrieverTest.java @@ -29,9 +29,13 @@ import com.netflix.priam.utils.FakeSleeper; import com.netflix.priam.utils.SystemUtils; import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; +import java.util.stream.IntStream; import mockit.Expectations; import mockit.Mocked; import mockit.Verifications; +import org.codehaus.jettison.json.JSONObject; import org.junit.Assert; import org.junit.Test; @@ -43,6 +47,17 @@ public class DeadTokenRetrieverTest { private InstanceInfo instanceInfo; private IConfiguration configuration; + private Map tokenToEndpointMap = + IntStream.range(0, 6) + .mapToObj(e -> Integer.valueOf(e)) + .collect( + Collectors.toMap( + e -> String.valueOf(e), e -> String.format("127.0.0.%s", e))); + private List liveInstances = + IntStream.range(0, 6) + .mapToObj(e -> String.format("127.0.0.%d", e)) + .collect(Collectors.toList()); + public DeadTokenRetrieverTest() { Injector injector = Guice.createInjector(new BRTestModule()); if (instanceInfo == null) instanceInfo = injector.getInstance(InstanceInfo.class); @@ -109,8 +124,7 @@ public void testNoReplacementNoGossipMatch(@Mocked SystemUtils systemUtils) thro membership.getRacMembership(); result = racMembership; SystemUtils.getDataFromUrl(anyString); - result = - "[{\"TOKENS\":\"[1]\",\"PUBLIC_IP\":\"\",\"RACK\":\"az1\",\"STATUS\":\"NORMAL\",\"DC\":\"us-east-1\"},{\"TOKENS\":\"[2]\",\"PUBLIC_IP\":\"\",\"RACK\":\"az1\",\"STATUS\":\"NORMAL\",\"DC\":\"us-east-1\"}]"; + result = getStatus(liveInstances, tokenToEndpointMap); times = 1; } }; @@ -125,29 +139,17 @@ public void testNoReplacementNoGossipMatch(@Mocked SystemUtils systemUtils) thro times = 1; } }; + } - // gossip info from another instance returns everything is OK and thus unable to replace. - String gossipInfo = - "[{\"TOKENS\":\"[1]\",\"PUBLIC_IP\":\"127.0.0.1\",\"RACK\":\"az1\",\"STATUS\":\"NORMAL\",\"DC\":\"us-east-1\"},{\"TOKENS\":\"[2]\",\"PUBLIC_IP\":\"127.0.0.2\",\"RACK\":\"az1\",\"STATUS\":\"NORMAL\",\"DC\":\"us-east-1\"}]"; - new Expectations() { - { - factory.getAllIds(anyString); - result = allInstances; - membership.getRacMembership(); - result = racMembership; - SystemUtils.getDataFromUrl(anyString); - result = gossipInfo; - times = 1; - } - }; - priamInstance = deadTokenRetriever.get(); - Assert.assertNull(priamInstance); - new Verifications() { - { - factory.delete(withAny(instance)); - times = 1; - } - }; + private String getStatus(List liveInstances, Map tokenToEndpointMap) { + JSONObject jsonObject = new JSONObject(); + try { + jsonObject.put("live", liveInstances); + jsonObject.put("tokenToEndpointMap", tokenToEndpointMap); + } catch (Exception e) { + + } + return jsonObject.toString(); } @Test @@ -155,8 +157,14 @@ public void testReplacementGossipMatch(@Mocked SystemUtils systemUtils) throws E List allInstances = getInstances(6); List racMembership = getRacMembership(2); racMembership.add(instanceInfo.getInstanceId()); - String gossipResponse = - "[{\"TOKENS\":\"[1]\",\"PUBLIC_IP\":\"127.0.0.1\",\"RACK\":\"az1\",\"STATUS\":\"NORMAL\",\"DC\":\"us-east-1\"},{\"TOKENS\":\"[2]\",\"PUBLIC_IP\":\"127.0.0.2\",\"RACK\":\"az1\",\"STATUS\":\"NORMAL\",\"DC\":\"us-east-1\"},{\"TOKENS\":\"[3]\",\"PUBLIC_IP\":\"127.0.0.3\",\"RACK\":\"az1\",\"STATUS\":\"shutdown\",\"DC\":\"us-east-1\"},{\"TOKENS\":\"[4]\",\"PUBLIC_IP\":\"127.0.0.4\",\"RACK\":\"az2\",\"STATUS\":\"NORMAL\",\"DC\":\"us-east-1\"},{\"TOKENS\":\"[5]\",\"PUBLIC_IP\":\"127.0.0.5\",\"RACK\":\"az2\",\"STATUS\":\"NORMAL\",\"DC\":\"us-east-1\"},{\"TOKENS\":\"[6]\",\"PUBLIC_IP\":\"127.0.0.6\",\"RACK\":\"az2\",\"STATUS\":\"NORMAL\",\"DC\":\"us-east-1\"}]"; + + List myliveInstances = + liveInstances + .stream() + .filter(x -> !x.equalsIgnoreCase("127.0.0.3")) + .collect(Collectors.toList()); + String gossipResponse = getStatus(myliveInstances, tokenToEndpointMap); + new Expectations() { { factory.getAllIds(anyString); @@ -164,7 +172,7 @@ public void testReplacementGossipMatch(@Mocked SystemUtils systemUtils) throws E membership.getRacMembership(); result = racMembership; SystemUtils.getDataFromUrl(anyString); - returns(gossipResponse, gossipResponse, null, gossipResponse); + returns(gossipResponse, gossipResponse, null, "random_value", gossipResponse); } }; deadTokenRetriever = diff --git a/priam/src/test/java/com/netflix/priam/identity/token/TokenRetrieverUtilsTest.java b/priam/src/test/java/com/netflix/priam/identity/token/TokenRetrieverUtilsTest.java index c8700611b..f78a66250 100644 --- a/priam/src/test/java/com/netflix/priam/identity/token/TokenRetrieverUtilsTest.java +++ b/priam/src/test/java/com/netflix/priam/identity/token/TokenRetrieverUtilsTest.java @@ -5,13 +5,14 @@ import com.netflix.priam.identity.PriamInstance; import com.netflix.priam.utils.SystemUtils; -import java.util.Arrays; import java.util.List; +import java.util.Map; import java.util.stream.Collectors; import java.util.stream.IntStream; -import junit.framework.Assert; import mockit.Expectations; import mockit.Mocked; +import org.codehaus.jettison.json.JSONObject; +import org.junit.Assert; import org.junit.Test; public class TokenRetrieverUtilsTest { @@ -33,44 +34,31 @@ public class TokenRetrieverUtilsTest { String.valueOf(e))) .collect(Collectors.toList()); - private String[] gossipInfos = + private Map tokenToEndpointMap = IntStream.range(0, 6) - .mapToObj( - e -> - newGossipRecord( - e, - String.format("127.0.0.%d", e), - "us-east-1", - (e < 3) ? "az1" : "az2", - "NORMAL")) - .collect(Collectors.toList()) - .toArray(new String[0]); + .mapToObj(e -> Integer.valueOf(e)) + .collect( + Collectors.toMap( + e -> String.valueOf(e), e -> String.format("127.0.0.%s", e))); + private List liveInstances = + IntStream.range(0, 6) + .mapToObj(e -> String.format("127.0.0.%d", e)) + .collect(Collectors.toList()); @Test public void testRetrieveTokenOwnerWhenGossipAgrees(@Mocked SystemUtils systemUtils) throws Exception { - // updates instances with new instance owning token 4 as per token database. - List newInstances = - instances.stream().filter(e -> e.getId() != 4).collect(Collectors.toList()); - newInstances.add( - newMockPriamInstance( - APP, - "us-east", - "az2", - 4, - "fakeHost-400", - "127.0.0.400", - "fakeHost-400", - "4")); - // mark previous instance with tokenNumber 4 as down in gossip. - String[] newGossipInfos = Arrays.copyOf(gossipInfos, gossipInfos.length); - newGossipInfos[4] = newGossipRecord(4, "127.0.0.4", "us-east-1", "az2", "shutdown"); + List myliveInstances = + liveInstances + .stream() + .filter(x -> !x.equalsIgnoreCase("127.0.0.4")) + .collect(Collectors.toList()); new Expectations() { { SystemUtils.getDataFromUrl(anyString); - result = Arrays.toString(newGossipInfos); + result = getStatus(myliveInstances, tokenToEndpointMap); } }; @@ -78,21 +66,15 @@ public void testRetrieveTokenOwnerWhenGossipAgrees(@Mocked SystemUtils systemUti Assert.assertEquals("127.0.0.4", replaceIp); } - @SuppressWarnings("unchecked") @Test public void testRetrieveTokenOwnerWhenGossipDisagrees(@Mocked SystemUtils systemUtils) throws Exception { - // updates instances with new instance owning token 4 as per token database. - List newInstances = - instances.stream().filter(e -> e.getId() != 4).collect(Collectors.toList()); - // mark previous instance as token owner for tokenNumber 4 in the gossip. - String[] gossipInfoSetOne = Arrays.copyOf(gossipInfos, gossipInfos.length); - gossipInfoSetOne[4] = newGossipRecord(4, "127.0.0.4", "us-east-1", "az2", "shutdown"); - - // mark empty as token owner for tokenNumber 4 in the gossip. - String[] gossipInfoSetTwo = Arrays.copyOf(gossipInfos, gossipInfos.length); - gossipInfoSetTwo[4] = newGossipRecord(4, "", "us-east-1", "az2", "shutdown"); + List myliveInstances = + liveInstances + .stream() + .filter(x -> !x.equalsIgnoreCase("127.0.0.4")) + .collect(Collectors.toList()); new Expectations() { { @@ -100,48 +82,36 @@ public void testRetrieveTokenOwnerWhenGossipDisagrees(@Mocked SystemUtils system withArgThat( allOf( not(String.format(STATUS_URL_FORMAT, "fakeHost-0")), + not(String.format(STATUS_URL_FORMAT, "fakeHost-2")), not(String.format(STATUS_URL_FORMAT, "fakeHost-5"))))); - result = Arrays.toString(gossipInfoSetTwo); + result = getStatus(myliveInstances, tokenToEndpointMap); + minTimes = 0; SystemUtils.getDataFromUrl(String.format(STATUS_URL_FORMAT, "fakeHost-0")); - result = Arrays.toString(gossipInfoSetOne); + result = getStatus(liveInstances, tokenToEndpointMap); minTimes = 0; + + SystemUtils.getDataFromUrl(String.format(STATUS_URL_FORMAT, "fakeHost-2")); + result = getStatus(liveInstances, tokenToEndpointMap); + minTimes = 0; + SystemUtils.getDataFromUrl(String.format(STATUS_URL_FORMAT, "fakeHost-5")); - result = Arrays.toString(gossipInfoSetOne); + result = getStatus(liveInstances, tokenToEndpointMap); minTimes = 0; } }; - String replaceIp = - TokenRetrieverUtils.inferTokenOwnerFromGossip(newInstances, "4", "us-east"); + String replaceIp = TokenRetrieverUtils.inferTokenOwnerFromGossip(instances, "4", "us-east"); Assert.assertEquals(null, replaceIp); } @Test public void testRetrieveTokenOwnerWhenAllHostsInGossipReturnsNull( @Mocked SystemUtils systemUtils) throws Exception { - // updates instances with new instance owning token 4 as per token database. - List newInstances = - instances.stream().filter(e -> e.getId() != 4).collect(Collectors.toList()); - newInstances.add( - newMockPriamInstance( - APP, - "us-east", - "az2", - 4, - "fakeInstance-400", - "127.0.0.400", - "fakeHost-400", - "4")); - - // mark empty as token owner for tokenNumber 4 in the gossip. - String[] gossipInfo = Arrays.copyOf(gossipInfos, gossipInfos.length); - gossipInfo[4] = newGossipRecord(4, "", "us-east-1", "az2", "shutdown"); - new Expectations() { { SystemUtils.getDataFromUrl(anyString); - result = Arrays.toString(gossipInfo); + result = getStatus(liveInstances, tokenToEndpointMap); } }; @@ -152,24 +122,11 @@ public void testRetrieveTokenOwnerWhenAllHostsInGossipReturnsNull( @Test(expected = TokenRetrieverUtils.GossipParseException.class) public void testRetrieveTokenOwnerWhenAllInstancesThrowGossipParseException( @Mocked SystemUtils systemUtils) throws TokenRetrieverUtils.GossipParseException { - // updates instances with new instance owning token 4 as per token database. - List newInstances = - instances.stream().filter(e -> e.getId() != 4).collect(Collectors.toList()); - newInstances.add( - newMockPriamInstance( - APP, - "us-east", - "az2", - 4, - "fakeInstance-400", - "127.0.0.400", - "fakeHost-400", - "4")); new Expectations() { { SystemUtils.getDataFromUrl(anyString); - result = new TokenRetrieverUtils.GossipParseException(); + result = new TokenRetrieverUtils.GossipParseException("Test"); } }; @@ -184,6 +141,17 @@ private String newGossipRecord( tokenNumber, ip, dc, status, rack); } + private String getStatus(List liveInstances, Map tokenToEndpointMap) { + JSONObject jsonObject = new JSONObject(); + try { + jsonObject.put("live", liveInstances); + jsonObject.put("tokenToEndpointMap", tokenToEndpointMap); + } catch (Exception e) { + + } + return jsonObject.toString(); + } + private PriamInstance newMockPriamInstance( String app, String dc, From 3a8c518c015838e7d34ec5ed98a29cc4f214a7e7 Mon Sep 17 00:00:00 2001 From: aagrawal Date: Fri, 23 Aug 2019 11:47:48 -0700 Subject: [PATCH 090/228] Migrate travis to openjdk8 instead of oraclejdk8 (available in trusty only) --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 2a1b23979..29dbc1298 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,6 +1,6 @@ language: java jdk: -- oraclejdk8 +- openjdk8 sudo: false git: depth: 100 From 8c4a1f2f8117f89ff2b7fc58056a515da7640dc8 Mon Sep 17 00:00:00 2001 From: aagrawal Date: Wed, 16 Oct 2019 14:22:21 -0700 Subject: [PATCH 091/228] changelog update --- CHANGELOG.md | 57 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index cdba4026a..df6373cc2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,61 @@ # Changelog +## 2019/10/16 3.11.54 +(#834) Removing functionality of creating incremental manifest file in backup V1 as it is not used. +(#834) Bug fix: When meta file do not exist for TTL in backup v2 we should not be throwing NPE. +(#834) Bug fix: Fix X-Y-Z issue using gossip status information instead of gossip state. Note that gossip status is (JOINING/LEAVING/NORMAL) while gossip state is (UP/DOWN). Gossip state is calculated individually by all the Cassandra instances using gossip status. + +## 2019/06/07 3.11.53 +(#826): Rollback the fixes to use Gossip info while grabbing dead and pre-assigned tokens. Gossip info doesn't not reflect the correct cluster state always. A node marked with status as NORMAL in the Gossip info could actually be down. This can be checked using nt ring. This change will unblock the nodes from joining the ring. + +## 2019/05/28 3.11.52 +(#824): Use replace_address instead of replace_address_first_boot. replace_address always try to bootstrap Cassandra in replace mode even when the previous bootstrap is successful. replace_address_first_boot tries to bootstrap normally if the node already bootstrapped successfully. + +## 2019/05/14 3.11.51 +(#818) Changing the list in TokenRetrievalUtils to use wildcards. + +## 2019/05/13 3.11.50 +(#816) Priam will check Cassandra gossip information while grabbing pre-assigned token to decide if it should start Cassandra in bootstrap mode or in replace mode. +(#816) At most 3 random nodes are used to get the gossip information. +(#816) Moved token owner inferring logic based on Cassandra gossip into a util class. +(#816) Refactored InstanceIdentity.init() method. + +## 2019/04/29 3.11.49 +(#815) Update the backup service based on configuration changes. +(#812) Expose the list of files from backups as API call. +(#809) Run TTL for backup based on a simple timer to avoid S3 delete API call throttle. +(#815) API to clear the local filesystem cache. +(#815) Bug fix: Increment backup failure metric when no backup is found. +(#809) Bug fix: No backup verification job during restore. + +## 2019/03/19 3.11.48 +(#807) Fix X->Y->Z issue. Replace nodes when gossip actually converges. + +## 2019/03/13 3.11.47 +(#804) Write-thru cache in AbstractFileSystem. +(#803) Take care of issue - C* snapshot w.r.t. filesystem is not "sync" in nature. + +## 2019/03/05 3.11.46 +(#794) Fix for forgotten file +(#798) Use older API for prefix filtering (backup TTL), if prefix is available. +(#800) Send notifications only when we upload a file. + +## 2019/02/27 3.11.45 +(#793) S3 - BucketLifecycleConfiguration has `prefix` method removed from latest library. + +## 2019/02/27 3.11.44 +(#791) BackupServlet had an API call of backup status which was producing output which was not JSON. + +## 2019/02/15 3.11.43 +(#784): BackupVerificationService +(#781): Put a cache for the getObjectExist API call to S3. This will help keep the cost of this call at bay. +(#781): Put a rate limiter for getObjectExist API call to S3 so we can limit the no. of calls. +(#784): Provide an override method to force Priam to replace a particular IP. + +## 2019/02/08 3.11.42 +(#777)Do not check existence of file if it is not SST_V2. S3 may decide to slow down and throw an error. Best not to do s3 object check (API) if it is not required. + +## 2019/02/07 3.11.41 +(#775) Do not throw NPE when no backup is found for the requested date. ## 2019/01/30 3.11.39 (#765) Add metrics on CassandraConfig resource calls From 169b16b8108f0687425266c7ab2105e8f089c527 Mon Sep 17 00:00:00 2001 From: aagrawal Date: Wed, 16 Oct 2019 15:09:26 -0700 Subject: [PATCH 092/228] update dependencies. --- build.gradle | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/build.gradle b/build.gradle index 0a21f0f54..f248c0050 100644 --- a/build.gradle +++ b/build.gradle @@ -1,6 +1,6 @@ plugins { - id 'nebula.netflixoss' version '7.1.2' - id 'com.github.sherter.google-java-format' version '0.7.1' + id 'nebula.netflixoss' version '7.1.3' + id 'com.github.sherter.google-java-format' version '0.8' } repositories { @@ -41,13 +41,13 @@ allprojects { compile 'com.sun.jersey.contribs:jersey-guice:1.19.4' compile 'com.google.guava:guava:21.0' compile 'com.google.code.findbugs:jsr305:3.0.2' - compile 'com.amazonaws:aws-java-sdk:1.11.501' + compile 'com.amazonaws:aws-java-sdk:1.11.653' compile 'com.google.inject:guice:4.2.2' compile 'com.google.inject.extensions:guice-servlet:4.2.2' compile 'org.quartz-scheduler:quartz:2.3.0' compile 'com.googlecode.json-simple:json-simple:1.1.1' - compile 'org.xerial.snappy:snappy-java:1.1.7.2' - compile 'org.yaml:snakeyaml:1.23' + compile 'org.xerial.snappy:snappy-java:1.1.7.3' + compile 'org.yaml:snakeyaml:1.25' compile 'org.apache.cassandra:cassandra-all:3.0.17' compile 'javax.ws.rs:jsr311-api:1.1.1' compile 'joda-time:joda-time:2.10.1' @@ -58,8 +58,8 @@ allprojects { compile 'org.apache.httpcomponents:httpcore:4.4.11' compile 'com.ning:compress-lzf:1.0.4' compile 'com.google.code.gson:gson:2.8.5' - compile 'org.slf4j:slf4j-api:1.7.25' - compile 'org.slf4j:slf4j-log4j12:1.7.25' + compile 'org.slf4j:slf4j-api:1.7.28' + compile 'org.slf4j:slf4j-log4j12:1.7.28' compile 'org.bouncycastle:bcprov-jdk16:1.46' compile 'org.bouncycastle:bcpg-jdk16:1.46' compile ('com.google.appengine.tools:appengine-gcs-client:0.8') { @@ -67,7 +67,7 @@ allprojects { } compile 'com.google.apis:google-api-services-storage:v1-rev141-1.25.0' compile 'com.google.http-client:google-http-client-jackson2:1.28.0' - compile 'com.netflix.spectator:spectator-api:0.83.0' + compile 'com.netflix.spectator:spectator-api:0.96.0' compileOnly 'javax.servlet:javax.servlet-api:3.1.0' testCompile 'org.jmockit:jmockit:1.31' testCompile "org.spockframework:spock-core:1.1-groovy-2.4" From fcc44642feb6a1a6e330b4f8756e74b3f4f8f223 Mon Sep 17 00:00:00 2001 From: aagrawal Date: Wed, 16 Oct 2019 15:35:29 -0700 Subject: [PATCH 093/228] bug fixes. 1> Change return type of partitioner call to text. 2> C* sometimes includes files which are 0 bytes long. 3> Chunk size needs to consider files which `inflate` after compression. --- .../java/com/netflix/priam/aws/S3FileSystem.java | 7 ++++++- .../com/netflix/priam/aws/S3FileSystemBase.java | 13 +++---------- .../com/netflix/priam/resources/CassandraAdmin.java | 2 +- 3 files changed, 10 insertions(+), 12 deletions(-) diff --git a/priam/src/main/java/com/netflix/priam/aws/S3FileSystem.java b/priam/src/main/java/com/netflix/priam/aws/S3FileSystem.java index a51794d61..bdb05bc75 100644 --- a/priam/src/main/java/com/netflix/priam/aws/S3FileSystem.java +++ b/priam/src/main/java/com/netflix/priam/aws/S3FileSystem.java @@ -213,7 +213,12 @@ protected long uploadFileImpl(Path localPath, Path remotePath) throws BackupRest } byte[] chunk = byteArrayOutputStream.toByteArray(); long compressedFileSize = chunk.length; - rateLimiter.acquire(chunk.length); + /** + * Weird, right that we are checking for length which is positive. You can thanks + * this to sometimes C* creating files which are zero bytes, and giving that in + * snapshot for some unknown reason. + */ + if (chunk.length > 0) rateLimiter.acquire(chunk.length); ObjectMetadata objectMetadata = getObjectMetadata(localPath); objectMetadata.setContentLength(chunk.length); PutObjectRequest putObjectRequest = diff --git a/priam/src/main/java/com/netflix/priam/aws/S3FileSystemBase.java b/priam/src/main/java/com/netflix/priam/aws/S3FileSystemBase.java index 302c8cda1..4df59bfa6 100755 --- a/priam/src/main/java/com/netflix/priam/aws/S3FileSystemBase.java +++ b/priam/src/main/java/com/netflix/priam/aws/S3FileSystemBase.java @@ -282,17 +282,10 @@ public void deleteFiles(List remotePaths) throws BackupRestoreException { } } - final long getChunkSize(Path localPath) throws BackupRestoreException { + final long getChunkSize(Path localPath) { long chunkSize = config.getBackupChunkSize(); - long fileSize = localPath.toFile().length(); - - // compute the size of each block we will upload to endpoint - if (fileSize > 0) - chunkSize = - (fileSize / chunkSize >= MAX_CHUNKS) - ? (fileSize / (MAX_CHUNKS - 1)) - : chunkSize; - + long proposedChunkSize = localPath.toFile().length() / (MAX_CHUNKS - 5); + if (proposedChunkSize > chunkSize) return proposedChunkSize; return chunkSize; } } diff --git a/priam/src/main/java/com/netflix/priam/resources/CassandraAdmin.java b/priam/src/main/java/com/netflix/priam/resources/CassandraAdmin.java index 34ed43bf6..6894ffd4d 100644 --- a/priam/src/main/java/com/netflix/priam/resources/CassandraAdmin.java +++ b/priam/src/main/java/com/netflix/priam/resources/CassandraAdmin.java @@ -135,7 +135,7 @@ public Response cassPartitioner() throws IOException, InterruptedException, JSON return Response.status(503).entity("JMXConnectionException").build(); } logger.debug("node tool getPartitioner being called"); - return Response.ok(nodeTool.getPartitioner(), MediaType.APPLICATION_JSON).build(); + return Response.ok(nodeTool.getPartitioner(), MediaType.TEXT_PLAIN).build(); } @GET From e761038b2a3c1d20a2c752cd2537976c67b1c776 Mon Sep 17 00:00:00 2001 From: aagrawal Date: Wed, 23 Oct 2019 13:48:00 -0700 Subject: [PATCH 094/228] Move flush and compactions to Service (cherry picked from commit 6b4ac135258c9269a846b780fbffa2079eb7b801) --- .../java/com/netflix/priam/PriamServer.java | 15 +++--- .../management/ClusterManagementService.java | 48 +++++++++++++++++++ .../token/AssignedTokenRetrieverTest.java | 2 +- 3 files changed, 56 insertions(+), 9 deletions(-) create mode 100644 priam/src/main/java/com/netflix/priam/cluster/management/ClusterManagementService.java diff --git a/priam/src/main/java/com/netflix/priam/PriamServer.java b/priam/src/main/java/com/netflix/priam/PriamServer.java index 5145680bd..91cf66ea7 100644 --- a/priam/src/main/java/com/netflix/priam/PriamServer.java +++ b/priam/src/main/java/com/netflix/priam/PriamServer.java @@ -21,8 +21,7 @@ import com.netflix.priam.aws.UpdateSecuritySettings; import com.netflix.priam.backup.BackupService; import com.netflix.priam.backupv2.BackupV2Service; -import com.netflix.priam.cluster.management.Compaction; -import com.netflix.priam.cluster.management.Flush; +import com.netflix.priam.cluster.management.ClusterManagementService; import com.netflix.priam.config.IConfiguration; import com.netflix.priam.config.PriamConfigurationPersister; import com.netflix.priam.defaultimpl.ICassandraProcess; @@ -50,6 +49,7 @@ public class PriamServer implements IService { private final IService backupV2Service; private final IService backupService; private final IService cassandraTunerService; + private final IService clusterManagementService; private static final int CASSANDRA_MONITORING_INITIAL_DELAY = 10; private static final Logger logger = LoggerFactory.getLogger(PriamServer.class); @@ -63,7 +63,8 @@ public PriamServer( RestoreContext restoreContext, BackupService backupService, BackupV2Service backupV2Service, - CassandraTunerService cassandraTunerService) { + CassandraTunerService cassandraTunerService, + ClusterManagementService clusterManagementService) { this.config = config; this.scheduler = scheduler; this.instanceIdentity = id; @@ -73,6 +74,7 @@ public PriamServer( this.backupService = backupService; this.backupV2Service = backupV2Service; this.cassandraTunerService = cassandraTunerService; + this.clusterManagementService = clusterManagementService; } private void createDirectories() throws IOException { @@ -135,11 +137,8 @@ public void scheduleService() throws Exception { CassandraMonitor.getTimer(), CASSANDRA_MONITORING_INITIAL_DELAY); - // Set up nodetool flush task - scheduleTask(scheduler, Flush.class, Flush.getTimer(config)); - - // Set up compaction task - scheduleTask(scheduler, Compaction.class, Compaction.getTimer(config)); + // Set up management services like flush, compactions etc. + clusterManagementService.scheduleService(); // Set up the background configuration dumping thread scheduleTask( diff --git a/priam/src/main/java/com/netflix/priam/cluster/management/ClusterManagementService.java b/priam/src/main/java/com/netflix/priam/cluster/management/ClusterManagementService.java new file mode 100644 index 000000000..eaa75680a --- /dev/null +++ b/priam/src/main/java/com/netflix/priam/cluster/management/ClusterManagementService.java @@ -0,0 +1,48 @@ +package com.netflix.priam.cluster.management; +/* + * Copyright 2019 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +import com.netflix.priam.config.IConfiguration; +import com.netflix.priam.defaultimpl.IService; +import com.netflix.priam.scheduler.PriamScheduler; +import javax.inject.Inject; + +public class ClusterManagementService implements IService { + private final PriamScheduler scheduler; + private final IConfiguration config; + + @Inject + public ClusterManagementService(IConfiguration configuration, PriamScheduler priamScheduler) { + this.scheduler = priamScheduler; + this.config = configuration; + } + + @Override + public void scheduleService() throws Exception { + // Set up nodetool flush task + scheduleTask(scheduler, Flush.class, Flush.getTimer(config)); + + // Set up compaction task + scheduleTask(scheduler, Compaction.class, Compaction.getTimer(config)); + } + + @Override + public void updateServicePre() throws Exception {} + + @Override + public void updateServicePost() throws Exception {} +} diff --git a/priam/src/test/java/com/netflix/priam/identity/token/AssignedTokenRetrieverTest.java b/priam/src/test/java/com/netflix/priam/identity/token/AssignedTokenRetrieverTest.java index 31d5ec1c5..a3a28e240 100644 --- a/priam/src/test/java/com/netflix/priam/identity/token/AssignedTokenRetrieverTest.java +++ b/priam/src/test/java/com/netflix/priam/identity/token/AssignedTokenRetrieverTest.java @@ -13,10 +13,10 @@ import java.util.List; import java.util.stream.Collectors; import java.util.stream.IntStream; -import junit.framework.Assert; import mockit.Expectations; import mockit.Mocked; import org.apache.commons.lang3.StringUtils; +import org.junit.Assert; import org.junit.Test; public class AssignedTokenRetrieverTest { From ba9c0aaad17f3dff133cd1549aec492dbb9eee7e Mon Sep 17 00:00:00 2001 From: aagrawal Date: Wed, 23 Oct 2019 15:48:31 -0700 Subject: [PATCH 095/228] Direct pinning to AWS Services --- build.gradle | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index f248c0050..af2997378 100644 --- a/build.gradle +++ b/build.gradle @@ -41,7 +41,15 @@ allprojects { compile 'com.sun.jersey.contribs:jersey-guice:1.19.4' compile 'com.google.guava:guava:21.0' compile 'com.google.code.findbugs:jsr305:3.0.2' - compile 'com.amazonaws:aws-java-sdk:1.11.653' + + // AWS Services + compile 'com.amazonaws:aws-java-sdk-s3:latest.release' + compile 'com.amazonaws:aws-java-sdk-sns:latest.release' + compile 'com.amazonaws:aws-java-sdk-ec2:latest.release' + compile 'com.amazonaws:aws-java-sdk-autoscaling:latest.release' + compile 'com.amazonaws:aws-java-sdk-sts:latest.release' + compile 'com.amazonaws:aws-java-sdk-simpledb:latest.release' + compile 'com.google.inject:guice:4.2.2' compile 'com.google.inject.extensions:guice-servlet:4.2.2' compile 'org.quartz-scheduler:quartz:2.3.0' From 3e7dd75d7299cd903a5bd9ed057af6f8be204d89 Mon Sep 17 00:00:00 2001 From: aagrawal Date: Thu, 24 Oct 2019 12:46:57 -0700 Subject: [PATCH 096/228] Send "SNAPSHOT_VERIFIED" message when the snapshot is successfully verified by the backup system. This will ensure that downward dependencies can start consuming the backup. (cherry picked from commit 2f1c7f26d3c663b54a2bb80c47e6f72c6d679dd0) --- .../priam/backup/AbstractBackupPath.java | 6 +- .../backupv2/BackupVerificationTask.java | 19 ++++-- .../notification/BackupNotificationMgr.java | 61 +++++++++++++++---- 3 files changed, 67 insertions(+), 19 deletions(-) diff --git a/priam/src/main/java/com/netflix/priam/backup/AbstractBackupPath.java b/priam/src/main/java/com/netflix/priam/backup/AbstractBackupPath.java index c3dd2afe2..47aca4698 100644 --- a/priam/src/main/java/com/netflix/priam/backup/AbstractBackupPath.java +++ b/priam/src/main/java/com/netflix/priam/backup/AbstractBackupPath.java @@ -43,12 +43,14 @@ public enum BackupFileType { CL, META, META_V2, - SST_V2; + SST_V2, + SNAPSHOT_VERIFIED; public static boolean isDataFile(BackupFileType type) { return type != BackupFileType.META && type != BackupFileType.META_V2 - && type != BackupFileType.CL; + && type != BackupFileType.CL + && type != SNAPSHOT_VERIFIED; } } diff --git a/priam/src/main/java/com/netflix/priam/backupv2/BackupVerificationTask.java b/priam/src/main/java/com/netflix/priam/backupv2/BackupVerificationTask.java index 6627a3261..e0b365962 100644 --- a/priam/src/main/java/com/netflix/priam/backupv2/BackupVerificationTask.java +++ b/priam/src/main/java/com/netflix/priam/backupv2/BackupVerificationTask.java @@ -19,14 +19,12 @@ import com.google.inject.Inject; import com.google.inject.Singleton; -import com.netflix.priam.backup.BackupVerification; -import com.netflix.priam.backup.BackupVerificationResult; -import com.netflix.priam.backup.BackupVersion; -import com.netflix.priam.backup.Status; +import com.netflix.priam.backup.*; import com.netflix.priam.config.IBackupRestoreConfig; import com.netflix.priam.config.IConfiguration; import com.netflix.priam.health.InstanceState; import com.netflix.priam.merics.BackupMetrics; +import com.netflix.priam.notification.BackupNotificationMgr; import com.netflix.priam.scheduler.CronTimer; import com.netflix.priam.scheduler.Task; import com.netflix.priam.scheduler.TaskTimer; @@ -47,6 +45,7 @@ public class BackupVerificationTask extends Task { private BackupVerification backupVerification; private BackupMetrics backupMetrics; private InstanceState instanceState; + private BackupNotificationMgr backupNotificationMgr; @Inject public BackupVerificationTask( @@ -54,12 +53,14 @@ public BackupVerificationTask( IBackupRestoreConfig backupRestoreConfig, BackupVerification backupVerification, BackupMetrics backupMetrics, - InstanceState instanceState) { + InstanceState instanceState, + BackupNotificationMgr backupNotificationMgr) { super(configuration); this.backupRestoreConfig = backupRestoreConfig; this.backupVerification = backupVerification; this.backupMetrics = backupMetrics; this.instanceState = instanceState; + this.backupNotificationMgr = backupNotificationMgr; } @Override @@ -95,6 +96,14 @@ public void execute() throws Exception { "Not able to find any snapshot which is valid in our SLO window: {} hours", backupRestoreConfig.getBackupVerificationSLOInHours()); backupMetrics.incrementBackupVerificationFailure(); + } else { + // verification result is available and is valid. + // send notification that backup is uploaded. + logger.info( + "Sending {} message for backup: {}", + AbstractBackupPath.BackupFileType.SNAPSHOT_VERIFIED, + verificationResult.get().snapshotInstant); + backupNotificationMgr.notify(verificationResult.get()); } } diff --git a/priam/src/main/java/com/netflix/priam/notification/BackupNotificationMgr.java b/priam/src/main/java/com/netflix/priam/notification/BackupNotificationMgr.java index ba775dce6..79da73191 100644 --- a/priam/src/main/java/com/netflix/priam/notification/BackupNotificationMgr.java +++ b/priam/src/main/java/com/netflix/priam/notification/BackupNotificationMgr.java @@ -16,7 +16,9 @@ import com.amazonaws.services.sns.model.MessageAttributeValue; import com.google.inject.Inject; import com.netflix.priam.backup.AbstractBackupPath; +import com.netflix.priam.backup.BackupVerificationResult; import com.netflix.priam.config.IConfiguration; +import com.netflix.priam.identity.InstanceIdentity; import com.netflix.priam.identity.config.InstanceInfo; import java.util.HashMap; import java.util.Map; @@ -39,15 +41,59 @@ public class BackupNotificationMgr implements EventObserver { private final IConfiguration config; private final INotificationService notificationService; private final InstanceInfo instanceInfo; + private final InstanceIdentity instanceIdentity; @Inject public BackupNotificationMgr( IConfiguration config, INotificationService notificationService, - InstanceInfo instanceInfo) { + InstanceInfo instanceInfo, + InstanceIdentity instanceIdentity) { this.config = config; this.notificationService = notificationService; this.instanceInfo = instanceInfo; + this.instanceIdentity = instanceIdentity; + } + + public void notify(BackupVerificationResult backupVerificationResult) { + JSONObject jsonObject = new JSONObject(); + try { + jsonObject.put("s3bucketname", this.config.getBackupPrefix()); + jsonObject.put("s3clustername", config.getAppName()); + jsonObject.put("s3namespace", backupVerificationResult.remotePath); + jsonObject.put("region", instanceInfo.getRegion()); + jsonObject.put("rack", instanceInfo.getRac()); + jsonObject.put("token", instanceIdentity.getInstance().getToken()); + jsonObject.put("backuptype", "SNAPSHOT_VERIFIED"); + jsonObject.put("snapshotInstant", backupVerificationResult.snapshotInstant); + // SNS Attributes for filtering messages. Cluster name and backup file type. + Map messageAttributes = + getMessageAttributes(AbstractBackupPath.BackupFileType.SNAPSHOT_VERIFIED); + + this.notificationService.notify(jsonObject.toString(), messageAttributes); + } catch (JSONException exception) { + logger.error( + "JSON exception during generation of notification for snapshot verification: {}. Msg: {}", + backupVerificationResult, + exception.getLocalizedMessage()); + } + } + + private Map getMessageAttributes( + AbstractBackupPath.BackupFileType backupFileType) { + // SNS Attributes for filtering messages. Cluster name and backup file type. + Map messageAttributes = new HashMap<>(); + messageAttributes.putIfAbsent( + "s3clustername", + new MessageAttributeValue() + .withDataType("String") + .withStringValue(config.getAppName())); + messageAttributes.putIfAbsent( + "backuptype", + new MessageAttributeValue() + .withDataType("String") + .withStringValue(backupFileType.name())); + return messageAttributes; } private void notify(AbstractBackupPath abp, String uploadStatus) { @@ -70,17 +116,8 @@ private void notify(AbstractBackupPath abp, String uploadStatus) { jsonObject.put("encryption", abp.getEncryption().name()); // SNS Attributes for filtering messages. Cluster name and backup file type. - Map messageAttributes = new HashMap<>(); - messageAttributes.putIfAbsent( - "s3clustername", - new MessageAttributeValue() - .withDataType("String") - .withStringValue(abp.getClusterName())); - messageAttributes.putIfAbsent( - "backuptype", - new MessageAttributeValue() - .withDataType("String") - .withStringValue(abp.getType().name())); + Map messageAttributes = + getMessageAttributes(abp.getType()); this.notificationService.notify(jsonObject.toString(), messageAttributes); } catch (JSONException exception) { From 1456ead975394e849d1f8213d920304ecf5b0d67 Mon Sep 17 00:00:00 2001 From: Sreeram Chakrovorthy Date: Thu, 13 Feb 2020 13:29:34 -0800 Subject: [PATCH 097/228] A single consolidated commit with all changes required to support filtering notifications for certain BackupFileTypes. --- .../priam/backup/AbstractBackupPath.java | 8 + .../priam/config/BackupRestoreConfig.java | 6 + .../priam/config/IBackupRestoreConfig.java | 16 + .../notification/BackupNotificationMgr.java | 83 +++-- .../TestBackupNotificationMgr.java | 294 ++++++++++++++++++ 5 files changed, 386 insertions(+), 21 deletions(-) create mode 100644 priam/src/test/java/com/netflix/priam/notification/TestBackupNotificationMgr.java diff --git a/priam/src/main/java/com/netflix/priam/backup/AbstractBackupPath.java b/priam/src/main/java/com/netflix/priam/backup/AbstractBackupPath.java index 47aca4698..021817d1f 100644 --- a/priam/src/main/java/com/netflix/priam/backup/AbstractBackupPath.java +++ b/priam/src/main/java/com/netflix/priam/backup/AbstractBackupPath.java @@ -52,6 +52,14 @@ public static boolean isDataFile(BackupFileType type) { && type != BackupFileType.CL && type != SNAPSHOT_VERIFIED; } + + public static BackupFileType fromString(String s) throws BackupRestoreException { + try { + return BackupFileType.valueOf(s); + } catch (IllegalArgumentException e) { + throw new BackupRestoreException(String.format("Unknown BackupFileType %s", s)); + } + } } protected BackupFileType type; diff --git a/priam/src/main/java/com/netflix/priam/config/BackupRestoreConfig.java b/priam/src/main/java/com/netflix/priam/config/BackupRestoreConfig.java index 21b088414..df19e2114 100644 --- a/priam/src/main/java/com/netflix/priam/config/BackupRestoreConfig.java +++ b/priam/src/main/java/com/netflix/priam/config/BackupRestoreConfig.java @@ -15,6 +15,7 @@ import com.netflix.priam.configSource.IConfigSource; import javax.inject.Inject; +import org.apache.commons.lang3.StringUtils; /** Implementation of IBackupRestoreConfig. Created by aagrawal on 6/26/18. */ public class BackupRestoreConfig implements IBackupRestoreConfig { @@ -55,4 +56,9 @@ public int getBackupVerificationSLOInHours() { public String getBackupVerificationCronExpression() { return config.get("priam.backupVerificationCronExpression", "0 30 0/1 1/1 * ? *"); } + + @Override + public String getBackupNotifyComponentIncludeList() { + return config.get("priam.backup.notify.component.include", StringUtils.EMPTY); + } } diff --git a/priam/src/main/java/com/netflix/priam/config/IBackupRestoreConfig.java b/priam/src/main/java/com/netflix/priam/config/IBackupRestoreConfig.java index ac39b6915..809791034 100644 --- a/priam/src/main/java/com/netflix/priam/config/IBackupRestoreConfig.java +++ b/priam/src/main/java/com/netflix/priam/config/IBackupRestoreConfig.java @@ -14,6 +14,7 @@ package com.netflix.priam.config; import com.google.inject.ImplementedBy; +import org.apache.commons.lang3.StringUtils; /** * This interface is to abstract out the backup and restore configuration used by Priam. Goal is to @@ -97,4 +98,19 @@ default int getBackupVerificationSLOInHours() { default boolean enableV2Restore() { return false; } + + /** + * Returns a csv of backup component file types {@link + * com.netflix.priam.backup.AbstractBackupPath.BackupFileType} on which to send backup + * notifications. Default value of this filter is an empty string which would imply that backup + * notifications will be sent for all component types see {@link + * com.netflix.priam.backup.AbstractBackupPath.BackupFileType}. Sample filter : + * "SNAPSHOT_VERIFIED, META_V2" + * + * @return A csv string that can be parsed to infer the component file types on which to send + * backup related notifications + */ + default String getBackupNotifyComponentIncludeList() { + return StringUtils.EMPTY; + } } diff --git a/priam/src/main/java/com/netflix/priam/notification/BackupNotificationMgr.java b/priam/src/main/java/com/netflix/priam/notification/BackupNotificationMgr.java index 79da73191..5d61389ba 100644 --- a/priam/src/main/java/com/netflix/priam/notification/BackupNotificationMgr.java +++ b/priam/src/main/java/com/netflix/priam/notification/BackupNotificationMgr.java @@ -16,12 +16,14 @@ import com.amazonaws.services.sns.model.MessageAttributeValue; import com.google.inject.Inject; import com.netflix.priam.backup.AbstractBackupPath; +import com.netflix.priam.backup.BackupRestoreException; import com.netflix.priam.backup.BackupVerificationResult; +import com.netflix.priam.config.IBackupRestoreConfig; import com.netflix.priam.config.IConfiguration; import com.netflix.priam.identity.InstanceIdentity; import com.netflix.priam.identity.config.InstanceInfo; -import java.util.HashMap; -import java.util.Map; +import java.util.*; +import org.apache.commons.lang3.StringUtils; import org.codehaus.jettison.json.JSONException; import org.codehaus.jettison.json.JSONObject; import org.slf4j.Logger; @@ -39,20 +41,28 @@ public class BackupNotificationMgr implements EventObserver { private static final String STARTED = "started"; private static final Logger logger = LoggerFactory.getLogger(BackupNotificationMgr.class); private final IConfiguration config; + private final IBackupRestoreConfig backupRestoreConfig; private final INotificationService notificationService; private final InstanceInfo instanceInfo; private final InstanceIdentity instanceIdentity; + private final Set notifiedBackupFileTypesSet; + private String notifiedBackupFileTypes; @Inject public BackupNotificationMgr( IConfiguration config, + IBackupRestoreConfig backupRestoreConfig, INotificationService notificationService, InstanceInfo instanceInfo, InstanceIdentity instanceIdentity) { this.config = config; + this.backupRestoreConfig = backupRestoreConfig; this.notificationService = notificationService; this.instanceInfo = instanceInfo; this.instanceIdentity = instanceIdentity; + this.notifiedBackupFileTypesSet = new HashSet<>(); + this.notifiedBackupFileTypes = + this.backupRestoreConfig.getBackupNotifyComponentIncludeList(); } public void notify(BackupVerificationResult backupVerificationResult) { @@ -99,27 +109,37 @@ private Map getMessageAttributes( private void notify(AbstractBackupPath abp, String uploadStatus) { JSONObject jsonObject = new JSONObject(); try { - jsonObject.put("s3bucketname", this.config.getBackupPrefix()); - jsonObject.put("s3clustername", abp.getClusterName()); - jsonObject.put("s3namespace", abp.getRemotePath()); - jsonObject.put("keyspace", abp.getKeyspace()); - jsonObject.put("cf", abp.getColumnFamily()); - jsonObject.put("region", abp.getRegion()); - jsonObject.put("rack", instanceInfo.getRac()); - jsonObject.put("token", abp.getToken()); - jsonObject.put("filename", abp.getFileName()); - jsonObject.put("uncompressfilesize", abp.getSize()); - jsonObject.put("compressfilesize", abp.getCompressedFileSize()); - jsonObject.put("backuptype", abp.getType().name()); - jsonObject.put("uploadstatus", uploadStatus); - jsonObject.put("compression", abp.getCompression().name()); - jsonObject.put("encryption", abp.getEncryption().name()); + Set updatedNotifiedBackupFileTypeSet = + getUpdatedNotifiedBackupFileTypesSet(this.notifiedBackupFileTypes); + if (updatedNotifiedBackupFileTypeSet.isEmpty() + || updatedNotifiedBackupFileTypeSet.contains(abp.getType())) { + jsonObject.put("s3bucketname", this.config.getBackupPrefix()); + jsonObject.put("s3clustername", abp.getClusterName()); + jsonObject.put("s3namespace", abp.getRemotePath()); + jsonObject.put("keyspace", abp.getKeyspace()); + jsonObject.put("cf", abp.getColumnFamily()); + jsonObject.put("region", abp.getRegion()); + jsonObject.put("rack", instanceInfo.getRac()); + jsonObject.put("token", abp.getToken()); + jsonObject.put("filename", abp.getFileName()); + jsonObject.put("uncompressfilesize", abp.getSize()); + jsonObject.put("compressfilesize", abp.getCompressedFileSize()); + jsonObject.put("backuptype", abp.getType().name()); + jsonObject.put("uploadstatus", uploadStatus); + jsonObject.put("compression", abp.getCompression().name()); + jsonObject.put("encryption", abp.getEncryption().name()); - // SNS Attributes for filtering messages. Cluster name and backup file type. - Map messageAttributes = - getMessageAttributes(abp.getType()); + // SNS Attributes for filtering messages. Cluster name and backup file type. + Map messageAttributes = + getMessageAttributes(abp.getType()); - this.notificationService.notify(jsonObject.toString(), messageAttributes); + this.notificationService.notify(jsonObject.toString(), messageAttributes); + } else { + logger.debug( + "BackupFileType {} is not in the list of notified component types {}", + abp.getType().name(), + StringUtils.join(notifiedBackupFileTypesSet, ", ")); + } } catch (JSONException exception) { logger.error( "JSON exception during generation of notification for upload {}. Local file {}. Ignoring to continue with rest of backup. Msg: {}", @@ -129,6 +149,27 @@ private void notify(AbstractBackupPath abp, String uploadStatus) { } } + private Set getUpdatedNotifiedBackupFileTypesSet( + String notifiedBackupFileTypes) { + if (!notifiedBackupFileTypes.equals( + this.backupRestoreConfig.getBackupNotifyComponentIncludeList())) { + this.notifiedBackupFileTypesSet.clear(); + this.notifiedBackupFileTypes = + this.backupRestoreConfig.getBackupNotifyComponentIncludeList(); + if (!StringUtils.isBlank(this.notifiedBackupFileTypes)) { + for (String s : this.notifiedBackupFileTypes.split(",")) { + try { + AbstractBackupPath.BackupFileType backupFileType = + AbstractBackupPath.BackupFileType.fromString(s.trim()); + notifiedBackupFileTypesSet.add(backupFileType); + } catch (BackupRestoreException ignored) { + } + } + } + } + return Collections.unmodifiableSet(this.notifiedBackupFileTypesSet); + } + @Override public void updateEventStart(BackupEvent event) { notify(event.getAbstractBackupPath(), STARTED); diff --git a/priam/src/test/java/com/netflix/priam/notification/TestBackupNotificationMgr.java b/priam/src/test/java/com/netflix/priam/notification/TestBackupNotificationMgr.java new file mode 100644 index 000000000..ce0ffb310 --- /dev/null +++ b/priam/src/test/java/com/netflix/priam/notification/TestBackupNotificationMgr.java @@ -0,0 +1,294 @@ +package com.netflix.priam.notification; + +import com.amazonaws.services.sns.model.MessageAttributeValue; +import com.google.inject.Guice; +import com.google.inject.Injector; +import com.google.inject.Provider; +import com.netflix.priam.backup.AbstractBackupPath; +import com.netflix.priam.backup.BRTestModule; +import com.netflix.priam.config.IBackupRestoreConfig; +import com.netflix.priam.config.IConfiguration; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.text.ParseException; +import java.util.Map; +import mockit.Capturing; +import mockit.Expectations; +import mockit.Mocked; +import mockit.Verifications; +import org.junit.Before; +import org.junit.Test; + +public class TestBackupNotificationMgr { + private Injector injector; + private BackupNotificationMgr backupNotificationMgr; + private Provider abstractBackupPathProvider; + private IConfiguration configuration; + + @Before + public void setUp() { + if (injector == null) { + injector = Guice.createInjector(new BRTestModule()); + } + + if (backupNotificationMgr == null) { + backupNotificationMgr = injector.getInstance(BackupNotificationMgr.class); + } + + if (abstractBackupPathProvider == null) { + abstractBackupPathProvider = injector.getProvider(AbstractBackupPath.class); + } + } + + @Test + public void testNotificationNonEmptyFilter( + @Mocked IBackupRestoreConfig backupRestoreConfig, + @Capturing INotificationService notificationService) + throws ParseException { + new Expectations() { + { + backupRestoreConfig.getBackupNotifyComponentIncludeList(); + result = "SNAPSHOT_VERIFIED, META_V2"; + maxTimes = 2; + } + }; + new Expectations() { + { + notificationService.notify(anyString, (Map) any); + maxTimes = 1; + } + }; + Path path = + Paths.get( + "fakeDataLocation", + "fakeKeyspace", + "fakeColumnFamily", + "fakeBackup", + "fakeData.db"); + AbstractBackupPath abstractBackupPath = abstractBackupPathProvider.get(); + abstractBackupPath.parseLocal(path.toFile(), AbstractBackupPath.BackupFileType.META_V2); + BackupEvent backupEvent = new BackupEvent(abstractBackupPath); + backupNotificationMgr.updateEventStart(backupEvent); + new Verifications() { + { + backupRestoreConfig.getBackupNotifyComponentIncludeList(); + maxTimes = 2; + } + + { + notificationService.notify(anyString, (Map) any); + maxTimes = 1; + } + }; + } + + @Test + public void testNoNotificationsNonEmptyFilter( + @Mocked IBackupRestoreConfig backupRestoreConfig, + @Capturing INotificationService notificationService) + throws ParseException { + new Expectations() { + { + backupRestoreConfig.getBackupNotifyComponentIncludeList(); + result = "META_V2"; + maxTimes = 2; + } + }; + new Expectations() { + { + notificationService.notify(anyString, (Map) any); + maxTimes = 0; + } + }; + Path path = + Paths.get( + "fakeDataLocation", + "fakeKeyspace", + "fakeColumnFamily", + "fakeBackup", + "fakeData.db"); + AbstractBackupPath abstractBackupPath = abstractBackupPathProvider.get(); + abstractBackupPath.parseLocal(path.toFile(), AbstractBackupPath.BackupFileType.SST); + BackupEvent backupEvent = new BackupEvent(abstractBackupPath); + backupNotificationMgr.updateEventStart(backupEvent); + new Verifications() { + { + backupRestoreConfig.getBackupNotifyComponentIncludeList(); + maxTimes = 2; + } + + { + notificationService.notify(anyString, (Map) any); + maxTimes = 0; + } + }; + } + + @Test + public void testNotificationsEmptyFilter( + @Mocked IBackupRestoreConfig backupRestoreConfig, + @Capturing INotificationService notificationService) + throws ParseException { + new Expectations() { + { + backupRestoreConfig.getBackupNotifyComponentIncludeList(); + result = ""; + maxTimes = 1; + } + }; + new Expectations() { + { + notificationService.notify(anyString, (Map) any); + maxTimes = 1; + } + }; + Path path = + Paths.get( + "fakeDataLocation", + "fakeKeyspace", + "fakeColumnFamily", + "fakeBackup", + "fakeData.db"); + AbstractBackupPath abstractBackupPath = abstractBackupPathProvider.get(); + abstractBackupPath.parseLocal(path.toFile(), AbstractBackupPath.BackupFileType.SST); + BackupEvent backupEvent = new BackupEvent(abstractBackupPath); + backupNotificationMgr.updateEventStart(backupEvent); + new Verifications() { + { + backupRestoreConfig.getBackupNotifyComponentIncludeList(); + maxTimes = 1; + } + + { + notificationService.notify(anyString, (Map) any); + maxTimes = 1; + } + }; + } + + @Test + public void testNotificationsInvalidFilter( + @Mocked IBackupRestoreConfig backupRestoreConfig, + @Capturing INotificationService notificationService) + throws ParseException { + new Expectations() { + { + backupRestoreConfig.getBackupNotifyComponentIncludeList(); + result = "SOME_FAKE_FILE_TYPE_1, SOME_FAKE_FILE_TYPE_2"; + maxTimes = 2; + } + }; + new Expectations() { + { + notificationService.notify(anyString, (Map) any); + maxTimes = 1; + } + }; + Path path = + Paths.get( + "fakeDataLocation", + "fakeKeyspace", + "fakeColumnFamily", + "fakeBackup", + "fakeData.db"); + AbstractBackupPath abstractBackupPath = abstractBackupPathProvider.get(); + abstractBackupPath.parseLocal(path.toFile(), AbstractBackupPath.BackupFileType.SST); + BackupEvent backupEvent = new BackupEvent(abstractBackupPath); + backupNotificationMgr.updateEventStart(backupEvent); + new Verifications() { + { + backupRestoreConfig.getBackupNotifyComponentIncludeList(); + maxTimes = 2; + } + + { + notificationService.notify(anyString, (Map) any); + maxTimes = 1; + } + }; + } + + @Test + public void testNotificationsPartiallyValidFilter( + @Mocked IBackupRestoreConfig backupRestoreConfig, + @Capturing INotificationService notificationService) + throws ParseException { + new Expectations() { + { + backupRestoreConfig.getBackupNotifyComponentIncludeList(); + result = "SOME_FAKE_FILE_TYPE_1, SOME_FAKE_FILE_TYPE_2, META_V2"; + maxTimes = 2; + } + }; + new Expectations() { + { + notificationService.notify(anyString, (Map) any); + maxTimes = 1; + } + }; + Path path = + Paths.get( + "fakeDataLocation", + "fakeKeyspace", + "fakeColumnFamily", + "fakeBackup", + "fakeData.db"); + AbstractBackupPath abstractBackupPath = abstractBackupPathProvider.get(); + abstractBackupPath.parseLocal(path.toFile(), AbstractBackupPath.BackupFileType.META_V2); + BackupEvent backupEvent = new BackupEvent(abstractBackupPath); + backupNotificationMgr.updateEventStart(backupEvent); + new Verifications() { + { + backupRestoreConfig.getBackupNotifyComponentIncludeList(); + maxTimes = 2; + } + + { + notificationService.notify(anyString, (Map) any); + maxTimes = 1; + } + }; + } + + @Test + public void testNoNotificationsPartiallyValidFilter( + @Mocked IBackupRestoreConfig backupRestoreConfig, + @Capturing INotificationService notificationService) + throws ParseException { + new Expectations() { + { + backupRestoreConfig.getBackupNotifyComponentIncludeList(); + result = "SOME_FAKE_FILE_TYPE_1, SOME_FAKE_FILE_TYPE_2, SST"; + maxTimes = 2; + } + }; + new Expectations() { + { + notificationService.notify(anyString, (Map) any); + maxTimes = 0; + } + }; + Path path = + Paths.get( + "fakeDataLocation", + "fakeKeyspace", + "fakeColumnFamily", + "fakeBackup", + "fakeData.db"); + AbstractBackupPath abstractBackupPath = abstractBackupPathProvider.get(); + abstractBackupPath.parseLocal(path.toFile(), AbstractBackupPath.BackupFileType.META_V2); + BackupEvent backupEvent = new BackupEvent(abstractBackupPath); + backupNotificationMgr.updateEventStart(backupEvent); + new Verifications() { + { + backupRestoreConfig.getBackupNotifyComponentIncludeList(); + maxTimes = 2; + } + + { + notificationService.notify(anyString, (Map) any); + maxTimes = 0; + } + }; + } +} From f73470cf7cea8ad191ae60bc29244e64b7dc5371 Mon Sep 17 00:00:00 2001 From: Sreeram Chakrovorthy Date: Thu, 20 Feb 2020 11:37:58 -0800 Subject: [PATCH 098/228] Fixing the issue with bootstrapping logic for updating the BackUpNotification component set. --- .../priam/notification/BackupNotificationMgr.java | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/priam/src/main/java/com/netflix/priam/notification/BackupNotificationMgr.java b/priam/src/main/java/com/netflix/priam/notification/BackupNotificationMgr.java index 5d61389ba..9ab37aca5 100644 --- a/priam/src/main/java/com/netflix/priam/notification/BackupNotificationMgr.java +++ b/priam/src/main/java/com/netflix/priam/notification/BackupNotificationMgr.java @@ -61,8 +61,7 @@ public BackupNotificationMgr( this.instanceInfo = instanceInfo; this.instanceIdentity = instanceIdentity; this.notifiedBackupFileTypesSet = new HashSet<>(); - this.notifiedBackupFileTypes = - this.backupRestoreConfig.getBackupNotifyComponentIncludeList(); + this.notifiedBackupFileTypes = ""; } public void notify(BackupVerificationResult backupVerificationResult) { @@ -132,7 +131,6 @@ private void notify(AbstractBackupPath abp, String uploadStatus) { // SNS Attributes for filtering messages. Cluster name and backup file type. Map messageAttributes = getMessageAttributes(abp.getType()); - this.notificationService.notify(jsonObject.toString(), messageAttributes); } else { logger.debug( @@ -151,8 +149,12 @@ private void notify(AbstractBackupPath abp, String uploadStatus) { private Set getUpdatedNotifiedBackupFileTypesSet( String notifiedBackupFileTypes) { - if (!notifiedBackupFileTypes.equals( - this.backupRestoreConfig.getBackupNotifyComponentIncludeList())) { + String propertyValue = this.backupRestoreConfig.getBackupNotifyComponentIncludeList(); + if (!notifiedBackupFileTypes.equals(propertyValue)) { + logger.info( + String.format( + "Notified BackupFileTypes changed from %s to %s", + this.notifiedBackupFileTypes, propertyValue)); this.notifiedBackupFileTypesSet.clear(); this.notifiedBackupFileTypes = this.backupRestoreConfig.getBackupNotifyComponentIncludeList(); From 17303276a2dabf117cf70439217eb47d0723eb63 Mon Sep 17 00:00:00 2001 From: Sreeram Chakrovorthy Date: Fri, 21 Feb 2020 11:19:10 -0800 Subject: [PATCH 099/228] Updating ChangeLog and the config name to "priam.backupNotifyComponentIncludeList" --- CHANGELOG.md | 12 ++++++++++++ .../netflix/priam/config/BackupRestoreConfig.java | 2 +- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index df6373cc2..fc259cbbb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,16 @@ # Changelog + +# Changelog +## 2020/02/21 3.11.57 +(#844, #839) Implementation of a filter for Backup Notification. The filter can be controlled using the configuration "priam.backupNotifyComponentIncludeList" + +## 2019/10/24 3.11.56 +(#836) Move flush and compactions to Service Layer. This allows us to "hot" reload the jobs when configurations change. +(#836) Send SNAPSHOT_VERIFIED message when a snapshot is verified and ready to be consumed by downward dependencies. + +## 2019/08/23 3.11.55 +(#832) Travis build fails for oraclejdk8. Migration to openjdk8 + ## 2019/10/16 3.11.54 (#834) Removing functionality of creating incremental manifest file in backup V1 as it is not used. (#834) Bug fix: When meta file do not exist for TTL in backup v2 we should not be throwing NPE. diff --git a/priam/src/main/java/com/netflix/priam/config/BackupRestoreConfig.java b/priam/src/main/java/com/netflix/priam/config/BackupRestoreConfig.java index df19e2114..0f1642a18 100644 --- a/priam/src/main/java/com/netflix/priam/config/BackupRestoreConfig.java +++ b/priam/src/main/java/com/netflix/priam/config/BackupRestoreConfig.java @@ -59,6 +59,6 @@ public String getBackupVerificationCronExpression() { @Override public String getBackupNotifyComponentIncludeList() { - return config.get("priam.backup.notify.component.include", StringUtils.EMPTY); + return config.get("priam.backupNotifyComponentIncludeList", StringUtils.EMPTY); } } From 4fcf2bdf94f44e03e0b2eeb0e75a98b93337fe22 Mon Sep 17 00:00:00 2001 From: Matt Lehman Date: Tue, 14 Apr 2020 09:13:12 -0700 Subject: [PATCH 100/228] Add hook in StandardTuner to allow for subclasses to add custom Cassandra parameters --- .../java/com/netflix/priam/tuner/StandardTuner.java | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/priam/src/main/java/com/netflix/priam/tuner/StandardTuner.java b/priam/src/main/java/com/netflix/priam/tuner/StandardTuner.java index 8c27f48ef..8a1f599f1 100644 --- a/priam/src/main/java/com/netflix/priam/tuner/StandardTuner.java +++ b/priam/src/main/java/com/netflix/priam/tuner/StandardTuner.java @@ -136,8 +136,12 @@ public void writeAllProperties(String yamlLocation, String hostname, String seed // force to 1 until vnodes are properly supported map.put("num_tokens", 1); + // Additional C* Yaml properties, which can be set via Priam.extra.params addExtraCassParams(map); + // Custom specific C* yaml properties which might not be available in Apache C* OSS + addCustomCassParams(map); + // remove troublesome properties map.remove("flush_largest_memtables_at"); map.remove("reduce_cache_capacity_to"); @@ -148,6 +152,15 @@ public void writeAllProperties(String yamlLocation, String hostname, String seed configureCommitLogBackups(); } + /** + * This method can be overwritten in child classes for any additional tunings to C* Yaml. + * Default implementation is left empty intentionally for child classes to override. This is + * useful when custom YAML properties are supported in deployed C*. + * + * @param map + */ + protected void addCustomCassParams(Map map) {} + /** * Overridable by derived classes to inject a wrapper snitch. * From 62374827d19009ceaccfc86ddf0dfbc124c10001 Mon Sep 17 00:00:00 2001 From: Sreeram Chakrovorthy Date: Tue, 31 Mar 2020 22:09:07 -0700 Subject: [PATCH 101/228] First commit of the changes necessary to support verifying all backups in the specified date range. --- .../priam/backup/BackupVerification.java | 48 +++++++++++ .../backupv2/BackupVerificationTask.java | 23 ++++-- .../priam/backup/TestBackupVerification.java | 80 +++++++++++++++++++ 3 files changed, 143 insertions(+), 8 deletions(-) diff --git a/priam/src/main/java/com/netflix/priam/backup/BackupVerification.java b/priam/src/main/java/com/netflix/priam/backup/BackupVerification.java index 877c6e363..04ff6cb1e 100644 --- a/priam/src/main/java/com/netflix/priam/backup/BackupVerification.java +++ b/priam/src/main/java/com/netflix/priam/backup/BackupVerification.java @@ -108,6 +108,54 @@ public Optional verifyBackup( return Optional.empty(); } + public List verifyAllBackups( + BackupVersion backupVersion, boolean force, DateRange dateRange) + throws UnsupportedTypeException, IllegalArgumentException { + IMetaProxy metaProxy = getMetaProxy(backupVersion); + if (metaProxy == null) { + throw new UnsupportedTypeException( + "BackupVersion type: " + backupVersion + " is not supported"); + } + + if (dateRange == null) { + throw new IllegalArgumentException("dateRange provided is null"); + } + + List result = new ArrayList<>(); + + List metadata = + backupStatusMgr.getLatestBackupMetadata(backupVersion, dateRange); + if (metadata == null || metadata.isEmpty()) return result; + for (BackupMetadata backupMetadata : metadata) { + if (backupMetadata.getLastValidated() != null && !force) { + // Backup is already validated. Nothing to do. + BackupVerificationResult backupVerificationResult = new BackupVerificationResult(); + backupVerificationResult.valid = true; + backupVerificationResult.manifestAvailable = true; + backupVerificationResult.snapshotInstant = backupMetadata.getStart().toInstant(); + Path snapshotLocation = Paths.get(backupMetadata.getSnapshotLocation()); + backupVerificationResult.remotePath = + snapshotLocation.subpath(1, snapshotLocation.getNameCount()).toString(); + result.add(backupVerificationResult); + } + else { + BackupVerificationResult backupVerificationResult = + verifyBackup(metaProxy, backupMetadata); + if (logger.isDebugEnabled()) + logger.debug( + "BackupVerification: metadata: {}, result: {}", + backupMetadata, + backupVerificationResult); + if (backupVerificationResult.valid) { + backupMetadata.setLastValidated(new Date(DateUtil.getInstant().toEpochMilli())); + backupStatusMgr.update(backupMetadata); + result.add(backupVerificationResult); + } + } + } + return result; + } + private BackupVerificationResult verifyBackup( IMetaProxy metaProxy, BackupMetadata latestBackupMetaData) { Path metadataLocation = Paths.get(latestBackupMetaData.getSnapshotLocation()); diff --git a/priam/src/main/java/com/netflix/priam/backupv2/BackupVerificationTask.java b/priam/src/main/java/com/netflix/priam/backupv2/BackupVerificationTask.java index e0b365962..7b44e679c 100644 --- a/priam/src/main/java/com/netflix/priam/backupv2/BackupVerificationTask.java +++ b/priam/src/main/java/com/netflix/priam/backupv2/BackupVerificationTask.java @@ -31,7 +31,10 @@ import com.netflix.priam.utils.DateUtil; import com.netflix.priam.utils.DateUtil.DateRange; import java.time.temporal.ChronoUnit; +import java.util.List; import java.util.Optional; +import java.util.stream.Collectors; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -88,10 +91,11 @@ public void execute() throws Exception { backupRestoreConfig.getBackupVerificationSLOInHours(), ChronoUnit.HOURS), DateUtil.getInstant()); - Optional verificationResult = - backupVerification.verifyBackup( + List verificationResults = + backupVerification.verifyAllBackups( BackupVersion.SNAPSHOT_META_SERVICE, false, dateRange); - if (!verificationResult.isPresent() || !verificationResult.get().valid) { + if (verificationResults.isEmpty() + || verificationResults.stream().filter(r -> r.valid).collect(Collectors.toList()).isEmpty()) { logger.error( "Not able to find any snapshot which is valid in our SLO window: {} hours", backupRestoreConfig.getBackupVerificationSLOInHours()); @@ -99,11 +103,14 @@ public void execute() throws Exception { } else { // verification result is available and is valid. // send notification that backup is uploaded. - logger.info( - "Sending {} message for backup: {}", - AbstractBackupPath.BackupFileType.SNAPSHOT_VERIFIED, - verificationResult.get().snapshotInstant); - backupNotificationMgr.notify(verificationResult.get()); + verificationResults.stream().forEach(r -> { + logger.info( + "Sending {} message for backup: {}", + AbstractBackupPath.BackupFileType.SNAPSHOT_VERIFIED, + r.snapshotInstant); + backupNotificationMgr.notify(r); + }); + } } diff --git a/priam/src/test/java/com/netflix/priam/backup/TestBackupVerification.java b/priam/src/test/java/com/netflix/priam/backup/TestBackupVerification.java index cef662187..abbed6380 100644 --- a/priam/src/test/java/com/netflix/priam/backup/TestBackupVerification.java +++ b/priam/src/test/java/com/netflix/priam/backup/TestBackupVerification.java @@ -31,7 +31,9 @@ import java.nio.file.Paths; import java.time.Instant; import java.time.temporal.ChronoUnit; +import java.util.ArrayList; import java.util.Date; +import java.util.List; import java.util.Optional; import mockit.Mock; import mockit.MockUp; @@ -48,6 +50,7 @@ public class TestBackupVerification { private final IConfiguration configuration; private final IBackupStatusMgr backupStatusMgr; private final String backupDate = "201812011000"; + private final String backupDateEnd = "201812021000"; private final Path location = Paths.get( "some_bucket/casstestbackup/1049_fake-app/1808575600", @@ -56,6 +59,7 @@ public class TestBackupVerification { "SNAPPY", "PLAINTEXT", "meta_v2_201812011000.json"); + private final int numFakeBackups = 10; public TestBackupVerification() { Injector injector = Guice.createInjector(new BRTestModule()); @@ -116,6 +120,12 @@ public void noBackup() throws Exception { private void setUp() throws Exception { Instant start = DateUtil.parseInstant(backupDate); + for(int i=0; i backupVerificationResults = + backupVerification.verifyAllBackups( + BackupVersion.SNAPSHOT_BACKUP, + false, + new DateRange(backupDate + "," + backupDateEnd)); + Assert.assertTrue(!backupVerificationResults.isEmpty()); + Assert.assertTrue(backupVerificationResults.size() == numFakeBackups); + backupVerificationResults.stream() + .forEach(b -> Assert.assertEquals(Instant.EPOCH, b.snapshotInstant)); + List backupMetadata = + backupStatusMgr + .getLatestBackupMetadata( + BackupVersion.SNAPSHOT_BACKUP, + new DateRange(backupDate + "," + backupDateEnd)); + Assert.assertTrue(!backupMetadata.isEmpty()); + Assert.assertTrue(backupMetadata.size() == numFakeBackups); + backupMetadata.stream().forEach(b -> Assert.assertNotNull(b.getLastValidated())); + + backupMetadata = + backupStatusMgr + .getLatestBackupMetadata( + BackupVersion.SNAPSHOT_META_SERVICE, + new DateRange(backupDate + "," + backupDateEnd)); + Assert.assertTrue(!backupMetadata.isEmpty()); + Assert.assertTrue(backupMetadata.size() == numFakeBackups); + backupMetadata.stream().forEach(b -> Assert.assertNull(b.getLastValidated())); + } + @Test public void verifyBackupVersion2() throws Exception { setUp(); @@ -208,6 +256,38 @@ public void verifyBackupVersion2() throws Exception { Assert.assertNull(backupMetadata.get().getLastValidated()); } + @Test + public void verifyBackupVersion2List() throws Exception { + setUp(); + // Verify for backup version 2.0 + List backupVerificationResults = + backupVerification.verifyAllBackups( + BackupVersion.SNAPSHOT_META_SERVICE, + false, + new DateRange(backupDate + "," + backupDateEnd)); + Assert.assertTrue(!backupVerificationResults.isEmpty()); + Assert.assertTrue(backupVerificationResults.size() == numFakeBackups); + backupVerificationResults.stream() + .forEach(b -> Assert.assertEquals(Instant.EPOCH, b.snapshotInstant)); + List backupMetadata = + backupStatusMgr + .getLatestBackupMetadata( + BackupVersion.SNAPSHOT_META_SERVICE, + new DateRange(backupDate + "," + backupDateEnd)); + Assert.assertTrue(!backupMetadata.isEmpty()); + Assert.assertTrue(backupMetadata.size() == numFakeBackups); + backupMetadata.stream().forEach(b -> Assert.assertNotNull(b.getLastValidated())); + + backupMetadata = + backupStatusMgr + .getLatestBackupMetadata( + BackupVersion.SNAPSHOT_BACKUP, + new DateRange(backupDate + "," + backupDateEnd)); + Assert.assertTrue(!backupMetadata.isEmpty()); + Assert.assertTrue(backupMetadata.size() == numFakeBackups); + backupMetadata.stream().forEach(b -> Assert.assertNull(b.getLastValidated())); + } + private BackupMetadata getBackupMetaData( BackupVersion backupVersion, Instant startTime, Status status) throws Exception { BackupMetadata backupMetadata = From 24d1b90c4c41e0e262506c2ace95585a21d19927 Mon Sep 17 00:00:00 2001 From: Sreeram Chakrovorthy Date: Tue, 31 Mar 2020 22:15:50 -0700 Subject: [PATCH 102/228] First pass of the changes for verifying all backups in the specified date range. --- .../backupv2/BackupVerificationTask.java | 26 +++++----- .../priam/backup/TestBackupVerification.java | 49 +++++++++---------- 2 files changed, 39 insertions(+), 36 deletions(-) diff --git a/priam/src/main/java/com/netflix/priam/backupv2/BackupVerificationTask.java b/priam/src/main/java/com/netflix/priam/backupv2/BackupVerificationTask.java index 7b44e679c..ee97aa288 100644 --- a/priam/src/main/java/com/netflix/priam/backupv2/BackupVerificationTask.java +++ b/priam/src/main/java/com/netflix/priam/backupv2/BackupVerificationTask.java @@ -32,9 +32,7 @@ import com.netflix.priam.utils.DateUtil.DateRange; import java.time.temporal.ChronoUnit; import java.util.List; -import java.util.Optional; import java.util.stream.Collectors; - import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -95,7 +93,11 @@ public void execute() throws Exception { backupVerification.verifyAllBackups( BackupVersion.SNAPSHOT_META_SERVICE, false, dateRange); if (verificationResults.isEmpty() - || verificationResults.stream().filter(r -> r.valid).collect(Collectors.toList()).isEmpty()) { + || verificationResults + .stream() + .filter(r -> r.valid) + .collect(Collectors.toList()) + .isEmpty()) { logger.error( "Not able to find any snapshot which is valid in our SLO window: {} hours", backupRestoreConfig.getBackupVerificationSLOInHours()); @@ -103,14 +105,16 @@ public void execute() throws Exception { } else { // verification result is available and is valid. // send notification that backup is uploaded. - verificationResults.stream().forEach(r -> { - logger.info( - "Sending {} message for backup: {}", - AbstractBackupPath.BackupFileType.SNAPSHOT_VERIFIED, - r.snapshotInstant); - backupNotificationMgr.notify(r); - }); - + verificationResults + .stream() + .forEach( + r -> { + logger.info( + "Sending {} message for backup: {}", + AbstractBackupPath.BackupFileType.SNAPSHOT_VERIFIED, + r.snapshotInstant); + backupNotificationMgr.notify(r); + }); } } diff --git a/priam/src/test/java/com/netflix/priam/backup/TestBackupVerification.java b/priam/src/test/java/com/netflix/priam/backup/TestBackupVerification.java index abbed6380..4aa8e8550 100644 --- a/priam/src/test/java/com/netflix/priam/backup/TestBackupVerification.java +++ b/priam/src/test/java/com/netflix/priam/backup/TestBackupVerification.java @@ -31,7 +31,6 @@ import java.nio.file.Paths; import java.time.Instant; import java.time.temporal.ChronoUnit; -import java.util.ArrayList; import java.util.Date; import java.util.List; import java.util.Optional; @@ -120,10 +119,11 @@ public void noBackup() throws Exception { private void setUp() throws Exception { Instant start = DateUtil.parseInstant(backupDate); - for(int i=0; i Assert.assertEquals(Instant.EPOCH, b.snapshotInstant)); List backupMetadata = - backupStatusMgr - .getLatestBackupMetadata( - BackupVersion.SNAPSHOT_BACKUP, - new DateRange(backupDate + "," + backupDateEnd)); + backupStatusMgr.getLatestBackupMetadata( + BackupVersion.SNAPSHOT_BACKUP, + new DateRange(backupDate + "," + backupDateEnd)); Assert.assertTrue(!backupMetadata.isEmpty()); Assert.assertTrue(backupMetadata.size() == numFakeBackups); backupMetadata.stream().forEach(b -> Assert.assertNotNull(b.getLastValidated())); backupMetadata = - backupStatusMgr - .getLatestBackupMetadata( - BackupVersion.SNAPSHOT_META_SERVICE, - new DateRange(backupDate + "," + backupDateEnd)); + backupStatusMgr.getLatestBackupMetadata( + BackupVersion.SNAPSHOT_META_SERVICE, + new DateRange(backupDate + "," + backupDateEnd)); Assert.assertTrue(!backupMetadata.isEmpty()); Assert.assertTrue(backupMetadata.size() == numFakeBackups); backupMetadata.stream().forEach(b -> Assert.assertNull(b.getLastValidated())); @@ -267,22 +267,21 @@ public void verifyBackupVersion2List() throws Exception { new DateRange(backupDate + "," + backupDateEnd)); Assert.assertTrue(!backupVerificationResults.isEmpty()); Assert.assertTrue(backupVerificationResults.size() == numFakeBackups); - backupVerificationResults.stream() + backupVerificationResults + .stream() .forEach(b -> Assert.assertEquals(Instant.EPOCH, b.snapshotInstant)); List backupMetadata = - backupStatusMgr - .getLatestBackupMetadata( - BackupVersion.SNAPSHOT_META_SERVICE, - new DateRange(backupDate + "," + backupDateEnd)); + backupStatusMgr.getLatestBackupMetadata( + BackupVersion.SNAPSHOT_META_SERVICE, + new DateRange(backupDate + "," + backupDateEnd)); Assert.assertTrue(!backupMetadata.isEmpty()); Assert.assertTrue(backupMetadata.size() == numFakeBackups); backupMetadata.stream().forEach(b -> Assert.assertNotNull(b.getLastValidated())); backupMetadata = - backupStatusMgr - .getLatestBackupMetadata( - BackupVersion.SNAPSHOT_BACKUP, - new DateRange(backupDate + "," + backupDateEnd)); + backupStatusMgr.getLatestBackupMetadata( + BackupVersion.SNAPSHOT_BACKUP, + new DateRange(backupDate + "," + backupDateEnd)); Assert.assertTrue(!backupMetadata.isEmpty()); Assert.assertTrue(backupMetadata.size() == numFakeBackups); backupMetadata.stream().forEach(b -> Assert.assertNull(b.getLastValidated())); From 5a7e01568f19e8fccbac2dda98c1dcf6c87efbf8 Mon Sep 17 00:00:00 2001 From: Sreeram Chakrovorthy Date: Tue, 31 Mar 2020 22:37:15 -0700 Subject: [PATCH 103/228] Fixing some broken tests. --- .../netflix/priam/backup/BackupVerification.java | 3 +-- .../priam/backupv2/TestBackupVerificationTask.java | 14 ++++++++------ 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/priam/src/main/java/com/netflix/priam/backup/BackupVerification.java b/priam/src/main/java/com/netflix/priam/backup/BackupVerification.java index 04ff6cb1e..d208044bf 100644 --- a/priam/src/main/java/com/netflix/priam/backup/BackupVerification.java +++ b/priam/src/main/java/com/netflix/priam/backup/BackupVerification.java @@ -137,8 +137,7 @@ public List verifyAllBackups( backupVerificationResult.remotePath = snapshotLocation.subpath(1, snapshotLocation.getNameCount()).toString(); result.add(backupVerificationResult); - } - else { + } else { BackupVerificationResult backupVerificationResult = verifyBackup(metaProxy, backupMetadata); if (logger.isDebugEnabled()) diff --git a/priam/src/test/java/com/netflix/priam/backupv2/TestBackupVerificationTask.java b/priam/src/test/java/com/netflix/priam/backupv2/TestBackupVerificationTask.java index fc13079d1..a34528ee4 100644 --- a/priam/src/test/java/com/netflix/priam/backupv2/TestBackupVerificationTask.java +++ b/priam/src/test/java/com/netflix/priam/backupv2/TestBackupVerificationTask.java @@ -28,7 +28,8 @@ import com.netflix.priam.health.InstanceState; import com.netflix.priam.scheduler.UnsupportedTypeException; import com.netflix.priam.utils.DateUtil.DateRange; -import java.util.Optional; +import java.util.ArrayList; +import java.util.List; import mockit.Expectations; import mockit.Mock; import mockit.MockUp; @@ -57,16 +58,17 @@ static class MockBackupVerification extends MockUp { public static boolean throwError = false; @Mock - public Optional verifyBackup( + public List verifyAllBackups( BackupVersion backupVersion, boolean force, DateRange dateRange) throws UnsupportedTypeException, IllegalArgumentException { if (throwError) throw new IllegalArgumentException("DummyError"); - if (failCall) return Optional.of(new BackupVerificationResult()); + if (failCall) return new ArrayList<>(); - BackupVerificationResult result = new BackupVerificationResult(); - result.valid = true; - return Optional.of(result); + BackupVerificationResult backupVerificationResult = new BackupVerificationResult(); + backupVerificationResult.valid = true; + List result = new ArrayList<>(); + return result; } } From ef1113de578b4775ffb0a57c99fc2a1fd08823d6 Mon Sep 17 00:00:00 2001 From: Sreeram Chakrovorthy Date: Tue, 21 Apr 2020 12:33:22 -0700 Subject: [PATCH 104/228] Addressing review comments that we should only verify any unverified backups in the given date range. --- .../com/netflix/priam/backup/BackupVerification.java | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/priam/src/main/java/com/netflix/priam/backup/BackupVerification.java b/priam/src/main/java/com/netflix/priam/backup/BackupVerification.java index d208044bf..6e0d4a393 100644 --- a/priam/src/main/java/com/netflix/priam/backup/BackupVerification.java +++ b/priam/src/main/java/com/netflix/priam/backup/BackupVerification.java @@ -127,17 +127,7 @@ public List verifyAllBackups( backupStatusMgr.getLatestBackupMetadata(backupVersion, dateRange); if (metadata == null || metadata.isEmpty()) return result; for (BackupMetadata backupMetadata : metadata) { - if (backupMetadata.getLastValidated() != null && !force) { - // Backup is already validated. Nothing to do. - BackupVerificationResult backupVerificationResult = new BackupVerificationResult(); - backupVerificationResult.valid = true; - backupVerificationResult.manifestAvailable = true; - backupVerificationResult.snapshotInstant = backupMetadata.getStart().toInstant(); - Path snapshotLocation = Paths.get(backupMetadata.getSnapshotLocation()); - backupVerificationResult.remotePath = - snapshotLocation.subpath(1, snapshotLocation.getNameCount()).toString(); - result.add(backupVerificationResult); - } else { + if (backupMetadata.getLastValidated() == null) { BackupVerificationResult backupVerificationResult = verifyBackup(metaProxy, backupMetadata); if (logger.isDebugEnabled()) From 767fb17d232ed164fa2efa6250d303905838518a Mon Sep 17 00:00:00 2001 From: Sreeram Chakrovorthy Date: Wed, 22 Apr 2020 14:44:40 -0700 Subject: [PATCH 105/228] Addressing some of the review comments. --- .../java/com/netflix/priam/backup/BackupVerification.java | 2 +- .../com/netflix/priam/backupv2/BackupVerificationTask.java | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/priam/src/main/java/com/netflix/priam/backup/BackupVerification.java b/priam/src/main/java/com/netflix/priam/backup/BackupVerification.java index 6e0d4a393..47e5337d7 100644 --- a/priam/src/main/java/com/netflix/priam/backup/BackupVerification.java +++ b/priam/src/main/java/com/netflix/priam/backup/BackupVerification.java @@ -109,7 +109,7 @@ public Optional verifyBackup( } public List verifyAllBackups( - BackupVersion backupVersion, boolean force, DateRange dateRange) + BackupVersion backupVersion, DateRange dateRange) throws UnsupportedTypeException, IllegalArgumentException { IMetaProxy metaProxy = getMetaProxy(backupVersion); if (metaProxy == null) { diff --git a/priam/src/main/java/com/netflix/priam/backupv2/BackupVerificationTask.java b/priam/src/main/java/com/netflix/priam/backupv2/BackupVerificationTask.java index ee97aa288..bf02a8040 100644 --- a/priam/src/main/java/com/netflix/priam/backupv2/BackupVerificationTask.java +++ b/priam/src/main/java/com/netflix/priam/backupv2/BackupVerificationTask.java @@ -108,12 +108,12 @@ public void execute() throws Exception { verificationResults .stream() .forEach( - r -> { + backupVerificationResult -> { logger.info( "Sending {} message for backup: {}", AbstractBackupPath.BackupFileType.SNAPSHOT_VERIFIED, - r.snapshotInstant); - backupNotificationMgr.notify(r); + backupVerificationResult.snapshotInstant); + backupNotificationMgr.notify(backupVerificationResult); }); } } From b244d97f1f4d39374278cca825ea3525ee881c40 Mon Sep 17 00:00:00 2001 From: Sreeram Chakrovorthy Date: Wed, 22 Apr 2020 17:45:06 -0700 Subject: [PATCH 106/228] Adding some tests to address code coverage. --- .../backupv2/BackupVerificationTask.java | 3 +- .../priam/backup/TestBackupVerification.java | 30 ++++++++++++++++--- .../backupv2/TestBackupVerificationTask.java | 2 +- 3 files changed, 28 insertions(+), 7 deletions(-) diff --git a/priam/src/main/java/com/netflix/priam/backupv2/BackupVerificationTask.java b/priam/src/main/java/com/netflix/priam/backupv2/BackupVerificationTask.java index bf02a8040..623eaeeb4 100644 --- a/priam/src/main/java/com/netflix/priam/backupv2/BackupVerificationTask.java +++ b/priam/src/main/java/com/netflix/priam/backupv2/BackupVerificationTask.java @@ -90,8 +90,7 @@ public void execute() throws Exception { ChronoUnit.HOURS), DateUtil.getInstant()); List verificationResults = - backupVerification.verifyAllBackups( - BackupVersion.SNAPSHOT_META_SERVICE, false, dateRange); + backupVerification.verifyAllBackups(BackupVersion.SNAPSHOT_META_SERVICE, dateRange); if (verificationResults.isEmpty() || verificationResults .stream() diff --git a/priam/src/test/java/com/netflix/priam/backup/TestBackupVerification.java b/priam/src/test/java/com/netflix/priam/backup/TestBackupVerification.java index 4aa8e8550..daa182dcd 100644 --- a/priam/src/test/java/com/netflix/priam/backup/TestBackupVerification.java +++ b/priam/src/test/java/com/netflix/priam/backup/TestBackupVerification.java @@ -100,6 +100,16 @@ public void illegalDateRange() throws UnsupportedTypeException { } } + @Test + public void illegalDateRangeBackupDateRange() throws UnsupportedTypeException { + try { + backupVerification.verifyAllBackups(BackupVersion.SNAPSHOT_BACKUP, null); + Assert.assertTrue(false); + } catch (IllegalArgumentException e) { + Assert.assertTrue(true); + } + } + @Test public void noBackup() throws Exception { Optional backupVerificationResultOptinal = @@ -117,6 +127,20 @@ public void noBackup() throws Exception { Assert.assertFalse(backupVerificationResultOptinal.isPresent()); } + @Test + public void noBackupDateRange() throws Exception { + List backupVerificationResults = + backupVerification.verifyAllBackups( + BackupVersion.SNAPSHOT_BACKUP, new DateRange(Instant.now(), Instant.now())); + Assert.assertFalse(backupVerificationResults.size() > 0); + + backupVerificationResults = + backupVerification.verifyAllBackups( + BackupVersion.SNAPSHOT_META_SERVICE, + new DateRange(Instant.now(), Instant.now())); + Assert.assertFalse(backupVerificationResults.size() > 0); + } + private void setUp() throws Exception { Instant start = DateUtil.parseInstant(backupDate); for (int i = 0; i < numFakeBackups - 1; i++) { @@ -177,13 +201,12 @@ public void verifyBackupVersion1() throws Exception { } @Test - public void verifyBackupVersion1List() throws Exception { + public void verifyBackupVersion1DateRange() throws Exception { setUp(); // Verify for backup version 1.0 List backupVerificationResults = backupVerification.verifyAllBackups( BackupVersion.SNAPSHOT_BACKUP, - false, new DateRange(backupDate + "," + backupDateEnd)); Assert.assertTrue(!backupVerificationResults.isEmpty()); Assert.assertTrue(backupVerificationResults.size() == numFakeBackups); @@ -257,13 +280,12 @@ public void verifyBackupVersion2() throws Exception { } @Test - public void verifyBackupVersion2List() throws Exception { + public void verifyBackupVersion2DateRange() throws Exception { setUp(); // Verify for backup version 2.0 List backupVerificationResults = backupVerification.verifyAllBackups( BackupVersion.SNAPSHOT_META_SERVICE, - false, new DateRange(backupDate + "," + backupDateEnd)); Assert.assertTrue(!backupVerificationResults.isEmpty()); Assert.assertTrue(backupVerificationResults.size() == numFakeBackups); diff --git a/priam/src/test/java/com/netflix/priam/backupv2/TestBackupVerificationTask.java b/priam/src/test/java/com/netflix/priam/backupv2/TestBackupVerificationTask.java index a34528ee4..dec1610e1 100644 --- a/priam/src/test/java/com/netflix/priam/backupv2/TestBackupVerificationTask.java +++ b/priam/src/test/java/com/netflix/priam/backupv2/TestBackupVerificationTask.java @@ -59,7 +59,7 @@ static class MockBackupVerification extends MockUp { @Mock public List verifyAllBackups( - BackupVersion backupVersion, boolean force, DateRange dateRange) + BackupVersion backupVersion, DateRange dateRange) throws UnsupportedTypeException, IllegalArgumentException { if (throwError) throw new IllegalArgumentException("DummyError"); From e1905332dc140f5e83a2634f408751ab083b3e26 Mon Sep 17 00:00:00 2001 From: Sreeram Chakrovorthy Date: Wed, 22 Apr 2020 18:33:21 -0700 Subject: [PATCH 107/228] Adding more tests to improve coverage and some code clean up in TestBackupVerificationTask. --- .../backupv2/BackupVerificationTask.java | 6 +-- .../backupv2/TestBackupVerificationTask.java | 52 +++++++++++++++++-- 2 files changed, 50 insertions(+), 8 deletions(-) diff --git a/priam/src/main/java/com/netflix/priam/backupv2/BackupVerificationTask.java b/priam/src/main/java/com/netflix/priam/backupv2/BackupVerificationTask.java index 623eaeeb4..b91f383f5 100644 --- a/priam/src/main/java/com/netflix/priam/backupv2/BackupVerificationTask.java +++ b/priam/src/main/java/com/netflix/priam/backupv2/BackupVerificationTask.java @@ -93,10 +93,8 @@ public void execute() throws Exception { backupVerification.verifyAllBackups(BackupVersion.SNAPSHOT_META_SERVICE, dateRange); if (verificationResults.isEmpty() || verificationResults - .stream() - .filter(r -> r.valid) - .collect(Collectors.toList()) - .isEmpty()) { + .stream() + .filter(backupVerificationResult -> backupVerificationResult.valid).count() == 0) { logger.error( "Not able to find any snapshot which is valid in our SLO window: {} hours", backupRestoreConfig.getBackupVerificationSLOInHours()); diff --git a/priam/src/test/java/com/netflix/priam/backupv2/TestBackupVerificationTask.java b/priam/src/test/java/com/netflix/priam/backupv2/TestBackupVerificationTask.java index dec1610e1..bc0e5db54 100644 --- a/priam/src/test/java/com/netflix/priam/backupv2/TestBackupVerificationTask.java +++ b/priam/src/test/java/com/netflix/priam/backupv2/TestBackupVerificationTask.java @@ -26,8 +26,11 @@ import com.netflix.priam.backup.Status; import com.netflix.priam.config.IConfiguration; import com.netflix.priam.health.InstanceState; +import com.netflix.priam.notification.BackupNotificationMgr; import com.netflix.priam.scheduler.UnsupportedTypeException; import com.netflix.priam.utils.DateUtil.DateRange; + +import java.time.Instant; import java.util.ArrayList; import java.util.List; import mockit.Expectations; @@ -42,20 +45,21 @@ public class TestBackupVerificationTask { private static BackupVerificationTask backupVerificationService; private static IConfiguration configuration; private static BackupVerification backupVerification; + private static BackupNotificationMgr backupNotificationMgr; public TestBackupVerificationTask() { new MockBackupVerification(); + new MockBackupNotificationMgr(); Injector injector = Guice.createInjector(new BRTestModule()); if (configuration == null) configuration = injector.getInstance(IConfiguration.class); if (backupVerificationService == null) backupVerificationService = injector.getInstance(BackupVerificationTask.class); - if (backupVerification == null) - backupVerification = injector.getInstance(BackupVerification.class); } static class MockBackupVerification extends MockUp { public static boolean failCall = false; public static boolean throwError = false; + public static boolean validBackupVerificationResult = true; @Mock public List verifyAllBackups( @@ -65,13 +69,25 @@ public List verifyAllBackups( if (failCall) return new ArrayList<>(); - BackupVerificationResult backupVerificationResult = new BackupVerificationResult(); - backupVerificationResult.valid = true; List result = new ArrayList<>(); + if(validBackupVerificationResult){ + result.add(getValidBackupVerificationResult()); + } + else{ + result.add(getInvalidBackupVerificationResult()); + } return result; } } + static class MockBackupNotificationMgr extends MockUp{ + @Mock + public void notify(BackupVerificationResult backupVerificationResult){ + //do nothing just return + return; + } + } + @Test public void throwError() throws Exception { MockBackupVerification.throwError = true; @@ -98,6 +114,14 @@ public void normalOperation() throws Exception { backupVerificationService.execute(); } + @Test + public void normalOperationNoValidBackups() throws Exception { + MockBackupVerification.throwError = false; + MockBackupVerification.failCall = false; + MockBackupVerification.validBackupVerificationResult = false; + backupVerificationService.execute(); + } + @Test public void testRestoreMode(@Mocked InstanceState state) throws Exception { new Expectations() { @@ -108,4 +132,24 @@ public void testRestoreMode(@Mocked InstanceState state) throws Exception { }; backupVerificationService.execute(); } + + private static BackupVerificationResult getInvalidBackupVerificationResult() { + BackupVerificationResult result = new BackupVerificationResult(); + result.valid = false; + result.manifestAvailable = true; + result.remotePath = "some_random"; + result.filesMatched = 123; + result.snapshotInstant = Instant.EPOCH; + return result; + } + + private static BackupVerificationResult getValidBackupVerificationResult() { + BackupVerificationResult result = new BackupVerificationResult(); + result.valid = true; + result.manifestAvailable = true; + result.remotePath = "some_random"; + result.filesMatched = 123; + result.snapshotInstant = Instant.EPOCH; + return result; + } } From af16d2579fb77cbfb82956ab2208556eeaf3acf2 Mon Sep 17 00:00:00 2001 From: Sreeram Chakrovorthy Date: Wed, 22 Apr 2020 18:34:00 -0700 Subject: [PATCH 108/228] Code formatting. --- .../priam/backupv2/BackupVerificationTask.java | 7 ++++--- .../priam/backupv2/TestBackupVerificationTask.java | 12 +++++------- 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/priam/src/main/java/com/netflix/priam/backupv2/BackupVerificationTask.java b/priam/src/main/java/com/netflix/priam/backupv2/BackupVerificationTask.java index b91f383f5..c0c655254 100644 --- a/priam/src/main/java/com/netflix/priam/backupv2/BackupVerificationTask.java +++ b/priam/src/main/java/com/netflix/priam/backupv2/BackupVerificationTask.java @@ -32,7 +32,6 @@ import com.netflix.priam.utils.DateUtil.DateRange; import java.time.temporal.ChronoUnit; import java.util.List; -import java.util.stream.Collectors; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -93,8 +92,10 @@ public void execute() throws Exception { backupVerification.verifyAllBackups(BackupVersion.SNAPSHOT_META_SERVICE, dateRange); if (verificationResults.isEmpty() || verificationResults - .stream() - .filter(backupVerificationResult -> backupVerificationResult.valid).count() == 0) { + .stream() + .filter(backupVerificationResult -> backupVerificationResult.valid) + .count() + == 0) { logger.error( "Not able to find any snapshot which is valid in our SLO window: {} hours", backupRestoreConfig.getBackupVerificationSLOInHours()); diff --git a/priam/src/test/java/com/netflix/priam/backupv2/TestBackupVerificationTask.java b/priam/src/test/java/com/netflix/priam/backupv2/TestBackupVerificationTask.java index bc0e5db54..341258fec 100644 --- a/priam/src/test/java/com/netflix/priam/backupv2/TestBackupVerificationTask.java +++ b/priam/src/test/java/com/netflix/priam/backupv2/TestBackupVerificationTask.java @@ -29,7 +29,6 @@ import com.netflix.priam.notification.BackupNotificationMgr; import com.netflix.priam.scheduler.UnsupportedTypeException; import com.netflix.priam.utils.DateUtil.DateRange; - import java.time.Instant; import java.util.ArrayList; import java.util.List; @@ -70,20 +69,19 @@ public List verifyAllBackups( if (failCall) return new ArrayList<>(); List result = new ArrayList<>(); - if(validBackupVerificationResult){ + if (validBackupVerificationResult) { result.add(getValidBackupVerificationResult()); - } - else{ + } else { result.add(getInvalidBackupVerificationResult()); } return result; } } - static class MockBackupNotificationMgr extends MockUp{ + static class MockBackupNotificationMgr extends MockUp { @Mock - public void notify(BackupVerificationResult backupVerificationResult){ - //do nothing just return + public void notify(BackupVerificationResult backupVerificationResult) { + // do nothing just return return; } } From 58635cd353d2c54a10795682ce1e7bd4b27f35eb Mon Sep 17 00:00:00 2001 From: Sreeram Chakrovorthy Date: Wed, 22 Apr 2020 18:59:58 -0700 Subject: [PATCH 109/228] Adding another test to improve Project code coverage. --- .../TestBackupNotificationMgr.java | 30 +++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/priam/src/test/java/com/netflix/priam/notification/TestBackupNotificationMgr.java b/priam/src/test/java/com/netflix/priam/notification/TestBackupNotificationMgr.java index ce0ffb310..08bbfc0dd 100644 --- a/priam/src/test/java/com/netflix/priam/notification/TestBackupNotificationMgr.java +++ b/priam/src/test/java/com/netflix/priam/notification/TestBackupNotificationMgr.java @@ -6,11 +6,13 @@ import com.google.inject.Provider; import com.netflix.priam.backup.AbstractBackupPath; import com.netflix.priam.backup.BRTestModule; +import com.netflix.priam.backup.BackupVerificationResult; import com.netflix.priam.config.IBackupRestoreConfig; import com.netflix.priam.config.IConfiguration; import java.nio.file.Path; import java.nio.file.Paths; import java.text.ParseException; +import java.time.Instant; import java.util.Map; import mockit.Capturing; import mockit.Expectations; @@ -291,4 +293,32 @@ public void testNoNotificationsPartiallyValidFilter( } }; } + + @Test + public void testNotify(@Capturing INotificationService notificationService){ + new Expectations() { + { + notificationService.notify(anyString, (Map) any); + maxTimes = 1; + } + }; + BackupVerificationResult backupVerificationResult = getBackupVerificationResult(); + backupNotificationMgr.notify(backupVerificationResult); + new Verifications() { + { + notificationService.notify(anyString, (Map) any); + maxTimes = 1; + } + }; + } + + private static BackupVerificationResult getBackupVerificationResult() { + BackupVerificationResult result = new BackupVerificationResult(); + result.valid = true; + result.manifestAvailable = true; + result.remotePath = "some_random"; + result.filesMatched = 123; + result.snapshotInstant = Instant.EPOCH; + return result; + } } From e7b81b8c450952268fe00b8de4c300fea7acd02c Mon Sep 17 00:00:00 2001 From: Sreeram Chakrovorthy Date: Wed, 22 Apr 2020 19:01:11 -0700 Subject: [PATCH 110/228] Code formatting. --- .../netflix/priam/notification/TestBackupNotificationMgr.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/priam/src/test/java/com/netflix/priam/notification/TestBackupNotificationMgr.java b/priam/src/test/java/com/netflix/priam/notification/TestBackupNotificationMgr.java index 08bbfc0dd..7fa6d6bb2 100644 --- a/priam/src/test/java/com/netflix/priam/notification/TestBackupNotificationMgr.java +++ b/priam/src/test/java/com/netflix/priam/notification/TestBackupNotificationMgr.java @@ -295,7 +295,7 @@ public void testNoNotificationsPartiallyValidFilter( } @Test - public void testNotify(@Capturing INotificationService notificationService){ + public void testNotify(@Capturing INotificationService notificationService) { new Expectations() { { notificationService.notify(anyString, (Map) any); From 07e1b7c3044a98d2b15f67f8251941568b7f02eb Mon Sep 17 00:00:00 2001 From: Sreeram Chakrovorthy Date: Thu, 23 Apr 2020 10:32:06 -0700 Subject: [PATCH 111/228] Changelog for the changes related to Verifying all backups in the desired date range. --- CHANGELOG.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fc259cbbb..ad5e747f9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,7 @@ # Changelog +## 2020/04/22 3.11.58 +(#850) Modifying the backup verification strategy to verify all unverified backups in the specified date range vs the old implementation that verified the latest backup in the specified date range. -# Changelog ## 2020/02/21 3.11.57 (#844, #839) Implementation of a filter for Backup Notification. The filter can be controlled using the configuration "priam.backupNotifyComponentIncludeList" From 6f20e6a6951a5c1133a29333f7c835df3cf5965a Mon Sep 17 00:00:00 2001 From: Sreeram Chakrovorthy Date: Fri, 24 Apr 2020 10:45:56 -0700 Subject: [PATCH 112/228] Committing an update to change log to include the changes made by Matt for adding a hook to the StandardTuner to allow subclasses to add custom Cassandra parameters. --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ad5e747f9..19a2171ef 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ # Changelog ## 2020/04/22 3.11.58 -(#850) Modifying the backup verification strategy to verify all unverified backups in the specified date range vs the old implementation that verified the latest backup in the specified date range. +(#850) Modifying the backup verification strategy to verify all unverified backups in the specified date range vs the old implementation that verified the latest backup in the specified date range. Also adding a hook in StandardTuner to allow for subclasses to add custom Cassandra parameters ## 2020/02/21 3.11.57 (#844, #839) Implementation of a filter for Backup Notification. The filter can be controlled using the configuration "priam.backupNotifyComponentIncludeList" From d7768ed4fe491afc73dd681899ade9000af8bcd5 Mon Sep 17 00:00:00 2001 From: Sreeram Chakrovorthy Date: Mon, 4 May 2020 12:24:56 -0700 Subject: [PATCH 113/228] CASS-1731 Handling the case when there are no backups taken yet and hence no verification results are available. --- .../netflix/priam/backupv2/BackupVerificationTask.java | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/priam/src/main/java/com/netflix/priam/backupv2/BackupVerificationTask.java b/priam/src/main/java/com/netflix/priam/backupv2/BackupVerificationTask.java index c0c655254..61c6014cd 100644 --- a/priam/src/main/java/com/netflix/priam/backupv2/BackupVerificationTask.java +++ b/priam/src/main/java/com/netflix/priam/backupv2/BackupVerificationTask.java @@ -90,8 +90,8 @@ public void execute() throws Exception { DateUtil.getInstant()); List verificationResults = backupVerification.verifyAllBackups(BackupVersion.SNAPSHOT_META_SERVICE, dateRange); - if (verificationResults.isEmpty() - || verificationResults + if (!verificationResults.isEmpty() + && verificationResults .stream() .filter(backupVerificationResult -> backupVerificationResult.valid) .count() @@ -101,8 +101,9 @@ public void execute() throws Exception { backupRestoreConfig.getBackupVerificationSLOInHours()); backupMetrics.incrementBackupVerificationFailure(); } else { - // verification result is available and is valid. - // send notification that backup is uploaded. + // we would be here if there are no backup verification results + // or backup verification results are available and are all valid. + // send notifications for each backup that was uploaded and verified. verificationResults .stream() .forEach( From d7d624fb8489e9c40e774066ac90e3717799a344 Mon Sep 17 00:00:00 2001 From: Sreeram Chakrovorthy Date: Mon, 4 May 2020 13:10:55 -0700 Subject: [PATCH 114/228] CASS-1731 Handling the case when there are no backups to verify. --- .../backupv2/BackupVerificationTask.java | 4 ++ .../backupv2/TestBackupVerificationTask.java | 61 +++++++++++++++---- 2 files changed, 53 insertions(+), 12 deletions(-) diff --git a/priam/src/main/java/com/netflix/priam/backupv2/BackupVerificationTask.java b/priam/src/main/java/com/netflix/priam/backupv2/BackupVerificationTask.java index 61c6014cd..3b1532811 100644 --- a/priam/src/main/java/com/netflix/priam/backupv2/BackupVerificationTask.java +++ b/priam/src/main/java/com/netflix/priam/backupv2/BackupVerificationTask.java @@ -117,6 +117,10 @@ public void execute() throws Exception { } } + public BackupMetrics getBackupMetrics() { + return backupMetrics; + } + /** * Interval between trying to verify data manifest file on Remote file system. * diff --git a/priam/src/test/java/com/netflix/priam/backupv2/TestBackupVerificationTask.java b/priam/src/test/java/com/netflix/priam/backupv2/TestBackupVerificationTask.java index 341258fec..d3659509c 100644 --- a/priam/src/test/java/com/netflix/priam/backupv2/TestBackupVerificationTask.java +++ b/priam/src/test/java/com/netflix/priam/backupv2/TestBackupVerificationTask.java @@ -32,10 +32,7 @@ import java.time.Instant; import java.util.ArrayList; import java.util.List; -import mockit.Expectations; -import mockit.Mock; -import mockit.MockUp; -import mockit.Mocked; +import mockit.*; import org.junit.Assert; import org.junit.Test; @@ -66,13 +63,17 @@ public List verifyAllBackups( throws UnsupportedTypeException, IllegalArgumentException { if (throwError) throw new IllegalArgumentException("DummyError"); - if (failCall) return new ArrayList<>(); + if (failCall) { + List results = new ArrayList<>(); + results.add(getInvalidBackupVerificationResult()); + return results; + } List result = new ArrayList<>(); if (validBackupVerificationResult) { result.add(getValidBackupVerificationResult()); } else { - result.add(getInvalidBackupVerificationResult()); + return result; // Return empty backup verification results } return result; } @@ -99,25 +100,61 @@ public void throwError() throws Exception { } @Test - public void failCalls() throws Exception { + public void normalOperation() throws Exception { MockBackupVerification.throwError = false; - MockBackupVerification.failCall = true; + MockBackupVerification.failCall = false; + new Expectations() { + { + backupVerificationService.getBackupMetrics().incrementBackupVerificationFailure(); + maxTimes = 0; + } + }; backupVerificationService.execute(); + new Verifications() { + { + backupVerificationService.getBackupMetrics().incrementBackupVerificationFailure(); + maxTimes = 0; + } + }; } @Test - public void normalOperation() throws Exception { + public void normalOperationEmptyBackups() throws Exception { MockBackupVerification.throwError = false; MockBackupVerification.failCall = false; + MockBackupVerification.validBackupVerificationResult = false; + new Expectations() { + { + backupVerificationService.getBackupMetrics().incrementBackupVerificationFailure(); + maxTimes = 0; + } + }; backupVerificationService.execute(); + new Verifications() { + { + backupVerificationService.getBackupMetrics().incrementBackupVerificationFailure(); + maxTimes = 0; + } + }; } @Test - public void normalOperationNoValidBackups() throws Exception { + public void failCalls() throws Exception { MockBackupVerification.throwError = false; - MockBackupVerification.failCall = false; - MockBackupVerification.validBackupVerificationResult = false; + MockBackupVerification.failCall = true; + new Expectations() { + { + backupVerificationService.getBackupMetrics().incrementBackupVerificationFailure(); + maxTimes = 1; + } + }; backupVerificationService.execute(); + new Verifications() { + { + backupVerificationService.getBackupMetrics().incrementBackupVerificationFailure(); + maxTimes = 1; + } + }; } @Test From a1480ff6c5ef0a8b6538c17d7c25b809283f82b3 Mon Sep 17 00:00:00 2001 From: Sreeram Chakrovorthy Date: Tue, 5 May 2020 12:27:03 -0700 Subject: [PATCH 115/228] CASS-1731 Addressing the review comments, we now have separated the scenarios where we would get empty backup verification results into 2 checks, 1) Due to all verified backups in our SLO date range, 2) Due to no backups in our SLO date range. We will continue to page for 2, but not for 1. --- .../priam/backup/BackupVerification.java | 2 +- .../backupv2/BackupVerificationTask.java | 28 +++++++--- .../priam/backup/TestBackupVerification.java | 7 +++ .../backupv2/TestBackupVerificationTask.java | 51 ++++++++++++------- 4 files changed, 62 insertions(+), 26 deletions(-) diff --git a/priam/src/main/java/com/netflix/priam/backup/BackupVerification.java b/priam/src/main/java/com/netflix/priam/backup/BackupVerification.java index 47e5337d7..71102f3a7 100644 --- a/priam/src/main/java/com/netflix/priam/backup/BackupVerification.java +++ b/priam/src/main/java/com/netflix/priam/backup/BackupVerification.java @@ -53,7 +53,7 @@ public class BackupVerification { this.abstractBackupPathProvider = abstractBackupPathProvider; } - private IMetaProxy getMetaProxy(BackupVersion backupVersion) { + public IMetaProxy getMetaProxy(BackupVersion backupVersion) { switch (backupVersion) { case SNAPSHOT_BACKUP: return metaV1Proxy; diff --git a/priam/src/main/java/com/netflix/priam/backupv2/BackupVerificationTask.java b/priam/src/main/java/com/netflix/priam/backupv2/BackupVerificationTask.java index 3b1532811..af14c164e 100644 --- a/priam/src/main/java/com/netflix/priam/backupv2/BackupVerificationTask.java +++ b/priam/src/main/java/com/netflix/priam/backupv2/BackupVerificationTask.java @@ -90,19 +90,31 @@ public void execute() throws Exception { DateUtil.getInstant()); List verificationResults = backupVerification.verifyAllBackups(BackupVersion.SNAPSHOT_META_SERVICE, dateRange); - if (!verificationResults.isEmpty() - && verificationResults - .stream() - .filter(backupVerificationResult -> backupVerificationResult.valid) - .count() - == 0) { + // There are no backups in our SLO window and hence verification results is empty + boolean backupVerificationFailed = + (verificationResults.isEmpty() + && !BackupRestoreUtil.getLatestValidMetaPath( + backupVerification.getMetaProxy( + BackupVersion.SNAPSHOT_META_SERVICE), + dateRange) + .isPresent()); + // There are no valid backups in our SLO window + backupVerificationFailed |= + !verificationResults.isEmpty() + && verificationResults + .stream() + .filter( + backupVerificationResult -> + backupVerificationResult.valid) + .count() + == 0; + if (backupVerificationFailed) { logger.error( "Not able to find any snapshot which is valid in our SLO window: {} hours", backupRestoreConfig.getBackupVerificationSLOInHours()); backupMetrics.incrementBackupVerificationFailure(); } else { - // we would be here if there are no backup verification results - // or backup verification results are available and are all valid. + // we would be here only if there are valid backup verification results // send notifications for each backup that was uploaded and verified. verificationResults .stream() diff --git a/priam/src/test/java/com/netflix/priam/backup/TestBackupVerification.java b/priam/src/test/java/com/netflix/priam/backup/TestBackupVerification.java index daa182dcd..c14c35c13 100644 --- a/priam/src/test/java/com/netflix/priam/backup/TestBackupVerification.java +++ b/priam/src/test/java/com/netflix/priam/backup/TestBackupVerification.java @@ -20,6 +20,7 @@ import com.google.inject.Guice; import com.google.inject.Injector; import com.netflix.priam.backup.AbstractBackupPath.BackupFileType; +import com.netflix.priam.backupv2.IMetaProxy; import com.netflix.priam.backupv2.MetaV1Proxy; import com.netflix.priam.backupv2.MetaV2Proxy; import com.netflix.priam.config.IConfiguration; @@ -329,4 +330,10 @@ private static BackupVerificationResult getBackupVerificationResult() { result.snapshotInstant = Instant.EPOCH; return result; } + + @Test + public void testGetMetaProxy() { + IMetaProxy metaProxy = backupVerification.getMetaProxy(BackupVersion.SNAPSHOT_META_SERVICE); + Assert.assertTrue(metaProxy != null); + } } diff --git a/priam/src/test/java/com/netflix/priam/backupv2/TestBackupVerificationTask.java b/priam/src/test/java/com/netflix/priam/backupv2/TestBackupVerificationTask.java index d3659509c..1b34b09dc 100644 --- a/priam/src/test/java/com/netflix/priam/backupv2/TestBackupVerificationTask.java +++ b/priam/src/test/java/com/netflix/priam/backupv2/TestBackupVerificationTask.java @@ -19,11 +19,7 @@ import com.google.inject.Guice; import com.google.inject.Injector; -import com.netflix.priam.backup.BRTestModule; -import com.netflix.priam.backup.BackupVerification; -import com.netflix.priam.backup.BackupVerificationResult; -import com.netflix.priam.backup.BackupVersion; -import com.netflix.priam.backup.Status; +import com.netflix.priam.backup.*; import com.netflix.priam.config.IConfiguration; import com.netflix.priam.health.InstanceState; import com.netflix.priam.notification.BackupNotificationMgr; @@ -32,6 +28,7 @@ import java.time.Instant; import java.util.ArrayList; import java.util.List; +import java.util.Optional; import mockit.*; import org.junit.Assert; import org.junit.Test; @@ -53,7 +50,7 @@ public TestBackupVerificationTask() { } static class MockBackupVerification extends MockUp { - public static boolean failCall = false; + public static boolean emptyBackupVerificationList = false; public static boolean throwError = false; public static boolean validBackupVerificationResult = true; @@ -63,17 +60,14 @@ public List verifyAllBackups( throws UnsupportedTypeException, IllegalArgumentException { if (throwError) throw new IllegalArgumentException("DummyError"); - if (failCall) { - List results = new ArrayList<>(); - results.add(getInvalidBackupVerificationResult()); - return results; + if (emptyBackupVerificationList) { + return new ArrayList<>(); } - List result = new ArrayList<>(); if (validBackupVerificationResult) { result.add(getValidBackupVerificationResult()); } else { - return result; // Return empty backup verification results + result.add(getInvalidBackupVerificationResult()); } return result; } @@ -90,7 +84,7 @@ public void notify(BackupVerificationResult backupVerificationResult) { @Test public void throwError() throws Exception { MockBackupVerification.throwError = true; - MockBackupVerification.failCall = false; + MockBackupVerification.emptyBackupVerificationList = false; try { backupVerificationService.execute(); Assert.assertTrue(false); @@ -102,7 +96,7 @@ public void throwError() throws Exception { @Test public void normalOperation() throws Exception { MockBackupVerification.throwError = false; - MockBackupVerification.failCall = false; + MockBackupVerification.emptyBackupVerificationList = false; new Expectations() { { backupVerificationService.getBackupMetrics().incrementBackupVerificationFailure(); @@ -119,9 +113,32 @@ public void normalOperation() throws Exception { } @Test - public void normalOperationEmptyBackups() throws Exception { + public void normalOperationPriorVerifiedBackups( + @Mocked BackupRestoreUtil backupRestoreUtil, + @Mocked AbstractBackupPath remoteBackupPath) + throws Exception { + MockBackupVerification.throwError = false; + MockBackupVerification.emptyBackupVerificationList = true; + new Expectations() { + { + backupRestoreUtil.getLatestValidMetaPath((IMetaProxy) any, (DateRange) any); + result = Optional.of(remoteBackupPath); + maxTimes = 1; + } + }; + backupVerificationService.execute(); + new Verifications() { + { + backupVerificationService.getBackupMetrics().incrementBackupVerificationFailure(); + maxTimes = 1; + } + }; + } + + @Test + public void normalOperationInvalidBackups() throws Exception { MockBackupVerification.throwError = false; - MockBackupVerification.failCall = false; + MockBackupVerification.emptyBackupVerificationList = false; MockBackupVerification.validBackupVerificationResult = false; new Expectations() { { @@ -141,7 +158,7 @@ public void normalOperationEmptyBackups() throws Exception { @Test public void failCalls() throws Exception { MockBackupVerification.throwError = false; - MockBackupVerification.failCall = true; + MockBackupVerification.emptyBackupVerificationList = true; new Expectations() { { backupVerificationService.getBackupMetrics().incrementBackupVerificationFailure(); From 4c8533d5f82213f89c05dcbc99ed9bb36890f60d Mon Sep 17 00:00:00 2001 From: Sreeram Chakrovorthy Date: Mon, 4 May 2020 16:15:36 -0700 Subject: [PATCH 116/228] CASS-1731 Adding a test for the get accessor of BackupMetrics in the BackUpVerificationTask class. --- .../netflix/priam/backupv2/TestBackupVerificationTask.java | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/priam/src/test/java/com/netflix/priam/backupv2/TestBackupVerificationTask.java b/priam/src/test/java/com/netflix/priam/backupv2/TestBackupVerificationTask.java index 1b34b09dc..0d6244c3e 100644 --- a/priam/src/test/java/com/netflix/priam/backupv2/TestBackupVerificationTask.java +++ b/priam/src/test/java/com/netflix/priam/backupv2/TestBackupVerificationTask.java @@ -22,6 +22,7 @@ import com.netflix.priam.backup.*; import com.netflix.priam.config.IConfiguration; import com.netflix.priam.health.InstanceState; +import com.netflix.priam.merics.BackupMetrics; import com.netflix.priam.notification.BackupNotificationMgr; import com.netflix.priam.scheduler.UnsupportedTypeException; import com.netflix.priam.utils.DateUtil.DateRange; @@ -185,6 +186,12 @@ public void testRestoreMode(@Mocked InstanceState state) throws Exception { backupVerificationService.execute(); } + @Test + public void testGetBackupMetrics() { + BackupMetrics backupMetrics = backupVerificationService.getBackupMetrics(); + Assert.assertTrue(backupMetrics != null); + } + private static BackupVerificationResult getInvalidBackupVerificationResult() { BackupVerificationResult result = new BackupVerificationResult(); result.valid = false; From 9ba1c47d21aae27e248e963d431245ee030158f1 Mon Sep 17 00:00:00 2001 From: Sreeram Chakrovorthy Date: Tue, 5 May 2020 14:24:41 -0700 Subject: [PATCH 117/228] CASS-1731 Modifying the logic based on review comments to 1) Notify for every verified backup, 2) If there are no verified backups in our SLO window we would page. --- .../backupv2/BackupVerificationTask.java | 47 +++++++------------ 1 file changed, 16 insertions(+), 31 deletions(-) diff --git a/priam/src/main/java/com/netflix/priam/backupv2/BackupVerificationTask.java b/priam/src/main/java/com/netflix/priam/backupv2/BackupVerificationTask.java index af14c164e..71337cdf9 100644 --- a/priam/src/main/java/com/netflix/priam/backupv2/BackupVerificationTask.java +++ b/priam/src/main/java/com/netflix/priam/backupv2/BackupVerificationTask.java @@ -90,42 +90,27 @@ public void execute() throws Exception { DateUtil.getInstant()); List verificationResults = backupVerification.verifyAllBackups(BackupVersion.SNAPSHOT_META_SERVICE, dateRange); + + verificationResults + .stream() + .forEach( + backupVerificationResult -> { + logger.info( + "Sending {} message for backup: {}", + AbstractBackupPath.BackupFileType.SNAPSHOT_VERIFIED, + backupVerificationResult.snapshotInstant); + backupNotificationMgr.notify(backupVerificationResult); + }); + // There are no backups in our SLO window and hence verification results is empty - boolean backupVerificationFailed = - (verificationResults.isEmpty() - && !BackupRestoreUtil.getLatestValidMetaPath( - backupVerification.getMetaProxy( - BackupVersion.SNAPSHOT_META_SERVICE), - dateRange) - .isPresent()); - // There are no valid backups in our SLO window - backupVerificationFailed |= - !verificationResults.isEmpty() - && verificationResults - .stream() - .filter( - backupVerificationResult -> - backupVerificationResult.valid) - .count() - == 0; - if (backupVerificationFailed) { + if (!BackupRestoreUtil.getLatestValidMetaPath( + backupVerification.getMetaProxy(BackupVersion.SNAPSHOT_META_SERVICE), + dateRange) + .isPresent()) { logger.error( "Not able to find any snapshot which is valid in our SLO window: {} hours", backupRestoreConfig.getBackupVerificationSLOInHours()); backupMetrics.incrementBackupVerificationFailure(); - } else { - // we would be here only if there are valid backup verification results - // send notifications for each backup that was uploaded and verified. - verificationResults - .stream() - .forEach( - backupVerificationResult -> { - logger.info( - "Sending {} message for backup: {}", - AbstractBackupPath.BackupFileType.SNAPSHOT_VERIFIED, - backupVerificationResult.snapshotInstant); - backupNotificationMgr.notify(backupVerificationResult); - }); } } From 168191e608fcd887091a961ee36a9f39e44293d3 Mon Sep 17 00:00:00 2001 From: Sreeram Chakrovorthy Date: Tue, 5 May 2020 14:43:34 -0700 Subject: [PATCH 118/228] CASS-1731 Removing a comment that is no longer needed. --- .../com/netflix/priam/backupv2/BackupVerificationTask.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/priam/src/main/java/com/netflix/priam/backupv2/BackupVerificationTask.java b/priam/src/main/java/com/netflix/priam/backupv2/BackupVerificationTask.java index 71337cdf9..5b50734c3 100644 --- a/priam/src/main/java/com/netflix/priam/backupv2/BackupVerificationTask.java +++ b/priam/src/main/java/com/netflix/priam/backupv2/BackupVerificationTask.java @@ -90,7 +90,7 @@ public void execute() throws Exception { DateUtil.getInstant()); List verificationResults = backupVerification.verifyAllBackups(BackupVersion.SNAPSHOT_META_SERVICE, dateRange); - + verificationResults .stream() .forEach( @@ -102,7 +102,6 @@ public void execute() throws Exception { backupNotificationMgr.notify(backupVerificationResult); }); - // There are no backups in our SLO window and hence verification results is empty if (!BackupRestoreUtil.getLatestValidMetaPath( backupVerification.getMetaProxy(BackupVersion.SNAPSHOT_META_SERVICE), dateRange) From 38fb8eacc9f1a4f6695debf5271f5ca7887434d2 Mon Sep 17 00:00:00 2001 From: Sreeram Chakrovorthy Date: Tue, 5 May 2020 17:19:02 -0700 Subject: [PATCH 119/228] CASS-1731 Fixing the code formatting errors. --- .../java/com/netflix/priam/backupv2/BackupVerificationTask.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/priam/src/main/java/com/netflix/priam/backupv2/BackupVerificationTask.java b/priam/src/main/java/com/netflix/priam/backupv2/BackupVerificationTask.java index 5b50734c3..7589b8992 100644 --- a/priam/src/main/java/com/netflix/priam/backupv2/BackupVerificationTask.java +++ b/priam/src/main/java/com/netflix/priam/backupv2/BackupVerificationTask.java @@ -90,7 +90,7 @@ public void execute() throws Exception { DateUtil.getInstant()); List verificationResults = backupVerification.verifyAllBackups(BackupVersion.SNAPSHOT_META_SERVICE, dateRange); - + verificationResults .stream() .forEach( From cb9e16d6017bc00bb5426117a6a86c3e306b8046 Mon Sep 17 00:00:00 2001 From: Sreeram Chakrovorthy Date: Tue, 5 May 2020 15:10:13 -0700 Subject: [PATCH 120/228] CASS-1730 Changes to disable lifecycle rule if backup1.0 is disabled. --- .../main/java/com/netflix/priam/backup/BackupService.java | 5 ++--- .../java/com/netflix/priam/backup/TestBackupService.java | 2 +- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/priam/src/main/java/com/netflix/priam/backup/BackupService.java b/priam/src/main/java/com/netflix/priam/backup/BackupService.java index 7a16d4cba..64c750f8d 100644 --- a/priam/src/main/java/com/netflix/priam/backup/BackupService.java +++ b/priam/src/main/java/com/netflix/priam/backup/BackupService.java @@ -60,6 +60,8 @@ public void scheduleService() throws Exception { scheduleTask(scheduler, SnapshotBackup.class, snapshotTimer); if (snapshotTimer != null) { + // Set cleanup + scheduleTask(scheduler, UpdateCleanupPolicy.class, UpdateCleanupPolicy.getTimer()); // Schedule commit log task scheduleTask( scheduler, CommitLogBackupTask.class, CommitLogBackupTask.getTimer(config)); @@ -70,9 +72,6 @@ public void scheduleService() throws Exception { scheduler, IncrementalBackup.class, IncrementalBackup.getTimer(config, backupRestoreConfig)); - - // Set cleanup - scheduleTask(scheduler, UpdateCleanupPolicy.class, UpdateCleanupPolicy.getTimer()); } @Override diff --git a/priam/src/test/java/com/netflix/priam/backup/TestBackupService.java b/priam/src/test/java/com/netflix/priam/backup/TestBackupService.java index 2e6171eb4..ed4a34978 100644 --- a/priam/src/test/java/com/netflix/priam/backup/TestBackupService.java +++ b/priam/src/test/java/com/netflix/priam/backup/TestBackupService.java @@ -98,7 +98,7 @@ public void testBackupDisabled( new BackupService( configuration, backupRestoreConfig, scheduler, cassandraTunerService); backupService.scheduleService(); - Assert.assertEquals(1, scheduler.getScheduler().getJobKeys(null).size()); + Assert.assertEquals(0, scheduler.getScheduler().getJobKeys(null).size()); // snapshot V1 name should not be there. Set backupPaths = From 6b3f0e18b1a16c4af762273d571b1aa2670e1dd8 Mon Sep 17 00:00:00 2001 From: Sreeram Chakrovorthy Date: Tue, 5 May 2020 18:09:59 -0700 Subject: [PATCH 121/228] CASS-1730 and CASS-1731 updating changelog. --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 19a2171ef..e7f166fd2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,7 @@ # Changelog +## 2020/05/05 3.11.59 +(#860, #864) Fixing the bug in the backup verification strategy to only page when there is no valid backup in the specified date range (SLO window) And also disable lifecyle rule for backup if backup v1 is disabled. + ## 2020/04/22 3.11.58 (#850) Modifying the backup verification strategy to verify all unverified backups in the specified date range vs the old implementation that verified the latest backup in the specified date range. Also adding a hook in StandardTuner to allow for subclasses to add custom Cassandra parameters From 05dd1d8c1c020478a8b247dbf1319eaeccc1d358 Mon Sep 17 00:00:00 2001 From: Sreeram Chakrovorthy Date: Fri, 15 May 2020 20:10:52 -0700 Subject: [PATCH 122/228] CASS-1799 Fixing the Priam config endpoints after they were broken due to a dependency update. --- .../netflix/priam/resources/PriamConfig.java | 20 +++++++++----- .../priam/resources/PriamConfigTest.java | 27 +++++++++++++++---- 2 files changed, 35 insertions(+), 12 deletions(-) diff --git a/priam/src/main/java/com/netflix/priam/resources/PriamConfig.java b/priam/src/main/java/com/netflix/priam/resources/PriamConfig.java index 49f8e683f..cbf781ac3 100644 --- a/priam/src/main/java/com/netflix/priam/resources/PriamConfig.java +++ b/priam/src/main/java/com/netflix/priam/resources/PriamConfig.java @@ -18,6 +18,7 @@ import com.google.inject.Inject; import com.netflix.priam.PriamServer; +import com.netflix.priam.utils.GsonJsonSerializer; import java.util.HashMap; import java.util.Map; import javax.ws.rs.GET; @@ -51,14 +52,17 @@ private Response doGetPriamConfig(String group, String name) { priamServer.getConfiguration().getStructuredConfiguration(group); if (name != null && value.containsKey(name)) { result.put(name, value.get(name)); - return Response.ok(result, MediaType.APPLICATION_JSON).build(); + return Response.ok(GsonJsonSerializer.getGson().toJson(result)).build(); } else if (name != null) { result.put("message", String.format("No such structured config: [%s]", name)); logger.error(String.format("No such structured config: [%s]", name)); - return Response.status(404).entity(result).type(MediaType.APPLICATION_JSON).build(); + return Response.status(404) + .entity(GsonJsonSerializer.getGson().toJson(result)) + .type(MediaType.TEXT_PLAIN) + .build(); } else { result.putAll(value); - return Response.ok(result, MediaType.APPLICATION_JSON).build(); + return Response.ok(GsonJsonSerializer.getGson().toJson(result)).build(); } } catch (Exception e) { logger.error("Error while executing getPriamConfig", e); @@ -68,8 +72,7 @@ private Response doGetPriamConfig(String group, String name) { @GET @Path("/structured/{group}") - public Response getPriamConfig( - @PathParam("group") String group, @PathParam("name") String name) { + public Response getPriamConfig(@PathParam("group") String group) { return doGetPriamConfig(group, null); } @@ -89,11 +92,14 @@ public Response getProperty( String value = priamServer.getConfiguration().getProperty(name, defaultValue); if (value != null) { result.put(name, value); - return Response.ok(result, MediaType.APPLICATION_JSON).build(); + return Response.ok(GsonJsonSerializer.getGson().toJson(result)).build(); } else { result.put("message", String.format("No such property: [%s]", name)); logger.error(String.format("No such property: [%s]", name)); - return Response.status(404).entity(result).type(MediaType.APPLICATION_JSON).build(); + return Response.status(404) + .entity(GsonJsonSerializer.getGson().toJson(result)) + .type(MediaType.TEXT_PLAIN) + .build(); } } catch (Exception e) { logger.error("Error while executing getPriamConfig", e); diff --git a/priam/src/test/java/com/netflix/priam/resources/PriamConfigTest.java b/priam/src/test/java/com/netflix/priam/resources/PriamConfigTest.java index 163c17f46..aaa407024 100644 --- a/priam/src/test/java/com/netflix/priam/resources/PriamConfigTest.java +++ b/priam/src/test/java/com/netflix/priam/resources/PriamConfigTest.java @@ -20,6 +20,7 @@ import com.netflix.priam.PriamServer; import com.netflix.priam.config.FakeConfiguration; +import com.netflix.priam.utils.GsonJsonSerializer; import java.util.HashMap; import java.util.Map; import javax.ws.rs.core.Response; @@ -49,17 +50,26 @@ public void setUp() { public void getPriamConfig() { final Map expected = new HashMap<>(); expected.put("backupLocation", "casstestbackup"); + String expectedJsonString = GsonJsonSerializer.getGson().toJson(expected); new Expectations() { { priamServer.getConfiguration(); result = fakeConfiguration; - times = 2; + times = 3; } }; - Response response = resource.getPriamConfigByName("all", "backupLocation"); + Response response = resource.getPriamConfig("all"); assertEquals(200, response.getStatus()); - assertEquals(expected, response.getEntity()); + + Map result = + GsonJsonSerializer.getGson().fromJson(response.getEntity().toString(), Map.class); + assertNotNull(result); + assertTrue(!result.isEmpty()); + + response = resource.getPriamConfigByName("all", "backupLocation"); + assertEquals(200, response.getStatus()); + assertEquals(expectedJsonString, response.getEntity()); Response badResponse = resource.getPriamConfigByName("all", "getUnrealThing"); assertEquals(404, badResponse.getStatus()); @@ -77,15 +87,22 @@ public void getProperty() { } }; + String expectedJsonString = GsonJsonSerializer.getGson().toJson(expected); Response response = resource.getProperty("test.prop", null); assertEquals(200, response.getStatus()); - assertEquals(expected, response.getEntity()); + assertEquals(expectedJsonString, response.getEntity()); + + Map result = + GsonJsonSerializer.getGson().fromJson(response.getEntity().toString(), Map.class); + assertNotNull(result); + assertTrue(!result.isEmpty()); Response defaultResponse = resource.getProperty("not.a.property", "NOVALUE"); expected.clear(); expected.put("not.a.property", "NOVALUE"); + expectedJsonString = GsonJsonSerializer.getGson().toJson(expected); assertEquals(200, defaultResponse.getStatus()); - assertEquals(expected, defaultResponse.getEntity()); + assertEquals(expectedJsonString, defaultResponse.getEntity()); Response badResponse = resource.getProperty("not.a.property", null); assertEquals(404, badResponse.getStatus()); From af0a886d1b26eebd3aa9edf7025848dc01864628 Mon Sep 17 00:00:00 2001 From: Sreeram Chakrovorthy Date: Sun, 17 May 2020 22:01:48 -0700 Subject: [PATCH 123/228] CASS-1799 Addressing review comments no need to use TEXT_PLAIN, in this case since we already serialize to JSON. --- .../main/java/com/netflix/priam/resources/PriamConfig.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/priam/src/main/java/com/netflix/priam/resources/PriamConfig.java b/priam/src/main/java/com/netflix/priam/resources/PriamConfig.java index cbf781ac3..f6118b71f 100644 --- a/priam/src/main/java/com/netflix/priam/resources/PriamConfig.java +++ b/priam/src/main/java/com/netflix/priam/resources/PriamConfig.java @@ -58,7 +58,7 @@ private Response doGetPriamConfig(String group, String name) { logger.error(String.format("No such structured config: [%s]", name)); return Response.status(404) .entity(GsonJsonSerializer.getGson().toJson(result)) - .type(MediaType.TEXT_PLAIN) + .type(MediaType.APPLICATION_JSON) .build(); } else { result.putAll(value); @@ -98,7 +98,7 @@ public Response getProperty( logger.error(String.format("No such property: [%s]", name)); return Response.status(404) .entity(GsonJsonSerializer.getGson().toJson(result)) - .type(MediaType.TEXT_PLAIN) + .type(MediaType.APPLICATION_JSON) .build(); } } catch (Exception e) { From 05d6130cf2b947cbcf62d765c203ad89cd557c0e Mon Sep 17 00:00:00 2001 From: Sreeram Chakrovorthy Date: Sun, 17 May 2020 23:22:53 -0700 Subject: [PATCH 124/228] CASS-1799 Adding some tests for PriamConfig endpoints. --- .../com/netflix/priam/resources/PriamConfigTest.java | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/priam/src/test/java/com/netflix/priam/resources/PriamConfigTest.java b/priam/src/test/java/com/netflix/priam/resources/PriamConfigTest.java index aaa407024..cead22ab1 100644 --- a/priam/src/test/java/com/netflix/priam/resources/PriamConfigTest.java +++ b/priam/src/test/java/com/netflix/priam/resources/PriamConfigTest.java @@ -48,9 +48,6 @@ public void setUp() { @Test public void getPriamConfig() { - final Map expected = new HashMap<>(); - expected.put("backupLocation", "casstestbackup"); - String expectedJsonString = GsonJsonSerializer.getGson().toJson(expected); new Expectations() { { priamServer.getConfiguration(); @@ -67,9 +64,14 @@ public void getPriamConfig() { assertNotNull(result); assertTrue(!result.isEmpty()); + final Map expected = new HashMap<>(); + expected.put("backupLocation", "casstestbackup"); + String expectedJsonString = GsonJsonSerializer.getGson().toJson(expected); response = resource.getPriamConfigByName("all", "backupLocation"); assertEquals(200, response.getStatus()); assertEquals(expectedJsonString, response.getEntity()); + result = GsonJsonSerializer.getGson().fromJson(response.getEntity().toString(), Map.class); + assertEquals(result, expected); Response badResponse = resource.getPriamConfigByName("all", "getUnrealThing"); assertEquals(404, badResponse.getStatus()); @@ -103,6 +105,10 @@ public void getProperty() { expectedJsonString = GsonJsonSerializer.getGson().toJson(expected); assertEquals(200, defaultResponse.getStatus()); assertEquals(expectedJsonString, defaultResponse.getEntity()); + result = + GsonJsonSerializer.getGson() + .fromJson(defaultResponse.getEntity().toString(), Map.class); + assertEquals(result, expected); Response badResponse = resource.getProperty("not.a.property", null); assertEquals(404, badResponse.getStatus()); From 7ecf60fca1eab29460448a0c038d1fc76696e0f0 Mon Sep 17 00:00:00 2001 From: Sreeram Chakrovorthy Date: Mon, 18 May 2020 13:22:46 -0700 Subject: [PATCH 125/228] CASS-1799 Changelog related commit. --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index e7f166fd2..ae1b76be2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,7 @@ # Changelog +## 2020/05/05 3.11.60 +(#870) Fixing PriamConfig endpoints that were broken because of an underlying dependency change from the last release. + ## 2020/05/05 3.11.59 (#860, #864) Fixing the bug in the backup verification strategy to only page when there is no valid backup in the specified date range (SLO window) And also disable lifecyle rule for backup if backup v1 is disabled. From f193cdc7310fcc067514c4ebe62aed932ae208e6 Mon Sep 17 00:00:00 2001 From: Sreeram Chakrovorthy Date: Mon, 18 May 2020 13:23:27 -0700 Subject: [PATCH 126/228] CASS-1799 Updating changelog commit date. --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ae1b76be2..d5793b49a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,5 @@ # Changelog -## 2020/05/05 3.11.60 +## 2020/05/18 3.11.60 (#870) Fixing PriamConfig endpoints that were broken because of an underlying dependency change from the last release. ## 2020/05/05 3.11.59 From 65304220f87fa83361978977b5b398ff0b401e11 Mon Sep 17 00:00:00 2001 From: Sreeram Chakrovorthy Date: Mon, 18 May 2020 14:11:10 -0700 Subject: [PATCH 127/228] CASS-1799 Changelog update for re-release of v3.11.60 --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index d5793b49a..538794c18 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,7 @@ # Changelog +## 2020/05/18 3.11.61 +This is a re-release of v3.11.60 that failed to be uploaded. + ## 2020/05/18 3.11.60 (#870) Fixing PriamConfig endpoints that were broken because of an underlying dependency change from the last release. From 3ab2e5a64bce4a96d9937242654e00a97b76d142 Mon Sep 17 00:00:00 2001 From: Sreeram Chakrovorthy Date: Mon, 18 May 2020 21:27:21 -0700 Subject: [PATCH 128/228] CASS-1799 Fixing the other Priam endpoints that were missed in the previous release. Adding tests for BackupServletV2 resource. --- .../priam/resources/BackupServletV2.java | 13 +- .../priam/resources/BackupServletV2Test.java | 282 ++++++++++++++++++ 2 files changed, 290 insertions(+), 5 deletions(-) create mode 100644 priam/src/test/java/com/netflix/priam/resources/BackupServletV2Test.java diff --git a/priam/src/main/java/com/netflix/priam/resources/BackupServletV2.java b/priam/src/main/java/com/netflix/priam/resources/BackupServletV2.java index 64ccff9e7..681306468 100644 --- a/priam/src/main/java/com/netflix/priam/resources/BackupServletV2.java +++ b/priam/src/main/java/com/netflix/priam/resources/BackupServletV2.java @@ -27,6 +27,7 @@ import com.netflix.priam.config.IConfiguration; import com.netflix.priam.utils.DateUtil; import com.netflix.priam.utils.DateUtil.DateRange; +import com.netflix.priam.utils.GsonJsonSerializer; import java.time.Instant; import java.time.temporal.ChronoUnit; import java.util.List; @@ -118,7 +119,7 @@ public Response info(@PathParam("date") String date) { new DateRange( instant, instant.plus(1, ChronoUnit.DAYS).truncatedTo(ChronoUnit.DAYS))); - return Response.ok(metadataList).build(); + return Response.ok(GsonJsonSerializer.getGson().toJson(metadataList)).build(); } @GET @@ -137,7 +138,7 @@ public Response validateV2SnapshotByDate( .build(); } - return Response.ok(result.get()).build(); + return Response.ok(result.get().toString()).build(); } @GET @@ -155,9 +156,11 @@ public Response list(@PathParam("daterange") String daterange) throws Exception latestValidMetaFile.get(), dateRange, metaProxy, pathProvider); return Response.ok( - allFiles.stream() - .map(AbstractBackupPath::getRemotePath) - .collect(Collectors.toList())) + GsonJsonSerializer.getGson() + .toJson( + allFiles.stream() + .map(AbstractBackupPath::getRemotePath) + .collect(Collectors.toList()))) .build(); } } diff --git a/priam/src/test/java/com/netflix/priam/resources/BackupServletV2Test.java b/priam/src/test/java/com/netflix/priam/resources/BackupServletV2Test.java new file mode 100644 index 000000000..e9f0d32b1 --- /dev/null +++ b/priam/src/test/java/com/netflix/priam/resources/BackupServletV2Test.java @@ -0,0 +1,282 @@ +package com.netflix.priam.resources; + +import static org.junit.Assert.assertEquals; + +import com.google.inject.Guice; +import com.google.inject.Injector; +import com.netflix.priam.backup.*; +import com.netflix.priam.backupv2.SnapshotMetaTask; +import com.netflix.priam.config.IConfiguration; +import com.netflix.priam.identity.config.InstanceInfo; +import com.netflix.priam.restore.Restore; +import com.netflix.priam.utils.DateUtil; +import com.netflix.priam.utils.GsonJsonSerializer; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.time.Instant; +import java.time.temporal.ChronoUnit; +import java.util.ArrayList; +import java.util.Date; +import java.util.List; +import java.util.Optional; +import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response; +import mockit.Expectations; +import mockit.Mocked; +import org.joda.time.DateTime; +import org.junit.Before; +import org.junit.Test; + +public class BackupServletV2Test { + private IConfiguration config; + private @Mocked Restore restoreObj; + private @Mocked SnapshotMetaTask snapshotBackup; + private @Mocked BackupVerification backupVerification; + private @Mocked FileSnapshotStatusMgr backupStatusMgr; + private BackupServletV2 resource; + private RestoreServlet restoreResource; + private InstanceInfo instanceInfo; + private static final String backupDate = "201812011000"; + private static final Path location = + Paths.get( + "some_bucket/casstestbackup/1049_fake-app/1808575600", + AbstractBackupPath.BackupFileType.META_V2.toString(), + "1859817645000", + "SNAPPY", + "PLAINTEXT", + "meta_v2_201812011000.json"); + + @Before + public void setUp() { + Injector injector = Guice.createInjector(new BRTestModule()); + config = injector.getInstance(IConfiguration.class); + instanceInfo = injector.getInstance(InstanceInfo.class); + resource = injector.getInstance(BackupServletV2.class); + restoreResource = injector.getInstance(RestoreServlet.class); + } + + @Test + public void testBackup() throws Exception { + new Expectations() { + { + snapshotBackup.execute(); + } + }; + + Response response = resource.backup(); + assertEquals(200, response.getStatus()); + assertEquals("[\"ok\"]", response.getEntity()); + assertEquals( + MediaType.APPLICATION_JSON_TYPE, response.getMetadata().get("Content-Type").get(0)); + } + + @Test + public void testRestoreMinimal() throws Exception { + final String dateRange = null; + final String oldRegion = "us-east-1"; + new Expectations() { + { + instanceInfo.getRegion(); + result = oldRegion; + + restoreObj.restore(new DateUtil.DateRange((Instant) any, (Instant) any)); + } + }; + + expectCassandraStartup(); + + Response response = restoreResource.restore(dateRange); + assertEquals(200, response.getStatus()); + assertEquals("[\"ok\"]", response.getEntity()); + assertEquals( + MediaType.APPLICATION_JSON_TYPE, response.getMetadata().get("Content-Type").get(0)); + } + + @Test + public void testRestoreWithDateRange() throws Exception { + final String dateRange = "201101010000,201112312359"; + + new Expectations() { + + { + DateUtil.getDate(dateRange.split(",")[0]); + result = new DateTime(2011, 1, 1, 0, 0).toDate(); + times = 1; + DateUtil.getDate(dateRange.split(",")[1]); + result = new DateTime(2011, 12, 31, 23, 59).toDate(); + times = 1; + restoreObj.restore(new DateUtil.DateRange(dateRange)); + } + }; + + expectCassandraStartup(); + + Response response = restoreResource.restore(dateRange); + assertEquals(200, response.getStatus()); + assertEquals("[\"ok\"]", response.getEntity()); + assertEquals( + MediaType.APPLICATION_JSON_TYPE, response.getMetadata().get("Content-Type").get(0)); + } + + // TODO: create CassandraController interface and inject, instead of static util method + private void expectCassandraStartup() { + new Expectations() { + { + config.getCassStartupScript(); + result = "/usr/bin/false"; + config.getHeapNewSize(); + result = "2G"; + config.getHeapSize(); + result = "8G"; + config.getDataFileLocation(); + result = "/var/lib/cassandra/data"; + config.getCommitLogLocation(); + result = "/var/lib/cassandra/commitlog"; + config.getBackupLocation(); + result = "backup"; + config.getCacheLocation(); + result = "/var/lib/cassandra/saved_caches"; + config.getJmxPort(); + result = 7199; + config.getMaxDirectMemory(); + result = "50G"; + } + }; + } + + @Test + public void testValidate() throws Exception { + new Expectations() { + { + backupVerification.verifyBackup( + BackupVersion.SNAPSHOT_META_SERVICE, + anyBoolean, + new DateUtil.DateRange((Instant) any, (Instant) any)); + result = Optional.of(getBackupVerificationResult()); + } + }; + Response response = + resource.validateV2SnapshotByDate( + new DateUtil.DateRange(Instant.now(), Instant.now()).toString(), true); + assertEquals(200, response.getStatus()); + assertEquals( + GsonJsonSerializer.getGson().toJson(getBackupVerificationResult()), + response.getEntity().toString()); + } + + @Test + public void testValidateNoBackups() throws Exception { + new Expectations() { + { + backupVerification.verifyBackup( + BackupVersion.SNAPSHOT_META_SERVICE, + anyBoolean, + new DateUtil.DateRange((Instant) any, (Instant) any)); + result = Optional.empty(); + } + }; + Response response = + resource.validateV2SnapshotByDate( + new DateUtil.DateRange(Instant.now(), Instant.now()).toString(), true); + assertEquals(204, response.getStatus()); + } + + @Test + public void testListDateRange() throws Exception { + new Expectations() { + { + backupVerification.verifyBackup( + BackupVersion.SNAPSHOT_META_SERVICE, + anyBoolean, + new DateUtil.DateRange((Instant) any, (Instant) any)); + result = Optional.of(getBackupVerificationResult()); + } + }; + Response response = + resource.validateV2SnapshotByDate( + new DateUtil.DateRange(Instant.now(), Instant.now()).toString(), true); + assertEquals(200, response.getStatus()); + assertEquals( + GsonJsonSerializer.getGson().toJson(getBackupVerificationResult()), + response.getEntity().toString()); + } + + @Test + public void testListDateRangeNoBackups() throws Exception { + new Expectations() { + { + backupVerification.verifyBackup( + BackupVersion.SNAPSHOT_META_SERVICE, + anyBoolean, + new DateUtil.DateRange((Instant) any, (Instant) any)); + result = Optional.empty(); + } + }; + Response response = + resource.validateV2SnapshotByDate( + new DateUtil.DateRange(Instant.now(), Instant.now()).toString(), true); + assertEquals(204, response.getStatus()); + } + + @Test + public void testBackUpInfo() throws Exception { + List backupMetadataList = new ArrayList<>(); + backupMetadataList.add(getBackupMetaData()); + new Expectations() { + { + backupStatusMgr.getLatestBackupMetadata( + BackupVersion.SNAPSHOT_META_SERVICE, + new DateUtil.DateRange((Instant) any, (Instant) any)); + result = backupMetadataList; + } + }; + Response response = resource.info(backupDate); + assertEquals(200, response.getStatus()); + assertEquals( + GsonJsonSerializer.getGson().toJson(backupMetadataList), + response.getEntity().toString()); + } + + @Test + public void testBackUpInfoNoBackups() { + new Expectations() { + { + backupStatusMgr.getLatestBackupMetadata( + BackupVersion.SNAPSHOT_META_SERVICE, + new DateUtil.DateRange((Instant) any, (Instant) any)); + result = new ArrayList<>(); + } + }; + Response response = resource.info(backupDate); + assertEquals(200, response.getStatus()); + assertEquals( + GsonJsonSerializer.getGson().toJson(new ArrayList<>()), + response.getEntity().toString()); + } + + private static BackupVerificationResult getBackupVerificationResult() { + BackupVerificationResult result = new BackupVerificationResult(); + result.valid = true; + result.manifestAvailable = true; + result.remotePath = "some_random"; + result.filesMatched = 123; + result.snapshotInstant = Instant.EPOCH; + return result; + } + + private static BackupMetadata getBackupMetaData() throws Exception { + BackupMetadata backupMetadata = + new BackupMetadata( + BackupVersion.SNAPSHOT_META_SERVICE, + "123", + new Date(DateUtil.parseInstant(backupDate).toEpochMilli())); + backupMetadata.setCompleted( + new Date( + DateUtil.parseInstant(backupDate) + .plus(30, ChronoUnit.MINUTES) + .toEpochMilli())); + backupMetadata.setStatus(Status.FINISHED); + backupMetadata.setSnapshotLocation(location.toString()); + return backupMetadata; + } +} From 57b573ce7d4d534eb4aeb0a1b5cbb2817f198b02 Mon Sep 17 00:00:00 2001 From: Sreeram Chakrovorthy Date: Tue, 19 May 2020 00:44:53 -0700 Subject: [PATCH 129/228] CASS-1799 Attempting to address patch code coverage failure. --- .../priam/resources/BackupServletV2Test.java | 94 +++++++++++++++++-- 1 file changed, 85 insertions(+), 9 deletions(-) diff --git a/priam/src/test/java/com/netflix/priam/resources/BackupServletV2Test.java b/priam/src/test/java/com/netflix/priam/resources/BackupServletV2Test.java index e9f0d32b1..c4729aa71 100644 --- a/priam/src/test/java/com/netflix/priam/resources/BackupServletV2Test.java +++ b/priam/src/test/java/com/netflix/priam/resources/BackupServletV2Test.java @@ -4,7 +4,9 @@ import com.google.inject.Guice; import com.google.inject.Injector; +import com.google.inject.Provider; import com.netflix.priam.backup.*; +import com.netflix.priam.backupv2.MetaV2Proxy; import com.netflix.priam.backupv2.SnapshotMetaTask; import com.netflix.priam.config.IConfiguration; import com.netflix.priam.identity.config.InstanceInfo; @@ -13,6 +15,7 @@ import com.netflix.priam.utils.GsonJsonSerializer; import java.nio.file.Path; import java.nio.file.Paths; +import java.text.SimpleDateFormat; import java.time.Instant; import java.time.temporal.ChronoUnit; import java.util.ArrayList; @@ -33,6 +36,8 @@ public class BackupServletV2Test { private @Mocked SnapshotMetaTask snapshotBackup; private @Mocked BackupVerification backupVerification; private @Mocked FileSnapshotStatusMgr backupStatusMgr; + private @Mocked BackupRestoreUtil backupRestoreUtil; + private @Mocked MetaV2Proxy metaV2Proxy; private BackupServletV2 resource; private RestoreServlet restoreResource; private InstanceInfo instanceInfo; @@ -45,6 +50,8 @@ public class BackupServletV2Test { "SNAPPY", "PLAINTEXT", "meta_v2_201812011000.json"); + private static Provider pathProvider; + private static IConfiguration configuration; @Before public void setUp() { @@ -53,6 +60,8 @@ public void setUp() { instanceInfo = injector.getInstance(InstanceInfo.class); resource = injector.getInstance(BackupServletV2.class); restoreResource = injector.getInstance(RestoreServlet.class); + pathProvider = injector.getProvider(AbstractBackupPath.class); + configuration = injector.getInstance(IConfiguration.class); } @Test @@ -179,10 +188,12 @@ public void testValidateNoBackups() throws Exception { resource.validateV2SnapshotByDate( new DateUtil.DateRange(Instant.now(), Instant.now()).toString(), true); assertEquals(204, response.getStatus()); + assertEquals( + response.getEntity().toString(), "No valid meta found for provided time range"); } @Test - public void testListDateRange() throws Exception { + public void testValidateV2SnapshotByDate() throws Exception { new Expectations() { { backupVerification.verifyBackup( @@ -201,21 +212,46 @@ public void testListDateRange() throws Exception { response.getEntity().toString()); } + // @Test + // public void testListDateRange() throws Exception { + // Optional abstractBackupPath = getAbstractBackupPath(); + // String dateRange = String.format("%s,%s", + // new SimpleDateFormat("yyyymmddhhmm").format(new Date()) + // , new SimpleDateFormat("yyyymmddhhmm").format(new Date())); + // new Expectations() {{ + // backupRestoreUtil.getLatestValidMetaPath(metaV2Proxy, + // new DateUtil.DateRange((Instant) any, (Instant) any)); result = + // abstractBackupPath; + // + // backupRestoreUtil.getAllFiles( + // abstractBackupPath.get(), + // new DateUtil.DateRange((Instant) any, (Instant) any), metaV2Proxy, + // pathProvider); result = getBackupPathList(); + // }}; + // + // Response response = + // resource.list(dateRange); + // assertEquals(200, response.getStatus()); + // } + @Test public void testListDateRangeNoBackups() throws Exception { + String dateRange = + String.format( + "%s,%s", + new SimpleDateFormat("yyyymmdd").format(new Date()), + new SimpleDateFormat("yyyymmdd").format(new Date())); + new Expectations() { { - backupVerification.verifyBackup( - BackupVersion.SNAPSHOT_META_SERVICE, - anyBoolean, - new DateUtil.DateRange((Instant) any, (Instant) any)); + backupRestoreUtil.getLatestValidMetaPath( + metaV2Proxy, new DateUtil.DateRange((Instant) any, (Instant) any)); result = Optional.empty(); } }; - Response response = - resource.validateV2SnapshotByDate( - new DateUtil.DateRange(Instant.now(), Instant.now()).toString(), true); - assertEquals(204, response.getStatus()); + Response response = resource.list(dateRange); + assertEquals(200, response.getStatus()); + assertEquals(response.getEntity().toString(), "No valid meta found!"); } @Test @@ -279,4 +315,44 @@ private static BackupMetadata getBackupMetaData() throws Exception { backupMetadata.setSnapshotLocation(location.toString()); return backupMetadata; } + + private static Optional getAbstractBackupPath() throws Exception { + Path path = + Paths.get( + configuration.getDataFileLocation(), + "keyspace1", + "columnfamily1", + "backup", + "mc-1234-Data.db"); + AbstractBackupPath abstractBackupPath = pathProvider.get(); + abstractBackupPath.parseLocal(path.toFile(), AbstractBackupPath.BackupFileType.SST_V2); + return Optional.of(abstractBackupPath); + } + + private static List getBackupPathList() throws Exception { + List abstractBackupPathList = new ArrayList<>(); + Path path = + Paths.get( + configuration.getDataFileLocation(), + "keyspace1", + "columnfamily1", + "backup", + "mc-1234-Data.db"); + AbstractBackupPath abstractBackupPath1 = pathProvider.get(); + abstractBackupPath1.parseLocal(path.toFile(), AbstractBackupPath.BackupFileType.SST_V2); + abstractBackupPathList.add(abstractBackupPath1); + + path = + Paths.get( + configuration.getDataFileLocation(), + "keyspace1", + "columnfamily1", + "backup", + "mc-1234-Data.db"); + AbstractBackupPath abstractBackupPath2 = pathProvider.get(); + abstractBackupPath2.parseLocal( + path.toFile(), AbstractBackupPath.BackupFileType.SNAPSHOT_VERIFIED); + abstractBackupPathList.add(abstractBackupPath2); + return abstractBackupPathList; + } } From b6ed4d86bbda61293215b1760476ca519b5082ad Mon Sep 17 00:00:00 2001 From: Sreeram Chakrovorthy Date: Tue, 19 May 2020 13:10:37 -0700 Subject: [PATCH 130/228] CASS-1799 Fixing other Priam endpoints changelog related commit. --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 538794c18..f5206d99e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,7 @@ # Changelog +## 2020/05/19 3.11.62 +(#878) Fixing BackupServletV2 endpoints that were broken because of an underlying dependency change from the release 3.11.59. + ## 2020/05/18 3.11.61 This is a re-release of v3.11.60 that failed to be uploaded. From 1e6229e97d3c0a310156782494674a4d8f4188b2 Mon Sep 17 00:00:00 2001 From: Sumanth Pasupuleti Date: Tue, 26 May 2020 10:58:48 -0700 Subject: [PATCH 131/228] Changing cass log dir env variable to converge with Cassandra upstream (#884) Co-authored-by: Sumanth Pasupuleti --- .../com/netflix/priam/defaultimpl/CassandraProcessManager.java | 1 + 1 file changed, 1 insertion(+) diff --git a/priam/src/main/java/com/netflix/priam/defaultimpl/CassandraProcessManager.java b/priam/src/main/java/com/netflix/priam/defaultimpl/CassandraProcessManager.java index 3ffe3b507..3ef8d06a6 100644 --- a/priam/src/main/java/com/netflix/priam/defaultimpl/CassandraProcessManager.java +++ b/priam/src/main/java/com/netflix/priam/defaultimpl/CassandraProcessManager.java @@ -58,6 +58,7 @@ protected void setEnv(Map env) { env.put("LOCAL_JMX", config.enableRemoteJMX() ? "no" : "yes"); env.put("MAX_DIRECT_MEMORY", config.getMaxDirectMemory()); env.put("CASS_LOGS_DIR", config.getLogDirLocation()); + env.put("CASSANDRA_LOG_DIR", config.getLogDirLocation()); env.put("CASSANDRA_HOME", config.getCassHome()); } From ac54bc56f3804735d3d9151d32aeff771c311314 Mon Sep 17 00:00:00 2001 From: Joseph Lynch Date: Thu, 30 Aug 2018 20:54:46 -0700 Subject: [PATCH 132/228] Support Tuning arbitrary Property Files in 3.11 --- build.gradle | 1 + .../netflix/priam/config/IConfiguration.java | 11 ++++ .../priam/tuner/PropertiesFileTuner.java | 56 +++++++++++++++++ .../netflix/priam/tuner/StandardTuner.java | 11 +++- .../priam/config/FakeConfiguration.java | 20 +++++- .../priam/tuner/StandardTunerTest.java | 62 +++++++++++++++++-- .../conf/cassandra-rackdc.properties | 39 ++++++++++++ 7 files changed, 192 insertions(+), 8 deletions(-) create mode 100644 priam/src/main/java/com/netflix/priam/tuner/PropertiesFileTuner.java create mode 100644 priam/src/test/resources/conf/cassandra-rackdc.properties diff --git a/build.gradle b/build.gradle index af2997378..debb8740d 100644 --- a/build.gradle +++ b/build.gradle @@ -31,6 +31,7 @@ allprojects { dependencies { compile 'org.apache.commons:commons-lang3:3.8.1' + compile 'org.apache.commons:commons-text:1.8' compile 'commons-logging:commons-logging:1.2' compile 'org.apache.commons:commons-collections4:4.2' compile 'commons-io:commons-io:2.6' diff --git a/priam/src/main/java/com/netflix/priam/config/IConfiguration.java b/priam/src/main/java/com/netflix/priam/config/IConfiguration.java index 7c9819708..872353ef0 100644 --- a/priam/src/main/java/com/netflix/priam/config/IConfiguration.java +++ b/priam/src/main/java/com/netflix/priam/config/IConfiguration.java @@ -18,6 +18,7 @@ import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.collect.ImmutableSet; import com.google.inject.ImplementedBy; import com.netflix.priam.scheduler.UnsupportedTypeException; import com.netflix.priam.tuner.GCType; @@ -1080,6 +1081,16 @@ default String getMergedConfigurationDirectory() { return "/tmp/priam_configuration"; } + /** + * Return a list of property file paths from the configuration directory by Priam that should be + * tuned. + * + * @return the files paths + */ + default ImmutableSet getTunablePropertyFiles() { + return ImmutableSet.of(); + } + /** * Escape hatch for getting any arbitrary property by key This is useful so we don't have to * keep adding methods to this interface for every single configuration option ever. Also diff --git a/priam/src/main/java/com/netflix/priam/tuner/PropertiesFileTuner.java b/priam/src/main/java/com/netflix/priam/tuner/PropertiesFileTuner.java new file mode 100644 index 000000000..818816707 --- /dev/null +++ b/priam/src/main/java/com/netflix/priam/tuner/PropertiesFileTuner.java @@ -0,0 +1,56 @@ +package com.netflix.priam.tuner; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.base.Splitter; +import com.google.inject.Inject; +import com.netflix.priam.config.IConfiguration; +import java.io.FileReader; +import java.io.FileWriter; +import java.io.IOException; +import java.util.Map; +import org.apache.commons.configuration2.PropertiesConfiguration; +import org.apache.commons.configuration2.ex.ConfigurationException; +import org.apache.commons.io.FilenameUtils; +import org.apache.commons.text.StringSubstitutor; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Support tuning standard .properties files + * + *

    + */ +public class PropertiesFileTuner { + private static final Logger logger = LoggerFactory.getLogger(PropertiesFileTuner.class); + protected final IConfiguration config; + + @Inject + public PropertiesFileTuner(IConfiguration config) { + this.config = config; + } + + @SuppressWarnings("unchecked") + public void updateAndSaveProperties(String propertyFile) + throws IOException, ConfigurationException { + try { + PropertiesConfiguration properties = new PropertiesConfiguration(); + properties.getLayout().load(properties, new FileReader(propertyFile)); + String overrides = + config.getProperty( + "propertyOverrides." + FilenameUtils.getBaseName(propertyFile), null); + if (overrides != null) { + // Allow use of the IConfiguration object as template strings + Map map = new ObjectMapper().convertValue(config, Map.class); + String resolvedOverrides = new StringSubstitutor(map).replace(overrides); + Splitter.on(",") + .withKeyValueSeparator("=") + .split(resolvedOverrides) + .forEach(properties::setProperty); + } + properties.getLayout().save(properties, new FileWriter(propertyFile)); + } catch (IOException | ConfigurationException e) { + logger.error("Could not tune " + propertyFile + ". Does it exist? Is it writable?", e); + throw e; + } + } +} diff --git a/priam/src/main/java/com/netflix/priam/tuner/StandardTuner.java b/priam/src/main/java/com/netflix/priam/tuner/StandardTuner.java index 8a1f599f1..ac37b3ad0 100644 --- a/priam/src/main/java/com/netflix/priam/tuner/StandardTuner.java +++ b/priam/src/main/java/com/netflix/priam/tuner/StandardTuner.java @@ -51,6 +51,7 @@ public StandardTuner( this.instanceInfo = instanceInfo; } + @SuppressWarnings("unchecked") public void writeAllProperties(String yamlLocation, String hostname, String seedProvider) throws Exception { DumperOptions options = new DumperOptions(); @@ -149,7 +150,13 @@ public void writeAllProperties(String yamlLocation, String hostname, String seed logger.info(yaml.dump(map)); yaml.dump(map, new FileWriter(yamlFile)); + // TODO: port commit log backups to the PropertiesFileTuner implementation configureCommitLogBackups(); + + PropertiesFileTuner propertyTuner = new PropertiesFileTuner(config); + for (String propertyFile : config.getTunablePropertyFiles()) { + propertyTuner.updateAndSaveProperties(propertyFile); + } } /** @@ -212,7 +219,7 @@ protected void configfureSecurity(Map map) { serverEnc.put("internode_encryption", config.getInternodeEncryption()); } - protected void configureCommitLogBackups() throws IOException { + protected void configureCommitLogBackups() { if (!config.isBackingUpCommitLogs()) return; Properties props = new Properties(); props.put("archive_command", config.getCommitLogBackupArchiveCmd()); @@ -223,6 +230,8 @@ protected void configureCommitLogBackups() throws IOException { try (FileOutputStream fos = new FileOutputStream(new File(config.getCommitLogBackupPropsFile()))) { props.store(fos, "cassandra commit log archive props, as written by priam"); + } catch (IOException e) { + logger.error("Could not store commitlog_archiving.properties", e); } } diff --git a/priam/src/test/java/com/netflix/priam/config/FakeConfiguration.java b/priam/src/test/java/com/netflix/priam/config/FakeConfiguration.java index 3b34b2e8b..edc05bfac 100644 --- a/priam/src/test/java/com/netflix/priam/config/FakeConfiguration.java +++ b/priam/src/test/java/com/netflix/priam/config/FakeConfiguration.java @@ -17,6 +17,7 @@ package com.netflix.priam.config; +import com.google.common.collect.ImmutableSet; import com.google.inject.Singleton; import java.io.File; import java.util.Arrays; @@ -31,7 +32,7 @@ public class FakeConfiguration implements IConfiguration { private String restorePrefix = ""; public Map fakeConfig; - public final Map fakeProperties = new HashMap<>(); + public Map fakeProperties = new HashMap<>(); public FakeConfiguration() { this("my_fake_cluster"); @@ -129,8 +130,9 @@ public int getBackupRetentionDays() { return 5; } + @Override public String getYamlLocation() { - return "conf/cassandra.yaml"; + return getCassHome() + "/conf/cassandra.yaml"; } @Override @@ -175,4 +177,18 @@ public String getProperty(String key, String defaultValue) { public String getMergedConfigurationDirectory() { return fakeProperties.getOrDefault("priam_test_config", "/tmp/priam_test_config"); } + + @Override + public ImmutableSet getTunablePropertyFiles() { + String path = new File(getYamlLocation()).getParentFile().getPath(); + return ImmutableSet.of(path + "/cassandra-rackdc.properties"); + } + + public String getRAC() { + return "my_zone"; + } + + public String getDC() { + return "us-east-1"; + } } diff --git a/priam/src/test/java/com/netflix/priam/tuner/StandardTunerTest.java b/priam/src/test/java/com/netflix/priam/tuner/StandardTunerTest.java index ec4d8efb8..3cdf0b162 100644 --- a/priam/src/test/java/com/netflix/priam/tuner/StandardTunerTest.java +++ b/priam/src/test/java/com/netflix/priam/tuner/StandardTunerTest.java @@ -20,15 +20,21 @@ import com.google.common.io.Files; import com.google.inject.Guice; +import com.google.inject.Injector; import com.netflix.priam.backup.BRTestModule; import com.netflix.priam.config.BackupRestoreConfig; import com.netflix.priam.config.FakeConfiguration; import com.netflix.priam.config.IBackupRestoreConfig; +import com.netflix.priam.config.IConfiguration; import com.netflix.priam.identity.config.InstanceInfo; import java.io.File; import java.io.FileInputStream; +import java.io.FileReader; +import java.nio.charset.Charset; +import java.nio.file.Paths; import java.util.HashMap; import java.util.Map; +import java.util.Properties; import org.apache.commons.io.FileUtils; import org.junit.After; import org.junit.Assert; @@ -48,13 +54,16 @@ public class StandardTunerTest { private final InstanceInfo instanceInfo; private final IBackupRestoreConfig backupRestoreConfig; private final File target = new File("/tmp/priam_test.yaml"); + private IConfiguration config; public StandardTunerTest() { - this.tuner = Guice.createInjector(new BRTestModule()).getInstance(StandardTuner.class); - this.instanceInfo = - Guice.createInjector(new BRTestModule()).getInstance(InstanceInfo.class); - this.backupRestoreConfig = - Guice.createInjector(new BRTestModule()).getInstance(BackupRestoreConfig.class); + Injector injector = Guice.createInjector(new BRTestModule()); + this.tuner = injector.getInstance(StandardTuner.class); + this.instanceInfo = injector.getInstance(InstanceInfo.class); + this.backupRestoreConfig = injector.getInstance(BackupRestoreConfig.class); + this.config = injector.getInstance(IConfiguration.class); + File targetDir = new File(config.getYamlLocation()).getParentFile(); + if (!targetDir.exists()) targetDir.mkdirs(); } @Test @@ -172,4 +181,47 @@ public String getExtraConfigParams() { return extraConfigParams; } } + + @Test + public void testPropertiesFiles() throws Exception { + FakeConfiguration fake = (FakeConfiguration) config; + File testRackDcFile = new File("src/test/resources/conf/cassandra-rackdc.properties"); + File testYamlFile = new File("src/main/resources/incr-restore-cassandra.yaml"); + String propertiesPath = new File(config.getYamlLocation()).getParentFile().getPath(); + File rackDcFile = + new File( + Paths.get(propertiesPath, "cassandra-rackdc.properties") + .normalize() + .toString()); + File configFile = + new File(Paths.get(propertiesPath, "properties_test.yaml").normalize().toString()); + System.out.println(testRackDcFile); + System.out.println(rackDcFile); + Files.copy(testRackDcFile, rackDcFile); + Files.copy(testYamlFile, configFile); + + try { + fake.fakeProperties.put( + "propertyOverrides.cassandra-rackdc", + "dc=${dc},rack=${rac},ec2_naming_scheme=legacy,dc_suffix=testsuffix"); + + tuner.writeAllProperties(configFile.getPath(), "your_host", "YourSeedProvider"); + Properties prop = new Properties(); + prop.load(new FileReader(rackDcFile)); + assertEquals("us-east-1", prop.getProperty("dc")); + assertEquals("my_zone", prop.getProperty("rack")); + assertEquals("legacy", prop.getProperty("ec2_naming_scheme")); + assertEquals("testsuffix", prop.getProperty("dc_suffix")); + + assertEquals(4, prop.stringPropertyNames().size()); + } finally { + fake.fakeProperties.clear(); + for (String line : Files.readLines(rackDcFile, Charset.defaultCharset())) { + System.out.println(line); + } + + Files.copy(testRackDcFile, rackDcFile); + Files.copy(testYamlFile, configFile); + } + } } diff --git a/priam/src/test/resources/conf/cassandra-rackdc.properties b/priam/src/test/resources/conf/cassandra-rackdc.properties new file mode 100644 index 000000000..26b9b9de5 --- /dev/null +++ b/priam/src/test/resources/conf/cassandra-rackdc.properties @@ -0,0 +1,39 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# These properties are used with GossipingPropertyFileSnitch and will +# indicate the rack and dc for this node +dc=dc1 +rack=rack1 + +# Add a suffix to a datacenter name. Used by the Ec2Snitch and Ec2MultiRegionSnitch +# to append a string to the EC2 region name. +#dc_suffix= + +# Uncomment the following line to make this snitch prefer the internal ip when possible, as the Ec2MultiRegionSnitch does. +# prefer_local=true + +# Datacenter and rack naming convention used by the Ec2Snitch and Ec2MultiRegionSnitch. +# Options are: +# legacy : datacenter name is the part of the availability zone name preceding the last "-" +# when the zone ends in -1 and includes the number if not -1. Rack is the portion of +# the availability zone name following the last "-". +# Examples: us-west-1a => dc: us-west, rack: 1a; us-west-2b => dc: us-west-2, rack: 2b; +# YOU MUST USE THIS VALUE IF YOU ARE UPGRADING A PRE-4.0 CLUSTER +# standard : Default value. datacenter name is the standard AWS region name, including the number. +# rack name is the region plus the availability zone letter. +# Examples: us-west-1a => dc: us-west-1, rack: us-west-1a; us-west-2b => dc: us-west-2, rack: us-west-2b; +# ec2_naming_scheme=standard From 1950bdc5a71d217d17b56bfb2df1f5ea7ce7c249 Mon Sep 17 00:00:00 2001 From: Matt Lehman Date: Wed, 3 Jun 2020 09:39:11 -0700 Subject: [PATCH 133/228] Update CHANGELOG for release 3.11.63 --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index f5206d99e..4a02a2267 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,7 @@ # Changelog +## 2020/06/03 3.11.63 +(#881) Ported PropertiesFileTuner to the 3.11 branch. + ## 2020/05/19 3.11.62 (#878) Fixing BackupServletV2 endpoints that were broken because of an underlying dependency change from the release 3.11.59. From ee67bba85f52279812da214bc9edbf9002364c78 Mon Sep 17 00:00:00 2001 From: Matt Lehman Date: Wed, 3 Jun 2020 10:33:57 -0700 Subject: [PATCH 134/228] Get CHANGELOG caught up to 3.11.66 --- CHANGELOG.md | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4a02a2267..c34338be5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,15 @@ # Changelog -## 2020/06/03 3.11.63 -(#881) Ported PropertiesFileTuner to the 3.11 branch. +## 2020/06/03 3.11.66 +(#881) Portng PropertiesFileTuner to the 3.11 branch. + +## 2020/05/26 3.11.65 +(#884) Adding support for upstream C* log directory env variable. + +## 2020/05/19 3.11.64 +Re-re-releasing 3.11.62 + +## 2020/05/19 3.11.63 +Re-releasing 3.11.62 ## 2020/05/19 3.11.62 (#878) Fixing BackupServletV2 endpoints that were broken because of an underlying dependency change from the release 3.11.59. From 2e03ca1f322f91d50d13048ac8618cbffcbcc51d Mon Sep 17 00:00:00 2001 From: Matt Lehman Date: Thu, 4 Jun 2020 07:08:04 -0700 Subject: [PATCH 135/228] Update CHANGELOG for 3.11.67 --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index c34338be5..5978fb2f2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,7 @@ # Changelog +## 2020/06/04 3.11.67 +Re-releasing 3.11.66 + ## 2020/06/03 3.11.66 (#881) Portng PropertiesFileTuner to the 3.11 branch. From a0864f6497f744137c22f99e98188c79ddbf821b Mon Sep 17 00:00:00 2001 From: Sreeram Chakrovorthy Date: Fri, 19 Jun 2020 11:25:01 -0700 Subject: [PATCH 136/228] CASS-1752 First pass of changes to throw an exception when a node is bootstrapping to an existing token. Per current thoughts this change is going to held as a PR and only committed after the root cause is addressed in Cassandra if any. --- .../identity/token/TokenRetrieverUtils.java | 27 ++++++++++++++++--- .../token/DeadTokenRetrieverTest.java | 3 +-- .../token/TokenRetrieverUtilsTest.java | 9 ++++--- 3 files changed, 29 insertions(+), 10 deletions(-) diff --git a/priam/src/main/java/com/netflix/priam/identity/token/TokenRetrieverUtils.java b/priam/src/main/java/com/netflix/priam/identity/token/TokenRetrieverUtils.java index 7011729c4..4367706ba 100644 --- a/priam/src/main/java/com/netflix/priam/identity/token/TokenRetrieverUtils.java +++ b/priam/src/main/java/com/netflix/priam/identity/token/TokenRetrieverUtils.java @@ -35,7 +35,7 @@ public class TokenRetrieverUtils { */ public static String inferTokenOwnerFromGossip( List allIds, String token, String dc) - throws GossipParseException { + throws GossipParseException, TokenAliveException { // Avoid using dead instance who we are trying to replace (duh!!) // Avoid other regions instances to avoid communication over public ip address. @@ -107,7 +107,8 @@ public static String inferTokenOwnerFromGossip( } // helper method to get the token owner IP from a Cassandra node. - private static String getIp(String host, String token) throws GossipParseException { + private static String getIp(String host, String token) + throws GossipParseException, TokenAliveException { String response = null; try { response = SystemUtils.getDataFromUrl(String.format(STATUS_URL_FORMAT, host)); @@ -119,8 +120,8 @@ private static String getIp(String host, String token) throws GossipParseExcepti // place to start. // We just verify that the endpoint we provide is not "live". if (liveNodes.contains(endpointInfo)) { - logger.warn("The token [{}] is considered as alive by [{}].", token, host); - return null; + throw new TokenAliveException( + String.format("The token %s is considered as alive by %s.", token, host)); } return endpointInfo; @@ -155,4 +156,22 @@ public GossipParseException(String message, Throwable t) { super(message, t); } } + + /** This exception is thrown either when a node is bootstrapping using a token that is alive. */ + public static class TokenAliveException extends Exception { + + private static final long serialVersionUID = 1038678311186020257L; + + public TokenAliveException() { + super(); + } + + public TokenAliveException(String message) { + super(message); + } + + public TokenAliveException(String message, Throwable t) { + super(message, t); + } + } } diff --git a/priam/src/test/java/com/netflix/priam/identity/token/DeadTokenRetrieverTest.java b/priam/src/test/java/com/netflix/priam/identity/token/DeadTokenRetrieverTest.java index 1c9ac7ab3..ed346e930 100644 --- a/priam/src/test/java/com/netflix/priam/identity/token/DeadTokenRetrieverTest.java +++ b/priam/src/test/java/com/netflix/priam/identity/token/DeadTokenRetrieverTest.java @@ -109,7 +109,7 @@ public void testNoReplacementNoSpotAvailable() throws Exception { }; } - @Test + @Test(expected = TokenRetrieverUtils.TokenAliveException.class) // There is a potential slot for dead token but we are unable to replace. public void testNoReplacementNoGossipMatch(@Mocked SystemUtils systemUtils) throws Exception { List allInstances = getInstances(2); @@ -132,7 +132,6 @@ public void testNoReplacementNoGossipMatch(@Mocked SystemUtils systemUtils) thro new DeadTokenRetriever( factory, membership, configuration, new FakeSleeper(), instanceInfo); PriamInstance priamInstance = deadTokenRetriever.get(); - Assert.assertNull(priamInstance); new Verifications() { { factory.delete(withAny(instance)); diff --git a/priam/src/test/java/com/netflix/priam/identity/token/TokenRetrieverUtilsTest.java b/priam/src/test/java/com/netflix/priam/identity/token/TokenRetrieverUtilsTest.java index f78a66250..fd406e151 100644 --- a/priam/src/test/java/com/netflix/priam/identity/token/TokenRetrieverUtilsTest.java +++ b/priam/src/test/java/com/netflix/priam/identity/token/TokenRetrieverUtilsTest.java @@ -66,7 +66,7 @@ public void testRetrieveTokenOwnerWhenGossipAgrees(@Mocked SystemUtils systemUti Assert.assertEquals("127.0.0.4", replaceIp); } - @Test + @Test(expected = TokenRetrieverUtils.TokenAliveException.class) public void testRetrieveTokenOwnerWhenGossipDisagrees(@Mocked SystemUtils systemUtils) throws Exception { @@ -102,10 +102,9 @@ public void testRetrieveTokenOwnerWhenGossipDisagrees(@Mocked SystemUtils system }; String replaceIp = TokenRetrieverUtils.inferTokenOwnerFromGossip(instances, "4", "us-east"); - Assert.assertEquals(null, replaceIp); } - @Test + @Test(expected = TokenRetrieverUtils.TokenAliveException.class) public void testRetrieveTokenOwnerWhenAllHostsInGossipReturnsNull( @Mocked SystemUtils systemUtils) throws Exception { new Expectations() { @@ -121,7 +120,9 @@ public void testRetrieveTokenOwnerWhenAllHostsInGossipReturnsNull( @Test(expected = TokenRetrieverUtils.GossipParseException.class) public void testRetrieveTokenOwnerWhenAllInstancesThrowGossipParseException( - @Mocked SystemUtils systemUtils) throws TokenRetrieverUtils.GossipParseException { + @Mocked SystemUtils systemUtils) + throws TokenRetrieverUtils.GossipParseException, + TokenRetrieverUtils.TokenAliveException { new Expectations() { { From e177c27929ff2f66cdc19ed9b7616cf6c2e69b40 Mon Sep 17 00:00:00 2001 From: Sreeram Chakrovorthy Date: Thu, 2 Jul 2020 16:41:02 -0700 Subject: [PATCH 137/228] CASS-1752 Changelog related commit for PR #891. --- CHANGELOG.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5978fb2f2..ac49da3fb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,9 +1,12 @@ # Changelog +## 2020/07/02/ 3.11.68 +(#891) Adding an exception in the replace-ip path when a node attempts to bootstrap to an existing token because of a stale state. + ## 2020/06/04 3.11.67 Re-releasing 3.11.66 ## 2020/06/03 3.11.66 -(#881) Portng PropertiesFileTuner to the 3.11 branch. +(#881) Porting PropertiesFileTuner to the 3.11 branch. ## 2020/05/26 3.11.65 (#884) Adding support for upstream C* log directory env variable. From 90e66ece54b8f6f17115d01175ebb09496a009b3 Mon Sep 17 00:00:00 2001 From: Matt Lehman Date: Wed, 15 Jul 2020 11:05:37 -0700 Subject: [PATCH 138/228] Make BackupVerificationTask log and emit when there is no verified backup within SLO. Cease requiring the backup to be fully in S3. Plus tidying tweaks. --- build.gradle | 4 +- .../backupv2/BackupVerificationTask.java | 47 ++--- .../backupv2/TestBackupVerificationTask.java | 170 ++++++++---------- 3 files changed, 92 insertions(+), 129 deletions(-) diff --git a/build.gradle b/build.gradle index debb8740d..c3f6b6b02 100644 --- a/build.gradle +++ b/build.gradle @@ -80,7 +80,9 @@ allprojects { compileOnly 'javax.servlet:javax.servlet-api:3.1.0' testCompile 'org.jmockit:jmockit:1.31' testCompile "org.spockframework:spock-core:1.1-groovy-2.4" - testCompile 'junit:junit:4.12' + testCompile "com.google.truth:truth:1.0.1" + testImplementation 'org.junit.jupiter:junit-jupiter-api:5.3.1' + testRuntimeOnly 'org.junit.jupiter:junit-jupiter-engine:5.3.1' } sourceCompatibility = JavaVersion.VERSION_1_8 diff --git a/priam/src/main/java/com/netflix/priam/backupv2/BackupVerificationTask.java b/priam/src/main/java/com/netflix/priam/backupv2/BackupVerificationTask.java index 7589b8992..926b8513c 100644 --- a/priam/src/main/java/com/netflix/priam/backupv2/BackupVerificationTask.java +++ b/priam/src/main/java/com/netflix/priam/backupv2/BackupVerificationTask.java @@ -30,6 +30,7 @@ import com.netflix.priam.scheduler.TaskTimer; import com.netflix.priam.utils.DateUtil; import com.netflix.priam.utils.DateUtil.DateRange; +import java.time.Instant; import java.time.temporal.ChronoUnit; import java.util.List; import org.slf4j.Logger; @@ -66,45 +67,37 @@ public BackupVerificationTask( @Override public void execute() throws Exception { // Ensure that backup version 2.0 is actually enabled. - if (backupRestoreConfig.getSnapshotMetaServiceCronExpression().equalsIgnoreCase("-1")) { - logger.info( - "Not executing the Verification Service for backups as V2 backups are not enabled."); + if (backupRestoreConfig.getSnapshotMetaServiceCronExpression().equals("-1")) { + logger.info("Skipping backup verification. V2 backups are not enabled."); return; } if (instanceState.getRestoreStatus() != null && instanceState.getRestoreStatus().getStatus() != null && instanceState.getRestoreStatus().getStatus() == Status.STARTED) { - logger.info( - "Not executing the Verification Service for backups as Priam is in restore mode."); + logger.info("Skipping backup verification. Priam is in restore mode."); return; } // Validate the backup done in last x hours. - DateRange dateRange = - new DateRange( - DateUtil.getInstant() - .minus( - backupRestoreConfig.getBackupVerificationSLOInHours(), - ChronoUnit.HOURS), - DateUtil.getInstant()); + Instant now = DateUtil.getInstant(); + Instant slo = + now.minus(backupRestoreConfig.getBackupVerificationSLOInHours(), ChronoUnit.HOURS); + DateRange dateRange = new DateRange(slo, now); List verificationResults = backupVerification.verifyAllBackups(BackupVersion.SNAPSHOT_META_SERVICE, dateRange); - verificationResults - .stream() - .forEach( - backupVerificationResult -> { - logger.info( - "Sending {} message for backup: {}", - AbstractBackupPath.BackupFileType.SNAPSHOT_VERIFIED, - backupVerificationResult.snapshotInstant); - backupNotificationMgr.notify(backupVerificationResult); - }); + verificationResults.forEach( + result -> { + logger.info( + "Sending {} message for backup: {}", + AbstractBackupPath.BackupFileType.SNAPSHOT_VERIFIED, + result.snapshotInstant); + backupNotificationMgr.notify(result); + }); - if (!BackupRestoreUtil.getLatestValidMetaPath( - backupVerification.getMetaProxy(BackupVersion.SNAPSHOT_META_SERVICE), - dateRange) + if (!backupVerification + .verifyBackup(BackupVersion.SNAPSHOT_META_SERVICE, false /* force */, dateRange) .isPresent()) { logger.error( "Not able to find any snapshot which is valid in our SLO window: {} hours", @@ -113,10 +106,6 @@ public void execute() throws Exception { } } - public BackupMetrics getBackupMetrics() { - return backupMetrics; - } - /** * Interval between trying to verify data manifest file on Remote file system. * diff --git a/priam/src/test/java/com/netflix/priam/backupv2/TestBackupVerificationTask.java b/priam/src/test/java/com/netflix/priam/backupv2/TestBackupVerificationTask.java index 0d6244c3e..390307eca 100644 --- a/priam/src/test/java/com/netflix/priam/backupv2/TestBackupVerificationTask.java +++ b/priam/src/test/java/com/netflix/priam/backupv2/TestBackupVerificationTask.java @@ -17,179 +17,151 @@ package com.netflix.priam.backupv2; +import com.google.common.collect.ImmutableList; +import com.google.common.truth.Truth; import com.google.inject.Guice; import com.google.inject.Injector; import com.netflix.priam.backup.*; -import com.netflix.priam.config.IConfiguration; import com.netflix.priam.health.InstanceState; -import com.netflix.priam.merics.BackupMetrics; +import com.netflix.priam.merics.Metrics; import com.netflix.priam.notification.BackupNotificationMgr; import com.netflix.priam.scheduler.UnsupportedTypeException; import com.netflix.priam.utils.DateUtil.DateRange; +import com.netflix.spectator.api.Counter; +import com.netflix.spectator.api.Registry; import java.time.Instant; -import java.util.ArrayList; import java.util.List; import java.util.Optional; +import javax.inject.Inject; import mockit.*; -import org.junit.Assert; +import org.junit.Before; import org.junit.Test; +import org.junit.jupiter.api.Assertions; /** Created by aagrawal on 2/1/19. */ public class TestBackupVerificationTask { - private static BackupVerificationTask backupVerificationService; - private static IConfiguration configuration; - private static BackupVerification backupVerification; - private static BackupNotificationMgr backupNotificationMgr; + @Inject private BackupVerificationTask backupVerificationService; + private Counter badVerifications; + @Mocked private BackupVerification backupVerification; + @Mocked private BackupNotificationMgr backupNotificationMgr; - public TestBackupVerificationTask() { + @Before + public void setUp() { new MockBackupVerification(); new MockBackupNotificationMgr(); Injector injector = Guice.createInjector(new BRTestModule()); - if (configuration == null) configuration = injector.getInstance(IConfiguration.class); - if (backupVerificationService == null) - backupVerificationService = injector.getInstance(BackupVerificationTask.class); + injector.injectMembers(this); + badVerifications = + injector.getInstance(Registry.class) + .counter(Metrics.METRIC_PREFIX + "backup.verification.failure"); } - static class MockBackupVerification extends MockUp { - public static boolean emptyBackupVerificationList = false; - public static boolean throwError = false; - public static boolean validBackupVerificationResult = true; + private static final class MockBackupVerification extends MockUp { + private static boolean throwError; + private static ImmutableList results; + + public static void setResults(BackupVerificationResult... newResults) { + results = ImmutableList.copyOf(newResults); + } + + public static void shouldThrow(boolean newThrowError) { + throwError = newThrowError; + } @Mock public List verifyAllBackups( BackupVersion backupVersion, DateRange dateRange) throws UnsupportedTypeException, IllegalArgumentException { if (throwError) throw new IllegalArgumentException("DummyError"); - - if (emptyBackupVerificationList) { - return new ArrayList<>(); - } - List result = new ArrayList<>(); - if (validBackupVerificationResult) { - result.add(getValidBackupVerificationResult()); - } else { - result.add(getInvalidBackupVerificationResult()); - } - return result; + return results; } - } - static class MockBackupNotificationMgr extends MockUp { @Mock - public void notify(BackupVerificationResult backupVerificationResult) { - // do nothing just return - return; + public Optional verifyBackup( + BackupVersion backupVersion, boolean force, DateRange dateRange) + throws UnsupportedTypeException, IllegalArgumentException { + if (throwError) throw new IllegalArgumentException("DummyError"); + return results.isEmpty() ? Optional.empty() : Optional.of(results.get(0)); } } + private static final class MockBackupNotificationMgr extends MockUp {} + @Test - public void throwError() throws Exception { - MockBackupVerification.throwError = true; - MockBackupVerification.emptyBackupVerificationList = false; - try { - backupVerificationService.execute(); - Assert.assertTrue(false); - } catch (IllegalArgumentException e) { - if (!e.getMessage().equalsIgnoreCase("DummyError")) Assert.assertTrue(false); - } + public void throwError() { + MockBackupVerification.shouldThrow(true); + Assertions.assertThrows( + IllegalArgumentException.class, () -> backupVerificationService.execute()); } @Test - public void normalOperation() throws Exception { - MockBackupVerification.throwError = false; - MockBackupVerification.emptyBackupVerificationList = false; - new Expectations() { - { - backupVerificationService.getBackupMetrics().incrementBackupVerificationFailure(); - maxTimes = 0; - } - }; + public void validBackups() throws Exception { + MockBackupVerification.shouldThrow(false); + MockBackupVerification.setResults(getValidBackupVerificationResult()); backupVerificationService.execute(); + Truth.assertThat(badVerifications.count()).isEqualTo(0); new Verifications() { { - backupVerificationService.getBackupMetrics().incrementBackupVerificationFailure(); - maxTimes = 0; + backupNotificationMgr.notify((BackupVerificationResult) any); + times = 1; } }; } @Test - public void normalOperationPriorVerifiedBackups( - @Mocked BackupRestoreUtil backupRestoreUtil, - @Mocked AbstractBackupPath remoteBackupPath) - throws Exception { - MockBackupVerification.throwError = false; - MockBackupVerification.emptyBackupVerificationList = true; - new Expectations() { - { - backupRestoreUtil.getLatestValidMetaPath((IMetaProxy) any, (DateRange) any); - result = Optional.of(remoteBackupPath); - maxTimes = 1; - } - }; + public void invalidBackups() throws Exception { + MockBackupVerification.shouldThrow(false); + MockBackupVerification.setResults(getInvalidBackupVerificationResult()); backupVerificationService.execute(); + Truth.assertThat(badVerifications.count()).isEqualTo(0); new Verifications() { { - backupVerificationService.getBackupMetrics().incrementBackupVerificationFailure(); - maxTimes = 1; + backupNotificationMgr.notify((BackupVerificationResult) any); + times = 1; } }; } @Test - public void normalOperationInvalidBackups() throws Exception { - MockBackupVerification.throwError = false; - MockBackupVerification.emptyBackupVerificationList = false; - MockBackupVerification.validBackupVerificationResult = false; - new Expectations() { - { - backupVerificationService.getBackupMetrics().incrementBackupVerificationFailure(); - maxTimes = 0; - } - }; + public void noBackups() throws Exception { + MockBackupVerification.shouldThrow(false); + MockBackupVerification.setResults(); backupVerificationService.execute(); + Truth.assertThat(badVerifications.count()).isEqualTo(1); new Verifications() { { - backupVerificationService.getBackupMetrics().incrementBackupVerificationFailure(); + backupNotificationMgr.notify((BackupVerificationResult) any); maxTimes = 0; } }; } @Test - public void failCalls() throws Exception { - MockBackupVerification.throwError = false; - MockBackupVerification.emptyBackupVerificationList = true; + public void testRestoreMode(@Mocked InstanceState state) throws Exception { new Expectations() { { - backupVerificationService.getBackupMetrics().incrementBackupVerificationFailure(); - maxTimes = 1; + state.getRestoreStatus().getStatus(); + result = Status.STARTED; } }; backupVerificationService.execute(); + Truth.assertThat(badVerifications.count()).isEqualTo(0); new Verifications() { { - backupVerificationService.getBackupMetrics().incrementBackupVerificationFailure(); - maxTimes = 1; + backupVerification.verifyBackup((BackupVersion) any, anyBoolean, (DateRange) any); + maxTimes = 0; } - }; - } - @Test - public void testRestoreMode(@Mocked InstanceState state) throws Exception { - new Expectations() { { - state.getRestoreStatus().getStatus(); - result = Status.STARTED; + backupVerification.verifyAllBackups((BackupVersion) any, (DateRange) any); + maxTimes = 0; } - }; - backupVerificationService.execute(); - } - @Test - public void testGetBackupMetrics() { - BackupMetrics backupMetrics = backupVerificationService.getBackupMetrics(); - Assert.assertTrue(backupMetrics != null); + { + backupNotificationMgr.notify((BackupVerificationResult) any); + maxTimes = 0; + } + }; } private static BackupVerificationResult getInvalidBackupVerificationResult() { From 14e93f08c7df8711f65d187a5bb7a24a45f1629a Mon Sep 17 00:00:00 2001 From: Arun Agrawal Date: Wed, 15 Jul 2020 15:58:41 -0700 Subject: [PATCH 139/228] Fix the inferTokenOwnership method. Expose the results out so caller identity can make decision accordingly. --- .../priam/identity/InstanceIdentity.java | 36 +++-- .../identity/token/DeadTokenRetriever.java | 65 +++++--- .../identity/token/TokenRetrieverUtils.java | 146 ++++++++++++------ .../token/AssignedTokenRetrieverTest.java | 26 +++- .../token/DeadTokenRetrieverTest.java | 3 +- .../token/TokenRetrieverUtilsTest.java | 47 +++--- 6 files changed, 215 insertions(+), 108 deletions(-) diff --git a/priam/src/main/java/com/netflix/priam/identity/InstanceIdentity.java b/priam/src/main/java/com/netflix/priam/identity/InstanceIdentity.java index ab4ad880c..59df9cca9 100644 --- a/priam/src/main/java/com/netflix/priam/identity/InstanceIdentity.java +++ b/priam/src/main/java/com/netflix/priam/identity/InstanceIdentity.java @@ -29,7 +29,6 @@ import com.netflix.priam.identity.token.INewTokenRetriever; import com.netflix.priam.identity.token.IPreGeneratedTokenRetriever; import com.netflix.priam.identity.token.TokenRetrieverUtils; -import com.netflix.priam.identity.token.TokenRetrieverUtils.GossipParseException; import com.netflix.priam.utils.ITokenManager; import com.netflix.priam.utils.RetryableCallable; import com.netflix.priam.utils.Sleeper; @@ -39,7 +38,6 @@ import java.util.LinkedList; import java.util.List; import java.util.Optional; -import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -182,18 +180,28 @@ public PriamInstance retriableCall() throws Exception { // Priam might have crashed before bootstrapping Cassandra in replace mode. // So, it is premature to use the assigned token without checking Cassandra // gossip. - try { - String replaceIp = - TokenRetrieverUtils.inferTokenOwnerFromGossip( - aliveInstances, instance.getToken(), instance.getDC()); - if (!StringUtils.isEmpty(replaceIp) - && !replaceIp.equals(instance.getHostIP())) { - setReplacedIp(replaceIp); - logger.info( - "Priam found that the token is not alive according to Cassandra and we should start Cassandra in replace mode with replace ip: " - + replaceIp); - } - } catch (GossipParseException e) { + + // Infer current ownership information from other instances using gossip. + TokenRetrieverUtils.InferredTokenOwnership inferredTokenInformation = + TokenRetrieverUtils.inferTokenOwnerFromGossip( + aliveInstances, instance.getToken(), instance.getDC()); + String inferredIp = + (inferredTokenInformation.getTokenInformation() == null) + ? null + : inferredTokenInformation + .getTokenInformation() + .getIpAddress(); + // if unreachable rely on token database. + // if mismatch rely on token database. + if (inferredTokenInformation.getTokenInformationStatus() + == TokenRetrieverUtils.InferredTokenOwnership + .TokenInformationStatus.GOOD + && !inferredIp.equalsIgnoreCase(instance.getHostIP()) + && !inferredTokenInformation.getTokenInformation().isLive()) { + setReplacedIp(inferredIp); + logger.info( + "Priam found that the token is not alive according to Cassandra and we should start Cassandra in replace mode with replace ip: " + + inferredIp); } } } diff --git a/priam/src/main/java/com/netflix/priam/identity/token/DeadTokenRetriever.java b/priam/src/main/java/com/netflix/priam/identity/token/DeadTokenRetriever.java index 784a46aec..918d214f2 100755 --- a/priam/src/main/java/com/netflix/priam/identity/token/DeadTokenRetriever.java +++ b/priam/src/main/java/com/netflix/priam/identity/token/DeadTokenRetriever.java @@ -112,27 +112,50 @@ public PriamInstance get() throws Exception { factory.delete(priamInstance); // find the replaced IP - try { - replacedIp = - TokenRetrieverUtils.inferTokenOwnerFromGossip( - allInstancesWithinCluster, - priamInstance.getToken(), - priamInstance.getDC()); - - // Lets not replace the instance if gossip info is not merging!! - if (replacedIp == null) return null; - logger.info( - "Will try to replace token: {} with replacedIp (from gossip info): {} instead of ip from Token database: {}", - priamInstance.getToken(), - replacedIp, - priamInstance.getHostIP()); - } catch (TokenRetrieverUtils.GossipParseException e) { - // In case of gossip exception, fallback to IP in token database. - this.replacedIp = priamInstance.getHostIP(); - logger.info( - "Will try to replace token: {} with replacedIp from Token database: {}", - priamInstance.getToken(), - priamInstance.getHostIP()); + + // Infer current ownership information from other instances using gossip. + TokenRetrieverUtils.InferredTokenOwnership inferredTokenInformation = + TokenRetrieverUtils.inferTokenOwnerFromGossip( + allInstancesWithinCluster, + priamInstance.getToken(), + priamInstance.getDC()); + + switch (inferredTokenInformation.getTokenInformationStatus()) { + case GOOD: + if (inferredTokenInformation.getTokenInformation() == null) { + logger.error( + "If you see this message, it should not have happened. We expect token ownership information if all nodes agree. This is a code bounty issue."); + return null; + } + // Everyone agreed to a value. Check if it is live node. + if (inferredTokenInformation.getTokenInformation().isLive()) { + logger.info( + "This token is considered alive unanimously! We will not replace this instance."); + return null; + } else + this.replacedIp = + inferredTokenInformation.getTokenInformation().getIpAddress(); + break; + case UNREACHABLE_NODES: + // In case of unable to reach sufficient nodes, fallback to IP in token + // database. This could be a genuine case of say missing security permissions. + this.replacedIp = priamInstance.getHostIP(); + logger.warn( + "Unable to reach sufficient nodes. Please check security group permissions or there might be a network partition."); + logger.info( + "Will try to replace token: {} with replacedIp from Token database: {}", + priamInstance.getToken(), + priamInstance.getHostIP()); + break; + case MISMATCH: + // Lets not replace the instance if gossip info is not merging!! + logger.info( + "Mismatch in gossip. We will not replace this instance, until gossip settles down."); + return null; + default: + throw new IllegalStateException( + "Unexpected value: " + + inferredTokenInformation.getTokenInformationStatus()); } PriamInstance result; diff --git a/priam/src/main/java/com/netflix/priam/identity/token/TokenRetrieverUtils.java b/priam/src/main/java/com/netflix/priam/identity/token/TokenRetrieverUtils.java index 4367706ba..700f3ef92 100644 --- a/priam/src/main/java/com/netflix/priam/identity/token/TokenRetrieverUtils.java +++ b/priam/src/main/java/com/netflix/priam/identity/token/TokenRetrieverUtils.java @@ -1,11 +1,11 @@ package com.netflix.priam.identity.token; import com.netflix.priam.identity.PriamInstance; +import com.netflix.priam.utils.GsonJsonSerializer; import com.netflix.priam.utils.SystemUtils; import java.util.Collections; import java.util.List; import java.util.stream.Collectors; -import org.apache.commons.lang3.StringUtils; import org.json.simple.JSONArray; import org.json.simple.JSONObject; import org.json.simple.parser.JSONParser; @@ -33,9 +33,8 @@ public class TokenRetrieverUtils { * @throws GossipParseException when required number of instances are not available to fetch the * gossip info. */ - public static String inferTokenOwnerFromGossip( - List allIds, String token, String dc) - throws GossipParseException, TokenAliveException { + public static InferredTokenOwnership inferTokenOwnerFromGossip( + List allIds, String token, String dc) { // Avoid using dead instance who we are trying to replace (duh!!) // Avoid other regions instances to avoid communication over public ip address. @@ -59,56 +58,64 @@ public static String inferTokenOwnerFromGossip( // the IP to be replaced, in large clusters it may affect the startup // performance. So we pick three random hosts from the ring and see if they all // agree on the IP to be replaced. If not, we don't replace. - String replaceIp = null; + InferredTokenOwnership inferredTokenOwnership = new InferredTokenOwnership(); int matchedGossipInstances = 0, reachableInstances = 0; for (PriamInstance instance : eligibleInstances) { logger.info( "Calling getIp on hostname[{}] and token[{}]", instance.getHostName(), token); try { - String ip = getIp(instance.getHostName(), token); + TokenInformation tokenInformation = + getTokenInformation(instance.getHostName(), token); reachableInstances++; - if (replaceIp == null) { - replaceIp = ip; + if (inferredTokenOwnership.getTokenInformation() == null) { + inferredTokenOwnership.setTokenInformation(tokenInformation); } - if (StringUtils.isEmpty(ip) || !replaceIp.equals(ip)) { - // If the IP address produced by getIp call is empty it means it was able to - // parse the status information and that token was still alive!! - // We do not want to do anything if token is considered as alive by Cassandra. + if (inferredTokenOwnership.getTokenInformation().equals(tokenInformation)) { + matchedGossipInstances++; + if (matchedGossipInstances == noOfInstancesGossipShouldMatch) { + inferredTokenOwnership.setTokenInformationStatus( + InferredTokenOwnership.TokenInformationStatus.GOOD); + return inferredTokenOwnership; + } + } else { + // Mismatch in the gossip information from Cassandra. + inferredTokenOwnership.setTokenInformationStatus( + InferredTokenOwnership.TokenInformationStatus.MISMATCH); logger.info( - "Not producing anything in replaceIp as according to C* that token is still alive or " - + "there is a mismatch in status information per Cassandra. ip: [{}], replaceIp: [{}]", - ip, - replaceIp); - return null; + "There is a mismatch in the status information reported by Cassandra. TokenInformation1: {}, TokenInformation2: {}", + inferredTokenOwnership.getTokenInformation(), + tokenInformation); + inferredTokenOwnership.setTokenInformation( + inferredTokenOwnership.getTokenInformation().isLive + ? inferredTokenOwnership.getTokenInformation() + : tokenInformation); + return inferredTokenOwnership; } - matchedGossipInstances++; - if (matchedGossipInstances == noOfInstancesGossipShouldMatch) { - return replaceIp; - } } catch (GossipParseException e) { logger.warn(e.getMessage()); } } - // Throw exception if we are not able to reach at least minimum required - // instances. + // If we are not able to reach at least minimum required instances. if (reachableInstances < noOfInstancesGossipShouldMatch) { - throw new GossipParseException( + inferredTokenOwnership.setTokenInformationStatus( + InferredTokenOwnership.TokenInformationStatus.UNREACHABLE_NODES); + logger.info( String.format( "Unable to find enough instances where gossip match. Required: [%d]", noOfInstancesGossipShouldMatch)); } - return null; + return inferredTokenOwnership; } // helper method to get the token owner IP from a Cassandra node. - private static String getIp(String host, String token) - throws GossipParseException, TokenAliveException { + private static TokenInformation getTokenInformation(String host, String token) + throws GossipParseException { String response = null; try { response = SystemUtils.getDataFromUrl(String.format(STATUS_URL_FORMAT, host)); @@ -119,12 +126,8 @@ private static String getIp(String host, String token) // We intentionally do not use the "unreachable" nodes as it may or may not be the best // place to start. // We just verify that the endpoint we provide is not "live". - if (liveNodes.contains(endpointInfo)) { - throw new TokenAliveException( - String.format("The token %s is considered as alive by %s.", token, host)); - } - - return endpointInfo; + boolean isLive = liveNodes.contains(endpointInfo); + return new TokenInformation(endpointInfo, isLive); } catch (RuntimeException e) { throw new GossipParseException( String.format("Error in reaching out to host: [%s]", host), e); @@ -137,40 +140,81 @@ private static String getIp(String host, String token) } } - /** - * This exception is thrown either when instances are not available or when they return invalid - * response. - */ - public static class GossipParseException extends Exception { - private static final long serialVersionUID = 1462488371031437486L; + public static class TokenInformation { + private String ipAddress; + private boolean isLive; - public GossipParseException() { - super(); + public TokenInformation(String ipAddress, boolean isLive) { + this.ipAddress = ipAddress; + this.isLive = isLive; } - public GossipParseException(String message) { - super(message); + public boolean isLive() { + return isLive; } - public GossipParseException(String message, Throwable t) { - super(message, t); + public String getIpAddress() { + return ipAddress; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) return true; + if (obj == null || this.getClass() != obj.getClass()) return false; + TokenInformation tokenInformation = (TokenInformation) obj; + return this.ipAddress.equalsIgnoreCase(tokenInformation.getIpAddress()) + && isLive == tokenInformation.isLive; + } + + public String toString() { + return GsonJsonSerializer.getGson().toJson(this); } } - /** This exception is thrown either when a node is bootstrapping using a token that is alive. */ - public static class TokenAliveException extends Exception { + public static class InferredTokenOwnership { + public enum TokenInformationStatus { + GOOD, + UNREACHABLE_NODES, + MISMATCH + } + + private TokenInformationStatus tokenInformationStatus = + TokenInformationStatus.UNREACHABLE_NODES; + private TokenInformation tokenInformation; + + public void setTokenInformationStatus(TokenInformationStatus tokenInformationStatus) { + this.tokenInformationStatus = tokenInformationStatus; + } + + public void setTokenInformation(TokenInformation tokenInformation) { + this.tokenInformation = tokenInformation; + } + + public TokenInformationStatus getTokenInformationStatus() { + return tokenInformationStatus; + } + + public TokenInformation getTokenInformation() { + return tokenInformation; + } + } - private static final long serialVersionUID = 1038678311186020257L; + /** + * This exception is thrown either when instances are not available or when they return invalid + * response. + */ + public static class GossipParseException extends Exception { + private static final long serialVersionUID = 1462488371031437486L; - public TokenAliveException() { + public GossipParseException() { super(); } - public TokenAliveException(String message) { + public GossipParseException(String message) { super(message); } - public TokenAliveException(String message, Throwable t) { + public GossipParseException(String message, Throwable t) { super(message, t); } } diff --git a/priam/src/test/java/com/netflix/priam/identity/token/AssignedTokenRetrieverTest.java b/priam/src/test/java/com/netflix/priam/identity/token/AssignedTokenRetrieverTest.java index a3a28e240..a4294e7c7 100644 --- a/priam/src/test/java/com/netflix/priam/identity/token/AssignedTokenRetrieverTest.java +++ b/priam/src/test/java/com/netflix/priam/identity/token/AssignedTokenRetrieverTest.java @@ -36,6 +36,13 @@ public void grabAssignedTokenStartDbInBootstrapModeWhenGossipAgreesCurrentInstan List liveHosts = newPriamInstances(); Collections.shuffle(liveHosts); + TokenRetrieverUtils.InferredTokenOwnership inferredTokenOwnership = + new TokenRetrieverUtils.InferredTokenOwnership(); + inferredTokenOwnership.setTokenInformationStatus( + TokenRetrieverUtils.InferredTokenOwnership.TokenInformationStatus.GOOD); + inferredTokenOwnership.setTokenInformation( + new TokenRetrieverUtils.TokenInformation(liveHosts.get(0).getHostIP(), false)); + new Expectations() { { config.getAppName(); @@ -52,7 +59,7 @@ public void grabAssignedTokenStartDbInBootstrapModeWhenGossipAgreesCurrentInstan TokenRetrieverUtils.inferTokenOwnerFromGossip( liveHosts, liveHosts.get(0).getToken(), liveHosts.get(0).getDC()); - result = liveHosts.get(0).getHostIP(); + result = inferredTokenOwnership; } }; @@ -106,6 +113,12 @@ public void grabAssignedTokenStartDbInReplaceModeWhenGossipAgreesOnPreviousToken // the case we are trying to test is when Priam restarted after it acquired the // token. new instance is already registered with token database. liveHosts.add(newInstance); + TokenRetrieverUtils.InferredTokenOwnership inferredTokenOwnership = + new TokenRetrieverUtils.InferredTokenOwnership(); + inferredTokenOwnership.setTokenInformationStatus( + TokenRetrieverUtils.InferredTokenOwnership.TokenInformationStatus.GOOD); + inferredTokenOwnership.setTokenInformation( + new TokenRetrieverUtils.TokenInformation(deadInstance.getHostIP(), false)); new Expectations() { { @@ -122,7 +135,7 @@ public void grabAssignedTokenStartDbInReplaceModeWhenGossipAgreesOnPreviousToken TokenRetrieverUtils.inferTokenOwnerFromGossip( liveHosts, newInstance.getToken(), newInstance.getDC()); - result = deadInstance.getHostIP(); + result = inferredTokenOwnership; } }; @@ -162,6 +175,13 @@ public void grabAssignedTokenStartDbInBootstrapModeWhenGossipDisagreesOnPrevious List liveHosts = newPriamInstances(); Collections.shuffle(liveHosts); + TokenRetrieverUtils.InferredTokenOwnership inferredTokenOwnership = + new TokenRetrieverUtils.InferredTokenOwnership(); + inferredTokenOwnership.setTokenInformationStatus( + TokenRetrieverUtils.InferredTokenOwnership.TokenInformationStatus.MISMATCH); + inferredTokenOwnership.setTokenInformation( + new TokenRetrieverUtils.TokenInformation(liveHosts.get(0).getHostIP(), false)); + new Expectations() { { config.getAppName(); @@ -177,7 +197,7 @@ public void grabAssignedTokenStartDbInBootstrapModeWhenGossipDisagreesOnPrevious TokenRetrieverUtils.inferTokenOwnerFromGossip( liveHosts, liveHosts.get(0).getToken(), liveHosts.get(0).getDC()); - result = null; + result = inferredTokenOwnership; } }; diff --git a/priam/src/test/java/com/netflix/priam/identity/token/DeadTokenRetrieverTest.java b/priam/src/test/java/com/netflix/priam/identity/token/DeadTokenRetrieverTest.java index ed346e930..1c9ac7ab3 100644 --- a/priam/src/test/java/com/netflix/priam/identity/token/DeadTokenRetrieverTest.java +++ b/priam/src/test/java/com/netflix/priam/identity/token/DeadTokenRetrieverTest.java @@ -109,7 +109,7 @@ public void testNoReplacementNoSpotAvailable() throws Exception { }; } - @Test(expected = TokenRetrieverUtils.TokenAliveException.class) + @Test // There is a potential slot for dead token but we are unable to replace. public void testNoReplacementNoGossipMatch(@Mocked SystemUtils systemUtils) throws Exception { List allInstances = getInstances(2); @@ -132,6 +132,7 @@ public void testNoReplacementNoGossipMatch(@Mocked SystemUtils systemUtils) thro new DeadTokenRetriever( factory, membership, configuration, new FakeSleeper(), instanceInfo); PriamInstance priamInstance = deadTokenRetriever.get(); + Assert.assertNull(priamInstance); new Verifications() { { factory.delete(withAny(instance)); diff --git a/priam/src/test/java/com/netflix/priam/identity/token/TokenRetrieverUtilsTest.java b/priam/src/test/java/com/netflix/priam/identity/token/TokenRetrieverUtilsTest.java index fd406e151..c5bd4f6cd 100644 --- a/priam/src/test/java/com/netflix/priam/identity/token/TokenRetrieverUtilsTest.java +++ b/priam/src/test/java/com/netflix/priam/identity/token/TokenRetrieverUtilsTest.java @@ -46,8 +46,7 @@ public class TokenRetrieverUtilsTest { .collect(Collectors.toList()); @Test - public void testRetrieveTokenOwnerWhenGossipAgrees(@Mocked SystemUtils systemUtils) - throws Exception { + public void testRetrieveTokenOwnerWhenGossipAgrees(@Mocked SystemUtils systemUtils) { // mark previous instance with tokenNumber 4 as down in gossip. List myliveInstances = liveInstances @@ -62,13 +61,14 @@ public void testRetrieveTokenOwnerWhenGossipAgrees(@Mocked SystemUtils systemUti } }; - String replaceIp = TokenRetrieverUtils.inferTokenOwnerFromGossip(instances, "4", "us-east"); - Assert.assertEquals("127.0.0.4", replaceIp); + TokenRetrieverUtils.InferredTokenOwnership inferredTokenOwnership = + TokenRetrieverUtils.inferTokenOwnerFromGossip(instances, "4", "us-east"); + Assert.assertEquals( + "127.0.0.4", inferredTokenOwnership.getTokenInformation().getIpAddress()); } - @Test(expected = TokenRetrieverUtils.TokenAliveException.class) - public void testRetrieveTokenOwnerWhenGossipDisagrees(@Mocked SystemUtils systemUtils) - throws Exception { + @Test + public void testRetrieveTokenOwnerWhenGossipDisagrees(@Mocked SystemUtils systemUtils) { List myliveInstances = liveInstances @@ -96,15 +96,20 @@ public void testRetrieveTokenOwnerWhenGossipDisagrees(@Mocked SystemUtils system minTimes = 0; SystemUtils.getDataFromUrl(String.format(STATUS_URL_FORMAT, "fakeHost-5")); - result = getStatus(liveInstances, tokenToEndpointMap); + result = null; minTimes = 0; } }; - String replaceIp = TokenRetrieverUtils.inferTokenOwnerFromGossip(instances, "4", "us-east"); + TokenRetrieverUtils.InferredTokenOwnership inferredTokenOwnership = + TokenRetrieverUtils.inferTokenOwnerFromGossip(instances, "4", "us-east"); + Assert.assertEquals( + TokenRetrieverUtils.InferredTokenOwnership.TokenInformationStatus.MISMATCH, + inferredTokenOwnership.getTokenInformationStatus()); + Assert.assertTrue(inferredTokenOwnership.getTokenInformation().isLive()); } - @Test(expected = TokenRetrieverUtils.TokenAliveException.class) + @Test public void testRetrieveTokenOwnerWhenAllHostsInGossipReturnsNull( @Mocked SystemUtils systemUtils) throws Exception { new Expectations() { @@ -114,15 +119,17 @@ public void testRetrieveTokenOwnerWhenAllHostsInGossipReturnsNull( } }; - String replaceIp = TokenRetrieverUtils.inferTokenOwnerFromGossip(instances, "4", "us-east"); - Assert.assertNull(replaceIp); + TokenRetrieverUtils.InferredTokenOwnership inferredTokenOwnership = + TokenRetrieverUtils.inferTokenOwnerFromGossip(instances, "4", "us-east"); + Assert.assertEquals( + TokenRetrieverUtils.InferredTokenOwnership.TokenInformationStatus.GOOD, + inferredTokenOwnership.getTokenInformationStatus()); + Assert.assertTrue(inferredTokenOwnership.getTokenInformation().isLive()); } - @Test(expected = TokenRetrieverUtils.GossipParseException.class) + @Test public void testRetrieveTokenOwnerWhenAllInstancesThrowGossipParseException( - @Mocked SystemUtils systemUtils) - throws TokenRetrieverUtils.GossipParseException, - TokenRetrieverUtils.TokenAliveException { + @Mocked SystemUtils systemUtils) { new Expectations() { { @@ -131,8 +138,12 @@ public void testRetrieveTokenOwnerWhenAllInstancesThrowGossipParseException( } }; - String replaceIp = TokenRetrieverUtils.inferTokenOwnerFromGossip(instances, "4", "us-east"); - Assert.assertNull(replaceIp); + TokenRetrieverUtils.InferredTokenOwnership inferredTokenOwnership = + TokenRetrieverUtils.inferTokenOwnerFromGossip(instances, "4", "us-east"); + Assert.assertEquals( + TokenRetrieverUtils.InferredTokenOwnership.TokenInformationStatus.UNREACHABLE_NODES, + inferredTokenOwnership.getTokenInformationStatus()); + Assert.assertNull(inferredTokenOwnership.getTokenInformation()); } private String newGossipRecord( From 8ebaccade9feeb46e4d4fd8d87fb6b4fb7a17ae9 Mon Sep 17 00:00:00 2001 From: Matt Lehman Date: Wed, 15 Jul 2020 19:01:13 -0700 Subject: [PATCH 140/228] Update CHANGELOG in advance of 3.11.69 --- CHANGELOG.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ac49da3fb..5260fd67c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # Changelog -## 2020/07/02/ 3.11.68 +## 2020/07/15 3.11.69 +(#894) Fix the inferTokenOwnership information. This will provide all the details to the caller method so they can make decision rather than throwing any exception. +(#897) Make BackupVerificationTask log and emit when there is no verified backup within SLO. Cease requiring the backup to be fully in S3. + +## 2020/07/02 3.11.68 (#891) Adding an exception in the replace-ip path when a node attempts to bootstrap to an existing token because of a stale state. ## 2020/06/04 3.11.67 From 3f3862e91b648a0cb34c2678fef9288d7b942653 Mon Sep 17 00:00:00 2001 From: Matt Lehman Date: Thu, 6 Aug 2020 08:20:08 -0700 Subject: [PATCH 141/228] Throw when gossip unanimously says token is already owned by a live node. C* will throw anyway in these cases and there is some chance that the current owner has yet to be taken offline. --- .../priam/identity/InstanceIdentity.java | 36 ++++--- .../token/AssignedTokenRetrieverTest.java | 98 ++++++++++++++++--- 2 files changed, 105 insertions(+), 29 deletions(-) diff --git a/priam/src/main/java/com/netflix/priam/identity/InstanceIdentity.java b/priam/src/main/java/com/netflix/priam/identity/InstanceIdentity.java index 59df9cca9..03ab09cb0 100644 --- a/priam/src/main/java/com/netflix/priam/identity/InstanceIdentity.java +++ b/priam/src/main/java/com/netflix/priam/identity/InstanceIdentity.java @@ -16,6 +16,7 @@ */ package com.netflix.priam.identity; +import com.google.common.base.Preconditions; import com.google.common.base.Predicate; import com.google.common.collect.Iterables; import com.google.common.collect.ListMultimap; @@ -29,6 +30,7 @@ import com.netflix.priam.identity.token.INewTokenRetriever; import com.netflix.priam.identity.token.IPreGeneratedTokenRetriever; import com.netflix.priam.identity.token.TokenRetrieverUtils; +import com.netflix.priam.identity.token.TokenRetrieverUtils.GossipParseException; import com.netflix.priam.utils.ITokenManager; import com.netflix.priam.utils.RetryableCallable; import com.netflix.priam.utils.Sleeper; @@ -182,26 +184,28 @@ public PriamInstance retriableCall() throws Exception { // gossip. // Infer current ownership information from other instances using gossip. - TokenRetrieverUtils.InferredTokenOwnership inferredTokenInformation = + TokenRetrieverUtils.InferredTokenOwnership inferredTokenOwnership = TokenRetrieverUtils.inferTokenOwnerFromGossip( aliveInstances, instance.getToken(), instance.getDC()); - String inferredIp = - (inferredTokenInformation.getTokenInformation() == null) - ? null - : inferredTokenInformation - .getTokenInformation() - .getIpAddress(); // if unreachable rely on token database. // if mismatch rely on token database. - if (inferredTokenInformation.getTokenInformationStatus() - == TokenRetrieverUtils.InferredTokenOwnership - .TokenInformationStatus.GOOD - && !inferredIp.equalsIgnoreCase(instance.getHostIP()) - && !inferredTokenInformation.getTokenInformation().isLive()) { - setReplacedIp(inferredIp); - logger.info( - "Priam found that the token is not alive according to Cassandra and we should start Cassandra in replace mode with replace ip: " - + inferredIp); + if (inferredTokenOwnership.getTokenInformationStatus() + == TokenRetrieverUtils.InferredTokenOwnership.TokenInformationStatus + .GOOD) { + Preconditions.checkNotNull( + inferredTokenOwnership.getTokenInformation()); + String inferredIp = + inferredTokenOwnership.getTokenInformation().getIpAddress(); + if (!inferredIp.equalsIgnoreCase(instance.getHostIP())) { + if (inferredTokenOwnership.getTokenInformation().isLive()) { + throw new GossipParseException( + "We have been assigned a token that C* thinks is alive. Throwing to buy time in the hopes that Gossip just needs to settle."); + } + setReplacedIp(inferredIp); + logger.info( + "Priam found that the token is not alive according to Cassandra and we should start Cassandra in replace mode with replace ip: " + + inferredIp); + } } } } diff --git a/priam/src/test/java/com/netflix/priam/identity/token/AssignedTokenRetrieverTest.java b/priam/src/test/java/com/netflix/priam/identity/token/AssignedTokenRetrieverTest.java index a4294e7c7..f61b584a8 100644 --- a/priam/src/test/java/com/netflix/priam/identity/token/AssignedTokenRetrieverTest.java +++ b/priam/src/test/java/com/netflix/priam/identity/token/AssignedTokenRetrieverTest.java @@ -1,5 +1,7 @@ package com.netflix.priam.identity.token; +import com.google.common.base.Strings; +import com.google.common.truth.Truth; import com.netflix.priam.config.IConfiguration; import com.netflix.priam.identity.IMembership; import com.netflix.priam.identity.IPriamInstanceFactory; @@ -15,9 +17,8 @@ import java.util.stream.IntStream; import mockit.Expectations; import mockit.Mocked; -import org.apache.commons.lang3.StringUtils; -import org.junit.Assert; import org.junit.Test; +import org.junit.jupiter.api.Assertions; public class AssignedTokenRetrieverTest { public static final String APP = "testapp"; @@ -82,11 +83,11 @@ public void grabAssignedTokenStartDbInBootstrapModeWhenGossipAgreesCurrentInstan newTokenRetriever, instanceInfo); - Assert.assertEquals(false, instanceIdentity.isReplace()); + Truth.assertThat(instanceIdentity.isReplace()).isFalse(); } @Test - public void grabAssignedTokenStartDbInReplaceModeWhenGossipAgreesOnPreviousTokenOwner( + public void grabAssignedTokenStartDbInReplaceModeWhenGossipAgreesPreviousTokenOwnerIsNotLive( @Mocked IPriamInstanceFactory factory, @Mocked IConfiguration config, @Mocked IMembership membership, @@ -101,7 +102,6 @@ public void grabAssignedTokenStartDbInReplaceModeWhenGossipAgreesOnPreviousToken PriamInstance deadInstance = liveHosts.remove(0); PriamInstance newInstance = newMockPriamInstance( - APP, deadInstance.getDC(), deadInstance.getRac(), deadInstance.getId(), @@ -158,8 +158,82 @@ public void grabAssignedTokenStartDbInReplaceModeWhenGossipAgreesOnPreviousToken newTokenRetriever, instanceInfo); - Assert.assertEquals(deadInstance.getHostIP(), instanceIdentity.getReplacedIp()); - Assert.assertEquals(true, instanceIdentity.isReplace()); + Truth.assertThat(instanceIdentity.getReplacedIp()).isEqualTo(deadInstance.getHostIP()); + Truth.assertThat(instanceIdentity.isReplace()).isTrue(); + } + + @Test + public void grabAssignedTokenThrowWhenGossipAgreesPreviousTokenOwnerIsLive( + @Mocked IPriamInstanceFactory factory, + @Mocked IConfiguration config, + @Mocked IMembership membership, + @Mocked Sleeper sleeper, + @Mocked ITokenManager tokenManager, + @Mocked InstanceInfo instanceInfo, + @Mocked TokenRetrieverUtils retrievalUtils) { + List liveHosts = newPriamInstances(); + Collections.shuffle(liveHosts); + + PriamInstance deadInstance = liveHosts.remove(0); + PriamInstance newInstance = + newMockPriamInstance( + deadInstance.getDC(), + deadInstance.getRac(), + deadInstance.getId(), + String.format("new-fakeInstance-%d", deadInstance.getId()), + String.format("127.1.1.%d", deadInstance.getId() + 100), + String.format("new-fakeHost-%d", deadInstance.getId()), + deadInstance.getToken()); + + // the case we are trying to test is when Priam restarted after it acquired the + // token. new instance is already registered with token database. + liveHosts.add(newInstance); + TokenRetrieverUtils.InferredTokenOwnership inferredTokenOwnership = + new TokenRetrieverUtils.InferredTokenOwnership(); + inferredTokenOwnership.setTokenInformationStatus( + TokenRetrieverUtils.InferredTokenOwnership.TokenInformationStatus.GOOD); + inferredTokenOwnership.setTokenInformation( + new TokenRetrieverUtils.TokenInformation(deadInstance.getHostIP(), true)); + + new Expectations() { + { + config.getAppName(); + result = APP; + + factory.getAllIds(DEAD_APP); + result = Collections.singletonList(deadInstance); + factory.getAllIds(APP); + result = liveHosts; + + instanceInfo.getInstanceId(); + result = newInstance.getInstanceId(); + + TokenRetrieverUtils.inferTokenOwnerFromGossip( + liveHosts, newInstance.getToken(), newInstance.getDC()); + result = inferredTokenOwnership; + } + }; + + IDeadTokenRetriever deadTokenRetriever = + new DeadTokenRetriever(factory, membership, config, sleeper, instanceInfo); + IPreGeneratedTokenRetriever preGeneratedTokenRetriever = + new PreGeneratedTokenRetriever(factory, membership, config, sleeper, instanceInfo); + INewTokenRetriever newTokenRetriever = + new NewTokenRetriever( + factory, membership, config, sleeper, tokenManager, instanceInfo); + Assertions.assertThrows( + TokenRetrieverUtils.GossipParseException.class, + () -> + new InstanceIdentity( + factory, + membership, + config, + sleeper, + tokenManager, + deadTokenRetriever, + preGeneratedTokenRetriever, + newTokenRetriever, + instanceInfo)); } @Test @@ -220,8 +294,8 @@ public void grabAssignedTokenStartDbInBootstrapModeWhenGossipDisagreesOnPrevious newTokenRetriever, instanceInfo); - Assert.assertTrue(StringUtils.isEmpty(instanceIdentity.getReplacedIp())); - Assert.assertEquals(false, instanceIdentity.isReplace()); + Truth.assertThat(Strings.isNullOrEmpty(instanceIdentity.getReplacedIp())).isTrue(); + Truth.assertThat(instanceIdentity.isReplace()).isFalse(); } private List newPriamInstances() { @@ -246,10 +320,9 @@ private List newPriamInstances( String dc, String rack, int seqNo, String ipRanges) { return IntStream.range(0, 3) .map(e -> seqNo + (e * 9)) - .mapToObj( + .mapToObj( e -> newMockPriamInstance( - APP, dc, rack, e, @@ -261,7 +334,6 @@ private List newPriamInstances( } private PriamInstance newMockPriamInstance( - String app, String dc, String rack, int id, @@ -270,7 +342,7 @@ private PriamInstance newMockPriamInstance( String hostName, String token) { PriamInstance priamInstance = new PriamInstance(); - priamInstance.setApp(app); + priamInstance.setApp(APP); priamInstance.setDC(dc); priamInstance.setRac(rack); priamInstance.setId(id); From 05c86369996030a0e8c974470b6e4eda7b7aabf7 Mon Sep 17 00:00:00 2001 From: Matt Lehman Date: Tue, 11 Aug 2020 16:37:13 -0700 Subject: [PATCH 142/228] Update CHANGELOG in advance of 3.11.70 --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5260fd67c..2e42c063c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,7 @@ # Changelog +## 2020/08/11 3.11.70 +(#901) Throw when gossip unanimously says token is already owned by a live node. + ## 2020/07/15 3.11.69 (#894) Fix the inferTokenOwnership information. This will provide all the details to the caller method so they can make decision rather than throwing any exception. (#897) Make BackupVerificationTask log and emit when there is no verified backup within SLO. Cease requiring the backup to be fully in S3. From 9908f29772575b9c2e53bcc1f1a4ecc2a3f53d7a Mon Sep 17 00:00:00 2001 From: mattl-netflix <63665634+mattl-netflix@users.noreply.github.com> Date: Tue, 8 Sep 2020 13:53:22 -0700 Subject: [PATCH 143/228] Remove redundant log statements from CassandraAdmin (#902) * CASS-1870 remove dead code, clean imports * CASS-1870 correct typos * CASS-1870 Remove noisy log statements from CassandraAdmin. All removed logs are redundant with what JMXNodeTool produces and sometimes print entire stack traces when the Exception is not a surprise (i.e., C* is not up yet) --- .../priam/resources/CassandraAdmin.java | 72 +++++-------------- 1 file changed, 18 insertions(+), 54 deletions(-) diff --git a/priam/src/main/java/com/netflix/priam/resources/CassandraAdmin.java b/priam/src/main/java/com/netflix/priam/resources/CassandraAdmin.java index 6894ffd4d..a9f2b895c 100644 --- a/priam/src/main/java/com/netflix/priam/resources/CassandraAdmin.java +++ b/priam/src/main/java/com/netflix/priam/resources/CassandraAdmin.java @@ -35,7 +35,6 @@ import javax.ws.rs.*; import javax.ws.rs.core.MediaType; import javax.ws.rs.core.Response; -import org.apache.cassandra.exceptions.ConfigurationException; import org.apache.commons.lang3.StringUtils; import org.codehaus.jettison.json.JSONArray; import org.codehaus.jettison.json.JSONException; @@ -44,12 +43,10 @@ import org.slf4j.LoggerFactory; /** Do general operations. Start/Stop and some JMX node tool commands */ -@SuppressWarnings("deprecation") @Path("/v1/cassadmin") @Produces(MediaType.APPLICATION_JSON) public class CassandraAdmin { private static final String REST_HEADER_KEYSPACES = "keyspaces"; - private static final String REST_HEADER_CFS = "cfnames"; private static final String REST_HEADER_TOKEN = "token"; private static final String REST_SUCCESS = "[\"ok\"]"; private static final Logger logger = LoggerFactory.getLogger(CassandraAdmin.class); @@ -75,7 +72,7 @@ public CassandraAdmin( @GET @Path("/start") - public Response cassStart() throws IOException, InterruptedException, JSONException { + public Response cassStart() throws IOException { cassProcess.start(true); return Response.ok(REST_SUCCESS, MediaType.APPLICATION_JSON).build(); } @@ -83,7 +80,7 @@ public Response cassStart() throws IOException, InterruptedException, JSONExcept @GET @Path("/stop") public Response cassStop(@DefaultValue("false") @QueryParam("force") boolean force) - throws IOException, InterruptedException, JSONException { + throws IOException { cassProcess.stop(force); return Response.ok(REST_SUCCESS, MediaType.APPLICATION_JSON).build(); } @@ -91,7 +88,7 @@ public Response cassStop(@DefaultValue("false") @QueryParam("force") boolean for @GET @Path("/refresh") public Response cassRefresh(@QueryParam(REST_HEADER_KEYSPACES) String keyspaces) - throws IOException, ExecutionException, InterruptedException, JSONException { + throws IOException, ExecutionException, InterruptedException { logger.debug("node tool refresh is being called"); if (StringUtils.isBlank(keyspaces)) return Response.status(400).entity("Missing keyspace in request").build(); @@ -100,8 +97,6 @@ public Response cassRefresh(@QueryParam(REST_HEADER_KEYSPACES) String keyspaces) try { nodeTool = JMXNodeTool.instance(config); } catch (JMXConnectionException e) { - logger.error( - "Exception in fetching c* jmx tool . Msgl: {}", e.getLocalizedMessage(), e); return Response.status(503).entity("JMXConnectionException").build(); } nodeTool.refresh(Lists.newArrayList(keyspaces.split(","))); @@ -110,13 +105,11 @@ public Response cassRefresh(@QueryParam(REST_HEADER_KEYSPACES) String keyspaces) @GET @Path("/info") - public Response cassInfo() throws IOException, InterruptedException, JSONException { + public Response cassInfo() throws JSONException { JMXNodeTool nodeTool; try { nodeTool = JMXNodeTool.instance(config); } catch (JMXConnectionException e) { - logger.error( - "Exception in fetching c* jmx tool . Msgl: {}", e.getLocalizedMessage(), e); return Response.status(503).entity("JMXConnectionException").build(); } logger.debug("node tool info being called"); @@ -125,13 +118,11 @@ public Response cassInfo() throws IOException, InterruptedException, JSONExcepti @GET @Path("/partitioner") - public Response cassPartitioner() throws IOException, InterruptedException, JSONException { + public Response cassPartitioner() { JMXNodeTool nodeTool; try { nodeTool = JMXNodeTool.instance(config); } catch (JMXConnectionException e) { - logger.error( - "Exception in fetching c* jmx tool . Msg: {}", e.getLocalizedMessage(), e); return Response.status(503).entity("JMXConnectionException").build(); } logger.debug("node tool getPartitioner being called"); @@ -140,14 +131,11 @@ public Response cassPartitioner() throws IOException, InterruptedException, JSON @GET @Path("/ring/{id}") - public Response cassRing(@PathParam("id") String keyspace) - throws IOException, InterruptedException, JSONException { + public Response cassRing(@PathParam("id") String keyspace) throws JSONException { JMXNodeTool nodeTool; try { nodeTool = JMXNodeTool.instance(config); } catch (JMXConnectionException e) { - logger.error( - "Exception in fetching c* jmx tool . Msg: {}", e.getLocalizedMessage(), e); return Response.status(503).entity("JMXConnectionException").build(); } logger.debug("node tool ring being called"); @@ -161,8 +149,6 @@ public Response statusInfo() throws JSONException { try { nodeTool = JMXNodeTool.instance(config); } catch (JMXConnectionException e) { - logger.error( - "Exception in fetching c* jmx tool . Msg: {}", e.getLocalizedMessage(), e); return Response.status(503).entity("JMXConnectionException").build(); } logger.debug("node tool status being called"); @@ -171,7 +157,7 @@ public Response statusInfo() throws JSONException { @GET @Path("/flush") - public Response cassFlush() throws IOException, InterruptedException, ExecutionException { + public Response cassFlush() { JSONObject rootObj = new JSONObject(); try { @@ -180,7 +166,7 @@ public Response cassFlush() throws IOException, InterruptedException, ExecutionE return Response.ok().entity(rootObj).build(); } catch (Exception e) { try { - rootObj.put("status", "ERRROR"); + rootObj.put("status", "ERROR"); rootObj.put("desc", e.getLocalizedMessage()); } catch (Exception e1) { return Response.status(503).entity("FlushError").build(); @@ -191,16 +177,16 @@ public Response cassFlush() throws IOException, InterruptedException, ExecutionE @GET @Path("/compact") - public Response cassCompact() throws IOException, ExecutionException, InterruptedException { + public Response cassCompact() { JSONObject rootObj = new JSONObject(); try { compaction.execute(); - rootObj.put("Compcated", true); + rootObj.put("Compacted", true); return Response.ok().entity(rootObj).build(); } catch (Exception e) { try { - rootObj.put("status", "ERRROR"); + rootObj.put("status", "ERROR"); rootObj.put("desc", e.getLocalizedMessage()); } catch (Exception e1) { return Response.status(503).entity("CompactionError").build(); @@ -216,8 +202,6 @@ public Response cassCleanup() throws IOException, ExecutionException, Interrupte try { nodeTool = JMXNodeTool.instance(config); } catch (JMXConnectionException e) { - logger.error( - "Exception in fetching c* jmx tool . Msgl: {}", e.getLocalizedMessage(), e); return Response.status(503).entity("JMXConnectionException").build(); } logger.debug("node tool cleanup being called"); @@ -236,8 +220,6 @@ public Response cassRepair( try { nodeTool = JMXNodeTool.instance(config); } catch (JMXConnectionException e) { - logger.error( - "Exception in fetching c* jmx tool . Msgl: {}", e.getLocalizedMessage(), e); return Response.status(503).entity("JMXConnectionException").build(); } logger.debug("node tool repair being called"); @@ -247,13 +229,11 @@ public Response cassRepair( @GET @Path("/version") - public Response version() throws IOException, ExecutionException, InterruptedException { + public Response version() { JMXNodeTool nodeTool; try { nodeTool = JMXNodeTool.instance(config); } catch (JMXConnectionException e) { - logger.error( - "Exception in fetching c* jmx tool . Msgl: {}", e.getLocalizedMessage(), e); return Response.status(503).entity("JMXConnectionException").build(); } return Response.ok( @@ -264,13 +244,11 @@ public Response version() throws IOException, ExecutionException, InterruptedExc @GET @Path("/disablegossip") - public Response disablegossip() throws IOException, ExecutionException, InterruptedException { + public Response disablegossip() { JMXNodeTool nodeTool; try { nodeTool = JMXNodeTool.instance(config); } catch (JMXConnectionException e) { - logger.error( - "Exception in fetching c* jmx tool . Msgl: {}", e.getLocalizedMessage(), e); return Response.status(503).entity("JMXConnectionException").build(); } nodeTool.stopGossiping(); @@ -279,13 +257,11 @@ public Response disablegossip() throws IOException, ExecutionException, Interrup @GET @Path("/enablegossip") - public Response enablegossip() throws IOException, ExecutionException, InterruptedException { + public Response enablegossip() { JMXNodeTool nodeTool; try { nodeTool = JMXNodeTool.instance(config); } catch (JMXConnectionException e) { - logger.error( - "Exception in fetching c* jmx tool . Msgl: {}", e.getLocalizedMessage(), e); return Response.status(503).entity("JMXConnectionException").build(); } nodeTool.startGossiping(); @@ -294,13 +270,11 @@ public Response enablegossip() throws IOException, ExecutionException, Interrupt @GET @Path("/disablethrift") - public Response disablethrift() throws IOException, ExecutionException, InterruptedException { + public Response disablethrift() { JMXNodeTool nodeTool; try { nodeTool = JMXNodeTool.instance(config); } catch (JMXConnectionException e) { - logger.error( - "Exception in fetching c* jmx tool . Msgl: {}", e.getLocalizedMessage(), e); return Response.status(503).entity("JMXConnectionException").build(); } nodeTool.stopThriftServer(); @@ -309,13 +283,11 @@ public Response disablethrift() throws IOException, ExecutionException, Interrup @GET @Path("/enablethrift") - public Response enablethrift() throws IOException, ExecutionException, InterruptedException { + public Response enablethrift() { JMXNodeTool nodeTool; try { nodeTool = JMXNodeTool.instance(config); } catch (JMXConnectionException e) { - logger.error( - "Exception in fetching c* jmx tool . Msgl: {}", e.getLocalizedMessage(), e); return Response.status(503).entity("JMXConnectionException").build(); } nodeTool.startThriftServer(); @@ -324,14 +296,11 @@ public Response enablethrift() throws IOException, ExecutionException, Interrupt @GET @Path("/statusthrift") - public Response statusthrift() - throws IOException, ExecutionException, InterruptedException, JSONException { + public Response statusthrift() throws JSONException { JMXNodeTool nodeTool; try { nodeTool = JMXNodeTool.instance(config); } catch (JMXConnectionException e) { - logger.error( - "Exception in fetching c* jmx tool . Msgl: {}", e.getLocalizedMessage(), e); return Response.status(503).entity("JMXConnectionException").build(); } return Response.ok( @@ -354,14 +323,11 @@ public Response gossipinfo() throws Exception { @GET @Path("/move") - public Response moveToken(@QueryParam(REST_HEADER_TOKEN) String newToken) - throws IOException, ExecutionException, InterruptedException, ConfigurationException { + public Response moveToken(@QueryParam(REST_HEADER_TOKEN) String newToken) throws IOException { JMXNodeTool nodeTool; try { nodeTool = JMXNodeTool.instance(config); } catch (JMXConnectionException e) { - logger.error( - "Exception in fetching c* jmx tool . Msgl: {}", e.getLocalizedMessage(), e); return Response.status(503).entity("JMXConnectionException").build(); } nodeTool.move(newToken); @@ -375,8 +341,6 @@ public Response cassDrain() throws IOException, ExecutionException, InterruptedE try { nodeTool = JMXNodeTool.instance(config); } catch (JMXConnectionException e) { - logger.error( - "Exception in fetching c* jmx tool . Msgl: {}", e.getLocalizedMessage(), e); return Response.status(503).entity("JMXConnectionException").build(); } logger.debug("node tool drain being called"); From b1d1551eebc8d4d018b6d6e5776f9467b5f8fcd9 Mon Sep 17 00:00:00 2001 From: Matt Lehman Date: Tue, 8 Sep 2020 13:56:54 -0700 Subject: [PATCH 144/228] Update CHANGELOG in advance of 3.11.71 --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2e42c063c..281b531d6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,7 @@ # Changelog +## 2020/09/08 3.11.71 +(#902) Remove noisy log statements from CassandraAdmin. + ## 2020/08/11 3.11.70 (#901) Throw when gossip unanimously says token is already owned by a live node. From 0fd48220496c9f2d4d392bc49214f3bc051d87d3 Mon Sep 17 00:00:00 2001 From: mattl-netflix <63665634+mattl-netflix@users.noreply.github.com> Date: Wed, 23 Sep 2020 09:56:08 -0700 Subject: [PATCH 145/228] CASS-1937 cease filtering out OpsCenter keyspace. (#908) --- .../netflix/priam/backup/AbstractBackup.java | 36 +++++-------------- 1 file changed, 9 insertions(+), 27 deletions(-) diff --git a/priam/src/main/java/com/netflix/priam/backup/AbstractBackup.java b/priam/src/main/java/com/netflix/priam/backup/AbstractBackup.java index c073c9a4c..b4a222d6d 100644 --- a/priam/src/main/java/com/netflix/priam/backup/AbstractBackup.java +++ b/priam/src/main/java/com/netflix/priam/backup/AbstractBackup.java @@ -144,20 +144,16 @@ protected final void initiateBackup( for (File columnFamilyDir : columnFamilyDirectories) { File backupDir = new File(columnFamilyDir, monitoringFolder); - - if (!isValidBackupDir(keyspaceDir, backupDir)) { - continue; - } - - String columnFamilyName = columnFamilyDir.getName().split("-")[0]; - if (backupRestoreUtil.isFiltered( - keyspaceDir.getName(), columnFamilyDir.getName())) { - // Clean the backup/snapshot directory else files will keep getting accumulated. - SystemUtils.cleanupDir(backupDir.getAbsolutePath(), null); - continue; + if (backupDir.exists() && backupDir.isDirectory() && backupDir.canRead()) { + String columnFamilyName = columnFamilyDir.getName().split("-")[0]; + if (backupRestoreUtil.isFiltered(keyspaceDir.getName(), columnFamilyName)) { + // Clean the backup/snapshot directory else files will keep getting + // accumulated. + SystemUtils.cleanupDir(backupDir.getAbsolutePath(), null); + } else { + processColumnFamily(keyspaceDir.getName(), columnFamilyName, backupDir); + } } - - processColumnFamily(keyspaceDir.getName(), columnFamilyName, backupDir); } // end processing all CFs for keyspace } // end processing keyspaces under the C* data dir } @@ -206,18 +202,4 @@ public static Set getBackupDirectories(IConfiguration config, String monit } return backupPaths; } - - /** Filters unwanted keyspaces */ - private boolean isValidBackupDir(File keyspaceDir, File backupDir) { - if (backupDir == null || !backupDir.isDirectory() || !backupDir.exists()) return false; - String keyspaceName = keyspaceDir.getName(); - if (BackupRestoreUtil.FILTER_KEYSPACE.contains(keyspaceName)) { - logger.debug( - "{} is not consider a valid keyspace backup directory, will be bypass.", - keyspaceName); - return false; - } - - return true; - } } From 762f10658ca9c92611f80f01be880b78f942e7fe Mon Sep 17 00:00:00 2001 From: mattl-netflix <63665634+mattl-netflix@users.noreply.github.com> Date: Wed, 30 Sep 2020 16:13:07 -0700 Subject: [PATCH 146/228] CASS-1870 Quieter logs when we try to create a JMX connection before C* has started. (#910) * CASS-1870 remove dead code, clean imports * CASS-1870 Remove noisy log statements from CassandraAdmin. All removed logs are redundant with what JMXNodeTool produces and sometimes print entire stack traces when the Exception is not a surprise (i.e., C* is not up yet) * CASS-1870 Quieter logs any time CassandraTunerService.updateServicePost is run before C* starts. --- .../com/netflix/priam/tuner/CassandraTunerService.java | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/priam/src/main/java/com/netflix/priam/tuner/CassandraTunerService.java b/priam/src/main/java/com/netflix/priam/tuner/CassandraTunerService.java index d04ba7927..ab9e6e611 100644 --- a/priam/src/main/java/com/netflix/priam/tuner/CassandraTunerService.java +++ b/priam/src/main/java/com/netflix/priam/tuner/CassandraTunerService.java @@ -38,9 +38,10 @@ public void updateServicePost() throws Exception { // Update the cassandra to enable/disable new incremental files. new RetryableCallable(6, 10000) { public Void retriableCall() throws Exception { - JMXNodeTool nodetool = JMXNodeTool.instance(configuration); - nodetool.setIncrementalBackupsEnabled( - IncrementalBackup.isEnabled(configuration, backupRestoreConfig)); + try (JMXNodeTool nodeTool = JMXNodeTool.instance(configuration)) { + nodeTool.setIncrementalBackupsEnabled( + IncrementalBackup.isEnabled(configuration, backupRestoreConfig)); + } return null; } }.call(); From 29f75c46c5a1dbb45ae2642b28847880da671d7c Mon Sep 17 00:00:00 2001 From: Matt Lehman Date: Wed, 30 Sep 2020 16:18:36 -0700 Subject: [PATCH 147/228] Update CHANGELOG in advance of 3.11.72 --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 281b531d6..8dfecd3a5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,7 @@ # Changelog +## 2020/09/30 3.11.72 +(#908, #910) Stop explicitly filtering OpsCenter keyspace when backing up. Remove more noisy log statements. + ## 2020/09/08 3.11.71 (#902) Remove noisy log statements from CassandraAdmin. From 9ebb4feb3bdf2e88652f82b778208e68d8b06aa0 Mon Sep 17 00:00:00 2001 From: Matt Lehman Date: Fri, 30 Oct 2020 16:05:00 -0700 Subject: [PATCH 148/228] Migrate to travis-ci.com per their recommendations. --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 983e49607..6e6071db0 100644 --- a/README.md +++ b/README.md @@ -71,5 +71,5 @@ References [release]:https://github.com/Netflix/Priam/releases/latest "Latest Release (external link) ➶" [wiki]:http://netflix.github.io/Priam/ [repo]:https://github.com/Netflix/Priam -[img-travis-ci]:https://travis-ci.org/Netflix/Priam.svg?branch=3.11 -[travis-ci]:https://travis-ci.org/Netflix/Priam +[img-travis-ci]:https://travis-ci.com/Netflix/Priam.svg?branch=3.11 +[travis-ci]:https://travis-ci.com/Netflix/Priam From ed5bb90705a7cb2b6dbefd9222dc5a91c1a47db4 Mon Sep 17 00:00:00 2001 From: mattl-netflix <63665634+mattl-netflix@users.noreply.github.com> Date: Mon, 9 Nov 2020 19:40:30 -0800 Subject: [PATCH 149/228] Backup Secondary Indexes (#911) * CASS-1906 Move CompressionAlgorithm and EncryptionAlgorithm to separate classes. * CASS-1906 Remove dead code from AbstractBackupPath * CASS-1906 Style tweaks * CASS-1906 Add support for secondary index file type * CASS-1906 Remove redundant methods from FileUploadResult. * CASS-1906 Remove redundant methods from ColumnFamilyResult * CASS-1906 Inline PrefixGenerator's only method, unnest try-catch blocks, Use ImmutableSetMultimap where applicable. * CASS-1906 Upload secondary index files for V2 backups * CASS-1906 Append '_V2' to 'SECONDARY_INDEX' enum value in BackupFileType to adhere to existing conventions. * CASS-1906 Fix file depth bug found in e2e testing. --- .../netflix/priam/aws/RemoteBackupPath.java | 236 ++++++++---------- .../netflix/priam/backup/AbstractBackup.java | 6 +- .../priam/backup/AbstractBackupPath.java | 120 +++++---- .../netflix/priam/backup/BackupFolder.java | 18 ++ .../priam/backup/IncrementalBackup.java | 15 +- .../priam/backupv2/ColumnfamilyResult.java | 28 +-- .../priam/backupv2/FileUploadResult.java | 98 +------- .../priam/backupv2/PrefixGenerator.java | 42 ---- .../priam/backupv2/SnapshotMetaTask.java | 178 ++++++++----- .../priam/compress/CompressionAlgorithm.java | 7 + .../netflix/priam/compress/ICompression.java | 6 - .../cryptography/CryptographyAlgorithm.java | 6 + .../priam/cryptography/IFileCryptography.java | 5 - .../priam/aws/TestRemoteBackupPath.java | 169 +++++++++---- .../com/netflix/priam/backup/TestBackup.java | 6 +- .../priam/backupv2/TestBackupUtils.java | 7 +- .../priam/backupv2/TestFileUploadResult.java | 73 ++++++ 17 files changed, 543 insertions(+), 477 deletions(-) create mode 100644 priam/src/main/java/com/netflix/priam/backup/BackupFolder.java delete mode 100644 priam/src/main/java/com/netflix/priam/backupv2/PrefixGenerator.java create mode 100644 priam/src/main/java/com/netflix/priam/compress/CompressionAlgorithm.java create mode 100644 priam/src/main/java/com/netflix/priam/cryptography/CryptographyAlgorithm.java create mode 100644 priam/src/test/java/com/netflix/priam/backupv2/TestFileUploadResult.java diff --git a/priam/src/main/java/com/netflix/priam/aws/RemoteBackupPath.java b/priam/src/main/java/com/netflix/priam/aws/RemoteBackupPath.java index b1e59e3d5..6a20ffc92 100644 --- a/priam/src/main/java/com/netflix/priam/aws/RemoteBackupPath.java +++ b/priam/src/main/java/com/netflix/priam/aws/RemoteBackupPath.java @@ -16,17 +16,20 @@ */ package com.netflix.priam.aws; +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; import com.google.inject.Inject; import com.netflix.priam.backup.AbstractBackupPath; -import com.netflix.priam.compress.ICompression; import com.netflix.priam.config.IConfiguration; -import com.netflix.priam.cryptography.IFileCryptography; import com.netflix.priam.identity.InstanceIdentity; import com.netflix.priam.utils.DateUtil; import java.nio.file.Path; import java.nio.file.Paths; import java.time.Instant; +import java.util.Arrays; import java.util.Date; +import java.util.Optional; /** * Represents location of an object on the remote file system. All the objects will be keyed with a @@ -34,42 +37,37 @@ * this instance. */ public class RemoteBackupPath extends AbstractBackupPath { + private static final ImmutableSet V2_ONLY_FILE_TYPES = + ImmutableSet.of( + BackupFileType.META_V2, + BackupFileType.SST_V2, + BackupFileType.SECONDARY_INDEX_V2); @Inject public RemoteBackupPath(IConfiguration config, InstanceIdentity factory) { super(config, factory); } - private Path getV2Prefix() { - return Paths.get(baseDir, getAppNameWithHash(clusterName), token); - } - - private void parseV2Prefix(Path remoteFilePath) { - if (remoteFilePath.getNameCount() < 3) - throw new RuntimeException( - "Not enough no. of elements to parseV2Prefix : " + remoteFilePath); - baseDir = remoteFilePath.getName(0).toString(); - clusterName = parseAndValidateAppNameWithHash(remoteFilePath.getName(1).toString()); - token = remoteFilePath.getName(2).toString(); + private ImmutableList.Builder getV2Prefix() { + ImmutableList.Builder prefix = ImmutableList.builder(); + prefix.add(baseDir, prependHash(clusterName), token); + return prefix; } /* This will ensure that there is some randomness in the path at the start so that remote file systems can hash the contents better when we have lot of clusters backing up at the same remote location. */ - private String getAppNameWithHash(String appName) { + private String prependHash(String appName) { return String.format("%d_%s", appName.hashCode() % 10000, appName); } - private String parseAndValidateAppNameWithHash(String appNameWithHash) { + private String removeHash(String appNameWithHash) { int hash = Integer.parseInt(appNameWithHash.substring(0, appNameWithHash.indexOf("_"))); String appName = appNameWithHash.substring(appNameWithHash.indexOf("_") + 1); - // Validate the hash - int calculatedHash = appName.hashCode() % 10000; - if (calculatedHash != hash) - throw new RuntimeException( - String.format( - "Hash for the app name: %s was calculated to be: %d but provided was: %d", - appName, calculatedHash, hash)); + Preconditions.checkArgument( + hash == appName.hashCode() % 10000, + "Prepended hash does not match app name. Should have received: %s", + prependHash(appName)); return appName; } @@ -81,93 +79,76 @@ private String parseAndValidateAppNameWithHash(String appNameWithHash) { * Another major difference w.r.t. V1 is having no distinction between SNAP and SST files as we upload SSTables only * once to remote file system. */ - private Path getV2Location() { - Path prefix = - Paths.get( - getV2Prefix().toString(), - type.toString(), - getLastModified().toEpochMilli() + ""); - - if (type == BackupFileType.SST_V2) { - prefix = Paths.get(prefix.toString(), keyspace, columnFamily); + private String getV2Location() { + ImmutableList.Builder parts = getV2Prefix(); + parts.add(type.toString(), getLastModified().toEpochMilli() + ""); + if (BackupFileType.isDataFile(type)) { + parts.add(keyspace, columnFamily); } - - return Paths.get( - prefix.toString(), - getCompression().toString(), - getEncryption().toString(), - fileName); + if (type == BackupFileType.SECONDARY_INDEX_V2) { + parts.add(indexDir); + } + parts.add(getCompression().toString(), getEncryption().toString(), fileName); + return toPath(parts.build()).toString(); + } + + private void parseV2Location(Path remotePath) { + Preconditions.checkArgument( + remotePath.getNameCount() >= 8, "%s has fewer than %d parts", remotePath, 8); + int index = 0; + baseDir = remotePath.getName(index++).toString(); + clusterName = removeHash(remotePath.getName(index++).toString()); + token = remotePath.getName(index++).toString(); + type = BackupFileType.valueOf(remotePath.getName(index++).toString()); + String lastModified = remotePath.getName(index++).toString(); + setLastModified(Instant.ofEpochMilli(Long.parseLong(lastModified))); + if (BackupFileType.isDataFile(type)) { + keyspace = remotePath.getName(index++).toString(); + columnFamily = remotePath.getName(index++).toString(); + } + if (type == BackupFileType.SECONDARY_INDEX_V2) { + indexDir = remotePath.getName(index++).toString(); + } + setCompression(remotePath.getName(index++).toString()); + setEncryption(remotePath.getName(index++).toString()); + fileName = remotePath.getName(index).toString(); } - private void parseV2Location(String remoteFile) { - Path remoteFilePath = Paths.get(remoteFile); - parseV2Prefix(remoteFilePath); - if (remoteFilePath.getNameCount() < 8) - throw new IndexOutOfBoundsException( - String.format( - "Too few elements (expected: [%d]) in path: %s", 8, remoteFilePath)); - int name_count_idx = 3; - - type = BackupFileType.valueOf(remoteFilePath.getName(name_count_idx++).toString()); - setLastModified( - Instant.ofEpochMilli( - Long.parseLong(remoteFilePath.getName(name_count_idx++).toString()))); - - if (type == BackupFileType.SST_V2) { - keyspace = remoteFilePath.getName(name_count_idx++).toString(); - columnFamily = remoteFilePath.getName(name_count_idx++).toString(); + private String getV1Location() { + ImmutableList.Builder parts = ImmutableList.builder(); + String timeString = DateUtil.formatyyyyMMddHHmm(time); + parts.add(baseDir, region, clusterName, token, timeString, type.toString()); + if (BackupFileType.isDataFile(type)) { + parts.add(keyspace, columnFamily); } - - setCompression( - ICompression.CompressionAlgorithm.valueOf( - remoteFilePath.getName(name_count_idx++).toString())); - - setEncryption( - IFileCryptography.CryptographyAlgorithm.valueOf( - remoteFilePath.getName(name_count_idx++).toString())); - fileName = remoteFilePath.getName(name_count_idx).toString(); + parts.add(fileName); + return toPath(parts.build()).toString(); } - private Path getV1Location() { - Path path = - Paths.get( - getV1Prefix().toString(), - DateUtil.formatyyyyMMddHHmm(time), - type.toString()); - if (BackupFileType.isDataFile(type)) - path = Paths.get(path.toString(), keyspace, columnFamily); - return Paths.get(path.toString(), fileName); + private Path toPath(ImmutableList parts) { + return Paths.get(parts.get(0), parts.subList(1, parts.size()).toArray(new String[0])); } - private void parseV1Location(Path remoteFilePath) { - parseV1Prefix(remoteFilePath); - if (remoteFilePath.getNameCount() < 7) - throw new IndexOutOfBoundsException( - String.format( - "Too few elements (expected: [%d]) in path: %s", 7, remoteFilePath)); - - time = DateUtil.getDate(remoteFilePath.getName(4).toString()); - type = BackupFileType.valueOf(remoteFilePath.getName(5).toString()); + private void parseV1Location(Path remotePath) { + Preconditions.checkArgument( + remotePath.getNameCount() >= 7, "%s has fewer than %d parts", remotePath, 7); + parseV1Prefix(remotePath); + time = DateUtil.getDate(remotePath.getName(4).toString()); + type = BackupFileType.valueOf(remotePath.getName(5).toString()); if (BackupFileType.isDataFile(type)) { - keyspace = remoteFilePath.getName(6).toString(); - columnFamily = remoteFilePath.getName(7).toString(); + keyspace = remotePath.getName(6).toString(); + columnFamily = remotePath.getName(7).toString(); } - // append the rest - fileName = remoteFilePath.getName(remoteFilePath.getNameCount() - 1).toString(); + fileName = remotePath.getName(remotePath.getNameCount() - 1).toString(); } - private Path getV1Prefix() { - return Paths.get(baseDir, region, clusterName, token); - } - - private void parseV1Prefix(Path remoteFilePath) { - if (remoteFilePath.getNameCount() < 4) - throw new RuntimeException( - "Not enough no. of elements to parseV1Prefix : " + remoteFilePath); - baseDir = remoteFilePath.getName(0).toString(); - region = remoteFilePath.getName(1).toString(); - clusterName = remoteFilePath.getName(2).toString(); - token = remoteFilePath.getName(3).toString(); + private void parseV1Prefix(Path remotePath) { + Preconditions.checkArgument( + remotePath.getNameCount() >= 4, "%s needs %d parts to parse prefix", remotePath, 4); + baseDir = remotePath.getName(0).toString(); + region = remotePath.getName(1).toString(); + clusterName = remotePath.getName(2).toString(); + token = remotePath.getName(3).toString(); } /** @@ -179,29 +160,21 @@ private void parseV1Prefix(Path remoteFilePath) { */ @Override public String getRemotePath() { - if (type == BackupFileType.SST_V2 || type == BackupFileType.META_V2) { - return getV2Location().toString(); - } else { - return getV1Location().toString(); - } + return V2_ONLY_FILE_TYPES.contains(type) ? getV2Location() : getV1Location(); } @Override - public void parseRemote(String remoteFilePath) { - // Check for all backup file types to ensure how we parse - // TODO: We should clean this hack to get backupFileType for parsing when we delete V1 of - // backups. - for (BackupFileType fileType : BackupFileType.values()) { - if (remoteFilePath.contains(PATH_SEP + fileType.toString() + PATH_SEP)) { - type = fileType; - break; - } - } - - if (type == BackupFileType.SST_V2 || type == BackupFileType.META_V2) { - parseV2Location(remoteFilePath); + public void parseRemote(String remotePath) { + // Hack to determine type in advance of parsing. Will disappear once v1 is retired + Optional inferredType = + Arrays.stream(BackupFileType.values()) + .filter(bft -> remotePath.contains(PATH_SEP + bft.toString() + PATH_SEP)) + .findAny() + .filter(V2_ONLY_FILE_TYPES::contains); + if (inferredType.isPresent()) { + parseV2Location(Paths.get(remotePath)); } else { - parseV1Location(Paths.get(remoteFilePath)); + parseV1Location(Paths.get(remotePath)); } } @@ -212,12 +185,10 @@ public void parsePartialPrefix(String remoteFilePath) { @Override public String remotePrefix(Date start, Date end, String location) { - StringBuilder buff = new StringBuilder(clusterPrefix(location)); - token = instanceIdentity.getInstance().getToken(); - buff.append(token).append(RemoteBackupPath.PATH_SEP); - // match the common characters to prefix. - buff.append(match(start, end)); - return buff.toString(); + return PATH_JOINER.join( + clusterPrefix(location), + instanceIdentity.getInstance().getToken(), + match(start, end)); } @Override @@ -227,33 +198,30 @@ public Path remoteV2Prefix(Path location, BackupFileType fileType) { clusterName = config.getAppName(); } else if (location.getNameCount() >= 3) { baseDir = location.getName(1).toString(); - clusterName = parseAndValidateAppNameWithHash(location.getName(2).toString()); + clusterName = removeHash(location.getName(2).toString()); } token = instanceIdentity.getInstance().getToken(); - return Paths.get(getV2Prefix().toString(), fileType.toString()); + ImmutableList.Builder parts = getV2Prefix(); + parts.add(fileType.toString()); + return toPath(parts.build()); } @Override public String clusterPrefix(String location) { - StringBuilder buff = new StringBuilder(); String[] elements = location.split(String.valueOf(RemoteBackupPath.PATH_SEP)); + Preconditions.checkArgument( + elements.length < 2 || elements.length > 3, + "Path must have fewer than 2 or greater than 3 elements. Saw %s", + location); if (elements.length <= 1) { baseDir = config.getBackupLocation(); region = instanceIdentity.getInstanceInfo().getRegion(); clusterName = config.getAppName(); } else { - if (elements.length < 4) - throw new IndexOutOfBoundsException( - String.format( - "Too few elements (expected: [%d]) in path: %s", 4, location)); baseDir = elements[1]; region = elements[2]; clusterName = elements[3]; } - buff.append(baseDir).append(RemoteBackupPath.PATH_SEP); - buff.append(region).append(RemoteBackupPath.PATH_SEP); - buff.append(clusterName).append(RemoteBackupPath.PATH_SEP); - - return buff.toString(); + return PATH_JOINER.join(baseDir, region, clusterName, ""); // "" ensures a trailing "/" } } diff --git a/priam/src/main/java/com/netflix/priam/backup/AbstractBackup.java b/priam/src/main/java/com/netflix/priam/backup/AbstractBackup.java index b4a222d6d..63e9b907f 100644 --- a/priam/src/main/java/com/netflix/priam/backup/AbstractBackup.java +++ b/priam/src/main/java/com/netflix/priam/backup/AbstractBackup.java @@ -144,7 +144,7 @@ protected final void initiateBackup( for (File columnFamilyDir : columnFamilyDirectories) { File backupDir = new File(columnFamilyDir, monitoringFolder); - if (backupDir.exists() && backupDir.isDirectory() && backupDir.canRead()) { + if (isAReadableDirectory(backupDir)) { String columnFamilyName = columnFamilyDir.getName().split("-")[0]; if (backupRestoreUtil.isFiltered(keyspaceDir.getName(), columnFamilyName)) { // Clean the backup/snapshot directory else files will keep getting @@ -202,4 +202,8 @@ public static Set getBackupDirectories(IConfiguration config, String monit } return backupPaths; } + + protected static boolean isAReadableDirectory(File dir) { + return dir.exists() && dir.isDirectory() && dir.canRead(); + } } diff --git a/priam/src/main/java/com/netflix/priam/backup/AbstractBackupPath.java b/priam/src/main/java/com/netflix/priam/backup/AbstractBackupPath.java index 021817d1f..d02a0502d 100644 --- a/priam/src/main/java/com/netflix/priam/backup/AbstractBackupPath.java +++ b/priam/src/main/java/com/netflix/priam/backup/AbstractBackupPath.java @@ -16,41 +16,45 @@ */ package com.netflix.priam.backup; +import com.google.common.base.Joiner; +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; import com.google.inject.ImplementedBy; import com.netflix.priam.aws.RemoteBackupPath; -import com.netflix.priam.compress.ICompression; +import com.netflix.priam.compress.CompressionAlgorithm; import com.netflix.priam.config.IConfiguration; -import com.netflix.priam.cryptography.IFileCryptography; +import com.netflix.priam.cryptography.CryptographyAlgorithm; import com.netflix.priam.identity.InstanceIdentity; import com.netflix.priam.utils.DateUtil; import java.io.File; import java.nio.file.Path; -import java.text.ParseException; import java.time.Instant; import java.util.Date; import org.apache.commons.lang3.StringUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; @ImplementedBy(RemoteBackupPath.class) public abstract class AbstractBackupPath implements Comparable { - private static final Logger logger = LoggerFactory.getLogger(AbstractBackupPath.class); public static final char PATH_SEP = File.separatorChar; + public static final Joiner PATH_JOINER = Joiner.on(PATH_SEP); + private static final ImmutableMap FOLDER_POSITIONS = + ImmutableMap.of(BackupFolder.BACKUPS, 3, BackupFolder.SNAPSHOTS, 4); public enum BackupFileType { - SNAP, - SST, CL, META, META_V2, - SST_V2, - SNAPSHOT_VERIFIED; + SECONDARY_INDEX_V2, + SNAP, + SNAPSHOT_VERIFIED, + SST, + SST_V2; + + private static ImmutableSet DATA_FILE_TYPES = + ImmutableSet.of(SECONDARY_INDEX_V2, SNAP, SST, SST_V2); public static boolean isDataFile(BackupFileType type) { - return type != BackupFileType.META - && type != BackupFileType.META_V2 - && type != BackupFileType.CL - && type != SNAPSHOT_VERIFIED; + return DATA_FILE_TYPES.contains(type); } public static BackupFileType fromString(String s) throws BackupRestoreException { @@ -70,6 +74,7 @@ public static BackupFileType fromString(String s) throws BackupRestoreException protected String baseDir; protected String token; protected String region; + protected String indexDir; protected Date time; private long size; // uncompressed file size private long compressedFileSize = 0; @@ -78,44 +83,47 @@ public static BackupFileType fromString(String s) throws BackupRestoreException private File backupFile; private Instant lastModified; private Date uploadedTs; - private ICompression.CompressionAlgorithm compression = - ICompression.CompressionAlgorithm.SNAPPY; - private IFileCryptography.CryptographyAlgorithm encryption = - IFileCryptography.CryptographyAlgorithm.PLAINTEXT; + private CompressionAlgorithm compression = CompressionAlgorithm.SNAPPY; + private CryptographyAlgorithm encryption = CryptographyAlgorithm.PLAINTEXT; public AbstractBackupPath(IConfiguration config, InstanceIdentity instanceIdentity) { this.instanceIdentity = instanceIdentity; this.config = config; } - public void parseLocal(File file, BackupFileType type) throws ParseException { + public void parseLocal(File file, BackupFileType type) { this.backupFile = file; - - String rpath = - new File(config.getDataFileLocation()).toURI().relativize(file.toURI()).getPath(); - String[] elements = rpath.split("" + PATH_SEP); - this.clusterName = config.getAppName(); this.baseDir = config.getBackupLocation(); + this.clusterName = config.getAppName(); + this.fileName = file.getName(); + this.lastModified = Instant.ofEpochMilli(file.lastModified()); this.region = instanceIdentity.getInstanceInfo().getRegion(); + this.size = file.length(); this.token = instanceIdentity.getInstance().getToken(); this.type = type; + + String rpath = + new File(config.getDataFileLocation()).toURI().relativize(file.toURI()).getPath(); + String[] parts = rpath.split("" + PATH_SEP); if (BackupFileType.isDataFile(type)) { - this.keyspace = elements[0]; - this.columnFamily = elements[1]; + this.keyspace = parts[0]; + this.columnFamily = parts[1]; + } + if (type == BackupFileType.SECONDARY_INDEX_V2) { + Integer index = BackupFolder.fromName(parts[2]).map(FOLDER_POSITIONS::get).orElse(null); + Preconditions.checkNotNull(index, "Unrecognized backup folder " + parts[2]); + this.indexDir = parts[index]; } - - time = new Date(file.lastModified()); /* 1. For old style snapshots, make this value to time at which backup was executed. 2. This is to ensure that all the files from the snapshot are uploaded under single directory in remote file system. - 3. For META file we always override the time field via @link{Metadata#decorateMetaJson} + 3. For META files we always override the time field via @link{Metadata#decorateMetaJson} */ - if (type == BackupFileType.SNAP) time = DateUtil.getDate(elements[3]); - - this.lastModified = Instant.ofEpochMilli(file.lastModified()); - this.fileName = file.getName(); - this.size = file.length(); + this.time = + type == BackupFileType.SNAP + ? DateUtil.getDate(parts[3]) + : new Date(file.lastModified()); } /** Given a date range, find a common string prefix Eg: 20120212, 20120213 = 2012021 */ @@ -129,18 +137,24 @@ protected String match(Date start, Date end) { /** Local restore file */ public File newRestoreFile() { - StringBuilder buff = new StringBuilder(); - if (type == BackupFileType.CL) { - buff.append(config.getBackupCommitLogLocation()).append(PATH_SEP); - } else { - buff.append(config.getDataFileLocation()).append(PATH_SEP); - if (type != BackupFileType.META && type != BackupFileType.META_V2) - buff.append(keyspace).append(PATH_SEP).append(columnFamily).append(PATH_SEP); + File return_; + String dataDir = config.getDataFileLocation(); + switch (type) { + case CL: + return_ = new File(PATH_JOINER.join(config.getBackupCommitLogLocation(), fileName)); + break; + case SECONDARY_INDEX_V2: + String restoreFileName = + PATH_JOINER.join(dataDir, keyspace, columnFamily, indexDir, fileName); + return_ = new File(restoreFileName); + break; + case META: + case META_V2: + return_ = new File(PATH_JOINER.join(config.getDataFileLocation(), fileName)); + break; + default: + return_ = new File(PATH_JOINER.join(dataDir, keyspace, columnFamily, fileName)); } - - buff.append(fileName); - - File return_ = new File(buff.toString()); File parent = new File(return_.getParent()); if (!parent.exists()) parent.mkdirs(); return return_; @@ -200,10 +214,6 @@ public String getFileName() { return fileName; } - public String getBaseDir() { - return baseDir; - } - public String getToken() { return token; } @@ -267,20 +277,20 @@ public void setLastModified(Instant instant) { this.lastModified = instant; } - public ICompression.CompressionAlgorithm getCompression() { + public CompressionAlgorithm getCompression() { return compression; } - public void setCompression(ICompression.CompressionAlgorithm compression) { - this.compression = compression; + public void setCompression(String compression) { + this.compression = CompressionAlgorithm.valueOf(compression); } - public IFileCryptography.CryptographyAlgorithm getEncryption() { + public CryptographyAlgorithm getEncryption() { return encryption; } - public void setEncryption(IFileCryptography.CryptographyAlgorithm encryption) { - this.encryption = encryption; + public void setEncryption(String encryption) { + this.encryption = CryptographyAlgorithm.valueOf(encryption); } @Override diff --git a/priam/src/main/java/com/netflix/priam/backup/BackupFolder.java b/priam/src/main/java/com/netflix/priam/backup/BackupFolder.java new file mode 100644 index 000000000..14a4cf6b1 --- /dev/null +++ b/priam/src/main/java/com/netflix/priam/backup/BackupFolder.java @@ -0,0 +1,18 @@ +package com.netflix.priam.backup; + +import java.util.Arrays; +import java.util.Optional; + +public enum BackupFolder { + SNAPSHOTS("snapshots"), + BACKUPS("backups"); + private String name; + + BackupFolder(String name) { + this.name = name; + } + + public static Optional fromName(String name) { + return Arrays.stream(values()).filter(b -> b.name.equals(name)).findFirst(); + } +} diff --git a/priam/src/main/java/com/netflix/priam/backup/IncrementalBackup.java b/priam/src/main/java/com/netflix/priam/backup/IncrementalBackup.java index ef8d9740a..dc6d338be 100644 --- a/priam/src/main/java/com/netflix/priam/backup/IncrementalBackup.java +++ b/priam/src/main/java/com/netflix/priam/backup/IncrementalBackup.java @@ -26,7 +26,9 @@ import com.netflix.priam.scheduler.SimpleTimer; import com.netflix.priam.scheduler.TaskTimer; import java.io.File; +import java.io.FileFilter; import java.nio.file.Path; +import java.util.Optional; import java.util.Set; import org.apache.commons.io.FileUtils; import org.slf4j.Logger; @@ -112,9 +114,18 @@ public String getName() { @Override protected void processColumnFamily(String keyspace, String columnFamily, File backupDir) throws Exception { - BackupFileType fileType = BackupFileType.SST; - if (backupRestoreConfig.enableV2Backups()) fileType = BackupFileType.SST_V2; + BackupFileType fileType = + backupRestoreConfig.enableV2Backups() ? BackupFileType.SST_V2 : BackupFileType.SST; + // upload SSTables and components upload(backupDir, fileType, config.enableAsyncIncremental(), true); + + // Next, upload secondary indexes + FileFilter filter = + (file) -> + file.getName().startsWith("." + columnFamily) && isAReadableDirectory(file); + for (File subDir : Optional.ofNullable(backupDir.listFiles(filter)).orElse(new File[] {})) { + upload(subDir, fileType, config.enableAsyncIncremental(), true); + } } } diff --git a/priam/src/main/java/com/netflix/priam/backupv2/ColumnfamilyResult.java b/priam/src/main/java/com/netflix/priam/backupv2/ColumnfamilyResult.java index a0d4242da..8f637d53e 100644 --- a/priam/src/main/java/com/netflix/priam/backupv2/ColumnfamilyResult.java +++ b/priam/src/main/java/com/netflix/priam/backupv2/ColumnfamilyResult.java @@ -13,9 +13,11 @@ */ package com.netflix.priam.backupv2; +import com.google.common.collect.ImmutableSet; import com.netflix.priam.utils.GsonJsonSerializer; import java.util.ArrayList; import java.util.List; +import java.util.Set; /** * This is a POJO to encapsulate all the SSTables for a given column family. Created by aagrawal on @@ -31,22 +33,6 @@ public ColumnfamilyResult(String keyspaceName, String columnfamilyName) { this.columnfamilyName = columnfamilyName; } - public String getKeyspaceName() { - return keyspaceName; - } - - public void setKeyspaceName(String keyspaceName) { - this.keyspaceName = keyspaceName; - } - - public String getColumnfamilyName() { - return columnfamilyName; - } - - public void setColumnfamilyName(String columnfamilyName) { - this.columnfamilyName = columnfamilyName; - } - public List getSstables() { return sstables; } @@ -68,21 +54,17 @@ public String toString() { /** This is a POJO to encapsulate a SSTable and all its components. */ public static class SSTableResult { private String prefix; - private List sstableComponents; - - public String getPrefix() { - return prefix; - } + private Set sstableComponents; public void setPrefix(String prefix) { this.prefix = prefix; } - public List getSstableComponents() { + public Set getSstableComponents() { return sstableComponents; } - public void setSstableComponents(List sstableComponents) { + public void setSstableComponents(ImmutableSet sstableComponents) { this.sstableComponents = sstableComponents; } diff --git a/priam/src/main/java/com/netflix/priam/backupv2/FileUploadResult.java b/priam/src/main/java/com/netflix/priam/backupv2/FileUploadResult.java index d0228cb4f..26e866af0 100644 --- a/priam/src/main/java/com/netflix/priam/backupv2/FileUploadResult.java +++ b/priam/src/main/java/com/netflix/priam/backupv2/FileUploadResult.java @@ -16,13 +16,10 @@ */ package com.netflix.priam.backupv2; -import com.netflix.priam.compress.ICompression; -import com.netflix.priam.cryptography.IFileCryptography; +import com.netflix.priam.compress.CompressionAlgorithm; +import com.netflix.priam.cryptography.CryptographyAlgorithm; import com.netflix.priam.utils.GsonJsonSerializer; -import java.io.File; -import java.nio.file.Files; import java.nio.file.Path; -import java.nio.file.attribute.BasicFileAttributes; import java.time.Instant; /** @@ -30,100 +27,32 @@ */ public class FileUploadResult { private final Path fileName; - @GsonJsonSerializer.PriamAnnotation.GsonIgnore private String keyspaceName; - @GsonJsonSerializer.PriamAnnotation.GsonIgnore private String columnFamilyName; - private Instant lastModifiedTime; + private final Instant lastModifiedTime; private final Instant fileCreationTime; private final long fileSizeOnDisk; // Size on disk in bytes - private Boolean isUploaded; // Valid compression technique for now is SNAPPY only. Future we need to support LZ4 and NONE - private ICompression.CompressionAlgorithm compression = - ICompression.CompressionAlgorithm.SNAPPY; + private final CompressionAlgorithm compression = CompressionAlgorithm.SNAPPY; // Valid encryption technique for now is PLAINTEXT only. In future we will support pgp and more. - private IFileCryptography.CryptographyAlgorithm encryption = - IFileCryptography.CryptographyAlgorithm.PLAINTEXT; + private final CryptographyAlgorithm encryption = CryptographyAlgorithm.PLAINTEXT; + + private Boolean isUploaded; private String backupPath; public FileUploadResult( Path fileName, - String keyspaceName, - String columnFamilyName, Instant lastModifiedTime, Instant fileCreationTime, long fileSizeOnDisk) { this.fileName = fileName; - this.keyspaceName = keyspaceName; - this.columnFamilyName = columnFamilyName; this.lastModifiedTime = lastModifiedTime; this.fileCreationTime = fileCreationTime; this.fileSizeOnDisk = fileSizeOnDisk; } - private static FileUploadResult getFileUploadResult( - String keyspaceName, String columnFamilyName, Path file) throws Exception { - BasicFileAttributes fileAttributes = Files.readAttributes(file, BasicFileAttributes.class); - return new FileUploadResult( - file, - keyspaceName, - columnFamilyName, - fileAttributes.lastModifiedTime().toInstant(), - fileAttributes.creationTime().toInstant(), - fileAttributes.size()); - } - - public static FileUploadResult getFileUploadResult( - String keyspaceName, String columnFamilyName, File file) throws Exception { - return getFileUploadResult(keyspaceName, columnFamilyName, file.toPath()); - } - - public Path getFileName() { - return fileName; - } - - public String getKeyspaceName() { - return keyspaceName; - } - - public String getColumnFamilyName() { - return columnFamilyName; - } - - public Instant getLastModifiedTime() { - return lastModifiedTime; - } - - public Instant getFileCreationTime() { - return fileCreationTime; - } - - public long getFileSizeOnDisk() { - return fileSizeOnDisk; - } - - public Boolean getUploaded() { - return isUploaded; - } - public void setUploaded(Boolean uploaded) { isUploaded = uploaded; } - public ICompression.CompressionAlgorithm getCompression() { - return compression; - } - - public void setCompression(ICompression.CompressionAlgorithm compression) { - this.compression = compression; - } - - public IFileCryptography.CryptographyAlgorithm getEncryption() { - return encryption; - } - - public void setEncryption(IFileCryptography.CryptographyAlgorithm encryption) { - this.encryption = encryption; - } - public String getBackupPath() { return backupPath; } @@ -132,19 +61,6 @@ public void setBackupPath(String backupPath) { this.backupPath = backupPath; } - public void setLastModifiedTime(Instant lastModifiedTime) { - this.lastModifiedTime = lastModifiedTime; - } - - public void setKeyspaceName(String keyspaceName) { - - this.keyspaceName = keyspaceName; - } - - public void setColumnFamilyName(String columnFamilyName) { - this.columnFamilyName = columnFamilyName; - } - @Override public String toString() { return GsonJsonSerializer.getGson().toJson(this); diff --git a/priam/src/main/java/com/netflix/priam/backupv2/PrefixGenerator.java b/priam/src/main/java/com/netflix/priam/backupv2/PrefixGenerator.java deleted file mode 100644 index 8cede4311..000000000 --- a/priam/src/main/java/com/netflix/priam/backupv2/PrefixGenerator.java +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Copyright 2018 Netflix, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ -package com.netflix.priam.backupv2; - -/** - * This is a utility class to get the backup location of the SSTables/Meta files with backup version - * 2.0. - */ -public class PrefixGenerator { - /** - * Gives the prefix (common name) of the sstable components. Returns null if it is not sstable - * component e.g. mc-3-big-Data.db or ks-cf-ka-7213-Index.db will return mc-3-big or - * ks-cf-ka-7213 - * - * @param fileName name of the file for common prefix - * @return common prefix of the file, or null, if not identified as sstable component. - */ - public static final String getSSTFileBase(String fileName) { - String prefix = null; - try { - prefix = fileName.substring(0, fileName.lastIndexOf("-")); - } catch (IndexOutOfBoundsException e) { - // Do nothing - } - - return prefix; - } -} diff --git a/priam/src/main/java/com/netflix/priam/backupv2/SnapshotMetaTask.java b/priam/src/main/java/com/netflix/priam/backupv2/SnapshotMetaTask.java index aea7b8238..5ac26dc8f 100644 --- a/priam/src/main/java/com/netflix/priam/backupv2/SnapshotMetaTask.java +++ b/priam/src/main/java/com/netflix/priam/backupv2/SnapshotMetaTask.java @@ -16,9 +16,9 @@ */ package com.netflix.priam.backupv2; +import com.google.common.collect.ImmutableSetMultimap; import com.google.inject.Provider; import com.netflix.priam.backup.*; -import com.netflix.priam.backup.BackupVersion; import com.netflix.priam.config.IBackupRestoreConfig; import com.netflix.priam.config.IConfiguration; import com.netflix.priam.connection.CassandraOperations; @@ -28,14 +28,20 @@ import com.netflix.priam.scheduler.TaskTimer; import com.netflix.priam.utils.DateUtil; import java.io.File; +import java.io.IOException; import java.nio.file.DirectoryStream; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; +import java.nio.file.attribute.BasicFileAttributes; import java.time.Instant; -import java.util.*; +import java.util.Date; +import java.util.List; +import java.util.Optional; +import java.util.Set; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; +import java.util.stream.Collectors; import javax.inject.Inject; import javax.inject.Named; import javax.inject.Singleton; @@ -252,22 +258,22 @@ public String getName() { private ColumnfamilyResult convertToColumnFamilyResult( String keyspace, String columnFamilyName, - Map> filePrefixToFileMap) { + ImmutableSetMultimap filePrefixToFileMap) { ColumnfamilyResult columnfamilyResult = new ColumnfamilyResult(keyspace, columnFamilyName); - filePrefixToFileMap.forEach( - (key, value) -> { - ColumnfamilyResult.SSTableResult ssTableResult = - new ColumnfamilyResult.SSTableResult(); - ssTableResult.setPrefix(key); - ssTableResult.setSstableComponents(value); - columnfamilyResult.addSstable(ssTableResult); - }); + filePrefixToFileMap + .keySet() + .forEach( + (key) -> { + ColumnfamilyResult.SSTableResult ssTableResult = + new ColumnfamilyResult.SSTableResult(); + ssTableResult.setPrefix(key); + ssTableResult.setSstableComponents(filePrefixToFileMap.get(key)); + columnfamilyResult.addSstable(ssTableResult); + }); return columnfamilyResult; } - private void uploadAllFiles( - final String keyspace, final String columnFamily, final File backupDir) - throws Exception { + private void uploadAllFiles(final String columnFamily, final File backupDir) throws Exception { // Process all the snapshots with SNAPSHOT_PREFIX. This will ensure that we "resume" the // uploads of previous snapshot leftover as Priam restarted or any failure for any reason // (like we exhausted the wait time for upload) @@ -288,6 +294,15 @@ private void uploadAllFiles( // We do not want to wait for completion and we just want to add them to queue. This // is to ensure that next run happens on time. upload(snapshotDirectory, AbstractBackupPath.BackupFileType.SST_V2, true, false); + + // Next, upload secondary indexes + for (File subDir : getSecondaryIndexDirectories(snapshotDirectory, columnFamily)) { + upload( + subDir, + AbstractBackupPath.BackupFileType.SECONDARY_INDEX_V2, + true, + false); + } } } } @@ -301,7 +316,7 @@ protected void processColumnFamily( generateMetaFile(keyspace, columnFamily, backupDir); break; case UPLOAD_FILES: - uploadAllFiles(keyspace, columnFamily, backupDir); + uploadAllFiles(columnFamily, backupDir); break; default: throw new Exception("Unknown meta file type: " + metaStep); @@ -319,48 +334,47 @@ private void generateMetaFile( } logger.debug("Scanning for all SSTables in: {}", snapshotDir.getAbsolutePath()); + ImmutableSetMultimap.Builder filePrefixToFileMap = + ImmutableSetMultimap.builder(); + filePrefixToFileMap.putAll( + getFileUploadResults(snapshotDir, AbstractBackupPath.BackupFileType.SST_V2)); + + // Next, add secondary indexes + for (File subDir : getSecondaryIndexDirectories(snapshotDir, columnFamily)) { + filePrefixToFileMap.putAll( + getFileUploadResults( + subDir, AbstractBackupPath.BackupFileType.SECONDARY_INDEX_V2)); + } - Map> filePrefixToFileMap = new HashMap<>(); - Collection files = - FileUtils.listFiles(snapshotDir, FileFilterUtils.fileFileFilter(), null); - - for (File file : files) { - if (!file.exists()) continue; - - try { - String prefix = PrefixGenerator.getSSTFileBase(file.getName()); - - if (prefix == null && file.getName().equalsIgnoreCase(CASSANDRA_MANIFEST_FILE)) - prefix = "manifest"; + ColumnfamilyResult columnfamilyResult = + convertToColumnFamilyResult(keyspace, columnFamily, filePrefixToFileMap.build()); - if (prefix == null && file.getName().equalsIgnoreCase(CASSANDRA_SCHEMA_FILE)) - prefix = "schema"; + int sstableCount = columnfamilyResult.getSstables().size(); + logger.debug("Processing {} sstables from {}.{}", keyspace, columnFamily, sstableCount); - if (prefix == null) { - logger.error( - "Unknown file type with no SSTFileBase found: {}", - file.getAbsolutePath()); - continue; - } + dataStep.addColumnfamilyResult(columnfamilyResult); + logger.debug("Finished processing KS: {}, CF: {}", keyspace, columnFamily); + } - FileUploadResult fileUploadResult = - FileUploadResult.getFileUploadResult(keyspace, columnFamily, file); - // Add isUploaded and remotePath here. - try { - AbstractBackupPath abstractBackupPath = pathFactory.get(); - abstractBackupPath.parseLocal(file, AbstractBackupPath.BackupFileType.SST_V2); - fileUploadResult.setBackupPath(abstractBackupPath.getRemotePath()); - fileUploadResult.setUploaded( - fs.checkObjectExists(Paths.get(fileUploadResult.getBackupPath()))); - } catch (Exception e) { - logger.error( - "Error while setting the remoteLocation or checking if file exists. Ignoring them as they are not fatal.", - e.getMessage()); - e.printStackTrace(); - } + ImmutableSetMultimap getFileUploadResults( + File snapshotDir, AbstractBackupPath.BackupFileType type) throws Exception { + ImmutableSetMultimap.Builder filePrefixToFileMap = + ImmutableSetMultimap.builder(); + for (File file : FileUtils.listFiles(snapshotDir, FileFilterUtils.fileFileFilter(), null)) { + if (!file.exists()) continue; + Optional prefix = getPrefix(file); + if (!prefix.isPresent()) continue; - filePrefixToFileMap.putIfAbsent(prefix, new ArrayList<>()); - filePrefixToFileMap.get(prefix).add(fileUploadResult); + FileUploadResult fileUploadResult; + try { + BasicFileAttributes fileAttributes = + Files.readAttributes(file.toPath(), BasicFileAttributes.class); + fileUploadResult = + new FileUploadResult( + file.toPath(), + fileAttributes.lastModifiedTime().toInstant(), + fileAttributes.creationTime().toInstant(), + fileAttributes.size()); } catch (Exception e) { /* If you are here it means either of the issues. In that case, do not upload the meta file. * @throws UnsupportedOperationException @@ -380,23 +394,53 @@ private void generateMetaFile( e); throw e; } + // Add isUploaded and remotePath here. + try { + AbstractBackupPath abstractBackupPath = pathFactory.get(); + abstractBackupPath.parseLocal(file, type); + fileUploadResult.setBackupPath(abstractBackupPath.getRemotePath()); + fileUploadResult.setUploaded( + fs.checkObjectExists(Paths.get(fileUploadResult.getBackupPath()))); + } catch (Exception e) { + logger.error( + "Error while setting the remoteLocation or checking if file exists. Ignoring them as they are not fatal.", + e.getMessage()); + e.printStackTrace(); + } + filePrefixToFileMap.put(prefix.get(), fileUploadResult); } + return filePrefixToFileMap.build(); + } + /** + * Gives the prefix (common name) of the sstable components. Returns an empty Optional if it is + * not an sstable component or a manifest or schema file. + * + *

    For example: mc-3-big-Data.db --> mc-3-big ks-cf-ka-7213-Index.db --> ks-cf-ka-7213 + * + * @param file the file from which to extract a common prefix. + * @return common prefix of the file, or empty, + */ + public static Optional getPrefix(File file) { + String fileName = file.getName(); + String prefix = null; + if (fileName.contains("-")) { + prefix = fileName.substring(0, fileName.lastIndexOf("-")); + } else if (fileName.equalsIgnoreCase(CASSANDRA_MANIFEST_FILE)) { + prefix = "manifest"; + } else if (fileName.equalsIgnoreCase(CASSANDRA_SCHEMA_FILE)) { + prefix = "schema"; + } else { + logger.error("Unknown file type with no SSTFileBase found: {}", file.getAbsolutePath()); + } + return Optional.ofNullable(prefix); + } - ColumnfamilyResult columnfamilyResult = - convertToColumnFamilyResult(keyspace, columnFamily, filePrefixToFileMap); - filePrefixToFileMap.clear(); // Release the resources. - - logger.debug( - "Starting the processing of KS: {}, CF: {}, No.of SSTables: {}", - columnfamilyResult.getKeyspaceName(), - columnfamilyResult.getColumnfamilyName(), - columnfamilyResult.getSstables().size()); - - dataStep.addColumnfamilyResult(columnfamilyResult); - logger.debug( - "Finished processing KS: {}, CF: {}", - columnfamilyResult.getKeyspaceName(), - columnfamilyResult.getColumnfamilyName()); + private List getSecondaryIndexDirectories(File snapshotDir, String columnFamily) + throws IOException { + return Files.walk(snapshotDir.toPath(), Integer.MAX_VALUE) + .map(Path::toFile) + .filter(f -> f.getName().startsWith("." + columnFamily) && isAReadableDirectory(f)) + .collect(Collectors.toList()); } // For testing purposes only. diff --git a/priam/src/main/java/com/netflix/priam/compress/CompressionAlgorithm.java b/priam/src/main/java/com/netflix/priam/compress/CompressionAlgorithm.java new file mode 100644 index 000000000..c9244899f --- /dev/null +++ b/priam/src/main/java/com/netflix/priam/compress/CompressionAlgorithm.java @@ -0,0 +1,7 @@ +package com.netflix.priam.compress; + +public enum CompressionAlgorithm { + SNAPPY, + LZ4, + NONE +} diff --git a/priam/src/main/java/com/netflix/priam/compress/ICompression.java b/priam/src/main/java/com/netflix/priam/compress/ICompression.java index bab9b0f0f..47f9f31b0 100644 --- a/priam/src/main/java/com/netflix/priam/compress/ICompression.java +++ b/priam/src/main/java/com/netflix/priam/compress/ICompression.java @@ -25,12 +25,6 @@ @ImplementedBy(SnappyCompression.class) public interface ICompression { - enum CompressionAlgorithm { - SNAPPY, - LZ4, - NONE - } - /** * Uncompress the input stream and write to the output stream. Closes both input and output * streams diff --git a/priam/src/main/java/com/netflix/priam/cryptography/CryptographyAlgorithm.java b/priam/src/main/java/com/netflix/priam/cryptography/CryptographyAlgorithm.java new file mode 100644 index 000000000..04940286b --- /dev/null +++ b/priam/src/main/java/com/netflix/priam/cryptography/CryptographyAlgorithm.java @@ -0,0 +1,6 @@ +package com.netflix.priam.cryptography; + +public enum CryptographyAlgorithm { + PLAINTEXT, + PGP +} diff --git a/priam/src/main/java/com/netflix/priam/cryptography/IFileCryptography.java b/priam/src/main/java/com/netflix/priam/cryptography/IFileCryptography.java index 7cf26ad5c..3e8d0d58b 100755 --- a/priam/src/main/java/com/netflix/priam/cryptography/IFileCryptography.java +++ b/priam/src/main/java/com/netflix/priam/cryptography/IFileCryptography.java @@ -18,11 +18,6 @@ public interface IFileCryptography { - enum CryptographyAlgorithm { - PLAINTEXT, - PGP - } - /** * @param in - a handle to the encrypted, compressed data stream * @param passwd - pass phrase used to extract the PGP private key from the encrypted content. diff --git a/priam/src/test/java/com/netflix/priam/aws/TestRemoteBackupPath.java b/priam/src/test/java/com/netflix/priam/aws/TestRemoteBackupPath.java index 67e1d0dc7..ad845914c 100644 --- a/priam/src/test/java/com/netflix/priam/aws/TestRemoteBackupPath.java +++ b/priam/src/test/java/com/netflix/priam/aws/TestRemoteBackupPath.java @@ -18,20 +18,18 @@ package com.netflix.priam.aws; import com.google.inject.Guice; -import com.google.inject.Injector; import com.google.inject.Provider; import com.netflix.priam.backup.AbstractBackupPath; import com.netflix.priam.backup.AbstractBackupPath.BackupFileType; import com.netflix.priam.backup.BRTestModule; -import com.netflix.priam.config.IConfiguration; -import com.netflix.priam.cryptography.IFileCryptography; +import com.netflix.priam.cryptography.CryptographyAlgorithm; import com.netflix.priam.utils.DateUtil; import java.nio.file.Path; import java.nio.file.Paths; -import java.text.ParseException; import java.time.Instant; import org.junit.Assert; import org.junit.Test; +import org.junit.jupiter.api.Assertions; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -39,23 +37,16 @@ public class TestRemoteBackupPath { private static final Logger logger = LoggerFactory.getLogger(TestRemoteBackupPath.class); private Provider pathFactory; - private IConfiguration configuration; public TestRemoteBackupPath() { - Injector injector = Guice.createInjector(new BRTestModule()); - pathFactory = injector.getProvider(AbstractBackupPath.class); - configuration = injector.getInstance(IConfiguration.class); + pathFactory = + Guice.createInjector(new BRTestModule()).getProvider(AbstractBackupPath.class); } @Test - public void testV1BackupPathsSST() throws ParseException { + public void testV1BackupPathsSST() { Path path = - Paths.get( - configuration.getDataFileLocation(), - "keyspace1", - "columnfamily1", - "backup", - "mc-1234-Data.db"); + Paths.get("target/data", "keyspace1", "columnfamily1", "backup", "mc-1234-Data.db"); AbstractBackupPath abstractBackupPath = pathFactory.get(); abstractBackupPath.parseLocal(path.toFile(), BackupFileType.SST); @@ -82,20 +73,11 @@ public void testV1BackupPathsSST() throws ParseException { Assert.assertEquals(abstractBackupPath.getTime(), abstractBackupPath2.getTime()); } - private void validateAbstractBackupPath(AbstractBackupPath abp1, AbstractBackupPath abp2) { - Assert.assertEquals(abp1.getKeyspace(), abp2.getKeyspace()); - Assert.assertEquals(abp1.getColumnFamily(), abp2.getColumnFamily()); - Assert.assertEquals(abp1.getFileName(), abp2.getFileName()); - Assert.assertEquals(abp1.getType(), abp2.getType()); - Assert.assertEquals(abp1.getCompression(), abp2.getCompression()); - Assert.assertEquals(abp1.getEncryption(), abp2.getEncryption()); - } - @Test - public void testV1BackupPathsSnap() throws ParseException { + public void testV1BackupPathsSnap() { Path path = Paths.get( - configuration.getDataFileLocation(), + "target/data", "keyspace1", "columnfamily1", "snapshot", @@ -125,16 +107,16 @@ public void testV1BackupPathsSnap() throws ParseException { } @Test - public void testV1BackupPathsMeta() throws ParseException { - Path path = Paths.get(configuration.getDataFileLocation(), "meta.json"); + public void testV1BackupPathsMeta() { + Path path = Paths.get("target/data", "meta.json"); AbstractBackupPath abstractBackupPath = pathFactory.get(); abstractBackupPath.parseLocal(path.toFile(), BackupFileType.META); // Verify parse local Assert.assertEquals( 0, abstractBackupPath.getLastModified().toEpochMilli()); // File do not exist. - Assert.assertEquals(null, abstractBackupPath.getKeyspace()); - Assert.assertEquals(null, abstractBackupPath.getColumnFamily()); + Assert.assertNull(abstractBackupPath.getKeyspace()); + Assert.assertNull(abstractBackupPath.getColumnFamily()); Assert.assertEquals(BackupFileType.META, abstractBackupPath.getType()); Assert.assertEquals(path.toFile(), abstractBackupPath.getBackupFile()); @@ -149,14 +131,9 @@ public void testV1BackupPathsMeta() throws ParseException { } @Test - public void testV2BackupPathSST() throws ParseException { + public void testV2BackupPathSST() { Path path = - Paths.get( - configuration.getDataFileLocation(), - "keyspace1", - "columnfamily1", - "backup", - "mc-1234-Data.db"); + Paths.get("target/data", "keyspace1", "columnfamily1", "backup", "mc-1234-Data.db"); AbstractBackupPath abstractBackupPath = pathFactory.get(); abstractBackupPath.parseLocal(path.toFile(), BackupFileType.SST_V2); @@ -182,29 +159,106 @@ public void testV2BackupPathSST() throws ParseException { } @Test - public void testV2BackupPathMeta() throws ParseException { - Path path = Paths.get(configuration.getDataFileLocation(), "meta_v2_201801011201.json"); + public void testV2BackupPathMeta() { + Path path = Paths.get("target/data", "meta_v2_201801011201.json"); AbstractBackupPath abstractBackupPath = pathFactory.get(); abstractBackupPath.parseLocal(path.toFile(), BackupFileType.META_V2); // Verify parse local Assert.assertEquals( 0, abstractBackupPath.getLastModified().toEpochMilli()); // File do not exist. - Assert.assertEquals(null, abstractBackupPath.getKeyspace()); - Assert.assertEquals(null, abstractBackupPath.getColumnFamily()); + Assert.assertNull(abstractBackupPath.getKeyspace()); + Assert.assertNull(abstractBackupPath.getColumnFamily()); Assert.assertEquals(BackupFileType.META_V2, abstractBackupPath.getType()); Assert.assertEquals(path.toFile(), abstractBackupPath.getBackupFile()); + Assert.assertEquals(CryptographyAlgorithm.PLAINTEXT, abstractBackupPath.getEncryption()); + + // Verify toRemote and parseRemote. + Instant now = DateUtil.getInstant(); + abstractBackupPath.setLastModified(now); + String remotePath = abstractBackupPath.getRemotePath(); + logger.info(remotePath); + + Assert.assertEquals("SNAPPY", abstractBackupPath.getCompression().name()); + + AbstractBackupPath abstractBackupPath2 = pathFactory.get(); + abstractBackupPath2.parseRemote(remotePath); + Assert.assertEquals(now, abstractBackupPath2.getLastModified()); + validateAbstractBackupPath(abstractBackupPath, abstractBackupPath2); + } + + @Test + public void testV2BackupPathSecondaryIndex() { + Path path = + Paths.get( + "target/data", + "keyspace1", + "columnfamily1", + "backups", + ".columnfamily1_field1_idx", + "mc-1234-Data.db"); + AbstractBackupPath abstractBackupPath = pathFactory.get(); + abstractBackupPath.parseLocal(path.toFile(), BackupFileType.SECONDARY_INDEX_V2); + + // Verify parse local Assert.assertEquals( - IFileCryptography.CryptographyAlgorithm.PLAINTEXT, - abstractBackupPath.getEncryption()); + 0, abstractBackupPath.getLastModified().toEpochMilli()); // File do not exist. + Assert.assertEquals("keyspace1", abstractBackupPath.getKeyspace()); + Assert.assertEquals("columnfamily1", abstractBackupPath.getColumnFamily()); + Assert.assertEquals("SNAPPY", abstractBackupPath.getCompression().name()); + Assert.assertEquals(BackupFileType.SECONDARY_INDEX_V2, abstractBackupPath.getType()); + Assert.assertEquals(path.toFile(), abstractBackupPath.getBackupFile()); // Verify toRemote and parseRemote. Instant now = DateUtil.getInstant(); abstractBackupPath.setLastModified(now); String remotePath = abstractBackupPath.getRemotePath(); logger.info(remotePath); + Assert.assertEquals( + "casstestbackup/1049_fake-app/1808575600/SECONDARY_INDEX_V2/" + + now.toEpochMilli() + + "/keyspace1/columnfamily1/.columnfamily1_field1_idx/SNAPPY/PLAINTEXT/mc-1234-Data.db", + remotePath); + + AbstractBackupPath abstractBackupPath2 = pathFactory.get(); + abstractBackupPath2.parseRemote(remotePath); + Assert.assertEquals(now, abstractBackupPath2.getLastModified()); + validateAbstractBackupPath(abstractBackupPath, abstractBackupPath2); + } + @Test + public void testV2SnapshotPathSecondaryIndex() { + Path path = + Paths.get( + "target/data", + "keyspace1", + "columnfamily1", + "snapshots", + "snap_v2_19700101000", + ".columnfamily1_field1_idx", + "mc-1234-Data.db"); + AbstractBackupPath abstractBackupPath = pathFactory.get(); + abstractBackupPath.parseLocal(path.toFile(), BackupFileType.SECONDARY_INDEX_V2); + + // Verify parse local + Assert.assertEquals( + 0, abstractBackupPath.getLastModified().toEpochMilli()); // File do not exist. + Assert.assertEquals("keyspace1", abstractBackupPath.getKeyspace()); + Assert.assertEquals("columnfamily1", abstractBackupPath.getColumnFamily()); Assert.assertEquals("SNAPPY", abstractBackupPath.getCompression().name()); + Assert.assertEquals(BackupFileType.SECONDARY_INDEX_V2, abstractBackupPath.getType()); + Assert.assertEquals(path.toFile(), abstractBackupPath.getBackupFile()); + + // Verify toRemote and parseRemote. + Instant now = DateUtil.getInstant(); + abstractBackupPath.setLastModified(now); + String remotePath = abstractBackupPath.getRemotePath(); + logger.info(remotePath); + Assert.assertEquals( + "casstestbackup/1049_fake-app/1808575600/SECONDARY_INDEX_V2/" + + now.toEpochMilli() + + "/keyspace1/columnfamily1/.columnfamily1_field1_idx/SNAPPY/PLAINTEXT/mc-1234-Data.db", + remotePath); AbstractBackupPath abstractBackupPath2 = pathFactory.get(); abstractBackupPath2.parseRemote(remotePath); @@ -213,7 +267,25 @@ public void testV2BackupPathMeta() throws ParseException { } @Test - public void testRemoteV2Prefix() throws ParseException { + public void testUnknownBackupFolder() { + Path path = + Paths.get( + "target/data", + "keyspace1", + "columnfamily1", + "foo", // foo is invalid + ".columnfamily1_field1_idx", + "mc-1234-Data.db"); + AbstractBackupPath abstractBackupPath = pathFactory.get(); + Assertions.assertThrows( + NullPointerException.class, + () -> + abstractBackupPath.parseLocal( + path.toFile(), BackupFileType.SECONDARY_INDEX_V2)); + } + + @Test + public void testRemoteV2Prefix() { Path path = Paths.get("test_backup"); AbstractBackupPath abstractBackupPath = pathFactory.get(); Assert.assertEquals( @@ -225,4 +297,13 @@ public void testRemoteV2Prefix() throws ParseException { "fake_base_dir/-6717_random_fake_app/1808575600/META_V2", abstractBackupPath.remoteV2Prefix(path, BackupFileType.META_V2).toString()); } + + private void validateAbstractBackupPath(AbstractBackupPath abp1, AbstractBackupPath abp2) { + Assert.assertEquals(abp1.getKeyspace(), abp2.getKeyspace()); + Assert.assertEquals(abp1.getColumnFamily(), abp2.getColumnFamily()); + Assert.assertEquals(abp1.getFileName(), abp2.getFileName()); + Assert.assertEquals(abp1.getType(), abp2.getType()); + Assert.assertEquals(abp1.getCompression(), abp2.getCompression()); + Assert.assertEquals(abp1.getEncryption(), abp2.getEncryption()); + } } diff --git a/priam/src/test/java/com/netflix/priam/backup/TestBackup.java b/priam/src/test/java/com/netflix/priam/backup/TestBackup.java index a79f63f06..f4cb1dc44 100644 --- a/priam/src/test/java/com/netflix/priam/backup/TestBackup.java +++ b/priam/src/test/java/com/netflix/priam/backup/TestBackup.java @@ -91,7 +91,7 @@ public void testIncrementalBackup() throws Exception { generateIncrementalFiles(); IncrementalBackup backup = injector.getInstance(IncrementalBackup.class); backup.execute(); - Assert.assertEquals(4, filesystem.uploadedFiles.size()); + Assert.assertEquals(5, filesystem.uploadedFiles.size()); for (String filePath : expectedFiles) Assert.assertTrue(filesystem.uploadedFiles.contains(filePath)); } @@ -142,7 +142,7 @@ private void testClusterSpecificColumnFamiliesSkipped(String[] columnFamilyDirs) } IncrementalBackup backup = injector.getInstance(IncrementalBackup.class); backup.execute(); - Assert.assertEquals(6, filesystem.uploadedFiles.size()); + Assert.assertEquals(7, filesystem.uploadedFiles.size()); for (String filePath : expectedFiles) Assert.assertTrue(filesystem.uploadedFiles.contains(filePath)); } @@ -156,6 +156,8 @@ private static void generateIncrementalFiles() { files.add("target/data/Keyspace1/Standard1/backups/Keyspace1-Standard1-ia-1-Index.db"); files.add("target/data/Keyspace1/Standard1/backups/Keyspace1-Standard1-ia-2-Data.db"); files.add("target/data/Keyspace1/Standard1/backups/Keyspace1-Standard1-ia-3-Data.db"); + files.add( + "target/data/Keyspace1/Standard1/backups/.Standard1_field1_idx_1/Keyspace1-Standard1-ia-4-Data.db"); expectedFiles.clear(); for (String filePath : files) { diff --git a/priam/src/test/java/com/netflix/priam/backupv2/TestBackupUtils.java b/priam/src/test/java/com/netflix/priam/backupv2/TestBackupUtils.java index 1cd63c960..30c9c5477 100644 --- a/priam/src/test/java/com/netflix/priam/backupv2/TestBackupUtils.java +++ b/priam/src/test/java/com/netflix/priam/backupv2/TestBackupUtils.java @@ -17,6 +17,7 @@ package com.netflix.priam.backupv2; +import com.google.common.collect.ImmutableSet; import com.google.inject.Guice; import com.google.inject.Inject; import com.google.inject.Injector; @@ -29,7 +30,6 @@ import java.nio.file.Path; import java.nio.file.Paths; import java.time.Instant; -import java.util.ArrayList; import java.util.List; import org.apache.commons.io.FileUtils; @@ -60,8 +60,7 @@ public Path createMeta(List filesToAdd, Instant snapshotTime) throws IOE ColumnfamilyResult.SSTableResult ssTableResult = new ColumnfamilyResult.SSTableResult(); - ssTableResult.setSstableComponents(new ArrayList<>()); - ssTableResult.getSstableComponents().add(getFileUploadResult(path)); + ssTableResult.setSstableComponents(ImmutableSet.of(getFileUploadResult(path))); columnfamilyResult.addSstable(ssTableResult); } } @@ -85,8 +84,6 @@ private FileUploadResult getFileUploadResult(AbstractBackupPath path) { FileUploadResult fileUploadResult = new FileUploadResult( Paths.get(path.getFileName()), - path.getKeyspace(), - path.getColumnFamily(), path.getLastModified(), path.getLastModified(), path.getSize()); diff --git a/priam/src/test/java/com/netflix/priam/backupv2/TestFileUploadResult.java b/priam/src/test/java/com/netflix/priam/backupv2/TestFileUploadResult.java new file mode 100644 index 000000000..49c45adfa --- /dev/null +++ b/priam/src/test/java/com/netflix/priam/backupv2/TestFileUploadResult.java @@ -0,0 +1,73 @@ +package com.netflix.priam.backupv2; + +import com.google.common.truth.Truth; +import java.nio.file.Paths; +import java.time.Instant; +import org.junit.Test; + +/** unit tests of FileUploadResult */ +public class TestFileUploadResult { + + @Test + public void standardInput() { + Truth.assertThat(getResult().toString()) + .isEqualTo( + "{\n" + + " \"fileName\": \"path\",\n" + + " \"lastModifiedTime\": 200000,\n" + + " \"fileCreationTime\": 100000,\n" + + " \"fileSizeOnDisk\": 100000,\n" + + " \"compression\": \"SNAPPY\",\n" + + " \"encryption\": \"PLAINTEXT\"\n" + + "}"); + } + + @Test + public void setIsUploaded() { + FileUploadResult result = getResult(); + result.setUploaded(true); + Truth.assertThat(result.toString()) + .isEqualTo( + "{\n" + + " \"fileName\": \"path\",\n" + + " \"lastModifiedTime\": 200000,\n" + + " \"fileCreationTime\": 100000,\n" + + " \"fileSizeOnDisk\": 100000,\n" + + " \"compression\": \"SNAPPY\",\n" + + " \"encryption\": \"PLAINTEXT\",\n" + + " \"isUploaded\": true\n" + + "}"); + } + + @Test + public void setBackupPath() { + FileUploadResult result = getResult(); + result.setBackupPath("foo"); + Truth.assertThat(result.toString()) + .isEqualTo( + "{\n" + + " \"fileName\": \"path\",\n" + + " \"lastModifiedTime\": 200000,\n" + + " \"fileCreationTime\": 100000,\n" + + " \"fileSizeOnDisk\": 100000,\n" + + " \"compression\": \"SNAPPY\",\n" + + " \"encryption\": \"PLAINTEXT\",\n" + + " \"backupPath\": \"foo\"\n" + + "}"); + } + + @Test + public void getBackupPath() { + FileUploadResult result = getResult(); + result.setBackupPath("foo"); + Truth.assertThat(result.getBackupPath()).isEqualTo("foo"); + } + + private FileUploadResult getResult() { + return new FileUploadResult( + Paths.get("/my/file/path"), + Instant.ofEpochMilli(200000L), + Instant.ofEpochMilli(100000L), + 100000L); + } +} From d4a9fead548605837283f82c4f0c6c1576448f1e Mon Sep 17 00:00:00 2001 From: Matt Lehman Date: Mon, 9 Nov 2020 19:46:28 -0800 Subject: [PATCH 150/228] Update CHANGELOG in advance of 3.11.73 --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8dfecd3a5..129803dbe 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,7 @@ # Changelog +## 2020/11/09 3.11.73 +(#911) Backup secondary index files. + ## 2020/09/30 3.11.72 (#908, #910) Stop explicitly filtering OpsCenter keyspace when backing up. Remove more noisy log statements. From 6919d340a5611277df67da38fa92f2f98ab5bfbe Mon Sep 17 00:00:00 2001 From: mattl-netflix <63665634+mattl-netflix@users.noreply.github.com> Date: Fri, 15 Jan 2021 09:45:58 -0800 Subject: [PATCH 151/228] Remove redundant information about bootstrapping from InstanceState (#915) --- .../netflix/priam/health/InstanceState.java | 56 ++----------------- 1 file changed, 4 insertions(+), 52 deletions(-) diff --git a/priam/src/main/java/com/netflix/priam/health/InstanceState.java b/priam/src/main/java/com/netflix/priam/health/InstanceState.java index 96f702371..402f331e0 100644 --- a/priam/src/main/java/com/netflix/priam/health/InstanceState.java +++ b/priam/src/main/java/com/netflix/priam/health/InstanceState.java @@ -24,8 +24,6 @@ import java.time.LocalDateTime; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLong; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; /** * Contains the state of the health of processed managed by Priam, and maintains the isHealthy flag @@ -35,20 +33,6 @@ */ @Singleton public class InstanceState { - private static final Logger logger = LoggerFactory.getLogger(InstanceState.class); - - public enum NODE_STATE { - // This state to be used when Priam is joining the ring for the first time or was - // already assigned this token. - JOIN, - // This state to be used when Priam replaces an instance from the token range. - REPLACE - } - - // Bootstrap status - private final AtomicBoolean isBootstrapping = new AtomicBoolean(false); - private NODE_STATE nodeState; - private LocalDateTime bootstrapTime; // Cassandra process status private final AtomicBoolean isCassandraProcessAlive = new AtomicBoolean(false); @@ -62,7 +46,7 @@ public enum NODE_STATE { private final AtomicBoolean isHealthy = new AtomicBoolean(false); private final AtomicBoolean isHealthyOverride = new AtomicBoolean(true); - // Backup status + // This is referenced when this class is serialized to a String1 private BackupMetadata backupStatus; // Restore status @@ -148,36 +132,7 @@ public long getLastAttemptedStartTime() { return this.lastAttemptedStartTime.get(); } - /* Boostrap */ - public boolean isBootstrapping() { - return isBootstrapping.get(); - } - - public void setBootstrapping(boolean isBootstrapping) { - this.isBootstrapping.set(isBootstrapping); - } - - public NODE_STATE getNodeState() { - return nodeState; - } - - public LocalDateTime getBootstrapTime() { - return bootstrapTime; - } - - public void setBootstrapTime(LocalDateTime bootstrapTime) { - this.bootstrapTime = bootstrapTime; - } - - public void setNodeState(NODE_STATE nodeState) { - this.nodeState = nodeState; - } - /* Backup */ - public BackupMetadata getBackupStatus() { - return backupStatus; - } - public void setBackupStatus(BackupMetadata backupMetadata) { this.backupStatus = backupMetadata; } @@ -225,8 +180,9 @@ public void setYmlWritten(boolean yml) { public static class RestoreStatus { private LocalDateTime startDateRange, endDateRange; // Date range to restore from - private LocalDateTime executionStartTime, - executionEndTime; // Start-end time of the actual restore execution + // Start-end time of the actual restore execution + // Note these are referenced when this class is serialized to a String. + private LocalDateTime executionStartTime, executionEndTime; private String snapshotMetaFile; // Location of the snapshot meta file selected for restore. // the state of a restore. Note: this is different than the "status" of a Task. private Status status; @@ -275,10 +231,6 @@ public LocalDateTime getExecutionStartTime() { return executionStartTime; } - public LocalDateTime getExecutionEndTime() { - return executionEndTime; - } - public String getSnapshotMetaFile() { return snapshotMetaFile; } From 243afcf1f19ddc052f372108aac25d795d98e4d1 Mon Sep 17 00:00:00 2001 From: Sumanth Pasupuleti Date: Wed, 17 Feb 2021 11:01:54 -0800 Subject: [PATCH 152/228] Adding support for custom override for role_manager (#918) Co-authored-by: Sumanth Pasupuleti --- .../netflix/priam/config/IConfiguration.java | 5 +++ .../priam/config/PriamConfiguration.java | 5 +++ .../netflix/priam/tuner/StandardTuner.java | 1 + .../priam/config/FakeConfiguration.java | 11 +++++++ .../priam/tuner/StandardTunerTest.java | 33 ++++++++++++------- 5 files changed, 44 insertions(+), 11 deletions(-) diff --git a/priam/src/main/java/com/netflix/priam/config/IConfiguration.java b/priam/src/main/java/com/netflix/priam/config/IConfiguration.java index 872353ef0..76f574767 100644 --- a/priam/src/main/java/com/netflix/priam/config/IConfiguration.java +++ b/priam/src/main/java/com/netflix/priam/config/IConfiguration.java @@ -560,6 +560,11 @@ default String getAuthorizer() { return "org.apache.cassandra.auth.AllowAllAuthorizer"; } + /** Defaults to 'CassandraRoleManager'. */ + default String getRoleManager() { + return "org.apache.cassandra.auth.CassandraRoleManager"; + } + /** @return true/false, if Cassandra needs to be started manually */ default boolean doesCassandraStartManually() { return false; diff --git a/priam/src/main/java/com/netflix/priam/config/PriamConfiguration.java b/priam/src/main/java/com/netflix/priam/config/PriamConfiguration.java index 83b48fd71..d9b77d224 100644 --- a/priam/src/main/java/com/netflix/priam/config/PriamConfiguration.java +++ b/priam/src/main/java/com/netflix/priam/config/PriamConfiguration.java @@ -427,6 +427,11 @@ public String getAuthorizer() { PRIAM_PRE + ".authorizer", "org.apache.cassandra.auth.AllowAllAuthorizer"); } + public String getRoleManager() { + return config.get( + PRIAM_PRE + ".roleManager", "org.apache.cassandra.auth.CassandraRoleManager"); + } + @Override public boolean doesCassandraStartManually() { return config.get(PRIAM_PRE + ".cass.manual.start.enable", false); diff --git a/priam/src/main/java/com/netflix/priam/tuner/StandardTuner.java b/priam/src/main/java/com/netflix/priam/tuner/StandardTuner.java index ac37b3ad0..d36f9cf09 100644 --- a/priam/src/main/java/com/netflix/priam/tuner/StandardTuner.java +++ b/priam/src/main/java/com/netflix/priam/tuner/StandardTuner.java @@ -105,6 +105,7 @@ public void writeAllProperties(String yamlLocation, String hostname, String seed map.put("hinted_handoff_throttle_in_kb", config.getHintedHandoffThrottleKb()); map.put("authenticator", config.getAuthenticator()); map.put("authorizer", config.getAuthorizer()); + map.put("role_manager", config.getRoleManager()); map.put("internode_compression", config.getInternodeCompression()); map.put("dynamic_snitch", config.isDynamicSnitchEnabled()); diff --git a/priam/src/test/java/com/netflix/priam/config/FakeConfiguration.java b/priam/src/test/java/com/netflix/priam/config/FakeConfiguration.java index edc05bfac..31a5f86e3 100644 --- a/priam/src/test/java/com/netflix/priam/config/FakeConfiguration.java +++ b/priam/src/test/java/com/netflix/priam/config/FakeConfiguration.java @@ -31,6 +31,7 @@ public class FakeConfiguration implements IConfiguration { private final String appName; private String restorePrefix = ""; public Map fakeConfig; + private String roleManager = ""; public Map fakeProperties = new HashMap<>(); @@ -184,6 +185,16 @@ public ImmutableSet getTunablePropertyFiles() { return ImmutableSet.of(path + "/cassandra-rackdc.properties"); } + @Override + public String getRoleManager() { + return this.roleManager; + } + + public FakeConfiguration setRoleManager(String roleManager) { + this.roleManager = roleManager; + return this; + } + public String getRAC() { return "my_zone"; } diff --git a/priam/src/test/java/com/netflix/priam/tuner/StandardTunerTest.java b/priam/src/test/java/com/netflix/priam/tuner/StandardTunerTest.java index 3cdf0b162..588c3541f 100644 --- a/priam/src/test/java/com/netflix/priam/tuner/StandardTunerTest.java +++ b/priam/src/test/java/com/netflix/priam/tuner/StandardTunerTest.java @@ -141,11 +141,28 @@ public void addExtraParams() throws Exception { extraParamValues.put(priamKeyName2, "test"); extraParamValues.put(priamKeyName3, "randomKeyValue"); extraParamValues.put(priamKeyName4, "randomGroupValue"); + + Map map = + applyFakeConfiguration(new TunerConfiguration(extraConfigParam, extraParamValues)); + Assert.assertEquals("your_host", map.get("listen_address")); + Assert.assertEquals("true", ((Map) map.get("client_encryption_options")).get("optional")); + Assert.assertEquals( + "test", ((Map) map.get("client_encryption_options")).get("keystore_password")); + Assert.assertEquals("randomKeyValue", map.get("randomKey")); + Assert.assertEquals("randomGroupValue", ((Map) map.get("randomGroup")).get("randomKey")); + } + + @Test + public void testRoleManagerOverride() throws Exception { + String roleManagerOverride = "org.apache.cassandra.auth.CustomRoleManager"; + Map map = + applyFakeConfiguration(new FakeConfiguration().setRoleManager(roleManagerOverride)); + Assert.assertEquals(roleManagerOverride, map.get("role_manager")); + } + + private Map applyFakeConfiguration(FakeConfiguration fakeConfiguration) throws Exception { StandardTuner tuner = - new StandardTuner( - new TunerConfiguration(extraConfigParam, extraParamValues), - backupRestoreConfig, - instanceInfo); + new StandardTuner(fakeConfiguration, backupRestoreConfig, instanceInfo); Files.copy(new File("src/main/resources/incr-restore-cassandra.yaml"), target); tuner.writeAllProperties(target.getAbsolutePath(), "your_host", "YourSeedProvider"); @@ -153,13 +170,7 @@ public void addExtraParams() throws Exception { DumperOptions options = new DumperOptions(); options.setDefaultFlowStyle(DumperOptions.FlowStyle.BLOCK); Yaml yaml = new Yaml(options); - Map map = yaml.load(new FileInputStream(target)); - Assert.assertEquals("your_host", map.get("listen_address")); - Assert.assertEquals("true", ((Map) map.get("client_encryption_options")).get("optional")); - Assert.assertEquals( - "test", ((Map) map.get("client_encryption_options")).get("keystore_password")); - Assert.assertEquals("randomKeyValue", map.get("randomKey")); - Assert.assertEquals("randomGroupValue", ((Map) map.get("randomGroup")).get("randomKey")); + return yaml.load(new FileInputStream(target)); } private class TunerConfiguration extends FakeConfiguration { From a83862687d736f458a226a367f5811a337581110 Mon Sep 17 00:00:00 2001 From: Roberto Perez Alcolea Date: Thu, 4 Mar 2021 18:56:27 -0800 Subject: [PATCH 153/228] Upgrade nebula.netflixoss to replace bintray publication and update TravisCI Secrets --- .gitignore | 4 ++++ .travis.yml | 16 +++++++++------- build.gradle | 4 ++-- buildViaTravis.sh | 11 ++++++----- gradle/wrapper/gradle-wrapper.properties | 2 +- installViaTravis.sh | 15 +++------------ priam/build.gradle | 9 ++++++++- .../priam/backupv2/SnapshotMetaTask.java | 2 +- .../identity/token/TokenRetrieverUtils.java | 2 -- secrets/signing-key.enc | Bin 0 -> 6800 bytes 10 files changed, 34 insertions(+), 31 deletions(-) create mode 100644 secrets/signing-key.enc diff --git a/.gitignore b/.gitignore index 8163af2a8..c1862c52c 100644 --- a/.gitignore +++ b/.gitignore @@ -41,6 +41,7 @@ Thumbs.db .gradle .m2 + # Build output directies /target */target @@ -67,3 +68,6 @@ atlassian-ide-plugin.xml # NetBeans specific files/directories .nbattrs >>>>>>> build/multi-project + +# publishing secrets +secrets/signing-key diff --git a/.travis.yml b/.travis.yml index 29dbc1298..9e6b9926a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,14 +4,16 @@ jdk: sudo: false git: depth: 100 -install: ./installViaTravis.sh -script: ./buildViaTravis.sh +install: "./installViaTravis.sh" +script: "./buildViaTravis.sh" cache: directories: - - $HOME/.gradle + - "$HOME/.gradle" env: global: - - secure: p81PYYYYhjhp1uJHOoT9EfUia8qPMphdykCxQannWMVvIFKXF5ibi/Qd52OUIYuJT9Q5MN9GhlwCtolxmCZu44YMLi99LsoITAv6n08LAFEux1oAOXmGt3ZBllvBFtlW+jlQ5lKmbuuPue3RVbTaWzL3KCCaYQ4uKW/sP31bXic= - - secure: AYWrIC/1JqAVyLhzlDhL163LypPpq7AliDltfey9emxzS73vB5VrGTHG45Em7xdkHN0kc1vED+UFsYks7ym1olXZPsbW29R0Gfd+p1VxCE3Cbhj7x690xNC3wIlSgWGpbgiPs7vhVN80OuNgnpz9/+WRFpKAVrYrcisbvugsDqk= - - secure: Ai75crFDh3imkmxzNQDXIwK49appZuZTrsmdZzgH1Zl/nD33RoZPWUWeeHP0yJoicNqZlU6aEXJnk8JGDWtPIT3VNO7Rsey+yrw041rHNV6bFGpwGdQgKv5CUhRHK+gShgFgnME5THCPKvK51+li1feIoByoYVmEn0gmvcxL0yM= - - secure: ZqBsyaFMBz3cqM7UHq0IRotTPaoBNUHrC7buiAqqhZ5JNM48a2m7ORuu/SENPFfr6xOT5KylCrvT6eD1bkQi5S2jT9Qyoju/9H/hXaAUG89dhwKOxqEE11ZGKF8DlNO/jcmchVnIkgWfEjVKc7vCAww1Y8lgxhyj4gRtnln+F+o= + - secure: bXzjPA0Ec3NBIN25knd9UYzGtxxmlhoLkO0LwI87iRSeCLzvbT+qBl/chvaXCk3DC2JnSq6GGWw2r0R6pjR4UezZx72KP+dm19cy606e7np1mHZmzziSkyjaBd84HPgOz8L4jh5G9qQBG8VhkjikIabAgsbiWG3oX6a/rOadobg= + - secure: COIxG87u7BSBAD0s8g0E9PTq4XLompgW/PL55bnIxuVkIo3tXXqShAxMzsJ0C++q5AbofNUICqTbGqN0ozk9GpMC78d5pgWS1O4hpdWsouK8Fxcju+KkKC5MUDC478G/zW0uSa0uZsZGaDy3Gx17jpvq2CdVCgHmaHmGGOxfDbA= + - secure: LMyJ2v4kN+07Gz5pPHkYLOvbQOAI05bHFGUVfaAbofyiR1DqKpKqB4hYaKSgb0kB/3YI6u9dLQ5AWzAqfx3HY0+p2iTmgseIxbHGrFfOH7bt2xusYNUtYPn1JXOAz1Nzxq/a+4rCgqD9lFI19XetrI5SNrRIRErDIZnWdXyhW/M= + - secure: dxJVfXpC/yORQWg2oaw+Q2UjeoAHiqf5hyLVSdGbIJYVrSu/fyFCuUNjQffikjleoyuaPeIHxaGQZoQZE3+9Cf3d7s1lXzugO3UOxmKR9UYi8tz6sLUFeLPjVhR6Etx603A8oCg659RhD2VyOCZvq1/R2gUa34tUtnVyoFyUzDA= + - secure: tl59nbZhOzWqW/PLp91+Paugond4Lwwj/qT1Tq4EFJ9Q2nNMFW3twlWzmUOcgn7GaJZTgfoDx2HGYFg1N3rrBtZIY/Wtzz9eWrGY4ywHfoXz/8lQGtbciDjiNePByuvSLOPWeTIGncOLGNUcCPNFOO9+kC4BzUGr9CjtTc+qnWg= + - secure: AZoBLO43P+GVjQq+vsT8mekOUNqifi+tRYukz19GLN2vv2vsgY4ejhXjeer1kwqm0l/1bXtfqo0uHyD+CFn2yINV20oKHScYNS4fwKnJ8aC60MZqAj6zddfe2VvEeDIKDiejpz4AfRJwgqpl9B0FrNtZ/1oU5n8oneTKpVFryTs= diff --git a/build.gradle b/build.gradle index c3f6b6b02..591a397c3 100644 --- a/build.gradle +++ b/build.gradle @@ -1,5 +1,5 @@ plugins { - id 'nebula.netflixoss' version '7.1.3' + id 'nebula.netflixoss' version '9.1.0' id 'com.github.sherter.google-java-format' version '0.8' } @@ -78,7 +78,7 @@ allprojects { compile 'com.google.http-client:google-http-client-jackson2:1.28.0' compile 'com.netflix.spectator:spectator-api:0.96.0' compileOnly 'javax.servlet:javax.servlet-api:3.1.0' - testCompile 'org.jmockit:jmockit:1.31' + testCompile 'org.jmockit:jmockit:1.38' testCompile "org.spockframework:spock-core:1.1-groovy-2.4" testCompile "com.google.truth:truth:1.0.1" testImplementation 'org.junit.jupiter:junit-jupiter-api:5.3.1' diff --git a/buildViaTravis.sh b/buildViaTravis.sh index 812026d7a..c3ddb674c 100755 --- a/buildViaTravis.sh +++ b/buildViaTravis.sh @@ -3,21 +3,22 @@ if [ "$TRAVIS_PULL_REQUEST" != "false" ]; then echo -e "Build Pull Request #$TRAVIS_PULL_REQUEST => Branch [$TRAVIS_BRANCH]" - ./gradlew build --stacktrace + ./gradlew build elif [ "$TRAVIS_PULL_REQUEST" == "false" ] && [ "$TRAVIS_TAG" == "" ]; then echo -e 'Build Branch with Snapshot => Branch ['$TRAVIS_BRANCH']' - ./gradlew -Prelease.travisci=true -PbintrayUser="${bintrayUser}" -PbintrayKey="${bintrayKey}" -PsonatypeUsername="${sonatypeUsername}" -PsonatypePassword="${sonatypePassword}" build snapshot --stacktrace + ./gradlew -Prelease.travisci=true -PnetflixOss.username="$NETFLIX_OSS_REPO_USERNAME" -PnetflixOss.password="$NETFLIX_OSS_REPO_PASSWORD" -Psonatype.signingPassword="$NETFLIX_OSS_SIGNING_PASSWORD" build snapshot elif [ "$TRAVIS_PULL_REQUEST" == "false" ] && [ "$TRAVIS_TAG" != "" ]; then echo -e 'Build Branch for Release => Branch ['$TRAVIS_BRANCH'] Tag ['$TRAVIS_TAG']' case "$TRAVIS_TAG" in *-rc\.*) - ./gradlew -Prelease.travisci=true -Prelease.useLastTag=true -PbintrayUser="${bintrayUser}" -PbintrayKey="${bintrayKey}" -PsonatypeUsername="${sonatypeUsername}" -PsonatypePassword="${sonatypePassword}" candidate --stacktrace + ./gradlew -Prelease.travisci=true -PnetflixOss.username="$NETFLIX_OSS_REPO_USERNAME" -PnetflixOss.password="$NETFLIX_OSS_REPO_PASSWORD" -Psonatype.signingPassword="$NETFLIX_OSS_SIGNING_PASSWORD" -Prelease.useLastTag=true candidate ;; *) - ./gradlew -Prelease.travisci=true -Prelease.useLastTag=true -PbintrayUser="${bintrayUser}" -PbintrayKey="${bintrayKey}" -PsonatypeUsername="${sonatypeUsername}" -PsonatypePassword="${sonatypePassword}" final --stacktrace + ./gradlew -Prelease.travisci=true -PnetflixOss.username="$NETFLIX_OSS_REPO_USERNAME" -PnetflixOss.password="$NETFLIX_OSS_REPO_PASSWORD" -Psonatype.username="$NETFLIX_OSS_SONATYPE_USERNAME" -Psonatype.password="$NETFLIX_OSS_SONATYPE_PASSWORD" -Psonatype.signingPassword="$NETFLIX_OSS_SIGNING_PASSWORD" -Prelease.useLastTag=true final ;; esac else echo -e 'WARN: Should not be here => Branch ['$TRAVIS_BRANCH'] Tag ['$TRAVIS_TAG'] Pull Request ['$TRAVIS_PULL_REQUEST']' - ./gradlew build --stacktrace + ./gradlew build fi + diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index 717f03890..68ca99ac4 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -2,4 +2,4 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-4.8.1-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-6.8.3-bin.zip diff --git a/installViaTravis.sh b/installViaTravis.sh index 06a86291c..82cf1b880 100755 --- a/installViaTravis.sh +++ b/installViaTravis.sh @@ -1,16 +1,7 @@ #!/bin/bash # This script will build the project. -if [ "$TRAVIS_PULL_REQUEST" != "false" ]; then - echo -e "Assemble Pull Request #$TRAVIS_PULL_REQUEST => Branch [$TRAVIS_BRANCH]" - ./gradlew assemble --stacktrace -elif [ "$TRAVIS_PULL_REQUEST" == "false" ] && [ "$TRAVIS_TAG" == "" ]; then - echo -e 'Assemble Branch with Snapshot => Branch ['$TRAVIS_BRANCH']' - ./gradlew -Prelease.travisci=true assemble --stacktrace -elif [ "$TRAVIS_PULL_REQUEST" == "false" ] && [ "$TRAVIS_TAG" != "" ]; then - echo -e 'Assemble Branch for Release => Branch ['$TRAVIS_BRANCH'] Tag ['$TRAVIS_TAG']' - ./gradlew -Prelease.travisci=true -Prelease.useLastTag=true assemble --stacktrace -else - echo -e 'WARN: Should not be here => Branch ['$TRAVIS_BRANCH'] Tag ['$TRAVIS_TAG'] Pull Request ['$TRAVIS_PULL_REQUEST']' - ./gradlew assemble --stacktrace +if [ "$TRAVIS_SECURE_ENV_VARS" = "true" ]; then + echo "Decrypting publishing credentials" + openssl aes-256-cbc -k "$NETFLIX_OSS_SIGNING_FILE_PASSWORD" -in secrets/signing-key.enc -out secrets/signing-key -d fi diff --git a/priam/build.gradle b/priam/build.gradle index 723f3d03d..bbafaa0aa 100644 --- a/priam/build.gradle +++ b/priam/build.gradle @@ -1 +1,8 @@ -apply plugin: 'groovy' \ No newline at end of file +apply plugin: 'groovy' + +/** + * This is from https://jmockit.github.io/tutorial/Introduction.html#runningTests + */ +test { + jvmArgs "-javaagent:${classpath.find { it.name.contains("jmockit") }.absolutePath}" +} diff --git a/priam/src/main/java/com/netflix/priam/backupv2/SnapshotMetaTask.java b/priam/src/main/java/com/netflix/priam/backupv2/SnapshotMetaTask.java index 5ac26dc8f..871f021e2 100644 --- a/priam/src/main/java/com/netflix/priam/backupv2/SnapshotMetaTask.java +++ b/priam/src/main/java/com/netflix/priam/backupv2/SnapshotMetaTask.java @@ -415,7 +415,7 @@ ImmutableSetMultimap getFileUploadResults( * Gives the prefix (common name) of the sstable components. Returns an empty Optional if it is * not an sstable component or a manifest or schema file. * - *

    For example: mc-3-big-Data.db --> mc-3-big ks-cf-ka-7213-Index.db --> ks-cf-ka-7213 + *

    For example: mc-3-big-Data.db -- mc-3-big ks-cf-ka-7213-Index.db -- ks-cf-ka-7213 * * @param file the file from which to extract a common prefix. * @return common prefix of the file, or empty, diff --git a/priam/src/main/java/com/netflix/priam/identity/token/TokenRetrieverUtils.java b/priam/src/main/java/com/netflix/priam/identity/token/TokenRetrieverUtils.java index 700f3ef92..1fd35b7ec 100644 --- a/priam/src/main/java/com/netflix/priam/identity/token/TokenRetrieverUtils.java +++ b/priam/src/main/java/com/netflix/priam/identity/token/TokenRetrieverUtils.java @@ -30,8 +30,6 @@ public class TokenRetrieverUtils { * @param dc * @return IP of the token owner based on gossip information or null if C* status doesn't * converge. - * @throws GossipParseException when required number of instances are not available to fetch the - * gossip info. */ public static InferredTokenOwnership inferTokenOwnerFromGossip( List allIds, String token, String dc) { diff --git a/secrets/signing-key.enc b/secrets/signing-key.enc new file mode 100644 index 0000000000000000000000000000000000000000..ce0833210e7c7d67f8e1ce477f263c20886463b7 GIT binary patch literal 6800 zcmV;B8gJ!OVQh3|WM5xs`pV8kFyX+=Gru0$5*eNlQx2_`Ym3npRuD}yfbqLdh6&BJ z6mq5A=!sT2ZBIfFi~_J)d#HX%)$*qa?Bk9#@8e1lEk2%XR?RO|p|Rh;9I!~45r^H- zq8$p@Z`f_{L#S{iTbmd}By-(NWF-(dQF*q|=?O}8dJBbz)+e@-8(nv%BdEg5d~=ux zilU1zEp3(v$HH=}Mk+H9TluL=MV1c4J=y{i4Wu>sD+@uoTZpN;_%RcvDWE7I_f z(c(xQEvsoQ?`K1LLLPeV1Rx$Z$pDgGp~EWPu00x?#Gu;R#!OwSpezeDM*$=bSDUhC zlt6dmYye*+`?!UlGn+<)BrB?b6{O_@XTBXQBz?s2Q9dXD)85WdFC8@7W-L>Mb@4sv zbwxi2+!D!+GW7M%{K5T!_WSw$3|JC-npr#Wa2^Rv|Fhigxr%5kX9F=G=Yci@>f}NP z_9j$Vo7P^DQo#gf6yQQPYLIdOUx2v3Jn$pKbtjQsXDE6OS!OnYj=UO*SQlFF2Rjom zHo13-*BS-qah@!sv`t&UmIt!l zL&hF!iG3*yvUKP^CDIZ%Q6j}SyTl;@%J09xE(ramXU7U2oKV|k3@*gRR{l#MOKFWa zku0nGvVFC4zwD%&Ckkujjnhxk)S zedH_sl`$_(QMTFUvuk%&e-Ng!t~b_UlUk zeT=PL@f*lJt237ZJ)ww1vzX=RtH~mXc zRf3&BtqxjH!@kX^?sum4fR|zAv<1Nkn!?FmPU#Y?HyH0f$HoW!QxhfWDbEV-q4$Ol zvy)d158*?9kn39ZMl`Y*9Ldo8&$54F^Pv8jWZlYM2{s*jW8EuCe4+M_Kd335V zT}ep#c^BQ14_LTnnG-k_sOBA;0A|!{=4% z%Sv3n#_AMn%UowlEbAQ08PRY_i1KVUKztw56<>;a+;eXIdx>ZuMNd{qL3OBD-)7qohYBwiXp^rHwn2S5o`(`ZkuCfP(I~nkyrJfrbqo?&m5m~lj&?cG_?P*91}w$qulaoUpsL#7 z#BE{TjtQRWiR23T=8j5H1iqdtc(?qn1?LF|W_nQ(!YOPGuc1UNV- zqeEvH!s%6cVYOScPE>vzNA>i|dRVW;ZkXQ+G`M8@=?#Py7_bDn>WhYJGVF|vpUXz` zaA@rLRB=KdW>thBL#G#dzxC6PCzHMDxrKk8veS+2qchc|ndlUn^ob@|7x))B(B+cS z6B9Df_@1h{e|zsC4jOz6I;4-R(gB&Qee>QK|Oarj<6RKR$oUcb0cPwT1>G$Brk zvp__+(`q$o7s@ttm1Ap|RNd?=OegNk(+JC}M(!K4C)gb;(3f@Fq&2@YW7e-G11?Ud zq@~S=RFMG1FZ{SR&rFbz8q{@VWon)#xqm-pE0eI4hcR-BL+orH)2<_ML;Rl*XXF2w&l>G5=+7`h&i_(H_z}m(B#1a6|`4w<%1wGOF&QzOZfrg!)K4)<%73{ zZ}aquQ_Mu*K?b$LfLAZ8sTAx(o`V9rY8sRXEg&^VJB#`jgc(UcopLKe;3U+;F;OA`;^YxxFI&{pwbRWxYBNGYZRNANn4ex z>AHpEl2E2L|SAVsTz~OsoDj{DfJJt^VOq*ic8qpYp=35`ES?K zsIueQlvL)?RSJw4re8kK*UU9zKnH~ZSE~Z~7HEb#XVUyvbs7mg=5wi~Fdm`AEAY6T zrGX3oHWZ+B2PPJ;NlB{#vDt@ErlF>$15v_DbAoO3fX?ujLSxa7zZ+pC-yPBtx)OAR zz{*pjL;XRmJHvLb)Z%wyw0ORQw_c?r56?Y=?@yCvu+abj(MWn=B-#*S%BJm7q9bG_ zgP>24hPA@Xm7DZ)ZP?UT_)=ABOhk@86CHZ$Ps?mm&GaJ&R1FUQJnS6F8jLxr6bw|C zuR1AP&|gC}`A2?By*S7T&qjEUn3J)<6Q@s5U`zDrPvcY4_u{r|Kj=~kC>?gR&2A5{ zLTUipjq&YKI!Y*rLu{d>ZEoR;*n8D{cCj9dW56-M=yoPmTPWg37z-kfF|BEDs z^$wv|v>U%Jkn@x*mbhA3W(ksMam)R4kdbPUNB!e$$vSmQq$lLa>*nAf^^Fpaqy2w{F}uAU2(66NWydRScLp>c=gz2Tvs zV@c;H-X{tDu1ym}%g~kbhSB~tJ=^6f(j*dUlwws+7Im9wV#1h4mbHyC7bWnOlgegW zwFc(u+8Ug%Fyo%yBRn70EnhgcPVXS)v$nO+IePGZ0?_|?S98c^NT?WxE7H&GnRgtm z5ijm;wbg#Aonduv2Vz2oLTVlQj=;U^RO6_nPm!o=_wCnmv8?Zu%yMoiGzMhG z`vkHIb%9c=Hv3#4z|0i8$>~88gNv5s&=I@lzAQI#h<%+x#6SU($U4RWO}-Z5-m9sp zU?ptn{~rB_lni=y`B;q_S$e_<-zeSXZfCJ z+?B`$)R&zN#cXQM2c69i(aq2e0cMZXjZ5*>D!Xs6PJVOU*$n^OQRV!@Zl!zl0%=2M z0?2{A+3GPFzuf#Wqf=wYVbi_R(7SE;P=93w z5t)cDCHi*zn}kdUOY$>pCYd?(UnYEsGafxQKAK9Jkt*k8peJm-s>^YowzE^A|>40NJ=2lu559S~qtr;96l>ul zmHO5o@D$dT6`Kq{Y7E%tqe$L>)$sJx1~+Ip_yFQX%|bH$?oPN6NgT{v*2YoCb~_z@ z)xh~j1LhePse}>sY{GvuJp>e+Rb`m$FDT4)5Ei4gNQ3in2n48Z5JxHQs2BfWtNdoE*ob|Xi6|n{X zIWggOKMgR+ag{I1q7tvIS`gW++iH*!%s$#|QVr4`s+%}zmQrx{GiDnoSNaQ<1qd;# z?V+TlC69X81kq_Ab9Jw&W2#1 zNHu=%aMb-9!voa0oif#cgjJSJF&@BsFO@enQ2P9y7}I8qe<8Y}w3d4f{?GbAQ<*X@ zzu?V6zp%WD>)x`D@V^N11rD(Y&02CT55N9(4jJrh$u<53 z^0#E|iu4nhgdqAHq@d7KN^Na}@2P7wFhat^(}gu;SWByr46UD4s~O>R>aob`p`rPI zk758&BFY3rD9SNMk|~v)Y51EMkZ31UVhchki&OK93|tpIYx02@rw8@*65@|arjDav!NO4u;b(hcwwy*(*lpAao9 z@vn+f&gL_FbdZVIno08ouQlI%b#3<=OL+>P~%n!>rnyV|00G|=#>IH2*V!BprGj1LhhkM8K;G8fdK%WIW)0vJDA z*XZl)kfSUo>Xym2o=qIe(aejHq$Eu)UTNCU{8!1L^!WBMqG%u%}k4=qOZ zfo4XKu=WzY7F*fG1Xi9p%qw!d_?&bjt-IXs-%%21&P{1Q43TqMzw9_oE1etC8zm?H zDg8F$^b(wn(tW>vyCm^S=u2ku`1}4pTie8Cj4}5l=&;2f_W~5#k!;#7chbk>4B`E> z={i}d9T&M&U53Q24-(IhSQ65Nk(c@>oo(8vjZdbz#$Wpz%bt4xp~XJ09GNRef%k^X z`_VqK1m|H*MiG@b3-^G)5*!{9(YKgXs7|k~1I-cr;eyiQUd8qqTQM?WhMRhiUA8 zVbrC!^Z19Qt(^P|{_9v*dRngN6?CrjOWg&gkz9j7SIH>4VfktYx&5ldD z?R+92&32~V)56U8!eJGf0?AC(Sz^fp4Ft%^Wip3Kq`|}yuDcxGk9b4P7x1x+#u%2eDh_Z*^bM!># zAIlf>;SdH8(eD=;2`&3ygS4rfZ@bv??{=#+K2DBL%hGsFK zo?Wk1gVy0{`i>lF4*#?n&^l}u=RQv$Ha zF6Nu@YQ-n=*^FisFq4z1I_Xv1YZJrKE_&MFI19NKQ=ncAvHk;#5h2b!+)blGa(j^+R^LytMX#_4Vza_*iDCbF^0* zgGIkYJSI}U>0iNXRHo4F&YWfJqbx@#T9T`9F5^i z(3yXdqcf$(4#){07Sc>wBXZbuo=A_Hw+?IW2Y_*IQN-8O zekdC6WNu)bNHZRw@ptv!;cp*=s_GA1tEpL*pGf%b^6hymCEE7U;|mPWz6pS&U}=3X zk0>&!1?}plTnfjbvVdy*=`L!&?Gna;p+_I#hUpuhNbRtD-P;I4SYL_?& z06hPMn4+xZOYt2OiGAVaaRVi^o5#4%@b~p+-*uwbk6DRJjAZ<4SFE+$2tYba>0;&) z-$(fjx=}$_%{3pgJ$0|j--W2wGl))EBBMKeqNlXtyuxb}j0yu{A&?LSVVJjaEP_Bl zIYJI00SFgtcAx{R&?8Q2E&Y=vD46wcHBFg;b^p&5UbX?EjV@*8T#-n-RH8lsi;&}& z$OWIA1jL*@#y?JMAL($M5K_%G1B=H(%{?({~E}h0gPw<(8v(#0rxw(H; zOuWFHU)6rxuLxzCpBU7-T#0{C(9GC2k!{E)fChlrU9)99%j~EJnSeG(U|u%@Vdxrn zob=L4?5`)=INHh5tarq&nXIjW_a$D2&1ZCNZzC`CDPzrmCH<-K5!f5>V1ffWT5mY) zwu6Y2i;;m0-?G9rY3VOF9{9PXfvQqGbPYagP(71GVIJF&ldxEsOW4nOBSx{o@k&Ai ziSm8YrH(Y-a;heO$WDT+_wLIm-q53gt-0$7U565wTA$1|3?_Gfd*W@+MJ^P7yyslr z2iGpzAW_W|oIj#LP-l_!v?lb$Wf4=Hqg6Qwx=#60<178)^>L3DtRxyak+g%H!VdvX zpv`%|S-NT!sU9~EOg{zuhShH$QdH^d50o3Q{L~y#VeL3zO#x1vb}X))JwD4nvJMMA zE71{J8WJ%RRmA$V2Sam7HCX*fJPXWxk1yyo$o^#`eP^iIsn?DWK{Yw4h5v#6*@q|8 z0L$2-sJV3pe+*e;Q|A@dd%;Pcf}Ig*L%E?7etQ9*=Xg_7Y-uI?W>Z! zFtn*i^J{aM4fCam3H;9-I%Z007ekfjsN|h{wb}d8aU6EjUThbmm$o*GuBjXpP+JP+ z>WB=o-*&K--634I*~dmizqx$#f& zFVCvJlJib49Pht=PUH0m3rj;oREzw4L1P)ZpK`Y9j*;PK2$sE0a$i$i>tgQ)Zwe^x z$lq=F1@L;;QsWzPfIa5G$Io*Fv5t%-qa?ILZ#M3ziNjX@;x| Date: Fri, 5 Mar 2021 05:58:18 -0800 Subject: [PATCH 154/228] Upgrade nebula.netflixoss to replace bintray publication and update TravisCI Secrets (#920) --- .gitignore | 4 ++++ .travis.yml | 16 +++++++++------- build.gradle | 4 ++-- buildViaTravis.sh | 11 ++++++----- gradle/wrapper/gradle-wrapper.properties | 2 +- installViaTravis.sh | 15 +++------------ priam/build.gradle | 9 ++++++++- .../priam/backupv2/SnapshotMetaTask.java | 2 +- .../identity/token/TokenRetrieverUtils.java | 2 -- secrets/signing-key.enc | Bin 0 -> 6800 bytes 10 files changed, 34 insertions(+), 31 deletions(-) create mode 100644 secrets/signing-key.enc diff --git a/.gitignore b/.gitignore index 8163af2a8..c1862c52c 100644 --- a/.gitignore +++ b/.gitignore @@ -41,6 +41,7 @@ Thumbs.db .gradle .m2 + # Build output directies /target */target @@ -67,3 +68,6 @@ atlassian-ide-plugin.xml # NetBeans specific files/directories .nbattrs >>>>>>> build/multi-project + +# publishing secrets +secrets/signing-key diff --git a/.travis.yml b/.travis.yml index 29dbc1298..9e6b9926a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,14 +4,16 @@ jdk: sudo: false git: depth: 100 -install: ./installViaTravis.sh -script: ./buildViaTravis.sh +install: "./installViaTravis.sh" +script: "./buildViaTravis.sh" cache: directories: - - $HOME/.gradle + - "$HOME/.gradle" env: global: - - secure: p81PYYYYhjhp1uJHOoT9EfUia8qPMphdykCxQannWMVvIFKXF5ibi/Qd52OUIYuJT9Q5MN9GhlwCtolxmCZu44YMLi99LsoITAv6n08LAFEux1oAOXmGt3ZBllvBFtlW+jlQ5lKmbuuPue3RVbTaWzL3KCCaYQ4uKW/sP31bXic= - - secure: AYWrIC/1JqAVyLhzlDhL163LypPpq7AliDltfey9emxzS73vB5VrGTHG45Em7xdkHN0kc1vED+UFsYks7ym1olXZPsbW29R0Gfd+p1VxCE3Cbhj7x690xNC3wIlSgWGpbgiPs7vhVN80OuNgnpz9/+WRFpKAVrYrcisbvugsDqk= - - secure: Ai75crFDh3imkmxzNQDXIwK49appZuZTrsmdZzgH1Zl/nD33RoZPWUWeeHP0yJoicNqZlU6aEXJnk8JGDWtPIT3VNO7Rsey+yrw041rHNV6bFGpwGdQgKv5CUhRHK+gShgFgnME5THCPKvK51+li1feIoByoYVmEn0gmvcxL0yM= - - secure: ZqBsyaFMBz3cqM7UHq0IRotTPaoBNUHrC7buiAqqhZ5JNM48a2m7ORuu/SENPFfr6xOT5KylCrvT6eD1bkQi5S2jT9Qyoju/9H/hXaAUG89dhwKOxqEE11ZGKF8DlNO/jcmchVnIkgWfEjVKc7vCAww1Y8lgxhyj4gRtnln+F+o= + - secure: bXzjPA0Ec3NBIN25knd9UYzGtxxmlhoLkO0LwI87iRSeCLzvbT+qBl/chvaXCk3DC2JnSq6GGWw2r0R6pjR4UezZx72KP+dm19cy606e7np1mHZmzziSkyjaBd84HPgOz8L4jh5G9qQBG8VhkjikIabAgsbiWG3oX6a/rOadobg= + - secure: COIxG87u7BSBAD0s8g0E9PTq4XLompgW/PL55bnIxuVkIo3tXXqShAxMzsJ0C++q5AbofNUICqTbGqN0ozk9GpMC78d5pgWS1O4hpdWsouK8Fxcju+KkKC5MUDC478G/zW0uSa0uZsZGaDy3Gx17jpvq2CdVCgHmaHmGGOxfDbA= + - secure: LMyJ2v4kN+07Gz5pPHkYLOvbQOAI05bHFGUVfaAbofyiR1DqKpKqB4hYaKSgb0kB/3YI6u9dLQ5AWzAqfx3HY0+p2iTmgseIxbHGrFfOH7bt2xusYNUtYPn1JXOAz1Nzxq/a+4rCgqD9lFI19XetrI5SNrRIRErDIZnWdXyhW/M= + - secure: dxJVfXpC/yORQWg2oaw+Q2UjeoAHiqf5hyLVSdGbIJYVrSu/fyFCuUNjQffikjleoyuaPeIHxaGQZoQZE3+9Cf3d7s1lXzugO3UOxmKR9UYi8tz6sLUFeLPjVhR6Etx603A8oCg659RhD2VyOCZvq1/R2gUa34tUtnVyoFyUzDA= + - secure: tl59nbZhOzWqW/PLp91+Paugond4Lwwj/qT1Tq4EFJ9Q2nNMFW3twlWzmUOcgn7GaJZTgfoDx2HGYFg1N3rrBtZIY/Wtzz9eWrGY4ywHfoXz/8lQGtbciDjiNePByuvSLOPWeTIGncOLGNUcCPNFOO9+kC4BzUGr9CjtTc+qnWg= + - secure: AZoBLO43P+GVjQq+vsT8mekOUNqifi+tRYukz19GLN2vv2vsgY4ejhXjeer1kwqm0l/1bXtfqo0uHyD+CFn2yINV20oKHScYNS4fwKnJ8aC60MZqAj6zddfe2VvEeDIKDiejpz4AfRJwgqpl9B0FrNtZ/1oU5n8oneTKpVFryTs= diff --git a/build.gradle b/build.gradle index c3f6b6b02..591a397c3 100644 --- a/build.gradle +++ b/build.gradle @@ -1,5 +1,5 @@ plugins { - id 'nebula.netflixoss' version '7.1.3' + id 'nebula.netflixoss' version '9.1.0' id 'com.github.sherter.google-java-format' version '0.8' } @@ -78,7 +78,7 @@ allprojects { compile 'com.google.http-client:google-http-client-jackson2:1.28.0' compile 'com.netflix.spectator:spectator-api:0.96.0' compileOnly 'javax.servlet:javax.servlet-api:3.1.0' - testCompile 'org.jmockit:jmockit:1.31' + testCompile 'org.jmockit:jmockit:1.38' testCompile "org.spockframework:spock-core:1.1-groovy-2.4" testCompile "com.google.truth:truth:1.0.1" testImplementation 'org.junit.jupiter:junit-jupiter-api:5.3.1' diff --git a/buildViaTravis.sh b/buildViaTravis.sh index 812026d7a..c3ddb674c 100755 --- a/buildViaTravis.sh +++ b/buildViaTravis.sh @@ -3,21 +3,22 @@ if [ "$TRAVIS_PULL_REQUEST" != "false" ]; then echo -e "Build Pull Request #$TRAVIS_PULL_REQUEST => Branch [$TRAVIS_BRANCH]" - ./gradlew build --stacktrace + ./gradlew build elif [ "$TRAVIS_PULL_REQUEST" == "false" ] && [ "$TRAVIS_TAG" == "" ]; then echo -e 'Build Branch with Snapshot => Branch ['$TRAVIS_BRANCH']' - ./gradlew -Prelease.travisci=true -PbintrayUser="${bintrayUser}" -PbintrayKey="${bintrayKey}" -PsonatypeUsername="${sonatypeUsername}" -PsonatypePassword="${sonatypePassword}" build snapshot --stacktrace + ./gradlew -Prelease.travisci=true -PnetflixOss.username="$NETFLIX_OSS_REPO_USERNAME" -PnetflixOss.password="$NETFLIX_OSS_REPO_PASSWORD" -Psonatype.signingPassword="$NETFLIX_OSS_SIGNING_PASSWORD" build snapshot elif [ "$TRAVIS_PULL_REQUEST" == "false" ] && [ "$TRAVIS_TAG" != "" ]; then echo -e 'Build Branch for Release => Branch ['$TRAVIS_BRANCH'] Tag ['$TRAVIS_TAG']' case "$TRAVIS_TAG" in *-rc\.*) - ./gradlew -Prelease.travisci=true -Prelease.useLastTag=true -PbintrayUser="${bintrayUser}" -PbintrayKey="${bintrayKey}" -PsonatypeUsername="${sonatypeUsername}" -PsonatypePassword="${sonatypePassword}" candidate --stacktrace + ./gradlew -Prelease.travisci=true -PnetflixOss.username="$NETFLIX_OSS_REPO_USERNAME" -PnetflixOss.password="$NETFLIX_OSS_REPO_PASSWORD" -Psonatype.signingPassword="$NETFLIX_OSS_SIGNING_PASSWORD" -Prelease.useLastTag=true candidate ;; *) - ./gradlew -Prelease.travisci=true -Prelease.useLastTag=true -PbintrayUser="${bintrayUser}" -PbintrayKey="${bintrayKey}" -PsonatypeUsername="${sonatypeUsername}" -PsonatypePassword="${sonatypePassword}" final --stacktrace + ./gradlew -Prelease.travisci=true -PnetflixOss.username="$NETFLIX_OSS_REPO_USERNAME" -PnetflixOss.password="$NETFLIX_OSS_REPO_PASSWORD" -Psonatype.username="$NETFLIX_OSS_SONATYPE_USERNAME" -Psonatype.password="$NETFLIX_OSS_SONATYPE_PASSWORD" -Psonatype.signingPassword="$NETFLIX_OSS_SIGNING_PASSWORD" -Prelease.useLastTag=true final ;; esac else echo -e 'WARN: Should not be here => Branch ['$TRAVIS_BRANCH'] Tag ['$TRAVIS_TAG'] Pull Request ['$TRAVIS_PULL_REQUEST']' - ./gradlew build --stacktrace + ./gradlew build fi + diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index 717f03890..68ca99ac4 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -2,4 +2,4 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-4.8.1-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-6.8.3-bin.zip diff --git a/installViaTravis.sh b/installViaTravis.sh index 06a86291c..82cf1b880 100755 --- a/installViaTravis.sh +++ b/installViaTravis.sh @@ -1,16 +1,7 @@ #!/bin/bash # This script will build the project. -if [ "$TRAVIS_PULL_REQUEST" != "false" ]; then - echo -e "Assemble Pull Request #$TRAVIS_PULL_REQUEST => Branch [$TRAVIS_BRANCH]" - ./gradlew assemble --stacktrace -elif [ "$TRAVIS_PULL_REQUEST" == "false" ] && [ "$TRAVIS_TAG" == "" ]; then - echo -e 'Assemble Branch with Snapshot => Branch ['$TRAVIS_BRANCH']' - ./gradlew -Prelease.travisci=true assemble --stacktrace -elif [ "$TRAVIS_PULL_REQUEST" == "false" ] && [ "$TRAVIS_TAG" != "" ]; then - echo -e 'Assemble Branch for Release => Branch ['$TRAVIS_BRANCH'] Tag ['$TRAVIS_TAG']' - ./gradlew -Prelease.travisci=true -Prelease.useLastTag=true assemble --stacktrace -else - echo -e 'WARN: Should not be here => Branch ['$TRAVIS_BRANCH'] Tag ['$TRAVIS_TAG'] Pull Request ['$TRAVIS_PULL_REQUEST']' - ./gradlew assemble --stacktrace +if [ "$TRAVIS_SECURE_ENV_VARS" = "true" ]; then + echo "Decrypting publishing credentials" + openssl aes-256-cbc -k "$NETFLIX_OSS_SIGNING_FILE_PASSWORD" -in secrets/signing-key.enc -out secrets/signing-key -d fi diff --git a/priam/build.gradle b/priam/build.gradle index 723f3d03d..bbafaa0aa 100644 --- a/priam/build.gradle +++ b/priam/build.gradle @@ -1 +1,8 @@ -apply plugin: 'groovy' \ No newline at end of file +apply plugin: 'groovy' + +/** + * This is from https://jmockit.github.io/tutorial/Introduction.html#runningTests + */ +test { + jvmArgs "-javaagent:${classpath.find { it.name.contains("jmockit") }.absolutePath}" +} diff --git a/priam/src/main/java/com/netflix/priam/backupv2/SnapshotMetaTask.java b/priam/src/main/java/com/netflix/priam/backupv2/SnapshotMetaTask.java index 5ac26dc8f..871f021e2 100644 --- a/priam/src/main/java/com/netflix/priam/backupv2/SnapshotMetaTask.java +++ b/priam/src/main/java/com/netflix/priam/backupv2/SnapshotMetaTask.java @@ -415,7 +415,7 @@ ImmutableSetMultimap getFileUploadResults( * Gives the prefix (common name) of the sstable components. Returns an empty Optional if it is * not an sstable component or a manifest or schema file. * - *

    For example: mc-3-big-Data.db --> mc-3-big ks-cf-ka-7213-Index.db --> ks-cf-ka-7213 + *

    For example: mc-3-big-Data.db -- mc-3-big ks-cf-ka-7213-Index.db -- ks-cf-ka-7213 * * @param file the file from which to extract a common prefix. * @return common prefix of the file, or empty, diff --git a/priam/src/main/java/com/netflix/priam/identity/token/TokenRetrieverUtils.java b/priam/src/main/java/com/netflix/priam/identity/token/TokenRetrieverUtils.java index 700f3ef92..1fd35b7ec 100644 --- a/priam/src/main/java/com/netflix/priam/identity/token/TokenRetrieverUtils.java +++ b/priam/src/main/java/com/netflix/priam/identity/token/TokenRetrieverUtils.java @@ -30,8 +30,6 @@ public class TokenRetrieverUtils { * @param dc * @return IP of the token owner based on gossip information or null if C* status doesn't * converge. - * @throws GossipParseException when required number of instances are not available to fetch the - * gossip info. */ public static InferredTokenOwnership inferTokenOwnerFromGossip( List allIds, String token, String dc) { diff --git a/secrets/signing-key.enc b/secrets/signing-key.enc new file mode 100644 index 0000000000000000000000000000000000000000..ce0833210e7c7d67f8e1ce477f263c20886463b7 GIT binary patch literal 6800 zcmV;B8gJ!OVQh3|WM5xs`pV8kFyX+=Gru0$5*eNlQx2_`Ym3npRuD}yfbqLdh6&BJ z6mq5A=!sT2ZBIfFi~_J)d#HX%)$*qa?Bk9#@8e1lEk2%XR?RO|p|Rh;9I!~45r^H- zq8$p@Z`f_{L#S{iTbmd}By-(NWF-(dQF*q|=?O}8dJBbz)+e@-8(nv%BdEg5d~=ux zilU1zEp3(v$HH=}Mk+H9TluL=MV1c4J=y{i4Wu>sD+@uoTZpN;_%RcvDWE7I_f z(c(xQEvsoQ?`K1LLLPeV1Rx$Z$pDgGp~EWPu00x?#Gu;R#!OwSpezeDM*$=bSDUhC zlt6dmYye*+`?!UlGn+<)BrB?b6{O_@XTBXQBz?s2Q9dXD)85WdFC8@7W-L>Mb@4sv zbwxi2+!D!+GW7M%{K5T!_WSw$3|JC-npr#Wa2^Rv|Fhigxr%5kX9F=G=Yci@>f}NP z_9j$Vo7P^DQo#gf6yQQPYLIdOUx2v3Jn$pKbtjQsXDE6OS!OnYj=UO*SQlFF2Rjom zHo13-*BS-qah@!sv`t&UmIt!l zL&hF!iG3*yvUKP^CDIZ%Q6j}SyTl;@%J09xE(ramXU7U2oKV|k3@*gRR{l#MOKFWa zku0nGvVFC4zwD%&Ckkujjnhxk)S zedH_sl`$_(QMTFUvuk%&e-Ng!t~b_UlUk zeT=PL@f*lJt237ZJ)ww1vzX=RtH~mXc zRf3&BtqxjH!@kX^?sum4fR|zAv<1Nkn!?FmPU#Y?HyH0f$HoW!QxhfWDbEV-q4$Ol zvy)d158*?9kn39ZMl`Y*9Ldo8&$54F^Pv8jWZlYM2{s*jW8EuCe4+M_Kd335V zT}ep#c^BQ14_LTnnG-k_sOBA;0A|!{=4% z%Sv3n#_AMn%UowlEbAQ08PRY_i1KVUKztw56<>;a+;eXIdx>ZuMNd{qL3OBD-)7qohYBwiXp^rHwn2S5o`(`ZkuCfP(I~nkyrJfrbqo?&m5m~lj&?cG_?P*91}w$qulaoUpsL#7 z#BE{TjtQRWiR23T=8j5H1iqdtc(?qn1?LF|W_nQ(!YOPGuc1UNV- zqeEvH!s%6cVYOScPE>vzNA>i|dRVW;ZkXQ+G`M8@=?#Py7_bDn>WhYJGVF|vpUXz` zaA@rLRB=KdW>thBL#G#dzxC6PCzHMDxrKk8veS+2qchc|ndlUn^ob@|7x))B(B+cS z6B9Df_@1h{e|zsC4jOz6I;4-R(gB&Qee>QK|Oarj<6RKR$oUcb0cPwT1>G$Brk zvp__+(`q$o7s@ttm1Ap|RNd?=OegNk(+JC}M(!K4C)gb;(3f@Fq&2@YW7e-G11?Ud zq@~S=RFMG1FZ{SR&rFbz8q{@VWon)#xqm-pE0eI4hcR-BL+orH)2<_ML;Rl*XXF2w&l>G5=+7`h&i_(H_z}m(B#1a6|`4w<%1wGOF&QzOZfrg!)K4)<%73{ zZ}aquQ_Mu*K?b$LfLAZ8sTAx(o`V9rY8sRXEg&^VJB#`jgc(UcopLKe;3U+;F;OA`;^YxxFI&{pwbRWxYBNGYZRNANn4ex z>AHpEl2E2L|SAVsTz~OsoDj{DfJJt^VOq*ic8qpYp=35`ES?K zsIueQlvL)?RSJw4re8kK*UU9zKnH~ZSE~Z~7HEb#XVUyvbs7mg=5wi~Fdm`AEAY6T zrGX3oHWZ+B2PPJ;NlB{#vDt@ErlF>$15v_DbAoO3fX?ujLSxa7zZ+pC-yPBtx)OAR zz{*pjL;XRmJHvLb)Z%wyw0ORQw_c?r56?Y=?@yCvu+abj(MWn=B-#*S%BJm7q9bG_ zgP>24hPA@Xm7DZ)ZP?UT_)=ABOhk@86CHZ$Ps?mm&GaJ&R1FUQJnS6F8jLxr6bw|C zuR1AP&|gC}`A2?By*S7T&qjEUn3J)<6Q@s5U`zDrPvcY4_u{r|Kj=~kC>?gR&2A5{ zLTUipjq&YKI!Y*rLu{d>ZEoR;*n8D{cCj9dW56-M=yoPmTPWg37z-kfF|BEDs z^$wv|v>U%Jkn@x*mbhA3W(ksMam)R4kdbPUNB!e$$vSmQq$lLa>*nAf^^Fpaqy2w{F}uAU2(66NWydRScLp>c=gz2Tvs zV@c;H-X{tDu1ym}%g~kbhSB~tJ=^6f(j*dUlwws+7Im9wV#1h4mbHyC7bWnOlgegW zwFc(u+8Ug%Fyo%yBRn70EnhgcPVXS)v$nO+IePGZ0?_|?S98c^NT?WxE7H&GnRgtm z5ijm;wbg#Aonduv2Vz2oLTVlQj=;U^RO6_nPm!o=_wCnmv8?Zu%yMoiGzMhG z`vkHIb%9c=Hv3#4z|0i8$>~88gNv5s&=I@lzAQI#h<%+x#6SU($U4RWO}-Z5-m9sp zU?ptn{~rB_lni=y`B;q_S$e_<-zeSXZfCJ z+?B`$)R&zN#cXQM2c69i(aq2e0cMZXjZ5*>D!Xs6PJVOU*$n^OQRV!@Zl!zl0%=2M z0?2{A+3GPFzuf#Wqf=wYVbi_R(7SE;P=93w z5t)cDCHi*zn}kdUOY$>pCYd?(UnYEsGafxQKAK9Jkt*k8peJm-s>^YowzE^A|>40NJ=2lu559S~qtr;96l>ul zmHO5o@D$dT6`Kq{Y7E%tqe$L>)$sJx1~+Ip_yFQX%|bH$?oPN6NgT{v*2YoCb~_z@ z)xh~j1LhePse}>sY{GvuJp>e+Rb`m$FDT4)5Ei4gNQ3in2n48Z5JxHQs2BfWtNdoE*ob|Xi6|n{X zIWggOKMgR+ag{I1q7tvIS`gW++iH*!%s$#|QVr4`s+%}zmQrx{GiDnoSNaQ<1qd;# z?V+TlC69X81kq_Ab9Jw&W2#1 zNHu=%aMb-9!voa0oif#cgjJSJF&@BsFO@enQ2P9y7}I8qe<8Y}w3d4f{?GbAQ<*X@ zzu?V6zp%WD>)x`D@V^N11rD(Y&02CT55N9(4jJrh$u<53 z^0#E|iu4nhgdqAHq@d7KN^Na}@2P7wFhat^(}gu;SWByr46UD4s~O>R>aob`p`rPI zk758&BFY3rD9SNMk|~v)Y51EMkZ31UVhchki&OK93|tpIYx02@rw8@*65@|arjDav!NO4u;b(hcwwy*(*lpAao9 z@vn+f&gL_FbdZVIno08ouQlI%b#3<=OL+>P~%n!>rnyV|00G|=#>IH2*V!BprGj1LhhkM8K;G8fdK%WIW)0vJDA z*XZl)kfSUo>Xym2o=qIe(aejHq$Eu)UTNCU{8!1L^!WBMqG%u%}k4=qOZ zfo4XKu=WzY7F*fG1Xi9p%qw!d_?&bjt-IXs-%%21&P{1Q43TqMzw9_oE1etC8zm?H zDg8F$^b(wn(tW>vyCm^S=u2ku`1}4pTie8Cj4}5l=&;2f_W~5#k!;#7chbk>4B`E> z={i}d9T&M&U53Q24-(IhSQ65Nk(c@>oo(8vjZdbz#$Wpz%bt4xp~XJ09GNRef%k^X z`_VqK1m|H*MiG@b3-^G)5*!{9(YKgXs7|k~1I-cr;eyiQUd8qqTQM?WhMRhiUA8 zVbrC!^Z19Qt(^P|{_9v*dRngN6?CrjOWg&gkz9j7SIH>4VfktYx&5ldD z?R+92&32~V)56U8!eJGf0?AC(Sz^fp4Ft%^Wip3Kq`|}yuDcxGk9b4P7x1x+#u%2eDh_Z*^bM!># zAIlf>;SdH8(eD=;2`&3ygS4rfZ@bv??{=#+K2DBL%hGsFK zo?Wk1gVy0{`i>lF4*#?n&^l}u=RQv$Ha zF6Nu@YQ-n=*^FisFq4z1I_Xv1YZJrKE_&MFI19NKQ=ncAvHk;#5h2b!+)blGa(j^+R^LytMX#_4Vza_*iDCbF^0* zgGIkYJSI}U>0iNXRHo4F&YWfJqbx@#T9T`9F5^i z(3yXdqcf$(4#){07Sc>wBXZbuo=A_Hw+?IW2Y_*IQN-8O zekdC6WNu)bNHZRw@ptv!;cp*=s_GA1tEpL*pGf%b^6hymCEE7U;|mPWz6pS&U}=3X zk0>&!1?}plTnfjbvVdy*=`L!&?Gna;p+_I#hUpuhNbRtD-P;I4SYL_?& z06hPMn4+xZOYt2OiGAVaaRVi^o5#4%@b~p+-*uwbk6DRJjAZ<4SFE+$2tYba>0;&) z-$(fjx=}$_%{3pgJ$0|j--W2wGl))EBBMKeqNlXtyuxb}j0yu{A&?LSVVJjaEP_Bl zIYJI00SFgtcAx{R&?8Q2E&Y=vD46wcHBFg;b^p&5UbX?EjV@*8T#-n-RH8lsi;&}& z$OWIA1jL*@#y?JMAL($M5K_%G1B=H(%{?({~E}h0gPw<(8v(#0rxw(H; zOuWFHU)6rxuLxzCpBU7-T#0{C(9GC2k!{E)fChlrU9)99%j~EJnSeG(U|u%@Vdxrn zob=L4?5`)=INHh5tarq&nXIjW_a$D2&1ZCNZzC`CDPzrmCH<-K5!f5>V1ffWT5mY) zwu6Y2i;;m0-?G9rY3VOF9{9PXfvQqGbPYagP(71GVIJF&ldxEsOW4nOBSx{o@k&Ai ziSm8YrH(Y-a;heO$WDT+_wLIm-q53gt-0$7U565wTA$1|3?_Gfd*W@+MJ^P7yyslr z2iGpzAW_W|oIj#LP-l_!v?lb$Wf4=Hqg6Qwx=#60<178)^>L3DtRxyak+g%H!VdvX zpv`%|S-NT!sU9~EOg{zuhShH$QdH^d50o3Q{L~y#VeL3zO#x1vb}X))JwD4nvJMMA zE71{J8WJ%RRmA$V2Sam7HCX*fJPXWxk1yyo$o^#`eP^iIsn?DWK{Yw4h5v#6*@q|8 z0L$2-sJV3pe+*e;Q|A@dd%;Pcf}Ig*L%E?7etQ9*=Xg_7Y-uI?W>Z! zFtn*i^J{aM4fCam3H;9-I%Z007ekfjsN|h{wb}d8aU6EjUThbmm$o*GuBjXpP+JP+ z>WB=o-*&K--634I*~dmizqx$#f& zFVCvJlJib49Pht=PUH0m3rj;oREzw4L1P)ZpK`Y9j*;PK2$sE0a$i$i>tgQ)Zwe^x z$lq=F1@L;;QsWzPfIa5G$Io*Fv5t%-qa?ILZ#M3ziNjX@;x| Date: Mon, 8 Mar 2021 21:45:56 -0800 Subject: [PATCH 155/228] Revert "Adding support for custom override for role_manager (#918)" This reverts commit 243afcf1f19ddc052f372108aac25d795d98e4d1. --- .../netflix/priam/config/IConfiguration.java | 5 --- .../priam/config/PriamConfiguration.java | 5 --- .../netflix/priam/tuner/StandardTuner.java | 1 - .../priam/config/FakeConfiguration.java | 11 ------- .../priam/tuner/StandardTunerTest.java | 33 +++++++------------ 5 files changed, 11 insertions(+), 44 deletions(-) diff --git a/priam/src/main/java/com/netflix/priam/config/IConfiguration.java b/priam/src/main/java/com/netflix/priam/config/IConfiguration.java index 76f574767..872353ef0 100644 --- a/priam/src/main/java/com/netflix/priam/config/IConfiguration.java +++ b/priam/src/main/java/com/netflix/priam/config/IConfiguration.java @@ -560,11 +560,6 @@ default String getAuthorizer() { return "org.apache.cassandra.auth.AllowAllAuthorizer"; } - /** Defaults to 'CassandraRoleManager'. */ - default String getRoleManager() { - return "org.apache.cassandra.auth.CassandraRoleManager"; - } - /** @return true/false, if Cassandra needs to be started manually */ default boolean doesCassandraStartManually() { return false; diff --git a/priam/src/main/java/com/netflix/priam/config/PriamConfiguration.java b/priam/src/main/java/com/netflix/priam/config/PriamConfiguration.java index d9b77d224..83b48fd71 100644 --- a/priam/src/main/java/com/netflix/priam/config/PriamConfiguration.java +++ b/priam/src/main/java/com/netflix/priam/config/PriamConfiguration.java @@ -427,11 +427,6 @@ public String getAuthorizer() { PRIAM_PRE + ".authorizer", "org.apache.cassandra.auth.AllowAllAuthorizer"); } - public String getRoleManager() { - return config.get( - PRIAM_PRE + ".roleManager", "org.apache.cassandra.auth.CassandraRoleManager"); - } - @Override public boolean doesCassandraStartManually() { return config.get(PRIAM_PRE + ".cass.manual.start.enable", false); diff --git a/priam/src/main/java/com/netflix/priam/tuner/StandardTuner.java b/priam/src/main/java/com/netflix/priam/tuner/StandardTuner.java index d36f9cf09..ac37b3ad0 100644 --- a/priam/src/main/java/com/netflix/priam/tuner/StandardTuner.java +++ b/priam/src/main/java/com/netflix/priam/tuner/StandardTuner.java @@ -105,7 +105,6 @@ public void writeAllProperties(String yamlLocation, String hostname, String seed map.put("hinted_handoff_throttle_in_kb", config.getHintedHandoffThrottleKb()); map.put("authenticator", config.getAuthenticator()); map.put("authorizer", config.getAuthorizer()); - map.put("role_manager", config.getRoleManager()); map.put("internode_compression", config.getInternodeCompression()); map.put("dynamic_snitch", config.isDynamicSnitchEnabled()); diff --git a/priam/src/test/java/com/netflix/priam/config/FakeConfiguration.java b/priam/src/test/java/com/netflix/priam/config/FakeConfiguration.java index 31a5f86e3..edc05bfac 100644 --- a/priam/src/test/java/com/netflix/priam/config/FakeConfiguration.java +++ b/priam/src/test/java/com/netflix/priam/config/FakeConfiguration.java @@ -31,7 +31,6 @@ public class FakeConfiguration implements IConfiguration { private final String appName; private String restorePrefix = ""; public Map fakeConfig; - private String roleManager = ""; public Map fakeProperties = new HashMap<>(); @@ -185,16 +184,6 @@ public ImmutableSet getTunablePropertyFiles() { return ImmutableSet.of(path + "/cassandra-rackdc.properties"); } - @Override - public String getRoleManager() { - return this.roleManager; - } - - public FakeConfiguration setRoleManager(String roleManager) { - this.roleManager = roleManager; - return this; - } - public String getRAC() { return "my_zone"; } diff --git a/priam/src/test/java/com/netflix/priam/tuner/StandardTunerTest.java b/priam/src/test/java/com/netflix/priam/tuner/StandardTunerTest.java index 588c3541f..3cdf0b162 100644 --- a/priam/src/test/java/com/netflix/priam/tuner/StandardTunerTest.java +++ b/priam/src/test/java/com/netflix/priam/tuner/StandardTunerTest.java @@ -141,28 +141,11 @@ public void addExtraParams() throws Exception { extraParamValues.put(priamKeyName2, "test"); extraParamValues.put(priamKeyName3, "randomKeyValue"); extraParamValues.put(priamKeyName4, "randomGroupValue"); - - Map map = - applyFakeConfiguration(new TunerConfiguration(extraConfigParam, extraParamValues)); - Assert.assertEquals("your_host", map.get("listen_address")); - Assert.assertEquals("true", ((Map) map.get("client_encryption_options")).get("optional")); - Assert.assertEquals( - "test", ((Map) map.get("client_encryption_options")).get("keystore_password")); - Assert.assertEquals("randomKeyValue", map.get("randomKey")); - Assert.assertEquals("randomGroupValue", ((Map) map.get("randomGroup")).get("randomKey")); - } - - @Test - public void testRoleManagerOverride() throws Exception { - String roleManagerOverride = "org.apache.cassandra.auth.CustomRoleManager"; - Map map = - applyFakeConfiguration(new FakeConfiguration().setRoleManager(roleManagerOverride)); - Assert.assertEquals(roleManagerOverride, map.get("role_manager")); - } - - private Map applyFakeConfiguration(FakeConfiguration fakeConfiguration) throws Exception { StandardTuner tuner = - new StandardTuner(fakeConfiguration, backupRestoreConfig, instanceInfo); + new StandardTuner( + new TunerConfiguration(extraConfigParam, extraParamValues), + backupRestoreConfig, + instanceInfo); Files.copy(new File("src/main/resources/incr-restore-cassandra.yaml"), target); tuner.writeAllProperties(target.getAbsolutePath(), "your_host", "YourSeedProvider"); @@ -170,7 +153,13 @@ private Map applyFakeConfiguration(FakeConfiguration fakeConfiguration) throws E DumperOptions options = new DumperOptions(); options.setDefaultFlowStyle(DumperOptions.FlowStyle.BLOCK); Yaml yaml = new Yaml(options); - return yaml.load(new FileInputStream(target)); + Map map = yaml.load(new FileInputStream(target)); + Assert.assertEquals("your_host", map.get("listen_address")); + Assert.assertEquals("true", ((Map) map.get("client_encryption_options")).get("optional")); + Assert.assertEquals( + "test", ((Map) map.get("client_encryption_options")).get("keystore_password")); + Assert.assertEquals("randomKeyValue", map.get("randomKey")); + Assert.assertEquals("randomGroupValue", ((Map) map.get("randomGroup")).get("randomKey")); } private class TunerConfiguration extends FakeConfiguration { From c563283f4088f958f4a8a32fc010488d55294c65 Mon Sep 17 00:00:00 2001 From: Sumanth Pasupuleti Date: Mon, 8 Mar 2021 21:54:26 -0800 Subject: [PATCH 156/228] Revert "Revert "Adding support for custom override for role_manager (#918)"" This reverts commit b1b9b11e3183f987b6f06691d3d2dca9b465d8f1. --- .../netflix/priam/config/IConfiguration.java | 5 +++ .../priam/config/PriamConfiguration.java | 5 +++ .../netflix/priam/tuner/StandardTuner.java | 1 + .../priam/config/FakeConfiguration.java | 11 +++++++ .../priam/tuner/StandardTunerTest.java | 33 ++++++++++++------- 5 files changed, 44 insertions(+), 11 deletions(-) diff --git a/priam/src/main/java/com/netflix/priam/config/IConfiguration.java b/priam/src/main/java/com/netflix/priam/config/IConfiguration.java index 872353ef0..76f574767 100644 --- a/priam/src/main/java/com/netflix/priam/config/IConfiguration.java +++ b/priam/src/main/java/com/netflix/priam/config/IConfiguration.java @@ -560,6 +560,11 @@ default String getAuthorizer() { return "org.apache.cassandra.auth.AllowAllAuthorizer"; } + /** Defaults to 'CassandraRoleManager'. */ + default String getRoleManager() { + return "org.apache.cassandra.auth.CassandraRoleManager"; + } + /** @return true/false, if Cassandra needs to be started manually */ default boolean doesCassandraStartManually() { return false; diff --git a/priam/src/main/java/com/netflix/priam/config/PriamConfiguration.java b/priam/src/main/java/com/netflix/priam/config/PriamConfiguration.java index 83b48fd71..d9b77d224 100644 --- a/priam/src/main/java/com/netflix/priam/config/PriamConfiguration.java +++ b/priam/src/main/java/com/netflix/priam/config/PriamConfiguration.java @@ -427,6 +427,11 @@ public String getAuthorizer() { PRIAM_PRE + ".authorizer", "org.apache.cassandra.auth.AllowAllAuthorizer"); } + public String getRoleManager() { + return config.get( + PRIAM_PRE + ".roleManager", "org.apache.cassandra.auth.CassandraRoleManager"); + } + @Override public boolean doesCassandraStartManually() { return config.get(PRIAM_PRE + ".cass.manual.start.enable", false); diff --git a/priam/src/main/java/com/netflix/priam/tuner/StandardTuner.java b/priam/src/main/java/com/netflix/priam/tuner/StandardTuner.java index ac37b3ad0..d36f9cf09 100644 --- a/priam/src/main/java/com/netflix/priam/tuner/StandardTuner.java +++ b/priam/src/main/java/com/netflix/priam/tuner/StandardTuner.java @@ -105,6 +105,7 @@ public void writeAllProperties(String yamlLocation, String hostname, String seed map.put("hinted_handoff_throttle_in_kb", config.getHintedHandoffThrottleKb()); map.put("authenticator", config.getAuthenticator()); map.put("authorizer", config.getAuthorizer()); + map.put("role_manager", config.getRoleManager()); map.put("internode_compression", config.getInternodeCompression()); map.put("dynamic_snitch", config.isDynamicSnitchEnabled()); diff --git a/priam/src/test/java/com/netflix/priam/config/FakeConfiguration.java b/priam/src/test/java/com/netflix/priam/config/FakeConfiguration.java index edc05bfac..31a5f86e3 100644 --- a/priam/src/test/java/com/netflix/priam/config/FakeConfiguration.java +++ b/priam/src/test/java/com/netflix/priam/config/FakeConfiguration.java @@ -31,6 +31,7 @@ public class FakeConfiguration implements IConfiguration { private final String appName; private String restorePrefix = ""; public Map fakeConfig; + private String roleManager = ""; public Map fakeProperties = new HashMap<>(); @@ -184,6 +185,16 @@ public ImmutableSet getTunablePropertyFiles() { return ImmutableSet.of(path + "/cassandra-rackdc.properties"); } + @Override + public String getRoleManager() { + return this.roleManager; + } + + public FakeConfiguration setRoleManager(String roleManager) { + this.roleManager = roleManager; + return this; + } + public String getRAC() { return "my_zone"; } diff --git a/priam/src/test/java/com/netflix/priam/tuner/StandardTunerTest.java b/priam/src/test/java/com/netflix/priam/tuner/StandardTunerTest.java index 3cdf0b162..588c3541f 100644 --- a/priam/src/test/java/com/netflix/priam/tuner/StandardTunerTest.java +++ b/priam/src/test/java/com/netflix/priam/tuner/StandardTunerTest.java @@ -141,11 +141,28 @@ public void addExtraParams() throws Exception { extraParamValues.put(priamKeyName2, "test"); extraParamValues.put(priamKeyName3, "randomKeyValue"); extraParamValues.put(priamKeyName4, "randomGroupValue"); + + Map map = + applyFakeConfiguration(new TunerConfiguration(extraConfigParam, extraParamValues)); + Assert.assertEquals("your_host", map.get("listen_address")); + Assert.assertEquals("true", ((Map) map.get("client_encryption_options")).get("optional")); + Assert.assertEquals( + "test", ((Map) map.get("client_encryption_options")).get("keystore_password")); + Assert.assertEquals("randomKeyValue", map.get("randomKey")); + Assert.assertEquals("randomGroupValue", ((Map) map.get("randomGroup")).get("randomKey")); + } + + @Test + public void testRoleManagerOverride() throws Exception { + String roleManagerOverride = "org.apache.cassandra.auth.CustomRoleManager"; + Map map = + applyFakeConfiguration(new FakeConfiguration().setRoleManager(roleManagerOverride)); + Assert.assertEquals(roleManagerOverride, map.get("role_manager")); + } + + private Map applyFakeConfiguration(FakeConfiguration fakeConfiguration) throws Exception { StandardTuner tuner = - new StandardTuner( - new TunerConfiguration(extraConfigParam, extraParamValues), - backupRestoreConfig, - instanceInfo); + new StandardTuner(fakeConfiguration, backupRestoreConfig, instanceInfo); Files.copy(new File("src/main/resources/incr-restore-cassandra.yaml"), target); tuner.writeAllProperties(target.getAbsolutePath(), "your_host", "YourSeedProvider"); @@ -153,13 +170,7 @@ public void addExtraParams() throws Exception { DumperOptions options = new DumperOptions(); options.setDefaultFlowStyle(DumperOptions.FlowStyle.BLOCK); Yaml yaml = new Yaml(options); - Map map = yaml.load(new FileInputStream(target)); - Assert.assertEquals("your_host", map.get("listen_address")); - Assert.assertEquals("true", ((Map) map.get("client_encryption_options")).get("optional")); - Assert.assertEquals( - "test", ((Map) map.get("client_encryption_options")).get("keystore_password")); - Assert.assertEquals("randomKeyValue", map.get("randomKey")); - Assert.assertEquals("randomGroupValue", ((Map) map.get("randomGroup")).get("randomKey")); + return yaml.load(new FileInputStream(target)); } private class TunerConfiguration extends FakeConfiguration { From ed9eb9f3d1693964490cde335c6978733050796f Mon Sep 17 00:00:00 2001 From: Matt Lehman Date: Tue, 9 Mar 2021 05:18:49 -0800 Subject: [PATCH 157/228] CHANGELOG commit for 3.11.76 --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 129803dbe..2eab578e4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,7 @@ # Changelog +## 2021/03/09 3.11.76 +(#918) Adding support for custom override for role_manager. + ## 2020/11/09 3.11.73 (#911) Backup secondary index files. From 21497b009f98627cd4a82f9de5d2d3d6680f066b Mon Sep 17 00:00:00 2001 From: mattl-netflix <63665634+mattl-netflix@users.noreply.github.com> Date: Wed, 17 Mar 2021 16:12:39 -0700 Subject: [PATCH 158/228] Store private IPs in the token database. (#913) * CASS-1828 Consolidate PriamInstance fetching into a single interface. * CASS-1828 Remove DeadTokenRetriever interface. * CASS-1828 Remove IPreGeneratedTokenRetriever interface * CASS-1828 Remove INewTokenRetriever interface * CASS-1828 remove populateRacMap * CASS-1828 Remove sameHostPredicate * CASS-1828 Move calls to gossip when finding preassigned token to separate function. * CASS-1828 Change PriamInstance toString to include IP and shrink a log statement. * CASS-1828 tighten up grabPreassignedToken. * CASS-1828 make TokenRetriever use the same logic to get rac instances both when generating a dead token and a pregenerated token. * CASS-1828 move gossip check from grabDeadToken to function. * CASS-1828 Move deletion to separate function. * CASS-1828 move token claiming to separate function * CASS-1828 Remove redundant comments and log statements, plus some minor rearranging. * CASS-1828 combine grabDeadToken and grabPreGeneratedToken * CASS-1828 add tests of new token generation * CASS-1828 Remove redundant method from ITokenManager interface and tighten up new token generation logic * CASS-1828 use nullity of replace ip to determine whether to replace. * CASS-1828 Ensure that pregenerated token is claimed when available and the dead token fails gossip check. This corrects a bug introduced when combining the erstwhile grabDeadToken and grabPregeneratedToken methods. * CASS-1828 stop marking tokens dead and deleting them, begin updating atomically and reading consistently with SimpleDB. * CASS-1828 Compare against both IPs when checking Gossip in assigned token case * CASS-1828 update database when getting preassigned tokens * CASS-1828 make DoubleRing inject InstanceInfo instead of InstanceIdentity. * CASS-1828 use private IP when dictated by configuration. * CASS-1828 remove redundant attachVolumes method from PriamInstanceFactory * CASS-1828 remove redundant sort method from PriamInstanceFactory * CASS-1828 remove redundant generics in PriamInstanceFactory * CASS-1828 Make PriamInstanceFactory return a Set of PriamInstances rather than a List. More generally, use a Set of PriamInstances where applicable. * CASS-1828 make IMembership return an ImmutableSet of IPs not a list. * CASS-1828 make UpdateSecuritySettings always add current instance's IP to account for possibility of stale data in instance database. Remove extra call to instance database as well. --- .../com/netflix/priam/aws/AWSMembership.java | 20 +- .../netflix/priam/aws/SDBInstanceData.java | 30 +- .../netflix/priam/aws/SDBInstanceFactory.java | 42 +- .../priam/aws/UpdateSecuritySettings.java | 63 ++- .../netflix/priam/cli/StaticMembership.java | 13 +- .../netflix/priam/config/IConfiguration.java | 7 + .../netflix/priam/identity/DoubleRing.java | 46 +- .../netflix/priam/identity/IMembership.java | 10 +- .../priam/identity/IPriamInstanceFactory.java | 27 +- .../priam/identity/InstanceIdentity.java | 236 +--------- .../netflix/priam/identity/PriamInstance.java | 9 +- .../identity/token/DeadTokenRetriever.java | 208 -------- .../identity/token/IDeadTokenRetriever.java | 34 -- .../identity/token/INewTokenRetriever.java | 29 -- .../token/IPreGeneratedTokenRetriever.java | 29 -- .../priam/identity/token/ITokenRetriever.java | 15 + .../identity/token/NewTokenRetriever.java | 119 ----- .../token/PreGeneratedTokenRetriever.java | 103 ---- .../priam/identity/token/TokenRetriever.java | 313 ++++++++++++ .../identity/token/TokenRetrieverUtils.java | 3 +- .../resources/PriamInstanceResource.java | 15 +- .../netflix/priam/utils/ITokenManager.java | 2 - .../com/netflix/priam/utils/TokenManager.java | 14 - .../java/com/netflix/priam/TestModule.java | 2 +- .../priam/aws/TestUpdateSecuritySettings.java | 111 +++++ .../priam/backup/identity/DoubleRingTest.java | 11 +- .../backup/identity/InstanceIdentityTest.java | 14 +- .../backup/identity/InstanceTestUtils.java | 25 +- .../token/FakeDeadTokenRetriever.java | 43 -- .../identity/token/FakeNewTokenRetriever.java | 37 -- .../token/FakePreGeneratedTokenRetriever.java | 37 -- .../priam/config/FakeConfiguration.java | 31 +- .../priam/identity/FakeMembership.java | 28 +- .../identity/FakePriamInstanceFactory.java | 41 +- .../identity/config/FakeInstanceInfo.java | 4 +- .../token/AssignedTokenRetrieverTest.java | 130 ++--- .../token/DeadTokenRetrieverTest.java | 220 --------- .../identity/token/TokenRetrieverTest.java | 444 ++++++++++++++++++ .../token/TokenRetrieverUtilsTest.java | 30 +- .../resources/PriamInstanceResourceTest.java | 6 +- .../priam/utils/Murmur3TokenManagerTest.java | 4 +- .../priam/utils/RandomTokenManagerTest.java | 4 +- 42 files changed, 1171 insertions(+), 1438 deletions(-) delete mode 100755 priam/src/main/java/com/netflix/priam/identity/token/DeadTokenRetriever.java delete mode 100755 priam/src/main/java/com/netflix/priam/identity/token/IDeadTokenRetriever.java delete mode 100755 priam/src/main/java/com/netflix/priam/identity/token/INewTokenRetriever.java delete mode 100755 priam/src/main/java/com/netflix/priam/identity/token/IPreGeneratedTokenRetriever.java create mode 100644 priam/src/main/java/com/netflix/priam/identity/token/ITokenRetriever.java delete mode 100755 priam/src/main/java/com/netflix/priam/identity/token/NewTokenRetriever.java delete mode 100755 priam/src/main/java/com/netflix/priam/identity/token/PreGeneratedTokenRetriever.java create mode 100644 priam/src/main/java/com/netflix/priam/identity/token/TokenRetriever.java create mode 100644 priam/src/test/java/com/netflix/priam/aws/TestUpdateSecuritySettings.java delete mode 100755 priam/src/test/java/com/netflix/priam/backup/identity/token/FakeDeadTokenRetriever.java delete mode 100755 priam/src/test/java/com/netflix/priam/backup/identity/token/FakeNewTokenRetriever.java delete mode 100755 priam/src/test/java/com/netflix/priam/backup/identity/token/FakePreGeneratedTokenRetriever.java delete mode 100644 priam/src/test/java/com/netflix/priam/identity/token/DeadTokenRetrieverTest.java create mode 100644 priam/src/test/java/com/netflix/priam/identity/token/TokenRetrieverTest.java diff --git a/priam/src/main/java/com/netflix/priam/aws/AWSMembership.java b/priam/src/main/java/com/netflix/priam/aws/AWSMembership.java index ec05f304d..8373d8991 100644 --- a/priam/src/main/java/com/netflix/priam/aws/AWSMembership.java +++ b/priam/src/main/java/com/netflix/priam/aws/AWSMembership.java @@ -24,7 +24,7 @@ import com.amazonaws.services.ec2.AmazonEC2ClientBuilder; import com.amazonaws.services.ec2.model.*; import com.amazonaws.services.ec2.model.Filter; -import com.google.common.collect.Lists; +import com.google.common.collect.ImmutableSet; import com.google.inject.Inject; import com.google.inject.name.Named; import com.netflix.priam.config.IConfiguration; @@ -60,7 +60,7 @@ public AWSMembership( } @Override - public List getRacMembership() { + public ImmutableSet getRacMembership() { AmazonAutoScaling client = null; try { List asgNames = new ArrayList<>(); @@ -73,7 +73,7 @@ public List getRacMembership() { asgNames.toArray(new String[asgNames.size()])); DescribeAutoScalingGroupsResult res = client.describeAutoScalingGroups(asgReq); - List instanceIds = Lists.newArrayList(); + ImmutableSet.Builder instanceIds = ImmutableSet.builder(); for (AutoScalingGroup asg : res.getAutoScalingGroups()) { for (Instance ins : asg.getInstances()) if (!(ins.getLifecycleState().equalsIgnoreCase("Terminating") @@ -89,7 +89,7 @@ public List getRacMembership() { StringUtils.join(asgNames, ","), StringUtils.join(instanceIds, ","))); } - return instanceIds; + return instanceIds.build(); } finally { if (client != null) client.shutdown(); } @@ -117,7 +117,7 @@ public int getRacMembershipSize() { } @Override - public List getCrossAccountRacMembership() { + public ImmutableSet getCrossAccountRacMembership() { AmazonAutoScaling client = null; try { List asgNames = new ArrayList<>(); @@ -130,7 +130,7 @@ public List getCrossAccountRacMembership() { asgNames.toArray(new String[asgNames.size()])); DescribeAutoScalingGroupsResult res = client.describeAutoScalingGroups(asgReq); - List instanceIds = Lists.newArrayList(); + ImmutableSet.Builder instanceIds = ImmutableSet.builder(); for (AutoScalingGroup asg : res.getAutoScalingGroups()) { for (Instance ins : asg.getInstances()) if (!(ins.getLifecycleState().equalsIgnoreCase("Terminating") @@ -144,7 +144,7 @@ public List getCrossAccountRacMembership() { "Querying Amazon returned following instance in the cross-account ASG: %s --> %s", instanceInfo.getRac(), StringUtils.join(instanceIds, ","))); } - return instanceIds; + return instanceIds.build(); } finally { if (client != null) client.shutdown(); } @@ -274,11 +274,11 @@ public void removeACL(Collection listIPs, int from, int to) { } /** List SG ACL's */ - public List listACL(int from, int to) { + public ImmutableSet listACL(int from, int to) { AmazonEC2 client = null; try { client = getEc2Client(); - List ipPermissions = new ArrayList<>(); + ImmutableSet.Builder ipPermissions = ImmutableSet.builder(); if (isClassic()) { @@ -316,7 +316,7 @@ public List listACL(int from, int to) { logger.debug("Fetch current permissions for vpc env of running instance"); } - return ipPermissions; + return ipPermissions.build(); } finally { if (client != null) client.shutdown(); } diff --git a/priam/src/main/java/com/netflix/priam/aws/SDBInstanceData.java b/priam/src/main/java/com/netflix/priam/aws/SDBInstanceData.java index 3c035c3d3..1d0fe8a08 100644 --- a/priam/src/main/java/com/netflix/priam/aws/SDBInstanceData.java +++ b/priam/src/main/java/com/netflix/priam/aws/SDBInstanceData.java @@ -74,7 +74,9 @@ public SDBInstanceData(ICredential provider, IConfiguration configuration) { */ public PriamInstance getInstance(String app, String dc, int id) { AmazonSimpleDB simpleDBClient = getSimpleDBClient(); - SelectRequest request = new SelectRequest(String.format(INSTANCE_QUERY, app, dc, id)); + SelectRequest request = + new SelectRequest(String.format(INSTANCE_QUERY, app, dc, id)) + .withConsistentRead(true); SelectResult result = simpleDBClient.select(request); if (result.getItems().size() == 0) return null; return transform(result.getItems().get(0)); @@ -91,8 +93,10 @@ public Set getAllIds(String app) { Set inslist = new HashSet<>(); String nextToken = null; do { - SelectRequest request = new SelectRequest(String.format(ALL_QUERY, app)); - request.setNextToken(nextToken); + SelectRequest request = + new SelectRequest(String.format(ALL_QUERY, app)) + .withConsistentRead(true) + .withNextToken(nextToken); SelectResult result = simpleDBClient.select(request); nextToken = result.getNextToken(); for (Item item : result.getItems()) { @@ -106,15 +110,23 @@ public Set getAllIds(String app) { /** * Create a new instance entry in SimpleDB * - * @param instance Instance entry to be created. + * @param orig Original instance used for validation + * @param inst Instance entry to be created. * @throws AmazonServiceException If unable to write to Simple DB because of any error. */ - public void createInstance(PriamInstance instance) throws AmazonServiceException { - AmazonSimpleDB simpleDBClient = getSimpleDBClient(); + public void updateInstance(PriamInstance orig, PriamInstance inst) + throws AmazonServiceException { PutAttributesRequest putReq = - new PutAttributesRequest( - DOMAIN, getKey(instance), createAttributesToRegister(instance)); - simpleDBClient.putAttributes(putReq); + new PutAttributesRequest(DOMAIN, getKey(inst), createAttributesToRegister(inst)) + .withExpected( + new UpdateCondition() + .withName(Attributes.INSTANCE_ID) + .withValue(orig.getInstanceId())) + .withExpected( + new UpdateCondition() + .withName(Attributes.TOKEN) + .withValue(orig.getToken())); + getSimpleDBClient().putAttributes(putReq); } /** diff --git a/priam/src/main/java/com/netflix/priam/aws/SDBInstanceFactory.java b/priam/src/main/java/com/netflix/priam/aws/SDBInstanceFactory.java index 53f05a281..df80c0bb5 100644 --- a/priam/src/main/java/com/netflix/priam/aws/SDBInstanceFactory.java +++ b/priam/src/main/java/com/netflix/priam/aws/SDBInstanceFactory.java @@ -17,13 +17,14 @@ package com.netflix.priam.aws; import com.amazonaws.AmazonServiceException; +import com.google.common.collect.ImmutableSet; import com.google.inject.Inject; import com.google.inject.Singleton; -import com.netflix.priam.config.IConfiguration; import com.netflix.priam.identity.IPriamInstanceFactory; import com.netflix.priam.identity.PriamInstance; import com.netflix.priam.identity.config.InstanceInfo; import java.util.*; +import java.util.stream.Collectors; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -31,27 +32,25 @@ * SimpleDB based instance instanceIdentity. Requires 'InstanceIdentity' domain to be created ahead */ @Singleton -public class SDBInstanceFactory implements IPriamInstanceFactory { +public class SDBInstanceFactory implements IPriamInstanceFactory { private static final Logger logger = LoggerFactory.getLogger(SDBInstanceFactory.class); - private final IConfiguration config; private final SDBInstanceData dao; private final InstanceInfo instanceInfo; @Inject - public SDBInstanceFactory( - IConfiguration config, SDBInstanceData dao, InstanceInfo instanceInfo) { - this.config = config; + public SDBInstanceFactory(SDBInstanceData dao, InstanceInfo instanceInfo) { this.dao = dao; this.instanceInfo = instanceInfo; } @Override - public List getAllIds(String appName) { - List return_ = new ArrayList<>(); - return_.addAll(dao.getAllIds(appName)); - sort(return_); - return return_; + public ImmutableSet getAllIds(String appName) { + return ImmutableSet.copyOf( + dao.getAllIds(appName) + .stream() + .sorted((Comparator.comparingInt(PriamInstance::getId))) + .collect(Collectors.toList())); } @Override @@ -104,31 +103,14 @@ public void delete(PriamInstance inst) { } @Override - public void update(PriamInstance inst) { + public void update(PriamInstance orig, PriamInstance inst) { try { - dao.createInstance(inst); + dao.updateInstance(orig, inst); } catch (AmazonServiceException e) { throw new RuntimeException("Unable to update/create priam instance", e); } } - @Override - public void sort(List return_) { - Comparator comparator = - (Comparator) - (o1, o2) -> { - Integer c1 = o1.getId(); - Integer c2 = o2.getId(); - return c1.compareTo(c2); - }; - return_.sort(comparator); - } - - @Override - public void attachVolumes(PriamInstance instance, String mountPath, String device) { - // TODO Auto-generated method stub - } - private PriamInstance makePriamInstance( String app, int id, diff --git a/priam/src/main/java/com/netflix/priam/aws/UpdateSecuritySettings.java b/priam/src/main/java/com/netflix/priam/aws/UpdateSecuritySettings.java index 6a0c4f51b..29f9dc653 100644 --- a/priam/src/main/java/com/netflix/priam/aws/UpdateSecuritySettings.java +++ b/priam/src/main/java/com/netflix/priam/aws/UpdateSecuritySettings.java @@ -16,21 +16,21 @@ */ package com.netflix.priam.aws; -import com.google.common.collect.Lists; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Sets; import com.google.inject.Inject; import com.google.inject.Singleton; import com.netflix.priam.config.IConfiguration; import com.netflix.priam.identity.IMembership; import com.netflix.priam.identity.IPriamInstanceFactory; import com.netflix.priam.identity.InstanceIdentity; -import com.netflix.priam.identity.PriamInstance; +import com.netflix.priam.identity.config.InstanceInfo; import com.netflix.priam.scheduler.SimpleTimer; import com.netflix.priam.scheduler.Task; import com.netflix.priam.scheduler.TaskTimer; -import java.util.HashSet; -import java.util.List; import java.util.Random; import java.util.Set; +import java.util.stream.Collectors; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -52,16 +52,21 @@ public class UpdateSecuritySettings extends Task { private static final Random ran = new Random(); private final IMembership membership; - private final IPriamInstanceFactory factory; + private final IPriamInstanceFactory factory; + private final InstanceInfo instanceInfo; @Inject // Note: do not parameterized the generic type variable to an implementation as it confuses // Guice in the binding. public UpdateSecuritySettings( - IConfiguration config, IMembership membership, IPriamInstanceFactory factory) { + IConfiguration config, + IMembership membership, + IPriamInstanceFactory factory, + InstanceInfo instanceInfo) { super(config); this.membership = membership; this.factory = factory; + this.instanceInfo = instanceInfo; } /** @@ -70,37 +75,27 @@ public UpdateSecuritySettings( */ @Override public void execute() { - // if seed dont execute. int port = config.getSSLStoragePort(); - List acls = membership.listACL(port, port); - List instances = factory.getAllIds(config.getAppName()); - - // iterate to add... - Set add = new HashSet<>(); - List allInstances = factory.getAllIds(config.getAppName()); - for (PriamInstance instance : allInstances) { - String range = instance.getHostIP() + "/32"; - if (!acls.contains(range)) add.add(range); - } - if (add.size() > 0) { - membership.addACL(add, port, port); + ImmutableSet currentAcl = membership.listACL(port, port); + Set desiredAcl = + factory.getAllIds(config.getAppName()) + .stream() + .map(i -> i.getHostIP() + "/32") + .collect(Collectors.toSet()); + // Make sure a hole is opened for my instance. + // This accommodates the eventually consistent CassandraInstanceFactory. + // Remove once IPs are all private as there won't be any chance of a discrepancy anymore. + String myIp = + config.usePrivateIP() ? instanceInfo.getPrivateIP() : instanceInfo.getHostIP(); + desiredAcl.add(myIp + "/32"); + Set aclToAdd = Sets.difference(desiredAcl, currentAcl); + if (!aclToAdd.isEmpty()) { + membership.addACL(aclToAdd, port, port); firstTimeUpdated = true; } - - // just iterate to generate ranges. - List currentRanges = Lists.newArrayList(); - for (PriamInstance instance : instances) { - String range = instance.getHostIP() + "/32"; - currentRanges.add(range); - } - - // iterate to remove... - List remove = Lists.newArrayList(); - for (String acl : acls) - if (!currentRanges.contains(acl)) // if not found then remove.... - remove.add(acl); - if (remove.size() > 0) { - membership.removeACL(remove, port, port); + Set aclToRemove = Sets.difference(currentAcl, desiredAcl); + if (!aclToRemove.isEmpty()) { + membership.removeACL(aclToRemove, port, port); firstTimeUpdated = true; } } diff --git a/priam/src/main/java/com/netflix/priam/cli/StaticMembership.java b/priam/src/main/java/com/netflix/priam/cli/StaticMembership.java index 3e6244d27..2f1626737 100644 --- a/priam/src/main/java/com/netflix/priam/cli/StaticMembership.java +++ b/priam/src/main/java/com/netflix/priam/cli/StaticMembership.java @@ -16,12 +16,11 @@ */ package com.netflix.priam.cli; +import com.google.common.collect.ImmutableSet; import com.netflix.priam.identity.IMembership; import java.io.FileInputStream; import java.io.IOException; -import java.util.Arrays; import java.util.Collection; -import java.util.List; import java.util.Properties; import org.apache.cassandra.io.util.FileUtils; import org.slf4j.Logger; @@ -36,7 +35,7 @@ public class StaticMembership implements IMembership { private static final Logger logger = LoggerFactory.getLogger(StaticMembership.class); - private List racMembership; + private ImmutableSet racMembership; private int racCount; public StaticMembership() throws IOException { @@ -57,18 +56,18 @@ public StaticMembership() throws IOException { if (name.startsWith(INSTANCES_PRE)) { racCount += 1; if (name.equals(INSTANCES_PRE + racName)) - racMembership = Arrays.asList(config.getProperty(name).split(",")); + racMembership = ImmutableSet.copyOf(config.getProperty(name).split(",")); } } } @Override - public List getRacMembership() { + public ImmutableSet getRacMembership() { return racMembership; } @Override - public List getCrossAccountRacMembership() { + public ImmutableSet getCrossAccountRacMembership() { return null; } @@ -90,7 +89,7 @@ public void addACL(Collection listIPs, int from, int to) {} public void removeACL(Collection listIPs, int from, int to) {} @Override - public List listACL(int from, int to) { + public ImmutableSet listACL(int from, int to) { return null; } diff --git a/priam/src/main/java/com/netflix/priam/config/IConfiguration.java b/priam/src/main/java/com/netflix/priam/config/IConfiguration.java index 76f574767..b45c09e51 100644 --- a/priam/src/main/java/com/netflix/priam/config/IConfiguration.java +++ b/priam/src/main/java/com/netflix/priam/config/IConfiguration.java @@ -1096,6 +1096,13 @@ default ImmutableSet getTunablePropertyFiles() { return ImmutableSet.of(); } + /** + * @return true to use private IPs for seeds and insertion into the Token DB false otherwise. + */ + default boolean usePrivateIP() { + return getSnitch().equals("org.apache.cassandra.locator.GossipingPropertyFileSnitch"); + } + /** * Escape hatch for getting any arbitrary property by key This is useful so we don't have to * keep adding methods to this interface for every single configuration option ever. Also diff --git a/priam/src/main/java/com/netflix/priam/identity/DoubleRing.java b/priam/src/main/java/com/netflix/priam/identity/DoubleRing.java index e0bb5898d..910b0f0d6 100644 --- a/priam/src/main/java/com/netflix/priam/identity/DoubleRing.java +++ b/priam/src/main/java/com/netflix/priam/identity/DoubleRing.java @@ -16,12 +16,13 @@ */ package com.netflix.priam.identity; -import com.google.common.collect.Lists; import com.google.inject.Inject; import com.netflix.priam.config.IConfiguration; +import com.netflix.priam.identity.config.InstanceInfo; import com.netflix.priam.utils.ITokenManager; import java.io.*; -import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -30,20 +31,20 @@ public class DoubleRing { private static final Logger logger = LoggerFactory.getLogger(DoubleRing.class); private static File TMP_BACKUP_FILE; private final IConfiguration config; - private final IPriamInstanceFactory factory; + private final IPriamInstanceFactory factory; private final ITokenManager tokenManager; - private final InstanceIdentity instanceIdentity; + private final InstanceInfo instanceInfo; @Inject public DoubleRing( IConfiguration config, IPriamInstanceFactory factory, ITokenManager tokenManager, - InstanceIdentity instanceIdentity) { + InstanceInfo instanceInfo) { this.config = config; this.factory = factory; this.tokenManager = tokenManager; - this.instanceIdentity = instanceIdentity; + this.instanceInfo = instanceInfo; } /** @@ -51,12 +52,12 @@ public DoubleRing( * nodes come up, they will get the unsed token assigned per token logic. */ public void doubleSlots() { - List local = filteredRemote(factory.getAllIds(config.getAppName())); + Set local = getInstancesInSameRegion(); // delete all for (PriamInstance data : local) factory.delete(data); - int hash = tokenManager.regionOffset(instanceIdentity.getInstanceInfo().getRegion()); + int hash = tokenManager.regionOffset(instanceInfo.getRegion()); // move existing slots. for (PriamInstance data : local) { int slot = (data.getId() - hash) * 2; @@ -72,7 +73,7 @@ public void doubleSlots() { } int new_ring_size = local.size() * 2; - for (PriamInstance data : filteredRemote(factory.getAllIds(config.getAppName()))) { + for (PriamInstance data : getInstancesInSameRegion()) { // if max then rotate. int currentSlot = data.getId() - hash; int new_slot = @@ -80,16 +81,13 @@ public void doubleSlots() { ? (currentSlot + 3) - new_ring_size : currentSlot + 3; String token = - tokenManager.createToken( - new_slot, - new_ring_size, - instanceIdentity.getInstanceInfo().getRegion()); + tokenManager.createToken(new_slot, new_ring_size, instanceInfo.getRegion()); factory.create( data.getApp(), new_slot + hash, InstanceIdentity.DUMMY_INSTANCE_ID, - instanceIdentity.getInstanceInfo().getHostname(), - instanceIdentity.getInstanceInfo().getHostIP(), + instanceInfo.getHostname(), + config.usePrivateIP() ? instanceInfo.getPrivateIP() : instanceInfo.getHostIP(), data.getRac(), null, token); @@ -97,12 +95,11 @@ public void doubleSlots() { } // filter other DC's - private List filteredRemote(List lst) { - List local = Lists.newArrayList(); - for (PriamInstance data : lst) - if (data.getDC().equals(instanceIdentity.getInstanceInfo().getRegion())) - local.add(data); - return local; + private Set getInstancesInSameRegion() { + return factory.getAllIds(config.getAppName()) + .stream() + .filter(i -> i.getDC().equals(instanceInfo.getRegion())) + .collect(Collectors.toSet()); } /** Backup the current state in case of failure */ @@ -111,7 +108,7 @@ public void backup() throws IOException { TMP_BACKUP_FILE = File.createTempFile("Backup-instance-data", ".dat"); try (ObjectOutputStream stream = new ObjectOutputStream(new FileOutputStream(TMP_BACKUP_FILE))) { - stream.writeObject(filteredRemote(factory.getAllIds(config.getAppName()))); + stream.writeObject(getInstancesInSameRegion()); logger.info( "Wrote the backup of the instances to: {}", TMP_BACKUP_FILE.getAbsolutePath()); } @@ -124,14 +121,13 @@ public void backup() throws IOException { * @throws ClassNotFoundException */ public void restore() throws IOException, ClassNotFoundException { - for (PriamInstance data : filteredRemote(factory.getAllIds(config.getAppName()))) - factory.delete(data); + for (PriamInstance data : getInstancesInSameRegion()) factory.delete(data); // read from the file. try (ObjectInputStream stream = new ObjectInputStream(new FileInputStream(TMP_BACKUP_FILE))) { @SuppressWarnings("unchecked") - List allInstances = (List) stream.readObject(); + Set allInstances = (Set) stream.readObject(); for (PriamInstance data : allInstances) factory.create( data.getApp(), diff --git a/priam/src/main/java/com/netflix/priam/identity/IMembership.java b/priam/src/main/java/com/netflix/priam/identity/IMembership.java index 03e30ed6a..ed6f72b18 100644 --- a/priam/src/main/java/com/netflix/priam/identity/IMembership.java +++ b/priam/src/main/java/com/netflix/priam/identity/IMembership.java @@ -16,10 +16,10 @@ */ package com.netflix.priam.identity; +import com.google.common.collect.ImmutableSet; import com.google.inject.ImplementedBy; import com.netflix.priam.aws.AWSMembership; import java.util.Collection; -import java.util.List; /** * Interface to manage membership meta information such as size of RAC, list of nodes in RAC etc. @@ -32,17 +32,17 @@ public interface IMembership { * * @return */ - List getRacMembership(); + ImmutableSet getRacMembership(); /** @return Size of current RAC */ int getRacMembershipSize(); /** - * Get a list of Instances in the cross-account but current RAC + * Get a set of Instances in the cross-account but current RAC * * @return */ - List getCrossAccountRacMembership(); + ImmutableSet getCrossAccountRacMembership(); /** * Number of RACs @@ -74,7 +74,7 @@ public interface IMembership { * * @return */ - List listACL(int from, int to); + ImmutableSet listACL(int from, int to); /** * Expand the membership size by 1. diff --git a/priam/src/main/java/com/netflix/priam/identity/IPriamInstanceFactory.java b/priam/src/main/java/com/netflix/priam/identity/IPriamInstanceFactory.java index 79f51f78e..af1e37d80 100644 --- a/priam/src/main/java/com/netflix/priam/identity/IPriamInstanceFactory.java +++ b/priam/src/main/java/com/netflix/priam/identity/IPriamInstanceFactory.java @@ -16,9 +16,9 @@ */ package com.netflix.priam.identity; +import com.google.common.collect.ImmutableSet; import com.google.inject.ImplementedBy; import com.netflix.priam.aws.SDBInstanceFactory; -import java.util.List; import java.util.Map; /** @@ -26,14 +26,14 @@ * delete or list instances from the registry */ @ImplementedBy(SDBInstanceFactory.class) -public interface IPriamInstanceFactory { +public interface IPriamInstanceFactory { /** * Return a list of all Cassandra server nodes registered. * * @param appName the cluster name * @return a list of all nodes in {@code appName} */ - List getAllIds(String appName); + ImmutableSet getAllIds(String appName); /** * Return the Cassandra server node with the given {@code id}. @@ -77,23 +77,8 @@ PriamInstance create( /** * Update the details of the server node in registry * - * @param inst the node to update + * @param orig the values that should exist in the database for the update to succeed + * @param inst the new values */ - void update(PriamInstance inst); - - /** - * Sort the list by instance ID - * - * @param return_ the list of nodes to sort - */ - void sort(List return_); - - /** - * Attach volumes if required - * - * @param instance - * @param mountPath - * @param device - */ - void attachVolumes(PriamInstance instance, String mountPath, String device); + void update(PriamInstance orig, PriamInstance inst); } diff --git a/priam/src/main/java/com/netflix/priam/identity/InstanceIdentity.java b/priam/src/main/java/com/netflix/priam/identity/InstanceIdentity.java index 03ab09cb0..458420e91 100644 --- a/priam/src/main/java/com/netflix/priam/identity/InstanceIdentity.java +++ b/priam/src/main/java/com/netflix/priam/identity/InstanceIdentity.java @@ -16,7 +16,6 @@ */ package com.netflix.priam.identity; -import com.google.common.base.Preconditions; import com.google.common.base.Predicate; import com.google.common.collect.Iterables; import com.google.common.collect.ListMultimap; @@ -26,22 +25,10 @@ import com.google.inject.Singleton; import com.netflix.priam.config.IConfiguration; import com.netflix.priam.identity.config.InstanceInfo; -import com.netflix.priam.identity.token.IDeadTokenRetriever; -import com.netflix.priam.identity.token.INewTokenRetriever; -import com.netflix.priam.identity.token.IPreGeneratedTokenRetriever; -import com.netflix.priam.identity.token.TokenRetrieverUtils; -import com.netflix.priam.identity.token.TokenRetrieverUtils.GossipParseException; -import com.netflix.priam.utils.ITokenManager; -import com.netflix.priam.utils.RetryableCallable; -import com.netflix.priam.utils.Sleeper; -import java.net.UnknownHostException; -import java.util.Collections; +import com.netflix.priam.identity.token.ITokenRetriever; import java.util.HashMap; import java.util.LinkedList; import java.util.List; -import java.util.Optional; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; /** * This class provides the central place to create and consume the identity of the instance - token, @@ -49,16 +36,13 @@ */ @Singleton public class InstanceIdentity { - private static final Logger logger = LoggerFactory.getLogger(InstanceIdentity.class); public static final String DUMMY_INSTANCE_ID = "new_slot"; private final ListMultimap locMap = Multimaps.newListMultimap(new HashMap<>(), Lists::newArrayList); - private final IPriamInstanceFactory factory; + private final IPriamInstanceFactory factory; private final IMembership membership; private final IConfiguration config; - private final Sleeper sleeper; - private final ITokenManager tokenManager; private final Predicate differentHostPredicate = new Predicate() { @@ -88,15 +72,9 @@ public boolean apply(PriamInstance instance) { private PriamInstance myInstance; // Instance information contains other information like ASG/vpc-id etc. private InstanceInfo myInstanceInfo; - private boolean isReplace = false; - private boolean isTokenPregenerated = false; - private String replacedIp = ""; - private final IDeadTokenRetriever deadTokenRetriever; - private final IPreGeneratedTokenRetriever preGeneratedTokenRetriever; - private final INewTokenRetriever newTokenRetriever; - - private final java.util.function.Predicate sameHostPredicate = - (i) -> i.getInstanceId().equals(myInstanceInfo.getInstanceId()); + private boolean isReplace; + private boolean isTokenPregenerated; + private String replacedIp; @Inject // Note: do not parameterized the generic type variable to an implementation as @@ -106,23 +84,17 @@ public InstanceIdentity( IPriamInstanceFactory factory, IMembership membership, IConfiguration config, - Sleeper sleeper, - ITokenManager tokenManager, - IDeadTokenRetriever deadTokenRetriever, - IPreGeneratedTokenRetriever preGeneratedTokenRetriever, - INewTokenRetriever newTokenRetriever, - InstanceInfo instanceInfo) + InstanceInfo instanceInfo, + ITokenRetriever tokenRetriever) throws Exception { this.factory = factory; this.membership = membership; this.config = config; - this.sleeper = sleeper; - this.tokenManager = tokenManager; - this.deadTokenRetriever = deadTokenRetriever; - this.preGeneratedTokenRetriever = preGeneratedTokenRetriever; - this.newTokenRetriever = newTokenRetriever; this.myInstanceInfo = instanceInfo; - init(); + this.myInstance = tokenRetriever.get(); + this.replacedIp = tokenRetriever.getReplacedIp().orElse(null); + this.isReplace = replacedIp != null; + this.isTokenPregenerated = tokenRetriever.isTokenPregenerated(); } public PriamInstance getInstance() { @@ -133,194 +105,12 @@ public InstanceInfo getInstanceInfo() { return myInstanceInfo; } - public void init() throws Exception { - // Grab the token which was preassigned. - logger.info("trying to grab preassigned token."); - myInstance = grabPreAssignedToken(); - - // Grab a dead token. - if (myInstance == null) { - logger.info("unable to grab preassigned token. trying to grab a dead token."); - myInstance = grabDeadToken(); - } - - // Grab a pre-generated token if there is such one. - if (myInstance == null) { - logger.info("unable to grab a dead token. trying to grab a pregenerated token."); - myInstance = grabPreGeneratedToken(); - } - - // Grab a new token - if (myInstance == null) { - logger.info("unable to grab a pregenerated token. trying to grab a new token."); - myInstance = grabNewToken(); - } - - logger.info("My token: {}", myInstance.getToken()); - } - - private PriamInstance grabPreAssignedToken() throws Exception { - return new RetryableCallable() { - @Override - public PriamInstance retriableCall() throws Exception { - // Check if this node is decommissioned. - List deadInstances = - factory.getAllIds(config.getAppName() + "-dead"); - PriamInstance instance = - findInstance(deadInstances, sameHostPredicate).orElse(null); - if (instance != null) { - instance.setOutOfService(true); - } - - if (instance == null) { - List aliveInstances = factory.getAllIds(config.getAppName()); - instance = findInstance(aliveInstances, sameHostPredicate).orElse(null); - - if (instance != null) { - instance.setOutOfService(false); - - // Priam might have crashed before bootstrapping Cassandra in replace mode. - // So, it is premature to use the assigned token without checking Cassandra - // gossip. - - // Infer current ownership information from other instances using gossip. - TokenRetrieverUtils.InferredTokenOwnership inferredTokenOwnership = - TokenRetrieverUtils.inferTokenOwnerFromGossip( - aliveInstances, instance.getToken(), instance.getDC()); - // if unreachable rely on token database. - // if mismatch rely on token database. - if (inferredTokenOwnership.getTokenInformationStatus() - == TokenRetrieverUtils.InferredTokenOwnership.TokenInformationStatus - .GOOD) { - Preconditions.checkNotNull( - inferredTokenOwnership.getTokenInformation()); - String inferredIp = - inferredTokenOwnership.getTokenInformation().getIpAddress(); - if (!inferredIp.equalsIgnoreCase(instance.getHostIP())) { - if (inferredTokenOwnership.getTokenInformation().isLive()) { - throw new GossipParseException( - "We have been assigned a token that C* thinks is alive. Throwing to buy time in the hopes that Gossip just needs to settle."); - } - setReplacedIp(inferredIp); - logger.info( - "Priam found that the token is not alive according to Cassandra and we should start Cassandra in replace mode with replace ip: " - + inferredIp); - } - } - } - } - - if (instance != null) { - logger.info( - "{} found that this node is {}." - + " application: {}," - + " id: {}," - + " instance: {}," - + " region: {}," - + " host ip: {}," - + " host name: {}," - + " token: {}", - instance.isOutOfService() ? "[Dead]" : "[Alive]", - instance.isOutOfService() ? "dead" : "alive", - instance.getApp(), - instance.getId(), - instance.getInstanceId(), - instance.getDC(), - instance.getHostIP(), - instance.getHostName(), - instance.getToken()); - } - - return instance; - } - }.call(); - } - - private PriamInstance grabDeadToken() throws Exception { - return new RetryableCallable() { - @Override - public PriamInstance retriableCall() throws Exception { - PriamInstance result = deadTokenRetriever.get(); - if (result != null) { - isReplace = true; // indicate that we are acquiring a dead instance's token. - - if (deadTokenRetriever.getReplaceIp() - != null) { // The IP address of the dead instance to which - // we will acquire its token - replacedIp = deadTokenRetriever.getReplaceIp(); - } - } - - return result; - } - - @Override - public void forEachExecution() { - populateRacMap(); - deadTokenRetriever.setLocMap(locMap); - } - }.call(); - } - - private PriamInstance grabPreGeneratedToken() throws Exception { - return new RetryableCallable() { - @Override - public PriamInstance retriableCall() throws Exception { - PriamInstance result = preGeneratedTokenRetriever.get(); - if (result != null) { - isTokenPregenerated = true; - } - return result; - } - - @Override - public void forEachExecution() { - populateRacMap(); - preGeneratedTokenRetriever.setLocMap(locMap); - } - }.call(); - } - - private PriamInstance grabNewToken() throws Exception { - if (!this.config.isCreateNewTokenEnable()) { - throw new IllegalStateException( - "Node attempted to erroneously create a new token when we should be grabbing an existing token."); - } - - return new RetryableCallable() { - @Override - public PriamInstance retriableCall() throws Exception { - set(100, 100); - newTokenRetriever.setLocMap(locMap); - return newTokenRetriever.get(); - } - - @Override - public void forEachExecution() { - populateRacMap(); - newTokenRetriever.setLocMap(locMap); - } - }.call(); - } - - private Optional findInstance( - List instances, java.util.function.Predicate predicate) { - return Optional.ofNullable(instances) - .orElse(Collections.emptyList()) - .stream() - .filter(predicate) - .findFirst(); - } - private void populateRacMap() { locMap.clear(); - List instances = factory.getAllIds(config.getAppName()); - for (PriamInstance ins : instances) { - locMap.put(ins.getRac(), ins); - } + factory.getAllIds(config.getAppName()).forEach(ins -> locMap.put(ins.getRac(), ins)); } - public List getSeeds() throws UnknownHostException { + public List getSeeds() { populateRacMap(); List seeds = new LinkedList<>(); // Handle single zone deployment diff --git a/priam/src/main/java/com/netflix/priam/identity/PriamInstance.java b/priam/src/main/java/com/netflix/priam/identity/PriamInstance.java index 19d1e50ea..5c5b7eb54 100644 --- a/priam/src/main/java/com/netflix/priam/identity/PriamInstance.java +++ b/priam/src/main/java/com/netflix/priam/identity/PriamInstance.java @@ -110,8 +110,8 @@ public void setVolumes(Map volumes) { @Override public String toString() { return String.format( - "Hostname: %s, InstanceId: %s, APP_NAME: %s, RAC : %s Location %s, Id: %s: Token: %s", - getHostName(), getInstanceId(), getApp(), getRac(), getDC(), getId(), getToken()); + "Hostname: %s, InstanceId: %s, APP_NAME: %s, RAC : %s, Location %s, Id: %s: Token: %s, IP: %s", + hostname, instanceId, app, availabilityZone, location, Id, token, publicip); } public String getDC() { @@ -134,7 +134,8 @@ public boolean isOutOfService() { return outOfService; } - public void setOutOfService(boolean outOfService) { - this.outOfService = outOfService; + public PriamInstance setOutOfService() { + this.outOfService = true; + return this; } } diff --git a/priam/src/main/java/com/netflix/priam/identity/token/DeadTokenRetriever.java b/priam/src/main/java/com/netflix/priam/identity/token/DeadTokenRetriever.java deleted file mode 100755 index 918d214f2..000000000 --- a/priam/src/main/java/com/netflix/priam/identity/token/DeadTokenRetriever.java +++ /dev/null @@ -1,208 +0,0 @@ -/** - * Copyright 2017 Netflix, Inc. - * - *

    Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file - * except in compliance with the License. You may obtain a copy of the License at - * - *

    http://www.apache.org/licenses/LICENSE-2.0 - * - *

    Unless required by applicable law or agreed to in writing, software distributed under the - * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.netflix.priam.identity.token; - -import com.google.common.collect.ListMultimap; -import com.google.inject.Inject; -import com.netflix.priam.config.IConfiguration; -import com.netflix.priam.identity.IMembership; -import com.netflix.priam.identity.IPriamInstanceFactory; -import com.netflix.priam.identity.PriamInstance; -import com.netflix.priam.identity.config.InstanceInfo; -import com.netflix.priam.utils.Sleeper; -import java.util.List; -import java.util.Random; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class DeadTokenRetriever extends TokenRetrieverBase implements IDeadTokenRetriever { - private static final Logger logger = LoggerFactory.getLogger(DeadTokenRetriever.class); - private final IPriamInstanceFactory factory; - private final IMembership membership; - private final IConfiguration config; - private final Sleeper sleeper; - // The IP address of the dead instance to which we will acquire its token - private String replacedIp; - private ListMultimap locMap; - private final InstanceInfo instanceInfo; - - @Inject - public DeadTokenRetriever( - IPriamInstanceFactory factory, - IMembership membership, - IConfiguration config, - Sleeper sleeper, - InstanceInfo instanceInfo) { - this.factory = factory; - this.membership = membership; - this.config = config; - this.sleeper = sleeper; - this.instanceInfo = instanceInfo; - } - - private List getDualAccountRacMembership(List asgInstances) { - logger.info("Dual Account cluster"); - - List crossAccountAsgInstances = membership.getCrossAccountRacMembership(); - - // Remove duplicates (probably there are not) - asgInstances.removeAll(crossAccountAsgInstances); - - // Merge the two lists - asgInstances.addAll(crossAccountAsgInstances); - logger.info("Combined Instances in the AZ: {}", asgInstances); - - return asgInstances; - } - - @Override - public PriamInstance get() throws Exception { - - logger.info("Looking for a token from any dead node"); - final List allInstancesWithinCluster = - factory.getAllIds(config.getAppName()); - List asgInstances = membership.getRacMembership(); - if (config.isDualAccount()) { - asgInstances = getDualAccountRacMembership(asgInstances); - } else { - logger.info("Single Account cluster"); - } - - // Sleep random interval - upto 15 sec - sleeper.sleep(new Random().nextInt(5000) + 10000); - logger.info( - "About to iterate through all instances within cluster " - + config.getAppName() - + " to find an available token to acquire."); - - for (PriamInstance priamInstance : allInstancesWithinCluster) { - // test same zone and is it alive. - if (!priamInstance.getRac().equals(instanceInfo.getRac()) - || asgInstances.contains(priamInstance.getInstanceId()) - || super.isInstanceDummy(priamInstance)) continue; - // TODO: If instance is in SHUTTING_DOWN mode, it might not show up in asg - // instances (if cloud control plane is having issues), thus, we should not try - // to replace the instance as it will lead to "Cannot replace a live node" - // issue. - - logger.info("Found dead instance: {}", priamInstance.toString()); - - PriamInstance markAsDead = - factory.create( - priamInstance.getApp() + "-dead", - priamInstance.getId(), - priamInstance.getInstanceId(), - priamInstance.getHostName(), - priamInstance.getHostIP(), - priamInstance.getRac(), - priamInstance.getVolumes(), - priamInstance.getToken()); - // remove it as we marked it down... - factory.delete(priamInstance); - - // find the replaced IP - - // Infer current ownership information from other instances using gossip. - TokenRetrieverUtils.InferredTokenOwnership inferredTokenInformation = - TokenRetrieverUtils.inferTokenOwnerFromGossip( - allInstancesWithinCluster, - priamInstance.getToken(), - priamInstance.getDC()); - - switch (inferredTokenInformation.getTokenInformationStatus()) { - case GOOD: - if (inferredTokenInformation.getTokenInformation() == null) { - logger.error( - "If you see this message, it should not have happened. We expect token ownership information if all nodes agree. This is a code bounty issue."); - return null; - } - // Everyone agreed to a value. Check if it is live node. - if (inferredTokenInformation.getTokenInformation().isLive()) { - logger.info( - "This token is considered alive unanimously! We will not replace this instance."); - return null; - } else - this.replacedIp = - inferredTokenInformation.getTokenInformation().getIpAddress(); - break; - case UNREACHABLE_NODES: - // In case of unable to reach sufficient nodes, fallback to IP in token - // database. This could be a genuine case of say missing security permissions. - this.replacedIp = priamInstance.getHostIP(); - logger.warn( - "Unable to reach sufficient nodes. Please check security group permissions or there might be a network partition."); - logger.info( - "Will try to replace token: {} with replacedIp from Token database: {}", - priamInstance.getToken(), - priamInstance.getHostIP()); - break; - case MISMATCH: - // Lets not replace the instance if gossip info is not merging!! - logger.info( - "Mismatch in gossip. We will not replace this instance, until gossip settles down."); - return null; - default: - throw new IllegalStateException( - "Unexpected value: " - + inferredTokenInformation.getTokenInformationStatus()); - } - - PriamInstance result; - try { - result = - factory.create( - config.getAppName(), - markAsDead.getId(), - instanceInfo.getInstanceId(), - instanceInfo.getHostname(), - instanceInfo.getHostIP(), - instanceInfo.getRac(), - markAsDead.getVolumes(), - markAsDead.getToken()); - } catch (Exception ex) { - long sleepTime = super.getSleepTime(); - logger.warn( - "Exception when acquiring dead token: " - + priamInstance.getToken() - + " , will sleep for " - + sleepTime - + " millisecs before we retry."); - Thread.sleep(sleepTime); - - throw ex; - } - - logger.info( - "Acquired token: " - + priamInstance.getToken() - + " and we will replace with replacedIp: " - + replacedIp); - - return result; - } - - logger.info("This node was NOT able to acquire any dead token"); - return null; - } - - @Override - public String getReplaceIp() { - return this.replacedIp; - } - - @Override - public void setLocMap(ListMultimap locMap) { - this.locMap = locMap; - } -} diff --git a/priam/src/main/java/com/netflix/priam/identity/token/IDeadTokenRetriever.java b/priam/src/main/java/com/netflix/priam/identity/token/IDeadTokenRetriever.java deleted file mode 100755 index d14b82d72..000000000 --- a/priam/src/main/java/com/netflix/priam/identity/token/IDeadTokenRetriever.java +++ /dev/null @@ -1,34 +0,0 @@ -/** - * Copyright 2017 Netflix, Inc. - * - *

    Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file - * except in compliance with the License. You may obtain a copy of the License at - * - *

    http://www.apache.org/licenses/LICENSE-2.0 - * - *

    Unless required by applicable law or agreed to in writing, software distributed under the - * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.netflix.priam.identity.token; - -import com.google.common.collect.ListMultimap; -import com.google.inject.ImplementedBy; -import com.netflix.priam.identity.PriamInstance; - -@ImplementedBy(DeadTokenRetriever.class) -public interface IDeadTokenRetriever { - - PriamInstance get() throws Exception; - - /* - * @return the IP address of the dead instance to which we will acquire its token - */ - String getReplaceIp(); - - /* - * @param A map of the rac for each instance. - */ - void setLocMap(ListMultimap locMap); -} diff --git a/priam/src/main/java/com/netflix/priam/identity/token/INewTokenRetriever.java b/priam/src/main/java/com/netflix/priam/identity/token/INewTokenRetriever.java deleted file mode 100755 index ccf320008..000000000 --- a/priam/src/main/java/com/netflix/priam/identity/token/INewTokenRetriever.java +++ /dev/null @@ -1,29 +0,0 @@ -/** - * Copyright 2017 Netflix, Inc. - * - *

    Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file - * except in compliance with the License. You may obtain a copy of the License at - * - *

    http://www.apache.org/licenses/LICENSE-2.0 - * - *

    Unless required by applicable law or agreed to in writing, software distributed under the - * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.netflix.priam.identity.token; - -import com.google.common.collect.ListMultimap; -import com.google.inject.ImplementedBy; -import com.netflix.priam.identity.PriamInstance; - -@ImplementedBy(NewTokenRetriever.class) -public interface INewTokenRetriever { - - PriamInstance get() throws Exception; - - /* - * @param A map of the rac for each instance. - */ - void setLocMap(ListMultimap locMap); -} diff --git a/priam/src/main/java/com/netflix/priam/identity/token/IPreGeneratedTokenRetriever.java b/priam/src/main/java/com/netflix/priam/identity/token/IPreGeneratedTokenRetriever.java deleted file mode 100755 index 74aed283f..000000000 --- a/priam/src/main/java/com/netflix/priam/identity/token/IPreGeneratedTokenRetriever.java +++ /dev/null @@ -1,29 +0,0 @@ -/** - * Copyright 2017 Netflix, Inc. - * - *

    Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file - * except in compliance with the License. You may obtain a copy of the License at - * - *

    http://www.apache.org/licenses/LICENSE-2.0 - * - *

    Unless required by applicable law or agreed to in writing, software distributed under the - * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.netflix.priam.identity.token; - -import com.google.common.collect.ListMultimap; -import com.google.inject.ImplementedBy; -import com.netflix.priam.identity.PriamInstance; - -@ImplementedBy(PreGeneratedTokenRetriever.class) -public interface IPreGeneratedTokenRetriever { - - PriamInstance get() throws Exception; - - /* - * @param A map of the rac for each instance. - */ - void setLocMap(ListMultimap locMap); -} diff --git a/priam/src/main/java/com/netflix/priam/identity/token/ITokenRetriever.java b/priam/src/main/java/com/netflix/priam/identity/token/ITokenRetriever.java new file mode 100644 index 000000000..38125bee9 --- /dev/null +++ b/priam/src/main/java/com/netflix/priam/identity/token/ITokenRetriever.java @@ -0,0 +1,15 @@ +package com.netflix.priam.identity.token; + +import com.google.inject.ImplementedBy; +import com.netflix.priam.identity.PriamInstance; +import java.util.Optional; + +/** Fetches PriamInstances and other data which is convenient at the time */ +@ImplementedBy(TokenRetriever.class) +public interface ITokenRetriever { + PriamInstance get() throws Exception; + /** Gets the IP address of the dead instance to which we will acquire its token */ + Optional getReplacedIp(); + + boolean isTokenPregenerated(); +} diff --git a/priam/src/main/java/com/netflix/priam/identity/token/NewTokenRetriever.java b/priam/src/main/java/com/netflix/priam/identity/token/NewTokenRetriever.java deleted file mode 100755 index e0e32127c..000000000 --- a/priam/src/main/java/com/netflix/priam/identity/token/NewTokenRetriever.java +++ /dev/null @@ -1,119 +0,0 @@ -/** - * Copyright 2017 Netflix, Inc. - * - *

    Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file - * except in compliance with the License. You may obtain a copy of the License at - * - *

    http://www.apache.org/licenses/LICENSE-2.0 - * - *

    Unless required by applicable law or agreed to in writing, software distributed under the - * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.netflix.priam.identity.token; - -import com.google.common.collect.ListMultimap; -import com.google.inject.Inject; -import com.netflix.priam.config.IConfiguration; -import com.netflix.priam.identity.IMembership; -import com.netflix.priam.identity.IPriamInstanceFactory; -import com.netflix.priam.identity.PriamInstance; -import com.netflix.priam.identity.config.InstanceInfo; -import com.netflix.priam.utils.ITokenManager; -import com.netflix.priam.utils.Sleeper; -import java.util.List; -import java.util.Random; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class NewTokenRetriever extends TokenRetrieverBase implements INewTokenRetriever { - - private static final Logger logger = LoggerFactory.getLogger(NewTokenRetriever.class); - private final IPriamInstanceFactory factory; - private final IMembership membership; - private final IConfiguration config; - private final Sleeper sleeper; - private final ITokenManager tokenManager; - private ListMultimap locMap; - private InstanceInfo instanceInfo; - - @Inject - // Note: do not parameterized the generic type variable to an implementation as it confuses - // Guice in the binding. - public NewTokenRetriever( - IPriamInstanceFactory factory, - IMembership membership, - IConfiguration config, - Sleeper sleeper, - ITokenManager tokenManager, - InstanceInfo instanceInfo) { - this.factory = factory; - this.membership = membership; - this.config = config; - this.sleeper = sleeper; - this.tokenManager = tokenManager; - this.instanceInfo = instanceInfo; - } - - @Override - public PriamInstance get() throws Exception { - - logger.info("Generating my own and new token"); - // Sleep random interval - upto 15 sec - sleeper.sleep(new Random().nextInt(15000)); - int hash = tokenManager.regionOffset(instanceInfo.getRegion()); - // use this hash so that the nodes are spread far away from the other - // regions. - - int max = hash; - List allInstances = factory.getAllIds(config.getAppName()); - for (PriamInstance data : allInstances) - max = - (data.getRac().equals(instanceInfo.getRac()) && (data.getId() > max)) - ? data.getId() - : max; - int maxSlot = max - hash; - int my_slot; - - if (hash == max && locMap.get(instanceInfo.getRac()).size() == 0) { - int idx = config.getRacs().indexOf(instanceInfo.getRac()); - if (idx < 0) - throw new Exception( - String.format( - "Rac %s is not in Racs %s", - instanceInfo.getRac(), config.getRacs())); - my_slot = idx + maxSlot; - } else my_slot = config.getRacs().size() + maxSlot; - - logger.info( - "Trying to createToken with slot {} with rac count {} with rac membership size {} with dc {}", - my_slot, - membership.getRacCount(), - membership.getRacMembershipSize(), - instanceInfo.getRegion()); - String payload = - tokenManager.createToken( - my_slot, - membership.getRacCount(), - membership.getRacMembershipSize(), - instanceInfo.getRegion()); - return factory.create( - config.getAppName(), - my_slot + hash, - instanceInfo.getInstanceId(), - instanceInfo.getHostname(), - instanceInfo.getHostIP(), - instanceInfo.getRac(), - null, - payload); - } - - /* - * @param A map of the rac for each instance. - */ - @Override - public void setLocMap(ListMultimap locMap) { - this.locMap = locMap; - } -} diff --git a/priam/src/main/java/com/netflix/priam/identity/token/PreGeneratedTokenRetriever.java b/priam/src/main/java/com/netflix/priam/identity/token/PreGeneratedTokenRetriever.java deleted file mode 100755 index dda193459..000000000 --- a/priam/src/main/java/com/netflix/priam/identity/token/PreGeneratedTokenRetriever.java +++ /dev/null @@ -1,103 +0,0 @@ -/** - * Copyright 2017 Netflix, Inc. - * - *

    Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file - * except in compliance with the License. You may obtain a copy of the License at - * - *

    http://www.apache.org/licenses/LICENSE-2.0 - * - *

    Unless required by applicable law or agreed to in writing, software distributed under the - * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.netflix.priam.identity.token; - -import com.google.common.collect.ListMultimap; -import com.google.inject.Inject; -import com.netflix.priam.config.IConfiguration; -import com.netflix.priam.identity.IMembership; -import com.netflix.priam.identity.IPriamInstanceFactory; -import com.netflix.priam.identity.PriamInstance; -import com.netflix.priam.identity.config.InstanceInfo; -import com.netflix.priam.utils.Sleeper; -import java.util.List; -import java.util.Random; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class PreGeneratedTokenRetriever extends TokenRetrieverBase - implements IPreGeneratedTokenRetriever { - - private static final Logger logger = LoggerFactory.getLogger(PreGeneratedTokenRetriever.class); - private final IPriamInstanceFactory factory; - private final IMembership membership; - private final IConfiguration config; - private final Sleeper sleeper; - private ListMultimap locMap; - private InstanceInfo instanceInfo; - - @Inject - public PreGeneratedTokenRetriever( - IPriamInstanceFactory factory, - IMembership membership, - IConfiguration config, - Sleeper sleeper, - InstanceInfo instanceInfo) { - this.factory = factory; - this.membership = membership; - this.config = config; - this.sleeper = sleeper; - this.instanceInfo = instanceInfo; - } - - @Override - public PriamInstance get() throws Exception { - logger.info("Looking for any pre-generated token"); - - final List allIds = factory.getAllIds(config.getAppName()); - List asgInstances = membership.getRacMembership(); - // Sleep random interval - upto 15 sec - sleeper.sleep(new Random().nextInt(5000) + 10000); - for (PriamInstance dead : allIds) { - // test same zone and is it is alive. - if (!dead.getRac().equals(instanceInfo.getRac()) - || asgInstances.contains(dead.getInstanceId()) - || !isInstanceDummy(dead)) continue; - logger.info("Found pre-generated token: {}", dead.getToken()); - PriamInstance markAsDead = - factory.create( - dead.getApp() + "-dead", - dead.getId(), - dead.getInstanceId(), - dead.getHostName(), - dead.getHostIP(), - dead.getRac(), - dead.getVolumes(), - dead.getToken()); - // remove it as we marked it down... - factory.delete(dead); - - String payLoad = markAsDead.getToken(); - logger.info( - "Trying to grab slot {} with availability zone {}", - markAsDead.getId(), - markAsDead.getRac()); - return factory.create( - config.getAppName(), - markAsDead.getId(), - instanceInfo.getInstanceId(), - instanceInfo.getHostname(), - instanceInfo.getHostIP(), - instanceInfo.getRac(), - markAsDead.getVolumes(), - payLoad); - } - return null; - } - - @Override - public void setLocMap(ListMultimap locMap) { - this.locMap = locMap; - } -} diff --git a/priam/src/main/java/com/netflix/priam/identity/token/TokenRetriever.java b/priam/src/main/java/com/netflix/priam/identity/token/TokenRetriever.java new file mode 100644 index 000000000..a2280f004 --- /dev/null +++ b/priam/src/main/java/com/netflix/priam/identity/token/TokenRetriever.java @@ -0,0 +1,313 @@ +package com.netflix.priam.identity.token; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Sets; +import com.netflix.priam.config.IConfiguration; +import com.netflix.priam.identity.IMembership; +import com.netflix.priam.identity.IPriamInstanceFactory; +import com.netflix.priam.identity.InstanceIdentity; +import com.netflix.priam.identity.PriamInstance; +import com.netflix.priam.identity.config.InstanceInfo; +import com.netflix.priam.utils.ITokenManager; +import com.netflix.priam.utils.RetryableCallable; +import com.netflix.priam.utils.Sleeper; +import java.util.List; +import java.util.Optional; +import java.util.Random; +import java.util.Set; +import java.util.stream.Collectors; +import javax.inject.Inject; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class TokenRetriever implements ITokenRetriever { + + public static final String NEW_SLOT = "new_slot"; + private static final int MAX_VALUE_IN_MILISECS = 300000; // sleep up to 5 minutes + private static final Logger logger = LoggerFactory.getLogger(InstanceIdentity.class); + + private final Random randomizer; + private final Sleeper sleeper; + private final IPriamInstanceFactory factory; + private final IMembership membership; + private final IConfiguration config; + private final ITokenManager tokenManager; + + // Instance information contains other information like ASG/vpc-id etc. + private InstanceInfo myInstanceInfo; + private boolean isTokenPregenerated = false; + private String replacedIp; + + @Inject + public TokenRetriever( + IPriamInstanceFactory factory, + IMembership membership, + IConfiguration config, + InstanceInfo instanceInfo, + Sleeper sleeper, + ITokenManager tokenManager) { + this.factory = factory; + this.membership = membership; + this.config = config; + this.myInstanceInfo = instanceInfo; + this.randomizer = new Random(); + this.sleeper = sleeper; + this.tokenManager = tokenManager; + } + + @Override + public PriamInstance get() throws Exception { + PriamInstance myInstance = grabPreAssignedToken(); + if (myInstance == null) { + myInstance = grabExistingToken(); + } + if (myInstance == null) { + myInstance = grabNewToken(); + } + logger.info("My instance: {}", myInstance); + return myInstance; + } + + @Override + public Optional getReplacedIp() { + return Optional.ofNullable(replacedIp); + } + + @Override + public boolean isTokenPregenerated() { + return isTokenPregenerated; + } + + private PriamInstance grabPreAssignedToken() throws Exception { + return new RetryableCallable() { + @Override + public PriamInstance retriableCall() throws Exception { + logger.info("Trying to grab a pre-assigned token."); + // Check if this node is decommissioned. + ImmutableSet allIds = + factory.getAllIds(config.getAppName() + "-dead"); + Optional instance = + findInstance(allIds).map(PriamInstance::setOutOfService); + if (!instance.isPresent()) { + ImmutableSet liveNodes = factory.getAllIds(config.getAppName()); + instance = instance.map(Optional::of).orElseGet(() -> findInstance(liveNodes)); + if (instance.isPresent()) { + // Why check gossip? Priam might have crashed before bootstrapping + // Cassandra in replace mode. + replacedIp = getReplacedIpForAssignedToken(liveNodes, instance.get()); + } + } + return instance.map(i -> claimToken(i)).orElse(null); + } + }.call(); + } + + @VisibleForTesting + public PriamInstance grabExistingToken() throws Exception { + return new RetryableCallable() { + @Override + public PriamInstance retriableCall() throws Exception { + logger.info("Trying to grab an existing token"); + sleeper.sleep(new Random().nextInt(5000) + 10000); + Set racInstanceIds = getRacInstanceIds(); + ImmutableSet allIds = factory.getAllIds(config.getAppName()); + List instances = + allIds.stream() + .filter(i -> i.getRac().equals(myInstanceInfo.getRac())) + .filter(i -> !racInstanceIds.contains(i.getInstanceId())) + .collect(Collectors.toList()); + Optional candidate = + instances.stream().filter(i -> !isNew(i)).findFirst(); + candidate.ifPresent(i -> replacedIp = getReplacedIpForExistingToken(allIds, i)); + if (replacedIp == null) { + candidate = instances.stream().filter(i -> isNew(i)).findFirst(); + candidate.ifPresent(i -> isTokenPregenerated = true); + } + return candidate.map(i -> claimToken(i)).orElse(null); + } + }.call(); + } + + private PriamInstance grabNewToken() throws Exception { + Preconditions.checkState(config.isCreateNewTokenEnable()); + return new RetryableCallable() { + @Override + public PriamInstance retriableCall() throws Exception { + set(100, 100); + logger.info("Trying to generate a new token"); + sleeper.sleep(new Random().nextInt(15000)); + String myRegion = myInstanceInfo.getRegion(); + // this offset ensures the nodes are spread far away from the other regions. + int regionOffset = tokenManager.regionOffset(myRegion); + String myRac = myInstanceInfo.getRac(); + List racs = config.getRacs(); + int mySlot = + factory.getAllIds(config.getAppName()) + .stream() + .filter(i -> i.getRac().equals(myRac)) + .map(PriamInstance::getId) + .max(Integer::compareTo) + .map(id -> racs.size() + Math.max(id, regionOffset) - regionOffset) + .orElseGet( + () -> { + Preconditions.checkState(racs.contains(myRac)); + return racs.indexOf(myRac); + }); + int instanceCount = membership.getRacCount() * membership.getRacMembershipSize(); + String newToken = tokenManager.createToken(mySlot, instanceCount, myRegion); + return createToken(mySlot + regionOffset, newToken); + } + }.call(); + } + + private String getReplacedIpForAssignedToken( + ImmutableSet aliveInstances, PriamInstance instance) + throws TokenRetrieverUtils.GossipParseException { + // Infer current ownership information from other instances using gossip. + TokenRetrieverUtils.InferredTokenOwnership inferredTokenOwnership = + TokenRetrieverUtils.inferTokenOwnerFromGossip( + aliveInstances, instance.getToken(), instance.getDC()); + // if unreachable rely on token database. + // if mismatch rely on token database. + String ipToReplace = null; + if (inferredTokenOwnership.getTokenInformationStatus() + == TokenRetrieverUtils.InferredTokenOwnership.TokenInformationStatus.GOOD) { + Preconditions.checkNotNull(inferredTokenOwnership.getTokenInformation()); + String inferredIp = inferredTokenOwnership.getTokenInformation().getIpAddress(); + if (!inferredIp.equals(myInstanceInfo.getHostIP()) + && !inferredIp.equals(myInstanceInfo.getPrivateIP())) { + if (inferredTokenOwnership.getTokenInformation().isLive()) { + throw new TokenRetrieverUtils.GossipParseException( + "We have been assigned a token that C* thinks is alive. Throwing to buy time in the hopes that Gossip just needs to settle."); + } + ipToReplace = inferredIp; + logger.info( + "Priam found that the token is not alive according to Cassandra and we should start Cassandra in replace mode with replace ip: " + + inferredIp); + } + } + return ipToReplace; + } + + private String getReplacedIpForExistingToken( + ImmutableSet allInstancesWithinCluster, PriamInstance priamInstance) { + + // Infer current ownership information from other instances using gossip. + TokenRetrieverUtils.InferredTokenOwnership inferredTokenInformation = + TokenRetrieverUtils.inferTokenOwnerFromGossip( + allInstancesWithinCluster, priamInstance.getToken(), priamInstance.getDC()); + + switch (inferredTokenInformation.getTokenInformationStatus()) { + case GOOD: + if (inferredTokenInformation.getTokenInformation() == null) { + logger.error( + "If you see this message, it should not have happened. We expect token ownership information if all nodes agree. This is a code bounty issue."); + return null; + } + // Everyone agreed to a value. Check if it is live node. + if (inferredTokenInformation.getTokenInformation().isLive()) { + logger.info( + "This token is considered alive unanimously! We will not replace this instance."); + return null; + } else { + String ip = inferredTokenInformation.getTokenInformation().getIpAddress(); + logger.info("Will try to replace token owned by {}", ip); + return ip; + } + case UNREACHABLE_NODES: + // In case of unable to reach sufficient nodes, fallback to IP in token + // database. This could be a genuine case of say missing security + // permissions. + logger.warn( + "Unable to reach sufficient nodes. Please check security group permissions or there might be a network partition."); + logger.info( + "Will try to replace token: {} with replacedIp from Token database: {}", + priamInstance.getToken(), + priamInstance.getHostIP()); + return priamInstance.getHostIP(); + case MISMATCH: + // Lets not replace the instance if gossip info is not merging!! + logger.info( + "Mismatch in gossip. We will not replace this instance, until gossip settles down."); + return null; + default: + throw new IllegalStateException( + "Unexpected value: " + + inferredTokenInformation.getTokenInformationStatus()); + } + } + + private PriamInstance claimToken(PriamInstance originalInstance) { + String hostIP = + config.usePrivateIP() ? myInstanceInfo.getPrivateIP() : myInstanceInfo.getHostIP(); + if (originalInstance.getInstanceId().equals(myInstanceInfo.getInstanceId()) + && originalInstance.getHostName().equals(myInstanceInfo.getHostname()) + && originalInstance.getHostIP().equals(hostIP) + && originalInstance.getRac().equals(myInstanceInfo.getRac())) { + return originalInstance; + } + PriamInstance newInstance = new PriamInstance(); + newInstance.setApp(config.getAppName()); + newInstance.setId(originalInstance.getId()); + newInstance.setInstanceId(myInstanceInfo.getInstanceId()); + newInstance.setHost(myInstanceInfo.getHostname()); + newInstance.setHostIP(hostIP); + newInstance.setRac(myInstanceInfo.getRac()); + newInstance.setVolumes(originalInstance.getVolumes()); + newInstance.setToken(originalInstance.getToken()); + newInstance.setDC(originalInstance.getDC()); + try { + factory.update(originalInstance, newInstance); + } catch (Exception ex) { + long sleepTime = randomizer.nextInt(MAX_VALUE_IN_MILISECS); + String token = newInstance.getToken(); + logger.warn("Failed updating token: {}; sleeping {} millis", token, sleepTime); + sleeper.sleepQuietly(sleepTime); + throw ex; + } + return newInstance; + } + + private PriamInstance createToken(int id, String token) { + try { + String hostIp = + config.usePrivateIP() + ? myInstanceInfo.getPrivateIP() + : myInstanceInfo.getHostIP(); + return factory.create( + config.getAppName(), + id, + myInstanceInfo.getInstanceId(), + myInstanceInfo.getHostname(), + hostIp, + myInstanceInfo.getRac(), + null /* volumes */, + token); + } catch (Exception ex) { + long sleepTime = randomizer.nextInt(MAX_VALUE_IN_MILISECS); + logger.warn("Failed updating token: {}; sleeping {} millis", token, sleepTime); + sleeper.sleepQuietly(sleepTime); + throw ex; + } + } + + private Optional findInstance(ImmutableSet instances) { + return instances + .stream() + .filter((i) -> i.getInstanceId().equals(myInstanceInfo.getInstanceId())) + .findFirst(); + } + + private Set getRacInstanceIds() { // TODO(CASS-1986) + ImmutableSet racMembership = membership.getRacMembership(); + return config.isDualAccount() + ? Sets.union(membership.getCrossAccountRacMembership(), racMembership) + : racMembership; + } + + private boolean isNew(PriamInstance instance) { + return instance.getInstanceId().equals(NEW_SLOT); + } +} diff --git a/priam/src/main/java/com/netflix/priam/identity/token/TokenRetrieverUtils.java b/priam/src/main/java/com/netflix/priam/identity/token/TokenRetrieverUtils.java index 1fd35b7ec..b7b6ee07e 100644 --- a/priam/src/main/java/com/netflix/priam/identity/token/TokenRetrieverUtils.java +++ b/priam/src/main/java/com/netflix/priam/identity/token/TokenRetrieverUtils.java @@ -1,5 +1,6 @@ package com.netflix.priam.identity.token; +import com.google.common.collect.ImmutableSet; import com.netflix.priam.identity.PriamInstance; import com.netflix.priam.utils.GsonJsonSerializer; import com.netflix.priam.utils.SystemUtils; @@ -32,7 +33,7 @@ public class TokenRetrieverUtils { * converge. */ public static InferredTokenOwnership inferTokenOwnerFromGossip( - List allIds, String token, String dc) { + ImmutableSet allIds, String token, String dc) { // Avoid using dead instance who we are trying to replace (duh!!) // Avoid other regions instances to avoid communication over public ip address. diff --git a/priam/src/main/java/com/netflix/priam/resources/PriamInstanceResource.java b/priam/src/main/java/com/netflix/priam/resources/PriamInstanceResource.java index 883f06fba..7c8eefc23 100644 --- a/priam/src/main/java/com/netflix/priam/resources/PriamInstanceResource.java +++ b/priam/src/main/java/com/netflix/priam/resources/PriamInstanceResource.java @@ -22,7 +22,7 @@ import com.netflix.priam.identity.PriamInstance; import com.netflix.priam.identity.config.InstanceInfo; import java.net.URI; -import java.util.List; +import java.util.stream.Collectors; import javax.ws.rs.*; import javax.ws.rs.core.MediaType; import javax.ws.rs.core.Response; @@ -37,7 +37,7 @@ public class PriamInstanceResource { private static final Logger log = LoggerFactory.getLogger(PriamInstanceResource.class); private final IConfiguration config; - private final IPriamInstanceFactory factory; + private final IPriamInstanceFactory factory; private final InstanceInfo instanceInfo; @Inject @@ -57,13 +57,10 @@ public PriamInstanceResource( */ @GET public String getInstances() { - StringBuilder response = new StringBuilder(); - List allInstances = factory.getAllIds(config.getAppName()); - for (PriamInstance node : allInstances) { - response.append(node.toString()); - response.append("\n"); - } - return response.toString(); + return factory.getAllIds(config.getAppName()) + .stream() + .map(PriamInstance::toString) + .collect(Collectors.joining("\n", "", "\n")); } /** diff --git a/priam/src/main/java/com/netflix/priam/utils/ITokenManager.java b/priam/src/main/java/com/netflix/priam/utils/ITokenManager.java index 7ac64288a..b3a669587 100644 --- a/priam/src/main/java/com/netflix/priam/utils/ITokenManager.java +++ b/priam/src/main/java/com/netflix/priam/utils/ITokenManager.java @@ -22,8 +22,6 @@ @ImplementedBy(TokenManager.class) public interface ITokenManager { - String createToken(int mySlot, int racCount, int racSize, String region); - String createToken(int mySlot, int totalCount, String region); BigInteger findClosestToken(BigInteger tokenToSearch, List tokenList); diff --git a/priam/src/main/java/com/netflix/priam/utils/TokenManager.java b/priam/src/main/java/com/netflix/priam/utils/TokenManager.java index 1d26077f2..d9276a4da 100644 --- a/priam/src/main/java/com/netflix/priam/utils/TokenManager.java +++ b/priam/src/main/java/com/netflix/priam/utils/TokenManager.java @@ -77,20 +77,6 @@ BigInteger initialToken(int size, int position, int offset) { .add(minimumToken); } - /** - * Creates a token given the following parameter - * - * @param my_slot -- Slot where this instance has to be. - * @param rac_count -- Rac count is the number of RAC's - * @param rac_size -- number of memberships in the rac - * @param region -- name of the DC where it this token is created. - */ - @Override - public String createToken(int my_slot, int rac_count, int rac_size, String region) { - int regionCount = rac_count * rac_size; - return initialToken(regionCount, my_slot, regionOffset(region)).toString(); - } - @Override public String createToken(int my_slot, int totalCount, String region) { return initialToken(totalCount, my_slot, regionOffset(region)).toString(); diff --git a/priam/src/test/java/com/netflix/priam/TestModule.java b/priam/src/test/java/com/netflix/priam/TestModule.java index 15529c61b..0f4ceffbe 100644 --- a/priam/src/test/java/com/netflix/priam/TestModule.java +++ b/priam/src/test/java/com/netflix/priam/TestModule.java @@ -52,7 +52,7 @@ protected void configure() { bind(InstanceInfo.class) .toInstance(new FakeInstanceInfo("fakeInstance1", "az1", "us-east-1")); - bind(IPriamInstanceFactory.class).to(FakePriamInstanceFactory.class); + bind(IPriamInstanceFactory.class).to(FakePriamInstanceFactory.class).in(Scopes.SINGLETON); bind(SchedulerFactory.class).to(StdSchedulerFactory.class).in(Scopes.SINGLETON); bind(IMembership.class) .toInstance( diff --git a/priam/src/test/java/com/netflix/priam/aws/TestUpdateSecuritySettings.java b/priam/src/test/java/com/netflix/priam/aws/TestUpdateSecuritySettings.java new file mode 100644 index 000000000..8f53e9c3c --- /dev/null +++ b/priam/src/test/java/com/netflix/priam/aws/TestUpdateSecuritySettings.java @@ -0,0 +1,111 @@ +package com.netflix.priam.aws; + +import com.google.common.collect.ImmutableSet; +import com.google.common.truth.Truth; +import com.google.inject.Guice; +import com.google.inject.Injector; +import com.netflix.priam.TestModule; +import com.netflix.priam.config.FakeConfiguration; +import com.netflix.priam.config.IConfiguration; +import com.netflix.priam.identity.IMembership; +import com.netflix.priam.identity.IPriamInstanceFactory; +import com.netflix.priam.identity.config.InstanceInfo; +import org.junit.Before; +import org.junit.Test; + +/* Tests of {@link UpdateSecuritySettings.java} */ +public class TestUpdateSecuritySettings { + private static final int PORT = 7103; + + private UpdateSecuritySettings updateSecuritySettings; + private IMembership membership; + private IPriamInstanceFactory factory; + private FakeConfiguration config; + private InstanceInfo instanceInfo; + + @Before + public void setUp() { + Injector injector = Guice.createInjector(new TestModule()); + factory = injector.getInstance(IPriamInstanceFactory.class); + membership = injector.getInstance(IMembership.class); + updateSecuritySettings = injector.getInstance(UpdateSecuritySettings.class); + config = (FakeConfiguration) injector.getInstance(IConfiguration.class); + instanceInfo = injector.getInstance(InstanceInfo.class); + } + + @Test + public void add_membershipEmpty() { // edge-case, not expected + addToFactory(1, "1.1.1.1"); + Truth.assertThat(membership.listACL(PORT, PORT)).doesNotContain("1.1.1.1/32"); + updateSecuritySettings.execute(); + Truth.assertThat(membership.listACL(PORT, PORT)).contains("1.1.1.1/32"); + } + + @Test + public void add() { + addToFactory(1, "1.1.1.1"); + membership.addACL(ImmutableSet.of("2.2.2.2/32"), PORT, PORT); + Truth.assertThat(membership.listACL(PORT, PORT)).doesNotContain("1.1.1.1/32"); + updateSecuritySettings.execute(); + Truth.assertThat(membership.listACL(PORT, PORT)).contains("1.1.1.1/32"); + } + + @Test + public void delete() { + addToFactory(1, "1.1.1.1"); + membership.addACL(ImmutableSet.of("2.2.2.2/32"), PORT, PORT); + Truth.assertThat(membership.listACL(PORT, PORT)).contains("2.2.2.2/32"); + updateSecuritySettings.execute(); + Truth.assertThat(membership.listACL(PORT, PORT)).doesNotContain("2.2.2.2/32"); + } + + @Test + public void delete_factoryEmpty() { // edge-case, not expected + membership.addACL(ImmutableSet.of("2.2.2.2/32"), PORT, PORT); + Truth.assertThat(membership.listACL(PORT, PORT)).contains("2.2.2.2/32"); + updateSecuritySettings.execute(); + Truth.assertThat(membership.listACL(PORT, PORT)).doesNotContain("2.2.2.2/32"); + } + + @Test + public void addMyPrivateIP() { + config.usePrivateIP(true); + String ingressRule = instanceInfo.getPrivateIP() + "/32"; + Truth.assertThat(membership.listACL(PORT, PORT)).doesNotContain(ingressRule); + updateSecuritySettings.execute(); + Truth.assertThat(membership.listACL(PORT, PORT)).contains(ingressRule); + } + + @Test + public void addMyPublicIP() { + config.usePrivateIP(false); + String ingressRule = instanceInfo.getHostIP() + "/32"; + Truth.assertThat(membership.listACL(PORT, PORT)).doesNotContain(ingressRule); + updateSecuritySettings.execute(); + Truth.assertThat(membership.listACL(PORT, PORT)).contains(ingressRule); + } + + @Test + public void keepMyPrivateIP() { + config.usePrivateIP(true); + String ingressRule = instanceInfo.getPrivateIP() + "/32"; + membership.addACL(ImmutableSet.of(ingressRule), PORT, PORT); + Truth.assertThat(membership.listACL(PORT, PORT)).contains(ingressRule); + updateSecuritySettings.execute(); + Truth.assertThat(membership.listACL(PORT, PORT)).contains(ingressRule); + } + + @Test + public void keepMyPublicIP() { + config.usePrivateIP(false); + String ingressRule = instanceInfo.getHostIP() + "/32"; + membership.addACL(ImmutableSet.of(ingressRule), PORT, PORT); + Truth.assertThat(membership.listACL(PORT, PORT)).contains(ingressRule); + updateSecuritySettings.execute(); + Truth.assertThat(membership.listACL(PORT, PORT)).contains(ingressRule); + } + + private void addToFactory(int id, String ip) { + factory.create("myApp", id, "i-1", "hostname", ip, "us-east-1a", null /* volumes */, "123"); + } +} diff --git a/priam/src/test/java/com/netflix/priam/backup/identity/DoubleRingTest.java b/priam/src/test/java/com/netflix/priam/backup/identity/DoubleRingTest.java index f340a03dc..74593f3c6 100644 --- a/priam/src/test/java/com/netflix/priam/backup/identity/DoubleRingTest.java +++ b/priam/src/test/java/com/netflix/priam/backup/identity/DoubleRingTest.java @@ -19,6 +19,7 @@ import static org.junit.Assert.assertEquals; +import com.google.common.collect.ImmutableSet; import com.google.common.collect.Lists; import com.netflix.priam.identity.DoubleRing; import com.netflix.priam.identity.InstanceIdentity; @@ -32,12 +33,10 @@ public class DoubleRingTest extends InstanceTestUtils { public void testDouble() throws Exception { createInstances(); int originalSize = factory.getAllIds(config.getAppName()).size(); - new DoubleRing(config, factory, tokenManager, identity).doubleSlots(); - List doubled = factory.getAllIds(config.getAppName()); - factory.sort(doubled); - + new DoubleRing(config, factory, tokenManager, instanceInfo).doubleSlots(); + ImmutableSet doubled = factory.getAllIds(config.getAppName()); assertEquals(originalSize * 2, doubled.size()); - validate(doubled); + validate(doubled.asList()); } private void validate(List doubled) { @@ -59,7 +58,7 @@ private void validate(List doubled) { public void testBR() throws Exception { createInstances(); int intialSize = factory.getAllIds(config.getAppName()).size(); - DoubleRing ring = new DoubleRing(config, factory, tokenManager, identity); + DoubleRing ring = new DoubleRing(config, factory, tokenManager, instanceInfo); ring.backup(); ring.doubleSlots(); assertEquals(intialSize * 2, factory.getAllIds(config.getAppName()).size()); diff --git a/priam/src/test/java/com/netflix/priam/backup/identity/InstanceIdentityTest.java b/priam/src/test/java/com/netflix/priam/backup/identity/InstanceIdentityTest.java index 4a72ce972..0b2ad7f20 100644 --- a/priam/src/test/java/com/netflix/priam/backup/identity/InstanceIdentityTest.java +++ b/priam/src/test/java/com/netflix/priam/backup/identity/InstanceIdentityTest.java @@ -17,14 +17,12 @@ package com.netflix.priam.backup.identity; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; +import static org.junit.Assert.*; +import com.google.common.collect.ImmutableList; import com.netflix.priam.identity.DoubleRing; import com.netflix.priam.identity.InstanceIdentity; import com.netflix.priam.identity.PriamInstance; -import java.util.List; import org.junit.Test; public class InstanceIdentityTest extends InstanceTestUtils { @@ -95,10 +93,8 @@ public void testGetSeedsAutobootstrapFalse() throws Exception { public void testDoubleSlots() throws Exception { createInstances(); int before = factory.getAllIds(config.getAppName()).size(); - new DoubleRing(config, factory, tokenManager, identity).doubleSlots(); - List lst = factory.getAllIds(config.getAppName()); - // sort it so it will look good if you want to print it. - factory.sort(lst); + new DoubleRing(config, factory, tokenManager, instanceInfo).doubleSlots(); + ImmutableList lst = factory.getAllIds(config.getAppName()).asList(); for (int i = 0; i < lst.size(); i++) { System.out.println(lst.get(i)); if (0 == i % 2) continue; @@ -110,7 +106,7 @@ public void testDoubleSlots() throws Exception { @Test public void testDoubleGrap() throws Exception { createInstances(); - new DoubleRing(config, factory, tokenManager, identity).doubleSlots(); + new DoubleRing(config, factory, tokenManager, instanceInfo).doubleSlots(); int hash = tokenManager.regionOffset(instanceInfo.getRegion()); identity = createInstanceIdentity("az1", "fakeinstancex"); printInstance(identity.getInstance(), hash); diff --git a/priam/src/test/java/com/netflix/priam/backup/identity/InstanceTestUtils.java b/priam/src/test/java/com/netflix/priam/backup/identity/InstanceTestUtils.java index 67d9433a5..1dca57e38 100644 --- a/priam/src/test/java/com/netflix/priam/backup/identity/InstanceTestUtils.java +++ b/priam/src/test/java/com/netflix/priam/backup/identity/InstanceTestUtils.java @@ -21,7 +21,8 @@ import com.netflix.priam.identity.*; import com.netflix.priam.identity.config.FakeInstanceInfo; import com.netflix.priam.identity.config.InstanceInfo; -import com.netflix.priam.identity.token.*; +import com.netflix.priam.identity.token.ITokenRetriever; +import com.netflix.priam.identity.token.TokenRetriever; import com.netflix.priam.utils.FakeSleeper; import com.netflix.priam.utils.ITokenManager; import com.netflix.priam.utils.Sleeper; @@ -81,23 +82,9 @@ public void createInstances() throws Exception { InstanceIdentity createInstanceIdentity(String zone, String instanceId) throws Exception { InstanceInfo newInstanceInfo = new FakeInstanceInfo(instanceId, zone, region); - IDeadTokenRetriever deadTokenRetriever = - new DeadTokenRetriever(factory, membership, config, sleeper, newInstanceInfo); - IPreGeneratedTokenRetriever preGeneratedTokenRetriever = - new PreGeneratedTokenRetriever( - factory, membership, config, sleeper, newInstanceInfo); - INewTokenRetriever newTokenRetriever = - new NewTokenRetriever( - factory, membership, config, sleeper, tokenManager, newInstanceInfo); - return new InstanceIdentity( - factory, - membership, - config, - sleeper, - this.tokenManager, - deadTokenRetriever, - preGeneratedTokenRetriever, - newTokenRetriever, - newInstanceInfo); + ITokenRetriever tokenRetriever = + new TokenRetriever( + factory, membership, config, newInstanceInfo, sleeper, tokenManager); + return new InstanceIdentity(factory, membership, config, newInstanceInfo, tokenRetriever); } } diff --git a/priam/src/test/java/com/netflix/priam/backup/identity/token/FakeDeadTokenRetriever.java b/priam/src/test/java/com/netflix/priam/backup/identity/token/FakeDeadTokenRetriever.java deleted file mode 100755 index ff0d9f041..000000000 --- a/priam/src/test/java/com/netflix/priam/backup/identity/token/FakeDeadTokenRetriever.java +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Copyright 2017 Netflix, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.netflix.priam.backup.identity.token; - -import com.google.common.collect.ListMultimap; -import com.netflix.priam.identity.PriamInstance; -import com.netflix.priam.identity.token.IDeadTokenRetriever; - -class FakeDeadTokenRetriever implements IDeadTokenRetriever { - - @Override - public PriamInstance get() throws Exception { - // TODO Auto-generated method stub - return null; - } - - @Override - public String getReplaceIp() { - // TODO Auto-generated method stub - return null; - } - - @Override - public void setLocMap(ListMultimap locMap) { - // TODO Auto-generated method stub - - } -} diff --git a/priam/src/test/java/com/netflix/priam/backup/identity/token/FakeNewTokenRetriever.java b/priam/src/test/java/com/netflix/priam/backup/identity/token/FakeNewTokenRetriever.java deleted file mode 100755 index 14a357e51..000000000 --- a/priam/src/test/java/com/netflix/priam/backup/identity/token/FakeNewTokenRetriever.java +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright 2017 Netflix, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.netflix.priam.backup.identity.token; - -import com.google.common.collect.ListMultimap; -import com.netflix.priam.identity.PriamInstance; -import com.netflix.priam.identity.token.INewTokenRetriever; - -class FakeNewTokenRetriever implements INewTokenRetriever { - - @Override - public PriamInstance get() throws Exception { - // TODO Auto-generated method stub - return null; - } - - @Override - public void setLocMap(ListMultimap locMap) { - // TODO Auto-generated method stub - - } -} diff --git a/priam/src/test/java/com/netflix/priam/backup/identity/token/FakePreGeneratedTokenRetriever.java b/priam/src/test/java/com/netflix/priam/backup/identity/token/FakePreGeneratedTokenRetriever.java deleted file mode 100755 index b4f827daa..000000000 --- a/priam/src/test/java/com/netflix/priam/backup/identity/token/FakePreGeneratedTokenRetriever.java +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright 2017 Netflix, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.netflix.priam.backup.identity.token; - -import com.google.common.collect.ListMultimap; -import com.netflix.priam.identity.PriamInstance; -import com.netflix.priam.identity.token.IPreGeneratedTokenRetriever; - -class FakePreGeneratedTokenRetriever implements IPreGeneratedTokenRetriever { - - @Override - public PriamInstance get() throws Exception { - // TODO Auto-generated method stub - return null; - } - - @Override - public void setLocMap(ListMultimap locMap) { - // TODO Auto-generated method stub - - } -} diff --git a/priam/src/test/java/com/netflix/priam/config/FakeConfiguration.java b/priam/src/test/java/com/netflix/priam/config/FakeConfiguration.java index 31a5f86e3..ebd3bd318 100644 --- a/priam/src/test/java/com/netflix/priam/config/FakeConfiguration.java +++ b/priam/src/test/java/com/netflix/priam/config/FakeConfiguration.java @@ -17,10 +17,10 @@ package com.netflix.priam.config; +import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; import com.google.inject.Singleton; import java.io.File; -import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -32,6 +32,9 @@ public class FakeConfiguration implements IConfiguration { private String restorePrefix = ""; public Map fakeConfig; private String roleManager = ""; + private boolean mayCreateNewToken; + private ImmutableList racs; + private boolean usePrivateIp; public Map fakeProperties = new HashMap<>(); @@ -43,6 +46,8 @@ public FakeConfiguration(String appName) { this.appName = appName; fakeConfig = new HashMap<>(); fakeConfig.put("auto_bootstrap", false); + this.mayCreateNewToken = true; // matches interface default + this.racs = ImmutableList.of("az1", "az2", "az3"); } public Object getFakeConfig(String key) { @@ -83,7 +88,11 @@ public String getCassandraBaseDirectory() { @Override public List getRacs() { - return Arrays.asList("az1", "az2", "az3"); + return racs; + } + + public void setRacs(String... racs) { + this.racs = ImmutableList.copyOf(racs); } @Override @@ -202,4 +211,22 @@ public String getRAC() { public String getDC() { return "us-east-1"; } + + @Override + public boolean isCreateNewTokenEnable() { + return mayCreateNewToken; + } + + public void setCreateNewToken(boolean mayCreateNewToken) { + this.mayCreateNewToken = mayCreateNewToken; + } + + @Override + public boolean usePrivateIP() { + return usePrivateIp; + } + + public void usePrivateIP(boolean usePrivateIp) { + this.usePrivateIp = usePrivateIp; + } } diff --git a/priam/src/test/java/com/netflix/priam/identity/FakeMembership.java b/priam/src/test/java/com/netflix/priam/identity/FakeMembership.java index e3185c914..b320442d4 100644 --- a/priam/src/test/java/com/netflix/priam/identity/FakeMembership.java +++ b/priam/src/test/java/com/netflix/priam/identity/FakeMembership.java @@ -17,28 +17,29 @@ package com.netflix.priam.identity; +import com.google.common.collect.ImmutableSet; import java.util.Collection; +import java.util.HashSet; import java.util.List; +import java.util.Set; public class FakeMembership implements IMembership { - private List instances; + private ImmutableSet instances; + private Set acl; public FakeMembership(List priamInstances) { - this.instances = priamInstances; - } - - public void setInstances(List priamInstances) { - this.instances = priamInstances; + this.instances = ImmutableSet.copyOf(priamInstances); + this.acl = new HashSet<>(); } @Override - public List getRacMembership() { + public ImmutableSet getRacMembership() { return instances; } @Override - public List getCrossAccountRacMembership() { + public ImmutableSet getCrossAccountRacMembership() { return null; } @@ -54,20 +55,17 @@ public int getRacCount() { @Override public void addACL(Collection listIPs, int from, int to) { - // TODO Auto-generated method stub - + acl.addAll(listIPs); } @Override public void removeACL(Collection listIPs, int from, int to) { - // TODO Auto-generated method stub - + acl.removeAll(listIPs); } @Override - public List listACL(int from, int to) { - // TODO Auto-generated method stub - return null; + public ImmutableSet listACL(int from, int to) { + return ImmutableSet.copyOf(acl); } @Override diff --git a/priam/src/test/java/com/netflix/priam/identity/FakePriamInstanceFactory.java b/priam/src/test/java/com/netflix/priam/identity/FakePriamInstanceFactory.java index bedaae527..45b826b78 100644 --- a/priam/src/test/java/com/netflix/priam/identity/FakePriamInstanceFactory.java +++ b/priam/src/test/java/com/netflix/priam/identity/FakePriamInstanceFactory.java @@ -17,12 +17,17 @@ package com.netflix.priam.identity; +import com.google.common.collect.ImmutableSet; import com.google.common.collect.Maps; import com.google.inject.Inject; import com.netflix.priam.identity.config.InstanceInfo; -import java.util.*; +import groovy.lang.Singleton; +import java.util.Comparator; +import java.util.Map; +import java.util.stream.Collectors; -public class FakePriamInstanceFactory implements IPriamInstanceFactory { +@Singleton +public class FakePriamInstanceFactory implements IPriamInstanceFactory { private final Map instances = Maps.newHashMap(); private final InstanceInfo instanceInfo; @@ -32,10 +37,15 @@ public FakePriamInstanceFactory(InstanceInfo instanceInfo) { } @Override - public List getAllIds(String appName) { - List result = new ArrayList<>(instances.values()); - sort(result); - return result; + public ImmutableSet getAllIds(String appName) { + return appName.endsWith("-dead") + ? ImmutableSet.of() + : ImmutableSet.copyOf( + instances + .values() + .stream() + .sorted(Comparator.comparingInt(PriamInstance::getId)) + .collect(Collectors.toList())); } @Override @@ -72,24 +82,7 @@ public void delete(PriamInstance inst) { } @Override - public void update(PriamInstance inst) { + public void update(PriamInstance orig, PriamInstance inst) { instances.put(inst.getId(), inst); } - - @Override - public void sort(List return_) { - Comparator comparator = - (Comparator) - (o1, o2) -> { - Integer c1 = o1.getId(); - Integer c2 = o2.getId(); - return c1.compareTo(c2); - }; - return_.sort(comparator); - } - - @Override - public void attachVolumes(PriamInstance instance, String mountPath, String device) { - // TODO Auto-generated method stub - } } diff --git a/priam/src/test/java/com/netflix/priam/identity/config/FakeInstanceInfo.java b/priam/src/test/java/com/netflix/priam/identity/config/FakeInstanceInfo.java index e9f064821..54b269dcf 100644 --- a/priam/src/test/java/com/netflix/priam/identity/config/FakeInstanceInfo.java +++ b/priam/src/test/java/com/netflix/priam/identity/config/FakeInstanceInfo.java @@ -57,12 +57,12 @@ public String getHostname() { @Override public String getHostIP() { - return instanceId; + return "127.0.0.0"; } @Override public String getPrivateIP() { - return instanceId; + return "127.1.1.0"; } @Override diff --git a/priam/src/test/java/com/netflix/priam/identity/token/AssignedTokenRetrieverTest.java b/priam/src/test/java/com/netflix/priam/identity/token/AssignedTokenRetrieverTest.java index f61b584a8..e77f88d6d 100644 --- a/priam/src/test/java/com/netflix/priam/identity/token/AssignedTokenRetrieverTest.java +++ b/priam/src/test/java/com/netflix/priam/identity/token/AssignedTokenRetrieverTest.java @@ -1,6 +1,7 @@ package com.netflix.priam.identity.token; import com.google.common.base.Strings; +import com.google.common.collect.ImmutableSet; import com.google.common.truth.Truth; import com.netflix.priam.config.IConfiguration; import com.netflix.priam.identity.IMembership; @@ -26,7 +27,7 @@ public class AssignedTokenRetrieverTest { @Test public void grabAssignedTokenStartDbInBootstrapModeWhenGossipAgreesCurrentInstanceIsTokenOwner( - @Mocked IPriamInstanceFactory factory, + @Mocked IPriamInstanceFactory factory, @Mocked IConfiguration config, @Mocked IMembership membership, @Mocked Sleeper sleeper, @@ -50,45 +51,36 @@ public void grabAssignedTokenStartDbInBootstrapModeWhenGossipAgreesCurrentInstan result = APP; factory.getAllIds(DEAD_APP); - result = Collections.emptyList(); + result = ImmutableSet.of(); factory.getAllIds(APP); - result = liveHosts; + result = ImmutableSet.copyOf(liveHosts); instanceInfo.getInstanceId(); result = liveHosts.get(0).getInstanceId(); + instanceInfo.getHostIP(); + result = liveHosts.get(0).getHostIP(); + TokenRetrieverUtils.inferTokenOwnerFromGossip( - liveHosts, liveHosts.get(0).getToken(), liveHosts.get(0).getDC()); + ImmutableSet.copyOf(liveHosts), + liveHosts.get(0).getToken(), + liveHosts.get(0).getDC()); result = inferredTokenOwnership; } }; - IDeadTokenRetriever deadTokenRetriever = - new DeadTokenRetriever(factory, membership, config, sleeper, instanceInfo); - IPreGeneratedTokenRetriever preGeneratedTokenRetriever = - new PreGeneratedTokenRetriever(factory, membership, config, sleeper, instanceInfo); - INewTokenRetriever newTokenRetriever = - new NewTokenRetriever( - factory, membership, config, sleeper, tokenManager, instanceInfo); + ITokenRetriever tokenRetriever = + new TokenRetriever( + factory, membership, config, instanceInfo, sleeper, tokenManager); InstanceIdentity instanceIdentity = - new InstanceIdentity( - factory, - membership, - config, - sleeper, - tokenManager, - deadTokenRetriever, - preGeneratedTokenRetriever, - newTokenRetriever, - instanceInfo); - + new InstanceIdentity(factory, membership, config, instanceInfo, tokenRetriever); Truth.assertThat(instanceIdentity.isReplace()).isFalse(); } @Test public void grabAssignedTokenStartDbInReplaceModeWhenGossipAgreesPreviousTokenOwnerIsNotLive( - @Mocked IPriamInstanceFactory factory, + @Mocked IPriamInstanceFactory factory, @Mocked IConfiguration config, @Mocked IMembership membership, @Mocked Sleeper sleeper, @@ -126,45 +118,33 @@ public void grabAssignedTokenStartDbInReplaceModeWhenGossipAgreesPreviousTokenOw result = APP; factory.getAllIds(DEAD_APP); - result = Collections.singletonList(deadInstance); + result = ImmutableSet.of(deadInstance); factory.getAllIds(APP); - result = liveHosts; + result = ImmutableSet.copyOf(liveHosts); instanceInfo.getInstanceId(); result = newInstance.getInstanceId(); TokenRetrieverUtils.inferTokenOwnerFromGossip( - liveHosts, newInstance.getToken(), newInstance.getDC()); + ImmutableSet.copyOf(liveHosts), + newInstance.getToken(), + newInstance.getDC()); result = inferredTokenOwnership; } }; - IDeadTokenRetriever deadTokenRetriever = - new DeadTokenRetriever(factory, membership, config, sleeper, instanceInfo); - IPreGeneratedTokenRetriever preGeneratedTokenRetriever = - new PreGeneratedTokenRetriever(factory, membership, config, sleeper, instanceInfo); - INewTokenRetriever newTokenRetriever = - new NewTokenRetriever( - factory, membership, config, sleeper, tokenManager, instanceInfo); + ITokenRetriever tokenRetriever = + new TokenRetriever( + factory, membership, config, instanceInfo, sleeper, tokenManager); InstanceIdentity instanceIdentity = - new InstanceIdentity( - factory, - membership, - config, - sleeper, - tokenManager, - deadTokenRetriever, - preGeneratedTokenRetriever, - newTokenRetriever, - instanceInfo); - + new InstanceIdentity(factory, membership, config, instanceInfo, tokenRetriever); Truth.assertThat(instanceIdentity.getReplacedIp()).isEqualTo(deadInstance.getHostIP()); Truth.assertThat(instanceIdentity.isReplace()).isTrue(); } @Test public void grabAssignedTokenThrowWhenGossipAgreesPreviousTokenOwnerIsLive( - @Mocked IPriamInstanceFactory factory, + @Mocked IPriamInstanceFactory factory, @Mocked IConfiguration config, @Mocked IMembership membership, @Mocked Sleeper sleeper, @@ -201,44 +181,34 @@ public void grabAssignedTokenThrowWhenGossipAgreesPreviousTokenOwnerIsLive( result = APP; factory.getAllIds(DEAD_APP); - result = Collections.singletonList(deadInstance); + result = ImmutableSet.of(deadInstance); factory.getAllIds(APP); - result = liveHosts; + result = ImmutableSet.copyOf(liveHosts); instanceInfo.getInstanceId(); result = newInstance.getInstanceId(); TokenRetrieverUtils.inferTokenOwnerFromGossip( - liveHosts, newInstance.getToken(), newInstance.getDC()); + ImmutableSet.copyOf(liveHosts), + newInstance.getToken(), + newInstance.getDC()); result = inferredTokenOwnership; } }; - IDeadTokenRetriever deadTokenRetriever = - new DeadTokenRetriever(factory, membership, config, sleeper, instanceInfo); - IPreGeneratedTokenRetriever preGeneratedTokenRetriever = - new PreGeneratedTokenRetriever(factory, membership, config, sleeper, instanceInfo); - INewTokenRetriever newTokenRetriever = - new NewTokenRetriever( - factory, membership, config, sleeper, tokenManager, instanceInfo); + ITokenRetriever tokenRetriever = + new TokenRetriever( + factory, membership, config, instanceInfo, sleeper, tokenManager); Assertions.assertThrows( TokenRetrieverUtils.GossipParseException.class, () -> new InstanceIdentity( - factory, - membership, - config, - sleeper, - tokenManager, - deadTokenRetriever, - preGeneratedTokenRetriever, - newTokenRetriever, - instanceInfo)); + factory, membership, config, instanceInfo, tokenRetriever)); } @Test public void grabAssignedTokenStartDbInBootstrapModeWhenGossipDisagreesOnPreviousTokenOwner( - @Mocked IPriamInstanceFactory factory, + @Mocked IPriamInstanceFactory factory, @Mocked IConfiguration config, @Mocked IMembership membership, @Mocked Sleeper sleeper, @@ -262,38 +232,26 @@ public void grabAssignedTokenStartDbInBootstrapModeWhenGossipDisagreesOnPrevious result = APP; factory.getAllIds(DEAD_APP); - result = Collections.emptyList(); + result = ImmutableSet.of(); factory.getAllIds(APP); - result = liveHosts; + result = ImmutableSet.copyOf(liveHosts); instanceInfo.getInstanceId(); result = liveHosts.get(0).getInstanceId(); TokenRetrieverUtils.inferTokenOwnerFromGossip( - liveHosts, liveHosts.get(0).getToken(), liveHosts.get(0).getDC()); + ImmutableSet.copyOf(liveHosts), + liveHosts.get(0).getToken(), + liveHosts.get(0).getDC()); result = inferredTokenOwnership; } }; - IDeadTokenRetriever deadTokenRetriever = - new DeadTokenRetriever(factory, membership, config, sleeper, instanceInfo); - IPreGeneratedTokenRetriever preGeneratedTokenRetriever = - new PreGeneratedTokenRetriever(factory, membership, config, sleeper, instanceInfo); - INewTokenRetriever newTokenRetriever = - new NewTokenRetriever( - factory, membership, config, sleeper, tokenManager, instanceInfo); + ITokenRetriever tokenRetriever = + new TokenRetriever( + factory, membership, config, instanceInfo, sleeper, tokenManager); InstanceIdentity instanceIdentity = - new InstanceIdentity( - factory, - membership, - config, - sleeper, - tokenManager, - deadTokenRetriever, - preGeneratedTokenRetriever, - newTokenRetriever, - instanceInfo); - + new InstanceIdentity(factory, membership, config, instanceInfo, tokenRetriever); Truth.assertThat(Strings.isNullOrEmpty(instanceIdentity.getReplacedIp())).isTrue(); Truth.assertThat(instanceIdentity.isReplace()).isFalse(); } diff --git a/priam/src/test/java/com/netflix/priam/identity/token/DeadTokenRetrieverTest.java b/priam/src/test/java/com/netflix/priam/identity/token/DeadTokenRetrieverTest.java deleted file mode 100644 index 1c9ac7ab3..000000000 --- a/priam/src/test/java/com/netflix/priam/identity/token/DeadTokenRetrieverTest.java +++ /dev/null @@ -1,220 +0,0 @@ -/* - * Copyright 2019 Netflix, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.netflix.priam.identity.token; - -import com.google.common.collect.Lists; -import com.google.inject.Guice; -import com.google.inject.Injector; -import com.netflix.priam.backup.BRTestModule; -import com.netflix.priam.config.IConfiguration; -import com.netflix.priam.identity.IMembership; -import com.netflix.priam.identity.IPriamInstanceFactory; -import com.netflix.priam.identity.PriamInstance; -import com.netflix.priam.identity.config.InstanceInfo; -import com.netflix.priam.utils.FakeSleeper; -import com.netflix.priam.utils.SystemUtils; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; -import java.util.stream.IntStream; -import mockit.Expectations; -import mockit.Mocked; -import mockit.Verifications; -import org.codehaus.jettison.json.JSONObject; -import org.junit.Assert; -import org.junit.Test; - -/** Created by aagrawal on 3/1/19. */ -public class DeadTokenRetrieverTest { - @Mocked private IPriamInstanceFactory factory; - @Mocked private IMembership membership; - private IDeadTokenRetriever deadTokenRetriever; - private InstanceInfo instanceInfo; - private IConfiguration configuration; - - private Map tokenToEndpointMap = - IntStream.range(0, 6) - .mapToObj(e -> Integer.valueOf(e)) - .collect( - Collectors.toMap( - e -> String.valueOf(e), e -> String.format("127.0.0.%s", e))); - private List liveInstances = - IntStream.range(0, 6) - .mapToObj(e -> String.format("127.0.0.%d", e)) - .collect(Collectors.toList()); - - public DeadTokenRetrieverTest() { - Injector injector = Guice.createInjector(new BRTestModule()); - if (instanceInfo == null) instanceInfo = injector.getInstance(InstanceInfo.class); - if (configuration == null) configuration = injector.getInstance(IConfiguration.class); - } - - @Test - public void testNoReplacementNormalScenario() throws Exception { - new Expectations() { - { - factory.getAllIds(anyString); - result = Lists.newArrayList(); - membership.getRacMembership(); - result = Lists.newArrayList(); - } - }; - deadTokenRetriever = - new DeadTokenRetriever( - factory, membership, configuration, new FakeSleeper(), instanceInfo); - PriamInstance priamInstance = deadTokenRetriever.get(); - Assert.assertNull(priamInstance); - } - - @Test - // There is no slot available for replacement as per Token Database. - public void testNoReplacementNoSpotAvailable() throws Exception { - List allInstances = getInstances(1); - List racMembership = getRacMembership(1); - racMembership.add(instanceInfo.getInstanceId()); - - new Expectations() { - { - factory.getAllIds(anyString); - result = allInstances; - membership.getRacMembership(); - result = racMembership; - } - }; - deadTokenRetriever = - new DeadTokenRetriever( - factory, membership, configuration, new FakeSleeper(), instanceInfo); - PriamInstance priamInstance = deadTokenRetriever.get(); - Assert.assertNull(priamInstance); - new Verifications() { - { - factory.delete(withAny(priamInstance)); - times = 0; - } - }; - } - - @Test - // There is a potential slot for dead token but we are unable to replace. - public void testNoReplacementNoGossipMatch(@Mocked SystemUtils systemUtils) throws Exception { - List allInstances = getInstances(2); - List racMembership = getRacMembership(1); - racMembership.add(instanceInfo.getInstanceId()); - PriamInstance instance = null; - // gossip info returns null, thus unable to replace the instance. - new Expectations() { - { - factory.getAllIds(anyString); - result = allInstances; - membership.getRacMembership(); - result = racMembership; - SystemUtils.getDataFromUrl(anyString); - result = getStatus(liveInstances, tokenToEndpointMap); - times = 1; - } - }; - deadTokenRetriever = - new DeadTokenRetriever( - factory, membership, configuration, new FakeSleeper(), instanceInfo); - PriamInstance priamInstance = deadTokenRetriever.get(); - Assert.assertNull(priamInstance); - new Verifications() { - { - factory.delete(withAny(instance)); - times = 1; - } - }; - } - - private String getStatus(List liveInstances, Map tokenToEndpointMap) { - JSONObject jsonObject = new JSONObject(); - try { - jsonObject.put("live", liveInstances); - jsonObject.put("tokenToEndpointMap", tokenToEndpointMap); - } catch (Exception e) { - - } - return jsonObject.toString(); - } - - @Test - public void testReplacementGossipMatch(@Mocked SystemUtils systemUtils) throws Exception { - List allInstances = getInstances(6); - List racMembership = getRacMembership(2); - racMembership.add(instanceInfo.getInstanceId()); - - List myliveInstances = - liveInstances - .stream() - .filter(x -> !x.equalsIgnoreCase("127.0.0.3")) - .collect(Collectors.toList()); - String gossipResponse = getStatus(myliveInstances, tokenToEndpointMap); - - new Expectations() { - { - factory.getAllIds(anyString); - result = allInstances; - membership.getRacMembership(); - result = racMembership; - SystemUtils.getDataFromUrl(anyString); - returns(gossipResponse, gossipResponse, null, "random_value", gossipResponse); - } - }; - deadTokenRetriever = - new DeadTokenRetriever( - factory, membership, configuration, new FakeSleeper(), instanceInfo); - PriamInstance priamInstance = deadTokenRetriever.get(); - Assert.assertNotNull(priamInstance); - Assert.assertEquals("127.0.0.3", deadTokenRetriever.getReplaceIp()); - } - - private List getInstances(int noOfInstances) { - List allInstances = Lists.newArrayList(); - for (int i = 1; i <= noOfInstances; i++) - allInstances.add( - create( - i, - String.format("instance_id_%d", i), - String.format("hostname_%d", i), - String.format("127.0.0.%d", i), - instanceInfo.getRac(), - i + "")); - return allInstances; - } - - private List getRacMembership(int noOfInstances) { - List racMembership = Lists.newArrayList(); - for (int i = 1; i <= noOfInstances; i++) - racMembership.add(String.format("instance_id_%d", i)); - return racMembership; - } - - private PriamInstance create( - int id, String instanceID, String hostname, String ip, String rac, String payload) { - PriamInstance ins = new PriamInstance(); - ins.setApp(configuration.getAppName()); - ins.setRac(rac); - ins.setHost(hostname); - ins.setHostIP(ip); - ins.setId(id); - ins.setInstanceId(instanceID); - ins.setDC(instanceInfo.getRegion()); - ins.setToken(payload); - return ins; - } -} diff --git a/priam/src/test/java/com/netflix/priam/identity/token/TokenRetrieverTest.java b/priam/src/test/java/com/netflix/priam/identity/token/TokenRetrieverTest.java new file mode 100644 index 000000000..94c2f8769 --- /dev/null +++ b/priam/src/test/java/com/netflix/priam/identity/token/TokenRetrieverTest.java @@ -0,0 +1,444 @@ +/* + * Copyright 2019 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.netflix.priam.identity.token; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Lists; +import com.google.common.truth.Truth; +import com.google.inject.Guice; +import com.google.inject.Injector; +import com.netflix.priam.backup.BRTestModule; +import com.netflix.priam.config.FakeConfiguration; +import com.netflix.priam.config.IConfiguration; +import com.netflix.priam.identity.IMembership; +import com.netflix.priam.identity.IPriamInstanceFactory; +import com.netflix.priam.identity.PriamInstance; +import com.netflix.priam.identity.config.InstanceInfo; +import com.netflix.priam.utils.FakeSleeper; +import com.netflix.priam.utils.SystemUtils; +import com.netflix.priam.utils.TokenManager; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.IntStream; +import mockit.Expectations; +import mockit.Mocked; +import org.codehaus.jettison.json.JSONObject; +import org.junit.Test; +import org.junit.jupiter.api.Assertions; + +/** Created by aagrawal on 3/1/19. */ +public class TokenRetrieverTest { + @Mocked private IMembership membership; + private IPriamInstanceFactory factory; + private InstanceInfo instanceInfo; + private IConfiguration configuration; + + private Map tokenToEndpointMap = + IntStream.range(0, 6) + .boxed() + .collect( + Collectors.toMap(String::valueOf, e -> String.format("127.0.0.%s", e))); + private ImmutableList liveInstances = ImmutableList.copyOf(tokenToEndpointMap.values()); + + public TokenRetrieverTest() { + Injector injector = Guice.createInjector(new BRTestModule()); + instanceInfo = injector.getInstance(InstanceInfo.class); + configuration = injector.getInstance(IConfiguration.class); + factory = injector.getInstance(IPriamInstanceFactory.class); + } + + @Test + public void testNoReplacementNormalScenario() throws Exception { + new Expectations() { + { + membership.getRacMembership(); + result = ImmutableSet.of(); + } + }; + PriamInstance priamInstance = getTokenRetriever().grabExistingToken(); + Truth.assertThat(priamInstance).isNull(); + } + + @Test + // There is no slot available for replacement as per Token Database. + public void testNoReplacementNoSpotAvailable() throws Exception { + List allInstances = getInstances(1); + Set racMembership = getRacMembership(1); + racMembership.add(instanceInfo.getInstanceId()); + new Expectations() { + { + membership.getRacMembership(); + result = ImmutableSet.copyOf(racMembership); + } + }; + TokenRetriever tokenRetriever = getTokenRetriever(); + Truth.assertThat(tokenRetriever.grabExistingToken()).isNull(); + Truth.assertThat(tokenRetriever.getReplacedIp().isPresent()).isFalse(); + Truth.assertThat(factory.getAllIds(configuration.getAppName())) + .containsExactlyElementsIn(allInstances); + } + + @Test + // There is a potential slot for dead token but we are unable to replace. + public void testNoReplacementNoGossipMatch(@Mocked SystemUtils systemUtils) throws Exception { + getInstances(2); + Set racMembership = getRacMembership(1); + racMembership.add(instanceInfo.getInstanceId()); + // gossip info returns null, thus unable to replace the instance. + new Expectations() { + { + membership.getRacMembership(); + result = ImmutableSet.copyOf(racMembership); + SystemUtils.getDataFromUrl(anyString); + result = getStatus(liveInstances, tokenToEndpointMap); + times = 1; + } + }; + TokenRetriever tokenRetriever = getTokenRetriever(); + Truth.assertThat(tokenRetriever.grabExistingToken()).isNull(); + Truth.assertThat(tokenRetriever.getReplacedIp().isPresent()).isFalse(); + } + + @Test + // There is a potential slot for dead token but we are unable to replace. + public void testUsePregeneratedTokenWhenThereIsNoGossipMatchForDeadToken( + @Mocked SystemUtils systemUtils) throws Exception { + create(0, "iid_0", "host_0", "127.0.0.0", instanceInfo.getRac(), 0 + ""); + create(1, "new_slot", "host_1", "127.0.0.1", instanceInfo.getRac(), 1 + ""); + // gossip info returns null, thus unable to replace the instance. + new Expectations() { + { + membership.getRacMembership(); + result = ImmutableSet.of(); + SystemUtils.getDataFromUrl(anyString); + result = getStatus(liveInstances, tokenToEndpointMap); + times = 1; + } + }; + TokenRetriever tokenRetriever = getTokenRetriever(); + PriamInstance instance = tokenRetriever.grabExistingToken(); + Truth.assertThat(instance).isNotNull(); + Truth.assertThat(instance.getId()).isEqualTo(1); + Truth.assertThat(tokenRetriever.getReplacedIp().isPresent()).isFalse(); + } + + @Test + public void testReplacementGossipMatch(@Mocked SystemUtils systemUtils) throws Exception { + getInstances(6); + Set racMembership = getRacMembership(2); + racMembership.add(instanceInfo.getInstanceId()); + + List myliveInstances = + liveInstances + .stream() + .filter(x -> !x.equalsIgnoreCase("127.0.0.3")) + .collect(Collectors.toList()); + String gossipResponse = getStatus(myliveInstances, tokenToEndpointMap); + + new Expectations() { + { + membership.getRacMembership(); + result = ImmutableSet.copyOf(racMembership); + SystemUtils.getDataFromUrl(anyString); + returns(gossipResponse, gossipResponse, null, "random_value", gossipResponse); + } + }; + TokenRetriever tokenRetriever = getTokenRetriever(); + Truth.assertThat(tokenRetriever.grabExistingToken()).isNotNull(); + Truth.assertThat(tokenRetriever.getReplacedIp().isPresent()).isTrue(); + Truth.assertThat(tokenRetriever.getReplacedIp().get()).isEqualTo("127.0.0.3"); + } + + @Test + public void testPrioritizeDeadTokens(@Mocked SystemUtils systemUtils) throws Exception { + create(0, "iid_0", "host_0", "127.0.0.0", instanceInfo.getRac(), 0 + ""); + create(1, "new_slot", "host_1", "127.0.0.1", instanceInfo.getRac(), 1 + ""); + new Expectations() { + { + membership.getRacMembership(); + result = ImmutableSet.of(); + SystemUtils.getDataFromUrl(anyString); + returns(null, null); + } + }; + TokenRetriever tokenRetriever = getTokenRetriever(); + Truth.assertThat(tokenRetriever.grabExistingToken()).isNotNull(); + Truth.assertThat(tokenRetriever.getReplacedIp().isPresent()).isTrue(); + Truth.assertThat(tokenRetriever.getReplacedIp().get()).isEqualTo("127.0.0.0"); + } + + @Test + public void testPrioritizeDeadInstancesEvenIfAfterANewSlot(@Mocked SystemUtils systemUtils) + throws Exception { + create(0, "new_slot", "host_0", "127.0.0.0", instanceInfo.getRac(), 0 + ""); + create(1, "iid_1", "host_1", "127.0.0.1", instanceInfo.getRac(), 1 + ""); + new Expectations() { + { + membership.getRacMembership(); + result = ImmutableSet.of(); + SystemUtils.getDataFromUrl(anyString); + returns(null, null); + } + }; + TokenRetriever tokenRetriever = getTokenRetriever(); + Truth.assertThat(tokenRetriever.grabExistingToken()).isNotNull(); + Truth.assertThat(tokenRetriever.getReplacedIp().isPresent()).isTrue(); + Truth.assertThat(tokenRetriever.getReplacedIp().get()).isEqualTo("127.0.0.1"); + } + + @Test + public void testNewTokenFailureIfProhibited() { + ((FakeConfiguration) configuration).setCreateNewToken(false); + create(0, "iid_0", "host_0", "127.0.0.0", instanceInfo.getRac(), 0 + ""); + create(1, "iid_1", "host_1", "127.0.0.1", instanceInfo.getRac(), 1 + ""); + new Expectations() { + { + membership.getRacMembership(); + result = ImmutableSet.of("iid_0", "iid_1"); + } + }; + Assertions.assertThrows(IllegalStateException.class, () -> getTokenRetriever().get()); + } + + @Test + public void testNewTokenNoInstancesInRac() throws Exception { + create(0, "iid_0", "host_0", "127.0.0.0", "az2", 0 + ""); + create(1, "iid_1", "host_1", "127.0.0.1", "az2", 1 + ""); + new Expectations() { + { + membership.getRacMembership(); + result = ImmutableSet.of("iid_0", "iid_1"); + membership.getRacCount(); + result = 1; + membership.getRacMembershipSize(); + result = 3; + } + }; + PriamInstance instance = getTokenRetriever().get(); + Truth.assertThat(instance.getToken()).isEqualTo("1808575600"); + // region offset for us-east-1 + index of rac az1 (1808575600 + 0) + Truth.assertThat(instance.getId()).isEqualTo(1808575600); + } + + @Test + public void testNewTokenGenerationNoInstancesWithLargeEnoughId() throws Exception { + create(0, "iid_0", "host_0", "127.0.0.0", "az1", 0 + ""); + create(1, "iid_1", "host_1", "127.0.0.1", "az1", 1 + ""); + new Expectations() { + { + membership.getRacMembership(); + result = ImmutableSet.of("iid_0", "iid_1"); + membership.getRacCount(); + result = 1; + membership.getRacMembershipSize(); + result = 3; + } + }; + PriamInstance instance = getTokenRetriever().get(); + Truth.assertThat(instance.getToken()).isEqualTo("170141183460469231731687303717692681326"); + // region offset for us-east-1 + number of racs in cluster (3) + Truth.assertThat(instance.getId()).isEqualTo(1808575603); + } + + @Test + public void testNewTokenFailureWhenMyRacIsNotInCluster() { + ((FakeConfiguration) configuration).setRacs("az2", "az3"); + create(0, "iid_0", "host_0", "127.0.0.0", "az2", 0 + ""); + create(1, "iid_1", "host_1", "127.0.0.1", "az2", 1 + ""); + new Expectations() { + { + membership.getRacMembership(); + result = ImmutableSet.of("iid_0", "iid_1"); + } + }; + Assertions.assertThrows(IllegalStateException.class, () -> getTokenRetriever().get()); + } + + @Test + public void testNewTokenGenerationMultipleInstancesWithLargetEnoughIds() throws Exception { + create(2000000000, "iid_0", "host_0", "127.0.0.0", "az1", 0 + ""); + create(2000000001, "iid_1", "host_1", "127.0.0.1", "az1", 1 + ""); + new Expectations() { + { + membership.getRacMembership(); + result = ImmutableSet.of("iid_0", "iid_1"); + membership.getRacCount(); + result = 1; + membership.getRacMembershipSize(); + result = 3; + } + }; + PriamInstance instance = getTokenRetriever().get(); + Truth.assertThat(instance.getToken()) + .isEqualTo("10856391546591660081525376676060033425699421368"); + // max id (2000000001) + total instances (3) + Truth.assertThat(instance.getId()).isEqualTo(2000000004); + } + + @Test + public void testPreassignedTokenNotReplacedIfPublicIPMatch(@Mocked SystemUtils systemUtils) + throws Exception { + // IP in DB doesn't matter so we make it different to confirm that + create(0, instanceInfo.getInstanceId(), "host_0", "1.2.3.4", "az1", 0 + ""); + getInstances(5); + String gossipResponse = getStatus(liveInstances, tokenToEndpointMap); + + new Expectations() { + { + SystemUtils.getDataFromUrl(anyString); + returns(gossipResponse, gossipResponse, null, "random_value", gossipResponse); + } + }; + TokenRetriever tokenRetriever = getTokenRetriever(); + tokenRetriever.get(); + Truth.assertThat(tokenRetriever.getReplacedIp().isPresent()).isFalse(); + } + + @Test + public void testPreassignedTokenNotReplacedIfPrivateIPMatch(@Mocked SystemUtils systemUtils) + throws Exception { + // IP in DB doesn't matter so we make it different to confirm that + create(0, instanceInfo.getInstanceId(), "host_0", "1.2.3.4", "az1", 0 + ""); + getInstances(5); + Map myTokenToEndpointMap = + IntStream.range(0, 7) + .boxed() + .collect( + Collectors.toMap( + String::valueOf, e -> String.format("127.1.1.%s", e))); + ImmutableList myLiveInstances = ImmutableList.copyOf(tokenToEndpointMap.values()); + String gossipResponse = getStatus(myLiveInstances, myTokenToEndpointMap); + + new Expectations() { + { + SystemUtils.getDataFromUrl(anyString); + returns(gossipResponse, gossipResponse, null, "random_value", gossipResponse); + } + }; + TokenRetriever tokenRetriever = getTokenRetriever(); + tokenRetriever.get(); + Truth.assertThat(tokenRetriever.getReplacedIp().isPresent()).isFalse(); + } + + @Test + public void testGetPreassignedTokenThrowsIfOwnerIPIsLive(@Mocked SystemUtils systemUtils) + throws Exception { + getInstances(5); + create(6, instanceInfo.getInstanceId(), "host_5", "1.2.3.4", "az1", 6 + ""); + Map myTokenToEndpointMap = + IntStream.range(0, 7) + .boxed() + .collect( + Collectors.toMap( + String::valueOf, e -> String.format("18.221.0.%s", e))); + ImmutableList myLiveInstances = ImmutableList.copyOf(myTokenToEndpointMap.values()); + String gossipResponse = getStatus(myLiveInstances, myTokenToEndpointMap); + + new Expectations() { + { + SystemUtils.getDataFromUrl(anyString); + returns(gossipResponse, gossipResponse, null, "random_value", gossipResponse); + } + }; + Assertions.assertThrows( + TokenRetrieverUtils.GossipParseException.class, () -> getTokenRetriever().get()); + } + + @Test + public void testGetPreassignedTokenReplacesIfOwnerIPIsNotLive(@Mocked SystemUtils systemUtils) + throws Exception { + getInstances(5); + create(6, instanceInfo.getInstanceId(), "host_0", "1.2.3.4", "az1", 6 + ""); + Map myTokenToEndpointMap = + IntStream.range(0, 7) + .boxed() + .collect( + Collectors.toMap( + String::valueOf, e -> String.format("18.221.0.%s", e))); + List myLiveInstances = + tokenToEndpointMap.values().stream().sorted().limit(6).collect(Collectors.toList()); + String gossipResponse = getStatus(myLiveInstances, myTokenToEndpointMap); + + new Expectations() { + { + SystemUtils.getDataFromUrl(anyString); + returns(gossipResponse, gossipResponse, null, "random_value", gossipResponse); + } + }; + TokenRetriever tokenRetriever = getTokenRetriever(); + tokenRetriever.get(); + Truth.assertThat(tokenRetriever.getReplacedIp().isPresent()).isTrue(); + } + + @Test + public void testIPIsUpdatedWhenGrabbingPreassignedToken(@Mocked SystemUtils systemUtils) + throws Exception { + create(0, instanceInfo.getInstanceId(), "host_0", "1.2.3.4", "az1", 0 + ""); + Truth.assertThat(getTokenRetriever().get().getHostIP()).isEqualTo("127.0.0.0"); + } + + private String getStatus(List liveInstances, Map tokenToEndpointMap) { + JSONObject jsonObject = new JSONObject(); + try { + jsonObject.put("live", liveInstances); + jsonObject.put("tokenToEndpointMap", tokenToEndpointMap); + } catch (Exception e) { + + } + return jsonObject.toString(); + } + + private List getInstances(int noOfInstances) { + List allInstances = Lists.newArrayList(); + for (int i = 1; i <= noOfInstances; i++) + allInstances.add( + create( + i, + String.format("instance_id_%d", i), + String.format("hostname_%d", i), + String.format("127.0.0.%d", i), + instanceInfo.getRac(), + i + "")); + return allInstances; + } + + private Set getRacMembership(int noOfInstances) { + return IntStream.range(1, noOfInstances + 1) + .mapToObj(i -> String.format("instance_id_%d", i)) + .collect(Collectors.toSet()); + } + + private PriamInstance create( + int id, String instanceID, String hostname, String ip, String rac, String payload) { + return factory.create( + configuration.getAppName(), id, instanceID, hostname, ip, rac, null, payload); + } + + private TokenRetriever getTokenRetriever() { + return new TokenRetriever( + factory, + membership, + configuration, + instanceInfo, + new FakeSleeper(), + new TokenManager(configuration)); + } +} diff --git a/priam/src/test/java/com/netflix/priam/identity/token/TokenRetrieverUtilsTest.java b/priam/src/test/java/com/netflix/priam/identity/token/TokenRetrieverUtilsTest.java index c5bd4f6cd..7d20e90f6 100644 --- a/priam/src/test/java/com/netflix/priam/identity/token/TokenRetrieverUtilsTest.java +++ b/priam/src/test/java/com/netflix/priam/identity/token/TokenRetrieverUtilsTest.java @@ -3,6 +3,7 @@ import static org.hamcrest.core.AllOf.allOf; import static org.hamcrest.core.IsNot.not; +import com.google.common.collect.ImmutableSet; import com.netflix.priam.identity.PriamInstance; import com.netflix.priam.utils.SystemUtils; import java.util.List; @@ -19,20 +20,21 @@ public class TokenRetrieverUtilsTest { private static final String APP = "testapp"; private static final String STATUS_URL_FORMAT = "http://%s:8080/Priam/REST/v1/cassadmin/status"; - private List instances = - IntStream.range(0, 6) - .mapToObj( - e -> - newMockPriamInstance( - APP, - "us-east", - (e < 3) ? "az1" : "az2", - e, - String.format("fakeInstance-%d", e), - String.format("127.0.0.%d", e), - String.format("fakeHost-%d", e), - String.valueOf(e))) - .collect(Collectors.toList()); + private ImmutableSet instances = + ImmutableSet.copyOf( + IntStream.range(0, 6) + .mapToObj( + e -> + newMockPriamInstance( + APP, + "us-east", + (e < 3) ? "az1" : "az2", + e, + String.format("fakeInstance-%d", e), + String.format("127.0.0.%d", e), + String.format("fakeHost-%d", e), + String.valueOf(e))) + .collect(Collectors.toList())); private Map tokenToEndpointMap = IntStream.range(0, 6) diff --git a/priam/src/test/java/com/netflix/priam/resources/PriamInstanceResourceTest.java b/priam/src/test/java/com/netflix/priam/resources/PriamInstanceResourceTest.java index d7470945b..9d1cde4cc 100644 --- a/priam/src/test/java/com/netflix/priam/resources/PriamInstanceResourceTest.java +++ b/priam/src/test/java/com/netflix/priam/resources/PriamInstanceResourceTest.java @@ -20,12 +20,11 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.fail; -import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; import com.netflix.priam.config.IConfiguration; import com.netflix.priam.identity.IPriamInstanceFactory; import com.netflix.priam.identity.PriamInstance; import com.netflix.priam.identity.config.InstanceInfo; -import java.util.List; import javax.ws.rs.WebApplicationException; import javax.ws.rs.core.Response; import mockit.Expectations; @@ -56,7 +55,8 @@ public void getInstances( @Mocked final PriamInstance instance2, @Mocked final PriamInstance instance3) { new Expectations() { - final List instances = ImmutableList.of(instance1, instance2, instance3); + final ImmutableSet instances = + ImmutableSet.of(instance1, instance2, instance3); { config.getAppName(); diff --git a/priam/src/test/java/com/netflix/priam/utils/Murmur3TokenManagerTest.java b/priam/src/test/java/com/netflix/priam/utils/Murmur3TokenManagerTest.java index 8a946eebd..a5bcc0e3f 100644 --- a/priam/src/test/java/com/netflix/priam/utils/Murmur3TokenManagerTest.java +++ b/priam/src/test/java/com/netflix/priam/utils/Murmur3TokenManagerTest.java @@ -91,12 +91,12 @@ public void createToken() { assertEquals( MAXIMUM_TOKEN_MURMUR3 .subtract(MINIMUM_TOKEN_MURMUR3) - .divide(BigInteger.valueOf(8 * 32)) + .divide(BigInteger.valueOf(256)) .multiply(BigInteger.TEN) .add(BigInteger.valueOf(tokenManager.regionOffset("region"))) .add(MINIMUM_TOKEN_MURMUR3) .toString(), - tokenManager.createToken(10, 8, 32, "region")); + tokenManager.createToken(10, 256, "region")); } @Test(expected = IllegalArgumentException.class) diff --git a/priam/src/test/java/com/netflix/priam/utils/RandomTokenManagerTest.java b/priam/src/test/java/com/netflix/priam/utils/RandomTokenManagerTest.java index 6a4aeda46..7bf791ede 100644 --- a/priam/src/test/java/com/netflix/priam/utils/RandomTokenManagerTest.java +++ b/priam/src/test/java/com/netflix/priam/utils/RandomTokenManagerTest.java @@ -90,11 +90,11 @@ public void initialToken_cannotExceedMaximumToken() { public void createToken() { assertEquals( MAXIMUM_TOKEN_RANDOM - .divide(BigInteger.valueOf(8 * 32)) + .divide(BigInteger.valueOf(256)) .multiply(BigInteger.TEN) .add(BigInteger.valueOf(tokenManager.regionOffset("region"))) .toString(), - tokenManager.createToken(10, 8, 32, "region")); + tokenManager.createToken(10, 256, "region")); } @Test(expected = IllegalArgumentException.class) From 179ef5ec47fd2b7c4bb7ea1a9985a310165eae3c Mon Sep 17 00:00:00 2001 From: Matt Lehman Date: Wed, 17 Mar 2021 18:20:53 -0700 Subject: [PATCH 159/228] Update CHANGELOG in advance of 3.11.77 --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2eab578e4..8aa289834 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,7 @@ # Changelog +## 2021/03/17 3.11.77 +(#913) Store Private IPs in the token database when the snitch is GPFS. + ## 2021/03/09 3.11.76 (#918) Adding support for custom override for role_manager. From 31b7b908e9f37dae18169b472c52789092ea73d9 Mon Sep 17 00:00:00 2001 From: Roberto Perez Alcolea Date: Thu, 18 Mar 2021 15:24:31 -0700 Subject: [PATCH 160/228] Replace JCenter with Maven Central --- build.gradle | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/build.gradle b/build.gradle index 591a397c3..db5eaa911 100644 --- a/build.gradle +++ b/build.gradle @@ -3,10 +3,6 @@ plugins { id 'com.github.sherter.google-java-format' version '0.8' } -repositories { - jcenter() -} - googleJavaFormat { options style: 'AOSP' } @@ -20,7 +16,7 @@ allprojects { group = 'com.netflix.priam' repositories { - jcenter() + mavenCentral() } configurations { From a48d1b7d2bf02fbaaa9028e03970cd5f9233c255 Mon Sep 17 00:00:00 2001 From: mattl-netflix <63665634+mattl-netflix@users.noreply.github.com> Date: Fri, 2 Apr 2021 17:46:54 -0700 Subject: [PATCH 161/228] Skip backup compression based on configuration (#922) * CASS-1264 remove dead code. * CASS-1264 Move buffer size constraint to the only place it is used and simplify its usage. * CASS-1264 simplify getChunkSize() * CASS-1264 style tweaks * CASS-1264 remove redundant comments * CASS-1264 change IBackupFileSystem download api to include compression information * CASS-1264 pushing AbstractBackupPath (read compression info) further down. Plus change the getFileSize method to take a String to better accommodate existing usage patterns. * CASS-1264 remove option to delete file after upload. It is always deleted in practice. * CASS-1264 remove redundant Path parameters from upload api. They are embedded in the AbstractBackupPath in practice. * CASS-1264 push compression info all the way down to where it is needed. * CASS-1264 skip uncompression on download. * CASS-1264 Skip compression on backup based on desires in path. * CASS-1264 choose compression behavior on backup based on fast property rather tahn just defaulting to Snappy always. * CASS-1264 a little reorganization in advance of making metafiles aware of varying compression. * CASS-1264 Make metafile display compression algorithm properly * CASS-1264 Limit conditional compression to V2 backups. * CASS-1264 Addressing review comments. * CASS-1264 ensure disctinction between creation time and last modified in FileUploadResult. They may not always be the same in practice. * CASS-1264 rename BackupsToCompress.UNCOMPRESSED to IF_REQUIRED per review feedback * CASS-1264 Rename CompressionAlgorithm enum to CompressionType per review comments --- .../netflix/priam/aws/RemoteBackupPath.java | 12 +- .../priam/aws/S3EncryptedFileSystem.java | 32 +-- .../com/netflix/priam/aws/S3FileSystem.java | 225 +++++++----------- .../netflix/priam/aws/S3FileSystemBase.java | 14 +- .../netflix/priam/backup/AbstractBackup.java | 99 ++++---- .../priam/backup/AbstractBackupPath.java | 48 +++- .../priam/backup/AbstractFileSystem.java | 68 +++--- .../netflix/priam/backup/CommitLogBackup.java | 8 +- .../priam/backup/IBackupFileSystem.java | 47 +--- .../com/netflix/priam/backup/MetaData.java | 13 +- .../priam/backupv2/FileUploadResult.java | 26 +- .../priam/backupv2/MetaFileWriterBuilder.java | 72 ++++-- .../netflix/priam/backupv2/MetaV1Proxy.java | 5 +- .../netflix/priam/backupv2/MetaV2Proxy.java | 5 +- .../priam/backupv2/SnapshotMetaTask.java | 108 ++------- .../netflix/priam/compress/ChunkedStream.java | 30 ++- ...ionAlgorithm.java => CompressionType.java} | 2 +- .../netflix/priam/compress/ICompression.java | 4 - .../priam/compress/SnappyCompression.java | 6 - .../priam/config/BackupsToCompress.java | 7 + .../netflix/priam/config/IConfiguration.java | 8 + .../priam/config/PriamConfiguration.java | 6 + .../google/GoogleEncryptedFileSystem.java | 13 +- .../priam/restore/AbstractRestore.java | 7 +- .../priam/restore/EncryptedRestoreBase.java | 68 +++--- .../com/netflix/priam/restore/Restore.java | 8 +- .../priam/backup/FakeBackupFileSystem.java | 22 +- .../priam/backup/NullBackupFileSystem.java | 6 +- .../priam/backup/TestAbstractBackup.java | 127 ++++++++++ .../priam/backup/TestAbstractFileSystem.java | 102 +++----- .../netflix/priam/backup/TestCompression.java | 7 +- .../priam/backup/TestS3FileSystem.java | 79 +++--- .../priam/backupv2/TestBackupUtils.java | 31 +-- .../priam/backupv2/TestMetaV2Proxy.java | 8 +- .../priam/config/FakeConfiguration.java | 5 + 35 files changed, 681 insertions(+), 647 deletions(-) rename priam/src/main/java/com/netflix/priam/compress/{CompressionAlgorithm.java => CompressionType.java} (66%) create mode 100644 priam/src/main/java/com/netflix/priam/config/BackupsToCompress.java create mode 100644 priam/src/test/java/com/netflix/priam/backup/TestAbstractBackup.java diff --git a/priam/src/main/java/com/netflix/priam/aws/RemoteBackupPath.java b/priam/src/main/java/com/netflix/priam/aws/RemoteBackupPath.java index 6a20ffc92..a7fb7098a 100644 --- a/priam/src/main/java/com/netflix/priam/aws/RemoteBackupPath.java +++ b/priam/src/main/java/com/netflix/priam/aws/RemoteBackupPath.java @@ -16,11 +16,13 @@ */ package com.netflix.priam.aws; +import com.google.api.client.util.Lists; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; import com.google.inject.Inject; import com.netflix.priam.backup.AbstractBackupPath; +import com.netflix.priam.compress.CompressionType; import com.netflix.priam.config.IConfiguration; import com.netflix.priam.identity.InstanceIdentity; import com.netflix.priam.utils.DateUtil; @@ -29,6 +31,7 @@ import java.time.Instant; import java.util.Arrays; import java.util.Date; +import java.util.List; import java.util.Optional; /** @@ -102,16 +105,23 @@ private void parseV2Location(Path remotePath) { type = BackupFileType.valueOf(remotePath.getName(index++).toString()); String lastModified = remotePath.getName(index++).toString(); setLastModified(Instant.ofEpochMilli(Long.parseLong(lastModified))); + List parts = Lists.newArrayListWithCapacity(4); if (BackupFileType.isDataFile(type)) { keyspace = remotePath.getName(index++).toString(); columnFamily = remotePath.getName(index++).toString(); + parts.add(keyspace); + parts.add(columnFamily); } if (type == BackupFileType.SECONDARY_INDEX_V2) { indexDir = remotePath.getName(index++).toString(); + parts.add(indexDir); } - setCompression(remotePath.getName(index++).toString()); + setCompression(CompressionType.valueOf(remotePath.getName(index++).toString())); setEncryption(remotePath.getName(index++).toString()); fileName = remotePath.getName(index).toString(); + parts.add(fileName); + this.backupFile = + Paths.get(config.getDataFileLocation(), parts.toArray(new String[] {})).toFile(); } private String getV1Location() { diff --git a/priam/src/main/java/com/netflix/priam/aws/S3EncryptedFileSystem.java b/priam/src/main/java/com/netflix/priam/aws/S3EncryptedFileSystem.java index 60dc33f57..9012af932 100755 --- a/priam/src/main/java/com/netflix/priam/aws/S3EncryptedFileSystem.java +++ b/priam/src/main/java/com/netflix/priam/aws/S3EncryptedFileSystem.java @@ -26,6 +26,7 @@ import com.netflix.priam.backup.AbstractBackupPath; import com.netflix.priam.backup.BackupRestoreException; import com.netflix.priam.backup.RangeReadInputStream; +import com.netflix.priam.compress.ChunkedStream; import com.netflix.priam.compress.ICompression; import com.netflix.priam.config.IConfiguration; import com.netflix.priam.cred.ICredential; @@ -35,6 +36,7 @@ import com.netflix.priam.notification.BackupNotificationMgr; import java.io.*; import java.nio.file.Path; +import java.nio.file.Paths; import java.util.Iterator; import java.util.List; import org.apache.commons.io.IOUtils; @@ -69,14 +71,14 @@ public S3EncryptedFileSystem( } @Override - protected void downloadFileImpl(Path remotePath, Path localPath) throws BackupRestoreException { + protected void downloadFileImpl(AbstractBackupPath path, String suffix) + throws BackupRestoreException { + String remotePath = path.getRemotePath(); + Path localPath = Paths.get(path.newRestoreFile().getAbsolutePath() + suffix); try (OutputStream os = new FileOutputStream(localPath.toFile()); RangeReadInputStream rris = new RangeReadInputStream( - s3Client, - getShard(), - super.getFileSize(remotePath), - remotePath.toString())) { + s3Client, getShard(), super.getFileSize(remotePath), remotePath)) { /* * To handle use cases where decompression should be done outside of the download. For example, the file have been compressed and then encrypted. * Hence, decompressing it here would compromise the decryption. @@ -95,18 +97,18 @@ protected void downloadFileImpl(Path remotePath, Path localPath) throws BackupRe } @Override - protected long uploadFileImpl(Path localPath, Path remotePath) throws BackupRestoreException { + protected long uploadFileImpl(AbstractBackupPath path) throws BackupRestoreException { + Path localPath = Paths.get(path.getBackupFile().getAbsolutePath()); + String remotePath = path.getRemotePath(); + long chunkSize = getChunkSize(localPath); // initialize chunking request to aws InitiateMultipartUploadRequest initRequest = - new InitiateMultipartUploadRequest(config.getBackupPrefix(), remotePath.toString()); + new InitiateMultipartUploadRequest(config.getBackupPrefix(), remotePath); // Fetch the aws generated upload id for this chunking request InitiateMultipartUploadResult initResponse = s3Client.initiateMultipartUpload(initRequest); DataPart part = - new DataPart( - config.getBackupPrefix(), - remotePath.toString(), - initResponse.getUploadId()); + new DataPart(config.getBackupPrefix(), remotePath, initResponse.getUploadId()); // Metadata on number of parts to be uploaded List partETags = Lists.newArrayList(); @@ -121,7 +123,8 @@ protected long uploadFileImpl(Path localPath, Path remotePath) throws BackupRest try (InputStream in = new FileInputStream(localPath.toFile()); BufferedOutputStream compressedBos = new BufferedOutputStream(new FileOutputStream(compressedDstFile))) { - Iterator compressedChunks = this.compress.compress(in, chunkSize); + Iterator compressedChunks = + new ChunkedStream(in, chunkSize, path.getCompression()); while (compressedChunks.hasNext()) { byte[] compressedChunk = compressedChunks.next(); compressedBos.write(compressedChunk); @@ -137,8 +140,7 @@ protected long uploadFileImpl(Path localPath, Path remotePath) throws BackupRest // == Read compressed data, encrypt each chunk, upload it to aws try (BufferedInputStream compressedBis = new BufferedInputStream(new FileInputStream(compressedDstFile))) { - Iterator chunks = - this.encryptor.encryptStream(compressedBis, remotePath.toString()); + Iterator chunks = this.encryptor.encryptStream(compressedBis, remotePath); // identifies this part position in the object we are uploading int partNum = 0; @@ -154,7 +156,7 @@ protected long uploadFileImpl(Path localPath, Path remotePath) throws BackupRest ++partNum, chunk, config.getBackupPrefix(), - remotePath.toString(), + remotePath, initResponse.getUploadId()); S3PartUploader partUploader = new S3PartUploader(s3Client, dp, partETags); encryptedFileSize += chunk.length; diff --git a/priam/src/main/java/com/netflix/priam/aws/S3FileSystem.java b/priam/src/main/java/com/netflix/priam/aws/S3FileSystem.java index bdb05bc75..b175ef56b 100644 --- a/priam/src/main/java/com/netflix/priam/aws/S3FileSystem.java +++ b/priam/src/main/java/com/netflix/priam/aws/S3FileSystem.java @@ -19,6 +19,7 @@ import com.amazonaws.services.s3.AmazonS3Client; import com.amazonaws.services.s3.S3ResponseMetadata; import com.amazonaws.services.s3.model.*; +import com.google.common.base.Preconditions; import com.google.inject.Inject; import com.google.inject.Provider; import com.google.inject.Singleton; @@ -27,6 +28,8 @@ import com.netflix.priam.backup.AbstractBackupPath; import com.netflix.priam.backup.BackupRestoreException; import com.netflix.priam.backup.RangeReadInputStream; +import com.netflix.priam.compress.ChunkedStream; +import com.netflix.priam.compress.CompressionType; import com.netflix.priam.compress.ICompression; import com.netflix.priam.config.IConfiguration; import com.netflix.priam.identity.config.InstanceInfo; @@ -35,12 +38,13 @@ import com.netflix.priam.utils.BoundedExponentialRetryCallable; import java.io.*; import java.nio.file.Path; +import java.nio.file.Paths; import java.util.ArrayList; import java.util.Collections; import java.util.Iterator; import java.util.List; -import java.util.concurrent.Future; import java.util.concurrent.atomic.AtomicInteger; +import org.apache.commons.io.IOUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -48,6 +52,7 @@ @Singleton public class S3FileSystem extends S3FileSystemBase { private static final Logger logger = LoggerFactory.getLogger(S3FileSystem.class); + private static final int MAX_BUFFER_SIZE = 5 * 1024 * 1024; @Inject public S3FileSystem( @@ -67,122 +72,88 @@ public S3FileSystem( } @Override - protected void downloadFileImpl(Path remotePath, Path localPath) throws BackupRestoreException { - try { - long remoteFileSize = super.getFileSize(remotePath); - RangeReadInputStream rris = - new RangeReadInputStream( - s3Client, getShard(), remoteFileSize, remotePath.toString()); - final long bufSize = - MAX_BUFFERED_IN_STREAM_SIZE > remoteFileSize - ? remoteFileSize - : MAX_BUFFERED_IN_STREAM_SIZE; - compress.decompressAndClose( - new BufferedInputStream(rris, (int) bufSize), - new BufferedOutputStream(new FileOutputStream(localPath.toFile()))); + protected void downloadFileImpl(AbstractBackupPath path, String suffix) + throws BackupRestoreException { + String remotePath = path.getRemotePath(); + File localFile = new File(path.newRestoreFile().getAbsolutePath() + suffix); + long size = super.getFileSize(remotePath); + final int bufferSize = Math.min(MAX_BUFFER_SIZE, Math.toIntExact(size)); + try (BufferedInputStream is = + new BufferedInputStream( + new RangeReadInputStream(s3Client, getShard(), size, remotePath), + bufferSize); + BufferedOutputStream os = + new BufferedOutputStream(new FileOutputStream(localFile))) { + if (path.getCompression() == CompressionType.NONE) { + IOUtils.copyLarge(is, os); + } else { + compress.decompressAndClose(is, os); + } } catch (Exception e) { - throw new BackupRestoreException( - "Exception encountered downloading " - + remotePath - + " from S3 bucket " - + getShard() - + ", Msg: " - + e.getMessage(), - e); + String err = + String.format( + "Failed to GET %s Bucket: %s Msg: %s", + remotePath, getShard(), e.getMessage()); + throw new BackupRestoreException(err); } } - private ObjectMetadata getObjectMetadata(Path path) { + private ObjectMetadata getObjectMetadata(File file) { ObjectMetadata ret = new ObjectMetadata(); - long lastModified = path.toFile().lastModified(); + long lastModified = file.lastModified(); if (lastModified != 0) { ret.addUserMetadata("local-modification-time", Long.toString(lastModified)); } - long fileSize = path.toFile().length(); + long fileSize = file.length(); if (fileSize != 0) { ret.addUserMetadata("local-size", Long.toString(fileSize)); } return ret; } - private long uploadMultipart(Path localPath, Path remotePath) throws BackupRestoreException { + private long uploadMultipart(AbstractBackupPath path) throws BackupRestoreException { + Path localPath = Paths.get(path.getBackupFile().getAbsolutePath()); + String remotePath = path.getRemotePath(); long chunkSize = getChunkSize(localPath); + String prefix = config.getBackupPrefix(); if (logger.isDebugEnabled()) - logger.debug( - "Uploading to {}/{} with chunk size {}", - config.getBackupPrefix(), - remotePath, - chunkSize); + logger.debug("Uploading to {}/{} with chunk size {}", prefix, remotePath, chunkSize); + File localFile = localPath.toFile(); InitiateMultipartUploadRequest initRequest = - new InitiateMultipartUploadRequest(config.getBackupPrefix(), remotePath.toString()); - initRequest.withObjectMetadata(getObjectMetadata(localPath)); - InitiateMultipartUploadResult initResponse = s3Client.initiateMultipartUpload(initRequest); - DataPart part = - new DataPart( - config.getBackupPrefix(), - remotePath.toString(), - initResponse.getUploadId()); - List partETags = Collections.synchronizedList(new ArrayList()); - - try (InputStream in = new FileInputStream(localPath.toFile())) { - Iterator chunks = compress.compress(in, chunkSize); - // Upload parts. + new InitiateMultipartUploadRequest(prefix, remotePath) + .withObjectMetadata(getObjectMetadata(localFile)); + String uploadId = s3Client.initiateMultipartUpload(initRequest).getUploadId(); + DataPart part = new DataPart(prefix, remotePath, uploadId); + List partETags = Collections.synchronizedList(new ArrayList<>()); + + try (InputStream in = new FileInputStream(localFile)) { + Iterator chunks = new ChunkedStream(in, chunkSize, path.getCompression()); int partNum = 0; - AtomicInteger partsUploaded = new AtomicInteger(0); + AtomicInteger partsPut = new AtomicInteger(0); long compressedFileSize = 0; while (chunks.hasNext()) { byte[] chunk = chunks.next(); rateLimiter.acquire(chunk.length); - DataPart dp = - new DataPart( - ++partNum, - chunk, - config.getBackupPrefix(), - remotePath.toString(), - initResponse.getUploadId()); - S3PartUploader partUploader = - new S3PartUploader(s3Client, dp, partETags, partsUploaded); + DataPart dp = new DataPart(++partNum, chunk, prefix, remotePath, uploadId); + S3PartUploader partUploader = new S3PartUploader(s3Client, dp, partETags, partsPut); compressedFileSize += chunk.length; - // TODO: Get the future over here and create a new arraylist. - Future future = executor.submit(partUploader); + // TODO: output Future instead, collect them here, wait for all below + executor.submit(partUploader); } - // TODO: Instead of waiting for executor thread to be empty we should wait for all the - // futures to finish. executor.sleepTillEmpty(); - logger.info( - "All chunks uploaded for file {}, num of expected parts:{}, num of actual uploaded parts: {}", - localPath.toFile().getName(), - partNum, - partsUploaded.get()); - - if (partNum != partETags.size()) - throw new BackupRestoreException( - "Number of parts(" - + partNum - + ") does not match the uploaded parts(" - + partETags.size() - + ")"); - + logger.info("{} done. part count: {} expected: {}", localFile, partsPut.get(), partNum); + Preconditions.checkState(partNum == partETags.size(), "part count mismatch"); CompleteMultipartUploadResult resultS3MultiPartUploadComplete = new S3PartUploader(s3Client, part, partETags).completeUpload(); checkSuccessfulUpload(resultS3MultiPartUploadComplete, localPath); if (logger.isDebugEnabled()) { - final S3ResponseMetadata responseMetadata = - s3Client.getCachedResponseMetadata(initRequest); - final String requestId = - responseMetadata.getRequestId(); // "x-amz-request-id" header - final String hostId = responseMetadata.getHostId(); // "x-amz-id-2" header - logger.debug( - "S3 AWS x-amz-request-id[" - + requestId - + "], and x-amz-id-2[" - + hostId - + "]"); + final S3ResponseMetadata info = s3Client.getCachedResponseMetadata(initRequest); + logger.debug("Request Id: {}, Host Id: {}", info.getRequestId(), info.getHostId()); } return compressedFileSize; @@ -192,61 +163,43 @@ private long uploadMultipart(Path localPath, Path remotePath) throws BackupResto } } - protected long uploadFileImpl(Path localPath, Path remotePath) throws BackupRestoreException { + protected long uploadFileImpl(AbstractBackupPath path) throws BackupRestoreException { + Path localPath = Paths.get(path.getBackupFile().getAbsolutePath()); + String remotePath = path.getRemotePath(); long chunkSize = config.getBackupChunkSize(); - long fileSize = localPath.toFile().length(); + File localFile = localPath.toFile(); + if (localFile.length() >= chunkSize) return uploadMultipart(path); - if (fileSize < chunkSize) { - // Upload file without using multipart upload as it will be more efficient. - if (logger.isDebugEnabled()) - logger.debug( - "Uploading to {}/{} using PUT operation", - config.getBackupPrefix(), - remotePath); - - try (ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); - InputStream in = - new BufferedInputStream(new FileInputStream(localPath.toFile()))) { - Iterator chunkedStream = compress.compress(in, chunkSize); - while (chunkedStream.hasNext()) { - byteArrayOutputStream.write(chunkedStream.next()); - } - byte[] chunk = byteArrayOutputStream.toByteArray(); - long compressedFileSize = chunk.length; - /** - * Weird, right that we are checking for length which is positive. You can thanks - * this to sometimes C* creating files which are zero bytes, and giving that in - * snapshot for some unknown reason. - */ - if (chunk.length > 0) rateLimiter.acquire(chunk.length); - ObjectMetadata objectMetadata = getObjectMetadata(localPath); - objectMetadata.setContentLength(chunk.length); - PutObjectRequest putObjectRequest = - new PutObjectRequest( - config.getBackupPrefix(), - remotePath.toString(), - new ByteArrayInputStream(chunk), - objectMetadata); - // Retry if failed. - PutObjectResult upload = - new BoundedExponentialRetryCallable(1000, 10000, 5) { - @Override - public PutObjectResult retriableCall() throws Exception { - return s3Client.putObject(putObjectRequest); - } - }.call(); - - if (logger.isDebugEnabled()) - logger.debug( - "Successfully uploaded file with putObject: {} and etag: {}", - remotePath, - upload.getETag()); - - return compressedFileSize; - } catch (Exception e) { - throw new BackupRestoreException( - "Error uploading file: " + localPath.toFile().getName(), e); + String prefix = config.getBackupPrefix(); + if (logger.isDebugEnabled()) logger.debug("PUTing {}/{}", prefix, remotePath); + + try (ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); + InputStream in = new BufferedInputStream(new FileInputStream(localFile))) { + Iterator chunks = new ChunkedStream(in, chunkSize, path.getCompression()); + while (chunks.hasNext()) { + byteArrayOutputStream.write(chunks.next()); } - } else return uploadMultipart(localPath, remotePath); + byte[] chunk = byteArrayOutputStream.toByteArray(); + long compressedFileSize = chunk.length; + // C* snapshots may have empty files. That is probably unintentional. + if (chunk.length > 0) rateLimiter.acquire(chunk.length); + ObjectMetadata objectMetadata = getObjectMetadata(localFile); + objectMetadata.setContentLength(chunk.length); + ByteArrayInputStream inputStream = new ByteArrayInputStream(chunk); + PutObjectRequest putObjectRequest = + new PutObjectRequest(prefix, remotePath, inputStream, objectMetadata); + PutObjectResult upload = + new BoundedExponentialRetryCallable(1000, 10000, 5) { + @Override + public PutObjectResult retriableCall() { + return s3Client.putObject(putObjectRequest); + } + }.call(); + if (logger.isDebugEnabled()) + logger.debug("Put: {} with etag: {}", remotePath, upload.getETag()); + return compressedFileSize; + } catch (Exception e) { + throw new BackupRestoreException("Error uploading file: " + localFile.getName(), e); + } } } diff --git a/priam/src/main/java/com/netflix/priam/aws/S3FileSystemBase.java b/priam/src/main/java/com/netflix/priam/aws/S3FileSystemBase.java index 4df59bfa6..3fe07736a 100755 --- a/priam/src/main/java/com/netflix/priam/aws/S3FileSystemBase.java +++ b/priam/src/main/java/com/netflix/priam/aws/S3FileSystemBase.java @@ -45,8 +45,7 @@ import org.slf4j.LoggerFactory; public abstract class S3FileSystemBase extends AbstractFileSystem { - private static final int MAX_CHUNKS = 10000; - static final long MAX_BUFFERED_IN_STREAM_SIZE = 5 * 1024 * 1024; + private static final int MAX_CHUNKS = 9995; // 10K is AWS limit, minus a small buffer private static final Logger logger = LoggerFactory.getLogger(S3FileSystemBase.class); AmazonS3 s3Client; final IConfiguration config; @@ -226,8 +225,8 @@ void checkSuccessfulUpload( } @Override - public long getFileSize(Path remotePath) throws BackupRestoreException { - return s3Client.getObjectMetadata(getShard(), remotePath.toString()).getContentLength(); + public long getFileSize(String remotePath) throws BackupRestoreException { + return s3Client.getObjectMetadata(getShard(), remotePath).getContentLength(); } @Override @@ -282,10 +281,7 @@ public void deleteFiles(List remotePaths) throws BackupRestoreException { } } - final long getChunkSize(Path localPath) { - long chunkSize = config.getBackupChunkSize(); - long proposedChunkSize = localPath.toFile().length() / (MAX_CHUNKS - 5); - if (proposedChunkSize > chunkSize) return proposedChunkSize; - return chunkSize; + final long getChunkSize(Path path) { + return Math.max(path.toFile().length() / MAX_CHUNKS, config.getBackupChunkSize()); } } diff --git a/priam/src/main/java/com/netflix/priam/backup/AbstractBackup.java b/priam/src/main/java/com/netflix/priam/backup/AbstractBackup.java index 63e9b907f..15d4f00a9 100644 --- a/priam/src/main/java/com/netflix/priam/backup/AbstractBackup.java +++ b/priam/src/main/java/com/netflix/priam/backup/AbstractBackup.java @@ -16,19 +16,24 @@ */ package com.netflix.priam.backup; +import static java.util.stream.Collectors.toSet; + +import com.google.common.collect.ImmutableSet; import com.google.common.collect.Lists; import com.google.inject.Inject; import com.google.inject.Provider; import com.netflix.priam.backup.AbstractBackupPath.BackupFileType; +import com.netflix.priam.compress.CompressionType; +import com.netflix.priam.config.BackupsToCompress; import com.netflix.priam.config.IConfiguration; import com.netflix.priam.scheduler.Task; import com.netflix.priam.utils.SystemUtils; import java.io.File; +import java.io.IOException; import java.nio.file.DirectoryStream; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; -import java.text.ParseException; import java.util.HashSet; import java.util.List; import java.util.Set; @@ -39,6 +44,7 @@ /** Abstract Backup class for uploading files to backup location */ public abstract class AbstractBackup extends Task { private static final Logger logger = LoggerFactory.getLogger(AbstractBackup.class); + private static final String COMPRESSION_SUFFIX = "-CompressionInfo.db"; static final String INCREMENTAL_BACKUP_FOLDER = "backups"; public static final String SNAPSHOT_FOLDER = "snapshots"; @@ -56,18 +62,6 @@ public AbstractBackup( this.fs = backupFileSystemCtx.getFileStrategy(config); } - /** A means to override the type of backup strategy chosen via BackupFileSystemContext */ - protected void setFileSystem(IBackupFileSystem fs) { - this.fs = fs; - } - - private AbstractBackupPath getAbstractBackupPath(final File file, final BackupFileType type) - throws ParseException { - final AbstractBackupPath bp = pathFactory.get(); - bp.parseLocal(file, type); - return bp; - } - /** * Upload files in the specified dir. Does not delete the file in case of error. The files are * uploaded serially or async based on flag provided. @@ -80,48 +74,67 @@ private AbstractBackupPath getAbstractBackupPath(final File file, final BackupFi * @return List of files that are successfully uploaded as part of backup * @throws Exception when there is failure in uploading files. */ - protected List upload( + protected ImmutableSet upload( final File parent, final BackupFileType type, boolean async, boolean waitForCompletion) throws Exception { - final List bps = Lists.newArrayList(); - final List> futures = Lists.newArrayList(); - - File[] files = parent.listFiles(); - if (files == null) return bps; - - for (File file : files) { - if (file.isFile() && file.exists()) { - AbstractBackupPath bp = getAbstractBackupPath(file, type); - - if (async) - futures.add( - fs.asyncUploadFile( - Paths.get(bp.getBackupFile().getAbsolutePath()), - Paths.get(bp.getRemotePath()), - bp, - 10, - true)); - else - fs.uploadFile( - Paths.get(bp.getBackupFile().getAbsolutePath()), - Paths.get(bp.getRemotePath()), - bp, - 10, - true); - - bps.add(bp); - } + ImmutableSet bps = getBackupPaths(parent, type); + final List> futures = Lists.newArrayList(); + for (AbstractBackupPath bp : bps) { + if (async) futures.add(fs.asyncUploadAndDelete(bp, 10)); + else fs.uploadAndDelete(bp, 10); } // Wait for all files to be uploaded. if (async && waitForCompletion) { - for (Future future : futures) + for (Future future : futures) future.get(); // This might throw exception if there is any error } return bps; } + protected ImmutableSet getBackupPaths(File dir, BackupFileType type) + throws IOException { + Set files = + Files.list(dir.toPath()).map(Path::toFile).filter(File::isFile).collect(toSet()); + Set compressedFilePrefixes = + files.stream() + .map(File::getName) + .filter(name -> name.endsWith(COMPRESSION_SUFFIX)) + .map(name -> name.substring(0, name.lastIndexOf('-'))) + .collect(toSet()); + final ImmutableSet.Builder bps = ImmutableSet.builder(); + for (File file : files) { + final AbstractBackupPath bp = pathFactory.get(); + bp.parseLocal(file, type); + bp.setCompression(getCorrectCompressionAlgorithm(bp, compressedFilePrefixes)); + bps.add(bp); + } + return bps.build(); + } + + private CompressionType getCorrectCompressionAlgorithm( + AbstractBackupPath path, Set compressedFiles) { + if (!BackupFileType.isV2(path.getType())) { + return CompressionType.SNAPPY; + } + String file = path.getFileName(); + BackupsToCompress which = config.getBackupsToCompress(); + switch (which) { + case NONE: + return CompressionType.NONE; + case ALL: + return CompressionType.SNAPPY; + case IF_REQUIRED: + int splitIndex = file.lastIndexOf('-'); + return splitIndex >= 0 && compressedFiles.contains(file.substring(0, splitIndex)) + ? CompressionType.NONE + : CompressionType.SNAPPY; + default: + throw new IllegalArgumentException("NONE, ALL, UNCOMPRESSED only. Saw: " + which); + } + } + protected final void initiateBackup( String monitoringFolder, BackupRestoreUtil backupRestoreUtil) throws Exception { diff --git a/priam/src/main/java/com/netflix/priam/backup/AbstractBackupPath.java b/priam/src/main/java/com/netflix/priam/backup/AbstractBackupPath.java index d02a0502d..3d4e77c14 100644 --- a/priam/src/main/java/com/netflix/priam/backup/AbstractBackupPath.java +++ b/priam/src/main/java/com/netflix/priam/backup/AbstractBackupPath.java @@ -16,19 +16,23 @@ */ package com.netflix.priam.backup; +import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Joiner; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import com.google.inject.ImplementedBy; import com.netflix.priam.aws.RemoteBackupPath; -import com.netflix.priam.compress.CompressionAlgorithm; +import com.netflix.priam.compress.CompressionType; import com.netflix.priam.config.IConfiguration; import com.netflix.priam.cryptography.CryptographyAlgorithm; import com.netflix.priam.identity.InstanceIdentity; import com.netflix.priam.utils.DateUtil; import java.io.File; +import java.io.IOException; +import java.nio.file.Files; import java.nio.file.Path; +import java.nio.file.attribute.BasicFileAttributes; import java.time.Instant; import java.util.Date; import org.apache.commons.lang3.StringUtils; @@ -53,10 +57,17 @@ public enum BackupFileType { private static ImmutableSet DATA_FILE_TYPES = ImmutableSet.of(SECONDARY_INDEX_V2, SNAP, SST, SST_V2); + private static ImmutableSet V2_FILE_TYPES = + ImmutableSet.of(SECONDARY_INDEX_V2, SST_V2, META_V2); + public static boolean isDataFile(BackupFileType type) { return DATA_FILE_TYPES.contains(type); } + public static boolean isV2(BackupFileType type) { + return V2_FILE_TYPES.contains(type); + } + public static BackupFileType fromString(String s) throws BackupRestoreException { try { return BackupFileType.valueOf(s); @@ -80,10 +91,11 @@ public static BackupFileType fromString(String s) throws BackupRestoreException private long compressedFileSize = 0; protected final InstanceIdentity instanceIdentity; protected final IConfiguration config; - private File backupFile; + protected File backupFile; private Instant lastModified; + private Instant creationTime; private Date uploadedTs; - private CompressionAlgorithm compression = CompressionAlgorithm.SNAPPY; + private CompressionType compression = CompressionType.SNAPPY; private CryptographyAlgorithm encryption = CryptographyAlgorithm.PLAINTEXT; public AbstractBackupPath(IConfiguration config, InstanceIdentity instanceIdentity) { @@ -96,9 +108,18 @@ public void parseLocal(File file, BackupFileType type) { this.baseDir = config.getBackupLocation(); this.clusterName = config.getAppName(); this.fileName = file.getName(); - this.lastModified = Instant.ofEpochMilli(file.lastModified()); + BasicFileAttributes fileAttributes; + try { + fileAttributes = Files.readAttributes(file.toPath(), BasicFileAttributes.class); + this.lastModified = fileAttributes.lastModifiedTime().toInstant(); + this.creationTime = fileAttributes.creationTime().toInstant(); + this.size = fileAttributes.size(); + } catch (IOException e) { + this.lastModified = Instant.ofEpochMilli(0L); + this.creationTime = Instant.ofEpochMilli(0L); + this.size = 0L; + } this.region = instanceIdentity.getInstanceInfo().getRegion(); - this.size = file.length(); this.token = instanceIdentity.getInstance().getToken(); this.type = type; @@ -123,7 +144,7 @@ public void parseLocal(File file, BackupFileType type) { this.time = type == BackupFileType.SNAP ? DateUtil.getDate(parts[3]) - : new Date(file.lastModified()); + : new Date(lastModified.toEpochMilli()); } /** Given a date range, find a common string prefix Eg: 20120212, 20120213 = 2012021 */ @@ -277,12 +298,21 @@ public void setLastModified(Instant instant) { this.lastModified = instant; } - public CompressionAlgorithm getCompression() { + public Instant getCreationTime() { + return creationTime; + } + + @VisibleForTesting + public void setCreationTime(Instant instant) { + this.creationTime = instant; + } + + public CompressionType getCompression() { return compression; } - public void setCompression(String compression) { - this.compression = CompressionAlgorithm.valueOf(compression); + public void setCompression(CompressionType compressionType) { + this.compression = compressionType; } public CryptographyAlgorithm getEncryption() { diff --git a/priam/src/main/java/com/netflix/priam/backup/AbstractFileSystem.java b/priam/src/main/java/com/netflix/priam/backup/AbstractFileSystem.java index 1646b06a5..2ac1df0db 100644 --- a/priam/src/main/java/com/netflix/priam/backup/AbstractFileSystem.java +++ b/priam/src/main/java/com/netflix/priam/backup/AbstractFileSystem.java @@ -17,6 +17,7 @@ package com.netflix.priam.backup; +import com.google.common.base.Preconditions; import com.google.common.cache.Cache; import com.google.common.cache.CacheBuilder; import com.google.inject.Inject; @@ -32,7 +33,6 @@ import com.netflix.priam.utils.BoundedExponentialRetryCallable; import com.netflix.spectator.api.patterns.PolledMeter; import java.io.File; -import java.io.FileNotFoundException; import java.nio.file.Path; import java.nio.file.Paths; import java.util.Date; @@ -111,28 +111,27 @@ Also, we may want to have different TIMEOUT for each kind of operation (upload/d } @Override - public Future asyncDownloadFile( - final Path remotePath, final Path localPath, final int retry) - throws BackupRestoreException, RejectedExecutionException { + public Future asyncDownloadFile(final AbstractBackupPath path, final int retry) + throws RejectedExecutionException { return fileDownloadExecutor.submit( () -> { - downloadFile(remotePath, localPath, retry); - return remotePath; + downloadFile(path, "" /* suffix */, retry); + return Paths.get(path.getRemotePath()); }); } @Override - public void downloadFile(final Path remotePath, final Path localPath, final int retry) + public void downloadFile(final AbstractBackupPath path, String suffix, final int retry) throws BackupRestoreException { // TODO: Should we download the file if localPath already exists? - if (remotePath == null || localPath == null) return; - localPath.toFile().getParentFile().mkdirs(); - logger.info("Downloading file: {} to location: {}", remotePath, localPath); + String remotePath = path.getRemotePath(); + String localPath = path.newRestoreFile().getAbsolutePath() + suffix; + logger.info("Downloading file: {} to location: {}", path.getRemotePath(), localPath); try { new BoundedExponentialRetryCallable(500, 10000, retry) { @Override public Void retriableCall() throws Exception { - downloadFileImpl(remotePath, localPath); + downloadFileImpl(path, suffix); return null; } }.call(); @@ -149,41 +148,28 @@ public Void retriableCall() throws Exception { } } - protected abstract void downloadFileImpl(final Path remotePath, final Path localPath) + protected abstract void downloadFileImpl(final AbstractBackupPath path, String suffix) throws BackupRestoreException; @Override - public Future asyncUploadFile( - final Path localPath, - final Path remotePath, - final AbstractBackupPath path, - final int retry, - final boolean deleteAfterSuccessfulUpload) - throws FileNotFoundException, RejectedExecutionException, BackupRestoreException { + public Future asyncUploadAndDelete( + final AbstractBackupPath path, final int retry) throws RejectedExecutionException { return fileUploadExecutor.submit( () -> { - uploadFile(localPath, remotePath, path, retry, deleteAfterSuccessfulUpload); - return localPath; + uploadAndDelete(path, retry); + return path; }); } @Override - public void uploadFile( - final Path localPath, - final Path remotePath, - final AbstractBackupPath path, - final int retry, - final boolean deleteAfterSuccessfulUpload) - throws FileNotFoundException, BackupRestoreException { - if (localPath == null - || remotePath == null - || !localPath.toFile().exists() - || localPath.toFile().isDirectory()) - throw new FileNotFoundException( - "File do not exist or is a directory. localPath: " - + localPath - + ", remotePath: " - + remotePath); + public void uploadAndDelete(final AbstractBackupPath path, final int retry) + throws BackupRestoreException { + Path localPath = Paths.get(path.getBackupFile().getAbsolutePath()); + File localFile = localPath.toFile(); + Preconditions.checkArgument(localFile.exists(), "Can't upload nonexistent {}", localPath); + Preconditions.checkArgument( + !localFile.isDirectory(), "Can only upload files {} is a directory", localPath); + Path remotePath = Paths.get(path.getRemotePath()); if (tasksQueued.add(localPath)) { logger.info("Uploading file: {} to location: {}", localPath, remotePath); @@ -197,7 +183,7 @@ public void uploadFile( new BoundedExponentialRetryCallable(500, 10000, retry) { @Override public Long retriableCall() throws Exception { - return uploadFileImpl(localPath, remotePath); + return uploadFileImpl(path); } }.call(); @@ -218,11 +204,11 @@ public Long retriableCall() throws Exception { logger.info( "Successfully uploaded file: {} to location: {}", localPath, remotePath); - if (deleteAfterSuccessfulUpload && !FileUtils.deleteQuietly(localPath.toFile())) + if (!FileUtils.deleteQuietly(localFile)) logger.warn( String.format( "Failed to delete local file %s.", - localPath.toFile().getAbsolutePath())); + localFile.getAbsolutePath())); } catch (Exception e) { backupMetrics.incrementInvalidUploads(); @@ -278,7 +264,7 @@ public void deleteRemoteFiles(List remotePaths) throws BackupRestoreExcept protected abstract boolean doesRemoteFileExist(Path remotePath); - protected abstract long uploadFileImpl(final Path localPath, final Path remotePath) + protected abstract long uploadFileImpl(final AbstractBackupPath path) throws BackupRestoreException; @Override diff --git a/priam/src/main/java/com/netflix/priam/backup/CommitLogBackup.java b/priam/src/main/java/com/netflix/priam/backup/CommitLogBackup.java index c667d3a54..52ac0fc3b 100644 --- a/priam/src/main/java/com/netflix/priam/backup/CommitLogBackup.java +++ b/priam/src/main/java/com/netflix/priam/backup/CommitLogBackup.java @@ -20,7 +20,6 @@ import com.netflix.priam.backup.AbstractBackupPath.BackupFileType; import com.netflix.priam.utils.DateUtil; import java.io.File; -import java.nio.file.Paths; import java.util.List; import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; @@ -66,12 +65,7 @@ public List upload(String archivedDir, final String snapshot if (snapshotName != null) bp.time = DateUtil.getDate(snapshotName); - fs.uploadFile( - Paths.get(bp.getBackupFile().getAbsolutePath()), - Paths.get(bp.getRemotePath()), - bp, - 10, - true); + fs.uploadAndDelete(bp, 10); bps.add(bp); addToRemotePath(bp.getRemotePath()); } catch (Exception e) { diff --git a/priam/src/main/java/com/netflix/priam/backup/IBackupFileSystem.java b/priam/src/main/java/com/netflix/priam/backup/IBackupFileSystem.java index ef8935a05..269189545 100644 --- a/priam/src/main/java/com/netflix/priam/backup/IBackupFileSystem.java +++ b/priam/src/main/java/com/netflix/priam/backup/IBackupFileSystem.java @@ -29,21 +29,20 @@ public interface IBackupFileSystem { /** * Download the file denoted by remotePath to the local file system denoted by local path. * - * @param remotePath fully qualified location of the file on remote file system. - * @param localPath location on the local file sytem where remote file should be downloaded. + * @param path Backup path representing a local and remote file pair * @param retry No. of times to retry to download a file from remote file system. If <1, it * will try to download file exactly once. * @throws BackupRestoreException if file is not available, downloadable or any other error from * remote file system. */ - void downloadFile(Path remotePath, Path localPath, int retry) throws BackupRestoreException; + void downloadFile(AbstractBackupPath path, String suffix, int retry) + throws BackupRestoreException; /** * Download the file denoted by remotePath in an async fashion to the local file system denoted * by local path. * - * @param remotePath fully qualified location of the file on remote file system. - * @param localPath location on the local file sytem where remote file should be downloaded. + * @param path Backup path representing a local and remote file pair * @param retry No. of times to retry to download a file from remote file system. If <1, it * will try to download file exactly once. * @return The future of the async job to monitor the progress of the job. @@ -52,52 +51,35 @@ public interface IBackupFileSystem { * @throws RejectedExecutionException if the queue is full and TIMEOUT is reached while trying * to add the work to the queue. */ - Future asyncDownloadFile(final Path remotePath, final Path localPath, final int retry) + Future asyncDownloadFile(final AbstractBackupPath path, final int retry) throws BackupRestoreException, RejectedExecutionException; /** - * Upload the local file denoted by localPath to the remote file system at location denoted by - * remotePath. De-duping of the file to upload will always be done by comparing the + * Upload the local file to its remote counterpart. Both locations are embedded within the path + * parameter. De-duping of the file to upload will always be done by comparing the * files-in-progress to be uploaded. This may result in this particular request to not to be * executed e.g. if any other thread has given the same file to upload and that file is in * internal queue. Note that de-duping is best effort and is not always guaranteed as we try to - * avoid lock on read/write of the files-in-progress. + * avoid lock on read/write of the files-in-progress. Once uploaded, files are deleted. * - * @param localPath Path of the local file that needs to be uploaded. - * @param remotePath Fully qualified path on the remote file system where file should be - * uploaded. - * @param path AbstractBackupPath to be used to send backup notifications only. + * @param path Backup path representing a local and remote file pair * @param retry No of times to retry to upload a file. If <1, it will try to upload file * exactly once. - * @param deleteAfterSuccessfulUpload If true, delete the file denoted by localPath after it is - * successfully uploaded to the filesystem. If there is any failure, file will not be - * deleted. * @throws BackupRestoreException in case of failure to upload for any reason including file not * readable or remote file system errors. * @throws FileNotFoundException If a file as denoted by localPath is not available or is a * directory. */ - void uploadFile( - Path localPath, - Path remotePath, - AbstractBackupPath path, - int retry, - boolean deleteAfterSuccessfulUpload) + void uploadAndDelete(AbstractBackupPath path, int retry) throws FileNotFoundException, BackupRestoreException; /** * Upload the local file denoted by localPath in async fashion to the remote file system at * location denoted by remotePath. * - * @param localPath Path of the local file that needs to be uploaded. - * @param remotePath Fully qualified path on the remote file system where file should be - * uploaded. * @param path AbstractBackupPath to be used to send backup notifications only. * @param retry No of times to retry to upload a file. If <1, it will try to upload file * exactly once. - * @param deleteAfterSuccessfulUpload If true, delete the file denoted by localPath after it is - * successfully uploaded to the filesystem. If there is any failure, file will not be - * deleted. * @return The future of the async job to monitor the progress of the job. This will be null if * file was de-duped for upload. * @throws BackupRestoreException in case of failure to upload for any reason including file not @@ -107,12 +89,7 @@ void uploadFile( * @throws RejectedExecutionException if the queue is full and TIMEOUT is reached while trying * to add the work to the queue. */ - Future asyncUploadFile( - final Path localPath, - final Path remotePath, - final AbstractBackupPath path, - final int retry, - final boolean deleteAfterSuccessfulUpload) + Future asyncUploadAndDelete(final AbstractBackupPath path, final int retry) throws FileNotFoundException, RejectedExecutionException, BackupRestoreException; /** @@ -172,7 +149,7 @@ default String getShard() { * @throws BackupRestoreException in case of failure to read object denoted by remotePath or any * other error. */ - long getFileSize(Path remotePath) throws BackupRestoreException; + long getFileSize(String remotePath) throws BackupRestoreException; /** * Checks if the file denoted by remotePath exists on the remote file system. It does not need diff --git a/priam/src/main/java/com/netflix/priam/backup/MetaData.java b/priam/src/main/java/com/netflix/priam/backup/MetaData.java index be7a0bdab..f20c18a54 100644 --- a/priam/src/main/java/com/netflix/priam/backup/MetaData.java +++ b/priam/src/main/java/com/netflix/priam/backup/MetaData.java @@ -24,7 +24,6 @@ import java.io.File; import java.io.FileWriter; import java.io.IOException; -import java.nio.file.Paths; import java.text.ParseException; import java.util.ArrayList; import java.util.List; @@ -62,12 +61,7 @@ public AbstractBackupPath set(List bps, String snapshotName) fr.write(jsonObj.toJSONString()); } AbstractBackupPath backupfile = decorateMetaJson(metafile, snapshotName); - fs.uploadFile( - Paths.get(backupfile.getBackupFile().getAbsolutePath()), - Paths.get(backupfile.getRemotePath()), - backupfile, - 10, - true); + fs.uploadAndDelete(backupfile, 10); addToRemotePath(backupfile.getRemotePath()); return backupfile; } @@ -92,10 +86,7 @@ public AbstractBackupPath decorateMetaJson(File metafile, String snapshotName) */ public Boolean doesExist(final AbstractBackupPath meta) { try { - fs.downloadFile( - Paths.get(meta.getRemotePath()), - Paths.get(meta.newRestoreFile().getAbsolutePath()), - 5); // download actual file to disk + fs.downloadFile(meta, "" /* suffix */, 5 /* retries */); } catch (Exception e) { logger.error("Error downloading the Meta data try with a different date...", e); } diff --git a/priam/src/main/java/com/netflix/priam/backupv2/FileUploadResult.java b/priam/src/main/java/com/netflix/priam/backupv2/FileUploadResult.java index 26e866af0..f9c5da37d 100644 --- a/priam/src/main/java/com/netflix/priam/backupv2/FileUploadResult.java +++ b/priam/src/main/java/com/netflix/priam/backupv2/FileUploadResult.java @@ -16,9 +16,13 @@ */ package com.netflix.priam.backupv2; -import com.netflix.priam.compress.CompressionAlgorithm; +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Preconditions; +import com.netflix.priam.backup.AbstractBackupPath; +import com.netflix.priam.compress.CompressionType; import com.netflix.priam.cryptography.CryptographyAlgorithm; import com.netflix.priam.utils.GsonJsonSerializer; +import java.io.File; import java.nio.file.Path; import java.time.Instant; @@ -31,13 +35,14 @@ public class FileUploadResult { private final Instant fileCreationTime; private final long fileSizeOnDisk; // Size on disk in bytes // Valid compression technique for now is SNAPPY only. Future we need to support LZ4 and NONE - private final CompressionAlgorithm compression = CompressionAlgorithm.SNAPPY; + private final CompressionType compression; // Valid encryption technique for now is PLAINTEXT only. In future we will support pgp and more. - private final CryptographyAlgorithm encryption = CryptographyAlgorithm.PLAINTEXT; + private final CryptographyAlgorithm encryption; private Boolean isUploaded; private String backupPath; + @VisibleForTesting public FileUploadResult( Path fileName, Instant lastModifiedTime, @@ -47,6 +52,21 @@ public FileUploadResult( this.lastModifiedTime = lastModifiedTime; this.fileCreationTime = fileCreationTime; this.fileSizeOnDisk = fileSizeOnDisk; + this.compression = CompressionType.SNAPPY; + this.encryption = CryptographyAlgorithm.PLAINTEXT; + } + + public FileUploadResult(AbstractBackupPath path) { + Preconditions.checkArgument(path.getLastModified().toEpochMilli() > 0); + Preconditions.checkArgument(path.getCreationTime().toEpochMilli() > 0); + File file = path.getBackupFile(); + this.fileName = file.toPath(); + this.backupPath = path.getRemotePath(); + this.lastModifiedTime = path.getLastModified(); + this.fileCreationTime = path.getCreationTime(); + this.fileSizeOnDisk = path.getSize(); + this.compression = path.getCompression(); + this.encryption = path.getEncryption(); } public void setUploaded(Boolean uploaded) { diff --git a/priam/src/main/java/com/netflix/priam/backupv2/MetaFileWriterBuilder.java b/priam/src/main/java/com/netflix/priam/backupv2/MetaFileWriterBuilder.java index a1a4fb878..489d087f8 100644 --- a/priam/src/main/java/com/netflix/priam/backupv2/MetaFileWriterBuilder.java +++ b/priam/src/main/java/com/netflix/priam/backupv2/MetaFileWriterBuilder.java @@ -16,6 +16,9 @@ */ package com.netflix.priam.backupv2; +import com.google.common.collect.ImmutableCollection; +import com.google.common.collect.ImmutableMultimap; +import com.google.common.collect.ImmutableSet; import com.google.gson.stream.JsonWriter; import com.google.inject.Provider; import com.netflix.priam.backup.AbstractBackupPath; @@ -30,6 +33,7 @@ import java.time.Instant; import java.util.ArrayList; import java.util.List; +import java.util.stream.Collectors; import javax.inject.Inject; import javax.inject.Named; import org.slf4j.Logger; @@ -61,13 +65,17 @@ public interface StartStep { } public interface DataStep { - DataStep addColumnfamilyResult(ColumnfamilyResult columnfamilyResult) throws IOException; + DataStep addColumnfamilyResult( + String keyspace, + String columnFamily, + ImmutableMultimap sstables) + throws IOException; UploadStep endMetaFileGeneration() throws IOException; } public interface UploadStep { - void uploadMetaFile(boolean deleteOnSuccess) throws Exception; + void uploadMetaFile() throws Exception; Path getMetaFilePath(); @@ -132,18 +140,18 @@ public DataStep startMetaFileGeneration(Instant snapshotInstant) throws IOExcept * Add {@link ColumnfamilyResult} after it has been processed so it can be streamed to * meta.json. Streaming write to meta.json is required so we don't get Priam OOM. * - * @param columnfamilyResult a POJO encapsulating the column family result * @throws IOException if unable to write to the file or if JSON is not valid */ - public MetaFileWriterBuilder.DataStep addColumnfamilyResult( - ColumnfamilyResult columnfamilyResult) throws IOException { + public DataStep addColumnfamilyResult( + String keyspace, + String columnFamily, + ImmutableMultimap sstables) + throws IOException { + if (jsonWriter == null) throw new NullPointerException( "addColumnfamilyResult: Json Writer in MetaFileWriter is null. This should not happen!"); - if (columnfamilyResult == null) - throw new NullPointerException( - "Column family result is null in MetaFileWriter. This should not happen!"); - jsonWriter.jsonValue(columnfamilyResult.toString()); + jsonWriter.jsonValue(toColumnFamilyResult(keyspace, columnFamily, sstables).toString()); return this; } @@ -182,20 +190,13 @@ public MetaFileWriterBuilder.UploadStep endMetaFileGeneration() throws IOExcepti /** * Upload the meta file generated to backup file system. * - * @param deleteOnSuccess delete the meta file from local file system if backup is - * successful. Useful for testing purposes * @throws Exception when unable to upload the meta file. */ - public void uploadMetaFile(boolean deleteOnSuccess) throws Exception { + public void uploadMetaFile() throws Exception { AbstractBackupPath abstractBackupPath = pathFactory.get(); abstractBackupPath.parseLocal( metaFilePath.toFile(), AbstractBackupPath.BackupFileType.META_V2); - backupFileSystem.uploadFile( - metaFilePath, - Paths.get(getRemoteMetaFilePath()), - abstractBackupPath, - 10, - deleteOnSuccess); + backupFileSystem.uploadAndDelete(abstractBackupPath, 10); } public Path getMetaFilePath() { @@ -208,5 +209,40 @@ public String getRemoteMetaFilePath() throws Exception { metaFilePath.toFile(), AbstractBackupPath.BackupFileType.META_V2); return abstractBackupPath.getRemotePath(); } + + private ColumnfamilyResult toColumnFamilyResult( + String keyspace, + String columnFamily, + ImmutableMultimap sstables) { + ColumnfamilyResult columnfamilyResult = new ColumnfamilyResult(keyspace, columnFamily); + sstables.keySet() + .stream() + .map(k -> toSSTableResult(k, sstables.get(k))) + .forEach(columnfamilyResult::addSstable); + return columnfamilyResult; + } + + private ColumnfamilyResult.SSTableResult toSSTableResult( + String prefix, ImmutableCollection sstable) { + ColumnfamilyResult.SSTableResult ssTableResult = new ColumnfamilyResult.SSTableResult(); + ssTableResult.setPrefix(prefix); + ssTableResult.setSstableComponents( + ImmutableSet.copyOf( + sstable.stream() + .map(this::toFileUploadResult) + .collect(Collectors.toSet()))); + return ssTableResult; + } + + private FileUploadResult toFileUploadResult(AbstractBackupPath path) { + FileUploadResult fileUploadResult = new FileUploadResult(path); + try { + Path backupPath = Paths.get(fileUploadResult.getBackupPath()); + fileUploadResult.setUploaded(backupFileSystem.checkObjectExists(backupPath)); + } catch (Exception e) { + logger.error("Error checking if file exists. Ignoring as it is not fatal.", e); + } + return fileUploadResult; + } } } diff --git a/priam/src/main/java/com/netflix/priam/backupv2/MetaV1Proxy.java b/priam/src/main/java/com/netflix/priam/backupv2/MetaV1Proxy.java index cfce6ae81..3569b7345 100644 --- a/priam/src/main/java/com/netflix/priam/backupv2/MetaV1Proxy.java +++ b/priam/src/main/java/com/netflix/priam/backupv2/MetaV1Proxy.java @@ -149,9 +149,8 @@ public BackupVerificationResult isMetaFileValid(AbstractBackupPath metaBackupPat @Override public Path downloadMetaFile(AbstractBackupPath meta) throws BackupRestoreException { - Path localFile = Paths.get(meta.newRestoreFile().getAbsolutePath() + ".download"); - fs.downloadFile(Paths.get(meta.getRemotePath()), localFile, 10); - return localFile; + fs.downloadFile(meta, ".download" /* suffix */, 10 /* retries */); + return Paths.get(meta.newRestoreFile().getAbsolutePath() + ".download"); } @Override diff --git a/priam/src/main/java/com/netflix/priam/backupv2/MetaV2Proxy.java b/priam/src/main/java/com/netflix/priam/backupv2/MetaV2Proxy.java index 5dafd2472..223c57624 100644 --- a/priam/src/main/java/com/netflix/priam/backupv2/MetaV2Proxy.java +++ b/priam/src/main/java/com/netflix/priam/backupv2/MetaV2Proxy.java @@ -152,9 +152,8 @@ public List findMetaFiles(DateUtil.DateRange dateRange) { @Override public Path downloadMetaFile(AbstractBackupPath meta) throws BackupRestoreException { - Path localFile = Paths.get(meta.newRestoreFile().getAbsolutePath()); - fs.downloadFile(Paths.get(meta.getRemotePath()), localFile, 10); - return localFile; + fs.downloadFile(meta, "" /* suffix */, 10 /* retries */); + return Paths.get(meta.newRestoreFile().getAbsolutePath()); } @Override diff --git a/priam/src/main/java/com/netflix/priam/backupv2/SnapshotMetaTask.java b/priam/src/main/java/com/netflix/priam/backupv2/SnapshotMetaTask.java index 871f021e2..bda0e6899 100644 --- a/priam/src/main/java/com/netflix/priam/backupv2/SnapshotMetaTask.java +++ b/priam/src/main/java/com/netflix/priam/backupv2/SnapshotMetaTask.java @@ -16,6 +16,7 @@ */ package com.netflix.priam.backupv2; +import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.ImmutableSetMultimap; import com.google.inject.Provider; import com.netflix.priam.backup.*; @@ -32,8 +33,6 @@ import java.nio.file.DirectoryStream; import java.nio.file.Files; import java.nio.file.Path; -import java.nio.file.Paths; -import java.nio.file.attribute.BasicFileAttributes; import java.time.Instant; import java.util.Date; import java.util.List; @@ -46,7 +45,6 @@ import javax.inject.Named; import javax.inject.Singleton; import org.apache.commons.io.FileUtils; -import org.apache.commons.io.filefilter.FileFilterUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -219,7 +217,7 @@ public void execute() throws Exception { MetaFileWriterBuilder.UploadStep uploadStep = processSnapshot(snapshotInstant); backupMetadata.setSnapshotLocation( config.getBackupPrefix() + File.separator + uploadStep.getRemoteMetaFilePath()); - uploadStep.uploadMetaFile(true); + uploadStep.uploadMetaFile(); logger.info("Finished processing snapshot meta service"); @@ -255,24 +253,6 @@ public String getName() { return JOBNAME; } - private ColumnfamilyResult convertToColumnFamilyResult( - String keyspace, - String columnFamilyName, - ImmutableSetMultimap filePrefixToFileMap) { - ColumnfamilyResult columnfamilyResult = new ColumnfamilyResult(keyspace, columnFamilyName); - filePrefixToFileMap - .keySet() - .forEach( - (key) -> { - ColumnfamilyResult.SSTableResult ssTableResult = - new ColumnfamilyResult.SSTableResult(); - ssTableResult.setPrefix(key); - ssTableResult.setSstableComponents(filePrefixToFileMap.get(key)); - columnfamilyResult.addSstable(ssTableResult); - }); - return columnfamilyResult; - } - private void uploadAllFiles(final String columnFamily, final File backupDir) throws Exception { // Process all the snapshots with SNAPSHOT_PREFIX. This will ensure that we "resume" the // uploads of previous snapshot leftover as Priam restarted or any failure for any reason @@ -334,83 +314,31 @@ private void generateMetaFile( } logger.debug("Scanning for all SSTables in: {}", snapshotDir.getAbsolutePath()); - ImmutableSetMultimap.Builder filePrefixToFileMap = + ImmutableSetMultimap.Builder builder = ImmutableSetMultimap.builder(); - filePrefixToFileMap.putAll( - getFileUploadResults(snapshotDir, AbstractBackupPath.BackupFileType.SST_V2)); + builder.putAll(getSSTables(snapshotDir, AbstractBackupPath.BackupFileType.SST_V2)); // Next, add secondary indexes for (File subDir : getSecondaryIndexDirectories(snapshotDir, columnFamily)) { - filePrefixToFileMap.putAll( - getFileUploadResults( - subDir, AbstractBackupPath.BackupFileType.SECONDARY_INDEX_V2)); + builder.putAll( + getSSTables(subDir, AbstractBackupPath.BackupFileType.SECONDARY_INDEX_V2)); } - ColumnfamilyResult columnfamilyResult = - convertToColumnFamilyResult(keyspace, columnFamily, filePrefixToFileMap.build()); - - int sstableCount = columnfamilyResult.getSstables().size(); - logger.debug("Processing {} sstables from {}.{}", keyspace, columnFamily, sstableCount); - - dataStep.addColumnfamilyResult(columnfamilyResult); + ImmutableSetMultimap sstables = builder.build(); + logger.debug("Processing {} sstables from {}.{}", keyspace, columnFamily, sstables.size()); + dataStep.addColumnfamilyResult(keyspace, columnFamily, sstables); logger.debug("Finished processing KS: {}, CF: {}", keyspace, columnFamily); } - ImmutableSetMultimap getFileUploadResults( - File snapshotDir, AbstractBackupPath.BackupFileType type) throws Exception { - ImmutableSetMultimap.Builder filePrefixToFileMap = + private ImmutableSetMultimap getSSTables( + File snapshotDir, AbstractBackupPath.BackupFileType type) throws IOException { + ImmutableSetMultimap.Builder ssTables = ImmutableSetMultimap.builder(); - for (File file : FileUtils.listFiles(snapshotDir, FileFilterUtils.fileFileFilter(), null)) { - if (!file.exists()) continue; - Optional prefix = getPrefix(file); - if (!prefix.isPresent()) continue; - - FileUploadResult fileUploadResult; - try { - BasicFileAttributes fileAttributes = - Files.readAttributes(file.toPath(), BasicFileAttributes.class); - fileUploadResult = - new FileUploadResult( - file.toPath(), - fileAttributes.lastModifiedTime().toInstant(), - fileAttributes.creationTime().toInstant(), - fileAttributes.size()); - } catch (Exception e) { - /* If you are here it means either of the issues. In that case, do not upload the meta file. - * @throws UnsupportedOperationException - * if an attributes of the given type are not supported - * @throws IOException - * if an I/O error occurs - * @throws SecurityException - * In the case of the default provider, a security manager is - * installed, its {@link SecurityManager#checkRead(String) checkRead} - * method is invoked to check read access to the file. If this - * method is invoked to read security sensitive attributes then the - * security manager may be invoke to check for additional permissions. - */ - logger.error( - "Internal error while trying to generate FileUploadResult and/or reading FileAttributes for file: " - + file.getAbsolutePath(), - e); - throw e; - } - // Add isUploaded and remotePath here. - try { - AbstractBackupPath abstractBackupPath = pathFactory.get(); - abstractBackupPath.parseLocal(file, type); - fileUploadResult.setBackupPath(abstractBackupPath.getRemotePath()); - fileUploadResult.setUploaded( - fs.checkObjectExists(Paths.get(fileUploadResult.getBackupPath()))); - } catch (Exception e) { - logger.error( - "Error while setting the remoteLocation or checking if file exists. Ignoring them as they are not fatal.", - e.getMessage()); - e.printStackTrace(); - } - filePrefixToFileMap.put(prefix.get(), fileUploadResult); - } - return filePrefixToFileMap.build(); + getBackupPaths(snapshotDir, type) + .forEach(bp -> getPrefix(bp.getBackupFile()).ifPresent(p -> ssTables.put(p, bp))); + return ssTables.build(); } + /** * Gives the prefix (common name) of the sstable components. Returns an empty Optional if it is * not an sstable component or a manifest or schema file. @@ -420,7 +348,7 @@ ImmutableSetMultimap getFileUploadResults( * @param file the file from which to extract a common prefix. * @return common prefix of the file, or empty, */ - public static Optional getPrefix(File file) { + private static Optional getPrefix(File file) { String fileName = file.getName(); String prefix = null; if (fileName.contains("-")) { @@ -443,7 +371,7 @@ private List getSecondaryIndexDirectories(File snapshotDir, String columnF .collect(Collectors.toList()); } - // For testing purposes only. + @VisibleForTesting void setSnapshotName(String snapshotName) { this.snapshotName = snapshotName; } diff --git a/priam/src/main/java/com/netflix/priam/compress/ChunkedStream.java b/priam/src/main/java/com/netflix/priam/compress/ChunkedStream.java index b23b29dd4..81f86c6ab 100644 --- a/priam/src/main/java/com/netflix/priam/compress/ChunkedStream.java +++ b/priam/src/main/java/com/netflix/priam/compress/ChunkedStream.java @@ -25,18 +25,25 @@ /** Byte iterator representing compressed data. Uses snappy compression */ public class ChunkedStream implements Iterator { + private static final int BYTES_TO_READ = 2048; + private boolean hasnext = true; private final ByteArrayOutputStream bos; - private final SnappyOutputStream compress; + private final SnappyOutputStream snappy; private final InputStream origin; private final long chunkSize; - private static final int BYTES_TO_READ = 2048; + private final CompressionType compression; + + public ChunkedStream(InputStream is, long chunkSize) { + this(is, chunkSize, CompressionType.NONE); + } - public ChunkedStream(InputStream is, long chunkSize) throws IOException { + public ChunkedStream(InputStream is, long chunkSize, CompressionType compression) { this.origin = is; this.bos = new ByteArrayOutputStream(); - this.compress = new SnappyOutputStream(bos); + this.snappy = new SnappyOutputStream(bos); this.chunkSize = chunkSize; + this.compression = compression; } @Override @@ -50,7 +57,16 @@ public byte[] next() { byte data[] = new byte[BYTES_TO_READ]; int count; while ((count = origin.read(data, 0, data.length)) != -1) { - compress.write(data, 0, count); + switch (compression) { + case NONE: + bos.write(data, 0, count); + break; + case SNAPPY: + snappy.write(data, 0, count); + break; + default: + throw new IllegalArgumentException("Snappy compression only."); + } if (bos.size() >= chunkSize) return returnSafe(); } // We don't have anything else to read hence set to false. @@ -61,10 +77,10 @@ public byte[] next() { } private byte[] done() throws IOException { - compress.flush(); + if (compression == CompressionType.SNAPPY) snappy.flush(); byte[] return_ = bos.toByteArray(); hasnext = false; - IOUtils.closeQuietly(compress); + IOUtils.closeQuietly(snappy); IOUtils.closeQuietly(bos); IOUtils.closeQuietly(origin); return return_; diff --git a/priam/src/main/java/com/netflix/priam/compress/CompressionAlgorithm.java b/priam/src/main/java/com/netflix/priam/compress/CompressionType.java similarity index 66% rename from priam/src/main/java/com/netflix/priam/compress/CompressionAlgorithm.java rename to priam/src/main/java/com/netflix/priam/compress/CompressionType.java index c9244899f..63d516b9e 100644 --- a/priam/src/main/java/com/netflix/priam/compress/CompressionAlgorithm.java +++ b/priam/src/main/java/com/netflix/priam/compress/CompressionType.java @@ -1,6 +1,6 @@ package com.netflix.priam.compress; -public enum CompressionAlgorithm { +public enum CompressionType { SNAPPY, LZ4, NONE diff --git a/priam/src/main/java/com/netflix/priam/compress/ICompression.java b/priam/src/main/java/com/netflix/priam/compress/ICompression.java index 47f9f31b0..2a0267893 100644 --- a/priam/src/main/java/com/netflix/priam/compress/ICompression.java +++ b/priam/src/main/java/com/netflix/priam/compress/ICompression.java @@ -20,7 +20,6 @@ import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; -import java.util.Iterator; @ImplementedBy(SnappyCompression.class) public interface ICompression { @@ -30,7 +29,4 @@ public interface ICompression { * streams */ void decompressAndClose(InputStream input, OutputStream output) throws IOException; - - /** Produces chunks of compressed data. */ - Iterator compress(InputStream is, long chunkSize) throws IOException; } diff --git a/priam/src/main/java/com/netflix/priam/compress/SnappyCompression.java b/priam/src/main/java/com/netflix/priam/compress/SnappyCompression.java index 084bcf04e..20d1d3f84 100644 --- a/priam/src/main/java/com/netflix/priam/compress/SnappyCompression.java +++ b/priam/src/main/java/com/netflix/priam/compress/SnappyCompression.java @@ -17,7 +17,6 @@ package com.netflix.priam.compress; import java.io.*; -import java.util.Iterator; import org.apache.commons.io.IOUtils; import org.xerial.snappy.SnappyInputStream; @@ -25,11 +24,6 @@ public class SnappyCompression implements ICompression { private static final int BUFFER = 2 * 1024; - @Override - public Iterator compress(InputStream is, long chunkSize) throws IOException { - return new ChunkedStream(is, chunkSize); - } - @Override public void decompressAndClose(InputStream input, OutputStream output) throws IOException { try { diff --git a/priam/src/main/java/com/netflix/priam/config/BackupsToCompress.java b/priam/src/main/java/com/netflix/priam/config/BackupsToCompress.java new file mode 100644 index 000000000..9a7591cc1 --- /dev/null +++ b/priam/src/main/java/com/netflix/priam/config/BackupsToCompress.java @@ -0,0 +1,7 @@ +package com.netflix.priam.config; + +public enum BackupsToCompress { + ALL, + IF_REQUIRED, + NONE +} diff --git a/priam/src/main/java/com/netflix/priam/config/IConfiguration.java b/priam/src/main/java/com/netflix/priam/config/IConfiguration.java index b45c09e51..eb19ba618 100644 --- a/priam/src/main/java/com/netflix/priam/config/IConfiguration.java +++ b/priam/src/main/java/com/netflix/priam/config/IConfiguration.java @@ -1103,6 +1103,14 @@ default boolean usePrivateIP() { return getSnitch().equals("org.apache.cassandra.locator.GossipingPropertyFileSnitch"); } + /** + * @return BackupsToCompress UNCOMPRESSED means compress backups only when the files are not + * already compressed by Cassandra + */ + default BackupsToCompress getBackupsToCompress() { + return BackupsToCompress.ALL; + } + /** * Escape hatch for getting any arbitrary property by key This is useful so we don't have to * keep adding methods to this interface for every single configuration option ever. Also diff --git a/priam/src/main/java/com/netflix/priam/config/PriamConfiguration.java b/priam/src/main/java/com/netflix/priam/config/PriamConfiguration.java index d9b77d224..83087d9cf 100644 --- a/priam/src/main/java/com/netflix/priam/config/PriamConfiguration.java +++ b/priam/src/main/java/com/netflix/priam/config/PriamConfiguration.java @@ -768,4 +768,10 @@ public int getForgottenFileGracePeriodDaysForRead() { public boolean isForgottenFileMoveEnabled() { return config.get(PRIAM_PRE + ".forgottenFileMoveEnabled", false); } + + @Override + public BackupsToCompress getBackupsToCompress() { + return BackupsToCompress.valueOf( + config.get("priam.backupsToCompress", BackupsToCompress.ALL.name())); + } } diff --git a/priam/src/main/java/com/netflix/priam/google/GoogleEncryptedFileSystem.java b/priam/src/main/java/com/netflix/priam/google/GoogleEncryptedFileSystem.java index 8a4f153e5..49455517b 100755 --- a/priam/src/main/java/com/netflix/priam/google/GoogleEncryptedFileSystem.java +++ b/priam/src/main/java/com/netflix/priam/google/GoogleEncryptedFileSystem.java @@ -175,12 +175,15 @@ private Credential constructGcsCredential() throws Exception { } @Override - protected void downloadFileImpl(Path remotePath, Path localPath) throws BackupRestoreException { + protected void downloadFileImpl(AbstractBackupPath path, String suffix) + throws BackupRestoreException { + String remotePath = path.getRemotePath(); + File localFile = new File(path.newRestoreFile().getAbsolutePath() + suffix); String objectName = parseObjectname(getPrefix().toString()); com.google.api.services.storage.Storage.Objects.Get get; try { - get = constructObjectResourceHandle().get(this.srcBucketName, remotePath.toString()); + get = constructObjectResourceHandle().get(this.srcBucketName, remotePath); } catch (IOException e) { throw new BackupRestoreException( "IO error retrieving metadata for: " @@ -193,7 +196,7 @@ protected void downloadFileImpl(Path remotePath, Path localPath) throws BackupRe // If you're not using GCS' AppEngine, download the whole thing (instead of chunks) in one // request, if possible. get.getMediaHttpDownloader().setDirectDownloadEnabled(true); - try (OutputStream os = new FileOutputStream(localPath.toFile()); + try (OutputStream os = new FileOutputStream(localFile); InputStream is = get.executeMediaAsInputStream()) { IOUtils.copyLarge(is, os); } catch (IOException e) { @@ -233,12 +236,12 @@ public void shutdown() { } @Override - protected long uploadFileImpl(Path localPath, Path remotePath) throws BackupRestoreException { + protected long uploadFileImpl(AbstractBackupPath path) throws BackupRestoreException { throw new UnsupportedOperationException(); } @Override - public long getFileSize(Path remotePath) throws BackupRestoreException { + public long getFileSize(String remotePath) throws BackupRestoreException { return 0; } diff --git a/priam/src/main/java/com/netflix/priam/restore/AbstractRestore.java b/priam/src/main/java/com/netflix/priam/restore/AbstractRestore.java index d69352f87..6b8c865c6 100644 --- a/priam/src/main/java/com/netflix/priam/restore/AbstractRestore.java +++ b/priam/src/main/java/com/netflix/priam/restore/AbstractRestore.java @@ -137,7 +137,7 @@ private List> download( + localFileHandler.getAbsolutePath() + File.pathSeparator + localFileHandler.getName()); - futureList.add(downloadFile(temp, localFileHandler)); + futureList.add(downloadFile(temp)); } // Wait for all download to finish that were started from this method. @@ -301,13 +301,10 @@ public void restore(DateUtil.DateRange dateRange) throws Exception { * decrypted(optionally) and decompressed before saving to final location. * * @param path - path of object to download from source S3/GCS. - * @param restoreLocation - path to the final location of the decompressed and/or decrypted - * file. * @return Future of the job to track the progress of the job. * @throws Exception If there is any error in downloading file from the remote file system. */ - protected abstract Future downloadFile( - final AbstractBackupPath path, final File restoreLocation) throws Exception; + protected abstract Future downloadFile(final AbstractBackupPath path) throws Exception; final class BoundedList extends LinkedList { diff --git a/priam/src/main/java/com/netflix/priam/restore/EncryptedRestoreBase.java b/priam/src/main/java/com/netflix/priam/restore/EncryptedRestoreBase.java index ee9ebfc6a..2e69d5870 100755 --- a/priam/src/main/java/com/netflix/priam/restore/EncryptedRestoreBase.java +++ b/priam/src/main/java/com/netflix/priam/restore/EncryptedRestoreBase.java @@ -14,7 +14,10 @@ package com.netflix.priam.restore; import com.google.inject.Provider; -import com.netflix.priam.backup.*; +import com.netflix.priam.backup.AbstractBackupPath; +import com.netflix.priam.backup.IBackupFileSystem; +import com.netflix.priam.backup.MetaData; +import com.netflix.priam.compress.CompressionType; import com.netflix.priam.compress.ICompression; import com.netflix.priam.config.IConfiguration; import com.netflix.priam.cred.ICredentialGeneric; @@ -26,6 +29,7 @@ import com.netflix.priam.utils.RetryableCallable; import com.netflix.priam.utils.Sleeper; import java.io.*; +import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; import java.util.concurrent.Future; @@ -37,6 +41,7 @@ /** Provides common functionality applicable to all restore strategies */ public abstract class EncryptedRestoreBase extends AbstractRestore { private static final Logger logger = LoggerFactory.getLogger(EncryptedRestoreBase.class); + private static final String TMP_SUFFIX = ".tmp"; private final String jobName; private final ICredentialGeneric pgpCredential; @@ -86,12 +91,12 @@ protected EncryptedRestoreBase( } @Override - protected final Future downloadFile( - final AbstractBackupPath path, final File restoreLocation) throws Exception { + protected final Future downloadFile(final AbstractBackupPath path) throws Exception { final char[] passPhrase = new String(this.pgpCredential.getValue(ICredentialGeneric.KEY.PGP_PASSWORD)) .toCharArray(); - File tempFile = new File(restoreLocation.getAbsolutePath() + ".tmp"); + File restoreLocation = path.newRestoreFile(); + File tempFile = new File(restoreLocation.getAbsolutePath() + TMP_SUFFIX); return executor.submit( new RetryableCallable() { @@ -102,10 +107,7 @@ public Path retriableCall() throws Exception { // == download object from source bucket try { // Not retrying to download file here as it is already in RetryCallable. - fs.downloadFile( - Paths.get(path.getRemotePath()), - Paths.get(tempFile.getAbsolutePath()), - 0); + fs.downloadFile(path, TMP_SUFFIX, 0 /* retries */); } catch (Exception ex) { // This behavior is retryable; therefore, lets get to a clean state // before each retry. @@ -157,31 +159,35 @@ public Path retriableCall() throws Exception { ex); } - // == object downloaded and decrypted successfully, now uncompress it - logger.info( - "Start uncompressing file: {} to the FINAL destination stream", - decryptedFile.getAbsolutePath()); + // == object is downloaded and decrypted, now uncompress it if necessary + if (path.getCompression() == CompressionType.NONE) { + Files.move(decryptedFile.toPath(), restoreLocation.toPath()); + } else { + logger.info( + "Start uncompressing file: {} to the FINAL destination stream", + decryptedFile.getAbsolutePath()); - try (InputStream is = - new BufferedInputStream( - new FileInputStream(decryptedFile)); - BufferedOutputStream finalDestination = - new BufferedOutputStream( - new FileOutputStream(restoreLocation))) { - compress.decompressAndClose(is, finalDestination); - } catch (Exception ex) { - throw new Exception( - "Exception uncompressing file: " - + decryptedFile.getAbsolutePath() - + " to the FINAL destination stream", - ex); - } + try (InputStream is = + new BufferedInputStream( + new FileInputStream(decryptedFile)); + BufferedOutputStream finalDestination = + new BufferedOutputStream( + new FileOutputStream(restoreLocation))) { + compress.decompressAndClose(is, finalDestination); + } catch (Exception ex) { + throw new Exception( + "Exception uncompressing file: " + + decryptedFile.getAbsolutePath() + + " to the FINAL destination stream", + ex); + } - logger.info( - "Completed uncompressing file: {} to the FINAL destination stream " - + " current worker: {}", - decryptedFile.getAbsolutePath(), - Thread.currentThread().getName()); + logger.info( + "Completed uncompressing file: {} to the FINAL destination stream " + + " current worker: {}", + decryptedFile.getAbsolutePath(), + Thread.currentThread().getName()); + } // if here, everything was successful for this object, lets remove unneeded // file(s) if (tempFile.exists()) tempFile.delete(); diff --git a/priam/src/main/java/com/netflix/priam/restore/Restore.java b/priam/src/main/java/com/netflix/priam/restore/Restore.java index ac0a27f66..c5cd04388 100644 --- a/priam/src/main/java/com/netflix/priam/restore/Restore.java +++ b/priam/src/main/java/com/netflix/priam/restore/Restore.java @@ -30,9 +30,7 @@ import com.netflix.priam.scheduler.SimpleTimer; import com.netflix.priam.scheduler.TaskTimer; import com.netflix.priam.utils.Sleeper; -import java.io.File; import java.nio.file.Path; -import java.nio.file.Paths; import java.util.concurrent.Future; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -70,10 +68,8 @@ public Restore( } @Override - protected final Future downloadFile( - final AbstractBackupPath path, final File restoreLocation) throws Exception { - return fs.asyncDownloadFile( - Paths.get(path.getRemotePath()), Paths.get(restoreLocation.getAbsolutePath()), 5); + protected final Future downloadFile(final AbstractBackupPath path) throws Exception { + return fs.asyncDownloadFile(path, 5 /* retries */); } public static TaskTimer getTimer() { diff --git a/priam/src/test/java/com/netflix/priam/backup/FakeBackupFileSystem.java b/priam/src/test/java/com/netflix/priam/backup/FakeBackupFileSystem.java index 4d9c64bc2..86d5d822e 100644 --- a/priam/src/test/java/com/netflix/priam/backup/FakeBackupFileSystem.java +++ b/priam/src/test/java/com/netflix/priam/backup/FakeBackupFileSystem.java @@ -24,6 +24,7 @@ import com.netflix.priam.config.IConfiguration; import com.netflix.priam.merics.BackupMetrics; import com.netflix.priam.notification.BackupNotificationMgr; +import java.io.File; import java.io.FileWriter; import java.io.IOException; import java.nio.file.Path; @@ -111,7 +112,7 @@ public void shutdown() { } @Override - public long getFileSize(Path remotePath) throws BackupRestoreException { + public long getFileSize(String remotePath) throws BackupRestoreException { return 0; } @@ -142,13 +143,12 @@ public void cleanup() { } @Override - protected void downloadFileImpl(Path remotePath, Path localPath) throws BackupRestoreException { - AbstractBackupPath path = pathProvider.get(); - path.parseRemote(remotePath.toString()); - + protected void downloadFileImpl(AbstractBackupPath path, String suffix) + throws BackupRestoreException { + File localFile = new File(path.newRestoreFile().getAbsolutePath() + suffix); if (path.getType() == AbstractBackupPath.BackupFileType.META) { // List all files and generate the file - try (FileWriter fr = new FileWriter(localPath.toFile())) { + try (FileWriter fr = new FileWriter(localFile)) { JSONArray jsonObj = new JSONArray(); for (AbstractBackupPath filePath : flist) { if (filePath.type == AbstractBackupPath.BackupFileType.SNAP @@ -162,13 +162,13 @@ protected void downloadFileImpl(Path remotePath, Path localPath) throws BackupRe throw new BackupRestoreException(io.getMessage(), io); } } - downloadedFiles.add(remotePath.toString()); + downloadedFiles.add(path.getRemotePath()); } @Override - protected long uploadFileImpl(Path localPath, Path remotePath) throws BackupRestoreException { - uploadedFiles.add(localPath.toFile().getAbsolutePath()); - addFile(remotePath.toString()); - return localPath.toFile().length(); + protected long uploadFileImpl(AbstractBackupPath path) throws BackupRestoreException { + uploadedFiles.add(path.getBackupFile().getAbsolutePath()); + addFile(path.getRemotePath()); + return path.getBackupFile().length(); } } diff --git a/priam/src/test/java/com/netflix/priam/backup/NullBackupFileSystem.java b/priam/src/test/java/com/netflix/priam/backup/NullBackupFileSystem.java index 04377f301..d074a5e99 100644 --- a/priam/src/test/java/com/netflix/priam/backup/NullBackupFileSystem.java +++ b/priam/src/test/java/com/netflix/priam/backup/NullBackupFileSystem.java @@ -43,7 +43,7 @@ public void shutdown() { } @Override - public long getFileSize(Path remotePath) throws BackupRestoreException { + public long getFileSize(String remotePath) throws BackupRestoreException { return 0; } @@ -63,7 +63,7 @@ public void cleanup() { } @Override - protected void downloadFileImpl(Path remotePath, Path localPath) + protected void downloadFileImpl(AbstractBackupPath path, String suffix) throws BackupRestoreException {} @Override @@ -72,7 +72,7 @@ protected boolean doesRemoteFileExist(Path remotePath) { } @Override - protected long uploadFileImpl(Path localPath, Path remotePath) throws BackupRestoreException { + protected long uploadFileImpl(AbstractBackupPath path) throws BackupRestoreException { return 0; } } diff --git a/priam/src/test/java/com/netflix/priam/backup/TestAbstractBackup.java b/priam/src/test/java/com/netflix/priam/backup/TestAbstractBackup.java new file mode 100644 index 000000000..94dc44412 --- /dev/null +++ b/priam/src/test/java/com/netflix/priam/backup/TestAbstractBackup.java @@ -0,0 +1,127 @@ +package com.netflix.priam.backup; + +import com.google.appengine.repackaged.com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; +import com.google.common.truth.Truth; +import com.google.inject.Guice; +import com.google.inject.Injector; +import com.google.inject.Provider; +import com.netflix.priam.compress.CompressionType; +import com.netflix.priam.config.BackupsToCompress; +import com.netflix.priam.config.FakeConfiguration; +import com.netflix.priam.config.IConfiguration; +import java.io.File; +import java.io.IOException; +import java.nio.file.Paths; +import java.util.Arrays; +import java.util.Collection; +import org.apache.commons.io.FileUtils; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; + +@RunWith(Parameterized.class) +public class TestAbstractBackup { + private static final String COMPRESSED_DATA = "compressed-1234-Data.db"; + private static final String COMPRESSION_INFO = "compressed-1234-CompressionInfo.db"; + private static final String UNCOMPRESSED_DATA = "uncompressed-1234-Data.db"; + private static final String RANDOM_DATA = "random-1234-Data.db"; + private static final String RANDOM_COMPONENT = "random-1234-compressioninfo.db"; + private static final ImmutableList TABLE_PARTS = + ImmutableList.of( + COMPRESSED_DATA, + COMPRESSION_INFO, + UNCOMPRESSED_DATA, + RANDOM_DATA, + RANDOM_COMPONENT); + + private static final String DIRECTORY = "target/data/ks/cf/backup/"; + private final AbstractBackup abstractBackup; + private final FakeConfiguration fakeConfiguration; + private final String tablePart; + private final CompressionType compressionAlgorithm; + + @BeforeClass + public static void setUp() throws IOException { + FileUtils.forceMkdir(new File(DIRECTORY)); + } + + @Before + public void createFiles() throws IOException { + for (String tablePart : TABLE_PARTS) { + File file = Paths.get(DIRECTORY, tablePart).toFile(); + if (file.createNewFile()) { + FileUtils.forceDeleteOnExit(file); + } else { + throw new IllegalStateException("failed to create " + tablePart); + } + } + } + + @AfterClass + public static void tearDown() throws IOException { + FileUtils.deleteDirectory(new File(DIRECTORY)); + } + + @Parameterized.Parameters + public static Collection data() { + return Arrays.asList( + new Object[][] { + {BackupsToCompress.NONE, COMPRESSED_DATA, CompressionType.NONE}, + {BackupsToCompress.NONE, COMPRESSION_INFO, CompressionType.NONE}, + {BackupsToCompress.NONE, UNCOMPRESSED_DATA, CompressionType.NONE}, + {BackupsToCompress.NONE, RANDOM_DATA, CompressionType.NONE}, + {BackupsToCompress.NONE, RANDOM_COMPONENT, CompressionType.NONE}, + {BackupsToCompress.ALL, COMPRESSED_DATA, CompressionType.SNAPPY}, + {BackupsToCompress.ALL, COMPRESSION_INFO, CompressionType.SNAPPY}, + {BackupsToCompress.ALL, UNCOMPRESSED_DATA, CompressionType.SNAPPY}, + {BackupsToCompress.ALL, RANDOM_DATA, CompressionType.SNAPPY}, + {BackupsToCompress.ALL, RANDOM_COMPONENT, CompressionType.SNAPPY}, + {BackupsToCompress.IF_REQUIRED, COMPRESSED_DATA, CompressionType.NONE}, + {BackupsToCompress.IF_REQUIRED, COMPRESSION_INFO, CompressionType.NONE}, + {BackupsToCompress.IF_REQUIRED, UNCOMPRESSED_DATA, CompressionType.SNAPPY}, + {BackupsToCompress.IF_REQUIRED, RANDOM_DATA, CompressionType.SNAPPY}, + {BackupsToCompress.IF_REQUIRED, RANDOM_COMPONENT, CompressionType.SNAPPY}, + }); + } + + public TestAbstractBackup(BackupsToCompress which, String tablePart, CompressionType algo) { + this.tablePart = tablePart; + this.compressionAlgorithm = algo; + Injector injector = Guice.createInjector(new BRTestModule()); + fakeConfiguration = (FakeConfiguration) injector.getInstance(IConfiguration.class); + fakeConfiguration.setFakeConfig("Priam.backupsToCompress", which); + IFileSystemContext context = injector.getInstance(IFileSystemContext.class); + Provider pathFactory = injector.getProvider(AbstractBackupPath.class); + abstractBackup = + new AbstractBackup(fakeConfiguration, context, pathFactory) { + @Override + protected void processColumnFamily(String ks, String cf, File dir) {} + + @Override + public void execute() {} + + @Override + public String getName() { + return null; + } + }; + } + + @Test + public void testCorrectCompressionType() throws Exception { + File parent = new File(DIRECTORY); + AbstractBackupPath.BackupFileType backupFileType = AbstractBackupPath.BackupFileType.SST_V2; + ImmutableSet paths = + abstractBackup.upload(parent, backupFileType, false, false); + AbstractBackupPath abstractBackupPath = + paths.stream() + .filter(path -> path.getFileName().equals(tablePart)) + .findAny() + .orElseThrow(IllegalStateException::new); + Truth.assertThat(abstractBackupPath.getCompression()).isEqualTo(compressionAlgorithm); + } +} diff --git a/priam/src/test/java/com/netflix/priam/backup/TestAbstractFileSystem.java b/priam/src/test/java/com/netflix/priam/backup/TestAbstractFileSystem.java index 695caf80b..e042cd0ac 100644 --- a/priam/src/test/java/com/netflix/priam/backup/TestAbstractFileSystem.java +++ b/priam/src/test/java/com/netflix/priam/backup/TestAbstractFileSystem.java @@ -82,8 +82,7 @@ public void testFailedRetriesUpload() throws Exception { try { Collection files = generateFiles(1, 1, 1); for (File file : files) { - failureFileSystem.uploadFile( - file.toPath(), file.toPath(), getDummyPath(file.toPath()), 2, true); + failureFileSystem.uploadAndDelete(getDummyPath(file.toPath()), 2); } } catch (BackupRestoreException e) { // Verify the failure metric for upload is incremented. @@ -91,9 +90,13 @@ public void testFailedRetriesUpload() throws Exception { } } + private AbstractBackupPath getDummyPath() throws ParseException { + return getDummyPath(Paths.get(configuration.getDataFileLocation() + "/ks/cf/file-Data.db")); + } + private AbstractBackupPath getDummyPath(Path localPath) throws ParseException { AbstractBackupPath path = injector.getInstance(AbstractBackupPath.class); - path.parseLocal(localPath.toFile(), AbstractBackupPath.BackupFileType.SNAP); + path.parseLocal(localPath.toFile(), AbstractBackupPath.BackupFileType.SST_V2); return path; } @@ -107,9 +110,9 @@ private Collection generateFiles(int noOfKeyspaces, int noOfCf, int noOfSs } @Test - public void testFailedRetriesDownload() { + public void testFailedRetriesDownload() throws Exception { try { - failureFileSystem.downloadFile(Paths.get(""), null, 2); + failureFileSystem.downloadFile(getDummyPath(), "", 2); } catch (BackupRestoreException e) { // Verify the failure metric for download is incremented. Assert.assertEquals(1, (int) backupMetrics.getInvalidDownloads().count()); @@ -118,51 +121,26 @@ public void testFailedRetriesDownload() { @Test public void testUpload() throws Exception { - Collection files = generateFiles(1, 1, 1); - // Dummy upload with compressed size. - for (File file : files) { - myFileSystem.uploadFile( - file.toPath(), - Paths.get(file.toString() + ".tmp"), - getDummyPath(file.toPath()), - 2, - true); - // Verify the success metric for upload is incremented. - Assert.assertEquals(1, (int) backupMetrics.getValidUploads().actualCount()); - - // Verify delete of the original file if flag provided. - Assert.assertFalse(file.exists()); - break; - } + File file = generateFiles(1, 1, 1).iterator().next(); + myFileSystem.uploadAndDelete(getDummyPath(file.toPath()), 2); + Assert.assertEquals(1, (int) backupMetrics.getValidUploads().actualCount()); + Assert.assertFalse(file.exists()); } @Test public void testDownload() throws Exception { // Dummy download - myFileSystem.downloadFile(Paths.get(""), Paths.get(configuration.getDataFileLocation()), 2); + myFileSystem.downloadFile(getDummyPath(), "", 2); // Verify the success metric for download is incremented. Assert.assertEquals(1, (int) backupMetrics.getValidDownloads().actualCount()); } @Test public void testAsyncUpload() throws Exception { - // Testing single async upload. - Collection files = generateFiles(1, 1, 1); - for (File file : files) { - myFileSystem - .asyncUploadFile( - file.toPath(), - Paths.get(file.toString() + ".tmp"), - getDummyPath(file.toPath()), - 2, - true) - .get(); - // 1. Verify the success metric for upload is incremented. - Assert.assertEquals(1, (int) backupMetrics.getValidUploads().actualCount()); - // 2. The task queue is empty after upload is finished. - Assert.assertEquals(0, myFileSystem.getUploadTasksQueued()); - break; - } + File file = generateFiles(1, 1, 1).iterator().next(); + myFileSystem.asyncUploadAndDelete(getDummyPath(file.toPath()), 2).get(); + Assert.assertEquals(1, (int) backupMetrics.getValidUploads().actualCount()); + Assert.assertEquals(0, myFileSystem.getUploadTasksQueued()); } @Test @@ -170,19 +148,13 @@ public void testAsyncUploadBulk() throws Exception { // Testing the queue feature works. // 1. Give 1000 dummy files to upload. File upload takes some random time to upload Collection files = generateFiles(1, 1, 20); - List> futures = new ArrayList<>(); + List> futures = new ArrayList<>(); for (File file : files) { - futures.add( - myFileSystem.asyncUploadFile( - file.toPath(), - Paths.get(file.toString() + ".tmp"), - getDummyPath(file.toPath()), - 2, - true)); + futures.add(myFileSystem.asyncUploadAndDelete(getDummyPath(file.toPath()), 2)); } // Verify all the work is finished. - for (Future future : futures) { + for (Future future : futures) { future.get(); } // 2. Success metric is incremented correctly @@ -205,8 +177,7 @@ public void testUploadDedup() throws Exception { for (int i = 0; i < size; i++) { torun.add( () -> { - myFileSystem.uploadFile( - file.toPath(), file.toPath(), abstractBackupPath, 2, true); + myFileSystem.uploadAndDelete(abstractBackupPath, 2); return Boolean.TRUE; }); } @@ -216,7 +187,7 @@ public void testUploadDedup() throws Exception { // no more need for the threadpool threads.shutdown(); - for (Future future : futures) { + for (Future future : futures) { try { future.get(); } catch (InterruptedException | ExecutionException e) { @@ -233,9 +204,8 @@ public void testAsyncUploadFailure() throws Exception { // Testing single async upload. Collection files = generateFiles(1, 1, 1); for (File file : files) { - Future future = - failureFileSystem.asyncUploadFile( - file.toPath(), file.toPath(), getDummyPath(file.toPath()), 2, true); + Future future = + failureFileSystem.asyncUploadAndDelete(getDummyPath(file.toPath()), 2); try { future.get(); } catch (Exception e) { @@ -254,9 +224,7 @@ public void testAsyncUploadFailure() throws Exception { @Test public void testAsyncDownload() throws Exception { // Testing single async download. - Future future = - myFileSystem.asyncDownloadFile( - Paths.get(""), Paths.get(configuration.getDataFileLocation()), 2); + Future future = myFileSystem.asyncDownloadFile(getDummyPath(), 2); future.get(); // 1. Verify the success metric for download is incremented. Assert.assertEquals(1, (int) backupMetrics.getValidDownloads().actualCount()); @@ -271,9 +239,7 @@ public void testAsyncDownloadBulk() throws Exception { int totalFiles = 1000; List> futureList = new ArrayList<>(); for (int i = 0; i < totalFiles; i++) - futureList.add( - myFileSystem.asyncDownloadFile( - Paths.get("" + i), Paths.get(configuration.getDataFileLocation()), 2)); + futureList.add(myFileSystem.asyncDownloadFile(getDummyPath(Paths.get("" + i)), 2)); // Ensure processing is finished. for (Future future1 : futureList) { @@ -289,7 +255,7 @@ public void testAsyncDownloadBulk() throws Exception { @Test public void testAsyncDownloadFailure() throws Exception { - Future future = failureFileSystem.asyncDownloadFile(Paths.get(""), null, 2); + Future future = failureFileSystem.asyncDownloadFile(getDummyPath(), 2); try { future.get(); } catch (Exception e) { @@ -310,19 +276,18 @@ public FailureFileSystem( } @Override - protected void downloadFileImpl(Path remotePath, Path localPath) + protected void downloadFileImpl(AbstractBackupPath path, String suffix) throws BackupRestoreException { throw new BackupRestoreException( "User injected failure file system error for testing download. Remote path: " - + remotePath); + + path.getRemotePath()); } @Override - protected long uploadFileImpl(Path localPath, Path remotePath) - throws BackupRestoreException { + protected long uploadFileImpl(AbstractBackupPath path) throws BackupRestoreException { throw new BackupRestoreException( "User injected failure file system error for testing upload. Local path: " - + localPath); + + path.getBackupFile().getAbsolutePath()); } } @@ -340,7 +305,7 @@ public MyFileSystem( } @Override - protected void downloadFileImpl(Path remotePath, Path localPath) + protected void downloadFileImpl(AbstractBackupPath path, String suffix) throws BackupRestoreException { try { Thread.sleep(random.nextInt(20)); @@ -350,8 +315,7 @@ protected void downloadFileImpl(Path remotePath, Path localPath) } @Override - protected long uploadFileImpl(Path localPath, Path remotePath) - throws BackupRestoreException { + protected long uploadFileImpl(AbstractBackupPath path) throws BackupRestoreException { try { Thread.sleep(random.nextInt(20)); } catch (InterruptedException e) { diff --git a/priam/src/test/java/com/netflix/priam/backup/TestCompression.java b/priam/src/test/java/com/netflix/priam/backup/TestCompression.java index 63fdb0dbd..73e616f47 100644 --- a/priam/src/test/java/com/netflix/priam/backup/TestCompression.java +++ b/priam/src/test/java/com/netflix/priam/backup/TestCompression.java @@ -20,6 +20,8 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; +import com.netflix.priam.compress.ChunkedStream; +import com.netflix.priam.compress.CompressionType; import com.netflix.priam.compress.ICompression; import com.netflix.priam.compress.SnappyCompression; import com.netflix.priam.utils.SystemUtils; @@ -118,7 +120,10 @@ private void testCompressor(ICompression compress) throws IOException { try { Iterator it = - compress.compress(new FileInputStream(randomContentFile), chunkSize); + new ChunkedStream( + new FileInputStream(randomContentFile), + chunkSize, + CompressionType.SNAPPY); try (FileOutputStream ostream = new FileOutputStream(compressedOutputFile)) { while (it.hasNext()) { byte[] chunk = it.next(); diff --git a/priam/src/test/java/com/netflix/priam/backup/TestS3FileSystem.java b/priam/src/test/java/com/netflix/priam/backup/TestS3FileSystem.java index 9fdd4fead..626ff6e51 100644 --- a/priam/src/test/java/com/netflix/priam/backup/TestS3FileSystem.java +++ b/priam/src/test/java/com/netflix/priam/backup/TestS3FileSystem.java @@ -25,6 +25,7 @@ import com.amazonaws.services.s3.model.*; import com.amazonaws.services.s3.model.lifecycle.LifecycleFilter; import com.amazonaws.services.s3.model.lifecycle.LifecyclePrefixPredicate; +import com.google.api.client.util.Preconditions; import com.google.common.collect.Lists; import com.google.inject.Guice; import com.google.inject.Injector; @@ -43,11 +44,13 @@ import java.nio.file.Path; import java.nio.file.Paths; import java.util.ArrayList; +import java.util.Arrays; import java.util.List; import java.util.Optional; import java.util.stream.Collectors; import mockit.Mock; import mockit.MockUp; +import org.apache.commons.io.FileUtils; import org.junit.AfterClass; import org.junit.Assert; import org.junit.BeforeClass; @@ -58,8 +61,7 @@ public class TestS3FileSystem { private static Injector injector; private static final Logger logger = LoggerFactory.getLogger(TestS3FileSystem.class); - private static final String FILE_PATH = - "target/data/Keyspace1/Standard1/backups/201108082320/Keyspace1-Standard1-ia-1-Data.db"; + private static final File DIR = new File("target/data/KS1/CF1/backups/201108082320/"); private static BackupMetrics backupMetrics; private static String region; private static IConfiguration configuration; @@ -76,26 +78,15 @@ public TestS3FileSystem() { } @BeforeClass - public static void setup() throws InterruptedException, IOException { + public static void setUp() { new MockS3PartUploader(); new MockAmazonS3Client(); - - File dir1 = new File("target/data/Keyspace1/Standard1/backups/201108082320"); - if (!dir1.exists()) dir1.mkdirs(); - File file = new File(FILE_PATH); - long fiveKB = (5L * 1024); - byte b = 8; - BufferedOutputStream bos1 = new BufferedOutputStream(new FileOutputStream(file)); - for (long i = 0; i < fiveKB; i++) { - bos1.write(b); - } - bos1.close(); + if (!DIR.exists()) DIR.mkdirs(); } @AfterClass - public static void cleanup() { - File file = new File(FILE_PATH); - file.delete(); + public static void cleanup() throws IOException { + FileUtils.cleanDirectory(DIR); } @Test @@ -103,14 +94,9 @@ public void testFileUpload() throws Exception { MockS3PartUploader.setup(); IBackupFileSystem fs = injector.getInstance(NullBackupFileSystem.class); RemoteBackupPath backupfile = injector.getInstance(RemoteBackupPath.class); - backupfile.parseLocal(new File(FILE_PATH), BackupFileType.SNAP); + backupfile.parseLocal(localFile(), BackupFileType.SNAP); long noOfFilesUploaded = backupMetrics.getUploadRate().count(); - fs.uploadFile( - Paths.get(backupfile.getBackupFile().getAbsolutePath()), - Paths.get(backupfile.getRemotePath()), - backupfile, - 0, - false); + fs.uploadAndDelete(backupfile, 0); Assert.assertEquals(1, backupMetrics.getUploadRate().count() - noOfFilesUploaded); } @@ -119,13 +105,8 @@ public void testFileUploadDeleteExists() throws Exception { MockS3PartUploader.setup(); IBackupFileSystem fs = injector.getInstance(NullBackupFileSystem.class); RemoteBackupPath backupfile = injector.getInstance(RemoteBackupPath.class); - backupfile.parseLocal(new File(FILE_PATH), BackupFileType.SST_V2); - fs.uploadFile( - Paths.get(backupfile.getBackupFile().getAbsolutePath()), - Paths.get(backupfile.getRemotePath()), - backupfile, - 0, - false); + backupfile.parseLocal(localFile(), BackupFileType.SST_V2); + fs.uploadAndDelete(backupfile, 0); Assert.assertTrue(fs.checkObjectExists(Paths.get(backupfile.getRemotePath()))); // Lets delete the file now. List deleteFiles = Lists.newArrayList(); @@ -140,17 +121,10 @@ public void testFileUploadFailures() throws Exception { MockS3PartUploader.partFailure = true; long noOfFailures = backupMetrics.getInvalidUploads().count(); S3FileSystem fs = injector.getInstance(S3FileSystem.class); - String snapshotfile = - "target/data/Keyspace1/Standard1/backups/201108082320/Keyspace1-Standard1-ia-1-Data.db"; RemoteBackupPath backupfile = injector.getInstance(RemoteBackupPath.class); - backupfile.parseLocal(new File(snapshotfile), BackupFileType.SNAP); + backupfile.parseLocal(localFile(), BackupFileType.SNAP); try { - fs.uploadFile( - Paths.get(backupfile.getBackupFile().getAbsolutePath()), - Paths.get(backupfile.getRemotePath()), - backupfile, - 0, - false); + fs.uploadAndDelete(backupfile, 0); } catch (BackupRestoreException e) { // ignore } @@ -164,17 +138,10 @@ public void testFileUploadCompleteFailure() throws Exception { MockS3PartUploader.completionFailure = true; S3FileSystem fs = injector.getInstance(S3FileSystem.class); fs.setS3Client(new MockAmazonS3Client().getMockInstance()); - String snapshotfile = - "target/data/Keyspace1/Standard1/backups/201108082320/Keyspace1-Standard1-ia-1-Data.db"; RemoteBackupPath backupfile = injector.getInstance(RemoteBackupPath.class); - backupfile.parseLocal(new File(snapshotfile), BackupFileType.SNAP); + backupfile.parseLocal(localFile(), BackupFileType.SNAP); try { - fs.uploadFile( - Paths.get(backupfile.getBackupFile().getAbsolutePath()), - Paths.get(backupfile.getRemotePath()), - backupfile, - 0, - false); + fs.uploadAndDelete(backupfile, 0); } catch (BackupRestoreException e) { // ignore } @@ -237,6 +204,20 @@ public void testDeleteObjects() throws Exception { } } + private File localFile() throws IOException { + String caller = Thread.currentThread().getStackTrace()[1].getMethodName(); + File file = new File(DIR + caller + "KS1-CF1-ia-1-Data.db"); + if (file.createNewFile()) { + byte[] data = new byte[5 << 10]; + Arrays.fill(data, (byte) 8); + try (BufferedOutputStream os = new BufferedOutputStream(new FileOutputStream(file))) { + os.write(data); + } + } + Preconditions.checkState(file.exists()); + return file; + } + // Mock Nodeprobe class static class MockS3PartUploader extends MockUp { static int compattempts = 0; diff --git a/priam/src/test/java/com/netflix/priam/backupv2/TestBackupUtils.java b/priam/src/test/java/com/netflix/priam/backupv2/TestBackupUtils.java index 30c9c5477..e14253a6f 100644 --- a/priam/src/test/java/com/netflix/priam/backupv2/TestBackupUtils.java +++ b/priam/src/test/java/com/netflix/priam/backupv2/TestBackupUtils.java @@ -17,7 +17,7 @@ package com.netflix.priam.backupv2; -import com.google.common.collect.ImmutableSet; +import com.google.common.collect.ImmutableSetMultimap; import com.google.inject.Guice; import com.google.inject.Inject; import com.google.inject.Injector; @@ -52,19 +52,19 @@ public TestBackupUtils() { public Path createMeta(List filesToAdd, Instant snapshotTime) throws IOException { MetaFileWriterBuilder.DataStep dataStep = metaFileWriterBuilder.newBuilder().startMetaFileGeneration(snapshotTime); - ColumnfamilyResult columnfamilyResult = new ColumnfamilyResult(keyspace, columnfamily); + ImmutableSetMultimap.Builder builder = + ImmutableSetMultimap.builder(); for (String file : filesToAdd) { + String basename = Paths.get(file).getFileName().toString(); + int lastIndex = basename.lastIndexOf('-'); + lastIndex = lastIndex < 0 ? basename.length() : lastIndex; + String prefix = basename.substring(0, lastIndex); AbstractBackupPath path = pathProvider.get(); path.parseRemote(file); - if (path.getType() == AbstractBackupPath.BackupFileType.SST_V2) { - ColumnfamilyResult.SSTableResult ssTableResult = - new ColumnfamilyResult.SSTableResult(); - - ssTableResult.setSstableComponents(ImmutableSet.of(getFileUploadResult(path))); - columnfamilyResult.addSstable(ssTableResult); - } + path.setCreationTime(path.getLastModified()); + builder.put(prefix, path); } - dataStep.addColumnfamilyResult(columnfamilyResult); + dataStep.addColumnfamilyResult(keyspace, columnfamily, builder.build()); Path metaPath = dataStep.endMetaFileGeneration().getMetaFilePath(); metaPath.toFile().setLastModified(snapshotTime.toEpochMilli()); return metaPath; @@ -79,15 +79,4 @@ public String createFile(String fileName, Instant lastModifiedTime) throws Excep path.toFile().setLastModified(lastModifiedTime.toEpochMilli()); return path.toString(); } - - private FileUploadResult getFileUploadResult(AbstractBackupPath path) { - FileUploadResult fileUploadResult = - new FileUploadResult( - Paths.get(path.getFileName()), - path.getLastModified(), - path.getLastModified(), - path.getSize()); - fileUploadResult.setBackupPath(path.getRemotePath()); - return fileUploadResult; - } } diff --git a/priam/src/test/java/com/netflix/priam/backupv2/TestMetaV2Proxy.java b/priam/src/test/java/com/netflix/priam/backupv2/TestMetaV2Proxy.java index f434204d7..68d56d895 100644 --- a/priam/src/test/java/com/netflix/priam/backupv2/TestMetaV2Proxy.java +++ b/priam/src/test/java/com/netflix/priam/backupv2/TestMetaV2Proxy.java @@ -171,7 +171,7 @@ private List getRemoteFakeFiles() { "columnfamily1", "SNAPPY", "PLAINTEXT", - "file1.Data.db")); + "file1-Data.db")); files.add( Paths.get( getPrefix(), @@ -181,7 +181,7 @@ private List getRemoteFakeFiles() { "columnfamily1", "SNAPPY", "PLAINTEXT", - "file2.Data.db")); + "file2-Data.db")); files.add( Paths.get( @@ -210,7 +210,7 @@ private List getRemoteFakeFiles() { "columnfamily1", "SNAPPY", "PLAINTEXT", - "file3.Data.db")); + "file3-Data.db")); files.add( Paths.get( getPrefix(), @@ -220,7 +220,7 @@ private List getRemoteFakeFiles() { "columnfamily1", "SNAPPY", "PLAINTEXT", - "file4.Data.db")); + "file4-Data.db")); files.add( Paths.get( getPrefix(), diff --git a/priam/src/test/java/com/netflix/priam/config/FakeConfiguration.java b/priam/src/test/java/com/netflix/priam/config/FakeConfiguration.java index ebd3bd318..40a6f069a 100644 --- a/priam/src/test/java/com/netflix/priam/config/FakeConfiguration.java +++ b/priam/src/test/java/com/netflix/priam/config/FakeConfiguration.java @@ -229,4 +229,9 @@ public boolean usePrivateIP() { public void usePrivateIP(boolean usePrivateIp) { this.usePrivateIp = usePrivateIp; } + + public BackupsToCompress getBackupsToCompress() { + return (BackupsToCompress) + fakeConfig.getOrDefault("Priam.backupsToCompress", BackupsToCompress.ALL); + } } From 9c8cfe292c55f2a8a684736ba8a67947071d7be3 Mon Sep 17 00:00:00 2001 From: Martin Chalupa Date: Thu, 20 May 2021 11:28:15 -0700 Subject: [PATCH 162/228] Update secrets from 3.x branch (#942) --- .travis.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.travis.yml b/.travis.yml index 9e6b9926a..864d3bc02 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,9 +11,9 @@ cache: - "$HOME/.gradle" env: global: - - secure: bXzjPA0Ec3NBIN25knd9UYzGtxxmlhoLkO0LwI87iRSeCLzvbT+qBl/chvaXCk3DC2JnSq6GGWw2r0R6pjR4UezZx72KP+dm19cy606e7np1mHZmzziSkyjaBd84HPgOz8L4jh5G9qQBG8VhkjikIabAgsbiWG3oX6a/rOadobg= - - secure: COIxG87u7BSBAD0s8g0E9PTq4XLompgW/PL55bnIxuVkIo3tXXqShAxMzsJ0C++q5AbofNUICqTbGqN0ozk9GpMC78d5pgWS1O4hpdWsouK8Fxcju+KkKC5MUDC478G/zW0uSa0uZsZGaDy3Gx17jpvq2CdVCgHmaHmGGOxfDbA= - - secure: LMyJ2v4kN+07Gz5pPHkYLOvbQOAI05bHFGUVfaAbofyiR1DqKpKqB4hYaKSgb0kB/3YI6u9dLQ5AWzAqfx3HY0+p2iTmgseIxbHGrFfOH7bt2xusYNUtYPn1JXOAz1Nzxq/a+4rCgqD9lFI19XetrI5SNrRIRErDIZnWdXyhW/M= - - secure: dxJVfXpC/yORQWg2oaw+Q2UjeoAHiqf5hyLVSdGbIJYVrSu/fyFCuUNjQffikjleoyuaPeIHxaGQZoQZE3+9Cf3d7s1lXzugO3UOxmKR9UYi8tz6sLUFeLPjVhR6Etx603A8oCg659RhD2VyOCZvq1/R2gUa34tUtnVyoFyUzDA= - - secure: tl59nbZhOzWqW/PLp91+Paugond4Lwwj/qT1Tq4EFJ9Q2nNMFW3twlWzmUOcgn7GaJZTgfoDx2HGYFg1N3rrBtZIY/Wtzz9eWrGY4ywHfoXz/8lQGtbciDjiNePByuvSLOPWeTIGncOLGNUcCPNFOO9+kC4BzUGr9CjtTc+qnWg= - - secure: AZoBLO43P+GVjQq+vsT8mekOUNqifi+tRYukz19GLN2vv2vsgY4ejhXjeer1kwqm0l/1bXtfqo0uHyD+CFn2yINV20oKHScYNS4fwKnJ8aC60MZqAj6zddfe2VvEeDIKDiejpz4AfRJwgqpl9B0FrNtZ/1oU5n8oneTKpVFryTs= + - secure: JUZT8CmNPjpmKguIioCY0quM5ZGzO03fv7YMDZqR909WbfiX57ArPJOorD9U1ri8REiRjhIe5dTgQMNtar6i1uJj0EwhCIahRn+CNJhtHBxA1siX3/E8HU8ANirLLziBQlWnnjwKlGoUKw04plMLkaA9Ybn3WV9sdi/kVHRpI2Y= + - secure: wxVMsVmsKEuksiQlkm47gAH6Pu5lM3zwWVXJw1eUf9zTqLVwqomuQ19Bgv2ZDcoWNBdcfRouhV98uCclCgFD3tPTVfJ3GSrXhpQLstGihNRtZE5EmEVj8rephVa4H1g0BxzuhbFCgV6VyQWECvAYpyyutk6+ygeS7uhjWtUCTus= + - secure: wDpYNAJGRtlnbJUfmnmmTSH5+TTf5AccjbaVa1NVblQPb0gdupEo8lGPeKGyaAWtteHlC+ih5nP6nh2geoWc6Svfv7fXRdffVyCF+WLrTPKvPOgC1A9IgrbVbjKRJv3Lqx+i6vYcRc+EJ3Bn9Z5AVACY4ag9KfDr5Xc1xmoJK6U= + - secure: XP08P6WaX92eH0lziOoWAAPz0HDpg8RjuxgNQBaQ5NmgyjRypxMnjro2Uqfcivn0tjV8tFQNzGrx2Y1LaC3w1ZJiINuLbSKWNfC/NP/zlMwTfQBwnwx0brmn5u/OwpqmU6l+nrMNyXKKR4hhH47O2ZD3ynrRAivt5tgDo/6xp9Y= + - secure: Gf497SakGlRoyYAqNrto/U+wrfJ9wtUzStwV0uac5CEJoDNN1DG3LA+HmRYgubwXZU3VKb/2BPEyflNV+LOWW54CkWu5Syw4i//iL+FEHv2eDR82NyvOcl11BtmUDLW/G+iBV5J3DlGTSmbgfC195kAYhozxx5FF+UVIYisS0Yc= + - secure: ky97xfe8gq2KqexRhLvaaEI85H/6HJ3iXh+z8eHnRHQpCkL4azVripdaUXwMj5H5EotWevB1nZVdttX/ZU5VIfkyA7H7fgWT9I17o4p8JKA8VItzUAIWopA4pUw2moxKiqhJZ8szqBCeEk5qgbUJyFbsAsHrTmoqSI/8u3WaL0U= From 47c14182501d0417ecc2b39168c8c5f73609eb35 Mon Sep 17 00:00:00 2001 From: ayushisingh29 Date: Thu, 20 May 2021 16:09:55 -0700 Subject: [PATCH 163/228] Verify thrift listening on port before returning its status (#943) --- .../netflix/priam/config/IConfiguration.java | 5 ++ .../priam/config/PriamConfiguration.java | 5 ++ .../priam/health/CassandraMonitor.java | 9 +- .../netflix/priam/health/IThriftChecker.java | 8 ++ .../netflix/priam/health/ThriftChecker.java | 52 +++++++++++ .../priam/config/FakeConfiguration.java | 10 +++ .../priam/health/TestCassandraMonitor.java | 7 +- .../priam/health/TestThriftChecker.java | 86 +++++++++++++++++++ 8 files changed, 179 insertions(+), 3 deletions(-) create mode 100644 priam/src/main/java/com/netflix/priam/health/IThriftChecker.java create mode 100644 priam/src/main/java/com/netflix/priam/health/ThriftChecker.java create mode 100644 priam/src/test/java/com/netflix/priam/health/TestThriftChecker.java diff --git a/priam/src/main/java/com/netflix/priam/config/IConfiguration.java b/priam/src/main/java/com/netflix/priam/config/IConfiguration.java index eb19ba618..4e6cdcf36 100644 --- a/priam/src/main/java/com/netflix/priam/config/IConfiguration.java +++ b/priam/src/main/java/com/netflix/priam/config/IConfiguration.java @@ -1103,6 +1103,11 @@ default boolean usePrivateIP() { return getSnitch().equals("org.apache.cassandra.locator.GossipingPropertyFileSnitch"); } + /** @return true to check is thrift listening on the rpc_port. */ + default boolean checkThriftServerIsListening() { + return false; + } + /** * @return BackupsToCompress UNCOMPRESSED means compress backups only when the files are not * already compressed by Cassandra diff --git a/priam/src/main/java/com/netflix/priam/config/PriamConfiguration.java b/priam/src/main/java/com/netflix/priam/config/PriamConfiguration.java index 83087d9cf..f0e9a883c 100644 --- a/priam/src/main/java/com/netflix/priam/config/PriamConfiguration.java +++ b/priam/src/main/java/com/netflix/priam/config/PriamConfiguration.java @@ -774,4 +774,9 @@ public BackupsToCompress getBackupsToCompress() { return BackupsToCompress.valueOf( config.get("priam.backupsToCompress", BackupsToCompress.ALL.name())); } + + @Override + public boolean checkThriftServerIsListening() { + return config.get(PRIAM_PRE + ".checkThriftServerIsListening", false); + } } diff --git a/priam/src/main/java/com/netflix/priam/health/CassandraMonitor.java b/priam/src/main/java/com/netflix/priam/health/CassandraMonitor.java index f55cd506c..f0f6c849c 100644 --- a/priam/src/main/java/com/netflix/priam/health/CassandraMonitor.java +++ b/priam/src/main/java/com/netflix/priam/health/CassandraMonitor.java @@ -47,17 +47,20 @@ public class CassandraMonitor extends Task { private final InstanceState instanceState; private final ICassandraProcess cassProcess; private final CassMonitorMetrics cassMonitorMetrics; + private final IThriftChecker thriftChecker; @Inject protected CassandraMonitor( IConfiguration config, InstanceState instanceState, ICassandraProcess cassProcess, - CassMonitorMetrics cassMonitorMetrics) { + CassMonitorMetrics cassMonitorMetrics, + IThriftChecker thriftChecker) { super(config); this.instanceState = instanceState; this.cassProcess = cassProcess; this.cassMonitorMetrics = cassMonitorMetrics; + this.thriftChecker = thriftChecker; } @Override @@ -92,7 +95,9 @@ public void execute() throws Exception { NodeProbe bean = JMXNodeTool.instance(this.config); instanceState.setIsGossipActive(bean.isGossipRunning()); instanceState.setIsNativeTransportActive(bean.isNativeTransportRunning()); - instanceState.setIsThriftActive(bean.isThriftServerRunning()); + instanceState.setIsThriftActive( + bean.isThriftServerRunning() && thriftChecker.isThriftServerListening()); + } else { // Setting cassandra flag to false instanceState.setCassandraProcessAlive(false); diff --git a/priam/src/main/java/com/netflix/priam/health/IThriftChecker.java b/priam/src/main/java/com/netflix/priam/health/IThriftChecker.java new file mode 100644 index 000000000..621faec30 --- /dev/null +++ b/priam/src/main/java/com/netflix/priam/health/IThriftChecker.java @@ -0,0 +1,8 @@ +package com.netflix.priam.health; + +import com.google.inject.ImplementedBy; + +@ImplementedBy(ThriftChecker.class) +public interface IThriftChecker { + boolean isThriftServerListening(); +} diff --git a/priam/src/main/java/com/netflix/priam/health/ThriftChecker.java b/priam/src/main/java/com/netflix/priam/health/ThriftChecker.java new file mode 100644 index 000000000..0390cbbca --- /dev/null +++ b/priam/src/main/java/com/netflix/priam/health/ThriftChecker.java @@ -0,0 +1,52 @@ +package com.netflix.priam.health; + +import com.google.inject.Inject; +import com.netflix.priam.config.IConfiguration; +import java.io.BufferedReader; +import java.io.InputStreamReader; +import java.util.concurrent.TimeUnit; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class ThriftChecker implements IThriftChecker { + private static final Logger logger = LoggerFactory.getLogger(ThriftChecker.class); + protected final IConfiguration config; + + @Inject + public ThriftChecker(IConfiguration config) { + this.config = config; + } + + public boolean isThriftServerListening() { + if (!config.checkThriftServerIsListening()) { + return true; + } + String[] cmd = + new String[] { + "/bin/sh", "-c", "ss -tuln | grep -c " + config.getThriftPort(), " 2>/dev/null" + }; + Process process = null; + try { + process = Runtime.getRuntime().exec(cmd); + process.waitFor(1, TimeUnit.SECONDS); + } catch (Exception e) { + logger.warn("Exception while executing the process: ", e); + } + if (process != null) { + try (BufferedReader reader = + new BufferedReader(new InputStreamReader(process.getInputStream())); ) { + if (Integer.parseInt(reader.readLine()) == 0) { + logger.info( + "Could not find anything listening on the rpc port {}!", + config.getThriftPort()); + return false; + } + } catch (Exception e) { + logger.warn("Exception while reading the input stream: ", e); + } + } + // A quiet on-call is our top priority, err on the side of avoiding false positives by + // default. + return true; + } +} diff --git a/priam/src/test/java/com/netflix/priam/config/FakeConfiguration.java b/priam/src/test/java/com/netflix/priam/config/FakeConfiguration.java index 40a6f069a..6a3358f07 100644 --- a/priam/src/test/java/com/netflix/priam/config/FakeConfiguration.java +++ b/priam/src/test/java/com/netflix/priam/config/FakeConfiguration.java @@ -35,6 +35,7 @@ public class FakeConfiguration implements IConfiguration { private boolean mayCreateNewToken; private ImmutableList racs; private boolean usePrivateIp; + private boolean checkThriftIsListening; public Map fakeProperties = new HashMap<>(); @@ -234,4 +235,13 @@ public BackupsToCompress getBackupsToCompress() { return (BackupsToCompress) fakeConfig.getOrDefault("Priam.backupsToCompress", BackupsToCompress.ALL); } + + @Override + public boolean checkThriftServerIsListening() { + return checkThriftIsListening; + } + + public void setCheckThriftServerIsListening(boolean checkThriftServerIsListening) { + this.checkThriftIsListening = checkThriftServerIsListening; + } } diff --git a/priam/src/test/java/com/netflix/priam/health/TestCassandraMonitor.java b/priam/src/test/java/com/netflix/priam/health/TestCassandraMonitor.java index 55fc76f64..f4a79fc97 100644 --- a/priam/src/test/java/com/netflix/priam/health/TestCassandraMonitor.java +++ b/priam/src/test/java/com/netflix/priam/health/TestCassandraMonitor.java @@ -37,6 +37,8 @@ public class TestCassandraMonitor { private static CassandraMonitor monitor; private static InstanceState instanceState; private static CassMonitorMetrics cassMonitorMetrics; + private static IThriftChecker thriftChecker; + private IConfiguration config; @Mocked private Process mockProcess; @@ -47,13 +49,16 @@ public class TestCassandraMonitor { public void setUp() { Injector injector = Guice.createInjector(new BRTestModule()); config = injector.getInstance(IConfiguration.class); + thriftChecker = injector.getInstance(ThriftChecker.class); if (instanceState == null) instanceState = injector.getInstance(InstanceState.class); if (cassMonitorMetrics == null) cassMonitorMetrics = injector.getInstance(CassMonitorMetrics.class); if (monitor == null) - monitor = new CassandraMonitor(config, instanceState, cassProcess, cassMonitorMetrics); + monitor = + new CassandraMonitor( + config, instanceState, cassProcess, cassMonitorMetrics, thriftChecker); } @Test diff --git a/priam/src/test/java/com/netflix/priam/health/TestThriftChecker.java b/priam/src/test/java/com/netflix/priam/health/TestThriftChecker.java new file mode 100644 index 000000000..0936686d4 --- /dev/null +++ b/priam/src/test/java/com/netflix/priam/health/TestThriftChecker.java @@ -0,0 +1,86 @@ +package com.netflix.priam.health; + +import com.netflix.priam.config.FakeConfiguration; +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import mockit.Expectations; +import mockit.Mocked; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +public class TestThriftChecker { + private FakeConfiguration config; + private ThriftChecker thriftChecker; + + @Mocked private Process mockProcess; + + @Before + public void TestThriftChecker() { + config = new FakeConfiguration(); + thriftChecker = new ThriftChecker(config); + } + + @Test + public void testThriftServerIsListeningDisabled() { + config.setCheckThriftServerIsListening(false); + Assert.assertTrue(thriftChecker.isThriftServerListening()); + } + + @Test + public void testThriftServerIsNotListening() { + config.setCheckThriftServerIsListening(true); + Assert.assertFalse(thriftChecker.isThriftServerListening()); + } + + @Test + public void testThriftServerIsListening() throws IOException { + config.setCheckThriftServerIsListening(true); + final InputStream mockOutput = new ByteArrayInputStream("1".getBytes()); + new Expectations() { + { + mockProcess.getInputStream(); + result = mockOutput; + } + }; + // Mock out the ps call + final Runtime r = Runtime.getRuntime(); + String[] cmd = { + "/bin/sh", "-c", "ss -tuln | grep -c " + config.getThriftPort(), " 2>/dev/null" + }; + new Expectations(r) { + { + r.exec(cmd); + result = mockProcess; + } + }; + + Assert.assertTrue(thriftChecker.isThriftServerListening()); + } + + @Test + public void testThriftServerIsListeningException() throws IOException { + config.setCheckThriftServerIsListening(true); + final IOException mockOutput = new IOException("Command exited with code 0"); + new Expectations() { + { + mockProcess.getInputStream(); + result = mockOutput; + } + }; + // Mock out the ps call + final Runtime r = Runtime.getRuntime(); + String[] cmd = { + "/bin/sh", "-c", "ss -tuln | grep -c " + config.getThriftPort(), " 2>/dev/null" + }; + new Expectations(r) { + { + r.exec(cmd); + result = mockProcess; + } + }; + + Assert.assertTrue(thriftChecker.isThriftServerListening()); + } +} From da450bf4c311a6748dd5c3766d7550e87f0fc0f2 Mon Sep 17 00:00:00 2001 From: ayushisingh29 Date: Fri, 21 May 2021 11:06:04 -0700 Subject: [PATCH 164/228] Make disk_acess_mode tunable via fast property (#944) --- .../java/com/netflix/priam/config/IConfiguration.java | 5 +++++ .../com/netflix/priam/config/PriamConfiguration.java | 5 +++++ .../java/com/netflix/priam/tuner/StandardTuner.java | 1 + .../com/netflix/priam/config/FakeConfiguration.java | 11 +++++++++++ .../com/netflix/priam/tuner/StandardTunerTest.java | 7 +++++++ 5 files changed, 29 insertions(+) diff --git a/priam/src/main/java/com/netflix/priam/config/IConfiguration.java b/priam/src/main/java/com/netflix/priam/config/IConfiguration.java index 4e6cdcf36..e1664187a 100644 --- a/priam/src/main/java/com/netflix/priam/config/IConfiguration.java +++ b/priam/src/main/java/com/netflix/priam/config/IConfiguration.java @@ -1116,6 +1116,11 @@ default BackupsToCompress getBackupsToCompress() { return BackupsToCompress.ALL; } + /** @return the way how data files are accessed. Default value is auto. */ + default String getDiskAccessMode() { + return "auto"; + } + /** * Escape hatch for getting any arbitrary property by key This is useful so we don't have to * keep adding methods to this interface for every single configuration option ever. Also diff --git a/priam/src/main/java/com/netflix/priam/config/PriamConfiguration.java b/priam/src/main/java/com/netflix/priam/config/PriamConfiguration.java index f0e9a883c..f2892d43d 100644 --- a/priam/src/main/java/com/netflix/priam/config/PriamConfiguration.java +++ b/priam/src/main/java/com/netflix/priam/config/PriamConfiguration.java @@ -775,6 +775,11 @@ public BackupsToCompress getBackupsToCompress() { config.get("priam.backupsToCompress", BackupsToCompress.ALL.name())); } + @Override + public String getDiskAccessMode() { + return config.get(PRIAM_PRE + ".diskAccessMode", "auto"); + } + @Override public boolean checkThriftServerIsListening() { return config.get(PRIAM_PRE + ".checkThriftServerIsListening", false); diff --git a/priam/src/main/java/com/netflix/priam/tuner/StandardTuner.java b/priam/src/main/java/com/netflix/priam/tuner/StandardTuner.java index d36f9cf09..4b511a672 100644 --- a/priam/src/main/java/com/netflix/priam/tuner/StandardTuner.java +++ b/priam/src/main/java/com/netflix/priam/tuner/StandardTuner.java @@ -128,6 +128,7 @@ public void writeAllProperties(String yamlLocation, String hostname, String seed map.put( "compaction_large_partition_warning_threshold_mb", config.getCompactionLargePartitionWarnThresholdInMB()); + map.put("disk_access_mode", config.getDiskAccessMode()); List seedp = (List) map.get("seed_provider"); Map m = (Map) seedp.get(0); diff --git a/priam/src/test/java/com/netflix/priam/config/FakeConfiguration.java b/priam/src/test/java/com/netflix/priam/config/FakeConfiguration.java index 6a3358f07..02c2cd72e 100644 --- a/priam/src/test/java/com/netflix/priam/config/FakeConfiguration.java +++ b/priam/src/test/java/com/netflix/priam/config/FakeConfiguration.java @@ -35,6 +35,7 @@ public class FakeConfiguration implements IConfiguration { private boolean mayCreateNewToken; private ImmutableList racs; private boolean usePrivateIp; + private String diskAccessMode; private boolean checkThriftIsListening; public Map fakeProperties = new HashMap<>(); @@ -236,6 +237,16 @@ public BackupsToCompress getBackupsToCompress() { fakeConfig.getOrDefault("Priam.backupsToCompress", BackupsToCompress.ALL); } + @Override + public String getDiskAccessMode() { + return this.diskAccessMode; + } + + public FakeConfiguration setDiskAccessMode(String diskAccessMode) { + this.diskAccessMode = diskAccessMode; + return this; + } + @Override public boolean checkThriftServerIsListening() { return checkThriftIsListening; diff --git a/priam/src/test/java/com/netflix/priam/tuner/StandardTunerTest.java b/priam/src/test/java/com/netflix/priam/tuner/StandardTunerTest.java index 588c3541f..b63cb9c4a 100644 --- a/priam/src/test/java/com/netflix/priam/tuner/StandardTunerTest.java +++ b/priam/src/test/java/com/netflix/priam/tuner/StandardTunerTest.java @@ -152,6 +152,13 @@ public void addExtraParams() throws Exception { Assert.assertEquals("randomGroupValue", ((Map) map.get("randomGroup")).get("randomKey")); } + @Test + public void testDiskAccessMode() throws Exception { + String diskAccessMode = "test_mode"; + Map map = applyFakeConfiguration(new FakeConfiguration().setDiskAccessMode(diskAccessMode)); + Assert.assertEquals(diskAccessMode, map.get("disk_access_mode")); + } + @Test public void testRoleManagerOverride() throws Exception { String roleManagerOverride = "org.apache.cassandra.auth.CustomRoleManager"; From 084fe6e5a780f29038a929d8c26a6012600c14af Mon Sep 17 00:00:00 2001 From: mattl-netflix <63665634+mattl-netflix@users.noreply.github.com> Date: Tue, 25 May 2021 10:23:16 -0700 Subject: [PATCH 165/228] Ingress rule configurability (#940) * Improve the configurability of setting ingress rules * CASS-1608 derive public ip, toggle ingressing public ips exclusively * CASS-1608 Responding to review comments. Publish a metric with the ACL size, publish log warnings when we receive unexpected http status codes when trying to update acls. --- .../com/netflix/priam/aws/AWSIPConverter.java | 59 +++++++++++++++++++ .../com/netflix/priam/aws/AWSMembership.java | 19 ++++-- .../com/netflix/priam/aws/IPConverter.java | 11 ++++ .../priam/aws/UpdateSecuritySettings.java | 54 +++++++++++------ .../netflix/priam/config/IConfiguration.java | 23 ++++++++ .../netflix/priam/merics/SecurityMetrics.java | 21 +++++++ .../java/com/netflix/priam/TestModule.java | 3 + .../netflix/priam/aws/FakeIPConverter.java | 19 ++++++ .../priam/aws/TestUpdateSecuritySettings.java | 51 ++++++++-------- .../priam/config/FakeConfiguration.java | 30 ++++++++++ 10 files changed, 240 insertions(+), 50 deletions(-) create mode 100644 priam/src/main/java/com/netflix/priam/aws/AWSIPConverter.java create mode 100644 priam/src/main/java/com/netflix/priam/aws/IPConverter.java create mode 100644 priam/src/main/java/com/netflix/priam/merics/SecurityMetrics.java create mode 100644 priam/src/test/java/com/netflix/priam/aws/FakeIPConverter.java diff --git a/priam/src/main/java/com/netflix/priam/aws/AWSIPConverter.java b/priam/src/main/java/com/netflix/priam/aws/AWSIPConverter.java new file mode 100644 index 000000000..eb0f60ca9 --- /dev/null +++ b/priam/src/main/java/com/netflix/priam/aws/AWSIPConverter.java @@ -0,0 +1,59 @@ +package com.netflix.priam.aws; + +import com.google.common.base.Splitter; +import com.netflix.priam.identity.PriamInstance; +import com.netflix.priam.identity.config.InstanceInfo; +import java.net.InetAddress; +import java.net.UnknownHostException; +import java.util.List; +import java.util.Optional; +import java.util.regex.Pattern; +import javax.inject.Inject; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** Derives public ips from PriamInstances with AWS hostnames. */ +public class AWSIPConverter implements IPConverter { + private static final Logger logger = LoggerFactory.getLogger(AWSIPConverter.class); + private static final Pattern IP_PART = Pattern.compile("^[0-9]{1,3}$"); + + private final InstanceInfo myInstance; + + @Inject + public AWSIPConverter(InstanceInfo myInstance) { + this.myInstance = myInstance; + } + + @Override + public Optional getPublicIP(PriamInstance instance) { + if (instance.getInstanceId().equals(myInstance.getInstanceId())) { + return Optional.ofNullable(myInstance.getHostIP()); + } + Optional ip = extractIPFromHostnameString(instance.getHostName()); + return !instance.getDC().equals(myInstance.getRegion()) && !ip.isPresent() + ? deriveIPFromHostname(instance.getHostName()) + : ip; + } + + private Optional extractIPFromHostnameString(String hostname) { + if (hostname.contains(".")) { + String ip = hostname.substring(4, hostname.indexOf('.')).replace('-', '.'); + List parts = Splitter.on(".").splitToList(ip); + return parts.size() == 4 && parts.stream().allMatch(p -> IP_PART.matcher(p).matches()) + ? Optional.of(ip) + : Optional.empty(); + } + return Optional.empty(); + } + + private Optional deriveIPFromHostname(String hostname) { + String ip = null; + try { + ip = InetAddress.getByName(hostname).getHostAddress(); + logger.info("Derived IP for " + hostname + ": " + ip); + } catch (UnknownHostException e) { + logger.warn("Could not resolve [" + hostname + "]"); + } + return Optional.ofNullable(ip); + } +} diff --git a/priam/src/main/java/com/netflix/priam/aws/AWSMembership.java b/priam/src/main/java/com/netflix/priam/aws/AWSMembership.java index 8373d8991..f9421aabe 100644 --- a/priam/src/main/java/com/netflix/priam/aws/AWSMembership.java +++ b/priam/src/main/java/com/netflix/priam/aws/AWSMembership.java @@ -183,14 +183,21 @@ public void addACL(Collection listIPs, int from, int to) { logger.info("Done adding ACL to classic: " + StringUtils.join(listIPs, ",")); } } else { + // Adding peers' IPs as ingress to the running instance SG AuthorizeSecurityGroupIngressRequest sgIngressRequest = - new AuthorizeSecurityGroupIngressRequest(); - sgIngressRequest.withGroupId(getVpcGoupId()); + new AuthorizeSecurityGroupIngressRequest() + .withGroupId(getVpcGoupId()) + .withIpPermissions(ipPermissions); // fetch SG group id for vpc account of the running instance. - client.authorizeSecurityGroupIngress( - sgIngressRequest.withIpPermissions( - ipPermissions)); // Adding peers' IPs as ingress to the running - // instance SG + int status = + client.authorizeSecurityGroupIngress(sgIngressRequest) + .getSdkHttpMetadata() + .getHttpStatusCode(); + if (status != 200) { + logger.warn( + "We might have too many ingress rules saw http status {} when updating", + status); + } if (logger.isInfoEnabled()) { logger.info("Done adding ACL to vpc: " + StringUtils.join(listIPs, ",")); } diff --git a/priam/src/main/java/com/netflix/priam/aws/IPConverter.java b/priam/src/main/java/com/netflix/priam/aws/IPConverter.java new file mode 100644 index 000000000..fde04ac83 --- /dev/null +++ b/priam/src/main/java/com/netflix/priam/aws/IPConverter.java @@ -0,0 +1,11 @@ +package com.netflix.priam.aws; + +import com.google.inject.ImplementedBy; +import com.netflix.priam.identity.PriamInstance; +import java.util.Optional; + +/** Derives public ip from hostname. */ +@ImplementedBy(AWSIPConverter.class) +public interface IPConverter { + Optional getPublicIP(PriamInstance instance); +} diff --git a/priam/src/main/java/com/netflix/priam/aws/UpdateSecuritySettings.java b/priam/src/main/java/com/netflix/priam/aws/UpdateSecuritySettings.java index 29f9dc653..1b6c6ddd3 100644 --- a/priam/src/main/java/com/netflix/priam/aws/UpdateSecuritySettings.java +++ b/priam/src/main/java/com/netflix/priam/aws/UpdateSecuritySettings.java @@ -16,6 +16,7 @@ */ package com.netflix.priam.aws; +import com.google.common.base.Joiner; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Sets; import com.google.inject.Inject; @@ -24,10 +25,13 @@ import com.netflix.priam.identity.IMembership; import com.netflix.priam.identity.IPriamInstanceFactory; import com.netflix.priam.identity.InstanceIdentity; -import com.netflix.priam.identity.config.InstanceInfo; +import com.netflix.priam.identity.PriamInstance; +import com.netflix.priam.merics.SecurityMetrics; import com.netflix.priam.scheduler.SimpleTimer; import com.netflix.priam.scheduler.Task; import com.netflix.priam.scheduler.TaskTimer; +import java.util.Objects; +import java.util.Optional; import java.util.Random; import java.util.Set; import java.util.stream.Collectors; @@ -53,7 +57,8 @@ public class UpdateSecuritySettings extends Task { private static final Random ran = new Random(); private final IMembership membership; private final IPriamInstanceFactory factory; - private final InstanceInfo instanceInfo; + private final IPConverter ipConverter; + private final SecurityMetrics securityMetrics; @Inject // Note: do not parameterized the generic type variable to an implementation as it confuses @@ -62,11 +67,13 @@ public UpdateSecuritySettings( IConfiguration config, IMembership membership, IPriamInstanceFactory factory, - InstanceInfo instanceInfo) { + IPConverter ipConverter, + SecurityMetrics securityMetrics) { super(config); this.membership = membership; this.factory = factory; - this.instanceInfo = instanceInfo; + this.ipConverter = ipConverter; + this.securityMetrics = securityMetrics; } /** @@ -77,29 +84,38 @@ public UpdateSecuritySettings( public void execute() { int port = config.getSSLStoragePort(); ImmutableSet currentAcl = membership.listACL(port, port); + securityMetrics.setIngressRules(currentAcl.size()); Set desiredAcl = factory.getAllIds(config.getAppName()) .stream() - .map(i -> i.getHostIP() + "/32") + .map(i -> getIngressRule(i).orElse(null)) + .filter(Objects::nonNull) + .map(ip -> ip + "/32") .collect(Collectors.toSet()); - // Make sure a hole is opened for my instance. - // This accommodates the eventually consistent CassandraInstanceFactory. - // Remove once IPs are all private as there won't be any chance of a discrepancy anymore. - String myIp = - config.usePrivateIP() ? instanceInfo.getPrivateIP() : instanceInfo.getHostIP(); - desiredAcl.add(myIp + "/32"); - Set aclToAdd = Sets.difference(desiredAcl, currentAcl); - if (!aclToAdd.isEmpty()) { - membership.addACL(aclToAdd, port, port); - firstTimeUpdated = true; + if (!config.skipDeletingOthersIngressRules()) { + Set aclToRemove = Sets.difference(currentAcl, desiredAcl); + logger.info("ingress rules to delete: {}", Joiner.on(",").join(aclToRemove)); + if (!aclToRemove.isEmpty()) { + membership.removeACL(aclToRemove, port, port); + firstTimeUpdated = true; + } } - Set aclToRemove = Sets.difference(currentAcl, desiredAcl); - if (!aclToRemove.isEmpty()) { - membership.removeACL(aclToRemove, port, port); - firstTimeUpdated = true; + if (!config.skipUpdatingOthersIngressRules()) { + Set aclToAdd = Sets.difference(desiredAcl, currentAcl); + logger.info("ingress rules to update: {}", Joiner.on(",").join(aclToAdd)); + if (!aclToAdd.isEmpty()) { + membership.addACL(aclToAdd, port, port); + firstTimeUpdated = true; + } } } + private Optional getIngressRule(PriamInstance instance) { + return config.skipIngressUnlessIPIsPublic() + ? ipConverter.getPublicIP(instance) + : Optional.of(instance.getHostIP()); + } + public static TaskTimer getTimer(InstanceIdentity id) { SimpleTimer return_; if (id.isSeed()) { diff --git a/priam/src/main/java/com/netflix/priam/config/IConfiguration.java b/priam/src/main/java/com/netflix/priam/config/IConfiguration.java index e1664187a..0111f7fa7 100644 --- a/priam/src/main/java/com/netflix/priam/config/IConfiguration.java +++ b/priam/src/main/java/com/netflix/priam/config/IConfiguration.java @@ -1121,6 +1121,29 @@ default String getDiskAccessMode() { return "auto"; } + /** + * @return true if Priam should skip deleting ingress rules for IPs not found in the token + * database. + */ + default boolean skipDeletingOthersIngressRules() { + return false; + } + + /** + * @return true if Priam should skip updating ingress rules for ips found in the token database. + */ + default boolean skipUpdatingOthersIngressRules() { + return false; + } + + /** + * @return true if Priam should skip ingress on an IP address from the token database unless it + * can confirm that it is public + */ + default boolean skipIngressUnlessIPIsPublic() { + return false; + } + /** * Escape hatch for getting any arbitrary property by key This is useful so we don't have to * keep adding methods to this interface for every single configuration option ever. Also diff --git a/priam/src/main/java/com/netflix/priam/merics/SecurityMetrics.java b/priam/src/main/java/com/netflix/priam/merics/SecurityMetrics.java new file mode 100644 index 000000000..7dac74222 --- /dev/null +++ b/priam/src/main/java/com/netflix/priam/merics/SecurityMetrics.java @@ -0,0 +1,21 @@ +package com.netflix.priam.merics; + +import com.google.inject.Inject; +import com.netflix.spectator.api.Gauge; +import com.netflix.spectator.api.Registry; +import javax.inject.Singleton; + +/** Metrics pertaining to network security. Currently just publishes a count of ingress rules. */ +@Singleton +public class SecurityMetrics { + private final Gauge ingressRules; + + @Inject + public SecurityMetrics(Registry registry) { + ingressRules = registry.gauge(Metrics.METRIC_PREFIX + "ingress.rules"); + } + + public void setIngressRules(int count) { + ingressRules.set(count); + } +} diff --git a/priam/src/test/java/com/netflix/priam/TestModule.java b/priam/src/test/java/com/netflix/priam/TestModule.java index 0f4ceffbe..bacb0800b 100644 --- a/priam/src/test/java/com/netflix/priam/TestModule.java +++ b/priam/src/test/java/com/netflix/priam/TestModule.java @@ -20,6 +20,8 @@ import com.google.common.collect.ImmutableList; import com.google.inject.AbstractModule; import com.google.inject.Scopes; +import com.netflix.priam.aws.FakeIPConverter; +import com.netflix.priam.aws.IPConverter; import com.netflix.priam.backup.FakeCredentials; import com.netflix.priam.backup.IBackupFileSystem; import com.netflix.priam.backup.NullBackupFileSystem; @@ -63,5 +65,6 @@ protected void configure() { bind(IBackupFileSystem.class).to(NullBackupFileSystem.class); bind(Sleeper.class).to(FakeSleeper.class); bind(Registry.class).toInstance(new DefaultRegistry()); + bind(IPConverter.class).toInstance(new FakeIPConverter()); } } diff --git a/priam/src/test/java/com/netflix/priam/aws/FakeIPConverter.java b/priam/src/test/java/com/netflix/priam/aws/FakeIPConverter.java new file mode 100644 index 000000000..77d41cee1 --- /dev/null +++ b/priam/src/test/java/com/netflix/priam/aws/FakeIPConverter.java @@ -0,0 +1,19 @@ +package com.netflix.priam.aws; + +import com.netflix.priam.identity.PriamInstance; +import groovy.lang.Singleton; +import java.util.Optional; + +@Singleton +public class FakeIPConverter implements IPConverter { + private boolean shouldFail; + + public void setShouldFail(boolean shouldFail) { + this.shouldFail = shouldFail; + } + + @Override + public Optional getPublicIP(PriamInstance instance) { + return shouldFail ? Optional.empty() : Optional.of("1.1.1.1"); + } +} diff --git a/priam/src/test/java/com/netflix/priam/aws/TestUpdateSecuritySettings.java b/priam/src/test/java/com/netflix/priam/aws/TestUpdateSecuritySettings.java index 8f53e9c3c..2a8cd3b3e 100644 --- a/priam/src/test/java/com/netflix/priam/aws/TestUpdateSecuritySettings.java +++ b/priam/src/test/java/com/netflix/priam/aws/TestUpdateSecuritySettings.java @@ -9,7 +9,6 @@ import com.netflix.priam.config.IConfiguration; import com.netflix.priam.identity.IMembership; import com.netflix.priam.identity.IPriamInstanceFactory; -import com.netflix.priam.identity.config.InstanceInfo; import org.junit.Before; import org.junit.Test; @@ -21,7 +20,7 @@ public class TestUpdateSecuritySettings { private IMembership membership; private IPriamInstanceFactory factory; private FakeConfiguration config; - private InstanceInfo instanceInfo; + private FakeIPConverter ipConverter; @Before public void setUp() { @@ -30,7 +29,7 @@ public void setUp() { membership = injector.getInstance(IMembership.class); updateSecuritySettings = injector.getInstance(UpdateSecuritySettings.class); config = (FakeConfiguration) injector.getInstance(IConfiguration.class); - instanceInfo = injector.getInstance(InstanceInfo.class); + ipConverter = (FakeIPConverter) injector.getInstance(IPConverter.class); } @Test @@ -68,41 +67,43 @@ public void delete_factoryEmpty() { // edge-case, not expected } @Test - public void addMyPrivateIP() { - config.usePrivateIP(true); - String ingressRule = instanceInfo.getPrivateIP() + "/32"; - Truth.assertThat(membership.listACL(PORT, PORT)).doesNotContain(ingressRule); + public void dontDeleteOthersIngress() { + config.setSkipDeletingOthersIngressRules(true); + membership.addACL(ImmutableSet.of("1.1.1.1/32"), PORT, PORT); + Truth.assertThat(membership.listACL(PORT, PORT)).contains("1.1.1.1/32"); updateSecuritySettings.execute(); - Truth.assertThat(membership.listACL(PORT, PORT)).contains(ingressRule); + Truth.assertThat(membership.listACL(PORT, PORT)).contains("1.1.1.1/32"); } @Test - public void addMyPublicIP() { - config.usePrivateIP(false); - String ingressRule = instanceInfo.getHostIP() + "/32"; - Truth.assertThat(membership.listACL(PORT, PORT)).doesNotContain(ingressRule); + public void dontUpdateOthersIngress() { + addToFactory(1, "1.1.1.1"); + config.setSkipUpdatingOthersIngressRules(true); + Truth.assertThat(membership.listACL(PORT, PORT)).isEmpty(); updateSecuritySettings.execute(); - Truth.assertThat(membership.listACL(PORT, PORT)).contains(ingressRule); + Truth.assertThat(membership.listACL(PORT, PORT)).isEmpty(); } @Test - public void keepMyPrivateIP() { - config.usePrivateIP(true); - String ingressRule = instanceInfo.getPrivateIP() + "/32"; - membership.addACL(ImmutableSet.of(ingressRule), PORT, PORT); - Truth.assertThat(membership.listACL(PORT, PORT)).contains(ingressRule); + public void dontUpdatePrivateIngress() { + addToFactory(1, "2.2.2.2"); + ipConverter.setShouldFail(false); + config.setSkipIngressUnlessIPIsPublic(true); + Truth.assertThat(membership.listACL(PORT, PORT)).isEmpty(); updateSecuritySettings.execute(); - Truth.assertThat(membership.listACL(PORT, PORT)).contains(ingressRule); + // The private IP produced by our FakeIPConverter + Truth.assertThat(membership.listACL(PORT, PORT)).contains("1.1.1.1/32"); + Truth.assertThat(membership.listACL(PORT, PORT)).doesNotContain("2.2.2.2/32"); } @Test - public void keepMyPublicIP() { - config.usePrivateIP(false); - String ingressRule = instanceInfo.getHostIP() + "/32"; - membership.addACL(ImmutableSet.of(ingressRule), PORT, PORT); - Truth.assertThat(membership.listACL(PORT, PORT)).contains(ingressRule); + public void dontUpdateUnknownIngress() { + addToFactory(1, "1.1.1.1"); + ipConverter.setShouldFail(true); + config.setSkipIngressUnlessIPIsPublic(true); + Truth.assertThat(membership.listACL(PORT, PORT)).isEmpty(); updateSecuritySettings.execute(); - Truth.assertThat(membership.listACL(PORT, PORT)).contains(ingressRule); + Truth.assertThat(membership.listACL(PORT, PORT)).isEmpty(); } private void addToFactory(int id, String ip) { diff --git a/priam/src/test/java/com/netflix/priam/config/FakeConfiguration.java b/priam/src/test/java/com/netflix/priam/config/FakeConfiguration.java index 02c2cd72e..e9b61cfff 100644 --- a/priam/src/test/java/com/netflix/priam/config/FakeConfiguration.java +++ b/priam/src/test/java/com/netflix/priam/config/FakeConfiguration.java @@ -37,6 +37,9 @@ public class FakeConfiguration implements IConfiguration { private boolean usePrivateIp; private String diskAccessMode; private boolean checkThriftIsListening; + private boolean skipDeletingOthersIngressRules; + private boolean skipUpdatingOthersIngressRules; + private boolean skipIngressUnlessIPIsPublic; public Map fakeProperties = new HashMap<>(); @@ -237,6 +240,24 @@ public BackupsToCompress getBackupsToCompress() { fakeConfig.getOrDefault("Priam.backupsToCompress", BackupsToCompress.ALL); } + @Override + public boolean skipDeletingOthersIngressRules() { + return this.skipDeletingOthersIngressRules; + } + + public void setSkipDeletingOthersIngressRules(boolean skipDeletingOthersIngressRules) { + this.skipDeletingOthersIngressRules = skipDeletingOthersIngressRules; + } + + @Override + public boolean skipUpdatingOthersIngressRules() { + return this.skipUpdatingOthersIngressRules; + } + + public void setSkipUpdatingOthersIngressRules(boolean skipUpdatingOthersIngressRules) { + this.skipUpdatingOthersIngressRules = skipUpdatingOthersIngressRules; + } + @Override public String getDiskAccessMode() { return this.diskAccessMode; @@ -255,4 +276,13 @@ public boolean checkThriftServerIsListening() { public void setCheckThriftServerIsListening(boolean checkThriftServerIsListening) { this.checkThriftIsListening = checkThriftServerIsListening; } + + @Override + public boolean skipIngressUnlessIPIsPublic() { + return this.skipIngressUnlessIPIsPublic; + } + + public void setSkipIngressUnlessIPIsPublic(boolean skipIngressUnlessIPIsPublic) { + this.skipIngressUnlessIPIsPublic = skipIngressUnlessIPIsPublic; + } } From 858fe1c92ff131fb51bd68f60ec45c639568da19 Mon Sep 17 00:00:00 2001 From: mattl-netflix <63665634+mattl-netflix@users.noreply.github.com> Date: Tue, 25 May 2021 13:55:09 -0700 Subject: [PATCH 166/228] Fix file descriptor leak introduced in PR #922 (#945) --- .../main/java/com/netflix/priam/backup/AbstractBackup.java | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/priam/src/main/java/com/netflix/priam/backup/AbstractBackup.java b/priam/src/main/java/com/netflix/priam/backup/AbstractBackup.java index 15d4f00a9..82402cde4 100644 --- a/priam/src/main/java/com/netflix/priam/backup/AbstractBackup.java +++ b/priam/src/main/java/com/netflix/priam/backup/AbstractBackup.java @@ -38,6 +38,7 @@ import java.util.List; import java.util.Set; import java.util.concurrent.Future; +import java.util.stream.Stream; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -95,8 +96,10 @@ protected ImmutableSet upload( protected ImmutableSet getBackupPaths(File dir, BackupFileType type) throws IOException { - Set files = - Files.list(dir.toPath()).map(Path::toFile).filter(File::isFile).collect(toSet()); + Set files; + try (Stream pathStream = Files.list(dir.toPath())) { + files = pathStream.map(Path::toFile).filter(File::isFile).collect(toSet()); + } Set compressedFilePrefixes = files.stream() .map(File::getName) From 392633baa5ca2c011da90dffde0b74ba22d9a450 Mon Sep 17 00:00:00 2001 From: mattl-netflix <63665634+mattl-netflix@users.noreply.github.com> Date: Tue, 25 May 2021 14:27:58 -0700 Subject: [PATCH 167/228] Delete secondary index backup directories after uploading files (#941) * CASS-2201 Consolidate logic to find secondary index directories * CASS-2201 use correct file type for secondary index files in IncrementalBackup * CASS-2201 Delete empty secondary index backup directories * Responding to review comments and ensuring we wait for completion in IncrementalBackup. --- .../netflix/priam/backup/AbstractBackup.java | 46 +++++++++++-------- .../priam/backup/AbstractFileSystem.java | 16 ++++--- .../priam/backup/IBackupFileSystem.java | 4 +- .../priam/backup/IncrementalBackup.java | 26 +++++++---- .../netflix/priam/backup/SnapshotBackup.java | 12 ++++- .../priam/backupv2/SnapshotMetaTask.java | 36 ++++++++------- .../priam/backup/TestAbstractBackup.java | 20 ++++---- .../com/netflix/priam/backup/TestBackup.java | 24 +++++++++- 8 files changed, 121 insertions(+), 63 deletions(-) diff --git a/priam/src/main/java/com/netflix/priam/backup/AbstractBackup.java b/priam/src/main/java/com/netflix/priam/backup/AbstractBackup.java index 82402cde4..2f1733772 100644 --- a/priam/src/main/java/com/netflix/priam/backup/AbstractBackup.java +++ b/priam/src/main/java/com/netflix/priam/backup/AbstractBackup.java @@ -18,8 +18,10 @@ import static java.util.stream.Collectors.toSet; +import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; -import com.google.common.collect.Lists; +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; import com.google.inject.Inject; import com.google.inject.Provider; import com.netflix.priam.backup.AbstractBackupPath.BackupFileType; @@ -29,15 +31,16 @@ import com.netflix.priam.scheduler.Task; import com.netflix.priam.utils.SystemUtils; import java.io.File; +import java.io.FileFilter; import java.io.IOException; import java.nio.file.DirectoryStream; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; import java.util.HashSet; -import java.util.List; +import java.util.Locale; +import java.util.Optional; import java.util.Set; -import java.util.concurrent.Future; import java.util.stream.Stream; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -70,28 +73,22 @@ public AbstractBackup( * @param parent Parent dir * @param type Type of file (META, SST, SNAP etc) * @param async Upload the file(s) in async fashion if enabled. - * @param waitForCompletion wait for completion for all files to upload if using async API. If - * `false` it will queue the files and return with no guarantee to upload. * @return List of files that are successfully uploaded as part of backup * @throws Exception when there is failure in uploading files. */ - protected ImmutableSet upload( - final File parent, final BackupFileType type, boolean async, boolean waitForCompletion) - throws Exception { - ImmutableSet bps = getBackupPaths(parent, type); - final List> futures = Lists.newArrayList(); - for (AbstractBackupPath bp : bps) { + protected ImmutableList> uploadAndDeleteAllFiles( + final File parent, final BackupFileType type, boolean async) throws Exception { + ImmutableSet backupPaths = getBackupPaths(parent, type); + final ImmutableList.Builder> futures = + ImmutableList.builder(); + for (AbstractBackupPath bp : backupPaths) { if (async) futures.add(fs.asyncUploadAndDelete(bp, 10)); - else fs.uploadAndDelete(bp, 10); - } - - // Wait for all files to be uploaded. - if (async && waitForCompletion) { - for (Future future : futures) - future.get(); // This might throw exception if there is any error + else { + fs.uploadAndDelete(bp, 10); + futures.add(Futures.immediateFuture(bp)); + } } - - return bps; + return futures.build(); } protected ImmutableSet getBackupPaths(File dir, BackupFileType type) @@ -219,6 +216,15 @@ public static Set getBackupDirectories(IConfiguration config, String monit return backupPaths; } + protected static File[] getSecondaryIndexDirectories(File backupDir, String columnFamily) { + String reference = "." + columnFamily.toLowerCase(Locale.ROOT); + FileFilter filter = + (file) -> + file.getName().toLowerCase(Locale.ROOT).startsWith(reference) + && isAReadableDirectory(file); + return Optional.ofNullable(backupDir.listFiles(filter)).orElse(new File[] {}); + } + protected static boolean isAReadableDirectory(File dir) { return dir.exists() && dir.isDirectory() && dir.canRead(); } diff --git a/priam/src/main/java/com/netflix/priam/backup/AbstractFileSystem.java b/priam/src/main/java/com/netflix/priam/backup/AbstractFileSystem.java index 2ac1df0db..3e28ecb81 100644 --- a/priam/src/main/java/com/netflix/priam/backup/AbstractFileSystem.java +++ b/priam/src/main/java/com/netflix/priam/backup/AbstractFileSystem.java @@ -20,6 +20,9 @@ import com.google.common.base.Preconditions; import com.google.common.cache.Cache; import com.google.common.cache.CacheBuilder; +import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.ListeningExecutorService; +import com.google.common.util.concurrent.MoreExecutors; import com.google.inject.Inject; import com.google.inject.Provider; import com.netflix.priam.backup.AbstractBackupPath.BackupFileType; @@ -61,7 +64,7 @@ public abstract class AbstractFileSystem implements IBackupFileSystem, EventGene private final IConfiguration configuration; protected final BackupMetrics backupMetrics; private final Set tasksQueued; - private final ThreadPoolExecutor fileUploadExecutor; + private final ListeningExecutorService fileUploadExecutor; private final ThreadPoolExecutor fileDownloadExecutor; // This is going to be a write-thru cache containing the most frequently used items from remote @@ -93,10 +96,11 @@ Also, we may want to have different TIMEOUT for each kind of operation (upload/d .withName(backupMetrics.uploadQueueSize) .monitorSize(uploadQueue); this.fileUploadExecutor = - new BlockingSubmitThreadPoolExecutor( - configuration.getBackupThreads(), - uploadQueue, - configuration.getUploadTimeout()); + MoreExecutors.listeningDecorator( + new BlockingSubmitThreadPoolExecutor( + configuration.getBackupThreads(), + uploadQueue, + configuration.getUploadTimeout())); BlockingQueue downloadQueue = new ArrayBlockingQueue<>(configuration.getDownloadQueueSize()); @@ -152,7 +156,7 @@ protected abstract void downloadFileImpl(final AbstractBackupPath path, String s throws BackupRestoreException; @Override - public Future asyncUploadAndDelete( + public ListenableFuture asyncUploadAndDelete( final AbstractBackupPath path, final int retry) throws RejectedExecutionException { return fileUploadExecutor.submit( () -> { diff --git a/priam/src/main/java/com/netflix/priam/backup/IBackupFileSystem.java b/priam/src/main/java/com/netflix/priam/backup/IBackupFileSystem.java index 269189545..7999958a2 100644 --- a/priam/src/main/java/com/netflix/priam/backup/IBackupFileSystem.java +++ b/priam/src/main/java/com/netflix/priam/backup/IBackupFileSystem.java @@ -16,6 +16,7 @@ */ package com.netflix.priam.backup; +import com.google.common.util.concurrent.ListenableFuture; import java.io.FileNotFoundException; import java.nio.file.Path; import java.util.Date; @@ -89,7 +90,8 @@ void uploadAndDelete(AbstractBackupPath path, int retry) * @throws RejectedExecutionException if the queue is full and TIMEOUT is reached while trying * to add the work to the queue. */ - Future asyncUploadAndDelete(final AbstractBackupPath path, final int retry) + ListenableFuture asyncUploadAndDelete( + final AbstractBackupPath path, final int retry) throws FileNotFoundException, RejectedExecutionException, BackupRestoreException; /** diff --git a/priam/src/main/java/com/netflix/priam/backup/IncrementalBackup.java b/priam/src/main/java/com/netflix/priam/backup/IncrementalBackup.java index dc6d338be..a466855c7 100644 --- a/priam/src/main/java/com/netflix/priam/backup/IncrementalBackup.java +++ b/priam/src/main/java/com/netflix/priam/backup/IncrementalBackup.java @@ -16,6 +16,10 @@ */ package com.netflix.priam.backup; +import com.google.common.collect.ImmutableList; +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.MoreExecutors; import com.google.inject.Inject; import com.google.inject.Provider; import com.google.inject.Singleton; @@ -26,9 +30,7 @@ import com.netflix.priam.scheduler.SimpleTimer; import com.netflix.priam.scheduler.TaskTimer; import java.io.File; -import java.io.FileFilter; import java.nio.file.Path; -import java.util.Optional; import java.util.Set; import org.apache.commons.io.FileUtils; import org.slf4j.Logger; @@ -118,14 +120,22 @@ protected void processColumnFamily(String keyspace, String columnFamily, File ba backupRestoreConfig.enableV2Backups() ? BackupFileType.SST_V2 : BackupFileType.SST; // upload SSTables and components - upload(backupDir, fileType, config.enableAsyncIncremental(), true); + ImmutableList> futures = + uploadAndDeleteAllFiles(backupDir, fileType, config.enableAsyncIncremental()); + Futures.whenAllComplete(futures).call(() -> null, MoreExecutors.directExecutor()); // Next, upload secondary indexes - FileFilter filter = - (file) -> - file.getName().startsWith("." + columnFamily) && isAReadableDirectory(file); - for (File subDir : Optional.ofNullable(backupDir.listFiles(filter)).orElse(new File[] {})) { - upload(subDir, fileType, config.enableAsyncIncremental(), true); + fileType = BackupFileType.SECONDARY_INDEX_V2; + for (File dir : getSecondaryIndexDirectories(backupDir, columnFamily)) { + futures = uploadAndDeleteAllFiles(dir, fileType, config.enableAsyncIncremental()); + Futures.whenAllComplete(futures) + .call( + () -> { + if (FileUtils.sizeOfDirectory(dir) == 0) + FileUtils.deleteQuietly(dir); + return null; + }, + MoreExecutors.directExecutor()); } } } diff --git a/priam/src/main/java/com/netflix/priam/backup/SnapshotBackup.java b/priam/src/main/java/com/netflix/priam/backup/SnapshotBackup.java index 5024b102d..6a16b0104 100644 --- a/priam/src/main/java/com/netflix/priam/backup/SnapshotBackup.java +++ b/priam/src/main/java/com/netflix/priam/backup/SnapshotBackup.java @@ -16,7 +16,9 @@ */ package com.netflix.priam.backup; +import com.google.common.collect.ImmutableList; import com.google.common.collect.Lists; +import com.google.common.util.concurrent.ListenableFuture; import com.google.inject.Inject; import com.google.inject.Provider; import com.google.inject.Singleton; @@ -36,6 +38,7 @@ import java.nio.file.Path; import java.time.Instant; import java.util.*; +import java.util.concurrent.Future; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; import org.apache.commons.io.FileUtils; @@ -209,8 +212,13 @@ protected void processColumnFamily(String keyspace, String columnFamily, File ba forgottenFilesManager.findAndMoveForgottenFiles(snapshotInstant, snapshotDir); // Add files to this dir - abstractBackupPaths.addAll( - upload(snapshotDir, BackupFileType.SNAP, config.enableAsyncSnapshot(), true)); + + ImmutableList> futures = + uploadAndDeleteAllFiles( + snapshotDir, BackupFileType.SNAP, config.enableAsyncSnapshot()); + for (Future future : futures) { + abstractBackupPaths.add(future.get()); + } } private static boolean isValidBackupDir(Path backupDir) { diff --git a/priam/src/main/java/com/netflix/priam/backupv2/SnapshotMetaTask.java b/priam/src/main/java/com/netflix/priam/backupv2/SnapshotMetaTask.java index bda0e6899..9af03613b 100644 --- a/priam/src/main/java/com/netflix/priam/backupv2/SnapshotMetaTask.java +++ b/priam/src/main/java/com/netflix/priam/backupv2/SnapshotMetaTask.java @@ -17,7 +17,10 @@ package com.netflix.priam.backupv2; import com.google.common.annotations.VisibleForTesting; +import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSetMultimap; +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; import com.google.inject.Provider; import com.netflix.priam.backup.*; import com.netflix.priam.config.IBackupRestoreConfig; @@ -35,12 +38,12 @@ import java.nio.file.Path; import java.time.Instant; import java.util.Date; -import java.util.List; import java.util.Optional; import java.util.Set; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; -import java.util.stream.Collectors; import javax.inject.Inject; import javax.inject.Named; import javax.inject.Singleton; @@ -79,6 +82,7 @@ public class SnapshotMetaTask extends AbstractBackup { private static final Lock lock = new ReentrantLock(); private final IBackupStatusMgr snapshotStatusMgr; private final InstanceIdentity instanceIdentity; + private final ExecutorService threadPool; private enum MetaStep { META_GENERATION, @@ -106,6 +110,7 @@ private enum MetaStep { config.getSnapshotIncludeCFList(), config.getSnapshotExcludeCFList()); this.metaFileWriter = metaFileWriter; this.metaProxy = metaProxy; + this.threadPool = Executors.newSingleThreadExecutor(); } /** @@ -273,15 +278,22 @@ private void uploadAllFiles(final String columnFamily, final File backupDir) thr // Process each snapshot of SNAPSHOT_PREFIX // We do not want to wait for completion and we just want to add them to queue. This // is to ensure that next run happens on time. - upload(snapshotDirectory, AbstractBackupPath.BackupFileType.SST_V2, true, false); + AbstractBackupPath.BackupFileType type = AbstractBackupPath.BackupFileType.SST_V2; + uploadAndDeleteAllFiles(snapshotDirectory, type, true); // Next, upload secondary indexes + type = AbstractBackupPath.BackupFileType.SECONDARY_INDEX_V2; + ImmutableList> futures; for (File subDir : getSecondaryIndexDirectories(snapshotDirectory, columnFamily)) { - upload( - subDir, - AbstractBackupPath.BackupFileType.SECONDARY_INDEX_V2, - true, - false); + futures = uploadAndDeleteAllFiles(subDir, type, true); + Futures.whenAllComplete(futures) + .call( + () -> { + if (FileUtils.sizeOfDirectory(subDir) == 0) + FileUtils.deleteQuietly(subDir); + return null; + }, + threadPool); } } } @@ -363,14 +375,6 @@ private static Optional getPrefix(File file) { return Optional.ofNullable(prefix); } - private List getSecondaryIndexDirectories(File snapshotDir, String columnFamily) - throws IOException { - return Files.walk(snapshotDir.toPath(), Integer.MAX_VALUE) - .map(Path::toFile) - .filter(f -> f.getName().startsWith("." + columnFamily) && isAReadableDirectory(f)) - .collect(Collectors.toList()); - } - @VisibleForTesting void setSnapshotName(String snapshotName) { this.snapshotName = snapshotName; diff --git a/priam/src/test/java/com/netflix/priam/backup/TestAbstractBackup.java b/priam/src/test/java/com/netflix/priam/backup/TestAbstractBackup.java index 94dc44412..f0abd4d94 100644 --- a/priam/src/test/java/com/netflix/priam/backup/TestAbstractBackup.java +++ b/priam/src/test/java/com/netflix/priam/backup/TestAbstractBackup.java @@ -1,8 +1,8 @@ package com.netflix.priam.backup; -import com.google.appengine.repackaged.com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableSet; +import com.google.common.collect.ImmutableList; import com.google.common.truth.Truth; +import com.google.common.util.concurrent.ListenableFuture; import com.google.inject.Guice; import com.google.inject.Injector; import com.google.inject.Provider; @@ -115,13 +115,15 @@ public String getName() { public void testCorrectCompressionType() throws Exception { File parent = new File(DIRECTORY); AbstractBackupPath.BackupFileType backupFileType = AbstractBackupPath.BackupFileType.SST_V2; - ImmutableSet paths = - abstractBackup.upload(parent, backupFileType, false, false); - AbstractBackupPath abstractBackupPath = - paths.stream() - .filter(path -> path.getFileName().equals(tablePart)) - .findAny() - .orElseThrow(IllegalStateException::new); + ImmutableList> futures = + abstractBackup.uploadAndDeleteAllFiles(parent, backupFileType, false); + AbstractBackupPath abstractBackupPath = null; + for (ListenableFuture future : futures) { + if (future.get().getFileName().equals(tablePart)) { + abstractBackupPath = future.get(); + break; + } + } Truth.assertThat(abstractBackupPath.getCompression()).isEqualTo(compressionAlgorithm); } } diff --git a/priam/src/test/java/com/netflix/priam/backup/TestBackup.java b/priam/src/test/java/com/netflix/priam/backup/TestBackup.java index f4cb1dc44..70e8b6d79 100644 --- a/priam/src/test/java/com/netflix/priam/backup/TestBackup.java +++ b/priam/src/test/java/com/netflix/priam/backup/TestBackup.java @@ -17,6 +17,7 @@ package com.netflix.priam.backup; +import com.google.common.collect.Iterators; import com.google.inject.Guice; import com.google.inject.Injector; import com.google.inject.Key; @@ -26,6 +27,7 @@ import java.io.FileOutputStream; import java.io.IOException; import java.util.HashSet; +import java.util.Iterator; import java.util.Set; import mockit.Mock; import mockit.MockUp; @@ -96,6 +98,25 @@ public void testIncrementalBackup() throws Exception { Assert.assertTrue(filesystem.uploadedFiles.contains(filePath)); } + @Test + public void testIncrementalBackupOfSecondaryIndexes() throws Exception { + filesystem.cleanup(); + generateIncrementalFiles(); + IncrementalBackup backup = injector.getInstance(IncrementalBackup.class); + File secondaryIndexBackupDir = + new File("target/data/Keyspace1/Standard1/backups/.STANDARD1_field1_idx_1/"); + Assert.assertTrue(secondaryIndexBackupDir.exists()); + backup.execute(); + Iterator paths = + filesystem.listFileSystem("casstestbackup", "/", null /* marker */); + String path = + Iterators.find(paths, p -> p.endsWith("Keyspace1-Standard1-ia-4-Data.db"), null); + Assert.assertNotNull(path); + Assert.assertTrue( + path.contains(AbstractBackupPath.BackupFileType.SECONDARY_INDEX_V2.name())); + Assert.assertFalse(secondaryIndexBackupDir.exists()); + } + @Test public void testClusterSpecificColumnFamiliesSkippedBefore21() throws Exception { String[] columnFamilyDirs = {"schema_columns", "local", "peers", "LocationInfo"}; @@ -156,8 +177,9 @@ private static void generateIncrementalFiles() { files.add("target/data/Keyspace1/Standard1/backups/Keyspace1-Standard1-ia-1-Index.db"); files.add("target/data/Keyspace1/Standard1/backups/Keyspace1-Standard1-ia-2-Data.db"); files.add("target/data/Keyspace1/Standard1/backups/Keyspace1-Standard1-ia-3-Data.db"); + // purposely testing case mismatch in secondary index directory which can exist in practice files.add( - "target/data/Keyspace1/Standard1/backups/.Standard1_field1_idx_1/Keyspace1-Standard1-ia-4-Data.db"); + "target/data/Keyspace1/Standard1/backups/.STANDARD1_field1_idx_1/Keyspace1-Standard1-ia-4-Data.db"); expectedFiles.clear(); for (String filePath : files) { From a0795dc86a87ee68811703e54268df2ca9178f98 Mon Sep 17 00:00:00 2001 From: Matt Lehman Date: Tue, 25 May 2021 15:16:35 -0700 Subject: [PATCH 168/228] Changelog commit in advance of 3.11.79 --- CHANGELOG.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8aa289834..d7d4cf370 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,14 @@ # Changelog +## 2021/05/25 3.11.79 +(#922) Optionally skip compression of backups +(#943) Optionally make extra check to ensure Thrift server is actually listening on the rpc_port +(#944) Make disk_access_mode a first class configuration option +(#941) Delete secondary index backup directories after uploading +(#940) Improve operator control over creation of ingress rules + +## 2021/05/12 3.11.78 +Improve operator control over creation of ingress rules + ## 2021/03/17 3.11.77 (#913) Store Private IPs in the token database when the snitch is GPFS. From c313fdddbbf15a91641cdd81ae88e1fb69f8367f Mon Sep 17 00:00:00 2001 From: mattl-netflix <63665634+mattl-netflix@users.noreply.github.com> Date: Mon, 7 Jun 2021 22:23:14 -0700 Subject: [PATCH 169/228] Revert behavior back to truncating milliseconds in backups last modified time. (#948) --- .../netflix/priam/aws/RemoteBackupPath.java | 5 ++++- .../priam/aws/TestRemoteBackupPath.java | 20 +++++++++++++------ 2 files changed, 18 insertions(+), 7 deletions(-) diff --git a/priam/src/main/java/com/netflix/priam/aws/RemoteBackupPath.java b/priam/src/main/java/com/netflix/priam/aws/RemoteBackupPath.java index a7fb7098a..f4e8cb07f 100644 --- a/priam/src/main/java/com/netflix/priam/aws/RemoteBackupPath.java +++ b/priam/src/main/java/com/netflix/priam/aws/RemoteBackupPath.java @@ -84,7 +84,10 @@ private String removeHash(String appNameWithHash) { */ private String getV2Location() { ImmutableList.Builder parts = getV2Prefix(); - parts.add(type.toString(), getLastModified().toEpochMilli() + ""); + // JDK-8177809 truncate to seconds to ensure consistent behavior with our old method of + // getting lastModified time (File::lastModified) in Java 8. + long lastModified = getLastModified().toEpochMilli() / 1_000L * 1_000L; + parts.add(type.toString(), lastModified + ""); if (BackupFileType.isDataFile(type)) { parts.add(keyspace, columnFamily); } diff --git a/priam/src/test/java/com/netflix/priam/aws/TestRemoteBackupPath.java b/priam/src/test/java/com/netflix/priam/aws/TestRemoteBackupPath.java index ad845914c..72ede456e 100644 --- a/priam/src/test/java/com/netflix/priam/aws/TestRemoteBackupPath.java +++ b/priam/src/test/java/com/netflix/priam/aws/TestRemoteBackupPath.java @@ -154,7 +154,9 @@ public void testV2BackupPathSST() { AbstractBackupPath abstractBackupPath2 = pathFactory.get(); abstractBackupPath2.parseRemote(remotePath); - Assert.assertEquals(now, abstractBackupPath2.getLastModified()); + Assert.assertEquals( + now.toEpochMilli() / 1_000L * 1_000L, + abstractBackupPath2.getLastModified().toEpochMilli()); validateAbstractBackupPath(abstractBackupPath, abstractBackupPath2); } @@ -183,7 +185,9 @@ public void testV2BackupPathMeta() { AbstractBackupPath abstractBackupPath2 = pathFactory.get(); abstractBackupPath2.parseRemote(remotePath); - Assert.assertEquals(now, abstractBackupPath2.getLastModified()); + Assert.assertEquals( + now.toEpochMilli() / 1_000L * 1_000L, + abstractBackupPath2.getLastModified().toEpochMilli()); validateAbstractBackupPath(abstractBackupPath, abstractBackupPath2); } @@ -214,15 +218,17 @@ public void testV2BackupPathSecondaryIndex() { abstractBackupPath.setLastModified(now); String remotePath = abstractBackupPath.getRemotePath(); logger.info(remotePath); + long correctLastModified = +now.toEpochMilli() / 1_000L * 1_000L; Assert.assertEquals( "casstestbackup/1049_fake-app/1808575600/SECONDARY_INDEX_V2/" - + now.toEpochMilli() + + correctLastModified + "/keyspace1/columnfamily1/.columnfamily1_field1_idx/SNAPPY/PLAINTEXT/mc-1234-Data.db", remotePath); AbstractBackupPath abstractBackupPath2 = pathFactory.get(); abstractBackupPath2.parseRemote(remotePath); - Assert.assertEquals(now, abstractBackupPath2.getLastModified()); + Assert.assertEquals( + correctLastModified, abstractBackupPath2.getLastModified().toEpochMilli()); validateAbstractBackupPath(abstractBackupPath, abstractBackupPath2); } @@ -254,15 +260,17 @@ public void testV2SnapshotPathSecondaryIndex() { abstractBackupPath.setLastModified(now); String remotePath = abstractBackupPath.getRemotePath(); logger.info(remotePath); + long correctLastModified = +now.toEpochMilli() / 1_000L * 1_000L; Assert.assertEquals( "casstestbackup/1049_fake-app/1808575600/SECONDARY_INDEX_V2/" - + now.toEpochMilli() + + correctLastModified + "/keyspace1/columnfamily1/.columnfamily1_field1_idx/SNAPPY/PLAINTEXT/mc-1234-Data.db", remotePath); AbstractBackupPath abstractBackupPath2 = pathFactory.get(); abstractBackupPath2.parseRemote(remotePath); - Assert.assertEquals(now, abstractBackupPath2.getLastModified()); + Assert.assertEquals( + correctLastModified, abstractBackupPath2.getLastModified().toEpochMilli()); validateAbstractBackupPath(abstractBackupPath, abstractBackupPath2); } From 2718dc4eb1abad7347e66d7f63b6d6806bc3927d Mon Sep 17 00:00:00 2001 From: Matt Lehman Date: Mon, 7 Jun 2021 22:24:42 -0700 Subject: [PATCH 170/228] CHANGELOG commit in advance of 3.11.80 --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index d7d4cf370..2fd960dbe 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,7 @@ # Changelog +## 2021/06/07 3.11.80 +(#948) Reverting back to previous behavior of omitting milliseconds in backup last modified times. + ## 2021/05/25 3.11.79 (#922) Optionally skip compression of backups (#943) Optionally make extra check to ensure Thrift server is actually listening on the rpc_port From 9fa2b6345739975400e3064e6eda3bc86e022fee Mon Sep 17 00:00:00 2001 From: mattl-netflix <63665634+mattl-netflix@users.noreply.github.com> Date: Fri, 11 Jun 2021 14:34:15 -0700 Subject: [PATCH 171/228] Fix overflow problem in restore. (#952) --- priam/src/main/java/com/netflix/priam/aws/S3FileSystem.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/priam/src/main/java/com/netflix/priam/aws/S3FileSystem.java b/priam/src/main/java/com/netflix/priam/aws/S3FileSystem.java index b175ef56b..363058d76 100644 --- a/priam/src/main/java/com/netflix/priam/aws/S3FileSystem.java +++ b/priam/src/main/java/com/netflix/priam/aws/S3FileSystem.java @@ -52,7 +52,7 @@ @Singleton public class S3FileSystem extends S3FileSystemBase { private static final Logger logger = LoggerFactory.getLogger(S3FileSystem.class); - private static final int MAX_BUFFER_SIZE = 5 * 1024 * 1024; + private static final long MAX_BUFFER_SIZE = 5L * 1024L * 1024L; @Inject public S3FileSystem( @@ -77,7 +77,7 @@ protected void downloadFileImpl(AbstractBackupPath path, String suffix) String remotePath = path.getRemotePath(); File localFile = new File(path.newRestoreFile().getAbsolutePath() + suffix); long size = super.getFileSize(remotePath); - final int bufferSize = Math.min(MAX_BUFFER_SIZE, Math.toIntExact(size)); + final int bufferSize = Math.toIntExact(Math.min(MAX_BUFFER_SIZE, size)); try (BufferedInputStream is = new BufferedInputStream( new RangeReadInputStream(s3Client, getShard(), size, remotePath), From 6afef5c41b546209a0c450fc35d64a7303e355e2 Mon Sep 17 00:00:00 2001 From: Matt Lehman Date: Fri, 11 Jun 2021 14:38:51 -0700 Subject: [PATCH 172/228] Updating CHANGELOG in advance of 3.11.81 --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2fd960dbe..fb631fa64 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,7 @@ # Changelog +## 2021/06/11 3.11.81 +(#952) Fixing integer overflow problem in restore. + ## 2021/06/07 3.11.80 (#948) Reverting back to previous behavior of omitting milliseconds in backup last modified times. From 3b55a4b57f4c9ac98df95c753c360e95d1aa4975 Mon Sep 17 00:00:00 2001 From: mattl-netflix <63665634+mattl-netflix@users.noreply.github.com> Date: Fri, 9 Jul 2021 15:03:24 -0700 Subject: [PATCH 173/228] Use different versions of some preconditions checks to keep code consistent with 2.1 branch. (#958) --- .../com/netflix/priam/aws/RemoteBackupPath.java | 16 +++++++++------- .../netflix/priam/backup/AbstractFileSystem.java | 6 ++++-- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/priam/src/main/java/com/netflix/priam/aws/RemoteBackupPath.java b/priam/src/main/java/com/netflix/priam/aws/RemoteBackupPath.java index f4e8cb07f..b3616d10a 100644 --- a/priam/src/main/java/com/netflix/priam/aws/RemoteBackupPath.java +++ b/priam/src/main/java/com/netflix/priam/aws/RemoteBackupPath.java @@ -69,8 +69,8 @@ private String removeHash(String appNameWithHash) { String appName = appNameWithHash.substring(appNameWithHash.indexOf("_") + 1); Preconditions.checkArgument( hash == appName.hashCode() % 10000, - "Prepended hash does not match app name. Should have received: %s", - prependHash(appName)); + "Prepended hash does not match app name. Should have received: " + + prependHash(appName)); return appName; } @@ -100,7 +100,8 @@ private String getV2Location() { private void parseV2Location(Path remotePath) { Preconditions.checkArgument( - remotePath.getNameCount() >= 8, "%s has fewer than %d parts", remotePath, 8); + remotePath.getNameCount() >= 8, + String.format("%s has fewer than %d parts", remotePath, 8)); int index = 0; baseDir = remotePath.getName(index++).toString(); clusterName = removeHash(remotePath.getName(index++).toString()); @@ -144,7 +145,8 @@ private Path toPath(ImmutableList parts) { private void parseV1Location(Path remotePath) { Preconditions.checkArgument( - remotePath.getNameCount() >= 7, "%s has fewer than %d parts", remotePath, 7); + remotePath.getNameCount() >= 7, + String.format("%s has fewer than %d parts", remotePath, 7)); parseV1Prefix(remotePath); time = DateUtil.getDate(remotePath.getName(4).toString()); type = BackupFileType.valueOf(remotePath.getName(5).toString()); @@ -157,7 +159,8 @@ private void parseV1Location(Path remotePath) { private void parseV1Prefix(Path remotePath) { Preconditions.checkArgument( - remotePath.getNameCount() >= 4, "%s needs %d parts to parse prefix", remotePath, 4); + remotePath.getNameCount() >= 4, + String.format("%s needs %d parts to parse prefix", remotePath, 4)); baseDir = remotePath.getName(0).toString(); region = remotePath.getName(1).toString(); clusterName = remotePath.getName(2).toString(); @@ -224,8 +227,7 @@ public String clusterPrefix(String location) { String[] elements = location.split(String.valueOf(RemoteBackupPath.PATH_SEP)); Preconditions.checkArgument( elements.length < 2 || elements.length > 3, - "Path must have fewer than 2 or greater than 3 elements. Saw %s", - location); + "Path must have fewer than 2 or greater than 3 elements. Saw " + location); if (elements.length <= 1) { baseDir = config.getBackupLocation(); region = instanceIdentity.getInstanceInfo().getRegion(); diff --git a/priam/src/main/java/com/netflix/priam/backup/AbstractFileSystem.java b/priam/src/main/java/com/netflix/priam/backup/AbstractFileSystem.java index 3e28ecb81..84b74d9bc 100644 --- a/priam/src/main/java/com/netflix/priam/backup/AbstractFileSystem.java +++ b/priam/src/main/java/com/netflix/priam/backup/AbstractFileSystem.java @@ -170,9 +170,11 @@ public void uploadAndDelete(final AbstractBackupPath path, final int retry) throws BackupRestoreException { Path localPath = Paths.get(path.getBackupFile().getAbsolutePath()); File localFile = localPath.toFile(); - Preconditions.checkArgument(localFile.exists(), "Can't upload nonexistent {}", localPath); Preconditions.checkArgument( - !localFile.isDirectory(), "Can only upload files {} is a directory", localPath); + localFile.exists(), String.format("Can't upload nonexistent %s", localPath)); + Preconditions.checkArgument( + !localFile.isDirectory(), + String.format("Can only upload files %s is a directory", localPath)); Path remotePath = Paths.get(path.getRemotePath()); if (tasksQueued.add(localPath)) { From 8da8e9404aa5d73013ced8f9baaa5d5e8850e27e Mon Sep 17 00:00:00 2001 From: mattl-netflix <63665634+mattl-netflix@users.noreply.github.com> Date: Fri, 9 Jul 2021 15:04:40 -0700 Subject: [PATCH 174/228] CASS-2153 Don't throw on failure to delete old backup directories. (#956) --- .../java/com/netflix/priam/backupv2/SnapshotMetaTask.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/priam/src/main/java/com/netflix/priam/backupv2/SnapshotMetaTask.java b/priam/src/main/java/com/netflix/priam/backupv2/SnapshotMetaTask.java index 9af03613b..07a322e48 100644 --- a/priam/src/main/java/com/netflix/priam/backupv2/SnapshotMetaTask.java +++ b/priam/src/main/java/com/netflix/priam/backupv2/SnapshotMetaTask.java @@ -269,9 +269,8 @@ private void uploadAllFiles(final String columnFamily, final File backupDir) thr if (!snapshotDirectory.getName().startsWith(SNAPSHOT_PREFIX) || !snapshotDirectory.isDirectory()) continue; - if (snapshotDirectory.list().length == 0) { - FileUtils.cleanDirectory(snapshotDirectory); - FileUtils.deleteDirectory(snapshotDirectory); + if (FileUtils.sizeOfDirectory(snapshotDirectory) == 0) { + FileUtils.deleteQuietly(snapshotDirectory); continue; } From b316c90cd7a9a32a2c4ca8245e5b1a36414cb345 Mon Sep 17 00:00:00 2001 From: mattl-netflix <63665634+mattl-netflix@users.noreply.github.com> Date: Sun, 11 Jul 2021 21:07:55 -0700 Subject: [PATCH 175/228] Only upload secondary index directories if they exist in the main data directory. (#955) * Preliminary refactoring in advance of fixing si directory name bug * Fix filtering issue with secondary index directory names. * Switch to secondary index backup policy of uploading all readable directories that start with a '.' --- .../netflix/priam/backup/AbstractBackup.java | 26 +++++++-------- .../priam/backup/CommitLogBackupTask.java | 3 +- .../priam/backup/IncrementalBackup.java | 23 ++++++------- .../netflix/priam/backup/SnapshotBackup.java | 4 +-- .../priam/backupv2/SnapshotMetaTask.java | 33 ++++++++++--------- .../priam/backup/TestAbstractBackup.java | 2 +- .../com/netflix/priam/backup/TestBackup.java | 8 +++++ 7 files changed, 53 insertions(+), 46 deletions(-) diff --git a/priam/src/main/java/com/netflix/priam/backup/AbstractBackup.java b/priam/src/main/java/com/netflix/priam/backup/AbstractBackup.java index 2f1733772..e64f7ec4f 100644 --- a/priam/src/main/java/com/netflix/priam/backup/AbstractBackup.java +++ b/priam/src/main/java/com/netflix/priam/backup/AbstractBackup.java @@ -38,7 +38,6 @@ import java.nio.file.Path; import java.nio.file.Paths; import java.util.HashSet; -import java.util.Locale; import java.util.Optional; import java.util.Set; import java.util.stream.Stream; @@ -158,29 +157,34 @@ protected final void initiateBackup( for (File columnFamilyDir : columnFamilyDirectories) { File backupDir = new File(columnFamilyDir, monitoringFolder); if (isAReadableDirectory(backupDir)) { - String columnFamilyName = columnFamilyDir.getName().split("-")[0]; + String columnFamilyName = getColumnFamily(backupDir); if (backupRestoreUtil.isFiltered(keyspaceDir.getName(), columnFamilyName)) { // Clean the backup/snapshot directory else files will keep getting // accumulated. SystemUtils.cleanupDir(backupDir.getAbsolutePath(), null); } else { - processColumnFamily(keyspaceDir.getName(), columnFamilyName, backupDir); + processColumnFamily(backupDir); } } } // end processing all CFs for keyspace } // end processing keyspaces under the C* data dir } + protected String getColumnFamily(File backupDir) { + return backupDir.getParentFile().getName().split("-")[0]; + } + + protected String getKeyspace(File backupDir) { + return backupDir.getParentFile().getName(); + } + /** * Process the columnfamily in a given snapshot/backup directory. * - * @param keyspace Name of the keyspace - * @param columnFamily Name of the columnfamily * @param backupDir Location of the backup/snapshot directory in that columnfamily. * @throws Exception throws exception if there is any error in process the directory. */ - protected abstract void processColumnFamily( - String keyspace, String columnFamily, File backupDir) throws Exception; + protected abstract void processColumnFamily(File backupDir) throws Exception; /** * Get all the backup directories for Cassandra. @@ -216,12 +220,8 @@ public static Set getBackupDirectories(IConfiguration config, String monit return backupPaths; } - protected static File[] getSecondaryIndexDirectories(File backupDir, String columnFamily) { - String reference = "." + columnFamily.toLowerCase(Locale.ROOT); - FileFilter filter = - (file) -> - file.getName().toLowerCase(Locale.ROOT).startsWith(reference) - && isAReadableDirectory(file); + protected static File[] getSecondaryIndexDirectories(File backupDir) { + FileFilter filter = (file) -> file.getName().startsWith(".") && isAReadableDirectory(file); return Optional.ofNullable(backupDir.listFiles(filter)).orElse(new File[] {}); } diff --git a/priam/src/main/java/com/netflix/priam/backup/CommitLogBackupTask.java b/priam/src/main/java/com/netflix/priam/backup/CommitLogBackupTask.java index 016675a10..6286f4ef8 100644 --- a/priam/src/main/java/com/netflix/priam/backup/CommitLogBackupTask.java +++ b/priam/src/main/java/com/netflix/priam/backup/CommitLogBackupTask.java @@ -65,8 +65,7 @@ public static TaskTimer getTimer(IConfiguration config) { } @Override - protected void processColumnFamily(String keyspace, String columnFamily, File backupDir) - throws Exception { + protected void processColumnFamily(File columnFamilyDirectory) { // Do nothing. } } diff --git a/priam/src/main/java/com/netflix/priam/backup/IncrementalBackup.java b/priam/src/main/java/com/netflix/priam/backup/IncrementalBackup.java index a466855c7..f0222f60a 100644 --- a/priam/src/main/java/com/netflix/priam/backup/IncrementalBackup.java +++ b/priam/src/main/java/com/netflix/priam/backup/IncrementalBackup.java @@ -114,8 +114,7 @@ public String getName() { } @Override - protected void processColumnFamily(String keyspace, String columnFamily, File backupDir) - throws Exception { + protected void processColumnFamily(File backupDir) throws Exception { BackupFileType fileType = backupRestoreConfig.enableV2Backups() ? BackupFileType.SST_V2 : BackupFileType.SST; @@ -126,16 +125,18 @@ protected void processColumnFamily(String keyspace, String columnFamily, File ba // Next, upload secondary indexes fileType = BackupFileType.SECONDARY_INDEX_V2; - for (File dir : getSecondaryIndexDirectories(backupDir, columnFamily)) { - futures = uploadAndDeleteAllFiles(dir, fileType, config.enableAsyncIncremental()); + for (File directory : getSecondaryIndexDirectories(backupDir)) { + futures = uploadAndDeleteAllFiles(directory, fileType, config.enableAsyncIncremental()); + if (futures.isEmpty()) { + deleteIfEmpty(directory); + } Futures.whenAllComplete(futures) - .call( - () -> { - if (FileUtils.sizeOfDirectory(dir) == 0) - FileUtils.deleteQuietly(dir); - return null; - }, - MoreExecutors.directExecutor()); + .call(() -> deleteIfEmpty(directory), MoreExecutors.directExecutor()); } } + + private Void deleteIfEmpty(File dir) { + if (FileUtils.sizeOfDirectory(dir) == 0) FileUtils.deleteQuietly(dir); + return null; + } } diff --git a/priam/src/main/java/com/netflix/priam/backup/SnapshotBackup.java b/priam/src/main/java/com/netflix/priam/backup/SnapshotBackup.java index 6a16b0104..babc61c9b 100644 --- a/priam/src/main/java/com/netflix/priam/backup/SnapshotBackup.java +++ b/priam/src/main/java/com/netflix/priam/backup/SnapshotBackup.java @@ -200,9 +200,7 @@ private static void cleanOldBackups(IConfiguration configuration) throws Excepti } @Override - protected void processColumnFamily(String keyspace, String columnFamily, File backupDir) - throws Exception { - + protected void processColumnFamily(File backupDir) throws Exception { File snapshotDir = getValidSnapshot(backupDir, snapshotName); if (snapshotDir == null) { diff --git a/priam/src/main/java/com/netflix/priam/backupv2/SnapshotMetaTask.java b/priam/src/main/java/com/netflix/priam/backupv2/SnapshotMetaTask.java index 07a322e48..4c85a6bb1 100644 --- a/priam/src/main/java/com/netflix/priam/backupv2/SnapshotMetaTask.java +++ b/priam/src/main/java/com/netflix/priam/backupv2/SnapshotMetaTask.java @@ -258,7 +258,7 @@ public String getName() { return JOBNAME; } - private void uploadAllFiles(final String columnFamily, final File backupDir) throws Exception { + private void uploadAllFiles(final File backupDir) throws Exception { // Process all the snapshots with SNAPSHOT_PREFIX. This will ensure that we "resume" the // uploads of previous snapshot leftover as Priam restarted or any failure for any reason // (like we exhausted the wait time for upload) @@ -283,31 +283,32 @@ private void uploadAllFiles(final String columnFamily, final File backupDir) thr // Next, upload secondary indexes type = AbstractBackupPath.BackupFileType.SECONDARY_INDEX_V2; ImmutableList> futures; - for (File subDir : getSecondaryIndexDirectories(snapshotDirectory, columnFamily)) { + for (File subDir : getSecondaryIndexDirectories(snapshotDirectory)) { futures = uploadAndDeleteAllFiles(subDir, type, true); - Futures.whenAllComplete(futures) - .call( - () -> { - if (FileUtils.sizeOfDirectory(subDir) == 0) - FileUtils.deleteQuietly(subDir); - return null; - }, - threadPool); + if (futures.isEmpty()) { + deleteIfEmpty(subDir); + } + Futures.whenAllComplete(futures).call(() -> deleteIfEmpty(subDir), threadPool); } } } } + private Void deleteIfEmpty(File dir) { + if (FileUtils.sizeOfDirectory(dir) == 0) FileUtils.deleteQuietly(dir); + return null; + } + @Override - protected void processColumnFamily( - final String keyspace, final String columnFamily, final File backupDir) - throws Exception { + protected void processColumnFamily(File backupDir) throws Exception { + String keyspace = getKeyspace(backupDir); + String columnFamily = getColumnFamily(backupDir); switch (metaStep) { case META_GENERATION: generateMetaFile(keyspace, columnFamily, backupDir); break; case UPLOAD_FILES: - uploadAllFiles(columnFamily, backupDir); + uploadAllFiles(backupDir); break; default: throw new Exception("Unknown meta file type: " + metaStep); @@ -330,9 +331,9 @@ private void generateMetaFile( builder.putAll(getSSTables(snapshotDir, AbstractBackupPath.BackupFileType.SST_V2)); // Next, add secondary indexes - for (File subDir : getSecondaryIndexDirectories(snapshotDir, columnFamily)) { + for (File directory : getSecondaryIndexDirectories(backupDir)) { builder.putAll( - getSSTables(subDir, AbstractBackupPath.BackupFileType.SECONDARY_INDEX_V2)); + getSSTables(directory, AbstractBackupPath.BackupFileType.SECONDARY_INDEX_V2)); } ImmutableSetMultimap sstables = builder.build(); diff --git a/priam/src/test/java/com/netflix/priam/backup/TestAbstractBackup.java b/priam/src/test/java/com/netflix/priam/backup/TestAbstractBackup.java index f0abd4d94..882373a48 100644 --- a/priam/src/test/java/com/netflix/priam/backup/TestAbstractBackup.java +++ b/priam/src/test/java/com/netflix/priam/backup/TestAbstractBackup.java @@ -99,7 +99,7 @@ public TestAbstractBackup(BackupsToCompress which, String tablePart, Compression abstractBackup = new AbstractBackup(fakeConfiguration, context, pathFactory) { @Override - protected void processColumnFamily(String ks, String cf, File dir) {} + protected void processColumnFamily(File dir) {} @Override public void execute() {} diff --git a/priam/src/test/java/com/netflix/priam/backup/TestBackup.java b/priam/src/test/java/com/netflix/priam/backup/TestBackup.java index 70e8b6d79..3c374213f 100644 --- a/priam/src/test/java/com/netflix/priam/backup/TestBackup.java +++ b/priam/src/test/java/com/netflix/priam/backup/TestBackup.java @@ -180,6 +180,14 @@ private static void generateIncrementalFiles() { // purposely testing case mismatch in secondary index directory which can exist in practice files.add( "target/data/Keyspace1/Standard1/backups/.STANDARD1_field1_idx_1/Keyspace1-Standard1-ia-4-Data.db"); + File fileToSkip = + new File( + "target/data/Keyspace1/Standard1/backups/.foo/Keyspace1-Standard1-ia-5-Data.db"); + if (!fileToSkip.exists()) fileToSkip.mkdirs(); + File siDir = + new File( + "target/data/Keyspace1/Standard1/.STANDARD1_field1_idx_1/Keyspace1-Standard1-ia-5-Data.db"); + if (!siDir.exists()) siDir.mkdirs(); expectedFiles.clear(); for (String filePath : files) { From b18afa379d9a0a17f501b8b1be126ecb88c51398 Mon Sep 17 00:00:00 2001 From: Matt Lehman Date: Sun, 11 Jul 2021 21:09:19 -0700 Subject: [PATCH 176/228] Update CHANGELOG in advance of 3.11.82 --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index fb631fa64..f9a5da7db 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,7 @@ # Changelog +## 2021/07/11 3.11.82 +(#955) Upload and delete any backup directory starting with a '.' + ## 2021/06/11 3.11.81 (#952) Fixing integer overflow problem in restore. From 74490a58a175ac01adcdfbf36ba4c023794b2613 Mon Sep 17 00:00:00 2001 From: mattl-netflix <63665634+mattl-netflix@users.noreply.github.com> Date: Wed, 28 Jul 2021 14:15:33 -0700 Subject: [PATCH 177/228] Cease printing column family names in metafiles in place of keyspace names. (#960) --- .../src/main/java/com/netflix/priam/backup/AbstractBackup.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/priam/src/main/java/com/netflix/priam/backup/AbstractBackup.java b/priam/src/main/java/com/netflix/priam/backup/AbstractBackup.java index e64f7ec4f..583a5d3f3 100644 --- a/priam/src/main/java/com/netflix/priam/backup/AbstractBackup.java +++ b/priam/src/main/java/com/netflix/priam/backup/AbstractBackup.java @@ -175,7 +175,7 @@ protected String getColumnFamily(File backupDir) { } protected String getKeyspace(File backupDir) { - return backupDir.getParentFile().getName(); + return backupDir.getParentFile().getParentFile().getName(); } /** From 7c9695b713ec59ce86cc03c8975484b4af8da89e Mon Sep 17 00:00:00 2001 From: Matt Lehman Date: Wed, 28 Jul 2021 14:17:01 -0700 Subject: [PATCH 178/228] Update CHANGELOG in advance of 3.11.83 --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index f9a5da7db..d0e6a8862 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,7 @@ # Changelog +## 2021/07/28 3.11.83 +(#960) Fix bug in metafile generation where column family names are printed in place of keyspace names. + ## 2021/07/11 3.11.82 (#955) Upload and delete any backup directory starting with a '.' From 2786739b35c891ba2f4bde4c8592c8adbbe3dae0 Mon Sep 17 00:00:00 2001 From: Roberto Perez Alcolea Date: Mon, 16 Aug 2021 23:09:06 -0700 Subject: [PATCH 179/228] rotate TravisCI secrets --- .travis.yml | 12 ++++++------ secrets/signing-key.enc | Bin 6800 -> 6800 bytes 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.travis.yml b/.travis.yml index 864d3bc02..0399928ed 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,9 +11,9 @@ cache: - "$HOME/.gradle" env: global: - - secure: JUZT8CmNPjpmKguIioCY0quM5ZGzO03fv7YMDZqR909WbfiX57ArPJOorD9U1ri8REiRjhIe5dTgQMNtar6i1uJj0EwhCIahRn+CNJhtHBxA1siX3/E8HU8ANirLLziBQlWnnjwKlGoUKw04plMLkaA9Ybn3WV9sdi/kVHRpI2Y= - - secure: wxVMsVmsKEuksiQlkm47gAH6Pu5lM3zwWVXJw1eUf9zTqLVwqomuQ19Bgv2ZDcoWNBdcfRouhV98uCclCgFD3tPTVfJ3GSrXhpQLstGihNRtZE5EmEVj8rephVa4H1g0BxzuhbFCgV6VyQWECvAYpyyutk6+ygeS7uhjWtUCTus= - - secure: wDpYNAJGRtlnbJUfmnmmTSH5+TTf5AccjbaVa1NVblQPb0gdupEo8lGPeKGyaAWtteHlC+ih5nP6nh2geoWc6Svfv7fXRdffVyCF+WLrTPKvPOgC1A9IgrbVbjKRJv3Lqx+i6vYcRc+EJ3Bn9Z5AVACY4ag9KfDr5Xc1xmoJK6U= - - secure: XP08P6WaX92eH0lziOoWAAPz0HDpg8RjuxgNQBaQ5NmgyjRypxMnjro2Uqfcivn0tjV8tFQNzGrx2Y1LaC3w1ZJiINuLbSKWNfC/NP/zlMwTfQBwnwx0brmn5u/OwpqmU6l+nrMNyXKKR4hhH47O2ZD3ynrRAivt5tgDo/6xp9Y= - - secure: Gf497SakGlRoyYAqNrto/U+wrfJ9wtUzStwV0uac5CEJoDNN1DG3LA+HmRYgubwXZU3VKb/2BPEyflNV+LOWW54CkWu5Syw4i//iL+FEHv2eDR82NyvOcl11BtmUDLW/G+iBV5J3DlGTSmbgfC195kAYhozxx5FF+UVIYisS0Yc= - - secure: ky97xfe8gq2KqexRhLvaaEI85H/6HJ3iXh+z8eHnRHQpCkL4azVripdaUXwMj5H5EotWevB1nZVdttX/ZU5VIfkyA7H7fgWT9I17o4p8JKA8VItzUAIWopA4pUw2moxKiqhJZ8szqBCeEk5qgbUJyFbsAsHrTmoqSI/8u3WaL0U= + - secure: V+K6B2s3kRV2y9eMJJ32y2vqVFIp8R+lXrjGq+mw3au147XEdbXpDMSh8pOo0AL4MaOWb85yVdNOCrwfvjPg5S98P6ZMt+tSACHj4gt+8odID8BkYWt6yJIgue+fCysVyLfsS9X+7bqHQA3p5KqmIueuqni3E6/c89TKTC0WS7w= + - secure: TlGo0nG+g1+7oW18uNKpt/EL+3BruhpLy4CUMmdZJ9ciyzL+iFtw2Iy8I9UHe/TYzu78D5NEcULdI+CToQp6ELKz2jvEao+iB4MtHkNw00Or7E/bci5QXW0RuEDYsNE0Y3pzYwULZjoV0IWfL2Jq4DJPRPhxCjJtfYHEVBCj/r8= + - secure: s6mhGCg2bn8aFpG1iScsnSHaem9+xrp2I9h4UOGtK+alafuoPnKUP8GLshp58wHYcdUs0zaG4wvw+4i2Ie2QyyUzKTEx1qRe6esxCqx1F8dyw8j51dhXC0lDS8aShHRCakJHefEs1Gi4ECmTfqc2w7A4DFa8ECjtz84cQ8UplqE= + - secure: EpNMulXv9VNVIG1WYPdNppSMVfY0JQ25j1m3Hzb3oqfOkqyhp1pLI54TlF3ks77qgDjyCEa1kk49g4ONvzzh6vp0DH6bLis1LYSvjUXowmGAhluOr8JDlPcBf6BeeVLOXYakxk37/ErYQRAyui2VVcABEspbFM7ANhI8r7Z99eQ= + - secure: jCP93MvQMoIDnyXi4v3tz8ZMVcT365rAMPL4JkzELXFXq91Z/CTKG5SQuh4F9JbKD+yYQPPC0gpABMieER1HqNuSeQ9DsPssD3cG2XZ2oomOdnljilYYj2GFXrFhG+7zfcdCKr8idacQFn33D/pd7O6c6vHrrz6a7KuydiPqy7g= + - secure: F6+6GMeG1HiG0pxUSwBzk6INyzurfPw1DCremO/UBulbGIX1a4JVuPePes9FUCp7l9FIf3b5PH4tOS8XlO9HYL3noh0r26FgB9atoh6xvOERLCtABrGml/qeN7E8VYXCTP7XvaNF/31Yo06rXGw7Owig57hnnEnkmgwgJKbAnIo= diff --git a/secrets/signing-key.enc b/secrets/signing-key.enc index ce0833210e7c7d67f8e1ce477f263c20886463b7..4d3da984f729a65c7e748558541a83b84ffb9900 100644 GIT binary patch literal 6800 zcmV;B8gJ!OVQh3|WM5zBgjpmVodjPkkC9P0sfQd{h3XfMqC~Xz4WOjjoQH?U`>z$t zM0D8MVkJh7VU7#5NPm(uF9;EZKF+67^YU4T4bos9<;`QG#Tg#WX)g@Ya7;OXY zsv69s_pwji{f39Q@f(pO{>E>w0Cvs&tHOT=6w$hHG$5os&vdjYqpgi1Eo9Pn_bWqv zem`8RO+A?ariu;qS$9Q(9_To_d12Dftdk;O2~g&8yf*87!^E%lzGK4(I-rczWU@Vxh@+-a zb}h|>eJcUsnQIS|2Q2iJI7$0vkG5hTkqttHMD33xJXTv6Hy-kH_=0kns9r<3e$OgjHr%Dwz{L^on z=Ywcfb`Nt=`ehYyWqXTBD+o~^>L6buBdam0?1wlE?7-VZo`TbCT|1Ix7-J6Q9Fk zjhZ?gyaZf`lB6wUSkey|B>H6Xn6pddFIsmdP-*(j<6SLE-%BLFAe#xPH7DV>x&V&pRY%8sz%x4^ScP}_wq3M(+rkhl(d}6EbkHDhMIfpes9wU?;d)|ds+iJM)p8oGJ@|68hE!vRb~l{>v5wP zWh{>MGtp7Hv zd!7+_Wq?y5;rGww?t}p9AUShU+umza0wc{~D(0OU$My_iC@NuEPWVND<8}+DMXfwR zX>>R65*Wuk#hF2l)e{hwNCjMIv!WqSLww$0p+2qieSB;(V)bFKM>-Po%Cl$N$elrMJ zmlYU!2Zd72K+ii5#B%LFahU*HL>@VEDXB=O!P~#@o*(%j?eAd9X5UY3gk;&ANCUr> zVJ8`@*Tl<$RGoI`Gdp(aojv|mB?Oi)ejUBAmGdN>M^V-^bv+D6_jJ1WR%lV{utpKA zJ?~SA1GFu2OnrigeVL`>?iv@2t4uX5+X`wrh)~UjE+Auf;h_-A zzMbdpv#8*jhpJtqYF*pm3u? z8ClBN9Bb34XQB4Wmy0mEUbUlvB*lnEvSC2;+Cn5od^aQB=Q$D(WMvG>%h;S*1+2Pz zJ#aWX#ui3uW7bNfO3)o91a`x^S1#u8aGY}Xsa6iT%&2|~#*LA7LP@^h5P6vO5lfG| zIC_W`&I?3;&*h3Cf!%uYC|7D@6l-f`iqGblman=J=Oj#DnV&H=z=$^>0JuO%REIrF zoq48}XS9n*d6+&T)E9oS+Coo;O=%$uox2i2h;HP|%G+}ViZ#Sp>b=`VGWNVz=iZ7BcZT+BEP~KXHwL{4frI%9Cp+#YiaO!CH!JEx$^z8)z&rVP?l+k z3-C?-j{TOXEJ*x=y=DoDF3Dd+0E{9Hn)RP&o` zQ$%4D2hmj1_TulpTU0C}{Z;;Wq#_Gy+yiX#Gwi z1lGO2j(cU<@N!1r@SH=mI`RP^1~7H6UEeHSqJv0S^OwsOsZeh1&nr8MZ7XXMXkV~> z%fmn?>KzCVlY{qbVoole=g%SZpVj;5CyuWv3MF6^EIX1s_cD6cpdaO3+G)2r1q@JF zRfO@+3zMe4qYQ$LQ}S6(D0Y1xb=l;bHx|YksNLtm3%g5x+ef^a0jfrK5M!<0Q&|VC zWTJ9bg4YW7&}+Q5QiC!83F+2phW?K{6`YfLi_EA%P{un>=vOCgDW6q*3h-Ym1ix7+ zH@5QrgmJW40=A7#X>Kk5k{Tb)4L?*i-6cZE|%9M^R9-aQy4pb|vq5}Rij0307| zev|_Ts{@?i9Sz1aiQxHyg&kBqp!0$Agfs^&(W~2#O@hd5IY1C#kv}hY!TSL273v8VZ8(FTHah0jQZKmwD~`ksfPG_9 zY_g;97|*U1F#HEpCPvv3!V}G%V-}aMQO0uAUDvhsL&|=deZVLN@T^js4CgPjp+aM{ zRW721r0W(f(*n(VXNgPCg4XFfw=4bWlqpEGd=oH$(1MthXucvn#1iUtS9RfMEjToA z!8e&JnnAqNsw%q)W2Q52@CpZ;u1WFBHtl1bap)16@8T(KKE6m{31K?W3n;B4h=Gew zl4)C$2cc4Ra{H;cZUZ6KyohI!<`{{K#ksGd;xgK$Fy|;(3-q0xQw!>%IPs#%m#rRp~B6DGHZ51TR7abn1^tev1I{3wpKOIiF_qh z-^Nt{;W(4^4mN1j_*96s=RS?T+~kVqhpz3~TCn?FsP$7?+_b4JTVPFnSL%?o{HHdn zPS35FDm;|B&$J)=j12Pe1j6eG?*_J2sD_OQ=6vSSQYoC|Pl`YxcNv)wXiAPOmco^Fj z{3?wPEMpw-C(9xdrxCAy9})?e%Z=E@=$ez|&r)$K3>!D=pmQJ5%(;W{!I9&4$fUy} zb3ke9B}y_Wq3%Lo8W>tAm%c;4S53gYD3BjqbeIm&rFR%7v#70r#Oa7{hjI;(eD`mt ziGC?ut9hZ&O-xj&>SVHBKFTCov$9puyjcJ+*NulAw;14Ady6wO5ISqcOO% zbp>+4PK5~wD=m5oz$cqrLFv>~@m9Zb`dq{xMP%wu=?2rWvl{fw)`uepJ%(ZvqZ{+# z&TMxl6MN^Ms3qeSD)I-n0L8%cY@eHIFc&hXMk5jtnnjioD$;>I<7?QFuq_6DomU0b zJNgFnDn5NNEe1;e!YomusD63?w9sH|e>dlNP9-t+`aN@F+}nw@qx5HR8*>B4n<*~~ zw}G+e8G7DM-|S{g5a<>xbDYwZrPG5fr3dOv9b0Lw89S+7aa-o;*=@O)!Pb!k_`m6C zumUW&iKz~IChko&l&v);`GI3~+sX(@?qyOwgLxD3K zY8BJ*yvqOxn@a6FdvnlGdUH;-r_5-LAzu8cS-mSXpBnF%b}ZeGYggUpLDq?Lyx_~q zvv{Nzd@Iy3q>Z<^3;3xQQtF`nQXB->k1rg)MEMQl1T}Dm?I2dd;wTsREV*VXKQZ2M zKXqCEW?i+RS>Gf7=>YpqC|8dNfXyuRz^zWpqI02$cEBH4Zv|2skgiirNUbu^M+?cX ztFY4<7=jP3y4jK5@(#~!(?#6fhjFqO@sjd;8i|8xW^@n|BVth?VySiv(0IH`jqZAE zJ?|X0Cpd`If{n7X66NKu*Nu+0<*GcufJ4P)RTC`CGF+sd+YeD&Osy!{poPH1KM8{M zFTy6ZR<(1L*_nbPHE|YENX^^pn9v(4_L<%0K=KZ{OHs_Vq@UEtn}0D8;=nE#79rzK zESW!FP_LpdD!<3()KSnTac^J&@{$SW{6dQXywnX=Zt8dmNp3OsiM_Wp77)VBQV4i; zE8!Om*L&@t%Z&(ldf!=yi1|dr+J&D*!`d9!Pw|LCk}@#tTFR?Wn$H|nks(Hkjrw6%n=@vnELH{4kdS=qPEU)d^#+4KNwusg5H2}*fz@8PNbj!W2%dBBTZw2uu?VzQ*D?O23h zmT(=h7vjJg4>P`!TP67ORow+NTnw4p3@R4I@9xVDWt)8@)1&d?ejPqu559RS(umof z=iu8eVT+F!2np4=@f*-8KeX^k2Y1r}i7<&f@?Hr)nK~;IlyxlwEGa9p{T)6Pn}r;R z{?J6Cn?j+H!exp^oY4dQw4bCj5}gnd&$=IEM-|q(`4ohuKB%G0vg2yp>r(WOcWt(@ zYX0(LKec%*bqV1f|H!Nc&N46?&Qn}9t$wb zY=<>W(1cPZ6>4NWxa8)s={slU1rtp8_-7 znC;*H7wXTMN!6PZsq{VNCYn1P_4{0nS{%K>=57!SYA4gn8AmS06z{7%y%Ie`&AuOP zMx#&4Cw3Rpe3;6)a)}J=ZY5Uv_S0(wJVVG$otgq5tH(Qfh<>d7G>m&z{QNwDq-gX& zlBQxWS4FJFPiURA*#&$dTiaXF3?scr(q4$_7iZZ0Z2H@SmynE{Zzz6bzf5#SpZu_)*2wF@+#{7}1|%B* literal 6800 zcmV;B8gJ!OVQh3|WM5xs`pV8kFyX+=Gru0$5*eNlQx2_`Ym3npRuD}yfbqLdh6&BJ z6mq5A=!sT2ZBIfFi~_J)d#HX%)$*qa?Bk9#@8e1lEk2%XR?RO|p|Rh;9I!~45r^H- zq8$p@Z`f_{L#S{iTbmd}By-(NWF-(dQF*q|=?O}8dJBbz)+e@-8(nv%BdEg5d~=ux zilU1zEp3(v$HH=}Mk+H9TluL=MV1c4J=y{i4Wu>sD+@uoTZpN;_%RcvDWE7I_f z(c(xQEvsoQ?`K1LLLPeV1Rx$Z$pDgGp~EWPu00x?#Gu;R#!OwSpezeDM*$=bSDUhC zlt6dmYye*+`?!UlGn+<)BrB?b6{O_@XTBXQBz?s2Q9dXD)85WdFC8@7W-L>Mb@4sv zbwxi2+!D!+GW7M%{K5T!_WSw$3|JC-npr#Wa2^Rv|Fhigxr%5kX9F=G=Yci@>f}NP z_9j$Vo7P^DQo#gf6yQQPYLIdOUx2v3Jn$pKbtjQsXDE6OS!OnYj=UO*SQlFF2Rjom zHo13-*BS-qah@!sv`t&UmIt!l zL&hF!iG3*yvUKP^CDIZ%Q6j}SyTl;@%J09xE(ramXU7U2oKV|k3@*gRR{l#MOKFWa zku0nGvVFC4zwD%&Ckkujjnhxk)S zedH_sl`$_(QMTFUvuk%&e-Ng!t~b_UlUk zeT=PL@f*lJt237ZJ)ww1vzX=RtH~mXc zRf3&BtqxjH!@kX^?sum4fR|zAv<1Nkn!?FmPU#Y?HyH0f$HoW!QxhfWDbEV-q4$Ol zvy)d158*?9kn39ZMl`Y*9Ldo8&$54F^Pv8jWZlYM2{s*jW8EuCe4+M_Kd335V zT}ep#c^BQ14_LTnnG-k_sOBA;0A|!{=4% z%Sv3n#_AMn%UowlEbAQ08PRY_i1KVUKztw56<>;a+;eXIdx>ZuMNd{qL3OBD-)7qohYBwiXp^rHwn2S5o`(`ZkuCfP(I~nkyrJfrbqo?&m5m~lj&?cG_?P*91}w$qulaoUpsL#7 z#BE{TjtQRWiR23T=8j5H1iqdtc(?qn1?LF|W_nQ(!YOPGuc1UNV- zqeEvH!s%6cVYOScPE>vzNA>i|dRVW;ZkXQ+G`M8@=?#Py7_bDn>WhYJGVF|vpUXz` zaA@rLRB=KdW>thBL#G#dzxC6PCzHMDxrKk8veS+2qchc|ndlUn^ob@|7x))B(B+cS z6B9Df_@1h{e|zsC4jOz6I;4-R(gB&Qee>QK|Oarj<6RKR$oUcb0cPwT1>G$Brk zvp__+(`q$o7s@ttm1Ap|RNd?=OegNk(+JC}M(!K4C)gb;(3f@Fq&2@YW7e-G11?Ud zq@~S=RFMG1FZ{SR&rFbz8q{@VWon)#xqm-pE0eI4hcR-BL+orH)2<_ML;Rl*XXF2w&l>G5=+7`h&i_(H_z}m(B#1a6|`4w<%1wGOF&QzOZfrg!)K4)<%73{ zZ}aquQ_Mu*K?b$LfLAZ8sTAx(o`V9rY8sRXEg&^VJB#`jgc(UcopLKe;3U+;F;OA`;^YxxFI&{pwbRWxYBNGYZRNANn4ex z>AHpEl2E2L|SAVsTz~OsoDj{DfJJt^VOq*ic8qpYp=35`ES?K zsIueQlvL)?RSJw4re8kK*UU9zKnH~ZSE~Z~7HEb#XVUyvbs7mg=5wi~Fdm`AEAY6T zrGX3oHWZ+B2PPJ;NlB{#vDt@ErlF>$15v_DbAoO3fX?ujLSxa7zZ+pC-yPBtx)OAR zz{*pjL;XRmJHvLb)Z%wyw0ORQw_c?r56?Y=?@yCvu+abj(MWn=B-#*S%BJm7q9bG_ zgP>24hPA@Xm7DZ)ZP?UT_)=ABOhk@86CHZ$Ps?mm&GaJ&R1FUQJnS6F8jLxr6bw|C zuR1AP&|gC}`A2?By*S7T&qjEUn3J)<6Q@s5U`zDrPvcY4_u{r|Kj=~kC>?gR&2A5{ zLTUipjq&YKI!Y*rLu{d>ZEoR;*n8D{cCj9dW56-M=yoPmTPWg37z-kfF|BEDs z^$wv|v>U%Jkn@x*mbhA3W(ksMam)R4kdbPUNB!e$$vSmQq$lLa>*nAf^^Fpaqy2w{F}uAU2(66NWydRScLp>c=gz2Tvs zV@c;H-X{tDu1ym}%g~kbhSB~tJ=^6f(j*dUlwws+7Im9wV#1h4mbHyC7bWnOlgegW zwFc(u+8Ug%Fyo%yBRn70EnhgcPVXS)v$nO+IePGZ0?_|?S98c^NT?WxE7H&GnRgtm z5ijm;wbg#Aonduv2Vz2oLTVlQj=;U^RO6_nPm!o=_wCnmv8?Zu%yMoiGzMhG z`vkHIb%9c=Hv3#4z|0i8$>~88gNv5s&=I@lzAQI#h<%+x#6SU($U4RWO}-Z5-m9sp zU?ptn{~rB_lni=y`B;q_S$e_<-zeSXZfCJ z+?B`$)R&zN#cXQM2c69i(aq2e0cMZXjZ5*>D!Xs6PJVOU*$n^OQRV!@Zl!zl0%=2M z0?2{A+3GPFzuf#Wqf=wYVbi_R(7SE;P=93w z5t)cDCHi*zn}kdUOY$>pCYd?(UnYEsGafxQKAK9Jkt*k8peJm-s>^YowzE^A|>40NJ=2lu559S~qtr;96l>ul zmHO5o@D$dT6`Kq{Y7E%tqe$L>)$sJx1~+Ip_yFQX%|bH$?oPN6NgT{v*2YoCb~_z@ z)xh~j1LhePse}>sY{GvuJp>e+Rb`m$FDT4)5Ei4gNQ3in2n48Z5JxHQs2BfWtNdoE*ob|Xi6|n{X zIWggOKMgR+ag{I1q7tvIS`gW++iH*!%s$#|QVr4`s+%}zmQrx{GiDnoSNaQ<1qd;# z?V+TlC69X81kq_Ab9Jw&W2#1 zNHu=%aMb-9!voa0oif#cgjJSJF&@BsFO@enQ2P9y7}I8qe<8Y}w3d4f{?GbAQ<*X@ zzu?V6zp%WD>)x`D@V^N11rD(Y&02CT55N9(4jJrh$u<53 z^0#E|iu4nhgdqAHq@d7KN^Na}@2P7wFhat^(}gu;SWByr46UD4s~O>R>aob`p`rPI zk758&BFY3rD9SNMk|~v)Y51EMkZ31UVhchki&OK93|tpIYx02@rw8@*65@|arjDav!NO4u;b(hcwwy*(*lpAao9 z@vn+f&gL_FbdZVIno08ouQlI%b#3<=OL+>P~%n!>rnyV|00G|=#>IH2*V!BprGj1LhhkM8K;G8fdK%WIW)0vJDA z*XZl)kfSUo>Xym2o=qIe(aejHq$Eu)UTNCU{8!1L^!WBMqG%u%}k4=qOZ zfo4XKu=WzY7F*fG1Xi9p%qw!d_?&bjt-IXs-%%21&P{1Q43TqMzw9_oE1etC8zm?H zDg8F$^b(wn(tW>vyCm^S=u2ku`1}4pTie8Cj4}5l=&;2f_W~5#k!;#7chbk>4B`E> z={i}d9T&M&U53Q24-(IhSQ65Nk(c@>oo(8vjZdbz#$Wpz%bt4xp~XJ09GNRef%k^X z`_VqK1m|H*MiG@b3-^G)5*!{9(YKgXs7|k~1I-cr;eyiQUd8qqTQM?WhMRhiUA8 zVbrC!^Z19Qt(^P|{_9v*dRngN6?CrjOWg&gkz9j7SIH>4VfktYx&5ldD z?R+92&32~V)56U8!eJGf0?AC(Sz^fp4Ft%^Wip3Kq`|}yuDcxGk9b4P7x1x+#u%2eDh_Z*^bM!># zAIlf>;SdH8(eD=;2`&3ygS4rfZ@bv??{=#+K2DBL%hGsFK zo?Wk1gVy0{`i>lF4*#?n&^l}u=RQv$Ha zF6Nu@YQ-n=*^FisFq4z1I_Xv1YZJrKE_&MFI19NKQ=ncAvHk;#5h2b!+)blGa(j^+R^LytMX#_4Vza_*iDCbF^0* zgGIkYJSI}U>0iNXRHo4F&YWfJqbx@#T9T`9F5^i z(3yXdqcf$(4#){07Sc>wBXZbuo=A_Hw+?IW2Y_*IQN-8O zekdC6WNu)bNHZRw@ptv!;cp*=s_GA1tEpL*pGf%b^6hymCEE7U;|mPWz6pS&U}=3X zk0>&!1?}plTnfjbvVdy*=`L!&?Gna;p+_I#hUpuhNbRtD-P;I4SYL_?& z06hPMn4+xZOYt2OiGAVaaRVi^o5#4%@b~p+-*uwbk6DRJjAZ<4SFE+$2tYba>0;&) z-$(fjx=}$_%{3pgJ$0|j--W2wGl))EBBMKeqNlXtyuxb}j0yu{A&?LSVVJjaEP_Bl zIYJI00SFgtcAx{R&?8Q2E&Y=vD46wcHBFg;b^p&5UbX?EjV@*8T#-n-RH8lsi;&}& z$OWIA1jL*@#y?JMAL($M5K_%G1B=H(%{?({~E}h0gPw<(8v(#0rxw(H; zOuWFHU)6rxuLxzCpBU7-T#0{C(9GC2k!{E)fChlrU9)99%j~EJnSeG(U|u%@Vdxrn zob=L4?5`)=INHh5tarq&nXIjW_a$D2&1ZCNZzC`CDPzrmCH<-K5!f5>V1ffWT5mY) zwu6Y2i;;m0-?G9rY3VOF9{9PXfvQqGbPYagP(71GVIJF&ldxEsOW4nOBSx{o@k&Ai ziSm8YrH(Y-a;heO$WDT+_wLIm-q53gt-0$7U565wTA$1|3?_Gfd*W@+MJ^P7yyslr z2iGpzAW_W|oIj#LP-l_!v?lb$Wf4=Hqg6Qwx=#60<178)^>L3DtRxyak+g%H!VdvX zpv`%|S-NT!sU9~EOg{zuhShH$QdH^d50o3Q{L~y#VeL3zO#x1vb}X))JwD4nvJMMA zE71{J8WJ%RRmA$V2Sam7HCX*fJPXWxk1yyo$o^#`eP^iIsn?DWK{Yw4h5v#6*@q|8 z0L$2-sJV3pe+*e;Q|A@dd%;Pcf}Ig*L%E?7etQ9*=Xg_7Y-uI?W>Z! zFtn*i^J{aM4fCam3H;9-I%Z007ekfjsN|h{wb}d8aU6EjUThbmm$o*GuBjXpP+JP+ z>WB=o-*&K--634I*~dmizqx$#f& zFVCvJlJib49Pht=PUH0m3rj;oREzw4L1P)ZpK`Y9j*;PK2$sE0a$i$i>tgQ)Zwe^x z$lq=F1@L;;QsWzPfIa5G$Io*Fv5t%-qa?ILZ#M3ziNjX@;x| Date: Tue, 17 Aug 2021 11:42:12 -0700 Subject: [PATCH 180/228] Remove TravisCI and use Github Actions --- .github/workflows/nebula-ci.yml | 45 +++++++++++++++++++++++ .github/workflows/nebula-publish.yml | 51 ++++++++++++++++++++++++++ .github/workflows/nebula-snapshot.yml | 37 +++++++++++++++++++ .travis.yml | 19 ---------- buildViaTravis.sh | 24 ------------ installViaTravis.sh | 7 ---- secrets/signing-key.enc | Bin 6800 -> 0 bytes 7 files changed, 133 insertions(+), 50 deletions(-) create mode 100644 .github/workflows/nebula-ci.yml create mode 100644 .github/workflows/nebula-publish.yml create mode 100644 .github/workflows/nebula-snapshot.yml delete mode 100644 .travis.yml delete mode 100755 buildViaTravis.sh delete mode 100755 installViaTravis.sh delete mode 100644 secrets/signing-key.enc diff --git a/.github/workflows/nebula-ci.yml b/.github/workflows/nebula-ci.yml new file mode 100644 index 000000000..314ca6f5b --- /dev/null +++ b/.github/workflows/nebula-ci.yml @@ -0,0 +1,45 @@ +name: "CI" +on: + push: + branches: + - '*' + tags-ignore: + - '*' + pull_request: + +jobs: + build: + runs-on: ubuntu-latest + strategy: + matrix: + # test against JDK 8 + java: [ 8 ] + name: CI with Java ${{ matrix.java }} + steps: + - uses: actions/checkout@v1 + - name: Setup jdk + uses: actions/setup-java@v1 + with: + java-version: ${{ matrix.java }} + - uses: actions/cache@v1 + id: gradle-cache + with: + path: ~/.gradle/caches + key: ${{ runner.os }}-gradle-${{ hashFiles('**/gradle/dependency-locks/*.lockfile') }} + restore-keys: | + - ${{ runner.os }}-gradle- + - uses: actions/cache@v1 + id: gradle-wrapper-cache + with: + path: ~/.gradle/wrapper + key: ${{ runner.os }}-gradlewrapper-${{ hashFiles('gradle/wrapper/*') }} + restore-keys: | + - ${{ runner.os }}-gradlewrapper- + - name: Build with Gradle + run: ./gradlew --info --stacktrace build + env: + CI_NAME: github_actions + CI_BUILD_NUMBER: ${{ github.sha }} + CI_BUILD_URL: 'https://github.com/${{ github.repository }}' + CI_BRANCH: ${{ github.ref }} + COVERALLS_REPO_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/nebula-publish.yml b/.github/workflows/nebula-publish.yml new file mode 100644 index 000000000..5e20218a1 --- /dev/null +++ b/.github/workflows/nebula-publish.yml @@ -0,0 +1,51 @@ +name: "Publish candidate/release to NetflixOSS and Maven Central" +on: + push: + tags: + - v*.*.* + - v*.*.*-rc.* + release: + types: + - published + +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v1 + - name: Setup jdk 8 + uses: actions/setup-java@v1 + with: + java-version: 1.8 + - uses: actions/cache@v1 + id: gradle-cache + with: + path: ~/.gradle/caches + key: ${{ runner.os }}-gradle-${{ hashFiles('**/gradle/dependency-locks/*.lockfile') }} + restore-keys: | + - ${{ runner.os }}-gradle- + - uses: actions/cache@v1 + id: gradle-wrapper-cache + with: + path: ~/.gradle/wrapper + key: ${{ runner.os }}-gradlewrapper-${{ hashFiles('gradle/wrapper/*') }} + restore-keys: | + - ${{ runner.os }}-gradlewrapper- + - name: Publish candidate + if: contains(github.ref, '-rc.') + run: ./gradlew --info --stacktrace -Prelease.useLastTag=true candidate + env: + NETFLIX_OSS_SIGNING_KEY: ${{ secrets.ORG_SIGNING_KEY }} + NETFLIX_OSS_SIGNING_PASSWORD: ${{ secrets.ORG_SIGNING_PASSWORD }} + NETFLIX_OSS_REPO_USERNAME: ${{ secrets.ORG_NETFLIXOSS_USERNAME }} + NETFLIX_OSS_REPO_PASSWORD: ${{ secrets.ORG_NETFLIXOSS_PASSWORD }} + - name: Publish release + if: (!contains(github.ref, '-rc.')) + run: ./gradlew --info -Prelease.useLastTag=true final + env: + NETFLIX_OSS_SONATYPE_USERNAME: ${{ secrets.ORG_SONATYPE_USERNAME }} + NETFLIX_OSS_SONATYPE_PASSWORD: ${{ secrets.ORG_SONATYPE_PASSWORD }} + NETFLIX_OSS_SIGNING_KEY: ${{ secrets.ORG_SIGNING_KEY }} + NETFLIX_OSS_SIGNING_PASSWORD: ${{ secrets.ORG_SIGNING_PASSWORD }} + NETFLIX_OSS_REPO_USERNAME: ${{ secrets.ORG_NETFLIXOSS_USERNAME }} + NETFLIX_OSS_REPO_PASSWORD: ${{ secrets.ORG_NETFLIXOSS_PASSWORD }} diff --git a/.github/workflows/nebula-snapshot.yml b/.github/workflows/nebula-snapshot.yml new file mode 100644 index 000000000..b4ee74093 --- /dev/null +++ b/.github/workflows/nebula-snapshot.yml @@ -0,0 +1,37 @@ +name: "Publish snapshot to NetflixOSS and Maven Central" + +on: + push: + branches: + - master + +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 0 + - name: Set up JDK + uses: actions/setup-java@v1 + with: + java-version: 8 + - uses: actions/cache@v2 + id: gradle-cache + with: + path: | + ~/.gradle/caches + key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle') }} + - uses: actions/cache@v2 + id: gradle-wrapper-cache + with: + path: | + ~/.gradle/wrapper + key: ${{ runner.os }}-gradlewrapper-${{ hashFiles('gradle/wrapper/*') }} + - name: Build + run: ./gradlew build snapshot + env: + NETFLIX_OSS_SIGNING_KEY: ${{ secrets.ORG_SIGNING_KEY }} + NETFLIX_OSS_SIGNING_PASSWORD: ${{ secrets.ORG_SIGNING_PASSWORD }} + NETFLIX_OSS_REPO_USERNAME: ${{ secrets.ORG_NETFLIXOSS_USERNAME }} + NETFLIX_OSS_REPO_PASSWORD: ${{ secrets.ORG_NETFLIXOSS_PASSWORD }} diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 0399928ed..000000000 --- a/.travis.yml +++ /dev/null @@ -1,19 +0,0 @@ -language: java -jdk: -- openjdk8 -sudo: false -git: - depth: 100 -install: "./installViaTravis.sh" -script: "./buildViaTravis.sh" -cache: - directories: - - "$HOME/.gradle" -env: - global: - - secure: V+K6B2s3kRV2y9eMJJ32y2vqVFIp8R+lXrjGq+mw3au147XEdbXpDMSh8pOo0AL4MaOWb85yVdNOCrwfvjPg5S98P6ZMt+tSACHj4gt+8odID8BkYWt6yJIgue+fCysVyLfsS9X+7bqHQA3p5KqmIueuqni3E6/c89TKTC0WS7w= - - secure: TlGo0nG+g1+7oW18uNKpt/EL+3BruhpLy4CUMmdZJ9ciyzL+iFtw2Iy8I9UHe/TYzu78D5NEcULdI+CToQp6ELKz2jvEao+iB4MtHkNw00Or7E/bci5QXW0RuEDYsNE0Y3pzYwULZjoV0IWfL2Jq4DJPRPhxCjJtfYHEVBCj/r8= - - secure: s6mhGCg2bn8aFpG1iScsnSHaem9+xrp2I9h4UOGtK+alafuoPnKUP8GLshp58wHYcdUs0zaG4wvw+4i2Ie2QyyUzKTEx1qRe6esxCqx1F8dyw8j51dhXC0lDS8aShHRCakJHefEs1Gi4ECmTfqc2w7A4DFa8ECjtz84cQ8UplqE= - - secure: EpNMulXv9VNVIG1WYPdNppSMVfY0JQ25j1m3Hzb3oqfOkqyhp1pLI54TlF3ks77qgDjyCEa1kk49g4ONvzzh6vp0DH6bLis1LYSvjUXowmGAhluOr8JDlPcBf6BeeVLOXYakxk37/ErYQRAyui2VVcABEspbFM7ANhI8r7Z99eQ= - - secure: jCP93MvQMoIDnyXi4v3tz8ZMVcT365rAMPL4JkzELXFXq91Z/CTKG5SQuh4F9JbKD+yYQPPC0gpABMieER1HqNuSeQ9DsPssD3cG2XZ2oomOdnljilYYj2GFXrFhG+7zfcdCKr8idacQFn33D/pd7O6c6vHrrz6a7KuydiPqy7g= - - secure: F6+6GMeG1HiG0pxUSwBzk6INyzurfPw1DCremO/UBulbGIX1a4JVuPePes9FUCp7l9FIf3b5PH4tOS8XlO9HYL3noh0r26FgB9atoh6xvOERLCtABrGml/qeN7E8VYXCTP7XvaNF/31Yo06rXGw7Owig57hnnEnkmgwgJKbAnIo= diff --git a/buildViaTravis.sh b/buildViaTravis.sh deleted file mode 100755 index c3ddb674c..000000000 --- a/buildViaTravis.sh +++ /dev/null @@ -1,24 +0,0 @@ -#!/bin/bash -# This script will build the project. - -if [ "$TRAVIS_PULL_REQUEST" != "false" ]; then - echo -e "Build Pull Request #$TRAVIS_PULL_REQUEST => Branch [$TRAVIS_BRANCH]" - ./gradlew build -elif [ "$TRAVIS_PULL_REQUEST" == "false" ] && [ "$TRAVIS_TAG" == "" ]; then - echo -e 'Build Branch with Snapshot => Branch ['$TRAVIS_BRANCH']' - ./gradlew -Prelease.travisci=true -PnetflixOss.username="$NETFLIX_OSS_REPO_USERNAME" -PnetflixOss.password="$NETFLIX_OSS_REPO_PASSWORD" -Psonatype.signingPassword="$NETFLIX_OSS_SIGNING_PASSWORD" build snapshot -elif [ "$TRAVIS_PULL_REQUEST" == "false" ] && [ "$TRAVIS_TAG" != "" ]; then - echo -e 'Build Branch for Release => Branch ['$TRAVIS_BRANCH'] Tag ['$TRAVIS_TAG']' - case "$TRAVIS_TAG" in - *-rc\.*) - ./gradlew -Prelease.travisci=true -PnetflixOss.username="$NETFLIX_OSS_REPO_USERNAME" -PnetflixOss.password="$NETFLIX_OSS_REPO_PASSWORD" -Psonatype.signingPassword="$NETFLIX_OSS_SIGNING_PASSWORD" -Prelease.useLastTag=true candidate - ;; - *) - ./gradlew -Prelease.travisci=true -PnetflixOss.username="$NETFLIX_OSS_REPO_USERNAME" -PnetflixOss.password="$NETFLIX_OSS_REPO_PASSWORD" -Psonatype.username="$NETFLIX_OSS_SONATYPE_USERNAME" -Psonatype.password="$NETFLIX_OSS_SONATYPE_PASSWORD" -Psonatype.signingPassword="$NETFLIX_OSS_SIGNING_PASSWORD" -Prelease.useLastTag=true final - ;; - esac -else - echo -e 'WARN: Should not be here => Branch ['$TRAVIS_BRANCH'] Tag ['$TRAVIS_TAG'] Pull Request ['$TRAVIS_PULL_REQUEST']' - ./gradlew build -fi - diff --git a/installViaTravis.sh b/installViaTravis.sh deleted file mode 100755 index 82cf1b880..000000000 --- a/installViaTravis.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/bash -# This script will build the project. - -if [ "$TRAVIS_SECURE_ENV_VARS" = "true" ]; then - echo "Decrypting publishing credentials" - openssl aes-256-cbc -k "$NETFLIX_OSS_SIGNING_FILE_PASSWORD" -in secrets/signing-key.enc -out secrets/signing-key -d -fi diff --git a/secrets/signing-key.enc b/secrets/signing-key.enc deleted file mode 100644 index 4d3da984f729a65c7e748558541a83b84ffb9900..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 6800 zcmV;B8gJ!OVQh3|WM5zBgjpmVodjPkkC9P0sfQd{h3XfMqC~Xz4WOjjoQH?U`>z$t zM0D8MVkJh7VU7#5NPm(uF9;EZKF+67^YU4T4bos9<;`QG#Tg#WX)g@Ya7;OXY zsv69s_pwji{f39Q@f(pO{>E>w0Cvs&tHOT=6w$hHG$5os&vdjYqpgi1Eo9Pn_bWqv zem`8RO+A?ariu;qS$9Q(9_To_d12Dftdk;O2~g&8yf*87!^E%lzGK4(I-rczWU@Vxh@+-a zb}h|>eJcUsnQIS|2Q2iJI7$0vkG5hTkqttHMD33xJXTv6Hy-kH_=0kns9r<3e$OgjHr%Dwz{L^on z=Ywcfb`Nt=`ehYyWqXTBD+o~^>L6buBdam0?1wlE?7-VZo`TbCT|1Ix7-J6Q9Fk zjhZ?gyaZf`lB6wUSkey|B>H6Xn6pddFIsmdP-*(j<6SLE-%BLFAe#xPH7DV>x&V&pRY%8sz%x4^ScP}_wq3M(+rkhl(d}6EbkHDhMIfpes9wU?;d)|ds+iJM)p8oGJ@|68hE!vRb~l{>v5wP zWh{>MGtp7Hv zd!7+_Wq?y5;rGww?t}p9AUShU+umza0wc{~D(0OU$My_iC@NuEPWVND<8}+DMXfwR zX>>R65*Wuk#hF2l)e{hwNCjMIv!WqSLww$0p+2qieSB;(V)bFKM>-Po%Cl$N$elrMJ zmlYU!2Zd72K+ii5#B%LFahU*HL>@VEDXB=O!P~#@o*(%j?eAd9X5UY3gk;&ANCUr> zVJ8`@*Tl<$RGoI`Gdp(aojv|mB?Oi)ejUBAmGdN>M^V-^bv+D6_jJ1WR%lV{utpKA zJ?~SA1GFu2OnrigeVL`>?iv@2t4uX5+X`wrh)~UjE+Auf;h_-A zzMbdpv#8*jhpJtqYF*pm3u? z8ClBN9Bb34XQB4Wmy0mEUbUlvB*lnEvSC2;+Cn5od^aQB=Q$D(WMvG>%h;S*1+2Pz zJ#aWX#ui3uW7bNfO3)o91a`x^S1#u8aGY}Xsa6iT%&2|~#*LA7LP@^h5P6vO5lfG| zIC_W`&I?3;&*h3Cf!%uYC|7D@6l-f`iqGblman=J=Oj#DnV&H=z=$^>0JuO%REIrF zoq48}XS9n*d6+&T)E9oS+Coo;O=%$uox2i2h;HP|%G+}ViZ#Sp>b=`VGWNVz=iZ7BcZT+BEP~KXHwL{4frI%9Cp+#YiaO!CH!JEx$^z8)z&rVP?l+k z3-C?-j{TOXEJ*x=y=DoDF3Dd+0E{9Hn)RP&o` zQ$%4D2hmj1_TulpTU0C}{Z;;Wq#_Gy+yiX#Gwi z1lGO2j(cU<@N!1r@SH=mI`RP^1~7H6UEeHSqJv0S^OwsOsZeh1&nr8MZ7XXMXkV~> z%fmn?>KzCVlY{qbVoole=g%SZpVj;5CyuWv3MF6^EIX1s_cD6cpdaO3+G)2r1q@JF zRfO@+3zMe4qYQ$LQ}S6(D0Y1xb=l;bHx|YksNLtm3%g5x+ef^a0jfrK5M!<0Q&|VC zWTJ9bg4YW7&}+Q5QiC!83F+2phW?K{6`YfLi_EA%P{un>=vOCgDW6q*3h-Ym1ix7+ zH@5QrgmJW40=A7#X>Kk5k{Tb)4L?*i-6cZE|%9M^R9-aQy4pb|vq5}Rij0307| zev|_Ts{@?i9Sz1aiQxHyg&kBqp!0$Agfs^&(W~2#O@hd5IY1C#kv}hY!TSL273v8VZ8(FTHah0jQZKmwD~`ksfPG_9 zY_g;97|*U1F#HEpCPvv3!V}G%V-}aMQO0uAUDvhsL&|=deZVLN@T^js4CgPjp+aM{ zRW721r0W(f(*n(VXNgPCg4XFfw=4bWlqpEGd=oH$(1MthXucvn#1iUtS9RfMEjToA z!8e&JnnAqNsw%q)W2Q52@CpZ;u1WFBHtl1bap)16@8T(KKE6m{31K?W3n;B4h=Gew zl4)C$2cc4Ra{H;cZUZ6KyohI!<`{{K#ksGd;xgK$Fy|;(3-q0xQw!>%IPs#%m#rRp~B6DGHZ51TR7abn1^tev1I{3wpKOIiF_qh z-^Nt{;W(4^4mN1j_*96s=RS?T+~kVqhpz3~TCn?FsP$7?+_b4JTVPFnSL%?o{HHdn zPS35FDm;|B&$J)=j12Pe1j6eG?*_J2sD_OQ=6vSSQYoC|Pl`YxcNv)wXiAPOmco^Fj z{3?wPEMpw-C(9xdrxCAy9})?e%Z=E@=$ez|&r)$K3>!D=pmQJ5%(;W{!I9&4$fUy} zb3ke9B}y_Wq3%Lo8W>tAm%c;4S53gYD3BjqbeIm&rFR%7v#70r#Oa7{hjI;(eD`mt ziGC?ut9hZ&O-xj&>SVHBKFTCov$9puyjcJ+*NulAw;14Ady6wO5ISqcOO% zbp>+4PK5~wD=m5oz$cqrLFv>~@m9Zb`dq{xMP%wu=?2rWvl{fw)`uepJ%(ZvqZ{+# z&TMxl6MN^Ms3qeSD)I-n0L8%cY@eHIFc&hXMk5jtnnjioD$;>I<7?QFuq_6DomU0b zJNgFnDn5NNEe1;e!YomusD63?w9sH|e>dlNP9-t+`aN@F+}nw@qx5HR8*>B4n<*~~ zw}G+e8G7DM-|S{g5a<>xbDYwZrPG5fr3dOv9b0Lw89S+7aa-o;*=@O)!Pb!k_`m6C zumUW&iKz~IChko&l&v);`GI3~+sX(@?qyOwgLxD3K zY8BJ*yvqOxn@a6FdvnlGdUH;-r_5-LAzu8cS-mSXpBnF%b}ZeGYggUpLDq?Lyx_~q zvv{Nzd@Iy3q>Z<^3;3xQQtF`nQXB->k1rg)MEMQl1T}Dm?I2dd;wTsREV*VXKQZ2M zKXqCEW?i+RS>Gf7=>YpqC|8dNfXyuRz^zWpqI02$cEBH4Zv|2skgiirNUbu^M+?cX ztFY4<7=jP3y4jK5@(#~!(?#6fhjFqO@sjd;8i|8xW^@n|BVth?VySiv(0IH`jqZAE zJ?|X0Cpd`If{n7X66NKu*Nu+0<*GcufJ4P)RTC`CGF+sd+YeD&Osy!{poPH1KM8{M zFTy6ZR<(1L*_nbPHE|YENX^^pn9v(4_L<%0K=KZ{OHs_Vq@UEtn}0D8;=nE#79rzK zESW!FP_LpdD!<3()KSnTac^J&@{$SW{6dQXywnX=Zt8dmNp3OsiM_Wp77)VBQV4i; zE8!Om*L&@t%Z&(ldf!=yi1|dr+J&D*!`d9!Pw|LCk}@#tTFR?Wn$H|nks(Hkjrw6%n=@vnELH{4kdS=qPEU)d^#+4KNwusg5H2}*fz@8PNbj!W2%dBBTZw2uu?VzQ*D?O23h zmT(=h7vjJg4>P`!TP67ORow+NTnw4p3@R4I@9xVDWt)8@)1&d?ejPqu559RS(umof z=iu8eVT+F!2np4=@f*-8KeX^k2Y1r}i7<&f@?Hr)nK~;IlyxlwEGa9p{T)6Pn}r;R z{?J6Cn?j+H!exp^oY4dQw4bCj5}gnd&$=IEM-|q(`4ohuKB%G0vg2yp>r(WOcWt(@ zYX0(LKec%*bqV1f|H!Nc&N46?&Qn}9t$wb zY=<>W(1cPZ6>4NWxa8)s={slU1rtp8_-7 znC;*H7wXTMN!6PZsq{VNCYn1P_4{0nS{%K>=57!SYA4gn8AmS06z{7%y%Ie`&AuOP zMx#&4Cw3Rpe3;6)a)}J=ZY5Uv_S0(wJVVG$otgq5tH(Qfh<>d7G>m&z{QNwDq-gX& zlBQxWS4FJFPiURA*#&$dTiaXF3?scr(q4$_7iZZ0Z2H@SmynE{Zzz6bzf5#SpZu_)*2wF@+#{7}1|%B* From 72b51e299f253901331e1c7303fcba520ab4942c Mon Sep 17 00:00:00 2001 From: mattl-netflix <63665634+mattl-netflix@users.noreply.github.com> Date: Thu, 19 Aug 2021 17:23:46 -0700 Subject: [PATCH 181/228] CASS-2201 Ensure SI backup directories are deleted when empty (#961) --- .../java/com/netflix/priam/backup/IncrementalBackup.java | 7 ++++--- .../test/java/com/netflix/priam/backup/TestBackup.java | 8 ++++++++ 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/priam/src/main/java/com/netflix/priam/backup/IncrementalBackup.java b/priam/src/main/java/com/netflix/priam/backup/IncrementalBackup.java index f0222f60a..92907859a 100644 --- a/priam/src/main/java/com/netflix/priam/backup/IncrementalBackup.java +++ b/priam/src/main/java/com/netflix/priam/backup/IncrementalBackup.java @@ -127,11 +127,12 @@ protected void processColumnFamily(File backupDir) throws Exception { fileType = BackupFileType.SECONDARY_INDEX_V2; for (File directory : getSecondaryIndexDirectories(backupDir)) { futures = uploadAndDeleteAllFiles(directory, fileType, config.enableAsyncIncremental()); - if (futures.isEmpty()) { + if (futures.stream().allMatch(ListenableFuture::isDone)) { deleteIfEmpty(directory); + } else { + Futures.whenAllComplete(futures) + .call(() -> deleteIfEmpty(directory), MoreExecutors.directExecutor()); } - Futures.whenAllComplete(futures) - .call(() -> deleteIfEmpty(directory), MoreExecutors.directExecutor()); } } diff --git a/priam/src/test/java/com/netflix/priam/backup/TestBackup.java b/priam/src/test/java/com/netflix/priam/backup/TestBackup.java index 3c374213f..276ace860 100644 --- a/priam/src/test/java/com/netflix/priam/backup/TestBackup.java +++ b/priam/src/test/java/com/netflix/priam/backup/TestBackup.java @@ -26,9 +26,13 @@ import java.io.File; import java.io.FileOutputStream; import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; import java.util.HashSet; import java.util.Iterator; import java.util.Set; +import java.util.stream.Stream; import mockit.Mock; import mockit.MockUp; import org.apache.cassandra.tools.NodeProbe; @@ -96,6 +100,10 @@ public void testIncrementalBackup() throws Exception { Assert.assertEquals(5, filesystem.uploadedFiles.size()); for (String filePath : expectedFiles) Assert.assertTrue(filesystem.uploadedFiles.contains(filePath)); + try (Stream entries = + Files.list(Paths.get("target/data/Keyspace1/Standard1/backups/"))) { + Assert.assertEquals(0, entries.count()); + } } @Test From 485c63ea07e1550dd11c099786430720dd5f867c Mon Sep 17 00:00:00 2001 From: Matt Lehman Date: Fri, 20 Aug 2021 16:30:36 -0700 Subject: [PATCH 182/228] Update CHANGELOG in advance of 3.11.84 release (ensure secondary index directories are deleted once empty) --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index d0e6a8862..236033042 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,7 @@ # Changelog +## 2021/08/18 3.11.84 +(#961) Ensure SI backup directories are deleted when empty + ## 2021/07/28 3.11.83 (#960) Fix bug in metafile generation where column family names are printed in place of keyspace names. From ab0952c092eed26e0345996339c6262a83ab0bff Mon Sep 17 00:00:00 2001 From: mattl-netflix <63665634+mattl-netflix@users.noreply.github.com> Date: Mon, 30 Aug 2021 10:44:59 -0700 Subject: [PATCH 183/228] Return private host name and ip in cases where public versions do not exist. Do this instead of throwing. (#970) --- .../identity/config/AWSInstanceInfo.java | 51 ++++++++------ .../identity/config/TestAWSInstanceInfo.java | 70 +++++++++++++++++++ 2 files changed, 99 insertions(+), 22 deletions(-) create mode 100644 priam/src/test/java/com/netflix/priam/identity/config/TestAWSInstanceInfo.java diff --git a/priam/src/main/java/com/netflix/priam/identity/config/AWSInstanceInfo.java b/priam/src/main/java/com/netflix/priam/identity/config/AWSInstanceInfo.java index b2c104daf..6102c9daa 100644 --- a/priam/src/main/java/com/netflix/priam/identity/config/AWSInstanceInfo.java +++ b/priam/src/main/java/com/netflix/priam/identity/config/AWSInstanceInfo.java @@ -24,6 +24,7 @@ import com.netflix.priam.utils.RetryableCallable; import com.netflix.priam.utils.SystemUtils; import java.util.List; +import java.util.Optional; import org.apache.commons.lang3.StringUtils; import org.codehaus.jettison.json.JSONException; import org.codehaus.jettison.json.JSONObject; @@ -33,11 +34,17 @@ @Singleton public class AWSInstanceInfo implements InstanceInfo { private static final Logger logger = LoggerFactory.getLogger(AWSInstanceInfo.class); + static final String PUBLIC_HOSTNAME_URL = + "http://169.254.169.254/latest/meta-data/public-hostname"; + static final String LOCAL_HOSTNAME_URL = + "http://169.254.169.254/latest/meta-data/local-hostname"; + static final String PUBLIC_HOSTIP_URL = "http://169.254.169.254/latest/meta-data/public-ipv4"; + static final String LOCAL_HOSTIP_URL = "http://169.254.169.254/latest/meta-data/local-ipv4"; private JSONObject identityDocument = null; private String privateIp; - private String publicIp; + private String hostIP; private String rac; - private String publicHostName; + private String hostName; private String instanceId; private String instanceType; private String mac; @@ -88,24 +95,6 @@ public List getDefaultRacks() { return ImmutableList.copyOf(zone); } - public String getPublicHostname() { - if (publicHostName == null) { - publicHostName = - SystemUtils.getDataFromUrl( - "http://169.254.169.254/latest/meta-data/public-hostname"); - } - return publicHostName; - } - - public String getPublicIP() { - if (publicIp == null) { - publicIp = - SystemUtils.getDataFromUrl( - "http://169.254.169.254/latest/meta-data/public-ipv4"); - } - return publicIp; - } - @Override public String getInstanceId() { if (instanceId == null) { @@ -221,11 +210,29 @@ public InstanceEnvironment getInstanceEnvironment() { @Override public String getHostname() { - return (getPublicHostname() == null) ? getPrivateIP() : getPublicHostname(); + if (hostName == null) { + hostName = + tryGetDataFromUrl(PUBLIC_HOSTNAME_URL) + .orElse(SystemUtils.getDataFromUrl(LOCAL_HOSTNAME_URL)); + } + return hostName; } @Override public String getHostIP() { - return (getPublicIP() == null) ? getPrivateIP() : getPublicIP(); + if (hostIP == null) { + hostIP = + tryGetDataFromUrl(PUBLIC_HOSTIP_URL) + .orElse(SystemUtils.getDataFromUrl(LOCAL_HOSTIP_URL)); + } + return hostIP; + } + + Optional tryGetDataFromUrl(String url) { + try { + return Optional.of(SystemUtils.getDataFromUrl(url)); + } catch (Exception e) { + return Optional.empty(); + } } } diff --git a/priam/src/test/java/com/netflix/priam/identity/config/TestAWSInstanceInfo.java b/priam/src/test/java/com/netflix/priam/identity/config/TestAWSInstanceInfo.java new file mode 100644 index 000000000..c71fa663c --- /dev/null +++ b/priam/src/test/java/com/netflix/priam/identity/config/TestAWSInstanceInfo.java @@ -0,0 +1,70 @@ +package com.netflix.priam.identity.config; + +import com.google.common.truth.Truth; +import com.netflix.priam.utils.SystemUtils; +import mockit.Expectations; +import mockit.Mocked; +import org.junit.Before; +import org.junit.Test; + +/** tests of {@link com.netflix.priam.identity.config.AWSInstanceInfo} */ +public class TestAWSInstanceInfo { + private AWSInstanceInfo instanceInfo; + + @Before + public void setUp() { + instanceInfo = + new AWSInstanceInfo( + () -> { + throw new RuntimeException("not implemented"); + }); + } + + @Test + public void testPublicHostIP(@Mocked SystemUtils systemUtils) { + new Expectations() { + { + SystemUtils.getDataFromUrl(AWSInstanceInfo.PUBLIC_HOSTIP_URL); + result = "1.2.3.4"; + } + }; + Truth.assertThat(instanceInfo.getHostIP()).isEqualTo("1.2.3.4"); + } + + @Test + public void testMissingPublicHostIP(@Mocked SystemUtils systemUtils) { + new Expectations() { + { + SystemUtils.getDataFromUrl(AWSInstanceInfo.PUBLIC_HOSTIP_URL); + result = new RuntimeException(); + SystemUtils.getDataFromUrl(AWSInstanceInfo.LOCAL_HOSTIP_URL); + result = "1.2.3.4"; + } + }; + Truth.assertThat(instanceInfo.getHostIP()).isEqualTo("1.2.3.4"); + } + + @Test + public void testPublicHostname(@Mocked SystemUtils systemUtils) { + new Expectations() { + { + SystemUtils.getDataFromUrl(AWSInstanceInfo.PUBLIC_HOSTNAME_URL); + result = "hostname"; + } + }; + Truth.assertThat(instanceInfo.getHostname()).isEqualTo("hostname"); + } + + @Test + public void testMissingPublicHostname(@Mocked SystemUtils systemUtils) { + new Expectations() { + { + SystemUtils.getDataFromUrl(AWSInstanceInfo.PUBLIC_HOSTNAME_URL); + result = new RuntimeException(); + SystemUtils.getDataFromUrl(AWSInstanceInfo.LOCAL_HOSTNAME_URL); + result = "hostname"; + } + }; + Truth.assertThat(instanceInfo.getHostname()).isEqualTo("hostname"); + } +} From 366440762e4f5e37ba910a2e9365d18f7964625b Mon Sep 17 00:00:00 2001 From: mattl-netflix <63665634+mattl-netflix@users.noreply.github.com> Date: Tue, 31 Aug 2021 12:05:19 -0700 Subject: [PATCH 184/228] Throw in case of gossip mismatch when determining replace_ip in assigned token case. (#973) * CASS-1752 Throw in case of gossip mismatch when determining replace_ip in assigned token case. * CASS-1752 Toggle throw-on-mismatch with a property. * Add test of two-node mismatch case --- .../netflix/priam/config/IConfiguration.java | 4 ++ .../priam/config/PriamConfiguration.java | 5 ++ .../priam/identity/token/TokenRetriever.java | 6 +++ .../token/AssignedTokenRetrieverTest.java | 52 +++++++++++++++++++ .../token/TokenRetrieverUtilsTest.java | 36 +++++++++++++ 5 files changed, 103 insertions(+) diff --git a/priam/src/main/java/com/netflix/priam/config/IConfiguration.java b/priam/src/main/java/com/netflix/priam/config/IConfiguration.java index 0111f7fa7..35abf7818 100644 --- a/priam/src/main/java/com/netflix/priam/config/IConfiguration.java +++ b/priam/src/main/java/com/netflix/priam/config/IConfiguration.java @@ -1144,6 +1144,10 @@ default boolean skipIngressUnlessIPIsPublic() { return false; } + default boolean permitDirectTokenAssignmentWithGossipMismatch() { + return false; + } + /** * Escape hatch for getting any arbitrary property by key This is useful so we don't have to * keep adding methods to this interface for every single configuration option ever. Also diff --git a/priam/src/main/java/com/netflix/priam/config/PriamConfiguration.java b/priam/src/main/java/com/netflix/priam/config/PriamConfiguration.java index f2892d43d..d0d449f50 100644 --- a/priam/src/main/java/com/netflix/priam/config/PriamConfiguration.java +++ b/priam/src/main/java/com/netflix/priam/config/PriamConfiguration.java @@ -784,4 +784,9 @@ public String getDiskAccessMode() { public boolean checkThriftServerIsListening() { return config.get(PRIAM_PRE + ".checkThriftServerIsListening", false); } + + @Override + public boolean permitDirectTokenAssignmentWithGossipMismatch() { + return config.get(PRIAM_PRE + ".permitDirectTokenAssignmentWithGossipMismatch", false); + } } diff --git a/priam/src/main/java/com/netflix/priam/identity/token/TokenRetriever.java b/priam/src/main/java/com/netflix/priam/identity/token/TokenRetriever.java index a2280f004..5db933ef8 100644 --- a/priam/src/main/java/com/netflix/priam/identity/token/TokenRetriever.java +++ b/priam/src/main/java/com/netflix/priam/identity/token/TokenRetriever.java @@ -187,6 +187,12 @@ private String getReplacedIpForAssignedToken( "Priam found that the token is not alive according to Cassandra and we should start Cassandra in replace mode with replace ip: " + inferredIp); } + } else if (inferredTokenOwnership.getTokenInformationStatus() + == TokenRetrieverUtils.InferredTokenOwnership.TokenInformationStatus + .MISMATCH + && !config.permitDirectTokenAssignmentWithGossipMismatch()) { + throw new TokenRetrieverUtils.GossipParseException( + "We saw inconsistent results from gossip. Throwing to buy time for it to settle."); } return ipToReplace; } diff --git a/priam/src/test/java/com/netflix/priam/identity/token/AssignedTokenRetrieverTest.java b/priam/src/test/java/com/netflix/priam/identity/token/AssignedTokenRetrieverTest.java index e77f88d6d..94c02cb79 100644 --- a/priam/src/test/java/com/netflix/priam/identity/token/AssignedTokenRetrieverTest.java +++ b/priam/src/test/java/com/netflix/priam/identity/token/AssignedTokenRetrieverTest.java @@ -206,6 +206,56 @@ public void grabAssignedTokenThrowWhenGossipAgreesPreviousTokenOwnerIsLive( factory, membership, config, instanceInfo, tokenRetriever)); } + @Test + public void grabAssignedTokenThrowToBuyTimeWhenGossipDisagreesOnPreviousTokenOwner( + @Mocked IPriamInstanceFactory factory, + @Mocked IConfiguration config, + @Mocked IMembership membership, + @Mocked Sleeper sleeper, + @Mocked ITokenManager tokenManager, + @Mocked InstanceInfo instanceInfo, + @Mocked TokenRetrieverUtils retrievalUtils) { + List liveHosts = newPriamInstances(); + Collections.shuffle(liveHosts); + + TokenRetrieverUtils.InferredTokenOwnership inferredTokenOwnership = + new TokenRetrieverUtils.InferredTokenOwnership(); + inferredTokenOwnership.setTokenInformationStatus( + TokenRetrieverUtils.InferredTokenOwnership.TokenInformationStatus.MISMATCH); + inferredTokenOwnership.setTokenInformation( + new TokenRetrieverUtils.TokenInformation(liveHosts.get(0).getHostIP(), false)); + + new Expectations() { + { + config.getAppName(); + result = APP; + + factory.getAllIds(DEAD_APP); + result = ImmutableSet.of(); + factory.getAllIds(APP); + result = ImmutableSet.copyOf(liveHosts); + + instanceInfo.getInstanceId(); + result = liveHosts.get(0).getInstanceId(); + + TokenRetrieverUtils.inferTokenOwnerFromGossip( + ImmutableSet.copyOf(liveHosts), + liveHosts.get(0).getToken(), + liveHosts.get(0).getDC()); + result = inferredTokenOwnership; + } + }; + + ITokenRetriever tokenRetriever = + new TokenRetriever( + factory, membership, config, instanceInfo, sleeper, tokenManager); + Assertions.assertThrows( + TokenRetrieverUtils.GossipParseException.class, + () -> + new InstanceIdentity( + factory, membership, config, instanceInfo, tokenRetriever)); + } + @Test public void grabAssignedTokenStartDbInBootstrapModeWhenGossipDisagreesOnPreviousTokenOwner( @Mocked IPriamInstanceFactory factory, @@ -230,6 +280,8 @@ public void grabAssignedTokenStartDbInBootstrapModeWhenGossipDisagreesOnPrevious { config.getAppName(); result = APP; + config.permitDirectTokenAssignmentWithGossipMismatch(); + result = true; factory.getAllIds(DEAD_APP); result = ImmutableSet.of(); diff --git a/priam/src/test/java/com/netflix/priam/identity/token/TokenRetrieverUtilsTest.java b/priam/src/test/java/com/netflix/priam/identity/token/TokenRetrieverUtilsTest.java index 7d20e90f6..54a5f4d9f 100644 --- a/priam/src/test/java/com/netflix/priam/identity/token/TokenRetrieverUtilsTest.java +++ b/priam/src/test/java/com/netflix/priam/identity/token/TokenRetrieverUtilsTest.java @@ -6,8 +6,10 @@ import com.google.common.collect.ImmutableSet; import com.netflix.priam.identity.PriamInstance; import com.netflix.priam.utils.SystemUtils; +import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.function.Function; import java.util.stream.Collectors; import java.util.stream.IntStream; import mockit.Expectations; @@ -111,6 +113,40 @@ public void testRetrieveTokenOwnerWhenGossipDisagrees(@Mocked SystemUtils system Assert.assertTrue(inferredTokenOwnership.getTokenInformation().isLive()); } + @Test + public void testRetrieveTokenOwnerWhenGossipDisagrees_2Nodes(@Mocked SystemUtils systemUtils) { + ImmutableSet myInstances = + ImmutableSet.copyOf(instances.stream().limit(3).collect(Collectors.toList())); + List myLiveInstances = liveInstances.stream().limit(3).collect(Collectors.toList()); + Map myTokenToEndpointMap = + IntStream.range(0, 3) + .mapToObj(String::valueOf) + .collect( + Collectors.toMap( + Function.identity(), (i) -> tokenToEndpointMap.get(i))); + Map alteredMap = new HashMap<>(myTokenToEndpointMap); + alteredMap.put("1", "1.2.3.4"); + + new Expectations() { + { + SystemUtils.getDataFromUrl(String.format(STATUS_URL_FORMAT, "fakeHost-0")); + result = getStatus(myLiveInstances, myTokenToEndpointMap); + minTimes = 0; + + SystemUtils.getDataFromUrl(String.format(STATUS_URL_FORMAT, "fakeHost-2")); + result = getStatus(myLiveInstances, alteredMap); + minTimes = 0; + } + }; + + TokenRetrieverUtils.InferredTokenOwnership inferredTokenOwnership = + TokenRetrieverUtils.inferTokenOwnerFromGossip(myInstances, "1", "us-east"); + Assert.assertEquals( + TokenRetrieverUtils.InferredTokenOwnership.TokenInformationStatus.MISMATCH, + inferredTokenOwnership.getTokenInformationStatus()); + Assert.assertTrue(inferredTokenOwnership.getTokenInformation().isLive()); + } + @Test public void testRetrieveTokenOwnerWhenAllHostsInGossipReturnsNull( @Mocked SystemUtils systemUtils) throws Exception { From fc6a46aa9502d074e0e6717367a993c81c7e41ff Mon Sep 17 00:00:00 2001 From: Matt Lehman Date: Tue, 31 Aug 2021 12:43:42 -0700 Subject: [PATCH 185/228] Update CHANGELOG in advance of 3.11.85 --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 236033042..6d875e2b8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,8 @@ # Changelog +## 2021/08/31 3.11.85 +(#973) Throw in case of gossip mismatch when determining replace_ip in assigned token case. +(#970) Make getPublicIP and getPublicHostname fail elegantly when those attributes are absent. + ## 2021/08/18 3.11.84 (#961) Ensure SI backup directories are deleted when empty From b2a161b9b78de5addecfde14304678d7cc3a4088 Mon Sep 17 00:00:00 2001 From: ayushisingh29 Date: Wed, 15 Dec 2021 16:00:46 -0800 Subject: [PATCH 186/228] Use IMDSV2 to get instance metadata (#977) Co-authored-by: ayushis --- build.gradle | 1 + .../netflix/priam/aws/S3FileSystemBase.java | 14 ++-- .../identity/config/AWSInstanceInfo.java | 76 +++++-------------- .../identity/config/TestAWSInstanceInfo.java | 34 ++++----- 4 files changed, 47 insertions(+), 78 deletions(-) diff --git a/build.gradle b/build.gradle index db5eaa911..5ae51364f 100644 --- a/build.gradle +++ b/build.gradle @@ -40,6 +40,7 @@ allprojects { compile 'com.google.code.findbugs:jsr305:3.0.2' // AWS Services + compile 'com.amazonaws:aws-java-sdk-core:latest.release' compile 'com.amazonaws:aws-java-sdk-s3:latest.release' compile 'com.amazonaws:aws-java-sdk-sns:latest.release' compile 'com.amazonaws:aws-java-sdk-ec2:latest.release' diff --git a/priam/src/main/java/com/netflix/priam/aws/S3FileSystemBase.java b/priam/src/main/java/com/netflix/priam/aws/S3FileSystemBase.java index 3fe07736a..4ff3d10df 100755 --- a/priam/src/main/java/com/netflix/priam/aws/S3FileSystemBase.java +++ b/priam/src/main/java/com/netflix/priam/aws/S3FileSystemBase.java @@ -19,11 +19,7 @@ import com.amazonaws.services.s3.model.BucketLifecycleConfiguration.Rule; import com.amazonaws.services.s3.model.CompleteMultipartUploadResult; import com.amazonaws.services.s3.model.DeleteObjectsRequest; -import com.amazonaws.services.s3.model.lifecycle.LifecycleAndOperator; -import com.amazonaws.services.s3.model.lifecycle.LifecycleFilter; -import com.amazonaws.services.s3.model.lifecycle.LifecyclePredicateVisitor; -import com.amazonaws.services.s3.model.lifecycle.LifecyclePrefixPredicate; -import com.amazonaws.services.s3.model.lifecycle.LifecycleTagPredicate; +import com.amazonaws.services.s3.model.lifecycle.*; import com.google.common.collect.Lists; import com.google.common.util.concurrent.RateLimiter; import com.google.inject.Provider; @@ -142,6 +138,14 @@ public void visit(LifecycleTagPredicate lifecycleTagPredicate) {} @Override public void visit(LifecycleAndOperator lifecycleAndOperator) {} + + @Override + public void visit( + LifecycleObjectSizeGreaterThanPredicate lifecycleObjectSizeGreaterThanPredicate) {} + + @Override + public void visit( + LifecycleObjectSizeLessThanPredicate lifecycleObjectSizeLessThanPredicate) {} } private Optional getBucketLifecycleRule(List rules, String prefix) { diff --git a/priam/src/main/java/com/netflix/priam/identity/config/AWSInstanceInfo.java b/priam/src/main/java/com/netflix/priam/identity/config/AWSInstanceInfo.java index 6102c9daa..78a9f0455 100644 --- a/priam/src/main/java/com/netflix/priam/identity/config/AWSInstanceInfo.java +++ b/priam/src/main/java/com/netflix/priam/identity/config/AWSInstanceInfo.java @@ -16,17 +16,15 @@ import com.amazonaws.services.ec2.AmazonEC2; import com.amazonaws.services.ec2.AmazonEC2ClientBuilder; import com.amazonaws.services.ec2.model.*; +import com.amazonaws.util.EC2MetadataUtils; import com.google.common.collect.ImmutableList; import com.google.common.collect.Lists; import com.google.inject.Inject; import com.google.inject.Singleton; import com.netflix.priam.cred.ICredential; import com.netflix.priam.utils.RetryableCallable; -import com.netflix.priam.utils.SystemUtils; import java.util.List; -import java.util.Optional; import org.apache.commons.lang3.StringUtils; -import org.codehaus.jettison.json.JSONException; import org.codehaus.jettison.json.JSONObject; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -34,12 +32,12 @@ @Singleton public class AWSInstanceInfo implements InstanceInfo { private static final Logger logger = LoggerFactory.getLogger(AWSInstanceInfo.class); - static final String PUBLIC_HOSTNAME_URL = - "http://169.254.169.254/latest/meta-data/public-hostname"; - static final String LOCAL_HOSTNAME_URL = - "http://169.254.169.254/latest/meta-data/local-hostname"; - static final String PUBLIC_HOSTIP_URL = "http://169.254.169.254/latest/meta-data/public-ipv4"; - static final String LOCAL_HOSTIP_URL = "http://169.254.169.254/latest/meta-data/local-ipv4"; + + static final String PUBLIC_HOSTNAME_URL = "/latest/meta-data/public-hostname"; + static final String LOCAL_HOSTNAME_URL = "/latest/meta-data/local-hostname"; + static final String PUBLIC_HOSTIP_URL = "/latest/meta-data/public-ipv4"; + static final String LOCAL_HOSTIP_URL = "/latest/meta-data/local-ipv4"; + private JSONObject identityDocument = null; private String privateIp; private String hostIP; @@ -61,9 +59,7 @@ public AWSInstanceInfo(ICredential credential) { @Override public String getPrivateIP() { if (privateIp == null) { - privateIp = - SystemUtils.getDataFromUrl( - "http://169.254.169.254/latest/meta-data/local-ipv4"); + privateIp = EC2MetadataUtils.getPrivateIpAddress(); } return privateIp; } @@ -71,9 +67,7 @@ public String getPrivateIP() { @Override public String getRac() { if (rac == null) { - rac = - SystemUtils.getDataFromUrl( - "http://169.254.169.254/latest/meta-data/placement/availability-zone"); + rac = EC2MetadataUtils.getAvailabilityZone(); } return rac; } @@ -98,9 +92,7 @@ public List getDefaultRacks() { @Override public String getInstanceId() { if (instanceId == null) { - instanceId = - SystemUtils.getDataFromUrl( - "http://169.254.169.254/latest/meta-data/instance-id"); + instanceId = EC2MetadataUtils.getInstanceId(); } return instanceId; } @@ -108,41 +100,21 @@ public String getInstanceId() { @Override public String getInstanceType() { if (instanceType == null) { - instanceType = - SystemUtils.getDataFromUrl( - "http://169.254.169.254/latest/meta-data/instance-type"); + instanceType = EC2MetadataUtils.getInstanceType(); } return instanceType; } private String getMac() { if (mac == null) { - mac = - SystemUtils.getDataFromUrl( - "http://169.254.169.254/latest/meta-data/network/interfaces/macs/") - .trim(); + mac = EC2MetadataUtils.getNetworkInterfaces().get(0).getMacAddress(); } return mac; } @Override public String getRegion() { - try { - getIdentityDocument(); - return this.identityDocument.getString("region"); - } catch (JSONException e) { - // If there is any issue in getting region, use AZ as backup. - return getRac().substring(0, getRac().length() - 1); - } - } - - private void getIdentityDocument() throws JSONException { - if (this.identityDocument == null) { - String jsonStr = - SystemUtils.getDataFromUrl( - "http://169.254.169.254/latest/dynamic/instance-identity/document"); - this.identityDocument = new JSONObject(jsonStr); - } + return EC2MetadataUtils.getEC2InstanceRegion(); } @Override @@ -152,12 +124,7 @@ public String getVpcId() { if (vpcId == null) try { - vpcId = - SystemUtils.getDataFromUrl( - "http://169.254.169.254/latest/meta-data/network/interfaces/macs/" - + nacId - + "vpc-id") - .trim(); + vpcId = EC2MetadataUtils.getNetworkInterfaces().get(0).getVpcId(); } catch (Exception e) { logger.info( "Vpc id does not exist for running instance, not fatal as running instance maybe not be in vpc. Msg: {}", @@ -211,9 +178,9 @@ public InstanceEnvironment getInstanceEnvironment() { @Override public String getHostname() { if (hostName == null) { + String publicHostName = tryGetDataFromUrl(PUBLIC_HOSTNAME_URL); hostName = - tryGetDataFromUrl(PUBLIC_HOSTNAME_URL) - .orElse(SystemUtils.getDataFromUrl(LOCAL_HOSTNAME_URL)); + publicHostName == null ? tryGetDataFromUrl(LOCAL_HOSTNAME_URL) : publicHostName; } return hostName; } @@ -221,18 +188,17 @@ public String getHostname() { @Override public String getHostIP() { if (hostIP == null) { - hostIP = - tryGetDataFromUrl(PUBLIC_HOSTIP_URL) - .orElse(SystemUtils.getDataFromUrl(LOCAL_HOSTIP_URL)); + String publicHostIP = tryGetDataFromUrl(PUBLIC_HOSTIP_URL); + hostIP = publicHostIP == null ? tryGetDataFromUrl(LOCAL_HOSTIP_URL) : publicHostIP; } return hostIP; } - Optional tryGetDataFromUrl(String url) { + String tryGetDataFromUrl(String url) { try { - return Optional.of(SystemUtils.getDataFromUrl(url)); + return EC2MetadataUtils.getData(url); } catch (Exception e) { - return Optional.empty(); + return null; } } } diff --git a/priam/src/test/java/com/netflix/priam/identity/config/TestAWSInstanceInfo.java b/priam/src/test/java/com/netflix/priam/identity/config/TestAWSInstanceInfo.java index c71fa663c..ffc3fd5cf 100644 --- a/priam/src/test/java/com/netflix/priam/identity/config/TestAWSInstanceInfo.java +++ b/priam/src/test/java/com/netflix/priam/identity/config/TestAWSInstanceInfo.java @@ -1,9 +1,7 @@ package com.netflix.priam.identity.config; import com.google.common.truth.Truth; -import com.netflix.priam.utils.SystemUtils; import mockit.Expectations; -import mockit.Mocked; import org.junit.Before; import org.junit.Test; @@ -21,10 +19,10 @@ public void setUp() { } @Test - public void testPublicHostIP(@Mocked SystemUtils systemUtils) { - new Expectations() { + public void testPublicHostIP() { + new Expectations(instanceInfo) { { - SystemUtils.getDataFromUrl(AWSInstanceInfo.PUBLIC_HOSTIP_URL); + instanceInfo.tryGetDataFromUrl(AWSInstanceInfo.PUBLIC_HOSTIP_URL); result = "1.2.3.4"; } }; @@ -32,12 +30,12 @@ public void testPublicHostIP(@Mocked SystemUtils systemUtils) { } @Test - public void testMissingPublicHostIP(@Mocked SystemUtils systemUtils) { - new Expectations() { + public void testMissingPublicHostIP() { + new Expectations(instanceInfo) { { - SystemUtils.getDataFromUrl(AWSInstanceInfo.PUBLIC_HOSTIP_URL); - result = new RuntimeException(); - SystemUtils.getDataFromUrl(AWSInstanceInfo.LOCAL_HOSTIP_URL); + instanceInfo.tryGetDataFromUrl(AWSInstanceInfo.PUBLIC_HOSTIP_URL); + result = null; + instanceInfo.tryGetDataFromUrl(AWSInstanceInfo.LOCAL_HOSTIP_URL); result = "1.2.3.4"; } }; @@ -45,10 +43,10 @@ public void testMissingPublicHostIP(@Mocked SystemUtils systemUtils) { } @Test - public void testPublicHostname(@Mocked SystemUtils systemUtils) { - new Expectations() { + public void testPublicHostname() { + new Expectations(instanceInfo) { { - SystemUtils.getDataFromUrl(AWSInstanceInfo.PUBLIC_HOSTNAME_URL); + instanceInfo.tryGetDataFromUrl(AWSInstanceInfo.PUBLIC_HOSTNAME_URL); result = "hostname"; } }; @@ -56,12 +54,12 @@ public void testPublicHostname(@Mocked SystemUtils systemUtils) { } @Test - public void testMissingPublicHostname(@Mocked SystemUtils systemUtils) { - new Expectations() { + public void testMissingPublicHostname() { + new Expectations(instanceInfo) { { - SystemUtils.getDataFromUrl(AWSInstanceInfo.PUBLIC_HOSTNAME_URL); - result = new RuntimeException(); - SystemUtils.getDataFromUrl(AWSInstanceInfo.LOCAL_HOSTNAME_URL); + instanceInfo.tryGetDataFromUrl(AWSInstanceInfo.PUBLIC_HOSTNAME_URL); + result = null; + instanceInfo.tryGetDataFromUrl(AWSInstanceInfo.LOCAL_HOSTNAME_URL); result = "hostname"; } }; From a0eef8faa3bb895d9164cc5337eecc49ebdbc0c2 Mon Sep 17 00:00:00 2001 From: Matt Lehman Date: Tue, 4 Jan 2022 12:46:46 -0800 Subject: [PATCH 187/228] Update CHANGELOG in advance of 3.11.86 --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6d875e2b8..c647e7abb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,7 @@ # Changelog +## 2022/01/04 3.11.86 +(#977) Use IMDSV2 to get instance metadata + ## 2021/08/31 3.11.85 (#973) Throw in case of gossip mismatch when determining replace_ip in assigned token case. (#970) Make getPublicIP and getPublicHostname fail elegantly when those attributes are absent. From 36bbabbb7998f75cc5663853a2f72f9c27000b91 Mon Sep 17 00:00:00 2001 From: ayushisingh29 Date: Wed, 26 Jan 2022 13:57:34 -0800 Subject: [PATCH 188/228] compute region if not cached (#979) Co-authored-by: ayushis --- .../com/netflix/priam/identity/config/AWSInstanceInfo.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/priam/src/main/java/com/netflix/priam/identity/config/AWSInstanceInfo.java b/priam/src/main/java/com/netflix/priam/identity/config/AWSInstanceInfo.java index 78a9f0455..0a74522da 100644 --- a/priam/src/main/java/com/netflix/priam/identity/config/AWSInstanceInfo.java +++ b/priam/src/main/java/com/netflix/priam/identity/config/AWSInstanceInfo.java @@ -46,6 +46,7 @@ public class AWSInstanceInfo implements InstanceInfo { private String instanceId; private String instanceType; private String mac; + private String region; private String availabilityZone; private ICredential credential; private String vpcId; @@ -114,7 +115,10 @@ private String getMac() { @Override public String getRegion() { - return EC2MetadataUtils.getEC2InstanceRegion(); + if (region == null) { + region = EC2MetadataUtils.getEC2InstanceRegion(); + } + return region; } @Override From 9554a8b6f87b7350f549ac078527ee3a914d34f2 Mon Sep 17 00:00:00 2001 From: Matt Lehman Date: Wed, 26 Jan 2022 13:59:44 -0800 Subject: [PATCH 189/228] Update CHANGELOG in advance of 3.11.87 --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index c647e7abb..307ffbbe3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,7 @@ # Changelog +## 2022/01/25 3.11.87 +(#979) Cache region in AWSInstanceInfo to avoid throttling. + ## 2022/01/04 3.11.86 (#977) Use IMDSV2 to get instance metadata From bc1de420a6bed2386bb6eb970c7f55d894afe539 Mon Sep 17 00:00:00 2001 From: Matt Lehman Date: Wed, 26 Jan 2022 14:51:31 -0800 Subject: [PATCH 190/228] Update CHANGELOG in advance of 3.11.88 --- CHANGELOG.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 307ffbbe3..a908f8e88 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,7 +1,10 @@ # Changelog -## 2022/01/25 3.11.87 +## 2022/01/25 3.11.88 (#979) Cache region in AWSInstanceInfo to avoid throttling. +## 2022/01/25 3.11.87 +Botched release. Disregard + ## 2022/01/04 3.11.86 (#977) Use IMDSV2 to get instance metadata From a52d833dcc1a08579bb753061a6283351cd72412 Mon Sep 17 00:00:00 2001 From: ayushis Date: Wed, 9 Feb 2022 10:16:13 -0800 Subject: [PATCH 191/228] Add Cassandra 4.0 --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 5ae51364f..b8eaa8273 100644 --- a/build.gradle +++ b/build.gradle @@ -54,7 +54,7 @@ allprojects { compile 'com.googlecode.json-simple:json-simple:1.1.1' compile 'org.xerial.snappy:snappy-java:1.1.7.3' compile 'org.yaml:snakeyaml:1.25' - compile 'org.apache.cassandra:cassandra-all:3.0.17' + compile 'org.apache.cassandra:cassandra-all:4.0.0' compile 'javax.ws.rs:jsr311-api:1.1.1' compile 'joda-time:joda-time:2.10.1' compile 'org.apache.commons:commons-configuration2:2.4' From 417c0bfa88522663c131f8703a0b40313de8064d Mon Sep 17 00:00:00 2001 From: ayushis Date: Wed, 9 Feb 2022 10:23:05 -0800 Subject: [PATCH 192/228] relocate jvm.options file and remove deprecated functions --- .../netflix/priam/config/IConfiguration.java | 21 ----------------- .../priam/config/PriamConfiguration.java | 23 +------------------ .../priam/config/FakeConfiguration.java | 2 +- 3 files changed, 2 insertions(+), 44 deletions(-) diff --git a/priam/src/main/java/com/netflix/priam/config/IConfiguration.java b/priam/src/main/java/com/netflix/priam/config/IConfiguration.java index 35abf7818..c6a9dd8e5 100644 --- a/priam/src/main/java/com/netflix/priam/config/IConfiguration.java +++ b/priam/src/main/java/com/netflix/priam/config/IConfiguration.java @@ -220,11 +220,6 @@ default int getSSLStoragePort() { return 7001; } - /** @return Cassandra's thrift port */ - default int getThriftPort() { - return 9160; - } - /** @return Port for CQL binary transport. */ default int getNativeTransportPort() { return 9042; @@ -621,10 +616,6 @@ default boolean isDynamicSnitchEnabled() { return true; } - default boolean isThriftEnabled() { - return true; - } - default boolean isNativeTransportEnabled() { return true; } @@ -641,18 +632,6 @@ default int getConcurrentCompactorsCnt() { return Runtime.getRuntime().availableProcessors(); } - default String getRpcServerType() { - return "hsha"; - } - - default int getRpcMinThreads() { - return 16; - } - - default int getRpcMaxThreads() { - return 2048; - } - /* * @return the warning threshold in MB's for large partitions encountered during compaction. * Default value of 100 is used (default from cassandra.yaml) diff --git a/priam/src/main/java/com/netflix/priam/config/PriamConfiguration.java b/priam/src/main/java/com/netflix/priam/config/PriamConfiguration.java index d0d449f50..a3f4e879d 100644 --- a/priam/src/main/java/com/netflix/priam/config/PriamConfiguration.java +++ b/priam/src/main/java/com/netflix/priam/config/PriamConfiguration.java @@ -165,11 +165,6 @@ public int getNativeTransportPort() { return config.get(PRIAM_PRE + ".nativeTransport.port", 9042); } - @Override - public int getThriftPort() { - return config.get(PRIAM_PRE + ".thrift.port", 9160); - } - @Override public int getStoragePort() { return config.get(PRIAM_PRE + ".storage.port", 7000); @@ -414,7 +409,7 @@ public String getYamlLocation() { @Override public String getJVMOptionsFileLocation() { - return config.get(PRIAM_PRE + ".jvm.options.location", getCassHome() + "/conf/jvm.options"); + return config.get(PRIAM_PRE + ".jvm.options.location", getCassHome() + "/conf/jvm-server.options"); } public String getAuthenticator() { @@ -491,10 +486,6 @@ public boolean isDynamicSnitchEnabled() { return config.get(PRIAM_PRE + ".dsnitchEnabled", true); } - public boolean isThriftEnabled() { - return config.get(PRIAM_PRE + ".thrift.enabled", true); - } - public boolean isNativeTransportEnabled() { return config.get(PRIAM_PRE + ".nativeTransport.enabled", false); } @@ -512,18 +503,6 @@ public int getConcurrentCompactorsCnt() { return config.get(PRIAM_PRE + ".concurrentCompactors", cpus); } - public String getRpcServerType() { - return config.get(PRIAM_PRE + ".rpc.server.type", "hsha"); - } - - public int getRpcMinThreads() { - return config.get(PRIAM_PRE + ".rpc.min.threads", 16); - } - - public int getRpcMaxThreads() { - return config.get(PRIAM_PRE + ".rpc.max.threads", 2048); - } - @Override public int getCompactionLargePartitionWarnThresholdInMB() { return config.get(PRIAM_PRE + ".compaction.large.partition.warn.threshold", 100); diff --git a/priam/src/test/java/com/netflix/priam/config/FakeConfiguration.java b/priam/src/test/java/com/netflix/priam/config/FakeConfiguration.java index e9b61cfff..545c8f126 100644 --- a/priam/src/test/java/com/netflix/priam/config/FakeConfiguration.java +++ b/priam/src/test/java/com/netflix/priam/config/FakeConfiguration.java @@ -152,7 +152,7 @@ public String getYamlLocation() { @Override public String getJVMOptionsFileLocation() { - return "src/test/resources/conf/jvm.options"; + return "src/test/resources/conf/jvm-server.options"; } public String getCommitLogBackupPropsFile() { From 3abe7a148a05c2bc49ef43b4a46cc2f339273476 Mon Sep 17 00:00:00 2001 From: Joseph Lynch Date: Thu, 30 Aug 2018 18:38:40 -0700 Subject: [PATCH 193/228] Updates to Healthchecks and Admin for thrift removal --- .../netflix/priam/health/InstanceState.java | 26 ++++------- .../priam/resources/CassandraAdmin.java | 46 ------------------- 2 files changed, 9 insertions(+), 63 deletions(-) diff --git a/priam/src/main/java/com/netflix/priam/health/InstanceState.java b/priam/src/main/java/com/netflix/priam/health/InstanceState.java index 402f331e0..8d1547354 100644 --- a/priam/src/main/java/com/netflix/priam/health/InstanceState.java +++ b/priam/src/main/java/com/netflix/priam/health/InstanceState.java @@ -39,7 +39,6 @@ public class InstanceState { private final AtomicBoolean shouldCassandraBeAlive = new AtomicBoolean(false); private final AtomicLong lastAttemptedStartTime = new AtomicLong(Long.MAX_VALUE); private final AtomicBoolean isGossipActive = new AtomicBoolean(false); - private final AtomicBoolean isThriftActive = new AtomicBoolean(false); private final AtomicBoolean isNativeTransportActive = new AtomicBoolean(false); private final AtomicBoolean isRequiredDirectoriesExist = new AtomicBoolean(false); private final AtomicBoolean isYmlWritten = new AtomicBoolean(false); @@ -71,15 +70,6 @@ public void setIsGossipActive(boolean isGossipActive) { setHealthy(); } - public boolean isThriftActive() { - return isThriftActive.get(); - } - - public void setIsThriftActive(boolean isThriftActive) { - this.isThriftActive.set(isThriftActive); - setHealthy(); - } - public boolean isNativeTransportActive() { return isNativeTransportActive.get(); } @@ -161,13 +151,15 @@ private boolean isRestoring() { private void setHealthy() { this.isHealthy.set( - isRestoring() - || (isCassandraProcessAlive() - && isRequiredDirectoriesExist() - && isGossipActive() - && isYmlWritten() - && isHealthyOverride() - && (isThriftActive() || isNativeTransportActive()))); + isRestoring() || + (isCassandraProcessAlive() && + isRequiredDirectoriesExist() && + isGossipActive() && + isYmlWritten() && + isHealthyOverride() && + isNativeTransportActive() + ) + ); } public boolean isYmlWritten() { diff --git a/priam/src/main/java/com/netflix/priam/resources/CassandraAdmin.java b/priam/src/main/java/com/netflix/priam/resources/CassandraAdmin.java index a9f2b895c..54208791c 100644 --- a/priam/src/main/java/com/netflix/priam/resources/CassandraAdmin.java +++ b/priam/src/main/java/com/netflix/priam/resources/CassandraAdmin.java @@ -268,52 +268,6 @@ public Response enablegossip() { return Response.ok(REST_SUCCESS, MediaType.APPLICATION_JSON).build(); } - @GET - @Path("/disablethrift") - public Response disablethrift() { - JMXNodeTool nodeTool; - try { - nodeTool = JMXNodeTool.instance(config); - } catch (JMXConnectionException e) { - return Response.status(503).entity("JMXConnectionException").build(); - } - nodeTool.stopThriftServer(); - return Response.ok(REST_SUCCESS, MediaType.APPLICATION_JSON).build(); - } - - @GET - @Path("/enablethrift") - public Response enablethrift() { - JMXNodeTool nodeTool; - try { - nodeTool = JMXNodeTool.instance(config); - } catch (JMXConnectionException e) { - return Response.status(503).entity("JMXConnectionException").build(); - } - nodeTool.startThriftServer(); - return Response.ok(REST_SUCCESS, MediaType.APPLICATION_JSON).build(); - } - - @GET - @Path("/statusthrift") - public Response statusthrift() throws JSONException { - JMXNodeTool nodeTool; - try { - nodeTool = JMXNodeTool.instance(config); - } catch (JMXConnectionException e) { - return Response.status(503).entity("JMXConnectionException").build(); - } - return Response.ok( - new JSONObject() - .put( - "status", - (nodeTool.isThriftServerRunning() - ? "running" - : "not running")), - MediaType.APPLICATION_JSON) - .build(); - } - @GET @Path("/gossipinfo") public Response gossipinfo() throws Exception { From fd7e144dd8092384a9ae2beeaf8ec666589aa12e Mon Sep 17 00:00:00 2001 From: ayushis Date: Wed, 9 Feb 2022 10:33:00 -0800 Subject: [PATCH 194/228] Remove deprecated Cassandra.yaml options from standard tuner --- .../src/main/java/com/netflix/priam/tuner/StandardTuner.java | 5 ----- 1 file changed, 5 deletions(-) diff --git a/priam/src/main/java/com/netflix/priam/tuner/StandardTuner.java b/priam/src/main/java/com/netflix/priam/tuner/StandardTuner.java index 4b511a672..fa542bfb1 100644 --- a/priam/src/main/java/com/netflix/priam/tuner/StandardTuner.java +++ b/priam/src/main/java/com/netflix/priam/tuner/StandardTuner.java @@ -62,8 +62,6 @@ public void writeAllProperties(String yamlLocation, String hostname, String seed map.put("cluster_name", config.getAppName()); map.put("storage_port", config.getStoragePort()); map.put("ssl_storage_port", config.getSSLStoragePort()); - map.put("start_rpc", config.isThriftEnabled()); - map.put("rpc_port", config.getThriftPort()); map.put("start_native_transport", config.isNativeTransportEnabled()); map.put("native_transport_port", config.getNativeTransportPort()); map.put("listen_address", hostname); @@ -113,9 +111,6 @@ public void writeAllProperties(String yamlLocation, String hostname, String seed map.put("concurrent_writes", config.getConcurrentWritesCnt()); map.put("concurrent_compactors", config.getConcurrentCompactorsCnt()); - map.put("rpc_server_type", config.getRpcServerType()); - map.put("rpc_min_threads", config.getRpcMinThreads()); - map.put("rpc_max_threads", config.getRpcMaxThreads()); // Add private ip address as broadcast_rpc_address. This will ensure that COPY function // works correctly. map.put("broadcast_rpc_address", instanceInfo.getPrivateIP()); From 7eb4767461004899905a731f43cbd1ae96d93b44 Mon Sep 17 00:00:00 2001 From: ayushis Date: Wed, 9 Feb 2022 10:39:49 -0800 Subject: [PATCH 195/228] changes in test health --- .../priam/health/TestCassandraMonitor.java | 2 -- .../priam/health/TestInstanceStatus.java | 30 +++++++++---------- 2 files changed, 14 insertions(+), 18 deletions(-) diff --git a/priam/src/test/java/com/netflix/priam/health/TestCassandraMonitor.java b/priam/src/test/java/com/netflix/priam/health/TestCassandraMonitor.java index f4a79fc97..026d25d33 100644 --- a/priam/src/test/java/com/netflix/priam/health/TestCassandraMonitor.java +++ b/priam/src/test/java/com/netflix/priam/health/TestCassandraMonitor.java @@ -91,8 +91,6 @@ NodeProbe instance(IConfiguration config) { result = true; nodeProbe.isNativeTransportRunning(); result = true; - nodeProbe.isThriftServerRunning(); - result = true; } }; // Mock out the ps call diff --git a/priam/src/test/java/com/netflix/priam/health/TestInstanceStatus.java b/priam/src/test/java/com/netflix/priam/health/TestInstanceStatus.java index 1005857cf..9ab55779c 100644 --- a/priam/src/test/java/com/netflix/priam/health/TestInstanceStatus.java +++ b/priam/src/test/java/com/netflix/priam/health/TestInstanceStatus.java @@ -40,61 +40,61 @@ public void testHealth() { // Verify good health. Assert.assertTrue( testInstanceState - .setParams(false, true, true, true, true, true, true, true) + .setParams(false, true, true, true, true, true, true) .isHealthy()); Assert.assertTrue( testInstanceState - .setParams(false, true, true, true, true, false, true, true) + .setParams(false, true, true, true, false, true, true) .isHealthy()); Assert.assertTrue( testInstanceState - .setParams(false, true, true, true, false, true, true, true) + .setParams(false, true, true, true, true, true, true) .isHealthy()); Assert.assertTrue( testInstanceState - .setParams(true, false, true, true, false, true, true, true) + .setParams(true, false, true, true, true, true, true) .isHealthy()); Assert.assertTrue( testInstanceState - .setParams(true, true, false, true, true, true, true, true) + .setParams(true, true, false, true, true, true, true) .isHealthy()); Assert.assertTrue( testInstanceState - .setParams(true, true, true, false, true, true, true, true) + .setParams(true, true, true, false, true, true, true) .isHealthy()); Assert.assertTrue( testInstanceState - .setParams(true, true, true, true, true, true, false, true) + .setParams(true, true, true, true, true, false, true) .isHealthy()); Assert.assertTrue( testInstanceState - .setParams(true, true, true, true, false, false, true, true) + .setParams(true, true, true, true, false, true, true) .isHealthy()); // Negative health case scenarios. Assert.assertFalse( testInstanceState - .setParams(false, false, true, true, false, true, true, true) + .setParams(false, false, true, true, true, true, true) .isHealthy()); Assert.assertFalse( testInstanceState - .setParams(false, true, false, true, true, true, true, true) + .setParams(false, true, false, true, true, true, true) .isHealthy()); Assert.assertFalse( testInstanceState - .setParams(false, true, true, false, true, true, true, true) + .setParams(false, true, true, false, true, true, true) .isHealthy()); Assert.assertFalse( testInstanceState - .setParams(false, true, true, true, true, true, false, true) + .setParams(false, true, true, true, true, false, true) .isHealthy()); Assert.assertFalse( testInstanceState - .setParams(false, true, true, true, false, false, true, true) + .setParams(false, true, true, true, false, true, true) .isHealthy()); Assert.assertFalse( testInstanceState - .setParams(false, true, true, true, false, false, true, false) + .setParams(false, true, true, true, false, true, false) .isHealthy()); } @@ -110,14 +110,12 @@ InstanceState setParams( boolean isYmlWritten, boolean isCassandraProcessAlive, boolean isGossipEnabled, - boolean isThriftEnabled, boolean isNativeEnabled, boolean isRequiredDirectoriesExist, boolean shouldCassandraBeAlive) { instanceState.setYmlWritten(isYmlWritten); instanceState.setCassandraProcessAlive(isCassandraProcessAlive); instanceState.setIsNativeTransportActive(isNativeEnabled); - instanceState.setIsThriftActive(isThriftEnabled); instanceState.setIsGossipActive(isGossipEnabled); instanceState.setIsRequiredDirectoriesExist(isRequiredDirectoriesExist); instanceState.setShouldCassandraBeAlive(shouldCassandraBeAlive); From 1928958a84aa12c92e1658754e36569d599a7e6b Mon Sep 17 00:00:00 2001 From: ayushis Date: Wed, 9 Feb 2022 11:00:33 -0800 Subject: [PATCH 196/228] clean up obsolete info in casandra.yaml --- .../resources/incr-restore-cassandra.yaml | 111 ---------------- .../conf/{jvm.options => jvm-server.options} | 3 - .../resources/incr-restore-cassandra.yaml | 118 ------------------ 3 files changed, 232 deletions(-) rename priam/src/test/resources/conf/{jvm.options => jvm-server.options} (98%) diff --git a/priam/src/main/resources/incr-restore-cassandra.yaml b/priam/src/main/resources/incr-restore-cassandra.yaml index d0d4688b0..f65e88363 100755 --- a/priam/src/main/resources/incr-restore-cassandra.yaml +++ b/priam/src/main/resources/incr-restore-cassandra.yaml @@ -333,65 +333,6 @@ native_transport_port: 9042 # native_transport_min_threads: 16 # native_transport_max_threads: 128 - -# Whether to start the thrift rpc server. -start_rpc: true -# The address to bind the Thrift RPC service to -- clients connect -# here. Unlike ListenAddress above, you *can* specify 0.0.0.0 here if -# you want Thrift to listen on all interfaces. -# -# Leaving this blank has the same effect it does for ListenAddress, -# (i.e. it will be based on the configured hostname of the node). -rpc_address: localhost -# port for Thrift to listen for clients on -rpc_port: 9160 - -# enable or disable keepalive on rpc connections -rpc_keepalive: true - -# Cassandra provides three out-of-the-box options for the RPC Server: -# -# sync -> One thread per thrift connection. For a very large number of clients, memory -# will be your limiting factor. On a 64 bit JVM, 128KB is the minimum stack size -# per thread, and that will correspond to your use of virtual memory (but physical memory -# may be limited depending on use of stack space). -# -# hsha -> Stands for "half synchronous, half asynchronous." All thrift clients are handled -# asynchronously using a small number of threads that does not vary with the amount -# of thrift clients (and thus scales well to many clients). The rpc requests are still -# synchronous (one thread per active request). -# -# The default is sync because on Windows hsha is about 30% slower. On Linux, -# sync/hsha performance is about the same, with hsha of course using less memory. -# -# Alternatively, can provide your own RPC server by providing the fully-qualified class name -# of an o.a.c.t.TServerFactory that can create an instance of it. -rpc_server_type: sync - -# Uncomment rpc_min|max_thread to set request pool size limits. -# -# Regardless of your choice of RPC server (see above), the number of maximum requests in the -# RPC thread pool dictates how many concurrent requests are possible (but if you are using the sync -# RPC server, it also dictates the number of clients that can be connected at all). -# -# The default is unlimited and thus provide no protection against clients overwhelming the server. You are -# encouraged to set a maximum that makes sense for you in production, but do keep in mind that -# rpc_max_threads represents the maximum number of client requests this server may execute concurrently. -# -# rpc_min_threads: 16 -# rpc_max_threads: 2048 - -# uncomment to set socket buffer sizes on rpc connections -# rpc_send_buff_size_in_bytes: -# rpc_recv_buff_size_in_bytes: - -# Frame size for thrift (maximum field length). -thrift_framed_transport_size_in_mb: 15 - -# The max length of a thrift message, including all fields and -# internal thrift overhead. -thrift_max_message_length_in_mb: 16 - # Set to true to have Cassandra create a hard link to each sstable # flushed or streamed locally in a backups/ subdirectory of the # Keyspace data. Removing these links is the operator's @@ -563,58 +504,6 @@ dynamic_snitch_reset_interval_in_ms: 600000 # until the pinned host was 20% worse than the fastest. dynamic_snitch_badness_threshold: 0.1 -# request_scheduler -- Set this to a class that implements -# RequestScheduler, which will schedule incoming client requests -# according to the specific policy. This is useful for multi-tenancy -# with a single Cassandra cluster. -# NOTE: This is specifically for requests from the client and does -# not affect inter node communication. -# org.apache.cassandra.scheduler.NoScheduler - No scheduling takes place -# org.apache.cassandra.scheduler.RoundRobinScheduler - Round robin of -# client requests to a node with a separate queue for each -# request_scheduler_id. The scheduler is further customized by -# request_scheduler_options as described below. -request_scheduler: org.apache.cassandra.scheduler.NoScheduler - -# Scheduler Options vary based on the type of scheduler -# NoScheduler - Has no options -# RoundRobin -# - throttle_limit -- The throttle_limit is the number of in-flight -# requests per client. Requests beyond -# that limit are queued up until -# running requests can complete. -# The value of 80 here is twice the number of -# concurrent_reads + concurrent_writes. -# - default_weight -- default_weight is optional and allows for -# overriding the default which is 1. -# - weights -- Weights are optional and will default to 1 or the -# overridden default_weight. The weight translates into how -# many requests are handled during each turn of the -# RoundRobin, based on the scheduler id. -# -# request_scheduler_options: -# throttle_limit: 80 -# default_weight: 5 -# weights: -# Keyspace1: 1 -# Keyspace2: 5 - -# request_scheduler_id -- An identifer based on which to perform -# the request scheduling. Currently the only valid option is keyspace. -# request_scheduler_id: keyspace - -# index_interval controls the sampling of entries from the primrary -# row index in terms of space versus time. The larger the interval, -# the smaller and less effective the sampling will be. In technicial -# terms, the interval coresponds to the number of index entries that -# are skipped between taking each sample. All the sampled entries -# must fit in memory. Generally, a value between 128 and 512 here -# coupled with a large key cache size on CFs results in the best trade -# offs. This value is not often changed, however if you have many -# very small rows (many to an OS page), then increasing this will -# often lower memory usage without a impact on performance. -index_interval: 128 - # Enable or disable inter-node encryption # Default settings are TLS v1, RSA 1024-bit keys (it is imperative that # users generate their own keys) TLS_RSA_WITH_AES_128_CBC_SHA as the cipher diff --git a/priam/src/test/resources/conf/jvm.options b/priam/src/test/resources/conf/jvm-server.options similarity index 98% rename from priam/src/test/resources/conf/jvm.options rename to priam/src/test/resources/conf/jvm-server.options index f91466ad0..bd61450c1 100644 --- a/priam/src/test/resources/conf/jvm.options +++ b/priam/src/test/resources/conf/jvm-server.options @@ -62,9 +62,6 @@ # Enable or disable the native transport server. See start_native_transport in cassandra.yaml. # cassandra.start_native_transport=true|false -# Enable or disable the Thrift RPC server. (Default: true) -#-Dcassandra.start_rpc=true/false - # Set the port for inter-node communication. (Default: 7000) #-Dcassandra.storage_port=port diff --git a/priam/src/test/resources/incr-restore-cassandra.yaml b/priam/src/test/resources/incr-restore-cassandra.yaml index 9594672e2..39a3c41aa 100755 --- a/priam/src/test/resources/incr-restore-cassandra.yaml +++ b/priam/src/test/resources/incr-restore-cassandra.yaml @@ -317,65 +317,6 @@ native_transport_port: 9042 # native_transport_min_threads: 16 # native_transport_max_threads: 128 - -# Whether to start the thrift rpc server. -start_rpc: true -# The address to bind the Thrift RPC service to -- clients connect -# here. Unlike ListenAddress above, you *can* specify 0.0.0.0 here if -# you want Thrift to listen on all interfaces. -# -# Leaving this blank has the same effect it does for ListenAddress, -# (i.e. it will be based on the configured hostname of the node). -rpc_address: localhost -# port for Thrift to listen for clients on -rpc_port: 9160 - -# enable or disable keepalive on rpc connections -rpc_keepalive: true - -# Cassandra provides three out-of-the-box options for the RPC Server: -# -# sync -> One thread per thrift connection. For a very large number of clients, memory -# will be your limiting factor. On a 64 bit JVM, 128KB is the minimum stack size -# per thread, and that will correspond to your use of virtual memory (but physical memory -# may be limited depending on use of stack space). -# -# hsha -> Stands for "half synchronous, half asynchronous." All thrift clients are handled -# asynchronously using a small number of threads that does not vary with the amount -# of thrift clients (and thus scales well to many clients). The rpc requests are still -# synchronous (one thread per active request). -# -# The default is sync because on Windows hsha is about 30% slower. On Linux, -# sync/hsha performance is about the same, with hsha of course using less memory. -# -# Alternatively, can provide your own RPC server by providing the fully-qualified class name -# of an o.a.c.t.TServerFactory that can create an instance of it. -rpc_server_type: sync - -# Uncomment rpc_min|max_thread to set request pool size limits. -# -# Regardless of your choice of RPC server (see above), the number of maximum requests in the -# RPC thread pool dictates how many concurrent requests are possible (but if you are using the sync -# RPC server, it also dictates the number of clients that can be connected at all). -# -# The default is unlimited and thus provide no protection against clients overwhelming the server. You are -# encouraged to set a maximum that makes sense for you in production, but do keep in mind that -# rpc_max_threads represents the maximum number of client requests this server may execute concurrently. -# -# rpc_min_threads: 16 -# rpc_max_threads: 2048 - -# uncomment to set socket buffer sizes on rpc connections -# rpc_send_buff_size_in_bytes: -# rpc_recv_buff_size_in_bytes: - -# Frame size for thrift (maximum field length). -thrift_framed_transport_size_in_mb: 15 - -# The max length of a thrift message, including all fields and -# internal thrift overhead. -thrift_max_message_length_in_mb: 16 - # Set to true to have Cassandra create a hard link to each sstable # flushed or streamed locally in a backups/ subdirectory of the # Keyspace data. Removing these links is the operator's @@ -470,13 +411,6 @@ rpc_timeout_in_ms: 10000 # and the times are synchronized between the nodes. cross_node_timeout: false -# Enable socket timeout for streaming operation. -# When a timeout occurs during streaming, streaming is retried from the start -# of the current file. This *can* involve re-streaming an important amount of -# data, so you should avoid setting the value too low. -# Default value is 0, which never timeout streams. -# streaming_socket_timeout_in_ms: 0 - # phi value that must be reached for a host to be marked down. # most users should never need to adjust this. # phi_convict_threshold: 8 @@ -547,58 +481,6 @@ dynamic_snitch_reset_interval_in_ms: 600000 # until the pinned host was 20% worse than the fastest. dynamic_snitch_badness_threshold: 0.1 -# request_scheduler -- Set this to a class that implements -# RequestScheduler, which will schedule incoming client requests -# according to the specific policy. This is useful for multi-tenancy -# with a single Cassandra cluster. -# NOTE: This is specifically for requests from the client and does -# not affect inter node communication. -# org.apache.cassandra.scheduler.NoScheduler - No scheduling takes place -# org.apache.cassandra.scheduler.RoundRobinScheduler - Round robin of -# client requests to a node with a separate queue for each -# request_scheduler_id. The scheduler is further customized by -# request_scheduler_options as described below. -request_scheduler: org.apache.cassandra.scheduler.NoScheduler - -# Scheduler Options vary based on the type of scheduler -# NoScheduler - Has no options -# RoundRobin -# - throttle_limit -- The throttle_limit is the number of in-flight -# requests per client. Requests beyond -# that limit are queued up until -# running requests can complete. -# The value of 80 here is twice the number of -# concurrent_reads + concurrent_writes. -# - default_weight -- default_weight is optional and allows for -# overriding the default which is 1. -# - weights -- Weights are optional and will default to 1 or the -# overridden default_weight. The weight translates into how -# many requests are handled during each turn of the -# RoundRobin, based on the scheduler id. -# -# request_scheduler_options: -# throttle_limit: 80 -# default_weight: 5 -# weights: -# Keyspace1: 1 -# Keyspace2: 5 - -# request_scheduler_id -- An identifer based on which to perform -# the request scheduling. Currently the only valid option is keyspace. -# request_scheduler_id: keyspace - -# index_interval controls the sampling of entries from the primrary -# row index in terms of space versus time. The larger the interval, -# the smaller and less effective the sampling will be. In technicial -# terms, the interval coresponds to the number of index entries that -# are skipped between taking each sample. All the sampled entries -# must fit in memory. Generally, a value between 128 and 512 here -# coupled with a large key cache size on CFs results in the best trade -# offs. This value is not often changed, however if you have many -# very small rows (many to an OS page), then increasing this will -# often lower memory usage without a impact on performance. -index_interval: 128 - # Enable or disable inter-node encryption # Default settings are TLS v1, RSA 1024-bit keys (it is imperative that # users generate their own keys) TLS_RSA_WITH_AES_128_CBC_SHA as the cipher From 04d7521ee0ac2deee559311504dc1fda6044b419 Mon Sep 17 00:00:00 2001 From: ayushis Date: Wed, 9 Feb 2022 12:02:00 -0800 Subject: [PATCH 197/228] refrences to thrift removed,changes realted to nodetool commands --- .../cassandra/extensions/NFSeedProvider.java | 8 +- .../priam/config/PriamConfiguration.java | 3 +- .../priam/connection/CassandraOperations.java | 4 +- .../netflix/priam/connection/JMXNodeTool.java | 27 +++--- .../priam/health/CassandraMonitor.java | 8 +- .../netflix/priam/health/IThriftChecker.java | 8 -- .../netflix/priam/health/InstanceState.java | 16 ++-- .../netflix/priam/health/ThriftChecker.java | 52 ----------- .../connection/TestCassandraOperations.java | 2 +- .../priam/health/TestCassandraMonitor.java | 6 +- .../priam/health/TestInstanceStatus.java | 40 +++------ .../priam/health/TestThriftChecker.java | 86 ------------------- 12 files changed, 43 insertions(+), 217 deletions(-) delete mode 100644 priam/src/main/java/com/netflix/priam/health/IThriftChecker.java delete mode 100644 priam/src/main/java/com/netflix/priam/health/ThriftChecker.java delete mode 100644 priam/src/test/java/com/netflix/priam/health/TestThriftChecker.java diff --git a/priam-cass-extensions/src/main/java/com/netflix/priam/cassandra/extensions/NFSeedProvider.java b/priam-cass-extensions/src/main/java/com/netflix/priam/cassandra/extensions/NFSeedProvider.java index 3a5102541..0f5b7fc2c 100644 --- a/priam-cass-extensions/src/main/java/com/netflix/priam/cassandra/extensions/NFSeedProvider.java +++ b/priam-cass-extensions/src/main/java/com/netflix/priam/cassandra/extensions/NFSeedProvider.java @@ -16,10 +16,10 @@ */ package com.netflix.priam.cassandra.extensions; -import java.net.InetAddress; import java.util.ArrayList; import java.util.List; import java.util.Map; +import org.apache.cassandra.locator.InetAddressAndPort; import org.apache.cassandra.locator.SeedProvider; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -31,13 +31,13 @@ public class NFSeedProvider implements SeedProvider { public NFSeedProvider(Map args) {} @Override - public List getSeeds() { - List seeds = new ArrayList(); + public List getSeeds() { + List seeds = new ArrayList(); try { String priamSeeds = DataFetcher.fetchData( "http://127.0.0.1:8080/Priam/REST/v1/cassconfig/get_seeds"); - for (String seed : priamSeeds.split(",")) seeds.add(InetAddress.getByName(seed)); + for (String seed : priamSeeds.split(",")) seeds.add(InetAddressAndPort.getByName(seed)); } catch (Exception e) { logger.error("Failed to load seed data", e); } diff --git a/priam/src/main/java/com/netflix/priam/config/PriamConfiguration.java b/priam/src/main/java/com/netflix/priam/config/PriamConfiguration.java index a3f4e879d..cc850a6a2 100644 --- a/priam/src/main/java/com/netflix/priam/config/PriamConfiguration.java +++ b/priam/src/main/java/com/netflix/priam/config/PriamConfiguration.java @@ -409,7 +409,8 @@ public String getYamlLocation() { @Override public String getJVMOptionsFileLocation() { - return config.get(PRIAM_PRE + ".jvm.options.location", getCassHome() + "/conf/jvm-server.options"); + return config.get( + PRIAM_PRE + ".jvm.options.location", getCassHome() + "/conf/jvm-server.options"); } public String getAuthenticator() { diff --git a/priam/src/main/java/com/netflix/priam/connection/CassandraOperations.java b/priam/src/main/java/com/netflix/priam/connection/CassandraOperations.java index f275f8212..79579de6c 100644 --- a/priam/src/main/java/com/netflix/priam/connection/CassandraOperations.java +++ b/priam/src/main/java/com/netflix/priam/connection/CassandraOperations.java @@ -48,7 +48,7 @@ public synchronized void takeSnapshot(final String snapshotName) throws Exceptio new RetryableCallable(6, 10000) { public Void retriableCall() throws Exception { JMXNodeTool nodetool = JMXNodeTool.instance(configuration); - nodetool.takeSnapshot(snapshotName, null); + nodetool.takeSnapshot(snapshotName, null, Collections.emptyMap()); return null; } }.call(); @@ -141,7 +141,7 @@ public List> gossipInfo() throws Exception { "Exception in fetching c* jmx tool . Msg: {}", e.getLocalizedMessage(), e); throw e; } - String gossipInfoLines[] = nodeTool.getGossipInfo().split("/"); + String gossipInfoLines[] = nodeTool.getGossipInfo(false).split("/"); Arrays.stream(gossipInfoLines) .forEach( gossipInfoLine -> { diff --git a/priam/src/main/java/com/netflix/priam/connection/JMXNodeTool.java b/priam/src/main/java/com/netflix/priam/connection/JMXNodeTool.java index 0be900d83..76d940e7d 100644 --- a/priam/src/main/java/com/netflix/priam/connection/JMXNodeTool.java +++ b/priam/src/main/java/com/netflix/priam/connection/JMXNodeTool.java @@ -263,7 +263,6 @@ public JSONObject estimateKeys() throws JSONException { public JSONObject info() throws JSONException { JSONObject object = new JSONObject(); object.put("gossip_active", isInitialized()); - object.put("thrift_active", isThriftServerRunning()); object.put("native_active", isNativeTransportRunning()); object.put("token", getTokens().toString()); object.put("load", getLoadString()); @@ -280,26 +279,26 @@ public JSONObject info() throws JSONException { public JSONObject statusInformation() throws JSONException { JSONObject jsonObject = new JSONObject(); - jsonObject.put("live", getLiveNodes()); - jsonObject.put("unreachable", getUnreachableNodes()); - jsonObject.put("joining", getJoiningNodes()); - jsonObject.put("leaving", getLeavingNodes()); - jsonObject.put("moving", getMovingNodes()); - jsonObject.put("tokenToEndpointMap", getTokenToEndpointMap()); + jsonObject.put("live", getLiveNodes(false)); + jsonObject.put("unreachable", getUnreachableNodes(false)); + jsonObject.put("joining", getJoiningNodes(false)); + jsonObject.put("leaving", getLeavingNodes(false)); + jsonObject.put("moving", getMovingNodes(false)); + jsonObject.put("tokenToEndpointMap", getTokenToEndpointMap(false)); return jsonObject; } public JSONArray ring(String keyspace) throws JSONException { JSONArray ring = new JSONArray(); - Map tokenToEndpoint = getTokenToEndpointMap(); + Map tokenToEndpoint = getTokenToEndpointMap(false); List sortedTokens = new ArrayList<>(tokenToEndpoint.keySet()); - Collection liveNodes = getLiveNodes(); - Collection deadNodes = getUnreachableNodes(); - Collection joiningNodes = getJoiningNodes(); - Collection leavingNodes = getLeavingNodes(); - Collection movingNodes = getMovingNodes(); - Map loadMap = getLoadMap(); + Collection liveNodes = getLiveNodes(false); + Collection deadNodes = getUnreachableNodes(false); + Collection joiningNodes = getJoiningNodes(false); + Collection leavingNodes = getLeavingNodes(false); + Collection movingNodes = getMovingNodes(false); + Map loadMap = getLoadMap(false); String format = "%-16s%-12s%-12s%-7s%-8s%-16s%-20s%-44s%n"; diff --git a/priam/src/main/java/com/netflix/priam/health/CassandraMonitor.java b/priam/src/main/java/com/netflix/priam/health/CassandraMonitor.java index f0f6c849c..846bbed81 100644 --- a/priam/src/main/java/com/netflix/priam/health/CassandraMonitor.java +++ b/priam/src/main/java/com/netflix/priam/health/CassandraMonitor.java @@ -47,20 +47,17 @@ public class CassandraMonitor extends Task { private final InstanceState instanceState; private final ICassandraProcess cassProcess; private final CassMonitorMetrics cassMonitorMetrics; - private final IThriftChecker thriftChecker; @Inject protected CassandraMonitor( IConfiguration config, InstanceState instanceState, ICassandraProcess cassProcess, - CassMonitorMetrics cassMonitorMetrics, - IThriftChecker thriftChecker) { + CassMonitorMetrics cassMonitorMetrics) { super(config); this.instanceState = instanceState; this.cassProcess = cassProcess; this.cassMonitorMetrics = cassMonitorMetrics; - this.thriftChecker = thriftChecker; } @Override @@ -95,9 +92,6 @@ public void execute() throws Exception { NodeProbe bean = JMXNodeTool.instance(this.config); instanceState.setIsGossipActive(bean.isGossipRunning()); instanceState.setIsNativeTransportActive(bean.isNativeTransportRunning()); - instanceState.setIsThriftActive( - bean.isThriftServerRunning() && thriftChecker.isThriftServerListening()); - } else { // Setting cassandra flag to false instanceState.setCassandraProcessAlive(false); diff --git a/priam/src/main/java/com/netflix/priam/health/IThriftChecker.java b/priam/src/main/java/com/netflix/priam/health/IThriftChecker.java deleted file mode 100644 index 621faec30..000000000 --- a/priam/src/main/java/com/netflix/priam/health/IThriftChecker.java +++ /dev/null @@ -1,8 +0,0 @@ -package com.netflix.priam.health; - -import com.google.inject.ImplementedBy; - -@ImplementedBy(ThriftChecker.class) -public interface IThriftChecker { - boolean isThriftServerListening(); -} diff --git a/priam/src/main/java/com/netflix/priam/health/InstanceState.java b/priam/src/main/java/com/netflix/priam/health/InstanceState.java index 8d1547354..b9cc3c00f 100644 --- a/priam/src/main/java/com/netflix/priam/health/InstanceState.java +++ b/priam/src/main/java/com/netflix/priam/health/InstanceState.java @@ -151,15 +151,13 @@ private boolean isRestoring() { private void setHealthy() { this.isHealthy.set( - isRestoring() || - (isCassandraProcessAlive() && - isRequiredDirectoriesExist() && - isGossipActive() && - isYmlWritten() && - isHealthyOverride() && - isNativeTransportActive() - ) - ); + isRestoring() + || (isCassandraProcessAlive() + && isRequiredDirectoriesExist() + && isGossipActive() + && isYmlWritten() + && isHealthyOverride() + && isNativeTransportActive())); } public boolean isYmlWritten() { diff --git a/priam/src/main/java/com/netflix/priam/health/ThriftChecker.java b/priam/src/main/java/com/netflix/priam/health/ThriftChecker.java deleted file mode 100644 index 0390cbbca..000000000 --- a/priam/src/main/java/com/netflix/priam/health/ThriftChecker.java +++ /dev/null @@ -1,52 +0,0 @@ -package com.netflix.priam.health; - -import com.google.inject.Inject; -import com.netflix.priam.config.IConfiguration; -import java.io.BufferedReader; -import java.io.InputStreamReader; -import java.util.concurrent.TimeUnit; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class ThriftChecker implements IThriftChecker { - private static final Logger logger = LoggerFactory.getLogger(ThriftChecker.class); - protected final IConfiguration config; - - @Inject - public ThriftChecker(IConfiguration config) { - this.config = config; - } - - public boolean isThriftServerListening() { - if (!config.checkThriftServerIsListening()) { - return true; - } - String[] cmd = - new String[] { - "/bin/sh", "-c", "ss -tuln | grep -c " + config.getThriftPort(), " 2>/dev/null" - }; - Process process = null; - try { - process = Runtime.getRuntime().exec(cmd); - process.waitFor(1, TimeUnit.SECONDS); - } catch (Exception e) { - logger.warn("Exception while executing the process: ", e); - } - if (process != null) { - try (BufferedReader reader = - new BufferedReader(new InputStreamReader(process.getInputStream())); ) { - if (Integer.parseInt(reader.readLine()) == 0) { - logger.info( - "Could not find anything listening on the rpc port {}!", - config.getThriftPort()); - return false; - } - } catch (Exception e) { - logger.warn("Exception while reading the input stream: ", e); - } - } - // A quiet on-call is our top priority, err on the side of avoiding false positives by - // default. - return true; - } -} diff --git a/priam/src/test/java/com/netflix/priam/connection/TestCassandraOperations.java b/priam/src/test/java/com/netflix/priam/connection/TestCassandraOperations.java index a7aa32ee3..89c9e5e59 100644 --- a/priam/src/test/java/com/netflix/priam/connection/TestCassandraOperations.java +++ b/priam/src/test/java/com/netflix/priam/connection/TestCassandraOperations.java @@ -58,7 +58,7 @@ public void testGossipInfo() throws Exception { String gossipInfoFromNodetool = FileUtils.getContentsAsString(new File(gossipInfo1)); new Expectations() { { - nodeProbe.getGossipInfo(); + nodeProbe.getGossipInfo(false); result = gossipInfoFromNodetool; nodeProbe.getTokens("127.0.0.1"); result = "123,234"; diff --git a/priam/src/test/java/com/netflix/priam/health/TestCassandraMonitor.java b/priam/src/test/java/com/netflix/priam/health/TestCassandraMonitor.java index 026d25d33..4bf3fa61c 100644 --- a/priam/src/test/java/com/netflix/priam/health/TestCassandraMonitor.java +++ b/priam/src/test/java/com/netflix/priam/health/TestCassandraMonitor.java @@ -37,7 +37,6 @@ public class TestCassandraMonitor { private static CassandraMonitor monitor; private static InstanceState instanceState; private static CassMonitorMetrics cassMonitorMetrics; - private static IThriftChecker thriftChecker; private IConfiguration config; @@ -49,16 +48,13 @@ public class TestCassandraMonitor { public void setUp() { Injector injector = Guice.createInjector(new BRTestModule()); config = injector.getInstance(IConfiguration.class); - thriftChecker = injector.getInstance(ThriftChecker.class); if (instanceState == null) instanceState = injector.getInstance(InstanceState.class); if (cassMonitorMetrics == null) cassMonitorMetrics = injector.getInstance(CassMonitorMetrics.class); if (monitor == null) - monitor = - new CassandraMonitor( - config, instanceState, cassProcess, cassMonitorMetrics, thriftChecker); + monitor = new CassandraMonitor(config, instanceState, cassProcess, cassMonitorMetrics); } @Test diff --git a/priam/src/test/java/com/netflix/priam/health/TestInstanceStatus.java b/priam/src/test/java/com/netflix/priam/health/TestInstanceStatus.java index 9ab55779c..c5839328e 100644 --- a/priam/src/test/java/com/netflix/priam/health/TestInstanceStatus.java +++ b/priam/src/test/java/com/netflix/priam/health/TestInstanceStatus.java @@ -39,37 +39,21 @@ public void setUp() { public void testHealth() { // Verify good health. Assert.assertTrue( - testInstanceState - .setParams(false, true, true, true, true, true, true) - .isHealthy()); + testInstanceState.setParams(false, true, true, true, true, true, true).isHealthy()); Assert.assertTrue( - testInstanceState - .setParams(false, true, true, true, false, true, true) - .isHealthy()); + testInstanceState.setParams(false, true, true, true, true, true, true).isHealthy()); Assert.assertTrue( - testInstanceState - .setParams(false, true, true, true, true, true, true) - .isHealthy()); + testInstanceState.setParams(false, true, true, true, true, true, true).isHealthy()); Assert.assertTrue( - testInstanceState - .setParams(true, false, true, true, true, true, true) - .isHealthy()); + testInstanceState.setParams(true, false, true, true, true, true, true).isHealthy()); Assert.assertTrue( - testInstanceState - .setParams(true, true, false, true, true, true, true) - .isHealthy()); + testInstanceState.setParams(true, true, false, true, true, true, true).isHealthy()); Assert.assertTrue( - testInstanceState - .setParams(true, true, true, false, true, true, true) - .isHealthy()); + testInstanceState.setParams(true, true, true, false, true, true, true).isHealthy()); Assert.assertTrue( - testInstanceState - .setParams(true, true, true, true, true, false, true) - .isHealthy()); + testInstanceState.setParams(true, true, true, true, true, false, true).isHealthy()); Assert.assertTrue( - testInstanceState - .setParams(true, true, true, true, false, true, true) - .isHealthy()); + testInstanceState.setParams(true, true, true, true, false, true, true).isHealthy()); // Negative health case scenarios. Assert.assertFalse( @@ -78,7 +62,7 @@ public void testHealth() { .isHealthy()); Assert.assertFalse( testInstanceState - .setParams(false, true, false, true, true, true, true) + .setParams(false, true, false, true, true, true, true) .isHealthy()); Assert.assertFalse( testInstanceState @@ -86,15 +70,15 @@ public void testHealth() { .isHealthy()); Assert.assertFalse( testInstanceState - .setParams(false, true, true, true, true, false, true) + .setParams(false, true, true, true, true, false, true) .isHealthy()); Assert.assertFalse( testInstanceState - .setParams(false, true, true, true, false, true, true) + .setParams(false, true, true, true, false, true, true) .isHealthy()); Assert.assertFalse( testInstanceState - .setParams(false, true, true, true, false, true, false) + .setParams(false, true, true, true, false, true, false) .isHealthy()); } diff --git a/priam/src/test/java/com/netflix/priam/health/TestThriftChecker.java b/priam/src/test/java/com/netflix/priam/health/TestThriftChecker.java deleted file mode 100644 index 0936686d4..000000000 --- a/priam/src/test/java/com/netflix/priam/health/TestThriftChecker.java +++ /dev/null @@ -1,86 +0,0 @@ -package com.netflix.priam.health; - -import com.netflix.priam.config.FakeConfiguration; -import java.io.ByteArrayInputStream; -import java.io.IOException; -import java.io.InputStream; -import mockit.Expectations; -import mockit.Mocked; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; - -public class TestThriftChecker { - private FakeConfiguration config; - private ThriftChecker thriftChecker; - - @Mocked private Process mockProcess; - - @Before - public void TestThriftChecker() { - config = new FakeConfiguration(); - thriftChecker = new ThriftChecker(config); - } - - @Test - public void testThriftServerIsListeningDisabled() { - config.setCheckThriftServerIsListening(false); - Assert.assertTrue(thriftChecker.isThriftServerListening()); - } - - @Test - public void testThriftServerIsNotListening() { - config.setCheckThriftServerIsListening(true); - Assert.assertFalse(thriftChecker.isThriftServerListening()); - } - - @Test - public void testThriftServerIsListening() throws IOException { - config.setCheckThriftServerIsListening(true); - final InputStream mockOutput = new ByteArrayInputStream("1".getBytes()); - new Expectations() { - { - mockProcess.getInputStream(); - result = mockOutput; - } - }; - // Mock out the ps call - final Runtime r = Runtime.getRuntime(); - String[] cmd = { - "/bin/sh", "-c", "ss -tuln | grep -c " + config.getThriftPort(), " 2>/dev/null" - }; - new Expectations(r) { - { - r.exec(cmd); - result = mockProcess; - } - }; - - Assert.assertTrue(thriftChecker.isThriftServerListening()); - } - - @Test - public void testThriftServerIsListeningException() throws IOException { - config.setCheckThriftServerIsListening(true); - final IOException mockOutput = new IOException("Command exited with code 0"); - new Expectations() { - { - mockProcess.getInputStream(); - result = mockOutput; - } - }; - // Mock out the ps call - final Runtime r = Runtime.getRuntime(); - String[] cmd = { - "/bin/sh", "-c", "ss -tuln | grep -c " + config.getThriftPort(), " 2>/dev/null" - }; - new Expectations(r) { - { - r.exec(cmd); - result = mockProcess; - } - }; - - Assert.assertTrue(thriftChecker.isThriftServerListening()); - } -} From 4daca2022fed55d6d73071c15225f7cf3fd95973 Mon Sep 17 00:00:00 2001 From: ayushis Date: Wed, 9 Feb 2022 12:35:10 -0800 Subject: [PATCH 198/228] Adding google in the repo list for dependencies --- build.gradle | 1 + 1 file changed, 1 insertion(+) diff --git a/build.gradle b/build.gradle index b8eaa8273..677a82ad1 100644 --- a/build.gradle +++ b/build.gradle @@ -17,6 +17,7 @@ allprojects { repositories { mavenCentral() + google() } configurations { From 99df803f4490b9edefd1832fabe64fa10856db7b Mon Sep 17 00:00:00 2001 From: mattl-netflix <63665634+mattl-netflix@users.noreply.github.com> Date: Wed, 30 Mar 2022 13:18:41 -0700 Subject: [PATCH 199/228] Remove TokenRetrieverBase (#981) --- .../identity/token/TokenRetrieverBase.java | 41 ------------------- .../identity/token/TokenRetrieverUtils.java | 2 +- 2 files changed, 1 insertion(+), 42 deletions(-) delete mode 100755 priam/src/main/java/com/netflix/priam/identity/token/TokenRetrieverBase.java diff --git a/priam/src/main/java/com/netflix/priam/identity/token/TokenRetrieverBase.java b/priam/src/main/java/com/netflix/priam/identity/token/TokenRetrieverBase.java deleted file mode 100755 index 04353f450..000000000 --- a/priam/src/main/java/com/netflix/priam/identity/token/TokenRetrieverBase.java +++ /dev/null @@ -1,41 +0,0 @@ -/** - * Copyright 2017 Netflix, Inc. - * - *

    Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file - * except in compliance with the License. You may obtain a copy of the License at - * - *

    http://www.apache.org/licenses/LICENSE-2.0 - * - *

    Unless required by applicable law or agreed to in writing, software distributed under the - * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.netflix.priam.identity.token; - -import com.netflix.priam.identity.PriamInstance; -import java.util.Random; - -public class TokenRetrieverBase { - - public static final String DUMMY_INSTANCE_ID = "new_slot"; - private static final int MAX_VALUE_IN_MILISECS = 300000; // sleep up to 5 minutes - protected final Random randomizer; - - public TokenRetrieverBase() { - this.randomizer = new Random(); - } - - protected boolean isInstanceDummy(PriamInstance instance) { - return instance.getInstanceId().equals(DUMMY_INSTANCE_ID); - } - - /* - * Return a random time for a thread to sleep. - * - * @return time in millisecs - */ - protected long getSleepTime() { - return (long) this.randomizer.nextInt(MAX_VALUE_IN_MILISECS); - } -} diff --git a/priam/src/main/java/com/netflix/priam/identity/token/TokenRetrieverUtils.java b/priam/src/main/java/com/netflix/priam/identity/token/TokenRetrieverUtils.java index b7b6ee07e..324388ca4 100644 --- a/priam/src/main/java/com/netflix/priam/identity/token/TokenRetrieverUtils.java +++ b/priam/src/main/java/com/netflix/priam/identity/token/TokenRetrieverUtils.java @@ -16,7 +16,7 @@ /** Common utilities for token retrieval. */ public class TokenRetrieverUtils { - private static final Logger logger = LoggerFactory.getLogger(TokenRetrieverBase.class); + private static final Logger logger = LoggerFactory.getLogger(TokenRetrieverUtils.class); private static final String STATUS_URL_FORMAT = "http://%s:8080/Priam/REST/v1/cassadmin/status"; /** From 27c267e47831c172f0007c84ddc7adedc3f0c893 Mon Sep 17 00:00:00 2001 From: mattl-netflix <63665634+mattl-netflix@users.noreply.github.com> Date: Thu, 2 Jun 2022 14:59:07 -0700 Subject: [PATCH 200/228] Dynamic Rate Limiting of Snapshots (#975) * CASS-2011 update interface to accommodate dependency change * CASS-2011 move backup cleaning outside of getTimer method * CASS-2011 introduce a target time for upload completion. Currently always the epoch which implies a no-op * CASS-2011 Change AbstractFileSystem API to pass target instant to fileUploadImpl * CASS-2011 Delete already-uploaded files to get accurate estimates of remaining bytes to upload. * CASS-2011 create a rate limiter that dynamically adjusts its throttle based on the bytes still to upload in all remaining snapshots and a user-specified target time. Adjust the throttle only when we've deviated by a user-configurable threshold to ensure the rate limiter is not constantly adjusted. In that case, it would be redundant as it does not throttle the subsequent file after an adjustment. Ensure that the target does not exceed the earlier of the next scheduled snapshot or the time at which we would fail to meet our backup verification SLO. * CASS-2011 Add unit tests and associated refactoring for BackupDynamicRateLimiter. --- .../priam/aws/S3EncryptedFileSystem.java | 11 +- .../com/netflix/priam/aws/S3FileSystem.java | 21 ++- .../netflix/priam/aws/S3FileSystemBase.java | 6 +- .../netflix/priam/backup/AbstractBackup.java | 15 +- .../priam/backup/AbstractFileSystem.java | 12 +- .../backup/BackupDynamicRateLimiter.java | 52 ++++++ .../priam/backup/BackupVerification.java | 21 ++- .../netflix/priam/backup/DirectorySize.java | 10 ++ .../priam/backup/DynamicRateLimiter.java | 9 + .../priam/backup/IBackupFileSystem.java | 20 ++- .../priam/backup/IncrementalBackup.java | 2 +- .../priam/backup/SnapshotDirectorySize.java | 50 ++++++ .../priam/backupv2/BackupV2Service.java | 5 +- .../priam/backupv2/FileUploadResult.java | 8 + .../priam/backupv2/MetaFileWriterBuilder.java | 9 +- .../priam/backupv2/SnapshotMetaTask.java | 109 ++++++++---- .../netflix/priam/config/IConfiguration.java | 13 ++ .../priam/config/PriamConfiguration.java | 10 ++ .../google/GoogleEncryptedFileSystem.java | 4 +- .../netflix/priam/backup/BRTestModule.java | 10 +- .../priam/backup/FakeBackupFileSystem.java | 4 +- .../priam/backup/FakeDynamicRateLimiter.java | 8 + .../priam/backup/NullBackupFileSystem.java | 4 +- .../priam/backup/TestAbstractFileSystem.java | 7 +- .../backup/TestBackupDynamicRateLimiter.java | 159 ++++++++++++++++++ .../priam/backupv2/TestSnapshotMetaTask.java | 2 +- .../priam/config/FakeConfiguration.java | 7 + 27 files changed, 518 insertions(+), 70 deletions(-) create mode 100644 priam/src/main/java/com/netflix/priam/backup/BackupDynamicRateLimiter.java create mode 100644 priam/src/main/java/com/netflix/priam/backup/DirectorySize.java create mode 100644 priam/src/main/java/com/netflix/priam/backup/DynamicRateLimiter.java create mode 100644 priam/src/main/java/com/netflix/priam/backup/SnapshotDirectorySize.java create mode 100644 priam/src/test/java/com/netflix/priam/backup/FakeDynamicRateLimiter.java create mode 100644 priam/src/test/java/com/netflix/priam/backup/TestBackupDynamicRateLimiter.java diff --git a/priam/src/main/java/com/netflix/priam/aws/S3EncryptedFileSystem.java b/priam/src/main/java/com/netflix/priam/aws/S3EncryptedFileSystem.java index 9012af932..eeeaa2ca4 100755 --- a/priam/src/main/java/com/netflix/priam/aws/S3EncryptedFileSystem.java +++ b/priam/src/main/java/com/netflix/priam/aws/S3EncryptedFileSystem.java @@ -25,6 +25,7 @@ import com.google.inject.name.Named; import com.netflix.priam.backup.AbstractBackupPath; import com.netflix.priam.backup.BackupRestoreException; +import com.netflix.priam.backup.DynamicRateLimiter; import com.netflix.priam.backup.RangeReadInputStream; import com.netflix.priam.compress.ChunkedStream; import com.netflix.priam.compress.ICompression; @@ -37,6 +38,7 @@ import java.io.*; import java.nio.file.Path; import java.nio.file.Paths; +import java.time.Instant; import java.util.Iterator; import java.util.List; import org.apache.commons.io.IOUtils; @@ -49,6 +51,7 @@ public class S3EncryptedFileSystem extends S3FileSystemBase { private static final Logger logger = LoggerFactory.getLogger(S3EncryptedFileSystem.class); private final IFileCryptography encryptor; + private final DynamicRateLimiter dynamicRateLimiter; @Inject public S3EncryptedFileSystem( @@ -59,10 +62,12 @@ public S3EncryptedFileSystem( @Named("filecryptoalgorithm") IFileCryptography fileCryptography, BackupMetrics backupMetrics, BackupNotificationMgr backupNotificationMgr, - InstanceInfo instanceInfo) { + InstanceInfo instanceInfo, + DynamicRateLimiter dynamicRateLimiter) { super(pathProvider, compress, config, backupMetrics, backupNotificationMgr); this.encryptor = fileCryptography; + this.dynamicRateLimiter = dynamicRateLimiter; super.s3Client = AmazonS3Client.builder() .withCredentials(cred.getAwsCredentialProvider()) @@ -97,7 +102,8 @@ s3Client, getShard(), super.getFileSize(remotePath), remotePath)) { } @Override - protected long uploadFileImpl(AbstractBackupPath path) throws BackupRestoreException { + protected long uploadFileImpl(AbstractBackupPath path, Instant target) + throws BackupRestoreException { Path localPath = Paths.get(path.getBackupFile().getAbsolutePath()); String remotePath = path.getRemotePath(); @@ -150,6 +156,7 @@ protected long uploadFileImpl(AbstractBackupPath path) throws BackupRestoreExcep byte[] chunk = chunks.next(); // throttle upload to endpoint rateLimiter.acquire(chunk.length); + dynamicRateLimiter.acquire(path, target, chunk.length); DataPart dp = new DataPart( diff --git a/priam/src/main/java/com/netflix/priam/aws/S3FileSystem.java b/priam/src/main/java/com/netflix/priam/aws/S3FileSystem.java index 363058d76..e24f68100 100644 --- a/priam/src/main/java/com/netflix/priam/aws/S3FileSystem.java +++ b/priam/src/main/java/com/netflix/priam/aws/S3FileSystem.java @@ -27,6 +27,7 @@ import com.netflix.priam.aws.auth.IS3Credential; import com.netflix.priam.backup.AbstractBackupPath; import com.netflix.priam.backup.BackupRestoreException; +import com.netflix.priam.backup.DynamicRateLimiter; import com.netflix.priam.backup.RangeReadInputStream; import com.netflix.priam.compress.ChunkedStream; import com.netflix.priam.compress.CompressionType; @@ -39,6 +40,7 @@ import java.io.*; import java.nio.file.Path; import java.nio.file.Paths; +import java.time.Instant; import java.util.ArrayList; import java.util.Collections; import java.util.Iterator; @@ -53,6 +55,7 @@ public class S3FileSystem extends S3FileSystemBase { private static final Logger logger = LoggerFactory.getLogger(S3FileSystem.class); private static final long MAX_BUFFER_SIZE = 5L * 1024L * 1024L; + private final DynamicRateLimiter dynamicRateLimiter; @Inject public S3FileSystem( @@ -62,13 +65,15 @@ public S3FileSystem( final IConfiguration config, BackupMetrics backupMetrics, BackupNotificationMgr backupNotificationMgr, - InstanceInfo instanceInfo) { + InstanceInfo instanceInfo, + DynamicRateLimiter dynamicRateLimiter) { super(pathProvider, compress, config, backupMetrics, backupNotificationMgr); s3Client = AmazonS3Client.builder() .withCredentials(cred.getAwsCredentialProvider()) .withRegion(instanceInfo.getRegion()) .build(); + this.dynamicRateLimiter = dynamicRateLimiter; } @Override @@ -113,7 +118,8 @@ private ObjectMetadata getObjectMetadata(File file) { return ret; } - private long uploadMultipart(AbstractBackupPath path) throws BackupRestoreException { + private long uploadMultipart(AbstractBackupPath path, Instant target) + throws BackupRestoreException { Path localPath = Paths.get(path.getBackupFile().getAbsolutePath()); String remotePath = path.getRemotePath(); long chunkSize = getChunkSize(localPath); @@ -137,6 +143,7 @@ private long uploadMultipart(AbstractBackupPath path) throws BackupRestoreExcept while (chunks.hasNext()) { byte[] chunk = chunks.next(); rateLimiter.acquire(chunk.length); + dynamicRateLimiter.acquire(path, target, chunk.length); DataPart dp = new DataPart(++partNum, chunk, prefix, remotePath, uploadId); S3PartUploader partUploader = new S3PartUploader(s3Client, dp, partETags, partsPut); compressedFileSize += chunk.length; @@ -163,12 +170,13 @@ private long uploadMultipart(AbstractBackupPath path) throws BackupRestoreExcept } } - protected long uploadFileImpl(AbstractBackupPath path) throws BackupRestoreException { + protected long uploadFileImpl(AbstractBackupPath path, Instant target) + throws BackupRestoreException { Path localPath = Paths.get(path.getBackupFile().getAbsolutePath()); String remotePath = path.getRemotePath(); long chunkSize = config.getBackupChunkSize(); File localFile = localPath.toFile(); - if (localFile.length() >= chunkSize) return uploadMultipart(path); + if (localFile.length() >= chunkSize) return uploadMultipart(path, target); String prefix = config.getBackupPrefix(); if (logger.isDebugEnabled()) logger.debug("PUTing {}/{}", prefix, remotePath); @@ -182,7 +190,10 @@ protected long uploadFileImpl(AbstractBackupPath path) throws BackupRestoreExcep byte[] chunk = byteArrayOutputStream.toByteArray(); long compressedFileSize = chunk.length; // C* snapshots may have empty files. That is probably unintentional. - if (chunk.length > 0) rateLimiter.acquire(chunk.length); + if (chunk.length > 0) { + rateLimiter.acquire(chunk.length); + dynamicRateLimiter.acquire(path, target, chunk.length); + } ObjectMetadata objectMetadata = getObjectMetadata(localFile); objectMetadata.setContentLength(chunk.length); ByteArrayInputStream inputStream = new ByteArrayInputStream(chunk); diff --git a/priam/src/main/java/com/netflix/priam/aws/S3FileSystemBase.java b/priam/src/main/java/com/netflix/priam/aws/S3FileSystemBase.java index 4ff3d10df..5703424f2 100755 --- a/priam/src/main/java/com/netflix/priam/aws/S3FileSystemBase.java +++ b/priam/src/main/java/com/netflix/priam/aws/S3FileSystemBase.java @@ -136,13 +136,13 @@ public void visit(LifecyclePrefixPredicate lifecyclePrefixPredicate) { @Override public void visit(LifecycleTagPredicate lifecycleTagPredicate) {} - @Override - public void visit(LifecycleAndOperator lifecycleAndOperator) {} - @Override public void visit( LifecycleObjectSizeGreaterThanPredicate lifecycleObjectSizeGreaterThanPredicate) {} + @Override + public void visit(LifecycleAndOperator lifecycleAndOperator) {} + @Override public void visit( LifecycleObjectSizeLessThanPredicate lifecycleObjectSizeLessThanPredicate) {} diff --git a/priam/src/main/java/com/netflix/priam/backup/AbstractBackup.java b/priam/src/main/java/com/netflix/priam/backup/AbstractBackup.java index 583a5d3f3..282eecf5c 100644 --- a/priam/src/main/java/com/netflix/priam/backup/AbstractBackup.java +++ b/priam/src/main/java/com/netflix/priam/backup/AbstractBackup.java @@ -37,6 +37,7 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; +import java.time.Instant; import java.util.HashSet; import java.util.Optional; import java.util.Set; @@ -65,6 +66,12 @@ public AbstractBackup( this.fs = backupFileSystemCtx.getFileStrategy(config); } + /** Overload that uploads files without any custom throttling */ + protected ImmutableList> uploadAndDeleteAllFiles( + final File parent, final BackupFileType type, boolean async) throws Exception { + return uploadAndDeleteAllFiles(parent, type, async, Instant.EPOCH); + } + /** * Upload files in the specified dir. Does not delete the file in case of error. The files are * uploaded serially or async based on flag provided. @@ -72,18 +79,20 @@ public AbstractBackup( * @param parent Parent dir * @param type Type of file (META, SST, SNAP etc) * @param async Upload the file(s) in async fashion if enabled. + * @param target target time of completion of the batch of files * @return List of files that are successfully uploaded as part of backup * @throws Exception when there is failure in uploading files. */ protected ImmutableList> uploadAndDeleteAllFiles( - final File parent, final BackupFileType type, boolean async) throws Exception { + final File parent, final BackupFileType type, boolean async, Instant target) + throws Exception { ImmutableSet backupPaths = getBackupPaths(parent, type); final ImmutableList.Builder> futures = ImmutableList.builder(); for (AbstractBackupPath bp : backupPaths) { - if (async) futures.add(fs.asyncUploadAndDelete(bp, 10)); + if (async) futures.add(fs.asyncUploadAndDelete(bp, 10, target)); else { - fs.uploadAndDelete(bp, 10); + fs.uploadAndDelete(bp, 10, target); futures.add(Futures.immediateFuture(bp)); } } diff --git a/priam/src/main/java/com/netflix/priam/backup/AbstractFileSystem.java b/priam/src/main/java/com/netflix/priam/backup/AbstractFileSystem.java index 84b74d9bc..0737ac639 100644 --- a/priam/src/main/java/com/netflix/priam/backup/AbstractFileSystem.java +++ b/priam/src/main/java/com/netflix/priam/backup/AbstractFileSystem.java @@ -38,6 +38,7 @@ import java.io.File; import java.nio.file.Path; import java.nio.file.Paths; +import java.time.Instant; import java.util.Date; import java.util.Iterator; import java.util.List; @@ -157,16 +158,17 @@ protected abstract void downloadFileImpl(final AbstractBackupPath path, String s @Override public ListenableFuture asyncUploadAndDelete( - final AbstractBackupPath path, final int retry) throws RejectedExecutionException { + final AbstractBackupPath path, final int retry, Instant target) + throws RejectedExecutionException { return fileUploadExecutor.submit( () -> { - uploadAndDelete(path, retry); + uploadAndDelete(path, retry, target); return path; }); } @Override - public void uploadAndDelete(final AbstractBackupPath path, final int retry) + public void uploadAndDelete(final AbstractBackupPath path, final int retry, Instant target) throws BackupRestoreException { Path localPath = Paths.get(path.getBackupFile().getAbsolutePath()); File localFile = localPath.toFile(); @@ -189,7 +191,7 @@ public void uploadAndDelete(final AbstractBackupPath path, final int retry) new BoundedExponentialRetryCallable(500, 10000, retry) { @Override public Long retriableCall() throws Exception { - return uploadFileImpl(path); + return uploadFileImpl(path, target); } }.call(); @@ -270,7 +272,7 @@ public void deleteRemoteFiles(List remotePaths) throws BackupRestoreExcept protected abstract boolean doesRemoteFileExist(Path remotePath); - protected abstract long uploadFileImpl(final AbstractBackupPath path) + protected abstract long uploadFileImpl(final AbstractBackupPath path, Instant target) throws BackupRestoreException; @Override diff --git a/priam/src/main/java/com/netflix/priam/backup/BackupDynamicRateLimiter.java b/priam/src/main/java/com/netflix/priam/backup/BackupDynamicRateLimiter.java new file mode 100644 index 000000000..4b301e321 --- /dev/null +++ b/priam/src/main/java/com/netflix/priam/backup/BackupDynamicRateLimiter.java @@ -0,0 +1,52 @@ +package com.netflix.priam.backup; + +import com.google.common.base.Preconditions; +import com.google.common.util.concurrent.RateLimiter; +import com.netflix.priam.config.IConfiguration; +import java.time.Clock; +import java.time.Duration; +import java.time.Instant; +import javax.inject.Inject; + +public class BackupDynamicRateLimiter implements DynamicRateLimiter { + + private final Clock clock; + private final IConfiguration config; + private final DirectorySize dirSize; + private final RateLimiter rateLimiter; + + @Inject + BackupDynamicRateLimiter(IConfiguration config, Clock clock, DirectorySize dirSize) { + this.clock = clock; + this.config = config; + this.dirSize = dirSize; + this.rateLimiter = RateLimiter.create(Double.MAX_VALUE); + } + + @Override + public void acquire(AbstractBackupPath path, Instant target, int permits) { + if (target.equals(Instant.EPOCH) + || !path.getBackupFile() + .getAbsolutePath() + .contains(AbstractBackup.SNAPSHOT_FOLDER)) { + return; + } + long secondsRemaining = Duration.between(clock.instant(), target).getSeconds(); + if (secondsRemaining < 1) { + // skip file system checks when unnecessary + return; + } + int backupThreads = config.getBackupThreads(); + Preconditions.checkState(backupThreads > 0); + long bytesPerThread = this.dirSize.getBytes(config.getDataFileLocation()) / backupThreads; + if (bytesPerThread < 1) { + return; + } + double newRate = (double) bytesPerThread / secondsRemaining; + double oldRate = rateLimiter.getRate(); + if ((Math.abs(newRate - oldRate) / oldRate) > config.getRateLimitChangeThreshold()) { + rateLimiter.setRate(newRate); + } + rateLimiter.acquire(permits); + } +} diff --git a/priam/src/main/java/com/netflix/priam/backup/BackupVerification.java b/priam/src/main/java/com/netflix/priam/backup/BackupVerification.java index 71102f3a7..3dfc1bdfb 100644 --- a/priam/src/main/java/com/netflix/priam/backup/BackupVerification.java +++ b/priam/src/main/java/com/netflix/priam/backup/BackupVerification.java @@ -23,6 +23,7 @@ import com.netflix.priam.utils.DateUtil.DateRange; import java.nio.file.Path; import java.nio.file.Paths; +import java.time.Instant; import java.util.*; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -40,6 +41,7 @@ public class BackupVerification { private final IMetaProxy metaV2Proxy; private final IBackupStatusMgr backupStatusMgr; private final Provider abstractBackupPathProvider; + private BackupVerificationResult latestResult; @Inject BackupVerification( @@ -83,14 +85,14 @@ public Optional verifyBackup( for (BackupMetadata backupMetadata : metadata) { if (backupMetadata.getLastValidated() != null && !force) { // Backup is already validated. Nothing to do. - BackupVerificationResult result = new BackupVerificationResult(); - result.valid = true; - result.manifestAvailable = true; - result.snapshotInstant = backupMetadata.getStart().toInstant(); + latestResult = new BackupVerificationResult(); + latestResult.valid = true; + latestResult.manifestAvailable = true; + latestResult.snapshotInstant = backupMetadata.getStart().toInstant(); Path snapshotLocation = Paths.get(backupMetadata.getSnapshotLocation()); - result.remotePath = + latestResult.remotePath = snapshotLocation.subpath(1, snapshotLocation.getNameCount()).toString(); - return Optional.of(result); + return Optional.of(latestResult); } BackupVerificationResult backupVerificationResult = verifyBackup(metaProxy, backupMetadata); @@ -102,9 +104,11 @@ public Optional verifyBackup( if (backupVerificationResult.valid) { backupMetadata.setLastValidated(new Date(DateUtil.getInstant().toEpochMilli())); backupStatusMgr.update(backupMetadata); + latestResult = backupVerificationResult; return Optional.of(backupVerificationResult); } } + latestResult = null; return Optional.empty(); } @@ -145,6 +149,11 @@ public List verifyAllBackups( return result; } + /** returns the latest valid backup verification result if we have found one within the SLO * */ + public Optional getLatestVerfifiedBackupTime() { + return latestResult == null ? Optional.empty() : Optional.of(latestResult.snapshotInstant); + } + private BackupVerificationResult verifyBackup( IMetaProxy metaProxy, BackupMetadata latestBackupMetaData) { Path metadataLocation = Paths.get(latestBackupMetaData.getSnapshotLocation()); diff --git a/priam/src/main/java/com/netflix/priam/backup/DirectorySize.java b/priam/src/main/java/com/netflix/priam/backup/DirectorySize.java new file mode 100644 index 000000000..1dc44ec35 --- /dev/null +++ b/priam/src/main/java/com/netflix/priam/backup/DirectorySize.java @@ -0,0 +1,10 @@ +package com.netflix.priam.backup; + +import com.google.inject.ImplementedBy; + +/** estimates the number of bytes remaining to upload in a snapshot */ +@ImplementedBy(SnapshotDirectorySize.class) +public interface DirectorySize { + /** return the total bytes of all snapshot files south of location in the filesystem */ + long getBytes(String location); +} diff --git a/priam/src/main/java/com/netflix/priam/backup/DynamicRateLimiter.java b/priam/src/main/java/com/netflix/priam/backup/DynamicRateLimiter.java new file mode 100644 index 000000000..dd1417d7f --- /dev/null +++ b/priam/src/main/java/com/netflix/priam/backup/DynamicRateLimiter.java @@ -0,0 +1,9 @@ +package com.netflix.priam.backup; + +import com.google.inject.ImplementedBy; +import java.time.Instant; + +@ImplementedBy(BackupDynamicRateLimiter.class) +public interface DynamicRateLimiter { + void acquire(AbstractBackupPath dir, Instant target, int tokens); +} diff --git a/priam/src/main/java/com/netflix/priam/backup/IBackupFileSystem.java b/priam/src/main/java/com/netflix/priam/backup/IBackupFileSystem.java index 7999958a2..176958e2b 100644 --- a/priam/src/main/java/com/netflix/priam/backup/IBackupFileSystem.java +++ b/priam/src/main/java/com/netflix/priam/backup/IBackupFileSystem.java @@ -19,6 +19,7 @@ import com.google.common.util.concurrent.ListenableFuture; import java.io.FileNotFoundException; import java.nio.file.Path; +import java.time.Instant; import java.util.Date; import java.util.Iterator; import java.util.List; @@ -55,6 +56,12 @@ void downloadFile(AbstractBackupPath path, String suffix, int retry) Future asyncDownloadFile(final AbstractBackupPath path, final int retry) throws BackupRestoreException, RejectedExecutionException; + /** Overload that uploads as fast as possible without any custom throttling */ + default void uploadAndDelete(AbstractBackupPath path, int retry) + throws FileNotFoundException, BackupRestoreException { + uploadAndDelete(path, retry, Instant.EPOCH); + } + /** * Upload the local file to its remote counterpart. Both locations are embedded within the path * parameter. De-duping of the file to upload will always be done by comparing the @@ -66,14 +73,22 @@ Future asyncDownloadFile(final AbstractBackupPath path, final int retry) * @param path Backup path representing a local and remote file pair * @param retry No of times to retry to upload a file. If <1, it will try to upload file * exactly once. + * @param target The target time of completion of all files in the upload. * @throws BackupRestoreException in case of failure to upload for any reason including file not * readable or remote file system errors. * @throws FileNotFoundException If a file as denoted by localPath is not available or is a * directory. */ - void uploadAndDelete(AbstractBackupPath path, int retry) + void uploadAndDelete(AbstractBackupPath path, int retry, Instant target) throws FileNotFoundException, BackupRestoreException; + /** Overload that uploads as fast as possible without any custom throttling */ + default ListenableFuture asyncUploadAndDelete( + final AbstractBackupPath path, final int retry) + throws FileNotFoundException, RejectedExecutionException, BackupRestoreException { + return asyncUploadAndDelete(path, retry, Instant.EPOCH); + } + /** * Upload the local file denoted by localPath in async fashion to the remote file system at * location denoted by remotePath. @@ -81,6 +96,7 @@ void uploadAndDelete(AbstractBackupPath path, int retry) * @param path AbstractBackupPath to be used to send backup notifications only. * @param retry No of times to retry to upload a file. If <1, it will try to upload file * exactly once. + * @param target The target time of completion of all files in the upload. * @return The future of the async job to monitor the progress of the job. This will be null if * file was de-duped for upload. * @throws BackupRestoreException in case of failure to upload for any reason including file not @@ -91,7 +107,7 @@ void uploadAndDelete(AbstractBackupPath path, int retry) * to add the work to the queue. */ ListenableFuture asyncUploadAndDelete( - final AbstractBackupPath path, final int retry) + final AbstractBackupPath path, final int retry, Instant target) throws FileNotFoundException, RejectedExecutionException, BackupRestoreException; /** diff --git a/priam/src/main/java/com/netflix/priam/backup/IncrementalBackup.java b/priam/src/main/java/com/netflix/priam/backup/IncrementalBackup.java index 92907859a..2326b1618 100644 --- a/priam/src/main/java/com/netflix/priam/backup/IncrementalBackup.java +++ b/priam/src/main/java/com/netflix/priam/backup/IncrementalBackup.java @@ -93,7 +93,7 @@ public static boolean isEnabled( && (SnapshotBackup.isBackupEnabled(configuration) || (backupRestoreConfig.enableV2Backups() && SnapshotMetaTask.isBackupEnabled( - configuration, backupRestoreConfig)))); + backupRestoreConfig)))); logger.info("Incremental backups are enabled: {}", enabled); if (!enabled) { diff --git a/priam/src/main/java/com/netflix/priam/backup/SnapshotDirectorySize.java b/priam/src/main/java/com/netflix/priam/backup/SnapshotDirectorySize.java new file mode 100644 index 000000000..ab77cfe4a --- /dev/null +++ b/priam/src/main/java/com/netflix/priam/backup/SnapshotDirectorySize.java @@ -0,0 +1,50 @@ +package com.netflix.priam.backup; + +import java.io.IOException; +import java.nio.file.*; +import java.nio.file.attribute.BasicFileAttributes; + +/** Estimates remaining bytes to upload in a backup by looking at the file system */ +public class SnapshotDirectorySize implements DirectorySize { + + public long getBytes(String location) { + SummingFileVisitor fileVisitor = new SummingFileVisitor(); + try { + Files.walkFileTree(Paths.get(location), fileVisitor); + } catch (IOException e) { + // BackupFileVisitor is happy with an estimate and won't produce these in practice. + } + return fileVisitor.getTotalBytes(); + } + + private static final class SummingFileVisitor implements FileVisitor { + private long totalBytes; + + @Override + public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) { + return FileVisitResult.CONTINUE; + } + + @Override + public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) { + if (file.toString().contains(AbstractBackup.SNAPSHOT_FOLDER) && attrs.isRegularFile()) { + totalBytes += attrs.size(); + } + return FileVisitResult.CONTINUE; + } + + @Override + public FileVisitResult visitFileFailed(Path file, IOException exc) { + return FileVisitResult.CONTINUE; + } + + @Override + public FileVisitResult postVisitDirectory(Path dir, IOException exc) { + return FileVisitResult.CONTINUE; + } + + long getTotalBytes() { + return totalBytes; + } + } +} diff --git a/priam/src/main/java/com/netflix/priam/backupv2/BackupV2Service.java b/priam/src/main/java/com/netflix/priam/backupv2/BackupV2Service.java index 9b74047cb..5c02b396e 100644 --- a/priam/src/main/java/com/netflix/priam/backupv2/BackupV2Service.java +++ b/priam/src/main/java/com/netflix/priam/backupv2/BackupV2Service.java @@ -56,7 +56,10 @@ public BackupV2Service( @Override public void scheduleService() throws Exception { - TaskTimer snapshotMetaTimer = SnapshotMetaTask.getTimer(configuration, backupRestoreConfig); + TaskTimer snapshotMetaTimer = SnapshotMetaTask.getTimer(backupRestoreConfig); + if (snapshotMetaTimer == null) { + SnapshotMetaTask.cleanOldBackups(configuration); + } scheduleTask(scheduler, SnapshotMetaTask.class, snapshotMetaTimer); if (snapshotMetaTimer != null) { diff --git a/priam/src/main/java/com/netflix/priam/backupv2/FileUploadResult.java b/priam/src/main/java/com/netflix/priam/backupv2/FileUploadResult.java index f9c5da37d..dc01a5a81 100644 --- a/priam/src/main/java/com/netflix/priam/backupv2/FileUploadResult.java +++ b/priam/src/main/java/com/netflix/priam/backupv2/FileUploadResult.java @@ -73,6 +73,14 @@ public void setUploaded(Boolean uploaded) { isUploaded = uploaded; } + public Boolean getIsUploaded() { + return isUploaded; + } + + public Path getFileName() { + return fileName; + } + public String getBackupPath() { return backupPath; } diff --git a/priam/src/main/java/com/netflix/priam/backupv2/MetaFileWriterBuilder.java b/priam/src/main/java/com/netflix/priam/backupv2/MetaFileWriterBuilder.java index 489d087f8..3bda21e45 100644 --- a/priam/src/main/java/com/netflix/priam/backupv2/MetaFileWriterBuilder.java +++ b/priam/src/main/java/com/netflix/priam/backupv2/MetaFileWriterBuilder.java @@ -65,7 +65,7 @@ public interface StartStep { } public interface DataStep { - DataStep addColumnfamilyResult( + ColumnfamilyResult addColumnfamilyResult( String keyspace, String columnFamily, ImmutableMultimap sstables) @@ -142,7 +142,7 @@ public DataStep startMetaFileGeneration(Instant snapshotInstant) throws IOExcept * * @throws IOException if unable to write to the file or if JSON is not valid */ - public DataStep addColumnfamilyResult( + public ColumnfamilyResult addColumnfamilyResult( String keyspace, String columnFamily, ImmutableMultimap sstables) @@ -151,8 +151,9 @@ public DataStep addColumnfamilyResult( if (jsonWriter == null) throw new NullPointerException( "addColumnfamilyResult: Json Writer in MetaFileWriter is null. This should not happen!"); - jsonWriter.jsonValue(toColumnFamilyResult(keyspace, columnFamily, sstables).toString()); - return this; + ColumnfamilyResult result = toColumnFamilyResult(keyspace, columnFamily, sstables); + jsonWriter.jsonValue(result.toString()); + return result; } /** diff --git a/priam/src/main/java/com/netflix/priam/backupv2/SnapshotMetaTask.java b/priam/src/main/java/com/netflix/priam/backupv2/SnapshotMetaTask.java index 4c85a6bb1..10f3d4821 100644 --- a/priam/src/main/java/com/netflix/priam/backupv2/SnapshotMetaTask.java +++ b/priam/src/main/java/com/netflix/priam/backupv2/SnapshotMetaTask.java @@ -36,10 +36,13 @@ import java.nio.file.DirectoryStream; import java.nio.file.Files; import java.nio.file.Path; +import java.text.ParseException; +import java.time.Clock; +import java.time.Duration; import java.time.Instant; -import java.util.Date; -import java.util.Optional; -import java.util.Set; +import java.time.ZoneId; +import java.time.temporal.ChronoUnit; +import java.util.*; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.locks.Lock; @@ -48,6 +51,7 @@ import javax.inject.Named; import javax.inject.Singleton; import org.apache.commons.io.FileUtils; +import org.quartz.CronExpression; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -73,6 +77,7 @@ public class SnapshotMetaTask extends AbstractBackup { private static final String SNAPSHOT_PREFIX = "snap_v2_"; private static final String CASSANDRA_MANIFEST_FILE = "manifest.json"; private static final String CASSANDRA_SCHEMA_FILE = "schema.cql"; + private static final TimeZone UTC = TimeZone.getTimeZone(ZoneId.of("UTC")); private final BackupRestoreUtil backupRestoreUtil; private final MetaFileWriterBuilder metaFileWriter; private MetaFileWriterBuilder.DataStep dataStep; @@ -83,6 +88,10 @@ public class SnapshotMetaTask extends AbstractBackup { private final IBackupStatusMgr snapshotStatusMgr; private final InstanceIdentity instanceIdentity; private final ExecutorService threadPool; + private final IConfiguration config; + private final Clock clock; + private final IBackupRestoreConfig backupRestoreConfig; + private final BackupVerification backupVerification; private enum MetaStep { META_GENERATION, @@ -100,11 +109,18 @@ private enum MetaStep { @Named("v2") IMetaProxy metaProxy, InstanceIdentity instanceIdentity, IBackupStatusMgr snapshotStatusMgr, - CassandraOperations cassandraOperations) { + CassandraOperations cassandraOperations, + Clock clock, + IBackupRestoreConfig backupRestoreConfig, + BackupVerification backupVerification) { super(config, backupFileSystemCtx, pathFactory); + this.config = config; this.instanceIdentity = instanceIdentity; this.snapshotStatusMgr = snapshotStatusMgr; this.cassandraOperations = cassandraOperations; + this.clock = clock; + this.backupRestoreConfig = backupRestoreConfig; + this.backupVerification = backupVerification; backupRestoreUtil = new BackupRestoreUtil( config.getSnapshotIncludeCFList(), config.getSnapshotExcludeCFList()); @@ -116,31 +132,22 @@ private enum MetaStep { /** * Interval between generating snapshot meta file using {@link SnapshotMetaTask}. * - * @param backupRestoreConfig {@link - * IBackupRestoreConfig#getSnapshotMetaServiceCronExpression()} to get configuration details - * from priam. Use "-1" to disable the service. - * @param config configuration to get the data folder. + * @param config {@link IBackupRestoreConfig#getSnapshotMetaServiceCronExpression()} to get + * configuration details from priam. Use "-1" to disable the service. * @return the timer to be used for snapshot meta service. - * @throws Exception if the configuration is not set correctly or are not valid. This is to - * ensure we fail-fast. + * @throws IllegalArgumentException if the configuration is not set correctly or are not valid. + * This is to ensure we fail-fast. */ - public static TaskTimer getTimer( - IConfiguration config, IBackupRestoreConfig backupRestoreConfig) throws Exception { - TaskTimer timer = - CronTimer.getCronTimer( - JOBNAME, backupRestoreConfig.getSnapshotMetaServiceCronExpression()); - if (timer == null) { - cleanOldBackups(config); - } - return timer; + public static TaskTimer getTimer(IBackupRestoreConfig config) throws IllegalArgumentException { + return CronTimer.getCronTimer(JOBNAME, config.getSnapshotMetaServiceCronExpression()); } - private static void cleanOldBackups(IConfiguration config) throws Exception { + static void cleanOldBackups(IConfiguration config) throws Exception { // Clean up all the backup directories, if any. Set backupPaths = AbstractBackup.getBackupDirectories(config, SNAPSHOT_FOLDER); for (Path backupDirPath : backupPaths) try (DirectoryStream directoryStream = - Files.newDirectoryStream(backupDirPath, path -> Files.isDirectory(path))) { + Files.newDirectoryStream(backupDirPath, Files::isDirectory)) { for (Path backupDir : directoryStream) { if (backupDir.toFile().getName().startsWith(SNAPSHOT_PREFIX)) { FileUtils.deleteDirectory(backupDir.toFile()); @@ -149,10 +156,9 @@ private static void cleanOldBackups(IConfiguration config) throws Exception { } } - public static boolean isBackupEnabled( - IConfiguration configuration, IBackupRestoreConfig backupRestoreConfig) + public static boolean isBackupEnabled(IBackupRestoreConfig backupRestoreConfig) throws Exception { - return (getTimer(configuration, backupRestoreConfig) != null); + return (getTimer(backupRestoreConfig) != null); } String generateSnapshotName(Instant snapshotInstant) { @@ -194,7 +200,7 @@ public void execute() throws Exception { } // Save start snapshot status - Instant snapshotInstant = DateUtil.getInstant(); + Instant snapshotInstant = clock.instant(); String token = instanceIdentity.getInstance().getToken(); BackupMetadata backupMetadata = new BackupMetadata( @@ -264,6 +270,7 @@ private void uploadAllFiles(final File backupDir) throws Exception { // (like we exhausted the wait time for upload) File[] snapshotDirectories = backupDir.listFiles(); if (snapshotDirectories != null) { + Instant target = getUploadTarget(); for (File snapshotDirectory : snapshotDirectories) { // Is it a valid SNAPSHOT_PREFIX if (!snapshotDirectory.getName().startsWith(SNAPSHOT_PREFIX) @@ -278,13 +285,13 @@ private void uploadAllFiles(final File backupDir) throws Exception { // We do not want to wait for completion and we just want to add them to queue. This // is to ensure that next run happens on time. AbstractBackupPath.BackupFileType type = AbstractBackupPath.BackupFileType.SST_V2; - uploadAndDeleteAllFiles(snapshotDirectory, type, true); + uploadAndDeleteAllFiles(snapshotDirectory, type, true, target); // Next, upload secondary indexes type = AbstractBackupPath.BackupFileType.SECONDARY_INDEX_V2; ImmutableList> futures; for (File subDir : getSecondaryIndexDirectories(snapshotDirectory)) { - futures = uploadAndDeleteAllFiles(subDir, type, true); + futures = uploadAndDeleteAllFiles(subDir, type, true, target); if (futures.isEmpty()) { deleteIfEmpty(subDir); } @@ -294,6 +301,35 @@ private void uploadAllFiles(final File backupDir) throws Exception { } } + private Instant getUploadTarget() { + Instant now = clock.instant(); + Instant target = + now.plus(config.getTargetMinutesToCompleteSnaphotUpload(), ChronoUnit.MINUTES); + Duration verificationSLO = + Duration.ofHours(backupRestoreConfig.getBackupVerificationSLOInHours()); + Instant verificationDeadline = + backupVerification + .getLatestVerfifiedBackupTime() + .map(backupTime -> backupTime.plus(verificationSLO)) + .orElse(Instant.MAX); + Instant nextSnapshotTime; + try { + CronExpression snapshotCron = + new CronExpression(backupRestoreConfig.getSnapshotMetaServiceCronExpression()); + snapshotCron.setTimeZone(UTC); + Date nextSnapshotDate = snapshotCron.getNextValidTimeAfter(Date.from(Instant.now())); + nextSnapshotTime = + nextSnapshotDate == null ? Instant.MAX : nextSnapshotDate.toInstant(); + } catch (ParseException e) { + nextSnapshotTime = Instant.MAX; + } + return earliest(target, verificationDeadline, nextSnapshotTime); + } + + private Instant earliest(Instant... instants) { + return Arrays.stream(instants).min(Instant::compareTo).get(); + } + private Void deleteIfEmpty(File dir) { if (FileUtils.sizeOfDirectory(dir) == 0) FileUtils.deleteQuietly(dir); return null; @@ -305,7 +341,8 @@ protected void processColumnFamily(File backupDir) throws Exception { String columnFamily = getColumnFamily(backupDir); switch (metaStep) { case META_GENERATION: - generateMetaFile(keyspace, columnFamily, backupDir); + generateMetaFile(keyspace, columnFamily, backupDir) + .ifPresent(this::deleteUploadedFiles); break; case UPLOAD_FILES: uploadAllFiles(backupDir); @@ -315,14 +352,14 @@ protected void processColumnFamily(File backupDir) throws Exception { } } - private void generateMetaFile( + private Optional generateMetaFile( final String keyspace, final String columnFamily, final File backupDir) throws Exception { File snapshotDir = getValidSnapshot(backupDir, snapshotName); // Process this snapshot folder for the given columnFamily if (snapshotDir == null) { logger.warn("{} folder does not contain {} snapshots", backupDir, snapshotName); - return; + return Optional.empty(); } logger.debug("Scanning for all SSTables in: {}", snapshotDir.getAbsolutePath()); @@ -338,8 +375,18 @@ private void generateMetaFile( ImmutableSetMultimap sstables = builder.build(); logger.debug("Processing {} sstables from {}.{}", keyspace, columnFamily, sstables.size()); - dataStep.addColumnfamilyResult(keyspace, columnFamily, sstables); + ColumnfamilyResult result = + dataStep.addColumnfamilyResult(keyspace, columnFamily, sstables); logger.debug("Finished processing KS: {}, CF: {}", keyspace, columnFamily); + return Optional.of(result); + } + + private void deleteUploadedFiles(ColumnfamilyResult result) { + result.getSstables() + .stream() + .flatMap(sstable -> sstable.getSstableComponents().stream()) + .filter(file -> Boolean.TRUE.equals(file.getIsUploaded())) + .forEach(file -> FileUtils.deleteQuietly(file.getFileName().toFile())); } private ImmutableSetMultimap getSSTables( diff --git a/priam/src/main/java/com/netflix/priam/config/IConfiguration.java b/priam/src/main/java/com/netflix/priam/config/IConfiguration.java index 35abf7818..b013c62dd 100644 --- a/priam/src/main/java/com/netflix/priam/config/IConfiguration.java +++ b/priam/src/main/java/com/netflix/priam/config/IConfiguration.java @@ -1148,6 +1148,19 @@ default boolean permitDirectTokenAssignmentWithGossipMismatch() { return false; } + /** returns how long a snapshot backup should take to upload in minutes */ + default int getTargetMinutesToCompleteSnaphotUpload() { + return 0; + } + + /** + * @return the percentage off of the old rate that the current rate must be to trigger a new + * rate in the dynamic rate limiter + */ + default double getRateLimitChangeThreshold() { + return 0.1; + } + /** * Escape hatch for getting any arbitrary property by key This is useful so we don't have to * keep adding methods to this interface for every single configuration option ever. Also diff --git a/priam/src/main/java/com/netflix/priam/config/PriamConfiguration.java b/priam/src/main/java/com/netflix/priam/config/PriamConfiguration.java index d0d449f50..a7193655e 100644 --- a/priam/src/main/java/com/netflix/priam/config/PriamConfiguration.java +++ b/priam/src/main/java/com/netflix/priam/config/PriamConfiguration.java @@ -789,4 +789,14 @@ public boolean checkThriftServerIsListening() { public boolean permitDirectTokenAssignmentWithGossipMismatch() { return config.get(PRIAM_PRE + ".permitDirectTokenAssignmentWithGossipMismatch", false); } + + @Override + public int getTargetMinutesToCompleteSnaphotUpload() { + return config.get(PRIAM_PRE + ".snapshotUploadDuration", 0); + } + + @Override + public double getRateLimitChangeThreshold() { + return config.get(PRIAM_PRE + ".rateLimitChangeThreshold", 0.1); + } } diff --git a/priam/src/main/java/com/netflix/priam/google/GoogleEncryptedFileSystem.java b/priam/src/main/java/com/netflix/priam/google/GoogleEncryptedFileSystem.java index 49455517b..f008e5726 100755 --- a/priam/src/main/java/com/netflix/priam/google/GoogleEncryptedFileSystem.java +++ b/priam/src/main/java/com/netflix/priam/google/GoogleEncryptedFileSystem.java @@ -34,6 +34,7 @@ import com.netflix.priam.notification.BackupNotificationMgr; import java.io.*; import java.nio.file.Path; +import java.time.Instant; import java.util.ArrayList; import java.util.Collection; import java.util.Iterator; @@ -236,7 +237,8 @@ public void shutdown() { } @Override - protected long uploadFileImpl(AbstractBackupPath path) throws BackupRestoreException { + protected long uploadFileImpl(AbstractBackupPath path, Instant target) + throws BackupRestoreException { throw new UnsupportedOperationException(); } diff --git a/priam/src/test/java/com/netflix/priam/backup/BRTestModule.java b/priam/src/test/java/com/netflix/priam/backup/BRTestModule.java index 29c87b051..6a1425f49 100644 --- a/priam/src/test/java/com/netflix/priam/backup/BRTestModule.java +++ b/priam/src/test/java/com/netflix/priam/backup/BRTestModule.java @@ -34,7 +34,10 @@ import com.netflix.priam.cryptography.pgp.PgpCryptography; import com.netflix.priam.defaultimpl.FakeCassandraProcess; import com.netflix.priam.defaultimpl.ICassandraProcess; -import com.netflix.priam.identity.*; +import com.netflix.priam.identity.FakeMembership; +import com.netflix.priam.identity.FakePriamInstanceFactory; +import com.netflix.priam.identity.IMembership; +import com.netflix.priam.identity.IPriamInstanceFactory; import com.netflix.priam.identity.config.FakeInstanceInfo; import com.netflix.priam.identity.config.InstanceInfo; import com.netflix.priam.restore.IPostRestoreHook; @@ -42,6 +45,9 @@ import com.netflix.priam.utils.Sleeper; import com.netflix.spectator.api.DefaultRegistry; import com.netflix.spectator.api.Registry; +import java.time.Clock; +import java.time.Instant; +import java.time.ZoneId; import java.util.Collections; import org.junit.Ignore; import org.quartz.SchedulerFactory; @@ -83,5 +89,7 @@ protected void configure() { bind(Registry.class).toInstance(new DefaultRegistry()); bind(IMetaProxy.class).annotatedWith(Names.named("v1")).to(MetaV1Proxy.class); bind(IMetaProxy.class).annotatedWith(Names.named("v2")).to(MetaV2Proxy.class); + bind(DynamicRateLimiter.class).to(FakeDynamicRateLimiter.class); + bind(Clock.class).toInstance(Clock.fixed(Instant.EPOCH, ZoneId.systemDefault())); } } diff --git a/priam/src/test/java/com/netflix/priam/backup/FakeBackupFileSystem.java b/priam/src/test/java/com/netflix/priam/backup/FakeBackupFileSystem.java index 86d5d822e..8807e4b4d 100644 --- a/priam/src/test/java/com/netflix/priam/backup/FakeBackupFileSystem.java +++ b/priam/src/test/java/com/netflix/priam/backup/FakeBackupFileSystem.java @@ -28,6 +28,7 @@ import java.io.FileWriter; import java.io.IOException; import java.nio.file.Path; +import java.time.Instant; import java.util.*; import org.json.simple.JSONArray; @@ -166,7 +167,8 @@ protected void downloadFileImpl(AbstractBackupPath path, String suffix) } @Override - protected long uploadFileImpl(AbstractBackupPath path) throws BackupRestoreException { + protected long uploadFileImpl(AbstractBackupPath path, Instant target) + throws BackupRestoreException { uploadedFiles.add(path.getBackupFile().getAbsolutePath()); addFile(path.getRemotePath()); return path.getBackupFile().length(); diff --git a/priam/src/test/java/com/netflix/priam/backup/FakeDynamicRateLimiter.java b/priam/src/test/java/com/netflix/priam/backup/FakeDynamicRateLimiter.java new file mode 100644 index 000000000..2108d3567 --- /dev/null +++ b/priam/src/test/java/com/netflix/priam/backup/FakeDynamicRateLimiter.java @@ -0,0 +1,8 @@ +package com.netflix.priam.backup; + +import java.time.Instant; + +public class FakeDynamicRateLimiter implements DynamicRateLimiter { + @Override + public void acquire(AbstractBackupPath dir, Instant target, int tokens) {} +} diff --git a/priam/src/test/java/com/netflix/priam/backup/NullBackupFileSystem.java b/priam/src/test/java/com/netflix/priam/backup/NullBackupFileSystem.java index d074a5e99..6dbf3941f 100644 --- a/priam/src/test/java/com/netflix/priam/backup/NullBackupFileSystem.java +++ b/priam/src/test/java/com/netflix/priam/backup/NullBackupFileSystem.java @@ -23,6 +23,7 @@ import com.netflix.priam.merics.BackupMetrics; import com.netflix.priam.notification.BackupNotificationMgr; import java.nio.file.Path; +import java.time.Instant; import java.util.Collections; import java.util.Iterator; import java.util.List; @@ -72,7 +73,8 @@ protected boolean doesRemoteFileExist(Path remotePath) { } @Override - protected long uploadFileImpl(AbstractBackupPath path) throws BackupRestoreException { + protected long uploadFileImpl(AbstractBackupPath path, Instant target) + throws BackupRestoreException { return 0; } } diff --git a/priam/src/test/java/com/netflix/priam/backup/TestAbstractFileSystem.java b/priam/src/test/java/com/netflix/priam/backup/TestAbstractFileSystem.java index e042cd0ac..47cf1d1fa 100644 --- a/priam/src/test/java/com/netflix/priam/backup/TestAbstractFileSystem.java +++ b/priam/src/test/java/com/netflix/priam/backup/TestAbstractFileSystem.java @@ -29,6 +29,7 @@ import java.nio.file.Path; import java.nio.file.Paths; import java.text.ParseException; +import java.time.Instant; import java.util.ArrayList; import java.util.Collection; import java.util.List; @@ -284,7 +285,8 @@ protected void downloadFileImpl(AbstractBackupPath path, String suffix) } @Override - protected long uploadFileImpl(AbstractBackupPath path) throws BackupRestoreException { + protected long uploadFileImpl(AbstractBackupPath path, Instant target) + throws BackupRestoreException { throw new BackupRestoreException( "User injected failure file system error for testing upload. Local path: " + path.getBackupFile().getAbsolutePath()); @@ -315,7 +317,8 @@ protected void downloadFileImpl(AbstractBackupPath path, String suffix) } @Override - protected long uploadFileImpl(AbstractBackupPath path) throws BackupRestoreException { + protected long uploadFileImpl(AbstractBackupPath path, Instant target) + throws BackupRestoreException { try { Thread.sleep(random.nextInt(20)); } catch (InterruptedException e) { diff --git a/priam/src/test/java/com/netflix/priam/backup/TestBackupDynamicRateLimiter.java b/priam/src/test/java/com/netflix/priam/backup/TestBackupDynamicRateLimiter.java new file mode 100644 index 000000000..1c607ba7d --- /dev/null +++ b/priam/src/test/java/com/netflix/priam/backup/TestBackupDynamicRateLimiter.java @@ -0,0 +1,159 @@ +package com.netflix.priam.backup; + +import com.google.common.base.Stopwatch; +import com.google.common.collect.ImmutableMap; +import com.google.common.truth.Truth; +import com.google.inject.Guice; +import com.google.inject.Injector; +import com.netflix.priam.aws.RemoteBackupPath; +import com.netflix.priam.config.FakeConfiguration; +import java.nio.file.Paths; +import java.time.Clock; +import java.time.Duration; +import java.time.Instant; +import java.time.ZoneId; +import java.util.Map; +import java.util.concurrent.TimeUnit; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +public class TestBackupDynamicRateLimiter { + private static final Instant NOW = Instant.ofEpochMilli(1 << 16); + private static final Instant LATER = NOW.plusMillis(Duration.ofHours(1).toMillis()); + private static final int DIR_SIZE = 1 << 16; + + private BackupDynamicRateLimiter rateLimiter; + private FakeConfiguration config; + private Injector injector; + + @Before + public void setUp() { + injector = Guice.createInjector(new BRTestModule()); + config = injector.getInstance(FakeConfiguration.class); + } + + @Test + public void sunnyDay() { + rateLimiter = getRateLimiter(ImmutableMap.of("Priam.backup.threads", 1), NOW, DIR_SIZE); + Stopwatch timer = timePermitAcquisition(getBackupPath(), LATER, 21); + Truth.assertThat(timer.elapsed(TimeUnit.MILLISECONDS)).isAtLeast(1_000); + Truth.assertThat(timer.elapsed(TimeUnit.MILLISECONDS)).isAtMost(2_000); + } + + @Test + public void targetSetToEpoch() { + rateLimiter = getRateLimiter(ImmutableMap.of("Priam.backup.threads", 1), NOW, DIR_SIZE); + Stopwatch timer = timePermitAcquisition(getBackupPath(), Instant.EPOCH, 20); + assertNoRateLimiting(timer); + } + + @Test + public void pathIsNotASnapshot() { + rateLimiter = getRateLimiter(ImmutableMap.of("Priam.backup.threads", 1), NOW, DIR_SIZE); + AbstractBackupPath path = + getBackupPath( + "target/data/Keyspace1/Standard1/backups/Keyspace1-Standard1-ia-4-Data.db"); + Stopwatch timer = timePermitAcquisition(path, LATER, 20); + assertNoRateLimiting(timer); + } + + @Test + public void targetIsNow() { + rateLimiter = getRateLimiter(ImmutableMap.of("Priam.backup.threads", 1), NOW, DIR_SIZE); + Stopwatch timer = timePermitAcquisition(getBackupPath(), NOW, 20); + assertNoRateLimiting(timer); + } + + @Test + public void targetIsInThePast() { + rateLimiter = getRateLimiter(ImmutableMap.of("Priam.backup.threads", 1), NOW, DIR_SIZE); + Instant target = NOW.minus(Duration.ofHours(1L)); + Stopwatch timer = timePermitAcquisition(getBackupPath(), target, 20); + assertNoRateLimiting(timer); + } + + @Test + public void noBackupThreads() { + rateLimiter = getRateLimiter(ImmutableMap.of("Priam.backup.threads", 0), NOW, DIR_SIZE); + Assert.assertThrows( + IllegalStateException.class, + () -> timePermitAcquisition(getBackupPath(), LATER, 20)); + } + + @Test + public void negativeBackupThreads() { + rateLimiter = getRateLimiter(ImmutableMap.of("Priam.backup.threads", -1), NOW, DIR_SIZE); + Assert.assertThrows( + IllegalStateException.class, + () -> timePermitAcquisition(getBackupPath(), LATER, 20)); + } + + @Test + public void noData() { + rateLimiter = getRateLimiter(ImmutableMap.of("Priam.backup.threads", 1), NOW, 0); + Stopwatch timer = timePermitAcquisition(getBackupPath(), LATER, 20); + assertNoRateLimiting(timer); + } + + @Test + public void noPermitsRequested() { + rateLimiter = getRateLimiter(ImmutableMap.of("Priam.backup.threads", 1), NOW, DIR_SIZE); + Assert.assertThrows( + IllegalArgumentException.class, + () -> timePermitAcquisition(getBackupPath(), LATER, 0)); + } + + @Test + public void negativePermitsRequested() { + rateLimiter = getRateLimiter(ImmutableMap.of("Priam.backup.threads", 1), NOW, DIR_SIZE); + Assert.assertThrows( + IllegalArgumentException.class, + () -> timePermitAcquisition(getBackupPath(), LATER, -1)); + } + + private RemoteBackupPath getBackupPath() { + return getBackupPath( + "target/data/Keyspace1/Standard1/snapshots/snap_v2_202201010000/.STANDARD1_field1_idx_1/Keyspace1-Standard1-ia-4-Data.db"); + } + + private RemoteBackupPath getBackupPath(String filePath) { + RemoteBackupPath path = injector.getInstance(RemoteBackupPath.class); + path.parseLocal(Paths.get(filePath).toFile(), AbstractBackupPath.BackupFileType.SST_V2); + return path; + } + + private Stopwatch timePermitAcquisition(AbstractBackupPath path, Instant now, int permits) { + rateLimiter.acquire(path, now, permits); // Do this once first or else it won't throttle. + Stopwatch timer = Stopwatch.createStarted(); + rateLimiter.acquire(path, now, permits); + timer.stop(); + return timer; + } + + private BackupDynamicRateLimiter getRateLimiter( + Map properties, Instant now, long directorySize) { + properties.forEach(config::setFakeConfig); + return new BackupDynamicRateLimiter( + config, + Clock.fixed(now, ZoneId.systemDefault()), + new FakeDirectorySize(directorySize)); + } + + private void assertNoRateLimiting(Stopwatch timer) { + Truth.assertThat(timer.elapsed(TimeUnit.MILLISECONDS)).isAtMost(1); + } + + private static final class FakeDirectorySize implements DirectorySize { + private final long size; + + FakeDirectorySize(long size) { + this.size = size; + } + + @Override + public long getBytes(String location) { + return size; + } + } +} diff --git a/priam/src/test/java/com/netflix/priam/backupv2/TestSnapshotMetaTask.java b/priam/src/test/java/com/netflix/priam/backupv2/TestSnapshotMetaTask.java index 4ae4de2c1..913b09364 100644 --- a/priam/src/test/java/com/netflix/priam/backupv2/TestSnapshotMetaTask.java +++ b/priam/src/test/java/com/netflix/priam/backupv2/TestSnapshotMetaTask.java @@ -64,7 +64,7 @@ public void setUp() { @Test public void testSnapshotMetaServiceEnabled() throws Exception { - TaskTimer taskTimer = SnapshotMetaTask.getTimer(configuration, backupRestoreConfig); + TaskTimer taskTimer = SnapshotMetaTask.getTimer(backupRestoreConfig); Assert.assertNotNull(taskTimer); } diff --git a/priam/src/test/java/com/netflix/priam/config/FakeConfiguration.java b/priam/src/test/java/com/netflix/priam/config/FakeConfiguration.java index e9b61cfff..046b367f5 100644 --- a/priam/src/test/java/com/netflix/priam/config/FakeConfiguration.java +++ b/priam/src/test/java/com/netflix/priam/config/FakeConfiguration.java @@ -285,4 +285,11 @@ public boolean skipIngressUnlessIPIsPublic() { public void setSkipIngressUnlessIPIsPublic(boolean skipIngressUnlessIPIsPublic) { this.skipIngressUnlessIPIsPublic = skipIngressUnlessIPIsPublic; } + + @Override + public int getBackupThreads() { + return (Integer) + fakeConfig.getOrDefault( + "Priam.backup.threads", IConfiguration.super.getBackupThreads()); + } } From 702e7ee0af027b79c655bd730393faf0b5f27128 Mon Sep 17 00:00:00 2001 From: mattl-netflix <63665634+mattl-netflix@users.noreply.github.com> Date: Thu, 2 Jun 2022 16:05:49 -0700 Subject: [PATCH 201/228] Optionally Add Content-MD5 header to uploads of files smaller than the backupChunkSize. (#985) * Preliminary refactoring in advance of adding Content-MD5 header to single part uploads. That header is required when objects have a retention period. * Optionally add md5 header for direct puts. --- .../com/netflix/priam/aws/S3FileSystem.java | 73 +++++++++++-------- .../netflix/priam/config/IConfiguration.java | 4 + 2 files changed, 46 insertions(+), 31 deletions(-) diff --git a/priam/src/main/java/com/netflix/priam/aws/S3FileSystem.java b/priam/src/main/java/com/netflix/priam/aws/S3FileSystem.java index e24f68100..ce597c13e 100644 --- a/priam/src/main/java/com/netflix/priam/aws/S3FileSystem.java +++ b/priam/src/main/java/com/netflix/priam/aws/S3FileSystem.java @@ -37,6 +37,7 @@ import com.netflix.priam.merics.BackupMetrics; import com.netflix.priam.notification.BackupNotificationMgr; import com.netflix.priam.utils.BoundedExponentialRetryCallable; +import com.netflix.priam.utils.SystemUtils; import java.io.*; import java.nio.file.Path; import java.nio.file.Paths; @@ -172,45 +173,55 @@ private long uploadMultipart(AbstractBackupPath path, Instant target) protected long uploadFileImpl(AbstractBackupPath path, Instant target) throws BackupRestoreException { - Path localPath = Paths.get(path.getBackupFile().getAbsolutePath()); - String remotePath = path.getRemotePath(); - long chunkSize = config.getBackupChunkSize(); - File localFile = localPath.toFile(); - if (localFile.length() >= chunkSize) return uploadMultipart(path, target); + File localFile = Paths.get(path.getBackupFile().getAbsolutePath()).toFile(); + if (localFile.length() >= config.getBackupChunkSize()) return uploadMultipart(path, target); + byte[] chunk = getFileContents(path); + // C* snapshots may have empty files. That is probably unintentional. + if (chunk.length > 0) { + rateLimiter.acquire(chunk.length); + dynamicRateLimiter.acquire(path, target, chunk.length); + } + try { + new BoundedExponentialRetryCallable(1000, 10000, 5) { + @Override + public PutObjectResult retriableCall() { + return s3Client.putObject(generatePut(path, chunk)); + } + }.call(); + } catch (Exception e) { + throw new BackupRestoreException("Error uploading file: " + localFile.getName(), e); + } + return chunk.length; + } - String prefix = config.getBackupPrefix(); - if (logger.isDebugEnabled()) logger.debug("PUTing {}/{}", prefix, remotePath); + private PutObjectRequest generatePut(AbstractBackupPath path, byte[] chunk) { + File localFile = Paths.get(path.getBackupFile().getAbsolutePath()).toFile(); + ObjectMetadata metadata = getObjectMetadata(localFile); + metadata.setContentLength(chunk.length); + PutObjectRequest put = + new PutObjectRequest( + config.getBackupPrefix(), + path.getRemotePath(), + new ByteArrayInputStream(chunk), + metadata); + if (config.addMD5ToBackupUploads()) { + put.getMetadata().setContentMD5(SystemUtils.toBase64(SystemUtils.md5(chunk))); + } + return put; + } + private byte[] getFileContents(AbstractBackupPath path) throws BackupRestoreException { + File localFile = Paths.get(path.getBackupFile().getAbsolutePath()).toFile(); try (ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); InputStream in = new BufferedInputStream(new FileInputStream(localFile))) { - Iterator chunks = new ChunkedStream(in, chunkSize, path.getCompression()); + Iterator chunks = + new ChunkedStream(in, config.getBackupChunkSize(), path.getCompression()); while (chunks.hasNext()) { byteArrayOutputStream.write(chunks.next()); } - byte[] chunk = byteArrayOutputStream.toByteArray(); - long compressedFileSize = chunk.length; - // C* snapshots may have empty files. That is probably unintentional. - if (chunk.length > 0) { - rateLimiter.acquire(chunk.length); - dynamicRateLimiter.acquire(path, target, chunk.length); - } - ObjectMetadata objectMetadata = getObjectMetadata(localFile); - objectMetadata.setContentLength(chunk.length); - ByteArrayInputStream inputStream = new ByteArrayInputStream(chunk); - PutObjectRequest putObjectRequest = - new PutObjectRequest(prefix, remotePath, inputStream, objectMetadata); - PutObjectResult upload = - new BoundedExponentialRetryCallable(1000, 10000, 5) { - @Override - public PutObjectResult retriableCall() { - return s3Client.putObject(putObjectRequest); - } - }.call(); - if (logger.isDebugEnabled()) - logger.debug("Put: {} with etag: {}", remotePath, upload.getETag()); - return compressedFileSize; + return byteArrayOutputStream.toByteArray(); } catch (Exception e) { - throw new BackupRestoreException("Error uploading file: " + localFile.getName(), e); + throw new BackupRestoreException("Error reading file: " + localFile.getName(), e); } } } diff --git a/priam/src/main/java/com/netflix/priam/config/IConfiguration.java b/priam/src/main/java/com/netflix/priam/config/IConfiguration.java index b013c62dd..e5365d410 100644 --- a/priam/src/main/java/com/netflix/priam/config/IConfiguration.java +++ b/priam/src/main/java/com/netflix/priam/config/IConfiguration.java @@ -1161,6 +1161,10 @@ default double getRateLimitChangeThreshold() { return 0.1; } + default boolean addMD5ToBackupUploads() { + return false; + } + /** * Escape hatch for getting any arbitrary property by key This is useful so we don't have to * keep adding methods to this interface for every single configuration option ever. Also From a45b62aca6c972514638c41d14dd84b620e93e7a Mon Sep 17 00:00:00 2001 From: Matt Lehman Date: Thu, 2 Jun 2022 17:19:39 -0700 Subject: [PATCH 202/228] Update CHANGELOG in advance of 3.11.89 --- CHANGELOG.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index a908f8e88..d0bbab97a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,9 @@ # Changelog +## 2022/06/02 3.11.89 +*UpdateSecuritySettings will be removed in the next release* +(#975) Dynamic Rate Limiting of Snapshots. +(#985) Optionally Add Content-MD5 header to uploads of files smaller than the backupChunkSize. + ## 2022/01/25 3.11.88 (#979) Cache region in AWSInstanceInfo to avoid throttling. From a3397eff62183f7a134a3e6206a6be96dbfca428 Mon Sep 17 00:00:00 2001 From: mattl-netflix <63665634+mattl-netflix@users.noreply.github.com> Date: Sun, 26 Jun 2022 08:15:46 -0700 Subject: [PATCH 203/228] Update comment with crucial part of contract. (#989) --- .../main/java/com/netflix/priam/backup/IBackupFileSystem.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/priam/src/main/java/com/netflix/priam/backup/IBackupFileSystem.java b/priam/src/main/java/com/netflix/priam/backup/IBackupFileSystem.java index 176958e2b..b8a69ec94 100644 --- a/priam/src/main/java/com/netflix/priam/backup/IBackupFileSystem.java +++ b/priam/src/main/java/com/netflix/priam/backup/IBackupFileSystem.java @@ -143,8 +143,8 @@ default String getShard() { Iterator listPrefixes(Date date); /** - * List all the files with the given prefix, delimiter, and marker. This should never return - * null. + * List all the files with the given prefix, delimiter, and marker. Files should be returned + * ordered by last modified time descending. This should never return null. * * @param prefix Common prefix of the elements to search in the backup file system. * @param delimiter All the object will end with this delimiter. From ddef6e75b3853a300a7ea91206a31df827f638ee Mon Sep 17 00:00:00 2001 From: mattl-netflix <63665634+mattl-netflix@users.noreply.github.com> Date: Sun, 26 Jun 2022 08:33:02 -0700 Subject: [PATCH 204/228] Capitalize 'f' in ColumnfamilyResult pursuant to old review comment. (#991) --- .../netflix/priam/backupv2/BackupTTLTask.java | 2 +- ...familyResult.java => ColumnFamilyResult.java} | 4 ++-- .../netflix/priam/backupv2/MetaFileReader.java | 6 +++--- .../priam/backupv2/MetaFileWriterBuilder.java | 16 ++++++++-------- .../com/netflix/priam/backupv2/MetaV2Proxy.java | 8 ++++---- .../netflix/priam/backupv2/SnapshotMetaTask.java | 6 +++--- .../priam/backupv2/TestSnapshotMetaTask.java | 2 +- 7 files changed, 22 insertions(+), 22 deletions(-) rename priam/src/main/java/com/netflix/priam/backupv2/{ColumnfamilyResult.java => ColumnFamilyResult.java} (95%) diff --git a/priam/src/main/java/com/netflix/priam/backupv2/BackupTTLTask.java b/priam/src/main/java/com/netflix/priam/backupv2/BackupTTLTask.java index 6f6e36419..71c43b839 100644 --- a/priam/src/main/java/com/netflix/priam/backupv2/BackupTTLTask.java +++ b/priam/src/main/java/com/netflix/priam/backupv2/BackupTTLTask.java @@ -246,7 +246,7 @@ public static TaskTimer getTimer(IBackupRestoreConfig backupRestoreConfig) throw private class MetaFileWalker extends MetaFileReader { @Override - public void process(ColumnfamilyResult columnfamilyResult) { + public void process(ColumnFamilyResult columnfamilyResult) { columnfamilyResult .getSstables() .forEach( diff --git a/priam/src/main/java/com/netflix/priam/backupv2/ColumnfamilyResult.java b/priam/src/main/java/com/netflix/priam/backupv2/ColumnFamilyResult.java similarity index 95% rename from priam/src/main/java/com/netflix/priam/backupv2/ColumnfamilyResult.java rename to priam/src/main/java/com/netflix/priam/backupv2/ColumnFamilyResult.java index 8f637d53e..5c273821e 100644 --- a/priam/src/main/java/com/netflix/priam/backupv2/ColumnfamilyResult.java +++ b/priam/src/main/java/com/netflix/priam/backupv2/ColumnFamilyResult.java @@ -23,12 +23,12 @@ * This is a POJO to encapsulate all the SSTables for a given column family. Created by aagrawal on * 7/1/18. */ -public class ColumnfamilyResult { +public class ColumnFamilyResult { private String keyspaceName; private String columnfamilyName; private List sstables = new ArrayList<>(); - public ColumnfamilyResult(String keyspaceName, String columnfamilyName) { + public ColumnFamilyResult(String keyspaceName, String columnfamilyName) { this.keyspaceName = keyspaceName; this.columnfamilyName = columnfamilyName; } diff --git a/priam/src/main/java/com/netflix/priam/backupv2/MetaFileReader.java b/priam/src/main/java/com/netflix/priam/backupv2/MetaFileReader.java index 17382f79f..701bf21d0 100644 --- a/priam/src/main/java/com/netflix/priam/backupv2/MetaFileReader.java +++ b/priam/src/main/java/com/netflix/priam/backupv2/MetaFileReader.java @@ -70,7 +70,7 @@ public void readMeta(Path metaFilePath) throws IOException { while (jsonReader.hasNext()) process( GsonJsonSerializer.getGson() - .fromJson(jsonReader, ColumnfamilyResult.class)); + .fromJson(jsonReader, ColumnFamilyResult.class)); jsonReader.endArray(); } } @@ -82,10 +82,10 @@ public void readMeta(Path metaFilePath) throws IOException { /** * Process the columnfamily result obtained after reading meta file. * - * @param columnfamilyResult {@link ColumnfamilyResult} POJO containing the column family data + * @param columnfamilyResult {@link ColumnFamilyResult} POJO containing the column family data * (all SSTables references) obtained from meta.json. */ - public abstract void process(ColumnfamilyResult columnfamilyResult); + public abstract void process(ColumnFamilyResult columnfamilyResult); /** * Returns if it is a valid meta file name. diff --git a/priam/src/main/java/com/netflix/priam/backupv2/MetaFileWriterBuilder.java b/priam/src/main/java/com/netflix/priam/backupv2/MetaFileWriterBuilder.java index 3bda21e45..7c5ec79ea 100644 --- a/priam/src/main/java/com/netflix/priam/backupv2/MetaFileWriterBuilder.java +++ b/priam/src/main/java/com/netflix/priam/backupv2/MetaFileWriterBuilder.java @@ -65,7 +65,7 @@ public interface StartStep { } public interface DataStep { - ColumnfamilyResult addColumnfamilyResult( + ColumnFamilyResult addColumnfamilyResult( String keyspace, String columnFamily, ImmutableMultimap sstables) @@ -137,12 +137,12 @@ public DataStep startMetaFileGeneration(Instant snapshotInstant) throws IOExcept } /** - * Add {@link ColumnfamilyResult} after it has been processed so it can be streamed to + * Add {@link ColumnFamilyResult} after it has been processed so it can be streamed to * meta.json. Streaming write to meta.json is required so we don't get Priam OOM. * * @throws IOException if unable to write to the file or if JSON is not valid */ - public ColumnfamilyResult addColumnfamilyResult( + public ColumnFamilyResult addColumnfamilyResult( String keyspace, String columnFamily, ImmutableMultimap sstables) @@ -151,7 +151,7 @@ public ColumnfamilyResult addColumnfamilyResult( if (jsonWriter == null) throw new NullPointerException( "addColumnfamilyResult: Json Writer in MetaFileWriter is null. This should not happen!"); - ColumnfamilyResult result = toColumnFamilyResult(keyspace, columnFamily, sstables); + ColumnFamilyResult result = toColumnFamilyResult(keyspace, columnFamily, sstables); jsonWriter.jsonValue(result.toString()); return result; } @@ -211,11 +211,11 @@ public String getRemoteMetaFilePath() throws Exception { return abstractBackupPath.getRemotePath(); } - private ColumnfamilyResult toColumnFamilyResult( + private ColumnFamilyResult toColumnFamilyResult( String keyspace, String columnFamily, ImmutableMultimap sstables) { - ColumnfamilyResult columnfamilyResult = new ColumnfamilyResult(keyspace, columnFamily); + ColumnFamilyResult columnfamilyResult = new ColumnFamilyResult(keyspace, columnFamily); sstables.keySet() .stream() .map(k -> toSSTableResult(k, sstables.get(k))) @@ -223,9 +223,9 @@ private ColumnfamilyResult toColumnFamilyResult( return columnfamilyResult; } - private ColumnfamilyResult.SSTableResult toSSTableResult( + private ColumnFamilyResult.SSTableResult toSSTableResult( String prefix, ImmutableCollection sstable) { - ColumnfamilyResult.SSTableResult ssTableResult = new ColumnfamilyResult.SSTableResult(); + ColumnFamilyResult.SSTableResult ssTableResult = new ColumnFamilyResult.SSTableResult(); ssTableResult.setPrefix(prefix); ssTableResult.setSstableComponents( ImmutableSet.copyOf( diff --git a/priam/src/main/java/com/netflix/priam/backupv2/MetaV2Proxy.java b/priam/src/main/java/com/netflix/priam/backupv2/MetaV2Proxy.java index 223c57624..1cfa44c73 100644 --- a/priam/src/main/java/com/netflix/priam/backupv2/MetaV2Proxy.java +++ b/priam/src/main/java/com/netflix/priam/backupv2/MetaV2Proxy.java @@ -217,8 +217,8 @@ private class MetaFileBackupValidator extends MetaFileReader { private BackupVerificationResult verificationResult = new BackupVerificationResult(); @Override - public void process(ColumnfamilyResult columnfamilyResult) { - for (ColumnfamilyResult.SSTableResult ssTableResult : + public void process(ColumnFamilyResult columnfamilyResult) { + for (ColumnFamilyResult.SSTableResult ssTableResult : columnfamilyResult.getSstables()) { for (FileUploadResult fileUploadResult : ssTableResult.getSstableComponents()) { if (fs.checkObjectExists(Paths.get(fileUploadResult.getBackupPath()))) { @@ -235,8 +235,8 @@ private class MetaFileBackupWalker extends MetaFileReader { private List backupRemotePaths = new ArrayList<>(); @Override - public void process(ColumnfamilyResult columnfamilyResult) { - for (ColumnfamilyResult.SSTableResult ssTableResult : + public void process(ColumnFamilyResult columnfamilyResult) { + for (ColumnFamilyResult.SSTableResult ssTableResult : columnfamilyResult.getSstables()) { for (FileUploadResult fileUploadResult : ssTableResult.getSstableComponents()) { backupRemotePaths.add(fileUploadResult.getBackupPath()); diff --git a/priam/src/main/java/com/netflix/priam/backupv2/SnapshotMetaTask.java b/priam/src/main/java/com/netflix/priam/backupv2/SnapshotMetaTask.java index 10f3d4821..a1643670f 100644 --- a/priam/src/main/java/com/netflix/priam/backupv2/SnapshotMetaTask.java +++ b/priam/src/main/java/com/netflix/priam/backupv2/SnapshotMetaTask.java @@ -352,7 +352,7 @@ protected void processColumnFamily(File backupDir) throws Exception { } } - private Optional generateMetaFile( + private Optional generateMetaFile( final String keyspace, final String columnFamily, final File backupDir) throws Exception { File snapshotDir = getValidSnapshot(backupDir, snapshotName); @@ -375,13 +375,13 @@ private Optional generateMetaFile( ImmutableSetMultimap sstables = builder.build(); logger.debug("Processing {} sstables from {}.{}", keyspace, columnFamily, sstables.size()); - ColumnfamilyResult result = + ColumnFamilyResult result = dataStep.addColumnfamilyResult(keyspace, columnFamily, sstables); logger.debug("Finished processing KS: {}, CF: {}", keyspace, columnFamily); return Optional.of(result); } - private void deleteUploadedFiles(ColumnfamilyResult result) { + private void deleteUploadedFiles(ColumnFamilyResult result) { result.getSstables() .stream() .flatMap(sstable -> sstable.getSstableComponents().stream()) diff --git a/priam/src/test/java/com/netflix/priam/backupv2/TestSnapshotMetaTask.java b/priam/src/test/java/com/netflix/priam/backupv2/TestSnapshotMetaTask.java index 913b09364..781282c79 100644 --- a/priam/src/test/java/com/netflix/priam/backupv2/TestSnapshotMetaTask.java +++ b/priam/src/test/java/com/netflix/priam/backupv2/TestSnapshotMetaTask.java @@ -132,7 +132,7 @@ void setNoOfSstables(int noOfSstables) { } @Override - public void process(ColumnfamilyResult columnfamilyResult) { + public void process(ColumnFamilyResult columnfamilyResult) { Assert.assertEquals(noOfSstables, columnfamilyResult.getSstables().size()); } } From 57b26a4f42dc5457ade7b0ebf68b8fcada2f1743 Mon Sep 17 00:00:00 2001 From: mattl-netflix <63665634+mattl-netflix@users.noreply.github.com> Date: Sun, 26 Jun 2022 08:33:32 -0700 Subject: [PATCH 205/228] CASS-2805 Add hook to optionally skip deletion for files added within a certain window. (#992) --- .../com/netflix/priam/backupv2/BackupTTLTask.java | 9 +++++++-- .../com/netflix/priam/backupv2/DeletionFilter.java | 12 ++++++++++++ .../netflix/priam/backupv2/NoOpDeletionFilter.java | 4 ++++ 3 files changed, 23 insertions(+), 2 deletions(-) create mode 100644 priam/src/main/java/com/netflix/priam/backupv2/DeletionFilter.java create mode 100644 priam/src/main/java/com/netflix/priam/backupv2/NoOpDeletionFilter.java diff --git a/priam/src/main/java/com/netflix/priam/backupv2/BackupTTLTask.java b/priam/src/main/java/com/netflix/priam/backupv2/BackupTTLTask.java index 71c43b839..ebb1fea45 100644 --- a/priam/src/main/java/com/netflix/priam/backupv2/BackupTTLTask.java +++ b/priam/src/main/java/com/netflix/priam/backupv2/BackupTTLTask.java @@ -69,6 +69,7 @@ public class BackupTTLTask extends Task { private static final Lock lock = new ReentrantLock(); private final int BATCH_SIZE = 1000; private final Instant start_of_feature = DateUtil.parseInstant("201801010000"); + private final DeletionFilter deletionFilter; @Inject public BackupTTLTask( @@ -77,13 +78,15 @@ public BackupTTLTask( @Named("v2") IMetaProxy metaProxy, IFileSystemContext backupFileSystemCtx, Provider abstractBackupPathProvider, - InstanceState instanceState) { + InstanceState instanceState, + DeletionFilter deletionFilter) { super(configuration); this.backupRestoreConfig = backupRestoreConfig; this.metaProxy = metaProxy; this.fileSystem = backupFileSystemCtx.getFileStrategy(configuration); this.abstractBackupPathProvider = abstractBackupPathProvider; this.instanceState = instanceState; + this.deletionFilter = deletionFilter; } @Override @@ -189,7 +192,9 @@ Since this is not the case, the TTL may end up deleting this file even though th } if (!filesInMeta.containsKey(abstractBackupPath.getRemotePath())) { - deleteFile(abstractBackupPath, false); + if (deletionFilter.shouldDelete(abstractBackupPath.getLastModified())) { + deleteFile(abstractBackupPath, false); + } } else { if (logger.isDebugEnabled()) logger.debug( diff --git a/priam/src/main/java/com/netflix/priam/backupv2/DeletionFilter.java b/priam/src/main/java/com/netflix/priam/backupv2/DeletionFilter.java new file mode 100644 index 000000000..6772a94db --- /dev/null +++ b/priam/src/main/java/com/netflix/priam/backupv2/DeletionFilter.java @@ -0,0 +1,12 @@ +package com.netflix.priam.backupv2; + +import com.google.inject.ImplementedBy; +import java.time.Instant; + +/** Hook to allow forks to optionally omit files from deletion. */ +@ImplementedBy(NoOpDeletionFilter.class) +public interface DeletionFilter { + default boolean shouldDelete(Instant lastModified) { + return true; + } +} diff --git a/priam/src/main/java/com/netflix/priam/backupv2/NoOpDeletionFilter.java b/priam/src/main/java/com/netflix/priam/backupv2/NoOpDeletionFilter.java new file mode 100644 index 000000000..25c015dcf --- /dev/null +++ b/priam/src/main/java/com/netflix/priam/backupv2/NoOpDeletionFilter.java @@ -0,0 +1,4 @@ +package com.netflix.priam.backupv2; + +/** Default implementation of DeletionFilter that does nothing. */ +public class NoOpDeletionFilter implements DeletionFilter {} From 270524bb7fb53ed5680ba68ad7612d59465f6bab Mon Sep 17 00:00:00 2001 From: Matt Lehman Date: Sun, 26 Jun 2022 08:35:35 -0700 Subject: [PATCH 206/228] Update CHANGELOG in advance of 3.11.90 --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index d0bbab97a..b9af426dd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,8 @@ # Changelog +## 2022/06/26 3.11.90 +*UpdateSecuritySettings will be removed in the next release* +(#992) Add hook to optionally skip deletion for files added within a certain window. + ## 2022/06/02 3.11.89 *UpdateSecuritySettings will be removed in the next release* (#975) Dynamic Rate Limiting of Snapshots. From 6e5f7290cda7a7e6e06228c3e682112454106885 Mon Sep 17 00:00:00 2001 From: ayushis Date: Thu, 30 Jun 2022 15:46:59 -0700 Subject: [PATCH 207/228] streaming_socket_timeout_in_ms removed --- priam/src/main/java/com/netflix/priam/tuner/StandardTuner.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/priam/src/main/java/com/netflix/priam/tuner/StandardTuner.java b/priam/src/main/java/com/netflix/priam/tuner/StandardTuner.java index fa542bfb1..c1eea3459 100644 --- a/priam/src/main/java/com/netflix/priam/tuner/StandardTuner.java +++ b/priam/src/main/java/com/netflix/priam/tuner/StandardTuner.java @@ -117,7 +117,7 @@ public void writeAllProperties(String yamlLocation, String hostname, String seed map.put("tombstone_warn_threshold", config.getTombstoneWarnThreshold()); map.put("tombstone_failure_threshold", config.getTombstoneFailureThreshold()); - map.put("streaming_socket_timeout_in_ms", config.getStreamingSocketTimeoutInMS()); + // map.put("streaming_socket_timeout_in_ms", config.getStreamingSocketTimeoutInMS()); map.put("memtable_cleanup_threshold", config.getMemtableCleanupThreshold()); map.put( From a4bc0a4e3a8473089b0157147431b89f93f30496 Mon Sep 17 00:00:00 2001 From: mattl-netflix <63665634+mattl-netflix@users.noreply.github.com> Date: Wed, 6 Jul 2022 23:08:45 -0700 Subject: [PATCH 208/228] Fix racy behavior and remove redundant system call to get the time. (#995) --- .../main/java/com/netflix/priam/backupv2/SnapshotMetaTask.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/priam/src/main/java/com/netflix/priam/backupv2/SnapshotMetaTask.java b/priam/src/main/java/com/netflix/priam/backupv2/SnapshotMetaTask.java index a1643670f..10a7fb712 100644 --- a/priam/src/main/java/com/netflix/priam/backupv2/SnapshotMetaTask.java +++ b/priam/src/main/java/com/netflix/priam/backupv2/SnapshotMetaTask.java @@ -317,7 +317,7 @@ private Instant getUploadTarget() { CronExpression snapshotCron = new CronExpression(backupRestoreConfig.getSnapshotMetaServiceCronExpression()); snapshotCron.setTimeZone(UTC); - Date nextSnapshotDate = snapshotCron.getNextValidTimeAfter(Date.from(Instant.now())); + Date nextSnapshotDate = snapshotCron.getNextValidTimeAfter(Date.from(now)); nextSnapshotTime = nextSnapshotDate == null ? Instant.MAX : nextSnapshotDate.toInstant(); } catch (ParseException e) { From 550c1de5d1209ce5a60b0513961ef05980cbf226 Mon Sep 17 00:00:00 2001 From: mattl-netflix <63665634+mattl-netflix@users.noreply.github.com> Date: Sun, 28 Aug 2022 21:45:17 -0700 Subject: [PATCH 209/228] Revert "CASS-2805 Add hook to optionally skip deletion for files added within a certain window. (#992)" (#997) This reverts commit 57b26a4f42dc5457ade7b0ebf68b8fcada2f1743. --- .../com/netflix/priam/backupv2/BackupTTLTask.java | 9 ++------- .../com/netflix/priam/backupv2/DeletionFilter.java | 12 ------------ .../netflix/priam/backupv2/NoOpDeletionFilter.java | 4 ---- 3 files changed, 2 insertions(+), 23 deletions(-) delete mode 100644 priam/src/main/java/com/netflix/priam/backupv2/DeletionFilter.java delete mode 100644 priam/src/main/java/com/netflix/priam/backupv2/NoOpDeletionFilter.java diff --git a/priam/src/main/java/com/netflix/priam/backupv2/BackupTTLTask.java b/priam/src/main/java/com/netflix/priam/backupv2/BackupTTLTask.java index ebb1fea45..71c43b839 100644 --- a/priam/src/main/java/com/netflix/priam/backupv2/BackupTTLTask.java +++ b/priam/src/main/java/com/netflix/priam/backupv2/BackupTTLTask.java @@ -69,7 +69,6 @@ public class BackupTTLTask extends Task { private static final Lock lock = new ReentrantLock(); private final int BATCH_SIZE = 1000; private final Instant start_of_feature = DateUtil.parseInstant("201801010000"); - private final DeletionFilter deletionFilter; @Inject public BackupTTLTask( @@ -78,15 +77,13 @@ public BackupTTLTask( @Named("v2") IMetaProxy metaProxy, IFileSystemContext backupFileSystemCtx, Provider abstractBackupPathProvider, - InstanceState instanceState, - DeletionFilter deletionFilter) { + InstanceState instanceState) { super(configuration); this.backupRestoreConfig = backupRestoreConfig; this.metaProxy = metaProxy; this.fileSystem = backupFileSystemCtx.getFileStrategy(configuration); this.abstractBackupPathProvider = abstractBackupPathProvider; this.instanceState = instanceState; - this.deletionFilter = deletionFilter; } @Override @@ -192,9 +189,7 @@ Since this is not the case, the TTL may end up deleting this file even though th } if (!filesInMeta.containsKey(abstractBackupPath.getRemotePath())) { - if (deletionFilter.shouldDelete(abstractBackupPath.getLastModified())) { - deleteFile(abstractBackupPath, false); - } + deleteFile(abstractBackupPath, false); } else { if (logger.isDebugEnabled()) logger.debug( diff --git a/priam/src/main/java/com/netflix/priam/backupv2/DeletionFilter.java b/priam/src/main/java/com/netflix/priam/backupv2/DeletionFilter.java deleted file mode 100644 index 6772a94db..000000000 --- a/priam/src/main/java/com/netflix/priam/backupv2/DeletionFilter.java +++ /dev/null @@ -1,12 +0,0 @@ -package com.netflix.priam.backupv2; - -import com.google.inject.ImplementedBy; -import java.time.Instant; - -/** Hook to allow forks to optionally omit files from deletion. */ -@ImplementedBy(NoOpDeletionFilter.class) -public interface DeletionFilter { - default boolean shouldDelete(Instant lastModified) { - return true; - } -} diff --git a/priam/src/main/java/com/netflix/priam/backupv2/NoOpDeletionFilter.java b/priam/src/main/java/com/netflix/priam/backupv2/NoOpDeletionFilter.java deleted file mode 100644 index 25c015dcf..000000000 --- a/priam/src/main/java/com/netflix/priam/backupv2/NoOpDeletionFilter.java +++ /dev/null @@ -1,4 +0,0 @@ -package com.netflix.priam.backupv2; - -/** Default implementation of DeletionFilter that does nothing. */ -public class NoOpDeletionFilter implements DeletionFilter {} From c690a671c75f946c781f581df86f9f7e42cfef2f Mon Sep 17 00:00:00 2001 From: mattl-netflix <63665634+mattl-netflix@users.noreply.github.com> Date: Sun, 28 Aug 2022 22:24:55 -0700 Subject: [PATCH 210/228] Remove UpdateSecuritySettings (#966) --- .../java/com/netflix/priam/PriamServer.java | 16 --- .../com/netflix/priam/aws/AWSIPConverter.java | 59 -------- .../com/netflix/priam/aws/IPConverter.java | 11 -- .../priam/aws/UpdateSecuritySettings.java | 136 ------------------ .../java/com/netflix/priam/TestModule.java | 3 - .../netflix/priam/aws/FakeIPConverter.java | 19 --- .../priam/aws/TestUpdateSecuritySettings.java | 112 --------------- 7 files changed, 356 deletions(-) delete mode 100644 priam/src/main/java/com/netflix/priam/aws/AWSIPConverter.java delete mode 100644 priam/src/main/java/com/netflix/priam/aws/IPConverter.java delete mode 100644 priam/src/main/java/com/netflix/priam/aws/UpdateSecuritySettings.java delete mode 100644 priam/src/test/java/com/netflix/priam/aws/FakeIPConverter.java delete mode 100644 priam/src/test/java/com/netflix/priam/aws/TestUpdateSecuritySettings.java diff --git a/priam/src/main/java/com/netflix/priam/PriamServer.java b/priam/src/main/java/com/netflix/priam/PriamServer.java index 91cf66ea7..36c8f5103 100644 --- a/priam/src/main/java/com/netflix/priam/PriamServer.java +++ b/priam/src/main/java/com/netflix/priam/PriamServer.java @@ -18,7 +18,6 @@ import com.google.inject.Inject; import com.google.inject.Singleton; -import com.netflix.priam.aws.UpdateSecuritySettings; import com.netflix.priam.backup.BackupService; import com.netflix.priam.backupv2.BackupV2Service; import com.netflix.priam.cluster.management.ClusterManagementService; @@ -97,21 +96,6 @@ public void scheduleService() throws Exception { // start to schedule jobs scheduler.start(); - // update security settings. - if (config.isMultiDC()) { - scheduler.runTaskNow(UpdateSecuritySettings.class); - // sleep for 150 sec if this is a new node with new IP for SG to be updated by other - // seed nodes - if (instanceIdentity.isReplace() || instanceIdentity.isTokenPregenerated()) - sleeper.sleep(150 * 1000); - else if (UpdateSecuritySettings.firstTimeUpdated) sleeper.sleep(60 * 1000); - - scheduler.addTask( - UpdateSecuritySettings.JOBNAME, - UpdateSecuritySettings.class, - UpdateSecuritySettings.getTimer(instanceIdentity)); - } - // Set up cassandra tuning. cassandraTunerService.scheduleService(); diff --git a/priam/src/main/java/com/netflix/priam/aws/AWSIPConverter.java b/priam/src/main/java/com/netflix/priam/aws/AWSIPConverter.java deleted file mode 100644 index eb0f60ca9..000000000 --- a/priam/src/main/java/com/netflix/priam/aws/AWSIPConverter.java +++ /dev/null @@ -1,59 +0,0 @@ -package com.netflix.priam.aws; - -import com.google.common.base.Splitter; -import com.netflix.priam.identity.PriamInstance; -import com.netflix.priam.identity.config.InstanceInfo; -import java.net.InetAddress; -import java.net.UnknownHostException; -import java.util.List; -import java.util.Optional; -import java.util.regex.Pattern; -import javax.inject.Inject; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** Derives public ips from PriamInstances with AWS hostnames. */ -public class AWSIPConverter implements IPConverter { - private static final Logger logger = LoggerFactory.getLogger(AWSIPConverter.class); - private static final Pattern IP_PART = Pattern.compile("^[0-9]{1,3}$"); - - private final InstanceInfo myInstance; - - @Inject - public AWSIPConverter(InstanceInfo myInstance) { - this.myInstance = myInstance; - } - - @Override - public Optional getPublicIP(PriamInstance instance) { - if (instance.getInstanceId().equals(myInstance.getInstanceId())) { - return Optional.ofNullable(myInstance.getHostIP()); - } - Optional ip = extractIPFromHostnameString(instance.getHostName()); - return !instance.getDC().equals(myInstance.getRegion()) && !ip.isPresent() - ? deriveIPFromHostname(instance.getHostName()) - : ip; - } - - private Optional extractIPFromHostnameString(String hostname) { - if (hostname.contains(".")) { - String ip = hostname.substring(4, hostname.indexOf('.')).replace('-', '.'); - List parts = Splitter.on(".").splitToList(ip); - return parts.size() == 4 && parts.stream().allMatch(p -> IP_PART.matcher(p).matches()) - ? Optional.of(ip) - : Optional.empty(); - } - return Optional.empty(); - } - - private Optional deriveIPFromHostname(String hostname) { - String ip = null; - try { - ip = InetAddress.getByName(hostname).getHostAddress(); - logger.info("Derived IP for " + hostname + ": " + ip); - } catch (UnknownHostException e) { - logger.warn("Could not resolve [" + hostname + "]"); - } - return Optional.ofNullable(ip); - } -} diff --git a/priam/src/main/java/com/netflix/priam/aws/IPConverter.java b/priam/src/main/java/com/netflix/priam/aws/IPConverter.java deleted file mode 100644 index fde04ac83..000000000 --- a/priam/src/main/java/com/netflix/priam/aws/IPConverter.java +++ /dev/null @@ -1,11 +0,0 @@ -package com.netflix.priam.aws; - -import com.google.inject.ImplementedBy; -import com.netflix.priam.identity.PriamInstance; -import java.util.Optional; - -/** Derives public ip from hostname. */ -@ImplementedBy(AWSIPConverter.class) -public interface IPConverter { - Optional getPublicIP(PriamInstance instance); -} diff --git a/priam/src/main/java/com/netflix/priam/aws/UpdateSecuritySettings.java b/priam/src/main/java/com/netflix/priam/aws/UpdateSecuritySettings.java deleted file mode 100644 index 1b6c6ddd3..000000000 --- a/priam/src/main/java/com/netflix/priam/aws/UpdateSecuritySettings.java +++ /dev/null @@ -1,136 +0,0 @@ -/* - * Copyright 2013 Netflix, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ -package com.netflix.priam.aws; - -import com.google.common.base.Joiner; -import com.google.common.collect.ImmutableSet; -import com.google.common.collect.Sets; -import com.google.inject.Inject; -import com.google.inject.Singleton; -import com.netflix.priam.config.IConfiguration; -import com.netflix.priam.identity.IMembership; -import com.netflix.priam.identity.IPriamInstanceFactory; -import com.netflix.priam.identity.InstanceIdentity; -import com.netflix.priam.identity.PriamInstance; -import com.netflix.priam.merics.SecurityMetrics; -import com.netflix.priam.scheduler.SimpleTimer; -import com.netflix.priam.scheduler.Task; -import com.netflix.priam.scheduler.TaskTimer; -import java.util.Objects; -import java.util.Optional; -import java.util.Random; -import java.util.Set; -import java.util.stream.Collectors; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * this class will associate an Public IP's with a new instance so they can talk across the regions. - * - *

    Requirement: 1) Nodes in the same region needs to be able to talk to each other. 2) Nodes in - * other regions needs to be able to talk to t`he others in the other region. - * - *

    Assumption: 1) IPriamInstanceFactory will provide the membership... and will be visible across - * the regions 2) IMembership amazon or any other implementation which can tell if the instance is - * part of the group (ASG in amazons case). - */ -@Singleton -public class UpdateSecuritySettings extends Task { - private static final Logger logger = LoggerFactory.getLogger(UpdateSecuritySettings.class); - public static final String JOBNAME = "Update_SG"; - public static boolean firstTimeUpdated = false; - - private static final Random ran = new Random(); - private final IMembership membership; - private final IPriamInstanceFactory factory; - private final IPConverter ipConverter; - private final SecurityMetrics securityMetrics; - - @Inject - // Note: do not parameterized the generic type variable to an implementation as it confuses - // Guice in the binding. - public UpdateSecuritySettings( - IConfiguration config, - IMembership membership, - IPriamInstanceFactory factory, - IPConverter ipConverter, - SecurityMetrics securityMetrics) { - super(config); - this.membership = membership; - this.factory = factory; - this.ipConverter = ipConverter; - this.securityMetrics = securityMetrics; - } - - /** - * Seeds nodes execute this at the specifed interval. Other nodes run only on startup. Seeds in - * cassandra are the first node in each Availablity Zone. - */ - @Override - public void execute() { - int port = config.getSSLStoragePort(); - ImmutableSet currentAcl = membership.listACL(port, port); - securityMetrics.setIngressRules(currentAcl.size()); - Set desiredAcl = - factory.getAllIds(config.getAppName()) - .stream() - .map(i -> getIngressRule(i).orElse(null)) - .filter(Objects::nonNull) - .map(ip -> ip + "/32") - .collect(Collectors.toSet()); - if (!config.skipDeletingOthersIngressRules()) { - Set aclToRemove = Sets.difference(currentAcl, desiredAcl); - logger.info("ingress rules to delete: {}", Joiner.on(",").join(aclToRemove)); - if (!aclToRemove.isEmpty()) { - membership.removeACL(aclToRemove, port, port); - firstTimeUpdated = true; - } - } - if (!config.skipUpdatingOthersIngressRules()) { - Set aclToAdd = Sets.difference(desiredAcl, currentAcl); - logger.info("ingress rules to update: {}", Joiner.on(",").join(aclToAdd)); - if (!aclToAdd.isEmpty()) { - membership.addACL(aclToAdd, port, port); - firstTimeUpdated = true; - } - } - } - - private Optional getIngressRule(PriamInstance instance) { - return config.skipIngressUnlessIPIsPublic() - ? ipConverter.getPublicIP(instance) - : Optional.of(instance.getHostIP()); - } - - public static TaskTimer getTimer(InstanceIdentity id) { - SimpleTimer return_; - if (id.isSeed()) { - logger.info( - "Seed node. Instance id: {}" + ", host ip: {}" + ", host name: {}", - id.getInstance().getInstanceId(), - id.getInstance().getHostIP(), - id.getInstance().getHostName()); - return_ = new SimpleTimer(JOBNAME, 120 * 1000 + ran.nextInt(120 * 1000)); - } else return_ = new SimpleTimer(JOBNAME); - return return_; - } - - @Override - public String getName() { - return JOBNAME; - } -} diff --git a/priam/src/test/java/com/netflix/priam/TestModule.java b/priam/src/test/java/com/netflix/priam/TestModule.java index bacb0800b..0f4ceffbe 100644 --- a/priam/src/test/java/com/netflix/priam/TestModule.java +++ b/priam/src/test/java/com/netflix/priam/TestModule.java @@ -20,8 +20,6 @@ import com.google.common.collect.ImmutableList; import com.google.inject.AbstractModule; import com.google.inject.Scopes; -import com.netflix.priam.aws.FakeIPConverter; -import com.netflix.priam.aws.IPConverter; import com.netflix.priam.backup.FakeCredentials; import com.netflix.priam.backup.IBackupFileSystem; import com.netflix.priam.backup.NullBackupFileSystem; @@ -65,6 +63,5 @@ protected void configure() { bind(IBackupFileSystem.class).to(NullBackupFileSystem.class); bind(Sleeper.class).to(FakeSleeper.class); bind(Registry.class).toInstance(new DefaultRegistry()); - bind(IPConverter.class).toInstance(new FakeIPConverter()); } } diff --git a/priam/src/test/java/com/netflix/priam/aws/FakeIPConverter.java b/priam/src/test/java/com/netflix/priam/aws/FakeIPConverter.java deleted file mode 100644 index 77d41cee1..000000000 --- a/priam/src/test/java/com/netflix/priam/aws/FakeIPConverter.java +++ /dev/null @@ -1,19 +0,0 @@ -package com.netflix.priam.aws; - -import com.netflix.priam.identity.PriamInstance; -import groovy.lang.Singleton; -import java.util.Optional; - -@Singleton -public class FakeIPConverter implements IPConverter { - private boolean shouldFail; - - public void setShouldFail(boolean shouldFail) { - this.shouldFail = shouldFail; - } - - @Override - public Optional getPublicIP(PriamInstance instance) { - return shouldFail ? Optional.empty() : Optional.of("1.1.1.1"); - } -} diff --git a/priam/src/test/java/com/netflix/priam/aws/TestUpdateSecuritySettings.java b/priam/src/test/java/com/netflix/priam/aws/TestUpdateSecuritySettings.java deleted file mode 100644 index 2a8cd3b3e..000000000 --- a/priam/src/test/java/com/netflix/priam/aws/TestUpdateSecuritySettings.java +++ /dev/null @@ -1,112 +0,0 @@ -package com.netflix.priam.aws; - -import com.google.common.collect.ImmutableSet; -import com.google.common.truth.Truth; -import com.google.inject.Guice; -import com.google.inject.Injector; -import com.netflix.priam.TestModule; -import com.netflix.priam.config.FakeConfiguration; -import com.netflix.priam.config.IConfiguration; -import com.netflix.priam.identity.IMembership; -import com.netflix.priam.identity.IPriamInstanceFactory; -import org.junit.Before; -import org.junit.Test; - -/* Tests of {@link UpdateSecuritySettings.java} */ -public class TestUpdateSecuritySettings { - private static final int PORT = 7103; - - private UpdateSecuritySettings updateSecuritySettings; - private IMembership membership; - private IPriamInstanceFactory factory; - private FakeConfiguration config; - private FakeIPConverter ipConverter; - - @Before - public void setUp() { - Injector injector = Guice.createInjector(new TestModule()); - factory = injector.getInstance(IPriamInstanceFactory.class); - membership = injector.getInstance(IMembership.class); - updateSecuritySettings = injector.getInstance(UpdateSecuritySettings.class); - config = (FakeConfiguration) injector.getInstance(IConfiguration.class); - ipConverter = (FakeIPConverter) injector.getInstance(IPConverter.class); - } - - @Test - public void add_membershipEmpty() { // edge-case, not expected - addToFactory(1, "1.1.1.1"); - Truth.assertThat(membership.listACL(PORT, PORT)).doesNotContain("1.1.1.1/32"); - updateSecuritySettings.execute(); - Truth.assertThat(membership.listACL(PORT, PORT)).contains("1.1.1.1/32"); - } - - @Test - public void add() { - addToFactory(1, "1.1.1.1"); - membership.addACL(ImmutableSet.of("2.2.2.2/32"), PORT, PORT); - Truth.assertThat(membership.listACL(PORT, PORT)).doesNotContain("1.1.1.1/32"); - updateSecuritySettings.execute(); - Truth.assertThat(membership.listACL(PORT, PORT)).contains("1.1.1.1/32"); - } - - @Test - public void delete() { - addToFactory(1, "1.1.1.1"); - membership.addACL(ImmutableSet.of("2.2.2.2/32"), PORT, PORT); - Truth.assertThat(membership.listACL(PORT, PORT)).contains("2.2.2.2/32"); - updateSecuritySettings.execute(); - Truth.assertThat(membership.listACL(PORT, PORT)).doesNotContain("2.2.2.2/32"); - } - - @Test - public void delete_factoryEmpty() { // edge-case, not expected - membership.addACL(ImmutableSet.of("2.2.2.2/32"), PORT, PORT); - Truth.assertThat(membership.listACL(PORT, PORT)).contains("2.2.2.2/32"); - updateSecuritySettings.execute(); - Truth.assertThat(membership.listACL(PORT, PORT)).doesNotContain("2.2.2.2/32"); - } - - @Test - public void dontDeleteOthersIngress() { - config.setSkipDeletingOthersIngressRules(true); - membership.addACL(ImmutableSet.of("1.1.1.1/32"), PORT, PORT); - Truth.assertThat(membership.listACL(PORT, PORT)).contains("1.1.1.1/32"); - updateSecuritySettings.execute(); - Truth.assertThat(membership.listACL(PORT, PORT)).contains("1.1.1.1/32"); - } - - @Test - public void dontUpdateOthersIngress() { - addToFactory(1, "1.1.1.1"); - config.setSkipUpdatingOthersIngressRules(true); - Truth.assertThat(membership.listACL(PORT, PORT)).isEmpty(); - updateSecuritySettings.execute(); - Truth.assertThat(membership.listACL(PORT, PORT)).isEmpty(); - } - - @Test - public void dontUpdatePrivateIngress() { - addToFactory(1, "2.2.2.2"); - ipConverter.setShouldFail(false); - config.setSkipIngressUnlessIPIsPublic(true); - Truth.assertThat(membership.listACL(PORT, PORT)).isEmpty(); - updateSecuritySettings.execute(); - // The private IP produced by our FakeIPConverter - Truth.assertThat(membership.listACL(PORT, PORT)).contains("1.1.1.1/32"); - Truth.assertThat(membership.listACL(PORT, PORT)).doesNotContain("2.2.2.2/32"); - } - - @Test - public void dontUpdateUnknownIngress() { - addToFactory(1, "1.1.1.1"); - ipConverter.setShouldFail(true); - config.setSkipIngressUnlessIPIsPublic(true); - Truth.assertThat(membership.listACL(PORT, PORT)).isEmpty(); - updateSecuritySettings.execute(); - Truth.assertThat(membership.listACL(PORT, PORT)).isEmpty(); - } - - private void addToFactory(int id, String ip) { - factory.create("myApp", id, "i-1", "hostname", ip, "us-east-1a", null /* volumes */, "123"); - } -} From 1b76cc0d0907c930c03a95250982242c9d2b6b09 Mon Sep 17 00:00:00 2001 From: mattl-netflix <63665634+mattl-netflix@users.noreply.github.com> Date: Wed, 31 Aug 2022 12:43:58 -0700 Subject: [PATCH 211/228] Use IP to get gossip info rather than hostname. (#1001) --- .../identity/token/TokenRetrieverUtils.java | 14 ++++++-------- .../identity/token/TokenRetrieverUtilsTest.java | 16 ++++++++-------- 2 files changed, 14 insertions(+), 16 deletions(-) diff --git a/priam/src/main/java/com/netflix/priam/identity/token/TokenRetrieverUtils.java b/priam/src/main/java/com/netflix/priam/identity/token/TokenRetrieverUtils.java index 324388ca4..5827e01ae 100644 --- a/priam/src/main/java/com/netflix/priam/identity/token/TokenRetrieverUtils.java +++ b/priam/src/main/java/com/netflix/priam/identity/token/TokenRetrieverUtils.java @@ -60,12 +60,11 @@ public static InferredTokenOwnership inferTokenOwnerFromGossip( InferredTokenOwnership inferredTokenOwnership = new InferredTokenOwnership(); int matchedGossipInstances = 0, reachableInstances = 0; for (PriamInstance instance : eligibleInstances) { - logger.info( - "Calling getIp on hostname[{}] and token[{}]", instance.getHostName(), token); + logger.info("Finding down nodes from ip[{}]; token[{}]", instance.getHostIP(), token); try { TokenInformation tokenInformation = - getTokenInformation(instance.getHostName(), token); + getTokenInformation(instance.getHostIP(), token); reachableInstances++; if (inferredTokenOwnership.getTokenInformation() == null) { @@ -113,11 +112,11 @@ public static InferredTokenOwnership inferTokenOwnerFromGossip( } // helper method to get the token owner IP from a Cassandra node. - private static TokenInformation getTokenInformation(String host, String token) + private static TokenInformation getTokenInformation(String ip, String token) throws GossipParseException { String response = null; try { - response = SystemUtils.getDataFromUrl(String.format(STATUS_URL_FORMAT, host)); + response = SystemUtils.getDataFromUrl(String.format(STATUS_URL_FORMAT, ip)); JSONObject jsonObject = (JSONObject) new JSONParser().parse(response); JSONArray liveNodes = (JSONArray) jsonObject.get("live"); JSONObject tokenToEndpointMap = (JSONObject) jsonObject.get("tokenToEndpointMap"); @@ -129,12 +128,11 @@ private static TokenInformation getTokenInformation(String host, String token) return new TokenInformation(endpointInfo, isLive); } catch (RuntimeException e) { throw new GossipParseException( - String.format("Error in reaching out to host: [%s]", host), e); + String.format("Error in reaching out to host: [%s]", ip), e); } catch (ParseException e) { throw new GossipParseException( String.format( - "Error in parsing gossip response [%s] from host: [%s]", - response, host), + "Error in parsing gossip response [%s] from host: [%s]", response, ip), e); } } diff --git a/priam/src/test/java/com/netflix/priam/identity/token/TokenRetrieverUtilsTest.java b/priam/src/test/java/com/netflix/priam/identity/token/TokenRetrieverUtilsTest.java index 54a5f4d9f..dbb00003f 100644 --- a/priam/src/test/java/com/netflix/priam/identity/token/TokenRetrieverUtilsTest.java +++ b/priam/src/test/java/com/netflix/priam/identity/token/TokenRetrieverUtilsTest.java @@ -85,21 +85,21 @@ public void testRetrieveTokenOwnerWhenGossipDisagrees(@Mocked SystemUtils system SystemUtils.getDataFromUrl( withArgThat( allOf( - not(String.format(STATUS_URL_FORMAT, "fakeHost-0")), - not(String.format(STATUS_URL_FORMAT, "fakeHost-2")), - not(String.format(STATUS_URL_FORMAT, "fakeHost-5"))))); + not(String.format(STATUS_URL_FORMAT, "127.0.0.0")), + not(String.format(STATUS_URL_FORMAT, "127.0.0.2")), + not(String.format(STATUS_URL_FORMAT, "127.0.0.5"))))); result = getStatus(myliveInstances, tokenToEndpointMap); minTimes = 0; - SystemUtils.getDataFromUrl(String.format(STATUS_URL_FORMAT, "fakeHost-0")); + SystemUtils.getDataFromUrl(String.format(STATUS_URL_FORMAT, "127.0.0.0")); result = getStatus(liveInstances, tokenToEndpointMap); minTimes = 0; - SystemUtils.getDataFromUrl(String.format(STATUS_URL_FORMAT, "fakeHost-2")); + SystemUtils.getDataFromUrl(String.format(STATUS_URL_FORMAT, "127.0.0.2")); result = getStatus(liveInstances, tokenToEndpointMap); minTimes = 0; - SystemUtils.getDataFromUrl(String.format(STATUS_URL_FORMAT, "fakeHost-5")); + SystemUtils.getDataFromUrl(String.format(STATUS_URL_FORMAT, "127.0.0.5")); result = null; minTimes = 0; } @@ -129,11 +129,11 @@ public void testRetrieveTokenOwnerWhenGossipDisagrees_2Nodes(@Mocked SystemUtils new Expectations() { { - SystemUtils.getDataFromUrl(String.format(STATUS_URL_FORMAT, "fakeHost-0")); + SystemUtils.getDataFromUrl(String.format(STATUS_URL_FORMAT, "127.0.0.0")); result = getStatus(myLiveInstances, myTokenToEndpointMap); minTimes = 0; - SystemUtils.getDataFromUrl(String.format(STATUS_URL_FORMAT, "fakeHost-2")); + SystemUtils.getDataFromUrl(String.format(STATUS_URL_FORMAT, "127.0.0.2")); result = getStatus(myLiveInstances, alteredMap); minTimes = 0; } From 8135e4fd8c8835e64eec4a346fc17be4958533ea Mon Sep 17 00:00:00 2001 From: mattl-netflix <63665634+mattl-netflix@users.noreply.github.com> Date: Wed, 14 Sep 2022 18:59:34 -0700 Subject: [PATCH 212/228] Spread deletes over the interval depending on where the node is in the ring. (#1002) --- .../netflix/priam/backupv2/BackupTTLTask.java | 27 ++++++--- .../priam/backupv2/BackupV2Service.java | 14 +++-- .../priam/identity/token/ITokenRetriever.java | 7 ++- .../priam/identity/token/TokenRetriever.java | 30 +++++++--- .../netflix/priam/scheduler/SimpleTimer.java | 22 +++++-- .../priam/backupv2/TestBackupV2Service.java | 15 +++-- .../priam/config/FakeBackupRestoreConfig.java | 5 ++ .../identity/token/TokenRetrieverTest.java | 50 +++++++++++++--- .../priam/scheduler/TestSimpleTimer.java | 59 +++++++++++++++++++ 9 files changed, 189 insertions(+), 40 deletions(-) create mode 100644 priam/src/test/java/com/netflix/priam/scheduler/TestSimpleTimer.java diff --git a/priam/src/main/java/com/netflix/priam/backupv2/BackupTTLTask.java b/priam/src/main/java/com/netflix/priam/backupv2/BackupTTLTask.java index 71c43b839..c67223bc2 100644 --- a/priam/src/main/java/com/netflix/priam/backupv2/BackupTTLTask.java +++ b/priam/src/main/java/com/netflix/priam/backupv2/BackupTTLTask.java @@ -20,14 +20,11 @@ import com.google.inject.Inject; import com.google.inject.Provider; import com.google.inject.Singleton; -import com.netflix.priam.backup.AbstractBackupPath; -import com.netflix.priam.backup.BackupRestoreException; -import com.netflix.priam.backup.IBackupFileSystem; -import com.netflix.priam.backup.IFileSystemContext; -import com.netflix.priam.backup.Status; +import com.netflix.priam.backup.*; import com.netflix.priam.config.IBackupRestoreConfig; import com.netflix.priam.config.IConfiguration; import com.netflix.priam.health.InstanceState; +import com.netflix.priam.identity.token.TokenRetriever; import com.netflix.priam.scheduler.SimpleTimer; import com.netflix.priam.scheduler.Task; import com.netflix.priam.scheduler.TaskTimer; @@ -41,6 +38,7 @@ import java.util.concurrent.locks.ReentrantLock; import javax.inject.Named; import org.apache.commons.io.FileUtils; +import org.apache.commons.lang3.math.Fraction; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -69,6 +67,7 @@ public class BackupTTLTask extends Task { private static final Lock lock = new ReentrantLock(); private final int BATCH_SIZE = 1000; private final Instant start_of_feature = DateUtil.parseInstant("201801010000"); + private final int maxWaitSeconds; @Inject public BackupTTLTask( @@ -77,13 +76,18 @@ public BackupTTLTask( @Named("v2") IMetaProxy metaProxy, IFileSystemContext backupFileSystemCtx, Provider abstractBackupPathProvider, - InstanceState instanceState) { + InstanceState instanceState, + TokenRetriever tokenRetriever) + throws Exception { super(configuration); this.backupRestoreConfig = backupRestoreConfig; this.metaProxy = metaProxy; this.fileSystem = backupFileSystemCtx.getFileStrategy(configuration); this.abstractBackupPathProvider = abstractBackupPathProvider; this.instanceState = instanceState; + this.maxWaitSeconds = + backupRestoreConfig.getBackupTTLMonitorPeriodInSec() + / tokenRetriever.getRingPosition().getDenominator(); } @Override @@ -102,6 +106,9 @@ public void execute() throws Exception { throw new Exception(JOBNAME + " already running"); } + // Sleep a random amount but not so long that it will spill into the next token's turn. + if (maxWaitSeconds > 0) Thread.sleep(new Random().nextInt(maxWaitSeconds)); + try { filesInMeta.clear(); filesToDelete.clear(); @@ -239,9 +246,11 @@ public String getName() { * @throws Exception if the configuration is not set correctly or are not valid. This is to * ensure we fail-fast. */ - public static TaskTimer getTimer(IBackupRestoreConfig backupRestoreConfig) throws Exception { - return SimpleTimer.getSimpleTimer( - JOBNAME, backupRestoreConfig.getBackupTTLMonitorPeriodInSec() * 1000); + public static TaskTimer getTimer( + IBackupRestoreConfig backupRestoreConfig, Fraction ringPosition) throws Exception { + int period = backupRestoreConfig.getBackupTTLMonitorPeriodInSec(); + Instant start = Instant.ofEpochSecond((long) (period * ringPosition.doubleValue())); + return new SimpleTimer(JOBNAME, period, start); } private class MetaFileWalker extends MetaFileReader { diff --git a/priam/src/main/java/com/netflix/priam/backupv2/BackupV2Service.java b/priam/src/main/java/com/netflix/priam/backupv2/BackupV2Service.java index 5c02b396e..7889eaea7 100644 --- a/priam/src/main/java/com/netflix/priam/backupv2/BackupV2Service.java +++ b/priam/src/main/java/com/netflix/priam/backupv2/BackupV2Service.java @@ -22,11 +22,10 @@ import com.netflix.priam.config.IBackupRestoreConfig; import com.netflix.priam.config.IConfiguration; import com.netflix.priam.defaultimpl.IService; +import com.netflix.priam.identity.token.ITokenRetriever; import com.netflix.priam.scheduler.PriamScheduler; import com.netflix.priam.scheduler.TaskTimer; import com.netflix.priam.tuner.CassandraTunerService; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; /** * Encapsulate the backup service 2.0 - Execute all the tasks required to run backup service. @@ -38,7 +37,7 @@ public class BackupV2Service implements IService { private final IBackupRestoreConfig backupRestoreConfig; private final SnapshotMetaTask snapshotMetaTask; private final CassandraTunerService cassandraTunerService; - private static final Logger logger = LoggerFactory.getLogger(BackupV2Service.class); + private final ITokenRetriever tokenRetriever; @Inject public BackupV2Service( @@ -46,12 +45,14 @@ public BackupV2Service( IBackupRestoreConfig backupRestoreConfig, PriamScheduler scheduler, SnapshotMetaTask snapshotMetaService, - CassandraTunerService cassandraTunerService) { + CassandraTunerService cassandraTunerService, + ITokenRetriever tokenRetriever) { this.configuration = configuration; this.backupRestoreConfig = backupRestoreConfig; this.scheduler = scheduler; this.snapshotMetaTask = snapshotMetaService; this.cassandraTunerService = cassandraTunerService; + this.tokenRetriever = tokenRetriever; } @Override @@ -68,8 +69,9 @@ public void scheduleService() throws Exception { snapshotMetaTask.uploadFiles(); // Schedule the TTL service - scheduleTask( - scheduler, BackupTTLTask.class, BackupTTLTask.getTimer(backupRestoreConfig)); + TaskTimer timer = + BackupTTLTask.getTimer(backupRestoreConfig, tokenRetriever.getRingPosition()); + scheduleTask(scheduler, BackupTTLTask.class, timer); // Schedule the backup verification service scheduleTask( diff --git a/priam/src/main/java/com/netflix/priam/identity/token/ITokenRetriever.java b/priam/src/main/java/com/netflix/priam/identity/token/ITokenRetriever.java index 38125bee9..b22ff5907 100644 --- a/priam/src/main/java/com/netflix/priam/identity/token/ITokenRetriever.java +++ b/priam/src/main/java/com/netflix/priam/identity/token/ITokenRetriever.java @@ -3,13 +3,18 @@ import com.google.inject.ImplementedBy; import com.netflix.priam.identity.PriamInstance; import java.util.Optional; +import org.apache.commons.lang3.math.Fraction; /** Fetches PriamInstances and other data which is convenient at the time */ @ImplementedBy(TokenRetriever.class) public interface ITokenRetriever { PriamInstance get() throws Exception; - /** Gets the IP address of the dead instance to which we will acquire its token */ + + /** Gets the IP address of the dead instance whose token we will acquire. */ Optional getReplacedIp(); boolean isTokenPregenerated(); + + /** returns the percentage of tokens in the ring which come before this node's token */ + Fraction getRingPosition() throws Exception; } diff --git a/priam/src/main/java/com/netflix/priam/identity/token/TokenRetriever.java b/priam/src/main/java/com/netflix/priam/identity/token/TokenRetriever.java index 5db933ef8..251044d2b 100644 --- a/priam/src/main/java/com/netflix/priam/identity/token/TokenRetriever.java +++ b/priam/src/main/java/com/netflix/priam/identity/token/TokenRetriever.java @@ -19,6 +19,7 @@ import java.util.Set; import java.util.stream.Collectors; import javax.inject.Inject; +import org.apache.commons.lang3.math.Fraction; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -39,6 +40,7 @@ public class TokenRetriever implements ITokenRetriever { private InstanceInfo myInstanceInfo; private boolean isTokenPregenerated = false; private String replacedIp; + private PriamInstance priamInstance; @Inject public TokenRetriever( @@ -59,15 +61,17 @@ public TokenRetriever( @Override public PriamInstance get() throws Exception { - PriamInstance myInstance = grabPreAssignedToken(); - if (myInstance == null) { - myInstance = grabExistingToken(); + if (priamInstance == null) { + priamInstance = grabPreAssignedToken(); } - if (myInstance == null) { - myInstance = grabNewToken(); + if (priamInstance == null) { + priamInstance = grabExistingToken(); } - logger.info("My instance: {}", myInstance); - return myInstance; + if (priamInstance == null) { + priamInstance = grabNewToken(); + } + logger.info("My instance: {}", priamInstance); + return priamInstance; } @Override @@ -80,6 +84,18 @@ public boolean isTokenPregenerated() { return isTokenPregenerated; } + @Override + public Fraction getRingPosition() throws Exception { + get(); + long tokenAsLong = Long.parseLong(priamInstance.getToken()); + ImmutableSet nodes = factory.getAllIds(config.getAppName()); + long ringPosition = + nodes.stream() + .filter(node -> Long.parseLong(node.getToken()) < tokenAsLong) + .count(); + return Fraction.getFraction(Math.toIntExact(ringPosition), nodes.size()); + } + private PriamInstance grabPreAssignedToken() throws Exception { return new RetryableCallable() { @Override diff --git a/priam/src/main/java/com/netflix/priam/scheduler/SimpleTimer.java b/priam/src/main/java/com/netflix/priam/scheduler/SimpleTimer.java index 8759e6ea0..4b584d05f 100644 --- a/priam/src/main/java/com/netflix/priam/scheduler/SimpleTimer.java +++ b/priam/src/main/java/com/netflix/priam/scheduler/SimpleTimer.java @@ -16,12 +16,11 @@ */ package com.netflix.priam.scheduler; +import com.google.common.base.Preconditions; import java.text.ParseException; +import java.time.Instant; import java.util.Date; -import org.quartz.Scheduler; -import org.quartz.SimpleScheduleBuilder; -import org.quartz.Trigger; -import org.quartz.TriggerBuilder; +import org.quartz.*; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -45,6 +44,21 @@ public SimpleTimer(String name, long interval) { .build(); } + /** Run forever every @period seconds starting at @start */ + public SimpleTimer(String name, int period, Instant start) { + Preconditions.checkArgument(period > 0); + Preconditions.checkArgument(start.compareTo(Instant.EPOCH) >= 0); + this.trigger = + TriggerBuilder.newTrigger() + .withIdentity(name) + .withSchedule( + CalendarIntervalScheduleBuilder.calendarIntervalSchedule() + .withMisfireHandlingInstructionFireAndProceed() + .withIntervalInSeconds(period)) + .startAt(Date.from(start)) + .build(); + } + /** Run once at given time... */ public SimpleTimer(String name, String group, long startTime) { this.trigger = diff --git a/priam/src/test/java/com/netflix/priam/backupv2/TestBackupV2Service.java b/priam/src/test/java/com/netflix/priam/backupv2/TestBackupV2Service.java index d69e779c0..064c26b8b 100644 --- a/priam/src/test/java/com/netflix/priam/backupv2/TestBackupV2Service.java +++ b/priam/src/test/java/com/netflix/priam/backupv2/TestBackupV2Service.java @@ -25,6 +25,7 @@ import com.netflix.priam.config.IConfiguration; import com.netflix.priam.connection.JMXNodeTool; import com.netflix.priam.defaultimpl.IService; +import com.netflix.priam.identity.token.ITokenRetriever; import com.netflix.priam.scheduler.PriamScheduler; import com.netflix.priam.tuner.CassandraTunerService; import com.netflix.priam.tuner.TuneCassandra; @@ -47,12 +48,14 @@ public class TestBackupV2Service { private final PriamScheduler scheduler; private final SnapshotMetaTask snapshotMetaTask; private final CassandraTunerService cassandraTunerService; + private final ITokenRetriever tokenRetriever; public TestBackupV2Service() { Injector injector = Guice.createInjector(new BRTestModule()); scheduler = injector.getInstance(PriamScheduler.class); snapshotMetaTask = injector.getInstance(SnapshotMetaTask.class); cassandraTunerService = injector.getInstance(CassandraTunerService.class); + tokenRetriever = injector.getInstance(ITokenRetriever.class); } @Before @@ -103,7 +106,8 @@ public void testBackupDisabled( backupRestoreConfig, scheduler, snapshotMetaTask, - cassandraTunerService); + cassandraTunerService, + tokenRetriever); backupService.scheduleService(); Assert.assertTrue(scheduler.getScheduler().getJobGroupNames().isEmpty()); @@ -142,7 +146,8 @@ public void testBackupEnabled( backupRestoreConfig, scheduler, snapshotMetaTask, - cassandraTunerService); + cassandraTunerService, + tokenRetriever); backupService.scheduleService(); Assert.assertEquals(4, scheduler.getScheduler().getJobKeys(null).size()); } @@ -171,7 +176,8 @@ public void testBackup( backupRestoreConfig, scheduler, snapshotMetaTask, - cassandraTunerService); + cassandraTunerService, + tokenRetriever); backupService.scheduleService(); Assert.assertEquals(3, scheduler.getScheduler().getJobKeys(null).size()); } @@ -208,7 +214,8 @@ public void updateService( backupRestoreConfig, scheduler, snapshotMetaTask, - cassandraTunerService); + cassandraTunerService, + tokenRetriever); backupService.scheduleService(); Assert.assertEquals(3, scheduler.getScheduler().getJobKeys(null).size()); diff --git a/priam/src/test/java/com/netflix/priam/config/FakeBackupRestoreConfig.java b/priam/src/test/java/com/netflix/priam/config/FakeBackupRestoreConfig.java index 266273025..99dd0e551 100644 --- a/priam/src/test/java/com/netflix/priam/config/FakeBackupRestoreConfig.java +++ b/priam/src/test/java/com/netflix/priam/config/FakeBackupRestoreConfig.java @@ -29,4 +29,9 @@ public boolean enableV2Backups() { public boolean enableV2Restore() { return false; } + + @Override + public int getBackupTTLMonitorPeriodInSec() { + return 0; // avoids sleeping altogether in tests. + } } diff --git a/priam/src/test/java/com/netflix/priam/identity/token/TokenRetrieverTest.java b/priam/src/test/java/com/netflix/priam/identity/token/TokenRetrieverTest.java index 94c2f8769..2d589d76f 100644 --- a/priam/src/test/java/com/netflix/priam/identity/token/TokenRetrieverTest.java +++ b/priam/src/test/java/com/netflix/priam/identity/token/TokenRetrieverTest.java @@ -40,6 +40,7 @@ import java.util.stream.IntStream; import mockit.Expectations; import mockit.Mocked; +import org.apache.commons.lang3.math.Fraction; import org.codehaus.jettison.json.JSONObject; import org.junit.Test; import org.junit.jupiter.api.Assertions; @@ -395,6 +396,35 @@ public void testIPIsUpdatedWhenGrabbingPreassignedToken(@Mocked SystemUtils syst Truth.assertThat(getTokenRetriever().get().getHostIP()).isEqualTo("127.0.0.0"); } + @Test + public void testRingPositionFirst(@Mocked SystemUtils systemUtils) throws Exception { + getInstances(6); + create(0, instanceInfo.getInstanceId(), "host_0", "1.2.3.4", "az1", 0 + ""); + TokenRetriever tokenRetriever = getTokenRetriever(); + tokenRetriever.get(); + Truth.assertThat(tokenRetriever.getRingPosition()).isEqualTo(Fraction.getFraction(0, 7)); + } + + @Test + public void testRingPositionMiddle(@Mocked SystemUtils systemUtils) throws Exception { + getInstances(3); + create(4, instanceInfo.getInstanceId(), "host_0", "1.2.3.4", "az1", 4 + ""); + createByIndex(5); + createByIndex(6); + TokenRetriever tokenRetriever = getTokenRetriever(); + tokenRetriever.get(); + Truth.assertThat(tokenRetriever.getRingPosition()).isEqualTo(Fraction.getFraction(3, 6)); + } + + @Test + public void testRingPositionLast(@Mocked SystemUtils systemUtils) throws Exception { + getInstances(6); + create(7, instanceInfo.getInstanceId(), "host_0", "1.2.3.4", "az1", 7 + ""); + TokenRetriever tokenRetriever = getTokenRetriever(); + tokenRetriever.get(); + Truth.assertThat(tokenRetriever.getRingPosition()).isEqualTo(Fraction.getFraction(6, 7)); + } + private String getStatus(List liveInstances, Map tokenToEndpointMap) { JSONObject jsonObject = new JSONObject(); try { @@ -408,18 +438,20 @@ private String getStatus(List liveInstances, Map tokenTo private List getInstances(int noOfInstances) { List allInstances = Lists.newArrayList(); - for (int i = 1; i <= noOfInstances; i++) - allInstances.add( - create( - i, - String.format("instance_id_%d", i), - String.format("hostname_%d", i), - String.format("127.0.0.%d", i), - instanceInfo.getRac(), - i + "")); + for (int i = 1; i <= noOfInstances; i++) allInstances.add(createByIndex(i)); return allInstances; } + private PriamInstance createByIndex(int index) { + return create( + index, + String.format("instance_id_%d", index), + String.format("hostname_%d", index), + String.format("127.0.0.%d", index), + instanceInfo.getRac(), + index + ""); + } + private Set getRacMembership(int noOfInstances) { return IntStream.range(1, noOfInstances + 1) .mapToObj(i -> String.format("instance_id_%d", i)) diff --git a/priam/src/test/java/com/netflix/priam/scheduler/TestSimpleTimer.java b/priam/src/test/java/com/netflix/priam/scheduler/TestSimpleTimer.java new file mode 100644 index 000000000..8aa3ddcc5 --- /dev/null +++ b/priam/src/test/java/com/netflix/priam/scheduler/TestSimpleTimer.java @@ -0,0 +1,59 @@ +package com.netflix.priam.scheduler; + +import com.google.common.truth.Truth; +import java.text.ParseException; +import java.time.Instant; +import java.time.temporal.ChronoUnit; +import java.util.Date; +import org.junit.Assert; +import org.junit.Test; +import org.quartz.Trigger; + +public class TestSimpleTimer { + private static final int PERIOD = 10; + private static final Instant START = Instant.EPOCH.plus(5, ChronoUnit.SECONDS); + + @Test + public void sunnyDay() throws ParseException { + assertions(new SimpleTimer("foo", PERIOD, START).getTrigger(), START); + } + + @Test + public void startBeforeEpoch() { + Assert.assertThrows( + IllegalArgumentException.class, + () -> new SimpleTimer("foo", PERIOD, Instant.EPOCH.minus(5, ChronoUnit.SECONDS))); + } + + @Test + public void startAtEpoch() throws ParseException { + assertions(new SimpleTimer("foo", PERIOD, Instant.EPOCH).getTrigger(), Instant.EPOCH); + } + + @Test + public void startMoreThanOnePeriodAfterEpoch() throws ParseException { + Instant start = Instant.EPOCH.plus(2 * PERIOD, ChronoUnit.SECONDS); + assertions(new SimpleTimer("foo", PERIOD, start).getTrigger(), start); + } + + @Test + public void negativePeriod() { + Assert.assertThrows( + IllegalArgumentException.class, () -> new SimpleTimer("foo", -PERIOD, START)); + } + + @Test + public void zeroPeriod() { + Assert.assertThrows(IllegalArgumentException.class, () -> new SimpleTimer("foo", 0, START)); + } + + private void assertions(Trigger trigger, Instant start) { + Instant now = Instant.now(); + Instant nextFireTime = trigger.getFireTimeAfter(Date.from(now)).toInstant(); + Truth.assertThat(nextFireTime.getEpochSecond() % PERIOD) + .isEqualTo(start.getEpochSecond() % PERIOD); + Truth.assertThat(nextFireTime).isAtMost(Instant.now().plus(PERIOD, ChronoUnit.SECONDS)); + Truth.assertThat(trigger.getFinalFireTime()).isNull(); + Truth.assertThat(trigger.getEndTime()).isNull(); + } +} From c6eb496cedde9ca694cf97e17b1bcbd491cdcc42 Mon Sep 17 00:00:00 2001 From: Matt Lehman Date: Wed, 14 Sep 2022 21:29:27 -0700 Subject: [PATCH 213/228] Update CHANGELOG in advance of 3.11.91 --- CHANGELOG.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index b9af426dd..947b1f333 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,10 @@ # Changelog +## 2022/09/14 3.11.91 +(#997) Revert "CASS-2805 Add hook to optionally skip deletion for files added within a certain window. (#992)" +(#1001) Use IP to get gossip info rather than hostname. +(#1002) Spread deletes over the interval depending on where the node is in the ring. +(#966) Remove UpdateSecuritySettings + ## 2022/06/26 3.11.90 *UpdateSecuritySettings will be removed in the next release* (#992) Add hook to optionally skip deletion for files added within a certain window. From 71d8415c261dd4f3fbd9c016b114e1b33c04718f Mon Sep 17 00:00:00 2001 From: mattl-netflix <63665634+mattl-netflix@users.noreply.github.com> Date: Thu, 15 Sep 2022 21:29:17 -0700 Subject: [PATCH 214/228] RandomPartitioner creates tokens that don't always fit into a long so we use BigInteger to store and compare them. (#1005) --- .../com/netflix/priam/identity/token/TokenRetriever.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/priam/src/main/java/com/netflix/priam/identity/token/TokenRetriever.java b/priam/src/main/java/com/netflix/priam/identity/token/TokenRetriever.java index 251044d2b..8ad3e02ae 100644 --- a/priam/src/main/java/com/netflix/priam/identity/token/TokenRetriever.java +++ b/priam/src/main/java/com/netflix/priam/identity/token/TokenRetriever.java @@ -13,6 +13,7 @@ import com.netflix.priam.utils.ITokenManager; import com.netflix.priam.utils.RetryableCallable; import com.netflix.priam.utils.Sleeper; +import java.math.BigInteger; import java.util.List; import java.util.Optional; import java.util.Random; @@ -87,11 +88,11 @@ public boolean isTokenPregenerated() { @Override public Fraction getRingPosition() throws Exception { get(); - long tokenAsLong = Long.parseLong(priamInstance.getToken()); + BigInteger token = new BigInteger(priamInstance.getToken()); ImmutableSet nodes = factory.getAllIds(config.getAppName()); long ringPosition = nodes.stream() - .filter(node -> Long.parseLong(node.getToken()) < tokenAsLong) + .filter(node -> token.compareTo(new BigInteger(node.getToken())) > 0) .count(); return Fraction.getFraction(Math.toIntExact(ringPosition), nodes.size()); } From 53fcbc7025d557745b93e67092584eb5673de222 Mon Sep 17 00:00:00 2001 From: Matt Lehman Date: Thu, 15 Sep 2022 21:31:27 -0700 Subject: [PATCH 215/228] Update CHANGELOG in advance of 3.11.92 --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 947b1f333..dc915d76d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,7 @@ # Changelog +## 2022/09/15 3.11.92 +(#1005) RandomPartitioner creates tokens that don't always fit into a long so we use BigInteger to store and compare them. + ## 2022/09/14 3.11.91 (#997) Revert "CASS-2805 Add hook to optionally skip deletion for files added within a certain window. (#992)" (#1001) Use IP to get gossip info rather than hostname. From 528e0ce425c8e679e17e11dd73be7eebd0220327 Mon Sep 17 00:00:00 2001 From: mattl-netflix <63665634+mattl-netflix@users.noreply.github.com> Date: Sun, 2 Oct 2022 22:35:40 -0700 Subject: [PATCH 216/228] Pass millis to sleep instead of seconds. (#1009) --- .../java/com/netflix/priam/backupv2/BackupTTLTask.java | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/priam/src/main/java/com/netflix/priam/backupv2/BackupTTLTask.java b/priam/src/main/java/com/netflix/priam/backupv2/BackupTTLTask.java index c67223bc2..534bbff0a 100644 --- a/priam/src/main/java/com/netflix/priam/backupv2/BackupTTLTask.java +++ b/priam/src/main/java/com/netflix/priam/backupv2/BackupTTLTask.java @@ -67,7 +67,7 @@ public class BackupTTLTask extends Task { private static final Lock lock = new ReentrantLock(); private final int BATCH_SIZE = 1000; private final Instant start_of_feature = DateUtil.parseInstant("201801010000"); - private final int maxWaitSeconds; + private final int maxWaitMillis; @Inject public BackupTTLTask( @@ -85,8 +85,9 @@ public BackupTTLTask( this.fileSystem = backupFileSystemCtx.getFileStrategy(configuration); this.abstractBackupPathProvider = abstractBackupPathProvider; this.instanceState = instanceState; - this.maxWaitSeconds = - backupRestoreConfig.getBackupTTLMonitorPeriodInSec() + this.maxWaitMillis = + 1_000 + * backupRestoreConfig.getBackupTTLMonitorPeriodInSec() / tokenRetriever.getRingPosition().getDenominator(); } @@ -107,7 +108,7 @@ public void execute() throws Exception { } // Sleep a random amount but not so long that it will spill into the next token's turn. - if (maxWaitSeconds > 0) Thread.sleep(new Random().nextInt(maxWaitSeconds)); + if (maxWaitMillis > 0) Thread.sleep(new Random().nextInt(maxWaitMillis)); try { filesInMeta.clear(); From 96ec37851da18bf8f60c20241b817d3ad34c3533 Mon Sep 17 00:00:00 2001 From: mattl-netflix <63665634+mattl-netflix@users.noreply.github.com> Date: Fri, 7 Oct 2022 14:29:33 -0700 Subject: [PATCH 217/228] Identify incrementals, optionally skip metafile compression, and upload data files last. (#999) * Reveal whether an sstable file is an incremental backup in SNS notification. * Remove retry parameter from upload method. It is always 10 in practice and there is no way to change it without a code change. * Move upload logic out of AbstractBackup into an interface called BackupHelper. * Upload data files last. * Don't compress anything when backupsToCompress is set to None. --- .../netflix/priam/backup/AbstractBackup.java | 102 +--------- .../priam/backup/AbstractBackupPath.java | 25 ++- .../priam/backup/AbstractFileSystem.java | 30 +-- .../netflix/priam/backup/BackupHelper.java | 29 +++ .../priam/backup/BackupHelperImpl.java | 110 +++++++++++ .../netflix/priam/backup/CommitLogBackup.java | 2 +- .../priam/backup/CommitLogBackupTask.java | 9 +- .../priam/backup/IBackupFileSystem.java | 48 ++--- .../priam/backup/IncrementalBackup.java | 15 +- .../com/netflix/priam/backup/MetaData.java | 2 +- .../netflix/priam/backup/SnapshotBackup.java | 10 +- .../priam/backupv2/MetaFileWriterBuilder.java | 2 +- .../priam/backupv2/SnapshotMetaTask.java | 15 +- .../notification/BackupNotificationMgr.java | 1 + .../priam/backup/TestAbstractBackup.java | 129 ------------- .../priam/backup/TestAbstractFileSystem.java | 17 +- .../priam/backup/TestBackupHelperImpl.java | 181 ++++++++++++++++++ .../priam/backup/TestS3FileSystem.java | 17 +- 18 files changed, 423 insertions(+), 321 deletions(-) create mode 100644 priam/src/main/java/com/netflix/priam/backup/BackupHelper.java create mode 100644 priam/src/main/java/com/netflix/priam/backup/BackupHelperImpl.java delete mode 100644 priam/src/test/java/com/netflix/priam/backup/TestAbstractBackup.java create mode 100644 priam/src/test/java/com/netflix/priam/backup/TestBackupHelperImpl.java diff --git a/priam/src/main/java/com/netflix/priam/backup/AbstractBackup.java b/priam/src/main/java/com/netflix/priam/backup/AbstractBackup.java index 282eecf5c..bd8cab2cc 100644 --- a/priam/src/main/java/com/netflix/priam/backup/AbstractBackup.java +++ b/priam/src/main/java/com/netflix/priam/backup/AbstractBackup.java @@ -16,131 +16,31 @@ */ package com.netflix.priam.backup; -import static java.util.stream.Collectors.toSet; - -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableSet; -import com.google.common.util.concurrent.Futures; -import com.google.common.util.concurrent.ListenableFuture; import com.google.inject.Inject; -import com.google.inject.Provider; -import com.netflix.priam.backup.AbstractBackupPath.BackupFileType; -import com.netflix.priam.compress.CompressionType; -import com.netflix.priam.config.BackupsToCompress; import com.netflix.priam.config.IConfiguration; import com.netflix.priam.scheduler.Task; import com.netflix.priam.utils.SystemUtils; import java.io.File; import java.io.FileFilter; -import java.io.IOException; import java.nio.file.DirectoryStream; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; -import java.time.Instant; import java.util.HashSet; import java.util.Optional; import java.util.Set; -import java.util.stream.Stream; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** Abstract Backup class for uploading files to backup location */ public abstract class AbstractBackup extends Task { private static final Logger logger = LoggerFactory.getLogger(AbstractBackup.class); - private static final String COMPRESSION_SUFFIX = "-CompressionInfo.db"; static final String INCREMENTAL_BACKUP_FOLDER = "backups"; public static final String SNAPSHOT_FOLDER = "snapshots"; - protected final Provider pathFactory; - - protected IBackupFileSystem fs; - @Inject - public AbstractBackup( - IConfiguration config, - IFileSystemContext backupFileSystemCtx, - Provider pathFactory) { + public AbstractBackup(IConfiguration config) { super(config); - this.pathFactory = pathFactory; - this.fs = backupFileSystemCtx.getFileStrategy(config); - } - - /** Overload that uploads files without any custom throttling */ - protected ImmutableList> uploadAndDeleteAllFiles( - final File parent, final BackupFileType type, boolean async) throws Exception { - return uploadAndDeleteAllFiles(parent, type, async, Instant.EPOCH); - } - - /** - * Upload files in the specified dir. Does not delete the file in case of error. The files are - * uploaded serially or async based on flag provided. - * - * @param parent Parent dir - * @param type Type of file (META, SST, SNAP etc) - * @param async Upload the file(s) in async fashion if enabled. - * @param target target time of completion of the batch of files - * @return List of files that are successfully uploaded as part of backup - * @throws Exception when there is failure in uploading files. - */ - protected ImmutableList> uploadAndDeleteAllFiles( - final File parent, final BackupFileType type, boolean async, Instant target) - throws Exception { - ImmutableSet backupPaths = getBackupPaths(parent, type); - final ImmutableList.Builder> futures = - ImmutableList.builder(); - for (AbstractBackupPath bp : backupPaths) { - if (async) futures.add(fs.asyncUploadAndDelete(bp, 10, target)); - else { - fs.uploadAndDelete(bp, 10, target); - futures.add(Futures.immediateFuture(bp)); - } - } - return futures.build(); - } - - protected ImmutableSet getBackupPaths(File dir, BackupFileType type) - throws IOException { - Set files; - try (Stream pathStream = Files.list(dir.toPath())) { - files = pathStream.map(Path::toFile).filter(File::isFile).collect(toSet()); - } - Set compressedFilePrefixes = - files.stream() - .map(File::getName) - .filter(name -> name.endsWith(COMPRESSION_SUFFIX)) - .map(name -> name.substring(0, name.lastIndexOf('-'))) - .collect(toSet()); - final ImmutableSet.Builder bps = ImmutableSet.builder(); - for (File file : files) { - final AbstractBackupPath bp = pathFactory.get(); - bp.parseLocal(file, type); - bp.setCompression(getCorrectCompressionAlgorithm(bp, compressedFilePrefixes)); - bps.add(bp); - } - return bps.build(); - } - - private CompressionType getCorrectCompressionAlgorithm( - AbstractBackupPath path, Set compressedFiles) { - if (!BackupFileType.isV2(path.getType())) { - return CompressionType.SNAPPY; - } - String file = path.getFileName(); - BackupsToCompress which = config.getBackupsToCompress(); - switch (which) { - case NONE: - return CompressionType.NONE; - case ALL: - return CompressionType.SNAPPY; - case IF_REQUIRED: - int splitIndex = file.lastIndexOf('-'); - return splitIndex >= 0 && compressedFiles.contains(file.substring(0, splitIndex)) - ? CompressionType.NONE - : CompressionType.SNAPPY; - default: - throw new IllegalArgumentException("NONE, ALL, UNCOMPRESSED only. Saw: " + which); - } } protected final void initiateBackup( diff --git a/priam/src/main/java/com/netflix/priam/backup/AbstractBackupPath.java b/priam/src/main/java/com/netflix/priam/backup/AbstractBackupPath.java index 3d4e77c14..e5c9a69fa 100644 --- a/priam/src/main/java/com/netflix/priam/backup/AbstractBackupPath.java +++ b/priam/src/main/java/com/netflix/priam/backup/AbstractBackupPath.java @@ -24,6 +24,7 @@ import com.google.inject.ImplementedBy; import com.netflix.priam.aws.RemoteBackupPath; import com.netflix.priam.compress.CompressionType; +import com.netflix.priam.config.BackupsToCompress; import com.netflix.priam.config.IConfiguration; import com.netflix.priam.cryptography.CryptographyAlgorithm; import com.netflix.priam.identity.InstanceIdentity; @@ -35,6 +36,7 @@ import java.nio.file.attribute.BasicFileAttributes; import java.time.Instant; import java.util.Date; +import java.util.Optional; import org.apache.commons.lang3.StringUtils; @ImplementedBy(RemoteBackupPath.class) @@ -95,12 +97,17 @@ public static BackupFileType fromString(String s) throws BackupRestoreException private Instant lastModified; private Instant creationTime; private Date uploadedTs; - private CompressionType compression = CompressionType.SNAPPY; + private CompressionType compression; private CryptographyAlgorithm encryption = CryptographyAlgorithm.PLAINTEXT; + private boolean isIncremental; public AbstractBackupPath(IConfiguration config, InstanceIdentity instanceIdentity) { this.instanceIdentity = instanceIdentity; this.config = config; + this.compression = + config.getBackupsToCompress() == BackupsToCompress.NONE + ? CompressionType.NONE + : CompressionType.SNAPPY; } public void parseLocal(File file, BackupFileType type) { @@ -130,10 +137,14 @@ public void parseLocal(File file, BackupFileType type) { this.keyspace = parts[0]; this.columnFamily = parts[1]; } - if (type == BackupFileType.SECONDARY_INDEX_V2) { - Integer index = BackupFolder.fromName(parts[2]).map(FOLDER_POSITIONS::get).orElse(null); - Preconditions.checkNotNull(index, "Unrecognized backup folder " + parts[2]); - this.indexDir = parts[index]; + if (BackupFileType.isDataFile(type)) { + Optional folder = BackupFolder.fromName(parts[2]); + this.isIncremental = folder.filter(BackupFolder.BACKUPS::equals).isPresent(); + if (type == BackupFileType.SECONDARY_INDEX_V2) { + Integer index = folder.map(FOLDER_POSITIONS::get).orElse(null); + Preconditions.checkNotNull(index, "Unrecognized backup folder " + parts[2]); + this.indexDir = parts[index]; + } } /* @@ -323,6 +334,10 @@ public void setEncryption(String encryption) { this.encryption = CryptographyAlgorithm.valueOf(encryption); } + public boolean isIncremental() { + return isIncremental; + } + @Override public String toString() { return "From: " + getRemotePath() + " To: " + newRestoreFile().getPath(); diff --git a/priam/src/main/java/com/netflix/priam/backup/AbstractFileSystem.java b/priam/src/main/java/com/netflix/priam/backup/AbstractFileSystem.java index 0737ac639..a39190ab5 100644 --- a/priam/src/main/java/com/netflix/priam/backup/AbstractFileSystem.java +++ b/priam/src/main/java/com/netflix/priam/backup/AbstractFileSystem.java @@ -17,9 +17,11 @@ package com.netflix.priam.backup; +import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; import com.google.common.cache.Cache; import com.google.common.cache.CacheBuilder; +import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.ListeningExecutorService; import com.google.common.util.concurrent.MoreExecutors; @@ -157,19 +159,21 @@ protected abstract void downloadFileImpl(final AbstractBackupPath path, String s throws BackupRestoreException; @Override - public ListenableFuture asyncUploadAndDelete( - final AbstractBackupPath path, final int retry, Instant target) - throws RejectedExecutionException { - return fileUploadExecutor.submit( - () -> { - uploadAndDelete(path, retry, target); - return path; - }); + public ListenableFuture uploadAndDelete( + final AbstractBackupPath path, Instant target, boolean async) + throws RejectedExecutionException, BackupRestoreException { + if (async) { + return fileUploadExecutor.submit( + () -> uploadAndDeleteInternal(path, target, 10 /* retries */)); + } else { + return Futures.immediateFuture(uploadAndDeleteInternal(path, target, 10 /* retries */)); + } } - @Override - public void uploadAndDelete(final AbstractBackupPath path, final int retry, Instant target) - throws BackupRestoreException { + @VisibleForTesting + public AbstractBackupPath uploadAndDeleteInternal( + final AbstractBackupPath path, Instant target, int retry) + throws RejectedExecutionException, BackupRestoreException { Path localPath = Paths.get(path.getBackupFile().getAbsolutePath()); File localFile = localPath.toFile(); Preconditions.checkArgument( @@ -188,7 +192,8 @@ public void uploadAndDelete(final AbstractBackupPath path, final int retry, Inst if (path.getType() != BackupFileType.SST_V2 || !checkObjectExists(remotePath)) { notifyEventStart(new BackupEvent(path)); uploadedFileSize = - new BoundedExponentialRetryCallable(500, 10000, retry) { + new BoundedExponentialRetryCallable( + 500 /* minSleep */, 10000 /* maxSleep */, retry) { @Override public Long retriableCall() throws Exception { return uploadFileImpl(path, target); @@ -233,6 +238,7 @@ public Long retriableCall() throws Exception { tasksQueued.remove(localPath); } } else logger.info("Already in queue, no-op. File: {}", localPath); + return path; } private void addObjectCache(Path remotePath) { diff --git a/priam/src/main/java/com/netflix/priam/backup/BackupHelper.java b/priam/src/main/java/com/netflix/priam/backup/BackupHelper.java new file mode 100644 index 000000000..38a1e98b1 --- /dev/null +++ b/priam/src/main/java/com/netflix/priam/backup/BackupHelper.java @@ -0,0 +1,29 @@ +package com.netflix.priam.backup; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; +import com.google.common.util.concurrent.ListenableFuture; +import com.google.inject.ImplementedBy; +import java.io.File; +import java.io.IOException; +import java.time.Instant; + +@ImplementedBy(BackupHelperImpl.class) +public interface BackupHelper { + + default ImmutableList> uploadAndDeleteAllFiles( + final File parent, final AbstractBackupPath.BackupFileType type, boolean async) + throws Exception { + return uploadAndDeleteAllFiles(parent, type, Instant.EPOCH, async); + } + + ImmutableList> uploadAndDeleteAllFiles( + final File parent, + final AbstractBackupPath.BackupFileType type, + Instant target, + boolean async) + throws Exception; + + ImmutableSet getBackupPaths( + File dir, AbstractBackupPath.BackupFileType type) throws IOException; +} diff --git a/priam/src/main/java/com/netflix/priam/backup/BackupHelperImpl.java b/priam/src/main/java/com/netflix/priam/backup/BackupHelperImpl.java new file mode 100644 index 000000000..e03b8001b --- /dev/null +++ b/priam/src/main/java/com/netflix/priam/backup/BackupHelperImpl.java @@ -0,0 +1,110 @@ +package com.netflix.priam.backup; + +import static java.util.stream.Collectors.toSet; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; +import com.google.common.util.concurrent.ListenableFuture; +import com.google.inject.Inject; +import com.google.inject.Provider; +import com.netflix.priam.compress.CompressionType; +import com.netflix.priam.config.BackupsToCompress; +import com.netflix.priam.config.IConfiguration; +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.Instant; +import java.util.Set; +import java.util.stream.Stream; + +public class BackupHelperImpl implements BackupHelper { + + private static final String COMPRESSION_SUFFIX = "-CompressionInfo.db"; + private static final String DATA_SUFFIX = "-Data.db"; + private final Provider pathFactory; + private final IBackupFileSystem fs; + private final IConfiguration config; + + @Inject + public BackupHelperImpl( + IConfiguration config, + IFileSystemContext backupFileSystemCtx, + Provider pathFactory) { + this.config = config; + this.pathFactory = pathFactory; + this.fs = backupFileSystemCtx.getFileStrategy(config); + } + + /** + * Upload files in the specified dir. Does not delete the file in case of error. The files are + * uploaded serially or async based on flag provided. + * + * @param parent Parent dir + * @param type Type of file (META, SST, SNAP etc) + * @param async Upload the file(s) in async fashion if enabled. + * @return List of files that are successfully uploaded as part of backup + * @throws Exception when there is failure in uploading files. + */ + @Override + public ImmutableList> uploadAndDeleteAllFiles( + final File parent, + final AbstractBackupPath.BackupFileType type, + Instant target, + boolean async) + throws Exception { + final ImmutableList.Builder> futures = + ImmutableList.builder(); + for (AbstractBackupPath bp : getBackupPaths(parent, type)) { + futures.add(fs.uploadAndDelete(bp, target, async)); + } + return futures.build(); + } + + @Override + public ImmutableSet getBackupPaths( + File dir, AbstractBackupPath.BackupFileType type) throws IOException { + Set files; + try (Stream pathStream = Files.list(dir.toPath())) { + files = pathStream.map(Path::toFile).filter(File::isFile).collect(toSet()); + } + Set compressedFilePrefixes = + files.stream() + .map(File::getName) + .filter(name -> name.endsWith(COMPRESSION_SUFFIX)) + .map(name -> name.substring(0, name.lastIndexOf('-'))) + .collect(toSet()); + final ImmutableSet.Builder bps = ImmutableSet.builder(); + ImmutableSet.Builder dataFiles = ImmutableSet.builder(); + for (File file : files) { + final AbstractBackupPath bp = pathFactory.get(); + bp.parseLocal(file, type); + bp.setCompression(getCorrectCompressionAlgorithm(bp, compressedFilePrefixes)); + (file.getAbsolutePath().endsWith(DATA_SUFFIX) ? dataFiles : bps).add(bp); + } + bps.addAll(dataFiles.build()); + return bps.build(); + } + + private CompressionType getCorrectCompressionAlgorithm( + AbstractBackupPath path, Set compressedFiles) { + if (!AbstractBackupPath.BackupFileType.isV2(path.getType())) { + return CompressionType.SNAPPY; + } + String file = path.getFileName(); + BackupsToCompress which = config.getBackupsToCompress(); + switch (which) { + case NONE: + return CompressionType.NONE; + case ALL: + return CompressionType.SNAPPY; + case IF_REQUIRED: + int splitIndex = file.lastIndexOf('-'); + return splitIndex >= 0 && compressedFiles.contains(file.substring(0, splitIndex)) + ? CompressionType.NONE + : CompressionType.SNAPPY; + default: + throw new IllegalArgumentException("NONE, ALL, UNCOMPRESSED only. Saw: " + which); + } + } +} diff --git a/priam/src/main/java/com/netflix/priam/backup/CommitLogBackup.java b/priam/src/main/java/com/netflix/priam/backup/CommitLogBackup.java index 52ac0fc3b..8de2b4dd0 100644 --- a/priam/src/main/java/com/netflix/priam/backup/CommitLogBackup.java +++ b/priam/src/main/java/com/netflix/priam/backup/CommitLogBackup.java @@ -65,7 +65,7 @@ public List upload(String archivedDir, final String snapshot if (snapshotName != null) bp.time = DateUtil.getDate(snapshotName); - fs.uploadAndDelete(bp, 10); + fs.uploadAndDelete(bp, false /* async */); bps.add(bp); addToRemotePath(bp.getRemotePath()); } catch (Exception e) { diff --git a/priam/src/main/java/com/netflix/priam/backup/CommitLogBackupTask.java b/priam/src/main/java/com/netflix/priam/backup/CommitLogBackupTask.java index 6286f4ef8..e4e059b11 100644 --- a/priam/src/main/java/com/netflix/priam/backup/CommitLogBackupTask.java +++ b/priam/src/main/java/com/netflix/priam/backup/CommitLogBackupTask.java @@ -14,7 +14,6 @@ package com.netflix.priam.backup; import com.google.inject.Inject; -import com.google.inject.Provider; import com.google.inject.Singleton; import com.netflix.priam.config.IConfiguration; import com.netflix.priam.scheduler.SimpleTimer; @@ -32,12 +31,8 @@ public class CommitLogBackupTask extends AbstractBackup { private final CommitLogBackup clBackup; @Inject - public CommitLogBackupTask( - IConfiguration config, - Provider pathFactory, - CommitLogBackup clBackup, - IFileSystemContext backupFileSystemCtx) { - super(config, backupFileSystemCtx, pathFactory); + public CommitLogBackupTask(IConfiguration config, CommitLogBackup clBackup) { + super(config); this.clBackup = clBackup; } diff --git a/priam/src/main/java/com/netflix/priam/backup/IBackupFileSystem.java b/priam/src/main/java/com/netflix/priam/backup/IBackupFileSystem.java index b8a69ec94..e33ff3bfc 100644 --- a/priam/src/main/java/com/netflix/priam/backup/IBackupFileSystem.java +++ b/priam/src/main/java/com/netflix/priam/backup/IBackupFileSystem.java @@ -57,46 +57,24 @@ Future asyncDownloadFile(final AbstractBackupPath path, final int retry) throws BackupRestoreException, RejectedExecutionException; /** Overload that uploads as fast as possible without any custom throttling */ - default void uploadAndDelete(AbstractBackupPath path, int retry) + default void uploadAndDelete(AbstractBackupPath path, boolean async) throws FileNotFoundException, BackupRestoreException { - uploadAndDelete(path, retry, Instant.EPOCH); + uploadAndDelete(path, Instant.EPOCH, async); } /** - * Upload the local file to its remote counterpart. Both locations are embedded within the path - * parameter. De-duping of the file to upload will always be done by comparing the - * files-in-progress to be uploaded. This may result in this particular request to not to be - * executed e.g. if any other thread has given the same file to upload and that file is in - * internal queue. Note that de-duping is best effort and is not always guaranteed as we try to - * avoid lock on read/write of the files-in-progress. Once uploaded, files are deleted. - * - * @param path Backup path representing a local and remote file pair - * @param retry No of times to retry to upload a file. If <1, it will try to upload file - * exactly once. - * @param target The target time of completion of all files in the upload. - * @throws BackupRestoreException in case of failure to upload for any reason including file not - * readable or remote file system errors. - * @throws FileNotFoundException If a file as denoted by localPath is not available or is a - * directory. - */ - void uploadAndDelete(AbstractBackupPath path, int retry, Instant target) - throws FileNotFoundException, BackupRestoreException; - - /** Overload that uploads as fast as possible without any custom throttling */ - default ListenableFuture asyncUploadAndDelete( - final AbstractBackupPath path, final int retry) - throws FileNotFoundException, RejectedExecutionException, BackupRestoreException { - return asyncUploadAndDelete(path, retry, Instant.EPOCH); - } - - /** - * Upload the local file denoted by localPath in async fashion to the remote file system at - * location denoted by remotePath. + * Upload the local file to its remote counterpart in an optionally async fashion. Both + * locations are embedded within the path parameter. De-duping of the file to upload will always + * be done by comparing the files-in-progress to be uploaded. This may result in this particular + * request to not to be executed e.g. if any other thread has given the same file to upload and + * that file is in internal queue. Note that de-duping is best effort and is not always + * guaranteed as we try to avoid lock on read/write of the files-in-progress. Once uploaded, + * files are deleted. Uploads are retried 10 times. * * @param path AbstractBackupPath to be used to send backup notifications only. - * @param retry No of times to retry to upload a file. If <1, it will try to upload file - * exactly once. * @param target The target time of completion of all files in the upload. + * @param async boolean to determine whether the call should block or return immediately and + * upload asynchronously * @return The future of the async job to monitor the progress of the job. This will be null if * file was de-duped for upload. * @throws BackupRestoreException in case of failure to upload for any reason including file not @@ -106,8 +84,8 @@ default ListenableFuture asyncUploadAndDelete( * @throws RejectedExecutionException if the queue is full and TIMEOUT is reached while trying * to add the work to the queue. */ - ListenableFuture asyncUploadAndDelete( - final AbstractBackupPath path, final int retry, Instant target) + ListenableFuture uploadAndDelete( + final AbstractBackupPath path, Instant target, boolean async) throws FileNotFoundException, RejectedExecutionException, BackupRestoreException; /** diff --git a/priam/src/main/java/com/netflix/priam/backup/IncrementalBackup.java b/priam/src/main/java/com/netflix/priam/backup/IncrementalBackup.java index 2326b1618..43d23bf66 100644 --- a/priam/src/main/java/com/netflix/priam/backup/IncrementalBackup.java +++ b/priam/src/main/java/com/netflix/priam/backup/IncrementalBackup.java @@ -21,7 +21,6 @@ import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.MoreExecutors; import com.google.inject.Inject; -import com.google.inject.Provider; import com.google.inject.Singleton; import com.netflix.priam.backup.AbstractBackupPath.BackupFileType; import com.netflix.priam.backupv2.SnapshotMetaTask; @@ -45,20 +44,21 @@ public class IncrementalBackup extends AbstractBackup { public static final String JOBNAME = "IncrementalBackup"; private final BackupRestoreUtil backupRestoreUtil; private final IBackupRestoreConfig backupRestoreConfig; + private final BackupHelper backupHelper; @Inject public IncrementalBackup( IConfiguration config, IBackupRestoreConfig backupRestoreConfig, - Provider pathFactory, - IFileSystemContext backupFileSystemCtx) { - super(config, backupFileSystemCtx, pathFactory); + BackupHelper backupHelper) { + super(config); // a means to upload audit trail (via meta_cf_yyyymmddhhmm.json) of files successfully // uploaded) this.backupRestoreConfig = backupRestoreConfig; backupRestoreUtil = new BackupRestoreUtil( config.getIncrementalIncludeCFList(), config.getIncrementalExcludeCFList()); + this.backupHelper = backupHelper; } @Override @@ -120,13 +120,16 @@ protected void processColumnFamily(File backupDir) throws Exception { // upload SSTables and components ImmutableList> futures = - uploadAndDeleteAllFiles(backupDir, fileType, config.enableAsyncIncremental()); + backupHelper.uploadAndDeleteAllFiles( + backupDir, fileType, config.enableAsyncIncremental()); Futures.whenAllComplete(futures).call(() -> null, MoreExecutors.directExecutor()); // Next, upload secondary indexes fileType = BackupFileType.SECONDARY_INDEX_V2; for (File directory : getSecondaryIndexDirectories(backupDir)) { - futures = uploadAndDeleteAllFiles(directory, fileType, config.enableAsyncIncremental()); + futures = + backupHelper.uploadAndDeleteAllFiles( + directory, fileType, config.enableAsyncIncremental()); if (futures.stream().allMatch(ListenableFuture::isDone)) { deleteIfEmpty(directory); } else { diff --git a/priam/src/main/java/com/netflix/priam/backup/MetaData.java b/priam/src/main/java/com/netflix/priam/backup/MetaData.java index f20c18a54..60dffbf62 100644 --- a/priam/src/main/java/com/netflix/priam/backup/MetaData.java +++ b/priam/src/main/java/com/netflix/priam/backup/MetaData.java @@ -61,7 +61,7 @@ public AbstractBackupPath set(List bps, String snapshotName) fr.write(jsonObj.toJSONString()); } AbstractBackupPath backupfile = decorateMetaJson(metafile, snapshotName); - fs.uploadAndDelete(backupfile, 10); + fs.uploadAndDelete(backupfile, false /* async */); addToRemotePath(backupfile.getRemotePath()); return backupfile; } diff --git a/priam/src/main/java/com/netflix/priam/backup/SnapshotBackup.java b/priam/src/main/java/com/netflix/priam/backup/SnapshotBackup.java index babc61c9b..fdde89fa8 100644 --- a/priam/src/main/java/com/netflix/priam/backup/SnapshotBackup.java +++ b/priam/src/main/java/com/netflix/priam/backup/SnapshotBackup.java @@ -20,7 +20,6 @@ import com.google.common.collect.Lists; import com.google.common.util.concurrent.ListenableFuture; import com.google.inject.Inject; -import com.google.inject.Provider; import com.google.inject.Singleton; import com.netflix.priam.backup.AbstractBackupPath.BackupFileType; import com.netflix.priam.backupv2.ForgottenFilesManager; @@ -61,19 +60,20 @@ public class SnapshotBackup extends AbstractBackup { private Instant snapshotInstant = DateUtil.getInstant(); private List abstractBackupPaths = null; private final CassandraOperations cassandraOperations; + private final BackupHelper backupHelper; private static final Lock lock = new ReentrantLock(); @Inject public SnapshotBackup( IConfiguration config, - Provider pathFactory, + BackupHelper backupHelper, MetaData metaData, - IFileSystemContext backupFileSystemCtx, IBackupStatusMgr snapshotStatusMgr, InstanceIdentity instanceIdentity, CassandraOperations cassandraOperations, ForgottenFilesManager forgottenFilesManager) { - super(config, backupFileSystemCtx, pathFactory); + super(config); + this.backupHelper = backupHelper; this.metaData = metaData; this.snapshotStatusMgr = snapshotStatusMgr; this.instanceIdentity = instanceIdentity; @@ -212,7 +212,7 @@ protected void processColumnFamily(File backupDir) throws Exception { // Add files to this dir ImmutableList> futures = - uploadAndDeleteAllFiles( + backupHelper.uploadAndDeleteAllFiles( snapshotDir, BackupFileType.SNAP, config.enableAsyncSnapshot()); for (Future future : futures) { abstractBackupPaths.add(future.get()); diff --git a/priam/src/main/java/com/netflix/priam/backupv2/MetaFileWriterBuilder.java b/priam/src/main/java/com/netflix/priam/backupv2/MetaFileWriterBuilder.java index 7c5ec79ea..cc7e77f2e 100644 --- a/priam/src/main/java/com/netflix/priam/backupv2/MetaFileWriterBuilder.java +++ b/priam/src/main/java/com/netflix/priam/backupv2/MetaFileWriterBuilder.java @@ -197,7 +197,7 @@ public void uploadMetaFile() throws Exception { AbstractBackupPath abstractBackupPath = pathFactory.get(); abstractBackupPath.parseLocal( metaFilePath.toFile(), AbstractBackupPath.BackupFileType.META_V2); - backupFileSystem.uploadAndDelete(abstractBackupPath, 10); + backupFileSystem.uploadAndDelete(abstractBackupPath, false /* async */); } public Path getMetaFilePath() { diff --git a/priam/src/main/java/com/netflix/priam/backupv2/SnapshotMetaTask.java b/priam/src/main/java/com/netflix/priam/backupv2/SnapshotMetaTask.java index 10a7fb712..065b8a6fe 100644 --- a/priam/src/main/java/com/netflix/priam/backupv2/SnapshotMetaTask.java +++ b/priam/src/main/java/com/netflix/priam/backupv2/SnapshotMetaTask.java @@ -21,7 +21,6 @@ import com.google.common.collect.ImmutableSetMultimap; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; -import com.google.inject.Provider; import com.netflix.priam.backup.*; import com.netflix.priam.config.IBackupRestoreConfig; import com.netflix.priam.config.IConfiguration; @@ -92,6 +91,7 @@ public class SnapshotMetaTask extends AbstractBackup { private final Clock clock; private final IBackupRestoreConfig backupRestoreConfig; private final BackupVerification backupVerification; + private final BackupHelper backupHelper; private enum MetaStep { META_GENERATION, @@ -103,8 +103,7 @@ private enum MetaStep { @Inject SnapshotMetaTask( IConfiguration config, - IFileSystemContext backupFileSystemCtx, - Provider pathFactory, + BackupHelper backupHelper, MetaFileWriterBuilder metaFileWriter, @Named("v2") IMetaProxy metaProxy, InstanceIdentity instanceIdentity, @@ -113,8 +112,9 @@ private enum MetaStep { Clock clock, IBackupRestoreConfig backupRestoreConfig, BackupVerification backupVerification) { - super(config, backupFileSystemCtx, pathFactory); + super(config); this.config = config; + this.backupHelper = backupHelper; this.instanceIdentity = instanceIdentity; this.snapshotStatusMgr = snapshotStatusMgr; this.cassandraOperations = cassandraOperations; @@ -285,13 +285,13 @@ private void uploadAllFiles(final File backupDir) throws Exception { // We do not want to wait for completion and we just want to add them to queue. This // is to ensure that next run happens on time. AbstractBackupPath.BackupFileType type = AbstractBackupPath.BackupFileType.SST_V2; - uploadAndDeleteAllFiles(snapshotDirectory, type, true, target); + backupHelper.uploadAndDeleteAllFiles(snapshotDirectory, type, target, true); // Next, upload secondary indexes type = AbstractBackupPath.BackupFileType.SECONDARY_INDEX_V2; ImmutableList> futures; for (File subDir : getSecondaryIndexDirectories(snapshotDirectory)) { - futures = uploadAndDeleteAllFiles(subDir, type, true, target); + futures = backupHelper.uploadAndDeleteAllFiles(subDir, type, target, true); if (futures.isEmpty()) { deleteIfEmpty(subDir); } @@ -393,7 +393,8 @@ private ImmutableSetMultimap getSSTables( File snapshotDir, AbstractBackupPath.BackupFileType type) throws IOException { ImmutableSetMultimap.Builder ssTables = ImmutableSetMultimap.builder(); - getBackupPaths(snapshotDir, type) + backupHelper + .getBackupPaths(snapshotDir, type) .forEach(bp -> getPrefix(bp.getBackupFile()).ifPresent(p -> ssTables.put(p, bp))); return ssTables.build(); } diff --git a/priam/src/main/java/com/netflix/priam/notification/BackupNotificationMgr.java b/priam/src/main/java/com/netflix/priam/notification/BackupNotificationMgr.java index 9ab37aca5..b15a8a341 100644 --- a/priam/src/main/java/com/netflix/priam/notification/BackupNotificationMgr.java +++ b/priam/src/main/java/com/netflix/priam/notification/BackupNotificationMgr.java @@ -127,6 +127,7 @@ private void notify(AbstractBackupPath abp, String uploadStatus) { jsonObject.put("uploadstatus", uploadStatus); jsonObject.put("compression", abp.getCompression().name()); jsonObject.put("encryption", abp.getEncryption().name()); + jsonObject.put("isincremental", abp.isIncremental()); // SNS Attributes for filtering messages. Cluster name and backup file type. Map messageAttributes = diff --git a/priam/src/test/java/com/netflix/priam/backup/TestAbstractBackup.java b/priam/src/test/java/com/netflix/priam/backup/TestAbstractBackup.java deleted file mode 100644 index 882373a48..000000000 --- a/priam/src/test/java/com/netflix/priam/backup/TestAbstractBackup.java +++ /dev/null @@ -1,129 +0,0 @@ -package com.netflix.priam.backup; - -import com.google.common.collect.ImmutableList; -import com.google.common.truth.Truth; -import com.google.common.util.concurrent.ListenableFuture; -import com.google.inject.Guice; -import com.google.inject.Injector; -import com.google.inject.Provider; -import com.netflix.priam.compress.CompressionType; -import com.netflix.priam.config.BackupsToCompress; -import com.netflix.priam.config.FakeConfiguration; -import com.netflix.priam.config.IConfiguration; -import java.io.File; -import java.io.IOException; -import java.nio.file.Paths; -import java.util.Arrays; -import java.util.Collection; -import org.apache.commons.io.FileUtils; -import org.junit.AfterClass; -import org.junit.Before; -import org.junit.BeforeClass; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.junit.runners.Parameterized; - -@RunWith(Parameterized.class) -public class TestAbstractBackup { - private static final String COMPRESSED_DATA = "compressed-1234-Data.db"; - private static final String COMPRESSION_INFO = "compressed-1234-CompressionInfo.db"; - private static final String UNCOMPRESSED_DATA = "uncompressed-1234-Data.db"; - private static final String RANDOM_DATA = "random-1234-Data.db"; - private static final String RANDOM_COMPONENT = "random-1234-compressioninfo.db"; - private static final ImmutableList TABLE_PARTS = - ImmutableList.of( - COMPRESSED_DATA, - COMPRESSION_INFO, - UNCOMPRESSED_DATA, - RANDOM_DATA, - RANDOM_COMPONENT); - - private static final String DIRECTORY = "target/data/ks/cf/backup/"; - private final AbstractBackup abstractBackup; - private final FakeConfiguration fakeConfiguration; - private final String tablePart; - private final CompressionType compressionAlgorithm; - - @BeforeClass - public static void setUp() throws IOException { - FileUtils.forceMkdir(new File(DIRECTORY)); - } - - @Before - public void createFiles() throws IOException { - for (String tablePart : TABLE_PARTS) { - File file = Paths.get(DIRECTORY, tablePart).toFile(); - if (file.createNewFile()) { - FileUtils.forceDeleteOnExit(file); - } else { - throw new IllegalStateException("failed to create " + tablePart); - } - } - } - - @AfterClass - public static void tearDown() throws IOException { - FileUtils.deleteDirectory(new File(DIRECTORY)); - } - - @Parameterized.Parameters - public static Collection data() { - return Arrays.asList( - new Object[][] { - {BackupsToCompress.NONE, COMPRESSED_DATA, CompressionType.NONE}, - {BackupsToCompress.NONE, COMPRESSION_INFO, CompressionType.NONE}, - {BackupsToCompress.NONE, UNCOMPRESSED_DATA, CompressionType.NONE}, - {BackupsToCompress.NONE, RANDOM_DATA, CompressionType.NONE}, - {BackupsToCompress.NONE, RANDOM_COMPONENT, CompressionType.NONE}, - {BackupsToCompress.ALL, COMPRESSED_DATA, CompressionType.SNAPPY}, - {BackupsToCompress.ALL, COMPRESSION_INFO, CompressionType.SNAPPY}, - {BackupsToCompress.ALL, UNCOMPRESSED_DATA, CompressionType.SNAPPY}, - {BackupsToCompress.ALL, RANDOM_DATA, CompressionType.SNAPPY}, - {BackupsToCompress.ALL, RANDOM_COMPONENT, CompressionType.SNAPPY}, - {BackupsToCompress.IF_REQUIRED, COMPRESSED_DATA, CompressionType.NONE}, - {BackupsToCompress.IF_REQUIRED, COMPRESSION_INFO, CompressionType.NONE}, - {BackupsToCompress.IF_REQUIRED, UNCOMPRESSED_DATA, CompressionType.SNAPPY}, - {BackupsToCompress.IF_REQUIRED, RANDOM_DATA, CompressionType.SNAPPY}, - {BackupsToCompress.IF_REQUIRED, RANDOM_COMPONENT, CompressionType.SNAPPY}, - }); - } - - public TestAbstractBackup(BackupsToCompress which, String tablePart, CompressionType algo) { - this.tablePart = tablePart; - this.compressionAlgorithm = algo; - Injector injector = Guice.createInjector(new BRTestModule()); - fakeConfiguration = (FakeConfiguration) injector.getInstance(IConfiguration.class); - fakeConfiguration.setFakeConfig("Priam.backupsToCompress", which); - IFileSystemContext context = injector.getInstance(IFileSystemContext.class); - Provider pathFactory = injector.getProvider(AbstractBackupPath.class); - abstractBackup = - new AbstractBackup(fakeConfiguration, context, pathFactory) { - @Override - protected void processColumnFamily(File dir) {} - - @Override - public void execute() {} - - @Override - public String getName() { - return null; - } - }; - } - - @Test - public void testCorrectCompressionType() throws Exception { - File parent = new File(DIRECTORY); - AbstractBackupPath.BackupFileType backupFileType = AbstractBackupPath.BackupFileType.SST_V2; - ImmutableList> futures = - abstractBackup.uploadAndDeleteAllFiles(parent, backupFileType, false); - AbstractBackupPath abstractBackupPath = null; - for (ListenableFuture future : futures) { - if (future.get().getFileName().equals(tablePart)) { - abstractBackupPath = future.get(); - break; - } - } - Truth.assertThat(abstractBackupPath.getCompression()).isEqualTo(compressionAlgorithm); - } -} diff --git a/priam/src/test/java/com/netflix/priam/backup/TestAbstractFileSystem.java b/priam/src/test/java/com/netflix/priam/backup/TestAbstractFileSystem.java index 47cf1d1fa..449a4a282 100644 --- a/priam/src/test/java/com/netflix/priam/backup/TestAbstractFileSystem.java +++ b/priam/src/test/java/com/netflix/priam/backup/TestAbstractFileSystem.java @@ -83,7 +83,7 @@ public void testFailedRetriesUpload() throws Exception { try { Collection files = generateFiles(1, 1, 1); for (File file : files) { - failureFileSystem.uploadAndDelete(getDummyPath(file.toPath()), 2); + failureFileSystem.uploadAndDelete(getDummyPath(file.toPath()), false /* async */); } } catch (BackupRestoreException e) { // Verify the failure metric for upload is incremented. @@ -123,7 +123,7 @@ public void testFailedRetriesDownload() throws Exception { @Test public void testUpload() throws Exception { File file = generateFiles(1, 1, 1).iterator().next(); - myFileSystem.uploadAndDelete(getDummyPath(file.toPath()), 2); + myFileSystem.uploadAndDelete(getDummyPath(file.toPath()), false /* async */); Assert.assertEquals(1, (int) backupMetrics.getValidUploads().actualCount()); Assert.assertFalse(file.exists()); } @@ -139,7 +139,9 @@ public void testDownload() throws Exception { @Test public void testAsyncUpload() throws Exception { File file = generateFiles(1, 1, 1).iterator().next(); - myFileSystem.asyncUploadAndDelete(getDummyPath(file.toPath()), 2).get(); + myFileSystem + .uploadAndDelete(getDummyPath(file.toPath()), Instant.EPOCH, true /* async */) + .get(); Assert.assertEquals(1, (int) backupMetrics.getValidUploads().actualCount()); Assert.assertEquals(0, myFileSystem.getUploadTasksQueued()); } @@ -151,7 +153,9 @@ public void testAsyncUploadBulk() throws Exception { Collection files = generateFiles(1, 1, 20); List> futures = new ArrayList<>(); for (File file : files) { - futures.add(myFileSystem.asyncUploadAndDelete(getDummyPath(file.toPath()), 2)); + futures.add( + myFileSystem.uploadAndDelete( + getDummyPath(file.toPath()), Instant.EPOCH, true /* async */)); } // Verify all the work is finished. @@ -178,7 +182,7 @@ public void testUploadDedup() throws Exception { for (int i = 0; i < size; i++) { torun.add( () -> { - myFileSystem.uploadAndDelete(abstractBackupPath, 2); + myFileSystem.uploadAndDelete(abstractBackupPath, false /* async */); return Boolean.TRUE; }); } @@ -206,7 +210,8 @@ public void testAsyncUploadFailure() throws Exception { Collection files = generateFiles(1, 1, 1); for (File file : files) { Future future = - failureFileSystem.asyncUploadAndDelete(getDummyPath(file.toPath()), 2); + failureFileSystem.uploadAndDelete( + getDummyPath(file.toPath()), Instant.EPOCH, true /* async */); try { future.get(); } catch (Exception e) { diff --git a/priam/src/test/java/com/netflix/priam/backup/TestBackupHelperImpl.java b/priam/src/test/java/com/netflix/priam/backup/TestBackupHelperImpl.java new file mode 100644 index 000000000..55b110043 --- /dev/null +++ b/priam/src/test/java/com/netflix/priam/backup/TestBackupHelperImpl.java @@ -0,0 +1,181 @@ +package com.netflix.priam.backup; + +import com.google.common.collect.ImmutableList; +import com.google.common.truth.Truth; +import com.google.common.util.concurrent.ListenableFuture; +import com.google.inject.Guice; +import com.google.inject.Injector; +import com.google.inject.Provider; +import com.netflix.priam.compress.CompressionType; +import com.netflix.priam.config.BackupsToCompress; +import com.netflix.priam.config.FakeConfiguration; +import com.netflix.priam.config.IConfiguration; +import java.io.File; +import java.io.IOException; +import java.nio.file.Paths; +import java.util.Arrays; +import java.util.Collection; +import java.util.Objects; +import org.apache.commons.io.FileUtils; +import org.junit.*; +import org.junit.experimental.runners.Enclosed; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; + +@RunWith(Enclosed.class) +public class TestBackupHelperImpl { + private static final String COMPRESSED_DATA = "compressed-1234-Data.db"; + private static final String COMPRESSION_INFO = "compressed-1234-CompressionInfo.db"; + private static final String UNCOMPRESSED_DATA = "uncompressed-1234-Data.db"; + private static final String RANDOM_DATA = "random-1234-Data.db"; + private static final String RANDOM_COMPONENT = "random-1234-compressioninfo.db"; + private static final ImmutableList TABLE_PARTS = + ImmutableList.of( + COMPRESSED_DATA, + COMPRESSION_INFO, + UNCOMPRESSED_DATA, + RANDOM_DATA, + RANDOM_COMPONENT); + + private static final String DIRECTORY = "target/data/ks/cf/backup/"; + + @RunWith(Parameterized.class) + public static class ParameterizedTests { + private final BackupHelperImpl backupHelper; + private final String tablePart; + private final CompressionType compressionAlgorithm; + + @BeforeClass + public static void setUp() throws IOException { + FileUtils.forceMkdir(new File(DIRECTORY)); + } + + @Before + public void createFiles() throws IOException { + for (String tablePart : TABLE_PARTS) { + File file = Paths.get(DIRECTORY, tablePart).toFile(); + if (file.createNewFile()) { + FileUtils.forceDeleteOnExit(file); + } else { + throw new IllegalStateException("failed to create " + tablePart); + } + } + } + + @AfterClass + public static void tearDown() throws IOException { + FileUtils.deleteDirectory(new File(DIRECTORY)); + } + + @Parameterized.Parameters + public static Collection data() { + return Arrays.asList( + new Object[][] { + {BackupsToCompress.NONE, COMPRESSED_DATA, CompressionType.NONE}, + {BackupsToCompress.NONE, COMPRESSION_INFO, CompressionType.NONE}, + {BackupsToCompress.NONE, UNCOMPRESSED_DATA, CompressionType.NONE}, + {BackupsToCompress.NONE, RANDOM_DATA, CompressionType.NONE}, + {BackupsToCompress.NONE, RANDOM_COMPONENT, CompressionType.NONE}, + {BackupsToCompress.ALL, COMPRESSED_DATA, CompressionType.SNAPPY}, + {BackupsToCompress.ALL, COMPRESSION_INFO, CompressionType.SNAPPY}, + {BackupsToCompress.ALL, UNCOMPRESSED_DATA, CompressionType.SNAPPY}, + {BackupsToCompress.ALL, RANDOM_DATA, CompressionType.SNAPPY}, + {BackupsToCompress.ALL, RANDOM_COMPONENT, CompressionType.SNAPPY}, + {BackupsToCompress.IF_REQUIRED, COMPRESSED_DATA, CompressionType.NONE}, + {BackupsToCompress.IF_REQUIRED, COMPRESSION_INFO, CompressionType.NONE}, + {BackupsToCompress.IF_REQUIRED, UNCOMPRESSED_DATA, CompressionType.SNAPPY}, + {BackupsToCompress.IF_REQUIRED, RANDOM_DATA, CompressionType.SNAPPY}, + {BackupsToCompress.IF_REQUIRED, RANDOM_COMPONENT, CompressionType.SNAPPY}, + }); + } + + public ParameterizedTests(BackupsToCompress which, String tablePart, CompressionType algo) { + this.tablePart = tablePart; + this.compressionAlgorithm = algo; + Injector injector = Guice.createInjector(new BRTestModule()); + FakeConfiguration fakeConfiguration = + (FakeConfiguration) injector.getInstance(IConfiguration.class); + fakeConfiguration.setFakeConfig("Priam.backupsToCompress", which); + IFileSystemContext context = injector.getInstance(IFileSystemContext.class); + Provider pathFactory = + injector.getProvider(AbstractBackupPath.class); + backupHelper = new BackupHelperImpl(fakeConfiguration, context, pathFactory); + } + + @Test + public void testCorrectCompressionType() throws Exception { + File parent = new File(DIRECTORY); + AbstractBackupPath.BackupFileType backupFileType = + AbstractBackupPath.BackupFileType.SST_V2; + ImmutableList> futures = + backupHelper.uploadAndDeleteAllFiles(parent, backupFileType, false); + AbstractBackupPath abstractBackupPath = null; + for (ListenableFuture future : futures) { + if (future.get().getFileName().equals(tablePart)) { + abstractBackupPath = future.get(); + break; + } + } + Truth.assertThat(Objects.requireNonNull(abstractBackupPath).getCompression()) + .isEqualTo(compressionAlgorithm); + } + } + + public static class ProgrammaticTests { + private final BackupHelperImpl backupHelper; + + @BeforeClass + public static void setUp() throws IOException { + FileUtils.forceMkdir(new File(DIRECTORY)); + for (String tablePart : TABLE_PARTS) { + File file = Paths.get(DIRECTORY, tablePart).toFile(); + if (file.createNewFile()) { + FileUtils.forceDeleteOnExit(file); + } else { + throw new IllegalStateException("failed to create " + tablePart); + } + } + } + + @AfterClass + public static void tearDown() throws IOException { + FileUtils.deleteDirectory(new File(DIRECTORY)); + } + + public ProgrammaticTests() { + Injector injector = Guice.createInjector(new BRTestModule()); + FakeConfiguration fakeConfiguration = + (FakeConfiguration) injector.getInstance(IConfiguration.class); + IFileSystemContext context = injector.getInstance(IFileSystemContext.class); + Provider pathFactory = + injector.getProvider(AbstractBackupPath.class); + backupHelper = new BackupHelperImpl(fakeConfiguration, context, pathFactory); + } + + @Test + public void testDataFilesAreLast() throws IOException { + AbstractBackupPath.BackupFileType fileType = AbstractBackupPath.BackupFileType.SST_V2; + boolean dataFilesAreLast = + backupHelper + .getBackupPaths(new File(DIRECTORY), fileType) + .asList() + .stream() + .skip(2) + .allMatch(p -> p.getBackupFile().getName().endsWith("-Data.db")); + Truth.assertThat(dataFilesAreLast).isTrue(); + } + + @Test + public void testNonDataFilesComeFirst() throws IOException { + AbstractBackupPath.BackupFileType fileType = AbstractBackupPath.BackupFileType.SST_V2; + boolean nonDataFilesComeFirst = + backupHelper + .getBackupPaths(new File(DIRECTORY), fileType) + .asList() + .stream() + .limit(2) + .noneMatch(p -> p.getBackupFile().getName().endsWith("-Data.db")); + Truth.assertThat(nonDataFilesComeFirst).isTrue(); + } + } +} diff --git a/priam/src/test/java/com/netflix/priam/backup/TestS3FileSystem.java b/priam/src/test/java/com/netflix/priam/backup/TestS3FileSystem.java index 626ff6e51..0a297433b 100644 --- a/priam/src/test/java/com/netflix/priam/backup/TestS3FileSystem.java +++ b/priam/src/test/java/com/netflix/priam/backup/TestS3FileSystem.java @@ -43,6 +43,7 @@ import java.io.IOException; import java.nio.file.Path; import java.nio.file.Paths; +import java.time.Instant; import java.util.ArrayList; import java.util.Arrays; import java.util.List; @@ -92,11 +93,13 @@ public static void cleanup() throws IOException { @Test public void testFileUpload() throws Exception { MockS3PartUploader.setup(); - IBackupFileSystem fs = injector.getInstance(NullBackupFileSystem.class); + AbstractFileSystem fs = injector.getInstance(NullBackupFileSystem.class); RemoteBackupPath backupfile = injector.getInstance(RemoteBackupPath.class); backupfile.parseLocal(localFile(), BackupFileType.SNAP); long noOfFilesUploaded = backupMetrics.getUploadRate().count(); - fs.uploadAndDelete(backupfile, 0); + // temporary hack to allow tests to complete in a timely fashion + // This will be removed once we stop inheriting from AbstractFileSystem + fs.uploadAndDeleteInternal(backupfile, Instant.EPOCH, 0 /* retries */); Assert.assertEquals(1, backupMetrics.getUploadRate().count() - noOfFilesUploaded); } @@ -106,7 +109,7 @@ public void testFileUploadDeleteExists() throws Exception { IBackupFileSystem fs = injector.getInstance(NullBackupFileSystem.class); RemoteBackupPath backupfile = injector.getInstance(RemoteBackupPath.class); backupfile.parseLocal(localFile(), BackupFileType.SST_V2); - fs.uploadAndDelete(backupfile, 0); + fs.uploadAndDelete(backupfile, false /* async */); Assert.assertTrue(fs.checkObjectExists(Paths.get(backupfile.getRemotePath()))); // Lets delete the file now. List deleteFiles = Lists.newArrayList(); @@ -124,7 +127,9 @@ public void testFileUploadFailures() throws Exception { RemoteBackupPath backupfile = injector.getInstance(RemoteBackupPath.class); backupfile.parseLocal(localFile(), BackupFileType.SNAP); try { - fs.uploadAndDelete(backupfile, 0); + // temporary hack to allow tests to complete in a timely fashion + // This will be removed once we stop inheriting from AbstractFileSystem + fs.uploadAndDeleteInternal(backupfile, Instant.EPOCH, 0 /* retries */); } catch (BackupRestoreException e) { // ignore } @@ -141,7 +146,9 @@ public void testFileUploadCompleteFailure() throws Exception { RemoteBackupPath backupfile = injector.getInstance(RemoteBackupPath.class); backupfile.parseLocal(localFile(), BackupFileType.SNAP); try { - fs.uploadAndDelete(backupfile, 0); + // temporary hack to allow tests to complete in a timely fashion + // This will be removed once we stop inheriting from AbstractFileSystem + fs.uploadAndDeleteInternal(backupfile, Instant.EPOCH, 0 /* retries */); } catch (BackupRestoreException e) { // ignore } From 33a5d517af536910d4a6c07dc732dcc5883bbaa5 Mon Sep 17 00:00:00 2001 From: Satyajit Thadeshwar Date: Fri, 7 Oct 2022 16:42:57 -0700 Subject: [PATCH 218/228] Adding ability to add more keys in the message attributes for backup notification messages (#1006) * Adding ability to add more keys in the message attributes for backup notification messages * Formatting as per google java format * Incorporating review comments * Fixing failing tests in TestBackupNotificationMgr --- .../priam/config/BackupRestoreConfig.java | 8 ++++ .../priam/config/IBackupRestoreConfig.java | 13 ++++++ .../notification/BackupNotificationMgr.java | 40 +++++++++++-------- .../priam/config/FakeBackupRestoreConfig.java | 8 ++++ 4 files changed, 52 insertions(+), 17 deletions(-) diff --git a/priam/src/main/java/com/netflix/priam/config/BackupRestoreConfig.java b/priam/src/main/java/com/netflix/priam/config/BackupRestoreConfig.java index 0f1642a18..d9b8fcac9 100644 --- a/priam/src/main/java/com/netflix/priam/config/BackupRestoreConfig.java +++ b/priam/src/main/java/com/netflix/priam/config/BackupRestoreConfig.java @@ -13,7 +13,9 @@ */ package com.netflix.priam.config; +import com.google.common.base.Splitter; import com.netflix.priam.configSource.IConfigSource; +import java.util.List; import javax.inject.Inject; import org.apache.commons.lang3.StringUtils; @@ -61,4 +63,10 @@ public String getBackupVerificationCronExpression() { public String getBackupNotifyComponentIncludeList() { return config.get("priam.backupNotifyComponentIncludeList", StringUtils.EMPTY); } + + @Override + public List getBackupNotificationAdditionalMessageAttrs() { + String value = config.get("priam.backupNotifyAdditionalMessageAttrs", StringUtils.EMPTY); + return Splitter.on(",").omitEmptyStrings().trimResults().splitToList(value); + } } diff --git a/priam/src/main/java/com/netflix/priam/config/IBackupRestoreConfig.java b/priam/src/main/java/com/netflix/priam/config/IBackupRestoreConfig.java index 809791034..2c4cad2fd 100644 --- a/priam/src/main/java/com/netflix/priam/config/IBackupRestoreConfig.java +++ b/priam/src/main/java/com/netflix/priam/config/IBackupRestoreConfig.java @@ -13,7 +13,9 @@ */ package com.netflix.priam.config; +import com.google.common.collect.ImmutableList; import com.google.inject.ImplementedBy; +import java.util.List; import org.apache.commons.lang3.StringUtils; /** @@ -113,4 +115,15 @@ default boolean enableV2Restore() { default String getBackupNotifyComponentIncludeList() { return StringUtils.EMPTY; } + + /** + * Returns ',' separated list of attribute names. If not empty, adds the specified attributes to + * MessageAttributes in the backup notifications. SNS filter policy needs keys in + * MessageAttributes in order to filter based on those keys. + * + * @return + */ + default List getBackupNotificationAdditionalMessageAttrs() { + return ImmutableList.of(); + } } diff --git a/priam/src/main/java/com/netflix/priam/notification/BackupNotificationMgr.java b/priam/src/main/java/com/netflix/priam/notification/BackupNotificationMgr.java index b15a8a341..1e6f0f43c 100644 --- a/priam/src/main/java/com/netflix/priam/notification/BackupNotificationMgr.java +++ b/priam/src/main/java/com/netflix/priam/notification/BackupNotificationMgr.java @@ -73,11 +73,11 @@ public void notify(BackupVerificationResult backupVerificationResult) { jsonObject.put("region", instanceInfo.getRegion()); jsonObject.put("rack", instanceInfo.getRac()); jsonObject.put("token", instanceIdentity.getInstance().getToken()); - jsonObject.put("backuptype", "SNAPSHOT_VERIFIED"); + jsonObject.put( + "backuptype", AbstractBackupPath.BackupFileType.SNAPSHOT_VERIFIED.name()); jsonObject.put("snapshotInstant", backupVerificationResult.snapshotInstant); // SNS Attributes for filtering messages. Cluster name and backup file type. - Map messageAttributes = - getMessageAttributes(AbstractBackupPath.BackupFileType.SNAPSHOT_VERIFIED); + Map messageAttributes = getMessageAttributes(jsonObject); this.notificationService.notify(jsonObject.toString(), messageAttributes); } catch (JSONException exception) { @@ -88,23 +88,28 @@ public void notify(BackupVerificationResult backupVerificationResult) { } } - private Map getMessageAttributes( - AbstractBackupPath.BackupFileType backupFileType) { - // SNS Attributes for filtering messages. Cluster name and backup file type. + private Map getMessageAttributes(JSONObject message) + throws JSONException { + // SNS Attributes for filtering messages Map messageAttributes = new HashMap<>(); - messageAttributes.putIfAbsent( - "s3clustername", - new MessageAttributeValue() - .withDataType("String") - .withStringValue(config.getAppName())); - messageAttributes.putIfAbsent( - "backuptype", - new MessageAttributeValue() - .withDataType("String") - .withStringValue(backupFileType.name())); + // Always include these default attributes + ArrayList attrs = new ArrayList<>(Arrays.asList("s3clustername", "backuptype")); + List additionalAttrs = + this.backupRestoreConfig.getBackupNotificationAdditionalMessageAttrs(); + attrs.addAll(additionalAttrs); + for (String attr : attrs) { + if (message.has(attr)) { + Object value = message.get(attr); + messageAttributes.putIfAbsent(attr, toStringAttribute(String.valueOf(value))); + } + } return messageAttributes; } + private MessageAttributeValue toStringAttribute(String value) { + return new MessageAttributeValue().withDataType("String").withStringValue(value); + } + private void notify(AbstractBackupPath abp, String uploadStatus) { JSONObject jsonObject = new JSONObject(); try { @@ -131,7 +136,8 @@ private void notify(AbstractBackupPath abp, String uploadStatus) { // SNS Attributes for filtering messages. Cluster name and backup file type. Map messageAttributes = - getMessageAttributes(abp.getType()); + getMessageAttributes(jsonObject); + this.notificationService.notify(jsonObject.toString(), messageAttributes); } else { logger.debug( diff --git a/priam/src/test/java/com/netflix/priam/config/FakeBackupRestoreConfig.java b/priam/src/test/java/com/netflix/priam/config/FakeBackupRestoreConfig.java index 99dd0e551..74ed3de5f 100644 --- a/priam/src/test/java/com/netflix/priam/config/FakeBackupRestoreConfig.java +++ b/priam/src/test/java/com/netflix/priam/config/FakeBackupRestoreConfig.java @@ -13,6 +13,9 @@ */ package com.netflix.priam.config; +import com.google.common.collect.ImmutableList; +import java.util.List; + /** Created by aagrawal on 6/26/18. */ public class FakeBackupRestoreConfig implements IBackupRestoreConfig { @Override @@ -34,4 +37,9 @@ public boolean enableV2Restore() { public int getBackupTTLMonitorPeriodInSec() { return 0; // avoids sleeping altogether in tests. } + + @Override + public List getBackupNotificationAdditionalMessageAttrs() { + return ImmutableList.of(); + } } From cfd0cf4aa27130d87787c24db957e7f31a008454 Mon Sep 17 00:00:00 2001 From: mattl-netflix <63665634+mattl-netflix@users.noreply.github.com> Date: Sun, 9 Oct 2022 22:17:34 -0700 Subject: [PATCH 219/228] Use ImmutableSet instead of ImmutableList to store additional message attributes. (#1010) * Use an ImmutableSet rather than an ImmutableList of additional message attributes when sending SNS backup notifications. * Use ImmutableSet rather than ImmutableList to store additional message attributes. --- .../priam/config/BackupRestoreConfig.java | 6 +++--- .../priam/config/IBackupRestoreConfig.java | 14 ++++++-------- .../notification/BackupNotificationMgr.java | 17 ++++++----------- .../priam/config/FakeBackupRestoreConfig.java | 7 +++---- 4 files changed, 18 insertions(+), 26 deletions(-) diff --git a/priam/src/main/java/com/netflix/priam/config/BackupRestoreConfig.java b/priam/src/main/java/com/netflix/priam/config/BackupRestoreConfig.java index d9b8fcac9..c1d320029 100644 --- a/priam/src/main/java/com/netflix/priam/config/BackupRestoreConfig.java +++ b/priam/src/main/java/com/netflix/priam/config/BackupRestoreConfig.java @@ -14,8 +14,8 @@ package com.netflix.priam.config; import com.google.common.base.Splitter; +import com.google.common.collect.ImmutableSet; import com.netflix.priam.configSource.IConfigSource; -import java.util.List; import javax.inject.Inject; import org.apache.commons.lang3.StringUtils; @@ -65,8 +65,8 @@ public String getBackupNotifyComponentIncludeList() { } @Override - public List getBackupNotificationAdditionalMessageAttrs() { + public ImmutableSet getBackupNotificationAdditionalMessageAttrs() { String value = config.get("priam.backupNotifyAdditionalMessageAttrs", StringUtils.EMPTY); - return Splitter.on(",").omitEmptyStrings().trimResults().splitToList(value); + return ImmutableSet.copyOf(Splitter.on(",").omitEmptyStrings().trimResults().split(value)); } } diff --git a/priam/src/main/java/com/netflix/priam/config/IBackupRestoreConfig.java b/priam/src/main/java/com/netflix/priam/config/IBackupRestoreConfig.java index 2c4cad2fd..5ded92836 100644 --- a/priam/src/main/java/com/netflix/priam/config/IBackupRestoreConfig.java +++ b/priam/src/main/java/com/netflix/priam/config/IBackupRestoreConfig.java @@ -13,9 +13,8 @@ */ package com.netflix.priam.config; -import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; import com.google.inject.ImplementedBy; -import java.util.List; import org.apache.commons.lang3.StringUtils; /** @@ -117,13 +116,12 @@ default String getBackupNotifyComponentIncludeList() { } /** - * Returns ',' separated list of attribute names. If not empty, adds the specified attributes to - * MessageAttributes in the backup notifications. SNS filter policy needs keys in - * MessageAttributes in order to filter based on those keys. + * Returns a set of attribute names to add to MessageAttributes in the backup notifications. SNS + * filter policy needs keys in MessageAttributes in order to filter based on those keys. * - * @return + * @return A set of attributes to include in MessageAttributes. */ - default List getBackupNotificationAdditionalMessageAttrs() { - return ImmutableList.of(); + default ImmutableSet getBackupNotificationAdditionalMessageAttrs() { + return ImmutableSet.of(); } } diff --git a/priam/src/main/java/com/netflix/priam/notification/BackupNotificationMgr.java b/priam/src/main/java/com/netflix/priam/notification/BackupNotificationMgr.java index 1e6f0f43c..2bef31e0c 100644 --- a/priam/src/main/java/com/netflix/priam/notification/BackupNotificationMgr.java +++ b/priam/src/main/java/com/netflix/priam/notification/BackupNotificationMgr.java @@ -90,20 +90,15 @@ public void notify(BackupVerificationResult backupVerificationResult) { private Map getMessageAttributes(JSONObject message) throws JSONException { - // SNS Attributes for filtering messages - Map messageAttributes = new HashMap<>(); - // Always include these default attributes - ArrayList attrs = new ArrayList<>(Arrays.asList("s3clustername", "backuptype")); - List additionalAttrs = - this.backupRestoreConfig.getBackupNotificationAdditionalMessageAttrs(); - attrs.addAll(additionalAttrs); - for (String attr : attrs) { + Map attributes = new HashMap<>(); + attributes.put("s3clustername", toStringAttribute(message.getString("s3clustername"))); + attributes.put("backuptype", toStringAttribute(message.getString("backuptype"))); + for (String attr : backupRestoreConfig.getBackupNotificationAdditionalMessageAttrs()) { if (message.has(attr)) { - Object value = message.get(attr); - messageAttributes.putIfAbsent(attr, toStringAttribute(String.valueOf(value))); + attributes.put(attr, toStringAttribute(String.valueOf(message.get(attr)))); } } - return messageAttributes; + return attributes; } private MessageAttributeValue toStringAttribute(String value) { diff --git a/priam/src/test/java/com/netflix/priam/config/FakeBackupRestoreConfig.java b/priam/src/test/java/com/netflix/priam/config/FakeBackupRestoreConfig.java index 74ed3de5f..e78987409 100644 --- a/priam/src/test/java/com/netflix/priam/config/FakeBackupRestoreConfig.java +++ b/priam/src/test/java/com/netflix/priam/config/FakeBackupRestoreConfig.java @@ -13,8 +13,7 @@ */ package com.netflix.priam.config; -import com.google.common.collect.ImmutableList; -import java.util.List; +import com.google.common.collect.ImmutableSet; /** Created by aagrawal on 6/26/18. */ public class FakeBackupRestoreConfig implements IBackupRestoreConfig { @@ -39,7 +38,7 @@ public int getBackupTTLMonitorPeriodInSec() { } @Override - public List getBackupNotificationAdditionalMessageAttrs() { - return ImmutableList.of(); + public ImmutableSet getBackupNotificationAdditionalMessageAttrs() { + return ImmutableSet.of(); } } From e589739046e98014438473e2e21baf9d0e93cd14 Mon Sep 17 00:00:00 2001 From: Matt Lehman Date: Mon, 10 Oct 2022 22:09:41 -0700 Subject: [PATCH 220/228] Update CHANGELOG in advance of 3.11.93 --- CHANGELOG.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index dc915d76d..d1f4d44f7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,9 @@ # Changelog +## 2022/10/10 3.11.93 +(#1006) Adding ability to add more keys in the message attributes for backup notification messages. +(#999) Identify incrementals, optionally skip metafile compression, and upload data files last. +(#1009) Pass millis to sleep instead of seconds. + ## 2022/09/15 3.11.92 (#1005) RandomPartitioner creates tokens that don't always fit into a long so we use BigInteger to store and compare them. From ad9450755204ad0ccdd60febcf0c0c35ad674dd4 Mon Sep 17 00:00:00 2001 From: mattl-netflix <63665634+mattl-netflix@users.noreply.github.com> Date: Sun, 13 Nov 2022 23:04:34 -0800 Subject: [PATCH 221/228] Ensure SI files get into meta file properly. (#1012) --- .../main/java/com/netflix/priam/backupv2/SnapshotMetaTask.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/priam/src/main/java/com/netflix/priam/backupv2/SnapshotMetaTask.java b/priam/src/main/java/com/netflix/priam/backupv2/SnapshotMetaTask.java index 065b8a6fe..2192caf01 100644 --- a/priam/src/main/java/com/netflix/priam/backupv2/SnapshotMetaTask.java +++ b/priam/src/main/java/com/netflix/priam/backupv2/SnapshotMetaTask.java @@ -368,7 +368,7 @@ private Optional generateMetaFile( builder.putAll(getSSTables(snapshotDir, AbstractBackupPath.BackupFileType.SST_V2)); // Next, add secondary indexes - for (File directory : getSecondaryIndexDirectories(backupDir)) { + for (File directory : getSecondaryIndexDirectories(snapshotDir)) { builder.putAll( getSSTables(directory, AbstractBackupPath.BackupFileType.SECONDARY_INDEX_V2)); } From a923510b03870f95787ede30412a87c47ebb80a9 Mon Sep 17 00:00:00 2001 From: mattl-netflix <63665634+mattl-netflix@users.noreply.github.com> Date: Tue, 15 Nov 2022 22:18:03 -0800 Subject: [PATCH 222/228] Create operator-specifiable time such that if a backup file was written before then it is automatically compressed using SNAPPY before upload. (#1013) --- .../priam/backup/BackupHelperImpl.java | 4 ++- .../netflix/priam/config/IConfiguration.java | 10 ++++++ .../priam/backup/TestBackupHelperImpl.java | 36 +++++++++++++++++-- .../priam/config/FakeConfiguration.java | 10 ++++++ 4 files changed, 56 insertions(+), 4 deletions(-) diff --git a/priam/src/main/java/com/netflix/priam/backup/BackupHelperImpl.java b/priam/src/main/java/com/netflix/priam/backup/BackupHelperImpl.java index e03b8001b..c51de5e0a 100644 --- a/priam/src/main/java/com/netflix/priam/backup/BackupHelperImpl.java +++ b/priam/src/main/java/com/netflix/priam/backup/BackupHelperImpl.java @@ -88,7 +88,9 @@ public ImmutableSet getBackupPaths( private CompressionType getCorrectCompressionAlgorithm( AbstractBackupPath path, Set compressedFiles) { - if (!AbstractBackupPath.BackupFileType.isV2(path.getType())) { + if (!AbstractBackupPath.BackupFileType.isV2(path.getType()) + || path.getLastModified().toEpochMilli() + < config.getCompressionTransitionEpochMillis()) { return CompressionType.SNAPPY; } String file = path.getFileName(); diff --git a/priam/src/main/java/com/netflix/priam/config/IConfiguration.java b/priam/src/main/java/com/netflix/priam/config/IConfiguration.java index e5365d410..6bad0aedc 100644 --- a/priam/src/main/java/com/netflix/priam/config/IConfiguration.java +++ b/priam/src/main/java/com/netflix/priam/config/IConfiguration.java @@ -1165,6 +1165,16 @@ default boolean addMD5ToBackupUploads() { return false; } + /** + * If a backup file's last-modified time is before this time, revert to SNAPPY compression. + * Otherwise, choose compression using the default logic based on getBackupsToCompress(). + * + * @return the milliseconds since the epoch of the transition time. + */ + default long getCompressionTransitionEpochMillis() { + return 0L; + } + /** * Escape hatch for getting any arbitrary property by key This is useful so we don't have to * keep adding methods to this interface for every single configuration option ever. Also diff --git a/priam/src/test/java/com/netflix/priam/backup/TestBackupHelperImpl.java b/priam/src/test/java/com/netflix/priam/backup/TestBackupHelperImpl.java index 55b110043..224633efd 100644 --- a/priam/src/test/java/com/netflix/priam/backup/TestBackupHelperImpl.java +++ b/priam/src/test/java/com/netflix/priam/backup/TestBackupHelperImpl.java @@ -13,6 +13,8 @@ import java.io.File; import java.io.IOException; import java.nio.file.Paths; +import java.time.Instant; +import java.time.temporal.ChronoUnit; import java.util.Arrays; import java.util.Collection; import java.util.Objects; @@ -123,6 +125,7 @@ public void testCorrectCompressionType() throws Exception { public static class ProgrammaticTests { private final BackupHelperImpl backupHelper; + private final FakeConfiguration config; @BeforeClass public static void setUp() throws IOException { @@ -144,12 +147,11 @@ public static void tearDown() throws IOException { public ProgrammaticTests() { Injector injector = Guice.createInjector(new BRTestModule()); - FakeConfiguration fakeConfiguration = - (FakeConfiguration) injector.getInstance(IConfiguration.class); + config = (FakeConfiguration) injector.getInstance(IConfiguration.class); IFileSystemContext context = injector.getInstance(IFileSystemContext.class); Provider pathFactory = injector.getProvider(AbstractBackupPath.class); - backupHelper = new BackupHelperImpl(fakeConfiguration, context, pathFactory); + backupHelper = new BackupHelperImpl(config, context, pathFactory); } @Test @@ -177,5 +179,33 @@ public void testNonDataFilesComeFirst() throws IOException { .noneMatch(p -> p.getBackupFile().getName().endsWith("-Data.db")); Truth.assertThat(nonDataFilesComeFirst).isTrue(); } + + @Test + public void testNeverCompressedOldFilesAreCompressed() throws IOException { + AbstractBackupPath.BackupFileType fileType = AbstractBackupPath.BackupFileType.SST_V2; + long transitionInstant = Instant.now().plus(1, ChronoUnit.DAYS).toEpochMilli(); + config.setCompressionTransitionEpochMillis(transitionInstant); + config.setFakeConfig("Priam.backupsToCompress", BackupsToCompress.NONE); + boolean backupsAreCompressed = + backupHelper + .getBackupPaths(new File(DIRECTORY), fileType) + .stream() + .allMatch(p -> p.getCompression() == CompressionType.SNAPPY); + Truth.assertThat(backupsAreCompressed).isTrue(); + } + + @Test + public void testOptionallyCompressedOldFilesAreCompressed() throws IOException { + AbstractBackupPath.BackupFileType fileType = AbstractBackupPath.BackupFileType.SST_V2; + long transitionInstant = Instant.now().plus(1, ChronoUnit.DAYS).toEpochMilli(); + config.setCompressionTransitionEpochMillis(transitionInstant); + config.setFakeConfig("Priam.backupsToCompress", BackupsToCompress.IF_REQUIRED); + boolean backupsAreCompressed = + backupHelper + .getBackupPaths(new File(DIRECTORY), fileType) + .stream() + .allMatch(p -> p.getCompression() == CompressionType.SNAPPY); + Truth.assertThat(backupsAreCompressed).isTrue(); + } } } diff --git a/priam/src/test/java/com/netflix/priam/config/FakeConfiguration.java b/priam/src/test/java/com/netflix/priam/config/FakeConfiguration.java index 046b367f5..e839ad56a 100644 --- a/priam/src/test/java/com/netflix/priam/config/FakeConfiguration.java +++ b/priam/src/test/java/com/netflix/priam/config/FakeConfiguration.java @@ -40,6 +40,7 @@ public class FakeConfiguration implements IConfiguration { private boolean skipDeletingOthersIngressRules; private boolean skipUpdatingOthersIngressRules; private boolean skipIngressUnlessIPIsPublic; + private long compressionTransitionEpochMillis; public Map fakeProperties = new HashMap<>(); @@ -292,4 +293,13 @@ public int getBackupThreads() { fakeConfig.getOrDefault( "Priam.backup.threads", IConfiguration.super.getBackupThreads()); } + + public void setCompressionTransitionEpochMillis(long transitionTime) { + compressionTransitionEpochMillis = transitionTime; + } + + @Override + public long getCompressionTransitionEpochMillis() { + return compressionTransitionEpochMillis; + } } From a85197a9ff6a503bd634242999fc4a913d545541 Mon Sep 17 00:00:00 2001 From: Matt Lehman Date: Tue, 15 Nov 2022 22:19:27 -0800 Subject: [PATCH 223/228] Update CHANGELOG in advance of 3.11.94 --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index d1f4d44f7..7be01db77 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,8 @@ # Changelog +## 2022/11/15 3.11.94 +(#1013) Create operator-specifiable time such that if a backup file was written before then it is automatically compressed using SNAPPY before upload. +(#1012) Ensure SI files get into meta file properly. + ## 2022/10/10 3.11.93 (#1006) Adding ability to add more keys in the message attributes for backup notification messages. (#999) Identify incrementals, optionally skip metafile compression, and upload data files last. From 8a29ce09a0bb59165e289b7cc321a8b26a78c8a1 Mon Sep 17 00:00:00 2001 From: ayushis Date: Mon, 19 Dec 2022 15:13:57 -0800 Subject: [PATCH 224/228] reply only on replace_address_first_boot for replacement --- .../netflix/priam/cassandra/extensions/PriamStartupAgent.java | 4 ---- 1 file changed, 4 deletions(-) diff --git a/priam-cass-extensions/src/main/java/com/netflix/priam/cassandra/extensions/PriamStartupAgent.java b/priam-cass-extensions/src/main/java/com/netflix/priam/cassandra/extensions/PriamStartupAgent.java index a875b5a79..92591b010 100644 --- a/priam-cass-extensions/src/main/java/com/netflix/priam/cassandra/extensions/PriamStartupAgent.java +++ b/priam-cass-extensions/src/main/java/com/netflix/priam/cassandra/extensions/PriamStartupAgent.java @@ -84,11 +84,7 @@ private void setPriamProperties() { if (isReplace) { System.out.println( "Detect cassandra version : " + FBUtilities.getReleaseVersionString()); - if (FBUtilities.getReleaseVersionString().compareTo(REPLACED_ADDRESS_MIN_VER) < 0) { - System.setProperty("cassandra.replace_token", token); - } else { System.setProperty("cassandra.replace_address_first_boot", replacedIp); - } } } From 73d794befbfb2cee15a11dbeac1fa084389c3714 Mon Sep 17 00:00:00 2001 From: ayushis Date: Mon, 19 Dec 2022 15:16:17 -0800 Subject: [PATCH 225/228] reply only on replace_address_first_boot for replacement --- .../netflix/priam/cassandra/extensions/PriamStartupAgent.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/priam-cass-extensions/src/main/java/com/netflix/priam/cassandra/extensions/PriamStartupAgent.java b/priam-cass-extensions/src/main/java/com/netflix/priam/cassandra/extensions/PriamStartupAgent.java index 92591b010..8a935a097 100644 --- a/priam-cass-extensions/src/main/java/com/netflix/priam/cassandra/extensions/PriamStartupAgent.java +++ b/priam-cass-extensions/src/main/java/com/netflix/priam/cassandra/extensions/PriamStartupAgent.java @@ -84,7 +84,7 @@ private void setPriamProperties() { if (isReplace) { System.out.println( "Detect cassandra version : " + FBUtilities.getReleaseVersionString()); - System.setProperty("cassandra.replace_address_first_boot", replacedIp); + System.setProperty("cassandra.replace_address_first_boot", replacedIp); } } From 0736325342a88371c54611f8fc23e5703540fb17 Mon Sep 17 00:00:00 2001 From: ayushis Date: Wed, 21 Dec 2022 12:35:49 -0800 Subject: [PATCH 226/228] cassandra-all version changed to 4.1.0 --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 677a82ad1..bbaa3da3f 100644 --- a/build.gradle +++ b/build.gradle @@ -55,7 +55,7 @@ allprojects { compile 'com.googlecode.json-simple:json-simple:1.1.1' compile 'org.xerial.snappy:snappy-java:1.1.7.3' compile 'org.yaml:snakeyaml:1.25' - compile 'org.apache.cassandra:cassandra-all:4.0.0' + compile 'org.apache.cassandra:cassandra-all:4.1.0' compile 'javax.ws.rs:jsr311-api:1.1.1' compile 'joda-time:joda-time:2.10.1' compile 'org.apache.commons:commons-configuration2:2.4' From d51660ed8cdd912fe82b3a805d04a1a565d93a57 Mon Sep 17 00:00:00 2001 From: ayushis Date: Mon, 9 Jan 2023 16:11:51 -0800 Subject: [PATCH 227/228] changes for 4.x --- .../com/netflix/priam/config/IConfiguration.java | 13 ++++--------- .../netflix/priam/config/PriamConfiguration.java | 9 ++------- .../com/netflix/priam/tuner/StandardTuner.java | 2 +- .../netflix/priam/config/FakeConfiguration.java | 15 +++++---------- .../configSource/PropertiesConfigSourceTest.java | 5 ----- priam/src/test/resources/conf/Priam.properties | 1 - priam/src/test/resources/conf/jvm-server.options | 3 --- 7 files changed, 12 insertions(+), 36 deletions(-) diff --git a/priam/src/main/java/com/netflix/priam/config/IConfiguration.java b/priam/src/main/java/com/netflix/priam/config/IConfiguration.java index a55658f56..db46690e2 100644 --- a/priam/src/main/java/com/netflix/priam/config/IConfiguration.java +++ b/priam/src/main/java/com/netflix/priam/config/IConfiguration.java @@ -46,7 +46,7 @@ default String getYamlLocation() { /** @return Path to jvm.options file. This is used to pass JVM options to Cassandra. */ default String getJVMOptionsFileLocation() { - return getCassHome() + "/conf/jvm.options"; + return getCassHome() + "/conf/jvm-server.options"; } /** @@ -853,9 +853,9 @@ default int getTombstoneFailureThreshold() { return 100000; } - /** @return streaming_socket_timeout_in_ms in C* yaml */ - default int getStreamingSocketTimeoutInMS() { - return 86400000; + /** @return streaming_keep_alive_period in seconds in C* yaml */ + default int getStreamingKeepAlivePeriod() { + return 300; } /** @@ -1082,11 +1082,6 @@ default boolean usePrivateIP() { return getSnitch().equals("org.apache.cassandra.locator.GossipingPropertyFileSnitch"); } - /** @return true to check is thrift listening on the rpc_port. */ - default boolean checkThriftServerIsListening() { - return false; - } - /** * @return BackupsToCompress UNCOMPRESSED means compress backups only when the files are not * already compressed by Cassandra diff --git a/priam/src/main/java/com/netflix/priam/config/PriamConfiguration.java b/priam/src/main/java/com/netflix/priam/config/PriamConfiguration.java index fc705958d..ebde99fe1 100644 --- a/priam/src/main/java/com/netflix/priam/config/PriamConfiguration.java +++ b/priam/src/main/java/com/netflix/priam/config/PriamConfiguration.java @@ -653,8 +653,8 @@ public int getTombstoneFailureThreshold() { } @Override - public int getStreamingSocketTimeoutInMS() { - return config.get(PRIAM_PRE + ".streaming.socket.timeout.ms", 86400000); + public int getStreamingKeepAlivePeriod() { + return config.get(PRIAM_PRE + ".streaming.socket.keepalive.s", 300); } @Override @@ -760,11 +760,6 @@ public String getDiskAccessMode() { return config.get(PRIAM_PRE + ".diskAccessMode", "auto"); } - @Override - public boolean checkThriftServerIsListening() { - return config.get(PRIAM_PRE + ".checkThriftServerIsListening", false); - } - @Override public boolean permitDirectTokenAssignmentWithGossipMismatch() { return config.get(PRIAM_PRE + ".permitDirectTokenAssignmentWithGossipMismatch", false); diff --git a/priam/src/main/java/com/netflix/priam/tuner/StandardTuner.java b/priam/src/main/java/com/netflix/priam/tuner/StandardTuner.java index c1eea3459..41482d509 100644 --- a/priam/src/main/java/com/netflix/priam/tuner/StandardTuner.java +++ b/priam/src/main/java/com/netflix/priam/tuner/StandardTuner.java @@ -117,7 +117,7 @@ public void writeAllProperties(String yamlLocation, String hostname, String seed map.put("tombstone_warn_threshold", config.getTombstoneWarnThreshold()); map.put("tombstone_failure_threshold", config.getTombstoneFailureThreshold()); - // map.put("streaming_socket_timeout_in_ms", config.getStreamingSocketTimeoutInMS()); + map.put("streaming_keep_alive_period", config.getStreamingKeepAlivePeriod() + "s"); map.put("memtable_cleanup_threshold", config.getMemtableCleanupThreshold()); map.put( diff --git a/priam/src/test/java/com/netflix/priam/config/FakeConfiguration.java b/priam/src/test/java/com/netflix/priam/config/FakeConfiguration.java index 336442a18..6f855b086 100644 --- a/priam/src/test/java/com/netflix/priam/config/FakeConfiguration.java +++ b/priam/src/test/java/com/netflix/priam/config/FakeConfiguration.java @@ -36,7 +36,6 @@ public class FakeConfiguration implements IConfiguration { private ImmutableList racs; private boolean usePrivateIp; private String diskAccessMode; - private boolean checkThriftIsListening; private boolean skipDeletingOthersIngressRules; private boolean skipUpdatingOthersIngressRules; private boolean skipIngressUnlessIPIsPublic; @@ -269,15 +268,6 @@ public FakeConfiguration setDiskAccessMode(String diskAccessMode) { return this; } - @Override - public boolean checkThriftServerIsListening() { - return checkThriftIsListening; - } - - public void setCheckThriftServerIsListening(boolean checkThriftServerIsListening) { - this.checkThriftIsListening = checkThriftServerIsListening; - } - @Override public boolean skipIngressUnlessIPIsPublic() { return this.skipIngressUnlessIPIsPublic; @@ -302,4 +292,9 @@ public void setCompressionTransitionEpochMillis(long transitionTime) { public long getCompressionTransitionEpochMillis() { return compressionTransitionEpochMillis; } + + @Override + public int getStreamingKeepAlivePeriod() { + return 300; + } } diff --git a/priam/src/test/java/com/netflix/priam/configSource/PropertiesConfigSourceTest.java b/priam/src/test/java/com/netflix/priam/configSource/PropertiesConfigSourceTest.java index 194fc841e..3cc6cdfec 100644 --- a/priam/src/test/java/com/netflix/priam/configSource/PropertiesConfigSourceTest.java +++ b/priam/src/test/java/com/netflix/priam/configSource/PropertiesConfigSourceTest.java @@ -33,7 +33,6 @@ public void readFile() { Assert.assertEquals( "\"/tmp/commitlog\"", configSource.get("Priam.backup.commitlog.location")); - Assert.assertEquals(7102, configSource.get("Priam.thrift.port", 0)); // File has 13 lines, but line 6 is "Priam.jmx.port7501", so it gets filtered out with empty // string check. Assert.assertEquals(12, configSource.size()); @@ -53,9 +52,5 @@ public void updateKey() { Assert.assertEquals(13, configSource.size()); Assert.assertEquals("bar", configSource.get("foo")); - - Assert.assertEquals(7102, configSource.get("Priam.thrift.port", 0)); - configSource.set("Priam.thrift.port", Integer.toString(10)); - Assert.assertEquals(10, configSource.get("Priam.thrift.port", 0)); } } diff --git a/priam/src/test/resources/conf/Priam.properties b/priam/src/test/resources/conf/Priam.properties index 324b5ec99..ec6bd81bc 100644 --- a/priam/src/test/resources/conf/Priam.properties +++ b/priam/src/test/resources/conf/Priam.properties @@ -8,6 +8,5 @@ Priam.restore.threads=5 Priam.zones.available="us-east-1a,us-east1c" Priam.backup.incremental.enable=true Priam.backup.commitlog.enable=true -Priam.thrift.port=7102 Priam.backup.commitlog.location="/tmp/commitlog" Priam.snapshot.meta.cron="0 0/2 * 1/1 * ? *" \ No newline at end of file diff --git a/priam/src/test/resources/conf/jvm-server.options b/priam/src/test/resources/conf/jvm-server.options index bd61450c1..03378b1d3 100644 --- a/priam/src/test/resources/conf/jvm-server.options +++ b/priam/src/test/resources/conf/jvm-server.options @@ -53,9 +53,6 @@ # before joining the ring. #-Dcassandra.ring_delay_ms=ms -# Set the port for the Thrift RPC service, which is used for client connections. (Default: 9160) -#-Dcassandra.rpc_port=port - # Set the SSL port for encrypted communication. (Default: 7001) #-Dcassandra.ssl_storage_port=port From e6730bdc3508e55447e73821e5401b56ed6f6dc8 Mon Sep 17 00:00:00 2001 From: ayushis Date: Thu, 12 Jan 2023 15:31:56 -0800 Subject: [PATCH 228/228] broken tests fixed --- .../priam/backup/TestBackupDynamicRateLimiter.java | 11 ++++++----- .../configSource/PropertiesConfigSourceTest.java | 8 ++++---- .../com/netflix/priam/scheduler/TestSimpleTimer.java | 10 +++++----- 3 files changed, 15 insertions(+), 14 deletions(-) diff --git a/priam/src/test/java/com/netflix/priam/backup/TestBackupDynamicRateLimiter.java b/priam/src/test/java/com/netflix/priam/backup/TestBackupDynamicRateLimiter.java index 1c607ba7d..cc754b58c 100644 --- a/priam/src/test/java/com/netflix/priam/backup/TestBackupDynamicRateLimiter.java +++ b/priam/src/test/java/com/netflix/priam/backup/TestBackupDynamicRateLimiter.java @@ -1,5 +1,7 @@ package com.netflix.priam.backup; +import static org.junit.jupiter.api.Assertions.assertThrows; + import com.google.common.base.Stopwatch; import com.google.common.collect.ImmutableMap; import com.google.common.truth.Truth; @@ -14,7 +16,6 @@ import java.time.ZoneId; import java.util.Map; import java.util.concurrent.TimeUnit; -import org.junit.Assert; import org.junit.Before; import org.junit.Test; @@ -76,7 +77,7 @@ public void targetIsInThePast() { @Test public void noBackupThreads() { rateLimiter = getRateLimiter(ImmutableMap.of("Priam.backup.threads", 0), NOW, DIR_SIZE); - Assert.assertThrows( + assertThrows( IllegalStateException.class, () -> timePermitAcquisition(getBackupPath(), LATER, 20)); } @@ -84,7 +85,7 @@ public void noBackupThreads() { @Test public void negativeBackupThreads() { rateLimiter = getRateLimiter(ImmutableMap.of("Priam.backup.threads", -1), NOW, DIR_SIZE); - Assert.assertThrows( + assertThrows( IllegalStateException.class, () -> timePermitAcquisition(getBackupPath(), LATER, 20)); } @@ -99,7 +100,7 @@ public void noData() { @Test public void noPermitsRequested() { rateLimiter = getRateLimiter(ImmutableMap.of("Priam.backup.threads", 1), NOW, DIR_SIZE); - Assert.assertThrows( + assertThrows( IllegalArgumentException.class, () -> timePermitAcquisition(getBackupPath(), LATER, 0)); } @@ -107,7 +108,7 @@ public void noPermitsRequested() { @Test public void negativePermitsRequested() { rateLimiter = getRateLimiter(ImmutableMap.of("Priam.backup.threads", 1), NOW, DIR_SIZE); - Assert.assertThrows( + assertThrows( IllegalArgumentException.class, () -> timePermitAcquisition(getBackupPath(), LATER, -1)); } diff --git a/priam/src/test/java/com/netflix/priam/configSource/PropertiesConfigSourceTest.java b/priam/src/test/java/com/netflix/priam/configSource/PropertiesConfigSourceTest.java index 3cc6cdfec..d78b26ca9 100644 --- a/priam/src/test/java/com/netflix/priam/configSource/PropertiesConfigSourceTest.java +++ b/priam/src/test/java/com/netflix/priam/configSource/PropertiesConfigSourceTest.java @@ -35,7 +35,7 @@ public void readFile() { "\"/tmp/commitlog\"", configSource.get("Priam.backup.commitlog.location")); // File has 13 lines, but line 6 is "Priam.jmx.port7501", so it gets filtered out with empty // string check. - Assert.assertEquals(12, configSource.size()); + Assert.assertEquals(11, configSource.size()); } @Test @@ -43,13 +43,13 @@ public void updateKey() { PropertiesConfigSource configSource = new PropertiesConfigSource("conf/Priam.properties"); configSource.initialize("asgName", "region"); - // File has 13 lines, but line 6 is "Priam.jmx.port7501", so it gets filtered out with empty + // File has 12 lines, but line 6 is "Priam.jmx.port7501", so it gets filtered out with empty // string check. - Assert.assertEquals(12, configSource.size()); + Assert.assertEquals(11, configSource.size()); configSource.set("foo", "bar"); - Assert.assertEquals(13, configSource.size()); + Assert.assertEquals(12, configSource.size()); Assert.assertEquals("bar", configSource.get("foo")); } diff --git a/priam/src/test/java/com/netflix/priam/scheduler/TestSimpleTimer.java b/priam/src/test/java/com/netflix/priam/scheduler/TestSimpleTimer.java index 8aa3ddcc5..648a483ca 100644 --- a/priam/src/test/java/com/netflix/priam/scheduler/TestSimpleTimer.java +++ b/priam/src/test/java/com/netflix/priam/scheduler/TestSimpleTimer.java @@ -1,11 +1,12 @@ package com.netflix.priam.scheduler; +import static org.junit.jupiter.api.Assertions.assertThrows; + import com.google.common.truth.Truth; import java.text.ParseException; import java.time.Instant; import java.time.temporal.ChronoUnit; import java.util.Date; -import org.junit.Assert; import org.junit.Test; import org.quartz.Trigger; @@ -20,7 +21,7 @@ public void sunnyDay() throws ParseException { @Test public void startBeforeEpoch() { - Assert.assertThrows( + assertThrows( IllegalArgumentException.class, () -> new SimpleTimer("foo", PERIOD, Instant.EPOCH.minus(5, ChronoUnit.SECONDS))); } @@ -38,13 +39,12 @@ public void startMoreThanOnePeriodAfterEpoch() throws ParseException { @Test public void negativePeriod() { - Assert.assertThrows( - IllegalArgumentException.class, () -> new SimpleTimer("foo", -PERIOD, START)); + assertThrows(IllegalArgumentException.class, () -> new SimpleTimer("foo", -PERIOD, START)); } @Test public void zeroPeriod() { - Assert.assertThrows(IllegalArgumentException.class, () -> new SimpleTimer("foo", 0, START)); + assertThrows(IllegalArgumentException.class, () -> new SimpleTimer("foo", 0, START)); } private void assertions(Trigger trigger, Instant start) {

!84drmts00mGky;c0^(T{kVQsROt##O+K0LqrIyT~p3Jq0y2ohNWQ-?kn!+!) zGx*(n9>1P0LF_6rZ1tv;JB$jG4`MYN#{b%7pm?L+oB0CyQoC^XQf5y%;b zoGHkeNiQr!`YN*iEm-4L{=c8zd4c@>D{@H#q2BzKR<#b}Ph_gLUj(lJgQO3UX%C8%r39i-8_7u^ca@x`jIn~Iiq2&XS zH56GRkTn50Gmx>Ey_?Z$H<#KIc!cxFGSC-%7mI?{YV7KZNX^e$1}}nWBLC0Zi3=nNy+9Fc+@r=+ei^Lb{vTV{KPj z>y15wkQXk6X0gIWr4w(Y>_D}sQ@unwkCAYfnG)){T!LM1 zkwCX&;_vpP__<#YfA{Yt!2K^d%2aqC^0r2^9x0AIC!QRG?W7pyw&*jMbA7Q(r8o5l z^v&*MC21X;B;^KoN$Th)37tYD&N)hAToNVPHA5oZ@+E>=OqhGMgm?^+AkR?};6+q< zFBKo}ZQ|u~R6KlGjpy^ec=&!Np44HE&{pI9ND25t3ckQpV+PytNld-bXCS)t!Y&oT zq=xiYx|bO>Bs+<9cabO$FNyF7kZ{j%3H6MV5U*4Tq81bAQ!4(xJ;c|qR=oX(i)X-8 zaSymjTm!d=bKo7)G3YVr5cDqF%i5CXuxf*()=cB4-hgU|c%$TaFDtmmb)q zEQ)zHw5dVVAN;H&%-2bRecdIH`;-FwLd4HMT6_bz{~<6-yaJ2FBdD9W2lM=ekYVB+ zIz>8!Ef$9`D!<`}#Wwt5u?c^J?N@yMT@KL)JCgCoO#Cqyf9y<5f-&qjFzxA&Uft29 z6kT%AB^g~J!_28C*@R!aSP(qplMzcR8pA?bfRXL>7 z=$MTzN$3)eE5c^XR(Q^5NncT%lIK;o-m%m z#XN8`H~<)#6PfHKGFwRaLH5IU8*N@&%-CE`Omrb8z^DrRu@rwqpKj<=j6T`W#iL6I z-|=JnfL}IX5B}I#6aB%hxL zpK|^O*@L~eAY%OrmpjQ8SHp+hA)pi?pWrK4vwy7;4uD?VeJVJnsy&SI9~ zMM8jcXA(%6iQK5b$t;MoVU#_PP8}4Vn1C(D7lU4)0U%`D3b2QU zo?;+;gMsK9Sp%C7zD&i^ezgEMAEp zV4~qbtwSjWNgxn(;FxiOyPEA}RWR0Ne@= zfyEO-nM@+?faRttH7 z{a3(S*yEi5o)ePB%?0@92Mu5rzyKep3c$78zj@!BV=hz)0?16G$pF%rf#-8?LK+4)zZQSUjQ;{T#JkL$-zMXFgTQ%}?C&Kq!sp2+o+V>CN9OV*MtO|F%p+7# z)cr5&J{0u~4poD=4d`|F9i4un{_rhx=P$@3J|&m=fEoJR%-dgMCjSzd#B*d8=h5Sw z3?WZ%qV6#Y8RId>jpVJfC`B#AOe>JQk&I>!In*(_>k0n<2)Tcd@iog+mZ$QXrdjHk=gQ$lCceT!-1 zYFua=U3v%?eh{~M4fotPyc8fzsI{Z7s zzZCvm;olQi>5Gg3$QX>vHqzpev}-(arXphwHHoE!>UylPo80|4p?wamJ|oos#!Agk z2UY7ZenIBv;BD|MI17|ZpWq&lquAmwesL>#|33O;H<{T^eB%}}s%_}81^?L01#}}B z*aj|Q>JEps=&=SpR#DJij&CeQ;u5r3gx2%%hPhOj=Fr=-kgcHBVSItS*T54%xw;Cm zBj8rCpj{o1=LLsg>=sRnl8}*U%p9Aag;9a`RgqQqLQX&A43Y)tF%LcFa$#wH7RC~^ zQO_;i!zJckjy{KmUy(x_$laUqCh}A`JpfersZiVl)Nr&BTdXCwTgmVJ%gFnekmoL> zHZ>nT=8{d$#!qLF`OP5noKB886(5;`9+T;piNwn|K8-;m^^}kiaBRY|jRf#eV^pc9 z?liB#IFGD{fEsSp5X8`69DW%l=`#!DIwI2p`}rd)j5fu5s0A9u;_D1#w!A9?Az)9~=N`XjbiC3b>SGUo}lI!=`q~bs+-1kr#w4-VBN! zwdm2Gc&R~;zUa{#j@3x&K`X1Es6@*OtW|~;OX!axBBp={&SzA~BMS4V2j!xJ8rs!R zyxWRCU}#?j7Fsq>6&Tg&Ri;B?Nvj;NryDYTkPt*`BZ$se>JUkILk4ZjMP3o|Dv;Nc zz5d7>N;Hq<%uGhPWq8q6G`@q;=22EUy+ea#?r71}0h#$nA(s z5A5lSykO)-Qq$oX5VS0fy%e-jm6-&3B@WBQn2nGqbDqCo!Sfd^dH#asZ4zjCS^_Lz z5r3;o;%EJn9HGs7xU_B#q8~!(hcKpCAxs7NJRE)M(4`Odsl+bDHs;h@Y$eOA6K(X6 z6f>R(X&xd8=1~%Fksz^_X%cOfCy~~r5^mF7LT!2ef?bmY*iR8ZhehJ;uu(i6ZxeUN zGvey_s<=6QF78gh7^fJ!khd{{Sd1nXVwm5v9T&+I7=6_8@oMZ+?#cWPebXH+B*E5B z;%uBH+SXGdZT%(OE>uG8Vkhs56PWwu|4liROy$e(En^4G`W3yJtb5_vb>u4{*9bF`#lehSH3KU=G2=Q^@{s-4|@uU{x&YO)~J^F~V=TPau+l(A}NS~ee zIQWh|R{kl9-N{)BUBu$1UGnt%=X<-<;c%#b=p*CU{ z(n)MWJ;gdKKrF)|#5^2=2)+>>meDbM6E1t?WSM4Un%u_JXdA&)t}3SCKlh9mr>OD@PY} zDnzGL=pyMSU;2q#DYT56_u`WG`3G_*TE*RT*po={sB+rTy_S95d00T)* zAi%+LaG+c|F&E`?VU#x%OahC*W{w?&<_T~CN53pvY2RApFGZeOhca~lV?!-EfLiv| z^GJ&OpgOulVIN;~=~%(}r44k3eDG5t}D0~cf95<0hpFP01j21%2nGxxChAZfT@o;9Fm!I{YEzATSTle# zc+el+9VirFi*DGWTQHzMy5Yp#3IK|C8wkdLIba=358-5IG4OlTWO)7qd{+`ai;y{c zI690%htZ5rYGdWw=)3U!vOFi0r=(Bbtv=a5SQ~HVcUNe-GnX~=oEoIo>TDwwoC(> zCs#Aswg5V&jLk4UFP46pjvmu}p^pUV0LPzN12C#u>@syDoqiOfp97!DirM(%LSkYG z&0B=2*soeZ_@GY=@CJ5(i7mzg&@dfoo^YydTUczvar9W?4`LaBa{;zkJOt37#jC)r z@OlI%cwd&^#AZ3_ETfx2uceeNITjE6fIXj?hN($*b1h^oTU^@&>p9jM8?2ZRpvQ(F zkOYbWzPVv6pdlM?VS6v%c}Z@8P9 zZiCkjpkyh&hX7wSPK57Zd-MwKWP2Al4o(1E&NyLyz?^!8JBTKc>r67kU~qa2z&9Vp zHy=JB6OdKEiD70lakvE^+{*KKIAv_BlDi)q1h<1@K=D2a?gyv9L*O(x1L$7kkH(L) zeF{7So&&F75B27n*AsXKNg1#H9L$3jr;+4h?03`(2z?DieRgEV?L$5sPNtqjPM$+X zQcUfmlB}Sbx<^0i9)rl1hB7-IPOd(hy2J!l7EDFtY(y`jUa(4@pz|KZcn@Qkhd6SI z3W~b_MZFWob@&^7egT)MKYYO)o|V7Ii{Y6V>B#5||5EsOqlvx98vDV2 z5KSL~j3!JmntW*jdFxcnG>2?z36j?$>lV7}cJk;mbm6;n<*($F&9c?=Ft)bR;a6mT z13m%oiUl$|Qfu)g9}bhV%+yblF+3mzw4sdd=m!64_^W4v4n#&hnaeOFjiPN6=(=g- zZS%0eGF)^cuDcIS?jetRj(Wk@#@fe5WUF-;tFP$rBXTbRMsH*HsqW;d( zyhtwlCsmr}H?*pC7~di9Bk(eK0z3%r21mt`Hg%%EylG1aE*b-;6!>SuzX1NF3@S()!WJnT(uSXmJx3*+B2?=hI0vQ}5XMo*pxh4I9YeuRVwJVdUOt zl-(Q#D$KXD?qD-H=6dq}we-m1625&%#jeY*p`%y@ky4PP9``bWzX2FyPJ2cmY{TzG}M)`@KL7Lxd}GEooOrTH=YU zK;(qeo;cc*LVL1kPiNXvOk2vS4Rk|RFJ$#Y)?j1}N7e-XpNVB|B7!#IH3!k~3?B3@ zm;Ybs!R9p>s_#|%Rrss6GdviFMm01q0rSZ*XOZ_$qfe#~Clkq1$D_ws^x&xm$jPHk zMMOq9t?x!9s27nk04;_PHN2q|E&E}qzQ!KggD2ld@4ZOA{w>xs@C{Y3Y#w?JuvJ47 zL#6TZuz>AsFqIgYNZvn|J{d`z@bo^~#oI~IqaHu4BikKF_FGH+fp>8dDOF_Leb8he z9y^Rr6Y$>I^w~%(yTw*1g-pHV*(&)t$T9!n7B;XVASU-+xU@Y~y7)E4u$=S!{ev2b>+fLX3 z%yMErWk>tbX9(x}gI@S&6>-vqbwa#j5qW;d3r1cf^5T$}Li@64S3Y$Q-inE)iFkv0 zdPyuE8;xg0;%VW;Xec9g2qSwi@exF=@-XsuIdeJpWQxFLWdWGx#e9ZKT@(7KH6ZFI zM6SB@r4{|>KxuCvL1{~9`whdj$E3Nx!xaN3?}ynHwH2#<}mPFk$ zj#^y|oTHEwK}*9p8VW@)778@$B7Ww*#mAyvyewH&WI0D%tyYV()gHOQisvs_JtGcQ zABw&8w{pOPcnBaKLz$jMkYhyRi)=@Q(GTcTgDzF*QsTz7y#wi-otea1SxcmaqlB5c zO0c=N1X=`&zh$`iTE>ZwRjPPd=ZS|+nYh~W`~|zg(!qX=I5^H0JEv7*I@~Q5 z9nOil(+6U~nwWjQj0wnJ!xUpl9KH~bFR&dE!`u>m`k)Ih-;hE-Y=FHIUCB8+noFR) zjriC)ikB_VU$FBLH@jeQwT}{Kha~CXlr4@OdH#ZP53z9>C|0hc#KLWcn7K3RxHAEB zClcJ5V7OfXyb})oo1=(HrWcEnnJ%Z2cc(DzPQn+^r#HHEL6-vb%z!o?UBc1D&yD&+ zCtK;zp@TSf@(>3Xf3b577hAV@v3Ac8OOGNk_pIUv`TTTFek}7mbm8$I5=I~S(nrp`)yUSWuP}-8UiL#BU#9H6F9>0N5MPL zeJ$H)<67h|MdsXc;<77o2^zXEHqnfp94jxSFF2nJT{yP!<~xow)tvPxl1R`dbihFF zZ2dtvpz{(jY$AInve@nAmK01k8REOh9Q@U?8g{%aszg4{XX$#wXOx)*0C+x|V! zfnyc8mU4+wAjyP5<{&Tz%mM2- zeh9jUN$B4O-^fPVx2lquK<2DIj19Pux)*0C=+AzS8sZY2QehR2o;)I29I%TS`bd!( zC#?w@d(a7Ed4fPXE}AWd%I^$do<9JL1hc>z&L5ztafXEQE%1%3hh#-BVgi{nQFk(C z8$ARaKtJ}o)G{`qOC0(IpbH@{me_@h2lWh4i7mQf3*(v;2C@oYI*f=QxL$((V^E!AS*Rb)dEW;7a0v4BsOK2=k~;LLqfZ98FtFek1F^-x zWNI7)RxL)Wr6IL2u3bdHZAZY}82Sa;@|CPahFXU*ZxT98WsIDL7^Ut+x^6sf%elbO z)L?+wC}b(1pAj50epbiO*LblXfgU69&*1?8TQm_LP4q_-25f?`X)XicMuPuNg6}+? z{+TRCwZ+I)&xDQ5AW*rV-D$)$jF~L+5y=FjXob(vaTR?*z zlRZEPNW!=U0B4$nK_^XTvBw&4hz38-pnTD|4rLKxtM0`ay8u(NUx}(&=o2{`9{^T> z2{noW&NIo@Vga99W4<|`(PO?BhyWR&0%H#ZG+He>nYSC9g6~_hbTNl;lSv#N3VJLd zY&aH!PF}#8ed7e1bH>T;Dz-G^-(oE^Ytdt^FNkd(2-jkZwK)A+8nkvZJnjPLjb)js zE9rw3xHl*Q3BV87a-0MTdce@abg+5yk4qUR;c9zJE9_ujVJFxHXsYaS0QkmU`ebh@ z``ALgL3l6B_a0y($@56z-?9#+jQy0G@j+m}p6_o2sB8SuQSnwh`5M#GD{D>I9tO9A zBUf+-Tinf9t2oZ~Zg5X0)`x_GERvZX)Gx5b$$4N4b&O*&4q0`yuM&BgAQYK5aLU+D z)t)_IKfr~I6X2_S`Fs-G4;}z?tua|S!}byID0l)q4W0wf6Du!7gF^0;89;(NiQ33= zer#6;)Bc%RSXx=z+Sxleb-1CEvx}>{ho`rXuU|l5P;h8icw|&`Ok8|IVsc7qdPZhe zPHtZ3g2JMb(z5a{T`Q})_vqQHcb~rf`VSa5sBZ9(p$&~q!$*u7J!b6q2@@wxxpC_B z88c_inLBU6!bOX3TDolc%2lh^-n?%8#!Z{IY}$(XPD zcIopkzxw){%in$f<4-^T^4sr!{Q0-xKl-O_J>yS%`~PS4znQt}aU13Gj`(~><@9cN zy_a%(f93ZfIDUlk{8;7sN%(%6a{g@P{rR|mvGV^40-&3UfNB*2HN-%z3W9nS1;Yq~ zktzGIuYzHTiiQ=0!x|M28&p7SB_ejHkl3qY;vhkByNZfqgvH$| zF78)>ahk|Dt3u;R6&vRXju%vPysEm?%*@R# zEG+REYa1I|TRZ&5!O_tP*SVo%$4+>Ui!1))?tuq+dE-Mk5nhB%@FN^ad6LnUnte&< zOh#YAmAX}_p_GxdcF^D<4a0_y96fHrq#LKroHK9X;-xEAtzEZa^VaP2d6Olgw7?JeZuGz$}4aS`~t_oGjI)j|Sv|FIv0__%Pw?MlE z{&!g5_pkpAmrvgKHv~U@_HPK)0__&~|6_qaF24Pb zUwi(*`kntYK{&r^Snr8<{&-y;-|VbS53kvD;kpD*PfvG|E-SAu(p< zZyj4SsI>dGOV=eZeEz_u`E&2O{&|DpuXoQq{^~yww4ZCY!2e7OeD;sOjlOq$^^|-6 z!7z_3tw~Ir|NM0+o}5w8L0ksjem#P}w-j3NV^i7YU#>&(uSR?IBUJ91%hw?|&Uz32 zvey1^9fFs}vI2p{JzKBa(35pO5aiu<9R>!&cZV9&!Xxq@x-Nm?t25gdEm{5A^$9rh z>*Zhn@qGKqb_@L1v%q)1{gY1bo>(>FW{vc54Gmlq7QH^@8{5 zk^U-p9YV2p={H6CJ*OAevzRzFzeQguGNHtx$u5`pbKuZ4MtbU15=R zJsqB_W5Er7NsqP7!C-iPR9XiJ;-0)7f#KZxNwwX|E1tV9f#J6gUw!WUv)8la^}pAC zrriSn2@72B%57x%#lzcYjy`!^?w=o>UJ)RkOa8nL#e3`3s{mOQ_fp#ge{CyQr&%_7 zo$3FJETc94g>Pw_!$Fp78vi;iZkwQm0V_S*ynO#{d4&P%7u&3Iu(-12Z$O)smdcjX zEYQ8smJ;gqVORc&+Rl?M^lKhqSoVCKImoB;DuRug7G7^8HJsi)Z**NvpTTdorNfnD zKY#Vf#Wr>PhwrqXYPZ03E%4##T?=wN#k6XYJ{a=q03 zr8T{b`GfV0wuhL9#spt+>z-}SFrFXj+~W4}ZO$-mGS(K1G@UlIq2>APmhZEj(FOsl z|F8Wun3gCmbGyK`zaC$iuvmZXEPI{bF(Gi#5X77O9Q3T_yabvgYnfUC*W-E-Beeuh zhxG*No`qI_jwappMXR&yx*XGUFi7f{Wz&cEE=&&hb9Z$KK5Ig7&UobO58rs<>Bk>^ z{E0722-*+)ziNS>{-I(2I=*&X_vDq=q4;qB*j!_T_j$cdihD-3gkOY-&#Eun*7Mqu z_<3z`IMb_@V-IR$7#n5NiXh)qiTk)~>Dph|850>puAOD?22(9NY6xVbp2J@4d0D6@ z$k7o@(i7C{2paSRt8@f4dIEKyQtQ7SdV*hCpJl(B$v~+c<)m8A!O)~-G024AMTw{w zI%dp>hPna$Y6c9NYC>Rm{~t?Xru6Mc{u@}}&v#DmS-AYi>u7p@`?#`TVmzhIxxL|` zX_@?dVf-_R8#sQyx$K(P_4>qA#_Hs2=OyN06M}nETb&oXohAf>TA$~pPCda#y0~c4 z6L5>8=C8Y+AWw6e-%<2Mj)6LYsCRWN=IaQ&^fk=ebOiSLDadO&0x_9_=m|{5!EhbL z0iBYDF**X1Tunbn9o2I%a7Sh9zY`_|cev!$jGDP@-RAAP_aDCNr0H;P@{RTb{}n9o z*$dZQW%bsHHDkNQb#Q2F@^3J_wrNo8l^Mc-HWq4ru~9uy;L2b4DU+sNo)*+{R&7_C z5UgscUN)1NMECbs%NxxE#g}y~wlyEW`hU2-I>?>Mnm5TGlQ{H`bH zpf8i?LX+2s>XfL~v(V2`bg|2WFm)Ur)^j+hBhZfzvRhA}AA?NlsBhk&6|!8<2h}J zcx6SmYjycb)A4>?M(ghOc}mY>o>S}dJV-@HaPPHWH<}42>IhP_-{;vSI)ZXNLCkkr z3X|zU2i>%IubzWEq~-9G34t!Mbk#-3^|Liymc*l~v?2_x%eY2ylXL{b^hylT5p2{G z=%*m(^aMpZ4u%LFMXH`)qK+V3Pw<3}z*SFRXjv}PLU>6}@orqpsphS&Gax^w(bHjp zo`b>gbwk6H1*^C2yW`|zFTMZePo@OzM@%fBPrI)YTzdNO%CX&IY(w9=dg$8D%Ny&H zTNXHpE-avWbXfD`<4TuKzxKuEJ@t0c)_?lK+DV%+^x0Q8&L^70x<%ho=2dD2e6d#)q$Qv30gO3TtBU*xD6M`dJg3%@f+q4AQpK%rSGA)5gXv$PA!Dr2# z*h-N8hnUW-&KY<0!}*PRf-J2F!@YWfU>$)$ZwV_s!7X|g-|8s-y5n%b247KcC2OZ*_C}; zP4~989B$d(TBzv=8P;;*%66dkG|xTM66oEDpExgTD6XvUYR%?Z4Z-x*r<(V*v1vL! z$QliYuT42j(h!(V5A>y&{w6F6wHy|j5X5Q;ZZ{$D))KsILSU;U_(M0BgI-Hu=xM@2 zSF<+NQddwiwdEsP8Nca5m*-h(2@EUrO3c?0e4uAhqa!fvch~ZW&=VL|=*CJD0>cjN zl}qb%M)LP24jegk;mx~`oqeN?v!)#XhadUtw_kqv^-ohHoSZeXwns@$N=&dqW0G7W)A`H=^+Y6!kF<)FLjX{s>I z)Y_}wC{q^N$l78`puH@fH6=Kv(ejc>jYD>72n;zUEOeJe-4Y7Or)xQEFySy%OYnrP zo}!DE;Ew`5LA;j0ut-nPK}Yc7mETCN(dh^bORk-2-up#I@l~CU;=GQ)@Mb?Pg>G64 z`R6yX^2XXFATGCR$kerWzNzQH=XYPZ_^qy#-3C?rp?_n6zrOkK&6l2e?9BZqj^2Lo zux{%9*1>rVm6=h#yk6nTU#W)O;(T2Rls7tP2r62>*J`WzdEr2-6D_+>v}SR+@Ly}=nMZ44c8FdX3C*TL-4wp2}Op6zz}0X5TYgMV?xkTOEAZTKsUo^ zLrX1(*St+wXbB9JCIp(!Fc=n_5WJ*i@pPD;;)IsKFiB6Koexss)c@+$sN-O`HA>4P zNl#!naBb<2nM-70oe6>A@%q7IXRX+F_`w%GF+JISN?)Wp9%Jgqv_Ph0fT2!rni6OyYnx38?$vO3SU=h0@%$QsOC}{w{h}ECm75neXgCTDY9UTY5SL4hzikj;Q4ByUjv+NWQ zo6~jR_{BT!Z>#ukc;@`upX(;nCT!Xd{NJ*`?>~I?>BaZodGqb}Kh}zT@6_()(?-?x z?plqJ6j2%igXwHd_e0EN6AoHa8utD>f|nCbD6~2l zOpCobf}-6Y2FWLF)}9c=bff&S|X@ren|nEy10^E#GX} zF4YozHLT@C%eF~NU^tfEa^}i5Uq@hA+e(N_Oj*+kJ;C3r!u;ZLD+i5RxaHV6&E@1O z<@pys&`uJso@w9t?`na+fBWH^ufNq@kPH`}I=*N9O|vJDZm8{DS(2BLoRr+UfuAm{ z%?oq3(E40!0)xq{8d?p3=~Yui^ZP>^)dP8_O8MWpZmB3@@1eBBf0*elz_ss9eLrv_2^bT7r`?*Yt1R)6H_eY0?vDZEY|djMGzS zMHs%H-ugEZbKk659f9GsVd|oBLqJkV?c_B_&VR2z&+QL0=dRsz;_)|4ryi!X?MMEf zx4L?lxcU()bu11H3;H|8y$FJ^a2=q6qD=KY@vex|S zH(PeJvvbqwfxN7dal!PboCh@ohWX6{a0`F7`?UmjXSX_M++D6E_-wM)6h67zwFHI} zJx`;K0C%l-e-B(yD@e%iU^j)!0VwEaia7K5~J|C1K@ z`^PUoe(zuZi@h&_hw@t=pH|vTid14Ml&mST#7w0L`ASlRm~7b+$!>;9vb2o7QcSjx z5QWITB-!`ujD0uOVHp222BE(9{=UC^|G(e;-CLixappbGd*1hb&htFyS^{nFR_udq(_EJtf0(KBR+X-1M%*&LP zRaeh{ZLra?eeycKXr+8y4oO^@h_k%wcl$R6{^=O#uKAXc_&y@Y-`C@*n~Uos@1VTp z_n30Gx}kF6>`6f$4i?6(RHy78%K_$YF#Y7JU77IyeC4`JVJ^BABjo-)FX_-v-o;gF z(A<-sqs1$3F&F={Dzy@z@uz!iSP3wPUEwj>DS}$%1D#7(ro7wyKNgAW52I>U4KO$H zgc7RR&L(>4)|2LL#aq-(_x(|1>Sd!WLr9E{-z>Ye95H}Wdq zk6tkV@ID$X#*1h;q;L6V^sVZjn|ndYi zWW;)@j(L&qR~Zbi1s6N!|I$eqyB?{KkKD{4gE@0S-Spr<2MHW z{V>p6l9Q4U8x;{25*!ltERhu3J!Y)##5CKeZbFEL!pt)L%hEX*(auI{W5K(JLCs!u?a%! zZcxsjIxOdga1Y*o?fG%R&#m^-KNT)N1OGnruZDr;TI~Nu`+plB;GO3NAUqfKP0UU1 z=$n}CKhAd)B63DXZr#8;M zz|`EpF%Z$W;f^Wt)tDadsEj~_m=q`fDHlugZ$Ko{vW+3TwF(6OxsLX z)7-?2dV59P+y5BOZW_QZToC7{pbIpq4KOH}w@sTiiSsYN-#pN=_g#Pn(*{~glMJrh z|3C2W_f7s1l7FM?H@f~32L2N8Z|eGuuD^tVzXbf7y8e%$>xZJKdkbiXRzMA$tA|KJ zHm={Wal`tJ8#ipEq1i~gdHZIdV{B%igKpi<#ISoe6T_}uds#SOdzsnx?b@}UXFuDa z!$-J|?1Awf<>fre!FhypE(5|qyYc?|SL-F|A#(H(6?%v)J!CEi0@Cc(Y*;h@Z&B4> zvvwW08x8HI&07FK{C3FNHS5-`UB7O_hV>vbXblGRL)O!8*l|GM^v0bEnluM*GaS3; z5lG8+CN`7tQrR&3aV@j^n>O!a+P!Bl$DzZVN4NxqP6&&LibJ-vjH(XsJ~N#fM>TpfgNy(B~qq5=Ua zehA3)o3x0Sw?@5woI!5?W(PN0P7`~%xWIK2q0ik>n&j~jX>#=2a#f-V^B-|N6Av@}^AdGtj$!)R=OAu~ zk7@By9~0o4pNttJJSM_2lickr=*fHFcsK5^{PA1rrwVIoteB4P^c+23oJTm6!hEU9 zV%vrG21DgZD=#lEj+`R4o#TfE71W_l_WP}0X73E{@eAo>6tBPVMf$m|u>?GrrYSjQ zJzA0A?08GtOR73rwNd`WGd|sW&Tq1;r=y{%49aCNXquqDJ^M(MkXiDDH{aLjnlyZM zsJ&B4Iw9d2^A65PE^T5aSb1qjdzZ%hW;wl>gOqicOt4FABKK*R?nE;7rrIheIC^-L z|GUlkzZnTrUx$miAkvOmh0o-$TT_!jQisp|x7bVaYa@G^5|WL*Yy6xFPb9cU_Z)hW zbr1^9yK!fy?Vqtph#$R~!m>T-rH^pM5&9U?qpGy9b>C}>(iss3^|FYQ@jGUj7Ul*C6l> zIstn-E6Q5c2ipP{B?f2Q;Fj|4tvlF)aqB78Mor(C@t!tIt5iUpCo2P*M)O4W-J6kY&}(KKpMzw}yEQQp9d89orEc#%aOV*2&Vv^iQ?a9y*GPZ~W1=U1 z6ZO9?2Dl6D_s?jRrU-PZ_Vg!5hbOUK<|lnBA(>PR?}2}c*+YyVowC-;46yL^E{Hag z^%*g4un`cR74-6$gRHq0kjJ2a61J|%+dK65R!Y$g!sE?%Sl*#{6lR%{CPm@nBd{EK zSNI{q?2tcx%Y>1IahXcC_kI4LNqE=cCO7Wq(Iyav1NwGq+B)7tT_7JuO~*=bkC|X|aIIr0twR@!W<#9{S%> zzWtwjDQ+J^I@TOUqvl%`oteyb!~j~vO=fXU%7|veOXWhh%)sM$qnkIT>NYYyNn^E< zfTLPj2krsMYLx?hlsxG+Q=q0NcEh_Crg!gY4-Ltra&{w4Y>c}-7utq^cq-n1Avv0;`W%Da$>^2+J=xr9FWq7`C4w8`K z4R2$cEkiO6R3~n4tk08pS5~y%(M*PUqY~@ur`s^hdKM-aBgQIk!Zo$?fo;Br>7GJO zrPeDTWfYg5{&(Yl4+DBebxki{8w}aKYsxmbnVx%~7X#^)f8QC0R=~@_vg*`&;Rh;> z`*|ve9XEHu-Na%iAby-NxU~If%ha^dn=@=XRn%bKP3PD6IFs0dW^EAXrxfvF1H$2k z8kV>gPSk~uqKe^vye@8=b!y)~DT){bSMN;vb1*#de0g7g9EsMJp(eZU0jsCzS*7~-*(6JC~b z5u%>uH$US!ihsY47wLq!pDHv(>sSKQ6^xzph@>zQC+aCu$DzM|%q_ zFmr698+UnRP58f^4fTIVG`|optIryhKZxA1ws0`>%Bew_ZmyFrU5t&}1A4Ev-WTE- zkp^dbWyIGY8QWM9#ICTOG+mq%o7|axUAc<(Q_iyKQt#@h)sbOqkY!Xg#2u$rhN)*o`knjgiV zt3)HiOEBiPdzd>;cfS=;=!XQk6}&so88S~EM+BCnH=tZVwTD(A87L)GC7KG)U6BSu z>(r$1cBuAltxNDuu~9?}vBCNgsEl@7t<;x5=~#m|*=Tb@HTXngP&8Bir#qcF>0`wg zchKB=@ip;%yYo4&y_r$a?PoXtZ?kW?Qf zHo=?XNS&m3g?exrihPJ4Zl?{Q(L^&KBhKZe~9R6w29_bc{~?&-kYYF}jXV;M{M;ILiM=XUouYt8YfAsXw?>ch9flbxM~j`T;1G;Qx-XMGy> z5T*dkKU8+}(`1d94i#ePPT4dO1=A#wxN~!|y(lX_EtaTElGU#p@(LrCi;NkcZAI33 z8tcH@M`3LqgPcLBen)obdrJFj2(R(+Za*qPD>TNiBTxK@;0|^Vv4>LUHe6+WKwqhV z6vHh&`0vI)83T;+%2hn(c6<1RWW$YlwMpgY>!m6hNOVRBjq0%yXpd*FSmSu=6K|BQ zK?CPBRI!`pkQ;YqiUXle1zmukIjbd~Y!ZfgJ_q^a99ETn0L6#CJR252 z^XD0BiCgeeF5-K*LY+!*VO{*Xrn2WSmXnHuh&)0U5U!+S-n_|L)QrD)QqdxZU&wm;Qj2{BKE<{*1}Z;*2Q zylXtSPU&A@bFn>0$AA%=EMia+U@ZRGzVlWFBZu}6}adzI30w&To z8<zefxNxDrSEjf&u$GkU{`*<+qfy=^4PtLr){2Cg#Eko|v%Hv;bINe z&m?0axLHq{NL-_IyK%e52rueS0;@=Iwa!eHuRo;nf*=-L)T~uC_LW-j)Az<2)Com4 zG|CiLwQvp=+Cn7cZW3j)#nKc*5s$8i-H|rz#H7fX7?sQRAQQ01S!5OK&IJy2!CegdxxKWpg z>sr-ZXycM9qYmYL!(X|LGnGQzV@!=C7X)ldPSC9w9Jkv;82R)<=Nc)JA90UCK|>fQ zV-K4454(7_OU(v{#vlw66-S^G6>n#4SjR_)a$)xZjtuYi?MKc=1;rqYKT48f=OA)B zJ#iY!!1Z3-_id#j1!$Jyfd|KjH@!@L>ix=o#)Z|13a>(9MU`MKk;(7~hoIi0%{{_i zwHjG>_LYjBL**zS?P{`{u_3@qDdf!NwAU&5&bSB7DC%3`;3^+L0n94?c=QC zRm8pXz;@y$OKdq~(B<*mysBSL79Hzlj#SyGboJFSTtRv~Mk1(p+CJ$`cDVjz!eqjw z$04O676_N2fl39WS|s?Oij?1^&_Dsb;F#qdsJ<{9$QD6qWKK{ zJf;Bx@G-zOW1o&kiDhR!I^-OB;_|mBvJdB^<|O=43F(|0cm9hKwp6%s2+<~4kue50#?$r-uISiH_m9YB(8bY@8IH^ zyTF>~IfD|>Z1@4AEahf{(xH&MUZk(sqJ05{VUFXyC+XFN@1ZU5{Ob%+H;0>od1JN2 zzsnafWS5l=+owutaR#Ni6YU7;=W1pmCD4aFBKAT0o4ZF~HxO3c%`al2$c*H&G#CNl zrHQ}C>J)i`;bh`8Pgre9!?ZAL0KAgA($+xl`yo2?O{uji>XHYxhUjcJqYZLXRi(;} z-=F_g47{Co8jtqZ|JjbenWtph-T^H*WGRO&8FiycEnoY)1U9^h`t*pJE3q;fw988hJ`k zL1X9JUghNY{%3yvl{LzjV16=JV-d3^hEopLs)bnWEu-AkJ?Q=y9Bk+O-(!q%evS3ME=}$XxF9s{hoOC zTaQ~{#cteVYc6v+U0i$bE{);r1>|Mq*)>*EbnC2B zh{yY^TIUR(HC=b<^>V~I?VW{?F^EH=2Ij!kBIy5NSGH#K*_ zzjlR{r3;`0fnE1p6IWabXDvS9F7d(C>L8=vGp>6hw!XwKEF&?`aPQiinNFQg?xe`hENtFu=aXE915M8uk{t5MXCnVeJLFn_#27 z=9uK@5Rut>`_AL!il^zwnOtj!T=}}g*#SF8-=QTws8=&_p~HZ*cTKOmJ&BF$Dg_?X z(Y!}!^(gDOp^|D7)+;muS?N3qbCBdz1Z&#(h%h#7zqMDMpK~2rDm-88NTE{xh$NI0 zE&@~ZwDUqTmed@XCweM3Q8ZQR!Lhg!QcoU`we>(P0~;Gil0G z=+0WV>v*WJ$S5i8-v1P+mOnHEfD3>XY49J27sX%?yoZLcj>Z1SBbhXnd(oG~kA--g zWOtF|K9YG!Y`Yy;s&v3QSD{pv5{cLu*_*HqB99w$$*Tl>g@um~%|SNLK@22JvdMDJ zk1xrY{IOfXjFxm+ve(*+yup!NW5^j3jON8-V7VvOj!LGv_3{aai&&QgA8Ch0VRT#4 z_vpg~%X8--563x84z$@u1{l8*oAGo`t+X#JwqV6b zdt^n9mc#y-P#ARBTC;|59AD!^hMmfP+k^>LACS-Tzt7R0Pa@lQYF zvpLdVOl%YQMB>?MuSu@AFbP{%2aLloLV7M-;#l2_^tg}bzV12^eU1Rk9a!Y72+EK% zdUHxNMJW7W9QU%tXY`(*FOL5byFCW5RCwwvgR8(WmsIBnhj!?W!HgY^T`A_%@f!%%nVMBZhDhQy{Iw^pN~rG-my<-1=QU{bCWlqC+Czs!%~ z^BTZHY~PhDaTHeTGr}E~CptnBUOMgdkK4&?m%qEiKGLIq1A}t%_a-$| z9nwPkHe>F`U^CgB4?&)a+qAM=`o}6B&D4rpuRni$VI?I<$Fk4%zKEcLV%5K4Vl*Jk zf9QCZPH*ifJKS+ErP@R*-kE3xdr=Y;-hnNDKZC*-K{Kb-3|ZMKU&kiysZZ%>hr0y( z!ZQ?*)-+iA6TMi z6M2h?)rUM~LTwINdP;G;0QQ_jeoR1^pPFmCA70AWVE2d5u{X3ps)!yVwcI!(cpch$VUu~I5&h|HBhcyYbf8`i_0G!u zO^ixivEXVs_C9mH4!v}D92;A>jwZ#d=>|)ZGai~T3%fRIA}*#VQ%NWD7&>MHO~xjT zYe+<9+H=_#O+#JLeRGgaA8lFR)@swD##H3zyAid3Eemq(8LH9Tmr?;-Tve&lP}mG~ubtNXM7%P}cA^$C*5|0#Z|1cpgy?sNw=(ii3gL zR8SYbKVuS26dNikGud-c`vq^vRG?RHT3Q=T(`Tr2I(>?43-b^nyUsCh-H9G!j;WSs zv+d)PyWwLBD47`vVp*W|%Woemzx9X(U%v_R&MiRTT6Yc-n3rJGm+s{75%w@eyZw6~ zPcm!VcZGdJBv-XrS11HGm;lxE`%% z)QB1vfvHZV1O5`#*6cDMuW>SoK6knIc*OR=If$lM(xeD%2v{W>tK*+YUQ4$tQXMq< z?AtqX!c+p%P4ga~rcB^>v2fVg>uD%P4!rGPJ*)HL!@cg{Agm_HXFL-ZzQQC~nx2Jq zkLoRH{_V3YrCjW4>#H(+wV#^0novdYFZD*Ou)4=|PhJSA~Nc5~T z+*P4=Eyt8_-bqFNlhEji$Tv+rdmmJbg@#O6x%U7Zv3m69uYA zOz6xO!j@BnkaZ-kZjP2ccB1d&Vq#!95?_!Qhv9lW&WXIgeqB(L%TU&!XI9%LF3KKqRM)Vrxd2K;^;3k@1PjAJAS`X#QW7BpI zEBot)CHL9Zb=a5OY%6e0?+JNzZwzFw1Bu-HMdL(`xccpMrDTL2jX+)QNDdA;h$)ss zI}w=idamUC8heELMkkOUGyOC*Gkw5o2+3Ha%!s7Ks($$0fMi<;+kzJLSYQvqMQOdl z_p!)HKl=$l;A{bMLBt~&d1GcUl4K&;LgLws{pmS~I@44c5;KL0__zoHxb<%y*CrpM znFEPwC10!iJ(Uz(y5xPY&OsV)rQiEH5v`av2Z5#tw-rD?SYt29_@6tNB=2eM&}a{O zB{Bh;KI5U8>UXjOjGLZ(R<2-Te_k_LhMjP?n2u@ZZZ~nmeMoni;pjr?uNiM-G0Vxq z_0aQ^6tF20HSiE1tT^{O-WQG8zW@H|_4G)Fz2}!jt*NX8&=ryxpk~k(P^WUh@7Nyp zCvI^b4fLgz)z6OCHsszm5OlwGHnYw55Z70!GqmkMr!)pE*EGd*rz=F2MP6Az^W5m!|kX zp^+4`$GTo}_(|HVYB%QfM>3iez$@3Tj;z>#$uSD@9O7i?fF*;wF z>?e~6`ul)xAsWLn&m&Y4S>UB5jytnzDkN8h0~WG{mUxNE9xFn6VDylKs~qUQHDowg zXweR00R?2l{y9kWqnYlYw3(R7x>Yy)J6~fYi1&tZrwG=bEmE`oZxazli3+2*iJD-H zHGI4S#XzPZqVj^&VwonC5^52*;!ls_ruARZOb<5!#Y~Vm)LH{FIF6qPHSYZG^&r)e zmwqh2H|%>`y%4Zkb4nhNIf-^BPsHeT^@PT<1{yzeN?jvy5mXa?Kr-ejuUT5Zm(WFh z+KSmsJn~5(~hTpsebRWN`pRXqJY=g*{uK3Z3 z{cfBwfn-*+?fXh}BtO%a1SYA?{A}y9x@H5bpq+4^aa2}%ozWjvXn#bW>6;g}ra|QG zaw*QUvH6liAO>&g#$6xP<U|eM0UXLq+6ID}S4V zfNH^l0mNRZMENOl>pSSh(=C9`L6PF{VYw>t`jX9J28qr*#;8Ly@WzVauM>=;HM#P& z!P3D$q# zTaq%%qYG-io_%=9yK|Z_bGYGpjRnvPpSi(d*Bpa1p1tQymfi9=1==?P%+AScuqaR* z0hV)ne&CsIQh!%0Sajt*P)W0ma1*d#3>`@1au|+jOLv;FXs>OM7Q`42!}HDBZa$f> zjS!|(PjJRa$EeLe*3w-4`!#OHu?vWMiRJBGTi9>PVC$vOAjkM|hum#0p7et+uA3l4#clz$P zmw8_k9?wBOG)azSOac>Q6Q0H4GJ%osx!Qyy`DMcB$m_8y`#2{UN1Pv}s6<<5UpAJ^ zvGty;IeZeGJ@Q3@UTU+%T?Pde3+H)WSPw{n%5hZUUZ^_&>mlGJ%Hox~Nc2=jqf*X~ zn%*DIegM6K+2$`_97S;cx+SK*?3+#)hmzW+Dko5cDVkYxR2PzQvcvvO8FKM4h;^{6*9F2xoQcDYS?w2h9DBpw;!J@9 zXle-0l%e(+0nECQ_>NBn%txMYQl9+~+f>$cd1uz3np&StRCJJWoa8YE zbNB;Ra5uNtL2eX{0^pX48}PZ1I?xN!-35)Bc7wdLV|59>pIEPzkiO@UptE#RZ@BC1 zLFv9d$1o_}e|Ox-kE`{pV){@JYJy$7#ld2!m}4I(=J+^ju9j=#eM8m&t`a;Q6GkcK zh$genNDbFu`_X`34L6U1YK~%*r*(Rhy!>qsU}RJ%YyLG<|Mw>7fBq`QEbBhJrE{wS zJF8y(xKLaF`=e9xB;zvsS2)9w7&nJ~;=C;fc*Fc>Bcr0Fz8hadJCpqo2jH&~<0<(j zLWGGVe2)qU5O5WU%1WEXV*i+fSd(zsjPttDBRD6~=_Fs2O}jzh2UeW1w-aUyT{Na1 z{yoU8z3=YgB)bE8_Fg3YB{WlFNYn%H6eXsnwMs#x5(uD^1s` z1!a^JEM~$JH zm!|Gij@51Shr?x`vDzNjlygiTYV0Q7(^u4Z4>P(_&p5VQrfpjIc7H8vr2@@|XjlcE4vYiB|2w4bzNA%wsyvc`8j{W4@)XA2_;rcBOcM+*l5i>Ye&}|~v_acg~ z)wrx^i?=!a^;3{clngwQB#Cq%-8N~t9F#dK_dZn3~S+6RtAFxPlZ|3oN%=Ie7vCMloyHB0QQCvI5`uJrgGIQmdxR+0Q!;Wx{nTsEt#)g7D zXVRhYlLo3KK6-d_Ij)(OO_?Ao!F^W#2(jI+rW!?wyUFd$VWot)C72XN6wEe}c*1&Z zu;dLM^C52CC*@bGrEAr9Z&XrrKRvh&vxdP{P=<#u?p({U}i(%!z1hVnuT3Xl7CLh+tV{{LTU%W zw9wg(;y^2os_~pereph3JY#QYj|+|RMMh}!PCuxr3vHNu=EXuO53z)|mm!sbvw1Zg z0E!04b}w5b`sLIDV1F1}%U7la8d{nFFbL@WiIcPzS2-%+!c|n~2}L0s{YfFL`tK`* z^Yl%9JIW_oh`=rTNQy9+s(mZ)y4ygZp%_WO#Zhpe45&3G(@VFnc+eB(ZG1W(pC%2yCJ zrZTl+L5c>$4BY2nMRZzCs2+b<|@nVBPMReW&6W>?%qI*6(u8DOWp^$4MkRj&&!Ai_Pvc+PDA^J*D7oEh zYSYWvSPShZL7t?>z;MjO69SFX%uDD_SD{7@)4;~`axKk3_F;*kB6A{*yeq49K(d)(T*ucw)kd(8Ot;@0;tUw!%Dz;=u+Yfc%` zL|N{KOjso%{sq4>=2_R{4>`9A>JQ!^U3g7XV6bMx9Hd1m2qfs-t)V37GY41Jk{(8rhTI-V=utW)8NFJF7cgIw0zjAhLNxk|x=m!Ob^ zRhn-Qo*C>l(lhby3_ZwXx-;HZ3bHu)Qpzb5ypV_sCihPiAu2k5&;al$EhoDL_rPiv zday(-)CQm>AWOkm3kwYO?4Pg*#eJ^%v-_O*PvDP`8=FI)-+is)Rp;?SK~{A0YaLHh z2uOZ*c!Fc%sko3-D$Z+`$h0%!Y3B%=_Y>%bi<8==9vLj6J)?&Qm|Mzl-l47G4BdQqMK1{WeXIz^ujB|3cv(F1ciIRp;|u9+<0i3 z8-eI=^=%&uPfHBs`t(a35u{Ua#-Fsa0@Xi5&Nx^m2v`}@ zK{pF+F`v~rJySCh6PCtEuE>}Yv&s=UYOU~AH7~DPI&V~-)CKkpOf&Fv3W(HY?0jjl zt?FIy5gxuh@_6o^BG~2k)gQY}#$JBqzxig*Q~iBniFkw$!m)@F|qzj0BrcebCVRA2uZy?GzE|S%g;;coGyTcT_A#K zM(vx?t`c%MR&z?Zf=7PiHS%m;C8i`BXFJ6Y^e&EQ;-ljC!*2P3J;pxA@58G&fk)sr ztjuN(VhD;g8ecTN`2Im4pS|5ngN7|7PSErC$ylDkJa(^2FC@b^V+x{vOtts?1OpnN zQvCe4#qI^}03e_rAp3`!PUS6N=upPmyV5^!1gs#-d5`jag+tD3elW_I%kK$BnpeJ< zR(s=e|54@ZwuS`Jr*#gp@k?1#113x%uR!;fq|CtQxOwU2kCrEbVj1T$aaE_9dZC-T zf}~4(QN~?CP$1jNkmxGPei!ilgh%2#d$eSlsB{kSnoI#(sM)&@*dWBNI3@}T@1^5K$pnjnuMQIM=IHMVDeP(iC zFUx4w?oMg7PB+w~H>G$*TV4L{j2#}awijjumKTmsT3ZtaMkR$m1h&lDQj&!+hr!Zj z?}~?Um8jQ-7#iQ*k~qkY=uaZ0pPMPdn1HqI(eRL4%1hV|8{1vc-!JMoj2Xz2`0RTo zObW}8wk`Ybo?g#-nsLxgl9TH+jk}cX^7;H1YKNQL8t6CHNlFdiwzK}CA#;^0eDcuS zUR$RCA5EdbFpCYLueg(C!c1Kp8v=lLTRRW2x9t%yX1vcRcbnWmc{_Y^j;SP{OVg*+ zj!4pg+7XGn^88#DC%va=Qf>#xSG|{b!l3NZo9r_fWjkr>Q<oD47{66i0A#KwGO9PFdqQ2u#2s4dE{ja^PU@ z4mC)!xdO_rrgvUMfGhs$p(N}kE*E00k5;D%mXrkaPC2yjbb-tvYWMAP4xRKZ_iLDJKklywB|>?T21 zMfs!=p{=_8(X=Ww(M^RCd2Mc_b*;~GOSYiTII)!vl8fFN7xYY?4gX)^SvP{X>Puo9z8d@5#kzm1W55%4|r6pA|a_fBn5FJqKW@JhH?Yl*^X% zK`PEJ(JyfdI04vf<8p=s@YDkgv@{Ah^DEs{)(EiF(K*OJg}W{@)mT>V@>EC3Gp2r& zyi>sKbcMCO%J@9FLN?ZK@>25s@^Z#T_6m`+r+a09@Ub?@fnkrTP5EkUa{Aw|-use$dovb#4FHbhhoUUaDe8+-7M--Lpi$P^lts+jT5Q~VZ2NzsG2 zcb*CIrM6JbTy!FSo7H8)t_Z&1XjB7ABY!f~zhIv&^mI4PS~sqH?^foEga|GQsK2R3 z3;aSRnp5jnqz-QX_RaM2(+7LeJM!1(^Xw2e3*IPoMv}78X<(4s1lAeHDkR7D?E~Qw zJVNVeP^&cQtad{HjF34AGVmG6<;lfP?LB0{r*e81b}N{NVBQK0%I~ljfp>z0sOGKK z&!h?G0{G2Oc1O3inJPY%?EYq>I3Dv3&4QoZ@-MyfTK|zf% zl`0(mH$NE*hy{6KhaceCE`A#2`}$2a`=LkbF~kGASN1}gGqS{}RtM*VexXNYCxVm( zdM3WxysTgg!dm7f=m(`a5b|;wO>$XvlM&wqtK0fZ1q!ee^^q76)+!K5%0)c2=Oonn zm5W~R2!cy}kGj^01QHCJ=@7OB>_?GV<5fm^agO;lL0W&26R7;K+Tl%wqDaxDWer^L z=YF^a?T|Xy5=$Jt>Zlqh!n!tAunp9) znm@QB4UIhF4$W5T&4!U3;l!9-fSFDcjWU}oo<0Tp0dxBZWPlBHT+{2_E!aqrUf)3_ zW*2l1u|}QX)co>Y)A5`zfqE&5w8LPEmC7GO_ zxj(yWtVg%h83`<7L~D_Q(r|SIYw+vlhN~KRUSTt*?%f=uV=z?e{^Jjao*+8q-eVFV2svrQ z{n@VXYV^%8H++)T@(LoZ5gO!KT5GPYpmG!Y~{zZG>z9mK|*r6@bIGF-0G0#EWHA_bKW+%a(wv)0|+-(RZj_)?sIdpZeBeUE+817)wFE+UrqkM?e66 zD;b(hhXT%lZ6R}CxPomVDTbSy8)cW7 z6%uvzT6l2Pa`#Crri4T9ba<1PNe=WoG_*0B^L>g}V(TI`g+!WbxN>UkO#uXeD2Uu4C`)+IF_ z1ZnCrBD`a}DQW6rG3@jD!jIPj#0a)y?>lOQ(ibTZYHe7ILP@=0Eu49&W3;9s-`dx;$KZ;{{JQ%W0!mI!rPx5v)H16%cchr>?P zX#>R88TycVTAq|6!CY&el&i8w+ef#=&d4z+3A>Va5G6J4)sXmj;VoESJ4Sjt2 zd=QMf2fbw&<~pE*>c1)&83dR07dpE^KDul~7|$7$rbF~<;g!uuep}zmznVp9K; z3Z$2|Kl>Bef{>Tn{7@_}Th7?nVlAIJ>G4b1?wU=UET>^v*^jYbLDg?w&ALS*Tksj_ z6Yb8X6L!U#yfHYsXRzxg*HZ{fzQo;qUaDk<;lVI`s%^!&(gDBD>5haT&{d1oUFQkr z4L6L)7^3S=AQVKut_HetN}krI(Q6!p4>m%B z!U#~yVhljGC3(y6=gjs*XTdwW1QPod9^3o$c%tFMH+`G+95!`t+^20afj$KdE64>p ztaV?@32a2pBrB6ce`He-6GzLPX&(^fHyq)2MSbAA_b!{G#*v+zTPgO)>V};GG)zQK(^Gi9iFKPyKmI8ZZXx_eWsrd#M8orU<67|P-qL-Xsu_BK5YoQtT z^6hNsyjv$H+VW>d2%spyWa^0fXe+QWa9?4W$?j8m$5abOB@b8G!EQSasXxn32fM}( zZy0*nN}5o+VO(BY^z1(B?F1YYJ%Ji|2WQ-#F(|{7lAj}=*V_adpX1~~Npkks3?1~1 zQC|s=%rpFg>}| zwU0Sz4M?^Rj>K4lx}gH#3ljz#lV>q?Trssye6^ z`H*~sqOP8ja6(7o9d;P89;t#U5H?cEhCu!$w+`o+LYdTiaLbGD=GUDNAkYBQFPMy= zNfmpJOG+yhS3m^+Fxdgr8LDsuz{_Oi1tLDl);{GrZ2AOY&JAl~Vh?&3Y)@iP=f zHm`ELami|uy?%0^#s8PY?G2)%dWE<|Sp47zyKSML0Iqm23>Mn`+^}4Y!)55hq3~cl z@42R|GF!o@0vuAI0CpWz&rVH(S_&5ac<8Z`tlRNU5mRyvclW$9Ywt6H1A7G+ zDORBe5E<@i&veY*kS}h_ah&z|`q*y7-djO~_qKTA@qD427@qPSCG3w?o=!YEBYCW{2EF8thD2{noAdWDnpxy>Wc< z^!QENyMVQ^!w!z-2wIh?y&Rrr3kBGRMr99^iog!|@t8X;Q0FG*VHW4w6wxqtzU&vS0kNOPg8vW00hOW!#o%H)RL8PPMc-yr9l(TzNAuKUU$GW6XKZ-3nS z9CR;Qv|2Dp8*27GJ~vp~+kXwG)EnS^(?&89w%MD`k3*euX?>9EV>O{M+@KV&C%LBM z)UBaVeQhmxzqIM?@ta+%#}qLrKS0%9>zOpGYpyRazC&NPSafzSEUd{uxdLok@}W-SOg-4PgtHrD zEK;h@K2oTi^3LFC&o8nN;4kh9I4fGKW`?6fwIQiW&?kc=K4|p?@!dYS+f2gAb`Fx2 zj@39kjPTO6>{2OX+A^W`>?9sFt#_CmLnrl`K>_s#_|+B~=NW@r%+jB?m~Yhi@N&~J zdlsbu7bbRqEy68#E7NYdT5_B`U$fR;_HZFsq9!`v2uCV+Pm? z;Nc@+XEZfF436MuJ>EvDA^)y<{SuGhfi zXL>Ya5M>>0R_C9;tEGcCXi7^y8~o^q&cmCVhI4xyh$r{`em zLDbbR`rx8S0vHDebyrP_x=5wJUu%8tfI) z_a@Bt*cQ( zvh2yy;h_-e49i35+;1S~TTIvO<$R+!-%X(rfa?kQjH5JJ>NKog4&77SV_PP!L(p4? zzifo7InF`6jscOq!PW-={y3C{+Ysx!H3UA6SbXO2MDBiyN6c66nE+Og^Z->0L)jYe zSQ5hJkRsa(6(gO6Mt2wZ@~^cgA0Rhqhr^n#O}hj4SIFbNgbV)QCC|@SOHuK@K%apA zLgPRAmE%u7QVDrzT@QXogi#k{a-NQOHyF(m9h4Lopl0{{xwu(Q2McZY>3tyCZ4d#4 z!!^DGqglW;Ch8%BMh@f!4N0~vFP)|jb#8`x_RaVc7jA(!3q@wjplN?b|-_ky)nqlOn8Rpg5q-!OgbB1zlit(7aKE(2>)85<7tdzZ?^V zb_O=`U4rlv@jhOGmuJh=xZBmzhjdZX28U_LYmc7X2X-Xtur68_zM#sSC5riB5qm2j z*Mrvsf559RhXNoAwk*|RULi!58@&rq52#2215rm=4g$IspjLG+IFG59QlNfP&rFC@ zk5+F&RQ>Bsh}`CRC}jxfSG&w%n{9kcKUKyff!FS6o%3vBH{9`XU5XbzEYD#Jk*WAY zqPx-FIvtE>?U(QMps4UG>UI7ESoB1!oAc@>AL|{yoP0!|?o#=_cd|qWddjLmlY?`! z=Oj9sT}VV<$~d9@KJnYB031I7W|>Hkl#LJyh=}x%RSZv!5 zPk-h7-@>CqGJ5H{#f!3`1UWT;e5s+Fh z!ypiHd;jKe$2F(AAuQb8YSsr6?u+o^r30Uw3Ar79<)Rpb259eas-RpZp(v=FAw8P~ z9{Cibku-<#5r-78C5$te9oipSYtPnWXE}Y{ zxYJ`4RtrALu%kE^6zvQGKL#*8n z?ew>j@E}EYRW;ooY}S90{D5!(UHPNZ z8zu|*tDoo>0Dzp~5Al>fnGChnX4+q~oWt%J(gk0E<5y{l{@QHg$TM^EVgQ_}CWC+M zUu4XP-^-Y{f_4wRaBJq$+kYlq^`WalGOoj4DU@vU6lL8ewaHe7Sm|&TqJdWSEcJ!` z;;V3w6}DbU>6Rlz1C0dZsJnWD_#g>gX^r~hI^cF~z@%#Qohc-1>FCBb=IL))eYd1JIz(H_f1n22%*bJKVNA!3O(g5VgXuwm#4vaC zhASX@GKq;wdOm}=Q1#?}!*viBYT(KaaiL5bATG2&h>5h?!m7Q&lAxZDazOtgE3+Ur zy?$-i9{Y!*2=Vgz&U9+n9ECnnu_khvNY^?Lf|Kj+@MLeBE`xo1kxOWI>T;(uhT}-& zD^ccc!PvHFsm=g$WVRG{sf2JuexxNT&c1_{~|Pvo5n& zZ@N|Q;mCI;yzg`?r7JX0xda*778*U!sK(60w)q-j2EKe}3Twj`W2)V#a+h5(^=~1V z;hte=8hH%%Cpx(Ne138EC`S%zd5+PJJ@a3AJ%8xfceqF3V65e}{J z7q448p%#GP5u79JP-7p_(Lho-R`%+vW{sS@W6{*M82hj)jjxRym`f(Y8Y4+4W`zK? zp+YWB`F*A6aiy($Cvp0)$oRVyObp5Nl(EpYJmcez#Rr<*VqQ19y%imP);%yn@P^^Rpv$1TlpVr?~5!v*6tFn?=v~TR-FESqvj3$6OxC_90yLXk+JlaH@tyuUa7c|gnr{?j5{*{V`{MRbI2ensqbm9 zqCf+gS@JSVTjs(N{VBxF3xE+|a&~{3I}d)e+p|l@cxr}gq625R$d7ks+Ctbbwi$$` z-~N@9F?`Hsx&Xa^LvDsnK3i>di#NzUtK5HpH)y{khch~>MB1*G>!o+>^JVwvEwJDq zKV!eQL!R&IdZ=pRb_4~=r^KGwLB?q1#p;Jv$=k%#XB%C4cH!Y`Dd%!1x3ND*eBe>@ zmz_GYI+-c)Z*YUh5B778@-U!lf5p}v%_2Vy*4GmP z(IoQPH4r14M;BGBMhb2)^V;OOxsa2UUW+K}^4eJ3uruHCrEj?Itel0!8;Gdrk;exl z2~K64BpZK`=`c=v7ii{tqP@N-dK*xzh3Y?cga;+GvHSg43|-$Q)E)6bciXhsBynB; zDk_`^nsEk6qngbM4fm4}N+t&dENKxl9u51URCG zqnR*EZC@$G-O1%VI!!6f{Fa8&chu9Rh>ol9TrKVHhD~L0gZ*5jExM}YSaX#Da2D5-fUv=0{#>wTN&lJ?ZV6RR5acB_;#s&-+LVu$%ElZ5#&k zry9-T`>O=O-(Bi={iZm`%$eCmo&kfd>9LAulkpL+n9gV;DQeD+@q7tBFIX#ul-TAY zxKz7@f+Li zZ>T3j{|4>((Gd1mza<7yoN1e!-DGk9+z{GDsPB>=_KH!-%RSE}Qde-~S2wj&3BsyN8kk6jpIhn`re-L|T=8)3 zJdv3OsSI8}!>so0b|+6X>t&&rSqc_plq;KO7+r5Pe0RA0SmWqdsY5`M+Q{dTdYV;T z^tJd8=S<92bOv_9(_W*9N>G}>LE_-uc-f|(2$_-P*L_YbKdQ?D;18{vT`gQG+$E6e>*P5kxh{!=``c+?u#%lT$%Fz zc0ShhP7p>ZkmQzd0iw}LhFF`Zna!AeJ@qT1BX%m)Y*b0zO|3CUD|-(JwxQjz^C(J9 zrsBgDdtY|l^Pkdr!##wb7Qy*|+Xskz>80s3eac__5VTjuu-%AT{+ke}~uKN7B z?Cw6KzTO(=xbC2hWt+>*j`!O1Zny6%KA=A;(-mEE=e3v#bzij>)GUZGSOG637v9a3$} z*Vwp8{~DS8Jy}D3ri}+COqUJj)G9SncxkgR!+e0bANCG>9AWMPwIPJ6Ac^@t~~v@mGnB`Qu&- zJaVj+{iWCKt$xi#h^o8XgXES`Q~1lsHAWS=lq*e33*{052HMGd#!peFt>+C`#_265*{Bcgi{^P2Vl=UK z;*FUX$5)nE9(T#F;ORghu{5d0MLx1>nN;t=rtf%uo>y-Iy!jTfR1l)T%(=T-(@NY8 z*P-3(?YiDPjcqAgakm-V$I zzD{wiso($)mXKqIyO^4m@<(5>roq$9lK%MuUsQmt$ehZ{6zu%Yq_=G00- zGji6b`p)!!$H2|ST>MUw8}ahtA=yBSNP%wg*b9@dBL~7U{gFyJ=g1xDms=wHrkjH~ zCo5A=zjBM)r~NfWsqRx_e|}a0b+~B)$j)SBUc|Pf0jYB3v$;3Q=g1RuD5v)=E_116>=|MN zDXkbEk^G?5n%VDx^`0QqWUiU+lA#xwP0vo~S(=^>_<9t#XP1R6J&Jv_A9mt!dv1z( zsc4ZvQBF+2$&l*{s*x~u&|6w5k=F8zjYUHGvExUE&`^}59e4B_*V1i5RFCbjCyk&> zK(2@ApK7*>V;VBc?(bQ*!AVlurtO-K%o~|cU0HkHJas}HSvQ`v9{L>>78ziZZLyhxP_LoP^^}X-5H*@6_wv3% z(9mq6noMeZ#z3HK3mE^K+tzJ>avWiuy&Eo1W=@ZQAt`Tg@ETG-&=l$izJ1Uh?(MOg zqG+4{^2#G}MVKfx&PuYsFMkijhhGApCbgRa4}Z`eI_}Wv1&wqdFvXxtbk+VWPqlc4 zNuon9P%y^BGp)ccv%daxsQ!!8%J$1^NF#q$(9M!ZdgJt`;Jk%)C1?19D9@0?xhTb; z+4rGo`s<9h;7fF=Sf$0Y;~|uklHcHh1SyZt4|n3cw$+Qvvf)hlxv0AlPws(E93;$m zOO_`4uox?gAbyLot3{iuZ*;z$=nM8zdX0Y$Qo)z?c`0o@Jo|hU(MkvOFFoFW8f80# zZuJFnNN^%{x+UV5SFuX2`6Q@`^*A^Al$^k$Ic*Fj^(>DcCEEL^V%kcpQ^c`+q0oshT14N+0e2mPi0nMwyxj4aQwe&Tta+Fp{ZsS6%i} zawapvW@yAMHYS1?hwTH3zl}xCAvRdI~{S-bYS5NkU8gm=B^qupRWTR*|%u`<9Z`B5On? z-`X8%V|h$=>2t2WF$h&72^sTY+W++-I6~GN!&8UwpTNzivVb~7HB&yZIcyHG#K%+T ze`hM#T8yoS;4#?(y%1~yit?an88+?4cczF{CDImSJt99%&)li{)3qj*pZFEm;we$x z>%Uv)=-1M06z7=whG?=n>V7&pgj95V9}i#IxeVuLF0MShJzioeqo4#51D1d-ri*IQ z>GEvsIGc~jN3`L6n|ayPbyDAkO(AJwtX8otB*b4N5l|7ZC$HRbJj(6$26@ssBuyS{ zds{qPyu%)H@*Poo=(?CUo?~RRM1BFD{bR|eJF&d?Ie4RZ<3ruOp3pi?U-_AZU;5dL zbBQ_&Bo|&T#vNvEcK%i5Cdc=Pd|XA~rkGWjL9A!a6*A%Jt_3diJO)Unf(g78FoSZ& z)b+zxCDh*-2b^v`_OcP`&RoJ^yXw9(B?961AwKdD(OXkxhd)B_YCpu)I`u{dvHJvFY*DRPlx3shl&S03x>pb8bx1#ZlVit?g1b2-($^|~p;Q2YpDgq91B6|?eR@o# zrqDt!`PP1Ca=fiPMwy2fCj}2Ve`rL7S|+jLldn{LG8zvTArz>51Pp2lfER ze!_n`Tq5{P$38lg{)GQ9%^*D?cr>G?p@S5nf`jL0+uQsL($O3L<&*H^EWXpGhZu1gBK|JWJ*r`-A0Dh0R~Y<-f_Ts>2j4!wiZw*@qtuAxL`X;F;Q( zo}H490mIgeK~%8M)FE?NJ`o-MZw&SHRr>pLOS&Dg z5~4IcgQ&HAWY=9O$xS<)%PcPO`%Sjnhgh6X_!g|K9erLpTra;AQ>z@a&G5YUDij$*hV(U3Mitf z2MofClophD_WOdZ9xpaTzvHuuQ6AeM3vy!t@IIiud-U=1KoRen+hRLT%1b3Rt@=2k zFs#te=XHE9ah8MD(5F&*rE}IT!Rguj-N-!CvthyWGd*Sy2R3@of1DYG(giA>MCx8ii9gew zTv}HzcilTD;_-K;V6BYM?)9$kqK`&)jo)f$_gCX*S$RP-Q^W236FZfm4wO?UE*8l&+c zZEcxd-p8J++RrBhmWmW|qm^9q6%w7keq~EOaPH9Eu&Sl}u^cp*p$&&VsDF1B9H9cg zFeW;whapd58Zckox4ZtrEihyMmRb1rUoZYczT5N2;GIMHY+6Cg8ZH0ceC7&eK+*rEHE#?zshdLc4jyCBW@3ZGCYfIjt z^iZ5qa)QO|#(T0kcUGu-XD^sN7tic9^jQT-6o5oSx~Xx!F#(k^CF5p0B)Fk~nNo6` zIy_3TlyXhYWTG4z+U78ZN+jJ<_E`YgBZ@|{_fLJa#3(x*WTVLo==-Tk9vyBc%TE~v z6cXbTE^UW=V7=4=$`br*dz7tRF~wYBy?X4rQqf5%)q8=v>h_0UwFb++)E%%+Mc6>1pkFWD?luRxoeUo#`nXKc0=n7jXu z@BUS(2cQ4x+dt6=1ebwQh#6`}Ko69gnO0WJRAiL_3ca=aCCtr|*>VuO}E^xxs&%%uHvfiIfOtjH|?tLawG@COwn z=dw!e~UyjZ_Vo27!2x90Ptg#AC&wo+al{f_dx|3 zfd4rdobhb#kG{?#`nEUU)ovB=NkRL9>LGl1fAC%!T55W2QXUHO|I`msIl=9F|6JerkM zcC6xqzc$y~4s-4^%*(Tf&f@ai1^Ww+KQc9O|oA|DeypYCIHA#}*=l`p-M%JqDOmRP3tWCgT?MEBwrcigmK0SxV zVfj%+0u%qMt2E*FuF~|`{paUNuJH#$%d{$U79}NJCmMOPp3iFI+;iqT)8-@7s+Aua zlOo+ihETbS-n@ArZxWWZqj3AjJ#C4P$*b6>q9L$8W=MfnGJvtS&a6890>)wuB=Ij> zbjgol(&35@R?x&OMA2#4Ed8Sc1;7Oy3@Fc~P5o#DiJVmtz{?Nd+w}YN zGa~H!(JtwIku#)yTH(iM=pFep8D%tHPNX`5I(`d(ILdih*qG`d0 zLz!8^+?Jv7_wlCsk{|jhL2s%%Hz=kTA#dcURCboj$x@skm+&F|Z|_ zz;RC`&WP__$;ed7$qOk(XIJ$0*QP|+WM=7kjgZza9u;0CD>~?wV&C4{p?%`%Q|1X@ ziT)lj-cs=FT+XUhX5PCny<~7xTC!=&eJG}9&Qdj^XJDz96M%6yODwmQjH+#{&K%0A z7O-1quF`i1Y#SW-pVwsW`uwx&-)#a6KV62jXLpZ?rpik>BY_A1FR;n0#nL3i$L5KF z>$<#7ADN{wlJXMsly&OfUp5iStt&Zfy6Tb7nX>}m6u->@JUVz=g7XD6O(IJvYc~b7 zzn3lFeFs-rSk!>&H{(u#>hVU?;XC^Kcbb{Hl?Hm`FA@_yd;Wmi(RSv-{Zl$D%0amt zBexO`?iJhd`KG)WUuD70e%^#MhNS)ivT)TPh=P>z_$^*?bj_du9q1WI^b4h?6Omc$ zae^NJaNK{OM!3ii;!9sX#<$MH1GEFM7kPR2FL=pNuL07{d>j^ema@}de^BxZUe0`j zK_q4X0~%tVX^Y;u=s2Os^mkx+=iC-_6fEz@_o5H6XvOPaG>>XHYNT%}`mOv+@&WxL z1D(b5ASv*;>3rLj@7@g$N@#CSb{k|ppvlQKJ>Hdh*mo36#T?Yn7LOWEK5v~2_*B*G z8h9=1b1UL32VUp(Ux}Q6G&iBu))$~& z&-ej~aU1HM-qLhsBDvY?*RU3(uJzf7>t;V{o5TDu=9E5_MkK6mIYrVwB|x+&Z2ZA~dj(}n%+9dpdKnw8=jC~gf;btKY}EP1NFvgCwx&?dD-FY*mugosDP4Nu zLucJTi)e9QVi3gczhsr%(?uFG3pK5s#4AhIRIaSo&X;@{|7C$YlNCg&d~3!=@Ds{|zIL{inI8QF z!Sv;!4~T9sTKUNcW@oll2SfCf25zi*dhY;Td9 z-g1e(L~7Be;vMWdd;0yyuR=(s_UrMpD5wJOlO_(moIPGBEwfmeu962i&|`U{F6wt3 z?p3bW&bBb!l2N`Jpk1xi`>3FE`ASPr73Zg05f!pejqIcdZLZScQH*Va8hO5u46#qJ z6RD87?@=ry1#pZ|6d<*G-VYBnLQQS+<)q|8x_F`__DDGeV_#vN`xLO)F@Kq)>ZtyJ zV7u@dOFb>3e+=2;Zen-ewMU^JGX-&K`q`-Kau5!^Io|LGgGKA~tVpYK*~!-K6-nxx zNNw<*&rdVLs|^DynMq1-^IQK@#77r?9!0e{U8B7=9mrhRjd7zYAal!02v5;h$Pg)g zIbVsDH25jOM>*Ws#Bp3Oa-DZ(cOa3gJKF|0$c5Iq|1G{{?Zw7nqkpn`Ms>|D1|gfo z2Iw6|*OY|+CVJ<;)Req3&I{UKoX&03(W~L^^ZN7c5=xo;s2&n?b=ECM_-?~`;&NPY z3-{sEw{=R+ivB({L2pGvv8i!^vAt06wMSQ^UmviEw%7C>qIoMr$gI>l%ixyfRfCCELs~n8zE}WJ?c~*2!As3B%d9N+3y2CST8(t*d84M56x*FtDkWr)G>EIS zc@__wLL8M=O*5hFrZZCZ_o=Ppn3xnWHP+lV3+V&@%~{<(4Pw*AoX_-(U$3uCA$}B7 z=|P>c%NO--!7Vpz5GZXtIke)LDb}#@y})qfH*I>oGNLJuEnE7V5n+k#yt=hsis5&l zTv-`{HgUGGp!2?1ZdP>g6mdgQ^qKqdWgQz&(#w?rsY*8tO~2H!#3a&(>v0fg*g;(} z_So&C8%Oa^@j};jyoyLpcS3^b!@U(;xEJ>B&-T9jyQ563I=clp+>U7}c9HKE6+Ry8E(S7(_9V5!MsjF#=xN%!F}`8L(2$t9zLa#NW=PYU zX@AXbtc&T)`zWB>tW}K;*js4-$ibp0&~Wjm7G_o_Tx1Sn0~pO(_~{>^z3R>n-<|y^ z{5Go>qJIVfM%pn;ko^AY7D#VwraQk1wzGXr6j4MgR|@+S$tU{vG-tZ*F1Ge9M%~Ia zwlv5N$agoF8y!S-gBLK4vhvk!oVp~tjSndPQ%P%^o<#&dvS@j?X5hxh#Jk?0yGgfB zDTA8BF$540{MTFfMp=$uDBqd4&WN~s#3Nq;(4V6uQ!P)@rmvgpBBgUho17zUA5pCkHZSE@%y}(nZrL{v{NoOnV@ou*SqY4G=CG+DyzCyRiei&S4dpJq(m8J`UHTmF_E#DB?wlPOC5cr8sERQXJrG#+D z+1lsjiMgFcj`kPO{gKLJoIGR0s9Yt727V&unT;`Lh1sz}yO`<~q({^Ma0ZI{IKi&0 zYoR-rM9lDfF!=&DT!qi11`ft@-7^_55R*TD9*IIof+&OlwckV`jDxUn%U^hBQ63wo zN(jNxUY|-{k*P-Uv=fG$=|MrLZAv7Uc|kkbuiP#e!CCQcK*!K3h?*lN^7DBIzwGSc ziq)8UbF9{_tZItKxJQyjyG-Kn3+bq?M->wlr>Cn!u4i}Ip`s+feE#tH3=I9}3l8Fj z?1^6-rIhJ`adZIGs0yBsY4c+Q{Qiz*sQKr{l@}ko*{=@o3FY7N#EXe1yzHX)rhDyA zDh?E{xoqGNV$KK2C3FuCv9ty~46{nM=T!Y>>U#KT@Tg@``OWW4t}GdrEI=SsjsXFE zrr*?t(<;wXt@E)jWFvcl80gT1O{#$>Iu;PG?YW;iV%9GRuBf2Xey%hJbTz>>J_`vwM zY_YkWNwtWH0K{mU_ayrdZbwgi6j45XC16o}c!-^feEE(C5R368ArCXuBw%tmdp-58 z<5(O@wXB%9exOv&ts_fW`VycVLM-S=5177%T^WhZLg*0CnOQv?gDAMuFdy#xL`3HH z7HN%bZi~=9C;N|K%d}hHt+6F-==j4X%L!*=6-cq z*j^&dHynN{Z&;rI=$Eb%)V;%U+D_*mO5%{r?;h--CIH0W`0-jY8q#J^gM6d&ga0-b03e3I z{5L!c0<2T3n`9|Io-EVVnWJV8$t+X5z+-b6F@GurO;#3S*RatI`$y}?@ty>12hNSM z;mu)@Mw9aV;|Fgq^!v_~T)%p#v#RnjMVVCAY|6AFxvXe9>}6D|Qqs9IO4B0O?vX~i zma}VoLF#<8pg2DwWj+E=epUZe_YhZW0U@GUOG#5L*gOgbuh>7-E!RLdG+09Yo%W-`g`6Q?{%Ww@Z0qDZ?{xRm825g(UkLlKGso6(K1@a`xAP zhU50PLp4@iS$n`Lu#zPdbupg+qD0J~;p_lsM*2@Ko~I|JWF-4E46o}Ue_YM!cc(b2{;UXoK#HPnSNMP+QIQ zo`0O+7`Y4{%>Sgs{t)h;qY%P_Il3YB&xyW6>OK{H*SW)<~hA=W% z##d!onX5zx7mG_9V`>tc?2W|=Aed>Mr0Vp6u|*lY&DcB))GYKlaP^vfeK_-o(ZsRD za~4(B&&jl+g(ZXqq=zA-jMIWvgo0?cFVEi3yZ4>x+^A_b%NE)UO#GHHPy>pB-pht3FL}u6L^^3wIc|;gDF1yAn!BtcCn@ zBOxT#!j~UnEu=h(rexd&3phP?UxE_0@`JkUpTFil_CJppZAzrK&?+-C(#6l>5~Rxt zP<3ZF;=9SAyy*Dq;&}~!2trDLy2h)cdkAWs?$1UGd0p2O;t~)1F$&4>rzj+MVx^l% z@UrffqNDNdrRbI_smoF?^ZhYe{K`d6QCJVU7}H&gRI>LBUbXU{SDWvJ+oTQ+8mt^5nEjb7E)Y`eT=m%fSl$ZoTo zMBxv8x3qaKk;w(9ccwM{ms|`qd_^XSPHzy}24L~4y8s=o0M>uv#b)?G##@Xwen1tY zF83_O{8i62^KAx2g%3j$2(g=cFqGXX>D!yRFXw9Lp2QiJZhk5;2o(sBl(tGq=9wre z56L$_1-!e713@D)ardr;Pi}OZ?(7cM_w9n5Uf-FrG8d1Uq9!f0Uev27Z|IOlsvr0b zj$t0)Ux5Xx2&ydOeC_ZHD$O5eFM5J0X)zt=m6W57TrQsn${F+itaInge(u%W9{%&| zf}w40X82W_fr?i?2|mue^E!JIgljmDzC?3cVh#@l`CpZgf01p&a*%bIgzUEsHzT`1 z(F8`Ug`$#zpVNqCQV|!#cCk&3@??koAz0&^!<6dLsug^cUD#?cVzr&l8W13w(%s6bM&GFuDx4N%MzVOSM*);4@)7~B>$JN-CaLRmwNvf(q&AI|8t{=YI%#Ox@=?Pb%jvCMwt{4`SObY z`kl{^aLO+Xi=B-v)iU8wPv zwJB>;-aJh&dv_aiufM;>T_8xna|Z2@i&<|t!D1;ig@1_U{xXaC!hO@*-|e?pD$`8; zFlLHpZpct+zWg4=P8!WM+e9ra(O%G~vxzHzdhbd6ShF3d$^D7w>|hjW&lVH$+OL^; zJma<^k(JzjHmH2jah7c1JA^ePoB4!?m&ZMouAxu8o7W+gX0vgemVfFE{?K36hcq}2 z*;srMV!t!JW1-$t0M}W#^l8OoD-W7ZvvETn`+(sei3}NV$dE~)EyQy56XBUbIDk@s zTn1eND)$>E3W&-U+6*0rGyQ7zDY!|yhn(s66Adj zqZbX}vtDDATv~rNv)*e82mZGZcpKpZ%VIjiZAWon3!CJ2p8(W74KFA4)tEmXIQ|mJ5A; zjWz0uxhdSQ`XG!NAgoHcDfxw6V~MT~Z4iEQ7LUGu>WOg*6Y?2eBPUV=Dta(!d?!?K zrjbd8hSpyrLVA?1tQYFf^`sT}RC7z=mW@JUWX*|d^F^?rE8BhbniF+F(^Tq;U^Pl? z4+_cGadQ&#bsSh=`oZ&bwUo*rpF!j|>>w~K1MBVjUxgWQe{;#v*jKtNJbu?j{XS_s z$SbL?MF?I#{?KA!`_pW*TZfuU+~T3whCu!A=U5{v0W4^LNvkp=zJZte0uVyC7ZV`w zaJS&-U-GJq|30q@ACvsXdF27?TuviltwH6=x=K(0NZq3uQhb&jtB$>L^nRFq%U+}y z^x6@Q=3pd+#|o0h*LY&X7YmKE(YrIrBC^wwXv-<#4v?I04Ow`MWv*$M4%5!CauD<4 zm^_bzB$HAZc8a#R)*>DfF5engLy?UtxX$j&kxF@;-S;D+_3ZRbx>B5u9(25-<4P(- zWFOQXGViP&J-+RsYKweryE|qgFobMA%JI#THH(LLsNwmz-U_4AlQD<)hRA^w?NN%o7j1W5;(ZBuhQHl z8zoBH)K5*f4m6$umGeso@LD!cw*?9GVus8Xzr_kYZfY_3E2x`eCpN;~wq1{z`VBjL z*1VD22}K7(c_DMEHBc=;eFk;f4X(F{avYlTqcsWAAA_3B9XJf1zK_9cKi&RicVpa! zR)1oUW*eP5r@-IMKDA}8S0McVJLEU1!D$u2R70uy+q;b1FIm5iX69ZCi15}AAjSR^99GKL|Ho2cFWF#Uf^!mf9 z%SXr*Vspkm zF1e397aXqnsPqh2nYJOSBHdI0M#tvR)c?URAr4tizec@iv=%vQNo92C7zT>O9U8f# zS0`szZr2z?l}U#ZwSAq zbCj`H(ZB=&7-aDPXv}z44cR9(AI!u^r&~Vsy5LAK32zL zL^(Igi1}AjEy0t<3WdJ=p8GB!1y`0Hr-Lw~y9Wv!vNGLEJzsxR<+w4%g%J1Ch*grL~Sm z*K!u2`%~1XS(0qAL&uSFmbRre1(@+m?scrJ>KhM$=#2YDeTq1aEJ{zFgMCf_VSpQe zA~2j-YxO{z*hbmQFSNdxn{WD%yUO3-J5$9(EO{#~syEe7%ii-M)Jom{;pqG|27~#B zX<=uL*@F%-peXa)(onbSwIhk=Yb(A4D^7L>E!l^AZ9V9fop7VHKfK9Cek1JdQ$6m2 z^`pJ4ihI5yRhYPpH4zn=0KQ4EJ8s*F_>xi5RkK*R9evGQ2CfShyCVh264bkXvntSM zI0y%)pMwMY%bxE{1BZb+ohGnM$G{J7do#zr)_!L?CJVV_$LP^WK%wY75RF>RkSb{B zH#x8n;H1MQ{Db&90CAaVfI)KTkIGYBH>)!8su2K8eo&1SYe`XIwaXwA;F zcM86$0~s8-dxHco_&1>6bhnLI9M~lHo#}$PX*TYL?jvGmQ>DB8Mu}V1wRX<3+d9g3 z`o*wOvwAwa^P?e9;jrbl<_5QBQF)W+Q|cis`)@gDVpGNW$+vnG?Hu9VMi&B_?^d<8 zU77|nyz5{zZ}7F*;nB`=$PAE|v6Vj0{}0as$r1WS(bWY;QMJ;yOg3iA3b?9})D97x+^sOV16buwnEoZO7E8bLF(Pk$^D%)0pl^I@(KpE zZOkB|UY;zwiK+xig>QrEwEwtLTInEPV!25~^1J)ZVGk8vQ?nUMc^&v@~>T>7ME7y-h9n(NB8YlBSUS2Z}w|! zuMf#O+*c2>7=Lv)-OjbU=p&{(a9E}L(-fpA=W4><6-V9Q%oV`_Lxp>}yS-jaF^j-o*H0LPFh1yS+O2#w04+({^9lVeK2h z9}au#;+fr-+&FHR?$z6TJ8;!X-3s~VSc8te--W+T!7!tla8oaEE>&vxqNIY|AccB>A0ppQO47NJYE%zTcIhteb zx8~HX*J6bq?bu{@*gF9i9DEnO!CX@Na{}>$)RPOA`xoFO^OFxKWp8UBoP@HB3cccx zy_>XCOvL0syJ8vJcr;7=23gXDDUiW8_{#>!`eO)%ws}pU+w&_nahRR5u!Oc%qT41>?hi88|As5&s zL#;K0M9{?S?~>vg zHl=8m^ajK<-`RpKIWP6e%D1Nun*cS_6rQsSIK(VsVrE)8@wUW3TCQAxf1a8Q6OWVo zb|b@|`Z%}0|MTnD6t!kWuH1dt%M~w6ucCaEhYQeCFU8O&gF@foxJkg}i7bSVJeoiYc>`Arkyn+W+RFBleH|!=oK3mo4r<9}3 zrsCh3vbQQ)$eoQprZ@W74dRwb5Vs6v*;KYKa9YgkwA@luy-JNAA9wpVu9>VyN=4)`76UNBr`m z%Pn{8WQwov)Znp3*%y(TrUJ-ofdvze?IJ+!0n4^;qeCuh?>XKAKQH6NU@z1=z#iso z!p2gH>^Wf(?W$KHlDE-MP+{BKmE!TS6yBB$RNi{1>ARu)93lBYeS2^Kt*CDwXj{tT zJ|VruF9*xlrdsJp#b!?)bN?Pw|M6pDY}+U!MH4veVedxVlgukc9_rVP)&d7U_NmC` z`t2)wk5y|Lux#CdYmuB@bhwSEtS4Ui(T@ScfIG-!jDnjkLCWKU_~&G;u!p7KUpa02 zgk>J}&UGwvO-AhaB~m2H!S<+BVBANIL*>=B1>zn?vaDM%sj|ySJmt!IYS}w(eeij| zS$=iY1v4?Ebb>j@glHlW@jy=m5A+yr!~;FLZKdo2dgTk(;hu(1HOB)BTAUhoziWT! z6oWghG#~Q0C8wJERR*|x-k1E~Rrl5>A2y@lcXz=YVD@TiDuIgH9_cVmHNA8i(g?1N zwL8fYj}L@Q;$_LVb#Dg~sSDHJa;ik1-OdK~nHPpurqNmBkiWf2s{tQn{Akc7wn5Rt zZ2H2(1V^O6o*Tz-(cQ6W?VQx`0Iv+}4Xn+!GncMDkYws&c?|`X&Wt_!f#~$vqE8wi zxnMGJZMGsd@1#%7;c=aV7}54DfnXxrH(VL+v|y>!I$Q1`r5w&x#;khrMhhr4F2#a8 z2m83sg0*Nf#Q0#CD7%U!Hbjf8oK8?-4lHzFWu|6z1-nUg!1ky^GyrzE^q_|^wtx8r zRN$%9Q#wl1dv)(hJtafB?0StCjFtPjYk&FC)@te~wm|dSccwaLU=1T|QB!K5XSh*` z?R}O)z3b&J_+?0UCRz??7aZ0L>Hq%V9TB$=xG^hq&h=OVsA0G^&#M3ZXw5LZWq-8w zr$nB!uhHK6o4mvdG(@G-q{uf3?=n-t;=JVAPQC+jEvG-{cL_8+e9_P|L9FV&?c1Px z;&%SB#X40}px5ssu#f}>+v`bhbUez>9H|SQs51kLlVEqS3pQcdn@MrEob8|GvzPCz zrW*9~IY*XY2cJU`1?x-CE=N@78%1hMn_LaPec+KHdD^bGhnrQV%xw7{A&hi9WK6%S zzsYfa)I@9a7?Q^P3cln&EXd4s{BJL=Hyk8g3?U^-3`>s-$&IEt9@>Ge)%B!I_kNbz zZp%(g!BuEK388+eczi*xCTdweDA{Bw*X?rIHNFOkk<20bMaHH|szm>(P#aUCeQjDa znYsW`(5)t{w2W*X#qW$T9?uXwpm?NwjYqA8nN&Uz9BVxB;&Zl+pElE2^3M3EE{j5| za9u^Co9U)AMVpaQFITAR%gas`i?Rl?<}}GO_kSGaea@2)I8h4favab0WCz(>rZ03} zqkH-8J;YTS&m(KR>Bc0Xy>|5b3l=J_nTDNEXU7UJ`?L+hZ6IW0?%4}lCg?ae?Z2U^ z{Wg1x&$4E#t?h2E3|o zY~Z&$vU3qqjhkIV+}LN1Bs*sz_~US_Bm;0&BPEvXhE82k)t#W{xZZ>y<{y(@@^j@V|&8KE|?x^50C&|SxzOdgq zHgv*+=JWIMoVC+I;A?n?2Hz|0)qpsieFZOcgH9~4Z!Rjn8SE;XWIGLRaKe|&KZPgJWb`lvLA@7ZbE;?I>9=-}|*h4rhB@3Rnhsk@C-sc=KMIp=}hus7@Z1b7)*hWqTecT}HVy;fi z;M?v|UAtVG6R&l3gD9R!_Hv%QBeQo(9OuC&k0cyV81telCE!Bl)s znrt~a_|_LY=@7$!%WjDu6Z==?HROE~PFuE^-A;+VMco8j5)FX=>6suo{XDbn<&}S+ zZkm;s@P~8fASVKC@J$A0&_2vS$ysGK9U|$T1G7TxPo4)eKtk!5p}%|#!}j-FRM z(#Gy$cS!Q=)K0r0z`lmXB%@GB!~ZmYykFjG>O_x}3iZRN9v+1FEbz8E_@QAwo6h6R zB=e;AQ|6_p2Z*;-+qO!7Addrrf2~FNqu2j*CH{xcM;otKbWJ)xZXFlC|8^0ieC6$t zM7`}uuss^mt;<*1UX}ztcT?M?z%!~jSh)^;188SD8q`piQy_|F)AqL>gIwDKJM|{G z!t7R{-r@ZTkYrXvot9+wcACRQNHW``Kqz_FOz?Cb03*4(u2!99y}XIlg4btdZl~Xj zR#)jdj1L&&;>y@FZY(KNW32sF`|5+wF5IZ+EMRwA4)tK@6m^!I($SQGn=||<9Vh@I z;5Ngf1Xdk+%kQQoY)fb++w~O``S$uA$HZs5Ut1_-W<0>Yc}9qYbdkoesYzP;=28{M zlcB(J;4+>I)09|h&gBqH1HHboC(*)L1hZJnN?a%)pLd!3_3um~Z?Nf&#pi(J>Y)hS zB;%}o7{nXJTcPmYwYJI5!0FD>@pT{#>2R2r&p_7%+zDjr)7ES52Va}oqq&a%o zWFF*rfAH8~xGyFui@A6yc}>IOxDYQ^Lmz&FQ&2RGLv@A2JP_zkMl(7~@ACg&&hLuW zSqdK;GQmH5sU@hei1!_?-5(QXZS#M~dlPUd+xCB4Pm4+=JtT!Gl1kE|v|v)nmZXh{ zsZ_Qkr0i44k}RR+Nil`8#UvzICLyG<@B5aWA;uUp)Bl{g$5_(yKHvBE{*UAP_C1cf z={EO0_i|pJbGgpzTm*egn(3>kJBax{-6^=m9i7ATlh3ByX$g`V#uQhyKxU^yUvvGf zUG=WiyyzXYGQB!G?LG?GI^hvv)uj zS?@N87&F#~icm1h2u@)1e~O@(>ab_hj1^3#b+W<7jWXVSX?F4;|R7Wf^t^wmho!3&QwzzZ%MC$pX1qbQ}zwG21YE5kqPMx26VGMTv^D&fY z?yKUbtES=UEhvhZL<6=8PgnB>6>zGDFu4DUroK62S`0*CENL*eu_RBR4`K+i4!|=^ zQfzbCe5g7ZXXBELLE>r88o!VTo0pJ_z_pNr$g4(3_IyhUT=~XB<;C3|VFqk_6DbV4#>4#y?)TG}lZzN;%rBoNTb{@45ljIBE|W z^r}c}YDJy+_XlS(-Twc&jzqq!Juzf&zs4i0KNiXyyW~{(hgS@?`j8u>U3BHp*HJl% z1aF6~Jfn|Z(P0c9T{@m!QWgVtwdqtC{rK5@*hmZ;*(&!m= z2621Fp#i+CmPmcA=ca_c5VKz;hZz+9n3e#FZ>zDL5)Aq`MmE$k?M&rF=qMC#mxC1} z!TRu*S(|k~+)ZDuxP&9z{n_z7-5l)_?T<}u%p{u=6I^$kMta!p`MVyr|9lY1qa+sy z62ngo8jFD%+pMUuCtALRlV@p+s4+%;#Bo|8oWcAYS56ZTPg0NYOI7Bt?-ovwe8B6< zMiVZCoCSmRbn01fCL#v3unjmDO1?Wa`OfoQSD)03K%cy3+O1A#K-Z<8LBuzpZd1)# z>grb&UOABA_@b5j(R#28caX(9FUxIZOzC)tm%JLaw|6|q$QR|dlKLV+mvejd9*<8G zq;4i=XN>L%Sm-X}nS4GC$1RIH(Jl9ICZwYU$CSaV{da8Zce{%u-DrG!Tg+78utzReNE*d+Ak_`CjGIFiP3 z8cgJ@V5w|N$Gfn=aGOQvilr85R~%GH1T*_p%{}=VpJA}>Z**0ydHxiW5EN@D`TPUa zl5}`m zhZ5Joe`{5NgXp3t={=n~0rO7gw2H_E`<&gB-PJA8>r<)wekpw!ap9XJtNDQjI-ZA$ zHhtT{z4__jtTR3lqENTi&c*&bqEh~!KDEF-7cWk|?}(x0pa)g8z$uo1p*|G4Mo+57 zw!6c!ab1cx10dt2Cs>6^SPor~FynR|@+hE~7`R4Y#S&3dULWBIIAdyY((&8CC16uF z8sjiTaUrIm#8V>9IRVF8bFrw$$8V8vlg5M9l%TeXbLGawuEwz2X)E<@?Pfc#IBxd3 zF8|(5si%Z)i?=ikz$IyMl^8Ax=9=$b#i=f~^ zSR~FuyH0|bcBkpfy1_mk;{r^T%&PPHhbd8uj*Z@22wY$@+~7jO0S}!xKuADsPPlsm zvKlp%XxwFKM}dd7n%HRo4=vRy-QcQS@oCRnPwZ=(CEPiq%h%#p1PPdZv$-T{^L$@I zVF^<0An;Q3xA-K#x=w+WL2K6b=R7~jDs~_c9zNOnfeAZM7$hTWxj<8qZ`j=JmcgFl zSvv+?;uSV`ygEzJpso?Y%x}*#_gQ&zuV}V<@*K44+NhzZ|G?Pr_5a}LnfHT33*|B7eUvz?jP)eDgKRxHY zP;LjTeOxUgR#j)uj@QgsU|^!O+~qiA$KM|;bqRcfGvs^jlCr(*RZK;r%O4`#dl>!b zRgS`wGluZLb+b&{PF(8JJeQjk_)6{7riYgs&tnc@-R%X-x%wpSl6!NV za9lnZmPbr0!v6y*AF8N6L$^eA{*6PgZ5z9qHG_}O0(DpumZ=UoaM&S?F^P3p6;Ii( ze*lgmi~)p!ZIVY0&SD9Rsnbm9G1}?-6H$A8j`d%Bn7CG0XxHZTH+OuW_kpf$F!V=i zF=-$1p72Vt*Tx9PWq#F)w@!(?%K2W=w8iPKtHPZSFXw}Qi#*(a|7ViVZGGW8nRN?F zUJA`TD7vNkiRJ5XF^i#&!8LDjo^?x{KjIcydu`OtsBzb6DvbSf==wd;V)T{K%@7&& zfw049LTw>21~|uzU1V(HQ+CIg$P1O8ExB^%LE<}~kVX3zb!I%{EGWyr<)9vAqjbty zt3XP)h$K8yYE}8WSM8r1LDv-4%^;VDb~)*Vx&L9Ta|3dOprHI`r?$ek!_b7Bk6yH# z`?l<3?xh>-gd;07x5_}qcco7aMF!OarclxAcdmN{lyg?L@aPYEzZQA7>D-yS?^}3a z+#Wmtp#1xr1cZ!M{rPt@d@I%rpN&#|7q&$-Y3T?aV|;NFd7`_8WAYLB8z>3XWXuNO z;3Soi#U%2$HUvgusga4VhQt%`_0-21t&rVo|E(UvowP~`uX2R7ycX@Oy#OIIZOdzc%asOqz2kcd^f5(;_;*!-am0jA3ssHJ`-x252i*ymSw`;?h`KG3*U_ zF(ctk`#s9AuRB*elwF&}nLjEX!W}06RI38bMc-nh+qA$U{k#=P=#G({vE|wO7(Q?W_cgh;lhj0Ty9uifiohJVq0Q8@E2K>#|9~=-_ z-nX2J$I%>ceW*4?-YR6?1b0wn-er8@m<>*bjcB=5G%AGe{LWc|i;WADvqe%a=5+3p zSeHKQ8YHfl%g*eNJ5lJ6tidDE`C#Uajack53XVYj-lsnTNh`kP4m(>aY?98EODBA| zoritCA6+M_GI$AF1<_&rAL!ymvYKZ%#;aBI9yYLBB)ql#lP%v+7}VZq{K2up_~vMa z-8~tL>#{lN)vJ{X3kJdlGe}_*#;<;(tOEGE8ctIf2B)!AZE_2!9AStx-VU^tAwE(@6 zvP4Z0ou?sAXP9cs&med4rs?54&+?#a z`3s~zIo5J&MAG-B1%umAV;|Z-jJo&wKJoe6)Xt4?r}%m+m0F@JVUFE+#?~xcN6Fg2 zf-fuHdEJIX0^0UG7_|;NuLsd*7WO`}o?ZJSkJ*o@xx#yGa9GqamR$OWLxoxCRDc~g zp786i5Jqk(z>RH@N)#~BhhT7m5@)fo^cZ3hA&5N9q!ls7#o#O;Gd$zOv*7PZP-P!x zOH#fbLfkb9VX-5mkddA{>Wwb=!4ck%8!f|rvRjKc#z%xR7~}WY(iTk37aFez%%mz` zYNHRQDP7!vcIU+xi)A~Ey_am0cX>QhzSi_5Q6nRt&!#e;Dr&bat7iSxg!IhI5rZ2u zH#)}QAblr`&S{f+rjB3ZLN+DV;=5A_D%wsE7bl*{`{w&`>4y%6pL7Y2J3if@Kog!c zQDRwaW3um@#ISs$o?$i5kRKfHGFH;(q-12#7xr^IcdsM2r-J)9^ zN;cw2uEhE~DUXG&yl{E>hId2$=xvSBCBnBL!7R3Y9xPwaCl%)8bO5^k>Bcv}-NOm~ z4&A@;f9wZVODuN*`Q( zby6qG%y7YF-OS`8J4)7v4{~VLzO(j=cWrwG!iZ2uoJ(jSqUUxuPpkNoyLD;Q69CC~Z z!c?yF5DOx-v9YA$5S7N0KVbef?KC(SH*K-4a?8ja#&5Wi9M)p5w5n^+oRI^UVGYq9 z+XvLt_txo}{`FOO-LWX{_vN1h(jGa6nEa`2Qf5_IxA$u9f_BXk+CtvrCSemy*omT7 z>nWMDDJHcMy|FfwUWg!rg6a$A$njK89Vk472YVH82$>4hCnN)m#k zq~+Y6XJ)EHo%%hDq@??fm(lL_o}-McB;$TvSTk5`R$??9EVQ4MmK4KzK)4I%ps8q` zQ1{Q+(7}0B0ivu=loEawb^X;er#*O*Svfg>1>r*A5*GU*dO~^vd$IKlLo@*2!GEw) z0lyn(-Ma`=vY(lC7Mv*t|HkwgGkNC!l1X(1`W$td4y0j+2CymEK;C24%v2)Qs)!au zn#2uxUd1hVd124Y+Pn>49hEJe2Nf{3^xSvnPM~JTC%c!-lWKdLbdU1NHv~oQHic?p zL*3GBGHYtfOA@{BG$o@&b>dW%mwBHbIqyR&>y!_aQ?#s;BXYaRUJIi=%(S~=_4=mB zTa2F(wg$3XEV(Zm1*h3S1Z1?~p6?q6&8lLa?A*;iTbS$X%s+#dbq3P%)X@uY)4ax=m=F!0ETGf={-so{EZ z*Ll+a->X& zBgaoOmZ>0a@R?z25P;}ZtTIT_4D1V zGfIrrdvAA7HUg@@M3A-(UAsj+8f=rSrl*wg&Xd0^EW(qT<0z!D-qD>`Q^)#WRj%M0 z;P08u&~PVy?cv|aa9|=Yk1z^f`wq5Ej3^%luQ2(GWi4j083Gx}DMVvkjE70Wanoo73jNB&EGPNK0d1(Vn7U$6eyEkeOP@>D&HkYK!fjMa=aM2VlH(MtukUaF z!J)qbnpn3<2|CD@r=Uc)(FlPCkl!F+l>fp$f`oQ81rOnE!aUidMA8Mv6U9Bd^t;zQ zGvWH+IP==R?h4WuUM(Sy?^O(x076*S5)(5Fq$NO`g!zV5W^WZlSBC&}9vW4J#_iOkB( zrakW@=)>A}B(Rfu2jwBK6N$Gt<}0ts;&q#LLIGskwT>T^MuBow(J`yuC3@SRGzgeC zr^Rpkf|qg%g3Bq}zjoG9$|R!JHNAave_!^7LqdYW4c>GAz5?Niq;J>K^f#aa7#tX}`_~q=F(a88E`&W}z5r9Sh6&Ja}0tBe2 z9~{4&_ACBg(|!j+7iyx8I=UVynk2#`r$iS4;UQ-Ih5TuH%pae&ZLB)f-4tik-Aq4m zLSU91&%vN&y{l{ty~I2e*SB{HXbV4eF40u9fh3%?xgGEDkFGVksc$E?JieNLx2^h} zlYsDtGzT#7q}wTNuzI{+dC+lxt5?XmsvO7I&sbL_=#~L7_(4LY4l+*682k%#AtVMr zqXQ#O&iJsa&pF{hF;(eBzJbNpqULoghYkpC8U|JwNTM+o84Gi0HO`ppISNCcz)`AJzYlB#gS>EF20RNT_8N5_0)HwvjEfqTNsWr z1+dT5I=ti2^i0WMpQ(}ZD>G;qcX`O19zBv4LvzKJ*yPi? z(5-U3TJc&XFyQb+zr}{(%%dmJUl1hPesEyKN3%x?6}N{lX#~K604~0v^7nlsVgaZk zbDH4L&vk#T|EBIQpOQ|IG|e)-JHb@>QV%XtIVpXJGhDtoAP*}24dVVZwluCH;mon; z#kIekYM?lmYP?N{S&|0IVL?&?Gh{yL)_JdcpJ&~nH(xP-mOi{~AREe;dJklOEs>qE zTfcfiZ1JUK^X_7cWk7T7bMF1Ql;Lb`G@is*j`QiY^^S7Gw{N^?f|;N ziEtxlFw0)%1Hp&BhKTV$AGy`kZhfP}9XI)lYCkv*wzNP&MP1QX5UE?3wxB=FrD@cB zt5WhDYXQG1%mZzDRE@o-j~(TEV|^xm0hylvg-D|$e2VF~_~L^Gf2qBXg)dYlEGj=g z6<&M;sGgZ%!D%5D9Pe~78oL-V8pzY%Eo9Dc?7+FLY&7;IYhok|@{yQDEQ0e9{(o1qnc*^kba?XTN{@KuGb#z6r$g_;z`Wa5aU!=l;`1>T7B- zUqULMWu4OPrFcul|(!wsvTzZmwhYGi|fC2;F~b7{)eZ^ zgg*QMhz>Tus5nT_!pt|M1frW}EwsI~Ddb4v9x=@y9O{cN*K0a=18>(~(KFB}->_s@ zImNZNlp?GC1hsdUbGI!%g1QsmAum?cTyjNdZyo*9A-@lm92A}slmWO4@FxPtCAC&o zJ%56B0`B(m*4pGbZeJINEV^~z*%mqP8u~Wq0{tC2B%%4K>R6M;9(sXf^J&Xv`|k)V z#je$8R^-br#{pA;CSqUCZU&Q&eyR(Es1r7i{Zyh6A>I_t4Ip9Jhsfcnv4U)#Wt~ML zAUf(5wnvqcOcDV1YBPHHz2n5h7N6#d* zJTStl5C*Nvf9{)C`VZUbXgwhhH*R!k^4apObk3tl8F>e;0mfuv7b-lRQhwLrD4DC# zXT?bQ8)a!d=-*`gMxpgn{a}&&48ZRZ@IzSj<8N5kbJp;cnyI`IpJ&bgvcfS?d=z<&!#y`%?iTF5OEjMq4!g(dyL4~*dwEiAi!g7aDszOlbmjb zR@3+WsSnD`o90`_$O&jHkWWZ$=;Jp{TUg3}nE36v-2Lno9|S|43rmO@oe;fqi^!S1 zaO;VIZt9`iFZ7?xxSTW~I6AD6En1fr3o|A)qpCibF%|6?G~3^(SmkGSPQK+ks@b;} zl%B`BB}RVAdgr`8w!$Kj zy;v91U{+YQuA$Lw&Y=v{wTPCp9unVF$)DhzS=+j~%sq($qhFS$ksecAh_Y0MGgXe%UwxDb(A7q-3k$ylB=rlh6@SWF&hzPr8p9|d>w{z_V=O;FFP3+u3H1EMb_`| z1uc#eZ74|9T7wIRR)p7IsLOY^dgFHgITJ@(;pbM-f_(=Wm|S{=+a%}Ex)bi~jNYh4 zUcD@5i1V3nv&V>TCS9;hRx&a-%*I*c?W+`xhWi$8?IPMUYL=`Gjx!Zu17UK^2OmDf z=JBaQ{tNz5n3LHhp+~nwK*O6hK0;%?0Qq5t(NTr+1c|w~b3M3RkBOJWSU`_-^(b;i zrd`S!AvN8SjVtBO+E$;Tgg2B1+Bb3DNf^DFSSG6ySzEm+WACtJYhtNFIyF4_mBd3P z{1K$GA0_n<<}O2{z~S);%OvLjTk_u<I5^Dk#lb-~YQ>m)|0;%^gs;qI%!>Z6 zY8<9$VG3$WGsBvz{r9aoHk$(Ww%@c2)K>v#Aq$&(KGXnzaU*%JLz%)uVq(UGd$%GQXW~Kl79f?K(rM^k4 zG7OK`*jTN_N7G?V`#&SG*%yVK7`1E^_aJK7heOZJTI+7yN^#~|PdNuGfTW$Itjm}s zbY2@w)52t-v-!Ctdz<9;R?pi)IX7t0T}gr|@mG0y@^QS)!Q9KSKRAqxP4las9=(U^ z!5^m#$aR5l*#kS%L9=12WYvCaGagH)Y7@83c$)MfRYzM41c1on$6s%Xs?VCT5&?0K z7h2*DqCpI8TxlU14jvDV@dE?__wFvZfAU^p=2t)?>^ zN`EiI;mA*h!@(@b17qd5^xSGnXi2)<|~Ij0-+&**`DvYY8W+4OsAjJjOmg%cQf zf>uI??QcGX>^T>C)csD<|}h&i$71PM1%o<&~-&A_g;be%9;zB$t>>N0E{k`8&+|2pE3LS{PTr?e~!_^m`YufrG7f~x>*)bsWxl9>j~k8QoW;x zUYEXH6I&{|GCA%qGd~CnVfyjYoz_tQRF-Tjp$e{EHN#e#B<8()#1L%CD{U2n(1-OS zuJX-pU9~{d#`gQSPUmj)Ye}lE&o#)+9zLYfZ3ok@*`l1B(GKwnvu`YbY<5`agW*HX;%(0iASXY5LI17}>X*}$|5rB?MPcR|# zerZB}`MsPL;!~X#A6N@g2=yuT4cPyr4T&)Sl+O_vP^xZc3sP)Xm$@&g*7Veyt$960 zWf5nS4EB9g1H|CrKWqYh?ODE*$Gv;VMzJ&4bM{@o*E5FBB-3X@QS5}~E6A9RMFxQ4 z1pFW(3M3PAdX9kSXR!sR{?uY>+&A>yM>db+f>N0l1re#rqvG)Ej;Zs zyvqHdYu#JvAs?}QCx39PYY$EhNc$)KY^RXyf9;>-Lzp4_(6$i_nlgf}Lx1&HmmtJ! zZHy2eADzaq_cs0~X6R$8WExuFpAbw=Hk;Y+gJQd})w^GjE