From f3dd1041a0c8a9c4b040238a940cf85acf4ebb1d Mon Sep 17 00:00:00 2001 From: Rob Zienert Date: Mon, 4 Dec 2017 13:03:31 -0800 Subject: [PATCH 01/91] ignoring IDEA files --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index e2f8b8c..aee7a2c 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,6 @@ dyno-queues-core/build build redis-3.0.7/ redis-3.0.7.tar.gz + +*.iml +.idea From b9caecc9dbe785220e496ec0d6e4c672954f02ec Mon Sep 17 00:00:00 2001 From: Rob Zienert Date: Mon, 4 Dec 2017 13:20:30 -0800 Subject: [PATCH 02/91] ignoring .gradle dir --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index aee7a2c..abf0c50 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,4 @@ redis-3.0.7.tar.gz *.iml .idea +.gradle From a022a1eef73d28fd723c57c113d4de5a035c5d34 Mon Sep 17 00:00:00 2001 From: Rob Zienert Date: Mon, 4 Dec 2017 13:21:02 -0800 Subject: [PATCH 03/91] refactoring queue implementations to accept a Clock --- .../dyno/queues/redis/MultiRedisQueue.java | 43 +++++++++----- .../dyno/queues/redis/RedisDynoQueue.java | 59 +++++++++++-------- .../netflix/dyno/queues/redis/RedisQueue.java | 52 ++++++++-------- .../dyno/queues/redis/RedisQueues.java | 31 +++++++--- 4 files changed, 112 insertions(+), 73 deletions(-) diff --git a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/MultiRedisQueue.java b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/MultiRedisQueue.java index 1499974..f4d3d20 100644 --- a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/MultiRedisQueue.java +++ b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/MultiRedisQueue.java @@ -15,17 +15,6 @@ */ package com.netflix.dyno.queues.redis; -import java.io.IOException; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import java.util.Map.Entry; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.stream.Collectors; - import com.google.common.base.Function; import com.google.common.collect.Collections2; import com.google.common.collect.Lists; @@ -38,10 +27,21 @@ import com.netflix.dyno.connectionpool.Host; import com.netflix.dyno.queues.DynoQueue; import com.netflix.dyno.queues.Message; - import redis.clients.jedis.JedisPool; import redis.clients.jedis.JedisPoolConfig; +import java.io.IOException; +import java.time.Clock; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.stream.Collectors; + /** * @author Viren * @@ -217,6 +217,8 @@ private String getNextShard() { public static class Builder { + + private Clock clock; private String queueName; @@ -239,6 +241,15 @@ public static class Builder { private int nonQuorumPort; private List hosts; + + /** + * @param clock the Clock instance to set + * @return instance of builder + */ + public Builder setClock(Clock clock) { + this.clock = clock; + return this; + } /** * @param queueName the queueName to set @@ -336,6 +347,9 @@ public Builder setHosts(List hosts) { } public MultiRedisQueue build() { + if (clock == null) { + clock = Clock.systemDefaultZone(); + } if(hosts == null) { hosts = getHostsFromEureka(ec, dynomiteClusterName); } @@ -352,15 +366,14 @@ public MultiRedisQueue build() { config.setMaxIdle(5); config.setMaxWaitMillis(60_000); - Map queues = new HashMap<>(); for(String queueShard : shardMap.keySet()) { String host = shardMap.get(queueShard).getIpAddress(); JedisPool pool = new JedisPool(config, host, quorumPort, 0); JedisPool readPool = new JedisPool(config, host, nonQuorumPort, 0); - - RedisQueue q = new RedisQueue(redisKeyPrefix, queueName, queueShard, unackTime, pool); + + RedisQueue q = new RedisQueue(clock, redisKeyPrefix, queueName, queueShard, unackTime, unackTime, pool); q.setNonQuorumPool(readPool); queues.put(queueShard, q); } diff --git a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/RedisDynoQueue.java b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/RedisDynoQueue.java index 4797b65..6a6dd48 100644 --- a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/RedisDynoQueue.java +++ b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/RedisDynoQueue.java @@ -15,7 +15,24 @@ */ package com.netflix.dyno.queues.redis; +import com.fasterxml.jackson.annotation.JsonInclude.Include; +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.SerializationFeature; +import com.google.common.annotations.VisibleForTesting; +import com.google.common.util.concurrent.Uninterruptibles; +import com.netflix.dyno.connectionpool.exception.DynoException; +import com.netflix.dyno.queues.DynoQueue; +import com.netflix.dyno.queues.Message; +import com.netflix.servo.monitor.Stopwatch; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import redis.clients.jedis.JedisCommands; +import redis.clients.jedis.Tuple; +import redis.clients.jedis.params.sortedset.ZAddParams; + import java.io.IOException; +import java.time.Clock; import java.util.Collections; import java.util.HashMap; import java.util.LinkedList; @@ -31,24 +48,6 @@ import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Collectors; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import com.fasterxml.jackson.annotation.JsonInclude.Include; -import com.fasterxml.jackson.databind.DeserializationFeature; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.SerializationFeature; -import com.google.common.annotations.VisibleForTesting; -import com.google.common.util.concurrent.Uninterruptibles; -import com.netflix.dyno.connectionpool.exception.DynoException; -import com.netflix.dyno.queues.DynoQueue; -import com.netflix.dyno.queues.Message; -import com.netflix.servo.monitor.Stopwatch; - -import redis.clients.jedis.JedisCommands; -import redis.clients.jedis.Tuple; -import redis.clients.jedis.params.sortedset.ZAddParams; - /** * * @author Viren @@ -58,6 +57,8 @@ public class RedisDynoQueue implements DynoQueue { private final Logger logger = LoggerFactory.getLogger(RedisDynoQueue.class); + private Clock clock; + private String queueName; private List allShards; @@ -91,7 +92,13 @@ public class RedisDynoQueue implements DynoQueue { public RedisDynoQueue(String redisKeyPrefix, String queueName, Set allShards, String shardName) { this(redisKeyPrefix, queueName, allShards, shardName, 60_000); } + public RedisDynoQueue(String redisKeyPrefix, String queueName, Set allShards, String shardName, int unackScheduleInMS) { + this(Clock.systemDefaultZone(), redisKeyPrefix, queueName, allShards, shardName, unackScheduleInMS); + } + + public RedisDynoQueue(Clock clock, String redisKeyPrefix, String queueName, Set allShards, String shardName, int unackScheduleInMS) { + this.clock = clock; this.redisKeyPrefix = redisKeyPrefix; this.queueName = queueName; this.allShards = allShards.stream().collect(Collectors.toList()); @@ -157,7 +164,7 @@ public List push(final List messages) { String json = om.writeValueAsString(message); quorumConn.hset(messageStoreKey, message.getId(), json); double priority = message.getPriority() / 100; - double score = Long.valueOf(System.currentTimeMillis() + message.getTimeout()).doubleValue() + priority; + double score = Long.valueOf(clock.millis() + message.getTimeout()).doubleValue() + priority; String shard = getNextShard(); String queueShard = getQueueShardKey(queueName, shard); quorumConn.zadd(queueShard, score, message.getId()); @@ -210,11 +217,11 @@ public List pop(int messageCount, int wait, TimeUnit unit) { Stopwatch sw = monitor.start(monitor.pop, messageCount); try { - long start = System.currentTimeMillis(); + long start = clock.millis(); long waitFor = unit.toMillis(wait); prefetch.addAndGet(messageCount); prefetchIds(); - while(prefetchedIds.size() < messageCount && ((System.currentTimeMillis() - start) < waitFor)) { + while(prefetchedIds.size() < messageCount && ((clock.millis() - start) < waitFor)) { Uninterruptibles.sleepUninterruptibly(200, TimeUnit.MILLISECONDS); prefetchIds(); } @@ -255,7 +262,7 @@ private void prefetchIds() { private List _pop(int messageCount) throws Exception { - double unackScore = Long.valueOf(System.currentTimeMillis() + unackTime).doubleValue(); + double unackScore = Long.valueOf(clock.millis() + unackTime).doubleValue(); String unackQueueName = getUnackKey(queueName, shardName); List popped = new LinkedList<>(); @@ -343,7 +350,7 @@ public boolean setUnackTimeout(String messageId, long timeout) { try { return execute(() -> { - double unackScore = Long.valueOf(System.currentTimeMillis() + timeout).doubleValue(); + double unackScore = Long.valueOf(clock.millis() + timeout).doubleValue(); for (String shard : allShards) { String unackShardKey = getUnackKey(queueName, shard); @@ -379,7 +386,7 @@ public boolean setTimeout(String messageId, long timeout) { Double score = quorumConn.zscore(queueShard, messageId); if(score != null) { double priorityd = message.getPriority() / 100; - double newScore = Long.valueOf(System.currentTimeMillis() + timeout).doubleValue() + priorityd; + double newScore = Long.valueOf(clock.millis() + timeout).doubleValue() + priorityd; ZAddParams params = ZAddParams.zAddParams().xx(); quorumConn.zadd(queueShard, newScore, messageId, params); json = om.writeValueAsString(message); @@ -510,7 +517,7 @@ public void clear() { private Set peekIds(int offset, int count) { return execute(() -> { - double now = Long.valueOf(System.currentTimeMillis() + 1).doubleValue(); + double now = Long.valueOf(clock.millis() + 1).doubleValue(); Set scanned = quorumConn.zrangeByScore(myQueueShard, 0, now, offset, count); return scanned; }); @@ -530,7 +537,7 @@ public void processUnacks() { int batchSize = 1_000; String unackQueueName = getUnackKey(queueName, shardName); - double now = Long.valueOf(System.currentTimeMillis()).doubleValue(); + double now = Long.valueOf(clock.millis()).doubleValue(); Set unacks = quorumConn.zrangeByScoreWithScores(unackQueueName, 0, now, 0, batchSize); diff --git a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/RedisQueue.java b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/RedisQueue.java index aeee37c..a040428 100644 --- a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/RedisQueue.java +++ b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/RedisQueue.java @@ -15,22 +15,6 @@ */ package com.netflix.dyno.queues.redis; -import java.io.IOException; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.TimeUnit; -import java.util.stream.Collectors; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import com.fasterxml.jackson.annotation.JsonInclude.Include; import com.fasterxml.jackson.databind.DeserializationFeature; import com.fasterxml.jackson.databind.ObjectMapper; @@ -40,7 +24,8 @@ import com.netflix.dyno.queues.DynoQueue; import com.netflix.dyno.queues.Message; import com.netflix.servo.monitor.Stopwatch; - +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import redis.clients.jedis.Jedis; import redis.clients.jedis.JedisPool; import redis.clients.jedis.Pipeline; @@ -48,6 +33,20 @@ import redis.clients.jedis.Tuple; import redis.clients.jedis.params.sortedset.ZAddParams; +import java.io.IOException; +import java.time.Clock; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; + /** * * @author Viren @@ -57,6 +56,8 @@ public class RedisQueue implements DynoQueue { private final Logger logger = LoggerFactory.getLogger(RedisQueue.class); + private Clock clock; + private String queueName; private String shardName; @@ -90,6 +91,11 @@ public RedisQueue(String redisKeyPrefix, String queueName, String shardName, int } public RedisQueue(String redisKeyPrefix, String queueName, String shardName, int unackScheduleInMS, int unackTime, JedisPool pool) { + this(Clock.systemDefaultZone(), redisKeyPrefix, queueName, shardName, unackScheduleInMS, unackTime, pool); + } + + public RedisQueue(Clock clock, String redisKeyPrefix, String queueName, String shardName, int unackScheduleInMS, int unackTime, JedisPool pool) { + this.clock = clock; this.queueName = queueName; this.shardName = shardName; this.messageStoreKeyPrefix = redisKeyPrefix + ".MESSAGE."; @@ -150,7 +156,7 @@ public List push(final List messages) { String json = om.writeValueAsString(message); pipe.hset(messageStoreKey(message.getId()), message.getId(), json); double priority = message.getPriority() / 100.0; - double score = Long.valueOf(System.currentTimeMillis() + message.getTimeout()).doubleValue() + priority; + double score = Long.valueOf(clock.millis() + message.getTimeout()).doubleValue() + priority; pipe.zadd(myQueueShard, score, message.getId()); } pipe.sync(); @@ -237,7 +243,7 @@ public synchronized List pop(int messageCount, int wait, TimeUnit unit) private List _pop(List batch) throws Exception { - double unackScore = Long.valueOf(System.currentTimeMillis() + unackTime).doubleValue(); + double unackScore = Long.valueOf(clock.millis() + unackTime).doubleValue(); List popped = new LinkedList<>(); ZAddParams zParams = ZAddParams.zAddParams().nx(); @@ -371,7 +377,7 @@ public boolean setUnackTimeout(String messageId, long timeout) { try { - double unackScore = Long.valueOf(System.currentTimeMillis() + timeout).doubleValue(); + double unackScore = Long.valueOf(clock.millis() + timeout).doubleValue(); Double score = jedis.zscore(unackShardKey(messageId), messageId); if (score != null) { jedis.zadd(unackShardKey(messageId), unackScore, messageId); @@ -402,7 +408,7 @@ public boolean setTimeout(String messageId, long timeout) { Double score = jedis.zscore(myQueueShard, messageId); if (score != null) { double priorityd = message.getPriority() / 100.0; - double newScore = Long.valueOf(System.currentTimeMillis() + timeout).doubleValue() + priorityd; + double newScore = Long.valueOf(clock.millis() + timeout).doubleValue() + priorityd; jedis.zadd(myQueueShard, newScore, messageId); json = om.writeValueAsString(message); jedis.hset(messageStoreKey(message.getId()), message.getId(), json); @@ -537,7 +543,7 @@ public void clear() { private Set peekIds(int offset, int count) { Jedis jedis = connPool.getResource(); try { - double now = Long.valueOf(System.currentTimeMillis() + 1).doubleValue(); + double now = Long.valueOf(clock.millis() + 1).doubleValue(); Set scanned = jedis.zrangeByScore(myQueueShard, 0, now, offset, count); return scanned; } finally { @@ -566,7 +572,7 @@ private void processUnacks(String unackShardKey) { int batchSize = 1_000; - double now = Long.valueOf(System.currentTimeMillis()).doubleValue(); + double now = Long.valueOf(clock.millis()).doubleValue(); Set unacks = jedis.zrangeByScoreWithScores(unackShardKey, 0, now, 0, batchSize); diff --git a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/RedisQueues.java b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/RedisQueues.java index 30bbbfe..291c06f 100644 --- a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/RedisQueues.java +++ b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/RedisQueues.java @@ -15,17 +15,17 @@ */ package com.netflix.dyno.queues.redis; +import com.netflix.dyno.queues.DynoQueue; +import com.netflix.dyno.queues.ShardSupplier; +import redis.clients.jedis.JedisCommands; + import java.io.Closeable; import java.io.IOException; +import java.time.Clock; import java.util.Collection; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; -import com.netflix.dyno.queues.DynoQueue; -import com.netflix.dyno.queues.ShardSupplier; - -import redis.clients.jedis.JedisCommands; - /** * @author Viren * @@ -34,6 +34,8 @@ */ public class RedisQueues implements Closeable { + private Clock clock; + private JedisCommands quorumConn; private JedisCommands nonQuorumConn; @@ -51,7 +53,6 @@ public class RedisQueues implements Closeable { private ConcurrentHashMap queues; /** - * * @param quorumConn Dyno connection with dc_quorum enabled * @param nonQuorumConn Dyno connection to local Redis * @param redisKeyPrefix prefix applied to the Redis keys @@ -59,9 +60,21 @@ public class RedisQueues implements Closeable { * @param unackTime Time in millisecond within which a message needs to be acknowledged by the client, after which the message is re-queued. * @param unackHandlerIntervalInMS Time in millisecond at which the un-acknowledgement processor runs */ - public RedisQueues(JedisCommands quorumConn, JedisCommands nonQuorumConn, String redisKeyPrefix, ShardSupplier shardSupplier, int unackTime, - int unackHandlerIntervalInMS) { + public RedisQueues(JedisCommands quorumConn, JedisCommands nonQuorumConn, String redisKeyPrefix, ShardSupplier shardSupplier, int unackTime, int unackHandlerIntervalInMS) { + this(Clock.systemDefaultZone(), quorumConn, nonQuorumConn, redisKeyPrefix, shardSupplier, unackTime, unackHandlerIntervalInMS); + } + /** + * @param clock Time provider + * @param quorumConn Dyno connection with dc_quorum enabled + * @param nonQuorumConn Dyno connection to local Redis + * @param redisKeyPrefix prefix applied to the Redis keys + * @param shardSupplier Provider for the shards for the queues created + * @param unackTime Time in millisecond within which a message needs to be acknowledged by the client, after which the message is re-queued. + * @param unackHandlerIntervalInMS Time in millisecond at which the un-acknowledgement processor runs + */ + public RedisQueues(Clock clock, JedisCommands quorumConn, JedisCommands nonQuorumConn, String redisKeyPrefix, ShardSupplier shardSupplier, int unackTime, int unackHandlerIntervalInMS) { + this.clock = clock; this.quorumConn = quorumConn; this.nonQuorumConn = nonQuorumConn; this.redisKeyPrefix = redisKeyPrefix; @@ -88,7 +101,7 @@ public DynoQueue get(String queueName) { } synchronized (this) { - queue = new RedisDynoQueue(redisKeyPrefix, queueName, allShards, shardName, unackHandlerIntervalInMS) + queue = new RedisDynoQueue(clock, redisKeyPrefix, queueName, allShards, shardName, unackHandlerIntervalInMS) .withUnackTime(unackTime) .withNonQuorumConn(nonQuorumConn) .withQuorumConn(quorumConn); From 267567b3692fc2f319663af2335fc986b73e5f1e Mon Sep 17 00:00:00 2001 From: Viren Baraiya Date: Sun, 11 Feb 2018 15:01:19 -0800 Subject: [PATCH 04/91] refactored for the new pipeline based queues --- .gitignore | 2 + dyno-queues-redis/build.gradle | 6 +- .../dyno/queues/redis/MultiRedisQueue.java | 222 +-------------- .../dyno/queues/redis/QueueBuilder.java | 253 ++++++++++++++++++ .../netflix/dyno/queues/redis/RedisQueue.java | 93 +++---- .../queues/redis/conn/DynoClientProxy.java | 91 +++++++ .../dyno/queues/redis/conn/DynoJedisPipe.java | 65 +++++ .../dyno/queues/redis/conn/JedisProxy.java | 97 +++++++ .../netflix/dyno/queues/redis/conn/Pipe.java | 83 ++++++ .../queues/redis/conn/RedisConnection.java | 52 ++++ .../dyno/queues/redis/conn/RedisPipe.java | 64 +++++ .../{redis => shard}/DynoShardSupplier.java | 2 +- .../{redis => shard}/SingleShardSupplier.java | 2 +- .../dyno/queues/redis/BenchmarkTests.java | 40 ++- .../queues/redis/DynoShardSupplierTest.java | 1 + .../queues/redis/RedisDynoQueueTest2.java | 4 +- gradle/wrapper/gradle-wrapper.properties | 4 +- 17 files changed, 793 insertions(+), 288 deletions(-) create mode 100644 dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/QueueBuilder.java create mode 100644 dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/conn/DynoClientProxy.java create mode 100644 dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/conn/DynoJedisPipe.java create mode 100644 dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/conn/JedisProxy.java create mode 100644 dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/conn/Pipe.java create mode 100644 dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/conn/RedisConnection.java create mode 100644 dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/conn/RedisPipe.java rename dyno-queues-redis/src/main/java/com/netflix/dyno/queues/{redis => shard}/DynoShardSupplier.java (97%) rename dyno-queues-redis/src/main/java/com/netflix/dyno/queues/{redis => shard}/SingleShardSupplier.java (96%) diff --git a/.gitignore b/.gitignore index abf0c50..342edea 100644 --- a/.gitignore +++ b/.gitignore @@ -7,3 +7,5 @@ redis-3.0.7.tar.gz *.iml .idea .gradle +.classpath +.project diff --git a/dyno-queues-redis/build.gradle b/dyno-queues-redis/build.gradle index 27074b8..21f8b4c 100644 --- a/dyno-queues-redis/build.gradle +++ b/dyno-queues-redis/build.gradle @@ -3,8 +3,10 @@ dependencies { compile project(':dyno-queues-core') compile "com.google.inject:guice:3.0" - compile 'com.netflix.dyno:dyno-core:1.5.9' - compile "com.netflix.dyno:dyno-jedis:1.5.9" + + compile 'com.netflix.dyno:dyno-core:1.6.2' + compile "com.netflix.dyno:dyno-jedis:1.6.2" + compile "com.netflix.archaius:archaius-core:0.7.5" compile "com.netflix.servo:servo-core:0.12.17" compile 'com.netflix.eureka:eureka-client:1.8.1' diff --git a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/MultiRedisQueue.java b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/MultiRedisQueue.java index f4d3d20..92bce97 100644 --- a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/MultiRedisQueue.java +++ b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/MultiRedisQueue.java @@ -15,24 +15,7 @@ */ package com.netflix.dyno.queues.redis; -import com.google.common.base.Function; -import com.google.common.collect.Collections2; -import com.google.common.collect.Lists; -import com.netflix.appinfo.AmazonInfo; -import com.netflix.appinfo.AmazonInfo.MetaDataKey; -import com.netflix.appinfo.InstanceInfo; -import com.netflix.appinfo.InstanceInfo.InstanceStatus; -import com.netflix.discovery.EurekaClient; -import com.netflix.discovery.shared.Application; -import com.netflix.dyno.connectionpool.Host; -import com.netflix.dyno.queues.DynoQueue; -import com.netflix.dyno.queues.Message; -import redis.clients.jedis.JedisPool; -import redis.clients.jedis.JedisPoolConfig; - import java.io.IOException; -import java.time.Clock; -import java.util.ArrayList; import java.util.HashMap; import java.util.LinkedList; import java.util.List; @@ -42,6 +25,9 @@ import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Collectors; +import com.netflix.dyno.queues.DynoQueue; +import com.netflix.dyno.queues.Message; + /** * @author Viren * @@ -214,207 +200,5 @@ private String getNextShard() { String s = shards.get(indx); return s; } - - - public static class Builder { - - private Clock clock; - - private String queueName; - - private EurekaClient ec; - - private String dynomiteClusterName; - - private String redisKeyPrefix; - - private int unackTime; - - private String currentShard; - - private Function hostToShardMap; - - private int redisPoolSize; - - private int quorumPort; - - private int nonQuorumPort; - - private List hosts; - - /** - * @param clock the Clock instance to set - * @return instance of builder - */ - public Builder setClock(Clock clock) { - this.clock = clock; - return this; - } - - /** - * @param queueName the queueName to set - * @return instance of builder - */ - public Builder setQueueName(String queueName) { - this.queueName = queueName; - return this; - } - - /** - * @param ec the ec to set - * @return instance of builder - */ - public Builder setEc(EurekaClient ec) { - this.ec = ec; - return this; - } - - /** - * @param dynomiteClusterName the dynomiteClusterName to set - * @return instance of builder - */ - public Builder setDynomiteClusterName(String dynomiteClusterName) { - this.dynomiteClusterName = dynomiteClusterName; - return this; - } - - /** - * @param redisKeyPrefix the redisKeyPrefix to set - * @return instance of builder - */ - public Builder setRedisKeyPrefix(String redisKeyPrefix) { - this.redisKeyPrefix = redisKeyPrefix; - return this; - } - - /** - * @param unackTime the unackTime to set - * @return instance of builder - */ - public Builder setUnackTime(int unackTime) { - this.unackTime = unackTime; - return this; - } - - /** - * @param currentShard the currentShard to set - * @return instance of builder - */ - public Builder setCurrentShard(String currentShard) { - this.currentShard = currentShard; - return this; - } - - /** - * @param hostToShardMap the hostToShardMap to set - * @return instance of builder - */ - public Builder setHostToShardMap(Function hostToShardMap) { - this.hostToShardMap = hostToShardMap; - return this; - } - - /** - * @param redisPoolSize the redisPoolSize to set - * @return instance of builder - */ - public Builder setRedisPoolSize(int redisPoolSize) { - this.redisPoolSize = redisPoolSize; - return this; - } - - /** - * @param quorumPort the quorumPort to set - * @return instance of builder - */ - public Builder setQuorumPort(int quorumPort) { - this.quorumPort = quorumPort; - return this; - } - - /** - * @param nonQuorumPort the nonQuorumPort to set - * @return instance of builder - */ - public Builder setNonQuorumPort(int nonQuorumPort) { - this.nonQuorumPort = nonQuorumPort; - return this; - } - - public Builder setHosts(List hosts) { - this.hosts = hosts; - return this; - } - - public MultiRedisQueue build() { - if (clock == null) { - clock = Clock.systemDefaultZone(); - } - if(hosts == null) { - hosts = getHostsFromEureka(ec, dynomiteClusterName); - } - Map shardMap = new HashMap<>(); - for(Host host : hosts) { - String shard = hostToShardMap.apply(host); - shardMap.put(shard, host); - } - - JedisPoolConfig config = new JedisPoolConfig(); - config.setTestOnBorrow(true); - config.setTestOnCreate(true); - config.setMaxTotal(redisPoolSize); - config.setMaxIdle(5); - config.setMaxWaitMillis(60_000); - - Map queues = new HashMap<>(); - for(String queueShard : shardMap.keySet()) { - String host = shardMap.get(queueShard).getIpAddress(); - - JedisPool pool = new JedisPool(config, host, quorumPort, 0); - JedisPool readPool = new JedisPool(config, host, nonQuorumPort, 0); - - RedisQueue q = new RedisQueue(clock, redisKeyPrefix, queueName, queueShard, unackTime, unackTime, pool); - q.setNonQuorumPool(readPool); - queues.put(queueShard, q); - } - MultiRedisQueue queue = new MultiRedisQueue(queueName, currentShard, queues); - return queue; - } - - private static List getHostsFromEureka(EurekaClient ec, String applicationName) { - - Application app = ec.getApplication(applicationName); - List hosts = new ArrayList(); - - if (app == null) { - return hosts; - } - - List ins = app.getInstances(); - - if (ins == null || ins.isEmpty()) { - return hosts; - } - - hosts = Lists.newArrayList(Collections2.transform(ins, - - new Function() { - @Override - public Host apply(InstanceInfo info) { - - Host.Status status = info.getStatus() == InstanceStatus.UP ? Host.Status.Up : Host.Status.Down; - String rack = null; - if (info.getDataCenterInfo() instanceof AmazonInfo) { - AmazonInfo amazonInfo = (AmazonInfo)info.getDataCenterInfo(); - rack = amazonInfo.get(MetaDataKey.availabilityZone); - } - Host host = new Host(info.getHostName(), info.getIPAddr(), rack, status); - return host; - } - })); - return hosts; - } - } - } diff --git a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/QueueBuilder.java b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/QueueBuilder.java new file mode 100644 index 0000000..722f188 --- /dev/null +++ b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/QueueBuilder.java @@ -0,0 +1,253 @@ +/** + * + */ +package com.netflix.dyno.queues.redis; + +import java.time.Clock; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import com.google.common.base.Function; +import com.google.common.collect.Collections2; +import com.google.common.collect.Lists; +import com.netflix.appinfo.AmazonInfo; +import com.netflix.appinfo.AmazonInfo.MetaDataKey; +import com.netflix.appinfo.InstanceInfo; +import com.netflix.appinfo.InstanceInfo.InstanceStatus; +import com.netflix.discovery.EurekaClient; +import com.netflix.discovery.shared.Application; +import com.netflix.dyno.connectionpool.Host; +import com.netflix.dyno.connectionpool.Host.Status; +import com.netflix.dyno.contrib.EurekaHostsSupplier; +import com.netflix.dyno.jedis.DynoJedisClient; +import com.netflix.dyno.queues.DynoQueue; +import com.netflix.dyno.queues.redis.conn.DynoClientProxy; +import com.netflix.dyno.queues.redis.conn.JedisProxy; +import com.netflix.dyno.queues.redis.conn.RedisConnection; + +import redis.clients.jedis.JedisPool; +import redis.clients.jedis.JedisPoolConfig; + +/** + * @author Viren + * + */ +public class QueueBuilder { + + private Clock clock; + + private String queueName; + + private EurekaClient ec; + + private String dynomiteClusterName; + + private String redisKeyPrefix; + + private int unackTime; + + private String currentShard; + + private Function hostToShardMap; + + private int nonQuorumPort; + + private List hosts; + + private JedisPoolConfig redisPoolConfig; + + /** + * @param clock the Clock instance to set + * @return instance of QueueBuilder + */ + public QueueBuilder setClock(Clock clock) { + this.clock = clock; + return this; + } + + /** + * @param queueName the queueName to set + * @return instance of QueueBuilder + */ + public QueueBuilder setQueueName(String queueName) { + this.queueName = queueName; + return this; + } + + /** + * @param redisKeyPrefix Prefix used for all the keys in Redis + * @return instance of QueueBuilder + */ + public QueueBuilder setRedisKeyPrefix(String redisKeyPrefix) { + this.redisKeyPrefix = redisKeyPrefix; + return this; + } + + /** + * @param ec the ec to set + * @return instance of QueueBuilder + */ + + /** + * + * @param ec EurekaClient instance used to discover the hosts in the dynomite cluster + * @param dynomiteClusterName Name of the Dynomite Cluster to be used + * @param nonQuorumPort Direct Redis port used to make non-quorumed queries - used when querying the message counts + * @return instance of QueueBuilder + */ + public QueueBuilder useDynomiteCluster(EurekaClient ec, String dynomiteClusterName, int nonQuorumPort) { + this.ec = ec; + this.dynomiteClusterName = dynomiteClusterName; + this.nonQuorumPort = nonQuorumPort; + return this; + } + + /** + * + * @param redisPoolConfig + * @return instance of QueueBuilder + */ + public QueueBuilder useNonDynomiteRedis(JedisPoolConfig redisPoolConfig, List redisHosts) { + this.redisPoolConfig = redisPoolConfig; + this.hosts = redisHosts; + return this; + } + + /** + * + * @param hostToShardMap Mapping from a Host to a queue shard + * @return instance of QueueBuilder + */ + public QueueBuilder setHostToShardMap(Function hostToShardMap) { + this.hostToShardMap = hostToShardMap; + return this; + } + + /** + * @param unackTime Time in millisecond, after which the uncked messages will be re-queued for the delivery + * @return instance of QueueBuilder + */ + public QueueBuilder setUnackTime(int unackTime) { + this.unackTime = unackTime; + return this; + } + + /** + * @param currentShard Name of the current shard + * @return instance of QueueBuilder + */ + public QueueBuilder setCurrentShard(String currentShard) { + this.currentShard = currentShard; + return this; + } + + public DynoQueue build() { + if (clock == null) { + clock = Clock.systemDefaultZone(); + } + + boolean useDynomite = false; + //When using Dynomite + if(dynomiteClusterName != null) { + hosts = getHostsFromEureka(ec, dynomiteClusterName); + useDynomite = true; + } + Map shardMap = new HashMap<>(); + for(Host host : hosts) { + String shard = hostToShardMap.apply(host); + shardMap.put(shard, host); + } + + DynoJedisClient dynoClientRead = null; + DynoJedisClient dynoClient = null; + if(useDynomite) { + String appId = queueName; + EurekaHostsSupplier hostSupplier = new EurekaHostsSupplier(dynomiteClusterName, ec) { + @Override + public List getHosts() { + List hosts = super.getHosts(); + List updatedHosts = new ArrayList<>(hosts.size()); + hosts.forEach(host -> { + updatedHosts.add(new Host(host.getHostName(), host.getIpAddress(), nonQuorumPort, host.getRack(), host.getDatacenter(), host.isUp() ? Status.Up : Status.Down)); + }); + return updatedHosts; + } + }; + + dynoClientRead = new DynoJedisClient.Builder().withApplicationName(appId).withDynomiteClusterName(dynomiteClusterName).withHostSupplier(hostSupplier).build(); + dynoClient = new DynoJedisClient.Builder().withApplicationName(appId).withDynomiteClusterName(dynomiteClusterName).withDiscoveryClient(ec).build(); + } + + + Map queues = new HashMap<>(); + for(String queueShard : shardMap.keySet()) { + + Host host = shardMap.get(queueShard); + String hostAddress = host.getIpAddress(); + if(hostAddress == null || "".equals(hostAddress)) { + hostAddress = host.getHostName(); + } + RedisConnection redisConn = null; + RedisConnection redisConnRead = null; + + if(useDynomite) { + redisConn = new DynoClientProxy(dynoClient); + redisConnRead = new DynoClientProxy(dynoClientRead); + } else{ + JedisPool pool = new JedisPool(redisPoolConfig, hostAddress, host.getPort(), 0); + redisConn = new JedisProxy(pool); + redisConnRead = new JedisProxy(pool); + } + + RedisQueue q = new RedisQueue(clock, redisKeyPrefix, queueName, queueShard, unackTime, unackTime, redisConn); + q.setNonQuorumPool(redisConnRead); + + queues.put(queueShard, q); + } + + if(queues.size() == 1) { + //This is a queue with a single shard + return queues.values().iterator().next(); + } + + MultiRedisQueue queue = new MultiRedisQueue(queueName, currentShard, queues); + return queue; + } + + + private static List getHostsFromEureka(EurekaClient ec, String applicationName) { + + Application app = ec.getApplication(applicationName); + List hosts = new ArrayList(); + + if (app == null) { + return hosts; + } + + List ins = app.getInstances(); + + if (ins == null || ins.isEmpty()) { + return hosts; + } + + hosts = Lists.newArrayList(Collections2.transform(ins, + + new Function() { + @Override + public Host apply(InstanceInfo info) { + + Host.Status status = info.getStatus() == InstanceStatus.UP ? Host.Status.Up : Host.Status.Down; + String rack = null; + if (info.getDataCenterInfo() instanceof AmazonInfo) { + AmazonInfo amazonInfo = (AmazonInfo)info.getDataCenterInfo(); + rack = amazonInfo.get(MetaDataKey.availabilityZone); + } + Host host = new Host(info.getHostName(), info.getIPAddr(), rack, status); + return host; + } + })); + return hosts; + } +} diff --git a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/RedisQueue.java b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/RedisQueue.java index a040428..1b0d033 100644 --- a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/RedisQueue.java +++ b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/RedisQueue.java @@ -15,24 +15,6 @@ */ package com.netflix.dyno.queues.redis; -import com.fasterxml.jackson.annotation.JsonInclude.Include; -import com.fasterxml.jackson.databind.DeserializationFeature; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.SerializationFeature; -import com.netflix.dyno.connectionpool.HashPartitioner; -import com.netflix.dyno.connectionpool.impl.hash.Murmur3HashPartitioner; -import com.netflix.dyno.queues.DynoQueue; -import com.netflix.dyno.queues.Message; -import com.netflix.servo.monitor.Stopwatch; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import redis.clients.jedis.Jedis; -import redis.clients.jedis.JedisPool; -import redis.clients.jedis.Pipeline; -import redis.clients.jedis.Response; -import redis.clients.jedis.Tuple; -import redis.clients.jedis.params.sortedset.ZAddParams; - import java.io.IOException; import java.time.Clock; import java.util.ArrayList; @@ -47,6 +29,25 @@ import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.fasterxml.jackson.annotation.JsonInclude.Include; +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.SerializationFeature; +import com.netflix.dyno.connectionpool.HashPartitioner; +import com.netflix.dyno.connectionpool.impl.hash.Murmur3HashPartitioner; +import com.netflix.dyno.queues.DynoQueue; +import com.netflix.dyno.queues.Message; +import com.netflix.dyno.queues.redis.conn.Pipe; +import com.netflix.dyno.queues.redis.conn.RedisConnection; +import com.netflix.servo.monitor.Stopwatch; + +import redis.clients.jedis.Response; +import redis.clients.jedis.Tuple; +import redis.clients.jedis.params.sortedset.ZAddParams; + /** * * @author Viren @@ -74,9 +75,9 @@ public class RedisQueue implements DynoQueue { private ObjectMapper om; - private JedisPool connPool; + private RedisConnection connPool; - private JedisPool nonQuorumPool; + private RedisConnection nonQuorumPool; private ScheduledExecutorService schedulerForUnacksProcessing; @@ -84,17 +85,13 @@ public class RedisQueue implements DynoQueue { private HashPartitioner partitioner = new Murmur3HashPartitioner(); - private int maxHashBuckets = 1024; + private int maxHashBuckets = 32; - public RedisQueue(String redisKeyPrefix, String queueName, String shardName, int unackTime, JedisPool pool) { - this(redisKeyPrefix, queueName, shardName, unackTime, unackTime, pool); - } - - public RedisQueue(String redisKeyPrefix, String queueName, String shardName, int unackScheduleInMS, int unackTime, JedisPool pool) { + public RedisQueue(String redisKeyPrefix, String queueName, String shardName, int unackScheduleInMS, int unackTime, RedisConnection pool) { this(Clock.systemDefaultZone(), redisKeyPrefix, queueName, shardName, unackScheduleInMS, unackTime, pool); } - public RedisQueue(Clock clock, String redisKeyPrefix, String queueName, String shardName, int unackScheduleInMS, int unackTime, JedisPool pool) { + public RedisQueue(Clock clock, String redisKeyPrefix, String queueName, String shardName, int unackScheduleInMS, int unackTime, RedisConnection pool) { this.clock = clock; this.queueName = queueName; this.shardName = shardName; @@ -121,7 +118,7 @@ public RedisQueue(Clock clock, String redisKeyPrefix, String queueName, String s schedulerForUnacksProcessing.scheduleAtFixedRate(() -> processUnacks(), unackScheduleInMS, unackScheduleInMS, TimeUnit.MILLISECONDS); - logger.info(RedisQueue.class.getName() + " is ready to serve " + queueName); + logger.info(RedisQueue.class.getName() + " is ready to serve " + queueName + ", shard=" + shardName); } @@ -129,7 +126,7 @@ public RedisQueue(Clock clock, String redisKeyPrefix, String queueName, String s * * @param nonQuorumPool When using a cluster like Dynomite, which relies on the quorum reads, supply a separate non-quorum read connection for ops like size etc. */ - public void setNonQuorumPool(JedisPool nonQuorumPool) { + public void setNonQuorumPool(RedisConnection nonQuorumPool) { this.nonQuorumPool = nonQuorumPool; } @@ -147,10 +144,10 @@ public int getUnackTime() { public List push(final List messages) { Stopwatch sw = monitor.start(monitor.push, messages.size()); - Jedis conn = connPool.getResource(); + RedisConnection conn = connPool.getResource(); try { - Pipeline pipe = conn.pipelined(); + Pipe pipe = conn.pipelined(); for (Message message : messages) { String json = om.writeValueAsString(message); @@ -188,7 +185,7 @@ private String unackShardKey(String messageId) { public List peek(final int messageCount) { Stopwatch sw = monitor.peek.start(); - Jedis jedis = connPool.getResource(); + RedisConnection jedis = connPool.getResource(); try { @@ -248,10 +245,10 @@ private List _pop(List batch) throws Exception { List popped = new LinkedList<>(); ZAddParams zParams = ZAddParams.zAddParams().nx(); - Jedis jedis = connPool.getResource(); + RedisConnection jedis = connPool.getResource(); try { - Pipeline pipe = jedis.pipelined(); + Pipe pipe = jedis.pipelined(); List> zadds = new ArrayList<>(batch.size()); for (int i = 0; i < batch.size(); i++) { String msgId = batch.get(i); @@ -317,7 +314,7 @@ private List _pop(List batch) throws Exception { public boolean ack(String messageId) { Stopwatch sw = monitor.ack.start(); - Jedis jedis = connPool.getResource(); + RedisConnection jedis = connPool.getResource(); try { @@ -339,8 +336,8 @@ public boolean ack(String messageId) { public void ack(List messages) { Stopwatch sw = monitor.ack.start(); - Jedis jedis = connPool.getResource(); - Pipeline pipe = jedis.pipelined(); + RedisConnection jedis = connPool.getResource(); + Pipe pipe = jedis.pipelined(); List> responses = new LinkedList<>(); try { for(Message msg : messages) { @@ -359,7 +356,7 @@ public void ack(List messages) { pipe.sync(); pipe.close(); - } catch (IOException e) { + } catch (Exception e) { throw new RuntimeException(e); } finally { jedis.close(); @@ -373,7 +370,7 @@ public void ack(List messages) { public boolean setUnackTimeout(String messageId, long timeout) { Stopwatch sw = monitor.ack.start(); - Jedis jedis = connPool.getResource(); + RedisConnection jedis = connPool.getResource(); try { @@ -395,7 +392,7 @@ public boolean setUnackTimeout(String messageId, long timeout) { @Override public boolean setTimeout(String messageId, long timeout) { - Jedis jedis = connPool.getResource(); + RedisConnection jedis = connPool.getResource(); try { String json = jedis.hget(messageStoreKey(messageId), messageId); @@ -429,7 +426,7 @@ public boolean setTimeout(String messageId, long timeout) { public boolean remove(String messageId) { Stopwatch sw = monitor.remove.start(); - Jedis jedis = connPool.getResource(); + RedisConnection jedis = connPool.getResource(); try { @@ -454,7 +451,7 @@ public boolean remove(String messageId) { public Message get(String messageId) { Stopwatch sw = monitor.get.start(); - Jedis jedis = connPool.getResource(); + RedisConnection jedis = connPool.getResource(); try { String json = jedis.hget(messageStoreKey(messageId), messageId); @@ -480,7 +477,7 @@ public Message get(String messageId) { public long size() { Stopwatch sw = monitor.size.start(); - Jedis jedis = nonQuorumPool.getResource(); + RedisConnection jedis = nonQuorumPool.getResource(); try { long size = jedis.zcard(myQueueShard); @@ -496,7 +493,7 @@ public Map> shardSizes() { Stopwatch sw = monitor.size.start(); Map> shardSizes = new HashMap<>(); - Jedis jedis = nonQuorumPool.getResource(); + RedisConnection jedis = nonQuorumPool.getResource(); try { long size = jedis.zcard(myQueueShard); @@ -521,7 +518,7 @@ public Map> shardSizes() { @Override public void clear() { - Jedis jedis = connPool.getResource(); + RedisConnection jedis = connPool.getResource(); try { jedis.del(myQueueShard); @@ -541,7 +538,7 @@ public void clear() { } private Set peekIds(int offset, int count) { - Jedis jedis = connPool.getResource(); + RedisConnection jedis = connPool.getResource(); try { double now = Long.valueOf(clock.millis() + 1).doubleValue(); Set scanned = jedis.zrangeByScore(myQueueShard, 0, now, offset, count); @@ -561,7 +558,7 @@ public void processUnacks() { private void processUnacks(String unackShardKey) { Stopwatch sw = monitor.processUnack.start(); - Jedis jedis = connPool.getResource(); + RedisConnection jedis = connPool.getResource(); try { @@ -613,6 +610,4 @@ public void close() throws IOException { schedulerForPrefetchProcessing.shutdown(); monitor.close(); } - - } diff --git a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/conn/DynoClientProxy.java b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/conn/DynoClientProxy.java new file mode 100644 index 0000000..18be436 --- /dev/null +++ b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/conn/DynoClientProxy.java @@ -0,0 +1,91 @@ +/** + * + */ +package com.netflix.dyno.queues.redis.conn; + +import java.util.Set; + +import com.netflix.dyno.jedis.DynoJedisClient; + +import redis.clients.jedis.Tuple; + +/** + * @author Viren + * + */ +public class DynoClientProxy implements RedisConnection { + + private DynoJedisClient jedis; + + + public DynoClientProxy(DynoJedisClient jedis) { + this.jedis = jedis; + } + + @Override + public RedisConnection getResource() { + return this; + } + + @Override + public void close() { + //nothing! + } + + @Override + public Pipe pipelined() { + return new DynoJedisPipe(jedis.pipelined()); + } + + @Override + public String hget(String key, String member) { + return jedis.hget(key, member); + } + + @Override + public Long zrem(String key, String member) { + return jedis.zrem(key, member); + } + + @Override + public Long hdel(String key, String member) { + return jedis.hdel(key, member); + + } + + @Override + public Double zscore(String key, String member) { + return jedis.zscore(key, member); + } + + @Override + public void zadd(String key, double score, String member) { + jedis.zadd(key, score, member); + } + + @Override + public void hset(String key, String member, String json) { + jedis.hset(key, member, json); + } + + @Override + public long zcard(String key) { + return jedis.zcard(key); + } + + @Override + public void del(String key) { + jedis.del(key); + } + + @Override + public Set zrangeByScore(String key, int min, double max, int offset, int count) { + return jedis.zrangeByScore(key, min, max, offset, count); + } + + @Override + public Set zrangeByScoreWithScores(String key, int min, double max, int offset, int count) { + return jedis.zrangeByScoreWithScores(key, min, max, offset, count); + } + +} diff --git a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/conn/DynoJedisPipe.java b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/conn/DynoJedisPipe.java new file mode 100644 index 0000000..90b9e05 --- /dev/null +++ b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/conn/DynoJedisPipe.java @@ -0,0 +1,65 @@ +/** + * + */ +package com.netflix.dyno.queues.redis.conn; + +import com.netflix.dyno.jedis.DynoJedisPipeline; + +import redis.clients.jedis.Response; +import redis.clients.jedis.params.sortedset.ZAddParams; + +/** + * @author Viren + * + */ +public class DynoJedisPipe implements Pipe { + + private DynoJedisPipeline pipe; + + public DynoJedisPipe(DynoJedisPipeline pipe) { + this.pipe = pipe; + } + + @Override + public void hset(String key, String field, String value) { + pipe.hset(key, field, value); + + } + + @Override + public Response zadd(String key, double score, String member) { + return pipe.zadd(key, score, member); + } + + @Override + public Response zadd(String key, double score, String member, ZAddParams zParams) { + return pipe.zadd(key, score, member, zParams); + } + + @Override + public Response zrem(String key, String member) { + return pipe.zrem(key, member); + } + + @Override + public Response hget(String key, String member) { + return pipe.hget(key, member); + } + + @Override + public Response hdel(String key, String member) { + return pipe.hdel(key, member); + } + + @Override + public void sync() { + pipe.sync(); + } + + @Override + public void close() throws Exception { + pipe.close(); + } + + +} diff --git a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/conn/JedisProxy.java b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/conn/JedisProxy.java new file mode 100644 index 0000000..0a11689 --- /dev/null +++ b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/conn/JedisProxy.java @@ -0,0 +1,97 @@ +/** + * + */ +package com.netflix.dyno.queues.redis.conn; + +import java.util.Set; + +import redis.clients.jedis.Jedis; +import redis.clients.jedis.JedisPool; +import redis.clients.jedis.Tuple; + +/** + * @author Viren + * + */ +public class JedisProxy implements RedisConnection { + + private JedisPool pool; + + private Jedis jedis; + + public JedisProxy(JedisPool pool) { + this.pool = pool; + } + + public JedisProxy(Jedis jedis) { + this.jedis = jedis; + } + + @Override + public RedisConnection getResource() { + Jedis jedis = pool.getResource(); + return new JedisProxy(jedis); + } + + @Override + public void close() { + jedis.close(); + } + + @Override + public Pipe pipelined() { + return new RedisPipe(jedis.pipelined()); + } + + @Override + public String hget(String key, String member) { + return jedis.hget(key, member); + } + + @Override + public Long zrem(String key, String member) { + return jedis.zrem(key, member); + } + + @Override + public Long hdel(String key, String member) { + return jedis.hdel(key, member); + + } + + @Override + public Double zscore(String key, String member) { + return jedis.zscore(key, member); + } + + @Override + public void zadd(String key, double unackScore, String member) { + jedis.zadd(key, unackScore, member); + } + + @Override + public void hset(String key, String member, String json) { + jedis.hset(key, member, json); + } + + @Override + public long zcard(String key) { + return jedis.zcard(key); + } + + @Override + public void del(String key) { + jedis.del(key); + } + + @Override + public Set zrangeByScore(String key, int min, double max, int offset, int count) { + return jedis.zrangeByScore(key, min, max, offset, count); + } + + @Override + public Set zrangeByScoreWithScores(String key, int min, double max, int offset, int count) { + return jedis.zrangeByScoreWithScores(key, min, max, offset, count); + } + +} diff --git a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/conn/Pipe.java b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/conn/Pipe.java new file mode 100644 index 0000000..2cd3e4f --- /dev/null +++ b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/conn/Pipe.java @@ -0,0 +1,83 @@ +package com.netflix.dyno.queues.redis.conn; + +import com.netflix.dyno.jedis.DynoJedisPipeline; + +import redis.clients.jedis.Pipeline; +import redis.clients.jedis.Response; +import redis.clients.jedis.params.sortedset.ZAddParams; + +/** + * + * @author Viren + *

+ * Abstraction of Redis Pipeline. + * The abstraction is required as there is no common interface between DynoJedisPipeline and Jedis' Pipeline classes. + *

+ * @see DynoJedisPipeline + * @see Pipeline + * + */ +public interface Pipe { + + /** + * + * @param key + * @param field + * @param value + */ + public void hset(String key, String field, String value); + + /** + * + * @param key + * @param score + * @param member + * @return + */ + public Response zadd(String key, double score, String member); + + /** + * + * @param key + * @param score + * @param member + * @param zParams + * @return + */ + public Response zadd(String key, double score, String member, ZAddParams zParams); + + /** + * + * @param key + * @param member + * @return + */ + public Response zrem(String key, String member); + + /** + * + * @param key + * @param member + * @return + */ + public Response hget(String key, String member); + + /** + * + * @param key + * @param member + * @return + */ + public Response hdel(String key, String member); + + /** + * + */ + public void sync(); + + /** + * + * @throws Exception + */ + public void close() throws Exception; +} \ No newline at end of file diff --git a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/conn/RedisConnection.java b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/conn/RedisConnection.java new file mode 100644 index 0000000..0f56192 --- /dev/null +++ b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/conn/RedisConnection.java @@ -0,0 +1,52 @@ +package com.netflix.dyno.queues.redis.conn; + +import java.util.Set; + +import com.netflix.dyno.jedis.DynoJedisClient; + +import redis.clients.jedis.Jedis; +import redis.clients.jedis.Tuple; + +/** + * Abstraction of Redis connection. + * + * @author viren + *

+ * The methods are 1-1 proxies from Jedis. See Jedis documentation for the details. + *

+ * @see Jedis + * @see DynoJedisClient + */ +public interface RedisConnection { + + /** + * + * @return Returns the underlying connection resource. For connection pool, returns the actual connection + */ + public RedisConnection getResource(); + + public String hget(String messkeyageStoreKey, String member); + + public Long zrem(String key, String member); + + public Long hdel(String key, String member); + + public Double zscore(String key, String member); + + public void zadd(String key, double score, String member); + + public void hset(String key, String id, String json); + + public long zcard(String key); + + public void del(String key); + + public Set zrangeByScore(String key, int min, double max, int offset, int count); + + public Set zrangeByScoreWithScores(String key, int min, double max, int offset, int count); + + public void close(); + + public Pipe pipelined(); + +} \ No newline at end of file diff --git a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/conn/RedisPipe.java b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/conn/RedisPipe.java new file mode 100644 index 0000000..216a6f2 --- /dev/null +++ b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/conn/RedisPipe.java @@ -0,0 +1,64 @@ +/** + * + */ +package com.netflix.dyno.queues.redis.conn; + +import redis.clients.jedis.Pipeline; +import redis.clients.jedis.Response; +import redis.clients.jedis.params.sortedset.ZAddParams; + +/** + * @author Viren + * + */ +public class RedisPipe implements Pipe { + + private Pipeline pipe; + + public RedisPipe(Pipeline pipe) { + this.pipe = pipe; + } + + @Override + public void hset(String key, String field, String value) { + pipe.hset(key, field, value); + + } + + @Override + public Response zadd(String key, double score, String member) { + return pipe.zadd(key, score, member); + } + + @Override + public Response zadd(String key, double score, String member, ZAddParams zParams) { + return pipe.zadd(key, score, member, zParams); + } + + @Override + public Response zrem(String key, String member) { + return pipe.zrem(key, member); + } + + @Override + public Response hget(String key, String member) { + return pipe.hget(key, member); + } + + @Override + public Response hdel(String key, String member) { + return pipe.hdel(key, member); + } + + @Override + public void sync() { + pipe.sync(); + } + + @Override + public void close() throws Exception { + pipe.close(); + } + + +} diff --git a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/DynoShardSupplier.java b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/shard/DynoShardSupplier.java similarity index 97% rename from dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/DynoShardSupplier.java rename to dyno-queues-redis/src/main/java/com/netflix/dyno/queues/shard/DynoShardSupplier.java index 54245e2..c1a43d2 100644 --- a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/DynoShardSupplier.java +++ b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/shard/DynoShardSupplier.java @@ -16,7 +16,7 @@ /** * */ -package com.netflix.dyno.queues.redis; +package com.netflix.dyno.queues.shard; import java.util.Set; import java.util.stream.Collectors; diff --git a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/SingleShardSupplier.java b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/shard/SingleShardSupplier.java similarity index 96% rename from dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/SingleShardSupplier.java rename to dyno-queues-redis/src/main/java/com/netflix/dyno/queues/shard/SingleShardSupplier.java index 716b423..0162812 100644 --- a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/SingleShardSupplier.java +++ b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/shard/SingleShardSupplier.java @@ -16,7 +16,7 @@ /** * */ -package com.netflix.dyno.queues.redis; +package com.netflix.dyno.queues.shard; import java.util.Set; diff --git a/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/BenchmarkTests.java b/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/BenchmarkTests.java index 63e99e5..c9f9c45 100644 --- a/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/BenchmarkTests.java +++ b/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/BenchmarkTests.java @@ -4,13 +4,15 @@ package com.netflix.dyno.queues.redis; import java.util.ArrayList; +import java.util.LinkedList; import java.util.List; import java.util.UUID; import java.util.concurrent.TimeUnit; +import com.netflix.dyno.connectionpool.Host; +import com.netflix.dyno.queues.DynoQueue; import com.netflix.dyno.queues.Message; -import redis.clients.jedis.JedisPool; import redis.clients.jedis.JedisPoolConfig; /** @@ -19,18 +21,31 @@ */ public class BenchmarkTests { - private RedisQueue queue; + private DynoQueue queue; public BenchmarkTests() { + List hosts = new LinkedList<>(); + hosts.add(new Host("localhost", 6379, "us-east-1a")); + QueueBuilder qb = new QueueBuilder(); + JedisPoolConfig config = new JedisPoolConfig(); config.setTestOnBorrow(true); config.setTestOnCreate(true); config.setMaxTotal(10); config.setMaxIdle(5); config.setMaxWaitMillis(60_000); - JedisPool pool = new JedisPool(config, "localhost", 6379); - queue = new RedisQueue("perf", "TEST_QUEUE", "x", 60000_000, pool); + + queue = qb + .setCurrentShard("a") + .setHostToShardMap((Host h) -> h.getRack().substring(h.getRack().length()-1)) + .setQueueName("testq") + .setRedisKeyPrefix("keyprefix") + .setUnackTime(60_000_000) + .useNonDynomiteRedis(config, hosts) + .build(); + + System.out.println("Instance: " + queue.getClass().getName()); } public void publish() { @@ -55,9 +70,10 @@ public void publish() { public void consume() { try { + long s = System.currentTimeMillis(); int loopCount = 100; - int batchSize = 2000; + int batchSize = 3500; int count = 0; for(int i = 0; i < loopCount; i++) { List popped = queue.pop(batchSize, 1, TimeUnit.MILLISECONDS); @@ -67,7 +83,7 @@ public void consume() { long e = System.currentTimeMillis(); long diff = e-s; long throughput = 1000 * ((count)/diff); - System.out.println("Consume time: " + diff + ", read throughput: " + throughput + " msg/sec, read: " + count); + System.out.println("Consume time: " + diff + ", read throughput: " + throughput + " msg/sec, messages read: " + count); }catch(Exception e) { e.printStackTrace(); } @@ -86,13 +102,11 @@ public static void main(String[] args) throws Exception { try { BenchmarkTests tests = new BenchmarkTests(); - - for(int i = 0; i < 20; i++) { - tests.publish(); - tests.consume(); - } - - } finally { + tests.publish(); + tests.consume(); + } catch(Exception e) { + e.printStackTrace(); + }finally { System.exit(0); } } diff --git a/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/DynoShardSupplierTest.java b/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/DynoShardSupplierTest.java index 874fb6e..65c3066 100644 --- a/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/DynoShardSupplierTest.java +++ b/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/DynoShardSupplierTest.java @@ -32,6 +32,7 @@ import com.netflix.dyno.connectionpool.Host; import com.netflix.dyno.connectionpool.Host.Status; +import com.netflix.dyno.queues.shard.DynoShardSupplier; import com.netflix.dyno.connectionpool.HostSupplier; /** diff --git a/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/RedisDynoQueueTest2.java b/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/RedisDynoQueueTest2.java index 12e1c6e..d8ba84a 100644 --- a/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/RedisDynoQueueTest2.java +++ b/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/RedisDynoQueueTest2.java @@ -43,6 +43,7 @@ import com.google.common.util.concurrent.Uninterruptibles; import com.netflix.dyno.queues.Message; +import com.netflix.dyno.queues.redis.conn.JedisProxy; import redis.clients.jedis.Jedis; import redis.clients.jedis.JedisPool; @@ -75,7 +76,8 @@ public static void setUpBeforeClass() throws Exception { JedisPool pool = new JedisPool(config, "localhost", 6379); dynoClient = new Jedis("localhost", 6379, 0, 0); dynoClient.flushAll(); - rdq = new RedisQueue(redisKeyPrefix, queueName, "x", 1_000, pool); + + rdq = new RedisQueue(redisKeyPrefix, queueName, "x", 1_000, 1_000, new JedisProxy(pool)); messageKeyPrefix = redisKeyPrefix + ".MESSAGE."; } diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index 53b60c3..9a4bb89 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,6 +1,6 @@ -#Wed Sep 20 15:04:48 PDT 2017 +#Sun Feb 04 11:53:34 PST 2018 distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-3.1-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-3.1-all.zip From 6335430c8119e0d23682bd102926ac97f5cdda68 Mon Sep 17 00:00:00 2001 From: Viren Baraiya Date: Sun, 11 Feb 2018 15:10:21 -0800 Subject: [PATCH 05/91] fixed https://github.com/Netflix/dyno-queues/issues/19 --- .../main/java/com/netflix/dyno/queues/redis/RedisDynoQueue.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/RedisDynoQueue.java b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/RedisDynoQueue.java index 6a6dd48..e19462e 100644 --- a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/RedisDynoQueue.java +++ b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/RedisDynoQueue.java @@ -163,7 +163,7 @@ public List push(final List messages) { for (Message message : messages) { String json = om.writeValueAsString(message); quorumConn.hset(messageStoreKey, message.getId(), json); - double priority = message.getPriority() / 100; + double priority = message.getPriority() / 100.0; double score = Long.valueOf(clock.millis() + message.getTimeout()).doubleValue() + priority; String shard = getNextShard(); String queueShard = getQueueShardKey(queueName, shard); From 647f812118a45c2ebf2c96eb90e600691f59b525 Mon Sep 17 00:00:00 2001 From: Viren Baraiya Date: Fri, 16 Mar 2018 11:25:20 -0700 Subject: [PATCH 06/91] v2 recipe that uses redis pipes --- .../com/netflix/dyno/queues/DynoQueue.java | 5 + .../dyno/queues/redis/QueueBuilder.java | 253 ---- .../dyno/queues/redis/QueueMonitor.java | 3 +- .../dyno/queues/redis/RedisDynoQueue.java | 4 +- .../netflix/dyno/queues/redis/RedisQueue.java | 1129 +++++++++-------- .../dyno/queues/redis/conn/DynoJedisPipe.java | 111 +- .../redis/{ => v2}/MultiRedisQueue.java | 10 +- .../dyno/queues/redis/v2/QueueBuilder.java | 235 ++++ .../dyno/queues/redis/BaseQueueTests.java | 299 +++++ .../dyno/queues/redis/BenchmarkTests.java | 114 -- .../queues/redis/DynoShardSupplierTest.java | 2 +- .../dyno/queues/redis/RedisDynoQueueTest.java | 2 +- .../queues/redis/RedisDynoQueueTest2.java | 4 +- .../benchmark/BenchmarkTestsDynoJedis.java | 89 ++ .../redis/benchmark/BenchmarkTestsJedis.java | 58 + .../benchmark/BenchmarkTestsNoPipelines.java | 103 ++ .../redis/benchmark/QueueBenchmark.java | 94 ++ .../queues/redis/pieline/DynoJedisTests.java | 99 ++ .../dyno/queues/redis/pieline/JedisTests.java | 74 ++ 19 files changed, 1705 insertions(+), 983 deletions(-) delete mode 100644 dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/QueueBuilder.java rename dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/{ => v2}/MultiRedisQueue.java (92%) create mode 100644 dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/v2/QueueBuilder.java create mode 100644 dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/BaseQueueTests.java delete mode 100644 dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/BenchmarkTests.java create mode 100644 dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/benchmark/BenchmarkTestsDynoJedis.java create mode 100644 dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/benchmark/BenchmarkTestsJedis.java create mode 100644 dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/benchmark/BenchmarkTestsNoPipelines.java create mode 100644 dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/benchmark/QueueBenchmark.java create mode 100644 dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/pieline/DynoJedisTests.java create mode 100644 dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/pieline/JedisTests.java diff --git a/dyno-queues-core/src/main/java/com/netflix/dyno/queues/DynoQueue.java b/dyno-queues-core/src/main/java/com/netflix/dyno/queues/DynoQueue.java index ea53058..398ce91 100644 --- a/dyno-queues-core/src/main/java/com/netflix/dyno/queues/DynoQueue.java +++ b/dyno-queues-core/src/main/java/com/netflix/dyno/queues/DynoQueue.java @@ -134,4 +134,9 @@ public interface DynoQueue extends Closeable { * Truncates the entire queue. Use with caution! */ public void clear(); + + /** + * Process un-acknowledged messages. The messages which are polled by the client but not ack'ed are moved back to queue + */ + public void processUnacks(); } diff --git a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/QueueBuilder.java b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/QueueBuilder.java deleted file mode 100644 index 722f188..0000000 --- a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/QueueBuilder.java +++ /dev/null @@ -1,253 +0,0 @@ -/** - * - */ -package com.netflix.dyno.queues.redis; - -import java.time.Clock; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -import com.google.common.base.Function; -import com.google.common.collect.Collections2; -import com.google.common.collect.Lists; -import com.netflix.appinfo.AmazonInfo; -import com.netflix.appinfo.AmazonInfo.MetaDataKey; -import com.netflix.appinfo.InstanceInfo; -import com.netflix.appinfo.InstanceInfo.InstanceStatus; -import com.netflix.discovery.EurekaClient; -import com.netflix.discovery.shared.Application; -import com.netflix.dyno.connectionpool.Host; -import com.netflix.dyno.connectionpool.Host.Status; -import com.netflix.dyno.contrib.EurekaHostsSupplier; -import com.netflix.dyno.jedis.DynoJedisClient; -import com.netflix.dyno.queues.DynoQueue; -import com.netflix.dyno.queues.redis.conn.DynoClientProxy; -import com.netflix.dyno.queues.redis.conn.JedisProxy; -import com.netflix.dyno.queues.redis.conn.RedisConnection; - -import redis.clients.jedis.JedisPool; -import redis.clients.jedis.JedisPoolConfig; - -/** - * @author Viren - * - */ -public class QueueBuilder { - - private Clock clock; - - private String queueName; - - private EurekaClient ec; - - private String dynomiteClusterName; - - private String redisKeyPrefix; - - private int unackTime; - - private String currentShard; - - private Function hostToShardMap; - - private int nonQuorumPort; - - private List hosts; - - private JedisPoolConfig redisPoolConfig; - - /** - * @param clock the Clock instance to set - * @return instance of QueueBuilder - */ - public QueueBuilder setClock(Clock clock) { - this.clock = clock; - return this; - } - - /** - * @param queueName the queueName to set - * @return instance of QueueBuilder - */ - public QueueBuilder setQueueName(String queueName) { - this.queueName = queueName; - return this; - } - - /** - * @param redisKeyPrefix Prefix used for all the keys in Redis - * @return instance of QueueBuilder - */ - public QueueBuilder setRedisKeyPrefix(String redisKeyPrefix) { - this.redisKeyPrefix = redisKeyPrefix; - return this; - } - - /** - * @param ec the ec to set - * @return instance of QueueBuilder - */ - - /** - * - * @param ec EurekaClient instance used to discover the hosts in the dynomite cluster - * @param dynomiteClusterName Name of the Dynomite Cluster to be used - * @param nonQuorumPort Direct Redis port used to make non-quorumed queries - used when querying the message counts - * @return instance of QueueBuilder - */ - public QueueBuilder useDynomiteCluster(EurekaClient ec, String dynomiteClusterName, int nonQuorumPort) { - this.ec = ec; - this.dynomiteClusterName = dynomiteClusterName; - this.nonQuorumPort = nonQuorumPort; - return this; - } - - /** - * - * @param redisPoolConfig - * @return instance of QueueBuilder - */ - public QueueBuilder useNonDynomiteRedis(JedisPoolConfig redisPoolConfig, List redisHosts) { - this.redisPoolConfig = redisPoolConfig; - this.hosts = redisHosts; - return this; - } - - /** - * - * @param hostToShardMap Mapping from a Host to a queue shard - * @return instance of QueueBuilder - */ - public QueueBuilder setHostToShardMap(Function hostToShardMap) { - this.hostToShardMap = hostToShardMap; - return this; - } - - /** - * @param unackTime Time in millisecond, after which the uncked messages will be re-queued for the delivery - * @return instance of QueueBuilder - */ - public QueueBuilder setUnackTime(int unackTime) { - this.unackTime = unackTime; - return this; - } - - /** - * @param currentShard Name of the current shard - * @return instance of QueueBuilder - */ - public QueueBuilder setCurrentShard(String currentShard) { - this.currentShard = currentShard; - return this; - } - - public DynoQueue build() { - if (clock == null) { - clock = Clock.systemDefaultZone(); - } - - boolean useDynomite = false; - //When using Dynomite - if(dynomiteClusterName != null) { - hosts = getHostsFromEureka(ec, dynomiteClusterName); - useDynomite = true; - } - Map shardMap = new HashMap<>(); - for(Host host : hosts) { - String shard = hostToShardMap.apply(host); - shardMap.put(shard, host); - } - - DynoJedisClient dynoClientRead = null; - DynoJedisClient dynoClient = null; - if(useDynomite) { - String appId = queueName; - EurekaHostsSupplier hostSupplier = new EurekaHostsSupplier(dynomiteClusterName, ec) { - @Override - public List getHosts() { - List hosts = super.getHosts(); - List updatedHosts = new ArrayList<>(hosts.size()); - hosts.forEach(host -> { - updatedHosts.add(new Host(host.getHostName(), host.getIpAddress(), nonQuorumPort, host.getRack(), host.getDatacenter(), host.isUp() ? Status.Up : Status.Down)); - }); - return updatedHosts; - } - }; - - dynoClientRead = new DynoJedisClient.Builder().withApplicationName(appId).withDynomiteClusterName(dynomiteClusterName).withHostSupplier(hostSupplier).build(); - dynoClient = new DynoJedisClient.Builder().withApplicationName(appId).withDynomiteClusterName(dynomiteClusterName).withDiscoveryClient(ec).build(); - } - - - Map queues = new HashMap<>(); - for(String queueShard : shardMap.keySet()) { - - Host host = shardMap.get(queueShard); - String hostAddress = host.getIpAddress(); - if(hostAddress == null || "".equals(hostAddress)) { - hostAddress = host.getHostName(); - } - RedisConnection redisConn = null; - RedisConnection redisConnRead = null; - - if(useDynomite) { - redisConn = new DynoClientProxy(dynoClient); - redisConnRead = new DynoClientProxy(dynoClientRead); - } else{ - JedisPool pool = new JedisPool(redisPoolConfig, hostAddress, host.getPort(), 0); - redisConn = new JedisProxy(pool); - redisConnRead = new JedisProxy(pool); - } - - RedisQueue q = new RedisQueue(clock, redisKeyPrefix, queueName, queueShard, unackTime, unackTime, redisConn); - q.setNonQuorumPool(redisConnRead); - - queues.put(queueShard, q); - } - - if(queues.size() == 1) { - //This is a queue with a single shard - return queues.values().iterator().next(); - } - - MultiRedisQueue queue = new MultiRedisQueue(queueName, currentShard, queues); - return queue; - } - - - private static List getHostsFromEureka(EurekaClient ec, String applicationName) { - - Application app = ec.getApplication(applicationName); - List hosts = new ArrayList(); - - if (app == null) { - return hosts; - } - - List ins = app.getInstances(); - - if (ins == null || ins.isEmpty()) { - return hosts; - } - - hosts = Lists.newArrayList(Collections2.transform(ins, - - new Function() { - @Override - public Host apply(InstanceInfo info) { - - Host.Status status = info.getStatus() == InstanceStatus.UP ? Host.Status.Up : Host.Status.Down; - String rack = null; - if (info.getDataCenterInfo() instanceof AmazonInfo) { - AmazonInfo amazonInfo = (AmazonInfo)info.getDataCenterInfo(); - rack = amazonInfo.get(MetaDataKey.availabilityZone); - } - Host host = new Host(info.getHostName(), info.getIPAddr(), rack, status); - return host; - } - })); - return hosts; - } -} diff --git a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/QueueMonitor.java b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/QueueMonitor.java index 9654d98..ded34c5 100644 --- a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/QueueMonitor.java +++ b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/QueueMonitor.java @@ -33,7 +33,8 @@ /** * @author Viren - * Monitoring for the queue + * Monitoring for the queue, publishes the metrics using servo + * https://github.com/Netflix/servo */ public class QueueMonitor implements Closeable { diff --git a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/RedisDynoQueue.java b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/RedisDynoQueue.java index e19462e..8941231 100644 --- a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/RedisDynoQueue.java +++ b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/RedisDynoQueue.java @@ -51,7 +51,8 @@ /** * * @author Viren - * + * Current Production (March 2018) recipe - well tested in production. + * Note, this recipe does not use redis pipelines and hence the throughput offered is less compared to v2 recipes. */ public class RedisDynoQueue implements DynoQueue { @@ -524,6 +525,7 @@ private Set peekIds(int offset, int count) { } + @Override public void processUnacks() { Stopwatch sw = monitor.processUnack.start(); diff --git a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/RedisQueue.java b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/RedisQueue.java index 1b0d033..059e586 100644 --- a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/RedisQueue.java +++ b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/RedisQueue.java @@ -1,12 +1,12 @@ /** * Copyright 2016 Netflix, Inc. - * + *

* Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -51,563 +51,578 @@ /** * * @author Viren + * Queue implementation that uses redis pipelines that improves the throughput under heavy load. * */ public class RedisQueue implements DynoQueue { - private final Logger logger = LoggerFactory.getLogger(RedisQueue.class); + private final Logger logger = LoggerFactory.getLogger(RedisQueue.class); + + private Clock clock; + + private String queueName; + + private String shardName; + + private String messageStoreKeyPrefix; + + private String myQueueShard; + + private String unackShardKeyPrefix; + + private int unackTime = 60; + + private QueueMonitor monitor; + + private ObjectMapper om; + + private RedisConnection connPool; + + private RedisConnection nonQuorumPool; + + private ScheduledExecutorService schedulerForUnacksProcessing; + + private ScheduledExecutorService schedulerForPrefetchProcessing; + + private HashPartitioner partitioner = new Murmur3HashPartitioner(); + + private int maxHashBuckets = 32; + + public RedisQueue(String redisKeyPrefix, String queueName, String shardName, int unackScheduleInMS, int unackTime, RedisConnection pool) { + this(Clock.systemDefaultZone(), redisKeyPrefix, queueName, shardName, unackScheduleInMS, unackTime, pool); + } + + public RedisQueue(Clock clock, String redisKeyPrefix, String queue, String shardName, int unackScheduleInMS, int unackTime, RedisConnection pool) { + this.clock = clock; + this.queueName = queue; + String qName = "{" + queue + "}"; + this.shardName = shardName; + this.messageStoreKeyPrefix = redisKeyPrefix + ".MESSAGE."; + this.myQueueShard = redisKeyPrefix + ".QUEUE." + qName + "." + shardName; + this.unackShardKeyPrefix = redisKeyPrefix + ".UNACK." + qName + "." + shardName + "."; + this.unackTime = unackTime; + this.connPool = pool; + this.nonQuorumPool = pool; + + ObjectMapper om = new ObjectMapper(); + om.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); + om.configure(DeserializationFeature.FAIL_ON_IGNORED_PROPERTIES, false); + om.configure(DeserializationFeature.FAIL_ON_NULL_FOR_PRIMITIVES, false); + om.setSerializationInclusion(Include.NON_NULL); + om.setSerializationInclusion(Include.NON_EMPTY); + om.disable(SerializationFeature.INDENT_OUTPUT); + + this.om = om; + this.monitor = new QueueMonitor(qName, shardName); + + schedulerForUnacksProcessing = Executors.newScheduledThreadPool(1); + schedulerForPrefetchProcessing = Executors.newScheduledThreadPool(1); + + schedulerForUnacksProcessing.scheduleAtFixedRate(() -> processUnacks(), unackScheduleInMS, unackScheduleInMS, TimeUnit.MILLISECONDS); + + logger.info(RedisQueue.class.getName() + " is ready to serve " + qName + ", shard=" + shardName); + + } + + /** + * + * @param nonQuorumPool When using a cluster like Dynomite, which relies on the quorum reads, supply a separate non-quorum read connection for ops like size etc. + */ + public void setNonQuorumPool(RedisConnection nonQuorumPool) { + this.nonQuorumPool = nonQuorumPool; + } + + @Override + public String getName() { + return queueName; + } + + @Override + public int getUnackTime() { + return unackTime; + } + + @Override + public List push(final List messages) { + + Stopwatch sw = monitor.start(monitor.push, messages.size()); + RedisConnection conn = connPool.getResource(); + try { + + Pipe pipe = conn.pipelined(); + + for (Message message : messages) { + String json = om.writeValueAsString(message); + pipe.hset(messageStoreKey(message.getId()), message.getId(), json); + double priority = message.getPriority() / 100.0; + double score = Long.valueOf(clock.millis() + message.getTimeout()).doubleValue() + priority; + pipe.zadd(myQueueShard, score, message.getId()); + } + pipe.sync(); + pipe.close(); + + return messages.stream().map(msg -> msg.getId()).collect(Collectors.toList()); + + } catch (Exception e) { + throw new RuntimeException(e); + } finally { + conn.close(); + sw.stop(); + } + } + + private String messageStoreKey(String msgId) { + Long hash = partitioner.hash(msgId); + long bucket = hash % maxHashBuckets; + return messageStoreKeyPrefix + bucket + ".{" + queueName + "}"; + } + + private String unackShardKey(String messageId) { + Long hash = partitioner.hash(messageId); + long bucket = hash % maxHashBuckets; + return unackShardKeyPrefix + bucket; + } + + @Override + public List peek(final int messageCount) { + + Stopwatch sw = monitor.peek.start(); + RedisConnection jedis = connPool.getResource(); + + try { + + Set ids = peekIds(0, messageCount); + if (ids == null) { + return Collections.emptyList(); + } + + List messages = new LinkedList(); + for (String id : ids) { + String json = jedis.hget(messageStoreKey(id), id); + Message message = om.readValue(json, Message.class); + messages.add(message); + } + return messages; + + } catch (Exception e) { + throw new RuntimeException(e); + } finally { + jedis.close(); + sw.stop(); + } + } + + + // + //Note: This implementation does NOT support long polling. The method itself is synchronized, so implementing long poll could potentially block other threads. + //When required, the long polling should be implemented on the caller side (ie the broker implementation using the recipe) + // + @Override + public synchronized List pop(int messageCount, int wait, TimeUnit unit) { + + if (messageCount < 1) { + return Collections.emptyList(); + } + + Stopwatch sw = monitor.start(monitor.pop, messageCount); + + try { + + List peeked = peekIds(0, messageCount).stream().collect(Collectors.toList()); + List popped = _pop(peeked); + return popped; + + } catch (Exception e) { + throw new RuntimeException(e); + } finally { + sw.stop(); + } + + } + + private List _pop(List batch) throws Exception { + + double unackScore = Long.valueOf(clock.millis() + unackTime).doubleValue(); + + List popped = new LinkedList<>(); + ZAddParams zParams = ZAddParams.zAddParams().nx(); + + RedisConnection jedis = connPool.getResource(); + try { + + Pipe pipe = jedis.pipelined(); + List> zadds = new ArrayList<>(batch.size()); + + for (int i = 0; i < batch.size(); i++) { + String msgId = batch.get(i); + if (msgId == null) { + break; + } + zadds.add(pipe.zadd(unackShardKey(msgId), unackScore, msgId, zParams)); + } + pipe.sync(); + + pipe = jedis.pipelined(); + int count = zadds.size(); + List zremIds = new ArrayList<>(count); + List> zremRes = new LinkedList<>(); + + for (int i = 0; i < count; i++) { + long added = zadds.get(i).get(); + if (added == 0) { + if (logger.isDebugEnabled()) { + logger.debug("Cannot add {} to unack queue shard", batch.get(i)); + } + monitor.misses.increment(); + continue; + } + String id = batch.get(i); + zremIds.add(id); + zremRes.add(pipe.zrem(myQueueShard, id)); + } + pipe.sync(); + + pipe = jedis.pipelined(); + List> getRes = new ArrayList<>(count); + for (int i = 0; i < zremRes.size(); i++) { + long removed = zremRes.get(i).get(); + if (removed == 0) { + if (logger.isDebugEnabled()) { + logger.debug("Cannot remove {} from queue shard", zremIds.get(i)); + } + monitor.misses.increment(); + continue; + } + getRes.add(pipe.hget(messageStoreKey(zremIds.get(i)), zremIds.get(i))); + } + pipe.sync(); + + for (int i = 0; i < getRes.size(); i++) { + String json = getRes.get(i).get(); + if (json == null) { + if (logger.isDebugEnabled()) { + logger.debug("Cannot read payload for {}", zremIds.get(i)); + } + monitor.misses.increment(); + continue; + } + Message msg = om.readValue(json, Message.class); + msg.setShard(shardName); + popped.add(msg); + } + return popped; + } finally { + jedis.close(); + } + } + + @Override + public boolean ack(String messageId) { + + Stopwatch sw = monitor.ack.start(); + RedisConnection jedis = connPool.getResource(); + + try { + + Long removed = jedis.zrem(unackShardKey(messageId), messageId); + if (removed > 0) { + jedis.hdel(messageStoreKey(messageId), messageId); + return true; + } + + return false; + + } finally { + jedis.close(); + sw.stop(); + } + } + + @Override + public void ack(List messages) { + + Stopwatch sw = monitor.ack.start(); + RedisConnection jedis = connPool.getResource(); + Pipe pipe = jedis.pipelined(); + List> responses = new LinkedList<>(); + try { + for (Message msg : messages) { + responses.add(pipe.zrem(unackShardKey(msg.getId()), msg.getId())); + } + pipe.sync(); + pipe = jedis.pipelined(); + + List> dels = new LinkedList<>(); + for (int i = 0; i < messages.size(); i++) { + Long removed = responses.get(i).get(); + if (removed > 0) { + dels.add(pipe.hdel(messageStoreKey(messages.get(i).getId()), messages.get(i).getId())); + } + } + pipe.sync(); + + } catch (Exception e) { + throw new RuntimeException(e); + } finally { + jedis.close(); + sw.stop(); + } + + + } + + @Override + public boolean setUnackTimeout(String messageId, long timeout) { + + Stopwatch sw = monitor.ack.start(); + RedisConnection jedis = connPool.getResource(); + + try { + + double unackScore = Long.valueOf(clock.millis() + timeout).doubleValue(); + Double score = jedis.zscore(unackShardKey(messageId), messageId); + if (score != null) { + jedis.zadd(unackShardKey(messageId), unackScore, messageId); + return true; + } + + return false; + + } finally { + jedis.close(); + sw.stop(); + } + } + + @Override + public boolean setTimeout(String messageId, long timeout) { + + RedisConnection jedis = connPool.getResource(); + + try { + String json = jedis.hget(messageStoreKey(messageId), messageId); + if (json == null) { + return false; + } + Message message = om.readValue(json, Message.class); + message.setTimeout(timeout); + + Double score = jedis.zscore(myQueueShard, messageId); + if (score != null) { + double priorityd = message.getPriority() / 100.0; + double newScore = Long.valueOf(clock.millis() + timeout).doubleValue() + priorityd; + jedis.zadd(myQueueShard, newScore, messageId); + json = om.writeValueAsString(message); + jedis.hset(messageStoreKey(message.getId()), message.getId(), json); + return true; + + } + + return false; + } catch (Exception e) { + throw new RuntimeException(e); + } finally { + jedis.close(); + } + + } + + @Override + public boolean remove(String messageId) { + + Stopwatch sw = monitor.remove.start(); + RedisConnection jedis = connPool.getResource(); + + try { + + jedis.zrem(unackShardKey(messageId), messageId); + + Long removed = jedis.zrem(myQueueShard, messageId); + Long msgRemoved = jedis.hdel(messageStoreKey(messageId), messageId); + + if (removed > 0 && msgRemoved > 0) { + return true; + } + + return false; + + } finally { + jedis.close(); + sw.stop(); + } + } + + @Override + public Message get(String messageId) { + + Stopwatch sw = monitor.get.start(); + RedisConnection jedis = connPool.getResource(); + try { + + String json = jedis.hget(messageStoreKey(messageId), messageId); + if (json == null) { + if (logger.isDebugEnabled()) { + logger.debug("Cannot get the message payload " + messageId); + } + return null; + } + + Message msg = om.readValue(json, Message.class); + return msg; + + } catch (Exception e) { + throw new RuntimeException(e); + } finally { + jedis.close(); + sw.stop(); + } + } + + @Override + public long size() { + + Stopwatch sw = monitor.size.start(); + RedisConnection jedis = nonQuorumPool.getResource(); + + try { + long size = jedis.zcard(myQueueShard); + return size; + } finally { + jedis.close(); + sw.stop(); + } + } + + @Override + public Map> shardSizes() { + + Stopwatch sw = monitor.size.start(); + Map> shardSizes = new HashMap<>(); + RedisConnection jedis = nonQuorumPool.getResource(); + try { + + long size = jedis.zcard(myQueueShard); + long uacked = 0; + for (int i = 0; i < maxHashBuckets; i++) { + String unackShardKey = unackShardKeyPrefix + i; + uacked += jedis.zcard(unackShardKey); + } + + Map shardDetails = new HashMap<>(); + shardDetails.put("size", size); + shardDetails.put("uacked", uacked); + shardSizes.put(shardName, shardDetails); + + return shardSizes; + + } finally { + jedis.close(); + sw.stop(); + } + } + + @Override + public void clear() { + RedisConnection jedis = connPool.getResource(); + try { + + jedis.del(myQueueShard); + + for (int i = 0; i < maxHashBuckets; i++) { + String unackShardKey = unackShardKeyPrefix + i; + jedis.del(unackShardKey); + + String messageStoreKey = messageStoreKeyPrefix + i + "." + queueName; + jedis.del(messageStoreKey); + + } + + } finally { + jedis.close(); + } + } + + private Set peekIds(int offset, int count) { + RedisConnection jedis = connPool.getResource(); + try { + double now = Long.valueOf(clock.millis() + 1).doubleValue(); + Set scanned = jedis.zrangeByScore(myQueueShard, 0, now, offset, count); + return scanned; + } finally { + jedis.close(); + } + } + + public void processUnacks() { + for (int i = 0; i < maxHashBuckets; i++) { + String unackShardKey = unackShardKeyPrefix + i; + processUnacks(unackShardKey); + } + } + + private void processUnacks(String unackShardKey) { + + Stopwatch sw = monitor.processUnack.start(); + RedisConnection jedis2 = connPool.getResource(); + + try { + + do { + + long queueDepth = size(); + monitor.queueDepth.record(queueDepth); + + int batchSize = 1_000; + + double now = Long.valueOf(clock.millis()).doubleValue(); + + Set unacks = jedis2.zrangeByScoreWithScores(unackShardKey, 0, now, 0, batchSize); + + if (unacks.size() > 0) { + logger.debug("Adding " + unacks.size() + " messages back to the queue for " + queueName); + } else { + //Nothing more to be processed + return; + } + + List requeue = new LinkedList<>(); + for (Tuple unack : unacks) { + + double score = unack.getScore(); + String member = unack.getElement(); + + String payload = jedis2.hget(messageStoreKey(member), member); + if (payload == null) { + jedis2.zrem(unackShardKey(member), member); + continue; + } + requeue.add(unack); + } + + Pipe pipe = jedis2.pipelined(); + + for (Tuple unack : requeue) { + double score = unack.getScore(); + String member = unack.getElement(); + + pipe.zadd(myQueueShard, score, member); + pipe.zrem(unackShardKey(member), member); + } + pipe.sync(); + + } while (true); + + } finally { + jedis2.close(); + sw.stop(); + } - private Clock clock; - - private String queueName; + } - private String shardName; - - private String messageStoreKeyPrefix; - - private String myQueueShard; - - private String unackShardKeyPrefix; - - private int unackTime = 60; - - private QueueMonitor monitor; - - private ObjectMapper om; - - private RedisConnection connPool; - - private RedisConnection nonQuorumPool; - - private ScheduledExecutorService schedulerForUnacksProcessing; - - private ScheduledExecutorService schedulerForPrefetchProcessing; - - private HashPartitioner partitioner = new Murmur3HashPartitioner(); - - private int maxHashBuckets = 32; - - public RedisQueue(String redisKeyPrefix, String queueName, String shardName, int unackScheduleInMS, int unackTime, RedisConnection pool) { - this(Clock.systemDefaultZone(), redisKeyPrefix, queueName, shardName, unackScheduleInMS, unackTime, pool); - } - - public RedisQueue(Clock clock, String redisKeyPrefix, String queueName, String shardName, int unackScheduleInMS, int unackTime, RedisConnection pool) { - this.clock = clock; - this.queueName = queueName; - this.shardName = shardName; - this.messageStoreKeyPrefix = redisKeyPrefix + ".MESSAGE."; - this.myQueueShard = redisKeyPrefix + ".QUEUE." + queueName + "." + shardName; - this.unackShardKeyPrefix = redisKeyPrefix + ".UNACK." + queueName + "." + shardName + "."; - this.unackTime = unackTime; - this.connPool = pool; - this.nonQuorumPool = pool; - - ObjectMapper om = new ObjectMapper(); - om.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); - om.configure(DeserializationFeature.FAIL_ON_IGNORED_PROPERTIES, false); - om.configure(DeserializationFeature.FAIL_ON_NULL_FOR_PRIMITIVES, false); - om.setSerializationInclusion(Include.NON_NULL); - om.setSerializationInclusion(Include.NON_EMPTY); - om.disable(SerializationFeature.INDENT_OUTPUT); - - this.om = om; - this.monitor = new QueueMonitor(queueName, shardName); - - schedulerForUnacksProcessing = Executors.newScheduledThreadPool(1); - schedulerForPrefetchProcessing = Executors.newScheduledThreadPool(1); - - schedulerForUnacksProcessing.scheduleAtFixedRate(() -> processUnacks(), unackScheduleInMS, unackScheduleInMS, TimeUnit.MILLISECONDS); - - logger.info(RedisQueue.class.getName() + " is ready to serve " + queueName + ", shard=" + shardName); - - } - - /** - * - * @param nonQuorumPool When using a cluster like Dynomite, which relies on the quorum reads, supply a separate non-quorum read connection for ops like size etc. - */ - public void setNonQuorumPool(RedisConnection nonQuorumPool) { - this.nonQuorumPool = nonQuorumPool; - } - - @Override - public String getName() { - return queueName; - } - - @Override - public int getUnackTime() { - return unackTime; - } - - @Override - public List push(final List messages) { - - Stopwatch sw = monitor.start(monitor.push, messages.size()); - RedisConnection conn = connPool.getResource(); - try { - - Pipe pipe = conn.pipelined(); - - for (Message message : messages) { - String json = om.writeValueAsString(message); - pipe.hset(messageStoreKey(message.getId()), message.getId(), json); - double priority = message.getPriority() / 100.0; - double score = Long.valueOf(clock.millis() + message.getTimeout()).doubleValue() + priority; - pipe.zadd(myQueueShard, score, message.getId()); - } - pipe.sync(); - pipe.close(); - - return messages.stream().map(msg -> msg.getId()).collect(Collectors.toList()); - - } catch (Exception e) { - throw new RuntimeException(e); - } finally { - conn.close(); - sw.stop(); - } - } - - private String messageStoreKey(String msgId) { - Long hash = partitioner.hash(msgId); - long bucket = hash % maxHashBuckets; - return messageStoreKeyPrefix + bucket + "." + queueName; - } - - private String unackShardKey(String messageId) { - Long hash = partitioner.hash(messageId); - long bucket = hash % maxHashBuckets; - return unackShardKeyPrefix + bucket; - } - - @Override - public List peek(final int messageCount) { - - Stopwatch sw = monitor.peek.start(); - RedisConnection jedis = connPool.getResource(); - - try { - - Set ids = peekIds(0, messageCount); - if (ids == null) { - return Collections.emptyList(); - } - - List messages = new LinkedList(); - for (String id : ids) { - String json = jedis.hget(messageStoreKey(id), id); - Message message = om.readValue(json, Message.class); - messages.add(message); - } - return messages; - - } catch (Exception e) { - throw new RuntimeException(e); - } finally { - jedis.close(); - sw.stop(); - } - } - - - // - //Note: This implementation does NOT support long polling. The method itself is synchronized, so implementing long poll could potentially block other threads. - //When required, the long polling should be implemented on the caller side (ie the broker implementation using the recipe) - // - @Override - public synchronized List pop(int messageCount, int wait, TimeUnit unit) { - - if (messageCount < 1) { - return Collections.emptyList(); - } - - Stopwatch sw = monitor.start(monitor.pop, messageCount); - - try { - - List peeked = peekIds(0, messageCount).stream().collect(Collectors.toList()); - List popped = _pop(peeked); - return popped; - - } catch (Exception e) { - throw new RuntimeException(e); - } finally { - sw.stop(); - } - - } - - private List _pop(List batch) throws Exception { - - double unackScore = Long.valueOf(clock.millis() + unackTime).doubleValue(); - - List popped = new LinkedList<>(); - ZAddParams zParams = ZAddParams.zAddParams().nx(); - - RedisConnection jedis = connPool.getResource(); - try { - - Pipe pipe = jedis.pipelined(); - List> zadds = new ArrayList<>(batch.size()); - for (int i = 0; i < batch.size(); i++) { - String msgId = batch.get(i); - if(msgId == null) { - break; - } - zadds.add(pipe.zadd(unackShardKey(msgId), unackScore, msgId, zParams)); - } - pipe.sync(); - - int count = zadds.size(); - List zremIds = new ArrayList<>(count); - List> zremRes = new LinkedList<>(); - for (int i = 0; i < count; i++) { - long added = zadds.get(i).get(); - if (added == 0) { - if(logger.isDebugEnabled()) { - logger.debug("Cannot add {} to unack queue shard", batch.get(i)); - } - monitor.misses.increment(); - continue; - } - String id = batch.get(i); - zremIds.add(id); - zremRes.add(pipe.zrem(myQueueShard, id)); - } - pipe.sync(); - - List> getRes = new ArrayList<>(count); - for (int i = 0; i < zremRes.size(); i++) { - long removed = zremRes.get(i).get(); - if (removed == 0) { - if(logger.isDebugEnabled()) { - logger.debug("Cannot remove {} from queue shard", zremIds.get(i)); - } - monitor.misses.increment(); - continue; - } - getRes.add(pipe.hget(messageStoreKey(zremIds.get(i)), zremIds.get(i))); - } - pipe.sync(); - - for (int i = 0; i < getRes.size(); i++) { - String json = getRes.get(i).get(); - if (json == null) { - if(logger.isDebugEnabled()) { - logger.debug("Cannot read payload for {}", zremIds.get(i)); - } - monitor.misses.increment(); - continue; - } - Message msg = om.readValue(json, Message.class); - msg.setShard(shardName); - popped.add(msg); - } - return popped; - } finally { - jedis.close(); - } - } - - @Override - public boolean ack(String messageId) { - - Stopwatch sw = monitor.ack.start(); - RedisConnection jedis = connPool.getResource(); - - try { - - Long removed = jedis.zrem(unackShardKey(messageId), messageId); - if (removed > 0) { - jedis.hdel(messageStoreKey(messageId), messageId); - return true; - } - - return false; - - } finally { - jedis.close(); - sw.stop(); - } - } - - @Override - public void ack(List messages) { - - Stopwatch sw = monitor.ack.start(); - RedisConnection jedis = connPool.getResource(); - Pipe pipe = jedis.pipelined(); - List> responses = new LinkedList<>(); - try { - for(Message msg : messages) { - responses.add(pipe.zrem(unackShardKey(msg.getId()), msg.getId())); - } - pipe.sync(); - pipe.close(); - - List> dels = new LinkedList<>(); - for(int i = 0; i < messages.size(); i++) { - Long removed = responses.get(i).get(); - if (removed > 0) { - dels.add(pipe.hdel(messageStoreKey(messages.get(i).getId()), messages.get(i).getId())); - } - } - pipe.sync(); - pipe.close(); - - } catch (Exception e) { - throw new RuntimeException(e); - } finally { - jedis.close(); - sw.stop(); - } - - - } - - @Override - public boolean setUnackTimeout(String messageId, long timeout) { - - Stopwatch sw = monitor.ack.start(); - RedisConnection jedis = connPool.getResource(); - - try { - - double unackScore = Long.valueOf(clock.millis() + timeout).doubleValue(); - Double score = jedis.zscore(unackShardKey(messageId), messageId); - if (score != null) { - jedis.zadd(unackShardKey(messageId), unackScore, messageId); - return true; - } - - return false; - - } finally { - jedis.close(); - sw.stop(); - } - } - - @Override - public boolean setTimeout(String messageId, long timeout) { - - RedisConnection jedis = connPool.getResource(); - - try { - String json = jedis.hget(messageStoreKey(messageId), messageId); - if (json == null) { - return false; - } - Message message = om.readValue(json, Message.class); - message.setTimeout(timeout); - - Double score = jedis.zscore(myQueueShard, messageId); - if (score != null) { - double priorityd = message.getPriority() / 100.0; - double newScore = Long.valueOf(clock.millis() + timeout).doubleValue() + priorityd; - jedis.zadd(myQueueShard, newScore, messageId); - json = om.writeValueAsString(message); - jedis.hset(messageStoreKey(message.getId()), message.getId(), json); - return true; - - } - - return false; - } catch (Exception e) { - throw new RuntimeException(e); - } finally { - jedis.close(); - } - - } - - @Override - public boolean remove(String messageId) { - - Stopwatch sw = monitor.remove.start(); - RedisConnection jedis = connPool.getResource(); - - try { - - jedis.zrem(unackShardKey(messageId), messageId); - - Long removed = jedis.zrem(myQueueShard, messageId); - Long msgRemoved = jedis.hdel(messageStoreKey(messageId), messageId); - - if (removed > 0 && msgRemoved > 0) { - return true; - } - - return false; - - } finally { - jedis.close(); - sw.stop(); - } - } - - @Override - public Message get(String messageId) { - - Stopwatch sw = monitor.get.start(); - RedisConnection jedis = connPool.getResource(); - try { - - String json = jedis.hget(messageStoreKey(messageId), messageId); - if (json == null) { - if (logger.isDebugEnabled()) { - logger.debug("Cannot get the message payload " + messageId); - } - return null; - } - - Message msg = om.readValue(json, Message.class); - return msg; - - } catch (Exception e) { - throw new RuntimeException(e); - } finally { - jedis.close(); - sw.stop(); - } - } - - @Override - public long size() { - - Stopwatch sw = monitor.size.start(); - RedisConnection jedis = nonQuorumPool.getResource(); - - try { - long size = jedis.zcard(myQueueShard); - return size; - } finally { - jedis.close(); - sw.stop(); - } - } - - @Override - public Map> shardSizes() { - - Stopwatch sw = monitor.size.start(); - Map> shardSizes = new HashMap<>(); - RedisConnection jedis = nonQuorumPool.getResource(); - try { - - long size = jedis.zcard(myQueueShard); - long uacked = 0; - for(int i = 0; i < maxHashBuckets; i++) { - String unackShardKey = unackShardKeyPrefix + i; - uacked += jedis.zcard(unackShardKey); - } - - Map shardDetails = new HashMap<>(); - shardDetails.put("size", size); - shardDetails.put("uacked", uacked); - shardSizes.put(shardName, shardDetails); - - return shardSizes; - - } finally { - jedis.close(); - sw.stop(); - } - } - - @Override - public void clear() { - RedisConnection jedis = connPool.getResource(); - try { - - jedis.del(myQueueShard); - - for(int i = 0; i < maxHashBuckets; i++) { - String unackShardKey = unackShardKeyPrefix + i; - jedis.del(unackShardKey); - - String messageStoreKey = messageStoreKeyPrefix + i + "." + queueName; - jedis.del(messageStoreKey); - - } - - } finally { - jedis.close(); - } - } - - private Set peekIds(int offset, int count) { - RedisConnection jedis = connPool.getResource(); - try { - double now = Long.valueOf(clock.millis() + 1).doubleValue(); - Set scanned = jedis.zrangeByScore(myQueueShard, 0, now, offset, count); - return scanned; - } finally { - jedis.close(); - } - } - - public void processUnacks() { - for(int i = 0; i < maxHashBuckets; i++) { - String unackShardKey = unackShardKeyPrefix + i; - processUnacks(unackShardKey); - } - } - - private void processUnacks(String unackShardKey) { - - Stopwatch sw = monitor.processUnack.start(); - RedisConnection jedis = connPool.getResource(); - - try { - - do { - - long queueDepth = size(); - monitor.queueDepth.record(queueDepth); - - int batchSize = 1_000; - - double now = Long.valueOf(clock.millis()).doubleValue(); - - Set unacks = jedis.zrangeByScoreWithScores(unackShardKey, 0, now, 0, batchSize); - - if (unacks.size() > 0) { - logger.debug("Adding " + unacks.size() + " messages back to the queue for " + queueName); - } else { - //Nothing more to be processed - return; - } - - for (Tuple unack : unacks) { - - double score = unack.getScore(); - String member = unack.getElement(); - - String payload = jedis.hget(messageStoreKey(member), member); - if (payload == null) { - jedis.zrem(unackShardKey(member), member); - continue; - } - - jedis.zadd(myQueueShard, score, member); - jedis.zrem(unackShardKey(member), member); - } - - } while (true); - - } finally { - jedis.close(); - sw.stop(); - } - - } - - @Override - public void close() throws IOException { - schedulerForUnacksProcessing.shutdown(); - schedulerForPrefetchProcessing.shutdown(); - monitor.close(); - } + @Override + public void close() throws IOException { + schedulerForUnacksProcessing.shutdown(); + schedulerForPrefetchProcessing.shutdown(); + monitor.close(); + } } diff --git a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/conn/DynoJedisPipe.java b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/conn/DynoJedisPipe.java index 90b9e05..5079581 100644 --- a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/conn/DynoJedisPipe.java +++ b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/conn/DynoJedisPipe.java @@ -1,65 +1,76 @@ /** - * + * */ package com.netflix.dyno.queues.redis.conn; -import com.netflix.dyno.jedis.DynoJedisPipeline; +import com.netflix.dyno.jedis.DynoJedisPipeline; import redis.clients.jedis.Response; import redis.clients.jedis.params.sortedset.ZAddParams; /** * @author Viren - * */ public class DynoJedisPipe implements Pipe { - private DynoJedisPipeline pipe; - - public DynoJedisPipe(DynoJedisPipeline pipe) { - this.pipe = pipe; - } - - @Override - public void hset(String key, String field, String value) { - pipe.hset(key, field, value); - - } - - @Override - public Response zadd(String key, double score, String member) { - return pipe.zadd(key, score, member); - } - - @Override - public Response zadd(String key, double score, String member, ZAddParams zParams) { - return pipe.zadd(key, score, member, zParams); - } - - @Override - public Response zrem(String key, String member) { - return pipe.zrem(key, member); - } - - @Override - public Response hget(String key, String member) { - return pipe.hget(key, member); - } - - @Override - public Response hdel(String key, String member) { - return pipe.hdel(key, member); - } - - @Override - public void sync() { - pipe.sync(); - } - - @Override - public void close() throws Exception { - pipe.close(); - } - + private DynoJedisPipeline pipe; + + private boolean modified; + + public DynoJedisPipe(DynoJedisPipeline pipe) { + this.pipe = pipe; + this.modified = false; + } + + @Override + public void hset(String key, String field, String value) { + pipe.hset(key, field, value); + this.modified = true; + + } + + @Override + public Response zadd(String key, double score, String member) { + this.modified = true; + return pipe.zadd(key, score, member); + } + + @Override + public Response zadd(String key, double score, String member, ZAddParams zParams) { + this.modified = true; + return pipe.zadd(key, score, member, zParams); + } + + @Override + public Response zrem(String key, String member) { + this.modified = true; + return pipe.zrem(key, member); + } + + @Override + public Response hget(String key, String member) { + this.modified = true; + return pipe.hget(key, member); + } + + @Override + public Response hdel(String key, String member) { + this.modified = true; + return pipe.hdel(key, member); + } + + @Override + public void sync() { + if (modified) { + pipe.sync(); + modified = false; + } + } + + @Override + public void close() throws Exception { + pipe.close(); + } + } diff --git a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/MultiRedisQueue.java b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/v2/MultiRedisQueue.java similarity index 92% rename from dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/MultiRedisQueue.java rename to dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/v2/MultiRedisQueue.java index 92bce97..e257bed 100644 --- a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/MultiRedisQueue.java +++ b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/v2/MultiRedisQueue.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.netflix.dyno.queues.redis; +package com.netflix.dyno.queues.redis.v2; import java.io.IOException; import java.util.HashMap; @@ -27,10 +27,13 @@ import com.netflix.dyno.queues.DynoQueue; import com.netflix.dyno.queues.Message; +import com.netflix.dyno.queues.redis.RedisQueue; /** * @author Viren - * + * MultiRedisQueue exposes a single queue using multiple redis queues. Each RedisQueue is a shard. + * When pushing elements to the queue, does a round robin to push the message to one of the shards. + * When polling, the message is polled from the current shard (shardName) the instance is associated with. */ public class MultiRedisQueue implements DynoQueue { @@ -182,7 +185,8 @@ public void close() throws IOException { queue.close(); } } - + + @Override public void processUnacks() { for(RedisQueue queue : queues.values()) { queue.processUnacks(); diff --git a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/v2/QueueBuilder.java b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/v2/QueueBuilder.java new file mode 100644 index 0000000..7542ab3 --- /dev/null +++ b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/v2/QueueBuilder.java @@ -0,0 +1,235 @@ +/** + * + */ +package com.netflix.dyno.queues.redis.v2; + +import java.time.Clock; +import java.util.*; + +import com.google.common.base.Function; +import com.google.common.collect.Collections2; +import com.google.common.collect.Lists; +import com.netflix.appinfo.AmazonInfo; +import com.netflix.appinfo.AmazonInfo.MetaDataKey; +import com.netflix.appinfo.InstanceInfo; +import com.netflix.appinfo.InstanceInfo.InstanceStatus; +import com.netflix.discovery.EurekaClient; +import com.netflix.discovery.shared.Application; +import com.netflix.dyno.connectionpool.Host; +import com.netflix.dyno.connectionpool.HostSupplier; +import com.netflix.dyno.jedis.DynoJedisClient; +import com.netflix.dyno.queues.DynoQueue; +import com.netflix.dyno.queues.redis.RedisQueue; +import com.netflix.dyno.queues.redis.conn.DynoClientProxy; +import com.netflix.dyno.queues.redis.conn.JedisProxy; +import com.netflix.dyno.queues.redis.conn.RedisConnection; + +import redis.clients.jedis.JedisPool; +import redis.clients.jedis.JedisPoolConfig; + +/** + * @author Viren + * Builder for the queues. + * + */ +public class QueueBuilder { + + private Clock clock; + + private String queueName; + + private EurekaClient ec; + + private String redisKeyPrefix; + + private int unackTime; + + private String currentShard; + + private Function hostToShardMap; + + private int nonQuorumPort; + + private HostSupplier hs; + + private Collection hosts; + + private JedisPoolConfig redisPoolConfig; + + private DynoJedisClient dynoQuorumClient; + + private DynoJedisClient dynoNonQuorumClient; + + /** + * @param clock the Clock instance to set + * @return instance of QueueBuilder + */ + public QueueBuilder setClock(Clock clock) { + this.clock = clock; + return this; + } + + /** + * @param queueName the queueName to set + * @return instance of QueueBuilder + */ + public QueueBuilder setQueueName(String queueName) { + this.queueName = queueName; + return this; + } + + /** + * @param redisKeyPrefix Prefix used for all the keys in Redis + * @return instance of QueueBuilder + */ + public QueueBuilder setRedisKeyPrefix(String redisKeyPrefix) { + this.redisKeyPrefix = redisKeyPrefix; + return this; + } + + /** + * @param redisPoolConfig + * @return instance of QueueBuilder + */ + public QueueBuilder useNonDynomiteRedis(JedisPoolConfig redisPoolConfig, List redisHosts) { + this.redisPoolConfig = redisPoolConfig; + this.hosts = redisHosts; + return this; + } + + /** + * + * @param dynoQuorumClient + * @param dynoNonQuorumClient + * @param hs + * @return + */ + public QueueBuilder useDynomite(DynoJedisClient dynoQuorumClient, DynoJedisClient dynoNonQuorumClient, HostSupplier hs) { + this.dynoQuorumClient = dynoQuorumClient; + this.dynoNonQuorumClient = dynoNonQuorumClient; + this.hs = hs; + return this; + } + + /** + * @param hostToShardMap Mapping from a Host to a queue shard + * @return instance of QueueBuilder + */ + public QueueBuilder setHostToShardMap(Function hostToShardMap) { + this.hostToShardMap = hostToShardMap; + return this; + } + + /** + * @param unackTime Time in millisecond, after which the uncked messages will be re-queued for the delivery + * @return instance of QueueBuilder + */ + public QueueBuilder setUnackTime(int unackTime) { + this.unackTime = unackTime; + return this; + } + + /** + * @param currentShard Name of the current shard + * @return instance of QueueBuilder + */ + public QueueBuilder setCurrentShard(String currentShard) { + this.currentShard = currentShard; + return this; + } + + /** + * + * @return Build an instance of the queue with supplied parameters. + * @see MultiRedisQueue + * @see RedisQueue + */ + public DynoQueue build() { + + if (clock == null) { + clock = Clock.systemDefaultZone(); + } + + boolean useDynomiteCluster = (dynoQuorumClient != null && hs != null); + if(useDynomiteCluster) { + this.hosts = hs.getHosts(); + } + + Map shardMap = new HashMap<>(); + for (Host host : hosts) { + String shard = hostToShardMap.apply(host); + shardMap.put(shard, host); + } + + + Map queues = new HashMap<>(); + + for (String queueShard : shardMap.keySet()) { + + Host host = shardMap.get(queueShard); + String hostAddress = host.getIpAddress(); + if (hostAddress == null || "".equals(hostAddress)) { + hostAddress = host.getHostName(); + } + RedisConnection redisConn = null; + RedisConnection redisConnRead = null; + + if (useDynomiteCluster) { + redisConn = new DynoClientProxy(dynoQuorumClient); + redisConnRead = new DynoClientProxy(dynoNonQuorumClient); + } else { + JedisPool pool = new JedisPool(redisPoolConfig, hostAddress, host.getPort(), 0); + redisConn = new JedisProxy(pool); + redisConnRead = new JedisProxy(pool); + } + + RedisQueue q = new RedisQueue(clock, redisKeyPrefix, queueName, queueShard, unackTime, unackTime, redisConn); + q.setNonQuorumPool(redisConnRead); + + queues.put(queueShard, q); + } + + if (queues.size() == 1) { + //This is a queue with a single shard + return queues.values().iterator().next(); + } + + MultiRedisQueue queue = new MultiRedisQueue(queueName, currentShard, queues); + return queue; + } + + + private static List getHostsFromEureka(EurekaClient ec, String applicationName) { + + Application app = ec.getApplication(applicationName); + List hosts = new ArrayList(); + + if (app == null) { + return hosts; + } + + List ins = app.getInstances(); + + if (ins == null || ins.isEmpty()) { + return hosts; + } + + hosts = Lists.newArrayList(Collections2.transform(ins, + + new Function() { + @Override + public Host apply(InstanceInfo info) { + + Host.Status status = info.getStatus() == InstanceStatus.UP ? Host.Status.Up : Host.Status.Down; + String rack = null; + if (info.getDataCenterInfo() instanceof AmazonInfo) { + AmazonInfo amazonInfo = (AmazonInfo) info.getDataCenterInfo(); + rack = amazonInfo.get(MetaDataKey.availabilityZone); + } + Host host = new Host(info.getHostName(), info.getIPAddr(), rack, status); + return host; + } + })); + return hosts; + } +} diff --git a/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/BaseQueueTests.java b/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/BaseQueueTests.java new file mode 100644 index 0000000..35ad23d --- /dev/null +++ b/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/BaseQueueTests.java @@ -0,0 +1,299 @@ +/** + * Copyright 2016 Netflix, Inc. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.netflix.dyno.queues.redis; + +import com.google.common.util.concurrent.Uninterruptibles; +import com.netflix.dyno.queues.DynoQueue; +import com.netflix.dyno.queues.Message; +import com.netflix.dyno.queues.redis.conn.JedisProxy; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; +import redis.clients.jedis.Jedis; +import redis.clients.jedis.JedisPool; +import redis.clients.jedis.JedisPoolConfig; + +import java.util.*; +import java.util.concurrent.*; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.stream.Collectors; + +import static org.junit.Assert.*; + +public abstract class BaseQueueTests { + + + protected static final String queueName = "test_queue"; + + protected static final String redisKeyPrefix = "testdynoqueues"; + + protected DynoQueue rdq; + + protected String messageKeyPrefix; + + + public abstract DynoQueue getQueue(String redisKeyPrefix, String queueName); + + public BaseQueueTests() { + + this.rdq = getQueue(redisKeyPrefix, queueName); + this.messageKeyPrefix = redisKeyPrefix + ".MESSAGE."; + } + + + @Test + public void testGetName() { + assertEquals(queueName, rdq.getName()); + } + + @Test + public void testGetUnackTime() { + assertEquals(1_000, rdq.getUnackTime()); + } + + @Test + public void testTimeoutUpdate() { + + rdq.clear(); + + String id = UUID.randomUUID().toString(); + Message msg = new Message(id, "Hello World-" + id); + msg.setTimeout(100, TimeUnit.MILLISECONDS); + rdq.push(Arrays.asList(msg)); + + List popped = rdq.pop(1, 10, TimeUnit.MILLISECONDS); + assertNotNull(popped); + assertEquals(0, popped.size()); + + Uninterruptibles.sleepUninterruptibly(500, TimeUnit.MILLISECONDS); + + popped = rdq.pop(1, 1, TimeUnit.SECONDS); + assertNotNull(popped); + assertEquals(1, popped.size()); + + boolean updated = rdq.setUnackTimeout(id, 500); + assertTrue(updated); + popped = rdq.pop(1, 1, TimeUnit.SECONDS); + assertNotNull(popped); + assertEquals(0, popped.size()); + + Uninterruptibles.sleepUninterruptibly(1000, TimeUnit.MILLISECONDS); + rdq.processUnacks(); + popped = rdq.pop(1, 1, TimeUnit.SECONDS); + assertNotNull(popped); + assertEquals(1, popped.size()); + + updated = rdq.setUnackTimeout(id, 10_000); //10 seconds! + assertTrue(updated); + rdq.processUnacks(); + popped = rdq.pop(1, 1, TimeUnit.SECONDS); + assertNotNull(popped); + assertEquals(0, popped.size()); + + updated = rdq.setUnackTimeout(id, 0); + assertTrue(updated); + rdq.processUnacks(); + popped = rdq.pop(1, 1, TimeUnit.SECONDS); + assertNotNull(popped); + assertEquals(1, popped.size()); + + rdq.ack(id); + Map> size = rdq.shardSizes(); + Map values = size.get("a"); + long total = values.values().stream().mapToLong(v -> v).sum(); + assertEquals(0, total); + + popped = rdq.pop(1, 1, TimeUnit.SECONDS); + assertNotNull(popped); + assertEquals(0, popped.size()); + } + + @Test + public void testConcurrency() throws InterruptedException, ExecutionException { + + rdq.clear(); + + final int count = 100; + final AtomicInteger published = new AtomicInteger(0); + + ScheduledExecutorService ses = Executors.newScheduledThreadPool(6); + CountDownLatch publishLatch = new CountDownLatch(1); + Runnable publisher = new Runnable() { + + @Override + public void run() { + List messages = new LinkedList<>(); + for (int i = 0; i < 10; i++) { + Message msg = new Message(UUID.randomUUID().toString(), "Hello World-" + i); + msg.setPriority(new Random().nextInt(98)); + messages.add(msg); + } + if (published.get() >= count) { + publishLatch.countDown(); + return; + } + + published.addAndGet(messages.size()); + rdq.push(messages); + } + }; + + for (int p = 0; p < 3; p++) { + ses.scheduleWithFixedDelay(publisher, 1, 1, TimeUnit.MILLISECONDS); + } + publishLatch.await(); + CountDownLatch latch = new CountDownLatch(count); + List allMsgs = new CopyOnWriteArrayList<>(); + AtomicInteger consumed = new AtomicInteger(0); + AtomicInteger counter = new AtomicInteger(0); + Runnable consumer = new Runnable() { + + @Override + public void run() { + if (consumed.get() >= count) { + return; + } + List popped = rdq.pop(100, 1, TimeUnit.MILLISECONDS); + allMsgs.addAll(popped); + consumed.addAndGet(popped.size()); + popped.stream().forEach(p -> latch.countDown()); + counter.incrementAndGet(); + } + }; + for (int c = 0; c < 2; c++) { + ses.scheduleWithFixedDelay(consumer, 1, 10, TimeUnit.MILLISECONDS); + } + Uninterruptibles.awaitUninterruptibly(latch); + System.out.println("Consumed: " + consumed.get() + ", all: " + allMsgs.size() + " counter: " + counter.get()); + Set uniqueMessages = allMsgs.stream().collect(Collectors.toSet()); + + assertEquals(count, allMsgs.size()); + assertEquals(count, uniqueMessages.size()); + List more = rdq.pop(1, 1, TimeUnit.SECONDS); + assertEquals(0, more.size()); + + ses.shutdownNow(); + } + + @Test + public void testSetTimeout() { + + rdq.clear(); + + Message msg = new Message("x001yx", "Hello World"); + msg.setPriority(3); + msg.setTimeout(10_000); + rdq.push(Arrays.asList(msg)); + + List popped = rdq.pop(1, 1, TimeUnit.SECONDS); + assertTrue(popped.isEmpty()); + + boolean updated = rdq.setTimeout(msg.getId(), 0); + assertTrue(updated); + popped = rdq.pop(2, 1, TimeUnit.SECONDS); + assertEquals(1, popped.size()); + assertEquals(0, popped.get(0).getTimeout()); + } + + @Test + public void testAll() { + + rdq.clear(); + assertEquals(0, rdq.size()); + + int count = 10; + List messages = new LinkedList<>(); + for (int i = 0; i < count; i++) { + Message msg = new Message("" + i, "Hello World-" + i); + msg.setPriority(count - i); + messages.add(msg); + } + rdq.push(messages); + + messages = rdq.peek(count); + + assertNotNull(messages); + assertEquals(count, messages.size()); + long size = rdq.size(); + assertEquals(count, size); + + // We did a peek - let's ensure the messages are still around! + List messages2 = rdq.peek(count); + assertNotNull(messages2); + assertEquals(messages, messages2); + + List poped = rdq.pop(count, 1, TimeUnit.SECONDS); + assertNotNull(poped); + assertEquals(count, poped.size()); + assertEquals(messages, poped); + + Uninterruptibles.sleepUninterruptibly(2, TimeUnit.SECONDS); + rdq.processUnacks(); + + for (Message msg : messages) { + Message found = rdq.get(msg.getId()); + assertNotNull(found); + assertEquals(msg.getId(), found.getId()); + assertEquals(msg.getTimeout(), found.getTimeout()); + } + assertNull(rdq.get("some fake id")); + + List messages3 = rdq.pop(count, 1, TimeUnit.SECONDS); + if (messages3.size() < count) { + List messages4 = rdq.pop(count, 1, TimeUnit.SECONDS); + messages3.addAll(messages4); + } + + assertNotNull(messages3); + assertEquals(10, messages3.size()); + assertEquals(messages.stream().map(msg -> msg.getId()).sorted().collect(Collectors.toList()), messages3.stream().map(msg -> msg.getId()).sorted().collect(Collectors.toList())); + assertEquals(10, messages3.stream().map(msg -> msg.getId()).collect(Collectors.toSet()).size()); + messages3.stream().forEach(System.out::println); + + for (Message msg : messages3) { + assertTrue(rdq.ack(msg.getId())); + assertFalse(rdq.ack(msg.getId())); + } + Uninterruptibles.sleepUninterruptibly(2, TimeUnit.SECONDS); + messages3 = rdq.pop(count, 1, TimeUnit.SECONDS); + assertNotNull(messages3); + assertEquals(0, messages3.size()); + } + + @Before + public void clear() { + rdq.clear(); + } + + @Test + public void testClearQueues() { + rdq.clear(); + int count = 10; + List messages = new LinkedList<>(); + for (int i = 0; i < count; i++) { + Message msg = new Message("x" + i, "Hello World-" + i); + msg.setPriority(count - i); + messages.add(msg); + } + + rdq.push(messages); + assertEquals(count, rdq.size()); + rdq.clear(); + assertEquals(0, rdq.size()); + + } + +} diff --git a/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/BenchmarkTests.java b/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/BenchmarkTests.java deleted file mode 100644 index c9f9c45..0000000 --- a/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/BenchmarkTests.java +++ /dev/null @@ -1,114 +0,0 @@ -/** - * - */ -package com.netflix.dyno.queues.redis; - -import java.util.ArrayList; -import java.util.LinkedList; -import java.util.List; -import java.util.UUID; -import java.util.concurrent.TimeUnit; - -import com.netflix.dyno.connectionpool.Host; -import com.netflix.dyno.queues.DynoQueue; -import com.netflix.dyno.queues.Message; - -import redis.clients.jedis.JedisPoolConfig; - -/** - * @author Viren - * - */ -public class BenchmarkTests { - - private DynoQueue queue; - - public BenchmarkTests() { - List hosts = new LinkedList<>(); - hosts.add(new Host("localhost", 6379, "us-east-1a")); - QueueBuilder qb = new QueueBuilder(); - - JedisPoolConfig config = new JedisPoolConfig(); - config.setTestOnBorrow(true); - config.setTestOnCreate(true); - config.setMaxTotal(10); - config.setMaxIdle(5); - config.setMaxWaitMillis(60_000); - - - queue = qb - .setCurrentShard("a") - .setHostToShardMap((Host h) -> h.getRack().substring(h.getRack().length()-1)) - .setQueueName("testq") - .setRedisKeyPrefix("keyprefix") - .setUnackTime(60_000_000) - .useNonDynomiteRedis(config, hosts) - .build(); - - System.out.println("Instance: " + queue.getClass().getName()); - } - - public void publish() { - - long s = System.currentTimeMillis(); - int loopCount = 100; - int batchSize = 3000; - for(int i = 0; i < loopCount; i++) { - List messages = new ArrayList<>(batchSize); - for(int k = 0; k < batchSize; k++) { - String id = UUID.randomUUID().toString(); - Message message = new Message(id, getPayload()); - messages.add(message); - } - queue.push(messages); - } - long e = System.currentTimeMillis(); - long diff = e-s; - long throughput = 1000 * ((loopCount * batchSize)/diff); - System.out.println("Publish time: " + diff + ", throughput: " + throughput + " msg/sec"); - } - - public void consume() { - try { - - long s = System.currentTimeMillis(); - int loopCount = 100; - int batchSize = 3500; - int count = 0; - for(int i = 0; i < loopCount; i++) { - List popped = queue.pop(batchSize, 1, TimeUnit.MILLISECONDS); - queue.ack(popped); - count += popped.size(); - } - long e = System.currentTimeMillis(); - long diff = e-s; - long throughput = 1000 * ((count)/diff); - System.out.println("Consume time: " + diff + ", read throughput: " + throughput + " msg/sec, messages read: " + count); - }catch(Exception e) { - e.printStackTrace(); - } - } - - private String getPayload() { - StringBuilder sb = new StringBuilder(); - for(int i = 0; i < 1; i++) { - sb.append(UUID.randomUUID().toString()); - sb.append(","); - } - return sb.toString(); - } - - public static void main(String[] args) throws Exception { - try { - - BenchmarkTests tests = new BenchmarkTests(); - tests.publish(); - tests.consume(); - } catch(Exception e) { - e.printStackTrace(); - }finally { - System.exit(0); - } - } - -} diff --git a/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/DynoShardSupplierTest.java b/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/DynoShardSupplierTest.java index 65c3066..d4827cd 100644 --- a/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/DynoShardSupplierTest.java +++ b/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/DynoShardSupplierTest.java @@ -45,7 +45,7 @@ public class DynoShardSupplierTest { public void test(){ HostSupplier hs = new HostSupplier() { @Override - public Collection getHosts() { + public List getHosts() { List hosts = new LinkedList<>(); hosts.add(new Host("host1", 8102, "us-east-1a", Status.Up)); hosts.add(new Host("host1", 8102, "us-east-1b", Status.Up)); diff --git a/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/RedisDynoQueueTest.java b/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/RedisDynoQueueTest.java index 35c58c8..617b47e 100644 --- a/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/RedisDynoQueueTest.java +++ b/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/RedisDynoQueueTest.java @@ -70,7 +70,7 @@ public static void setUpBeforeClass() throws Exception { HostSupplier hs = new HostSupplier() { @Override - public Collection getHosts() { + public List getHosts() { List hosts = new LinkedList<>(); hosts.add(new Host("ec2-11-22-33-444.compute-0.amazonaws.com", 8102, "us-east-1d", Status.Up)); return hosts; diff --git a/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/RedisDynoQueueTest2.java b/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/RedisDynoQueueTest2.java index d8ba84a..fa90292 100644 --- a/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/RedisDynoQueueTest2.java +++ b/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/RedisDynoQueueTest2.java @@ -61,7 +61,7 @@ public class RedisDynoQueueTest2 { private static String messageKeyPrefix; - private static int maxHashBuckets = 1024; + private static int maxHashBuckets = 32; @BeforeClass public static void setUpBeforeClass() throws Exception { @@ -291,7 +291,7 @@ public void testAll() { messages3.stream().forEach(System.out::println); int bucketCounts = 0; for(int i = 0; i < maxHashBuckets; i++) { - bucketCounts += dynoClient.hlen(messageKeyPrefix + i + "." + queueName); + bucketCounts += dynoClient.hlen(messageKeyPrefix + i + ".{" + queueName + "}"); } assertEquals(10, bucketCounts); diff --git a/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/benchmark/BenchmarkTestsDynoJedis.java b/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/benchmark/BenchmarkTestsDynoJedis.java new file mode 100644 index 0000000..c532bfe --- /dev/null +++ b/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/benchmark/BenchmarkTestsDynoJedis.java @@ -0,0 +1,89 @@ +/** + * + */ +package com.netflix.dyno.queues.redis.benchmark; + +import com.netflix.dyno.connectionpool.Host; +import com.netflix.dyno.connectionpool.HostSupplier; +import com.netflix.dyno.connectionpool.TokenMapSupplier; +import com.netflix.dyno.connectionpool.impl.ConnectionPoolConfigurationImpl; +import com.netflix.dyno.connectionpool.impl.lb.HostToken; +import com.netflix.dyno.jedis.DynoJedisClient; +import com.netflix.dyno.queues.redis.v2.QueueBuilder; + +import java.util.*; + +/** + * @author Viren + */ +public class BenchmarkTestsDynoJedis extends QueueBenchmark { + + public BenchmarkTestsDynoJedis() { + + List hosts = new ArrayList<>(1); + hosts.add(new Host("localhost", "127.0.0.1", 6379, "us-east-1c", "us-east-1", Host.Status.Up)); + + + QueueBuilder qb = new QueueBuilder(); + + DynoJedisClient.Builder builder = new DynoJedisClient.Builder(); + HostSupplier hs = new HostSupplier() { + @Override + public List getHosts() { + return hosts; + } + }; + + ConnectionPoolConfigurationImpl cp = new ConnectionPoolConfigurationImpl("test").withTokenSupplier(new TokenMapSupplier() { + + HostToken token = new HostToken(1L, hosts.get(0)); + + @Override + public List getTokens(Set activeHosts) { + return Arrays.asList(token); + } + + @Override + public HostToken getTokenForHost(Host host, Set activeHosts) { + return token; + } + + + }).setLocalRack("us-east-1c").setLocalDataCenter("us-east-1"); + cp.setSocketTimeout(0); + cp.setConnectTimeout(0); + cp.setMaxConnsPerHost(10); + cp.withHashtag("{}"); + + DynoJedisClient client = builder.withApplicationName("test") + .withDynomiteClusterName("test") + .withCPConfig(cp) + .withHostSupplier(hs) + .build(); + + + queue = qb + .setCurrentShard("a") + .setHostToShardMap((Host h) -> h.getRack().substring(h.getRack().length() - 1)) + .setQueueName("testq") + .setRedisKeyPrefix("keyprefix") + .setUnackTime(60_000) + .useDynomite(client, client, hs) + .build(); + } + + + public static void main(String[] args) throws Exception { + try { + + System.out.println("Start"); + BenchmarkTestsDynoJedis tests = new BenchmarkTestsDynoJedis(); + tests.run(); + } catch (Exception e) { + e.printStackTrace(); + } finally { + System.exit(0); + } + } + +} diff --git a/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/benchmark/BenchmarkTestsJedis.java b/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/benchmark/BenchmarkTestsJedis.java new file mode 100644 index 0000000..8b7258e --- /dev/null +++ b/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/benchmark/BenchmarkTestsJedis.java @@ -0,0 +1,58 @@ +/** + * + */ +package com.netflix.dyno.queues.redis.benchmark; + +import java.util.LinkedList; +import java.util.List; + +import com.netflix.dyno.connectionpool.Host; + +import com.netflix.dyno.queues.redis.v2.QueueBuilder; +import redis.clients.jedis.JedisPoolConfig; + +/** + * @author Viren + * + */ +public class BenchmarkTestsJedis extends QueueBenchmark { + + public BenchmarkTestsJedis() { + List hosts = new LinkedList<>(); + hosts.add(new Host("localhost", 6379, "us-east-1a")); + QueueBuilder qb = new QueueBuilder(); + + JedisPoolConfig config = new JedisPoolConfig(); + config.setTestOnBorrow(true); + config.setTestOnCreate(true); + config.setMaxTotal(10); + config.setMaxIdle(5); + config.setMaxWaitMillis(60_000); + + + queue = qb + .setCurrentShard("a") + .setHostToShardMap((Host h) -> h.getRack().substring(h.getRack().length()-1)) + .setQueueName("testq") + .setRedisKeyPrefix("keyprefix") + .setUnackTime(60_000_000) + .useNonDynomiteRedis(config, hosts) + .build(); + + System.out.println("Instance: " + queue.getClass().getName()); + } + + public static void main(String[] args) throws Exception { + try { + + BenchmarkTestsJedis tests = new BenchmarkTestsJedis(); + tests.run(); + + } catch(Exception e) { + e.printStackTrace(); + }finally { + System.exit(0); + } + } + +} diff --git a/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/benchmark/BenchmarkTestsNoPipelines.java b/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/benchmark/BenchmarkTestsNoPipelines.java new file mode 100644 index 0000000..c3147b7 --- /dev/null +++ b/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/benchmark/BenchmarkTestsNoPipelines.java @@ -0,0 +1,103 @@ +/** + * + */ +package com.netflix.dyno.queues.redis.benchmark; + +import com.netflix.dyno.connectionpool.Host; +import com.netflix.dyno.connectionpool.HostSupplier; +import com.netflix.dyno.connectionpool.TokenMapSupplier; +import com.netflix.dyno.connectionpool.impl.ConnectionPoolConfigurationImpl; +import com.netflix.dyno.connectionpool.impl.lb.HostToken; +import com.netflix.dyno.jedis.DynoJedisClient; +import com.netflix.dyno.queues.ShardSupplier; +import com.netflix.dyno.queues.redis.RedisQueues; + +import java.util.*; +import java.util.stream.Collectors; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; + +/** + * @author Viren + */ +public class BenchmarkTestsNoPipelines extends QueueBenchmark { + + + public BenchmarkTestsNoPipelines() { + + String redisKeyPrefix = "perftestnopipe"; + String queueName = "nopipequeue"; + + List hosts = new ArrayList<>(1); + hosts.add(new Host("localhost", "127.0.0.1", 6379, "us-east-1c", "us-east-1", Host.Status.Up)); + + DynoJedisClient.Builder builder = new DynoJedisClient.Builder(); + HostSupplier hs = new HostSupplier() { + @Override + public List getHosts() { + return hosts; + } + }; + + ConnectionPoolConfigurationImpl cp = new ConnectionPoolConfigurationImpl("test").withTokenSupplier(new TokenMapSupplier() { + + HostToken token = new HostToken(1L, hosts.get(0)); + + @Override + public List getTokens(Set activeHosts) { + return Arrays.asList(token); + } + + @Override + public HostToken getTokenForHost(Host host, Set activeHosts) { + return token; + } + + + }).setLocalRack("us-east-1c").setLocalDataCenter("us-east-1"); + cp.setSocketTimeout(0); + cp.setConnectTimeout(0); + cp.setMaxConnsPerHost(10); + + + DynoJedisClient client = builder.withApplicationName("test") + .withDynomiteClusterName("test") + .withCPConfig(cp) + .withHostSupplier(hs) + .build(); + + Set allShards = hs.getHosts().stream().map(host -> host.getRack().substring(host.getRack().length() - 2)).collect(Collectors.toSet()); + String shardName = allShards.iterator().next(); + ShardSupplier ss = new ShardSupplier() { + + @Override + public Set getQueueShards() { + return allShards; + } + + @Override + public String getCurrentShard() { + return shardName; + } + }; + + RedisQueues rq = new RedisQueues(client, client, redisKeyPrefix, ss, 60_000, 1_000_000); + queue = rq.get(queueName); + } + + + public static void main(String[] args) throws Exception { + try { + + BenchmarkTestsNoPipelines tests = new BenchmarkTestsNoPipelines(); + tests.run(); + + } catch (Exception e) { + e.printStackTrace(); + } finally { + System.exit(0); + } + } + +} diff --git a/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/benchmark/QueueBenchmark.java b/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/benchmark/QueueBenchmark.java new file mode 100644 index 0000000..b3367b7 --- /dev/null +++ b/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/benchmark/QueueBenchmark.java @@ -0,0 +1,94 @@ +package com.netflix.dyno.queues.redis.benchmark; + +import com.netflix.dyno.queues.DynoQueue; +import com.netflix.dyno.queues.Message; +import com.netflix.dyno.queues.redis.RedisQueue; + +import java.util.*; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; + +public abstract class QueueBenchmark { + + protected DynoQueue queue; + + public void publish() { + + long s = System.currentTimeMillis(); + int loopCount = 100; + int batchSize = 3000; + for (int i = 0; i < loopCount; i++) { + List messages = new ArrayList<>(batchSize); + for (int k = 0; k < batchSize; k++) { + String id = UUID.randomUUID().toString(); + Message message = new Message(id, getPayload()); + messages.add(message); + } + queue.push(messages); + } + long e = System.currentTimeMillis(); + long diff = e - s; + long throughput = 1000 * ((loopCount * batchSize) / diff); + System.out.println("Publish time: " + diff + ", throughput: " + throughput + " msg/sec"); + } + + public void consume() { + try { + Set ids = new HashSet<>(); + long s = System.currentTimeMillis(); + int loopCount = 100; + int batchSize = 3500; + int count = 0; + for (int i = 0; i < loopCount; i++) { + List popped = queue.pop(batchSize, 1, TimeUnit.MILLISECONDS); + queue.ack(popped); + Set poppedIds = popped.stream().map(Message::getId).collect(Collectors.toSet()); + if (popped.size() != poppedIds.size()) { + //We consumed dups + throw new RuntimeException("Count does not match. expected: " + popped.size() + ", but actual was : " + poppedIds.size() + ", i: " + i); + } + ids.addAll(poppedIds); + count += popped.size(); + } + long e = System.currentTimeMillis(); + long diff = e - s; + long throughput = 1000 * ((count) / diff); + if (count != ids.size()) { + //We consumed dups + throw new RuntimeException("There were duplicate messages consumed... expected messages to be consumed " + count + ", but actual was : " + ids.size()); + } + System.out.println("Consume time: " + diff + ", read throughput: " + throughput + " msg/sec, messages read: " + count); + } catch (Exception e) { + e.printStackTrace(); + } + } + + private String getPayload() { + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < 1; i++) { + sb.append(UUID.randomUUID().toString()); + sb.append(","); + } + return sb.toString(); + } + + public void run() throws Exception { + + ExecutorService es = Executors.newFixedThreadPool(2); + List> futures = new LinkedList<>(); + for (int i = 0; i < 2; i++) { + Future future = es.submit(() -> { + publish(); + consume(); + return null; + }); + futures.add(future); + } + for (Future future : futures) { + future.get(); + } + } +} diff --git a/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/pieline/DynoJedisTests.java b/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/pieline/DynoJedisTests.java new file mode 100644 index 0000000..0c4d3a6 --- /dev/null +++ b/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/pieline/DynoJedisTests.java @@ -0,0 +1,99 @@ +/** + * Copyright 2016 Netflix, Inc. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.netflix.dyno.queues.redis.pieline; + +import com.netflix.dyno.connectionpool.Host; +import com.netflix.dyno.connectionpool.HostSupplier; +import com.netflix.dyno.connectionpool.TokenMapSupplier; +import com.netflix.dyno.connectionpool.impl.ConnectionPoolConfigurationImpl; +import com.netflix.dyno.connectionpool.impl.lb.HostToken; +import com.netflix.dyno.jedis.DynoJedisClient; +import com.netflix.dyno.queues.DynoQueue; +import com.netflix.dyno.queues.redis.BaseQueueTests; +import com.netflix.dyno.queues.redis.v2.QueueBuilder; +import com.netflix.dyno.queues.redis.RedisQueue; +import redis.clients.jedis.Jedis; + +import java.util.*; + +public class DynoJedisTests extends BaseQueueTests { + + private static Jedis dynoClient; + + + private static RedisQueue rdq; + + private static String messageKeyPrefix; + + private static int maxHashBuckets = 32; + + @Override + public DynoQueue getQueue(String redisKeyPrefix, String queueName) { + + List hosts = new ArrayList<>(1); + hosts.add(new Host("localhost", "127.0.0.1", 6379, "us-east-1a", "us-east-1", Host.Status.Up)); + + + QueueBuilder qb = new QueueBuilder(); + + DynoJedisClient.Builder builder = new DynoJedisClient.Builder(); + HostSupplier hs = new HostSupplier() { + @Override + public List getHosts() { + return hosts; + } + }; + + ConnectionPoolConfigurationImpl cp = new ConnectionPoolConfigurationImpl("test").withTokenSupplier(new TokenMapSupplier() { + + HostToken token = new HostToken(1L, hosts.get(0)); + + @Override + public List getTokens(Set activeHosts) { + return Arrays.asList(token); + } + + @Override + public HostToken getTokenForHost(Host host, Set activeHosts) { + return token; + } + + + }).setLocalRack("us-east-1a").setLocalDataCenter("us-east-1"); + cp.setSocketTimeout(0); + cp.setConnectTimeout(0); + cp.setMaxConnsPerHost(10); + cp.withHashtag("{}"); + + DynoJedisClient client = builder.withApplicationName("test") + .withDynomiteClusterName("test") + .withCPConfig(cp) + .withHostSupplier(hs) + .build(); + + return qb + .setCurrentShard("a") + .setHostToShardMap((Host h) -> h.getRack().substring(h.getRack().length() - 1)) + .setQueueName(queueName) + .setRedisKeyPrefix(redisKeyPrefix) + .setUnackTime(1_000) + .useDynomite(client, client, hs) + .build(); + } + + + +} diff --git a/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/pieline/JedisTests.java b/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/pieline/JedisTests.java new file mode 100644 index 0000000..6e15225 --- /dev/null +++ b/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/pieline/JedisTests.java @@ -0,0 +1,74 @@ +/** + * Copyright 2016 Netflix, Inc. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.netflix.dyno.queues.redis.pieline; + +import com.netflix.dyno.connectionpool.Host; +import com.netflix.dyno.queues.DynoQueue; +import com.netflix.dyno.queues.redis.BaseQueueTests; +import com.netflix.dyno.queues.redis.v2.QueueBuilder; +import com.netflix.dyno.queues.redis.RedisQueue; +import redis.clients.jedis.Jedis; +import redis.clients.jedis.JedisPool; +import redis.clients.jedis.JedisPoolConfig; + +import java.util.*; + +/** + * + */ +public class JedisTests extends BaseQueueTests { + + private static Jedis dynoClient; + + + private static RedisQueue rdq; + + private static String messageKeyPrefix; + + private static int maxHashBuckets = 32; + + @Override + public DynoQueue getQueue(String redisKeyPrefix, String queueName) { + JedisPoolConfig config = new JedisPoolConfig(); + config.setTestOnBorrow(true); + config.setTestOnCreate(true); + config.setMaxTotal(10); + config.setMaxIdle(5); + config.setMaxWaitMillis(60_000); + JedisPool pool = new JedisPool(config, "localhost", 6379); + dynoClient = new Jedis("localhost", 6379, 0, 0); + dynoClient.flushAll(); + + List hosts = new LinkedList<>(); + hosts.add(new Host("localhost", 6379, "us-east-1a")); + + QueueBuilder qb = new QueueBuilder(); + DynoQueue queue = qb + .setCurrentShard("a") + .setHostToShardMap((Host h) -> h.getRack().substring(h.getRack().length()-1)) + .setQueueName(queueName) + .setRedisKeyPrefix(redisKeyPrefix) + .setUnackTime(1_000) + .useNonDynomiteRedis(config, hosts) + .build(); + + return queue; + + } + + + +} From 6d9ce47db30115798e3eb02a59b064309fd79299 Mon Sep 17 00:00:00 2001 From: Viren Baraiya Date: Fri, 16 Mar 2018 11:53:59 -0700 Subject: [PATCH 07/91] update dyno version --- dyno-queues-redis/build.gradle | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/dyno-queues-redis/build.gradle b/dyno-queues-redis/build.gradle index 21f8b4c..0171893 100644 --- a/dyno-queues-redis/build.gradle +++ b/dyno-queues-redis/build.gradle @@ -4,11 +4,11 @@ dependencies { compile "com.google.inject:guice:3.0" - compile 'com.netflix.dyno:dyno-core:1.6.2' - compile "com.netflix.dyno:dyno-jedis:1.6.2" + compile 'com.netflix.dyno:dyno-core:1.6.4-rc.1' + compile 'com.netflix.dyno:dyno-jedis:1.6.4-rc.1' - compile "com.netflix.archaius:archaius-core:0.7.5" - compile "com.netflix.servo:servo-core:0.12.17" + compile 'com.netflix.archaius:archaius-core:0.7.5' + compile 'com.netflix.servo:servo-core:0.12.17' compile 'com.netflix.eureka:eureka-client:1.8.1' compile 'com.fasterxml.jackson.core:jackson-databind:2.4.4' From e74db616c764b2ae55ab3dc6ebdff27e647e8e75 Mon Sep 17 00:00:00 2001 From: Viren Baraiya Date: Sat, 17 Mar 2018 11:54:02 -0700 Subject: [PATCH 08/91] Apply license header to the files. --- .../dyno/queues/redis/conn/DynoClientProxy.java | 15 +++++++++++++++ .../dyno/queues/redis/conn/DynoJedisPipe.java | 15 +++++++++++++++ .../dyno/queues/redis/conn/JedisProxy.java | 15 +++++++++++++++ .../com/netflix/dyno/queues/redis/conn/Pipe.java | 15 +++++++++++++++ .../dyno/queues/redis/conn/RedisConnection.java | 15 +++++++++++++++ .../netflix/dyno/queues/redis/conn/RedisPipe.java | 15 +++++++++++++++ .../dyno/queues/redis/v2/QueueBuilder.java | 15 +++++++++++++++ 7 files changed, 105 insertions(+) diff --git a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/conn/DynoClientProxy.java b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/conn/DynoClientProxy.java index 18be436..e207e35 100644 --- a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/conn/DynoClientProxy.java +++ b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/conn/DynoClientProxy.java @@ -1,3 +1,18 @@ +/** + * Copyright 2018 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ /** * */ diff --git a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/conn/DynoJedisPipe.java b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/conn/DynoJedisPipe.java index 5079581..47fdc98 100644 --- a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/conn/DynoJedisPipe.java +++ b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/conn/DynoJedisPipe.java @@ -1,3 +1,18 @@ +/** + * Copyright 2018 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ /** * */ diff --git a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/conn/JedisProxy.java b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/conn/JedisProxy.java index 0a11689..23937b6 100644 --- a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/conn/JedisProxy.java +++ b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/conn/JedisProxy.java @@ -1,3 +1,18 @@ +/** + * Copyright 2018 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ /** * */ diff --git a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/conn/Pipe.java b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/conn/Pipe.java index 2cd3e4f..6d085f4 100644 --- a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/conn/Pipe.java +++ b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/conn/Pipe.java @@ -1,3 +1,18 @@ +/** + * Copyright 2018 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package com.netflix.dyno.queues.redis.conn; import com.netflix.dyno.jedis.DynoJedisPipeline; diff --git a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/conn/RedisConnection.java b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/conn/RedisConnection.java index 0f56192..54dab2c 100644 --- a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/conn/RedisConnection.java +++ b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/conn/RedisConnection.java @@ -1,3 +1,18 @@ +/** + * Copyright 2018 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package com.netflix.dyno.queues.redis.conn; import java.util.Set; diff --git a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/conn/RedisPipe.java b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/conn/RedisPipe.java index 216a6f2..f76fbcc 100644 --- a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/conn/RedisPipe.java +++ b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/conn/RedisPipe.java @@ -1,3 +1,18 @@ +/** + * Copyright 2018 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ /** * */ diff --git a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/v2/QueueBuilder.java b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/v2/QueueBuilder.java index 7542ab3..3b5516b 100644 --- a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/v2/QueueBuilder.java +++ b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/v2/QueueBuilder.java @@ -1,3 +1,18 @@ +/** + * Copyright 2018 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ /** * */ From 309ef570890c6a82aca0e1037f9942895afb6264 Mon Sep 17 00:00:00 2001 From: Viren Baraiya Date: Sat, 17 Mar 2018 17:52:25 -0700 Subject: [PATCH 09/91] more tests and implement long polling for the v2 recipe --- .../dyno/queues/redis/QueueMonitor.java | 204 +++++------ .../dyno/queues/redis/v2/MultiRedisQueue.java | 345 +++++++++--------- .../dyno/queues/redis/v2/QueueBuilder.java | 1 - .../queues/redis/{ => v2}/RedisQueue.java | 71 ++-- .../redis/benchmark/QueueBenchmark.java | 1 - .../redis/{pieline => v2}/DynoJedisTests.java | 4 +- .../redis/{pieline => v2}/JedisTests.java | 4 +- .../dyno/queues/redis/v2/MultiQueueTests.java | 139 +++++++ .../RedisDynoQueueTest.java} | 5 +- 9 files changed, 461 insertions(+), 313 deletions(-) rename dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/{ => v2}/RedisQueue.java (94%) rename dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/{pieline => v2}/DynoJedisTests.java (97%) rename dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/{pieline => v2}/JedisTests.java (95%) create mode 100644 dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/v2/MultiQueueTests.java rename dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/{RedisDynoQueueTest2.java => v2/RedisDynoQueueTest.java} (98%) diff --git a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/QueueMonitor.java b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/QueueMonitor.java index ded34c5..085d9e1 100644 --- a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/QueueMonitor.java +++ b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/QueueMonitor.java @@ -1,12 +1,12 @@ /** * Copyright 2016 Netflix, Inc. - * + *

* Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -38,102 +38,102 @@ */ public class QueueMonitor implements Closeable { - BasicTimer peek; - - BasicTimer ack; - - BasicTimer size; - - BasicTimer processUnack; - - BasicTimer remove; - - BasicTimer get; - - StatsMonitor queueDepth; - - StatsMonitor batchSize; - - StatsMonitor pop; - - StatsMonitor push; - - BasicCounter misses; - - StatsMonitor prefetch; - - private String queueName; - - private String shardName; - - private ScheduledExecutorService executor; - - private static final String className = QueueMonitor.class.getSimpleName(); - - QueueMonitor(String queueName, String shardName){ - - String totalTagName = "total"; - executor = Executors.newScheduledThreadPool(1); - - this.queueName = queueName; - this.shardName = shardName; - - peek = new BasicTimer(create("peek"), TimeUnit.MILLISECONDS); - ack = new BasicTimer(create("ack"), TimeUnit.MILLISECONDS); - size = new BasicTimer(create("size"), TimeUnit.MILLISECONDS); - processUnack = new BasicTimer(create("processUnack"), TimeUnit.MILLISECONDS); - remove = new BasicTimer(create("remove"), TimeUnit.MILLISECONDS); - get = new BasicTimer(create("get"), TimeUnit.MILLISECONDS); - misses = new BasicCounter(create("queue_miss")); - - - StatsConfig statsConfig = new StatsConfig.Builder().withPublishCount(true).withPublishMax(true).withPublishMean(true).withPublishMin(true).withPublishTotal(true).build(); - - queueDepth = new StatsMonitor(create("queueDepth"), statsConfig, executor, totalTagName, true); - batchSize = new StatsMonitor(create("batchSize"), statsConfig, executor, totalTagName, true); - pop = new StatsMonitor(create("pop"), statsConfig, executor, totalTagName, true); - push = new StatsMonitor(create("push"), statsConfig, executor, totalTagName, true); - prefetch = new StatsMonitor(create("prefetch"), statsConfig, executor, totalTagName, true); - - MonitorRegistry registry = DefaultMonitorRegistry.getInstance(); - - registry.register(pop); - registry.register(push); - registry.register(peek); - registry.register(ack); - registry.register(size); - registry.register(processUnack); - registry.register(remove); - registry.register(get); - registry.register(queueDepth); - registry.register(misses); - registry.register(batchSize); - registry.register(prefetch); - } - - private MonitorConfig create(String name){ - return MonitorConfig.builder(name).withTag("class", className).withTag("shard", shardName).withTag("queueName", queueName).build(); - } - - Stopwatch start(StatsMonitor sm, int batchCount){ - int count = (batchCount == 0) ? 1 : batchCount; - Stopwatch sw = new BasicStopwatch(){ - - @Override - public void stop() { - super.stop(); - long duration = getDuration(TimeUnit.MILLISECONDS)/count; - sm.record(duration); - batchSize.record(count); - } - - }; - sw.start(); - return sw; - } - - @Override - public void close() throws IOException { - executor.shutdown(); - } + public BasicTimer peek; + + public BasicTimer ack; + + public BasicTimer size; + + public BasicTimer processUnack; + + public BasicTimer remove; + + public BasicTimer get; + + public StatsMonitor queueDepth; + + public StatsMonitor batchSize; + + public StatsMonitor pop; + + public StatsMonitor push; + + public BasicCounter misses; + + public StatsMonitor prefetch; + + private String queueName; + + private String shardName; + + private ScheduledExecutorService executor; + + private static final String className = QueueMonitor.class.getSimpleName(); + + public QueueMonitor(String queueName, String shardName) { + + String totalTagName = "total"; + executor = Executors.newScheduledThreadPool(1); + + this.queueName = queueName; + this.shardName = shardName; + + peek = new BasicTimer(create("peek"), TimeUnit.MILLISECONDS); + ack = new BasicTimer(create("ack"), TimeUnit.MILLISECONDS); + size = new BasicTimer(create("size"), TimeUnit.MILLISECONDS); + processUnack = new BasicTimer(create("processUnack"), TimeUnit.MILLISECONDS); + remove = new BasicTimer(create("remove"), TimeUnit.MILLISECONDS); + get = new BasicTimer(create("get"), TimeUnit.MILLISECONDS); + misses = new BasicCounter(create("queue_miss")); + + + StatsConfig statsConfig = new StatsConfig.Builder().withPublishCount(true).withPublishMax(true).withPublishMean(true).withPublishMin(true).withPublishTotal(true).build(); + + queueDepth = new StatsMonitor(create("queueDepth"), statsConfig, executor, totalTagName, true); + batchSize = new StatsMonitor(create("batchSize"), statsConfig, executor, totalTagName, true); + pop = new StatsMonitor(create("pop"), statsConfig, executor, totalTagName, true); + push = new StatsMonitor(create("push"), statsConfig, executor, totalTagName, true); + prefetch = new StatsMonitor(create("prefetch"), statsConfig, executor, totalTagName, true); + + MonitorRegistry registry = DefaultMonitorRegistry.getInstance(); + + registry.register(pop); + registry.register(push); + registry.register(peek); + registry.register(ack); + registry.register(size); + registry.register(processUnack); + registry.register(remove); + registry.register(get); + registry.register(queueDepth); + registry.register(misses); + registry.register(batchSize); + registry.register(prefetch); + } + + private MonitorConfig create(String name) { + return MonitorConfig.builder(name).withTag("class", className).withTag("shard", shardName).withTag("queueName", queueName).build(); + } + + public Stopwatch start(StatsMonitor sm, int batchCount) { + int count = (batchCount == 0) ? 1 : batchCount; + Stopwatch sw = new BasicStopwatch() { + + @Override + public void stop() { + super.stop(); + long duration = getDuration(TimeUnit.MILLISECONDS) / count; + sm.record(duration); + batchSize.record(count); + } + + }; + sw.start(); + return sw; + } + + @Override + public void close() throws IOException { + executor.shutdown(); + } } diff --git a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/v2/MultiRedisQueue.java b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/v2/MultiRedisQueue.java index e257bed..4507cd6 100644 --- a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/v2/MultiRedisQueue.java +++ b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/v2/MultiRedisQueue.java @@ -1,12 +1,12 @@ /** * Copyright 2017 Netflix, Inc. - * + *

* Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -27,7 +27,6 @@ import com.netflix.dyno.queues.DynoQueue; import com.netflix.dyno.queues.Message; -import com.netflix.dyno.queues.redis.RedisQueue; /** * @author Viren @@ -37,172 +36,172 @@ */ public class MultiRedisQueue implements DynoQueue { - private List shards; - - private String name; - - private Map queues = new HashMap<>(); - - private RedisQueue me; - - public MultiRedisQueue(String queueName, String shardName, Map queues) { - this.name = queueName; - this.queues = queues; - this.me = queues.get(shardName); - if(me == null) { - throw new IllegalArgumentException("List of shards supplied (" + queues.keySet() + ") does not contain current shard name: " + shardName); - } - this.shards = queues.keySet().stream().collect(Collectors.toList()); - } - - @Override - public String getName() { - return name; - } - - @Override - public int getUnackTime() { - return me.getUnackTime(); - } - - @Override - public List push(List messages) { - int size = queues.size(); - int partitionSize = messages.size()/size; - List ids = new LinkedList<>(); - - for(int i = 0; i < size-1; i++) { - RedisQueue queue = queues.get(getNextShard()); - int start = i * partitionSize; - int end = start + partitionSize; - ids.addAll(queue.push(messages.subList(start, end))); - } - RedisQueue queue = queues.get(getNextShard()); - int start = (size-1) * partitionSize; - - ids.addAll(queue.push(messages.subList(start, messages.size()))); - return ids; - } - - @Override - public List pop(int messageCount, int wait, TimeUnit unit) { - return me.pop(messageCount, wait, unit); - } - - @Override - public List peek(int messageCount) { - return me.peek(messageCount); - } - - @Override - public boolean ack(String messageId) { - for(DynoQueue q : queues.values()) { - if(q.ack(messageId)) { - return true; - } - } - return false; - } - - @Override - public void ack(List messages) { - Map> byShard = messages.stream().collect(Collectors.groupingBy(Message::getShard)); - for(Entry> e: byShard.entrySet()) { - queues.get(e.getKey()).ack(e.getValue()); - } - } - - @Override - public boolean setUnackTimeout(String messageId, long timeout) { - for(DynoQueue q : queues.values()) { - if(q.setUnackTimeout(messageId, timeout)) { - return true; - } - } - return false; - } - - @Override - public boolean setTimeout(String messageId, long timeout) { - for(DynoQueue q : queues.values()) { - if(q.setTimeout(messageId, timeout)) { - return true; - } - } - return false; - } - - @Override - public boolean remove(String messageId) { - for(DynoQueue q : queues.values()) { - if(q.remove(messageId)) { - return true; - } - } - return false; - } - - @Override - public Message get(String messageId) { - for(DynoQueue q : queues.values()) { - Message msg = q.get(messageId); - if(msg != null) { - return msg; - } - } - return null; - } - - @Override - public long size() { - long size = 0; - for(DynoQueue q : queues.values()) { - size += q.size(); - } - return size; - } - - @Override - public Map> shardSizes() { - Map> sizes = new HashMap<>(); - for(Entry e : queues.entrySet()) { - sizes.put(e.getKey(), e.getValue().shardSizes().get(e.getKey())); - } - return sizes; - } - - @Override - public void clear() { - for(DynoQueue q : queues.values()) { - q.clear(); - } - - } - - @Override - public void close() throws IOException { - for(RedisQueue queue : queues.values()) { - queue.close(); - } - } - - @Override - public void processUnacks() { - for(RedisQueue queue : queues.values()) { - queue.processUnacks(); - } - } - - private AtomicInteger nextShardIndex = new AtomicInteger(0); - - private String getNextShard() { - int indx = nextShardIndex.incrementAndGet(); - if (indx >= shards.size()) { - nextShardIndex.set(0); - indx = 0; - } - String s = shards.get(indx); - return s; - } - + private List shards; + + private String name; + + private Map queues = new HashMap<>(); + + private RedisQueue me; + + public MultiRedisQueue(String queueName, String shardName, Map queues) { + this.name = queueName; + this.queues = queues; + this.me = queues.get(shardName); + if (me == null) { + throw new IllegalArgumentException("List of shards supplied (" + queues.keySet() + ") does not contain current shard name: " + shardName); + } + this.shards = queues.keySet().stream().collect(Collectors.toList()); + } + + @Override + public String getName() { + return name; + } + + @Override + public int getUnackTime() { + return me.getUnackTime(); + } + + @Override + public List push(List messages) { + int size = queues.size(); + int partitionSize = messages.size() / size; + List ids = new LinkedList<>(); + + for (int i = 0; i < size - 1; i++) { + RedisQueue queue = queues.get(getNextShard()); + int start = i * partitionSize; + int end = start + partitionSize; + ids.addAll(queue.push(messages.subList(start, end))); + } + RedisQueue queue = queues.get(getNextShard()); + int start = (size - 1) * partitionSize; + + ids.addAll(queue.push(messages.subList(start, messages.size()))); + return ids; + } + + @Override + public List pop(int messageCount, int wait, TimeUnit unit) { + return me.pop(messageCount, wait, unit); + } + + @Override + public List peek(int messageCount) { + return me.peek(messageCount); + } + + @Override + public boolean ack(String messageId) { + for (DynoQueue q : queues.values()) { + if (q.ack(messageId)) { + return true; + } + } + return false; + } + + @Override + public void ack(List messages) { + Map> byShard = messages.stream().collect(Collectors.groupingBy(Message::getShard)); + for (Entry> e : byShard.entrySet()) { + queues.get(e.getKey()).ack(e.getValue()); + } + } + + @Override + public boolean setUnackTimeout(String messageId, long timeout) { + for (DynoQueue q : queues.values()) { + if (q.setUnackTimeout(messageId, timeout)) { + return true; + } + } + return false; + } + + @Override + public boolean setTimeout(String messageId, long timeout) { + for (DynoQueue q : queues.values()) { + if (q.setTimeout(messageId, timeout)) { + return true; + } + } + return false; + } + + @Override + public boolean remove(String messageId) { + for (DynoQueue q : queues.values()) { + if (q.remove(messageId)) { + return true; + } + } + return false; + } + + @Override + public Message get(String messageId) { + for (DynoQueue q : queues.values()) { + Message msg = q.get(messageId); + if (msg != null) { + return msg; + } + } + return null; + } + + @Override + public long size() { + long size = 0; + for (DynoQueue q : queues.values()) { + size += q.size(); + } + return size; + } + + @Override + public Map> shardSizes() { + Map> sizes = new HashMap<>(); + for (Entry e : queues.entrySet()) { + sizes.put(e.getKey(), e.getValue().shardSizes().get(e.getKey())); + } + return sizes; + } + + @Override + public void clear() { + for (DynoQueue q : queues.values()) { + q.clear(); + } + + } + + @Override + public void close() throws IOException { + for (RedisQueue queue : queues.values()) { + queue.close(); + } + } + + @Override + public void processUnacks() { + for (RedisQueue queue : queues.values()) { + queue.processUnacks(); + } + } + + private AtomicInteger nextShardIndex = new AtomicInteger(0); + + private String getNextShard() { + int indx = nextShardIndex.incrementAndGet(); + if (indx >= shards.size()) { + nextShardIndex.set(0); + indx = 0; + } + String s = shards.get(indx); + return s; + } + } diff --git a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/v2/QueueBuilder.java b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/v2/QueueBuilder.java index 3b5516b..3d13495 100644 --- a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/v2/QueueBuilder.java +++ b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/v2/QueueBuilder.java @@ -34,7 +34,6 @@ import com.netflix.dyno.connectionpool.HostSupplier; import com.netflix.dyno.jedis.DynoJedisClient; import com.netflix.dyno.queues.DynoQueue; -import com.netflix.dyno.queues.redis.RedisQueue; import com.netflix.dyno.queues.redis.conn.DynoClientProxy; import com.netflix.dyno.queues.redis.conn.JedisProxy; import com.netflix.dyno.queues.redis.conn.RedisConnection; diff --git a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/RedisQueue.java b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/v2/RedisQueue.java similarity index 94% rename from dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/RedisQueue.java rename to dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/v2/RedisQueue.java index 059e586..d2addc5 100644 --- a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/RedisQueue.java +++ b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/v2/RedisQueue.java @@ -13,24 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.netflix.dyno.queues.redis; - -import java.io.IOException; -import java.time.Clock; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.TimeUnit; -import java.util.stream.Collectors; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +package com.netflix.dyno.queues.redis.v2; import com.fasterxml.jackson.annotation.JsonInclude.Include; import com.fasterxml.jackson.databind.DeserializationFeature; @@ -40,19 +23,27 @@ import com.netflix.dyno.connectionpool.impl.hash.Murmur3HashPartitioner; import com.netflix.dyno.queues.DynoQueue; import com.netflix.dyno.queues.Message; +import com.netflix.dyno.queues.redis.QueueMonitor; import com.netflix.dyno.queues.redis.conn.Pipe; import com.netflix.dyno.queues.redis.conn.RedisConnection; import com.netflix.servo.monitor.Stopwatch; - +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import redis.clients.jedis.Response; import redis.clients.jedis.Tuple; import redis.clients.jedis.params.sortedset.ZAddParams; +import java.io.IOException; +import java.time.Clock; +import java.util.*; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; + /** - * * @author Viren * Queue implementation that uses redis pipelines that improves the throughput under heavy load. - * */ public class RedisQueue implements DynoQueue { @@ -88,6 +79,8 @@ public class RedisQueue implements DynoQueue { private int maxHashBuckets = 32; + private int longPollWaitIntervalInMillis = 10; + public RedisQueue(String redisKeyPrefix, String queueName, String shardName, int unackScheduleInMS, int unackTime, RedisConnection pool) { this(Clock.systemDefaultZone(), redisKeyPrefix, queueName, shardName, unackScheduleInMS, unackTime, pool); } @@ -125,7 +118,6 @@ public RedisQueue(Clock clock, String redisKeyPrefix, String queue, String shard } /** - * * @param nonQuorumPool When using a cluster like Dynomite, which relies on the quorum reads, supply a separate non-quorum read connection for ops like size etc. */ public void setNonQuorumPool(RedisConnection nonQuorumPool) { @@ -212,11 +204,6 @@ public List peek(final int messageCount) { } } - - // - //Note: This implementation does NOT support long polling. The method itself is synchronized, so implementing long poll could potentially block other threads. - //When required, the long polling should be implemented on the caller side (ie the broker implementation using the recipe) - // @Override public synchronized List pop(int messageCount, int wait, TimeUnit unit) { @@ -225,12 +212,36 @@ public synchronized List pop(int messageCount, int wait, TimeUnit unit) } Stopwatch sw = monitor.start(monitor.pop, messageCount); + List messages = new LinkedList<>(); + int remaining = messageCount; + long time = clock.millis() + unit.toMillis(wait); try { - List peeked = peekIds(0, messageCount).stream().collect(Collectors.toList()); - List popped = _pop(peeked); - return popped; + do { + + List peeked = peekIds(0, remaining).stream().collect(Collectors.toList()); + List popped = _pop(peeked); + int poppedCount = popped.size(); + if (poppedCount == messageCount) { + messages = popped; + break; + } + messages.addAll(popped); + remaining -= poppedCount; + if(clock.millis() > time) { + break; + } + + try { + Thread.sleep(longPollWaitIntervalInMillis); + } catch (InterruptedException ie) { + logger.error(ie.getMessage(), ie); + } + + } while (remaining > 0); + + return messages; } catch (Exception e) { throw new RuntimeException(e); diff --git a/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/benchmark/QueueBenchmark.java b/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/benchmark/QueueBenchmark.java index b3367b7..db65611 100644 --- a/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/benchmark/QueueBenchmark.java +++ b/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/benchmark/QueueBenchmark.java @@ -2,7 +2,6 @@ import com.netflix.dyno.queues.DynoQueue; import com.netflix.dyno.queues.Message; -import com.netflix.dyno.queues.redis.RedisQueue; import java.util.*; import java.util.concurrent.ExecutorService; diff --git a/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/pieline/DynoJedisTests.java b/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/v2/DynoJedisTests.java similarity index 97% rename from dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/pieline/DynoJedisTests.java rename to dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/v2/DynoJedisTests.java index 0c4d3a6..138c6a9 100644 --- a/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/pieline/DynoJedisTests.java +++ b/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/v2/DynoJedisTests.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.netflix.dyno.queues.redis.pieline; +package com.netflix.dyno.queues.redis.v2; import com.netflix.dyno.connectionpool.Host; import com.netflix.dyno.connectionpool.HostSupplier; @@ -24,7 +24,7 @@ import com.netflix.dyno.queues.DynoQueue; import com.netflix.dyno.queues.redis.BaseQueueTests; import com.netflix.dyno.queues.redis.v2.QueueBuilder; -import com.netflix.dyno.queues.redis.RedisQueue; +import com.netflix.dyno.queues.redis.v2.RedisQueue; import redis.clients.jedis.Jedis; import java.util.*; diff --git a/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/pieline/JedisTests.java b/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/v2/JedisTests.java similarity index 95% rename from dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/pieline/JedisTests.java rename to dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/v2/JedisTests.java index 6e15225..441447a 100644 --- a/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/pieline/JedisTests.java +++ b/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/v2/JedisTests.java @@ -13,13 +13,13 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.netflix.dyno.queues.redis.pieline; +package com.netflix.dyno.queues.redis.v2; import com.netflix.dyno.connectionpool.Host; import com.netflix.dyno.queues.DynoQueue; import com.netflix.dyno.queues.redis.BaseQueueTests; import com.netflix.dyno.queues.redis.v2.QueueBuilder; -import com.netflix.dyno.queues.redis.RedisQueue; +import com.netflix.dyno.queues.redis.v2.RedisQueue; import redis.clients.jedis.Jedis; import redis.clients.jedis.JedisPool; import redis.clients.jedis.JedisPoolConfig; diff --git a/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/v2/MultiQueueTests.java b/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/v2/MultiQueueTests.java new file mode 100644 index 0000000..44bc27c --- /dev/null +++ b/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/v2/MultiQueueTests.java @@ -0,0 +1,139 @@ +/** + * Copyright 2016 Netflix, Inc. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.netflix.dyno.queues.redis.v2; + +import com.netflix.dyno.connectionpool.Host; +import com.netflix.dyno.queues.DynoQueue; +import com.netflix.dyno.queues.Message; +import org.junit.Test; +import redis.clients.jedis.Jedis; +import redis.clients.jedis.JedisPool; +import redis.clients.jedis.JedisPoolConfig; + +import java.util.Arrays; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; + +/** + * + */ +public class MultiQueueTests { + + private static Jedis dynoClient; + + + private static RedisQueue rdq; + + private static String messageKeyPrefix; + + private static int maxHashBuckets = 32; + + public DynoQueue getQueue(String redisKeyPrefix, String queueName) { + JedisPoolConfig config = new JedisPoolConfig(); + config.setTestOnBorrow(true); + config.setTestOnCreate(true); + config.setMaxTotal(10); + config.setMaxIdle(5); + config.setMaxWaitMillis(60_000); + JedisPool pool = new JedisPool(config, "localhost", 6379); + dynoClient = new Jedis("localhost", 6379, 0, 0); + dynoClient.flushAll(); + + List hosts = new LinkedList<>(); + hosts.add(new Host("localhost", 6379, "us-east-1a")); + hosts.add(new Host("localhost", 6379, "us-east-1b")); + + QueueBuilder qb = new QueueBuilder(); + DynoQueue queue = qb + .setCurrentShard("a") + .setHostToShardMap((Host h) -> h.getRack().substring(h.getRack().length() - 1)) + .setQueueName(queueName) + .setRedisKeyPrefix(redisKeyPrefix) + .setUnackTime(50_000) + .useNonDynomiteRedis(config, hosts) + .build(); + + queue.clear(); //clear the queue + + return queue; + + } + + @Test + public void testAll() { + DynoQueue queue = getQueue("test", "multi_queue"); + assertEquals(MultiRedisQueue.class, queue.getClass()); + + long start = System.currentTimeMillis(); + List popped = queue.pop(1, 1, TimeUnit.SECONDS); + assertTrue(popped.isEmpty()); //we have not pushed anything!!!! + long elapsedTime = System.currentTimeMillis() - start; + System.out.println("elapsed Time " + elapsedTime); + assertTrue(elapsedTime > 1000); + + List messages = new LinkedList<>(); + for (int i = 0; i < 10; i++) { + Message msg = new Message(); + msg.setId("" + i); + msg.setPayload("" + i); + messages.add(msg); + } + queue.push(messages); + + assertEquals(10, queue.size()); + Map> shards = queue.shardSizes(); + assertEquals(2, shards.keySet().size()); //a and b + + Map shardA = shards.get("a"); + Map shardB = shards.get("b"); + + assertNotNull(shardA); + assertNotNull(shardB); + + Long sizeA = shardA.get("size"); + Long sizeB = shardB.get("size"); + + assertNotNull(sizeA); + assertNotNull(sizeB); + + assertEquals(5L, sizeA.longValue()); + assertEquals(5L, sizeB.longValue()); + + start = System.currentTimeMillis(); + popped = queue.pop(2, 1, TimeUnit.SECONDS); + elapsedTime = System.currentTimeMillis() - start; + assertEquals(2, popped.size()); + System.out.println("elapsed Time " + elapsedTime); + assertTrue(elapsedTime < 1000); + + + start = System.currentTimeMillis(); + popped = queue.pop(5, 5, TimeUnit.SECONDS); + elapsedTime = System.currentTimeMillis() - start; + assertEquals(3, popped.size()); //3 remaining in the current shard + System.out.println("elapsed Time " + elapsedTime); + assertTrue(elapsedTime > 5000); //we would have waited for at least 5 second for the last 2 elements! + + } + + +} diff --git a/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/RedisDynoQueueTest2.java b/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/v2/RedisDynoQueueTest.java similarity index 98% rename from dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/RedisDynoQueueTest2.java rename to dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/v2/RedisDynoQueueTest.java index fa90292..eb51424 100644 --- a/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/RedisDynoQueueTest2.java +++ b/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/v2/RedisDynoQueueTest.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.netflix.dyno.queues.redis; +package com.netflix.dyno.queues.redis.v2; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; @@ -37,6 +37,7 @@ import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Collectors; +import com.netflix.dyno.queues.redis.v2.RedisQueue; import org.junit.Before; import org.junit.BeforeClass; import org.junit.Test; @@ -49,7 +50,7 @@ import redis.clients.jedis.JedisPool; import redis.clients.jedis.JedisPoolConfig; -public class RedisDynoQueueTest2 { +public class RedisDynoQueueTest { private static Jedis dynoClient; From a1a8e39caa9cb2249a2dc5c39580d9c38c55aabc Mon Sep 17 00:00:00 2001 From: Viren Baraiya Date: Sat, 17 Mar 2018 18:03:56 -0700 Subject: [PATCH 10/91] refactored tests so the queue names are distinct across tests --- .../com/netflix/dyno/queues/redis/BaseQueueTests.java | 9 ++++++--- .../com/netflix/dyno/queues/redis/v2/DynoJedisTests.java | 4 ++++ .../com/netflix/dyno/queues/redis/v2/JedisTests.java | 6 ++++++ 3 files changed, 16 insertions(+), 3 deletions(-) diff --git a/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/BaseQueueTests.java b/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/BaseQueueTests.java index 35ad23d..d440605 100644 --- a/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/BaseQueueTests.java +++ b/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/BaseQueueTests.java @@ -36,7 +36,7 @@ public abstract class BaseQueueTests { - protected static final String queueName = "test_queue"; + private String queueName; protected static final String redisKeyPrefix = "testdynoqueues"; @@ -47,10 +47,13 @@ public abstract class BaseQueueTests { public abstract DynoQueue getQueue(String redisKeyPrefix, String queueName); - public BaseQueueTests() { + public BaseQueueTests(String queueName) { + this.queueName = queueName; + this.messageKeyPrefix = redisKeyPrefix + ".MESSAGE."; this.rdq = getQueue(redisKeyPrefix, queueName); - this.messageKeyPrefix = redisKeyPrefix + ".MESSAGE."; + this.rdq.clear(); + } diff --git a/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/v2/DynoJedisTests.java b/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/v2/DynoJedisTests.java index 138c6a9..b4dc01a 100644 --- a/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/v2/DynoJedisTests.java +++ b/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/v2/DynoJedisTests.java @@ -40,6 +40,10 @@ public class DynoJedisTests extends BaseQueueTests { private static int maxHashBuckets = 32; + public DynoJedisTests() { + super("dyno_queue_tests"); + } + @Override public DynoQueue getQueue(String redisKeyPrefix, String queueName) { diff --git a/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/v2/JedisTests.java b/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/v2/JedisTests.java index 441447a..ca541f9 100644 --- a/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/v2/JedisTests.java +++ b/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/v2/JedisTests.java @@ -40,6 +40,10 @@ public class JedisTests extends BaseQueueTests { private static int maxHashBuckets = 32; + public JedisTests() { + super("jedis_queue_tests"); + } + @Override public DynoQueue getQueue(String redisKeyPrefix, String queueName) { JedisPoolConfig config = new JedisPoolConfig(); @@ -65,6 +69,8 @@ public DynoQueue getQueue(String redisKeyPrefix, String queueName) { .useNonDynomiteRedis(config, hosts) .build(); + queue.clear(); + return queue; } From 3ac3dbe196dc5535bf59e4ad0c241aea0d1701c2 Mon Sep 17 00:00:00 2001 From: Viren Baraiya Date: Sat, 17 Mar 2018 18:28:25 -0700 Subject: [PATCH 11/91] javadocs --- .../queues/redis/conn/DynoClientProxy.java | 1 + .../dyno/queues/redis/conn/DynoJedisPipe.java | 1 + .../dyno/queues/redis/conn/JedisProxy.java | 167 +++++++++--------- .../netflix/dyno/queues/redis/conn/Pipe.java | 29 +-- .../dyno/queues/redis/conn/RedisPipe.java | 1 + 5 files changed, 102 insertions(+), 97 deletions(-) diff --git a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/conn/DynoClientProxy.java b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/conn/DynoClientProxy.java index e207e35..a7200a7 100644 --- a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/conn/DynoClientProxy.java +++ b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/conn/DynoClientProxy.java @@ -27,6 +27,7 @@ /** * @author Viren * + * Dynomite connection */ public class DynoClientProxy implements RedisConnection { diff --git a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/conn/DynoJedisPipe.java b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/conn/DynoJedisPipe.java index 47fdc98..4d5db74 100644 --- a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/conn/DynoJedisPipe.java +++ b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/conn/DynoJedisPipe.java @@ -25,6 +25,7 @@ /** * @author Viren + * Pipeline abstraction for Dynomite Pipeline. */ public class DynoJedisPipe implements Pipe { diff --git a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/conn/JedisProxy.java b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/conn/JedisProxy.java index 23937b6..e686ff9 100644 --- a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/conn/JedisProxy.java +++ b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/conn/JedisProxy.java @@ -1,12 +1,12 @@ /** * Copyright 2018 Netflix, Inc. - * + *

* Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -14,7 +14,7 @@ * limitations under the License. */ /** - * + * */ package com.netflix.dyno.queues.redis.conn; @@ -27,86 +27,87 @@ /** * @author Viren * + * This class provides the abstraction of a Jedis Connection Pool. Used when using Redis directly without Dynomite. */ public class JedisProxy implements RedisConnection { - private JedisPool pool; - - private Jedis jedis; - - public JedisProxy(JedisPool pool) { - this.pool = pool; - } - - public JedisProxy(Jedis jedis) { - this.jedis = jedis; - } - - @Override - public RedisConnection getResource() { - Jedis jedis = pool.getResource(); - return new JedisProxy(jedis); - } - - @Override - public void close() { - jedis.close(); - } - - @Override - public Pipe pipelined() { - return new RedisPipe(jedis.pipelined()); - } - - @Override - public String hget(String key, String member) { - return jedis.hget(key, member); - } - - @Override - public Long zrem(String key, String member) { - return jedis.zrem(key, member); - } - - @Override - public Long hdel(String key, String member) { - return jedis.hdel(key, member); - - } - - @Override - public Double zscore(String key, String member) { - return jedis.zscore(key, member); - } - - @Override - public void zadd(String key, double unackScore, String member) { - jedis.zadd(key, unackScore, member); - } - - @Override - public void hset(String key, String member, String json) { - jedis.hset(key, member, json); - } - - @Override - public long zcard(String key) { - return jedis.zcard(key); - } - - @Override - public void del(String key) { - jedis.del(key); - } - - @Override - public Set zrangeByScore(String key, int min, double max, int offset, int count) { - return jedis.zrangeByScore(key, min, max, offset, count); - } - - @Override - public Set zrangeByScoreWithScores(String key, int min, double max, int offset, int count) { - return jedis.zrangeByScoreWithScores(key, min, max, offset, count); - } + private JedisPool pool; + + private Jedis jedis; + + public JedisProxy(JedisPool pool) { + this.pool = pool; + } + + public JedisProxy(Jedis jedis) { + this.jedis = jedis; + } + + @Override + public RedisConnection getResource() { + Jedis jedis = pool.getResource(); + return new JedisProxy(jedis); + } + + @Override + public void close() { + jedis.close(); + } + + @Override + public Pipe pipelined() { + return new RedisPipe(jedis.pipelined()); + } + + @Override + public String hget(String key, String member) { + return jedis.hget(key, member); + } + + @Override + public Long zrem(String key, String member) { + return jedis.zrem(key, member); + } + + @Override + public Long hdel(String key, String member) { + return jedis.hdel(key, member); + + } + + @Override + public Double zscore(String key, String member) { + return jedis.zscore(key, member); + } + + @Override + public void zadd(String key, double unackScore, String member) { + jedis.zadd(key, unackScore, member); + } + + @Override + public void hset(String key, String member, String json) { + jedis.hset(key, member, json); + } + + @Override + public long zcard(String key) { + return jedis.zcard(key); + } + + @Override + public void del(String key) { + jedis.del(key); + } + + @Override + public Set zrangeByScore(String key, int min, double max, int offset, int count) { + return jedis.zrangeByScore(key, min, max, offset, count); + } + + @Override + public Set zrangeByScoreWithScores(String key, int min, double max, int offset, int count) { + return jedis.zrangeByScoreWithScores(key, min, max, offset, count); + } } diff --git a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/conn/Pipe.java b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/conn/Pipe.java index 6d085f4..65b121a 100644 --- a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/conn/Pipe.java +++ b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/conn/Pipe.java @@ -30,49 +30,50 @@ *

* @see DynoJedisPipeline * @see Pipeline + * The commands here reflects the RedisCommand structure. * */ public interface Pipe { /** * - * @param key - * @param field - * @param value + * @param key The Key + * @param field Field + * @param value Value of the Field */ public void hset(String key, String field, String value); /** * - * @param key - * @param score - * @param member + * @param key The Key + * @param score Score for the member + * @param member Member to be added within the key * @return */ public Response zadd(String key, double score, String member); /** * - * @param key - * @param score - * @param member - * @param zParams + * @param key The Key + * @param score Score for the member + * @param member Member to be added within the key + * @param zParams Parameters * @return */ public Response zadd(String key, double score, String member, ZAddParams zParams); /** * - * @param key - * @param member + * @param key The Key + * @param member Member * @return */ public Response zrem(String key, String member); /** * - * @param key - * @param member + * @param key The Key + * @param member Member * @return */ public Response hget(String key, String member); diff --git a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/conn/RedisPipe.java b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/conn/RedisPipe.java index f76fbcc..6c24aa8 100644 --- a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/conn/RedisPipe.java +++ b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/conn/RedisPipe.java @@ -25,6 +25,7 @@ /** * @author Viren * + * Pipeline abstraction for direct redis connection - when not using Dynomite. */ public class RedisPipe implements Pipe { From efb295ae089c671c098b716cf0b1ed83595b38a4 Mon Sep 17 00:00:00 2001 From: Viren Baraiya Date: Tue, 20 Mar 2018 08:23:47 -0700 Subject: [PATCH 12/91] use hashtags to separate messages per shard --- .../java/com/netflix/dyno/queues/Message.java | 282 +++++++++--------- .../dyno/queues/redis/v2/RedisQueue.java | 17 +- .../queues/redis/v2/RedisDynoQueueTest.java | 6 +- 3 files changed, 153 insertions(+), 152 deletions(-) diff --git a/dyno-queues-core/src/main/java/com/netflix/dyno/queues/Message.java b/dyno-queues-core/src/main/java/com/netflix/dyno/queues/Message.java index 024e447..c768d09 100644 --- a/dyno-queues-core/src/main/java/com/netflix/dyno/queues/Message.java +++ b/dyno-queues-core/src/main/java/com/netflix/dyno/queues/Message.java @@ -1,12 +1,12 @@ /** * Copyright 2016 Netflix, Inc. - * + *

* Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -14,7 +14,7 @@ * limitations under the License. */ /** - * + * */ package com.netflix.dyno.queues; @@ -26,140 +26,140 @@ */ public class Message { - private String id; - - private String payload; - - private long timeout; - - private int priority; - - private String shard; - - public Message() { - - } - - public Message(String id, String payload) { - this.id = id; - this.payload = payload; - } - - /** - * @return the id - */ - public String getId() { - return id; - } - - /** - * @param id - * the id to set - */ - public void setId(String id) { - this.id = id; - } - - /** - * @return the payload - */ - public String getPayload() { - return payload; - } - - /** - * @param payload the payload to set - * - */ - public void setPayload(String payload) { - this.payload = payload; - } - - /** - * - * @param timeout Timeout in milliseconds - The message is only given to the consumer after the specified milliseconds have elapsed. - */ - public void setTimeout(long timeout) { - this.timeout = timeout; - } - - /** - * Helper method for the {@link #setTimeout(long)} - * @param time timeout time - * @param unit unit for the time - * @see #setTimeout(long) - */ - public void setTimeout(long time, TimeUnit unit) { - this.timeout = TimeUnit.MILLISECONDS.convert(time, unit); - } - - /** - * - * @return Returns the timeout for the message - */ - public long getTimeout() { - return timeout; - } - - /** - * Sets the message priority. Higher priority message is retrieved ahead of lower priority ones - * @param priority priority for the message. - */ - public void setPriority(int priority) { - if(priority < 0 || priority > 99){ - throw new IllegalArgumentException("prioirty MUST be between 0 and 99 (inclusive)"); - } - this.priority = priority; - } - - public int getPriority() { - return priority; - } - - /** - * @return the shard - */ - public String getShard() { - return shard; - } - - /** - * @param shard the shard to set - * - */ - public void setShard(String shard) { - this.shard = shard; - } - - @Override - public int hashCode() { - final int prime = 31; - int result = 1; - result = prime * result + ((id == null) ? 0 : id.hashCode()); - return result; - } - - @Override - public boolean equals(Object obj) { - if (this == obj) - return true; - if (obj == null) - return false; - if (getClass() != obj.getClass()) - return false; - Message other = (Message) obj; - if (id == null) { - if (other.id != null) - return false; - } else if (!id.equals(other.id)) - return false; - return true; - } - - @Override - public String toString() { - return "Message [id=" + id + ", payload=" + payload + ", timeout=" + timeout + ", priority=" + priority + "]"; - } - - + private String id; + + private String payload; + + private long timeout; + + private int priority; + + private String shard; + + public Message() { + + } + + public Message(String id, String payload) { + this.id = id; + this.payload = payload; + } + + /** + * @return the id + */ + public String getId() { + return id; + } + + /** + * @param id + * the id to set + */ + public void setId(String id) { + this.id = id; + } + + /** + * @return the payload + */ + public String getPayload() { + return payload; + } + + /** + * @param payload the payload to set + * + */ + public void setPayload(String payload) { + this.payload = payload; + } + + /** + * + * @param timeout Timeout in milliseconds - The message is only given to the consumer after the specified milliseconds have elapsed. + */ + public void setTimeout(long timeout) { + this.timeout = timeout; + } + + /** + * Helper method for the {@link #setTimeout(long)} + * @param time timeout time + * @param unit unit for the time + * @see #setTimeout(long) + */ + public void setTimeout(long time, TimeUnit unit) { + this.timeout = TimeUnit.MILLISECONDS.convert(time, unit); + } + + /** + * + * @return Returns the timeout for the message + */ + public long getTimeout() { + return timeout; + } + + /** + * Sets the message priority. Higher priority message is retrieved ahead of lower priority ones + * @param priority priority for the message. + */ + public void setPriority(int priority) { + if (priority < 0 || priority > 99) { + throw new IllegalArgumentException("prioirty MUST be between 0 and 99 (inclusive)"); + } + this.priority = priority; + } + + public int getPriority() { + return priority; + } + + /** + * @return the shard + */ + public String getShard() { + return shard; + } + + /** + * @param shard the shard to set + * + */ + public void setShard(String shard) { + this.shard = shard; + } + + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + ((id == null) ? 0 : id.hashCode()); + return result; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) + return true; + if (obj == null) + return false; + if (getClass() != obj.getClass()) + return false; + Message other = (Message) obj; + if (id == null) { + if (other.id != null) + return false; + } else if (!id.equals(other.id)) + return false; + return true; + } + + @Override + public String toString() { + return "Message [id=" + id + ", payload=" + payload + ", timeout=" + timeout + ", priority=" + priority + "]"; + } + + } diff --git a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/v2/RedisQueue.java b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/v2/RedisQueue.java index d2addc5..0749a34 100644 --- a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/v2/RedisQueue.java +++ b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/v2/RedisQueue.java @@ -88,11 +88,12 @@ public RedisQueue(String redisKeyPrefix, String queueName, String shardName, int public RedisQueue(Clock clock, String redisKeyPrefix, String queue, String shardName, int unackScheduleInMS, int unackTime, RedisConnection pool) { this.clock = clock; this.queueName = queue; - String qName = "{" + queue + "}"; + String qName = "{" + queue + "." + shardName + "}"; this.shardName = shardName; - this.messageStoreKeyPrefix = redisKeyPrefix + ".MESSAGE."; - this.myQueueShard = redisKeyPrefix + ".QUEUE." + qName + "." + shardName; - this.unackShardKeyPrefix = redisKeyPrefix + ".UNACK." + qName + "." + shardName + "."; + + this.messageStoreKeyPrefix = redisKeyPrefix + ".MSG." + qName; + this.myQueueShard = redisKeyPrefix + ".QUEUE." + qName; + this.unackShardKeyPrefix = redisKeyPrefix + ".UNACK." + qName + "."; this.unackTime = unackTime; this.connPool = pool; this.nonQuorumPool = pool; @@ -166,7 +167,7 @@ public List push(final List messages) { private String messageStoreKey(String msgId) { Long hash = partitioner.hash(msgId); long bucket = hash % maxHashBuckets; - return messageStoreKeyPrefix + bucket + ".{" + queueName + "}"; + return messageStoreKeyPrefix + "." + bucket; } private String unackShardKey(String messageId) { @@ -539,11 +540,11 @@ public void clear() { jedis.del(myQueueShard); - for (int i = 0; i < maxHashBuckets; i++) { - String unackShardKey = unackShardKeyPrefix + i; + for (int bucket = 0; bucket < maxHashBuckets; bucket++) { + String unackShardKey = unackShardKeyPrefix + bucket; jedis.del(unackShardKey); - String messageStoreKey = messageStoreKeyPrefix + i + "." + queueName; + String messageStoreKey = messageStoreKeyPrefix + "." + bucket; jedis.del(messageStoreKey); } diff --git a/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/v2/RedisDynoQueueTest.java b/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/v2/RedisDynoQueueTest.java index eb51424..7efc46c 100644 --- a/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/v2/RedisDynoQueueTest.java +++ b/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/v2/RedisDynoQueueTest.java @@ -79,7 +79,7 @@ public static void setUpBeforeClass() throws Exception { dynoClient.flushAll(); rdq = new RedisQueue(redisKeyPrefix, queueName, "x", 1_000, 1_000, new JedisProxy(pool)); - messageKeyPrefix = redisKeyPrefix + ".MESSAGE."; + messageKeyPrefix = redisKeyPrefix + ".MSG." + "{" + queueName + ".x}"; } @Test @@ -292,7 +292,7 @@ public void testAll() { messages3.stream().forEach(System.out::println); int bucketCounts = 0; for(int i = 0; i < maxHashBuckets; i++) { - bucketCounts += dynoClient.hlen(messageKeyPrefix + i + ".{" + queueName + "}"); + bucketCounts += dynoClient.hlen(messageKeyPrefix + "." + i); } assertEquals(10, bucketCounts); @@ -311,7 +311,7 @@ public void clear(){ rdq.clear(); int bucketCounts = 0; for(int i = 0; i < maxHashBuckets; i++) { - bucketCounts += dynoClient.hlen(messageKeyPrefix + i + "." + queueName); + bucketCounts += dynoClient.hlen(messageKeyPrefix + "." + i); } assertEquals(0, bucketCounts); } From 3329399e36c01aadf1000e8f2728991a835a5ee5 Mon Sep 17 00:00:00 2001 From: Michel Onstein Date: Thu, 19 Apr 2018 17:22:28 -0700 Subject: [PATCH 13/91] Fixed typo --- .../src/main/java/com/netflix/dyno/queues/Message.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dyno-queues-core/src/main/java/com/netflix/dyno/queues/Message.java b/dyno-queues-core/src/main/java/com/netflix/dyno/queues/Message.java index c768d09..6d62af8 100644 --- a/dyno-queues-core/src/main/java/com/netflix/dyno/queues/Message.java +++ b/dyno-queues-core/src/main/java/com/netflix/dyno/queues/Message.java @@ -107,7 +107,7 @@ public long getTimeout() { */ public void setPriority(int priority) { if (priority < 0 || priority > 99) { - throw new IllegalArgumentException("prioirty MUST be between 0 and 99 (inclusive)"); + throw new IllegalArgumentException("priority MUST be between 0 and 99 (inclusive)"); } this.priority = priority; } From e92d2aa584de32e4169f78c8311e04da2d5b3784 Mon Sep 17 00:00:00 2001 From: Ioannis Papapanagiotou Date: Thu, 20 Sep 2018 05:07:40 -0700 Subject: [PATCH 14/91] update dyno version to the latest stable --- dyno-queues-redis/build.gradle | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dyno-queues-redis/build.gradle b/dyno-queues-redis/build.gradle index 0171893..0024a41 100644 --- a/dyno-queues-redis/build.gradle +++ b/dyno-queues-redis/build.gradle @@ -4,8 +4,8 @@ dependencies { compile "com.google.inject:guice:3.0" - compile 'com.netflix.dyno:dyno-core:1.6.4-rc.1' - compile 'com.netflix.dyno:dyno-jedis:1.6.4-rc.1' + compile 'com.netflix.dyno:dyno-core:1.6.4' + compile 'com.netflix.dyno:dyno-jedis:1.6.4' compile 'com.netflix.archaius:archaius-core:0.7.5' compile 'com.netflix.servo:servo-core:0.12.17' From 4f008f75f77c15fa9f54fc680084697bed7d5681 Mon Sep 17 00:00:00 2001 From: Ioannis Papapanagiotou Date: Thu, 20 Sep 2018 05:07:40 -0700 Subject: [PATCH 15/91] update dyno version to the latest stable --- dyno-queues-redis/build.gradle | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dyno-queues-redis/build.gradle b/dyno-queues-redis/build.gradle index 0171893..0024a41 100644 --- a/dyno-queues-redis/build.gradle +++ b/dyno-queues-redis/build.gradle @@ -4,8 +4,8 @@ dependencies { compile "com.google.inject:guice:3.0" - compile 'com.netflix.dyno:dyno-core:1.6.4-rc.1' - compile 'com.netflix.dyno:dyno-jedis:1.6.4-rc.1' + compile 'com.netflix.dyno:dyno-core:1.6.4' + compile 'com.netflix.dyno:dyno-jedis:1.6.4' compile 'com.netflix.archaius:archaius-core:0.7.5' compile 'com.netflix.servo:servo-core:0.12.17' From d1185bda08aa6e35eea5dd88a9e3e265ad2c7fa5 Mon Sep 17 00:00:00 2001 From: ipapapa Date: Mon, 1 Oct 2018 10:52:49 -0700 Subject: [PATCH 16/91] Gradle, nebula and REdis pipeline gradle updates Upgrade plugins Updating Redis Pipeline Queue v2 with proper names --- build.gradle | 15 +++++------- .../dyno/queues/redis/v2/MultiRedisQueue.java | 16 ++++++------ .../dyno/queues/redis/v2/QueueBuilder.java | 10 +++----- ...edisQueue.java => RedisPipelineQueue.java} | 16 +++++------- .../dyno/queues/redis/v2/DynoJedisTests.java | 5 ++-- .../dyno/queues/redis/v2/JedisTests.java | 4 +-- .../dyno/queues/redis/v2/MultiQueueTests.java | 2 +- .../queues/redis/v2/RedisDynoQueueTest.java | 6 ++--- gradle/wrapper/gradle-wrapper.jar | Bin 52928 -> 54329 bytes gradle/wrapper/gradle-wrapper.properties | 3 +-- gradlew | 23 ++++++++++-------- 11 files changed, 45 insertions(+), 55 deletions(-) rename dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/v2/{RedisQueue.java => RedisPipelineQueue.java} (96%) diff --git a/build.gradle b/build.gradle index d56c935..0a3136b 100644 --- a/build.gradle +++ b/build.gradle @@ -1,16 +1,16 @@ buildscript { - - repositories { + repositories { jcenter() } dependencies { - classpath 'com.netflix.nebula:gradle-aggregate-javadocs-plugin:2.2.+' - classpath 'com.netflix.nebula:gradle-extra-configurations-plugin:3.0.3' + classpath 'com.netflix.nebula:gradle-aggregate-javadocs-plugin:3.0.1' + classpath 'com.netflix.nebula:gradle-extra-configurations-plugin:4.0.1' } } + plugins { - id 'nebula.netflixoss' version '3.6.0' + id 'nebula.netflixoss' version '6.0.3' } // Establish version and status @@ -19,17 +19,14 @@ ext.githubProjectName = rootProject.name // Change if github project name is not apply plugin: 'project-report' subprojects { - apply plugin: 'nebula.netflixoss' apply plugin: 'java' - apply plugin: 'idea' - apply plugin: 'eclipse' apply plugin: 'project-report' sourceCompatibility = 1.8 targetCompatibility = 1.8 - repositories { + repositories { jcenter() } diff --git a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/v2/MultiRedisQueue.java b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/v2/MultiRedisQueue.java index 4507cd6..ac0d607 100644 --- a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/v2/MultiRedisQueue.java +++ b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/v2/MultiRedisQueue.java @@ -40,11 +40,11 @@ public class MultiRedisQueue implements DynoQueue { private String name; - private Map queues = new HashMap<>(); + private Map queues = new HashMap<>(); - private RedisQueue me; + private RedisPipelineQueue me; - public MultiRedisQueue(String queueName, String shardName, Map queues) { + public MultiRedisQueue(String queueName, String shardName, Map queues) { this.name = queueName; this.queues = queues; this.me = queues.get(shardName); @@ -71,12 +71,12 @@ public List push(List messages) { List ids = new LinkedList<>(); for (int i = 0; i < size - 1; i++) { - RedisQueue queue = queues.get(getNextShard()); + RedisPipelineQueue queue = queues.get(getNextShard()); int start = i * partitionSize; int end = start + partitionSize; ids.addAll(queue.push(messages.subList(start, end))); } - RedisQueue queue = queues.get(getNextShard()); + RedisPipelineQueue queue = queues.get(getNextShard()); int start = (size - 1) * partitionSize; ids.addAll(queue.push(messages.subList(start, messages.size()))); @@ -164,7 +164,7 @@ public long size() { @Override public Map> shardSizes() { Map> sizes = new HashMap<>(); - for (Entry e : queues.entrySet()) { + for (Entry e : queues.entrySet()) { sizes.put(e.getKey(), e.getValue().shardSizes().get(e.getKey())); } return sizes; @@ -180,14 +180,14 @@ public void clear() { @Override public void close() throws IOException { - for (RedisQueue queue : queues.values()) { + for (RedisPipelineQueue queue : queues.values()) { queue.close(); } } @Override public void processUnacks() { - for (RedisQueue queue : queues.values()) { + for (RedisPipelineQueue queue : queues.values()) { queue.processUnacks(); } } diff --git a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/v2/QueueBuilder.java b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/v2/QueueBuilder.java index 3d13495..0195312 100644 --- a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/v2/QueueBuilder.java +++ b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/v2/QueueBuilder.java @@ -52,8 +52,6 @@ public class QueueBuilder { private String queueName; - private EurekaClient ec; - private String redisKeyPrefix; private int unackTime; @@ -62,8 +60,6 @@ public class QueueBuilder { private Function hostToShardMap; - private int nonQuorumPort; - private HostSupplier hs; private Collection hosts; @@ -156,7 +152,7 @@ public QueueBuilder setCurrentShard(String currentShard) { * * @return Build an instance of the queue with supplied parameters. * @see MultiRedisQueue - * @see RedisQueue + * @see RedisPipelineQueue */ public DynoQueue build() { @@ -176,7 +172,7 @@ public DynoQueue build() { } - Map queues = new HashMap<>(); + Map queues = new HashMap<>(); for (String queueShard : shardMap.keySet()) { @@ -197,7 +193,7 @@ public DynoQueue build() { redisConnRead = new JedisProxy(pool); } - RedisQueue q = new RedisQueue(clock, redisKeyPrefix, queueName, queueShard, unackTime, unackTime, redisConn); + RedisPipelineQueue q = new RedisPipelineQueue(clock, redisKeyPrefix, queueName, queueShard, unackTime, unackTime, redisConn); q.setNonQuorumPool(redisConnRead); queues.put(queueShard, q); diff --git a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/v2/RedisQueue.java b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/v2/RedisPipelineQueue.java similarity index 96% rename from dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/v2/RedisQueue.java rename to dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/v2/RedisPipelineQueue.java index 0749a34..9e1998c 100644 --- a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/v2/RedisQueue.java +++ b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/v2/RedisPipelineQueue.java @@ -43,11 +43,11 @@ /** * @author Viren - * Queue implementation that uses redis pipelines that improves the throughput under heavy load. + * Queue implementation that uses Redis pipelines that improves the throughput under heavy load. */ -public class RedisQueue implements DynoQueue { +public class RedisPipelineQueue implements DynoQueue { - private final Logger logger = LoggerFactory.getLogger(RedisQueue.class); + private final Logger logger = LoggerFactory.getLogger(RedisPipelineQueue.class); private Clock clock; @@ -73,19 +73,17 @@ public class RedisQueue implements DynoQueue { private ScheduledExecutorService schedulerForUnacksProcessing; - private ScheduledExecutorService schedulerForPrefetchProcessing; - private HashPartitioner partitioner = new Murmur3HashPartitioner(); private int maxHashBuckets = 32; private int longPollWaitIntervalInMillis = 10; - public RedisQueue(String redisKeyPrefix, String queueName, String shardName, int unackScheduleInMS, int unackTime, RedisConnection pool) { + public RedisPipelineQueue(String redisKeyPrefix, String queueName, String shardName, int unackScheduleInMS, int unackTime, RedisConnection pool) { this(Clock.systemDefaultZone(), redisKeyPrefix, queueName, shardName, unackScheduleInMS, unackTime, pool); } - public RedisQueue(Clock clock, String redisKeyPrefix, String queue, String shardName, int unackScheduleInMS, int unackTime, RedisConnection pool) { + public RedisPipelineQueue(Clock clock, String redisKeyPrefix, String queue, String shardName, int unackScheduleInMS, int unackTime, RedisConnection pool) { this.clock = clock; this.queueName = queue; String qName = "{" + queue + "." + shardName + "}"; @@ -110,11 +108,10 @@ public RedisQueue(Clock clock, String redisKeyPrefix, String queue, String shard this.monitor = new QueueMonitor(qName, shardName); schedulerForUnacksProcessing = Executors.newScheduledThreadPool(1); - schedulerForPrefetchProcessing = Executors.newScheduledThreadPool(1); schedulerForUnacksProcessing.scheduleAtFixedRate(() -> processUnacks(), unackScheduleInMS, unackScheduleInMS, TimeUnit.MILLISECONDS); - logger.info(RedisQueue.class.getName() + " is ready to serve " + qName + ", shard=" + shardName); + logger.info(RedisPipelineQueue.class.getName() + " is ready to serve " + qName + ", shard=" + shardName); } @@ -634,7 +631,6 @@ private void processUnacks(String unackShardKey) { @Override public void close() throws IOException { schedulerForUnacksProcessing.shutdown(); - schedulerForPrefetchProcessing.shutdown(); monitor.close(); } } diff --git a/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/v2/DynoJedisTests.java b/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/v2/DynoJedisTests.java index b4dc01a..efbc5f0 100644 --- a/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/v2/DynoJedisTests.java +++ b/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/v2/DynoJedisTests.java @@ -24,7 +24,7 @@ import com.netflix.dyno.queues.DynoQueue; import com.netflix.dyno.queues.redis.BaseQueueTests; import com.netflix.dyno.queues.redis.v2.QueueBuilder; -import com.netflix.dyno.queues.redis.v2.RedisQueue; +import com.netflix.dyno.queues.redis.v2.RedisPipelineQueue; import redis.clients.jedis.Jedis; import java.util.*; @@ -33,8 +33,7 @@ public class DynoJedisTests extends BaseQueueTests { private static Jedis dynoClient; - - private static RedisQueue rdq; + private static RedisPipelineQueue rdq; private static String messageKeyPrefix; diff --git a/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/v2/JedisTests.java b/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/v2/JedisTests.java index ca541f9..085033a 100644 --- a/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/v2/JedisTests.java +++ b/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/v2/JedisTests.java @@ -19,7 +19,7 @@ import com.netflix.dyno.queues.DynoQueue; import com.netflix.dyno.queues.redis.BaseQueueTests; import com.netflix.dyno.queues.redis.v2.QueueBuilder; -import com.netflix.dyno.queues.redis.v2.RedisQueue; +import com.netflix.dyno.queues.redis.v2.RedisPipelineQueue; import redis.clients.jedis.Jedis; import redis.clients.jedis.JedisPool; import redis.clients.jedis.JedisPoolConfig; @@ -34,7 +34,7 @@ public class JedisTests extends BaseQueueTests { private static Jedis dynoClient; - private static RedisQueue rdq; + private static RedisPipelineQueue rdq; private static String messageKeyPrefix; diff --git a/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/v2/MultiQueueTests.java b/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/v2/MultiQueueTests.java index 44bc27c..58653fa 100644 --- a/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/v2/MultiQueueTests.java +++ b/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/v2/MultiQueueTests.java @@ -41,7 +41,7 @@ public class MultiQueueTests { private static Jedis dynoClient; - private static RedisQueue rdq; + private static RedisPipelineQueue rdq; private static String messageKeyPrefix; diff --git a/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/v2/RedisDynoQueueTest.java b/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/v2/RedisDynoQueueTest.java index 7efc46c..f5263c0 100644 --- a/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/v2/RedisDynoQueueTest.java +++ b/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/v2/RedisDynoQueueTest.java @@ -37,7 +37,7 @@ import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Collectors; -import com.netflix.dyno.queues.redis.v2.RedisQueue; +import com.netflix.dyno.queues.redis.v2.RedisPipelineQueue; import org.junit.Before; import org.junit.BeforeClass; import org.junit.Test; @@ -58,7 +58,7 @@ public class RedisDynoQueueTest { private static final String redisKeyPrefix = "testdynoqueues"; - private static RedisQueue rdq; + private static RedisPipelineQueue rdq; private static String messageKeyPrefix; @@ -78,7 +78,7 @@ public static void setUpBeforeClass() throws Exception { dynoClient = new Jedis("localhost", 6379, 0, 0); dynoClient.flushAll(); - rdq = new RedisQueue(redisKeyPrefix, queueName, "x", 1_000, 1_000, new JedisProxy(pool)); + rdq = new RedisPipelineQueue(redisKeyPrefix, queueName, "x", 1_000, 1_000, new JedisProxy(pool)); messageKeyPrefix = redisKeyPrefix + ".MSG." + "{" + queueName + ".x}"; } diff --git a/gradle/wrapper/gradle-wrapper.jar b/gradle/wrapper/gradle-wrapper.jar index 6ffa237849ef3607e39c3b334a92a65367962071..01b8bf6b1f99cad9213fc495b33ad5bbab8efd20 100644 GIT binary patch delta 37264 zcmZ6Sb8Kf(*Y7*!)V6KgQ`@#}_g9^1r{>hQZDXdMnp4}hdFQ^l$@@I_B>N;QS^Jyw z$4S=SpY>h8vmx5cAdr;hARsZoz`lM30|OHQi$@|x`rk;J2s$X9Uz0!MDE-Ry+Vu?e!v6J7 z^F7RQyA<@Yd^oIi5(Z1UUXn!P#Yb|*T0m>FF(br>wWq2B?AP?TTAWuUF+lLESQjh< zxzG8vx$NS%A%daW3S3bAs_V#uryJXW%1%!r-%wkb@CBB3N4Z~JSV~p{QtJpQr}l>I~v3?oel}E%s(dF*LeXl%lx|ISUKio0w>XqC*&C%=e<{ zX4Y^6zqR-S)WtK;Z{8xW!G}CpCemSDuzfcX6mIouH-(NPNXBtJM8Q2XSXM*4p5@*x zRm?Al-ysQ+sBnp82pw!$$Tl60erusm3h_mXygDNI1TSrerq!Y0Z;OK|=@)~K!-@R# zN<&B&t|=m+`^wz(?yKI)d}Xjv$RWGQ3^FheNLE_RlPQU*7{O znbDsJ5(PUyoVW0(CO&d-eSK1VG3aF%*W?1|u>h&NY<&LSAbhWCbc&vxKGw4-wS6Il zq_Nmhan!WFW|#I-b8<6A`<9gPaFzwrE=+yml4_)9Zs+tWIYTrk{h*ji^LWKK=>Psz ze_o?bA|ru;Q4u7@YET2&I`GCAiyneXE91T^V7V3fApANXJRKlb=ebzk<}AO3nio&sN@Wx@fZw>kn%y43qOtjuq9 zmb(}-wf0V!kDjsujRu%)o-81s9dW!df0=G+U4xe2GK!u(mfu1HSFbZdH|MYPpfj#k zq5GZ)A$~<|CXW4L&J>=$hJpaXg%c)-#}1;PdxC{ouS%cTm)}5$68<{FMmSG$Z^@yx zM^APTde__{e*ggQ3n5^)b$RvJ`~=hGxJ7)eLCEfUkh`a7l*n#0cQvJ&4#ZFV8a>!F zzrAdDMqc(a;QFM+d|@X1bl8IYkQ(N8f2$33!dnVUb@z)h>fNOZ{@G2pBCU?kLk8aQ z6_1R9!{8{REM6^?$=1Wf);pEz1VM!v&wb{P!62SOFeflm6xG~ey&xXNw1A6)ed>Wa zGYH*&DQQdQI6FvXV81Xp(Gh-P8pY%|GpO0#%cWqyI8dF@rT_+hkhi;J2#m^>o;Q8- zo_Awryt91MS-nnFOS$v`zb`Z!Mvg-qFhzbK#wF&ZB7ufFc`=|q*NYW4DQ5F*ZYuod zu=$iP1_78M4?!y-t3y<0!4%tyN@ z8=VsG_G`LZhWhvt`^e5fqB=RFDr=dQ6y?YZeIDpCp*db6Pj8o5<+o8Nnb%pPrROvO z$?#4pLBB0iD0JI!CYKj=#KJ{#wnpEb^Njn#f2gS(cPiidIbCsSs1Mb6d)-pL%QR4ih(I;{qu? z^u`sN6VjxR&nl|%J;s`u`d?gCR`16K*iZ<3Wh-Q%D%PW!r&R?eucL)?uWig%Gj?4y z63!+D|B95ltH|liJ+!4_Ea_V=6YekUUIU34*2NUMZBw$;rK8==g#~V!6%6a<@|eT! zbsXDy?B{H}^13#@k2PPcWVc@z7U0^-F{)$OTXM~ALLGpG;(e~9Y0!gvKFK`@W7r< zzjyX%5yK`CbH;Yc=!SNHr3IIi1&eKa_l6b2X+H%idb>IfeOT^#tM# zuG7Slx9R767L*Q!2Wkm^6g+qlQB)iAe^v)1pxd-79m4XvS=Z+}u1a;PqKZD6i7BecEy$Ka!gT*JwVUC%eIPaLf}bm z?_k=yO-m=nJYv`U)6Q3PaS>ht9X8qg|=Z{Sd6%%Yc(t+aWVmLFT- zBxLq)T5_;x0ot~-1X=lpF0@1_`Oh2hcrH8!h0kJqC;<&MXuYXBnF2KVtgQQ+eaoT5 z9-8RXRIqlP(Q3I8m{g{?Sy>nLlJJ561=w-QzCM`EepAYs50^WvqUUhja?ueii6vjH0Lh%@@~TljAN7;27PKMn`6$C# z>I%*bKWAk$&D-pe=klb?6i!q^oXm`73&WWu;hmCqeWxT?=FP?;Baa6~bNh5Ri!Gv? zs&rnB;)!3R#?@K4Z=dxe?7`Ny^vO)O#ywZbq$|&%bXRW^bWy6myz58YD&$mjeWXC| zo!r<&0A+I>!WE|ZAX|I;4){3}VJ3FtJ3oyc3WL7}%{#LC7H?xvse3ABT*UpRW6XmB zANJa9@PHuZnQuwZ2)&?i5$P+I-?N>JTrv8nm*wiol1W!5w#HSNm6zS8tx!^#zE4ze ziGkDP%~-5*uPSEj&(c0D3KoPf_@wr~d9$p90jNAUHv)c2Gi;M*absZyBiNJE!(V>% zS9U)i$e8L4wyX(6AfKvj?63*5qfnU39EencHS7zgX+M0I?8Z=ce6RgayxQ<6rOR?K zG>UyxwVTndtqljz_2Yl7IXLQxYv{ddsJD6;O5`!3wM8214Qm9qtboUDX@r?Wof(tNGNqsH1(^7g3O^9ed% zoK73e$eH#C-d&u*DBWs~vL=cMZm>RvZEBVRkJ6TstG%2d3v{L^u&5_R`(tXR&_RGKGh3=o3nhF-Ww+1J5o0U@eqH6MhqJBF3dHP=sqd*Qzz~Yz?t+wv!7#S zKJ@2R%p2L3_8M*KfoT4kqIO5>$`H4>pazWk;l|mN8K#s@C93pFNF6?6k3dTK$Xi(G z+ZaQjLMrpSKtr5LNAOkw&OXzDGY4PVlvXkLrER+x=!aBPE_%iW=tA#<3)zux~=BfM1ON$ST25-1I)Uv}zRH{qCI-T2A)3{!}JQ z4prUinxeF!kB9n7LphL44f^)MMkIcw)7p|DB`i-|Z;j;0oW=#*`DX7e`B#EfNDf%f2)#Wg}eK@>zID*5b;Pv?V zzUYZ3-caGB`bp&h0MF8iBksR-U*XN*gUHRlwI67M&^vu^lJp|%ho5HG!Rm#Kx zs)Ur|;oTLIv1q*_=6}zzDbKK%;azjr+lCColX6uPgky9x$ka_TmXTsQ-hLTM_dN;! z_dRl6k$FA;ALA zt7++$DAC7iA^^_@N-Z#^pDcY}c|64KV^L3z^-evIPjduLlvFCk8(Bqc8L+ETt7Wul zvctv4c&t?HbW*AYg+o|g4sNKJkrt!LPtm>-Hj#%-hs+7x;dyQa{C}7xAe;r&5E=|@ zKauFG0HEiFeux_&q%*s6%E^VqgF4JPEzY(SboY&}lR~mroy4udWj9!KVK%JWyScJP z^ShjqjND3*gOBV=sOwzj0oh!-B^4^lRIq+Cd$(ihue{XuV+AW8l^4IxhIr&Q^ryn7^zZY z&tG|P>Z|utA~Lu3tB&Ck3ep^NXe^Mdy66mRQubgr@*NmgX43N=Y&R7aoVA}{@u>`Q zyFZm3e^plc_U=LKUxTjYqR{Vw2#?o8JFO;=Eex`$QWo&E*?C*i-i8X?*ugxlNcLdx z0=W25pIZ3&8K#pT*L02Bdz1F(i@!-3YZYlB57~hikkpl*@GABq76E@<=S^G(JkPhn zzmXZxn3q8AfNJFjFAB~_)$T{5@2IPBrRV^|5!VuiP0iN>6M?iw8%n*qx8<`cr)0wP z?gPEQ6?cupHK|!X6Fqdp_ENDavJIN}K&!r;<_ly-Qy)a&j5|iyxkd7tb5kWuOClr4 zvHaq77O?>|=8`nLEiX4;YE|<`H7|PPRAbf1IGpT2XFsvXH^5kh55BWj%=>8QCVEU(_MfL9}-bRZp+8GN3PeohK$vAFo!p zi=R@j_#+ca-Y7`_Lx#;|&68j-wZ%O&c}2e-%hG(~+V;kNJteT;{a)}6)lgi5nwei# zPA_nHA>M=o-Ef_m^R)eS<_upI7=;dX?*AP?z`_y^0WN804c!ys1TWC@5<*zLs;Kh< z-SguT?}^BeR4FuTwkuHUZ!b!Z(-usd%@!x;8w$71Ss2&kUxRys)I=s{mdn4{4aYu# zIyc3b(v7y|6dC)J;OVJ?o9az$%;>2$TT=0bi-=jjWdf;*Bq#&ZfQjGuz*Bu_F3uC} z28Z%MTWBus6K??NG%s#5^jxQXg?7M*l|7ocjbh4#T?!$~#`Nor9BU!B9kk(BDl6Nk zLe>%| zs{)4cBouWp4IO2y7VOexfc;HIbPb>Ld)*i+sQG^6t|MvV%MNC<5Yhq#%>$jq@DJut zgRvT@HP4mil+YQ4--Ya)Q{^x>(U{|~_6Eo}=f_Fo%pOr6#YbF6*!wTK91*< zL0r?qSuP`GdKS4|+Ar~m_=ZvJc%l61b7^d@(Tz#OOqlArM&8OOpk?j4RS=J$|Eie0 zIzvCu)ifXL4_xJeZcS;~)JJ1U6iayBL|!iOY3UQKqZpixISI>e0|Y)Fjj62;t!BhY zmkmVHv1<~qxg;C4+^ax-6o^$}aeL$8Fl%37bs>z3LEZ#YcVwq$z}OgNz@z?Tk$y{Za}#4+P}%hdAg zbqn3XwC?Xc(u+D1-qLbs)ag6DF+p3{IykT3AOt&yw0gb*xZe>;@h9zKdyo9#d)5q@ z^_tbkXM~ajx7!jUS0%0mDtfx}FQbX^riw+oDop7R!J0fMCeM5BHZ867ZX4qyH=3yzT&vAW?rQPb`rC9cc9bgrU9$lR)ja$=^m$ zA;9*h2|wo{^{j8d+&2jIf^nhR`vb*R66t$)g}~wCFs&48wbzU0)P`3gg<8`SM7}pc zwBnayyLAT?6J_7N(Ob2v3)rp}!La1NJ&x@0b=wkt^i__sr;QknbG ze*V07rk7S@rTw@u0;SNyL$Ps5in)Q(N+o5rPliM7C-sl$nn2$r>MN$5!AB55sWJAt zw)|C{~{^|6N-l%YD}M|M7m6MlYx&@c(E8 zoL1Q;mcadM0^opw(Iv)1VkOe_V<(=7AOiM3>{-YVM8KHsL*j#%g6HcIam?}U%}k|5 zk)G=$D6-=fqb)rs2xT%`+^N>tWvFHJmSYlsPBtAYK5o#j`^J5C@{cvRe+S;7gx6m; zS{GjpbKa)<RqT#s#lhnoJdcMbS=Bp35TnDH=gZXVuSu|M!gVl8^^<) z6i4fHDZ7Gj8ObWlQiDSgLotRLIOow?*V?_BnXAd$f97MIcZaoNA^vGOWwT2 zLx|%>-_77AE4&GaHLp?q)CY&JTma16-{^2cCffwu(nyChzUZh=sTuylDnpIN-wU0g8|b@7d5&ZP~)+Qd}K(?g{jz~vuuQ_lW-RfLzV9v$~a>oHC)eZSo7HV&%5$6<=RYy389!9 zi<=YdBD`Z(Utkm|8G(26Nt|O@t?{4nva9;GB^DS-J1#K2PMY&MDJ)Kwr*J>O1ZFi&V<0p^fO;^7-$?RN`?aP1ii{p6vMU2R`%$C9rxdL16JbE@Hu6(pkg&}Y`$u0<25nikk582p@#2-%n?nEQ zH8P^jTKP6cPI+&i0o+IVw$W4TZ&l<+kFg$txW+93B>oBXk+$-HxR` z1I8B#b-Myd2Gs#mh51NI_Z5(B!bi?X>WBPnBxUqkKA=us7G9juCVg_sFUrnYFVBFz zw2)7oSb~YidK3$ZPKpt8dVI$th|gzQB3jcyxg77 z?>8T=o1smrDH}nmP?fL~WYU?CsisIfbu(k@!Ya0E7iypMGYT!^0!^JXksv<_uQsJ7 zg448`3RsBX3mp5?f6kd-v~DkY>_U#mkH#)3TEEBp)Jjueh>qA6d%j%Q&EiUE8S36B z$Z?LYWf;ZLqJpZBq0OA36B}gTpV0={N^kn3Do*b>*o_Nard!1j`MM2cYIHM1F_!1_ zGOMk&$&wn?I`4kpz2+vsgB;a!Eaa2ZGJnX%3=q%FotMwZ_etwwt1=F2O+reU7N%ps z+c~JLGIB{8f+!#6S+(p0hJysyI*+gE@Hng9^jt>nVl)J3Xf6oIo$m>lA_bVvq8$*S ze0jw!syvtP!S6%bI6Ut|q+sdBbk~f$8~N95-Dad&N|()BETA6s>KD7iCPP9RU^_p$ zfaq?fkmI3C5MSl+DmX@eOHc{%-6*mW>SXsW=)i+{$(8AWU;@ry`lz?zDrbE}J!0&- zG=tZB8@G<<$JyFnwJLt<1B>?VE(XL(t8`Jrn}lfo3_MGoT#8|WCzWE2FF>1l)>Ry}}y?rZE$oGMy{UH#@h zUbjg!R3A-L`U`76kzdPNum1~K-5xsMu6KwO3zE;YH(#WVnv&2qh$^ox;;EP`-ZXMC}EJ*yM4~( zKL_fFVO>5ce+ltEyy)|A>)*SLMsoW+3|XYihWpqL@=bn)0KLKi7sv@|yl)-&y3JA-~8Hg#g|G zdG^Lj*vc>sG*hCPU$ms1iy<#_MKNiyn^GX zL3!n2GWTg_4JGqLd9Hc^yRksH*8tK6xgS^TOs+-Rw`e0a66EVIyrIuYw_uLqS=zoP zWPud^0T7?;BgIJsNIN*H$~2YP{f9VjNiFGdjR@`8C5FQ;LDxJsW2i^l+(u50X@z^7 zalbt7_ApLD@geTft(a#vP=)4q@Fo)9%&7^W3)ETu#Y_h1oH)M$a{X4lw zs@}mnVD~pgO~!9_V6tPhNonhZy`>83ZeqWNik=1nhr~X7T4dgsSj|c}@d!`z@{3+S zieSfgpmkek?JtY>)R5aB8{Q0qG^3t4gX&yM7P%*W@oES{9rp`IKXA+zOM_Ux)${>= zGKF~?E44iSCzC2lelvDnU%7<4TFzS73WexOV_3&bMC~l5PnjVO5 z|6%xZy@GJ6{}$Fr<_JD*GAM2T?l=6ob+JpS-6Iiy#PQ6~HyteB3z3+_mbJ5q0wW&) z4T7C_Yp+~F02Y_Qf^>YZ53ch&g%`Y6`&w(;8ht5?bO~k>OQ3%Uu7?^?s%&bM@r+Cr z_9yaxA9oXjA-6F8*X@#BiSso7kHBytC&ohJCHBap0I$9nTDSp94{T$W-%)g(^{i(k zmlzP?^Oa!(;b0xaQorGgLYjFr@Y_REv7eqXq*mOAwzk+-Y8Kmet4Dd2Cs1NIoK~;A zM16k$^`iAucFEnCxo)5G^#Jsc#k=X3`slaGzsU=F-K_0_Hpc&GLD0KuLm*Dt7GuL< zlOOr11dv4qL9<6dD%#Re#fytz&E1$&I?Eo+k=0wI#rMYdDwv`bQwSk22R2Ci#jB(7 zkW*PD@`T-zfeO%xd!;0{%^Lg~(20ks`L1RBWJ8sxoka$6Wp)~$^wDxtjT4_l!Idbl zngoQ%`>_W<Sd-la-DoRz=PYd>}d@*$LQHxS}BvK1UYKOA)x0rMtpBWubIws^SDMDK3 zr5$7KZ!|YRm$XQQQ=XP02t&0m)aRmYw~Ls#v)}D0bRQD+DE8Gm&AhXcz5N@9QS^sU zGp^4epRCczO3E*>(^*g@m$hrI1iOM6@RDy-Ci0x(Yq7(?n1^L6vd`iO)59%zb)>b* zVRdK-_wLNdIxM6AD!6!$xxldY_^j%GiLWaUjqPY&Ys~M=sGh^px?{nl~dTcJC=?A2AXPuej;8>=K?49mHYd9V};QM=I z)iHfv$oyrFNpEn`T_CRZ6z8{N_It6>uqW}F){X5hS`p#4e-v!XN|)x!Pc+`65&K|elUK%B7kQ(>&|_SE=R+ikE0T?A7^U9VU9 z6V*q-ZmqZ6@aoRZcl&^KRR-!xUAywSfDuzP^M3Oj0XVNU9;Ds_!MQ64Ad>#fc2#nP z>$`-JAq69$F=*y44hNi`;^)wP8LL^#Z0pR)Kx=<(HS>B&DFfQyCz1Z(%TTfq-`R5? zT|2jvrqTZUSt#dc&~UYDxErO*MJ8RR=OA|+&Ibp&+1{;Xx`8+StMf(5t6io8hpUGb zzEw<5q0AKSBLi38UEjkRu||}DrfwMT=PT6BR<#&Wxt0ARqENAd%fgN!>wJxU5h{M`L#*P30#-m&GA647G{a|i|6O2R-vWVYwL z$GiYNnf;bO=h*6`=0Io1UL7QlQ|*KR+mG~8OP1R$1k0SNt`4au`~f1}%l$c2_iwvh ztc@TnI=4#(J~sbGAlzJaMn{~ze2Kv{UEL4sdR)W0R=E)Z**SWSDEiYd*5|4V^_2yo z{l~R*0amw{KuDUm7PFDmj`sl^Uxl_O;mEGm3~tRGU5Z!jmh#|eW!JkhUb*RCiK`tp-SpokriWx`VCFkr!up*MvGQ$P}4z0jmkWtUzlz-N6;uYoRc& zdG$^;^`puiBsvWjt0&j&HYJH#6Mx8Z>!b6U2)-)bH9?fF{bvCbxxt$4>`3fb!J9j# z5bmEi;jNJEVIrz77nI^^@JW$V`Bpo#fMG2tR^$4DuCmYj?Rviii`yn6%z8k@%c(88 z>7|;NWLlmOVE1Z^2N8rrxtO^vjq7jn$443M};1X9~o*A1lLmJMXbetqJiRjicR9?4Z~sPt4hHr>3kF8=KW5Uz6iBpxbr3p{ zr4Ra_V-MC3bH7=b`te|LVxkU8w5^I9ZG^~^g9a~wq@I+-gq386!Yd(qjdC(u4((Kv zSX1}Bz@rt@tE^Z<9gTyo#ba?rx4LCv!OG76ywbnc>Daf{iS`n(baZq~5c$2A>SWoo zU-jwMbL$VfcTy6R6Ni5;M7ekQh9{Ac#{mLngKwtyN~)$W4|0oF!pSJah$6 zAKh{J$;Bn~(f2TK1lZ%Ic9KbjGN)j-t2MVZNOh46aVir_3^Snf>vU~j2iCFM3=-iJceVkjGkCiY->na zIx`!0s${yFbgi*}|d!q=F zlt@eKyGHT&onVdjyf%?4Yet)eFr(=Bb0{T(HpGi|4ZXlIJ#))YXPP?F5Nl`ZcK-&{ z-eo{4)ULlxpB^sb5Iu}o$y7m|D49U3zggGesWvx6cpt-v&&^S@(p2waa0|rVH{&m| zU~(7~T+UV&`gApR47QAWAHtQmctm*5F6=&73T6Z`X|X~TL`*97n`X=;)$vvGGYX<{ zR`@b>cdph|WM+v}{quGTUR#UCu}nNFt_!cbaC@T zJv0dq^d=miLm30n3-wYcF+af9f_fj+F%Blsyd6iR>52_!c2C)S(w7E|$OV?Se@;b^ zQdqXnB})2^vYq#wGy}iMzH2*q;>qY~dWO%zvkTzuuG8YXBMOG9gyMobl*b|k`b@TH zgtxQ^c^kEQlnAhRP#R}&Nmgh~(oLr=+blI-|FP_6<9}&zXeNyZ3&o0_b*l0)%Tr4_ zU-cnhc1&Nd8GA#W3*tjx$Ha-#&PAGPHs5IDr|UYG225YLJ?o)5uLpX=KG zSzTe&S_S%=;81ve5Qpc2u$N|iqdPv?=2*&LRs}CPcgHf)RAj<^2d>_t?b2;BIBu@` zJz5!0!J)BE-V$I53c$vfH2;w=wU(9EL>`fp{iUGa$D=RuxaC~_N-DnpTWKfaGtXGr zX?3vqq6RJdiE3YtWa6;mPgFC=$wiAGCkbfC$;crdZQXK59ARJCKMIg`Xspw?hXM1H zrmTXoG-~x^Ce;((f8`xncb8K0nrn;P=`rLfGCg$Cu~~tF&^@KGe$})x3bo0AFQ98o z`^zBHlrPpw8wa>gsH91mkw%uH*StCPN$27EWsgrB_5`5HvP(WryD&sAej)g!9r}*rK#%j8Yb1u zc{M00!zF8y{+bQC32unWA=QnFQ#x&Hw+LTA%hv$0HMJly8`4Eh2?q{RXKK0mU?aQf zQ0Z%CHFY)?T7=KvBra%TwelRVNpxl8Xi?6o`{V$I%Eu5dzR52(-->hRnALqUa#W7- zUig!1N|?*;z6w^%67@hwD^>I*N#tEibSM_DyI1#hYV@T1D!cLMlG6JsfL&>{cx>BCJnobZv zYUZTXK#!-)a1?2B=Yo(V@I2nv4NDHU>9Eu*FIpO7%&QP*W$c`tS62IL zJvg%Sq^vt%M36+<%5Qf4;O?Nc%L8ZLq+9w8pFQt}JZMNtL+DQ&?OWTZQur>ii*U=y zc9V{sj?X_QdNRxrVKVY;3V$E3#g8Jh&gs_#8iAS&b zeN&Zh;FrXaE36s>lW#Ms{h!t9|_ErxSzN6%>2X5}oe;`HcDprr-E40PAd=;5^!cJhIhhnU>dA`t2&P4N7mC_!vDzSpAvX zFJ|=D{%X8lTsL*z1K~;hPpSLXqdlb&u^f-J)P45(QPsi}4z5wR4_L#mdA0+nZWQkk zl*NTkkR>r0NghKvsym>Hh1gQux!A<^FVFIeuuz2qo{C7g_o4JNCc zS5cey(uX0hbc(Gx2L>Hxc?o`1?az}-X9rTBOmJ-5t~0k54bQF`6PvSf=u2$Cwdc{V z>0M((pRCNEBSgpW$6w>Zja^qOW+knYPGXN*l7x?(P4L2tB(74EMiEDf;?qakDaTvU z?LwL5;N;#UM^ zI%kYlQ{=_fT1c-sTocd4EWm_c>Lko_KNrO^wj0mP2PTafIJ{!!d^Z@4P>Z2xflG>y03DaW*$^u?=ig;692a;G zhA#Q*0!mD04pPa9a}i7KBaD+mRkG1y76U~sl)2=m(frD?Z<7n&%s+p$+zy~FgLb;2 z56#Ed#f(H>E&L1+;Q|B>lM$5-mq|i&;cuvC#vqD_3S2Dq!=z~|dBg)7M#VTj^7^-Z zn&xP~dopR-0vsWVAqcnDT$sD(935ZCEtnKu;QW*SfY8&IKqGi! zttZ(!OE_&0eb<$E(O1>Zk=PT_3g!=A`K}G(UfAk7Iq8o%f;ZV)39<>J?ehFt(wc%i z9c{VE_E{8h=UzwEI-T2?YN;9O_G)v$2IPNU6aqD04`1N&_p1D*ge#6pmpP|h6;Huz zI3t|^ojMjABmrMy#x!xM{=mMi9eU(BT29b*S3t(;udti`(E+G;_mh-5GK1TeGo+iI zLZ{&506HSBG24vb2k`K9S#)dqym~~YuAc|fo*7Emy~_MY?zTU8$=sQ6n*#ZcOiw5h zJC0O4Ag9^En$pbrkZ8PFH89$TKy>pK+B8BScy1u|k@0G`hY}SIi(JCqRm5Tz9isR&c$}05HYra12JdUm zzmce*uf)ka!2^o(0(!HXaK3A|6&uIzgd1+mo=6Svq%5qBrYwA0Vy>9q6OxpJRFZoS zvsrc?pY+Ibg~BnEcn%+?4gbSZaP9sT0FVZ1VIG{EO-a5BVb*&?L5@gK2pm@heZ%F% zKcb~R{FAvQJPg8j-=WYeFU^mxoX_-@t@?{J>uUV$4tv3VkvHf)&!HvMIew{^q8D!6 z7S}Y|w^%n=SH7A?eO&!SS7ux zcIO(vgS$e@d;dZ$U8>A0C?{1Hfam;?^`is#>iF?Gg}%=Mqe|hG^{eepcUJV{T8`M! zbD-G@@y;d4r&po$O{i@z|I>`*gkdl8#}D;+UBf!p?~{gO6F=M)?Y3|$9PTlzf==O1 zws@Xudn~@=t-ZD2JtNleKPvKJ0MVu^}X1$>IJ6;zKqWAfECA-I&6A@cS!Xl4q zHsBpBTlgD3g<6ww?+~8Ecx!nx)e{+JNOI5w(|mt#O2A436i5QC4{GAox?;?e-tcyP z5kB)v=dAs~suX)_Z&1{gD@q48S-~TD+@D*QJydLaY!%cMU*mUQlls?s;8@^0F7k}2 zxIf}St>}0wX`e63p%!^ht(5$w#DG9J%@Q&na{or$&by2D3yP^3in9PXNe}YYJ5n9m zrYA&}VE7RBF{Spqt-Ubw9Q{^XnB`36cT$iuv9y1>i6)C@2SK?Ys7O((_Vi- zzH@;O!?;s`ys(FWaQa3caA^=5>NLBG)a16uM&Iq?IGFl7z^6C1-~HOG=uu5i4J6_# zEa6I&r~mn7Gf@5rlSd#HOn7QWJ#xMS1q4l?$} zr1C|iJ@cS=)3Qdzz|K$kZoq2UE%hsSX&03!VcK8Y4R8FDUTSY2*Nw5%i2q|EaVd}T%K`}{*!9H% zsu_g)F_;bwSYN;~AR7t+VGh5h_G+C4|E8|Z+i-a_(dVA(4xQcVeDz1fx`_}!ZJ)wp z6%_gtxtOT~iF3xIu;q^z>kE-W@P>)C5Hg?Ux3R_}G4t`bZ^vfSFM^au$a#0Z^Cq8P zb#wD-p3!OigZy5UVkNkL6W!LW2T1=4bZ9lpd}q{z`PK?!41D0UgjBO(bw_-Vc89D% zW`O_3zuupO|FQ;vVFOfSEv9dATl<#Fr!&;Xv^jfH9y9ue5fud<>7Pt^Mk>ei7=o{{ zAFy>t5(1@?scxS^51aW9M#}SV*l?Y{M2|ivzk!RhlVOSVzQ=|SDqoXZrDpD)tqXyb zgnulLRN^hH6Vs zT$E4vcPT#Yit8ku_Jo2inaT^uR1U09YInj;9%s|2PdJG7D7)EXk}V9fjmmXrWb-Dt zHr>F<&*@K}nQLMm8NN>`KrRy`SVb;F{gu6t?Sb&lqG&ILxfh|Bgr@z~2gTF#@t%Z` zu)S6Zp-=Jbc#^=0EB9iuvtbRk${FdkE4p{1Db568)I^0b%)nEI9iA_PbU-!oCdu8#+%)tmfeX;u%0X2!C+u zSR=q7L=#xO+7dAE)z)5=8S!oBhIHiYd?TtbD>1h?Y(#yEjty3tNV}DMhMSCM+F%9q z6)s&+D*iNp*zbcJ> z-N{0LRl1(k6ZN*kaf06l&e?K?_{}-CaB-ttn1a1VwIkgSDV~{#HYWOgpSFBK-o$R3 z#6vLNLR0+2dw;|Fyi*R5Z?EE6a(17=8*>l;O7%bCOc5V$qnr3894cL39is@L4_PgL zpQ?X4v^X2UUK)t%dLgUr!5B(ieygqfttIw^H)Uk?z)splVupMGr1{GXabGDwcQfq5 z9ato}a7H`HSo5xB5)qx8u zlBCfop>bI~V8=}L?4dvtCj_`B`|vb)lZ<_)?JCxa3~1j(-yjS z1+^Vp`?8ctgrA}KmTN^&wU)}Q!cqIRK#Mrp zpw*#m{QyymT6=l`YqxCqzwmTVhC9Z}T~nQcPaQ{((!`cWtf?wSD9Mpx3vga;$zf3r zds;g{v>6=n0}yR#TItNDD80-8PR>CDdXERXJL0CU|3-PEZQ@+<7Q0ie52;hDPw|o3 zBXtUIC^dj!H>8l_f@+Drsg$ZU8V4Gh@XK#7y_{63oULkig!T@aC8KNdu~EuiuAQ}D zmx5hfvQJ)9##4o%@Qj#MdtfH@{ecRZZe!>k(tZzYqgO2j@~t0ya}h z&K7VMXnp$jZg5?7{xdqDv*j01UaGeURTn&9O#hOs{7kQ6NlQ(ero4z9a{ zc7DAPk7~OGItzeN>l;l~d#q2lz~bdv!)iecH6dF(KZFyARx{r-AHQ!#RNNXOnvt8= zmQ=L>D90XPp|tl&zID1=eh8cUY5EM0-Wp@o`D8wJ&;KGz9SJT@F91yc#w`JYF!;TJ zqFv#o+%_a{u_1b5E-4{;A}%c<-65v~_+ET-eE41@#GAR6czgJCX6Ngy3`ASLrxar# z`R$7GG`XfA^QWNa;sV~trdZJaDv z+SWF{X$tFVdfMiD+U`P4J>^|q%ec6jf)lRY+TO6Zu|@XIV90kB^ymQ$J@NN#yC)*` zxP@KD>}&Fdq#n3qB+s=*b*>2tfV3EcyD7N1$1mEokMj+-#Z-GU=*9S^_W5NFfN+2P zAi^C4-;BXHT~{V{0?Ai50viz}J#omK7Odp-i`Q^8TAU-`TVuViGCkA5o4P{H6(ta( z^9)oDnBQYgKb{lf(WU~>9nW#4ly0htiuPpqb>l+-5GpC~Y=)$8Cg$)4yEy$hVy`)n z(Q;pj_G3EQ7mN^yYvV?nJ#BhGxbWB1hNBN)R=KE?JH}UaGsyoHy8MtWtdA$Zz%4>k zWd>w(mS^WW+uW~%Ut*D|-(1L(p46p$KI;bY_Bumu$>Ev0kfZ}7Ytwuz-Ikh_;QsVt zQ?^GHk_#i*6t_kB7^39&S2o!dSEI!hS4%yY83icVr6=V9yk@e}*MwMb2$~R5CnFV{vmr-IIk?|9U?Kf9D5RzwH*Kdk6^E zO&W5M5t3Zj%eMib(EXzmgr$Q>or%GTNi#|snA2<(6hQ5MaNeE4j5_Y-Civ}n2tz9* zExjq+7OJi;VFIbR@k!9|3Q2+*N5!Syh%~b!+R?()(W}nE%*Oh7# zC-=H8wtyYSuOP`xS9SdXdIa3N%9WpL%WL#%-uos?y(#L34;df!9cajEt^z5-z7i>- zud3<(*Dso`lRJuqIJzq6B{y`r5~mBbNhsM0W$XAecfnTtr%oq_ww)|)_LFGi)Y8`R zY4WIEGGGDlKYFZ$OX}axPH`2M$9P+|v4z6sv#MgnU|t@dIi{cfwtNBSle1 zT0Q3<-Y2|kj-&(%`FtXoW~TE~w(slHw)st(z}M#$!jGE*qCI4O^t-U22#JTI=pfb2 zq*|bN=0{@Awe;vN3sepXPoyir2G$rw+vn8?QD5qoh$9qoAhlOLbM<&4I<%(kvpO_{ z7@rBYkX5%q9rGjZls&N4Dp3VES6N(v^Vdp%*~&AXU8{OJ-j0~WvTjudWB?BVd&Q(ogl0`fd~2CH%(W;v zGCRM_i7ha^mszE- zLb$Z9dW>8$!nWudUpZLnN%?~lBJFuHf+p+^4It8K@~!T+$+o}}C*il&^p%Fo^2$C)QPtukkM zqj>MtDAs38xCQ_(V;)J*Xcu7UY?nTT3;S-FXwamxqsEx zE$yXeWKDH#BZ%io;jlTcu1Y;vcUI3_vcO92o<)lvm^b4iW zV`bqDT@h-%i~~vI1^ki=&nGqCwbvBL2)Er*{e%7hn(&toYwNp>bQJE3r;xKHT;jFC1VfDNY~F%Zha~PB}QM z_*V-(nZD9mu@mE91u-FsGTdkcRYVeYCAftQSP!c$E*4;uY_mn1Oy(k|r)*!k7G8z3 zra6V5G|OLs5nira#|q_r;XYEXEJBA~-_Try0aGw?npdipSD%k|)wV&#ZF`@E&9Uf| z_Pd$2I;tqxoEK1&+K3rlK+HhYOrN1dYitQi-gA=F^L9MnL?h8R2DJ=}zDcrao08dQM=MN;<1 z0Ae{Ig#<)^sNA*fsbtwz=I-t=iO1th*{{z|v%|j2gd|z58naVV-{9FqjN?9Cp;x>~ zWj_3sstTZ^22-iUzI1SSQY@fuJqHU zw`zMC)xInRcFJ9JPBM2{H&gEAO+D{oP3t`x>~DQFgj^wZA;r8Svz$!2B99&XVL5EU`Yh%XmlSK03gqYVB{>pGHT`0nd)VhasW)2NMHg z9Mpx?jhRe*DtA$Q#^;xEDgH7KCD;%_g1^`5_i_0Y69%Ftc$Gb6>#8|Gn4{sA+AVI& z+4)2}g6r=}@4Y7l)Z9gV&+SjQW~5Z`?$U~^wWi$$8GFPx)>1iK)p&W`?>;P(DPO83 zLeBx$Fr&OHN!!oN8i>ed=;Wj}@=;$Y#zCyBOg@Mo*mxXpfhs|W;5gmbG{p<0%@6XR`%b_=)-UF1FN_sXP0j(qZIo^ev1K(v{&%o&rjLlP6WmY0`26*HK{U4> zd(;Z47|j!wBTC^l7ifFf1ygY70$c`@=bb^Z%*GVU>;XP4unc8kFv<9t1g9{9axMep zaETuV8qxeJXnuFTcgb8(Dwem5JCRS@ATHu_Le|S65AE|)SfH5%;^aaz4>zehOoDp` z>I(_WnTwL+dpw78P5g9`<5A#SGR9Y5tCEST&^UW;#dq)T99InDnnZIf=Ig?`YZ4^V z&n0R%_s!hc5=53`1z%f*HCf$TfjtC}4(rD9R0cXfQE_WLPD`5ATp&RkKcv+a?lW4-UBxypxpqNT>YsTy8xiUC&i&i(8z;)~updL|WlKaPcl{19#UA2j)yKUNXY&ZboWjvK8W>@j=5Z-?mzaSiK^Ly zQ}65!>}N#}(C%fiRr+6xr?)htrYB@G#TYoeu@8q!`Yc{qbsV_;mci$Af=R)D5e!%P zyalD{4Y8*;QPc+N=ZaeDm1xoCSjFSF6u#H4WW&_OKV?U7Y1wlxfBcUEk&tV+{~z^X z6`p&O_vg0(!Z+8)^IacE6GW!50YhpRawv+3pVQ5?wR8%I)Z0ij{(7tEqP@gg3iL`^ z5=mfRbf#4>rYSZUcW6F8c7CG3QHR~-gvIQOV4hr})1tQr$G)6QW~OoNq@`u0@d*F` z{~!qcO`)n{Gc3cTEdpudY(_&2H&K{~_xV>MB@vnE_x(emp!j8^Vs9k|0mevFK=F$v zHRvHMuLYoHEKG5|p30GrVnRt&6`h#AymfnfVJg{@O#I)VV(Vqn0h^wV*BIvYhMNeq6HlZJ(n3fSyG~6MK7Vo zt1e)I$eiA+Ys~Kil-fL$Im!s)=_t?vyT^&*bTBPVH1ZV0(ZhdNQ zTV#4R^z5ybGwR%k`*fB(YT& ze6?eWnG*?c&!J`^Ro|rUbDpnl*4}CEH22vcnlKovawwVP(>~8*I=fkg173EebRY3~ z9&A*{o|hvgx9s1F4-hZ$Vsjz0Nc7`GRF_mfxPJO=Xg#-`a74k1BowKC&~f0=TOGIG ztmQYeHVwt@>dBI?u#e52g#QvAVgvdYZ$?Tb@Gk*sm;5)0)O?U^ZQ8!5p|DncUrACq9F@7XY6Fgn<|bp5n`KWsa%Q(%rZG9?^wt3+juU4 z2kw9*!~84Jjx2Se>6kr{61^`3I279oLVQf$htSWsRgCNw~E8Y~hi69Fs65?t)}HBWp7@DmFv&2XlK zq*v+(Evv3}uga97z$12U$>0bq3pICvtBjFRP|w_wHziR(@F!J@ z1?_C8Px_}ngjw#t|6!`CT+DuU{#^@vV1E4I{ol19sE!U`Q7A;n5cGpuQUPYM1v$(% zk-gNG!%$W|LD#&Y>@)Z?=o5x#0bXmkd53#};UB?2q^#@?91%u;xM-GtPWscn*IYBc zPn!+THQzM=P7oWxG;?sC$%>h|Ut@oCEVh=GBtjZQXBaJ7e$XV?ShbfwiRgfOq=`u9 zq?PD^oIe4;6zLk>8Hvcw360x=1#xxrDDDz%NNlgQq_U*cVy#uKw4iKl)>VzY;kh+y zSD_}-;1r~{47b%i>EOGL^GV*%WbeM~#F7C!pWar*l!*w8GV7|;bYFCeqRs2nRJ6`0 zFN*M-poD6N-BZ2Fv$R}ssW8OII7*g>I4U-@Sk?l_+Oo0+(0Rs3KDmfez^Odt@(rHx z!wl3~qS`Li!^qogv2u1BU^8hyuPVJEf0`ToTA=UHuD*TdWwGB%2>ja}uF#VrrAC8RH5W=hHFu7!!1pdO1q>wDPVlWMAPZ5`1=y2)vw9Ki2^xB>Kjk=gB2>LKMps-|um z(qf_>Wr_Q}h&GkpBrN>(mPf+2Bu6W*z+C&tZ5O`nS7z{ zLmdIqdazFT0xKd(=e-ZcK+&w@8Jb+;$Jjm$m^eD*-e z7;=%H67tLu?fxURoMOx_;-Ir9+&)w05M6(8*_Y1!9>`yP>3hI`(+54xQ008zRKfB$ zVJZH9wl>UM2@-yelL3m9H|!P!5&153HX%ZQjc^JoT2i@fiX6T|l!{=Y!r`bOO6A@5 z@lk8zbu=4gFKC}c44tT9kb^9Mpw{ViHAGoL3Ci*5%a_SCu7kUhn;U^2r3aKh@rGd4 z7UTN)TsK$hYWm6W(C}38*nlp|+{_kCq*6yHK$8tTXkb&d3t;eJtd*hDc)Nuv{-*t> zU5QDQCBX57wyj91QQ7hON8LFdu#^e&C!V?p~Aa^?Lwvwwvw@ zGhGaVts%UPC9>OOR$lpAym$9)xHKgW>>FXHBF=l!HI=|g@f6-fWNg%{M4JfLwI-o% zN%i9-h*?aG1Yk`eS{%00IqaXzodyoKLl znvZE@d1YL4Q@I(E$?QMM#(6C4$L3;-SgHapS*qjhMSoNhAvDpj_H%lQo|3NlG`abU zTED|7Z73g6l&}ULdW2);JRM*aU()wCctnSskahNMFx)>$i+NymkahOw*GPHD z_FsV3&skdow=FZ$@l$Te71<*TX6t2FR2$gxG?J~L7F%%4IsyE4)OA5x1hN)?tHkAF z@<-O$bhk=VldY1YQuN6bPPGtQkBRfEVI48VMnW}3aZ&0V14xw&zWFYxRgw8>m@FB@ z(uu1JPCvIMc}ddO#G$K=B>p0POY5*Vx2bwdd+`$V6*L~p`2BjxjfyQ-;kkpq+ie+| zSzA?_rf?VnZGa09=bB}gDLZsd{j?0HC3 z*v*RzA;(X?uZ;|`e#K7>z=qZ+iDNA|r^7CJ?S#$dhEI<)+oUvRv)f(M26Y}5UNcG` zhwnD`09fZlUeqdyvz?o=<|>M#B#&zV(U{lgh!lJcbCl8=-3ZIQ_VmwfSI8{{4)vDu z!ksFJQCU=I3US;<)$o->!ttd{eekwojInwQH{95yvb&=GU2f+_{^X>rZ5Kl*%pEm2*3!RAQ62oFmR@1k~R{^`=@XYMvy?HP^@GW1udHi~BoSOBG=zvD);NH=no z2Oy?r1*`E?SqtZyakI(Ua=5!fs~mPucXs@-)5nIPGT?*O9j0k$jH|jzfC=Q5PQF%% zuL7^T4=NwIvKvvuVfzy{14~R(w@P}Jh-C#UbdEt67BrV*hqWkAHl;P;>M=LyFhf=A z3T5kf zMcQRVYfG`F!^{Ta6m8VJ=IshWQ^CBG1v4)st4nqeaIw-!FYjv_aNBxKFj=Ui=0bxm zAUkTf^*UKqIW9vDuSv`P^cg#rqAg3Z)**I4n#I>=QZf$8LV+kTsNx;j{pdKs06>34 zUC%(uClD}vb5so_Q5_oE6vi9ly)JEZMyYxt z%ApjLZT0%)X~-1taf|iRF0Q~B)bB8bSm2c9fr+$WMPx%h(zl3f-5L^ z`8$j=6nr1C8aYIfJ~~$mcL;l5695EVj!~-w&%}DqI+vo4E!(xy}u(*UWe-nA(R{HoXOzd z;z?)uC{lx}o5u7~W})D*0Sa2vr;=w6vHb~ZIhAyIBI^&nK99<&2#OdZ?{u^GD?)2U z2DLmtb_6W3F$e0y#Ah;r@4B96Cc>s2%6aZg%%y=lk>m5)Z08s zeE1g~45@IEs~{o9`bP`QfzCJO{pBkjWK#y$eI8@TZ?fb@?pB}qr^f)s%m1*C;F8{Z zE=-6sk_6bwFDRgne7WOnEu&#khQ`PQ)I*xU#u4R)O^0Aoq#H2ysbefpreCAm!_2)5 zbG%vl5QRk#V)*?KNW9x@o`g$=O(wtMYB~9sHeUbV#G68Yc{JEPcRVGjWV+(HEFRTl zc|(5u@qTK|0eax^rAIQ!zJ>v&-`)g6n~Q84;M#GsYXgC>r4O-!lWp8w0;~Fpy;I~wm)ij>GYZp&!Ol{{5 zz^B*PR*JF0Cuv91CJUrYaZTE(OMwlHJ{7ca6LRr?TKShqvW)A9Nxj6h1G(jNgICNV zxQ=8?9wQoMHFM0q-#YaJ$l(ju)gnKpA2}Mr?=wO(d&z&_!!5x{$p(K#KRWiqGRA2y+DqmR)yU=J)x zPUCtH{_L~Y-9|Y$Z6e#t(4_~h15VikHcfAObQY2ps!58(3Y*k2v=nM?jL}kFB4U;$ z$s98D5JmuLi-Jw~nDBwp^V;Cgx8)J(^!%EJL9{bzUkYmSoGbnTkqcLl)vGWHpw6By z8=`5^+ZMNKXwCry!x!FGa@14&j_N1uF@SJ=P|&a#>gpY7lY$@x3Rb(d53z(atMJ#D zIwMlR4G@YPE%keaCfLtzf@v*)T${_xflf5ZmrqnNy5_4zUj6zK2Kllgh6QxsM+$_j zA$1Th>4e2>?a$hOBx9`W)He33!md4cQtkehJ^7V4X$9&2l7kkUWSC5UQD_WN&oR{c zRp%+r!d&(V{=1zEd_yct`vwl>-!0w$ZRaL&0adDNb_)uKd>oDkgN~Avwi@F~^NHlw z@$99E^t46!rAjKX0mMly8G~RLBjPLY6D7Ccowq-b!9~gV|HKGS%F0S45vGKvM4BH@ zawYIMUl9oO_(Il(c0iFbl?nZp}HG+%~)d*OHtut&IVQ1kEiv*2U^7_DKMzS544&>?cY9kPUp<}SXCL0Yiz z4+r-4?t`fV>(AIr*Aihb8@1V#d5_sLh)GOarmv_mk{<(ei8o9~d42Y@s~hgZBLu7V zopbKrnh#Qq7LK#ck?qJDCpHbp-)hd#0RUx|F_cdtpdqdP_5p+E+_7AP#$E6c>k70y zX1)%q>42{!cG$&b>$H8qVVv0wTt3Xri$#XkkZnYJo9WoW*kj&FQ9v~6Jhz;&dAmMr)yTLl*^&@-3rO9{iKN*V*1jfS$V3 zY&Bhu0}!_1%PE=q`kQ~|_GxmC$Xbfh-oELR3%cSSV6Qaq6)BG)*i2r{a6g z>a16jk)5Kgl^en18d*V^^7sm9s~_RZxZ-b0a5f!q90%(na`kc-XQWogl_)uyS0;rH zezFK6_Xc({6XlyM3{Uc>_H`2ij3%#1-k3_kmz1(i^XeoEHQh7~-nuf}o-}dv{U` zo7F^j5R21K12uz+Bh)6{zA%Z4{`wz)WN}yu(Dxm|Ro@ZJ@qYlxBt(Lx6DHt#c}|fX zvu4~!$Z;Gxl35Tj{MWQaGOYo#$o>d(x~Jqyd^5=>EZ~>^2dFsmfcaOHVh{F=E>5P6 zl1%fq^Ni=T^Yk~F`*cw=qyGc-PFM(=1^R8M0HJ~T7*!%weYBa-SaCQ$pm3Fx*bLM@i>0mr zFVgI;nIVtJSG0JbAE^Q0Lc3+ZNfwzMx#`+Q8L_S$K=gAP*jsP0%K^BUzkz00Zn_n3 z^eq=&1|*{@-BYa%?gq;~UCIT?rj;zY!7;SbiWs(GT{-5Xx{y<|+DEP%w;B0%NdU6N z$F`z>R8L+mbnKy8W!$nYX5EMRuf}+LCt$A7Sg^&ns5s2oOqDtfcS3XI1xPguQEHu`F@umJTY9%gG#G8+f{0?a(d@}h)Fn7_mJ^i-C>=qD6DZqB|g41`WF z#oL>rA-KF~OI=+T5CUdhaVINmjTP8@|BW|*(ZUv7-O)ujL7(okeyTkA6lHY1_nuju znCP_&eiK=XrKXb$+LbwWvp>3ik_`@y&e?^!VOP1d#isI$T>?m;m|zzgep9Y-M8;-9 z|G*rQ4Yih_gU%8_)C^cYN<$f(A?z|Ok?;|o;X1rpgI`K_0T!-4qYoFMP>x{Cy&+NT z-(Z>vFb?@O=hK5v8mbU&^|@?ffss_Rj0*m1xOKsT{)3)bc9#f0PWO=P*0Z$A6`R8%t1!Jt zk+6n#3_LuERMsCmu1H=EBqr`3+<)ikEE1Q-1RFm`FhHI_NyDC>*MEvd;9Mew6ThE8 z=C>Y%<^O*IN%^RN2HWo;mMNWnz6G?JWlLgFXbvH^wM-fy3({ic66&*Nnn^YF%&G z0UKPyHv?Zzk06raR)r>Dp%L|GMR%jTEn4+&hV1o{SVRl&uPZ_Twmc{^9FNyo5M2sD z^GNobMTZjLhLRrr(Ok2S^b>PiN5b5mAB_v<=pmC4&F~oO9;1&5_Eu{Oe!$1nFnV{X zpKQq;t-R{JLuktHC^Rp|O(hi#5l`Llbks;9b%Yzj5b?oDa|i`5Lc<7?NB?c8`e6A= za73B*qBRT(LT%acCkk~oQzC;{R^N+0z}9-CS_~GT++{9q-$poWn`!>LeIOK4@nt)# z8B4rf$L{WDm(#Y@nqJna8_#T9INO4Xb5|?TZvlFd9@V_CdCswkaH@s$_%F>dT^l4}i*_rQrh#~ZwCMZ_&Amx#>E#|NTUTn6-Kp_P-)!~Be zpe;XO98-h2ha^oFFaL&y+)sFmS6D0PU!r*H+c1v(^tf(<%^-h#ezvMFJlx#46b=I`=Xs_#h@Ixn zzwf|R{SMs!*L^@FL=K}TY?wj;+%|%Xg2NTi=!n%UsOnJ>g{49Z!z2^LWf)1laK;N^ zHQmOplV56j$1)(j??A)IhZ_KicT>&zN{zz<)9*|>-ZvQ>_D>frF*zWKbymiR4eT~u z8k8ET4TeT~3^8=Hr?&CY!D29`z(e4@O5>OTYIp~%+0sr5EQ#yU*hv+jfDfw^)tlZ^ z{K8M9wS!n;{hE8~R4^EJ?dm|{bz2UVvPSc7Otcswv4DZZ60bajs(Mf!>895fLb@Ca z;(N6VcD2(lZ_-X*-OJ!#!dQnjGb)C)*mYuJ<}iOn9Eow&J!DG)D<8eW*~0G*Y)q%R zAsyPfL}h;=d=L`jdjA^$XtXO?c;6_n=S$U;Xfy~K48C-q(5Snlyk>|2KYDVpAly|J zP!y#Jd=KP9pCiQdH!=_bqVl962pFK zN*1lfZ$k!g+3B**Ori)2U@K8%)~U21q*1WQ&KSD{5__0=Y-L%Sg1|==$Uiukf%x=R z@*FRGu=hW;i<~i!wKa};QiqQ}YL?5>c!K%tvWfaO+E}Wxi4B_t`h14yV=0li*ScwW zB>~X4U_eh~E;~a25FxkMHt`iP*^`^Rah__OtguPD6)$14A8(#BPJ0;a8o3MizV&x{ z+8XhG8~q*II+H~4iIl#<)Pj4o|C7F4&Nb?Ug?D*|Nfzl%bF}VGyA7nn7^^>IYO(K6 z1Y;lk7$dI03{r|wfly*`$qdr4$sb#4NMtfT|JM&2_WvE@4AYc-Pq1r|CIoLOb5`6` zFtAyCHDgSdU}YlOK%T^Za`OkeQneg6vm~swkOEf3(RtXNR>umg4xqBNO0_Kzofv%2 zMrwYgc^w29xb^4#*0_3`=jp_J=KCx3`12{TV^0u_Lilw(n{(=AGUivXaUQY!Un8W3 zh8I04c~?**F?@pN_B`x15;CCw2|Vb|RnS3QtcWhrRE1no3)HlR&zZMP*~_W%Wk`Ij z-VGo-LcSj^2)C_ESP|`_<(|5ye~^lPSFYKcv)UD?1 z!bG>FfP{VzR`JPgb=i`R{uI9YdrUZ@wgn*Ly}8u`g7xeq&CagG-}QzS`ICKUVI^++DK`|QFCG~dWa7t~rqik~ zc)1>i|EW~*s(~K0`r9p@z~a&fpz4EriDl88K%{I~H%TV{rtkIEmP(CVGRw@&!4GYY zvdDF(nBGbYwWD*|0AC}V`E2xlLi0>4LUBcLsB@}iY%hcvESE_v zB6S}5O*L6wS1cr1sI|-I;lq_o7wMN0Jh_HFSPk{lTU4VHTC3l;4!X!vyB!yji=*f| zw6W81-0iJa>x=F>(S;lU1NaZf4b2Ql&@020b{B$I-*2djm?Qc$%j7D1#N+YiEY7N=6 zg-Hj__8u;m-gB9D>D&*tExnm=4EN%c(#y~_`wJ1r20oj)u+Jnwo&;r`)-Dx|d1BW^ zepd2)4n@=Bm351eHfmn3gGl25tUa4C(cf&HM{+OWMBXpsMb}JOQ{OHta?UQB5fAT) z5vkfWZk9s@VZD)4nkh#l5e+NeS;Hcw6XCx4df0OEVQDJQgPhdSOb#lnL-L znr)Iy%Wn9B-MF6Enw*_)<1}q;Y7j}m^{)>XF&o}XI(#bNj_{nYy?}vVI#&VbAhS7OP~)>l|+=FfLUU|r6A6Z5~)%P zfGshrsm#QkKkaIyb@bc3G(!wbi?ckvG6nLeihVHVYie+lU|%#El@U9sC24grRZPH( zSrrkARKex7&M!8soKx877y;isN@v1Jjn2E~up;W28VB{%bmqE)qy*LBH2pt}E*t!L zvcI=2WsrK89X_1?D-Pg0%*fkh%*kCXqrTMB001zzxX0B=5690%Udmn{10v ztmb;!K!7eQZZr2~Xh%^ajK*u|C-rEWM01@lLNMiK1INLcLL^^cUa0n2^1#ma%2JUL zv-hTu^%b70PRiS?IUwvecy*-$u*$JT!d7m+R34x+cc!#B6%9{=&!E|E5trL&V}wX~ zQo?=Go3A)+8vd|2<`8YophTQZgXEuN;@o}iW!=cI+=nljcQff2Z!$#!YdcvlSU=Whi~5z=+DjHvV#WTkRYdH08jOXlr6u5CkfkQyUS67QDR)F!iK_Zd ztHV;Yl*rqmkqCX(LTK*TYo3iLs5!E$QwkxS5qZBv!R!Rd9Bv>iP;g1KzeF;F5Kb8i6cBs0^5tu?qK zeJ`AQ55B*)J$LLNav#m2;z?M=TC;W~XNf)HlH4=4X9I4MxtpiWBSxhCgHB_6B?h1r zl&|&*?B&35mEw9xTr(X?JGGEMW53k-ti`?$*Ov6H$G)o$drBV^#Z?YE=g!8~NG!$< z+^ga%S^Ykg3Z4F37*H^dm8;uQ7Wm~LaA&IldU%kAfjW)YYKa46= zZSl!F2BE5t0kWo4PbpcY?f~cOu z@y;~=UhASA0eR`=XUkBIk-cwO$Vt<0`MZ3Ckw!YS@(yRXN4{8<9X4A{7gEl3%vn5)UA2KiS(ZEDHqPZQ*DSBV}w6FXHL19ZZ=s3`?HS)<4y4y3E%5 z1T1a{O(lI6@0pna?EbwNe}d!)xxjgjQ%g;@>ks6(oTxw1A}iW=(HfM08`NvsCm5XX z#z?ykcdY+zT#pfK*{~T4m?xlOv41o&MUH zWpBN=7rB8Za;p~EyTT6kwJ_?rdE6N|LLdT)qXX6kNNt{V#;;kthdx8|7aovb>wbf}+~k+CHg$+%??nK|Dc&$^pOB~66g2qG(4&{W z>nuko7Ua=fZcVKPA`3#k0a(L2>cwR;NL$K8pHQX)B1J6oDal<}AZOV&!6SkZJQO`e zpH4+xBJY$AtuZW&%ccD#DUMv6|-)AjGK zamv!pS|h}y@Kz^8RV&^yeneQ<{5L}AK>x%x6>ttf*-<*WzOJ2OwUg$RA#H)GkFp!u z%EcP--$DsKy87ayb{#Cvr_CSB3K(N59Bl%B=EEM9MBDM|#lEjo(Us#5?o%hp#@Fl7YS{;0C(Mb8X;c-@3X*{H?9$R)6CMc&Ya=Sg+vx%=U^N2L?mGhp|DiLr-K z==1Xm;6Z@L36x6%l{U+hKF-w+pCdV1B`GnkX(XoQsg-{sr0zORqH(C(3`*q@|^XBMvc1~z;>I{OEEK!6E;SikI z9{92HVG(P7>pJaInwYao#5-watXp&*+aU4+VQt^s8TI>?vckpx1AAw%<@8}hDcD|_ zFI%+={*<@xHQ}-5C!KU#KDA!OEkB(65GCTN+ue>Zncb8i3SC){!;)%HB#I-C*ybKF zVBd;YmSKZHn+iN}!rLx^Plbphb$-pa=;Fh^#?@%Hj{Ao+3-2 zxFL%OhUEupSZYuKflw-f2qHd71O=*W1+gqP;zEEBWHSlqFUrz`OTYkXkyTWz|0FJX zLv+r0=e&33`|iE7y*KC0%)O)@$EWoL>;1E<+7F-6^Bw|0GOAp9?v>l9wT9XB9?Z{r z^(L-oE#q6C3gTK$<=865UBe^g>Mf&kZG)9+{Ek=CmLp6|f;n@Jl@>@0tj;wXY&{bm zvM8tFKP#%P@#OT4)?37;&55*o8IS0SqxENwZf_N>CH?K1PO;^(JkA8R@dC+C6wyx^ zy}c{;KA~%L-LWWlvN3iH*H27!8PJ@d7TaDZpO3=b>NiTQKKH`g-D}V7EqGs>BOaul zuy|tY&({NR`$n!?jIn#JSdAT{IMeGd{MR{kj%cRr!t+V5$v&ytXsQ=vm-0g`y>^|W zOT=0$!{Cc~dfGmoE4M#oYfb%zwaQr0nv1tgiY*#&5Y}bcsV%}69KeN4;VZg}TN;vV zch72vrih|A8a(AOuJ!$*k>zIEcT)p?tMv{Od3NmGXXVL1)gR`tA;q2L*zGF{j5A;` zCQR62jLc3YZg0@#fz(1TkE?xe=q7@FQ-5Uc+VphgmG0k%Jq!;E$_yQF>`N&Fgra!w zq42@tA5$lK`L0|gP4}{gC9WpbF*NUnO^gVex*7=WyM)99v1c;RZh&~v2H!4Y!0-d%sb2{?K^9D+YiKBHBu7zwLY*IrVZQF zCu`LzCQsrMe^j>h(s}vNqq&C3pxcqz9=YWgY%+_AME1oq^Nz_Cw0rJXpR`-&F(!5} zO8!jDbepwreC%WwB~a{F<2~mO_T;Q-_Ky)9O{Y9-r@C&eI(7w~IOaFE7+9j8S zX9vr3LuQ(4?yL1FIdV$24v-Unfsx|8dl5UUBZ!e7{hP(?boNgRpQ6h}a zQ?o94$+aDiU6TGw2F;ga#&NF666&39|s$F;60t*7dDk$Fc&m#gK z>=^KO2|Q!J!egWko5V7}FLOx(hrbpiH;0{@NX|cP0bboWdea80l(I1Dsv*06|0)8pP(VjW{u&(#-hfL$ z<3-QBXcYiA04^N}Pb5k3WjGa+g@up?)LBSUFqM#!xOlXZg#iepdNkI3z$<`I1WITQ z6Uti4v+mA%$FV*H}!JWOj3n(82=d!i-12W&(9_V?)B!-9F%z%}oQT z6dG+|Ss@Q>5OvE=3DtJ%lt`(0v3zhZCK`{@U z*VS5#YKZqo4YNC!0{h8v62z^}<>-D19rJ=DZcG694Kk)LJQO139QjqEK&}(&_|VpV zDHz=89g*O-wrQgcl5BkhE%Oo#7F!7Ke_VoR2WZF`1#LKkNPAAaSHgpoK-+sD{I={5 zc=2wZ#WIS-J0{9N-zfzrK*(vfvblDAajs(%;?3U(4f$|^I77V|IVR9GdOzp+IN?H1 eqtqZ9_BDg)7Xkk@=(*zhHplpb(zP^;pZ*Ul3H;^& delta 35919 zcmZ6yb8x27vo#uPV%x^Vwr$(C-|)@Eb~3ST8xz~MZDWEv=YHqj^E=Zhu!`{~+$ zbk*)!y;ge}_@8WWL?u~p2sjWB7#NUi3s&g_L?Xoh@mgwy;}Uo^cyTXq?p^L&KtcW| zdG}v3$iD;jU!Q*m;-5LXSTchDUpWbrByj(y;D7y*@Ivz+UXmHqI@bRLhqgnO(f`$R z`d3RV$r*|WS=Gzc&D>te#nH*!#m(B>HECxM8(6Gr?}#dl?2kzuHZr$gZd9GJzU-}b z6zMo;gbN!J0n033AulIrZwzi(<7wuYa9{fvWzTj8{aP5^D3n4Bigq(FeaZJQ$;aFI za&WS812PVIJF&@zEb$uB-)%xr*qKj6&hrd=*8-i6ndg1uTCjY*7Fv+ZP&(ZL};yFly9DZ5D(B_ z_ZK)|KKiM_4O%sw1tRJp^yoF~66S|J|DF@N^$xGA;72bm&1j__2@J!|k(d6E1loA8 zNRX;&*}7_{BqoIitMXMaAC@+8Rh(TIq&v%1%^6w<1DyvStLMpd3bJKW4bU-LW|05N zDZp#%Bn1(VnCnyhp%Q@~tXWc=T~nMB91 z`yZ{=m`aolhW0RJq~LjZMlwi%vFWLoS9NcaawK2L7Em6Tso~X7bg#BSs1{^3^1<0oNRwi@P?tf`W)({kf*;rWAUt1I{~1`R zY6yu~_L*1kSa}q$gIIYK9$HH4q(%@oR1dE>Vr(KL+5;BCr9u0g67P&ZVw5Ejx?ks1 z05%|#tN$y3mdoh?dx%y2R%vn9mfD(&rjC9kWzDsUEOW!X$`U9k`J5-De;w>KFSl00 zns$E2KY?S1Q8wMCFvm&IJJTeqk_Vh`l_=7^=Zyj*rBh-(U-wwPEB2gk!eCyu;7;uo zCC-?%$b5$&#c37lThBTg?JiCLJ$uRkZ&7>R?6Q7I&ndoE9Sj#VQyXERF|@L=tx7-qP|4#fhb!?`dfyVR6Vd!lk$LRdD3RKI z3`3+zo{5>6#caMqgRJhd_aG8cep&havK3EW=RfzARw$M7G74l;)q(#*w6C7i5Ghcr zuHn4AT1K(^{HwtBx9^($yXa-;P{7j^tn3V6?l+lTYq^0M_f?bK23)l>*Kj!pE58wL zsc8LRxOO8sIQWInLA%lJwwJ2CDw!wd>^4a%W)K0xx=L(=Yhq^#9m3e9O1?qkVOKsT zqNtC5sAUgor2`N#)&Ve|YN0%J3;7Vt6!a@d!oP|0)-xX;b~B`tq%NVAWl*`a1bW8Z zCIA9IFs41-_J`x=Ad0ERn;S%+&6y3Mol(9uHW$7F$yN^s)~e zg04l|shz^hXs_!ZcDaM&U#T7MQEmJL@?um@l02#5^28`khTrVMd+HI?Sdq!O?ZHci zhPi6qIu<}Yhdl9O7ckrN>qD;veB?|<(`q4}9O6D0bX3*6q+17Usli@(O@?VzclPQT zlf_z=ci?V@5=s?jvkf$znjQSLyp3g+ddoD6s?qrhGafBb9LthqI%rZYq{X<2g5BoK z**6IE6-VsgRa-#VEyesRT=grI{cDcO?>U92PA6bY(yOA`9hS#|<7J=VDD#j$X{18| zK(9(jgS<|c%_z?5a6_WUUf5HKL{f_pvu1W3eAdc6@wp{wm^Q5jH;1su9VKhN$uQJ? zaZN&pJFx%?=>eVOeWxBgj#!4spRB_}V?iPEf;ACGMnFO)IN5!$x}r5`Xy#^AEuZMv z08<71U9rreCtBiHEy|dmq)Tx068euHJvkNupGAbkpU8%Y4X1GsKbMk9Lw!>BU}XJ} zT*i^ypeSFyp#S%yGl^gzk??PVd4o#IBf?3NAH!{Q)=mNY&v!^oZ5iS}{V1u3{}rc} zCx!lxnTTt-`w21q+#oL zan;@Z^1QwAm-Yo{;lY_{hP=<%`)P6PlXdc)b>nMj@cDN^GQx@kh@2-d{d<@yhBO9f zm=OrZ25LP#R*IX)LRfnXfpnKS14;b{&D<`LEeTw2RJAA_cb7B=Gmp1TE8`3f8k{Y~ za`Yn|Zsz{s5@g0$es2NX4R;3N7KcyKTc(Ixb?*%{W8eT>Z`>TT;=tvtD!xP|qrzEl ziXD#e_CX{BYb<&QG-F1#2w0x`d(0Ks9`2JGolTHuQuFix=T$?$>k3d0m^pd?15~O4 z+yhdBu!hfzt>a6SsjE=nPX2mTKeEyI{_4-YcWumFk(QJVm_B%#w}ZJ2=~CWX2B|)} z^WP?)nVMIF1pIJRe}ysXn;^wb_fp*hb0I*xTs;{raZz)+U!h8}bAMX?E`H z&j+zT@emCAw0mtAJ-njd{mCI-Iq})0nuCHuS>Z%4L{LYb@yd05OjjgLodrLu;E+6= z7(o-_NwbNT=a7zfVxT=s6={I8Tls49)}^y8z#!1BvuB$Yh!A6Y&cW%}(Uf2AzA#*} zl*!3n?bz-nx>eFF7c%wxqVo}$5l(nfW!;QwQRND`@cIs*w?1RGjUtp5zPC;HO#oPy zCKzSR;VZf=yHP$m4VIlrpKfhQ6qKwH4$1(zw$Y*npO_ZxS|j8U&Ybt=t9A9=$BIzw z<6w^39e!{ePWKfoL~kpRr?7eA*iMEkW5MB!9rEy(Y&0yjDO>UFfyDzKs@Y6Snj_B= z=i}xs4%oTdSIu?Px}NUp`q=8%qH!3?tvVA{i%gaHkde+37`*$FZ`?{(&RLS=SoOK1 z97$26h+rt-w^(o|i-z1xt?{MoQ+DMamr`0^hi}IAQ zFVQS~WO_KMNafJ*@0o!E6>DAfu&R+RrA;1ZQK>&|<;Bw}CB>dXvxGxxMLi!z9VnY= zFu`xa5exg7WEE^4y$ZvHki{Xanb5Rx!sPPS8|+v+kMK$$?V=sDQ`KHsZ&9H{%VDar z$-Xz|_teHW1IHH3A6e=n4AJm6jFV=fNJT8T7m_5XQ<$oIGTeaylum0#9=Ou78XhoE zkqvwqCR;Q=+z|L9l|$8iugYT)h5fgmmtcCH1pu)j1j6EKY)@@Ar5TzKf^LMPCBuL7+R+bPXSdy7K36Q)8EWpm?Tr`aMR2J;ky%q2X zLH)MUsb?i7^|^sr8?@S;6AAnT(NFpkR+cKhjZJP%O;k%z&J|(=jDJGCM`g}XHWnbq z0JJD9c7Qlm@6MQb^WZBK;NUNGwS~>S_{s`RMe<`3iiOiq(Qrg+N&8Y^~Q(~cAw z>_UB}#F8lK5-NnsdaS&L!AQJlu|+vb$kFhsgvAz^2{Rxj*{NrpaWE&Y_o{S)G#$R8 zwl?v4)yg26@>ato>^6)F?x|ZV8UG2kyC~_;F%S3gXs+bC{JG>gG_yhM(1)kBUEKu8 zU)=>XP^l4WoYA?w6uabs&3DwN(Vhb52XkeFqRHi2D2i1R-b->0?53+FdCjFwc6jK~ zRJv~3+Q>i-($J!X;HY9IW$BX3EAadM38M2r6Ik%soQ5s-+W`@3L5$A%;C)v;Q<@c& zH;GA^M{_^p4IEk9_#|9&u|#zIgeg!|E*#Pe*Y}X-sd+pOGgGDD8XE0Jt}qu1K%IwMfk#tX#M zwd>)m$a`-rsYay?anUMFIJ%c9mVt5izZ1(`I3iKOv*Yz>csewhNih&LwluUYDV!kgFFkhIq?gP; zJO%<~dh8gM6?L=O^4a!V`S4p4x2_G@u|IQPyNrX9FZgy8M zQ$_!3825SOfbh5u&T@i8@fP0OQ11!qEw~42zA2=<-06@0yfzQ&_pJP2|LPcjx%E%E zIdYV8%^iPzmz=DNi$(h_UOp=(zONrDAB+m_7bYeqy`Vmau5zEGsH( zb7mp8VCt8a`o|ydH@-$SdkLrERHQ9|)M3V4lpoK0DZ}LSMWsYK^B~ag@6oBoTc$5i z2>v;Kd}setj#q(NEW;C(eqZM?NjmR-;KM%Unk4O%7$|upX@qgzK*zvpR;tgio!>q- zn?E4Dw;*dmQP1&b(Op69W})LzKXlmvn|W%!aI3-6mcL|4T_3ek_EAi&vBT|0O#YN7 z!=Dq|tq3km9Y0+9Evhd3I`!f{jch_-*m}ldbw+xyGim`DK3Xb8Moxg$Suj-e>gD`I zbF3I&#aTv*6vdE%0)je2q;>mHNoYfb2Ay>3AWOP(QlwI7)^*j8=-GYEvIqL8mAgj_ zhmFvYBxFonxo_Z%?5+_}ua zL!nL_S?K&fCpdPqR29yleLq$GPr^D1oEJ%Qj%0sVe9+&Z&`PQc4A0Pf=%@5j8PO9G z*jh*GR2qXCr@JUz5LTQt#R3X6l1_aL_OR5x1O$^K)|Er>RcFmBJ%-i%6AV$y4N{L& zj4{VCy=i;LeUwzg1<6wlCgkvd26g2h;OQKH4{SL~?%x5ls4b%WUZ{}ux{l%4-YQNd zW9?cds0xyU=>!~5K96Y>b7}qfeTFf`=;@UkT%a$GK%6s{Y`n z{0bCh_nGjyy+eYMIOjTFWsfA^B73dsU&VO`njZ?o^M&JVDL@3G2S*4XFRLVv>1W-$ zy|;tRGy$q$7oCHbVzJg(azlyZSRaHX=@HZSLiMCEFoTUs@3We`Y77c{^jTu z{-`0jyjWiTJ_{JB2|bJiEi7MtKLmV-vb{A!&u*l&t#dr9G8>Z zZBouxU*oXwsli7!61;~Ykj#=4Gw1L_Qp_+cJN3$SlKOl@vU5Z|my~ntu!P0h02a-A zw?TP$v}40k?-zO&zA(jrg2K|Y@p{Bp$Fmt0YK85iR)ZTo4243!D*n5f0C^ zg7aA@cgGPOsirJYqSo4H^^U9Oa#535lNHSB?{T5sV)g_V>sma`DsdoTDC-Q@TMCwz zT%f{}H$9;!4VH6qhQ_%ItkH>Vo`A6~%SybcCsC55`}lBCCT5GlUo%&k+p*1h2t|NP z3Yn*#k$ISj0zAfvS>>K)^upH6OTkide*U9;!s9yye&!YMA}_^MXFsY8F-y)}I!SI%hKpcorNM7-@6u zXZPZsJgxP>J`&E@j;ivp=Dxb9GPFJO3Ot7L05WH08-FC0PT^8jq7)Qm0mw6%j>xo{ zVNF8647eje%!>%O?S^7j;uTfwBtW5W6Yq?&$>-LcfEsX@r70i;c@%_hMTgIsZa_%< z#1z9=%Oxk{b!2>>qDfE!8)k=?E9!&$Hak5e6H@}5$B8#4oGL->%9iFdE!xbNZk??b zd(Ipke9n|gvFwe|BYgDF=?NfAIEI0g>T^{W7nIivvrv(JOJW0MOtS43b`-=|KAtd@ zsYd!d2s6S3R2?t<*!E zs7n$#h!}#9aiS{-iuAb$x#{Q*flOK5(LX5Uh|Rp(K16kS{k67GXX?$h`1ye8b;JGj zd)%m}6h>y6$LS+z+&@GA2g6AdLcB&LAoUFx^&z}7_t&b=jEC?M;|x6)Z@B1vOa{+Ly($YqojX)`G9#%})=TiDN% zw+VM_OT46DDho($r4XdXg+6Ls$f*yRB~^NJr4I^sTn z1#Zh#oU@-ysH9Mpgn7^;Lt^XVJ4N73wj>m1BvrS862U#43_#<~C}`zr5*^9efWvXW(8iZVkdECwiIOa}(NqSXS;rWUWc z`nG0<-iqcatj9>&=C`;7)V>9k8x$Q5^hQ_vQayywm02z>6IxQ{a4o(c1-W5BY&(#9 z#}?B+VFD>o#Qp*nIWc8j;~I_3l@?_D<|p`GXdfqQj*1)3iIt1Q_=nU_XnYjQ=TO$% zb-t+(FKd6TmND)_u_eA$o$QL$uOC7$MhhT7%qZ)&F!z*8B4~`A$_Q`T{k&u}-7|c7zEjr&(YWQk!tz1e- z@4A1jFEkD@q+zg{@$;+^H|zlfCtU`cV5=!dEIZ7J?eLfefxks(yJ7A=kX^$0a)l6` z|9HS+6mB71?EHw@3X?wp`uhx`?D`Mk@fo^pRADdI10<%NDlIaQv+b0{jr+} z2mYefJsN)?fW?l040?ie?HtJZZXh^46XW(OGWMUZ82|N(Fn}82XNxibMgQDQ&C@(% z2ehYp%p;wK-8Pt|tei=n^4j%&u7*B`Y0m&*$|6rnXYn3Y@z%wGr(Of09yxjz#hlaj zuc)iJ8h|sDj!WnHzvbQS1x$;IdtK1fJ3h0nOnBZQ9C3wzW|%Xen&hEv#Dc65L)aWs zC}(v&w2sC?yfS8j;#KO}yZU3pZ1|$xbJQDY>tVAD90n>~mPvPQ1G*Z18R_2RZ(b#6 z^(}^oig4Y1lfpj&QMJ0GjR268J{UBY?BpN6tx*=y^HT%#VJ*7ldxKZekO>oK`L$dr zKj9FO6MkDxHD~$TXVNJ;{9WkW7T+ZouJerNG3&i>i(jHRTT6bl+dpy%CGp^Kin}Z{ znJZEeD3~qIH4!?)3qpq;?A!>AL72q=PZk12ew5KCQs1?3JWPm~rpJY~k!8{j4!e?Y^$iQjJe6LlTox+{|7T&t4FT_fUp_{37 zXGXvHZLVivV8tH?)bm0l;U#dK=YRns5+BOe7UK`a-QB}w-B~JRn)FL?`8gHk5S;@1 z^cRN=C2ey}RDS5#Zycjjar9CNOlP3*Uq_|qT59tzw4==(;&Y(+FOULl6xqCGP~AaP z>gS&B)~R|=J0v3&xo@+Rg{ttRkH?w3>+scZZbCj_$~~N&)mOgCpe0(CG3#O$%_F0A zDBP#*IwhP~@)TQVYrw1F+0FuqN>J(K`0v3(1s!)vuA=%^!Muypp&nJ;4^He+ZM{qd zFoDhwC&Md23 z;juOOsn5E`#>~HGHv^uox7^c*=hJ+i97H$Moi3ZORZOlqnar!%&#=+LUi5qqt5G~c z_YQC}Y?#3p?V>^@c!QZtj-K%ot>*2l<#A9~{ zln?ihsc@IR1Og$_E-a3J)^CzMp!BwvB*+c?zCI`)#ep>+Ss~%fxK`i;XqWCVOI9t_ z*vsv$0pUM9e`jU@I_+;CvB2O1tnGnNR^V6j7FoJq)CDuBYj}m(TsXs`64<&>2+y=9r@HbzYcKFX zc3fTS>7=kWXW7_DKJrn_i~SOokkjwKAJxzm)@bCB2=#sgyO-#bOQ|iHOpt{-1b^i? zDkv|v2DOe(!{LbsGB3LIr$7nw%SKd|vr-6-o^9LJ7_DtUg9FmR_=~k221aSu%iBy~ z>qoznkU-^M6%?|fv-I>N45YgCG}5D~YA2ZJ8x*IWDlZdkbTVihMoTkP4=oGB5LrZ8 zNhq4OrSpSR&*^2QA)fm0Zo-ym>6+0&%XM zQOV1cOCrY_aqB=$DbpM!VO3uJyhH_6?r7=T+Rg0{^OXm$L^@tND)IYuiq?h- z~>uSu}04&!F+hNAy*huj%@(b{_{zuGi^ zq6UC(m+ev;)b3(?WAF_g$C}9XA>E43CwZjU{LWZG|A65{KNR}fGLy$ZgK@LEDL9+UfMN7m9NDE(b zz@<0ApmUbqYR9Ez3`d8bal|gxqDME@_FC-j?6$)Bk5ujiB|2(%6!=-J?k>PtIKON3 zaCIkcOMAYz@eo$N1_cW4D}sJ#_Ze~#&13)At%dsv?S;NYg8UxrC3i!_Hcq(MYwTbr z@Ye!nHsA2)E~va0)f5kSFE3>3;)u}HFWalSaM+fg=;d3}vmXktTuaQXlo#xm-z-T0 zs?gehV;yRVL#-E(3*YQT+jzBtcrG-lE_Y_Kd9gF8^wJxCI#h+L324(VX+3o~i75L? z@Ts#OcL!IJa6Ms8tasX0n90&8RDIZMvt$Et7hGF-xNj)!ViB&L>`1*&YIykUaV_7_ z#KrP+j1tSOWAOf}_!FL*65FnIS3e>seZkn$Mo?Y(%FcmyNc5VAKN={1nh!aWU}1aV z-1(jjsRNi`SW}o+7g9HR6Ary1gBk8VNfXZ;1R*iKj+daix78Zx9AR@e_-OZJ|#qsb^q!AVp#G+#x%)IDPl@c!2>Au9B>2GLwmwc&Du9 zrZf1*&-IQXQSy_*>~<>@gy7cb+f>7$NUIN8k-1fMw&Bt6s5ay%!-4TrjD7Suwm5YY z9G$5JzGd_|zBo)D$Lj7C+UMmppO;mi7Zs!a2Q5cOhdNG9xIPXa$E5C7?MtA2dvGe~ z-lxXKUu>Tp1F^?<^9I4_lUqL0*d2W>g#(X!3$oAmjbACN(sNtlDn6&QFQ_L19oK#U z)@M^FXeUDGvkzktuHdLPBF4F9DzQUfvD$u^*eI##J&5dITFxKj_4|Wm`GVfn)`z;X z3+=cW8`MJe0+Bi&l%){G0~kQI>B4M}7G<5uLerlbJ=O8y{+ypK{9s?`t3o`hk?I_& zg&T1`&Zv$gTaFatk`FJKOnt*m@dcopf5R?4)lf0`(|XFMj& zgR&)#yJS>(Ca0>8eVFf;d1p=6k6mEac_!<1*9v0hjtIQ(0AcSzX0Cv4yzI$y2v($h zl=sEuMJ*vVAP^dgbN0mSB#}mFQ`Lt*GLbPYs_D$nc~Gp+E8KEvDPfLMwQ1f==V`U(v4d%N2+58JiVB0(Y{`bYT&z%BI6P8xsbK*j_*#rozJC;A7Q+);bo zVlOLUw(P_13!X)q>W-zhBr@2?GIa3-HIl|M1iuD^FWb?0!`cMVFVS{PefpPfM8CHI z9Gf>QxcIIjt!$+|D+_f=eYC&qzajtkDAXEZbW!k+Btwh+&;FaxW$9vUW@nz{4+WcK zL`n|ii${G|}#`P(y+^zHS^43asBiNWjC8aV}s z&@-U@{hNlXus|v>>edCa_Vysfzf0!Ypn4z3-Zm2F^9iU9Tq!S$*TMjF^K~f>;oV9= zI;tpN1?*IrP;HZX^{D@JOq$qG-;DF51_<@>egb|&bn6bmbxHR5s8q`N=uxJ@li+uf<&H^AA3;br+z7%W1el`nahq7WZ>;wG}-d_-_xME~xTL2UZ2%DAVK9 zGPP&YT57kou%b#Q5`xlbsdbi!-sK|=ZD898gTP7easUH*_S#TFITCkT+4 zh>cX!XG{rbPE5IDDI|eC)MjAK>2c>jZ&+}>m`nGC*e8Oq>qT9#Mel1MwC-9Uj5x7Y z&V~F;3^m-f?x5Ytj>ZLktMe4$EevjQ4HwPc%6`c7)v|v;^-@eb|0*~*D-+JOe-o_9 z3*Fg!!Tkylq+t4RORtg9H~esO^N71?Y3%KMd162i-#HO$Jc)h62~$o}!; z8&m6euSY@qw<)s$SKNTdgBhvCIaw9x^q4fl%+S?*)OY$-i*+Q>uY-L#Lk7oLW3g!P zFC$h5NHgK6W@AY)y@Q$WJHr--226cGd1@vh+e*G&22E7rm5BH&n=*MMdp=V^zv7Y@ ztb-*^d+MG-4F>fT1xf{ViW~04kgrV_={TG6Pg->4@(6L#yl`51wWy&;t%3;L`ecR7 zgP!>v*0=fJ3R-i(wgmoU8wiYgIW#P3KR-$963Wyl3^3mHN3s4U5i+PP6id=zQfB@X z+)_(X$tkl@bzf2qiNX0nnR*!+cx#hpbS(G{ zPA}8>_VGsFCJDXC+2=G8Jc`4D;F3xRbtx(d>f-gvB3lz+wN`1wQ-Q`V%e|AWJhd@a zrjpHOxLBM-Oo9@dGvSwfWZ8lke~qcvGc{whUS>MwmK~D(-@%j>@)&z1u|i6Y@R+XH zK)ZpAHi%X_lcX-9)_K?y2FgV5MmwbXG%pt>XVbuyJkJj6s_M#I#RCQoeLuz*e57Ne zIbMCq6KYvt_s@rUX8xrNeEE@>V0V(G2Hzt&CpMl8g?p1~I_l1l;n7r0Gev@j=3!U* zQIf_eLfyGC<0Fru;st_ACncp#Zi(G4xHl3BYGyjlXpPHbm>CrQ#Fsk5sWs{x* zyU8z`Xx?y!X*8}4u&!x@8csZxe(P`SPMRoMfYk+>P1fxY4&OBq7WQJEB=Rye6@L+< z0FDlL4gevNOo(2y07#d5=xV>PONfTyEGOU-2K>>Jf!hhrqTlpvOVjj&o(x?!O?T~^AOaA~!Ql9zwR;YrGHpcDYn7WslWV}= z&OmBQLj^bzA|}0BDsp&kEpTxJG-RZ8>acdqjXx=EzbcQ3g+@@mQi%gmOcJ!R^4nDU z33CSnYlfxiFOT;x(naQOAtE#Vgbks_f92JlD!+zdLCL&_`GGbI2F=4a;W(v3{Fc4_ zMWYtq(9lk;MQ)7+>4$&=kr73uyi^5*zz~TlRgzC0!l56_qZ~~tp8CoI9*_sRG*-2n z4Uqq|(Fn2`Ku<-2mMy6Zv^6>+*N`V>3t1Wv{)SkV+uf(Bq9}F8smiVQK?2iTnVY(> zwr1E;kHcj%g2b)Iz^lWPUsWvC50ZbJ!o&{F={Ijf*l>c*xFfY2!O@FwwQ2_XabCZM z0VJ<>`TH#ous^i}Va47k!0(ZX;!U-rxRG`x2E&_)DEFmiPdIPmq<;<_I>C%21dC`_ z;|q-JmNr%UW)+WJ!9Vf*N{Sw#(Tx@2?0xfg9l$54h5V>4rYcq9-9#?S6#67T-NwaQ zOasqwmg2?$|F;)K>@X{}Uql6lVK@|^*7e4&NtEAHT*7rNPF1Qj*t&h?Oik2GXI;`Z ze}Xl%QA`vp4OP}-OdElJ+rzU#_T%WW{E~mW>~LLOGF!V#xo+r;t4Z>{DepgqZ!xQ5 z50YhVv%C4EzR2ir*&Bw{bB<-n0B69l>&23#U~($HQNF}BSe+G&sH6nmM!VLqRHL=M zS20v3_Ox#aD$^ou z)2Cw|?b8f;0+k(&*RPHHugx1|@bIYf6H>Q@H_T&fh8}Kx`!!MC!6@sAEok^}MR+A> zHqX|(XB70`>q(Rn`+9touE^;)|FWJY3FL}>%N=H4Hc^wrn9GTLr^9Pm8{&;^_3Qt9 zvH$sUPXixsjYA=4#3UH!Futd9?Sj%}gL*Oaf1`YB(Ed6&JbBwT*J=IWWUWF%2+@c@l}lLiE9oRy1Gq`+ z;h=PCg=s!Z^B!s+1>EA8sQ(~`90J`p$^7w~hx!d(=5a87*OV)<#I#tHXuT~v!Y}OPaG0tiFQ4=tlw)UsQ z1h1(#TxOk}y>c7O-Tmkq_kL=Vbd#HCL#jOk5ICL!R0nkneDH3lgMHvaPcB}I%YY$< ztUA>uZXF7IpOk@~?p?vpzkoHAxx5wVOP^?G(&-P@gK*xMz85{!VjGCF4@ZjKg{V*ErAwaVCb=O3`uMl z`F$b}1lE$&f$fWN5wcKDNvuy3S?q7@`JXFraI<5US9I+Tmfb+nhFl~~DO3+1lupqi zF3M>Bnn2jJ!2r*qi*P;BnroKrXIeBve1~o^dEMprnB<4nSDzR6B?+u8 zeXv9@F1z?d;|-BKVC!2c6qomPsT+A1!pYU#l|*fJfo?&c=is}y9(@c*)Dq2zZjjFc zuX|yvo+jJV5OjXjX@fm}?l`7;YqsjU;2y#Mh#e$(C@ZUrMH(X%5S_QOcl_OnV6>rY z?=_&VW8jjK`l45SBtClL7XIzGI=r^Awh@IXwz{3BBM6?-;RVG|evK2X0&5SYLyM9l z%#~u5Eo^%ZPf9e`0Le?O%S{yR3temgJlhF(EFik)+Lsrh^LD&LDWO=@JwfJAe#d4o zr{@h@&1bdxA>oZPaX?2suH_XPhhZaXUX?MN@nk+&TAu+&F}^!s-q`lx#>D!COS(>D zm48LBkft^Njp6zg)-j>QPby&PY`(LqOEcwvA-iJvsr#6%;htCSDAHxe}N4ep9O6%XUhC z02ML(y{xTao}ZZsR>ttF5Ws>Riue$wFzT1<5+qpGmLzj%8{~){2rmd zI#-p)o`H$AaTm}SDHOTjM7=G}klh;*8{4-t*Z)PiP?>eJC&~W>TJO++TNxuKRMmXb z*Bh2JtP; zVfB@q8+g~~wRErl zX$69jqK?T-W`bx5`z6hN<%UPp*$&WXGj63ZkPcK$-WK2pmG5wTHP~`h@4S44hp_G_ z=J;?6u13&(wTCos+oEe}Uo#@7UcEuKXT&-K9m@bRpKC*)mRAS+%^b+GZ3-=Lbd{}}zv{+mE%hu{F z<4&A`z0=o-y)ItIGzZG^&Qykc=kLWs0@c6!cVva03W}Qy!jaNi{5I*am6_MkIa7ED zq%yFT5+x(U&+-HjBZyapR=SPBM$0m#2s&$A4r^z>nyO2I4a0v5s!vXxjW%`h6K4m# zG`J?Uj)QxQq+sHRmg3I9wYTJu2K9MOOWGOqn(c?{2t3@JZHFh@e8{84Y7Y){Tvuc| zEXZFatI@YOV=U!NNqd=WSD_!*s<7fy78_YRvQZuqXK$u0(?;Z<@Nb$ir>crev0o6R zC$c7r9$M;wEPdOsxCQ-jgd-YP){d6 z?ED=R0xo#@`r{`*`jbie9mm%+bi_DqjzW}Gj(_R^o9}CaTU_{(yf}Y*rhAJT=D7>H z4gywI02C8)WHf>Ptd86r!v{e9wdUGxVS2)}P#5om5tE4Uci#;y?u`69|{8)9|i+NVoXGTh|>Q8qjw{-XWAuXhI84vLRToNXa@MTDQkj9 zT`J7z2*~<-wbJ;&29YXXY>aQ2#yD|7`5O`#^sGb z+)8@K4np1*#?t8r-IR_Bt4Megp7~JS{W(}z-9mR-({9^E<<@Nt(pxxn-8Y`v*Q&DE zy=(U4N8eN|cT@Gg%C(cV*^0LjABz4Z`)iu%%-@b)#iNi|*RtkzBXreasR+q3MZp2K z^(|^N8a8^ekLx;k5zDpODYG)~U1M^<0&N-{RE{bqW!2gB^V+&JW=_@NPcO`SxSt#6 zyDHy!%cBtL>XNV*6)bAH94m_`jhGt-1V~~?ac)C4)UHN7ZB0Dst&m_759>0z$@3rQ z(P))=)92^ixpT{~g)(yA+{}d<{1R|RUneIy@W`r3KOM_82(wqN+N7$|*=(eM{Fhgl zMD7xeErgr!mUGi>hnbU2gC?W_P(2RRiWUqbW!>5a@zEyJ4Oni3#-7<^_=JYq*ymfU zo1-rW9z9#z5g;5!oKW0^`<=64y4=0hHa*bBcYb?Z^a9KVg6B`5*K1LJb|YM_?JG0> zelMs+*Ex;1?k_pwqMke2uQYJLth=hMJ`eCfJ5J{pi{LuQ7^@Lni6%sXwj{bOao+H~ zpEuPV+YgLhWGKI8@FK8rxsFF_(EG4YOjg#NZt)K%thzubNEk@m5N(Eh$UH%_eO(-j zSISP5>CRL5WZIz7HW};`Tw7On7IptrfHehe@psE=bJW`m_K zU-a1ay3-0QvP=ajpt*7kJSab-X!NI2D(8dQuup07oS6Wi@)19*6lQHFQ2Wsmj6iLn z#3fb`$5DW7?4($;n0Hup4otHR)o%qXWx*ch2|{8tMt&xiqc7x?GbAd1LaAoo`8Yxy zsf`vO?G%Dr>GMBTGS_i|-da1wE#m1SxU_i> zkHDN~jaJM$2Gllq3RG(1bx)ivKS;OiVfi4PPj~Gq$tGD#z#@pkeF(^x{^Pp|DG@Q= zAW%+#g*Cv_7{kMa%p(EqhEk@pon>?mx!nmbVP_zCI~Kw#KM4)^l7N8N{z|;|3Zwc) zZOkQY!6)v(C(4#f0m7(sz-L~o;t|rYLhTdO&E$2i3qit~5`pO*MMnV-OIHjYlK^z!I5N`R828Pm+%HWI*`t zCTr14<<+gEOML?YZJBC4RGR}Hq?;2SvqaZu`ox=z9Yz2lH<#B`iz9P8MK+~MlAPO@ zL9|~|#5ph$fp@|92?wynoHl}V&+3t;9{ z{Wc)N?BbFWVUUT6o4oJ zqAVh8z{bnIGNykjtzU4*+mAe`Fht{m2!&huVhvjyR-Ym{4yY@=Q9Jg`!82M8J61dk z7hF`a$?39rBHPvXN`%~jdGT)Zo8s!$*`WoT?;axd@h@JY|GD~q-8ISAoZ`P_b^qly zR+|6%|73OlZ+O*x|G%fE|8iACUGWRy0RntUa~|YLIZ*^aeZ>VM^pWowj*jXJ zvpnJ+7OdoMV$<+|0Lnl{W(BrBqd-lhs&w`()p^Vv#;&_zyiZ~SsRFXQex%(nmYh-s zS3KF_RLU!E#hJJNhpTsr&IDSQMPu8x(XnkOosQYDZU3=t+qP}nwrzBhlfCb_Z`A+X4Xn0;OpxhJ~t2@o#{|zP%$)tm3*m!1|>DwRE&B`lnHl;4Pt2B8D2(| zX@v1{H6h1t12A_5pVGk{@GJn+wV$q1VXsJyEBx|F z5iOWZJswUB_kuAX+)OBFlR#W=qC%A;WZv4Xwy}Iw;NM!DpJmItD}^nZ&J_4 z5P9BYP*V(>LcBGFZT{{%M6716E+0S!PRvEjjZJdJZ%*t%aeoPv=e0-K|4|q`if2SX zG2@D*cav9-$SGx7(9IXn zlL46)30J@-O8fS+!v_s2<4(M>ZdD-ob%o1rHwHvWo>Yi5hRI0aYYPEAOrGix*(@T! zafD?nco2oR(2ACe!OJJ87(MgVqL6fb32Sg!5whSVLYM*~r%&R$Njy=zQwv)A*=(fC zA*`~xSv#S7P~-%jpev>^!y>^3iY;o!Np6PxhDxgsc8{srFE-EeC`$8;_}`bmvr>-v ze`2Qr33_RAfOcbj>%98{y(WLnyx&IDM3T~>g<;Tj$f$5(+r?MKa2hUS*U7Iny<-`W zo_8Q&6vGXG#Jee%Dq%|tPN(gRmu~l)4cqO^Ahf&5UZi7qEQ`6iT*EovVDhF^ z?qrp8E5(uUXq8|;`m<5J4C3S*0lnFAn=J#ZGRYASz!Gn@<~FWk6E(A@y=6Z?QnvC3 zRVjGLys6|(7TCP=GpGW0RK2Gc2`_Rhfr!D1Jzb3z>`n_*i-vty@Y=2DR`X;!y<9&5 zY)N%`7e=IlU%iD;8BadW-OGfUrzlqvanTAEQu4*e+faOa;N~|FcJQ20 zT-|VoWlMYn3*JcO2L)ghu_lr_CsVcd1Lp020G0F@18xjWSOm{K(CJtk4mf@Kufb+G zChu8&%zVND;`y_B~r9K0DPHU|>9%U0(H+2hpPW(bJxfF5BQB|Jf!h7LMm>6(OK>s2JV-H8(*L;(2IZ zhEI|m71~oM8)@Z`cZ>f{^7)H#_5AIL#agdKBc4h=WGX%}LzR(@O>s}Io;N<>DVb#p z!U<-$OfUMLi5KCnAr;n|z|e>VjZhf40QZoOC}NRlOAlBf_U%9X1^@djZ&*bFqy1lm z5x_M29)bKp7}Ny)G|B{vX*|G|x}_W15xP&0QGx~qXmAjapK3{ZtdS!kaKvmQaeiU2 zAg*j=UNNqHb;e({jGT_p|#uulM^- zP8FZ2iAzJqR&AKT$4SOR*3D+OIqytRRJjXeiu|jJ>j33J31W^cck0t?tLLHzDiPBD>$yzu`&dr84Z{#TV&m zY&GD`{FAf0A8Rr4!rjsDzX`tM8{YiucXI&Q`R{js%f!dbuXBi&FA%UTHr##qz6;?E9&!(1Q7vAcP3V}sD`)6~R&V$`qTLw`0nw;bv(qv!7c7zl5 zW1qA?vl{YcGO(~2r|4sYjdp$i2sDXqgm}3&u)i+dTWr+%I_?IS?OI@jZ;{y*_O`z_ z7SC%viQw^YpSHq6gL+c8zh6d|ozKDHDcpc;6oKv+eXmSS##*!rcmZY}+(uD%O)!^B z?N<4b4|F$u`vH}QT5HT5S88C4UCh(W8yoIo7 zq!=2{taa;;#P~;%pkrVKcXz%K&onaIw`BI>Nv%5@9{}@x6BxSO?rY=Tde8<`dZ4P~ z(Ptc3na3CRW}2L4FwePaeekVud6ZzwlOgpR%}oWHrmv?fFfMr#eX3S!cK9w-t6?$J zr@}4%LU7QmzigDa6I=+x&V*iDz(%H^xz#>ZtCp9zoh>%!@s}F4mzJ&Dn z^0b@*J2b7}9JFp2A~4eilKqLj&H2A}EIAR3lXAG6j&cFY=z|SY3uH|@??k~gseN3~ z4KUM&@_mkO;{6XWy1fxUF2%mn^$h7&h$Rd0c0hsrX>+&H8^hiU%2)1g%w>SUo^bTd z=e^=BbI0e5n^>P%Crzyn@lbVQT<8V+(<*2|d-*Qgzo3*AaQ4C_>}Z^ogk}QLoWUg% z5)aTCNjIfFExLc!|5W;Sz!nXX%6Cz=kz1cZpDp~N$(y6`ikHCcrY6pXK;$OOi5`nF z835jX?fYHKW~__r=JX|2SPP{KwhXu8_?KGh&W~2;ZXV?@UAA<*c^RbFr6m<9x+$C3 z!LypY-M&0k!Ma&ceBxVSfoyT^F{gFOaHbrXZ^oztJte(;@Ew2kVru9@l2 z)9e;&d@{>P*g!CD4tquSy7)M-W9DZikAP;7Xoj!GqbTqQzFy_-XnV}F%}!Pb=4bL>O9x z4Z|#THC>q!^JN@RNj~u|^9tJzqHcvmRQ*apFGYR*AM;jw-sac2Z#N$PGPND`@ge9j z4S}YZ+;3!81ftaXzQtw-Xwvxx5P+gNuE1c^-|Rv6W(#PAW)EonbLX>Ys-hLpv^jTV z+oBgqv1Wt3z#6>aG=sfoZa69lr6%4D>XeLONsj6Qlha#v+-grW-Q9ZS55I>Tv$=;b(Mo?JNf6T9 zyhXWYuhQ9*qtvFIZqyM1nSi|>a_d4jUpy&!4CBjDsYPRrQ_rgQ+!fW^FFf{lO^&BG zE5iIaXB@(3-8=M$Ma9Xn}G#G zZa{9l<0@|mxAe%nvJJXzZ8O+&3+>7|zD{1h08AFo2!R}Zqy(i*1!*{|p)ArKY2&ne? zg!tVX9m#rZ^mwm^cuY<=id08s2t9$KE3FkKUhjf8LL8+v25QiR|*;kdP(#XX^5lxXQ6KKOfV zM-b61-7x9B@qVLsTQ4K>&8$DeybdnJ3r;DXIe;?V!a`N2$|G8TDZ(#`NU4qeV7o6) z_aQtB#|P%8j>Nfqc9Qf1`hX5 zW1Y;R&4MEyxOEh)KJqL%t^IU z0)wGnw`~_IulJ)f8u4v%!4}Y!vZi@;EFhCVul4Zd2rx`0s@W{CzL<7A z#t_A}N2r+3gvwoLsm%wjY=cOzl2d~NvamF3?yZJfC&Dq|PDzCs?R?YZY!<4LIBhjN zMBgesb6F&=ha#!?K)6?|ph3$<82xE=r|Ej&T(MNLe!Ml*(T`X2$=3@yf>~}McUdLs z`ouNNMv?@QC~1Lu#do1H^`Xj^he=xv^QYo6oC8(DJJn4 z(~JiP0!o8Vn8_nb_^?F;I4-CneI;T|)QO?M4ek*T`5K~bVt^zT{)TE1l=Ll>63G>- zQ;|@P)7Chi|LFRJ?R=~{Dne{j>MxvhdLI-qeW!dES@)4pHCn5eDNYER&R)vivUTm@ zyC(#EzvKF*+*#xCFp?W`N3+X1>+x)#_QIJ2Jr=u7lG}kkKyMBL=kB&c?x`jE=W?U&IIWyGb1JyDD*4o{Ocl4sd`B)Te& zX`*coi&;(8g+~%~Syf65EK#19d+^I_;Fsu*i}AELA_gekg%z(1jHOtz&LGJN){$y) z*cKhqC^B%INwMt#%!;$)?pm&Wl9It&N*c4$&BiJ?L51Jpaw4uUADRAI3h(FW0 z3Gs5+i%xdc{xey*@A7w2o!gTzExW@bnh)kLIEq2Lt9jk5a~Gnv$yjZ24$i%&aK}t3 z@`BDeAfyN;EM#Rx(HCpUKi81kiWUlAMTUH;uupHn+?>xwQG_MJzpt_{HSp?e?z=vk zOI3xO1Oo<-7{M{Nsvub(DIvEIc49fXvIEEE;h`g7=bb9v-&n7p)v};l4ryxVjrvi0px7nNO&(IIe*< zRI1g%4jdnazyLR>b?_l9Y?2`qu3FH1;IaVl^&Ymp9(Uk~XyA2S_>I;+8$obfpP?Qc zcEnr6q}eGN0Y@Jkw|kTJ$(weZp=Ou*8VV!92-Z3;k4JuQn227jGG#ANqeh*Tn_{5~ z;JOZ+;s(B|vy63&pi2DM%vGVh=mb)sBwg$ZPRLHM)CPy*N|CJwAXo3 zp7N@!UrEDJ%{zOk%XS#(oGyWKjKtAjh+7LCw8(81%oX9yhSj^i1(D+1S{!>fkw@rw zP3PhYP@Ead(d}PnajHMlIjGcqair1!)crCXpj$a~ENqxqeZ}8jyY4o&N}rWXUasvcu*UIfY}BJWcjULE{(?Xec3LpST=Vi1c>pr2iXe5~=MGt8w; z>3(b%*CWgTK3?MvJ!BW*a-t2AIRehmq|~SD5>n36rz_ulT9it&cS-W5>Fm-EpmpJJ zjC7Fi4#9H|l>CT+7tMfK?AWMzDLVF9Z6VZrO|w2fadcg%N3mx8A3 zY;C@`Krt`@-=JEqE|stDMPVSMOIWG{QIURI{j)u!g=?BO_I}`%G|#5i{`DWFqyIx} zGOe^{65JE*!M+Q63LCByJ^oYd&~md$P5co%Mv#Brb@YVa^Oy~54XKd-sapGWj|2aQ zOwDYi)-3(QlfZt^K>UAY>gOr)pZ;5&pZ;4kJ{B*!U4yvhSB;_YWQLBScLSm*X#-(0 z3*m(qV2DhUHEWlY;hufK?odkltDe`h1ojObt}HABsz|Em$#mPvG@tA9Nsa9eFzYQZ zD5wInOWjL1+>!CEQ&zPa+(y&e5nVU_t7y$4I`d z64{P3WIsQ0_WTDG79u}Q&YqCgYyH9N{GCefv@yeNB8yE&0?S@l?-SN!^nqlJHSA7G z;%)ndhoR+VSRQ?qUlm=yd?3rBiU8D1imo}laY7$fAW!?SZks2~bdWKusnt~6)RAP7 z2-menu69Y@hXx2;G~&;4GY=;UOZ^1q3gAmuYsUsOY@t! z(oFs%gJ7_eLKHr-Y!gaVkB&1A)y6yNU(2=+(){tq%`%Qy+e88z{j$w&dFHr2HdIKn zR8BDJv<_)w15_=DY^KhCJ4}BKaySNSo61Rpx1KDRti)rESFa-R0V@gZ`~%P7kcE)x zy1Cfl7pGDv0sI`}`G|}N)z0}@QallK@PuLKYPSrBc1GrM=R^wo+;7KtzVL^6InYvA zTk3`jidZ;^x3vLAPA{?s+dyg_6NqysFBZP@jyCuztP&&`TZv`6*{yz;J$ z_OLC;6J7b~uKDSfTqW}Eq zViNT#lBgzuVf(?5`3oPgTtxnnCJKLGarE1ij`G22G$kcTl`ixS5Mno8=khF8y=QgXFbnpX_K-D6we0- zmwcWzUCZkicQ;M0+y7qMP(@iI6prT+-T`m_0Al=vV<$B@pY$nT@`M+Mc}m<@dh=yr z8V7h;PUbq?*q6&|1G;I;tk33|86(^$XUe_7@^8Gt2XWl!ccr}3C;q&@4p|OaG!IC! z5N@#uW0uT$V-{9x7RU&04p@#}z}{rh!lk%%1`@or=}1k-Ov58(pYB-A#umK2nul4O z05FUZhbU1p&gBNr_Gyhr_t2x^FmK8*0W^C&O8gC?3QaaH1UUnahf2A3+F2NmO~VgP zn&S^_y1DjI@)K@(@=cwRp%|8(q9qSQFfz{VzfCY6+Hh9Ro$461haJ7NDOI@?Op-s$ zN+h}d@?)NQ5!b|Si{bpul(=v!in(C30mNR@Ah)fLSc=}rcx_VktXO!J$*VyfIBOGi zJ?)44K7!!ZbT4>he{*Y4KXJ%vxkcyB=v*3^%7^|pm)|f{#cz`^<`NYk zJ7`o16;frUGa5NmiM5cbl69v?wuu!3F<|W+7cI|XF-)o$)xsoK&0@G5ZZ`=t0`R}v zm8Z=JrQXra>bBuoXq6e*7>Lu+_jG8&j?$PLZpCh4MG)E_DoMN0qp`z6U_oDkG-L{G zHnl9Uwt(?0vyReI^kglz4>WJY;S#`#8p7>p7Op{H*#v=@s9cs@9LHH{Wij?$*D+Zf zkB#6Roxtk#iE^@J%Br(?T1T(I0W`BNE;H=Zx4T@eAEmG|zNy);xTdS<-079lSTt#L zRH$ayo1yEDveFpPc2x3=87kvAcdbFEZw~8etwy?v)eRh_HAGzI5VLHm8ndXkGNV{m zSVW}L6IYaTp;by>S{pN3t&RWE2FV6Dp+Z|zA>6cXBAXl<#H|Ho%id#{1fVsmXDL>! zDR=v#Q6b{4*9XMWur>1I3^J%wi zu){Xtv~xFW&NT@`DbkWonV4l^!;jw#=JIS41mY<1>)SS)ih4;&*91zFFbH_YjxA8j zq#h?(;aLsMTPcPEWYPug0Hc-!uh4x6RzO)3t{hrpzJmnEDTRtvz7+KXdtoez{m=lW zDeQTi1qfrrR<|lUSZ0W8UBB6IvI8}`o7`^SKoiud{&h*=<_RWa00_hUGRux7_|;YO;C7-#!=MAjx2{c_2)DZn z1MZh*NhIXvQZj3A9Wu1~^rVq2;dS6jUQUX^u`|hLiwkx}0P?0%LR}(ly>Tx(6Rpaq zTZ>CmbGX7VJW8YNRA{RzH6ioN3vh76xRIeumMdApz~*Af^r#$bVez)4+vmk`!inc18q@ z?RviP0|IU^MRNE4s28YS zIV+WdI4u}FFNRo(M@-Nl4Tnsipl0WFHvLJ6teTX=F&M(qddzQI$Z##rC+#><7}&Km z0pXFS%2!x66F2UkNZG$zV8POw|$cM}jj{WTW<(v1e6WN|~r>Nrsl zoH02h`t4b;g7dSkOt3b>ATlB{HumaF0@dN}sOyy*jKz;AAu6xDqW*Hc!YdjHj};er zvODXl;DYRQ;VN~7(`QUb!}}y4cPa%(0?4%yrt#y=Mo$93Md~@XWcuy)x8Ab+%%4ko zB;X;#6&}QyT;1%rPsa>e-!eb#`p{lkQ;PO>y^jkd-phFfh8+c$8$vMP zTGTs^bb6!ctB?q5?6%XZiKNMC7nmtmyfoL`!&qot>NeB6irXJSbnx7BiZv7b1qh_J zwq)>B2-i%86rbQqYc|l8Mp0NFf-CK`I(346k}R0e9HRYFkjJ#+hwRA_0H131vdL+e zaK4gWc~DrD`8K<3($w|*xa#RAp*XEn1o-MJ- zXZeOAWg!9nk$5eHdg3M|xz@7}mFkYHsa{tXiIkCV2<+I!Nu3*2Hpba;pb1aK>LaAU z&^@n0r>pSv}3)nLLN|Xz!zr>)To)voRe9A|jW}0Ea(IOU zkz0cwz5WVCmL|V>@=3L;E6$lsLR2;Ha_(G%(5l6L)^5Iv_%y@fyA|j*&CFIt3^gs7 z^iI3lIfooHQ5pZHpi<`zhWr&P$ej^U!A|=Fmm}>|S{@W6JwkfI0kay70qLQmG6zL& z;tUctH-nF2OFj7C_Hk;T0p;|Gp8SemoNcuGFF$7G_2`?FTT7<4xdAf`0r&Li_?DDL z-Fv<&tj&m;=wvhMs6xjx{ztfs% zF4ldlJD>w$Neazizzs-kMFF%^HVWggNjwr!SH~%F=&A(bLJE_qWmZ9{!^C*Z!YBdG zlX7^K+F1)tl!{Hb$}c|Q7=tgU_xh`v-S2@%thT_i3!=vvP+V7mBaSB}A*Hp3!}9{+ zVqNZn6!Dc8B}F9-Ur%#c5qH94*OxiYViB)D9zMmkfZ10Zg`k}U5mSPF%75AY z(0r_SNJlRVg%ldxZF^8V5qtXt+!shuKSffY`_z1_O~V&2&$FyZm%x2aV!zK`4jHcV z8Y;TjjghHLpsJ2n{=FV`3s=$i<-^vND${ez%(e4D|EYcQ?4)iVlPJ4nME7gQTMVJD z$pN`l9*R&00p#Pb6RI579BZyCsYl|I#p2(Tt1nrM-=e0j>ae-i1HUIQNllv zS^11Hxll1h^4XF+t*QCi()QUBI-8O_?xa`y){9eFPUJU56|X>7dlKdh0?|7oW`qI? zD|Z!*fSjFT+*i={IDi7j1L2wK7wy(C;ldSH{1?gxpJc6#(SX{S6(Rw+=bkLuH}%dPvB(_ zlpf1BE)NJk#H0tF+Ce6%Cp=eX#5qr}_r2v_K#!OWzC^P8(O*SEKFEw0N+xkdzMQx! z_Q5XSFK7=YKW;}R66~6!HsWF{ASkQdc3f#ZY0zKv;M$N`8i#NO_6+KqC%p!Q@t@=w z;2n8w!F6)}xLu98-JN;+qD>nrSJd}tZG^(o815!3ZlGhEog->jdF?{T zC;)zYM#_Dhz+D9UPVT+g8hM507Bu}Hp!M0`4eqrJqME`641VEp))^d**8M!^ zkeZ(}9dh=o)LxCsauH|%<(ZSZD^RGDJp(pq3Z_<3GBmP>YV6Q}Q6p?}+3Hu@*Ctqs z^Sq2;<1eB=S3bVQuio-l3ldLyE`bz?vIyOD#)2vT4|UrNBF0E9c%BzKpfU{TF+#nZ2a)1{NQ5Ub>g9>2hWpB)3| zKA=)|hy(eBXT4-8HA-55!glt&JTgvZUEj3d30CG_+M>4ibRK{0NDCaQosRtD{tQ&` zmqFP^$T#4m))QL+N+maPB>rIXfbC?H?8)jDu_fi*DKQ7?pk(^s34DH81cWcmK2h~= z$*>uTBtq81gZtnp{4uwOn_Y!y{pRnAsAJnL9VRPHF!$U3DGh{r)AmX85fuHsoB*k4Ei*8hJh+h#9q<(v{bHC1dp}7r z%Y)QeR8Z?pGwJe_db2#F2WU6cSH?6KV^Xm4DW)#EQ4kbw0!{xregfvY)GRy!ss9I+ z-2~BF^k-i6(T{?}k*pA7wg#Bs8zI-7Q1Bs#3j7;3X~Wix%R9HT4s0Flx6LQ*C+_pH zgaGh6Ck-IVxzOWgpcw>n;#8oQ6mm0YChP1c4-E)d4t_@js+Txx0YJxEhZq~JP>x@K zQYN>SZ;4%pAi50Qgz+1M(+MZK+7U!XBPf?h41+v^vj8`=W63U$r*|(d)GxeaKqF@Xc zfov61wUVp|f_Fmw(4lXc9#{3t@n7>h_CTsdRq1iA>U&*XOzJcyYIggiquaU z#8K|>6d1~hDs;?;=ROT%d=>j{>;rB?nD7n6tw-x7Th5b|+KpD0&O4?-#EuhpnY-)* zhfei?57{K*u5&Fg2FZbUmj;uFlFs9J}hoy0d$Qae@UwBwhqI1u@0`1@(nNLQl;aE-W9C(k_p8kH-Yt7kD-vsU?{l%9&ky}#7#xpUYAC2_m(4=tlckLy0 z{Cn;wKj{RJIii%3+2olOVz`Z$kVT3zc)%lpXt?U~?8y0SY7oep_{KHWJ{XcT%ET#G zSR7QlXu;#3KZHD$aEIiZ;Y^7LBE%DbvRJ0z2IK?es@zj5FNycIJb<{g$?D4)P%Soy zjP(jaEfR2%Q#4u7;X^A{-i z{wnMA#?d98F8CyfA1+QRVNuoq^gOIm*k*>wG!W&?(jwjnZ zj@NuA(~~y>0zSZ{`)fqixrr2p#c@I!!H(3*^Qj?4`o!??#^&Yobj5aR**f!fe+NK= zIbSm1+M4*j;cVZmwYQ3)YeR77P=ygMk`io zl{YOef#1nDoWd^m=bLN$-Sh`kX;7|LzCt^{;dzXyypG!YYq)r!z^*hcvv(TP`E5D- z%Xykr9yIB%*^MGNHW35=az;@Crxio}T@~8c%UN-cc#p)8`TSve!Zin$ZlgfKE12`{ z{5j?6VwVg4tSaUh*f5xVOmKG|I$%P1?Tv!Bns@Q%A24#Z9v?nBosCDcBc3#qGNIA> zSlkX+amR7dbZ`c`WoB-bhS;=ls9QxjHyiRIuiV*sDkJc+OtsX-g26F;3$~hy$A-XeJ79#Zf`~LV%NHPCNF` zW22~&frL`%F+Hn8y8MuBa~yOEDPtuUt%E0sT!kAxM~=e94^Xx?O(V=q zIGuJO%p5qs+iP2{N_QG?l1XmuN%9b~CPBM{H|D5FavE}}|2V*XSS#5WrS7YKh96yP zi`!9ybs^cuP#ix`ad#}@DY!b?2k7+csxi2i+u;G88^ovRN?qBq`^^BD-hLbLckC2c z;tK0YcfX2_C-pQ34Y6-TkmFJm1r>+s{wmhd<`1epi<&V%--(K9h=aF7iHM``j5GGt z^wMLqTfoKI``#EW+K*=3U{5;@(USVRbNiLhXrVphGF~lT&q}8iY5wR2&tWF1J~~Z# z4!8iH(`os_oFBXD!N`)@wo;{`y6Bm9E5pD*V@1W`ui$UW(CUl+*vpY-%fA}~Ltm_A zwX+rb%1ZmhNY|A>*i-vzHmzFyjJZueZBt#<@c2t?yUBmX$iJ4C+OK#A5Kr}&vZP*= zR=dyhi!qnphkuFGVxpUWku92iMAdq_<_rN$X3OtH$4kD>A6kK+Kp5cV1s2Xt-4(cR^s9{< zrPp=O8N|JN`Wg_qtbq57Zm$VS0AwZZ``a148G z)Ur^1dl2Fn-ad zSD}zmVF&v|21UYeuaN2`0wW}9MMD7?NJX}hvq5&~DQ3U);9jV}Q?gziu3jA;Z682G zp71m}r0;YrPyA#sq#Uw_L8NSUNke)v3aIasa*V=Dk>-Q4Zk2a-Jrw6Ew=M zG=%GqH2Ud?cWy#{`&<=$m&Gz`~gp*AMk7l>9_?!{J*Jz=H6Pz)C9X| zX~2~{sv;6!i%$EMOC5Ti&n~0^ayuFf+h_?43KJQ8`XX!J&)15Ym@OU0*UK>ychLV1 z5amRp$oPE`jWe~9HRsdC_H(AEC(|5@U5~H7{ylv{`np$A#ty>6bJyRS3m4>M;7~9X z5kra$HA0~O$gXT;j3oxTVsXS8ju~>-ivXEqL*~d7?Kvt|u$m2%JQJ;;5p6dU?*~Ma zB;sR%md;CULMO~>9V#-9DF?dmUjqYWc5ba+)&03v?amhL5f z0|BA>SZOp@cWRw_jwOmzAN>wE6>TMOiobB`_p-^N2Ay5BPeO^@E7v|gPy2cJw~JL> zwDL`CH16G)`A#*;a!G?4qZ`y`nL=5(Z48x%@}tS=D%9ud!r6j@eji1dc2hUg?Y$kT zvPrVQy}-&e@xZZA9`N!K;-qe0Ndpkpn}(F8V~M{ROw0BOK(QzW)OdJybhlcM@ZnwP ztX8lg+`GPZqZr(RHs~K1CX~;L&J|129yF&OJ1_h3bT-3KwZ+;B8NMcx_|Dd>UP`mF zEqU=uBIj`bl(?FL4LA(>Icfgl6U+Z?787h%e=z7)&-Bx55PbcZeget>yHL1!uM5qz87ix8?-sf(ao-V57f zuKu09M>7nP%?H=0D0aY-FUlsE>@9MM&kXt7wTO~s(fuFaA4thdi0bFMf-)(H=1@+p z^Lcs~rwBhAu$2wAgI2+1q9bZD@4wTu$lMcn)ztU22_o4@(DmT^U1At#ueSevog3_J zRR7c2Au@+?d=3f(3h+4fk6I(qpKklMQxvb@!#u-wjbkb+f26S^VidjEdx-Y{oi4Y z$*g=6U2!Z&JZx+f3}d=oO(2=p>ASF0I>$<=Rd{S>Rjy@-+G%^GVF>;tCP1iMbY>({ zGv_^C3-2Bt~{ zkAc4uoX1=2+9nnmLgVQjI0M-Fs}_sW>CO`LDq|?)X~s?MRhsRo98vMP6G*yq*BQyp z#ahlCvs7%}%v89+c^Yfwnt&Qhu#5*X?5p!oc^oQ2!H&=ccZ^`61*+9@;}gQQc5^$g zX~x6oE%ipTgqp0eP0%MuHWig9>2f`-cEvF#&En27*H;PzpQUoBGwy$LEiCxFVTTBt z;OWPRa!OqgHe3W22{b#A_S)=isc;g0z;K8=(lqW3@;qn9p3J1_jQ}$f?EJ1v!&q38 z0UD+BP&~ZrURl^S^vr=!fBnu0`w(Jx66~Sl4w&XU@R1al@Kya#H>uKHf^LJFjr@BqMRq)80Y_R3G<*kd?n?Owq5QqBjGI4U5!kQ_vQ38>OvG3FIu zm?-%4o$0!lT-1df_+znq(g=6y(rD19G5rLpF zLZ32eU6o&$N>uaQkb{If1`oK?#k#bM&Gr*H|5}wxYWeN4ZV7OzNr(7rDvX}_r=d8W z?m@?Gn@6`ywF$v>jduU@0m69oU}g6c@wI1AE_0BUWyv%`)Zfmb1QK5TzqRs3q*HGXiOZTg=vPnCLZtt=L44J0Q`z z`uNTPF)Pv4j0h;hQmEy7v>x*DyTN`(o$2#_1qFqj4}GMZuYOxpfVvX-HEfO(imdD2 z9@A%V+W5#bjrKX#&jR+s_~<~Gm%kc^7pF_SLTP3;ZC_(R%tJ-5X zfRV+>ggFm0z$A0dV(K(DClmSW!0^OV+uloiC}goF*2VOU%PX5s@qC2*PFq(nQ+J5^ z92w8o?UyMr!2oQlSS#aGjugL_+$Tr;zyCL}48)V>1q=lA?+4|${?`QGVA|XApN|AR zx|u)Z&uPaG6KDS40a8DcfT(v%LEIl>W8rlTWvqNy8JyFYXgFYcA zlMFB-jcn7A1$Q@;ND89XmS9w79E!+}Geq}^lmDIi zXkK;fp-Rd8Mqt0X*I9>E+3Bg$A$WYP?hPP2LT2w(I670ekeADXLmpa#p%X?5nr)uW zbRUY=FH^XoR)&h=9e%sSF?^D`qcscL>NpnmP@kw~v^#5MX5JCzPY#ov z9=rVse1J;}VfAIwm@qzCm#qMzUJXLQBE$qiQzt99o@s1$YC`>lf4ri_n87qDjuPKH zxQ=t}hh>M4Xs{bXCGs~~V&yWuZc6dS+Jr}GlFEY&9g#Tc8Kzf6Cqk*gs1gO0F4l=uVlzhOhX8R~`a2PVd$f95dsgxX0uz>w<2 zpM~E@U-DW|g}~C_gFFVtx%7?T$hB_ND!=|vsf*41%^Sp_f=+jQx(SPXe8lyx2jea=5rvR;TG*iNRm z)^|gjg+{nH)t3ikSj^2{Au#j<1V8{K$+(zdyC_UFKASZkrA=l!TH#I}KUEP|*j-0# zSwXF$gke;Wi)CL=pvJb;Il{PfVf9rLM6OF;7x6;Rcui&o*?%-a zq>qu3b0>q7Yfok}9+0(7FRzA{WNuvV^hT$a>-Sj8&`j7FkL#>FBEeqx-K5>AJ%4iP zj?MnIn;eCwpp8x7&9zleSt2bggK50f-DQo6c^eB1cVT+Jmo*TfF|_A_c_b6b5CG?= zIn5M`?L9F{lR#6G8qJ&m49F&c_Q_1x6{+x*K%rnF*VF8WBVcjjG7iw6oQpGEZ!fmB z97z;3B~J>JX`RH+C|Gyy)N2Ouk{}lYM6nsKz&R_OC3mpRq_7U$maUg3r(WWsZq(=D zMU}-H+#Y%hA6;vUsYLX$=W{1CXcYa1iWD!C4zVzcK4Y|AOb1oID|!v(qSeMx#^uA5sQb2o=_ zKZEHe3qg)?OG-R*0%9V3hvtk(um{lFr^G*fA>2YK{Bw5SK@?tT&$VCu-rv4*Zf}It z5#ox8M_uMx8zAk|Hm>?Q@L&=o&v(?=ukc#;Pkb-AG}mAo8;rqLX(zIBz32RRptU79 z+WOjX2ivGlsX$txTVe_xF?3{=%>EVX4NW2#R-c5d2}8HqDFXe!&6VZ zPSaE4>_vKINFZg|RcD&sVW<_RW~M#F9%M-dr5gslAZen&z{1i{B$2jzRk6EtTh_$ZA>{4DsF||oY5hNaSzFNxz*lftk%CHZP%xReY{5EK z@0NY9^MQ30u-Xu)$cKT7tiI$om z;UK2b*&Bl%cJ$K+zC@@KEv@=YRHB3jO?%1l0h2xnFwDOUOCn=j+Rwuy^s@3p?nWQkl zT@uT)nS6#bkhn6!bwp?nJp`=G1z;3xsTY7+dw_+)lL8V~Ivr`F%l_J3a} zo>_v_L4%5u=sxo@+6`Y8qD07zQwuxC4@WYAGtDUz7Fhhc6sL~D7})I4pktli7%@plLA%GA*`DdZAVzF8NSc?V?U zd`a7>5G|RSFQO$|Q$sqg%rf#rHf!#|A==z?50evJ$ut&tm!ez}rLJ(7M6aXFir)jX zd3iMxK+E279OSJea45hikB^kx{Yv&ne<)axe~xL48wleyqW59Qf~>{~Ke%6y`RaO= z`bI(!79wzB6B+4S$`kQxhw(`EOA9#83MrNPR$_~a5Du*{=;F{GZ=0~r@idYrn(3Wp zm;x=k8F2I~UMZ6Ju^liLa;qztwnH%IC%W=ORt=cIO^yzxxO#u2uDV1sIaD Date: Mon, 8 Oct 2018 21:43:20 +0200 Subject: [PATCH 17/91] Custom sharding strategy --- .../dyno/queues/redis/RedisDynoQueue.java | 58 ++++++++----------- .../dyno/queues/redis/RedisQueues.java | 48 +++++++-------- .../redis/sharding/RoundRobinStrategy.java | 28 +++++++++ .../redis/sharding/ShardingStrategy.java | 12 ++++ 4 files changed, 87 insertions(+), 59 deletions(-) create mode 100644 dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/sharding/RoundRobinStrategy.java create mode 100644 dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/sharding/ShardingStrategy.java diff --git a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/RedisDynoQueue.java b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/RedisDynoQueue.java index 8941231..e900e68 100644 --- a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/RedisDynoQueue.java +++ b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/RedisDynoQueue.java @@ -20,10 +20,12 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.SerializationFeature; import com.google.common.annotations.VisibleForTesting; +import com.google.common.collect.ImmutableList; import com.google.common.util.concurrent.Uninterruptibles; import com.netflix.dyno.connectionpool.exception.DynoException; import com.netflix.dyno.queues.DynoQueue; import com.netflix.dyno.queues.Message; +import com.netflix.dyno.queues.redis.sharding.ShardingStrategy; import com.netflix.servo.monitor.Stopwatch; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -58,37 +60,37 @@ public class RedisDynoQueue implements DynoQueue { private final Logger logger = LoggerFactory.getLogger(RedisDynoQueue.class); - private Clock clock; + private final Clock clock; - private String queueName; + private final String queueName; - private List allShards; + private final List allShards; - private String shardName; + private final String shardName; - private String redisKeyPrefix; + private final String redisKeyPrefix; - private String messageStoreKey; + private final String messageStoreKey; - private String myQueueShard; + private final String myQueueShard; - private int unackTime = 60; + private volatile int unackTime = 60; - private QueueMonitor monitor; + private final QueueMonitor monitor; - private ObjectMapper om; + private final ObjectMapper om; - private JedisCommands quorumConn; + private volatile JedisCommands quorumConn; - private JedisCommands nonQuorumConn; + private volatile JedisCommands nonQuorumConn; - private ConcurrentLinkedQueue prefetchedIds; + private final ConcurrentLinkedQueue prefetchedIds; - private ScheduledExecutorService schedulerForUnacksProcessing; + private final ScheduledExecutorService schedulerForUnacksProcessing; - private ScheduledExecutorService schedulerForPrefetchProcessing; + private final int retryCount = 2; - private int retryCount = 2; + private final ShardingStrategy shardingStrategy; public RedisDynoQueue(String redisKeyPrefix, String queueName, Set allShards, String shardName) { this(redisKeyPrefix, queueName, allShards, shardName, 60_000); @@ -99,13 +101,18 @@ public RedisDynoQueue(String redisKeyPrefix, String queueName, Set allSh } public RedisDynoQueue(Clock clock, String redisKeyPrefix, String queueName, Set allShards, String shardName, int unackScheduleInMS) { + this(clock, redisKeyPrefix, queueName, allShards, shardName, unackScheduleInMS, null); + } + + public RedisDynoQueue(Clock clock, String redisKeyPrefix, String queueName, Set allShards, String shardName, int unackScheduleInMS, ShardingStrategy shardingStrategy) { this.clock = clock; this.redisKeyPrefix = redisKeyPrefix; this.queueName = queueName; - this.allShards = allShards.stream().collect(Collectors.toList()); + this.allShards = ImmutableList.copyOf(allShards.stream().collect(Collectors.toList())); this.shardName = shardName; this.messageStoreKey = redisKeyPrefix + ".MESSAGE." + queueName; this.myQueueShard = getQueueShardKey(queueName, shardName); + this.shardingStrategy = shardingStrategy; ObjectMapper om = new ObjectMapper(); om.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); @@ -120,12 +127,10 @@ public RedisDynoQueue(Clock clock, String redisKeyPrefix, String queueName, Set< this.prefetchedIds = new ConcurrentLinkedQueue<>(); schedulerForUnacksProcessing = Executors.newScheduledThreadPool(1); - schedulerForPrefetchProcessing = Executors.newScheduledThreadPool(1); schedulerForUnacksProcessing.scheduleAtFixedRate(() -> processUnacks(), unackScheduleInMS, unackScheduleInMS, TimeUnit.MILLISECONDS); logger.info(RedisDynoQueue.class.getName() + " is ready to serve " + queueName); - } public RedisDynoQueue withQuorumConn(JedisCommands quorumConn){ @@ -166,7 +171,7 @@ public List push(final List messages) { quorumConn.hset(messageStoreKey, message.getId(), json); double priority = message.getPriority() / 100.0; double score = Long.valueOf(clock.millis() + message.getTimeout()).doubleValue() + priority; - String shard = getNextShard(); + String shard = shardingStrategy.getNextShard(allShards, message); String queueShard = getQueueShardKey(queueName, shard); quorumConn.zadd(queueShard, score, message.getId()); } @@ -570,18 +575,6 @@ public void processUnacks() { } - private AtomicInteger nextShardIndex = new AtomicInteger(0); - - private String getNextShard() { - int indx = nextShardIndex.incrementAndGet(); - if (indx >= allShards.size()) { - nextShardIndex.set(0); - indx = 0; - } - String s = allShards.get(indx); - return s; - } - private String getQueueShardKey(String queueName, String shard) { return redisKeyPrefix + ".QUEUE." + queueName + "." + shard; } @@ -616,7 +609,6 @@ private R executeWithRetry(Callable r, int retryCount) { @Override public void close() throws IOException { schedulerForUnacksProcessing.shutdown(); - schedulerForPrefetchProcessing.shutdown(); monitor.close(); } } diff --git a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/RedisQueues.java b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/RedisQueues.java index 291c06f..7f468b7 100644 --- a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/RedisQueues.java +++ b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/RedisQueues.java @@ -17,6 +17,8 @@ import com.netflix.dyno.queues.DynoQueue; import com.netflix.dyno.queues.ShardSupplier; +import com.netflix.dyno.queues.redis.sharding.RoundRobinStrategy; +import com.netflix.dyno.queues.redis.sharding.ShardingStrategy; import redis.clients.jedis.JedisCommands; import java.io.Closeable; @@ -34,23 +36,25 @@ */ public class RedisQueues implements Closeable { - private Clock clock; + private final Clock clock; - private JedisCommands quorumConn; + private final JedisCommands quorumConn; - private JedisCommands nonQuorumConn; + private final JedisCommands nonQuorumConn; - private Set allShards; + private final Set allShards; - private String shardName; + private final String shardName; - private String redisKeyPrefix; + private final String redisKeyPrefix; - private int unackTime; + private final int unackTime; - private int unackHandlerIntervalInMS; + private final int unackHandlerIntervalInMS; - private ConcurrentHashMap queues; + private final ConcurrentHashMap queues; + + private final ShardingStrategy shardingStrategy; /** * @param quorumConn Dyno connection with dc_quorum enabled @@ -61,7 +65,7 @@ public class RedisQueues implements Closeable { * @param unackHandlerIntervalInMS Time in millisecond at which the un-acknowledgement processor runs */ public RedisQueues(JedisCommands quorumConn, JedisCommands nonQuorumConn, String redisKeyPrefix, ShardSupplier shardSupplier, int unackTime, int unackHandlerIntervalInMS) { - this(Clock.systemDefaultZone(), quorumConn, nonQuorumConn, redisKeyPrefix, shardSupplier, unackTime, unackHandlerIntervalInMS); + this(Clock.systemDefaultZone(), quorumConn, nonQuorumConn, redisKeyPrefix, shardSupplier, unackTime, unackHandlerIntervalInMS, new RoundRobinStrategy()); } /** @@ -73,7 +77,7 @@ public RedisQueues(JedisCommands quorumConn, JedisCommands nonQuorumConn, String * @param unackTime Time in millisecond within which a message needs to be acknowledged by the client, after which the message is re-queued. * @param unackHandlerIntervalInMS Time in millisecond at which the un-acknowledgement processor runs */ - public RedisQueues(Clock clock, JedisCommands quorumConn, JedisCommands nonQuorumConn, String redisKeyPrefix, ShardSupplier shardSupplier, int unackTime, int unackHandlerIntervalInMS) { + public RedisQueues(Clock clock, JedisCommands quorumConn, JedisCommands nonQuorumConn, String redisKeyPrefix, ShardSupplier shardSupplier, int unackTime, int unackHandlerIntervalInMS, ShardingStrategy shardingStrategy) { this.clock = clock; this.quorumConn = quorumConn; this.nonQuorumConn = nonQuorumConn; @@ -83,8 +87,9 @@ public RedisQueues(Clock clock, JedisCommands quorumConn, JedisCommands nonQuoru this.unackTime = unackTime; this.unackHandlerIntervalInMS = unackHandlerIntervalInMS; this.queues = new ConcurrentHashMap<>(); + this.shardingStrategy = shardingStrategy; } - + /** * * @param queueName Name of the queue @@ -95,20 +100,11 @@ public RedisQueues(Clock clock, JedisCommands quorumConn, JedisCommands nonQuoru public DynoQueue get(String queueName) { String key = queueName.intern(); - DynoQueue queue = this.queues.get(key); - if (queue != null) { - return queue; - } - - synchronized (this) { - queue = new RedisDynoQueue(clock, redisKeyPrefix, queueName, allShards, shardName, unackHandlerIntervalInMS) - .withUnackTime(unackTime) - .withNonQuorumConn(nonQuorumConn) - .withQuorumConn(quorumConn); - this.queues.put(key, queue); - } - - return queue; + + return queues.computeIfAbsent(key, (keyToCompute) -> new RedisDynoQueue(clock, redisKeyPrefix, queueName, allShards, shardName, unackHandlerIntervalInMS, shardingStrategy) + .withUnackTime(unackTime) + .withNonQuorumConn(nonQuorumConn) + .withQuorumConn(quorumConn)); } /** diff --git a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/sharding/RoundRobinStrategy.java b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/sharding/RoundRobinStrategy.java new file mode 100644 index 0000000..d05fac4 --- /dev/null +++ b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/sharding/RoundRobinStrategy.java @@ -0,0 +1,28 @@ +package com.netflix.dyno.queues.redis.sharding; + +import com.netflix.dyno.queues.Message; + +import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; + +public class RoundRobinStrategy implements ShardingStrategy { + + private final AtomicInteger nextShardIndex = new AtomicInteger(0); + + /** + * Get shard based on round robin strategy. + * @param allShards + * @param message is ignored in round robin strategy + * @return + */ + @Override + public String getNextShard(List allShards, Message message) { + int index = nextShardIndex.incrementAndGet(); + if (index >= allShards.size()) { + nextShardIndex.set(0); + index = 0; + } + String shard = allShards.get(index); + return shard; + } +} diff --git a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/sharding/ShardingStrategy.java b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/sharding/ShardingStrategy.java new file mode 100644 index 0000000..5e203e0 --- /dev/null +++ b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/sharding/ShardingStrategy.java @@ -0,0 +1,12 @@ +package com.netflix.dyno.queues.redis.sharding; + +import com.netflix.dyno.queues.Message; + +import java.util.List; + +/** + * Expose common interface that allow to apply custom sharding strategy. + */ +public interface ShardingStrategy { + String getNextShard(List allShards, Message message); +} From 82e5ae6cc89355a0868b6bb15c40c5c6677fa82d Mon Sep 17 00:00:00 2001 From: Kishore Kasi Date: Tue, 9 Oct 2018 15:23:47 -0700 Subject: [PATCH 18/91] fix bundle.gradle to pick up RCs and upgrade dyno-core and dyno-jedis to 1.6.5-rc.1 --- build.gradle | 5 +++++ dyno-queues-redis/build.gradle | 4 ++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/build.gradle b/build.gradle index 0a3136b..cbf1a63 100644 --- a/build.gradle +++ b/build.gradle @@ -28,6 +28,11 @@ subprojects { repositories { jcenter() + + // oss-candidate for -rc.* verions: + maven { + url "https://dl.bintray.com/netflixoss/oss-candidate" + } } group = "com.netflix.${githubProjectName}" diff --git a/dyno-queues-redis/build.gradle b/dyno-queues-redis/build.gradle index 0024a41..d4d1ed9 100644 --- a/dyno-queues-redis/build.gradle +++ b/dyno-queues-redis/build.gradle @@ -4,8 +4,8 @@ dependencies { compile "com.google.inject:guice:3.0" - compile 'com.netflix.dyno:dyno-core:1.6.4' - compile 'com.netflix.dyno:dyno-jedis:1.6.4' + compile 'com.netflix.dyno:dyno-core:1.6.5-rc.1' + compile 'com.netflix.dyno:dyno-jedis:1.6.5-rc.1' compile 'com.netflix.archaius:archaius-core:0.7.5' compile 'com.netflix.servo:servo-core:0.12.17' From ec33a3a1082bb9399bb3383f6a4dabfb5b59273d Mon Sep 17 00:00:00 2001 From: KowalczykBartek Date: Wed, 10 Oct 2018 22:36:34 +0200 Subject: [PATCH 19/91] Review fix - test and improvments --- .gitignore | 4 + .../dyno/queues/redis/RedisDynoQueue.java | 12 +- .../dyno/queues/redis/RedisQueues.java | 15 ++ .../redis/sharding/RoundRobinStrategy.java | 15 ++ .../redis/sharding/ShardingStrategy.java | 15 ++ .../redis/DefaultShardingStrategyTest.java | 170 +++++++++++++++++ .../redis/v2/CustomShardingStrategyTest.java | 174 ++++++++++++++++++ 7 files changed, 397 insertions(+), 8 deletions(-) create mode 100644 dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/DefaultShardingStrategyTest.java create mode 100644 dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/v2/CustomShardingStrategyTest.java diff --git a/.gitignore b/.gitignore index 342edea..6e9cfde 100644 --- a/.gitignore +++ b/.gitignore @@ -9,3 +9,7 @@ redis-3.0.7.tar.gz .gradle .classpath .project + + +dyno-queues-core/out/ +dyno-queues-redis/out/ \ No newline at end of file diff --git a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/RedisDynoQueue.java b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/RedisDynoQueue.java index e900e68..c6d3447 100644 --- a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/RedisDynoQueue.java +++ b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/RedisDynoQueue.java @@ -92,16 +92,12 @@ public class RedisDynoQueue implements DynoQueue { private final ShardingStrategy shardingStrategy; - public RedisDynoQueue(String redisKeyPrefix, String queueName, Set allShards, String shardName) { - this(redisKeyPrefix, queueName, allShards, shardName, 60_000); + public RedisDynoQueue(String redisKeyPrefix, String queueName, Set allShards, String shardName, ShardingStrategy shardingStrategy) { + this(redisKeyPrefix, queueName, allShards, shardName, 60_000, shardingStrategy); } - public RedisDynoQueue(String redisKeyPrefix, String queueName, Set allShards, String shardName, int unackScheduleInMS) { - this(Clock.systemDefaultZone(), redisKeyPrefix, queueName, allShards, shardName, unackScheduleInMS); - } - - public RedisDynoQueue(Clock clock, String redisKeyPrefix, String queueName, Set allShards, String shardName, int unackScheduleInMS) { - this(clock, redisKeyPrefix, queueName, allShards, shardName, unackScheduleInMS, null); + public RedisDynoQueue(String redisKeyPrefix, String queueName, Set allShards, String shardName, int unackScheduleInMS, ShardingStrategy shardingStrategy) { + this(Clock.systemDefaultZone(), redisKeyPrefix, queueName, allShards, shardName, unackScheduleInMS, shardingStrategy); } public RedisDynoQueue(Clock clock, String redisKeyPrefix, String queueName, Set allShards, String shardName, int unackScheduleInMS, ShardingStrategy shardingStrategy) { diff --git a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/RedisQueues.java b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/RedisQueues.java index 7f468b7..add0339 100644 --- a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/RedisQueues.java +++ b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/RedisQueues.java @@ -68,6 +68,20 @@ public RedisQueues(JedisCommands quorumConn, JedisCommands nonQuorumConn, String this(Clock.systemDefaultZone(), quorumConn, nonQuorumConn, redisKeyPrefix, shardSupplier, unackTime, unackHandlerIntervalInMS, new RoundRobinStrategy()); } + /** + * @param quorumConn Dyno connection with dc_quorum enabled + * @param nonQuorumConn Dyno connection to local Redis + * @param redisKeyPrefix prefix applied to the Redis keys + * @param shardSupplier Provider for the shards for the queues created + * @param unackTime Time in millisecond within which a message needs to be acknowledged by the client, after which the message is re-queued. + * @param unackHandlerIntervalInMS Time in millisecond at which the un-acknowledgement processor runs + * @param shardingStrategy sharding strategy responsible for calculating message's destination shard + */ + public RedisQueues(JedisCommands quorumConn, JedisCommands nonQuorumConn, String redisKeyPrefix, ShardSupplier shardSupplier, int unackTime, int unackHandlerIntervalInMS, ShardingStrategy shardingStrategy) { + this(Clock.systemDefaultZone(), quorumConn, nonQuorumConn, redisKeyPrefix, shardSupplier, unackTime, unackHandlerIntervalInMS, shardingStrategy); + } + + /** * @param clock Time provider * @param quorumConn Dyno connection with dc_quorum enabled @@ -76,6 +90,7 @@ public RedisQueues(JedisCommands quorumConn, JedisCommands nonQuorumConn, String * @param shardSupplier Provider for the shards for the queues created * @param unackTime Time in millisecond within which a message needs to be acknowledged by the client, after which the message is re-queued. * @param unackHandlerIntervalInMS Time in millisecond at which the un-acknowledgement processor runs + * @param shardingStrategy sharding strategy responsible for calculating message's destination shard */ public RedisQueues(Clock clock, JedisCommands quorumConn, JedisCommands nonQuorumConn, String redisKeyPrefix, ShardSupplier shardSupplier, int unackTime, int unackHandlerIntervalInMS, ShardingStrategy shardingStrategy) { this.clock = clock; diff --git a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/sharding/RoundRobinStrategy.java b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/sharding/RoundRobinStrategy.java index d05fac4..5ad511d 100644 --- a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/sharding/RoundRobinStrategy.java +++ b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/sharding/RoundRobinStrategy.java @@ -1,3 +1,18 @@ +/** + * Copyright 2018 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package com.netflix.dyno.queues.redis.sharding; import com.netflix.dyno.queues.Message; diff --git a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/sharding/ShardingStrategy.java b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/sharding/ShardingStrategy.java index 5e203e0..71744af 100644 --- a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/sharding/ShardingStrategy.java +++ b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/sharding/ShardingStrategy.java @@ -1,3 +1,18 @@ +/** + * Copyright 2018 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package com.netflix.dyno.queues.redis.sharding; import com.netflix.dyno.queues.Message; diff --git a/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/DefaultShardingStrategyTest.java b/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/DefaultShardingStrategyTest.java new file mode 100644 index 0000000..d0f42bc --- /dev/null +++ b/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/DefaultShardingStrategyTest.java @@ -0,0 +1,170 @@ +/** + * Copyright 2016 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.netflix.dyno.queues.redis; + +import com.netflix.dyno.connectionpool.Host; +import com.netflix.dyno.connectionpool.HostSupplier; +import com.netflix.dyno.queues.Message; +import com.netflix.dyno.queues.ShardSupplier; +import com.netflix.dyno.queues.jedis.JedisMock; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.util.Iterator; +import java.util.LinkedList; +import java.util.List; +import java.util.Set; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; + +import static org.junit.Assert.assertEquals; + +public class DefaultShardingStrategyTest { + + private static JedisMock dynoClient; + + private static final String queueName = "test_queue"; + + private static final String redisKeyPrefix = "testdynoqueues"; + + private static RedisDynoQueue shard1DynoQueue; + private static RedisDynoQueue shard2DynoQueue; + private static RedisDynoQueue shard3DynoQueue; + + private static RedisQueues shard1Queue; + private static RedisQueues shard2Queue; + private static RedisQueues shard3Queue; + + private static String messageKey; + + @BeforeClass + public static void setUpBeforeClass() throws Exception { + + HostSupplier hs = new HostSupplier() { + @Override + public List getHosts() { + List hosts = new LinkedList<>(); + hosts.add(new Host("localhost", 8102, "us-east-1d", Host.Status.Up)); + hosts.add(new Host("localhost", 8102, "us-east-2d", Host.Status.Up)); + hosts.add(new Host("localhost", 8102, "us-east-3d", Host.Status.Up)); + return hosts; + } + }; + + dynoClient = new JedisMock(); + + Set allShards = hs.getHosts().stream().map(host -> host.getRack().substring(host.getRack().length() - 2)).collect(Collectors.toSet()); + Iterator iterator = allShards.iterator(); + String shard1Name = iterator.next(); + String shard2Name = iterator.next(); + String shard3Name = iterator.next(); + + ShardSupplier shard1Supplier = new ShardSupplier() { + + @Override + public Set getQueueShards() { + return allShards; + } + + @Override + public String getCurrentShard() { + return shard1Name; + } + }; + + ShardSupplier shard2Supplier = new ShardSupplier() { + + @Override + public Set getQueueShards() { + return allShards; + } + + @Override + public String getCurrentShard() { + return shard2Name; + } + }; + + + ShardSupplier shard3Supplier = new ShardSupplier() { + + @Override + public Set getQueueShards() { + return allShards; + } + + @Override + public String getCurrentShard() { + return shard3Name; + } + }; + + messageKey = redisKeyPrefix + ".MESSAGE." + queueName; + + shard1Queue = new RedisQueues(dynoClient, dynoClient, redisKeyPrefix, shard1Supplier, 1_000, 1_000_000); + shard2Queue = new RedisQueues(dynoClient, dynoClient, redisKeyPrefix, shard2Supplier, 1_000, 1_000_000); + shard3Queue = new RedisQueues(dynoClient, dynoClient, redisKeyPrefix, shard3Supplier, 1_000, 1_000_000); + + + shard1DynoQueue = (RedisDynoQueue)shard1Queue.get(queueName); + shard2DynoQueue = (RedisDynoQueue)shard2Queue.get(queueName); + shard3DynoQueue = (RedisDynoQueue)shard3Queue.get(queueName); + } + + @Before + public void clearAll() + { + shard1DynoQueue.clear(); + shard2DynoQueue.clear(); + shard3DynoQueue.clear(); + } + + @Test + public void testAll() { + + List messages = new LinkedList<>(); + + Message msg = new Message("1", "Hello World"); + msg.setPriority(1); + messages.add(msg); + + /** + * Because of sharding strategy works in round-robin manner, single client, for shard1, should + * push message(even the same) to three different shards. + */ + shard1DynoQueue.push(messages); + shard1DynoQueue.push(messages); + shard1DynoQueue.push(messages); + + List popedFromShard1 = shard1DynoQueue.pop(1, 1, TimeUnit.SECONDS); + + List popedFromShard2 = shard2DynoQueue.pop(1, 1, TimeUnit.SECONDS); + + List popedFromShard3 = shard3DynoQueue.pop(1, 1, TimeUnit.SECONDS); + + + assertEquals(1, popedFromShard1.size()); + assertEquals(1, popedFromShard2.size()); + assertEquals(1, popedFromShard3.size()); + + assertEquals(msg, popedFromShard1.get(0)); + assertEquals(msg, popedFromShard2.get(0)); + assertEquals(msg, popedFromShard3.get(0)); + + } + +} diff --git a/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/v2/CustomShardingStrategyTest.java b/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/v2/CustomShardingStrategyTest.java new file mode 100644 index 0000000..c57a4fd --- /dev/null +++ b/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/v2/CustomShardingStrategyTest.java @@ -0,0 +1,174 @@ +/** + * Copyright 2016 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.netflix.dyno.queues.redis.v2; + +import com.netflix.dyno.connectionpool.Host; +import com.netflix.dyno.connectionpool.HostSupplier; +import com.netflix.dyno.queues.Message; +import com.netflix.dyno.queues.ShardSupplier; +import com.netflix.dyno.queues.jedis.JedisMock; +import com.netflix.dyno.queues.redis.RedisDynoQueue; +import com.netflix.dyno.queues.redis.RedisQueues; +import com.netflix.dyno.queues.redis.sharding.ShardingStrategy; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.util.LinkedList; +import java.util.List; +import java.util.Set; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; + +import static org.junit.Assert.assertEquals; + +public class CustomShardingStrategyTest { + + public static class HashBasedStrategy implements ShardingStrategy { + @Override + public String getNextShard(List allShards, Message message) { + int hashCodeAbs = Math.abs(message.getId().hashCode()); + int calculatedShard = (hashCodeAbs % allShards.size()) + 1; + return Integer.toString(calculatedShard); + } + } + + private static JedisMock dynoClient; + + private static final String queueName = "test_queue"; + + private static final String redisKeyPrefix = "testdynoqueues"; + + private static RedisDynoQueue shard1DynoQueue; + private static RedisDynoQueue shard2DynoQueue; + private static RedisDynoQueue shard3DynoQueue; + + private static RedisQueues shard1Queue; + private static RedisQueues shard2Queue; + private static RedisQueues shard3Queue; + + private static String messageKey; + + @BeforeClass + public static void setUpBeforeClass() throws Exception { + + HostSupplier hs = new HostSupplier() { + @Override + public List getHosts() { + List hosts = new LinkedList<>(); + hosts.add(new Host("host1", 8102, "rack1", Host.Status.Up)); + hosts.add(new Host("host2", 8102, "rack2", Host.Status.Up)); + hosts.add(new Host("host3", 8102, "rack3", Host.Status.Up)); + return hosts; + } + }; + + dynoClient = new JedisMock(); + + Set allShards = hs.getHosts().stream().map(host -> host.getRack().substring(host.getRack().length() - 2)).collect(Collectors.toSet()); + + ShardSupplier shard1Supplier = new ShardSupplier() { + + @Override + public Set getQueueShards() { + return allShards; + } + + @Override + public String getCurrentShard() { + return "1"; + } + }; + + ShardSupplier shard2Supplier = new ShardSupplier() { + + @Override + public Set getQueueShards() { + return allShards; + } + + @Override + public String getCurrentShard() { + return "2"; + } + }; + + + ShardSupplier shard3Supplier = new ShardSupplier() { + + @Override + public Set getQueueShards() { + return allShards; + } + + @Override + public String getCurrentShard() { + return "3"; + } + }; + + messageKey = redisKeyPrefix + ".MESSAGE." + queueName; + + HashBasedStrategy hashBasedStrategy = new HashBasedStrategy(); + + shard1Queue = new RedisQueues(dynoClient, dynoClient, redisKeyPrefix, shard1Supplier, 1_000, 1_000_000, hashBasedStrategy); + shard2Queue = new RedisQueues(dynoClient, dynoClient, redisKeyPrefix, shard2Supplier, 1_000, 1_000_000, hashBasedStrategy); + shard3Queue = new RedisQueues(dynoClient, dynoClient, redisKeyPrefix, shard3Supplier, 1_000, 1_000_000, hashBasedStrategy); + + shard1DynoQueue = (RedisDynoQueue) shard1Queue.get(queueName); + shard2DynoQueue = (RedisDynoQueue) shard2Queue.get(queueName); + shard3DynoQueue = (RedisDynoQueue) shard3Queue.get(queueName); + } + + @Before + public void clearAll() { + shard1DynoQueue.clear(); + shard2DynoQueue.clear(); + shard3DynoQueue.clear(); + } + + @Test + public void testAll() { + + List messages = new LinkedList<>(); + + Message msg = new Message("1", "Hello World"); + msg.setPriority(1); + messages.add(msg); + + /** + * Because my custom sharding strategy that depends on message id, and calculated hash (just Java's hashCode), + * message will always ends on the same shard, so message never duplicates, in test case, I expect that + * message will be received only once. + */ + shard1DynoQueue.push(messages); + shard1DynoQueue.push(messages); + shard1DynoQueue.push(messages); + + List popedFromShard1 = shard1DynoQueue.pop(1, 1, TimeUnit.SECONDS); + + List popedFromShard2 = shard2DynoQueue.pop(1, 1, TimeUnit.SECONDS); + + List popedFromShard3 = shard3DynoQueue.pop(1, 1, TimeUnit.SECONDS); + + + assertEquals(0, popedFromShard1.size()); + assertEquals(1, popedFromShard2.size()); + assertEquals(0, popedFromShard3.size()); + + assertEquals(msg, popedFromShard2.get(0)); + } +} From b8e5168631c6656f1105c7304120b85c52084b77 Mon Sep 17 00:00:00 2001 From: KowalczykBartek Date: Fri, 12 Oct 2018 23:38:02 +0200 Subject: [PATCH 20/91] change custom shard strategy logic --- .../{v2 => }/CustomShardingStrategyTest.java | 23 ++++++++++--------- 1 file changed, 12 insertions(+), 11 deletions(-) rename dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/{v2 => }/CustomShardingStrategyTest.java (93%) diff --git a/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/v2/CustomShardingStrategyTest.java b/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/CustomShardingStrategyTest.java similarity index 93% rename from dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/v2/CustomShardingStrategyTest.java rename to dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/CustomShardingStrategyTest.java index c57a4fd..44f1cbc 100644 --- a/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/v2/CustomShardingStrategyTest.java +++ b/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/CustomShardingStrategyTest.java @@ -13,20 +13,19 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.netflix.dyno.queues.redis.v2; +package com.netflix.dyno.queues.redis; import com.netflix.dyno.connectionpool.Host; import com.netflix.dyno.connectionpool.HostSupplier; import com.netflix.dyno.queues.Message; import com.netflix.dyno.queues.ShardSupplier; import com.netflix.dyno.queues.jedis.JedisMock; -import com.netflix.dyno.queues.redis.RedisDynoQueue; -import com.netflix.dyno.queues.redis.RedisQueues; import com.netflix.dyno.queues.redis.sharding.ShardingStrategy; import org.junit.Before; import org.junit.BeforeClass; import org.junit.Test; +import java.util.Iterator; import java.util.LinkedList; import java.util.List; import java.util.Set; @@ -41,8 +40,8 @@ public static class HashBasedStrategy implements ShardingStrategy { @Override public String getNextShard(List allShards, Message message) { int hashCodeAbs = Math.abs(message.getId().hashCode()); - int calculatedShard = (hashCodeAbs % allShards.size()) + 1; - return Integer.toString(calculatedShard); + int calculatedShard = (hashCodeAbs % allShards.size()); + return allShards.get(calculatedShard); } } @@ -80,6 +79,11 @@ public List getHosts() { Set allShards = hs.getHosts().stream().map(host -> host.getRack().substring(host.getRack().length() - 2)).collect(Collectors.toSet()); + Iterator iterator = allShards.iterator(); + String shard1Name = iterator.next(); + String shard2Name = iterator.next(); + String shard3Name = iterator.next(); + ShardSupplier shard1Supplier = new ShardSupplier() { @Override @@ -89,7 +93,7 @@ public Set getQueueShards() { @Override public String getCurrentShard() { - return "1"; + return shard1Name; } }; @@ -102,7 +106,7 @@ public Set getQueueShards() { @Override public String getCurrentShard() { - return "2"; + return shard2Name; } }; @@ -116,7 +120,7 @@ public Set getQueueShards() { @Override public String getCurrentShard() { - return "3"; + return shard3Name; } }; @@ -159,12 +163,9 @@ public void testAll() { shard1DynoQueue.push(messages); List popedFromShard1 = shard1DynoQueue.pop(1, 1, TimeUnit.SECONDS); - List popedFromShard2 = shard2DynoQueue.pop(1, 1, TimeUnit.SECONDS); - List popedFromShard3 = shard3DynoQueue.pop(1, 1, TimeUnit.SECONDS); - assertEquals(0, popedFromShard1.size()); assertEquals(1, popedFromShard2.size()); assertEquals(0, popedFromShard3.size()); From 8af9b442795937059755398630335705f0cdb606 Mon Sep 17 00:00:00 2001 From: Ricool06 Date: Sat, 13 Oct 2018 02:28:15 +0100 Subject: [PATCH 21/91] Remove some unused lines (solves #29) --- .../java/com/netflix/dyno/queues/redis/RedisDynoQueue.java | 4 ---- .../com/netflix/dyno/queues/redis/RedisDynoQueueTest.java | 1 - 2 files changed, 5 deletions(-) diff --git a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/RedisDynoQueue.java b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/RedisDynoQueue.java index 8941231..76afcbf 100644 --- a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/RedisDynoQueue.java +++ b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/RedisDynoQueue.java @@ -86,8 +86,6 @@ public class RedisDynoQueue implements DynoQueue { private ScheduledExecutorService schedulerForUnacksProcessing; - private ScheduledExecutorService schedulerForPrefetchProcessing; - private int retryCount = 2; public RedisDynoQueue(String redisKeyPrefix, String queueName, Set allShards, String shardName) { @@ -120,7 +118,6 @@ public RedisDynoQueue(Clock clock, String redisKeyPrefix, String queueName, Set< this.prefetchedIds = new ConcurrentLinkedQueue<>(); schedulerForUnacksProcessing = Executors.newScheduledThreadPool(1); - schedulerForPrefetchProcessing = Executors.newScheduledThreadPool(1); schedulerForUnacksProcessing.scheduleAtFixedRate(() -> processUnacks(), unackScheduleInMS, unackScheduleInMS, TimeUnit.MILLISECONDS); @@ -616,7 +613,6 @@ private R executeWithRetry(Callable r, int retryCount) { @Override public void close() throws IOException { schedulerForUnacksProcessing.shutdown(); - schedulerForPrefetchProcessing.shutdown(); monitor.close(); } } diff --git a/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/RedisDynoQueueTest.java b/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/RedisDynoQueueTest.java index 617b47e..112a26a 100644 --- a/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/RedisDynoQueueTest.java +++ b/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/RedisDynoQueueTest.java @@ -22,7 +22,6 @@ import static org.junit.Assert.assertTrue; import java.util.Arrays; -import java.util.Collection; import java.util.LinkedList; import java.util.List; import java.util.Map; From 0eaebf653b336aea9e2e8584ed39a2db6a979203 Mon Sep 17 00:00:00 2001 From: KowalczykBartek Date: Sat, 20 Oct 2018 00:21:13 +0200 Subject: [PATCH 22/91] Make v2 queue more safe --- .../queues/redis/v2/RedisPipelineQueue.java | 30 +++++++++---------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/v2/RedisPipelineQueue.java b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/v2/RedisPipelineQueue.java index 9e1998c..17092bd 100644 --- a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/v2/RedisPipelineQueue.java +++ b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/v2/RedisPipelineQueue.java @@ -49,35 +49,35 @@ public class RedisPipelineQueue implements DynoQueue { private final Logger logger = LoggerFactory.getLogger(RedisPipelineQueue.class); - private Clock clock; + private final Clock clock; - private String queueName; + private final String queueName; - private String shardName; + private final String shardName; - private String messageStoreKeyPrefix; + private final String messageStoreKeyPrefix; - private String myQueueShard; + private final String myQueueShard; - private String unackShardKeyPrefix; + private final String unackShardKeyPrefix; - private int unackTime = 60; + private final int unackTime; - private QueueMonitor monitor; + private final QueueMonitor monitor; - private ObjectMapper om; + private final ObjectMapper om; - private RedisConnection connPool; + private final RedisConnection connPool; - private RedisConnection nonQuorumPool; + private volatile RedisConnection nonQuorumPool; - private ScheduledExecutorService schedulerForUnacksProcessing; + private final ScheduledExecutorService schedulerForUnacksProcessing; - private HashPartitioner partitioner = new Murmur3HashPartitioner(); + private final HashPartitioner partitioner = new Murmur3HashPartitioner(); - private int maxHashBuckets = 32; + private final int maxHashBuckets = 32; - private int longPollWaitIntervalInMillis = 10; + private final int longPollWaitIntervalInMillis = 10; public RedisPipelineQueue(String redisKeyPrefix, String queueName, String shardName, int unackScheduleInMS, int unackTime, RedisConnection pool) { this(Clock.systemDefaultZone(), redisKeyPrefix, queueName, shardName, unackScheduleInMS, unackTime, pool); From bc8569e93bac95800bc295c57927ad17738144a2 Mon Sep 17 00:00:00 2001 From: Sailesh Mukil Date: Wed, 7 Nov 2018 09:06:44 -0800 Subject: [PATCH 23/91] Add information about operation and key in exception strings When exceptions are thrown, we don't have information about what operation failed and on what key. This patch attempts to add this information as part of the final exception string thrown to the application. Note that in some cases, we cannot get granular information about a specific shard that the operation failed on. This is due to how the code is strtuctured, and would require a bigger rehaul of the code to capture that information. This is a good starting point regardless to help diagnose issues when they happen. --- .../dyno/queues/redis/RedisDynoQueue.java | 39 +++++++++---------- 1 file changed, 19 insertions(+), 20 deletions(-) diff --git a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/RedisDynoQueue.java b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/RedisDynoQueue.java index ec789fc..20bccb3 100644 --- a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/RedisDynoQueue.java +++ b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/RedisDynoQueue.java @@ -160,8 +160,7 @@ public List push(final List messages) { Stopwatch sw = monitor.start(monitor.push, messages.size()); try { - - execute(() -> { + execute("push", "(a shard in) " + queueName, () -> { for (Message message : messages) { String json = om.writeValueAsString(message); quorumConn.hset(messageStoreKey, message.getId(), json); @@ -193,7 +192,7 @@ public List peek(final int messageCount) { return Collections.emptyList(); } - List msgs = execute(() -> { + List msgs = execute("peek", messageStoreKey, () -> { List messages = new LinkedList(); for (String id : ids) { String json = nonQuorumConn.hget(messageStoreKey, id); @@ -318,8 +317,7 @@ public boolean ack(String messageId) { Stopwatch sw = monitor.ack.start(); try { - - return execute(() -> { + return execute("ack", "(a shard in) " + queueName, () -> { for (String shard : allShards) { String unackShardKey = getUnackKey(queueName, shard); @@ -350,8 +348,7 @@ public boolean setUnackTimeout(String messageId, long timeout) { Stopwatch sw = monitor.ack.start(); try { - - return execute(() -> { + return execute("setUnackTimeout", "(a shard in) " + queueName, () -> { double unackScore = Long.valueOf(clock.millis() + timeout).doubleValue(); for (String shard : allShards) { @@ -373,7 +370,7 @@ public boolean setUnackTimeout(String messageId, long timeout) { @Override public boolean setTimeout(String messageId, long timeout) { - return execute(() -> { + return execute("setTimeout", "(a shard in) " + queueName, () -> { String json = nonQuorumConn.hget(messageStoreKey, messageId); if(json == null) { @@ -407,7 +404,7 @@ public boolean remove(String messageId) { try { - return execute(() -> { + return execute("remove", "(a shard in) " + queueName, () -> { for (String shard : allShards) { @@ -439,7 +436,7 @@ public Message get(String messageId) { try { - return execute(() -> { + return execute("get", messageStoreKey, () -> { String json = quorumConn.hget(messageStoreKey, messageId); if(json == null){ if (logger.isDebugEnabled()) { @@ -464,7 +461,7 @@ public long size() { try { - return execute(() -> { + return execute("size", "(a shard in) " + queueName, () -> { long size = 0; for (String shard : allShards) { size += nonQuorumConn.zcard(getQueueShardKey(queueName, shard)); @@ -484,7 +481,7 @@ public Map> shardSizes() { Map> shardSizes = new HashMap<>(); try { - return execute(() -> { + return execute("shardSizes", "(a shard in) " + queueName, () -> { for (String shard : allShards) { long size = nonQuorumConn.zcard(getQueueShardKey(queueName, shard)); long uacked = nonQuorumConn.zcard(getUnackKey(queueName, shard)); @@ -503,7 +500,7 @@ public Map> shardSizes() { @Override public void clear() { - execute(() -> { + execute("clear", "(a shard in) " + queueName, () -> { for (String shard : allShards) { String queueShard = getQueueShardKey(queueName, shard); String unackShard = getUnackKey(queueName, shard); @@ -518,7 +515,7 @@ public void clear() { private Set peekIds(int offset, int count) { - return execute(() -> { + return execute("peekIds", myQueueShard, () -> { double now = Long.valueOf(clock.millis() + 1).doubleValue(); Set scanned = quorumConn.zrangeByScore(myQueueShard, 0, now, offset, count); return scanned; @@ -535,7 +532,8 @@ public void processUnacks() { long queueDepth = size(); monitor.queueDepth.record(queueDepth); - execute(() -> { + String keyName = getUnackKey(queueName, shardName); + execute("processUnacks", keyName, () -> { int batchSize = 1_000; String unackQueueName = getUnackKey(queueName, shardName); @@ -579,11 +577,11 @@ private String getUnackKey(String queueName, String shard) { return redisKeyPrefix + ".UNACK." + queueName + "." + shard; } - private R execute(Callable r) { - return executeWithRetry(r, 0); + private R execute(String opName, String keyName, Callable r) { + return executeWithRetry(opName, keyName, r, 0); } - private R executeWithRetry(Callable r, int retryCount) { + private R executeWithRetry(String opName, String keyName, Callable r, int retryCount) { try { @@ -593,12 +591,13 @@ private R executeWithRetry(Callable r, int retryCount) { if (e.getCause() instanceof DynoException) { if (retryCount < this.retryCount) { - return executeWithRetry(r, ++retryCount); + return executeWithRetry(opName, keyName, r, ++retryCount); } } throw new RuntimeException(e.getCause()); } catch (Exception e) { - throw new RuntimeException(e); + throw new RuntimeException( + "Operation: ( " + opName + " ) failed on key: [" + keyName + " ].", e); } } From 7f93610ff8bab77dd9eaa0da270f3f1808d87ee9 Mon Sep 17 00:00:00 2001 From: David Wadden Date: Wed, 12 Dec 2018 22:43:14 -0800 Subject: [PATCH 24/91] Updates dyno to latest stable release --- dyno-queues-redis/build.gradle | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dyno-queues-redis/build.gradle b/dyno-queues-redis/build.gradle index d4d1ed9..d8a12f8 100644 --- a/dyno-queues-redis/build.gradle +++ b/dyno-queues-redis/build.gradle @@ -4,8 +4,8 @@ dependencies { compile "com.google.inject:guice:3.0" - compile 'com.netflix.dyno:dyno-core:1.6.5-rc.1' - compile 'com.netflix.dyno:dyno-jedis:1.6.5-rc.1' + compile 'com.netflix.dyno:dyno-core:1.6.5' + compile 'com.netflix.dyno:dyno-jedis:1.6.5' compile 'com.netflix.archaius:archaius-core:0.7.5' compile 'com.netflix.servo:servo-core:0.12.17' From f8b68a8529b53dfb7a30abd22c6eec86d2ae9470 Mon Sep 17 00:00:00 2001 From: saurabh garg Date: Thu, 28 Feb 2019 21:48:30 -0800 Subject: [PATCH 25/91] returning shard directly without assigning to variable --- .../netflix/dyno/queues/redis/sharding/RoundRobinStrategy.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/sharding/RoundRobinStrategy.java b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/sharding/RoundRobinStrategy.java index 5ad511d..0e2ea1e 100644 --- a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/sharding/RoundRobinStrategy.java +++ b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/sharding/RoundRobinStrategy.java @@ -37,7 +37,6 @@ public String getNextShard(List allShards, Message message) { nextShardIndex.set(0); index = 0; } - String shard = allShards.get(index); - return shard; + return allShards.get(index); } } From cf18218726d66e857a00998616d333ec0a1fd1e3 Mon Sep 17 00:00:00 2001 From: Raghuram Onti Srinivasan Date: Tue, 30 Apr 2019 10:02:37 -0700 Subject: [PATCH 26/91] Adding simple demo for queues --- dyno-queues-redis/build.gradle | 3 +- .../dyno/queues/demo/DynoQueueDemo.java | 85 +++++++++++++++++++ .../netflix/dyno/queues/redis/QueueUtils.java | 50 +++++++++++ .../dyno/queues/redis/RedisDynoQueue.java | 41 +-------- .../dyno/queues/redis/v2/MultiRedisQueue.java | 6 +- .../dyno/queues/redis/v2/QueueBuilder.java | 13 +-- .../queues/redis/v2/RedisPipelineQueue.java | 22 ++--- .../dyno/queues/shard/DynoShardSupplier.java | 16 ++-- .../src/main/resources/demo.properties | 32 +++++++ gradle/wrapper/gradle-wrapper.properties | 3 +- 10 files changed, 202 insertions(+), 69 deletions(-) create mode 100644 dyno-queues-redis/src/main/java/com/netflix/dyno/queues/demo/DynoQueueDemo.java create mode 100644 dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/QueueUtils.java create mode 100644 dyno-queues-redis/src/main/resources/demo.properties diff --git a/dyno-queues-redis/build.gradle b/dyno-queues-redis/build.gradle index d8a12f8..1f4d40b 100644 --- a/dyno-queues-redis/build.gradle +++ b/dyno-queues-redis/build.gradle @@ -1,11 +1,12 @@ dependencies { compile project(':dyno-queues-core') - + compile "com.google.inject:guice:3.0" compile 'com.netflix.dyno:dyno-core:1.6.5' compile 'com.netflix.dyno:dyno-jedis:1.6.5' + compile 'com.netflix.dyno:dyno-demo:1.6.5' compile 'com.netflix.archaius:archaius-core:0.7.5' compile 'com.netflix.servo:servo-core:0.12.17' diff --git a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/demo/DynoQueueDemo.java b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/demo/DynoQueueDemo.java new file mode 100644 index 0000000..94dddf8 --- /dev/null +++ b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/demo/DynoQueueDemo.java @@ -0,0 +1,85 @@ +package com.netflix.dyno.queues.demo; + +import com.netflix.dyno.demo.redis.DynoJedisDemo; +import com.netflix.dyno.jedis.DynoJedisClient; +import com.netflix.dyno.queues.DynoQueue; +import com.netflix.dyno.queues.Message; +import com.netflix.dyno.queues.redis.v2.QueueBuilder; +import com.netflix.dyno.queues.shard.DynoShardSupplier; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.util.Arrays; +import java.util.List; +import java.util.Properties; +import java.util.concurrent.TimeUnit; + +public class DynoQueueDemo extends DynoJedisDemo { + + private static final Logger logger = LoggerFactory.getLogger(DynoQueue.class); + public DynoQueueDemo(String clusterName, String localRack) { + super(clusterName, localRack); + } + + public DynoQueueDemo(String primaryCluster, String shadowCluster, String localRack) { + super(primaryCluster, shadowCluster, localRack); + } + + public static void main(String[] args) throws IOException { + final String clusterName = args[0]; + + final DynoQueueDemo demo = new DynoQueueDemo(clusterName, "us-east-1e"); + int numCounters = (args.length == 2) ? Integer.valueOf(args[1]) : 1; + Properties props = new Properties(); + props.load(DynoQueueDemo.class.getResourceAsStream("/demo.properties")); + for (String name : props.stringPropertyNames()) { + System.setProperty(name, props.getProperty(name)); + } + + try { + demo.initWithRemoteClusterFromEurekaUrl(args[0], 8102); + + demo.runSimpleQueueDemo(demo.client); + Thread.sleep(10000); + + } catch (Exception ex) { + ex.printStackTrace(); + } finally { + demo.stop(); + logger.info("Done"); + } + } + + private void runQueueDemoWithTimeOut(DynoJedisClient dyno) throws IOException { + + } + + private void runSimpleQueueDemo(DynoJedisClient dyno) throws IOException { + String region = System.getProperty("LOCAL_DATACENTER"); + String localRack = System.getProperty("LOCAL_RACK"); + + String prefix = "dynoQueue_"; + + + DynoShardSupplier ss = new DynoShardSupplier(dyno.getConnPool().getConfiguration().getHostSupplier(), region, localRack); + + DynoQueue queue = new QueueBuilder() + .setQueueName("test") + .setCurrentShard(ss.getCurrentShard()) + .setRedisKeyPrefix(prefix) + .useDynomite(dyno, dyno, dyno.getConnPool().getConfiguration().getHostSupplier()) + .setUnackTime(50_000) + .build(); + + Message msg = new Message("id1", "message payload"); + queue.push(Arrays.asList(msg)); + + int count = 10; + List polled = queue.pop(count, 1, TimeUnit.SECONDS); + logger.info(polled.toString()); + + queue.ack("id1"); + queue.close(); + } +} diff --git a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/QueueUtils.java b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/QueueUtils.java new file mode 100644 index 0000000..0eaf775 --- /dev/null +++ b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/QueueUtils.java @@ -0,0 +1,50 @@ +package com.netflix.dyno.queues.redis; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.SerializationFeature; +import com.netflix.dyno.connectionpool.exception.DynoException; + +import java.util.concurrent.Callable; +import java.util.concurrent.ExecutionException; + +public class QueueUtils { + + private static final int retryCount = 2; + + public static R execute(String opName, String keyName, Callable r) { + return executeWithRetry(opName, keyName, r, 0); + } + + public static R executeWithRetry(String opName, String keyName, Callable r, int retryNum) { + + try { + + return r.call(); + + } catch (ExecutionException e) { + + if (e.getCause() instanceof DynoException) { + if (retryNum < retryCount) { + return executeWithRetry(opName, keyName, r, ++retryNum); + } + } + throw new RuntimeException(e.getCause()); + } catch (Exception e) { + throw new RuntimeException( + "Operation: ( " + opName + " ) failed on key: [" + keyName + " ].", e); + } + } + + public static ObjectMapper constructObjectMapper() { + ObjectMapper om = new ObjectMapper(); + om.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); + om.configure(DeserializationFeature.FAIL_ON_IGNORED_PROPERTIES, false); + om.configure(DeserializationFeature.FAIL_ON_NULL_FOR_PRIMITIVES, false); + om.setSerializationInclusion(JsonInclude.Include.NON_NULL); + om.setSerializationInclusion(JsonInclude.Include.NON_EMPTY); + om.disable(SerializationFeature.INDENT_OUTPUT); + return om; + } +} diff --git a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/RedisDynoQueue.java b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/RedisDynoQueue.java index 20bccb3..fa76698 100644 --- a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/RedisDynoQueue.java +++ b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/RedisDynoQueue.java @@ -15,14 +15,10 @@ */ package com.netflix.dyno.queues.redis; -import com.fasterxml.jackson.annotation.JsonInclude.Include; -import com.fasterxml.jackson.databind.DeserializationFeature; import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.SerializationFeature; import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.ImmutableList; import com.google.common.util.concurrent.Uninterruptibles; -import com.netflix.dyno.connectionpool.exception.DynoException; import com.netflix.dyno.queues.DynoQueue; import com.netflix.dyno.queues.Message; import com.netflix.dyno.queues.redis.sharding.ShardingStrategy; @@ -41,15 +37,15 @@ import java.util.List; import java.util.Map; import java.util.Set; -import java.util.concurrent.Callable; import java.util.concurrent.ConcurrentLinkedQueue; -import java.util.concurrent.ExecutionException; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Collectors; +import static com.netflix.dyno.queues.redis.QueueUtils.execute; + /** * * @author Viren @@ -110,15 +106,8 @@ public RedisDynoQueue(Clock clock, String redisKeyPrefix, String queueName, Set< this.myQueueShard = getQueueShardKey(queueName, shardName); this.shardingStrategy = shardingStrategy; - ObjectMapper om = new ObjectMapper(); - om.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); - om.configure(DeserializationFeature.FAIL_ON_IGNORED_PROPERTIES, false); - om.configure(DeserializationFeature.FAIL_ON_NULL_FOR_PRIMITIVES, false); - om.setSerializationInclusion(Include.NON_NULL); - om.setSerializationInclusion(Include.NON_EMPTY); - om.disable(SerializationFeature.INDENT_OUTPUT); - this.om = om; + this.om = QueueUtils.constructObjectMapper(); this.monitor = new QueueMonitor(queueName, shardName); this.prefetchedIds = new ConcurrentLinkedQueue<>(); @@ -577,30 +566,6 @@ private String getUnackKey(String queueName, String shard) { return redisKeyPrefix + ".UNACK." + queueName + "." + shard; } - private R execute(String opName, String keyName, Callable r) { - return executeWithRetry(opName, keyName, r, 0); - } - - private R executeWithRetry(String opName, String keyName, Callable r, int retryCount) { - - try { - - return r.call(); - - } catch (ExecutionException e) { - - if (e.getCause() instanceof DynoException) { - if (retryCount < this.retryCount) { - return executeWithRetry(opName, keyName, r, ++retryCount); - } - } - throw new RuntimeException(e.getCause()); - } catch (Exception e) { - throw new RuntimeException( - "Operation: ( " + opName + " ) failed on key: [" + keyName + " ].", e); - } - } - @Override public void close() throws IOException { schedulerForUnacksProcessing.shutdown(); diff --git a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/v2/MultiRedisQueue.java b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/v2/MultiRedisQueue.java index ac0d607..08000dc 100644 --- a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/v2/MultiRedisQueue.java +++ b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/v2/MultiRedisQueue.java @@ -15,6 +15,9 @@ */ package com.netflix.dyno.queues.redis.v2; +import com.netflix.dyno.queues.DynoQueue; +import com.netflix.dyno.queues.Message; + import java.io.IOException; import java.util.HashMap; import java.util.LinkedList; @@ -25,9 +28,6 @@ import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Collectors; -import com.netflix.dyno.queues.DynoQueue; -import com.netflix.dyno.queues.Message; - /** * @author Viren * MultiRedisQueue exposes a single queue using multiple redis queues. Each RedisQueue is a shard. diff --git a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/v2/QueueBuilder.java b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/v2/QueueBuilder.java index 0195312..23ca6da 100644 --- a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/v2/QueueBuilder.java +++ b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/v2/QueueBuilder.java @@ -18,9 +18,6 @@ */ package com.netflix.dyno.queues.redis.v2; -import java.time.Clock; -import java.util.*; - import com.google.common.base.Function; import com.google.common.collect.Collections2; import com.google.common.collect.Lists; @@ -37,10 +34,16 @@ import com.netflix.dyno.queues.redis.conn.DynoClientProxy; import com.netflix.dyno.queues.redis.conn.JedisProxy; import com.netflix.dyno.queues.redis.conn.RedisConnection; - import redis.clients.jedis.JedisPool; import redis.clients.jedis.JedisPoolConfig; +import java.time.Clock; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + /** * @author Viren * Builder for the queues. @@ -58,7 +61,7 @@ public class QueueBuilder { private String currentShard; - private Function hostToShardMap; + private Function hostToShardMap = (Host h) -> h.getRack(); private HostSupplier hs; diff --git a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/v2/RedisPipelineQueue.java b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/v2/RedisPipelineQueue.java index 17092bd..d0f1417 100644 --- a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/v2/RedisPipelineQueue.java +++ b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/v2/RedisPipelineQueue.java @@ -15,15 +15,13 @@ */ package com.netflix.dyno.queues.redis.v2; -import com.fasterxml.jackson.annotation.JsonInclude.Include; -import com.fasterxml.jackson.databind.DeserializationFeature; import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.SerializationFeature; import com.netflix.dyno.connectionpool.HashPartitioner; import com.netflix.dyno.connectionpool.impl.hash.Murmur3HashPartitioner; import com.netflix.dyno.queues.DynoQueue; import com.netflix.dyno.queues.Message; import com.netflix.dyno.queues.redis.QueueMonitor; +import com.netflix.dyno.queues.redis.QueueUtils; import com.netflix.dyno.queues.redis.conn.Pipe; import com.netflix.dyno.queues.redis.conn.RedisConnection; import com.netflix.servo.monitor.Stopwatch; @@ -35,7 +33,13 @@ import java.io.IOException; import java.time.Clock; -import java.util.*; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Set; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; @@ -96,15 +100,7 @@ public RedisPipelineQueue(Clock clock, String redisKeyPrefix, String queue, Stri this.connPool = pool; this.nonQuorumPool = pool; - ObjectMapper om = new ObjectMapper(); - om.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); - om.configure(DeserializationFeature.FAIL_ON_IGNORED_PROPERTIES, false); - om.configure(DeserializationFeature.FAIL_ON_NULL_FOR_PRIMITIVES, false); - om.setSerializationInclusion(Include.NON_NULL); - om.setSerializationInclusion(Include.NON_EMPTY); - om.disable(SerializationFeature.INDENT_OUTPUT); - - this.om = om; + this.om = QueueUtils.constructObjectMapper(); this.monitor = new QueueMonitor(qName, shardName); schedulerForUnacksProcessing = Executors.newScheduledThreadPool(1); diff --git a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/shard/DynoShardSupplier.java b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/shard/DynoShardSupplier.java index c1a43d2..34f7450 100644 --- a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/shard/DynoShardSupplier.java +++ b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/shard/DynoShardSupplier.java @@ -18,12 +18,12 @@ */ package com.netflix.dyno.queues.shard; -import java.util.Set; -import java.util.stream.Collectors; - import com.netflix.dyno.connectionpool.HostSupplier; import com.netflix.dyno.queues.ShardSupplier; +import java.util.Set; +import java.util.stream.Collectors; + /** * @author Viren * @@ -34,23 +34,23 @@ public class DynoShardSupplier implements ShardSupplier { private String region; - private String localDC; + private String localRack; /** * Dynomite based shard supplier. Keeps the number of shards in parity with the hosts and regions * @param hs Host supplier * @param region current region - * @param localDC local data center identifier + * @param localRack local data center identifier */ - public DynoShardSupplier(HostSupplier hs, String region, String localDC) { + public DynoShardSupplier(HostSupplier hs, String region, String localRack) { this.hs = hs; this.region = region; - this.localDC = localDC; + this.localRack = localRack; } @Override public String getCurrentShard() { - return localDC; + return localRack; } @Override diff --git a/dyno-queues-redis/src/main/resources/demo.properties b/dyno-queues-redis/src/main/resources/demo.properties new file mode 100644 index 0000000..66c524b --- /dev/null +++ b/dyno-queues-redis/src/main/resources/demo.properties @@ -0,0 +1,32 @@ +#### +## Properties to initialize the demo app +# +LOCAL_DATACENTER=us-east-1 +LOCAL_RACK=us-east-1c +NETFLIX_STACK=dyno_demo +#EC2_AVAILABILITY_ZONE=us-east-1 + +dyno.demo.lbStrategy=TokenAware + +netflix.appinfo.name=dyno_demo +netflix.environment=test +netflix.appinfo.metadata.enableRoute53=false +netflix.discovery.registration.enabled=false +netflix.appinfo.validateInstanceId=false + +dyno.demo.retryPolicy=RetryNTimes:2 +dyno.demo.port=8102 +dyno.demo.discovery.prod=discoveryreadonly.%s.dynprod.netflix.net:7001/v2/apps +dyno.demo.discovery.test=discoveryreadonly.%s.dyntest.netflix.net:7001/v2/apps + +## +# These properties apply to runScanTest(). Note that .prefix will apply only if .populateKeys is true. To query for a +# pattern w/out writing anything to the cluster, ensure .populateKeys is false and set the pattern. +dyno.demo.scan.populateKeys=false +dyno.demo.scan.key.prefix=DynoClientTest_key- +dyno.demo.scan.key.pattern=DynoClientTest_key-* + +## Uncomment for hosts file usage +#dyno.demo.hostsFile=file:/// + + diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index fb7ef98..6daac93 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,5 +1,6 @@ +#Wed Apr 24 09:46:53 PDT 2019 distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-4.10.2-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-4.10.2-all.zip From 2a2a3e44d9a837b551a3fc1813518f2aeec655f2 Mon Sep 17 00:00:00 2001 From: Sailesh Mukil Date: Tue, 30 Apr 2019 09:53:30 -0700 Subject: [PATCH 27/91] Create ensure() API No-op if the message exists either in the queue or the unack set. Adds the message to the queue otherwise with a timeout. --- .../com/netflix/dyno/queues/DynoQueue.java | 7 ++++++ .../dyno/queues/redis/RedisDynoQueue.java | 23 +++++++++++++++++++ .../dyno/queues/redis/v2/MultiRedisQueue.java | 6 +++++ .../queues/redis/v2/RedisPipelineQueue.java | 6 +++++ 4 files changed, 42 insertions(+) diff --git a/dyno-queues-core/src/main/java/com/netflix/dyno/queues/DynoQueue.java b/dyno-queues-core/src/main/java/com/netflix/dyno/queues/DynoQueue.java index 398ce91..74015b7 100644 --- a/dyno-queues-core/src/main/java/com/netflix/dyno/queues/DynoQueue.java +++ b/dyno-queues-core/src/main/java/com/netflix/dyno/queues/DynoQueue.java @@ -108,6 +108,13 @@ public interface DynoQueue extends Closeable { */ public boolean remove(String messageId); + /** + * Enqueues 'message' if it doesn't exist in any of the shards or unack sets. + * + * @param message Message to enqueue if it doesn't exist. + * @return true if message was enqueued. False if messageId already exists. + */ + public boolean ensure(Message message); /** * diff --git a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/RedisDynoQueue.java b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/RedisDynoQueue.java index 20bccb3..96ab3ce 100644 --- a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/RedisDynoQueue.java +++ b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/RedisDynoQueue.java @@ -429,6 +429,29 @@ public boolean remove(String messageId) { } } + @Override + public boolean ensure(Message message) { + return execute("ensure", "(a shard in) " + queueName, () -> { + + String messageId = message.getId(); + for (String shard : allShards) { + + String queueShard = getQueueShardKey(queueName, shard); + Double score = quorumConn.zscore(queueShard, messageId); + if(score != null) { + return false; + } + String unackShardKey = getUnackKey(queueName, shard); + score = quorumConn.zscore(unackShardKey, messageId); + if(score != null) { + return false; + } + } + push(Collections.singletonList(message)); + return true; + }); + } + @Override public Message get(String messageId) { diff --git a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/v2/MultiRedisQueue.java b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/v2/MultiRedisQueue.java index ac0d607..f6e8500 100644 --- a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/v2/MultiRedisQueue.java +++ b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/v2/MultiRedisQueue.java @@ -16,6 +16,7 @@ package com.netflix.dyno.queues.redis.v2; import java.io.IOException; +import java.lang.UnsupportedOperationException; import java.util.HashMap; import java.util.LinkedList; import java.util.List; @@ -141,6 +142,11 @@ public boolean remove(String messageId) { return false; } + @Override + public boolean ensure(Message message) { + throw new UnsupportedOperationException(); + } + @Override public Message get(String messageId) { for (DynoQueue q : queues.values()) { diff --git a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/v2/RedisPipelineQueue.java b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/v2/RedisPipelineQueue.java index 17092bd..6520038 100644 --- a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/v2/RedisPipelineQueue.java +++ b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/v2/RedisPipelineQueue.java @@ -34,6 +34,7 @@ import redis.clients.jedis.params.sortedset.ZAddParams; import java.io.IOException; +import java.lang.UnsupportedOperationException; import java.time.Clock; import java.util.*; import java.util.concurrent.Executors; @@ -461,6 +462,11 @@ public boolean remove(String messageId) { } } + @Override + public boolean ensure(Message message) { + throw new UnsupportedOperationException(); + } + @Override public Message get(String messageId) { From d090cb2bb3181a2a9ac5abe7af83c97c843c86ea Mon Sep 17 00:00:00 2001 From: Raghuram Onti Srinivasan Date: Tue, 30 Apr 2019 10:55:51 -0700 Subject: [PATCH 28/91] Trying to fix flaky test --- .../dyno/queues/redis/RedisDynoQueueTest.java | 35 +++++++++---------- 1 file changed, 17 insertions(+), 18 deletions(-) diff --git a/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/RedisDynoQueueTest.java b/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/RedisDynoQueueTest.java index 112a26a..5976e92 100644 --- a/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/RedisDynoQueueTest.java +++ b/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/RedisDynoQueueTest.java @@ -15,11 +15,17 @@ */ package com.netflix.dyno.queues.redis; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertTrue; +import com.google.common.util.concurrent.Uninterruptibles; +import com.netflix.dyno.connectionpool.Host; +import com.netflix.dyno.connectionpool.Host.Status; +import com.netflix.dyno.connectionpool.HostSupplier; +import com.netflix.dyno.queues.DynoQueue; +import com.netflix.dyno.queues.Message; +import com.netflix.dyno.queues.ShardSupplier; +import com.netflix.dyno.queues.jedis.JedisMock; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; import java.util.Arrays; import java.util.LinkedList; @@ -37,18 +43,11 @@ import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Collectors; -import org.junit.Before; -import org.junit.BeforeClass; -import org.junit.Test; - -import com.google.common.util.concurrent.Uninterruptibles; -import com.netflix.dyno.connectionpool.Host; -import com.netflix.dyno.connectionpool.Host.Status; -import com.netflix.dyno.connectionpool.HostSupplier; -import com.netflix.dyno.queues.DynoQueue; -import com.netflix.dyno.queues.Message; -import com.netflix.dyno.queues.ShardSupplier; -import com.netflix.dyno.queues.jedis.JedisMock; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; public class RedisDynoQueueTest { @@ -238,7 +237,7 @@ public void run() { long start = System.currentTimeMillis(); List more = rdq.pop(1, 1, TimeUnit.SECONDS); long elapsedTime = System.currentTimeMillis() - start; - assertTrue(elapsedTime > 1000); + assertTrue(elapsedTime >= 1000); assertEquals(0, more.size()); assertEquals(0, rdq.prefetch.get()); From a69810c47b2d37c21497720b218cdd733657f927 Mon Sep 17 00:00:00 2001 From: Raghuram Onti Srinivasan Date: Tue, 30 Apr 2019 14:10:24 -0700 Subject: [PATCH 29/91] Fixes from comments --- .../dyno/queues/demo/DynoQueueDemo.java | 8 ++------ .../netflix/dyno/queues/redis/QueueUtils.java | 17 ++++++++++++++++- .../dyno/queues/shard/DynoShardSupplier.java | 2 +- .../src/main/resources/demo.properties | 18 ------------------ 4 files changed, 19 insertions(+), 26 deletions(-) diff --git a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/demo/DynoQueueDemo.java b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/demo/DynoQueueDemo.java index 94dddf8..05ee6f6 100644 --- a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/demo/DynoQueueDemo.java +++ b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/demo/DynoQueueDemo.java @@ -40,7 +40,7 @@ public static void main(String[] args) throws IOException { try { demo.initWithRemoteClusterFromEurekaUrl(args[0], 8102); - demo.runSimpleQueueDemo(demo.client); + demo.runSimpleV2QueueDemo(demo.client); Thread.sleep(10000); } catch (Exception ex) { @@ -51,11 +51,7 @@ public static void main(String[] args) throws IOException { } } - private void runQueueDemoWithTimeOut(DynoJedisClient dyno) throws IOException { - - } - - private void runSimpleQueueDemo(DynoJedisClient dyno) throws IOException { + private void runSimpleV2QueueDemo(DynoJedisClient dyno) throws IOException { String region = System.getProperty("LOCAL_DATACENTER"); String localRack = System.getProperty("LOCAL_RACK"); diff --git a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/QueueUtils.java b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/QueueUtils.java index 0eaf775..b3c2c5f 100644 --- a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/QueueUtils.java +++ b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/QueueUtils.java @@ -9,15 +9,26 @@ import java.util.concurrent.Callable; import java.util.concurrent.ExecutionException; +/** + * Helper class to consolidate functions which might be reused across different DynoQueue implementations. + */ public class QueueUtils { private static final int retryCount = 2; + /** + * Execute function with retries if required + * @param opName + * @param keyName + * @param r + * @param + * @return + */ public static R execute(String opName, String keyName, Callable r) { return executeWithRetry(opName, keyName, r, 0); } - public static R executeWithRetry(String opName, String keyName, Callable r, int retryNum) { + private static R executeWithRetry(String opName, String keyName, Callable r, int retryNum) { try { @@ -37,6 +48,10 @@ public static R executeWithRetry(String opName, String keyName, Callable } } + /** + * Construct standard objectmapper to use within the DynoQueue instances to read/write Message objects + * @return + */ public static ObjectMapper constructObjectMapper() { ObjectMapper om = new ObjectMapper(); om.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); diff --git a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/shard/DynoShardSupplier.java b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/shard/DynoShardSupplier.java index 34f7450..5c14e79 100644 --- a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/shard/DynoShardSupplier.java +++ b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/shard/DynoShardSupplier.java @@ -40,7 +40,7 @@ public class DynoShardSupplier implements ShardSupplier { * Dynomite based shard supplier. Keeps the number of shards in parity with the hosts and regions * @param hs Host supplier * @param region current region - * @param localRack local data center identifier + * @param localRack local rack identifier */ public DynoShardSupplier(HostSupplier hs, String region, String localRack) { this.hs = hs; diff --git a/dyno-queues-redis/src/main/resources/demo.properties b/dyno-queues-redis/src/main/resources/demo.properties index 66c524b..d171cff 100644 --- a/dyno-queues-redis/src/main/resources/demo.properties +++ b/dyno-queues-redis/src/main/resources/demo.properties @@ -8,25 +8,7 @@ NETFLIX_STACK=dyno_demo dyno.demo.lbStrategy=TokenAware -netflix.appinfo.name=dyno_demo -netflix.environment=test -netflix.appinfo.metadata.enableRoute53=false -netflix.discovery.registration.enabled=false -netflix.appinfo.validateInstanceId=false - dyno.demo.retryPolicy=RetryNTimes:2 dyno.demo.port=8102 dyno.demo.discovery.prod=discoveryreadonly.%s.dynprod.netflix.net:7001/v2/apps dyno.demo.discovery.test=discoveryreadonly.%s.dyntest.netflix.net:7001/v2/apps - -## -# These properties apply to runScanTest(). Note that .prefix will apply only if .populateKeys is true. To query for a -# pattern w/out writing anything to the cluster, ensure .populateKeys is false and set the pattern. -dyno.demo.scan.populateKeys=false -dyno.demo.scan.key.prefix=DynoClientTest_key- -dyno.demo.scan.key.pattern=DynoClientTest_key-* - -## Uncomment for hosts file usage -#dyno.demo.hostsFile=file:/// - - From 095f39412861c29a0c9a4b7eb981cc084a585e7e Mon Sep 17 00:00:00 2001 From: Raghuram Onti Srinivasan Date: Thu, 2 May 2019 14:07:31 -0700 Subject: [PATCH 30/91] Added doc comment --- .../java/com/netflix/dyno/queues/demo/DynoQueueDemo.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/demo/DynoQueueDemo.java b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/demo/DynoQueueDemo.java index 05ee6f6..7683e81 100644 --- a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/demo/DynoQueueDemo.java +++ b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/demo/DynoQueueDemo.java @@ -26,6 +26,11 @@ public DynoQueueDemo(String primaryCluster, String shadowCluster, String localRa super(primaryCluster, shadowCluster, localRack); } + /** + * Provide the cluster name to connect to as an argument to the function. + * throws java.lang.RuntimeException: java.net.ConnectException: Connection timed out (Connection timed out) + * if the cluster is not reachable. + */ public static void main(String[] args) throws IOException { final String clusterName = args[0]; From e080716407c9905e199b605069ee87c948a11d89 Mon Sep 17 00:00:00 2001 From: Sailesh Mukil Date: Thu, 2 May 2019 18:49:01 -0700 Subject: [PATCH 31/91] Create containsPredicate() API Checks the message bodies (i.e. the data in the hash map), and returns true on the first match with 'predicate'. --- .../com/netflix/dyno/queues/DynoQueue.java | 14 +++++++- .../dyno/queues/redis/RedisDynoQueue.java | 35 +++++++++++++++++++ .../dyno/queues/redis/v2/MultiRedisQueue.java | 3 ++ .../queues/redis/v2/RedisPipelineQueue.java | 3 ++ 4 files changed, 54 insertions(+), 1 deletion(-) diff --git a/dyno-queues-core/src/main/java/com/netflix/dyno/queues/DynoQueue.java b/dyno-queues-core/src/main/java/com/netflix/dyno/queues/DynoQueue.java index 74015b7..a6b3fd3 100644 --- a/dyno-queues-core/src/main/java/com/netflix/dyno/queues/DynoQueue.java +++ b/dyno-queues-core/src/main/java/com/netflix/dyno/queues/DynoQueue.java @@ -115,7 +115,19 @@ public interface DynoQueue extends Closeable { * @return true if message was enqueued. False if messageId already exists. */ public boolean ensure(Message message); - + + /** + * Checks the message bodies (i.e. the data in the hash map), and returns true on the first match with + * 'predicate'. + * + * Disclaimer: This is a potentially expensive call, since we will iterate over the entire hash map in the + * worst case. Use mindfully. + * + * @param predicate The predicate to check against. + * @return 'true' if any of the messages contain 'predicate'; 'false' otherwise. + */ + public boolean containsPredicate(String predicate); + /** * * @param messageId message to be retrieved. diff --git a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/RedisDynoQueue.java b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/RedisDynoQueue.java index 1c2a7ed..08794ba 100644 --- a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/RedisDynoQueue.java +++ b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/RedisDynoQueue.java @@ -26,6 +26,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import redis.clients.jedis.JedisCommands; +import redis.clients.jedis.Jedis; import redis.clients.jedis.Tuple; import redis.clients.jedis.params.sortedset.ZAddParams; @@ -441,6 +442,40 @@ public boolean ensure(Message message) { }); } + @Override + public boolean containsPredicate(String predicate) { + return execute("containsPredicate", messageStoreKey, () -> { + + // We use a Lua script here to do predicate matching since we only want to find whether the predicate + // exists in any of the message bodies or not, and the only way to do that is to check for the predicate + // match on the server side. + // The alternative is to have the server return all the hash values back to us and we filter it here on + // the client side. This is not desirable since we would potentially be sending large amounts of data + // over the network only to return a 'true' or 'false' value back to the calling application. + String predicateCheckLuaScript = "local hkey=KEYS[1]\n" + + "local predicate=ARGV[1]\n" + + "local cursor=0\n" + + "local begin=false\n" + + "while (cursor ~= 0 or begin==false) do\n" + + " local ret = redis.call('hscan', hkey, cursor)\n" + + " for i, content in ipairs(ret[2]) do\n" + + " if (string.find(content, predicate)) then\n" + + " return 1\n" + + " end\n" + + " end\n" + + " cursor=tonumber(ret[1])\n" + + " begin=true\n" + + "end\n" + + "return 0"; + + // Cast from 'JedisCommands' to 'Jedis' here since the former does not expose 'eval()'. + int retval = (int) ((Jedis)quorumConn).eval(predicateCheckLuaScript, + Collections.singletonList(messageStoreKey), Collections.singletonList(predicate)); + + return (retval == 1); + }); + } + @Override public Message get(String messageId) { diff --git a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/v2/MultiRedisQueue.java b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/v2/MultiRedisQueue.java index 6acfa98..09dc77c 100644 --- a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/v2/MultiRedisQueue.java +++ b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/v2/MultiRedisQueue.java @@ -147,6 +147,9 @@ public boolean ensure(Message message) { throw new UnsupportedOperationException(); } + @Override + public boolean containsPredicate(String predicate) { throw new UnsupportedOperationException(); } + @Override public Message get(String messageId) { for (DynoQueue q : queues.values()) { diff --git a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/v2/RedisPipelineQueue.java b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/v2/RedisPipelineQueue.java index db9f38b..cfb28a3 100644 --- a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/v2/RedisPipelineQueue.java +++ b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/v2/RedisPipelineQueue.java @@ -463,6 +463,9 @@ public boolean ensure(Message message) { throw new UnsupportedOperationException(); } + @Override + public boolean containsPredicate(String predicate) { throw new UnsupportedOperationException(); } + @Override public Message get(String messageId) { From d3c5506c99a371e9668ae0f54c24c410afb1d092 Mon Sep 17 00:00:00 2001 From: Sailesh Mukil Date: Tue, 7 May 2019 16:02:57 -0700 Subject: [PATCH 32/91] Cast from 'JedisCommands' to 'DynoJedisClient' instead of 'Jedis' in containsPredicate() The dyno-queues are always built using a 'DynoJedisClient' object and not a 'Jedis' object. This patch fixes a bug in the previous patch that would throw a runtime error if a DynoJedisClient object was used to build the queue. --- .../java/com/netflix/dyno/queues/redis/RedisDynoQueue.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/RedisDynoQueue.java b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/RedisDynoQueue.java index 08794ba..a683a83 100644 --- a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/RedisDynoQueue.java +++ b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/RedisDynoQueue.java @@ -19,6 +19,7 @@ import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.ImmutableList; import com.google.common.util.concurrent.Uninterruptibles; +import com.netflix.dyno.jedis.DynoJedisClient; import com.netflix.dyno.queues.DynoQueue; import com.netflix.dyno.queues.Message; import com.netflix.dyno.queues.redis.sharding.ShardingStrategy; @@ -468,8 +469,8 @@ public boolean containsPredicate(String predicate) { "end\n" + "return 0"; - // Cast from 'JedisCommands' to 'Jedis' here since the former does not expose 'eval()'. - int retval = (int) ((Jedis)quorumConn).eval(predicateCheckLuaScript, + // Cast from 'JedisCommands' to 'DynoJedisClient' here since the former does not expose 'eval()'. + int retval = (int) ((DynoJedisClient)quorumConn).eval(predicateCheckLuaScript, Collections.singletonList(messageStoreKey), Collections.singletonList(predicate)); return (retval == 1); From 6506c4d4e5a553930f71868ea48a118a8d37c802 Mon Sep 17 00:00:00 2001 From: Sailesh Mukil Date: Tue, 7 May 2019 16:15:21 -0700 Subject: [PATCH 33/91] DynoShardSupplier should return consistent results through its APIs The getQueueShards() call returns all the shards with the region filtered out from the strings in the final list. However, the getCurrentShard() API returns the shard name as is. This can cause items to get lost in the queue depending on how the regions and racks (which the shards derive their names from) are supplied. This patch fixes the above issue and makes both the APIs consistent with their results. --- .../java/com/netflix/dyno/queues/shard/DynoShardSupplier.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/shard/DynoShardSupplier.java b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/shard/DynoShardSupplier.java index 5c14e79..59619cd 100644 --- a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/shard/DynoShardSupplier.java +++ b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/shard/DynoShardSupplier.java @@ -50,7 +50,7 @@ public DynoShardSupplier(HostSupplier hs, String region, String localRack) { @Override public String getCurrentShard() { - return localRack; + return localRack.replaceAll(region, ""); } @Override From 016be83c8303c811705d1a7bb96dc664feec4417 Mon Sep 17 00:00:00 2001 From: Sailesh Mukil Date: Tue, 7 May 2019 15:59:09 -0700 Subject: [PATCH 34/91] Add a demo for the V1 queue --- .../dyno/queues/demo/DynoQueueDemo.java | 56 ++++++++++++++++++- .../dyno/queues/redis/RedisDynoQueue.java | 2 +- 2 files changed, 55 insertions(+), 3 deletions(-) diff --git a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/demo/DynoQueueDemo.java b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/demo/DynoQueueDemo.java index 7683e81..6b5bd4c 100644 --- a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/demo/DynoQueueDemo.java +++ b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/demo/DynoQueueDemo.java @@ -1,7 +1,9 @@ package com.netflix.dyno.queues.demo; +import com.google.common.collect.ImmutableList; import com.netflix.dyno.demo.redis.DynoJedisDemo; import com.netflix.dyno.jedis.DynoJedisClient; +import com.netflix.dyno.queues.redis.RedisQueues; import com.netflix.dyno.queues.DynoQueue; import com.netflix.dyno.queues.Message; import com.netflix.dyno.queues.redis.v2.QueueBuilder; @@ -10,6 +12,7 @@ import org.slf4j.LoggerFactory; import java.io.IOException; +import java.sql.Time; import java.util.Arrays; import java.util.List; import java.util.Properties; @@ -30,12 +33,23 @@ public DynoQueueDemo(String primaryCluster, String shadowCluster, String localRa * Provide the cluster name to connect to as an argument to the function. * throws java.lang.RuntimeException: java.net.ConnectException: Connection timed out (Connection timed out) * if the cluster is not reachable. + * + * @param args: + * + * + * cluster-name: Name of cluster to run demo against + * version: Possible values = 1 or 2; (for V1 or V2) + * */ public static void main(String[] args) throws IOException { final String clusterName = args[0]; + if (args.length < 2) { + throw new IllegalArgumentException("Need to pass in cluster-name and version of dyno-queues to run as arguments"); + } + + int version = Integer.parseInt(args[1]); final DynoQueueDemo demo = new DynoQueueDemo(clusterName, "us-east-1e"); - int numCounters = (args.length == 2) ? Integer.valueOf(args[1]) : 1; Properties props = new Properties(); props.load(DynoQueueDemo.class.getResourceAsStream("/demo.properties")); for (String name : props.stringPropertyNames()) { @@ -45,7 +59,11 @@ public static void main(String[] args) throws IOException { try { demo.initWithRemoteClusterFromEurekaUrl(args[0], 8102); - demo.runSimpleV2QueueDemo(demo.client); + if (version == 1) { + demo.runSimpleV1Demo(demo.client); + } else if (version == 2) { + demo.runSimpleV2QueueDemo(demo.client); + } Thread.sleep(10000); } catch (Exception ex) { @@ -56,6 +74,40 @@ public static void main(String[] args) throws IOException { } } + private void runSimpleV1Demo(DynoJedisClient dyno) throws IOException { + String region = System.getProperty("LOCAL_DATACENTER"); + String localRack = System.getProperty("LOCAL_RACK"); + + String prefix = "dynoQueue_"; + + DynoShardSupplier ss = new DynoShardSupplier(dyno.getConnPool().getConfiguration().getHostSupplier(), region, localRack); + + RedisQueues queues = new RedisQueues(dyno, dyno, prefix, ss, 50_000, 50_000); + + Message msg1 = new Message("id1", "searchable payload"); + Message msg2 = new Message("id2", "payload 2"); + Message msg3 = new Message("id2", "payload 3"); + DynoQueue V1Queue = queues.get("simpleQueue"); + + // Test push() API + //List pushed_msgs = V1Queue.push(Arrays.asList(msg1)); + List pushed_msgs = V1Queue.push(ImmutableList.of(msg1, msg2, msg3)); + + // Test ensure() API + System.out.println("Does Message with ID '" + msg1.getId() + "' already exist? " + !V1Queue.ensure(msg1)); + + // Test containsPredicate() API + System.out.println("Does the predicate 'searchable' exist in the queue ? " + V1Queue.containsPredicate("searchable")); + + // Test pop() + List popped_msgs = V1Queue.pop(3, 1000, TimeUnit.MILLISECONDS); + + // Test ack() + assert(V1Queue.ack(popped_msgs.get(0).getId())); + + V1Queue.close(); + } + private void runSimpleV2QueueDemo(DynoJedisClient dyno) throws IOException { String region = System.getProperty("LOCAL_DATACENTER"); String localRack = System.getProperty("LOCAL_RACK"); diff --git a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/RedisDynoQueue.java b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/RedisDynoQueue.java index a683a83..552c937 100644 --- a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/RedisDynoQueue.java +++ b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/RedisDynoQueue.java @@ -470,7 +470,7 @@ public boolean containsPredicate(String predicate) { "return 0"; // Cast from 'JedisCommands' to 'DynoJedisClient' here since the former does not expose 'eval()'. - int retval = (int) ((DynoJedisClient)quorumConn).eval(predicateCheckLuaScript, + Long retval = (Long) ((DynoJedisClient)quorumConn).eval(predicateCheckLuaScript, Collections.singletonList(messageStoreKey), Collections.singletonList(predicate)); return (retval == 1); From c432aefb71aaefda0494076e1ce3811b425132ab Mon Sep 17 00:00:00 2001 From: Sailesh Mukil Date: Tue, 7 May 2019 21:34:27 -0700 Subject: [PATCH 35/91] Use logger instead of System.out in demo --- .../java/com/netflix/dyno/queues/demo/DynoQueueDemo.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/demo/DynoQueueDemo.java b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/demo/DynoQueueDemo.java index 6b5bd4c..b330e0b 100644 --- a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/demo/DynoQueueDemo.java +++ b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/demo/DynoQueueDemo.java @@ -90,14 +90,13 @@ private void runSimpleV1Demo(DynoJedisClient dyno) throws IOException { DynoQueue V1Queue = queues.get("simpleQueue"); // Test push() API - //List pushed_msgs = V1Queue.push(Arrays.asList(msg1)); List pushed_msgs = V1Queue.push(ImmutableList.of(msg1, msg2, msg3)); // Test ensure() API - System.out.println("Does Message with ID '" + msg1.getId() + "' already exist? " + !V1Queue.ensure(msg1)); + logger.info("Does Message with ID '" + msg1.getId() + "' already exist? " + !V1Queue.ensure(msg1)); // Test containsPredicate() API - System.out.println("Does the predicate 'searchable' exist in the queue ? " + V1Queue.containsPredicate("searchable")); + logger.info("Does the predicate 'searchable' exist in the queue ? " + V1Queue.containsPredicate("searchable")); // Test pop() List popped_msgs = V1Queue.pop(3, 1000, TimeUnit.MILLISECONDS); From cbafb5c091027aef18c1b8232944e2392fe463f5 Mon Sep 17 00:00:00 2001 From: Raghuram Onti Srinivasan Date: Wed, 15 May 2019 11:10:36 -0700 Subject: [PATCH 36/91] Fix issue with javadoc not building --- .../java/com/netflix/dyno/queues/demo/DynoQueueDemo.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/demo/DynoQueueDemo.java b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/demo/DynoQueueDemo.java index b330e0b..6798a17 100644 --- a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/demo/DynoQueueDemo.java +++ b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/demo/DynoQueueDemo.java @@ -3,16 +3,15 @@ import com.google.common.collect.ImmutableList; import com.netflix.dyno.demo.redis.DynoJedisDemo; import com.netflix.dyno.jedis.DynoJedisClient; -import com.netflix.dyno.queues.redis.RedisQueues; import com.netflix.dyno.queues.DynoQueue; import com.netflix.dyno.queues.Message; +import com.netflix.dyno.queues.redis.RedisQueues; import com.netflix.dyno.queues.redis.v2.QueueBuilder; import com.netflix.dyno.queues.shard.DynoShardSupplier; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.IOException; -import java.sql.Time; import java.util.Arrays; import java.util.List; import java.util.Properties; @@ -35,7 +34,7 @@ public DynoQueueDemo(String primaryCluster, String shadowCluster, String localRa * if the cluster is not reachable. * * @param args: - * + * cluster-name version * * cluster-name: Name of cluster to run demo against * version: Possible values = 1 or 2; (for V1 or V2) From ce4b87e0029cc604259e8137dcf13bf3977106cd Mon Sep 17 00:00:00 2001 From: Raghuram Onti Srinivasan Date: Tue, 28 May 2019 10:58:43 -0700 Subject: [PATCH 37/91] Auto formatting and fixing host to shard map Setting hostToShardMap explicitly in DynoQueueDemo.java:120 --- .../com/netflix/dyno/queues/DynoQueue.java | 270 +- .../netflix/dyno/queues/ShardSupplier.java | 34 +- .../com/netflix/dyno/queues/TestMessage.java | 52 +- .../dyno/queues/demo/DynoQueueDemo.java | 14 +- .../netflix/dyno/queues/redis/QueueUtils.java | 2 + .../dyno/queues/redis/RedisDynoQueue.java | 74 +- .../dyno/queues/redis/RedisQueues.java | 207 +- .../queues/redis/conn/DynoClientProxy.java | 154 +- .../dyno/queues/redis/conn/DynoJedisPipe.java | 8 +- .../netflix/dyno/queues/redis/conn/Pipe.java | 118 +- .../queues/redis/conn/RedisConnection.java | 48 +- .../dyno/queues/redis/conn/RedisPipe.java | 88 +- .../redis/sharding/RoundRobinStrategy.java | 8 +- .../redis/sharding/ShardingStrategy.java | 8 +- .../dyno/queues/redis/v2/MultiRedisQueue.java | 4 +- .../dyno/queues/redis/v2/QueueBuilder.java | 12 +- .../queues/redis/v2/RedisPipelineQueue.java | 6 +- .../dyno/queues/shard/DynoShardSupplier.java | 70 +- .../queues/shard/SingleShardSupplier.java | 41 +- .../src/main/resources/demo.properties | 2 - .../netflix/dyno/queues/jedis/JedisMock.java | 2240 ++++++++--------- .../redis/CustomShardingStrategyTest.java | 8 +- .../redis/DefaultShardingStrategyTest.java | 17 +- .../queues/redis/DynoShardSupplierTest.java | 54 +- .../dyno/queues/redis/RedisDynoQueueTest.java | 640 ++--- .../redis/benchmark/BenchmarkTestsJedis.java | 76 +- .../dyno/queues/redis/v2/DynoJedisTests.java | 1 - .../dyno/queues/redis/v2/JedisTests.java | 3 +- .../queues/redis/v2/RedisDynoQueueTest.java | 570 ++--- 29 files changed, 2415 insertions(+), 2414 deletions(-) diff --git a/dyno-queues-core/src/main/java/com/netflix/dyno/queues/DynoQueue.java b/dyno-queues-core/src/main/java/com/netflix/dyno/queues/DynoQueue.java index a6b3fd3..5bd0c64 100644 --- a/dyno-queues-core/src/main/java/com/netflix/dyno/queues/DynoQueue.java +++ b/dyno-queues-core/src/main/java/com/netflix/dyno/queues/DynoQueue.java @@ -1,12 +1,12 @@ /** * Copyright 2016 Netflix, Inc. - * + *

* Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -14,7 +14,7 @@ * limitations under the License. */ /** - * + * */ package com.netflix.dyno.queues; @@ -28,134 +28,134 @@ * Abstraction of a dyno queue. */ public interface DynoQueue extends Closeable { - - /** - * - * @return Returns the name of the queue - */ - public String getName(); - - /** - * - * @return Time in milliseconds before the messages that are popped and not acknowledge are pushed back into the queue. - * @see #ack(String) - */ - public int getUnackTime(); - - /** - * - * @param messages messages to be pushed onto the queue - * @return Returns the list of message ids - */ - public List push(List messages); - - /** - * - * @param messageCount number of messages to be popped out of the queue. - * @param wait Amount of time to wait if there are no messages in queue - * @param unit Time unit for the wait period - * @return messages. Can be less than the messageCount if there are fewer messages available than the message count. If the popped messages are not acknowledge in a timely manner, they are pushed back into the queue. - * @see #peek(int) - * @see #ack(String) - * @see #getUnackTime() - * - */ - public List pop(int messageCount, int wait, TimeUnit unit); - - /** - * Provides a peek into the queue without taking messages out. - * @param messageCount number of messages to be peeked. - * @return List of peeked messages. - * @see #pop(int, int, TimeUnit) - */ - public List peek(int messageCount); - - /** - * Provides an acknowledgement for the message. Once ack'ed the message is removed from the queue forever. - * @param messageId ID of the message to be acknowledged - * @return true if the message was found pending acknowledgement and is now ack'ed. false if the message id is invalid or message is no longer present in the queue. - */ - public boolean ack(String messageId); - - - /** - * Bulk version for {@link #ack(String)} - * @param messages Messages to be acknowledged. Each message MUST be populated with id and shard information. - */ - public void ack(List messages); - - /** - * Sets the unack timeout on the message (changes the default timeout to the new value). Useful when extended lease is required for a message by consumer before sending ack. - * @param messageId ID of the message to be acknowledged - * @param timeout time in milliseconds for which the message will remain in un-ack state. If no ack is received after the timeout period has expired, the message is put back into the queue - * @return true if the message id was found and updated with new timeout. false otherwise. - */ - public boolean setUnackTimeout(String messageId, long timeout); - - - /** - * Updates the timeout for the message. - * @param messageId ID of the message to be acknowledged - * @param timeout time in milliseconds for which the message will remain invisible and not popped out of the queue. - * @return true if the message id was found and updated with new timeout. false otherwise. - */ - public boolean setTimeout(String messageId, long timeout); - - /** - * - * @param messageId Remove the message from the queue - * @return true if the message id was found and removed. False otherwise. - */ - public boolean remove(String messageId); - - /** - * Enqueues 'message' if it doesn't exist in any of the shards or unack sets. - * - * @param message Message to enqueue if it doesn't exist. - * @return true if message was enqueued. False if messageId already exists. - */ - public boolean ensure(Message message); - - /** - * Checks the message bodies (i.e. the data in the hash map), and returns true on the first match with - * 'predicate'. - * - * Disclaimer: This is a potentially expensive call, since we will iterate over the entire hash map in the - * worst case. Use mindfully. - * - * @param predicate The predicate to check against. - * @return 'true' if any of the messages contain 'predicate'; 'false' otherwise. - */ - public boolean containsPredicate(String predicate); - - /** - * - * @param messageId message to be retrieved. - * @return Retrieves the message stored in the queue by the messageId. Null if not found. - */ - public Message get(String messageId); - - /** - * - * @return Size of the queue. - * @see #shardSizes() - */ - public long size(); - - /** - * - * @return Map of shard name to the # of messages in the shard. - * @see #size() - */ - public Map> shardSizes(); - - /** - * Truncates the entire queue. Use with caution! - */ - public void clear(); - - /** - * Process un-acknowledged messages. The messages which are polled by the client but not ack'ed are moved back to queue - */ - public void processUnacks(); + + /** + * + * @return Returns the name of the queue + */ + public String getName(); + + /** + * + * @return Time in milliseconds before the messages that are popped and not acknowledge are pushed back into the queue. + * @see #ack(String) + */ + public int getUnackTime(); + + /** + * + * @param messages messages to be pushed onto the queue + * @return Returns the list of message ids + */ + public List push(List messages); + + /** + * + * @param messageCount number of messages to be popped out of the queue. + * @param wait Amount of time to wait if there are no messages in queue + * @param unit Time unit for the wait period + * @return messages. Can be less than the messageCount if there are fewer messages available than the message count. If the popped messages are not acknowledge in a timely manner, they are pushed back into the queue. + * @see #peek(int) + * @see #ack(String) + * @see #getUnackTime() + * + */ + public List pop(int messageCount, int wait, TimeUnit unit); + + /** + * Provides a peek into the queue without taking messages out. + * @param messageCount number of messages to be peeked. + * @return List of peeked messages. + * @see #pop(int, int, TimeUnit) + */ + public List peek(int messageCount); + + /** + * Provides an acknowledgement for the message. Once ack'ed the message is removed from the queue forever. + * @param messageId ID of the message to be acknowledged + * @return true if the message was found pending acknowledgement and is now ack'ed. false if the message id is invalid or message is no longer present in the queue. + */ + public boolean ack(String messageId); + + + /** + * Bulk version for {@link #ack(String)} + * @param messages Messages to be acknowledged. Each message MUST be populated with id and shard information. + */ + public void ack(List messages); + + /** + * Sets the unack timeout on the message (changes the default timeout to the new value). Useful when extended lease is required for a message by consumer before sending ack. + * @param messageId ID of the message to be acknowledged + * @param timeout time in milliseconds for which the message will remain in un-ack state. If no ack is received after the timeout period has expired, the message is put back into the queue + * @return true if the message id was found and updated with new timeout. false otherwise. + */ + public boolean setUnackTimeout(String messageId, long timeout); + + + /** + * Updates the timeout for the message. + * @param messageId ID of the message to be acknowledged + * @param timeout time in milliseconds for which the message will remain invisible and not popped out of the queue. + * @return true if the message id was found and updated with new timeout. false otherwise. + */ + public boolean setTimeout(String messageId, long timeout); + + /** + * + * @param messageId Remove the message from the queue + * @return true if the message id was found and removed. False otherwise. + */ + public boolean remove(String messageId); + + /** + * Enqueues 'message' if it doesn't exist in any of the shards or unack sets. + * + * @param message Message to enqueue if it doesn't exist. + * @return true if message was enqueued. False if messageId already exists. + */ + public boolean ensure(Message message); + + /** + * Checks the message bodies (i.e. the data in the hash map), and returns true on the first match with + * 'predicate'. + * + * Disclaimer: This is a potentially expensive call, since we will iterate over the entire hash map in the + * worst case. Use mindfully. + * + * @param predicate The predicate to check against. + * @return 'true' if any of the messages contain 'predicate'; 'false' otherwise. + */ + public boolean containsPredicate(String predicate); + + /** + * + * @param messageId message to be retrieved. + * @return Retrieves the message stored in the queue by the messageId. Null if not found. + */ + public Message get(String messageId); + + /** + * + * @return Size of the queue. + * @see #shardSizes() + */ + public long size(); + + /** + * + * @return Map of shard name to the # of messages in the shard. + * @see #size() + */ + public Map> shardSizes(); + + /** + * Truncates the entire queue. Use with caution! + */ + public void clear(); + + /** + * Process un-acknowledged messages. The messages which are polled by the client but not ack'ed are moved back to queue + */ + public void processUnacks(); } diff --git a/dyno-queues-core/src/main/java/com/netflix/dyno/queues/ShardSupplier.java b/dyno-queues-core/src/main/java/com/netflix/dyno/queues/ShardSupplier.java index f6a2b6a..a29ed58 100644 --- a/dyno-queues-core/src/main/java/com/netflix/dyno/queues/ShardSupplier.java +++ b/dyno-queues-core/src/main/java/com/netflix/dyno/queues/ShardSupplier.java @@ -1,12 +1,12 @@ /** * Copyright 2016 Netflix, Inc. - * + *

* Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -14,7 +14,7 @@ * limitations under the License. */ /** - * + * */ package com.netflix.dyno.queues; @@ -27,16 +27,16 @@ */ public interface ShardSupplier { - /** - * - * @return Provides the set of all the available queue shards. The elements are evenly distributed amongst these shards - */ - public Set getQueueShards(); - - /** - * - * @return Name of the current shard. Used when popping elements out of the queue - */ - public String getCurrentShard(); - + /** + * + * @return Provides the set of all the available queue shards. The elements are evenly distributed amongst these shards + */ + public Set getQueueShards(); + + /** + * + * @return Name of the current shard. Used when popping elements out of the queue + */ + public String getCurrentShard(); + } diff --git a/dyno-queues-core/src/test/java/com/netflix/dyno/queues/TestMessage.java b/dyno-queues-core/src/test/java/com/netflix/dyno/queues/TestMessage.java index a59b5f2..4b3c510 100644 --- a/dyno-queues-core/src/test/java/com/netflix/dyno/queues/TestMessage.java +++ b/dyno-queues-core/src/test/java/com/netflix/dyno/queues/TestMessage.java @@ -1,12 +1,12 @@ /** * Copyright 2016 Netflix, Inc. - * + *

* Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -14,7 +14,7 @@ * limitations under the License. */ /** - * + * */ package com.netflix.dyno.queues; @@ -30,25 +30,25 @@ */ public class TestMessage { - @Test - public void test(){ - Message msg = new Message(); - msg.setPayload("payload"); - msg.setTimeout(10, TimeUnit.SECONDS); - assertEquals(msg.toString(), 10*1000, msg.getTimeout()); - msg.setTimeout(10); - assertEquals(msg.toString(), 10, msg.getTimeout()); - } - - @Test(expected=IllegalArgumentException.class) - public void testPrioirty(){ - Message msg = new Message(); - msg.setPriority(-1); - } - - @Test(expected=IllegalArgumentException.class) - public void testPrioirty2(){ - Message msg = new Message(); - msg.setPriority(100); - } + @Test + public void test() { + Message msg = new Message(); + msg.setPayload("payload"); + msg.setTimeout(10, TimeUnit.SECONDS); + assertEquals(msg.toString(), 10 * 1000, msg.getTimeout()); + msg.setTimeout(10); + assertEquals(msg.toString(), 10, msg.getTimeout()); + } + + @Test(expected = IllegalArgumentException.class) + public void testPrioirty() { + Message msg = new Message(); + msg.setPriority(-1); + } + + @Test(expected = IllegalArgumentException.class) + public void testPrioirty2() { + Message msg = new Message(); + msg.setPriority(100); + } } diff --git a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/demo/DynoQueueDemo.java b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/demo/DynoQueueDemo.java index 6798a17..83779f2 100644 --- a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/demo/DynoQueueDemo.java +++ b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/demo/DynoQueueDemo.java @@ -20,6 +20,7 @@ public class DynoQueueDemo extends DynoJedisDemo { private static final Logger logger = LoggerFactory.getLogger(DynoQueue.class); + public DynoQueueDemo(String clusterName, String localRack) { super(clusterName, localRack); } @@ -33,12 +34,10 @@ public DynoQueueDemo(String primaryCluster, String shadowCluster, String localRa * throws java.lang.RuntimeException: java.net.ConnectException: Connection timed out (Connection timed out) * if the cluster is not reachable. * - * @param args: - * cluster-name version - * - * cluster-name: Name of cluster to run demo against - * version: Possible values = 1 or 2; (for V1 or V2) - * + * @param args: cluster-name version + *

+ * cluster-name: Name of cluster to run demo against + * version: Possible values = 1 or 2; (for V1 or V2) */ public static void main(String[] args) throws IOException { final String clusterName = args[0]; @@ -101,7 +100,7 @@ private void runSimpleV1Demo(DynoJedisClient dyno) throws IOException { List popped_msgs = V1Queue.pop(3, 1000, TimeUnit.MILLISECONDS); // Test ack() - assert(V1Queue.ack(popped_msgs.get(0).getId())); + assert (V1Queue.ack(popped_msgs.get(0).getId())); V1Queue.close(); } @@ -118,6 +117,7 @@ private void runSimpleV2QueueDemo(DynoJedisClient dyno) throws IOException { DynoQueue queue = new QueueBuilder() .setQueueName("test") .setCurrentShard(ss.getCurrentShard()) + .setHostToShardMap(h -> h.getRack().replaceAll(region, "")) .setRedisKeyPrefix(prefix) .useDynomite(dyno, dyno, dyno.getConnPool().getConfiguration().getHostSupplier()) .setUnackTime(50_000) diff --git a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/QueueUtils.java b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/QueueUtils.java index b3c2c5f..c83402e 100644 --- a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/QueueUtils.java +++ b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/QueueUtils.java @@ -18,6 +18,7 @@ public class QueueUtils { /** * Execute function with retries if required + * * @param opName * @param keyName * @param r @@ -50,6 +51,7 @@ private static R executeWithRetry(String opName, String keyName, Callable /** * Construct standard objectmapper to use within the DynoQueue instances to read/write Message objects + * * @return */ public static ObjectMapper constructObjectMapper() { diff --git a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/RedisDynoQueue.java b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/RedisDynoQueue.java index 552c937..2b92b9c 100644 --- a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/RedisDynoQueue.java +++ b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/RedisDynoQueue.java @@ -1,12 +1,12 @@ /** * Copyright 2016 Netflix, Inc. - * + *

* Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -120,17 +120,17 @@ public RedisDynoQueue(Clock clock, String redisKeyPrefix, String queueName, Set< logger.info(RedisDynoQueue.class.getName() + " is ready to serve " + queueName); } - public RedisDynoQueue withQuorumConn(JedisCommands quorumConn){ + public RedisDynoQueue withQuorumConn(JedisCommands quorumConn) { this.quorumConn = quorumConn; return this; } - public RedisDynoQueue withNonQuorumConn(JedisCommands nonQuorumConn){ + public RedisDynoQueue withNonQuorumConn(JedisCommands nonQuorumConn) { this.nonQuorumConn = nonQuorumConn; return this; } - public RedisDynoQueue withUnackTime(int unackTime){ + public RedisDynoQueue withUnackTime(int unackTime) { this.unackTime = unackTime; return this; } @@ -213,13 +213,13 @@ public List pop(int messageCount, int wait, TimeUnit unit) { long waitFor = unit.toMillis(wait); prefetch.addAndGet(messageCount); prefetchIds(); - while(prefetchedIds.size() < messageCount && ((clock.millis() - start) < waitFor)) { + while (prefetchedIds.size() < messageCount && ((clock.millis() - start) < waitFor)) { Uninterruptibles.sleepUninterruptibly(200, TimeUnit.MILLISECONDS); prefetchIds(); } return _pop(messageCount); - } catch(Exception e) { + } catch (Exception e) { throw new RuntimeException(e); } finally { sw.stop(); @@ -243,7 +243,7 @@ private void prefetchIds() { Set ids = peekIds(0, prefetchCount); prefetchedIds.addAll(ids); prefetch.addAndGet((-1 * ids.size())); - if(prefetch.get() < 0 || ids.isEmpty()) { + if (prefetch.get() < 0 || ids.isEmpty()) { prefetch.set(0); } } finally { @@ -260,14 +260,14 @@ private List _pop(int messageCount) throws Exception { List popped = new LinkedList<>(); ZAddParams zParams = ZAddParams.zAddParams().nx(); - for (;popped.size() != messageCount;) { + for (; popped.size() != messageCount; ) { String msgId = prefetchedIds.poll(); - if(msgId == null) { + if (msgId == null) { break; } long added = quorumConn.zadd(unackQueueName, unackScore, msgId, zParams); - if(added == 0){ + if (added == 0) { if (logger.isDebugEnabled()) { logger.debug("cannot add {} to the unack shard ", queueName, msgId); } @@ -285,7 +285,7 @@ private List _pop(int messageCount) throws Exception { } String json = quorumConn.hget(messageStoreKey, msgId); - if(json == null){ + if (json == null) { if (logger.isDebugEnabled()) { logger.debug("Cannot get the message payload for {}", msgId); } @@ -328,7 +328,7 @@ public boolean ack(String messageId) { @Override public void ack(List messages) { - for(Message message : messages) { + for (Message message : messages) { ack(message.getId()); } } @@ -345,7 +345,7 @@ public boolean setUnackTimeout(String messageId, long timeout) { String unackShardKey = getUnackKey(queueName, shard); Double score = quorumConn.zscore(unackShardKey, messageId); - if(score != null) { + if (score != null) { quorumConn.zadd(unackShardKey, unackScore, messageId); return true; } @@ -364,7 +364,7 @@ public boolean setTimeout(String messageId, long timeout) { return execute("setTimeout", "(a shard in) " + queueName, () -> { String json = nonQuorumConn.hget(messageStoreKey, messageId); - if(json == null) { + if (json == null) { return false; } Message message = om.readValue(json, Message.class); @@ -374,7 +374,7 @@ public boolean setTimeout(String messageId, long timeout) { String queueShard = getQueueShardKey(queueName, shard); Double score = quorumConn.zscore(queueShard, messageId); - if(score != null) { + if (score != null) { double priorityd = message.getPriority() / 100; double newScore = Long.valueOf(clock.millis() + timeout).doubleValue() + priorityd; ZAddParams params = ZAddParams.zAddParams().xx(); @@ -422,25 +422,25 @@ public boolean remove(String messageId) { @Override public boolean ensure(Message message) { - return execute("ensure", "(a shard in) " + queueName, () -> { + return execute("ensure", "(a shard in) " + queueName, () -> { - String messageId = message.getId(); - for (String shard : allShards) { + String messageId = message.getId(); + for (String shard : allShards) { - String queueShard = getQueueShardKey(queueName, shard); - Double score = quorumConn.zscore(queueShard, messageId); - if(score != null) { - return false; - } - String unackShardKey = getUnackKey(queueName, shard); - score = quorumConn.zscore(unackShardKey, messageId); - if(score != null) { - return false; - } - } - push(Collections.singletonList(message)); - return true; - }); + String queueShard = getQueueShardKey(queueName, shard); + Double score = quorumConn.zscore(queueShard, messageId); + if (score != null) { + return false; + } + String unackShardKey = getUnackKey(queueName, shard); + score = quorumConn.zscore(unackShardKey, messageId); + if (score != null) { + return false; + } + } + push(Collections.singletonList(message)); + return true; + }); } @Override @@ -470,7 +470,7 @@ public boolean containsPredicate(String predicate) { "return 0"; // Cast from 'JedisCommands' to 'DynoJedisClient' here since the former does not expose 'eval()'. - Long retval = (Long) ((DynoJedisClient)quorumConn).eval(predicateCheckLuaScript, + Long retval = (Long) ((DynoJedisClient) quorumConn).eval(predicateCheckLuaScript, Collections.singletonList(messageStoreKey), Collections.singletonList(predicate)); return (retval == 1); @@ -486,7 +486,7 @@ public Message get(String messageId) { return execute("get", messageStoreKey, () -> { String json = quorumConn.hget(messageStoreKey, messageId); - if(json == null){ + if (json == null) { if (logger.isDebugEnabled()) { logger.debug("Cannot get the message payload " + messageId); } diff --git a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/RedisQueues.java b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/RedisQueues.java index add0339..c0d5d8f 100644 --- a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/RedisQueues.java +++ b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/RedisQueues.java @@ -1,12 +1,12 @@ /** * Copyright 2016 Netflix, Inc. - * + *

* Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -36,109 +36,108 @@ */ public class RedisQueues implements Closeable { - private final Clock clock; - - private final JedisCommands quorumConn; - - private final JedisCommands nonQuorumConn; - - private final Set allShards; - - private final String shardName; - - private final String redisKeyPrefix; - - private final int unackTime; - - private final int unackHandlerIntervalInMS; - - private final ConcurrentHashMap queues; - - private final ShardingStrategy shardingStrategy; - - /** - * @param quorumConn Dyno connection with dc_quorum enabled - * @param nonQuorumConn Dyno connection to local Redis - * @param redisKeyPrefix prefix applied to the Redis keys - * @param shardSupplier Provider for the shards for the queues created - * @param unackTime Time in millisecond within which a message needs to be acknowledged by the client, after which the message is re-queued. - * @param unackHandlerIntervalInMS Time in millisecond at which the un-acknowledgement processor runs - */ - public RedisQueues(JedisCommands quorumConn, JedisCommands nonQuorumConn, String redisKeyPrefix, ShardSupplier shardSupplier, int unackTime, int unackHandlerIntervalInMS) { - this(Clock.systemDefaultZone(), quorumConn, nonQuorumConn, redisKeyPrefix, shardSupplier, unackTime, unackHandlerIntervalInMS, new RoundRobinStrategy()); - } - - /** - * @param quorumConn Dyno connection with dc_quorum enabled - * @param nonQuorumConn Dyno connection to local Redis - * @param redisKeyPrefix prefix applied to the Redis keys - * @param shardSupplier Provider for the shards for the queues created - * @param unackTime Time in millisecond within which a message needs to be acknowledged by the client, after which the message is re-queued. - * @param unackHandlerIntervalInMS Time in millisecond at which the un-acknowledgement processor runs - * @param shardingStrategy sharding strategy responsible for calculating message's destination shard - */ - public RedisQueues(JedisCommands quorumConn, JedisCommands nonQuorumConn, String redisKeyPrefix, ShardSupplier shardSupplier, int unackTime, int unackHandlerIntervalInMS, ShardingStrategy shardingStrategy) { - this(Clock.systemDefaultZone(), quorumConn, nonQuorumConn, redisKeyPrefix, shardSupplier, unackTime, unackHandlerIntervalInMS, shardingStrategy); - } - - - /** - * @param clock Time provider - * @param quorumConn Dyno connection with dc_quorum enabled - * @param nonQuorumConn Dyno connection to local Redis - * @param redisKeyPrefix prefix applied to the Redis keys - * @param shardSupplier Provider for the shards for the queues created - * @param unackTime Time in millisecond within which a message needs to be acknowledged by the client, after which the message is re-queued. - * @param unackHandlerIntervalInMS Time in millisecond at which the un-acknowledgement processor runs - * @param shardingStrategy sharding strategy responsible for calculating message's destination shard - */ - public RedisQueues(Clock clock, JedisCommands quorumConn, JedisCommands nonQuorumConn, String redisKeyPrefix, ShardSupplier shardSupplier, int unackTime, int unackHandlerIntervalInMS, ShardingStrategy shardingStrategy) { - this.clock = clock; - this.quorumConn = quorumConn; - this.nonQuorumConn = nonQuorumConn; - this.redisKeyPrefix = redisKeyPrefix; - this.allShards = shardSupplier.getQueueShards(); - this.shardName = shardSupplier.getCurrentShard(); - this.unackTime = unackTime; - this.unackHandlerIntervalInMS = unackHandlerIntervalInMS; - this.queues = new ConcurrentHashMap<>(); - this.shardingStrategy = shardingStrategy; - } - - /** - * - * @param queueName Name of the queue - * @return Returns the DynoQueue hosting the given queue by name - * @see DynoQueue - * @see RedisDynoQueue - */ - public DynoQueue get(String queueName) { - - String key = queueName.intern(); + private final Clock clock; + + private final JedisCommands quorumConn; + + private final JedisCommands nonQuorumConn; + + private final Set allShards; + + private final String shardName; + + private final String redisKeyPrefix; + + private final int unackTime; + + private final int unackHandlerIntervalInMS; + + private final ConcurrentHashMap queues; + + private final ShardingStrategy shardingStrategy; + + /** + * @param quorumConn Dyno connection with dc_quorum enabled + * @param nonQuorumConn Dyno connection to local Redis + * @param redisKeyPrefix prefix applied to the Redis keys + * @param shardSupplier Provider for the shards for the queues created + * @param unackTime Time in millisecond within which a message needs to be acknowledged by the client, after which the message is re-queued. + * @param unackHandlerIntervalInMS Time in millisecond at which the un-acknowledgement processor runs + */ + public RedisQueues(JedisCommands quorumConn, JedisCommands nonQuorumConn, String redisKeyPrefix, ShardSupplier shardSupplier, int unackTime, int unackHandlerIntervalInMS) { + this(Clock.systemDefaultZone(), quorumConn, nonQuorumConn, redisKeyPrefix, shardSupplier, unackTime, unackHandlerIntervalInMS, new RoundRobinStrategy()); + } + + /** + * @param quorumConn Dyno connection with dc_quorum enabled + * @param nonQuorumConn Dyno connection to local Redis + * @param redisKeyPrefix prefix applied to the Redis keys + * @param shardSupplier Provider for the shards for the queues created + * @param unackTime Time in millisecond within which a message needs to be acknowledged by the client, after which the message is re-queued. + * @param unackHandlerIntervalInMS Time in millisecond at which the un-acknowledgement processor runs + * @param shardingStrategy sharding strategy responsible for calculating message's destination shard + */ + public RedisQueues(JedisCommands quorumConn, JedisCommands nonQuorumConn, String redisKeyPrefix, ShardSupplier shardSupplier, int unackTime, int unackHandlerIntervalInMS, ShardingStrategy shardingStrategy) { + this(Clock.systemDefaultZone(), quorumConn, nonQuorumConn, redisKeyPrefix, shardSupplier, unackTime, unackHandlerIntervalInMS, shardingStrategy); + } + + + /** + * @param clock Time provider + * @param quorumConn Dyno connection with dc_quorum enabled + * @param nonQuorumConn Dyno connection to local Redis + * @param redisKeyPrefix prefix applied to the Redis keys + * @param shardSupplier Provider for the shards for the queues created + * @param unackTime Time in millisecond within which a message needs to be acknowledged by the client, after which the message is re-queued. + * @param unackHandlerIntervalInMS Time in millisecond at which the un-acknowledgement processor runs + * @param shardingStrategy sharding strategy responsible for calculating message's destination shard + */ + public RedisQueues(Clock clock, JedisCommands quorumConn, JedisCommands nonQuorumConn, String redisKeyPrefix, ShardSupplier shardSupplier, int unackTime, int unackHandlerIntervalInMS, ShardingStrategy shardingStrategy) { + this.clock = clock; + this.quorumConn = quorumConn; + this.nonQuorumConn = nonQuorumConn; + this.redisKeyPrefix = redisKeyPrefix; + this.allShards = shardSupplier.getQueueShards(); + this.shardName = shardSupplier.getCurrentShard(); + this.unackTime = unackTime; + this.unackHandlerIntervalInMS = unackHandlerIntervalInMS; + this.queues = new ConcurrentHashMap<>(); + this.shardingStrategy = shardingStrategy; + } + + /** + * + * @param queueName Name of the queue + * @return Returns the DynoQueue hosting the given queue by name + * @see DynoQueue + * @see RedisDynoQueue + */ + public DynoQueue get(String queueName) { + + String key = queueName.intern(); return queues.computeIfAbsent(key, (keyToCompute) -> new RedisDynoQueue(clock, redisKeyPrefix, queueName, allShards, shardName, unackHandlerIntervalInMS, shardingStrategy) .withUnackTime(unackTime) .withNonQuorumConn(nonQuorumConn) .withQuorumConn(quorumConn)); - } - - /** - * - * @return Collection of all the registered queues - */ - public Collection queues(){ - return this.queues.values(); - } - - @Override - public void close() throws IOException { - queues.values().forEach(queue -> { - try { - queue.close(); - } - catch (final IOException e) { - throw new RuntimeException(e.getCause()); - } - }); - } + } + + /** + * + * @return Collection of all the registered queues + */ + public Collection queues() { + return this.queues.values(); + } + + @Override + public void close() throws IOException { + queues.values().forEach(queue -> { + try { + queue.close(); + } catch (final IOException e) { + throw new RuntimeException(e.getCause()); + } + }); + } } diff --git a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/conn/DynoClientProxy.java b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/conn/DynoClientProxy.java index a7200a7..88b6a36 100644 --- a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/conn/DynoClientProxy.java +++ b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/conn/DynoClientProxy.java @@ -1,12 +1,12 @@ /** * Copyright 2018 Netflix, Inc. - * + *

* Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -14,7 +14,7 @@ * limitations under the License. */ /** - * + * */ package com.netflix.dyno.queues.redis.conn; @@ -31,77 +31,77 @@ */ public class DynoClientProxy implements RedisConnection { - private DynoJedisClient jedis; - - - public DynoClientProxy(DynoJedisClient jedis) { - this.jedis = jedis; - } - - @Override - public RedisConnection getResource() { - return this; - } - - @Override - public void close() { - //nothing! - } - - @Override - public Pipe pipelined() { - return new DynoJedisPipe(jedis.pipelined()); - } - - @Override - public String hget(String key, String member) { - return jedis.hget(key, member); - } - - @Override - public Long zrem(String key, String member) { - return jedis.zrem(key, member); - } - - @Override - public Long hdel(String key, String member) { - return jedis.hdel(key, member); - - } - - @Override - public Double zscore(String key, String member) { - return jedis.zscore(key, member); - } - - @Override - public void zadd(String key, double score, String member) { - jedis.zadd(key, score, member); - } - - @Override - public void hset(String key, String member, String json) { - jedis.hset(key, member, json); - } - - @Override - public long zcard(String key) { - return jedis.zcard(key); - } - - @Override - public void del(String key) { - jedis.del(key); - } - - @Override - public Set zrangeByScore(String key, int min, double max, int offset, int count) { - return jedis.zrangeByScore(key, min, max, offset, count); - } - - @Override - public Set zrangeByScoreWithScores(String key, int min, double max, int offset, int count) { - return jedis.zrangeByScoreWithScores(key, min, max, offset, count); - } + private DynoJedisClient jedis; + + + public DynoClientProxy(DynoJedisClient jedis) { + this.jedis = jedis; + } + + @Override + public RedisConnection getResource() { + return this; + } + + @Override + public void close() { + //nothing! + } + + @Override + public Pipe pipelined() { + return new DynoJedisPipe(jedis.pipelined()); + } + + @Override + public String hget(String key, String member) { + return jedis.hget(key, member); + } + + @Override + public Long zrem(String key, String member) { + return jedis.zrem(key, member); + } + + @Override + public Long hdel(String key, String member) { + return jedis.hdel(key, member); + + } + + @Override + public Double zscore(String key, String member) { + return jedis.zscore(key, member); + } + + @Override + public void zadd(String key, double score, String member) { + jedis.zadd(key, score, member); + } + + @Override + public void hset(String key, String member, String json) { + jedis.hset(key, member, json); + } + + @Override + public long zcard(String key) { + return jedis.zcard(key); + } + + @Override + public void del(String key) { + jedis.del(key); + } + + @Override + public Set zrangeByScore(String key, int min, double max, int offset, int count) { + return jedis.zrangeByScore(key, min, max, offset, count); + } + + @Override + public Set zrangeByScoreWithScores(String key, int min, double max, int offset, int count) { + return jedis.zrangeByScoreWithScores(key, min, max, offset, count); + } } diff --git a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/conn/DynoJedisPipe.java b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/conn/DynoJedisPipe.java index 4d5db74..b32502d 100644 --- a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/conn/DynoJedisPipe.java +++ b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/conn/DynoJedisPipe.java @@ -1,12 +1,12 @@ /** * Copyright 2018 Netflix, Inc. - * + *

* Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. diff --git a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/conn/Pipe.java b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/conn/Pipe.java index 65b121a..3c3ae86 100644 --- a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/conn/Pipe.java +++ b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/conn/Pipe.java @@ -1,12 +1,12 @@ /** * Copyright 2018 Netflix, Inc. - * + *

* Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -22,7 +22,7 @@ import redis.clients.jedis.params.sortedset.ZAddParams; /** - * + * * @author Viren *

* Abstraction of Redis Pipeline. @@ -35,65 +35,65 @@ */ public interface Pipe { - /** - * - * @param key The Key - * @param field Field - * @param value Value of the Field - */ - public void hset(String key, String field, String value); + /** + * + * @param key The Key + * @param field Field + * @param value Value of the Field + */ + public void hset(String key, String field, String value); - /** - * - * @param key The Key - * @param score Score for the member - * @param member Member to be added within the key - * @return - */ - public Response zadd(String key, double score, String member); + /** + * + * @param key The Key + * @param score Score for the member + * @param member Member to be added within the key + * @return + */ + public Response zadd(String key, double score, String member); - /** - * - * @param key The Key - * @param score Score for the member - * @param member Member to be added within the key - * @param zParams Parameters - * @return - */ - public Response zadd(String key, double score, String member, ZAddParams zParams); + /** + * + * @param key The Key + * @param score Score for the member + * @param member Member to be added within the key + * @param zParams Parameters + * @return + */ + public Response zadd(String key, double score, String member, ZAddParams zParams); - /** - * - * @param key The Key - * @param member Member - * @return - */ - public Response zrem(String key, String member); + /** + * + * @param key The Key + * @param member Member + * @return + */ + public Response zrem(String key, String member); - /** - * - * @param key The Key - * @param member Member - * @return - */ - public Response hget(String key, String member); + /** + * + * @param key The Key + * @param member Member + * @return + */ + public Response hget(String key, String member); - /** - * - * @param key - * @param member - * @return - */ - public Response hdel(String key, String member); + /** + * + * @param key + * @param member + * @return + */ + public Response hdel(String key, String member); - /** - * - */ - public void sync(); + /** + * + */ + public void sync(); - /** - * - * @throws Exception - */ - public void close() throws Exception; + /** + * + * @throws Exception + */ + public void close() throws Exception; } \ No newline at end of file diff --git a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/conn/RedisConnection.java b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/conn/RedisConnection.java index 54dab2c..b854c43 100644 --- a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/conn/RedisConnection.java +++ b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/conn/RedisConnection.java @@ -1,12 +1,12 @@ /** * Copyright 2018 Netflix, Inc. - * + *

* Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -24,7 +24,7 @@ /** * Abstraction of Redis connection. - * + * * @author viren *

* The methods are 1-1 proxies from Jedis. See Jedis documentation for the details. @@ -34,34 +34,34 @@ */ public interface RedisConnection { - /** - * - * @return Returns the underlying connection resource. For connection pool, returns the actual connection - */ - public RedisConnection getResource(); + /** + * + * @return Returns the underlying connection resource. For connection pool, returns the actual connection + */ + public RedisConnection getResource(); + + public String hget(String messkeyageStoreKey, String member); + + public Long zrem(String key, String member); - public String hget(String messkeyageStoreKey, String member); + public Long hdel(String key, String member); - public Long zrem(String key, String member); - - public Long hdel(String key, String member); + public Double zscore(String key, String member); - public Double zscore(String key, String member); + public void zadd(String key, double score, String member); - public void zadd(String key, double score, String member); + public void hset(String key, String id, String json); - public void hset(String key, String id, String json); + public long zcard(String key); - public long zcard(String key); + public void del(String key); - public void del(String key); + public Set zrangeByScore(String key, int min, double max, int offset, int count); - public Set zrangeByScore(String key, int min, double max, int offset, int count); + public Set zrangeByScoreWithScores(String key, int min, double max, int offset, int count); - public Set zrangeByScoreWithScores(String key, int min, double max, int offset, int count); - - public void close(); + public void close(); - public Pipe pipelined(); + public Pipe pipelined(); } \ No newline at end of file diff --git a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/conn/RedisPipe.java b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/conn/RedisPipe.java index 6c24aa8..e8bc502 100644 --- a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/conn/RedisPipe.java +++ b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/conn/RedisPipe.java @@ -1,12 +1,12 @@ /** * Copyright 2018 Netflix, Inc. - * + *

* Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -14,7 +14,7 @@ * limitations under the License. */ /** - * + * */ package com.netflix.dyno.queues.redis.conn; @@ -29,52 +29,52 @@ */ public class RedisPipe implements Pipe { - private Pipeline pipe; - - public RedisPipe(Pipeline pipe) { - this.pipe = pipe; - } + private Pipeline pipe; + + public RedisPipe(Pipeline pipe) { + this.pipe = pipe; + } + + @Override + public void hset(String key, String field, String value) { + pipe.hset(key, field, value); + + } - @Override - public void hset(String key, String field, String value) { - pipe.hset(key, field, value); - - } + @Override + public Response zadd(String key, double score, String member) { + return pipe.zadd(key, score, member); + } - @Override - public Response zadd(String key, double score, String member) { - return pipe.zadd(key, score, member); - } + @Override + public Response zadd(String key, double score, String member, ZAddParams zParams) { + return pipe.zadd(key, score, member, zParams); + } - @Override - public Response zadd(String key, double score, String member, ZAddParams zParams) { - return pipe.zadd(key, score, member, zParams); - } - - @Override - public Response zrem(String key, String member) { - return pipe.zrem(key, member); - } + @Override + public Response zrem(String key, String member) { + return pipe.zrem(key, member); + } - @Override - public Response hget(String key, String member) { - return pipe.hget(key, member); - } + @Override + public Response hget(String key, String member) { + return pipe.hget(key, member); + } - @Override - public Response hdel(String key, String member) { - return pipe.hdel(key, member); - } + @Override + public Response hdel(String key, String member) { + return pipe.hdel(key, member); + } - @Override - public void sync() { - pipe.sync(); - } + @Override + public void sync() { + pipe.sync(); + } - @Override - public void close() throws Exception { - pipe.close(); - } + @Override + public void close() throws Exception { + pipe.close(); + } } diff --git a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/sharding/RoundRobinStrategy.java b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/sharding/RoundRobinStrategy.java index 0e2ea1e..ff5ac4a 100644 --- a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/sharding/RoundRobinStrategy.java +++ b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/sharding/RoundRobinStrategy.java @@ -1,12 +1,12 @@ /** * Copyright 2018 Netflix, Inc. - * + *

* Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. diff --git a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/sharding/ShardingStrategy.java b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/sharding/ShardingStrategy.java index 71744af..e27a2ba 100644 --- a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/sharding/ShardingStrategy.java +++ b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/sharding/ShardingStrategy.java @@ -1,12 +1,12 @@ /** * Copyright 2018 Netflix, Inc. - * + *

* Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. diff --git a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/v2/MultiRedisQueue.java b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/v2/MultiRedisQueue.java index 09dc77c..f934889 100644 --- a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/v2/MultiRedisQueue.java +++ b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/v2/MultiRedisQueue.java @@ -148,7 +148,9 @@ public boolean ensure(Message message) { } @Override - public boolean containsPredicate(String predicate) { throw new UnsupportedOperationException(); } + public boolean containsPredicate(String predicate) { + throw new UnsupportedOperationException(); + } @Override public Message get(String messageId) { diff --git a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/v2/QueueBuilder.java b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/v2/QueueBuilder.java index 23ca6da..aed659c 100644 --- a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/v2/QueueBuilder.java +++ b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/v2/QueueBuilder.java @@ -1,12 +1,12 @@ /** * Copyright 2018 Netflix, Inc. - * + *

* Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -61,7 +61,7 @@ public class QueueBuilder { private String currentShard; - private Function hostToShardMap = (Host h) -> h.getRack(); + private Function hostToShardMap; private HostSupplier hs; @@ -164,7 +164,7 @@ public DynoQueue build() { } boolean useDynomiteCluster = (dynoQuorumClient != null && hs != null); - if(useDynomiteCluster) { + if (useDynomiteCluster) { this.hosts = hs.getHosts(); } diff --git a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/v2/RedisPipelineQueue.java b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/v2/RedisPipelineQueue.java index cfb28a3..74518ee 100644 --- a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/v2/RedisPipelineQueue.java +++ b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/v2/RedisPipelineQueue.java @@ -224,7 +224,7 @@ public synchronized List pop(int messageCount, int wait, TimeUnit unit) } messages.addAll(popped); remaining -= poppedCount; - if(clock.millis() > time) { + if (clock.millis() > time) { break; } @@ -464,7 +464,9 @@ public boolean ensure(Message message) { } @Override - public boolean containsPredicate(String predicate) { throw new UnsupportedOperationException(); } + public boolean containsPredicate(String predicate) { + throw new UnsupportedOperationException(); + } @Override public Message get(String messageId) { diff --git a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/shard/DynoShardSupplier.java b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/shard/DynoShardSupplier.java index 59619cd..b849bac 100644 --- a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/shard/DynoShardSupplier.java +++ b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/shard/DynoShardSupplier.java @@ -1,12 +1,12 @@ /** * Copyright 2016 Netflix, Inc. - * + *

* Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -14,7 +14,7 @@ * limitations under the License. */ /** - * + * */ package com.netflix.dyno.queues.shard; @@ -29,34 +29,34 @@ * */ public class DynoShardSupplier implements ShardSupplier { - - private HostSupplier hs; - - private String region; - - private String localRack; - - /** - * Dynomite based shard supplier. Keeps the number of shards in parity with the hosts and regions - * @param hs Host supplier - * @param region current region - * @param localRack local rack identifier - */ - public DynoShardSupplier(HostSupplier hs, String region, String localRack) { - this.hs = hs; - this.region = region; - this.localRack = localRack; - } - - @Override - public String getCurrentShard() { - return localRack.replaceAll(region, ""); - } - - @Override - public Set getQueueShards() { - return hs.getHosts().stream().map(host -> host.getRack()).map(rack -> rack.replaceAll(region, "")).collect(Collectors.toSet()); - } - - + + private HostSupplier hs; + + private String region; + + private String localRack; + + /** + * Dynomite based shard supplier. Keeps the number of shards in parity with the hosts and regions + * @param hs Host supplier + * @param region current region + * @param localRack local rack identifier + */ + public DynoShardSupplier(HostSupplier hs, String region, String localRack) { + this.hs = hs; + this.region = region; + this.localRack = localRack; + } + + @Override + public String getCurrentShard() { + return localRack.replaceAll(region, ""); + } + + @Override + public Set getQueueShards() { + return hs.getHosts().stream().map(host -> host.getRack()).map(rack -> rack.replaceAll(region, "")).collect(Collectors.toSet()); + } + + } diff --git a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/shard/SingleShardSupplier.java b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/shard/SingleShardSupplier.java index 0162812..9b1884d 100644 --- a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/shard/SingleShardSupplier.java +++ b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/shard/SingleShardSupplier.java @@ -1,12 +1,12 @@ /** * Copyright 2016 Netflix, Inc. - * + *

* Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -14,7 +14,7 @@ * limitations under the License. */ /** - * + * */ package com.netflix.dyno.queues.shard; @@ -28,19 +28,20 @@ * */ public class SingleShardSupplier implements ShardSupplier { - - private String shardName; - - public SingleShardSupplier(String shardName){ - this.shardName = shardName; - } - @Override - public String getCurrentShard() { - return shardName; - } - - @Override - public Set getQueueShards() { - return Sets.newHashSet(shardName); - } + + private String shardName; + + public SingleShardSupplier(String shardName) { + this.shardName = shardName; + } + + @Override + public String getCurrentShard() { + return shardName; + } + + @Override + public Set getQueueShards() { + return Sets.newHashSet(shardName); + } } diff --git a/dyno-queues-redis/src/main/resources/demo.properties b/dyno-queues-redis/src/main/resources/demo.properties index d171cff..ae861b7 100644 --- a/dyno-queues-redis/src/main/resources/demo.properties +++ b/dyno-queues-redis/src/main/resources/demo.properties @@ -5,9 +5,7 @@ LOCAL_DATACENTER=us-east-1 LOCAL_RACK=us-east-1c NETFLIX_STACK=dyno_demo #EC2_AVAILABILITY_ZONE=us-east-1 - dyno.demo.lbStrategy=TokenAware - dyno.demo.retryPolicy=RetryNTimes:2 dyno.demo.port=8102 dyno.demo.discovery.prod=discoveryreadonly.%s.dynprod.netflix.net:7001/v2/apps diff --git a/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/jedis/JedisMock.java b/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/jedis/JedisMock.java index f4732b0..9acbdbc 100644 --- a/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/jedis/JedisMock.java +++ b/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/jedis/JedisMock.java @@ -1,12 +1,12 @@ /** * Copyright 2016 Netflix, Inc. - * + *

* Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -14,7 +14,7 @@ * limitations under the License. */ /** - * + * */ package com.netflix.dyno.queues.jedis; @@ -44,1119 +44,1119 @@ */ public class JedisMock extends Jedis { - private IRedisClient redis; - - public JedisMock() { - super(""); - this.redis = new RedisMock(); - } - - private Set toTupleSet(Set pairs) { - Set set = new HashSet(); - for (ZsetPair pair : pairs) { - set.add(new Tuple(pair.member, pair.score)); - } - return set; - } - - @Override - public String set(final String key, String value) { - try { - return redis.set(key, value); - } catch (Exception e) { - throw new JedisException(e); - } - } - - @Override - public String set(final String key, final String value, final String nxxx, final String expx, final long time) { - try { - return redis.set(key, value, nxxx, expx, String.valueOf(time)); - } catch (Exception e) { - throw new JedisException(e); - } - } - - @Override - public String get(final String key) { - try { - return redis.get(key); - } catch (Exception e) { - throw new JedisException(e); - } - } - - @Override - public Boolean exists(final String key) { - try { - return redis.exists(key); - } catch (Exception e) { - throw new JedisException(e); - } - } - - @Override - public Long del(final String... keys) { - try { - return redis.del(keys); - } catch (Exception e) { - throw new JedisException(e); - } - } - - @Override - public Long del(String key) { - try { - return redis.del(key); - } catch (Exception e) { - throw new JedisException(e); - } - } - - @Override - public String type(final String key) { - try { - return redis.type(key); - } catch (Exception e) { - throw new JedisException(e); - } - } - - /* - * public Set keys(final String pattern) { checkIsInMulti(); - * client.keys(pattern); return - * BuilderFactory.STRING_SET.build(client.getBinaryMultiBulkReply()); } - * - * public String randomKey() { checkIsInMulti(); client.randomKey(); return - * client.getBulkReply(); } - * - * public String rename(final String oldkey, final String newkey) { - * checkIsInMulti(); client.rename(oldkey, newkey); return - * client.getStatusCodeReply(); } - * - * public Long renamenx(final String oldkey, final String newkey) { - * checkIsInMulti(); client.renamenx(oldkey, newkey); return - * client.getIntegerReply(); } - */ - @Override - public Long expire(final String key, final int seconds) { - try { - return redis.expire(key, seconds) ? 1L : 0L; - } catch (Exception e) { - throw new JedisException(e); - } - } - - @Override - public Long expireAt(final String key, final long unixTime) { - try { - return redis.expireat(key, unixTime) ? 1L : 0L; - } catch (Exception e) { - throw new JedisException(e); - } - } - - @Override - public Long ttl(final String key) { - try { - return redis.ttl(key); - } catch (Exception e) { - throw new JedisException(e); - } - } - - @Override - public Long move(final String key, final int dbIndex) { - try { - return redis.move(key, dbIndex); - } catch (Exception e) { - throw new JedisException(e); - } - } - - @Override - public String getSet(final String key, final String value) { - try { - return redis.getset(key, value); - } catch (Exception e) { - throw new JedisException(e); - } - } - - @Override - public List mget(final String... keys) { - try { - String[] mget = redis.mget(keys); - List lst = new ArrayList(mget.length); - for (String get : mget) { - lst.add(get); - } - return lst; - } catch (Exception e) { - throw new JedisException(e); - } - } - - @Override - public Long setnx(final String key, final String value) { - try { - return redis.setnx(key, value); - } catch (Exception e) { - throw new JedisException(e); - } - } - - @Override - public String setex(final String key, final int seconds, final String value) { - try { - return redis.setex(key, seconds, value); - } catch (Exception e) { - throw new JedisException(e); - } - } - - @Override - public String mset(final String... keysvalues) { - try { - return redis.mset(keysvalues); - } catch (Exception e) { - throw new JedisException(e); - } - } - - @Override - public Long msetnx(final String... keysvalues) { - try { - return redis.msetnx(keysvalues) ? 1L : 0L; - } catch (Exception e) { - throw new JedisException(e); - } - } - - @Override - public Long decrBy(final String key, final long integer) { - try { - return redis.decrby(key, integer); - } catch (Exception e) { - throw new JedisException(e); - } - } - - @Override - public Long decr(final String key) { - try { - return redis.decr(key); - } catch (Exception e) { - throw new JedisException(e); - } - } - - @Override - public Long incrBy(final String key, final long integer) { - try { - return redis.incrby(key, integer); - } catch (Exception e) { - throw new JedisException(e); - } - } - - @Override - public Double incrByFloat(final String key, final double value) { - try { - return Double.parseDouble(redis.incrbyfloat(key, value)); - } catch (Exception e) { - throw new JedisException(e); - } - } - - @Override - public Long incr(final String key) { - try { - return redis.incr(key); - } catch (Exception e) { - throw new JedisException(e); - } - } - - @Override - public Long append(final String key, final String value) { - try { - return redis.append(key, value); - } catch (Exception e) { - throw new JedisException(e); - } - } - - @Override - public String substr(final String key, final int start, final int end) { - try { - return redis.getrange(key, start, end); - } catch (Exception e) { - throw new JedisException(e); - } - } - - @Override - public Long hset(final String key, final String field, final String value) { - try { - return redis.hset(key, field, value) ? 1L : 0L; - } catch (Exception e) { - throw new JedisException(e); - } - } - - @Override - public String hget(final String key, final String field) { - try { - return redis.hget(key, field); - } catch (Exception e) { - throw new JedisException(e); - } - } - - @Override - public Long hsetnx(final String key, final String field, final String value) { - try { - return redis.hsetnx(key, field, value) ? 1L : 0L; - } catch (Exception e) { - throw new JedisException(e); - } - } - - @Override - public String hmset(final String key, final Map hash) { - try { - String field = null, value = null; - String[] args = new String[(hash.size() - 1) * 2]; - int idx = 0; - for (String f : hash.keySet()) { - if (field == null) { - field = f; - value = hash.get(f); - continue; - } - args[idx] = f; - args[idx + 1] = hash.get(f); - idx += 2; - } - return redis.hmset(key, field, value, args); - } catch (Exception e) { - throw new JedisException(e); - } - } - - @Override - public List hmget(final String key, final String... fields) { - try { - String field = fields[0]; - String[] f = new String[fields.length - 1]; - for (int idx = 1; idx < fields.length; ++idx) { - f[idx - 1] = fields[idx]; - } - return redis.hmget(key, field, f); - } catch (Exception e) { - throw new JedisException(e); - } - } - - @Override - public Long hincrBy(final String key, final String field, final long value) { - try { - return redis.hincrby(key, field, value); - } catch (Exception e) { - throw new JedisException(e); - } - } - - @Override - public Double hincrByFloat(final String key, final String field, final double value) { - try { - return Double.parseDouble(redis.hincrbyfloat(key, field, value)); - } catch (Exception e) { - throw new JedisException(e); - } - } - - @Override - public Boolean hexists(final String key, final String field) { - try { - return redis.hexists(key, field); - } catch (Exception e) { - throw new JedisException(e); - } - } - - @Override - public Long hdel(final String key, final String... fields) { - try { - String field = fields[0]; - String[] f = new String[fields.length - 1]; - for (int idx = 1; idx < fields.length; ++idx) { - f[idx - 1] = fields[idx]; - } - return redis.hdel(key, field, f); - } catch (Exception e) { - throw new JedisException(e); - } - } - - @Override - public Long hlen(final String key) { - try { - return redis.hlen(key); - } catch (Exception e) { - throw new JedisException(e); - } - } - - @Override - public Set hkeys(final String key) { - try { - return redis.hkeys(key); - } catch (Exception e) { - throw new JedisException(e); - } - } - - @Override - public List hvals(final String key) { - try { - return redis.hvals(key); - } catch (Exception e) { - throw new JedisException(e); - } - } - - @Override - public Map hgetAll(final String key) { - try { - return redis.hgetall(key); - } catch (Exception e) { - throw new JedisException(e); - } - } - - @Override - public Long rpush(final String key, final String... strings) { - try { - String element = strings[0]; - String[] elements = new String[strings.length - 1]; - for (int idx = 1; idx < strings.length; ++idx) { - elements[idx - 1] = strings[idx]; - } - return redis.rpush(key, element, elements); - } catch (Exception e) { - throw new JedisException(e); - } - } - - @Override - public Long lpush(final String key, final String... strings) { - try { - String element = strings[0]; - String[] elements = new String[strings.length - 1]; - for (int idx = 1; idx < strings.length; ++idx) { - elements[idx - 1] = strings[idx]; - } - return redis.lpush(key, element, elements); - } catch (Exception e) { - throw new JedisException(e); - } - } - - @Override - public Long llen(final String key) { - try { - return redis.llen(key); - } catch (Exception e) { - throw new JedisException(e); - } - } - - @Override - public List lrange(final String key, final long start, final long end) { - try { - return redis.lrange(key, start, end); - } catch (Exception e) { - throw new JedisException(e); - } - } - - @Override - public String ltrim(final String key, final long start, final long end) { - try { - return redis.ltrim(key, start, end); - } catch (Exception e) { - throw new JedisException(e); - } - } - - @Override - public String lindex(final String key, final long index) { - try { - return redis.lindex(key, index); - } catch (Exception e) { - throw new JedisException(e); - } - } - - @Override - public String lset(final String key, final long index, final String value) { - try { - return redis.lset(key, index, value); - } catch (Exception e) { - throw new JedisException(e); - } - } - - @Override - public Long lrem(final String key, final long count, final String value) { - try { - return redis.lrem(key, count, value); - } catch (Exception e) { - throw new JedisException(e); - } - } - - @Override - public String lpop(final String key) { - try { - return redis.lpop(key); - } catch (Exception e) { - throw new JedisException(e); - } - } - - @Override - public String rpop(final String key) { - try { - return redis.rpop(key); - } catch (Exception e) { - throw new JedisException(e); - } - } - - @Override - public String rpoplpush(final String srckey, final String dstkey) { - try { - return redis.rpoplpush(srckey, dstkey); - } catch (Exception e) { - throw new JedisException(e); - } - } - - @Override - public Long sadd(final String key, final String... members) { - try { - String member = members[0]; - String[] m = new String[members.length - 1]; - for (int idx = 1; idx < members.length; ++idx) { - m[idx - 1] = members[idx]; - } - return redis.sadd(key, member, m); - } catch (Exception e) { - throw new JedisException(e); - } - } - - @Override - public Set smembers(final String key) { - try { - return redis.smembers(key); - } catch (Exception e) { - throw new JedisException(e); - } - } - - @Override - public Long srem(final String key, final String... members) { - try { - String member = members[0]; - String[] m = new String[members.length - 1]; - for (int idx = 1; idx < members.length; ++idx) { - m[idx - 1] = members[idx]; - } - return redis.srem(key, member, m); - } catch (Exception e) { - throw new JedisException(e); - } - } - - @Override - public String spop(final String key) { - try { - return redis.spop(key); - } catch (Exception e) { - throw new JedisException(e); - } - } - - @Override - public Long smove(final String srckey, final String dstkey, final String member) { - try { - return redis.smove(srckey, dstkey, member) ? 1L : 0L; - } catch (Exception e) { - throw new JedisException(e); - } - } - - @Override - public Long scard(final String key) { - try { - return redis.scard(key); - } catch (Exception e) { - throw new JedisException(e); - } - } - - @Override - public Boolean sismember(final String key, final String member) { - try { - return redis.sismember(key, member); - } catch (Exception e) { - throw new JedisException(e); - } - } - - @Override - public Set sinter(final String... keys) { - try { - String key = keys[0]; - String[] k = new String[keys.length - 1]; - for (int idx = 0; idx < keys.length; ++idx) { - k[idx - 1] = keys[idx]; - } - return redis.sinter(key, k); - } catch (Exception e) { - throw new JedisException(e); - } - } - - @Override - public Long sinterstore(final String dstkey, final String... keys) { - try { - String key = keys[0]; - String[] k = new String[keys.length - 1]; - for (int idx = 0; idx < keys.length; ++idx) { - k[idx - 1] = keys[idx]; - } - return redis.sinterstore(dstkey, key, k); - } catch (Exception e) { - throw new JedisException(e); - } - } - - @Override - public Set sunion(final String... keys) { - try { - String key = keys[0]; - String[] k = new String[keys.length - 1]; - for (int idx = 0; idx < keys.length; ++idx) { - k[idx - 1] = keys[idx]; - } - return redis.sunion(key, k); - } catch (Exception e) { - throw new JedisException(e); - } - } - - @Override - public Long sunionstore(final String dstkey, final String... keys) { - try { - String key = keys[0]; - String[] k = new String[keys.length - 1]; - for (int idx = 0; idx < keys.length; ++idx) { - k[idx - 1] = keys[idx]; - } - return redis.sunionstore(dstkey, key, k); - } catch (Exception e) { - throw new JedisException(e); - } - } - - @Override - public Set sdiff(final String... keys) { - try { - String key = keys[0]; - String[] k = new String[keys.length - 1]; - for (int idx = 0; idx < keys.length; ++idx) { - k[idx - 1] = keys[idx]; - } - return redis.sdiff(key, k); - } catch (Exception e) { - throw new JedisException(e); - } - } - - @Override - public Long sdiffstore(final String dstkey, final String... keys) { - try { - String key = keys[0]; - String[] k = new String[keys.length - 1]; - for (int idx = 0; idx < keys.length; ++idx) { - k[idx - 1] = keys[idx]; - } - return redis.sdiffstore(dstkey, key, k); - } catch (Exception e) { - throw new JedisException(e); - } - } - - @Override - public String srandmember(final String key) { - try { - return redis.srandmember(key); - } catch (Exception e) { - throw new JedisException(e); - } - } - - @Override - public List srandmember(final String key, final int count) { - try { - return redis.srandmember(key, count); - } catch (Exception e) { - throw new JedisException(e); - } - } - - @Override - public Long zadd(final String key, final double score, final String member) { - try { - return redis.zadd(key, new ZsetPair(member, score)); - } catch (Exception e) { - throw new JedisException(e); - } - } - - @Override - public Long zadd(String key, double score, String member, ZAddParams params) { - - try { - - if(params.contains("xx")) { - Double existing = redis.zscore(key, member); - if(existing == null) { - return 0L; - } - redis.zadd(key, new ZsetPair(member, score)); - return 1L; - }else { - return redis.zadd(key, new ZsetPair(member, score)); - } - - } catch (Exception e) { - throw new JedisException(e); - } - } - - - @Override - public Long zadd(final String key, final Map scoreMembers) { - try { - Double score = null; - String member = null; - List scoresmembers = new ArrayList((scoreMembers.size() - 1) * 2); - for (String m : scoreMembers.keySet()) { - if (m == null) { - member = m; - score = scoreMembers.get(m); - continue; - } - scoresmembers.add(new ZsetPair(m, scoreMembers.get(m))); - } - return redis.zadd(key, new ZsetPair(member, score), (ZsetPair[]) scoresmembers.toArray()); - } catch (Exception e) { - throw new JedisException(e); - } - } - - @Override - public Set zrange(final String key, final long start, final long end) { - try { - return ZsetPair.members(redis.zrange(key, start, end)); - } catch (Exception e) { - throw new JedisException(e); - } - } - - @Override - public Long zrem(final String key, final String... members) { - try { - String member = members[0]; - String[] ms = new String[members.length - 1]; - for (int idx = 1; idx < members.length; ++idx) { - ms[idx - 1] = members[idx]; - } - return redis.zrem(key, member, ms); - } catch (Exception e) { - throw new JedisException(e); - } - } - - @Override - public Double zincrby(final String key, final double score, final String member) { - try { - return Double.parseDouble(redis.zincrby(key, score, member)); - } catch (Exception e) { - throw new JedisException(e); - } - } - - @Override - public Long zrank(final String key, final String member) { - try { - return redis.zrank(key, member); - } catch (Exception e) { - throw new JedisException(e); - } - } - - @Override - public Long zrevrank(final String key, final String member) { - try { - return redis.zrevrank(key, member); - } catch (Exception e) { - throw new JedisException(e); - } - } - - @Override - public Set zrevrange(final String key, final long start, final long end) { - try { - return ZsetPair.members(redis.zrevrange(key, start, end)); - } catch (Exception e) { - throw new JedisException(e); - } - } - - @Override - public Set zrangeWithScores(final String key, final long start, final long end) { - try { - return toTupleSet(redis.zrange(key, start, end, "withscores")); - } catch (Exception e) { - throw new JedisException(e); - } - } - - @Override - public Set zrevrangeWithScores(final String key, final long start, final long end) { - try { - return toTupleSet(redis.zrevrange(key, start, end, "withscores")); - } catch (Exception e) { - throw new JedisException(e); - } - } - - @Override - public Long zcard(final String key) { - try { - return redis.zcard(key); - } catch (Exception e) { - throw new JedisException(e); - } - } - - @Override - public Double zscore(final String key, final String member) { - try { - return redis.zscore(key, member); - } catch (Exception e) { - throw new JedisException(e); - } - } - - @Override - public String watch(final String... keys) { - try { - for (String key : keys) { - redis.watch(key); - } - return "OK"; - } catch (Exception e) { - throw new JedisException(e); - } - } - - /* - * public List sort(final String key) { checkIsInMulti(); - * client.sort(key); return client.getMultiBulkReply(); } - * - * public List sort(final String key, final SortingParams - * sortingParameters) { checkIsInMulti(); client.sort(key, - * sortingParameters); return client.getMultiBulkReply(); } - * - * public List blpop(final int timeout, final String... keys) { - * return blpop(getArgsAddTimeout(timeout, keys)); } - * - * private String[] getArgsAddTimeout(int timeout, String[] keys) { final - * int keyCount = keys.length; final String[] args = new String[keyCount + - * 1]; for (int at = 0; at != keyCount; ++at) { args[at] = keys[at]; } - * - * args[keyCount] = String.valueOf(timeout); return args; } - * - * public List blpop(String... args) { checkIsInMulti(); - * client.blpop(args); client.setTimeoutInfinite(); try { return - * client.getMultiBulkReply(); } finally { client.rollbackTimeout(); } } - * - * public List brpop(String... args) { checkIsInMulti(); - * client.brpop(args); client.setTimeoutInfinite(); try { return - * client.getMultiBulkReply(); } finally { client.rollbackTimeout(); } } - * - * @Deprecated public List blpop(String arg) { return blpop(new - * String[] { arg }); } - * - * public List brpop(String arg) { return brpop(new String[] { arg - * }); } - * - * public Long sort(final String key, final SortingParams sortingParameters, - * final String dstkey) { checkIsInMulti(); client.sort(key, - * sortingParameters, dstkey); return client.getIntegerReply(); } - * - * public Long sort(final String key, final String dstkey) { - * checkIsInMulti(); client.sort(key, dstkey); return - * client.getIntegerReply(); } - * - * public List brpop(final int timeout, final String... keys) { - * return brpop(getArgsAddTimeout(timeout, keys)); } - */ - @Override - public Long zcount(final String key, final double min, final double max) { - try { - return redis.zcount(key, min, max); - } catch (Exception e) { - throw new JedisException(e); - } - } - - @Override - public Long zcount(final String key, final String min, final String max) { - try { - return redis.zcount(key, Double.parseDouble(min), Double.parseDouble(max)); - } catch (Exception e) { - throw new JedisException(e); - } - } - - @Override - public Set zrangeByScore(final String key, final double min, final double max) { - try { - return ZsetPair.members(redis.zrangebyscore(key, String.valueOf(min), String.valueOf(max))); - } catch (Exception e) { - throw new JedisException(e); - } - } - - @Override - public Set zrangeByScore(final String key, final String min, final String max) { - try { - return ZsetPair.members(redis.zrangebyscore(key, min, max)); - } catch (Exception e) { - throw new JedisException(e); - } - } - - @Override - public Set zrangeByScore(final String key, final double min, final double max, final int offset, final int count) { - try { - return ZsetPair.members(redis.zrangebyscore(key, String.valueOf(min), String.valueOf(max), "limit", String.valueOf(offset), - String.valueOf(count))); - } catch (Exception e) { - throw new JedisException(e); - } - } - - @Override - public Set zrangeByScore(final String key, final String min, final String max, final int offset, final int count) { - try { - return ZsetPair.members(redis.zrangebyscore(key, min, max, "limit", String.valueOf(offset), String.valueOf(count))); - } catch (Exception e) { - throw new JedisException(e); - } - } - - @Override - public Set zrangeByScoreWithScores(final String key, final double min, final double max) { - try { - return toTupleSet(redis.zrangebyscore(key, String.valueOf(min), String.valueOf(max), "withscores")); - } catch (Exception e) { - throw new JedisException(e); - } - } - - @Override - public Set zrangeByScoreWithScores(final String key, final String min, final String max) { - try { - return toTupleSet(redis.zrangebyscore(key, min, max, "withscores")); - } catch (Exception e) { - throw new JedisException(e); - } - } - - @Override - public Set zrangeByScoreWithScores(final String key, final double min, final double max, final int offset, final int count) { - try { - return toTupleSet(redis.zrangebyscore(key, String.valueOf(min), String.valueOf(max), "limit", String.valueOf(offset), - String.valueOf(count), "withscores")); - } catch (Exception e) { - throw new JedisException(e); - } - } - - @Override - public Set zrangeByScoreWithScores(final String key, final String min, final String max, final int offset, final int count) { - try { - return toTupleSet(redis.zrangebyscore(key, min, max, "limit", String.valueOf(offset), String.valueOf(count), "withscores")); - } catch (Exception e) { - throw new JedisException(e); - } - } - - @Override - public Set zrevrangeByScore(final String key, final double max, final double min) { - try { - return ZsetPair.members(redis.zrevrangebyscore(key, String.valueOf(max), String.valueOf(min))); - } catch (Exception e) { - throw new JedisException(e); - } - } - - @Override - public Set zrevrangeByScore(final String key, final String max, final String min) { - try { - return ZsetPair.members(redis.zrevrangebyscore(key, max, min)); - } catch (Exception e) { - throw new JedisException(e); - } - } - - @Override - public Set zrevrangeByScore(final String key, final double max, final double min, final int offset, final int count) { - try { - return ZsetPair.members(redis.zrevrangebyscore(key, String.valueOf(max), String.valueOf(min), "limit", String.valueOf(offset), - String.valueOf(count))); - } catch (Exception e) { - throw new JedisException(e); - } - } - - @Override - public Set zrevrangeByScoreWithScores(final String key, final double max, final double min) { - try { - return toTupleSet(redis.zrevrangebyscore(key, String.valueOf(max), String.valueOf(min), "withscores")); - } catch (Exception e) { - throw new JedisException(e); - } - } - - @Override - public Set zrevrangeByScoreWithScores(final String key, final double max, final double min, final int offset, final int count) { - try { - return toTupleSet(redis.zrevrangebyscore(key, String.valueOf(max), String.valueOf(min), "limit", String.valueOf(offset), - String.valueOf(count), "withscores")); - } catch (Exception e) { - throw new JedisException(e); - } - } - - @Override - public Set zrevrangeByScoreWithScores(final String key, final String max, final String min, final int offset, final int count) { - try { - return toTupleSet(redis.zrevrangebyscore(key, max, min, "limit", String.valueOf(offset), String.valueOf(count), "withscores")); - } catch (Exception e) { - throw new JedisException(e); - } - } - - @Override - public Set zrevrangeByScore(final String key, final String max, final String min, final int offset, final int count) { - try { - return ZsetPair.members(redis.zrevrangebyscore(key, max, min, "limit", String.valueOf(offset), String.valueOf(count))); - } catch (Exception e) { - throw new JedisException(e); - } - } - - @Override - public Set zrevrangeByScoreWithScores(final String key, final String max, final String min) { - try { - return toTupleSet(redis.zrevrangebyscore(key, max, min, "withscores")); - } catch (Exception e) { - throw new JedisException(e); - } - } - - @Override - public Long zremrangeByRank(final String key, final long start, final long end) { - try { - return redis.zremrangebyrank(key, start, end); - } catch (Exception e) { - throw new JedisException(e); - } - } - - @Override - public Long zremrangeByScore(final String key, final double start, final double end) { - try { - return redis.zremrangebyscore(key, String.valueOf(start), String.valueOf(end)); - } catch (Exception e) { - throw new JedisException(e); - } - } - - @Override - public Long zremrangeByScore(final String key, final String start, final String end) { - try { - return redis.zremrangebyscore(key, start, end); - } catch (Exception e) { - throw new JedisException(e); - } - } - - @Override - public Long zunionstore(final String dstkey, final String... sets) { - try { - return redis.zunionstore(dstkey, sets.length, sets); - } catch (Exception e) { - throw new JedisException(e); - } - } - - @Override - public ScanResult sscan(String key, String cursor, ScanParams params) { - try { - org.rarefiedredis.redis.ScanResult> sr = redis.sscan(key, Long.valueOf(cursor), "count", "1000000"); - List list = sr.results.stream().collect(Collectors.toList()); - ScanResult result = new ScanResult("0", list); - return result; - } catch (Exception e) { - throw new JedisException(e); - } - } - - public ScanResult> hscan(final String key, final String cursor) { - try { - org.rarefiedredis.redis.ScanResult> mockr = redis.hscan(key, Long.valueOf(cursor), "count", "1000000"); - Map results = mockr.results; - List> list = results.entrySet().stream().collect(Collectors.toList()); - ScanResult> result = new ScanResult>("0", list); - - return result; - } catch (Exception e) { - throw new JedisException(e); - } - } - - public ScanResult zscan(final String key, final String cursor) { - try { - org.rarefiedredis.redis.ScanResult> sr = redis.zscan(key, Long.valueOf(cursor), "count", "1000000"); - List list = sr.results.stream().collect(Collectors.toList()); - List tl = new LinkedList(); - list.forEach(p -> tl.add(new Tuple(p.member, p.score))); - ScanResult result = new ScanResult("0", tl); - return result; - } catch (Exception e) { - throw new JedisException(e); - } - } + private IRedisClient redis; + + public JedisMock() { + super(""); + this.redis = new RedisMock(); + } + + private Set toTupleSet(Set pairs) { + Set set = new HashSet(); + for (ZsetPair pair : pairs) { + set.add(new Tuple(pair.member, pair.score)); + } + return set; + } + + @Override + public String set(final String key, String value) { + try { + return redis.set(key, value); + } catch (Exception e) { + throw new JedisException(e); + } + } + + @Override + public String set(final String key, final String value, final String nxxx, final String expx, final long time) { + try { + return redis.set(key, value, nxxx, expx, String.valueOf(time)); + } catch (Exception e) { + throw new JedisException(e); + } + } + + @Override + public String get(final String key) { + try { + return redis.get(key); + } catch (Exception e) { + throw new JedisException(e); + } + } + + @Override + public Boolean exists(final String key) { + try { + return redis.exists(key); + } catch (Exception e) { + throw new JedisException(e); + } + } + + @Override + public Long del(final String... keys) { + try { + return redis.del(keys); + } catch (Exception e) { + throw new JedisException(e); + } + } + + @Override + public Long del(String key) { + try { + return redis.del(key); + } catch (Exception e) { + throw new JedisException(e); + } + } + + @Override + public String type(final String key) { + try { + return redis.type(key); + } catch (Exception e) { + throw new JedisException(e); + } + } + + /* + * public Set keys(final String pattern) { checkIsInMulti(); + * client.keys(pattern); return + * BuilderFactory.STRING_SET.build(client.getBinaryMultiBulkReply()); } + * + * public String randomKey() { checkIsInMulti(); client.randomKey(); return + * client.getBulkReply(); } + * + * public String rename(final String oldkey, final String newkey) { + * checkIsInMulti(); client.rename(oldkey, newkey); return + * client.getStatusCodeReply(); } + * + * public Long renamenx(final String oldkey, final String newkey) { + * checkIsInMulti(); client.renamenx(oldkey, newkey); return + * client.getIntegerReply(); } + */ + @Override + public Long expire(final String key, final int seconds) { + try { + return redis.expire(key, seconds) ? 1L : 0L; + } catch (Exception e) { + throw new JedisException(e); + } + } + + @Override + public Long expireAt(final String key, final long unixTime) { + try { + return redis.expireat(key, unixTime) ? 1L : 0L; + } catch (Exception e) { + throw new JedisException(e); + } + } + + @Override + public Long ttl(final String key) { + try { + return redis.ttl(key); + } catch (Exception e) { + throw new JedisException(e); + } + } + + @Override + public Long move(final String key, final int dbIndex) { + try { + return redis.move(key, dbIndex); + } catch (Exception e) { + throw new JedisException(e); + } + } + + @Override + public String getSet(final String key, final String value) { + try { + return redis.getset(key, value); + } catch (Exception e) { + throw new JedisException(e); + } + } + + @Override + public List mget(final String... keys) { + try { + String[] mget = redis.mget(keys); + List lst = new ArrayList(mget.length); + for (String get : mget) { + lst.add(get); + } + return lst; + } catch (Exception e) { + throw new JedisException(e); + } + } + + @Override + public Long setnx(final String key, final String value) { + try { + return redis.setnx(key, value); + } catch (Exception e) { + throw new JedisException(e); + } + } + + @Override + public String setex(final String key, final int seconds, final String value) { + try { + return redis.setex(key, seconds, value); + } catch (Exception e) { + throw new JedisException(e); + } + } + + @Override + public String mset(final String... keysvalues) { + try { + return redis.mset(keysvalues); + } catch (Exception e) { + throw new JedisException(e); + } + } + + @Override + public Long msetnx(final String... keysvalues) { + try { + return redis.msetnx(keysvalues) ? 1L : 0L; + } catch (Exception e) { + throw new JedisException(e); + } + } + + @Override + public Long decrBy(final String key, final long integer) { + try { + return redis.decrby(key, integer); + } catch (Exception e) { + throw new JedisException(e); + } + } + + @Override + public Long decr(final String key) { + try { + return redis.decr(key); + } catch (Exception e) { + throw new JedisException(e); + } + } + + @Override + public Long incrBy(final String key, final long integer) { + try { + return redis.incrby(key, integer); + } catch (Exception e) { + throw new JedisException(e); + } + } + + @Override + public Double incrByFloat(final String key, final double value) { + try { + return Double.parseDouble(redis.incrbyfloat(key, value)); + } catch (Exception e) { + throw new JedisException(e); + } + } + + @Override + public Long incr(final String key) { + try { + return redis.incr(key); + } catch (Exception e) { + throw new JedisException(e); + } + } + + @Override + public Long append(final String key, final String value) { + try { + return redis.append(key, value); + } catch (Exception e) { + throw new JedisException(e); + } + } + + @Override + public String substr(final String key, final int start, final int end) { + try { + return redis.getrange(key, start, end); + } catch (Exception e) { + throw new JedisException(e); + } + } + + @Override + public Long hset(final String key, final String field, final String value) { + try { + return redis.hset(key, field, value) ? 1L : 0L; + } catch (Exception e) { + throw new JedisException(e); + } + } + + @Override + public String hget(final String key, final String field) { + try { + return redis.hget(key, field); + } catch (Exception e) { + throw new JedisException(e); + } + } + + @Override + public Long hsetnx(final String key, final String field, final String value) { + try { + return redis.hsetnx(key, field, value) ? 1L : 0L; + } catch (Exception e) { + throw new JedisException(e); + } + } + + @Override + public String hmset(final String key, final Map hash) { + try { + String field = null, value = null; + String[] args = new String[(hash.size() - 1) * 2]; + int idx = 0; + for (String f : hash.keySet()) { + if (field == null) { + field = f; + value = hash.get(f); + continue; + } + args[idx] = f; + args[idx + 1] = hash.get(f); + idx += 2; + } + return redis.hmset(key, field, value, args); + } catch (Exception e) { + throw new JedisException(e); + } + } + + @Override + public List hmget(final String key, final String... fields) { + try { + String field = fields[0]; + String[] f = new String[fields.length - 1]; + for (int idx = 1; idx < fields.length; ++idx) { + f[idx - 1] = fields[idx]; + } + return redis.hmget(key, field, f); + } catch (Exception e) { + throw new JedisException(e); + } + } + + @Override + public Long hincrBy(final String key, final String field, final long value) { + try { + return redis.hincrby(key, field, value); + } catch (Exception e) { + throw new JedisException(e); + } + } + + @Override + public Double hincrByFloat(final String key, final String field, final double value) { + try { + return Double.parseDouble(redis.hincrbyfloat(key, field, value)); + } catch (Exception e) { + throw new JedisException(e); + } + } + + @Override + public Boolean hexists(final String key, final String field) { + try { + return redis.hexists(key, field); + } catch (Exception e) { + throw new JedisException(e); + } + } + + @Override + public Long hdel(final String key, final String... fields) { + try { + String field = fields[0]; + String[] f = new String[fields.length - 1]; + for (int idx = 1; idx < fields.length; ++idx) { + f[idx - 1] = fields[idx]; + } + return redis.hdel(key, field, f); + } catch (Exception e) { + throw new JedisException(e); + } + } + + @Override + public Long hlen(final String key) { + try { + return redis.hlen(key); + } catch (Exception e) { + throw new JedisException(e); + } + } + + @Override + public Set hkeys(final String key) { + try { + return redis.hkeys(key); + } catch (Exception e) { + throw new JedisException(e); + } + } + + @Override + public List hvals(final String key) { + try { + return redis.hvals(key); + } catch (Exception e) { + throw new JedisException(e); + } + } + + @Override + public Map hgetAll(final String key) { + try { + return redis.hgetall(key); + } catch (Exception e) { + throw new JedisException(e); + } + } + + @Override + public Long rpush(final String key, final String... strings) { + try { + String element = strings[0]; + String[] elements = new String[strings.length - 1]; + for (int idx = 1; idx < strings.length; ++idx) { + elements[idx - 1] = strings[idx]; + } + return redis.rpush(key, element, elements); + } catch (Exception e) { + throw new JedisException(e); + } + } + + @Override + public Long lpush(final String key, final String... strings) { + try { + String element = strings[0]; + String[] elements = new String[strings.length - 1]; + for (int idx = 1; idx < strings.length; ++idx) { + elements[idx - 1] = strings[idx]; + } + return redis.lpush(key, element, elements); + } catch (Exception e) { + throw new JedisException(e); + } + } + + @Override + public Long llen(final String key) { + try { + return redis.llen(key); + } catch (Exception e) { + throw new JedisException(e); + } + } + + @Override + public List lrange(final String key, final long start, final long end) { + try { + return redis.lrange(key, start, end); + } catch (Exception e) { + throw new JedisException(e); + } + } + + @Override + public String ltrim(final String key, final long start, final long end) { + try { + return redis.ltrim(key, start, end); + } catch (Exception e) { + throw new JedisException(e); + } + } + + @Override + public String lindex(final String key, final long index) { + try { + return redis.lindex(key, index); + } catch (Exception e) { + throw new JedisException(e); + } + } + + @Override + public String lset(final String key, final long index, final String value) { + try { + return redis.lset(key, index, value); + } catch (Exception e) { + throw new JedisException(e); + } + } + + @Override + public Long lrem(final String key, final long count, final String value) { + try { + return redis.lrem(key, count, value); + } catch (Exception e) { + throw new JedisException(e); + } + } + + @Override + public String lpop(final String key) { + try { + return redis.lpop(key); + } catch (Exception e) { + throw new JedisException(e); + } + } + + @Override + public String rpop(final String key) { + try { + return redis.rpop(key); + } catch (Exception e) { + throw new JedisException(e); + } + } + + @Override + public String rpoplpush(final String srckey, final String dstkey) { + try { + return redis.rpoplpush(srckey, dstkey); + } catch (Exception e) { + throw new JedisException(e); + } + } + + @Override + public Long sadd(final String key, final String... members) { + try { + String member = members[0]; + String[] m = new String[members.length - 1]; + for (int idx = 1; idx < members.length; ++idx) { + m[idx - 1] = members[idx]; + } + return redis.sadd(key, member, m); + } catch (Exception e) { + throw new JedisException(e); + } + } + + @Override + public Set smembers(final String key) { + try { + return redis.smembers(key); + } catch (Exception e) { + throw new JedisException(e); + } + } + + @Override + public Long srem(final String key, final String... members) { + try { + String member = members[0]; + String[] m = new String[members.length - 1]; + for (int idx = 1; idx < members.length; ++idx) { + m[idx - 1] = members[idx]; + } + return redis.srem(key, member, m); + } catch (Exception e) { + throw new JedisException(e); + } + } + + @Override + public String spop(final String key) { + try { + return redis.spop(key); + } catch (Exception e) { + throw new JedisException(e); + } + } + + @Override + public Long smove(final String srckey, final String dstkey, final String member) { + try { + return redis.smove(srckey, dstkey, member) ? 1L : 0L; + } catch (Exception e) { + throw new JedisException(e); + } + } + + @Override + public Long scard(final String key) { + try { + return redis.scard(key); + } catch (Exception e) { + throw new JedisException(e); + } + } + + @Override + public Boolean sismember(final String key, final String member) { + try { + return redis.sismember(key, member); + } catch (Exception e) { + throw new JedisException(e); + } + } + + @Override + public Set sinter(final String... keys) { + try { + String key = keys[0]; + String[] k = new String[keys.length - 1]; + for (int idx = 0; idx < keys.length; ++idx) { + k[idx - 1] = keys[idx]; + } + return redis.sinter(key, k); + } catch (Exception e) { + throw new JedisException(e); + } + } + + @Override + public Long sinterstore(final String dstkey, final String... keys) { + try { + String key = keys[0]; + String[] k = new String[keys.length - 1]; + for (int idx = 0; idx < keys.length; ++idx) { + k[idx - 1] = keys[idx]; + } + return redis.sinterstore(dstkey, key, k); + } catch (Exception e) { + throw new JedisException(e); + } + } + + @Override + public Set sunion(final String... keys) { + try { + String key = keys[0]; + String[] k = new String[keys.length - 1]; + for (int idx = 0; idx < keys.length; ++idx) { + k[idx - 1] = keys[idx]; + } + return redis.sunion(key, k); + } catch (Exception e) { + throw new JedisException(e); + } + } + + @Override + public Long sunionstore(final String dstkey, final String... keys) { + try { + String key = keys[0]; + String[] k = new String[keys.length - 1]; + for (int idx = 0; idx < keys.length; ++idx) { + k[idx - 1] = keys[idx]; + } + return redis.sunionstore(dstkey, key, k); + } catch (Exception e) { + throw new JedisException(e); + } + } + + @Override + public Set sdiff(final String... keys) { + try { + String key = keys[0]; + String[] k = new String[keys.length - 1]; + for (int idx = 0; idx < keys.length; ++idx) { + k[idx - 1] = keys[idx]; + } + return redis.sdiff(key, k); + } catch (Exception e) { + throw new JedisException(e); + } + } + + @Override + public Long sdiffstore(final String dstkey, final String... keys) { + try { + String key = keys[0]; + String[] k = new String[keys.length - 1]; + for (int idx = 0; idx < keys.length; ++idx) { + k[idx - 1] = keys[idx]; + } + return redis.sdiffstore(dstkey, key, k); + } catch (Exception e) { + throw new JedisException(e); + } + } + + @Override + public String srandmember(final String key) { + try { + return redis.srandmember(key); + } catch (Exception e) { + throw new JedisException(e); + } + } + + @Override + public List srandmember(final String key, final int count) { + try { + return redis.srandmember(key, count); + } catch (Exception e) { + throw new JedisException(e); + } + } + + @Override + public Long zadd(final String key, final double score, final String member) { + try { + return redis.zadd(key, new ZsetPair(member, score)); + } catch (Exception e) { + throw new JedisException(e); + } + } + + @Override + public Long zadd(String key, double score, String member, ZAddParams params) { + + try { + + if (params.contains("xx")) { + Double existing = redis.zscore(key, member); + if (existing == null) { + return 0L; + } + redis.zadd(key, new ZsetPair(member, score)); + return 1L; + } else { + return redis.zadd(key, new ZsetPair(member, score)); + } + + } catch (Exception e) { + throw new JedisException(e); + } + } + + + @Override + public Long zadd(final String key, final Map scoreMembers) { + try { + Double score = null; + String member = null; + List scoresmembers = new ArrayList((scoreMembers.size() - 1) * 2); + for (String m : scoreMembers.keySet()) { + if (m == null) { + member = m; + score = scoreMembers.get(m); + continue; + } + scoresmembers.add(new ZsetPair(m, scoreMembers.get(m))); + } + return redis.zadd(key, new ZsetPair(member, score), (ZsetPair[]) scoresmembers.toArray()); + } catch (Exception e) { + throw new JedisException(e); + } + } + + @Override + public Set zrange(final String key, final long start, final long end) { + try { + return ZsetPair.members(redis.zrange(key, start, end)); + } catch (Exception e) { + throw new JedisException(e); + } + } + + @Override + public Long zrem(final String key, final String... members) { + try { + String member = members[0]; + String[] ms = new String[members.length - 1]; + for (int idx = 1; idx < members.length; ++idx) { + ms[idx - 1] = members[idx]; + } + return redis.zrem(key, member, ms); + } catch (Exception e) { + throw new JedisException(e); + } + } + + @Override + public Double zincrby(final String key, final double score, final String member) { + try { + return Double.parseDouble(redis.zincrby(key, score, member)); + } catch (Exception e) { + throw new JedisException(e); + } + } + + @Override + public Long zrank(final String key, final String member) { + try { + return redis.zrank(key, member); + } catch (Exception e) { + throw new JedisException(e); + } + } + + @Override + public Long zrevrank(final String key, final String member) { + try { + return redis.zrevrank(key, member); + } catch (Exception e) { + throw new JedisException(e); + } + } + + @Override + public Set zrevrange(final String key, final long start, final long end) { + try { + return ZsetPair.members(redis.zrevrange(key, start, end)); + } catch (Exception e) { + throw new JedisException(e); + } + } + + @Override + public Set zrangeWithScores(final String key, final long start, final long end) { + try { + return toTupleSet(redis.zrange(key, start, end, "withscores")); + } catch (Exception e) { + throw new JedisException(e); + } + } + + @Override + public Set zrevrangeWithScores(final String key, final long start, final long end) { + try { + return toTupleSet(redis.zrevrange(key, start, end, "withscores")); + } catch (Exception e) { + throw new JedisException(e); + } + } + + @Override + public Long zcard(final String key) { + try { + return redis.zcard(key); + } catch (Exception e) { + throw new JedisException(e); + } + } + + @Override + public Double zscore(final String key, final String member) { + try { + return redis.zscore(key, member); + } catch (Exception e) { + throw new JedisException(e); + } + } + + @Override + public String watch(final String... keys) { + try { + for (String key : keys) { + redis.watch(key); + } + return "OK"; + } catch (Exception e) { + throw new JedisException(e); + } + } + + /* + * public List sort(final String key) { checkIsInMulti(); + * client.sort(key); return client.getMultiBulkReply(); } + * + * public List sort(final String key, final SortingParams + * sortingParameters) { checkIsInMulti(); client.sort(key, + * sortingParameters); return client.getMultiBulkReply(); } + * + * public List blpop(final int timeout, final String... keys) { + * return blpop(getArgsAddTimeout(timeout, keys)); } + * + * private String[] getArgsAddTimeout(int timeout, String[] keys) { final + * int keyCount = keys.length; final String[] args = new String[keyCount + + * 1]; for (int at = 0; at != keyCount; ++at) { args[at] = keys[at]; } + * + * args[keyCount] = String.valueOf(timeout); return args; } + * + * public List blpop(String... args) { checkIsInMulti(); + * client.blpop(args); client.setTimeoutInfinite(); try { return + * client.getMultiBulkReply(); } finally { client.rollbackTimeout(); } } + * + * public List brpop(String... args) { checkIsInMulti(); + * client.brpop(args); client.setTimeoutInfinite(); try { return + * client.getMultiBulkReply(); } finally { client.rollbackTimeout(); } } + * + * @Deprecated public List blpop(String arg) { return blpop(new + * String[] { arg }); } + * + * public List brpop(String arg) { return brpop(new String[] { arg + * }); } + * + * public Long sort(final String key, final SortingParams sortingParameters, + * final String dstkey) { checkIsInMulti(); client.sort(key, + * sortingParameters, dstkey); return client.getIntegerReply(); } + * + * public Long sort(final String key, final String dstkey) { + * checkIsInMulti(); client.sort(key, dstkey); return + * client.getIntegerReply(); } + * + * public List brpop(final int timeout, final String... keys) { + * return brpop(getArgsAddTimeout(timeout, keys)); } + */ + @Override + public Long zcount(final String key, final double min, final double max) { + try { + return redis.zcount(key, min, max); + } catch (Exception e) { + throw new JedisException(e); + } + } + + @Override + public Long zcount(final String key, final String min, final String max) { + try { + return redis.zcount(key, Double.parseDouble(min), Double.parseDouble(max)); + } catch (Exception e) { + throw new JedisException(e); + } + } + + @Override + public Set zrangeByScore(final String key, final double min, final double max) { + try { + return ZsetPair.members(redis.zrangebyscore(key, String.valueOf(min), String.valueOf(max))); + } catch (Exception e) { + throw new JedisException(e); + } + } + + @Override + public Set zrangeByScore(final String key, final String min, final String max) { + try { + return ZsetPair.members(redis.zrangebyscore(key, min, max)); + } catch (Exception e) { + throw new JedisException(e); + } + } + + @Override + public Set zrangeByScore(final String key, final double min, final double max, final int offset, final int count) { + try { + return ZsetPair.members(redis.zrangebyscore(key, String.valueOf(min), String.valueOf(max), "limit", String.valueOf(offset), + String.valueOf(count))); + } catch (Exception e) { + throw new JedisException(e); + } + } + + @Override + public Set zrangeByScore(final String key, final String min, final String max, final int offset, final int count) { + try { + return ZsetPair.members(redis.zrangebyscore(key, min, max, "limit", String.valueOf(offset), String.valueOf(count))); + } catch (Exception e) { + throw new JedisException(e); + } + } + + @Override + public Set zrangeByScoreWithScores(final String key, final double min, final double max) { + try { + return toTupleSet(redis.zrangebyscore(key, String.valueOf(min), String.valueOf(max), "withscores")); + } catch (Exception e) { + throw new JedisException(e); + } + } + + @Override + public Set zrangeByScoreWithScores(final String key, final String min, final String max) { + try { + return toTupleSet(redis.zrangebyscore(key, min, max, "withscores")); + } catch (Exception e) { + throw new JedisException(e); + } + } + + @Override + public Set zrangeByScoreWithScores(final String key, final double min, final double max, final int offset, final int count) { + try { + return toTupleSet(redis.zrangebyscore(key, String.valueOf(min), String.valueOf(max), "limit", String.valueOf(offset), + String.valueOf(count), "withscores")); + } catch (Exception e) { + throw new JedisException(e); + } + } + + @Override + public Set zrangeByScoreWithScores(final String key, final String min, final String max, final int offset, final int count) { + try { + return toTupleSet(redis.zrangebyscore(key, min, max, "limit", String.valueOf(offset), String.valueOf(count), "withscores")); + } catch (Exception e) { + throw new JedisException(e); + } + } + + @Override + public Set zrevrangeByScore(final String key, final double max, final double min) { + try { + return ZsetPair.members(redis.zrevrangebyscore(key, String.valueOf(max), String.valueOf(min))); + } catch (Exception e) { + throw new JedisException(e); + } + } + + @Override + public Set zrevrangeByScore(final String key, final String max, final String min) { + try { + return ZsetPair.members(redis.zrevrangebyscore(key, max, min)); + } catch (Exception e) { + throw new JedisException(e); + } + } + + @Override + public Set zrevrangeByScore(final String key, final double max, final double min, final int offset, final int count) { + try { + return ZsetPair.members(redis.zrevrangebyscore(key, String.valueOf(max), String.valueOf(min), "limit", String.valueOf(offset), + String.valueOf(count))); + } catch (Exception e) { + throw new JedisException(e); + } + } + + @Override + public Set zrevrangeByScoreWithScores(final String key, final double max, final double min) { + try { + return toTupleSet(redis.zrevrangebyscore(key, String.valueOf(max), String.valueOf(min), "withscores")); + } catch (Exception e) { + throw new JedisException(e); + } + } + + @Override + public Set zrevrangeByScoreWithScores(final String key, final double max, final double min, final int offset, final int count) { + try { + return toTupleSet(redis.zrevrangebyscore(key, String.valueOf(max), String.valueOf(min), "limit", String.valueOf(offset), + String.valueOf(count), "withscores")); + } catch (Exception e) { + throw new JedisException(e); + } + } + + @Override + public Set zrevrangeByScoreWithScores(final String key, final String max, final String min, final int offset, final int count) { + try { + return toTupleSet(redis.zrevrangebyscore(key, max, min, "limit", String.valueOf(offset), String.valueOf(count), "withscores")); + } catch (Exception e) { + throw new JedisException(e); + } + } + + @Override + public Set zrevrangeByScore(final String key, final String max, final String min, final int offset, final int count) { + try { + return ZsetPair.members(redis.zrevrangebyscore(key, max, min, "limit", String.valueOf(offset), String.valueOf(count))); + } catch (Exception e) { + throw new JedisException(e); + } + } + + @Override + public Set zrevrangeByScoreWithScores(final String key, final String max, final String min) { + try { + return toTupleSet(redis.zrevrangebyscore(key, max, min, "withscores")); + } catch (Exception e) { + throw new JedisException(e); + } + } + + @Override + public Long zremrangeByRank(final String key, final long start, final long end) { + try { + return redis.zremrangebyrank(key, start, end); + } catch (Exception e) { + throw new JedisException(e); + } + } + + @Override + public Long zremrangeByScore(final String key, final double start, final double end) { + try { + return redis.zremrangebyscore(key, String.valueOf(start), String.valueOf(end)); + } catch (Exception e) { + throw new JedisException(e); + } + } + + @Override + public Long zremrangeByScore(final String key, final String start, final String end) { + try { + return redis.zremrangebyscore(key, start, end); + } catch (Exception e) { + throw new JedisException(e); + } + } + + @Override + public Long zunionstore(final String dstkey, final String... sets) { + try { + return redis.zunionstore(dstkey, sets.length, sets); + } catch (Exception e) { + throw new JedisException(e); + } + } + + @Override + public ScanResult sscan(String key, String cursor, ScanParams params) { + try { + org.rarefiedredis.redis.ScanResult> sr = redis.sscan(key, Long.valueOf(cursor), "count", "1000000"); + List list = sr.results.stream().collect(Collectors.toList()); + ScanResult result = new ScanResult("0", list); + return result; + } catch (Exception e) { + throw new JedisException(e); + } + } + + public ScanResult> hscan(final String key, final String cursor) { + try { + org.rarefiedredis.redis.ScanResult> mockr = redis.hscan(key, Long.valueOf(cursor), "count", "1000000"); + Map results = mockr.results; + List> list = results.entrySet().stream().collect(Collectors.toList()); + ScanResult> result = new ScanResult>("0", list); + + return result; + } catch (Exception e) { + throw new JedisException(e); + } + } + + public ScanResult zscan(final String key, final String cursor) { + try { + org.rarefiedredis.redis.ScanResult> sr = redis.zscan(key, Long.valueOf(cursor), "count", "1000000"); + List list = sr.results.stream().collect(Collectors.toList()); + List tl = new LinkedList(); + list.forEach(p -> tl.add(new Tuple(p.member, p.score))); + ScanResult result = new ScanResult("0", tl); + return result; + } catch (Exception e) { + throw new JedisException(e); + } + } } diff --git a/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/CustomShardingStrategyTest.java b/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/CustomShardingStrategyTest.java index 44f1cbc..23d09e9 100644 --- a/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/CustomShardingStrategyTest.java +++ b/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/CustomShardingStrategyTest.java @@ -1,12 +1,12 @@ /** * Copyright 2016 Netflix, Inc. - * + *

* Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. diff --git a/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/DefaultShardingStrategyTest.java b/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/DefaultShardingStrategyTest.java index d0f42bc..85d04ba 100644 --- a/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/DefaultShardingStrategyTest.java +++ b/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/DefaultShardingStrategyTest.java @@ -1,12 +1,12 @@ /** * Copyright 2016 Netflix, Inc. - * + *

* Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -120,14 +120,13 @@ public String getCurrentShard() { shard3Queue = new RedisQueues(dynoClient, dynoClient, redisKeyPrefix, shard3Supplier, 1_000, 1_000_000); - shard1DynoQueue = (RedisDynoQueue)shard1Queue.get(queueName); - shard2DynoQueue = (RedisDynoQueue)shard2Queue.get(queueName); - shard3DynoQueue = (RedisDynoQueue)shard3Queue.get(queueName); + shard1DynoQueue = (RedisDynoQueue) shard1Queue.get(queueName); + shard2DynoQueue = (RedisDynoQueue) shard2Queue.get(queueName); + shard3DynoQueue = (RedisDynoQueue) shard3Queue.get(queueName); } @Before - public void clearAll() - { + public void clearAll() { shard1DynoQueue.clear(); shard2DynoQueue.clear(); shard3DynoQueue.clear(); diff --git a/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/DynoShardSupplierTest.java b/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/DynoShardSupplierTest.java index d4827cd..5c4576d 100644 --- a/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/DynoShardSupplierTest.java +++ b/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/DynoShardSupplierTest.java @@ -1,12 +1,12 @@ /** * Copyright 2016 Netflix, Inc. - * + *

* Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -14,7 +14,7 @@ * limitations under the License. */ /** - * + * */ package com.netflix.dyno.queues.redis; @@ -41,26 +41,26 @@ */ public class DynoShardSupplierTest { - @Test - public void test(){ - HostSupplier hs = new HostSupplier() { - @Override - public List getHosts() { - List hosts = new LinkedList<>(); - hosts.add(new Host("host1", 8102, "us-east-1a", Status.Up)); - hosts.add(new Host("host1", 8102, "us-east-1b", Status.Up)); - hosts.add(new Host("host1", 8102, "us-east-1d", Status.Up)); - - return hosts; - } - }; - DynoShardSupplier supplier = new DynoShardSupplier(hs, "us-east-1", "a"); - String localShard = supplier.getCurrentShard(); - Set allShards = supplier.getQueueShards(); - - assertNotNull(localShard); - assertEquals("a", localShard); - assertNotNull(allShards); - assertEquals(Arrays.asList("a","b","d").stream().collect(Collectors.toSet()), allShards); - } + @Test + public void test() { + HostSupplier hs = new HostSupplier() { + @Override + public List getHosts() { + List hosts = new LinkedList<>(); + hosts.add(new Host("host1", 8102, "us-east-1a", Status.Up)); + hosts.add(new Host("host1", 8102, "us-east-1b", Status.Up)); + hosts.add(new Host("host1", 8102, "us-east-1d", Status.Up)); + + return hosts; + } + }; + DynoShardSupplier supplier = new DynoShardSupplier(hs, "us-east-1", "a"); + String localShard = supplier.getCurrentShard(); + Set allShards = supplier.getQueueShards(); + + assertNotNull(localShard); + assertEquals("a", localShard); + assertNotNull(allShards); + assertEquals(Arrays.asList("a", "b", "d").stream().collect(Collectors.toSet()), allShards); + } } diff --git a/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/RedisDynoQueueTest.java b/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/RedisDynoQueueTest.java index 5976e92..a0b70f9 100644 --- a/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/RedisDynoQueueTest.java +++ b/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/RedisDynoQueueTest.java @@ -1,12 +1,12 @@ /** * Copyright 2016 Netflix, Inc. - * + *

* Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -51,321 +51,321 @@ public class RedisDynoQueueTest { - private static JedisMock dynoClient; - - private static final String queueName = "test_queue"; - - private static final String redisKeyPrefix = "testdynoqueues"; - - private static RedisDynoQueue rdq; - - private static RedisQueues rq; - - private static String messageKey; - - @BeforeClass - public static void setUpBeforeClass() throws Exception { - - HostSupplier hs = new HostSupplier() { - @Override - public List getHosts() { - List hosts = new LinkedList<>(); - hosts.add(new Host("ec2-11-22-33-444.compute-0.amazonaws.com", 8102, "us-east-1d", Status.Up)); - return hosts; - } - }; - - dynoClient = new JedisMock(); - - Set allShards = hs.getHosts().stream().map(host -> host.getRack().substring(host.getRack().length() - 2)).collect(Collectors.toSet()); - String shardName = allShards.iterator().next(); - ShardSupplier ss = new ShardSupplier() { - - @Override - public Set getQueueShards() { - return allShards; - } - - @Override - public String getCurrentShard() { - return shardName; - } - }; - messageKey = redisKeyPrefix + ".MESSAGE." + queueName; - - rq = new RedisQueues(dynoClient, dynoClient, redisKeyPrefix, ss, 1_000, 1_000_000); - DynoQueue rdq1 = rq.get(queueName); - assertNotNull(rdq1); - - rdq = (RedisDynoQueue)rq.get(queueName); - assertNotNull(rdq); - - assertEquals(rdq1, rdq); // should be the same instance. - - } - - @Test - public void testGetName() { - assertEquals(queueName, rdq.getName()); - } - - @Test - public void testGetUnackTime() { - assertEquals(1_000, rdq.getUnackTime()); - } - - @Test - public void testTimeoutUpdate() { - - rdq.clear(); - - String id = UUID.randomUUID().toString(); - Message msg = new Message(id, "Hello World-" + id); - msg.setTimeout(100, TimeUnit.MILLISECONDS); - rdq.push(Arrays.asList(msg)); - - List popped = rdq.pop(1, 10, TimeUnit.MILLISECONDS); - assertNotNull(popped); - assertEquals(0, popped.size()); - - Uninterruptibles.sleepUninterruptibly(500, TimeUnit.MILLISECONDS); - - popped = rdq.pop(1, 1, TimeUnit.SECONDS); - assertNotNull(popped); - assertEquals(1, popped.size()); - - boolean updated = rdq.setUnackTimeout(id, 500); - assertTrue(updated); - popped = rdq.pop(1, 1, TimeUnit.SECONDS); - assertNotNull(popped); - assertEquals(0, popped.size()); - - Uninterruptibles.sleepUninterruptibly(1000, TimeUnit.MILLISECONDS); - rdq.processUnacks(); - popped = rdq.pop(1, 1, TimeUnit.SECONDS); - assertNotNull(popped); - assertEquals(1, popped.size()); - - updated = rdq.setUnackTimeout(id, 10_000); //10 seconds! - assertTrue(updated); - rdq.processUnacks(); - popped = rdq.pop(1, 1, TimeUnit.SECONDS); - assertNotNull(popped); - assertEquals(0, popped.size()); - - updated = rdq.setUnackTimeout(id, 0); - assertTrue(updated); - rdq.processUnacks(); - popped = rdq.pop(1, 1, TimeUnit.SECONDS); - assertNotNull(popped); - assertEquals(1, popped.size()); - - rdq.ack(id); - Map> size = rdq.shardSizes(); - Map values = size.get("1d"); - long total = values.values().stream().mapToLong(v -> v).sum(); - assertEquals(0, total); - - popped = rdq.pop(1, 1, TimeUnit.SECONDS); - assertNotNull(popped); - assertEquals(0, popped.size()); - } - - @Test - public void testConcurrency() throws InterruptedException, ExecutionException { - - rdq.clear(); - - final int count = 10_000; - final AtomicInteger published = new AtomicInteger(0); - - ScheduledExecutorService ses = Executors.newScheduledThreadPool(6); - CountDownLatch publishLatch = new CountDownLatch(1); - Runnable publisher = new Runnable() { - - @Override - public void run() { - List messages = new LinkedList<>(); - for (int i = 0; i < 10; i++) { - Message msg = new Message(UUID.randomUUID().toString(), "Hello World-" + i); - msg.setPriority(new Random().nextInt(98)); - messages.add(msg); - } - if(published.get() >= count) { - publishLatch.countDown(); - return; - } - - published.addAndGet(messages.size()); - rdq.push(messages); - - } - }; - - for(int p = 0; p < 3; p++) { - ses.scheduleWithFixedDelay(publisher, 1, 1, TimeUnit.MILLISECONDS); - } - publishLatch.await(); - CountDownLatch latch = new CountDownLatch(count); - List allMsgs = new CopyOnWriteArrayList<>(); - AtomicInteger consumed = new AtomicInteger(0); - AtomicInteger counter = new AtomicInteger(0); - Runnable consumer = new Runnable() { - - @Override - public void run() { - if(consumed.get() >= count) { - return; - } - List popped = rdq.pop(100, 1, TimeUnit.MILLISECONDS); - allMsgs.addAll(popped); - consumed.addAndGet(popped.size()); - popped.stream().forEach(p -> latch.countDown()); - counter.incrementAndGet(); - } - }; - - for(int c = 0; c < 2; c++) { - ses.scheduleWithFixedDelay(consumer, 1, 10, TimeUnit.MILLISECONDS); - } - Uninterruptibles.awaitUninterruptibly(latch); - System.out.println("Consumed: " + consumed.get() + ", all: " + allMsgs.size() + " counter: " + counter.get()); - Set uniqueMessages = allMsgs.stream().collect(Collectors.toSet()); - - assertEquals(count, allMsgs.size()); - assertEquals(count, uniqueMessages.size()); - long start = System.currentTimeMillis(); - List more = rdq.pop(1, 1, TimeUnit.SECONDS); - long elapsedTime = System.currentTimeMillis() - start; - assertTrue(elapsedTime >= 1000); - assertEquals(0, more.size()); - assertEquals(0, rdq.prefetch.get()); - - ses.shutdownNow(); - } - - @Test - public void testSetTimeout() { - - rdq.clear(); - - Message msg = new Message("x001", "Hello World"); - msg.setPriority(3); - msg.setTimeout(20_000); - rdq.push(Arrays.asList(msg)); - - List popped = rdq.pop(1, 1, TimeUnit.SECONDS); - assertTrue(popped.isEmpty()); - - boolean updated = rdq.setTimeout(msg.getId(), 1); - assertTrue(updated); - popped = rdq.pop(1, 1, TimeUnit.SECONDS); - assertEquals(1, popped.size()); - assertEquals(1, popped.get(0).getTimeout()); - updated = rdq.setTimeout(msg.getId(), 1); - assertTrue(!updated); - } - - @Test - public void testAll() { - - rdq.clear(); - - int count = 10; - List messages = new LinkedList<>(); - for (int i = 0; i < count; i++) { - Message msg = new Message("" + i, "Hello World-" + i); - msg.setPriority(count - i); - messages.add(msg); - } - rdq.push(messages); - - messages = rdq.peek(count); - - assertNotNull(messages); - assertEquals(count, messages.size()); - long size = rdq.size(); - assertEquals(count, size); - - // We did a peek - let's ensure the messages are still around! - List messages2 = rdq.peek(count); - assertNotNull(messages2); - assertEquals(messages, messages2); - - List poped = rdq.pop(count, 1, TimeUnit.SECONDS); - assertNotNull(poped); - assertEquals(count, poped.size()); - assertEquals(messages, poped); - - Uninterruptibles.sleepUninterruptibly(2, TimeUnit.SECONDS); - ((RedisDynoQueue)rdq).processUnacks(); - - for (Message msg : messages) { - Message found = rdq.get(msg.getId()); - assertNotNull(found); - assertEquals(msg.getId(), found.getId()); - assertEquals(msg.getTimeout(), found.getTimeout()); - } - assertNull(rdq.get("some fake id")); - - List messages3 = rdq.pop(count, 1, TimeUnit.SECONDS); - if(messages3.size() < count){ - List messages4 = rdq.pop(count, 1, TimeUnit.SECONDS); - messages3.addAll(messages4); - } - - assertNotNull(messages3); - assertEquals(10, messages3.size()); - assertEquals(messages, messages3); - assertEquals(10, messages3.stream().map(msg -> msg.getId()).collect(Collectors.toSet()).size()); - messages3.stream().forEach(System.out::println); - assertTrue(dynoClient.hlen(messageKey) == 10); - - for (Message msg : messages3) { - assertTrue(rdq.ack(msg.getId())); - assertFalse(rdq.ack(msg.getId())); - } - Uninterruptibles.sleepUninterruptibly(2, TimeUnit.SECONDS); - messages3 = rdq.pop(count, 1, TimeUnit.SECONDS); - assertNotNull(messages3); - assertEquals(0, messages3.size()); - - int max = 10; - for (Message msg : messages) { - assertEquals(max, msg.getPriority()); - rdq.remove(msg.getId()); - max--; - } - - size = rdq.size(); - assertEquals(0, size); - - assertTrue(dynoClient.hlen(messageKey) == 0); - - } - - @Before - public void clear(){ - rdq.clear(); - assertTrue(dynoClient.hlen(messageKey) == 0); - } - - @Test - public void testClearQueues() { - rdq.clear(); - int count = 10; - List messages = new LinkedList<>(); - for (int i = 0; i < count; i++) { - Message msg = new Message("x" + i, "Hello World-" + i); - msg.setPriority(count - i); - messages.add(msg); - } - - rdq.push(messages); - assertEquals(count, rdq.size()); - rdq.clear(); - assertEquals(0, rdq.size()); - - } + private static JedisMock dynoClient; + + private static final String queueName = "test_queue"; + + private static final String redisKeyPrefix = "testdynoqueues"; + + private static RedisDynoQueue rdq; + + private static RedisQueues rq; + + private static String messageKey; + + @BeforeClass + public static void setUpBeforeClass() throws Exception { + + HostSupplier hs = new HostSupplier() { + @Override + public List getHosts() { + List hosts = new LinkedList<>(); + hosts.add(new Host("ec2-11-22-33-444.compute-0.amazonaws.com", 8102, "us-east-1d", Status.Up)); + return hosts; + } + }; + + dynoClient = new JedisMock(); + + Set allShards = hs.getHosts().stream().map(host -> host.getRack().substring(host.getRack().length() - 2)).collect(Collectors.toSet()); + String shardName = allShards.iterator().next(); + ShardSupplier ss = new ShardSupplier() { + + @Override + public Set getQueueShards() { + return allShards; + } + + @Override + public String getCurrentShard() { + return shardName; + } + }; + messageKey = redisKeyPrefix + ".MESSAGE." + queueName; + + rq = new RedisQueues(dynoClient, dynoClient, redisKeyPrefix, ss, 1_000, 1_000_000); + DynoQueue rdq1 = rq.get(queueName); + assertNotNull(rdq1); + + rdq = (RedisDynoQueue) rq.get(queueName); + assertNotNull(rdq); + + assertEquals(rdq1, rdq); // should be the same instance. + + } + + @Test + public void testGetName() { + assertEquals(queueName, rdq.getName()); + } + + @Test + public void testGetUnackTime() { + assertEquals(1_000, rdq.getUnackTime()); + } + + @Test + public void testTimeoutUpdate() { + + rdq.clear(); + + String id = UUID.randomUUID().toString(); + Message msg = new Message(id, "Hello World-" + id); + msg.setTimeout(100, TimeUnit.MILLISECONDS); + rdq.push(Arrays.asList(msg)); + + List popped = rdq.pop(1, 10, TimeUnit.MILLISECONDS); + assertNotNull(popped); + assertEquals(0, popped.size()); + + Uninterruptibles.sleepUninterruptibly(500, TimeUnit.MILLISECONDS); + + popped = rdq.pop(1, 1, TimeUnit.SECONDS); + assertNotNull(popped); + assertEquals(1, popped.size()); + + boolean updated = rdq.setUnackTimeout(id, 500); + assertTrue(updated); + popped = rdq.pop(1, 1, TimeUnit.SECONDS); + assertNotNull(popped); + assertEquals(0, popped.size()); + + Uninterruptibles.sleepUninterruptibly(1000, TimeUnit.MILLISECONDS); + rdq.processUnacks(); + popped = rdq.pop(1, 1, TimeUnit.SECONDS); + assertNotNull(popped); + assertEquals(1, popped.size()); + + updated = rdq.setUnackTimeout(id, 10_000); //10 seconds! + assertTrue(updated); + rdq.processUnacks(); + popped = rdq.pop(1, 1, TimeUnit.SECONDS); + assertNotNull(popped); + assertEquals(0, popped.size()); + + updated = rdq.setUnackTimeout(id, 0); + assertTrue(updated); + rdq.processUnacks(); + popped = rdq.pop(1, 1, TimeUnit.SECONDS); + assertNotNull(popped); + assertEquals(1, popped.size()); + + rdq.ack(id); + Map> size = rdq.shardSizes(); + Map values = size.get("1d"); + long total = values.values().stream().mapToLong(v -> v).sum(); + assertEquals(0, total); + + popped = rdq.pop(1, 1, TimeUnit.SECONDS); + assertNotNull(popped); + assertEquals(0, popped.size()); + } + + @Test + public void testConcurrency() throws InterruptedException, ExecutionException { + + rdq.clear(); + + final int count = 10_000; + final AtomicInteger published = new AtomicInteger(0); + + ScheduledExecutorService ses = Executors.newScheduledThreadPool(6); + CountDownLatch publishLatch = new CountDownLatch(1); + Runnable publisher = new Runnable() { + + @Override + public void run() { + List messages = new LinkedList<>(); + for (int i = 0; i < 10; i++) { + Message msg = new Message(UUID.randomUUID().toString(), "Hello World-" + i); + msg.setPriority(new Random().nextInt(98)); + messages.add(msg); + } + if (published.get() >= count) { + publishLatch.countDown(); + return; + } + + published.addAndGet(messages.size()); + rdq.push(messages); + + } + }; + + for (int p = 0; p < 3; p++) { + ses.scheduleWithFixedDelay(publisher, 1, 1, TimeUnit.MILLISECONDS); + } + publishLatch.await(); + CountDownLatch latch = new CountDownLatch(count); + List allMsgs = new CopyOnWriteArrayList<>(); + AtomicInteger consumed = new AtomicInteger(0); + AtomicInteger counter = new AtomicInteger(0); + Runnable consumer = new Runnable() { + + @Override + public void run() { + if (consumed.get() >= count) { + return; + } + List popped = rdq.pop(100, 1, TimeUnit.MILLISECONDS); + allMsgs.addAll(popped); + consumed.addAndGet(popped.size()); + popped.stream().forEach(p -> latch.countDown()); + counter.incrementAndGet(); + } + }; + + for (int c = 0; c < 2; c++) { + ses.scheduleWithFixedDelay(consumer, 1, 10, TimeUnit.MILLISECONDS); + } + Uninterruptibles.awaitUninterruptibly(latch); + System.out.println("Consumed: " + consumed.get() + ", all: " + allMsgs.size() + " counter: " + counter.get()); + Set uniqueMessages = allMsgs.stream().collect(Collectors.toSet()); + + assertEquals(count, allMsgs.size()); + assertEquals(count, uniqueMessages.size()); + long start = System.currentTimeMillis(); + List more = rdq.pop(1, 1, TimeUnit.SECONDS); + long elapsedTime = System.currentTimeMillis() - start; + assertTrue(elapsedTime >= 1000); + assertEquals(0, more.size()); + assertEquals(0, rdq.prefetch.get()); + + ses.shutdownNow(); + } + + @Test + public void testSetTimeout() { + + rdq.clear(); + + Message msg = new Message("x001", "Hello World"); + msg.setPriority(3); + msg.setTimeout(20_000); + rdq.push(Arrays.asList(msg)); + + List popped = rdq.pop(1, 1, TimeUnit.SECONDS); + assertTrue(popped.isEmpty()); + + boolean updated = rdq.setTimeout(msg.getId(), 1); + assertTrue(updated); + popped = rdq.pop(1, 1, TimeUnit.SECONDS); + assertEquals(1, popped.size()); + assertEquals(1, popped.get(0).getTimeout()); + updated = rdq.setTimeout(msg.getId(), 1); + assertTrue(!updated); + } + + @Test + public void testAll() { + + rdq.clear(); + + int count = 10; + List messages = new LinkedList<>(); + for (int i = 0; i < count; i++) { + Message msg = new Message("" + i, "Hello World-" + i); + msg.setPriority(count - i); + messages.add(msg); + } + rdq.push(messages); + + messages = rdq.peek(count); + + assertNotNull(messages); + assertEquals(count, messages.size()); + long size = rdq.size(); + assertEquals(count, size); + + // We did a peek - let's ensure the messages are still around! + List messages2 = rdq.peek(count); + assertNotNull(messages2); + assertEquals(messages, messages2); + + List poped = rdq.pop(count, 1, TimeUnit.SECONDS); + assertNotNull(poped); + assertEquals(count, poped.size()); + assertEquals(messages, poped); + + Uninterruptibles.sleepUninterruptibly(2, TimeUnit.SECONDS); + ((RedisDynoQueue) rdq).processUnacks(); + + for (Message msg : messages) { + Message found = rdq.get(msg.getId()); + assertNotNull(found); + assertEquals(msg.getId(), found.getId()); + assertEquals(msg.getTimeout(), found.getTimeout()); + } + assertNull(rdq.get("some fake id")); + + List messages3 = rdq.pop(count, 1, TimeUnit.SECONDS); + if (messages3.size() < count) { + List messages4 = rdq.pop(count, 1, TimeUnit.SECONDS); + messages3.addAll(messages4); + } + + assertNotNull(messages3); + assertEquals(10, messages3.size()); + assertEquals(messages, messages3); + assertEquals(10, messages3.stream().map(msg -> msg.getId()).collect(Collectors.toSet()).size()); + messages3.stream().forEach(System.out::println); + assertTrue(dynoClient.hlen(messageKey) == 10); + + for (Message msg : messages3) { + assertTrue(rdq.ack(msg.getId())); + assertFalse(rdq.ack(msg.getId())); + } + Uninterruptibles.sleepUninterruptibly(2, TimeUnit.SECONDS); + messages3 = rdq.pop(count, 1, TimeUnit.SECONDS); + assertNotNull(messages3); + assertEquals(0, messages3.size()); + + int max = 10; + for (Message msg : messages) { + assertEquals(max, msg.getPriority()); + rdq.remove(msg.getId()); + max--; + } + + size = rdq.size(); + assertEquals(0, size); + + assertTrue(dynoClient.hlen(messageKey) == 0); + + } + + @Before + public void clear() { + rdq.clear(); + assertTrue(dynoClient.hlen(messageKey) == 0); + } + + @Test + public void testClearQueues() { + rdq.clear(); + int count = 10; + List messages = new LinkedList<>(); + for (int i = 0; i < count; i++) { + Message msg = new Message("x" + i, "Hello World-" + i); + msg.setPriority(count - i); + messages.add(msg); + } + + rdq.push(messages); + assertEquals(count, rdq.size()); + rdq.clear(); + assertEquals(0, rdq.size()); + + } } diff --git a/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/benchmark/BenchmarkTestsJedis.java b/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/benchmark/BenchmarkTestsJedis.java index 8b7258e..ec38b28 100644 --- a/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/benchmark/BenchmarkTestsJedis.java +++ b/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/benchmark/BenchmarkTestsJedis.java @@ -1,5 +1,5 @@ /** - * + * */ package com.netflix.dyno.queues.redis.benchmark; @@ -17,42 +17,42 @@ */ public class BenchmarkTestsJedis extends QueueBenchmark { - public BenchmarkTestsJedis() { - List hosts = new LinkedList<>(); - hosts.add(new Host("localhost", 6379, "us-east-1a")); - QueueBuilder qb = new QueueBuilder(); - - JedisPoolConfig config = new JedisPoolConfig(); - config.setTestOnBorrow(true); - config.setTestOnCreate(true); - config.setMaxTotal(10); - config.setMaxIdle(5); - config.setMaxWaitMillis(60_000); - - - queue = qb - .setCurrentShard("a") - .setHostToShardMap((Host h) -> h.getRack().substring(h.getRack().length()-1)) - .setQueueName("testq") - .setRedisKeyPrefix("keyprefix") - .setUnackTime(60_000_000) - .useNonDynomiteRedis(config, hosts) - .build(); - - System.out.println("Instance: " + queue.getClass().getName()); - } - - public static void main(String[] args) throws Exception { - try { - - BenchmarkTestsJedis tests = new BenchmarkTestsJedis(); - tests.run(); - - } catch(Exception e) { - e.printStackTrace(); - }finally { - System.exit(0); - } - } + public BenchmarkTestsJedis() { + List hosts = new LinkedList<>(); + hosts.add(new Host("localhost", 6379, "us-east-1a")); + QueueBuilder qb = new QueueBuilder(); + + JedisPoolConfig config = new JedisPoolConfig(); + config.setTestOnBorrow(true); + config.setTestOnCreate(true); + config.setMaxTotal(10); + config.setMaxIdle(5); + config.setMaxWaitMillis(60_000); + + + queue = qb + .setCurrentShard("a") + .setHostToShardMap((Host h) -> h.getRack().substring(h.getRack().length() - 1)) + .setQueueName("testq") + .setRedisKeyPrefix("keyprefix") + .setUnackTime(60_000_000) + .useNonDynomiteRedis(config, hosts) + .build(); + + System.out.println("Instance: " + queue.getClass().getName()); + } + + public static void main(String[] args) throws Exception { + try { + + BenchmarkTestsJedis tests = new BenchmarkTestsJedis(); + tests.run(); + + } catch (Exception e) { + e.printStackTrace(); + } finally { + System.exit(0); + } + } } diff --git a/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/v2/DynoJedisTests.java b/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/v2/DynoJedisTests.java index efbc5f0..5240b22 100644 --- a/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/v2/DynoJedisTests.java +++ b/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/v2/DynoJedisTests.java @@ -98,5 +98,4 @@ public HostToken getTokenForHost(Host host, Set activeHosts) { } - } diff --git a/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/v2/JedisTests.java b/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/v2/JedisTests.java index 085033a..52a5ea0 100644 --- a/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/v2/JedisTests.java +++ b/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/v2/JedisTests.java @@ -62,7 +62,7 @@ public DynoQueue getQueue(String redisKeyPrefix, String queueName) { QueueBuilder qb = new QueueBuilder(); DynoQueue queue = qb .setCurrentShard("a") - .setHostToShardMap((Host h) -> h.getRack().substring(h.getRack().length()-1)) + .setHostToShardMap((Host h) -> h.getRack().substring(h.getRack().length() - 1)) .setQueueName(queueName) .setRedisKeyPrefix(redisKeyPrefix) .setUnackTime(1_000) @@ -76,5 +76,4 @@ public DynoQueue getQueue(String redisKeyPrefix, String queueName) { } - } diff --git a/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/v2/RedisDynoQueueTest.java b/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/v2/RedisDynoQueueTest.java index f5263c0..df6bb66 100644 --- a/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/v2/RedisDynoQueueTest.java +++ b/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/v2/RedisDynoQueueTest.java @@ -1,12 +1,12 @@ /** * Copyright 2016 Netflix, Inc. - * + *

* Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -52,286 +52,286 @@ public class RedisDynoQueueTest { - private static Jedis dynoClient; - - private static final String queueName = "test_queue"; - - private static final String redisKeyPrefix = "testdynoqueues"; - - private static RedisPipelineQueue rdq; - - private static String messageKeyPrefix; - - private static int maxHashBuckets = 32; - - @BeforeClass - public static void setUpBeforeClass() throws Exception { - - - JedisPoolConfig config = new JedisPoolConfig(); - config.setTestOnBorrow(true); - config.setTestOnCreate(true); - config.setMaxTotal(10); - config.setMaxIdle(5); - config.setMaxWaitMillis(60_000); - JedisPool pool = new JedisPool(config, "localhost", 6379); - dynoClient = new Jedis("localhost", 6379, 0, 0); - dynoClient.flushAll(); - - rdq = new RedisPipelineQueue(redisKeyPrefix, queueName, "x", 1_000, 1_000, new JedisProxy(pool)); - messageKeyPrefix = redisKeyPrefix + ".MSG." + "{" + queueName + ".x}"; - } - - @Test - public void testGetName() { - assertEquals(queueName, rdq.getName()); - } - - @Test - public void testGetUnackTime() { - assertEquals(1_000, rdq.getUnackTime()); - } - - @Test - public void testTimeoutUpdate() { - - rdq.clear(); - - String id = UUID.randomUUID().toString(); - Message msg = new Message(id, "Hello World-" + id); - msg.setTimeout(100, TimeUnit.MILLISECONDS); - rdq.push(Arrays.asList(msg)); - - List popped = rdq.pop(1, 10, TimeUnit.MILLISECONDS); - assertNotNull(popped); - assertEquals(0, popped.size()); - - Uninterruptibles.sleepUninterruptibly(500, TimeUnit.MILLISECONDS); - - popped = rdq.pop(1, 1, TimeUnit.SECONDS); - assertNotNull(popped); - assertEquals(1, popped.size()); - - boolean updated = rdq.setUnackTimeout(id, 500); - assertTrue(updated); - popped = rdq.pop(1, 1, TimeUnit.SECONDS); - assertNotNull(popped); - assertEquals(0, popped.size()); - - Uninterruptibles.sleepUninterruptibly(1000, TimeUnit.MILLISECONDS); - rdq.processUnacks(); - popped = rdq.pop(1, 1, TimeUnit.SECONDS); - assertNotNull(popped); - assertEquals(1, popped.size()); - - updated = rdq.setUnackTimeout(id, 10_000); //10 seconds! - assertTrue(updated); - rdq.processUnacks(); - popped = rdq.pop(1, 1, TimeUnit.SECONDS); - assertNotNull(popped); - assertEquals(0, popped.size()); - - updated = rdq.setUnackTimeout(id, 0); - assertTrue(updated); - rdq.processUnacks(); - popped = rdq.pop(1, 1, TimeUnit.SECONDS); - assertNotNull(popped); - assertEquals(1, popped.size()); - - rdq.ack(id); - Map> size = rdq.shardSizes(); - Map values = size.get("x"); - long total = values.values().stream().mapToLong(v -> v).sum(); - assertEquals(0, total); - - popped = rdq.pop(1, 1, TimeUnit.SECONDS); - assertNotNull(popped); - assertEquals(0, popped.size()); - } - - @Test - public void testConcurrency() throws InterruptedException, ExecutionException { - - rdq.clear(); - - final int count = 100; - final AtomicInteger published = new AtomicInteger(0); - - ScheduledExecutorService ses = Executors.newScheduledThreadPool(6); - CountDownLatch publishLatch = new CountDownLatch(1); - Runnable publisher = new Runnable() { - - @Override - public void run() { - List messages = new LinkedList<>(); - for (int i = 0; i < 10; i++) { - Message msg = new Message(UUID.randomUUID().toString(), "Hello World-" + i); - msg.setPriority(new Random().nextInt(98)); - messages.add(msg); - } - if(published.get() >= count) { - publishLatch.countDown(); - return; - } - - published.addAndGet(messages.size()); - rdq.push(messages); - } - }; - - for(int p = 0; p < 3; p++) { - ses.scheduleWithFixedDelay(publisher, 1, 1, TimeUnit.MILLISECONDS); - } - publishLatch.await(); - CountDownLatch latch = new CountDownLatch(count); - List allMsgs = new CopyOnWriteArrayList<>(); - AtomicInteger consumed = new AtomicInteger(0); - AtomicInteger counter = new AtomicInteger(0); - Runnable consumer = new Runnable() { - - @Override - public void run() { - if(consumed.get() >= count) { - return; - } - List popped = rdq.pop(100, 1, TimeUnit.MILLISECONDS); - allMsgs.addAll(popped); - consumed.addAndGet(popped.size()); - popped.stream().forEach(p -> latch.countDown()); - counter.incrementAndGet(); - } - }; - for(int c = 0; c < 2; c++) { - ses.scheduleWithFixedDelay(consumer, 1, 10, TimeUnit.MILLISECONDS); - } - Uninterruptibles.awaitUninterruptibly(latch); - System.out.println("Consumed: " + consumed.get() + ", all: " + allMsgs.size() + " counter: " + counter.get()); - Set uniqueMessages = allMsgs.stream().collect(Collectors.toSet()); - - assertEquals(count, allMsgs.size()); - assertEquals(count, uniqueMessages.size()); - List more = rdq.pop(1, 1, TimeUnit.SECONDS); - assertEquals(0, more.size()); - - ses.shutdownNow(); - } - - @Test - public void testSetTimeout() { - - rdq.clear(); - - Message msg = new Message("x001yx", "Hello World"); - msg.setPriority(3); - msg.setTimeout(10_000); - rdq.push(Arrays.asList(msg)); - - List popped = rdq.pop(1, 1, TimeUnit.SECONDS); - assertTrue(popped.isEmpty()); - - boolean updated = rdq.setTimeout(msg.getId(), 0); - assertTrue(updated); - popped = rdq.pop(2, 1, TimeUnit.SECONDS); - assertEquals(1, popped.size()); - assertEquals(0, popped.get(0).getTimeout()); - } - - @Test - public void testAll() { - - rdq.clear(); - assertEquals(0, rdq.size()); - - int count = 10; - List messages = new LinkedList<>(); - for (int i = 0; i < count; i++) { - Message msg = new Message("" + i, "Hello World-" + i); - msg.setPriority(count - i); - messages.add(msg); - } - rdq.push(messages); - - messages = rdq.peek(count); - - assertNotNull(messages); - assertEquals(count, messages.size()); - long size = rdq.size(); - assertEquals(count, size); - - // We did a peek - let's ensure the messages are still around! - List messages2 = rdq.peek(count); - assertNotNull(messages2); - assertEquals(messages, messages2); - - List poped = rdq.pop(count, 1, TimeUnit.SECONDS); - assertNotNull(poped); - assertEquals(count, poped.size()); - assertEquals(messages, poped); - - Uninterruptibles.sleepUninterruptibly(2, TimeUnit.SECONDS); - rdq.processUnacks(); - - for (Message msg : messages) { - Message found = rdq.get(msg.getId()); - assertNotNull(found); - assertEquals(msg.getId(), found.getId()); - assertEquals(msg.getTimeout(), found.getTimeout()); - } - assertNull(rdq.get("some fake id")); - - List messages3 = rdq.pop(count, 1, TimeUnit.SECONDS); - if(messages3.size() < count){ - List messages4 = rdq.pop(count, 1, TimeUnit.SECONDS); - messages3.addAll(messages4); - } - - assertNotNull(messages3); - assertEquals(10, messages3.size()); - assertEquals(messages.stream().map(msg -> msg.getId()).sorted().collect(Collectors.toList()), messages3.stream().map(msg -> msg.getId()).sorted().collect(Collectors.toList())); - assertEquals(10, messages3.stream().map(msg -> msg.getId()).collect(Collectors.toSet()).size()); - messages3.stream().forEach(System.out::println); - int bucketCounts = 0; - for(int i = 0; i < maxHashBuckets; i++) { - bucketCounts += dynoClient.hlen(messageKeyPrefix + "." + i); - } - assertEquals(10, bucketCounts); - - for (Message msg : messages3) { - assertTrue(rdq.ack(msg.getId())); - assertFalse(rdq.ack(msg.getId())); - } - Uninterruptibles.sleepUninterruptibly(2, TimeUnit.SECONDS); - messages3 = rdq.pop(count, 1, TimeUnit.SECONDS); - assertNotNull(messages3); - assertEquals(0, messages3.size()); - } - - @Before - public void clear(){ - rdq.clear(); - int bucketCounts = 0; - for(int i = 0; i < maxHashBuckets; i++) { - bucketCounts += dynoClient.hlen(messageKeyPrefix + "." + i); - } - assertEquals(0, bucketCounts); - } - - @Test - public void testClearQueues() { - rdq.clear(); - int count = 10; - List messages = new LinkedList<>(); - for (int i = 0; i < count; i++) { - Message msg = new Message("x" + i, "Hello World-" + i); - msg.setPriority(count - i); - messages.add(msg); - } - - rdq.push(messages); - assertEquals(count, rdq.size()); - rdq.clear(); - assertEquals(0, rdq.size()); - - } + private static Jedis dynoClient; + + private static final String queueName = "test_queue"; + + private static final String redisKeyPrefix = "testdynoqueues"; + + private static RedisPipelineQueue rdq; + + private static String messageKeyPrefix; + + private static int maxHashBuckets = 32; + + @BeforeClass + public static void setUpBeforeClass() throws Exception { + + + JedisPoolConfig config = new JedisPoolConfig(); + config.setTestOnBorrow(true); + config.setTestOnCreate(true); + config.setMaxTotal(10); + config.setMaxIdle(5); + config.setMaxWaitMillis(60_000); + JedisPool pool = new JedisPool(config, "localhost", 6379); + dynoClient = new Jedis("localhost", 6379, 0, 0); + dynoClient.flushAll(); + + rdq = new RedisPipelineQueue(redisKeyPrefix, queueName, "x", 1_000, 1_000, new JedisProxy(pool)); + messageKeyPrefix = redisKeyPrefix + ".MSG." + "{" + queueName + ".x}"; + } + + @Test + public void testGetName() { + assertEquals(queueName, rdq.getName()); + } + + @Test + public void testGetUnackTime() { + assertEquals(1_000, rdq.getUnackTime()); + } + + @Test + public void testTimeoutUpdate() { + + rdq.clear(); + + String id = UUID.randomUUID().toString(); + Message msg = new Message(id, "Hello World-" + id); + msg.setTimeout(100, TimeUnit.MILLISECONDS); + rdq.push(Arrays.asList(msg)); + + List popped = rdq.pop(1, 10, TimeUnit.MILLISECONDS); + assertNotNull(popped); + assertEquals(0, popped.size()); + + Uninterruptibles.sleepUninterruptibly(500, TimeUnit.MILLISECONDS); + + popped = rdq.pop(1, 1, TimeUnit.SECONDS); + assertNotNull(popped); + assertEquals(1, popped.size()); + + boolean updated = rdq.setUnackTimeout(id, 500); + assertTrue(updated); + popped = rdq.pop(1, 1, TimeUnit.SECONDS); + assertNotNull(popped); + assertEquals(0, popped.size()); + + Uninterruptibles.sleepUninterruptibly(1000, TimeUnit.MILLISECONDS); + rdq.processUnacks(); + popped = rdq.pop(1, 1, TimeUnit.SECONDS); + assertNotNull(popped); + assertEquals(1, popped.size()); + + updated = rdq.setUnackTimeout(id, 10_000); //10 seconds! + assertTrue(updated); + rdq.processUnacks(); + popped = rdq.pop(1, 1, TimeUnit.SECONDS); + assertNotNull(popped); + assertEquals(0, popped.size()); + + updated = rdq.setUnackTimeout(id, 0); + assertTrue(updated); + rdq.processUnacks(); + popped = rdq.pop(1, 1, TimeUnit.SECONDS); + assertNotNull(popped); + assertEquals(1, popped.size()); + + rdq.ack(id); + Map> size = rdq.shardSizes(); + Map values = size.get("x"); + long total = values.values().stream().mapToLong(v -> v).sum(); + assertEquals(0, total); + + popped = rdq.pop(1, 1, TimeUnit.SECONDS); + assertNotNull(popped); + assertEquals(0, popped.size()); + } + + @Test + public void testConcurrency() throws InterruptedException, ExecutionException { + + rdq.clear(); + + final int count = 100; + final AtomicInteger published = new AtomicInteger(0); + + ScheduledExecutorService ses = Executors.newScheduledThreadPool(6); + CountDownLatch publishLatch = new CountDownLatch(1); + Runnable publisher = new Runnable() { + + @Override + public void run() { + List messages = new LinkedList<>(); + for (int i = 0; i < 10; i++) { + Message msg = new Message(UUID.randomUUID().toString(), "Hello World-" + i); + msg.setPriority(new Random().nextInt(98)); + messages.add(msg); + } + if (published.get() >= count) { + publishLatch.countDown(); + return; + } + + published.addAndGet(messages.size()); + rdq.push(messages); + } + }; + + for (int p = 0; p < 3; p++) { + ses.scheduleWithFixedDelay(publisher, 1, 1, TimeUnit.MILLISECONDS); + } + publishLatch.await(); + CountDownLatch latch = new CountDownLatch(count); + List allMsgs = new CopyOnWriteArrayList<>(); + AtomicInteger consumed = new AtomicInteger(0); + AtomicInteger counter = new AtomicInteger(0); + Runnable consumer = new Runnable() { + + @Override + public void run() { + if (consumed.get() >= count) { + return; + } + List popped = rdq.pop(100, 1, TimeUnit.MILLISECONDS); + allMsgs.addAll(popped); + consumed.addAndGet(popped.size()); + popped.stream().forEach(p -> latch.countDown()); + counter.incrementAndGet(); + } + }; + for (int c = 0; c < 2; c++) { + ses.scheduleWithFixedDelay(consumer, 1, 10, TimeUnit.MILLISECONDS); + } + Uninterruptibles.awaitUninterruptibly(latch); + System.out.println("Consumed: " + consumed.get() + ", all: " + allMsgs.size() + " counter: " + counter.get()); + Set uniqueMessages = allMsgs.stream().collect(Collectors.toSet()); + + assertEquals(count, allMsgs.size()); + assertEquals(count, uniqueMessages.size()); + List more = rdq.pop(1, 1, TimeUnit.SECONDS); + assertEquals(0, more.size()); + + ses.shutdownNow(); + } + + @Test + public void testSetTimeout() { + + rdq.clear(); + + Message msg = new Message("x001yx", "Hello World"); + msg.setPriority(3); + msg.setTimeout(10_000); + rdq.push(Arrays.asList(msg)); + + List popped = rdq.pop(1, 1, TimeUnit.SECONDS); + assertTrue(popped.isEmpty()); + + boolean updated = rdq.setTimeout(msg.getId(), 0); + assertTrue(updated); + popped = rdq.pop(2, 1, TimeUnit.SECONDS); + assertEquals(1, popped.size()); + assertEquals(0, popped.get(0).getTimeout()); + } + + @Test + public void testAll() { + + rdq.clear(); + assertEquals(0, rdq.size()); + + int count = 10; + List messages = new LinkedList<>(); + for (int i = 0; i < count; i++) { + Message msg = new Message("" + i, "Hello World-" + i); + msg.setPriority(count - i); + messages.add(msg); + } + rdq.push(messages); + + messages = rdq.peek(count); + + assertNotNull(messages); + assertEquals(count, messages.size()); + long size = rdq.size(); + assertEquals(count, size); + + // We did a peek - let's ensure the messages are still around! + List messages2 = rdq.peek(count); + assertNotNull(messages2); + assertEquals(messages, messages2); + + List poped = rdq.pop(count, 1, TimeUnit.SECONDS); + assertNotNull(poped); + assertEquals(count, poped.size()); + assertEquals(messages, poped); + + Uninterruptibles.sleepUninterruptibly(2, TimeUnit.SECONDS); + rdq.processUnacks(); + + for (Message msg : messages) { + Message found = rdq.get(msg.getId()); + assertNotNull(found); + assertEquals(msg.getId(), found.getId()); + assertEquals(msg.getTimeout(), found.getTimeout()); + } + assertNull(rdq.get("some fake id")); + + List messages3 = rdq.pop(count, 1, TimeUnit.SECONDS); + if (messages3.size() < count) { + List messages4 = rdq.pop(count, 1, TimeUnit.SECONDS); + messages3.addAll(messages4); + } + + assertNotNull(messages3); + assertEquals(10, messages3.size()); + assertEquals(messages.stream().map(msg -> msg.getId()).sorted().collect(Collectors.toList()), messages3.stream().map(msg -> msg.getId()).sorted().collect(Collectors.toList())); + assertEquals(10, messages3.stream().map(msg -> msg.getId()).collect(Collectors.toSet()).size()); + messages3.stream().forEach(System.out::println); + int bucketCounts = 0; + for (int i = 0; i < maxHashBuckets; i++) { + bucketCounts += dynoClient.hlen(messageKeyPrefix + "." + i); + } + assertEquals(10, bucketCounts); + + for (Message msg : messages3) { + assertTrue(rdq.ack(msg.getId())); + assertFalse(rdq.ack(msg.getId())); + } + Uninterruptibles.sleepUninterruptibly(2, TimeUnit.SECONDS); + messages3 = rdq.pop(count, 1, TimeUnit.SECONDS); + assertNotNull(messages3); + assertEquals(0, messages3.size()); + } + + @Before + public void clear() { + rdq.clear(); + int bucketCounts = 0; + for (int i = 0; i < maxHashBuckets; i++) { + bucketCounts += dynoClient.hlen(messageKeyPrefix + "." + i); + } + assertEquals(0, bucketCounts); + } + + @Test + public void testClearQueues() { + rdq.clear(); + int count = 10; + List messages = new LinkedList<>(); + for (int i = 0; i < count; i++) { + Message msg = new Message("x" + i, "Hello World-" + i); + msg.setPriority(count - i); + messages.add(msg); + } + + rdq.push(messages); + assertEquals(count, rdq.size()); + rdq.clear(); + assertEquals(0, rdq.size()); + + } } From 270ead193c0652e1a855fa61614ef901cbbc8988 Mon Sep 17 00:00:00 2001 From: Raghuram Onti Srinivasan Date: Tue, 4 Jun 2019 16:52:34 -0700 Subject: [PATCH 38/91] Bumping up the dyno dependency --- dyno-queues-redis/build.gradle | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/dyno-queues-redis/build.gradle b/dyno-queues-redis/build.gradle index 1f4d40b..24b7744 100644 --- a/dyno-queues-redis/build.gradle +++ b/dyno-queues-redis/build.gradle @@ -4,9 +4,9 @@ dependencies { compile "com.google.inject:guice:3.0" - compile 'com.netflix.dyno:dyno-core:1.6.5' - compile 'com.netflix.dyno:dyno-jedis:1.6.5' - compile 'com.netflix.dyno:dyno-demo:1.6.5' + compile 'com.netflix.dyno:dyno-core:1.6.6' + compile 'com.netflix.dyno:dyno-jedis:1.6.6' + compile 'com.netflix.dyno:dyno-demo:1.6.6' compile 'com.netflix.archaius:archaius-core:0.7.5' compile 'com.netflix.servo:servo-core:0.12.17' From b5443e3ed419f8bea01582a638acbef78062a396 Mon Sep 17 00:00:00 2001 From: Raghuram Onti Srinivasan Date: Tue, 4 Jun 2019 17:14:23 -0700 Subject: [PATCH 39/91] Fix for jedis version change --- .../main/java/com/netflix/dyno/queues/redis/RedisQueues.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/RedisQueues.java b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/RedisQueues.java index c0d5d8f..3cc2b93 100644 --- a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/RedisQueues.java +++ b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/RedisQueues.java @@ -19,7 +19,7 @@ import com.netflix.dyno.queues.ShardSupplier; import com.netflix.dyno.queues.redis.sharding.RoundRobinStrategy; import com.netflix.dyno.queues.redis.sharding.ShardingStrategy; -import redis.clients.jedis.JedisCommands; +import redis.clients.jedis.commands.JedisCommands; import java.io.Closeable; import java.io.IOException; From 45c5b13cdb4b6112fbae72338618d36907632710 Mon Sep 17 00:00:00 2001 From: Raghuram Onti Srinivasan Date: Tue, 4 Jun 2019 17:31:09 -0700 Subject: [PATCH 40/91] Fixing issues with moving to the new jedis --- .../dyno/queues/redis/RedisDynoQueue.java | 5 ++-- .../dyno/queues/redis/conn/DynoJedisPipe.java | 2 +- .../netflix/dyno/queues/redis/conn/Pipe.java | 3 +-- .../dyno/queues/redis/conn/RedisPipe.java | 2 +- .../queues/redis/v2/RedisPipelineQueue.java | 3 +-- .../netflix/dyno/queues/jedis/JedisMock.java | 25 ++++++++----------- 6 files changed, 17 insertions(+), 23 deletions(-) diff --git a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/RedisDynoQueue.java b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/RedisDynoQueue.java index 2b92b9c..df395e5 100644 --- a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/RedisDynoQueue.java +++ b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/RedisDynoQueue.java @@ -26,10 +26,9 @@ import com.netflix.servo.monitor.Stopwatch; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import redis.clients.jedis.JedisCommands; -import redis.clients.jedis.Jedis; import redis.clients.jedis.Tuple; -import redis.clients.jedis.params.sortedset.ZAddParams; +import redis.clients.jedis.commands.JedisCommands; +import redis.clients.jedis.params.ZAddParams; import java.io.IOException; import java.time.Clock; diff --git a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/conn/DynoJedisPipe.java b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/conn/DynoJedisPipe.java index b32502d..2b279b5 100644 --- a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/conn/DynoJedisPipe.java +++ b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/conn/DynoJedisPipe.java @@ -21,7 +21,7 @@ import com.netflix.dyno.jedis.DynoJedisPipeline; import redis.clients.jedis.Response; -import redis.clients.jedis.params.sortedset.ZAddParams; +import redis.clients.jedis.params.ZAddParams; /** * @author Viren diff --git a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/conn/Pipe.java b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/conn/Pipe.java index 3c3ae86..ef8624c 100644 --- a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/conn/Pipe.java +++ b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/conn/Pipe.java @@ -16,10 +16,9 @@ package com.netflix.dyno.queues.redis.conn; import com.netflix.dyno.jedis.DynoJedisPipeline; - import redis.clients.jedis.Pipeline; import redis.clients.jedis.Response; -import redis.clients.jedis.params.sortedset.ZAddParams; +import redis.clients.jedis.params.ZAddParams; /** * diff --git a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/conn/RedisPipe.java b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/conn/RedisPipe.java index e8bc502..3210453 100644 --- a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/conn/RedisPipe.java +++ b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/conn/RedisPipe.java @@ -20,7 +20,7 @@ import redis.clients.jedis.Pipeline; import redis.clients.jedis.Response; -import redis.clients.jedis.params.sortedset.ZAddParams; +import redis.clients.jedis.params.ZAddParams; /** * @author Viren diff --git a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/v2/RedisPipelineQueue.java b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/v2/RedisPipelineQueue.java index 74518ee..65f5249 100644 --- a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/v2/RedisPipelineQueue.java +++ b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/v2/RedisPipelineQueue.java @@ -29,10 +29,9 @@ import org.slf4j.LoggerFactory; import redis.clients.jedis.Response; import redis.clients.jedis.Tuple; -import redis.clients.jedis.params.sortedset.ZAddParams; +import redis.clients.jedis.params.ZAddParams; import java.io.IOException; -import java.lang.UnsupportedOperationException; import java.time.Clock; import java.util.ArrayList; import java.util.Collections; diff --git a/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/jedis/JedisMock.java b/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/jedis/JedisMock.java index 9acbdbc..89a391a 100644 --- a/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/jedis/JedisMock.java +++ b/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/jedis/JedisMock.java @@ -18,25 +18,24 @@ */ package com.netflix.dyno.queues.jedis; -import java.util.ArrayList; -import java.util.HashSet; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import java.util.Map.Entry; -import java.util.Set; -import java.util.stream.Collectors; - import org.rarefiedredis.redis.IRedisClient; import org.rarefiedredis.redis.IRedisSortedSet.ZsetPair; import org.rarefiedredis.redis.RedisMock; - import redis.clients.jedis.Jedis; import redis.clients.jedis.ScanParams; import redis.clients.jedis.ScanResult; import redis.clients.jedis.Tuple; import redis.clients.jedis.exceptions.JedisException; -import redis.clients.jedis.params.sortedset.ZAddParams; +import redis.clients.jedis.params.ZAddParams; + +import java.util.ArrayList; +import java.util.HashSet; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Set; +import java.util.stream.Collectors; /** * @author Viren @@ -68,7 +67,6 @@ public String set(final String key, String value) { } } - @Override public String set(final String key, final String value, final String nxxx, final String expx, final long time) { try { return redis.set(key, value, nxxx, expx, String.valueOf(time)); @@ -733,8 +731,7 @@ public Long zadd(final String key, final double score, final String member) { public Long zadd(String key, double score, String member, ZAddParams params) { try { - - if (params.contains("xx")) { + if (params.getParam("xx") != null) { Double existing = redis.zscore(key, member); if (existing == null) { return 0L; From 39d99dbe4311932f78561e140545eaba27074bcb Mon Sep 17 00:00:00 2001 From: Raghuram Onti Srinivasan Date: Wed, 12 Jun 2019 13:18:39 -0700 Subject: [PATCH 41/91] Changing api for queuebuilder to be simpler --- dyno-queues-core/build.gradle | 1 + .../netflix/dyno/queues/ShardSupplier.java | 8 ++ .../dyno/queues/demo/DynoQueueDemo.java | 10 +- .../dyno/queues/redis/v2/QueueBuilder.java | 103 +++++++++++------- .../dyno/queues/shard/DynoShardSupplier.java | 13 ++- .../queues/shard/SingleShardSupplier.java | 10 +- .../src/main/resources/demo.properties | 2 +- .../dyno/queues/redis/BaseQueueTests.java | 53 +++++---- .../redis/CustomShardingStrategyTest.java | 15 +++ .../redis/DefaultShardingStrategyTest.java | 15 +++ .../dyno/queues/redis/RedisDynoQueueTest.java | 5 + .../benchmark/BenchmarkTestsDynoJedis.java | 4 +- .../redis/benchmark/BenchmarkTestsJedis.java | 8 +- .../benchmark/BenchmarkTestsNoPipelines.java | 13 ++- .../dyno/queues/redis/v2/DynoJedisTests.java | 10 +- .../dyno/queues/redis/v2/JedisTests.java | 6 +- .../dyno/queues/redis/v2/MultiQueueTests.java | 2 - 17 files changed, 182 insertions(+), 96 deletions(-) diff --git a/dyno-queues-core/build.gradle b/dyno-queues-core/build.gradle index 5820ad6..edf46a5 100644 --- a/dyno-queues-core/build.gradle +++ b/dyno-queues-core/build.gradle @@ -1,4 +1,5 @@ dependencies { + compile 'com.netflix.dyno:dyno-core:1.6.6' testCompile "junit:junit:4.11" } \ No newline at end of file diff --git a/dyno-queues-core/src/main/java/com/netflix/dyno/queues/ShardSupplier.java b/dyno-queues-core/src/main/java/com/netflix/dyno/queues/ShardSupplier.java index a29ed58..e8bf518 100644 --- a/dyno-queues-core/src/main/java/com/netflix/dyno/queues/ShardSupplier.java +++ b/dyno-queues-core/src/main/java/com/netflix/dyno/queues/ShardSupplier.java @@ -18,6 +18,8 @@ */ package com.netflix.dyno.queues; +import com.netflix.dyno.connectionpool.Host; + import java.util.Set; @@ -39,4 +41,10 @@ public interface ShardSupplier { */ public String getCurrentShard(); + /** + * + * @param host + * @return shard for this host based on the rack + */ + public String getShardForHost(Host host); } diff --git a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/demo/DynoQueueDemo.java b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/demo/DynoQueueDemo.java index 83779f2..973d704 100644 --- a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/demo/DynoQueueDemo.java +++ b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/demo/DynoQueueDemo.java @@ -106,20 +106,12 @@ private void runSimpleV1Demo(DynoJedisClient dyno) throws IOException { } private void runSimpleV2QueueDemo(DynoJedisClient dyno) throws IOException { - String region = System.getProperty("LOCAL_DATACENTER"); - String localRack = System.getProperty("LOCAL_RACK"); - String prefix = "dynoQueue_"; - - DynoShardSupplier ss = new DynoShardSupplier(dyno.getConnPool().getConfiguration().getHostSupplier(), region, localRack); - DynoQueue queue = new QueueBuilder() .setQueueName("test") - .setCurrentShard(ss.getCurrentShard()) - .setHostToShardMap(h -> h.getRack().replaceAll(region, "")) .setRedisKeyPrefix(prefix) - .useDynomite(dyno, dyno, dyno.getConnPool().getConfiguration().getHostSupplier()) + .useDynomite(dyno, dyno) .setUnackTime(50_000) .build(); diff --git a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/v2/QueueBuilder.java b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/v2/QueueBuilder.java index aed659c..6813178 100644 --- a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/v2/QueueBuilder.java +++ b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/v2/QueueBuilder.java @@ -18,7 +18,6 @@ */ package com.netflix.dyno.queues.redis.v2; -import com.google.common.base.Function; import com.google.common.collect.Collections2; import com.google.common.collect.Lists; import com.netflix.appinfo.AmazonInfo; @@ -29,11 +28,14 @@ import com.netflix.discovery.shared.Application; import com.netflix.dyno.connectionpool.Host; import com.netflix.dyno.connectionpool.HostSupplier; +import com.netflix.dyno.connectionpool.impl.utils.ConfigUtils; import com.netflix.dyno.jedis.DynoJedisClient; import com.netflix.dyno.queues.DynoQueue; +import com.netflix.dyno.queues.ShardSupplier; import com.netflix.dyno.queues.redis.conn.DynoClientProxy; import com.netflix.dyno.queues.redis.conn.JedisProxy; import com.netflix.dyno.queues.redis.conn.RedisConnection; +import com.netflix.dyno.queues.shard.DynoShardSupplier; import redis.clients.jedis.JedisPool; import redis.clients.jedis.JedisPoolConfig; @@ -61,10 +63,14 @@ public class QueueBuilder { private String currentShard; - private Function hostToShardMap; + private ShardSupplier shardSupplier; private HostSupplier hs; + private EurekaClient eurekaClient; + + private String applicationName; + private Collection hosts; private JedisPoolConfig redisPoolConfig; @@ -82,6 +88,16 @@ public QueueBuilder setClock(Clock clock) { return this; } + public QueueBuilder setApplicationName(String appName) { + this.applicationName = appName; + return this; + } + + public QueueBuilder setEurekaClient(EurekaClient eurekaClient) { + this.eurekaClient = eurekaClient; + return this; + } + /** * @param queueName the queueName to set * @return instance of QueueBuilder @@ -114,22 +130,12 @@ public QueueBuilder useNonDynomiteRedis(JedisPoolConfig redisPoolConfig, List hostToShardMap) { - this.hostToShardMap = hostToShardMap; + this.hs = dynoQuorumClient.getConnPool().getConfiguration().getHostSupplier(); return this; } @@ -151,6 +157,15 @@ public QueueBuilder setCurrentShard(String currentShard) { return this; } + /** + * @param shardSupplier + * @return + */ + public QueueBuilder setShardSupplier(ShardSupplier shardSupplier) { + this.shardSupplier = shardSupplier; + return this; + } + /** * * @return Build an instance of the queue with supplied parameters. @@ -159,18 +174,29 @@ public QueueBuilder setCurrentShard(String currentShard) { */ public DynoQueue build() { - if (clock == null) { - clock = Clock.systemDefaultZone(); - } - - boolean useDynomiteCluster = (dynoQuorumClient != null && hs != null); + boolean useDynomiteCluster = dynoQuorumClient != null; if (useDynomiteCluster) { + if(hs == null) { + hs = dynoQuorumClient.getConnPool().getConfiguration().getHostSupplier(); + } this.hosts = hs.getHosts(); } + if (shardSupplier == null) { + String region = ConfigUtils.getDataCenter(); + String az = ConfigUtils.getLocalZone(); + shardSupplier = new DynoShardSupplier(hs, region, az); + } + if(currentShard == null) + currentShard = shardSupplier.getCurrentShard(); + + if (clock == null) { + clock = Clock.systemDefaultZone(); + } + Map shardMap = new HashMap<>(); for (Host host : hosts) { - String shard = hostToShardMap.apply(host); + String shard = shardSupplier.getShardForHost(host); shardMap.put(shard, host); } @@ -189,6 +215,9 @@ public DynoQueue build() { if (useDynomiteCluster) { redisConn = new DynoClientProxy(dynoQuorumClient); + if(dynoNonQuorumClient == null) { + dynoNonQuorumClient = dynoQuorumClient; + } redisConnRead = new DynoClientProxy(dynoNonQuorumClient); } else { JedisPool pool = new JedisPool(redisPoolConfig, hostAddress, host.getPort(), 0); @@ -212,26 +241,24 @@ public DynoQueue build() { } - private static List getHostsFromEureka(EurekaClient ec, String applicationName) { - - Application app = ec.getApplication(applicationName); - List hosts = new ArrayList(); + private HostSupplier getHostSupplierFromEureka(String applicationName) { + return () -> { + Application app = eurekaClient.getApplication(applicationName); + List hosts = new ArrayList<>(); - if (app == null) { - return hosts; - } + if (app == null) { + return hosts; + } - List ins = app.getInstances(); + List ins = app.getInstances(); - if (ins == null || ins.isEmpty()) { - return hosts; - } + if (ins == null || ins.isEmpty()) { + return hosts; + } - hosts = Lists.newArrayList(Collections2.transform(ins, + hosts = Lists.newArrayList(Collections2.transform(ins, - new Function() { - @Override - public Host apply(InstanceInfo info) { + info -> { Host.Status status = info.getStatus() == InstanceStatus.UP ? Host.Status.Up : Host.Status.Down; String rack = null; @@ -241,8 +268,8 @@ public Host apply(InstanceInfo info) { } Host host = new Host(info.getHostName(), info.getIPAddr(), rack, status); return host; - } - })); - return hosts; + })); + return hosts; + }; } } diff --git a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/shard/DynoShardSupplier.java b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/shard/DynoShardSupplier.java index b849bac..6716257 100644 --- a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/shard/DynoShardSupplier.java +++ b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/shard/DynoShardSupplier.java @@ -18,10 +18,12 @@ */ package com.netflix.dyno.queues.shard; +import com.netflix.dyno.connectionpool.Host; import com.netflix.dyno.connectionpool.HostSupplier; import com.netflix.dyno.queues.ShardSupplier; import java.util.Set; +import java.util.function.Function; import java.util.stream.Collectors; /** @@ -36,6 +38,8 @@ public class DynoShardSupplier implements ShardSupplier { private String localRack; + private Function rackToShardMap = rack -> rack.substring(rack.length()-1); + /** * Dynomite based shard supplier. Keeps the number of shards in parity with the hosts and regions * @param hs Host supplier @@ -50,13 +54,16 @@ public DynoShardSupplier(HostSupplier hs, String region, String localRack) { @Override public String getCurrentShard() { - return localRack.replaceAll(region, ""); + return rackToShardMap.apply(localRack); } @Override public Set getQueueShards() { - return hs.getHosts().stream().map(host -> host.getRack()).map(rack -> rack.replaceAll(region, "")).collect(Collectors.toSet()); + return hs.getHosts().stream().map(host -> host.getRack()).map(rackToShardMap).collect(Collectors.toSet()); } - + @Override + public String getShardForHost(Host host) { + return rackToShardMap.apply(host.getRack()); + } } diff --git a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/shard/SingleShardSupplier.java b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/shard/SingleShardSupplier.java index 9b1884d..fc3f1f4 100644 --- a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/shard/SingleShardSupplier.java +++ b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/shard/SingleShardSupplier.java @@ -18,11 +18,12 @@ */ package com.netflix.dyno.queues.shard; -import java.util.Set; - import com.google.common.collect.Sets; +import com.netflix.dyno.connectionpool.Host; import com.netflix.dyno.queues.ShardSupplier; +import java.util.Set; + /** * @author Viren * @@ -40,6 +41,11 @@ public String getCurrentShard() { return shardName; } + @Override + public String getShardForHost(Host host) { + return shardName; + } + @Override public Set getQueueShards() { return Sets.newHashSet(shardName); diff --git a/dyno-queues-redis/src/main/resources/demo.properties b/dyno-queues-redis/src/main/resources/demo.properties index ae861b7..05408a3 100644 --- a/dyno-queues-redis/src/main/resources/demo.properties +++ b/dyno-queues-redis/src/main/resources/demo.properties @@ -4,7 +4,7 @@ LOCAL_DATACENTER=us-east-1 LOCAL_RACK=us-east-1c NETFLIX_STACK=dyno_demo -#EC2_AVAILABILITY_ZONE=us-east-1 +EC2_AVAILABILITY_ZONE=us-east-1c dyno.demo.lbStrategy=TokenAware dyno.demo.retryPolicy=RetryNTimes:2 dyno.demo.port=8102 diff --git a/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/BaseQueueTests.java b/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/BaseQueueTests.java index d440605..a021455 100644 --- a/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/BaseQueueTests.java +++ b/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/BaseQueueTests.java @@ -18,20 +18,30 @@ import com.google.common.util.concurrent.Uninterruptibles; import com.netflix.dyno.queues.DynoQueue; import com.netflix.dyno.queues.Message; -import com.netflix.dyno.queues.redis.conn.JedisProxy; import org.junit.Before; -import org.junit.BeforeClass; import org.junit.Test; -import redis.clients.jedis.Jedis; -import redis.clients.jedis.JedisPool; -import redis.clients.jedis.JedisPoolConfig; -import java.util.*; -import java.util.concurrent.*; +import java.util.Arrays; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Random; +import java.util.Set; +import java.util.UUID; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Collectors; -import static org.junit.Assert.*; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; public abstract class BaseQueueTests { @@ -162,19 +172,15 @@ public void run() { List allMsgs = new CopyOnWriteArrayList<>(); AtomicInteger consumed = new AtomicInteger(0); AtomicInteger counter = new AtomicInteger(0); - Runnable consumer = new Runnable() { - - @Override - public void run() { - if (consumed.get() >= count) { - return; - } - List popped = rdq.pop(100, 1, TimeUnit.MILLISECONDS); - allMsgs.addAll(popped); - consumed.addAndGet(popped.size()); - popped.stream().forEach(p -> latch.countDown()); - counter.incrementAndGet(); + Runnable consumer = () -> { + if (consumed.get() >= count) { + return; } + List popped = rdq.pop(100, 1, TimeUnit.MILLISECONDS); + allMsgs.addAll(popped); + consumed.addAndGet(popped.size()); + popped.stream().forEach(p -> latch.countDown()); + counter.incrementAndGet(); }; for (int c = 0; c < 2; c++) { ses.scheduleWithFixedDelay(consumer, 1, 10, TimeUnit.MILLISECONDS); @@ -186,7 +192,12 @@ public void run() { assertEquals(count, allMsgs.size()); assertEquals(count, uniqueMessages.size()); List more = rdq.pop(1, 1, TimeUnit.SECONDS); - assertEquals(0, more.size()); + // If we published more than we consumed since we could've published more than we consumed in which case this + // will not be empty + if(published.get() == consumed.get()) + assertEquals(0, more.size()); + else + assertEquals(1, more.size()); ses.shutdownNow(); } diff --git a/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/CustomShardingStrategyTest.java b/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/CustomShardingStrategyTest.java index 23d09e9..34ba1f9 100644 --- a/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/CustomShardingStrategyTest.java +++ b/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/CustomShardingStrategyTest.java @@ -95,6 +95,11 @@ public Set getQueueShards() { public String getCurrentShard() { return shard1Name; } + + @Override + public String getShardForHost(Host host) { + return null; + } }; ShardSupplier shard2Supplier = new ShardSupplier() { @@ -108,6 +113,11 @@ public Set getQueueShards() { public String getCurrentShard() { return shard2Name; } + + @Override + public String getShardForHost(Host host) { + return null; + } }; @@ -122,6 +132,11 @@ public Set getQueueShards() { public String getCurrentShard() { return shard3Name; } + + @Override + public String getShardForHost(Host host) { + return null; + } }; messageKey = redisKeyPrefix + ".MESSAGE." + queueName; diff --git a/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/DefaultShardingStrategyTest.java b/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/DefaultShardingStrategyTest.java index 85d04ba..2157f01 100644 --- a/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/DefaultShardingStrategyTest.java +++ b/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/DefaultShardingStrategyTest.java @@ -84,6 +84,11 @@ public Set getQueueShards() { public String getCurrentShard() { return shard1Name; } + + @Override + public String getShardForHost(Host host) { + return null; + } }; ShardSupplier shard2Supplier = new ShardSupplier() { @@ -97,6 +102,11 @@ public Set getQueueShards() { public String getCurrentShard() { return shard2Name; } + + @Override + public String getShardForHost(Host host) { + return null; + } }; @@ -111,6 +121,11 @@ public Set getQueueShards() { public String getCurrentShard() { return shard3Name; } + + @Override + public String getShardForHost(Host host) { + return null; + } }; messageKey = redisKeyPrefix + ".MESSAGE." + queueName; diff --git a/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/RedisDynoQueueTest.java b/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/RedisDynoQueueTest.java index a0b70f9..8a4de0f 100644 --- a/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/RedisDynoQueueTest.java +++ b/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/RedisDynoQueueTest.java @@ -90,6 +90,11 @@ public Set getQueueShards() { public String getCurrentShard() { return shardName; } + + @Override + public String getShardForHost(Host host) { + return null; + } }; messageKey = redisKeyPrefix + ".MESSAGE." + queueName; diff --git a/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/benchmark/BenchmarkTestsDynoJedis.java b/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/benchmark/BenchmarkTestsDynoJedis.java index c532bfe..3fbf0e4 100644 --- a/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/benchmark/BenchmarkTestsDynoJedis.java +++ b/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/benchmark/BenchmarkTestsDynoJedis.java @@ -64,11 +64,11 @@ public HostToken getTokenForHost(Host host, Set activeHosts) { queue = qb .setCurrentShard("a") - .setHostToShardMap((Host h) -> h.getRack().substring(h.getRack().length() - 1)) +// .setHostToShardMap((Host h) -> h.getRack().substring(h.getRack().length() - 1)) .setQueueName("testq") .setRedisKeyPrefix("keyprefix") .setUnackTime(60_000) - .useDynomite(client, client, hs) + .useDynomite(client, client) .build(); } diff --git a/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/benchmark/BenchmarkTestsJedis.java b/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/benchmark/BenchmarkTestsJedis.java index ec38b28..020db32 100644 --- a/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/benchmark/BenchmarkTestsJedis.java +++ b/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/benchmark/BenchmarkTestsJedis.java @@ -3,14 +3,13 @@ */ package com.netflix.dyno.queues.redis.benchmark; -import java.util.LinkedList; -import java.util.List; - import com.netflix.dyno.connectionpool.Host; - import com.netflix.dyno.queues.redis.v2.QueueBuilder; import redis.clients.jedis.JedisPoolConfig; +import java.util.LinkedList; +import java.util.List; + /** * @author Viren * @@ -32,7 +31,6 @@ public BenchmarkTestsJedis() { queue = qb .setCurrentShard("a") - .setHostToShardMap((Host h) -> h.getRack().substring(h.getRack().length() - 1)) .setQueueName("testq") .setRedisKeyPrefix("keyprefix") .setUnackTime(60_000_000) diff --git a/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/benchmark/BenchmarkTestsNoPipelines.java b/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/benchmark/BenchmarkTestsNoPipelines.java index c3147b7..5c275e2 100644 --- a/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/benchmark/BenchmarkTestsNoPipelines.java +++ b/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/benchmark/BenchmarkTestsNoPipelines.java @@ -12,12 +12,12 @@ import com.netflix.dyno.queues.ShardSupplier; import com.netflix.dyno.queues.redis.RedisQueues; -import java.util.*; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Set; import java.util.stream.Collectors; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; - /** * @author Viren */ @@ -80,6 +80,11 @@ public Set getQueueShards() { public String getCurrentShard() { return shardName; } + + @Override + public String getShardForHost(Host host) { + return null; + } }; RedisQueues rq = new RedisQueues(client, client, redisKeyPrefix, ss, 60_000, 1_000_000); diff --git a/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/v2/DynoJedisTests.java b/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/v2/DynoJedisTests.java index 5240b22..31905ed 100644 --- a/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/v2/DynoJedisTests.java +++ b/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/v2/DynoJedisTests.java @@ -23,11 +23,12 @@ import com.netflix.dyno.jedis.DynoJedisClient; import com.netflix.dyno.queues.DynoQueue; import com.netflix.dyno.queues.redis.BaseQueueTests; -import com.netflix.dyno.queues.redis.v2.QueueBuilder; -import com.netflix.dyno.queues.redis.v2.RedisPipelineQueue; import redis.clients.jedis.Jedis; -import java.util.*; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Set; public class DynoJedisTests extends BaseQueueTests { @@ -89,11 +90,10 @@ public HostToken getTokenForHost(Host host, Set activeHosts) { return qb .setCurrentShard("a") - .setHostToShardMap((Host h) -> h.getRack().substring(h.getRack().length() - 1)) .setQueueName(queueName) .setRedisKeyPrefix(redisKeyPrefix) .setUnackTime(1_000) - .useDynomite(client, client, hs) + .useDynomite(client, client) .build(); } diff --git a/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/v2/JedisTests.java b/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/v2/JedisTests.java index 52a5ea0..77477f6 100644 --- a/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/v2/JedisTests.java +++ b/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/v2/JedisTests.java @@ -18,13 +18,12 @@ import com.netflix.dyno.connectionpool.Host; import com.netflix.dyno.queues.DynoQueue; import com.netflix.dyno.queues.redis.BaseQueueTests; -import com.netflix.dyno.queues.redis.v2.QueueBuilder; -import com.netflix.dyno.queues.redis.v2.RedisPipelineQueue; import redis.clients.jedis.Jedis; import redis.clients.jedis.JedisPool; import redis.clients.jedis.JedisPoolConfig; -import java.util.*; +import java.util.LinkedList; +import java.util.List; /** * @@ -62,7 +61,6 @@ public DynoQueue getQueue(String redisKeyPrefix, String queueName) { QueueBuilder qb = new QueueBuilder(); DynoQueue queue = qb .setCurrentShard("a") - .setHostToShardMap((Host h) -> h.getRack().substring(h.getRack().length() - 1)) .setQueueName(queueName) .setRedisKeyPrefix(redisKeyPrefix) .setUnackTime(1_000) diff --git a/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/v2/MultiQueueTests.java b/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/v2/MultiQueueTests.java index 58653fa..243f3a1 100644 --- a/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/v2/MultiQueueTests.java +++ b/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/v2/MultiQueueTests.java @@ -23,7 +23,6 @@ import redis.clients.jedis.JedisPool; import redis.clients.jedis.JedisPoolConfig; -import java.util.Arrays; import java.util.LinkedList; import java.util.List; import java.util.Map; @@ -65,7 +64,6 @@ public DynoQueue getQueue(String redisKeyPrefix, String queueName) { QueueBuilder qb = new QueueBuilder(); DynoQueue queue = qb .setCurrentShard("a") - .setHostToShardMap((Host h) -> h.getRack().substring(h.getRack().length() - 1)) .setQueueName(queueName) .setRedisKeyPrefix(redisKeyPrefix) .setUnackTime(50_000) From 02b16e693b886d13c7bfff8a2a0f44d88e03b677 Mon Sep 17 00:00:00 2001 From: Raghuram Onti Srinivasan Date: Thu, 20 Jun 2019 21:20:28 -0700 Subject: [PATCH 42/91] Fix from comments --- .../java/com/netflix/dyno/queues/redis/v2/QueueBuilder.java | 3 ++- .../dyno/queues/redis/benchmark/BenchmarkTestsDynoJedis.java | 1 - 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/v2/QueueBuilder.java b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/v2/QueueBuilder.java index 6813178..b9047c5 100644 --- a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/v2/QueueBuilder.java +++ b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/v2/QueueBuilder.java @@ -187,8 +187,9 @@ public DynoQueue build() { String az = ConfigUtils.getLocalZone(); shardSupplier = new DynoShardSupplier(hs, region, az); } - if(currentShard == null) + if(currentShard == null) { currentShard = shardSupplier.getCurrentShard(); + } if (clock == null) { clock = Clock.systemDefaultZone(); diff --git a/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/benchmark/BenchmarkTestsDynoJedis.java b/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/benchmark/BenchmarkTestsDynoJedis.java index 3fbf0e4..05b657f 100644 --- a/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/benchmark/BenchmarkTestsDynoJedis.java +++ b/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/benchmark/BenchmarkTestsDynoJedis.java @@ -64,7 +64,6 @@ public HostToken getTokenForHost(Host host, Set activeHosts) { queue = qb .setCurrentShard("a") -// .setHostToShardMap((Host h) -> h.getRack().substring(h.getRack().length() - 1)) .setQueueName("testq") .setRedisKeyPrefix("keyprefix") .setUnackTime(60_000) From 0133961016d7dafbeaeb88af009369b767d10dc2 Mon Sep 17 00:00:00 2001 From: Raghuram Onti Srinivasan Date: Thu, 20 Jun 2019 21:36:52 -0700 Subject: [PATCH 43/91] Pin to dyno 1.6.7 --- dyno-queues-core/build.gradle | 2 +- dyno-queues-redis/build.gradle | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/dyno-queues-core/build.gradle b/dyno-queues-core/build.gradle index edf46a5..494c968 100644 --- a/dyno-queues-core/build.gradle +++ b/dyno-queues-core/build.gradle @@ -1,5 +1,5 @@ dependencies { - compile 'com.netflix.dyno:dyno-core:1.6.6' + compile 'com.netflix.dyno:dyno-core:1.6.7' testCompile "junit:junit:4.11" } \ No newline at end of file diff --git a/dyno-queues-redis/build.gradle b/dyno-queues-redis/build.gradle index 24b7744..b967118 100644 --- a/dyno-queues-redis/build.gradle +++ b/dyno-queues-redis/build.gradle @@ -4,9 +4,9 @@ dependencies { compile "com.google.inject:guice:3.0" - compile 'com.netflix.dyno:dyno-core:1.6.6' - compile 'com.netflix.dyno:dyno-jedis:1.6.6' - compile 'com.netflix.dyno:dyno-demo:1.6.6' + compile 'com.netflix.dyno:dyno-core:1.6.7' + compile 'com.netflix.dyno:dyno-jedis:1.6.7' + compile 'com.netflix.dyno:dyno-demo:1.6.7' compile 'com.netflix.archaius:archaius-core:0.7.5' compile 'com.netflix.servo:servo-core:0.12.17' From 64adfda7cd99def139b6041a93a2c3c697815bf2 Mon Sep 17 00:00:00 2001 From: Sailesh Mukil Date: Fri, 16 Aug 2019 08:42:00 -0700 Subject: [PATCH 44/91] Switch Travis CI from using Oracle JDK 8 to Open JDK 8 --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 5a13010..ab84c2e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,6 +1,6 @@ language: java jdk: -- oraclejdk8 +- openjdk8 install: - export REDIS_BIN=$HOME/redis/3.0.7/bin - export TMPDIR=/tmp From 71a529b76438aa17c0afde454ba340873613092c Mon Sep 17 00:00:00 2001 From: Sailesh Mukil Date: Fri, 16 Aug 2019 08:37:32 -0700 Subject: [PATCH 45/91] Update dyno dependency to 1.6.9 1.6.7 has bugs in it which has been fixed in the new version. --- dyno-queues-core/build.gradle | 4 ++-- dyno-queues-redis/build.gradle | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/dyno-queues-core/build.gradle b/dyno-queues-core/build.gradle index 494c968..81c163e 100644 --- a/dyno-queues-core/build.gradle +++ b/dyno-queues-core/build.gradle @@ -1,5 +1,5 @@ dependencies { - compile 'com.netflix.dyno:dyno-core:1.6.7' + compile 'com.netflix.dyno:dyno-core:1.6.9' testCompile "junit:junit:4.11" -} \ No newline at end of file +} diff --git a/dyno-queues-redis/build.gradle b/dyno-queues-redis/build.gradle index b967118..1e45162 100644 --- a/dyno-queues-redis/build.gradle +++ b/dyno-queues-redis/build.gradle @@ -4,9 +4,9 @@ dependencies { compile "com.google.inject:guice:3.0" - compile 'com.netflix.dyno:dyno-core:1.6.7' - compile 'com.netflix.dyno:dyno-jedis:1.6.7' - compile 'com.netflix.dyno:dyno-demo:1.6.7' + compile 'com.netflix.dyno:dyno-core:1.6.9' + compile 'com.netflix.dyno:dyno-jedis:1.6.9' + compile 'com.netflix.dyno:dyno-demo:1.6.9' compile 'com.netflix.archaius:archaius-core:0.7.5' compile 'com.netflix.servo:servo-core:0.12.17' From be985f3b0e0ad47e5cc3032f283a341510f11e8c Mon Sep 17 00:00:00 2001 From: Sailesh Mukil Date: Thu, 29 Aug 2019 09:12:01 -0700 Subject: [PATCH 46/91] Create getMsgWithPredicate() API Checks the message bodies (i.e. the data in the hash map), and returns the message ID of the first message with 'predicate'. Returns null if not found. --- .../com/netflix/dyno/queues/DynoQueue.java | 12 ++++++++++ .../dyno/queues/demo/DynoQueueDemo.java | 13 ++++++---- .../dyno/queues/redis/RedisDynoQueue.java | 24 ++++++++++++------- .../dyno/queues/redis/v2/MultiRedisQueue.java | 5 ++++ .../queues/redis/v2/RedisPipelineQueue.java | 5 ++++ 5 files changed, 47 insertions(+), 12 deletions(-) diff --git a/dyno-queues-core/src/main/java/com/netflix/dyno/queues/DynoQueue.java b/dyno-queues-core/src/main/java/com/netflix/dyno/queues/DynoQueue.java index 5bd0c64..ed811db 100644 --- a/dyno-queues-core/src/main/java/com/netflix/dyno/queues/DynoQueue.java +++ b/dyno-queues-core/src/main/java/com/netflix/dyno/queues/DynoQueue.java @@ -128,6 +128,18 @@ public interface DynoQueue extends Closeable { */ public boolean containsPredicate(String predicate); + /** + * Checks the message bodies (i.e. the data in the hash map), and returns the ID of the first message to match with + * 'predicate'. + * + * Disclaimer: This is a potentially expensive call, since we will iterate over the entire hash map in the + * worst case. Use mindfully. + * + * @param predicate The predicate to check against. + * @return Message ID as string if any of the messages contain 'predicate'; 'null' otherwise. + */ + public String getMsgWithPredicate(String predicate); + /** * * @param messageId message to be retrieved. diff --git a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/demo/DynoQueueDemo.java b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/demo/DynoQueueDemo.java index 973d704..354a7fe 100644 --- a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/demo/DynoQueueDemo.java +++ b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/demo/DynoQueueDemo.java @@ -84,24 +84,29 @@ private void runSimpleV1Demo(DynoJedisClient dyno) throws IOException { Message msg1 = new Message("id1", "searchable payload"); Message msg2 = new Message("id2", "payload 2"); - Message msg3 = new Message("id2", "payload 3"); + Message msg3 = new Message("id3", "payload 3"); DynoQueue V1Queue = queues.get("simpleQueue"); // Test push() API List pushed_msgs = V1Queue.push(ImmutableList.of(msg1, msg2, msg3)); // Test ensure() API - logger.info("Does Message with ID '" + msg1.getId() + "' already exist? " + !V1Queue.ensure(msg1)); + logger.info("Does Message with ID '" + msg1.getId() + "' already exist? -> " + !V1Queue.ensure(msg1)); // Test containsPredicate() API - logger.info("Does the predicate 'searchable' exist in the queue ? " + V1Queue.containsPredicate("searchable")); + logger.info("Does the predicate 'searchable' exist in the queue? -> " + V1Queue.containsPredicate("searchable")); + + // Test getMsgWithPredicate() API + logger.info("Get MSG ID that contains 'searchable' in the queue -> " + V1Queue.getMsgWithPredicate("searchable")); // Test pop() List popped_msgs = V1Queue.pop(3, 1000, TimeUnit.MILLISECONDS); // Test ack() - assert (V1Queue.ack(popped_msgs.get(0).getId())); + boolean ack_successful = V1Queue.ack(popped_msgs.get(0).getId()); + assert(ack_successful); + V1Queue.clear(); V1Queue.close(); } diff --git a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/RedisDynoQueue.java b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/RedisDynoQueue.java index df395e5..142d452 100644 --- a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/RedisDynoQueue.java +++ b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/RedisDynoQueue.java @@ -268,7 +268,7 @@ private List _pop(int messageCount) throws Exception { long added = quorumConn.zadd(unackQueueName, unackScore, msgId, zParams); if (added == 0) { if (logger.isDebugEnabled()) { - logger.debug("cannot add {} to the unack shard ", queueName, msgId); + logger.debug("cannot add {} to the unack shard {}", msgId, unackQueueName); } monitor.misses.increment(); continue; @@ -444,35 +444,43 @@ public boolean ensure(Message message) { @Override public boolean containsPredicate(String predicate) { - return execute("containsPredicate", messageStoreKey, () -> { + return execute("containsPredicate", messageStoreKey, () -> getMsgWithPredicate(predicate) != null); + } + + @Override + public String getMsgWithPredicate(String predicate) { + return execute("getMsgWithPredicate", messageStoreKey, () -> { // We use a Lua script here to do predicate matching since we only want to find whether the predicate // exists in any of the message bodies or not, and the only way to do that is to check for the predicate // match on the server side. // The alternative is to have the server return all the hash values back to us and we filter it here on // the client side. This is not desirable since we would potentially be sending large amounts of data - // over the network only to return a 'true' or 'false' value back to the calling application. + // over the network only to return a single string value back to the calling application. String predicateCheckLuaScript = "local hkey=KEYS[1]\n" + "local predicate=ARGV[1]\n" + "local cursor=0\n" + "local begin=false\n" + "while (cursor ~= 0 or begin==false) do\n" + " local ret = redis.call('hscan', hkey, cursor)\n" + + " local curmsgid\n" + " for i, content in ipairs(ret[2]) do\n" + - " if (string.find(content, predicate)) then\n" + - " return 1\n" + + " if (i % 2 ~= 0) then\n" + + " curmsgid = content\n" + + " elseif (string.find(content, predicate)) then\n" + + " return curmsgid\n" + " end\n" + " end\n" + " cursor=tonumber(ret[1])\n" + " begin=true\n" + "end\n" + - "return 0"; + "return nil"; // Cast from 'JedisCommands' to 'DynoJedisClient' here since the former does not expose 'eval()'. - Long retval = (Long) ((DynoJedisClient) quorumConn).eval(predicateCheckLuaScript, + String retval = (String) ((DynoJedisClient) quorumConn).eval(predicateCheckLuaScript, Collections.singletonList(messageStoreKey), Collections.singletonList(predicate)); - return (retval == 1); + return retval; }); } diff --git a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/v2/MultiRedisQueue.java b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/v2/MultiRedisQueue.java index f934889..3c1b766 100644 --- a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/v2/MultiRedisQueue.java +++ b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/v2/MultiRedisQueue.java @@ -152,6 +152,11 @@ public boolean containsPredicate(String predicate) { throw new UnsupportedOperationException(); } + @Override + public String getMsgWithPredicate(String predicate) { + throw new UnsupportedOperationException(); + } + @Override public Message get(String messageId) { for (DynoQueue q : queues.values()) { diff --git a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/v2/RedisPipelineQueue.java b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/v2/RedisPipelineQueue.java index 65f5249..6d41034 100644 --- a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/v2/RedisPipelineQueue.java +++ b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/v2/RedisPipelineQueue.java @@ -467,6 +467,11 @@ public boolean containsPredicate(String predicate) { throw new UnsupportedOperationException(); } + @Override + public String getMsgWithPredicate(String predicate) { + throw new UnsupportedOperationException(); + } + @Override public Message get(String messageId) { From f4813353275411428c9eed9f799839e2e3eab75f Mon Sep 17 00:00:00 2001 From: Sailesh Mukil Date: Fri, 30 Aug 2019 11:00:48 -0700 Subject: [PATCH 47/91] getMsgWithPredicate() should use nonQuorumConn Each hashmap across replicas can have the same items in different orders. Therefore, it's possible that getMsgWithPredicate() can have a quorum failure since each replica matched the predicate with a different message ID. This patch changes it to nonQuorumConn, as any one valid message ID would do. --- .../main/java/com/netflix/dyno/queues/redis/RedisDynoQueue.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/RedisDynoQueue.java b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/RedisDynoQueue.java index 142d452..fff885a 100644 --- a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/RedisDynoQueue.java +++ b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/RedisDynoQueue.java @@ -477,7 +477,7 @@ public String getMsgWithPredicate(String predicate) { "return nil"; // Cast from 'JedisCommands' to 'DynoJedisClient' here since the former does not expose 'eval()'. - String retval = (String) ((DynoJedisClient) quorumConn).eval(predicateCheckLuaScript, + String retval = (String) ((DynoJedisClient) nonQuorumConn).eval(predicateCheckLuaScript, Collections.singletonList(messageStoreKey), Collections.singletonList(predicate)); return retval; From 6bf4bc2429aefb1a04b2be6f77830534c4ff131d Mon Sep 17 00:00:00 2001 From: Sailesh Mukil Date: Wed, 4 Sep 2019 18:32:00 -0700 Subject: [PATCH 48/91] Pin to dyno 1.7.0 --- dyno-queues-core/build.gradle | 2 +- dyno-queues-redis/build.gradle | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/dyno-queues-core/build.gradle b/dyno-queues-core/build.gradle index 81c163e..62c8855 100644 --- a/dyno-queues-core/build.gradle +++ b/dyno-queues-core/build.gradle @@ -1,5 +1,5 @@ dependencies { - compile 'com.netflix.dyno:dyno-core:1.6.9' + compile 'com.netflix.dyno:dyno-core:1.7.0' testCompile "junit:junit:4.11" } diff --git a/dyno-queues-redis/build.gradle b/dyno-queues-redis/build.gradle index 1e45162..f8da556 100644 --- a/dyno-queues-redis/build.gradle +++ b/dyno-queues-redis/build.gradle @@ -4,9 +4,9 @@ dependencies { compile "com.google.inject:guice:3.0" - compile 'com.netflix.dyno:dyno-core:1.6.9' - compile 'com.netflix.dyno:dyno-jedis:1.6.9' - compile 'com.netflix.dyno:dyno-demo:1.6.9' + compile 'com.netflix.dyno:dyno-core:1.7.0' + compile 'com.netflix.dyno:dyno-jedis:1.7.0' + compile 'com.netflix.dyno:dyno-demo:1.7.0' compile 'com.netflix.archaius:archaius-core:0.7.5' compile 'com.netflix.servo:servo-core:0.12.17' From 79f246a12505dd796e451e3891b511ae39bb0e70 Mon Sep 17 00:00:00 2001 From: Sailesh Mukil Date: Mon, 26 Aug 2019 16:52:02 -0700 Subject: [PATCH 49/91] Update APIs to compile with dyno-1.7.1 - All Host() objects are changed to be created with a HostBuilder. - Slight Demo API change --- dyno-queues-core/build.gradle | 2 +- dyno-queues-redis/build.gradle | 6 +- .../dyno/queues/demo/DynoQueueDemo.java | 2 +- .../dyno/queues/redis/v2/QueueBuilder.java | 8 ++- .../redis/CustomShardingStrategyTest.java | 28 +++++++- .../redis/DefaultShardingStrategyTest.java | 28 +++++++- .../queues/redis/DynoShardSupplierTest.java | 66 ++++++++++++------- .../dyno/queues/redis/RedisDynoQueueTest.java | 11 +++- .../benchmark/BenchmarkTestsDynoJedis.java | 12 +++- .../redis/benchmark/BenchmarkTestsJedis.java | 12 +++- .../benchmark/BenchmarkTestsNoPipelines.java | 12 +++- .../dyno/queues/redis/v2/DynoJedisTests.java | 12 +++- .../dyno/queues/redis/v2/JedisTests.java | 9 ++- .../dyno/queues/redis/v2/MultiQueueTests.java | 17 ++++- 14 files changed, 181 insertions(+), 44 deletions(-) diff --git a/dyno-queues-core/build.gradle b/dyno-queues-core/build.gradle index 62c8855..b95efc8 100644 --- a/dyno-queues-core/build.gradle +++ b/dyno-queues-core/build.gradle @@ -1,5 +1,5 @@ dependencies { - compile 'com.netflix.dyno:dyno-core:1.7.0' + compile 'com.netflix.dyno:dyno-core:1.7.1' testCompile "junit:junit:4.11" } diff --git a/dyno-queues-redis/build.gradle b/dyno-queues-redis/build.gradle index f8da556..682f43d 100644 --- a/dyno-queues-redis/build.gradle +++ b/dyno-queues-redis/build.gradle @@ -4,9 +4,9 @@ dependencies { compile "com.google.inject:guice:3.0" - compile 'com.netflix.dyno:dyno-core:1.7.0' - compile 'com.netflix.dyno:dyno-jedis:1.7.0' - compile 'com.netflix.dyno:dyno-demo:1.7.0' + compile 'com.netflix.dyno:dyno-core:1.7.1' + compile 'com.netflix.dyno:dyno-jedis:1.7.1' + compile 'com.netflix.dyno:dyno-demo:1.7.1' compile 'com.netflix.archaius:archaius-core:0.7.5' compile 'com.netflix.servo:servo-core:0.12.17' diff --git a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/demo/DynoQueueDemo.java b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/demo/DynoQueueDemo.java index 354a7fe..9a9cefa 100644 --- a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/demo/DynoQueueDemo.java +++ b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/demo/DynoQueueDemo.java @@ -55,7 +55,7 @@ public static void main(String[] args) throws IOException { } try { - demo.initWithRemoteClusterFromEurekaUrl(args[0], 8102); + demo.initWithRemoteClusterFromEurekaUrl(args[0], 8102, false); if (version == 1) { demo.runSimpleV1Demo(demo.client); diff --git a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/v2/QueueBuilder.java b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/v2/QueueBuilder.java index b9047c5..3875bb9 100644 --- a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/v2/QueueBuilder.java +++ b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/v2/QueueBuilder.java @@ -27,6 +27,7 @@ import com.netflix.discovery.EurekaClient; import com.netflix.discovery.shared.Application; import com.netflix.dyno.connectionpool.Host; +import com.netflix.dyno.connectionpool.HostBuilder; import com.netflix.dyno.connectionpool.HostSupplier; import com.netflix.dyno.connectionpool.impl.utils.ConfigUtils; import com.netflix.dyno.jedis.DynoJedisClient; @@ -267,7 +268,12 @@ private HostSupplier getHostSupplierFromEureka(String applicationName) { AmazonInfo amazonInfo = (AmazonInfo) info.getDataCenterInfo(); rack = amazonInfo.get(MetaDataKey.availabilityZone); } - Host host = new Host(info.getHostName(), info.getIPAddr(), rack, status); + //Host host = new Host(info.getHostName(), info.getIPAddr(), rack, status); + Host host = new HostBuilder() + .setHostname(info.getHostName()) + .setIpAddress(info.getIPAddr()) + .setRack(rack).setStatus(status) + .createHost(); return host; })); return hosts; diff --git a/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/CustomShardingStrategyTest.java b/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/CustomShardingStrategyTest.java index 34ba1f9..b2c4415 100644 --- a/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/CustomShardingStrategyTest.java +++ b/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/CustomShardingStrategyTest.java @@ -16,6 +16,7 @@ package com.netflix.dyno.queues.redis; import com.netflix.dyno.connectionpool.Host; +import com.netflix.dyno.connectionpool.HostBuilder; import com.netflix.dyno.connectionpool.HostSupplier; import com.netflix.dyno.queues.Message; import com.netflix.dyno.queues.ShardSupplier; @@ -68,9 +69,30 @@ public static void setUpBeforeClass() throws Exception { @Override public List getHosts() { List hosts = new LinkedList<>(); - hosts.add(new Host("host1", 8102, "rack1", Host.Status.Up)); - hosts.add(new Host("host2", 8102, "rack2", Host.Status.Up)); - hosts.add(new Host("host3", 8102, "rack3", Host.Status.Up)); + hosts.add( + new HostBuilder() + .setHostname("localhost") + .setPort(8102) + .setRack("rack1") + .setStatus(Host.Status.Up) + .createHost() + ); + hosts.add( + new HostBuilder() + .setHostname("localhost") + .setPort(8102) + .setRack("rack2") + .setStatus(Host.Status.Up) + .createHost() + ); + hosts.add( + new HostBuilder() + .setHostname("localhost") + .setPort(8102) + .setRack("rack3") + .setStatus(Host.Status.Up) + .createHost() + ); return hosts; } }; diff --git a/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/DefaultShardingStrategyTest.java b/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/DefaultShardingStrategyTest.java index 2157f01..aca49e5 100644 --- a/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/DefaultShardingStrategyTest.java +++ b/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/DefaultShardingStrategyTest.java @@ -16,6 +16,7 @@ package com.netflix.dyno.queues.redis; import com.netflix.dyno.connectionpool.Host; +import com.netflix.dyno.connectionpool.HostBuilder; import com.netflix.dyno.connectionpool.HostSupplier; import com.netflix.dyno.queues.Message; import com.netflix.dyno.queues.ShardSupplier; @@ -58,9 +59,30 @@ public static void setUpBeforeClass() throws Exception { @Override public List getHosts() { List hosts = new LinkedList<>(); - hosts.add(new Host("localhost", 8102, "us-east-1d", Host.Status.Up)); - hosts.add(new Host("localhost", 8102, "us-east-2d", Host.Status.Up)); - hosts.add(new Host("localhost", 8102, "us-east-3d", Host.Status.Up)); + hosts.add( + new HostBuilder() + .setHostname("localhost") + .setPort(8102) + .setRack("us-east-1d") + .setStatus(Host.Status.Up) + .createHost() + ); + hosts.add( + new HostBuilder() + .setHostname("localhost") + .setPort(8102) + .setRack("us-east-2d") + .setStatus(Host.Status.Up) + .createHost() + ); + hosts.add( + new HostBuilder() + .setHostname("localhost") + .setPort(8102) + .setRack("us-east-3d") + .setStatus(Host.Status.Up) + .createHost() + ); return hosts; } }; diff --git a/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/DynoShardSupplierTest.java b/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/DynoShardSupplierTest.java index 5c4576d..09914cd 100644 --- a/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/DynoShardSupplierTest.java +++ b/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/DynoShardSupplierTest.java @@ -28,6 +28,7 @@ import java.util.Set; import java.util.stream.Collectors; +import com.netflix.dyno.connectionpool.HostBuilder; import org.junit.Test; import com.netflix.dyno.connectionpool.Host; @@ -41,26 +42,47 @@ */ public class DynoShardSupplierTest { - @Test - public void test() { - HostSupplier hs = new HostSupplier() { - @Override - public List getHosts() { - List hosts = new LinkedList<>(); - hosts.add(new Host("host1", 8102, "us-east-1a", Status.Up)); - hosts.add(new Host("host1", 8102, "us-east-1b", Status.Up)); - hosts.add(new Host("host1", 8102, "us-east-1d", Status.Up)); - - return hosts; - } - }; - DynoShardSupplier supplier = new DynoShardSupplier(hs, "us-east-1", "a"); - String localShard = supplier.getCurrentShard(); - Set allShards = supplier.getQueueShards(); - - assertNotNull(localShard); - assertEquals("a", localShard); - assertNotNull(allShards); - assertEquals(Arrays.asList("a", "b", "d").stream().collect(Collectors.toSet()), allShards); - } + @Test + public void test(){ + HostSupplier hs = new HostSupplier() { + @Override + public List getHosts() { + List hosts = new LinkedList<>(); + hosts.add( + new HostBuilder() + .setHostname("host1") + .setPort(8102) + .setRack("us-east-1a") + .setStatus(Host.Status.Up) + .createHost() + ); + hosts.add( + new HostBuilder() + .setHostname("host1") + .setPort(8102) + .setRack("us-east-1b") + .setStatus(Host.Status.Up) + .createHost() + ); + hosts.add( + new HostBuilder() + .setHostname("host1") + .setPort(8102) + .setRack("us-east-1d") + .setStatus(Host.Status.Up) + .createHost() + ); + + return hosts; + } + }; + DynoShardSupplier supplier = new DynoShardSupplier(hs, "us-east-1", "a"); + String localShard = supplier.getCurrentShard(); + Set allShards = supplier.getQueueShards(); + + assertNotNull(localShard); + assertEquals("a", localShard); + assertNotNull(allShards); + assertEquals(Arrays.asList("a","b","d").stream().collect(Collectors.toSet()), allShards); + } } diff --git a/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/RedisDynoQueueTest.java b/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/RedisDynoQueueTest.java index 8a4de0f..45f32f4 100644 --- a/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/RedisDynoQueueTest.java +++ b/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/RedisDynoQueueTest.java @@ -17,7 +17,7 @@ import com.google.common.util.concurrent.Uninterruptibles; import com.netflix.dyno.connectionpool.Host; -import com.netflix.dyno.connectionpool.Host.Status; +import com.netflix.dyno.connectionpool.HostBuilder; import com.netflix.dyno.connectionpool.HostSupplier; import com.netflix.dyno.queues.DynoQueue; import com.netflix.dyno.queues.Message; @@ -70,7 +70,14 @@ public static void setUpBeforeClass() throws Exception { @Override public List getHosts() { List hosts = new LinkedList<>(); - hosts.add(new Host("ec2-11-22-33-444.compute-0.amazonaws.com", 8102, "us-east-1d", Status.Up)); + hosts.add( + new HostBuilder() + .setHostname("ec2-11-22-33-444.compute-0.amazonaws.com") + .setPort(8102) + .setRack("us-east-1d") + .setStatus(Host.Status.Up) + .createHost() + ); return hosts; } }; diff --git a/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/benchmark/BenchmarkTestsDynoJedis.java b/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/benchmark/BenchmarkTestsDynoJedis.java index 05b657f..65b16cc 100644 --- a/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/benchmark/BenchmarkTestsDynoJedis.java +++ b/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/benchmark/BenchmarkTestsDynoJedis.java @@ -4,6 +4,7 @@ package com.netflix.dyno.queues.redis.benchmark; import com.netflix.dyno.connectionpool.Host; +import com.netflix.dyno.connectionpool.HostBuilder; import com.netflix.dyno.connectionpool.HostSupplier; import com.netflix.dyno.connectionpool.TokenMapSupplier; import com.netflix.dyno.connectionpool.impl.ConnectionPoolConfigurationImpl; @@ -21,7 +22,16 @@ public class BenchmarkTestsDynoJedis extends QueueBenchmark { public BenchmarkTestsDynoJedis() { List hosts = new ArrayList<>(1); - hosts.add(new Host("localhost", "127.0.0.1", 6379, "us-east-1c", "us-east-1", Host.Status.Up)); + hosts.add( + new HostBuilder() + .setHostname("localhost") + .setIpAddress("127.0.0.1") + .setPort(6379) + .setRack("us-east-1c") + .setDatacenter("us-east-1") + .setStatus(Host.Status.Up) + .createHost() + ); QueueBuilder qb = new QueueBuilder(); diff --git a/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/benchmark/BenchmarkTestsJedis.java b/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/benchmark/BenchmarkTestsJedis.java index 020db32..c1ab9a7 100644 --- a/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/benchmark/BenchmarkTestsJedis.java +++ b/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/benchmark/BenchmarkTestsJedis.java @@ -4,6 +4,8 @@ package com.netflix.dyno.queues.redis.benchmark; import com.netflix.dyno.connectionpool.Host; +import com.netflix.dyno.connectionpool.HostBuilder; + import com.netflix.dyno.queues.redis.v2.QueueBuilder; import redis.clients.jedis.JedisPoolConfig; @@ -18,7 +20,14 @@ public class BenchmarkTestsJedis extends QueueBenchmark { public BenchmarkTestsJedis() { List hosts = new LinkedList<>(); - hosts.add(new Host("localhost", 6379, "us-east-1a")); + hosts.add( + new HostBuilder() + .setHostname("localhost") + .setPort(6379) + .setRack("us-east-1a") + .createHost() + ); + QueueBuilder qb = new QueueBuilder(); JedisPoolConfig config = new JedisPoolConfig(); @@ -52,5 +61,4 @@ public static void main(String[] args) throws Exception { System.exit(0); } } - } diff --git a/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/benchmark/BenchmarkTestsNoPipelines.java b/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/benchmark/BenchmarkTestsNoPipelines.java index 5c275e2..11262cd 100644 --- a/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/benchmark/BenchmarkTestsNoPipelines.java +++ b/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/benchmark/BenchmarkTestsNoPipelines.java @@ -4,6 +4,7 @@ package com.netflix.dyno.queues.redis.benchmark; import com.netflix.dyno.connectionpool.Host; +import com.netflix.dyno.connectionpool.HostBuilder; import com.netflix.dyno.connectionpool.HostSupplier; import com.netflix.dyno.connectionpool.TokenMapSupplier; import com.netflix.dyno.connectionpool.impl.ConnectionPoolConfigurationImpl; @@ -30,7 +31,16 @@ public BenchmarkTestsNoPipelines() { String queueName = "nopipequeue"; List hosts = new ArrayList<>(1); - hosts.add(new Host("localhost", "127.0.0.1", 6379, "us-east-1c", "us-east-1", Host.Status.Up)); + hosts.add( + new HostBuilder() + .setHostname("localhost") + .setIpAddress("127.0.0.1") + .setPort(6379) + .setRack("us-east-1c") + .setDatacenter("us-east-1") + .setStatus(Host.Status.Up) + .createHost() + ); DynoJedisClient.Builder builder = new DynoJedisClient.Builder(); HostSupplier hs = new HostSupplier() { diff --git a/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/v2/DynoJedisTests.java b/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/v2/DynoJedisTests.java index 31905ed..7e25234 100644 --- a/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/v2/DynoJedisTests.java +++ b/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/v2/DynoJedisTests.java @@ -16,6 +16,7 @@ package com.netflix.dyno.queues.redis.v2; import com.netflix.dyno.connectionpool.Host; +import com.netflix.dyno.connectionpool.HostBuilder; import com.netflix.dyno.connectionpool.HostSupplier; import com.netflix.dyno.connectionpool.TokenMapSupplier; import com.netflix.dyno.connectionpool.impl.ConnectionPoolConfigurationImpl; @@ -48,7 +49,16 @@ public DynoJedisTests() { public DynoQueue getQueue(String redisKeyPrefix, String queueName) { List hosts = new ArrayList<>(1); - hosts.add(new Host("localhost", "127.0.0.1", 6379, "us-east-1a", "us-east-1", Host.Status.Up)); + hosts.add( + new HostBuilder() + .setHostname("localhost") + .setIpAddress("127.0.0.1") + .setPort(6379) + .setRack("us-east-1a") + .setDatacenter("us-east-1") + .setStatus(Host.Status.Up) + .createHost() + ); QueueBuilder qb = new QueueBuilder(); diff --git a/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/v2/JedisTests.java b/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/v2/JedisTests.java index 77477f6..4c71903 100644 --- a/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/v2/JedisTests.java +++ b/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/v2/JedisTests.java @@ -16,6 +16,7 @@ package com.netflix.dyno.queues.redis.v2; import com.netflix.dyno.connectionpool.Host; +import com.netflix.dyno.connectionpool.HostBuilder; import com.netflix.dyno.queues.DynoQueue; import com.netflix.dyno.queues.redis.BaseQueueTests; import redis.clients.jedis.Jedis; @@ -56,7 +57,13 @@ public DynoQueue getQueue(String redisKeyPrefix, String queueName) { dynoClient.flushAll(); List hosts = new LinkedList<>(); - hosts.add(new Host("localhost", 6379, "us-east-1a")); + hosts.add( + new HostBuilder() + .setHostname("localhost") + .setPort(6379) + .setRack("us-east-1a") + .createHost() + ); QueueBuilder qb = new QueueBuilder(); DynoQueue queue = qb diff --git a/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/v2/MultiQueueTests.java b/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/v2/MultiQueueTests.java index 243f3a1..8f94647 100644 --- a/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/v2/MultiQueueTests.java +++ b/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/v2/MultiQueueTests.java @@ -16,6 +16,7 @@ package com.netflix.dyno.queues.redis.v2; import com.netflix.dyno.connectionpool.Host; +import com.netflix.dyno.connectionpool.HostBuilder; import com.netflix.dyno.queues.DynoQueue; import com.netflix.dyno.queues.Message; import org.junit.Test; @@ -58,8 +59,20 @@ public DynoQueue getQueue(String redisKeyPrefix, String queueName) { dynoClient.flushAll(); List hosts = new LinkedList<>(); - hosts.add(new Host("localhost", 6379, "us-east-1a")); - hosts.add(new Host("localhost", 6379, "us-east-1b")); + hosts.add( + new HostBuilder() + .setHostname("localhost") + .setPort(6379) + .setRack("us-east-1a") + .createHost() + ); + hosts.add( + new HostBuilder() + .setHostname("localhost") + .setPort(6379) + .setRack("us-east-2b") + .createHost() + ); QueueBuilder qb = new QueueBuilder(); DynoQueue queue = qb From 251f94ebbe7135eab8dfc6d3ef244e87d981fc8b Mon Sep 17 00:00:00 2001 From: Sailesh Mukil Date: Tue, 17 Sep 2019 08:17:55 -0700 Subject: [PATCH 50/91] Add a popWithMsgId() API This allows users to pop a specific message ID from the queue. Note that if the message will be popped only if the local shard contains the message. Test added to DynoQueueDemo. --- .../com/netflix/dyno/queues/DynoQueue.java | 9 +++ .../dyno/queues/demo/DynoQueueDemo.java | 41 ++++++++++---- .../dyno/queues/redis/RedisDynoQueue.java | 55 +++++++++++++++++++ .../dyno/queues/redis/v2/MultiRedisQueue.java | 4 ++ .../queues/redis/v2/RedisPipelineQueue.java | 4 ++ 5 files changed, 102 insertions(+), 11 deletions(-) diff --git a/dyno-queues-core/src/main/java/com/netflix/dyno/queues/DynoQueue.java b/dyno-queues-core/src/main/java/com/netflix/dyno/queues/DynoQueue.java index ed811db..60275f1 100644 --- a/dyno-queues-core/src/main/java/com/netflix/dyno/queues/DynoQueue.java +++ b/dyno-queues-core/src/main/java/com/netflix/dyno/queues/DynoQueue.java @@ -62,6 +62,15 @@ public interface DynoQueue extends Closeable { */ public List pop(int messageCount, int wait, TimeUnit unit); + /** + * Pops "messageId" from the local shard if it exists. + * Note that if "messageId" is present in a different shard, we will be unable to pop it. + * + * @param messageId ID of message to pop + * @return Returns a "Message" object if pop was successful. 'null' otherwise. + */ + public Message popWithMsgId(String messageId); + /** * Provides a peek into the queue without taking messages out. * @param messageCount number of messages to be peeked. diff --git a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/demo/DynoQueueDemo.java b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/demo/DynoQueueDemo.java index 9a9cefa..df11851 100644 --- a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/demo/DynoQueueDemo.java +++ b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/demo/DynoQueueDemo.java @@ -1,6 +1,5 @@ package com.netflix.dyno.queues.demo; -import com.google.common.collect.ImmutableList; import com.netflix.dyno.demo.redis.DynoJedisDemo; import com.netflix.dyno.jedis.DynoJedisClient; import com.netflix.dyno.queues.DynoQueue; @@ -11,10 +10,11 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import javax.xml.parsers.SAXParser; +import javax.xml.parsers.SAXParserFactory; import java.io.IOException; -import java.util.Arrays; -import java.util.List; -import java.util.Properties; +import java.io.InputStream; +import java.util.*; import java.util.concurrent.TimeUnit; public class DynoQueueDemo extends DynoJedisDemo { @@ -82,15 +82,23 @@ private void runSimpleV1Demo(DynoJedisClient dyno) throws IOException { RedisQueues queues = new RedisQueues(dyno, dyno, prefix, ss, 50_000, 50_000); - Message msg1 = new Message("id1", "searchable payload"); - Message msg2 = new Message("id2", "payload 2"); - Message msg3 = new Message("id3", "payload 3"); + List payloads = new ArrayList<>(); + payloads.add(new Message("id1", "searchable payload")); + payloads.add(new Message("id2", "payload 2")); + payloads.add(new Message("id3", "payload 3")); + payloads.add(new Message("id4", "payload 4")); + payloads.add(new Message("id5", "payload 5")); + payloads.add(new Message("id6", "payload 6")); DynoQueue V1Queue = queues.get("simpleQueue"); + // Clear the queue in case the server already has the above key. + V1Queue.clear(); + // Test push() API - List pushed_msgs = V1Queue.push(ImmutableList.of(msg1, msg2, msg3)); + List pushed_msgs = V1Queue.push(payloads); // Test ensure() API + Message msg1 = payloads.get(0); logger.info("Does Message with ID '" + msg1.getId() + "' already exist? -> " + !V1Queue.ensure(msg1)); // Test containsPredicate() API @@ -99,13 +107,24 @@ private void runSimpleV1Demo(DynoJedisClient dyno) throws IOException { // Test getMsgWithPredicate() API logger.info("Get MSG ID that contains 'searchable' in the queue -> " + V1Queue.getMsgWithPredicate("searchable")); - // Test pop() - List popped_msgs = V1Queue.pop(3, 1000, TimeUnit.MILLISECONDS); + List specific_pops = new ArrayList<>(); + // We'd only be able to pop from the local shard, so try to pop the first payload ID we see in the local shard. + for (int i = 0; i < payloads.size(); ++i) { + Message popWithMsgId = V1Queue.popWithMsgId(payloads.get(i).getId()); + if (popWithMsgId != null) { + specific_pops.add(popWithMsgId); + break; + } + } // Test ack() - boolean ack_successful = V1Queue.ack(popped_msgs.get(0).getId()); + boolean ack_successful = V1Queue.ack(specific_pops.get(0).getId()); assert(ack_successful); + // Test pop(). Even though we try to pop 3 messages, there will only be one remaining message in our local shard. + List popped_msgs = V1Queue.pop(3, 1000, TimeUnit.MILLISECONDS); + V1Queue.ack(popped_msgs.get(0).getId()); + V1Queue.clear(); V1Queue.close(); } diff --git a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/RedisDynoQueue.java b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/RedisDynoQueue.java index fff885a..e595e67 100644 --- a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/RedisDynoQueue.java +++ b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/RedisDynoQueue.java @@ -226,6 +226,61 @@ public List pop(int messageCount, int wait, TimeUnit unit) { } + @Override + public Message popWithMsgId(String messageId) { + + return execute("popWithMsgId", myQueueShard, () -> { + double unackScore = Long.valueOf(clock.millis() + unackTime).doubleValue(); + String unackQueueName = getUnackKey(queueName, shardName); + + ZAddParams zParams = ZAddParams.zAddParams().nx(); + + try { + long exists = nonQuorumConn.zrank(myQueueShard, messageId); + // If an exception wasn't thrown, the element has to exist. + assert(exists >= 0); + } catch (NullPointerException e) { + // If we get a NPE, that means "messageId" does not exist in the sorted set. + if (logger.isDebugEnabled()) { + logger.debug("Cannot get the message payload for {}", messageId); + } + monitor.misses.increment(); + return null; + } + + String json = quorumConn.hget(messageStoreKey, messageId); + if (json == null) { + if (logger.isDebugEnabled()) { + logger.debug("Cannot get the message payload for {}", messageId); + } + monitor.misses.increment(); + return null; + } + + long added = quorumConn.zadd(unackQueueName, unackScore, messageId, zParams); + if (added == 0) { + if (logger.isDebugEnabled()) { + logger.debug("cannot add {} to the unack shard {}", messageId, unackQueueName); + } + monitor.misses.increment(); + return null; + } + + long removed = quorumConn.zrem(myQueueShard, messageId); + if (removed == 0) { + if (logger.isDebugEnabled()) { + logger.debug("cannot remove {} from the queue shard ", queueName, messageId); + } + monitor.misses.increment(); + return null; + } + + Message msg = om.readValue(json, Message.class); + return msg; + }); + + } + @VisibleForTesting AtomicInteger prefetch = new AtomicInteger(0); diff --git a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/v2/MultiRedisQueue.java b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/v2/MultiRedisQueue.java index 3c1b766..e246d8c 100644 --- a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/v2/MultiRedisQueue.java +++ b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/v2/MultiRedisQueue.java @@ -89,6 +89,10 @@ public List pop(int messageCount, int wait, TimeUnit unit) { return me.pop(messageCount, wait, unit); } + @Override + public Message popWithMsgId(String messageId) { + throw new UnsupportedOperationException(); + } @Override public List peek(int messageCount) { return me.peek(messageCount); diff --git a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/v2/RedisPipelineQueue.java b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/v2/RedisPipelineQueue.java index 6d41034..2387123 100644 --- a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/v2/RedisPipelineQueue.java +++ b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/v2/RedisPipelineQueue.java @@ -245,6 +245,10 @@ public synchronized List pop(int messageCount, int wait, TimeUnit unit) } + @Override + public Message popWithMsgId(String messageId) { + throw new UnsupportedOperationException(); + } private List _pop(List batch) throws Exception { double unackScore = Long.valueOf(clock.millis() + unackTime).doubleValue(); From 761e2315d959dd8ed20e31ca9d8fbd4b38507b29 Mon Sep 17 00:00:00 2001 From: Sailesh Mukil Date: Tue, 17 Sep 2019 10:29:30 -0700 Subject: [PATCH 51/91] Allow Lua pattern matching in containsPredicate() and getMsgWithPredicate() Previously we used string.find() which would match only a letter by letter match. This patch converts it to use LUA pattern matching which can do more complicated comparisons. Lua Pattern matching: http://lua-users.org/wiki/PatternsTutorial --- .../src/main/java/com/netflix/dyno/queues/DynoQueue.java | 6 ++++++ .../java/com/netflix/dyno/queues/demo/DynoQueueDemo.java | 4 ++-- .../java/com/netflix/dyno/queues/redis/RedisDynoQueue.java | 2 +- 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/dyno-queues-core/src/main/java/com/netflix/dyno/queues/DynoQueue.java b/dyno-queues-core/src/main/java/com/netflix/dyno/queues/DynoQueue.java index 60275f1..2aa5d7e 100644 --- a/dyno-queues-core/src/main/java/com/netflix/dyno/queues/DynoQueue.java +++ b/dyno-queues-core/src/main/java/com/netflix/dyno/queues/DynoQueue.java @@ -129,6 +129,9 @@ public interface DynoQueue extends Closeable { * Checks the message bodies (i.e. the data in the hash map), and returns true on the first match with * 'predicate'. * + * Matching is done based on 'lua pattern' matching. + * http://lua-users.org/wiki/PatternsTutorial + * * Disclaimer: This is a potentially expensive call, since we will iterate over the entire hash map in the * worst case. Use mindfully. * @@ -141,6 +144,9 @@ public interface DynoQueue extends Closeable { * Checks the message bodies (i.e. the data in the hash map), and returns the ID of the first message to match with * 'predicate'. * + * Matching is done based on 'lua pattern' matching. + * http://lua-users.org/wiki/PatternsTutorial + * * Disclaimer: This is a potentially expensive call, since we will iterate over the entire hash map in the * worst case. Use mindfully. * diff --git a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/demo/DynoQueueDemo.java b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/demo/DynoQueueDemo.java index df11851..b8ae6a6 100644 --- a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/demo/DynoQueueDemo.java +++ b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/demo/DynoQueueDemo.java @@ -83,7 +83,7 @@ private void runSimpleV1Demo(DynoJedisClient dyno) throws IOException { RedisQueues queues = new RedisQueues(dyno, dyno, prefix, ss, 50_000, 50_000); List payloads = new ArrayList<>(); - payloads.add(new Message("id1", "searchable payload")); + payloads.add(new Message("id1", "searchable payload123")); payloads.add(new Message("id2", "payload 2")); payloads.add(new Message("id3", "payload 3")); payloads.add(new Message("id4", "payload 4")); @@ -105,7 +105,7 @@ private void runSimpleV1Demo(DynoJedisClient dyno) throws IOException { logger.info("Does the predicate 'searchable' exist in the queue? -> " + V1Queue.containsPredicate("searchable")); // Test getMsgWithPredicate() API - logger.info("Get MSG ID that contains 'searchable' in the queue -> " + V1Queue.getMsgWithPredicate("searchable")); + logger.info("Get MSG ID that contains 'searchable' in the queue -> " + V1Queue.getMsgWithPredicate("searchable pay*")); List specific_pops = new ArrayList<>(); // We'd only be able to pop from the local shard, so try to pop the first payload ID we see in the local shard. diff --git a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/RedisDynoQueue.java b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/RedisDynoQueue.java index e595e67..8f0783b 100644 --- a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/RedisDynoQueue.java +++ b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/RedisDynoQueue.java @@ -522,7 +522,7 @@ public String getMsgWithPredicate(String predicate) { " for i, content in ipairs(ret[2]) do\n" + " if (i % 2 ~= 0) then\n" + " curmsgid = content\n" + - " elseif (string.find(content, predicate)) then\n" + + " elseif (string.match(content, predicate)) then\n" + " return curmsgid\n" + " end\n" + " end\n" + From 52f79ee435924aa2ce38e8c3bd18f2dbaab9b9cb Mon Sep 17 00:00:00 2001 From: Sailesh Mukil Date: Tue, 17 Sep 2019 10:49:54 -0700 Subject: [PATCH 52/91] remove() may return 'false' even though it removed the element remove() was implemented incorrectly. This fixes it. --- .../java/com/netflix/dyno/queues/demo/DynoQueueDemo.java | 7 +++++++ .../java/com/netflix/dyno/queues/redis/RedisDynoQueue.java | 5 +++-- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/demo/DynoQueueDemo.java b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/demo/DynoQueueDemo.java index b8ae6a6..bf2c0da 100644 --- a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/demo/DynoQueueDemo.java +++ b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/demo/DynoQueueDemo.java @@ -89,6 +89,9 @@ private void runSimpleV1Demo(DynoJedisClient dyno) throws IOException { payloads.add(new Message("id4", "payload 4")); payloads.add(new Message("id5", "payload 5")); payloads.add(new Message("id6", "payload 6")); + payloads.add(new Message("id7", "payload 7")); + payloads.add(new Message("id8", "payload 8")); + payloads.add(new Message("id9", "payload 9")); DynoQueue V1Queue = queues.get("simpleQueue"); // Clear the queue in case the server already has the above key. @@ -121,6 +124,10 @@ private void runSimpleV1Demo(DynoJedisClient dyno) throws IOException { boolean ack_successful = V1Queue.ack(specific_pops.get(0).getId()); assert(ack_successful); + // Test remove() + boolean removed = V1Queue.remove("id9"); + assert(removed); + // Test pop(). Even though we try to pop 3 messages, there will only be one remaining message in our local shard. List popped_msgs = V1Queue.pop(3, 1000, TimeUnit.MILLISECONDS); V1Queue.ack(popped_msgs.get(0).getId()); diff --git a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/RedisDynoQueue.java b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/RedisDynoQueue.java index 8f0783b..83ac8aa 100644 --- a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/RedisDynoQueue.java +++ b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/RedisDynoQueue.java @@ -458,9 +458,10 @@ public boolean remove(String messageId) { String queueShardKey = getQueueShardKey(queueName, shard); Long removed = quorumConn.zrem(queueShardKey, messageId); - Long msgRemoved = quorumConn.hdel(messageStoreKey, messageId); - if (removed > 0 && msgRemoved > 0) { + if (removed > 0) { + // Ignoring return value since we just want to get rid of it. + Long msgRemoved = quorumConn.hdel(messageStoreKey, messageId); return true; } } From bb124c93159ae28707346fab7ee1c36ca5dd7c0f Mon Sep 17 00:00:00 2001 From: Sailesh Mukil Date: Tue, 17 Sep 2019 19:50:56 -0700 Subject: [PATCH 53/91] Add logging for processUnacks() We suspect that processUnacks() doesn't get the job done in some cases. Adding logging to help debug further. --- .../netflix/dyno/queues/redis/RedisDynoQueue.java | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/RedisDynoQueue.java b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/RedisDynoQueue.java index 83ac8aa..df05baa 100644 --- a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/RedisDynoQueue.java +++ b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/RedisDynoQueue.java @@ -650,11 +650,13 @@ public void processUnacks() { String unackQueueName = getUnackKey(queueName, shardName); double now = Long.valueOf(clock.millis()).doubleValue(); + int num_moved_back = 0; + int num_stale = 0; Set unacks = quorumConn.zrangeByScoreWithScores(unackQueueName, 0, now, 0, batchSize); if (unacks.size() > 0) { - logger.debug("Adding " + unacks.size() + " messages back to the queue for " + queueName); + logger.info("processUnacks: Adding " + unacks.size() + " messages back to shard of queue: " + unackQueueName); } for (Tuple unack : unacks) { @@ -665,15 +667,21 @@ public void processUnacks() { String payload = quorumConn.hget(messageStoreKey, member); if (payload == null) { quorumConn.zrem(unackQueueName, member); + ++num_stale; continue; } - quorumConn.zadd(myQueueShard, score, member); - quorumConn.zrem(unackQueueName, member); + long added_back = quorumConn.zadd(myQueueShard, score, member); + long removed_from_unack = quorumConn.zrem(unackQueueName, member); + if (added_back > 0 && removed_from_unack > 0) ++num_moved_back; } + + logger.info("processUnacks: Moved back " + num_moved_back + " items. Got rid of " + num_stale + " stale items."); return null; }); + } catch (Exception e) { + logger.error("Error while processing unacks. " + e.getMessage()); } finally { sw.stop(); } From 7ccd0b93e3aaa456707ece9e51a1bb39c1ba4184 Mon Sep 17 00:00:00 2001 From: Sailesh Mukil Date: Tue, 17 Sep 2019 19:53:28 -0700 Subject: [PATCH 54/91] Make peekIds() use nonQuorumConn. peekIds() just looks at what's currently in the queue. There's no reason it needs to be done using a quorum connection. --- .../main/java/com/netflix/dyno/queues/redis/RedisDynoQueue.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/RedisDynoQueue.java b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/RedisDynoQueue.java index df05baa..9278ca3 100644 --- a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/RedisDynoQueue.java +++ b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/RedisDynoQueue.java @@ -628,7 +628,7 @@ private Set peekIds(int offset, int count) { return execute("peekIds", myQueueShard, () -> { double now = Long.valueOf(clock.millis() + 1).doubleValue(); - Set scanned = quorumConn.zrangeByScore(myQueueShard, 0, now, offset, count); + Set scanned = nonQuorumConn.zrangeByScore(myQueueShard, 0, now, offset, count); return scanned; }); From 34fea81cedd4eeebd891a8e0e2d273688b06a972 Mon Sep 17 00:00:00 2001 From: Sailesh Mukil Date: Tue, 17 Sep 2019 22:33:19 -0700 Subject: [PATCH 55/91] processUnacks() should use nonQuorumConn to look over a range in the zset processUnacks() uses the ZRANGEBYSCORE command to return a large list of items. It's possible that all 3 of the replicas may not have the exact same items all the time causing the command to fail with a quorum error. The following update commands (HDEL, ZREM, ZADD) will still use quorum since they modify the sets, but since ZRANGEBYSCORE is only doing a read quorum is not necessary. --- .../main/java/com/netflix/dyno/queues/redis/RedisDynoQueue.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/RedisDynoQueue.java b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/RedisDynoQueue.java index 9278ca3..db793d7 100644 --- a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/RedisDynoQueue.java +++ b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/RedisDynoQueue.java @@ -653,7 +653,7 @@ public void processUnacks() { int num_moved_back = 0; int num_stale = 0; - Set unacks = quorumConn.zrangeByScoreWithScores(unackQueueName, 0, now, 0, batchSize); + Set unacks = nonQuorumConn.zrangeByScoreWithScores(unackQueueName, 0, now, 0, batchSize); if (unacks.size() > 0) { logger.info("processUnacks: Adding " + unacks.size() + " messages back to shard of queue: " + unackQueueName); From 964f57aa7fa7a5aaf99d575df3424471f43016ec Mon Sep 17 00:00:00 2001 From: Sailesh Mukil Date: Thu, 9 May 2019 13:51:00 -0700 Subject: [PATCH 56/91] Add unsafePeekAllShards() and unsafePopAllShards() functions This patch exposes 2 new APIs that allow the calling application to peek into and pop from all shards of the queue. The APIs are prepended with unsafe* because if multiple client app instances call these APIs concurrently, it's possible that duplicate items may be returned to the application across client instances. Hence, these APIs must not be used unless the calling application's use case is okay with handling duplicate items. The DynoQueueDemo is updated to use these new APIs. Additionally, a lot of the code is refactored in V1. --- .../com/netflix/dyno/queues/DynoQueue.java | 50 ++- .../dyno/queues/demo/DynoQueueDemo.java | 23 +- .../dyno/queues/redis/RedisDynoQueue.java | 328 ++++++++++++++---- .../dyno/queues/redis/v2/MultiRedisQueue.java | 10 + .../queues/redis/v2/RedisPipelineQueue.java | 10 + .../dyno/queues/redis/RedisDynoQueueTest.java | 2 +- 6 files changed, 352 insertions(+), 71 deletions(-) diff --git a/dyno-queues-core/src/main/java/com/netflix/dyno/queues/DynoQueue.java b/dyno-queues-core/src/main/java/com/netflix/dyno/queues/DynoQueue.java index 2aa5d7e..6a77c0c 100644 --- a/dyno-queues-core/src/main/java/com/netflix/dyno/queues/DynoQueue.java +++ b/dyno-queues-core/src/main/java/com/netflix/dyno/queues/DynoQueue.java @@ -21,6 +21,7 @@ import java.io.Closeable; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.concurrent.TimeUnit; /** @@ -54,7 +55,8 @@ public interface DynoQueue extends Closeable { * @param messageCount number of messages to be popped out of the queue. * @param wait Amount of time to wait if there are no messages in queue * @param unit Time unit for the wait period - * @return messages. Can be less than the messageCount if there are fewer messages available than the message count. If the popped messages are not acknowledge in a timely manner, they are pushed back into the queue. + * @return messages. Can be less than the messageCount if there are fewer messages available than the message count. + * If the popped messages are not acknowledge in a timely manner, they are pushed back into the queue. * @see #peek(int) * @see #ack(String) * @see #getUnackTime() @@ -73,6 +75,9 @@ public interface DynoQueue extends Closeable { /** * Provides a peek into the queue without taking messages out. + * + * Note: This peeks only into the 'local' shard. + * * @param messageCount number of messages to be peeked. * @return List of peeked messages. * @see #pop(int, int, TimeUnit) @@ -185,4 +190,47 @@ public interface DynoQueue extends Closeable { * Process un-acknowledged messages. The messages which are polled by the client but not ack'ed are moved back to queue */ public void processUnacks(); + + /* + * <=== Begin unsafe* functions. ===> + * + * The unsafe functions listed below are not advisable to use. + * The reason they are listed as unsafe is that they operate over all shards of a queue which means that + * due to the eventually consistent nature of Dynomite, the calling application may see duplicate item(s) that + * may have already been popped in a different rack, by another instance of the same application. + * + * Why are these functions made available then? + * There are some users of dyno-queues who have use-cases that are completely okay with dealing with duplicate + * items. + */ + + /** + * Provides a peek into all shards of the queue without taking messages out. + * Note: The local shard will always be looked into first and other shards will be filled behind it (if 'messageCount' is + * greater than the number of elements in the local shard). This way we ensure the chances of duplicates are less. + * + * @param count The number of messages to peek. + * @return A list of up to 'count' messages. + */ + public List unsafePeekAllShards(final int messageCount); + + + /** + * Allows popping from all shards of the queue. + * + * Note: The local shard will always be looked into first and other shards will be filled behind it (if 'messageCount' is + * greater than the number of elements in the local shard). This way we ensure the chances of duplicates are less. + * + * @param messageCount number of messages to be popped out of the queue. + * @param wait Amount of time to wait for each shard if there are no messages in shard. + * @param unit Time unit for the wait period + * @return messages. Can be less than the messageCount if there are fewer messages available than the message count. + * If the popped messages are not acknowledge in a timely manner, they are pushed back into + * the queue. + * @see #peek(int) + * @see #ack(String) + * @see #getUnackTime() + * + */ + public List unsafePopAllShards(int messageCount, int wait, TimeUnit unit); } diff --git a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/demo/DynoQueueDemo.java b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/demo/DynoQueueDemo.java index bf2c0da..da34ac7 100644 --- a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/demo/DynoQueueDemo.java +++ b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/demo/DynoQueueDemo.java @@ -10,11 +10,11 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import javax.xml.parsers.SAXParser; -import javax.xml.parsers.SAXParserFactory; import java.io.IOException; -import java.io.InputStream; -import java.util.*; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Properties; import java.util.concurrent.TimeUnit; public class DynoQueueDemo extends DynoJedisDemo { @@ -92,6 +92,7 @@ private void runSimpleV1Demo(DynoJedisClient dyno) throws IOException { payloads.add(new Message("id7", "payload 7")); payloads.add(new Message("id8", "payload 8")); payloads.add(new Message("id9", "payload 9")); + DynoQueue V1Queue = queues.get("simpleQueue"); // Clear the queue in case the server already has the above key. @@ -125,6 +126,8 @@ private void runSimpleV1Demo(DynoJedisClient dyno) throws IOException { assert(ack_successful); // Test remove() + // Note: This checks for "id9" specifically as it implicitly expects every 3rd element we push to be in our + // local shard. boolean removed = V1Queue.remove("id9"); assert(removed); @@ -132,6 +135,18 @@ private void runSimpleV1Demo(DynoJedisClient dyno) throws IOException { List popped_msgs = V1Queue.pop(3, 1000, TimeUnit.MILLISECONDS); V1Queue.ack(popped_msgs.get(0).getId()); + // Test unsafePeekAllShards() + List peek_all_msgs = V1Queue.unsafePeekAllShards(5); + for (Message msg : peek_all_msgs) { + logger.info("Message peeked (ID : payload) -> " + msg.getId() + " : " + msg.getPayload()); + } + + // Test unsafePopAllShards() + List pop_all_msgs = V1Queue.unsafePopAllShards(2, 1000, TimeUnit.MILLISECONDS); + for (Message msg : pop_all_msgs) { + logger.info("Message popped (ID : payload) -> " + msg.getId() + " : " + msg.getPayload()); + } + V1Queue.clear(); V1Queue.close(); } diff --git a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/RedisDynoQueue.java b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/RedisDynoQueue.java index db793d7..54c7de9 100644 --- a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/RedisDynoQueue.java +++ b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/RedisDynoQueue.java @@ -32,12 +32,7 @@ import java.io.IOException; import java.time.Clock; -import java.util.Collections; -import java.util.HashMap; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import java.util.Set; +import java.util.*; import java.util.concurrent.ConcurrentLinkedQueue; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; @@ -69,7 +64,7 @@ public class RedisDynoQueue implements DynoQueue { private final String messageStoreKey; - private final String myQueueShard; + private final String localQueueShard; private volatile int unackTime = 60; @@ -83,12 +78,23 @@ public class RedisDynoQueue implements DynoQueue { private final ConcurrentLinkedQueue prefetchedIds; + private final Map> unsafePrefetchedIdsAllShardsMap; + private final ScheduledExecutorService schedulerForUnacksProcessing; private final int retryCount = 2; private final ShardingStrategy shardingStrategy; + // Tracks the number of message IDs to prefetch based on the message counts requested by the caller via pop(). + @VisibleForTesting + AtomicInteger numIdsToPrefetch; + + // Tracks the number of message IDs to prefetch based on the message counts requested by the caller via + // unsafePopAllShards(). + @VisibleForTesting + AtomicInteger unsafeNumIdsToPrefetchAllShards; + public RedisDynoQueue(String redisKeyPrefix, String queueName, Set allShards, String shardName, ShardingStrategy shardingStrategy) { this(redisKeyPrefix, queueName, allShards, shardName, 60_000, shardingStrategy); } @@ -104,13 +110,19 @@ public RedisDynoQueue(Clock clock, String redisKeyPrefix, String queueName, Set< this.allShards = ImmutableList.copyOf(allShards.stream().collect(Collectors.toList())); this.shardName = shardName; this.messageStoreKey = redisKeyPrefix + ".MESSAGE." + queueName; - this.myQueueShard = getQueueShardKey(queueName, shardName); + this.localQueueShard = getQueueShardKey(queueName, shardName); this.shardingStrategy = shardingStrategy; + this.numIdsToPrefetch = new AtomicInteger(0); + this.unsafeNumIdsToPrefetchAllShards = new AtomicInteger(0); this.om = QueueUtils.constructObjectMapper(); this.monitor = new QueueMonitor(queueName, shardName); this.prefetchedIds = new ConcurrentLinkedQueue<>(); + this.unsafePrefetchedIdsAllShardsMap = new HashMap<>(); + for (String shard : allShards) { + unsafePrefetchedIdsAllShardsMap.put(getQueueShardKey(queueName, shard), new ConcurrentLinkedQueue<>()); + } schedulerForUnacksProcessing = Executors.newScheduledThreadPool(1); @@ -134,6 +146,17 @@ public RedisDynoQueue withUnackTime(int unackTime) { return this; } + /** + * @return Number of items in each ConcurrentLinkedQueue from 'unsafePrefetchedIdsAllShardsMap'. + */ + private int unsafeGetNumPrefetchedIds() { + // Note: We use an AtomicInteger due to Java's limitation of not allowing the modification of local native + // data types in lambdas (Java 8). + AtomicInteger totalSize = new AtomicInteger(0); + unsafePrefetchedIdsAllShardsMap.forEach((k,v)->totalSize.addAndGet(v.size())); + return totalSize.get(); + } + @Override public String getName() { return queueName; @@ -181,24 +204,113 @@ public List peek(final int messageCount) { if (ids == null) { return Collections.emptyList(); } + return doPeekBodyHelper(ids); - List msgs = execute("peek", messageStoreKey, () -> { - List messages = new LinkedList(); - for (String id : ids) { - String json = nonQuorumConn.hget(messageStoreKey, id); - Message message = om.readValue(json, Message.class); - messages.add(message); - } - return messages; - }); + } finally { + sw.stop(); + } + } - return msgs; + @Override + public List unsafePeekAllShards(final int messageCount) { + + Stopwatch sw = monitor.peek.start(); + try { + + Set ids = peekIdsAllShards(0, messageCount); + if (ids == null) { + return Collections.emptyList(); + } + return doPeekBodyHelper(ids); } finally { sw.stop(); } } + /** + * + * Peeks into 'this.localQueueShard' and returns up to 'count' items starting at position 'offset' in the shard. + * + * + * @param offset Number of items to skip over in 'this.localQueueShard' + * @param count Number of items to return. + * @return Up to 'count' number of message IDs in a set. + */ + private Set peekIds(final int offset, final int count, final double peekTillTs) { + + return execute("peekIds", localQueueShard, () -> { + double peekTillTsOrNow = (peekTillTs == 0.0) ? Long.valueOf(clock.millis() + 1).doubleValue() : peekTillTs; + return doPeekIdsFromShardHelper(localQueueShard, peekTillTsOrNow, offset, count); + }); + + } + + private Set peekIds(final int offset, final int count) { + return peekIds(offset, count, 0.0); + } + + /** + * + * Same as 'peekIds()' but looks into all shards of the queue ('this.allShards'). + * + * @param count Number of items to return. + * @return Up to 'count' number of message IDs in a set. + */ + private Set peekIdsAllShards(final int offset, final int count) { + return execute("peekIdsAllShards", localQueueShard, () -> { + Set scanned = new HashSet<>(); + double now = Long.valueOf(clock.millis() + 1).doubleValue(); + int remaining_count = count; + + // Try to get as many items from 'this.localQueueShard' first to reduce chances of returning duplicate items. + // (See unsafe* functions disclaimer in DynoQueue.java) + scanned.addAll(peekIds(offset, count, now)); + remaining_count -= scanned.size(); + + for (String shard : allShards) { + String queueShardName = getQueueShardKey(queueName, shard); + // Skip 'localQueueShard'. + if (queueShardName.equals(localQueueShard)) continue; + + Set elems = doPeekIdsFromShardHelper(queueShardName, now, offset, count); + scanned.addAll(elems); + remaining_count -= elems.size(); + if (remaining_count <= 0) break; + } + + return scanned; + + }); + } + + private Set doPeekIdsFromShardHelper(final String queueShardName, final double peekTillTs, final int offset, + final int count) { + return nonQuorumConn.zrangeByScore(queueShardName, 0, peekTillTs, offset, count); + } + + /** + * Takes a set of message IDs, 'message_ids', and returns a list of Message objects + * corresponding to 'message_ids'. Read only, does not make any updates. + * + * @param message_ids Set of message IDs to peek. + * @return a list of Message objects corresponding to 'message_ids' + * + */ + private List doPeekBodyHelper(Set message_ids) { + List msgs = execute("peek", messageStoreKey, () -> { + List messages = new LinkedList(); + for (String id : message_ids) { + String json = nonQuorumConn.hget(messageStoreKey, id); + Message message = om.readValue(json, Message.class); + messages.add(message); + } + return messages; + }); + + return msgs; + } + @Override public List pop(int messageCount, int wait, TimeUnit unit) { @@ -210,13 +322,21 @@ public List pop(int messageCount, int wait, TimeUnit unit) { try { long start = clock.millis(); long waitFor = unit.toMillis(wait); - prefetch.addAndGet(messageCount); + numIdsToPrefetch.addAndGet(messageCount); + + // We prefetch message IDs here first before attempting to pop them off the sorted set. + // The reason we do this (as opposed to just popping from the head of the sorted set), + // is that due to the eventually consistent nature of Dynomite, the different replicas of the same + // sorted set _may_ not look exactly the same at any given time, i.e. they may have a different number of + // items due to replication lag. + // So, we first peek into the sorted set to find the list of message IDs that we know for sure are + // replicated across all replicas and then attempt to pop them based on those message IDs. prefetchIds(); while (prefetchedIds.size() < messageCount && ((clock.millis() - start) < waitFor)) { Uninterruptibles.sleepUninterruptibly(200, TimeUnit.MILLISECONDS); prefetchIds(); } - return _pop(messageCount); + return _pop(shardName, messageCount, prefetchedIds); } catch (Exception e) { throw new RuntimeException(e); @@ -229,14 +349,14 @@ public List pop(int messageCount, int wait, TimeUnit unit) { @Override public Message popWithMsgId(String messageId) { - return execute("popWithMsgId", myQueueShard, () -> { + return execute("popWithMsgId", localQueueShard, () -> { double unackScore = Long.valueOf(clock.millis() + unackTime).doubleValue(); - String unackQueueName = getUnackKey(queueName, shardName); + String unackShardName = getUnackKey(queueName, shardName); ZAddParams zParams = ZAddParams.zAddParams().nx(); try { - long exists = nonQuorumConn.zrank(myQueueShard, messageId); + long exists = nonQuorumConn.zrank(localQueueShard, messageId); // If an exception wasn't thrown, the element has to exist. assert(exists >= 0); } catch (NullPointerException e) { @@ -257,16 +377,16 @@ public Message popWithMsgId(String messageId) { return null; } - long added = quorumConn.zadd(unackQueueName, unackScore, messageId, zParams); + long added = quorumConn.zadd(unackShardName, unackScore, messageId, zParams); if (added == 0) { if (logger.isDebugEnabled()) { - logger.debug("cannot add {} to the unack shard {}", messageId, unackQueueName); + logger.debug("cannot add {} to the unack shard {}", messageId, unackShardName); } monitor.misses.increment(); return null; } - long removed = quorumConn.zrem(myQueueShard, messageId); + long removed = quorumConn.zrem(localQueueShard, messageId); if (removed == 0) { if (logger.isDebugEnabled()) { logger.debug("cannot remove {} from the queue shard ", queueName, messageId); @@ -281,58 +401,146 @@ public Message popWithMsgId(String messageId) { } - @VisibleForTesting - AtomicInteger prefetch = new AtomicInteger(0); + public List unsafePopAllShards(int messageCount, int wait, TimeUnit unit) { + if (messageCount < 1) { + return Collections.emptyList(); + } + + Stopwatch sw = monitor.start(monitor.pop, messageCount); + try { + long start = clock.millis(); + long waitFor = unit.toMillis(wait); + unsafeNumIdsToPrefetchAllShards.addAndGet(messageCount); + prefetchIdsAllShards(); + while(unsafeGetNumPrefetchedIds() < messageCount && ((clock.millis() - start) < waitFor)) { + Uninterruptibles.sleepUninterruptibly(200, TimeUnit.MILLISECONDS); + prefetchIdsAllShards(); + } + + int remainingCount = messageCount; + // Pop as much as possible from the local shard first to reduce chances of returning duplicate items. + // (See unsafe* functions disclaimer in DynoQueue.java) + List popped = _pop(shardName, remainingCount, prefetchedIds); + remainingCount -= popped.size(); + + for (String shard : allShards) { + String queueShardName = getQueueShardKey(queueName, shard); + List elems = _pop(shard, remainingCount, unsafePrefetchedIdsAllShardsMap.get(queueShardName)); + popped.addAll(elems); + remainingCount -= elems.size(); + } + return popped; + } catch(Exception e) { + throw new RuntimeException(e); + } finally { + sw.stop(); + } + } + + /** + * Prefetch message IDs from the local shard. + */ private void prefetchIds() { + double now = Long.valueOf(clock.millis() + 1).doubleValue(); + int numPrefetched = doPrefetchIdsHelper(localQueueShard, numIdsToPrefetch, prefetchedIds, now); + if (numPrefetched == 0) { + numIdsToPrefetch.set(0); + } + } + + + /** + * Prefetch message IDs from all shards. + */ + private void prefetchIdsAllShards() { + double now = Long.valueOf(clock.millis() + 1).doubleValue(); + + // Try to prefetch as many items from 'this.localQueueShard' first to reduce chances of returning duplicate items. + // (See unsafe* functions disclaimer in DynoQueue.java) + doPrefetchIdsHelper(localQueueShard, unsafeNumIdsToPrefetchAllShards, + unsafePrefetchedIdsAllShardsMap.get(localQueueShard), now); + + if (unsafeNumIdsToPrefetchAllShards.get() < 1) return; + + for (String shard : allShards) { + String queueShardName = getQueueShardKey(queueName, shard); + if (queueShardName.equals(localQueueShard)) continue; // Skip since we've already serviced the local shard. - if (prefetch.get() < 1) { - return; + doPrefetchIdsHelper(queueShardName, unsafeNumIdsToPrefetchAllShards, + unsafePrefetchedIdsAllShardsMap.get(queueShardName), now); + } + } + + /** + * Attempts to prefetch up to 'prefetchCounter' message IDs, by peeking into a queue based on 'peekFunction', + * and store it in a concurrent linked queue. + * + * @param prefetchCounter Number of message IDs to attempt prefetch. + * @param prefetchedIdQueue Concurrent Linked Queue where message IDs are stored. + * @param peekFunction Function to call to peek into the queue. + */ + private int doPrefetchIdsHelper(String queueShardName, AtomicInteger prefetchCounter, + ConcurrentLinkedQueue prefetchedIdQueue, double prefetchFromTs) { + + if (prefetchCounter.get() < 1) { + return 0; } - int prefetchCount = prefetch.get(); - Stopwatch sw = monitor.start(monitor.prefetch, prefetchCount); + int numSuccessfullyPrefetched = 0; + int numToPrefetch = prefetchCounter.get(); + Stopwatch sw = monitor.start(monitor.prefetch, numToPrefetch); try { + // Attempt to peek up to 'numToPrefetch' message Ids. + Set ids = doPeekIdsFromShardHelper(queueShardName, prefetchFromTs, 0, numToPrefetch); + + // Store prefetched IDs in a queue. + prefetchedIdQueue.addAll(ids); + + numSuccessfullyPrefetched = ids.size(); - Set ids = peekIds(0, prefetchCount); - prefetchedIds.addAll(ids); - prefetch.addAndGet((-1 * ids.size())); - if (prefetch.get() < 0 || ids.isEmpty()) { - prefetch.set(0); + // Account for number of IDs successfully prefetched. + prefetchCounter.addAndGet((-1 * ids.size())); + if(prefetchCounter.get() < 0) { + prefetchCounter.set(0); } } finally { sw.stop(); } - + return numSuccessfullyPrefetched; } - private List _pop(int messageCount) throws Exception { + private List _pop(String shard, int messageCount, + ConcurrentLinkedQueue prefetchedIdQueue) throws Exception { + String queueShardName = getQueueShardKey(queueName, shard); + String unackShardName = getUnackKey(queueName, shard); double unackScore = Long.valueOf(clock.millis() + unackTime).doubleValue(); - String unackQueueName = getUnackKey(queueName, shardName); - List popped = new LinkedList<>(); + // NX option indicates add only if it doesn't exist. + // https://redis.io/commands/zadd#zadd-options-redis-302-or-greater ZAddParams zParams = ZAddParams.zAddParams().nx(); - for (; popped.size() != messageCount; ) { - String msgId = prefetchedIds.poll(); - if (msgId == null) { + List popped = new LinkedList<>(); + for (;popped.size() != messageCount;) { + String msgId = prefetchedIdQueue.poll(); + if(msgId == null) { break; } - long added = quorumConn.zadd(unackQueueName, unackScore, msgId, zParams); - if (added == 0) { + long added = quorumConn.zadd(unackShardName, unackScore, msgId, zParams); + if(added == 0){ if (logger.isDebugEnabled()) { - logger.debug("cannot add {} to the unack shard {}", msgId, unackQueueName); + logger.debug("cannot add {} to the unack shard {}", msgId, unackShardName); } monitor.misses.increment(); continue; } - long removed = quorumConn.zrem(myQueueShard, msgId); + long removed = quorumConn.zrem(queueShardName, msgId); if (removed == 0) { if (logger.isDebugEnabled()) { - logger.debug("cannot remove {} from the queue shard ", queueName, msgId); + logger.debug("cannot remove {} from the queue shard {}", msgId, queueShardName); } monitor.misses.increment(); continue; @@ -624,16 +832,6 @@ public void clear() { } - private Set peekIds(int offset, int count) { - - return execute("peekIds", myQueueShard, () -> { - double now = Long.valueOf(clock.millis() + 1).doubleValue(); - Set scanned = nonQuorumConn.zrangeByScore(myQueueShard, 0, now, offset, count); - return scanned; - }); - - } - @Override public void processUnacks() { @@ -647,16 +845,16 @@ public void processUnacks() { execute("processUnacks", keyName, () -> { int batchSize = 1_000; - String unackQueueName = getUnackKey(queueName, shardName); + String unackShardName = getUnackKey(queueName, shardName); double now = Long.valueOf(clock.millis()).doubleValue(); int num_moved_back = 0; int num_stale = 0; - Set unacks = nonQuorumConn.zrangeByScoreWithScores(unackQueueName, 0, now, 0, batchSize); + Set unacks = nonQuorumConn.zrangeByScoreWithScores(unackShardName, 0, now, 0, batchSize); if (unacks.size() > 0) { - logger.info("processUnacks: Adding " + unacks.size() + " messages back to shard of queue: " + unackQueueName); + logger.info("processUnacks: Adding " + unacks.size() + " messages back to shard of queue: " + unackShardName); } for (Tuple unack : unacks) { @@ -666,13 +864,13 @@ public void processUnacks() { String payload = quorumConn.hget(messageStoreKey, member); if (payload == null) { - quorumConn.zrem(unackQueueName, member); + quorumConn.zrem(unackShardName, member); ++num_stale; continue; } - long added_back = quorumConn.zadd(myQueueShard, score, member); - long removed_from_unack = quorumConn.zrem(unackQueueName, member); + long added_back = quorumConn.zadd(localQueueShard, score, member); + long removed_from_unack = quorumConn.zrem(unackShardName, member); if (added_back > 0 && removed_from_unack > 0) ++num_moved_back; } diff --git a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/v2/MultiRedisQueue.java b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/v2/MultiRedisQueue.java index e246d8c..effe881 100644 --- a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/v2/MultiRedisQueue.java +++ b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/v2/MultiRedisQueue.java @@ -224,4 +224,14 @@ private String getNextShard() { return s; } + @Override + public List unsafePeekAllShards(final int messageCount) { + throw new UnsupportedOperationException(); + } + + @Override + public List unsafePopAllShards(int messageCount, int wait, TimeUnit unit) { + throw new UnsupportedOperationException(); + } + } diff --git a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/v2/RedisPipelineQueue.java b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/v2/RedisPipelineQueue.java index 2387123..db200d2 100644 --- a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/v2/RedisPipelineQueue.java +++ b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/v2/RedisPipelineQueue.java @@ -648,4 +648,14 @@ public void close() throws IOException { schedulerForUnacksProcessing.shutdown(); monitor.close(); } + + @Override + public List unsafePeekAllShards(final int messageCount) { + throw new UnsupportedOperationException(); + } + + @Override + public List unsafePopAllShards(int messageCount, int wait, TimeUnit unit) { + throw new UnsupportedOperationException(); + } } diff --git a/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/RedisDynoQueueTest.java b/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/RedisDynoQueueTest.java index 45f32f4..c4ded37 100644 --- a/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/RedisDynoQueueTest.java +++ b/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/RedisDynoQueueTest.java @@ -251,7 +251,7 @@ public void run() { long elapsedTime = System.currentTimeMillis() - start; assertTrue(elapsedTime >= 1000); assertEquals(0, more.size()); - assertEquals(0, rdq.prefetch.get()); + assertEquals(0, rdq.numIdsToPrefetch.get()); ses.shutdownNow(); } From 7b1b4ea3dad38c8f38802befdb90e1f236cc7084 Mon Sep 17 00:00:00 2001 From: Sailesh Mukil Date: Thu, 19 Sep 2019 08:53:36 -0700 Subject: [PATCH 57/91] unsafePopAllShards() should use the prefetched IDs from the unsafe Map To preserve the order of having local shard IDs show up first, use the 'unsafePrefetchedIdsAllShardsMap' instead of 'prefetchedIds' as we don't refresh 'prefetchedIds' on unsafePopAllShards() but rather only during pop(). Modified the demo to confirm this behavior. TODO: unsafePeekIdsAllShards() does not respect the local shard order. This needs to be fixed by moving internal representation of elements in a LIst as opposed to a Set. --- .../java/com/netflix/dyno/queues/demo/DynoQueueDemo.java | 8 +++++++- .../com/netflix/dyno/queues/redis/RedisDynoQueue.java | 2 +- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/demo/DynoQueueDemo.java b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/demo/DynoQueueDemo.java index da34ac7..1af4e1f 100644 --- a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/demo/DynoQueueDemo.java +++ b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/demo/DynoQueueDemo.java @@ -92,6 +92,12 @@ private void runSimpleV1Demo(DynoJedisClient dyno) throws IOException { payloads.add(new Message("id7", "payload 7")); payloads.add(new Message("id8", "payload 8")); payloads.add(new Message("id9", "payload 9")); + payloads.add(new Message("id10", "payload 10")); + payloads.add(new Message("id11", "payload 11")); + payloads.add(new Message("id12", "payload 12")); + payloads.add(new Message("id13", "payload 13")); + payloads.add(new Message("id14", "payload 14")); + payloads.add(new Message("id15", "payload 15")); DynoQueue V1Queue = queues.get("simpleQueue"); @@ -132,7 +138,7 @@ private void runSimpleV1Demo(DynoJedisClient dyno) throws IOException { assert(removed); // Test pop(). Even though we try to pop 3 messages, there will only be one remaining message in our local shard. - List popped_msgs = V1Queue.pop(3, 1000, TimeUnit.MILLISECONDS); + List popped_msgs = V1Queue.pop(1, 1000, TimeUnit.MILLISECONDS); V1Queue.ack(popped_msgs.get(0).getId()); // Test unsafePeekAllShards() diff --git a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/RedisDynoQueue.java b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/RedisDynoQueue.java index 54c7de9..bcea117 100644 --- a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/RedisDynoQueue.java +++ b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/RedisDynoQueue.java @@ -421,7 +421,7 @@ public List unsafePopAllShards(int messageCount, int wait, TimeUnit uni int remainingCount = messageCount; // Pop as much as possible from the local shard first to reduce chances of returning duplicate items. // (See unsafe* functions disclaimer in DynoQueue.java) - List popped = _pop(shardName, remainingCount, prefetchedIds); + List popped = _pop(shardName, remainingCount, unsafePrefetchedIdsAllShardsMap.get(localQueueShard)); remainingCount -= popped.size(); for (String shard : allShards) { From 41376a400bd4f4c4f31dfb330c99c15f4848036c Mon Sep 17 00:00:00 2001 From: Sailesh Mukil Date: Thu, 19 Sep 2019 10:57:27 -0700 Subject: [PATCH 58/91] Clarify ordering guarantees with unsafePeekAllShards() --- .../src/main/java/com/netflix/dyno/queues/DynoQueue.java | 3 +-- .../main/java/com/netflix/dyno/queues/demo/DynoQueueDemo.java | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/dyno-queues-core/src/main/java/com/netflix/dyno/queues/DynoQueue.java b/dyno-queues-core/src/main/java/com/netflix/dyno/queues/DynoQueue.java index 6a77c0c..469d80e 100644 --- a/dyno-queues-core/src/main/java/com/netflix/dyno/queues/DynoQueue.java +++ b/dyno-queues-core/src/main/java/com/netflix/dyno/queues/DynoQueue.java @@ -206,8 +206,7 @@ public interface DynoQueue extends Closeable { /** * Provides a peek into all shards of the queue without taking messages out. - * Note: The local shard will always be looked into first and other shards will be filled behind it (if 'messageCount' is - * greater than the number of elements in the local shard). This way we ensure the chances of duplicates are less. + * Note: This function does not guarantee ordering of items based on shards like unsafePopAllShards(). * * @param count The number of messages to peek. * @return A list of up to 'count' messages. diff --git a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/demo/DynoQueueDemo.java b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/demo/DynoQueueDemo.java index 1af4e1f..7035cd7 100644 --- a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/demo/DynoQueueDemo.java +++ b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/demo/DynoQueueDemo.java @@ -148,7 +148,7 @@ private void runSimpleV1Demo(DynoJedisClient dyno) throws IOException { } // Test unsafePopAllShards() - List pop_all_msgs = V1Queue.unsafePopAllShards(2, 1000, TimeUnit.MILLISECONDS); + List pop_all_msgs = V1Queue.unsafePopAllShards(7, 1000, TimeUnit.MILLISECONDS); for (Message msg : pop_all_msgs) { logger.info("Message popped (ID : payload) -> " + msg.getId() + " : " + msg.getPayload()); } From 9f086829a2e8fb9faf66033799844e3df4e188f0 Mon Sep 17 00:00:00 2001 From: Sailesh Mukil Date: Fri, 20 Sep 2019 10:36:48 -0700 Subject: [PATCH 59/91] Add a new ShardSupplier to address issues with DynoShardSupplier DynoShardSupplier supplies the shard names based on AWS regions and zones of the calling application. This works fine if the calling application is present only in one region (across all zones in the region). However, if we have calling applications that are distributed across regions, we will get into trouble with DynoShardSupplier. For example: - Assume a queue called 'simple_queue'. - App in us-east-1d pushes to the queue and it ends up in shard 'simple_queue.d'. - The counterpart for us-east-1d in the EU-west region is eu-west-1b. - When eu-west-1b tries to pop, because of the DynoShardSupplier, it will attempt to pop from 'simple_queue.b', which does not even exist at this point. ConsistentDynoShardSupplier is added as an abstract class to deal with this issue. ConsistentAWSDynoShardSupplier inherits from it and hardcodes mappings for all racks we'd use in the AWS environment. Non-AWS users may extend ConsistentDynoShardSupplier and create different mappings. The DynoShardSupplier is left in the codebase and marked as deprecated for legacy applications using it. --- .../dyno/queues/demo/DynoQueueDemo.java | 4 +- .../shard/ConsistentAWSDynoShardSupplier.java | 37 +++++++++++++ .../shard/ConsistentDynoShardSupplier.java | 54 +++++++++++++++++++ .../dyno/queues/shard/DynoShardSupplier.java | 4 ++ 4 files changed, 97 insertions(+), 2 deletions(-) create mode 100644 dyno-queues-redis/src/main/java/com/netflix/dyno/queues/shard/ConsistentAWSDynoShardSupplier.java create mode 100644 dyno-queues-redis/src/main/java/com/netflix/dyno/queues/shard/ConsistentDynoShardSupplier.java diff --git a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/demo/DynoQueueDemo.java b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/demo/DynoQueueDemo.java index 7035cd7..233eb09 100644 --- a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/demo/DynoQueueDemo.java +++ b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/demo/DynoQueueDemo.java @@ -6,7 +6,7 @@ import com.netflix.dyno.queues.Message; import com.netflix.dyno.queues.redis.RedisQueues; import com.netflix.dyno.queues.redis.v2.QueueBuilder; -import com.netflix.dyno.queues.shard.DynoShardSupplier; +import com.netflix.dyno.queues.shard.ConsistentAWSDynoShardSupplier; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -78,7 +78,7 @@ private void runSimpleV1Demo(DynoJedisClient dyno) throws IOException { String prefix = "dynoQueue_"; - DynoShardSupplier ss = new DynoShardSupplier(dyno.getConnPool().getConfiguration().getHostSupplier(), region, localRack); + ConsistentAWSDynoShardSupplier ss = new ConsistentAWSDynoShardSupplier(dyno.getConnPool().getConfiguration().getHostSupplier(), region, localRack); RedisQueues queues = new RedisQueues(dyno, dyno, prefix, ss, 50_000, 50_000); diff --git a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/shard/ConsistentAWSDynoShardSupplier.java b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/shard/ConsistentAWSDynoShardSupplier.java new file mode 100644 index 0000000..3b815ea --- /dev/null +++ b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/shard/ConsistentAWSDynoShardSupplier.java @@ -0,0 +1,37 @@ +package com.netflix.dyno.queues.shard; + +import com.netflix.dyno.connectionpool.HostSupplier; + +import java.util.HashMap; +import java.util.Map; + +public class ConsistentAWSDynoShardSupplier extends ConsistentDynoShardSupplier { + + /** + * Dynomite based shard supplier. Keeps the number of shards in parity with the hosts and regions + * + * Note: This ensures that all racks use the same shard names. This fixes issues with the now deprecated DynoShardSupplier + * that would write to the wrong shard if there are cross-region writers/readers. + * + * @param hs Host supplier + * @param region current region + * @param localRack local rack identifier + */ + public ConsistentAWSDynoShardSupplier(HostSupplier hs, String region, String localRack) { + super(hs, region, localRack); + Map rackToHashMapEntries = new HashMap() {{ + this.put("us-east-1c", "c"); + this.put("us-east-1d", "d"); + this.put("us-east-1e", "e"); + + this.put("eu-west-1a", "c"); + this.put("eu-west-1b", "d"); + this.put("eu-west-1c", "e"); + + this.put("us-west-2a", "c"); + this.put("us-west-2b", "d"); + this.put("us-west-2c", "e"); + }}; + setRackToShardMap(rackToHashMapEntries); + } +} \ No newline at end of file diff --git a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/shard/ConsistentDynoShardSupplier.java b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/shard/ConsistentDynoShardSupplier.java new file mode 100644 index 0000000..ed4c797 --- /dev/null +++ b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/shard/ConsistentDynoShardSupplier.java @@ -0,0 +1,54 @@ +package com.netflix.dyno.queues.shard; + +import com.netflix.dyno.connectionpool.Host; +import com.netflix.dyno.connectionpool.HostSupplier; +import com.netflix.dyno.queues.ShardSupplier; + +import java.util.*; + +abstract class ConsistentDynoShardSupplier implements ShardSupplier { + + protected HostSupplier hs; + + protected String region; + + protected String localRack; + + protected Map rackToShardMap; + + /** + * Dynomite based shard supplier. Keeps the number of shards in parity with the hosts and regions + * @param hs Host supplier + * @param region current region + * @param localRack local rack identifier + */ + public ConsistentDynoShardSupplier(HostSupplier hs, String region, String localRack) { + this.hs = hs; + this.region = region; + this.localRack = localRack; + } + + public void setRackToShardMap(Map rackToShardMapEntries) { + rackToShardMap = new HashMap<>(rackToShardMapEntries); + } + + @Override + public String getCurrentShard() { + return rackToShardMap.get(localRack); + } + + @Override + public Set getQueueShards() { + Set queueShards = new HashSet<>(); + List hosts = hs.getHosts(); + for (Host host : hosts) { + queueShards.add(rackToShardMap.get(host.getRack())); + } + return queueShards; + } + + @Override + public String getShardForHost(Host host) { + return rackToShardMap.get(host.getRack()); + } +} \ No newline at end of file diff --git a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/shard/DynoShardSupplier.java b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/shard/DynoShardSupplier.java index 6716257..1de8823 100644 --- a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/shard/DynoShardSupplier.java +++ b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/shard/DynoShardSupplier.java @@ -29,7 +29,11 @@ /** * @author Viren * + * NOTE: This class is deprecated and should not be used. It still remains for backwards compatibility for legacy applications + * New applications must use 'ConsistentAWSDynoShardSupplier' or extend 'ConsistentDynoShardSupplier' for non-AWS environments. + * */ +@Deprecated public class DynoShardSupplier implements ShardSupplier { private HostSupplier hs; From 75f9c4b6d792334b77e94d67b7a3b6af7bb78d61 Mon Sep 17 00:00:00 2001 From: Sailesh Mukil Date: Mon, 23 Sep 2019 10:51:57 -0700 Subject: [PATCH 60/91] Allow containsPredicate()/getMsgWithPredicate() to filter on local shard NOTE: This will only work against Dynomite clusters that have a single ring size. This extends the APIs to only return if the predicate belongs to a message ID that belongs to the local shard of the calling instance. Testing: Added to the Demo and confirmed working. --- .../com/netflix/dyno/queues/DynoQueue.java | 8 +++ .../dyno/queues/demo/DynoQueueDemo.java | 5 ++ .../dyno/queues/redis/RedisDynoQueue.java | 51 +++++++++++++++++-- .../dyno/queues/redis/v2/MultiRedisQueue.java | 13 ++++- .../queues/redis/v2/RedisPipelineQueue.java | 12 ++++- .../shard/ConsistentDynoShardSupplier.java | 2 +- 6 files changed, 83 insertions(+), 8 deletions(-) diff --git a/dyno-queues-core/src/main/java/com/netflix/dyno/queues/DynoQueue.java b/dyno-queues-core/src/main/java/com/netflix/dyno/queues/DynoQueue.java index 469d80e..b6a0095 100644 --- a/dyno-queues-core/src/main/java/com/netflix/dyno/queues/DynoQueue.java +++ b/dyno-queues-core/src/main/java/com/netflix/dyno/queues/DynoQueue.java @@ -141,9 +141,13 @@ public interface DynoQueue extends Closeable { * worst case. Use mindfully. * * @param predicate The predicate to check against. + * @param localShardOnly If this is true, it will only check if the message exists in the local shard as opposed to + * all shards. Note that this will only work if the Dynomite cluster ring size is 1 (i.e. one + * instance per AZ). * @return 'true' if any of the messages contain 'predicate'; 'false' otherwise. */ public boolean containsPredicate(String predicate); + public boolean containsPredicate(String predicate, boolean localShardOnly); /** * Checks the message bodies (i.e. the data in the hash map), and returns the ID of the first message to match with @@ -156,9 +160,13 @@ public interface DynoQueue extends Closeable { * worst case. Use mindfully. * * @param predicate The predicate to check against. + * @param localShardOnly If this is true, it will only check if the message exists in the local shard as opposed to + * all shards. Note that this will only work if the Dynomite cluster ring size is 1 (i.e. one + * instance per AZ). * @return Message ID as string if any of the messages contain 'predicate'; 'null' otherwise. */ public String getMsgWithPredicate(String predicate); + public String getMsgWithPredicate(String predicate, boolean localShardOnly); /** * diff --git a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/demo/DynoQueueDemo.java b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/demo/DynoQueueDemo.java index 233eb09..b884425 100644 --- a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/demo/DynoQueueDemo.java +++ b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/demo/DynoQueueDemo.java @@ -117,6 +117,11 @@ private void runSimpleV1Demo(DynoJedisClient dyno) throws IOException { // Test getMsgWithPredicate() API logger.info("Get MSG ID that contains 'searchable' in the queue -> " + V1Queue.getMsgWithPredicate("searchable pay*")); + // Test getMsgWithPredicate(predicate, localShardOnly=true) API + // NOTE: This only works on single ring sized Dynomite clusters. + logger.info("Get MSG ID that contains 'searchable' in the queue -> " + V1Queue.getMsgWithPredicate("searchable pay*", true)); + logger.info("Get MSG ID that contains '3' in the queue -> " + V1Queue.getMsgWithPredicate("3", true)); + List specific_pops = new ArrayList<>(); // We'd only be able to pop from the local shard, so try to pop the first payload ID we see in the local shard. for (int i = 0; i < payloads.size(); ++i) { diff --git a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/RedisDynoQueue.java b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/RedisDynoQueue.java index bcea117..a9a05b4 100644 --- a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/RedisDynoQueue.java +++ b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/RedisDynoQueue.java @@ -706,13 +706,24 @@ public boolean ensure(Message message) { }); } + @Override public boolean containsPredicate(String predicate) { - return execute("containsPredicate", messageStoreKey, () -> getMsgWithPredicate(predicate) != null); + return containsPredicate(predicate, false); } @Override public String getMsgWithPredicate(String predicate) { + return getMsgWithPredicate(predicate, false); + } + + @Override + public boolean containsPredicate(String predicate, boolean localShardOnly) { + return execute("containsPredicate", messageStoreKey, () -> getMsgWithPredicate(predicate, localShardOnly) != null); + } + + @Override + public String getMsgWithPredicate(String predicate, boolean localShardOnly) { return execute("getMsgWithPredicate", messageStoreKey, () -> { // We use a Lua script here to do predicate matching since we only want to find whether the predicate @@ -721,7 +732,7 @@ public String getMsgWithPredicate(String predicate) { // The alternative is to have the server return all the hash values back to us and we filter it here on // the client side. This is not desirable since we would potentially be sending large amounts of data // over the network only to return a single string value back to the calling application. - String predicateCheckLuaScript = "local hkey=KEYS[1]\n" + + String predicateCheckAllLuaScript = "local hkey=KEYS[1]\n" + "local predicate=ARGV[1]\n" + "local cursor=0\n" + "local begin=false\n" + @@ -740,9 +751,39 @@ public String getMsgWithPredicate(String predicate) { "end\n" + "return nil"; - // Cast from 'JedisCommands' to 'DynoJedisClient' here since the former does not expose 'eval()'. - String retval = (String) ((DynoJedisClient) nonQuorumConn).eval(predicateCheckLuaScript, - Collections.singletonList(messageStoreKey), Collections.singletonList(predicate)); + String predicateCheckLocalOnlyLuaScript = "local hkey=KEYS[1]\n" + + "local predicate=ARGV[1]\n" + + "local shard_name=ARGV[2]\n" + + "local cursor=0\n" + + "local begin=false\n" + + "while (cursor ~= 0 or begin==false) do\n" + + " local ret = redis.call('hscan', hkey, cursor)\n" + + "local curmsgid\n" + + "for i, content in ipairs(ret[2]) do\n" + + " if (i % 2 ~= 0) then\n" + + " curmsgid = content\n" + + "elseif (string.match(content, predicate)) then\n" + + "local in_local_shard = redis.call('zrank', shard_name, curmsgid)\n" + + "if (type(in_local_shard) ~= 'boolean' and in_local_shard >= 0) then\n" + + "return curmsgid\n" + + "end\n" + + " end\n" + + "end\n" + + " cursor=tonumber(ret[1])\n" + + "begin=true\n" + + "end\n" + + "return nil"; + + String retval; + if (localShardOnly) { + // Cast from 'JedisCommands' to 'DynoJedisClient' here since the former does not expose 'eval()'. + retval = (String) ((DynoJedisClient) nonQuorumConn).eval(predicateCheckLocalOnlyLuaScript, + Collections.singletonList(messageStoreKey), ImmutableList.of(predicate, localQueueShard)); + } else { + // Cast from 'JedisCommands' to 'DynoJedisClient' here since the former does not expose 'eval()'. + retval = (String) ((DynoJedisClient) nonQuorumConn).eval(predicateCheckAllLuaScript, + Collections.singletonList(messageStoreKey), Collections.singletonList(predicate)); + } return retval; }); diff --git a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/v2/MultiRedisQueue.java b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/v2/MultiRedisQueue.java index effe881..0ba5828 100644 --- a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/v2/MultiRedisQueue.java +++ b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/v2/MultiRedisQueue.java @@ -151,13 +151,24 @@ public boolean ensure(Message message) { throw new UnsupportedOperationException(); } + @Override public boolean containsPredicate(String predicate) { - throw new UnsupportedOperationException(); + return containsPredicate(predicate, false); } @Override public String getMsgWithPredicate(String predicate) { + return getMsgWithPredicate(predicate, false); + } + + @Override + public boolean containsPredicate(String predicate, boolean localShardOnly) { + throw new UnsupportedOperationException(); + } + + @Override + public String getMsgWithPredicate(String predicate, boolean localShardOnly) { throw new UnsupportedOperationException(); } diff --git a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/v2/RedisPipelineQueue.java b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/v2/RedisPipelineQueue.java index db200d2..ac54fee 100644 --- a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/v2/RedisPipelineQueue.java +++ b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/v2/RedisPipelineQueue.java @@ -468,11 +468,21 @@ public boolean ensure(Message message) { @Override public boolean containsPredicate(String predicate) { - throw new UnsupportedOperationException(); + return containsPredicate(predicate, false); } @Override public String getMsgWithPredicate(String predicate) { + return getMsgWithPredicate(predicate, false); + } + + @Override + public boolean containsPredicate(String predicate, boolean localShardOnly) { + throw new UnsupportedOperationException(); + } + + @Override + public String getMsgWithPredicate(String predicate, boolean localShardOnly) { throw new UnsupportedOperationException(); } diff --git a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/shard/ConsistentDynoShardSupplier.java b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/shard/ConsistentDynoShardSupplier.java index ed4c797..d432db3 100644 --- a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/shard/ConsistentDynoShardSupplier.java +++ b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/shard/ConsistentDynoShardSupplier.java @@ -10,7 +10,7 @@ abstract class ConsistentDynoShardSupplier implements ShardSupplier { protected HostSupplier hs; - protected String region; + protected String region; protected String localRack; From 6975b462e44145a2bf1a268bf99ae7bf087bdde5 Mon Sep 17 00:00:00 2001 From: Sailesh Mukil Date: Mon, 23 Sep 2019 12:55:47 -0700 Subject: [PATCH 61/91] Add unsafePopWithMsgIdAllShards() API Similar to popWithMsgId(), but attempts to pop from any shard. Hence, prefixed with unsafe. Testing: Added a test to the DynoQueueDemo. --- .../com/netflix/dyno/queues/DynoQueue.java | 10 +++++++++ .../dyno/queues/demo/DynoQueueDemo.java | 12 +++++++++- .../dyno/queues/redis/RedisDynoQueue.java | 22 +++++++++++++++---- .../dyno/queues/redis/v2/MultiRedisQueue.java | 6 +++++ .../queues/redis/v2/RedisPipelineQueue.java | 6 +++++ 5 files changed, 51 insertions(+), 5 deletions(-) diff --git a/dyno-queues-core/src/main/java/com/netflix/dyno/queues/DynoQueue.java b/dyno-queues-core/src/main/java/com/netflix/dyno/queues/DynoQueue.java index b6a0095..db62841 100644 --- a/dyno-queues-core/src/main/java/com/netflix/dyno/queues/DynoQueue.java +++ b/dyno-queues-core/src/main/java/com/netflix/dyno/queues/DynoQueue.java @@ -240,4 +240,14 @@ public interface DynoQueue extends Closeable { * */ public List unsafePopAllShards(int messageCount, int wait, TimeUnit unit); + + + /** + * Same as popWithMsgId(), but allows popping from any shard. + * + * @param messageId ID of message to pop + * @return Returns a "Message" object if pop was successful. 'null' otherwise. + */ + public Message unsafePopWithMsgIdAllShards(String messageId); + } diff --git a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/demo/DynoQueueDemo.java b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/demo/DynoQueueDemo.java index b884425..72f8308 100644 --- a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/demo/DynoQueueDemo.java +++ b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/demo/DynoQueueDemo.java @@ -123,12 +123,20 @@ private void runSimpleV1Demo(DynoJedisClient dyno) throws IOException { logger.info("Get MSG ID that contains '3' in the queue -> " + V1Queue.getMsgWithPredicate("3", true)); List specific_pops = new ArrayList<>(); - // We'd only be able to pop from the local shard, so try to pop the first payload ID we see in the local shard. + // We'd only be able to pop from the local shard with popWithMsgId(), so try to pop the first payload ID we see in the local shard. + // Until then pop all messages not in the local shard with unsafePopWithMsgIdAllShards(). for (int i = 0; i < payloads.size(); ++i) { Message popWithMsgId = V1Queue.popWithMsgId(payloads.get(i).getId()); if (popWithMsgId != null) { specific_pops.add(popWithMsgId); break; + } else { + // If we were unable to pop using popWithMsgId(), that means the message ID does not exist in the local shard. + // Ensure that we can pop with unsafePopWithMsgIdAllShards(). + Message unsafeSpecificPop = V1Queue.unsafePopWithMsgIdAllShards(payloads.get(i).getId()); + assert(unsafeSpecificPop != null); + boolean ack = V1Queue.ack(unsafeSpecificPop.getId()); + assert(ack); } } @@ -156,6 +164,8 @@ private void runSimpleV1Demo(DynoJedisClient dyno) throws IOException { List pop_all_msgs = V1Queue.unsafePopAllShards(7, 1000, TimeUnit.MILLISECONDS); for (Message msg : pop_all_msgs) { logger.info("Message popped (ID : payload) -> " + msg.getId() + " : " + msg.getPayload()); + boolean ack = V1Queue.ack(msg.getId()); + assert(ack); } V1Queue.clear(); diff --git a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/RedisDynoQueue.java b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/RedisDynoQueue.java index a9a05b4..cfd3cee 100644 --- a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/RedisDynoQueue.java +++ b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/RedisDynoQueue.java @@ -348,15 +348,29 @@ public List pop(int messageCount, int wait, TimeUnit unit) { @Override public Message popWithMsgId(String messageId) { + return popWithMsgIdHelper(messageId, shardName); + } + + @Override + public Message unsafePopWithMsgIdAllShards(String messageId) { + for (String shard : allShards) { + Message msg = popWithMsgIdHelper(messageId, shard); + if (msg != null) return msg; + } + return null; + } + + public Message popWithMsgIdHelper(String messageId, String targetShard) { - return execute("popWithMsgId", localQueueShard, () -> { + return execute("popWithMsgId", targetShard, () -> { + String queueShardName = getQueueShardKey(queueName, targetShard); double unackScore = Long.valueOf(clock.millis() + unackTime).doubleValue(); - String unackShardName = getUnackKey(queueName, shardName); + String unackShardName = getUnackKey(queueName, targetShard); ZAddParams zParams = ZAddParams.zAddParams().nx(); try { - long exists = nonQuorumConn.zrank(localQueueShard, messageId); + long exists = nonQuorumConn.zrank(queueShardName, messageId); // If an exception wasn't thrown, the element has to exist. assert(exists >= 0); } catch (NullPointerException e) { @@ -386,7 +400,7 @@ public Message popWithMsgId(String messageId) { return null; } - long removed = quorumConn.zrem(localQueueShard, messageId); + long removed = quorumConn.zrem(queueShardName, messageId); if (removed == 0) { if (logger.isDebugEnabled()) { logger.debug("cannot remove {} from the queue shard ", queueName, messageId); diff --git a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/v2/MultiRedisQueue.java b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/v2/MultiRedisQueue.java index 0ba5828..fa0618e 100644 --- a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/v2/MultiRedisQueue.java +++ b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/v2/MultiRedisQueue.java @@ -93,6 +93,12 @@ public List pop(int messageCount, int wait, TimeUnit unit) { public Message popWithMsgId(String messageId) { throw new UnsupportedOperationException(); } + + @Override + public Message unsafePopWithMsgIdAllShards(String messageId) { + throw new UnsupportedOperationException(); + } + @Override public List peek(int messageCount) { return me.peek(messageCount); diff --git a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/v2/RedisPipelineQueue.java b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/v2/RedisPipelineQueue.java index ac54fee..fc4138c 100644 --- a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/v2/RedisPipelineQueue.java +++ b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/v2/RedisPipelineQueue.java @@ -249,6 +249,12 @@ public synchronized List pop(int messageCount, int wait, TimeUnit unit) public Message popWithMsgId(String messageId) { throw new UnsupportedOperationException(); } + + @Override + public Message unsafePopWithMsgIdAllShards(String messageId) { + throw new UnsupportedOperationException(); + } + private List _pop(List batch) throws Exception { double unackScore = Long.valueOf(clock.millis() + unackTime).doubleValue(); From c3fc2315c8ac5a4bdadae20faf4e5cffff625873 Mon Sep 17 00:00:00 2001 From: Sailesh Mukil Date: Wed, 2 Oct 2019 14:20:48 -0400 Subject: [PATCH 62/91] Make debug message clearer in popWithMsgId() --- .../main/java/com/netflix/dyno/queues/redis/RedisDynoQueue.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/RedisDynoQueue.java b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/RedisDynoQueue.java index cfd3cee..c5375af 100644 --- a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/RedisDynoQueue.java +++ b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/RedisDynoQueue.java @@ -376,7 +376,7 @@ public Message popWithMsgIdHelper(String messageId, String targetShard) { } catch (NullPointerException e) { // If we get a NPE, that means "messageId" does not exist in the sorted set. if (logger.isDebugEnabled()) { - logger.debug("Cannot get the message payload for {}", messageId); + logger.debug("Cannot find the message with ID {}", messageId); } monitor.misses.increment(); return null; From d9cfc62fac0c1239f6028abfbd811109f6b78bc4 Mon Sep 17 00:00:00 2001 From: Sailesh Mukil Date: Wed, 2 Oct 2019 15:37:22 -0400 Subject: [PATCH 63/91] Use object form of Long for zrank() return type The Jedis docs for each command are just a copy paste of Redis docs which doesn't translate well to Java. https://github.com/xetorthio/jedis/issues/978 From some stack overflow googling, we see the right way to use zrank(), and this patch changes the code to comply with that. --- .../com/netflix/dyno/queues/redis/RedisDynoQueue.java | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/RedisDynoQueue.java b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/RedisDynoQueue.java index c5375af..ad54b9a 100644 --- a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/RedisDynoQueue.java +++ b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/RedisDynoQueue.java @@ -369,12 +369,9 @@ public Message popWithMsgIdHelper(String messageId, String targetShard) { ZAddParams zParams = ZAddParams.zAddParams().nx(); - try { - long exists = nonQuorumConn.zrank(queueShardName, messageId); - // If an exception wasn't thrown, the element has to exist. - assert(exists >= 0); - } catch (NullPointerException e) { - // If we get a NPE, that means "messageId" does not exist in the sorted set. + Long exists = nonQuorumConn.zrank(queueShardName, messageId); + // If we get back a null type, then the element doesn't exist. + if (exists == null) { if (logger.isDebugEnabled()) { logger.debug("Cannot find the message with ID {}", messageId); } From d84fd5c541e3f1bcae794d04db9448c3d86e8149 Mon Sep 17 00:00:00 2001 From: Sailesh Mukil Date: Tue, 8 Oct 2019 14:11:56 -0400 Subject: [PATCH 64/91] Move debug messages from DEBUG to WARN Some log messages should be showed as warnings and not as DEBUG messages. --- .../dyno/queues/redis/RedisDynoQueue.java | 38 ++++++------------- 1 file changed, 12 insertions(+), 26 deletions(-) diff --git a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/RedisDynoQueue.java b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/RedisDynoQueue.java index ad54b9a..f71e0ac 100644 --- a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/RedisDynoQueue.java +++ b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/RedisDynoQueue.java @@ -372,36 +372,28 @@ public Message popWithMsgIdHelper(String messageId, String targetShard) { Long exists = nonQuorumConn.zrank(queueShardName, messageId); // If we get back a null type, then the element doesn't exist. if (exists == null) { - if (logger.isDebugEnabled()) { - logger.debug("Cannot find the message with ID {}", messageId); - } + logger.warn("Cannot find the message with ID {}", messageId); monitor.misses.increment(); return null; } String json = quorumConn.hget(messageStoreKey, messageId); if (json == null) { - if (logger.isDebugEnabled()) { - logger.debug("Cannot get the message payload for {}", messageId); - } + logger.warn("Cannot get the message payload for {}", messageId); monitor.misses.increment(); return null; } long added = quorumConn.zadd(unackShardName, unackScore, messageId, zParams); if (added == 0) { - if (logger.isDebugEnabled()) { - logger.debug("cannot add {} to the unack shard {}", messageId, unackShardName); - } + logger.warn("cannot add {} to the unack shard {}", messageId, unackShardName); monitor.misses.increment(); return null; } long removed = quorumConn.zrem(queueShardName, messageId); if (removed == 0) { - if (logger.isDebugEnabled()) { - logger.debug("cannot remove {} from the queue shard ", queueName, messageId); - } + logger.warn("cannot remove {} from the queue shard ", queueName, messageId); monitor.misses.increment(); return null; } @@ -541,27 +533,21 @@ private List _pop(String shard, int messageCount, long added = quorumConn.zadd(unackShardName, unackScore, msgId, zParams); if(added == 0){ - if (logger.isDebugEnabled()) { - logger.debug("cannot add {} to the unack shard {}", msgId, unackShardName); - } + logger.warn("cannot add {} to the unack shard {}", msgId, unackShardName); monitor.misses.increment(); continue; } long removed = quorumConn.zrem(queueShardName, msgId); if (removed == 0) { - if (logger.isDebugEnabled()) { - logger.debug("cannot remove {} from the queue shard {}", msgId, queueShardName); - } + logger.warn("cannot remove {} from the queue shard {}", msgId, queueShardName); monitor.misses.increment(); continue; } String json = quorumConn.hget(messageStoreKey, msgId); if (json == null) { - if (logger.isDebugEnabled()) { - logger.debug("Cannot get the message payload for {}", msgId); - } + logger.warn("Cannot get the message payload for {}", msgId); monitor.misses.increment(); continue; } @@ -810,9 +796,7 @@ public Message get(String messageId) { return execute("get", messageStoreKey, () -> { String json = quorumConn.hget(messageStoreKey, messageId); if (json == null) { - if (logger.isDebugEnabled()) { - logger.debug("Cannot get the message payload " + messageId); - } + logger.warn("Cannot get the message payload " + messageId); return null; } @@ -906,7 +890,7 @@ public void processUnacks() { Set unacks = nonQuorumConn.zrangeByScoreWithScores(unackShardName, 0, now, 0, batchSize); if (unacks.size() > 0) { - logger.info("processUnacks: Adding " + unacks.size() + " messages back to shard of queue: " + unackShardName); + logger.info("processUnacks: Attempting to add " + unacks.size() + " messages back to shard of queue: " + unackShardName); } for (Tuple unack : unacks) { @@ -926,7 +910,9 @@ public void processUnacks() { if (added_back > 0 && removed_from_unack > 0) ++num_moved_back; } - logger.info("processUnacks: Moved back " + num_moved_back + " items. Got rid of " + num_stale + " stale items."); + if (num_moved_back > 0 || num_stale > 0) { + logger.info("processUnacks: Moved back " + num_moved_back + " items. Got rid of " + num_stale + " stale items."); + } return null; }); From 2218bb3d46eae744b7c41526cb4ff3bfc07db561 Mon Sep 17 00:00:00 2001 From: Sailesh Mukil Date: Fri, 11 Oct 2019 14:49:06 -0400 Subject: [PATCH 65/91] popWithMsgId() and unsafePopWithMsgId() should report metrics --- .../dyno/queues/redis/RedisDynoQueue.java | 73 ++++++++++--------- 1 file changed, 40 insertions(+), 33 deletions(-) diff --git a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/RedisDynoQueue.java b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/RedisDynoQueue.java index f71e0ac..c5775e7 100644 --- a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/RedisDynoQueue.java +++ b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/RedisDynoQueue.java @@ -362,45 +362,52 @@ public Message unsafePopWithMsgIdAllShards(String messageId) { public Message popWithMsgIdHelper(String messageId, String targetShard) { - return execute("popWithMsgId", targetShard, () -> { - String queueShardName = getQueueShardKey(queueName, targetShard); - double unackScore = Long.valueOf(clock.millis() + unackTime).doubleValue(); - String unackShardName = getUnackKey(queueName, targetShard); + Stopwatch sw = monitor.start(monitor.pop, 1); - ZAddParams zParams = ZAddParams.zAddParams().nx(); + try { + return execute("popWithMsgId", targetShard, () -> { - Long exists = nonQuorumConn.zrank(queueShardName, messageId); - // If we get back a null type, then the element doesn't exist. - if (exists == null) { - logger.warn("Cannot find the message with ID {}", messageId); - monitor.misses.increment(); - return null; - } + String queueShardName = getQueueShardKey(queueName, targetShard); + double unackScore = Long.valueOf(clock.millis() + unackTime).doubleValue(); + String unackShardName = getUnackKey(queueName, targetShard); - String json = quorumConn.hget(messageStoreKey, messageId); - if (json == null) { - logger.warn("Cannot get the message payload for {}", messageId); - monitor.misses.increment(); - return null; - } + ZAddParams zParams = ZAddParams.zAddParams().nx(); - long added = quorumConn.zadd(unackShardName, unackScore, messageId, zParams); - if (added == 0) { - logger.warn("cannot add {} to the unack shard {}", messageId, unackShardName); - monitor.misses.increment(); - return null; - } + Long exists = nonQuorumConn.zrank(queueShardName, messageId); + // If we get back a null type, then the element doesn't exist. + if (exists == null) { + logger.warn("Cannot find the message with ID {}", messageId); + monitor.misses.increment(); + return null; + } - long removed = quorumConn.zrem(queueShardName, messageId); - if (removed == 0) { - logger.warn("cannot remove {} from the queue shard ", queueName, messageId); - monitor.misses.increment(); - return null; - } + String json = quorumConn.hget(messageStoreKey, messageId); + if (json == null) { + logger.warn("Cannot get the message payload for {}", messageId); + monitor.misses.increment(); + return null; + } - Message msg = om.readValue(json, Message.class); - return msg; - }); + long added = quorumConn.zadd(unackShardName, unackScore, messageId, zParams); + if (added == 0) { + logger.warn("cannot add {} to the unack shard {}", messageId, unackShardName); + monitor.misses.increment(); + return null; + } + + long removed = quorumConn.zrem(queueShardName, messageId); + if (removed == 0) { + logger.warn("cannot remove {} from the queue shard ", queueName, messageId); + monitor.misses.increment(); + return null; + } + + Message msg = om.readValue(json, Message.class); + return msg; + }); + } finally { + sw.stop(); + } } From 9ccba780b29711c1b189a23c8eae3b6903ec06c6 Mon Sep 17 00:00:00 2001 From: Sailesh Mukil Date: Mon, 21 Oct 2019 17:39:08 -0400 Subject: [PATCH 66/91] Pin to dyno-1.7.2 --- dyno-queues-core/build.gradle | 2 +- dyno-queues-redis/build.gradle | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/dyno-queues-core/build.gradle b/dyno-queues-core/build.gradle index b95efc8..94fd0a8 100644 --- a/dyno-queues-core/build.gradle +++ b/dyno-queues-core/build.gradle @@ -1,5 +1,5 @@ dependencies { - compile 'com.netflix.dyno:dyno-core:1.7.1' + compile 'com.netflix.dyno:dyno-core:1.7.2' testCompile "junit:junit:4.11" } diff --git a/dyno-queues-redis/build.gradle b/dyno-queues-redis/build.gradle index 682f43d..dc0ea61 100644 --- a/dyno-queues-redis/build.gradle +++ b/dyno-queues-redis/build.gradle @@ -4,9 +4,9 @@ dependencies { compile "com.google.inject:guice:3.0" - compile 'com.netflix.dyno:dyno-core:1.7.1' - compile 'com.netflix.dyno:dyno-jedis:1.7.1' - compile 'com.netflix.dyno:dyno-demo:1.7.1' + compile 'com.netflix.dyno:dyno-core:1.7.2' + compile 'com.netflix.dyno:dyno-jedis:1.7.2' + compile 'com.netflix.dyno:dyno-demo:1.7.2' compile 'com.netflix.archaius:archaius-core:0.7.5' compile 'com.netflix.servo:servo-core:0.12.17' From 8824bfdb5af7f056ad2f3a3719c36424d5815b9c Mon Sep 17 00:00:00 2001 From: Sailesh Mukil Date: Tue, 22 Oct 2019 13:19:07 -0400 Subject: [PATCH 67/91] Pin to dyno-1.7.2-rc2 --- dyno-queues-core/build.gradle | 2 +- dyno-queues-redis/build.gradle | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/dyno-queues-core/build.gradle b/dyno-queues-core/build.gradle index 94fd0a8..6071082 100644 --- a/dyno-queues-core/build.gradle +++ b/dyno-queues-core/build.gradle @@ -1,5 +1,5 @@ dependencies { - compile 'com.netflix.dyno:dyno-core:1.7.2' + compile 'com.netflix.dyno:dyno-core:1.7.2-rc2' testCompile "junit:junit:4.11" } diff --git a/dyno-queues-redis/build.gradle b/dyno-queues-redis/build.gradle index dc0ea61..d83a66b 100644 --- a/dyno-queues-redis/build.gradle +++ b/dyno-queues-redis/build.gradle @@ -4,9 +4,9 @@ dependencies { compile "com.google.inject:guice:3.0" - compile 'com.netflix.dyno:dyno-core:1.7.2' - compile 'com.netflix.dyno:dyno-jedis:1.7.2' - compile 'com.netflix.dyno:dyno-demo:1.7.2' + compile 'com.netflix.dyno:dyno-core:1.7.2-rc2' + compile 'com.netflix.dyno:dyno-jedis:1.7.2-rc2' + compile 'com.netflix.dyno:dyno-demo:1.7.2-rc2' compile 'com.netflix.archaius:archaius-core:0.7.5' compile 'com.netflix.servo:servo-core:0.12.17' From a6f5592499c7280c8a3ac8fb8832977303214b6f Mon Sep 17 00:00:00 2001 From: Sailesh Mukil Date: Wed, 23 Oct 2019 12:38:17 -0400 Subject: [PATCH 68/91] unsafePopWithMsgId() shouldn't spam the logs unsafePopWithMsgId() checks all 3 shards for a msgId, but only one of them can have it. So make sure that the checks against the other 2 shards do not spam the logs with "Cannot find message with ID ...". --- .../dyno/queues/redis/RedisDynoQueue.java | 22 +++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/RedisDynoQueue.java b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/RedisDynoQueue.java index c5775e7..75782e1 100644 --- a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/RedisDynoQueue.java +++ b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/RedisDynoQueue.java @@ -348,19 +348,27 @@ public List pop(int messageCount, int wait, TimeUnit unit) { @Override public Message popWithMsgId(String messageId) { - return popWithMsgIdHelper(messageId, shardName); + return popWithMsgIdHelper(messageId, shardName, true); } @Override public Message unsafePopWithMsgIdAllShards(String messageId) { + int numShards = allShards.size(); for (String shard : allShards) { - Message msg = popWithMsgIdHelper(messageId, shard); + boolean warnIfNotExists = false; + + // Only one of the shards will have the message, so we don't want the check in the other 2 shards + // to spam the logs. So make sure only the last shard emits a warning log which means that none of the + // shards have 'messageId'. + if (--numShards == 0) warnIfNotExists = true; + + Message msg = popWithMsgIdHelper(messageId, shard, warnIfNotExists); if (msg != null) return msg; } return null; } - public Message popWithMsgIdHelper(String messageId, String targetShard) { + public Message popWithMsgIdHelper(String messageId, String targetShard, boolean warnIfNotExists) { Stopwatch sw = monitor.start(monitor.pop, 1); @@ -376,7 +384,13 @@ public Message popWithMsgIdHelper(String messageId, String targetShard) { Long exists = nonQuorumConn.zrank(queueShardName, messageId); // If we get back a null type, then the element doesn't exist. if (exists == null) { - logger.warn("Cannot find the message with ID {}", messageId); + // We only have a 'warnIfNotExists' check for this call since not all messages are present in + // all shards. So we want to avoid a log spam. If any of the following calls return 'null' or '0', + // we may have hit an inconsistency (because it's in the queue, but other calls have failed), + // so make sure to log those. + if (warnIfNotExists) { + logger.warn("Cannot find the message with ID {}", messageId); + } monitor.misses.increment(); return null; } From 749596c0973b6b6e20719829fbbf2ef71dfe926c Mon Sep 17 00:00:00 2001 From: Sailesh Mukil Date: Tue, 29 Oct 2019 13:35:08 -0700 Subject: [PATCH 69/91] Add popMsgWithPredicate() API This API looks for a value with a matching predicate in the hashmap and pops the first message ID that matches if it does find one. Also added a atomicPopWithMsgIdHelper() helper function to do a pop in one round trip. This should only be used if the ring size of the underlying Dynomite cluster is 1. Testing: Tested with DynoQueueDemo --- .../com/netflix/dyno/queues/DynoQueue.java | 11 +++ .../dyno/queues/demo/DynoQueueDemo.java | 6 +- .../dyno/queues/redis/RedisDynoQueue.java | 95 +++++++++++++++++++ .../dyno/queues/redis/v2/MultiRedisQueue.java | 5 + .../queues/redis/v2/RedisPipelineQueue.java | 5 + 5 files changed, 121 insertions(+), 1 deletion(-) diff --git a/dyno-queues-core/src/main/java/com/netflix/dyno/queues/DynoQueue.java b/dyno-queues-core/src/main/java/com/netflix/dyno/queues/DynoQueue.java index db62841..343b9e5 100644 --- a/dyno-queues-core/src/main/java/com/netflix/dyno/queues/DynoQueue.java +++ b/dyno-queues-core/src/main/java/com/netflix/dyno/queues/DynoQueue.java @@ -168,6 +168,17 @@ public interface DynoQueue extends Closeable { public String getMsgWithPredicate(String predicate); public String getMsgWithPredicate(String predicate, boolean localShardOnly); + /** + * Same as getMsgWithPredicate(), but it also pops the item if found. + * + * @param predicate The predicate to check against. + * @param localShardOnly If this is true, it will only check if the message exists in the local shard as opposed to + * all shards. Note that this will only work if the Dynomite cluster ring size is 1 (i.e. one + * instance per AZ). + * @return + */ + public Message popMsgWithPredicate(String predicate, boolean localShardOnly); + /** * * @param messageId message to be retrieved. diff --git a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/demo/DynoQueueDemo.java b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/demo/DynoQueueDemo.java index 72f8308..ea46ba0 100644 --- a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/demo/DynoQueueDemo.java +++ b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/demo/DynoQueueDemo.java @@ -72,6 +72,7 @@ public static void main(String[] args) throws IOException { } } + private void runSimpleV1Demo(DynoJedisClient dyno) throws IOException { String region = System.getProperty("LOCAL_DATACENTER"); String localRack = System.getProperty("LOCAL_RACK"); @@ -122,10 +123,13 @@ private void runSimpleV1Demo(DynoJedisClient dyno) throws IOException { logger.info("Get MSG ID that contains 'searchable' in the queue -> " + V1Queue.getMsgWithPredicate("searchable pay*", true)); logger.info("Get MSG ID that contains '3' in the queue -> " + V1Queue.getMsgWithPredicate("3", true)); + Message poppedWithPredicate = V1Queue.popMsgWithPredicate("searchable pay*", false); + V1Queue.ack(poppedWithPredicate.getId()); + List specific_pops = new ArrayList<>(); // We'd only be able to pop from the local shard with popWithMsgId(), so try to pop the first payload ID we see in the local shard. // Until then pop all messages not in the local shard with unsafePopWithMsgIdAllShards(). - for (int i = 0; i < payloads.size(); ++i) { + for (int i = 1; i < payloads.size(); ++i) { Message popWithMsgId = V1Queue.popWithMsgId(payloads.get(i).getId()); if (popWithMsgId != null) { specific_pops.add(popWithMsgId); diff --git a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/RedisDynoQueue.java b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/RedisDynoQueue.java index 75782e1..0032560 100644 --- a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/RedisDynoQueue.java +++ b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/RedisDynoQueue.java @@ -31,6 +31,7 @@ import redis.clients.jedis.params.ZAddParams; import java.io.IOException; +import java.text.NumberFormat; import java.time.Clock; import java.util.*; import java.util.concurrent.ConcurrentLinkedQueue; @@ -807,6 +808,100 @@ public String getMsgWithPredicate(String predicate, boolean localShardOnly) { }); } + /** + * + * Similar to popWithMsgId() but completes all the operations in one round trip. + * + * NOTE: This function assumes that the ring size in the cluster is 1. DO NOT use for APIs that support a ring + * size larger than 1. + * + * @param messageId + * @param localShardOnly + * @return + */ + private String atomicPopWithMsgIdHelper(String messageId, boolean localShardOnly) { + + double now = Long.valueOf(clock.millis() + 1).doubleValue(); + double unackScore = Long.valueOf(clock.millis() + unackTime).doubleValue(); + + // The script requires the scores as whole numbers + NumberFormat fmt = NumberFormat.getIntegerInstance(); + fmt.setGroupingUsed(false); + String nowScoreString = fmt.format(now); + String unackScoreString = fmt.format(unackScore); + + String atomicPopScript = "local hkey=KEYS[1]\n" + + "local message_id=ARGV[1]\n" + + "local num_shards=ARGV[2]\n" + + "local peek_until=ARGV[3]\n" + + "local unack_score=ARGV[4]\n" + + "for i=0,num_shards-1 do\n" + + " local queue_shard_name=ARGV[(i*2)+5]\n" + + " local unack_shard_name=ARGV[(i*2)+5+1]\n" + + " local exists = redis.call('zscore', queue_shard_name, message_id)\n" + + " if (exists) then\n" + + " if (exists <= peek_until) then\n" + + " local value = redis.call('hget', hkey, message_id)\n" + + " if (value) then\n" + + " local zadd_ret = redis.call('zadd', unack_shard_name, 'NX', unack_score, message_id )\n" + + " if (zadd_ret) then\n" + + " redis.call('zrem', queue_shard_name, message_id)\n" + + " return value\n" + + " end\n" + + " end\n" + + " end\n" + + " end\n" + + "end\n" + + "return nil"; + + String retval; + if (localShardOnly) { + String unackShardName = getUnackKey(queueName, shardName); + + retval = (String) ((DynoJedisClient) quorumConn).eval(atomicPopScript, Collections.singletonList(messageStoreKey), + ImmutableList.of(messageId, Integer.toString(1), nowScoreString, + unackScoreString, localQueueShard, unackShardName)); + } else { + int numShards = allShards.size(); + ImmutableList.Builder builder = ImmutableList.builder(); + builder.add(messageId); + builder.add(Integer.toString(numShards)); + builder.add(nowScoreString); + builder.add(unackScoreString); + + List arguments = Arrays.asList(messageId, Integer.toString(numShards), nowScoreString, + unackScoreString); + for (String shard : allShards) { + String queueShard = getQueueShardKey(queueName, shard); + String unackShardName = getUnackKey(queueName, shard); + builder.add(queueShard); + builder.add(unackShardName); + } + retval = (String) ((DynoJedisClient) quorumConn).eval(atomicPopScript, Collections.singletonList(messageStoreKey), builder.build()); + } + + return retval; + } + + @Override + public Message popMsgWithPredicate(String predicate, boolean localShardOnly) { + Stopwatch sw = monitor.get.start(); + + try { + String messageId = getMsgWithPredicate(predicate, localShardOnly); + if (messageId == null) { + return null; + } + String payload = execute("atomicPop", messageStoreKey, () -> atomicPopWithMsgIdHelper(messageId, localShardOnly)); + + return new Message(messageId, payload); + + } finally { + sw.stop(); + } + + } + @Override public Message get(String messageId) { diff --git a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/v2/MultiRedisQueue.java b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/v2/MultiRedisQueue.java index fa0618e..c15756f 100644 --- a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/v2/MultiRedisQueue.java +++ b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/v2/MultiRedisQueue.java @@ -178,6 +178,11 @@ public String getMsgWithPredicate(String predicate, boolean localShardOnly) { throw new UnsupportedOperationException(); } + @Override + public Message popMsgWithPredicate(String predicate, boolean localShardOnly) { + throw new UnsupportedOperationException(); + } + @Override public Message get(String messageId) { for (DynoQueue q : queues.values()) { diff --git a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/v2/RedisPipelineQueue.java b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/v2/RedisPipelineQueue.java index fc4138c..5deeb9c 100644 --- a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/v2/RedisPipelineQueue.java +++ b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/v2/RedisPipelineQueue.java @@ -492,6 +492,11 @@ public String getMsgWithPredicate(String predicate, boolean localShardOnly) { throw new UnsupportedOperationException(); } + @Override + public Message popMsgWithPredicate(String predicate, boolean localShardOnly) { + throw new UnsupportedOperationException(); + } + @Override public Message get(String messageId) { From d4305d402747939ce0d018bb9db6127b957b6028 Mon Sep 17 00:00:00 2001 From: Sailesh Mukil Date: Mon, 11 Nov 2019 06:53:46 -0800 Subject: [PATCH 70/91] Add a atomic bulkPop() API Using queues with DC_EACH_SAFE_QUORUM is quite expensive. The bulk pop operation is meant to pop more within a single round trip. --- .../com/netflix/dyno/queues/DynoQueue.java | 2 + .../dyno/queues/redis/RedisDynoQueue.java | 94 +++++++++++++++++++ .../dyno/queues/redis/v2/MultiRedisQueue.java | 5 + .../queues/redis/v2/RedisPipelineQueue.java | 5 + 4 files changed, 106 insertions(+) diff --git a/dyno-queues-core/src/main/java/com/netflix/dyno/queues/DynoQueue.java b/dyno-queues-core/src/main/java/com/netflix/dyno/queues/DynoQueue.java index 343b9e5..2c3c7fd 100644 --- a/dyno-queues-core/src/main/java/com/netflix/dyno/queues/DynoQueue.java +++ b/dyno-queues-core/src/main/java/com/netflix/dyno/queues/DynoQueue.java @@ -186,6 +186,8 @@ public interface DynoQueue extends Closeable { */ public Message get(String messageId); + public List bulkPop(int messageCount, int wait, TimeUnit unit); + /** * * @return Size of the queue. diff --git a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/RedisDynoQueue.java b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/RedisDynoQueue.java index 0032560..e9054ba 100644 --- a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/RedisDynoQueue.java +++ b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/RedisDynoQueue.java @@ -808,6 +808,100 @@ public String getMsgWithPredicate(String predicate, boolean localShardOnly) { }); } + @Override + public List bulkPop(int messageCount, int wait, TimeUnit unit) { + + if (messageCount < 1) { + return Collections.emptyList(); + } + + Stopwatch sw = monitor.start(monitor.pop, messageCount); + try { + long start = clock.millis(); + long waitFor = unit.toMillis(wait); + numIdsToPrefetch.addAndGet(messageCount); + + prefetchIds(); + while (prefetchedIds.size() < messageCount && ((clock.millis() - start) < waitFor)) { + Uninterruptibles.sleepUninterruptibly(200, TimeUnit.MILLISECONDS); + prefetchIds(); + } + return atomicBulkPopHelper(shardName, messageCount, prefetchedIds); + + } catch (Exception e) { + throw new RuntimeException(e); + } finally { + sw.stop(); + } + + } + + private List atomicBulkPopHelper(String shard, int messageCount, + ConcurrentLinkedQueue prefetchedIdQueue) { + + double now = Long.valueOf(clock.millis() + 1).doubleValue(); + double unackScore = Long.valueOf(clock.millis() + unackTime).doubleValue(); + + // The script requires the scores as whole numbers + NumberFormat fmt = NumberFormat.getIntegerInstance(); + fmt.setGroupingUsed(false); + String nowScoreString = fmt.format(now); + String unackScoreString = fmt.format(unackScore); + + List messageIds = new ArrayList<>(); + for (int i = 0; i < messageCount; ++i) { + messageIds.add(prefetchedIdQueue.poll()); + } + + String atomicBulkPopScript="local hkey=KEYS[1]\n" + + "local num_msgs=ARGV[1]\n" + + "local peek_until=ARGV[2]\n" + + "local unack_score=ARGV[3]\n" + + "local queue_shard_name=ARGV[4]\n" + + "local unack_shard_name=ARGV[5]\n" + + "local msg_start_idx = 6\n" + + "local idx = 1\n" + + "local return_vals={}\n" + + "for i=0,num_msgs-1 do\n" + + " local message_id=ARGV[msg_start_idx + i]\n" + + " local exists = redis.call('zscore', queue_shard_name, message_id)\n" + + " if (exists) then\n" + + " if (exists <=peek_until) then\n" + + " local value = redis.call('hget', hkey, message_id)\n" + + " if (value) then\n" + + " local zadd_ret = redis.call('zadd', unack_shard_name, 'NX', unack_score, message_id)\n" + + " if (zadd_ret) then\n" + + " redis.call('zrem', queue_shard_name, message_id)\n" + + " return_vals[idx]=value\n" + + " idx=idx+1\n" + + " end\n" + + " end\n" + + " end\n" + + " else\n" + + " return {}\n" + + " end\n" + + "end\n" + + "return return_vals"; + + String unackShardName = getUnackKey(queueName, shardName); + + ImmutableList.Builder builder = ImmutableList.builder(); + builder.add(Integer.toString(messageCount)); + builder.add(nowScoreString); + builder.add(unackScoreString); + builder.add(localQueueShard); + builder.add(unackShardName); + for (int i = 0; i < messageCount; ++i) { + builder.add(messageIds.get(i)); + } + + List payloads; + // Cast from 'JedisCommands' to 'DynoJedisClient' here since the former does not expose 'eval()'. + payloads = (List) ((DynoJedisClient) quorumConn).eval(atomicBulkPopScript, + Collections.singletonList(messageStoreKey), builder.build()); + + return payloads; + } /** * * Similar to popWithMsgId() but completes all the operations in one round trip. diff --git a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/v2/MultiRedisQueue.java b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/v2/MultiRedisQueue.java index c15756f..94b69f9 100644 --- a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/v2/MultiRedisQueue.java +++ b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/v2/MultiRedisQueue.java @@ -183,6 +183,11 @@ public Message popMsgWithPredicate(String predicate, boolean localShardOnly) { throw new UnsupportedOperationException(); } + @Override + public List bulkPop(int messageCount, int wait, TimeUnit unit) { + throw new UnsupportedOperationException(); + } + @Override public Message get(String messageId) { for (DynoQueue q : queues.values()) { diff --git a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/v2/RedisPipelineQueue.java b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/v2/RedisPipelineQueue.java index 5deeb9c..1c3b466 100644 --- a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/v2/RedisPipelineQueue.java +++ b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/v2/RedisPipelineQueue.java @@ -497,6 +497,11 @@ public Message popMsgWithPredicate(String predicate, boolean localShardOnly) { throw new UnsupportedOperationException(); } + @Override + public List bulkPop(int messageCount, int wait, TimeUnit unit) { + throw new UnsupportedOperationException(); + } + @Override public Message get(String messageId) { From 5089a7c6cc0c91b973a13c0882b33c3757c492e9 Mon Sep 17 00:00:00 2001 From: Sailesh Mukil Date: Thu, 14 Nov 2019 08:50:26 -0800 Subject: [PATCH 71/91] Add unsafeBulkPop() and localGet() unsafeBulkPop() allows bulk popping from all shards. loaclGet() does a get() with a non quorum connection. TODO: unsafeBulkPop() will return nil if messageCount > size(). Fix this. TODO 2: Do code cleanup. --- .../com/netflix/dyno/queues/DynoQueue.java | 9 + .../dyno/queues/redis/RedisDynoQueue.java | 159 ++++++++++++++++-- .../dyno/queues/redis/v2/MultiRedisQueue.java | 10 ++ .../queues/redis/v2/RedisPipelineQueue.java | 10 ++ 4 files changed, 170 insertions(+), 18 deletions(-) diff --git a/dyno-queues-core/src/main/java/com/netflix/dyno/queues/DynoQueue.java b/dyno-queues-core/src/main/java/com/netflix/dyno/queues/DynoQueue.java index 2c3c7fd..30b8591 100644 --- a/dyno-queues-core/src/main/java/com/netflix/dyno/queues/DynoQueue.java +++ b/dyno-queues-core/src/main/java/com/netflix/dyno/queues/DynoQueue.java @@ -186,7 +186,16 @@ public interface DynoQueue extends Closeable { */ public Message get(String messageId); + /** + * + * Same as get(), but uses the non quorum connection. + * @param messageId message to be retrieved. + * @return Retrieves the message stored in the queue by the messageId. Null if not found. + */ + public Message localGet(String messageId); + public List bulkPop(int messageCount, int wait, TimeUnit unit); + public List unsafeBulkPop(int messageCount, int wait, TimeUnit unit); /** * diff --git a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/RedisDynoQueue.java b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/RedisDynoQueue.java index e9054ba..d81a648 100644 --- a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/RedisDynoQueue.java +++ b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/RedisDynoQueue.java @@ -519,6 +519,7 @@ private int doPrefetchIdsHelper(String queueShardName, AtomicInteger prefetchCou // Attempt to peek up to 'numToPrefetch' message Ids. Set ids = doPeekIdsFromShardHelper(queueShardName, prefetchFromTs, 0, numToPrefetch); + // TODO: Check for duplicates. // Store prefetched IDs in a queue. prefetchedIdQueue.addAll(ids); @@ -826,7 +827,8 @@ public List bulkPop(int messageCount, int wait, TimeUnit unit) { Uninterruptibles.sleepUninterruptibly(200, TimeUnit.MILLISECONDS); prefetchIds(); } - return atomicBulkPopHelper(shardName, messageCount, prefetchedIds); + int numToPop = (prefetchedIds.size() > messageCount) ? messageCount : prefetchedIds.size(); + return atomicBulkPopHelper(numToPop, prefetchedIds, true); } catch (Exception e) { throw new RuntimeException(e); @@ -836,8 +838,47 @@ public List bulkPop(int messageCount, int wait, TimeUnit unit) { } - private List atomicBulkPopHelper(String shard, int messageCount, - ConcurrentLinkedQueue prefetchedIdQueue) { + @Override + public List unsafeBulkPop(int messageCount, int wait, TimeUnit unit) { + if (messageCount < 1) { + return Collections.emptyList(); + } + + Stopwatch sw = monitor.start(monitor.pop, messageCount); + try { + long start = clock.millis(); + long waitFor = unit.toMillis(wait); + unsafeNumIdsToPrefetchAllShards.addAndGet(messageCount); + + prefetchIdsAllShards(); + while(unsafeGetNumPrefetchedIds() < messageCount && ((clock.millis() - start) < waitFor)) { + Uninterruptibles.sleepUninterruptibly(200, TimeUnit.MILLISECONDS); + prefetchIdsAllShards(); + } + + int numToPop = (unsafeGetNumPrefetchedIds() > messageCount) ? messageCount : unsafeGetNumPrefetchedIds(); + ConcurrentLinkedQueue messageIds = new ConcurrentLinkedQueue<>(); + int numPrefetched = 0; + for (String shard : allShards) { + String queueShardName = getQueueShardKey(queueName, shard); + int prefetchedIdsSize = unsafePrefetchedIdsAllShardsMap.get(queueShardName).size(); + for (int i = 0; i < prefetchedIdsSize; ++i) { + messageIds.add(unsafePrefetchedIdsAllShardsMap.get(queueShardName).poll()); + if (++numPrefetched == numToPop) break; + } + if (numPrefetched == numToPop) break; + } + return atomicBulkPopHelper(numToPop, messageIds, false); + } catch(Exception e) { + throw new RuntimeException(e); + } finally { + sw.stop(); + } + } + + // TODO: Do code cleanup/consolidation + private List atomicBulkPopHelper(int messageCount, + ConcurrentLinkedQueue prefetchedIdQueue, boolean localShardOnly) { double now = Long.valueOf(clock.millis() + 1).doubleValue(); double unackScore = Long.valueOf(clock.millis() + unackTime).doubleValue(); @@ -853,7 +894,7 @@ private List atomicBulkPopHelper(String shard, int messageCount, messageIds.add(prefetchedIdQueue.poll()); } - String atomicBulkPopScript="local hkey=KEYS[1]\n" + + String atomicBulkPopScriptLocalOnly="local hkey=KEYS[1]\n" + "local num_msgs=ARGV[1]\n" + "local peek_until=ARGV[2]\n" + "local unack_score=ARGV[3]\n" + @@ -883,25 +924,84 @@ private List atomicBulkPopHelper(String shard, int messageCount, "end\n" + "return return_vals"; - String unackShardName = getUnackKey(queueName, shardName); - - ImmutableList.Builder builder = ImmutableList.builder(); - builder.add(Integer.toString(messageCount)); - builder.add(nowScoreString); - builder.add(unackScoreString); - builder.add(localQueueShard); - builder.add(unackShardName); - for (int i = 0; i < messageCount; ++i) { - builder.add(messageIds.get(i)); - } + String atomicBulkPopScript="local hkey=KEYS[1]\n" + + "local num_msgs=ARGV[1]\n" + + "local num_shards=ARGV[2]\n" + + "local peek_until=ARGV[3]\n" + + "local unack_score=ARGV[4]\n" + + "local shard_start_idx = 5\n" + + "local msg_start_idx = 5 + (num_shards * 2)\n" + + "local out_idx = 1\n" + + "local return_vals={}\n" + + "for i=0,num_msgs-1 do\n" + + " local found_msg=false\n" + + " local message_id=ARGV[msg_start_idx + i]\n" + + " for j=0,num_shards-1 do\n" + + " local queue_shard_name=ARGV[shard_start_idx + (j*2)]\n" + + " local unack_shard_name=ARGV[shard_start_idx + (j*2) + 1]\n" + + " local exists = redis.call('zscore', queue_shard_name, message_id)\n" + + " if (exists) then\n" + + " found_msg=true\n" + + " if (exists <=peek_until) then\n" + + " local value = redis.call('hget', hkey, message_id)\n" + + " if (value) then\n" + + " local zadd_ret = redis.call('zadd', unack_shard_name, 'NX', unack_score, message_id)\n" + + " if (zadd_ret) then\n" + + " redis.call('zrem', queue_shard_name, message_id)\n" + + " return_vals[out_idx]=value\n" + + " out_idx=out_idx+1\n" + + " break\n" + + " end\n" + + " end\n" + + " end\n" + + " end\n" + + " end\n" + + " if (found_msg == false) then\n" + + " return {}\n" + + " end\n" + + "end\n" + + "return return_vals"; List payloads; - // Cast from 'JedisCommands' to 'DynoJedisClient' here since the former does not expose 'eval()'. - payloads = (List) ((DynoJedisClient) quorumConn).eval(atomicBulkPopScript, - Collections.singletonList(messageStoreKey), builder.build()); + if (localShardOnly) { + String unackShardName = getUnackKey(queueName, shardName); + + ImmutableList.Builder builder = ImmutableList.builder(); + builder.add(Integer.toString(messageCount)); + builder.add(nowScoreString); + builder.add(unackScoreString); + builder.add(localQueueShard); + builder.add(unackShardName); + for (int i = 0; i < messageCount; ++i) { + builder.add(messageIds.get(i)); + } + // Cast from 'JedisCommands' to 'DynoJedisClient' here since the former does not expose 'eval()'. + payloads = (List) ((DynoJedisClient) quorumConn).eval(atomicBulkPopScriptLocalOnly, + Collections.singletonList(messageStoreKey), builder.build()); + } else { + ImmutableList.Builder builder = ImmutableList.builder(); + builder.add(Integer.toString(messageCount)); + builder.add(Integer.toString(allShards.size())); + builder.add(nowScoreString); + builder.add(unackScoreString); + for (String shard : allShards) { + String queueShard = getQueueShardKey(queueName, shard); + String unackShardName = getUnackKey(queueName, shard); + builder.add(queueShard); + builder.add(unackShardName); + } + for (int i = 0; i < messageCount; ++i) { + builder.add(messageIds.get(i)); + } + + // Cast from 'JedisCommands' to 'DynoJedisClient' here since the former does not expose 'eval()'. + payloads = (List) ((DynoJedisClient) quorumConn).eval(atomicBulkPopScript, + Collections.singletonList(messageStoreKey), builder.build()); + } return payloads; } + /** * * Similar to popWithMsgId() but completes all the operations in one round trip. @@ -1019,6 +1119,29 @@ public Message get(String messageId) { } } + @Override + public Message localGet(String messageId) { + + Stopwatch sw = monitor.get.start(); + + try { + + return execute("localGet", messageStoreKey, () -> { + String json = nonQuorumConn.hget(messageStoreKey, messageId); + if (json == null) { + logger.warn("Cannot get the message payload " + messageId); + return null; + } + + Message msg = om.readValue(json, Message.class); + return msg; + }); + + } finally { + sw.stop(); + } + } + @Override public long size() { diff --git a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/v2/MultiRedisQueue.java b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/v2/MultiRedisQueue.java index 94b69f9..c94dc46 100644 --- a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/v2/MultiRedisQueue.java +++ b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/v2/MultiRedisQueue.java @@ -188,6 +188,11 @@ public List bulkPop(int messageCount, int wait, TimeUnit unit) { throw new UnsupportedOperationException(); } + @Override + public List unsafeBulkPop(int messageCount, int wait, TimeUnit unit) { + throw new UnsupportedOperationException(); + } + @Override public Message get(String messageId) { for (DynoQueue q : queues.values()) { @@ -199,6 +204,11 @@ public Message get(String messageId) { return null; } + @Override + public Message localGet(String messageId) { + throw new UnsupportedOperationException(); + } + @Override public long size() { long size = 0; diff --git a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/v2/RedisPipelineQueue.java b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/v2/RedisPipelineQueue.java index 1c3b466..903c426 100644 --- a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/v2/RedisPipelineQueue.java +++ b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/v2/RedisPipelineQueue.java @@ -502,6 +502,11 @@ public List bulkPop(int messageCount, int wait, TimeUnit unit) { throw new UnsupportedOperationException(); } + @Override + public List unsafeBulkPop(int messageCount, int wait, TimeUnit unit) { + throw new UnsupportedOperationException(); + } + @Override public Message get(String messageId) { @@ -528,6 +533,11 @@ public Message get(String messageId) { } } + @Override + public Message localGet(String messageId) { + throw new UnsupportedOperationException(); + } + @Override public long size() { From 6d8a91a4ba94762fec161fef6aad71b7867c6fe0 Mon Sep 17 00:00:00 2001 From: Sailesh Mukil Date: Thu, 14 Nov 2019 18:07:59 -0800 Subject: [PATCH 72/91] atomicBulkPopHelper() should cast to Message before returning --- .../dyno/queues/redis/RedisDynoQueue.java | 21 +++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/RedisDynoQueue.java b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/RedisDynoQueue.java index d81a648..4c651d4 100644 --- a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/RedisDynoQueue.java +++ b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/RedisDynoQueue.java @@ -878,7 +878,7 @@ public List unsafeBulkPop(int messageCount, int wait, TimeUnit unit) { // TODO: Do code cleanup/consolidation private List atomicBulkPopHelper(int messageCount, - ConcurrentLinkedQueue prefetchedIdQueue, boolean localShardOnly) { + ConcurrentLinkedQueue prefetchedIdQueue, boolean localShardOnly) throws IOException { double now = Long.valueOf(clock.millis() + 1).doubleValue(); double unackScore = Long.valueOf(clock.millis() + unackTime).doubleValue(); @@ -962,7 +962,7 @@ private List atomicBulkPopHelper(int messageCount, "end\n" + "return return_vals"; - List payloads; + List payloads = new ArrayList<>(); if (localShardOnly) { String unackShardName = getUnackKey(queueName, shardName); @@ -975,9 +975,16 @@ private List atomicBulkPopHelper(int messageCount, for (int i = 0; i < messageCount; ++i) { builder.add(messageIds.get(i)); } + + List jsonPayloads; // Cast from 'JedisCommands' to 'DynoJedisClient' here since the former does not expose 'eval()'. - payloads = (List) ((DynoJedisClient) quorumConn).eval(atomicBulkPopScriptLocalOnly, + jsonPayloads = (List) ((DynoJedisClient) quorumConn).eval(atomicBulkPopScriptLocalOnly, Collections.singletonList(messageStoreKey), builder.build()); + + for (String p : jsonPayloads) { + Message msg = om.readValue(p, Message.class); + payloads.add(msg); + } } else { ImmutableList.Builder builder = ImmutableList.builder(); builder.add(Integer.toString(messageCount)); @@ -994,9 +1001,15 @@ private List atomicBulkPopHelper(int messageCount, builder.add(messageIds.get(i)); } + List jsonPayloads; // Cast from 'JedisCommands' to 'DynoJedisClient' here since the former does not expose 'eval()'. - payloads = (List) ((DynoJedisClient) quorumConn).eval(atomicBulkPopScript, + jsonPayloads = (List) ((DynoJedisClient) quorumConn).eval(atomicBulkPopScript, Collections.singletonList(messageStoreKey), builder.build()); + + for (String p : jsonPayloads) { + Message msg = om.readValue(p, Message.class); + payloads.add(msg); + } } return payloads; From 84325c0172bee3dd48576bfb4abaada4a8329fb4 Mon Sep 17 00:00:00 2001 From: Sailesh Mukil Date: Fri, 15 Nov 2019 11:53:40 -0800 Subject: [PATCH 73/91] Make popWithMsgPredicate() obey queue priority Previously popWithMsgPredicate() would pop the first item it found in the hashmap that matches the given predicate. With this patch, it will obey the queueing priority. --- .../com/netflix/dyno/queues/DynoQueue.java | 4 +- .../dyno/queues/redis/RedisDynoQueue.java | 148 +++++++++++++++--- 2 files changed, 133 insertions(+), 19 deletions(-) diff --git a/dyno-queues-core/src/main/java/com/netflix/dyno/queues/DynoQueue.java b/dyno-queues-core/src/main/java/com/netflix/dyno/queues/DynoQueue.java index 30b8591..ce448e7 100644 --- a/dyno-queues-core/src/main/java/com/netflix/dyno/queues/DynoQueue.java +++ b/dyno-queues-core/src/main/java/com/netflix/dyno/queues/DynoQueue.java @@ -169,7 +169,9 @@ public interface DynoQueue extends Closeable { public String getMsgWithPredicate(String predicate, boolean localShardOnly); /** - * Same as getMsgWithPredicate(), but it also pops the item if found. + * Pops the message with the highest priority that matches 'predicate'. + * + * Note: Can be slow for large queues. * * @param predicate The predicate to check against. * @param localShardOnly If this is true, it will only check if the message exists in the local shard as opposed to diff --git a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/RedisDynoQueue.java b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/RedisDynoQueue.java index 4c651d4..c28e29c 100644 --- a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/RedisDynoQueue.java +++ b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/RedisDynoQueue.java @@ -809,6 +809,136 @@ public String getMsgWithPredicate(String predicate, boolean localShardOnly) { }); } + private Message popMsgWithPredicateObeyPriority(String predicate, boolean localShardOnly) { + String popPredicateObeyPriority = "local hkey=KEYS[1]\n" + + "local predicate=ARGV[1]\n" + + "local num_shards=ARGV[2]\n" + + "local peek_until=tonumber(ARGV[3])\n" + + "local unack_score=tonumber(ARGV[4])\n" + + "\n" + + "local shard_names={}\n" + + "local unack_names={}\n" + + "local shard_lengths={}\n" + + "local largest_shard=-1\n" + + "for i=0,num_shards-1 do\n" + + " shard_names[i+1]=ARGV[5+(i*2)]\n" + + " shard_lengths[i+1] = redis.call('zcard', shard_names[i+1])\n" + + " unack_names[i+1]=ARGV[5+(i*2)+1]\n" + + "\n" + + " if (shard_lengths[i+1] > largest_shard) then\n" + + " largest_shard = shard_lengths[i+1]\n" + + " end\n" + + "end\n" + + "\n" + + "local min_score=-1\n" + + "local min_member\n" + + "local matching_value\n" + + "local owning_shard_idx=-1\n" + + "\n" + + "local num_complete_shards=0\n" + + "for j=0,largest_shard-1 do\n" + + " for i=1,num_shards do\n" + + " local skiploop=false\n" + + " if (shard_lengths[i] < j+1) then\n" + + " skiploop=true\n" + + " end\n" + + "\n" + + " if (skiploop == false) then\n" + + " local element = redis.call('zrange', shard_names[i], j, j, 'WITHSCORES')\n" + + " if ((min_score ~= -1 and min_score < tonumber(element[2])) or peek_until < tonumber(element[2])) then\n" + + " -- This is to make sure we don't process this shard again\n" + + " -- since all elements henceforth are of lower priority than min_member\n" + + " shard_lengths[i]=0\n" + + " num_complete_shards = num_complete_shards + 1\n" + + " else\n" + + " local value = redis.call('hget', hkey, tostring(element[1]))\n" + + " if (string.match(value, predicate)) then\n" + + " if (min_score == -1 or tonumber(element[2]) < min_score) then\n" + + " min_score = tonumber(element[2])\n" + + " owning_shard_idx=i\n" + + " min_member = element[1]\n" + + " matching_value = value\n" + + " end\n" + + " end\n" + + " end\n" + + " end\n" + + " end\n" + + " if (num_complete_shards == num_shards) then\n" + + " break\n" + + " end\n" + + "end\n" + + "\n" + + "if (min_member) then\n" + + " local queue_shard_name=shard_names[owning_shard_idx]\n" + + " local unack_shard_name=unack_names[owning_shard_idx]\n" + + " local zadd_ret = redis.call('zadd', unack_shard_name, 'NX', unack_score, min_member)\n" + + " if (zadd_ret) then\n" + + " redis.call('zrem', queue_shard_name, min_member)\n" + + " end\n" + + "end\n" + + "return {min_member, matching_value}"; + + double now = Long.valueOf(clock.millis() + 1).doubleValue(); + double unackScore = Long.valueOf(clock.millis() + unackTime).doubleValue(); + + // The script requires the scores as whole numbers + NumberFormat fmt = NumberFormat.getIntegerInstance(); + fmt.setGroupingUsed(false); + String nowScoreString = fmt.format(now); + String unackScoreString = fmt.format(unackScore); + + ArrayList retval; + if (localShardOnly) { + String unackShardName = getUnackKey(queueName, shardName); + + ImmutableList.Builder builder = ImmutableList.builder(); + builder.add(predicate); + builder.add(Integer.toString(1)); + builder.add(nowScoreString); + builder.add(unackScoreString); + builder.add(localQueueShard); + builder.add(unackShardName); + + // Cast from 'JedisCommands' to 'DynoJedisClient' here since the former does not expose 'eval()'. + retval = (ArrayList) ((DynoJedisClient) quorumConn).eval(popPredicateObeyPriority, + Collections.singletonList(messageStoreKey), builder.build()); + } else { + + ImmutableList.Builder builder = ImmutableList.builder(); + builder.add(predicate); + builder.add(Integer.toString(allShards.size())); + builder.add(nowScoreString); + builder.add(unackScoreString); + for (String shard : allShards) { + String queueShard = getQueueShardKey(queueName, shard); + String unackShardName = getUnackKey(queueName, shard); + builder.add(queueShard); + builder.add(unackShardName); + } + + // Cast from 'JedisCommands' to 'DynoJedisClient' here since the former does not expose 'eval()'. + retval = (ArrayList) ((DynoJedisClient) quorumConn).eval(popPredicateObeyPriority, + Collections.singletonList(messageStoreKey), builder.build()); + } + + return new Message(retval.get(0), retval.get(1)); + + } + + @Override + public Message popMsgWithPredicate(String predicate, boolean localShardOnly) { + Stopwatch sw = monitor.start(monitor.pop, 1); + + try { + Message payload = execute("popMsgWithPredicateObeyPriority", messageStoreKey, () -> popMsgWithPredicateObeyPriority(predicate, localShardOnly)); + return payload; + + } finally { + sw.stop(); + } + + } + @Override public List bulkPop(int messageCount, int wait, TimeUnit unit) { @@ -1090,24 +1220,6 @@ private String atomicPopWithMsgIdHelper(String messageId, boolean localShardOnly return retval; } - @Override - public Message popMsgWithPredicate(String predicate, boolean localShardOnly) { - Stopwatch sw = monitor.get.start(); - - try { - String messageId = getMsgWithPredicate(predicate, localShardOnly); - if (messageId == null) { - return null; - } - String payload = execute("atomicPop", messageStoreKey, () -> atomicPopWithMsgIdHelper(messageId, localShardOnly)); - - return new Message(messageId, payload); - - } finally { - sw.stop(); - } - - } @Override public Message get(String messageId) { From ad6c57f5ff196fd2744c64d74f0ce4e9c0829870 Mon Sep 17 00:00:00 2001 From: Sailesh Mukil Date: Fri, 15 Nov 2019 15:44:45 -0800 Subject: [PATCH 74/91] Fix null return from popMsgWithPredicateObeyPriority --- .../main/java/com/netflix/dyno/queues/redis/RedisDynoQueue.java | 1 + 1 file changed, 1 insertion(+) diff --git a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/RedisDynoQueue.java b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/RedisDynoQueue.java index c28e29c..fb0825e 100644 --- a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/RedisDynoQueue.java +++ b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/RedisDynoQueue.java @@ -921,6 +921,7 @@ private Message popMsgWithPredicateObeyPriority(String predicate, boolean localS Collections.singletonList(messageStoreKey), builder.build()); } + if (retval.size() == 0) return null; return new Message(retval.get(0), retval.get(1)); } From 12d242ab3fe431ffa9bf2f6a88e3a209b0d269a2 Mon Sep 17 00:00:00 2001 From: Sailesh Mukil Date: Fri, 17 Jan 2020 14:50:41 -0800 Subject: [PATCH 75/91] Add atomicProcessUnacks() and getAllMessages() --- .../com/netflix/dyno/queues/DynoQueue.java | 10 ++ .../dyno/queues/redis/RedisDynoQueue.java | 120 +++++++++++++++++- .../dyno/queues/redis/RedisQueues.java | 6 +- .../dyno/queues/redis/v2/MultiRedisQueue.java | 10 ++ .../queues/redis/v2/RedisPipelineQueue.java | 10 ++ 5 files changed, 149 insertions(+), 7 deletions(-) diff --git a/dyno-queues-core/src/main/java/com/netflix/dyno/queues/DynoQueue.java b/dyno-queues-core/src/main/java/com/netflix/dyno/queues/DynoQueue.java index ce448e7..1db74f8 100644 --- a/dyno-queues-core/src/main/java/com/netflix/dyno/queues/DynoQueue.java +++ b/dyno-queues-core/src/main/java/com/netflix/dyno/queues/DynoQueue.java @@ -188,6 +188,15 @@ public interface DynoQueue extends Closeable { */ public Message get(String messageId); + /** + * + * Attempts to return all the messages found in the hashmap. It's a best-effort return of all payloads, i.e. it may + * not 100% match with what's in the queue metadata at any given time and is read with a non-quorum connection. + * + * @return Returns a list of all messages found in the message hashmap. + */ + public List getAllMessages(); + /** * * Same as get(), but uses the non quorum connection. @@ -222,6 +231,7 @@ public interface DynoQueue extends Closeable { * Process un-acknowledged messages. The messages which are polled by the client but not ack'ed are moved back to queue */ public void processUnacks(); + public void atomicProcessUnacks(); /* * <=== Begin unsafe* functions. ===> diff --git a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/RedisDynoQueue.java b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/RedisDynoQueue.java index fb0825e..fca9c96 100644 --- a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/RedisDynoQueue.java +++ b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/RedisDynoQueue.java @@ -87,6 +87,8 @@ public class RedisDynoQueue implements DynoQueue { private final ShardingStrategy shardingStrategy; + private final boolean singleRingTopology; + // Tracks the number of message IDs to prefetch based on the message counts requested by the caller via pop(). @VisibleForTesting AtomicInteger numIdsToPrefetch; @@ -96,15 +98,15 @@ public class RedisDynoQueue implements DynoQueue { @VisibleForTesting AtomicInteger unsafeNumIdsToPrefetchAllShards; - public RedisDynoQueue(String redisKeyPrefix, String queueName, Set allShards, String shardName, ShardingStrategy shardingStrategy) { - this(redisKeyPrefix, queueName, allShards, shardName, 60_000, shardingStrategy); + public RedisDynoQueue(String redisKeyPrefix, String queueName, Set allShards, String shardName, ShardingStrategy shardingStrategy, boolean singleRingTopology) { + this(redisKeyPrefix, queueName, allShards, shardName, 60_000, shardingStrategy, singleRingTopology); } - public RedisDynoQueue(String redisKeyPrefix, String queueName, Set allShards, String shardName, int unackScheduleInMS, ShardingStrategy shardingStrategy) { - this(Clock.systemDefaultZone(), redisKeyPrefix, queueName, allShards, shardName, unackScheduleInMS, shardingStrategy); + public RedisDynoQueue(String redisKeyPrefix, String queueName, Set allShards, String shardName, int unackScheduleInMS, ShardingStrategy shardingStrategy, boolean singleRingTopology) { + this(Clock.systemDefaultZone(), redisKeyPrefix, queueName, allShards, shardName, unackScheduleInMS, shardingStrategy, singleRingTopology); } - public RedisDynoQueue(Clock clock, String redisKeyPrefix, String queueName, Set allShards, String shardName, int unackScheduleInMS, ShardingStrategy shardingStrategy) { + public RedisDynoQueue(Clock clock, String redisKeyPrefix, String queueName, Set allShards, String shardName, int unackScheduleInMS, ShardingStrategy shardingStrategy, boolean singleRingTopology) { this.clock = clock; this.redisKeyPrefix = redisKeyPrefix; this.queueName = queueName; @@ -116,6 +118,7 @@ public RedisDynoQueue(Clock clock, String redisKeyPrefix, String queueName, Set< this.numIdsToPrefetch = new AtomicInteger(0); this.unsafeNumIdsToPrefetchAllShards = new AtomicInteger(0); + this.singleRingTopology = singleRingTopology; this.om = QueueUtils.constructObjectMapper(); this.monitor = new QueueMonitor(queueName, shardName); @@ -127,7 +130,11 @@ public RedisDynoQueue(Clock clock, String redisKeyPrefix, String queueName, Set< schedulerForUnacksProcessing = Executors.newScheduledThreadPool(1); - schedulerForUnacksProcessing.scheduleAtFixedRate(() -> processUnacks(), unackScheduleInMS, unackScheduleInMS, TimeUnit.MILLISECONDS); + if (this.singleRingTopology) { + schedulerForUnacksProcessing.scheduleAtFixedRate(() -> atomicProcessUnacks(), unackScheduleInMS, unackScheduleInMS, TimeUnit.MILLISECONDS); + } else { + schedulerForUnacksProcessing.scheduleAtFixedRate(() -> processUnacks(), unackScheduleInMS, unackScheduleInMS, TimeUnit.MILLISECONDS); + } logger.info(RedisDynoQueue.class.getName() + " is ready to serve " + queueName); } @@ -1245,6 +1252,18 @@ public Message get(String messageId) { } } + @Override + public List getAllMessages() { + Map allMsgs = nonQuorumConn.hgetAll(messageStoreKey); + List retList = new ArrayList<>(); + for (Map.Entry entry: allMsgs.entrySet()) { + Message msg = new Message(entry.getKey(), entry.getValue()); + retList.add(msg); + } + + return retList; + } + @Override public Message localGet(String messageId) { @@ -1330,6 +1349,7 @@ public void clear() { @Override public void processUnacks() { + logger.info("processUnacks() will NOT be atomic."); Stopwatch sw = monitor.processUnack.start(); try { @@ -1383,6 +1403,94 @@ public void processUnacks() { } + @Override + public void atomicProcessUnacks() { + + logger.info("processUnacks() will be atomic."); + Stopwatch sw = monitor.processUnack.start(); + try { + + long queueDepth = size(); + monitor.queueDepth.record(queueDepth); + + String keyName = getUnackKey(queueName, shardName); + execute("processUnacks", keyName, () -> { + + int batchSize = 1_000; + String unackShardName = getUnackKey(queueName, shardName); + + double now = Long.valueOf(clock.millis()).doubleValue(); + long num_moved_back = 0; + long num_stale = 0; + + Set unacks = nonQuorumConn.zrangeByScoreWithScores(unackShardName, 0, now, 0, batchSize); + + if (unacks.size() > 0) { + logger.info("processUnacks: Attempting to add " + unacks.size() + " messages back to shard of queue: " + unackShardName); + } else { + return null; + } + + String atomicProcessUnacksScript = "local hkey=KEYS[1]\n" + + "local unack_shard=ARGV[1]\n" + + "local queue_shard=ARGV[2]\n" + + "local num_unacks=ARGV[3]\n" + + "\n" + + "local unacks={}\n" + + "local unack_scores={}\n" + + "local unack_start_idx = 4\n" + + "for i=0,num_unacks-1 do\n" + + " unacks[i]=ARGV[4 + (i*2)]\n" + + " unack_scores[i]=ARGV[4+(i*2)+1]\n" + + "end\n" + + "\n" + + "local num_moved=0\n" + + "local num_stale=0\n" + + "for i=0,num_unacks-1 do\n" + + " local mem_val = redis.call('hget', hkey, unacks[i])\n" + + " if (mem_val) then\n" + + " redis.call('zadd', queue_shard, unack_scores[i], unacks[i])\n" + + " redis.call('zrem', unack_shard, unacks[i])\n" + + " num_moved=num_moved+1\n" + + " else\n" + + " redis.call('zrem', unack_shard, unacks[i])\n" + + " num_stale=num_stale+1\n" + + " end\n" + + "end\n" + + "\n" + + "return {num_moved, num_stale}\n"; + + ImmutableList.Builder builder = ImmutableList.builder(); + builder.add(unackShardName); + builder.add(localQueueShard); + builder.add(Integer.toString(unacks.size())); + for (Tuple unack : unacks) { + builder.add(unack.getElement()); + + // The script requires the scores as whole numbers + NumberFormat fmt = NumberFormat.getIntegerInstance(); + fmt.setGroupingUsed(false); + String unackScoreString = fmt.format(unack.getScore()); + builder.add(unackScoreString); + } + + ArrayList retval = (ArrayList) ((DynoJedisClient)quorumConn).eval(atomicProcessUnacksScript, Collections.singletonList(messageStoreKey), builder.build()); + num_moved_back = retval.get(0).longValue(); + num_stale = retval.get(1).longValue(); + if (num_moved_back > 0 || num_stale > 0) { + logger.info("processUnacks: Moved back " + num_moved_back + " items. Got rid of " + num_stale + " stale items."); + } + return null; + }); + + } catch (Exception e) { + logger.error("Error while processing unacks. " + e.getMessage()); + } finally { + sw.stop(); + } + + } + private String getQueueShardKey(String queueName, String shard) { return redisKeyPrefix + ".QUEUE." + queueName + "." + shard; } diff --git a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/RedisQueues.java b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/RedisQueues.java index 3cc2b93..08567f7 100644 --- a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/RedisQueues.java +++ b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/RedisQueues.java @@ -15,6 +15,7 @@ */ package com.netflix.dyno.queues.redis; +import com.netflix.dyno.jedis.DynoJedisClient; import com.netflix.dyno.queues.DynoQueue; import com.netflix.dyno.queues.ShardSupplier; import com.netflix.dyno.queues.redis.sharding.RoundRobinStrategy; @@ -56,6 +57,8 @@ public class RedisQueues implements Closeable { private final ShardingStrategy shardingStrategy; + private final boolean singleRingTopology; + /** * @param quorumConn Dyno connection with dc_quorum enabled * @param nonQuorumConn Dyno connection to local Redis @@ -103,6 +106,7 @@ public RedisQueues(Clock clock, JedisCommands quorumConn, JedisCommands nonQuoru this.unackHandlerIntervalInMS = unackHandlerIntervalInMS; this.queues = new ConcurrentHashMap<>(); this.shardingStrategy = shardingStrategy; + this.singleRingTopology = ((DynoJedisClient) quorumConn).getConnPool().getPools().size() == 3; } /** @@ -116,7 +120,7 @@ public DynoQueue get(String queueName) { String key = queueName.intern(); - return queues.computeIfAbsent(key, (keyToCompute) -> new RedisDynoQueue(clock, redisKeyPrefix, queueName, allShards, shardName, unackHandlerIntervalInMS, shardingStrategy) + return queues.computeIfAbsent(key, (keyToCompute) -> new RedisDynoQueue(clock, redisKeyPrefix, queueName, allShards, shardName, unackHandlerIntervalInMS, shardingStrategy, singleRingTopology) .withUnackTime(unackTime) .withNonQuorumConn(nonQuorumConn) .withQuorumConn(quorumConn)); diff --git a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/v2/MultiRedisQueue.java b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/v2/MultiRedisQueue.java index c94dc46..93e84d9 100644 --- a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/v2/MultiRedisQueue.java +++ b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/v2/MultiRedisQueue.java @@ -242,6 +242,11 @@ public void close() throws IOException { } } + @Override + public List getAllMessages() { + throw new UnsupportedOperationException(); + } + @Override public void processUnacks() { for (RedisPipelineQueue queue : queues.values()) { @@ -249,6 +254,11 @@ public void processUnacks() { } } + @Override + public void atomicProcessUnacks() { + throw new UnsupportedOperationException(); + } + private AtomicInteger nextShardIndex = new AtomicInteger(0); private String getNextShard() { diff --git a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/v2/RedisPipelineQueue.java b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/v2/RedisPipelineQueue.java index 903c426..7235b37 100644 --- a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/v2/RedisPipelineQueue.java +++ b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/v2/RedisPipelineQueue.java @@ -679,6 +679,16 @@ private void processUnacks(String unackShardKey) { } + @Override + public List getAllMessages() { + throw new UnsupportedOperationException(); + } + + @Override + public void atomicProcessUnacks() { + throw new UnsupportedOperationException(); + } + @Override public void close() throws IOException { schedulerForUnacksProcessing.shutdown(); From ee400669349f92c0ee6e8c59f9d0f806c9e7933e Mon Sep 17 00:00:00 2001 From: Sailesh Mukil Date: Fri, 17 Jan 2020 15:09:38 -0800 Subject: [PATCH 76/91] JedisMock cannot be cast to DynoJedisClient Tests fail without this fix. --- .../java/com/netflix/dyno/queues/redis/RedisQueues.java | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/RedisQueues.java b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/RedisQueues.java index 08567f7..e0df5d7 100644 --- a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/RedisQueues.java +++ b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/RedisQueues.java @@ -106,7 +106,12 @@ public RedisQueues(Clock clock, JedisCommands quorumConn, JedisCommands nonQuoru this.unackHandlerIntervalInMS = unackHandlerIntervalInMS; this.queues = new ConcurrentHashMap<>(); this.shardingStrategy = shardingStrategy; - this.singleRingTopology = ((DynoJedisClient) quorumConn).getConnPool().getPools().size() == 3; + + if (quorumConn instanceof DynoJedisClient) { + this.singleRingTopology = ((DynoJedisClient) quorumConn).getConnPool().getPools().size() == 3; + } else { + this.singleRingTopology = false; + } } /** From 3da0ec15591df6c6032fcd4a45a68943898bde48 Mon Sep 17 00:00:00 2001 From: Sailesh Mukil Date: Tue, 21 Jan 2020 09:16:24 -0800 Subject: [PATCH 77/91] Make popMsgWithPredicateObeyPriority() check if hash exists In some weird cases, we find the message in the queue but without a payload in the hashmap. Although this is never expected, it seems to happen, and this patch should stop the bleeding until we find out the root cause. --- .../netflix/dyno/queues/redis/RedisDynoQueue.java | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/RedisDynoQueue.java b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/RedisDynoQueue.java index fca9c96..0d32606 100644 --- a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/RedisDynoQueue.java +++ b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/RedisDynoQueue.java @@ -859,12 +859,14 @@ private Message popMsgWithPredicateObeyPriority(String predicate, boolean localS " num_complete_shards = num_complete_shards + 1\n" + " else\n" + " local value = redis.call('hget', hkey, tostring(element[1]))\n" + - " if (string.match(value, predicate)) then\n" + - " if (min_score == -1 or tonumber(element[2]) < min_score) then\n" + - " min_score = tonumber(element[2])\n" + - " owning_shard_idx=i\n" + - " min_member = element[1]\n" + - " matching_value = value\n" + + " if (value) then\n" + + " if (string.match(value, predicate)) then\n" + + " if (min_score == -1 or tonumber(element[2]) < min_score) then\n" + + " min_score = tonumber(element[2])\n" + + " owning_shard_idx=i\n" + + " min_member = element[1]\n" + + " matching_value = value\n" + + " end\n" + " end\n" + " end\n" + " end\n" + From de3787f622597cd7e3e3a28642950bd37d5542c9 Mon Sep 17 00:00:00 2001 From: Sailesh Mukil Date: Wed, 22 Jan 2020 14:33:10 -0800 Subject: [PATCH 78/91] Add an atomicRemove() API --- .../com/netflix/dyno/queues/DynoQueue.java | 1 + .../dyno/queues/redis/RedisDynoQueue.java | 56 +++++++++++++++++++ .../dyno/queues/redis/v2/MultiRedisQueue.java | 5 ++ .../queues/redis/v2/RedisPipelineQueue.java | 5 ++ 4 files changed, 67 insertions(+) diff --git a/dyno-queues-core/src/main/java/com/netflix/dyno/queues/DynoQueue.java b/dyno-queues-core/src/main/java/com/netflix/dyno/queues/DynoQueue.java index 1db74f8..691f2f2 100644 --- a/dyno-queues-core/src/main/java/com/netflix/dyno/queues/DynoQueue.java +++ b/dyno-queues-core/src/main/java/com/netflix/dyno/queues/DynoQueue.java @@ -121,6 +121,7 @@ public interface DynoQueue extends Closeable { * @return true if the message id was found and removed. False otherwise. */ public boolean remove(String messageId); + public boolean atomicRemove(String messageId); /** * Enqueues 'message' if it doesn't exist in any of the shards or unack sets. diff --git a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/RedisDynoQueue.java b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/RedisDynoQueue.java index 0d32606..528b4ab 100644 --- a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/RedisDynoQueue.java +++ b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/RedisDynoQueue.java @@ -710,6 +710,62 @@ public boolean remove(String messageId) { } } + @Override + public boolean atomicRemove(String messageId) { + + Stopwatch sw = monitor.remove.start(); + + try { + + return execute("remove", "(a shard in) " + queueName, () -> { + + + String atomicRemoveScript = "local hkey=KEYS[1]\n" + + "local msg_id=ARGV[1]\n" + + "local num_shards=ARGV[2]\n" + + "\n" + + "local removed_shard=0\n" + + "local removed_unack=0\n" + + "local removed_hash=0\n" + + "for i=0,num_shards-1 do\n" + + " local shard_name = ARGV[3+(i*2)]\n" + + " local unack_name = ARGV[3+(i*2)+1]\n" + + "\n" + + " removed_shard = removed_shard + redis.call('zrem', shard_name, msg_id)\n" + + " removed_unack = removed_unack + redis.call('zrem', unack_name, msg_id)\n" + + "end\n" + + "\n" + + "removed_hash = redis.call('hdel', hkey, msg_id)\n" + + "if (removed_shard==1 or removed_unack==1 or removed_hash==1) then\n" + + " return 1\n" + + "end\n" + + "return removed_unack\n"; + + ImmutableList.Builder builder = ImmutableList.builder(); + builder.add(messageId); + builder.add(Integer.toString(allShards.size())); + + for (String shard : allShards) { + + String queueShardKey = getQueueShardKey(queueName, shard); + String unackShardKey = getUnackKey(queueName, shard); + + builder.add(queueShardKey); + builder.add(unackShardKey); + } + + Long removed = (Long) ((DynoJedisClient)quorumConn).eval(atomicRemoveScript, Collections.singletonList(messageStoreKey), builder.build()); + if (removed == 1) return true; + + return false; + + }); + + } finally { + sw.stop(); + } + } + @Override public boolean ensure(Message message) { return execute("ensure", "(a shard in) " + queueName, () -> { diff --git a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/v2/MultiRedisQueue.java b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/v2/MultiRedisQueue.java index 93e84d9..cf32bf4 100644 --- a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/v2/MultiRedisQueue.java +++ b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/v2/MultiRedisQueue.java @@ -259,6 +259,11 @@ public void atomicProcessUnacks() { throw new UnsupportedOperationException(); } + @Override + public boolean atomicRemove(String messageId) { + throw new UnsupportedOperationException(); + } + private AtomicInteger nextShardIndex = new AtomicInteger(0); private String getNextShard() { diff --git a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/v2/RedisPipelineQueue.java b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/v2/RedisPipelineQueue.java index 7235b37..afc97fd 100644 --- a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/v2/RedisPipelineQueue.java +++ b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/v2/RedisPipelineQueue.java @@ -689,6 +689,11 @@ public void atomicProcessUnacks() { throw new UnsupportedOperationException(); } + @Override + public boolean atomicRemove(String messageId) { + throw new UnsupportedOperationException(); + } + @Override public void close() throws IOException { schedulerForUnacksProcessing.shutdown(); From 5b25b6928ff53ef98266724d8e57391a9ea2d0e0 Mon Sep 17 00:00:00 2001 From: Sailesh Mukil Date: Thu, 23 Jan 2020 11:25:27 -0800 Subject: [PATCH 79/91] Add findStaleMessages() API Attempts to return the items present in the local queue shard but not in the hashmap, if any. (Ideally, we would not require this function, however, in some configurations, especially with multi-region write traffic sharing the same queue, we may find ourselves with stale items in the queue shards) --- .../com/netflix/dyno/queues/DynoQueue.java | 10 +++ .../dyno/queues/redis/RedisDynoQueue.java | 64 +++++++++++++++++++ .../dyno/queues/redis/v2/MultiRedisQueue.java | 3 + .../queues/redis/v2/RedisPipelineQueue.java | 3 + 4 files changed, 80 insertions(+) diff --git a/dyno-queues-core/src/main/java/com/netflix/dyno/queues/DynoQueue.java b/dyno-queues-core/src/main/java/com/netflix/dyno/queues/DynoQueue.java index 691f2f2..db9581d 100644 --- a/dyno-queues-core/src/main/java/com/netflix/dyno/queues/DynoQueue.java +++ b/dyno-queues-core/src/main/java/com/netflix/dyno/queues/DynoQueue.java @@ -234,6 +234,16 @@ public interface DynoQueue extends Closeable { public void processUnacks(); public void atomicProcessUnacks(); + /** + * + * Attempts to return the items present in the local queue shard but not in the hashmap, if any. + * (Ideally, we would not require this function, however, in some configurations, especially with multi-region write + * traffic sharing the same queue, we may find ourselves with stale items in the queue shards) + * + * @return List of stale messages IDs. + */ + public List findStaleMessages(); + /* * <=== Begin unsafe* functions. ===> * diff --git a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/RedisDynoQueue.java b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/RedisDynoQueue.java index 528b4ab..8c99fc9 100644 --- a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/RedisDynoQueue.java +++ b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/RedisDynoQueue.java @@ -1461,6 +1461,70 @@ public void processUnacks() { } + @Override + public List findStaleMessages() { + return execute("findStaleMessages", localQueueShard, () -> { + + List stale_msgs = new ArrayList<>(); + + int batchSize = 1_000; + + double now = Long.valueOf(clock.millis()).doubleValue(); + long num_stale = 0; + + Set elems = nonQuorumConn.zrangeByScore(localQueueShard, 0, now, 0, batchSize); + + if (elems.size() == 0) { + return stale_msgs; + } + + String findStaleMsgsScript = "local hkey=KEYS[1]\n" + + "local queue_shard=ARGV[1]\n" + + "local unack_shard=ARGV[2]\n" + + "local num_msgs=ARGV[3]\n" + + "\n" + + "local stale_msgs={}\n" + + "local num_stale_idx = 1\n" + + "for i=0,num_msgs-1 do\n" + + " local msg_id=ARGV[4+i]\n" + + "\n" + + " local exists_hash = redis.call('hget', hkey, msg_id)\n" + + " local exists_queue = redis.call('zscore', queue_shard, msg_id)\n" + + " local exists_unack = redis.call('zscore', unack_shard, msg_id)\n" + + "\n" + + " if (exists_hash and exists_queue) then\n" + + " elseif (not (exists_unack)) then\n" + + " stale_msgs[num_stale_idx] = msg_id\n" + + " num_stale_idx = num_stale_idx + 1\n" + + " end\n" + + "end\n" + + "\n" + + "return stale_msgs\n"; + + String unackKey = getUnackKey(queueName, shardName); + ImmutableList.Builder builder = ImmutableList.builder(); + builder.add(localQueueShard); + builder.add(unackKey); + builder.add(Integer.toString(elems.size())); + for (String msg : elems) { + builder.add(msg); + } + + ArrayList stale_msg_ids = (ArrayList) ((DynoJedisClient)quorumConn).eval(findStaleMsgsScript, Collections.singletonList(messageStoreKey), builder.build()); + num_stale = stale_msg_ids.size(); + if (num_stale > 0) { + logger.info("findStaleMsgs(): Found " + num_stale + " messages present in queue but not in hashmap"); + } + + for (String m : stale_msg_ids) { + Message msg = new Message(); + msg.setId(m); + stale_msgs.add(msg); + } + return stale_msgs; + }); + } + @Override public void atomicProcessUnacks() { diff --git a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/v2/MultiRedisQueue.java b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/v2/MultiRedisQueue.java index cf32bf4..47bd906 100644 --- a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/v2/MultiRedisQueue.java +++ b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/v2/MultiRedisQueue.java @@ -259,6 +259,9 @@ public void atomicProcessUnacks() { throw new UnsupportedOperationException(); } + @Override + public List findStaleMessages() { throw new UnsupportedOperationException(); } + @Override public boolean atomicRemove(String messageId) { throw new UnsupportedOperationException(); diff --git a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/v2/RedisPipelineQueue.java b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/v2/RedisPipelineQueue.java index afc97fd..33ce187 100644 --- a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/v2/RedisPipelineQueue.java +++ b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/v2/RedisPipelineQueue.java @@ -689,6 +689,9 @@ public void atomicProcessUnacks() { throw new UnsupportedOperationException(); } + @Override + public List findStaleMessages() { throw new UnsupportedOperationException(); } + @Override public boolean atomicRemove(String messageId) { throw new UnsupportedOperationException(); From 03af6f5359fa45c4d18e31ed30103f885f09767d Mon Sep 17 00:00:00 2001 From: Sailesh Mukil Date: Fri, 24 Jan 2020 18:40:13 -0800 Subject: [PATCH 80/91] Make findStaleMessages() return stale messages from all shards Note: All items returned MUST be checked at the app level if they've already been processed before acting on them (eg: removing them) --- .../dyno/queues/redis/RedisDynoQueue.java | 92 ++++++++++--------- 1 file changed, 48 insertions(+), 44 deletions(-) diff --git a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/RedisDynoQueue.java b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/RedisDynoQueue.java index 8c99fc9..8ac7887 100644 --- a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/RedisDynoQueue.java +++ b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/RedisDynoQueue.java @@ -1467,60 +1467,64 @@ public List findStaleMessages() { List stale_msgs = new ArrayList<>(); - int batchSize = 1_000; + int batchSize = 10; double now = Long.valueOf(clock.millis()).doubleValue(); long num_stale = 0; - Set elems = nonQuorumConn.zrangeByScore(localQueueShard, 0, now, 0, batchSize); + for (String shard : allShards) { + String queueShardName = getQueueShardKey(queueName, shard); + Set elems = nonQuorumConn.zrangeByScore(queueShardName, 0, now, 0, batchSize); - if (elems.size() == 0) { - return stale_msgs; - } + if (elems.size() == 0) { + continue; + } - String findStaleMsgsScript = "local hkey=KEYS[1]\n" + - "local queue_shard=ARGV[1]\n" + - "local unack_shard=ARGV[2]\n" + - "local num_msgs=ARGV[3]\n" + - "\n" + - "local stale_msgs={}\n" + - "local num_stale_idx = 1\n" + - "for i=0,num_msgs-1 do\n" + - " local msg_id=ARGV[4+i]\n" + - "\n" + - " local exists_hash = redis.call('hget', hkey, msg_id)\n" + - " local exists_queue = redis.call('zscore', queue_shard, msg_id)\n" + - " local exists_unack = redis.call('zscore', unack_shard, msg_id)\n" + - "\n" + - " if (exists_hash and exists_queue) then\n" + - " elseif (not (exists_unack)) then\n" + - " stale_msgs[num_stale_idx] = msg_id\n" + - " num_stale_idx = num_stale_idx + 1\n" + - " end\n" + - "end\n" + - "\n" + - "return stale_msgs\n"; + String findStaleMsgsScript = "local hkey=KEYS[1]\n" + + "local queue_shard=ARGV[1]\n" + + "local unack_shard=ARGV[2]\n" + + "local num_msgs=ARGV[3]\n" + + "\n" + + "local stale_msgs={}\n" + + "local num_stale_idx = 1\n" + + "for i=0,num_msgs-1 do\n" + + " local msg_id=ARGV[4+i]\n" + + "\n" + + " local exists_hash = redis.call('hget', hkey, msg_id)\n" + + " local exists_queue = redis.call('zscore', queue_shard, msg_id)\n" + + " local exists_unack = redis.call('zscore', unack_shard, msg_id)\n" + + "\n" + + " if (exists_hash and exists_queue) then\n" + + " elseif (not (exists_unack)) then\n" + + " stale_msgs[num_stale_idx] = msg_id\n" + + " num_stale_idx = num_stale_idx + 1\n" + + " end\n" + + "end\n" + + "\n" + + "return stale_msgs\n"; - String unackKey = getUnackKey(queueName, shardName); - ImmutableList.Builder builder = ImmutableList.builder(); - builder.add(localQueueShard); - builder.add(unackKey); - builder.add(Integer.toString(elems.size())); - for (String msg : elems) { - builder.add(msg); - } + String unackKey = getUnackKey(queueName, shard); + ImmutableList.Builder builder = ImmutableList.builder(); + builder.add(queueShardName); + builder.add(unackKey); + builder.add(Integer.toString(elems.size())); + for (String msg : elems) { + builder.add(msg); + } - ArrayList stale_msg_ids = (ArrayList) ((DynoJedisClient)quorumConn).eval(findStaleMsgsScript, Collections.singletonList(messageStoreKey), builder.build()); - num_stale = stale_msg_ids.size(); - if (num_stale > 0) { - logger.info("findStaleMsgs(): Found " + num_stale + " messages present in queue but not in hashmap"); - } + ArrayList stale_msg_ids = (ArrayList) ((DynoJedisClient)quorumConn).eval(findStaleMsgsScript, Collections.singletonList(messageStoreKey), builder.build()); + num_stale = stale_msg_ids.size(); + if (num_stale > 0) { + logger.info("findStaleMsgs(): Found " + num_stale + " messages present in queue but not in hashmap"); + } - for (String m : stale_msg_ids) { - Message msg = new Message(); - msg.setId(m); - stale_msgs.add(msg); + for (String m : stale_msg_ids) { + Message msg = new Message(); + msg.setId(m); + stale_msgs.add(msg); + } } + return stale_msgs; }); } From 7f8e6816c0cb32bce8ac4af3303be8356d90f14e Mon Sep 17 00:00:00 2001 From: Sailesh Mukil Date: Tue, 25 Aug 2020 08:39:32 -0700 Subject: [PATCH 81/91] Update README.md --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index 82068e8..8abf2c7 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,7 @@ +## DISCLAIMER: THIS PROJECT IS NO LONGER ACTIVELY MAINTAINED + + + # Dyno Queues [![Build Status](https://travis-ci.org/Netflix/dyno-queues.svg)](https://travis-ci.org/Netflix/dyno-queues) [![Dev chat at https://gitter.im/Netflix/dynomite](https://badges.gitter.im/Netflix/dynomite.svg)](https://gitter.im/Netflix/dynomite?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) From 6722cb25ac5d62d9f1454b0c6dee0ad6cb11c1af Mon Sep 17 00:00:00 2001 From: Sailesh Mukil Date: Tue, 17 Nov 2020 11:43:15 -0800 Subject: [PATCH 82/91] Comment out Global properties from demo.properties --- dyno-queues-redis/src/main/resources/demo.properties | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/dyno-queues-redis/src/main/resources/demo.properties b/dyno-queues-redis/src/main/resources/demo.properties index 05408a3..d85b54e 100644 --- a/dyno-queues-redis/src/main/resources/demo.properties +++ b/dyno-queues-redis/src/main/resources/demo.properties @@ -1,10 +1,10 @@ #### ## Properties to initialize the demo app # -LOCAL_DATACENTER=us-east-1 -LOCAL_RACK=us-east-1c -NETFLIX_STACK=dyno_demo -EC2_AVAILABILITY_ZONE=us-east-1c +#LOCAL_DATACENTER=us-east-1 +#LOCAL_RACK=us-east-1c +#NETFLIX_STACK=dyno_demo +#EC2_AVAILABILITY_ZONE=us-east-1c dyno.demo.lbStrategy=TokenAware dyno.demo.retryPolicy=RetryNTimes:2 dyno.demo.port=8102 From 1089056b13af0c3d3f75b68481b290ebd4f5390c Mon Sep 17 00:00:00 2001 From: Sailesh Mukil Date: Tue, 17 Nov 2020 12:33:02 -0800 Subject: [PATCH 83/91] Update build status location --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 8abf2c7..e6a5c6b 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,7 @@ # Dyno Queues -[![Build Status](https://travis-ci.org/Netflix/dyno-queues.svg)](https://travis-ci.org/Netflix/dyno-queues) +[![Build Status](https://travis-ci.com/Netflix/dyno-queues.svg)](https://travis-ci.com/Netflix/dyno-queues) [![Dev chat at https://gitter.im/Netflix/dynomite](https://badges.gitter.im/Netflix/dynomite.svg)](https://gitter.im/Netflix/dynomite?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) Dyno Queues is a recipe that provides task queues utilizing [Dynomite](https://github.com/Netflix/dynomite). From 826888fd1441c7f54f21bd30ecd6208d1189af2c Mon Sep 17 00:00:00 2001 From: Roberto Perez Alcolea Date: Wed, 3 Mar 2021 16:42:57 -0800 Subject: [PATCH 84/91] Upgrade nebula.netflixoss to replace bintray publication and update TravisCi secrets --- .gitignore | 5 +- .travis.yml | 45 ++++++++++-------- build.gradle | 2 +- buildViaTravis.sh | 9 ++-- .../com/netflix/dyno/queues/DynoQueue.java | 44 +++++++++++++---- gradle/wrapper/gradle-wrapper.properties | 2 +- secrets/signing-key.enc | Bin 0 -> 6800 bytes 7 files changed, 72 insertions(+), 35 deletions(-) create mode 100644 secrets/signing-key.enc diff --git a/.gitignore b/.gitignore index 6e9cfde..72f70c3 100644 --- a/.gitignore +++ b/.gitignore @@ -12,4 +12,7 @@ redis-3.0.7.tar.gz dyno-queues-core/out/ -dyno-queues-redis/out/ \ No newline at end of file +dyno-queues-redis/out/ + +# publishing secrets +secrets/signing-key diff --git a/.travis.yml b/.travis.yml index ab84c2e..3875ac9 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,33 +2,36 @@ language: java jdk: - openjdk8 install: - - export REDIS_BIN=$HOME/redis/3.0.7/bin - - export TMPDIR=/tmp - - wget -c https://github.com/antirez/redis/archive/3.0.7.tar.gz -O redis-3.0.7.tar.gz - - mv redis-3.0.7.tar.gz $TMPDIR - - cd $TMPDIR - - tar -xvf redis-3.0.7.tar.gz - - make -C redis-3.0.7 PREFIX=$HOME/redis/3.0.7 install +- export REDIS_BIN=$HOME/redis/3.0.7/bin +- export TMPDIR=/tmp +- wget -c https://github.com/antirez/redis/archive/3.0.7.tar.gz -O redis-3.0.7.tar.gz +- mv redis-3.0.7.tar.gz $TMPDIR +- cd $TMPDIR +- tar -xvf redis-3.0.7.tar.gz +- make -C redis-3.0.7 PREFIX=$HOME/redis/3.0.7 install before_script: - - $REDIS_BIN/redis-server --daemonize yes - - sleep 3 - - $REDIS_BIN/redis-cli PING - - export REDIS_VERSION="$(redis-cli INFO SERVER | sed -n 2p)" - - echo $REDIS_VERSION - - cd $TRAVIS_BUILD_DIR - - ls -la - - git status . +- "$REDIS_BIN/redis-server --daemonize yes" +- sleep 3 +- "$REDIS_BIN/redis-cli PING" +- export REDIS_VERSION="$(redis-cli INFO SERVER | sed -n 2p)" +- echo $REDIS_VERSION +- cd $TRAVIS_BUILD_DIR +- ls -la +- git status . +- openssl aes-256-cbc -k "$NETFLIX_OSS_SIGNING_FILE_PASSWORD" -in secrets/signing-key.enc -out secrets/signing-key -d after_success: - - redis-cli SHUTDOWN NOSAVE +- redis-cli SHUTDOWN NOSAVE after_failure: - - redis-cli SHUTDOWN NOSAVE +- redis-cli SHUTDOWN NOSAVE script: "./buildViaTravis.sh" cache: directories: - "$HOME/.gradle/caches" env: global: - - secure: NrYUZ8/zJLMeNPilbr32XzbA7159FE8Xi5c+GOodrxz7+5+W/RkIuW1e1INu6iA/PfC6qB3IhHVicH+HuREwnBnBDnz9F2HvcOEetmylLDJ9rV71Qq4Bbna5jORuwTqGf8EnGFYzxFMqgpjcIkXHOLnNeJtJSBdcLLQcIuc7JPKbk/i63k9sc8prRZs2G4jrsmFaBqJtRWmS4hWcOR5GJUn/URgDrlrKne6PkMprTzXdhrAURoGUKLgleGGNuzxGb0Bxb/Y1oRne9eWbR8FqwlDVY7RV7KjxF3dWGWJTdW8O8nYzH83G6AhqBlC93Y3Xi6nDWQHnKWi7JdPw6ohPGiikiqOV3deX7gwrXzLr1JjFGIsl8LFE74lzIjp9YBsM2O7l0Vm37YUM/UJorpjtLmTMB7fXZSrge8Cmu1nekq7btw2JS1fia5hk/RMUGGEzbSiOxG4/dgfE7qHwbG22OWHM3dyunDs8sJUQlkSBHnBukr5F5fgz3HemG+nH4x31JqHiajWrztYI7brwVWvndzs7fpyMBGGLIS0Ql/jV53Mktw5G/8N4tZgMstf7mfQtF3m2yqNHW3+iaB9R14DyobBcOrJi5A4opM70KwSR9QbyHAi+i833pc03CcMV13kaMQ5/YDDck8ywyZ/qfsR7J68TcLO1Wa/UHS7SamuTG5k= - - secure: FXIh9ibMdmI7YMaE4k3H1eF6n8LiP9KdwuOuSagx81Rr43cpqIH5JFWPx5yY0Mws1scfXt3hRzPFZ1HWhCLoaAxedGnhT9NKWzj/RyTB+6AQR+3tpwh4OeJkvxC8QPBWWmC0hag7ahRkKfhiSHbQ53w6aw4/Hpn/TrnX/UEIhpIrDLbzMBAgF0P0FavX87A2rpZefgP0ugJo/GG3fpJ1qg4FWah1svSI6cFw+w3WKv1Imk42EHWEPFPzPCQjiXh5yCR+90oswf7oiImBpGY7M45YEZTr4Y6JaVOPVbTTpNdkhlevtTPFJXDrskqNxA1JlVpRXzEGjTQlezEZ4WYf0BFbFcyPGaryUtDXeCkkt9VoBfV3VE+p9btVNHMXOnxLBj20XXz6MQo2ucf0sqLtqnGTsaPYY2Ej9az9N8/DTnx0tR/Df6cwyYOQ6TUfEZdtyVVxCzgFvS6439/47BaEBdqx4AlmEL4fEe/WnEFHUGHDKQP68NMy0RTvGJSCxxEFYKiy/OKlFs2esRvEulmEceWnSlr0NFGtySuqO5ebt8bCy80K/5NCDT2TIMNF7wcg05tKEhd3rIqluUyAZ0gAZx3brQhWvVkDrBOoWV5PTrs/5iicrP8LtXYrsm8+OHAs7rpXbK3OtWEuBvjyMZptCtt08lvOVQZ02etWAg96+2U= - - secure: iK6dRuLfnV+ZWhUxfcwMN3cQBlJowRysibtnV0c58j2UsP7cC4gkDeFnwIw3ggok751cdVXYQrRk2mJb4xhCiw90p1TU4VEHtfTjHKS/45jknrrFbScHplBcDoux6NLp4fUYdc2QswsPCsCFefIFZcpejumyZ4cAsJlfFTU/Kv3UT2SZyTkapcCmn/5FiIr2H3Ups+ShJOnxwNkE+svEcKdV/g67ZCzZfpPM79L4LpgA89/o3kItJSRHDWaECtTOoh759nJh+i7J4eCDbZTrZb27xuneeMgr1ffxHHN75z2Sf2PVwTQdnAdKbhVfmHx0nqms+ICLu3PZ8Wfb3wWOwGFlMXIsbpiZpOdKXQD+yTaR8NC9B8PbHFOFD/Q4kBJ5fdOh5tw4u8ejF4vro9l7z5RhEDbRtmrSH1OfM2RatmKRgx87hva1AzKhLOPSGbYSZ7OpVO/0slJSOXlcNsahIPCaiEmKvVQv2sCLJZdpVWXSA9HpmMb2S+wI8LI88sFZXIuz1mFRYtvxBDs3aIKMYWUXqjewI2ZXipi9INYiliTyKqp20RQNRvViTZLKYPmrN4tfbnYR9lBkEoX8UP7gKcS0CfuIjCBshkwEKfzXDbv0tlceRsi8fW4Dr93MDNvvIlI8vxIe2rW5WtcZdie3s1X0dg6j14nRBKIE9DpyCIA= - - secure: QZ9O23vPVd4EoV5rKXGnHpkI4Ehn6fYy1Sj6YBTsWPdBBw/AZukkj6wDl9sNvDeoe3ub/U7VcbjhJBkpXAuPk8hWZ3Qk+2L9kkMkCSF+osfOmOI8ufoEtPT4cCbLwv2T41g+8Tp224S4AtoWQzSy91yTgvUOqfm3SXA200iJF5DkCv+i+Am+ySKYT4T+ZcPWyqpV6Y/5XPE3X99UbdJ4oasju1cPsMnYRfKNhIjIunaFgXIo/KtBF+IPEgDsfHwDRQ+7Lt55QGFK2yqsP50Zb6s9IqLgTngEu41pTwnoJCg0z5+q1ZmVbL8EHYOpMA3iopB+k5oY9vRbM/8Y83oQeT1fntSHxgpDuWax7m+1kxGOlv9FLUVoSBKq0dVjiE9WXLn4gdV/Dvkc5c1fPj82PhvkCJmVmlL90gf4/tDWIKvjGg2EIhSUzRkB1Me8RTBOD607OTViKVlzX0+T5y9ILOb1C2krEcrW+lEg49rdzvG/nUZqWXfhX5x7OJmmBpPnAnCVJnTkVfxs6Eq6cqIQc0oMZlYm7Rl6nOJCnqbYm33rwUIrRHxwtmyXEXXWyTxXoKpe+LZvXzT6+qhFuQnz1iFZjL+C5BbJ1nSBCKSAbLBsgteIMZWDZQhkwPfZPhpHxRIejNo6NnXXu6brn4JZgpi0g3No9DgBz/TLLfiS8g8= + - secure: eEjRoGTY9T+eN7Cpym9XG7ZwRlYfTFYBI2SJHHJnpfNKLJs6Tw3kyuNz4j7TL5iTkUbgSmmE/Nbb9UDKrmLHO+HlmQ6znpVzsp/RRp9MqytROXirfvYYxJmWmJBJlUYafIp5cEQVFbsdbFh2+nyqansJe7JSNtXDofBNEf8dkT3zgrkEf6iy5QwnRPVJVtNjuCPFL1LV8pPY/IGB66zsCevpCdfATrOCVJGOF9GnmbObfC651Ig6hvNZxkbejxGmj7GTxu7xhgAdzFQ+Bi6wZ573vV8MwwGwo5PxbA8lJhmOUrFT+ToMKxitT0eRGsma/4L8zvev8R8OR9r+BcsbFfZrf4JDykcWztfbTJmlWiCOG09Rn4sRuB/a3aNfqokqXQMOLH82hK0/gD60wtCq7mXMsV5G3H6D93i/5+cJEkNlRBLM8cJiqV0GnfR8nMGTg9gOFsRF5HyzFvLrlYVOY8LdlKFJE3FX/rio0cNi0ilDeq2dmYBGnu0gG9Ey55eEFNDbE9YDa1vD8dQ0eBFfiib+SiF13mdk++v4Bl3Dmook2yfamg99x9zuJ20vRUPSq6OOzaFO+Wpnl/QMPs9VHOBHJtth1Du4WK1FsV5Gayj7uPdrtj8MA4KKLEKtWQ6F5C3AfnU2MQCd8VUNTJtmjdtJ5wS0mAPPjYB4KJqETC4= + - secure: ZRt6jZtYfkMHaUOrm5K1e7jlrx/a4pThCP4yq5u6q0iBO1psEMS9QIVKF/pvmWbtqy32g/Z0uS/2ESxRIChZJU5L/xZ9noO58xfBfKEHlBUjwFwlaN3MTAXp0QzSHFhYSg5mpFTUg1c5Wd0b6DDfeHKbGoWmjNEF5WJAhSZyTW92CfFka9T4pg1Z35eqXhNvwp4sWEVhOmt0PlarhJIdFxtr5eaodEbKf6icTo8Ep47LZbAisgL93P5cEesi6oBK6PXEEpK4mGsKp9PSl2J2V5Rrci5uNjQGLporcQf/fCLmpozV7H13qHsfn3wgFz8X4nQVHRpT3NwG9OkkbNBDf95nWMcDrSafhINJ+RhgO5OP0jldy32pkYiFK6cNR9XLjSRYjJsPN+ELTa4F5Ql98oNDbIIT+P9MT7Fco/Q8N1irgTZhFZXLgweqyGql+5Exw+au005b+drM/T2ssP5i173BVyok32KIW1D+5dfae0hat6OVKkVNc20u9sifbBVHhBqBFGBXrxZXMH+RDM8RRe7kEi0U33J2p2kKxqDWYuoT/c6BplC9i4+Ee5HNWeG0Hxn2WLK4Ttt7CdVwlS1U+1UFB2N7n1zX4zRh2dxKRRrAzsbS8YlcgawYEE+fQxGbkUnqJ0gWs0YU0hHcpu/sHNddhHuHekrx8Uten8A/Fbc= + - secure: k6WGpdbdfIyVBRoKJKuu7VaPEh5ux8fuaW9xxl0wG2tqWP/tfyWvxURNr3Qx3VwPNsiiTV7jhod1kFOzVT0LGanNBEP2N2qiJYTvGaRPLx4tE67HhAqeinnvvUro1k019yHCMgT5tMtVD1HEi7J0Jbi3vB7nVVql0pRFQXB0NrMK4P+J8DD3ClbKX0iie4wvwNNfNq03R4Vn3fF7CDih3WJafcDDdYqaipq9h/BqbkZ/x96DaOQW+o3+rSN976qo/8NvqmZL6cfyR1yiOagMISzJwwjWBW9rT9/a7LK3yOCUQtb+W2wtEN117ggRnFbFS7QLS0e7eLh2OcL20yg6WhVigpFOlrcK0BgdPm18ZGf3icnmcMODXGutxcEECpZF5BuHSNIpK4Fm2a4E4tNEvrEy8rwhqJVQBBGOoWpIHfOL+86RDd6sA0CYu4rdrB5gYROnQkaavYIyOCuTBUMSXaqIL/SKJi9Mz5r1Svuf4K9UQK9wQo7a9MFajlm8QWepngrqkC1G3AmqG1edYCVUwWx3eubf++gNhPV1oGTBOa2gqWQTp9+3C7qTmy3YhD6WUvUcLOOfgiURQhJFSxsS+aSQvB+2ZxVjxL5U6RSx0eB60CnDCZaRm/dLChK1BnBUaYJE59yvVgy7C0yW8E8AtR8VjgnTLHMwUrt2D8ejyXU= + - secure: Ey9NK5fxSVajZHGV8O41/ZEHRIu0xdp7FalhAdYaGAflMiuMvdP+iuwfv1RONVvFVMLfuBc7wu7e1XZlRfA1G5zIhQM8LNdckPWKZVhcHpbRka9c6IlOPArwQTiHfwYWlZfWfAK9akVXcc/1tK4ePU5JZDIAKer4xw3AvtsH302CkLMYIQInm8Xf8YSTVMfWtFTANA3pU6zbiDvJLTj4/NE/9tNeL0xoG0AGRnhyXb27N9nbiLCekweGLipuqogR5I1lYsK8VHFlkLeSbwjEcsbV++eS3cGSsDW557RfbIRQXE7cP7buIxyFzPgiNh3beDihBEv0PNkYR5U4lXxn2/RReOTaAB37r2BFN/WnppCDkWKqT4HPaWYnVlm+0XVA6m4gSVoC8EC7kvc2DseKUn7PiAVED+UeA/PCfRYk213vgDxKor3O4iRXmbfoR/AbLgs6AKYK6RHNrL/Be21QT2Wyv1yT+82TTzY3xkesz6+f+NQU21AvNahdvgcXGSYJ7NaGY8kDUvZ6NJtAdpWnI5H5l6PPiWRlmsIr/qWwmGLjHyEI+fdQJdJlZg639J5kzjHYX5brN6/Qjvc8++NmBg6OiIyJ7+DreEoco5npoDHTNOF16HpPZ89JIof8oq5bJKrvsxFjgQMmf2yWnS7nJRP8g0/0mkDoAyYb0N7CKls= + - secure: rVxtKHNl7PrBHtWnKLlOi3yhTnlMHWG9fZbm9m4eMcEf6pUnD54KBu5kJDuQv5dyFYcHoI9Pbzfhq0f4aAll37vThwXHpfYKRAAs8SkkNcUsmM6w2ALkM8Xz+WKsUx1x0U+wWvBFIErzyB2sDY7+QIzfNRCuS9zs0h+1tH9350AlurgLwuHQVjwVSmdEkicR1vvILap+RiDwpVQrqEB+imhIgH4G826NpkQjdmbBszEOgfCUeFp1N0MmA2ln+MbFhKWaoKDAdJlsHh2R5vn4rR2jl9VuFa7zgmk0hIycM2p9OxJuO//Nz9jm4h5mFsvirSSu0v25rWe+WU8tc3auzR6t3RkwTiizZhYH/CXRwPMeXIs0wvZmStkwwsiROHBboCV9HPjHG/98ziXjL2NbNP/EmJDj9wKqy2SmggVVxnNAE9DoIX73+Vk8aDEOgE/Xy3FZ838IsCVmodz1v01bFL3Hac+oL2Wd92DlOgtQaSIqoZE5XTxbtAGvFm1nyFhOCHv6B33IbuJjtMrpzqrZb+WpvPU45pJXFfqM5m9TflBOBrwersFKe1+zyw6oDUpMqzD3F6B1b0e4nVL4J8gn8iO8Am4/7Xk1bXfWrySkDF8TEEJV+XLrUcnQULDzjhp3IVYCB0tAdybeNVHJ0ReLEBBB3XPaRxc0BGqpghWdn5c= + - secure: TZFMcFcEkxV9puUgZh4tVxqD0biCmESRyluIcyzYfjbVGRbH2p0+oaU1NtMgGGyCfRaHx1UizjKAtXqsjoK/f31bKmBFC1CB9rWJZF8HHxekVi+fhCy7IL9I7dhs9NBl6pZe9SZVr4gmiAszFYfLIzh1BSg1sZf//M9p5DdMcQ6EaAn0iELq6XwtzCbzEL/mhLZyJYkySa0k8gTktopZef1yuk8E0nFAMafvFWGDmFriAe7HUMeh7i50RrmxiPGBCGycv2Xof5XDm7ppOtpBn/TMonY3C3I4y4fg71sW/uDFi5ndDUOgqPzFO618Uppf2+4tqBIj0ksltm989HEc58S/UGl5KaR2ne3Y6NVPngMC8P9jmm3fIfAdHvQ78qBlIkP3VrfW+7DYOOvoVQ4Cwjfb8wZTwXuBeLHdiyNnwnywEARcoxNU/bpcNI7KliK2WWXxZ3bAvQVUzRmHBi5O+LA0EfTajmGt6IXz6DVaAFxjJ/ekznw40ybnI4grwNOB3t5TWCG2AIelc+2h8allM5Nsoj43uhIgVGhqzqTYW1dHyROnHIyNQrX3xk7tPQD5Ux0U9zDVRGKuJShQgUha0018PdNJF+wFlO8dhxjhUEncHKjFGcjL/jx8awFRuR5SO6tKw3LQXzVxGXqp/0wlrtPwJhOVVOqQEa3FjQgXBwE= diff --git a/build.gradle b/build.gradle index cbf1a63..57996a5 100644 --- a/build.gradle +++ b/build.gradle @@ -10,7 +10,7 @@ buildscript { } plugins { - id 'nebula.netflixoss' version '6.0.3' + id 'nebula.netflixoss' version '9.1.0' } // Establish version and status diff --git a/buildViaTravis.sh b/buildViaTravis.sh index 9cc169e..36b9f97 100755 --- a/buildViaTravis.sh +++ b/buildViaTravis.sh @@ -1,22 +1,25 @@ #!/bin/bash # This script will build the project. + if [ "$TRAVIS_PULL_REQUEST" != "false" ]; then echo -e "Build Pull Request #$TRAVIS_PULL_REQUEST => Branch [$TRAVIS_BRANCH]" ./gradlew build elif [ "$TRAVIS_PULL_REQUEST" == "false" ] && [ "$TRAVIS_TAG" == "" ]; then echo -e 'Build Branch with Snapshot => Branch ['$TRAVIS_BRANCH']' - ./gradlew -Prelease.travisci=true -PbintrayUser="${bintrayUser}" -PbintrayKey="${bintrayKey}" -PsonatypeUsername="${sonatypeUsername}" -PsonatypePassword="${sonatypePassword}" build snapshot --info --stacktrace + ./gradlew -Prelease.travisci=true -PnetflixOss.username="$NETFLIX_OSS_REPO_USERNAME" -PnetflixOss.password="$NETFLIX_OSS_REPO_PASSWORD" -Psonatype.signingPassword="$NETFLIX_OSS_SIGNING_PASSWORD" build snapshot elif [ "$TRAVIS_PULL_REQUEST" == "false" ] && [ "$TRAVIS_TAG" != "" ]; then echo -e 'Build Branch for Release => Branch ['$TRAVIS_BRANCH'] Tag ['$TRAVIS_TAG']' case "$TRAVIS_TAG" in *-rc\.*) - ./gradlew -Prelease.travisci=true -Prelease.useLastTag=true -PbintrayUser="${bintrayUser}" -PbintrayKey="${bintrayKey}" -PsonatypeUsername="${sonatypeUsername}" -PsonatypePassword="${sonatypePassword}" candidate --info --stacktrace + ./gradlew -Prelease.travisci=true -PnetflixOss.username="$NETFLIX_OSS_REPO_USERNAME" -PnetflixOss.password="$NETFLIX_OSS_REPO_PASSWORD" -Psonatype.signingPassword="$NETFLIX_OSS_SIGNING_PASSWORD" -Prelease.useLastTag=true candidate ;; *) - ./gradlew -Prelease.travisci=true -Prelease.useLastTag=true -PbintrayUser="${bintrayUser}" -PbintrayKey="${bintrayKey}" -PsonatypeUsername="${sonatypeUsername}" -PsonatypePassword="${sonatypePassword}" final --info --stacktrace + ./gradlew -Prelease.travisci=true -PnetflixOss.username="$NETFLIX_OSS_REPO_USERNAME" -PnetflixOss.password="$NETFLIX_OSS_REPO_PASSWORD" -Psonatype.username="$NETFLIX_OSS_SONATYPE_USERNAME" -Psonatype.password="$NETFLIX_OSS_SONATYPE_PASSWORD" -Psonatype.signingPassword="$NETFLIX_OSS_SIGNING_PASSWORD" -Prelease.useLastTag=true final ;; esac else echo -e 'WARN: Should not be here => Branch ['$TRAVIS_BRANCH'] Tag ['$TRAVIS_TAG'] Pull Request ['$TRAVIS_PULL_REQUEST']' ./gradlew build fi + + diff --git a/dyno-queues-core/src/main/java/com/netflix/dyno/queues/DynoQueue.java b/dyno-queues-core/src/main/java/com/netflix/dyno/queues/DynoQueue.java index db9581d..1ded585 100644 --- a/dyno-queues-core/src/main/java/com/netflix/dyno/queues/DynoQueue.java +++ b/dyno-queues-core/src/main/java/com/netflix/dyno/queues/DynoQueue.java @@ -142,12 +142,26 @@ public interface DynoQueue extends Closeable { * worst case. Use mindfully. * * @param predicate The predicate to check against. - * @param localShardOnly If this is true, it will only check if the message exists in the local shard as opposed to - * all shards. Note that this will only work if the Dynomite cluster ring size is 1 (i.e. one - * instance per AZ). * @return 'true' if any of the messages contain 'predicate'; 'false' otherwise. */ - public boolean containsPredicate(String predicate); + public boolean containsPredicate(String predicate); + + /** + * Checks the message bodies (i.e. the data in the hash map), and returns true on the first match with + * 'predicate'. + * + * Matching is done based on 'lua pattern' matching. + * http://lua-users.org/wiki/PatternsTutorial + * + * Disclaimer: This is a potentially expensive call, since we will iterate over the entire hash map in the + * worst case. Use mindfully. + * + * @param predicate The predicate to check against. + * @param localShardOnly If this is true, it will only check if the message exists in the local shard as opposed to + * all shards. Note that this will only work if the Dynomite cluster ring size is 1 (i.e. one + * instance per AZ). + * @return 'true' if any of the messages contain 'predicate'; 'false' otherwise. + */ public boolean containsPredicate(String predicate, boolean localShardOnly); /** @@ -161,12 +175,26 @@ public interface DynoQueue extends Closeable { * worst case. Use mindfully. * * @param predicate The predicate to check against. - * @param localShardOnly If this is true, it will only check if the message exists in the local shard as opposed to - * all shards. Note that this will only work if the Dynomite cluster ring size is 1 (i.e. one - * instance per AZ). * @return Message ID as string if any of the messages contain 'predicate'; 'null' otherwise. */ public String getMsgWithPredicate(String predicate); + + /** + * Checks the message bodies (i.e. the data in the hash map), and returns the ID of the first message to match with + * 'predicate'. + * + * Matching is done based on 'lua pattern' matching. + * http://lua-users.org/wiki/PatternsTutorial + * + * Disclaimer: This is a potentially expensive call, since we will iterate over the entire hash map in the + * worst case. Use mindfully. + * + * @param predicate The predicate to check against. + * @param localShardOnly If this is true, it will only check if the message exists in the local shard as opposed to + * all shards. Note that this will only work if the Dynomite cluster ring size is 1 (i.e. one + * instance per AZ). + * @return Message ID as string if any of the messages contain 'predicate'; 'null' otherwise. + */ public String getMsgWithPredicate(String predicate, boolean localShardOnly); /** @@ -261,7 +289,7 @@ public interface DynoQueue extends Closeable { * Provides a peek into all shards of the queue without taking messages out. * Note: This function does not guarantee ordering of items based on shards like unsafePopAllShards(). * - * @param count The number of messages to peek. + * @param messageCount The number of messages to peek. * @return A list of up to 'count' messages. */ public List unsafePeekAllShards(final int messageCount); diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index 6daac93..470eeda 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -3,4 +3,4 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-4.10.2-all.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-6.8.3-all.zip diff --git a/secrets/signing-key.enc b/secrets/signing-key.enc new file mode 100644 index 0000000000000000000000000000000000000000..5c42699863e3eb25615bf69e4ca51f31bcec866a GIT binary patch literal 6800 zcmV;B8gJ!OVQh3|WM5yw;QEdAjO>dq)B2kYah>UM^=W(eGBS!&cZc!J==fj(W~Bd$ zLu&$wNsW8)LY&6TXlCq5UvBs=0XWLW;9KeZ{CNgzaj%oATp9Mahk^Q<*uVry=&+R% zA*tVz!WGTd>t-u7E8T&hm$tv`lb3)m=xi>OM=v$q|C6ccnoC{|nDFStw|h_Zy)$WB z82tR?sFT5pyl+r{=NrFRwI@H;;mb{bD~OQS4PGfoOP<%+&lbYEJbK;-sL;R$tZa0| zn5!8S?Af@caGts?ypOxF&*uWv?#zvJcDq~P_FR5(F}Kp#?`FffKg{S!(Tc`BK!b57 zQ*0^OCLl3UYcLe}>kxeWWPr%E@H1d_;o@dyy#720p0CeX2!`92^;JC^O_sa}?2B25@)1n#fv^th5 z{ATnH3o=DQg^)|B{yIy?-$R}L83Db^oO~<^15A%#%%Fx2$=wO9T?3I$dZRFTHd2Wl z@QSF#W9Y;5iQ}kV87go)$Ld7)2fx2IJhPaz>?kbt7!@hHIB}G z7WIcZf-u=gF(y??sRIQKSe(>;BRqs`3QoI%=vPC^3QNf1t1N^VhhZb*b0R$!t!UH# z+eE_RVir3vd8M3`B*Cma>gT*7kkG)zVs)@nQlkn^U?2~K^`qup$~ZUyA51Io%sQEg zY6ai2o9+$7iL6D7od6fLc{2%n-Pc?y?o(V6ZsL zpOOcyvNJ@Q5wli^wnYw zZjQ5+dWbA9V1*&T&PJ!_2sdnz(g>A?Y%r^+cR0h<-~C|sISt0vG}1CvkmL}H=s_nR z#+El=cP|H9jejcVv6zpC_oHA=sKIthYIy{@f8;?D(aWvv9h1nV@4XFrD#12@ix{#+ z%gm^F8apD|2gXbK2?6?l5+O=ohKxLCvZXd(r^Clz zp%Gg54`*-gx#cJ@Cseyko%ti&K7TTcP#a=7ge`7cO{^%Sa6k|MKk`yChKa!@8Eshhg}r~1Jn|Sf(iAe?-2xedB+~_VqCNM87pnv%y0Y?3 z=)fGO9xkIiTS;N_R0CfJ7`IRhpvP62p$ZmAVU~XAvn53sp0`e?B##@ZyzBN5_n`KX zt=37`c7;z9|92TTfr|BVX1Mpc{{Mb6o0%B)uy}wi?-i$gFl8VWvv_y6LjQZ+2t^w9I~PD)ZydAv(MZ`f$be)+i>zZ2EYhym>G zf8l#eB5D_k?w(LnW_1-BcB!OYU~dbb4aqhlm73iy7jBkO1HU*Fu*{#`9N}cPQGTW8 zOeDhmeO+&EqFURgG4@g#V?FKbYKE8#>~c>%kPfF~7R0cjR_acnY$ayL)h;l+_I+pcusln=imhwAI996Na!ENr>-;~`&S(qZ z*$yU3wdHDQP=bL(JeQ?0pYSSpl4MXZC#gJ)%iH$*Nej(!MSGgZ$AmF%SB^3b67&cL zU^Jacj`UZ|=z8j~4Rc8O!PoVBSf9xz-zqaUvoBxlKytzaA%pSWGS_%3f;!_#UcSA` z6HUq0s2zu~FGZb>kpQ=3w$mhMapQ{!o*oySg0K@OGANqo+6x5xk?ORr-GbWoexWQ9 zfM`s82h)wqo$SN9k8XqFlS80#eb7C~m1`ZISmAJVuLBPl|7HN$TlLPylFI|ESyMZm z4SkES24Qxi3bv9nbh0;dgo;9IR#oSmFTJ*yXZX- z;3o;{-2YduVaZoilN&pQ#Yx?Qtt!r)bV6s)OAQ+|;F!WR3q5X(uWbja1mQ3uj49fg zWCFd*C!Ilvei)W)py45I$mPVWQXC^_wGl4530v=#&Azzb*&53M2AN1mfC+4*wo?zB z=EZj^TaQt>y8K6pJmU=CWICUnCcJ`*bT% zt<}e`^;x0<0Dj&}{QSJ2ja_eSZzugOuTpzY8sWY#WsMYwFlNnoCBJELEfGv1MXoxObtV9%6MGy*O^<4y!Y70s`Uro?QVMa;xFPiWF*2jV@b}Vm!EK zhQg{fA&{flv}rYJ6a^6X@qc3{5MYn7pXYZa+zK6O;O(wpv*B0?k2+&1SR|x;+CzV6 zah+0+_j480C-l#2Em^o#WbkHL{Fm)Pux{HPWiIkg?BZHk%4T-kJzqDfN8u8Cot2UN z$?A(mzo+3nO*5rZnSeYj#lMC8d#GhYdRMTD!k}kEHvg91`WT|8(&HoD;x$q!JDDs4 z6C4Zxc6^c~2z6`j*ED;~YDSI+6XH5}LR;|(MhgeUakcKj)z@&-Dd!5-kcj~iH_quz zslJD|J?v8+mfkZdnsYP zw9wb754G?S%_;yMjy#ug2k&{w#vu>ceK6K4jx(EXp&#;(wh-07e|(8Hsoyfxn2}KW zeF6F7_L|c}1C9a2ouaqlixbjaex2IMe6>{}qq)ypEy9jMJ$Tfic;JtodAAR_8{7Z- zE2S#m#jV>c`#sdw4~>2HT!g;q5VO+FtJ}<7Spw-_wO#1ADNHT6hFl1bU#~>p*FWTI z&4?1Ue`9wsocSk2FxI`FsHPQFj`pg>)l2Qk^(fc{X4|$3zw`zUe`vClm1!O81>~YG$ z4m?fid@-cwk0}q7qf}(bk*m8C((Dtp4%gg2B`4@!JW?s(Y z0JQz$W-s#k2D%$xbam$l9_MK4boM$Pih_zrq8V!m5x-LV&92^DJzTd{HV>!x`-?~TP_-)!I& zm-Kjr&+Bo7ZtkR&?QC!qlOb5ha|j79ffLh&wTZ78%&unJkgQ|uGRTX9HKEY63~)KR zzR447V3NP{_M?W$25EbrQkYm`IXL6eTjoQ|0JW8>maxKOS1+$j%jWD zOD`JL^CnSA-4QM&!tsiKmN&Y6#ss#LG|#-(r_(&RKf#F?rFfdEC>6q;*B^D7#c;#q z0;W^KUxd_eD8w55d#*k)3`aBxk8{2(zMo==v+C?>QbbpvU)b;n1(DjoKo9d@q4V4b zxGq2y!U#0XYm1B3FJwIV!CR=+fD*|+Lq<^pJKULeisX-iIU6EeS~N}yfP&7V zLFJWH-?b0{up_mGtjoz|WRJkcS~f)0TpXYnxH@cqoFo>uFWU^KbjHdcIhSBHbp|Sn zx!m5uZbLAN-hPNN6G(z<8QCV(y{!C@_%`k<}lg6CTaCyR!PD*p=j0|WDC@2>)9 zyTQMra^lcNR1Dn9GIo$D@8dgS^rw9lKMJ=e&~RJcFOE)fU5$HSSYp*d>V{EAfU|uO z{FJ4ROwgrO`W)DXw0y@+tXxTVW1%^1z_G7~t*kN37L~qF8DQoV$60!+ns)0`M{)&I zQzfF-seLfW$a$TT>x;&7Xi+h7oC$GJI?2a9n;VgoeXWb?cP13>!pk z{uFmO8`7Os`AVw7TcW}`8ml;x9G(v%vDz(TcDTE7iHGrQMBMzvXg%H&K>1*;0TS%u zw17|Q2n{#|({pZ`6kO3tJ$VrVEhNQID%eZso0!=6ri}0<4OmJZQvF(;{rv+xW`%+b zRi^l4hI|dz!v%mf8uBlJO_n31yxhp-Qd23yc&A3MtsI=AjI$jHt>`e>``fnj$h$VI zM~7(ICIp_Q<$$g{8qr8U`x;@2&)T)C5U>pBhU{eo979#QK7)1kuSi*Azb1FDwlP$6 zU$q7oz5_kAz+y|K$Wz*_h*L{8nL%V+O||7vWn?rn z;!f02K1twD<>|Le>}lgb*{AEF$f|da2%%OE`I?c3FM@`qr*H z1CA>6ItMg;yy9Y*OmW?=Dp#kZgkuZU5sFxgHgUjI-?57%joUU%LPUl36GV6dDZ^sv zD5qUoZ227c4S8Mtf*%yk4GrFK7EYB`i4r64^WddSx5}N*8vKV8zOXRggWEc^f~i{W zZgS_Ia07Uf__4Q4DAHw)P59ZF>5A6*R9mx7N(ZXlyLwTzT5wy}*dYuYG(#-S%R=U9 zp+E?ooKmm_Y|rp3Qbne80GRDYd|}92IapJkdNsC{2BUw+ONE2Kh_=vWp#UlXeZ*E( zMk0~^#*>+;-nR!jZ{M67GsA=qz;2ngr#=zhxLp>u`oIB3cubP74VtB{>vFE&Dx!D| z=<*;|`zqhh*X%R=AQt)@GX$X2`cdoTnGi3>IjTJqf5B>0pQbc`C`3x^=N~5|sJ)-P zi#%13^S~HvmUULi?)jf4#o0%r73C1l33bOUoCOXmXK#_1>&bJ?nasSN&Zy<5tDt() zpsay#LH0u9jflXR7scts5R7|NyG`1XxeBqnOo)Hb*-~wkeSf;9&|CT!m+v)J?eB1k zO{miJk6^7#>com(1ePA~c)vs=rzabF9pWaQb~nC595Vf>RqU1P4!x34tbwg8Nzd@; z)QX>g4b{c02b_~Wk7@5V#4^b9<2w{q2XIkPVe5y~Zjlm1(vnP{i5XIEh_DUwqE1ys z#26&5YxwnG`lL+8|_;GdKc zR-GYeRHNZaXDRUa(q15@51D#7g)+aKdv zbQD1Y!%eZ{GOz{ilDSxG#~uj)K~uRW+n_drKi*x_IAlvrf9ATk-}|+)3bZNcJ}adM z{b~5-N_b?Kp)%*Y(h0Lv$`)s2=rI@lWD%#>{&vl0)A?Ynq(aOKKKe1vASFpk3XDnw zm`rW~rTYzEfRPp7HjeN1ov+&lG2V)Rq+zOR z@@Tfs5zhDIO}a`dtDq*N=@4)z8XtXpvC2t{sx2-tcmm9Uu{iX@hS??q|X)^lTtEtYli_mL)N7s4%WM zQXG_$)AGPw%uQtCpB6yDP`0qJ-nN z-NxB_7j-ftF}~Ipd+1*79NA7#Y|d1yxP8GIO?WW0Ps%O;mcv{cb%Rp75HGD>{!$vG zIo|`EA_{I7*4sT>9qB^FkBaDz%poWad)c&cxL9otX#g+nffb*mGhkm{>X|@W=ZTgW zu=vvv7d&&66@~Y#CA^8p{QDaSB@mSI!v;uhRCUh;-Lnmm>ItLnG zLUyvfg+(Hdz9LQF{l?bJdn;c%P4o8WPN9d}OVVOJMeKiEBqi|_D*hzvTK*4zmRB*d zB`?DH&JpjXs{E1?tvThy8SSs3ny|L5w(X&-^op%ACmTSnw$}REVB(<9l_>VMe3}nz z5M@^aC=L3CFnII|E$Gn zd!7`0+3JoN?7DlN((ayTZd=70L19yiz1h!i z08DvvPwSsj#jKr#B>{PoYIF$IHwdK0rKRX-jK(3(#zM^`Qk3M6NfxEMQLQ*IcL#K> z-L|Q8qI!>HLwE*yO-6Y5x3UGNVVu~?Alq*=llF)n@LOn;r$N~Y*2~CYoM|HWY>Ya! zS>O;x%B0h0t>UD#!rwy>{d@gaWp};&=dC&RD}gx?_l3J>YgBopAQtth{;*v`=#-`- z24)totgmy&&k-HDG1~OZIeh(S7tZ*F=&H7)7uE8d-@dfO6p3vGA}*sUO?M=_1+j}Pq%zN-F^?$h%9Gg;ua7&sU~GFe2_GyZBbWD|vx0tLCzEzS zZ(w-2MJ{`IwecM`fDe4tQif0qW&bGz@}rn=-&a#X z-2zl@*{u$XM+!u5Jcl9Z6e_KEs6f&?GQW@ZKz~EnVm$z1f^{ Date: Thu, 18 Mar 2021 14:47:37 -0700 Subject: [PATCH 85/91] Replace JCenter with Maven Central --- build.gradle | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/build.gradle b/build.gradle index 57996a5..e0c7081 100644 --- a/build.gradle +++ b/build.gradle @@ -1,6 +1,9 @@ buildscript { repositories { - jcenter() + mavenCentral() + maven { + url = 'https://plugins.gradle.org/m2' + } } dependencies { @@ -27,7 +30,7 @@ subprojects { targetCompatibility = 1.8 repositories { - jcenter() + mavenCentral() // oss-candidate for -rc.* verions: maven { From e20195af69dd6eb8d88242a23cdf344bee74a978 Mon Sep 17 00:00:00 2001 From: Roberto Perez Alcolea Date: Tue, 20 Apr 2021 17:12:07 -0700 Subject: [PATCH 86/91] Rotate publishing credentials --- .travis.yml | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/.travis.yml b/.travis.yml index 3875ac9..fc4fd50 100644 --- a/.travis.yml +++ b/.travis.yml @@ -18,7 +18,8 @@ before_script: - cd $TRAVIS_BUILD_DIR - ls -la - git status . -- openssl aes-256-cbc -k "$NETFLIX_OSS_SIGNING_FILE_PASSWORD" -in secrets/signing-key.enc -out secrets/signing-key -d +- openssl aes-256-cbc -k "$NETFLIX_OSS_SIGNING_FILE_PASSWORD" -in secrets/signing-key.enc + -out secrets/signing-key -d after_success: - redis-cli SHUTDOWN NOSAVE after_failure: @@ -29,9 +30,9 @@ cache: - "$HOME/.gradle/caches" env: global: - - secure: eEjRoGTY9T+eN7Cpym9XG7ZwRlYfTFYBI2SJHHJnpfNKLJs6Tw3kyuNz4j7TL5iTkUbgSmmE/Nbb9UDKrmLHO+HlmQ6znpVzsp/RRp9MqytROXirfvYYxJmWmJBJlUYafIp5cEQVFbsdbFh2+nyqansJe7JSNtXDofBNEf8dkT3zgrkEf6iy5QwnRPVJVtNjuCPFL1LV8pPY/IGB66zsCevpCdfATrOCVJGOF9GnmbObfC651Ig6hvNZxkbejxGmj7GTxu7xhgAdzFQ+Bi6wZ573vV8MwwGwo5PxbA8lJhmOUrFT+ToMKxitT0eRGsma/4L8zvev8R8OR9r+BcsbFfZrf4JDykcWztfbTJmlWiCOG09Rn4sRuB/a3aNfqokqXQMOLH82hK0/gD60wtCq7mXMsV5G3H6D93i/5+cJEkNlRBLM8cJiqV0GnfR8nMGTg9gOFsRF5HyzFvLrlYVOY8LdlKFJE3FX/rio0cNi0ilDeq2dmYBGnu0gG9Ey55eEFNDbE9YDa1vD8dQ0eBFfiib+SiF13mdk++v4Bl3Dmook2yfamg99x9zuJ20vRUPSq6OOzaFO+Wpnl/QMPs9VHOBHJtth1Du4WK1FsV5Gayj7uPdrtj8MA4KKLEKtWQ6F5C3AfnU2MQCd8VUNTJtmjdtJ5wS0mAPPjYB4KJqETC4= - - secure: ZRt6jZtYfkMHaUOrm5K1e7jlrx/a4pThCP4yq5u6q0iBO1psEMS9QIVKF/pvmWbtqy32g/Z0uS/2ESxRIChZJU5L/xZ9noO58xfBfKEHlBUjwFwlaN3MTAXp0QzSHFhYSg5mpFTUg1c5Wd0b6DDfeHKbGoWmjNEF5WJAhSZyTW92CfFka9T4pg1Z35eqXhNvwp4sWEVhOmt0PlarhJIdFxtr5eaodEbKf6icTo8Ep47LZbAisgL93P5cEesi6oBK6PXEEpK4mGsKp9PSl2J2V5Rrci5uNjQGLporcQf/fCLmpozV7H13qHsfn3wgFz8X4nQVHRpT3NwG9OkkbNBDf95nWMcDrSafhINJ+RhgO5OP0jldy32pkYiFK6cNR9XLjSRYjJsPN+ELTa4F5Ql98oNDbIIT+P9MT7Fco/Q8N1irgTZhFZXLgweqyGql+5Exw+au005b+drM/T2ssP5i173BVyok32KIW1D+5dfae0hat6OVKkVNc20u9sifbBVHhBqBFGBXrxZXMH+RDM8RRe7kEi0U33J2p2kKxqDWYuoT/c6BplC9i4+Ee5HNWeG0Hxn2WLK4Ttt7CdVwlS1U+1UFB2N7n1zX4zRh2dxKRRrAzsbS8YlcgawYEE+fQxGbkUnqJ0gWs0YU0hHcpu/sHNddhHuHekrx8Uten8A/Fbc= - - secure: k6WGpdbdfIyVBRoKJKuu7VaPEh5ux8fuaW9xxl0wG2tqWP/tfyWvxURNr3Qx3VwPNsiiTV7jhod1kFOzVT0LGanNBEP2N2qiJYTvGaRPLx4tE67HhAqeinnvvUro1k019yHCMgT5tMtVD1HEi7J0Jbi3vB7nVVql0pRFQXB0NrMK4P+J8DD3ClbKX0iie4wvwNNfNq03R4Vn3fF7CDih3WJafcDDdYqaipq9h/BqbkZ/x96DaOQW+o3+rSN976qo/8NvqmZL6cfyR1yiOagMISzJwwjWBW9rT9/a7LK3yOCUQtb+W2wtEN117ggRnFbFS7QLS0e7eLh2OcL20yg6WhVigpFOlrcK0BgdPm18ZGf3icnmcMODXGutxcEECpZF5BuHSNIpK4Fm2a4E4tNEvrEy8rwhqJVQBBGOoWpIHfOL+86RDd6sA0CYu4rdrB5gYROnQkaavYIyOCuTBUMSXaqIL/SKJi9Mz5r1Svuf4K9UQK9wQo7a9MFajlm8QWepngrqkC1G3AmqG1edYCVUwWx3eubf++gNhPV1oGTBOa2gqWQTp9+3C7qTmy3YhD6WUvUcLOOfgiURQhJFSxsS+aSQvB+2ZxVjxL5U6RSx0eB60CnDCZaRm/dLChK1BnBUaYJE59yvVgy7C0yW8E8AtR8VjgnTLHMwUrt2D8ejyXU= - - secure: Ey9NK5fxSVajZHGV8O41/ZEHRIu0xdp7FalhAdYaGAflMiuMvdP+iuwfv1RONVvFVMLfuBc7wu7e1XZlRfA1G5zIhQM8LNdckPWKZVhcHpbRka9c6IlOPArwQTiHfwYWlZfWfAK9akVXcc/1tK4ePU5JZDIAKer4xw3AvtsH302CkLMYIQInm8Xf8YSTVMfWtFTANA3pU6zbiDvJLTj4/NE/9tNeL0xoG0AGRnhyXb27N9nbiLCekweGLipuqogR5I1lYsK8VHFlkLeSbwjEcsbV++eS3cGSsDW557RfbIRQXE7cP7buIxyFzPgiNh3beDihBEv0PNkYR5U4lXxn2/RReOTaAB37r2BFN/WnppCDkWKqT4HPaWYnVlm+0XVA6m4gSVoC8EC7kvc2DseKUn7PiAVED+UeA/PCfRYk213vgDxKor3O4iRXmbfoR/AbLgs6AKYK6RHNrL/Be21QT2Wyv1yT+82TTzY3xkesz6+f+NQU21AvNahdvgcXGSYJ7NaGY8kDUvZ6NJtAdpWnI5H5l6PPiWRlmsIr/qWwmGLjHyEI+fdQJdJlZg639J5kzjHYX5brN6/Qjvc8++NmBg6OiIyJ7+DreEoco5npoDHTNOF16HpPZ89JIof8oq5bJKrvsxFjgQMmf2yWnS7nJRP8g0/0mkDoAyYb0N7CKls= - - secure: rVxtKHNl7PrBHtWnKLlOi3yhTnlMHWG9fZbm9m4eMcEf6pUnD54KBu5kJDuQv5dyFYcHoI9Pbzfhq0f4aAll37vThwXHpfYKRAAs8SkkNcUsmM6w2ALkM8Xz+WKsUx1x0U+wWvBFIErzyB2sDY7+QIzfNRCuS9zs0h+1tH9350AlurgLwuHQVjwVSmdEkicR1vvILap+RiDwpVQrqEB+imhIgH4G826NpkQjdmbBszEOgfCUeFp1N0MmA2ln+MbFhKWaoKDAdJlsHh2R5vn4rR2jl9VuFa7zgmk0hIycM2p9OxJuO//Nz9jm4h5mFsvirSSu0v25rWe+WU8tc3auzR6t3RkwTiizZhYH/CXRwPMeXIs0wvZmStkwwsiROHBboCV9HPjHG/98ziXjL2NbNP/EmJDj9wKqy2SmggVVxnNAE9DoIX73+Vk8aDEOgE/Xy3FZ838IsCVmodz1v01bFL3Hac+oL2Wd92DlOgtQaSIqoZE5XTxbtAGvFm1nyFhOCHv6B33IbuJjtMrpzqrZb+WpvPU45pJXFfqM5m9TflBOBrwersFKe1+zyw6oDUpMqzD3F6B1b0e4nVL4J8gn8iO8Am4/7Xk1bXfWrySkDF8TEEJV+XLrUcnQULDzjhp3IVYCB0tAdybeNVHJ0ReLEBBB3XPaRxc0BGqpghWdn5c= - - secure: TZFMcFcEkxV9puUgZh4tVxqD0biCmESRyluIcyzYfjbVGRbH2p0+oaU1NtMgGGyCfRaHx1UizjKAtXqsjoK/f31bKmBFC1CB9rWJZF8HHxekVi+fhCy7IL9I7dhs9NBl6pZe9SZVr4gmiAszFYfLIzh1BSg1sZf//M9p5DdMcQ6EaAn0iELq6XwtzCbzEL/mhLZyJYkySa0k8gTktopZef1yuk8E0nFAMafvFWGDmFriAe7HUMeh7i50RrmxiPGBCGycv2Xof5XDm7ppOtpBn/TMonY3C3I4y4fg71sW/uDFi5ndDUOgqPzFO618Uppf2+4tqBIj0ksltm989HEc58S/UGl5KaR2ne3Y6NVPngMC8P9jmm3fIfAdHvQ78qBlIkP3VrfW+7DYOOvoVQ4Cwjfb8wZTwXuBeLHdiyNnwnywEARcoxNU/bpcNI7KliK2WWXxZ3bAvQVUzRmHBi5O+LA0EfTajmGt6IXz6DVaAFxjJ/ekznw40ybnI4grwNOB3t5TWCG2AIelc+2h8allM5Nsoj43uhIgVGhqzqTYW1dHyROnHIyNQrX3xk7tPQD5Ux0U9zDVRGKuJShQgUha0018PdNJF+wFlO8dhxjhUEncHKjFGcjL/jx8awFRuR5SO6tKw3LQXzVxGXqp/0wlrtPwJhOVVOqQEa3FjQgXBwE= + - secure: jtJ2i3GtP6z6V9eT3dCXeZKPd/iSZS/AFmx70mxXukYXS6TNDA4SH65B9WIWlO9WkzJ20WpUrJWkCT7z/V0UjL9uXRo8eMBt7QPIV8/Uo4JB8R/uZeVEKHQnwNb6h6nVqcGRV5Z+OQk1wbfJtYVigxyGDSMT3ANl02KTPv2Z5I2GhViggjonnEh09iNtyf36xDA315molUh3dV8F8iAFCDuKax4s2uUuOWgHg9P9nKkvjT1Ekl00iS/vMsUQg1umY0WivGBFYmqGWR3QmD0se3DPA/ORkDoD0vsI5FFf3WLcJ4pS0gt1HAPR5lynMcJw2lfwAypRDP/h3xC8AsKvwOJZbkdKi6MZTi69C7bRSpTn42SIF++qoPCcEU9MC26AqejOy2jXCqe0qgmmonc0hiQ4FIduJ046+YlDjJGa/xuKP9IOYlS65RhMkFrfDBzmPlfIAV+6/NkZ37A6pj/8Iwl4pi3TbQuiNqadTWULedlS7mox3d/eswkn7SHLBQ5A/d5cH7tHkB/6xWro+enXPrgtnsFm6otWk18lZYFvMKxLDo20HAOPJdoaSe+VLqiRnX/FIYrLTxZFDBZ+zMAu6ex/3gzCsXkTVTx0f1QX7q2u2m0R5mgmwRWRqjgociP+drSpiEdj0Ye0kQQkGrJATTIAYHy6hSTZ72NsBCNEvkk= + - secure: n01LBFj4j+GguGgWknTH9SHJYoF7vbFX+2emL1GojFYQTZwzQmBjGgkHcw9HO3rFGlxVBKE3rVFvfRC0Yk5U+edKEMnfFh44521PbNjHEl+kNlJRwWoUrfGod7Fdb1AV+so+3dlCrJVnflkAVLBjN7qNFW9yTRfVyJ4fd2s06q4OL/1vZIPdmOelXyXB+duwhoo/XC6jNQ0NVv6IEtgJdZvhgC94Mdkk78b/Yme6zTfHrTpcknv6YAEl+TpInIAyx/q27rQdfam5O9YPYUC99iXGpZPH1vo+mXHI7ICX7/nFVZqaJHlLCoInM8sEwJoEN66XCPJNA8ZzSxPf6VTQdYe1ZkSZjbcjP9kReGfXdlcmpXK1C8Ljj810ZIudQKzLJn4QW97KfVXaUWOsI8vCEZAl1r598oboEGU/Y1KJwGHujMrvn6zuCfmxMdbvl7f5sI160o/Yjg4TOz7hisJyRImi/S13G+WlFLOa5eZFDCoXhdu9Ni7kzfwPnyxvqmCodlFcQwZUVnkwXQCehcQsz9Up+mKOXNV7bZrocRNiyHZvG0ok5yW7bu5WI21FWc8/FLChXmTNEBQQb6pNIiQiMaNa40JQbssxvorHusI7QclOQ0v//56J8hAPJ3pCy3syAKnFpqhrnUOEFdPVYBMG9rRsWYVLrMnO3bSrUWSJDzY= + - secure: kH0koPpkkWwCHtCzXhHj4js/anq8bYM6wW6TV2AeujH9pGVqVJLsX15LYB8gfRfVKclTKTmNBFxKyfShQnFXeGGgX8QLf9rs8GJkPGKBCXjRlnbx3FzLpIJXys4vCuuVk44srAVFLJ3CzU1laQrlJvw9ecKLsgUY6aPMGcFt1cuI4kpJ5FJR91e+qKXbikYYjfQEMxhuIeoiVIhoFtfrcLTUdyCcJ3SW9V9KdQGcm+WTJr3HTRBWM/3bjbMYifYiUaHcz5cxffRPbdcE1QDw0Zza49MEDsq4wlkx3iZhZBGWUqH7GAX6vda+tdiR67k1NPLBLnUx1qRoJylYzlgT+JyOGz0vE1prxGH5M6JpKB0LnMgqmn/SR84tS8I3otNAET28EMCyPPyCCLl6inYEnSNWv+1J3c5li7WLgHxKKb+ZGqbVQSMyLKJxRgiZluh/Gby6vTtSUmokAyKsORUACN8sq9nqEY/P7rZIwKxH/7X8nIUj3XKP4QzL9taxr5LkwWEL5x/w4jNSj/72rhcXj2qEPKWp0SZRtLzfKeN/q/4ZrscIP5p4zCP2chXI/18SJP+ObCZAaMR9cUxpiKSPgew9CkQnIydhIfisV9+ov/xBdNg4bUvKvbu8kSxub1RAv+h6q/2JFIewzSte7D3RCXRnq4sCDT3ROIOaXd3bKxc= + - secure: fvT1mzchzSty+f6uXWt3bXefkkzRpUW7//GAEi9XuiEiIl/Tat/Nrtd5hmzhSOpHE72XtcwZq4dCFw1mhdGkM3WTwXOvRslFr9TnRsJLRJRdm3Tisn/lTBtJWCt9WF8Ax7LWbm81esL0VS4Uyj5udpbQgIe9E78rW0nSd1PxpIqm8N/s5g+Nw8pPZTEGwxAgbIXeWZKzT3Nv/HE/uvm4cifURwlQSsSn29fbIrVkQwbwFn9ZIh6z6WX4QNfQCDXAEgTmxqG+YvPjfVYsInLfU/GfnxkqhtWKV7TtlwID1lG5RJpTfclT87ES/cqga+MuEXjEAeBa055Q4to5kb4PcSVAbbPiBU9dO2xeQE5vhnEU6i8my8bhOzbAjrtOlKw6mRSlS8ER0Dqbw/ZPSbtlXRxYbGHOgM5Ql/g2cTw9rYCyuJzOXhcAF+1Ex7m2E8AFmpZxW+eK3NQN/OMZew3JmAEBMbUMHxv9XITLod9/ggnfCRGniCYDQ5UfCdCK8ZfJFrYzY/TXAhgB29eVvsd0xcD+XDbwx6vXwAKg+whGEmd9cxOZhPJZWOcTeVU6nGtTyRgVHnGb8P5cEoaf8Hw/85Paja/DG2/rcsyzZuqG+XjcwNVrnnqbsh8rlRXiTQMIew/gQEvcermL9KtmbnZA382bceGqSBZaQkA7dtgD/rA= + - secure: aFbzxNtLjlRgwWCOzrl06+OuCdIEcIW3XOoAMw02LSOusRMmz0uiZDLQ+sxRlfr7tb7/ERSXuUZLT+7KctzMlSxyho3Nq/RqYHcGUzHrB4Bv8BE+WoA+V0CYQruAAIms+VIoYozYJJyMsFHR7eM6Xso/LaNWat+0udtADSlYjQOFCs0fnTpEuy0cTxWcTbXxDPtmDzIl+0tRMZQMY7PKCBxxqKR544w/Q2xifwrSJsFF7fSa5ELQEEEm5BMMqRTx/7RfOXMhs41UrgL5QOWysow+SLwFNdlRammZhY5oliSX/cxN9G6tSRGRENVnQPJHps75lBBF/tjKFwfMeT7eFeqrcr5EFl/1+FgILhqlSYWnuEyxHE+F8K72o7bwTLxagoBovAOJ32qYI6VDgnCFGhMVeJ14AAshlkjM99yuahi4AR/V73oAbNnXsG2IJbrvp0SlFAGYJJ0cd2EyKdM2JvgRD82ZYXePYTpwvHm8mo88bmUFTCMkXkw6ieGUXin+6JKNt9KCixP/Uz+lNMXgC03Z9kJGu0oLHF/RBBl80eqw6jUgismDCC50OrTSGosnu/rWT0U3JE6UsQHgGIGLz9LZUYsz9yDh3HlYNAdAkpZfZSKFxsEkIbmmTMGIsROWm2Fi3HqY3Wqk74z57fpvYhPXtVotNweEuRaDbckv9wg= + - secure: QHXqKywwywvhUpyic/2aX6du3u5FjaYjRDzC9SnMr2Vr4oZ8gqsbc1YF+AoJf42k5mbdfB8IjHLBcRwtMRdU89+Dn11X5tXoGqBaBlSwvrBBUFCHycd8qJLRh0XD+dXopp42yGGXZTQTaSu2UFOGpnJyAI56l0lG1PB/KMY7ASM3uQ5sziRIwjzMUdUlIjpC33HyWm5D9ldU44rY/WL74i60BoGSkWCLcmCLleLrIhcysF94/cN74QOefPyw/cYghqE9MAdofpRcmci74agnyh+5ZGV+cXQx3FldZJnyqTecIyKJYPqgAzIFk1JV7r0sGuxisEFKptuPQmpdea92fPhMo+wqnVmvctGOd9BbKbqzeQGikXlP5MjtWvNktq/fiNEW3kL/il1/GU9W4camoo/EaslxMcjIGMdFsuhsV7sGrtpjTq2vjNT1L4HGhCcRbAItrGAuUY3nXeB/UM/nGWu1e2j0iZOeY0mMNzd1x+kqTnS+lgDkrEpGjGGYHfXewGuWJudOoegcE68uKtS5Tdubz4AY4T9GPo0n+TdEyzx5LMRYCqyVm/l4Rzf2NVhv0wUcijOYvVE2cM0GktL4dRS5/82cQKLOojNWVf6oaXO1YicWrD3Imy/yNlW8bjxbkLgFDJXUCqYTHK9hEayYhG33a8mxtgHrY5dd7MqmhYQ= From 2b193c1100bb7c4a0b7e32a93572dd9a2983388f Mon Sep 17 00:00:00 2001 From: Roberto Perez Alcolea Date: Mon, 16 Aug 2021 22:56:49 -0700 Subject: [PATCH 87/91] rotate TravisCI secrets --- .travis.yml | 12 ++++++------ secrets/signing-key.enc | Bin 6800 -> 6800 bytes 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.travis.yml b/.travis.yml index fc4fd50..0d1a778 100644 --- a/.travis.yml +++ b/.travis.yml @@ -30,9 +30,9 @@ cache: - "$HOME/.gradle/caches" env: global: - - secure: jtJ2i3GtP6z6V9eT3dCXeZKPd/iSZS/AFmx70mxXukYXS6TNDA4SH65B9WIWlO9WkzJ20WpUrJWkCT7z/V0UjL9uXRo8eMBt7QPIV8/Uo4JB8R/uZeVEKHQnwNb6h6nVqcGRV5Z+OQk1wbfJtYVigxyGDSMT3ANl02KTPv2Z5I2GhViggjonnEh09iNtyf36xDA315molUh3dV8F8iAFCDuKax4s2uUuOWgHg9P9nKkvjT1Ekl00iS/vMsUQg1umY0WivGBFYmqGWR3QmD0se3DPA/ORkDoD0vsI5FFf3WLcJ4pS0gt1HAPR5lynMcJw2lfwAypRDP/h3xC8AsKvwOJZbkdKi6MZTi69C7bRSpTn42SIF++qoPCcEU9MC26AqejOy2jXCqe0qgmmonc0hiQ4FIduJ046+YlDjJGa/xuKP9IOYlS65RhMkFrfDBzmPlfIAV+6/NkZ37A6pj/8Iwl4pi3TbQuiNqadTWULedlS7mox3d/eswkn7SHLBQ5A/d5cH7tHkB/6xWro+enXPrgtnsFm6otWk18lZYFvMKxLDo20HAOPJdoaSe+VLqiRnX/FIYrLTxZFDBZ+zMAu6ex/3gzCsXkTVTx0f1QX7q2u2m0R5mgmwRWRqjgociP+drSpiEdj0Ye0kQQkGrJATTIAYHy6hSTZ72NsBCNEvkk= - - secure: n01LBFj4j+GguGgWknTH9SHJYoF7vbFX+2emL1GojFYQTZwzQmBjGgkHcw9HO3rFGlxVBKE3rVFvfRC0Yk5U+edKEMnfFh44521PbNjHEl+kNlJRwWoUrfGod7Fdb1AV+so+3dlCrJVnflkAVLBjN7qNFW9yTRfVyJ4fd2s06q4OL/1vZIPdmOelXyXB+duwhoo/XC6jNQ0NVv6IEtgJdZvhgC94Mdkk78b/Yme6zTfHrTpcknv6YAEl+TpInIAyx/q27rQdfam5O9YPYUC99iXGpZPH1vo+mXHI7ICX7/nFVZqaJHlLCoInM8sEwJoEN66XCPJNA8ZzSxPf6VTQdYe1ZkSZjbcjP9kReGfXdlcmpXK1C8Ljj810ZIudQKzLJn4QW97KfVXaUWOsI8vCEZAl1r598oboEGU/Y1KJwGHujMrvn6zuCfmxMdbvl7f5sI160o/Yjg4TOz7hisJyRImi/S13G+WlFLOa5eZFDCoXhdu9Ni7kzfwPnyxvqmCodlFcQwZUVnkwXQCehcQsz9Up+mKOXNV7bZrocRNiyHZvG0ok5yW7bu5WI21FWc8/FLChXmTNEBQQb6pNIiQiMaNa40JQbssxvorHusI7QclOQ0v//56J8hAPJ3pCy3syAKnFpqhrnUOEFdPVYBMG9rRsWYVLrMnO3bSrUWSJDzY= - - secure: kH0koPpkkWwCHtCzXhHj4js/anq8bYM6wW6TV2AeujH9pGVqVJLsX15LYB8gfRfVKclTKTmNBFxKyfShQnFXeGGgX8QLf9rs8GJkPGKBCXjRlnbx3FzLpIJXys4vCuuVk44srAVFLJ3CzU1laQrlJvw9ecKLsgUY6aPMGcFt1cuI4kpJ5FJR91e+qKXbikYYjfQEMxhuIeoiVIhoFtfrcLTUdyCcJ3SW9V9KdQGcm+WTJr3HTRBWM/3bjbMYifYiUaHcz5cxffRPbdcE1QDw0Zza49MEDsq4wlkx3iZhZBGWUqH7GAX6vda+tdiR67k1NPLBLnUx1qRoJylYzlgT+JyOGz0vE1prxGH5M6JpKB0LnMgqmn/SR84tS8I3otNAET28EMCyPPyCCLl6inYEnSNWv+1J3c5li7WLgHxKKb+ZGqbVQSMyLKJxRgiZluh/Gby6vTtSUmokAyKsORUACN8sq9nqEY/P7rZIwKxH/7X8nIUj3XKP4QzL9taxr5LkwWEL5x/w4jNSj/72rhcXj2qEPKWp0SZRtLzfKeN/q/4ZrscIP5p4zCP2chXI/18SJP+ObCZAaMR9cUxpiKSPgew9CkQnIydhIfisV9+ov/xBdNg4bUvKvbu8kSxub1RAv+h6q/2JFIewzSte7D3RCXRnq4sCDT3ROIOaXd3bKxc= - - secure: fvT1mzchzSty+f6uXWt3bXefkkzRpUW7//GAEi9XuiEiIl/Tat/Nrtd5hmzhSOpHE72XtcwZq4dCFw1mhdGkM3WTwXOvRslFr9TnRsJLRJRdm3Tisn/lTBtJWCt9WF8Ax7LWbm81esL0VS4Uyj5udpbQgIe9E78rW0nSd1PxpIqm8N/s5g+Nw8pPZTEGwxAgbIXeWZKzT3Nv/HE/uvm4cifURwlQSsSn29fbIrVkQwbwFn9ZIh6z6WX4QNfQCDXAEgTmxqG+YvPjfVYsInLfU/GfnxkqhtWKV7TtlwID1lG5RJpTfclT87ES/cqga+MuEXjEAeBa055Q4to5kb4PcSVAbbPiBU9dO2xeQE5vhnEU6i8my8bhOzbAjrtOlKw6mRSlS8ER0Dqbw/ZPSbtlXRxYbGHOgM5Ql/g2cTw9rYCyuJzOXhcAF+1Ex7m2E8AFmpZxW+eK3NQN/OMZew3JmAEBMbUMHxv9XITLod9/ggnfCRGniCYDQ5UfCdCK8ZfJFrYzY/TXAhgB29eVvsd0xcD+XDbwx6vXwAKg+whGEmd9cxOZhPJZWOcTeVU6nGtTyRgVHnGb8P5cEoaf8Hw/85Paja/DG2/rcsyzZuqG+XjcwNVrnnqbsh8rlRXiTQMIew/gQEvcermL9KtmbnZA382bceGqSBZaQkA7dtgD/rA= - - secure: aFbzxNtLjlRgwWCOzrl06+OuCdIEcIW3XOoAMw02LSOusRMmz0uiZDLQ+sxRlfr7tb7/ERSXuUZLT+7KctzMlSxyho3Nq/RqYHcGUzHrB4Bv8BE+WoA+V0CYQruAAIms+VIoYozYJJyMsFHR7eM6Xso/LaNWat+0udtADSlYjQOFCs0fnTpEuy0cTxWcTbXxDPtmDzIl+0tRMZQMY7PKCBxxqKR544w/Q2xifwrSJsFF7fSa5ELQEEEm5BMMqRTx/7RfOXMhs41UrgL5QOWysow+SLwFNdlRammZhY5oliSX/cxN9G6tSRGRENVnQPJHps75lBBF/tjKFwfMeT7eFeqrcr5EFl/1+FgILhqlSYWnuEyxHE+F8K72o7bwTLxagoBovAOJ32qYI6VDgnCFGhMVeJ14AAshlkjM99yuahi4AR/V73oAbNnXsG2IJbrvp0SlFAGYJJ0cd2EyKdM2JvgRD82ZYXePYTpwvHm8mo88bmUFTCMkXkw6ieGUXin+6JKNt9KCixP/Uz+lNMXgC03Z9kJGu0oLHF/RBBl80eqw6jUgismDCC50OrTSGosnu/rWT0U3JE6UsQHgGIGLz9LZUYsz9yDh3HlYNAdAkpZfZSKFxsEkIbmmTMGIsROWm2Fi3HqY3Wqk74z57fpvYhPXtVotNweEuRaDbckv9wg= - - secure: QHXqKywwywvhUpyic/2aX6du3u5FjaYjRDzC9SnMr2Vr4oZ8gqsbc1YF+AoJf42k5mbdfB8IjHLBcRwtMRdU89+Dn11X5tXoGqBaBlSwvrBBUFCHycd8qJLRh0XD+dXopp42yGGXZTQTaSu2UFOGpnJyAI56l0lG1PB/KMY7ASM3uQ5sziRIwjzMUdUlIjpC33HyWm5D9ldU44rY/WL74i60BoGSkWCLcmCLleLrIhcysF94/cN74QOefPyw/cYghqE9MAdofpRcmci74agnyh+5ZGV+cXQx3FldZJnyqTecIyKJYPqgAzIFk1JV7r0sGuxisEFKptuPQmpdea92fPhMo+wqnVmvctGOd9BbKbqzeQGikXlP5MjtWvNktq/fiNEW3kL/il1/GU9W4camoo/EaslxMcjIGMdFsuhsV7sGrtpjTq2vjNT1L4HGhCcRbAItrGAuUY3nXeB/UM/nGWu1e2j0iZOeY0mMNzd1x+kqTnS+lgDkrEpGjGGYHfXewGuWJudOoegcE68uKtS5Tdubz4AY4T9GPo0n+TdEyzx5LMRYCqyVm/l4Rzf2NVhv0wUcijOYvVE2cM0GktL4dRS5/82cQKLOojNWVf6oaXO1YicWrD3Imy/yNlW8bjxbkLgFDJXUCqYTHK9hEayYhG33a8mxtgHrY5dd7MqmhYQ= + - secure: huCeSvKQ168V/v8Em3q9VxjEPMPQV6hEqbdCSDHY+4x/0rxQ6DzhLrlOkS7z8+tfWsnzK3mhcpYOF3iLe1wX1xtTe4k3wuTYjF7rKLgNyON7luBglHapDa9/bRr8OLWc9EZwxZt1J6LgZM6DVsXEcjO+tJL4Mce3Y6Knz4bgNc36fvjg4mhQ+SjvVG93TWjOYqqHqu1QH/3dq31LWepBDQ009iy7TUmMOkAuWb6Fuihk333WzJi8aICx1BXTJCDfWrzpyy8+on65b0rtIdFMRUQmF3gR+uV4RRAatKhCELSInW45146gMMugewDwJScHMbPVO6MCtvQuHJeAipeQa/naPWspAAmD9Oi5jxt31YrDsLk2k3tLmrG5S0CcKuXMAEjSVWfJAFg9N0q3+TWuJ3dEcmIcVfJsQ90kPMHkMi8GnQBEGvTond4ItsePAycRcFhtXAWTdgcadEQ0N60TtHopUtKfblPp6SxKEUNr4pGxAOmUIRl44HsY27O/Nxm0kRd8RqrFY2ZEaKvZtwVZIPEzozRdF35S36hhl7FtEjOdeyvxhot1vNL1WrJh4nEGAhqhHvGiD6nHbMvXjnXpWFQn3icteVt2IU+SvQ/Qkq5kDm4Ytc4PkSuavlmkMSUDSrsOA/IRAuj/xWxkQCH5MO/oubIt6ka/Nn0lS9NUjIg= + - secure: nGBmnEARBcVOn4Eq1rbaCBCLqayx+bBv1MBmoK1fig95dw9JN5JVfYcSj5t3IG88uaaE995za813AXPZlNVs4+9Zr3Dt6JDnyF+FVR0m5fhJa46nYiOlyH6WFr7qCd43UEBn7XIj4E+v1ax//bWUpRy+vNB8/n7xjyxJ1rfoZcW8+lDjz7ijwPKwxr0A+6wkJsTb+h+3gC8MVbUMQnNCxOnDfFBf9XaoXSX4YqlMMSK22gGqgqIUxQ6H5p7NXexM/T3rvhnd1H9U1YHI/PSCur7UAjXcEw92vpI+DV1/IHAmbUviFGf2lhVSLmp1sz6RYBlB3fqwyOJDPRRcvaoV/ZVmHYsDK4RsMvcBzGNc5Y7w1ZyQpyiRItjWNLuO8I6sZo8ELCAz6snEiG4ym6E7IKLjhzxfd2Vm5zCWlfGUAORhQEm7ooZVm2x8gl99rzud/c87xbE0Ju0k0Zs8jTFR39L9JTkY8eHiMCjbNu71yumbmsLuiI0fvt5hFpsgWIWWYXs3IzvN+CFFJyCxNkVhZXAyy8JPvSSencxzfgV61GeemmEerzFpfREDYE57D+xNfNL6ib2NmHVDcinolvPb7NBYP+DEtZiAZoopA70fjzsndcH/EKTQb1X5ccEH9hCbvn0UVejQgsm82kUUhG1BC24QSMIftOnsJRPzyH6pz4o= + - secure: AXhVr18dG+qm1IbcTPYkoUS9q3/BIk4mDON7uf7YwuhAYy5r8RY9QWOeYAwT2yL+WaUvzGacpsYc5NKU8Cwrndv+f2qJerUCk/sgS1WoiGut377RvOQk/rOZnNvxi24K1T+iMgSUS0OQb4HIpSciKYGB6sYKmrwpXJD6b1tOp3x+P7NWv4b4rTamWlyf+NF0ortha91fu/KBt0vfUJ8GjcUKzLVLyqcP6dCd3xvUUWWYJZaffMn82Dh7PnhnxR03BWhVwRLU9lw7W90OEX5rcC2/mpcjR0etscnZl9VX5BbwUmJcPHVkEPVQGSfB7VxwcXpwfaq4uh944uJYM/wKJ0pAphq53M4fyHvoX6xPVmf3PN9AELMFfe+KzoStON627AXmsdWbaLJu3/iB9SdvHwETjOjrejgbsAxIC4B0oPQI320d8RhBF6xrkcl1oC0dNH/fZZAuP8/YoA6HzFGXeFSNx9TqtR+nSUxLYZWXAS01IAcKEn37aMhXTmNf4QkZViN3NI6HgM24WQsA3XpQZCZeDcXqOEdTPZG0OMJ3Jt+OfpNeDVfw0+m0Ir5nZEjfLo2lSMaL1Q1McWNdY86ZoJdHNBhUvliKvYf6oHJ4rnkUA8Q53KoPIEMtmSDRZKFj5et+RFfhqgR7B1jsIzu/2OEYS0aE0INdeBq1BzjvFWE= + - secure: nRQooOJYmuudAbqexGR/+GAJ7fSu3JIFJDqyfmIsZDb4cg6DlUCkr8rrSH9DFyR5akFhQNNMiR+eCCSG90s4GJImHIAhDRdodrcAFUIHAX7M94b4b6TD0wTwhzXClZRcBHt+VQ/EaYAP1N9czOsw//DPV2oQ2KNGAR7gpMmBSZ2Tw5Kiwv3zkBJr3ttOEy53rhgX+DskTtut1g5Cfq8rKBaGtS0AXNBX/aJyVWRMdlaomebbs7StKEESf5x5tus38iTwTyuQpHOKGD9oFDxaVnmw6NrAPTfVT+vEwhSttAoevG7nZrsjiCy5EoJEKxL6dUKttT494Xl1G9I7ubxZf4mkVnBsVL1GDYiRv3bYsfdkIJ38b57vCodmHebtjjBiRaC75iGn6D/fDIWTR8KIqOZIuL05LLKhuBDk6V5xifiw9zCcRNeZUSAX0q9LC+V0RLxExIBouzYKwcmv0fE9KBhvLPEfOpB4THBACLX1FCRybijNwqU5bb9T8SnFVH2Ruk9Ocan9sEvtaycXGrLlnFIj6DdHY4uaUlQzMfQuqFdVo1gu9GneX09oIMjKW1tN2MyUcBEBZHZkN+LQaa/Gt7c7TR3kAs9yyvDQffGYtC/MKyefnYeIFba7LiS/8eU5TtkvMcuauzq+yeuNhWKzY8NzLEz4wq1okz3iSQIYvz0= + - secure: lAq40Ixu+vQG5NfomDwR4Yld1+9y1BRFMmgqe75vcB4JSoJU0Kf+OgUvHYt2Q1OZOrHllD3hYBzx4JNUc05rQrDd+wDF6r24fay9JW6FcVv8yhfhukU0WE4AhjFvBS9UQoWloFYHXFGcofuSTqShoumTuAW+NheQIj/Ialhoy7/D+aezWMOhNx5PFhsEII0MqjUaoAeg0FI/ao1yoewONQNYpSiBHsmBOgzBnrjlYs0gWfxYMUOoj8rEuNrcX2kCCMdSGp9cy27nydPK3vYsJFUrlcI/SK+PBMQtgopPts6kRFxViFb3fgilKxYezrcCYmqeSkTU1CYS8J+5JahKemucaBesqX8UVAnywRCw1wx2jipbJcffhJa/+rOEi+8TUnNgbkHBhYa9mOix45zJR7YrvdptwAlRaan2zKeMVkrZdzvVOXnrLSsJN4vUviD6kh1whqX6OFKH0S2chMLbepKMkkM+L8qnHrsiDWCdYkAgETPcwM976uXi/vCN33D2JSV4JtFvgrxY+VoaKVuoMMNDFG6LptUuSOpU4h9N5e4spJ73/kmvJ9ZKrCOy/23EM5YTFFM/joestQB4eszD+TgaLLQCnebmth6sab2WAC4lseyJmYo8v1jfGtN+vhlN4hs1sweF6rY0aNzl0w018B/LDG5CbFQ7/9kv6XSvvOk= + - secure: cYwUuptciaQDevzUOOsulwDsRGMDOwhbS/Cma+MJMZ17WH7XkRNLnBiK3EQDzH3Kwz8DHDXMKHA6B0NxeNDMK/zo1hjndzutnhbYLSKTIjgLikvufyDddHBHRawMfKHwKgVQfc3FmoVWKtZIJtPcLpFpRsUOlpdvzJW3L+uHMkeoViHoCIvzMqSLNhTEGcdmgY+dgM3cX32ndBwgFLv9E5a23qCELX8icMmjrdtW8NpEJlxkcIa0yNpe7pX2h5d2JuwcWcwEYfKHC39FG2jIC2Jn07bECHQIm45y5msGd9M+Qq21hNdfvWzv9h2YJKlhZeUEpJkAwjQRDgBR1T9DY9427UFdRQ20JX1e8SNFEkFAvMs5kn+Is7QiVDWsOLUl+ZQgS4oYg8QyEvU+/srTQ2lSaiipDq6+QaAHRO3pSF+y3TH6uI6Q/fc08gaw1mtmj7OiU3lWK9EyIdgc60Nm4BOpAFkzytaYpJEaprU9WiqNjLjejpRiDjYNWLr3HPy1AoyTeqj83i10KRwE97gkc5mhoPL/dI85T9mgR2nOboEcI2ZNiMyIPHc8GV563jlKdTeALQW+KvbVia1NjtrC1PxkCTxdNeFkgeLCGB97W7o6swLyCo36EqLvxUPNmayCXE+GhsIH/n27PahtneinVr7urKx+l3U8+WifI4XmHnc= diff --git a/secrets/signing-key.enc b/secrets/signing-key.enc index 5c42699863e3eb25615bf69e4ca51f31bcec866a..1a4fdd1839ab3ee1d2646b50ec1745f76339188f 100644 GIT binary patch literal 6800 zcmV;B8gJ!OVQh3|WM5wgXVxXpOWpYCv!AZnF+elZ&kD16b!=0%o>*g)5|T4gNAXw> zFDGTeAdk!`LTS#$Vmm>H>eYM+D4^L790x{@ZOjgaFSio_f-hyWoO#iusa)_Y!)?+# z%dip+t;I2eUncxlT+9>$gLC?abq3DvO(5Y-w8*Rn5pls&uT9ETSHrgaH{uJS>E1Pu z=Ai0^n3sCz*fe8G-0IAovmOdkbsh{#RfKlaP%m>B6rl+|JUY}n+NkFq!VP-P7Lme) zg#N|eXoDl9-SvyxL~n*Yd&Z(nd0TceWIg(4`86rm06WnoJ?=5!x{6T+%GTN)?1mcYV&Nsxs3~Cv5Bm#UwFp` zbc7KN;W-(Q4aBa_g+dV`7?Q;GanyOM8UYnGXUZ^Z<5An(Q2wlk#!uvKf(|^aSJ&2; zx8!%Z2KVdMT;%#JaW8C{MCSAAI9lyY7oA%N-UjQ_7lVs`vh0_9UQt5);dl9s{s`8~ zQS#>e*?278y`tjo5(Z4E?Jr95^A>?vN?A;ROUjhml@Gw}h*}jl*xJ$R{7|eWzBRbW z7uv&v&NB26`*35Gw9<`QflJZV@C3oKEsCwmxIY{e?4zBGk4z@D940!yoYDign<&yF z_G(SK`*K%cqv&YTQHL0*tcS_xWM7cKq!P9hp+uH%P!NkTU!lwD^!s`fA9Up-cCge! zX9iS#(73ddUUs`6b4}cri*FI7tpoTSocOg2W2MF2hXdhD_nm|BNGN<6jS;ND@cXLs{WTy6j_|U16OGY z=ikMe=FJf1Q_$tb=TyB|hEw78`I711WsVdylv`TXGr~k_vus*ygGJzZ+A)NCd_dV` z?z^fB6_{l=I(bIIA0OB*%AyzMh!H>$kLPsk^l}W)e{$DbH`HV^@ZzVhzpj%4w zH9^e4M<}g&lbK&OAb(`8@k!9fi@tN(yX2U9-$jO7b7Z?jN?&|WTJ4C(xqOo~z(A$A zs)s^a6EV%agyn4xMY$8IEq?M5xulBeq_NRmB%LVl!!vNo(mfJdF!Nv-dgy}~*clh( zVl9h++?e4zrKq-oRMws<2N)`cdP3nxm?W*Z%*5Y-yLW zzn0aW6UlY!&c|A%u`e|fGORgEXUi`QJWQtwu`hmaMtzW8()Zd#!TaI%i5(@Lahs<| zqSJB~cFcBlhN)U4UNaG3cz)%HbwStp`7{-sv$NM*FKQs3a7Fi_O}R zn49{Fx~qs>1@kRz6l$flCn1y*$tnHZ#;~z2e7o&4b$)bE?zBXV z-nl5LuUkX~09|Xrmr=dl64XM!;yvpQ z8EMuh9%-1W8!T8DP5@gyaIOVqQZ9Gt{fI!J%S2xgw)TWha<=SEe+Tc?#!PvfeQ5yE zE#m^@-y9JvzZ74cQmu#2)KWZnV89w}27iW5;Jjf!{TkLQ1(eGG&d_Ug`h$pIafq0m z_6FPULne~^x5h274zUmb9uL)LvU(OV2AU~?2%`904Ycw9pWm57}$+gTn)^3E+YoY_Awl0sWdW&G?bo(q}?w3vvkg~;2^lKizQ(w;f zK@XP6M({?~{QD8^nhqX8&MP!EQb7FkqCPN>c%5d|ZND|2Hs~lONNXP^noO!yrsf)w zSzuaM4X`u)YVx3y$9r%}Ty_^i{!`)Yd{GiyqBjsC1tGWpF!Q`D_KO}r$)7&an($$d zo0NJ^Vg^3NLnxEX^lobIKf}6ZiuEC^!H@m@fekShK+q@s<}c=X*cSWzB|?hU40Cb5 zyADGu?db}uSTW1I*>2tIZpcZPcki-!T{IU|qyGwbf5f6fCZ4qZIA7<-B^8ZMfC}-< ztf}h>0>J*SxMrPS>rZ~$1p8Huw`xYK66}b>GpOvFi%C@ALDI$bMfSM=gB6iTBGI76 zggVstOcyaYKIN${;;zyX?ckk)=epOgg!Tfncint4cHL?s_yUUZyeG7-0}LIs5WaK# z%=|d3^+*9l2rHk#?D_N?5tBjDi35Kh0WORbe*_0Z=dViIaTU|f8&WR?FI^4Ga%-;L zTIdBgu#$tm0$cLCTbD+M_xX|(l|70kETrd&dydq7++<%1=DUCg9wW+!Gov0ENw0_T zy#oAXjHz_sQY*hL8P8@``%riCn#GFBT~J zR+aAmNeT&wE;dG|g^r22P{!nqC>uCA1`bLyNNy&!h64maIy3H^5`Il7DzMwK{l|1K zH8nhbTPr}lSA!oKbQnchym=bWN*Bna-VLoo{KBISGv1VwL=0(yPO0AE9NEVrSuj9O zl$?O2{kZZectjH9sWGhg=?vv#EJeYfX{qi&q_FeEHGtWsq%t!{1!%u(SRt_nxAaGb z)Fp;>S0-XDX(c6~;Izn2z*Ye4$hfzXE*_q=>i$_p zYjib7*G9OQggrT$lv4{O?k{oDQNh~8M7h~%?;Lr-&$&2Fkr!3Q1xLAU! zFOz8)~}7 zfl_XRhhf5UPE=0z0$Fi!Wb(Pg1X>dL~Z=#VVbz?R4Vjex!0P)r)J5em18{}VAzb9VgU&F zU-1wEU>8aDWFhzPr%0JL_SR|;5Rl3R`jR7)!bVsd~+gSRe8P>N3z z(|4bujq;R{C8XsQtheikP!`|Vjk~U3#d1I=|1oiIe4w_hGUh*1$gJ;3hm0kD2)wz# zA8}>(9TwSOiS4BymzIZLOI2h$_H674nv2d{1$`|a%S><4=EauZd&%YeIf{kf%AkIh zYrsa%gKO|vzRHe%SyZKdq;o2_5m{$zeB!BIH6^N8@)7Jn$U3#xCA?G=vMB`;dTF~d zZ9B3%Ka>$K{E<3X{0DE9+yF#CoEVudDjlEgo&_DDi*kXtNt5=Sw6d88dHfC%44L&m zL;UT>oV8w3hl~38g&q@d=b#Ff==O9Tf!d@}m}MZM*xbvbkAnV55R4r*r4nCk{mUwg z6q0LF?+4WDjQrduUO^d}y9c=dOat`LTyg#6J;7?HxzmE%ck~)V0+m_vlX7DUfYK3M zv=T}zip;Ebf6Dq&PB<0i;RGy=e8*?&9X`$&`l@Vb-%wDV-(L_cE;T&q^)!;XwTz-m z)n#D11`PvCS+p6y3$XVZPF`-p^dppGI^_cM`dB4yQOdWdKJ^$ZH58r@OP*xYr@tp2 zQmPIS*=%A+K-uEuP@*rjq_yUx^#Zu_5vWeQnx4XtwKs{*PFj0i1MxoXqf|%Ez(c zGM%cvVEf~Tge+K>AzPfm8OdHra4dlbRCNEe!C&E27LYmIP_6XJqcvs8Cl?BJx)Jdt z{WOyZ@Z2Qo=h`!BSovE7KYY0IdpWQTmwPPYIbP-+6Bq?Qm5J?H?B^IQZZmzi3PfKa zLFY_*vp8W3two@pXevtK=s8UQ+kna(95e0c6#cR?vVqZ1jvlN;#2UlughMWAgiJIhbTP%BKssB&JU7B)8x|`$oRVnNHHSwD z?gxmObtuZ{Rt%=2E!X%?GaNDc3d}!e4WNlEQ1MN?VXz6haX{5tTL0tY=+)WkYo=1+ zaqEiu&i|8c9nCtFxD8~Q+b|)lrQniL_SfoHexrd+^cvMAb{vSA>le>J78m)1xuB{l z(@A85S8r99r{DJF;IU-F_pIn;G(?7Ju4*5^sXokvjVoafCm}s1mO`CG{^ha|zaz6? za{|oXBVi#=Y!Hr15_+3?3ffF<2t=g8RDv86k4iX(vtx>f5Dh!}}1uf{;aE#>XwHRW%5s?mjeEb7L?_%~- zm&DU#B8%X{XFTA%_t`G#WwW=Hu+C*keb_K`4O^M`GRM!%N@Q^p|1T=mk)|tJ_EuEr zPZ#bBk8M^YpoN}q&&&bscSZ1>y0d@qE?tP?X37KPaLa5K+ea-YymOVN8oyg90@U%+ zrxy@x2!{i6o5K;XZl}7R;RRpM@#LoemHMY#Of2fW)b{wap&$?M6O6`b zFzP;Z8j;vHGIJnA-yBoQ_UtC}&xX+a^~!AcC5Vr%JSf2*yQ);O_~3j!U0p`Gb}#$==MDqjgcPMC-7X|?)7@We2) zeiq5VmK49q#K_JYlSgSwAT-_xeI|3uqljHPtR8xM5iokTYN3>8jN2?$cMVdokZ!|` zj+mc~%X60$RMk8=U@B*z=gJ0|g0hC_Nu4k-9yRFst|>{c{^baioGG20=!9+gMB8<` zF*1w9T8eM!)M_Z8$6G`1G1w6PC52owT5pak^!MN7xm!4nU!XjPJONBYJ8z7T8={g@ z07Gv?G4{d+O1?VCHR^lcI4N8(cyD1=ZU@$ER6@<&J$E~6fjK|pc+)B@9hS%*jEP%@ zKk}Gt!u-&ZQ>l~gTcNVKAMu%*6vi$ik8#>k@-G#ojf)ulDu@6Z3vWvP#eC_?vL5&k z<(*#m^AlM3dfxX0xe`flkwpeZ1CeHIm z_#kVHA1ht<==)poBDti15H_!tOW#APQaJrSTGgb;$DcrrZ_MxE0Nrckmh(3JEvPIm ze~11q+@KSY`MMlEiB_BL;Y54UP*yJ&{k1FkWJnD zGKZ}gp87F$0n$4#M3czKb#^Sl_Y^s{}Cgz3zTU42tD(ln>LB7qjYm1Oc(+bpqafIJ%~RrohK z8K;~jHr>gFPWkl&h<~?6jc_C-nzjkT8Q%v#FwvWtmqpODV9Lir97*+abTUmjwLUC}qek~v~7oAI;+8q@am z!dXp$!ECi(yJ4w0fxr7aYn(8jvzIE9%1+d^>)2%Pu+Q`XqsD1zh*3)#9TKV7Rme>v zoxw|c4{ESZWBDEtIe}<)s4rX%!SaPek+fjlphLW%8SYY3lqvm@t>?vnCi|>^)ema% z9wUlC3kAuq(3oMgGyK&Qm?vLEk~*m^Dv~IX&FWMlby=oKw%5N_w3igYec8Z3Ez;|} zK{O`cTreG%RJrh6(cC)+$vBvQ9Sk)n>aKUEc*S0V`(UmeXB-?r)Q=X{{8y8>HC(ih ztaG5E`15~f>eXf~WVV9-0gwo+HlQtLBmn4>xwxk?CEVB%`w{S1cja3}LL$`XX6@V~ z6T-&TNb(rVwJ`*u%jJp#*_~pg`i+F28uipb2F*_)sMwVe7_A)8!pcpA$e}3{lS}hpI!bt0c`VYU=*fJhrR`d zf5$*20)~u~pI1-9jvW_gXW7ZLfJ#8_f#~~QBl}JYZ6s5=yu%^MllyWQ5m!?>Ae{J@ zyFzoUKZQwgbu06Y?Zd5PjfRqX%L2j3*K{H+Osi+#2LXiy#RN6ycBU9+wmjY;rD(_G zp0l~+3z6MQr_B;*m|tcOJoGsrK~ihH^xnDPdzpZkAqEPuW=60HBcVOK3Wc0niCW|S y1bceG)Idq)B2kYah>UM^=W(eGBS!&cZc!J==fj(W~Bd$ zLu&$wNsW8)LY&6TXlCq5UvBs=0XWLW;9KeZ{CNgzaj%oATp9Mahk^Q<*uVry=&+R% zA*tVz!WGTd>t-u7E8T&hm$tv`lb3)m=xi>OM=v$q|C6ccnoC{|nDFStw|h_Zy)$WB z82tR?sFT5pyl+r{=NrFRwI@H;;mb{bD~OQS4PGfoOP<%+&lbYEJbK;-sL;R$tZa0| zn5!8S?Af@caGts?ypOxF&*uWv?#zvJcDq~P_FR5(F}Kp#?`FffKg{S!(Tc`BK!b57 zQ*0^OCLl3UYcLe}>kxeWWPr%E@H1d_;o@dyy#720p0CeX2!`92^;JC^O_sa}?2B25@)1n#fv^th5 z{ATnH3o=DQg^)|B{yIy?-$R}L83Db^oO~<^15A%#%%Fx2$=wO9T?3I$dZRFTHd2Wl z@QSF#W9Y;5iQ}kV87go)$Ld7)2fx2IJhPaz>?kbt7!@hHIB}G z7WIcZf-u=gF(y??sRIQKSe(>;BRqs`3QoI%=vPC^3QNf1t1N^VhhZb*b0R$!t!UH# z+eE_RVir3vd8M3`B*Cma>gT*7kkG)zVs)@nQlkn^U?2~K^`qup$~ZUyA51Io%sQEg zY6ai2o9+$7iL6D7od6fLc{2%n-Pc?y?o(V6ZsL zpOOcyvNJ@Q5wli^wnYw zZjQ5+dWbA9V1*&T&PJ!_2sdnz(g>A?Y%r^+cR0h<-~C|sISt0vG}1CvkmL}H=s_nR z#+El=cP|H9jejcVv6zpC_oHA=sKIthYIy{@f8;?D(aWvv9h1nV@4XFrD#12@ix{#+ z%gm^F8apD|2gXbK2?6?l5+O=ohKxLCvZXd(r^Clz zp%Gg54`*-gx#cJ@Cseyko%ti&K7TTcP#a=7ge`7cO{^%Sa6k|MKk`yChKa!@8Eshhg}r~1Jn|Sf(iAe?-2xedB+~_VqCNM87pnv%y0Y?3 z=)fGO9xkIiTS;N_R0CfJ7`IRhpvP62p$ZmAVU~XAvn53sp0`e?B##@ZyzBN5_n`KX zt=37`c7;z9|92TTfr|BVX1Mpc{{Mb6o0%B)uy}wi?-i$gFl8VWvv_y6LjQZ+2t^w9I~PD)ZydAv(MZ`f$be)+i>zZ2EYhym>G zf8l#eB5D_k?w(LnW_1-BcB!OYU~dbb4aqhlm73iy7jBkO1HU*Fu*{#`9N}cPQGTW8 zOeDhmeO+&EqFURgG4@g#V?FKbYKE8#>~c>%kPfF~7R0cjR_acnY$ayL)h;l+_I+pcusln=imhwAI996Na!ENr>-;~`&S(qZ z*$yU3wdHDQP=bL(JeQ?0pYSSpl4MXZC#gJ)%iH$*Nej(!MSGgZ$AmF%SB^3b67&cL zU^Jacj`UZ|=z8j~4Rc8O!PoVBSf9xz-zqaUvoBxlKytzaA%pSWGS_%3f;!_#UcSA` z6HUq0s2zu~FGZb>kpQ=3w$mhMapQ{!o*oySg0K@OGANqo+6x5xk?ORr-GbWoexWQ9 zfM`s82h)wqo$SN9k8XqFlS80#eb7C~m1`ZISmAJVuLBPl|7HN$TlLPylFI|ESyMZm z4SkES24Qxi3bv9nbh0;dgo;9IR#oSmFTJ*yXZX- z;3o;{-2YduVaZoilN&pQ#Yx?Qtt!r)bV6s)OAQ+|;F!WR3q5X(uWbja1mQ3uj49fg zWCFd*C!Ilvei)W)py45I$mPVWQXC^_wGl4530v=#&Azzb*&53M2AN1mfC+4*wo?zB z=EZj^TaQt>y8K6pJmU=CWICUnCcJ`*bT% zt<}e`^;x0<0Dj&}{QSJ2ja_eSZzugOuTpzY8sWY#WsMYwFlNnoCBJELEfGv1MXoxObtV9%6MGy*O^<4y!Y70s`Uro?QVMa;xFPiWF*2jV@b}Vm!EK zhQg{fA&{flv}rYJ6a^6X@qc3{5MYn7pXYZa+zK6O;O(wpv*B0?k2+&1SR|x;+CzV6 zah+0+_j480C-l#2Em^o#WbkHL{Fm)Pux{HPWiIkg?BZHk%4T-kJzqDfN8u8Cot2UN z$?A(mzo+3nO*5rZnSeYj#lMC8d#GhYdRMTD!k}kEHvg91`WT|8(&HoD;x$q!JDDs4 z6C4Zxc6^c~2z6`j*ED;~YDSI+6XH5}LR;|(MhgeUakcKj)z@&-Dd!5-kcj~iH_quz zslJD|J?v8+mfkZdnsYP zw9wb754G?S%_;yMjy#ug2k&{w#vu>ceK6K4jx(EXp&#;(wh-07e|(8Hsoyfxn2}KW zeF6F7_L|c}1C9a2ouaqlixbjaex2IMe6>{}qq)ypEy9jMJ$Tfic;JtodAAR_8{7Z- zE2S#m#jV>c`#sdw4~>2HT!g;q5VO+FtJ}<7Spw-_wO#1ADNHT6hFl1bU#~>p*FWTI z&4?1Ue`9wsocSk2FxI`FsHPQFj`pg>)l2Qk^(fc{X4|$3zw`zUe`vClm1!O81>~YG$ z4m?fid@-cwk0}q7qf}(bk*m8C((Dtp4%gg2B`4@!JW?s(Y z0JQz$W-s#k2D%$xbam$l9_MK4boM$Pih_zrq8V!m5x-LV&92^DJzTd{HV>!x`-?~TP_-)!I& zm-Kjr&+Bo7ZtkR&?QC!qlOb5ha|j79ffLh&wTZ78%&unJkgQ|uGRTX9HKEY63~)KR zzR447V3NP{_M?W$25EbrQkYm`IXL6eTjoQ|0JW8>maxKOS1+$j%jWD zOD`JL^CnSA-4QM&!tsiKmN&Y6#ss#LG|#-(r_(&RKf#F?rFfdEC>6q;*B^D7#c;#q z0;W^KUxd_eD8w55d#*k)3`aBxk8{2(zMo==v+C?>QbbpvU)b;n1(DjoKo9d@q4V4b zxGq2y!U#0XYm1B3FJwIV!CR=+fD*|+Lq<^pJKULeisX-iIU6EeS~N}yfP&7V zLFJWH-?b0{up_mGtjoz|WRJkcS~f)0TpXYnxH@cqoFo>uFWU^KbjHdcIhSBHbp|Sn zx!m5uZbLAN-hPNN6G(z<8QCV(y{!C@_%`k<}lg6CTaCyR!PD*p=j0|WDC@2>)9 zyTQMra^lcNR1Dn9GIo$D@8dgS^rw9lKMJ=e&~RJcFOE)fU5$HSSYp*d>V{EAfU|uO z{FJ4ROwgrO`W)DXw0y@+tXxTVW1%^1z_G7~t*kN37L~qF8DQoV$60!+ns)0`M{)&I zQzfF-seLfW$a$TT>x;&7Xi+h7oC$GJI?2a9n;VgoeXWb?cP13>!pk z{uFmO8`7Os`AVw7TcW}`8ml;x9G(v%vDz(TcDTE7iHGrQMBMzvXg%H&K>1*;0TS%u zw17|Q2n{#|({pZ`6kO3tJ$VrVEhNQID%eZso0!=6ri}0<4OmJZQvF(;{rv+xW`%+b zRi^l4hI|dz!v%mf8uBlJO_n31yxhp-Qd23yc&A3MtsI=AjI$jHt>`e>``fnj$h$VI zM~7(ICIp_Q<$$g{8qr8U`x;@2&)T)C5U>pBhU{eo979#QK7)1kuSi*Azb1FDwlP$6 zU$q7oz5_kAz+y|K$Wz*_h*L{8nL%V+O||7vWn?rn z;!f02K1twD<>|Le>}lgb*{AEF$f|da2%%OE`I?c3FM@`qr*H z1CA>6ItMg;yy9Y*OmW?=Dp#kZgkuZU5sFxgHgUjI-?57%joUU%LPUl36GV6dDZ^sv zD5qUoZ227c4S8Mtf*%yk4GrFK7EYB`i4r64^WddSx5}N*8vKV8zOXRggWEc^f~i{W zZgS_Ia07Uf__4Q4DAHw)P59ZF>5A6*R9mx7N(ZXlyLwTzT5wy}*dYuYG(#-S%R=U9 zp+E?ooKmm_Y|rp3Qbne80GRDYd|}92IapJkdNsC{2BUw+ONE2Kh_=vWp#UlXeZ*E( zMk0~^#*>+;-nR!jZ{M67GsA=qz;2ngr#=zhxLp>u`oIB3cubP74VtB{>vFE&Dx!D| z=<*;|`zqhh*X%R=AQt)@GX$X2`cdoTnGi3>IjTJqf5B>0pQbc`C`3x^=N~5|sJ)-P zi#%13^S~HvmUULi?)jf4#o0%r73C1l33bOUoCOXmXK#_1>&bJ?nasSN&Zy<5tDt() zpsay#LH0u9jflXR7scts5R7|NyG`1XxeBqnOo)Hb*-~wkeSf;9&|CT!m+v)J?eB1k zO{miJk6^7#>com(1ePA~c)vs=rzabF9pWaQb~nC595Vf>RqU1P4!x34tbwg8Nzd@; z)QX>g4b{c02b_~Wk7@5V#4^b9<2w{q2XIkPVe5y~Zjlm1(vnP{i5XIEh_DUwqE1ys z#26&5YxwnG`lL+8|_;GdKc zR-GYeRHNZaXDRUa(q15@51D#7g)+aKdv zbQD1Y!%eZ{GOz{ilDSxG#~uj)K~uRW+n_drKi*x_IAlvrf9ATk-}|+)3bZNcJ}adM z{b~5-N_b?Kp)%*Y(h0Lv$`)s2=rI@lWD%#>{&vl0)A?Ynq(aOKKKe1vASFpk3XDnw zm`rW~rTYzEfRPp7HjeN1ov+&lG2V)Rq+zOR z@@Tfs5zhDIO}a`dtDq*N=@4)z8XtXpvC2t{sx2-tcmm9Uu{iX@hS??q|X)^lTtEtYli_mL)N7s4%WM zQXG_$)AGPw%uQtCpB6yDP`0qJ-nN z-NxB_7j-ftF}~Ipd+1*79NA7#Y|d1yxP8GIO?WW0Ps%O;mcv{cb%Rp75HGD>{!$vG zIo|`EA_{I7*4sT>9qB^FkBaDz%poWad)c&cxL9otX#g+nffb*mGhkm{>X|@W=ZTgW zu=vvv7d&&66@~Y#CA^8p{QDaSB@mSI!v;uhRCUh;-Lnmm>ItLnG zLUyvfg+(Hdz9LQF{l?bJdn;c%P4o8WPN9d}OVVOJMeKiEBqi|_D*hzvTK*4zmRB*d zB`?DH&JpjXs{E1?tvThy8SSs3ny|L5w(X&-^op%ACmTSnw$}REVB(<9l_>VMe3}nz z5M@^aC=L3CFnII|E$Gn zd!7`0+3JoN?7DlN((ayTZd=70L19yiz1h!i z08DvvPwSsj#jKr#B>{PoYIF$IHwdK0rKRX-jK(3(#zM^`Qk3M6NfxEMQLQ*IcL#K> z-L|Q8qI!>HLwE*yO-6Y5x3UGNVVu~?Alq*=llF)n@LOn;r$N~Y*2~CYoM|HWY>Ya! zS>O;x%B0h0t>UD#!rwy>{d@gaWp};&=dC&RD}gx?_l3J>YgBopAQtth{;*v`=#-`- z24)totgmy&&k-HDG1~OZIeh(S7tZ*F=&H7)7uE8d-@dfO6p3vGA}*sUO?M=_1+j}Pq%zN-F^?$h%9Gg;ua7&sU~GFe2_GyZBbWD|vx0tLCzEzS zZ(w-2MJ{`IwecM`fDe4tQif0qW&bGz@}rn=-&a#X z-2zl@*{u$XM+!u5Jcl9Z6e_KEs6f&?GQW@ZKz~EnVm$z1f^{ Date: Tue, 17 Aug 2021 11:59:08 -0700 Subject: [PATCH 88/91] Remove TravisCI and use Github Actions --- .github/workflows/nebula-ci.yml | 55 +++++++++++++++++++++++ .github/workflows/nebula-publish.yml | 61 ++++++++++++++++++++++++++ .github/workflows/nebula-snapshot.yml | 47 ++++++++++++++++++++ .travis.yml | 38 ---------------- buildViaTravis.sh | 25 ----------- secrets/signing-key.enc | Bin 6800 -> 0 bytes 6 files changed, 163 insertions(+), 63 deletions(-) create mode 100644 .github/workflows/nebula-ci.yml create mode 100644 .github/workflows/nebula-publish.yml create mode 100644 .github/workflows/nebula-snapshot.yml delete mode 100644 .travis.yml delete mode 100755 buildViaTravis.sh delete mode 100644 secrets/signing-key.enc diff --git a/.github/workflows/nebula-ci.yml b/.github/workflows/nebula-ci.yml new file mode 100644 index 0000000..05826f9 --- /dev/null +++ b/.github/workflows/nebula-ci.yml @@ -0,0 +1,55 @@ +name: "CI" +on: + push: + branches: + - '*' + tags-ignore: + - '*' + pull_request: + +jobs: + build: + runs-on: ubuntu-latest + services: + redis: + image: redis + options: >- + --health-cmd "redis-cli ping" + --health-interval 10s + --health-timeout 5s + --health-retries 5 + ports: + - 6379:6379 + strategy: + matrix: + # test against JDK 8 + java: [ 8 ] + name: CI with Java ${{ matrix.java }} + steps: + - uses: actions/checkout@v1 + - name: Setup jdk + uses: actions/setup-java@v1 + with: + java-version: ${{ matrix.java }} + - uses: actions/cache@v1 + id: gradle-cache + with: + path: ~/.gradle/caches + key: ${{ runner.os }}-gradle-${{ hashFiles('**/gradle/dependency-locks/*.lockfile') }} + restore-keys: | + - ${{ runner.os }}-gradle- + - uses: actions/cache@v1 + id: gradle-wrapper-cache + with: + path: ~/.gradle/wrapper + key: ${{ runner.os }}-gradlewrapper-${{ hashFiles('gradle/wrapper/*') }} + restore-keys: | + - ${{ runner.os }}-gradlewrapper- + - name: Build with Gradle + run: ./gradlew --info --stacktrace build + env: + CI_NAME: github_actions + CI_BUILD_NUMBER: ${{ github.sha }} + CI_BUILD_URL: 'https://github.com/${{ github.repository }}' + CI_BRANCH: ${{ github.ref }} + COVERALLS_REPO_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/nebula-publish.yml b/.github/workflows/nebula-publish.yml new file mode 100644 index 0000000..18f982d --- /dev/null +++ b/.github/workflows/nebula-publish.yml @@ -0,0 +1,61 @@ +name: "Publish candidate/release to NetflixOSS and Maven Central" +on: + push: + tags: + - v*.*.* + - v*.*.*-rc.* + release: + types: + - published + +jobs: + build: + runs-on: ubuntu-latest + services: + redis: + image: redis + options: >- + --health-cmd "redis-cli ping" + --health-interval 10s + --health-timeout 5s + --health-retries 5 + ports: + - 6379:6379 + steps: + - uses: actions/checkout@v1 + - name: Setup jdk 8 + uses: actions/setup-java@v1 + with: + java-version: 1.8 + - uses: actions/cache@v1 + id: gradle-cache + with: + path: ~/.gradle/caches + key: ${{ runner.os }}-gradle-${{ hashFiles('**/gradle/dependency-locks/*.lockfile') }} + restore-keys: | + - ${{ runner.os }}-gradle- + - uses: actions/cache@v1 + id: gradle-wrapper-cache + with: + path: ~/.gradle/wrapper + key: ${{ runner.os }}-gradlewrapper-${{ hashFiles('gradle/wrapper/*') }} + restore-keys: | + - ${{ runner.os }}-gradlewrapper- + - name: Publish candidate + if: contains(github.ref, '-rc.') + run: ./gradlew --info --stacktrace -Prelease.useLastTag=true candidate + env: + NETFLIX_OSS_SIGNING_KEY: ${{ secrets.ORG_SIGNING_KEY }} + NETFLIX_OSS_SIGNING_PASSWORD: ${{ secrets.ORG_SIGNING_PASSWORD }} + NETFLIX_OSS_REPO_USERNAME: ${{ secrets.ORG_NETFLIXOSS_USERNAME }} + NETFLIX_OSS_REPO_PASSWORD: ${{ secrets.ORG_NETFLIXOSS_PASSWORD }} + - name: Publish release + if: (!contains(github.ref, '-rc.')) + run: ./gradlew --info -Prelease.useLastTag=true final + env: + NETFLIX_OSS_SONATYPE_USERNAME: ${{ secrets.ORG_SONATYPE_USERNAME }} + NETFLIX_OSS_SONATYPE_PASSWORD: ${{ secrets.ORG_SONATYPE_PASSWORD }} + NETFLIX_OSS_SIGNING_KEY: ${{ secrets.ORG_SIGNING_KEY }} + NETFLIX_OSS_SIGNING_PASSWORD: ${{ secrets.ORG_SIGNING_PASSWORD }} + NETFLIX_OSS_REPO_USERNAME: ${{ secrets.ORG_NETFLIXOSS_USERNAME }} + NETFLIX_OSS_REPO_PASSWORD: ${{ secrets.ORG_NETFLIXOSS_PASSWORD }} diff --git a/.github/workflows/nebula-snapshot.yml b/.github/workflows/nebula-snapshot.yml new file mode 100644 index 0000000..6e5cc87 --- /dev/null +++ b/.github/workflows/nebula-snapshot.yml @@ -0,0 +1,47 @@ +name: "Publish snapshot to NetflixOSS and Maven Central" + +on: + push: + branches: + - master + +jobs: + build: + runs-on: ubuntu-latest + services: + redis: + image: redis + options: >- + --health-cmd "redis-cli ping" + --health-interval 10s + --health-timeout 5s + --health-retries 5 + ports: + - 6379:6379 + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 0 + - name: Set up JDK + uses: actions/setup-java@v1 + with: + java-version: 8 + - uses: actions/cache@v2 + id: gradle-cache + with: + path: | + ~/.gradle/caches + key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle') }} + - uses: actions/cache@v2 + id: gradle-wrapper-cache + with: + path: | + ~/.gradle/wrapper + key: ${{ runner.os }}-gradlewrapper-${{ hashFiles('gradle/wrapper/*') }} + - name: Build + run: ./gradlew build snapshot + env: + NETFLIX_OSS_SIGNING_KEY: ${{ secrets.ORG_SIGNING_KEY }} + NETFLIX_OSS_SIGNING_PASSWORD: ${{ secrets.ORG_SIGNING_PASSWORD }} + NETFLIX_OSS_REPO_USERNAME: ${{ secrets.ORG_NETFLIXOSS_USERNAME }} + NETFLIX_OSS_REPO_PASSWORD: ${{ secrets.ORG_NETFLIXOSS_PASSWORD }} diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 0d1a778..0000000 --- a/.travis.yml +++ /dev/null @@ -1,38 +0,0 @@ -language: java -jdk: -- openjdk8 -install: -- export REDIS_BIN=$HOME/redis/3.0.7/bin -- export TMPDIR=/tmp -- wget -c https://github.com/antirez/redis/archive/3.0.7.tar.gz -O redis-3.0.7.tar.gz -- mv redis-3.0.7.tar.gz $TMPDIR -- cd $TMPDIR -- tar -xvf redis-3.0.7.tar.gz -- make -C redis-3.0.7 PREFIX=$HOME/redis/3.0.7 install -before_script: -- "$REDIS_BIN/redis-server --daemonize yes" -- sleep 3 -- "$REDIS_BIN/redis-cli PING" -- export REDIS_VERSION="$(redis-cli INFO SERVER | sed -n 2p)" -- echo $REDIS_VERSION -- cd $TRAVIS_BUILD_DIR -- ls -la -- git status . -- openssl aes-256-cbc -k "$NETFLIX_OSS_SIGNING_FILE_PASSWORD" -in secrets/signing-key.enc - -out secrets/signing-key -d -after_success: -- redis-cli SHUTDOWN NOSAVE -after_failure: -- redis-cli SHUTDOWN NOSAVE -script: "./buildViaTravis.sh" -cache: - directories: - - "$HOME/.gradle/caches" -env: - global: - - secure: huCeSvKQ168V/v8Em3q9VxjEPMPQV6hEqbdCSDHY+4x/0rxQ6DzhLrlOkS7z8+tfWsnzK3mhcpYOF3iLe1wX1xtTe4k3wuTYjF7rKLgNyON7luBglHapDa9/bRr8OLWc9EZwxZt1J6LgZM6DVsXEcjO+tJL4Mce3Y6Knz4bgNc36fvjg4mhQ+SjvVG93TWjOYqqHqu1QH/3dq31LWepBDQ009iy7TUmMOkAuWb6Fuihk333WzJi8aICx1BXTJCDfWrzpyy8+on65b0rtIdFMRUQmF3gR+uV4RRAatKhCELSInW45146gMMugewDwJScHMbPVO6MCtvQuHJeAipeQa/naPWspAAmD9Oi5jxt31YrDsLk2k3tLmrG5S0CcKuXMAEjSVWfJAFg9N0q3+TWuJ3dEcmIcVfJsQ90kPMHkMi8GnQBEGvTond4ItsePAycRcFhtXAWTdgcadEQ0N60TtHopUtKfblPp6SxKEUNr4pGxAOmUIRl44HsY27O/Nxm0kRd8RqrFY2ZEaKvZtwVZIPEzozRdF35S36hhl7FtEjOdeyvxhot1vNL1WrJh4nEGAhqhHvGiD6nHbMvXjnXpWFQn3icteVt2IU+SvQ/Qkq5kDm4Ytc4PkSuavlmkMSUDSrsOA/IRAuj/xWxkQCH5MO/oubIt6ka/Nn0lS9NUjIg= - - secure: nGBmnEARBcVOn4Eq1rbaCBCLqayx+bBv1MBmoK1fig95dw9JN5JVfYcSj5t3IG88uaaE995za813AXPZlNVs4+9Zr3Dt6JDnyF+FVR0m5fhJa46nYiOlyH6WFr7qCd43UEBn7XIj4E+v1ax//bWUpRy+vNB8/n7xjyxJ1rfoZcW8+lDjz7ijwPKwxr0A+6wkJsTb+h+3gC8MVbUMQnNCxOnDfFBf9XaoXSX4YqlMMSK22gGqgqIUxQ6H5p7NXexM/T3rvhnd1H9U1YHI/PSCur7UAjXcEw92vpI+DV1/IHAmbUviFGf2lhVSLmp1sz6RYBlB3fqwyOJDPRRcvaoV/ZVmHYsDK4RsMvcBzGNc5Y7w1ZyQpyiRItjWNLuO8I6sZo8ELCAz6snEiG4ym6E7IKLjhzxfd2Vm5zCWlfGUAORhQEm7ooZVm2x8gl99rzud/c87xbE0Ju0k0Zs8jTFR39L9JTkY8eHiMCjbNu71yumbmsLuiI0fvt5hFpsgWIWWYXs3IzvN+CFFJyCxNkVhZXAyy8JPvSSencxzfgV61GeemmEerzFpfREDYE57D+xNfNL6ib2NmHVDcinolvPb7NBYP+DEtZiAZoopA70fjzsndcH/EKTQb1X5ccEH9hCbvn0UVejQgsm82kUUhG1BC24QSMIftOnsJRPzyH6pz4o= - - secure: AXhVr18dG+qm1IbcTPYkoUS9q3/BIk4mDON7uf7YwuhAYy5r8RY9QWOeYAwT2yL+WaUvzGacpsYc5NKU8Cwrndv+f2qJerUCk/sgS1WoiGut377RvOQk/rOZnNvxi24K1T+iMgSUS0OQb4HIpSciKYGB6sYKmrwpXJD6b1tOp3x+P7NWv4b4rTamWlyf+NF0ortha91fu/KBt0vfUJ8GjcUKzLVLyqcP6dCd3xvUUWWYJZaffMn82Dh7PnhnxR03BWhVwRLU9lw7W90OEX5rcC2/mpcjR0etscnZl9VX5BbwUmJcPHVkEPVQGSfB7VxwcXpwfaq4uh944uJYM/wKJ0pAphq53M4fyHvoX6xPVmf3PN9AELMFfe+KzoStON627AXmsdWbaLJu3/iB9SdvHwETjOjrejgbsAxIC4B0oPQI320d8RhBF6xrkcl1oC0dNH/fZZAuP8/YoA6HzFGXeFSNx9TqtR+nSUxLYZWXAS01IAcKEn37aMhXTmNf4QkZViN3NI6HgM24WQsA3XpQZCZeDcXqOEdTPZG0OMJ3Jt+OfpNeDVfw0+m0Ir5nZEjfLo2lSMaL1Q1McWNdY86ZoJdHNBhUvliKvYf6oHJ4rnkUA8Q53KoPIEMtmSDRZKFj5et+RFfhqgR7B1jsIzu/2OEYS0aE0INdeBq1BzjvFWE= - - secure: nRQooOJYmuudAbqexGR/+GAJ7fSu3JIFJDqyfmIsZDb4cg6DlUCkr8rrSH9DFyR5akFhQNNMiR+eCCSG90s4GJImHIAhDRdodrcAFUIHAX7M94b4b6TD0wTwhzXClZRcBHt+VQ/EaYAP1N9czOsw//DPV2oQ2KNGAR7gpMmBSZ2Tw5Kiwv3zkBJr3ttOEy53rhgX+DskTtut1g5Cfq8rKBaGtS0AXNBX/aJyVWRMdlaomebbs7StKEESf5x5tus38iTwTyuQpHOKGD9oFDxaVnmw6NrAPTfVT+vEwhSttAoevG7nZrsjiCy5EoJEKxL6dUKttT494Xl1G9I7ubxZf4mkVnBsVL1GDYiRv3bYsfdkIJ38b57vCodmHebtjjBiRaC75iGn6D/fDIWTR8KIqOZIuL05LLKhuBDk6V5xifiw9zCcRNeZUSAX0q9LC+V0RLxExIBouzYKwcmv0fE9KBhvLPEfOpB4THBACLX1FCRybijNwqU5bb9T8SnFVH2Ruk9Ocan9sEvtaycXGrLlnFIj6DdHY4uaUlQzMfQuqFdVo1gu9GneX09oIMjKW1tN2MyUcBEBZHZkN+LQaa/Gt7c7TR3kAs9yyvDQffGYtC/MKyefnYeIFba7LiS/8eU5TtkvMcuauzq+yeuNhWKzY8NzLEz4wq1okz3iSQIYvz0= - - secure: lAq40Ixu+vQG5NfomDwR4Yld1+9y1BRFMmgqe75vcB4JSoJU0Kf+OgUvHYt2Q1OZOrHllD3hYBzx4JNUc05rQrDd+wDF6r24fay9JW6FcVv8yhfhukU0WE4AhjFvBS9UQoWloFYHXFGcofuSTqShoumTuAW+NheQIj/Ialhoy7/D+aezWMOhNx5PFhsEII0MqjUaoAeg0FI/ao1yoewONQNYpSiBHsmBOgzBnrjlYs0gWfxYMUOoj8rEuNrcX2kCCMdSGp9cy27nydPK3vYsJFUrlcI/SK+PBMQtgopPts6kRFxViFb3fgilKxYezrcCYmqeSkTU1CYS8J+5JahKemucaBesqX8UVAnywRCw1wx2jipbJcffhJa/+rOEi+8TUnNgbkHBhYa9mOix45zJR7YrvdptwAlRaan2zKeMVkrZdzvVOXnrLSsJN4vUviD6kh1whqX6OFKH0S2chMLbepKMkkM+L8qnHrsiDWCdYkAgETPcwM976uXi/vCN33D2JSV4JtFvgrxY+VoaKVuoMMNDFG6LptUuSOpU4h9N5e4spJ73/kmvJ9ZKrCOy/23EM5YTFFM/joestQB4eszD+TgaLLQCnebmth6sab2WAC4lseyJmYo8v1jfGtN+vhlN4hs1sweF6rY0aNzl0w018B/LDG5CbFQ7/9kv6XSvvOk= - - secure: cYwUuptciaQDevzUOOsulwDsRGMDOwhbS/Cma+MJMZ17WH7XkRNLnBiK3EQDzH3Kwz8DHDXMKHA6B0NxeNDMK/zo1hjndzutnhbYLSKTIjgLikvufyDddHBHRawMfKHwKgVQfc3FmoVWKtZIJtPcLpFpRsUOlpdvzJW3L+uHMkeoViHoCIvzMqSLNhTEGcdmgY+dgM3cX32ndBwgFLv9E5a23qCELX8icMmjrdtW8NpEJlxkcIa0yNpe7pX2h5d2JuwcWcwEYfKHC39FG2jIC2Jn07bECHQIm45y5msGd9M+Qq21hNdfvWzv9h2YJKlhZeUEpJkAwjQRDgBR1T9DY9427UFdRQ20JX1e8SNFEkFAvMs5kn+Is7QiVDWsOLUl+ZQgS4oYg8QyEvU+/srTQ2lSaiipDq6+QaAHRO3pSF+y3TH6uI6Q/fc08gaw1mtmj7OiU3lWK9EyIdgc60Nm4BOpAFkzytaYpJEaprU9WiqNjLjejpRiDjYNWLr3HPy1AoyTeqj83i10KRwE97gkc5mhoPL/dI85T9mgR2nOboEcI2ZNiMyIPHc8GV563jlKdTeALQW+KvbVia1NjtrC1PxkCTxdNeFkgeLCGB97W7o6swLyCo36EqLvxUPNmayCXE+GhsIH/n27PahtneinVr7urKx+l3U8+WifI4XmHnc= diff --git a/buildViaTravis.sh b/buildViaTravis.sh deleted file mode 100755 index 36b9f97..0000000 --- a/buildViaTravis.sh +++ /dev/null @@ -1,25 +0,0 @@ -#!/bin/bash -# This script will build the project. - -if [ "$TRAVIS_PULL_REQUEST" != "false" ]; then - echo -e "Build Pull Request #$TRAVIS_PULL_REQUEST => Branch [$TRAVIS_BRANCH]" - ./gradlew build -elif [ "$TRAVIS_PULL_REQUEST" == "false" ] && [ "$TRAVIS_TAG" == "" ]; then - echo -e 'Build Branch with Snapshot => Branch ['$TRAVIS_BRANCH']' - ./gradlew -Prelease.travisci=true -PnetflixOss.username="$NETFLIX_OSS_REPO_USERNAME" -PnetflixOss.password="$NETFLIX_OSS_REPO_PASSWORD" -Psonatype.signingPassword="$NETFLIX_OSS_SIGNING_PASSWORD" build snapshot -elif [ "$TRAVIS_PULL_REQUEST" == "false" ] && [ "$TRAVIS_TAG" != "" ]; then - echo -e 'Build Branch for Release => Branch ['$TRAVIS_BRANCH'] Tag ['$TRAVIS_TAG']' - case "$TRAVIS_TAG" in - *-rc\.*) - ./gradlew -Prelease.travisci=true -PnetflixOss.username="$NETFLIX_OSS_REPO_USERNAME" -PnetflixOss.password="$NETFLIX_OSS_REPO_PASSWORD" -Psonatype.signingPassword="$NETFLIX_OSS_SIGNING_PASSWORD" -Prelease.useLastTag=true candidate - ;; - *) - ./gradlew -Prelease.travisci=true -PnetflixOss.username="$NETFLIX_OSS_REPO_USERNAME" -PnetflixOss.password="$NETFLIX_OSS_REPO_PASSWORD" -Psonatype.username="$NETFLIX_OSS_SONATYPE_USERNAME" -Psonatype.password="$NETFLIX_OSS_SONATYPE_PASSWORD" -Psonatype.signingPassword="$NETFLIX_OSS_SIGNING_PASSWORD" -Prelease.useLastTag=true final - ;; - esac -else - echo -e 'WARN: Should not be here => Branch ['$TRAVIS_BRANCH'] Tag ['$TRAVIS_TAG'] Pull Request ['$TRAVIS_PULL_REQUEST']' - ./gradlew build -fi - - diff --git a/secrets/signing-key.enc b/secrets/signing-key.enc deleted file mode 100644 index 1a4fdd1839ab3ee1d2646b50ec1745f76339188f..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 6800 zcmV;B8gJ!OVQh3|WM5wgXVxXpOWpYCv!AZnF+elZ&kD16b!=0%o>*g)5|T4gNAXw> zFDGTeAdk!`LTS#$Vmm>H>eYM+D4^L790x{@ZOjgaFSio_f-hyWoO#iusa)_Y!)?+# z%dip+t;I2eUncxlT+9>$gLC?abq3DvO(5Y-w8*Rn5pls&uT9ETSHrgaH{uJS>E1Pu z=Ai0^n3sCz*fe8G-0IAovmOdkbsh{#RfKlaP%m>B6rl+|JUY}n+NkFq!VP-P7Lme) zg#N|eXoDl9-SvyxL~n*Yd&Z(nd0TceWIg(4`86rm06WnoJ?=5!x{6T+%GTN)?1mcYV&Nsxs3~Cv5Bm#UwFp` zbc7KN;W-(Q4aBa_g+dV`7?Q;GanyOM8UYnGXUZ^Z<5An(Q2wlk#!uvKf(|^aSJ&2; zx8!%Z2KVdMT;%#JaW8C{MCSAAI9lyY7oA%N-UjQ_7lVs`vh0_9UQt5);dl9s{s`8~ zQS#>e*?278y`tjo5(Z4E?Jr95^A>?vN?A;ROUjhml@Gw}h*}jl*xJ$R{7|eWzBRbW z7uv&v&NB26`*35Gw9<`QflJZV@C3oKEsCwmxIY{e?4zBGk4z@D940!yoYDign<&yF z_G(SK`*K%cqv&YTQHL0*tcS_xWM7cKq!P9hp+uH%P!NkTU!lwD^!s`fA9Up-cCge! zX9iS#(73ddUUs`6b4}cri*FI7tpoTSocOg2W2MF2hXdhD_nm|BNGN<6jS;ND@cXLs{WTy6j_|U16OGY z=ikMe=FJf1Q_$tb=TyB|hEw78`I711WsVdylv`TXGr~k_vus*ygGJzZ+A)NCd_dV` z?z^fB6_{l=I(bIIA0OB*%AyzMh!H>$kLPsk^l}W)e{$DbH`HV^@ZzVhzpj%4w zH9^e4M<}g&lbK&OAb(`8@k!9fi@tN(yX2U9-$jO7b7Z?jN?&|WTJ4C(xqOo~z(A$A zs)s^a6EV%agyn4xMY$8IEq?M5xulBeq_NRmB%LVl!!vNo(mfJdF!Nv-dgy}~*clh( zVl9h++?e4zrKq-oRMws<2N)`cdP3nxm?W*Z%*5Y-yLW zzn0aW6UlY!&c|A%u`e|fGORgEXUi`QJWQtwu`hmaMtzW8()Zd#!TaI%i5(@Lahs<| zqSJB~cFcBlhN)U4UNaG3cz)%HbwStp`7{-sv$NM*FKQs3a7Fi_O}R zn49{Fx~qs>1@kRz6l$flCn1y*$tnHZ#;~z2e7o&4b$)bE?zBXV z-nl5LuUkX~09|Xrmr=dl64XM!;yvpQ z8EMuh9%-1W8!T8DP5@gyaIOVqQZ9Gt{fI!J%S2xgw)TWha<=SEe+Tc?#!PvfeQ5yE zE#m^@-y9JvzZ74cQmu#2)KWZnV89w}27iW5;Jjf!{TkLQ1(eGG&d_Ug`h$pIafq0m z_6FPULne~^x5h274zUmb9uL)LvU(OV2AU~?2%`904Ycw9pWm57}$+gTn)^3E+YoY_Awl0sWdW&G?bo(q}?w3vvkg~;2^lKizQ(w;f zK@XP6M({?~{QD8^nhqX8&MP!EQb7FkqCPN>c%5d|ZND|2Hs~lONNXP^noO!yrsf)w zSzuaM4X`u)YVx3y$9r%}Ty_^i{!`)Yd{GiyqBjsC1tGWpF!Q`D_KO}r$)7&an($$d zo0NJ^Vg^3NLnxEX^lobIKf}6ZiuEC^!H@m@fekShK+q@s<}c=X*cSWzB|?hU40Cb5 zyADGu?db}uSTW1I*>2tIZpcZPcki-!T{IU|qyGwbf5f6fCZ4qZIA7<-B^8ZMfC}-< ztf}h>0>J*SxMrPS>rZ~$1p8Huw`xYK66}b>GpOvFi%C@ALDI$bMfSM=gB6iTBGI76 zggVstOcyaYKIN${;;zyX?ckk)=epOgg!Tfncint4cHL?s_yUUZyeG7-0}LIs5WaK# z%=|d3^+*9l2rHk#?D_N?5tBjDi35Kh0WORbe*_0Z=dViIaTU|f8&WR?FI^4Ga%-;L zTIdBgu#$tm0$cLCTbD+M_xX|(l|70kETrd&dydq7++<%1=DUCg9wW+!Gov0ENw0_T zy#oAXjHz_sQY*hL8P8@``%riCn#GFBT~J zR+aAmNeT&wE;dG|g^r22P{!nqC>uCA1`bLyNNy&!h64maIy3H^5`Il7DzMwK{l|1K zH8nhbTPr}lSA!oKbQnchym=bWN*Bna-VLoo{KBISGv1VwL=0(yPO0AE9NEVrSuj9O zl$?O2{kZZectjH9sWGhg=?vv#EJeYfX{qi&q_FeEHGtWsq%t!{1!%u(SRt_nxAaGb z)Fp;>S0-XDX(c6~;Izn2z*Ye4$hfzXE*_q=>i$_p zYjib7*G9OQggrT$lv4{O?k{oDQNh~8M7h~%?;Lr-&$&2Fkr!3Q1xLAU! zFOz8)~}7 zfl_XRhhf5UPE=0z0$Fi!Wb(Pg1X>dL~Z=#VVbz?R4Vjex!0P)r)J5em18{}VAzb9VgU&F zU-1wEU>8aDWFhzPr%0JL_SR|;5Rl3R`jR7)!bVsd~+gSRe8P>N3z z(|4bujq;R{C8XsQtheikP!`|Vjk~U3#d1I=|1oiIe4w_hGUh*1$gJ;3hm0kD2)wz# zA8}>(9TwSOiS4BymzIZLOI2h$_H674nv2d{1$`|a%S><4=EauZd&%YeIf{kf%AkIh zYrsa%gKO|vzRHe%SyZKdq;o2_5m{$zeB!BIH6^N8@)7Jn$U3#xCA?G=vMB`;dTF~d zZ9B3%Ka>$K{E<3X{0DE9+yF#CoEVudDjlEgo&_DDi*kXtNt5=Sw6d88dHfC%44L&m zL;UT>oV8w3hl~38g&q@d=b#Ff==O9Tf!d@}m}MZM*xbvbkAnV55R4r*r4nCk{mUwg z6q0LF?+4WDjQrduUO^d}y9c=dOat`LTyg#6J;7?HxzmE%ck~)V0+m_vlX7DUfYK3M zv=T}zip;Ebf6Dq&PB<0i;RGy=e8*?&9X`$&`l@Vb-%wDV-(L_cE;T&q^)!;XwTz-m z)n#D11`PvCS+p6y3$XVZPF`-p^dppGI^_cM`dB4yQOdWdKJ^$ZH58r@OP*xYr@tp2 zQmPIS*=%A+K-uEuP@*rjq_yUx^#Zu_5vWeQnx4XtwKs{*PFj0i1MxoXqf|%Ez(c zGM%cvVEf~Tge+K>AzPfm8OdHra4dlbRCNEe!C&E27LYmIP_6XJqcvs8Cl?BJx)Jdt z{WOyZ@Z2Qo=h`!BSovE7KYY0IdpWQTmwPPYIbP-+6Bq?Qm5J?H?B^IQZZmzi3PfKa zLFY_*vp8W3two@pXevtK=s8UQ+kna(95e0c6#cR?vVqZ1jvlN;#2UlughMWAgiJIhbTP%BKssB&JU7B)8x|`$oRVnNHHSwD z?gxmObtuZ{Rt%=2E!X%?GaNDc3d}!e4WNlEQ1MN?VXz6haX{5tTL0tY=+)WkYo=1+ zaqEiu&i|8c9nCtFxD8~Q+b|)lrQniL_SfoHexrd+^cvMAb{vSA>le>J78m)1xuB{l z(@A85S8r99r{DJF;IU-F_pIn;G(?7Ju4*5^sXokvjVoafCm}s1mO`CG{^ha|zaz6? za{|oXBVi#=Y!Hr15_+3?3ffF<2t=g8RDv86k4iX(vtx>f5Dh!}}1uf{;aE#>XwHRW%5s?mjeEb7L?_%~- zm&DU#B8%X{XFTA%_t`G#WwW=Hu+C*keb_K`4O^M`GRM!%N@Q^p|1T=mk)|tJ_EuEr zPZ#bBk8M^YpoN}q&&&bscSZ1>y0d@qE?tP?X37KPaLa5K+ea-YymOVN8oyg90@U%+ zrxy@x2!{i6o5K;XZl}7R;RRpM@#LoemHMY#Of2fW)b{wap&$?M6O6`b zFzP;Z8j;vHGIJnA-yBoQ_UtC}&xX+a^~!AcC5Vr%JSf2*yQ);O_~3j!U0p`Gb}#$==MDqjgcPMC-7X|?)7@We2) zeiq5VmK49q#K_JYlSgSwAT-_xeI|3uqljHPtR8xM5iokTYN3>8jN2?$cMVdokZ!|` zj+mc~%X60$RMk8=U@B*z=gJ0|g0hC_Nu4k-9yRFst|>{c{^baioGG20=!9+gMB8<` zF*1w9T8eM!)M_Z8$6G`1G1w6PC52owT5pak^!MN7xm!4nU!XjPJONBYJ8z7T8={g@ z07Gv?G4{d+O1?VCHR^lcI4N8(cyD1=ZU@$ER6@<&J$E~6fjK|pc+)B@9hS%*jEP%@ zKk}Gt!u-&ZQ>l~gTcNVKAMu%*6vi$ik8#>k@-G#ojf)ulDu@6Z3vWvP#eC_?vL5&k z<(*#m^AlM3dfxX0xe`flkwpeZ1CeHIm z_#kVHA1ht<==)poBDti15H_!tOW#APQaJrSTGgb;$DcrrZ_MxE0Nrckmh(3JEvPIm ze~11q+@KSY`MMlEiB_BL;Y54UP*yJ&{k1FkWJnD zGKZ}gp87F$0n$4#M3czKb#^Sl_Y^s{}Cgz3zTU42tD(ln>LB7qjYm1Oc(+bpqafIJ%~RrohK z8K;~jHr>gFPWkl&h<~?6jc_C-nzjkT8Q%v#FwvWtmqpODV9Lir97*+abTUmjwLUC}qek~v~7oAI;+8q@am z!dXp$!ECi(yJ4w0fxr7aYn(8jvzIE9%1+d^>)2%Pu+Q`XqsD1zh*3)#9TKV7Rme>v zoxw|c4{ESZWBDEtIe}<)s4rX%!SaPek+fjlphLW%8SYY3lqvm@t>?vnCi|>^)ema% z9wUlC3kAuq(3oMgGyK&Qm?vLEk~*m^Dv~IX&FWMlby=oKw%5N_w3igYec8Z3Ez;|} zK{O`cTreG%RJrh6(cC)+$vBvQ9Sk)n>aKUEc*S0V`(UmeXB-?r)Q=X{{8y8>HC(ih ztaG5E`15~f>eXf~WVV9-0gwo+HlQtLBmn4>xwxk?CEVB%`w{S1cja3}LL$`XX6@V~ z6T-&TNb(rVwJ`*u%jJp#*_~pg`i+F28uipb2F*_)sMwVe7_A)8!pcpA$e}3{lS}hpI!bt0c`VYU=*fJhrR`d zf5$*20)~u~pI1-9jvW_gXW7ZLfJ#8_f#~~QBl}JYZ6s5=yu%^MllyWQ5m!?>Ae{J@ zyFzoUKZQwgbu06Y?Zd5PjfRqX%L2j3*K{H+Osi+#2LXiy#RN6ycBU9+wmjY;rD(_G zp0l~+3z6MQr_B;*m|tcOJoGsrK~ihH^xnDPdzpZkAqEPuW=60HBcVOK3Wc0niCW|S y1bceG)I Date: Tue, 10 Dec 2024 21:04:41 +0000 Subject: [PATCH 89/91] refactor: Change GitHub Action version to address deprecation Use this link to re-run the recipe: https://app.moderne.io/recipes/org.openrewrite.github.ChangeActionVersion?organizationId=TmV0ZmxpeA%3D%3D#defaults=W3sidmFsdWUiOiJhY3Rpb25zL2NhY2hlIiwibmFtZSI6ImFjdGlvbiJ9LHsidmFsdWUiOiJ2NCIsIm5hbWUiOiJ2ZXJzaW9uIn1d Co-authored-by: Moderne --- .github/workflows/nebula-ci.yml | 4 ++-- .github/workflows/nebula-publish.yml | 4 ++-- .github/workflows/nebula-snapshot.yml | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/nebula-ci.yml b/.github/workflows/nebula-ci.yml index 05826f9..2b579d6 100644 --- a/.github/workflows/nebula-ci.yml +++ b/.github/workflows/nebula-ci.yml @@ -31,14 +31,14 @@ jobs: uses: actions/setup-java@v1 with: java-version: ${{ matrix.java }} - - uses: actions/cache@v1 + - uses: actions/cache@v4 id: gradle-cache with: path: ~/.gradle/caches key: ${{ runner.os }}-gradle-${{ hashFiles('**/gradle/dependency-locks/*.lockfile') }} restore-keys: | - ${{ runner.os }}-gradle- - - uses: actions/cache@v1 + - uses: actions/cache@v4 id: gradle-wrapper-cache with: path: ~/.gradle/wrapper diff --git a/.github/workflows/nebula-publish.yml b/.github/workflows/nebula-publish.yml index 18f982d..d997f54 100644 --- a/.github/workflows/nebula-publish.yml +++ b/.github/workflows/nebula-publish.yml @@ -27,14 +27,14 @@ jobs: uses: actions/setup-java@v1 with: java-version: 1.8 - - uses: actions/cache@v1 + - uses: actions/cache@v4 id: gradle-cache with: path: ~/.gradle/caches key: ${{ runner.os }}-gradle-${{ hashFiles('**/gradle/dependency-locks/*.lockfile') }} restore-keys: | - ${{ runner.os }}-gradle- - - uses: actions/cache@v1 + - uses: actions/cache@v4 id: gradle-wrapper-cache with: path: ~/.gradle/wrapper diff --git a/.github/workflows/nebula-snapshot.yml b/.github/workflows/nebula-snapshot.yml index 6e5cc87..ff02e9e 100644 --- a/.github/workflows/nebula-snapshot.yml +++ b/.github/workflows/nebula-snapshot.yml @@ -26,13 +26,13 @@ jobs: uses: actions/setup-java@v1 with: java-version: 8 - - uses: actions/cache@v2 + - uses: actions/cache@v4 id: gradle-cache with: path: | ~/.gradle/caches key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle') }} - - uses: actions/cache@v2 + - uses: actions/cache@v4 id: gradle-wrapper-cache with: path: | From 6104363527d0c6801101b8516306212e68d3517d Mon Sep 17 00:00:00 2001 From: Roberto Perez Alcolea Date: Tue, 10 Dec 2024 13:54:15 -0800 Subject: [PATCH 90/91] Upgrade Gradle --- .github/workflows/nebula-ci.yml | 5 ++++- .github/workflows/nebula-publish.yml | 5 ++++- .github/workflows/nebula-snapshot.yml | 5 ++++- build.gradle | 4 ++-- dyno-queues-core/build.gradle | 4 ++-- dyno-queues-redis/build.gradle | 24 ++++++++++++------------ gradle/wrapper/gradle-wrapper.properties | 2 +- 7 files changed, 29 insertions(+), 20 deletions(-) diff --git a/.github/workflows/nebula-ci.yml b/.github/workflows/nebula-ci.yml index 2b579d6..f0abc31 100644 --- a/.github/workflows/nebula-ci.yml +++ b/.github/workflows/nebula-ci.yml @@ -26,7 +26,10 @@ jobs: java: [ 8 ] name: CI with Java ${{ matrix.java }} steps: - - uses: actions/checkout@v1 + - uses: actions/checkout@v4 + - run: | + git config --global user.name "Netflix OSS Maintainers" + git config --global user.email "netflixoss@netflix.com" - name: Setup jdk uses: actions/setup-java@v1 with: diff --git a/.github/workflows/nebula-publish.yml b/.github/workflows/nebula-publish.yml index d997f54..c053f40 100644 --- a/.github/workflows/nebula-publish.yml +++ b/.github/workflows/nebula-publish.yml @@ -22,7 +22,10 @@ jobs: ports: - 6379:6379 steps: - - uses: actions/checkout@v1 + - uses: actions/checkout@v4 + - run: | + git config --global user.name "Netflix OSS Maintainers" + git config --global user.email "netflixoss@netflix.com" - name: Setup jdk 8 uses: actions/setup-java@v1 with: diff --git a/.github/workflows/nebula-snapshot.yml b/.github/workflows/nebula-snapshot.yml index ff02e9e..28df07c 100644 --- a/.github/workflows/nebula-snapshot.yml +++ b/.github/workflows/nebula-snapshot.yml @@ -19,9 +19,12 @@ jobs: ports: - 6379:6379 steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 with: fetch-depth: 0 + - run: | + git config --global user.name "Netflix OSS Maintainers" + git config --global user.email "netflixoss@netflix.com" - name: Set up JDK uses: actions/setup-java@v1 with: diff --git a/build.gradle b/build.gradle index e0c7081..0229160 100644 --- a/build.gradle +++ b/build.gradle @@ -13,7 +13,7 @@ buildscript { } plugins { - id 'nebula.netflixoss' version '9.1.0' + id 'com.netflix.nebula.netflixoss' version '11.5.0' } // Establish version and status @@ -23,7 +23,7 @@ apply plugin: 'project-report' subprojects { apply plugin: 'nebula.netflixoss' - apply plugin: 'java' + apply plugin: 'java-library' apply plugin: 'project-report' sourceCompatibility = 1.8 diff --git a/dyno-queues-core/build.gradle b/dyno-queues-core/build.gradle index 6071082..5764bb8 100644 --- a/dyno-queues-core/build.gradle +++ b/dyno-queues-core/build.gradle @@ -1,5 +1,5 @@ dependencies { - compile 'com.netflix.dyno:dyno-core:1.7.2-rc2' - testCompile "junit:junit:4.11" + api 'com.netflix.dyno:dyno-core:1.7.2-rc2' + testImplementation "junit:junit:4.11" } diff --git a/dyno-queues-redis/build.gradle b/dyno-queues-redis/build.gradle index d83a66b..614cbf7 100644 --- a/dyno-queues-redis/build.gradle +++ b/dyno-queues-redis/build.gradle @@ -1,20 +1,20 @@ dependencies { - compile project(':dyno-queues-core') + api project(':dyno-queues-core') - compile "com.google.inject:guice:3.0" + api "com.google.inject:guice:3.0" - compile 'com.netflix.dyno:dyno-core:1.7.2-rc2' - compile 'com.netflix.dyno:dyno-jedis:1.7.2-rc2' - compile 'com.netflix.dyno:dyno-demo:1.7.2-rc2' + api 'com.netflix.dyno:dyno-core:1.7.2-rc2' + api 'com.netflix.dyno:dyno-jedis:1.7.2-rc2' + api 'com.netflix.dyno:dyno-demo:1.7.2-rc2' - compile 'com.netflix.archaius:archaius-core:0.7.5' - compile 'com.netflix.servo:servo-core:0.12.17' - compile 'com.netflix.eureka:eureka-client:1.8.1' - compile 'com.fasterxml.jackson.core:jackson-databind:2.4.4' - - testCompile 'org.rarefiedredis.redis:redis-java:0.0.17' - testCompile "junit:junit:4.11" + api 'com.netflix.archaius:archaius-core:0.7.5' + api 'com.netflix.servo:servo-core:0.12.17' + api 'com.netflix.eureka:eureka-client:1.8.1' + api 'com.fasterxml.jackson.core:jackson-databind:2.4.4' + + testImplementation 'org.rarefiedredis.redis:redis-java:0.0.17' + testImplementation "junit:junit:4.11" } tasks.withType(Test) { diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index 470eeda..587ed3c 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -3,4 +3,4 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-6.8.3-all.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-8.11-all.zip From 3ba0d2a2a5ef80b1f03f90647a9517251e7b18e4 Mon Sep 17 00:00:00 2001 From: Akshay Shah Date: Mon, 9 Jun 2025 13:17:39 +0530 Subject: [PATCH 91/91] Fix lingering messages in the hash after remove() is invoked on the dyno queue --- .../dyno/queues/redis/RedisDynoQueue.java | 10 ++++---- .../dyno/queues/redis/RedisDynoQueueTest.java | 24 ++++++++++++++++++- 2 files changed, 28 insertions(+), 6 deletions(-) diff --git a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/RedisDynoQueue.java b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/RedisDynoQueue.java index 8ac7887..40e42c3 100644 --- a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/RedisDynoQueue.java +++ b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/RedisDynoQueue.java @@ -689,14 +689,14 @@ public boolean remove(String messageId) { for (String shard : allShards) { String unackShardKey = getUnackKey(queueName, shard); - quorumConn.zrem(unackShardKey, messageId); + Long removedFromUnack = quorumConn.zrem(unackShardKey, messageId); String queueShardKey = getQueueShardKey(queueName, shard); - Long removed = quorumConn.zrem(queueShardKey, messageId); + Long removedFromQueue = quorumConn.zrem(queueShardKey, messageId); - if (removed > 0) { + if ((removedFromUnack != null && removedFromUnack > 0) || (removedFromQueue != null && removedFromQueue > 0)) { // Ignoring return value since we just want to get rid of it. - Long msgRemoved = quorumConn.hdel(messageStoreKey, messageId); + quorumConn.hdel(messageStoreKey, messageId); return true; } } @@ -709,7 +709,7 @@ public boolean remove(String messageId) { sw.stop(); } } - + @Override public boolean atomicRemove(String messageId) { diff --git a/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/RedisDynoQueueTest.java b/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/RedisDynoQueueTest.java index c4ded37..9de4b1d 100644 --- a/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/RedisDynoQueueTest.java +++ b/dyno-queues-redis/src/test/java/com/netflix/dyno/queues/redis/RedisDynoQueueTest.java @@ -362,6 +362,29 @@ public void clear() { assertTrue(dynoClient.hlen(messageKey) == 0); } + @Test + public void testRemoveWhenMessageHasBeenPopped() { + List messages = new LinkedList<>(); + Message msg = new Message("1", "Hello World"); + msg.setPriority(1); + messages.add(msg); + rdq.push(messages); + rdq.pop(1, 1, TimeUnit.SECONDS); + rdq.remove(msg.getId()); + assertEquals(0, (long) dynoClient.hlen(messageKey)); + } + + @Test + public void testRemoveWhenMessageHasNotBeenPopped() { + List messages = new LinkedList<>(); + Message msg = new Message("1", "Hello World"); + msg.setPriority(1); + messages.add(msg); + rdq.push(messages); + rdq.remove(msg.getId()); + assertEquals(0, (long) dynoClient.hlen(messageKey)); + } + @Test public void testClearQueues() { rdq.clear(); @@ -379,5 +402,4 @@ public void testClearQueues() { assertEquals(0, rdq.size()); } - }