diff --git a/.github/workflows/nebula-ci.yml b/.github/workflows/nebula-ci.yml
new file mode 100644
index 0000000..f0abc31
--- /dev/null
+++ b/.github/workflows/nebula-ci.yml
@@ -0,0 +1,58 @@
+name: "CI"
+on:
+ push:
+ branches:
+ - '*'
+ tags-ignore:
+ - '*'
+ pull_request:
+
+jobs:
+ build:
+ runs-on: ubuntu-latest
+ services:
+ redis:
+ image: redis
+ options: >-
+ --health-cmd "redis-cli ping"
+ --health-interval 10s
+ --health-timeout 5s
+ --health-retries 5
+ ports:
+ - 6379:6379
+ strategy:
+ matrix:
+ # test against JDK 8
+ java: [ 8 ]
+ name: CI with Java ${{ matrix.java }}
+ steps:
+ - uses: actions/checkout@v4
+ - run: |
+ git config --global user.name "Netflix OSS Maintainers"
+ git config --global user.email "netflixoss@netflix.com"
+ - name: Setup jdk
+ uses: actions/setup-java@v1
+ with:
+ java-version: ${{ matrix.java }}
+ - uses: actions/cache@v4
+ id: gradle-cache
+ with:
+ path: ~/.gradle/caches
+ key: ${{ runner.os }}-gradle-${{ hashFiles('**/gradle/dependency-locks/*.lockfile') }}
+ restore-keys: |
+ - ${{ runner.os }}-gradle-
+ - uses: actions/cache@v4
+ id: gradle-wrapper-cache
+ with:
+ path: ~/.gradle/wrapper
+ key: ${{ runner.os }}-gradlewrapper-${{ hashFiles('gradle/wrapper/*') }}
+ restore-keys: |
+ - ${{ runner.os }}-gradlewrapper-
+ - name: Build with Gradle
+ run: ./gradlew --info --stacktrace build
+ env:
+ CI_NAME: github_actions
+ CI_BUILD_NUMBER: ${{ github.sha }}
+ CI_BUILD_URL: 'https://github.com/${{ github.repository }}'
+ CI_BRANCH: ${{ github.ref }}
+ COVERALLS_REPO_TOKEN: ${{ secrets.GITHUB_TOKEN }}
diff --git a/.github/workflows/nebula-publish.yml b/.github/workflows/nebula-publish.yml
new file mode 100644
index 0000000..c053f40
--- /dev/null
+++ b/.github/workflows/nebula-publish.yml
@@ -0,0 +1,64 @@
+name: "Publish candidate/release to NetflixOSS and Maven Central"
+on:
+ push:
+ tags:
+ - v*.*.*
+ - v*.*.*-rc.*
+ release:
+ types:
+ - published
+
+jobs:
+ build:
+ runs-on: ubuntu-latest
+ services:
+ redis:
+ image: redis
+ options: >-
+ --health-cmd "redis-cli ping"
+ --health-interval 10s
+ --health-timeout 5s
+ --health-retries 5
+ ports:
+ - 6379:6379
+ steps:
+ - uses: actions/checkout@v4
+ - run: |
+ git config --global user.name "Netflix OSS Maintainers"
+ git config --global user.email "netflixoss@netflix.com"
+ - name: Setup jdk 8
+ uses: actions/setup-java@v1
+ with:
+ java-version: 1.8
+ - uses: actions/cache@v4
+ id: gradle-cache
+ with:
+ path: ~/.gradle/caches
+ key: ${{ runner.os }}-gradle-${{ hashFiles('**/gradle/dependency-locks/*.lockfile') }}
+ restore-keys: |
+ - ${{ runner.os }}-gradle-
+ - uses: actions/cache@v4
+ id: gradle-wrapper-cache
+ with:
+ path: ~/.gradle/wrapper
+ key: ${{ runner.os }}-gradlewrapper-${{ hashFiles('gradle/wrapper/*') }}
+ restore-keys: |
+ - ${{ runner.os }}-gradlewrapper-
+ - name: Publish candidate
+ if: contains(github.ref, '-rc.')
+ run: ./gradlew --info --stacktrace -Prelease.useLastTag=true candidate
+ env:
+ NETFLIX_OSS_SIGNING_KEY: ${{ secrets.ORG_SIGNING_KEY }}
+ NETFLIX_OSS_SIGNING_PASSWORD: ${{ secrets.ORG_SIGNING_PASSWORD }}
+ NETFLIX_OSS_REPO_USERNAME: ${{ secrets.ORG_NETFLIXOSS_USERNAME }}
+ NETFLIX_OSS_REPO_PASSWORD: ${{ secrets.ORG_NETFLIXOSS_PASSWORD }}
+ - name: Publish release
+ if: (!contains(github.ref, '-rc.'))
+ run: ./gradlew --info -Prelease.useLastTag=true final
+ env:
+ NETFLIX_OSS_SONATYPE_USERNAME: ${{ secrets.ORG_SONATYPE_USERNAME }}
+ NETFLIX_OSS_SONATYPE_PASSWORD: ${{ secrets.ORG_SONATYPE_PASSWORD }}
+ NETFLIX_OSS_SIGNING_KEY: ${{ secrets.ORG_SIGNING_KEY }}
+ NETFLIX_OSS_SIGNING_PASSWORD: ${{ secrets.ORG_SIGNING_PASSWORD }}
+ NETFLIX_OSS_REPO_USERNAME: ${{ secrets.ORG_NETFLIXOSS_USERNAME }}
+ NETFLIX_OSS_REPO_PASSWORD: ${{ secrets.ORG_NETFLIXOSS_PASSWORD }}
diff --git a/.github/workflows/nebula-snapshot.yml b/.github/workflows/nebula-snapshot.yml
new file mode 100644
index 0000000..28df07c
--- /dev/null
+++ b/.github/workflows/nebula-snapshot.yml
@@ -0,0 +1,50 @@
+name: "Publish snapshot to NetflixOSS and Maven Central"
+
+on:
+ push:
+ branches:
+ - master
+
+jobs:
+ build:
+ runs-on: ubuntu-latest
+ services:
+ redis:
+ image: redis
+ options: >-
+ --health-cmd "redis-cli ping"
+ --health-interval 10s
+ --health-timeout 5s
+ --health-retries 5
+ ports:
+ - 6379:6379
+ steps:
+ - uses: actions/checkout@v4
+ with:
+ fetch-depth: 0
+ - run: |
+ git config --global user.name "Netflix OSS Maintainers"
+ git config --global user.email "netflixoss@netflix.com"
+ - name: Set up JDK
+ uses: actions/setup-java@v1
+ with:
+ java-version: 8
+ - uses: actions/cache@v4
+ id: gradle-cache
+ with:
+ path: |
+ ~/.gradle/caches
+ key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle') }}
+ - uses: actions/cache@v4
+ id: gradle-wrapper-cache
+ with:
+ path: |
+ ~/.gradle/wrapper
+ key: ${{ runner.os }}-gradlewrapper-${{ hashFiles('gradle/wrapper/*') }}
+ - name: Build
+ run: ./gradlew build snapshot
+ env:
+ NETFLIX_OSS_SIGNING_KEY: ${{ secrets.ORG_SIGNING_KEY }}
+ NETFLIX_OSS_SIGNING_PASSWORD: ${{ secrets.ORG_SIGNING_PASSWORD }}
+ NETFLIX_OSS_REPO_USERNAME: ${{ secrets.ORG_NETFLIXOSS_USERNAME }}
+ NETFLIX_OSS_REPO_PASSWORD: ${{ secrets.ORG_NETFLIXOSS_PASSWORD }}
diff --git a/.gitignore b/.gitignore
index e2f8b8c..72f70c3 100644
--- a/.gitignore
+++ b/.gitignore
@@ -3,3 +3,16 @@ dyno-queues-core/build
build
redis-3.0.7/
redis-3.0.7.tar.gz
+
+*.iml
+.idea
+.gradle
+.classpath
+.project
+
+
+dyno-queues-core/out/
+dyno-queues-redis/out/
+
+# publishing secrets
+secrets/signing-key
diff --git a/.travis.yml b/.travis.yml
deleted file mode 100644
index 5a13010..0000000
--- a/.travis.yml
+++ /dev/null
@@ -1,34 +0,0 @@
-language: java
-jdk:
-- oraclejdk8
-install:
- - export REDIS_BIN=$HOME/redis/3.0.7/bin
- - export TMPDIR=/tmp
- - wget -c https://github.com/antirez/redis/archive/3.0.7.tar.gz -O redis-3.0.7.tar.gz
- - mv redis-3.0.7.tar.gz $TMPDIR
- - cd $TMPDIR
- - tar -xvf redis-3.0.7.tar.gz
- - make -C redis-3.0.7 PREFIX=$HOME/redis/3.0.7 install
-before_script:
- - $REDIS_BIN/redis-server --daemonize yes
- - sleep 3
- - $REDIS_BIN/redis-cli PING
- - export REDIS_VERSION="$(redis-cli INFO SERVER | sed -n 2p)"
- - echo $REDIS_VERSION
- - cd $TRAVIS_BUILD_DIR
- - ls -la
- - git status .
-after_success:
- - redis-cli SHUTDOWN NOSAVE
-after_failure:
- - redis-cli SHUTDOWN NOSAVE
-script: "./buildViaTravis.sh"
-cache:
- directories:
- - "$HOME/.gradle/caches"
-env:
- global:
- - secure: NrYUZ8/zJLMeNPilbr32XzbA7159FE8Xi5c+GOodrxz7+5+W/RkIuW1e1INu6iA/PfC6qB3IhHVicH+HuREwnBnBDnz9F2HvcOEetmylLDJ9rV71Qq4Bbna5jORuwTqGf8EnGFYzxFMqgpjcIkXHOLnNeJtJSBdcLLQcIuc7JPKbk/i63k9sc8prRZs2G4jrsmFaBqJtRWmS4hWcOR5GJUn/URgDrlrKne6PkMprTzXdhrAURoGUKLgleGGNuzxGb0Bxb/Y1oRne9eWbR8FqwlDVY7RV7KjxF3dWGWJTdW8O8nYzH83G6AhqBlC93Y3Xi6nDWQHnKWi7JdPw6ohPGiikiqOV3deX7gwrXzLr1JjFGIsl8LFE74lzIjp9YBsM2O7l0Vm37YUM/UJorpjtLmTMB7fXZSrge8Cmu1nekq7btw2JS1fia5hk/RMUGGEzbSiOxG4/dgfE7qHwbG22OWHM3dyunDs8sJUQlkSBHnBukr5F5fgz3HemG+nH4x31JqHiajWrztYI7brwVWvndzs7fpyMBGGLIS0Ql/jV53Mktw5G/8N4tZgMstf7mfQtF3m2yqNHW3+iaB9R14DyobBcOrJi5A4opM70KwSR9QbyHAi+i833pc03CcMV13kaMQ5/YDDck8ywyZ/qfsR7J68TcLO1Wa/UHS7SamuTG5k=
- - secure: FXIh9ibMdmI7YMaE4k3H1eF6n8LiP9KdwuOuSagx81Rr43cpqIH5JFWPx5yY0Mws1scfXt3hRzPFZ1HWhCLoaAxedGnhT9NKWzj/RyTB+6AQR+3tpwh4OeJkvxC8QPBWWmC0hag7ahRkKfhiSHbQ53w6aw4/Hpn/TrnX/UEIhpIrDLbzMBAgF0P0FavX87A2rpZefgP0ugJo/GG3fpJ1qg4FWah1svSI6cFw+w3WKv1Imk42EHWEPFPzPCQjiXh5yCR+90oswf7oiImBpGY7M45YEZTr4Y6JaVOPVbTTpNdkhlevtTPFJXDrskqNxA1JlVpRXzEGjTQlezEZ4WYf0BFbFcyPGaryUtDXeCkkt9VoBfV3VE+p9btVNHMXOnxLBj20XXz6MQo2ucf0sqLtqnGTsaPYY2Ej9az9N8/DTnx0tR/Df6cwyYOQ6TUfEZdtyVVxCzgFvS6439/47BaEBdqx4AlmEL4fEe/WnEFHUGHDKQP68NMy0RTvGJSCxxEFYKiy/OKlFs2esRvEulmEceWnSlr0NFGtySuqO5ebt8bCy80K/5NCDT2TIMNF7wcg05tKEhd3rIqluUyAZ0gAZx3brQhWvVkDrBOoWV5PTrs/5iicrP8LtXYrsm8+OHAs7rpXbK3OtWEuBvjyMZptCtt08lvOVQZ02etWAg96+2U=
- - secure: iK6dRuLfnV+ZWhUxfcwMN3cQBlJowRysibtnV0c58j2UsP7cC4gkDeFnwIw3ggok751cdVXYQrRk2mJb4xhCiw90p1TU4VEHtfTjHKS/45jknrrFbScHplBcDoux6NLp4fUYdc2QswsPCsCFefIFZcpejumyZ4cAsJlfFTU/Kv3UT2SZyTkapcCmn/5FiIr2H3Ups+ShJOnxwNkE+svEcKdV/g67ZCzZfpPM79L4LpgA89/o3kItJSRHDWaECtTOoh759nJh+i7J4eCDbZTrZb27xuneeMgr1ffxHHN75z2Sf2PVwTQdnAdKbhVfmHx0nqms+ICLu3PZ8Wfb3wWOwGFlMXIsbpiZpOdKXQD+yTaR8NC9B8PbHFOFD/Q4kBJ5fdOh5tw4u8ejF4vro9l7z5RhEDbRtmrSH1OfM2RatmKRgx87hva1AzKhLOPSGbYSZ7OpVO/0slJSOXlcNsahIPCaiEmKvVQv2sCLJZdpVWXSA9HpmMb2S+wI8LI88sFZXIuz1mFRYtvxBDs3aIKMYWUXqjewI2ZXipi9INYiliTyKqp20RQNRvViTZLKYPmrN4tfbnYR9lBkEoX8UP7gKcS0CfuIjCBshkwEKfzXDbv0tlceRsi8fW4Dr93MDNvvIlI8vxIe2rW5WtcZdie3s1X0dg6j14nRBKIE9DpyCIA=
- - secure: QZ9O23vPVd4EoV5rKXGnHpkI4Ehn6fYy1Sj6YBTsWPdBBw/AZukkj6wDl9sNvDeoe3ub/U7VcbjhJBkpXAuPk8hWZ3Qk+2L9kkMkCSF+osfOmOI8ufoEtPT4cCbLwv2T41g+8Tp224S4AtoWQzSy91yTgvUOqfm3SXA200iJF5DkCv+i+Am+ySKYT4T+ZcPWyqpV6Y/5XPE3X99UbdJ4oasju1cPsMnYRfKNhIjIunaFgXIo/KtBF+IPEgDsfHwDRQ+7Lt55QGFK2yqsP50Zb6s9IqLgTngEu41pTwnoJCg0z5+q1ZmVbL8EHYOpMA3iopB+k5oY9vRbM/8Y83oQeT1fntSHxgpDuWax7m+1kxGOlv9FLUVoSBKq0dVjiE9WXLn4gdV/Dvkc5c1fPj82PhvkCJmVmlL90gf4/tDWIKvjGg2EIhSUzRkB1Me8RTBOD607OTViKVlzX0+T5y9ILOb1C2krEcrW+lEg49rdzvG/nUZqWXfhX5x7OJmmBpPnAnCVJnTkVfxs6Eq6cqIQc0oMZlYm7Rl6nOJCnqbYm33rwUIrRHxwtmyXEXXWyTxXoKpe+LZvXzT6+qhFuQnz1iFZjL+C5BbJ1nSBCKSAbLBsgteIMZWDZQhkwPfZPhpHxRIejNo6NnXXu6brn4JZgpi0g3No9DgBz/TLLfiS8g8=
diff --git a/README.md b/README.md
index 82068e8..e6a5c6b 100644
--- a/README.md
+++ b/README.md
@@ -1,5 +1,9 @@
+## DISCLAIMER: THIS PROJECT IS NO LONGER ACTIVELY MAINTAINED
+
+
+
# Dyno Queues
-[](https://travis-ci.org/Netflix/dyno-queues)
+[](https://travis-ci.com/Netflix/dyno-queues)
[](https://gitter.im/Netflix/dynomite?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge)
Dyno Queues is a recipe that provides task queues utilizing [Dynomite](https://github.com/Netflix/dynomite).
diff --git a/build.gradle b/build.gradle
index d56c935..0229160 100644
--- a/build.gradle
+++ b/build.gradle
@@ -1,16 +1,19 @@
buildscript {
-
- repositories {
- jcenter()
+ repositories {
+ mavenCentral()
+ maven {
+ url = 'https://plugins.gradle.org/m2'
+ }
}
dependencies {
- classpath 'com.netflix.nebula:gradle-aggregate-javadocs-plugin:2.2.+'
- classpath 'com.netflix.nebula:gradle-extra-configurations-plugin:3.0.3'
+ classpath 'com.netflix.nebula:gradle-aggregate-javadocs-plugin:3.0.1'
+ classpath 'com.netflix.nebula:gradle-extra-configurations-plugin:4.0.1'
}
}
+
plugins {
- id 'nebula.netflixoss' version '3.6.0'
+ id 'com.netflix.nebula.netflixoss' version '11.5.0'
}
// Establish version and status
@@ -19,18 +22,20 @@ ext.githubProjectName = rootProject.name // Change if github project name is not
apply plugin: 'project-report'
subprojects {
-
apply plugin: 'nebula.netflixoss'
- apply plugin: 'java'
- apply plugin: 'idea'
- apply plugin: 'eclipse'
+ apply plugin: 'java-library'
apply plugin: 'project-report'
sourceCompatibility = 1.8
targetCompatibility = 1.8
- repositories {
- jcenter()
+ repositories {
+ mavenCentral()
+
+ // oss-candidate for -rc.* verions:
+ maven {
+ url "https://dl.bintray.com/netflixoss/oss-candidate"
+ }
}
group = "com.netflix.${githubProjectName}"
diff --git a/buildViaTravis.sh b/buildViaTravis.sh
deleted file mode 100755
index 9cc169e..0000000
--- a/buildViaTravis.sh
+++ /dev/null
@@ -1,22 +0,0 @@
-#!/bin/bash
-# This script will build the project.
-if [ "$TRAVIS_PULL_REQUEST" != "false" ]; then
- echo -e "Build Pull Request #$TRAVIS_PULL_REQUEST => Branch [$TRAVIS_BRANCH]"
- ./gradlew build
-elif [ "$TRAVIS_PULL_REQUEST" == "false" ] && [ "$TRAVIS_TAG" == "" ]; then
- echo -e 'Build Branch with Snapshot => Branch ['$TRAVIS_BRANCH']'
- ./gradlew -Prelease.travisci=true -PbintrayUser="${bintrayUser}" -PbintrayKey="${bintrayKey}" -PsonatypeUsername="${sonatypeUsername}" -PsonatypePassword="${sonatypePassword}" build snapshot --info --stacktrace
-elif [ "$TRAVIS_PULL_REQUEST" == "false" ] && [ "$TRAVIS_TAG" != "" ]; then
- echo -e 'Build Branch for Release => Branch ['$TRAVIS_BRANCH'] Tag ['$TRAVIS_TAG']'
- case "$TRAVIS_TAG" in
- *-rc\.*)
- ./gradlew -Prelease.travisci=true -Prelease.useLastTag=true -PbintrayUser="${bintrayUser}" -PbintrayKey="${bintrayKey}" -PsonatypeUsername="${sonatypeUsername}" -PsonatypePassword="${sonatypePassword}" candidate --info --stacktrace
- ;;
- *)
- ./gradlew -Prelease.travisci=true -Prelease.useLastTag=true -PbintrayUser="${bintrayUser}" -PbintrayKey="${bintrayKey}" -PsonatypeUsername="${sonatypeUsername}" -PsonatypePassword="${sonatypePassword}" final --info --stacktrace
- ;;
- esac
-else
- echo -e 'WARN: Should not be here => Branch ['$TRAVIS_BRANCH'] Tag ['$TRAVIS_TAG'] Pull Request ['$TRAVIS_PULL_REQUEST']'
- ./gradlew build
-fi
diff --git a/dyno-queues-core/build.gradle b/dyno-queues-core/build.gradle
index 5820ad6..5764bb8 100644
--- a/dyno-queues-core/build.gradle
+++ b/dyno-queues-core/build.gradle
@@ -1,4 +1,5 @@
dependencies {
- testCompile "junit:junit:4.11"
-}
\ No newline at end of file
+ api 'com.netflix.dyno:dyno-core:1.7.2-rc2'
+ testImplementation "junit:junit:4.11"
+}
diff --git a/dyno-queues-core/src/main/java/com/netflix/dyno/queues/DynoQueue.java b/dyno-queues-core/src/main/java/com/netflix/dyno/queues/DynoQueue.java
index ea53058..1ded585 100644
--- a/dyno-queues-core/src/main/java/com/netflix/dyno/queues/DynoQueue.java
+++ b/dyno-queues-core/src/main/java/com/netflix/dyno/queues/DynoQueue.java
@@ -1,12 +1,12 @@
/**
* Copyright 2016 Netflix, Inc.
- *
+ *
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -14,13 +14,14 @@
* limitations under the License.
*/
/**
- *
+ *
*/
package com.netflix.dyno.queues;
import java.io.Closeable;
import java.util.List;
import java.util.Map;
+import java.util.Set;
import java.util.concurrent.TimeUnit;
/**
@@ -28,110 +29,298 @@
* Abstraction of a dyno queue.
*/
public interface DynoQueue extends Closeable {
-
- /**
- *
- * @return Returns the name of the queue
- */
- public String getName();
-
- /**
- *
- * @return Time in milliseconds before the messages that are popped and not acknowledge are pushed back into the queue.
- * @see #ack(String)
- */
- public int getUnackTime();
-
- /**
- *
- * @param messages messages to be pushed onto the queue
- * @return Returns the list of message ids
- */
- public List push(List messages);
-
- /**
- *
- * @param messageCount number of messages to be popped out of the queue.
- * @param wait Amount of time to wait if there are no messages in queue
- * @param unit Time unit for the wait period
- * @return messages. Can be less than the messageCount if there are fewer messages available than the message count. If the popped messages are not acknowledge in a timely manner, they are pushed back into the queue.
- * @see #peek(int)
- * @see #ack(String)
- * @see #getUnackTime()
- *
- */
- public List pop(int messageCount, int wait, TimeUnit unit);
-
+
+ /**
+ *
+ * @return Returns the name of the queue
+ */
+ public String getName();
+
+ /**
+ *
+ * @return Time in milliseconds before the messages that are popped and not acknowledge are pushed back into the queue.
+ * @see #ack(String)
+ */
+ public int getUnackTime();
+
+ /**
+ *
+ * @param messages messages to be pushed onto the queue
+ * @return Returns the list of message ids
+ */
+ public List push(List messages);
+
+ /**
+ *
+ * @param messageCount number of messages to be popped out of the queue.
+ * @param wait Amount of time to wait if there are no messages in queue
+ * @param unit Time unit for the wait period
+ * @return messages. Can be less than the messageCount if there are fewer messages available than the message count.
+ * If the popped messages are not acknowledge in a timely manner, they are pushed back into the queue.
+ * @see #peek(int)
+ * @see #ack(String)
+ * @see #getUnackTime()
+ *
+ */
+ public List pop(int messageCount, int wait, TimeUnit unit);
+
+ /**
+ * Pops "messageId" from the local shard if it exists.
+ * Note that if "messageId" is present in a different shard, we will be unable to pop it.
+ *
+ * @param messageId ID of message to pop
+ * @return Returns a "Message" object if pop was successful. 'null' otherwise.
+ */
+ public Message popWithMsgId(String messageId);
+
+ /**
+ * Provides a peek into the queue without taking messages out.
+ *
+ * Note: This peeks only into the 'local' shard.
+ *
+ * @param messageCount number of messages to be peeked.
+ * @return List of peeked messages.
+ * @see #pop(int, int, TimeUnit)
+ */
+ public List peek(int messageCount);
+
+ /**
+ * Provides an acknowledgement for the message. Once ack'ed the message is removed from the queue forever.
+ * @param messageId ID of the message to be acknowledged
+ * @return true if the message was found pending acknowledgement and is now ack'ed. false if the message id is invalid or message is no longer present in the queue.
+ */
+ public boolean ack(String messageId);
+
+
+ /**
+ * Bulk version for {@link #ack(String)}
+ * @param messages Messages to be acknowledged. Each message MUST be populated with id and shard information.
+ */
+ public void ack(List messages);
+
+ /**
+ * Sets the unack timeout on the message (changes the default timeout to the new value). Useful when extended lease is required for a message by consumer before sending ack.
+ * @param messageId ID of the message to be acknowledged
+ * @param timeout time in milliseconds for which the message will remain in un-ack state. If no ack is received after the timeout period has expired, the message is put back into the queue
+ * @return true if the message id was found and updated with new timeout. false otherwise.
+ */
+ public boolean setUnackTimeout(String messageId, long timeout);
+
+
+ /**
+ * Updates the timeout for the message.
+ * @param messageId ID of the message to be acknowledged
+ * @param timeout time in milliseconds for which the message will remain invisible and not popped out of the queue.
+ * @return true if the message id was found and updated with new timeout. false otherwise.
+ */
+ public boolean setTimeout(String messageId, long timeout);
+
+ /**
+ *
+ * @param messageId Remove the message from the queue
+ * @return true if the message id was found and removed. False otherwise.
+ */
+ public boolean remove(String messageId);
+ public boolean atomicRemove(String messageId);
+
+ /**
+ * Enqueues 'message' if it doesn't exist in any of the shards or unack sets.
+ *
+ * @param message Message to enqueue if it doesn't exist.
+ * @return true if message was enqueued. False if messageId already exists.
+ */
+ public boolean ensure(Message message);
+
+ /**
+ * Checks the message bodies (i.e. the data in the hash map), and returns true on the first match with
+ * 'predicate'.
+ *
+ * Matching is done based on 'lua pattern' matching.
+ * http://lua-users.org/wiki/PatternsTutorial
+ *
+ * Disclaimer: This is a potentially expensive call, since we will iterate over the entire hash map in the
+ * worst case. Use mindfully.
+ *
+ * @param predicate The predicate to check against.
+ * @return 'true' if any of the messages contain 'predicate'; 'false' otherwise.
+ */
+ public boolean containsPredicate(String predicate);
+
/**
- * Provides a peek into the queue without taking messages out.
- * @param messageCount number of messages to be peeked.
- * @return List of peeked messages.
- * @see #pop(int, int, TimeUnit)
+ * Checks the message bodies (i.e. the data in the hash map), and returns true on the first match with
+ * 'predicate'.
+ *
+ * Matching is done based on 'lua pattern' matching.
+ * http://lua-users.org/wiki/PatternsTutorial
+ *
+ * Disclaimer: This is a potentially expensive call, since we will iterate over the entire hash map in the
+ * worst case. Use mindfully.
+ *
+ * @param predicate The predicate to check against.
+ * @param localShardOnly If this is true, it will only check if the message exists in the local shard as opposed to
+ * all shards. Note that this will only work if the Dynomite cluster ring size is 1 (i.e. one
+ * instance per AZ).
+ * @return 'true' if any of the messages contain 'predicate'; 'false' otherwise.
*/
- public List peek(int messageCount);
-
+ public boolean containsPredicate(String predicate, boolean localShardOnly);
+
+ /**
+ * Checks the message bodies (i.e. the data in the hash map), and returns the ID of the first message to match with
+ * 'predicate'.
+ *
+ * Matching is done based on 'lua pattern' matching.
+ * http://lua-users.org/wiki/PatternsTutorial
+ *
+ * Disclaimer: This is a potentially expensive call, since we will iterate over the entire hash map in the
+ * worst case. Use mindfully.
+ *
+ * @param predicate The predicate to check against.
+ * @return Message ID as string if any of the messages contain 'predicate'; 'null' otherwise.
+ */
+ public String getMsgWithPredicate(String predicate);
+
/**
- * Provides an acknowledgement for the message. Once ack'ed the message is removed from the queue forever.
- * @param messageId ID of the message to be acknowledged
- * @return true if the message was found pending acknowledgement and is now ack'ed. false if the message id is invalid or message is no longer present in the queue.
+ * Checks the message bodies (i.e. the data in the hash map), and returns the ID of the first message to match with
+ * 'predicate'.
+ *
+ * Matching is done based on 'lua pattern' matching.
+ * http://lua-users.org/wiki/PatternsTutorial
+ *
+ * Disclaimer: This is a potentially expensive call, since we will iterate over the entire hash map in the
+ * worst case. Use mindfully.
+ *
+ * @param predicate The predicate to check against.
+ * @param localShardOnly If this is true, it will only check if the message exists in the local shard as opposed to
+ * all shards. Note that this will only work if the Dynomite cluster ring size is 1 (i.e. one
+ * instance per AZ).
+ * @return Message ID as string if any of the messages contain 'predicate'; 'null' otherwise.
*/
- public boolean ack(String messageId);
-
-
+ public String getMsgWithPredicate(String predicate, boolean localShardOnly);
+
/**
- * Bulk version for {@link #ack(String)}
- * @param messages Messages to be acknowledged. Each message MUST be populated with id and shard information.
+ * Pops the message with the highest priority that matches 'predicate'.
+ *
+ * Note: Can be slow for large queues.
+ *
+ * @param predicate The predicate to check against.
+ * @param localShardOnly If this is true, it will only check if the message exists in the local shard as opposed to
+ * all shards. Note that this will only work if the Dynomite cluster ring size is 1 (i.e. one
+ * instance per AZ).
+ * @return
*/
- public void ack(List messages);
-
+ public Message popMsgWithPredicate(String predicate, boolean localShardOnly);
+
+ /**
+ *
+ * @param messageId message to be retrieved.
+ * @return Retrieves the message stored in the queue by the messageId. Null if not found.
+ */
+ public Message get(String messageId);
+
/**
- * Sets the unack timeout on the message (changes the default timeout to the new value). Useful when extended lease is required for a message by consumer before sending ack.
- * @param messageId ID of the message to be acknowledged
- * @param timeout time in milliseconds for which the message will remain in un-ack state. If no ack is received after the timeout period has expired, the message is put back into the queue
- * @return true if the message id was found and updated with new timeout. false otherwise.
+ *
+ * Attempts to return all the messages found in the hashmap. It's a best-effort return of all payloads, i.e. it may
+ * not 100% match with what's in the queue metadata at any given time and is read with a non-quorum connection.
+ *
+ * @return Returns a list of all messages found in the message hashmap.
*/
- public boolean setUnackTimeout(String messageId, long timeout);
-
-
+ public List getAllMessages();
+
/**
- * Updates the timeout for the message.
- * @param messageId ID of the message to be acknowledged
- * @param timeout time in milliseconds for which the message will remain invisible and not popped out of the queue.
- * @return true if the message id was found and updated with new timeout. false otherwise.
+ *
+ * Same as get(), but uses the non quorum connection.
+ * @param messageId message to be retrieved.
+ * @return Retrieves the message stored in the queue by the messageId. Null if not found.
*/
- public boolean setTimeout(String messageId, long timeout);
-
+ public Message localGet(String messageId);
+
+ public List bulkPop(int messageCount, int wait, TimeUnit unit);
+ public List unsafeBulkPop(int messageCount, int wait, TimeUnit unit);
+
+ /**
+ *
+ * @return Size of the queue.
+ * @see #shardSizes()
+ */
+ public long size();
+
+ /**
+ *
+ * @return Map of shard name to the # of messages in the shard.
+ * @see #size()
+ */
+ public Map> shardSizes();
+
+ /**
+ * Truncates the entire queue. Use with caution!
+ */
+ public void clear();
+
+ /**
+ * Process un-acknowledged messages. The messages which are polled by the client but not ack'ed are moved back to queue
+ */
+ public void processUnacks();
+ public void atomicProcessUnacks();
+
/**
- *
- * @param messageId Remove the message from the queue
- * @return true if the message id was found and removed. False otherwise.
+ *
+ * Attempts to return the items present in the local queue shard but not in the hashmap, if any.
+ * (Ideally, we would not require this function, however, in some configurations, especially with multi-region write
+ * traffic sharing the same queue, we may find ourselves with stale items in the queue shards)
+ *
+ * @return List of stale messages IDs.
*/
- public boolean remove(String messageId);
-
-
- /**
- *
- * @param messageId message to be retrieved.
- * @return Retrieves the message stored in the queue by the messageId. Null if not found.
+ public List findStaleMessages();
+
+ /*
+ * <=== Begin unsafe* functions. ===>
+ *
+ * The unsafe functions listed below are not advisable to use.
+ * The reason they are listed as unsafe is that they operate over all shards of a queue which means that
+ * due to the eventually consistent nature of Dynomite, the calling application may see duplicate item(s) that
+ * may have already been popped in a different rack, by another instance of the same application.
+ *
+ * Why are these functions made available then?
+ * There are some users of dyno-queues who have use-cases that are completely okay with dealing with duplicate
+ * items.
*/
- public Message get(String messageId);
-
+
/**
- *
- * @return Size of the queue.
- * @see #shardSizes()
+ * Provides a peek into all shards of the queue without taking messages out.
+ * Note: This function does not guarantee ordering of items based on shards like unsafePopAllShards().
+ *
+ * @param messageCount The number of messages to peek.
+ * @return A list of up to 'count' messages.
*/
- public long size();
+ public List unsafePeekAllShards(final int messageCount);
+
/**
- *
- * @return Map of shard name to the # of messages in the shard.
- * @see #size()
+ * Allows popping from all shards of the queue.
+ *
+ * Note: The local shard will always be looked into first and other shards will be filled behind it (if 'messageCount' is
+ * greater than the number of elements in the local shard). This way we ensure the chances of duplicates are less.
+ *
+ * @param messageCount number of messages to be popped out of the queue.
+ * @param wait Amount of time to wait for each shard if there are no messages in shard.
+ * @param unit Time unit for the wait period
+ * @return messages. Can be less than the messageCount if there are fewer messages available than the message count.
+ * If the popped messages are not acknowledge in a timely manner, they are pushed back into
+ * the queue.
+ * @see #peek(int)
+ * @see #ack(String)
+ * @see #getUnackTime()
+ *
*/
- public Map> shardSizes();
-
+ public List unsafePopAllShards(int messageCount, int wait, TimeUnit unit);
+
+
/**
- * Truncates the entire queue. Use with caution!
+ * Same as popWithMsgId(), but allows popping from any shard.
+ *
+ * @param messageId ID of message to pop
+ * @return Returns a "Message" object if pop was successful. 'null' otherwise.
*/
- public void clear();
+ public Message unsafePopWithMsgIdAllShards(String messageId);
+
}
diff --git a/dyno-queues-core/src/main/java/com/netflix/dyno/queues/Message.java b/dyno-queues-core/src/main/java/com/netflix/dyno/queues/Message.java
index 024e447..6d62af8 100644
--- a/dyno-queues-core/src/main/java/com/netflix/dyno/queues/Message.java
+++ b/dyno-queues-core/src/main/java/com/netflix/dyno/queues/Message.java
@@ -1,12 +1,12 @@
/**
* Copyright 2016 Netflix, Inc.
- *
+ *
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -14,7 +14,7 @@
* limitations under the License.
*/
/**
- *
+ *
*/
package com.netflix.dyno.queues;
@@ -26,140 +26,140 @@
*/
public class Message {
- private String id;
-
- private String payload;
-
- private long timeout;
-
- private int priority;
-
- private String shard;
-
- public Message() {
-
- }
-
- public Message(String id, String payload) {
- this.id = id;
- this.payload = payload;
- }
-
- /**
- * @return the id
- */
- public String getId() {
- return id;
- }
-
- /**
- * @param id
- * the id to set
- */
- public void setId(String id) {
- this.id = id;
- }
-
- /**
- * @return the payload
- */
- public String getPayload() {
- return payload;
- }
-
- /**
- * @param payload the payload to set
- *
- */
- public void setPayload(String payload) {
- this.payload = payload;
- }
-
- /**
- *
- * @param timeout Timeout in milliseconds - The message is only given to the consumer after the specified milliseconds have elapsed.
- */
- public void setTimeout(long timeout) {
- this.timeout = timeout;
- }
-
- /**
- * Helper method for the {@link #setTimeout(long)}
- * @param time timeout time
- * @param unit unit for the time
- * @see #setTimeout(long)
- */
- public void setTimeout(long time, TimeUnit unit) {
- this.timeout = TimeUnit.MILLISECONDS.convert(time, unit);
- }
-
- /**
- *
- * @return Returns the timeout for the message
- */
- public long getTimeout() {
- return timeout;
- }
-
- /**
- * Sets the message priority. Higher priority message is retrieved ahead of lower priority ones
- * @param priority priority for the message.
- */
- public void setPriority(int priority) {
- if(priority < 0 || priority > 99){
- throw new IllegalArgumentException("prioirty MUST be between 0 and 99 (inclusive)");
- }
- this.priority = priority;
- }
-
- public int getPriority() {
- return priority;
- }
-
- /**
- * @return the shard
- */
- public String getShard() {
- return shard;
- }
-
- /**
- * @param shard the shard to set
- *
- */
- public void setShard(String shard) {
- this.shard = shard;
- }
-
- @Override
- public int hashCode() {
- final int prime = 31;
- int result = 1;
- result = prime * result + ((id == null) ? 0 : id.hashCode());
- return result;
- }
-
- @Override
- public boolean equals(Object obj) {
- if (this == obj)
- return true;
- if (obj == null)
- return false;
- if (getClass() != obj.getClass())
- return false;
- Message other = (Message) obj;
- if (id == null) {
- if (other.id != null)
- return false;
- } else if (!id.equals(other.id))
- return false;
- return true;
- }
-
- @Override
- public String toString() {
- return "Message [id=" + id + ", payload=" + payload + ", timeout=" + timeout + ", priority=" + priority + "]";
- }
-
-
+ private String id;
+
+ private String payload;
+
+ private long timeout;
+
+ private int priority;
+
+ private String shard;
+
+ public Message() {
+
+ }
+
+ public Message(String id, String payload) {
+ this.id = id;
+ this.payload = payload;
+ }
+
+ /**
+ * @return the id
+ */
+ public String getId() {
+ return id;
+ }
+
+ /**
+ * @param id
+ * the id to set
+ */
+ public void setId(String id) {
+ this.id = id;
+ }
+
+ /**
+ * @return the payload
+ */
+ public String getPayload() {
+ return payload;
+ }
+
+ /**
+ * @param payload the payload to set
+ *
+ */
+ public void setPayload(String payload) {
+ this.payload = payload;
+ }
+
+ /**
+ *
+ * @param timeout Timeout in milliseconds - The message is only given to the consumer after the specified milliseconds have elapsed.
+ */
+ public void setTimeout(long timeout) {
+ this.timeout = timeout;
+ }
+
+ /**
+ * Helper method for the {@link #setTimeout(long)}
+ * @param time timeout time
+ * @param unit unit for the time
+ * @see #setTimeout(long)
+ */
+ public void setTimeout(long time, TimeUnit unit) {
+ this.timeout = TimeUnit.MILLISECONDS.convert(time, unit);
+ }
+
+ /**
+ *
+ * @return Returns the timeout for the message
+ */
+ public long getTimeout() {
+ return timeout;
+ }
+
+ /**
+ * Sets the message priority. Higher priority message is retrieved ahead of lower priority ones
+ * @param priority priority for the message.
+ */
+ public void setPriority(int priority) {
+ if (priority < 0 || priority > 99) {
+ throw new IllegalArgumentException("priority MUST be between 0 and 99 (inclusive)");
+ }
+ this.priority = priority;
+ }
+
+ public int getPriority() {
+ return priority;
+ }
+
+ /**
+ * @return the shard
+ */
+ public String getShard() {
+ return shard;
+ }
+
+ /**
+ * @param shard the shard to set
+ *
+ */
+ public void setShard(String shard) {
+ this.shard = shard;
+ }
+
+ @Override
+ public int hashCode() {
+ final int prime = 31;
+ int result = 1;
+ result = prime * result + ((id == null) ? 0 : id.hashCode());
+ return result;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj)
+ return true;
+ if (obj == null)
+ return false;
+ if (getClass() != obj.getClass())
+ return false;
+ Message other = (Message) obj;
+ if (id == null) {
+ if (other.id != null)
+ return false;
+ } else if (!id.equals(other.id))
+ return false;
+ return true;
+ }
+
+ @Override
+ public String toString() {
+ return "Message [id=" + id + ", payload=" + payload + ", timeout=" + timeout + ", priority=" + priority + "]";
+ }
+
+
}
diff --git a/dyno-queues-core/src/main/java/com/netflix/dyno/queues/ShardSupplier.java b/dyno-queues-core/src/main/java/com/netflix/dyno/queues/ShardSupplier.java
index f6a2b6a..e8bf518 100644
--- a/dyno-queues-core/src/main/java/com/netflix/dyno/queues/ShardSupplier.java
+++ b/dyno-queues-core/src/main/java/com/netflix/dyno/queues/ShardSupplier.java
@@ -1,12 +1,12 @@
/**
* Copyright 2016 Netflix, Inc.
- *
+ *
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -14,10 +14,12 @@
* limitations under the License.
*/
/**
- *
+ *
*/
package com.netflix.dyno.queues;
+import com.netflix.dyno.connectionpool.Host;
+
import java.util.Set;
@@ -27,16 +29,22 @@
*/
public interface ShardSupplier {
- /**
- *
- * @return Provides the set of all the available queue shards. The elements are evenly distributed amongst these shards
- */
- public Set getQueueShards();
-
- /**
- *
- * @return Name of the current shard. Used when popping elements out of the queue
- */
- public String getCurrentShard();
-
+ /**
+ *
+ * @return Provides the set of all the available queue shards. The elements are evenly distributed amongst these shards
+ */
+ public Set getQueueShards();
+
+ /**
+ *
+ * @return Name of the current shard. Used when popping elements out of the queue
+ */
+ public String getCurrentShard();
+
+ /**
+ *
+ * @param host
+ * @return shard for this host based on the rack
+ */
+ public String getShardForHost(Host host);
}
diff --git a/dyno-queues-core/src/test/java/com/netflix/dyno/queues/TestMessage.java b/dyno-queues-core/src/test/java/com/netflix/dyno/queues/TestMessage.java
index a59b5f2..4b3c510 100644
--- a/dyno-queues-core/src/test/java/com/netflix/dyno/queues/TestMessage.java
+++ b/dyno-queues-core/src/test/java/com/netflix/dyno/queues/TestMessage.java
@@ -1,12 +1,12 @@
/**
* Copyright 2016 Netflix, Inc.
- *
+ *
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -14,7 +14,7 @@
* limitations under the License.
*/
/**
- *
+ *
*/
package com.netflix.dyno.queues;
@@ -30,25 +30,25 @@
*/
public class TestMessage {
- @Test
- public void test(){
- Message msg = new Message();
- msg.setPayload("payload");
- msg.setTimeout(10, TimeUnit.SECONDS);
- assertEquals(msg.toString(), 10*1000, msg.getTimeout());
- msg.setTimeout(10);
- assertEquals(msg.toString(), 10, msg.getTimeout());
- }
-
- @Test(expected=IllegalArgumentException.class)
- public void testPrioirty(){
- Message msg = new Message();
- msg.setPriority(-1);
- }
-
- @Test(expected=IllegalArgumentException.class)
- public void testPrioirty2(){
- Message msg = new Message();
- msg.setPriority(100);
- }
+ @Test
+ public void test() {
+ Message msg = new Message();
+ msg.setPayload("payload");
+ msg.setTimeout(10, TimeUnit.SECONDS);
+ assertEquals(msg.toString(), 10 * 1000, msg.getTimeout());
+ msg.setTimeout(10);
+ assertEquals(msg.toString(), 10, msg.getTimeout());
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testPrioirty() {
+ Message msg = new Message();
+ msg.setPriority(-1);
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testPrioirty2() {
+ Message msg = new Message();
+ msg.setPriority(100);
+ }
}
diff --git a/dyno-queues-redis/build.gradle b/dyno-queues-redis/build.gradle
index 27074b8..614cbf7 100644
--- a/dyno-queues-redis/build.gradle
+++ b/dyno-queues-redis/build.gradle
@@ -1,17 +1,20 @@
dependencies {
- compile project(':dyno-queues-core')
-
- compile "com.google.inject:guice:3.0"
- compile 'com.netflix.dyno:dyno-core:1.5.9'
- compile "com.netflix.dyno:dyno-jedis:1.5.9"
- compile "com.netflix.archaius:archaius-core:0.7.5"
- compile "com.netflix.servo:servo-core:0.12.17"
- compile 'com.netflix.eureka:eureka-client:1.8.1'
- compile 'com.fasterxml.jackson.core:jackson-databind:2.4.4'
-
- testCompile 'org.rarefiedredis.redis:redis-java:0.0.17'
- testCompile "junit:junit:4.11"
+ api project(':dyno-queues-core')
+
+ api "com.google.inject:guice:3.0"
+
+ api 'com.netflix.dyno:dyno-core:1.7.2-rc2'
+ api 'com.netflix.dyno:dyno-jedis:1.7.2-rc2'
+ api 'com.netflix.dyno:dyno-demo:1.7.2-rc2'
+
+ api 'com.netflix.archaius:archaius-core:0.7.5'
+ api 'com.netflix.servo:servo-core:0.12.17'
+ api 'com.netflix.eureka:eureka-client:1.8.1'
+ api 'com.fasterxml.jackson.core:jackson-databind:2.4.4'
+
+ testImplementation 'org.rarefiedredis.redis:redis-java:0.0.17'
+ testImplementation "junit:junit:4.11"
}
tasks.withType(Test) {
diff --git a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/demo/DynoQueueDemo.java b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/demo/DynoQueueDemo.java
new file mode 100644
index 0000000..ea46ba0
--- /dev/null
+++ b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/demo/DynoQueueDemo.java
@@ -0,0 +1,199 @@
+package com.netflix.dyno.queues.demo;
+
+import com.netflix.dyno.demo.redis.DynoJedisDemo;
+import com.netflix.dyno.jedis.DynoJedisClient;
+import com.netflix.dyno.queues.DynoQueue;
+import com.netflix.dyno.queues.Message;
+import com.netflix.dyno.queues.redis.RedisQueues;
+import com.netflix.dyno.queues.redis.v2.QueueBuilder;
+import com.netflix.dyno.queues.shard.ConsistentAWSDynoShardSupplier;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Properties;
+import java.util.concurrent.TimeUnit;
+
+public class DynoQueueDemo extends DynoJedisDemo {
+
+ private static final Logger logger = LoggerFactory.getLogger(DynoQueue.class);
+
+ public DynoQueueDemo(String clusterName, String localRack) {
+ super(clusterName, localRack);
+ }
+
+ public DynoQueueDemo(String primaryCluster, String shadowCluster, String localRack) {
+ super(primaryCluster, shadowCluster, localRack);
+ }
+
+ /**
+ * Provide the cluster name to connect to as an argument to the function.
+ * throws java.lang.RuntimeException: java.net.ConnectException: Connection timed out (Connection timed out)
+ * if the cluster is not reachable.
+ *
+ * @param args: cluster-name version
+ *
+ * cluster-name: Name of cluster to run demo against
+ * version: Possible values = 1 or 2; (for V1 or V2)
+ */
+ public static void main(String[] args) throws IOException {
+ final String clusterName = args[0];
+
+ if (args.length < 2) {
+ throw new IllegalArgumentException("Need to pass in cluster-name and version of dyno-queues to run as arguments");
+ }
+
+ int version = Integer.parseInt(args[1]);
+ final DynoQueueDemo demo = new DynoQueueDemo(clusterName, "us-east-1e");
+ Properties props = new Properties();
+ props.load(DynoQueueDemo.class.getResourceAsStream("/demo.properties"));
+ for (String name : props.stringPropertyNames()) {
+ System.setProperty(name, props.getProperty(name));
+ }
+
+ try {
+ demo.initWithRemoteClusterFromEurekaUrl(args[0], 8102, false);
+
+ if (version == 1) {
+ demo.runSimpleV1Demo(demo.client);
+ } else if (version == 2) {
+ demo.runSimpleV2QueueDemo(demo.client);
+ }
+ Thread.sleep(10000);
+
+ } catch (Exception ex) {
+ ex.printStackTrace();
+ } finally {
+ demo.stop();
+ logger.info("Done");
+ }
+ }
+
+
+ private void runSimpleV1Demo(DynoJedisClient dyno) throws IOException {
+ String region = System.getProperty("LOCAL_DATACENTER");
+ String localRack = System.getProperty("LOCAL_RACK");
+
+ String prefix = "dynoQueue_";
+
+ ConsistentAWSDynoShardSupplier ss = new ConsistentAWSDynoShardSupplier(dyno.getConnPool().getConfiguration().getHostSupplier(), region, localRack);
+
+ RedisQueues queues = new RedisQueues(dyno, dyno, prefix, ss, 50_000, 50_000);
+
+ List payloads = new ArrayList<>();
+ payloads.add(new Message("id1", "searchable payload123"));
+ payloads.add(new Message("id2", "payload 2"));
+ payloads.add(new Message("id3", "payload 3"));
+ payloads.add(new Message("id4", "payload 4"));
+ payloads.add(new Message("id5", "payload 5"));
+ payloads.add(new Message("id6", "payload 6"));
+ payloads.add(new Message("id7", "payload 7"));
+ payloads.add(new Message("id8", "payload 8"));
+ payloads.add(new Message("id9", "payload 9"));
+ payloads.add(new Message("id10", "payload 10"));
+ payloads.add(new Message("id11", "payload 11"));
+ payloads.add(new Message("id12", "payload 12"));
+ payloads.add(new Message("id13", "payload 13"));
+ payloads.add(new Message("id14", "payload 14"));
+ payloads.add(new Message("id15", "payload 15"));
+
+ DynoQueue V1Queue = queues.get("simpleQueue");
+
+ // Clear the queue in case the server already has the above key.
+ V1Queue.clear();
+
+ // Test push() API
+ List pushed_msgs = V1Queue.push(payloads);
+
+ // Test ensure() API
+ Message msg1 = payloads.get(0);
+ logger.info("Does Message with ID '" + msg1.getId() + "' already exist? -> " + !V1Queue.ensure(msg1));
+
+ // Test containsPredicate() API
+ logger.info("Does the predicate 'searchable' exist in the queue? -> " + V1Queue.containsPredicate("searchable"));
+
+ // Test getMsgWithPredicate() API
+ logger.info("Get MSG ID that contains 'searchable' in the queue -> " + V1Queue.getMsgWithPredicate("searchable pay*"));
+
+ // Test getMsgWithPredicate(predicate, localShardOnly=true) API
+ // NOTE: This only works on single ring sized Dynomite clusters.
+ logger.info("Get MSG ID that contains 'searchable' in the queue -> " + V1Queue.getMsgWithPredicate("searchable pay*", true));
+ logger.info("Get MSG ID that contains '3' in the queue -> " + V1Queue.getMsgWithPredicate("3", true));
+
+ Message poppedWithPredicate = V1Queue.popMsgWithPredicate("searchable pay*", false);
+ V1Queue.ack(poppedWithPredicate.getId());
+
+ List specific_pops = new ArrayList<>();
+ // We'd only be able to pop from the local shard with popWithMsgId(), so try to pop the first payload ID we see in the local shard.
+ // Until then pop all messages not in the local shard with unsafePopWithMsgIdAllShards().
+ for (int i = 1; i < payloads.size(); ++i) {
+ Message popWithMsgId = V1Queue.popWithMsgId(payloads.get(i).getId());
+ if (popWithMsgId != null) {
+ specific_pops.add(popWithMsgId);
+ break;
+ } else {
+ // If we were unable to pop using popWithMsgId(), that means the message ID does not exist in the local shard.
+ // Ensure that we can pop with unsafePopWithMsgIdAllShards().
+ Message unsafeSpecificPop = V1Queue.unsafePopWithMsgIdAllShards(payloads.get(i).getId());
+ assert(unsafeSpecificPop != null);
+ boolean ack = V1Queue.ack(unsafeSpecificPop.getId());
+ assert(ack);
+ }
+ }
+
+ // Test ack()
+ boolean ack_successful = V1Queue.ack(specific_pops.get(0).getId());
+ assert(ack_successful);
+
+ // Test remove()
+ // Note: This checks for "id9" specifically as it implicitly expects every 3rd element we push to be in our
+ // local shard.
+ boolean removed = V1Queue.remove("id9");
+ assert(removed);
+
+ // Test pop(). Even though we try to pop 3 messages, there will only be one remaining message in our local shard.
+ List popped_msgs = V1Queue.pop(1, 1000, TimeUnit.MILLISECONDS);
+ V1Queue.ack(popped_msgs.get(0).getId());
+
+ // Test unsafePeekAllShards()
+ List peek_all_msgs = V1Queue.unsafePeekAllShards(5);
+ for (Message msg : peek_all_msgs) {
+ logger.info("Message peeked (ID : payload) -> " + msg.getId() + " : " + msg.getPayload());
+ }
+
+ // Test unsafePopAllShards()
+ List pop_all_msgs = V1Queue.unsafePopAllShards(7, 1000, TimeUnit.MILLISECONDS);
+ for (Message msg : pop_all_msgs) {
+ logger.info("Message popped (ID : payload) -> " + msg.getId() + " : " + msg.getPayload());
+ boolean ack = V1Queue.ack(msg.getId());
+ assert(ack);
+ }
+
+ V1Queue.clear();
+ V1Queue.close();
+ }
+
+ private void runSimpleV2QueueDemo(DynoJedisClient dyno) throws IOException {
+ String prefix = "dynoQueue_";
+
+ DynoQueue queue = new QueueBuilder()
+ .setQueueName("test")
+ .setRedisKeyPrefix(prefix)
+ .useDynomite(dyno, dyno)
+ .setUnackTime(50_000)
+ .build();
+
+ Message msg = new Message("id1", "message payload");
+ queue.push(Arrays.asList(msg));
+
+ int count = 10;
+ List polled = queue.pop(count, 1, TimeUnit.SECONDS);
+ logger.info(polled.toString());
+
+ queue.ack("id1");
+ queue.close();
+ }
+}
diff --git a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/DynoShardSupplier.java b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/DynoShardSupplier.java
deleted file mode 100644
index 54245e2..0000000
--- a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/DynoShardSupplier.java
+++ /dev/null
@@ -1,62 +0,0 @@
-/**
- * Copyright 2016 Netflix, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-/**
- *
- */
-package com.netflix.dyno.queues.redis;
-
-import java.util.Set;
-import java.util.stream.Collectors;
-
-import com.netflix.dyno.connectionpool.HostSupplier;
-import com.netflix.dyno.queues.ShardSupplier;
-
-/**
- * @author Viren
- *
- */
-public class DynoShardSupplier implements ShardSupplier {
-
- private HostSupplier hs;
-
- private String region;
-
- private String localDC;
-
- /**
- * Dynomite based shard supplier. Keeps the number of shards in parity with the hosts and regions
- * @param hs Host supplier
- * @param region current region
- * @param localDC local data center identifier
- */
- public DynoShardSupplier(HostSupplier hs, String region, String localDC) {
- this.hs = hs;
- this.region = region;
- this.localDC = localDC;
- }
-
- @Override
- public String getCurrentShard() {
- return localDC;
- }
-
- @Override
- public Set getQueueShards() {
- return hs.getHosts().stream().map(host -> host.getRack()).map(rack -> rack.replaceAll(region, "")).collect(Collectors.toSet());
- }
-
-
-}
diff --git a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/MultiRedisQueue.java b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/MultiRedisQueue.java
deleted file mode 100644
index 1499974..0000000
--- a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/MultiRedisQueue.java
+++ /dev/null
@@ -1,407 +0,0 @@
-/**
- * Copyright 2017 Netflix, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.netflix.dyno.queues.redis;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.stream.Collectors;
-
-import com.google.common.base.Function;
-import com.google.common.collect.Collections2;
-import com.google.common.collect.Lists;
-import com.netflix.appinfo.AmazonInfo;
-import com.netflix.appinfo.AmazonInfo.MetaDataKey;
-import com.netflix.appinfo.InstanceInfo;
-import com.netflix.appinfo.InstanceInfo.InstanceStatus;
-import com.netflix.discovery.EurekaClient;
-import com.netflix.discovery.shared.Application;
-import com.netflix.dyno.connectionpool.Host;
-import com.netflix.dyno.queues.DynoQueue;
-import com.netflix.dyno.queues.Message;
-
-import redis.clients.jedis.JedisPool;
-import redis.clients.jedis.JedisPoolConfig;
-
-/**
- * @author Viren
- *
- */
-public class MultiRedisQueue implements DynoQueue {
-
- private List shards;
-
- private String name;
-
- private Map queues = new HashMap<>();
-
- private RedisQueue me;
-
- public MultiRedisQueue(String queueName, String shardName, Map queues) {
- this.name = queueName;
- this.queues = queues;
- this.me = queues.get(shardName);
- if(me == null) {
- throw new IllegalArgumentException("List of shards supplied (" + queues.keySet() + ") does not contain current shard name: " + shardName);
- }
- this.shards = queues.keySet().stream().collect(Collectors.toList());
- }
-
- @Override
- public String getName() {
- return name;
- }
-
- @Override
- public int getUnackTime() {
- return me.getUnackTime();
- }
-
- @Override
- public List push(List messages) {
- int size = queues.size();
- int partitionSize = messages.size()/size;
- List ids = new LinkedList<>();
-
- for(int i = 0; i < size-1; i++) {
- RedisQueue queue = queues.get(getNextShard());
- int start = i * partitionSize;
- int end = start + partitionSize;
- ids.addAll(queue.push(messages.subList(start, end)));
- }
- RedisQueue queue = queues.get(getNextShard());
- int start = (size-1) * partitionSize;
-
- ids.addAll(queue.push(messages.subList(start, messages.size())));
- return ids;
- }
-
- @Override
- public List pop(int messageCount, int wait, TimeUnit unit) {
- return me.pop(messageCount, wait, unit);
- }
-
- @Override
- public List peek(int messageCount) {
- return me.peek(messageCount);
- }
-
- @Override
- public boolean ack(String messageId) {
- for(DynoQueue q : queues.values()) {
- if(q.ack(messageId)) {
- return true;
- }
- }
- return false;
- }
-
- @Override
- public void ack(List messages) {
- Map> byShard = messages.stream().collect(Collectors.groupingBy(Message::getShard));
- for(Entry> e: byShard.entrySet()) {
- queues.get(e.getKey()).ack(e.getValue());
- }
- }
-
- @Override
- public boolean setUnackTimeout(String messageId, long timeout) {
- for(DynoQueue q : queues.values()) {
- if(q.setUnackTimeout(messageId, timeout)) {
- return true;
- }
- }
- return false;
- }
-
- @Override
- public boolean setTimeout(String messageId, long timeout) {
- for(DynoQueue q : queues.values()) {
- if(q.setTimeout(messageId, timeout)) {
- return true;
- }
- }
- return false;
- }
-
- @Override
- public boolean remove(String messageId) {
- for(DynoQueue q : queues.values()) {
- if(q.remove(messageId)) {
- return true;
- }
- }
- return false;
- }
-
- @Override
- public Message get(String messageId) {
- for(DynoQueue q : queues.values()) {
- Message msg = q.get(messageId);
- if(msg != null) {
- return msg;
- }
- }
- return null;
- }
-
- @Override
- public long size() {
- long size = 0;
- for(DynoQueue q : queues.values()) {
- size += q.size();
- }
- return size;
- }
-
- @Override
- public Map> shardSizes() {
- Map> sizes = new HashMap<>();
- for(Entry e : queues.entrySet()) {
- sizes.put(e.getKey(), e.getValue().shardSizes().get(e.getKey()));
- }
- return sizes;
- }
-
- @Override
- public void clear() {
- for(DynoQueue q : queues.values()) {
- q.clear();
- }
-
- }
-
- @Override
- public void close() throws IOException {
- for(RedisQueue queue : queues.values()) {
- queue.close();
- }
- }
-
- public void processUnacks() {
- for(RedisQueue queue : queues.values()) {
- queue.processUnacks();
- }
- }
-
- private AtomicInteger nextShardIndex = new AtomicInteger(0);
-
- private String getNextShard() {
- int indx = nextShardIndex.incrementAndGet();
- if (indx >= shards.size()) {
- nextShardIndex.set(0);
- indx = 0;
- }
- String s = shards.get(indx);
- return s;
- }
-
-
- public static class Builder {
-
- private String queueName;
-
- private EurekaClient ec;
-
- private String dynomiteClusterName;
-
- private String redisKeyPrefix;
-
- private int unackTime;
-
- private String currentShard;
-
- private Function hostToShardMap;
-
- private int redisPoolSize;
-
- private int quorumPort;
-
- private int nonQuorumPort;
-
- private List hosts;
-
- /**
- * @param queueName the queueName to set
- * @return instance of builder
- */
- public Builder setQueueName(String queueName) {
- this.queueName = queueName;
- return this;
- }
-
- /**
- * @param ec the ec to set
- * @return instance of builder
- */
- public Builder setEc(EurekaClient ec) {
- this.ec = ec;
- return this;
- }
-
- /**
- * @param dynomiteClusterName the dynomiteClusterName to set
- * @return instance of builder
- */
- public Builder setDynomiteClusterName(String dynomiteClusterName) {
- this.dynomiteClusterName = dynomiteClusterName;
- return this;
- }
-
- /**
- * @param redisKeyPrefix the redisKeyPrefix to set
- * @return instance of builder
- */
- public Builder setRedisKeyPrefix(String redisKeyPrefix) {
- this.redisKeyPrefix = redisKeyPrefix;
- return this;
- }
-
- /**
- * @param unackTime the unackTime to set
- * @return instance of builder
- */
- public Builder setUnackTime(int unackTime) {
- this.unackTime = unackTime;
- return this;
- }
-
- /**
- * @param currentShard the currentShard to set
- * @return instance of builder
- */
- public Builder setCurrentShard(String currentShard) {
- this.currentShard = currentShard;
- return this;
- }
-
- /**
- * @param hostToShardMap the hostToShardMap to set
- * @return instance of builder
- */
- public Builder setHostToShardMap(Function hostToShardMap) {
- this.hostToShardMap = hostToShardMap;
- return this;
- }
-
- /**
- * @param redisPoolSize the redisPoolSize to set
- * @return instance of builder
- */
- public Builder setRedisPoolSize(int redisPoolSize) {
- this.redisPoolSize = redisPoolSize;
- return this;
- }
-
- /**
- * @param quorumPort the quorumPort to set
- * @return instance of builder
- */
- public Builder setQuorumPort(int quorumPort) {
- this.quorumPort = quorumPort;
- return this;
- }
-
- /**
- * @param nonQuorumPort the nonQuorumPort to set
- * @return instance of builder
- */
- public Builder setNonQuorumPort(int nonQuorumPort) {
- this.nonQuorumPort = nonQuorumPort;
- return this;
- }
-
- public Builder setHosts(List hosts) {
- this.hosts = hosts;
- return this;
- }
-
- public MultiRedisQueue build() {
- if(hosts == null) {
- hosts = getHostsFromEureka(ec, dynomiteClusterName);
- }
- Map shardMap = new HashMap<>();
- for(Host host : hosts) {
- String shard = hostToShardMap.apply(host);
- shardMap.put(shard, host);
- }
-
- JedisPoolConfig config = new JedisPoolConfig();
- config.setTestOnBorrow(true);
- config.setTestOnCreate(true);
- config.setMaxTotal(redisPoolSize);
- config.setMaxIdle(5);
- config.setMaxWaitMillis(60_000);
-
-
- Map queues = new HashMap<>();
- for(String queueShard : shardMap.keySet()) {
- String host = shardMap.get(queueShard).getIpAddress();
-
- JedisPool pool = new JedisPool(config, host, quorumPort, 0);
- JedisPool readPool = new JedisPool(config, host, nonQuorumPort, 0);
-
- RedisQueue q = new RedisQueue(redisKeyPrefix, queueName, queueShard, unackTime, pool);
- q.setNonQuorumPool(readPool);
- queues.put(queueShard, q);
- }
- MultiRedisQueue queue = new MultiRedisQueue(queueName, currentShard, queues);
- return queue;
- }
-
- private static List getHostsFromEureka(EurekaClient ec, String applicationName) {
-
- Application app = ec.getApplication(applicationName);
- List hosts = new ArrayList();
-
- if (app == null) {
- return hosts;
- }
-
- List ins = app.getInstances();
-
- if (ins == null || ins.isEmpty()) {
- return hosts;
- }
-
- hosts = Lists.newArrayList(Collections2.transform(ins,
-
- new Function() {
- @Override
- public Host apply(InstanceInfo info) {
-
- Host.Status status = info.getStatus() == InstanceStatus.UP ? Host.Status.Up : Host.Status.Down;
- String rack = null;
- if (info.getDataCenterInfo() instanceof AmazonInfo) {
- AmazonInfo amazonInfo = (AmazonInfo)info.getDataCenterInfo();
- rack = amazonInfo.get(MetaDataKey.availabilityZone);
- }
- Host host = new Host(info.getHostName(), info.getIPAddr(), rack, status);
- return host;
- }
- }));
- return hosts;
- }
- }
-
-
-}
diff --git a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/QueueMonitor.java b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/QueueMonitor.java
index 9654d98..085d9e1 100644
--- a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/QueueMonitor.java
+++ b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/QueueMonitor.java
@@ -1,12 +1,12 @@
/**
* Copyright 2016 Netflix, Inc.
- *
+ *
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -33,106 +33,107 @@
/**
* @author Viren
- * Monitoring for the queue
+ * Monitoring for the queue, publishes the metrics using servo
+ * https://github.com/Netflix/servo
*/
public class QueueMonitor implements Closeable {
- BasicTimer peek;
-
- BasicTimer ack;
-
- BasicTimer size;
-
- BasicTimer processUnack;
-
- BasicTimer remove;
-
- BasicTimer get;
-
- StatsMonitor queueDepth;
-
- StatsMonitor batchSize;
-
- StatsMonitor pop;
-
- StatsMonitor push;
-
- BasicCounter misses;
-
- StatsMonitor prefetch;
-
- private String queueName;
-
- private String shardName;
-
- private ScheduledExecutorService executor;
-
- private static final String className = QueueMonitor.class.getSimpleName();
-
- QueueMonitor(String queueName, String shardName){
-
- String totalTagName = "total";
- executor = Executors.newScheduledThreadPool(1);
-
- this.queueName = queueName;
- this.shardName = shardName;
-
- peek = new BasicTimer(create("peek"), TimeUnit.MILLISECONDS);
- ack = new BasicTimer(create("ack"), TimeUnit.MILLISECONDS);
- size = new BasicTimer(create("size"), TimeUnit.MILLISECONDS);
- processUnack = new BasicTimer(create("processUnack"), TimeUnit.MILLISECONDS);
- remove = new BasicTimer(create("remove"), TimeUnit.MILLISECONDS);
- get = new BasicTimer(create("get"), TimeUnit.MILLISECONDS);
- misses = new BasicCounter(create("queue_miss"));
-
-
- StatsConfig statsConfig = new StatsConfig.Builder().withPublishCount(true).withPublishMax(true).withPublishMean(true).withPublishMin(true).withPublishTotal(true).build();
-
- queueDepth = new StatsMonitor(create("queueDepth"), statsConfig, executor, totalTagName, true);
- batchSize = new StatsMonitor(create("batchSize"), statsConfig, executor, totalTagName, true);
- pop = new StatsMonitor(create("pop"), statsConfig, executor, totalTagName, true);
- push = new StatsMonitor(create("push"), statsConfig, executor, totalTagName, true);
- prefetch = new StatsMonitor(create("prefetch"), statsConfig, executor, totalTagName, true);
-
- MonitorRegistry registry = DefaultMonitorRegistry.getInstance();
-
- registry.register(pop);
- registry.register(push);
- registry.register(peek);
- registry.register(ack);
- registry.register(size);
- registry.register(processUnack);
- registry.register(remove);
- registry.register(get);
- registry.register(queueDepth);
- registry.register(misses);
- registry.register(batchSize);
- registry.register(prefetch);
- }
-
- private MonitorConfig create(String name){
- return MonitorConfig.builder(name).withTag("class", className).withTag("shard", shardName).withTag("queueName", queueName).build();
- }
-
- Stopwatch start(StatsMonitor sm, int batchCount){
- int count = (batchCount == 0) ? 1 : batchCount;
- Stopwatch sw = new BasicStopwatch(){
-
- @Override
- public void stop() {
- super.stop();
- long duration = getDuration(TimeUnit.MILLISECONDS)/count;
- sm.record(duration);
- batchSize.record(count);
- }
-
- };
- sw.start();
- return sw;
- }
-
- @Override
- public void close() throws IOException {
- executor.shutdown();
- }
+ public BasicTimer peek;
+
+ public BasicTimer ack;
+
+ public BasicTimer size;
+
+ public BasicTimer processUnack;
+
+ public BasicTimer remove;
+
+ public BasicTimer get;
+
+ public StatsMonitor queueDepth;
+
+ public StatsMonitor batchSize;
+
+ public StatsMonitor pop;
+
+ public StatsMonitor push;
+
+ public BasicCounter misses;
+
+ public StatsMonitor prefetch;
+
+ private String queueName;
+
+ private String shardName;
+
+ private ScheduledExecutorService executor;
+
+ private static final String className = QueueMonitor.class.getSimpleName();
+
+ public QueueMonitor(String queueName, String shardName) {
+
+ String totalTagName = "total";
+ executor = Executors.newScheduledThreadPool(1);
+
+ this.queueName = queueName;
+ this.shardName = shardName;
+
+ peek = new BasicTimer(create("peek"), TimeUnit.MILLISECONDS);
+ ack = new BasicTimer(create("ack"), TimeUnit.MILLISECONDS);
+ size = new BasicTimer(create("size"), TimeUnit.MILLISECONDS);
+ processUnack = new BasicTimer(create("processUnack"), TimeUnit.MILLISECONDS);
+ remove = new BasicTimer(create("remove"), TimeUnit.MILLISECONDS);
+ get = new BasicTimer(create("get"), TimeUnit.MILLISECONDS);
+ misses = new BasicCounter(create("queue_miss"));
+
+
+ StatsConfig statsConfig = new StatsConfig.Builder().withPublishCount(true).withPublishMax(true).withPublishMean(true).withPublishMin(true).withPublishTotal(true).build();
+
+ queueDepth = new StatsMonitor(create("queueDepth"), statsConfig, executor, totalTagName, true);
+ batchSize = new StatsMonitor(create("batchSize"), statsConfig, executor, totalTagName, true);
+ pop = new StatsMonitor(create("pop"), statsConfig, executor, totalTagName, true);
+ push = new StatsMonitor(create("push"), statsConfig, executor, totalTagName, true);
+ prefetch = new StatsMonitor(create("prefetch"), statsConfig, executor, totalTagName, true);
+
+ MonitorRegistry registry = DefaultMonitorRegistry.getInstance();
+
+ registry.register(pop);
+ registry.register(push);
+ registry.register(peek);
+ registry.register(ack);
+ registry.register(size);
+ registry.register(processUnack);
+ registry.register(remove);
+ registry.register(get);
+ registry.register(queueDepth);
+ registry.register(misses);
+ registry.register(batchSize);
+ registry.register(prefetch);
+ }
+
+ private MonitorConfig create(String name) {
+ return MonitorConfig.builder(name).withTag("class", className).withTag("shard", shardName).withTag("queueName", queueName).build();
+ }
+
+ public Stopwatch start(StatsMonitor sm, int batchCount) {
+ int count = (batchCount == 0) ? 1 : batchCount;
+ Stopwatch sw = new BasicStopwatch() {
+
+ @Override
+ public void stop() {
+ super.stop();
+ long duration = getDuration(TimeUnit.MILLISECONDS) / count;
+ sm.record(duration);
+ batchSize.record(count);
+ }
+
+ };
+ sw.start();
+ return sw;
+ }
+
+ @Override
+ public void close() throws IOException {
+ executor.shutdown();
+ }
}
diff --git a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/QueueUtils.java b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/QueueUtils.java
new file mode 100644
index 0000000..c83402e
--- /dev/null
+++ b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/QueueUtils.java
@@ -0,0 +1,67 @@
+package com.netflix.dyno.queues.redis;
+
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.databind.DeserializationFeature;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.SerializationFeature;
+import com.netflix.dyno.connectionpool.exception.DynoException;
+
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutionException;
+
+/**
+ * Helper class to consolidate functions which might be reused across different DynoQueue implementations.
+ */
+public class QueueUtils {
+
+ private static final int retryCount = 2;
+
+ /**
+ * Execute function with retries if required
+ *
+ * @param opName
+ * @param keyName
+ * @param r
+ * @param
+ * @return
+ */
+ public static R execute(String opName, String keyName, Callable r) {
+ return executeWithRetry(opName, keyName, r, 0);
+ }
+
+ private static R executeWithRetry(String opName, String keyName, Callable r, int retryNum) {
+
+ try {
+
+ return r.call();
+
+ } catch (ExecutionException e) {
+
+ if (e.getCause() instanceof DynoException) {
+ if (retryNum < retryCount) {
+ return executeWithRetry(opName, keyName, r, ++retryNum);
+ }
+ }
+ throw new RuntimeException(e.getCause());
+ } catch (Exception e) {
+ throw new RuntimeException(
+ "Operation: ( " + opName + " ) failed on key: [" + keyName + " ].", e);
+ }
+ }
+
+ /**
+ * Construct standard objectmapper to use within the DynoQueue instances to read/write Message objects
+ *
+ * @return
+ */
+ public static ObjectMapper constructObjectMapper() {
+ ObjectMapper om = new ObjectMapper();
+ om.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
+ om.configure(DeserializationFeature.FAIL_ON_IGNORED_PROPERTIES, false);
+ om.configure(DeserializationFeature.FAIL_ON_NULL_FOR_PRIMITIVES, false);
+ om.setSerializationInclusion(JsonInclude.Include.NON_NULL);
+ om.setSerializationInclusion(JsonInclude.Include.NON_EMPTY);
+ om.disable(SerializationFeature.INDENT_OUTPUT);
+ return om;
+ }
+}
diff --git a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/RedisDynoQueue.java b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/RedisDynoQueue.java
index 4797b65..40e42c3 100644
--- a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/RedisDynoQueue.java
+++ b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/RedisDynoQueue.java
@@ -1,12 +1,12 @@
/**
* Copyright 2016 Netflix, Inc.
- *
+ *
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -15,599 +15,1619 @@
*/
package com.netflix.dyno.queues.redis;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.ImmutableList;
+import com.google.common.util.concurrent.Uninterruptibles;
+import com.netflix.dyno.jedis.DynoJedisClient;
+import com.netflix.dyno.queues.DynoQueue;
+import com.netflix.dyno.queues.Message;
+import com.netflix.dyno.queues.redis.sharding.ShardingStrategy;
+import com.netflix.servo.monitor.Stopwatch;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import redis.clients.jedis.Tuple;
+import redis.clients.jedis.commands.JedisCommands;
+import redis.clients.jedis.params.ZAddParams;
+
import java.io.IOException;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.concurrent.Callable;
+import java.text.NumberFormat;
+import java.time.Clock;
+import java.util.*;
import java.util.concurrent.ConcurrentLinkedQueue;
-import java.util.concurrent.ExecutionException;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.stream.Collectors;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.fasterxml.jackson.annotation.JsonInclude.Include;
-import com.fasterxml.jackson.databind.DeserializationFeature;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.SerializationFeature;
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.util.concurrent.Uninterruptibles;
-import com.netflix.dyno.connectionpool.exception.DynoException;
-import com.netflix.dyno.queues.DynoQueue;
-import com.netflix.dyno.queues.Message;
-import com.netflix.servo.monitor.Stopwatch;
-
-import redis.clients.jedis.JedisCommands;
-import redis.clients.jedis.Tuple;
-import redis.clients.jedis.params.sortedset.ZAddParams;
+import static com.netflix.dyno.queues.redis.QueueUtils.execute;
/**
*
* @author Viren
- *
+ * Current Production (March 2018) recipe - well tested in production.
+ * Note, this recipe does not use redis pipelines and hence the throughput offered is less compared to v2 recipes.
*/
public class RedisDynoQueue implements DynoQueue {
- private final Logger logger = LoggerFactory.getLogger(RedisDynoQueue.class);
-
- private String queueName;
-
- private List allShards;
+ private final Logger logger = LoggerFactory.getLogger(RedisDynoQueue.class);
- private String shardName;
+ private final Clock clock;
- private String redisKeyPrefix;
+ private final String queueName;
- private String messageStoreKey;
+ private final List allShards;
- private String myQueueShard;
+ private final String shardName;
- private int unackTime = 60;
+ private final String redisKeyPrefix;
- private QueueMonitor monitor;
+ private final String messageStoreKey;
- private ObjectMapper om;
+ private final String localQueueShard;
- private JedisCommands quorumConn;
+ private volatile int unackTime = 60;
- private JedisCommands nonQuorumConn;
-
- private ConcurrentLinkedQueue prefetchedIds;
+ private final QueueMonitor monitor;
- private ScheduledExecutorService schedulerForUnacksProcessing;
+ private final ObjectMapper om;
- private ScheduledExecutorService schedulerForPrefetchProcessing;
+ private volatile JedisCommands quorumConn;
- private int retryCount = 2;
-
- public RedisDynoQueue(String redisKeyPrefix, String queueName, Set allShards, String shardName) {
- this(redisKeyPrefix, queueName, allShards, shardName, 60_000);
- }
- public RedisDynoQueue(String redisKeyPrefix, String queueName, Set allShards, String shardName, int unackScheduleInMS) {
- this.redisKeyPrefix = redisKeyPrefix;
- this.queueName = queueName;
- this.allShards = allShards.stream().collect(Collectors.toList());
- this.shardName = shardName;
- this.messageStoreKey = redisKeyPrefix + ".MESSAGE." + queueName;
- this.myQueueShard = getQueueShardKey(queueName, shardName);
+ private volatile JedisCommands nonQuorumConn;
- ObjectMapper om = new ObjectMapper();
- om.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
- om.configure(DeserializationFeature.FAIL_ON_IGNORED_PROPERTIES, false);
- om.configure(DeserializationFeature.FAIL_ON_NULL_FOR_PRIMITIVES, false);
- om.setSerializationInclusion(Include.NON_NULL);
- om.setSerializationInclusion(Include.NON_EMPTY);
- om.disable(SerializationFeature.INDENT_OUTPUT);
+ private final ConcurrentLinkedQueue prefetchedIds;
- this.om = om;
- this.monitor = new QueueMonitor(queueName, shardName);
- this.prefetchedIds = new ConcurrentLinkedQueue<>();
+ private final Map> unsafePrefetchedIdsAllShardsMap;
- schedulerForUnacksProcessing = Executors.newScheduledThreadPool(1);
- schedulerForPrefetchProcessing = Executors.newScheduledThreadPool(1);
+ private final ScheduledExecutorService schedulerForUnacksProcessing;
- schedulerForUnacksProcessing.scheduleAtFixedRate(() -> processUnacks(), unackScheduleInMS, unackScheduleInMS, TimeUnit.MILLISECONDS);
+ private final int retryCount = 2;
- logger.info(RedisDynoQueue.class.getName() + " is ready to serve " + queueName);
+ private final ShardingStrategy shardingStrategy;
- }
+ private final boolean singleRingTopology;
- public RedisDynoQueue withQuorumConn(JedisCommands quorumConn){
- this.quorumConn = quorumConn;
- return this;
- }
+ // Tracks the number of message IDs to prefetch based on the message counts requested by the caller via pop().
+ @VisibleForTesting
+ AtomicInteger numIdsToPrefetch;
- public RedisDynoQueue withNonQuorumConn(JedisCommands nonQuorumConn){
- this.nonQuorumConn = nonQuorumConn;
- return this;
- }
+ // Tracks the number of message IDs to prefetch based on the message counts requested by the caller via
+ // unsafePopAllShards().
+ @VisibleForTesting
+ AtomicInteger unsafeNumIdsToPrefetchAllShards;
- public RedisDynoQueue withUnackTime(int unackTime){
- this.unackTime = unackTime;
- return this;
- }
+ public RedisDynoQueue(String redisKeyPrefix, String queueName, Set allShards, String shardName, ShardingStrategy shardingStrategy, boolean singleRingTopology) {
+ this(redisKeyPrefix, queueName, allShards, shardName, 60_000, shardingStrategy, singleRingTopology);
+ }
- @Override
- public String getName() {
- return queueName;
- }
+ public RedisDynoQueue(String redisKeyPrefix, String queueName, Set allShards, String shardName, int unackScheduleInMS, ShardingStrategy shardingStrategy, boolean singleRingTopology) {
+ this(Clock.systemDefaultZone(), redisKeyPrefix, queueName, allShards, shardName, unackScheduleInMS, shardingStrategy, singleRingTopology);
+ }
- @Override
- public int getUnackTime() {
- return unackTime;
- }
+ public RedisDynoQueue(Clock clock, String redisKeyPrefix, String queueName, Set allShards, String shardName, int unackScheduleInMS, ShardingStrategy shardingStrategy, boolean singleRingTopology) {
+ this.clock = clock;
+ this.redisKeyPrefix = redisKeyPrefix;
+ this.queueName = queueName;
+ this.allShards = ImmutableList.copyOf(allShards.stream().collect(Collectors.toList()));
+ this.shardName = shardName;
+ this.messageStoreKey = redisKeyPrefix + ".MESSAGE." + queueName;
+ this.localQueueShard = getQueueShardKey(queueName, shardName);
+ this.shardingStrategy = shardingStrategy;
- @Override
- public List push(final List messages) {
+ this.numIdsToPrefetch = new AtomicInteger(0);
+ this.unsafeNumIdsToPrefetchAllShards = new AtomicInteger(0);
+ this.singleRingTopology = singleRingTopology;
- Stopwatch sw = monitor.start(monitor.push, messages.size());
+ this.om = QueueUtils.constructObjectMapper();
+ this.monitor = new QueueMonitor(queueName, shardName);
+ this.prefetchedIds = new ConcurrentLinkedQueue<>();
+ this.unsafePrefetchedIdsAllShardsMap = new HashMap<>();
+ for (String shard : allShards) {
+ unsafePrefetchedIdsAllShardsMap.put(getQueueShardKey(queueName, shard), new ConcurrentLinkedQueue<>());
+ }
- try {
+ schedulerForUnacksProcessing = Executors.newScheduledThreadPool(1);
- execute(() -> {
- for (Message message : messages) {
- String json = om.writeValueAsString(message);
- quorumConn.hset(messageStoreKey, message.getId(), json);
- double priority = message.getPriority() / 100;
- double score = Long.valueOf(System.currentTimeMillis() + message.getTimeout()).doubleValue() + priority;
- String shard = getNextShard();
- String queueShard = getQueueShardKey(queueName, shard);
- quorumConn.zadd(queueShard, score, message.getId());
- }
- return messages;
- });
-
- return messages.stream().map(msg -> msg.getId()).collect(Collectors.toList());
-
- } finally {
- sw.stop();
- }
- }
-
- @Override
- public List peek(final int messageCount) {
-
- Stopwatch sw = monitor.peek.start();
-
- try {
-
- Set ids = peekIds(0, messageCount);
- if (ids == null) {
- return Collections.emptyList();
- }
-
- List msgs = execute(() -> {
- List messages = new LinkedList();
- for (String id : ids) {
- String json = nonQuorumConn.hget(messageStoreKey, id);
- Message message = om.readValue(json, Message.class);
- messages.add(message);
- }
- return messages;
- });
-
- return msgs;
-
- } finally {
- sw.stop();
- }
- }
-
- @Override
- public List pop(int messageCount, int wait, TimeUnit unit) {
-
- if (messageCount < 1) {
- return Collections.emptyList();
- }
-
- Stopwatch sw = monitor.start(monitor.pop, messageCount);
- try {
- long start = System.currentTimeMillis();
- long waitFor = unit.toMillis(wait);
- prefetch.addAndGet(messageCount);
- prefetchIds();
- while(prefetchedIds.size() < messageCount && ((System.currentTimeMillis() - start) < waitFor)) {
- Uninterruptibles.sleepUninterruptibly(200, TimeUnit.MILLISECONDS);
- prefetchIds();
- }
- return _pop(messageCount);
-
- } catch(Exception e) {
- throw new RuntimeException(e);
- } finally {
- sw.stop();
- }
-
- }
-
- @VisibleForTesting
- AtomicInteger prefetch = new AtomicInteger(0);
-
- private void prefetchIds() {
-
- if (prefetch.get() < 1) {
- return;
- }
-
- int prefetchCount = prefetch.get();
- Stopwatch sw = monitor.start(monitor.prefetch, prefetchCount);
- try {
-
- Set ids = peekIds(0, prefetchCount);
- prefetchedIds.addAll(ids);
- prefetch.addAndGet((-1 * ids.size()));
- if(prefetch.get() < 0 || ids.isEmpty()) {
- prefetch.set(0);
- }
- } finally {
- sw.stop();
- }
-
- }
-
- private List _pop(int messageCount) throws Exception {
-
- double unackScore = Long.valueOf(System.currentTimeMillis() + unackTime).doubleValue();
- String unackQueueName = getUnackKey(queueName, shardName);
-
- List popped = new LinkedList<>();
- ZAddParams zParams = ZAddParams.zAddParams().nx();
-
- for (;popped.size() != messageCount;) {
- String msgId = prefetchedIds.poll();
- if(msgId == null) {
- break;
- }
-
- long added = quorumConn.zadd(unackQueueName, unackScore, msgId, zParams);
- if(added == 0){
- if (logger.isDebugEnabled()) {
- logger.debug("cannot add {} to the unack shard ", queueName, msgId);
- }
- monitor.misses.increment();
- continue;
- }
-
- long removed = quorumConn.zrem(myQueueShard, msgId);
- if (removed == 0) {
- if (logger.isDebugEnabled()) {
- logger.debug("cannot remove {} from the queue shard ", queueName, msgId);
- }
- monitor.misses.increment();
- continue;
- }
-
- String json = quorumConn.hget(messageStoreKey, msgId);
- if(json == null){
- if (logger.isDebugEnabled()) {
- logger.debug("Cannot get the message payload for {}", msgId);
- }
- monitor.misses.increment();
- continue;
- }
- Message msg = om.readValue(json, Message.class);
- popped.add(msg);
-
- if (popped.size() == messageCount) {
- return popped;
- }
- }
- return popped;
- }
-
- @Override
- public boolean ack(String messageId) {
-
- Stopwatch sw = monitor.ack.start();
-
- try {
-
- return execute(() -> {
-
- for (String shard : allShards) {
- String unackShardKey = getUnackKey(queueName, shard);
- Long removed = quorumConn.zrem(unackShardKey, messageId);
- if (removed > 0) {
- quorumConn.hdel(messageStoreKey, messageId);
- return true;
- }
- }
- return false;
- });
-
- } finally {
- sw.stop();
- }
- }
-
- @Override
- public void ack(List messages) {
- for(Message message : messages) {
- ack(message.getId());
- }
- }
-
- @Override
- public boolean setUnackTimeout(String messageId, long timeout) {
-
- Stopwatch sw = monitor.ack.start();
-
- try {
-
- return execute(() -> {
- double unackScore = Long.valueOf(System.currentTimeMillis() + timeout).doubleValue();
- for (String shard : allShards) {
-
- String unackShardKey = getUnackKey(queueName, shard);
- Double score = quorumConn.zscore(unackShardKey, messageId);
- if(score != null) {
- quorumConn.zadd(unackShardKey, unackScore, messageId);
- return true;
- }
- }
- return false;
- });
-
- } finally {
- sw.stop();
- }
- }
-
- @Override
- public boolean setTimeout(String messageId, long timeout) {
-
- return execute(() -> {
-
- String json = nonQuorumConn.hget(messageStoreKey, messageId);
- if(json == null) {
- return false;
- }
- Message message = om.readValue(json, Message.class);
- message.setTimeout(timeout);
-
- for (String shard : allShards) {
-
- String queueShard = getQueueShardKey(queueName, shard);
- Double score = quorumConn.zscore(queueShard, messageId);
- if(score != null) {
- double priorityd = message.getPriority() / 100;
- double newScore = Long.valueOf(System.currentTimeMillis() + timeout).doubleValue() + priorityd;
- ZAddParams params = ZAddParams.zAddParams().xx();
- quorumConn.zadd(queueShard, newScore, messageId, params);
- json = om.writeValueAsString(message);
- quorumConn.hset(messageStoreKey, message.getId(), json);
- return true;
- }
- }
- return false;
- });
- }
-
- @Override
- public boolean remove(String messageId) {
-
- Stopwatch sw = monitor.remove.start();
-
- try {
-
- return execute(() -> {
-
- for (String shard : allShards) {
-
- String unackShardKey = getUnackKey(queueName, shard);
- quorumConn.zrem(unackShardKey, messageId);
-
- String queueShardKey = getQueueShardKey(queueName, shard);
- Long removed = quorumConn.zrem(queueShardKey, messageId);
- Long msgRemoved = quorumConn.hdel(messageStoreKey, messageId);
-
- if (removed > 0 && msgRemoved > 0) {
- return true;
- }
- }
-
- return false;
-
- });
-
- } finally {
- sw.stop();
- }
- }
-
- @Override
- public Message get(String messageId) {
-
- Stopwatch sw = monitor.get.start();
-
- try {
-
- return execute(() -> {
- String json = quorumConn.hget(messageStoreKey, messageId);
- if(json == null){
- if (logger.isDebugEnabled()) {
- logger.debug("Cannot get the message payload " + messageId);
- }
- return null;
- }
-
- Message msg = om.readValue(json, Message.class);
- return msg;
- });
-
- } finally {
- sw.stop();
- }
- }
-
- @Override
- public long size() {
-
- Stopwatch sw = monitor.size.start();
-
- try {
+ if (this.singleRingTopology) {
+ schedulerForUnacksProcessing.scheduleAtFixedRate(() -> atomicProcessUnacks(), unackScheduleInMS, unackScheduleInMS, TimeUnit.MILLISECONDS);
+ } else {
+ schedulerForUnacksProcessing.scheduleAtFixedRate(() -> processUnacks(), unackScheduleInMS, unackScheduleInMS, TimeUnit.MILLISECONDS);
+ }
- return execute(() -> {
- long size = 0;
- for (String shard : allShards) {
- size += nonQuorumConn.zcard(getQueueShardKey(queueName, shard));
- }
- return size;
- });
+ logger.info(RedisDynoQueue.class.getName() + " is ready to serve " + queueName);
+ }
- } finally {
- sw.stop();
- }
- }
-
- @Override
- public Map> shardSizes() {
-
- Stopwatch sw = monitor.size.start();
- Map> shardSizes = new HashMap<>();
- try {
-
- return execute(() -> {
- for (String shard : allShards) {
- long size = nonQuorumConn.zcard(getQueueShardKey(queueName, shard));
- long uacked = nonQuorumConn.zcard(getUnackKey(queueName, shard));
- Map shardDetails = new HashMap<>();
- shardDetails.put("size", size);
- shardDetails.put("uacked", uacked);
- shardSizes.put(shard, shardDetails);
- }
- return shardSizes;
- });
+ public RedisDynoQueue withQuorumConn(JedisCommands quorumConn) {
+ this.quorumConn = quorumConn;
+ return this;
+ }
- } finally {
- sw.stop();
- }
- }
+ public RedisDynoQueue withNonQuorumConn(JedisCommands nonQuorumConn) {
+ this.nonQuorumConn = nonQuorumConn;
+ return this;
+ }
- @Override
- public void clear() {
- execute(() -> {
- for (String shard : allShards) {
- String queueShard = getQueueShardKey(queueName, shard);
- String unackShard = getUnackKey(queueName, shard);
- quorumConn.del(queueShard);
- quorumConn.del(unackShard);
- }
- quorumConn.del(messageStoreKey);
- return null;
- });
-
- }
-
- private Set peekIds(int offset, int count) {
-
- return execute(() -> {
- double now = Long.valueOf(System.currentTimeMillis() + 1).doubleValue();
- Set scanned = quorumConn.zrangeByScore(myQueueShard, 0, now, offset, count);
- return scanned;
- });
-
- }
-
- public void processUnacks() {
-
- Stopwatch sw = monitor.processUnack.start();
- try {
-
- long queueDepth = size();
- monitor.queueDepth.record(queueDepth);
-
- execute(() -> {
-
- int batchSize = 1_000;
- String unackQueueName = getUnackKey(queueName, shardName);
-
- double now = Long.valueOf(System.currentTimeMillis()).doubleValue();
-
- Set unacks = quorumConn.zrangeByScoreWithScores(unackQueueName, 0, now, 0, batchSize);
-
- if (unacks.size() > 0) {
- logger.debug("Adding " + unacks.size() + " messages back to the queue for " + queueName);
- }
-
- for (Tuple unack : unacks) {
-
- double score = unack.getScore();
- String member = unack.getElement();
-
- String payload = quorumConn.hget(messageStoreKey, member);
- if (payload == null) {
- quorumConn.zrem(unackQueueName, member);
- continue;
- }
-
- quorumConn.zadd(myQueueShard, score, member);
- quorumConn.zrem(unackQueueName, member);
- }
- return null;
- });
-
- } finally {
- sw.stop();
- }
-
- }
-
- private AtomicInteger nextShardIndex = new AtomicInteger(0);
-
- private String getNextShard() {
- int indx = nextShardIndex.incrementAndGet();
- if (indx >= allShards.size()) {
- nextShardIndex.set(0);
- indx = 0;
- }
- String s = allShards.get(indx);
- return s;
- }
-
- private String getQueueShardKey(String queueName, String shard) {
- return redisKeyPrefix + ".QUEUE." + queueName + "." + shard;
- }
-
- private String getUnackKey(String queueName, String shard) {
- return redisKeyPrefix + ".UNACK." + queueName + "." + shard;
- }
-
- private R execute(Callable r) {
- return executeWithRetry(r, 0);
- }
-
- private R executeWithRetry(Callable r, int retryCount) {
-
- try {
-
- return r.call();
-
- } catch (ExecutionException e) {
-
- if (e.getCause() instanceof DynoException) {
- if (retryCount < this.retryCount) {
- return executeWithRetry(r, ++retryCount);
- }
- }
- throw new RuntimeException(e.getCause());
- } catch (Exception e) {
- throw new RuntimeException(e);
- }
- }
+ public RedisDynoQueue withUnackTime(int unackTime) {
+ this.unackTime = unackTime;
+ return this;
+ }
- @Override
- public void close() throws IOException {
- schedulerForUnacksProcessing.shutdown();
- schedulerForPrefetchProcessing.shutdown();
- monitor.close();
- }
+ /**
+ * @return Number of items in each ConcurrentLinkedQueue from 'unsafePrefetchedIdsAllShardsMap'.
+ */
+ private int unsafeGetNumPrefetchedIds() {
+ // Note: We use an AtomicInteger due to Java's limitation of not allowing the modification of local native
+ // data types in lambdas (Java 8).
+ AtomicInteger totalSize = new AtomicInteger(0);
+ unsafePrefetchedIdsAllShardsMap.forEach((k,v)->totalSize.addAndGet(v.size()));
+ return totalSize.get();
+ }
+
+ @Override
+ public String getName() {
+ return queueName;
+ }
+
+ @Override
+ public int getUnackTime() {
+ return unackTime;
+ }
+
+ @Override
+ public List push(final List messages) {
+
+ Stopwatch sw = monitor.start(monitor.push, messages.size());
+
+ try {
+ execute("push", "(a shard in) " + queueName, () -> {
+ for (Message message : messages) {
+ String json = om.writeValueAsString(message);
+ quorumConn.hset(messageStoreKey, message.getId(), json);
+ double priority = message.getPriority() / 100.0;
+ double score = Long.valueOf(clock.millis() + message.getTimeout()).doubleValue() + priority;
+ String shard = shardingStrategy.getNextShard(allShards, message);
+ String queueShard = getQueueShardKey(queueName, shard);
+ quorumConn.zadd(queueShard, score, message.getId());
+ }
+ return messages;
+ });
+
+ return messages.stream().map(msg -> msg.getId()).collect(Collectors.toList());
+
+ } finally {
+ sw.stop();
+ }
+ }
+
+ @Override
+ public List peek(final int messageCount) {
+
+ Stopwatch sw = monitor.peek.start();
+
+ try {
+
+ Set ids = peekIds(0, messageCount);
+ if (ids == null) {
+ return Collections.emptyList();
+ }
+ return doPeekBodyHelper(ids);
+
+ } finally {
+ sw.stop();
+ }
+ }
+
+ @Override
+ public List unsafePeekAllShards(final int messageCount) {
+
+ Stopwatch sw = monitor.peek.start();
+
+ try {
+
+ Set ids = peekIdsAllShards(0, messageCount);
+ if (ids == null) {
+ return Collections.emptyList();
+ }
+ return doPeekBodyHelper(ids);
+ } finally {
+ sw.stop();
+ }
+ }
+
+ /**
+ *
+ * Peeks into 'this.localQueueShard' and returns up to 'count' items starting at position 'offset' in the shard.
+ *
+ *
+ * @param offset Number of items to skip over in 'this.localQueueShard'
+ * @param count Number of items to return.
+ * @return Up to 'count' number of message IDs in a set.
+ */
+ private Set peekIds(final int offset, final int count, final double peekTillTs) {
+
+ return execute("peekIds", localQueueShard, () -> {
+ double peekTillTsOrNow = (peekTillTs == 0.0) ? Long.valueOf(clock.millis() + 1).doubleValue() : peekTillTs;
+ return doPeekIdsFromShardHelper(localQueueShard, peekTillTsOrNow, offset, count);
+ });
+
+ }
+
+ private Set peekIds(final int offset, final int count) {
+ return peekIds(offset, count, 0.0);
+ }
+
+ /**
+ *
+ * Same as 'peekIds()' but looks into all shards of the queue ('this.allShards').
+ *
+ * @param count Number of items to return.
+ * @return Up to 'count' number of message IDs in a set.
+ */
+ private Set peekIdsAllShards(final int offset, final int count) {
+ return execute("peekIdsAllShards", localQueueShard, () -> {
+ Set scanned = new HashSet<>();
+ double now = Long.valueOf(clock.millis() + 1).doubleValue();
+ int remaining_count = count;
+
+ // Try to get as many items from 'this.localQueueShard' first to reduce chances of returning duplicate items.
+ // (See unsafe* functions disclaimer in DynoQueue.java)
+ scanned.addAll(peekIds(offset, count, now));
+ remaining_count -= scanned.size();
+
+ for (String shard : allShards) {
+ String queueShardName = getQueueShardKey(queueName, shard);
+ // Skip 'localQueueShard'.
+ if (queueShardName.equals(localQueueShard)) continue;
+
+ Set elems = doPeekIdsFromShardHelper(queueShardName, now, offset, count);
+ scanned.addAll(elems);
+ remaining_count -= elems.size();
+ if (remaining_count <= 0) break;
+ }
+
+ return scanned;
+
+ });
+ }
+
+ private Set doPeekIdsFromShardHelper(final String queueShardName, final double peekTillTs, final int offset,
+ final int count) {
+ return nonQuorumConn.zrangeByScore(queueShardName, 0, peekTillTs, offset, count);
+ }
+
+ /**
+ * Takes a set of message IDs, 'message_ids', and returns a list of Message objects
+ * corresponding to 'message_ids'. Read only, does not make any updates.
+ *
+ * @param message_ids Set of message IDs to peek.
+ * @return a list of Message objects corresponding to 'message_ids'
+ *
+ */
+ private List doPeekBodyHelper(Set message_ids) {
+ List msgs = execute("peek", messageStoreKey, () -> {
+ List messages = new LinkedList();
+ for (String id : message_ids) {
+ String json = nonQuorumConn.hget(messageStoreKey, id);
+ Message message = om.readValue(json, Message.class);
+ messages.add(message);
+ }
+ return messages;
+ });
+
+ return msgs;
+ }
+
+ @Override
+ public List pop(int messageCount, int wait, TimeUnit unit) {
+
+ if (messageCount < 1) {
+ return Collections.emptyList();
+ }
+
+ Stopwatch sw = monitor.start(monitor.pop, messageCount);
+ try {
+ long start = clock.millis();
+ long waitFor = unit.toMillis(wait);
+ numIdsToPrefetch.addAndGet(messageCount);
+
+ // We prefetch message IDs here first before attempting to pop them off the sorted set.
+ // The reason we do this (as opposed to just popping from the head of the sorted set),
+ // is that due to the eventually consistent nature of Dynomite, the different replicas of the same
+ // sorted set _may_ not look exactly the same at any given time, i.e. they may have a different number of
+ // items due to replication lag.
+ // So, we first peek into the sorted set to find the list of message IDs that we know for sure are
+ // replicated across all replicas and then attempt to pop them based on those message IDs.
+ prefetchIds();
+ while (prefetchedIds.size() < messageCount && ((clock.millis() - start) < waitFor)) {
+ Uninterruptibles.sleepUninterruptibly(200, TimeUnit.MILLISECONDS);
+ prefetchIds();
+ }
+ return _pop(shardName, messageCount, prefetchedIds);
+
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ } finally {
+ sw.stop();
+ }
+
+ }
+
+ @Override
+ public Message popWithMsgId(String messageId) {
+ return popWithMsgIdHelper(messageId, shardName, true);
+ }
+
+ @Override
+ public Message unsafePopWithMsgIdAllShards(String messageId) {
+ int numShards = allShards.size();
+ for (String shard : allShards) {
+ boolean warnIfNotExists = false;
+
+ // Only one of the shards will have the message, so we don't want the check in the other 2 shards
+ // to spam the logs. So make sure only the last shard emits a warning log which means that none of the
+ // shards have 'messageId'.
+ if (--numShards == 0) warnIfNotExists = true;
+
+ Message msg = popWithMsgIdHelper(messageId, shard, warnIfNotExists);
+ if (msg != null) return msg;
+ }
+ return null;
+ }
+
+ public Message popWithMsgIdHelper(String messageId, String targetShard, boolean warnIfNotExists) {
+
+ Stopwatch sw = monitor.start(monitor.pop, 1);
+
+ try {
+ return execute("popWithMsgId", targetShard, () -> {
+
+ String queueShardName = getQueueShardKey(queueName, targetShard);
+ double unackScore = Long.valueOf(clock.millis() + unackTime).doubleValue();
+ String unackShardName = getUnackKey(queueName, targetShard);
+
+ ZAddParams zParams = ZAddParams.zAddParams().nx();
+
+ Long exists = nonQuorumConn.zrank(queueShardName, messageId);
+ // If we get back a null type, then the element doesn't exist.
+ if (exists == null) {
+ // We only have a 'warnIfNotExists' check for this call since not all messages are present in
+ // all shards. So we want to avoid a log spam. If any of the following calls return 'null' or '0',
+ // we may have hit an inconsistency (because it's in the queue, but other calls have failed),
+ // so make sure to log those.
+ if (warnIfNotExists) {
+ logger.warn("Cannot find the message with ID {}", messageId);
+ }
+ monitor.misses.increment();
+ return null;
+ }
+
+ String json = quorumConn.hget(messageStoreKey, messageId);
+ if (json == null) {
+ logger.warn("Cannot get the message payload for {}", messageId);
+ monitor.misses.increment();
+ return null;
+ }
+
+ long added = quorumConn.zadd(unackShardName, unackScore, messageId, zParams);
+ if (added == 0) {
+ logger.warn("cannot add {} to the unack shard {}", messageId, unackShardName);
+ monitor.misses.increment();
+ return null;
+ }
+
+ long removed = quorumConn.zrem(queueShardName, messageId);
+ if (removed == 0) {
+ logger.warn("cannot remove {} from the queue shard ", queueName, messageId);
+ monitor.misses.increment();
+ return null;
+ }
+
+ Message msg = om.readValue(json, Message.class);
+ return msg;
+ });
+ } finally {
+ sw.stop();
+ }
+
+ }
+
+ public List unsafePopAllShards(int messageCount, int wait, TimeUnit unit) {
+ if (messageCount < 1) {
+ return Collections.emptyList();
+ }
+
+ Stopwatch sw = monitor.start(monitor.pop, messageCount);
+ try {
+ long start = clock.millis();
+ long waitFor = unit.toMillis(wait);
+ unsafeNumIdsToPrefetchAllShards.addAndGet(messageCount);
+
+ prefetchIdsAllShards();
+ while(unsafeGetNumPrefetchedIds() < messageCount && ((clock.millis() - start) < waitFor)) {
+ Uninterruptibles.sleepUninterruptibly(200, TimeUnit.MILLISECONDS);
+ prefetchIdsAllShards();
+ }
+
+ int remainingCount = messageCount;
+ // Pop as much as possible from the local shard first to reduce chances of returning duplicate items.
+ // (See unsafe* functions disclaimer in DynoQueue.java)
+ List popped = _pop(shardName, remainingCount, unsafePrefetchedIdsAllShardsMap.get(localQueueShard));
+ remainingCount -= popped.size();
+
+ for (String shard : allShards) {
+ String queueShardName = getQueueShardKey(queueName, shard);
+ List elems = _pop(shard, remainingCount, unsafePrefetchedIdsAllShardsMap.get(queueShardName));
+ popped.addAll(elems);
+ remainingCount -= elems.size();
+ }
+ return popped;
+ } catch(Exception e) {
+ throw new RuntimeException(e);
+ } finally {
+ sw.stop();
+ }
+ }
+
+ /**
+ * Prefetch message IDs from the local shard.
+ */
+ private void prefetchIds() {
+ double now = Long.valueOf(clock.millis() + 1).doubleValue();
+ int numPrefetched = doPrefetchIdsHelper(localQueueShard, numIdsToPrefetch, prefetchedIds, now);
+ if (numPrefetched == 0) {
+ numIdsToPrefetch.set(0);
+ }
+ }
+
+
+ /**
+ * Prefetch message IDs from all shards.
+ */
+ private void prefetchIdsAllShards() {
+ double now = Long.valueOf(clock.millis() + 1).doubleValue();
+
+ // Try to prefetch as many items from 'this.localQueueShard' first to reduce chances of returning duplicate items.
+ // (See unsafe* functions disclaimer in DynoQueue.java)
+ doPrefetchIdsHelper(localQueueShard, unsafeNumIdsToPrefetchAllShards,
+ unsafePrefetchedIdsAllShardsMap.get(localQueueShard), now);
+
+ if (unsafeNumIdsToPrefetchAllShards.get() < 1) return;
+
+ for (String shard : allShards) {
+ String queueShardName = getQueueShardKey(queueName, shard);
+ if (queueShardName.equals(localQueueShard)) continue; // Skip since we've already serviced the local shard.
+
+ doPrefetchIdsHelper(queueShardName, unsafeNumIdsToPrefetchAllShards,
+ unsafePrefetchedIdsAllShardsMap.get(queueShardName), now);
+ }
+ }
+
+ /**
+ * Attempts to prefetch up to 'prefetchCounter' message IDs, by peeking into a queue based on 'peekFunction',
+ * and store it in a concurrent linked queue.
+ *
+ * @param prefetchCounter Number of message IDs to attempt prefetch.
+ * @param prefetchedIdQueue Concurrent Linked Queue where message IDs are stored.
+ * @param peekFunction Function to call to peek into the queue.
+ */
+ private int doPrefetchIdsHelper(String queueShardName, AtomicInteger prefetchCounter,
+ ConcurrentLinkedQueue prefetchedIdQueue, double prefetchFromTs) {
+
+ if (prefetchCounter.get() < 1) {
+ return 0;
+ }
+
+ int numSuccessfullyPrefetched = 0;
+ int numToPrefetch = prefetchCounter.get();
+ Stopwatch sw = monitor.start(monitor.prefetch, numToPrefetch);
+ try {
+ // Attempt to peek up to 'numToPrefetch' message Ids.
+ Set ids = doPeekIdsFromShardHelper(queueShardName, prefetchFromTs, 0, numToPrefetch);
+
+ // TODO: Check for duplicates.
+ // Store prefetched IDs in a queue.
+ prefetchedIdQueue.addAll(ids);
+
+ numSuccessfullyPrefetched = ids.size();
+
+ // Account for number of IDs successfully prefetched.
+ prefetchCounter.addAndGet((-1 * ids.size()));
+ if(prefetchCounter.get() < 0) {
+ prefetchCounter.set(0);
+ }
+ } finally {
+ sw.stop();
+ }
+ return numSuccessfullyPrefetched;
+ }
+
+ private List _pop(String shard, int messageCount,
+ ConcurrentLinkedQueue prefetchedIdQueue) throws Exception {
+
+ String queueShardName = getQueueShardKey(queueName, shard);
+ String unackShardName = getUnackKey(queueName, shard);
+ double unackScore = Long.valueOf(clock.millis() + unackTime).doubleValue();
+
+ // NX option indicates add only if it doesn't exist.
+ // https://redis.io/commands/zadd#zadd-options-redis-302-or-greater
+ ZAddParams zParams = ZAddParams.zAddParams().nx();
+
+ List popped = new LinkedList<>();
+ for (;popped.size() != messageCount;) {
+ String msgId = prefetchedIdQueue.poll();
+ if(msgId == null) {
+ break;
+ }
+
+ long added = quorumConn.zadd(unackShardName, unackScore, msgId, zParams);
+ if(added == 0){
+ logger.warn("cannot add {} to the unack shard {}", msgId, unackShardName);
+ monitor.misses.increment();
+ continue;
+ }
+
+ long removed = quorumConn.zrem(queueShardName, msgId);
+ if (removed == 0) {
+ logger.warn("cannot remove {} from the queue shard {}", msgId, queueShardName);
+ monitor.misses.increment();
+ continue;
+ }
+
+ String json = quorumConn.hget(messageStoreKey, msgId);
+ if (json == null) {
+ logger.warn("Cannot get the message payload for {}", msgId);
+ monitor.misses.increment();
+ continue;
+ }
+ Message msg = om.readValue(json, Message.class);
+ popped.add(msg);
+
+ if (popped.size() == messageCount) {
+ return popped;
+ }
+ }
+ return popped;
+ }
+
+ @Override
+ public boolean ack(String messageId) {
+
+ Stopwatch sw = monitor.ack.start();
+
+ try {
+ return execute("ack", "(a shard in) " + queueName, () -> {
+
+ for (String shard : allShards) {
+ String unackShardKey = getUnackKey(queueName, shard);
+ Long removed = quorumConn.zrem(unackShardKey, messageId);
+ if (removed > 0) {
+ quorumConn.hdel(messageStoreKey, messageId);
+ return true;
+ }
+ }
+ return false;
+ });
+
+ } finally {
+ sw.stop();
+ }
+ }
+
+ @Override
+ public void ack(List messages) {
+ for (Message message : messages) {
+ ack(message.getId());
+ }
+ }
+
+ @Override
+ public boolean setUnackTimeout(String messageId, long timeout) {
+
+ Stopwatch sw = monitor.ack.start();
+
+ try {
+ return execute("setUnackTimeout", "(a shard in) " + queueName, () -> {
+ double unackScore = Long.valueOf(clock.millis() + timeout).doubleValue();
+ for (String shard : allShards) {
+
+ String unackShardKey = getUnackKey(queueName, shard);
+ Double score = quorumConn.zscore(unackShardKey, messageId);
+ if (score != null) {
+ quorumConn.zadd(unackShardKey, unackScore, messageId);
+ return true;
+ }
+ }
+ return false;
+ });
+
+ } finally {
+ sw.stop();
+ }
+ }
+
+ @Override
+ public boolean setTimeout(String messageId, long timeout) {
+
+ return execute("setTimeout", "(a shard in) " + queueName, () -> {
+
+ String json = nonQuorumConn.hget(messageStoreKey, messageId);
+ if (json == null) {
+ return false;
+ }
+ Message message = om.readValue(json, Message.class);
+ message.setTimeout(timeout);
+
+ for (String shard : allShards) {
+
+ String queueShard = getQueueShardKey(queueName, shard);
+ Double score = quorumConn.zscore(queueShard, messageId);
+ if (score != null) {
+ double priorityd = message.getPriority() / 100;
+ double newScore = Long.valueOf(clock.millis() + timeout).doubleValue() + priorityd;
+ ZAddParams params = ZAddParams.zAddParams().xx();
+ quorumConn.zadd(queueShard, newScore, messageId, params);
+ json = om.writeValueAsString(message);
+ quorumConn.hset(messageStoreKey, message.getId(), json);
+ return true;
+ }
+ }
+ return false;
+ });
+ }
+
+ @Override
+ public boolean remove(String messageId) {
+
+ Stopwatch sw = monitor.remove.start();
+
+ try {
+
+ return execute("remove", "(a shard in) " + queueName, () -> {
+
+ for (String shard : allShards) {
+
+ String unackShardKey = getUnackKey(queueName, shard);
+ Long removedFromUnack = quorumConn.zrem(unackShardKey, messageId);
+
+ String queueShardKey = getQueueShardKey(queueName, shard);
+ Long removedFromQueue = quorumConn.zrem(queueShardKey, messageId);
+
+ if ((removedFromUnack != null && removedFromUnack > 0) || (removedFromQueue != null && removedFromQueue > 0)) {
+ // Ignoring return value since we just want to get rid of it.
+ quorumConn.hdel(messageStoreKey, messageId);
+ return true;
+ }
+ }
+
+ return false;
+
+ });
+
+ } finally {
+ sw.stop();
+ }
+ }
+
+ @Override
+ public boolean atomicRemove(String messageId) {
+
+ Stopwatch sw = monitor.remove.start();
+
+ try {
+
+ return execute("remove", "(a shard in) " + queueName, () -> {
+
+
+ String atomicRemoveScript = "local hkey=KEYS[1]\n" +
+ "local msg_id=ARGV[1]\n" +
+ "local num_shards=ARGV[2]\n" +
+ "\n" +
+ "local removed_shard=0\n" +
+ "local removed_unack=0\n" +
+ "local removed_hash=0\n" +
+ "for i=0,num_shards-1 do\n" +
+ " local shard_name = ARGV[3+(i*2)]\n" +
+ " local unack_name = ARGV[3+(i*2)+1]\n" +
+ "\n" +
+ " removed_shard = removed_shard + redis.call('zrem', shard_name, msg_id)\n" +
+ " removed_unack = removed_unack + redis.call('zrem', unack_name, msg_id)\n" +
+ "end\n" +
+ "\n" +
+ "removed_hash = redis.call('hdel', hkey, msg_id)\n" +
+ "if (removed_shard==1 or removed_unack==1 or removed_hash==1) then\n" +
+ " return 1\n" +
+ "end\n" +
+ "return removed_unack\n";
+
+ ImmutableList.Builder builder = ImmutableList.builder();
+ builder.add(messageId);
+ builder.add(Integer.toString(allShards.size()));
+
+ for (String shard : allShards) {
+
+ String queueShardKey = getQueueShardKey(queueName, shard);
+ String unackShardKey = getUnackKey(queueName, shard);
+
+ builder.add(queueShardKey);
+ builder.add(unackShardKey);
+ }
+
+ Long removed = (Long) ((DynoJedisClient)quorumConn).eval(atomicRemoveScript, Collections.singletonList(messageStoreKey), builder.build());
+ if (removed == 1) return true;
+
+ return false;
+
+ });
+
+ } finally {
+ sw.stop();
+ }
+ }
+
+ @Override
+ public boolean ensure(Message message) {
+ return execute("ensure", "(a shard in) " + queueName, () -> {
+
+ String messageId = message.getId();
+ for (String shard : allShards) {
+
+ String queueShard = getQueueShardKey(queueName, shard);
+ Double score = quorumConn.zscore(queueShard, messageId);
+ if (score != null) {
+ return false;
+ }
+ String unackShardKey = getUnackKey(queueName, shard);
+ score = quorumConn.zscore(unackShardKey, messageId);
+ if (score != null) {
+ return false;
+ }
+ }
+ push(Collections.singletonList(message));
+ return true;
+ });
+ }
+
+
+ @Override
+ public boolean containsPredicate(String predicate) {
+ return containsPredicate(predicate, false);
+ }
+
+ @Override
+ public String getMsgWithPredicate(String predicate) {
+ return getMsgWithPredicate(predicate, false);
+ }
+
+ @Override
+ public boolean containsPredicate(String predicate, boolean localShardOnly) {
+ return execute("containsPredicate", messageStoreKey, () -> getMsgWithPredicate(predicate, localShardOnly) != null);
+ }
+
+ @Override
+ public String getMsgWithPredicate(String predicate, boolean localShardOnly) {
+ return execute("getMsgWithPredicate", messageStoreKey, () -> {
+
+ // We use a Lua script here to do predicate matching since we only want to find whether the predicate
+ // exists in any of the message bodies or not, and the only way to do that is to check for the predicate
+ // match on the server side.
+ // The alternative is to have the server return all the hash values back to us and we filter it here on
+ // the client side. This is not desirable since we would potentially be sending large amounts of data
+ // over the network only to return a single string value back to the calling application.
+ String predicateCheckAllLuaScript = "local hkey=KEYS[1]\n" +
+ "local predicate=ARGV[1]\n" +
+ "local cursor=0\n" +
+ "local begin=false\n" +
+ "while (cursor ~= 0 or begin==false) do\n" +
+ " local ret = redis.call('hscan', hkey, cursor)\n" +
+ " local curmsgid\n" +
+ " for i, content in ipairs(ret[2]) do\n" +
+ " if (i % 2 ~= 0) then\n" +
+ " curmsgid = content\n" +
+ " elseif (string.match(content, predicate)) then\n" +
+ " return curmsgid\n" +
+ " end\n" +
+ " end\n" +
+ " cursor=tonumber(ret[1])\n" +
+ " begin=true\n" +
+ "end\n" +
+ "return nil";
+
+ String predicateCheckLocalOnlyLuaScript = "local hkey=KEYS[1]\n" +
+ "local predicate=ARGV[1]\n" +
+ "local shard_name=ARGV[2]\n" +
+ "local cursor=0\n" +
+ "local begin=false\n" +
+ "while (cursor ~= 0 or begin==false) do\n" +
+ " local ret = redis.call('hscan', hkey, cursor)\n" +
+ "local curmsgid\n" +
+ "for i, content in ipairs(ret[2]) do\n" +
+ " if (i % 2 ~= 0) then\n" +
+ " curmsgid = content\n" +
+ "elseif (string.match(content, predicate)) then\n" +
+ "local in_local_shard = redis.call('zrank', shard_name, curmsgid)\n" +
+ "if (type(in_local_shard) ~= 'boolean' and in_local_shard >= 0) then\n" +
+ "return curmsgid\n" +
+ "end\n" +
+ " end\n" +
+ "end\n" +
+ " cursor=tonumber(ret[1])\n" +
+ "begin=true\n" +
+ "end\n" +
+ "return nil";
+
+ String retval;
+ if (localShardOnly) {
+ // Cast from 'JedisCommands' to 'DynoJedisClient' here since the former does not expose 'eval()'.
+ retval = (String) ((DynoJedisClient) nonQuorumConn).eval(predicateCheckLocalOnlyLuaScript,
+ Collections.singletonList(messageStoreKey), ImmutableList.of(predicate, localQueueShard));
+ } else {
+ // Cast from 'JedisCommands' to 'DynoJedisClient' here since the former does not expose 'eval()'.
+ retval = (String) ((DynoJedisClient) nonQuorumConn).eval(predicateCheckAllLuaScript,
+ Collections.singletonList(messageStoreKey), Collections.singletonList(predicate));
+ }
+
+ return retval;
+ });
+ }
+
+ private Message popMsgWithPredicateObeyPriority(String predicate, boolean localShardOnly) {
+ String popPredicateObeyPriority = "local hkey=KEYS[1]\n" +
+ "local predicate=ARGV[1]\n" +
+ "local num_shards=ARGV[2]\n" +
+ "local peek_until=tonumber(ARGV[3])\n" +
+ "local unack_score=tonumber(ARGV[4])\n" +
+ "\n" +
+ "local shard_names={}\n" +
+ "local unack_names={}\n" +
+ "local shard_lengths={}\n" +
+ "local largest_shard=-1\n" +
+ "for i=0,num_shards-1 do\n" +
+ " shard_names[i+1]=ARGV[5+(i*2)]\n" +
+ " shard_lengths[i+1] = redis.call('zcard', shard_names[i+1])\n" +
+ " unack_names[i+1]=ARGV[5+(i*2)+1]\n" +
+ "\n" +
+ " if (shard_lengths[i+1] > largest_shard) then\n" +
+ " largest_shard = shard_lengths[i+1]\n" +
+ " end\n" +
+ "end\n" +
+ "\n" +
+ "local min_score=-1\n" +
+ "local min_member\n" +
+ "local matching_value\n" +
+ "local owning_shard_idx=-1\n" +
+ "\n" +
+ "local num_complete_shards=0\n" +
+ "for j=0,largest_shard-1 do\n" +
+ " for i=1,num_shards do\n" +
+ " local skiploop=false\n" +
+ " if (shard_lengths[i] < j+1) then\n" +
+ " skiploop=true\n" +
+ " end\n" +
+ "\n" +
+ " if (skiploop == false) then\n" +
+ " local element = redis.call('zrange', shard_names[i], j, j, 'WITHSCORES')\n" +
+ " if ((min_score ~= -1 and min_score < tonumber(element[2])) or peek_until < tonumber(element[2])) then\n" +
+ " -- This is to make sure we don't process this shard again\n" +
+ " -- since all elements henceforth are of lower priority than min_member\n" +
+ " shard_lengths[i]=0\n" +
+ " num_complete_shards = num_complete_shards + 1\n" +
+ " else\n" +
+ " local value = redis.call('hget', hkey, tostring(element[1]))\n" +
+ " if (value) then\n" +
+ " if (string.match(value, predicate)) then\n" +
+ " if (min_score == -1 or tonumber(element[2]) < min_score) then\n" +
+ " min_score = tonumber(element[2])\n" +
+ " owning_shard_idx=i\n" +
+ " min_member = element[1]\n" +
+ " matching_value = value\n" +
+ " end\n" +
+ " end\n" +
+ " end\n" +
+ " end\n" +
+ " end\n" +
+ " end\n" +
+ " if (num_complete_shards == num_shards) then\n" +
+ " break\n" +
+ " end\n" +
+ "end\n" +
+ "\n" +
+ "if (min_member) then\n" +
+ " local queue_shard_name=shard_names[owning_shard_idx]\n" +
+ " local unack_shard_name=unack_names[owning_shard_idx]\n" +
+ " local zadd_ret = redis.call('zadd', unack_shard_name, 'NX', unack_score, min_member)\n" +
+ " if (zadd_ret) then\n" +
+ " redis.call('zrem', queue_shard_name, min_member)\n" +
+ " end\n" +
+ "end\n" +
+ "return {min_member, matching_value}";
+
+ double now = Long.valueOf(clock.millis() + 1).doubleValue();
+ double unackScore = Long.valueOf(clock.millis() + unackTime).doubleValue();
+
+ // The script requires the scores as whole numbers
+ NumberFormat fmt = NumberFormat.getIntegerInstance();
+ fmt.setGroupingUsed(false);
+ String nowScoreString = fmt.format(now);
+ String unackScoreString = fmt.format(unackScore);
+
+ ArrayList retval;
+ if (localShardOnly) {
+ String unackShardName = getUnackKey(queueName, shardName);
+
+ ImmutableList.Builder builder = ImmutableList.builder();
+ builder.add(predicate);
+ builder.add(Integer.toString(1));
+ builder.add(nowScoreString);
+ builder.add(unackScoreString);
+ builder.add(localQueueShard);
+ builder.add(unackShardName);
+
+ // Cast from 'JedisCommands' to 'DynoJedisClient' here since the former does not expose 'eval()'.
+ retval = (ArrayList) ((DynoJedisClient) quorumConn).eval(popPredicateObeyPriority,
+ Collections.singletonList(messageStoreKey), builder.build());
+ } else {
+
+ ImmutableList.Builder builder = ImmutableList.builder();
+ builder.add(predicate);
+ builder.add(Integer.toString(allShards.size()));
+ builder.add(nowScoreString);
+ builder.add(unackScoreString);
+ for (String shard : allShards) {
+ String queueShard = getQueueShardKey(queueName, shard);
+ String unackShardName = getUnackKey(queueName, shard);
+ builder.add(queueShard);
+ builder.add(unackShardName);
+ }
+
+ // Cast from 'JedisCommands' to 'DynoJedisClient' here since the former does not expose 'eval()'.
+ retval = (ArrayList) ((DynoJedisClient) quorumConn).eval(popPredicateObeyPriority,
+ Collections.singletonList(messageStoreKey), builder.build());
+ }
+
+ if (retval.size() == 0) return null;
+ return new Message(retval.get(0), retval.get(1));
+
+ }
+
+ @Override
+ public Message popMsgWithPredicate(String predicate, boolean localShardOnly) {
+ Stopwatch sw = monitor.start(monitor.pop, 1);
+
+ try {
+ Message payload = execute("popMsgWithPredicateObeyPriority", messageStoreKey, () -> popMsgWithPredicateObeyPriority(predicate, localShardOnly));
+ return payload;
+
+ } finally {
+ sw.stop();
+ }
+
+ }
+
+ @Override
+ public List bulkPop(int messageCount, int wait, TimeUnit unit) {
+
+ if (messageCount < 1) {
+ return Collections.emptyList();
+ }
+
+ Stopwatch sw = monitor.start(monitor.pop, messageCount);
+ try {
+ long start = clock.millis();
+ long waitFor = unit.toMillis(wait);
+ numIdsToPrefetch.addAndGet(messageCount);
+
+ prefetchIds();
+ while (prefetchedIds.size() < messageCount && ((clock.millis() - start) < waitFor)) {
+ Uninterruptibles.sleepUninterruptibly(200, TimeUnit.MILLISECONDS);
+ prefetchIds();
+ }
+ int numToPop = (prefetchedIds.size() > messageCount) ? messageCount : prefetchedIds.size();
+ return atomicBulkPopHelper(numToPop, prefetchedIds, true);
+
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ } finally {
+ sw.stop();
+ }
+
+ }
+
+ @Override
+ public List unsafeBulkPop(int messageCount, int wait, TimeUnit unit) {
+ if (messageCount < 1) {
+ return Collections.emptyList();
+ }
+
+ Stopwatch sw = monitor.start(monitor.pop, messageCount);
+ try {
+ long start = clock.millis();
+ long waitFor = unit.toMillis(wait);
+ unsafeNumIdsToPrefetchAllShards.addAndGet(messageCount);
+
+ prefetchIdsAllShards();
+ while(unsafeGetNumPrefetchedIds() < messageCount && ((clock.millis() - start) < waitFor)) {
+ Uninterruptibles.sleepUninterruptibly(200, TimeUnit.MILLISECONDS);
+ prefetchIdsAllShards();
+ }
+
+ int numToPop = (unsafeGetNumPrefetchedIds() > messageCount) ? messageCount : unsafeGetNumPrefetchedIds();
+ ConcurrentLinkedQueue messageIds = new ConcurrentLinkedQueue<>();
+ int numPrefetched = 0;
+ for (String shard : allShards) {
+ String queueShardName = getQueueShardKey(queueName, shard);
+ int prefetchedIdsSize = unsafePrefetchedIdsAllShardsMap.get(queueShardName).size();
+ for (int i = 0; i < prefetchedIdsSize; ++i) {
+ messageIds.add(unsafePrefetchedIdsAllShardsMap.get(queueShardName).poll());
+ if (++numPrefetched == numToPop) break;
+ }
+ if (numPrefetched == numToPop) break;
+ }
+ return atomicBulkPopHelper(numToPop, messageIds, false);
+ } catch(Exception e) {
+ throw new RuntimeException(e);
+ } finally {
+ sw.stop();
+ }
+ }
+
+ // TODO: Do code cleanup/consolidation
+ private List atomicBulkPopHelper(int messageCount,
+ ConcurrentLinkedQueue prefetchedIdQueue, boolean localShardOnly) throws IOException {
+
+ double now = Long.valueOf(clock.millis() + 1).doubleValue();
+ double unackScore = Long.valueOf(clock.millis() + unackTime).doubleValue();
+
+ // The script requires the scores as whole numbers
+ NumberFormat fmt = NumberFormat.getIntegerInstance();
+ fmt.setGroupingUsed(false);
+ String nowScoreString = fmt.format(now);
+ String unackScoreString = fmt.format(unackScore);
+
+ List messageIds = new ArrayList<>();
+ for (int i = 0; i < messageCount; ++i) {
+ messageIds.add(prefetchedIdQueue.poll());
+ }
+
+ String atomicBulkPopScriptLocalOnly="local hkey=KEYS[1]\n" +
+ "local num_msgs=ARGV[1]\n" +
+ "local peek_until=ARGV[2]\n" +
+ "local unack_score=ARGV[3]\n" +
+ "local queue_shard_name=ARGV[4]\n" +
+ "local unack_shard_name=ARGV[5]\n" +
+ "local msg_start_idx = 6\n" +
+ "local idx = 1\n" +
+ "local return_vals={}\n" +
+ "for i=0,num_msgs-1 do\n" +
+ " local message_id=ARGV[msg_start_idx + i]\n" +
+ " local exists = redis.call('zscore', queue_shard_name, message_id)\n" +
+ " if (exists) then\n" +
+ " if (exists <=peek_until) then\n" +
+ " local value = redis.call('hget', hkey, message_id)\n" +
+ " if (value) then\n" +
+ " local zadd_ret = redis.call('zadd', unack_shard_name, 'NX', unack_score, message_id)\n" +
+ " if (zadd_ret) then\n" +
+ " redis.call('zrem', queue_shard_name, message_id)\n" +
+ " return_vals[idx]=value\n" +
+ " idx=idx+1\n" +
+ " end\n" +
+ " end\n" +
+ " end\n" +
+ " else\n" +
+ " return {}\n" +
+ " end\n" +
+ "end\n" +
+ "return return_vals";
+
+ String atomicBulkPopScript="local hkey=KEYS[1]\n" +
+ "local num_msgs=ARGV[1]\n" +
+ "local num_shards=ARGV[2]\n" +
+ "local peek_until=ARGV[3]\n" +
+ "local unack_score=ARGV[4]\n" +
+ "local shard_start_idx = 5\n" +
+ "local msg_start_idx = 5 + (num_shards * 2)\n" +
+ "local out_idx = 1\n" +
+ "local return_vals={}\n" +
+ "for i=0,num_msgs-1 do\n" +
+ " local found_msg=false\n" +
+ " local message_id=ARGV[msg_start_idx + i]\n" +
+ " for j=0,num_shards-1 do\n" +
+ " local queue_shard_name=ARGV[shard_start_idx + (j*2)]\n" +
+ " local unack_shard_name=ARGV[shard_start_idx + (j*2) + 1]\n" +
+ " local exists = redis.call('zscore', queue_shard_name, message_id)\n" +
+ " if (exists) then\n" +
+ " found_msg=true\n" +
+ " if (exists <=peek_until) then\n" +
+ " local value = redis.call('hget', hkey, message_id)\n" +
+ " if (value) then\n" +
+ " local zadd_ret = redis.call('zadd', unack_shard_name, 'NX', unack_score, message_id)\n" +
+ " if (zadd_ret) then\n" +
+ " redis.call('zrem', queue_shard_name, message_id)\n" +
+ " return_vals[out_idx]=value\n" +
+ " out_idx=out_idx+1\n" +
+ " break\n" +
+ " end\n" +
+ " end\n" +
+ " end\n" +
+ " end\n" +
+ " end\n" +
+ " if (found_msg == false) then\n" +
+ " return {}\n" +
+ " end\n" +
+ "end\n" +
+ "return return_vals";
+
+ List payloads = new ArrayList<>();
+ if (localShardOnly) {
+ String unackShardName = getUnackKey(queueName, shardName);
+
+ ImmutableList.Builder builder = ImmutableList.builder();
+ builder.add(Integer.toString(messageCount));
+ builder.add(nowScoreString);
+ builder.add(unackScoreString);
+ builder.add(localQueueShard);
+ builder.add(unackShardName);
+ for (int i = 0; i < messageCount; ++i) {
+ builder.add(messageIds.get(i));
+ }
+
+ List jsonPayloads;
+ // Cast from 'JedisCommands' to 'DynoJedisClient' here since the former does not expose 'eval()'.
+ jsonPayloads = (List) ((DynoJedisClient) quorumConn).eval(atomicBulkPopScriptLocalOnly,
+ Collections.singletonList(messageStoreKey), builder.build());
+
+ for (String p : jsonPayloads) {
+ Message msg = om.readValue(p, Message.class);
+ payloads.add(msg);
+ }
+ } else {
+ ImmutableList.Builder builder = ImmutableList.builder();
+ builder.add(Integer.toString(messageCount));
+ builder.add(Integer.toString(allShards.size()));
+ builder.add(nowScoreString);
+ builder.add(unackScoreString);
+ for (String shard : allShards) {
+ String queueShard = getQueueShardKey(queueName, shard);
+ String unackShardName = getUnackKey(queueName, shard);
+ builder.add(queueShard);
+ builder.add(unackShardName);
+ }
+ for (int i = 0; i < messageCount; ++i) {
+ builder.add(messageIds.get(i));
+ }
+
+ List jsonPayloads;
+ // Cast from 'JedisCommands' to 'DynoJedisClient' here since the former does not expose 'eval()'.
+ jsonPayloads = (List) ((DynoJedisClient) quorumConn).eval(atomicBulkPopScript,
+ Collections.singletonList(messageStoreKey), builder.build());
+
+ for (String p : jsonPayloads) {
+ Message msg = om.readValue(p, Message.class);
+ payloads.add(msg);
+ }
+ }
+
+ return payloads;
+ }
+
+ /**
+ *
+ * Similar to popWithMsgId() but completes all the operations in one round trip.
+ *
+ * NOTE: This function assumes that the ring size in the cluster is 1. DO NOT use for APIs that support a ring
+ * size larger than 1.
+ *
+ * @param messageId
+ * @param localShardOnly
+ * @return
+ */
+ private String atomicPopWithMsgIdHelper(String messageId, boolean localShardOnly) {
+
+ double now = Long.valueOf(clock.millis() + 1).doubleValue();
+ double unackScore = Long.valueOf(clock.millis() + unackTime).doubleValue();
+
+ // The script requires the scores as whole numbers
+ NumberFormat fmt = NumberFormat.getIntegerInstance();
+ fmt.setGroupingUsed(false);
+ String nowScoreString = fmt.format(now);
+ String unackScoreString = fmt.format(unackScore);
+
+ String atomicPopScript = "local hkey=KEYS[1]\n" +
+ "local message_id=ARGV[1]\n" +
+ "local num_shards=ARGV[2]\n" +
+ "local peek_until=ARGV[3]\n" +
+ "local unack_score=ARGV[4]\n" +
+ "for i=0,num_shards-1 do\n" +
+ " local queue_shard_name=ARGV[(i*2)+5]\n" +
+ " local unack_shard_name=ARGV[(i*2)+5+1]\n" +
+ " local exists = redis.call('zscore', queue_shard_name, message_id)\n" +
+ " if (exists) then\n" +
+ " if (exists <= peek_until) then\n" +
+ " local value = redis.call('hget', hkey, message_id)\n" +
+ " if (value) then\n" +
+ " local zadd_ret = redis.call('zadd', unack_shard_name, 'NX', unack_score, message_id )\n" +
+ " if (zadd_ret) then\n" +
+ " redis.call('zrem', queue_shard_name, message_id)\n" +
+ " return value\n" +
+ " end\n" +
+ " end\n" +
+ " end\n" +
+ " end\n" +
+ "end\n" +
+ "return nil";
+
+ String retval;
+ if (localShardOnly) {
+ String unackShardName = getUnackKey(queueName, shardName);
+
+ retval = (String) ((DynoJedisClient) quorumConn).eval(atomicPopScript, Collections.singletonList(messageStoreKey),
+ ImmutableList.of(messageId, Integer.toString(1), nowScoreString,
+ unackScoreString, localQueueShard, unackShardName));
+ } else {
+ int numShards = allShards.size();
+ ImmutableList.Builder builder = ImmutableList.builder();
+ builder.add(messageId);
+ builder.add(Integer.toString(numShards));
+ builder.add(nowScoreString);
+ builder.add(unackScoreString);
+
+ List arguments = Arrays.asList(messageId, Integer.toString(numShards), nowScoreString,
+ unackScoreString);
+ for (String shard : allShards) {
+ String queueShard = getQueueShardKey(queueName, shard);
+ String unackShardName = getUnackKey(queueName, shard);
+ builder.add(queueShard);
+ builder.add(unackShardName);
+ }
+ retval = (String) ((DynoJedisClient) quorumConn).eval(atomicPopScript, Collections.singletonList(messageStoreKey), builder.build());
+ }
+
+ return retval;
+ }
+
+
+ @Override
+ public Message get(String messageId) {
+
+ Stopwatch sw = monitor.get.start();
+
+ try {
+
+ return execute("get", messageStoreKey, () -> {
+ String json = quorumConn.hget(messageStoreKey, messageId);
+ if (json == null) {
+ logger.warn("Cannot get the message payload " + messageId);
+ return null;
+ }
+
+ Message msg = om.readValue(json, Message.class);
+ return msg;
+ });
+
+ } finally {
+ sw.stop();
+ }
+ }
+
+ @Override
+ public List getAllMessages() {
+ Map allMsgs = nonQuorumConn.hgetAll(messageStoreKey);
+ List retList = new ArrayList<>();
+ for (Map.Entry entry: allMsgs.entrySet()) {
+ Message msg = new Message(entry.getKey(), entry.getValue());
+ retList.add(msg);
+ }
+
+ return retList;
+ }
+
+ @Override
+ public Message localGet(String messageId) {
+
+ Stopwatch sw = monitor.get.start();
+
+ try {
+
+ return execute("localGet", messageStoreKey, () -> {
+ String json = nonQuorumConn.hget(messageStoreKey, messageId);
+ if (json == null) {
+ logger.warn("Cannot get the message payload " + messageId);
+ return null;
+ }
+
+ Message msg = om.readValue(json, Message.class);
+ return msg;
+ });
+
+ } finally {
+ sw.stop();
+ }
+ }
+
+ @Override
+ public long size() {
+
+ Stopwatch sw = monitor.size.start();
+
+ try {
+
+ return execute("size", "(a shard in) " + queueName, () -> {
+ long size = 0;
+ for (String shard : allShards) {
+ size += nonQuorumConn.zcard(getQueueShardKey(queueName, shard));
+ }
+ return size;
+ });
+
+ } finally {
+ sw.stop();
+ }
+ }
+
+ @Override
+ public Map> shardSizes() {
+
+ Stopwatch sw = monitor.size.start();
+ Map> shardSizes = new HashMap<>();
+ try {
+
+ return execute("shardSizes", "(a shard in) " + queueName, () -> {
+ for (String shard : allShards) {
+ long size = nonQuorumConn.zcard(getQueueShardKey(queueName, shard));
+ long uacked = nonQuorumConn.zcard(getUnackKey(queueName, shard));
+ Map shardDetails = new HashMap<>();
+ shardDetails.put("size", size);
+ shardDetails.put("uacked", uacked);
+ shardSizes.put(shard, shardDetails);
+ }
+ return shardSizes;
+ });
+
+ } finally {
+ sw.stop();
+ }
+ }
+
+ @Override
+ public void clear() {
+ execute("clear", "(a shard in) " + queueName, () -> {
+ for (String shard : allShards) {
+ String queueShard = getQueueShardKey(queueName, shard);
+ String unackShard = getUnackKey(queueName, shard);
+ quorumConn.del(queueShard);
+ quorumConn.del(unackShard);
+ }
+ quorumConn.del(messageStoreKey);
+ return null;
+ });
+
+ }
+
+ @Override
+ public void processUnacks() {
+
+ logger.info("processUnacks() will NOT be atomic.");
+ Stopwatch sw = monitor.processUnack.start();
+ try {
+
+ long queueDepth = size();
+ monitor.queueDepth.record(queueDepth);
+
+ String keyName = getUnackKey(queueName, shardName);
+ execute("processUnacks", keyName, () -> {
+
+ int batchSize = 1_000;
+ String unackShardName = getUnackKey(queueName, shardName);
+
+ double now = Long.valueOf(clock.millis()).doubleValue();
+ int num_moved_back = 0;
+ int num_stale = 0;
+
+ Set unacks = nonQuorumConn.zrangeByScoreWithScores(unackShardName, 0, now, 0, batchSize);
+
+ if (unacks.size() > 0) {
+ logger.info("processUnacks: Attempting to add " + unacks.size() + " messages back to shard of queue: " + unackShardName);
+ }
+
+ for (Tuple unack : unacks) {
+
+ double score = unack.getScore();
+ String member = unack.getElement();
+
+ String payload = quorumConn.hget(messageStoreKey, member);
+ if (payload == null) {
+ quorumConn.zrem(unackShardName, member);
+ ++num_stale;
+ continue;
+ }
+
+ long added_back = quorumConn.zadd(localQueueShard, score, member);
+ long removed_from_unack = quorumConn.zrem(unackShardName, member);
+ if (added_back > 0 && removed_from_unack > 0) ++num_moved_back;
+ }
+
+ if (num_moved_back > 0 || num_stale > 0) {
+ logger.info("processUnacks: Moved back " + num_moved_back + " items. Got rid of " + num_stale + " stale items.");
+ }
+ return null;
+ });
+
+ } catch (Exception e) {
+ logger.error("Error while processing unacks. " + e.getMessage());
+ } finally {
+ sw.stop();
+ }
+
+ }
+
+ @Override
+ public List findStaleMessages() {
+ return execute("findStaleMessages", localQueueShard, () -> {
+
+ List stale_msgs = new ArrayList<>();
+
+ int batchSize = 10;
+
+ double now = Long.valueOf(clock.millis()).doubleValue();
+ long num_stale = 0;
+
+ for (String shard : allShards) {
+ String queueShardName = getQueueShardKey(queueName, shard);
+ Set elems = nonQuorumConn.zrangeByScore(queueShardName, 0, now, 0, batchSize);
+
+ if (elems.size() == 0) {
+ continue;
+ }
+
+ String findStaleMsgsScript = "local hkey=KEYS[1]\n" +
+ "local queue_shard=ARGV[1]\n" +
+ "local unack_shard=ARGV[2]\n" +
+ "local num_msgs=ARGV[3]\n" +
+ "\n" +
+ "local stale_msgs={}\n" +
+ "local num_stale_idx = 1\n" +
+ "for i=0,num_msgs-1 do\n" +
+ " local msg_id=ARGV[4+i]\n" +
+ "\n" +
+ " local exists_hash = redis.call('hget', hkey, msg_id)\n" +
+ " local exists_queue = redis.call('zscore', queue_shard, msg_id)\n" +
+ " local exists_unack = redis.call('zscore', unack_shard, msg_id)\n" +
+ "\n" +
+ " if (exists_hash and exists_queue) then\n" +
+ " elseif (not (exists_unack)) then\n" +
+ " stale_msgs[num_stale_idx] = msg_id\n" +
+ " num_stale_idx = num_stale_idx + 1\n" +
+ " end\n" +
+ "end\n" +
+ "\n" +
+ "return stale_msgs\n";
+
+ String unackKey = getUnackKey(queueName, shard);
+ ImmutableList.Builder builder = ImmutableList.builder();
+ builder.add(queueShardName);
+ builder.add(unackKey);
+ builder.add(Integer.toString(elems.size()));
+ for (String msg : elems) {
+ builder.add(msg);
+ }
+
+ ArrayList stale_msg_ids = (ArrayList) ((DynoJedisClient)quorumConn).eval(findStaleMsgsScript, Collections.singletonList(messageStoreKey), builder.build());
+ num_stale = stale_msg_ids.size();
+ if (num_stale > 0) {
+ logger.info("findStaleMsgs(): Found " + num_stale + " messages present in queue but not in hashmap");
+ }
+
+ for (String m : stale_msg_ids) {
+ Message msg = new Message();
+ msg.setId(m);
+ stale_msgs.add(msg);
+ }
+ }
+
+ return stale_msgs;
+ });
+ }
+
+ @Override
+ public void atomicProcessUnacks() {
+
+ logger.info("processUnacks() will be atomic.");
+ Stopwatch sw = monitor.processUnack.start();
+ try {
+
+ long queueDepth = size();
+ monitor.queueDepth.record(queueDepth);
+
+ String keyName = getUnackKey(queueName, shardName);
+ execute("processUnacks", keyName, () -> {
+
+ int batchSize = 1_000;
+ String unackShardName = getUnackKey(queueName, shardName);
+
+ double now = Long.valueOf(clock.millis()).doubleValue();
+ long num_moved_back = 0;
+ long num_stale = 0;
+
+ Set unacks = nonQuorumConn.zrangeByScoreWithScores(unackShardName, 0, now, 0, batchSize);
+
+ if (unacks.size() > 0) {
+ logger.info("processUnacks: Attempting to add " + unacks.size() + " messages back to shard of queue: " + unackShardName);
+ } else {
+ return null;
+ }
+
+ String atomicProcessUnacksScript = "local hkey=KEYS[1]\n" +
+ "local unack_shard=ARGV[1]\n" +
+ "local queue_shard=ARGV[2]\n" +
+ "local num_unacks=ARGV[3]\n" +
+ "\n" +
+ "local unacks={}\n" +
+ "local unack_scores={}\n" +
+ "local unack_start_idx = 4\n" +
+ "for i=0,num_unacks-1 do\n" +
+ " unacks[i]=ARGV[4 + (i*2)]\n" +
+ " unack_scores[i]=ARGV[4+(i*2)+1]\n" +
+ "end\n" +
+ "\n" +
+ "local num_moved=0\n" +
+ "local num_stale=0\n" +
+ "for i=0,num_unacks-1 do\n" +
+ " local mem_val = redis.call('hget', hkey, unacks[i])\n" +
+ " if (mem_val) then\n" +
+ " redis.call('zadd', queue_shard, unack_scores[i], unacks[i])\n" +
+ " redis.call('zrem', unack_shard, unacks[i])\n" +
+ " num_moved=num_moved+1\n" +
+ " else\n" +
+ " redis.call('zrem', unack_shard, unacks[i])\n" +
+ " num_stale=num_stale+1\n" +
+ " end\n" +
+ "end\n" +
+ "\n" +
+ "return {num_moved, num_stale}\n";
+
+ ImmutableList.Builder builder = ImmutableList.builder();
+ builder.add(unackShardName);
+ builder.add(localQueueShard);
+ builder.add(Integer.toString(unacks.size()));
+ for (Tuple unack : unacks) {
+ builder.add(unack.getElement());
+
+ // The script requires the scores as whole numbers
+ NumberFormat fmt = NumberFormat.getIntegerInstance();
+ fmt.setGroupingUsed(false);
+ String unackScoreString = fmt.format(unack.getScore());
+ builder.add(unackScoreString);
+ }
+
+ ArrayList retval = (ArrayList) ((DynoJedisClient)quorumConn).eval(atomicProcessUnacksScript, Collections.singletonList(messageStoreKey), builder.build());
+ num_moved_back = retval.get(0).longValue();
+ num_stale = retval.get(1).longValue();
+ if (num_moved_back > 0 || num_stale > 0) {
+ logger.info("processUnacks: Moved back " + num_moved_back + " items. Got rid of " + num_stale + " stale items.");
+ }
+ return null;
+ });
+
+ } catch (Exception e) {
+ logger.error("Error while processing unacks. " + e.getMessage());
+ } finally {
+ sw.stop();
+ }
+
+ }
+
+ private String getQueueShardKey(String queueName, String shard) {
+ return redisKeyPrefix + ".QUEUE." + queueName + "." + shard;
+ }
+
+ private String getUnackKey(String queueName, String shard) {
+ return redisKeyPrefix + ".UNACK." + queueName + "." + shard;
+ }
+
+ @Override
+ public void close() throws IOException {
+ schedulerForUnacksProcessing.shutdown();
+ monitor.close();
+ }
}
diff --git a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/RedisQueue.java b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/RedisQueue.java
deleted file mode 100644
index aeee37c..0000000
--- a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/RedisQueue.java
+++ /dev/null
@@ -1,612 +0,0 @@
-/**
- * Copyright 2016 Netflix, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.netflix.dyno.queues.redis;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.concurrent.Executors;
-import java.util.concurrent.ScheduledExecutorService;
-import java.util.concurrent.TimeUnit;
-import java.util.stream.Collectors;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.fasterxml.jackson.annotation.JsonInclude.Include;
-import com.fasterxml.jackson.databind.DeserializationFeature;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.SerializationFeature;
-import com.netflix.dyno.connectionpool.HashPartitioner;
-import com.netflix.dyno.connectionpool.impl.hash.Murmur3HashPartitioner;
-import com.netflix.dyno.queues.DynoQueue;
-import com.netflix.dyno.queues.Message;
-import com.netflix.servo.monitor.Stopwatch;
-
-import redis.clients.jedis.Jedis;
-import redis.clients.jedis.JedisPool;
-import redis.clients.jedis.Pipeline;
-import redis.clients.jedis.Response;
-import redis.clients.jedis.Tuple;
-import redis.clients.jedis.params.sortedset.ZAddParams;
-
-/**
- *
- * @author Viren
- *
- */
-public class RedisQueue implements DynoQueue {
-
- private final Logger logger = LoggerFactory.getLogger(RedisQueue.class);
-
- private String queueName;
-
- private String shardName;
-
- private String messageStoreKeyPrefix;
-
- private String myQueueShard;
-
- private String unackShardKeyPrefix;
-
- private int unackTime = 60;
-
- private QueueMonitor monitor;
-
- private ObjectMapper om;
-
- private JedisPool connPool;
-
- private JedisPool nonQuorumPool;
-
- private ScheduledExecutorService schedulerForUnacksProcessing;
-
- private ScheduledExecutorService schedulerForPrefetchProcessing;
-
- private HashPartitioner partitioner = new Murmur3HashPartitioner();
-
- private int maxHashBuckets = 1024;
-
- public RedisQueue(String redisKeyPrefix, String queueName, String shardName, int unackTime, JedisPool pool) {
- this(redisKeyPrefix, queueName, shardName, unackTime, unackTime, pool);
- }
-
- public RedisQueue(String redisKeyPrefix, String queueName, String shardName, int unackScheduleInMS, int unackTime, JedisPool pool) {
- this.queueName = queueName;
- this.shardName = shardName;
- this.messageStoreKeyPrefix = redisKeyPrefix + ".MESSAGE.";
- this.myQueueShard = redisKeyPrefix + ".QUEUE." + queueName + "." + shardName;
- this.unackShardKeyPrefix = redisKeyPrefix + ".UNACK." + queueName + "." + shardName + ".";
- this.unackTime = unackTime;
- this.connPool = pool;
- this.nonQuorumPool = pool;
-
- ObjectMapper om = new ObjectMapper();
- om.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
- om.configure(DeserializationFeature.FAIL_ON_IGNORED_PROPERTIES, false);
- om.configure(DeserializationFeature.FAIL_ON_NULL_FOR_PRIMITIVES, false);
- om.setSerializationInclusion(Include.NON_NULL);
- om.setSerializationInclusion(Include.NON_EMPTY);
- om.disable(SerializationFeature.INDENT_OUTPUT);
-
- this.om = om;
- this.monitor = new QueueMonitor(queueName, shardName);
-
- schedulerForUnacksProcessing = Executors.newScheduledThreadPool(1);
- schedulerForPrefetchProcessing = Executors.newScheduledThreadPool(1);
-
- schedulerForUnacksProcessing.scheduleAtFixedRate(() -> processUnacks(), unackScheduleInMS, unackScheduleInMS, TimeUnit.MILLISECONDS);
-
- logger.info(RedisQueue.class.getName() + " is ready to serve " + queueName);
-
- }
-
- /**
- *
- * @param nonQuorumPool When using a cluster like Dynomite, which relies on the quorum reads, supply a separate non-quorum read connection for ops like size etc.
- */
- public void setNonQuorumPool(JedisPool nonQuorumPool) {
- this.nonQuorumPool = nonQuorumPool;
- }
-
- @Override
- public String getName() {
- return queueName;
- }
-
- @Override
- public int getUnackTime() {
- return unackTime;
- }
-
- @Override
- public List push(final List messages) {
-
- Stopwatch sw = monitor.start(monitor.push, messages.size());
- Jedis conn = connPool.getResource();
- try {
-
- Pipeline pipe = conn.pipelined();
-
- for (Message message : messages) {
- String json = om.writeValueAsString(message);
- pipe.hset(messageStoreKey(message.getId()), message.getId(), json);
- double priority = message.getPriority() / 100.0;
- double score = Long.valueOf(System.currentTimeMillis() + message.getTimeout()).doubleValue() + priority;
- pipe.zadd(myQueueShard, score, message.getId());
- }
- pipe.sync();
- pipe.close();
-
- return messages.stream().map(msg -> msg.getId()).collect(Collectors.toList());
-
- } catch (Exception e) {
- throw new RuntimeException(e);
- } finally {
- conn.close();
- sw.stop();
- }
- }
-
- private String messageStoreKey(String msgId) {
- Long hash = partitioner.hash(msgId);
- long bucket = hash % maxHashBuckets;
- return messageStoreKeyPrefix + bucket + "." + queueName;
- }
-
- private String unackShardKey(String messageId) {
- Long hash = partitioner.hash(messageId);
- long bucket = hash % maxHashBuckets;
- return unackShardKeyPrefix + bucket;
- }
-
- @Override
- public List peek(final int messageCount) {
-
- Stopwatch sw = monitor.peek.start();
- Jedis jedis = connPool.getResource();
-
- try {
-
- Set ids = peekIds(0, messageCount);
- if (ids == null) {
- return Collections.emptyList();
- }
-
- List messages = new LinkedList();
- for (String id : ids) {
- String json = jedis.hget(messageStoreKey(id), id);
- Message message = om.readValue(json, Message.class);
- messages.add(message);
- }
- return messages;
-
- } catch (Exception e) {
- throw new RuntimeException(e);
- } finally {
- jedis.close();
- sw.stop();
- }
- }
-
-
- //
- //Note: This implementation does NOT support long polling. The method itself is synchronized, so implementing long poll could potentially block other threads.
- //When required, the long polling should be implemented on the caller side (ie the broker implementation using the recipe)
- //
- @Override
- public synchronized List pop(int messageCount, int wait, TimeUnit unit) {
-
- if (messageCount < 1) {
- return Collections.emptyList();
- }
-
- Stopwatch sw = monitor.start(monitor.pop, messageCount);
-
- try {
-
- List peeked = peekIds(0, messageCount).stream().collect(Collectors.toList());
- List popped = _pop(peeked);
- return popped;
-
- } catch (Exception e) {
- throw new RuntimeException(e);
- } finally {
- sw.stop();
- }
-
- }
-
- private List _pop(List batch) throws Exception {
-
- double unackScore = Long.valueOf(System.currentTimeMillis() + unackTime).doubleValue();
-
- List popped = new LinkedList<>();
- ZAddParams zParams = ZAddParams.zAddParams().nx();
-
- Jedis jedis = connPool.getResource();
- try {
-
- Pipeline pipe = jedis.pipelined();
- List> zadds = new ArrayList<>(batch.size());
- for (int i = 0; i < batch.size(); i++) {
- String msgId = batch.get(i);
- if(msgId == null) {
- break;
- }
- zadds.add(pipe.zadd(unackShardKey(msgId), unackScore, msgId, zParams));
- }
- pipe.sync();
-
- int count = zadds.size();
- List zremIds = new ArrayList<>(count);
- List> zremRes = new LinkedList<>();
- for (int i = 0; i < count; i++) {
- long added = zadds.get(i).get();
- if (added == 0) {
- if(logger.isDebugEnabled()) {
- logger.debug("Cannot add {} to unack queue shard", batch.get(i));
- }
- monitor.misses.increment();
- continue;
- }
- String id = batch.get(i);
- zremIds.add(id);
- zremRes.add(pipe.zrem(myQueueShard, id));
- }
- pipe.sync();
-
- List> getRes = new ArrayList<>(count);
- for (int i = 0; i < zremRes.size(); i++) {
- long removed = zremRes.get(i).get();
- if (removed == 0) {
- if(logger.isDebugEnabled()) {
- logger.debug("Cannot remove {} from queue shard", zremIds.get(i));
- }
- monitor.misses.increment();
- continue;
- }
- getRes.add(pipe.hget(messageStoreKey(zremIds.get(i)), zremIds.get(i)));
- }
- pipe.sync();
-
- for (int i = 0; i < getRes.size(); i++) {
- String json = getRes.get(i).get();
- if (json == null) {
- if(logger.isDebugEnabled()) {
- logger.debug("Cannot read payload for {}", zremIds.get(i));
- }
- monitor.misses.increment();
- continue;
- }
- Message msg = om.readValue(json, Message.class);
- msg.setShard(shardName);
- popped.add(msg);
- }
- return popped;
- } finally {
- jedis.close();
- }
- }
-
- @Override
- public boolean ack(String messageId) {
-
- Stopwatch sw = monitor.ack.start();
- Jedis jedis = connPool.getResource();
-
- try {
-
- Long removed = jedis.zrem(unackShardKey(messageId), messageId);
- if (removed > 0) {
- jedis.hdel(messageStoreKey(messageId), messageId);
- return true;
- }
-
- return false;
-
- } finally {
- jedis.close();
- sw.stop();
- }
- }
-
- @Override
- public void ack(List messages) {
-
- Stopwatch sw = monitor.ack.start();
- Jedis jedis = connPool.getResource();
- Pipeline pipe = jedis.pipelined();
- List> responses = new LinkedList<>();
- try {
- for(Message msg : messages) {
- responses.add(pipe.zrem(unackShardKey(msg.getId()), msg.getId()));
- }
- pipe.sync();
- pipe.close();
-
- List> dels = new LinkedList<>();
- for(int i = 0; i < messages.size(); i++) {
- Long removed = responses.get(i).get();
- if (removed > 0) {
- dels.add(pipe.hdel(messageStoreKey(messages.get(i).getId()), messages.get(i).getId()));
- }
- }
- pipe.sync();
- pipe.close();
-
- } catch (IOException e) {
- throw new RuntimeException(e);
- } finally {
- jedis.close();
- sw.stop();
- }
-
-
- }
-
- @Override
- public boolean setUnackTimeout(String messageId, long timeout) {
-
- Stopwatch sw = monitor.ack.start();
- Jedis jedis = connPool.getResource();
-
- try {
-
- double unackScore = Long.valueOf(System.currentTimeMillis() + timeout).doubleValue();
- Double score = jedis.zscore(unackShardKey(messageId), messageId);
- if (score != null) {
- jedis.zadd(unackShardKey(messageId), unackScore, messageId);
- return true;
- }
-
- return false;
-
- } finally {
- jedis.close();
- sw.stop();
- }
- }
-
- @Override
- public boolean setTimeout(String messageId, long timeout) {
-
- Jedis jedis = connPool.getResource();
-
- try {
- String json = jedis.hget(messageStoreKey(messageId), messageId);
- if (json == null) {
- return false;
- }
- Message message = om.readValue(json, Message.class);
- message.setTimeout(timeout);
-
- Double score = jedis.zscore(myQueueShard, messageId);
- if (score != null) {
- double priorityd = message.getPriority() / 100.0;
- double newScore = Long.valueOf(System.currentTimeMillis() + timeout).doubleValue() + priorityd;
- jedis.zadd(myQueueShard, newScore, messageId);
- json = om.writeValueAsString(message);
- jedis.hset(messageStoreKey(message.getId()), message.getId(), json);
- return true;
-
- }
-
- return false;
- } catch (Exception e) {
- throw new RuntimeException(e);
- } finally {
- jedis.close();
- }
-
- }
-
- @Override
- public boolean remove(String messageId) {
-
- Stopwatch sw = monitor.remove.start();
- Jedis jedis = connPool.getResource();
-
- try {
-
- jedis.zrem(unackShardKey(messageId), messageId);
-
- Long removed = jedis.zrem(myQueueShard, messageId);
- Long msgRemoved = jedis.hdel(messageStoreKey(messageId), messageId);
-
- if (removed > 0 && msgRemoved > 0) {
- return true;
- }
-
- return false;
-
- } finally {
- jedis.close();
- sw.stop();
- }
- }
-
- @Override
- public Message get(String messageId) {
-
- Stopwatch sw = monitor.get.start();
- Jedis jedis = connPool.getResource();
- try {
-
- String json = jedis.hget(messageStoreKey(messageId), messageId);
- if (json == null) {
- if (logger.isDebugEnabled()) {
- logger.debug("Cannot get the message payload " + messageId);
- }
- return null;
- }
-
- Message msg = om.readValue(json, Message.class);
- return msg;
-
- } catch (Exception e) {
- throw new RuntimeException(e);
- } finally {
- jedis.close();
- sw.stop();
- }
- }
-
- @Override
- public long size() {
-
- Stopwatch sw = monitor.size.start();
- Jedis jedis = nonQuorumPool.getResource();
-
- try {
- long size = jedis.zcard(myQueueShard);
- return size;
- } finally {
- jedis.close();
- sw.stop();
- }
- }
-
- @Override
- public Map> shardSizes() {
-
- Stopwatch sw = monitor.size.start();
- Map> shardSizes = new HashMap<>();
- Jedis jedis = nonQuorumPool.getResource();
- try {
-
- long size = jedis.zcard(myQueueShard);
- long uacked = 0;
- for(int i = 0; i < maxHashBuckets; i++) {
- String unackShardKey = unackShardKeyPrefix + i;
- uacked += jedis.zcard(unackShardKey);
- }
-
- Map shardDetails = new HashMap<>();
- shardDetails.put("size", size);
- shardDetails.put("uacked", uacked);
- shardSizes.put(shardName, shardDetails);
-
- return shardSizes;
-
- } finally {
- jedis.close();
- sw.stop();
- }
- }
-
- @Override
- public void clear() {
- Jedis jedis = connPool.getResource();
- try {
-
- jedis.del(myQueueShard);
-
- for(int i = 0; i < maxHashBuckets; i++) {
- String unackShardKey = unackShardKeyPrefix + i;
- jedis.del(unackShardKey);
-
- String messageStoreKey = messageStoreKeyPrefix + i + "." + queueName;
- jedis.del(messageStoreKey);
-
- }
-
- } finally {
- jedis.close();
- }
- }
-
- private Set peekIds(int offset, int count) {
- Jedis jedis = connPool.getResource();
- try {
- double now = Long.valueOf(System.currentTimeMillis() + 1).doubleValue();
- Set scanned = jedis.zrangeByScore(myQueueShard, 0, now, offset, count);
- return scanned;
- } finally {
- jedis.close();
- }
- }
-
- public void processUnacks() {
- for(int i = 0; i < maxHashBuckets; i++) {
- String unackShardKey = unackShardKeyPrefix + i;
- processUnacks(unackShardKey);
- }
- }
-
- private void processUnacks(String unackShardKey) {
-
- Stopwatch sw = monitor.processUnack.start();
- Jedis jedis = connPool.getResource();
-
- try {
-
- do {
-
- long queueDepth = size();
- monitor.queueDepth.record(queueDepth);
-
- int batchSize = 1_000;
-
- double now = Long.valueOf(System.currentTimeMillis()).doubleValue();
-
- Set unacks = jedis.zrangeByScoreWithScores(unackShardKey, 0, now, 0, batchSize);
-
- if (unacks.size() > 0) {
- logger.debug("Adding " + unacks.size() + " messages back to the queue for " + queueName);
- } else {
- //Nothing more to be processed
- return;
- }
-
- for (Tuple unack : unacks) {
-
- double score = unack.getScore();
- String member = unack.getElement();
-
- String payload = jedis.hget(messageStoreKey(member), member);
- if (payload == null) {
- jedis.zrem(unackShardKey(member), member);
- continue;
- }
-
- jedis.zadd(myQueueShard, score, member);
- jedis.zrem(unackShardKey(member), member);
- }
-
- } while (true);
-
- } finally {
- jedis.close();
- sw.stop();
- }
-
- }
-
- @Override
- public void close() throws IOException {
- schedulerForUnacksProcessing.shutdown();
- schedulerForPrefetchProcessing.shutdown();
- monitor.close();
- }
-
-
-}
diff --git a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/RedisQueues.java b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/RedisQueues.java
index 30bbbfe..e0df5d7 100644
--- a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/RedisQueues.java
+++ b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/RedisQueues.java
@@ -1,12 +1,12 @@
/**
* Copyright 2016 Netflix, Inc.
- *
+ *
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -15,17 +15,20 @@
*/
package com.netflix.dyno.queues.redis;
+import com.netflix.dyno.jedis.DynoJedisClient;
+import com.netflix.dyno.queues.DynoQueue;
+import com.netflix.dyno.queues.ShardSupplier;
+import com.netflix.dyno.queues.redis.sharding.RoundRobinStrategy;
+import com.netflix.dyno.queues.redis.sharding.ShardingStrategy;
+import redis.clients.jedis.commands.JedisCommands;
+
import java.io.Closeable;
import java.io.IOException;
+import java.time.Clock;
import java.util.Collection;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
-import com.netflix.dyno.queues.DynoQueue;
-import com.netflix.dyno.queues.ShardSupplier;
-
-import redis.clients.jedis.JedisCommands;
-
/**
* @author Viren
*
@@ -34,87 +37,116 @@
*/
public class RedisQueues implements Closeable {
- private JedisCommands quorumConn;
-
- private JedisCommands nonQuorumConn;
-
- private Set allShards;
-
- private String shardName;
-
- private String redisKeyPrefix;
-
- private int unackTime;
-
- private int unackHandlerIntervalInMS;
-
- private ConcurrentHashMap queues;
-
- /**
- *
- * @param quorumConn Dyno connection with dc_quorum enabled
- * @param nonQuorumConn Dyno connection to local Redis
- * @param redisKeyPrefix prefix applied to the Redis keys
- * @param shardSupplier Provider for the shards for the queues created
- * @param unackTime Time in millisecond within which a message needs to be acknowledged by the client, after which the message is re-queued.
- * @param unackHandlerIntervalInMS Time in millisecond at which the un-acknowledgement processor runs
- */
- public RedisQueues(JedisCommands quorumConn, JedisCommands nonQuorumConn, String redisKeyPrefix, ShardSupplier shardSupplier, int unackTime,
- int unackHandlerIntervalInMS) {
-
- this.quorumConn = quorumConn;
- this.nonQuorumConn = nonQuorumConn;
- this.redisKeyPrefix = redisKeyPrefix;
- this.allShards = shardSupplier.getQueueShards();
- this.shardName = shardSupplier.getCurrentShard();
- this.unackTime = unackTime;
- this.unackHandlerIntervalInMS = unackHandlerIntervalInMS;
- this.queues = new ConcurrentHashMap<>();
- }
-
- /**
- *
- * @param queueName Name of the queue
- * @return Returns the DynoQueue hosting the given queue by name
- * @see DynoQueue
- * @see RedisDynoQueue
- */
- public DynoQueue get(String queueName) {
-
- String key = queueName.intern();
- DynoQueue queue = this.queues.get(key);
- if (queue != null) {
- return queue;
- }
-
- synchronized (this) {
- queue = new RedisDynoQueue(redisKeyPrefix, queueName, allShards, shardName, unackHandlerIntervalInMS)
- .withUnackTime(unackTime)
- .withNonQuorumConn(nonQuorumConn)
- .withQuorumConn(quorumConn);
- this.queues.put(key, queue);
- }
-
- return queue;
- }
-
- /**
- *
- * @return Collection of all the registered queues
- */
- public Collection queues(){
- return this.queues.values();
- }
-
- @Override
- public void close() throws IOException {
- queues.values().forEach(queue -> {
- try {
- queue.close();
- }
- catch (final IOException e) {
- throw new RuntimeException(e.getCause());
- }
- });
- }
+ private final Clock clock;
+
+ private final JedisCommands quorumConn;
+
+ private final JedisCommands nonQuorumConn;
+
+ private final Set allShards;
+
+ private final String shardName;
+
+ private final String redisKeyPrefix;
+
+ private final int unackTime;
+
+ private final int unackHandlerIntervalInMS;
+
+ private final ConcurrentHashMap queues;
+
+ private final ShardingStrategy shardingStrategy;
+
+ private final boolean singleRingTopology;
+
+ /**
+ * @param quorumConn Dyno connection with dc_quorum enabled
+ * @param nonQuorumConn Dyno connection to local Redis
+ * @param redisKeyPrefix prefix applied to the Redis keys
+ * @param shardSupplier Provider for the shards for the queues created
+ * @param unackTime Time in millisecond within which a message needs to be acknowledged by the client, after which the message is re-queued.
+ * @param unackHandlerIntervalInMS Time in millisecond at which the un-acknowledgement processor runs
+ */
+ public RedisQueues(JedisCommands quorumConn, JedisCommands nonQuorumConn, String redisKeyPrefix, ShardSupplier shardSupplier, int unackTime, int unackHandlerIntervalInMS) {
+ this(Clock.systemDefaultZone(), quorumConn, nonQuorumConn, redisKeyPrefix, shardSupplier, unackTime, unackHandlerIntervalInMS, new RoundRobinStrategy());
+ }
+
+ /**
+ * @param quorumConn Dyno connection with dc_quorum enabled
+ * @param nonQuorumConn Dyno connection to local Redis
+ * @param redisKeyPrefix prefix applied to the Redis keys
+ * @param shardSupplier Provider for the shards for the queues created
+ * @param unackTime Time in millisecond within which a message needs to be acknowledged by the client, after which the message is re-queued.
+ * @param unackHandlerIntervalInMS Time in millisecond at which the un-acknowledgement processor runs
+ * @param shardingStrategy sharding strategy responsible for calculating message's destination shard
+ */
+ public RedisQueues(JedisCommands quorumConn, JedisCommands nonQuorumConn, String redisKeyPrefix, ShardSupplier shardSupplier, int unackTime, int unackHandlerIntervalInMS, ShardingStrategy shardingStrategy) {
+ this(Clock.systemDefaultZone(), quorumConn, nonQuorumConn, redisKeyPrefix, shardSupplier, unackTime, unackHandlerIntervalInMS, shardingStrategy);
+ }
+
+
+ /**
+ * @param clock Time provider
+ * @param quorumConn Dyno connection with dc_quorum enabled
+ * @param nonQuorumConn Dyno connection to local Redis
+ * @param redisKeyPrefix prefix applied to the Redis keys
+ * @param shardSupplier Provider for the shards for the queues created
+ * @param unackTime Time in millisecond within which a message needs to be acknowledged by the client, after which the message is re-queued.
+ * @param unackHandlerIntervalInMS Time in millisecond at which the un-acknowledgement processor runs
+ * @param shardingStrategy sharding strategy responsible for calculating message's destination shard
+ */
+ public RedisQueues(Clock clock, JedisCommands quorumConn, JedisCommands nonQuorumConn, String redisKeyPrefix, ShardSupplier shardSupplier, int unackTime, int unackHandlerIntervalInMS, ShardingStrategy shardingStrategy) {
+ this.clock = clock;
+ this.quorumConn = quorumConn;
+ this.nonQuorumConn = nonQuorumConn;
+ this.redisKeyPrefix = redisKeyPrefix;
+ this.allShards = shardSupplier.getQueueShards();
+ this.shardName = shardSupplier.getCurrentShard();
+ this.unackTime = unackTime;
+ this.unackHandlerIntervalInMS = unackHandlerIntervalInMS;
+ this.queues = new ConcurrentHashMap<>();
+ this.shardingStrategy = shardingStrategy;
+
+ if (quorumConn instanceof DynoJedisClient) {
+ this.singleRingTopology = ((DynoJedisClient) quorumConn).getConnPool().getPools().size() == 3;
+ } else {
+ this.singleRingTopology = false;
+ }
+ }
+
+ /**
+ *
+ * @param queueName Name of the queue
+ * @return Returns the DynoQueue hosting the given queue by name
+ * @see DynoQueue
+ * @see RedisDynoQueue
+ */
+ public DynoQueue get(String queueName) {
+
+ String key = queueName.intern();
+
+ return queues.computeIfAbsent(key, (keyToCompute) -> new RedisDynoQueue(clock, redisKeyPrefix, queueName, allShards, shardName, unackHandlerIntervalInMS, shardingStrategy, singleRingTopology)
+ .withUnackTime(unackTime)
+ .withNonQuorumConn(nonQuorumConn)
+ .withQuorumConn(quorumConn));
+ }
+
+ /**
+ *
+ * @return Collection of all the registered queues
+ */
+ public Collection queues() {
+ return this.queues.values();
+ }
+
+ @Override
+ public void close() throws IOException {
+ queues.values().forEach(queue -> {
+ try {
+ queue.close();
+ } catch (final IOException e) {
+ throw new RuntimeException(e.getCause());
+ }
+ });
+ }
}
diff --git a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/conn/DynoClientProxy.java b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/conn/DynoClientProxy.java
new file mode 100644
index 0000000..88b6a36
--- /dev/null
+++ b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/conn/DynoClientProxy.java
@@ -0,0 +1,107 @@
+/**
+ * Copyright 2018 Netflix, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/**
+ *
+ */
+package com.netflix.dyno.queues.redis.conn;
+
+import java.util.Set;
+
+import com.netflix.dyno.jedis.DynoJedisClient;
+
+import redis.clients.jedis.Tuple;
+
+/**
+ * @author Viren
+ *
+ * Dynomite connection
+ */
+public class DynoClientProxy implements RedisConnection {
+
+ private DynoJedisClient jedis;
+
+
+ public DynoClientProxy(DynoJedisClient jedis) {
+ this.jedis = jedis;
+ }
+
+ @Override
+ public RedisConnection getResource() {
+ return this;
+ }
+
+ @Override
+ public void close() {
+ //nothing!
+ }
+
+ @Override
+ public Pipe pipelined() {
+ return new DynoJedisPipe(jedis.pipelined());
+ }
+
+ @Override
+ public String hget(String key, String member) {
+ return jedis.hget(key, member);
+ }
+
+ @Override
+ public Long zrem(String key, String member) {
+ return jedis.zrem(key, member);
+ }
+
+ @Override
+ public Long hdel(String key, String member) {
+ return jedis.hdel(key, member);
+
+ }
+
+ @Override
+ public Double zscore(String key, String member) {
+ return jedis.zscore(key, member);
+ }
+
+ @Override
+ public void zadd(String key, double score, String member) {
+ jedis.zadd(key, score, member);
+ }
+
+ @Override
+ public void hset(String key, String member, String json) {
+ jedis.hset(key, member, json);
+ }
+
+ @Override
+ public long zcard(String key) {
+ return jedis.zcard(key);
+ }
+
+ @Override
+ public void del(String key) {
+ jedis.del(key);
+ }
+
+ @Override
+ public Set zrangeByScore(String key, int min, double max, int offset, int count) {
+ return jedis.zrangeByScore(key, min, max, offset, count);
+ }
+
+ @Override
+ public Set zrangeByScoreWithScores(String key, int min, double max, int offset, int count) {
+ return jedis.zrangeByScoreWithScores(key, min, max, offset, count);
+ }
+
+}
diff --git a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/conn/DynoJedisPipe.java b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/conn/DynoJedisPipe.java
new file mode 100644
index 0000000..2b279b5
--- /dev/null
+++ b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/conn/DynoJedisPipe.java
@@ -0,0 +1,92 @@
+/**
+ * Copyright 2018 Netflix, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/**
+ *
+ */
+package com.netflix.dyno.queues.redis.conn;
+
+
+import com.netflix.dyno.jedis.DynoJedisPipeline;
+import redis.clients.jedis.Response;
+import redis.clients.jedis.params.ZAddParams;
+
+/**
+ * @author Viren
+ * Pipeline abstraction for Dynomite Pipeline.
+ */
+public class DynoJedisPipe implements Pipe {
+
+ private DynoJedisPipeline pipe;
+
+ private boolean modified;
+
+ public DynoJedisPipe(DynoJedisPipeline pipe) {
+ this.pipe = pipe;
+ this.modified = false;
+ }
+
+ @Override
+ public void hset(String key, String field, String value) {
+ pipe.hset(key, field, value);
+ this.modified = true;
+
+ }
+
+ @Override
+ public Response zadd(String key, double score, String member) {
+ this.modified = true;
+ return pipe.zadd(key, score, member);
+ }
+
+ @Override
+ public Response zadd(String key, double score, String member, ZAddParams zParams) {
+ this.modified = true;
+ return pipe.zadd(key, score, member, zParams);
+ }
+
+ @Override
+ public Response zrem(String key, String member) {
+ this.modified = true;
+ return pipe.zrem(key, member);
+ }
+
+ @Override
+ public Response hget(String key, String member) {
+ this.modified = true;
+ return pipe.hget(key, member);
+ }
+
+ @Override
+ public Response hdel(String key, String member) {
+ this.modified = true;
+ return pipe.hdel(key, member);
+ }
+
+ @Override
+ public void sync() {
+ if (modified) {
+ pipe.sync();
+ modified = false;
+ }
+ }
+
+ @Override
+ public void close() throws Exception {
+ pipe.close();
+ }
+
+
+}
diff --git a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/conn/JedisProxy.java b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/conn/JedisProxy.java
new file mode 100644
index 0000000..e686ff9
--- /dev/null
+++ b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/conn/JedisProxy.java
@@ -0,0 +1,113 @@
+/**
+ * Copyright 2018 Netflix, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/**
+ *
+ */
+package com.netflix.dyno.queues.redis.conn;
+
+import java.util.Set;
+
+import redis.clients.jedis.Jedis;
+import redis.clients.jedis.JedisPool;
+import redis.clients.jedis.Tuple;
+
+/**
+ * @author Viren
+ *
+ * This class provides the abstraction of a Jedis Connection Pool. Used when using Redis directly without Dynomite.
+ */
+public class JedisProxy implements RedisConnection {
+
+ private JedisPool pool;
+
+ private Jedis jedis;
+
+ public JedisProxy(JedisPool pool) {
+ this.pool = pool;
+ }
+
+ public JedisProxy(Jedis jedis) {
+ this.jedis = jedis;
+ }
+
+ @Override
+ public RedisConnection getResource() {
+ Jedis jedis = pool.getResource();
+ return new JedisProxy(jedis);
+ }
+
+ @Override
+ public void close() {
+ jedis.close();
+ }
+
+ @Override
+ public Pipe pipelined() {
+ return new RedisPipe(jedis.pipelined());
+ }
+
+ @Override
+ public String hget(String key, String member) {
+ return jedis.hget(key, member);
+ }
+
+ @Override
+ public Long zrem(String key, String member) {
+ return jedis.zrem(key, member);
+ }
+
+ @Override
+ public Long hdel(String key, String member) {
+ return jedis.hdel(key, member);
+
+ }
+
+ @Override
+ public Double zscore(String key, String member) {
+ return jedis.zscore(key, member);
+ }
+
+ @Override
+ public void zadd(String key, double unackScore, String member) {
+ jedis.zadd(key, unackScore, member);
+ }
+
+ @Override
+ public void hset(String key, String member, String json) {
+ jedis.hset(key, member, json);
+ }
+
+ @Override
+ public long zcard(String key) {
+ return jedis.zcard(key);
+ }
+
+ @Override
+ public void del(String key) {
+ jedis.del(key);
+ }
+
+ @Override
+ public Set zrangeByScore(String key, int min, double max, int offset, int count) {
+ return jedis.zrangeByScore(key, min, max, offset, count);
+ }
+
+ @Override
+ public Set zrangeByScoreWithScores(String key, int min, double max, int offset, int count) {
+ return jedis.zrangeByScoreWithScores(key, min, max, offset, count);
+ }
+
+}
diff --git a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/conn/Pipe.java b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/conn/Pipe.java
new file mode 100644
index 0000000..ef8624c
--- /dev/null
+++ b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/conn/Pipe.java
@@ -0,0 +1,98 @@
+/**
+ * Copyright 2018 Netflix, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.netflix.dyno.queues.redis.conn;
+
+import com.netflix.dyno.jedis.DynoJedisPipeline;
+import redis.clients.jedis.Pipeline;
+import redis.clients.jedis.Response;
+import redis.clients.jedis.params.ZAddParams;
+
+/**
+ *
+ * @author Viren
+ *
+ * Abstraction of Redis Pipeline.
+ * The abstraction is required as there is no common interface between DynoJedisPipeline and Jedis' Pipeline classes.
+ *
+ * @see DynoJedisPipeline
+ * @see Pipeline
+ * The commands here reflects the RedisCommand structure.
+ *
+ */
+public interface Pipe {
+
+ /**
+ *
+ * @param key The Key
+ * @param field Field
+ * @param value Value of the Field
+ */
+ public void hset(String key, String field, String value);
+
+ /**
+ *
+ * @param key The Key
+ * @param score Score for the member
+ * @param member Member to be added within the key
+ * @return
+ */
+ public Response zadd(String key, double score, String member);
+
+ /**
+ *
+ * @param key The Key
+ * @param score Score for the member
+ * @param member Member to be added within the key
+ * @param zParams Parameters
+ * @return
+ */
+ public Response zadd(String key, double score, String member, ZAddParams zParams);
+
+ /**
+ *
+ * @param key The Key
+ * @param member Member
+ * @return
+ */
+ public Response zrem(String key, String member);
+
+ /**
+ *
+ * @param key The Key
+ * @param member Member
+ * @return
+ */
+ public Response hget(String key, String member);
+
+ /**
+ *
+ * @param key
+ * @param member
+ * @return
+ */
+ public Response hdel(String key, String member);
+
+ /**
+ *
+ */
+ public void sync();
+
+ /**
+ *
+ * @throws Exception
+ */
+ public void close() throws Exception;
+}
\ No newline at end of file
diff --git a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/conn/RedisConnection.java b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/conn/RedisConnection.java
new file mode 100644
index 0000000..b854c43
--- /dev/null
+++ b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/conn/RedisConnection.java
@@ -0,0 +1,67 @@
+/**
+ * Copyright 2018 Netflix, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.netflix.dyno.queues.redis.conn;
+
+import java.util.Set;
+
+import com.netflix.dyno.jedis.DynoJedisClient;
+
+import redis.clients.jedis.Jedis;
+import redis.clients.jedis.Tuple;
+
+/**
+ * Abstraction of Redis connection.
+ *
+ * @author viren
+ *
+ * The methods are 1-1 proxies from Jedis. See Jedis documentation for the details.
+ *
+ * @see Jedis
+ * @see DynoJedisClient
+ */
+public interface RedisConnection {
+
+ /**
+ *
+ * @return Returns the underlying connection resource. For connection pool, returns the actual connection
+ */
+ public RedisConnection getResource();
+
+ public String hget(String messkeyageStoreKey, String member);
+
+ public Long zrem(String key, String member);
+
+ public Long hdel(String key, String member);
+
+ public Double zscore(String key, String member);
+
+ public void zadd(String key, double score, String member);
+
+ public void hset(String key, String id, String json);
+
+ public long zcard(String key);
+
+ public void del(String key);
+
+ public Set zrangeByScore(String key, int min, double max, int offset, int count);
+
+ public Set zrangeByScoreWithScores(String key, int min, double max, int offset, int count);
+
+ public void close();
+
+ public Pipe pipelined();
+
+}
\ No newline at end of file
diff --git a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/conn/RedisPipe.java b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/conn/RedisPipe.java
new file mode 100644
index 0000000..3210453
--- /dev/null
+++ b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/conn/RedisPipe.java
@@ -0,0 +1,80 @@
+/**
+ * Copyright 2018 Netflix, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/**
+ *
+ */
+package com.netflix.dyno.queues.redis.conn;
+
+import redis.clients.jedis.Pipeline;
+import redis.clients.jedis.Response;
+import redis.clients.jedis.params.ZAddParams;
+
+/**
+ * @author Viren
+ *
+ * Pipeline abstraction for direct redis connection - when not using Dynomite.
+ */
+public class RedisPipe implements Pipe {
+
+ private Pipeline pipe;
+
+ public RedisPipe(Pipeline pipe) {
+ this.pipe = pipe;
+ }
+
+ @Override
+ public void hset(String key, String field, String value) {
+ pipe.hset(key, field, value);
+
+ }
+
+ @Override
+ public Response zadd(String key, double score, String member) {
+ return pipe.zadd(key, score, member);
+ }
+
+ @Override
+ public Response zadd(String key, double score, String member, ZAddParams zParams) {
+ return pipe.zadd(key, score, member, zParams);
+ }
+
+ @Override
+ public Response zrem(String key, String member) {
+ return pipe.zrem(key, member);
+ }
+
+ @Override
+ public Response hget(String key, String member) {
+ return pipe.hget(key, member);
+ }
+
+ @Override
+ public Response hdel(String key, String member) {
+ return pipe.hdel(key, member);
+ }
+
+ @Override
+ public void sync() {
+ pipe.sync();
+ }
+
+ @Override
+ public void close() throws Exception {
+ pipe.close();
+ }
+
+
+}
diff --git a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/sharding/RoundRobinStrategy.java b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/sharding/RoundRobinStrategy.java
new file mode 100644
index 0000000..ff5ac4a
--- /dev/null
+++ b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/sharding/RoundRobinStrategy.java
@@ -0,0 +1,42 @@
+/**
+ * Copyright 2018 Netflix, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.netflix.dyno.queues.redis.sharding;
+
+import com.netflix.dyno.queues.Message;
+
+import java.util.List;
+import java.util.concurrent.atomic.AtomicInteger;
+
+public class RoundRobinStrategy implements ShardingStrategy {
+
+ private final AtomicInteger nextShardIndex = new AtomicInteger(0);
+
+ /**
+ * Get shard based on round robin strategy.
+ * @param allShards
+ * @param message is ignored in round robin strategy
+ * @return
+ */
+ @Override
+ public String getNextShard(List allShards, Message message) {
+ int index = nextShardIndex.incrementAndGet();
+ if (index >= allShards.size()) {
+ nextShardIndex.set(0);
+ index = 0;
+ }
+ return allShards.get(index);
+ }
+}
diff --git a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/sharding/ShardingStrategy.java b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/sharding/ShardingStrategy.java
new file mode 100644
index 0000000..e27a2ba
--- /dev/null
+++ b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/sharding/ShardingStrategy.java
@@ -0,0 +1,27 @@
+/**
+ * Copyright 2018 Netflix, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.netflix.dyno.queues.redis.sharding;
+
+import com.netflix.dyno.queues.Message;
+
+import java.util.List;
+
+/**
+ * Expose common interface that allow to apply custom sharding strategy.
+ */
+public interface ShardingStrategy {
+ String getNextShard(List allShards, Message message);
+}
diff --git a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/v2/MultiRedisQueue.java b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/v2/MultiRedisQueue.java
new file mode 100644
index 0000000..47bd906
--- /dev/null
+++ b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/v2/MultiRedisQueue.java
@@ -0,0 +1,292 @@
+/**
+ * Copyright 2017 Netflix, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.netflix.dyno.queues.redis.v2;
+
+import com.netflix.dyno.queues.DynoQueue;
+import com.netflix.dyno.queues.Message;
+
+import java.io.IOException;
+import java.lang.UnsupportedOperationException;
+import java.util.HashMap;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.stream.Collectors;
+
+/**
+ * @author Viren
+ * MultiRedisQueue exposes a single queue using multiple redis queues. Each RedisQueue is a shard.
+ * When pushing elements to the queue, does a round robin to push the message to one of the shards.
+ * When polling, the message is polled from the current shard (shardName) the instance is associated with.
+ */
+public class MultiRedisQueue implements DynoQueue {
+
+ private List shards;
+
+ private String name;
+
+ private Map queues = new HashMap<>();
+
+ private RedisPipelineQueue me;
+
+ public MultiRedisQueue(String queueName, String shardName, Map queues) {
+ this.name = queueName;
+ this.queues = queues;
+ this.me = queues.get(shardName);
+ if (me == null) {
+ throw new IllegalArgumentException("List of shards supplied (" + queues.keySet() + ") does not contain current shard name: " + shardName);
+ }
+ this.shards = queues.keySet().stream().collect(Collectors.toList());
+ }
+
+ @Override
+ public String getName() {
+ return name;
+ }
+
+ @Override
+ public int getUnackTime() {
+ return me.getUnackTime();
+ }
+
+ @Override
+ public List push(List messages) {
+ int size = queues.size();
+ int partitionSize = messages.size() / size;
+ List ids = new LinkedList<>();
+
+ for (int i = 0; i < size - 1; i++) {
+ RedisPipelineQueue queue = queues.get(getNextShard());
+ int start = i * partitionSize;
+ int end = start + partitionSize;
+ ids.addAll(queue.push(messages.subList(start, end)));
+ }
+ RedisPipelineQueue queue = queues.get(getNextShard());
+ int start = (size - 1) * partitionSize;
+
+ ids.addAll(queue.push(messages.subList(start, messages.size())));
+ return ids;
+ }
+
+ @Override
+ public List pop(int messageCount, int wait, TimeUnit unit) {
+ return me.pop(messageCount, wait, unit);
+ }
+
+ @Override
+ public Message popWithMsgId(String messageId) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public Message unsafePopWithMsgIdAllShards(String messageId) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public List peek(int messageCount) {
+ return me.peek(messageCount);
+ }
+
+ @Override
+ public boolean ack(String messageId) {
+ for (DynoQueue q : queues.values()) {
+ if (q.ack(messageId)) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ @Override
+ public void ack(List messages) {
+ Map> byShard = messages.stream().collect(Collectors.groupingBy(Message::getShard));
+ for (Entry> e : byShard.entrySet()) {
+ queues.get(e.getKey()).ack(e.getValue());
+ }
+ }
+
+ @Override
+ public boolean setUnackTimeout(String messageId, long timeout) {
+ for (DynoQueue q : queues.values()) {
+ if (q.setUnackTimeout(messageId, timeout)) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ @Override
+ public boolean setTimeout(String messageId, long timeout) {
+ for (DynoQueue q : queues.values()) {
+ if (q.setTimeout(messageId, timeout)) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ @Override
+ public boolean remove(String messageId) {
+ for (DynoQueue q : queues.values()) {
+ if (q.remove(messageId)) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ @Override
+ public boolean ensure(Message message) {
+ throw new UnsupportedOperationException();
+ }
+
+
+ @Override
+ public boolean containsPredicate(String predicate) {
+ return containsPredicate(predicate, false);
+ }
+
+ @Override
+ public String getMsgWithPredicate(String predicate) {
+ return getMsgWithPredicate(predicate, false);
+ }
+
+ @Override
+ public boolean containsPredicate(String predicate, boolean localShardOnly) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public String getMsgWithPredicate(String predicate, boolean localShardOnly) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public Message popMsgWithPredicate(String predicate, boolean localShardOnly) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public List bulkPop(int messageCount, int wait, TimeUnit unit) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public List unsafeBulkPop(int messageCount, int wait, TimeUnit unit) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public Message get(String messageId) {
+ for (DynoQueue q : queues.values()) {
+ Message msg = q.get(messageId);
+ if (msg != null) {
+ return msg;
+ }
+ }
+ return null;
+ }
+
+ @Override
+ public Message localGet(String messageId) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public long size() {
+ long size = 0;
+ for (DynoQueue q : queues.values()) {
+ size += q.size();
+ }
+ return size;
+ }
+
+ @Override
+ public Map> shardSizes() {
+ Map> sizes = new HashMap<>();
+ for (Entry e : queues.entrySet()) {
+ sizes.put(e.getKey(), e.getValue().shardSizes().get(e.getKey()));
+ }
+ return sizes;
+ }
+
+ @Override
+ public void clear() {
+ for (DynoQueue q : queues.values()) {
+ q.clear();
+ }
+
+ }
+
+ @Override
+ public void close() throws IOException {
+ for (RedisPipelineQueue queue : queues.values()) {
+ queue.close();
+ }
+ }
+
+ @Override
+ public List getAllMessages() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void processUnacks() {
+ for (RedisPipelineQueue queue : queues.values()) {
+ queue.processUnacks();
+ }
+ }
+
+ @Override
+ public void atomicProcessUnacks() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public List findStaleMessages() { throw new UnsupportedOperationException(); }
+
+ @Override
+ public boolean atomicRemove(String messageId) {
+ throw new UnsupportedOperationException();
+ }
+
+ private AtomicInteger nextShardIndex = new AtomicInteger(0);
+
+ private String getNextShard() {
+ int indx = nextShardIndex.incrementAndGet();
+ if (indx >= shards.size()) {
+ nextShardIndex.set(0);
+ indx = 0;
+ }
+ String s = shards.get(indx);
+ return s;
+ }
+
+ @Override
+ public List unsafePeekAllShards(final int messageCount) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public List unsafePopAllShards(int messageCount, int wait, TimeUnit unit) {
+ throw new UnsupportedOperationException();
+ }
+
+}
diff --git a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/v2/QueueBuilder.java b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/v2/QueueBuilder.java
new file mode 100644
index 0000000..3875bb9
--- /dev/null
+++ b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/v2/QueueBuilder.java
@@ -0,0 +1,282 @@
+/**
+ * Copyright 2018 Netflix, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/**
+ *
+ */
+package com.netflix.dyno.queues.redis.v2;
+
+import com.google.common.collect.Collections2;
+import com.google.common.collect.Lists;
+import com.netflix.appinfo.AmazonInfo;
+import com.netflix.appinfo.AmazonInfo.MetaDataKey;
+import com.netflix.appinfo.InstanceInfo;
+import com.netflix.appinfo.InstanceInfo.InstanceStatus;
+import com.netflix.discovery.EurekaClient;
+import com.netflix.discovery.shared.Application;
+import com.netflix.dyno.connectionpool.Host;
+import com.netflix.dyno.connectionpool.HostBuilder;
+import com.netflix.dyno.connectionpool.HostSupplier;
+import com.netflix.dyno.connectionpool.impl.utils.ConfigUtils;
+import com.netflix.dyno.jedis.DynoJedisClient;
+import com.netflix.dyno.queues.DynoQueue;
+import com.netflix.dyno.queues.ShardSupplier;
+import com.netflix.dyno.queues.redis.conn.DynoClientProxy;
+import com.netflix.dyno.queues.redis.conn.JedisProxy;
+import com.netflix.dyno.queues.redis.conn.RedisConnection;
+import com.netflix.dyno.queues.shard.DynoShardSupplier;
+import redis.clients.jedis.JedisPool;
+import redis.clients.jedis.JedisPoolConfig;
+
+import java.time.Clock;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * @author Viren
+ * Builder for the queues.
+ *
+ */
+public class QueueBuilder {
+
+ private Clock clock;
+
+ private String queueName;
+
+ private String redisKeyPrefix;
+
+ private int unackTime;
+
+ private String currentShard;
+
+ private ShardSupplier shardSupplier;
+
+ private HostSupplier hs;
+
+ private EurekaClient eurekaClient;
+
+ private String applicationName;
+
+ private Collection hosts;
+
+ private JedisPoolConfig redisPoolConfig;
+
+ private DynoJedisClient dynoQuorumClient;
+
+ private DynoJedisClient dynoNonQuorumClient;
+
+ /**
+ * @param clock the Clock instance to set
+ * @return instance of QueueBuilder
+ */
+ public QueueBuilder setClock(Clock clock) {
+ this.clock = clock;
+ return this;
+ }
+
+ public QueueBuilder setApplicationName(String appName) {
+ this.applicationName = appName;
+ return this;
+ }
+
+ public QueueBuilder setEurekaClient(EurekaClient eurekaClient) {
+ this.eurekaClient = eurekaClient;
+ return this;
+ }
+
+ /**
+ * @param queueName the queueName to set
+ * @return instance of QueueBuilder
+ */
+ public QueueBuilder setQueueName(String queueName) {
+ this.queueName = queueName;
+ return this;
+ }
+
+ /**
+ * @param redisKeyPrefix Prefix used for all the keys in Redis
+ * @return instance of QueueBuilder
+ */
+ public QueueBuilder setRedisKeyPrefix(String redisKeyPrefix) {
+ this.redisKeyPrefix = redisKeyPrefix;
+ return this;
+ }
+
+ /**
+ * @param redisPoolConfig
+ * @return instance of QueueBuilder
+ */
+ public QueueBuilder useNonDynomiteRedis(JedisPoolConfig redisPoolConfig, List redisHosts) {
+ this.redisPoolConfig = redisPoolConfig;
+ this.hosts = redisHosts;
+ return this;
+ }
+
+ /**
+ *
+ * @param dynoQuorumClient
+ * @param dynoNonQuorumClient
+ * @return
+ */
+ public QueueBuilder useDynomite(DynoJedisClient dynoQuorumClient, DynoJedisClient dynoNonQuorumClient) {
+ this.dynoQuorumClient = dynoQuorumClient;
+ this.dynoNonQuorumClient = dynoNonQuorumClient;
+ this.hs = dynoQuorumClient.getConnPool().getConfiguration().getHostSupplier();
+ return this;
+ }
+
+ /**
+ * @param unackTime Time in millisecond, after which the uncked messages will be re-queued for the delivery
+ * @return instance of QueueBuilder
+ */
+ public QueueBuilder setUnackTime(int unackTime) {
+ this.unackTime = unackTime;
+ return this;
+ }
+
+ /**
+ * @param currentShard Name of the current shard
+ * @return instance of QueueBuilder
+ */
+ public QueueBuilder setCurrentShard(String currentShard) {
+ this.currentShard = currentShard;
+ return this;
+ }
+
+ /**
+ * @param shardSupplier
+ * @return
+ */
+ public QueueBuilder setShardSupplier(ShardSupplier shardSupplier) {
+ this.shardSupplier = shardSupplier;
+ return this;
+ }
+
+ /**
+ *
+ * @return Build an instance of the queue with supplied parameters.
+ * @see MultiRedisQueue
+ * @see RedisPipelineQueue
+ */
+ public DynoQueue build() {
+
+ boolean useDynomiteCluster = dynoQuorumClient != null;
+ if (useDynomiteCluster) {
+ if(hs == null) {
+ hs = dynoQuorumClient.getConnPool().getConfiguration().getHostSupplier();
+ }
+ this.hosts = hs.getHosts();
+ }
+
+ if (shardSupplier == null) {
+ String region = ConfigUtils.getDataCenter();
+ String az = ConfigUtils.getLocalZone();
+ shardSupplier = new DynoShardSupplier(hs, region, az);
+ }
+ if(currentShard == null) {
+ currentShard = shardSupplier.getCurrentShard();
+ }
+
+ if (clock == null) {
+ clock = Clock.systemDefaultZone();
+ }
+
+ Map shardMap = new HashMap<>();
+ for (Host host : hosts) {
+ String shard = shardSupplier.getShardForHost(host);
+ shardMap.put(shard, host);
+ }
+
+
+ Map queues = new HashMap<>();
+
+ for (String queueShard : shardMap.keySet()) {
+
+ Host host = shardMap.get(queueShard);
+ String hostAddress = host.getIpAddress();
+ if (hostAddress == null || "".equals(hostAddress)) {
+ hostAddress = host.getHostName();
+ }
+ RedisConnection redisConn = null;
+ RedisConnection redisConnRead = null;
+
+ if (useDynomiteCluster) {
+ redisConn = new DynoClientProxy(dynoQuorumClient);
+ if(dynoNonQuorumClient == null) {
+ dynoNonQuorumClient = dynoQuorumClient;
+ }
+ redisConnRead = new DynoClientProxy(dynoNonQuorumClient);
+ } else {
+ JedisPool pool = new JedisPool(redisPoolConfig, hostAddress, host.getPort(), 0);
+ redisConn = new JedisProxy(pool);
+ redisConnRead = new JedisProxy(pool);
+ }
+
+ RedisPipelineQueue q = new RedisPipelineQueue(clock, redisKeyPrefix, queueName, queueShard, unackTime, unackTime, redisConn);
+ q.setNonQuorumPool(redisConnRead);
+
+ queues.put(queueShard, q);
+ }
+
+ if (queues.size() == 1) {
+ //This is a queue with a single shard
+ return queues.values().iterator().next();
+ }
+
+ MultiRedisQueue queue = new MultiRedisQueue(queueName, currentShard, queues);
+ return queue;
+ }
+
+
+ private HostSupplier getHostSupplierFromEureka(String applicationName) {
+ return () -> {
+ Application app = eurekaClient.getApplication(applicationName);
+ List hosts = new ArrayList<>();
+
+ if (app == null) {
+ return hosts;
+ }
+
+ List ins = app.getInstances();
+
+ if (ins == null || ins.isEmpty()) {
+ return hosts;
+ }
+
+ hosts = Lists.newArrayList(Collections2.transform(ins,
+
+ info -> {
+
+ Host.Status status = info.getStatus() == InstanceStatus.UP ? Host.Status.Up : Host.Status.Down;
+ String rack = null;
+ if (info.getDataCenterInfo() instanceof AmazonInfo) {
+ AmazonInfo amazonInfo = (AmazonInfo) info.getDataCenterInfo();
+ rack = amazonInfo.get(MetaDataKey.availabilityZone);
+ }
+ //Host host = new Host(info.getHostName(), info.getIPAddr(), rack, status);
+ Host host = new HostBuilder()
+ .setHostname(info.getHostName())
+ .setIpAddress(info.getIPAddr())
+ .setRack(rack).setStatus(status)
+ .createHost();
+ return host;
+ }));
+ return hosts;
+ };
+ }
+}
diff --git a/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/v2/RedisPipelineQueue.java b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/v2/RedisPipelineQueue.java
new file mode 100644
index 0000000..33ce187
--- /dev/null
+++ b/dyno-queues-redis/src/main/java/com/netflix/dyno/queues/redis/v2/RedisPipelineQueue.java
@@ -0,0 +1,715 @@
+/**
+ * Copyright 2016 Netflix, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.netflix.dyno.queues.redis.v2;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.netflix.dyno.connectionpool.HashPartitioner;
+import com.netflix.dyno.connectionpool.impl.hash.Murmur3HashPartitioner;
+import com.netflix.dyno.queues.DynoQueue;
+import com.netflix.dyno.queues.Message;
+import com.netflix.dyno.queues.redis.QueueMonitor;
+import com.netflix.dyno.queues.redis.QueueUtils;
+import com.netflix.dyno.queues.redis.conn.Pipe;
+import com.netflix.dyno.queues.redis.conn.RedisConnection;
+import com.netflix.servo.monitor.Stopwatch;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import redis.clients.jedis.Response;
+import redis.clients.jedis.Tuple;
+import redis.clients.jedis.params.ZAddParams;
+
+import java.io.IOException;
+import java.time.Clock;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
+
+/**
+ * @author Viren
+ * Queue implementation that uses Redis pipelines that improves the throughput under heavy load.
+ */
+public class RedisPipelineQueue implements DynoQueue {
+
+ private final Logger logger = LoggerFactory.getLogger(RedisPipelineQueue.class);
+
+ private final Clock clock;
+
+ private final String queueName;
+
+ private final String shardName;
+
+ private final String messageStoreKeyPrefix;
+
+ private final String myQueueShard;
+
+ private final String unackShardKeyPrefix;
+
+ private final int unackTime;
+
+ private final QueueMonitor monitor;
+
+ private final ObjectMapper om;
+
+ private final RedisConnection connPool;
+
+ private volatile RedisConnection nonQuorumPool;
+
+ private final ScheduledExecutorService schedulerForUnacksProcessing;
+
+ private final HashPartitioner partitioner = new Murmur3HashPartitioner();
+
+ private final int maxHashBuckets = 32;
+
+ private final int longPollWaitIntervalInMillis = 10;
+
+ public RedisPipelineQueue(String redisKeyPrefix, String queueName, String shardName, int unackScheduleInMS, int unackTime, RedisConnection pool) {
+ this(Clock.systemDefaultZone(), redisKeyPrefix, queueName, shardName, unackScheduleInMS, unackTime, pool);
+ }
+
+ public RedisPipelineQueue(Clock clock, String redisKeyPrefix, String queue, String shardName, int unackScheduleInMS, int unackTime, RedisConnection pool) {
+ this.clock = clock;
+ this.queueName = queue;
+ String qName = "{" + queue + "." + shardName + "}";
+ this.shardName = shardName;
+
+ this.messageStoreKeyPrefix = redisKeyPrefix + ".MSG." + qName;
+ this.myQueueShard = redisKeyPrefix + ".QUEUE." + qName;
+ this.unackShardKeyPrefix = redisKeyPrefix + ".UNACK." + qName + ".";
+ this.unackTime = unackTime;
+ this.connPool = pool;
+ this.nonQuorumPool = pool;
+
+ this.om = QueueUtils.constructObjectMapper();
+ this.monitor = new QueueMonitor(qName, shardName);
+
+ schedulerForUnacksProcessing = Executors.newScheduledThreadPool(1);
+
+ schedulerForUnacksProcessing.scheduleAtFixedRate(() -> processUnacks(), unackScheduleInMS, unackScheduleInMS, TimeUnit.MILLISECONDS);
+
+ logger.info(RedisPipelineQueue.class.getName() + " is ready to serve " + qName + ", shard=" + shardName);
+
+ }
+
+ /**
+ * @param nonQuorumPool When using a cluster like Dynomite, which relies on the quorum reads, supply a separate non-quorum read connection for ops like size etc.
+ */
+ public void setNonQuorumPool(RedisConnection nonQuorumPool) {
+ this.nonQuorumPool = nonQuorumPool;
+ }
+
+ @Override
+ public String getName() {
+ return queueName;
+ }
+
+ @Override
+ public int getUnackTime() {
+ return unackTime;
+ }
+
+ @Override
+ public List push(final List messages) {
+
+ Stopwatch sw = monitor.start(monitor.push, messages.size());
+ RedisConnection conn = connPool.getResource();
+ try {
+
+ Pipe pipe = conn.pipelined();
+
+ for (Message message : messages) {
+ String json = om.writeValueAsString(message);
+ pipe.hset(messageStoreKey(message.getId()), message.getId(), json);
+ double priority = message.getPriority() / 100.0;
+ double score = Long.valueOf(clock.millis() + message.getTimeout()).doubleValue() + priority;
+ pipe.zadd(myQueueShard, score, message.getId());
+ }
+ pipe.sync();
+ pipe.close();
+
+ return messages.stream().map(msg -> msg.getId()).collect(Collectors.toList());
+
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ } finally {
+ conn.close();
+ sw.stop();
+ }
+ }
+
+ private String messageStoreKey(String msgId) {
+ Long hash = partitioner.hash(msgId);
+ long bucket = hash % maxHashBuckets;
+ return messageStoreKeyPrefix + "." + bucket;
+ }
+
+ private String unackShardKey(String messageId) {
+ Long hash = partitioner.hash(messageId);
+ long bucket = hash % maxHashBuckets;
+ return unackShardKeyPrefix + bucket;
+ }
+
+ @Override
+ public List peek(final int messageCount) {
+
+ Stopwatch sw = monitor.peek.start();
+ RedisConnection jedis = connPool.getResource();
+
+ try {
+
+ Set ids = peekIds(0, messageCount);
+ if (ids == null) {
+ return Collections.emptyList();
+ }
+
+ List messages = new LinkedList();
+ for (String id : ids) {
+ String json = jedis.hget(messageStoreKey(id), id);
+ Message message = om.readValue(json, Message.class);
+ messages.add(message);
+ }
+ return messages;
+
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ } finally {
+ jedis.close();
+ sw.stop();
+ }
+ }
+
+ @Override
+ public synchronized List pop(int messageCount, int wait, TimeUnit unit) {
+
+ if (messageCount < 1) {
+ return Collections.emptyList();
+ }
+
+ Stopwatch sw = monitor.start(monitor.pop, messageCount);
+ List messages = new LinkedList<>();
+ int remaining = messageCount;
+ long time = clock.millis() + unit.toMillis(wait);
+
+ try {
+
+ do {
+
+ List peeked = peekIds(0, remaining).stream().collect(Collectors.toList());
+ List popped = _pop(peeked);
+ int poppedCount = popped.size();
+ if (poppedCount == messageCount) {
+ messages = popped;
+ break;
+ }
+ messages.addAll(popped);
+ remaining -= poppedCount;
+ if (clock.millis() > time) {
+ break;
+ }
+
+ try {
+ Thread.sleep(longPollWaitIntervalInMillis);
+ } catch (InterruptedException ie) {
+ logger.error(ie.getMessage(), ie);
+ }
+
+ } while (remaining > 0);
+
+ return messages;
+
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ } finally {
+ sw.stop();
+ }
+
+ }
+
+ @Override
+ public Message popWithMsgId(String messageId) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public Message unsafePopWithMsgIdAllShards(String messageId) {
+ throw new UnsupportedOperationException();
+ }
+
+ private List _pop(List batch) throws Exception {
+
+ double unackScore = Long.valueOf(clock.millis() + unackTime).doubleValue();
+
+ List popped = new LinkedList<>();
+ ZAddParams zParams = ZAddParams.zAddParams().nx();
+
+ RedisConnection jedis = connPool.getResource();
+ try {
+
+ Pipe pipe = jedis.pipelined();
+ List> zadds = new ArrayList<>(batch.size());
+
+ for (int i = 0; i < batch.size(); i++) {
+ String msgId = batch.get(i);
+ if (msgId == null) {
+ break;
+ }
+ zadds.add(pipe.zadd(unackShardKey(msgId), unackScore, msgId, zParams));
+ }
+ pipe.sync();
+
+ pipe = jedis.pipelined();
+ int count = zadds.size();
+ List zremIds = new ArrayList<>(count);
+ List> zremRes = new LinkedList<>();
+
+ for (int i = 0; i < count; i++) {
+ long added = zadds.get(i).get();
+ if (added == 0) {
+ if (logger.isDebugEnabled()) {
+ logger.debug("Cannot add {} to unack queue shard", batch.get(i));
+ }
+ monitor.misses.increment();
+ continue;
+ }
+ String id = batch.get(i);
+ zremIds.add(id);
+ zremRes.add(pipe.zrem(myQueueShard, id));
+ }
+ pipe.sync();
+
+ pipe = jedis.pipelined();
+ List