diff --git a/benchmark-framework/pom.xml b/benchmark-framework/pom.xml index 5f976a95..e63b05fa 100644 --- a/benchmark-framework/pom.xml +++ b/benchmark-framework/pom.xml @@ -87,87 +87,9 @@ - ${project.groupId} - driver-pulsar - ${project.version} - - - - ${project.groupId} - driver-jms - ${project.version} - - - - ${project.groupId} - driver-kafka - ${project.version} - - - - ${project.groupId} - driver-redpanda - ${project.version} - - - - ${project.groupId} - driver-kop - ${project.version} - - - - ${project.groupId} - driver-pravega - ${project.version} - - - - ${project.groupId} - driver-rocketmq - ${project.version} - - - - ${project.groupId} - driver-rabbitmq - ${project.version} - - - - ${project.groupId} - driver-artemis - ${project.version} - - - - ${project.groupId} - driver-bookkeeper - ${project.version} - - - - ${project.groupId} - driver-nats - ${project.version} - - - - ${project.groupId} - driver-nats-streaming - ${project.version} - - - - ${project.groupId} - driver-nsq - ${project.version} - - - - ${project.groupId} - driver-redis - ${project.version} + org.apache.commons + commons-lang3 + 3.7 diff --git a/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/FutureUtil.java b/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/FutureUtil.java new file mode 100644 index 00000000..8bc821b5 --- /dev/null +++ b/benchmark-framework/src/main/java/io/openmessaging/benchmark/utils/FutureUtil.java @@ -0,0 +1,44 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.openmessaging.benchmark.utils; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.concurrent.CompletableFuture; +import java.util.stream.Stream; + +public class FutureUtil { + + /* + Originally from the Apache Pulsar project + https://github.com/apache/pulsar/blob/master/pulsar-common/src/main/java/org/apache/pulsar/common/util/FutureUtil.java + */ + public static CompletableFuture waitForAll(Collection> futures) { + return CompletableFuture.allOf(futures.toArray(new CompletableFuture[0])); + } + + /* + Originally from the Apache Pulsar project + https://github.com/apache/pulsar/blob/master/pulsar-common/src/main/java/org/apache/pulsar/common/util/FutureUtil.java + */ + public static CompletableFuture> waitForAll(Stream>> futures) { + return futures.reduce(CompletableFuture.completedFuture(new ArrayList<>()), + (pre, curr) -> pre.thenCompose(preV -> curr.thenApply(currV -> { + preV.addAll(currV); + return preV; + }))); + } +} \ No newline at end of file diff --git a/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/DistributedWorkersEnsemble.java b/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/DistributedWorkersEnsemble.java index 0919ad73..1a86e393 100644 --- a/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/DistributedWorkersEnsemble.java +++ b/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/DistributedWorkersEnsemble.java @@ -21,12 +21,12 @@ import com.google.common.collect.Lists; import io.netty.buffer.ByteBufUtil; import io.netty.buffer.Unpooled; +import io.openmessaging.benchmark.utils.FutureUtil; import io.openmessaging.benchmark.utils.ListPartition; import io.openmessaging.benchmark.worker.commands.*; import org.HdrHistogram.Histogram; import org.apache.commons.lang3.tuple.Pair; import org.apache.commons.lang3.tuple.Triple; -import org.apache.pulsar.common.util.FutureUtil; import org.asynchttpclient.AsyncHttpClient; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/SwarmWorker.java b/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/SwarmWorker.java index c06f188f..ee8cd7f2 100644 --- a/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/SwarmWorker.java +++ b/benchmark-framework/src/main/java/io/openmessaging/benchmark/worker/SwarmWorker.java @@ -34,7 +34,6 @@ import com.fasterxml.jackson.core.type.TypeReference; import org.HdrHistogram.Histogram; -import org.apache.pulsar.common.util.FutureUtil; import org.asynchttpclient.AsyncHttpClient; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -46,6 +45,7 @@ import io.netty.buffer.ByteBufUtil; import io.netty.buffer.Unpooled; +import io.openmessaging.benchmark.utils.FutureUtil; import io.openmessaging.benchmark.worker.commands.ConsumerAssignment; import io.openmessaging.benchmark.worker.commands.CountersStats; import io.openmessaging.benchmark.worker.commands.CumulativeLatencies; diff --git a/driver-redpanda/README.md b/driver-redpanda/README.md index b5c99d09..1443c11c 100644 --- a/driver-redpanda/README.md +++ b/driver-redpanda/README.md @@ -17,7 +17,7 @@ ## Setup -1. In the top level directory run `mvn clean install -Dlicense.skip=true`. This will build the benchmark client needed during deployment. +1. In the top level directory run `mvn clean install -Dlicense.skip=true -pl '.,benchmark-framework,driver-api,driver-redpanda,package'`. This will build the benchmark client needed during deployment. 2. Create an ssh key for the benchmark using the following: `ssh-keygen -f ~/.ssh/redpanda_aws`. Set the password to blank. diff --git a/package/src/assemble/bin.xml b/package/src/assemble/bin.xml index 585b5ecf..0eeb2ca7 100644 --- a/package/src/assemble/bin.xml +++ b/package/src/assemble/bin.xml @@ -23,6 +23,41 @@ true + + *:* + + + lib + false + + ${module.groupId}-${module.artifactId}-${module.version}${dashClassifier?}.${module.extension} + + + compile + false + + ${artifact.groupId}-${artifact.artifactId}-${artifact.version}${dashClassifier?}.${artifact.extension} + + + io.netty:netty-buffer + io.netty:netty-codec + io.netty:netty-codec-dns + io.netty:netty-codec-http + io.netty:netty-common + io.netty:netty-handler + io.netty:netty-transport + io.netty:netty-resolver + io.netty:netty-resolver-dns + io.netty:netty-transport-native-epoll + io.netty:netty-transport-native-kqueue + io.netty:netty-transport-native-unix-common + ch.qos.logback:* + + + + @@ -69,33 +104,4 @@ 644 - - - lib - false - compile - false - - ${artifact.groupId}-${artifact.artifactId}-${artifact.version}${dashClassifier?}.${artifact.extension} - - - - io.netty:netty-common - io.netty:netty-buffer - io.netty:netty-codec-http - io.netty:netty-codec - io.netty:netty-transport - io.netty:netty-handler - io.netty:netty-transport-native-epoll - io.netty:netty-transport-native-kqueue - io.netty:netty-codec-http - io.netty:netty-codec-dns - io.netty:netty-resolver - io.netty:netty-resolver-dns - io.netty:netty-transport-native-unix-common - ch.qos.logback:* - - - diff --git a/pom.xml b/pom.xml index 599f86e6..31e72c4a 100644 --- a/pom.xml +++ b/pom.xml @@ -38,10 +38,10 @@ driver-nsq driver-jms driver-redis + driver-kop package deployment docker - driver-kop