From d8aedcb3753b404e621983f195023c7c1cae4870 Mon Sep 17 00:00:00 2001 From: Jose Bolina Date: Sun, 7 Jan 2024 18:51:01 -0300 Subject: [PATCH] Adding replication benchmark This is not utilizing maven submodules, that would require a lot of work. This commits add a new subproject in the `tests` folder to run benchmarks written with JMH. The benchmark we added here is not "micro". It uses `RaftHandle` object to replicate data with Raft, pretty much the way a user would. We vary the cluster and data size in the benchmark. Manually, we can tune the RAFT instance, changing the log implementation, fsync, etc. --- .../protocols/raft/election/BaseElection.java | 3 +- tests/benchmark/README.md | 39 +++++ tests/benchmark/pom.xml | 132 +++++++++++++++ .../raft/DataReplicationBenchmark.java | 159 ++++++++++++++++++ 4 files changed, 332 insertions(+), 1 deletion(-) create mode 100644 tests/benchmark/README.md create mode 100644 tests/benchmark/pom.xml create mode 100644 tests/benchmark/src/main/java/org/jgroups/raft/DataReplicationBenchmark.java diff --git a/src/org/jgroups/protocols/raft/election/BaseElection.java b/src/org/jgroups/protocols/raft/election/BaseElection.java index 30dbedcf..0efaceff 100644 --- a/src/org/jgroups/protocols/raft/election/BaseElection.java +++ b/src/org/jgroups/protocols/raft/election/BaseElection.java @@ -109,7 +109,8 @@ public void init() throws Exception { public void stop() { stopVotingThread(); - raft.setLeaderAndTerm(null); + if (raft != null) + raft.setLeaderAndTerm(null); } public Object down(Event evt) { diff --git a/tests/benchmark/README.md b/tests/benchmark/README.md new file mode 100644 index 00000000..d34371ba --- /dev/null +++ b/tests/benchmark/README.md @@ -0,0 +1,39 @@ +# JGroups Raft benchmarks + +This subproject includes all the benchmarks created utilizing JMH. + +## How to + +This specific repository uses the latest Java version available. Currently, Java 21. +To compile the project and runs the benchmarks. +First, execute: + +```bash +$ mvn clean verify +``` + +The output is located at `./target/benchmarks.jar`. +Now, to execute a benchmark, check the current implementations and select the name. +For example, for `MyBenchmark`: + +```bash +$ java -jar target/benchmarks.jar "MyBenchmark" +``` + +To pass configuration for `MyBenchmark`, execute: + +```bash +$ java -jar target/benchmarks.jar -pkey1=value1 -pkey2=value2 "MyBenchmark" +``` + +For more options, related to JMH: + +```bash +$ java -jar target/benchmarks.jar -h +``` + +## Current benchmarks + +The current list of benchmarks include: + +1. `DataReplicationBenchmark`: Benchmark the complete data replication utilizing `RaftHandle`; diff --git a/tests/benchmark/pom.xml b/tests/benchmark/pom.xml new file mode 100644 index 00000000..4d4ded22 --- /dev/null +++ b/tests/benchmark/pom.xml @@ -0,0 +1,132 @@ + + 4.0.0 + + org.jgroups.raft + benchmark + 1.0.13.Final-SNAPSHOT + jar + + JMH benchmark sample: Java + + + UTF-8 + 21 + 21 + + 1.37 + benchmarks + + + + + org.openjdk.jmh + jmh-core + ${jmh.version} + + + org.openjdk.jmh + jmh-generator-annprocess + ${jmh.version} + provided + + + org.jgroups + jgroups-raft + ${project.version} + compile + + + + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.8.0 + + ${javac.target} + ${javac.target} + ${javac.target} + + + + org.apache.maven.plugins + maven-shade-plugin + 3.2.1 + + + package + + shade + + + ${uberjar.name} + + + org.openjdk.jmh.Main + + + + + + + *:* + + META-INF/*.SF + META-INF/*.DSA + META-INF/*.RSA + + + + + + + + + + + + maven-clean-plugin + 2.5 + + + maven-deploy-plugin + 2.8.1 + + + maven-install-plugin + 2.5.1 + + + maven-jar-plugin + 2.4 + + + maven-javadoc-plugin + 2.9.1 + + + maven-resources-plugin + 2.6 + + + maven-site-plugin + 3.3 + + + maven-source-plugin + 2.2.1 + + + maven-surefire-plugin + 2.17 + + + + + + diff --git a/tests/benchmark/src/main/java/org/jgroups/raft/DataReplicationBenchmark.java b/tests/benchmark/src/main/java/org/jgroups/raft/DataReplicationBenchmark.java new file mode 100644 index 00000000..2afa1186 --- /dev/null +++ b/tests/benchmark/src/main/java/org/jgroups/raft/DataReplicationBenchmark.java @@ -0,0 +1,159 @@ +package org.jgroups.raft; + +import org.jgroups.JChannel; +import org.jgroups.protocols.raft.FileBasedLog; +import org.jgroups.protocols.raft.RAFT; +import org.jgroups.raft.testfwk.RaftTestUtils; +import org.jgroups.util.Util; + +import java.io.DataInput; +import java.io.DataOutput; +import java.util.Arrays; +import java.util.List; +import java.util.concurrent.ThreadLocalRandom; +import java.util.concurrent.TimeUnit; +import java.util.function.BooleanSupplier; +import java.util.function.Supplier; +import java.util.stream.Collectors; +import java.util.stream.IntStream; + +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.Param; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.TearDown; +import org.openjdk.jmh.annotations.Warmup; + +/** + * Benchmark the replication throughput. + * + * @author José Bolina + * @since 1.0.13 + */ +@BenchmarkMode({Mode.Throughput}) +@Warmup(iterations = 10, time = 5) +@Measurement(iterations = 5, time = 10) +@Fork(value = 3, jvmArgsPrepend = "-Djava.net.preferIPv4Stack=true -Djgroups.udp.ip_ttl=0") +public class DataReplicationBenchmark { + + @Benchmark + public byte[] testReplication(ClusterState state) throws Exception { + return state.leader.set(state.data, 0, state.dataSize); + } + + @State(Scope.Benchmark) + public static class ClusterState { + + // Change the majority between values. + @Param({"3", "5"}) + public int clusterSize; + + // Keep a factor of 4 between values. + @Param({"256", "1024", "4096"}) + public int dataSize; + + // Storage implementations that write to disk can enable/disable fsync. + @Param({"false", "true"}) + public boolean useFsync; + + public byte[] data; + + public RaftHandle[] members; + + public RaftHandle leader; + + + @Setup + public void initialize() throws Exception { + List memberList = IntStream.range(0, clusterSize) + .mapToObj(i -> Character.toString('A' + i)) + .collect(Collectors.toList()); + + members = new RaftHandle[clusterSize]; + for (int i = 0; i < clusterSize; i++) { + String name = Character.toString('A' + i); + + // Utilize the default configuration shipped with jgroups-raft. + JChannel ch = new JChannel("raft.xml"); + ch.name(name); + + // A no-op state machine. + RaftHandle handler = new RaftHandle(ch, new EmptyStateMachine(dataSize)); + RAFT raft = handler.raft(); + + // Default configuration. + raft.raftId(name); + raft.members(memberList); + + // Fine-tune the RAFT protocol below. + raft.logClass(FileBasedLog.class.getCanonicalName()); + raft.logUseFsync(useFsync); + + members[i] = handler; + ch.connect("jmh-replication"); + } + + data = new byte[dataSize]; + ThreadLocalRandom.current().nextBytes(data); + + // Block until ALL members have a leader installed. + BooleanSupplier bs = () -> Arrays.stream(members) + .map(RaftHandle::raft) + .allMatch(r -> r.leader() != null); + Supplier message = () -> Arrays.stream(members) + .map(RaftHandle::raft) + .map(r -> String.format("%s: %s", r.raftId(), r.leader())) + .collect(Collectors.joining(System.lineSeparator())); + assert RaftTestUtils.eventually(bs, 1, TimeUnit.MINUTES) : message.get(); + + for (RaftHandle member : members) { + if (member.isLeader()) { + leader = member; + break; + } + } + + assert leader != null : "Leader not found"; + } + + @TearDown + public void tearDown() throws Exception { + for (int i = clusterSize - 1; i >= 0; i--) { + RaftHandle rh = members[i]; + Util.close(rh.channel()); + } + + for (RaftHandle member : members) { + RaftTestUtils.deleteRaftLog(member.raft()); + } + } + } + + private static final class EmptyStateMachine implements StateMachine { + private static final byte[] RESPONSE = new byte[0]; + + private final int dataSize; + + public EmptyStateMachine(int dataSize) { + this.dataSize = dataSize; + } + + @Override + public byte[] apply(byte[] data, int offset, int length, boolean serialize_response) throws Exception { + if (data.length != dataSize) throw new IllegalArgumentException("Data size does not match"); + + return RESPONSE; + } + + @Override + public void readContentFrom(DataInput in) throws Exception { } + + @Override + public void writeContentTo(DataOutput out) throws Exception { } + } +}