Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[CLEANUP]: Update clickhouse libs & actualise benchmarks #443

Merged
merged 1 commit into from
Aug 26, 2023
Merged

[CLEANUP]: Update clickhouse libs & actualise benchmarks #443

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions clickhouse-integration/clickhouse-integration-spark/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -61,9 +61,9 @@
<scope>test</scope>
</dependency>
<dependency>
<groupId>ru.yandex.clickhouse</groupId>
<groupId>com.clickhouse</groupId>
<artifactId>clickhouse-jdbc</artifactId>
<classifier>shaded</classifier>
<classifier>all</classifier>
<scope>test</scope>
</dependency>
<dependency>
Expand Down
4 changes: 2 additions & 2 deletions clickhouse-native-jdbc/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -57,9 +57,9 @@
<scope>test</scope>
</dependency>
<dependency>
<groupId>ru.yandex.clickhouse</groupId>
<groupId>com.clickhouse</groupId>
<artifactId>clickhouse-jdbc</artifactId>
<classifier>shaded</classifier>
<classifier>all</classifier>
<scope>test</scope>
</dependency>
<dependency>
Expand Down
17 changes: 8 additions & 9 deletions clickhouse-native-jdbc/src/test/java/com/github/housepower/jdbc/AbstractITest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@
import com.github.housepower.misc.StrUtil;
import com.github.housepower.misc.SystemUtil;
import org.junit.jupiter.api.BeforeAll;
import org.testcontainers.containers.ClickHouseContainer;
import org.testcontainers.clickhouse.ClickHouseContainer;
import org.testcontainers.junit.jupiter.Container;
import org.testcontainers.junit.jupiter.Testcontainers;
import org.testcontainers.utility.MountableFile;
Expand All @@ -35,33 +35,32 @@ public abstract class AbstractITest implements Serializable {
protected static final ZoneId SERVER_TZ = ZoneId.of("UTC");
protected static final String DRIVER_CLASS_NAME = "com.github.housepower.jdbc.ClickHouseDriver";

public static final String CLICKHOUSE_IMAGE = System.getProperty("CLICKHOUSE_IMAGE", "yandex/clickhouse-server:21.9");
public static final String CLICKHOUSE_IMAGE = System.getProperty("CLICKHOUSE_IMAGE", "clickhouse/clickhouse-server:21.9");

protected static final String CLICKHOUSE_USER = SystemUtil.loadProp("CLICKHOUSE_USER", "default");
protected static final String CLICKHOUSE_PASSWORD = SystemUtil.loadProp("CLICKHOUSE_PASSWORD", "");
protected static final String CLICKHOUSE_DB = SystemUtil.loadProp("CLICKHOUSE_DB", "");

protected static final int CLICKHOUSE_GRPC_PORT = 9100;
protected static final int CLICKHOUSE_HTTP_PORT = 8123;
protected static final int CLICKHOUSE_NATIVE_PORT = 9000;

@Container
public static ClickHouseContainer container = (ClickHouseContainer) new ClickHouseContainer(CLICKHOUSE_IMAGE)
public static ClickHouseContainer container = new ClickHouseContainer(CLICKHOUSE_IMAGE)
.withEnv("CLICKHOUSE_USER", CLICKHOUSE_USER)
.withEnv("CLICKHOUSE_PASSWORD", CLICKHOUSE_PASSWORD)
.withEnv("CLICKHOUSE_DB", CLICKHOUSE_DB)
.withExposedPorts(CLICKHOUSE_GRPC_PORT)
.withExposedPorts(CLICKHOUSE_HTTP_PORT, CLICKHOUSE_NATIVE_PORT, CLICKHOUSE_GRPC_PORT)
.withCopyFileToContainer(MountableFile.forClasspathResource("grpc_config.xml"), "/etc/clickhouse-server/config.d/grpc_config.xml");


protected static String CK_HOST;
protected static String CK_IP;
protected static int CK_PORT;
protected static int CK_GRPC_PORT;

@BeforeAll
public static void extractContainerInfo() {
CK_HOST = container.getHost();
CK_IP = container.getContainerIpAddress();
CK_PORT = container.getMappedPort(ClickHouseContainer.NATIVE_PORT);
CK_PORT = container.getMappedPort(CLICKHOUSE_NATIVE_PORT);
CK_GRPC_PORT = container.getMappedPort(CLICKHOUSE_GRPC_PORT);
}

Expand All @@ -74,7 +73,7 @@ protected String getJdbcUrl() {

protected String getJdbcUrl(Object... params) {
StringBuilder sb = new StringBuilder();
int port = container.getMappedPort(ClickHouseContainer.NATIVE_PORT);
int port = container.getMappedPort(CLICKHOUSE_NATIVE_PORT);
sb.append("jdbc:clickhouse://").append(container.getHost()).append(":").append(port);
if (StrUtil.isNotEmpty(CLICKHOUSE_DB)) {
sb.append("/").append(container.getDatabaseName());
Expand Down
12 changes: 6 additions & 6 deletions ...tive-jdbc/src/test/java/com/github/housepower/jdbc/BalancedClickhouseDataSourceITest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ public class BalancedClickhouseDataSourceITest extends AbstractITest {
@BeforeEach
public void reset() {
singleDs = new BalancedClickhouseDataSource(String.format(Locale.ROOT, "jdbc:clickhouse://%s:%s", CK_HOST, CK_PORT));
dualDs = new BalancedClickhouseDataSource(String.format(Locale.ROOT, "jdbc:clickhouse://%s:%s,%s:%s", CK_HOST, CK_PORT, CK_IP, CK_PORT));
dualDs = new BalancedClickhouseDataSource(String.format(Locale.ROOT, "jdbc:clickhouse://%s:%s,%s:%s", CK_HOST, CK_PORT, CK_HOST, CK_PORT));
}

@Test
Expand Down Expand Up @@ -121,7 +121,7 @@ public void testDisableConnection() {
@Test
public void testWorkWithEnabledUrl() throws Exception {
BalancedClickhouseDataSource halfDatasource = new BalancedClickhouseDataSource(
String.format(Locale.ROOT, "jdbc:clickhouse://%s:%s,%s:%s", "not.existed.url", CK_PORT, CK_IP, CK_PORT), new Properties());
String.format(Locale.ROOT, "jdbc:clickhouse://%s:%s,%s:%s", "not.existed.url", CK_PORT, CK_HOST, CK_PORT), new Properties());

halfDatasource.actualize();

Expand Down Expand Up @@ -173,20 +173,20 @@ public void testConstructWithProperties() {

// without connection parameters
BalancedClickhouseDataSource dataSource = new BalancedClickhouseDataSource(
String.format(Locale.ROOT, "jdbc:clickhouse://%s:%s,%s:%s/click", CK_HOST, CK_PORT, CK_IP, CK_PORT), properties);
String.format(Locale.ROOT, "jdbc:clickhouse://%s:%s,%s:%s/click", CK_HOST, CK_PORT, CK_HOST, CK_PORT), properties);
ClickHouseConfig cfg = dataSource.getCfg();
assertEquals(Duration.ofSeconds(6789), cfg.queryTimeout());
assertEquals("888888", cfg.password());
assertEquals("click", cfg.database());
assertEquals(2, dataSource.getAllClickhouseUrls().size());
assertEquals(String.format(Locale.ROOT, "jdbc:clickhouse://%s:%s/click", CK_HOST, CK_PORT),
dataSource.getAllClickhouseUrls().get(0));
assertEquals(String.format(Locale.ROOT, "jdbc:clickhouse://%s:%s/click", CK_IP, CK_PORT),
assertEquals(String.format(Locale.ROOT, "jdbc:clickhouse://%s:%s/click", CK_HOST, CK_PORT),
dataSource.getAllClickhouseUrls().get(1));

// with connection parameters
dataSource = new BalancedClickhouseDataSource(
String.format(Locale.ROOT, "jdbc:clickhouse://%s:%s,%s:%s/click?query_timeout=12345&user=readonly", CK_HOST, CK_PORT, CK_IP, CK_PORT), properties);
String.format(Locale.ROOT, "jdbc:clickhouse://%s:%s,%s:%s/click?query_timeout=12345&user=readonly", CK_HOST, CK_PORT, CK_HOST, CK_PORT), properties);
cfg = dataSource.getCfg();
assertEquals(Duration.ofSeconds(6789), cfg.queryTimeout());
assertEquals("readonly", cfg.user());
Expand All @@ -195,7 +195,7 @@ public void testConstructWithProperties() {
assertEquals(2, dataSource.getAllClickhouseUrls().size());
assertEquals(String.format(Locale.ROOT, "jdbc:clickhouse://%s:%s/click?query_timeout=12345&user=readonly", CK_HOST, CK_PORT),
dataSource.getAllClickhouseUrls().get(0));
assertEquals(String.format(Locale.ROOT, "jdbc:clickhouse://%s:%s/click?query_timeout=12345&user=readonly", CK_IP, CK_PORT),
assertEquals(String.format(Locale.ROOT, "jdbc:clickhouse://%s:%s/click?query_timeout=12345&user=readonly", CK_HOST, CK_PORT),
dataSource.getAllClickhouseUrls().get(1));
}
}
22 changes: 14 additions & 8 deletions ...se-native-jdbc/src/test/java/com/github/housepower/jdbc/benchmark/AbstractIBenchmark.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@
import org.openjdk.jmh.runner.RunnerException;
import org.openjdk.jmh.runner.options.Options;
import org.openjdk.jmh.runner.options.OptionsBuilder;
import org.testcontainers.containers.ClickHouseContainer;
import org.testcontainers.clickhouse.ClickHouseContainer;

import java.sql.*;
import java.util.Enumeration;
Expand All @@ -31,14 +31,20 @@
@State(Scope.Thread)
public class AbstractIBenchmark {

/*
* HTTP API Port for http requests. used by JDBC, ODBC and web interfaces.
* */
protected static final int CLICKHOUSE_HTTP_PORT = 8123;
protected static final int CLICKHOUSE_NATIVE_PORT = 9000;

public static final ClickHouseContainer container;

static {
container = new ClickHouseContainer(AbstractITest.CLICKHOUSE_IMAGE);
container.start();
}

private final Driver httpDriver = new ru.yandex.clickhouse.ClickHouseDriver();
private final Driver clickhouseJdbcDriver = new com.clickhouse.jdbc.ClickHouseDriver();
private final Driver nativeDriver = new com.github.housepower.jdbc.ClickHouseDriver();

public static void main(String[] args) throws RunnerException {
Expand All @@ -64,13 +70,13 @@ protected void withConnection(WithConnection withConnection, ConnectionType conn
case NATIVE:
Class.forName("com.github.housepower.jdbc.ClickHouseDriver");
DriverManager.registerDriver(nativeDriver);
port = container.getMappedPort(ClickHouseContainer.NATIVE_PORT);
port = container.getMappedPort(CLICKHOUSE_NATIVE_PORT);
break;

case HTTP:
Class.forName("ru.yandex.clickhouse.ClickHouseDriver");
DriverManager.registerDriver(httpDriver);
port = container.getMappedPort(ClickHouseContainer.HTTP_PORT);
case JDBC:
Class.forName("com.clickhouse.jdbc.ClickHouseDriver");
DriverManager.registerDriver(clickhouseJdbcDriver);
port = container.getMappedPort(CLICKHOUSE_HTTP_PORT);
break;

default:
Expand Down Expand Up @@ -111,6 +117,6 @@ public interface WithPreparedStatement {
}

public enum ConnectionType {
NATIVE, HTTP
NATIVE, JDBC
}
}
29 changes: 15 additions & 14 deletions .../benchmark/RowBinaryDoubleIBenchmark.java → ...ower/jdbc/benchmark/DoubleIBenchmark.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,12 +16,10 @@

import com.google.common.base.Strings;
import org.openjdk.jmh.annotations.Benchmark;
import ru.yandex.clickhouse.ClickHouseStatement;
import ru.yandex.clickhouse.domain.ClickHouseFormat;

import static org.junit.jupiter.api.Assertions.assertEquals;

public class RowBinaryDoubleIBenchmark extends AbstractInsertIBenchmark {
public class DoubleIBenchmark extends AbstractInsertIBenchmark {
private final String columnType = "Float64";

@Benchmark
Expand All @@ -46,20 +44,23 @@ public void benchInsertNative() throws Exception {
}

@Benchmark
public void benchInsertHttpRowBinary() throws Exception {
public void benchInsertJdbc() throws Exception {
withConnection(connection -> {
wideColumnPrepare(connection, columnType);
withStatement(connection, stmt -> {
ClickHouseStatement sth = (ClickHouseStatement) stmt;
sth.write().send("INSERT INTO " + getTableName(), stream -> {
for (int i = 0; i < batchSize; i++) {
for (int j = 0; j < columnNum; j++) {
stream.writeFloat64(j + 1.0);
String params = Strings.repeat("?, ", columnNum);
withPreparedStatement(connection,
"INSERT INTO " + getTableName() + " values(" + params.substring(0, params.length() - 2) + ")",
pstmt -> {
for (int i = 0; i < batchSize; i++) {
for (int j = 0; j < columnNum; j++) {
pstmt.setDouble(j + 1, j + 1.0);
}
pstmt.addBatch();
}
}
}, ClickHouseFormat.RowBinary);
});
int[] res = pstmt.executeBatch();
assertEquals(res.length, batchSize);
});
wideColumnAfter(connection);
}, ConnectionType.HTTP);
}, ConnectionType.JDBC);
}
}
4 changes: 2 additions & 2 deletions ...ouse-native-jdbc/src/test/java/com/github/housepower/jdbc/benchmark/InsertIBenchmark.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -61,7 +61,7 @@ public void benchInsertNative() throws Exception {
}

@Benchmark
public void benchInsertHttp() throws Exception {
withConnection(benchInsert, ConnectionType.HTTP);
public void benchInsertJdbc() throws Exception {
withConnection(benchInsert, ConnectionType.JDBC);
}
}
29 changes: 15 additions & 14 deletions ...dbc/benchmark/RowBinaryIntIBenchmark.java → ...sepower/jdbc/benchmark/IntIBenchmark.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,12 +16,10 @@

import com.google.common.base.Strings;
import org.openjdk.jmh.annotations.Benchmark;
import ru.yandex.clickhouse.ClickHouseStatement;
import ru.yandex.clickhouse.domain.ClickHouseFormat;

import static org.junit.jupiter.api.Assertions.assertEquals;

public class RowBinaryIntIBenchmark extends AbstractInsertIBenchmark {
public class IntIBenchmark extends AbstractInsertIBenchmark {
private final String columnType = "Int32";

@Benchmark
Expand All @@ -46,21 +44,24 @@ public void benchInsertNative() throws Exception {
}

@Benchmark
public void benchInsertHttpRowBinary() throws Exception {
public void benchInsertJdbc() throws Exception {
withConnection(connection -> {
wideColumnPrepare(connection, columnType);
withStatement(connection, stmt -> {
ClickHouseStatement sth = (ClickHouseStatement) stmt;
sth.write().send("INSERT INTO " + getTableName(), stream -> {
for (int i = 0; i < batchSize; i++) {
for (int j = 0; j < columnNum; j++) {
stream.writeInt32(j + 1);
String params = Strings.repeat("?, ", columnNum);
withPreparedStatement(connection,
"INSERT INTO " + getTableName() + " values(" + params.substring(0, params.length() - 2) + ")",
pstmt -> {
for (int i = 0; i < batchSize; i++) {
for (int j = 0; j < columnNum; j++) {
pstmt.setObject(j + 1, j + 1);
}
pstmt.addBatch();
}
}
}, ClickHouseFormat.RowBinary);
});
int[] res = pstmt.executeBatch();
assertEquals(res.length, batchSize);
});
wideColumnAfter(connection);
}, ConnectionType.HTTP);
}, ConnectionType.JDBC);
}

}
4 changes: 2 additions & 2 deletions ...ouse-native-jdbc/src/test/java/com/github/housepower/jdbc/benchmark/SelectIBenchmark.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,7 @@ public void benchSelectNative() throws Exception {
}

@Benchmark
public void benchSelectHTTP() throws Exception {
withConnection(benchSelect, ConnectionType.HTTP);
public void benchSelectJdbc() throws Exception {
withConnection(benchSelect, ConnectionType.JDBC);
}
}
29 changes: 15 additions & 14 deletions .../benchmark/RowBinaryStringIBenchmark.java → ...ower/jdbc/benchmark/StringIBenchmark.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,12 +16,10 @@

import com.google.common.base.Strings;
import org.openjdk.jmh.annotations.Benchmark;
import ru.yandex.clickhouse.ClickHouseStatement;
import ru.yandex.clickhouse.domain.ClickHouseFormat;

import static org.junit.jupiter.api.Assertions.assertEquals;

public class RowBinaryStringIBenchmark extends AbstractInsertIBenchmark {
public class StringIBenchmark extends AbstractInsertIBenchmark {
private final String columnType = "String";

@Benchmark
Expand All @@ -46,21 +44,24 @@ public void benchInsertNative() throws Exception {
}

@Benchmark
public void benchInsertHttpRowBinary() throws Exception {
public void benchInsertJdbc() throws Exception {
withConnection(connection -> {
wideColumnPrepare(connection, columnType);
withStatement(connection, stmt -> {
ClickHouseStatement sth = (ClickHouseStatement) stmt;
sth.write().send("INSERT INTO " + getTableName(), stream -> {
for (int i = 0; i < batchSize; i++) {
for (int j = 0; j < columnNum; j++) {
stream.writeString(j + 1 + "");
String params = Strings.repeat("?, ", columnNum);
withPreparedStatement(connection,
"INSERT INTO " + getTableName() + " values(" + params.substring(0, params.length() - 2) + ")",
pstmt -> {
for (int i = 0; i < batchSize; i++) {
for (int j = 0; j < columnNum; j++) {
pstmt.setObject(j + 1, j + 1 + "");
}
pstmt.addBatch();
}
}
}, ClickHouseFormat.RowBinary);
});
int[] res = pstmt.executeBatch();
assertEquals(res.length, batchSize);
});
wideColumnAfter(connection);
}, ConnectionType.HTTP);
}, ConnectionType.JDBC);
}

}
4 changes: 2 additions & 2 deletions .../src/test/java/com/github/housepower/jdbc/benchmark/WideColumnDoubleInsertIBenchmark.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,8 +27,8 @@ public void benchInsertNative() throws Exception {
}

@Benchmark
public void benchInsertHttp() throws Exception {
withConnection(benchInsert, ConnectionType.HTTP);
public void benchInsertJdbc() throws Exception {
withConnection(benchInsert, ConnectionType.JDBC);
}

public WithConnection benchInsert = connection -> {
Expand Down
4 changes: 2 additions & 2 deletions ...dbc/src/test/java/com/github/housepower/jdbc/benchmark/WideColumnIntInsertIBenchmark.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,8 +27,8 @@ public void benchInsertNative() throws Exception {
}

@Benchmark
public void benchInsertHttp() throws Exception {
withConnection(benchInsert, ConnectionType.HTTP);
public void benchInsertJdbc() throws Exception {
withConnection(benchInsert, ConnectionType.JDBC);
}

public WithConnection benchInsert = connection -> {
Expand Down
4 changes: 2 additions & 2 deletions .../src/test/java/com/github/housepower/jdbc/benchmark/WideColumnStringInsertIBenchmark.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,8 +27,8 @@ public void benchInsertNative() throws Exception {
}

@Benchmark
public void benchInsertHttp() throws Exception {
withConnection(benchInsert, ConnectionType.HTTP);
public void benchInsertJdbc() throws Exception {
withConnection(benchInsert, ConnectionType.JDBC);
}

public WithConnection benchInsert = connection -> {
Expand Down
Loading
Loading