Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

PMM-12551 Clickhouse v2. #2920

Closed
wants to merge 4 commits into from
Closed

PMM-12551 Clickhouse v2. #2920

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
3f417f2
PMM-12551 Clickhouse v2.
JiriCtvrtka Mar 21, 2024
05c74f6
Merge branch 'main' into PMM-12551-clickhouse-v2
JiriCtvrtka Mar 21, 2024
2d55e0b
PMM-12551 Fix for new connection string format.
JiriCtvrtka Mar 21, 2024
c7da376
PMM-12551 Another fix for new connection string.
JiriCtvrtka Mar 21, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 0 additions & 6 deletions go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,11 +13,8 @@ replace gopkg.in/alecthomas/kingpin.v2 => github.com/Percona-Lab/kingpin v2.2.6-

replace golang.org/x/crypto => github.com/percona-lab/crypto v0.0.0-20231108144114-756dfb24eaf2

replace github.com/ClickHouse/clickhouse-go/151 => github.com/ClickHouse/clickhouse-go v1.5.1 // clickhouse-go/v2 cannot work with 1.5.1 which we need for QAN-API

require (
github.com/AlekSi/pointer v1.2.0
github.com/ClickHouse/clickhouse-go/151 v0.0.0-00010101000000-000000000000
github.com/ClickHouse/clickhouse-go/v2 v2.22.2
github.com/DATA-DOG/go-sqlmock v1.5.0
github.com/alecthomas/kong v0.9.0
Expand Down Expand Up @@ -134,7 +131,6 @@ require (
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
github.com/mwitkow/go-proto-validators v0.3.2 // indirect
github.com/percona/percona-backup-mongodb v1.8.1 // indirect
github.com/pierrec/lz4 v2.6.1+incompatible // indirect
github.com/posener/complete v1.2.3 // indirect
github.com/riywo/loginshell v0.0.0-20200815045211-7d26008be1ab // indirect
github.com/sergi/go-diff v1.2.0 // indirect
Expand Down Expand Up @@ -165,7 +161,6 @@ require (
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resourcegraph/armresourcegraph v0.9.0
github.com/AzureAD/microsoft-authentication-library-for-go v1.2.1 // indirect
github.com/ClickHouse/ch-go v0.61.5 // indirect
github.com/ClickHouse/clickhouse-go v1.5.4 // indirect
github.com/HdrHistogram/hdrhistogram-go v1.1.2
github.com/Microsoft/go-winio v0.6.1 // indirect
github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 // indirect
Expand All @@ -175,7 +170,6 @@ require (
github.com/cenkalti/backoff/v4 v4.2.1 // indirect
github.com/cespare/xxhash/v2 v2.2.0 // indirect
github.com/charmbracelet/harmonica v0.2.0 // indirect
github.com/cloudflare/golz4 v0.0.0-20150217214814-ef862a3cdc58 // indirect
github.com/containerd/console v1.0.4-0.20230313162750-1ae8d489ac81 // indirect
github.com/docker/go-units v0.5.0 // indirect
github.com/dustin/go-humanize v1.0.1 // indirect
Expand Down
11 changes: 0 additions & 11 deletions go.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -70,8 +70,6 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03
github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
github.com/ClickHouse/ch-go v0.61.5 h1:zwR8QbYI0tsMiEcze/uIMK+Tz1D3XZXLdNrlaOpeEI4=
github.com/ClickHouse/ch-go v0.61.5/go.mod h1:s1LJW/F/LcFs5HJnuogFMta50kKDO0lf9zzfrbl0RQg=
github.com/ClickHouse/clickhouse-go v1.5.1 h1:I8zVFZTz80crCs0FFEBJooIxsPcV0xfthzK1YrkpJTc=
github.com/ClickHouse/clickhouse-go v1.5.1/go.mod h1:EaI/sW7Azgz9UATzd5ZdZHRUhHgv5+JMS9NSr2smCJI=
github.com/ClickHouse/clickhouse-go v1.5.4 h1:cKjXeYLNWVJIx2J1K6H2CqyRmfwVJVY1OV1coaaFcI0=
github.com/ClickHouse/clickhouse-go v1.5.4/go.mod h1:EaI/sW7Azgz9UATzd5ZdZHRUhHgv5+JMS9NSr2smCJI=
github.com/ClickHouse/clickhouse-go/v2 v2.22.2 h1:T1BljsIjj+3aQog80jKMTeF4EqAUG4P6TVcCvmakYAc=
Expand Down Expand Up @@ -130,8 +128,6 @@ github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24
github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
github.com/bkaradzic/go-lz4 v1.0.0 h1:RXc4wYsyz985CkXXeX04y4VnZFGG8Rd43pRaHsOXAKk=
github.com/bkaradzic/go-lz4 v1.0.0/go.mod h1:0YdlkowM3VswSROI7qDxhRvJ3sLhlFrRRwjwegp5jy4=
github.com/blang/semver v3.5.1+incompatible h1:cQNTCjp13qL8KC3Nbxr/y2Bqb63oX6wdnnjpJbkM4JQ=
github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk=
github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM=
Expand Down Expand Up @@ -272,7 +268,6 @@ github.com/go-openapi/swag v0.23.0 h1:vsEVJDUo2hPJ2tu0/Xc+4noaxyEffXNIs3cOULZ+Gr
github.com/go-openapi/swag v0.23.0/go.mod h1:esZ8ITTYEsH1V2trKHjAN8Ai7xHb8RV+YSZ577vPjgQ=
github.com/go-openapi/validate v0.24.0 h1:LdfDKwNbpB6Vn40xhTdNZAnfLECL81w+VX3BumrGD58=
github.com/go-openapi/validate v0.24.0/go.mod h1:iyeX1sEufmv3nPbBdX3ieNviWnOZaJ1+zquzJEf2BAQ=
github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
github.com/go-sql-driver/mysql v1.7.1 h1:lUIinVbN1DY0xBg0eMOzmmtGoHwWBbvnWubQUrtU8EI=
Expand Down Expand Up @@ -465,7 +460,6 @@ github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9Y
github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8=
github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
github.com/jmoiron/sqlx v1.2.0/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks=
github.com/jmoiron/sqlx v1.3.5 h1:vFFPA71p1o5gAeqtEAwLU4dnX2napprKtHr7PYIcN3g=
github.com/jmoiron/sqlx v1.3.5/go.mod h1:nRVWtLre0KfCLJvgxzCsLVMogSvQ1zNJtpYr2Ccp0mQ=
github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg=
Expand Down Expand Up @@ -519,7 +513,6 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
github.com/lib/pq v1.8.0/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw=
Expand Down Expand Up @@ -551,7 +544,6 @@ github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzp
github.com/mattn/go-runewidth v0.0.12/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk=
github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZgg3U=
github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
github.com/mattn/go-sqlite3 v1.9.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
github.com/mattn/go-sqlite3 v1.14.0/go.mod h1:JIl7NbARA7phWnGvh0LKTyg7S9BA+6gx71ShQilpsus=
github.com/mattn/go-sqlite3 v1.14.6/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU=
github.com/mattn/go-sqlite3 v1.14.16 h1:yOQRA0RpS5PFz/oikGwBEqvAWhWg5ufRz4ETLjwpU1Y=
Expand Down Expand Up @@ -658,10 +650,7 @@ github.com/percona/promconfig v0.2.5 h1:f/HN/CbECQs7d9RIB6MKVkuXstsrsqEDxRvf6yig
github.com/percona/promconfig v0.2.5/go.mod h1:Y2uXi5QNk71+ceJHuI9poank+0S1kjxd3K105fXKVkg=
github.com/pganalyze/pg_query_go/v2 v2.2.0 h1:OW+reH+ZY7jdEuPyuLGlf1m7dLbE+fDudKXhLs0Ttpk=
github.com/pganalyze/pg_query_go/v2 v2.2.0/go.mod h1:XAxmVqz1tEGqizcQ3YSdN90vCOHBWjJi8URL1er5+cA=
github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY=
github.com/pierrec/lz4 v2.3.0+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY=
github.com/pierrec/lz4 v2.6.1+incompatible h1:9UY3+iC23yxF0UfGaYrGplQ+79Rg+h/q9FV9ix19jjM=
github.com/pierrec/lz4 v2.6.1+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY=
github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ=
github.com/pierrec/lz4/v4 v4.1.21/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ=
Expand Down
2 changes: 1 addition & 1 deletion managed/services/telemetry/telemetry_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,7 @@ func TestRunTelemetryService(t *testing.T) {
if ok {
pgHostPort = pgHostPortFromEnv
}
qanDSN := "tcp://localhost:9000?database=pmm"
qanDSN := "tcp://localhost:9000/pmm"
qanDSNFromEnv, ok := os.LookupEnv(envQanDSN)
if ok {
qanDSN = qanDSNFromEnv
Expand Down
9 changes: 3 additions & 6 deletions qan-api2/db.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ import (
"net/url"
"strings"

clickhouse "github.com/ClickHouse/clickhouse-go/151" // register database/sql driver
clickhouse "github.com/ClickHouse/clickhouse-go/v2" // register database/sql driver
"github.com/golang-migrate/migrate/v4"
_ "github.com/golang-migrate/migrate/v4/database/clickhouse" // register golang-migrate driver
"github.com/golang-migrate/migrate/v4/source/iofs"
Expand Down Expand Up @@ -78,11 +78,8 @@ func createDB(dsn string) error {
if err != nil {
return err
}
q := clickhouseURL.Query()
databaseName := q.Get("database")
q.Set("database", "default")

clickhouseURL.RawQuery = q.Encode()
databaseName := strings.Replace(clickhouseURL.Path, "/", "", 1)
clickhouseURL.Path = "/default"

defaultDB, err := sqlx.Connect("clickhouse", clickhouseURL.String())
if err != nil {
Expand Down
12 changes: 6 additions & 6 deletions qan-api2/db_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ import (
"testing"
"time"

_ "github.com/ClickHouse/clickhouse-go/151" // register database/sql driver
_ "github.com/ClickHouse/clickhouse-go/v2" // register database/sql driver
_ "github.com/golang-migrate/migrate/v4/database/clickhouse"
"github.com/jmoiron/sqlx"
"github.com/stretchr/testify/assert"
Expand All @@ -39,9 +39,9 @@ func setup() *sqlx.DB {
}

dsn, ok := os.LookupEnv("QANAPI_DSN_TEST")
dsn = strings.Replace(dsn, "?database=pmm_test", "?database=pmm_test_parts", 1)
dsn = strings.Replace(dsn, "/pmm_test", "/pmm_test_parts", 1)
if !ok {
dsn = "clickhouse://127.0.0.1:19000?database=pmm_test_parts"
dsn = "clickhouse://127.0.0.1:19000/pmm_test_parts"
}
db, err := sqlx.Connect("clickhouse", dsn)
if err != nil {
Expand Down Expand Up @@ -113,13 +113,13 @@ func TestCreateDbIfNotExists(t *testing.T) {
t.Run("connect to db that doesnt exist", func(t *testing.T) {
dsn, ok := os.LookupEnv("QANAPI_DSN_TEST")

dsn = strings.Replace(dsn, "?database=pmm_test", "?database=pmm_created_db", 1)
dsn = strings.Replace(dsn, "/pmm_test", "/pmm_created_db", 1)
if !ok {
dsn = "clickhouse://127.0.0.1:19000?database=pmm_created_db"
dsn = "clickhouse://127.0.0.1:19000/pmm_created_db"
}

db := createDB(dsn)

require.Equal(t, db, nil, "Check connection after we create database")
require.Equal(t, nil, db, "Check connection after we create database")
})
}
2 changes: 1 addition & 1 deletion qan-api2/main.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -61,7 +61,7 @@ import (

const (
shutdownTimeout = 3 * time.Second
defaultDsnF = "clickhouse://%s?database=%s&block_size=%s&pool_size=%s"
defaultDsnF = "clickhouse://%s/%s?block_size=%s&pool_size=%s"
maxIdleConns = 5
maxOpenConns = 10
)
Expand Down
4 changes: 2 additions & 2 deletions qan-api2/services/analytics/filters_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ import (
"testing"
"time"

_ "github.com/ClickHouse/clickhouse-go/151" // register database/sql driver
_ "github.com/ClickHouse/clickhouse-go/v2" // register database/sql driver
// TODO replace with 'google.golang.org/protobuf/encoding/protojson' since this one is deprecated.
"github.com/golang/protobuf/ptypes/timestamp"
"github.com/jmoiron/sqlx"
Expand Down Expand Up @@ -54,7 +54,7 @@ type testValuesUnmarshal struct {
func TestService_GetFilters(t *testing.T) {
dsn, ok := os.LookupEnv("QANAPI_DSN_TEST")
if !ok {
dsn = "clickhouse://127.0.0.1:19000?database=pmm_test"
dsn = "clickhouse://127.0.0.1:19000/pmm_test"
}
db, err := sqlx.Connect("clickhouse", dsn)
if err != nil {
Expand Down
2 changes: 1 addition & 1 deletion qan-api2/services/analytics/profile_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@ import (
func setup() *sqlx.DB {
dsn, ok := os.LookupEnv("QANAPI_DSN_TEST")
if !ok {
dsn = "clickhouse://127.0.0.1:19000?database=pmm_test"
dsn = "clickhouse://127.0.0.1:19000/pmm_test"
}
db, err := sqlx.Connect("clickhouse", dsn)
if err != nil {
Expand Down
Loading