From 3f417f242bf662f6754ee836428d4abcc16ab3ca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20=C4=8Ctvrtka?= Date: Thu, 21 Mar 2024 12:03:43 +0100 Subject: [PATCH 1/3] PMM-12551 Clickhouse v2. --- go.mod | 6 ------ go.sum | 11 ----------- qan-api2/db.go | 2 +- qan-api2/db_test.go | 2 +- qan-api2/services/analytics/filters_test.go | 2 +- 5 files changed, 3 insertions(+), 20 deletions(-) diff --git a/go.mod b/go.mod index a26378bdbc..2e910a1911 100644 --- a/go.mod +++ b/go.mod @@ -13,11 +13,8 @@ replace gopkg.in/alecthomas/kingpin.v2 => github.com/Percona-Lab/kingpin v2.2.6- replace golang.org/x/crypto => github.com/percona-lab/crypto v0.0.0-20231108144114-756dfb24eaf2 -replace github.com/ClickHouse/clickhouse-go/151 => github.com/ClickHouse/clickhouse-go v1.5.1 // clickhouse-go/v2 cannot work with 1.5.1 which we need for QAN-API - require ( github.com/AlekSi/pointer v1.2.0 - github.com/ClickHouse/clickhouse-go/151 v0.0.0-00010101000000-000000000000 github.com/ClickHouse/clickhouse-go/v2 v2.22.2 github.com/DATA-DOG/go-sqlmock v1.5.0 github.com/alecthomas/kong v0.9.0 @@ -134,7 +131,6 @@ require ( github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/mwitkow/go-proto-validators v0.3.2 // indirect github.com/percona/percona-backup-mongodb v1.8.1 // indirect - github.com/pierrec/lz4 v2.6.1+incompatible // indirect github.com/posener/complete v1.2.3 // indirect github.com/riywo/loginshell v0.0.0-20200815045211-7d26008be1ab // indirect github.com/sergi/go-diff v1.2.0 // indirect @@ -165,7 +161,6 @@ require ( github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resourcegraph/armresourcegraph v0.9.0 github.com/AzureAD/microsoft-authentication-library-for-go v1.2.1 // indirect github.com/ClickHouse/ch-go v0.61.5 // indirect - github.com/ClickHouse/clickhouse-go v1.5.4 // indirect github.com/HdrHistogram/hdrhistogram-go v1.1.2 github.com/Microsoft/go-winio v0.6.1 // indirect github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 // indirect @@ -175,7 +170,6 @@ require ( github.com/cenkalti/backoff/v4 v4.2.1 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/charmbracelet/harmonica v0.2.0 // indirect - github.com/cloudflare/golz4 v0.0.0-20150217214814-ef862a3cdc58 // indirect github.com/containerd/console v1.0.4-0.20230313162750-1ae8d489ac81 // indirect github.com/docker/go-units v0.5.0 // indirect github.com/dustin/go-humanize v1.0.1 // indirect diff --git a/go.sum b/go.sum index 8aa5745a68..5e577e3cc0 100644 --- a/go.sum +++ b/go.sum @@ -70,8 +70,6 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/ClickHouse/ch-go v0.61.5 h1:zwR8QbYI0tsMiEcze/uIMK+Tz1D3XZXLdNrlaOpeEI4= github.com/ClickHouse/ch-go v0.61.5/go.mod h1:s1LJW/F/LcFs5HJnuogFMta50kKDO0lf9zzfrbl0RQg= -github.com/ClickHouse/clickhouse-go v1.5.1 h1:I8zVFZTz80crCs0FFEBJooIxsPcV0xfthzK1YrkpJTc= -github.com/ClickHouse/clickhouse-go v1.5.1/go.mod h1:EaI/sW7Azgz9UATzd5ZdZHRUhHgv5+JMS9NSr2smCJI= github.com/ClickHouse/clickhouse-go v1.5.4 h1:cKjXeYLNWVJIx2J1K6H2CqyRmfwVJVY1OV1coaaFcI0= github.com/ClickHouse/clickhouse-go v1.5.4/go.mod h1:EaI/sW7Azgz9UATzd5ZdZHRUhHgv5+JMS9NSr2smCJI= github.com/ClickHouse/clickhouse-go/v2 v2.22.2 h1:T1BljsIjj+3aQog80jKMTeF4EqAUG4P6TVcCvmakYAc= @@ -130,8 +128,6 @@ github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= -github.com/bkaradzic/go-lz4 v1.0.0 h1:RXc4wYsyz985CkXXeX04y4VnZFGG8Rd43pRaHsOXAKk= -github.com/bkaradzic/go-lz4 v1.0.0/go.mod h1:0YdlkowM3VswSROI7qDxhRvJ3sLhlFrRRwjwegp5jy4= github.com/blang/semver v3.5.1+incompatible h1:cQNTCjp13qL8KC3Nbxr/y2Bqb63oX6wdnnjpJbkM4JQ= github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM= @@ -272,7 +268,6 @@ github.com/go-openapi/swag v0.23.0 h1:vsEVJDUo2hPJ2tu0/Xc+4noaxyEffXNIs3cOULZ+Gr github.com/go-openapi/swag v0.23.0/go.mod h1:esZ8ITTYEsH1V2trKHjAN8Ai7xHb8RV+YSZ577vPjgQ= github.com/go-openapi/validate v0.24.0 h1:LdfDKwNbpB6Vn40xhTdNZAnfLECL81w+VX3BumrGD58= github.com/go-openapi/validate v0.24.0/go.mod h1:iyeX1sEufmv3nPbBdX3ieNviWnOZaJ1+zquzJEf2BAQ= -github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= github.com/go-sql-driver/mysql v1.7.1 h1:lUIinVbN1DY0xBg0eMOzmmtGoHwWBbvnWubQUrtU8EI= @@ -465,7 +460,6 @@ github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9Y github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= -github.com/jmoiron/sqlx v1.2.0/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks= github.com/jmoiron/sqlx v1.3.5 h1:vFFPA71p1o5gAeqtEAwLU4dnX2napprKtHr7PYIcN3g= github.com/jmoiron/sqlx v1.3.5/go.mod h1:nRVWtLre0KfCLJvgxzCsLVMogSvQ1zNJtpYr2Ccp0mQ= github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg= @@ -519,7 +513,6 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= -github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.8.0/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= @@ -551,7 +544,6 @@ github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzp github.com/mattn/go-runewidth v0.0.12/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk= github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZgg3U= github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= -github.com/mattn/go-sqlite3 v1.9.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= github.com/mattn/go-sqlite3 v1.14.0/go.mod h1:JIl7NbARA7phWnGvh0LKTyg7S9BA+6gx71ShQilpsus= github.com/mattn/go-sqlite3 v1.14.6/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= github.com/mattn/go-sqlite3 v1.14.16 h1:yOQRA0RpS5PFz/oikGwBEqvAWhWg5ufRz4ETLjwpU1Y= @@ -658,10 +650,7 @@ github.com/percona/promconfig v0.2.5 h1:f/HN/CbECQs7d9RIB6MKVkuXstsrsqEDxRvf6yig github.com/percona/promconfig v0.2.5/go.mod h1:Y2uXi5QNk71+ceJHuI9poank+0S1kjxd3K105fXKVkg= github.com/pganalyze/pg_query_go/v2 v2.2.0 h1:OW+reH+ZY7jdEuPyuLGlf1m7dLbE+fDudKXhLs0Ttpk= github.com/pganalyze/pg_query_go/v2 v2.2.0/go.mod h1:XAxmVqz1tEGqizcQ3YSdN90vCOHBWjJi8URL1er5+cA= -github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pierrec/lz4 v2.3.0+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= -github.com/pierrec/lz4 v2.6.1+incompatible h1:9UY3+iC23yxF0UfGaYrGplQ+79Rg+h/q9FV9ix19jjM= -github.com/pierrec/lz4 v2.6.1+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ= github.com/pierrec/lz4/v4 v4.1.21/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ= diff --git a/qan-api2/db.go b/qan-api2/db.go index 060e137f70..44f55d76a7 100644 --- a/qan-api2/db.go +++ b/qan-api2/db.go @@ -22,7 +22,7 @@ import ( "net/url" "strings" - clickhouse "github.com/ClickHouse/clickhouse-go/151" // register database/sql driver + clickhouse "github.com/ClickHouse/clickhouse-go/v2" // register database/sql driver "github.com/golang-migrate/migrate/v4" _ "github.com/golang-migrate/migrate/v4/database/clickhouse" // register golang-migrate driver "github.com/golang-migrate/migrate/v4/source/iofs" diff --git a/qan-api2/db_test.go b/qan-api2/db_test.go index 841803965e..e77a325963 100644 --- a/qan-api2/db_test.go +++ b/qan-api2/db_test.go @@ -25,7 +25,7 @@ import ( "testing" "time" - _ "github.com/ClickHouse/clickhouse-go/151" // register database/sql driver + _ "github.com/ClickHouse/clickhouse-go/v2" // register database/sql driver _ "github.com/golang-migrate/migrate/v4/database/clickhouse" "github.com/jmoiron/sqlx" "github.com/stretchr/testify/assert" diff --git a/qan-api2/services/analytics/filters_test.go b/qan-api2/services/analytics/filters_test.go index 31b4fb81f4..0c866b082c 100644 --- a/qan-api2/services/analytics/filters_test.go +++ b/qan-api2/services/analytics/filters_test.go @@ -23,7 +23,7 @@ import ( "testing" "time" - _ "github.com/ClickHouse/clickhouse-go/151" // register database/sql driver + _ "github.com/ClickHouse/clickhouse-go/v2" // register database/sql driver // TODO replace with 'google.golang.org/protobuf/encoding/protojson' since this one is deprecated. "github.com/golang/protobuf/ptypes/timestamp" "github.com/jmoiron/sqlx" From 2d55e0b63dacfc55862b926f4e81bb9588ecc84a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20=C4=8Ctvrtka?= Date: Thu, 21 Mar 2024 14:35:18 +0100 Subject: [PATCH 2/3] PMM-12551 Fix for new connection string format. --- managed/services/telemetry/telemetry_test.go | 2 +- qan-api2/db_test.go | 8 ++++---- qan-api2/main.go | 2 +- qan-api2/services/analytics/filters_test.go | 2 +- qan-api2/services/analytics/profile_test.go | 2 +- 5 files changed, 8 insertions(+), 8 deletions(-) diff --git a/managed/services/telemetry/telemetry_test.go b/managed/services/telemetry/telemetry_test.go index 62bbcec44e..4de3244245 100644 --- a/managed/services/telemetry/telemetry_test.go +++ b/managed/services/telemetry/telemetry_test.go @@ -50,7 +50,7 @@ func TestRunTelemetryService(t *testing.T) { if ok { pgHostPort = pgHostPortFromEnv } - qanDSN := "tcp://localhost:9000?database=pmm" + qanDSN := "tcp://localhost:9000/pmm" qanDSNFromEnv, ok := os.LookupEnv(envQanDSN) if ok { qanDSN = qanDSNFromEnv diff --git a/qan-api2/db_test.go b/qan-api2/db_test.go index e77a325963..7adcaa506e 100644 --- a/qan-api2/db_test.go +++ b/qan-api2/db_test.go @@ -39,9 +39,9 @@ func setup() *sqlx.DB { } dsn, ok := os.LookupEnv("QANAPI_DSN_TEST") - dsn = strings.Replace(dsn, "?database=pmm_test", "?database=pmm_test_parts", 1) + dsn = strings.Replace(dsn, "/pmm_test", "/pmm_test_parts", 1) if !ok { - dsn = "clickhouse://127.0.0.1:19000?database=pmm_test_parts" + dsn = "clickhouse://127.0.0.1:19000/pmm_test_parts" } db, err := sqlx.Connect("clickhouse", dsn) if err != nil { @@ -113,9 +113,9 @@ func TestCreateDbIfNotExists(t *testing.T) { t.Run("connect to db that doesnt exist", func(t *testing.T) { dsn, ok := os.LookupEnv("QANAPI_DSN_TEST") - dsn = strings.Replace(dsn, "?database=pmm_test", "?database=pmm_created_db", 1) + dsn = strings.Replace(dsn, "/pmm_test", "/pmm_created_db", 1) if !ok { - dsn = "clickhouse://127.0.0.1:19000?database=pmm_created_db" + dsn = "clickhouse://127.0.0.1:19000/pmm_created_db" } db := createDB(dsn) diff --git a/qan-api2/main.go b/qan-api2/main.go index df5457964d..24e1e9af4b 100644 --- a/qan-api2/main.go +++ b/qan-api2/main.go @@ -61,7 +61,7 @@ import ( const ( shutdownTimeout = 3 * time.Second - defaultDsnF = "clickhouse://%s?database=%s&block_size=%s&pool_size=%s" + defaultDsnF = "clickhouse://%s/%s?block_size=%s&pool_size=%s" maxIdleConns = 5 maxOpenConns = 10 ) diff --git a/qan-api2/services/analytics/filters_test.go b/qan-api2/services/analytics/filters_test.go index 0c866b082c..fbb714f182 100644 --- a/qan-api2/services/analytics/filters_test.go +++ b/qan-api2/services/analytics/filters_test.go @@ -54,7 +54,7 @@ type testValuesUnmarshal struct { func TestService_GetFilters(t *testing.T) { dsn, ok := os.LookupEnv("QANAPI_DSN_TEST") if !ok { - dsn = "clickhouse://127.0.0.1:19000?database=pmm_test" + dsn = "clickhouse://127.0.0.1:19000/pmm_test" } db, err := sqlx.Connect("clickhouse", dsn) if err != nil { diff --git a/qan-api2/services/analytics/profile_test.go b/qan-api2/services/analytics/profile_test.go index 95e22f1f81..3db4b7cb55 100644 --- a/qan-api2/services/analytics/profile_test.go +++ b/qan-api2/services/analytics/profile_test.go @@ -38,7 +38,7 @@ import ( func setup() *sqlx.DB { dsn, ok := os.LookupEnv("QANAPI_DSN_TEST") if !ok { - dsn = "clickhouse://127.0.0.1:19000?database=pmm_test" + dsn = "clickhouse://127.0.0.1:19000/pmm_test" } db, err := sqlx.Connect("clickhouse", dsn) if err != nil { From c7da376fbed98de7905afc4f1790a405cebfa369 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20=C4=8Ctvrtka?= Date: Thu, 21 Mar 2024 15:11:19 +0100 Subject: [PATCH 3/3] PMM-12551 Another fix for new connection string. --- qan-api2/db.go | 7 ++----- qan-api2/db_test.go | 2 +- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/qan-api2/db.go b/qan-api2/db.go index 44f55d76a7..e97e3a90aa 100644 --- a/qan-api2/db.go +++ b/qan-api2/db.go @@ -78,11 +78,8 @@ func createDB(dsn string) error { if err != nil { return err } - q := clickhouseURL.Query() - databaseName := q.Get("database") - q.Set("database", "default") - - clickhouseURL.RawQuery = q.Encode() + databaseName := strings.Replace(clickhouseURL.Path, "/", "", 1) + clickhouseURL.Path = "/default" defaultDB, err := sqlx.Connect("clickhouse", clickhouseURL.String()) if err != nil { diff --git a/qan-api2/db_test.go b/qan-api2/db_test.go index 7adcaa506e..812e5f2321 100644 --- a/qan-api2/db_test.go +++ b/qan-api2/db_test.go @@ -120,6 +120,6 @@ func TestCreateDbIfNotExists(t *testing.T) { db := createDB(dsn) - require.Equal(t, db, nil, "Check connection after we create database") + require.Equal(t, nil, db, "Check connection after we create database") }) }