From c184de467b100525e1f8aa2e4e24a4378f0662e7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Lindstr=C3=B6m?= Date: Wed, 11 Dec 2024 09:34:18 +0200 Subject: [PATCH] MDEV-34924 : gtid_slave_pos table neven been deleted on non replica nodes (wsrep_gtid_mode = 1) Problem was caused by MDEV-31413 commit 277968aa where mysql.gtid_slave_pos table was replicated by Galera. However, as not all nodes in Galera cluster are replica nodes, rows were not deleted from table. In this fix this is corrected so that mysql.gtid_slave_pos table is not replicated by Galera. Instead when Galera node receives GTID event and wsrep_gtid_mode=1, this event is stored to mysql.gtid_slave_pos table Added test case galera_2primary_replica for 2 async primaries replicating to galera cluster. Added test case galera_circular_replication where async primary replicates to galera cluster and one of the galera cluster nodes is master to async replica. Modified test case galera_restart_replica to monitor gtid positions and rows in mysql.gtid_pos_table --- .../galera_2nodes_as_replica_2primary.cnf | 70 ++++++ .../galera/r/galera_2primary_replica.result | 95 +++++++ .../r/galera_circular_replication.result | 138 +++++++++++ .../galera/r/galera_restart_replica.result | 191 ++++++++------ .../galera/t/galera_2primary_replica.cnf | 23 ++ .../galera/t/galera_2primary_replica.test | 170 +++++++++++++ .../galera/t/galera_circular_replication.cnf | 25 ++ .../galera/t/galera_circular_replication.test | 234 ++++++++++++++++++ .../galera/t/galera_restart_replica.test | 195 ++++++++++----- sql/rpl_gtid.cc | 50 +--- sql/wsrep_applier.cc | 5 + sql/wsrep_schema.cc | 80 +++++- sql/wsrep_schema.h | 14 +- 13 files changed, 1110 insertions(+), 180 deletions(-) create mode 100644 mysql-test/suite/galera/galera_2nodes_as_replica_2primary.cnf create mode 100644 mysql-test/suite/galera/r/galera_2primary_replica.result create mode 100644 mysql-test/suite/galera/r/galera_circular_replication.result create mode 100644 mysql-test/suite/galera/t/galera_2primary_replica.cnf create mode 100644 mysql-test/suite/galera/t/galera_2primary_replica.test create mode 100644 mysql-test/suite/galera/t/galera_circular_replication.cnf create mode 100644 mysql-test/suite/galera/t/galera_circular_replication.test diff --git a/mysql-test/suite/galera/galera_2nodes_as_replica_2primary.cnf b/mysql-test/suite/galera/galera_2nodes_as_replica_2primary.cnf new file mode 100644 index 0000000000000..714caf6ee6766 --- /dev/null +++ b/mysql-test/suite/galera/galera_2nodes_as_replica_2primary.cnf @@ -0,0 +1,70 @@ +# +# This .cnf file creates a setup with 2 standard MariaDB servers, followed by a 2-node Galera cluster +# + +# Use default setting for mysqld processes +!include include/default_mysqld.cnf + +[mysqld] +loose-innodb +log-bin=mysqld-bin +log-slave-updates +binlog-format=row +innodb-autoinc-lock-mode=2 +default-storage-engine=innodb +# enforce read-committed characteristics across the cluster +# wsrep-causal-reads=ON +wsrep-sync-wait=15 + +[mysqld.1] +wsrep-on=1 +server-id=1 +#galera_port=@OPT.port +#ist_port=@OPT.port +#sst_port=@OPT.port +wsrep_provider=@ENV.WSREP_PROVIDER +wsrep_cluster_address=gcomm:// +wsrep_provider_options='repl.causal_read_timeout=PT90S;base_port=@mysqld.1.#galera_port;evs.suspect_timeout=PT10S;evs.inactive_timeout=PT30S;evs.install_timeout=PT15S;gcache.size=10M' +wsrep_node_address='127.0.0.1:@mysqld.1.#galera_port' +wsrep_node_incoming_address=127.0.0.1:@mysqld.1.port +wsrep_sst_receive_address='127.0.0.1:@mysqld.1.#sst_port' + +[mysqld.2] +wsrep-on=1 +server-id=2 +#galera_port=@OPT.port +#ist_port=@OPT.port +#sst_port=@OPT.port +wsrep_provider=@ENV.WSREP_PROVIDER +wsrep_cluster_address='gcomm://127.0.0.1:@mysqld.1.#galera_port' +wsrep_provider_options='repl.causal_read_timeout=PT90S;base_port=@mysqld.2.#galera_port;evs.suspect_timeout=PT10S;evs.inactive_timeout=PT30S;evs.install_timeout=PT15S;gcache.size=10M' +wsrep_node_address='127.0.0.1:@mysqld.2.#galera_port' +wsrep_node_incoming_address=127.0.0.1:@mysqld.2.port +wsrep_sst_receive_address='127.0.0.1:@mysqld.2.#sst_port' + +[mysqld.3] +wsrep-on=OFF +server-id=3 +gtid_domain_id=3 + +[mysqld.4] +wsrep-on=OFF +server-id=4 +gtid_domain_id=4 + + +[sst] +sst-log-archive-dir=@ENV.MYSQLTEST_VARDIR/log + +[ENV] +NODE_MYPORT_1= @mysqld.1.port +NODE_MYSOCK_1= @mysqld.1.socket + +NODE_MYPORT_2= @mysqld.2.port +NODE_MYSOCK_2= @mysqld.2.socket + +NODE_MYPORT_3= @mysqld.3.port +NODE_MYSOCK_3= @mysqld.3.socket + +NODE_MYPORT_4= @mysqld.4.port +NODE_MYSOCK_4= @mysqld.4.socket diff --git a/mysql-test/suite/galera/r/galera_2primary_replica.result b/mysql-test/suite/galera/r/galera_2primary_replica.result new file mode 100644 index 0000000000000..8bdbf5be962a0 --- /dev/null +++ b/mysql-test/suite/galera/r/galera_2primary_replica.result @@ -0,0 +1,95 @@ +connection node_2; +connection node_1; +connect primary1, 127.0.0.1, root, , test, $NODE_MYPORT_3; +connect primary2, 127.0.0.1, root, , test, $NODE_MYPORT_4; +connection primary1; +# Primary1 creating user for replication +create user repl@'%' identified by 'repl'; +grant all on *.* to repl@'%'; +connection primary2; +# Primary2 creating user for replication +create user repl2@'%' identified by 'repl2'; +grant all on *.* to repl2@'%'; +connect replica, 127.0.0.1, root, , test, $NODE_MYPORT_1; +connection replica; +connection node_2; +connection replica; +# Galera replica changing master to primary1 +SET @@default_master_connection='stream2'; +# Primary node changing master to primary2 +START ALL SLAVES; +Warnings: +Note 1937 SLAVE 'stream1' started +Note 1937 SLAVE 'stream2' started +connection primary1; +# Primary 1: Creating table and populating it with data +CREATE TABLE t1 (id bigint auto_increment primary key, msg varchar(100)) engine=innodb; +# Intentionally generate 1k GTID-events +SELECT COUNT(*) AS EXPECT_1000 FROM t1; +EXPECT_1000 +1000 +connection primary2; +# Primary 2: Creating table and populating it with data +CREATE TABLE t2 (id bigint auto_increment primary key, msg varchar(100)) engine=innodb; +# Intentionally generate 1k GTID-events +SELECT COUNT(*) AS EXPECT_1000 FROM t2; +EXPECT_1000 +1000 +connection replica; +# Waiting for data to replicate to node_1 +SELECT COUNT(*) AS EXPECT_1000 FROM t1; +EXPECT_1000 +1000 +SELECT COUNT(*) AS EXPECT_1000 FROM t2; +EXPECT_1000 +1000 +SELECT COUNT(*) > 0 AS EXPECT_1 FROM mysql.gtid_slave_pos; +EXPECT_1 +1 +SELECT COUNT(*) < 1000 AS EXPECT_1 FROM mysql.gtid_slave_pos; +EXPECT_1 +1 +SELECT @@gtid_slave_pos,@@gtid_binlog_pos,@@gtid_current_pos; +@@gtid_slave_pos @@gtid_binlog_pos @@gtid_current_pos +3-3-1003,4-4-1003 3-3-1003,4-4-1003 3-3-1003,4-4-1003 +connection node_2; +# Waiting for data to replicate to node_2 +SELECT COUNT(*) AS EXPECT_1000 FROM t1; +EXPECT_1000 +1000 +SELECT COUNT(*) AS EXPECT_1000 FROM t2; +EXPECT_1000 +1000 +SELECT COUNT(*) > 0 AS EXPECT_1 FROM mysql.gtid_slave_pos; +EXPECT_1 +1 +SELECT COUNT(*) < 1000 AS EXPECT_1 FROM mysql.gtid_slave_pos; +EXPECT_1 +1 +SELECT @@gtid_slave_pos,@@gtid_binlog_pos,@@gtid_current_pos; +@@gtid_slave_pos @@gtid_binlog_pos @@gtid_current_pos +3-3-1003,4-4-1003 3-3-1003,4-4-1003 3-3-1003,4-4-1003 +connection primary1; +drop table t1; +connection primary2; +drop table t2; +# Wait until drop table is replicated on Galera +connection replica; +connection node_2; +connection replica; +STOP ALL SLAVES; +Warnings: +Note 1938 SLAVE 'stream1' stopped +Note 1938 SLAVE 'stream2' stopped +RESET SLAVE ALL; +connection primary1; +RESET MASTER; +connection primary2; +RESET MASTER; +connection node_1; +disconnect primary1; +disconnect primary2; +disconnect replica; +disconnect node_2; +disconnect node_1; +# End of test diff --git a/mysql-test/suite/galera/r/galera_circular_replication.result b/mysql-test/suite/galera/r/galera_circular_replication.result new file mode 100644 index 0000000000000..72340977005d3 --- /dev/null +++ b/mysql-test/suite/galera/r/galera_circular_replication.result @@ -0,0 +1,138 @@ +connection node_2; +connection node_1; +connect replica1, 127.0.0.1, root, , test, $NODE_MYPORT_1; +connect primary2, 127.0.0.1, root, , test, $NODE_MYPORT_3; +connect primary1, 127.0.0.1, root, , test, $NODE_MYPORT_4; +connect replica2, 127.0.0.1, root, , test, $NODE_MYPORT_4; +connection primary1; +# Primary1 node creating user for replication +create user repl@'%' identified by 'repl'; +grant all on *.* to repl@'%'; +ALTER TABLE mysql.gtid_slave_pos ENGINE=InnoDB; +connection replica1; +connection node_2; +connection primary2; +connection replica1; +# Galera replica changing master to primary1 +START SLAVE; +connection primary2; +# Primary2 creating user for replication +create user repl2@'%' identified by 'repl2'; +grant all on *.* to repl2@'%'; +connection replica2; +# replica2 changing master to primary2 +START SLAVE; +connection primary1; +# Primary1: Creating table and populating it with data +CREATE TABLE t1 (id bigint auto_increment primary key, msg varchar(100)) engine=innodb; +# Intentionally generate 1k GTID-events +SELECT COUNT(*) AS EXPECT_1000 FROM t1; +EXPECT_1000 +1000 +connection replica1; +# Waiting for data to replicate to replica +SELECT COUNT(*) AS EXPECT_1000 FROM t1; +EXPECT_1000 +1000 +# Writing more data to table +# Intentionally generate 1k GTID-events +SELECT COUNT(*) AS EXPECT_2000 FROM t1; +EXPECT_2000 +2000 +connection node_2; +# Waiting for data to replicate to Galera node_2 +SELECT COUNT(*) AS EXPECT_2000 FROM t1; +EXPECT_2000 +2000 +# Writing more data to table +# Intentionally generate 1k GTID-events +SELECT COUNT(*) AS EXPECT_3000 FROM t1; +EXPECT_3000 +3000 +connection primary2; +# Waiting for data to replicate to primary2 +SELECT COUNT(*) AS EXPECT_3000 FROM t1; +EXPECT_3000 +3000 +# Writing more data to table +# Intentionally generate 1k GTID-events +SELECT COUNT(*) AS EXPECT_4000 FROM t1; +EXPECT_4000 +4000 +connection primary1; +# Waiting for data to replicate to primary1 +SELECT COUNT(*) AS EXPECT_4000 FROM t1; +EXPECT_4000 +4000 +SELECT COUNT(*) > 0 AS EXPECT_1 FROM mysql.gtid_slave_pos; +EXPECT_1 +1 +SELECT COUNT(*) < 1000 AS EXPECT_1 FROM mysql.gtid_slave_pos; +EXPECT_1 +1 +SELECT @@gtid_slave_pos,@@gtid_binlog_pos,@@gtid_current_pos; +@@gtid_slave_pos @@gtid_binlog_pos @@gtid_current_pos +0-4-1004,16-15-3002 0-4-1004,16-15-3002 0-4-1004,16-15-3002 +connection replica1; +# Waiting for data to replicate to replica +SELECT COUNT(*) AS EXPECT_4000 FROM t1; +EXPECT_4000 +4000 +SELECT COUNT(*) > 0 AS EXPECT_1 FROM mysql.gtid_slave_pos; +EXPECT_1 +1 +SELECT COUNT(*) < 1000 AS EXPECT_1 FROM mysql.gtid_slave_pos; +EXPECT_1 +1 +SELECT @@gtid_slave_pos,@@gtid_binlog_pos,@@gtid_current_pos; +@@gtid_slave_pos @@gtid_binlog_pos @@gtid_current_pos +0-4-1004,16-15-3002 0-4-1004,16-15-3002 0-4-1004,16-15-3002 +connection node_2; +# Waiting for data to replicate to node_2 +SELECT COUNT(*) AS EXPECT_4000 FROM t1; +EXPECT_4000 +4000 +SELECT COUNT(*) > 0 AS EXPECT_1 FROM mysql.gtid_slave_pos; +EXPECT_1 +1 +SELECT COUNT(*) < 1000 AS EXPECT_1 FROM mysql.gtid_slave_pos; +EXPECT_1 +1 +SELECT @@gtid_slave_pos,@@gtid_binlog_pos,@@gtid_current_pos; +@@gtid_slave_pos @@gtid_binlog_pos @@gtid_current_pos +0-4-1004 0-4-1004,16-15-3002 0-4-1004,16-15-3002 +connection primary2; +# Waiting for data to replicate to node_3 +SELECT COUNT(*) AS EXPECT_4000 FROM t1; +EXPECT_4000 +4000 +SELECT COUNT(*) > 0 AS EXPECT_1 FROM mysql.gtid_slave_pos; +EXPECT_1 +1 +SELECT COUNT(*) < 1000 AS EXPECT_1 FROM mysql.gtid_slave_pos; +EXPECT_1 +1 +SELECT @@gtid_slave_pos,@@gtid_binlog_pos,@@gtid_current_pos; +@@gtid_slave_pos @@gtid_binlog_pos @@gtid_current_pos +0-4-1004 0-4-1004,16-15-3002 0-4-1004,16-15-3002 +connection primary1; +drop table t1; +# Wait until drop table is replicated on Galera +connection replica1; +connection node_2; +connection primary2; +connection replica1; +STOP SLAVE; +RESET SLAVE ALL; +connection replica2; +STOP SLAVE; +RESET SLAVE ALL; +RESET MASTER; +connection node_1; +disconnect primary1; +disconnect replica1; +disconnect primary2; +disconnect replica2; +disconnect node_2; +disconnect node_1; +# End of test diff --git a/mysql-test/suite/galera/r/galera_restart_replica.result b/mysql-test/suite/galera/r/galera_restart_replica.result index 9b7e9fd259f0b..efc9a83a16872 100644 --- a/mysql-test/suite/galera/r/galera_restart_replica.result +++ b/mysql-test/suite/galera/r/galera_restart_replica.result @@ -1,122 +1,169 @@ connection node_2; connection node_1; -connect node_3, 127.0.0.1, root, , test, $NODE_MYPORT_3; +connect replica, 127.0.0.1, root, , test, $NODE_MYPORT_2; +connect primary, 127.0.0.1, root, , test, $NODE_MYPORT_3; create user repl@'%' identified by 'repl'; grant all on *.* to repl@'%'; ALTER TABLE mysql.gtid_slave_pos ENGINE=InnoDB; connection node_1; -connection node_2; -connection node_2; +connection replica; +connection replica; START SLAVE; -connection node_3; -CREATE TABLE t1 (id bigint primary key, msg varchar(100)) engine=innodb; -SELECT COUNT(*) AS EXPECT_10000 FROM t1; -EXPECT_10000 -10000 -connection node_2; +connection primary; +CREATE TABLE t1 (id bigint auto_increment primary key, msg varchar(100)) engine=innodb; +# Intentionally generate 1k GTID-events +SELECT COUNT(*) AS EXPECT_1000 FROM t1; +EXPECT_1000 +1000 +connection replica; SELECT COUNT(*) > 0 AS EXPECT_1 FROM mysql.gtid_slave_pos; EXPECT_1 1 -SELECT COUNT(*) AS EXPECT_10000 FROM t1; -EXPECT_10000 -10000 +SELECT COUNT(*) < 1000 AS EXPECT_1 FROM mysql.gtid_slave_pos; +EXPECT_1 +1 +SELECT @@gtid_slave_pos,@@gtid_binlog_pos,@@gtid_current_pos; +@@gtid_slave_pos @@gtid_binlog_pos @@gtid_current_pos +0-3-1004 0-3-1004 0-3-1004 +SELECT COUNT(*) AS EXPECT_1000 FROM t1; +EXPECT_1000 +1000 connection node_1; SELECT COUNT(*) > 0 AS EXPECT_1 FROM mysql.gtid_slave_pos; EXPECT_1 1 -SELECT COUNT(*) AS EXPECT_10000 FROM t1; -EXPECT_10000 -10000 -connection node_2; -# Verify that graceful shutdown succeeds. +SELECT COUNT(*) < 1000 AS EXPECT_1 FROM mysql.gtid_slave_pos; +EXPECT_1 +1 +SELECT @@gtid_slave_pos,@@gtid_binlog_pos,@@gtid_current_pos; +@@gtid_slave_pos @@gtid_binlog_pos @@gtid_current_pos +0-3-1004 0-3-1004 0-3-1004 +SELECT COUNT(*) AS EXPECT_1000 FROM t1; +EXPECT_1000 +1000 +connection replica; +# Verify that graceful shutdown succeeds in replica. # Force SST connection node_1; -# Waiting until node_2 is not part of cluster anymore -connection node_2; -# Start node_2 again -¤ Wait until node_2 is back on cluster -connection node_2; +# Waiting until replica is not part of cluster anymore +connection replica; +# Start replica again +# Wait until replica is back on cluster SELECT COUNT(*) > 0 AS EXPECT_1 FROM mysql.gtid_slave_pos; EXPECT_1 1 -SELECT COUNT(*) AS EXPECT_10000 FROM t1; -EXPECT_10000 -10000 +SELECT COUNT(*) < 1000 AS EXPECT_1 FROM mysql.gtid_slave_pos; +EXPECT_1 +1 +SELECT @@gtid_slave_pos,@@gtid_binlog_pos,@@gtid_current_pos; +@@gtid_slave_pos @@gtid_binlog_pos @@gtid_current_pos +0-3-1004 0-3-1004 0-3-1004 +SELECT COUNT(*) AS EXPECT_1000 FROM t1; +EXPECT_1000 +1000 connection node_1; SELECT COUNT(*) > 0 AS EXPECT_1 FROM mysql.gtid_slave_pos; EXPECT_1 1 -SELECT COUNT(*) AS EXPECT_10000 FROM t1; -EXPECT_10000 -10000 -connection node_3; -SELECT COUNT(*) AS EXPECT_10000 FROM t1; -EXPECT_10000 -10000 -connection node_3; +SELECT COUNT(*) < 1000 AS EXPECT_1 FROM mysql.gtid_slave_pos; +EXPECT_1 +1 +SELECT @@gtid_slave_pos,@@gtid_binlog_pos,@@gtid_current_pos; +@@gtid_slave_pos @@gtid_binlog_pos @@gtid_current_pos +0-3-1004 0-3-1004 0-3-1004 +SELECT COUNT(*) AS EXPECT_1000 FROM t1; +EXPECT_1000 +1000 +connection primary; +SELECT COUNT(*) AS EXPECT_1000 FROM t1; +EXPECT_1000 +1000 drop table t1; -connection node_2; +connection replica; connection node_1; -connection node_3; -CREATE TABLE t1 (id bigint primary key, msg varchar(100)) engine=innodb; -SELECT COUNT(*) AS EXPECT_10000 FROM t1; -EXPECT_10000 -10000 -connection node_2; +connection primary; +CREATE TABLE t1 (id bigint auto_increment primary key, msg varchar(100)) engine=innodb; +# Intentionally generate 1k GTID-events +SELECT COUNT(*) AS EXPECT_1000 FROM t1; +EXPECT_1000 +1000 +connection replica; SELECT COUNT(*) > 0 AS EXPECT_1 FROM mysql.gtid_slave_pos; EXPECT_1 1 -SELECT COUNT(*) AS EXPECT_10000 FROM t1; -EXPECT_10000 -10000 +SELECT COUNT(*) < 1000 AS EXPECT_1 FROM mysql.gtid_slave_pos; +EXPECT_1 +1 +SELECT @@gtid_slave_pos,@@gtid_binlog_pos,@@gtid_current_pos; +@@gtid_slave_pos @@gtid_binlog_pos @@gtid_current_pos +0-3-2006 0-3-2006 0-3-2006 +SELECT COUNT(*) AS EXPECT_1000 FROM t1; +EXPECT_1000 +1000 connection node_1; SELECT COUNT(*) > 0 AS EXPECT_1 FROM mysql.gtid_slave_pos; EXPECT_1 1 -SELECT COUNT(*) AS EXPECT_10000 FROM t1; -EXPECT_10000 -10000 -connection node_2; -# Verify that graceful shutdown succeeds. +SELECT COUNT(*) < 1000 AS EXPECT_1 FROM mysql.gtid_slave_pos; +EXPECT_1 +1 +SELECT @@gtid_slave_pos,@@gtid_binlog_pos,@@gtid_current_pos; +@@gtid_slave_pos @@gtid_binlog_pos @@gtid_current_pos +0-3-2006 0-3-2006 0-3-2006 +connection replica; +# Verify that graceful shutdown succeeds in replica. # Force SST connection node_1; -# Waiting until node_2 is not part of cluster anymore -connection node_3; -SELECT COUNT(*) AS EXPECT_20000 FROM t1; -EXPECT_20000 -20000 -connection node_2; -# Start node_2 again -¤ Wait until node_2 is back on cluster -connection node_2; +# Waiting until replica is not part of cluster anymore +# Add writes to primary +connection primary; +# Intentionally generate 1k GTID-events +SELECT COUNT(*) AS EXPECT_2000 FROM t1; +EXPECT_2000 +2000 +connection replica; +# Start replica again +# Wait until replica is back on cluster SELECT COUNT(*) > 0 AS EXPECT_1 FROM mysql.gtid_slave_pos; EXPECT_1 1 -SELECT COUNT(*) AS EXPECT_20000 FROM t1; -EXPECT_20000 -20000 +SELECT COUNT(*) < 1000 AS EXPECT_1 FROM mysql.gtid_slave_pos; +EXPECT_1 +1 +SELECT @@gtid_slave_pos,@@gtid_binlog_pos,@@gtid_current_pos; +@@gtid_slave_pos @@gtid_binlog_pos @@gtid_current_pos +0-3-3006 0-3-3006 0-3-3006 +SELECT COUNT(*) AS EXPECT_2000 FROM t1; +EXPECT_2000 +2000 connection node_1; SELECT COUNT(*) > 0 AS EXPECT_1 FROM mysql.gtid_slave_pos; EXPECT_1 1 -SELECT COUNT(*) AS EXPECT_20000 FROM t1; -EXPECT_20000 -20000 -connection node_3; -SELECT COUNT(*) AS EXPECT_20000 FROM t1; -EXPECT_20000 -20000 -connection node_3; +SELECT COUNT(*) < 1000 AS EXPECT_1 FROM mysql.gtid_slave_pos; +EXPECT_1 +1 +SELECT @@gtid_slave_pos,@@gtid_binlog_pos,@@gtid_current_pos; +@@gtid_slave_pos @@gtid_binlog_pos @@gtid_current_pos +0-3-3006 0-3-3006 0-3-3006 +SELECT COUNT(*) AS EXPECT_2000 FROM t1; +EXPECT_2000 +2000 +connection primary; +SELECT COUNT(*) AS EXPECT_2000 FROM t1; +EXPECT_2000 +2000 drop table t1; -connection node_2; +connection replica; connection node_1; -connection node_2; +connection replica; STOP SLAVE; RESET SLAVE ALL; -connection node_3; +connection primary; RESET MASTER; connection node_1; -disconnect node_3; +disconnect primary; +disconnect replica; disconnect node_2; disconnect node_1; # End of test diff --git a/mysql-test/suite/galera/t/galera_2primary_replica.cnf b/mysql-test/suite/galera/t/galera_2primary_replica.cnf new file mode 100644 index 0000000000000..21afd1945b105 --- /dev/null +++ b/mysql-test/suite/galera/t/galera_2primary_replica.cnf @@ -0,0 +1,23 @@ +!include ../galera_2nodes_as_replica_2primary.cnf + +[mysqld] +wsrep-debug=1 + +[mysqld.1] +server_id=15 +wsrep_gtid_mode=1 +wsrep_gtid_domain_id=16 +gtid_domain_id=11 +gtid_strict_mode=1 +wsrep-slave-threads=4 +slave-parallel-threads=2 + +[mysqld.2] +skip-slave-start=OFF +server_id=15 +wsrep_gtid_mode=1 +wsrep_gtid_domain_id=16 +gtid_domain_id=11 +gtid_strict_mode=1 +wsrep-slave-threads=4 + diff --git a/mysql-test/suite/galera/t/galera_2primary_replica.test b/mysql-test/suite/galera/t/galera_2primary_replica.test new file mode 100644 index 0000000000000..fb57c6637d099 --- /dev/null +++ b/mysql-test/suite/galera/t/galera_2primary_replica.test @@ -0,0 +1,170 @@ +# +# Test two primary nodes async replication to Galera cluster +# +# primary1 primary2 +# #3 #4 +# | | +# | async replication v +# +-------------------+ +----------------+ +# | | +# v v +# galera replica <------galera replication-------->galera node_2 +# #1 #2 +# +# Test outline +# +# - Create user for async replication and table with rows in both primaries +# - Verify that tables and rows are replicated to all Galera nodes +# - Verify that gtid position is same in all Galera nodes +# +# The galera/galera_2nodes_as_replica_2primary.cnf describes the setup of the nodes +# +--source include/force_restart.inc +--source include/galera_cluster.inc +--source include/have_innodb.inc + +# As node #3 and #4 are not a Galera node, and galera_cluster.inc does not open connetion to it +# we open the connections here +--connect primary1, 127.0.0.1, root, , test, $NODE_MYPORT_3 +--connect primary2, 127.0.0.1, root, , test, $NODE_MYPORT_4 +--connection primary1 +--echo # Primary1 creating user for replication +create user repl@'%' identified by 'repl'; +grant all on *.* to repl@'%'; + +--connection primary2 +--echo # Primary2 creating user for replication +create user repl2@'%' identified by 'repl2'; +grant all on *.* to repl2@'%'; + +--connect replica, 127.0.0.1, root, , test, $NODE_MYPORT_1 + +--let $node_1 = replica +--let $node_2 = node_2 +--source include/auto_increment_offset_save.inc + +--connection replica +--echo # Galera replica changing master to primary1 +--disable_query_log +SET @@default_master_connection='stream1'; +--eval CHANGE MASTER 'stream1' TO master_host='127.0.0.1', master_user='repl', master_password='repl', master_port=$NODE_MYPORT_3, master_use_gtid=slave_pos; +--enable_query_log + +SET @@default_master_connection='stream2'; +--echo # Primary node changing master to primary2 +--disable_query_log +--eval CHANGE MASTER 'stream2' TO master_host='127.0.0.1', master_user='repl2', master_password='repl2', master_port=$NODE_MYPORT_4, master_use_gtid=slave_pos; +--enable_query_log + +START ALL SLAVES; + +--connection primary1 +--echo # Primary 1: Creating table and populating it with data +CREATE TABLE t1 (id bigint auto_increment primary key, msg varchar(100)) engine=innodb; +--disable_query_log +--echo # Intentionally generate 1k GTID-events +--let $inserts=1000 +--let $count=0 +--disable_query_log +while($count < $inserts) +{ + --eval insert into t1 values (NULL,'test1') + --inc $count +} +--enable_query_log + +SELECT COUNT(*) AS EXPECT_1000 FROM t1; + +--connection primary2 +--echo # Primary 2: Creating table and populating it with data +CREATE TABLE t2 (id bigint auto_increment primary key, msg varchar(100)) engine=innodb; +--echo # Intentionally generate 1k GTID-events +--let $inserts=1000 +--let $count=0 +--disable_query_log +while($count < $inserts) +{ + --eval insert into t2 values (NULL,'test1') + --inc $count +} +--enable_query_log + +SELECT COUNT(*) AS EXPECT_1000 FROM t2; + +--connection replica +--echo # Waiting for data to replicate to node_1 +--let $wait_condition = SELECT COUNT(*) = 1 FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_NAME = 't1'; +--let $wait_condition_on_error_output = SHOW ALL SLAVES STATUS; +--source include/wait_condition_with_debug.inc +--let $wait_condition = SELECT COUNT(*) = 1 FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_NAME = 't2'; +--let $wait_condition_on_error_output = SHOW ALL SLAVES STATUS; +--source include/wait_condition_with_debug.inc + +--let $wait_condition = SELECT COUNT(*) = 1000 FROM t1; +--source include/wait_condition.inc +--let $wait_condition = SELECT COUNT(*) = 1000 FROM t2; +--source include/wait_condition.inc + +SELECT COUNT(*) AS EXPECT_1000 FROM t1; +SELECT COUNT(*) AS EXPECT_1000 FROM t2; + +SELECT COUNT(*) > 0 AS EXPECT_1 FROM mysql.gtid_slave_pos; +SELECT COUNT(*) < 1000 AS EXPECT_1 FROM mysql.gtid_slave_pos; +SELECT @@gtid_slave_pos,@@gtid_binlog_pos,@@gtid_current_pos; + +--connection node_2 +--echo # Waiting for data to replicate to node_2 +--let $wait_condition = SELECT COUNT(*) = 1 FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_NAME = 't1'; +--source include/wait_condition.inc +--let $wait_condition = SELECT COUNT(*) = 1 FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_NAME = 't2'; +--source include/wait_condition.inc + +--let $wait_condition = SELECT COUNT(*) = 1000 FROM t1; +--source include/wait_condition.inc +--let $wait_condition = SELECT COUNT(*) = 1000 FROM t2; +--source include/wait_condition.inc + +SELECT COUNT(*) AS EXPECT_1000 FROM t1; +SELECT COUNT(*) AS EXPECT_1000 FROM t2; +SELECT COUNT(*) > 0 AS EXPECT_1 FROM mysql.gtid_slave_pos; +SELECT COUNT(*) < 1000 AS EXPECT_1 FROM mysql.gtid_slave_pos; +SELECT @@gtid_slave_pos,@@gtid_binlog_pos,@@gtid_current_pos; +# +# Cleanup +# +--connection primary1 +drop table t1; +--connection primary2 +drop table t2; + +--echo # Wait until drop table is replicated on Galera +--connection replica +--let $wait_condition = SELECT COUNT(*) = 0 FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_NAME = 't1'; +--source include/wait_condition.inc +--let $wait_condition = SELECT COUNT(*) = 0 FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_NAME = 't2'; +--source include/wait_condition.inc + +--connection node_2 +--let $wait_condition = SELECT COUNT(*) = 0 FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_NAME = 't1'; +--source include/wait_condition.inc +--let $wait_condition = SELECT COUNT(*) = 0 FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_NAME = 't2'; +--source include/wait_condition.inc + +--connection replica +STOP ALL SLAVES; +RESET SLAVE ALL; + +--connection primary1 +RESET MASTER; +--connection primary2 +RESET MASTER; + +--source include/auto_increment_offset_restore.inc + +--connection node_1 +--disconnect primary1 +--disconnect primary2 +--disconnect replica + +--source include/galera_end.inc +--echo # End of test diff --git a/mysql-test/suite/galera/t/galera_circular_replication.cnf b/mysql-test/suite/galera/t/galera_circular_replication.cnf new file mode 100644 index 0000000000000..b1c6320d4df7b --- /dev/null +++ b/mysql-test/suite/galera/t/galera_circular_replication.cnf @@ -0,0 +1,25 @@ +!include ../galera_3nodes_as_slave.cnf + +[mysqld] +wsrep-debug=1 + +[mysqld.1] +server_id=15 +wsrep_gtid_mode=1 +wsrep_gtid_domain_id=16 +gtid_domain_id=11 +gtid_strict_mode=1 + +[mysqld.2] +server_id=15 +wsrep_gtid_mode=1 +wsrep_gtid_domain_id=16 +gtid_domain_id=11 +gtid_strict_mode=1 + +[mysqld.3] +server_id=15 +wsrep_gtid_mode=1 +wsrep_gtid_domain_id=16 +gtid_domain_id=11 +gtid_strict_mode=1 diff --git a/mysql-test/suite/galera/t/galera_circular_replication.test b/mysql-test/suite/galera/t/galera_circular_replication.test new file mode 100644 index 0000000000000..dbe85da4b23b7 --- /dev/null +++ b/mysql-test/suite/galera/t/galera_circular_replication.test @@ -0,0 +1,234 @@ +# +# Test circular replication where galera cluster is async replica and master +# +# mariadb #4 galera galera +# primary1 +# replica2 +# ---async replication-->replica1 #1 <--galera replication--> node_2 #2 +# ^ ^ +# | | galera replication +# | v +# +<------------------async replication----------------------primary2 (galera) #3 +# +# Test outline: +# +# - Create user for async replication in primary1 +# - Create user for async replication in primary2 +# - Create table and some data in primary1 +# - Verify that table and data is replicated to galera nodes +# - Verify that mysql.gtid_slave_pos has some rows in all Galera nodes +# - Verify that gtid_slave_pos, gtid_binlog_pos and gtid_current_pos are +# same in all Galera nodes and primary1 +# - Verify that writes on Galera nodes are replicated to all nodes +# and to primary1 +# +# The galera/galera_3nodes_as_slave.cnf describes the setup of the nodes +# +--source include/force_restart.inc +--source include/galera_cluster.inc +--source include/have_innodb.inc + +--connect replica1, 127.0.0.1, root, , test, $NODE_MYPORT_1 +--connect primary2, 127.0.0.1, root, , test, $NODE_MYPORT_3 + +# As node #4 is not a Galera node, and galera_cluster.inc does not open connetion to it +# because it is both primary and replica we open both connections here +--connect primary1, 127.0.0.1, root, , test, $NODE_MYPORT_4 +--connect replica2, 127.0.0.1, root, , test, $NODE_MYPORT_4 + +--connection primary1 +--echo # Primary1 node creating user for replication +create user repl@'%' identified by 'repl'; +grant all on *.* to repl@'%'; +ALTER TABLE mysql.gtid_slave_pos ENGINE=InnoDB; + +--let $node_1 = replica1 +--let $node_2 = node_2 +--let $node_3 = primary2 +--source include/auto_increment_offset_save.inc + +--connection replica1 +--echo # Galera replica changing master to primary1 +--disable_query_log +--eval CHANGE MASTER TO master_host='127.0.0.1', master_user='repl', master_password='repl', master_port=$NODE_MYPORT_4, master_use_gtid=slave_pos; +--enable_query_log +START SLAVE; + +--connection primary2 +--echo # Primary2 creating user for replication +create user repl2@'%' identified by 'repl2'; +grant all on *.* to repl2@'%'; + +--connection replica2 +--echo # replica2 changing master to primary2 +--disable_query_log +--eval CHANGE MASTER TO master_host='127.0.0.1', master_user='repl2', master_password='repl2', master_port=$NODE_MYPORT_3, master_use_gtid=slave_pos; +--enable_query_log +START SLAVE; + +--connection primary1 +--echo # Primary1: Creating table and populating it with data +CREATE TABLE t1 (id bigint auto_increment primary key, msg varchar(100)) engine=innodb; +--echo # Intentionally generate 1k GTID-events +--let $inserts=1000 +--let $count=0 +--disable_query_log +while($count < $inserts) +{ + --eval insert into t1 values (NULL,'test1') + --inc $count +} +--enable_query_log + +SELECT COUNT(*) AS EXPECT_1000 FROM t1; + +--connection replica1 +--echo # Waiting for data to replicate to replica +--let $wait_condition = SELECT COUNT(*) = 1 FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_NAME = 't1'; +--source include/wait_condition.inc + +--let $wait_condition = SELECT COUNT(*) = 1000 FROM t1; +--source include/wait_condition.inc + +SELECT COUNT(*) AS EXPECT_1000 FROM t1; + +--echo # Writing more data to table +--echo # Intentionally generate 1k GTID-events +--let $inserts=1000 +--let $count=0 +--disable_query_log +while($count < $inserts) +{ + --eval insert into t1 values (NULL,'test1') + --inc $count +} +--enable_query_log + +SELECT COUNT(*) AS EXPECT_2000 FROM t1; + +--connection node_2 +--echo # Waiting for data to replicate to Galera node_2 +--let $wait_condition = SELECT COUNT(*) = 1 FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_NAME = 't1'; +--source include/wait_condition.inc + +--let $wait_condition = SELECT COUNT(*) = 2000 FROM t1; +--source include/wait_condition.inc + +SELECT COUNT(*) AS EXPECT_2000 FROM t1; + +--echo # Writing more data to table +--echo # Intentionally generate 1k GTID-events +--let $inserts=1000 +--let $count=0 +--disable_query_log +while($count < $inserts) +{ + --eval insert into t1 values (NULL,'test1') + --inc $count +} +--enable_query_log + +SELECT COUNT(*) AS EXPECT_3000 FROM t1; + +--connection primary2 +--echo # Waiting for data to replicate to primary2 +--let $wait_condition = SELECT COUNT(*) = 1 FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_NAME = 't1'; +--source include/wait_condition.inc + +--let $wait_condition = SELECT COUNT(*) = 3000 FROM t1; +--source include/wait_condition.inc + +SELECT COUNT(*) AS EXPECT_3000 FROM t1; + +--echo # Writing more data to table +--echo # Intentionally generate 1k GTID-events +--let $inserts=1000 +--let $count=0 +--disable_query_log +while($count < $inserts) +{ + --eval insert into t1 values (NULL,'test1') + --inc $count +} +--enable_query_log + +SELECT COUNT(*) AS EXPECT_4000 FROM t1; + +--connection primary1 +--echo # Waiting for data to replicate to primary1 +--let $wait_condition = SELECT COUNT(*) = 4000 FROM t1; +--let $wait_condition_on_error_output = SHOW SLAVE STATUS; +--source include/wait_condition_with_debug.inc + +SELECT COUNT(*) AS EXPECT_4000 FROM t1; +SELECT COUNT(*) > 0 AS EXPECT_1 FROM mysql.gtid_slave_pos; +SELECT COUNT(*) < 1000 AS EXPECT_1 FROM mysql.gtid_slave_pos; +SELECT @@gtid_slave_pos,@@gtid_binlog_pos,@@gtid_current_pos; + +--connection replica1 +--echo # Waiting for data to replicate to replica +--let $wait_condition = SELECT COUNT(*) = 4000 FROM t1; +--source include/wait_condition.inc + +SELECT COUNT(*) AS EXPECT_4000 FROM t1; +SELECT COUNT(*) > 0 AS EXPECT_1 FROM mysql.gtid_slave_pos; +SELECT COUNT(*) < 1000 AS EXPECT_1 FROM mysql.gtid_slave_pos; +SELECT @@gtid_slave_pos,@@gtid_binlog_pos,@@gtid_current_pos; + +--connection node_2 +--echo # Waiting for data to replicate to node_2 +--let $wait_condition = SELECT COUNT(*) = 4000 FROM t1; +--source include/wait_condition.inc + +SELECT COUNT(*) AS EXPECT_4000 FROM t1; +SELECT COUNT(*) > 0 AS EXPECT_1 FROM mysql.gtid_slave_pos; +SELECT COUNT(*) < 1000 AS EXPECT_1 FROM mysql.gtid_slave_pos; +SELECT @@gtid_slave_pos,@@gtid_binlog_pos,@@gtid_current_pos; + +--connection primary2 +--echo # Waiting for data to replicate to node_3 +--let $wait_condition = SELECT COUNT(*) = 4000 FROM t1; +--source include/wait_condition.inc + +SELECT COUNT(*) AS EXPECT_4000 FROM t1; +SELECT COUNT(*) > 0 AS EXPECT_1 FROM mysql.gtid_slave_pos; +SELECT COUNT(*) < 1000 AS EXPECT_1 FROM mysql.gtid_slave_pos; +SELECT @@gtid_slave_pos,@@gtid_binlog_pos,@@gtid_current_pos; +# +# Cleanup +# +--connection primary1 +drop table t1; + +--echo # Wait until drop table is replicated on Galera +--connection replica1 +--let $wait_condition = SELECT COUNT(*) = 0 FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_NAME = 't1'; +--source include/wait_condition.inc + +--connection node_2 +--let $wait_condition = SELECT COUNT(*) = 0 FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_NAME = 't1'; +--source include/wait_condition.inc + +--connection primary2 +--let $wait_condition = SELECT COUNT(*) = 0 FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_NAME = 't1'; +--source include/wait_condition.inc + +--connection replica1 +STOP SLAVE; +RESET SLAVE ALL; + +--connection replica2 +STOP SLAVE; +RESET SLAVE ALL; +RESET MASTER; + +--source include/auto_increment_offset_restore.inc + +--connection node_1 +--disconnect primary1 +--disconnect replica1 +--disconnect primary2 +--disconnect replica2 + +--source include/galera_end.inc +--echo # End of test diff --git a/mysql-test/suite/galera/t/galera_restart_replica.test b/mysql-test/suite/galera/t/galera_restart_replica.test index 37cfd9bc0f936..05ab77f251935 100644 --- a/mysql-test/suite/galera/t/galera_restart_replica.test +++ b/mysql-test/suite/galera/t/galera_restart_replica.test @@ -1,77 +1,115 @@ # -# Test Galera as a replica to a MySQL async replication +# Test Galera as a replica to a MariaDB async replication +# +# MariaDB +# primary ---async replication--->galera node_2 (replica)<----galera replication---> galera node1 +# +# Test outline: +# +# - Create user for async replication +# - Create table and some data in primary +# - Verify that table and data is replicated to galera nodes +# - Verify that mysql.gtid_slave_pos has some rows in all Galera nodes +# - Verify that gtid_slave_pos, gtid_binlog_pos and gtid_current_pos are +# same in all Galera nodes +# - Verify that we can shutdown and restart Galera replica (node #2) +# - Verify that gtid_slave_pos, gtid_binlog_pos and gtid_current_pos are +# same in all Galera nodes +# - Verify that mysql.gtid_slave_pos table has limited amount of rows +# - Veruft that ddl works (drop table) +# +# Similar test is done so that new rows are added to table in +# primary while async replica (node #2) is down. # # The galera/galera_2node_slave.cnf describes the setup of the nodes # --source include/force_restart.inc --source include/galera_cluster.inc --source include/have_innodb.inc ---source include/have_sequence.inc + +# In this test we mark node #2 as replica +--connect replica, 127.0.0.1, root, , test, $NODE_MYPORT_2 # As node #3 is not a Galera node, and galera_cluster.inc does not open connetion to it -# we open the node_3 connection here ---connect node_3, 127.0.0.1, root, , test, $NODE_MYPORT_3 +# we open the primary connection her +--connect primary, 127.0.0.1, root, , test, $NODE_MYPORT_3 create user repl@'%' identified by 'repl'; grant all on *.* to repl@'%'; ALTER TABLE mysql.gtid_slave_pos ENGINE=InnoDB; --let $node_1 = node_1 ---let $node_2 = node_2 +--let $node_2 = replica --source include/auto_increment_offset_save.inc ---connection node_2 +--connection replica --disable_query_log --eval CHANGE MASTER TO master_host='127.0.0.1', master_user='repl', master_password='repl', master_port=$NODE_MYPORT_3, master_use_gtid=slave_pos; --enable_query_log START SLAVE; ---connection node_3 +--connection primary +CREATE TABLE t1 (id bigint auto_increment primary key, msg varchar(100)) engine=innodb; -CREATE TABLE t1 (id bigint primary key, msg varchar(100)) engine=innodb; +--echo # Intentionally generate 1k GTID-events +--let $inserts=1000 +--let $count=0 --disable_query_log -INSERT INTO t1 SELECT seq, 'test' from seq_1_to_10000; +while($count < $inserts) +{ + --eval insert into t1 values (NULL,'test1') + --inc $count +} --enable_query_log -SELECT COUNT(*) AS EXPECT_10000 FROM t1; ---connection node_2 +SELECT COUNT(*) AS EXPECT_1000 FROM t1; + +--connection replica --let $wait_condition = SELECT COUNT(*) = 1 FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_NAME = 't1'; --source include/wait_condition.inc ---let $wait_condition = SELECT COUNT(*) = 10000 FROM t1; +--let $wait_condition = SELECT COUNT(*) = 1000 FROM t1; --source include/wait_condition.inc +--let $wait_condition = SELECT COUNT(*) < 1000 FROM mysql.gtid_slave_pos; +--source include/wait_condition.inc SELECT COUNT(*) > 0 AS EXPECT_1 FROM mysql.gtid_slave_pos; -SELECT COUNT(*) AS EXPECT_10000 FROM t1; +SELECT COUNT(*) < 1000 AS EXPECT_1 FROM mysql.gtid_slave_pos; +SELECT @@gtid_slave_pos,@@gtid_binlog_pos,@@gtid_current_pos; +SELECT COUNT(*) AS EXPECT_1000 FROM t1; --connection node_1 --let $wait_condition = SELECT COUNT(*) = 1 FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_NAME = 't1'; --source include/wait_condition.inc ---let $wait_condition = SELECT COUNT(*) = 10000 FROM t1; +--let $wait_condition = SELECT COUNT(*) = 1000 FROM t1; --source include/wait_condition.inc +--let $wait_condition = SELECT COUNT(*) < 1000 FROM mysql.gtid_slave_pos; +--source include/wait_condition.inc SELECT COUNT(*) > 0 AS EXPECT_1 FROM mysql.gtid_slave_pos; -SELECT COUNT(*) AS EXPECT_10000 FROM t1; +SELECT COUNT(*) < 1000 AS EXPECT_1 FROM mysql.gtid_slave_pos; +SELECT @@gtid_slave_pos,@@gtid_binlog_pos,@@gtid_current_pos; +SELECT COUNT(*) AS EXPECT_1000 FROM t1; ---connection node_2 ---echo # Verify that graceful shutdown succeeds. +--connection replica +--echo # Verify that graceful shutdown succeeds in replica. --source include/shutdown_mysqld.inc --echo # Force SST --remove_file $MYSQLTEST_VARDIR/mysqld.2/data/grastate.dat --connection node_1 ---echo # Waiting until node_2 is not part of cluster anymore +--echo # Waiting until replica is not part of cluster anymore --let $wait_condition = SELECT VARIABLE_VALUE = 1 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_size'; --source include/wait_condition.inc --let $wait_condition = SELECT VARIABLE_VALUE = 'Primary' FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_status'; --source include/wait_condition.inc ---connection node_2 ---echo # Start node_2 again +--connection replica +--echo # Start replica again --source include/start_mysqld.inc ---echo ¤ Wait until node_2 is back on cluster +--echo # Wait until replica is back on cluster --let $wait_condition = SELECT VARIABLE_VALUE = 'Primary' FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_status'; --source include/wait_condition.inc --let $wait_condition = SELECT VARIABLE_VALUE = 2 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_size'; @@ -79,24 +117,30 @@ SELECT COUNT(*) AS EXPECT_10000 FROM t1; --let $wait_condition = SELECT VARIABLE_VALUE = 'ON' FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_ready'; --source include/wait_condition.inc ---connection node_2 +--let $wait_condition = SELECT COUNT(*) < 1000 FROM mysql.gtid_slave_pos; +--source include/wait_condition.inc SELECT COUNT(*) > 0 AS EXPECT_1 FROM mysql.gtid_slave_pos; -SELECT COUNT(*) AS EXPECT_10000 FROM t1; +SELECT COUNT(*) < 1000 AS EXPECT_1 FROM mysql.gtid_slave_pos; +SELECT @@gtid_slave_pos,@@gtid_binlog_pos,@@gtid_current_pos; +SELECT COUNT(*) AS EXPECT_1000 FROM t1; --connection node_1 +--let $wait_condition = SELECT COUNT(*) < 1000 FROM mysql.gtid_slave_pos; +--source include/wait_condition.inc SELECT COUNT(*) > 0 AS EXPECT_1 FROM mysql.gtid_slave_pos; -SELECT COUNT(*) AS EXPECT_10000 FROM t1; +SELECT COUNT(*) < 1000 AS EXPECT_1 FROM mysql.gtid_slave_pos; +SELECT @@gtid_slave_pos,@@gtid_binlog_pos,@@gtid_current_pos; +SELECT COUNT(*) AS EXPECT_1000 FROM t1; ---connection node_3 -SELECT COUNT(*) AS EXPECT_10000 FROM t1; +--connection primary +SELECT COUNT(*) AS EXPECT_1000 FROM t1; # # Cleanup # ---connection node_3 drop table t1; ---connection node_2 +--connection replica --let $wait_condition = SELECT COUNT(*) = 0 FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_NAME = 't1'; --source include/wait_condition.inc @@ -108,59 +152,80 @@ drop table t1; # Case 2 : While slave is down add writes to master # ---connection node_3 - -CREATE TABLE t1 (id bigint primary key, msg varchar(100)) engine=innodb; +--connection primary +CREATE TABLE t1 (id bigint auto_increment primary key, msg varchar(100)) engine=innodb; +--echo # Intentionally generate 1k GTID-events +--let $inserts=1000 +--let $count=0 --disable_query_log -INSERT INTO t1 SELECT seq, 'test' from seq_1_to_10000; +while($count < $inserts) +{ + --eval insert into t1 values (NULL,'test1') + --inc $count +} --enable_query_log -SELECT COUNT(*) AS EXPECT_10000 FROM t1; +SELECT COUNT(*) AS EXPECT_1000 FROM t1; ---connection node_2 +--connection replica --let $wait_condition = SELECT COUNT(*) = 1 FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_NAME = 't1'; --source include/wait_condition.inc ---let $wait_condition = SELECT COUNT(*) = 10000 FROM t1; +--let $wait_condition = SELECT COUNT(*) = 1000 FROM t1; --source include/wait_condition.inc +--let $wait_condition = SELECT COUNT(*) < 1000 FROM mysql.gtid_slave_pos; +--source include/wait_condition.inc SELECT COUNT(*) > 0 AS EXPECT_1 FROM mysql.gtid_slave_pos; -SELECT COUNT(*) AS EXPECT_10000 FROM t1; +SELECT COUNT(*) < 1000 AS EXPECT_1 FROM mysql.gtid_slave_pos; +SELECT @@gtid_slave_pos,@@gtid_binlog_pos,@@gtid_current_pos; +SELECT COUNT(*) AS EXPECT_1000 FROM t1; --connection node_1 --let $wait_condition = SELECT COUNT(*) = 1 FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_NAME = 't1'; --source include/wait_condition.inc ---let $wait_condition = SELECT COUNT(*) = 10000 FROM t1; +--let $wait_condition = SELECT COUNT(*) = 1000 FROM t1; --source include/wait_condition.inc +--let $wait_condition = SELECT COUNT(*) < 1000 FROM mysql.gtid_slave_pos; +--source include/wait_condition.inc SELECT COUNT(*) > 0 AS EXPECT_1 FROM mysql.gtid_slave_pos; -SELECT COUNT(*) AS EXPECT_10000 FROM t1; +SELECT COUNT(*) < 1000 AS EXPECT_1 FROM mysql.gtid_slave_pos; +SELECT @@gtid_slave_pos,@@gtid_binlog_pos,@@gtid_current_pos; ---connection node_2 ---echo # Verify that graceful shutdown succeeds. +--connection replica +--echo # Verify that graceful shutdown succeeds in replica. --source include/shutdown_mysqld.inc --echo # Force SST --remove_file $MYSQLTEST_VARDIR/mysqld.2/data/grastate.dat --connection node_1 ---echo # Waiting until node_2 is not part of cluster anymore +--echo # Waiting until replica is not part of cluster anymore --let $wait_condition = SELECT VARIABLE_VALUE = 1 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_size'; --source include/wait_condition.inc --let $wait_condition = SELECT VARIABLE_VALUE = 'Primary' FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_status'; --source include/wait_condition.inc -# Add writes to master ---connection node_3 +--echo # Add writes to primary +--connection primary +--echo # Intentionally generate 1k GTID-events +--let $inserts=1000 +--let $count=0 --disable_query_log -INSERT INTO t1 SELECT seq, 'test' from seq_20001_to_30000; +while($count < $inserts) +{ + --eval insert into t1 values (NULL,'test1') + --inc $count +} --enable_query_log -SELECT COUNT(*) AS EXPECT_20000 FROM t1; ---connection node_2 ---echo # Start node_2 again +SELECT COUNT(*) AS EXPECT_2000 FROM t1; + +--connection replica +--echo # Start replica again --source include/start_mysqld.inc ---echo ¤ Wait until node_2 is back on cluster +--echo # Wait until replica is back on cluster --let $wait_condition = SELECT VARIABLE_VALUE = 'Primary' FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_status'; --source include/wait_condition.inc --let $wait_condition = SELECT VARIABLE_VALUE = 2 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_size'; @@ -168,28 +233,34 @@ SELECT COUNT(*) AS EXPECT_20000 FROM t1; --let $wait_condition = SELECT VARIABLE_VALUE = 'ON' FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_ready'; --source include/wait_condition.inc ---connection node_2 ---let $wait_condition = SELECT COUNT(*) = 20000 FROM t1; +--let $wait_condition = SELECT COUNT(*) = 2000 FROM t1; +--source include/wait_condition.inc +--let $wait_condition = SELECT COUNT(*) < 1000 FROM mysql.gtid_slave_pos; --source include/wait_condition.inc SELECT COUNT(*) > 0 AS EXPECT_1 FROM mysql.gtid_slave_pos; -SELECT COUNT(*) AS EXPECT_20000 FROM t1; +SELECT COUNT(*) < 1000 AS EXPECT_1 FROM mysql.gtid_slave_pos; +SELECT @@gtid_slave_pos,@@gtid_binlog_pos,@@gtid_current_pos; +SELECT COUNT(*) AS EXPECT_2000 FROM t1; --connection node_1 ---let $wait_condition = SELECT COUNT(*) = 20000 FROM t1; +--let $wait_condition = SELECT COUNT(*) = 2000 FROM t1; +--source include/wait_condition.inc +--let $wait_condition = SELECT COUNT(*) < 1000 FROM mysql.gtid_slave_pos; --source include/wait_condition.inc SELECT COUNT(*) > 0 AS EXPECT_1 FROM mysql.gtid_slave_pos; -SELECT COUNT(*) AS EXPECT_20000 FROM t1; +SELECT COUNT(*) < 1000 AS EXPECT_1 FROM mysql.gtid_slave_pos; +SELECT @@gtid_slave_pos,@@gtid_binlog_pos,@@gtid_current_pos; +SELECT COUNT(*) AS EXPECT_2000 FROM t1; ---connection node_3 -SELECT COUNT(*) AS EXPECT_20000 FROM t1; +--connection primary +SELECT COUNT(*) AS EXPECT_2000 FROM t1; # # Cleanup # ---connection node_3 drop table t1; ---connection node_2 +--connection replica --let $wait_condition = SELECT COUNT(*) = 0 FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_NAME = 't1'; --source include/wait_condition.inc @@ -197,16 +268,18 @@ drop table t1; --let $wait_condition = SELECT COUNT(*) = 0 FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_NAME = 't1'; --source include/wait_condition.inc ---connection node_2 +--connection replica STOP SLAVE; RESET SLAVE ALL; ---connection node_3 +--connection primary RESET MASTER; +--source include/auto_increment_offset_restore.inc + --connection node_1 ---disconnect node_3 +--disconnect primary +--disconnect replica ---source include/auto_increment_offset_restore.inc --source include/galera_end.inc --echo # End of test diff --git a/sql/rpl_gtid.cc b/sql/rpl_gtid.cc index 93a83426d37be..8112a5b911db8 100644 --- a/sql/rpl_gtid.cc +++ b/sql/rpl_gtid.cc @@ -28,10 +28,6 @@ #include "rpl_rli.h" #include "slave.h" #include "log_event.h" -#ifdef WITH_WSREP -#include "wsrep_mysqld.h" // wsrep_thd_is_local -#include "wsrep_trans_observer.h" // wsrep_start_trx_if_not_started -#endif const LEX_CSTRING rpl_gtid_slave_state_table_name= { STRING_WITH_LEN("gtid_slave_pos") }; @@ -696,23 +692,7 @@ rpl_slave_state::record_gtid(THD *thd, const rpl_gtid *gtid, uint64 sub_id, goto end; #ifdef WITH_WSREP - /* - We should replicate local gtid_slave_pos updates to other nodes if - wsrep gtid mode is set. - In applier we should not append them to galera writeset. - */ - if (WSREP_ON_ && wsrep_gtid_mode && wsrep_thd_is_local(thd)) - { - thd->wsrep_ignore_table= false; - table->file->row_logging= 1; // replication requires binary logging - if (thd->wsrep_next_trx_id() == WSREP_UNDEFINED_TRX_ID) - thd->set_query_id(next_query_id()); - wsrep_start_trx_if_not_started(thd); - } - else - { - thd->wsrep_ignore_table= true; - } + thd->wsrep_ignore_table= true; // Do not replicate mysql.gtid_slave_pos table #endif if (!in_transaction) @@ -749,10 +729,6 @@ rpl_slave_state::record_gtid(THD *thd, const rpl_gtid *gtid, uint64 sub_id, } end: -#ifdef WITH_WSREP - thd->wsrep_ignore_table= false; -#endif - if (table_opened) { if (err || (err= ha_commit_trans(thd, FALSE))) @@ -764,6 +740,10 @@ rpl_slave_state::record_gtid(THD *thd, const rpl_gtid *gtid, uint64 sub_id, else thd->release_transactional_locks(); } + +#ifdef WITH_WSREP + thd->wsrep_ignore_table= false; +#endif thd->lex->restore_backup_query_tables_list(&lex_backup); thd->variables.option_bits= thd_saved_option; thd->resume_subsequent_commits(suspended_wfc); @@ -877,25 +857,7 @@ rpl_slave_state::gtid_delete_pending(THD *thd, return; #ifdef WITH_WSREP - /* - We should replicate local gtid_slave_pos updates to other nodes if - wsrep gtid mode is set. - In applier we should not append them to galera writeset. - */ - if (WSREP_ON_ && wsrep_gtid_mode && - wsrep_thd_is_local(thd) && - thd->wsrep_cs().state() != wsrep::client_state::s_none) - { - if (thd->wsrep_trx().active() == false) - { - if (thd->wsrep_next_trx_id() == WSREP_UNDEFINED_TRX_ID) - thd->set_query_id(next_query_id()); - wsrep_start_transaction(thd, thd->wsrep_next_trx_id()); - } - thd->wsrep_ignore_table= false; - } - else - thd->wsrep_ignore_table= true; + thd->wsrep_ignore_table= true; // No Galera replication for mysql.gtid_pos_table #endif thd_saved_option= thd->variables.option_bits; diff --git a/sql/wsrep_applier.cc b/sql/wsrep_applier.cc index feb78507bf93e..39df306a0e21a 100644 --- a/sql/wsrep_applier.cc +++ b/sql/wsrep_applier.cc @@ -22,6 +22,7 @@ #include "wsrep_xid.h" #include "wsrep_thd.h" #include "wsrep_trans_observer.h" +#include "wsrep_schema.h" // wsrep_schema #include "slave.h" // opt_log_slave_updates #include "debug_sync.h" @@ -180,6 +181,10 @@ int wsrep_apply_events(THD* thd, { thd->variables.gtid_seq_no= gtid_ev->seq_no; } + + if (wsrep_gtid_mode) + wsrep_schema->store_gtid_event(thd, gtid_ev); + delete ev; } continue; diff --git a/sql/wsrep_schema.cc b/sql/wsrep_schema.cc index 08e516b2a7a83..f29442abac1f1 100644 --- a/sql/wsrep_schema.cc +++ b/sql/wsrep_schema.cc @@ -1,4 +1,4 @@ -/* Copyright (C) 2015-2023 Codership Oy +/* Copyright (C) 2015-2025 Codership Oy This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -31,6 +31,8 @@ #include "wsrep_storage_service.h" #include "wsrep_thd.h" #include "wsrep_server_state.h" +#include "log_event.h" +#include "sql_class.h" #include #include @@ -165,6 +167,24 @@ class wsrep_off my_bool m_wsrep_on; }; +class wsrep_ignore_table +{ +public: + wsrep_ignore_table(THD* thd) + : m_thd(thd) + , m_wsrep_ignore_table(thd->wsrep_ignore_table) + { + thd->wsrep_ignore_table= true; + } + ~wsrep_ignore_table() + { + m_thd->wsrep_ignore_table= m_wsrep_ignore_table; + } +private: + THD* m_thd; + my_bool m_wsrep_ignore_table; +}; + class thd_server_status { public: @@ -1535,3 +1555,61 @@ int Wsrep_schema::recover_sr_transactions(THD *orig_thd) out: DBUG_RETURN(ret); } + +int Wsrep_schema::store_gtid_event(THD* thd, + const Gtid_log_event *gtid) +{ + DBUG_ENTER("Wsrep_schema::store_gtid_event"); + int error=0; + void *hton= NULL; + const bool in_transaction= (gtid->flags2 & Gtid_log_event::FL_TRANSACTIONAL); + const bool in_ddl= (gtid->flags2 & Gtid_log_event::FL_DDL); + + DBUG_PRINT("info", ("thd: %p, in_transaction: %d, in_ddl: %d " + "in_active_multi_stmt_transaction: %d", + thd, in_transaction, in_ddl, + thd->in_active_multi_stmt_transaction())); + + Wsrep_schema_impl::wsrep_ignore_table ignore_table(thd); + Wsrep_schema_impl::binlog_off binlog_off(thd); + Wsrep_schema_impl::sql_safe_updates sql_safe_updates(thd); + + rpl_group_info *rgi= thd->wsrep_rgi; + const uint64 sub_id= rpl_global_gtid_slave_state->next_sub_id(gtid->domain_id); + rpl_gtid current_gtid; + current_gtid.domain_id= gtid->domain_id; + current_gtid.server_id= gtid->server_id; + current_gtid.seq_no= gtid->seq_no; + rgi->gtid_pending= false; + + DBUG_ASSERT(!in_transaction || thd->in_active_multi_stmt_transaction()); + + if ((error= rpl_global_gtid_slave_state->record_gtid(thd, ¤t_gtid, + sub_id, + in_transaction, false, &hton))) + goto out; + + rpl_global_gtid_slave_state->update_state_hash(sub_id, ¤t_gtid, hton, rgi); + + if (in_ddl) + { + // Commit transaction if this GTID is part of DDL-clause because + // DDL causes implicit commit assuming there is no multi statement + // transaction ongoing. + if((error= trans_commit_stmt(thd))) + goto out; + + (void)trans_commit(thd); + } + +out: + if (error) + { + WSREP_DEBUG("Wsrep_schema::store_gtid_event %llu-%llu-%llu failed error=%s (%d).", + gtid->domain_id, gtid->server_id, gtid->seq_no, strerror(error), error); + (void)trans_rollback_stmt(thd); + (void)trans_rollback(thd); + } + + DBUG_RETURN(error); +} diff --git a/sql/wsrep_schema.h b/sql/wsrep_schema.h index f0f7904676887..46acba2961b88 100644 --- a/sql/wsrep_schema.h +++ b/sql/wsrep_schema.h @@ -1,4 +1,4 @@ -/* Copyright (C) 2015-2023 Codership Oy +/* Copyright (C) 2015-2024 Codership Oy This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -22,7 +22,6 @@ #include "mysqld.h" #include "wsrep_mysqld.h" - /* Forward decls */ @@ -32,6 +31,7 @@ struct TABLE; struct TABLE_LIST; struct st_mysql_lex_string; typedef struct st_mysql_lex_string LEX_STRING; +class Gtid_log_event; /** Name of the table in `wsrep_schema_str` used for storing streaming replication data. In an InnoDB full format, e.g. "database/tablename". */ @@ -133,6 +133,16 @@ class Wsrep_schema */ int recover_sr_transactions(THD* orig_thd); + /** + Store GTID-event to mysql.gtid_slave_pos table. + + @param thd The THD object of the calling thread. + @param gtid GTID event from binlog. + + @return Zero on success, non-zero on failure. + */ + int store_gtid_event(THD* thd, const Gtid_log_event *gtid); + private: /* Non-copyable */ Wsrep_schema(const Wsrep_schema&);