Merge pull request #1114 from France-ioi/mysql_8_0_28_on_circle

Use MySQL 8.0.28 on CircleCI

.circleci/config.yml

@@ -22,7 +22,7 @@ jobs:
   tests:
     docker:
       - image: cimg/go:1.20.2
-      - image: circleci/mysql:8.0.20
+      - image: circleci/mysql:8.0.28
         command: --default-authentication-plugin=mysql_native_password --max-allowed-packet=10485760
         environment:
           MYSQL_USER: algorea
@@ -86,7 +86,7 @@ jobs:
   migration-tests:
     docker:
       - image: cimg/go:1.20.2
-      - image: circleci/mysql:8.0.20
+      - image: circleci/mysql:8.0.28
         command: --default-authentication-plugin=mysql_native_password --max-allowed-packet=10485760
         environment:
           MYSQL_USER: franceioi
@@ -128,6 +128,9 @@ jobs:
       - run:
           name: Download a large dump of data
           command: curl -o - $LARGE_DUMP_URL | gunzip > dump.sql
+      - run:
+          name: Patch the dump data
+          command: sed -i "s/\`custom_prefix\` varchar(14) NOT NULL COMMENT \'Second part of the full login prefix, given by the user that created the batch\',/\`custom_prefix\` VARCHAR(14) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL COMMENT \'Second part of the full login prefix, given by the user that created the batch\',/g; s/CONSTRAINT \`ck_user_batches_custom_prefix\` CHECK (regexp_like(cast(\`custom_prefix\` as char charset binary),_utf8mb4'\\^\\[a-z0-9-\\]+\\$'))/CONSTRAINT \`ck_user_batches_custom_prefix\` CHECK (REGEXP_LIKE(\`custom_prefix\`, '^[a-z0-9-]+$'))/g" dump.sql
       - run:
           name: Seed database
           command: mysql -h 127.0.0.1 -u root -proot --protocol=TCP algorea_example < dump.sql

app/api/groups/create_user_batch.feature

@@ -90,7 +90,7 @@ Feature: Create a user batch
         ]
       }
       """
-    And the table "user_batches" should be:
+    And the table "user_batches_v2" should be:
       | group_prefix | custom_prefix | size | creator_id | created_at          |
       | test         | custom        | 2    | 21         | 2019-07-16 22:02:28 |
     And the table "users" should be:

app/api/groups/create_user_batch.robustness.feature

@@ -24,7 +24,7 @@ Feature: Create a user batch - robustness
       | test         | 3        | 1         | 2         |
       | test2        | 2        | 1         | 2         |
       | test3        | 3        | 0         | 2         |
-    And the database has the following table 'user_batches':
+    And the database has the following table 'user_batches_v2':
       | group_prefix | custom_prefix | size |
       | test         | custom        | 1    |
     And the application config is:
@@ -61,7 +61,7 @@ Feature: Create a user batch - robustness
       }
     }
     """
-    And the table "user_batches" should stay unchanged
+    And the table "user_batches_v2" should stay unchanged
     And the table "users" should stay unchanged
     And the table "groups" should stay unchanged
     And the table "groups_groups" should stay unchanged
@@ -95,7 +95,7 @@ Feature: Create a user batch - robustness
       }
     }
     """
-    And the table "user_batches" should stay unchanged
+    And the table "user_batches_v2" should stay unchanged
     And the table "users" should stay unchanged
     And the table "groups" should stay unchanged
     And the table "groups_groups" should stay unchanged
@@ -130,7 +130,7 @@ Feature: Create a user batch - robustness
       }
     }
     """
-    And the table "user_batches" should stay unchanged
+    And the table "user_batches_v2" should stay unchanged
     And the table "users" should stay unchanged
     And the table "groups" should stay unchanged
     And the table "groups_groups" should stay unchanged
@@ -164,7 +164,7 @@ Feature: Create a user batch - robustness
       }
     }
     """
-    And the table "user_batches" should stay unchanged
+    And the table "user_batches_v2" should stay unchanged
     And the table "users" should stay unchanged
     And the table "groups" should stay unchanged
     And the table "groups_groups" should stay unchanged
@@ -184,7 +184,7 @@ Feature: Create a user batch - robustness
     """
     Then the response code should be 403
     And the response error message should contain "Insufficient access rights"
-    And the table "user_batches" should stay unchanged
+    And the table "user_batches_v2" should stay unchanged
     And the table "users" should stay unchanged
     And the table "groups" should stay unchanged
     And the table "groups_groups" should stay unchanged
@@ -209,7 +209,7 @@ Feature: Create a user batch - robustness
     """
     Then the response code should be 400
     And the response error message should contain "'postfix_length' is too small"
-    And the table "user_batches" should stay unchanged
+    And the table "user_batches_v2" should stay unchanged
     And the table "users" should stay unchanged
     And the table "groups" should stay unchanged
     And the table "groups_groups" should stay unchanged
@@ -229,7 +229,7 @@ Feature: Create a user batch - robustness
     """
     Then the response code should be 403
     And the response error message should contain "Insufficient access rights"
-    And the table "user_batches" should stay unchanged
+    And the table "user_batches_v2" should stay unchanged
     And the table "users" should stay unchanged
     And the table "groups" should stay unchanged
     And the table "groups_groups" should stay unchanged
@@ -249,7 +249,7 @@ Feature: Create a user batch - robustness
     """
     Then the response code should be 403
     And the response error message should contain "Insufficient access rights"
-    And the table "user_batches" should stay unchanged
+    And the table "user_batches_v2" should stay unchanged
     And the table "users" should stay unchanged
     And the table "groups" should stay unchanged
     And the table "groups_groups" should stay unchanged
@@ -269,7 +269,7 @@ Feature: Create a user batch - robustness
     """
     Then the response code should be 400
     And the response error message should contain "'user_batch_prefix.max_users' exceeded"
-    And the table "user_batches" should stay unchanged
+    And the table "user_batches_v2" should stay unchanged
     And the table "users" should stay unchanged
     And the table "groups" should stay unchanged
     And the table "groups_groups" should stay unchanged
@@ -289,7 +289,7 @@ Feature: Create a user batch - robustness
     """
     Then the response code should be 400
     And the response error message should contain "'custom_prefix' already exists for the given 'group_prefix'"
-    And the table "user_batches" should stay unchanged
+    And the table "user_batches_v2" should stay unchanged
     And the table "users" should stay unchanged
     And the table "groups" should stay unchanged
     And the table "groups_groups" should stay unchanged
@@ -313,7 +313,7 @@ Feature: Create a user batch - robustness
     """
     Then the response code should be 500
     And the response error message should contain "Login module failed"
-    And the table "user_batches" should stay unchanged
+    And the table "user_batches_v2" should stay unchanged
     And the table "users" should stay unchanged
     And the table "groups" should stay unchanged
     And the table "groups_groups" should stay unchanged

app/api/groups/get_user_batch_prefixes.feature

@@ -28,7 +28,7 @@ Feature: List user-batch prefixes (userBatchPrefixesView)
       | test3        | 15       | 1         | 70        |
       | test4        | 14       | 1         | 60        |
       | test5        | 16       | 1         | 15        |
-    And the database has the following table 'user_batches':
+    And the database has the following table 'user_batches_v2':
       | group_prefix | custom_prefix | size | creator_id |
       | test         | custom        | 100  | null       |
       | test         | custom1       | 200  | 13         |

app/api/groups/get_user_batch_prefixes.go

@@ -97,8 +97,8 @@ func (srv *Service) getUserBatchPrefixes(w http.ResponseWriter, r *http.Request)
 		Where("user_batch_prefixes.group_id IN (?)", managedByUser.QueryExpr()).
 		Select(`
 			group_prefix, group_id, max_users,
-			(SELECT COUNT(*) FROM user_batches
-			 WHERE user_batches.group_prefix = user_batch_prefixes.group_prefix) AS total_size`)
+			(SELECT COUNT(*) FROM user_batches_v2
+			 WHERE user_batches_v2.group_prefix = user_batch_prefixes.group_prefix) AS total_size`)
 
 	query, apiErr := service.ApplySortingAndPaging(
 		r, query,

app/api/groups/get_user_batches.feature

@@ -22,7 +22,7 @@ Feature: List user batches (userBatchesView)
       | test2        | 13       | 0         |
       | test3        | 21       | 1         |
       | test4        | 14       | 1         |
-    And the database has the following table 'user_batches':
+    And the database has the following table 'user_batches_v2':
       | group_prefix | custom_prefix | size | creator_id |
       | test         | custom        | 100  | null       |
       | test         | custom1       | 200  | 13         |

app/api/groups/get_user_batches.go

@@ -92,15 +92,15 @@ func (srv *Service) getUserBatches(w http.ResponseWriter, r *http.Request) servi
 		Joins("JOIN user_batch_prefixes USING(group_prefix)").
 		Where(`user_batch_prefixes.group_id IN(?)`, managedByUser.QueryExpr()).
 		Where(`user_batch_prefixes.group_id IN(?)`, prefixAncestors.QueryExpr()).
-		Select("user_batches.group_prefix, user_batches.custom_prefix, user_batches.size, user_batches.creator_id")
+		Select("user_batches_v2.group_prefix, user_batches_v2.custom_prefix, user_batches_v2.size, user_batches_v2.creator_id")
 
 	query, apiErr := service.ApplySortingAndPaging(
 		r, query,
 		&service.SortingAndPagingParameters{
 			Fields: service.SortingAndPagingFields{
-				"group_prefix":  {ColumnName: "user_batches.group_prefix"},
-				"custom_prefix": {ColumnName: "user_batches.custom_prefix"},
-				"size":          {ColumnName: "user_batches.size"},
+				"group_prefix":  {ColumnName: "user_batches_v2.group_prefix"},
+				"custom_prefix": {ColumnName: "user_batches_v2.custom_prefix"},
+				"size":          {ColumnName: "user_batches_v2.size"},
 			},
 			DefaultRules: "group_prefix,custom_prefix",
 			TieBreakers: service.SortingAndPagingTieBreakers{

app/api/groups/remove_user_batch.feature

@@ -32,7 +32,7 @@ Feature: Remove user batch (userBatchRemove)
       | test         | 13       | 1         |
       | test1        | 13       | 1         |
       | test2        | 13       | 0         |
-    And the database has the following table 'user_batches':
+    And the database has the following table 'user_batches_v2':
       | group_prefix | custom_prefix | size | creator_id |
       | test         | custom        | 100  | null       |
       | test         | custom1       | 200  | 13         |
@@ -57,8 +57,8 @@ Feature: Remove user batch (userBatchRemove)
     """
     {"success": true, "message": "deleted"}
     """
-    And the table "user_batches" should stay unchanged but the rows with group_prefix "test"
-    And the table "user_batches" at group_prefix "test" should be:
+    And the table "user_batches_v2" should stay unchanged but the rows with group_prefix "test"
+    And the table "user_batches_v2" at group_prefix "test" should be:
       | group_prefix | custom_prefix | size | creator_id |
       | test         | custom1       | 200  | 13         |
     And the table "users" should stay unchanged but the rows with login "test_custom_user,test_custom_another_user"

app/api/groups/remove_user_batch.robustness.feature

@@ -37,7 +37,7 @@ Feature: Remove user batch (userBatchRemove) - robustness
       | test2        | 13       | 0         |
       | test3        | 21       | 1         |
       | test4        | 14       | 1         |
-    And the database has the following table 'user_batches':
+    And the database has the following table 'user_batches_v2':
       | group_prefix | custom_prefix | size | creator_id |
       | test         | custom        | 100  | null       |
       | test         | custom1       | 200  | 13         |
@@ -55,7 +55,7 @@ Feature: Remove user batch (userBatchRemove) - robustness
     When I send a DELETE request to "/user-batches/<group_prefix>/<custom_prefix>"
     Then the response code should be 403
     And the response error message should contain "Insufficient access rights"
-    And the table "user_batches" should stay unchanged
+    And the table "user_batches_v2" should stay unchanged
     And the table "users" should stay unchanged
     And the table "groups" should stay unchanged
   Examples:
@@ -68,7 +68,7 @@ Feature: Remove user batch (userBatchRemove) - robustness
     When I send a DELETE request to "/user-batches/test/custom"
     Then the response code should be 403
     And the response error message should contain "Insufficient access rights"
-    And the table "user_batches" should stay unchanged
+    And the table "user_batches_v2" should stay unchanged
     And the table "users" should stay unchanged
     And the table "groups" should stay unchanged
 
@@ -77,7 +77,7 @@ Feature: Remove user batch (userBatchRemove) - robustness
     When I send a DELETE request to "/user-batches/test/custom"
     Then the response code should be 422
     And the response error message should contain "There are users with locked membership"
-    And the table "user_batches" should stay unchanged
+    And the table "user_batches_v2" should stay unchanged
     And the table "users" should stay unchanged
     And the table "groups" should stay unchanged
     And logs should contain:
@@ -94,7 +94,7 @@ Feature: Remove user batch (userBatchRemove) - robustness
     When I send a DELETE request to "/user-batches/test1/custom"
     Then the response code should be 500
     And the response error message should contain "Login module failed"
-    And the table "user_batches" should stay unchanged
+    And the table "user_batches_v2" should stay unchanged
     And the table "users" should stay unchanged
     And the table "groups" should stay unchanged
     And logs should contain:

app/database/data_store.go

@@ -160,7 +160,7 @@ func (s *DataStore) Threads() *ThreadStore {
 
 // UserBatches returns a UserBatchStore.
 func (s *DataStore) UserBatches() *UserBatchStore {
-	return &UserBatchStore{NewDataStoreWithTable(s.DB, "user_batches")}
+	return &UserBatchStore{NewDataStoreWithTable(s.DB, "user_batches_v2")}
 }
 
 // UserBatchPrefixes returns a UserBatchPrefixStore.

app/database/data_store_test.go

@@ -44,7 +44,7 @@ func TestDataStore_StoreConstructorsSetTablesCorrectly(t *testing.T) {
 		{"AccessTokens", func(store *DataStore) *DB { return store.AccessTokens().Where("") }, "`access_tokens`"},
 		{"Threads", func(store *DataStore) *DB { return store.Threads().Where("") }, "`threads`"},
 		{"Users", func(store *DataStore) *DB { return store.Users().Where("") }, "`users`"},
-		{"UserBatches", func(store *DataStore) *DB { return store.UserBatches().Where("") }, "`user_batches`"},
+		{"UserBatches", func(store *DataStore) *DB { return store.UserBatches().Where("") }, "`user_batches_v2`"},
 		{"UserBatchPrefixes", func(store *DataStore) *DB { return store.UserBatchPrefixes().Where("") }, "`user_batch_prefixes`"},
 	}
 	for _, tt := range tests {

db/migrations/2002121859_create_tables_for_batches_of_users.sql

@@ -3,7 +3,7 @@ UPDATE `groups_login_prefixes`
     SET `prefix` = CONCAT(`prefix`, '_')
 WHERE `prefix` NOT LIKE '%\_';
 
-CREATE TABLE `user_batch_prefixes` (
+CREATE TABLE IF NOT EXISTS `user_batch_prefixes` (
     `group_prefix` VARCHAR(13) NOT NULL PRIMARY KEY COMMENT 'Prefix used in front of all batches',
     `group_id` BIGINT(20) DEFAULT NULL COMMENT 'Group (and its descendants) in which managers can create users in batch. NULL if the group was deleted, in which case batches should be cleaned manually.',
     `max_users` MEDIUMINT UNSIGNED NOT NULL DEFAULT 1000 COMMENT 'Maximum number of users that can be created under this prefix',
@@ -13,21 +13,21 @@ CREATE TABLE `user_batch_prefixes` (
 ) ENGINE=InnoDB CHARSET=utf8
     COMMENT='Authorized login prefixes for user batch creation. A prefix cannot be deleted without deleting batches using it.';
 
-CREATE TABLE `user_batches` (
+CREATE TABLE `user_batches_v2` (
     `group_prefix` VARCHAR(13) NOT NULL COMMENT 'Authorized (first) part of the full login prefix',
-    `custom_prefix` VARCHAR(14) NOT NULL
+    `custom_prefix` VARCHAR(14) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL
         COMMENT 'Second part of the full login prefix, given by the user that created the batch',
     `size` MEDIUMINT UNSIGNED NOT NULL COMMENT 'Number of users created in this batch',
     `creator_id` BIGINT(20) DEFAULT NULL,
     `created_at` DATETIME NOT NULL DEFAULT NOW(),
     PRIMARY KEY (`group_prefix`, `custom_prefix`),
-    CONSTRAINT `ck_user_batches_custom_prefix` CHECK (BINARY `custom_prefix` REGEXP '^[a-z0-9-]+$'),
-    CONSTRAINT `fk_user_batches_group_prefix_user_batch_prefixes_group_prefix`
+    CONSTRAINT `ck_user_batches_v2_custom_prefix` CHECK (REGEXP_LIKE(`custom_prefix`, '^[a-z0-9-]+$')),
+    CONSTRAINT `fk_user_batches_v2_group_prefix_user_batch_prefixes_group_pref`
         FOREIGN KEY (`group_prefix`) REFERENCES `user_batch_prefixes`(`group_prefix`) ON DELETE RESTRICT,
-    CONSTRAINT `fk_user_batches_creator_id_users_group_id`
+    CONSTRAINT `fk_user_batches_v2_creator_id_users_group_id`
         FOREIGN KEY (`creator_id`) REFERENCES `users`(`group_id`) ON DELETE SET NULL
 ) ENGINE=InnoDB CHARSET=utf8
-    COMMENT='Batches of users that were created';
+    COMMENT='Batches of users that were created (replaces user_batches which has been broken by a MySQL update)';
 
 INSERT INTO `user_batch_prefixes` (`group_prefix`, `group_id`, `max_users`)
 SELECT LEFT(LEFT(`prefix`, CHAR_LENGTH(`prefix`) - LOCATE('_', REVERSE(`prefix`), 2)), 13) AS group_prefix,
@@ -36,7 +36,7 @@ SELECT LEFT(LEFT(`prefix`, CHAR_LENGTH(`prefix`) - LOCATE('_', REVERSE(`prefix`)
 FROM `groups_login_prefixes`
 GROUP BY group_prefix;
 
-INSERT INTO `user_batches` (`group_prefix`, `custom_prefix`, `size`, `creator_id`, `created_at`)
+INSERT INTO `user_batches_v2` (`group_prefix`, `custom_prefix`, `size`, `creator_id`, `created_at`)
 SELECT LEFT(LEFT(`prefix`, CHAR_LENGTH(`prefix`) - LOCATE('_', REVERSE(`prefix`), 2)), 13) AS `group_prefix`,
        LEFT(REVERSE(SUBSTR(REVERSE(SUBSTRING_INDEX(`prefix`, '_', -2)), 2)), 14) AS `custom_prefix`,
        (SELECT COUNT(*) FROM `users` WHERE BINARY `login` LIKE REPLACE(CONCAT(`prefix`, '%'), '_', '\_') LIMIT 1) AS `size`,
@@ -67,13 +67,18 @@ INSERT INTO `groups_login_prefixes` (`group_id`, `prefix`)
 SELECT
     user_batch_prefixes.group_id,
     CONCAT(`group_prefix`, '_', `custom_prefix`) AS `prefix`
-FROM `user_batches` LEFT JOIN `user_batch_prefixes` USING (`group_prefix`)
+FROM `user_batches_v2` LEFT JOIN `user_batch_prefixes` USING (`group_prefix`)
 GROUP BY group_prefix, custom_prefix;
 
 ALTER TABLE `users`
     ADD COLUMN `login_module_prefix` varchar(100) DEFAULT NULL
         COMMENT 'Set to enable login module accounts manager'
             AFTER `notifications_read_at`;
 
-DROP TABLE `user_batches`;
-DROP TABLE `user_batch_prefixes`;
+DROP TABLE IF EXISTS `user_batches_v2`;
+/* We don't drop the table `user_batch_prefixes` because the broken user_batches references it */
+-- DROP TABLE `user_batch_prefixes`;
+/* Instead we empty it */
+SET FOREIGN_KEY_CHECKS = 0;
+DELETE FROM `user_batch_prefixes`;
+SET FOREIGN_KEY_CHECKS = 1;

db/migrations/2407171717_create_table_user_batches_new.sql

@@ -0,0 +1,21 @@
+-- +migrate Up
+
+CREATE TABLE IF NOT EXISTS `user_batches_v2` (
+    `group_prefix` VARCHAR(13) NOT NULL COMMENT 'Authorized (first) part of the full login prefix',
+    `custom_prefix` VARCHAR(14) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL
+        COMMENT 'Second part of the full login prefix, given by the user that created the batch',
+    `size` MEDIUMINT UNSIGNED NOT NULL COMMENT 'Number of users created in this batch',
+    `creator_id` BIGINT(20) DEFAULT NULL,
+    `created_at` DATETIME NOT NULL DEFAULT NOW(),
+    PRIMARY KEY (`group_prefix`, `custom_prefix`),
+    CONSTRAINT `ck_user_batches_v2_custom_prefix` CHECK (REGEXP_LIKE(`custom_prefix`, '^[a-z0-9-]+$')),
+    CONSTRAINT `fk_user_batches_v2_group_prefix_user_batch_prefixes_group_pref`
+        FOREIGN KEY (`group_prefix`) REFERENCES `user_batch_prefixes`(`group_prefix`) ON DELETE RESTRICT,
+    CONSTRAINT `fk_user_batches_v2_creator_id_users_group_id`
+        FOREIGN KEY (`creator_id`) REFERENCES `users`(`group_id`) ON DELETE SET NULL
+) ENGINE=InnoDB CHARSET=utf8
+    COMMENT='Batches of users that were created (replaces user_batches which has been broken by a MySQL update)';
+
+-- +migrate Down
+/* we pretend the table was always named `user_batches_v2`, so we don't need to drop it */
+;