From 2776c8e7c931fcad8d9796ac8052248d26eb81c2 Mon Sep 17 00:00:00 2001 From: Woongbae Yun Date: Thu, 22 Aug 2024 13:44:43 +0900 Subject: [PATCH] =?UTF-8?q?test:=20=ED=85=8C=EC=8A=A4=ED=8A=B8=EC=9A=A9=20?= =?UTF-8?q?bulk=20insert=EB=A5=BC=20=EC=B6=94=EA=B0=80=ED=95=9C=EB=8B=A4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../backend/common/BulkInsertRepository.kt | 72 +++++++++++++++++++ .../backend/config/DatabaseInitializer.kt | 56 +++++++++++++-- .../src/main/resources/application-dev.yml | 3 +- 3 files changed, 126 insertions(+), 5 deletions(-) create mode 100644 backend/src/main/kotlin/com/dclass/backend/common/BulkInsertRepository.kt diff --git a/backend/src/main/kotlin/com/dclass/backend/common/BulkInsertRepository.kt b/backend/src/main/kotlin/com/dclass/backend/common/BulkInsertRepository.kt new file mode 100644 index 0000000000..2fdc74ba40 --- /dev/null +++ b/backend/src/main/kotlin/com/dclass/backend/common/BulkInsertRepository.kt @@ -0,0 +1,72 @@ +package com.dclass.backend.common + +import com.dclass.backend.domain.blacklist.Blacklist +import com.dclass.backend.domain.post.Post +import jakarta.transaction.Transactional +import org.springframework.jdbc.core.BatchPreparedStatementSetter +import org.springframework.jdbc.core.JdbcTemplate +import org.springframework.stereotype.Repository +import java.sql.PreparedStatement + +@Repository +class BulkInsertRepository( + private val jdbcTemplate: JdbcTemplate +) { + + @Transactional + fun bulkInsert(values: List): Long? { + val sql = + """INSERT INTO post (community_id, content, created_date_time, deleted, is_anonymous, is_question, modified_date_time, comment_reply_count, like_count, scrap_count, title, user_id, version,id) + |VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""".trimMargin() + jdbcTemplate.batchUpdate( + sql, + object : BatchPreparedStatementSetter { + override fun setValues(ps: PreparedStatement, i: Int) { + val post = values[i] + ps.setLong(1, post.communityId) + ps.setString(2, post.content) + ps.setObject(3, post.createdDateTime) + ps.setBoolean(4, false) + ps.setBoolean(5, post.isAnonymous) + ps.setBoolean(6, post.isQuestion) + ps.setObject(7, post.modifiedDateTime) + ps.setInt(8, post.postCount.commentReplyCount) + ps.setInt(9, post.postCount.likeCount) + ps.setInt(10, post.postCount.scrapCount) + ps.setString(11, post.title) + ps.setLong(12, post.userId) + ps.setLong(13, post.version) + ps.setLong(14, post.id) + } + + override fun getBatchSize(): Int { + return values.size + } + }, + ) + return jdbcTemplate.queryForObject("SELECT LAST_INSERT_ID()", Long::class.java)!! + } + + @Transactional + fun bulkBlacklistInsert(values: List): Long? { + val sql = + """INSERT INTO blacklist (deleted, id, invalid_refresh_token) + |VALUES (?, ?, ?)""".trimMargin() + jdbcTemplate.batchUpdate( + sql, + object : BatchPreparedStatementSetter { + override fun setValues(ps: PreparedStatement, i: Int) { + val blacklists = values[i] + ps.setBoolean(1, false) + ps.setLong(2, blacklists.id) + ps.setString(3, blacklists.invalidRefreshToken) + } + + override fun getBatchSize(): Int { + return values.size + } + }, + ) + return jdbcTemplate.queryForObject("SELECT LAST_INSERT_ID()", Long::class.java)!! + } +} diff --git a/backend/src/main/kotlin/com/dclass/backend/config/DatabaseInitializer.kt b/backend/src/main/kotlin/com/dclass/backend/config/DatabaseInitializer.kt index cd1a10e555..2ced647373 100644 --- a/backend/src/main/kotlin/com/dclass/backend/config/DatabaseInitializer.kt +++ b/backend/src/main/kotlin/com/dclass/backend/config/DatabaseInitializer.kt @@ -3,8 +3,10 @@ package com.dclass.backend.config import com.dclass.backend.application.CommentService import com.dclass.backend.application.PostService import com.dclass.backend.application.ReplyService +import com.dclass.backend.common.BulkInsertRepository import com.dclass.backend.domain.belong.Belong import com.dclass.backend.domain.belong.BelongRepository +import com.dclass.backend.domain.blacklist.Blacklist import com.dclass.backend.domain.comment.Comment import com.dclass.backend.domain.comment.CommentRepository import com.dclass.backend.domain.community.Community @@ -20,6 +22,8 @@ import com.dclass.backend.domain.user.University import com.dclass.backend.domain.user.UniversityRepository import com.dclass.backend.domain.user.User import com.dclass.backend.domain.user.UserRepository +import com.dclass.backend.security.JwtTokenProvider +import jakarta.persistence.EntityManager import jakarta.transaction.Transactional import org.springframework.boot.CommandLineRunner import org.springframework.context.annotation.Profile @@ -31,6 +35,7 @@ import java.time.LocalDateTime @Component class DatabaseInitializer( private val database: Database, + private val entityManager: EntityManager, private val universityRepository: UniversityRepository, private val departmentRepository: DepartmentRepository, private val communityRepository: CommunityRepository, @@ -42,8 +47,20 @@ class DatabaseInitializer( private val commentService: CommentService, private val replyService: ReplyService, private val postService: PostService, + private val bulkInsertRepository: BulkInsertRepository, ) : CommandLineRunner { + private fun prev(){ + entityManager.createNativeQuery("set foreign_key_checks = 0").executeUpdate() + entityManager.createNativeQuery("set autocommit = 0").executeUpdate() + } + + private fun nxt(){ + entityManager.createNativeQuery("commit").executeUpdate() + entityManager.createNativeQuery("set foreign_key_checks = 1").executeUpdate() + entityManager.createNativeQuery("set autocommit = 1").executeUpdate() + } + override fun run(vararg args: String) { cleanUp() populate() @@ -58,6 +75,7 @@ class DatabaseInitializer( populateDepartment() populateCommunity() populateUser() +// populateBlacklist() // populateDummyPosts() // populateDummyComments() // populateDummyReplies() @@ -775,20 +793,50 @@ class DatabaseInitializer( val communityIds = (445L..450L).toList() val dummyPosts = mutableListOf() - + val batchSize = 10000 var number = 1 for (user in users) { - for (i in 1..10) { + for (i in 1..100000) { val communityId = communityIds.random() val community = communityRepository.findById(communityId).get() val post = createDummyPost(user, community, number) dummyPosts.add(post) number++ + + + if (dummyPosts.size >= batchSize) { + prev() + bulkInsertRepository.bulkInsert(dummyPosts) + nxt() + dummyPosts.clear() + } + } + } + } + + private fun populateBlacklist(){ + val blacklists = mutableListOf() + var number = 1L + val batchSize = 100000 + val totalSize = 100000 * 30 + for(i in 1..totalSize){ + val blacklist= Blacklist(invalidRefreshToken = generateRandomString(100),id=i.toLong()) + blacklists.add(blacklist) + if(blacklists.size >= batchSize){ + prev() + bulkInsertRepository.bulkBlacklistInsert(blacklists) + nxt() + blacklists.clear() } } + } - postRepository.saveAll(dummyPosts) + fun generateRandomString(length: Int): String { + val chars = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789" + return (1..length) + .map { chars.random() } + .joinToString("") } private fun createDummyPost(user: User, community: Community, number: Int): Post { @@ -796,7 +844,7 @@ class DatabaseInitializer( val content = "Dummy Post Content" val createdDateTime = LocalDateTime.now() - return Post(user.id, community.id, title, content, PostLikes(), createdDateTime = createdDateTime) + return Post(user.id, community.id, title, content, PostLikes(), createdDateTime = createdDateTime,id=number.toLong()) } private fun populateDummyComments() { diff --git a/backend/src/main/resources/application-dev.yml b/backend/src/main/resources/application-dev.yml index 2877b4133f..fbd95dae8e 100644 --- a/backend/src/main/resources/application-dev.yml +++ b/backend/src/main/resources/application-dev.yml @@ -9,6 +9,7 @@ spring: datasource: driver-class-name: com.mysql.cj.jdbc.Driver url: jdbc:mysql://localhost:3306/dclass?serverTimezone=UTC +# url: jdbc:mysql://localhost:3306/dclass?rewriteBatchedStatements=true&profileSQL=true&logger=Slf4JLogger&maxQuerySizeToLog=999999 username: dclass-user password: password @@ -39,4 +40,4 @@ springdoc: disable-swagger-default-url: true display-request-duration: true operations-sorter: alpha ---- \ No newline at end of file +---