From c346720e3b228d24da97b824b3748a0e63eefd46 Mon Sep 17 00:00:00 2001 From: songyuyuyu <88360486+songyuyuyu@users.noreply.github.com> Date: Mon, 20 Nov 2023 14:53:57 +0800 Subject: [PATCH] Feature/rebalance keeper (#744) * add keeper container comparator * add keeper instance in check action * add keeper info stats action * get all dc meta from console * add info action for redis * add redis instance for used memory * add redis info action when leader changes * temp * auto migrate keepers * add overload migration pages * fix keeper cotainer meta comparator test fail * fix DefaultDcMetaChangeManagerTest fail * add test for keeper info stats action * add test for redisInfoAction * add test for keeperContainerMetaComparator * temp * temp * add test for DefaultKeeperContainerUsedInfoAnalyzer * add test for AutoMigrateOverloadKeeperContainerAction * add keeper session manager * fix bean circle * add keeper related action when start * fix checker not report keeper result * add log and update choose redis strategy * add keeper session manager * adjust interval of keeper related actions * add alert when keeper migrate * clean expired keeper container used info * update session manager * update all dc meta of current dc with metacache --- .gitignore | 1 + .../redis/checker/CheckerConsoleService.java | 5 + .../xpipe/redis/checker/alert/ALERT_TYPE.java | 33 ++ .../redis/checker/config/CheckerConfig.java | 8 + .../controller/CheckerHealthController.java | 40 ++ .../AbstractHealthCheckAction.java | 1 + .../HealthCheckInstanceManager.java | 17 + .../KeeperHealthCheckActionFactory.java | 4 + .../KeeperHealthCheckActionListener.java | 4 + .../KeeperHealthCheckInstance.java | 11 + .../healthcheck/KeeperInstanceInfo.java | 19 + .../checker/healthcheck/KeeperSupport.java | 4 + .../keeper/AbstractKeeperInfoCommand.java | 29 + ...bstractKeeperInfoCommandActionFactory.java | 25 + .../keeper/AbstractKeeperInfoContext.java | 11 + .../keeper/KeeperStatsCheckAction.java | 14 + .../actions/keeper/info/RedisInfoAction.java | 31 + .../keeper/info/RedisInfoActionContext.java | 12 + .../info/RedisInfoActionController.java | 4 + .../keeper/info/RedisInfoActionFactory.java | 38 ++ .../keeper/info/RedisInfoActionListener.java | 7 + .../keeper/info/RedisUsedMemoryCollector.java | 71 +++ .../keeper/infoStats/KeeperFlowCollector.java | 49 ++ .../infoStats/KeeperInfoStatsAction.java | 31 + .../KeeperInfoStatsActionContext.java | 13 + .../KeeperInfoStatsActionController.java | 4 + .../KeeperInfoStatsActionFactory.java | 32 ++ .../KeeperInfoStatsActionListener.java | 7 + .../redisstats/AbstractInfoContext.java | 6 +- .../AbstractInstanceStatsCheckAction.java | 52 ++ .../redisstats/RedisStatsCheckAction.java | 42 +- .../CrdtInfoReplicationContext.java | 5 +- .../crdtinfostats/CrdtInfoStatsContext.java | 4 +- .../InfoReplicationContext.java | 2 +- .../infostats/InfoStatsContext.java | 2 +- .../config/AbstractHealthCheckConfig.java | 5 + .../config/CompositeHealthCheckConfig.java | 12 + .../healthcheck/config/HealthCheckConfig.java | 2 + .../DefaultHealthCheckEndpointFactory.java | 7 + .../DefaultHealthCheckInstanceFactory.java | 106 +++- .../DefaultHealthCheckInstanceManager.java | 67 ++- .../impl/DefaultHealthChecker.java | 31 + .../DefaultKeeperHealthCheckInstance.java | 56 ++ .../impl/DefaultKeeperInstanceInfo.java | 96 ++++ .../impl/HealthCheckEndpointFactory.java | 3 + .../impl/HealthCheckInstanceFactory.java | 7 + ...erLeaderAwareHealthCheckActionFactory.java | 33 ++ ...ctLeaderAwareHealthCheckActionFactory.java | 26 +- ...edLeaderAwareHealthCheckActionFactory.java | 25 + .../healthcheck/meta/DcMetaChangeManager.java | 2 +- .../meta/DefaultDcMetaChangeManager.java | 85 ++- .../meta/DefaultMetaChangeManager.java | 15 +- .../AbstractInstanceSessionManager.java | 225 ++++++++ .../session/DefaultKeeperSessionManager.java | 33 ++ .../session/DefaultRedisSessionManager.java | 184 +----- .../session/InstanceSessionManager.java | 16 + .../session/KeeperInstanceManager.java | 10 + .../session/KeeperSessionManager.java | 9 + .../session/RedisSessionManager.java | 8 +- .../stability/DefaultStabilityHolder.java | 20 +- .../stability/StabilityInspector.java | 27 +- .../impl/KeeperContainerInfoReporter.java | 127 +++++ .../impl/TestCurrentDcAllMetaCache.java | 16 + .../redis/checker/model/DcClusterShard.java | 21 +- .../checker/model/DcClusterShardPeer.java | 1 - .../model/KeeperContainerUsedInfoModel.java | 98 ++++ .../DefaultCheckerConsoleService.java | 24 +- .../AbstractCheckerIntegrationTest.java | 12 +- .../redis/checker/AbstractCheckerTest.java | 25 +- .../ctrip/xpipe/redis/checker/AllTests.java | 21 +- .../ctrip/xpipe/redis/checker/TestConfig.java | 12 +- .../info/RedisInfoActionFactoryTest.java | 30 + .../keeper/info/RedisInfoActionTest.java | 119 ++++ .../info/RedisUsedMemoryCollectorTest.java | 232 ++++++++ .../infoStats/KeeperFlowCollectorTest.java | 65 +++ .../KeeperInfoStatsActionFactoryTest.java | 58 ++ .../infoStats/KeeperInfoStatsActionTest.java | 92 +++ ...DefaultHealthCheckInstanceFactoryTest.java | 44 +- .../impl/DefaultHealthCheckerMockTest.java | 18 +- .../meta/DefaultDcMetaChangeManagerTest.java | 123 +++- .../DefaultKeeperSessionManagerTest.java | 167 ++++++ .../DefaultRedisSessionManagerTest.java | 169 ++++++ .../impl/KeeperContainerInfoReporterTest.java | 98 ++++ .../src/test/resources/dc-meta-test.xml | 35 +- .../redis/console/config/ConsoleConfig.java | 9 +- .../config/impl/DefaultConsoleConfig.java | 32 ++ .../KeeperContainerOverloadController.java | 34 ++ .../api/checker/ConsoleCheckerController.java | 20 + .../api/metaserver/ConsoleController.java | 1 - .../KeeperContainerInfoController.java | 32 ++ .../consoleportal/RedisController.java | 97 +--- ...oMigrateOverloadKeeperContainerAction.java | 127 +++++ .../keeper/KeeperContainerOverloadCause.java | 20 + .../KeeperContainerUsedInfoAnalyzer.java | 18 + ...efaultKeeperContainerUsedInfoAnalyzer.java | 305 ++++++++++ .../KeeperContainerOverloadStandardModel.java | 34 ++ .../MigrationKeeperContainerDetailModel.java | 119 ++++ .../model/MigrationKeeperDetailModel.java | 36 ++ .../console/resources/AbstractMetaCache.java | 9 +- .../resources/CheckerCurrentDcAllMeta.java | 95 ++++ .../console/resources/CheckerMetaCache.java | 1 - .../console/resources/DefaultMetaCache.java | 32 +- .../redis/console/service/AzService.java | 2 + .../service/KeeperAdvancedService.java | 7 +- .../KeeperContainerMigrationService.java | 11 + .../service/KeeperContainerService.java | 5 + .../console/service/impl/AzServiceImpl.java | 7 + .../impl/DefaultKeeperAdvancedService.java | 65 ++- ...efaultKeeperContainerMigrationService.java | 67 +++ .../impl/KeeperContainerServiceImpl.java | 25 + .../service/model/ShardModelService.java | 2 + .../model/impl/ShardModelServiceImpl.java | 32 ++ .../console/spring/CheckerContextConfig.java | 22 +- .../console/spring/ConsoleContextConfig.java | 7 + .../resources/META-INF/dal/jdbc/meta-dal.xml | 11 + .../src/main/resources/static/dist/bundle.js | 16 +- .../controllers/KeepercontainerOverloadCtl.ts | 103 ++++ .../main/resources/static/scripts/router.ts | 5 + .../services/KeeperContainerService.ts | 54 ++ .../views/index/keepercontainer_list.html | 3 +- .../views/index/keepercontainer_overload.html | 64 +++ .../ctrip/xpipe/redis/console/AllTests.java | 6 + ...rateOverloadKeeperContainerActionTest.java | 164 ++++++ ...ltKeeperContainerUsedInfoAnalyzerTest.java | 536 ++++++++++++++++++ .../CheckerCurrentDcAllMetaTest.java | 15 + ...ltKeeperContainerMigrationServiceTest.java | 79 +++ .../core/console/ConsoleCheckerPath.java | 4 + .../redis/core/meta/CurrentDcAllMeta.java | 14 + .../core/meta/KeeperContainerDetailInfo.java | 66 +++ .../AbstractInstanceNodeComparator.java | 63 ++ .../KeeperContainerMetaComparator.java | 116 ++++ .../meta/comparator/ShardMetaComparator.java | 53 +- .../protocal/cmd/InfoResultExtractor.java | 18 + .../xpipe/redis/core/AbstractRedisTest.java | 12 + .../com/ctrip/xpipe/redis/core/AllTests.java | 10 +- .../KeeperContainerMetaComparatorTest.java | 103 ++++ .../resources/KeeperContainerComparator.xml | 39 ++ .../console/TestCheckerContextConfig.java | 14 + .../console/TestConsoleContextConfig.java | 10 +- 139 files changed, 5691 insertions(+), 515 deletions(-) create mode 100644 redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/healthcheck/KeeperHealthCheckActionFactory.java create mode 100644 redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/healthcheck/KeeperHealthCheckActionListener.java create mode 100644 redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/healthcheck/KeeperHealthCheckInstance.java create mode 100644 redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/healthcheck/KeeperInstanceInfo.java create mode 100644 redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/healthcheck/KeeperSupport.java create mode 100644 redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/healthcheck/actions/keeper/AbstractKeeperInfoCommand.java create mode 100644 redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/healthcheck/actions/keeper/AbstractKeeperInfoCommandActionFactory.java create mode 100644 redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/healthcheck/actions/keeper/AbstractKeeperInfoContext.java create mode 100644 redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/healthcheck/actions/keeper/KeeperStatsCheckAction.java create mode 100644 redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/healthcheck/actions/keeper/info/RedisInfoAction.java create mode 100644 redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/healthcheck/actions/keeper/info/RedisInfoActionContext.java create mode 100644 redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/healthcheck/actions/keeper/info/RedisInfoActionController.java create mode 100644 redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/healthcheck/actions/keeper/info/RedisInfoActionFactory.java create mode 100644 redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/healthcheck/actions/keeper/info/RedisInfoActionListener.java create mode 100644 redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/healthcheck/actions/keeper/info/RedisUsedMemoryCollector.java create mode 100644 redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/healthcheck/actions/keeper/infoStats/KeeperFlowCollector.java create mode 100644 redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/healthcheck/actions/keeper/infoStats/KeeperInfoStatsAction.java create mode 100644 redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/healthcheck/actions/keeper/infoStats/KeeperInfoStatsActionContext.java create mode 100644 redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/healthcheck/actions/keeper/infoStats/KeeperInfoStatsActionController.java create mode 100644 redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/healthcheck/actions/keeper/infoStats/KeeperInfoStatsActionFactory.java create mode 100644 redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/healthcheck/actions/keeper/infoStats/KeeperInfoStatsActionListener.java create mode 100644 redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/healthcheck/actions/redisstats/AbstractInstanceStatsCheckAction.java create mode 100644 redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/healthcheck/impl/DefaultKeeperHealthCheckInstance.java create mode 100644 redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/healthcheck/impl/DefaultKeeperInstanceInfo.java create mode 100644 redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/healthcheck/leader/AbstractKeeperLeaderAwareHealthCheckActionFactory.java create mode 100644 redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/healthcheck/leader/AbstractRedisWithAssignedLeaderAwareHealthCheckActionFactory.java create mode 100644 redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/healthcheck/session/AbstractInstanceSessionManager.java create mode 100644 redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/healthcheck/session/DefaultKeeperSessionManager.java create mode 100644 redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/healthcheck/session/InstanceSessionManager.java create mode 100644 redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/healthcheck/session/KeeperInstanceManager.java create mode 100644 redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/healthcheck/session/KeeperSessionManager.java create mode 100644 redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/impl/KeeperContainerInfoReporter.java create mode 100644 redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/impl/TestCurrentDcAllMetaCache.java create mode 100644 redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/model/KeeperContainerUsedInfoModel.java create mode 100644 redis/redis-checker/src/test/java/com/ctrip/xpipe/redis/checker/healthcheck/actions/keeper/info/RedisInfoActionFactoryTest.java create mode 100644 redis/redis-checker/src/test/java/com/ctrip/xpipe/redis/checker/healthcheck/actions/keeper/info/RedisInfoActionTest.java create mode 100644 redis/redis-checker/src/test/java/com/ctrip/xpipe/redis/checker/healthcheck/actions/keeper/info/RedisUsedMemoryCollectorTest.java create mode 100644 redis/redis-checker/src/test/java/com/ctrip/xpipe/redis/checker/healthcheck/actions/keeper/infoStats/KeeperFlowCollectorTest.java create mode 100644 redis/redis-checker/src/test/java/com/ctrip/xpipe/redis/checker/healthcheck/actions/keeper/infoStats/KeeperInfoStatsActionFactoryTest.java create mode 100644 redis/redis-checker/src/test/java/com/ctrip/xpipe/redis/checker/healthcheck/actions/keeper/infoStats/KeeperInfoStatsActionTest.java create mode 100644 redis/redis-checker/src/test/java/com/ctrip/xpipe/redis/checker/healthcheck/session/DefaultKeeperSessionManagerTest.java create mode 100644 redis/redis-checker/src/test/java/com/ctrip/xpipe/redis/checker/healthcheck/session/DefaultRedisSessionManagerTest.java create mode 100644 redis/redis-checker/src/test/java/com/ctrip/xpipe/redis/checker/impl/KeeperContainerInfoReporterTest.java create mode 100644 redis/redis-console/src/main/java/com/ctrip/xpipe/redis/console/controller/api/KeeperContainerOverloadController.java create mode 100644 redis/redis-console/src/main/java/com/ctrip/xpipe/redis/console/keeper/AutoMigrateOverloadKeeperContainerAction.java create mode 100644 redis/redis-console/src/main/java/com/ctrip/xpipe/redis/console/keeper/KeeperContainerOverloadCause.java create mode 100644 redis/redis-console/src/main/java/com/ctrip/xpipe/redis/console/keeper/KeeperContainerUsedInfoAnalyzer.java create mode 100644 redis/redis-console/src/main/java/com/ctrip/xpipe/redis/console/keeper/impl/DefaultKeeperContainerUsedInfoAnalyzer.java create mode 100644 redis/redis-console/src/main/java/com/ctrip/xpipe/redis/console/model/KeeperContainerOverloadStandardModel.java create mode 100644 redis/redis-console/src/main/java/com/ctrip/xpipe/redis/console/model/MigrationKeeperContainerDetailModel.java create mode 100644 redis/redis-console/src/main/java/com/ctrip/xpipe/redis/console/model/MigrationKeeperDetailModel.java create mode 100644 redis/redis-console/src/main/java/com/ctrip/xpipe/redis/console/resources/CheckerCurrentDcAllMeta.java create mode 100644 redis/redis-console/src/main/java/com/ctrip/xpipe/redis/console/service/KeeperContainerMigrationService.java create mode 100644 redis/redis-console/src/main/java/com/ctrip/xpipe/redis/console/service/impl/DefaultKeeperContainerMigrationService.java create mode 100644 redis/redis-console/src/main/resources/static/scripts/controllers/KeepercontainerOverloadCtl.ts create mode 100644 redis/redis-console/src/main/resources/static/views/index/keepercontainer_overload.html create mode 100644 redis/redis-console/src/test/java/com/ctrip/xpipe/redis/console/keeper/AutoMigrateOverloadKeeperContainerActionTest.java create mode 100644 redis/redis-console/src/test/java/com/ctrip/xpipe/redis/console/keeper/impl/DefaultKeeperContainerUsedInfoAnalyzerTest.java create mode 100644 redis/redis-console/src/test/java/com/ctrip/xpipe/redis/console/resources/CheckerCurrentDcAllMetaTest.java create mode 100644 redis/redis-console/src/test/java/com/ctrip/xpipe/redis/console/service/impl/DefaultKeeperContainerMigrationServiceTest.java create mode 100644 redis/redis-core/src/main/java/com/ctrip/xpipe/redis/core/meta/CurrentDcAllMeta.java create mode 100644 redis/redis-core/src/main/java/com/ctrip/xpipe/redis/core/meta/KeeperContainerDetailInfo.java create mode 100644 redis/redis-core/src/main/java/com/ctrip/xpipe/redis/core/meta/comparator/AbstractInstanceNodeComparator.java create mode 100644 redis/redis-core/src/main/java/com/ctrip/xpipe/redis/core/meta/comparator/KeeperContainerMetaComparator.java create mode 100644 redis/redis-core/src/test/java/com/ctrip/xpipe/redis/core/meta/comparator/KeeperContainerMetaComparatorTest.java create mode 100644 redis/redis-core/src/test/resources/KeeperContainerComparator.xml diff --git a/.gitignore b/.gitignore index 546fbb43c..ee86ab4f9 100644 --- a/.gitignore +++ b/.gitignore @@ -40,3 +40,4 @@ hs_err_pid* # npm node_modules +*.js diff --git a/redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/CheckerConsoleService.java b/redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/CheckerConsoleService.java index c2c34035c..fa37e6ca4 100644 --- a/redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/CheckerConsoleService.java +++ b/redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/CheckerConsoleService.java @@ -7,6 +7,7 @@ import com.ctrip.xpipe.redis.checker.healthcheck.RedisHealthCheckInstance; import com.ctrip.xpipe.redis.checker.model.CheckerStatus; import com.ctrip.xpipe.redis.checker.model.HealthCheckResult; +import com.ctrip.xpipe.redis.checker.model.KeeperContainerUsedInfoModel; import com.ctrip.xpipe.redis.checker.model.ProxyTunnelInfo; import com.ctrip.xpipe.redis.core.entity.SentinelMeta; import com.ctrip.xpipe.redis.core.entity.XpipeMeta; @@ -25,12 +26,16 @@ public interface CheckerConsoleService { XpipeMeta getXpipeMeta(String console, int clusterPartIndex) throws SAXException, IOException; XpipeMeta getXpipeAllMeta(String console) throws SAXException, IOException; + + XpipeMeta getXpipeDcAllMeta(String console, String dcName) throws SAXException, IOException; List getProxyTunnelInfos(String console); void ack(String console, CheckerStatus checkerStatus); void report(String console, HealthCheckResult result); + + void reportKeeperContainerInfo(String console, List keeperContainerUsedInfoModels, int index); boolean isClusterOnMigration(String console, String clusterId); diff --git a/redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/alert/ALERT_TYPE.java b/redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/alert/ALERT_TYPE.java index 222e4fa68..b195d7c1c 100644 --- a/redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/alert/ALERT_TYPE.java +++ b/redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/alert/ALERT_TYPE.java @@ -570,8 +570,41 @@ public boolean reportRecovery() { public DetailDesc detailDesc() { return new DetailDesc("keepers should in different available zones", "keepers in the same available zone found"); } + }, + KEEPER_MIGRATION_SUCCESS("keeper migration success", EMAIL_XPIPE_ADMIN) { + @Override + public boolean urgent() { + return false; + } + + @Override + public boolean reportRecovery() { + return false; + } + + @Override + public DetailDesc detailDesc() { + return new DetailDesc("keeper migration success", "keeper migration success"); + } + }, + KEEPER_MIGRATION_FAIL("keeper migration fail", EMAIL_XPIPE_ADMIN) { + @Override + public boolean urgent() { + return false; + } + + @Override + public boolean reportRecovery() { + return false; + } + + @Override + public DetailDesc detailDesc() { + return new DetailDesc("keeper migration fail", "keeper migration fail"); + } }; + private String simpleDesc; private int alertMethod; diff --git a/redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/config/CheckerConfig.java b/redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/config/CheckerConfig.java index b3c5aa0ec..3c6648e3e 100644 --- a/redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/config/CheckerConfig.java +++ b/redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/config/CheckerConfig.java @@ -83,6 +83,8 @@ public interface CheckerConfig { String KEY_CHECKER_META_REFRESH_INTERVAL = "checker.meta.refresh.interval.milli"; + String KEY_CHECKER_CURRENT_DC_ALL_META_REFRESH_INTERVAL = "checker.current_dc_all_meta.refresh.interval.milli"; + String KEY_CONSOLE_ADDRESS = "console.address"; String KEY_CHECKER_ACK_INTERVAL = "checker.ack.interval.milli"; @@ -101,8 +103,12 @@ public interface CheckerConfig { String KEY_SUBSCRIBE_TIMEOUT_MILLI = "checker.subscribe.timeout.milli"; + String KEY_KEEPER_CHECKER_INTERVAL = "keeper.checker.interval"; + int getRedisReplicationHealthCheckInterval(); + int getCheckerCurrentDcAllMetaRefreshIntervalMilli(); + int getClusterHealthCheckInterval(); int getDownAfterCheckNums(); @@ -201,4 +207,6 @@ public interface CheckerConfig { Set getMigrationUnsupportedClusters(); + int getKeeperCheckerIntervalMilli(); + } diff --git a/redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/controller/CheckerHealthController.java b/redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/controller/CheckerHealthController.java index 173b1707b..acf6b2e29 100644 --- a/redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/controller/CheckerHealthController.java +++ b/redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/controller/CheckerHealthController.java @@ -8,8 +8,11 @@ import com.ctrip.xpipe.redis.checker.healthcheck.actions.interaction.DefaultDelayPingActionCollector; import com.ctrip.xpipe.redis.checker.healthcheck.actions.interaction.HEALTH_STATE; import com.ctrip.xpipe.redis.checker.healthcheck.actions.interaction.HealthStatusDesc; +import com.ctrip.xpipe.redis.checker.healthcheck.actions.keeper.info.RedisUsedMemoryCollector; +import com.ctrip.xpipe.redis.checker.healthcheck.actions.keeper.infoStats.KeeperFlowCollector; import com.ctrip.xpipe.redis.checker.healthcheck.actions.redisconf.AbstractRedisConfigRuleAction; import com.ctrip.xpipe.redis.checker.healthcheck.stability.StabilityHolder; +import com.ctrip.xpipe.redis.checker.model.DcClusterShard; import com.google.common.collect.Lists; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; @@ -18,6 +21,7 @@ import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.concurrent.ConcurrentMap; /** * @author lishanglin @@ -31,6 +35,12 @@ public class CheckerHealthController { @Autowired private DefaultDelayPingActionCollector defaultDelayPingActionCollector; + @Autowired + private RedisUsedMemoryCollector redisUsedMemoryCollector; + + @Autowired + private KeeperFlowCollector keeperFlowCollector; + @Autowired private HealthCheckInstanceManager instanceManager; @@ -66,6 +76,26 @@ public String getClusterHealthCheckInstance(@PathVariable String clusterId) { return Codec.DEFAULT.encode(model); } + @RequestMapping(value = "/health/check/keeper/{ip}/{port}", method = RequestMethod.GET) + public String getHealthCheckKeeper(@PathVariable String ip, @PathVariable int port) { + KeeperHealthCheckInstance instance = instanceManager.findKeeperHealthCheckInstance(new HostPort(ip, port)); + if(instance == null) { + return "Not found"; + } + HealthCheckInstanceModel model = buildHealthCheckInfo(instance); + return Codec.DEFAULT.encode(model); + } + + @RequestMapping(value = "/health/check/redis-for-assigned-action/{ip}/{port}", method = RequestMethod.GET) + public String getHealthCheckRedisInstanceForAssignedAction(@PathVariable String ip, @PathVariable int port) { + RedisHealthCheckInstance instance = instanceManager.findRedisInstanceForAssignedAction(new HostPort(ip, port)); + if(instance == null) { + return "Not found"; + } + HealthCheckInstanceModel model = buildHealthCheckInfo(instance); + return Codec.DEFAULT.encode(model); + } + @RequestMapping(value = "/health/redis/info/{ip}/{port}", method = RequestMethod.GET) public ActionContextRetMessage> getRedisInfo(@PathVariable String ip, @PathVariable int port) { return ActionContextRetMessage.from(redisInfoManager.getInfoByHostPort(new HostPort(ip, port))); @@ -82,6 +112,16 @@ public Map getAllHealthStatusDesc() { else return Collections.emptyMap(); } + @GetMapping("/health/keeper/status/all") + public ConcurrentMap> getAllKeeperFlows() { + return keeperFlowCollector.getHostPort2InputFlow(); + } + + @GetMapping("/health/redis/used-memory/all") + public ConcurrentMap getAllDclusterShardUsedMemory() { + return redisUsedMemoryCollector.getDcClusterShardUsedMemory(); + } + private HealthCheckInstanceModel buildHealthCheckInfo(HealthCheckInstance instance) { HealthCheckInstanceModel model = new HealthCheckInstanceModel(instance.toString()); for(HealthCheckAction action : instance.getHealthCheckActions()) { diff --git a/redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/healthcheck/AbstractHealthCheckAction.java b/redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/healthcheck/AbstractHealthCheckAction.java index 277e7fcda..205044ade 100644 --- a/redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/healthcheck/AbstractHealthCheckAction.java +++ b/redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/healthcheck/AbstractHealthCheckAction.java @@ -51,6 +51,7 @@ public void doStart() { @Override public void doStop() { + logger.debug("[stopped][{}][{}], listener:{}, future:{}", getClass().getSimpleName(), instance.getCheckInfo(), listeners, future); if(future != null) { future.cancel(true); } diff --git a/redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/healthcheck/HealthCheckInstanceManager.java b/redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/healthcheck/HealthCheckInstanceManager.java index c6f94bf02..30f1e0aa8 100644 --- a/redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/healthcheck/HealthCheckInstanceManager.java +++ b/redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/healthcheck/HealthCheckInstanceManager.java @@ -2,6 +2,7 @@ import com.ctrip.xpipe.endpoint.HostPort; import com.ctrip.xpipe.redis.core.entity.ClusterMeta; +import com.ctrip.xpipe.redis.core.entity.KeeperMeta; import com.ctrip.xpipe.redis.core.entity.RedisMeta; import java.util.List; @@ -15,18 +16,34 @@ public interface HealthCheckInstanceManager { RedisHealthCheckInstance getOrCreate(RedisMeta redis); + RedisHealthCheckInstance getOrCreateRedisInstanceForAssignedAction(RedisMeta redis); + + KeeperHealthCheckInstance getOrCreate(KeeperMeta keeper); + ClusterHealthCheckInstance getOrCreate(ClusterMeta cluster); RedisHealthCheckInstance findRedisHealthCheckInstance(HostPort hostPort); + RedisHealthCheckInstance findRedisInstanceForAssignedAction(HostPort hostPort); + + KeeperHealthCheckInstance findKeeperHealthCheckInstance(HostPort hostPort); + ClusterHealthCheckInstance findClusterHealthCheckInstance(String clusterId); RedisHealthCheckInstance remove(HostPort hostPort); + KeeperHealthCheckInstance removeKeeper(HostPort hostPort); + + RedisHealthCheckInstance removeRedisInstanceForAssignedAction(HostPort hostPort); + ClusterHealthCheckInstance remove(String cluster); List getAllRedisInstance(); + List getAllKeeperInstance(); + + List getAllRedisInstanceForAssignedAction(); + List getAllClusterInstance(); } diff --git a/redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/healthcheck/KeeperHealthCheckActionFactory.java b/redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/healthcheck/KeeperHealthCheckActionFactory.java new file mode 100644 index 000000000..893952d79 --- /dev/null +++ b/redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/healthcheck/KeeperHealthCheckActionFactory.java @@ -0,0 +1,4 @@ +package com.ctrip.xpipe.redis.checker.healthcheck; + +public interface KeeperHealthCheckActionFactory extends HealthCheckActionFactory{ +} diff --git a/redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/healthcheck/KeeperHealthCheckActionListener.java b/redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/healthcheck/KeeperHealthCheckActionListener.java new file mode 100644 index 000000000..7fae0bbf9 --- /dev/null +++ b/redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/healthcheck/KeeperHealthCheckActionListener.java @@ -0,0 +1,4 @@ +package com.ctrip.xpipe.redis.checker.healthcheck; + +public interface KeeperHealthCheckActionListener extends HealthCheckActionListener> { +} diff --git a/redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/healthcheck/KeeperHealthCheckInstance.java b/redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/healthcheck/KeeperHealthCheckInstance.java new file mode 100644 index 000000000..13f7da5a3 --- /dev/null +++ b/redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/healthcheck/KeeperHealthCheckInstance.java @@ -0,0 +1,11 @@ +package com.ctrip.xpipe.redis.checker.healthcheck; + +import com.ctrip.xpipe.api.endpoint.Endpoint; +import com.ctrip.xpipe.redis.checker.healthcheck.session.RedisSession; + +public interface KeeperHealthCheckInstance extends HealthCheckInstance{ + + Endpoint getEndpoint(); + + RedisSession getRedisSession(); +} diff --git a/redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/healthcheck/KeeperInstanceInfo.java b/redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/healthcheck/KeeperInstanceInfo.java new file mode 100644 index 000000000..9d89158a8 --- /dev/null +++ b/redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/healthcheck/KeeperInstanceInfo.java @@ -0,0 +1,19 @@ +package com.ctrip.xpipe.redis.checker.healthcheck; + +import com.ctrip.xpipe.endpoint.ClusterShardHostPort; +import com.ctrip.xpipe.endpoint.HostPort; + + +public interface KeeperInstanceInfo extends CheckInfo { + + ClusterShardHostPort getClusterShardHostport(); + + String getShardId(); + + String getDcId(); + + boolean isActive(); + + HostPort getHostPort(); + +} diff --git a/redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/healthcheck/KeeperSupport.java b/redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/healthcheck/KeeperSupport.java new file mode 100644 index 000000000..90d9effbd --- /dev/null +++ b/redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/healthcheck/KeeperSupport.java @@ -0,0 +1,4 @@ +package com.ctrip.xpipe.redis.checker.healthcheck; + +public interface KeeperSupport { +} diff --git a/redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/healthcheck/actions/keeper/AbstractKeeperInfoCommand.java b/redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/healthcheck/actions/keeper/AbstractKeeperInfoCommand.java new file mode 100644 index 000000000..febdd1ae9 --- /dev/null +++ b/redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/healthcheck/actions/keeper/AbstractKeeperInfoCommand.java @@ -0,0 +1,29 @@ +package com.ctrip.xpipe.redis.checker.healthcheck.actions.keeper; + +import com.ctrip.xpipe.redis.checker.healthcheck.AbstractActionContext; +import com.ctrip.xpipe.redis.checker.healthcheck.KeeperHealthCheckInstance; +import com.ctrip.xpipe.redis.core.protocal.cmd.InfoResultExtractor; +import org.slf4j.Logger; + +import java.util.concurrent.ExecutorService; +import java.util.concurrent.ScheduledExecutorService; + +public abstract class AbstractKeeperInfoCommand extends KeeperStatsCheckAction { + + public AbstractKeeperInfoCommand(ScheduledExecutorService scheduled, KeeperHealthCheckInstance instance, ExecutorService executors) { + super(scheduled, instance, executors); + } + + @Override + protected Logger getHealthCheckLogger() { + return logger; + } + + @Override + protected T generateActionContext(String result) { + return createActionContext(result); + } + + protected abstract T createActionContext(String extractor); + +} diff --git a/redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/healthcheck/actions/keeper/AbstractKeeperInfoCommandActionFactory.java b/redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/healthcheck/actions/keeper/AbstractKeeperInfoCommandActionFactory.java new file mode 100644 index 000000000..fda987a72 --- /dev/null +++ b/redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/healthcheck/actions/keeper/AbstractKeeperInfoCommandActionFactory.java @@ -0,0 +1,25 @@ +package com.ctrip.xpipe.redis.checker.healthcheck.actions.keeper; + +import com.ctrip.xpipe.redis.checker.healthcheck.KeeperHealthCheckInstance; +import com.ctrip.xpipe.redis.checker.healthcheck.actions.redisstats.AbstractInfoListener; +import com.ctrip.xpipe.redis.checker.healthcheck.leader.AbstractKeeperLeaderAwareHealthCheckActionFactory; +import com.ctrip.xpipe.redis.checker.healthcheck.leader.SiteLeaderAwareHealthCheckAction; +import org.springframework.beans.factory.annotation.Autowired; + +import java.util.List; + +public abstract class AbstractKeeperInfoCommandActionFactory + extends AbstractKeeperLeaderAwareHealthCheckActionFactory { + @Autowired + private List listeners; + + @Override + public SiteLeaderAwareHealthCheckAction create(KeeperHealthCheckInstance instance) { + C action = createAction(instance); + action.addListeners(listeners); + return action; + } + + protected abstract C createAction(KeeperHealthCheckInstance instance); + +} diff --git a/redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/healthcheck/actions/keeper/AbstractKeeperInfoContext.java b/redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/healthcheck/actions/keeper/AbstractKeeperInfoContext.java new file mode 100644 index 000000000..83b820e71 --- /dev/null +++ b/redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/healthcheck/actions/keeper/AbstractKeeperInfoContext.java @@ -0,0 +1,11 @@ +package com.ctrip.xpipe.redis.checker.healthcheck.actions.keeper; + +import com.ctrip.xpipe.redis.checker.healthcheck.AbstractActionContext; +import com.ctrip.xpipe.redis.checker.healthcheck.KeeperHealthCheckInstance; +import com.ctrip.xpipe.redis.core.protocal.cmd.InfoResultExtractor; + +public abstract class AbstractKeeperInfoContext extends AbstractActionContext { + public AbstractKeeperInfoContext(KeeperHealthCheckInstance instance, InfoResultExtractor infoResultExtractor) { + super(instance, infoResultExtractor); + } +} diff --git a/redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/healthcheck/actions/keeper/KeeperStatsCheckAction.java b/redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/healthcheck/actions/keeper/KeeperStatsCheckAction.java new file mode 100644 index 000000000..73d8e80f1 --- /dev/null +++ b/redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/healthcheck/actions/keeper/KeeperStatsCheckAction.java @@ -0,0 +1,14 @@ +package com.ctrip.xpipe.redis.checker.healthcheck.actions.keeper; + +import com.ctrip.xpipe.redis.checker.healthcheck.KeeperHealthCheckInstance; +import com.ctrip.xpipe.redis.checker.healthcheck.actions.redisstats.AbstractInstanceStatsCheckAction; + +import java.util.concurrent.ExecutorService; +import java.util.concurrent.ScheduledExecutorService; + +public abstract class KeeperStatsCheckAction extends AbstractInstanceStatsCheckAction { + + public KeeperStatsCheckAction(ScheduledExecutorService scheduled, KeeperHealthCheckInstance instance, ExecutorService executors) { + super(scheduled, instance, executors); + } +} diff --git a/redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/healthcheck/actions/keeper/info/RedisInfoAction.java b/redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/healthcheck/actions/keeper/info/RedisInfoAction.java new file mode 100644 index 000000000..d9faf2970 --- /dev/null +++ b/redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/healthcheck/actions/keeper/info/RedisInfoAction.java @@ -0,0 +1,31 @@ +package com.ctrip.xpipe.redis.checker.healthcheck.actions.keeper.info; + +import com.ctrip.xpipe.api.command.CommandFuture; +import com.ctrip.xpipe.redis.checker.healthcheck.RedisHealthCheckInstance; +import com.ctrip.xpipe.redis.checker.healthcheck.actions.redisstats.AbstractInfoCommandAction; +import com.ctrip.xpipe.redis.checker.healthcheck.session.Callbackable; + +import java.util.concurrent.ExecutorService; +import java.util.concurrent.ScheduledExecutorService; + +public class RedisInfoAction extends AbstractInfoCommandAction { + + public RedisInfoAction(ScheduledExecutorService scheduled, RedisHealthCheckInstance instance, ExecutorService executors) { + super(scheduled, instance, executors); + } + + @Override + protected RedisInfoActionContext createActionContext(String extractor) { + return new RedisInfoActionContext(instance, extractor); + } + + @Override + protected CommandFuture executeRedisCommandForStats(Callbackable callback) { + return getActionInstance().getRedisSession().info("", callback); + } + + @Override + protected int getBaseCheckInterval() { + return getActionInstance().getHealthCheckConfig().getKeeperCheckerIntervalMilli(); + } +} diff --git a/redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/healthcheck/actions/keeper/info/RedisInfoActionContext.java b/redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/healthcheck/actions/keeper/info/RedisInfoActionContext.java new file mode 100644 index 000000000..f23a7dd49 --- /dev/null +++ b/redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/healthcheck/actions/keeper/info/RedisInfoActionContext.java @@ -0,0 +1,12 @@ +package com.ctrip.xpipe.redis.checker.healthcheck.actions.keeper.info; + +import com.ctrip.xpipe.redis.checker.healthcheck.RedisHealthCheckInstance; +import com.ctrip.xpipe.redis.checker.healthcheck.actions.redisstats.AbstractInfoContext; +import com.ctrip.xpipe.redis.core.protocal.cmd.InfoResultExtractor; + +public class RedisInfoActionContext extends AbstractInfoContext { + + public RedisInfoActionContext(RedisHealthCheckInstance instance, String extractor) { + super(instance, new InfoResultExtractor(extractor)); + } +} diff --git a/redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/healthcheck/actions/keeper/info/RedisInfoActionController.java b/redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/healthcheck/actions/keeper/info/RedisInfoActionController.java new file mode 100644 index 000000000..f474d7e3c --- /dev/null +++ b/redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/healthcheck/actions/keeper/info/RedisInfoActionController.java @@ -0,0 +1,4 @@ +package com.ctrip.xpipe.redis.checker.healthcheck.actions.keeper.info; + +public interface RedisInfoActionController { +} diff --git a/redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/healthcheck/actions/keeper/info/RedisInfoActionFactory.java b/redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/healthcheck/actions/keeper/info/RedisInfoActionFactory.java new file mode 100644 index 000000000..b47020afc --- /dev/null +++ b/redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/healthcheck/actions/keeper/info/RedisInfoActionFactory.java @@ -0,0 +1,38 @@ +package com.ctrip.xpipe.redis.checker.healthcheck.actions.keeper.info; + +import com.ctrip.xpipe.redis.checker.alert.ALERT_TYPE; +import com.ctrip.xpipe.redis.checker.healthcheck.KeeperSupport; +import com.ctrip.xpipe.redis.checker.healthcheck.OneWaySupport; +import com.ctrip.xpipe.redis.checker.healthcheck.RedisHealthCheckInstance; +import com.ctrip.xpipe.redis.checker.healthcheck.leader.AbstractRedisWithAssignedLeaderAwareHealthCheckActionFactory; +import com.ctrip.xpipe.redis.checker.healthcheck.leader.SiteLeaderAwareHealthCheckAction; +import com.google.common.collect.Lists; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +import java.util.List; + +@Component +public class RedisInfoActionFactory extends AbstractRedisWithAssignedLeaderAwareHealthCheckActionFactory + implements KeeperSupport, OneWaySupport { + + @Autowired + private List listeners; + + @Override + protected List alertTypes() { + return Lists.newArrayList(); + } + + @Override + public RedisInfoAction create(RedisHealthCheckInstance instance) { + RedisInfoAction action = new RedisInfoAction(scheduled, instance, executors); + action.addListeners(listeners); + return action; + } + + @Override + public Class support() { + return RedisInfoAction.class; + } +} diff --git a/redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/healthcheck/actions/keeper/info/RedisInfoActionListener.java b/redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/healthcheck/actions/keeper/info/RedisInfoActionListener.java new file mode 100644 index 000000000..20daa45d0 --- /dev/null +++ b/redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/healthcheck/actions/keeper/info/RedisInfoActionListener.java @@ -0,0 +1,7 @@ +package com.ctrip.xpipe.redis.checker.healthcheck.actions.keeper.info; + +import com.ctrip.xpipe.redis.checker.healthcheck.actions.redisstats.AbstractInfoListener; + +public interface RedisInfoActionListener extends AbstractInfoListener { + +} diff --git a/redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/healthcheck/actions/keeper/info/RedisUsedMemoryCollector.java b/redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/healthcheck/actions/keeper/info/RedisUsedMemoryCollector.java new file mode 100644 index 000000000..724a1943d --- /dev/null +++ b/redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/healthcheck/actions/keeper/info/RedisUsedMemoryCollector.java @@ -0,0 +1,71 @@ +package com.ctrip.xpipe.redis.checker.healthcheck.actions.keeper.info; + +import com.ctrip.xpipe.redis.checker.healthcheck.HealthCheckAction; +import com.ctrip.xpipe.redis.checker.healthcheck.KeeperSupport; +import com.ctrip.xpipe.redis.checker.healthcheck.RedisInstanceInfo; +import com.ctrip.xpipe.redis.checker.healthcheck.impl.DefaultRedisInstanceInfo; +import com.ctrip.xpipe.redis.checker.model.DcClusterShard; +import com.ctrip.xpipe.redis.core.protocal.cmd.InfoResultExtractor; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Component; + +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; + +@Component +public class RedisUsedMemoryCollector implements RedisInfoActionListener, KeeperSupport { + + private final Logger logger = LoggerFactory.getLogger(getClass()); + + protected ConcurrentMap dcClusterShardUsedMemory = new ConcurrentHashMap<>(); + + @Override + public void onAction(RedisInfoActionContext context) { + try { + InfoResultExtractor extractor = context.getResult(); + RedisInstanceInfo info = context.instance().getCheckInfo(); + + long usedMemory = getUsedMemory(extractor); + dcClusterShardUsedMemory.put(new DcClusterShard(info.getDcId(), info.getClusterId(), info.getShardId()), usedMemory); + } catch (Throwable throwable) { + logger.error("get used memory of redis:{} error: ", context.instance().getCheckInfo().getHostPort(), throwable); + } + } + + private long getUsedMemory(InfoResultExtractor extractor) { + Long maxMemory = extractor.getMaxMemory(); + Long usedMemory = extractor.getUsedMemory(); + Long dbSize = extractor.getSwapUsedDbSize(); + + if (dbSize == null || usedMemory < maxMemory) return usedMemory; + + if (dbSize < maxMemory >> 1) return maxMemory; + + String keysSpaceDb0 = extractor.extract("db0"); + if (keysSpaceDb0 == null) return maxMemory * 3; + + + String[] keySpaces = keysSpaceDb0.split(","); + String[] keys1 = keySpaces[0].split("="); + String[] keys2 = keySpaces[1].split("="); + if (!keys1[0].equalsIgnoreCase("keys") || !keys2[0].equalsIgnoreCase("evicts")) { + return maxMemory * 3; + } + + long evicts = Long.parseLong(keys2[1]); + long keys = Long.parseLong(keys1[1]); + return keys == 0 ? maxMemory * 3 : (keys + evicts) / keys * maxMemory; + } + + @Override + public void stopWatch(HealthCheckAction action) { + DefaultRedisInstanceInfo info = (DefaultRedisInstanceInfo) action.getActionInstance().getCheckInfo(); + logger.debug("[stopWatch] DcClusterShard: {}", new DcClusterShard(info.getDcId(), info.getClusterId(), info.getShardId())); + dcClusterShardUsedMemory.remove(new DcClusterShard(info.getDcId(), info.getClusterId(), info.getShardId())); + } + + public ConcurrentMap getDcClusterShardUsedMemory() { + return dcClusterShardUsedMemory; + } +} diff --git a/redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/healthcheck/actions/keeper/infoStats/KeeperFlowCollector.java b/redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/healthcheck/actions/keeper/infoStats/KeeperFlowCollector.java new file mode 100644 index 000000000..027308d85 --- /dev/null +++ b/redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/healthcheck/actions/keeper/infoStats/KeeperFlowCollector.java @@ -0,0 +1,49 @@ +package com.ctrip.xpipe.redis.checker.healthcheck.actions.keeper.infoStats; + +import com.ctrip.xpipe.redis.checker.healthcheck.HealthCheckAction; +import com.ctrip.xpipe.redis.checker.healthcheck.KeeperInstanceInfo; +import com.ctrip.xpipe.redis.checker.healthcheck.KeeperSupport; +import com.ctrip.xpipe.redis.checker.model.DcClusterShard; +import com.ctrip.xpipe.redis.core.protocal.cmd.InfoResultExtractor; +import com.ctrip.xpipe.utils.MapUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Component; + +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; + +@Component +public class KeeperFlowCollector implements KeeperInfoStatsActionListener, KeeperSupport { + + private final Logger logger = LoggerFactory.getLogger(getClass()); + + protected ConcurrentMap> hostPort2InputFlow = new ConcurrentHashMap<>(); + + @Override + public void onAction(KeeperInfoStatsActionContext context) { + try { + InfoResultExtractor extractor = context.getResult(); + KeeperInstanceInfo info = context.instance().getCheckInfo(); + long keeperFlow = (long) extractor.getKeeperInstantaneousInputKbps(); + Map keeperContainerResult = MapUtils.getOrCreate(hostPort2InputFlow, info.getHostPort().getHost(), ()->new ConcurrentHashMap<>()); + keeperContainerResult.put(new DcClusterShard(info.getDcId(), info.getClusterId(), info.getShardId()), keeperFlow); + } catch (Throwable throwable) { + logger.error("get instantaneous input kbps of keeper:{} error: ", context.instance().getCheckInfo().getHostPort(), throwable); + } + + } + + @Override + public void stopWatch(HealthCheckAction action) { + KeeperInstanceInfo instanceInfo = (KeeperInstanceInfo) action.getActionInstance().getCheckInfo(); + logger.debug("stopWatch: {}", new DcClusterShard(instanceInfo.getDcId(), instanceInfo.getClusterId(), instanceInfo.getShardId())); + hostPort2InputFlow.get(instanceInfo.getHostPort().getHost()) + .remove(new DcClusterShard(instanceInfo.getDcId(), instanceInfo.getClusterId(), instanceInfo.getShardId())); + } + + public ConcurrentMap> getHostPort2InputFlow() { + return hostPort2InputFlow; + } +} diff --git a/redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/healthcheck/actions/keeper/infoStats/KeeperInfoStatsAction.java b/redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/healthcheck/actions/keeper/infoStats/KeeperInfoStatsAction.java new file mode 100644 index 000000000..37f363c3b --- /dev/null +++ b/redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/healthcheck/actions/keeper/infoStats/KeeperInfoStatsAction.java @@ -0,0 +1,31 @@ +package com.ctrip.xpipe.redis.checker.healthcheck.actions.keeper.infoStats; + +import com.ctrip.xpipe.api.command.CommandFuture; +import com.ctrip.xpipe.redis.checker.healthcheck.KeeperHealthCheckInstance; +import com.ctrip.xpipe.redis.checker.healthcheck.actions.keeper.AbstractKeeperInfoCommand; +import com.ctrip.xpipe.redis.checker.healthcheck.session.Callbackable; + +import java.util.concurrent.ExecutorService; +import java.util.concurrent.ScheduledExecutorService; + +public class KeeperInfoStatsAction extends AbstractKeeperInfoCommand { + + public KeeperInfoStatsAction(ScheduledExecutorService scheduled, KeeperHealthCheckInstance instance, ExecutorService executors) { + super(scheduled, instance, executors); + } + + @Override + protected KeeperInfoStatsActionContext createActionContext(String extractor) { + return new KeeperInfoStatsActionContext(instance, extractor); + } + + @Override + protected CommandFuture executeRedisCommandForStats(Callbackable callback) { + return getActionInstance().getRedisSession().infoStats(callback); + } + + @Override + protected int getBaseCheckInterval() { + return getActionInstance().getHealthCheckConfig().getKeeperCheckerIntervalMilli(); + } +} diff --git a/redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/healthcheck/actions/keeper/infoStats/KeeperInfoStatsActionContext.java b/redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/healthcheck/actions/keeper/infoStats/KeeperInfoStatsActionContext.java new file mode 100644 index 000000000..e6f100de4 --- /dev/null +++ b/redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/healthcheck/actions/keeper/infoStats/KeeperInfoStatsActionContext.java @@ -0,0 +1,13 @@ +package com.ctrip.xpipe.redis.checker.healthcheck.actions.keeper.infoStats; + +import com.ctrip.xpipe.redis.checker.healthcheck.KeeperHealthCheckInstance; +import com.ctrip.xpipe.redis.checker.healthcheck.actions.redisstats.AbstractInfoContext; +import com.ctrip.xpipe.redis.core.protocal.cmd.InfoResultExtractor; + + +public class KeeperInfoStatsActionContext extends AbstractInfoContext { + + public KeeperInfoStatsActionContext(KeeperHealthCheckInstance instance, String extractor) { + super(instance, new InfoResultExtractor(extractor)); + } +} diff --git a/redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/healthcheck/actions/keeper/infoStats/KeeperInfoStatsActionController.java b/redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/healthcheck/actions/keeper/infoStats/KeeperInfoStatsActionController.java new file mode 100644 index 000000000..9e55df2f0 --- /dev/null +++ b/redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/healthcheck/actions/keeper/infoStats/KeeperInfoStatsActionController.java @@ -0,0 +1,4 @@ +package com.ctrip.xpipe.redis.checker.healthcheck.actions.keeper.infoStats; + +public interface KeeperInfoStatsActionController { +} diff --git a/redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/healthcheck/actions/keeper/infoStats/KeeperInfoStatsActionFactory.java b/redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/healthcheck/actions/keeper/infoStats/KeeperInfoStatsActionFactory.java new file mode 100644 index 000000000..c0ab2f290 --- /dev/null +++ b/redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/healthcheck/actions/keeper/infoStats/KeeperInfoStatsActionFactory.java @@ -0,0 +1,32 @@ +package com.ctrip.xpipe.redis.checker.healthcheck.actions.keeper.infoStats; + +import com.ctrip.xpipe.redis.checker.alert.ALERT_TYPE; +import com.ctrip.xpipe.redis.checker.healthcheck.KeeperHealthCheckInstance; +import com.ctrip.xpipe.redis.checker.healthcheck.KeeperSupport; +import com.ctrip.xpipe.redis.checker.healthcheck.actions.keeper.AbstractKeeperInfoCommandActionFactory; +import com.ctrip.xpipe.redis.checker.healthcheck.leader.SiteLeaderAwareHealthCheckAction; +import com.google.common.collect.Lists; +import org.springframework.stereotype.Component; + +import java.util.List; + +@Component +public class KeeperInfoStatsActionFactory + extends AbstractKeeperInfoCommandActionFactory + implements KeeperSupport { + + @Override + protected KeeperInfoStatsAction createAction(KeeperHealthCheckInstance instance) { + return new KeeperInfoStatsAction(scheduled, instance, executors); + } + + @Override + protected List alertTypes() { + return Lists.newArrayList(); + } + + @Override + public Class support() { + return KeeperInfoStatsAction.class; + } +} diff --git a/redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/healthcheck/actions/keeper/infoStats/KeeperInfoStatsActionListener.java b/redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/healthcheck/actions/keeper/infoStats/KeeperInfoStatsActionListener.java new file mode 100644 index 000000000..9f8d4e698 --- /dev/null +++ b/redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/healthcheck/actions/keeper/infoStats/KeeperInfoStatsActionListener.java @@ -0,0 +1,7 @@ +package com.ctrip.xpipe.redis.checker.healthcheck.actions.keeper.infoStats; + +import com.ctrip.xpipe.redis.checker.healthcheck.actions.redisstats.AbstractInfoListener; + +public interface KeeperInfoStatsActionListener extends AbstractInfoListener { + +} diff --git a/redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/healthcheck/actions/redisstats/AbstractInfoContext.java b/redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/healthcheck/actions/redisstats/AbstractInfoContext.java index ccd9aee7c..f98bc52eb 100644 --- a/redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/healthcheck/actions/redisstats/AbstractInfoContext.java +++ b/redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/healthcheck/actions/redisstats/AbstractInfoContext.java @@ -1,11 +1,11 @@ package com.ctrip.xpipe.redis.checker.healthcheck.actions.redisstats; import com.ctrip.xpipe.redis.checker.healthcheck.AbstractActionContext; -import com.ctrip.xpipe.redis.checker.healthcheck.RedisHealthCheckInstance; +import com.ctrip.xpipe.redis.checker.healthcheck.HealthCheckInstance; import com.ctrip.xpipe.redis.core.protocal.cmd.InfoResultExtractor; -public abstract class AbstractInfoContext extends AbstractActionContext { - public AbstractInfoContext(RedisHealthCheckInstance instance, InfoResultExtractor extractor) { +public abstract class AbstractInfoContext extends AbstractActionContext { + public AbstractInfoContext(T instance, InfoResultExtractor extractor) { super(instance, extractor); } } diff --git a/redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/healthcheck/actions/redisstats/AbstractInstanceStatsCheckAction.java b/redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/healthcheck/actions/redisstats/AbstractInstanceStatsCheckAction.java new file mode 100644 index 000000000..9f5119731 --- /dev/null +++ b/redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/healthcheck/actions/redisstats/AbstractInstanceStatsCheckAction.java @@ -0,0 +1,52 @@ +package com.ctrip.xpipe.redis.checker.healthcheck.actions.redisstats; + +import com.ctrip.xpipe.api.command.CommandFuture; +import com.ctrip.xpipe.redis.checker.healthcheck.*; +import com.ctrip.xpipe.redis.checker.healthcheck.leader.AbstractLeaderAwareHealthCheckAction; +import com.ctrip.xpipe.redis.checker.healthcheck.session.Callbackable; + +import java.util.concurrent.ExecutorService; +import java.util.concurrent.ScheduledExecutorService; + +public abstract class AbstractInstanceStatsCheckAction extends AbstractLeaderAwareHealthCheckAction { + + + protected CommandFuture commandFuture; + + protected static final int METRIC_CHECK_INTERVAL = 60 * 1000; + + public AbstractInstanceStatsCheckAction(ScheduledExecutorService scheduled, V instance, ExecutorService executors) { + super(scheduled, instance, executors); + } + + @Override + protected void doTask() { + getHealthCheckLogger().debug("[doTask] begin for {}", instance); + + if (null != commandFuture && !commandFuture.isDone()) { + getHealthCheckLogger().info("[doTask] last command {} hasn't finished!", commandFuture); + return; + } + + commandFuture = executeRedisCommandForStats(new Callbackable() { + @Override + public void success(T message) { + notifyListeners(generateActionContext(message)); + } + + @Override + public void fail(Throwable throwable) { + getHealthCheckLogger().info("[doTask] cmd execute fail for {}", instance, throwable); + } + }); + } + + @Override + protected int getBaseCheckInterval() { + return METRIC_CHECK_INTERVAL; + } + + protected abstract CommandFuture executeRedisCommandForStats(Callbackable callback); + + protected abstract ActionContext generateActionContext(T result); +} diff --git a/redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/healthcheck/actions/redisstats/RedisStatsCheckAction.java b/redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/healthcheck/actions/redisstats/RedisStatsCheckAction.java index 844653405..30c4eef2c 100644 --- a/redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/healthcheck/actions/redisstats/RedisStatsCheckAction.java +++ b/redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/healthcheck/actions/redisstats/RedisStatsCheckAction.java @@ -1,53 +1,13 @@ package com.ctrip.xpipe.redis.checker.healthcheck.actions.redisstats; -import com.ctrip.xpipe.api.command.CommandFuture; -import com.ctrip.xpipe.redis.checker.healthcheck.ActionContext; import com.ctrip.xpipe.redis.checker.healthcheck.RedisHealthCheckInstance; -import com.ctrip.xpipe.redis.checker.healthcheck.leader.AbstractLeaderAwareHealthCheckAction; -import com.ctrip.xpipe.redis.checker.healthcheck.session.Callbackable; import java.util.concurrent.ExecutorService; import java.util.concurrent.ScheduledExecutorService; -public abstract class RedisStatsCheckAction extends AbstractLeaderAwareHealthCheckAction { - - protected CommandFuture commandFuture; - - protected static final int METRIC_CHECK_INTERVAL = 60 * 1000; +public abstract class RedisStatsCheckAction extends AbstractInstanceStatsCheckAction { public RedisStatsCheckAction(ScheduledExecutorService scheduled, RedisHealthCheckInstance instance, ExecutorService executors) { super(scheduled, instance, executors); } - - @Override - protected void doTask() { - getHealthCheckLogger().debug("[doTask] begin for {}", instance); - - if (null != commandFuture && !commandFuture.isDone()) { - getHealthCheckLogger().info("[doTask] last command {} hasn't finished!", commandFuture); - return; - } - - commandFuture = executeRedisCommandForStats(new Callbackable() { - @Override - public void success(T message) { - notifyListeners(generateActionContext(message)); - } - - @Override - public void fail(Throwable throwable) { - getHealthCheckLogger().info("[doTask] cmd execute fail for {}", instance, throwable); - } - }); - } - - @Override - protected int getBaseCheckInterval() { - return METRIC_CHECK_INTERVAL; - } - - protected abstract CommandFuture executeRedisCommandForStats(Callbackable callback); - - protected abstract ActionContext generateActionContext(T result); - } diff --git a/redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/healthcheck/actions/redisstats/crdtInforeplication/CrdtInfoReplicationContext.java b/redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/healthcheck/actions/redisstats/crdtInforeplication/CrdtInfoReplicationContext.java index 8aa776e60..e7ee1b8d1 100644 --- a/redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/healthcheck/actions/redisstats/crdtInforeplication/CrdtInfoReplicationContext.java +++ b/redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/healthcheck/actions/redisstats/crdtInforeplication/CrdtInfoReplicationContext.java @@ -1,13 +1,10 @@ package com.ctrip.xpipe.redis.checker.healthcheck.actions.redisstats.crdtInforeplication; -import com.ctrip.xpipe.redis.checker.healthcheck.AbstractActionContext; -import com.ctrip.xpipe.redis.checker.healthcheck.ActionContext; import com.ctrip.xpipe.redis.checker.healthcheck.RedisHealthCheckInstance; import com.ctrip.xpipe.redis.checker.healthcheck.actions.redisstats.AbstractInfoContext; import com.ctrip.xpipe.redis.core.protocal.cmd.CRDTInfoResultExtractor; -import com.ctrip.xpipe.redis.core.protocal.cmd.InfoResultExtractor; -public class CrdtInfoReplicationContext extends AbstractInfoContext { +public class CrdtInfoReplicationContext extends AbstractInfoContext { public CrdtInfoReplicationContext(RedisHealthCheckInstance instance, String extractor) { super(instance, new CRDTInfoResultExtractor(extractor)); } diff --git a/redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/healthcheck/actions/redisstats/crdtinfostats/CrdtInfoStatsContext.java b/redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/healthcheck/actions/redisstats/crdtinfostats/CrdtInfoStatsContext.java index 864128b39..4da0f08ae 100644 --- a/redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/healthcheck/actions/redisstats/crdtinfostats/CrdtInfoStatsContext.java +++ b/redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/healthcheck/actions/redisstats/crdtinfostats/CrdtInfoStatsContext.java @@ -1,12 +1,10 @@ package com.ctrip.xpipe.redis.checker.healthcheck.actions.redisstats.crdtinfostats; -import com.ctrip.xpipe.redis.checker.healthcheck.AbstractActionContext; import com.ctrip.xpipe.redis.checker.healthcheck.RedisHealthCheckInstance; import com.ctrip.xpipe.redis.checker.healthcheck.actions.redisstats.AbstractInfoContext; import com.ctrip.xpipe.redis.core.protocal.cmd.CRDTInfoResultExtractor; -import com.ctrip.xpipe.redis.core.protocal.cmd.InfoResultExtractor; -public class CrdtInfoStatsContext extends AbstractInfoContext { +public class CrdtInfoStatsContext extends AbstractInfoContext { public CrdtInfoStatsContext(RedisHealthCheckInstance instance, String extractor) { super(instance, new CRDTInfoResultExtractor(extractor)); } diff --git a/redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/healthcheck/actions/redisstats/inforeplication/InfoReplicationContext.java b/redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/healthcheck/actions/redisstats/inforeplication/InfoReplicationContext.java index 1691b26ff..65a799ee8 100644 --- a/redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/healthcheck/actions/redisstats/inforeplication/InfoReplicationContext.java +++ b/redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/healthcheck/actions/redisstats/inforeplication/InfoReplicationContext.java @@ -4,7 +4,7 @@ import com.ctrip.xpipe.redis.checker.healthcheck.actions.redisstats.AbstractInfoContext; import com.ctrip.xpipe.redis.core.protocal.cmd.InfoResultExtractor; -public class InfoReplicationContext extends AbstractInfoContext { +public class InfoReplicationContext extends AbstractInfoContext { public InfoReplicationContext(RedisHealthCheckInstance instance, String extractor) { super(instance, new InfoResultExtractor(extractor)); diff --git a/redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/healthcheck/actions/redisstats/infostats/InfoStatsContext.java b/redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/healthcheck/actions/redisstats/infostats/InfoStatsContext.java index 22a1c9ae2..367aff120 100644 --- a/redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/healthcheck/actions/redisstats/infostats/InfoStatsContext.java +++ b/redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/healthcheck/actions/redisstats/infostats/InfoStatsContext.java @@ -4,7 +4,7 @@ import com.ctrip.xpipe.redis.checker.healthcheck.actions.redisstats.AbstractInfoContext; import com.ctrip.xpipe.redis.core.protocal.cmd.InfoResultExtractor; -public class InfoStatsContext extends AbstractInfoContext { +public class InfoStatsContext extends AbstractInfoContext { public InfoStatsContext(RedisHealthCheckInstance instance, String extractor) { super(instance, new InfoResultExtractor(extractor)); diff --git a/redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/healthcheck/config/AbstractHealthCheckConfig.java b/redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/healthcheck/config/AbstractHealthCheckConfig.java index 206e7b009..083b8e912 100644 --- a/redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/healthcheck/config/AbstractHealthCheckConfig.java +++ b/redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/healthcheck/config/AbstractHealthCheckConfig.java @@ -61,6 +61,11 @@ public String getMinDiskLessReplVersion() { return checkerConfig.getReplDisklessMinRedisVersion(); } + @Override + public int getKeeperCheckerIntervalMilli() { + return checkerConfig.getKeeperCheckerIntervalMilli(); + } + @Override public boolean supportSentinelHealthCheck(ClusterType clusterType, String clusterName) { return checkerConfig.supportSentinelHealthCheck(clusterType, clusterName); diff --git a/redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/healthcheck/config/CompositeHealthCheckConfig.java b/redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/healthcheck/config/CompositeHealthCheckConfig.java index f7e3e5f76..9bfe46e78 100644 --- a/redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/healthcheck/config/CompositeHealthCheckConfig.java +++ b/redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/healthcheck/config/CompositeHealthCheckConfig.java @@ -3,6 +3,7 @@ import com.ctrip.xpipe.cluster.ClusterType; import com.ctrip.xpipe.redis.checker.DcRelationsService; import com.ctrip.xpipe.redis.checker.config.CheckerConfig; +import com.ctrip.xpipe.redis.checker.healthcheck.KeeperInstanceInfo; import com.ctrip.xpipe.redis.checker.healthcheck.RedisInstanceInfo; import com.ctrip.xpipe.redis.checker.healthcheck.actions.delay.DelayConfig; import org.slf4j.Logger; @@ -30,6 +31,12 @@ public CompositeHealthCheckConfig(RedisInstanceInfo instanceInfo, CheckerConfig logger.info("[CompositeHealthCheckConfig][{}] [config: {}]", instanceInfo, config.getClass().getSimpleName()); } + public CompositeHealthCheckConfig(KeeperInstanceInfo instanceInfo, CheckerConfig checkerConfig, DcRelationsService dcRelationsService) { + logger.info("[CompositeHealthCheckConfig] {}", instanceInfo); + config = new DefaultHealthCheckConfig(checkerConfig, dcRelationsService); + logger.info("[CompositeHealthCheckConfig][{}] [config: {}]", instanceInfo, config.getClass().getSimpleName()); + } + @Override public int instanceLongDelayMilli() { return config.instanceLongDelayMilli(); @@ -45,6 +52,11 @@ public int checkIntervalMilli() { return config.checkIntervalMilli(); } + @Override + public int getKeeperCheckerIntervalMilli() { + return config.getKeeperCheckerIntervalMilli(); + } + @Override public int clusterCheckIntervalMilli() { return config.clusterCheckIntervalMilli(); diff --git a/redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/healthcheck/config/HealthCheckConfig.java b/redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/healthcheck/config/HealthCheckConfig.java index f748cb814..03a8f2375 100644 --- a/redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/healthcheck/config/HealthCheckConfig.java +++ b/redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/healthcheck/config/HealthCheckConfig.java @@ -16,6 +16,8 @@ public interface HealthCheckConfig { int checkIntervalMilli(); + int getKeeperCheckerIntervalMilli(); + int clusterCheckIntervalMilli(); int getRedisConfCheckIntervalMilli(); diff --git a/redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/healthcheck/impl/DefaultHealthCheckEndpointFactory.java b/redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/healthcheck/impl/DefaultHealthCheckEndpointFactory.java index 1206b0745..dac2505d7 100644 --- a/redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/healthcheck/impl/DefaultHealthCheckEndpointFactory.java +++ b/redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/healthcheck/impl/DefaultHealthCheckEndpointFactory.java @@ -11,6 +11,7 @@ import com.ctrip.xpipe.endpoint.DefaultEndPoint; import com.ctrip.xpipe.endpoint.HostPort; import com.ctrip.xpipe.redis.checker.config.CheckerConfig; +import com.ctrip.xpipe.redis.core.entity.KeeperMeta; import com.ctrip.xpipe.redis.core.entity.RedisMeta; import com.ctrip.xpipe.redis.core.entity.RouteMeta; import com.ctrip.xpipe.redis.core.meta.MetaCache; @@ -171,6 +172,12 @@ public Endpoint getOrCreateEndpoint(RedisMeta redisMeta) { return getOrCreateEndpoint(hostPort); } + @Override + public Endpoint getOrCreateEndpoint(KeeperMeta keeperMeta) { + HostPort hostPort = new HostPort(keeperMeta.getIp(), keeperMeta.getPort()); + return getOrCreateEndpoint(hostPort); + } + @Override public void remove(HostPort hostPort) { logger.info("[ProxyRegistry] unregisterProxy {}:{}", hostPort.getHost(),hostPort.getPort()); diff --git a/redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/healthcheck/impl/DefaultHealthCheckInstanceFactory.java b/redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/healthcheck/impl/DefaultHealthCheckInstanceFactory.java index 164a1d1c5..63b1e3d98 100644 --- a/redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/healthcheck/impl/DefaultHealthCheckInstanceFactory.java +++ b/redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/healthcheck/impl/DefaultHealthCheckInstanceFactory.java @@ -15,9 +15,11 @@ import com.ctrip.xpipe.redis.checker.healthcheck.config.DefaultHealthCheckConfig; import com.ctrip.xpipe.redis.checker.healthcheck.config.HealthCheckConfig; import com.ctrip.xpipe.redis.checker.healthcheck.leader.SiteLeaderAwareHealthCheckActionFactory; +import com.ctrip.xpipe.redis.checker.healthcheck.session.KeeperSessionManager; import com.ctrip.xpipe.redis.checker.healthcheck.session.RedisSessionManager; import com.ctrip.xpipe.redis.checker.healthcheck.util.ClusterTypeSupporterSeparator; import com.ctrip.xpipe.redis.core.entity.ClusterMeta; +import com.ctrip.xpipe.redis.core.entity.KeeperMeta; import com.ctrip.xpipe.redis.core.entity.RedisCheckRuleMeta; import com.ctrip.xpipe.redis.core.entity.RedisMeta; import com.ctrip.xpipe.redis.core.meta.MetaCache; @@ -51,6 +53,10 @@ public class DefaultHealthCheckInstanceFactory implements HealthCheckInstanceFac private RedisSessionManager redisSessionManager; + private KeeperSessionManager keeperSessionManager; + + private List> keeperHealthCheckActionFactories; + private Map>> factoriesByClusterType; private Map>> clusterHealthCheckFactoriesByClusterType; @@ -64,25 +70,32 @@ public class DefaultHealthCheckInstanceFactory implements HealthCheckInstanceFac @Autowired(required = false) public DefaultHealthCheckInstanceFactory(CheckerConfig checkerConfig, HealthCheckEndpointFactory endpointFactory, - RedisSessionManager redisSessionManager, List> factories, + RedisSessionManager redisSessionManager, KeeperSessionManager keeperSessionManager, + List> factories, + List> keeperHealthCheckActionFactories, List> clusterHealthCheckFactories, GroupCheckerLeaderElector clusterServer, MetaCache metaCache, DcRelationsService dcRelationsService) { this.checkerConfig = checkerConfig; this.dcRelationsService = dcRelationsService; this.endpointFactory = endpointFactory; this.redisSessionManager = redisSessionManager; + this.keeperSessionManager = keeperSessionManager; this.clusterServer = clusterServer; this.metaCache = metaCache; this.factoriesByClusterType = ClusterTypeSupporterSeparator.divideByClusterType(factories); this.clusterHealthCheckFactoriesByClusterType = ClusterTypeSupporterSeparator.divideByClusterType(clusterHealthCheckFactories); + this.keeperHealthCheckActionFactories = keeperHealthCheckActionFactories; } @Autowired(required = false) public DefaultHealthCheckInstanceFactory(CheckerConfig checkerConfig, HealthCheckEndpointFactory endpointFactory, - RedisSessionManager redisSessionManager, List> factories, + RedisSessionManager redisSessionManager, KeeperSessionManager keeperSessionManager, + List> factories, + List> keeperHealthCheckActionFactories, List> clusterHealthCheckFactories, MetaCache metaCache, DcRelationsService dcRelationsService) { - this(checkerConfig, endpointFactory, redisSessionManager, factories, clusterHealthCheckFactories, null, metaCache, dcRelationsService); + this(checkerConfig, endpointFactory, redisSessionManager, keeperSessionManager, factories, + keeperHealthCheckActionFactories, clusterHealthCheckFactories, null, metaCache, dcRelationsService); } @Override @@ -92,6 +105,13 @@ public void remove(RedisHealthCheckInstance instance) { stopCheck(instance); } + @Override + public void remove(KeeperHealthCheckInstance instance) { + Endpoint endpoint = instance.getEndpoint(); + endpointFactory.remove(new HostPort(endpoint.getHost(), endpoint.getPort())); + stopCheck(instance); + } + @Override public void remove(ClusterHealthCheckInstance instance) { stopCheck(instance); @@ -167,15 +187,83 @@ public ClusterHealthCheckInstance create(ClusterMeta clusterMeta) { return instance; } + @Override + public KeeperHealthCheckInstance create(KeeperMeta keeperMeta) { + DefaultKeeperHealthCheckInstance instance = new DefaultKeeperHealthCheckInstance(); + + KeeperInstanceInfo keeper = createKeeperInstanceInfo(keeperMeta); + HealthCheckConfig config = new CompositeHealthCheckConfig(keeper, checkerConfig, dcRelationsService); + Endpoint endpoint = endpointFactory.getOrCreateEndpoint(keeperMeta); + + instance.setEndpoint(endpoint) + .setSession(keeperSessionManager.findOrCreateSession(endpoint)) + .setInstanceInfo(keeper) + .setHealthCheckConfig(config); + initKeeperActions(instance); + startCheck(instance); + + return instance; + } + + private KeeperInstanceInfo createKeeperInstanceInfo(KeeperMeta keeperMeta) { + ClusterType clusterType = ClusterType.lookup(((ClusterMeta)keeperMeta.parent().parent()).getType()); + + DefaultKeeperInstanceInfo info = new DefaultKeeperInstanceInfo( + ((ClusterMeta)keeperMeta.parent().parent()).parent().getId(), + ((ClusterMeta)keeperMeta.parent().parent()).getId(), + keeperMeta.parent().getId(), + new HostPort(keeperMeta.getIp(), keeperMeta.getPort()), + keeperMeta.parent().getActiveDc(), clusterType); + info.setActive(keeperMeta.isActive()); + + return info; + } + + private void initKeeperActions(DefaultKeeperHealthCheckInstance instance) { + keeperHealthCheckActionFactories.forEach(keeperHealthCheckActionFactory -> { + initActions(instance, keeperHealthCheckActionFactory); + }); + } + + @Override + public RedisHealthCheckInstance createRedisInstanceForAssignedAction(RedisMeta redis) { + DefaultRedisHealthCheckInstance instance = new DefaultRedisHealthCheckInstance(); + + RedisInstanceInfo info = createRedisInstanceInfo(redis); + HealthCheckConfig config = new CompositeHealthCheckConfig(info, checkerConfig, dcRelationsService); + Endpoint endpoint = endpointFactory.getOrCreateEndpoint(redis); + + instance.setEndpoint(endpoint) + .setSession(redisSessionManager.findOrCreateSession(endpoint)) + .setInstanceInfo(info) + .setHealthCheckConfig(config); + + initActionsForRedisForAssignedAction(instance); + startCheck(instance); + + return instance; + } + + private void initActionsForRedisForAssignedAction(DefaultRedisHealthCheckInstance instance) { + for(RedisHealthCheckActionFactory factory : factoriesByClusterType.get(instance.getCheckInfo().getClusterType())) { + if (factory instanceof KeeperSupport) + initActions(instance, factory); + } + } + @SuppressWarnings("unchecked") private void initActions(DefaultRedisHealthCheckInstance instance) { for(RedisHealthCheckActionFactory factory : factoriesByClusterType.get(instance.getCheckInfo().getClusterType())) { - if (factory.supportInstnace(instance)) { - if (factory instanceof SiteLeaderAwareHealthCheckActionFactory) { - installActionIfNeeded((SiteLeaderAwareHealthCheckActionFactory) factory, instance); - } else { - instance.register(factory.create(instance)); - } + initActions(instance, factory); + } + } + + private void initActions(HealthCheckInstance instance, HealthCheckActionFactory factory) { + if (factory.supportInstnace(instance)) { + if (factory instanceof SiteLeaderAwareHealthCheckActionFactory) { + installActionIfNeeded((SiteLeaderAwareHealthCheckActionFactory) factory, instance); + } else { + instance.register(factory.create(instance)); } } } diff --git a/redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/healthcheck/impl/DefaultHealthCheckInstanceManager.java b/redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/healthcheck/impl/DefaultHealthCheckInstanceManager.java index 4300d6ff3..390ecbc75 100644 --- a/redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/healthcheck/impl/DefaultHealthCheckInstanceManager.java +++ b/redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/healthcheck/impl/DefaultHealthCheckInstanceManager.java @@ -1,10 +1,9 @@ package com.ctrip.xpipe.redis.checker.healthcheck.impl; import com.ctrip.xpipe.endpoint.HostPort; -import com.ctrip.xpipe.redis.checker.healthcheck.ClusterHealthCheckInstance; -import com.ctrip.xpipe.redis.checker.healthcheck.HealthCheckInstanceManager; -import com.ctrip.xpipe.redis.checker.healthcheck.RedisHealthCheckInstance; +import com.ctrip.xpipe.redis.checker.healthcheck.*; import com.ctrip.xpipe.redis.core.entity.ClusterMeta; +import com.ctrip.xpipe.redis.core.entity.KeeperMeta; import com.ctrip.xpipe.redis.core.entity.RedisMeta; import com.ctrip.xpipe.utils.MapUtils; import com.ctrip.xpipe.utils.StringUtil; @@ -32,6 +31,10 @@ public class DefaultHealthCheckInstanceManager implements HealthCheckInstanceMan private ConcurrentMap clusterHealthCheckerInstances = Maps.newConcurrentMap(); + private ConcurrentMap keeperInstances = Maps.newConcurrentMap(); + + private ConcurrentMap redisInstanceForAssignedAction = Maps.newConcurrentMap(); + @Autowired private HealthCheckInstanceFactory instanceFactory; @@ -46,6 +49,29 @@ public RedisHealthCheckInstance getOrCreate(RedisMeta redis) { return null; } + @Override + public RedisHealthCheckInstance getOrCreateRedisInstanceForAssignedAction(RedisMeta redis) { + try { + HostPort key = new HostPort(redis.getIp(), redis.getPort()); + return MapUtils.getOrCreate(redisInstanceForAssignedAction, key, + () -> instanceFactory.createRedisInstanceForAssignedAction(redis)); + } catch (Throwable th) { + logger.error("getOrCreate health check redis instance:{}:{}", redis.getIp(), redis.getPort()); + } + return null; + } + + @Override + public KeeperHealthCheckInstance getOrCreate(KeeperMeta keeper) { + try { + HostPort key = new HostPort(keeper.getIp(), keeper.getPort()); + return MapUtils.getOrCreate(keeperInstances, key, () -> instanceFactory.create(keeper)); + } catch (Throwable th) { + logger.error("getOrCreate health check keeper instance:{}:{}", keeper.getIp(), keeper.getPort()); + } + return null; + } + @Override public ClusterHealthCheckInstance getOrCreate(ClusterMeta cluster) { try { @@ -62,6 +88,16 @@ public RedisHealthCheckInstance findRedisHealthCheckInstance(HostPort hostPort) return instances.get(hostPort); } + @Override + public RedisHealthCheckInstance findRedisInstanceForAssignedAction(HostPort hostPort) { + return redisInstanceForAssignedAction.get(hostPort); + } + + @Override + public KeeperHealthCheckInstance findKeeperHealthCheckInstance(HostPort hostPort) { + return keeperInstances.get(hostPort); + } + @Override public ClusterHealthCheckInstance findClusterHealthCheckInstance(String clusterId) { if (StringUtil.isEmpty(clusterId)) return null; @@ -75,6 +111,21 @@ public RedisHealthCheckInstance remove(HostPort hostPort) { return instance; } + @Override + public KeeperHealthCheckInstance removeKeeper(HostPort hostPort) { + KeeperHealthCheckInstance instance = keeperInstances.remove(hostPort); + if (null != instance) instanceFactory.remove(instance); + return instance; + } + + @Override + public RedisHealthCheckInstance removeRedisInstanceForAssignedAction(HostPort hostPort) { + RedisHealthCheckInstance instance = redisInstanceForAssignedAction.remove(hostPort); + if (null != instance) instanceFactory.remove(instance); + return instance; + } + + @Override public ClusterHealthCheckInstance remove(String cluster) { ClusterHealthCheckInstance instance = clusterHealthCheckerInstances.remove(cluster.toLowerCase()); @@ -87,6 +138,16 @@ public List getAllRedisInstance() { return Lists.newLinkedList(instances.values()); } + @Override + public List getAllKeeperInstance() { + return Lists.newLinkedList(keeperInstances.values()); + } + + @Override + public List getAllRedisInstanceForAssignedAction() { + return Lists.newLinkedList(redisInstanceForAssignedAction.values()); + } + @Override public List getAllClusterInstance() { return Lists.newLinkedList(clusterHealthCheckerInstances.values()); diff --git a/redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/healthcheck/impl/DefaultHealthChecker.java b/redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/healthcheck/impl/DefaultHealthChecker.java index 1313525a1..69417a3f7 100644 --- a/redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/healthcheck/impl/DefaultHealthChecker.java +++ b/redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/healthcheck/impl/DefaultHealthChecker.java @@ -4,11 +4,14 @@ import com.ctrip.xpipe.cluster.ClusterType; import com.ctrip.xpipe.lifecycle.AbstractLifecycle; import com.ctrip.xpipe.lifecycle.LifecycleHelper; +import com.ctrip.xpipe.redis.checker.CheckerConsoleService; import com.ctrip.xpipe.redis.checker.config.CheckerConfig; import com.ctrip.xpipe.redis.checker.healthcheck.HealthCheckInstanceManager; import com.ctrip.xpipe.redis.checker.healthcheck.HealthChecker; import com.ctrip.xpipe.redis.checker.healthcheck.meta.MetaChangeManager; import com.ctrip.xpipe.redis.core.entity.*; +import com.ctrip.xpipe.redis.core.meta.CurrentDcAllMeta; +import com.ctrip.xpipe.redis.core.meta.KeeperContainerDetailInfo; import com.ctrip.xpipe.redis.core.meta.MetaCache; import com.ctrip.xpipe.utils.StringUtil; import org.springframework.beans.factory.annotation.Autowired; @@ -18,9 +21,11 @@ import javax.annotation.PostConstruct; import javax.annotation.PreDestroy; import javax.annotation.Resource; +import java.util.Map; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; +import static com.ctrip.xpipe.redis.core.meta.comparator.KeeperContainerMetaComparator.getAllKeeperContainerDetailInfoFromDcMeta; import static com.ctrip.xpipe.spring.AbstractSpringConfigContext.SCHEDULED_EXECUTOR; /** @@ -44,9 +49,15 @@ public class DefaultHealthChecker extends AbstractLifecycle implements HealthChe @Autowired private CheckerConfig checkerConfig; + @Autowired + private CheckerConsoleService checkerConsoleService; + @Resource(name = SCHEDULED_EXECUTOR) private ScheduledExecutorService scheduled; + @Resource + private CurrentDcAllMeta currentDcAllMeta; + private static final String currentDcId = FoundationService.DEFAULT.getDataCenter(); @PostConstruct @@ -113,6 +124,16 @@ void generateHealthCheckInstances() { if(checkerConfig.getIgnoredHealthCheckDc().contains(dcMeta.getId())) { continue; } + + if (currentDcId.equalsIgnoreCase(dcMeta.getId())) { + Map keeperContainerDetailInfoMap + = getAllKeeperContainerDetailInfoFromDcMeta(dcMeta, currentDcAllMeta.getCurrentDcAllMeta()); + + keeperContainerDetailInfoMap.values().forEach(keeperContainerDetailInfo -> { + generateHealthCheckInstances(keeperContainerDetailInfo); + }); + } + for(ClusterMeta cluster : dcMeta.getClusters().values()) { ClusterType clusterType = ClusterType.lookup(cluster.getType()); @@ -131,6 +152,15 @@ void generateHealthCheckInstances() { } } + private void generateHealthCheckInstances(KeeperContainerDetailInfo keeperContainerDetailInfo) { + for (KeeperMeta keeperMeta : keeperContainerDetailInfo.getKeeperInstances()) { + instanceManager.getOrCreate(keeperMeta); + } + for (RedisMeta redisMeta : keeperContainerDetailInfo.getRedisInstances()) { + instanceManager.getOrCreateRedisInstanceForAssignedAction(redisMeta); + } + } + void generateHealthCheckInstances(ClusterMeta clusterMeta){ for(ShardMeta shard : clusterMeta.getShards().values()) { for(RedisMeta redis : shard.getRedises()) { @@ -156,6 +186,7 @@ private boolean isClusterInCurrentIdc(ClusterMeta cluster) { return false; } + private boolean clusterDcIsCurrentDc(ClusterMeta clusterMeta) { return clusterMeta.parent().getId().equalsIgnoreCase(currentDcId); } diff --git a/redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/healthcheck/impl/DefaultKeeperHealthCheckInstance.java b/redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/healthcheck/impl/DefaultKeeperHealthCheckInstance.java new file mode 100644 index 000000000..f64c45da3 --- /dev/null +++ b/redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/healthcheck/impl/DefaultKeeperHealthCheckInstance.java @@ -0,0 +1,56 @@ +package com.ctrip.xpipe.redis.checker.healthcheck.impl; + +import com.ctrip.xpipe.api.endpoint.Endpoint; +import com.ctrip.xpipe.redis.checker.healthcheck.KeeperHealthCheckInstance; +import com.ctrip.xpipe.redis.checker.healthcheck.KeeperInstanceInfo; +import com.ctrip.xpipe.redis.checker.healthcheck.session.RedisSession; +import com.ctrip.xpipe.utils.ObjectUtils; + +public class DefaultKeeperHealthCheckInstance extends AbstractHealthCheckInstance implements KeeperHealthCheckInstance { + + private Endpoint endpoint; + + private RedisSession session; + + @Override + public Endpoint getEndpoint() { + return endpoint; + } + + + + public DefaultKeeperHealthCheckInstance setEndpoint(Endpoint endpoint) { + this.endpoint = endpoint; + return this; + } + + @Override + public RedisSession getRedisSession() { + return session; + } + + public DefaultKeeperHealthCheckInstance setSession(RedisSession session) { + this.session = session; + return this; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + DefaultKeeperHealthCheckInstance that = (DefaultKeeperHealthCheckInstance) o; + return ObjectUtils.equals(that.getCheckInfo().getHostPort(), + this.getCheckInfo().getHostPort()); + } + + @Override + public int hashCode() { + return getCheckInfo().getHostPort().hashCode(); + } + + @Override + public String toString() { + return String.format("HealthCheckInstanceInfo: [%s]", getCheckInfo().toString()); + } + +} diff --git a/redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/healthcheck/impl/DefaultKeeperInstanceInfo.java b/redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/healthcheck/impl/DefaultKeeperInstanceInfo.java new file mode 100644 index 000000000..955890577 --- /dev/null +++ b/redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/healthcheck/impl/DefaultKeeperInstanceInfo.java @@ -0,0 +1,96 @@ +package com.ctrip.xpipe.redis.checker.healthcheck.impl; + +import com.ctrip.xpipe.cluster.ClusterType; +import com.ctrip.xpipe.endpoint.ClusterShardHostPort; +import com.ctrip.xpipe.endpoint.HostPort; +import com.ctrip.xpipe.redis.checker.healthcheck.KeeperInstanceInfo; +import com.ctrip.xpipe.redis.checker.healthcheck.actions.redisconf.RedisCheckRule; +import com.fasterxml.jackson.annotation.JsonIgnore; + +import java.util.List; + +public class DefaultKeeperInstanceInfo extends AbstractCheckInfo implements KeeperInstanceInfo { + + private String dcId; + + private String shardId; + + private HostPort hostPort; + + private boolean isActive; + + public DefaultKeeperInstanceInfo() { + super(); + } + + + public DefaultKeeperInstanceInfo(String dcId, String clusterId, String shardId, HostPort hostPort, String activeDc, ClusterType clusterType) { + this(dcId, clusterId, shardId, hostPort, activeDc, clusterType, null); + } + + public DefaultKeeperInstanceInfo(String dcId, String clusterId, String shardId, HostPort hostPort, String activeDc, ClusterType clusterType, List redisCheckRules) { + super(clusterId, activeDc, clusterType, redisCheckRules); + this.dcId = dcId; + this.shardId = shardId; + this.hostPort = hostPort; + } + + public DefaultKeeperInstanceInfo setDcId(String dcId) { + this.dcId = dcId; + return this; + } + + public DefaultKeeperInstanceInfo setShardId(String shardId) { + this.shardId = shardId; + return this; + } + + public DefaultKeeperInstanceInfo setHostPort(HostPort hostPort) { + this.hostPort = hostPort; + return this; + } + + public DefaultKeeperInstanceInfo setActive(boolean active) { + isActive = active; + return this; + } + + @Override + public String toString() { + return "DefaultKeeperInstanceInfo{" + + "dcId='" + dcId + '\'' + + ", shardId='" + shardId + '\'' + + ", hostPort=" + hostPort + + ", isMaster=" + isActive + + ", clusterId='" + clusterId + '\'' + + ", activeDc='" + activeDc + '\'' + + ", clusterType=" + clusterType + + '}'; + } + + @JsonIgnore + @Override + public ClusterShardHostPort getClusterShardHostport() { + return new ClusterShardHostPort(clusterId, shardId, hostPort); + } + + @Override + public String getShardId() { + return shardId; + } + + @Override + public String getDcId() { + return dcId; + } + + @Override + public boolean isActive() { + return isActive; + } + + @Override + public HostPort getHostPort() { + return hostPort; + } +} diff --git a/redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/healthcheck/impl/HealthCheckEndpointFactory.java b/redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/healthcheck/impl/HealthCheckEndpointFactory.java index ec7963f14..7d9eb758c 100644 --- a/redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/healthcheck/impl/HealthCheckEndpointFactory.java +++ b/redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/healthcheck/impl/HealthCheckEndpointFactory.java @@ -2,6 +2,7 @@ import com.ctrip.xpipe.api.endpoint.Endpoint; import com.ctrip.xpipe.endpoint.HostPort; +import com.ctrip.xpipe.redis.core.entity.KeeperMeta; import com.ctrip.xpipe.redis.core.entity.RedisMeta; /** @@ -13,6 +14,8 @@ public interface HealthCheckEndpointFactory { Endpoint getOrCreateEndpoint(RedisMeta redisMeta); + Endpoint getOrCreateEndpoint(KeeperMeta keeperMeta); + Endpoint getOrCreateEndpoint(HostPort hostPort); void updateRoutes(); diff --git a/redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/healthcheck/impl/HealthCheckInstanceFactory.java b/redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/healthcheck/impl/HealthCheckInstanceFactory.java index 9afdb3303..0b82cb610 100644 --- a/redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/healthcheck/impl/HealthCheckInstanceFactory.java +++ b/redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/healthcheck/impl/HealthCheckInstanceFactory.java @@ -1,8 +1,10 @@ package com.ctrip.xpipe.redis.checker.healthcheck.impl; import com.ctrip.xpipe.redis.checker.healthcheck.ClusterHealthCheckInstance; +import com.ctrip.xpipe.redis.checker.healthcheck.KeeperHealthCheckInstance; import com.ctrip.xpipe.redis.checker.healthcheck.RedisHealthCheckInstance; import com.ctrip.xpipe.redis.core.entity.ClusterMeta; +import com.ctrip.xpipe.redis.core.entity.KeeperMeta; import com.ctrip.xpipe.redis.core.entity.RedisMeta; /** @@ -14,10 +16,15 @@ public interface HealthCheckInstanceFactory { RedisHealthCheckInstance create(RedisMeta redisMeta); + KeeperHealthCheckInstance create(KeeperMeta keeperMeta); + ClusterHealthCheckInstance create(ClusterMeta clusterMeta); void remove(RedisHealthCheckInstance instance); + + void remove(KeeperHealthCheckInstance instance); void remove(ClusterHealthCheckInstance instance); + RedisHealthCheckInstance createRedisInstanceForAssignedAction(RedisMeta redis); } diff --git a/redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/healthcheck/leader/AbstractKeeperLeaderAwareHealthCheckActionFactory.java b/redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/healthcheck/leader/AbstractKeeperLeaderAwareHealthCheckActionFactory.java new file mode 100644 index 000000000..b8eba1ca8 --- /dev/null +++ b/redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/healthcheck/leader/AbstractKeeperLeaderAwareHealthCheckActionFactory.java @@ -0,0 +1,33 @@ +package com.ctrip.xpipe.redis.checker.healthcheck.leader; + +import com.ctrip.xpipe.cluster.ClusterType; +import com.ctrip.xpipe.redis.checker.healthcheck.HealthCheckInstanceManager; +import com.ctrip.xpipe.redis.checker.healthcheck.KeeperHealthCheckActionFactory; +import com.ctrip.xpipe.redis.checker.healthcheck.KeeperHealthCheckInstance; +import com.ctrip.xpipe.redis.checker.healthcheck.KeeperSupport; +import org.springframework.beans.factory.annotation.Autowired; + +import java.util.List; + +public abstract class AbstractKeeperLeaderAwareHealthCheckActionFactory + extends AbstractLeaderAwareHealthCheckActionFactory + implements KeeperHealthCheckActionFactory { + + @Autowired + private HealthCheckInstanceManager healthCheckInstanceManager; + + @Override + protected List getAllInstances() { + return healthCheckInstanceManager.getAllKeeperInstance(); + } + + @Override + protected void registerInstance(KeeperHealthCheckInstance instance) { + ClusterType clusterType = instance.getCheckInfo().getClusterType(); + + if ((clusterType.equals(ClusterType.ONE_WAY) || clusterType.equals(ClusterType.HETERO)) + && AbstractKeeperLeaderAwareHealthCheckActionFactory.this instanceof KeeperSupport){ + registerTo(instance); + } + } +} diff --git a/redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/healthcheck/leader/AbstractLeaderAwareHealthCheckActionFactory.java b/redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/healthcheck/leader/AbstractLeaderAwareHealthCheckActionFactory.java index ac2c58dc9..96634ffdc 100644 --- a/redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/healthcheck/leader/AbstractLeaderAwareHealthCheckActionFactory.java +++ b/redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/healthcheck/leader/AbstractLeaderAwareHealthCheckActionFactory.java @@ -17,8 +17,7 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.ScheduledExecutorService; -import static com.ctrip.xpipe.spring.AbstractSpringConfigContext.GLOBAL_EXECUTOR; -import static com.ctrip.xpipe.spring.AbstractSpringConfigContext.SCHEDULED_EXECUTOR; +import static com.ctrip.xpipe.spring.AbstractSpringConfigContext.*; /** * @author chen.zhu @@ -70,18 +69,23 @@ public void isleader() { new SafeLoop(executors, getAllInstances()) { @Override public void doRun0(V instance) { - ClusterType clusterType = instance.getCheckInfo().getClusterType(); - if ((clusterType.equals(ClusterType.BI_DIRECTION) && AbstractLeaderAwareHealthCheckActionFactory.this instanceof BiDirectionSupport) - || (clusterType.equals(ClusterType.ONE_WAY) && AbstractLeaderAwareHealthCheckActionFactory.this instanceof OneWaySupport - || clusterType.equals(ClusterType.LOCAL_DC) && AbstractLeaderAwareHealthCheckActionFactory.this instanceof LocalDcSupport - || clusterType.equals(ClusterType.SINGLE_DC) && AbstractLeaderAwareHealthCheckActionFactory.this instanceof SingleDcSupport - || clusterType.equals(ClusterType.CROSS_DC) && AbstractLeaderAwareHealthCheckActionFactory.this instanceof CrossDcSupport)) { - registerTo(instance); - } + registerInstance(instance); + } }.run(); } + protected void registerInstance(V instance) { + ClusterType clusterType = instance.getCheckInfo().getClusterType(); + if ((clusterType.equals(ClusterType.BI_DIRECTION) && AbstractLeaderAwareHealthCheckActionFactory.this instanceof BiDirectionSupport) + || (clusterType.equals(ClusterType.ONE_WAY) && AbstractLeaderAwareHealthCheckActionFactory.this instanceof OneWaySupport + || clusterType.equals(ClusterType.LOCAL_DC) && AbstractLeaderAwareHealthCheckActionFactory.this instanceof LocalDcSupport + || clusterType.equals(ClusterType.SINGLE_DC) && AbstractLeaderAwareHealthCheckActionFactory.this instanceof SingleDcSupport + || clusterType.equals(ClusterType.CROSS_DC) && AbstractLeaderAwareHealthCheckActionFactory.this instanceof CrossDcSupport)) { + registerTo(instance); + } + } + @Override public void notLeader() { new SafeLoop(executors, getAllInstances()) { @@ -92,7 +96,7 @@ public void doRun0(V instance) { }.run(); } - private void registerTo(V instance) { + protected void registerTo(V instance) { if(!supportInstnace(instance)) return; SiteLeaderAwareHealthCheckAction action = create(instance); diff --git a/redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/healthcheck/leader/AbstractRedisWithAssignedLeaderAwareHealthCheckActionFactory.java b/redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/healthcheck/leader/AbstractRedisWithAssignedLeaderAwareHealthCheckActionFactory.java new file mode 100644 index 000000000..851f79d3f --- /dev/null +++ b/redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/healthcheck/leader/AbstractRedisWithAssignedLeaderAwareHealthCheckActionFactory.java @@ -0,0 +1,25 @@ +package com.ctrip.xpipe.redis.checker.healthcheck.leader; + +import com.ctrip.xpipe.redis.checker.healthcheck.HealthCheckInstanceManager; +import com.ctrip.xpipe.redis.checker.healthcheck.RedisHealthCheckActionFactory; +import com.ctrip.xpipe.redis.checker.healthcheck.RedisHealthCheckInstance; +import org.springframework.beans.factory.annotation.Autowired; + +import java.util.List; + +/** + * @author lishanglin + * date 2021/1/15 + */ +public abstract class AbstractRedisWithAssignedLeaderAwareHealthCheckActionFactory extends AbstractLeaderAwareHealthCheckActionFactory + implements RedisHealthCheckActionFactory { + + @Autowired + private HealthCheckInstanceManager healthCheckInstanceManager; + + @Override + protected List getAllInstances() { + return healthCheckInstanceManager.getAllRedisInstanceForAssignedAction(); + } + +} diff --git a/redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/healthcheck/meta/DcMetaChangeManager.java b/redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/healthcheck/meta/DcMetaChangeManager.java index 35f09c425..c9bc57b2b 100644 --- a/redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/healthcheck/meta/DcMetaChangeManager.java +++ b/redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/healthcheck/meta/DcMetaChangeManager.java @@ -11,6 +11,6 @@ */ public interface DcMetaChangeManager extends Startable, Stoppable { - void compare(DcMeta future); + void compare(DcMeta future, DcMeta allFutureDcMeta); } diff --git a/redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/healthcheck/meta/DefaultDcMetaChangeManager.java b/redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/healthcheck/meta/DefaultDcMetaChangeManager.java index 5fdf5ed59..796505132 100644 --- a/redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/healthcheck/meta/DefaultDcMetaChangeManager.java +++ b/redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/healthcheck/meta/DefaultDcMetaChangeManager.java @@ -4,17 +4,17 @@ import com.ctrip.xpipe.cluster.ClusterType; import com.ctrip.xpipe.endpoint.HostPort; import com.ctrip.xpipe.lifecycle.AbstractStartStoppable; +import com.ctrip.xpipe.redis.checker.CheckerConsoleService; +import com.ctrip.xpipe.redis.checker.config.CheckerConfig; import com.ctrip.xpipe.redis.checker.healthcheck.HealthCheckInstanceManager; import com.ctrip.xpipe.redis.checker.healthcheck.impl.HealthCheckEndpointFactory; -import com.ctrip.xpipe.redis.core.entity.ClusterMeta; -import com.ctrip.xpipe.redis.core.entity.DcMeta; -import com.ctrip.xpipe.redis.core.entity.RedisMeta; -import com.ctrip.xpipe.redis.core.entity.Route; +import com.ctrip.xpipe.redis.core.entity.*; import com.ctrip.xpipe.redis.core.meta.MetaComparator; import com.ctrip.xpipe.redis.core.meta.MetaComparatorVisitor; import com.ctrip.xpipe.redis.core.meta.comparator.ClusterMetaComparator; import com.ctrip.xpipe.redis.core.meta.comparator.DcMetaComparator; import com.ctrip.xpipe.redis.core.meta.comparator.DcRouteMetaComparator; +import com.ctrip.xpipe.redis.core.meta.comparator.KeeperContainerMetaComparator; import com.ctrip.xpipe.tuple.Pair; import com.ctrip.xpipe.utils.StringUtil; import org.slf4j.Logger; @@ -36,12 +36,18 @@ public class DefaultDcMetaChangeManager extends AbstractStartStoppable implement private DcMeta current; + private DcMeta currentDcAllMeta; + private HealthCheckInstanceManager instanceManager; private static final String currentDcId = FoundationService.DEFAULT.getDataCenter(); private HealthCheckEndpointFactory healthCheckEndpointFactory; + private CheckerConsoleService checkerConsoleService; + + private CheckerConfig checkerConfig; + private final String dcId; private final List clustersToDelete = new ArrayList<>(); @@ -49,18 +55,24 @@ public class DefaultDcMetaChangeManager extends AbstractStartStoppable implement private final List redisListToDelete = new ArrayList<>(); private final List redisListToAdd = new ArrayList<>(); - public DefaultDcMetaChangeManager(String dcId, HealthCheckInstanceManager instanceManager, HealthCheckEndpointFactory healthCheckEndpointFactory) { + public DefaultDcMetaChangeManager(String dcId, HealthCheckInstanceManager instanceManager, + HealthCheckEndpointFactory healthCheckEndpointFactory, + CheckerConsoleService checkerConsoleService, + CheckerConfig checkerConfig) { this.dcId = dcId; this.instanceManager = instanceManager; this.healthCheckEndpointFactory = healthCheckEndpointFactory; + this.checkerConsoleService = checkerConsoleService; + this.checkerConfig = checkerConfig; } @Override - public void compare(DcMeta future) { + public void compare(DcMeta future, DcMeta allFutureDcMeta) { // init if(current == null) { healthCheckEndpointFactory.updateRoutes(); current = future; + currentDcAllMeta = currentDcId.equalsIgnoreCase(dcId) ? allFutureDcMeta : null; return; } @@ -74,6 +86,15 @@ public void compare(DcMeta future) { || !dcRouteMetaComparator.getRemoved().isEmpty()) { healthCheckEndpointFactory.updateRoutes(); } + + if (currentDcId.equalsIgnoreCase(dcId)) { + KeeperContainerMetaComparator keeperContainerMetaComparator + = new KeeperContainerMetaComparator(current, future, currentDcAllMeta, allFutureDcMeta); + keeperContainerMetaComparator.compare(); + keeperContainerMetaComparator.accept(new KeeperContainerMetaComparatorVisitor()); + currentDcAllMeta = allFutureDcMeta; + } + comparator.accept(this); removeAndAdd(); clearUp(); @@ -232,4 +253,56 @@ protected void doStop() { visitRemoved(cluster); } } + + private void removeKeeper(KeeperMeta removed) { + if (null != instanceManager.removeKeeper(new HostPort(removed.getIp(), removed.getPort()))) { + logger.info("[removeKeeper][{}:{}] {}", removed.getIp(), removed.getPort(), removed); + } + } + + private void addKeeper(KeeperMeta added) { + logger.info("[addKeeper][{}:{}] {}", added.getIp(), added.getPort(), added); + instanceManager.getOrCreate(added); + } + + private void removeRedisOnlyForUsedMemory(RedisMeta removed) { + if (null != instanceManager.removeRedisInstanceForAssignedAction(new HostPort(removed.getIp(), removed.getPort()))) { + logger.info("[removeRedisOnlyForUsedMemory][{}:{}] {}", removed.getIp(), removed.getPort(), removed); + } + } + + private void addRedisOnlyForUsedMemory(RedisMeta added) { + logger.info("[addRedisOnlyForUsedMemory][{}:{}] {}", added.getIp(), added.getPort(), added); + instanceManager.getOrCreateRedisInstanceForAssignedAction(added); + } + + private class KeeperContainerMetaComparatorVisitor implements MetaComparatorVisitor { + + @Override + public void visitAdded(InstanceNode added) { + if (added instanceof KeeperMeta) { + addKeeper((KeeperMeta) added); + } else if (added instanceof RedisMeta) { + addRedisOnlyForUsedMemory((RedisMeta) added); + } else { + logger.debug("[visitAdded][do nothing]{}", added); + } + } + + @Override + public void visitModified(MetaComparator comparator) { + // nothing to do + } + + @Override + public void visitRemoved(InstanceNode removed) { + if (removed instanceof KeeperMeta) { + removeKeeper((KeeperMeta) removed); + } else if (removed instanceof RedisMeta){ + removeRedisOnlyForUsedMemory((RedisMeta) removed); + } else { + logger.debug("[visitRemoved][do nothing]{}", removed); + } + } + } } diff --git a/redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/healthcheck/meta/DefaultMetaChangeManager.java b/redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/healthcheck/meta/DefaultMetaChangeManager.java index 66fa0b148..82a349e89 100644 --- a/redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/healthcheck/meta/DefaultMetaChangeManager.java +++ b/redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/healthcheck/meta/DefaultMetaChangeManager.java @@ -2,11 +2,13 @@ import com.ctrip.xpipe.api.factory.ObjectFactory; import com.ctrip.xpipe.concurrent.AbstractExceptionLogTask; +import com.ctrip.xpipe.redis.checker.CheckerConsoleService; import com.ctrip.xpipe.redis.checker.config.CheckerConfig; import com.ctrip.xpipe.redis.checker.healthcheck.HealthCheckInstanceManager; import com.ctrip.xpipe.redis.checker.healthcheck.impl.HealthCheckEndpointFactory; import com.ctrip.xpipe.redis.core.entity.DcMeta; import com.ctrip.xpipe.redis.core.entity.XpipeMeta; +import com.ctrip.xpipe.redis.core.meta.CurrentDcAllMeta; import com.ctrip.xpipe.redis.core.meta.MetaCache; import com.ctrip.xpipe.spring.AbstractSpringConfigContext; import com.ctrip.xpipe.utils.MapUtils; @@ -48,6 +50,12 @@ public class DefaultMetaChangeManager implements MetaChangeManager { @Autowired private MetaCache metaCache; + @Resource + private CurrentDcAllMeta currentDcAllMeta; + + @Autowired + private CheckerConsoleService checkerConsoleService; + private ScheduledFuture future; private ConcurrentMap dcMetaChangeManagers = Maps.newConcurrentMap(); @@ -79,7 +87,7 @@ private void checkDcMetaChange() { ignore(dcId); continue; } - getOrCreate(dcId).compare(entry.getValue()); + getOrCreate(dcId).compare(entry.getValue(), currentDcAllMeta.getCurrentDcAllMeta()); } } @@ -88,7 +96,8 @@ public DcMetaChangeManager getOrCreate(String dcId) { return MapUtils.getOrCreate(dcMetaChangeManagers, dcId, new ObjectFactory() { @Override public DcMetaChangeManager create() { - return new DefaultDcMetaChangeManager(dcId, instanceManager, healthCheckEndpointFactory); + return new DefaultDcMetaChangeManager(dcId, instanceManager, healthCheckEndpointFactory, + checkerConsoleService, checkerConfig); } }); } @@ -115,7 +124,7 @@ public void startIfPossible(String dcId) { } try { DcMetaChangeManager manager = getOrCreate(dcId); - manager.compare(metaCache.getXpipeMeta().findDc(dcId)); + manager.compare(metaCache.getXpipeMeta().findDc(dcId), currentDcAllMeta.getCurrentDcAllMeta()); manager.start(); } catch (Exception e) { logger.error("[start]", e); diff --git a/redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/healthcheck/session/AbstractInstanceSessionManager.java b/redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/healthcheck/session/AbstractInstanceSessionManager.java new file mode 100644 index 000000000..318d56de1 --- /dev/null +++ b/redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/healthcheck/session/AbstractInstanceSessionManager.java @@ -0,0 +1,225 @@ +package com.ctrip.xpipe.redis.checker.healthcheck.session; + +import com.ctrip.xpipe.api.endpoint.Endpoint; +import com.ctrip.xpipe.api.foundation.FoundationService; +import com.ctrip.xpipe.concurrent.AbstractExceptionLogTask; +import com.ctrip.xpipe.endpoint.HostPort; +import com.ctrip.xpipe.pool.XpipeNettyClientKeyedObjectPool; +import com.ctrip.xpipe.redis.checker.CheckerConsoleService; +import com.ctrip.xpipe.redis.checker.config.CheckerConfig; +import com.ctrip.xpipe.redis.checker.healthcheck.impl.HealthCheckEndpointFactory; +import com.ctrip.xpipe.redis.core.entity.*; +import com.ctrip.xpipe.redis.core.meta.CurrentDcAllMeta; +import com.ctrip.xpipe.redis.core.meta.MetaCache; +import com.ctrip.xpipe.utils.VisibleForTesting; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; + +import javax.annotation.PostConstruct; +import javax.annotation.PreDestroy; +import javax.annotation.Resource; +import java.util.HashSet; +import java.util.LinkedList; +import java.util.List; +import java.util.Set; +import java.util.concurrent.*; +import java.util.stream.Collectors; + +import static com.ctrip.xpipe.redis.checker.resource.Resource.REDIS_COMMAND_EXECUTOR; +import static com.ctrip.xpipe.redis.checker.resource.Resource.REDIS_SESSION_NETTY_CLIENT_POOL; +import static com.ctrip.xpipe.spring.AbstractSpringConfigContext.GLOBAL_EXECUTOR; + +/** + * @author yu + *

+ * 2023/10/31 + */ +public abstract class AbstractInstanceSessionManager implements InstanceSessionManager{ + + protected Logger logger = LoggerFactory.getLogger(getClass()); + + private ConcurrentMap sessions = new ConcurrentHashMap<>(); + + @Autowired + protected MetaCache metaCache; + + @Resource + protected CurrentDcAllMeta currentDcAllMeta; + + @Autowired + protected CheckerConsoleService checkerConsoleService; + + @Autowired + protected CheckerConfig checkerConfig; + + @Autowired + private HealthCheckEndpointFactory endpointFactory; + + @Resource(name = REDIS_SESSION_NETTY_CLIENT_POOL) + private XpipeNettyClientKeyedObjectPool keyedObjectPool; + + @Resource(name = REDIS_COMMAND_EXECUTOR) + private ScheduledExecutorService scheduled; + + @Resource(name = GLOBAL_EXECUTOR) + private ExecutorService executors; + + protected String currentDcId = FoundationService.DEFAULT.getDataCenter(); + + @Autowired + private CheckerConfig config; + + @VisibleForTesting + public static long checkUnusedRedisDelaySeconds = 4; + + + @PostConstruct + public void postConstruct(){ + scheduled.scheduleAtFixedRate(new AbstractExceptionLogTask() { + @Override + protected void doRun() throws Exception { + try { + removeUnusedInstances(); + } catch (Exception e) { + logger.error("[removeUnusedInstances]", e); + } + + for(RedisSession redisSession : sessions.values()){ + try{ + redisSession.check(); + }catch (Exception e){ + logger.error("[check]" + redisSession, e); + } + } + } + }, checkUnusedRedisDelaySeconds, checkUnusedRedisDelaySeconds, TimeUnit.SECONDS); + } + + @Override + public RedisSession findOrCreateSession(Endpoint endpoint) { + RedisSession session = sessions.get(endpoint); + + if (session == null) { + synchronized (this) { + session = sessions.get(endpoint); + if (session == null) { + session = new RedisSession(endpoint, scheduled, keyedObjectPool, config); + sessions.put(endpoint, session); + } + } + } + + return session; + } + + @Override + public RedisSession findOrCreateSession(HostPort hostPort) { + return findOrCreateSession(endpointFactory.getOrCreateEndpoint(hostPort)); + } + + @VisibleForTesting + protected void removeUnusedInstances() { + Set currentStoredRedises = sessions.keySet(); + if(currentStoredRedises.isEmpty()) + return; + + Set redisInUse = getInUseInstances(); + if(redisInUse == null || redisInUse.isEmpty()) { + return; + } + List unusedRedises = new LinkedList<>(); + + for(Endpoint endpoint : currentStoredRedises) { + if(!redisInUse.contains(new HostPort(endpoint.getHost(), endpoint.getPort()))) { + unusedRedises.add(endpoint); + } + } + + if(unusedRedises.isEmpty()) { + return; + } + unusedRedises.forEach(endpoint -> { + RedisSession redisSession = sessions.getOrDefault(endpoint, null); + if(redisSession != null) { + logger.info("[removeUnusedRedises]Redis: {} not in use, remove from session manager", endpoint); + // add try logic to continue working on others + try { + redisSession.closeConnection(); + } catch (Exception ignore) { + + } + sessions.remove(endpoint); + } + }); + } + + protected abstract Set getInUseInstances(); + + @VisibleForTesting + protected Set getSessionsForKeeper(DcMeta dcMeta, DcMeta dcAllMeta) { + Set instanceInUse = new HashSet<>(); + Set keeperContainerSet = dcMeta.getKeeperContainers().stream().map(KeeperContainerMeta::getId).collect(Collectors.toSet()); + if (dcAllMeta == null || dcAllMeta.getClusters() == null) return instanceInUse; + + dcAllMeta.getClusters().values().forEach(clusterMeta -> { + for (ShardMeta shardMeta : clusterMeta.getAllShards().values()){ + if (shardMeta.getKeepers() == null || shardMeta.getKeepers().isEmpty()) continue; + shardMeta.getKeepers().forEach(keeperMeta -> { + if (keeperContainerSet.contains(keeperMeta.getKeeperContainerId())) { + HostPort monitorInstance = getMonitorInstance(shardMeta.getRedises(), keeperMeta); + if (monitorInstance != null) instanceInUse.add(monitorInstance); + } + }); + } + }); + return instanceInUse; + } + + protected abstract HostPort getMonitorInstance(List redises, KeeperMeta keeper); + + private void closeAllConnections() { + try { + executors.execute(new Runnable() { + @Override + public void run() { + for (RedisSession session : sessions.values()) { + session.closeConnection(); + } + } + }); + } catch (Exception e) { + logger.error("[closeAllConnections] {}", e); + } + } + + @PreDestroy + public void preDestroy(){ + closeAllConnections(); + } + + public AbstractInstanceSessionManager setKeyedObjectPool(XpipeNettyClientKeyedObjectPool keyedObjectPool) { + this.keyedObjectPool = keyedObjectPool; + return this; + } + + public AbstractInstanceSessionManager setScheduled(ScheduledExecutorService scheduled) { + this.scheduled = scheduled; + return this; + } + + public AbstractInstanceSessionManager setExecutors(ExecutorService executors) { + this.executors = executors; + return this; + } + + public AbstractInstanceSessionManager setEndpointFactory(HealthCheckEndpointFactory endpointFactory) { + this.endpointFactory = endpointFactory; + return this; + } + + public void setConfig(CheckerConfig config) { + this.config = config; + } + +} diff --git a/redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/healthcheck/session/DefaultKeeperSessionManager.java b/redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/healthcheck/session/DefaultKeeperSessionManager.java new file mode 100644 index 000000000..a75b75e50 --- /dev/null +++ b/redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/healthcheck/session/DefaultKeeperSessionManager.java @@ -0,0 +1,33 @@ +package com.ctrip.xpipe.redis.checker.healthcheck.session; + +import com.ctrip.xpipe.endpoint.HostPort; +import com.ctrip.xpipe.redis.core.entity.DcMeta; +import com.ctrip.xpipe.redis.core.entity.KeeperMeta; +import com.ctrip.xpipe.redis.core.entity.RedisMeta; +import org.springframework.stereotype.Component; + +import java.util.List; +import java.util.Set; + +/** + * @author yu + *

+ * 2023/10/31 + */ +@Component +public class DefaultKeeperSessionManager extends AbstractInstanceSessionManager implements KeeperSessionManager { + + @Override + protected Set getInUseInstances() { + DcMeta currentDcAllMeta = metaCache.getXpipeMeta().getDcs().get(currentDcId); + if (currentDcAllMeta != null) + return getSessionsForKeeper(currentDcAllMeta, this.currentDcAllMeta.getCurrentDcAllMeta()); + + return null; + } + + @Override + protected HostPort getMonitorInstance(List redises, KeeperMeta keeper) { + return new HostPort(keeper.getIp(), keeper.getPort()); + } +} diff --git a/redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/healthcheck/session/DefaultRedisSessionManager.java b/redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/healthcheck/session/DefaultRedisSessionManager.java index 8120eedce..f3a4e3eaf 100644 --- a/redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/healthcheck/session/DefaultRedisSessionManager.java +++ b/redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/healthcheck/session/DefaultRedisSessionManager.java @@ -1,155 +1,37 @@ package com.ctrip.xpipe.redis.checker.healthcheck.session; -import com.ctrip.xpipe.api.endpoint.Endpoint; -import com.ctrip.xpipe.concurrent.AbstractExceptionLogTask; import com.ctrip.xpipe.endpoint.HostPort; -import com.ctrip.xpipe.pool.XpipeNettyClientKeyedObjectPool; -import com.ctrip.xpipe.redis.checker.config.CheckerConfig; -import com.ctrip.xpipe.redis.checker.healthcheck.impl.HealthCheckEndpointFactory; -import com.ctrip.xpipe.redis.core.entity.ClusterMeta; -import com.ctrip.xpipe.redis.core.entity.DcMeta; -import com.ctrip.xpipe.redis.core.entity.RedisMeta; -import com.ctrip.xpipe.redis.core.entity.ShardMeta; -import com.ctrip.xpipe.redis.core.meta.MetaCache; -import com.ctrip.xpipe.utils.VisibleForTesting; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Autowired; +import com.ctrip.xpipe.redis.core.entity.*; import org.springframework.stereotype.Component; -import javax.annotation.PostConstruct; -import javax.annotation.PreDestroy; -import javax.annotation.Resource; import java.util.HashSet; import java.util.LinkedList; import java.util.List; import java.util.Set; -import java.util.concurrent.*; -import static com.ctrip.xpipe.redis.checker.resource.Resource.REDIS_COMMAND_EXECUTOR; -import static com.ctrip.xpipe.redis.checker.resource.Resource.REDIS_SESSION_NETTY_CLIENT_POOL; -import static com.ctrip.xpipe.spring.AbstractSpringConfigContext.GLOBAL_EXECUTOR; +import static com.ctrip.xpipe.redis.core.meta.comparator.KeeperContainerMetaComparator.getMonitorRedisMeta; /** * @author marsqing * * Dec 1, 2016 6:42:01 PM */ -@Component -public class DefaultRedisSessionManager implements RedisSessionManager { - - private Logger logger = LoggerFactory.getLogger(getClass()); - - private ConcurrentMap sessions = new ConcurrentHashMap<>(); - - @Autowired - private MetaCache metaCache; - - @Autowired - private HealthCheckEndpointFactory endpointFactory; - - @Resource(name = REDIS_SESSION_NETTY_CLIENT_POOL) - private XpipeNettyClientKeyedObjectPool keyedObjectPool; - - @Resource(name = REDIS_COMMAND_EXECUTOR) - private ScheduledExecutorService scheduled; - - @Resource(name = GLOBAL_EXECUTOR) - private ExecutorService executors; - @Autowired - private CheckerConfig config; - - @VisibleForTesting - public static long checkUnusedRedisDelaySeconds = 4; - - @PostConstruct - public void postConstruct(){ - scheduled.scheduleAtFixedRate(new AbstractExceptionLogTask() { - @Override - protected void doRun() throws Exception { - try { - removeUnusedRedises(); - } catch (Exception e) { - logger.error("[removeUnusedRedises]", e); - } - - for(RedisSession redisSession : sessions.values()){ - try{ - redisSession.check(); - }catch (Exception e){ - logger.error("[check]" + redisSession, e); - } - } - } - }, checkUnusedRedisDelaySeconds, checkUnusedRedisDelaySeconds, TimeUnit.SECONDS); - } - - @Override - public RedisSession findOrCreateSession(Endpoint endpoint) { - RedisSession session = sessions.get(endpoint); - - if (session == null) { - synchronized (this) { - session = sessions.get(endpoint); - if (session == null) { - session = new RedisSession(endpoint, scheduled, keyedObjectPool, config); - sessions.put(endpoint, session); - } - } - } - - return session; - } +@Component +public class DefaultRedisSessionManager extends AbstractInstanceSessionManager implements RedisSessionManager { @Override - public RedisSession findOrCreateSession(HostPort hostPort) { - return findOrCreateSession(endpointFactory.getOrCreateEndpoint(hostPort)); - } - - @VisibleForTesting - protected void removeUnusedRedises() { - Set currentStoredRedises = sessions.keySet(); - if(currentStoredRedises.isEmpty()) - return; - - Set redisInUse = getInUseRedises(); - if(redisInUse == null || redisInUse.isEmpty()) { - return; - } - List unusedRedises = new LinkedList<>(); - - for(Endpoint endpoint : currentStoredRedises) { - if(!redisInUse.contains(new HostPort(endpoint.getHost(), endpoint.getPort()))) { - unusedRedises.add(endpoint); - } - } - - if(unusedRedises.isEmpty()) { - return; - } - unusedRedises.forEach(endpoint -> { - RedisSession redisSession = sessions.getOrDefault(endpoint, null); - if(redisSession != null) { - logger.info("[removeUnusedRedises]Redis: {} not in use, remove from session manager", endpoint); - // add try logic to continue working on others - try { - redisSession.closeConnection(); - } catch (Exception ignore) { - - } - sessions.remove(endpoint); - } - }); - } - - - private Set getInUseRedises() { + protected Set getInUseInstances() { Set redisInUse = new HashSet<>(); List dcMetas = new LinkedList<>(metaCache.getXpipeMeta().getDcs().values()); if(dcMetas.isEmpty()) return null; for (DcMeta dcMeta : dcMetas) { if(dcMeta == null) break; + + if (dcMeta.getId().equalsIgnoreCase(currentDcId)) { + redisInUse.addAll(getSessionsForKeeper(dcMeta, currentDcAllMeta.getCurrentDcAllMeta())); + } + for (ClusterMeta clusterMeta : dcMeta.getClusters().values()) { for (ShardMeta shardMeta : clusterMeta.getShards().values()) { for (RedisMeta redisMeta : shardMeta.getRedises()) { @@ -161,47 +43,9 @@ private Set getInUseRedises() { return redisInUse; } - private void closeAllConnections() { - try { - executors.execute(new Runnable() { - @Override - public void run() { - for (RedisSession session : sessions.values()) { - session.closeConnection(); - } - } - }); - } catch (Exception e) { - logger.error("[closeAllConnections] {}", e); - } - } - - @PreDestroy - public void preDestroy(){ - closeAllConnections(); - } - - public DefaultRedisSessionManager setKeyedObjectPool(XpipeNettyClientKeyedObjectPool keyedObjectPool) { - this.keyedObjectPool = keyedObjectPool; - return this; - } - - public DefaultRedisSessionManager setScheduled(ScheduledExecutorService scheduled) { - this.scheduled = scheduled; - return this; - } - - public DefaultRedisSessionManager setExecutors(ExecutorService executors) { - this.executors = executors; - return this; - } - - public DefaultRedisSessionManager setEndpointFactory(HealthCheckEndpointFactory endpointFactory) { - this.endpointFactory = endpointFactory; - return this; - } - - public void setConfig(CheckerConfig config) { - this.config = config; + @Override + protected HostPort getMonitorInstance(List redises, KeeperMeta keeper) { + RedisMeta monitorRedisMeta = getMonitorRedisMeta(redises); + return monitorRedisMeta == null ? null : new HostPort(monitorRedisMeta.getIp(), monitorRedisMeta.getPort()); } } diff --git a/redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/healthcheck/session/InstanceSessionManager.java b/redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/healthcheck/session/InstanceSessionManager.java new file mode 100644 index 000000000..ed2f23313 --- /dev/null +++ b/redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/healthcheck/session/InstanceSessionManager.java @@ -0,0 +1,16 @@ +package com.ctrip.xpipe.redis.checker.healthcheck.session; + +import com.ctrip.xpipe.api.endpoint.Endpoint; +import com.ctrip.xpipe.endpoint.HostPort; + +/** + * @author yu + *

+ * 2023/10/31 + */ +public interface InstanceSessionManager { + + RedisSession findOrCreateSession(Endpoint endpoint); + + RedisSession findOrCreateSession(HostPort hostPort); +} diff --git a/redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/healthcheck/session/KeeperInstanceManager.java b/redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/healthcheck/session/KeeperInstanceManager.java new file mode 100644 index 000000000..9a87fc8ce --- /dev/null +++ b/redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/healthcheck/session/KeeperInstanceManager.java @@ -0,0 +1,10 @@ +package com.ctrip.xpipe.redis.checker.healthcheck.session; + +/** + * @author yu + *

+ * 2023/10/31 + */ +public interface KeeperInstanceManager extends InstanceSessionManager{ + +} diff --git a/redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/healthcheck/session/KeeperSessionManager.java b/redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/healthcheck/session/KeeperSessionManager.java new file mode 100644 index 000000000..55513b3bb --- /dev/null +++ b/redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/healthcheck/session/KeeperSessionManager.java @@ -0,0 +1,9 @@ +package com.ctrip.xpipe.redis.checker.healthcheck.session; + +/** + * @author yu + *

+ * 2023/10/31 + */ +public interface KeeperSessionManager extends InstanceSessionManager{ +} diff --git a/redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/healthcheck/session/RedisSessionManager.java b/redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/healthcheck/session/RedisSessionManager.java index 419444faa..50db8e756 100644 --- a/redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/healthcheck/session/RedisSessionManager.java +++ b/redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/healthcheck/session/RedisSessionManager.java @@ -1,16 +1,10 @@ package com.ctrip.xpipe.redis.checker.healthcheck.session; -import com.ctrip.xpipe.api.endpoint.Endpoint; -import com.ctrip.xpipe.endpoint.HostPort; - /** * @author marsqing * * Dec 1, 2016 6:40:43 PM */ -public interface RedisSessionManager { - - RedisSession findOrCreateSession(Endpoint endpoint); +public interface RedisSessionManager extends InstanceSessionManager { - RedisSession findOrCreateSession(HostPort hostPort); } diff --git a/redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/healthcheck/stability/DefaultStabilityHolder.java b/redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/healthcheck/stability/DefaultStabilityHolder.java index 2f1ef9c17..398381994 100644 --- a/redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/healthcheck/stability/DefaultStabilityHolder.java +++ b/redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/healthcheck/stability/DefaultStabilityHolder.java @@ -15,16 +15,16 @@ public class DefaultStabilityHolder implements StabilityHolder { private CheckerConfig config; - private boolean stable; + private boolean stable = true; - private long expiredAtMill; + private long expiredAtMill = 0L; + + public DefaultStabilityHolder() { + } - @Autowired public DefaultStabilityHolder(StabilityInspector inspector, CheckerConfig config) { this.inspector = inspector; this.config = config; - this.stable = true; - this.expiredAtMill = 0L; } @Override @@ -54,6 +54,16 @@ public void setStaticStable(boolean stable, int ttl) { this.expiredAtMill = System.currentTimeMillis() + (ttl * 1000); } + @Autowired + public void setInspector(StabilityInspector inspector) { + this.inspector = inspector; + } + + @Autowired + public void setConfig(CheckerConfig config) { + this.config = config; + } + @Override public void useDynamicStable() { this.expiredAtMill = 0; diff --git a/redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/healthcheck/stability/StabilityInspector.java b/redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/healthcheck/stability/StabilityInspector.java index 99d30ee09..372269223 100644 --- a/redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/healthcheck/stability/StabilityInspector.java +++ b/redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/healthcheck/stability/StabilityInspector.java @@ -37,11 +37,11 @@ public class StabilityInspector extends AbstractLifecycle implements TopElement private CheckerConfig config; - private AtomicBoolean stable; + private AtomicBoolean stable = new AtomicBoolean(true); - private AtomicInteger continuousMismatchTimes; + private AtomicInteger continuousMismatchTimes = new AtomicInteger(); - private AtomicInteger continuousNoInterested; + private AtomicInteger continuousNoInterested = new AtomicInteger(); private DynamicDelayPeriodTask task; @@ -49,15 +49,14 @@ public class StabilityInspector extends AbstractLifecycle implements TopElement private static final String TYPE = "stability"; - @Autowired + public StabilityInspector() { + } + public StabilityInspector(DefaultDelayPingActionCollector defaultDelayPingActionCollector, MetaCache metaCache, CheckerConfig checkerConfig) { this.collector = defaultDelayPingActionCollector; this.metaCache = metaCache; this.config = checkerConfig; - this.stable = new AtomicBoolean(true); - this.continuousMismatchTimes = new AtomicInteger(); - this.continuousNoInterested = new AtomicInteger(); } protected boolean isSiteStable() { @@ -174,4 +173,18 @@ public int getOrder() { return LOWEST_PRECEDENCE; } + @Autowired + public void setCollector(DefaultDelayPingActionCollector collector) { + this.collector = collector; + } + + @Autowired + public void setMetaCache(MetaCache metaCache) { + this.metaCache = metaCache; + } + + @Autowired + public void setConfig(CheckerConfig config) { + this.config = config; + } } diff --git a/redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/impl/KeeperContainerInfoReporter.java b/redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/impl/KeeperContainerInfoReporter.java new file mode 100644 index 000000000..8169c49ce --- /dev/null +++ b/redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/impl/KeeperContainerInfoReporter.java @@ -0,0 +1,127 @@ +package com.ctrip.xpipe.redis.checker.impl; + +import com.ctrip.xpipe.api.foundation.FoundationService; +import com.ctrip.xpipe.redis.checker.CheckerConsoleService; +import com.ctrip.xpipe.redis.checker.cluster.GroupCheckerLeaderAware; +import com.ctrip.xpipe.redis.checker.config.CheckerConfig; +import com.ctrip.xpipe.redis.checker.healthcheck.actions.keeper.info.RedisUsedMemoryCollector; +import com.ctrip.xpipe.redis.checker.healthcheck.actions.keeper.infoStats.KeeperFlowCollector; +import com.ctrip.xpipe.redis.checker.model.DcClusterShard; +import com.ctrip.xpipe.redis.checker.model.KeeperContainerUsedInfoModel; +import com.ctrip.xpipe.tuple.Pair; +import com.ctrip.xpipe.utils.VisibleForTesting; +import com.ctrip.xpipe.utils.XpipeThreadFactory; +import com.ctrip.xpipe.utils.job.DynamicDelayPeriodTask; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.annotation.PostConstruct; +import javax.annotation.PreDestroy; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; + +public class KeeperContainerInfoReporter implements GroupCheckerLeaderAware { + + private RedisUsedMemoryCollector redisUsedMemoryCollector; + + private KeeperFlowCollector keeperFlowCollector; + + private ScheduledExecutorService scheduled; + + private DynamicDelayPeriodTask keeperContainerInfoReportTask; + + private CheckerConsoleService checkerConsoleService; + + private CheckerConfig config; + + private static final String CURRENT_IDC = FoundationService.DEFAULT.getDataCenter(); + + private static final Logger logger = LoggerFactory.getLogger(HealthCheckReporter.class); + + + public KeeperContainerInfoReporter(RedisUsedMemoryCollector redisUsedMemoryCollector, CheckerConsoleService + checkerConsoleService, KeeperFlowCollector keeperFlowCollector, CheckerConfig config) { + this.redisUsedMemoryCollector = redisUsedMemoryCollector; + this.keeperFlowCollector = keeperFlowCollector; + this.checkerConsoleService = checkerConsoleService; + this.config = config; + } + + @PostConstruct + public void init() { + logger.debug("[postConstruct] start"); + this.scheduled = Executors.newScheduledThreadPool(1, XpipeThreadFactory.create("KeeperContainerInfoReporter")); + this.keeperContainerInfoReportTask = new DynamicDelayPeriodTask("KeeperContainerInfoReporter", + this::reportKeeperContainerInfo, () -> 2 * config.getKeeperCheckerIntervalMilli(), scheduled); + } + + @PreDestroy + public void destroy() { + try { + keeperContainerInfoReportTask.stop(); + this.scheduled.shutdownNow(); + } catch (Throwable th) { + logger.info("[preDestroy] fail", th); + } + } + + @Override + public void isleader() { + try { + logger.debug("[isleader] become leader"); + keeperContainerInfoReportTask.start(); + } catch (Throwable th) { + logger.info("[isleader] keeperContainerInfoReportTask start fail", th); + } + } + + @Override + public void notLeader() { + try { + logger.debug("[notLeader] loss leader"); + keeperContainerInfoReportTask.stop(); + } catch (Throwable th) { + logger.info("[notLeader] keeperContainerInfoReportTask stop fail", th); + } + } + + @VisibleForTesting + void reportKeeperContainerInfo() { + try { + logger.debug("[reportKeeperContainerInfo] start"); + Map> hostPort2InputFlow = keeperFlowCollector.getHostPort2InputFlow(); + Map dcClusterShardUsedMemory = redisUsedMemoryCollector.getDcClusterShardUsedMemory(); + List result = new ArrayList<>(hostPort2InputFlow.keySet().size()); + + hostPort2InputFlow.forEach((keeperIp, inputFlowMap) -> { + KeeperContainerUsedInfoModel model = new KeeperContainerUsedInfoModel(); + model.setKeeperIp(keeperIp).setDcName(CURRENT_IDC); + long totalInputFlow = 0; + long totalRedisUsedMemory = 0; + Map> detailInfo = new HashMap<>(); + for (Map.Entry entry : inputFlowMap.entrySet()) { + totalInputFlow += entry.getValue(); + Long redisUsedMemory = dcClusterShardUsedMemory.get(entry.getKey()); + if (redisUsedMemory == null) { + logger.warn("[reportKeeperContainerInfo] redisUsedMemory is null, dcClusterShard: {}", entry.getKey()); + redisUsedMemory = 0L; + } + totalRedisUsedMemory += redisUsedMemory; + + detailInfo.put(entry.getKey(), new Pair<>(entry.getValue(), redisUsedMemory)); + } + + model.setDetailInfo(detailInfo).setTotalInputFlow(totalInputFlow).setTotalRedisUsedMemory(totalRedisUsedMemory); + result.add(model); + }); + logger.debug("[reportKeeperContainerInfo] result: {}", result); + checkerConsoleService.reportKeeperContainerInfo(config.getConsoleAddress(), result, config.getClustersPartIndex()); + } catch (Throwable th) { + logger.error("[reportKeeperContainerInfo] fail", th); + } + } +} diff --git a/redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/impl/TestCurrentDcAllMetaCache.java b/redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/impl/TestCurrentDcAllMetaCache.java new file mode 100644 index 000000000..a2aed6c7b --- /dev/null +++ b/redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/impl/TestCurrentDcAllMetaCache.java @@ -0,0 +1,16 @@ +package com.ctrip.xpipe.redis.checker.impl; + +import com.ctrip.xpipe.redis.core.entity.DcMeta; +import com.ctrip.xpipe.redis.core.meta.CurrentDcAllMeta; + +/** + * @author yu + *

+ * 2023/11/14 + */ +public class TestCurrentDcAllMetaCache implements CurrentDcAllMeta { + @Override + public DcMeta getCurrentDcAllMeta() { + return null; + } +} diff --git a/redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/model/DcClusterShard.java b/redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/model/DcClusterShard.java index 3c353adfb..0873e1de8 100644 --- a/redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/model/DcClusterShard.java +++ b/redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/model/DcClusterShard.java @@ -1,8 +1,11 @@ package com.ctrip.xpipe.redis.checker.model; +import java.io.Serializable; import java.util.Objects; -public class DcClusterShard { +public class DcClusterShard implements Serializable { + + protected static final String SPLITTER = ":"; protected String dcId; @@ -20,6 +23,15 @@ public DcClusterShard(String dcId, String clusterId, String shardId) { this.shardId = shardId; } + public DcClusterShard(String info) { + String[] split = info.split(SPLITTER); + if (split.length >= 3) { + this.dcId = split[0]; + this.clusterId = split[1]; + this.shardId = split[2]; + } + } + public DcClusterShard setDcId(String dcId) { this.dcId = dcId; return this; @@ -59,16 +71,11 @@ public boolean equals(Object o) { @Override public int hashCode() { - return Objects.hash(dcId, clusterId, shardId); } @Override public String toString() { - return "DcClusterShard{" + - "dcId='" + dcId + '\'' + - ", clusterId='" + clusterId + '\'' + - ", shardId='" + shardId + '\'' + - '}'; + return getDcId() + SPLITTER + getClusterId() + SPLITTER + getShardId(); } } diff --git a/redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/model/DcClusterShardPeer.java b/redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/model/DcClusterShardPeer.java index 15dce4069..980fbd93c 100644 --- a/redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/model/DcClusterShardPeer.java +++ b/redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/model/DcClusterShardPeer.java @@ -4,7 +4,6 @@ import java.util.Objects; public final class DcClusterShardPeer extends DcClusterShard implements Serializable { - private static final String SPLITTER = ":"; private String peerDcId; diff --git a/redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/model/KeeperContainerUsedInfoModel.java b/redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/model/KeeperContainerUsedInfoModel.java new file mode 100644 index 000000000..746971484 --- /dev/null +++ b/redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/model/KeeperContainerUsedInfoModel.java @@ -0,0 +1,98 @@ +package com.ctrip.xpipe.redis.checker.model; + +import com.ctrip.xpipe.tuple.Pair; + +import java.util.Map; +import java.util.Objects; + +public class KeeperContainerUsedInfoModel { + + private String keeperIp; + + private String dcName; + + private long totalInputFlow; + + private long totalRedisUsedMemory; + + private Map> detailInfo; + + public KeeperContainerUsedInfoModel() { + } + + public KeeperContainerUsedInfoModel(String keeperIp, String dcName, long totalInputFlow, long totalRedisUsedMemory) { + this.keeperIp = keeperIp; + this.dcName = dcName; + this.totalInputFlow = totalInputFlow; + this.totalRedisUsedMemory = totalRedisUsedMemory; + } + + public String getDcName() { + return dcName; + } + + public KeeperContainerUsedInfoModel setDcName(String dcName) { + this.dcName = dcName; + return this; + } + + public String getKeeperIp() { + return keeperIp; + } + + public KeeperContainerUsedInfoModel setKeeperIp(String keeperIp) { + this.keeperIp = keeperIp; + return this; + } + + public long getTotalInputFlow() { + return totalInputFlow; + } + + public KeeperContainerUsedInfoModel setTotalInputFlow(long totalInputFlow) { + this.totalInputFlow = totalInputFlow; + return this; + } + + public long getTotalRedisUsedMemory() { + return totalRedisUsedMemory; + } + + public KeeperContainerUsedInfoModel setTotalRedisUsedMemory(long totalRedisUsedMemory) { + this.totalRedisUsedMemory = totalRedisUsedMemory; + return this; + } + + public Map> getDetailInfo() { + return detailInfo; + } + + public KeeperContainerUsedInfoModel setDetailInfo(Map> detailInfo) { + this.detailInfo = detailInfo; + return this; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + KeeperContainerUsedInfoModel that = (KeeperContainerUsedInfoModel) o; + return Objects.equals(keeperIp, that.keeperIp) && Objects.equals(dcName, that.dcName); + } + + @Override + public int hashCode() { + return Objects.hash(keeperIp, dcName); + } + + @Override + public String toString() { + return "KeeperContainerUsedInfoModel{" + + "keeperIp='" + keeperIp + '\'' + + ", dcName='" + dcName + '\'' + + ", totalInputFlow=" + totalInputFlow + + ", totalRedisUsedMemory=" + totalRedisUsedMemory + + ", detailInfo=" + detailInfo + + '}'; + } +} diff --git a/redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/resource/DefaultCheckerConsoleService.java b/redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/resource/DefaultCheckerConsoleService.java index c78a3ae06..1a22c2a85 100644 --- a/redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/resource/DefaultCheckerConsoleService.java +++ b/redis/redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/resource/DefaultCheckerConsoleService.java @@ -3,7 +3,6 @@ import com.ctrip.xpipe.api.email.EmailResponse; import com.ctrip.xpipe.api.migration.OuterClientService; import com.ctrip.xpipe.api.server.Server; -import com.ctrip.xpipe.endpoint.HostPort; import com.ctrip.xpipe.exception.XpipeRuntimeException; import com.ctrip.xpipe.redis.checker.CheckerConsoleService; import com.ctrip.xpipe.redis.checker.alert.AlertMessageEntity; @@ -11,16 +10,15 @@ import com.ctrip.xpipe.redis.checker.healthcheck.RedisHealthCheckInstance; import com.ctrip.xpipe.redis.checker.model.CheckerStatus; import com.ctrip.xpipe.redis.checker.model.HealthCheckResult; +import com.ctrip.xpipe.redis.checker.model.KeeperContainerUsedInfoModel; import com.ctrip.xpipe.redis.checker.model.ProxyTunnelInfo; import com.ctrip.xpipe.redis.core.console.ConsoleCheckerPath; import com.ctrip.xpipe.redis.core.entity.SentinelMeta; import com.ctrip.xpipe.redis.core.entity.XpipeMeta; import com.ctrip.xpipe.redis.core.service.AbstractService; import com.ctrip.xpipe.redis.core.transform.DefaultSaxParser; -import com.ctrip.xpipe.tuple.Pair; import com.ctrip.xpipe.utils.StringUtil; import com.ctrip.xpipe.utils.VisibleForTesting; -import com.google.common.collect.Maps; import org.springframework.context.annotation.Lazy; import org.springframework.core.ParameterizedTypeReference; import org.springframework.http.HttpMethod; @@ -71,6 +69,15 @@ public XpipeMeta getXpipeAllMeta(String console) throws SAXException, IOExcepti return DefaultSaxParser.parse(raw); } + public XpipeMeta getXpipeDcAllMeta(String console, String dcName) throws SAXException, IOException { + UriComponents comp = UriComponentsBuilder.fromHttpUrl(console + ConsoleCheckerPath.PATH_GET_DC_ALL_META) + .queryParam("format", "xml").buildAndExpand(dcName); + + String raw = restTemplate.getForObject(comp.toString(), String.class); + if (StringUtil.isEmpty(raw)) return null; + return DefaultSaxParser.parse(raw); + } + public List getProxyTunnelInfos(String console) { ResponseEntity> resp = restTemplate.exchange(console + ConsoleCheckerPath.PATH_GET_PROXY_CHAINS, HttpMethod.GET, null, proxyTunnelInfosTypeDef); @@ -87,6 +94,17 @@ public void report(String console, HealthCheckResult result) { restTemplate.put(console + ConsoleCheckerPath.PATH_PUT_HEALTH_CHECK_RESULT, result); } + @Override + public void reportKeeperContainerInfo(String console, List keeperContainerUsedInfoModels, int index) { + try { + restTemplate.postForEntity(console + ConsoleCheckerPath.PATH_POST_KEEPER_CONTAINER_INFO_RESULT, + keeperContainerUsedInfoModels, RetMessage.class, index); + + } catch (Throwable th) { + logger.error("report keeper used info fail : {}", index, th); + } + } + @VisibleForTesting protected void setRestOperations(RestOperations restOperations) { this.restTemplate = restOperations; diff --git a/redis/redis-checker/src/test/java/com/ctrip/xpipe/redis/checker/AbstractCheckerIntegrationTest.java b/redis/redis-checker/src/test/java/com/ctrip/xpipe/redis/checker/AbstractCheckerIntegrationTest.java index 26c99ffe5..1348237fc 100644 --- a/redis/redis-checker/src/test/java/com/ctrip/xpipe/redis/checker/AbstractCheckerIntegrationTest.java +++ b/redis/redis-checker/src/test/java/com/ctrip/xpipe/redis/checker/AbstractCheckerIntegrationTest.java @@ -12,10 +12,8 @@ import com.ctrip.xpipe.redis.checker.healthcheck.HealthChecker; import com.ctrip.xpipe.redis.checker.healthcheck.actions.ping.DefaultPingService; import com.ctrip.xpipe.redis.checker.healthcheck.actions.ping.PingService; -import com.ctrip.xpipe.redis.checker.impl.CheckerClusterHealthManager; -import com.ctrip.xpipe.redis.checker.impl.CheckerRedisInfoManager; -import com.ctrip.xpipe.redis.checker.impl.DefaultRemoteCheckerManager; -import com.ctrip.xpipe.redis.checker.impl.TestMetaCache; +import com.ctrip.xpipe.redis.checker.impl.*; +import com.ctrip.xpipe.redis.core.meta.CurrentDcAllMeta; import com.ctrip.xpipe.redis.core.meta.MetaCache; import com.ctrip.xpipe.redis.core.spring.AbstractRedisConfigContext; import com.ctrip.xpipe.utils.OsUtils; @@ -72,6 +70,12 @@ public MetaCache metaCache() { return new TestMetaCache(); } + @Bean + public CurrentDcAllMeta testCurrentDcAllMeta() { + return new TestCurrentDcAllMetaCache(); + } + + @Bean public TestPersistenceCache persistence() { return new TestPersistenceCache(); diff --git a/redis/redis-checker/src/test/java/com/ctrip/xpipe/redis/checker/AbstractCheckerTest.java b/redis/redis-checker/src/test/java/com/ctrip/xpipe/redis/checker/AbstractCheckerTest.java index 79c1ab70f..dec7e5d14 100644 --- a/redis/redis-checker/src/test/java/com/ctrip/xpipe/redis/checker/AbstractCheckerTest.java +++ b/redis/redis-checker/src/test/java/com/ctrip/xpipe/redis/checker/AbstractCheckerTest.java @@ -8,13 +8,11 @@ import com.ctrip.xpipe.redis.checker.healthcheck.actions.redisconf.RedisCheckRule; import com.ctrip.xpipe.redis.checker.healthcheck.config.DefaultHealthCheckConfig; import com.ctrip.xpipe.redis.checker.healthcheck.config.HealthCheckConfig; -import com.ctrip.xpipe.redis.checker.healthcheck.impl.DefaultClusterHealthCheckInstance; -import com.ctrip.xpipe.redis.checker.healthcheck.impl.DefaultClusterInstanceInfo; -import com.ctrip.xpipe.redis.checker.healthcheck.impl.DefaultRedisHealthCheckInstance; -import com.ctrip.xpipe.redis.checker.healthcheck.impl.DefaultRedisInstanceInfo; +import com.ctrip.xpipe.redis.checker.healthcheck.impl.*; import com.ctrip.xpipe.redis.checker.healthcheck.session.RedisSession; import com.ctrip.xpipe.redis.core.AbstractRedisTest; import com.ctrip.xpipe.redis.core.entity.ClusterMeta; +import com.ctrip.xpipe.redis.core.entity.KeeperMeta; import com.ctrip.xpipe.redis.core.entity.RedisMeta; import org.junit.BeforeClass; @@ -126,6 +124,25 @@ protected ClusterHealthCheckInstance newRandomClusterHealthCheckInstance(String return instance; } + protected KeeperHealthCheckInstance newRandomKeeperHealthCheckInstance(String ip, int port) throws Exception { + KeeperMeta keeperMeta = newRandomFakeKeeperMeta(ip, port); + DefaultKeeperInstanceInfo info = new DefaultKeeperInstanceInfo(((ClusterMeta) keeperMeta.parent().parent()).parent().getId(), + ((ClusterMeta) keeperMeta.parent().parent()).getId(), keeperMeta.parent().getId(), + new HostPort(keeperMeta.getIp(), keeperMeta.getPort()), + keeperMeta.parent().getActiveDc(), ClusterType.ONE_WAY); + + return newRandomRedisHealthCheckInstance(info); + } + + protected KeeperHealthCheckInstance newRandomRedisHealthCheckInstance(KeeperInstanceInfo info) throws Exception { + DefaultKeeperHealthCheckInstance instance = new DefaultKeeperHealthCheckInstance(); + instance.setInstanceInfo(info); + instance.setEndpoint(new DefaultEndPoint(info.getHostPort().getHost(), info.getHostPort().getPort())); + instance.setHealthCheckConfig(new DefaultHealthCheckConfig(buildCheckerConfig(), buildDcRelationsService())); + instance.setSession(new RedisSession(instance.getEndpoint(), scheduled, getXpipeNettyClientKeyedObjectPool(), buildCheckerConfig())); + return instance; + } + protected CheckerConfig buildCheckerConfig() { return new TestConfig(); } diff --git a/redis/redis-checker/src/test/java/com/ctrip/xpipe/redis/checker/AllTests.java b/redis/redis-checker/src/test/java/com/ctrip/xpipe/redis/checker/AllTests.java index 023ae9700..9cfce5346 100644 --- a/redis/redis-checker/src/test/java/com/ctrip/xpipe/redis/checker/AllTests.java +++ b/redis/redis-checker/src/test/java/com/ctrip/xpipe/redis/checker/AllTests.java @@ -27,6 +27,12 @@ import com.ctrip.xpipe.redis.checker.healthcheck.actions.interaction.processor.OuterClientServiceProcessorTest; import com.ctrip.xpipe.redis.checker.healthcheck.actions.interaction.processor.route.DefaultRouteHealthEventProcessorTest; import com.ctrip.xpipe.redis.checker.healthcheck.actions.interaction.processor.route.HeteroRouteHealthEventProcessorTest; +import com.ctrip.xpipe.redis.checker.healthcheck.actions.keeper.info.RedisInfoActionFactoryTest; +import com.ctrip.xpipe.redis.checker.healthcheck.actions.keeper.info.RedisInfoActionTest; +import com.ctrip.xpipe.redis.checker.healthcheck.actions.keeper.info.RedisUsedMemoryCollectorTest; +import com.ctrip.xpipe.redis.checker.healthcheck.actions.keeper.infoStats.KeeperFlowCollectorTest; +import com.ctrip.xpipe.redis.checker.healthcheck.actions.keeper.infoStats.KeeperInfoStatsActionFactoryTest; +import com.ctrip.xpipe.redis.checker.healthcheck.actions.keeper.infoStats.KeeperInfoStatsActionTest; import com.ctrip.xpipe.redis.checker.healthcheck.actions.ping.PingActionContextTest; import com.ctrip.xpipe.redis.checker.healthcheck.actions.ping.PingActionTest; import com.ctrip.xpipe.redis.checker.healthcheck.actions.redisconf.RedisCheckRuleTest; @@ -81,6 +87,8 @@ import com.ctrip.xpipe.redis.checker.healthcheck.factory.HealthCheckEndpointFactoryTest; import com.ctrip.xpipe.redis.checker.healthcheck.impl.DefaultHealthCheckerMockTest; import com.ctrip.xpipe.redis.checker.healthcheck.meta.DefaultDcMetaChangeManagerTest; +import com.ctrip.xpipe.redis.checker.healthcheck.session.DefaultKeeperSessionManagerTest; +import com.ctrip.xpipe.redis.checker.healthcheck.session.DefaultRedisSessionManagerTest; import com.ctrip.xpipe.redis.checker.healthcheck.stability.StabilityInspectorTest; import com.ctrip.xpipe.redis.checker.impl.*; import com.ctrip.xpipe.redis.checker.model.HealthCheckResultSerializeTest; @@ -196,7 +204,18 @@ GtidGapCheckActionTest.class, GtidGapCheckActionControllerTest.class, + KeeperFlowCollectorTest.class, + KeeperInfoStatsActionTest.class, + KeeperInfoStatsActionFactoryTest.class, + + RedisInfoActionFactoryTest.class, + RedisInfoActionTest.class, + RedisUsedMemoryCollectorTest.class, + + DefaultKeeperSessionManagerTest.class, + DefaultRedisSessionManagerTest.class, + DefaultHealthCheckConfigTest.class }) public class AllTests { -} +} \ No newline at end of file diff --git a/redis/redis-checker/src/test/java/com/ctrip/xpipe/redis/checker/TestConfig.java b/redis/redis-checker/src/test/java/com/ctrip/xpipe/redis/checker/TestConfig.java index a1e9412da..9c5fcc1e5 100644 --- a/redis/redis-checker/src/test/java/com/ctrip/xpipe/redis/checker/TestConfig.java +++ b/redis/redis-checker/src/test/java/com/ctrip/xpipe/redis/checker/TestConfig.java @@ -70,6 +70,11 @@ public int getRedisReplicationHealthCheckInterval() { return 2000; } + @Override + public int getCheckerCurrentDcAllMetaRefreshIntervalMilli() { + return 300000; + } + @Override public int getClusterHealthCheckInterval() { return 300000; @@ -287,7 +292,7 @@ public Set getOuterClusterTypes() { @Override public Map sentinelMasterConfig() { - return null; + return new HashMap<>(); } @Override @@ -320,4 +325,9 @@ public Set getMigrationUnsupportedClusters() { return new HashSet<>(); } + @Override + public int getKeeperCheckerIntervalMilli() { + return 1000; + } + } diff --git a/redis/redis-checker/src/test/java/com/ctrip/xpipe/redis/checker/healthcheck/actions/keeper/info/RedisInfoActionFactoryTest.java b/redis/redis-checker/src/test/java/com/ctrip/xpipe/redis/checker/healthcheck/actions/keeper/info/RedisInfoActionFactoryTest.java new file mode 100644 index 000000000..d15b0857a --- /dev/null +++ b/redis/redis-checker/src/test/java/com/ctrip/xpipe/redis/checker/healthcheck/actions/keeper/info/RedisInfoActionFactoryTest.java @@ -0,0 +1,30 @@ +package com.ctrip.xpipe.redis.checker.healthcheck.actions.keeper.info; + +import com.ctrip.xpipe.api.foundation.FoundationService; +import com.ctrip.xpipe.cluster.ClusterType; +import com.ctrip.xpipe.redis.checker.AbstractCheckerIntegrationTest; +import com.ctrip.xpipe.redis.checker.healthcheck.KeeperSupport; +import com.ctrip.xpipe.redis.checker.healthcheck.RedisHealthCheckInstance; +import org.junit.Assert; +import org.junit.Test; +import org.springframework.beans.factory.annotation.Autowired; + +/** + * Created by yu + * 2023/8/29 + */ +public class RedisInfoActionFactoryTest extends AbstractCheckerIntegrationTest { + @Autowired + RedisInfoActionFactory factory; + + @Test + public void testCreate() throws Exception { + RedisHealthCheckInstance instance = newRandomRedisHealthCheckInstance(FoundationService.DEFAULT.getDataCenter(), ClusterType.ONE_WAY, 6379); + RedisInfoAction action = factory.create(instance); + action.doStart(); + + Assert.assertTrue(factory.support().isInstance(action)); + Assert.assertEquals(1, action.getListeners().size()); + Assert.assertTrue(action.getListeners().stream().allMatch(listener -> listener instanceof KeeperSupport)); + } +} diff --git a/redis/redis-checker/src/test/java/com/ctrip/xpipe/redis/checker/healthcheck/actions/keeper/info/RedisInfoActionTest.java b/redis/redis-checker/src/test/java/com/ctrip/xpipe/redis/checker/healthcheck/actions/keeper/info/RedisInfoActionTest.java new file mode 100644 index 000000000..5b6d8dc55 --- /dev/null +++ b/redis/redis-checker/src/test/java/com/ctrip/xpipe/redis/checker/healthcheck/actions/keeper/info/RedisInfoActionTest.java @@ -0,0 +1,119 @@ +package com.ctrip.xpipe.redis.checker.healthcheck.actions.keeper.info; + +import com.ctrip.xpipe.api.foundation.FoundationService; +import com.ctrip.xpipe.cluster.ClusterType; +import com.ctrip.xpipe.redis.checker.AbstractCheckerTest; +import com.ctrip.xpipe.redis.checker.healthcheck.*; +import com.ctrip.xpipe.simpleserver.Server; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import java.util.concurrent.TimeoutException; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Function; + +/** + * Created by yu + * 2023/8/29 + */ +public class RedisInfoActionTest extends AbstractCheckerTest { + private RedisHealthCheckInstance instance; + + private Server redis; + + private RedisInfoAction action; + + private AtomicInteger redisCallCnt = new AtomicInteger(0); + + private AtomicInteger listenerCallCnt = new AtomicInteger(0); + + AtomicReference contextRef = new AtomicReference(); + + private static final String INFO_RESPONSE = "# Memory\n" + + "used_memory:550515888\n" + + "used_memory_human:525.01M\n" + + "used_memory_rss:544837632\n" + + "used_memory_rss_human:519.60M\n" + + "used_memory_peak:550862048\n" + + "used_memory_peak_human:525.34M\n" + + "used_memory_peak_perc:99.94%\n" + + "used_memory_overhead:212334872\n" + + "used_memory_startup:803592\n" + + "used_memory_dataset:338181016\n" + + "used_memory_dataset_perc:61.52%\n" + + "allocator_allocated:550579680\n" + + "allocator_active:551002112\n" + + "allocator_resident:560062464\n" + + "total_system_memory:8201191424\n" + + "total_system_memory_human:7.64G\n" + + "used_memory_lua:45056\n" + + "used_memory_lua_human:44.00K\n" + + "used_memory_scripts:816\n" + + "used_memory_scripts_human:816B\n" + + "number_of_cached_scripts:2"; + + @Before + public void beforeInfoStatsActionTest() throws Exception { + redis = startServer(randomPort(), new Function() { + @Override + public String apply(String s) { + redisCallCnt.incrementAndGet(); + + if (s.trim().toLowerCase().startsWith("info")) { + return String.format("$%d\r\n%s\r\n", INFO_RESPONSE.length(), INFO_RESPONSE); + } else { + return "+OK\r\n"; + } + } + }); + + instance = newRandomRedisHealthCheckInstance(FoundationService.DEFAULT.getDataCenter(), ClusterType.ONE_WAY, redis.getPort()); + action = new RedisInfoAction(scheduled, instance, executors); + + action.addListener(new HealthCheckActionListener() { + @Override + public void onAction(ActionContext context) { + logger.info("[onAction]{}", context); + listenerCallCnt.incrementAndGet(); + contextRef.set(context); + } + + @Override + public boolean worksfor(ActionContext t) { + return true; + } + + @Override + public void stopWatch(HealthCheckAction action) { + + } + }); + } + + @Test + public void testMaster() throws TimeoutException { + AbstractHealthCheckAction.ScheduledHealthCheckTask task = action.new ScheduledHealthCheckTask(); + task.run(); + + waitConditionUntilTimeOut(() -> listenerCallCnt.get() == 1, 3000); + + RedisInfoActionContext context = (RedisInfoActionContext)contextRef.get(); + Assert.assertNotNull(context); + Assert.assertEquals(550515888, context.getResult().getUsedMemory()); + + } + + @After + public void afterInfoStatsActionTest() { + if(redis != null) { + try { + redis.stop(); + } catch (Throwable th) { + th.printStackTrace(); + } + } + } +} \ No newline at end of file diff --git a/redis/redis-checker/src/test/java/com/ctrip/xpipe/redis/checker/healthcheck/actions/keeper/info/RedisUsedMemoryCollectorTest.java b/redis/redis-checker/src/test/java/com/ctrip/xpipe/redis/checker/healthcheck/actions/keeper/info/RedisUsedMemoryCollectorTest.java new file mode 100644 index 000000000..9a86bac10 --- /dev/null +++ b/redis/redis-checker/src/test/java/com/ctrip/xpipe/redis/checker/healthcheck/actions/keeper/info/RedisUsedMemoryCollectorTest.java @@ -0,0 +1,232 @@ +package com.ctrip.xpipe.redis.checker.healthcheck.actions.keeper.info; + +import com.ctrip.xpipe.api.foundation.FoundationService; +import com.ctrip.xpipe.cluster.ClusterType; +import com.ctrip.xpipe.redis.checker.AbstractCheckerTest; +import com.ctrip.xpipe.redis.checker.healthcheck.RedisHealthCheckInstance; +import com.ctrip.xpipe.redis.checker.model.DcClusterShard; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +/** + * Created by yu + * 2023/8/29 + */ +public class RedisUsedMemoryCollectorTest extends AbstractCheckerTest { + + private RedisHealthCheckInstance instance; + + private RedisUsedMemoryCollector listener; + + private RedisInfoActionContext context; + + private static final String INFO_RESPONSE_OF_REDIS = "# Memory\n" + + "used_memory:550515888\n" + + "used_memory_human:525.01M\n" + + "used_memory_rss:544837632\n" + + "used_memory_rss_human:519.60M\n" + + "used_memory_peak:550862048\n" + + "used_memory_peak_human:525.34M\n" + + "used_memory_peak_perc:99.94%\n" + + "used_memory_overhead:212334872\n" + + "used_memory_startup:803592\n" + + "used_memory_dataset:338181016\n" + + "used_memory_dataset_perc:61.52%\n" + + "allocator_allocated:550579680\n" + + "allocator_active:551002112\n" + + "allocator_resident:560062464\n" + + "total_system_memory:8201191424\n" + + "total_system_memory_human:7.64G\n" + + "used_memory_lua:45056\n" + + "used_memory_lua_human:44.00K\n" + + "used_memory_scripts:816\n" + + "used_memory_scripts_human:816B\n" + + "number_of_cached_scripts:2\n" + + "used_memory_scripts_human:0B\n" + + "number_of_cached_scripts:0\n" + + "maxmemory:2415919104"; + + private static final String INFO_RESPONSE_OF_ROR = "# Memory\n" + + "used_memory:109632408\n" + + "used_memory_human:104.55M\n" + + "used_memory_rss:92024832\n" + + "used_memory_rss_human:87.76M\n" + + "used_memory_peak:109773720\n" + + "used_memory_peak_human:104.69M\n" + + "used_memory_peak_perc:99.87%\n" + + "used_memory_overhead:76041118\n" + + "used_memory_startup:8819264\n" + + "used_memory_dataset:33601768\n" + + "used_memory_dataset_perc:33.33%\n" + + "allocator_allocated:115857800\n" + + "allocator_active:118788096\n" + + "allocator_resident:127303680\n" + + "total_system_memory:270332620800\n" + + "total_system_memory_human:251.77G\n" + + "used_memory_scripts_human:0B\n" + + "number_of_cached_scripts:0\n" + + "maxmemory:2415919104\n" + + "# Swap\n" + + "swap_used_db_size:2472\n" + + "swap_max_db_size:32212254720\n" + + "swap_used_db_percent:0.00%\n" + + "swap_used_disk_size:96530432\n" + + "swap_disk_capacity:214748364800\n" + + "swap_used_disk_percent:0.04%\n" + + "swap_error_count:1\n" + + "swap_swapin_attempt_count:2\n" + + "swap_swapin_not_found_count:1\n" + + "swap_swapin_no_io_count:0\n" + + "swap_swapin_memory_hit_perc:0.00%\n" + + "swap_swapin_keyspace_hit_perc:50.00%\n" + + "swap_swapin_not_found_coldfilter_cuckoofilter_filt_count:1\n" + + "swap_swapin_not_found_coldfilter_absentcache_filt_count:0\n" + + "swap_swapin_not_found_coldfilter_miss:0\n" + + "swap_swapin_not_found_coldfilter_filt_perc:0.00%" ; + + + private static final String INFO_RESPONSE_OF_ROR2 = "# Memory\n" + + "used_memory:2415919104\n" + + "maxmemory:2415919104\n" + + "# Swap\n" + + "swap_used_db_size:1207959551\n" + + "# Keyspace\n" + + "db0:keys=2,evicts=1,metas=0,expires=0,avg_ttl=0"; + + + private static final String INFO_RESPONSE_OF_ROR3 = "# Memory\n" + + "used_memory:2415919104\n" + + "maxmemory:2415919104\n" + + "# Swap\n" + + "swap_used_db_size:1207959552\n" + + "# Keyspace\n" + + "db0:keys=10,evicts=30,metas=0,expires=0,avg_ttl=0"; + + private static final String INFO_RESPONSE_OF_ROR4 = "# Memory\n" + + "used_memory:2415919104\n" + + "maxmemory:2415919104\n" + + "# Swap\n" + + "swap_used_db_size:1207959552\n" + + "# Keyspace\n"; + + private static final String INFO_RESPONSE_OF_ROR5 = "# Memory\n" + + "used_memory:2415919104\n" + + "maxmemory:2415919104\n" + + "# Swap\n" + + "swap_used_db_size:1207959552\n" + + "# Keyspace\n" + + "db0:evicts=30,metas=0,expires=0,avg_ttl=0"; + + private static final String INFO_RESPONSE_OF_ROR6 = "# Memory\n" + + "used_memory:2415919104\n" + + "maxmemory:2415919104\n" + + "# Swap\n" + + "swap_used_db_size:1207959552\n" + + "# Keyspace\n" + + "db0:keys=10,metas=0,expires=0,avg_ttl=0"; + + private static final String INFO_RESPONSE_OF_ROR7 = "# Memory\n" + + "used_memory:2415919104\n" + + "maxmemory:2415919104\n" + + "# Swap\n" + + "swap_used_db_size:1207959552\n" + + "# Keyspace\n" + + "db0:keys=0,evicts=30,metas=0,expires=0,avg_ttl=0"; + + + @Before + public void before() throws Exception { + listener = new RedisUsedMemoryCollector(); + instance = newRandomRedisHealthCheckInstance(FoundationService.DEFAULT.getDataCenter(), ClusterType.ONE_WAY, randomPort()); + } + + @Test + public void testGetUsedMemoryInfoWithRor() { + // test used_memory < maxmemory + context = new RedisInfoActionContext(instance, INFO_RESPONSE_OF_ROR); + listener.onAction(context); + Assert.assertTrue(listener.worksfor(context)); + Assert.assertEquals(1, listener.getDcClusterShardUsedMemory().size()); + Assert.assertEquals(109632408, (long) listener.getDcClusterShardUsedMemory().get(new DcClusterShard("jq", "cluster", "shard"))); + + } + + + @Test + public void testGetUsedMemoryInfoWithRor2() { + // test dbSize < maxmemory * 0.5 + context = new RedisInfoActionContext(instance, INFO_RESPONSE_OF_ROR2); + listener.onAction(context); + Assert.assertTrue(listener.worksfor(context)); + Assert.assertEquals(1, listener.getDcClusterShardUsedMemory().size()); + Assert.assertEquals(2415919104L, (long) listener.getDcClusterShardUsedMemory().get(new DcClusterShard("jq", "cluster", "shard"))); + + } + + @Test + public void testGetUsedMemoryInfoWithRor3() { + // test dbSize >= maxmemory * 0.5, keyspace normal + context = new RedisInfoActionContext(instance, INFO_RESPONSE_OF_ROR3); + listener.onAction(context); + Assert.assertTrue(listener.worksfor(context)); + Assert.assertEquals(1, listener.getDcClusterShardUsedMemory().size()); + Assert.assertEquals(2415919104L * 4, (long) listener.getDcClusterShardUsedMemory().get(new DcClusterShard("jq", "cluster", "shard"))); + + } + + @Test + public void testGetUsedMemoryInfoWithRor4() { + // test dbSize >= maxmemory * 0.5, keyspace Exception db0 == null + context = new RedisInfoActionContext(instance, INFO_RESPONSE_OF_ROR4); + listener.onAction(context); + Assert.assertTrue(listener.worksfor(context)); + Assert.assertEquals(1, listener.getDcClusterShardUsedMemory().size()); + Assert.assertEquals(2415919104L * 3, (long) listener.getDcClusterShardUsedMemory().get(new DcClusterShard("jq", "cluster", "shard"))); + + } + + @Test + public void testGetUsedMemoryInfoWithRor5() { + // test dbSize >= maxmemory * 0.5, keyspace Exception keys==null + context = new RedisInfoActionContext(instance, INFO_RESPONSE_OF_ROR5); + listener.onAction(context); + Assert.assertTrue(listener.worksfor(context)); + Assert.assertEquals(1, listener.getDcClusterShardUsedMemory().size()); + Assert.assertEquals(2415919104L * 3, (long) listener.getDcClusterShardUsedMemory().get(new DcClusterShard("jq", "cluster", "shard"))); + + } + + @Test + public void testGetUsedMemoryInfoWithRor6() { + // test dbSize >= maxmemory * 0.5, keyspace Exception evict==null + context = new RedisInfoActionContext(instance, INFO_RESPONSE_OF_ROR6); + listener.onAction(context); + Assert.assertTrue(listener.worksfor(context)); + Assert.assertEquals(1, listener.getDcClusterShardUsedMemory().size()); + Assert.assertEquals(2415919104L * 3, (long) listener.getDcClusterShardUsedMemory().get(new DcClusterShard("jq", "cluster", "shard"))); + + } + + + @Test + public void testGetUsedMemoryInfoWithRor7() { + // test dbSize >= maxmemory * 0.5, keyspace Exception keys == 0 + context = new RedisInfoActionContext(instance, INFO_RESPONSE_OF_ROR7); + listener.onAction(context); + Assert.assertTrue(listener.worksfor(context)); + Assert.assertEquals(1, listener.getDcClusterShardUsedMemory().size()); + Assert.assertEquals(2415919104L * 3, (long) listener.getDcClusterShardUsedMemory().get(new DcClusterShard("jq", "cluster", "shard"))); + + } + + @Test + public void testGetUsedMemoryInfoWithRedis() { + context = new RedisInfoActionContext(instance, INFO_RESPONSE_OF_REDIS); + listener.onAction(context); + Assert.assertTrue(listener.worksfor(context)); + Assert.assertEquals(1, listener.getDcClusterShardUsedMemory().size()); + Assert.assertEquals(550515888, (long) listener.getDcClusterShardUsedMemory().get(new DcClusterShard("jq", "cluster", "shard"))); + + } +} \ No newline at end of file diff --git a/redis/redis-checker/src/test/java/com/ctrip/xpipe/redis/checker/healthcheck/actions/keeper/infoStats/KeeperFlowCollectorTest.java b/redis/redis-checker/src/test/java/com/ctrip/xpipe/redis/checker/healthcheck/actions/keeper/infoStats/KeeperFlowCollectorTest.java new file mode 100644 index 000000000..ed610c296 --- /dev/null +++ b/redis/redis-checker/src/test/java/com/ctrip/xpipe/redis/checker/healthcheck/actions/keeper/infoStats/KeeperFlowCollectorTest.java @@ -0,0 +1,65 @@ +package com.ctrip.xpipe.redis.checker.healthcheck.actions.keeper.infoStats; + +import com.ctrip.xpipe.redis.checker.AbstractCheckerTest; +import com.ctrip.xpipe.redis.checker.healthcheck.KeeperHealthCheckInstance; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.mockito.MockitoAnnotations; + +/** + * Created by yu + * 2023/8/29 + */ +public class KeeperFlowCollectorTest extends AbstractCheckerTest { + + private KeeperFlowCollector listener; + + private KeeperHealthCheckInstance instance; + + private KeeperInfoStatsActionContext context; + + @Before + public void before() throws Exception { + listener = new KeeperFlowCollector(); + int keeperPort = 6380; + String keeperIp = "127.0.0.1"; + MockitoAnnotations.initMocks(this); + instance = newRandomKeeperHealthCheckInstance(keeperIp, keeperPort); + String info = "# Stats\n" + + "sync_full:0\n" + + "sync_partial_ok:0\n" + + "sync_partial_err:0\n" + + "total_net_input_bytes:1716640145\n" + + "instantaneous_input_kbps:1.584961\n" + + "total_net_output_bytes:97539279\n" + + "instantaneous_output_kbps:0.030273\n" + + "peak_input_kbps:60315\n" + + "peak_output_kbps:2\n" + + "psync_fail_send:0"; + +// InfoResultExtractor extractors = new InfoResultExtractor(info); + context = new KeeperInfoStatsActionContext(instance, info); + } + + @Test + public void testParseResult() { + listener.onAction(context); + Assert.assertTrue(listener.worksfor(context)); + Assert.assertEquals(1, listener.getHostPort2InputFlow().size()); + } + + @Test + public void testWithNonResult() { + String info = "# Stats\n" + + "sync_full:0\n" + + "sync_partial_ok:0\n" + + "sync_partial_err:0\n" + + "psync_fail_send:0"; + context = new KeeperInfoStatsActionContext(instance, info); + listener.onAction(context); + Assert.assertTrue(listener.worksfor(context)); + Assert.assertEquals(0, listener.getHostPort2InputFlow().size()); + } + +} \ No newline at end of file diff --git a/redis/redis-checker/src/test/java/com/ctrip/xpipe/redis/checker/healthcheck/actions/keeper/infoStats/KeeperInfoStatsActionFactoryTest.java b/redis/redis-checker/src/test/java/com/ctrip/xpipe/redis/checker/healthcheck/actions/keeper/infoStats/KeeperInfoStatsActionFactoryTest.java new file mode 100644 index 000000000..fb0b683e6 --- /dev/null +++ b/redis/redis-checker/src/test/java/com/ctrip/xpipe/redis/checker/healthcheck/actions/keeper/infoStats/KeeperInfoStatsActionFactoryTest.java @@ -0,0 +1,58 @@ +package com.ctrip.xpipe.redis.checker.healthcheck.actions.keeper.infoStats; + +import com.ctrip.xpipe.redis.checker.AbstractCheckerIntegrationTest; +import com.ctrip.xpipe.redis.checker.healthcheck.HealthCheckAction; +import com.ctrip.xpipe.redis.checker.healthcheck.KeeperHealthCheckInstance; +import com.ctrip.xpipe.redis.checker.healthcheck.leader.SiteLeaderAwareHealthCheckAction; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.springframework.beans.factory.annotation.Autowired; + +/** + * Created by yu + * 2023/8/28 + */ +public class KeeperInfoStatsActionFactoryTest extends AbstractCheckerIntegrationTest { + + @Autowired + private KeeperInfoStatsActionFactory factory; + + private KeeperHealthCheckInstance instance; + + @Before + public void init() throws Exception { + instance = newRandomKeeperHealthCheckInstance(randomString(), randomPort()); + instance.register(factory.create(instance)); + } + + @Test + public void testCreate() { + Assert.assertTrue(instance.getHealthCheckActions().size() > 0); + KeeperInfoStatsAction action = (KeeperInfoStatsAction) instance.getHealthCheckActions().get(0); + Assert.assertTrue(action.getLifecycleState().isEmpty()); + } + + @Test + public void testSupport() { + Assert.assertEquals(KeeperInfoStatsAction.class, factory.support()); + } + + @Test + public void testDestroy() { + HealthCheckAction target = null; + for(HealthCheckAction action : instance.getHealthCheckActions()) { + if(action.getClass().isAssignableFrom(factory.support())) { + target = action; + factory.destroy((SiteLeaderAwareHealthCheckAction) target); + } + } + + + logger.info("{}", instance.getHealthCheckActions()); + instance.unregister(target); + instance.unregister(target); + Assert.assertTrue(instance.getHealthCheckActions().isEmpty()); + } + +} \ No newline at end of file diff --git a/redis/redis-checker/src/test/java/com/ctrip/xpipe/redis/checker/healthcheck/actions/keeper/infoStats/KeeperInfoStatsActionTest.java b/redis/redis-checker/src/test/java/com/ctrip/xpipe/redis/checker/healthcheck/actions/keeper/infoStats/KeeperInfoStatsActionTest.java new file mode 100644 index 000000000..a7b271cf0 --- /dev/null +++ b/redis/redis-checker/src/test/java/com/ctrip/xpipe/redis/checker/healthcheck/actions/keeper/infoStats/KeeperInfoStatsActionTest.java @@ -0,0 +1,92 @@ +package com.ctrip.xpipe.redis.checker.healthcheck.actions.keeper.infoStats; + +import com.ctrip.xpipe.redis.checker.AbstractCheckerTest; +import com.ctrip.xpipe.redis.checker.healthcheck.AbstractHealthCheckAction; +import com.ctrip.xpipe.redis.checker.healthcheck.KeeperHealthCheckInstance; +import com.ctrip.xpipe.simpleserver.Server; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.MockitoAnnotations; + +import java.util.function.Function; + +import static org.mockito.Mockito.doNothing; + +/** + * Created by yu + * 2023/8/28 + */ +public class KeeperInfoStatsActionTest extends AbstractCheckerTest { + + private KeeperInfoStatsAction action; + + @Mock + private KeeperInfoStatsActionListener listener; + + private KeeperInfoStatsActionContext context = null; + + Server redis; + + @Before + public void beforeVersionCheckActionTest() throws Exception { + int keeperPort = 6380; + String keeperIp = "127.0.0.1"; + MockitoAnnotations.initMocks(this); + KeeperHealthCheckInstance instance = newRandomKeeperHealthCheckInstance(keeperIp, keeperPort); + action = new KeeperInfoStatsAction(scheduled, instance, executors); + String info = "# Stats\n" + + "sync_full:0\n" + + "sync_partial_ok:0\n" + + "sync_partial_err:0\n" + + "total_net_input_bytes:1716640145\n" + + "total_net_output_bytes:97539279\n" + + "instantaneous_input_kbps:0.584961\n" + + "instantaneous_output_kbps:0.030273\n" + + "peak_input_kbps:60315\n" + + "peak_output_kbps:2\n" + + "psync_fail_send:0"; + + redis = startServer(keeperPort, new Function() { + @Override + public String apply(String s) { + + if (s.trim().toLowerCase().startsWith("info stats")) { + return String.format("$%d\r\n%s\r\n", info.length(), info); + } else { + return "+OK\r\n"; + } + } + }); + action.addListener(listener); + + Mockito.when(listener.worksfor(Mockito.any())).thenReturn(true); + doNothing().when(listener).onAction(Mockito.any()); + Mockito.doAnswer(invocation -> { + context = invocation.getArgument(0, KeeperInfoStatsActionContext.class); + return null; + }).when(listener).onAction(Mockito.any()); + } + + @Test + public void testDoScheduledTask0Positive() throws Exception { + AbstractHealthCheckAction.ScheduledHealthCheckTask task = action.new ScheduledHealthCheckTask(); + task.run(); + waitConditionUntilTimeOut(() -> null != context, 3000); + Assert.assertEquals(0, (long) context.getResult().getKeeperInstantaneousInputKbps()); + } + + @After + public void stopRedis() { + if(redis != null) { + try { + redis.stop(); + } catch (Exception e) { + e.printStackTrace(); + } + } + } +} \ No newline at end of file diff --git a/redis/redis-checker/src/test/java/com/ctrip/xpipe/redis/checker/healthcheck/factory/DefaultHealthCheckInstanceFactoryTest.java b/redis/redis-checker/src/test/java/com/ctrip/xpipe/redis/checker/healthcheck/factory/DefaultHealthCheckInstanceFactoryTest.java index 41823187a..19e0cee69 100644 --- a/redis/redis-checker/src/test/java/com/ctrip/xpipe/redis/checker/healthcheck/factory/DefaultHealthCheckInstanceFactoryTest.java +++ b/redis/redis-checker/src/test/java/com/ctrip/xpipe/redis/checker/healthcheck/factory/DefaultHealthCheckInstanceFactoryTest.java @@ -5,6 +5,7 @@ import com.ctrip.xpipe.endpoint.DefaultEndPoint; import com.ctrip.xpipe.endpoint.HostPort; import com.ctrip.xpipe.redis.checker.AbstractCheckerIntegrationTest; +import com.ctrip.xpipe.redis.checker.healthcheck.KeeperHealthCheckInstance; import com.ctrip.xpipe.redis.checker.healthcheck.RedisHealthCheckInstance; import com.ctrip.xpipe.redis.checker.healthcheck.impl.DefaultHealthCheckEndpointFactory; import com.ctrip.xpipe.redis.checker.healthcheck.impl.DefaultHealthCheckInstanceFactory; @@ -66,6 +67,40 @@ public void testCreate() { factory.remove(instance); } + @Test + public void testCreateRedisForAssignedAction() { + RedisMeta redisMeta = normalRedisMeta(); + when(metaCache.getDc(new HostPort(redisMeta.getIp(), redisMeta.getPort()))).thenReturn("oy"); + RedisHealthCheckInstance instance = factory.createRedisInstanceForAssignedAction(redisMeta); + + Assert.assertNotNull(instance.getEndpoint()); + Assert.assertNotNull(instance.getHealthCheckConfig()); + Assert.assertNotNull(instance.getCheckInfo()); + Assert.assertNotNull(instance.getCheckInfo().getRedisCheckRules()); + Assert.assertNotNull(instance.getRedisSession()); + + Assert.assertEquals(instance.getEndpoint(), new DefaultEndPoint(redisMeta.getIp(), redisMeta.getPort())); + Assert.assertTrue(instance.getLifecycleState().isStarted()); + factory.remove(instance); + } + + @Test + public void testCreateKeeper() { + KeeperMeta keeperMeta = normalKeeperMeta(); + when(metaCache.getDc(new HostPort(keeperMeta.getIp(), keeperMeta.getPort()))).thenReturn("oy"); + + KeeperHealthCheckInstance instance = factory.create(keeperMeta); + + Assert.assertNotNull(instance.getEndpoint()); + Assert.assertNotNull(instance.getHealthCheckConfig()); + Assert.assertNotNull(instance.getCheckInfo()); + Assert.assertNotNull(instance.getRedisSession()); + + Assert.assertEquals(instance.getEndpoint(), new DefaultEndPoint(keeperMeta.getIp(), keeperMeta.getPort())); + Assert.assertTrue(instance.getLifecycleState().isStarted()); + factory.remove(instance); + } + @Test public void testCreateProxyEnabledInstance() { XpipeMeta meta = new XpipeMeta(); @@ -113,6 +148,13 @@ protected RedisMeta normalRedisMeta() { RedisMeta redisMeta = new RedisMeta().setParent(shardMeta).setIp("localhost").setPort(randomPort()); return redisMeta; } - + + protected KeeperMeta normalKeeperMeta() { + DcMeta dcMeta = new DcMeta().setId("dc"); + ClusterMeta clusterMeta = new ClusterMeta().setId("cluster").setParent(dcMeta).setType(ClusterType.ONE_WAY.toString()); + ShardMeta shardMeta = new ShardMeta().setParent(clusterMeta).setId("shard"); + KeeperMeta keeperMeta = new KeeperMeta().setParent(shardMeta).setIp("127.0.0.1").setPort(randomPort()); + return keeperMeta; + } } \ No newline at end of file diff --git a/redis/redis-checker/src/test/java/com/ctrip/xpipe/redis/checker/healthcheck/impl/DefaultHealthCheckerMockTest.java b/redis/redis-checker/src/test/java/com/ctrip/xpipe/redis/checker/healthcheck/impl/DefaultHealthCheckerMockTest.java index 3e5f15931..9f5221a94 100644 --- a/redis/redis-checker/src/test/java/com/ctrip/xpipe/redis/checker/healthcheck/impl/DefaultHealthCheckerMockTest.java +++ b/redis/redis-checker/src/test/java/com/ctrip/xpipe/redis/checker/healthcheck/impl/DefaultHealthCheckerMockTest.java @@ -2,14 +2,12 @@ import com.ctrip.xpipe.endpoint.HostPort; import com.ctrip.xpipe.redis.checker.AbstractCheckerTest; +import com.ctrip.xpipe.redis.checker.CheckerConsoleService; import com.ctrip.xpipe.redis.checker.config.CheckerConfig; import com.ctrip.xpipe.redis.checker.healthcheck.HealthCheckInstanceManager; import com.ctrip.xpipe.redis.checker.healthcheck.meta.MetaChangeManager; -import com.ctrip.xpipe.redis.core.entity.ClusterMeta; -import com.ctrip.xpipe.redis.core.entity.DcMeta; -import com.ctrip.xpipe.redis.core.entity.RedisMeta; -import com.ctrip.xpipe.redis.core.entity.SourceMeta; -import com.ctrip.xpipe.redis.core.entity.XpipeMeta; +import com.ctrip.xpipe.redis.core.entity.*; +import com.ctrip.xpipe.redis.core.meta.CurrentDcAllMeta; import com.ctrip.xpipe.redis.core.meta.MetaCache; import com.google.common.collect.Sets; import org.junit.Assert; @@ -20,7 +18,9 @@ import org.mockito.Mock; import org.mockito.Mockito; import org.mockito.junit.MockitoJUnitRunner; +import org.xml.sax.SAXException; +import java.io.IOException; import java.util.Collections; import java.util.HashSet; import java.util.Set; @@ -42,11 +42,17 @@ public class DefaultHealthCheckerMockTest extends AbstractCheckerTest { @Mock private MetaChangeManager metaChangeManager; + @Mock + private CheckerConsoleService checkerConsoleService; + @Mock private CheckerConfig checkerConfig; + @Mock + private CurrentDcAllMeta currentDcAllMeta; + @Before - public void setupDefaultHealthCheckerMockTest() { + public void setupDefaultHealthCheckerMockTest() throws IOException, SAXException { when(checkerConfig.getIgnoredHealthCheckDc()).thenReturn(Collections.emptySet()); when(metaCache.getXpipeMeta()).thenReturn(getXpipeMeta()); } diff --git a/redis/redis-checker/src/test/java/com/ctrip/xpipe/redis/checker/healthcheck/meta/DefaultDcMetaChangeManagerTest.java b/redis/redis-checker/src/test/java/com/ctrip/xpipe/redis/checker/healthcheck/meta/DefaultDcMetaChangeManagerTest.java index 002601b9a..3a9e8d065 100644 --- a/redis/redis-checker/src/test/java/com/ctrip/xpipe/redis/checker/healthcheck/meta/DefaultDcMetaChangeManagerTest.java +++ b/redis/redis-checker/src/test/java/com/ctrip/xpipe/redis/checker/healthcheck/meta/DefaultDcMetaChangeManagerTest.java @@ -1,7 +1,9 @@ package com.ctrip.xpipe.redis.checker.healthcheck.meta; +import com.ctrip.xpipe.api.foundation.FoundationService; import com.ctrip.xpipe.cluster.ClusterType; import com.ctrip.xpipe.endpoint.HostPort; +import com.ctrip.xpipe.redis.checker.CheckerConsoleService; import com.ctrip.xpipe.redis.checker.config.CheckerConfig; import com.ctrip.xpipe.redis.checker.healthcheck.ClusterHealthCheckInstance; import com.ctrip.xpipe.redis.checker.healthcheck.HealthCheckInstanceManager; @@ -21,11 +23,10 @@ import org.mockito.Mock; import org.mockito.Mockito; import org.mockito.MockitoAnnotations; +import org.xml.sax.SAXException; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Map; -import java.util.Set; +import java.io.IOException; +import java.util.*; import static org.mockito.Matchers.any; import static org.mockito.Mockito.*; @@ -46,6 +47,9 @@ public class DefaultDcMetaChangeManagerTest extends AbstractRedisTest { @Mock private CheckerConfig checkerConfig; + @Mock + CheckerConsoleService checkerConsoleService; + private RedisHealthCheckInstance instance = null; private Set addedRedises = new HashSet<>(); @@ -53,7 +57,7 @@ public class DefaultDcMetaChangeManagerTest extends AbstractRedisTest { private Set deletedRedised = new HashSet<>(); @Before - public void beforeDefaultDcMetaChangeManagerTest() { + public void beforeDefaultDcMetaChangeManagerTest() throws IOException, SAXException { MockitoAnnotations.initMocks(this); Mockito.doAnswer(invocation -> { RedisMeta redis = invocation.getArgument(0, RedisMeta.class); @@ -66,12 +70,15 @@ public void beforeDefaultDcMetaChangeManagerTest() { deletedRedised.add(redis); return null; }).when(instanceManager).remove(any(HostPort.class)); + + when(checkerConfig.getConsoleAddress()).thenReturn("127.0.0.1"); + when(checkerConsoleService.getXpipeDcAllMeta(Mockito.anyString(), Mockito.anyString())).thenReturn(getXpipeMeta()); - manager = new DefaultDcMetaChangeManager("oy", instanceManager, factory); + manager = new DefaultDcMetaChangeManager("oy", instanceManager, factory, checkerConsoleService, checkerConfig); } private void prepareData(String dc) { - manager.compare(getDcMeta(dc)); + manager.compare(getDcMeta(dc), getDcMeta(dc)); } private DcMeta cloneDcMeta(String dc) { @@ -110,7 +117,7 @@ public void testMasterChange() throws Exception { prepareData("oy"); DcMeta future = cloneDcMeta("oy"); future.findCluster("cluster1").getShards().values().iterator().next().getRedises().get(0).setMaster(""); - manager.compare(future); + manager.compare(future, null); // only changed redis reload Mockito.verify(instanceManager, times(1)).remove(any(HostPort.class)); @@ -122,7 +129,7 @@ public void testShardConfigChange() throws Exception { prepareData("oy"); DcMeta future = cloneDcMeta("oy"); future.findCluster("cluster1").getShards().values().iterator().next().setSentinelId(100L); - manager.compare(future); + manager.compare(future, null); // only redis in changed shard reload Mockito.verify(instanceManager, times(2)).remove(any(HostPort.class)); @@ -153,7 +160,7 @@ public void testActiveDcOY2JQ() { prepareData("oy"); DcMeta future = cloneDcMeta("oy"); future.findCluster("cluster2").setActiveDc("jq").setBackupDcs("oy"); - manager.compare(future); + manager.compare(future, null); Mockito.verify(instanceManager, times(2)).getOrCreate(any(RedisMeta.class)); Mockito.verify(instanceManager, never()).remove(any(HostPort.class)); @@ -165,7 +172,7 @@ public void testActiveDcJQ2OY() { prepareData("oy"); DcMeta future = cloneDcMeta("oy"); future.findCluster("cluster1").setActiveDc("oy").setBackupDcs("jq"); - manager.compare(future); + manager.compare(future, null); Mockito.verify(instanceManager, never()).getOrCreate(any(RedisMeta.class)); Mockito.verify(instanceManager, times(4)).remove(any(HostPort.class)); @@ -199,7 +206,7 @@ public void testDcsAddCurrentDc() { prepareData("oy"); DcMeta future = cloneDcMeta("oy"); future.findCluster("cluster4").setDcs("jq,oy"); - manager.compare(future); + manager.compare(future, null); Mockito.verify(instanceManager).getOrCreate(any(RedisMeta.class)); Mockito.verify(instanceManager).remove(any(HostPort.class)); // delete anyway @@ -211,7 +218,7 @@ public void testDcsDeleteCurrentDc() { prepareData("oy"); DcMeta future = cloneDcMeta("oy"); future.findCluster("cluster3").setDcs("oy"); - manager.compare(future); + manager.compare(future, null); Mockito.verify(instanceManager, never()).getOrCreate(any(RedisMeta.class)); Mockito.verify(instanceManager, times(1)).remove(any(HostPort.class)); @@ -223,7 +230,7 @@ public void testClusterOrgChange() { prepareData("oy"); DcMeta future = cloneDcMeta("oy"); future.findCluster("cluster3").setOrgId(2); - manager.compare(future); + manager.compare(future, null); Mockito.verify(instanceManager).remove(any(HostPort.class)); Mockito.verify(instanceManager).getOrCreate(any(RedisMeta.class)); @@ -232,7 +239,7 @@ public void testClusterOrgChange() { @Test public void visitRemoved() { manager = spy(manager); - manager.compare(getDcMeta("oy")); + manager.compare(getDcMeta("oy"), null); verify(manager, never()).visitModified(any()); verify(manager, never()).visitAdded(any()); verify(manager, never()).visitRemoved(any()); @@ -247,7 +254,7 @@ public void visitRemoved() { } }); dcMeta.addCluster(clusterMeta); - manager.compare(dcMeta); + manager.compare(dcMeta, null); verify(manager, atLeastOnce()).visitRemoved(any()); verify(manager, atLeastOnce()).visitAdded(any()); verify(manager, never()).visitModified(any()); @@ -255,14 +262,14 @@ public void visitRemoved() { } @Test - public void visitRemovedClusterActiveDc() { - manager = spy(new DefaultDcMetaChangeManager("jq", instanceManager, factory)); - manager.compare(getDcMeta("jq")); + public void visitRemovedClusterActiveDc(){ + manager = spy(new DefaultDcMetaChangeManager("jq", instanceManager, factory, checkerConsoleService, checkerConfig)); + manager.compare(getDcMeta("jq"), null); DcMeta dcMeta = MetaClone.clone(getDcMeta("jq")); dcMeta.getClusters().remove("cluster1"); - manager.compare(dcMeta); + manager.compare(dcMeta, null); verify(manager, atLeastOnce()).visitRemoved(any()); verify(manager, never()).visitAdded(any()); verify(manager, never()).visitModified(any()); @@ -283,7 +290,7 @@ public void testRouteChange() { Mockito.verify(factory, times(1)).updateRoutes(); DcMeta future = cloneDcMeta("oy"); future.getRoutes().get(0).setIsPublic(false); - manager.compare(future); + manager.compare(future, null); Mockito.verify(factory, times(2)).updateRoutes(); } @@ -297,7 +304,7 @@ public void testSwitchClusterName() throws Exception { future.addCluster(cluster1.setId("cluster2")); future.addCluster(cluster2.setId("cluster1")); - manager.compare(future); + manager.compare(future, null); Mockito.verify(instanceManager, times(1)).getOrCreate(any(ClusterMeta.class)); Mockito.verify(instanceManager, times(4)).getOrCreate(any(RedisMeta.class)); @@ -331,7 +338,7 @@ public void testSwitchClusterShards() throws Exception { ClusterMeta cluster2 = dcMeta.findCluster("cluster2"); changeClusterShardId(cluster2); - manager.compare(dcMeta); + manager.compare(dcMeta, null); DcMeta future = cloneDcMeta(dcMeta); ClusterMeta cluster1Future = future.findCluster("cluster1"); @@ -342,7 +349,7 @@ public void testSwitchClusterShards() throws Exception { changeClusterShards(cluster1Future,cluster2ShardsCopy); changeClusterShards(cluster2Future,cluster1ShardsCopy); - manager.compare(future); + manager.compare(future, null); Mockito.verify(instanceManager, never()).getOrCreate(any(ClusterMeta.class)); Mockito.verify(instanceManager, times(2)).getOrCreate(any(RedisMeta.class)); @@ -352,12 +359,12 @@ public void testSwitchClusterShards() throws Exception { @Test public void testHeteroClusterModified() throws Exception { - DefaultDcMetaChangeManager manager = new DefaultDcMetaChangeManager("jq", instanceManager, factory); + DefaultDcMetaChangeManager manager = new DefaultDcMetaChangeManager("jq", instanceManager, factory, checkerConsoleService, checkerConfig); DcMeta dcMeta= getDcMeta("jq"); ClusterMeta cluster1 = dcMeta.findCluster("cluster2"); cluster1.setBackupDcs("ali"); cluster1.setAzGroupType(ClusterType.ONE_WAY.toString()); - manager.compare(dcMeta); + manager.compare(dcMeta, null); DcMeta future = cloneDcMeta(dcMeta); @@ -365,7 +372,7 @@ public void testHeteroClusterModified() throws Exception { futureCluster.setBackupDcs("ali"); futureCluster.setAzGroupType(ClusterType.SINGLE_DC.toString()); - manager.compare(future); + manager.compare(future, null); Mockito.verify(instanceManager, times(1)).getOrCreate(any(ClusterMeta.class)); Mockito.verify(instanceManager, times(2)).getOrCreate(any(RedisMeta.class)); @@ -377,18 +384,18 @@ public void testHeteroClusterModified() throws Exception { @Test public void testBackupDcClusterModified() throws Exception { - DefaultDcMetaChangeManager manager = new DefaultDcMetaChangeManager("jq", instanceManager, factory); + DefaultDcMetaChangeManager manager = new DefaultDcMetaChangeManager("jq", instanceManager, factory, checkerConsoleService, checkerConfig); DcMeta dcMeta= getDcMeta("jq"); ClusterMeta cluster1 = dcMeta.findCluster("cluster2"); cluster1.setBackupDcs("jq,ali"); - manager.compare(dcMeta); + manager.compare(dcMeta, null); DcMeta future = cloneDcMeta(dcMeta); ClusterMeta futureCluster = future.findCluster("cluster2"); futureCluster.setBackupDcs("jq"); - manager.compare(future); + manager.compare(future, null); Mockito.verify(instanceManager, never()).getOrCreate(any(ClusterMeta.class)); Mockito.verify(instanceManager, never()).getOrCreate(any(RedisMeta.class)); @@ -472,6 +479,62 @@ public void generateHeteroHealthCheckInstancesTest() throws Exception { Assert.assertFalse(manager.isInterestedInCluster(awsClusterMeta)); } + @Test + public void visitModified1() { + ClusterMeta clusterMeta = getDcMeta("oy").findCluster("cluster2"); + ClusterMeta clone = MetaClone.clone(clusterMeta); + clone.getShards().get("shard2").addRedis(new RedisMeta()); + manager.visitModified(new ClusterMetaComparator(clusterMeta, clone)); + verify(instanceManager, never()).getOrCreate(any(RedisMeta.class)); + } + + @Test + public void testKeeperChange() throws Exception { + String dcId = FoundationService.DEFAULT.getDataCenter(); + manager = new DefaultDcMetaChangeManager(dcId, instanceManager, factory, checkerConsoleService, checkerConfig); + prepareData(dcId); + DcMeta future = cloneDcMeta(dcId); + future.addKeeperContainer(new KeeperContainerMeta().setId(4L).setIp("1.1.1.4").setPort(8080)); + KeeperMeta keeperMeta = new KeeperMeta().setKeeperContainerId(4L).setIp("1.1.1.4").setPort(6389); + future.findCluster("cluster1").getShards().values().iterator().next().getKeepers().remove(0); + future.findCluster("cluster1").getShards().values().iterator().next().getKeepers().add(keeperMeta); + future.findCluster("cluster2").getShards().values().iterator().next().getKeepers().remove(1); + manager.compare(future, future); + + // only change keeper reload + Mockito.verify(instanceManager, times(2)).removeKeeper(any(HostPort.class)); + Mockito.verify(instanceManager, times(1)).getOrCreate(any(KeeperMeta.class)); + Mockito.verify(instanceManager, times(0)).removeRedisInstanceForAssignedAction(any(HostPort.class)); + Mockito.verify(instanceManager, times(0)).getOrCreateRedisInstanceForAssignedAction(any(RedisMeta.class)); + + } + + @Test + public void testRedisChange() throws Exception { + String dcId = FoundationService.DEFAULT.getDataCenter(); + manager = new DefaultDcMetaChangeManager(dcId, instanceManager, factory, checkerConsoleService, checkerConfig); + prepareData(dcId); + DcMeta future = cloneDcMeta(dcId); + future.addKeeperContainer(new KeeperContainerMeta().setId(4L).setIp("1.1.1.4").setPort(8080)); + KeeperMeta keeperMeta = new KeeperMeta().setKeeperContainerId(4L).setIp("1.1.1.4").setPort(6389); + ShardMeta shardMeta = future.findCluster("cluster1").getShards().get("shard1"); + List redises = shardMeta.getRedises(); + redises.clear(); + redises.add(new RedisMeta().setIp("1.1.1.1").setPort(6379).setParent(shardMeta).setMaster("127.0.0.1")); + redises.add(new RedisMeta().setIp("2.2.2.2").setPort(6379).setParent(shardMeta)); + when(checkerConsoleService.getXpipeDcAllMeta(Mockito.anyString(), Mockito.anyString())).thenReturn(new XpipeMeta().addDc(future)); + manager.compare(future, future); + + // only change redis changed + Mockito.verify(instanceManager, times(0)).removeKeeper(any(HostPort.class)); + Mockito.verify(instanceManager, times(0)).getOrCreate(any(KeeperMeta.class)); + Mockito.verify(instanceManager, times(1)).removeRedisInstanceForAssignedAction(any(HostPort.class)); + Mockito.verify(instanceManager, times(1)).getOrCreateRedisInstanceForAssignedAction(any(RedisMeta.class)); + + } + + + protected DcMeta getDcMeta(String dc) { Map dcMetaMap = getXpipeMeta().getDcs(); DcMeta dcMeta = dcMetaMap.get(dc); diff --git a/redis/redis-checker/src/test/java/com/ctrip/xpipe/redis/checker/healthcheck/session/DefaultKeeperSessionManagerTest.java b/redis/redis-checker/src/test/java/com/ctrip/xpipe/redis/checker/healthcheck/session/DefaultKeeperSessionManagerTest.java new file mode 100644 index 000000000..a27fe1cff --- /dev/null +++ b/redis/redis-checker/src/test/java/com/ctrip/xpipe/redis/checker/healthcheck/session/DefaultKeeperSessionManagerTest.java @@ -0,0 +1,167 @@ +package com.ctrip.xpipe.redis.checker.healthcheck.session; + +import com.ctrip.xpipe.cluster.ClusterType; +import com.ctrip.xpipe.endpoint.HostPort; +import com.ctrip.xpipe.redis.checker.CheckerConsoleService; +import com.ctrip.xpipe.redis.checker.config.CheckerConfig; +import com.ctrip.xpipe.redis.core.entity.*; +import com.ctrip.xpipe.redis.core.meta.CurrentDcAllMeta; +import com.ctrip.xpipe.redis.core.meta.MetaCache; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.junit.MockitoJUnitRunner; +import org.xml.sax.SAXException; + +import java.io.IOException; +import java.util.Arrays; +import java.util.List; +import java.util.Set; + +/** + * @author yu + *

+ * 2023/10/31 + */ + +@RunWith(MockitoJUnitRunner.class) +public class DefaultKeeperSessionManagerTest { + + @InjectMocks + private DefaultKeeperSessionManager sessionManager; + + @Mock + private CheckerConfig checkerConfig; + + @Mock + private CheckerConsoleService checkerConsoleService; + + @Mock + private MetaCache metaCache; + + @Mock + private CurrentDcAllMeta currentDcAllMeta; + + + private List mockDcs = Arrays.asList("jq"); + + private List mockClusters = Arrays.asList("cluster1", "cluster2"); + + private List mockShards = Arrays.asList("shard1", "shard2"); + + private List keepercontainerIps = Arrays.asList("127.0.0.1", "127.0.0.2", "127.0.0.3", "127.0.0.4"); + private List redisIps = Arrays.asList("127.0.1.1", "127.0.1.2", "127.0.1.3", "127.0.1.4", "127.0.1.5", "127.0.1.6", "127.0.1.7", "127.0.1.8"); + + private List azIds = Arrays.asList("1", "2"); + + + + @Before + public void before() throws IOException, SAXException { + Mockito.when(metaCache.getXpipeMeta()).thenReturn(mockXpipeMeta()); + Mockito.when(currentDcAllMeta.getCurrentDcAllMeta()).thenReturn(MockAllCurrentDcMeta().getDcs().get("jq")); + } + + + @Test + public void getInUseRedises() { + Set useRedises = sessionManager.getInUseInstances(); + Assert.assertEquals(4, useRedises.size()); + } + + private XpipeMeta mockXpipeMeta() { + XpipeMeta meta = new XpipeMeta(); + + for (String dc: mockDcs) { + meta.addDc(mockDcMeta(dc)); + } + + return meta; + } + + private DcMeta mockDcMeta(String dc) { + DcMeta dcMeta = new DcMeta(); + dcMeta.setId(dc); + + dcMeta.addAz(new AzMeta(azIds.get(0))); + dcMeta.addAz(new AzMeta(azIds.get(1))); + + dcMeta.addKeeperContainer(mockKeepercontainerMeta(1, keepercontainerIps.get(0), Long.valueOf(azIds.get(0)))); + dcMeta.addKeeperContainer(mockKeepercontainerMeta(3, keepercontainerIps.get(2), Long.valueOf(azIds.get(1)))); + + dcMeta.addCluster(mockClusterMeta(mockClusters.get(0))); + + return dcMeta; + } + + private XpipeMeta MockAllCurrentDcMeta() { + XpipeMeta meta = new XpipeMeta(); + String dc = mockDcs.get(0); + DcMeta dcMeta = new DcMeta(); + dcMeta.setId(dc); + + dcMeta.addAz(new AzMeta(azIds.get(0))); + dcMeta.addAz(new AzMeta(azIds.get(1))); + + dcMeta.addKeeperContainer(mockKeepercontainerMeta(1, keepercontainerIps.get(0), Long.valueOf(azIds.get(0)))); + dcMeta.addKeeperContainer(mockKeepercontainerMeta(2, keepercontainerIps.get(1), Long.valueOf(azIds.get(0)))); + dcMeta.addKeeperContainer(mockKeepercontainerMeta(3, keepercontainerIps.get(2), Long.valueOf(azIds.get(1)))); + dcMeta.addKeeperContainer(mockKeepercontainerMeta(4, keepercontainerIps.get(3), Long.valueOf(azIds.get(1)))); + + dcMeta.addCluster(mockClusterMeta(mockClusters.get(0))); + dcMeta.addCluster(mockClusterMeta(mockClusters.get(1))); + + meta.addDc(dcMeta); + + return meta; + } + + private KeeperContainerMeta mockKeepercontainerMeta(long id, String ip, Long azId) { + KeeperContainerMeta keeperContainerMeta = new KeeperContainerMeta(); + keeperContainerMeta.setId(id); + keeperContainerMeta.setAzId(azId); + keeperContainerMeta.setIp(ip); + return keeperContainerMeta; + } + + private ClusterMeta mockClusterMeta(String cluster) { + ClusterMeta clusterMeta = new ClusterMeta(); + clusterMeta.setId(cluster); + clusterMeta.setType(ClusterType.ONE_WAY.toString()); + if (cluster.equalsIgnoreCase(mockClusters.get(0))) { + ShardMeta shard1 = new ShardMeta("shard1"); + shard1.addKeeper(new KeeperMeta().setIp(keepercontainerIps.get(0)).setPort(6380).setKeeperContainerId(1L)); + shard1.addKeeper(new KeeperMeta().setIp(keepercontainerIps.get(1)).setPort(6380).setKeeperContainerId(2L)); + shard1.addRedis(new RedisMeta().setIp(redisIps.get(0)).setPort(6379).setMaster("127.0.0.1")); + shard1.addRedis(new RedisMeta().setIp(redisIps.get(1)).setPort(6379)); + clusterMeta.addShard(shard1); + + ShardMeta shard2 = new ShardMeta("shard2"); + shard2.addKeeper(new KeeperMeta().setIp(keepercontainerIps.get(2)).setPort(6380).setKeeperContainerId(3L)); + shard2.addKeeper(new KeeperMeta().setIp(keepercontainerIps.get(3)).setPort(6380).setKeeperContainerId(4L)); + shard2.addRedis(new RedisMeta().setIp(redisIps.get(2)).setPort(6379)); + shard2.addRedis(new RedisMeta().setIp(redisIps.get(3)).setPort(6379).setMaster("127.0.0.1")); + clusterMeta.addShard(shard2); + } else { + ShardMeta shard3 = new ShardMeta("shard3"); + shard3.addKeeper(new KeeperMeta().setIp(keepercontainerIps.get(0)).setPort(6381).setKeeperContainerId(1L)); + shard3.addKeeper(new KeeperMeta().setIp(keepercontainerIps.get(1)).setPort(6381).setKeeperContainerId(2L)); + shard3.addRedis(new RedisMeta().setIp(redisIps.get(4)).setPort(6379).setMaster("127/.0.0.1")); + shard3.addRedis(new RedisMeta().setIp(redisIps.get(5)).setPort(6379)); + clusterMeta.addShard(shard3); + + ShardMeta shard4 = new ShardMeta("shard4"); + shard4.addKeeper(new KeeperMeta().setIp(keepercontainerIps.get(2)).setPort(6381).setKeeperContainerId(3L)); + shard4.addKeeper(new KeeperMeta().setIp(keepercontainerIps.get(3)).setPort(6381).setKeeperContainerId(4L)); + shard4.addRedis(new RedisMeta().setIp(redisIps.get(6)).setPort(6379).setMaster("127.0.0.1")); + shard4.addRedis(new RedisMeta().setIp(redisIps.get(7)).setPort(6379)); + clusterMeta.addShard(shard4); + } + + return clusterMeta; + } +} \ No newline at end of file diff --git a/redis/redis-checker/src/test/java/com/ctrip/xpipe/redis/checker/healthcheck/session/DefaultRedisSessionManagerTest.java b/redis/redis-checker/src/test/java/com/ctrip/xpipe/redis/checker/healthcheck/session/DefaultRedisSessionManagerTest.java new file mode 100644 index 000000000..939e4c08e --- /dev/null +++ b/redis/redis-checker/src/test/java/com/ctrip/xpipe/redis/checker/healthcheck/session/DefaultRedisSessionManagerTest.java @@ -0,0 +1,169 @@ +package com.ctrip.xpipe.redis.checker.healthcheck.session; + +import com.ctrip.xpipe.cluster.ClusterType; +import com.ctrip.xpipe.endpoint.HostPort; +import com.ctrip.xpipe.redis.checker.CheckerConsoleService; +import com.ctrip.xpipe.redis.checker.config.CheckerConfig; +import com.ctrip.xpipe.redis.core.entity.*; +import com.ctrip.xpipe.redis.core.meta.CurrentDcAllMeta; +import com.ctrip.xpipe.redis.core.meta.MetaCache; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.runners.MockitoJUnitRunner; +import org.xml.sax.SAXException; + +import java.io.IOException; +import java.util.Arrays; +import java.util.List; +import java.util.Set; + +/** + * @author yu + *

+ * 2023/9/28 + */ + +@RunWith(MockitoJUnitRunner.class) +public class DefaultRedisSessionManagerTest { + + @InjectMocks + private DefaultRedisSessionManager sessionManager; + + @Mock + private CheckerConfig checkerConfig; + + @Mock + private CheckerConsoleService checkerConsoleService; + + @Mock + private MetaCache metaCache; + + @Mock + private CurrentDcAllMeta currentDcAllMeta; + + + private List mockDcs = Arrays.asList("jq"); + + private List mockClusters = Arrays.asList("cluster1", "cluster2"); + + private List mockShards = Arrays.asList("shard1", "shard2"); + + private List keepercontainerIps = Arrays.asList("127.0.0.1", "127.0.0.2", "127.0.0.3", "127.0.0.4"); + private List redisIps = Arrays.asList("127.0.1.1", "127.0.1.2", "127.0.1.3", "127.0.1.4", "127.0.1.5", "127.0.1.6", "127.0.1.7", "127.0.1.8"); + + private List azIds = Arrays.asList("1", "2"); + + + + @Before + public void before() throws IOException, SAXException { + Mockito.when(metaCache.getXpipeMeta()).thenReturn(mockXpipeMeta()); + Mockito.when(currentDcAllMeta.getCurrentDcAllMeta()).thenReturn(MockAllCurrentDcMeta().getDcs().get("jq")); + } + + + @Test + public void getInUseRedises() { + Set useRedises = sessionManager.getInUseInstances(); + Assert.assertEquals(6, useRedises.size()); + } + + + + private XpipeMeta mockXpipeMeta() { + XpipeMeta meta = new XpipeMeta(); + + for (String dc: mockDcs) { + meta.addDc(mockDcMeta(dc)); + } + + return meta; + } + + private DcMeta mockDcMeta(String dc) { + DcMeta dcMeta = new DcMeta(); + dcMeta.setId(dc); + + dcMeta.addAz(new AzMeta(azIds.get(0))); + dcMeta.addAz(new AzMeta(azIds.get(1))); + + dcMeta.addKeeperContainer(mockKeepercontainerMeta(1, keepercontainerIps.get(0), Long.valueOf(azIds.get(0)))); + dcMeta.addKeeperContainer(mockKeepercontainerMeta(3, keepercontainerIps.get(2), Long.valueOf(azIds.get(1)))); + + dcMeta.addCluster(mockClusterMeta(mockClusters.get(0))); + + return dcMeta; + } + + private XpipeMeta MockAllCurrentDcMeta() { + XpipeMeta meta = new XpipeMeta(); + String dc = mockDcs.get(0); + DcMeta dcMeta = new DcMeta(); + dcMeta.setId(dc); + + dcMeta.addAz(new AzMeta(azIds.get(0))); + dcMeta.addAz(new AzMeta(azIds.get(1))); + + dcMeta.addKeeperContainer(mockKeepercontainerMeta(1, keepercontainerIps.get(0), Long.valueOf(azIds.get(0)))); + dcMeta.addKeeperContainer(mockKeepercontainerMeta(2, keepercontainerIps.get(1), Long.valueOf(azIds.get(0)))); + dcMeta.addKeeperContainer(mockKeepercontainerMeta(3, keepercontainerIps.get(2), Long.valueOf(azIds.get(1)))); + dcMeta.addKeeperContainer(mockKeepercontainerMeta(4, keepercontainerIps.get(3), Long.valueOf(azIds.get(1)))); + + dcMeta.addCluster(mockClusterMeta(mockClusters.get(0))); + dcMeta.addCluster(mockClusterMeta(mockClusters.get(1))); + + meta.addDc(dcMeta); + + return meta; + } + + private KeeperContainerMeta mockKeepercontainerMeta(long id, String ip, Long azId) { + KeeperContainerMeta keeperContainerMeta = new KeeperContainerMeta(); + keeperContainerMeta.setId(id); + keeperContainerMeta.setAzId(azId); + keeperContainerMeta.setIp(ip); + return keeperContainerMeta; + } + + private ClusterMeta mockClusterMeta(String cluster) { + ClusterMeta clusterMeta = new ClusterMeta(); + clusterMeta.setId(cluster); + clusterMeta.setType(ClusterType.ONE_WAY.toString()); + if (cluster.equalsIgnoreCase(mockClusters.get(0))) { + ShardMeta shard1 = new ShardMeta("shard1"); + shard1.addKeeper(new KeeperMeta().setIp(keepercontainerIps.get(0)).setPort(6380).setKeeperContainerId(1L)); + shard1.addKeeper(new KeeperMeta().setIp(keepercontainerIps.get(1)).setPort(6380).setKeeperContainerId(2L)); + shard1.addRedis(new RedisMeta().setIp(redisIps.get(0)).setPort(6379).setMaster("127.0.0.1")); + shard1.addRedis(new RedisMeta().setIp(redisIps.get(1)).setPort(6379)); + clusterMeta.addShard(shard1); + + ShardMeta shard2 = new ShardMeta("shard2"); + shard2.addKeeper(new KeeperMeta().setIp(keepercontainerIps.get(2)).setPort(6380).setKeeperContainerId(3L)); + shard2.addKeeper(new KeeperMeta().setIp(keepercontainerIps.get(3)).setPort(6380).setKeeperContainerId(4L)); + shard2.addRedis(new RedisMeta().setIp(redisIps.get(2)).setPort(6379)); + shard2.addRedis(new RedisMeta().setIp(redisIps.get(3)).setPort(6379).setMaster("127.0.0.1")); + clusterMeta.addShard(shard2); + } else { + ShardMeta shard3 = new ShardMeta("shard3"); + shard3.addKeeper(new KeeperMeta().setIp(keepercontainerIps.get(0)).setPort(6381).setKeeperContainerId(1L)); + shard3.addKeeper(new KeeperMeta().setIp(keepercontainerIps.get(1)).setPort(6381).setKeeperContainerId(2L)); + shard3.addRedis(new RedisMeta().setIp(redisIps.get(4)).setPort(6379).setMaster("127/.0.0.1")); + shard3.addRedis(new RedisMeta().setIp(redisIps.get(5)).setPort(6379)); + clusterMeta.addShard(shard3); + + ShardMeta shard4 = new ShardMeta("shard4"); + shard4.addKeeper(new KeeperMeta().setIp(keepercontainerIps.get(2)).setPort(6381).setKeeperContainerId(3L)); + shard4.addKeeper(new KeeperMeta().setIp(keepercontainerIps.get(3)).setPort(6381).setKeeperContainerId(4L)); + shard4.addRedis(new RedisMeta().setIp(redisIps.get(6)).setPort(6379).setMaster("127.0.0.1")); + shard4.addRedis(new RedisMeta().setIp(redisIps.get(7)).setPort(6379)); + clusterMeta.addShard(shard4); + } + + return clusterMeta; + } +} \ No newline at end of file diff --git a/redis/redis-checker/src/test/java/com/ctrip/xpipe/redis/checker/impl/KeeperContainerInfoReporterTest.java b/redis/redis-checker/src/test/java/com/ctrip/xpipe/redis/checker/impl/KeeperContainerInfoReporterTest.java new file mode 100644 index 000000000..af984adab --- /dev/null +++ b/redis/redis-checker/src/test/java/com/ctrip/xpipe/redis/checker/impl/KeeperContainerInfoReporterTest.java @@ -0,0 +1,98 @@ +package com.ctrip.xpipe.redis.checker.impl; + +import com.ctrip.xpipe.redis.checker.CheckerConsoleService; +import com.ctrip.xpipe.redis.checker.config.CheckerConfig; +import com.ctrip.xpipe.redis.checker.healthcheck.actions.keeper.info.RedisUsedMemoryCollector; +import com.ctrip.xpipe.redis.checker.healthcheck.actions.keeper.infoStats.KeeperFlowCollector; +import com.ctrip.xpipe.redis.checker.model.DcClusterShard; +import com.ctrip.xpipe.redis.checker.model.KeeperContainerUsedInfoModel; +import com.google.common.collect.Maps; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.*; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentMap; + +/** + * @author yu + *

+ * 2023/10/18 + */ +@RunWith(org.mockito.junit.MockitoJUnitRunner.class) +public class KeeperContainerInfoReporterTest { + + @InjectMocks + KeeperContainerInfoReporter keeperContainerInfoReporter; + + @Mock + RedisUsedMemoryCollector redisUsedMemoryCollector; + + @Mock + KeeperFlowCollector keeperFlowCollector; + + @Mock + CheckerConsoleService checkerConsoleService; + + @Mock + CheckerConfig config; + + @Captor + ArgumentCaptor> resultCaptor; + + @Before + public void befor() { + DcClusterShard dcClusterShard1 = new DcClusterShard("jq", "cluster1", "shard1"); + DcClusterShard dcClusterShard2 = new DcClusterShard("jq", "cluster1", "shard2"); + DcClusterShard dcClusterShard3 = new DcClusterShard("jq", "cluster2", "shard1"); + DcClusterShard dcClusterShard4 = new DcClusterShard("jq", "cluster2", "shard2"); + DcClusterShard dcClusterShard5 = new DcClusterShard("jq", "cluster3", "shard1"); + DcClusterShard dcClusterShard6 = new DcClusterShard("jq", "cluster3", "shard2"); + DcClusterShard dcClusterShard7 = new DcClusterShard("jq", "cluster3", "shard3"); + + ConcurrentMap> keeperFlowMap = Maps.newConcurrentMap(); + Map map1 = new HashMap<>(); + map1.put(dcClusterShard1, 2L); + map1.put(dcClusterShard4, 2L); + map1.put(dcClusterShard5, 2L); + keeperFlowMap.put("127.0.0.1", map1); + + Map map2 = new HashMap<>(); + map2.put(dcClusterShard2, 2L); + map2.put(dcClusterShard6, 2L); + keeperFlowMap.put("127.0.0.2", map2); + + Map map3 = new HashMap<>(); + map3.put(dcClusterShard3, 2L); + map3.put(dcClusterShard7, 2L); + keeperFlowMap.put("127.0.0.3", map3); + + Mockito.when(keeperFlowCollector.getHostPort2InputFlow()).thenReturn(keeperFlowMap); + + ConcurrentMap redisUsedMemoryMap = Maps.newConcurrentMap(); + redisUsedMemoryMap.put(dcClusterShard1, 1L); + redisUsedMemoryMap.put(dcClusterShard2, 2L); + redisUsedMemoryMap.put(dcClusterShard3, 3L); + redisUsedMemoryMap.put(dcClusterShard4, 4L); + redisUsedMemoryMap.put(dcClusterShard6, 6L); + redisUsedMemoryMap.put(dcClusterShard7, 7L); + + Mockito.when(redisUsedMemoryCollector.getDcClusterShardUsedMemory()).thenReturn(redisUsedMemoryMap); + Mockito.when(config.getConsoleAddress()).thenReturn("127.0.0.1"); + Mockito.when(config.getClustersPartIndex()).thenReturn(0); + + } + + @Test + public void testReportKeeperContainerInfo() { + keeperContainerInfoReporter.reportKeeperContainerInfo(); + Mockito.verify(checkerConsoleService, Mockito.times(1)) + .reportKeeperContainerInfo(Mockito.anyString(), resultCaptor.capture(), Mockito.anyInt()); + + Assert.assertEquals(3, resultCaptor.getValue().size()); + } +} diff --git a/redis/redis-checker/src/test/resources/dc-meta-test.xml b/redis/redis-checker/src/test/resources/dc-meta-test.xml index 58bd14168..6a46481ff 100644 --- a/redis/redis-checker/src/test/resources/dc-meta-test.xml +++ b/redis/redis-checker/src/test/resources/dc-meta-test.xml @@ -11,22 +11,22 @@ - - + + - - + + - - + + @@ -36,28 +36,34 @@ + + + + + + - - + + - - + + - - + + @@ -72,6 +78,11 @@ + + + + + diff --git a/redis/redis-console/src/main/java/com/ctrip/xpipe/redis/console/config/ConsoleConfig.java b/redis/redis-console/src/main/java/com/ctrip/xpipe/redis/console/config/ConsoleConfig.java index f2445ebad..796da30d8 100644 --- a/redis/redis-console/src/main/java/com/ctrip/xpipe/redis/console/config/ConsoleConfig.java +++ b/redis/redis-console/src/main/java/com/ctrip/xpipe/redis/console/config/ConsoleConfig.java @@ -4,6 +4,7 @@ import com.ctrip.xpipe.redis.checker.config.CheckerConfig; import com.ctrip.xpipe.redis.checker.healthcheck.actions.interaction.DcClusterDelayMarkDown; import com.ctrip.xpipe.redis.console.config.model.BeaconOrgRoute; +import com.ctrip.xpipe.redis.console.model.KeeperContainerOverloadStandardModel; import com.ctrip.xpipe.redis.console.util.HickwallMetricInfo; import com.ctrip.xpipe.redis.core.config.CoreConfig; import com.ctrip.xpipe.redis.core.meta.QuorumConfig; @@ -142,7 +143,7 @@ public interface ConsoleConfig extends CoreConfig, CheckerConfig, AlertConfig { int getCheckerAckTimeoutMilli(); - long getMigrationTimeoutMilli(); + long getMigrationTimeoutMilli(); long getServletMethodTimeoutMilli(); @@ -177,4 +178,10 @@ public interface ConsoleConfig extends CoreConfig, CheckerConfig, AlertConfig { long getMigrationProcessReportIntervalMill(); long getMigrationResultReportIntervalMill(); + + boolean isAutoMigrateOverloadKeeperContainerOpen(); + + long getAutoMigrateOverloadKeeperContainerIntervalMilli(); + + Map getKeeperContainerOverloadStandards(); } diff --git a/redis/redis-console/src/main/java/com/ctrip/xpipe/redis/console/config/impl/DefaultConsoleConfig.java b/redis/redis-console/src/main/java/com/ctrip/xpipe/redis/console/config/impl/DefaultConsoleConfig.java index 38b74bc77..d8990a1e5 100644 --- a/redis/redis-console/src/main/java/com/ctrip/xpipe/redis/console/config/impl/DefaultConsoleConfig.java +++ b/redis/redis-console/src/main/java/com/ctrip/xpipe/redis/console/config/impl/DefaultConsoleConfig.java @@ -7,6 +7,7 @@ import com.ctrip.xpipe.redis.checker.healthcheck.actions.interaction.DcClusterDelayMarkDown; import com.ctrip.xpipe.redis.console.config.ConsoleConfig; import com.ctrip.xpipe.redis.console.config.model.BeaconOrgRoute; +import com.ctrip.xpipe.redis.console.model.KeeperContainerOverloadStandardModel; import com.ctrip.xpipe.redis.console.util.HickwallMetricInfo; import com.ctrip.xpipe.redis.core.config.AbstractCoreConfig; import com.ctrip.xpipe.redis.core.meta.QuorumConfig; @@ -122,6 +123,11 @@ public class DefaultConsoleConfig extends AbstractCoreConfig implements ConsoleC private static final String KEY_MIGRATION_RESULT_REPORT_OPEN = "migration.result.report.open"; private static final String KEY_MIGRATION_RESULT_REPORT_INTERVAL_MILLI = "migration.result.report.interval.milli"; + private static final String KEY_CONSOLE_KEEPER_CONTAINER_OVERLOAD_STANDARD = "console.keepercontainer.overlaod.standard"; + + private static final String KEY_CONSOLE_AUTO_MIGRATE_OVERLOAD_KEEPER_CONTAINER_OPEN = "console.auto.migrate.overload.keeper.container.open"; + private static final String KEY_CONSOLE_AUTO_MIGRATE_OVERLOAD_KEEPER_CONTAINER_INTERVAL_MILLI = "console.auto.migrate.overload.keeper.container.interval.milli"; + private String defaultRouteChooseStrategyType = RouteChooseStrategyFactory.RouteStrategyType.CRC32_HASH.name(); private Map> listeners = Maps.newConcurrentMap(); @@ -581,6 +587,11 @@ public int getCheckerReportIntervalMilli() { return getIntProperty(KEY_CHECKER_REPORT_INTERVAL, 10000); } + @Override + public int getCheckerCurrentDcAllMetaRefreshIntervalMilli() { + return getIntProperty(KEY_CHECKER_CURRENT_DC_ALL_META_REFRESH_INTERVAL, 600000); + } + @Override public int getCheckerMetaRefreshIntervalMilli() { return getIntProperty(KEY_CHECKER_META_REFRESH_INTERVAL, 30000); @@ -706,6 +717,11 @@ public Set getMigrationUnsupportedClusters() { return getSplitStringSet(raw); } + @Override + public int getKeeperCheckerIntervalMilli() { + return getIntProperty(KEY_KEEPER_CHECKER_INTERVAL, 1800 * 1000); + } + @Override public int monitorUnregisterProtectCount() { return getIntProperty(KEY_MONITOR_UNREGISTER_PROTECT_COUNT, 10); @@ -754,4 +770,20 @@ public long getMigrationResultReportIntervalMill() { return getLongProperty(KEY_MIGRATION_RESULT_REPORT_INTERVAL_MILLI, 10000L); } + @Override + public boolean isAutoMigrateOverloadKeeperContainerOpen() { + return getBooleanProperty(KEY_CONSOLE_AUTO_MIGRATE_OVERLOAD_KEEPER_CONTAINER_OPEN, false); + } + + @Override + public long getAutoMigrateOverloadKeeperContainerIntervalMilli() { + return getLongProperty(KEY_CONSOLE_AUTO_MIGRATE_OVERLOAD_KEEPER_CONTAINER_INTERVAL_MILLI, 60 * 60 * 1000L); + } + + @Override + public Map getKeeperContainerOverloadStandards() { + String property = getProperty(KEY_CONSOLE_KEEPER_CONTAINER_OVERLOAD_STANDARD, "{}"); + return JsonCodec.INSTANCE.decode(property, new GenericTypeReference>() {}); + } + } diff --git a/redis/redis-console/src/main/java/com/ctrip/xpipe/redis/console/controller/api/KeeperContainerOverloadController.java b/redis/redis-console/src/main/java/com/ctrip/xpipe/redis/console/controller/api/KeeperContainerOverloadController.java new file mode 100644 index 000000000..2d36e995c --- /dev/null +++ b/redis/redis-console/src/main/java/com/ctrip/xpipe/redis/console/controller/api/KeeperContainerOverloadController.java @@ -0,0 +1,34 @@ +package com.ctrip.xpipe.redis.console.controller.api; + + +import com.ctrip.xpipe.redis.checker.model.KeeperContainerUsedInfoModel; +import com.ctrip.xpipe.redis.console.controller.AbstractConsoleController; +import com.ctrip.xpipe.redis.console.keeper.KeeperContainerUsedInfoAnalyzer; +import com.ctrip.xpipe.redis.console.model.MigrationKeeperContainerDetailModel; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestMethod; +import org.springframework.web.bind.annotation.RestController; + +import java.util.ArrayList; +import java.util.List; + +@RestController +@RequestMapping(AbstractConsoleController.API_PREFIX) +public class KeeperContainerOverloadController extends AbstractConsoleController{ + + @Autowired + KeeperContainerUsedInfoAnalyzer analyzer; + + @RequestMapping(value = "/keepercontainer/overload/info/all", method = RequestMethod.GET) + public List getAllReadyToMigrateKeeperContainers() { + return analyzer.getAllDcReadyToMigrationKeeperContainers(); + } + + @RequestMapping(value = "/keepercontainer/overload/info/current", method = RequestMethod.GET) + public List getCurrentReadyToMigrateKeeperContainers() { + List result = new ArrayList<>(); + analyzer.getAllKeeperContainerUsedInfoModels().values().forEach(result::addAll); + return result; + } +} diff --git a/redis/redis-console/src/main/java/com/ctrip/xpipe/redis/console/controller/api/checker/ConsoleCheckerController.java b/redis/redis-console/src/main/java/com/ctrip/xpipe/redis/console/controller/api/checker/ConsoleCheckerController.java index 287be2c43..180133fd6 100644 --- a/redis/redis-console/src/main/java/com/ctrip/xpipe/redis/console/controller/api/checker/ConsoleCheckerController.java +++ b/redis/redis-console/src/main/java/com/ctrip/xpipe/redis/console/controller/api/checker/ConsoleCheckerController.java @@ -13,16 +13,19 @@ import com.ctrip.xpipe.redis.checker.healthcheck.impl.DefaultRedisInstanceInfo; import com.ctrip.xpipe.redis.checker.model.CheckerStatus; import com.ctrip.xpipe.redis.checker.model.HealthCheckResult; +import com.ctrip.xpipe.redis.checker.model.KeeperContainerUsedInfoModel; import com.ctrip.xpipe.redis.checker.model.ProxyTunnelInfo; import com.ctrip.xpipe.redis.checker.spring.ConsoleServerMode; import com.ctrip.xpipe.redis.checker.spring.ConsoleServerModeCondition; import com.ctrip.xpipe.redis.console.checker.CheckerManager; import com.ctrip.xpipe.redis.console.controller.AbstractConsoleController; import com.ctrip.xpipe.redis.console.healthcheck.nonredis.cluster.ClusterHealthMonitorManager; +import com.ctrip.xpipe.redis.console.keeper.KeeperContainerUsedInfoAnalyzer; import com.ctrip.xpipe.redis.console.service.CrossMasterDelayService; import com.ctrip.xpipe.redis.console.service.DelayService; import com.ctrip.xpipe.redis.console.service.impl.AlertEventService; import com.ctrip.xpipe.redis.core.console.ConsoleCheckerPath; +import com.ctrip.xpipe.redis.core.entity.DcMeta; import com.ctrip.xpipe.redis.core.entity.XpipeMeta; import com.ctrip.xpipe.redis.core.meta.MetaCache; import com.fasterxml.jackson.databind.ObjectMapper; @@ -73,6 +76,9 @@ public class ConsoleCheckerController extends AbstractConsoleController { @Autowired private OuterClientCache outerClientCache; + @Autowired + private KeeperContainerUsedInfoAnalyzer keeperContainerUsedInfoAnalyzer; + private Logger logger = LoggerFactory.getLogger(ConsoleCheckerController.class); @GetMapping(ConsoleCheckerPath.PATH_GET_META) @@ -89,6 +95,13 @@ public String getDividedMeta(@RequestParam(value="format", required = false) Str return (format != null && format.equals("xml"))? xpipeMeta.toString() : coder.encode(xpipeMeta); } + @GetMapping(ConsoleCheckerPath.PATH_GET_DC_ALL_META) + public String getDcAllMeta(@PathVariable String dcName, @RequestParam(value="format", required = false) String format) { + DcMeta dcMeta = metaCache.getXpipeMeta().getDcs().get(dcName); + XpipeMeta xpipeMeta = new XpipeMeta().addDc(dcMeta); + return (format != null && format.equals("xml"))? xpipeMeta.toString() : coder.encode(xpipeMeta); + } + @GetMapping(ConsoleCheckerPath.PATH_GET_PROXY_CHAINS) public List getProxyTunnels() { return proxyManager.getAllProxyTunnels(); @@ -113,6 +126,13 @@ public void reportHealthCheckResult(HttpServletRequest request, @RequestBody Hea if (null != checkResult.getHeteroShardsDelay()) delayService.updateHeteroShardsDelays(checkResult.getHeteroShardsDelay()); } + @PostMapping(ConsoleCheckerPath.PATH_POST_KEEPER_CONTAINER_INFO_RESULT) + public void updateKeeperContainerUsedInfo(HttpServletRequest request, @PathVariable int index, @RequestBody List keeperContainerUsedInfoModels) { + logger.debug("[updateKeeperContainerUsedInfo][{}] {}", request.getRemoteAddr(), keeperContainerUsedInfoModels); + keeperContainerUsedInfoAnalyzer.updateKeeperContainerUsedInfo(index, keeperContainerUsedInfoModels); + } + + @Resource PersistenceCache persistenceCache; diff --git a/redis/redis-console/src/main/java/com/ctrip/xpipe/redis/console/controller/api/metaserver/ConsoleController.java b/redis/redis-console/src/main/java/com/ctrip/xpipe/redis/console/controller/api/metaserver/ConsoleController.java index 655140d3d..aa8633c68 100644 --- a/redis/redis-console/src/main/java/com/ctrip/xpipe/redis/console/controller/api/metaserver/ConsoleController.java +++ b/redis/redis-console/src/main/java/com/ctrip/xpipe/redis/console/controller/api/metaserver/ConsoleController.java @@ -79,7 +79,6 @@ public String getDcMeta(@PathVariable String dcId, @RequestParam(value="format", } }); toRemoveClusters.forEach(clusterName -> dcMeta.getClusters().remove(clusterName)); - return (format != null && format.equals("xml"))? dcMeta.toString() : coder.encode(dcMeta); } diff --git a/redis/redis-console/src/main/java/com/ctrip/xpipe/redis/console/controller/consoleportal/KeeperContainerInfoController.java b/redis/redis-console/src/main/java/com/ctrip/xpipe/redis/console/controller/consoleportal/KeeperContainerInfoController.java index 986049f1c..22f8a6362 100644 --- a/redis/redis-console/src/main/java/com/ctrip/xpipe/redis/console/controller/consoleportal/KeeperContainerInfoController.java +++ b/redis/redis-console/src/main/java/com/ctrip/xpipe/redis/console/controller/consoleportal/KeeperContainerInfoController.java @@ -2,7 +2,10 @@ import com.ctrip.xpipe.redis.checker.controller.result.RetMessage; import com.ctrip.xpipe.redis.console.controller.AbstractConsoleController; +import com.ctrip.xpipe.redis.console.keeper.KeeperContainerUsedInfoAnalyzer; import com.ctrip.xpipe.redis.console.model.KeeperContainerInfoModel; +import com.ctrip.xpipe.redis.console.model.MigrationKeeperContainerDetailModel; +import com.ctrip.xpipe.redis.console.service.KeeperContainerMigrationService; import com.ctrip.xpipe.redis.console.service.KeeperContainerService; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.web.bind.annotation.*; @@ -16,6 +19,12 @@ public class KeeperContainerInfoController extends AbstractConsoleController { @Autowired KeeperContainerService keeperContainerService; + @Autowired + KeeperContainerUsedInfoAnalyzer analyzer; + + @Autowired + KeeperContainerMigrationService keeperContainerMigrationService; + @RequestMapping(value = "/keepercontainer/infos/all", method = RequestMethod.GET) public List getAllKeeperContainerInfos() { return keeperContainerService.findAllInfos(); @@ -60,4 +69,27 @@ public List getAvailableKeeperContainersByDcAzAndOrg(@ @PathVariable(required = false) String azName, @PathVariable(required = false) String orgName) { return keeperContainerService.findAvailableKeeperContainerInfoModelsByDcAzAndOrg(dcName, azName, orgName); } + + @RequestMapping(value = "/keepercontainers/overload/all", method = RequestMethod.GET) + public List getAllOverloadKeeperContainers() { + return analyzer.getAllReadyToMigrationKeeperContainers(); + } + + + @RequestMapping(value = "/keepercontainer/overload/migration/process", method = RequestMethod.GET) + public List getOverloadKeeperContainerMigrationProcess() { + return keeperContainerMigrationService.getMigrationProcess(); + } + + @RequestMapping(value = "/keepercontainer/overload/migration/begin", method = RequestMethod.POST) + public RetMessage beginToMigrateOverloadKeeperContainers(@RequestBody List keeperContainerDetailModels) { + logger.info("begin to migrate over load keeper containers {}", keeperContainerDetailModels); + try { + keeperContainerMigrationService.beginMigrateKeeperContainers(keeperContainerDetailModels); + } catch (Throwable th) { + logger.warn("migrate over load keeper containers {} fail by {}", keeperContainerDetailModels, th.getMessage()); + return RetMessage.createFailMessage(th.getMessage()); + } + return RetMessage.createSuccessMessage(); + } } diff --git a/redis/redis-console/src/main/java/com/ctrip/xpipe/redis/console/controller/consoleportal/RedisController.java b/redis/redis-console/src/main/java/com/ctrip/xpipe/redis/console/controller/consoleportal/RedisController.java index 6bdd1fafe..189916e88 100644 --- a/redis/redis-console/src/main/java/com/ctrip/xpipe/redis/console/controller/consoleportal/RedisController.java +++ b/redis/redis-console/src/main/java/com/ctrip/xpipe/redis/console/controller/consoleportal/RedisController.java @@ -1,19 +1,16 @@ package com.ctrip.xpipe.redis.console.controller.consoleportal; -import com.ctrip.xpipe.redis.console.constant.XPipeConsoleConstant; import com.ctrip.xpipe.redis.console.controller.AbstractConsoleController; -import com.ctrip.xpipe.redis.console.model.*; +import com.ctrip.xpipe.redis.console.model.ClusterTbl; +import com.ctrip.xpipe.redis.console.model.MigrationKeeperModel; +import com.ctrip.xpipe.redis.console.model.ShardModel; import com.ctrip.xpipe.redis.console.service.*; import com.ctrip.xpipe.redis.console.service.model.ShardModelService; -import com.ctrip.xpipe.utils.ObjectUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.web.bind.annotation.*; -import java.util.ArrayList; import java.util.List; -import static com.ctrip.xpipe.redis.core.protocal.RedisProtocol.KEEPER_PORT_DEFAULT; - /** * @author zhangle 16/8/24 */ @@ -26,10 +23,6 @@ public class RedisController extends AbstractConsoleController { private ClusterService clusterService; @Autowired private ShardModelService shardModelService; - @Autowired - private KeeperAdvancedService keeperAdvancedService; - @Autowired - private KeeperContainerService keeperContainerService; @RequestMapping(value = "/clusters/" + CLUSTER_NAME_PATH_VARIABLE + "/dcs/{dcName}/shards/" + SHARD_NAME_PATH_VARIABLE, method = RequestMethod.POST) public void updateRedises(@PathVariable String clusterName, @PathVariable String dcName, @@ -66,7 +59,10 @@ public void migrateKeepers(@RequestBody MigrationKeeperModel model) { .getAllShardModel(model.getSrcKeeperContainer().getDcName(), clusterTbl.getClusterName()); for (ShardModel shardModel : allShardModel) { - if (!migrateShardKeepers(clusterTbl.getClusterName(), model, shardModel)) { + if (!shardModelService.migrateShardKeepers(model.getSrcKeeperContainer().getDcName(), + clusterTbl.getClusterName(), shardModel, model.getSrcKeeperContainer().getAddr().getHost(), + (model.getTargetKeeperContainer() == null || model.getTargetKeeperContainer().getAddr() == null) + ? null : model.getTargetKeeperContainer().getAddr().getHost())) { continue; } if (model.getMaxMigrationKeeperNum() != 0 && (++count) >= model.getMaxMigrationKeeperNum()) { @@ -76,83 +72,4 @@ public void migrateKeepers(@RequestBody MigrationKeeperModel model) { } } } - - private boolean migrateShardKeepers(String clusterName, MigrationKeeperModel model, ShardModel shardModel) { - List newKeepers = - getNewKeepers(clusterName, shardModel, model.getSrcKeeperContainer(), model.getTargetKeeperContainer()); - - if (newKeepers == null) { - logger.debug("[migrateKeepers] no need to replace keepers"); - return false; - }else if (newKeepers.size() == 2) { - return doMigrateKeepers(shardModel, newKeepers, clusterName, - model.getSrcKeeperContainer().getDcName(), shardModel.getShardTbl().getShardName()); - } else { - logger.info("[migrateKeepers] fail to migrate keepers with unexpected newKeepers {}", newKeepers); - return false; - } - } - - private List getNewKeepers(String clusterName, ShardModel shardModel, - KeeperContainerInfoModel srcKeeperContainer, KeeperContainerInfoModel targetKeeperContainer) { - List newKeepers = new ArrayList<>(); - for (RedisTbl keeper : shardModel.getKeepers()) { - if (!ObjectUtils.equals(keeper.getKeepercontainerId(), srcKeeperContainer.getId())) { - newKeepers.add(keeper); - } - } - - if (newKeepers.size() == 2) { - return null; - } - - if (newKeepers.size() < 1) { - logger.info("[migrateKeepers] unexpected keepers {} from cluster:{}, dc:{}, shard:{}", - newKeepers, clusterName, srcKeeperContainer.getDcName(), shardModel.getShardTbl().getShardName()); - return newKeepers; - } - - long alreadyUsedAzId = keeperContainerService.find(newKeepers.get(0).getKeepercontainerId()).getAzId(); - List bestKeepers = findBestKeepers(clusterName, srcKeeperContainer.getDcName(), targetKeeperContainer); - for (KeeperBasicInfo keeperSelected : bestKeepers) { - if (keeperSelected.getKeeperContainerId() != srcKeeperContainer.getId() - && keeperSelected.getKeeperContainerId() != newKeepers.get(0).getKeepercontainerId() - && isDifferentAz(keeperSelected, alreadyUsedAzId)) { - newKeepers.add(new RedisTbl().setKeepercontainerId(keeperSelected.getKeeperContainerId()) - .setRedisIp(keeperSelected.getHost()) - .setRedisPort(keeperSelected.getPort()) - .setRedisRole(XPipeConsoleConstant.ROLE_KEEPER)); - break; - } - } - return newKeepers; - } - - private boolean isDifferentAz(KeeperBasicInfo keeperSelected, long alreadyUsedAzId) { - if (alreadyUsedAzId == 0) return true; - long newAzId = keeperContainerService.find(keeperSelected.getKeeperContainerId()).getAzId(); - return newAzId != alreadyUsedAzId; - } - - private boolean doMigrateKeepers(ShardModel shardModel, List newKeepers, String clusterName, - String dcName, String shardName) { - try { - shardModel.setKeepers(newKeepers); - logger.info("[Update Redises][construct]{},{},{},{}", clusterName, dcName, shardName, shardModel); - redisService.updateRedises(dcName, clusterName, shardName, shardModel); - logger.info("[Update Redises][success]{},{},{},{}", clusterName, dcName, shardName, shardModel); - return true; - } catch (Exception e) { - logger.error("[Update Redises][failed]{},{},{},{}", clusterName, dcName, shardName, shardModel, e); - return false; - } - } - - private List findBestKeepers(String clusterName, String dcName, KeeperContainerInfoModel targetKeeperContainer) { - if (targetKeeperContainer == null || targetKeeperContainer.getId() == 0) { - return keeperAdvancedService.findBestKeepers(dcName, KEEPER_PORT_DEFAULT, (ip, port) -> true, clusterName); - } else { - return keeperAdvancedService.findBestKeepersByKeeperContainer(targetKeeperContainer, KEEPER_PORT_DEFAULT, (ip, port) -> true, 1); - } - } } diff --git a/redis/redis-console/src/main/java/com/ctrip/xpipe/redis/console/keeper/AutoMigrateOverloadKeeperContainerAction.java b/redis/redis-console/src/main/java/com/ctrip/xpipe/redis/console/keeper/AutoMigrateOverloadKeeperContainerAction.java new file mode 100644 index 000000000..9c4a72bb4 --- /dev/null +++ b/redis/redis-console/src/main/java/com/ctrip/xpipe/redis/console/keeper/AutoMigrateOverloadKeeperContainerAction.java @@ -0,0 +1,127 @@ +package com.ctrip.xpipe.redis.console.keeper; + +import com.ctrip.xpipe.endpoint.HostPort; +import com.ctrip.xpipe.monitor.CatEventMonitor; +import com.ctrip.xpipe.redis.checker.alert.ALERT_TYPE; +import com.ctrip.xpipe.redis.checker.alert.AlertManager; +import com.ctrip.xpipe.redis.checker.model.DcClusterShard; +import com.ctrip.xpipe.redis.console.AbstractCrossDcIntervalAction; +import com.ctrip.xpipe.redis.console.model.MigrationKeeperContainerDetailModel; +import com.ctrip.xpipe.redis.console.model.ShardModel; +import com.ctrip.xpipe.redis.console.service.model.ShardModelService; +import com.ctrip.xpipe.utils.VisibleForTesting; +import com.google.common.collect.Lists; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; +import org.springframework.util.CollectionUtils; + +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Set; + +/** + * @author yu + *

+ * 2023/9/27 + */ +@Component +public class AutoMigrateOverloadKeeperContainerAction extends AbstractCrossDcIntervalAction { + + @Autowired + private KeeperContainerUsedInfoAnalyzer analyzer; + + @Autowired + private ShardModelService shardModelService; + + @Autowired + private AlertManager alertManager; + + private final List alertType = Lists.newArrayList(ALERT_TYPE.KEEPER_MIGRATION_FAIL, ALERT_TYPE.KEEPER_MIGRATION_SUCCESS); + + private final static String KEEPER_MIGRATION_SUCCESS = "keeper_migration_success"; + + private final static String KEEPER_MIGRATION_FAIL = "keeper_migration_fail"; + + @Override + protected void doAction() { + List readyToMigrationKeeperContainers = analyzer.getAllDcReadyToMigrationKeeperContainers(); + if (CollectionUtils.isEmpty(readyToMigrationKeeperContainers)) return; + + migrateAllKeepers(readyToMigrationKeeperContainers); + } + + @VisibleForTesting + void migrateAllKeepers(List readyToMigrationKeeperContainers) { + Set alreadyMigrateShards = new HashSet<>(); + for (MigrationKeeperContainerDetailModel migrationKeeperContainerDetailModel : readyToMigrationKeeperContainers) { + List migrateShards = migrationKeeperContainerDetailModel.getMigrateShards(); + if (CollectionUtils.isEmpty(migrateShards)) continue; + + String srcKeeperContainerIp = migrationKeeperContainerDetailModel.getSrcKeeperContainer().getKeeperIp(); + Iterator iterator= migrateShards.iterator(); + while (iterator.hasNext()) { + DcClusterShard migrateShard = iterator.next(); + if (!alreadyMigrateShards.add(migrateShard)) { + logger.info("[migrateAllKeepers] shard {} has already migrated, should not migrate in the same time", migrateShard); + continue; + } + + ShardModel shardModel = shardModelService.getShardModel(migrateShard.getDcId(), + migrateShard.getClusterId(), migrateShard.getShardId(), false, null); + + if (!shardModelService.migrateShardKeepers(migrateShard.getDcId(), migrateShard.getClusterId(), shardModel, + migrationKeeperContainerDetailModel.getTargetKeeperContainer().getKeeperIp(), srcKeeperContainerIp)) { + logger.warn("[migrateAllKeepers] migrate shard keepers failed, shard: {}", migrateShard); + alertForKeeperMigrationFail(migrateShard, srcKeeperContainerIp, + migrationKeeperContainerDetailModel.getTargetKeeperContainer().getKeeperIp()); + continue; + } + + alertForKeeperMigrationSuccess(migrateShard, srcKeeperContainerIp, + migrationKeeperContainerDetailModel.getTargetKeeperContainer().getKeeperIp()); + migrationKeeperContainerDetailModel.migrateKeeperCountIncrease(); + iterator.remove(); + } + } + } + + @Override + protected List alertTypes() { + return alertType; + } + + private void alertForKeeperMigrationSuccess(DcClusterShard dcClusterShard, String srcKeeperContainerIp, String targetKeeperContainerIp) { + CatEventMonitor.DEFAULT.logEvent(KEEPER_MIGRATION_SUCCESS, String.format("dc:%s, cluster:%s, shard:%s, src:%s, target:%s", + dcClusterShard.getDcId(), dcClusterShard.getClusterId(), dcClusterShard.getShardId(), srcKeeperContainerIp, + targetKeeperContainerIp)); + + alertManager.alert(dcClusterShard.getDcId(), dcClusterShard.getClusterId(), dcClusterShard.getShardId(), + new HostPort(srcKeeperContainerIp, 0), ALERT_TYPE.KEEPER_MIGRATION_SUCCESS, "keeper migration success"); + } + + private void alertForKeeperMigrationFail(DcClusterShard dcClusterShard, String srcKeeperContainerIp, String targetKeeperContainerIp) { + CatEventMonitor.DEFAULT.logEvent(KEEPER_MIGRATION_FAIL, String.format("dc:%s, cluster:%s, shard:%s, src:%s, target:%s", + dcClusterShard.getDcId(), dcClusterShard.getClusterId(), dcClusterShard.getShardId(), srcKeeperContainerIp, + targetKeeperContainerIp)); + + alertManager.alert(dcClusterShard.getDcId(), dcClusterShard.getClusterId(), dcClusterShard.getShardId(), + new HostPort(srcKeeperContainerIp, 0), ALERT_TYPE.KEEPER_MIGRATION_FAIL, "keeper migration fail"); + } + + + @Override + protected boolean shouldDoAction() { + return consoleConfig.isAutoMigrateOverloadKeeperContainerOpen() && super.shouldDoAction(); + } + + @Override + protected long getLeastIntervalMilli() { + return consoleConfig.getAutoMigrateOverloadKeeperContainerIntervalMilli(); + } + + @Override + protected long getIntervalMilli() { + return consoleConfig.getAutoMigrateOverloadKeeperContainerIntervalMilli(); + } +} diff --git a/redis/redis-console/src/main/java/com/ctrip/xpipe/redis/console/keeper/KeeperContainerOverloadCause.java b/redis/redis-console/src/main/java/com/ctrip/xpipe/redis/console/keeper/KeeperContainerOverloadCause.java new file mode 100644 index 000000000..8e0e513c9 --- /dev/null +++ b/redis/redis-console/src/main/java/com/ctrip/xpipe/redis/console/keeper/KeeperContainerOverloadCause.java @@ -0,0 +1,20 @@ +package com.ctrip.xpipe.redis.console.keeper; + +import com.ctrip.xpipe.utils.StringUtil; + +public enum KeeperContainerOverloadCause { + PEER_DATA_OVERLOAD, + INPUT_FLOW_OVERLOAD, + BOTH; + + public static KeeperContainerOverloadCause findByValue(String value) { + if(StringUtil.isEmpty(value)) return null; + String upperValue = value.toUpperCase(); + for (KeeperContainerOverloadCause keeperContainerOverloadCause : KeeperContainerOverloadCause.values()) { + if (keeperContainerOverloadCause.name().equals(upperValue)) { + return keeperContainerOverloadCause; + } + } + throw new IllegalArgumentException("no DcGroupType for value " + value); + } +} diff --git a/redis/redis-console/src/main/java/com/ctrip/xpipe/redis/console/keeper/KeeperContainerUsedInfoAnalyzer.java b/redis/redis-console/src/main/java/com/ctrip/xpipe/redis/console/keeper/KeeperContainerUsedInfoAnalyzer.java new file mode 100644 index 000000000..8f0feca3b --- /dev/null +++ b/redis/redis-console/src/main/java/com/ctrip/xpipe/redis/console/keeper/KeeperContainerUsedInfoAnalyzer.java @@ -0,0 +1,18 @@ +package com.ctrip.xpipe.redis.console.keeper; + +import com.ctrip.xpipe.redis.checker.model.KeeperContainerUsedInfoModel; +import com.ctrip.xpipe.redis.console.model.MigrationKeeperContainerDetailModel; + +import java.util.List; +import java.util.Map; + +public interface KeeperContainerUsedInfoAnalyzer { + + void updateKeeperContainerUsedInfo(int index, List keeperContainerUsedInfoModels); + + List getAllReadyToMigrationKeeperContainers(); + + List getAllDcReadyToMigrationKeeperContainers(); + + Map> getAllKeeperContainerUsedInfoModels(); +} diff --git a/redis/redis-console/src/main/java/com/ctrip/xpipe/redis/console/keeper/impl/DefaultKeeperContainerUsedInfoAnalyzer.java b/redis/redis-console/src/main/java/com/ctrip/xpipe/redis/console/keeper/impl/DefaultKeeperContainerUsedInfoAnalyzer.java new file mode 100644 index 000000000..bc72e014e --- /dev/null +++ b/redis/redis-console/src/main/java/com/ctrip/xpipe/redis/console/keeper/impl/DefaultKeeperContainerUsedInfoAnalyzer.java @@ -0,0 +1,305 @@ +package com.ctrip.xpipe.redis.console.keeper.impl; + +import com.ctrip.xpipe.api.foundation.FoundationService; +import com.ctrip.xpipe.command.AbstractCommand; +import com.ctrip.xpipe.command.ParallelCommandChain; +import com.ctrip.xpipe.concurrent.AbstractExceptionLogTask; +import com.ctrip.xpipe.monitor.CatEventMonitor; +import com.ctrip.xpipe.redis.checker.model.DcClusterShard; +import com.ctrip.xpipe.redis.checker.model.KeeperContainerUsedInfoModel; +import com.ctrip.xpipe.redis.console.config.ConsoleConfig; +import com.ctrip.xpipe.redis.console.keeper.KeeperContainerOverloadCause; +import com.ctrip.xpipe.redis.console.keeper.KeeperContainerUsedInfoAnalyzer; +import com.ctrip.xpipe.redis.console.model.KeeperContainerOverloadStandardModel; +import com.ctrip.xpipe.redis.console.model.MigrationKeeperContainerDetailModel; +import com.ctrip.xpipe.redis.core.service.AbstractService; +import com.ctrip.xpipe.spring.AbstractSpringConfigContext; +import com.ctrip.xpipe.tuple.Pair; +import com.ctrip.xpipe.utils.VisibleForTesting; +import com.google.common.util.concurrent.MoreExecutors; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.core.ParameterizedTypeReference; +import org.springframework.http.HttpMethod; +import org.springframework.http.ResponseEntity; +import org.springframework.web.client.RestOperations; + +import javax.annotation.Resource; +import java.util.*; +import java.util.concurrent.Executor; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; + + +public class DefaultKeeperContainerUsedInfoAnalyzer extends AbstractService implements KeeperContainerUsedInfoAnalyzer { + + private static final Logger logger = LoggerFactory.getLogger(DefaultKeeperContainerUsedInfoAnalyzer.class); + + private static final int DEFAULT_PEER_DATA_OVERLOAD = 474 * 1024 * 1024 * 1024; + + private static final int DEFAULT_KEEPER_FLOW_OVERLOAD = 270 * 1024; + + @Autowired + private ConsoleConfig config; + + @Resource(name = AbstractSpringConfigContext.GLOBAL_EXECUTOR) + private Executor executors; + + private Map checkerIndexes = new TreeMap<>(); + + private Map> allKeeperContainerUsedInfoModels = new HashMap<>(); + + private List allDcKeeperContainerDetailModel = new ArrayList<>(); + + private static final String currentDc = FoundationService.DEFAULT.getDataCenter(); + + private static final String KEEPER_RESOURCE_LACK = "keeper_resource_lack"; + + @Override + public List getAllReadyToMigrationKeeperContainers() { + List result = Collections.synchronizedList(new ArrayList<>()); + ParallelCommandChain commandChain = new ParallelCommandChain(MoreExecutors.directExecutor(), false); + config.getConsoleDomains().forEach((dc, domains) -> { + if (currentDc.equalsIgnoreCase(dc)) { + result.addAll(getAllDcReadyToMigrationKeeperContainers()); + } else { + MigrationKeeperContainerDetailInfoGetCommand command = new MigrationKeeperContainerDetailInfoGetCommand(domains, restTemplate); + command.future().addListener(commandFuture -> { + if (commandFuture.isSuccess() && commandFuture.get() != null) result.addAll(commandFuture.get()); + }); + commandChain.add(command); + } + }); + + try { + commandChain.execute().get(10, TimeUnit.SECONDS); + } catch (Throwable th) { + logger.warn("[getAllReadyToMigrationKeeperContainers] error:", th); + } + + return result; + } + + @Override + public List getAllDcReadyToMigrationKeeperContainers() { + return allDcKeeperContainerDetailModel; + } + + @Override + public synchronized void updateKeeperContainerUsedInfo(int index, List keeperContainerUsedInfoModels) { + if (keeperContainerUsedInfoModels != null && !keeperContainerUsedInfoModels.isEmpty()){ + allKeeperContainerUsedInfoModels.put(index, keeperContainerUsedInfoModels); + } + + Date currentTime = new Date(); + checkerIndexes.put(currentTime, index); + removeExpireData(currentTime); + if (!checkDataIntegrity()) return; + + List newKeeperContainerUsedInfoModels = new ArrayList<>(); + allKeeperContainerUsedInfoModels.values().forEach(newKeeperContainerUsedInfoModels::addAll); + + allKeeperContainerUsedInfoModels.clear(); + checkerIndexes.clear(); + + executors.execute(new AbstractExceptionLogTask() { + @Override + protected void doRun() throws Exception { + analyzeKeeperContainerUsedInfo(newKeeperContainerUsedInfoModels); + } + }); + } + + private void removeExpireData(Date currentTime) { + Iterator> iterator = checkerIndexes.entrySet().iterator(); + while (iterator.hasNext()) { + Map.Entry entry = iterator.next(); + logger.warn("[removeExpireData] remove expire index:{} time:{}, expire time:{}", entry.getValue(), entry.getKey(), config.getKeeperCheckerIntervalMilli()); + if (currentTime.getTime() - entry.getKey().getTime() > config.getKeeperCheckerIntervalMilli()) { + allKeeperContainerUsedInfoModels.remove(entry.getValue()); + iterator.remove(); + } + break; + } + } + + private boolean checkDataIntegrity() { + return checkerIndexes.size() == config.getClusterDividedParts(); + } + + @VisibleForTesting + void analyzeKeeperContainerUsedInfo(List newKeeperContainerUsedInfoModels) { + logger.debug("[analyzeKeeperContainerUsedInfo] newKeeperContainerUsedInfoModels: {}", newKeeperContainerUsedInfoModels); + PriorityQueue minInputFlowKeeperContainers = new PriorityQueue<>(newKeeperContainerUsedInfoModels.size(), + (keeper1, keeper2) -> (int)(keeper1.getTotalInputFlow() - keeper2.getTotalInputFlow())); + PriorityQueue minPeerDataKeeperContainers = new PriorityQueue<>(newKeeperContainerUsedInfoModels.size(), + (keeper1, keeper2) -> (int)(keeper1.getTotalRedisUsedMemory() - keeper2.getTotalRedisUsedMemory())); + newKeeperContainerUsedInfoModels.forEach(keeperContainerInfoModel -> { + minPeerDataKeeperContainers.add(keeperContainerInfoModel); + minInputFlowKeeperContainers.add(keeperContainerInfoModel); + }); + + KeeperContainerOverloadStandardModel keeperContainerOverloadStandard = config.getKeeperContainerOverloadStandards().get(currentDc); + if (keeperContainerOverloadStandard == null) { + logger.warn("[analyzeKeeperContainerUsedInfo] keeperContainerOverloadStandard from dc {} is null," + + " use default config", currentDc); + keeperContainerOverloadStandard = new KeeperContainerOverloadStandardModel() + .setFlowOverload(DEFAULT_KEEPER_FLOW_OVERLOAD).setPeerDataOverload(DEFAULT_PEER_DATA_OVERLOAD); + } + logger.debug("[analyzeKeeperContainerUsedInfo] keeperContainerOverloadStandard: {}", keeperContainerOverloadStandard); + List result = new ArrayList<>(); + for (KeeperContainerUsedInfoModel keeperContainerUsedInfoModel : newKeeperContainerUsedInfoModels) { + long overloadInputFlow = keeperContainerUsedInfoModel.getTotalInputFlow() - keeperContainerOverloadStandard.getFlowOverload(); + long overloadPeerData = keeperContainerUsedInfoModel.getTotalRedisUsedMemory() - keeperContainerOverloadStandard.getPeerDataOverload(); + + KeeperContainerOverloadCause overloadCause = getKeeperContainerOverloadCause(overloadInputFlow, overloadPeerData); + if (overloadCause == null) continue; + + result.addAll(getMigrationKeeperDetails(keeperContainerUsedInfoModel, overloadInputFlow, + overloadPeerData, overloadCause, keeperContainerOverloadStandard, minInputFlowKeeperContainers, minPeerDataKeeperContainers)); + } + allDcKeeperContainerDetailModel = result; + } + + + private List getMigrationKeeperDetails(KeeperContainerUsedInfoModel src, + long overloadInputFlow, long overloadPeerData, KeeperContainerOverloadCause overloadCause, + KeeperContainerOverloadStandardModel keeperContainerOverloadStandard, + PriorityQueue minInputFlowKeeperContainers, + PriorityQueue minPeerDataKeeperContainers) { + + switch (overloadCause) { + case PEER_DATA_OVERLOAD: + return getMigrationKeeperDetails(src, true, overloadPeerData, keeperContainerOverloadStandard, minPeerDataKeeperContainers); + case INPUT_FLOW_OVERLOAD: + case BOTH: + return getMigrationKeeperDetails(src, false, overloadInputFlow, keeperContainerOverloadStandard, minInputFlowKeeperContainers); + default: + logger.warn("invalid keeper container overload cause {}", overloadCause); + return null; + } + } + + private List getMigrationKeeperDetails(KeeperContainerUsedInfoModel src, boolean isPeerDataOverload, + long overloadData, KeeperContainerOverloadStandardModel keeperContainerOverloadStandard, + PriorityQueue availableKeeperContainers) { + List>> allDcClusterShards = null; + if (isPeerDataOverload) { + allDcClusterShards = src.getDetailInfo().entrySet().stream() + .sorted((o1, o2) -> (int) (o2.getValue().getValue() - o1.getValue().getValue())).collect(Collectors.toList()); + } else { + allDcClusterShards = src.getDetailInfo().entrySet().stream() + .sorted((o1, o2) -> (int) (o2.getValue().getKey() - o1.getValue().getKey())).collect(Collectors.toList()); + } + + List result = new ArrayList<>(); + KeeperContainerUsedInfoModel target = null; + MigrationKeeperContainerDetailModel keeperContainerDetailModel = null; + + logger.debug("[analyzeKeeperContainerUsedInfo] src: {}, overlaodCause:{}, overloadData:{}, availableKeeperContainers:{} ", + src, isPeerDataOverload, overloadData, availableKeeperContainers); + + for (Map.Entry> dcClusterShard : allDcClusterShards) { + if (target == null ) { + target = availableKeeperContainers.poll(); + if (target == null) { + logger.warn("[analyzeKeeperContainerUsedInfo] no available keeper containers {} for overload keeper container {}", + availableKeeperContainers, src); + CatEventMonitor.DEFAULT.logEvent(KEEPER_RESOURCE_LACK, currentDc); + return result; + } + + keeperContainerDetailModel = new MigrationKeeperContainerDetailModel( + new KeeperContainerUsedInfoModel(src.getKeeperIp(),src.getDcName(), src.getTotalInputFlow(), src.getTotalRedisUsedMemory()), + new KeeperContainerUsedInfoModel(target.getKeeperIp(),src.getDcName(), target.getTotalInputFlow(), target.getTotalRedisUsedMemory()), + 0, new ArrayList<>()); + } + + long currentOverLoadData = isPeerDataOverload ? dcClusterShard.getValue().getValue() : dcClusterShard.getValue().getKey(); + + long targetInputFlow = target.getTotalInputFlow() + dcClusterShard.getValue().getKey(); + long targetPeerData = target.getTotalRedisUsedMemory() + dcClusterShard.getValue().getValue(); + if (targetInputFlow > keeperContainerOverloadStandard.getFlowOverload() + || targetPeerData > keeperContainerOverloadStandard.getPeerDataOverload()) { + logger.debug("[analyzeKeeperContainerUsedInfo] target keeper container {} can not add shard {}", + target, dcClusterShard); + if (keeperContainerDetailModel.getMigrateKeeperCount() != 0) result.add(keeperContainerDetailModel); + target = null; + keeperContainerDetailModel = null; + continue; + } + keeperContainerDetailModel.addReadyToMigrateShard(dcClusterShard.getKey()); + target.setTotalInputFlow(targetInputFlow).setTotalRedisUsedMemory(targetPeerData); + + if ((overloadData -= currentOverLoadData) <= 0) break; + } + + if (keeperContainerDetailModel != null && keeperContainerDetailModel.getMigrateKeeperCount() != 0) { + result.add(keeperContainerDetailModel); + } + + if (target != null && target.getTotalInputFlow() < keeperContainerOverloadStandard.getPeerDataOverload() + && target.getTotalRedisUsedMemory() < keeperContainerOverloadStandard.getPeerDataOverload()) { + availableKeeperContainers.add(target); + } + + return result; + } + + private KeeperContainerOverloadCause getKeeperContainerOverloadCause(long overloadInputFlow, long overloadPeerData) { + if (overloadInputFlow <= 0 && overloadPeerData <= 0) { + return null; + } else if (overloadInputFlow > 0 && overloadPeerData > 0) { + return KeeperContainerOverloadCause.BOTH; + } else if (overloadInputFlow > 0) { + return KeeperContainerOverloadCause.INPUT_FLOW_OVERLOAD; + } else { + return KeeperContainerOverloadCause.PEER_DATA_OVERLOAD; + } + } + + @VisibleForTesting + Map getCheckerIndexes() { + return checkerIndexes; + } + + @Override + public Map> getAllKeeperContainerUsedInfoModels() { + return allKeeperContainerUsedInfoModels; + } + + class MigrationKeeperContainerDetailInfoGetCommand extends AbstractCommand> { + + private String domain; + private RestOperations restTemplate; + + public MigrationKeeperContainerDetailInfoGetCommand(String domain, RestOperations restTemplate) { + this.domain = domain; + this.restTemplate = restTemplate; + } + + @Override + public String getName() { + return "getMigrationKeeperContainerDetailInfo"; + } + + @Override + protected void doExecute() throws Throwable { + try { + ResponseEntity> result = + restTemplate.exchange(domain + "/api/keepercontainer/overload/info/all", HttpMethod.GET, null, + new ParameterizedTypeReference>() {}); + future().setSuccess(result.getBody()); + } catch (Throwable th) { + getLogger().error("get migration keeper container detail:{} fail", domain, th); + future().setFailure(th); + } + } + + @Override + protected void doReset() { + + } + } +} diff --git a/redis/redis-console/src/main/java/com/ctrip/xpipe/redis/console/model/KeeperContainerOverloadStandardModel.java b/redis/redis-console/src/main/java/com/ctrip/xpipe/redis/console/model/KeeperContainerOverloadStandardModel.java new file mode 100644 index 000000000..4376ba641 --- /dev/null +++ b/redis/redis-console/src/main/java/com/ctrip/xpipe/redis/console/model/KeeperContainerOverloadStandardModel.java @@ -0,0 +1,34 @@ +package com.ctrip.xpipe.redis.console.model; + +public class KeeperContainerOverloadStandardModel { + + private long peerDataOverload; + + private long flowOverload; + + public long getPeerDataOverload() { + return peerDataOverload; + } + + public KeeperContainerOverloadStandardModel setPeerDataOverload(long peerDataOverload) { + this.peerDataOverload = peerDataOverload; + return this; + } + + public long getFlowOverload() { + return flowOverload; + } + + public KeeperContainerOverloadStandardModel setFlowOverload(long flowOverload) { + this.flowOverload = flowOverload; + return this; + } + + @Override + public String toString() { + return "KeeperContainerOverloadStandardModel{" + + "peerDataOverload=" + peerDataOverload + + ", flowOverload=" + flowOverload + + '}'; + } +} diff --git a/redis/redis-console/src/main/java/com/ctrip/xpipe/redis/console/model/MigrationKeeperContainerDetailModel.java b/redis/redis-console/src/main/java/com/ctrip/xpipe/redis/console/model/MigrationKeeperContainerDetailModel.java new file mode 100644 index 000000000..47b8af1f2 --- /dev/null +++ b/redis/redis-console/src/main/java/com/ctrip/xpipe/redis/console/model/MigrationKeeperContainerDetailModel.java @@ -0,0 +1,119 @@ +package com.ctrip.xpipe.redis.console.model; + +import com.ctrip.xpipe.redis.checker.model.DcClusterShard; +import com.ctrip.xpipe.redis.checker.model.KeeperContainerUsedInfoModel; + +import java.io.Serializable; +import java.util.List; +import java.util.Objects; + +public class MigrationKeeperContainerDetailModel implements Serializable { + + private KeeperContainerUsedInfoModel srcKeeperContainer; + + private KeeperContainerUsedInfoModel targetKeeperContainer; + + private int migrateKeeperCount = 0; + + private int migrateKeeperCompleteCount = 0; + + List migrateShards; + + public MigrationKeeperContainerDetailModel() { + + } + + public MigrationKeeperContainerDetailModel(KeeperContainerUsedInfoModel srcKeeperContainer, KeeperContainerUsedInfoModel targetKeeperContainer, + int migrateKeeperCount, List migrateShards) { + this.srcKeeperContainer = srcKeeperContainer; + this.targetKeeperContainer = targetKeeperContainer; + this.migrateKeeperCount = migrateKeeperCount; + this.migrateShards = migrateShards; + } + + public void addReadyToMigrateShard( DcClusterShard shard) { + migrateShards.add(shard); + migrateKeeperCount++; + } + + public void migrateShardCompletion(DcClusterShard dcClusterShard) { + migrateShards.remove(dcClusterShard); + migrateKeeperCompleteCount++; + } + + public KeeperContainerUsedInfoModel getSrcKeeperContainer() { + return srcKeeperContainer; + } + + public MigrationKeeperContainerDetailModel setSrcKeeperContainer(KeeperContainerUsedInfoModel srcKeeperContainer) { + this.srcKeeperContainer = srcKeeperContainer; + return this; + } + + public KeeperContainerUsedInfoModel getTargetKeeperContainer() { + return targetKeeperContainer; + } + + public MigrationKeeperContainerDetailModel setTargetKeeperContainer(KeeperContainerUsedInfoModel targetKeeperContainer) { + this.targetKeeperContainer = targetKeeperContainer; + return this; + } + + public int getMigrateKeeperCount() { + return migrateKeeperCount; + } + + public MigrationKeeperContainerDetailModel setMigrateKeeperCount(int migrateKeeperCount) { + this.migrateKeeperCount = migrateKeeperCount; + return this; + } + + public List getMigrateShards() { + return migrateShards; + } + + public MigrationKeeperContainerDetailModel setMigrateShards(List migrateShards) { + this.migrateShards = migrateShards; + return this; + } + + public void migrateKeeperCountIncrease() { + this.migrateKeeperCount++; + } + + public void migrateKeeperCompleteCountIncrease() { + this.migrateKeeperCompleteCount++; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + MigrationKeeperContainerDetailModel that = (MigrationKeeperContainerDetailModel) o; + return srcKeeperContainer.equals(that.srcKeeperContainer) && targetKeeperContainer.equals(that.targetKeeperContainer); + } + + public void setMigrateKeeperCompleteCount(int migrateKeeperCompleteCount) { + this.migrateKeeperCompleteCount = migrateKeeperCompleteCount; + } + + public int getMigrateKeeperCompleteCount() { + return migrateKeeperCompleteCount; + } + + @Override + public int hashCode() { + return Objects.hash(srcKeeperContainer, targetKeeperContainer); + } + + @Override + public String toString() { + return "MigrationKeeperContainerDetailModel{" + + "srcKeeperContainer=" + srcKeeperContainer + + ", targetKeeperContainer=" + targetKeeperContainer + + ", migrateKeeperCount=" + migrateKeeperCount + + ", migrateKeeperCompleteCount=" + migrateKeeperCompleteCount + + ", migrateShards=" + migrateShards + + '}'; + } +} diff --git a/redis/redis-console/src/main/java/com/ctrip/xpipe/redis/console/model/MigrationKeeperDetailModel.java b/redis/redis-console/src/main/java/com/ctrip/xpipe/redis/console/model/MigrationKeeperDetailModel.java new file mode 100644 index 000000000..004e9456f --- /dev/null +++ b/redis/redis-console/src/main/java/com/ctrip/xpipe/redis/console/model/MigrationKeeperDetailModel.java @@ -0,0 +1,36 @@ +package com.ctrip.xpipe.redis.console.model; + +import com.ctrip.xpipe.redis.checker.model.DcClusterShard; + +public class MigrationKeeperDetailModel { + + private DcClusterShard migrateShad; + + private String targetKeeperContainerIp; + + public DcClusterShard getMigrateShad() { + return migrateShad; + } + + public MigrationKeeperDetailModel setMigrateShad(DcClusterShard migrateShad) { + this.migrateShad = migrateShad; + return this; + } + + public String getTargetKeeperContainerIp() { + return targetKeeperContainerIp; + } + + public MigrationKeeperDetailModel setTargetKeeperContainerIp(String targetKeeperContainerIp) { + this.targetKeeperContainerIp = targetKeeperContainerIp; + return this; + } + + @Override + public String toString() { + return "MigrationKeeperDetailModel{" + + "migrateShad=" + migrateShad + + ", targetKeeperContainerIp='" + targetKeeperContainerIp + '\'' + + '}'; + } +} diff --git a/redis/redis-console/src/main/java/com/ctrip/xpipe/redis/console/resources/AbstractMetaCache.java b/redis/redis-console/src/main/java/com/ctrip/xpipe/redis/console/resources/AbstractMetaCache.java index 86fcabe89..0528f19bb 100644 --- a/redis/redis-console/src/main/java/com/ctrip/xpipe/redis/console/resources/AbstractMetaCache.java +++ b/redis/redis-console/src/main/java/com/ctrip/xpipe/redis/console/resources/AbstractMetaCache.java @@ -157,7 +157,7 @@ private int dcMasterCount(ClusterMeta dcCluster) { return masterCount.get(); } - protected XpipeMeta createDividedMeta(XpipeMeta full, Set reqClusters) { + protected XpipeMeta createDividedMeta(XpipeMeta full, Set reqClusters, Set requestKeeperContainers) { XpipeMeta part = new XpipeMeta(); for (DcMeta dcMeta: full.getDcs().values()) { DcMeta partDcMeta = new DcMeta(dcMeta.getId()).setLastModifiedTime(dcMeta.getLastModifiedTime()).setZone(dcMeta.getZone()); @@ -193,7 +193,12 @@ protected XpipeMeta createDividedMeta(XpipeMeta full, Set reqClusters) { partDcMeta.addCluster(partClusterMeta); } dcMeta.getSentinels().values().forEach(partDcMeta::addSentinel); - dcMeta.getKeeperContainers().forEach(partDcMeta::addKeeperContainer); + + dcMeta.getKeeperContainers().forEach(keeperContainerMeta -> { + if (requestKeeperContainers.contains(keeperContainerMeta.getId())) { + partDcMeta.addKeeperContainer(keeperContainerMeta); + } + }); dcMeta.getRoutes().forEach(partDcMeta::addRoute); dcMeta.getMetaServers().forEach(partDcMeta::addMetaServer); } diff --git a/redis/redis-console/src/main/java/com/ctrip/xpipe/redis/console/resources/CheckerCurrentDcAllMeta.java b/redis/redis-console/src/main/java/com/ctrip/xpipe/redis/console/resources/CheckerCurrentDcAllMeta.java new file mode 100644 index 000000000..80c600349 --- /dev/null +++ b/redis/redis-console/src/main/java/com/ctrip/xpipe/redis/console/resources/CheckerCurrentDcAllMeta.java @@ -0,0 +1,95 @@ +package com.ctrip.xpipe.redis.console.resources; + +import com.ctrip.xpipe.api.foundation.FoundationService; +import com.ctrip.xpipe.redis.checker.CheckerConsoleService; +import com.ctrip.xpipe.redis.checker.config.CheckerConfig; +import com.ctrip.xpipe.redis.core.entity.DcMeta; +import com.ctrip.xpipe.redis.core.meta.CurrentDcAllMeta; +import com.ctrip.xpipe.utils.VisibleForTesting; +import com.ctrip.xpipe.utils.XpipeThreadFactory; +import com.ctrip.xpipe.utils.job.DynamicDelayPeriodTask; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; + +import javax.annotation.PostConstruct; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; + +/** + * @author yu + *

+ * 2023/11/14 + */ +public class CheckerCurrentDcAllMeta implements CurrentDcAllMeta { + + private static final Logger logger = LoggerFactory.getLogger(CheckerCurrentDcAllMeta.class); + + @Autowired + private CheckerConfig config; + + @Autowired + private CheckerConsoleService checkerConsoleService; + + private DcMeta currentDcAllMeta; + + private DynamicDelayPeriodTask metaLoadTask; + + private ScheduledExecutorService scheduled; + + private String currentDcId; + + @PostConstruct + public void init(){ + scheduled = Executors.newScheduledThreadPool(1, XpipeThreadFactory.create("CheckerCurrentDcAllMetaLoader")); + metaLoadTask = new DynamicDelayPeriodTask("CheckerCurrentDcAllMetaLoader", this::loadMeta, + config::getCheckerCurrentDcAllMetaRefreshIntervalMilli, scheduled); + this.currentDcId = FoundationService.DEFAULT.getDataCenter(); + + try { + metaLoadTask.start(); + } catch (Throwable th) { + logger.error("start to load current dc all meta fail", th); + } + } + + + public void preDestroy() { + try { + this.metaLoadTask.stop(); + this.scheduled.shutdownNow(); + } catch (Throwable th) { + logger.error("[preDestroy] fail", th); + } + } + + private void loadMeta() { + logger.debug("[loadMeta] start to load current dc all meta"); + DcMeta dcMeta = getCurrentDcAllMeta(currentDcId); + if (dcMeta == null) { + logger.warn("[loadMeta] get current dc meta null"); + return; + } + this.currentDcAllMeta = dcMeta; + } + + private DcMeta getCurrentDcAllMeta(String dcId) { + try { + return checkerConsoleService.getXpipeDcAllMeta(config.getConsoleAddress(), dcId) + .getDcs().get(dcId); + } catch (Throwable th) { + logger.error("[getCurrentDcAllMeta] get dcMeta from dc {} fail", dcId, th); + } + return null; + } + + @Override + public DcMeta getCurrentDcAllMeta() { + return this.currentDcAllMeta; + } + + @VisibleForTesting + public void setCurrentDcAllMeta(DcMeta currentDcAllMeta) { + this.currentDcAllMeta = currentDcAllMeta; + } +} diff --git a/redis/redis-console/src/main/java/com/ctrip/xpipe/redis/console/resources/CheckerMetaCache.java b/redis/redis-console/src/main/java/com/ctrip/xpipe/redis/console/resources/CheckerMetaCache.java index be56788a4..2818abffe 100644 --- a/redis/redis-console/src/main/java/com/ctrip/xpipe/redis/console/resources/CheckerMetaCache.java +++ b/redis/redis-console/src/main/java/com/ctrip/xpipe/redis/console/resources/CheckerMetaCache.java @@ -81,5 +81,4 @@ public Map chooseDefaultRoutes(String clusterName, String src public Map chooseRoutes(String clusterName, String backUpDcName, List peerDcs, int orgId) { throw new UnsupportedOperationException(); } - } diff --git a/redis/redis-console/src/main/java/com/ctrip/xpipe/redis/console/resources/DefaultMetaCache.java b/redis/redis-console/src/main/java/com/ctrip/xpipe/redis/console/resources/DefaultMetaCache.java index ce6cba402..6125eb6d1 100644 --- a/redis/redis-console/src/main/java/com/ctrip/xpipe/redis/console/resources/DefaultMetaCache.java +++ b/redis/redis-console/src/main/java/com/ctrip/xpipe/redis/console/resources/DefaultMetaCache.java @@ -8,6 +8,7 @@ import com.ctrip.xpipe.redis.console.exception.DataNotFoundException; import com.ctrip.xpipe.redis.console.model.RedisCheckRuleTbl; import com.ctrip.xpipe.redis.console.service.ClusterService; +import com.ctrip.xpipe.redis.console.service.KeeperContainerService; import com.ctrip.xpipe.redis.console.service.RedisCheckRuleService; import com.ctrip.xpipe.redis.console.service.meta.DcMetaService; import com.ctrip.xpipe.redis.core.entity.DcMeta; @@ -48,6 +49,9 @@ public class DefaultMetaCache extends AbstractMetaCache implements MetaCache, Co @Autowired private ClusterService clusterService; + @Autowired + private KeeperContainerService keeperContainerService; + @Autowired private ConsoleConfig consoleConfig; @@ -58,6 +62,8 @@ public class DefaultMetaCache extends AbstractMetaCache implements MetaCache, Co private List> clusterParts; + private List> keeperContainerParts; + private ScheduledExecutorService scheduled = Executors.newScheduledThreadPool(1); private ScheduledFuture future; @@ -138,6 +144,7 @@ public void go() throws Exception { synchronized (this) { refreshClusterParts(); + refreshKeeperContainerParts(); XpipeMeta xpipeMeta = createXpipeMeta(dcMetas, redisCheckRuleMetas); refreshMeta(xpipeMeta); } @@ -167,6 +174,24 @@ private void refreshClusterParts() { } } + private void refreshKeeperContainerParts() { + try { + int parts = Math.max(1, consoleConfig.getClusterDividedParts()); + logger.debug("[refreshKeeperContainerParts] start parts {}", parts); + + List> newKeeperContainerParts = keeperContainerService.divideKeeperContainers(parts); + if (newKeeperContainerParts.size() < parts) { + logger.info("[refreshKeeperContainerParts] skip for parts miss, expect {}, actual {}", + parts, newKeeperContainerParts.size()); + return; + } + + this.keeperContainerParts = newKeeperContainerParts; + } catch (Throwable th) { + logger.warn("[refreshKeeperContainerParts] fail", th); + } + } + @Override public Map chooseDefaultRoutes(String clusterName, String srcDc, List dstDcs, int orgId) { return chooseRoutes(clusterName, srcDc, dstDcs, orgId, false); @@ -198,12 +223,15 @@ private RouteChooseStrategy getRouteChooseStrategy(RouteChooseStrategyFactory.Ro @Override public synchronized XpipeMeta getDividedXpipeMeta(int partIndex) { if (null == meta || null == clusterParts) throw new DataNotFoundException("data not ready"); - if (partIndex >= clusterParts.size()) throw new DataNotFoundException("no part " + partIndex); + if (partIndex >= clusterParts.size() || partIndex >= keeperContainerParts.size()) + throw new DataNotFoundException("no part " + partIndex); XpipeMeta xpipeMeta = getXpipeMeta(); Set requestClusters = clusterParts.get(partIndex); + Set requestKeeperContainers = keeperContainerParts.get(partIndex); + - return createDividedMeta(xpipeMeta, requestClusters); + return createDividedMeta(xpipeMeta, requestClusters, requestKeeperContainers); } @VisibleForTesting diff --git a/redis/redis-console/src/main/java/com/ctrip/xpipe/redis/console/service/AzService.java b/redis/redis-console/src/main/java/com/ctrip/xpipe/redis/console/service/AzService.java index df4f74a69..83f417eff 100644 --- a/redis/redis-console/src/main/java/com/ctrip/xpipe/redis/console/service/AzService.java +++ b/redis/redis-console/src/main/java/com/ctrip/xpipe/redis/console/service/AzService.java @@ -16,6 +16,8 @@ public interface AzService { void updateAvailableZone(AzCreateInfo createInfo); + boolean isDcSupportMultiAz(String dcName); + List getDcActiveAvailableZoneTbls(String dcName); List getDcAvailableZoneTbls(String dcName); diff --git a/redis/redis-console/src/main/java/com/ctrip/xpipe/redis/console/service/KeeperAdvancedService.java b/redis/redis-console/src/main/java/com/ctrip/xpipe/redis/console/service/KeeperAdvancedService.java index 0a83f6b73..ea4b7d6a8 100644 --- a/redis/redis-console/src/main/java/com/ctrip/xpipe/redis/console/service/KeeperAdvancedService.java +++ b/redis/redis-console/src/main/java/com/ctrip/xpipe/redis/console/service/KeeperAdvancedService.java @@ -1,6 +1,7 @@ package com.ctrip.xpipe.redis.console.service; -import com.ctrip.xpipe.redis.console.model.KeeperContainerInfoModel; +import com.ctrip.xpipe.redis.console.model.RedisTbl; +import com.ctrip.xpipe.redis.console.model.ShardModel; import java.util.List; import java.util.function.BiPredicate; @@ -17,6 +18,8 @@ List findBestKeepers(String dcName, int beginPort, BiPredicate< List findBestKeepers(String dcName, String clusterName); - List findBestKeepersByKeeperContainer(KeeperContainerInfoModel targetKeeperContainer, int beginPort, + List getNewKeepers(String dcName, String clusterName, ShardModel shardModel, String srcKeeperContainerIp, String targetKeeperContainerIp); + + List findBestKeepersByKeeperContainer(String targetKeeperContainerIp, int beginPort, BiPredicate keeperGood, int returnCount); } diff --git a/redis/redis-console/src/main/java/com/ctrip/xpipe/redis/console/service/KeeperContainerMigrationService.java b/redis/redis-console/src/main/java/com/ctrip/xpipe/redis/console/service/KeeperContainerMigrationService.java new file mode 100644 index 000000000..4fd2f8353 --- /dev/null +++ b/redis/redis-console/src/main/java/com/ctrip/xpipe/redis/console/service/KeeperContainerMigrationService.java @@ -0,0 +1,11 @@ +package com.ctrip.xpipe.redis.console.service; + +import com.ctrip.xpipe.redis.console.model.MigrationKeeperContainerDetailModel; + +import java.util.List; + +public interface KeeperContainerMigrationService { + void beginMigrateKeeperContainers(List keeperContainerDetailModels); + + List getMigrationProcess(); +} diff --git a/redis/redis-console/src/main/java/com/ctrip/xpipe/redis/console/service/KeeperContainerService.java b/redis/redis-console/src/main/java/com/ctrip/xpipe/redis/console/service/KeeperContainerService.java index a813ee6ad..2194cb5aa 100644 --- a/redis/redis-console/src/main/java/com/ctrip/xpipe/redis/console/service/KeeperContainerService.java +++ b/redis/redis-console/src/main/java/com/ctrip/xpipe/redis/console/service/KeeperContainerService.java @@ -6,10 +6,13 @@ import java.util.List; import java.util.Map; +import java.util.Set; public interface KeeperContainerService { KeepercontainerTbl find(long id); + KeepercontainerTbl find(String ip); + List findAll(); List findAllByDcName(String dcName); List findAllActiveByDcName(String dcName); @@ -17,6 +20,8 @@ public interface KeeperContainerService { List findBestKeeperContainersByDcCluster(String dcName, String clusterName); List getKeeperContainerByAz(Long azId); + List> divideKeeperContainers(int partsCount); + List findAllInfos(); KeeperContainerInfoModel findKeeperContainerInfoModelById(long id); List findAvailableKeeperContainerInfoModelsByDcAzAndOrg(String dcName, String azName, String orgName); diff --git a/redis/redis-console/src/main/java/com/ctrip/xpipe/redis/console/service/impl/AzServiceImpl.java b/redis/redis-console/src/main/java/com/ctrip/xpipe/redis/console/service/impl/AzServiceImpl.java index 0764c3154..d048fbe9d 100644 --- a/redis/redis-console/src/main/java/com/ctrip/xpipe/redis/console/service/impl/AzServiceImpl.java +++ b/redis/redis-console/src/main/java/com/ctrip/xpipe/redis/console/service/impl/AzServiceImpl.java @@ -12,6 +12,7 @@ import com.google.common.collect.Lists; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; +import org.springframework.util.CollectionUtils; import java.util.HashMap; import java.util.List; @@ -76,6 +77,12 @@ public void updateAvailableZone(AzCreateInfo createInfo) { azDao.updateAvailableZone(azTbl); } + @Override + public boolean isDcSupportMultiAz(String dcName) { + List dcAvailableZoneInfos = getDcAvailableZoneInfos(dcName); + return !CollectionUtils.isEmpty(dcAvailableZoneInfos) && dcAvailableZoneInfos.size() > 1; + } + @Override public List getDcActiveAvailableZoneTbls(String dcName) { DcTbl dcTbl = dcService.find(dcName); diff --git a/redis/redis-console/src/main/java/com/ctrip/xpipe/redis/console/service/impl/DefaultKeeperAdvancedService.java b/redis/redis-console/src/main/java/com/ctrip/xpipe/redis/console/service/impl/DefaultKeeperAdvancedService.java index 8f399c1b0..2bd95ca4c 100644 --- a/redis/redis-console/src/main/java/com/ctrip/xpipe/redis/console/service/impl/DefaultKeeperAdvancedService.java +++ b/redis/redis-console/src/main/java/com/ctrip/xpipe/redis/console/service/impl/DefaultKeeperAdvancedService.java @@ -1,11 +1,13 @@ package com.ctrip.xpipe.redis.console.service.impl; -import com.ctrip.xpipe.redis.console.model.KeeperContainerInfoModel; +import com.ctrip.xpipe.redis.console.constant.XPipeConsoleConstant; import com.ctrip.xpipe.redis.console.model.KeepercontainerTbl; import com.ctrip.xpipe.redis.console.model.RedisTbl; import com.ctrip.xpipe.redis.console.model.RedisTblDao; +import com.ctrip.xpipe.redis.console.model.ShardModel; import com.ctrip.xpipe.redis.console.service.*; import com.ctrip.xpipe.spring.AbstractSpringConfigContext; +import com.ctrip.xpipe.utils.ObjectUtils; import com.ctrip.xpipe.utils.VisibleForTesting; import com.google.common.collect.Maps; import org.springframework.beans.factory.annotation.Autowired; @@ -34,6 +36,9 @@ public class DefaultKeeperAdvancedService extends AbstractConsoleService findBestKeepers(String dcName, int beginPort, BiPre } + @Override + public List getNewKeepers(String dcName, String clusterName, ShardModel shardModel, String srcKeeperContainerIp, String targetKeeperContainerIp) { + List newKeepers = new ArrayList<>(); + logger.debug("[migrateKeepers] origin keepers {} from cluster:{}, dc:{}, shard:{}",shardModel.getKeepers(), clusterName, dcName, shardModel.getShardTbl().getShardName()); + for (RedisTbl keeper : shardModel.getKeepers()) { + if (!ObjectUtils.equals(keeper.getRedisIp(), srcKeeperContainerIp)) { + newKeepers.add(keeper); + } + } + + if (newKeepers.size() == 2) { + return null; + } + + if (newKeepers.size() < 1) { + logger.warn("[migrateKeepers] unexpected keepers {} from cluster:{}, dc:{}, shard:{}", + newKeepers, clusterName, dcName, shardModel.getShardTbl().getShardName()); + return newKeepers; + } + + long alreadyUsedAzId = keeperContainerService.find(newKeepers.get(0).getKeepercontainerId()).getAzId(); + List bestKeepers = findBestKeepers(clusterName, dcName, targetKeeperContainerIp); + logger.debug("[migrateKeepers] best keepers {} from cluster:{}, dc:{}, shard:{}, targetKeeperContainerIp:{}, srcKeeperContainerIp:{}", + bestKeepers, clusterName, dcName, shardModel.getShardTbl().getShardName(), targetKeeperContainerIp, srcKeeperContainerIp); + for (KeeperBasicInfo keeperSelected : bestKeepers) { + logger.debug("[migrateKeepers] keeperSelected {} , result:{}",keeperSelected, ObjectUtils.equals(keeperSelected.getHost(), srcKeeperContainerIp)); + if (!ObjectUtils.equals(keeperSelected.getHost(), srcKeeperContainerIp) + && keeperSelected.getKeeperContainerId() != newKeepers.get(0).getKeepercontainerId() + && isDifferentAz(keeperSelected, alreadyUsedAzId, dcName)) { + newKeepers.add(new RedisTbl().setKeepercontainerId(keeperSelected.getKeeperContainerId()) + .setRedisIp(keeperSelected.getHost()) + .setRedisPort(keeperSelected.getPort()) + .setRedisRole(XPipeConsoleConstant.ROLE_KEEPER)); + break; + } + } + logger.debug("[migrateKeepers] new keepers {} from cluster:{}, dc:{}, shard:{}", + newKeepers, clusterName, dcName, shardModel.getShardTbl().getShardName()); + return newKeepers; + } + + private boolean isDifferentAz(KeeperBasicInfo keeperSelected, long alreadyUsedAzId, String dcName) { + if (alreadyUsedAzId == 0 || !azService.isDcSupportMultiAz(dcName)) return true; + long newAzId = keeperContainerService.find(keeperSelected.getKeeperContainerId()).getAzId(); + return newAzId != alreadyUsedAzId; + } + + private List findBestKeepers(String clusterName, String dcName, String targetKeeperContainerIp) { + if (targetKeeperContainerIp == null ) { + return findBestKeepers(dcName, KEEPER_PORT_DEFAULT, (ip, port) -> true, clusterName); + } else { + return findBestKeepersByKeeperContainer(targetKeeperContainerIp, KEEPER_PORT_DEFAULT, (ip, port) -> true, 1); + } + } @Override - public List findBestKeepersByKeeperContainer(KeeperContainerInfoModel targetKeeperContainer, + public List findBestKeepersByKeeperContainer(String targetKeeperContainerIp, int beginPort, BiPredicate keeperGood, int returnCount) { List result = new LinkedList<>(); List keepercontainerTbls = new LinkedList<>(); - keepercontainerTbls.add(keeperContainerService.find(targetKeeperContainer.getId())); + keepercontainerTbls.add(keeperContainerService.find(targetKeeperContainerIp)); fillInResult(keepercontainerTbls, result, beginPort, keeperGood, returnCount); return result; } diff --git a/redis/redis-console/src/main/java/com/ctrip/xpipe/redis/console/service/impl/DefaultKeeperContainerMigrationService.java b/redis/redis-console/src/main/java/com/ctrip/xpipe/redis/console/service/impl/DefaultKeeperContainerMigrationService.java new file mode 100644 index 000000000..e99f25074 --- /dev/null +++ b/redis/redis-console/src/main/java/com/ctrip/xpipe/redis/console/service/impl/DefaultKeeperContainerMigrationService.java @@ -0,0 +1,67 @@ +package com.ctrip.xpipe.redis.console.service.impl; + +import com.ctrip.xpipe.redis.checker.model.DcClusterShard; +import com.ctrip.xpipe.redis.console.model.MigrationKeeperContainerDetailModel; +import com.ctrip.xpipe.redis.console.model.ShardModel; +import com.ctrip.xpipe.redis.console.service.KeeperContainerMigrationService; +import com.ctrip.xpipe.redis.console.service.model.ShardModelService; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; +import org.springframework.util.CollectionUtils; + +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.concurrent.atomic.AtomicBoolean; + +@Component +public class DefaultKeeperContainerMigrationService implements KeeperContainerMigrationService { + + private Logger logger = LoggerFactory.getLogger(DefaultKeeperContainerMigrationService.class); + + @Autowired + private ShardModelService shardModelService; + + private volatile List readyToMigrationKeeperContainers; + + private volatile AtomicBoolean isBegin = new AtomicBoolean(false); + + @Override + public void beginMigrateKeeperContainers(List keeperContainerDetailModels) { + if (!isBegin.compareAndSet(false, true)) { + logger.info("[beginMigrateKeeperContainers] has already begin!!"); + return; + } + logger.debug("[beginMigrateKeeperContainers] begin migrate keeper containers {}", keeperContainerDetailModels); + readyToMigrationKeeperContainers = keeperContainerDetailModels; + Set alreadyMigrateShards = new HashSet<>(); + for (MigrationKeeperContainerDetailModel keeperContainer : readyToMigrationKeeperContainers) { + List migrateShards = keeperContainer.getMigrateShards(); + if (CollectionUtils.isEmpty(migrateShards)) continue; + + + String srcKeeperContainerIp = keeperContainer.getSrcKeeperContainer().getKeeperIp(); + for (DcClusterShard migrateShard : migrateShards) { + ShardModel shardModel = shardModelService.getShardModel(migrateShard.getDcId(), + migrateShard.getClusterId(), migrateShard.getShardId(), false, null); + if (!alreadyMigrateShards.add(migrateShard)) { + logger.info("[beginMigrateKeeperContainers] shard {} has already migrated, should not migrate in the same time", migrateShard); + continue; + } + logger.debug("[beginMigrateKeeperContainers] begin migrate shard {} from srcKeeperContainer:{} to targetKeeperContainer:{}", + migrateShard, srcKeeperContainerIp, keeperContainer.getTargetKeeperContainer().getKeeperIp()); + if (shardModelService.migrateShardKeepers(migrateShard.getDcId(), migrateShard.getClusterId(), shardModel, + srcKeeperContainerIp, keeperContainer.getTargetKeeperContainer().getKeeperIp())) + keeperContainer.migrateKeeperCompleteCountIncrease(); + } + } + isBegin.set(false); + } + + @Override + public List getMigrationProcess() { + return readyToMigrationKeeperContainers; + } +} diff --git a/redis/redis-console/src/main/java/com/ctrip/xpipe/redis/console/service/impl/KeeperContainerServiceImpl.java b/redis/redis-console/src/main/java/com/ctrip/xpipe/redis/console/service/impl/KeeperContainerServiceImpl.java index dd3a932e9..e75b41953 100644 --- a/redis/redis-console/src/main/java/com/ctrip/xpipe/redis/console/service/impl/KeeperContainerServiceImpl.java +++ b/redis/redis-console/src/main/java/com/ctrip/xpipe/redis/console/service/impl/KeeperContainerServiceImpl.java @@ -22,6 +22,7 @@ import java.util.*; import java.util.stream.Collectors; +import java.util.stream.IntStream; @Service public class KeeperContainerServiceImpl extends AbstractConsoleService @@ -54,6 +55,16 @@ public KeepercontainerTbl doQuery() throws DalException { }); } + @Override + public KeepercontainerTbl find(final String ip) { + return queryHandler.handleQuery(new DalQuery() { + @Override + public KeepercontainerTbl doQuery() throws DalException { + return dao.findByIp(ip, KeepercontainerTblEntity.READSET_FULL); + } + }); + } + @Override public List findAll() { return queryHandler.handleQuery(new DalQuery>() { @@ -428,6 +439,20 @@ public Map keeperContainerIdDcMap() { return keeperContainerIdDcMap; } + @Override + public List> divideKeeperContainers(int partsCount) { + List all = findAll(); + if (all == null) return Collections.emptyList(); + + List> result = new ArrayList<>(partsCount); + IntStream.range(0, partsCount).forEach(i -> result.add(new HashSet<>())); + + all.forEach(keeperContainer -> result.get((int) keeperContainer.getKeepercontainerId() % partsCount) + .add(keeperContainer.getKeepercontainerId())); + + return result; + } + @Override public List findAllInfos() { diff --git a/redis/redis-console/src/main/java/com/ctrip/xpipe/redis/console/service/model/ShardModelService.java b/redis/redis-console/src/main/java/com/ctrip/xpipe/redis/console/service/model/ShardModelService.java index 6728f7631..aa020886f 100644 --- a/redis/redis-console/src/main/java/com/ctrip/xpipe/redis/console/service/model/ShardModelService.java +++ b/redis/redis-console/src/main/java/com/ctrip/xpipe/redis/console/service/model/ShardModelService.java @@ -18,4 +18,6 @@ ShardModel getShardModel(String dcName, String clusterName, String shardName, ShardModel getSourceShardModel(String clusterName, String srcDcName, String toDcName, String shardName); + boolean migrateShardKeepers(String dcName, String clusterName, ShardModel shardModel, + String srcKeeperContainerIp, String targetKeeperContainerIp); } diff --git a/redis/redis-console/src/main/java/com/ctrip/xpipe/redis/console/service/model/impl/ShardModelServiceImpl.java b/redis/redis-console/src/main/java/com/ctrip/xpipe/redis/console/service/model/impl/ShardModelServiceImpl.java index 0b97ba019..fee44ad0c 100644 --- a/redis/redis-console/src/main/java/com/ctrip/xpipe/redis/console/service/model/impl/ShardModelServiceImpl.java +++ b/redis/redis-console/src/main/java/com/ctrip/xpipe/redis/console/service/model/impl/ShardModelServiceImpl.java @@ -46,6 +46,8 @@ public class ShardModelServiceImpl implements ShardModelService{ private KeeperContainerService keeperContainerService; @Autowired private AzGroupClusterRepository azGroupClusterRepository; + @Autowired + private KeeperAdvancedService keeperAdvancedService; private static final Logger logger = LoggerFactory.getLogger(ShardModelServiceImpl.class); @@ -219,4 +221,34 @@ private void addRedisesAndKeepersToNormalShard(ShardModel shardModel, long dcClu } } + @Override + public boolean migrateShardKeepers(String dcName, String clusterName, ShardModel shardModel, + String srcKeeperContainerIp, String targetKeeperContainerIp) { + List newKeepers = keeperAdvancedService.getNewKeepers(dcName, clusterName, shardModel, + srcKeeperContainerIp, targetKeeperContainerIp); + if (newKeepers == null) { + logger.debug("[migrateKeepers] no need to replace keepers"); + return false; + }else if (newKeepers.size() == 2) { + return doMigrateKeepers(shardModel, newKeepers, clusterName, dcName, shardModel.getShardTbl().getShardName()); + } else { + logger.info("[migrateKeepers] fail to migrate keepers with unexpected newKeepers {}", newKeepers); + return false; + } + } + + private boolean doMigrateKeepers(ShardModel shardModel, List newKeepers, String clusterName, + String dcName, String shardName) { + try { + shardModel.setKeepers(newKeepers); + logger.info("[Update Redises][construct]{},{},{},{}", clusterName, dcName, shardName, shardModel); + redisService.updateRedises(dcName, clusterName, shardName, shardModel); + logger.info("[Update Redises][success]{},{},{},{}", clusterName, dcName, shardName, shardModel); + return true; + } catch (Exception e) { + logger.error("[Update Redises][failed]{},{},{},{}", clusterName, dcName, shardName, shardModel, e); + return false; + } + } + } diff --git a/redis/redis-console/src/main/java/com/ctrip/xpipe/redis/console/spring/CheckerContextConfig.java b/redis/redis-console/src/main/java/com/ctrip/xpipe/redis/console/spring/CheckerContextConfig.java index aa5790cb8..e69e2dfdb 100644 --- a/redis/redis-console/src/main/java/com/ctrip/xpipe/redis/console/spring/CheckerContextConfig.java +++ b/redis/redis-console/src/main/java/com/ctrip/xpipe/redis/console/spring/CheckerContextConfig.java @@ -9,6 +9,8 @@ import com.ctrip.xpipe.redis.checker.config.CheckerDbConfig; import com.ctrip.xpipe.redis.checker.config.impl.DefaultCheckerDbConfig; import com.ctrip.xpipe.redis.checker.healthcheck.actions.interaction.HealthStateService; +import com.ctrip.xpipe.redis.checker.healthcheck.actions.keeper.info.RedisUsedMemoryCollector; +import com.ctrip.xpipe.redis.checker.healthcheck.actions.keeper.infoStats.KeeperFlowCollector; import com.ctrip.xpipe.redis.checker.healthcheck.actions.ping.DefaultPingService; import com.ctrip.xpipe.redis.checker.healthcheck.actions.ping.PingService; import com.ctrip.xpipe.redis.checker.healthcheck.allleader.SentinelMonitorsCheckCrossDc; @@ -23,16 +25,14 @@ import com.ctrip.xpipe.redis.console.migration.auto.DefaultMonitorManager; import com.ctrip.xpipe.redis.console.migration.auto.MonitorManager; import com.ctrip.xpipe.redis.console.redis.DefaultSentinelManager; -import com.ctrip.xpipe.redis.console.resources.CheckerAllMetaCache; -import com.ctrip.xpipe.redis.console.resources.CheckerMetaCache; -import com.ctrip.xpipe.redis.console.resources.CheckerOuterClientCache; -import com.ctrip.xpipe.redis.console.resources.CheckerPersistenceCache; +import com.ctrip.xpipe.redis.console.resources.*; import com.ctrip.xpipe.redis.console.service.DcClusterShardService; import com.ctrip.xpipe.redis.console.service.impl.DcClusterShardServiceImpl; import com.ctrip.xpipe.redis.console.service.impl.DefaultDcRelationsService; import com.ctrip.xpipe.redis.console.service.meta.BeaconMetaService; import com.ctrip.xpipe.redis.console.service.meta.impl.BeaconMetaServiceImpl; import com.ctrip.xpipe.redis.console.util.DefaultMetaServerConsoleServiceManagerWrapper; +import com.ctrip.xpipe.redis.core.meta.CurrentDcAllMeta; import com.ctrip.xpipe.redis.core.meta.MetaCache; import com.ctrip.xpipe.spring.AbstractProfile; import org.springframework.beans.factory.annotation.Qualifier; @@ -74,12 +74,18 @@ public CheckerAllMetaCache checkerAllMetaCache() { return new CheckerAllMetaCache(); } + @Bean + public CurrentDcAllMeta currentDcAllMeta() { + return new CheckerCurrentDcAllMeta(); + } + @Bean @Profile(AbstractProfile.PROFILE_NAME_TEST) public MetaCache testMetaCache() { return new TestMetaCache(); } + @Bean public ProxyManager proxyManager(GroupCheckerLeaderElector clusterServer, CheckerConfig checkerConfig, CheckerConsoleService checkerConsoleService) { return new CheckerProxyManager(clusterServer, checkerConfig, checkerConsoleService); @@ -177,6 +183,13 @@ public HealthCheckReporter healthCheckReporter(CheckerConfig checkerConfig, Chec return new HealthCheckReporter(healthStateService, checkerConfig, checkerConsoleService, clusterServer, allCheckerLeaderElector, redisDelayManager, crossMasterDelayManager, pingService, clusterHealthManager, serverPort); } + + @Bean + @Profile(AbstractProfile.PROFILE_NAME_PRODUCTION) + public KeeperContainerInfoReporter keeperContainerInfoReporter(RedisUsedMemoryCollector redisUsedMemoryCollector, + CheckerConsoleService checkerConsoleService, KeeperFlowCollector keeperFlowCollector, CheckerConfig config) { + return new KeeperContainerInfoReporter(redisUsedMemoryCollector, checkerConsoleService, keeperFlowCollector, config); + } @Bean(name = "ALLCHECKER") @Profile(AbstractProfile.PROFILE_NAME_PRODUCTION) @@ -209,5 +222,4 @@ public SentinelShardBind sentinelShardBind(CheckerAllMetaCache metaCache, Checke public OuterClientCache outerClientCache(CheckerConsoleService service, CheckerConfig config) { return new CheckerOuterClientCache(service, config); } - } diff --git a/redis/redis-console/src/main/java/com/ctrip/xpipe/redis/console/spring/ConsoleContextConfig.java b/redis/redis-console/src/main/java/com/ctrip/xpipe/redis/console/spring/ConsoleContextConfig.java index fca8e189a..a8c465a9b 100644 --- a/redis/redis-console/src/main/java/com/ctrip/xpipe/redis/console/spring/ConsoleContextConfig.java +++ b/redis/redis-console/src/main/java/com/ctrip/xpipe/redis/console/spring/ConsoleContextConfig.java @@ -20,6 +20,8 @@ import com.ctrip.xpipe.redis.console.dao.RedisDao; import com.ctrip.xpipe.redis.console.healthcheck.nonredis.cluster.ClusterHealthMonitorManager; import com.ctrip.xpipe.redis.console.healthcheck.nonredis.cluster.impl.DefaultClusterHealthMonitorManager; +import com.ctrip.xpipe.redis.console.keeper.KeeperContainerUsedInfoAnalyzer; +import com.ctrip.xpipe.redis.console.keeper.impl.DefaultKeeperContainerUsedInfoAnalyzer; import com.ctrip.xpipe.redis.console.resources.DefaultMetaCache; import com.ctrip.xpipe.redis.console.resources.DefaultPersistenceCache; import com.ctrip.xpipe.redis.console.service.DcClusterShardService; @@ -120,6 +122,11 @@ public PingService pingService() { return new ConsoleCachedPingService(); } + @Bean + public KeeperContainerUsedInfoAnalyzer KeeperContainerUsedInfoAnalyzer() { + return new DefaultKeeperContainerUsedInfoAnalyzer(); + } + @Bean public RedisInfoService redisInfoService() { return new ConsoleRedisInfoService(); diff --git a/redis/redis-console/src/main/resources/META-INF/dal/jdbc/meta-dal.xml b/redis/redis-console/src/main/resources/META-INF/dal/jdbc/meta-dal.xml index d1e506dfa..1fa084107 100644 --- a/redis/redis-console/src/main/resources/META-INF/dal/jdbc/meta-dal.xml +++ b/redis/redis-console/src/main/resources/META-INF/dal/jdbc/meta-dal.xml @@ -1702,6 +1702,17 @@ ]]> + + + + + FROM + WHERE = ${keepercontainer-ip} + AND = 0 + ]]> + + diff --git a/redis/redis-console/src/main/resources/static/dist/bundle.js b/redis/redis-console/src/main/resources/static/dist/bundle.js index 12e3bbae6..6a168dd52 100644 --- a/redis/redis-console/src/main/resources/static/dist/bundle.js +++ b/redis/redis-console/src/main/resources/static/dist/bundle.js @@ -9167,6 +9167,16 @@ eval("angular\n .module('index')\n .controller('KeepercontainerFormCtl', K /***/ }), +/***/ "./scripts/controllers/KeepercontainerOverloadCtl.ts": +/*!***********************************************************!*\ + !*** ./scripts/controllers/KeepercontainerOverloadCtl.ts ***! + \***********************************************************/ +/***/ (() => { + +eval("angular\n .module('index')\n .controller('KeepercontainerOverloadCtl', KeepercontainerOverloadCtl);\nKeepercontainerOverloadCtl.$inject = ['$rootScope', '$scope', '$window', '$stateParams', 'KeeperContainerService',\n 'toastr', 'NgTableParams', '$interval'];\nfunction KeepercontainerOverloadCtl($rootScope, $scope, $window, $stateParams, KeeperContainerService, toastr, NgTableParams, $interval) {\n $scope.overloadKeeperContainer = [];\n $scope.tableParams = new NgTableParams({}, {});\n $scope.migratingTableParams = new NgTableParams({}, {});\n $scope.selectAll = false;\n $scope.toggleAll = toggleAll;\n $scope.isChecked = isChecked;\n var OPERATE_TYPE = {\n DETAIL: 'detail',\n MIGRATING: 'migrating',\n STOPPED: 'stopped'\n };\n $scope.operateType = $stateParams.type;\n $scope.migratingKeeperContainers = [];\n $scope.scheduledWork;\n $scope.beginToMigrateOverloadKeeperContainers = beginToMigrateOverloadKeeperContainers;\n KeeperContainerService.getAllOverloadKeepercontainer()\n .then(function (result) {\n if (Array.isArray(result))\n $scope.overloadKeeperContainer = result;\n $scope.tableParams = new NgTableParams({\n page: 1,\n count: 10,\n }, {\n filterDelay: 100,\n counts: [10, 25, 50],\n dataset: $scope.overloadKeeperContainer\n });\n });\n function beginToMigrateOverloadKeeperContainers() {\n $scope.migratingKeeperContainers = $scope.overloadKeeperContainer.filter(function (keeperContainer) {\n return keeperContainer.selected;\n });\n $scope.tableParams = new NgTableParams({}, {});\n $scope.migratingTableParams = new NgTableParams({\n page: 1,\n count: 10,\n }, {\n filterDelay: 100,\n counts: [10, 25, 50],\n dataset: $scope.migratingKeeperContainers\n });\n $scope.operateType = OPERATE_TYPE.MIGRATING;\n KeeperContainerService.beginToMigrateOverloadKeeperContainers.apply(KeeperContainerService, $scope.migratingKeeperContainers)\n .then(function (result) {\n if (result.message == 'success') {\n toastr.success(\"迁移成功\");\n }\n else {\n toastr.error(result.message, \"迁移失败\");\n }\n getOverloadKeeperContainerMigrationProcess();\n $interval.cancel($scope.scheduledWork);\n });\n }\n function getOverloadKeeperContainerMigrationProcess() {\n if ($scope.operateType == OPERATE_TYPE.MIGRATING) {\n KeeperContainerService.getOverloadKeeperContainerMigrationProcess()\n .then(function (result) {\n if (result == null)\n return;\n $scope.migratingKeeperContainers = result;\n $scope.migratingTableParams = new NgTableParams({\n page: 1,\n count: 10,\n }, {\n filterDelay: 100,\n counts: [10, 25, 50],\n dataset: $scope.migratingKeeperContainers\n });\n });\n }\n }\n $scope.scheduledWork = $interval(getOverloadKeeperContainerMigrationProcess, 1000);\n function toggleAll() {\n $scope.selectAll = !$scope.selectAll;\n $scope.overloadKeeperContainer.forEach(function (keeperContainer) {\n keeperContainer.selected = !keeperContainer.selected;\n });\n }\n function isChecked() {\n return $scope.selectAll;\n }\n}\n\n\n//# sourceURL=webpack://XPipe-Console/./scripts/controllers/KeepercontainerOverloadCtl.ts?"); + +/***/ }), + /***/ "./scripts/controllers/ProxyChainCtl.ts": /*!**********************************************!*\ !*** ./scripts/controllers/ProxyChainCtl.ts ***! @@ -9313,7 +9323,7 @@ eval("/** 确认框 */\nangular\n .module('directive')\n .directive('xpip \***************************/ /***/ (() => { -eval("angular\n .module('index')\n .config(router)\n .config(['$locationProvider', function ($locationProvider) {\n $locationProvider.hashPrefix('');\n }]);\nfunction router($stateProvider, $urlRouterProvider) {\n $urlRouterProvider.otherwise(\"/cluster_list\");\n $stateProvider\n .state('cluster_shards', {\n url: '/cluster_shards?clusterName',\n templateUrl: 'views/index/cluster_shards.html',\n controller: 'ClusterShardCtl'\n })\n .state('cluster_dc_shards', {\n url: '/cluster_dc_shards/:clusterName/:currentDcName',\n params: {\n clusterName: {\n value: '',\n squash: false\n },\n currentDcName: {\n value: '',\n squash: false\n }\n },\n templateUrl: 'views/index/cluster_dc_shards.html',\n controller: 'ClusterCtl'\n })\n .state('dc_list', {\n url: '/dc_list?dcName',\n templateUrl: 'views/index/dc_list.html',\n controller: 'DcListCtl'\n })\n .state('cluster_dc_proxy_chains', {\n url: '/chain/:clusterName/:currentDcName',\n params: {\n clusterName: {\n value: '',\n squash: false\n },\n currentDcName: {\n value: '',\n squash: false\n }\n },\n templateUrl: 'views/index/cluster_proxy_chain.html',\n controller: 'ProxyChainCtl'\n })\n .state('proxy_tunnels', {\n url: '/proxy/:proxyIp/:dcId',\n params: {\n proxyIp: {\n value: '',\n squash: false\n },\n dcId: {\n value: '',\n squash: false\n }\n },\n templateUrl: 'views/index/proxy_tunnel.html',\n controller: 'TunnelsCtl'\n })\n .state('proxy_pings', {\n url: '/proxy/pings',\n params: {},\n templateUrl: 'views/index/proxy_ping.html',\n controller: 'ProxyPingCtl'\n })\n .state('proxy_overview', {\n url: '/proxy/overview',\n params: {},\n templateUrl: 'views/index/proxy_list.html',\n controller: 'ProxyListCtl'\n })\n .state('route_overview', {\n url: '/route/overview?srcDcName&dstDcName',\n params: {},\n templateUrl: 'views/index/route_list.html',\n controller: 'RouteListCtl'\n })\n .state('route_form', {\n url: '/route_form?routeId&type',\n templateUrl: 'views/index/route_form.html',\n controller: 'RouteFormCtl'\n })\n .state('route_direction', {\n url: '/route_direction/route',\n templateUrl: 'views/index/route_direction.html',\n controller: 'RouteDirectionCtl'\n })\n .state('route_switch', {\n url: '/route_switch?tag&srcDcName&dstDcName',\n templateUrl: 'views/index/route_switch.html',\n controller: 'RouteSwitchCtl'\n })\n .state('cluster_routes', {\n url: '/cluster_routes?clusterName&dcName',\n templateUrl: 'views/index/cluster_routes.html',\n controller: 'ClusterRoutesCtl'\n })\n .state('cluster_designated_routes_update', {\n url: '/cluster_designated_routes_update?clusterName&srcDcName',\n templateUrl: 'views/index/cluster_designated_routes_update.html',\n controller: 'ClusterDesignatedRoutesUpdateCtl'\n })\n .state('cluster_dc_shard_update', {\n url: '/cluster_dc_shard_update?clusterName&shardName¤tDcName&srcDcName',\n templateUrl: 'views/index/cluster_dc_shard_update.html',\n controller: 'ClusterDcShardUpdateCtl'\n })\n .state('cluster_dc', {\n url: '/cluster_dc?clusterName',\n templateUrl: 'views/index/cluster_dc.html',\n controller: 'ClusterDcCtl'\n })\n .state('cluster_list', {\n url: '/cluster_list?clusterName&dcName&type&clusterType?keepercontainer',\n templateUrl: 'views/index/cluster_list.html',\n controller: 'ClusterListCtl'\n })\n .state('shard_list', {\n url: '/shard_list',\n templateUrl: 'views/index/shard_list.html',\n controller: 'ShardListCtl'\n })\n .state('cluster_form', {\n url: '/cluster_form?clusterName&type',\n templateUrl: 'views/index/cluster_form.html',\n controller: 'ClusterFromCtl'\n })\n .state('migration_index', {\n url: '/active_dc_migration?:clusterName',\n params: {\n clusters: {\n value: [],\n squash: false\n }\n },\n templateUrl: 'views/index/migration_index.html',\n controller: 'ActiveDcMigrationIndexCtl'\n })\n .state('migration_event_list', {\n url: '/migration_event_list?clusterName',\n templateUrl: 'views/index/migration_list.html',\n controller: 'ActiveDcMigrationEventListCtl'\n })\n .state('migration_event_details', {\n url: '/migration_event_details/:eventId',\n params: {\n eventId: {\n value: '',\n squash: false\n }\n },\n templateUrl: 'views/index/migration_details.html',\n controller: 'ActiveDcMigrationEventDetailsCtl'\n })\n .state('migration_event_details.details', {\n url: '/details',\n params: {\n migrationCluster: {\n value: {},\n squash: false\n }\n },\n templateUrl: 'views/index/migration_details_content.html',\n controller: 'ActiveDcMigrationEventDetailsContentCtl'\n })\n .state('repl_direction_list', {\n url: '/repl_directions',\n templateUrl: 'views/index/repl_direction_list.html',\n controller: 'ReplDirectionListCtl',\n })\n .state('keepercontainer_list', {\n url: '/keepercontainers',\n templateUrl: 'views/index/keepercontainer_list.html',\n controller: 'KeeperContainerListCtl',\n })\n .state('keepercontainer_form', {\n url: '/keepercontainer_form?id&type',\n templateUrl: 'views/index/keepercontainer_form.html',\n controller: 'KeepercontainerFormCtl'\n })\n .state('keeper_migration', {\n url: '/keeper_migration?keepercontainer',\n templateUrl: 'views/index/keeper_migration.html',\n controller: 'KeeperMigrationCtl'\n })\n .state('appliercontainer_list', {\n url: '/appliercontainers',\n templateUrl: 'views/index/appliercontainer_list.html',\n controller: 'AppliercontainerListCtl',\n })\n .state('appliercontainer_form', {\n url: '/appliercontainer_form?id&type',\n templateUrl: 'views/index/appliercontainer_form.html',\n controller: 'AppliercontainerFormCtl'\n });\n}\n\n\n//# sourceURL=webpack://XPipe-Console/./scripts/router.ts?"); +eval("angular\n .module('index')\n .config(router)\n .config(['$locationProvider', function ($locationProvider) {\n $locationProvider.hashPrefix('');\n }]);\nfunction router($stateProvider, $urlRouterProvider) {\n $urlRouterProvider.otherwise(\"/cluster_list\");\n $stateProvider\n .state('cluster_shards', {\n url: '/cluster_shards?clusterName',\n templateUrl: 'views/index/cluster_shards.html',\n controller: 'ClusterShardCtl'\n })\n .state('cluster_dc_shards', {\n url: '/cluster_dc_shards/:clusterName/:currentDcName',\n params: {\n clusterName: {\n value: '',\n squash: false\n },\n currentDcName: {\n value: '',\n squash: false\n }\n },\n templateUrl: 'views/index/cluster_dc_shards.html',\n controller: 'ClusterCtl'\n })\n .state('dc_list', {\n url: '/dc_list?dcName',\n templateUrl: 'views/index/dc_list.html',\n controller: 'DcListCtl'\n })\n .state('cluster_dc_proxy_chains', {\n url: '/chain/:clusterName/:currentDcName',\n params: {\n clusterName: {\n value: '',\n squash: false\n },\n currentDcName: {\n value: '',\n squash: false\n }\n },\n templateUrl: 'views/index/cluster_proxy_chain.html',\n controller: 'ProxyChainCtl'\n })\n .state('proxy_tunnels', {\n url: '/proxy/:proxyIp/:dcId',\n params: {\n proxyIp: {\n value: '',\n squash: false\n },\n dcId: {\n value: '',\n squash: false\n }\n },\n templateUrl: 'views/index/proxy_tunnel.html',\n controller: 'TunnelsCtl'\n })\n .state('proxy_pings', {\n url: '/proxy/pings',\n params: {},\n templateUrl: 'views/index/proxy_ping.html',\n controller: 'ProxyPingCtl'\n })\n .state('proxy_overview', {\n url: '/proxy/overview',\n params: {},\n templateUrl: 'views/index/proxy_list.html',\n controller: 'ProxyListCtl'\n })\n .state('route_overview', {\n url: '/route/overview?srcDcName&dstDcName',\n params: {},\n templateUrl: 'views/index/route_list.html',\n controller: 'RouteListCtl'\n })\n .state('route_form', {\n url: '/route_form?routeId&type',\n templateUrl: 'views/index/route_form.html',\n controller: 'RouteFormCtl'\n })\n .state('route_direction', {\n url: '/route_direction/route',\n templateUrl: 'views/index/route_direction.html',\n controller: 'RouteDirectionCtl'\n })\n .state('route_switch', {\n url: '/route_switch?tag&srcDcName&dstDcName',\n templateUrl: 'views/index/route_switch.html',\n controller: 'RouteSwitchCtl'\n })\n .state('cluster_routes', {\n url: '/cluster_routes?clusterName&dcName',\n templateUrl: 'views/index/cluster_routes.html',\n controller: 'ClusterRoutesCtl'\n })\n .state('cluster_designated_routes_update', {\n url: '/cluster_designated_routes_update?clusterName&srcDcName',\n templateUrl: 'views/index/cluster_designated_routes_update.html',\n controller: 'ClusterDesignatedRoutesUpdateCtl'\n })\n .state('cluster_dc_shard_update', {\n url: '/cluster_dc_shard_update?clusterName&shardName¤tDcName&srcDcName',\n templateUrl: 'views/index/cluster_dc_shard_update.html',\n controller: 'ClusterDcShardUpdateCtl'\n })\n .state('cluster_dc', {\n url: '/cluster_dc?clusterName',\n templateUrl: 'views/index/cluster_dc.html',\n controller: 'ClusterDcCtl'\n })\n .state('cluster_list', {\n url: '/cluster_list?clusterName&dcName&type&clusterType?keepercontainer',\n templateUrl: 'views/index/cluster_list.html',\n controller: 'ClusterListCtl'\n })\n .state('shard_list', {\n url: '/shard_list',\n templateUrl: 'views/index/shard_list.html',\n controller: 'ShardListCtl'\n })\n .state('cluster_form', {\n url: '/cluster_form?clusterName&type',\n templateUrl: 'views/index/cluster_form.html',\n controller: 'ClusterFromCtl'\n })\n .state('migration_index', {\n url: '/active_dc_migration?:clusterName',\n params: {\n clusters: {\n value: [],\n squash: false\n }\n },\n templateUrl: 'views/index/migration_index.html',\n controller: 'ActiveDcMigrationIndexCtl'\n })\n .state('migration_event_list', {\n url: '/migration_event_list?clusterName',\n templateUrl: 'views/index/migration_list.html',\n controller: 'ActiveDcMigrationEventListCtl'\n })\n .state('migration_event_details', {\n url: '/migration_event_details/:eventId',\n params: {\n eventId: {\n value: '',\n squash: false\n }\n },\n templateUrl: 'views/index/migration_details.html',\n controller: 'ActiveDcMigrationEventDetailsCtl'\n })\n .state('migration_event_details.details', {\n url: '/details',\n params: {\n migrationCluster: {\n value: {},\n squash: false\n }\n },\n templateUrl: 'views/index/migration_details_content.html',\n controller: 'ActiveDcMigrationEventDetailsContentCtl'\n })\n .state('repl_direction_list', {\n url: '/repl_directions',\n templateUrl: 'views/index/repl_direction_list.html',\n controller: 'ReplDirectionListCtl',\n })\n .state('keepercontainer_list', {\n url: '/keepercontainers',\n templateUrl: 'views/index/keepercontainer_list.html',\n controller: 'KeeperContainerListCtl',\n })\n .state('keepercontainer_form', {\n url: '/keepercontainer_form?id&type',\n templateUrl: 'views/index/keepercontainer_form.html',\n controller: 'KeepercontainerFormCtl'\n })\n .state('keeper_migration', {\n url: '/keeper_migration?keepercontainer',\n templateUrl: 'views/index/keeper_migration.html',\n controller: 'KeeperMigrationCtl'\n })\n .state('keeper_overload', {\n url: '/keepercontainer_overload?type',\n templateUrl: 'views/index/keepercontainer_overload.html',\n controller: 'KeepercontainerOverloadCtl'\n })\n .state('appliercontainer_list', {\n url: '/appliercontainers',\n templateUrl: 'views/index/appliercontainer_list.html',\n controller: 'AppliercontainerListCtl',\n })\n .state('appliercontainer_form', {\n url: '/appliercontainer_form?id&type',\n templateUrl: 'views/index/appliercontainer_form.html',\n controller: 'AppliercontainerFormCtl'\n });\n}\n\n\n//# sourceURL=webpack://XPipe-Console/./scripts/router.ts?"); /***/ }), @@ -9403,7 +9413,7 @@ eval("angular\n .module('services')\n .service('HealthCheckService', Healt \****************************************************/ /***/ (() => { -eval("angular\n .module('services')\n .service('KeeperContainerService', ['$resource', '$q', function ($resource, $q) {\n var resource = $resource('', {}, {\n find_availablekeepers_by_dc: {\n method: 'POST',\n url: '/console/dcs/:dcName/availablekeepers',\n isArray: true\n },\n find_active_kcs_by_dc_and_cluster: {\n method: 'GET',\n url: '/console/dcs/:dcName/cluster/:clusterName/activekeepercontainers',\n isArray: true\n },\n find_keepercontainer_by_id: {\n method: 'GET',\n url: '/console/keepercontainer/:id',\n },\n find_available_keepers_by_dc_az_and_org: {\n method: 'GET',\n url: '/console/keepercontainers/dc/:dcName/az/:azName/org/:orgName',\n isArray: true\n },\n get_all_infos: {\n method: 'GET',\n url: '/console/keepercontainer/infos/all',\n isArray: true\n },\n get_all_organizations: {\n method: 'GET',\n url: '/console/organizations',\n isArray: true\n },\n add_keepercontainer: {\n method: 'POST',\n url: '/console/keepercontainer'\n },\n update_keepercontainer: {\n method: 'PUT',\n url: '/console/keepercontainer'\n }\n });\n function findActiveKeeperContainersByDc(dcName) {\n var d = $q.defer();\n resource.find_activekeepercontainers_by_dc({\n dcName: dcName\n }, function (result) {\n d.resolve(result);\n }, function (result) {\n d.reject(result);\n });\n return d.promise;\n }\n function findAvailableKeepersByDc(dcName, shard) {\n var d = $q.defer();\n resource.find_availablekeepers_by_dc({\n dcName: dcName\n }, shard, function (result) {\n d.resolve(result);\n }, function (result) {\n d.reject(result);\n });\n return d.promise;\n }\n function findKeepercontainerById(id) {\n var d = $q.defer();\n resource.find_keepercontainer_by_id({\n id: id\n }, function (result) {\n d.resolve(result);\n }, function (result) {\n d.reject(result);\n });\n return d.promise;\n }\n function findAvailableKeepersByDcAzAndOrg(dcName, azName, orgName) {\n var d = $q.defer();\n resource.find_available_keepers_by_dc_az_and_org({\n dcName: dcName,\n azName: azName,\n orgName: orgName\n }, function (result) {\n d.resolve(result);\n }, function (result) {\n d.reject(result);\n });\n return d.promise;\n }\n function findAvailableKeepersByDcAndCluster(dcName, clusterName) {\n var d = $q.defer();\n resource.find_active_kcs_by_dc_and_cluster({\n dcName: dcName,\n clusterName: clusterName\n }, function (result) {\n d.resolve(result);\n }, function (result) {\n d.reject(result);\n });\n return d.promise;\n }\n function getAllInfos() {\n var d = $q.defer();\n resource.get_all_infos({}, function (result) {\n d.resolve(result);\n }, function (result) {\n d.reject(result);\n });\n return d.promise;\n }\n function getAllOrganizations() {\n var d = $q.defer();\n resource.get_all_organizations({}, function (result) {\n d.resolve(result);\n }, function (result) {\n d.reject(result);\n });\n return d.promise;\n }\n function addKeepercontainer(addr, dcName, orgName, azName, active) {\n var d = $q.defer();\n resource.add_keepercontainer({}, {\n addr: addr,\n dcName: dcName,\n orgName: orgName,\n azName: azName,\n active: active\n }, function (result) {\n d.resolve(result);\n }, function (result) {\n d.reject(result);\n });\n return d.promise;\n }\n function updateKeepercontainer(addr, dcName, orgName, azName, active) {\n var d = $q.defer();\n resource.update_keepercontainer({}, {\n addr: addr,\n dcName: dcName,\n orgName: orgName,\n azName: azName,\n active: active\n }, function (result) {\n d.resolve(result);\n }, function (result) {\n d.reject(result);\n });\n return d.promise;\n }\n return {\n findAvailableKeepersByDc: findAvailableKeepersByDc,\n findAvailableKeepersByDcAndCluster: findAvailableKeepersByDcAndCluster,\n findKeepercontainerById: findKeepercontainerById,\n findAvailableKeepersByDcAzAndOrg: findAvailableKeepersByDcAzAndOrg,\n getAllInfos: getAllInfos,\n getAllOrganizations: getAllOrganizations,\n addKeepercontainer: addKeepercontainer,\n updateKeepercontainer: updateKeepercontainer,\n };\n }]);\n\n\n//# sourceURL=webpack://XPipe-Console/./scripts/services/KeeperContainerService.ts?"); +eval("angular\n .module('services')\n .service('KeeperContainerService', ['$resource', '$q', function ($resource, $q) {\n var resource = $resource('', {}, {\n find_availablekeepers_by_dc: {\n method: 'POST',\n url: '/console/dcs/:dcName/availablekeepers',\n isArray: true\n },\n find_active_kcs_by_dc_and_cluster: {\n method: 'GET',\n url: '/console/dcs/:dcName/cluster/:clusterName/activekeepercontainers',\n isArray: true\n },\n find_keepercontainer_by_id: {\n method: 'GET',\n url: '/console/keepercontainer/:id',\n },\n find_available_keepers_by_dc_az_and_org: {\n method: 'GET',\n url: '/console/keepercontainers/dc/:dcName/az/:azName/org/:orgName',\n isArray: true\n },\n get_all_infos: {\n method: 'GET',\n url: '/console/keepercontainer/infos/all',\n isArray: true\n },\n get_all_organizations: {\n method: 'GET',\n url: '/console/organizations',\n isArray: true\n },\n add_keepercontainer: {\n method: 'POST',\n url: '/console/keepercontainer'\n },\n update_keepercontainer: {\n method: 'PUT',\n url: '/console/keepercontainer'\n },\n get_all_overload_keepercontainer: {\n method: 'GET',\n url: '/console/keepercontainers/overload/all',\n isArray: true\n },\n get_overload_keepercontainer_migration_process: {\n method: 'GET',\n url: '/console/keepercontainer/overload/migration/process',\n isArray: true\n },\n begin_to_migrate_overload_keepercontainer: {\n method: 'POST',\n url: '/console/keepercontainer/overload/migration/begin'\n },\n stop_to_migrate_overload_keepercontainer: {\n method: 'POST',\n url: '/console/keepercontainer/overload/migration/stop'\n }\n });\n function findActiveKeeperContainersByDc(dcName) {\n var d = $q.defer();\n resource.find_activekeepercontainers_by_dc({\n dcName: dcName\n }, function (result) {\n d.resolve(result);\n }, function (result) {\n d.reject(result);\n });\n return d.promise;\n }\n function findAvailableKeepersByDc(dcName, shard) {\n var d = $q.defer();\n resource.find_availablekeepers_by_dc({\n dcName: dcName\n }, shard, function (result) {\n d.resolve(result);\n }, function (result) {\n d.reject(result);\n });\n return d.promise;\n }\n function findKeepercontainerById(id) {\n var d = $q.defer();\n resource.find_keepercontainer_by_id({\n id: id\n }, function (result) {\n d.resolve(result);\n }, function (result) {\n d.reject(result);\n });\n return d.promise;\n }\n function findAvailableKeepersByDcAzAndOrg(dcName, azName, orgName) {\n var d = $q.defer();\n resource.find_available_keepers_by_dc_az_and_org({\n dcName: dcName,\n azName: azName,\n orgName: orgName\n }, function (result) {\n d.resolve(result);\n }, function (result) {\n d.reject(result);\n });\n return d.promise;\n }\n function findAvailableKeepersByDcAndCluster(dcName, clusterName) {\n var d = $q.defer();\n resource.find_active_kcs_by_dc_and_cluster({\n dcName: dcName,\n clusterName: clusterName\n }, function (result) {\n d.resolve(result);\n }, function (result) {\n d.reject(result);\n });\n return d.promise;\n }\n function getAllInfos() {\n var d = $q.defer();\n resource.get_all_infos({}, function (result) {\n d.resolve(result);\n }, function (result) {\n d.reject(result);\n });\n return d.promise;\n }\n function getAllOrganizations() {\n var d = $q.defer();\n resource.get_all_organizations({}, function (result) {\n d.resolve(result);\n }, function (result) {\n d.reject(result);\n });\n return d.promise;\n }\n function addKeepercontainer(addr, dcName, orgName, azName, active) {\n var d = $q.defer();\n resource.add_keepercontainer({}, {\n addr: addr,\n dcName: dcName,\n orgName: orgName,\n azName: azName,\n active: active\n }, function (result) {\n d.resolve(result);\n }, function (result) {\n d.reject(result);\n });\n return d.promise;\n }\n function updateKeepercontainer(addr, dcName, orgName, azName, active) {\n var d = $q.defer();\n resource.update_keepercontainer({}, {\n addr: addr,\n dcName: dcName,\n orgName: orgName,\n azName: azName,\n active: active\n }, function (result) {\n d.resolve(result);\n }, function (result) {\n d.reject(result);\n });\n return d.promise;\n }\n function getAllOverloadKeepercontainer() {\n var d = $q.defer();\n resource.get_all_overload_keepercontainer({}, function (result) {\n d.resolve(result);\n }, function (result) {\n d.reject(result);\n });\n return d.promise;\n }\n function getOverloadKeeperContainerMigrationProcess() {\n var d = $q.defer();\n resource.get_overload_keepercontainer_migration_process({}, function (result) {\n d.resolve(result);\n }, function (result) {\n d.reject(result);\n });\n return d.promise;\n }\n function beginToMigrateOverloadKeeperContainers() {\n var d = $q.defer();\n resource.begin_to_migrate_overload_keepercontainer(Array.from(arguments), function (result) {\n d.resolve(result);\n }, function (result) {\n d.reject(result);\n });\n return d.promise;\n }\n return {\n findAvailableKeepersByDc: findAvailableKeepersByDc,\n findAvailableKeepersByDcAndCluster: findAvailableKeepersByDcAndCluster,\n findKeepercontainerById: findKeepercontainerById,\n findAvailableKeepersByDcAzAndOrg: findAvailableKeepersByDcAzAndOrg,\n getAllInfos: getAllInfos,\n getAllOrganizations: getAllOrganizations,\n addKeepercontainer: addKeepercontainer,\n updateKeepercontainer: updateKeepercontainer,\n getAllOverloadKeepercontainer: getAllOverloadKeepercontainer,\n getOverloadKeeperContainerMigrationProcess: getOverloadKeeperContainerMigrationProcess,\n beginToMigrateOverloadKeeperContainers: beginToMigrateOverloadKeeperContainers,\n };\n }]);\n\n\n//# sourceURL=webpack://XPipe-Console/./scripts/services/KeeperContainerService.ts?"); /***/ }), @@ -9544,7 +9554,7 @@ eval("var __WEBPACK_AMD_DEFINE_FACTORY__, __WEBPACK_AMD_DEFINE_RESULT__;/**\n* @ \******************************************************/ /***/ ((module, __unused_webpack_exports, __webpack_require__) => { -eval("var map = {\n\t\"./ActiveDcMigrationEventDetailsContentCtl.ts\": \"./scripts/controllers/ActiveDcMigrationEventDetailsContentCtl.ts\",\n\t\"./ActiveDcMigrationEventDetailsCtl.ts\": \"./scripts/controllers/ActiveDcMigrationEventDetailsCtl.ts\",\n\t\"./ActiveDcMigrationEventListCtl.ts\": \"./scripts/controllers/ActiveDcMigrationEventListCtl.ts\",\n\t\"./ActiveDcMigrationIndexCtl.ts\": \"./scripts/controllers/ActiveDcMigrationIndexCtl.ts\",\n\t\"./AppliercontainerFormCtl.ts\": \"./scripts/controllers/AppliercontainerFormCtl.ts\",\n\t\"./AppliercontainerListCtl.ts\": \"./scripts/controllers/AppliercontainerListCtl.ts\",\n\t\"./ClusterDcCtl.ts\": \"./scripts/controllers/ClusterDcCtl.ts\",\n\t\"./ClusterDcShardCtl.ts\": \"./scripts/controllers/ClusterDcShardCtl.ts\",\n\t\"./ClusterDcShardUpdateCtl.ts\": \"./scripts/controllers/ClusterDcShardUpdateCtl.ts\",\n\t\"./ClusterDesignatedRoutesUpdateCtl.ts\": \"./scripts/controllers/ClusterDesignatedRoutesUpdateCtl.ts\",\n\t\"./ClusterFormCtl.ts\": \"./scripts/controllers/ClusterFormCtl.ts\",\n\t\"./ClusterListCtl.ts\": \"./scripts/controllers/ClusterListCtl.ts\",\n\t\"./ClusterRoutesCtl.ts\": \"./scripts/controllers/ClusterRoutesCtl.ts\",\n\t\"./ClusterShardsCtl.ts\": \"./scripts/controllers/ClusterShardsCtl.ts\",\n\t\"./DcListCtl.ts\": \"./scripts/controllers/DcListCtl.ts\",\n\t\"./IndexCtl.ts\": \"./scripts/controllers/IndexCtl.ts\",\n\t\"./KeeperContainerListCtl.ts\": \"./scripts/controllers/KeeperContainerListCtl.ts\",\n\t\"./KeeperMigrationCtl.ts\": \"./scripts/controllers/KeeperMigrationCtl.ts\",\n\t\"./KeepercontainerFormCtl.ts\": \"./scripts/controllers/KeepercontainerFormCtl.ts\",\n\t\"./ProxyChainCtl.ts\": \"./scripts/controllers/ProxyChainCtl.ts\",\n\t\"./ProxyListCtl.ts\": \"./scripts/controllers/ProxyListCtl.ts\",\n\t\"./ProxyPingCtl.ts\": \"./scripts/controllers/ProxyPingCtl.ts\",\n\t\"./ReplDirectionListCtl.ts\": \"./scripts/controllers/ReplDirectionListCtl.ts\",\n\t\"./RouteDirectionCtl.ts\": \"./scripts/controllers/RouteDirectionCtl.ts\",\n\t\"./RouteFormCtl.ts\": \"./scripts/controllers/RouteFormCtl.ts\",\n\t\"./RouteListCtl.ts\": \"./scripts/controllers/RouteListCtl.ts\",\n\t\"./RouteSwitchCtl.ts\": \"./scripts/controllers/RouteSwitchCtl.ts\",\n\t\"./ShardListCtl.ts\": \"./scripts/controllers/ShardListCtl.ts\",\n\t\"./TunnelsCtl.ts\": \"./scripts/controllers/TunnelsCtl.ts\"\n};\n\n\nfunction webpackContext(req) {\n\tvar id = webpackContextResolve(req);\n\treturn __webpack_require__(id);\n}\nfunction webpackContextResolve(req) {\n\tif(!__webpack_require__.o(map, req)) {\n\t\tvar e = new Error(\"Cannot find module '\" + req + \"'\");\n\t\te.code = 'MODULE_NOT_FOUND';\n\t\tthrow e;\n\t}\n\treturn map[req];\n}\nwebpackContext.keys = function webpackContextKeys() {\n\treturn Object.keys(map);\n};\nwebpackContext.resolve = webpackContextResolve;\nmodule.exports = webpackContext;\nwebpackContext.id = \"./scripts/controllers sync \\\\.ts$\";\n\n//# sourceURL=webpack://XPipe-Console/./scripts/controllers/_sync_nonrecursive_\\.ts$?"); +eval("var map = {\n\t\"./ActiveDcMigrationEventDetailsContentCtl.ts\": \"./scripts/controllers/ActiveDcMigrationEventDetailsContentCtl.ts\",\n\t\"./ActiveDcMigrationEventDetailsCtl.ts\": \"./scripts/controllers/ActiveDcMigrationEventDetailsCtl.ts\",\n\t\"./ActiveDcMigrationEventListCtl.ts\": \"./scripts/controllers/ActiveDcMigrationEventListCtl.ts\",\n\t\"./ActiveDcMigrationIndexCtl.ts\": \"./scripts/controllers/ActiveDcMigrationIndexCtl.ts\",\n\t\"./AppliercontainerFormCtl.ts\": \"./scripts/controllers/AppliercontainerFormCtl.ts\",\n\t\"./AppliercontainerListCtl.ts\": \"./scripts/controllers/AppliercontainerListCtl.ts\",\n\t\"./ClusterDcCtl.ts\": \"./scripts/controllers/ClusterDcCtl.ts\",\n\t\"./ClusterDcShardCtl.ts\": \"./scripts/controllers/ClusterDcShardCtl.ts\",\n\t\"./ClusterDcShardUpdateCtl.ts\": \"./scripts/controllers/ClusterDcShardUpdateCtl.ts\",\n\t\"./ClusterDesignatedRoutesUpdateCtl.ts\": \"./scripts/controllers/ClusterDesignatedRoutesUpdateCtl.ts\",\n\t\"./ClusterFormCtl.ts\": \"./scripts/controllers/ClusterFormCtl.ts\",\n\t\"./ClusterListCtl.ts\": \"./scripts/controllers/ClusterListCtl.ts\",\n\t\"./ClusterRoutesCtl.ts\": \"./scripts/controllers/ClusterRoutesCtl.ts\",\n\t\"./ClusterShardsCtl.ts\": \"./scripts/controllers/ClusterShardsCtl.ts\",\n\t\"./DcListCtl.ts\": \"./scripts/controllers/DcListCtl.ts\",\n\t\"./IndexCtl.ts\": \"./scripts/controllers/IndexCtl.ts\",\n\t\"./KeeperContainerListCtl.ts\": \"./scripts/controllers/KeeperContainerListCtl.ts\",\n\t\"./KeeperMigrationCtl.ts\": \"./scripts/controllers/KeeperMigrationCtl.ts\",\n\t\"./KeepercontainerFormCtl.ts\": \"./scripts/controllers/KeepercontainerFormCtl.ts\",\n\t\"./KeepercontainerOverloadCtl.ts\": \"./scripts/controllers/KeepercontainerOverloadCtl.ts\",\n\t\"./ProxyChainCtl.ts\": \"./scripts/controllers/ProxyChainCtl.ts\",\n\t\"./ProxyListCtl.ts\": \"./scripts/controllers/ProxyListCtl.ts\",\n\t\"./ProxyPingCtl.ts\": \"./scripts/controllers/ProxyPingCtl.ts\",\n\t\"./ReplDirectionListCtl.ts\": \"./scripts/controllers/ReplDirectionListCtl.ts\",\n\t\"./RouteDirectionCtl.ts\": \"./scripts/controllers/RouteDirectionCtl.ts\",\n\t\"./RouteFormCtl.ts\": \"./scripts/controllers/RouteFormCtl.ts\",\n\t\"./RouteListCtl.ts\": \"./scripts/controllers/RouteListCtl.ts\",\n\t\"./RouteSwitchCtl.ts\": \"./scripts/controllers/RouteSwitchCtl.ts\",\n\t\"./ShardListCtl.ts\": \"./scripts/controllers/ShardListCtl.ts\",\n\t\"./TunnelsCtl.ts\": \"./scripts/controllers/TunnelsCtl.ts\"\n};\n\n\nfunction webpackContext(req) {\n\tvar id = webpackContextResolve(req);\n\treturn __webpack_require__(id);\n}\nfunction webpackContextResolve(req) {\n\tif(!__webpack_require__.o(map, req)) {\n\t\tvar e = new Error(\"Cannot find module '\" + req + \"'\");\n\t\te.code = 'MODULE_NOT_FOUND';\n\t\tthrow e;\n\t}\n\treturn map[req];\n}\nwebpackContext.keys = function webpackContextKeys() {\n\treturn Object.keys(map);\n};\nwebpackContext.resolve = webpackContextResolve;\nmodule.exports = webpackContext;\nwebpackContext.id = \"./scripts/controllers sync \\\\.ts$\";\n\n//# sourceURL=webpack://XPipe-Console/./scripts/controllers/_sync_nonrecursive_\\.ts$?"); /***/ }), diff --git a/redis/redis-console/src/main/resources/static/scripts/controllers/KeepercontainerOverloadCtl.ts b/redis/redis-console/src/main/resources/static/scripts/controllers/KeepercontainerOverloadCtl.ts new file mode 100644 index 000000000..97875a6e3 --- /dev/null +++ b/redis/redis-console/src/main/resources/static/scripts/controllers/KeepercontainerOverloadCtl.ts @@ -0,0 +1,103 @@ +angular + .module('index') + .controller('KeepercontainerOverloadCtl', KeepercontainerOverloadCtl); + +KeepercontainerOverloadCtl.$inject = ['$rootScope', '$scope', '$window', '$stateParams', 'KeeperContainerService', + 'toastr', 'NgTableParams', '$interval']; + +function KeepercontainerOverloadCtl($rootScope, $scope, $window, $stateParams, KeeperContainerService, + toastr, NgTableParams, $interval) { + $scope.overloadKeeperContainer = []; + $scope.tableParams = new NgTableParams({}, {}); + $scope.migratingTableParams = new NgTableParams({}, {}); + $scope.selectAll = false; + $scope.toggleAll = toggleAll; + $scope.isChecked = isChecked; + + var OPERATE_TYPE = { + DETAIL: 'detail', + MIGRATING: 'migrating', + STOPPED : 'stopped' + }; + $scope.operateType = $stateParams.type; + $scope.migratingKeeperContainers = []; + $scope.scheduledWork; + + $scope.beginToMigrateOverloadKeeperContainers = beginToMigrateOverloadKeeperContainers; + + KeeperContainerService.getAllOverloadKeepercontainer() + .then(function (result) { + if (Array.isArray(result)) $scope.overloadKeeperContainer = result; + + $scope.tableParams = new NgTableParams({ + page : 1, + count : 10, + }, { + filterDelay: 100, + counts: [10, 25, 50], + dataset: $scope.overloadKeeperContainer + }); + }); + + + function beginToMigrateOverloadKeeperContainers() { + $scope.migratingKeeperContainers = $scope.overloadKeeperContainer.filter(function(keeperContainer){ + return keeperContainer.selected; + }); + + $scope.tableParams = new NgTableParams({}, {}); + + $scope.migratingTableParams = new NgTableParams({ + page : 1, + count : 10, + }, { + filterDelay: 100, + counts: [10, 25, 50], + dataset: $scope.migratingKeeperContainers + }); + + $scope.operateType = OPERATE_TYPE.MIGRATING; + + KeeperContainerService.beginToMigrateOverloadKeeperContainers.apply(KeeperContainerService, $scope.migratingKeeperContainers) + .then(result => { + if(result.message == 'success' ) { + toastr.success("迁移成功"); + } else { + toastr.error(result.message, "迁移失败"); + } + getOverloadKeeperContainerMigrationProcess(); + $interval.cancel($scope.scheduledWork); + }); + } + + function getOverloadKeeperContainerMigrationProcess() { + if ($scope.operateType == OPERATE_TYPE.MIGRATING) { + KeeperContainerService.getOverloadKeeperContainerMigrationProcess() + .then(function (result) { + if (result == null) return; + $scope.migratingKeeperContainers = result; + $scope.migratingTableParams = new NgTableParams({ + page : 1, + count : 10, + }, { + filterDelay: 100, + counts: [10, 25, 50], + dataset: $scope.migratingKeeperContainers + }); + }); + } + } + + $scope.scheduledWork = $interval(getOverloadKeeperContainerMigrationProcess, 1000); + + function toggleAll() { + $scope.selectAll = !$scope.selectAll; + $scope.overloadKeeperContainer.forEach(function (keeperContainer) { + keeperContainer.selected = !keeperContainer.selected; + }); + } + + function isChecked() { + return $scope.selectAll; + } +} \ No newline at end of file diff --git a/redis/redis-console/src/main/resources/static/scripts/router.ts b/redis/redis-console/src/main/resources/static/scripts/router.ts index 3d5732630..60c5725ba 100644 --- a/redis/redis-console/src/main/resources/static/scripts/router.ts +++ b/redis/redis-console/src/main/resources/static/scripts/router.ts @@ -199,6 +199,11 @@ function router($stateProvider, $urlRouterProvider) { templateUrl: 'views/index/keeper_migration.html', controller: 'KeeperMigrationCtl' }) + .state('keeper_overload', { + url: '/keepercontainer_overload?type', + templateUrl: 'views/index/keepercontainer_overload.html', + controller: 'KeepercontainerOverloadCtl' + }) .state('appliercontainer_list', { url: '/appliercontainers', templateUrl: 'views/index/appliercontainer_list.html', diff --git a/redis/redis-console/src/main/resources/static/scripts/services/KeeperContainerService.ts b/redis/redis-console/src/main/resources/static/scripts/services/KeeperContainerService.ts index af20b3547..dd7c2466e 100644 --- a/redis/redis-console/src/main/resources/static/scripts/services/KeeperContainerService.ts +++ b/redis/redis-console/src/main/resources/static/scripts/services/KeeperContainerService.ts @@ -39,6 +39,24 @@ angular update_keepercontainer:{ method:'PUT', url:'/console/keepercontainer' + }, + get_all_overload_keepercontainer: { + method: 'GET', + url: '/console/keepercontainers/overload/all', + isArray: true + }, + get_overload_keepercontainer_migration_process: { + method: 'GET', + url: '/console/keepercontainer/overload/migration/process', + isArray: true + }, + begin_to_migrate_overload_keepercontainer:{ + method:'POST', + url:'/console/keepercontainer/overload/migration/begin' + }, + stop_to_migrate_overload_keepercontainer:{ + method:'POST', + url:'/console/keepercontainer/overload/migration/stop' } }); @@ -166,6 +184,39 @@ angular return d.promise; } + function getAllOverloadKeepercontainer() { + var d = $q.defer(); + resource.get_all_overload_keepercontainer({}, + function (result) { + d.resolve(result); + }, function (result) { + d.reject(result); + }); + return d.promise; + } + + function getOverloadKeeperContainerMigrationProcess() { + var d = $q.defer(); + resource.get_overload_keepercontainer_migration_process({}, + function (result) { + d.resolve(result); + }, function (result) { + d.reject(result); + }); + return d.promise; + } + + function beginToMigrateOverloadKeeperContainers() { + var d = $q.defer(); + resource.begin_to_migrate_overload_keepercontainer( + Array.from(arguments), + function (result) { + d.resolve(result); + }, function (result) { + d.reject(result); + }); + return d.promise; + } return { findAvailableKeepersByDc : findAvailableKeepersByDc, @@ -176,5 +227,8 @@ angular getAllOrganizations: getAllOrganizations, addKeepercontainer: addKeepercontainer, updateKeepercontainer: updateKeepercontainer, + getAllOverloadKeepercontainer : getAllOverloadKeepercontainer, + getOverloadKeeperContainerMigrationProcess : getOverloadKeeperContainerMigrationProcess, + beginToMigrateOverloadKeeperContainers : beginToMigrateOverloadKeeperContainers, } }]); diff --git a/redis/redis-console/src/main/resources/static/views/index/keepercontainer_list.html b/redis/redis-console/src/main/resources/static/views/index/keepercontainer_list.html index bc1ae6ff2..20b05470d 100644 --- a/redis/redis-console/src/main/resources/static/views/index/keepercontainer_list.html +++ b/redis/redis-console/src/main/resources/static/views/index/keepercontainer_list.html @@ -8,7 +8,8 @@
- AddKeepercontainer + OverloadKeeperContainer + AddKeeperContainer
diff --git a/redis/redis-console/src/main/resources/static/views/index/keepercontainer_overload.html b/redis/redis-console/src/main/resources/static/views/index/keepercontainer_overload.html new file mode 100644 index 000000000..3d6cfd5ca --- /dev/null +++ b/redis/redis-console/src/main/resources/static/views/index/keepercontainer_overload.html @@ -0,0 +1,64 @@ + +
+ +

+ OverloadKeeperContainer + >> 迁移中 +

+ +
+
+
+
+
+ 取消全选全选 + +
+ + 迁移 + +
+ + + +
+
+ +
+ + + + + + + + + + + + + +
+ + {{info.srcKeeperContainer.keeperIp}}{{info.srcKeeperContainer.dcName}}{{info.srcKeeperContainer.totalRedisUsedMemory / 1024}}{{info.srcKeeperContainer.totalInputFlow}}{{info.migrateKeeperCount}}{{info.targetKeeperContainer.keeperIp}}{{info.targetKeeperContainer.totalRedisUsedMemory / 1024}}{{info.targetKeeperContainer.totalInputFlow}}
+ + + +

+ + + + + + + + +
{{info.srcKeeperContainer.keeperIp}}{{info.srcKeeperContainer.dcName}}{{info.targetKeeperContainer.keeperIp}}{{info.migrateKeeperCount}}{{info.migrateKeeperCompleteCount}}
+
+ + + + + + + diff --git a/redis/redis-console/src/test/java/com/ctrip/xpipe/redis/console/AllTests.java b/redis/redis-console/src/test/java/com/ctrip/xpipe/redis/console/AllTests.java index eaafeb39c..fb5730849 100644 --- a/redis/redis-console/src/test/java/com/ctrip/xpipe/redis/console/AllTests.java +++ b/redis/redis-console/src/test/java/com/ctrip/xpipe/redis/console/AllTests.java @@ -38,6 +38,8 @@ import com.ctrip.xpipe.redis.console.healthcheck.nonredis.redisconfig.RedisConfigCheckMonitorTest; import com.ctrip.xpipe.redis.console.healthcheck.nonredis.sentinelconfig.SentinelConfigCheckTest; import com.ctrip.xpipe.redis.console.healthcheck.nonredis.unhealthycluster.UnhealthyClusterCheckerTest; +import com.ctrip.xpipe.redis.console.keeper.AutoMigrateOverloadKeeperContainerActionTest; +import com.ctrip.xpipe.redis.console.keeper.impl.DefaultKeeperContainerUsedInfoAnalyzerTest; import com.ctrip.xpipe.redis.console.migration.MigrationShardRollbackTest; import com.ctrip.xpipe.redis.console.migration.MultiClusterMigrationTest; import com.ctrip.xpipe.redis.console.migration.SingleShardMigrationTest; @@ -225,6 +227,10 @@ ClusterTypeUpdateEventListenerTest.class, + DefaultKeeperContainerMigrationServiceTest.class, + AutoMigrateOverloadKeeperContainerActionTest.class, + DefaultKeeperContainerUsedInfoAnalyzerTest.class, + RouteInfoControllerTest.class, RedisControllerTest.class, DcRelationsServiceTest.class, diff --git a/redis/redis-console/src/test/java/com/ctrip/xpipe/redis/console/keeper/AutoMigrateOverloadKeeperContainerActionTest.java b/redis/redis-console/src/test/java/com/ctrip/xpipe/redis/console/keeper/AutoMigrateOverloadKeeperContainerActionTest.java new file mode 100644 index 000000000..f9eb7175b --- /dev/null +++ b/redis/redis-console/src/test/java/com/ctrip/xpipe/redis/console/keeper/AutoMigrateOverloadKeeperContainerActionTest.java @@ -0,0 +1,164 @@ +package com.ctrip.xpipe.redis.console.keeper; + +import com.ctrip.xpipe.redis.checker.model.DcClusterShard; +import com.ctrip.xpipe.redis.checker.model.KeeperContainerUsedInfoModel; +import com.ctrip.xpipe.redis.console.model.MigrationKeeperContainerDetailModel; +import com.ctrip.xpipe.redis.console.model.ShardModel; +import com.ctrip.xpipe.redis.console.service.model.ShardModelService; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.Mockito; + +import java.util.ArrayList; +import java.util.List; + +/** + * @author yu + *

+ * 2023/9/27 + */ + +@RunWith(org.mockito.junit.MockitoJUnitRunner.class) +public class AutoMigrateOverloadKeeperContainerActionTest { + + @InjectMocks + AutoMigrateOverloadKeeperContainerAction action; + + @Mock + private ShardModelService shardModelService; + + @Before + public void beforeAutoMigrateOverloadKeeperContainerActionTest() { + ShardModel shardModel = new ShardModel(); + Mockito.when(shardModelService.getShardModel(Mockito.anyString(), Mockito.anyString(), Mockito.anyString(), Mockito.anyBoolean(), Mockito.anyObject())) + .thenReturn(shardModel); + Mockito.when(shardModelService.migrateShardKeepers(Mockito.anyString(), Mockito.anyString(), Mockito.any(), Mockito.anyString(), Mockito.anyString())) + .thenReturn(true); + } + + @Test + public void testMigrateAllKeepersSuccess() { + KeeperContainerUsedInfoModel model1 = new KeeperContainerUsedInfoModel("1.1.1.1", "jq", 14, 14); + KeeperContainerUsedInfoModel model2 = new KeeperContainerUsedInfoModel("2.2.2.2", "jq", 13, 13); + KeeperContainerUsedInfoModel model3 = new KeeperContainerUsedInfoModel("3.3.3.3", "jq", 5, 5); + KeeperContainerUsedInfoModel model4 = new KeeperContainerUsedInfoModel("4.4.4.4", "jq", 6, 6); + + List readyToMigrationKeeperContainers = new ArrayList<>(); + List migrationShards1 = new ArrayList<>(); + migrationShards1.add(new DcClusterShard("jq", "cluster2", "shard2")); + migrationShards1.add(new DcClusterShard("jq", "cluster4", "shard2")); + migrationShards1.add(new DcClusterShard("jq", "cluster5", "shard2")); + MigrationKeeperContainerDetailModel migrationKeeperContainerDetailModel1 = new MigrationKeeperContainerDetailModel() + .setSrcKeeperContainer(model1).setTargetKeeperContainer(model3).setMigrateKeeperCount(3).setMigrateShards(migrationShards1); + readyToMigrationKeeperContainers.add(migrationKeeperContainerDetailModel1); + + List migrationShards2 = new ArrayList<>(); + migrationShards2.add(new DcClusterShard("jq", "cluster14", "shard2")); + migrationShards2.add(new DcClusterShard("jq", "cluster15", "shard2")); + migrationShards2.add(new DcClusterShard("jq", "cluster16", "shard2")); + migrationShards2.add(new DcClusterShard("jq", "cluster17", "shard2")); + MigrationKeeperContainerDetailModel migrationKeeperContainerDetailModel2 = new MigrationKeeperContainerDetailModel() + .setSrcKeeperContainer(model2).setTargetKeeperContainer(model4).setMigrateKeeperCount(4).setMigrateShards(migrationShards2); + readyToMigrationKeeperContainers.add(migrationKeeperContainerDetailModel2); + + action.migrateAllKeepers(readyToMigrationKeeperContainers); + + Assert.assertEquals(3, migrationKeeperContainerDetailModel1.getMigrateKeeperCompleteCount()); + Assert.assertEquals(4, migrationKeeperContainerDetailModel2.getMigrateKeeperCompleteCount()); + } + + @Test + public void testMigrateAllKeepersFail() { + KeeperContainerUsedInfoModel model1 = new KeeperContainerUsedInfoModel("1.1.1.1", "jq", 14, 14); + KeeperContainerUsedInfoModel model2 = new KeeperContainerUsedInfoModel("2.2.2.2", "jq", 13, 13); + KeeperContainerUsedInfoModel model3 = new KeeperContainerUsedInfoModel("3.3.3.3", "jq", 5, 5); + KeeperContainerUsedInfoModel model4 = new KeeperContainerUsedInfoModel("4.4.4.4", "jq", 6, 6); + + List readyToMigrationKeeperContainers = new ArrayList<>(); + List migrationShards1 = new ArrayList<>(); + migrationShards1.add(new DcClusterShard("jq", "cluster2", "shard2")); + migrationShards1.add(new DcClusterShard("jq", "cluster4", "shard2")); + migrationShards1.add(new DcClusterShard("jq", "cluster5", "shard2")); + MigrationKeeperContainerDetailModel migrationKeeperContainerDetailModel1 = new MigrationKeeperContainerDetailModel() + .setSrcKeeperContainer(model1).setTargetKeeperContainer(model3).setMigrateKeeperCount(3).setMigrateShards(migrationShards1); + readyToMigrationKeeperContainers.add(migrationKeeperContainerDetailModel1); + + List migrationShards2 = new ArrayList<>(); + migrationShards2.add(new DcClusterShard("jq", "cluster14", "shard2")); + migrationShards2.add(new DcClusterShard("jq", "cluster15", "shard2")); + migrationShards2.add(new DcClusterShard("jq", "cluster16", "shard2")); + migrationShards2.add(new DcClusterShard("jq", "cluster17", "shard2")); + MigrationKeeperContainerDetailModel migrationKeeperContainerDetailModel2 = new MigrationKeeperContainerDetailModel() + .setSrcKeeperContainer(model2).setTargetKeeperContainer(model4).setMigrateKeeperCount(4).setMigrateShards(migrationShards2); + readyToMigrationKeeperContainers.add(migrationKeeperContainerDetailModel2); + + Mockito.when(shardModelService.migrateShardKeepers(Mockito.anyString(), Mockito.anyString(), Mockito.any(), Mockito.anyString(), Mockito.anyString())).thenReturn(false); + action.migrateAllKeepers(readyToMigrationKeeperContainers); + + Assert.assertEquals(0, migrationKeeperContainerDetailModel1.getMigrateKeeperCompleteCount()); + Assert.assertEquals(0, migrationKeeperContainerDetailModel2.getMigrateKeeperCompleteCount()); + } + + @Test + public void testMigrateAllKeepersWithSameShard() { + KeeperContainerUsedInfoModel model1 = new KeeperContainerUsedInfoModel("1.1.1.1", "jq", 14, 14); + KeeperContainerUsedInfoModel model2 = new KeeperContainerUsedInfoModel("2.2.2.2", "jq", 13, 13); + KeeperContainerUsedInfoModel model3 = new KeeperContainerUsedInfoModel("3.3.3.3", "jq", 5, 5); + KeeperContainerUsedInfoModel model4 = new KeeperContainerUsedInfoModel("4.4.4.4", "jq", 6, 6); + + List readyToMigrationKeeperContainers = new ArrayList<>(); + List migrationShards1 = new ArrayList<>(); + migrationShards1.add(new DcClusterShard("jq", "cluster2", "shard2")); + MigrationKeeperContainerDetailModel migrationKeeperContainerDetailModel1 = new MigrationKeeperContainerDetailModel() + .setSrcKeeperContainer(model1).setTargetKeeperContainer(model3).setMigrateKeeperCount(1).setMigrateShards(migrationShards1); + readyToMigrationKeeperContainers.add(migrationKeeperContainerDetailModel1); + + List migrationShards2 = new ArrayList<>(); + migrationShards2.add(new DcClusterShard("jq", "cluster2", "shard2")); + MigrationKeeperContainerDetailModel migrationKeeperContainerDetailModel2 = new MigrationKeeperContainerDetailModel() + .setSrcKeeperContainer(model2).setTargetKeeperContainer(model4).setMigrateKeeperCount(1).setMigrateShards(migrationShards2); + readyToMigrationKeeperContainers.add(migrationKeeperContainerDetailModel2); + + action.migrateAllKeepers(readyToMigrationKeeperContainers); + + Assert.assertEquals(1, migrationKeeperContainerDetailModel1.getMigrateKeeperCompleteCount()); + Assert.assertEquals(0, migrationKeeperContainerDetailModel2.getMigrateKeeperCompleteCount()); + } + + @Test + public void testMigrateAllKeepersWithEmptyShard() { + KeeperContainerUsedInfoModel model1 = new KeeperContainerUsedInfoModel("1.1.1.1", "jq", 14, 14); + KeeperContainerUsedInfoModel model2 = new KeeperContainerUsedInfoModel("2.2.2.2", "jq", 13, 13); + KeeperContainerUsedInfoModel model3 = new KeeperContainerUsedInfoModel("3.3.3.3", "jq", 5, 5); + KeeperContainerUsedInfoModel model4 = new KeeperContainerUsedInfoModel("4.4.4.4", "jq", 6, 6); + + List readyToMigrationKeeperContainers = new ArrayList<>(); + List migrationShards1 = new ArrayList<>(); + migrationShards1.add(new DcClusterShard("jq", "cluster2", "shard2")); + migrationShards1.add(new DcClusterShard("jq", "cluster4", "shard2")); + migrationShards1.add(new DcClusterShard("jq", "cluster5", "shard2")); + MigrationKeeperContainerDetailModel migrationKeeperContainerDetailModel1 = new MigrationKeeperContainerDetailModel() + .setSrcKeeperContainer(model1).setTargetKeeperContainer(model3).setMigrateKeeperCount(3).setMigrateShards(migrationShards1); + readyToMigrationKeeperContainers.add(migrationKeeperContainerDetailModel1); + + List migrationShards2 = new ArrayList<>(); + migrationShards2.add(new DcClusterShard("jq", "cluster4", "shard2")); + migrationShards2.add(new DcClusterShard("jq", "cluster5", "shard2")); + migrationShards2.add(new DcClusterShard("jq", "cluster16", "shard2")); + migrationShards2.add(new DcClusterShard("jq", "cluster17", "shard2")); + MigrationKeeperContainerDetailModel migrationKeeperContainerDetailModel2 = new MigrationKeeperContainerDetailModel() + .setSrcKeeperContainer(model2).setTargetKeeperContainer(model4).setMigrateKeeperCount(4).setMigrateShards(migrationShards2); + readyToMigrationKeeperContainers.add(migrationKeeperContainerDetailModel2); + + action.migrateAllKeepers(readyToMigrationKeeperContainers); + + Assert.assertEquals(3, migrationKeeperContainerDetailModel1.getMigrateKeeperCompleteCount()); + Assert.assertEquals(2, migrationKeeperContainerDetailModel2.getMigrateKeeperCompleteCount()); + } + + +} \ No newline at end of file diff --git a/redis/redis-console/src/test/java/com/ctrip/xpipe/redis/console/keeper/impl/DefaultKeeperContainerUsedInfoAnalyzerTest.java b/redis/redis-console/src/test/java/com/ctrip/xpipe/redis/console/keeper/impl/DefaultKeeperContainerUsedInfoAnalyzerTest.java new file mode 100644 index 000000000..fa6fb8988 --- /dev/null +++ b/redis/redis-console/src/test/java/com/ctrip/xpipe/redis/console/keeper/impl/DefaultKeeperContainerUsedInfoAnalyzerTest.java @@ -0,0 +1,536 @@ +package com.ctrip.xpipe.redis.console.keeper.impl; + +import com.ctrip.xpipe.api.foundation.FoundationService; +import com.ctrip.xpipe.redis.checker.model.DcClusterShard; +import com.ctrip.xpipe.redis.checker.model.KeeperContainerUsedInfoModel; +import com.ctrip.xpipe.redis.console.config.ConsoleConfig; +import com.ctrip.xpipe.redis.console.model.KeeperContainerOverloadStandardModel; +import com.ctrip.xpipe.redis.console.model.MigrationKeeperContainerDetailModel; +import com.ctrip.xpipe.tuple.Pair; +import com.google.common.collect.Maps; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.Mockito; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; + +/** + * @author yu + *

+ * 2023/9/20 + */ +@RunWith(org.mockito.junit.MockitoJUnitRunner.class) +public class DefaultKeeperContainerUsedInfoAnalyzerTest { + + @InjectMocks + private DefaultKeeperContainerUsedInfoAnalyzer analyzer; + + @Mock + private ConsoleConfig config; + + @Mock + private ThreadPoolExecutor executor; + + @Mock + private FoundationService service; + + + @Before + public void before() { + Mockito.when(config.getClusterDividedParts()).thenReturn(2); + Map standards = Maps.newHashMap(); + standards.put(FoundationService.DEFAULT.getDataCenter(), new KeeperContainerOverloadStandardModel().setFlowOverload(10).setPeerDataOverload(10)); + Mockito.when(config.getKeeperContainerOverloadStandards()).thenReturn(standards); + Mockito.when(config.getKeeperCheckerIntervalMilli()).thenReturn(10 * 1000); + Mockito.doNothing().when(executor).execute(Mockito.any()); + } + + @Test + public void testUpdateKeeperContainerUsedInfo() { + List models1 = new ArrayList<>(); + KeeperContainerUsedInfoModel model1 = new KeeperContainerUsedInfoModel("1.1.1.1", "jq", 14, 14); + Map> detailInfo1 = Maps.newHashMap(); + detailInfo1.put(new DcClusterShard("jq", "cluster1", "shard1"), new Pair<>(2L, 2L)); + detailInfo1.put(new DcClusterShard("jq", "cluster1", "shard2"), new Pair<>(3L, 3L)); + detailInfo1.put(new DcClusterShard("jq", "cluster2", "shard1"), new Pair<>(4L, 4L)); + detailInfo1.put(new DcClusterShard("jq", "cluster2", "shard2"), new Pair<>(5L, 5L)); + model1.setDetailInfo(detailInfo1); + models1.add(model1); + + analyzer.updateKeeperContainerUsedInfo(0, models1); + Assert.assertEquals(1, analyzer.getCheckerIndexes().size()); + + List models2 = new ArrayList<>(); + KeeperContainerUsedInfoModel model3 = new KeeperContainerUsedInfoModel("3.3.3.3", "jq", 5, 5); + Map> detailInfo3 = Maps.newHashMap(); + detailInfo3.put(new DcClusterShard("jq", "cluster3", "shard1"), new Pair<>(2L, 2L)); + detailInfo3.put(new DcClusterShard("jq", "cluster4", "shard2"), new Pair<>(3L, 3L)); + model3.setDetailInfo(detailInfo3); + models2.add(model3); + + analyzer.updateKeeperContainerUsedInfo(1, models2); + Assert.assertEquals(0, analyzer.getCheckerIndexes().size()); + Assert.assertEquals(0, analyzer.getAllKeeperContainerUsedInfoModels().size()); + } + + @Test + public void testUpdateKeeperContainerUsedInfoExpired() throws InterruptedException { + List models1 = new ArrayList<>(); + KeeperContainerUsedInfoModel model1 = new KeeperContainerUsedInfoModel("1.1.1.1", "jq", 14, 14); + Map> detailInfo1 = Maps.newHashMap(); + detailInfo1.put(new DcClusterShard("jq", "cluster1", "shard1"), new Pair<>(2L, 2L)); + detailInfo1.put(new DcClusterShard("jq", "cluster1", "shard2"), new Pair<>(3L, 3L)); + detailInfo1.put(new DcClusterShard("jq", "cluster2", "shard1"), new Pair<>(4L, 4L)); + detailInfo1.put(new DcClusterShard("jq", "cluster2", "shard2"), new Pair<>(5L, 5L)); + model1.setDetailInfo(detailInfo1); + models1.add(model1); + + analyzer.updateKeeperContainerUsedInfo(0, models1); + Assert.assertEquals(1, analyzer.getCheckerIndexes().size()); + + TimeUnit.MILLISECONDS.sleep(11 * 1000); + + List models2 = new ArrayList<>(); + KeeperContainerUsedInfoModel model3 = new KeeperContainerUsedInfoModel("3.3.3.3", "jq", 5, 5); + Map> detailInfo3 = Maps.newHashMap(); + detailInfo3.put(new DcClusterShard("jq", "cluster3", "shard1"), new Pair<>(2L, 2L)); + detailInfo3.put(new DcClusterShard("jq", "cluster4", "shard2"), new Pair<>(3L, 3L)); + model3.setDetailInfo(detailInfo3); + models2.add(model3); + + analyzer.updateKeeperContainerUsedInfo(1, models2); + Assert.assertEquals(1, analyzer.getCheckerIndexes().size()); + Assert.assertEquals(1, analyzer.getAllKeeperContainerUsedInfoModels().size()); + } + + @Test + public void testGetAllDcReadyToMigrationKeeperContainersWithBoth() { + List models1 = new ArrayList<>(); + KeeperContainerUsedInfoModel model1 = new KeeperContainerUsedInfoModel("1.1.1.1", "jq", 14, 14); + Map> detailInfo1 = Maps.newHashMap(); + detailInfo1.put(new DcClusterShard("jq", "cluster1", "shard1"), new Pair<>(2L, 2L)); + detailInfo1.put(new DcClusterShard("jq", "cluster1", "shard2"), new Pair<>(3L, 3L)); + detailInfo1.put(new DcClusterShard("jq", "cluster2", "shard1"), new Pair<>(4L, 4L)); + detailInfo1.put(new DcClusterShard("jq", "cluster2", "shard2"), new Pair<>(5L, 5L)); + model1.setDetailInfo(detailInfo1); + models1.add(model1); + + KeeperContainerUsedInfoModel model2 = new KeeperContainerUsedInfoModel("2.2.2.2", "jq", 13, 13); + Map> detailInfo2 = Maps.newHashMap(); + detailInfo2.put(new DcClusterShard("jq", "cluster3", "shard1"), new Pair<>(2L, 2L)); + detailInfo2.put(new DcClusterShard("jq", "cluster3", "shard2"), new Pair<>(3L, 3L)); + detailInfo2.put(new DcClusterShard("jq", "cluster4", "shard1"), new Pair<>(4L, 4L)); + detailInfo2.put(new DcClusterShard("jq", "cluster4", "shard2"), new Pair<>(3L, 3L)); + model2.setDetailInfo(detailInfo2); + models1.add(model2); + + KeeperContainerUsedInfoModel model3 = new KeeperContainerUsedInfoModel("3.3.3.3", "jq", 5, 5); + Map> detailInfo3 = Maps.newHashMap(); + detailInfo3.put(new DcClusterShard("jq", "cluster3", "shard1"), new Pair<>(2L, 2L)); + detailInfo3.put(new DcClusterShard("jq", "cluster4", "shard2"), new Pair<>(3L, 3L)); + model3.setDetailInfo(detailInfo3); + models1.add(model3); + + + KeeperContainerUsedInfoModel model4 = new KeeperContainerUsedInfoModel("4.4.4.4", "jq", 6, 6); + Map> detailInfo4 = Maps.newHashMap(); + detailInfo3.put(new DcClusterShard("jq", "cluster1", "shard1"), new Pair<>(3L, 3L)); + detailInfo3.put(new DcClusterShard("jq", "cluster2", "shard2"), new Pair<>(3L, 3L)); + model4.setDetailInfo(detailInfo4); + models1.add(model4); + + analyzer.updateKeeperContainerUsedInfo(0, models1); + analyzer.analyzeKeeperContainerUsedInfo(models1); + List allDcReadyToMigrationKeeperContainers = analyzer.getAllDcReadyToMigrationKeeperContainers(); + Assert.assertEquals(2, allDcReadyToMigrationKeeperContainers.size()); + } + + @Test + public void testMultiSrcKeeperSingleTargetWithBoth() { + List models1 = new ArrayList<>(); + KeeperContainerUsedInfoModel model1 = new KeeperContainerUsedInfoModel("1.1.1.1", "jq", 14, 14); + Map> detailInfo1 = Maps.newHashMap(); + detailInfo1.put(new DcClusterShard("jq", "cluster1", "shard1"), new Pair<>(2L, 2L)); + detailInfo1.put(new DcClusterShard("jq", "cluster1", "shard2"), new Pair<>(3L, 3L)); + detailInfo1.put(new DcClusterShard("jq", "cluster2", "shard1"), new Pair<>(4L, 4L)); + detailInfo1.put(new DcClusterShard("jq", "cluster2", "shard2"), new Pair<>(5L, 5L)); + model1.setDetailInfo(detailInfo1); + models1.add(model1); + + KeeperContainerUsedInfoModel model2 = new KeeperContainerUsedInfoModel("2.2.2.2", "jq", 13, 13); + Map> detailInfo2 = Maps.newHashMap(); + detailInfo2.put(new DcClusterShard("jq", "cluster3", "shard1"), new Pair<>(2L, 2L)); + detailInfo2.put(new DcClusterShard("jq", "cluster3", "shard2"), new Pair<>(3L, 3L)); + detailInfo2.put(new DcClusterShard("jq", "cluster4", "shard1"), new Pair<>(4L, 4L)); + detailInfo2.put(new DcClusterShard("jq", "cluster4", "shard2"), new Pair<>(3L, 3L)); + model2.setDetailInfo(detailInfo2); + models1.add(model2); + + KeeperContainerUsedInfoModel model3 = new KeeperContainerUsedInfoModel("3.3.3.3", "jq", 1, 1); + Map> detailInfo3 = Maps.newHashMap(); + detailInfo3.put(new DcClusterShard("jq", "cluster3", "shard1"), new Pair<>(1L, 1L)); + model3.setDetailInfo(detailInfo3); + models1.add(model3); + + analyzer.analyzeKeeperContainerUsedInfo(models1); + + List allDcReadyToMigrationKeeperContainers = analyzer.getAllDcReadyToMigrationKeeperContainers(); + Assert.assertEquals(2, allDcReadyToMigrationKeeperContainers.size()); + Assert.assertEquals("3.3.3.3", allDcReadyToMigrationKeeperContainers.get(0).getTargetKeeperContainer().getKeeperIp()); + Assert.assertEquals("3.3.3.3", allDcReadyToMigrationKeeperContainers.get(1).getTargetKeeperContainer().getKeeperIp()); + + } + + @Test + public void testSingleSrcKeeperMultiTargetWithBoth() { + List models1 = new ArrayList<>(); + KeeperContainerUsedInfoModel model1 = new KeeperContainerUsedInfoModel("1.1.1.1", "jq", 17, 17); + Map> detailInfo1 = Maps.newHashMap(); + detailInfo1.put(new DcClusterShard("jq", "cluster1", "shard1"), new Pair<>(2L, 2L)); + detailInfo1.put(new DcClusterShard("jq", "cluster1", "shard2"), new Pair<>(3L, 3L)); + detailInfo1.put(new DcClusterShard("jq", "cluster2", "shard1"), new Pair<>(3L, 3L)); + detailInfo1.put(new DcClusterShard("jq", "cluster2", "shard2"), new Pair<>(3L, 3L)); + detailInfo1.put(new DcClusterShard("jq", "cluster3", "shard1"), new Pair<>(3L, 3L)); + detailInfo1.put(new DcClusterShard("jq", "cluster3", "shard2"), new Pair<>(3L, 3L)); + model1.setDetailInfo(detailInfo1); + models1.add(model1); + + KeeperContainerUsedInfoModel model2 = new KeeperContainerUsedInfoModel("2.2.2.2", "jq", 5, 5); + Map> detailInfo2 = Maps.newHashMap(); + detailInfo2.put(new DcClusterShard("jq", "cluster3", "shard1"), new Pair<>(5L, 5L)); + model2.setDetailInfo(detailInfo2); + models1.add(model2); + + KeeperContainerUsedInfoModel model3 = new KeeperContainerUsedInfoModel("3.3.3.3", "jq", 4, 4); + Map> detailInfo3 = Maps.newHashMap(); + detailInfo3.put(new DcClusterShard("jq", "cluster3", "shard1"), new Pair<>(4L, 4L)); + model3.setDetailInfo(detailInfo3); + models1.add(model3); + + analyzer.analyzeKeeperContainerUsedInfo(models1); + + List allDcReadyToMigrationKeeperContainers = analyzer.getAllDcReadyToMigrationKeeperContainers(); + Assert.assertEquals(2, allDcReadyToMigrationKeeperContainers.size()); + Assert.assertEquals("3.3.3.3", allDcReadyToMigrationKeeperContainers.get(0).getTargetKeeperContainer().getKeeperIp()); + Assert.assertEquals(2, allDcReadyToMigrationKeeperContainers.get(0).getMigrateKeeperCount()); + Assert.assertEquals("2.2.2.2", allDcReadyToMigrationKeeperContainers.get(1).getTargetKeeperContainer().getKeeperIp()); + Assert.assertEquals(1, allDcReadyToMigrationKeeperContainers.get(1).getMigrateKeeperCount()); + } + + @Test + public void testKeeperResourceLackWithBoth() { + List models1 = new ArrayList<>(); + KeeperContainerUsedInfoModel model1 = new KeeperContainerUsedInfoModel("1.1.1.1", "jq", 14, 14); + Map> detailInfo1 = Maps.newHashMap(); + detailInfo1.put(new DcClusterShard("jq", "cluster1", "shard1"), new Pair<>(2L, 2L)); + detailInfo1.put(new DcClusterShard("jq", "cluster1", "shard2"), new Pair<>(3L, 3L)); + detailInfo1.put(new DcClusterShard("jq", "cluster2", "shard1"), new Pair<>(4L, 4L)); + detailInfo1.put(new DcClusterShard("jq", "cluster2", "shard2"), new Pair<>(5L, 5L)); + model1.setDetailInfo(detailInfo1); + models1.add(model1); + + KeeperContainerUsedInfoModel model2 = new KeeperContainerUsedInfoModel("2.2.2.2", "jq", 13, 13); + Map> detailInfo2 = Maps.newHashMap(); + detailInfo2.put(new DcClusterShard("jq", "cluster3", "shard1"), new Pair<>(2L, 2L)); + detailInfo2.put(new DcClusterShard("jq", "cluster3", "shard2"), new Pair<>(3L, 3L)); + detailInfo2.put(new DcClusterShard("jq", "cluster4", "shard1"), new Pair<>(4L, 4L)); + detailInfo2.put(new DcClusterShard("jq", "cluster4", "shard2"), new Pair<>(3L, 3L)); + model2.setDetailInfo(detailInfo2); + models1.add(model2); + + KeeperContainerUsedInfoModel model3 = new KeeperContainerUsedInfoModel("3.3.3.3", "jq", 5, 5); + Map> detailInfo3 = Maps.newHashMap(); + detailInfo3.put(new DcClusterShard("jq", "cluster3", "shard1"), new Pair<>(2L, 2L)); + detailInfo3.put(new DcClusterShard("jq", "cluster4", "shard2"), new Pair<>(3L, 3L)); + model3.setDetailInfo(detailInfo3); + models1.add(model3); + + + analyzer.updateKeeperContainerUsedInfo(0, models1); + analyzer.analyzeKeeperContainerUsedInfo(models1); + List allDcReadyToMigrationKeeperContainers = analyzer.getAllDcReadyToMigrationKeeperContainers(); + Assert.assertEquals(1, allDcReadyToMigrationKeeperContainers.size()); + } + + @Test + public void testGetAllDcReadyToMigrationKeeperContainersWithPeerDataOverLoad() { + List models1 = new ArrayList<>(); + KeeperContainerUsedInfoModel model1 = new KeeperContainerUsedInfoModel("1.1.1.1", "jq", 4, 14); + Map> detailInfo1 = Maps.newHashMap(); + detailInfo1.put(new DcClusterShard("jq", "cluster1", "shard1"), new Pair<>(1L, 2L)); + detailInfo1.put(new DcClusterShard("jq", "cluster1", "shard2"), new Pair<>(1L, 3L)); + detailInfo1.put(new DcClusterShard("jq", "cluster2", "shard1"), new Pair<>(1L, 4L)); + detailInfo1.put(new DcClusterShard("jq", "cluster2", "shard2"), new Pair<>(1L, 5L)); + model1.setDetailInfo(detailInfo1); + models1.add(model1); + + KeeperContainerUsedInfoModel model2 = new KeeperContainerUsedInfoModel("2.2.2.2", "jq", 4, 13); + Map> detailInfo2 = Maps.newHashMap(); + detailInfo2.put(new DcClusterShard("jq", "cluster3", "shard1"), new Pair<>(1L, 2L)); + detailInfo2.put(new DcClusterShard("jq", "cluster3", "shard2"), new Pair<>(1L, 3L)); + detailInfo2.put(new DcClusterShard("jq", "cluster4", "shard1"), new Pair<>(1L, 4L)); + detailInfo2.put(new DcClusterShard("jq", "cluster4", "shard2"), new Pair<>(1L, 3L)); + model2.setDetailInfo(detailInfo2); + models1.add(model2); + + KeeperContainerUsedInfoModel model3 = new KeeperContainerUsedInfoModel("3.3.3.3", "jq", 2, 5); + Map> detailInfo3 = Maps.newHashMap(); + detailInfo3.put(new DcClusterShard("jq", "cluster3", "shard1"), new Pair<>(1L, 2L)); + detailInfo3.put(new DcClusterShard("jq", "cluster4", "shard2"), new Pair<>(1L, 3L)); + model3.setDetailInfo(detailInfo3); + models1.add(model3); + + KeeperContainerUsedInfoModel model4 = new KeeperContainerUsedInfoModel("4.4.4.4", "jq", 2, 6); + Map> detailInfo4 = Maps.newHashMap(); + detailInfo3.put(new DcClusterShard("jq", "cluster1", "shard1"), new Pair<>(1L, 3L)); + detailInfo3.put(new DcClusterShard("jq", "cluster2", "shard2"), new Pair<>(1L, 3L)); + model4.setDetailInfo(detailInfo4); + models1.add(model4); + + analyzer.updateKeeperContainerUsedInfo(0, models1); + analyzer.analyzeKeeperContainerUsedInfo(models1); + List allDcReadyToMigrationKeeperContainers = analyzer.getAllDcReadyToMigrationKeeperContainers(); + Assert.assertEquals(2, allDcReadyToMigrationKeeperContainers.size()); + } + + @Test + public void testMultiSrcKeeperSingleTargetWithPeerDataOverLoad() { + List models1 = new ArrayList<>(); + KeeperContainerUsedInfoModel model1 = new KeeperContainerUsedInfoModel("1.1.1.1", "jq", 4, 14); + Map> detailInfo1 = Maps.newHashMap(); + detailInfo1.put(new DcClusterShard("jq", "cluster1", "shard1"), new Pair<>(1L, 2L)); + detailInfo1.put(new DcClusterShard("jq", "cluster1", "shard2"), new Pair<>(1L, 3L)); + detailInfo1.put(new DcClusterShard("jq", "cluster2", "shard1"), new Pair<>(1L, 4L)); + detailInfo1.put(new DcClusterShard("jq", "cluster2", "shard2"), new Pair<>(1L, 5L)); + model1.setDetailInfo(detailInfo1); + models1.add(model1); + + KeeperContainerUsedInfoModel model2 = new KeeperContainerUsedInfoModel("2.2.2.2", "jq", 4, 13); + Map> detailInfo2 = Maps.newHashMap(); + detailInfo2.put(new DcClusterShard("jq", "cluster3", "shard1"), new Pair<>(1L, 2L)); + detailInfo2.put(new DcClusterShard("jq", "cluster3", "shard2"), new Pair<>(1L, 3L)); + detailInfo2.put(new DcClusterShard("jq", "cluster4", "shard1"), new Pair<>(1L, 4L)); + detailInfo2.put(new DcClusterShard("jq", "cluster4", "shard2"), new Pair<>(1L, 3L)); + model2.setDetailInfo(detailInfo2); + models1.add(model2); + + List models2 = new ArrayList<>(); + KeeperContainerUsedInfoModel model3 = new KeeperContainerUsedInfoModel("3.3.3.3", "jq", 1, 1); + Map> detailInfo3 = Maps.newHashMap(); + detailInfo3.put(new DcClusterShard("jq", "cluster3", "shard1"), new Pair<>(1L, 1L)); + model3.setDetailInfo(detailInfo3); + models1.add(model3); + + analyzer.analyzeKeeperContainerUsedInfo(models1); + + List allDcReadyToMigrationKeeperContainers = analyzer.getAllDcReadyToMigrationKeeperContainers(); + Assert.assertEquals(2, allDcReadyToMigrationKeeperContainers.size()); + Assert.assertEquals("3.3.3.3", allDcReadyToMigrationKeeperContainers.get(0).getTargetKeeperContainer().getKeeperIp()); + Assert.assertEquals("3.3.3.3", allDcReadyToMigrationKeeperContainers.get(1).getTargetKeeperContainer().getKeeperIp()); + + } + + @Test + public void testSingleSrcKeeperMultiTargetWithPeerDataOverLoad() { + List models1 = new ArrayList<>(); + KeeperContainerUsedInfoModel model1 = new KeeperContainerUsedInfoModel("1.1.1.1", "jq", 6, 17); + Map> detailInfo1 = Maps.newHashMap(); + detailInfo1.put(new DcClusterShard("jq", "cluster1", "shard1"), new Pair<>(1L, 2L)); + detailInfo1.put(new DcClusterShard("jq", "cluster1", "shard2"), new Pair<>(1L, 3L)); + detailInfo1.put(new DcClusterShard("jq", "cluster2", "shard1"), new Pair<>(1L, 3L)); + detailInfo1.put(new DcClusterShard("jq", "cluster2", "shard2"), new Pair<>(1L, 3L)); + detailInfo1.put(new DcClusterShard("jq", "cluster3", "shard1"), new Pair<>(1L, 3L)); + detailInfo1.put(new DcClusterShard("jq", "cluster3", "shard2"), new Pair<>(1L, 3L)); + model1.setDetailInfo(detailInfo1); + models1.add(model1); + + KeeperContainerUsedInfoModel model2 = new KeeperContainerUsedInfoModel("2.2.2.2", "jq", 1, 5); + Map> detailInfo2 = Maps.newHashMap(); + detailInfo2.put(new DcClusterShard("jq", "cluster3", "shard1"), new Pair<>(1L, 5L)); + model2.setDetailInfo(detailInfo2); + models1.add(model2); + + KeeperContainerUsedInfoModel model3 = new KeeperContainerUsedInfoModel("3.3.3.3", "jq", 1, 4); + Map> detailInfo3 = Maps.newHashMap(); + detailInfo3.put(new DcClusterShard("jq", "cluster3", "shard1"), new Pair<>(1L, 4L)); + model3.setDetailInfo(detailInfo3); + models1.add(model3); + + analyzer.analyzeKeeperContainerUsedInfo(models1); + + List allDcReadyToMigrationKeeperContainers = analyzer.getAllDcReadyToMigrationKeeperContainers(); + Assert.assertEquals(2, allDcReadyToMigrationKeeperContainers.size()); + Assert.assertEquals("3.3.3.3", allDcReadyToMigrationKeeperContainers.get(0).getTargetKeeperContainer().getKeeperIp()); + Assert.assertEquals(2, allDcReadyToMigrationKeeperContainers.get(0).getMigrateKeeperCount()); + Assert.assertEquals("2.2.2.2", allDcReadyToMigrationKeeperContainers.get(1).getTargetKeeperContainer().getKeeperIp()); + Assert.assertEquals(1, allDcReadyToMigrationKeeperContainers.get(1).getMigrateKeeperCount()); + } + + @Test + public void testKeeperResourceLackWithPeerDataOverLoad() { + List models1 = new ArrayList<>(); + KeeperContainerUsedInfoModel model1 = new KeeperContainerUsedInfoModel("1.1.1.1", "jq", 4, 14); + Map> detailInfo1 = Maps.newHashMap(); + detailInfo1.put(new DcClusterShard("jq", "cluster1", "shard1"), new Pair<>(1L, 2L)); + detailInfo1.put(new DcClusterShard("jq", "cluster1", "shard2"), new Pair<>(1L, 3L)); + detailInfo1.put(new DcClusterShard("jq", "cluster2", "shard1"), new Pair<>(1L, 4L)); + detailInfo1.put(new DcClusterShard("jq", "cluster2", "shard2"), new Pair<>(1L, 5L)); + model1.setDetailInfo(detailInfo1); + models1.add(model1); + + KeeperContainerUsedInfoModel model2 = new KeeperContainerUsedInfoModel("2.2.2.2", "jq", 4, 13); + Map> detailInfo2 = Maps.newHashMap(); + detailInfo2.put(new DcClusterShard("jq", "cluster3", "shard1"), new Pair<>(1L, 2L)); + detailInfo2.put(new DcClusterShard("jq", "cluster3", "shard2"), new Pair<>(1L, 3L)); + detailInfo2.put(new DcClusterShard("jq", "cluster4", "shard1"), new Pair<>(1L, 4L)); + detailInfo2.put(new DcClusterShard("jq", "cluster4", "shard2"), new Pair<>(1L, 3L)); + model2.setDetailInfo(detailInfo2); + models1.add(model2); + + KeeperContainerUsedInfoModel model3 = new KeeperContainerUsedInfoModel("3.3.3.3", "jq", 2, 5); + Map> detailInfo3 = Maps.newHashMap(); + detailInfo3.put(new DcClusterShard("jq", "cluster3", "shard1"), new Pair<>(1L, 2L)); + detailInfo3.put(new DcClusterShard("jq", "cluster4", "shard2"), new Pair<>(1L, 3L)); + model3.setDetailInfo(detailInfo3); + models1.add(model3); + + + analyzer.updateKeeperContainerUsedInfo(0, models1); + analyzer.analyzeKeeperContainerUsedInfo(models1); + List allDcReadyToMigrationKeeperContainers = analyzer.getAllDcReadyToMigrationKeeperContainers(); + Assert.assertEquals(1, allDcReadyToMigrationKeeperContainers.size()); + } + + @Test + public void testGetAllDcReadyToMigrationKeeperContainersWithMixed() { + List models1 = new ArrayList<>(); + // inputOverLoad + KeeperContainerUsedInfoModel model1 = new KeeperContainerUsedInfoModel("1.1.1.1", "jq", 14, 8); + Map> detailInfo1 = Maps.newHashMap(); + detailInfo1.put(new DcClusterShard("jq", "cluster1", "shard1"), new Pair<>(2L, 2L)); + detailInfo1.put(new DcClusterShard("jq", "cluster1", "shard2"), new Pair<>(3L, 2L)); + detailInfo1.put(new DcClusterShard("jq", "cluster2", "shard1"), new Pair<>(4L, 2L)); + detailInfo1.put(new DcClusterShard("jq", "cluster2", "shard2"), new Pair<>(5L, 2L)); + model1.setDetailInfo(detailInfo1); + models1.add(model1); + + //PeerDataOverLoad + KeeperContainerUsedInfoModel model2 = new KeeperContainerUsedInfoModel("2.2.2.2", "jq", 8, 13); + Map> detailInfo2 = Maps.newHashMap(); + detailInfo2.put(new DcClusterShard("jq", "cluster3", "shard1"), new Pair<>(2L, 2L)); + detailInfo2.put(new DcClusterShard("jq", "cluster3", "shard2"), new Pair<>(2L, 3L)); + detailInfo2.put(new DcClusterShard("jq", "cluster4", "shard1"), new Pair<>(2L, 4L)); + detailInfo2.put(new DcClusterShard("jq", "cluster4", "shard2"), new Pair<>(2L, 3L)); + model2.setDetailInfo(detailInfo2); + models1.add(model2); + + KeeperContainerUsedInfoModel model3 = new KeeperContainerUsedInfoModel("3.3.3.3", "jq", 5, 5); + Map> detailInfo3 = Maps.newHashMap(); + detailInfo3.put(new DcClusterShard("jq", "cluster3", "shard1"), new Pair<>(2L, 2L)); + detailInfo3.put(new DcClusterShard("jq", "cluster4", "shard2"), new Pair<>(3L, 3L)); + model3.setDetailInfo(detailInfo3); + models1.add(model3); + + + KeeperContainerUsedInfoModel model4 = new KeeperContainerUsedInfoModel("4.4.4.4", "jq", 6, 6); + Map> detailInfo4 = Maps.newHashMap(); + detailInfo3.put(new DcClusterShard("jq", "cluster1", "shard1"), new Pair<>(3L, 3L)); + detailInfo3.put(new DcClusterShard("jq", "cluster2", "shard2"), new Pair<>(3L, 3L)); + model4.setDetailInfo(detailInfo4); + models1.add(model4); + + analyzer.updateKeeperContainerUsedInfo(0, models1); + analyzer.analyzeKeeperContainerUsedInfo(models1); + List allDcReadyToMigrationKeeperContainers = analyzer.getAllDcReadyToMigrationKeeperContainers(); + Assert.assertEquals(2, allDcReadyToMigrationKeeperContainers.size()); + } + + @Test + public void testMultiSrcKeeperSingleTargetWithMixed() { + List models1 = new ArrayList<>(); + KeeperContainerUsedInfoModel model1 = new KeeperContainerUsedInfoModel("1.1.1.1", "jq", 5, 15); + Map> detailInfo1 = Maps.newHashMap(); + detailInfo1.put(new DcClusterShard("jq", "cluster1", "shard1"), new Pair<>(1L, 3L)); + detailInfo1.put(new DcClusterShard("jq", "cluster1", "shard2"), new Pair<>(1L, 3L)); + detailInfo1.put(new DcClusterShard("jq", "cluster2", "shard1"), new Pair<>(1L, 3L)); + detailInfo1.put(new DcClusterShard("jq", "cluster2", "shard2"), new Pair<>(1L, 3L)); + detailInfo1.put(new DcClusterShard("jq", "cluster2", "shard3"), new Pair<>(1L, 3L)); + model1.setDetailInfo(detailInfo1); + models1.add(model1); + + KeeperContainerUsedInfoModel model2 = new KeeperContainerUsedInfoModel("2.2.2.2", "jq", 15, 5); + Map> detailInfo2 = Maps.newHashMap(); + detailInfo2.put(new DcClusterShard("jq", "cluster3", "shard1"), new Pair<>(3L, 1L)); + detailInfo2.put(new DcClusterShard("jq", "cluster3", "shard2"), new Pair<>(3L, 1L)); + detailInfo2.put(new DcClusterShard("jq", "cluster4", "shard1"), new Pair<>(3L, 1L)); + detailInfo2.put(new DcClusterShard("jq", "cluster4", "shard2"), new Pair<>(3L, 1L)); + detailInfo2.put(new DcClusterShard("jq", "cluster5", "shard2"), new Pair<>(3L, 1L)); + model2.setDetailInfo(detailInfo2); + models1.add(model2); + + KeeperContainerUsedInfoModel model3 = new KeeperContainerUsedInfoModel("3.3.3.3", "jq", 0, 0); + Map> detailInfo3 = Maps.newHashMap(); + model3.setDetailInfo(detailInfo3); + models1.add(model3); + + analyzer.analyzeKeeperContainerUsedInfo(models1); + + List allDcReadyToMigrationKeeperContainers = analyzer.getAllDcReadyToMigrationKeeperContainers(); + Assert.assertEquals(2, allDcReadyToMigrationKeeperContainers.size()); + Assert.assertEquals("3.3.3.3", allDcReadyToMigrationKeeperContainers.get(0).getTargetKeeperContainer().getKeeperIp()); + Assert.assertEquals(2, allDcReadyToMigrationKeeperContainers.get(0).getMigrateKeeperCount()); + Assert.assertEquals("3.3.3.3", allDcReadyToMigrationKeeperContainers.get(1).getTargetKeeperContainer().getKeeperIp()); + Assert.assertEquals(2, allDcReadyToMigrationKeeperContainers.get(1).getMigrateKeeperCount()); + } + + @Test + public void testMultiSrcMultiTargetWithFixed() { + List models1 = new ArrayList<>(); + KeeperContainerUsedInfoModel model1 = new KeeperContainerUsedInfoModel("1.1.1.1", "jq", 5, 15); + Map> detailInfo1 = Maps.newHashMap(); + detailInfo1.put(new DcClusterShard("jq", "cluster1", "shard1"), new Pair<>(1L, 3L)); + detailInfo1.put(new DcClusterShard("jq", "cluster1", "shard2"), new Pair<>(1L, 3L)); + detailInfo1.put(new DcClusterShard("jq", "cluster2", "shard1"), new Pair<>(1L, 3L)); + detailInfo1.put(new DcClusterShard("jq", "cluster2", "shard2"), new Pair<>(1L, 3L)); + detailInfo1.put(new DcClusterShard("jq", "cluster2", "shard3"), new Pair<>(1L, 3L)); + model1.setDetailInfo(detailInfo1); + models1.add(model1); + + KeeperContainerUsedInfoModel model2 = new KeeperContainerUsedInfoModel("2.2.2.2", "jq", 15, 5); + Map> detailInfo2 = Maps.newHashMap(); + detailInfo2.put(new DcClusterShard("jq", "cluster3", "shard1"), new Pair<>(3L, 1L)); + detailInfo2.put(new DcClusterShard("jq", "cluster3", "shard2"), new Pair<>(3L, 1L)); + detailInfo2.put(new DcClusterShard("jq", "cluster4", "shard1"), new Pair<>(3L, 1L)); + detailInfo2.put(new DcClusterShard("jq", "cluster4", "shard2"), new Pair<>(3L, 1L)); + detailInfo2.put(new DcClusterShard("jq", "cluster5", "shard2"), new Pair<>(3L, 1L)); + model2.setDetailInfo(detailInfo2); + models1.add(model2); + + KeeperContainerUsedInfoModel model3 = new KeeperContainerUsedInfoModel("3.3.3.3", "jq", 6, 4); + Map> detailInfo3 = Maps.newHashMap(); + model3.setDetailInfo(detailInfo3); + models1.add(model3); + + KeeperContainerUsedInfoModel model4 = new KeeperContainerUsedInfoModel("4.4.4.4", "jq", 4, 6); + Map> detailInfo4 = Maps.newHashMap(); + model4.setDetailInfo(detailInfo4); + models1.add(model4); + + analyzer.analyzeKeeperContainerUsedInfo(models1); + + List allDcReadyToMigrationKeeperContainers = analyzer.getAllDcReadyToMigrationKeeperContainers(); + Assert.assertEquals(2, allDcReadyToMigrationKeeperContainers.size()); + Assert.assertEquals("3.3.3.3", allDcReadyToMigrationKeeperContainers.get(0).getTargetKeeperContainer().getKeeperIp()); + Assert.assertEquals(2, allDcReadyToMigrationKeeperContainers.get(0).getMigrateKeeperCount()); + Assert.assertEquals("4.4.4.4", allDcReadyToMigrationKeeperContainers.get(1).getTargetKeeperContainer().getKeeperIp()); + Assert.assertEquals(2, allDcReadyToMigrationKeeperContainers.get(1).getMigrateKeeperCount()); + } +} \ No newline at end of file diff --git a/redis/redis-console/src/test/java/com/ctrip/xpipe/redis/console/resources/CheckerCurrentDcAllMetaTest.java b/redis/redis-console/src/test/java/com/ctrip/xpipe/redis/console/resources/CheckerCurrentDcAllMetaTest.java new file mode 100644 index 000000000..0d4061ede --- /dev/null +++ b/redis/redis-console/src/test/java/com/ctrip/xpipe/redis/console/resources/CheckerCurrentDcAllMetaTest.java @@ -0,0 +1,15 @@ +package com.ctrip.xpipe.redis.console.resources; + +import static org.junit.jupiter.api.Assertions.*; + +/** + * @author yu + *

+ * 2023/11/16 + */ + +public class CheckerCurrentDcAllMetaTest { + + + +} \ No newline at end of file diff --git a/redis/redis-console/src/test/java/com/ctrip/xpipe/redis/console/service/impl/DefaultKeeperContainerMigrationServiceTest.java b/redis/redis-console/src/test/java/com/ctrip/xpipe/redis/console/service/impl/DefaultKeeperContainerMigrationServiceTest.java new file mode 100644 index 000000000..95a174bc9 --- /dev/null +++ b/redis/redis-console/src/test/java/com/ctrip/xpipe/redis/console/service/impl/DefaultKeeperContainerMigrationServiceTest.java @@ -0,0 +1,79 @@ +package com.ctrip.xpipe.redis.console.service.impl; + +import com.ctrip.xpipe.redis.checker.model.DcClusterShard; +import com.ctrip.xpipe.redis.checker.model.KeeperContainerUsedInfoModel; +import com.ctrip.xpipe.redis.console.model.MigrationKeeperContainerDetailModel; +import com.ctrip.xpipe.redis.console.model.ShardModel; +import com.ctrip.xpipe.redis.console.service.model.ShardModelService; +import com.ctrip.xpipe.tuple.Pair; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.junit.MockitoJUnitRunner; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * @author yu + *

+ * 2023/9/20 + */ +@RunWith(MockitoJUnitRunner.class) +public class DefaultKeeperContainerMigrationServiceTest { + + @InjectMocks + private DefaultKeeperContainerMigrationService service; + + @Mock + private ShardModelService shardModelService; + + @Before + public void before() { + ShardModel shardModel = new ShardModel(); + Mockito.when(shardModelService.getShardModel(Mockito.anyString(), Mockito.anyString(), Mockito.anyString(), Mockito.anyBoolean(), Mockito.anyObject())) + .thenReturn(shardModel); + Mockito.when(shardModelService.migrateShardKeepers(Mockito.anyString(), Mockito.anyString(), Mockito.any(), Mockito.anyString(), Mockito.anyString())) + .thenReturn(true); + } + + @Test + public void testMigrationKeeperContainer() { + List models = new ArrayList<>(); + + MigrationKeeperContainerDetailModel model = new MigrationKeeperContainerDetailModel(); + KeeperContainerUsedInfoModel src = new KeeperContainerUsedInfoModel() + .setKeeperIp("1.1.1.1").setDcName("jq").setTotalInputFlow(300 * 1024 * 1024L) + .setTotalRedisUsedMemory(500 * 1024 * 1024 * 1024L); + Map> detailInfo = new HashMap<>(); + detailInfo.put(new DcClusterShard("jq", "cluster1", "shard1"), new Pair<>(200 * 1024 * 1024L, 400 * 1024 * 1024L)); + detailInfo.put(new DcClusterShard("jq", "cluster1", "shard2"), new Pair<>(20 * 1024 * 1024L, 20 * 1024 * 1024L)); + detailInfo.put(new DcClusterShard("jq", "cluster2", "shard1"), new Pair<>(30 * 1024 * 1024L, 30 * 1024 * 1024L)); + detailInfo.put(new DcClusterShard("jq", "cluster2", "shard2"), new Pair<>(40 * 1024 * 1024L, 40 * 1024 * 1024L)); + src.setDetailInfo(detailInfo); + + KeeperContainerUsedInfoModel target = new KeeperContainerUsedInfoModel() + .setKeeperIp("2.2.2.2").setDcName("jq").setTotalInputFlow(300 * 1024 * 1024L) + .setTotalRedisUsedMemory(500 * 1024 * 1024 * 1024L); + Map> detailInfo2 = new HashMap<>(); + detailInfo2.put(new DcClusterShard("jq", "cluster1", "shard1"), new Pair<>(200 * 1024 * 1024L, 400 * 1024 * 1024L)); + target.setDetailInfo(detailInfo2); + + List migrationShards = new ArrayList<>(); + migrationShards.add(new DcClusterShard("jq", "cluster1", "shard2")); + migrationShards.add(new DcClusterShard("jq", "cluster2", "shard1")); + migrationShards.add(new DcClusterShard("jq", "cluster2", "shard2")); + + model.setSrcKeeperContainer(src).setTargetKeeperContainer(target).setMigrateKeeperCount(3).setMigrateShards(migrationShards); + models.add(model); + service.beginMigrateKeeperContainers(models); + + Assert.assertEquals(3, service.getMigrationProcess().get(0).getMigrateKeeperCompleteCount()); + } +} \ No newline at end of file diff --git a/redis/redis-core/src/main/java/com/ctrip/xpipe/redis/core/console/ConsoleCheckerPath.java b/redis/redis-core/src/main/java/com/ctrip/xpipe/redis/core/console/ConsoleCheckerPath.java index eb3c2e8b1..5fa2f577a 100644 --- a/redis/redis-core/src/main/java/com/ctrip/xpipe/redis/core/console/ConsoleCheckerPath.java +++ b/redis/redis-core/src/main/java/com/ctrip/xpipe/redis/core/console/ConsoleCheckerPath.java @@ -12,6 +12,8 @@ private ConsoleCheckerPath() {} public static final String PATH_GET_ALL_META = "/api/meta/divide"; + public static final String PATH_GET_DC_ALL_META = "/api/meta/{dcName}/all"; + public static final String PATH_GET_META = "/api/meta/divide/{index}"; public static final String PATH_GET_PROXY_CHAINS = "/api/proxy/chains"; @@ -19,6 +21,8 @@ private ConsoleCheckerPath() {} public static final String PATH_PUT_CHECKER_STATUS = "/api/health/checker/status"; public static final String PATH_PUT_HEALTH_CHECK_RESULT = "/api/health/check/result"; + + public static final String PATH_POST_KEEPER_CONTAINER_INFO_RESULT = "/api/keeperContainer/info/result/{index}"; public static final String PATH_PERSISTENCE = "/api/persistence/"; diff --git a/redis/redis-core/src/main/java/com/ctrip/xpipe/redis/core/meta/CurrentDcAllMeta.java b/redis/redis-core/src/main/java/com/ctrip/xpipe/redis/core/meta/CurrentDcAllMeta.java new file mode 100644 index 000000000..f9d21faad --- /dev/null +++ b/redis/redis-core/src/main/java/com/ctrip/xpipe/redis/core/meta/CurrentDcAllMeta.java @@ -0,0 +1,14 @@ +package com.ctrip.xpipe.redis.core.meta; + +import com.ctrip.xpipe.redis.core.entity.DcMeta; + +/** + * @author yu + *

+ * 2023/11/14 + */ +public interface CurrentDcAllMeta { + + DcMeta getCurrentDcAllMeta(); + +} diff --git a/redis/redis-core/src/main/java/com/ctrip/xpipe/redis/core/meta/KeeperContainerDetailInfo.java b/redis/redis-core/src/main/java/com/ctrip/xpipe/redis/core/meta/KeeperContainerDetailInfo.java new file mode 100644 index 000000000..26e62a16b --- /dev/null +++ b/redis/redis-core/src/main/java/com/ctrip/xpipe/redis/core/meta/KeeperContainerDetailInfo.java @@ -0,0 +1,66 @@ +package com.ctrip.xpipe.redis.core.meta; + +import com.ctrip.xpipe.redis.core.entity.KeeperContainerMeta; +import com.ctrip.xpipe.redis.core.entity.KeeperMeta; +import com.ctrip.xpipe.redis.core.entity.RedisMeta; + +import java.util.List; +import java.util.Objects; + +public class KeeperContainerDetailInfo { + + private KeeperContainerMeta keeperContainer; + + private List keeperInstances; + + private List redisInstances; + + public KeeperContainerDetailInfo() { + } + + public KeeperContainerDetailInfo(KeeperContainerMeta keeperContainer, List keeperInstances, List redisInstances) { + this.keeperContainer = keeperContainer; + this.keeperInstances = keeperInstances; + this.redisInstances = redisInstances; + } + + public KeeperContainerMeta getKeeperContainer() { + return keeperContainer; + } + + public KeeperContainerDetailInfo setKeeperContainer(KeeperContainerMeta keeperContainer) { + this.keeperContainer = keeperContainer; + return this; + } + + public List getKeeperInstances() { + return keeperInstances; + } + + public KeeperContainerDetailInfo setKeeperInstances(List keeperInstances) { + this.keeperInstances = keeperInstances; + return this; + } + + public List getRedisInstances() { + return redisInstances; + } + + public KeeperContainerDetailInfo setRedisInstances(List redisInstances) { + this.redisInstances = redisInstances; + return this; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + KeeperContainerDetailInfo that = (KeeperContainerDetailInfo) o; + return Objects.equals(keeperContainer, that.keeperContainer); + } + + @Override + public int hashCode() { + return Objects.hash(keeperContainer); + } +} diff --git a/redis/redis-core/src/main/java/com/ctrip/xpipe/redis/core/meta/comparator/AbstractInstanceNodeComparator.java b/redis/redis-core/src/main/java/com/ctrip/xpipe/redis/core/meta/comparator/AbstractInstanceNodeComparator.java new file mode 100644 index 000000000..864cd8427 --- /dev/null +++ b/redis/redis-core/src/main/java/com/ctrip/xpipe/redis/core/meta/comparator/AbstractInstanceNodeComparator.java @@ -0,0 +1,63 @@ +package com.ctrip.xpipe.redis.core.meta.comparator; + +import com.ctrip.xpipe.redis.core.entity.ApplierMeta; +import com.ctrip.xpipe.redis.core.entity.InstanceNode; +import com.ctrip.xpipe.redis.core.meta.MetaUtils; +import com.ctrip.xpipe.tuple.Pair; + +import java.util.LinkedList; +import java.util.List; +import java.util.Objects; + +public abstract class AbstractInstanceNodeComparator extends AbstractMetaComparator{ + + + protected void compareConfigConfig(List> allModified) { + + for(Pair pair : allModified){ + InstanceNode current = pair.getValue(); + InstanceNode future = pair.getKey(); + if(current.equals(future)){ + continue; + } + InstanceNodeComparator instanceNodeComparator = new InstanceNodeComparator(current, future); + instanceNodeComparator.compare(); + modified.add(instanceNodeComparator); + } + + } + + protected Pair, List>> sub(List all1, List all2) { + + List subResult = new LinkedList<>(); + List> intersectResult = new LinkedList<>(); + + for(InstanceNode instanceNode1 : all1){ + + InstanceNode instanceNode2Equal = null; + for(InstanceNode instanceNode2 : all2){ + if(instanceNodeTheSame(instanceNode1, instanceNode2)){ + instanceNode2Equal = instanceNode2; + break; + } + } + if(instanceNode2Equal == null){ + subResult.add(instanceNode1); + }else{ + intersectResult.add(new Pair<>(instanceNode1, instanceNode2Equal)); + } + } + return new Pair, List>>(subResult, intersectResult); + } + + private boolean instanceNodeTheSame(InstanceNode instanceNode1, InstanceNode instanceNode2) { + if (!MetaUtils.theSame(instanceNode1, instanceNode2)) { + return false; + } + if (instanceNode1 instanceof ApplierMeta && instanceNode2 instanceof ApplierMeta) { + return Objects.equals(((ApplierMeta) instanceNode1).getTargetClusterName(), + ((ApplierMeta) instanceNode2).getTargetClusterName()); + } + return true; + } +} diff --git a/redis/redis-core/src/main/java/com/ctrip/xpipe/redis/core/meta/comparator/KeeperContainerMetaComparator.java b/redis/redis-core/src/main/java/com/ctrip/xpipe/redis/core/meta/comparator/KeeperContainerMetaComparator.java new file mode 100644 index 000000000..aa3d44e5d --- /dev/null +++ b/redis/redis-core/src/main/java/com/ctrip/xpipe/redis/core/meta/comparator/KeeperContainerMetaComparator.java @@ -0,0 +1,116 @@ +package com.ctrip.xpipe.redis.core.meta.comparator; + +import com.ctrip.xpipe.redis.core.entity.*; +import com.ctrip.xpipe.redis.core.meta.KeeperContainerDetailInfo; +import com.ctrip.xpipe.tuple.Pair; +import org.unidal.tuple.Triple; + +import java.util.*; +import java.util.stream.Collectors; + +/** + * @author yu + */ +public class KeeperContainerMetaComparator extends AbstractInstanceNodeComparator { + + private DcMeta current, future, currentAllDcMeta, futureAllDcMeta; + + public KeeperContainerMetaComparator(DcMeta current, DcMeta future, DcMeta currentAllDcMeta, DcMeta futureAllDcMeta) { + this.current = current; + this.future = future; + this.currentAllDcMeta = currentAllDcMeta; + this.futureAllDcMeta = futureAllDcMeta; + } + + @Override + public void compare() { + Map currentDetailInfo = getAllKeeperContainerDetailInfoFromDcMeta(current, currentAllDcMeta); + Map futureDetailInfo= getAllKeeperContainerDetailInfoFromDcMeta(future, futureAllDcMeta); + + Triple, Set, Set> result = getDiff(currentDetailInfo.keySet(), futureDetailInfo.keySet()); + result.getFirst().forEach(id -> added.addAll(futureDetailInfo.get(id).getKeeperInstances())); + result.getLast().forEach(id -> removed.addAll(currentDetailInfo.get(id).getKeeperInstances())); + result.getMiddle().forEach(id -> { + compareInstanceNode(getAllKeeperInstances(currentDetailInfo.get(id).getKeeperInstances()), + getAllKeeperInstances(futureDetailInfo.get(id).getKeeperInstances())); + }); + + Set currentRedises = getAllRedisesFromKeeperDetailInfo(currentDetailInfo); + Set futureRedises = getAllRedisesFromKeeperDetailInfo(futureDetailInfo); + Triple, Set, Set> redisResult = getDiff(currentRedises, futureRedises); + redisResult.getFirst().forEach(redis -> added.add(redis)); + redisResult.getLast().forEach(redis -> removed.add(redis)); + } + + private Set getAllRedisesFromKeeperDetailInfo(Map currentDetailInfo) { + Set result = new HashSet<>(); + currentDetailInfo.values().forEach(keeperContainerDetailInfo -> { + result.addAll(keeperContainerDetailInfo.getRedisInstances()); + }); + return result; + } + + public void compareInstanceNode(List current, List future) { + Pair, List>> subResult = sub(future, current); + List tAdded = subResult.getKey(); + added.addAll(tAdded); + + List> modified = subResult.getValue(); + compareConfigConfig(modified); + + List tRemoved = sub(current, future).getKey(); + removed.addAll(tRemoved); + + } + + private List getAllKeeperInstances(List keeperInstances) { + List result = new LinkedList<>(); + result.addAll(keeperInstances); + return result; + } + + private List getAllRedisInstances(List redisInstances) { + List result = new LinkedList<>(); + result.addAll(redisInstances); + return result; + } + + @Override + public String idDesc() { + return current.getId(); + } + + public static Map getAllKeeperContainerDetailInfoFromDcMeta(DcMeta dcMeta, DcMeta dcAllMeta) { + Map map = dcMeta.getKeeperContainers().stream() + .collect(Collectors.toMap(KeeperContainerMeta::getId, + keeperContainerMeta -> new KeeperContainerDetailInfo(keeperContainerMeta, new ArrayList<>(), new ArrayList<>()))); + if (dcAllMeta == null || dcAllMeta.getClusters() == null) return map; + + dcAllMeta.getClusters().values().forEach(clusterMeta -> { + for (ShardMeta shardMeta : clusterMeta.getAllShards().values()){ + if (shardMeta.getKeepers() == null || shardMeta.getKeepers().isEmpty()) continue; + RedisMeta monitorRedis = getMonitorRedisMeta(shardMeta.getRedises()); + shardMeta.getKeepers().forEach(keeperMeta -> { + if (map.containsKey(keeperMeta.getKeeperContainerId())) { + map.get(keeperMeta.getKeeperContainerId()).getKeeperInstances().add(keeperMeta); + if (monitorRedis != null) + map.get(keeperMeta.getKeeperContainerId()).getRedisInstances().add(monitorRedis); + } + }); + } + }); + + return map; + } + + public static RedisMeta getMonitorRedisMeta(List redisMetas) { + if (redisMetas == null || redisMetas.isEmpty()) return null; + + List candidates = redisMetas.stream().filter(r -> !r.isMaster()) + .sorted((r1, r2) -> (r1.getIp().hashCode() - r2.getIp().hashCode())) + .collect(Collectors.toList()); + return candidates.isEmpty() ? null : candidates.get(0); + } +} + + diff --git a/redis/redis-core/src/main/java/com/ctrip/xpipe/redis/core/meta/comparator/ShardMetaComparator.java b/redis/redis-core/src/main/java/com/ctrip/xpipe/redis/core/meta/comparator/ShardMetaComparator.java index a2dbb11ca..f1823f0bb 100644 --- a/redis/redis-core/src/main/java/com/ctrip/xpipe/redis/core/meta/comparator/ShardMetaComparator.java +++ b/redis/redis-core/src/main/java/com/ctrip/xpipe/redis/core/meta/comparator/ShardMetaComparator.java @@ -1,22 +1,19 @@ package com.ctrip.xpipe.redis.core.meta.comparator; -import com.ctrip.xpipe.redis.core.entity.ApplierMeta; import com.ctrip.xpipe.redis.core.entity.InstanceNode; import com.ctrip.xpipe.redis.core.entity.ShardMeta; -import com.ctrip.xpipe.redis.core.meta.MetaUtils; import com.ctrip.xpipe.tuple.Pair; import java.util.LinkedList; import java.util.List; -import java.util.Objects; /** * @author wenchao.meng * * Sep 2, 2016 */ -public class ShardMetaComparator extends AbstractMetaComparator{ +public class ShardMetaComparator extends AbstractInstanceNodeComparator{ private ShardMeta current, future; @@ -43,54 +40,6 @@ public void compare() { removed.addAll(tRemoved); } - private void compareConfigConfig(List> allModified) { - - for(Pair pair : allModified){ - InstanceNode current = pair.getValue(); - InstanceNode future = pair.getKey(); - if(current.equals(future)){ - continue; - } - InstanceNodeComparator instanceNodeComparator = new InstanceNodeComparator(current, future); - instanceNodeComparator.compare(); - modified.add(instanceNodeComparator); - } - - } - - private Pair, List>> sub(List all1, List all2) { - - List subResult = new LinkedList<>(); - List> intersectResult = new LinkedList<>(); - - for(InstanceNode instanceNode1 : all1){ - - InstanceNode instanceNode2Equal = null; - for(InstanceNode instanceNode2 : all2){ - if(instanceNodeTheSame(instanceNode1, instanceNode2)){ - instanceNode2Equal = instanceNode2; - break; - } - } - if(instanceNode2Equal == null){ - subResult.add(instanceNode1); - }else{ - intersectResult.add(new Pair<>(instanceNode1, instanceNode2Equal)); - } - } - return new Pair, List>>(subResult, intersectResult); - } - - private boolean instanceNodeTheSame(InstanceNode instanceNode1, InstanceNode instanceNode2) { - if (!MetaUtils.theSame(instanceNode1, instanceNode2)) { - return false; - } - if (instanceNode1 instanceof ApplierMeta && instanceNode2 instanceof ApplierMeta) { - return Objects.equals(((ApplierMeta) instanceNode1).getTargetClusterName(), - ((ApplierMeta) instanceNode2).getTargetClusterName()); - } - return true; - } private List getAll(ShardMeta shardMeta) { diff --git a/redis/redis-core/src/main/java/com/ctrip/xpipe/redis/core/protocal/cmd/InfoResultExtractor.java b/redis/redis-core/src/main/java/com/ctrip/xpipe/redis/core/protocal/cmd/InfoResultExtractor.java index 51ffc7b14..f0fa41d8e 100644 --- a/redis/redis-core/src/main/java/com/ctrip/xpipe/redis/core/protocal/cmd/InfoResultExtractor.java +++ b/redis/redis-core/src/main/java/com/ctrip/xpipe/redis/core/protocal/cmd/InfoResultExtractor.java @@ -21,6 +21,12 @@ public class InfoResultExtractor { private static final String KEY_MASTER_REPL_OFFSET = "master_repl_offset"; private static final String KEY_SLAVE_REPL_OFFSET = "slave_repl_offset"; + private static final String KEY_INSTANTANEOUS_INPUT_KBPS = "instantaneous_input_kbps"; + + private static final String KEY_SWAP_USED_DB_SIZE = "swap_used_db_size"; + private static final String KEY_USED_MEMORY ="used_memory"; + private static final String KEY_MAX_MEMORY ="maxmemory"; + protected static Logger logger = LoggerFactory.getLogger(InfoResultExtractor.class); private String result; @@ -49,6 +55,10 @@ public Long extractAsLong(String key) { return extract(key, (value) -> value == null ? null : Long.parseLong(value)); } + public Float extractAsFloat(String key) { + return extract(key, (value) -> value == null ? null : Float.parseFloat(value)); + } + public Map extract(String[] keys) { genKeyValues(); @@ -101,6 +111,14 @@ public long getSyncPartialErr() { return extractAsLong(KEY_SYNC_PARTIAL_ERR); } + public float getKeeperInstantaneousInputKbps() { return extractAsFloat(KEY_INSTANTANEOUS_INPUT_KBPS);} + + public long getUsedMemory() { return extractAsLong(KEY_USED_MEMORY);} + + public long getMaxMemory() { return extractAsLong(KEY_MAX_MEMORY);} + + public Long getSwapUsedDbSize() { return extractAsLong(KEY_SWAP_USED_DB_SIZE);} + public long getMasterReplOffset() { Long result = extractAsLong(KEY_MASTER_REPL_OFFSET); return result == null ? 0L : result; diff --git a/redis/redis-core/src/test/java/com/ctrip/xpipe/redis/core/AbstractRedisTest.java b/redis/redis-core/src/test/java/com/ctrip/xpipe/redis/core/AbstractRedisTest.java index 2f34b4f12..1b7c1d1fe 100644 --- a/redis/redis-core/src/test/java/com/ctrip/xpipe/redis/core/AbstractRedisTest.java +++ b/redis/redis-core/src/test/java/com/ctrip/xpipe/redis/core/AbstractRedisTest.java @@ -602,4 +602,16 @@ protected RedisMeta newRandomFakeRedisMeta(String ip, int port) { dcMeta.addCluster(clusterMeta); return redis; } + + protected KeeperMeta newRandomFakeKeeperMeta(String ip, int port) { + DcMeta dcMeta = new DcMeta("dc"); + ClusterMeta clusterMeta = new ClusterMeta("cluster"); + clusterMeta.setActiveDc("dc").setType(ClusterType.ONE_WAY.toString()).setActiveRedisCheckRules("0,1"); + ShardMeta shardMeta = new ShardMeta("shard"); + KeeperMeta keeper = new KeeperMeta().setIp(ip).setPort(port); + shardMeta.addKeeper(keeper); + clusterMeta.addShard(shardMeta); + dcMeta.addCluster(clusterMeta); + return keeper; + } } diff --git a/redis/redis-core/src/test/java/com/ctrip/xpipe/redis/core/AllTests.java b/redis/redis-core/src/test/java/com/ctrip/xpipe/redis/core/AllTests.java index e8026546e..07fc3710c 100644 --- a/redis/redis-core/src/test/java/com/ctrip/xpipe/redis/core/AllTests.java +++ b/redis/redis-core/src/test/java/com/ctrip/xpipe/redis/core/AllTests.java @@ -2,10 +2,7 @@ import com.ctrip.xpipe.redis.core.entity.DiskIOStatInfoTest; import com.ctrip.xpipe.redis.core.meta.*; -import com.ctrip.xpipe.redis.core.meta.comparator.ClusterMetaComparatorTest; -import com.ctrip.xpipe.redis.core.meta.comparator.DcMetaComparatorTest; -import com.ctrip.xpipe.redis.core.meta.comparator.DcRouteMetaComparatorTest; -import com.ctrip.xpipe.redis.core.meta.comparator.ShardMetaComparatorTest; +import com.ctrip.xpipe.redis.core.meta.comparator.*; import com.ctrip.xpipe.redis.core.meta.impl.DefaultXpipeMetaManagerTest; import com.ctrip.xpipe.redis.core.metaserver.META_SERVER_SERVICETest; import com.ctrip.xpipe.redis.core.metaserver.impl.DefaultReactorMetaServerConsoleServiceTest; @@ -28,12 +25,12 @@ import com.ctrip.xpipe.redis.core.proxy.parser.content.DefaultProxyContentParserTest; import com.ctrip.xpipe.redis.core.proxy.protocols.DefaultProxyConnectProtocolTest; import com.ctrip.xpipe.redis.core.redis.DefaultRunIdGeneratorTest; -import com.ctrip.xpipe.redis.core.redis.rdb.AllRdbTests; -import com.ctrip.xpipe.redis.core.route.impl.Crc32HashRouteChooseStrategyTest; import com.ctrip.xpipe.redis.core.redis.op.RedisOpDelTest; import com.ctrip.xpipe.redis.core.redis.op.RedisOpMsetTest; import com.ctrip.xpipe.redis.core.redis.parser.GeneralRedisOpParserTest; import com.ctrip.xpipe.redis.core.redis.parser.RedisReplStreamParseTest; +import com.ctrip.xpipe.redis.core.redis.rdb.AllRdbTests; +import com.ctrip.xpipe.redis.core.route.impl.Crc32HashRouteChooseStrategyTest; import com.ctrip.xpipe.redis.core.store.ReplicationStoreMetaTest; import com.ctrip.xpipe.redis.core.util.SentinelUtilTest; import org.junit.runner.RunWith; @@ -61,6 +58,7 @@ ReplicationStoreMetaTest.class, DcMetaComparatorTest.class, ClusterMetaComparatorTest.class, + KeeperContainerMetaComparatorTest.class, ShardMetaComparatorTest.class, InfoResultExtractorTest.class, CrdtInfoResultExtractorTest.class, diff --git a/redis/redis-core/src/test/java/com/ctrip/xpipe/redis/core/meta/comparator/KeeperContainerMetaComparatorTest.java b/redis/redis-core/src/test/java/com/ctrip/xpipe/redis/core/meta/comparator/KeeperContainerMetaComparatorTest.java new file mode 100644 index 000000000..15b508118 --- /dev/null +++ b/redis/redis-core/src/test/java/com/ctrip/xpipe/redis/core/meta/comparator/KeeperContainerMetaComparatorTest.java @@ -0,0 +1,103 @@ +package com.ctrip.xpipe.redis.core.meta.comparator; + +import com.ctrip.xpipe.redis.core.entity.*; +import com.ctrip.xpipe.redis.core.meta.MetaClone; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +public class KeeperContainerMetaComparatorTest extends AbstractComparatorTest { + + private DcMeta current; + private DcMeta future; + private DcMeta currentAllDcMeta; + private DcMeta futureAllDcMeta; + + @Before + public void init() { + current = getDc(); + future = MetaClone.clone(current); + currentAllDcMeta = MetaClone.clone(current); + futureAllDcMeta = MetaClone.clone(current); + + } + + @Test + public void testEquals() { + KeeperContainerMetaComparator comparator = new KeeperContainerMetaComparator(current, future, currentAllDcMeta, futureAllDcMeta); + comparator.compare(); + + Assert.assertEquals(0, comparator.getAdded().size()); + Assert.assertEquals(0, comparator.getRemoved().size()); + Assert.assertEquals(0, comparator.getMofified().size()); + } + + @Test + public void testAddOrRemoveKeeperContainer() { + KeeperContainerMeta newKeeperContainer = generateKeeperContainerMeta(4L, "127.0.0.4", 7080, future); + future.addKeeperContainer(newKeeperContainer); + futureAllDcMeta.addCluster(generateClusterMeta().addShard(generateShardMeta().addKeeper(generateKeeperMeta(newKeeperContainer)))); + + KeeperContainerMetaComparator comparator = new KeeperContainerMetaComparator(current, future, currentAllDcMeta, futureAllDcMeta); + comparator.compare(); + + Assert.assertEquals(1, comparator.getAdded().size()); + Assert.assertEquals(0, comparator.getRemoved().size()); + Assert.assertEquals(0, comparator.getMofified().size()); + + comparator = new KeeperContainerMetaComparator(future, current, futureAllDcMeta, currentAllDcMeta); + comparator.compare(); + + Assert.assertEquals(0, comparator.getAdded().size()); + Assert.assertEquals(1, comparator.getRemoved().size()); + Assert.assertEquals(0, comparator.getMofified().size()); + + } + + @Test + public void testWhenKeeperChange() { + ClusterMeta cluster1 = current.getClusters().get("cluster1"); + ClusterMeta newCluster1 = MetaClone.clone(cluster1); + newCluster1.getShards().get("shard1").addKeeper(new KeeperMeta().setKeeperContainerId(3L).setIp("127.0.0.3").setPort(randomPort())); + futureAllDcMeta.addCluster(newCluster1); + + KeeperContainerMetaComparator comparator = new KeeperContainerMetaComparator(current, future, currentAllDcMeta, futureAllDcMeta); + comparator.compare(); + + Assert.assertEquals(1, comparator.getAdded().size()); + Assert.assertEquals(0, comparator.getRemoved().size()); + Assert.assertEquals(0, comparator.getMofified().size()); + + comparator = new KeeperContainerMetaComparator(future, current,futureAllDcMeta, currentAllDcMeta); + comparator.compare(); + + Assert.assertEquals(0, comparator.getAdded().size()); + Assert.assertEquals(1, comparator.getRemoved().size()); + Assert.assertEquals(0, comparator.getMofified().size()); + + } + + protected KeeperContainerMeta generateKeeperContainerMeta(long id, String ip, int port, DcMeta parent) { + return new KeeperContainerMeta().setId(id).setIp(ip).setPort(port).setParent(parent); + } + + protected ClusterMeta generateClusterMeta() { + return new ClusterMeta().setId(randomString()).setDbId(Math.abs(randomLong())); + } + + protected KeeperMeta generateKeeperMeta(KeeperContainerMeta keeperContainer) { + return new KeeperMeta().setIp(keeperContainer.getIp()).setKeeperContainerId(keeperContainer.getId()) + .setPort(randomPort()).setActive(false); + } + + protected ShardMeta generateShardMeta() { + String shardName = randomString(); + return new ShardMeta().setId(shardName).setSentinelId(randomLong()).setSentinelMonitorName(shardName); + } + + + @Override + protected String getXpipeMetaConfigFile() { + return "KeeperContainerComparator.xml"; + } +} \ No newline at end of file diff --git a/redis/redis-core/src/test/resources/KeeperContainerComparator.xml b/redis/redis-core/src/test/resources/KeeperContainerComparator.xml new file mode 100644 index 000000000..2c5908ff6 --- /dev/null +++ b/redis/redis-core/src/test/resources/KeeperContainerComparator.xml @@ -0,0 +1,39 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/redis/redis-integration-test/src/test/java/com/ctrip/xpipe/redis/integratedtest/console/spring/console/TestCheckerContextConfig.java b/redis/redis-integration-test/src/test/java/com/ctrip/xpipe/redis/integratedtest/console/spring/console/TestCheckerContextConfig.java index 9901cc1bd..ad4000ff4 100644 --- a/redis/redis-integration-test/src/test/java/com/ctrip/xpipe/redis/integratedtest/console/spring/console/TestCheckerContextConfig.java +++ b/redis/redis-integration-test/src/test/java/com/ctrip/xpipe/redis/integratedtest/console/spring/console/TestCheckerContextConfig.java @@ -26,6 +26,7 @@ import com.ctrip.xpipe.redis.console.migration.auto.MonitorManager; import com.ctrip.xpipe.redis.console.redis.DefaultSentinelManager; import com.ctrip.xpipe.redis.console.resources.CheckerAllMetaCache; +import com.ctrip.xpipe.redis.console.resources.CheckerCurrentDcAllMeta; import com.ctrip.xpipe.redis.console.resources.CheckerMetaCache; import com.ctrip.xpipe.redis.console.resources.CheckerPersistenceCache; import com.ctrip.xpipe.redis.console.service.DcClusterShardService; @@ -33,6 +34,7 @@ import com.ctrip.xpipe.redis.console.service.meta.BeaconMetaService; import com.ctrip.xpipe.redis.console.service.meta.impl.BeaconMetaServiceImpl; import com.ctrip.xpipe.redis.console.util.DefaultMetaServerConsoleServiceManagerWrapper; +import com.ctrip.xpipe.redis.core.meta.CurrentDcAllMeta; import com.ctrip.xpipe.redis.core.meta.MetaCache; import com.ctrip.xpipe.redis.integratedtest.console.config.SpringEnvConsoleConfig; import com.ctrip.xpipe.redis.integratedtest.console.config.TestFoundationService; @@ -60,6 +62,12 @@ public PersistenceCache persistenceCache(CheckerConfig checkerConfig, CheckerCon return new CheckerPersistenceCache(checkerConfig, checkerConsoleService); } + @Bean + @Profile(AbstractProfile.PROFILE_NAME_PRODUCTION) + public CurrentDcAllMeta currentDcAllMeta() { + return new CheckerCurrentDcAllMeta(); + } + @Bean @Profile(AbstractProfile.PROFILE_NAME_PRODUCTION) public MetaCache metaCache(CheckerConfig checkerConfig, CheckerConsoleService checkerConsoleService) { @@ -77,6 +85,12 @@ public MetaCache testMetaCache() { return new TestMetaCache(); } + @Bean + @Profile(AbstractProfile.PROFILE_NAME_TEST) + public CurrentDcAllMeta testCurrentDcAllMeta() { + return new TestCurrentDcAllMetaCache(); + } + @Bean public ProxyManager proxyManager(GroupCheckerLeaderElector clusterServer, CheckerConfig checkerConfig, CheckerConsoleService checkerConsoleService) { return new CheckerProxyManager(clusterServer, checkerConfig, checkerConsoleService); diff --git a/redis/redis-integration-test/src/test/java/com/ctrip/xpipe/redis/integratedtest/console/spring/console/TestConsoleContextConfig.java b/redis/redis-integration-test/src/test/java/com/ctrip/xpipe/redis/integratedtest/console/spring/console/TestConsoleContextConfig.java index 3e8fae2f6..a274948f4 100644 --- a/redis/redis-integration-test/src/test/java/com/ctrip/xpipe/redis/integratedtest/console/spring/console/TestConsoleContextConfig.java +++ b/redis/redis-integration-test/src/test/java/com/ctrip/xpipe/redis/integratedtest/console/spring/console/TestConsoleContextConfig.java @@ -6,6 +6,7 @@ import com.ctrip.xpipe.redis.checker.PersistenceCache; import com.ctrip.xpipe.redis.checker.config.CheckerConfig; import com.ctrip.xpipe.redis.checker.healthcheck.actions.ping.PingService; +import com.ctrip.xpipe.redis.checker.impl.TestCurrentDcAllMetaCache; import com.ctrip.xpipe.redis.checker.impl.TestMetaCache; import com.ctrip.xpipe.redis.checker.spring.ConsoleServerMode; import com.ctrip.xpipe.redis.checker.spring.ConsoleServerModeCondition; @@ -28,13 +29,13 @@ import com.ctrip.xpipe.redis.console.service.impl.DefaultCrossMasterDelayService; import com.ctrip.xpipe.redis.console.sso.UserAccessFilter; import com.ctrip.xpipe.redis.console.util.DefaultMetaServerConsoleServiceManagerWrapper; +import com.ctrip.xpipe.redis.core.meta.CurrentDcAllMeta; import com.ctrip.xpipe.redis.core.meta.MetaCache; import com.ctrip.xpipe.redis.core.route.RouteChooseStrategyFactory; import com.ctrip.xpipe.redis.core.route.impl.DefaultRouteChooseStrategyFactory; import com.ctrip.xpipe.redis.integratedtest.console.config.SpringEnvConsoleConfig; import com.ctrip.xpipe.redis.integratedtest.console.config.TestFoundationService; import com.ctrip.xpipe.spring.AbstractProfile; -import org.springframework.boot.SpringBootConfiguration; import org.springframework.boot.autoconfigure.SpringBootApplication; import org.springframework.boot.test.context.SpringBootTest; import org.springframework.boot.web.servlet.FilterRegistrationBean; @@ -109,6 +110,13 @@ public MetaCache testMetaCache() { return new TestMetaCache(); } + @Bean + @Profile(AbstractProfile.PROFILE_NAME_TEST) + public CurrentDcAllMeta testCurrentDcAllMeta() { + return new TestCurrentDcAllMetaCache(); + } + + @Bean public ConsoleDbConfig consoleDbConfig() { return new DefaultConsoleDbConfig();