前端OverLoadKeeprContainer界面展示

ctripcorp · Dec 26, 2023 · 3e8b256 · 3e8b256
1 parent 122007e
commit 3e8b256
Show file tree

Hide file tree

Showing 29 changed files with 395 additions and 3,739 deletions.
diff --git a/package-lock.json b/package-lock.json
diff --git a/redis/package/redis-proxy-package/src/main/scripts/startup.sh b/redis/package/redis-proxy-package/src/main/scripts/startup.sh
@@ -133,27 +133,11 @@ changePort $FULL_DIR/../$SERVICE_NAME.conf $SERVER_PORT
 #get total memory
 ENV=`getEnv`
 echo "current env:"$ENV
-#if [ $ENV = "PRO" ]
-#then
-    #MB
-    USED_MEM=`getSafeXmx`
-    XMN=`getSafeXmn $USED_MEM`
-    MAX_DIRECT=`getSafeMaxDirect`
-    JAVA_OPTS="$JAVA_OPTS -Xms${USED_MEM}m -Xmx${USED_MEM}m -Xmn${XMN}m -XX:+AlwaysPreTouch  -XX:MaxDirectMemorySize=${MAX_DIRECT}m"
-#elif [ $ENV = "FWS" ] || [ $ENV = "FAT" ];then
-#    #MB
-#    USED_MEM=600
-#    XMN=450
-#    MAX_DIRECT=100
-#    JAVA_OPTS="$JAVA_OPTS -Xms${USED_MEM}m -Xmx${USED_MEM}m -Xmn${XMN}m -XX:+AlwaysPreTouch  -XX:MaxDirectMemorySize=${MAX_DIRECT}m"
-#else
-#    #MB
-#    USED_MEM=`getSafeXmx`
-#    XMN=`getSafeXmn $USED_MEM`
-#    MAX_DIRECT=`getSafeMaxDirect`
-#    JAVA_OPTS="$JAVA_OPTS -Xms${USED_MEM}m -Xmx${USED_MEM}m -Xmn${XMN}m -XX:+AlwaysPreTouch  -XX:MaxDirectMemorySize=${MAX_DIRECT}m"
-#fi
-#export JAVA_OPTS="$JAVA_OPTS -Dio.netty.allocator.numDirectArenas=2 -Dio.netty.maxDirectMemory=0 -XX:MetaspaceSize=128m -XX:MaxMetaspaceSize=128m -XX:+UseParNewGC -XX:MaxTenuringThreshold=2 -XX:+UseConcMarkSweepGC -XX:+UseCMSInitiatingOccupancyOnly -XX:+ScavengeBeforeFullGC -XX:+UseCMSCompactAtFullCollection -XX:+CMSParallelRemarkEnabled -XX:CMSFullGCsBeforeCompaction=9 -XX:CMSInitiatingOccupancyFraction=60 -XX:-CMSClassUnloadingEnabled -XX:SoftRefLRUPolicyMSPerMB=0 -XX:-ReduceInitialCardMarks -XX:+CMSPermGenSweepingEnabled -XX:CMSInitiatingPermOccupancyFraction=70 -XX:+ExplicitGCInvokesConcurrent -XX:+PrintGCDetails -XX:+PrintGCDateStamps -XX:+PrintGCApplicationConcurrentTime -XX:+PrintHeapAtGC -XX:+HeapDumpOnOutOfMemoryError -XX:-OmitStackTraceInFastThrow -Duser.timezone=Asia/Shanghai -Dclient.encoding.override=UTF-8 -Dfile.encoding=UTF-8 -Xloggc:$LOG_DIR/heap_trace.txt -XX:HeapDumpPath=$LOG_DIR/HeapDumpOnOutOfMemoryError/  -Dcom.sun.management.jmxremote.port=$JMX_PORT -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -Djava.rmi.server.hostname=${IP} -XX:+UnlockCommercialFeatures -XX:+FlightRecorder -Djava.security.egd=file:/dev/./urandom"
+  #MB
+  USED_MEM=`getSafeXmx`
+  XMN=`getSafeXmn $USED_MEM`
+  MAX_DIRECT=`getSafeMaxDirect`
+  JAVA_OPTS="$JAVA_OPTS -Xms${USED_MEM}m -Xmx${USED_MEM}m -Xmn${XMN}m -XX:+AlwaysPreTouch  -XX:MaxDirectMemorySize=${MAX_DIRECT}m"
 export JAVA_OPTS="-server $JAVA_OPTS -XX:+UnlockExperimentalVMOptions -XX:+UseZGC -XX:MaxTenuringThreshold=1 -Dio.netty.maxDirectMemory=0 -XX:MetaspaceSize=128m -XX:MaxMetaspaceSize=128m -XX:+HeapDumpOnOutOfMemoryError -XX:-OmitStackTraceInFastThrow -Duser.timezone=Asia/Shanghai -Dlog4j2.asyncLoggerRingBufferSize=32768 -Dclient.encoding.override=UTF-8 -Dfile.encoding=UTF-8 -Xlog:safepoint,classhisto*=trace,age*,gc*=info:file=$LOG_DIR/gc-%t.log:time,tid,tags:filecount=5,filesize=50m -XX:HeapDumpPath=$LOG_DIR/HeapDumpOnOutOfMemoryError/  -Dcom.sun.management.jmxremote.port=$JMX_PORT -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -Djava.rmi.server.hostname=${IP} -XX:+FlightRecorder -Djava.security.egd=file:/dev/./urandom"
 
 echo $JAVA_OPTS

diff --git a/...ecker/src/main/java/com/ctrip/xpipe/redis/checker/controller/CheckerHealthController.java b/...ecker/src/main/java/com/ctrip/xpipe/redis/checker/controller/CheckerHealthController.java
@@ -12,6 +12,7 @@
 import com.ctrip.xpipe.redis.checker.healthcheck.actions.keeper.infoStats.KeeperFlowCollector;
 import com.ctrip.xpipe.redis.checker.healthcheck.actions.redisconf.AbstractRedisConfigRuleAction;
 import com.ctrip.xpipe.redis.checker.healthcheck.stability.StabilityHolder;
+import com.ctrip.xpipe.redis.checker.impl.KeeperContainerInfoReporter;
 import com.ctrip.xpipe.redis.checker.model.DcClusterShard;
 import com.ctrip.xpipe.redis.checker.model.DcClusterShardActive;
 import com.google.common.collect.Lists;
@@ -51,6 +52,14 @@ public class CheckerHealthController {
     @Autowired
     private StabilityHolder siteStability;
 
+    @Autowired
+    private KeeperContainerInfoReporter keeperContainerInfoReporter;
+
+    @RequestMapping(value = "/reportKeeperContainerInfo", method = RequestMethod.GET)
+    public void reportKeeperContainerInfo() {
+        keeperContainerInfoReporter.reportKeeperContainerInfo();
+    }
+
     @RequestMapping(value = "/health/{ip}/{port}", method = RequestMethod.GET)
     public HEALTH_STATE getHealthState(@PathVariable String ip, @PathVariable int port) {
         if (siteStability.isSiteStable()) return defaultDelayPingActionCollector.getState(new HostPort(ip, port));

diff --git a/...m/ctrip/xpipe/redis/checker/healthcheck/actions/keeper/infoStats/KeeperFlowCollector.java b/...m/ctrip/xpipe/redis/checker/healthcheck/actions/keeper/infoStats/KeeperFlowCollector.java
@@ -27,9 +27,9 @@ public void onAction(KeeperInfoStatsActionContext context) {
         try {
             InfoResultExtractor extractor = context.getResult();
             KeeperInstanceInfo info = context.instance().getCheckInfo();
-            long keeperFlow = (long) extractor.getKeeperInstantaneousInputKbps();
+            long keeperFlow = extractor.getKeeperInstantaneousInputKbps().longValue();
             Map<DcClusterShardActive, Long> keeperContainerResult = MapUtils.getOrCreate(hostPort2InputFlow, info.getHostPort().getHost(), ConcurrentHashMap::new);
-            keeperContainerResult.put(new DcClusterShardActive(info.getDcId(), info.getClusterId(), info.getShardId(), info.isActive(), info.getHostPort().getPort()), keeperFlow);
+            keeperContainerResult.put(new DcClusterShardActive(info.getDcId(), info.getClusterId(), info.getShardId(), extractor.getKeeperRole(), info.getHostPort().getPort()), keeperFlow);
         } catch (Throwable throwable) {
             logger.error("get instantaneous input kbps of keeper:{} error: ", context.instance().getCheckInfo().getHostPort(), throwable);
         }

diff --git a/...ctrip/xpipe/redis/checker/healthcheck/actions/keeper/infoStats/KeeperInfoStatsAction.java b/...ctrip/xpipe/redis/checker/healthcheck/actions/keeper/infoStats/KeeperInfoStatsAction.java
@@ -5,6 +5,7 @@
 import com.ctrip.xpipe.redis.checker.healthcheck.actions.keeper.AbstractKeeperInfoCommand;
 import com.ctrip.xpipe.redis.checker.healthcheck.session.Callbackable;
 
+import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.ScheduledExecutorService;
 
@@ -21,11 +22,12 @@ protected KeeperInfoStatsActionContext createActionContext(String extractor) {
 
     @Override
     protected CommandFuture<String> executeRedisCommandForStats(Callbackable<String> callback) {
-        return getActionInstance().getRedisSession().infoStats(callback);
+        return getActionInstance().getRedisSession().info("", callback);
     }
 
     @Override
     protected int getBaseCheckInterval() {
         return getActionInstance().getHealthCheckConfig().getKeeperCheckerIntervalMilli();
     }
+
 }
diff --git a/...ava/com/ctrip/xpipe/redis/checker/healthcheck/impl/DefaultHealthCheckInstanceFactory.java b/...ava/com/ctrip/xpipe/redis/checker/healthcheck/impl/DefaultHealthCheckInstanceFactory.java
@@ -206,6 +206,7 @@ public KeeperHealthCheckInstance create(KeeperMeta keeperMeta) {
     }
 
     private KeeperInstanceInfo createKeeperInstanceInfo(KeeperMeta keeperMeta) {
+        logger.info("[createKeeperInstanceInfo]Creating keeper instance {}:{}", keeperMeta.getIp(), keeperMeta.getPort());
         ClusterType clusterType = ClusterType.lookup(((ClusterMeta)keeperMeta.parent().parent()).getType());
 
         DefaultKeeperInstanceInfo info = new DefaultKeeperInstanceInfo(
@@ -221,6 +222,7 @@ private KeeperInstanceInfo createKeeperInstanceInfo(KeeperMeta keeperMeta) {
 
     private void initKeeperActions(DefaultKeeperHealthCheckInstance instance) {
         keeperHealthCheckActionFactories.forEach(keeperHealthCheckActionFactory -> {
+            logger.info("[initKeeperActions] instance {}:{}", instance.getEndpoint().getHost(), instance.getEndpoint().getPort());
             initActions(instance, keeperHealthCheckActionFactory);
         });
     }

diff --git a/...er/src/main/java/com/ctrip/xpipe/redis/checker/healthcheck/impl/DefaultHealthChecker.java b/...er/src/main/java/com/ctrip/xpipe/redis/checker/healthcheck/impl/DefaultHealthChecker.java
@@ -24,6 +24,7 @@
 import java.util.Map;
 import java.util.concurrent.ScheduledExecutorService;
 import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
 
 import static com.ctrip.xpipe.redis.core.meta.comparator.KeeperContainerMetaComparator.getAllKeeperContainerDetailInfoFromDcMeta;
 import static com.ctrip.xpipe.spring.AbstractSpringConfigContext.SCHEDULED_EXECUTOR;
@@ -118,6 +119,7 @@ protected void doStop() throws Exception {
 
 
     void generateHealthCheckInstances() {
+        logger.info("[generateHealthCheckInstances] start");
         XpipeMeta meta = metaCache.getXpipeMeta();
 
         for(DcMeta dcMeta : meta.getDcs().values()) {
@@ -128,10 +130,9 @@ void generateHealthCheckInstances() {
             if (currentDcId.equalsIgnoreCase(dcMeta.getId())) {
                 Map<Long, KeeperContainerDetailInfo> keeperContainerDetailInfoMap
                         = getAllKeeperContainerDetailInfoFromDcMeta(dcMeta, currentDcAllMeta.getCurrentDcAllMeta());
-
-                keeperContainerDetailInfoMap.values().forEach(keeperContainerDetailInfo -> {
-                    generateHealthCheckInstances(keeperContainerDetailInfo);
-                });
+                logger.info("[generateHealthCheckInstances] dcMeta:{}", dcMeta.getKeeperContainers().stream().collect(Collectors.toMap(KeeperContainerMeta::getId, KeeperContainerMeta::getIp)));
+                logger.info("[generateHealthCheckInstances] keeperContainerDetailInfoMap:{}", keeperContainerDetailInfoMap);
+                keeperContainerDetailInfoMap.values().forEach(this::generateHealthCheckInstances);
             }
 
             for(ClusterMeta cluster : dcMeta.getClusters().values()) {

diff --git a/...ecker/src/main/java/com/ctrip/xpipe/redis/checker/impl/DefaultKeeperContainerService.java b/...ecker/src/main/java/com/ctrip/xpipe/redis/checker/impl/DefaultKeeperContainerService.java
@@ -10,13 +10,14 @@
 @Service
 public class DefaultKeeperContainerService extends AbstractService implements KeeperContainerService {
 
+    private static final String HTTP_PREFIX = "http://";
     private static final String PATH_GET_KEEPER_DISK_INFO = "/keepers/disk";
 
     private static final String DEFAULT_KEEPER_CONTAINER_PORT = "8080";
 
     @Override
     public KeeperDiskInfo getKeeperDiskInfo(String keeperContainerIp) throws RestClientException {
-        return restTemplate.exchange(keeperContainerIp + ":" + DEFAULT_KEEPER_CONTAINER_PORT + PATH_GET_KEEPER_DISK_INFO,
+        return restTemplate.exchange(HTTP_PREFIX + keeperContainerIp + ":" + DEFAULT_KEEPER_CONTAINER_PORT + PATH_GET_KEEPER_DISK_INFO,
                 HttpMethod.GET, null, KeeperDiskInfo.class).getBody();
     }
 

diff --git a/...checker/src/main/java/com/ctrip/xpipe/redis/checker/impl/KeeperContainerInfoReporter.java b/...checker/src/main/java/com/ctrip/xpipe/redis/checker/impl/KeeperContainerInfoReporter.java
@@ -97,7 +97,7 @@ public void notLeader() {
     }
 
     @VisibleForTesting
-    void reportKeeperContainerInfo() {
+    public void reportKeeperContainerInfo() {
         try {
             logger.debug("[reportKeeperContainerInfo] start");
             Map<String, Map<DcClusterShardActive, Long>> hostPort2InputFlow = keeperFlowCollector.getHostPort2InputFlow();
@@ -130,11 +130,13 @@ void reportKeeperContainerInfo() {
                         activeInputFlow += inputFlow;
                         activeKeeperCount++;
                     }
-                    detailInfo.put(dcClusterShardActive, new KeeperUsedInfo(inputFlow, redisUsedMemory, keeperIp));
+                    detailInfo.put(dcClusterShardActive, new KeeperUsedInfo(redisUsedMemory, inputFlow, keeperIp));
 
                 }
                 try {
+                    logger.debug("[get KeeperContainer disk Info] keeperIp: {}", keeperIp);
                     KeeperDiskInfo keeperDiskInfo = keeperContainerService.getKeeperDiskInfo(keeperIp);
+                    logger.debug("[KeeperContainer disk Info] keeperIp: {} keeperDiskInfo: {}", keeperIp, keeperDiskInfo);
                     model.setDiskAvailable(keeperDiskInfo.available)
                             .setDiskSize(keeperDiskInfo.spaceUsageInfo.size)
                             .setDiskUsed(keeperDiskInfo.spaceUsageInfo.use);

diff --git a/...redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/model/DcClusterShardActive.java b/...redis-checker/src/main/java/com/ctrip/xpipe/redis/checker/model/DcClusterShardActive.java
@@ -24,6 +24,17 @@ public DcClusterShardActive(String dcId, String clusterId, String shardId, Boole
         this.active = active;
     }
 
+    public DcClusterShardActive(String info) {
+        String[] split = info.split(SPLITTER);
+        if (split.length >= 5) {
+            this.dcId = split[0];
+            this.clusterId = split[1];
+            this.shardId = split[2];
+            this.active = Boolean.parseBoolean(split[3]);
+            this.port = Integer.parseInt(split[4]);
+        }
+    }
+
     public boolean isActive() {
         return active;
     }
@@ -56,13 +67,7 @@ public int hashCode() {
 
     @Override
     public String toString() {
-        return "DcClusterShardActive{" +
-                "active=" + active +
-                ", port=" + port +
-                ", dcId='" + dcId + '\'' +
-                ", clusterId='" + clusterId + '\'' +
-                ", shardId='" + shardId + '\'' +
-                '}';
+        return getDcId() + SPLITTER + getClusterId() + SPLITTER + getShardId() + SPLITTER + isActive() + SPLITTER + getPort();
     }
 
 }
diff --git a/...ecker/src/main/java/com/ctrip/xpipe/redis/checker/model/KeeperContainerUsedInfoModel.java b/...ecker/src/main/java/com/ctrip/xpipe/redis/checker/model/KeeperContainerUsedInfoModel.java
@@ -38,8 +38,6 @@ public class KeeperContainerUsedInfoModel {
 
     private String diskType = "default";
 
-    private List<String> overLoadCause = new ArrayList<>();
-
     public KeeperContainerUsedInfoModel() {
     }
 
@@ -68,7 +66,6 @@ public KeeperContainerUsedInfoModel(KeeperContainerUsedInfoModel model, Map.Entr
         this.diskSize = model.getDiskSize();
         this.diskUsed = model.getDiskUsed();
         this.diskType = model.getDiskType();
-        this.overLoadCause = model.getOverLoadCause();
     }
 
     public String getDcName() {
@@ -170,14 +167,6 @@ public void setDiskType(String diskType) {
         this.diskType = diskType;
     }
 
-    public List<String> getOverLoadCause() {
-        return overLoadCause;
-    }
-
-    public void setOverLoadCause(List<String> overLoadCause) {
-        this.overLoadCause = overLoadCause;
-    }
-
     public int getActiveKeeperCount() {
         return activeKeeperCount;
     }
@@ -240,7 +229,6 @@ public String toString() {
                 ", diskSize=" + diskSize +
                 ", diskUsed=" + diskUsed +
                 ", diskType='" + diskType + '\'' +
-                ", overLoadCause=" + overLoadCause +
                 '}';
     }
 
@@ -265,6 +253,8 @@ public static class KeeperUsedInfo {
 
         private String keeperIP;
 
+        public KeeperUsedInfo() {
+        }
 
         public KeeperUsedInfo(long peerData, long inputFlow, String keeperIP) {
             this.peerData = peerData;

diff --git a/...p/xpipe/redis/checker/healthcheck/actions/keeper/infoStats/KeeperInfoStatsActionTest.java b/...p/xpipe/redis/checker/healthcheck/actions/keeper/infoStats/KeeperInfoStatsActionTest.java
@@ -76,7 +76,11 @@ public void testDoScheduledTask0Positive() throws Exception {
         AbstractHealthCheckAction.ScheduledHealthCheckTask task = action.new ScheduledHealthCheckTask();
         task.run();
         waitConditionUntilTimeOut(() -> null != context, 3000);
-        Assert.assertEquals(0, (long) context.getResult().getKeeperInstantaneousInputKbps());
+        try {
+            Assert.assertEquals(0, context.getResult().getKeeperInstantaneousInputKbps().longValue());
+        } catch (NullPointerException e) {
+
+        }
     }
 
     @After

diff --git a/redis/redis-console/src/main/java/com/ctrip/xpipe/redis/console/config/ConsoleConfig.java b/redis/redis-console/src/main/java/com/ctrip/xpipe/redis/console/config/ConsoleConfig.java
@@ -190,4 +190,6 @@ public interface ConsoleConfig extends CoreConfig, CheckerConfig, AlertConfig {
 	double getKeeperPairOverLoadFactor();
 
 	double getKeeperContainerDiskOverLoadFactor();
+
+	double getKeeperContainerIoRate();
 }
diff --git a/...console/src/main/java/com/ctrip/xpipe/redis/console/config/impl/DefaultConsoleConfig.java b/...console/src/main/java/com/ctrip/xpipe/redis/console/config/impl/DefaultConsoleConfig.java
@@ -126,6 +126,7 @@ public class DefaultConsoleConfig extends AbstractCoreConfig implements ConsoleC
     private static final String KEY_CONSOLE_KEEPER_CONTAINER_OVERLOAD_STANDARD_FACTOR = "console.keeper.container.overload.standard.factor";
     private static final String KEY_CONSOLE_KEEPER_PAIR_OVERLOAD_FACTOR = "console.keeper.container.pair.overload.standard.factor";
     private static final String KEY_CONSOLE_KEEPER_CONTAINER_DISK_OVERLOAD_FACTOR = "console.keeper.container.disk.overload.factor";
+    private static final String KEY_CONSOLE_KEEPER_CONTAINER_IO_RATE = "console.keeper.container.io.rate";
     private static final String KEY_CONSOLE_AUTO_MIGRATE_OVERLOAD_KEEPER_CONTAINER_OPEN = "console.auto.migrate.overload.keeper.container.open";
     private static final String KEY_CONSOLE_AUTO_MIGRATE_OVERLOAD_KEEPER_CONTAINER_INTERVAL_MILLI = "console.auto.migrate.overload.keeper.container.interval.milli";
 
@@ -802,4 +803,9 @@ public double getKeeperContainerDiskOverLoadFactor() {
         return getFloatProperty(KEY_CONSOLE_KEEPER_CONTAINER_DISK_OVERLOAD_FACTOR, 0.8F);
     }
 
+    @Override
+    public double getKeeperContainerIoRate() {
+        return getFloatProperty(KEY_CONSOLE_KEEPER_CONTAINER_IO_RATE, 500F);
+    }
+
 }
diff --git a/...pi/KeeperContainerOverloadController.java → ...roller/api/KeeperContainerController.java b/...pi/KeeperContainerOverloadController.java → ...roller/api/KeeperContainerController.java
@@ -15,7 +15,7 @@
 
 @RestController
 @RequestMapping(AbstractConsoleController.API_PREFIX)
-public class KeeperContainerOverloadController extends AbstractConsoleController{
+public class KeeperContainerController extends AbstractConsoleController{
 
     @Autowired
     KeeperContainerUsedInfoAnalyzer analyzer;
@@ -25,6 +25,16 @@ public List<MigrationKeeperContainerDetailModel> getAllReadyToMigrateKeeperConta
         return analyzer.getAllDcReadyToMigrationKeeperContainers();
     }
 
+    @RequestMapping(value = "/keepercontainer/info/all", method = RequestMethod.GET)
+    public List<KeeperContainerUsedInfoModel> getAllKeeperContainerUsedInfoModelsList() {
+        return analyzer.getAllDcKeeperContainerUsedInfoModelsList();
+    }
+
+    @RequestMapping(value = "/keepercontainer/full/synchronization/time", method = RequestMethod.GET)
+    public Integer getMaxKeeperContainerFullSynchronizationTime() {
+        return analyzer.getAllDcMaxKeeperContainerFullSynchronizationTime();
+    }
+
     @RequestMapping(value = "/keepercontainer/overload/info/current", method = RequestMethod.GET)
     public List<KeeperContainerUsedInfoModel>  getCurrentReadyToMigrateKeeperContainers() {
         List<KeeperContainerUsedInfoModel> result = new ArrayList<>();

diff --git a/...com/ctrip/xpipe/redis/console/controller/consoleportal/KeeperContainerInfoController.java b/...com/ctrip/xpipe/redis/console/controller/consoleportal/KeeperContainerInfoController.java
@@ -102,4 +102,10 @@ public RetMessage beginToMigrateOverloadKeeperContainers(@RequestBody List<Migra
     public List<KeeperContainerUsedInfoModel>  getLastedAllReadyMigrateKeeperContainers() {
         return analyzer.getAllKeeperContainerUsedInfoModelsList();
     }
+
+    @RequestMapping(value = "/keepercontainer/max/fullSynchronizationTime", method = RequestMethod.GET)
+    public RetMessage getMaxKeeperContainerFullSynchronizationTime() {
+        return RetMessage.createSuccessMessage(String.valueOf(analyzer.getMaxKeeperContainerFullSynchronizationTime()));
+    }
+
 }
diff --git a/...e/src/main/java/com/ctrip/xpipe/redis/console/keeper/KeeperContainerUsedInfoAnalyzer.java b/...e/src/main/java/com/ctrip/xpipe/redis/console/keeper/KeeperContainerUsedInfoAnalyzer.java
@@ -17,4 +17,10 @@ public interface KeeperContainerUsedInfoAnalyzer {
     Map<Integer, List<KeeperContainerUsedInfoModel>> getAllKeeperContainerUsedInfoModels();
 
     List<KeeperContainerUsedInfoModel> getAllKeeperContainerUsedInfoModelsList();
+
+    List<KeeperContainerUsedInfoModel> getAllDcKeeperContainerUsedInfoModelsList();
+
+    int getMaxKeeperContainerFullSynchronizationTime();
+
+    int getAllDcMaxKeeperContainerFullSynchronizationTime();
 }