Skip to content

Commit

Permalink
跨region拉入拉出检测优化
Browse files Browse the repository at this point in the history
  • Loading branch information
yifuzhou committed Aug 15, 2024
1 parent 5032c49 commit 47f2335
Show file tree
Hide file tree
Showing 62 changed files with 1,357 additions and 152 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,9 @@ public interface CheckerConsoleService {

Map<String, Date> loadAllClusterCreateTime(String console);

Map<String, OuterClientService.ClusterInfo> loadAllActiveDcOneWayClusterInfo(String console, String activeDc);
Map<String, OuterClientService.ClusterInfo> loadAllDcOneWayClusterInfo(String console, String dc);

Map<String, OuterClientService.ClusterInfo> loadCurrentDcOneWayClusterInfo(String console, String dc);

void bindShardSentinel(String console, String dc, String cluster, String shard, SentinelMeta sentinelMeta);

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,10 @@ class AllInstanceHealthStatus extends HashMap<HostPort, HealthStatusDesc> {}

HEALTH_STATE getInstanceStatus(String ip, int port);

HEALTH_STATE getCrossRegionInstanceStatus(String ip, int port);

Map<HostPort, HealthStatusDesc> getAllInstanceHealthStatus();

Map<HostPort, HealthStatusDesc> getAllInstanceCrossRegionHealthStatus();

}
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,8 @@ public interface OuterClientCache {

OuterClientService.ClusterInfo getClusterInfo(String clusterName) throws Exception;

Map<String, OuterClientService.ClusterInfo> getAllActiveDcClusters(String activeDc);
Map<String, OuterClientService.ClusterInfo> getAllDcClusters(String dc);

Map<String, OuterClientService.ClusterInfo> getAllCurrentDcClusters(String dc);

}
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
import com.ctrip.xpipe.redis.checker.controller.result.ActionContextRetMessage;
import com.ctrip.xpipe.redis.checker.healthcheck.*;
import com.ctrip.xpipe.redis.checker.healthcheck.actions.interaction.DefaultDelayPingActionCollector;
import com.ctrip.xpipe.redis.checker.healthcheck.actions.interaction.DefaultPsubPingActionCollector;
import com.ctrip.xpipe.redis.checker.healthcheck.actions.interaction.HEALTH_STATE;
import com.ctrip.xpipe.redis.checker.healthcheck.actions.interaction.HealthStatusDesc;
import com.ctrip.xpipe.redis.checker.healthcheck.actions.keeper.info.RedisUsedMemoryCollector;
Expand Down Expand Up @@ -36,6 +37,9 @@ public class CheckerHealthController {
@Autowired
private DefaultDelayPingActionCollector defaultDelayPingActionCollector;

@Autowired
private DefaultPsubPingActionCollector defaultPsubPingActionCollector;

@Autowired
private RedisUsedMemoryCollector redisUsedMemoryCollector;

Expand All @@ -57,6 +61,12 @@ public HEALTH_STATE getHealthState(@PathVariable String ip, @PathVariable int po
else return HEALTH_STATE.UNKNOWN;
}

@RequestMapping(value = "/health/cross/region/{ip}/{port}", method = RequestMethod.GET)
public HEALTH_STATE getCrossRegionHealthState(@PathVariable String ip, @PathVariable int port) {
if (siteStability.isSiteStable()) return defaultPsubPingActionCollector.getHealthState(new HostPort(ip, port));
else return HEALTH_STATE.UNKNOWN;
}

@RequestMapping(value = "/health/check/instance/{ip}/{port}", method = RequestMethod.GET)
public String getHealthCheckInstance(@PathVariable String ip, @PathVariable int port) {
RedisHealthCheckInstance instance = instanceManager.findRedisHealthCheckInstance(new HostPort(ip, port));
Expand All @@ -67,6 +77,16 @@ public String getHealthCheckInstance(@PathVariable String ip, @PathVariable int
return Codec.DEFAULT.encode(model);
}

@RequestMapping(value = "/health/check/cross/region//instance/{ip}/{port}", method = RequestMethod.GET)
public String getCrossRegionHealthCheckInstance(@PathVariable String ip, @PathVariable int port) {
RedisHealthCheckInstance instance = instanceManager.findRedisInstanceForPsubPingAction(new HostPort(ip, port));
if(instance == null) {
return "Not found";
}
HealthCheckInstanceModel model = buildHealthCheckInfo(instance);
return Codec.DEFAULT.encode(model);
}

@RequestMapping(value = "/health/check/cluster/{clusterId}", method = RequestMethod.GET)
public String getClusterHealthCheckInstance(@PathVariable String clusterId) {
ClusterHealthCheckInstance instance = instanceManager.findClusterHealthCheckInstance(clusterId);
Expand Down Expand Up @@ -97,6 +117,16 @@ public String getHealthCheckRedisInstanceForAssignedAction(@PathVariable String
return Codec.DEFAULT.encode(model);
}

@RequestMapping(value = "/health/check/redis-for-ping-action/{ip}/{port}", method = RequestMethod.GET)
public String getHealthCheckRedisInstanceForPingAction(@PathVariable String ip, @PathVariable int port) {
RedisHealthCheckInstance instance = instanceManager.findRedisInstanceForPsubPingAction(new HostPort(ip, port));
if(instance == null) {
return "Not found";
}
HealthCheckInstanceModel model = buildHealthCheckInfo(instance);
return Codec.DEFAULT.encode(model);
}

@RequestMapping(value = "/health/redis/info/{ip}/{port}", method = RequestMethod.GET)
public ActionContextRetMessage<Map<String, String>> getRedisInfo(@PathVariable String ip, @PathVariable int port) {
return ActionContextRetMessage.from(redisInfoManager.getInfoByHostPort(new HostPort(ip, port)));
Expand All @@ -113,6 +143,12 @@ public Map<HostPort, HealthStatusDesc> getAllHealthStatusDesc() {
else return Collections.emptyMap();
}

@GetMapping("/health/check/cross/region/status/all")
public Map<HostPort, HealthStatusDesc> getAllCrossRegionHealthStatusDesc() {
if (siteStability.isSiteStable()) return defaultPsubPingActionCollector.getAllHealthStatus();
else return Collections.emptyMap();
}

@GetMapping("/health/keeper/status/all")
public ConcurrentMap<String, Map<DcClusterShardKeeper, Long>> getAllKeeperFlows() {
return keeperFlowCollector.getHostPort2InputFlow();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,8 @@ public interface HealthCheckInstanceManager {

RedisHealthCheckInstance getOrCreateRedisInstanceForAssignedAction(RedisMeta redis);

RedisHealthCheckInstance getOrCreateRedisInstanceForPsubPingAction(RedisMeta redis);

KeeperHealthCheckInstance getOrCreate(KeeperMeta keeper);

ClusterHealthCheckInstance getOrCreate(ClusterMeta cluster);
Expand All @@ -26,6 +28,8 @@ public interface HealthCheckInstanceManager {

RedisHealthCheckInstance findRedisInstanceForAssignedAction(HostPort hostPort);

RedisHealthCheckInstance findRedisInstanceForPsubPingAction(HostPort hostPort);

KeeperHealthCheckInstance findKeeperHealthCheckInstance(HostPort hostPort);

ClusterHealthCheckInstance findClusterHealthCheckInstance(String clusterId);
Expand All @@ -36,6 +40,8 @@ public interface HealthCheckInstanceManager {

RedisHealthCheckInstance removeRedisInstanceForAssignedAction(HostPort hostPort);

RedisHealthCheckInstance removeRedisInstanceForPingAction(HostPort hostPort);

ClusterHealthCheckInstance remove(String cluster);

List<RedisHealthCheckInstance> getAllRedisInstance();
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,94 @@
package com.ctrip.xpipe.redis.checker.healthcheck.actions.interaction;

import com.ctrip.xpipe.redis.checker.healthcheck.ActionContext;
import com.ctrip.xpipe.redis.checker.healthcheck.HealthCheckAction;
import com.ctrip.xpipe.redis.checker.healthcheck.RedisHealthCheckInstance;
import com.ctrip.xpipe.redis.checker.healthcheck.actions.ping.PingActionContext;
import com.ctrip.xpipe.redis.checker.healthcheck.actions.ping.PingActionListener;
import com.ctrip.xpipe.redis.checker.healthcheck.actions.psubscribe.PsubActionContext;
import com.ctrip.xpipe.redis.checker.healthcheck.actions.psubscribe.PsubActionListener;
import com.ctrip.xpipe.redis.checker.healthcheck.actions.psubscribe.PsubPingActionCollector;
import com.google.common.collect.Maps;

import java.util.Map;

public abstract class AbstractPsubPingActionCollector implements PsubPingActionCollector {

protected Map<RedisHealthCheckInstance, HealthStatus> allHealthStatus = Maps.newConcurrentMap();

protected PingActionListener pingActionListener = new AbstractPsubPingActionCollector.CollectorPingActionListener();

protected PsubActionListener psubActionListener = new AbstractPsubPingActionCollector.CollectorPsubActionListener();

protected abstract HealthStatus createOrGetHealthStatus(RedisHealthCheckInstance instance);

protected void removeHealthStatus(HealthCheckAction<RedisHealthCheckInstance> action) {
HealthStatus healthStatus = allHealthStatus.remove(action.getActionInstance());
if(healthStatus != null) {
healthStatus.stop();
}
}

@Override
public boolean supportInstance(RedisHealthCheckInstance instance) {
return true;
}

@Override
public PingActionListener createPingActionListener() {
return pingActionListener;
}

@Override
public PsubActionListener createPsubActionListener() {
return psubActionListener;
}

protected class CollectorPingActionListener implements PingActionListener {

@Override
public void onAction(PingActionContext pingActionContext) {
HealthStatus healthStatus = createOrGetHealthStatus(pingActionContext.instance());
if (!pingActionContext.isSuccess()) {
if (pingActionContext.getCause().getMessage().contains("LOADING")) {
healthStatus.loading();
}
return;
}

if (pingActionContext.getResult()) {
healthStatus.pong();
} else {
if(healthStatus.getState() == HEALTH_STATE.UNKNOWN) {
healthStatus.pongInit();
}
}
}

@Override
public boolean worksfor(ActionContext t) {
return t instanceof PingActionContext;
}

@Override
public void stopWatch(HealthCheckAction action) {
removeHealthStatus(action);
}
}

protected class CollectorPsubActionListener implements PsubActionListener {

@Override
public void onAction(PsubActionContext psubActionContext) {
HealthStatus healthStatus = createOrGetHealthStatus(psubActionContext.instance());
if (!psubActionContext.getResult().isEmpty()) {
healthStatus.subSuccess();
}
}

@Override
public void stopWatch(HealthCheckAction<RedisHealthCheckInstance> action) {
removeHealthStatus(action);
}
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,100 @@
package com.ctrip.xpipe.redis.checker.healthcheck.actions.interaction;

import com.ctrip.xpipe.redis.checker.healthcheck.RedisHealthCheckInstance;
import com.ctrip.xpipe.redis.checker.healthcheck.actions.interaction.event.InstanceDown;
import com.ctrip.xpipe.redis.checker.healthcheck.actions.interaction.event.InstanceLoading;
import com.ctrip.xpipe.redis.checker.healthcheck.actions.interaction.event.InstanceUp;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.util.concurrent.ScheduledExecutorService;

/**
* UNKNOWN
* pingSuccess -> INSTANCEUP + start subAction
* pingFail -> DOWN + markDown
* subSuccess -> throw exception
* <p>
* INSTANCEUP
* pingSuccess,do nothing
* pingFail -> DOWN + markDown
* subSuccess -> HEALTHY + markUp + stop subAction
* <p>
* HEALTHY
* pingSuccess,do nothing
* pingFail -> DOWN + markDown
* subSuccess -> throw exception
* <p>
* DOWN
* pingSuccess -> INSTANCEUP + start subAction
* pingFail,do nothing
* subSuccess -> throw exception
*/
public class CrossRegionRedisHealthStatus extends HealthStatus {

protected static final Logger logger = LoggerFactory.getLogger(CrossRegionRedisHealthStatus.class);

public CrossRegionRedisHealthStatus(RedisHealthCheckInstance instance, ScheduledExecutorService scheduled) {
super(instance, scheduled);
}

@Override
protected void loading() {
HEALTH_STATE preState = state.get();
if(state.compareAndSet(preState, HEALTH_STATE.DOWN)) {
logStateChange(preState, state.get());
}
if (!preState.equals(HEALTH_STATE.DOWN)) {
logger.info("[setLoading] {}", this);
notifyObservers(new InstanceLoading(instance));
}
}

@Override
protected void pong() {
lastPongTime.set(System.currentTimeMillis());
HEALTH_STATE preState = state.get();
if (preState.equals(HEALTH_STATE.UNKNOWN) || preState.equals(HEALTH_STATE.DOWN)) {
if(state.compareAndSet(preState, HEALTH_STATE.INSTANCEUP)) {
logStateChange(preState, state.get());
}
}
}

@Override
protected void subSuccess() {
HEALTH_STATE preState = state.get();
if (preState.equals(HEALTH_STATE.INSTANCEUP)) {
if(state.compareAndSet(preState, HEALTH_STATE.HEALTHY)) {
logStateChange(preState, state.get());
}
logger.info("[setUp] {}", this);
notifyObservers(new InstanceUp(instance));
}
}

@Override
protected void healthStatusUpdate() {
long currentTime = System.currentTimeMillis();

if(lastPongTime.get() != UNSET_TIME) {
long pingDownTime = currentTime - lastPongTime.get();
final int pingDownAfter = pingDownAfterMilli.getAsInt();
if (pingDownTime > pingDownAfter) {
doMarkDown();
}
}
}

protected void doMarkDown() {
HEALTH_STATE preState = state.get();
if(state.compareAndSet(preState, HEALTH_STATE.DOWN)) {
logStateChange(preState, state.get());
}
if (!preState.equals(HEALTH_STATE.DOWN)) {
logger.info("[setDown] {}", this);
notifyObservers(new InstanceDown(instance));
}
}

}
Loading

0 comments on commit 47f2335

Please sign in to comment.