Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,10 @@ public class IsolationServerEvent extends AlarmEvent {

private final long singleTestTime;

private int instancesTotalNum;

private int isolationInstancesNum;

public IsolationServerEvent(Invocation invocation, MicroserviceInstance instance,
ServiceCombServerStats serverStats,
IsolationServerListFilterExt.Settings settings, Type type, Endpoint endpoint) {
Expand Down Expand Up @@ -110,4 +114,20 @@ public int getMinIsolationTime() {
public Endpoint getEndpoint() {
return endpoint;
}

public int getInstancesTotalNum() {
return instancesTotalNum;
}

public void setInstancesTotalNum(int instancesTotalNum) {
this.instancesTotalNum = instancesTotalNum;
}

public int getIsolationInstancesNum() {
return isolationInstancesNum;
}

public void setIsolationInstancesNum(int isolationInstancesNum) {
this.isolationInstancesNum = isolationInstancesNum;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -88,11 +88,19 @@ public List<ServiceCombServer> getFilteredListOfServers(List<ServiceCombServer>

List<ServiceCombServer> filteredServers = new ArrayList<>();
Settings settings = createSettings(invocation);

// record instance isolation event
List<IsolationServerEvent> isolationEvents = new ArrayList<>();
servers.forEach((server) -> {
if (allowVisit(invocation, server, settings)) {
if (allowVisit(invocation, server, settings, isolationEvents)) {
filteredServers.add(server);
}
});
isolationEvents.forEach((event) -> {
event.setInstancesTotalNum(servers.size());
event.setIsolationInstancesNum(servers.size() - filteredServers.size());
eventBus.post(event);
});
if (filteredServers.isEmpty() && emptyProtection.get()) {
LOGGER.warn("All servers have been isolated, allow one of them based on load balance rule.");
return servers;
Expand All @@ -114,7 +122,8 @@ private Settings createSettings(Invocation invocation) {
return settings;
}

private boolean allowVisit(Invocation invocation, ServiceCombServer server, Settings settings) {
private boolean allowVisit(Invocation invocation, ServiceCombServer server, Settings settings,
List<IsolationServerEvent> isolationEvents) {
ServiceCombServerStats serverStats = ServiceCombLoadBalancerStats.INSTANCE.getServiceCombServerStats(server);
if (!checkThresholdAllowed(settings, serverStats)) {
if (serverStats.isIsolated()
Expand All @@ -124,11 +133,9 @@ private boolean allowVisit(Invocation invocation, ServiceCombServer server, Sett
if (!serverStats.isIsolated()) {
// checkThresholdAllowed is not concurrent control, may print several logs/events in current access.
serverStats.markIsolated(true);
eventBus.post(
new IsolationServerEvent(invocation, server.getInstance(), serverStats,
settings, Type.OPEN, server.getEndpoint()));
LOGGER.warn("Isolate service {}'s instance {}.",
invocation.getMicroserviceName(),
isolationEvents.add(new IsolationServerEvent(invocation, server.getInstance(), serverStats, settings,
Type.OPEN, server.getEndpoint()));
LOGGER.warn("Isolate service {}'s instance {}.", invocation.getMicroserviceName(),
server.getInstance().getInstanceId());
}
return false;
Expand All @@ -140,10 +147,9 @@ private boolean allowVisit(Invocation invocation, ServiceCombServer server, Sett
return false;
}
serverStats.markIsolated(false);
eventBus.post(new IsolationServerEvent(invocation, server.getInstance(), serverStats,
settings, Type.CLOSE, server.getEndpoint()));
LOGGER.warn("Recover service {}'s instance {} from isolation.",
invocation.getMicroserviceName(),
isolationEvents.add(new IsolationServerEvent(invocation, server.getInstance(), serverStats, settings,
Type.CLOSE, server.getEndpoint()));
LOGGER.warn("Recover service {}'s instance {} from isolation.", invocation.getMicroserviceName(),
server.getInstance().getInstanceId());
}
return true;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -474,6 +474,9 @@ public void testZoneAwareAndIsolationFilterWorksEmptyInstanceProtectionEnabled()
Assertions.assertEquals("rest://localhost:9090", server.getEndpoint().getEndpoint());
ServiceCombLoadBalancerStats.INSTANCE.markFailure(server2);
ServiceCombLoadBalancerStats.INSTANCE.markFailure(server2);
ServiceCombLoadBalancerStats.INSTANCE.markFailure(server2);
ServiceCombLoadBalancerStats.INSTANCE.markFailure(server2);
ServiceCombLoadBalancerStats.INSTANCE.markFailure(server2);
loadBalancer = handler.getOrCreateLoadBalancer(invocation);
server = loadBalancer.chooseServer(invocation);
Assertions.assertEquals("rest://localhost:9091", server.getEndpoint().getEndpoint());
Expand Down
Loading