From b4ad4a3ffd93b839a5addd43beaaf40228c7751d Mon Sep 17 00:00:00 2001 From: chengyouling Date: Thu, 11 Sep 2025 11:54:12 +0800 Subject: [PATCH 1/4] [#4936] isolation event add instance total and isolation num --- .../event/IsolationServerEvent.java | 20 +++++++++++ .../IsolationServerListFilterExt.java | 36 ++++++++++++------- 2 files changed, 43 insertions(+), 13 deletions(-) diff --git a/handlers/handler-loadbalance/src/main/java/org/apache/servicecomb/loadbalance/event/IsolationServerEvent.java b/handlers/handler-loadbalance/src/main/java/org/apache/servicecomb/loadbalance/event/IsolationServerEvent.java index fa027e10fc6..e9559de0e96 100644 --- a/handlers/handler-loadbalance/src/main/java/org/apache/servicecomb/loadbalance/event/IsolationServerEvent.java +++ b/handlers/handler-loadbalance/src/main/java/org/apache/servicecomb/loadbalance/event/IsolationServerEvent.java @@ -50,6 +50,10 @@ public class IsolationServerEvent extends AlarmEvent { private final long singleTestTime; + private int instancesTotalNum; + + private int isolationInstancesNum; + public IsolationServerEvent(Invocation invocation, MicroserviceInstance instance, ServiceCombServerStats serverStats, IsolationServerListFilterExt.Settings settings, Type type, Endpoint endpoint) { @@ -110,4 +114,20 @@ public int getMinIsolationTime() { public Endpoint getEndpoint() { return endpoint; } + + public int getInstancesTotalNum() { + return instancesTotalNum; + } + + public void setInstancesTotalNum(int instancesTotalNum) { + this.instancesTotalNum = instancesTotalNum; + } + + public int getIsolationInstancesNum() { + return isolationInstancesNum; + } + + public void setIsolationInstancesNum(int isolationInstancesNum) { + this.isolationInstancesNum = isolationInstancesNum; + } } diff --git a/handlers/handler-loadbalance/src/main/java/org/apache/servicecomb/loadbalance/filterext/IsolationServerListFilterExt.java b/handlers/handler-loadbalance/src/main/java/org/apache/servicecomb/loadbalance/filterext/IsolationServerListFilterExt.java index 78985aa2ee3..1facf6a56cf 100644 --- a/handlers/handler-loadbalance/src/main/java/org/apache/servicecomb/loadbalance/filterext/IsolationServerListFilterExt.java +++ b/handlers/handler-loadbalance/src/main/java/org/apache/servicecomb/loadbalance/filterext/IsolationServerListFilterExt.java @@ -21,6 +21,7 @@ import java.util.List; import org.apache.servicecomb.core.Invocation; +import org.apache.servicecomb.foundation.common.Holder; import org.apache.servicecomb.foundation.common.event.AlarmEvent.Type; import org.apache.servicecomb.foundation.common.event.EventManager; import org.apache.servicecomb.loadbalance.Configuration; @@ -88,11 +89,26 @@ public List getFilteredListOfServers(List List filteredServers = new ArrayList<>(); Settings settings = createSettings(invocation); + + // record instance isolation event + List> eventHolder = new ArrayList<>(); servers.forEach((server) -> { - if (allowVisit(invocation, server, settings)) { + if (allowVisit(invocation, server, settings, eventHolder)) { filteredServers.add(server); } }); + eventHolder.forEach((holder) -> { + IsolationServerEvent event = holder.value; + event.setInstancesTotalNum(servers.size()); + event.setIsolationInstancesNum(servers.size() - filteredServers.size()); + if (Type.OPEN == event.getType()) { + LOGGER.warn("Isolate service {}'s instance {}.", invocation.getMicroserviceName(), + event.getInstance().getInstanceId()); + } else { + LOGGER.warn("Recover service {}'s instance {} from isolation.", invocation.getMicroserviceName(), + event.getInstance().getInstanceId()); + } + }); if (filteredServers.isEmpty() && emptyProtection.get()) { LOGGER.warn("All servers have been isolated, allow one of them based on load balance rule."); return servers; @@ -114,7 +130,8 @@ private Settings createSettings(Invocation invocation) { return settings; } - private boolean allowVisit(Invocation invocation, ServiceCombServer server, Settings settings) { + private boolean allowVisit(Invocation invocation, ServiceCombServer server, Settings settings, + List> eventHolder) { ServiceCombServerStats serverStats = ServiceCombLoadBalancerStats.INSTANCE.getServiceCombServerStats(server); if (!checkThresholdAllowed(settings, serverStats)) { if (serverStats.isIsolated() @@ -124,12 +141,8 @@ private boolean allowVisit(Invocation invocation, ServiceCombServer server, Sett if (!serverStats.isIsolated()) { // checkThresholdAllowed is not concurrent control, may print several logs/events in current access. serverStats.markIsolated(true); - eventBus.post( - new IsolationServerEvent(invocation, server.getInstance(), serverStats, - settings, Type.OPEN, server.getEndpoint())); - LOGGER.warn("Isolate service {}'s instance {}.", - invocation.getMicroserviceName(), - server.getInstance().getInstanceId()); + eventHolder.add(new Holder<>(new IsolationServerEvent(invocation, server.getInstance(), serverStats, settings, + Type.OPEN, server.getEndpoint()))); } return false; } @@ -140,11 +153,8 @@ private boolean allowVisit(Invocation invocation, ServiceCombServer server, Sett return false; } serverStats.markIsolated(false); - eventBus.post(new IsolationServerEvent(invocation, server.getInstance(), serverStats, - settings, Type.CLOSE, server.getEndpoint())); - LOGGER.warn("Recover service {}'s instance {} from isolation.", - invocation.getMicroserviceName(), - server.getInstance().getInstanceId()); + eventHolder.add(new Holder<>(new IsolationServerEvent(invocation, server.getInstance(), serverStats, settings, + Type.CLOSE, server.getEndpoint()))); } return true; } From 9dfead824671fda0882284855431333d202a001c Mon Sep 17 00:00:00 2001 From: chengyouling Date: Thu, 11 Sep 2025 15:46:27 +0800 Subject: [PATCH 2/4] fixed testcase --- .../IsolationServerListFilterExt.java | 29 +++++++++---------- .../loadbalance/TestLoadBalanceHandler2.java | 3 ++ 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/handlers/handler-loadbalance/src/main/java/org/apache/servicecomb/loadbalance/filterext/IsolationServerListFilterExt.java b/handlers/handler-loadbalance/src/main/java/org/apache/servicecomb/loadbalance/filterext/IsolationServerListFilterExt.java index 1facf6a56cf..009e5dfd2fe 100644 --- a/handlers/handler-loadbalance/src/main/java/org/apache/servicecomb/loadbalance/filterext/IsolationServerListFilterExt.java +++ b/handlers/handler-loadbalance/src/main/java/org/apache/servicecomb/loadbalance/filterext/IsolationServerListFilterExt.java @@ -91,23 +91,16 @@ public List getFilteredListOfServers(List Settings settings = createSettings(invocation); // record instance isolation event - List> eventHolder = new ArrayList<>(); + List isolationEvents = new ArrayList<>(); servers.forEach((server) -> { - if (allowVisit(invocation, server, settings, eventHolder)) { + if (allowVisit(invocation, server, settings, isolationEvents)) { filteredServers.add(server); } }); - eventHolder.forEach((holder) -> { - IsolationServerEvent event = holder.value; + isolationEvents.forEach((event) -> { event.setInstancesTotalNum(servers.size()); event.setIsolationInstancesNum(servers.size() - filteredServers.size()); - if (Type.OPEN == event.getType()) { - LOGGER.warn("Isolate service {}'s instance {}.", invocation.getMicroserviceName(), - event.getInstance().getInstanceId()); - } else { - LOGGER.warn("Recover service {}'s instance {} from isolation.", invocation.getMicroserviceName(), - event.getInstance().getInstanceId()); - } + eventBus.post(event); }); if (filteredServers.isEmpty() && emptyProtection.get()) { LOGGER.warn("All servers have been isolated, allow one of them based on load balance rule."); @@ -131,7 +124,7 @@ private Settings createSettings(Invocation invocation) { } private boolean allowVisit(Invocation invocation, ServiceCombServer server, Settings settings, - List> eventHolder) { + List isolationEvents) { ServiceCombServerStats serverStats = ServiceCombLoadBalancerStats.INSTANCE.getServiceCombServerStats(server); if (!checkThresholdAllowed(settings, serverStats)) { if (serverStats.isIsolated() @@ -141,8 +134,10 @@ private boolean allowVisit(Invocation invocation, ServiceCombServer server, Sett if (!serverStats.isIsolated()) { // checkThresholdAllowed is not concurrent control, may print several logs/events in current access. serverStats.markIsolated(true); - eventHolder.add(new Holder<>(new IsolationServerEvent(invocation, server.getInstance(), serverStats, settings, - Type.OPEN, server.getEndpoint()))); + isolationEvents.add(new IsolationServerEvent(invocation, server.getInstance(), serverStats, settings, + Type.OPEN, server.getEndpoint())); + LOGGER.warn("Isolate service {}'s instance {}, endpoint {}.", invocation.getMicroserviceName(), + server.getInstance().getInstanceId(), server.getInstance().getEndpoints()); } return false; } @@ -153,8 +148,10 @@ private boolean allowVisit(Invocation invocation, ServiceCombServer server, Sett return false; } serverStats.markIsolated(false); - eventHolder.add(new Holder<>(new IsolationServerEvent(invocation, server.getInstance(), serverStats, settings, - Type.CLOSE, server.getEndpoint()))); + isolationEvents.add(new IsolationServerEvent(invocation, server.getInstance(), serverStats, settings, + Type.CLOSE, server.getEndpoint())); + LOGGER.warn("Recover service {}'s instance {}, endpoint {}, from isolation.", invocation.getMicroserviceName(), + server.getInstance().getInstanceId(), server.getInstance().getEndpoints()); } return true; } diff --git a/handlers/handler-loadbalance/src/test/java/org/apache/servicecomb/loadbalance/TestLoadBalanceHandler2.java b/handlers/handler-loadbalance/src/test/java/org/apache/servicecomb/loadbalance/TestLoadBalanceHandler2.java index f83d0b86ba7..a422efe1252 100644 --- a/handlers/handler-loadbalance/src/test/java/org/apache/servicecomb/loadbalance/TestLoadBalanceHandler2.java +++ b/handlers/handler-loadbalance/src/test/java/org/apache/servicecomb/loadbalance/TestLoadBalanceHandler2.java @@ -474,6 +474,9 @@ public void testZoneAwareAndIsolationFilterWorksEmptyInstanceProtectionEnabled() Assertions.assertEquals("rest://localhost:9090", server.getEndpoint().getEndpoint()); ServiceCombLoadBalancerStats.INSTANCE.markFailure(server2); ServiceCombLoadBalancerStats.INSTANCE.markFailure(server2); + ServiceCombLoadBalancerStats.INSTANCE.markFailure(server2); + ServiceCombLoadBalancerStats.INSTANCE.markFailure(server2); + ServiceCombLoadBalancerStats.INSTANCE.markFailure(server2); loadBalancer = handler.getOrCreateLoadBalancer(invocation); server = loadBalancer.chooseServer(invocation); Assertions.assertEquals("rest://localhost:9091", server.getEndpoint().getEndpoint()); From c1ce2003e8b749f7f0929431363e94d40b984edf Mon Sep 17 00:00:00 2001 From: chengyouling Date: Thu, 11 Sep 2025 15:55:08 +0800 Subject: [PATCH 3/4] delete not use import --- .../loadbalance/filterext/IsolationServerListFilterExt.java | 1 - 1 file changed, 1 deletion(-) diff --git a/handlers/handler-loadbalance/src/main/java/org/apache/servicecomb/loadbalance/filterext/IsolationServerListFilterExt.java b/handlers/handler-loadbalance/src/main/java/org/apache/servicecomb/loadbalance/filterext/IsolationServerListFilterExt.java index 009e5dfd2fe..adef0f659c4 100644 --- a/handlers/handler-loadbalance/src/main/java/org/apache/servicecomb/loadbalance/filterext/IsolationServerListFilterExt.java +++ b/handlers/handler-loadbalance/src/main/java/org/apache/servicecomb/loadbalance/filterext/IsolationServerListFilterExt.java @@ -21,7 +21,6 @@ import java.util.List; import org.apache.servicecomb.core.Invocation; -import org.apache.servicecomb.foundation.common.Holder; import org.apache.servicecomb.foundation.common.event.AlarmEvent.Type; import org.apache.servicecomb.foundation.common.event.EventManager; import org.apache.servicecomb.loadbalance.Configuration; From 359bba3d0f45f11b69ff722799777e746a982fa6 Mon Sep 17 00:00:00 2001 From: chengyouling Date: Thu, 11 Sep 2025 16:35:49 +0800 Subject: [PATCH 4/4] adjust log --- .../filterext/IsolationServerListFilterExt.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/handlers/handler-loadbalance/src/main/java/org/apache/servicecomb/loadbalance/filterext/IsolationServerListFilterExt.java b/handlers/handler-loadbalance/src/main/java/org/apache/servicecomb/loadbalance/filterext/IsolationServerListFilterExt.java index adef0f659c4..dab5e0b54b7 100644 --- a/handlers/handler-loadbalance/src/main/java/org/apache/servicecomb/loadbalance/filterext/IsolationServerListFilterExt.java +++ b/handlers/handler-loadbalance/src/main/java/org/apache/servicecomb/loadbalance/filterext/IsolationServerListFilterExt.java @@ -135,8 +135,8 @@ private boolean allowVisit(Invocation invocation, ServiceCombServer server, Sett serverStats.markIsolated(true); isolationEvents.add(new IsolationServerEvent(invocation, server.getInstance(), serverStats, settings, Type.OPEN, server.getEndpoint())); - LOGGER.warn("Isolate service {}'s instance {}, endpoint {}.", invocation.getMicroserviceName(), - server.getInstance().getInstanceId(), server.getInstance().getEndpoints()); + LOGGER.warn("Isolate service {}'s instance {}.", invocation.getMicroserviceName(), + server.getInstance().getInstanceId()); } return false; } @@ -149,8 +149,8 @@ private boolean allowVisit(Invocation invocation, ServiceCombServer server, Sett serverStats.markIsolated(false); isolationEvents.add(new IsolationServerEvent(invocation, server.getInstance(), serverStats, settings, Type.CLOSE, server.getEndpoint())); - LOGGER.warn("Recover service {}'s instance {}, endpoint {}, from isolation.", invocation.getMicroserviceName(), - server.getInstance().getInstanceId(), server.getInstance().getEndpoints()); + LOGGER.warn("Recover service {}'s instance {} from isolation.", invocation.getMicroserviceName(), + server.getInstance().getInstanceId()); } return true; }