From 16797703cc86c1a0cca69d680be4003633515ccf Mon Sep 17 00:00:00 2001 From: chengyouling Date: Tue, 15 Jul 2025 14:59:10 +0800 Subject: [PATCH 1/3] [#4877] not clear service instances cache when pull instance exception and add telnet check --- .../center/client/ServiceCenterDiscovery.java | 58 +++++++++++++------ 1 file changed, 41 insertions(+), 17 deletions(-) diff --git a/clients/service-center-client/src/main/java/org/apache/servicecomb/service/center/client/ServiceCenterDiscovery.java b/clients/service-center-client/src/main/java/org/apache/servicecomb/service/center/client/ServiceCenterDiscovery.java index 74c73efcab8..18417d8246d 100644 --- a/clients/service-center-client/src/main/java/org/apache/servicecomb/service/center/client/ServiceCenterDiscovery.java +++ b/clients/service-center-client/src/main/java/org/apache/servicecomb/service/center/client/ServiceCenterDiscovery.java @@ -18,6 +18,8 @@ package org.apache.servicecomb.service.center.client; import java.io.IOException; +import java.net.InetSocketAddress; +import java.net.Socket; import java.util.ArrayList; import java.util.Collections; import java.util.List; @@ -153,13 +155,12 @@ public void onPullInstanceEvent(PullInstanceEvent event) { startTask(new PullInstanceOnceTask()); } - private List pullInstance(SubscriptionKey k, SubscriptionValue v, boolean sendChangedEvent) { + private void pullInstance(SubscriptionKey k, SubscriptionValue v, boolean sendChangedEvent) { if (myselfServiceId == null) { // registration not ready - return Collections.emptyList(); + return; } - List failedKeys = new ArrayList<>(); try { FindMicroserviceInstancesResponse instancesResponse = serviceCenterClient .findMicroserviceInstance(myselfServiceId, k.appId, k.serviceName, ALL_VERSION, v.revision); @@ -186,17 +187,9 @@ private List pullInstance(SubscriptionKey k, SubscriptionValue } } } catch (Exception e) { - if (!(e.getCause() instanceof IOException)) { - // for IOException, do not remove cache, or when service center - // not available, invocation between microservices will fail. - failedKeys.add(k); - LOGGER.error("find service {}#{} instance failed and remove local cache.", k.appId, k.serviceName, e); - } else { - LOGGER.warn("find service {}#{} instance failed, remaining local instances cache, cause message: {}", - k.appId, k.serviceName, e.getMessage()); - } + LOGGER.warn("find service {}#{} instance failed, remaining local instances cache [{}], cause message: {}", + k.appId, k.serviceName, instanceToString(v.instancesCache), e.getMessage()); } - return failedKeys; } private void setMicroserviceInfo(List instances) { @@ -236,10 +229,13 @@ public void execute() { private synchronized void pullAllInstance() { List failedInstances = new ArrayList<>(); - instancesCache.forEach((k, v) -> failedInstances.addAll(pullInstance(k, v, true))); - if (failedInstances.isEmpty()) { - return; - } + instancesCache.forEach((k, v) -> { + pullInstance(k, v, true); + v.instancesCache.removeIf(instance -> isInstanceUnavailable(instance.getServiceName(), instance.getEndpoints())); + if (v.instancesCache.isEmpty()) { + failedInstances.add(k); + } + }); failedInstances.forEach(instancesCache::remove); failedInstances.clear(); } @@ -261,4 +257,32 @@ private static String instanceToString(List instances) { sb.append("#"); return sb.toString(); } + + protected boolean isInstanceUnavailable(String serviceName, List endpoints) { + for (String endpoint : endpoints) { + String[] hostPort = getHostPort(endpoint); + if (hostPort == null) { + continue; + } + for (int k = 0; k < 3; k++) { + try (Socket s = new Socket()) { + s.connect(new InetSocketAddress(hostPort[0], Integer.parseInt(hostPort[1])), 3000); + return false; + } catch (IOException e) { + LOGGER.warn("telnet endpoint [{}] failed, It will be try again.", endpoint); + } + } + LOGGER.warn("telnet three times failed, remove service [{}] endpoint [{}].", serviceName, endpoint); + return true; + } + return true; + } + + private String[] getHostPort(String endpoint) { + String hostPort = endpoint.substring("rest://".length()); + if (hostPort.split(":").length == 2) { + return hostPort.split(":"); + } + return null; + } } From e390f8d59fc9a29f7b73133417875f3cc55b535f Mon Sep 17 00:00:00 2001 From: chengyouling Date: Tue, 29 Jul 2025 10:45:53 +0800 Subject: [PATCH 2/3] adjust endpoint health check --- .../center/client/ServiceCenterDiscovery.java | 35 +++++++++++++++---- 1 file changed, 28 insertions(+), 7 deletions(-) diff --git a/clients/service-center-client/src/main/java/org/apache/servicecomb/service/center/client/ServiceCenterDiscovery.java b/clients/service-center-client/src/main/java/org/apache/servicecomb/service/center/client/ServiceCenterDiscovery.java index 18417d8246d..a89fee02781 100644 --- a/clients/service-center-client/src/main/java/org/apache/servicecomb/service/center/client/ServiceCenterDiscovery.java +++ b/clients/service-center-client/src/main/java/org/apache/servicecomb/service/center/client/ServiceCenterDiscovery.java @@ -26,6 +26,9 @@ import java.util.Map; import java.util.Objects; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; import org.apache.servicecomb.http.client.task.AbstractTask; import org.apache.servicecomb.http.client.task.Task; @@ -126,6 +129,7 @@ public void startDiscovery() { if (!started) { started = true; startTask(new PullInstanceTask()); + startCheckInstancesHealth(); } } @@ -228,16 +232,33 @@ public void execute() { } private synchronized void pullAllInstance() { - List failedInstances = new ArrayList<>(); instancesCache.forEach((k, v) -> { pullInstance(k, v, true); - v.instancesCache.removeIf(instance -> isInstanceUnavailable(instance.getServiceName(), instance.getEndpoints())); - if (v.instancesCache.isEmpty()) { - failedInstances.add(k); - } }); - failedInstances.forEach(instancesCache::remove); - failedInstances.clear(); + } + + private void startCheckInstancesHealth() { + ScheduledExecutorService executor = + Executors.newScheduledThreadPool(1, (t) -> new Thread(t, "instance-health-check")); + executor.scheduleWithFixedDelay(new CheckInstancesHealthTask(), 0, pollInterval, TimeUnit.MILLISECONDS); + } + + class CheckInstancesHealthTask implements Runnable { + @Override + public void run() { + if (instancesCache.isEmpty()) { + return; + } + List failedInstances = new ArrayList<>(); + instancesCache.forEach((k, v) -> { + v.instancesCache.removeIf(item -> isInstanceUnavailable(item.getServiceName(), item.getEndpoints())); + if (v.instancesCache.isEmpty()) { + failedInstances.add(k); + } + }); + failedInstances.forEach(instancesCache::remove); + failedInstances.clear(); + } } private static String instanceToString(List instances) { From 679e26697d55b9d20ca55598fa2c7920c258d111 Mon Sep 17 00:00:00 2001 From: chengyouling Date: Mon, 25 Aug 2025 21:13:55 +0800 Subject: [PATCH 3/3] remove instance healthcheck --- .../center/client/ServiceCenterDiscovery.java | 60 ------------------- 1 file changed, 60 deletions(-) diff --git a/clients/service-center-client/src/main/java/org/apache/servicecomb/service/center/client/ServiceCenterDiscovery.java b/clients/service-center-client/src/main/java/org/apache/servicecomb/service/center/client/ServiceCenterDiscovery.java index a89fee02781..67fcfcf606b 100644 --- a/clients/service-center-client/src/main/java/org/apache/servicecomb/service/center/client/ServiceCenterDiscovery.java +++ b/clients/service-center-client/src/main/java/org/apache/servicecomb/service/center/client/ServiceCenterDiscovery.java @@ -17,18 +17,11 @@ package org.apache.servicecomb.service.center.client; -import java.io.IOException; -import java.net.InetSocketAddress; -import java.net.Socket; -import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.Map; import java.util.Objects; import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.TimeUnit; import org.apache.servicecomb.http.client.task.AbstractTask; import org.apache.servicecomb.http.client.task.Task; @@ -129,7 +122,6 @@ public void startDiscovery() { if (!started) { started = true; startTask(new PullInstanceTask()); - startCheckInstancesHealth(); } } @@ -237,30 +229,6 @@ private synchronized void pullAllInstance() { }); } - private void startCheckInstancesHealth() { - ScheduledExecutorService executor = - Executors.newScheduledThreadPool(1, (t) -> new Thread(t, "instance-health-check")); - executor.scheduleWithFixedDelay(new CheckInstancesHealthTask(), 0, pollInterval, TimeUnit.MILLISECONDS); - } - - class CheckInstancesHealthTask implements Runnable { - @Override - public void run() { - if (instancesCache.isEmpty()) { - return; - } - List failedInstances = new ArrayList<>(); - instancesCache.forEach((k, v) -> { - v.instancesCache.removeIf(item -> isInstanceUnavailable(item.getServiceName(), item.getEndpoints())); - if (v.instancesCache.isEmpty()) { - failedInstances.add(k); - } - }); - failedInstances.forEach(instancesCache::remove); - failedInstances.clear(); - } - } - private static String instanceToString(List instances) { if (instances == null) { return ""; @@ -278,32 +246,4 @@ private static String instanceToString(List instances) { sb.append("#"); return sb.toString(); } - - protected boolean isInstanceUnavailable(String serviceName, List endpoints) { - for (String endpoint : endpoints) { - String[] hostPort = getHostPort(endpoint); - if (hostPort == null) { - continue; - } - for (int k = 0; k < 3; k++) { - try (Socket s = new Socket()) { - s.connect(new InetSocketAddress(hostPort[0], Integer.parseInt(hostPort[1])), 3000); - return false; - } catch (IOException e) { - LOGGER.warn("telnet endpoint [{}] failed, It will be try again.", endpoint); - } - } - LOGGER.warn("telnet three times failed, remove service [{}] endpoint [{}].", serviceName, endpoint); - return true; - } - return true; - } - - private String[] getHostPort(String endpoint) { - String hostPort = endpoint.substring("rest://".length()); - if (hostPort.split(":").length == 2) { - return hostPort.split(":"); - } - return null; - } }