From a5ccfeb1a9e5a9d07ac66929d3a1af69a8188e26 Mon Sep 17 00:00:00 2001 From: Alex <97039406+chengyouling@users.noreply.github.com> Date: Tue, 20 May 2025 17:36:20 +0800 Subject: [PATCH] [#4813] Adjust the log information about scheduled tasks and limiting to improve product experience (#4814) --- .../config/center/client/ConfigCenterClient.java | 5 +++++ .../config/center/client/ConfigCenterManager.java | 3 ++- .../apache/servicecomb/config/kie/client/KieClient.java | 6 ++++++ .../servicecomb/config/kie/client/KieConfigManager.java | 3 ++- .../service/center/client/ServiceCenterClient.java | 5 +++++ .../service/center/client/ServiceCenterDiscovery.java | 5 ++++- .../service/center/client/ServiceCenterRegistration.java | 9 +++++++-- 7 files changed, 31 insertions(+), 5 deletions(-) diff --git a/clients/config-center-client/src/main/java/org/apache/servicecomb/config/center/client/ConfigCenterClient.java b/clients/config-center-client/src/main/java/org/apache/servicecomb/config/center/client/ConfigCenterClient.java index 00889cbf63b..25aedec6623 100644 --- a/clients/config-center-client/src/main/java/org/apache/servicecomb/config/center/client/ConfigCenterClient.java +++ b/clients/config-center-client/src/main/java/org/apache/servicecomb/config/center/client/ConfigCenterClient.java @@ -116,6 +116,11 @@ public QueryConfigurationsResponse queryConfigurations(QueryConfigurationsReques queryConfigurationsResponse.setChanged(false); addressManager.recordSuccessState(address); return queryConfigurationsResponse; + } else if (httpResponse.getStatusCode() == HttpStatus.SC_TOO_MANY_REQUESTS) { + LOGGER.warn("rate limited, keep the local dimension [{}] configs unchanged.", dimensionsInfo); + queryConfigurationsResponse.setChanged(false); + addressManager.recordSuccessState(address); + return queryConfigurationsResponse; } else if (httpResponse.getStatusCode() == HttpStatus.SC_BAD_REQUEST) { throw new OperationException("Bad request for query configurations."); } else { diff --git a/clients/config-center-client/src/main/java/org/apache/servicecomb/config/center/client/ConfigCenterManager.java b/clients/config-center-client/src/main/java/org/apache/servicecomb/config/center/client/ConfigCenterManager.java index 7e972e289db..3d60a14683d 100644 --- a/clients/config-center-client/src/main/java/org/apache/servicecomb/config/center/client/ConfigCenterManager.java +++ b/clients/config-center-client/src/main/java/org/apache/servicecomb/config/center/client/ConfigCenterManager.java @@ -89,7 +89,8 @@ public void execute() { } startTask(new BackOffSleepTask(configCenterConfiguration.getRefreshIntervalInMillis(), new PollConfigurationTask(0))); } catch (Exception e) { - LOGGER.error("get configurations from ConfigCenter failed, and will try again.", e); + LOGGER.warn("get configurations from ConfigCenter failed, and will try again, cause message: {}. current " + + "fail does not affect the obtained historical configuration.", e.getCause().getMessage()); startTask(new BackOffSleepTask(failCount + 1, new PollConfigurationTask(failCount + 1))); } } diff --git a/clients/config-kie-client/src/main/java/org/apache/servicecomb/config/kie/client/KieClient.java b/clients/config-kie-client/src/main/java/org/apache/servicecomb/config/kie/client/KieClient.java index 49f81989ad8..8379e046b5b 100644 --- a/clients/config-kie-client/src/main/java/org/apache/servicecomb/config/kie/client/KieClient.java +++ b/clients/config-kie-client/src/main/java/org/apache/servicecomb/config/kie/client/KieClient.java @@ -105,6 +105,12 @@ public ConfigurationsResponse queryConfigurations(ConfigurationsRequest request, addressManager.recordSuccessState(address); return configurationsResponse; } + if (httpResponse.getStatusCode() == HttpStatus.SC_TOO_MANY_REQUESTS) { + LOGGER.warn("rate limited, keep the local dimension [{}] configs unchanged.", request.getLabelsQuery()); + configurationsResponse.setChanged(false); + addressManager.recordSuccessState(address); + return configurationsResponse; + } addressManager.recordFailState(address); throw new OperationException( "read response failed. status:" + httpResponse.getStatusCode() + "; message:" + diff --git a/clients/config-kie-client/src/main/java/org/apache/servicecomb/config/kie/client/KieConfigManager.java b/clients/config-kie-client/src/main/java/org/apache/servicecomb/config/kie/client/KieConfigManager.java index 3c4af57cd35..72adb86a52f 100644 --- a/clients/config-kie-client/src/main/java/org/apache/servicecomb/config/kie/client/KieConfigManager.java +++ b/clients/config-kie-client/src/main/java/org/apache/servicecomb/config/kie/client/KieConfigManager.java @@ -155,7 +155,8 @@ public void execute() { startTask(new BackOffSleepTask(kieConfiguration.getRefreshIntervalInMillis(), new PollConfigurationTask(0, this.configurationsRequest))); } } catch (Exception e) { - LOGGER.error("get configurations from KieConfigCenter failed, and will try again.", e); + LOGGER.warn("get configurations from KieConfigCenter failed, and will try again, cause message: {}. current " + + "fail does not affect the obtained historical configuration.", e.getCause().getMessage()); startTask( new BackOffSleepTask(failCount + 1, new PollConfigurationTask(failCount + 1, this.configurationsRequest))); } diff --git a/clients/service-center-client/src/main/java/org/apache/servicecomb/service/center/client/ServiceCenterClient.java b/clients/service-center-client/src/main/java/org/apache/servicecomb/service/center/client/ServiceCenterClient.java index 4abc2411fe4..cbab90217b6 100755 --- a/clients/service-center-client/src/main/java/org/apache/servicecomb/service/center/client/ServiceCenterClient.java +++ b/clients/service-center-client/src/main/java/org/apache/servicecomb/service/center/client/ServiceCenterClient.java @@ -249,6 +249,11 @@ public FindMicroserviceInstancesResponse findMicroserviceInstance(String consume result.setModified(false); return result; } + if (response.getStatusCode() == HttpStatus.SC_TOO_MANY_REQUESTS) { + LOGGER.warn("rate limited, keep the local service {}#{} instance cache unchanged.", appId, serviceName); + result.setModified(false); + return result; + } sendUnAuthorizedEvent(response); throw new OperationException( "get service instances list fails, statusCode = " + response.getStatusCode() + "; message = " + response diff --git a/clients/service-center-client/src/main/java/org/apache/servicecomb/service/center/client/ServiceCenterDiscovery.java b/clients/service-center-client/src/main/java/org/apache/servicecomb/service/center/client/ServiceCenterDiscovery.java index ccc8fb1697a..74c73efcab8 100644 --- a/clients/service-center-client/src/main/java/org/apache/servicecomb/service/center/client/ServiceCenterDiscovery.java +++ b/clients/service-center-client/src/main/java/org/apache/servicecomb/service/center/client/ServiceCenterDiscovery.java @@ -186,11 +186,14 @@ private List pullInstance(SubscriptionKey k, SubscriptionValue } } } catch (Exception e) { - LOGGER.error("find service {}#{} instance failed.", k.appId, k.serviceName, e); if (!(e.getCause() instanceof IOException)) { // for IOException, do not remove cache, or when service center // not available, invocation between microservices will fail. failedKeys.add(k); + LOGGER.error("find service {}#{} instance failed and remove local cache.", k.appId, k.serviceName, e); + } else { + LOGGER.warn("find service {}#{} instance failed, remaining local instances cache, cause message: {}", + k.appId, k.serviceName, e.getMessage()); } } return failedKeys; diff --git a/clients/service-center-client/src/main/java/org/apache/servicecomb/service/center/client/ServiceCenterRegistration.java b/clients/service-center-client/src/main/java/org/apache/servicecomb/service/center/client/ServiceCenterRegistration.java index 02d76ebbbd4..eafd58f48ae 100644 --- a/clients/service-center-client/src/main/java/org/apache/servicecomb/service/center/client/ServiceCenterRegistration.java +++ b/clients/service-center-client/src/main/java/org/apache/servicecomb/service/center/client/ServiceCenterRegistration.java @@ -277,7 +277,7 @@ public void execute() { } if (!serviceCenterClient.sendHeartBeat(microservice.getServiceId(), microserviceInstance.getInstanceId())) { - LOGGER.error("send heart failed, and will try again."); + LOGGER.warn("send heart failed, and will try again."); eventBus.post(new HeartBeatEvent(false, microservice, microserviceInstance)); startTask(new BackOffSleepTask(failedCount + 1, new SendHeartBeatTask(failedCount + 1))); } else { @@ -287,7 +287,12 @@ public void execute() { new BackOffSleepTask(Math.max(heartBeatInterval, heartBeatRequestTimeout), new SendHeartBeatTask(0))); } } catch (Exception e) { - LOGGER.error("send heart failed, and will try again.", e); + // If heartbeat failures three times, log error stack help troubleshooting. Others just log message as warn. + if (failedCount == 2) { + LOGGER.error("send heart failed, and will try again.", e); + } else { + LOGGER.warn("send heart failed, and will try again. message [{}]", e.getCause().getMessage()); + } eventBus.post(new HeartBeatEvent(false, microservice, microserviceInstance)); startTask(new BackOffSleepTask(failedCount + 1, new SendHeartBeatTask(failedCount + 1))); }