Merge pull request #780 from sadam21/ast-failover_retry

Add retry support when GD endpoint is unvailable

Reviewed-by: Libor Ryšavý
             https://github.com/liry

Author: gdgate

README.md

@@ -37,6 +37,19 @@ The *GoodData Java SDK* uses:
 * the *Slf4j API* version 1.7.*
 * the *Java Development Kit (JDK)* version 8 or later
 
+##### Retry of failed API calls
+
+You can retry your failed requests since version *2.34.0*. Turn it on by configuring
+[RetrySettings](https://github.com/gooddata/gooddata-java/blob/master/src/main/java/com/gooddata/retry/RetrySettings.java)
+and add [Spring retry](https://github.com/spring-projects/spring-retry) to your classpath:
+```xml
+        <dependency>
+            <groupId>org.springframework.retry</groupId>
+            <artifactId>spring-retry</artifactId>
+            <version>${spring.retry.version}</version>
+        </dependency>
+```
+
 ### Logging
 
 The *GoodData Java SDK* logs using `slf4j-api`. Please adjust your logging configuration for 

pom.xml

@@ -3,7 +3,7 @@
     <modelVersion>4.0.0</modelVersion>
 
     <artifactId>gooddata-java</artifactId>
-    <version>2.33.1-SNAPSHOT</version>
+    <version>2.34.0-SNAPSHOT</version>
     <name>${project.artifactId}</name>
     <description>GoodData Java SDK</description>
     <url>https://github.com/gooddata/gooddata-java</url>
@@ -311,6 +311,13 @@
             <version>${spring.version}</version>
             <optional>true</optional>
         </dependency>
+        <!-- Required only if you want to use retry of failed requests which is configured in RetrySettings. -->
+        <dependency>
+            <groupId>org.springframework.retry</groupId>
+            <artifactId>spring-retry</artifactId>
+            <version>1.2.4.RELEASE</version>
+            <optional>true</optional>
+        </dependency>
         <dependency>
             <groupId>com.fasterxml.jackson.core</groupId>
             <artifactId>jackson-core</artifactId>

src/main/java/com/gooddata/GoodData.java

@@ -18,6 +18,9 @@
 import com.gooddata.md.maintenance.ExportImportService;
 import com.gooddata.notification.NotificationService;
 import com.gooddata.projecttemplate.ProjectTemplateService;
+import com.gooddata.retry.RetrySettings;
+import com.gooddata.retry.GetServerErrorRetryStrategy;
+import com.gooddata.retry.RetryableRestTemplate;
 import com.gooddata.util.ResponseErrorHandler;
 import com.gooddata.authentication.LoginPasswordAuthentication;
 import com.gooddata.warehouse.WarehouseService;
@@ -38,6 +41,10 @@
 import org.springframework.context.annotation.Bean;
 import org.springframework.http.MediaType;
 import org.springframework.http.client.HttpComponentsClientHttpRequestFactory;
+import org.springframework.retry.backoff.ExponentialBackOffPolicy;
+import org.springframework.retry.backoff.FixedBackOffPolicy;
+import org.springframework.retry.policy.SimpleRetryPolicy;
+import org.springframework.retry.support.RetryTemplate;
 import org.springframework.util.StreamUtils;
 import org.springframework.web.client.RestTemplate;
 
@@ -48,7 +55,6 @@
 
 import static com.gooddata.util.Validate.notNull;
 import static java.util.Arrays.asList;
-import static java.util.Collections.singletonList;
 import static org.apache.http.util.VersionInfo.loadVersionInfo;
 
 /**
@@ -216,7 +222,7 @@ protected GoodData(GoodDataEndpoint endpoint, Authentication authentication) {
     protected GoodData(GoodDataEndpoint endpoint, Authentication authentication, GoodDataSettings settings) {
         httpClient = authentication.createHttpClient(endpoint, createHttpClientBuilder(settings));
 
-        restTemplate = createRestTemplate(endpoint, httpClient);
+        restTemplate = createRestTemplate(endpoint, httpClient, settings.getRetrySettings());
 
         accountService = new AccountService(getRestTemplate(), settings);
         projectService = new ProjectService(getRestTemplate(), accountService, settings);
@@ -240,7 +246,7 @@ protected GoodData(GoodDataEndpoint endpoint, Authentication authentication, Goo
         lcmService = new LcmService(getRestTemplate(), settings);
     }
 
-    static RestTemplate createRestTemplate(GoodDataEndpoint endpoint, HttpClient httpClient) {
+    static RestTemplate createRestTemplate(GoodDataEndpoint endpoint, HttpClient httpClient, RetrySettings retrySettings) {
         notNull(endpoint, "endpoint");
         notNull(httpClient, "httpClient");
 
@@ -253,7 +259,12 @@ static RestTemplate createRestTemplate(GoodDataEndpoint endpoint, HttpClient htt
         presetHeaders.put("Accept", MediaType.APPLICATION_JSON_VALUE);
         presetHeaders.put(Header.GDC_VERSION, readApiVersion());
 
-        final RestTemplate restTemplate = new RestTemplate(factory);
+        final RestTemplate restTemplate;
+        if (retrySettings == null) {
+            restTemplate = new RestTemplate(factory);
+        } else {
+            restTemplate = createRetryRestTemplate(retrySettings, factory);
+        }
         restTemplate.setInterceptors(asList(
                 new HeaderSettingRequestInterceptor(presetHeaders),
                 new DeprecationWarningRequestInterceptor()));
@@ -263,6 +274,30 @@ static RestTemplate createRestTemplate(GoodDataEndpoint endpoint, HttpClient htt
         return restTemplate;
     }
 
+    private static RestTemplate createRetryRestTemplate(RetrySettings retrySettings, UriPrefixingClientHttpRequestFactory factory) {
+        final RetryTemplate retryTemplate = new RetryTemplate();
+
+        if (retrySettings.getRetryCount() != null) {
+            retryTemplate.setRetryPolicy(new SimpleRetryPolicy(retrySettings.getRetryCount()));
+        }
+
+        if (retrySettings.getRetryInitialInterval() != null) {
+            if (retrySettings.getRetryMultiplier() != null) {
+                final ExponentialBackOffPolicy exponentialBackOffPolicy = new ExponentialBackOffPolicy();
+                exponentialBackOffPolicy.setInitialInterval(retrySettings.getRetryInitialInterval());
+                exponentialBackOffPolicy.setMultiplier(retrySettings.getRetryMultiplier());
+                exponentialBackOffPolicy.setMaxInterval(retrySettings.getRetryMaxInterval());
+                retryTemplate.setBackOffPolicy(exponentialBackOffPolicy);
+            } else {
+                final FixedBackOffPolicy backOffPolicy = new FixedBackOffPolicy();
+                backOffPolicy.setBackOffPeriod(retrySettings.getRetryInitialInterval());
+                retryTemplate.setBackOffPolicy(backOffPolicy);
+            }
+        }
+
+        return new RetryableRestTemplate(factory, retryTemplate, new GetServerErrorRetryStrategy());
+    }
+
     private HttpClientBuilder createHttpClientBuilder(final GoodDataSettings settings) {
         final PoolingHttpClientConnectionManager connectionManager = new PoolingHttpClientConnectionManager();
         connectionManager.setDefaultMaxPerRoute(settings.getMaxConnections());

src/main/java/com/gooddata/GoodDataSettings.java

@@ -5,8 +5,10 @@
  */
 package com.gooddata;
 
+import com.gooddata.retry.RetrySettings;
 import com.gooddata.util.GoodDataToStringBuilder;
 
+import java.util.Objects;
 import java.util.concurrent.TimeUnit;
 
 import static org.springframework.util.Assert.isTrue;
@@ -26,6 +28,7 @@ public class GoodDataSettings {
     private int socketTimeout = secondsToMillis(60);
     private int pollSleep = secondsToMillis(5);
     private String userAgent;
+    private RetrySettings retrySettings;
 
 
     /**
@@ -205,30 +208,36 @@ public void setUserAgent(String userAgent) {
         this.userAgent = userAgent;
     }
 
+    public RetrySettings getRetrySettings() {
+        return retrySettings;
+    }
+
+    /**
+     * Set retry settings
+     * @param retrySettings retry settings
+     */
+    public void setRetrySettings(RetrySettings retrySettings) {
+        this.retrySettings = retrySettings;
+    }
+
     @Override
-    public boolean equals(final Object o) {
+    public boolean equals(Object o) {
         if (this == o) return true;
         if (o == null || getClass() != o.getClass()) return false;
-
         final GoodDataSettings that = (GoodDataSettings) o;
-
-        if (maxConnections != that.maxConnections) return false;
-        if (connectionTimeout != that.connectionTimeout) return false;
-        if (connectionRequestTimeout != that.connectionRequestTimeout) return false;
-        if (socketTimeout != that.socketTimeout) return false;
-        if (pollSleep != that.pollSleep) return false;
-        return userAgent != null ? userAgent.equals(that.userAgent) : that.userAgent == null;
+        return maxConnections == that.maxConnections
+                && connectionTimeout == that.connectionTimeout
+                && connectionRequestTimeout == that.connectionRequestTimeout
+                && socketTimeout == that.socketTimeout
+                && pollSleep == that.pollSleep
+                && Objects.equals(userAgent, that.userAgent)
+                && Objects.equals(retrySettings, that.retrySettings);
     }
 
     @Override
     public int hashCode() {
-        int result = maxConnections;
-        result = 31 * result + connectionTimeout;
-        result = 31 * result + connectionRequestTimeout;
-        result = 31 * result + socketTimeout;
-        result = 31 * result + pollSleep;
-        result = 31 * result + (userAgent != null ? userAgent.hashCode() : 0);
-        return result;
+        return Objects.hash(maxConnections, connectionTimeout, connectionRequestTimeout, socketTimeout, pollSleep,
+                userAgent, retrySettings);
     }
 
     @Override

src/main/java/com/gooddata/retry/GetServerErrorRetryStrategy.java

@@ -0,0 +1,27 @@
+/*
+ * Copyright (C) 2007-2019, GoodData(R) Corporation. All rights reserved.
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE.txt file in the root directory of this source tree.
+ */
+package com.gooddata.retry;
+
+import java.net.URI;
+import java.util.Collection;
+import java.util.Collections;
+
+import static java.util.Arrays.asList;
+
+/**
+ * Allows retry for GET method and some HTTP 5XX states mentioned in {@link GetServerErrorRetryStrategy#RETRYABLE_STATES}.
+ */
+public class GetServerErrorRetryStrategy implements RetryStrategy {
+
+    public static final Collection<Integer> RETRYABLE_STATES = Collections.unmodifiableCollection(asList(500, 502, 503, 504, 507));
+    public static final Collection<String> RETRYABLE_METHODS = Collections.unmodifiableCollection(asList("GET"));
+
+    @Override
+    public boolean retryAllowed(String method, int statusCode, URI uri) {
+        return RETRYABLE_STATES.contains(statusCode) && RETRYABLE_METHODS.contains(method);
+    }
+
+}

src/main/java/com/gooddata/retry/RetrySettings.java

@@ -0,0 +1,95 @@
+/*
+ * Copyright (C) 2007-2019, GoodData(R) Corporation. All rights reserved.
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE.txt file in the root directory of this source tree.
+ */
+package com.gooddata.retry;
+
+import java.util.Objects;
+
+import static org.springframework.util.Assert.isTrue;
+
+/**
+ * Contains settings for HTTP requests retry.
+ */
+public class RetrySettings {
+
+    public Integer DEFAULT_RETRY_COUNT =  6;
+    private Long DEFAULT_RETRY_INITIAL_INTERVAL = 1 * 1000l;  // 1s
+    private Long DEFAULT_RETRY_MAX_INTERVAL = 1 * 60 * 1000l; // 1min
+    private Double DEFAULT_RETRY_MULTIPLIER = 2d;
+
+    private Integer retryCount = DEFAULT_RETRY_COUNT;
+    private Long retryInitialInterval = DEFAULT_RETRY_INITIAL_INTERVAL;
+    private Long retryMaxInterval = DEFAULT_RETRY_MAX_INTERVAL;
+    private Double retryMultiplier = DEFAULT_RETRY_MULTIPLIER;
+
+    /**
+     * Total retry count. Should be > 0. No retry if not set.
+     * @return retry count
+     */
+    public Integer getRetryCount() {
+        return retryCount;
+    }
+
+    public void setRetryCount(Integer retryCount) {
+        isTrue(retryCount == null || retryCount > 0, "retryCount hast to be greater than 0");
+        this.retryCount = retryCount;
+    }
+
+    /**
+     *
+     * @return retry initial interval
+     */
+    public Long getRetryInitialInterval() {
+        return retryInitialInterval;
+    }
+
+    public void setRetryInitialInterval(Long retryInitialInterval) {
+        isTrue(retryInitialInterval > 0, "retryInitialInterval has to be greater than 0");
+        this.retryInitialInterval = retryInitialInterval;
+    }
+
+    /**
+     *
+     * @return maximum retry interval
+     */
+    public Long getRetryMaxInterval() {
+        return retryMaxInterval;
+    }
+
+    public void setRetryMaxInterval(Long retryMaxInterval) {
+        isTrue(retryMaxInterval > 0, "retryMaxInterval has to be greater than 0");
+        this.retryMaxInterval = retryMaxInterval;
+    }
+
+    /**
+     * If set, exponential strategy is used. Every next retry interval will be computed as previos interval multiplied
+     * by this number.
+     * @return retry multiplier
+     */
+    public Double getRetryMultiplier() {
+        return retryMultiplier;
+    }
+
+    public void setRetryMultiplier(Double retryMultiplier) {
+        isTrue(retryMultiplier > 1.0, "retryMultiplier has to be greater than 1.0");
+        this.retryMultiplier = retryMultiplier;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        final RetrySettings that = (RetrySettings) o;
+        return Objects.equals(retryCount, that.retryCount) &&
+                Objects.equals(retryInitialInterval, that.retryInitialInterval) &&
+                Objects.equals(retryMaxInterval, that.retryMaxInterval) &&
+                Objects.equals(retryMultiplier, that.retryMultiplier);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(retryCount, retryInitialInterval, retryMaxInterval, retryMultiplier);
+    }
+}

src/main/java/com/gooddata/retry/RetryStrategy.java

@@ -0,0 +1,24 @@
+/*
+ * Copyright (C) 2007-2019, GoodData(R) Corporation. All rights reserved.
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE.txt file in the root directory of this source tree.
+ */
+package com.gooddata.retry;
+
+import java.net.URI;
+
+/**
+ * Interface for describing retry strategy.
+ */
+public interface RetryStrategy {
+
+    /**
+     * Method says if retry is allowed for given parameter combination.
+     * @param method HTTP method
+     * @param statusCode HTTP response code
+     * @param uri requested URL
+     * @return {@code true} it retry is allowed
+     */
+    boolean retryAllowed(String method, int statusCode, URI uri);
+
+}