Skip to content

Commit afd57b0

Browse files
mamirovlspeedmamirovlspeed
committed
ECWID-165744 Update Apache httpclient to httpclient5 5.5
1 parent 85b4f59 commit afd57b0

9 files changed

+443
-1
lines changed

‎src/main/kotlin/com/ecwid/apiclient/v3/httptransport/HttpTransport.kt‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,7 @@
11
package com.ecwid.apiclient.v3.httptransport
22

3-
interface HttpTransport {
3+
import java.io.Closeable
4+
5+
interface HttpTransport : Closeable {
46
fun makeHttpRequest(httpRequest: HttpRequest): HttpResponse
57
}

‎src/main/kotlin/com/ecwid/apiclient/v3/httptransport/impl/ApacheCommonsHttpClientTransport.kt‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ import org.apache.http.client.HttpClient
88
import org.apache.http.client.config.RequestConfig
99
import org.apache.http.impl.client.HttpClientBuilder
1010
import org.apache.http.impl.conn.PoolingHttpClientConnectionManager
11+
import java.io.Closeable
1112
import java.io.IOException
1213

1314
private const val DEFAULT_CONNECTION_TIMEOUT = 10_000 // 10 sec
@@ -33,6 +34,7 @@ internal const val MAX_RATE_LIMIT_RETRY_INTERVAL_SECONDS = 60L
3334
val EMPTY_WAITING_REACTION: (Long) -> Unit = { }
3435
val EMPTY_BEFORE_REQUEST_ACTION: () -> Unit = { }
3536

37+
@Deprecated("Use ApacheCommonsHttpClientTransport from client5 package")
3638
open class ApacheCommonsHttpClientTransport(
3739
private val httpClient: HttpClient = buildHttpClient(),
3840
private val rateLimitRetryStrategy: RateLimitRetryStrategy = SleepForRetryAfterRateLimitRetryStrategy(),
@@ -106,6 +108,12 @@ open class ApacheCommonsHttpClientTransport(
106108
}
107109
}
108110

111+
override fun close() {
112+
if (httpClient is Closeable) {
113+
httpClient.close()
114+
}
115+
}
116+
109117
companion object {
110118

111119
private fun buildHttpClient(

‎src/main/kotlin/com/ecwid/apiclient/v3/httptransport/impl/client5/ApacheCommonsHttpClient5Transport.kt‎

Lines changed: 154 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,154 @@
1+
package com.ecwid.apiclient.v3.httptransport.impl.client5
2+
3+
import com.ecwid.apiclient.v3.httptransport.HttpRequest
4+
import com.ecwid.apiclient.v3.httptransport.HttpResponse
5+
import com.ecwid.apiclient.v3.httptransport.HttpTransport
6+
import org.apache.hc.client5.http.classic.HttpClient
7+
import org.apache.hc.client5.http.config.ConnectionConfig
8+
import org.apache.hc.client5.http.config.RequestConfig
9+
import org.apache.hc.client5.http.impl.classic.HttpClients
10+
import org.apache.hc.client5.http.impl.io.PoolingHttpClientConnectionManagerBuilder
11+
import org.apache.hc.core5.http.Header
12+
import java.io.Closeable
13+
import java.io.IOException
14+
import java.util.concurrent.TimeUnit
15+
16+
private const val DEFAULT_CONNECTION_TIMEOUT = 10_000L // 10 sec
17+
private const val DEFAULT_READ_TIMEOUT = 60_000 // 1 min
18+
19+
private const val DEFAULT_MAX_CONNECTIONS = 10
20+
21+
/**
22+
* Number of attempts to retry request if server responded with 429
23+
*/
24+
internal const val DEFAULT_RATE_LIMIT_ATTEMPTS = 2
25+
26+
/**
27+
* Number of seconds to wait until next attempt, if server didn't send Retry-After header
28+
*/
29+
internal const val DEFAULT_RATE_LIMIT_RETRY_INTERVAL_SECONDS = 10L
30+
31+
/**
32+
* Maximal delay in seconds before next attempt
33+
*/
34+
internal const val MAX_RATE_LIMIT_RETRY_INTERVAL_SECONDS = 60L
35+
36+
val EMPTY_WAITING_REACTION: (Long) -> Unit = { }
37+
val EMPTY_BEFORE_REQUEST_ACTION: () -> Unit = { }
38+
39+
open class ApacheCommonsHttpClient5Transport(
40+
private val httpClient: HttpClient = buildHttpClient(),
41+
private val rateLimitRetryStrategy: RateLimitRetryStrategy = SleepForRetryAfterRateLimitRetryStrategy(),
42+
) : HttpTransport {
43+
44+
constructor(
45+
httpClient: HttpClient,
46+
defaultRateLimitAttempts: Int = DEFAULT_RATE_LIMIT_ATTEMPTS,
47+
defaultRateLimitRetryInterval: Long = DEFAULT_RATE_LIMIT_RETRY_INTERVAL_SECONDS,
48+
maxRateLimitRetryInterval: Long = MAX_RATE_LIMIT_RETRY_INTERVAL_SECONDS,
49+
onEverySecondOfWaiting: (Long) -> Unit = EMPTY_WAITING_REACTION,
50+
beforeEachRequestAttempt: () -> Unit = EMPTY_BEFORE_REQUEST_ACTION,
51+
) : this(
52+
httpClient,
53+
rateLimitRetryStrategy = SleepForRetryAfterRateLimitRetryStrategy(
54+
defaultRateLimitAttempts = defaultRateLimitAttempts,
55+
defaultRateLimitRetryInterval = defaultRateLimitRetryInterval,
56+
maxRateLimitRetryInterval = maxRateLimitRetryInterval,
57+
onEverySecondOfWaiting = onEverySecondOfWaiting,
58+
beforeEachRequestAttempt = beforeEachRequestAttempt
59+
)
60+
)
61+
62+
constructor(
63+
defaultConnectionTimeout: Long = DEFAULT_CONNECTION_TIMEOUT,
64+
defaultReadTimeout: Int = DEFAULT_READ_TIMEOUT,
65+
defaultMaxConnections: Int = DEFAULT_MAX_CONNECTIONS,
66+
defaultHeaders: List<Header> = emptyList(),
67+
rateLimitRetryStrategy: RateLimitRetryStrategy
68+
) : this(
69+
httpClient = buildHttpClient(
70+
defaultConnectionTimeout = defaultConnectionTimeout,
71+
defaultReadTimeout = defaultReadTimeout,
72+
defaultMaxConnections = defaultMaxConnections,
73+
defaultHeaders = defaultHeaders
74+
),
75+
rateLimitRetryStrategy = rateLimitRetryStrategy
76+
)
77+
78+
constructor(
79+
defaultConnectionTimeout: Long = DEFAULT_CONNECTION_TIMEOUT,
80+
defaultReadTimeout: Int = DEFAULT_READ_TIMEOUT,
81+
defaultMaxConnections: Int = DEFAULT_MAX_CONNECTIONS,
82+
defaultRateLimitAttempts: Int = DEFAULT_RATE_LIMIT_ATTEMPTS,
83+
defaultRateLimitRetryInterval: Long = DEFAULT_RATE_LIMIT_RETRY_INTERVAL_SECONDS,
84+
maxRateLimitRetryInterval: Long = MAX_RATE_LIMIT_RETRY_INTERVAL_SECONDS,
85+
defaultHeaders: List<Header> = emptyList(),
86+
onEverySecondOfWaiting: (Long) -> Unit = EMPTY_WAITING_REACTION,
87+
beforeEachRequestAttempt: () -> Unit = EMPTY_BEFORE_REQUEST_ACTION,
88+
) : this(
89+
httpClient = buildHttpClient(
90+
defaultConnectionTimeout = defaultConnectionTimeout,
91+
defaultReadTimeout = defaultReadTimeout,
92+
defaultMaxConnections = defaultMaxConnections,
93+
defaultHeaders = defaultHeaders
94+
),
95+
rateLimitRetryStrategy = SleepForRetryAfterRateLimitRetryStrategy(
96+
defaultRateLimitAttempts = defaultRateLimitAttempts,
97+
defaultRateLimitRetryInterval = defaultRateLimitRetryInterval,
98+
maxRateLimitRetryInterval = maxRateLimitRetryInterval,
99+
onEverySecondOfWaiting = onEverySecondOfWaiting,
100+
beforeEachRequestAttempt = beforeEachRequestAttempt,
101+
)
102+
)
103+
104+
override fun makeHttpRequest(httpRequest: HttpRequest): HttpResponse {
105+
return try {
106+
rateLimitRetryStrategy.makeHttpRequest(httpClient, httpRequest)
107+
} catch (e: IOException) {
108+
HttpResponse.TransportError(e)
109+
}
110+
}
111+
112+
override fun close() {
113+
if (httpClient is Closeable) {
114+
httpClient.close()
115+
}
116+
}
117+
118+
companion object {
119+
120+
private fun buildHttpClient(
121+
defaultConnectionTimeout: Long = DEFAULT_CONNECTION_TIMEOUT,
122+
defaultReadTimeout: Int = DEFAULT_READ_TIMEOUT,
123+
defaultMaxConnections: Int = DEFAULT_MAX_CONNECTIONS,
124+
defaultHeaders: List<Header> = emptyList(),
125+
): HttpClient {
126+
val connectionConfig = ConnectionConfig.custom()
127+
.setConnectTimeout(defaultConnectionTimeout, TimeUnit.SECONDS)
128+
.setSocketTimeout(defaultReadTimeout, TimeUnit.SECONDS)
129+
.build()
130+
131+
val connectionManager = PoolingHttpClientConnectionManagerBuilder.create()
132+
.setMaxConnTotal(defaultMaxConnections)
133+
.setMaxConnPerRoute(defaultMaxConnections)
134+
.setDefaultConnectionConfig(connectionConfig)
135+
.build()
136+
137+
val requestConfig = RequestConfig.custom()
138+
.setConnectionRequestTimeout(defaultConnectionTimeout, TimeUnit.SECONDS)
139+
.build()
140+
141+
val httpClientBuilder = HttpClients.custom()
142+
.setConnectionManager(connectionManager)
143+
.setDefaultRequestConfig(requestConfig)
144+
.setRedirectStrategy(RemoveDisallowedHeadersRedirectStrategy())
145+
// TODO .setRetryHandler()
146+
// TODO .setServiceUnavailableRetryStrategy()
147+
if (defaultHeaders.isNotEmpty()) {
148+
httpClientBuilder.setDefaultHeaders(defaultHeaders)
149+
}
150+
return httpClientBuilder.build()
151+
}
152+
}
153+
}
154+

‎src/main/kotlin/com/ecwid/apiclient/v3/httptransport/impl/client5/ConvertUtils.kt‎

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,63 @@
1+
package com.ecwid.apiclient.v3.httptransport.impl.client5
2+
3+
import com.ecwid.apiclient.v3.httptransport.HttpRequest
4+
import com.ecwid.apiclient.v3.httptransport.HttpResponse
5+
import com.ecwid.apiclient.v3.httptransport.TransportHttpBody
6+
import org.apache.hc.core5.http.ClassicHttpRequest
7+
import org.apache.hc.core5.http.ClassicHttpResponse
8+
import org.apache.hc.core5.http.ContentType
9+
import org.apache.hc.core5.http.HttpEntity
10+
import org.apache.hc.core5.http.HttpStatus
11+
import org.apache.hc.core5.http.io.entity.*
12+
import org.apache.hc.core5.http.io.support.ClassicRequestBuilder
13+
14+
internal fun HttpRequest.toHttpUriRequest(): ClassicHttpRequest {
15+
val requestBuilder = when (this) {
16+
is HttpRequest.HttpGetRequest -> {
17+
ClassicRequestBuilder.get(uri)
18+
}
19+
is HttpRequest.HttpPostRequest -> {
20+
ClassicRequestBuilder
21+
.post(uri)
22+
.setEntity(transportHttpBody.toEntity())
23+
}
24+
is HttpRequest.HttpPutRequest -> {
25+
ClassicRequestBuilder
26+
.put(uri)
27+
.setEntity(transportHttpBody.toEntity())
28+
}
29+
is HttpRequest.HttpDeleteRequest -> {
30+
ClassicRequestBuilder.delete(uri)
31+
}
32+
}
33+
34+
return requestBuilder
35+
.apply updated@{
36+
this@updated.charset = Charsets.UTF_8
37+
this@toHttpUriRequest.params.forEach(this@updated::addParameter)
38+
this@toHttpUriRequest.headers.forEach(this@updated::addHeader)
39+
}
40+
.build()
41+
}
42+
43+
internal fun ClassicHttpResponse.toApiResponse(): HttpResponse {
44+
val responseBytes = EntityUtils.toByteArray(entity)
45+
return if (code == HttpStatus.SC_OK) {
46+
HttpResponse.Success(responseBytes)
47+
} else {
48+
HttpResponse.Error(code, reasonPhrase, responseBytes)
49+
}
50+
}
51+
52+
private fun TransportHttpBody.toEntity(): HttpEntity? = when (this) {
53+
is TransportHttpBody.EmptyBody ->
54+
null
55+
is TransportHttpBody.InputStreamBody ->
56+
BufferedHttpEntity(InputStreamEntity(stream, mimeType.toContentType()))
57+
is TransportHttpBody.ByteArrayBody ->
58+
ByteArrayEntity(byteArray, mimeType.toContentType())
59+
is TransportHttpBody.LocalFileBody ->
60+
FileEntity(file, mimeType.toContentType())
61+
}
62+
63+
private fun String.toContentType(): ContentType = ContentType.create(this, Charsets.UTF_8)

‎src/main/kotlin/com/ecwid/apiclient/v3/httptransport/impl/client5/NoRetryRateLimitRetryStrategy.kt‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
package com.ecwid.apiclient.v3.httptransport.impl.client5
2+
3+
import com.ecwid.apiclient.v3.httptransport.HttpRequest
4+
import com.ecwid.apiclient.v3.httptransport.HttpResponse
5+
import org.apache.hc.client5.http.classic.HttpClient
6+
7+
class NoRetryRateLimitRetryStrategy : RateLimitRetryStrategy {
8+
override fun makeHttpRequest(httpClient: HttpClient, httpRequest: HttpRequest): HttpResponse {
9+
return httpClient.execute(httpRequest.toHttpUriRequest()) { response ->
10+
response.toApiResponse()
11+
}
12+
}
13+
}

‎src/main/kotlin/com/ecwid/apiclient/v3/httptransport/impl/client5/RateLimitRetryStrategy.kt‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
package com.ecwid.apiclient.v3.httptransport.impl.client5
2+
3+
import com.ecwid.apiclient.v3.httptransport.HttpRequest
4+
import com.ecwid.apiclient.v3.httptransport.HttpResponse
5+
import org.apache.hc.client5.http.classic.HttpClient
6+
7+
interface RateLimitRetryStrategy {
8+
fun makeHttpRequest(httpClient: HttpClient, httpRequest: HttpRequest): HttpResponse
9+
}
10+

‎src/main/kotlin/com/ecwid/apiclient/v3/httptransport/impl/client5/RateLimitedHttpClientWrapper.kt‎

Lines changed: 89 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,89 @@
1+
package com.ecwid.apiclient.v3.httptransport.impl.client5
2+
3+
import com.ecwid.apiclient.v3.metric.RequestRetrySleepMetric
4+
import org.apache.hc.client5.http.classic.HttpClient
5+
import org.apache.hc.core5.http.ClassicHttpRequest
6+
import org.apache.hc.core5.http.ClassicHttpResponse
7+
import org.apache.hc.core5.http.io.HttpClientResponseHandler
8+
import org.apache.hc.core5.http.io.entity.EntityUtils
9+
import java.io.IOException
10+
import java.util.concurrent.TimeUnit
11+
import java.util.logging.Logger
12+
13+
private val log = Logger.getLogger(RateLimitedHttpClientWrapper::class.qualifiedName)
14+
15+
private const val SC_TOO_MANY_REQUESTS = 429
16+
17+
/**
18+
* Wrapper for httpClient, which retries requests if faces 429 error response.
19+
* Respects server's Retry-After header if provided.
20+
*/
21+
open class RateLimitedHttpClientWrapper(
22+
private val httpClient: HttpClient,
23+
private val defaultRateLimitRetryInterval: Long = DEFAULT_RATE_LIMIT_RETRY_INTERVAL_SECONDS,
24+
private val maxRateLimitRetryInterval: Long = MAX_RATE_LIMIT_RETRY_INTERVAL_SECONDS,
25+
private val totalAttempts: Int = DEFAULT_RATE_LIMIT_ATTEMPTS,
26+
private val onEverySecondOfWaiting: (Long) -> Unit = { },
27+
private val beforeEachRequestAttempt: () -> Unit = { },
28+
) {
29+
30+
@Throws(IOException::class)
31+
fun <T> execute(request: ClassicHttpRequest, responseHandler: HttpClientResponseHandler<T>): T {
32+
return executeWithRetry(request, totalAttempts, responseHandler)
33+
}
34+
35+
private fun <T> executeWithRetry(
36+
request: ClassicHttpRequest,
37+
attemptsLeft: Int,
38+
responseHandler: HttpClientResponseHandler<T>
39+
): T {
40+
beforeEachRequestAttempt()
41+
return httpClient.execute(request) { response ->
42+
if (response.code == SC_TOO_MANY_REQUESTS && attemptsLeft > 0) {
43+
process429(
44+
request,
45+
response,
46+
attemptsLeft - 1, // decrement attempts reminder
47+
responseHandler
48+
)
49+
} else if (response.code == SC_TOO_MANY_REQUESTS && attemptsLeft <= 0) {
50+
log.warning("Request ${request.uri.path} rate-limited: no more attempts.")
51+
responseHandler.handleResponse(response)
52+
} else {
53+
responseHandler.handleResponse(response)
54+
}
55+
}
56+
}
57+
58+
private fun <T> process429(
59+
request: ClassicHttpRequest,
60+
response: ClassicHttpResponse,
61+
attemptsLeft: Int,
62+
responseHandler: HttpClientResponseHandler<T>
63+
): T {
64+
// server must inform how long to wait
65+
val waitInterval = response.getFirstHeader("Retry-After")?.value?.toLong()
66+
?: defaultRateLimitRetryInterval
67+
return if (waitInterval <= maxRateLimitRetryInterval) {
68+
// return used http connection to pool before retry
69+
EntityUtils.consume(response.entity)
70+
// if server requested acceptable time, we'll wait
71+
log.info("Request ${request.uri.path} rate-limited: waiting $waitInterval seconds...")
72+
waitSeconds(waitInterval, onEverySecondOfWaiting)
73+
log.info("Retrying ${request.uri.path} after $waitInterval-s pause...")
74+
executeWithRetry(request, attemptsLeft, responseHandler)
75+
} else {
76+
// too long to wait - let's return the original error
77+
log.warning("Request ${request.uri.path} rate-limited: too long to wait ($waitInterval).")
78+
responseHandler.handleResponse(response)
79+
}
80+
}
81+
82+
private fun waitSeconds(waitInterval: Long, onEverySecondOfWaiting: (Long) -> Unit) {
83+
for (second in 1..waitInterval) {
84+
TimeUnit.SECONDS.sleep(1)
85+
onEverySecondOfWaiting(second)
86+
RequestRetrySleepMetric.inc()
87+
}
88+
}
89+
}

0 commit comments

Comments
 (0)