✨ [http/proxy] Add helpers for proxying requests and responses

changes/20251031150814.feature

@@ -0,0 +1 @@
+:sparkles: `[http/proxy]` Add helpers for proxying requests and responses

utils/http/httptest/testing.go

@@ -19,8 +19,9 @@ import (
 
 // NewTestServer creates a test server
 func NewTestServer(t *testing.T, ctx context.Context, handler http.Handler, port string) {
+	t.Helper()
 	list, err := net.Listen("tcp", fmt.Sprintf(":%v", port))
-	require.Nil(t, err)
+	require.NoError(t, err)
 	srv := &http.Server{
 		Handler:           handler,
 		ReadHeaderTimeout: time.Minute,

utils/http/proxy/proxy.go

@@ -0,0 +1,157 @@
+package proxy
+
+import (
+	"bytes"
+	"context"
+	"io"
+	"net/http"
+	"strconv"
+	"strings"
+
+	"github.com/go-http-utils/headers"
+
+	"github.com/ARM-software/golang-utils/utils/commonerrors"
+	httpheaders "github.com/ARM-software/golang-utils/utils/http/headers"
+	"github.com/ARM-software/golang-utils/utils/reflection"
+	"github.com/ARM-software/golang-utils/utils/safecast"
+	"github.com/ARM-software/golang-utils/utils/safeio"
+)
+
+// ProxyDisallowList describes headers which are not proxied back.
+var ProxyDisallowList = []string{
+	headers.AccessControlAllowOrigin,
+	headers.AccessControlAllowMethods,
+	headers.AccessControlAllowHeaders,
+	headers.AccessControlExposeHeaders,
+	headers.AccessControlMaxAge,
+	headers.AccessControlAllowCredentials,
+}
+
+// ProxyRequest proxies a request to a new endpoint. The method can also be changed. Headers are sanitised during the process.
+func ProxyRequest(r *http.Request, proxyMethod, endpoint string) (proxiedRequest *http.Request, err error) {
+	if reflection.IsEmpty(r) {
+		err = commonerrors.UndefinedVariable("request to proxy")
+		return
+	}
+	ctx := r.Context()
+	// Note: It is important to know that an 0 or -1 content length does not mean there is no body. This is likely the case but it could also be because the body was never read and its size never assessed.
+	contentLength := determineRequestContentLength(r)
+	h := httpheaders.FromRequest(r).AllowList(headers.Authorization)
+	if reflection.IsEmpty(proxyMethod) {
+		proxyMethod = http.MethodGet
+	}
+	proxiedRequest, err = http.NewRequestWithContext(ctx, proxyMethod, endpoint, r.Body)
+	if err != nil {
+		err = commonerrors.WrapError(commonerrors.ErrUnexpected, err, "could not create a proxied request")
+		return
+	}
+
+	if proxiedRequest.ContentLength <= 0 {
+		if proxiedRequest.Body == nil || proxiedRequest.Body == http.NoBody {
+			if contentLength > 0 {
+				// In this case, NewRequestWithContext does not understand/expect the request body type (not a string/byte buffer as it may be wrapped into a bigger structure) and so, the body of the proxied request is set to nil
+				// This makes sure this does not happen without performing a copy of the body and the use of unnecessary memory.
+				proxiedRequest.Body = r.Body
+				proxiedRequest.GetBody = r.GetBody
+			} else {
+				// In this case, it will attempt a copy of the request body which should not be costly as the request is unlikely to have a body. Although it may still do as contentlength may not have actually been evaluated. However, we want to make sure it is set to the same type as the original request.
+				proxiedRequest, err = http.NewRequestWithContext(ctx, proxyMethod, endpoint, convertBody(ctx, r.Body))
+				if err != nil {
+					err = commonerrors.WrapError(commonerrors.ErrUnexpected, err, "could not create a proxied request")
+					return
+				}
+			}
+		} else {
+			// In this case, the original request is unlikely to have a body but we want to make sure that the body is of the same type.
+			if contentLength <= 0 {
+				proxiedRequest, err = http.NewRequestWithContext(ctx, proxyMethod, endpoint, convertBody(ctx, r.Body))
+				if err != nil {
+					err = commonerrors.WrapError(commonerrors.ErrUnexpected, err, "could not create a proxied request")
+					return
+				}
+			}
+		}
+		if contentLength > 0 && proxiedRequest.ContentLength <= 0 {
+			proxiedRequest.ContentLength = contentLength
+			h.AppendHeader(headers.ContentLength, strconv.FormatInt(contentLength, 10))
+		}
+	}
+	if contentLength > 0 && contentLength != proxiedRequest.ContentLength {
+		err = commonerrors.Newf(commonerrors.ErrUnexpected, "proxied request does not have the same content length `%v` as original request `%v`", proxiedRequest.ContentLength, contentLength)
+		return
+	}
+	h.AppendToRequest(proxiedRequest)
+	return
+}
+
+func determineRequestContentLength(r *http.Request) int64 {
+	if reflection.IsEmpty(r) {
+		return -1
+	}
+	if r.ContentLength > 0 {
+		return r.ContentLength
+	}
+	// Following what was done in https://github.com/luraproject/lura/blob/b9ad9ab654dd6149aeb58a5d6ffe731aba41717e/proxy/http.go#L99C1-L105C4
+	v := r.Header.Values(headers.ContentLength)
+	if len(v) == 1 && v[0] != "chunked" {
+		if size, err := strconv.Atoi(v[0]); err == nil {
+			return safecast.ToInt64(size)
+		}
+	}
+	return -1
+}
+
+func convertBody(_ context.Context, body io.Reader) io.Reader {
+	if body == nil || body == http.NoBody {
+		return http.NoBody
+	}
+	switch v := body.(type) {
+	case *bytes.Buffer:
+		return body
+	case *bytes.Reader:
+		return body
+	case *strings.Reader:
+		return body
+	default:
+		// see example https://github.com/luraproject/lura/blob/b9ad9ab654dd6149aeb58a5d6ffe731aba41717e/proxy/http.go#L73
+		buf := new(bytes.Buffer)
+		_, err := buf.ReadFrom(v)
+		if err != nil {
+			return http.NoBody
+		}
+		if b, ok := body.(io.ReadCloser); ok {
+			_ = b.Close()
+		}
+		return buf
+	}
+}
+
+// ProxyResponse proxies a response to a writer. Headers are sanitised and some headers such as CORS headers will be removed from the response.
+func ProxyResponse(ctx context.Context, resp *http.Response, w http.ResponseWriter) (err error) {
+	if w == nil {
+		err = commonerrors.UndefinedVariable("response writer")
+		return
+	}
+	if reflection.IsEmpty(resp) {
+		err = commonerrors.UndefinedVariable("response")
+		return
+	}
+	h := httpheaders.FromResponse(resp)
+	h.Sanitise()
+
+	var written int64
+	_, err = safeio.CopyDataWithContext(ctx, resp.Body, w)
+	if resp.Body != nil && resp.Body != http.NoBody {
+		written, err = safeio.CopyDataWithContext(ctx, resp.Body, w)
+		if err != nil {
+			err = commonerrors.DescribeCircumstance(err, "failed copying response body")
+		}
+	}
+	if written >= 0 {
+		h.AppendHeader(headers.ContentLength, strconv.FormatInt(written, 10))
+	}
+	h.RemoveHeaders(ProxyDisallowList...)
+	h.AppendToResponse(w)
+	w.WriteHeader(resp.StatusCode)
+	return
+}

utils/http/proxy/proxy_test.go

@@ -0,0 +1,164 @@
+package proxy
+
+import (
+	"bytes"
+	"context"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"strconv"
+	"strings"
+	"testing"
+
+	"github.com/go-faker/faker/v4"
+	"github.com/go-http-utils/headers"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/ARM-software/golang-utils/utils/commonerrors"
+	"github.com/ARM-software/golang-utils/utils/commonerrors/errortest"
+	"github.com/ARM-software/golang-utils/utils/safecast"
+	"github.com/ARM-software/golang-utils/utils/safeio"
+)
+
+func TestProxy(t *testing.T) {
+	content := faker.Paragraph()
+	path := faker.URL()
+	password := faker.Password()
+	tests := []struct {
+		request *http.Request
+	}{
+		{
+			request: httptest.NewRequest(http.MethodGet, faker.URL(), io.NopCloser(strings.NewReader(content))),
+		},
+		{
+			request: httptest.NewRequest(http.MethodGet, faker.URL(), strings.NewReader(content)),
+		},
+		{
+			request: httptest.NewRequest(http.MethodGet, faker.URL(), io.NopCloser(bytes.NewReader([]byte(content)))),
+		},
+		{
+			request: httptest.NewRequest(http.MethodGet, faker.URL(), bytes.NewReader([]byte(content))),
+		},
+		{
+			request: httptest.NewRequest(http.MethodGet, faker.URL(), io.NopCloser(bytes.NewBuffer([]byte(content)))),
+		},
+		{
+			request: httptest.NewRequest(http.MethodGet, faker.URL(), bytes.NewBuffer([]byte(content))),
+		},
+	}
+	for i := range tests {
+		test := tests[i]
+		t.Run(strconv.Itoa(i), func(t *testing.T) {
+			req := test.request
+			req.Header.Set(headers.AccessControlAllowOrigin, faker.Word())
+			req.Header.Set(headers.XHTTPMethodOverride, http.MethodPut)
+			req.Header.Set(headers.Authorization, password)
+			assert.NotEqual(t, req.URL.String(), path)
+			_, err := ProxyRequest(nil, http.MethodPost, "/")
+			errortest.AssertError(t, err, commonerrors.ErrUndefined)
+			preq, err := ProxyRequest(req, "     ", path)
+			require.NoError(t, err)
+			require.NotNil(t, preq)
+			assert.Equal(t, path, preq.URL.String())
+			assert.Equal(t, http.MethodGet, preq.Method)
+			assert.NotEmpty(t, preq.Header.Get(headers.AccessControlAllowOrigin))
+			assert.NotEmpty(t, preq.Header.Get(headers.Authorization))
+			assert.NotZero(t, preq.ContentLength)
+			resp := generateTestResponseBasedOnRequest(t, preq)
+			defer func() {
+				if resp != nil {
+					_ = resp.Body.Close()
+				}
+			}()
+			w := httptest.NewRecorder()
+			require.NoError(t, ProxyResponse(context.Background(), resp, w))
+			proxiedResp := w.Result()
+			defer func() { _ = proxiedResp.Body.Close() }()
+			assert.Empty(t, w.Header().Get(headers.AccessControlAllowOrigin))
+			assert.Equal(t, http.MethodPut, w.Header().Get(headers.XHTTPMethodOverride))
+			assert.Equal(t, http.StatusOK, resp.StatusCode)
+			responseContent, err := safeio.ReadAll(context.Background(), proxiedResp.Body)
+			require.NoError(t, err)
+			assert.Equal(t, content, string(responseContent))
+		})
+	}
+}
+
+func TestEmptyResponse(t *testing.T) {
+	path := faker.URL()
+	tests := []struct {
+		request *http.Request
+	}{
+		{
+			httptest.NewRequest(http.MethodGet, faker.URL(), nil),
+		},
+		{
+			request: httptest.NewRequest(http.MethodGet, faker.URL(), http.NoBody),
+		},
+		{
+			request: httptest.NewRequest(http.MethodGet, faker.URL(), io.NopCloser(http.NoBody)),
+		},
+		{
+			request: httptest.NewRequest(http.MethodGet, faker.URL(), bytes.NewReader(nil)),
+		},
+		{
+			request: httptest.NewRequest(http.MethodGet, faker.URL(), io.NopCloser(bytes.NewBuffer(nil))),
+		},
+		{
+			request: httptest.NewRequest(http.MethodGet, faker.URL(), strings.NewReader("")),
+		},
+		{
+			request: httptest.NewRequest(http.MethodGet, faker.URL(), io.NopCloser(strings.NewReader(""))),
+		},
+	}
+	for i := range tests {
+		test := tests[i]
+		t.Run(strconv.Itoa(i), func(t *testing.T) {
+			req := test.request
+			assert.NotEqual(t, req.URL.String(), path)
+			preq, err := ProxyRequest(req, http.MethodPost, path)
+			require.NoError(t, err)
+			require.NotNil(t, preq)
+			assert.Equal(t, path, preq.URL.String())
+			assert.Equal(t, http.MethodPost, preq.Method)
+			assert.Zero(t, preq.ContentLength)
+
+			resp := generateTestResponseBasedOnRequest(t, preq)
+			defer func() {
+				if resp != nil {
+					_ = resp.Body.Close()
+				}
+			}()
+			w := httptest.NewRecorder()
+			require.NoError(t, ProxyResponse(context.Background(), resp, w))
+			require.NoError(t, err)
+			returnedResp := w.Result()
+			assert.LessOrEqual(t, returnedResp.ContentLength, safecast.ToInt64(0))
+			assert.Equal(t, http.StatusOK, returnedResp.StatusCode)
+		})
+	}
+}
+
+func loopTestHandler(t *testing.T, w http.ResponseWriter, r *http.Request) {
+	t.Helper()
+	require.NotNil(t, r)
+	require.NotNil(t, w)
+	for k, v := range r.Header {
+		for h := range v {
+			w.Header().Add(k, v[h])
+		}
+	}
+	written, err := safeio.CopyDataWithContext(r.Context(), r.Body, w)
+	require.NoError(t, err)
+	w.Header().Add(headers.ContentLength, strconv.FormatInt(written, 10))
+	w.WriteHeader(http.StatusOK)
+}
+
+func generateTestResponseBasedOnRequest(t *testing.T, r *http.Request) *http.Response {
+	t.Helper()
+	require.NotNil(t, r)
+	w := httptest.NewRecorder()
+	loopTestHandler(t, w, r)
+	return w.Result()
+}