390 lines
12 KiB
Go
390 lines
12 KiB
Go
package providerapi
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/proxy-pool/proxy-pool/internal/config"
|
|
controllerProvider "github.com/proxy-pool/proxy-pool/internal/controller/provider"
|
|
)
|
|
|
|
var _ controllerProvider.ProviderAdapter = (*HTTPAdapter)(nil)
|
|
|
|
func TestHTTPAdapterBuildsStructuredRequestAndReturnsBody(t *testing.T) {
|
|
server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
|
|
if request.Method != http.MethodPost {
|
|
t.Errorf("method = %q, want POST", request.Method)
|
|
}
|
|
if got := request.URL.Query().Get("count"); got != "100" {
|
|
t.Errorf("count query = %q, want 100", got)
|
|
}
|
|
if got := request.Header.Get("X-Provider-Key"); got != "secret" {
|
|
t.Errorf("X-Provider-Key = %q, want secret", got)
|
|
}
|
|
if got := request.Header.Get("X-Request-Mode"); got != "batch" {
|
|
t.Errorf("X-Request-Mode = %q, want batch", got)
|
|
}
|
|
var body map[string]string
|
|
if err := json.NewDecoder(request.Body).Decode(&body); err != nil {
|
|
t.Errorf("decode request body: %v", err)
|
|
}
|
|
if body["protocol"] != "http" {
|
|
t.Errorf("protocol body = %q, want http", body["protocol"])
|
|
}
|
|
_, _ = writer.Write([]byte("http://192.0.2.20:8080"))
|
|
}))
|
|
defer server.Close()
|
|
|
|
adapter, err := NewHTTPAdapter(config.ProviderAPI{
|
|
URL: server.URL + "/proxies",
|
|
Method: http.MethodPost,
|
|
Headers: map[string]string{"X-Request-Mode": "batch"},
|
|
Query: map[string]string{"count": "100"},
|
|
Auth: config.ProviderAuth{
|
|
Type: "apiKey",
|
|
Location: "header",
|
|
Name: "X-Provider-Key",
|
|
Value: "secret",
|
|
},
|
|
Body: config.APIBody{Type: "json", Value: map[string]string{"protocol": "http"}},
|
|
}, config.Fetch{MaxResponseBytes: 1024}, server.Client())
|
|
if err != nil {
|
|
t.Fatalf("NewHTTPAdapter(): %v", err)
|
|
}
|
|
|
|
response, err := adapter.Fetch(context.Background())
|
|
if err != nil {
|
|
t.Fatalf("Fetch(): %v", err)
|
|
}
|
|
if got := string(response.Body); got != "http://192.0.2.20:8080" {
|
|
t.Fatalf("response body = %q", got)
|
|
}
|
|
}
|
|
|
|
func TestHTTPAdapterFormattingDoesNotExposeProviderSecrets(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
adapter, err := NewHTTPAdapter(config.ProviderAPI{
|
|
URL: "https://provider.test/fetch",
|
|
Method: http.MethodGet,
|
|
Auth: config.ProviderAuth{
|
|
Type: "apiKey", Location: "query", Name: "token", Value: "provider-secret",
|
|
},
|
|
}, config.Fetch{MaxResponseBytes: 1024}, doerFunc(func(*http.Request) (*http.Response, error) {
|
|
return nil, errors.New("unused")
|
|
}))
|
|
if err != nil {
|
|
t.Fatalf("NewHTTPAdapter(): %v", err)
|
|
}
|
|
|
|
formatted := fmt.Sprintf("%v %+v %#v", adapter, adapter, adapter)
|
|
for _, secret := range []string{"provider-secret", "token=", "provider.test/fetch"} {
|
|
if strings.Contains(formatted, secret) {
|
|
t.Fatalf("formatted adapter contains %q: %s", secret, formatted)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestHTTPAdapterRedactsTransportErrorURLAndQuerySecret(t *testing.T) {
|
|
transportErr := errors.New("dial failed")
|
|
adapter, err := NewHTTPAdapter(config.ProviderAPI{
|
|
URL: "https://provider.example/proxies",
|
|
Method: http.MethodGet,
|
|
Auth: config.ProviderAuth{
|
|
Type: "apiKey",
|
|
Location: "query",
|
|
Name: "api_key",
|
|
Value: "top-secret",
|
|
},
|
|
}, config.Fetch{MaxResponseBytes: 1024}, doerFunc(func(request *http.Request) (*http.Response, error) {
|
|
return nil, fmt.Errorf("request %s: %w", request.URL.String(), transportErr)
|
|
}))
|
|
if err != nil {
|
|
t.Fatalf("NewHTTPAdapter(): %v", err)
|
|
}
|
|
|
|
_, err = adapter.Fetch(context.Background())
|
|
if !errors.Is(err, transportErr) {
|
|
t.Fatalf("Fetch() error = %v, want wrapped transport cause", err)
|
|
}
|
|
if strings.Contains(err.Error(), "top-secret") || strings.Contains(err.Error(), "provider.example") {
|
|
t.Fatalf("transport error leaked Provider URL or API key: %v", err)
|
|
}
|
|
}
|
|
|
|
func TestHTTPAdapterClosesResponseBodyReturnedWithTransportError(t *testing.T) {
|
|
body := &trackingBody{Reader: strings.NewReader("unused")}
|
|
adapter, err := NewHTTPAdapter(
|
|
config.ProviderAPI{URL: "https://provider.example/proxies", Method: http.MethodGet},
|
|
config.Fetch{MaxResponseBytes: 1024},
|
|
doerFunc(func(*http.Request) (*http.Response, error) {
|
|
return &http.Response{StatusCode: http.StatusBadGateway, Body: body}, errors.New("transport failed")
|
|
}),
|
|
)
|
|
if err != nil {
|
|
t.Fatalf("NewHTTPAdapter(): %v", err)
|
|
}
|
|
|
|
_, _ = adapter.Fetch(context.Background())
|
|
if !body.closed {
|
|
t.Fatal("response body returned with transport error was not closed")
|
|
}
|
|
}
|
|
|
|
func TestHTTPAdapterRejectsConflictingAuthenticationFields(t *testing.T) {
|
|
tests := []struct {
|
|
name string
|
|
api config.ProviderAPI
|
|
}{
|
|
{
|
|
name: "bearer conflicts with authorization header",
|
|
api: config.ProviderAPI{
|
|
URL: "https://provider.example/proxies",
|
|
Headers: map[string]string{"Authorization": "configured"},
|
|
Auth: config.ProviderAuth{Type: "bearer", Token: "resolved-token"},
|
|
},
|
|
},
|
|
{
|
|
name: "API key conflicts with configured header",
|
|
api: config.ProviderAPI{
|
|
URL: "https://provider.example/proxies",
|
|
Headers: map[string]string{"X-API-Key": "configured"},
|
|
Auth: config.ProviderAuth{
|
|
Type: "apiKey", Location: "header", Name: "x-api-key", Value: "resolved-key",
|
|
},
|
|
},
|
|
},
|
|
{
|
|
name: "API key conflicts with URL query",
|
|
api: config.ProviderAPI{
|
|
URL: "https://provider.example/proxies?api_key=configured",
|
|
Auth: config.ProviderAuth{
|
|
Type: "apiKey", Location: "query", Name: "api_key", Value: "resolved-key",
|
|
},
|
|
},
|
|
},
|
|
}
|
|
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
if _, err := NewHTTPAdapter(tt.api, config.Fetch{MaxResponseBytes: 1024}, nil); err == nil {
|
|
t.Fatal("NewHTTPAdapter() error = nil, want authentication conflict")
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
type doerFunc func(*http.Request) (*http.Response, error)
|
|
|
|
func (f doerFunc) Do(request *http.Request) (*http.Response, error) { return f(request) }
|
|
|
|
func TestHTTPAdapterClassifiesStatusAndRetryAfter(t *testing.T) {
|
|
tests := []struct {
|
|
name string
|
|
status int
|
|
retryAfter string
|
|
retryMax config.Duration
|
|
wantRetry bool
|
|
wantDelay time.Duration
|
|
}{
|
|
{name: "bad request is permanent", status: http.StatusBadRequest},
|
|
{name: "authentication is permanent", status: http.StatusUnauthorized},
|
|
{name: "request timeout is transient", status: http.StatusRequestTimeout, wantRetry: true},
|
|
{name: "rate limit honors bounded retry after", status: http.StatusTooManyRequests, retryAfter: "60", retryMax: config.Duration(2 * time.Second), wantRetry: true, wantDelay: 2 * time.Second},
|
|
{name: "server error is transient", status: http.StatusServiceUnavailable, retryAfter: "1", retryMax: config.Duration(5 * time.Second), wantRetry: true, wantDelay: time.Second},
|
|
}
|
|
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, _ *http.Request) {
|
|
writer.Header().Set("Retry-After", tt.retryAfter)
|
|
writer.WriteHeader(tt.status)
|
|
_, _ = writer.Write([]byte("sensitive provider response"))
|
|
}))
|
|
defer server.Close()
|
|
|
|
adapter, err := NewHTTPAdapter(
|
|
config.ProviderAPI{URL: server.URL, Method: http.MethodGet},
|
|
config.Fetch{MaxResponseBytes: 1024, Retry: config.Backoff{Max: tt.retryMax}},
|
|
server.Client(),
|
|
)
|
|
if err != nil {
|
|
t.Fatalf("NewHTTPAdapter(): %v", err)
|
|
}
|
|
|
|
response, err := adapter.Fetch(context.Background())
|
|
if err == nil {
|
|
t.Fatal("Fetch() error = nil, want HTTP status error")
|
|
}
|
|
var statusErr *HTTPStatusError
|
|
if !errors.As(err, &statusErr) || statusErr.StatusCode != tt.status {
|
|
t.Fatalf("Fetch() error = %v, want HTTPStatusError{%d}", err, tt.status)
|
|
}
|
|
var retryable controllerProvider.RetryableError
|
|
if !errors.As(err, &retryable) || retryable.Retryable() != tt.wantRetry {
|
|
t.Fatalf("Retryable() = %v, want %v", retryable != nil && retryable.Retryable(), tt.wantRetry)
|
|
}
|
|
if response.RetryAfter != tt.wantDelay {
|
|
t.Fatalf("RetryAfter = %s, want %s", response.RetryAfter, tt.wantDelay)
|
|
}
|
|
if strings.Contains(err.Error(), "sensitive") || strings.Contains(err.Error(), server.URL) {
|
|
t.Fatalf("status error leaked response or URL: %v", err)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestHTTPAdapterRejectsResponseOverMaxBytes(t *testing.T) {
|
|
server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, _ *http.Request) {
|
|
_, _ = writer.Write([]byte("12345"))
|
|
}))
|
|
defer server.Close()
|
|
|
|
adapter, err := NewHTTPAdapter(
|
|
config.ProviderAPI{URL: server.URL, Method: http.MethodGet},
|
|
config.Fetch{MaxResponseBytes: 4},
|
|
server.Client(),
|
|
)
|
|
if err != nil {
|
|
t.Fatalf("NewHTTPAdapter(): %v", err)
|
|
}
|
|
|
|
response, err := adapter.Fetch(context.Background())
|
|
if !errors.Is(err, ErrResponseTooLarge) {
|
|
t.Fatalf("Fetch() error = %v, want ErrResponseTooLarge", err)
|
|
}
|
|
if response.Body != nil {
|
|
t.Fatalf("response body = %q, want nil on overflow", response.Body)
|
|
}
|
|
}
|
|
|
|
func TestHTTPAdapterPreservesPermanentStatusWhenErrorBodyIsOversized(t *testing.T) {
|
|
server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, _ *http.Request) {
|
|
writer.WriteHeader(http.StatusUnauthorized)
|
|
_, _ = writer.Write([]byte("12345"))
|
|
}))
|
|
defer server.Close()
|
|
|
|
adapter, err := NewHTTPAdapter(
|
|
config.ProviderAPI{URL: server.URL, Method: http.MethodGet},
|
|
config.Fetch{MaxResponseBytes: 4},
|
|
server.Client(),
|
|
)
|
|
if err != nil {
|
|
t.Fatalf("NewHTTPAdapter(): %v", err)
|
|
}
|
|
|
|
_, err = adapter.Fetch(context.Background())
|
|
var statusErr *HTTPStatusError
|
|
if !errors.As(err, &statusErr) || statusErr.StatusCode != http.StatusUnauthorized {
|
|
t.Fatalf("Fetch() error = %v, want HTTPStatusError{401}", err)
|
|
}
|
|
if !errors.Is(err, ErrResponseTooLarge) {
|
|
t.Fatalf("Fetch() error = %v, want joined ErrResponseTooLarge", err)
|
|
}
|
|
if statusErr.Retryable() {
|
|
t.Fatal("HTTPStatusError{401}.Retryable() = true, want false")
|
|
}
|
|
}
|
|
|
|
func TestHTTPAdapterAcceptsResponseAtExactByteLimitAndClosesBody(t *testing.T) {
|
|
body := &trackingBody{Reader: strings.NewReader("1234")}
|
|
adapter, err := NewHTTPAdapter(
|
|
config.ProviderAPI{URL: "https://provider.example/proxies", Method: http.MethodGet},
|
|
config.Fetch{MaxResponseBytes: 4},
|
|
doerFunc(func(*http.Request) (*http.Response, error) {
|
|
return &http.Response{StatusCode: http.StatusOK, Body: body, Header: make(http.Header)}, nil
|
|
}),
|
|
)
|
|
if err != nil {
|
|
t.Fatalf("NewHTTPAdapter(): %v", err)
|
|
}
|
|
|
|
response, err := adapter.Fetch(context.Background())
|
|
if err != nil {
|
|
t.Fatalf("Fetch(): %v", err)
|
|
}
|
|
if string(response.Body) != "1234" {
|
|
t.Fatalf("response body = %q, want exact-limit body", response.Body)
|
|
}
|
|
if !body.closed {
|
|
t.Fatal("response body was not closed")
|
|
}
|
|
}
|
|
|
|
func TestHTTPAdapterParsesHTTPDateAndUsesSafeDefaultRetryAfterCap(t *testing.T) {
|
|
server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, _ *http.Request) {
|
|
writer.Header().Set("Retry-After", time.Now().Add(time.Hour).UTC().Format(http.TimeFormat))
|
|
writer.WriteHeader(http.StatusTooManyRequests)
|
|
}))
|
|
defer server.Close()
|
|
adapter, err := NewHTTPAdapter(
|
|
config.ProviderAPI{URL: server.URL, Method: http.MethodGet},
|
|
config.Fetch{MaxResponseBytes: 1024},
|
|
server.Client(),
|
|
)
|
|
if err != nil {
|
|
t.Fatalf("NewHTTPAdapter(): %v", err)
|
|
}
|
|
|
|
response, err := adapter.Fetch(context.Background())
|
|
if err == nil {
|
|
t.Fatal("Fetch() error = nil, want rate limit status")
|
|
}
|
|
if response.RetryAfter != defaultRetryAfterMax {
|
|
t.Fatalf("RetryAfter = %s, want safe default cap %s", response.RetryAfter, defaultRetryAfterMax)
|
|
}
|
|
}
|
|
|
|
func TestHTTPAdapterRejectsMalformedDoerResponse(t *testing.T) {
|
|
tests := []struct {
|
|
name string
|
|
response *http.Response
|
|
}{
|
|
{name: "nil response"},
|
|
{name: "nil body", response: &http.Response{StatusCode: http.StatusOK}},
|
|
}
|
|
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
adapter, err := NewHTTPAdapter(
|
|
config.ProviderAPI{URL: "https://provider.example/proxies", Method: http.MethodGet},
|
|
config.Fetch{MaxResponseBytes: 1024},
|
|
doerFunc(func(*http.Request) (*http.Response, error) { return tt.response, nil }),
|
|
)
|
|
if err != nil {
|
|
t.Fatalf("NewHTTPAdapter(): %v", err)
|
|
}
|
|
|
|
_, err = adapter.Fetch(context.Background())
|
|
if !errors.Is(err, ErrInvalidHTTPResponse) {
|
|
t.Fatalf("Fetch() error = %v, want ErrInvalidHTTPResponse", err)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestRetryableHTTPStatusRejectsOutOfRangeStatus(t *testing.T) {
|
|
if retryableHTTPStatus(600) {
|
|
t.Fatal("retryableHTTPStatus(600) = true, want false")
|
|
}
|
|
}
|
|
|
|
type trackingBody struct {
|
|
io.Reader
|
|
closed bool
|
|
}
|
|
|
|
func (b *trackingBody) Close() error {
|
|
b.closed = true
|
|
return nil
|
|
}
|