package providerapi import ( "context" "encoding/json" "errors" "fmt" "io" "net/http" "net/http/httptest" "strings" "testing" "time" "github.com/proxy-pool/proxy-pool/internal/config" controllerProvider "github.com/proxy-pool/proxy-pool/internal/controller/provider" ) var _ controllerProvider.ProviderAdapter = (*HTTPAdapter)(nil) func TestHTTPAdapterBuildsStructuredRequestAndReturnsBody(t *testing.T) { server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { if request.Method != http.MethodPost { t.Errorf("method = %q, want POST", request.Method) } if got := request.URL.Query().Get("count"); got != "100" { t.Errorf("count query = %q, want 100", got) } if got := request.Header.Get("X-Provider-Key"); got != "secret" { t.Errorf("X-Provider-Key = %q, want secret", got) } if got := request.Header.Get("X-Request-Mode"); got != "batch" { t.Errorf("X-Request-Mode = %q, want batch", got) } var body map[string]string if err := json.NewDecoder(request.Body).Decode(&body); err != nil { t.Errorf("decode request body: %v", err) } if body["protocol"] != "http" { t.Errorf("protocol body = %q, want http", body["protocol"]) } _, _ = writer.Write([]byte("http://192.0.2.20:8080")) })) defer server.Close() adapter, err := NewHTTPAdapter(config.ProviderAPI{ URL: server.URL + "/proxies", Method: http.MethodPost, Headers: map[string]string{"X-Request-Mode": "batch"}, Query: map[string]string{"count": "100"}, Auth: config.ProviderAuth{ Type: "apiKey", Location: "header", Name: "X-Provider-Key", Value: "secret", }, Body: config.APIBody{Type: "json", Value: map[string]string{"protocol": "http"}}, }, config.Fetch{MaxResponseBytes: 1024}, server.Client()) if err != nil { t.Fatalf("NewHTTPAdapter(): %v", err) } response, err := adapter.Fetch(context.Background()) if err != nil { t.Fatalf("Fetch(): %v", err) } if got := string(response.Body); got != "http://192.0.2.20:8080" { t.Fatalf("response body = %q", got) } } func TestHTTPAdapterFormattingDoesNotExposeProviderSecrets(t *testing.T) { t.Parallel() adapter, err := NewHTTPAdapter(config.ProviderAPI{ URL: "https://provider.test/fetch", Method: http.MethodGet, Auth: config.ProviderAuth{ Type: "apiKey", Location: "query", Name: "token", Value: "provider-secret", }, }, config.Fetch{MaxResponseBytes: 1024}, doerFunc(func(*http.Request) (*http.Response, error) { return nil, errors.New("unused") })) if err != nil { t.Fatalf("NewHTTPAdapter(): %v", err) } formatted := fmt.Sprintf("%v %+v %#v", adapter, adapter, adapter) for _, secret := range []string{"provider-secret", "token=", "provider.test/fetch"} { if strings.Contains(formatted, secret) { t.Fatalf("formatted adapter contains %q: %s", secret, formatted) } } } func TestHTTPAdapterRedactsTransportErrorURLAndQuerySecret(t *testing.T) { transportErr := errors.New("dial failed") adapter, err := NewHTTPAdapter(config.ProviderAPI{ URL: "https://provider.example/proxies", Method: http.MethodGet, Auth: config.ProviderAuth{ Type: "apiKey", Location: "query", Name: "api_key", Value: "top-secret", }, }, config.Fetch{MaxResponseBytes: 1024}, doerFunc(func(request *http.Request) (*http.Response, error) { return nil, fmt.Errorf("request %s: %w", request.URL.String(), transportErr) })) if err != nil { t.Fatalf("NewHTTPAdapter(): %v", err) } _, err = adapter.Fetch(context.Background()) if !errors.Is(err, transportErr) { t.Fatalf("Fetch() error = %v, want wrapped transport cause", err) } if strings.Contains(err.Error(), "top-secret") || strings.Contains(err.Error(), "provider.example") { t.Fatalf("transport error leaked Provider URL or API key: %v", err) } } func TestHTTPAdapterClosesResponseBodyReturnedWithTransportError(t *testing.T) { body := &trackingBody{Reader: strings.NewReader("unused")} adapter, err := NewHTTPAdapter( config.ProviderAPI{URL: "https://provider.example/proxies", Method: http.MethodGet}, config.Fetch{MaxResponseBytes: 1024}, doerFunc(func(*http.Request) (*http.Response, error) { return &http.Response{StatusCode: http.StatusBadGateway, Body: body}, errors.New("transport failed") }), ) if err != nil { t.Fatalf("NewHTTPAdapter(): %v", err) } _, _ = adapter.Fetch(context.Background()) if !body.closed { t.Fatal("response body returned with transport error was not closed") } } func TestHTTPAdapterRejectsConflictingAuthenticationFields(t *testing.T) { tests := []struct { name string api config.ProviderAPI }{ { name: "bearer conflicts with authorization header", api: config.ProviderAPI{ URL: "https://provider.example/proxies", Headers: map[string]string{"Authorization": "configured"}, Auth: config.ProviderAuth{Type: "bearer", Token: "resolved-token"}, }, }, { name: "API key conflicts with configured header", api: config.ProviderAPI{ URL: "https://provider.example/proxies", Headers: map[string]string{"X-API-Key": "configured"}, Auth: config.ProviderAuth{ Type: "apiKey", Location: "header", Name: "x-api-key", Value: "resolved-key", }, }, }, { name: "API key conflicts with URL query", api: config.ProviderAPI{ URL: "https://provider.example/proxies?api_key=configured", Auth: config.ProviderAuth{ Type: "apiKey", Location: "query", Name: "api_key", Value: "resolved-key", }, }, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { if _, err := NewHTTPAdapter(tt.api, config.Fetch{MaxResponseBytes: 1024}, nil); err == nil { t.Fatal("NewHTTPAdapter() error = nil, want authentication conflict") } }) } } type doerFunc func(*http.Request) (*http.Response, error) func (f doerFunc) Do(request *http.Request) (*http.Response, error) { return f(request) } func TestHTTPAdapterClassifiesStatusAndRetryAfter(t *testing.T) { tests := []struct { name string status int retryAfter string retryMax config.Duration wantRetry bool wantDelay time.Duration }{ {name: "bad request is permanent", status: http.StatusBadRequest}, {name: "authentication is permanent", status: http.StatusUnauthorized}, {name: "request timeout is transient", status: http.StatusRequestTimeout, wantRetry: true}, {name: "rate limit honors bounded retry after", status: http.StatusTooManyRequests, retryAfter: "60", retryMax: config.Duration(2 * time.Second), wantRetry: true, wantDelay: 2 * time.Second}, {name: "server error is transient", status: http.StatusServiceUnavailable, retryAfter: "1", retryMax: config.Duration(5 * time.Second), wantRetry: true, wantDelay: time.Second}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, _ *http.Request) { writer.Header().Set("Retry-After", tt.retryAfter) writer.WriteHeader(tt.status) _, _ = writer.Write([]byte("sensitive provider response")) })) defer server.Close() adapter, err := NewHTTPAdapter( config.ProviderAPI{URL: server.URL, Method: http.MethodGet}, config.Fetch{MaxResponseBytes: 1024, Retry: config.Backoff{Max: tt.retryMax}}, server.Client(), ) if err != nil { t.Fatalf("NewHTTPAdapter(): %v", err) } response, err := adapter.Fetch(context.Background()) if err == nil { t.Fatal("Fetch() error = nil, want HTTP status error") } var statusErr *HTTPStatusError if !errors.As(err, &statusErr) || statusErr.StatusCode != tt.status { t.Fatalf("Fetch() error = %v, want HTTPStatusError{%d}", err, tt.status) } var retryable controllerProvider.RetryableError if !errors.As(err, &retryable) || retryable.Retryable() != tt.wantRetry { t.Fatalf("Retryable() = %v, want %v", retryable != nil && retryable.Retryable(), tt.wantRetry) } if response.RetryAfter != tt.wantDelay { t.Fatalf("RetryAfter = %s, want %s", response.RetryAfter, tt.wantDelay) } if strings.Contains(err.Error(), "sensitive") || strings.Contains(err.Error(), server.URL) { t.Fatalf("status error leaked response or URL: %v", err) } }) } } func TestHTTPAdapterRejectsResponseOverMaxBytes(t *testing.T) { server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, _ *http.Request) { _, _ = writer.Write([]byte("12345")) })) defer server.Close() adapter, err := NewHTTPAdapter( config.ProviderAPI{URL: server.URL, Method: http.MethodGet}, config.Fetch{MaxResponseBytes: 4}, server.Client(), ) if err != nil { t.Fatalf("NewHTTPAdapter(): %v", err) } response, err := adapter.Fetch(context.Background()) if !errors.Is(err, ErrResponseTooLarge) { t.Fatalf("Fetch() error = %v, want ErrResponseTooLarge", err) } if response.Body != nil { t.Fatalf("response body = %q, want nil on overflow", response.Body) } } func TestHTTPAdapterPreservesPermanentStatusWhenErrorBodyIsOversized(t *testing.T) { server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, _ *http.Request) { writer.WriteHeader(http.StatusUnauthorized) _, _ = writer.Write([]byte("12345")) })) defer server.Close() adapter, err := NewHTTPAdapter( config.ProviderAPI{URL: server.URL, Method: http.MethodGet}, config.Fetch{MaxResponseBytes: 4}, server.Client(), ) if err != nil { t.Fatalf("NewHTTPAdapter(): %v", err) } _, err = adapter.Fetch(context.Background()) var statusErr *HTTPStatusError if !errors.As(err, &statusErr) || statusErr.StatusCode != http.StatusUnauthorized { t.Fatalf("Fetch() error = %v, want HTTPStatusError{401}", err) } if !errors.Is(err, ErrResponseTooLarge) { t.Fatalf("Fetch() error = %v, want joined ErrResponseTooLarge", err) } if statusErr.Retryable() { t.Fatal("HTTPStatusError{401}.Retryable() = true, want false") } } func TestHTTPAdapterAcceptsResponseAtExactByteLimitAndClosesBody(t *testing.T) { body := &trackingBody{Reader: strings.NewReader("1234")} adapter, err := NewHTTPAdapter( config.ProviderAPI{URL: "https://provider.example/proxies", Method: http.MethodGet}, config.Fetch{MaxResponseBytes: 4}, doerFunc(func(*http.Request) (*http.Response, error) { return &http.Response{StatusCode: http.StatusOK, Body: body, Header: make(http.Header)}, nil }), ) if err != nil { t.Fatalf("NewHTTPAdapter(): %v", err) } response, err := adapter.Fetch(context.Background()) if err != nil { t.Fatalf("Fetch(): %v", err) } if string(response.Body) != "1234" { t.Fatalf("response body = %q, want exact-limit body", response.Body) } if !body.closed { t.Fatal("response body was not closed") } } func TestHTTPAdapterParsesHTTPDateAndUsesSafeDefaultRetryAfterCap(t *testing.T) { server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, _ *http.Request) { writer.Header().Set("Retry-After", time.Now().Add(time.Hour).UTC().Format(http.TimeFormat)) writer.WriteHeader(http.StatusTooManyRequests) })) defer server.Close() adapter, err := NewHTTPAdapter( config.ProviderAPI{URL: server.URL, Method: http.MethodGet}, config.Fetch{MaxResponseBytes: 1024}, server.Client(), ) if err != nil { t.Fatalf("NewHTTPAdapter(): %v", err) } response, err := adapter.Fetch(context.Background()) if err == nil { t.Fatal("Fetch() error = nil, want rate limit status") } if response.RetryAfter != defaultRetryAfterMax { t.Fatalf("RetryAfter = %s, want safe default cap %s", response.RetryAfter, defaultRetryAfterMax) } } func TestHTTPAdapterRejectsMalformedDoerResponse(t *testing.T) { tests := []struct { name string response *http.Response }{ {name: "nil response"}, {name: "nil body", response: &http.Response{StatusCode: http.StatusOK}}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { adapter, err := NewHTTPAdapter( config.ProviderAPI{URL: "https://provider.example/proxies", Method: http.MethodGet}, config.Fetch{MaxResponseBytes: 1024}, doerFunc(func(*http.Request) (*http.Response, error) { return tt.response, nil }), ) if err != nil { t.Fatalf("NewHTTPAdapter(): %v", err) } _, err = adapter.Fetch(context.Background()) if !errors.Is(err, ErrInvalidHTTPResponse) { t.Fatalf("Fetch() error = %v, want ErrInvalidHTTPResponse", err) } }) } } func TestRetryableHTTPStatusRejectsOutOfRangeStatus(t *testing.T) { if retryableHTTPStatus(600) { t.Fatal("retryableHTTPStatus(600) = true, want false") } } type trackingBody struct { io.Reader closed bool } func (b *trackingBody) Close() error { b.closed = true return nil }