1066 lines
33 KiB
Go
1066 lines
33 KiB
Go
package provider
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"sync"
|
|
"sync/atomic"
|
|
"testing"
|
|
"time"
|
|
|
|
"proxy-pool/internal/domain/activitypool"
|
|
proxyDomain "proxy-pool/internal/domain/proxy"
|
|
"proxy-pool/internal/domain/upstream"
|
|
)
|
|
|
|
func TestReconcilerWritesProviderTTLPolicyToEphemeralPool(t *testing.T) {
|
|
now := time.Date(2026, 7, 29, 12, 0, 0, 0, time.UTC)
|
|
clock := newFakeClock(now)
|
|
results := make(chan Result, 1)
|
|
batches := make(chan activitypool.FetchedBatch, 1)
|
|
ports := successfulPorts(func() {}, results)
|
|
ports.Activity = activitySinkFunc(func(_ context.Context, upstreamID string, batch activitypool.FetchedBatch) (activitypool.UpsertResult, error) {
|
|
if upstreamID != "provider-a" {
|
|
t.Errorf("upstream ID = %q, want provider-a", upstreamID)
|
|
}
|
|
batches <- batch
|
|
return activitypool.UpsertResult{Accepted: 1, Inserted: 1}, nil
|
|
})
|
|
reconciler, err := NewReconciler(Config{
|
|
UpstreamID: "provider-a",
|
|
Timeout: time.Second,
|
|
MaxAttempts: 1,
|
|
MaxInFlight: 1,
|
|
MaxSize: 321,
|
|
TTL: 30 * time.Second,
|
|
AllocationSafetyMargin: 3 * time.Second,
|
|
}, ports, Runtime{Clock: clock})
|
|
if err != nil {
|
|
t.Fatalf("NewReconciler(): %v", err)
|
|
}
|
|
|
|
result := runSingleReconcile(t, reconciler, results)
|
|
if result.ValidCount != 1 || result.NewCount != 1 {
|
|
t.Fatalf("result = %+v, want valid=1 new=1", result)
|
|
}
|
|
batch := <-batches
|
|
if !batch.ObservedAt.Equal(now) || batch.ConfiguredTTL != 30*time.Second ||
|
|
batch.AllocationSafetyMargin != 3*time.Second || batch.MaxSize != 321 {
|
|
t.Fatalf("activity batch = %+v", batch)
|
|
}
|
|
}
|
|
|
|
func TestReconcilerCoalescesConcurrentNotifications(t *testing.T) {
|
|
var calls atomic.Int64
|
|
result := make(chan Result, 1)
|
|
reconciler, err := NewReconciler(Config{
|
|
UpstreamID: "provider-a",
|
|
Timeout: time.Second,
|
|
MaxAttempts: 1,
|
|
MaxInFlight: 1,
|
|
MaxSize: 100,
|
|
}, Ports{
|
|
Adapter: adapterFunc(func(context.Context) (FetchResponse, error) {
|
|
calls.Add(1)
|
|
return FetchResponse{Body: []byte("fixture")}, nil
|
|
}),
|
|
Parser: parserFunc(func(context.Context, []byte) ([]proxyDomain.Proxy, error) {
|
|
return []proxyDomain.Proxy{{ID: "proxy-1"}}, nil
|
|
}),
|
|
Activity: activitySinkFunc(func(context.Context, string, activitypool.FetchedBatch) (activitypool.UpsertResult, error) {
|
|
return activitypool.UpsertResult{Accepted: 1, Inserted: 1}, nil
|
|
}),
|
|
Results: resultRecorderFunc(func(got Result) { result <- got }),
|
|
Capacity: unlimitedFetchCapacity{},
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("NewReconciler(): %v", err)
|
|
}
|
|
|
|
var wg sync.WaitGroup
|
|
for range 100 {
|
|
wg.Add(1)
|
|
go func() {
|
|
defer wg.Done()
|
|
reconciler.Notify()
|
|
}()
|
|
}
|
|
wg.Wait()
|
|
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
done := make(chan error, 1)
|
|
go func() { done <- reconciler.RunLeader(ctx, unlimitedLeaderSession{}) }()
|
|
|
|
select {
|
|
case got := <-result:
|
|
if got.Class != upstream.FetchValid {
|
|
t.Fatalf("result class = %q, want %q", got.Class, upstream.FetchValid)
|
|
}
|
|
case <-time.After(time.Second):
|
|
t.Fatal("timed out waiting for reconcile result")
|
|
}
|
|
cancel()
|
|
if err := <-done; err != nil {
|
|
t.Fatalf("Run(): %v", err)
|
|
}
|
|
if got := calls.Load(); got != 1 {
|
|
t.Fatalf("ProviderAdapter.Fetch() calls = %d, want 1", got)
|
|
}
|
|
}
|
|
|
|
func TestReconcilerEnforcesRequestInterval(t *testing.T) {
|
|
clock := newFakeClock(time.Date(2026, 7, 28, 12, 0, 0, 0, time.UTC))
|
|
sleeper := &fakeSleeper{clock: clock}
|
|
calledAt := make(chan time.Time, 2)
|
|
results := make(chan Result, 2)
|
|
reconciler, err := NewReconciler(Config{
|
|
UpstreamID: "provider-a",
|
|
RequestInterval: 250 * time.Millisecond,
|
|
Timeout: time.Second,
|
|
MaxAttempts: 1,
|
|
MaxInFlight: 1,
|
|
MaxSize: 100,
|
|
}, successfulPorts(func() { calledAt <- clock.Now() }, results), Runtime{
|
|
Clock: clock,
|
|
Sleeper: sleeper,
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("NewReconciler(): %v", err)
|
|
}
|
|
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
done := make(chan error, 1)
|
|
go func() { done <- reconciler.RunLeader(ctx, unlimitedLeaderSession{}) }()
|
|
|
|
reconciler.Notify()
|
|
<-results
|
|
reconciler.Notify()
|
|
<-results
|
|
cancel()
|
|
if err := <-done; err != nil {
|
|
t.Fatalf("Run(): %v", err)
|
|
}
|
|
|
|
first, second := <-calledAt, <-calledAt
|
|
if got := second.Sub(first); got != 250*time.Millisecond {
|
|
t.Fatalf("Provider calls separated by %s, want 250ms", got)
|
|
}
|
|
if got := sleeper.Durations(); len(got) != 1 || got[0] != 250*time.Millisecond {
|
|
t.Fatalf("Sleep durations = %v, want [250ms]", got)
|
|
}
|
|
}
|
|
|
|
func TestReconcilerRetriesErrorsWithExponentialBackoffAndJitter(t *testing.T) {
|
|
clock := newFakeClock(time.Date(2026, 7, 28, 12, 0, 0, 0, time.UTC))
|
|
sleeper := &fakeSleeper{clock: clock}
|
|
results := make(chan Result, 3)
|
|
var calls atomic.Int64
|
|
ports := successfulPorts(func() {}, results)
|
|
ports.Adapter = adapterFunc(func(context.Context) (FetchResponse, error) {
|
|
if calls.Add(1) < 3 {
|
|
return FetchResponse{}, errors.New("provider unavailable")
|
|
}
|
|
return FetchResponse{Body: []byte("fixture")}, nil
|
|
})
|
|
ports.Activity = activitySinkFunc(func(context.Context, string, activitypool.FetchedBatch) (activitypool.UpsertResult, error) {
|
|
return activitypool.UpsertResult{Accepted: 1}, nil
|
|
})
|
|
reconciler, err := NewReconciler(Config{
|
|
UpstreamID: "provider-a",
|
|
Timeout: time.Second,
|
|
MaxAttempts: 3,
|
|
MaxInFlight: 1,
|
|
MaxSize: 100,
|
|
Retry: RetryConfig{
|
|
Initial: 100 * time.Millisecond,
|
|
Max: time.Second,
|
|
Jitter: 20,
|
|
},
|
|
}, ports, Runtime{Clock: clock, Sleeper: sleeper, Random: fixedRandom(0.75)})
|
|
if err != nil {
|
|
t.Fatalf("NewReconciler(): %v", err)
|
|
}
|
|
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
done := make(chan error, 1)
|
|
go func() { done <- reconciler.RunLeader(ctx, unlimitedLeaderSession{}) }()
|
|
reconciler.Notify()
|
|
|
|
wantClasses := []upstream.FetchClass{
|
|
upstream.FetchError,
|
|
upstream.FetchError,
|
|
upstream.FetchDuplicateOnly,
|
|
}
|
|
for attempt, want := range wantClasses {
|
|
select {
|
|
case got := <-results:
|
|
if got.Class != want || got.Attempt != attempt+1 {
|
|
t.Fatalf("result %d = {class:%q attempt:%d}, want {class:%q attempt:%d}", attempt, got.Class, got.Attempt, want, attempt+1)
|
|
}
|
|
case <-time.After(time.Second):
|
|
t.Fatalf("timed out waiting for result %d", attempt)
|
|
}
|
|
}
|
|
cancel()
|
|
if err := <-done; err != nil {
|
|
t.Fatalf("Run(): %v", err)
|
|
}
|
|
|
|
if got := sleeper.Durations(); len(got) != 2 || got[0] != 110*time.Millisecond || got[1] != 220*time.Millisecond {
|
|
t.Fatalf("Sleep durations = %v, want [110ms 220ms]", got)
|
|
}
|
|
}
|
|
|
|
func TestReconcilerClassifiesFetchResults(t *testing.T) {
|
|
tests := []struct {
|
|
name string
|
|
callErr error
|
|
parseErr error
|
|
candidates int
|
|
newCount int
|
|
want upstream.FetchClass
|
|
}{
|
|
{name: "valid", candidates: 2, newCount: 1, want: upstream.FetchValid},
|
|
{name: "empty", want: upstream.FetchEmpty},
|
|
{name: "duplicate only", candidates: 2, want: upstream.FetchDuplicateOnly},
|
|
{name: "provider error", callErr: errors.New("HTTP 500"), want: upstream.FetchError},
|
|
{name: "parser error", parseErr: errors.New("invalid template output"), want: upstream.FetchError},
|
|
}
|
|
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
results := make(chan Result, 1)
|
|
ports := Ports{
|
|
Adapter: adapterFunc(func(context.Context) (FetchResponse, error) {
|
|
return FetchResponse{Body: []byte("fixture")}, tt.callErr
|
|
}),
|
|
Parser: parserFunc(func(context.Context, []byte) ([]proxyDomain.Proxy, error) {
|
|
return make([]proxyDomain.Proxy, tt.candidates), tt.parseErr
|
|
}),
|
|
Activity: activitySinkFunc(func(context.Context, string, activitypool.FetchedBatch) (activitypool.UpsertResult, error) {
|
|
return activitypool.UpsertResult{Accepted: tt.candidates, Inserted: tt.newCount}, nil
|
|
}),
|
|
Results: resultRecorderFunc(func(got Result) { results <- got }),
|
|
Capacity: unlimitedFetchCapacity{},
|
|
}
|
|
reconciler, err := NewReconciler(Config{
|
|
UpstreamID: "provider-a",
|
|
Timeout: time.Second,
|
|
MaxAttempts: 1,
|
|
MaxInFlight: 1,
|
|
MaxSize: 100,
|
|
}, ports)
|
|
if err != nil {
|
|
t.Fatalf("NewReconciler(): %v", err)
|
|
}
|
|
|
|
got := runSingleReconcile(t, reconciler, results)
|
|
if got.Class != tt.want {
|
|
t.Fatalf("result class = %q, want %q", got.Class, tt.want)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestReconcilerHonorsRetryAfterBeforeBackoff(t *testing.T) {
|
|
clock := newFakeClock(time.Date(2026, 7, 28, 12, 0, 0, 0, time.UTC))
|
|
sleeper := &fakeSleeper{clock: clock}
|
|
calledAt := make(chan time.Time, 2)
|
|
results := make(chan Result, 2)
|
|
var calls atomic.Int64
|
|
ports := successfulPorts(func() { calledAt <- clock.Now() }, results)
|
|
ports.Adapter = adapterFunc(func(context.Context) (FetchResponse, error) {
|
|
calledAt <- clock.Now()
|
|
if calls.Add(1) == 1 {
|
|
return FetchResponse{RetryAfter: 700 * time.Millisecond}, errors.New("rate limited")
|
|
}
|
|
return FetchResponse{Body: []byte("fixture")}, nil
|
|
})
|
|
reconciler, err := NewReconciler(Config{
|
|
UpstreamID: "provider-a",
|
|
RequestInterval: 100 * time.Millisecond,
|
|
Timeout: time.Second,
|
|
MaxAttempts: 2,
|
|
MaxInFlight: 1,
|
|
MaxSize: 100,
|
|
Retry: RetryConfig{Initial: 50 * time.Millisecond, Max: time.Second, Jitter: 20},
|
|
}, ports, Runtime{Clock: clock, Sleeper: sleeper, Random: fixedRandom(1)})
|
|
if err != nil {
|
|
t.Fatalf("NewReconciler(): %v", err)
|
|
}
|
|
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
done := make(chan error, 1)
|
|
go func() { done <- reconciler.RunLeader(ctx, unlimitedLeaderSession{}) }()
|
|
reconciler.Notify()
|
|
<-results
|
|
<-results
|
|
cancel()
|
|
if err := <-done; err != nil {
|
|
t.Fatalf("Run(): %v", err)
|
|
}
|
|
|
|
first, second := <-calledAt, <-calledAt
|
|
if got := second.Sub(first); got != 700*time.Millisecond {
|
|
t.Fatalf("Provider calls separated by %s, want Retry-After 700ms", got)
|
|
}
|
|
}
|
|
|
|
func TestReconcilerCapsRetryAfter(t *testing.T) {
|
|
clock := newFakeClock(time.Date(2026, 7, 28, 12, 0, 0, 0, time.UTC))
|
|
sleeper := &fakeSleeper{clock: clock}
|
|
results := make(chan Result, 2)
|
|
var calls atomic.Int64
|
|
ports := successfulPorts(func() {}, results)
|
|
ports.Adapter = adapterFunc(func(context.Context) (FetchResponse, error) {
|
|
if calls.Add(1) == 1 {
|
|
return FetchResponse{RetryAfter: time.Minute}, errors.New("rate limited")
|
|
}
|
|
return FetchResponse{Body: []byte("fixture")}, nil
|
|
})
|
|
reconciler, err := NewReconciler(Config{
|
|
UpstreamID: "provider-a",
|
|
Timeout: time.Second,
|
|
MaxAttempts: 2,
|
|
MaxInFlight: 1,
|
|
MaxSize: 100,
|
|
Retry: RetryConfig{Initial: 50 * time.Millisecond, Max: 2 * time.Second},
|
|
}, ports, Runtime{Clock: clock, Sleeper: sleeper})
|
|
if err != nil {
|
|
t.Fatalf("NewReconciler(): %v", err)
|
|
}
|
|
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
done := make(chan error, 1)
|
|
go func() { done <- reconciler.RunLeader(ctx, unlimitedLeaderSession{}) }()
|
|
reconciler.Notify()
|
|
<-results
|
|
<-results
|
|
cancel()
|
|
if err := <-done; err != nil {
|
|
t.Fatalf("Run(): %v", err)
|
|
}
|
|
if got := sleeper.Durations(); len(got) != 1 || got[0] != 2*time.Second {
|
|
t.Fatalf("Sleep durations = %v, want capped Retry-After [2s]", got)
|
|
}
|
|
}
|
|
|
|
func TestReconcilerAppliesAttemptTimeoutToEveryPort(t *testing.T) {
|
|
results := make(chan Result, 1)
|
|
ports := successfulPorts(func() {}, results)
|
|
var stages atomic.Int64
|
|
assertDeadline := func(ctx context.Context, stage string) {
|
|
deadline, ok := ctx.Deadline()
|
|
if !ok {
|
|
t.Errorf("%s context has no deadline", stage)
|
|
return
|
|
}
|
|
remaining := time.Until(deadline)
|
|
if remaining <= 0 || remaining > 250*time.Millisecond {
|
|
t.Errorf("%s deadline remaining = %s, want (0, 250ms]", stage, remaining)
|
|
}
|
|
stages.Add(1)
|
|
}
|
|
ports.Adapter = adapterFunc(func(ctx context.Context) (FetchResponse, error) {
|
|
assertDeadline(ctx, "ProviderAdapter.Fetch")
|
|
return FetchResponse{Body: []byte("fixture")}, nil
|
|
})
|
|
ports.Parser = parserFunc(func(ctx context.Context, _ []byte) ([]proxyDomain.Proxy, error) {
|
|
assertDeadline(ctx, "Parser.Parse")
|
|
return []proxyDomain.Proxy{{ID: "proxy-1"}}, nil
|
|
})
|
|
ports.Activity = activitySinkFunc(func(ctx context.Context, _ string, _ activitypool.FetchedBatch) (activitypool.UpsertResult, error) {
|
|
assertDeadline(ctx, "ActivitySink.UpsertFetched")
|
|
return activitypool.UpsertResult{Accepted: 1, Inserted: 1}, nil
|
|
})
|
|
reconciler, err := NewReconciler(Config{
|
|
UpstreamID: "provider-a",
|
|
Timeout: 250 * time.Millisecond,
|
|
MaxAttempts: 1,
|
|
MaxInFlight: 1,
|
|
MaxSize: 100,
|
|
}, ports)
|
|
if err != nil {
|
|
t.Fatalf("NewReconciler(): %v", err)
|
|
}
|
|
|
|
if got := runSingleReconcile(t, reconciler, results); got.Class != upstream.FetchValid {
|
|
t.Fatalf("result class = %q, want %q", got.Class, upstream.FetchValid)
|
|
}
|
|
if got := stages.Load(); got != 3 {
|
|
t.Fatalf("ports observing timeout = %d, want 3", got)
|
|
}
|
|
}
|
|
|
|
func TestReconcilerDropsNotificationFanoutWhileFetchIsInFlight(t *testing.T) {
|
|
started := make(chan struct{})
|
|
release := make(chan struct{})
|
|
results := make(chan Result, 2)
|
|
var calls atomic.Int64
|
|
ports := successfulPorts(func() {}, results)
|
|
ports.Adapter = adapterFunc(func(context.Context) (FetchResponse, error) {
|
|
calls.Add(1)
|
|
close(started)
|
|
<-release
|
|
return FetchResponse{Body: []byte("fixture")}, nil
|
|
})
|
|
reconciler, err := NewReconciler(Config{
|
|
UpstreamID: "provider-a",
|
|
Timeout: time.Second,
|
|
MaxAttempts: 1,
|
|
MaxInFlight: 1,
|
|
MaxSize: 100,
|
|
}, ports)
|
|
if err != nil {
|
|
t.Fatalf("NewReconciler(): %v", err)
|
|
}
|
|
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
done := make(chan error, 1)
|
|
go func() { done <- reconciler.RunLeader(ctx, unlimitedLeaderSession{}) }()
|
|
reconciler.Notify()
|
|
<-started
|
|
|
|
var wg sync.WaitGroup
|
|
for range 100 {
|
|
wg.Add(1)
|
|
go func() {
|
|
defer wg.Done()
|
|
reconciler.Notify()
|
|
}()
|
|
}
|
|
wg.Wait()
|
|
cancel()
|
|
close(release)
|
|
if err := <-done; err != nil {
|
|
t.Fatalf("Run(): %v", err)
|
|
}
|
|
if got := calls.Load(); got != 1 {
|
|
t.Fatalf("ProviderAdapter.Fetch() calls = %d, want 1", got)
|
|
}
|
|
}
|
|
|
|
func TestReconcilerEnforcesMaxInFlightAcrossRunConsumers(t *testing.T) {
|
|
started := make(chan struct{}, 2)
|
|
release := make(chan struct{}, 2)
|
|
results := make(chan Result, 2)
|
|
var active atomic.Int64
|
|
var maximum atomic.Int64
|
|
ports := successfulPorts(func() {}, results)
|
|
ports.Adapter = adapterFunc(func(context.Context) (FetchResponse, error) {
|
|
current := active.Add(1)
|
|
for {
|
|
observed := maximum.Load()
|
|
if current <= observed || maximum.CompareAndSwap(observed, current) {
|
|
break
|
|
}
|
|
}
|
|
started <- struct{}{}
|
|
<-release
|
|
active.Add(-1)
|
|
return FetchResponse{Body: []byte("fixture")}, nil
|
|
})
|
|
reconciler, err := NewReconciler(Config{
|
|
UpstreamID: "provider-a",
|
|
Timeout: time.Second,
|
|
MaxAttempts: 1,
|
|
MaxInFlight: 1,
|
|
MaxSize: 100,
|
|
}, ports)
|
|
if err != nil {
|
|
t.Fatalf("NewReconciler(): %v", err)
|
|
}
|
|
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
done := make(chan error, 2)
|
|
go func() { done <- reconciler.RunLeader(ctx, unlimitedLeaderSession{}) }()
|
|
go func() { done <- reconciler.RunLeader(ctx, unlimitedLeaderSession{}) }()
|
|
|
|
reconciler.Notify()
|
|
<-started
|
|
reconciler.Notify()
|
|
release <- struct{}{}
|
|
<-started
|
|
release <- struct{}{}
|
|
<-results
|
|
<-results
|
|
cancel()
|
|
if err := <-done; err != nil {
|
|
t.Fatalf("first Run(): %v", err)
|
|
}
|
|
if err := <-done; err != nil {
|
|
t.Fatalf("second Run(): %v", err)
|
|
}
|
|
if got := maximum.Load(); got != 1 {
|
|
t.Fatalf("maximum ProviderAdapter.Fetch() in flight = %d, want 1", got)
|
|
}
|
|
}
|
|
|
|
func TestReconcilerUsesConfiguredMaxInFlight(t *testing.T) {
|
|
started := make(chan struct{}, 2)
|
|
release := make(chan struct{}, 2)
|
|
results := make(chan Result, 2)
|
|
ports := successfulPorts(func() {}, results)
|
|
ports.Adapter = adapterFunc(func(context.Context) (FetchResponse, error) {
|
|
started <- struct{}{}
|
|
<-release
|
|
return FetchResponse{Body: []byte("fixture")}, nil
|
|
})
|
|
reconciler, err := NewReconciler(Config{
|
|
UpstreamID: "provider-a",
|
|
Timeout: time.Second,
|
|
MaxAttempts: 1,
|
|
MaxInFlight: 2,
|
|
MaxSize: 100,
|
|
}, ports)
|
|
if err != nil {
|
|
t.Fatalf("NewReconciler(): %v", err)
|
|
}
|
|
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
done := make(chan error, 1)
|
|
go func() { done <- reconciler.RunLeader(ctx, unlimitedLeaderSession{}) }()
|
|
reconciler.Notify()
|
|
<-started
|
|
reconciler.Notify()
|
|
select {
|
|
case <-started:
|
|
case <-time.After(time.Second):
|
|
release <- struct{}{}
|
|
cancel()
|
|
<-done
|
|
t.Fatal("second ProviderAdapter.Fetch did not use available in-flight slot")
|
|
}
|
|
release <- struct{}{}
|
|
release <- struct{}{}
|
|
<-results
|
|
<-results
|
|
cancel()
|
|
if err := <-done; err != nil {
|
|
t.Fatalf("Run(): %v", err)
|
|
}
|
|
}
|
|
|
|
func TestReconcilerDoesNotRefetchWhenActivitySinkFails(t *testing.T) {
|
|
clock := newFakeClock(time.Date(2026, 7, 28, 12, 0, 0, 0, time.UTC))
|
|
sleeper := &fakeSleeper{clock: clock}
|
|
results := make(chan Result, 3)
|
|
globalCompleted := make(chan int, 1)
|
|
var calls atomic.Int64
|
|
ports := successfulPorts(func() { calls.Add(1) }, results)
|
|
ports.Activity = activitySinkFunc(func(context.Context, string, activitypool.FetchedBatch) (activitypool.UpsertResult, error) {
|
|
return activitypool.UpsertResult{}, errors.New("activity pool unavailable")
|
|
})
|
|
reconciler, err := NewReconciler(Config{
|
|
UpstreamID: "provider-a",
|
|
Timeout: time.Second,
|
|
MaxAttempts: 3,
|
|
MaxInFlight: 1,
|
|
MaxSize: 100,
|
|
Retry: RetryConfig{Initial: 100 * time.Millisecond, Max: time.Second},
|
|
}, ports, Runtime{Clock: clock, Sleeper: sleeper})
|
|
if err != nil {
|
|
t.Fatalf("NewReconciler(): %v", err)
|
|
}
|
|
|
|
result := runSingleReconcile(t, reconciler, results, leaderSessionFunc(
|
|
func(context.Context, int) (RequestPermit, bool, error) {
|
|
return &recordingRequestPermit{completed: globalCompleted}, true, nil
|
|
},
|
|
))
|
|
if result.Class != upstream.FetchError {
|
|
t.Fatalf("result class = %q, want %q", result.Class, upstream.FetchError)
|
|
}
|
|
if got := calls.Load(); got != 1 {
|
|
t.Fatalf("ProviderAdapter.Fetch() calls = %d, want 1", got)
|
|
}
|
|
if got := <-globalCompleted; got != 1 {
|
|
t.Fatalf("global charged count = %d, want fetched=1", got)
|
|
}
|
|
if got := sleeper.Durations(); len(got) != 0 {
|
|
t.Fatalf("Sleep durations = %v, want no retry backoff", got)
|
|
}
|
|
}
|
|
|
|
func TestReconcilerDoesNotRetryPermanentAdapterError(t *testing.T) {
|
|
results := make(chan Result, 1)
|
|
sleeper := &errorSleeper{}
|
|
var calls atomic.Int64
|
|
ports := successfulPorts(func() {}, results)
|
|
ports.Adapter = adapterFunc(func(context.Context) (FetchResponse, error) {
|
|
calls.Add(1)
|
|
return FetchResponse{}, permanentFetchError("authentication rejected")
|
|
})
|
|
reconciler, err := NewReconciler(Config{
|
|
UpstreamID: "provider-a",
|
|
Timeout: time.Second,
|
|
MaxAttempts: 3,
|
|
MaxInFlight: 1,
|
|
MaxSize: 100,
|
|
Retry: RetryConfig{Initial: time.Second, Max: time.Second},
|
|
}, ports, Runtime{Sleeper: sleeper})
|
|
if err != nil {
|
|
t.Fatalf("NewReconciler(): %v", err)
|
|
}
|
|
|
|
result := runSingleReconcile(t, reconciler, results)
|
|
if result.Class != upstream.FetchError {
|
|
t.Fatalf("result class = %q, want %q", result.Class, upstream.FetchError)
|
|
}
|
|
if got := calls.Load(); got != 1 {
|
|
t.Fatalf("ProviderAdapter.Fetch() calls = %d, want 1", got)
|
|
}
|
|
if got := sleeper.calls.Load(); got != 0 {
|
|
t.Fatalf("Sleeper.Sleep() calls = %d, want 0", got)
|
|
}
|
|
}
|
|
|
|
func TestReconcilerDoesNotCallProviderWithoutFetchCapacity(t *testing.T) {
|
|
results := make(chan Result, 1)
|
|
var calls atomic.Int64
|
|
ports := successfulPorts(func() { calls.Add(1) }, results)
|
|
ports.Capacity = fetchCapacityFunc(func(string) (upstream.FetchPermit, bool, error) {
|
|
return nil, false, nil
|
|
})
|
|
reconciler, err := NewReconciler(Config{
|
|
UpstreamID: "provider-a", Timeout: time.Second, MaxAttempts: 1, MaxInFlight: 1, MaxSize: 100,
|
|
}, ports)
|
|
if err != nil {
|
|
t.Fatalf("NewReconciler(): %v", err)
|
|
}
|
|
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
done := make(chan error, 1)
|
|
go func() { done <- reconciler.RunLeader(ctx, unlimitedLeaderSession{}) }()
|
|
reconciler.Notify()
|
|
time.Sleep(20 * time.Millisecond)
|
|
cancel()
|
|
if err := <-done; err != nil {
|
|
t.Fatalf("Run(): %v", err)
|
|
}
|
|
if got := calls.Load(); got != 0 {
|
|
t.Fatalf("ProviderAdapter.Fetch() calls = %d, want 0", got)
|
|
}
|
|
select {
|
|
case result := <-results:
|
|
t.Fatalf("unexpected fetch result: %+v", result)
|
|
default:
|
|
}
|
|
}
|
|
|
|
func TestReconcilerDoesNotCallProviderWhenDistributedQuotaIsExhausted(t *testing.T) {
|
|
results := make(chan Result, 1)
|
|
var calls atomic.Int64
|
|
ports := successfulPorts(func() { calls.Add(1) }, results)
|
|
ports.Capacity = fetchCapacityFunc(func(string) (upstream.FetchPermit, bool, error) {
|
|
return &recordingFetchPermit{expected: 2}, true, nil
|
|
})
|
|
reconciler, err := NewReconciler(Config{
|
|
UpstreamID: "provider-a", Timeout: time.Second, MaxAttempts: 1, MaxInFlight: 1, MaxSize: 100,
|
|
}, ports)
|
|
if err != nil {
|
|
t.Fatalf("NewReconciler(): %v", err)
|
|
}
|
|
session := leaderSessionFunc(func(_ context.Context, expected int) (RequestPermit, bool, error) {
|
|
if expected != 2 {
|
|
t.Errorf("distributed expected = %d, want 2", expected)
|
|
}
|
|
return nil, false, nil
|
|
})
|
|
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
done := make(chan error, 1)
|
|
go func() { done <- reconciler.RunLeader(ctx, session) }()
|
|
reconciler.Notify()
|
|
time.Sleep(20 * time.Millisecond)
|
|
cancel()
|
|
if err := <-done; err != nil {
|
|
t.Fatalf("RunLeader(): %v", err)
|
|
}
|
|
if got := calls.Load(); got != 0 {
|
|
t.Fatalf("ProviderAdapter.Fetch() calls = %d, want 0", got)
|
|
}
|
|
select {
|
|
case result := <-results:
|
|
t.Fatalf("unexpected fetch result: %+v", result)
|
|
default:
|
|
}
|
|
}
|
|
|
|
func TestReconcilerChargesExpectedWhenSuccessfulResponseCannotBeParsed(t *testing.T) {
|
|
results := make(chan Result, 1)
|
|
globalCompleted := make(chan int, 1)
|
|
localCancelled := make(chan struct{}, 1)
|
|
ports := successfulPorts(func() {}, results)
|
|
ports.Parser = parserFunc(func(context.Context, []byte) ([]proxyDomain.Proxy, error) {
|
|
return nil, errors.New("invalid provider payload")
|
|
})
|
|
ports.Capacity = fetchCapacityFunc(func(string) (upstream.FetchPermit, bool, error) {
|
|
return &recordingFetchPermit{expected: 2, cancelled: localCancelled}, true, nil
|
|
})
|
|
reconciler, err := NewReconciler(Config{
|
|
UpstreamID: "provider-a", Timeout: time.Second, MaxAttempts: 1, MaxInFlight: 1, MaxSize: 100,
|
|
}, ports)
|
|
if err != nil {
|
|
t.Fatalf("NewReconciler(): %v", err)
|
|
}
|
|
session := leaderSessionFunc(func(context.Context, int) (RequestPermit, bool, error) {
|
|
return &recordingRequestPermit{completed: globalCompleted}, true, nil
|
|
})
|
|
|
|
result := runSingleReconcile(t, reconciler, results, session)
|
|
if result.Class != upstream.FetchError {
|
|
t.Fatalf("result class = %q, want %q", result.Class, upstream.FetchError)
|
|
}
|
|
if got := <-globalCompleted; got != 2 {
|
|
t.Fatalf("global charged count = %d, want expected=2", got)
|
|
}
|
|
select {
|
|
case <-localCancelled:
|
|
default:
|
|
t.Fatal("local pool reservation was not cancelled")
|
|
}
|
|
}
|
|
|
|
func TestReconcilerCancelsDistributedPermitWhenProviderCallFails(t *testing.T) {
|
|
results := make(chan Result, 1)
|
|
globalCancelled := make(chan struct{}, 1)
|
|
ports := successfulPorts(func() {}, results)
|
|
ports.Adapter = adapterFunc(func(context.Context) (FetchResponse, error) {
|
|
return FetchResponse{}, errors.New("provider connection failed")
|
|
})
|
|
reconciler, err := NewReconciler(Config{
|
|
UpstreamID: "provider-a", Timeout: time.Second, MaxAttempts: 1, MaxInFlight: 1, MaxSize: 100,
|
|
}, ports)
|
|
if err != nil {
|
|
t.Fatalf("NewReconciler(): %v", err)
|
|
}
|
|
session := leaderSessionFunc(func(context.Context, int) (RequestPermit, bool, error) {
|
|
return &recordingRequestPermit{cancelled: globalCancelled}, true, nil
|
|
})
|
|
|
|
result := runSingleReconcile(t, reconciler, results, session)
|
|
if result.Class != upstream.FetchError {
|
|
t.Fatalf("result class = %q, want %q", result.Class, upstream.FetchError)
|
|
}
|
|
select {
|
|
case <-globalCancelled:
|
|
default:
|
|
t.Fatal("distributed request reservation was not cancelled")
|
|
}
|
|
}
|
|
|
|
func TestReconcilerChargesFetchedGloballyAndCompletesRetainedLocally(t *testing.T) {
|
|
results := make(chan Result, 1)
|
|
localCompleted := make(chan fetchCompletion, 1)
|
|
globalCompleted := make(chan int, 1)
|
|
ports := successfulPorts(func() {}, results)
|
|
ports.Parser = parserFunc(func(context.Context, []byte) ([]proxyDomain.Proxy, error) {
|
|
return []proxyDomain.Proxy{{ID: "p1"}, {ID: "p2"}, {ID: "p3"}}, nil
|
|
})
|
|
ports.Activity = activitySinkFunc(func(_ context.Context, _ string, batch activitypool.FetchedBatch) (activitypool.UpsertResult, error) {
|
|
if len(batch.Proxies) != 2 {
|
|
t.Errorf("activity batch proxies = %d, want permit limit 2", len(batch.Proxies))
|
|
}
|
|
return activitypool.UpsertResult{Accepted: 2, Inserted: 1}, nil
|
|
})
|
|
ports.Capacity = fetchCapacityFunc(func(string) (upstream.FetchPermit, bool, error) {
|
|
return &recordingFetchPermit{expected: 2, completed: localCompleted}, true, nil
|
|
})
|
|
reconciler, err := NewReconciler(Config{
|
|
UpstreamID: "provider-a", Timeout: time.Second, MaxAttempts: 1, MaxInFlight: 1, MaxSize: 100,
|
|
}, ports)
|
|
if err != nil {
|
|
t.Fatalf("NewReconciler(): %v", err)
|
|
}
|
|
|
|
result := runSingleReconcile(t, reconciler, results, leaderSessionFunc(
|
|
func(_ context.Context, expected int) (RequestPermit, bool, error) {
|
|
if expected != 2 {
|
|
t.Errorf("distributed expected = %d, want 2", expected)
|
|
}
|
|
return &recordingRequestPermit{completed: globalCompleted}, true, nil
|
|
},
|
|
))
|
|
if result.ValidCount != 3 || result.NewCount != 1 {
|
|
t.Fatalf("result = %+v, want valid=3 new=1", result)
|
|
}
|
|
if got := <-globalCompleted; got != 3 {
|
|
t.Fatalf("global fetch completion = %d, want fetched=3", got)
|
|
}
|
|
if got := <-localCompleted; got.retained != 1 {
|
|
t.Fatalf("local fetch completion = %+v, want retained=1", got)
|
|
}
|
|
}
|
|
|
|
func TestReconcilerRetriesParserErrorsAsFetchErrors(t *testing.T) {
|
|
clock := newFakeClock(time.Date(2026, 7, 28, 12, 0, 0, 0, time.UTC))
|
|
sleeper := &fakeSleeper{clock: clock}
|
|
results := make(chan Result, 2)
|
|
var calls atomic.Int64
|
|
var parses atomic.Int64
|
|
ports := successfulPorts(func() { calls.Add(1) }, results)
|
|
ports.Parser = parserFunc(func(context.Context, []byte) ([]proxyDomain.Proxy, error) {
|
|
if parses.Add(1) == 1 {
|
|
return nil, errors.New("temporary parser failure")
|
|
}
|
|
return []proxyDomain.Proxy{{ID: "proxy-1"}}, nil
|
|
})
|
|
reconciler, err := NewReconciler(Config{
|
|
UpstreamID: "provider-a",
|
|
Timeout: time.Second,
|
|
MaxAttempts: 2,
|
|
MaxInFlight: 1,
|
|
MaxSize: 100,
|
|
Retry: RetryConfig{Initial: 100 * time.Millisecond, Max: time.Second},
|
|
}, ports, Runtime{Clock: clock, Sleeper: sleeper})
|
|
if err != nil {
|
|
t.Fatalf("NewReconciler(): %v", err)
|
|
}
|
|
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
done := make(chan error, 1)
|
|
go func() { done <- reconciler.RunLeader(ctx, unlimitedLeaderSession{}) }()
|
|
reconciler.Notify()
|
|
first, second := <-results, <-results
|
|
cancel()
|
|
if err := <-done; err != nil {
|
|
t.Fatalf("Run(): %v", err)
|
|
}
|
|
if first.Class != upstream.FetchError || second.Class != upstream.FetchValid {
|
|
t.Fatalf("result classes = [%q %q], want [%q %q]", first.Class, second.Class, upstream.FetchError, upstream.FetchValid)
|
|
}
|
|
if got := calls.Load(); got != 2 {
|
|
t.Fatalf("ProviderAdapter.Fetch() calls = %d, want 2", got)
|
|
}
|
|
if got := sleeper.Durations(); len(got) != 1 || got[0] != 100*time.Millisecond {
|
|
t.Fatalf("Sleep durations = %v, want [100ms]", got)
|
|
}
|
|
}
|
|
|
|
func TestNewReconcilerRejectsInvalidSchedulingConfiguration(t *testing.T) {
|
|
results := make(chan Result, 1)
|
|
ports := successfulPorts(func() {}, results)
|
|
tests := []struct {
|
|
name string
|
|
config Config
|
|
}{
|
|
{name: "missing upstream", config: Config{Timeout: time.Second, MaxAttempts: 1, MaxInFlight: 1, MaxSize: 100}},
|
|
{name: "negative interval", config: Config{UpstreamID: "a", RequestInterval: -1, Timeout: time.Second, MaxAttempts: 1, MaxInFlight: 1, MaxSize: 100}},
|
|
{name: "missing timeout", config: Config{UpstreamID: "a", MaxAttempts: 1, MaxInFlight: 1, MaxSize: 100}},
|
|
{name: "missing attempts", config: Config{UpstreamID: "a", Timeout: time.Second, MaxInFlight: 1, MaxSize: 100}},
|
|
{name: "missing in flight", config: Config{UpstreamID: "a", Timeout: time.Second, MaxAttempts: 1, MaxSize: 100}},
|
|
{name: "missing max size", config: Config{UpstreamID: "a", Timeout: time.Second, MaxAttempts: 1, MaxInFlight: 1}},
|
|
{name: "invalid jitter", config: Config{UpstreamID: "a", Timeout: time.Second, MaxAttempts: 1, MaxInFlight: 1, MaxSize: 100, Retry: RetryConfig{Jitter: 101}}},
|
|
{name: "initial exceeds max", config: Config{UpstreamID: "a", Timeout: time.Second, MaxAttempts: 1, MaxInFlight: 1, MaxSize: 100, Retry: RetryConfig{Initial: time.Second, Max: time.Millisecond}}},
|
|
{name: "incomplete retry pair", config: Config{UpstreamID: "a", Timeout: time.Second, MaxAttempts: 1, MaxInFlight: 1, MaxSize: 100, Retry: RetryConfig{Max: time.Second}}},
|
|
}
|
|
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
if _, err := NewReconciler(tt.config, ports); err == nil {
|
|
t.Fatal("NewReconciler() error = nil, want invalid configuration error")
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func runSingleReconcile(
|
|
t *testing.T,
|
|
reconciler *Reconciler,
|
|
results <-chan Result,
|
|
sessions ...LeaderSession,
|
|
) Result {
|
|
t.Helper()
|
|
session := LeaderSession(unlimitedLeaderSession{})
|
|
if len(sessions) > 1 {
|
|
t.Fatal("runSingleReconcile accepts at most one LeaderSession")
|
|
}
|
|
if len(sessions) == 1 {
|
|
session = sessions[0]
|
|
}
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
done := make(chan error, 1)
|
|
go func() { done <- reconciler.RunLeader(ctx, session) }()
|
|
reconciler.Notify()
|
|
var result Result
|
|
select {
|
|
case result = <-results:
|
|
case <-time.After(time.Second):
|
|
t.Fatal("timed out waiting for reconcile result")
|
|
}
|
|
cancel()
|
|
if err := <-done; err != nil {
|
|
t.Fatalf("Run(): %v", err)
|
|
}
|
|
return result
|
|
}
|
|
|
|
func successfulPorts(onFetch func(), results chan<- Result) Ports {
|
|
return Ports{
|
|
Adapter: adapterFunc(func(context.Context) (FetchResponse, error) {
|
|
onFetch()
|
|
return FetchResponse{Body: []byte("fixture")}, nil
|
|
}),
|
|
Parser: parserFunc(func(context.Context, []byte) ([]proxyDomain.Proxy, error) {
|
|
return []proxyDomain.Proxy{{ID: "proxy-1"}}, nil
|
|
}),
|
|
Activity: activitySinkFunc(func(context.Context, string, activitypool.FetchedBatch) (activitypool.UpsertResult, error) {
|
|
return activitypool.UpsertResult{Accepted: 1, Inserted: 1}, nil
|
|
}),
|
|
Results: resultRecorderFunc(func(got Result) { results <- got }),
|
|
Capacity: unlimitedFetchCapacity{},
|
|
}
|
|
}
|
|
|
|
type fakeClock struct {
|
|
mu sync.Mutex
|
|
now time.Time
|
|
}
|
|
|
|
func newFakeClock(now time.Time) *fakeClock { return &fakeClock{now: now} }
|
|
|
|
func (c *fakeClock) Now() time.Time {
|
|
c.mu.Lock()
|
|
defer c.mu.Unlock()
|
|
return c.now
|
|
}
|
|
|
|
func (c *fakeClock) Advance(duration time.Duration) {
|
|
c.mu.Lock()
|
|
c.now = c.now.Add(duration)
|
|
c.mu.Unlock()
|
|
}
|
|
|
|
type fakeSleeper struct {
|
|
mu sync.Mutex
|
|
clock *fakeClock
|
|
durations []time.Duration
|
|
}
|
|
|
|
type fixedRandom float64
|
|
|
|
func (r fixedRandom) Float64() float64 { return float64(r) }
|
|
|
|
type permanentFetchError string
|
|
|
|
func (e permanentFetchError) Error() string { return string(e) }
|
|
func (permanentFetchError) Retryable() bool { return false }
|
|
|
|
type errorSleeper struct{ calls atomic.Int64 }
|
|
|
|
func (s *errorSleeper) Sleep(context.Context, time.Duration) error {
|
|
s.calls.Add(1)
|
|
return errors.New("unexpected sleep")
|
|
}
|
|
|
|
func (s *fakeSleeper) Sleep(ctx context.Context, duration time.Duration) error {
|
|
if err := ctx.Err(); err != nil {
|
|
return err
|
|
}
|
|
s.mu.Lock()
|
|
s.durations = append(s.durations, duration)
|
|
s.mu.Unlock()
|
|
s.clock.Advance(duration)
|
|
return nil
|
|
}
|
|
|
|
func (s *fakeSleeper) Durations() []time.Duration {
|
|
s.mu.Lock()
|
|
defer s.mu.Unlock()
|
|
return append([]time.Duration(nil), s.durations...)
|
|
}
|
|
|
|
type adapterFunc func(context.Context) (FetchResponse, error)
|
|
|
|
func (f adapterFunc) Fetch(ctx context.Context) (FetchResponse, error) { return f(ctx) }
|
|
|
|
type parserFunc func(context.Context, []byte) ([]proxyDomain.Proxy, error)
|
|
|
|
func (f parserFunc) Parse(ctx context.Context, body []byte) ([]proxyDomain.Proxy, error) {
|
|
return f(ctx, body)
|
|
}
|
|
|
|
type activitySinkFunc func(context.Context, string, activitypool.FetchedBatch) (activitypool.UpsertResult, error)
|
|
|
|
func (f activitySinkFunc) UpsertFetched(ctx context.Context, upstreamID string, batch activitypool.FetchedBatch) (activitypool.UpsertResult, error) {
|
|
return f(ctx, upstreamID, batch)
|
|
}
|
|
|
|
type fetchCapacityFunc func(string) (upstream.FetchPermit, bool, error)
|
|
|
|
func (f fetchCapacityFunc) ReserveFetch(upstreamID string) (upstream.FetchPermit, bool, error) {
|
|
return f(upstreamID)
|
|
}
|
|
|
|
type unlimitedFetchCapacity struct{}
|
|
|
|
func (unlimitedFetchCapacity) ReserveFetch(string) (upstream.FetchPermit, bool, error) {
|
|
return &recordingFetchPermit{expected: int(^uint(0) >> 1)}, true, nil
|
|
}
|
|
|
|
type fetchCompletion struct {
|
|
retained int
|
|
}
|
|
|
|
type recordingFetchPermit struct {
|
|
expected int
|
|
completed chan<- fetchCompletion
|
|
cancelled chan<- struct{}
|
|
}
|
|
|
|
func (p *recordingFetchPermit) Expected() int { return p.expected }
|
|
|
|
func (p *recordingFetchPermit) Complete(retained int) error {
|
|
if p.completed != nil {
|
|
p.completed <- fetchCompletion{retained: retained}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (p *recordingFetchPermit) Cancel() error {
|
|
if p.cancelled != nil {
|
|
p.cancelled <- struct{}{}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
type resultRecorderFunc func(Result)
|
|
|
|
func (f resultRecorderFunc) Record(result Result) { f(result) }
|
|
|
|
type unlimitedLeaderSession struct{}
|
|
|
|
func (unlimitedLeaderSession) Fence() Fence { return Fence{Generation: "test", Epoch: 1} }
|
|
|
|
func (unlimitedLeaderSession) AcquireFetch(context.Context, int) (RequestPermit, bool, error) {
|
|
return &recordingRequestPermit{}, true, nil
|
|
}
|
|
|
|
type leaderSessionFunc func(context.Context, int) (RequestPermit, bool, error)
|
|
|
|
func (leaderSessionFunc) Fence() Fence { return Fence{Generation: "test", Epoch: 1} }
|
|
|
|
func (f leaderSessionFunc) AcquireFetch(ctx context.Context, expected int) (RequestPermit, bool, error) {
|
|
return f(ctx, expected)
|
|
}
|
|
|
|
type recordingRequestPermit struct {
|
|
completed chan<- int
|
|
cancelled chan<- struct{}
|
|
}
|
|
|
|
func (p *recordingRequestPermit) Complete(_ context.Context, fetched int) error {
|
|
if p.completed != nil {
|
|
p.completed <- fetched
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (p *recordingRequestPermit) Cancel(context.Context) error {
|
|
if p.cancelled != nil {
|
|
p.cancelled <- struct{}{}
|
|
}
|
|
return nil
|
|
}
|