package provider import ( "context" "errors" "fmt" "math/rand" "sync" "time" "proxy-pool/internal/domain/activitypool" "proxy-pool/internal/domain/upstream" "proxy-pool/internal/platform/coalesce" ) type Config struct { UpstreamID string RequestInterval time.Duration Timeout time.Duration MaxAttempts int MaxInFlight int MaxTotal int64 MaxSize int TTL time.Duration AllocationSafetyMargin time.Duration Retry RetryConfig } type RetryConfig struct { Initial time.Duration Max time.Duration Jitter int } const requestPermitSettlementTimeout = 5 * time.Second type Reconciler struct { config Config ports Ports runtime Runtime signal *coalesce.Signal inFlight chan struct{} rateMu sync.Mutex nextRequest time.Time } type Clock interface { Now() time.Time } type Sleeper interface { Sleep(context.Context, time.Duration) error } type Random interface { Float64() float64 } type Runtime struct { Clock Clock Sleeper Sleeper Random Random } func NewReconciler(config Config, ports Ports, runtimes ...Runtime) (*Reconciler, error) { if config.UpstreamID == "" { return nil, fmt.Errorf("new provider reconciler: upstream ID is required") } if config.RequestInterval < 0 || config.Timeout <= 0 || config.MaxAttempts <= 0 || config.MaxInFlight <= 0 || config.MaxTotal < 0 || config.MaxSize <= 0 { return nil, fmt.Errorf("new provider reconciler: fetch limits must be positive") } if config.TTL < 0 || config.AllocationSafetyMargin < 0 || (config.TTL > 0 && config.AllocationSafetyMargin >= config.TTL) { return nil, fmt.Errorf("new provider reconciler: lifecycle settings are invalid") } if config.Retry.Initial < 0 || config.Retry.Max < 0 || config.Retry.Jitter < 0 || config.Retry.Jitter > 100 { return nil, fmt.Errorf("new provider reconciler: retry settings are invalid") } if (config.Retry.Initial == 0) != (config.Retry.Max == 0) { return nil, fmt.Errorf("new provider reconciler: retry initial and max must be configured together") } if config.Retry.Max > 0 && config.Retry.Initial > config.Retry.Max { return nil, fmt.Errorf("new provider reconciler: retry initial delay exceeds maximum") } if ports.Adapter == nil || ports.Parser == nil || ports.Activity == nil || ports.Results == nil || ports.Capacity == nil { return nil, fmt.Errorf("new provider reconciler: all ports are required") } runtime := Runtime{Clock: systemClock{}, Sleeper: timerSleeper{}, Random: globalRandom{}} if len(runtimes) > 1 { return nil, fmt.Errorf("new provider reconciler: at most one runtime is allowed") } if len(runtimes) == 1 { if runtimes[0].Clock != nil { runtime.Clock = runtimes[0].Clock } if runtimes[0].Sleeper != nil { runtime.Sleeper = runtimes[0].Sleeper } if runtimes[0].Random != nil { runtime.Random = runtimes[0].Random } } return &Reconciler{ config: config, ports: ports, runtime: runtime, signal: coalesce.NewSignal(), inFlight: make(chan struct{}, config.MaxInFlight), }, nil } func (r *Reconciler) Notify() { r.signal.Notify() } func (r *Reconciler) RunLeader(ctx context.Context, session LeaderSession) error { if ctx == nil || session == nil { return ErrInvalidCoordination } var workers sync.WaitGroup defer workers.Wait() for { if ctx.Err() != nil { return nil } if err := r.signal.Wait(ctx); err != nil { return nil } select { case r.inFlight <- struct{}{}: case <-ctx.Done(): return nil } workers.Add(1) go func() { defer workers.Done() defer func() { <-r.inFlight }() r.reconcile(ctx, session) }() } } func (r *Reconciler) reconcile(ctx context.Context, session LeaderSession) { permit, available, err := r.ports.Capacity.ReserveFetch(r.config.UpstreamID) if err != nil { r.ports.Results.Record(Result{ UpstreamID: r.config.UpstreamID, Class: upstream.FetchError, Err: fmt.Errorf("reserve fetch capacity: %w", err), Attempt: 1, }) return } if !available { return } if permit == nil || permit.Expected() <= 0 { if permit != nil { _ = permit.Cancel() } r.ports.Results.Record(Result{ UpstreamID: r.config.UpstreamID, Class: upstream.FetchError, Err: fmt.Errorf("reserve fetch capacity: invalid permit"), Attempt: 1, }) return } permitFinished := false defer func() { if !permitFinished { _ = permit.Cancel() } }() for attempt := 1; attempt <= r.config.MaxAttempts; attempt++ { response, result, retryable, ok := r.fetchAttempt(ctx, session, permit, attempt) if !ok { return } if result.Class != upstream.FetchError { if capacityErr := permit.Complete(result.NewCount); capacityErr != nil { result.Class = upstream.FetchError result.Err = errors.Join(result.Err, capacityErr) } else { permitFinished = true } } r.ports.Results.Record(result) if result.Class != upstream.FetchError || !retryable || attempt == r.config.MaxAttempts { return } if delay := r.retryDelay(attempt, response.RetryAfter); delay > 0 { if err := r.runtime.Sleeper.Sleep(ctx, delay); err != nil { return } } if ctx.Err() != nil { return } } } func (r *Reconciler) fetchAttempt( ctx context.Context, session LeaderSession, permit upstream.FetchPermit, attempt int, ) (FetchResponse, Result, bool, bool) { if err := r.waitForRequestSlot(ctx); err != nil { return FetchResponse{}, Result{}, false, false } requestPermit, available, err := session.AcquireFetch(ctx, permit.Expected()) if err != nil { if ctx.Err() != nil || errors.Is(err, ErrLeadershipLost) { return FetchResponse{}, Result{}, false, false } return FetchResponse{}, Result{ UpstreamID: r.config.UpstreamID, Class: upstream.FetchError, Err: fmt.Errorf("acquire distributed fetch capacity: %w", err), Attempt: attempt, }, false, true } if !available { return FetchResponse{}, Result{}, false, false } if requestPermit == nil { return FetchResponse{}, Result{ UpstreamID: r.config.UpstreamID, Class: upstream.FetchError, Err: fmt.Errorf("acquire distributed fetch capacity: invalid permit"), Attempt: attempt, }, false, true } requestSettlementAttempted := false defer func() { if !requestSettlementAttempted { _ = r.settleRequestPermit(requestPermit, false, 0) } }() callCtx := ctx cancel := func() {} if r.config.Timeout > 0 { callCtx, cancel = context.WithTimeout(ctx, r.config.Timeout) } defer cancel() response, callErr := r.ports.Adapter.Fetch(callCtx) var parseErr, candidateErr, coordinationErr error var validCount, newCount int if callErr != nil { requestSettlementAttempted = true coordinationErr = r.settleRequestPermit(requestPermit, true, permit.Expected()) } else { candidates, err := r.ports.Parser.Parse(callCtx, response.Body) defer r.ports.Parser.ReleaseCandidates(candidates) parseErr = err validCount = len(candidates) charged := validCount if parseErr != nil { charged = permit.Expected() } requestSettlementAttempted = true coordinationErr = r.settleRequestPermit(requestPermit, true, charged) if parseErr == nil && coordinationErr == nil && validCount > 0 { retained := candidates if expected := permit.Expected(); expected < len(retained) { retained = retained[:expected] } upserted, err := r.ports.Activity.UpsertFetched(callCtx, r.config.UpstreamID, activitypool.FetchedBatch{ ObservedAt: r.runtime.Clock.Now().UTC(), ConfiguredTTL: r.config.TTL, AllocationSafetyMargin: r.config.AllocationSafetyMargin, MaxSize: r.config.MaxSize, Proxies: retained, }) candidateErr = err newCount = upserted.Inserted } } resultErr := errors.Join(callErr, parseErr, candidateErr, coordinationErr) class := upstream.ClassifyFetchResult( callErr, errors.Join(parseErr, candidateErr, coordinationErr), validCount, newCount, ) return response, Result{ UpstreamID: r.config.UpstreamID, Class: class, ValidCount: validCount, NewCount: newCount, Err: resultErr, Attempt: attempt, }, isRetryable(callErr) || parseErr != nil, true } func (r *Reconciler) settleRequestPermit(permit RequestPermit, complete bool, fetched int) error { ctx, cancel := context.WithTimeout(context.Background(), requestPermitSettlementTimeout) defer cancel() if complete { return permit.Complete(ctx, fetched) } return permit.Cancel(ctx) } func isRetryable(err error) bool { if err == nil { return false } var classified RetryableError if errors.As(err, &classified) { return classified.Retryable() } return true } func (r *Reconciler) waitForRequestSlot(ctx context.Context) error { r.rateMu.Lock() now := r.runtime.Clock.Now() requestAt := now if r.nextRequest.After(requestAt) { requestAt = r.nextRequest } r.nextRequest = requestAt.Add(r.config.RequestInterval) r.rateMu.Unlock() if delay := requestAt.Sub(now); delay > 0 { return r.runtime.Sleeper.Sleep(ctx, delay) } return nil } func (r *Reconciler) retryDelay(failedAttempt int, retryAfter time.Duration) time.Duration { if retryAfter > 0 { if r.config.Retry.Max > 0 && retryAfter > r.config.Retry.Max { return r.config.Retry.Max } return retryAfter } delay := r.config.Retry.Initial for attempt := 1; attempt < failedAttempt; attempt++ { if r.config.Retry.Max > 0 && delay >= r.config.Retry.Max/2 { delay = r.config.Retry.Max break } delay *= 2 } if delay <= 0 || r.config.Retry.Jitter == 0 { return delay } random := r.runtime.Random.Float64() if random < 0 { random = 0 } else if random > 1 { random = 1 } spread := float64(r.config.Retry.Jitter) / 100 delay = time.Duration(float64(delay) * (1 + (2*random-1)*spread)) if r.config.Retry.Max > 0 && delay > r.config.Retry.Max { return r.config.Retry.Max } return delay } type systemClock struct{} func (systemClock) Now() time.Time { return time.Now() } type timerSleeper struct{} func (timerSleeper) Sleep(ctx context.Context, duration time.Duration) error { timer := time.NewTimer(duration) defer timer.Stop() select { case <-timer.C: return nil case <-ctx.Done(): return ctx.Err() } } type globalRandom struct{} func (globalRandom) Float64() float64 { return rand.Float64() }