proxy-pool/internal/controller/provider/reconciler.go

package provider

import (
	"context"
	"errors"
	"fmt"
	"math/rand"
	"sync"
	"time"

	"proxy-pool/internal/domain/activitypool"
	"proxy-pool/internal/domain/upstream"
	"proxy-pool/internal/platform/coalesce"
)

type Config struct {
	UpstreamID             string
	RequestInterval        time.Duration
	Timeout                time.Duration
	MaxAttempts            int
	MaxInFlight            int
	MaxSize                int
	TTL                    time.Duration
	AllocationSafetyMargin time.Duration
	Retry                  RetryConfig
}

type RetryConfig struct {
	Initial time.Duration
	Max     time.Duration
	Jitter  int
}

type Reconciler struct {
	config      Config
	ports       Ports
	runtime     Runtime
	signal      *coalesce.Signal
	inFlight    chan struct{}
	rateMu      sync.Mutex
	nextRequest time.Time
}

type Clock interface {
	Now() time.Time
}

type Sleeper interface {
	Sleep(context.Context, time.Duration) error
}

type Random interface {
	Float64() float64
}

type Runtime struct {
	Clock   Clock
	Sleeper Sleeper
	Random  Random
}

func NewReconciler(config Config, ports Ports, runtimes ...Runtime) (*Reconciler, error) {
	if config.UpstreamID == "" {
		return nil, fmt.Errorf("new provider reconciler: upstream ID is required")
	}
	if config.RequestInterval < 0 || config.Timeout <= 0 || config.MaxAttempts <= 0 ||
		config.MaxInFlight <= 0 || config.MaxSize <= 0 {
		return nil, fmt.Errorf("new provider reconciler: fetch limits must be positive")
	}
	if config.TTL < 0 || config.AllocationSafetyMargin < 0 ||
		(config.TTL > 0 && config.AllocationSafetyMargin >= config.TTL) {
		return nil, fmt.Errorf("new provider reconciler: lifecycle settings are invalid")
	}
	if config.Retry.Initial < 0 || config.Retry.Max < 0 || config.Retry.Jitter < 0 || config.Retry.Jitter > 100 {
		return nil, fmt.Errorf("new provider reconciler: retry settings are invalid")
	}
	if (config.Retry.Initial == 0) != (config.Retry.Max == 0) {
		return nil, fmt.Errorf("new provider reconciler: retry initial and max must be configured together")
	}
	if config.Retry.Max > 0 && config.Retry.Initial > config.Retry.Max {
		return nil, fmt.Errorf("new provider reconciler: retry initial delay exceeds maximum")
	}
	if ports.Adapter == nil || ports.Parser == nil || ports.Activity == nil ||
		ports.Results == nil || ports.Capacity == nil {
		return nil, fmt.Errorf("new provider reconciler: all ports are required")
	}
	runtime := Runtime{Clock: systemClock{}, Sleeper: timerSleeper{}, Random: globalRandom{}}
	if len(runtimes) > 1 {
		return nil, fmt.Errorf("new provider reconciler: at most one runtime is allowed")
	}
	if len(runtimes) == 1 {
		if runtimes[0].Clock != nil {
			runtime.Clock = runtimes[0].Clock
		}
		if runtimes[0].Sleeper != nil {
			runtime.Sleeper = runtimes[0].Sleeper
		}
		if runtimes[0].Random != nil {
			runtime.Random = runtimes[0].Random
		}
	}
	return &Reconciler{
		config:   config,
		ports:    ports,
		runtime:  runtime,
		signal:   coalesce.NewSignal(),
		inFlight: make(chan struct{}, config.MaxInFlight),
	}, nil
}

func (r *Reconciler) Notify() {
	r.signal.Notify()
}

func (r *Reconciler) Run(ctx context.Context) error {
	var workers sync.WaitGroup
	defer workers.Wait()
	for {
		if ctx.Err() != nil {
			return nil
		}
		if err := r.signal.Wait(ctx); err != nil {
			return nil
		}
		select {
		case r.inFlight <- struct{}{}:
		case <-ctx.Done():
			return nil
		}
		workers.Add(1)
		go func() {
			defer workers.Done()
			defer func() { <-r.inFlight }()
			r.reconcile(ctx)
		}()
	}
}

func (r *Reconciler) reconcile(ctx context.Context) {
	for attempt := 1; attempt <= r.config.MaxAttempts; attempt++ {
		response, result, retryable, ok := r.fetchAttempt(ctx, attempt)
		if !ok {
			return
		}
		r.ports.Results.Record(result)
		if result.Class != upstream.FetchError || !retryable || attempt == r.config.MaxAttempts {
			return
		}
		if delay := r.retryDelay(attempt, response.RetryAfter); delay > 0 {
			if err := r.runtime.Sleeper.Sleep(ctx, delay); err != nil {
				return
			}
		}
		if ctx.Err() != nil {
			return
		}
	}
}

func (r *Reconciler) fetchAttempt(ctx context.Context, attempt int) (FetchResponse, Result, bool, bool) {
	if err := r.waitForRequestSlot(ctx); err != nil {
		return FetchResponse{}, Result{}, false, false
	}
	permit, available, err := r.ports.Capacity.ReserveFetch(r.config.UpstreamID)
	if err != nil {
		resultErr := fmt.Errorf("reserve fetch capacity: %w", err)
		return FetchResponse{}, Result{
			UpstreamID: r.config.UpstreamID,
			Class:      upstream.FetchError,
			Err:        resultErr,
			Attempt:    attempt,
		}, false, true
	}
	if !available {
		return FetchResponse{}, Result{}, false, false
	}
	if permit == nil || permit.Expected() <= 0 {
		if permit != nil {
			_ = permit.Cancel()
		}
		return FetchResponse{}, Result{
			UpstreamID: r.config.UpstreamID,
			Class:      upstream.FetchError,
			Err:        fmt.Errorf("reserve fetch capacity: invalid permit"),
			Attempt:    attempt,
		}, false, true
	}
	permitFinished := false
	defer func() {
		if !permitFinished {
			_ = permit.Cancel()
		}
	}()

	callCtx := ctx
	cancel := func() {}
	if r.config.Timeout > 0 {
		callCtx, cancel = context.WithTimeout(ctx, r.config.Timeout)
	}
	defer cancel()
	response, callErr := r.ports.Adapter.Fetch(callCtx)

	var parseErr, candidateErr, capacityErr error
	var validCount, newCount int
	if callErr == nil {
		candidates, err := r.ports.Parser.Parse(callCtx, response.Body)
		parseErr = err
		validCount = len(candidates)
		if parseErr == nil && validCount > 0 {
			retained := candidates
			if expected := permit.Expected(); expected < len(retained) {
				retained = retained[:expected]
			}
			upserted, err := r.ports.Activity.UpsertFetched(callCtx, r.config.UpstreamID, activitypool.FetchedBatch{
				ObservedAt:             r.runtime.Clock.Now().UTC(),
				ConfiguredTTL:          r.config.TTL,
				AllocationSafetyMargin: r.config.AllocationSafetyMargin,
				MaxSize:                r.config.MaxSize,
				Proxies:                retained,
			})
			candidateErr = err
			newCount = upserted.Inserted
		}
	}
	if callErr == nil && parseErr == nil && candidateErr == nil {
		capacityErr = permit.Complete(validCount, newCount)
		permitFinished = capacityErr == nil
	}
	resultErr := errors.Join(callErr, parseErr, candidateErr, capacityErr)
	class := upstream.ClassifyFetchResult(callErr, errors.Join(parseErr, candidateErr, capacityErr), validCount, newCount)
	return response, Result{
		UpstreamID: r.config.UpstreamID,
		Class:      class,
		ValidCount: validCount,
		NewCount:   newCount,
		Err:        resultErr,
		Attempt:    attempt,
	}, isRetryable(callErr) || parseErr != nil, true
}

func isRetryable(err error) bool {
	if err == nil {
		return false
	}
	var classified RetryableError
	if errors.As(err, &classified) {
		return classified.Retryable()
	}
	return true
}

func (r *Reconciler) waitForRequestSlot(ctx context.Context) error {
	r.rateMu.Lock()
	now := r.runtime.Clock.Now()
	requestAt := now
	if r.nextRequest.After(requestAt) {
		requestAt = r.nextRequest
	}
	r.nextRequest = requestAt.Add(r.config.RequestInterval)
	r.rateMu.Unlock()

	if delay := requestAt.Sub(now); delay > 0 {
		return r.runtime.Sleeper.Sleep(ctx, delay)
	}
	return nil
}

func (r *Reconciler) retryDelay(failedAttempt int, retryAfter time.Duration) time.Duration {
	if retryAfter > 0 {
		if r.config.Retry.Max > 0 && retryAfter > r.config.Retry.Max {
			return r.config.Retry.Max
		}
		return retryAfter
	}

	delay := r.config.Retry.Initial
	for attempt := 1; attempt < failedAttempt; attempt++ {
		if r.config.Retry.Max > 0 && delay >= r.config.Retry.Max/2 {
			delay = r.config.Retry.Max
			break
		}
		delay *= 2
	}
	if delay <= 0 || r.config.Retry.Jitter == 0 {
		return delay
	}

	random := r.runtime.Random.Float64()
	if random < 0 {
		random = 0
	} else if random > 1 {
		random = 1
	}
	spread := float64(r.config.Retry.Jitter) / 100
	delay = time.Duration(float64(delay) * (1 + (2*random-1)*spread))
	if r.config.Retry.Max > 0 && delay > r.config.Retry.Max {
		return r.config.Retry.Max
	}
	return delay
}

type systemClock struct{}

func (systemClock) Now() time.Time { return time.Now() }

type timerSleeper struct{}

func (timerSleeper) Sleep(ctx context.Context, duration time.Duration) error {
	timer := time.NewTimer(duration)
	defer timer.Stop()
	select {
	case <-timer.C:
		return nil
	case <-ctx.Done():
		return ctx.Err()
	}
}

type globalRandom struct{}

func (globalRandom) Float64() float64 { return rand.Float64() }