package routing import ( "context" "sync" "testing" "time" "proxy-pool/internal/config" "proxy-pool/internal/controller/provider" "proxy-pool/internal/domain/adminstate" ) func TestSequentialCoordinatorSwitchesOnlyAtConfiguredThreshold(t *testing.T) { state := newRoutingState("provider-a", map[string]bool{"provider-a": true, "provider-b": true}) stats := &routingStats{byUpstream: map[string]provider.Stats{ "provider-a": {UpstreamID: "provider-a", ConsecutiveEmptyFetch: 4, EmptyGeneration: 1}, }} refresh := &refreshRecorder{} coordinator := newCoordinator(t, routingConfiguration("stop", []string{"provider-a", "provider-b"}), state, stats, refresh) if result, err := coordinator.Tick(context.Background()); err != nil || result.Switched != 0 || state.current() != "provider-a" { t.Fatalf("Tick(before threshold) = (%+v, %v), current=%q", result, err, state.current()) } stats.set("provider-a", provider.Stats{UpstreamID: "provider-a", ConsecutiveEmptyFetch: 5, EmptyGeneration: 1}) if result, err := coordinator.Tick(context.Background()); err != nil || result.Switched != 1 || state.current() != "provider-b" { t.Fatalf("Tick(at threshold) = (%+v, %v), current=%q", result, err, state.current()) } if refresh.count != 1 || state.switches != 1 { t.Fatalf("refresh=%d switches=%d, want 1", refresh.count, state.switches) } } func TestSequentialCoordinatorStopsAtTerminalUpstream(t *testing.T) { state := newRoutingState("provider-b", map[string]bool{"provider-a": true, "provider-b": true}) stats := &routingStats{byUpstream: map[string]provider.Stats{ "provider-b": {UpstreamID: "provider-b", ConsecutiveEmptyFetch: 5, EmptyGeneration: 9}, }} refresh := &refreshRecorder{} coordinator := newCoordinator(t, routingConfiguration("stop", []string{"provider-a", "provider-b"}), state, stats, refresh) if result, err := coordinator.Tick(context.Background()); err != nil || result != (TickResult{Stopped: 1}) || state.enabled() || refresh.count != 1 { t.Fatalf("Tick(terminal stop) = (%+v, %v), enabled=%v refresh=%d", result, err, state.enabled(), refresh.count) } if result, err := coordinator.Tick(context.Background()); err != nil || result != (TickResult{}) || state.stops != 1 || refresh.count != 1 { t.Fatalf("Tick(reused terminal generation) = (%+v, %v), stops=%d refresh=%d", result, err, state.stops, refresh.count) } } func TestSequentialCoordinatorStopsWhenDisabledCandidatesLeaveNoAlternative(t *testing.T) { state := newRoutingState("provider-a", map[string]bool{"provider-a": true, "provider-b": false}) stats := &routingStats{byUpstream: map[string]provider.Stats{ "provider-a": {UpstreamID: "provider-a", ConsecutiveEmptyFetch: 5, EmptyGeneration: 4}, }} coordinator := newCoordinator(t, routingConfiguration("stop", []string{"provider-a", "provider-b"}), state, stats, &refreshRecorder{}) if result, err := coordinator.Tick(context.Background()); err != nil || result != (TickResult{Stopped: 1}) || state.enabled() { t.Fatalf("Tick(last enabled candidate) = (%+v, %v), enabled=%v", result, err, state.enabled()) } } func TestSequentialCoordinatorAdvancesWhenCurrentUpstreamIsDisabledWithoutProviderStats(t *testing.T) { state := newRoutingState("provider-a", map[string]bool{"provider-a": false, "provider-b": true, "provider-c": true}) stats := &routingStats{byUpstream: map[string]provider.Stats{}} refresh := &refreshRecorder{} coordinator := newCoordinator(t, routingConfiguration("stop", []string{"provider-a", "provider-b", "provider-c"}), state, stats, refresh) if result, err := coordinator.Tick(context.Background()); err != nil || result != (TickResult{Switched: 1}) || state.current() != "provider-b" { t.Fatalf("Tick(disabled current) = (%+v, %v), current=%q", result, err, state.current()) } if stats.readCount() != 0 || state.switches != 1 || refresh.count != 1 { t.Fatalf("provider reads=%d switches=%d refresh=%d, want 0/1/1", stats.readCount(), state.switches, refresh.count) } } func TestSequentialCoordinatorStopsWhenDisabledCurrentHasNoSuccessor(t *testing.T) { state := newRoutingState("provider-b", map[string]bool{"provider-a": true, "provider-b": false}) coordinator := newCoordinator(t, routingConfiguration("stayLast", []string{"provider-a", "provider-b"}), state, &routingStats{byUpstream: map[string]provider.Stats{}}, &refreshRecorder{}) if result, err := coordinator.Tick(context.Background()); err != nil || result != (TickResult{Stopped: 1}) || state.enabled() { t.Fatalf("Tick(disabled terminal current) = (%+v, %v), enabled=%v", result, err, state.enabled()) } } func TestSequentialCoordinatorLoopsWhenDisabledCurrentHasEarlierSuccessor(t *testing.T) { state := newRoutingState("provider-c", map[string]bool{"provider-a": true, "provider-b": false, "provider-c": false}) coordinator := newCoordinator(t, routingConfiguration("loop", []string{"provider-a", "provider-b", "provider-c"}), state, &routingStats{byUpstream: map[string]provider.Stats{}}, &refreshRecorder{}) if result, err := coordinator.Tick(context.Background()); err != nil || result != (TickResult{Switched: 1}) || state.current() != "provider-a" { t.Fatalf("Tick(disabled loop current) = (%+v, %v), current=%q", result, err, state.current()) } } func TestSequentialCoordinatorConsumesStatsRecorderNotifications(t *testing.T) { state := newRoutingState("provider-a", map[string]bool{"provider-a": true, "provider-b": true}) stats, err := provider.NewStatsRecorder(2) if err != nil { t.Fatalf("NewStatsRecorder() = %v", err) } refresh := &refreshRecorder{} coordinator, err := NewSequentialCoordinator( staticConfiguration{configuration: routingConfiguration("stop", []string{"provider-a", "provider-b"}), revision: 7}, state, stats, refresh, Options{Now: time.Now}, ) if err != nil { t.Fatalf("NewSequentialCoordinator() = %v", err) } stats.AddResultObserver(coordinator) for range 5 { stats.Record(provider.Result{UpstreamID: "provider-a", Class: "empty"}) } select { case <-coordinator.notify: case <-time.After(time.Second): t.Fatal("StatsRecorder did not notify SequentialCoordinator") } if result, err := coordinator.Tick(context.Background()); err != nil || result.Switched != 1 || state.current() != "provider-b" || refresh.count != 1 { t.Fatalf("Tick() = (%+v, %v), current=%q refresh=%d", result, err, state.current(), refresh.count) } } func TestSequentialCoordinatorSkipsDisabledCandidatesAndStaysAtConfiguredEnd(t *testing.T) { state := newRoutingState("provider-a", map[string]bool{ "provider-a": true, "provider-b": false, "provider-c": true, }) stats := &routingStats{byUpstream: map[string]provider.Stats{ "provider-a": {UpstreamID: "provider-a", ConsecutiveEmptyFetch: 5, EmptyGeneration: 1}, "provider-c": {UpstreamID: "provider-c", ConsecutiveEmptyFetch: 5, EmptyGeneration: 2}, }} coordinator := newCoordinator(t, routingConfiguration("stayLast", []string{"provider-a", "provider-b", "provider-c"}), state, stats, &refreshRecorder{}) if result, err := coordinator.Tick(context.Background()); err != nil || result.Switched != 1 || state.current() != "provider-c" { t.Fatalf("Tick(skip disabled) = (%+v, %v), current=%q", result, err, state.current()) } if result, err := coordinator.Tick(context.Background()); err != nil || result.Switched != 0 || state.current() != "provider-c" { t.Fatalf("Tick(stay last) = (%+v, %v), current=%q", result, err, state.current()) } } func TestSequentialCoordinatorDoesNotReuseAnEmptyGenerationAfterLoop(t *testing.T) { state := newRoutingState("provider-a", map[string]bool{"provider-a": true, "provider-b": true}) stats := &routingStats{byUpstream: map[string]provider.Stats{ "provider-a": {UpstreamID: "provider-a", ConsecutiveEmptyFetch: 5, EmptyGeneration: 1}, "provider-b": {UpstreamID: "provider-b", ConsecutiveEmptyFetch: 5, EmptyGeneration: 2}, }} coordinator := newCoordinator(t, routingConfiguration("loop", []string{"provider-a", "provider-b"}), state, stats, &refreshRecorder{}) for _, want := range []string{"provider-b", "provider-a"} { if result, err := coordinator.Tick(context.Background()); err != nil || result.Switched != 1 || state.current() != want { t.Fatalf("Tick() = (%+v, %v), current=%q, want %q", result, err, state.current(), want) } } if result, err := coordinator.Tick(context.Background()); err != nil || result.Switched != 0 || state.current() != "provider-a" { t.Fatalf("Tick(reused generation) = (%+v, %v), current=%q", result, err, state.current()) } stats.set("provider-a", provider.Stats{UpstreamID: "provider-a", ConsecutiveEmptyFetch: 5, EmptyGeneration: 3}) if result, err := coordinator.Tick(context.Background()); err != nil || result.Switched != 1 || state.current() != "provider-b" { t.Fatalf("Tick(new generation) = (%+v, %v), current=%q", result, err, state.current()) } } func TestSequentialCoordinatorUsesCompareAndSwapUnderConcurrentTicks(t *testing.T) { state := newRoutingState("provider-a", map[string]bool{"provider-a": true, "provider-b": true}) stats := &routingStats{byUpstream: map[string]provider.Stats{ "provider-a": {UpstreamID: "provider-a", ConsecutiveEmptyFetch: 5, EmptyGeneration: 1}, }} coordinator := newCoordinator(t, routingConfiguration("stop", []string{"provider-a", "provider-b"}), state, stats, &refreshRecorder{}) var wait sync.WaitGroup for range 32 { wait.Add(1) go func() { defer wait.Done() if _, err := coordinator.Tick(context.Background()); err != nil { t.Errorf("Tick() = %v", err) } }() } wait.Wait() if state.switches != 1 || state.current() != "provider-b" { t.Fatalf("switches=%d current=%q, want one switch to provider-b", state.switches, state.current()) } } func TestSequentialCoordinatorUsesCompareAndSwapForDisabledCurrentAcrossReplicas(t *testing.T) { state := newRoutingState("provider-a", map[string]bool{"provider-a": false, "provider-b": true}) configuration := routingConfiguration("stop", []string{"provider-a", "provider-b"}) stats := &routingStats{byUpstream: map[string]provider.Stats{}} first := newCoordinator(t, configuration, state, stats, &refreshRecorder{}) second := newCoordinator(t, configuration, state, stats, &refreshRecorder{}) var wait sync.WaitGroup for index := range 32 { wait.Add(1) go func() { defer wait.Done() coordinator := first if index%2 == 1 { coordinator = second } if _, err := coordinator.Tick(context.Background()); err != nil { t.Errorf("Tick() = %v", err) } }() } wait.Wait() if state.switches != 1 || state.current() != "provider-b" || !state.enabled() { t.Fatalf("switches=%d current=%q enabled=%v, want one switch to provider-b", state.switches, state.current(), state.enabled()) } } func TestSequentialCoordinatorFailsClosedWhenConfigurationRevisionIsStale(t *testing.T) { state := newRoutingState("provider-a", map[string]bool{"provider-a": true, "provider-b": true}) state.snapshot.Config.Revision = 8 stats := &routingStats{byUpstream: map[string]provider.Stats{ "provider-a": {UpstreamID: "provider-a", ConsecutiveEmptyFetch: 5, EmptyGeneration: 1}, }} coordinator := newCoordinator(t, routingConfiguration("stop", []string{"provider-a", "provider-b"}), state, stats, &refreshRecorder{}) if _, err := coordinator.Tick(context.Background()); err == nil || state.current() != "provider-a" { t.Fatalf("Tick(stale revision) error=%v current=%q, want error and unchanged state", err, state.current()) } } func TestSequentialCoordinatorRejectsMalformedStatsAndCoalescesNotifications(t *testing.T) { state := newRoutingState("provider-a", map[string]bool{"provider-a": true, "provider-b": true}) configuration := staticConfiguration{configuration: routingConfiguration("stop", []string{"provider-a", "provider-b"}), revision: 7} coordinator, err := NewSequentialCoordinator(configuration, state, emptyStats{}, &refreshRecorder{}, Options{Now: time.Now}) if err != nil { t.Fatalf("NewSequentialCoordinator() = %v", err) } if _, err := coordinator.Tick(context.Background()); err == nil { t.Fatal("Tick() with malformed StatsReader response error = nil") } for range 4 { coordinator.ObserveProviderResult(provider.Result{UpstreamID: "provider-a", Class: "empty"}) } select { case <-coordinator.notify: default: t.Fatal("ObserveProviderResult() did not notify") } select { case <-coordinator.notify: t.Fatal("ObserveProviderResult() did not coalesce notifications") default: } } func newCoordinator( t *testing.T, configuration *config.Config, state *routingState, stats *routingStats, refresh *refreshRecorder, ) *SequentialCoordinator { t.Helper() coordinator, err := NewSequentialCoordinator(staticConfiguration{configuration: configuration, revision: 7}, state, stats, refresh, Options{ Now: func() time.Time { return time.Date(2026, 8, 2, 12, 0, 0, 0, time.UTC) }, }) if err != nil { t.Fatalf("NewSequentialCoordinator() = %v", err) } return coordinator } func routingConfiguration(endBehavior string, upstreams []string) *config.Config { configured := make(map[string]config.Upstream, len(upstreams)) for _, upstream := range upstreams { configured[upstream] = config.Upstream{Enabled: true} } return &config.Config{ Routing: []config.Routing{{ Name: "checkout", Enabled: true, Purpose: "gateway", Upstreams: append([]string(nil), upstreams...), Strategy: config.Strategy{Type: "sequential", SwitchAfterEmptyFetch: 5, EndBehavior: endBehavior}, }}, Upstreams: configured, } } type staticConfiguration struct { configuration *config.Config revision uint64 } func (source staticConfiguration) Snapshot() (*config.Config, uint64) { return source.configuration, source.revision } type routingState struct { mu sync.Mutex snapshot adminstate.Snapshot switches int stops int } func newRoutingState(current string, enabled map[string]bool) *routingState { upstreams := make([]adminstate.UpstreamState, 0, len(enabled)) for name, isEnabled := range enabled { upstreams = append(upstreams, adminstate.UpstreamState{Name: name, Enabled: isEnabled}) } return &routingState{snapshot: adminstate.Snapshot{ Config: &adminstate.ConfigRevision{Revision: 7}, Upstreams: upstreams, Routings: []adminstate.RoutingState{{Name: "checkout", Enabled: true, CurrentUpstream: current}}, }} } func (state *routingState) Snapshot(context.Context) (adminstate.Snapshot, error) { state.mu.Lock() defer state.mu.Unlock() snapshot := state.snapshot snapshot.Upstreams = append([]adminstate.UpstreamState(nil), state.snapshot.Upstreams...) snapshot.Routings = append([]adminstate.RoutingState(nil), state.snapshot.Routings...) return snapshot, nil } func (state *routingState) SwitchRouting(_ context.Context, command adminstate.SwitchRoutingCommand) (adminstate.MutationResult, error) { state.mu.Lock() defer state.mu.Unlock() routing := &state.snapshot.Routings[0] if command.Name != routing.Name || command.ExpectedCurrent != routing.CurrentUpstream { return adminstate.MutationResult{}, adminstate.ErrConflict } routing.CurrentUpstream = command.Target state.switches++ return adminstate.MutationResult{RequestID: command.RequestID, Changed: true, Revision: uint64(state.switches)}, nil } func (state *routingState) DisableRouting(_ context.Context, command adminstate.DisableRoutingCommand) (adminstate.MutationResult, error) { state.mu.Lock() defer state.mu.Unlock() routing := &state.snapshot.Routings[0] if command.Name != routing.Name || command.ExpectedCurrent != routing.CurrentUpstream { return adminstate.MutationResult{}, adminstate.ErrConflict } if !routing.Enabled { return adminstate.MutationResult{RequestID: command.RequestID}, nil } routing.Enabled = false state.stops++ return adminstate.MutationResult{RequestID: command.RequestID, Changed: true, Revision: uint64(state.stops)}, nil } func (state *routingState) current() string { state.mu.Lock() defer state.mu.Unlock() return state.snapshot.Routings[0].CurrentUpstream } func (state *routingState) enabled() bool { state.mu.Lock() defer state.mu.Unlock() return state.snapshot.Routings[0].Enabled } type routingStats struct { mu sync.Mutex byUpstream map[string]provider.Stats reads int } type emptyStats struct{} func (emptyStats) ReadProviderStats([]string) []provider.Stats { return nil } func (stats *routingStats) ReadProviderStats(upstreams []string) []provider.Stats { stats.mu.Lock() defer stats.mu.Unlock() stats.reads++ result := make([]provider.Stats, len(upstreams)) for index, upstream := range upstreams { result[index] = stats.byUpstream[upstream] result[index].UpstreamID = upstream } return result } func (stats *routingStats) readCount() int { stats.mu.Lock() defer stats.mu.Unlock() return stats.reads } func (stats *routingStats) set(upstream string, value provider.Stats) { stats.mu.Lock() defer stats.mu.Unlock() stats.byUpstream[upstream] = value } type refreshRecorder struct{ count int } func (recorder *refreshRecorder) NotifySnapshotRefresh() { recorder.count++ }