397 lines
17 KiB
Go
397 lines
17 KiB
Go
package routing
|
|
|
|
import (
|
|
"context"
|
|
"sync"
|
|
"testing"
|
|
"time"
|
|
|
|
"proxy-pool/internal/config"
|
|
"proxy-pool/internal/controller/provider"
|
|
"proxy-pool/internal/domain/adminstate"
|
|
)
|
|
|
|
func TestSequentialCoordinatorSwitchesOnlyAtConfiguredThreshold(t *testing.T) {
|
|
state := newRoutingState("provider-a", map[string]bool{"provider-a": true, "provider-b": true})
|
|
stats := &routingStats{byUpstream: map[string]provider.Stats{
|
|
"provider-a": {UpstreamID: "provider-a", ConsecutiveEmptyFetch: 4, EmptyGeneration: 1},
|
|
}}
|
|
refresh := &refreshRecorder{}
|
|
coordinator := newCoordinator(t, routingConfiguration("stop", []string{"provider-a", "provider-b"}), state, stats, refresh)
|
|
|
|
if result, err := coordinator.Tick(context.Background()); err != nil || result.Switched != 0 || state.current() != "provider-a" {
|
|
t.Fatalf("Tick(before threshold) = (%+v, %v), current=%q", result, err, state.current())
|
|
}
|
|
stats.set("provider-a", provider.Stats{UpstreamID: "provider-a", ConsecutiveEmptyFetch: 5, EmptyGeneration: 1})
|
|
if result, err := coordinator.Tick(context.Background()); err != nil || result.Switched != 1 || state.current() != "provider-b" {
|
|
t.Fatalf("Tick(at threshold) = (%+v, %v), current=%q", result, err, state.current())
|
|
}
|
|
if refresh.count != 1 || state.switches != 1 {
|
|
t.Fatalf("refresh=%d switches=%d, want 1", refresh.count, state.switches)
|
|
}
|
|
}
|
|
|
|
func TestSequentialCoordinatorStopsAtTerminalUpstream(t *testing.T) {
|
|
state := newRoutingState("provider-b", map[string]bool{"provider-a": true, "provider-b": true})
|
|
stats := &routingStats{byUpstream: map[string]provider.Stats{
|
|
"provider-b": {UpstreamID: "provider-b", ConsecutiveEmptyFetch: 5, EmptyGeneration: 9},
|
|
}}
|
|
refresh := &refreshRecorder{}
|
|
coordinator := newCoordinator(t, routingConfiguration("stop", []string{"provider-a", "provider-b"}), state, stats, refresh)
|
|
|
|
if result, err := coordinator.Tick(context.Background()); err != nil || result != (TickResult{Stopped: 1}) || state.enabled() || refresh.count != 1 {
|
|
t.Fatalf("Tick(terminal stop) = (%+v, %v), enabled=%v refresh=%d", result, err, state.enabled(), refresh.count)
|
|
}
|
|
if result, err := coordinator.Tick(context.Background()); err != nil || result != (TickResult{}) || state.stops != 1 || refresh.count != 1 {
|
|
t.Fatalf("Tick(reused terminal generation) = (%+v, %v), stops=%d refresh=%d", result, err, state.stops, refresh.count)
|
|
}
|
|
}
|
|
|
|
func TestSequentialCoordinatorStopsWhenDisabledCandidatesLeaveNoAlternative(t *testing.T) {
|
|
state := newRoutingState("provider-a", map[string]bool{"provider-a": true, "provider-b": false})
|
|
stats := &routingStats{byUpstream: map[string]provider.Stats{
|
|
"provider-a": {UpstreamID: "provider-a", ConsecutiveEmptyFetch: 5, EmptyGeneration: 4},
|
|
}}
|
|
coordinator := newCoordinator(t, routingConfiguration("stop", []string{"provider-a", "provider-b"}), state, stats, &refreshRecorder{})
|
|
|
|
if result, err := coordinator.Tick(context.Background()); err != nil || result != (TickResult{Stopped: 1}) || state.enabled() {
|
|
t.Fatalf("Tick(last enabled candidate) = (%+v, %v), enabled=%v", result, err, state.enabled())
|
|
}
|
|
}
|
|
|
|
func TestSequentialCoordinatorAdvancesWhenCurrentUpstreamIsDisabledWithoutProviderStats(t *testing.T) {
|
|
state := newRoutingState("provider-a", map[string]bool{"provider-a": false, "provider-b": true, "provider-c": true})
|
|
stats := &routingStats{byUpstream: map[string]provider.Stats{}}
|
|
refresh := &refreshRecorder{}
|
|
coordinator := newCoordinator(t, routingConfiguration("stop", []string{"provider-a", "provider-b", "provider-c"}), state, stats, refresh)
|
|
|
|
if result, err := coordinator.Tick(context.Background()); err != nil || result != (TickResult{Switched: 1}) || state.current() != "provider-b" {
|
|
t.Fatalf("Tick(disabled current) = (%+v, %v), current=%q", result, err, state.current())
|
|
}
|
|
if stats.readCount() != 0 || state.switches != 1 || refresh.count != 1 {
|
|
t.Fatalf("provider reads=%d switches=%d refresh=%d, want 0/1/1", stats.readCount(), state.switches, refresh.count)
|
|
}
|
|
}
|
|
|
|
func TestSequentialCoordinatorStopsWhenDisabledCurrentHasNoSuccessor(t *testing.T) {
|
|
state := newRoutingState("provider-b", map[string]bool{"provider-a": true, "provider-b": false})
|
|
coordinator := newCoordinator(t, routingConfiguration("stayLast", []string{"provider-a", "provider-b"}), state,
|
|
&routingStats{byUpstream: map[string]provider.Stats{}}, &refreshRecorder{})
|
|
|
|
if result, err := coordinator.Tick(context.Background()); err != nil || result != (TickResult{Stopped: 1}) || state.enabled() {
|
|
t.Fatalf("Tick(disabled terminal current) = (%+v, %v), enabled=%v", result, err, state.enabled())
|
|
}
|
|
}
|
|
|
|
func TestSequentialCoordinatorLoopsWhenDisabledCurrentHasEarlierSuccessor(t *testing.T) {
|
|
state := newRoutingState("provider-c", map[string]bool{"provider-a": true, "provider-b": false, "provider-c": false})
|
|
coordinator := newCoordinator(t, routingConfiguration("loop", []string{"provider-a", "provider-b", "provider-c"}), state,
|
|
&routingStats{byUpstream: map[string]provider.Stats{}}, &refreshRecorder{})
|
|
|
|
if result, err := coordinator.Tick(context.Background()); err != nil || result != (TickResult{Switched: 1}) || state.current() != "provider-a" {
|
|
t.Fatalf("Tick(disabled loop current) = (%+v, %v), current=%q", result, err, state.current())
|
|
}
|
|
}
|
|
|
|
func TestSequentialCoordinatorConsumesStatsRecorderNotifications(t *testing.T) {
|
|
state := newRoutingState("provider-a", map[string]bool{"provider-a": true, "provider-b": true})
|
|
stats, err := provider.NewStatsRecorder(2)
|
|
if err != nil {
|
|
t.Fatalf("NewStatsRecorder() = %v", err)
|
|
}
|
|
refresh := &refreshRecorder{}
|
|
coordinator, err := NewSequentialCoordinator(
|
|
staticConfiguration{configuration: routingConfiguration("stop", []string{"provider-a", "provider-b"}), revision: 7},
|
|
state,
|
|
stats,
|
|
refresh,
|
|
Options{Now: time.Now},
|
|
)
|
|
if err != nil {
|
|
t.Fatalf("NewSequentialCoordinator() = %v", err)
|
|
}
|
|
stats.AddResultObserver(coordinator)
|
|
for range 5 {
|
|
stats.Record(provider.Result{UpstreamID: "provider-a", Class: "empty"})
|
|
}
|
|
select {
|
|
case <-coordinator.notify:
|
|
case <-time.After(time.Second):
|
|
t.Fatal("StatsRecorder did not notify SequentialCoordinator")
|
|
}
|
|
if result, err := coordinator.Tick(context.Background()); err != nil || result.Switched != 1 || state.current() != "provider-b" || refresh.count != 1 {
|
|
t.Fatalf("Tick() = (%+v, %v), current=%q refresh=%d", result, err, state.current(), refresh.count)
|
|
}
|
|
}
|
|
|
|
func TestSequentialCoordinatorSkipsDisabledCandidatesAndStaysAtConfiguredEnd(t *testing.T) {
|
|
state := newRoutingState("provider-a", map[string]bool{
|
|
"provider-a": true, "provider-b": false, "provider-c": true,
|
|
})
|
|
stats := &routingStats{byUpstream: map[string]provider.Stats{
|
|
"provider-a": {UpstreamID: "provider-a", ConsecutiveEmptyFetch: 5, EmptyGeneration: 1},
|
|
"provider-c": {UpstreamID: "provider-c", ConsecutiveEmptyFetch: 5, EmptyGeneration: 2},
|
|
}}
|
|
coordinator := newCoordinator(t, routingConfiguration("stayLast", []string{"provider-a", "provider-b", "provider-c"}), state, stats, &refreshRecorder{})
|
|
|
|
if result, err := coordinator.Tick(context.Background()); err != nil || result.Switched != 1 || state.current() != "provider-c" {
|
|
t.Fatalf("Tick(skip disabled) = (%+v, %v), current=%q", result, err, state.current())
|
|
}
|
|
if result, err := coordinator.Tick(context.Background()); err != nil || result.Switched != 0 || state.current() != "provider-c" {
|
|
t.Fatalf("Tick(stay last) = (%+v, %v), current=%q", result, err, state.current())
|
|
}
|
|
}
|
|
|
|
func TestSequentialCoordinatorDoesNotReuseAnEmptyGenerationAfterLoop(t *testing.T) {
|
|
state := newRoutingState("provider-a", map[string]bool{"provider-a": true, "provider-b": true})
|
|
stats := &routingStats{byUpstream: map[string]provider.Stats{
|
|
"provider-a": {UpstreamID: "provider-a", ConsecutiveEmptyFetch: 5, EmptyGeneration: 1},
|
|
"provider-b": {UpstreamID: "provider-b", ConsecutiveEmptyFetch: 5, EmptyGeneration: 2},
|
|
}}
|
|
coordinator := newCoordinator(t, routingConfiguration("loop", []string{"provider-a", "provider-b"}), state, stats, &refreshRecorder{})
|
|
|
|
for _, want := range []string{"provider-b", "provider-a"} {
|
|
if result, err := coordinator.Tick(context.Background()); err != nil || result.Switched != 1 || state.current() != want {
|
|
t.Fatalf("Tick() = (%+v, %v), current=%q, want %q", result, err, state.current(), want)
|
|
}
|
|
}
|
|
if result, err := coordinator.Tick(context.Background()); err != nil || result.Switched != 0 || state.current() != "provider-a" {
|
|
t.Fatalf("Tick(reused generation) = (%+v, %v), current=%q", result, err, state.current())
|
|
}
|
|
stats.set("provider-a", provider.Stats{UpstreamID: "provider-a", ConsecutiveEmptyFetch: 5, EmptyGeneration: 3})
|
|
if result, err := coordinator.Tick(context.Background()); err != nil || result.Switched != 1 || state.current() != "provider-b" {
|
|
t.Fatalf("Tick(new generation) = (%+v, %v), current=%q", result, err, state.current())
|
|
}
|
|
}
|
|
|
|
func TestSequentialCoordinatorUsesCompareAndSwapUnderConcurrentTicks(t *testing.T) {
|
|
state := newRoutingState("provider-a", map[string]bool{"provider-a": true, "provider-b": true})
|
|
stats := &routingStats{byUpstream: map[string]provider.Stats{
|
|
"provider-a": {UpstreamID: "provider-a", ConsecutiveEmptyFetch: 5, EmptyGeneration: 1},
|
|
}}
|
|
coordinator := newCoordinator(t, routingConfiguration("stop", []string{"provider-a", "provider-b"}), state, stats, &refreshRecorder{})
|
|
|
|
var wait sync.WaitGroup
|
|
for range 32 {
|
|
wait.Add(1)
|
|
go func() {
|
|
defer wait.Done()
|
|
if _, err := coordinator.Tick(context.Background()); err != nil {
|
|
t.Errorf("Tick() = %v", err)
|
|
}
|
|
}()
|
|
}
|
|
wait.Wait()
|
|
if state.switches != 1 || state.current() != "provider-b" {
|
|
t.Fatalf("switches=%d current=%q, want one switch to provider-b", state.switches, state.current())
|
|
}
|
|
}
|
|
|
|
func TestSequentialCoordinatorUsesCompareAndSwapForDisabledCurrentAcrossReplicas(t *testing.T) {
|
|
state := newRoutingState("provider-a", map[string]bool{"provider-a": false, "provider-b": true})
|
|
configuration := routingConfiguration("stop", []string{"provider-a", "provider-b"})
|
|
stats := &routingStats{byUpstream: map[string]provider.Stats{}}
|
|
first := newCoordinator(t, configuration, state, stats, &refreshRecorder{})
|
|
second := newCoordinator(t, configuration, state, stats, &refreshRecorder{})
|
|
|
|
var wait sync.WaitGroup
|
|
for index := range 32 {
|
|
wait.Add(1)
|
|
go func() {
|
|
defer wait.Done()
|
|
coordinator := first
|
|
if index%2 == 1 {
|
|
coordinator = second
|
|
}
|
|
if _, err := coordinator.Tick(context.Background()); err != nil {
|
|
t.Errorf("Tick() = %v", err)
|
|
}
|
|
}()
|
|
}
|
|
wait.Wait()
|
|
if state.switches != 1 || state.current() != "provider-b" || !state.enabled() {
|
|
t.Fatalf("switches=%d current=%q enabled=%v, want one switch to provider-b", state.switches, state.current(), state.enabled())
|
|
}
|
|
}
|
|
|
|
func TestSequentialCoordinatorFailsClosedWhenConfigurationRevisionIsStale(t *testing.T) {
|
|
state := newRoutingState("provider-a", map[string]bool{"provider-a": true, "provider-b": true})
|
|
state.snapshot.Config.Revision = 8
|
|
stats := &routingStats{byUpstream: map[string]provider.Stats{
|
|
"provider-a": {UpstreamID: "provider-a", ConsecutiveEmptyFetch: 5, EmptyGeneration: 1},
|
|
}}
|
|
coordinator := newCoordinator(t, routingConfiguration("stop", []string{"provider-a", "provider-b"}), state, stats, &refreshRecorder{})
|
|
if _, err := coordinator.Tick(context.Background()); err == nil || state.current() != "provider-a" {
|
|
t.Fatalf("Tick(stale revision) error=%v current=%q, want error and unchanged state", err, state.current())
|
|
}
|
|
}
|
|
|
|
func TestSequentialCoordinatorRejectsMalformedStatsAndCoalescesNotifications(t *testing.T) {
|
|
state := newRoutingState("provider-a", map[string]bool{"provider-a": true, "provider-b": true})
|
|
configuration := staticConfiguration{configuration: routingConfiguration("stop", []string{"provider-a", "provider-b"}), revision: 7}
|
|
coordinator, err := NewSequentialCoordinator(configuration, state, emptyStats{}, &refreshRecorder{}, Options{Now: time.Now})
|
|
if err != nil {
|
|
t.Fatalf("NewSequentialCoordinator() = %v", err)
|
|
}
|
|
if _, err := coordinator.Tick(context.Background()); err == nil {
|
|
t.Fatal("Tick() with malformed StatsReader response error = nil")
|
|
}
|
|
for range 4 {
|
|
coordinator.ObserveProviderResult(provider.Result{UpstreamID: "provider-a", Class: "empty"})
|
|
}
|
|
select {
|
|
case <-coordinator.notify:
|
|
default:
|
|
t.Fatal("ObserveProviderResult() did not notify")
|
|
}
|
|
select {
|
|
case <-coordinator.notify:
|
|
t.Fatal("ObserveProviderResult() did not coalesce notifications")
|
|
default:
|
|
}
|
|
}
|
|
|
|
func newCoordinator(
|
|
t *testing.T,
|
|
configuration *config.Config,
|
|
state *routingState,
|
|
stats *routingStats,
|
|
refresh *refreshRecorder,
|
|
) *SequentialCoordinator {
|
|
t.Helper()
|
|
coordinator, err := NewSequentialCoordinator(staticConfiguration{configuration: configuration, revision: 7}, state, stats, refresh, Options{
|
|
Now: func() time.Time { return time.Date(2026, 8, 2, 12, 0, 0, 0, time.UTC) },
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("NewSequentialCoordinator() = %v", err)
|
|
}
|
|
return coordinator
|
|
}
|
|
|
|
func routingConfiguration(endBehavior string, upstreams []string) *config.Config {
|
|
configured := make(map[string]config.Upstream, len(upstreams))
|
|
for _, upstream := range upstreams {
|
|
configured[upstream] = config.Upstream{Enabled: true}
|
|
}
|
|
return &config.Config{
|
|
Routing: []config.Routing{{
|
|
Name: "checkout", Enabled: true, Purpose: "gateway", Upstreams: append([]string(nil), upstreams...),
|
|
Strategy: config.Strategy{Type: "sequential", SwitchAfterEmptyFetch: 5, EndBehavior: endBehavior},
|
|
}},
|
|
Upstreams: configured,
|
|
}
|
|
}
|
|
|
|
type staticConfiguration struct {
|
|
configuration *config.Config
|
|
revision uint64
|
|
}
|
|
|
|
func (source staticConfiguration) Snapshot() (*config.Config, uint64) {
|
|
return source.configuration, source.revision
|
|
}
|
|
|
|
type routingState struct {
|
|
mu sync.Mutex
|
|
snapshot adminstate.Snapshot
|
|
switches int
|
|
stops int
|
|
}
|
|
|
|
func newRoutingState(current string, enabled map[string]bool) *routingState {
|
|
upstreams := make([]adminstate.UpstreamState, 0, len(enabled))
|
|
for name, isEnabled := range enabled {
|
|
upstreams = append(upstreams, adminstate.UpstreamState{Name: name, Enabled: isEnabled})
|
|
}
|
|
return &routingState{snapshot: adminstate.Snapshot{
|
|
Config: &adminstate.ConfigRevision{Revision: 7}, Upstreams: upstreams,
|
|
Routings: []adminstate.RoutingState{{Name: "checkout", Enabled: true, CurrentUpstream: current}},
|
|
}}
|
|
}
|
|
|
|
func (state *routingState) Snapshot(context.Context) (adminstate.Snapshot, error) {
|
|
state.mu.Lock()
|
|
defer state.mu.Unlock()
|
|
snapshot := state.snapshot
|
|
snapshot.Upstreams = append([]adminstate.UpstreamState(nil), state.snapshot.Upstreams...)
|
|
snapshot.Routings = append([]adminstate.RoutingState(nil), state.snapshot.Routings...)
|
|
return snapshot, nil
|
|
}
|
|
|
|
func (state *routingState) SwitchRouting(_ context.Context, command adminstate.SwitchRoutingCommand) (adminstate.MutationResult, error) {
|
|
state.mu.Lock()
|
|
defer state.mu.Unlock()
|
|
routing := &state.snapshot.Routings[0]
|
|
if command.Name != routing.Name || command.ExpectedCurrent != routing.CurrentUpstream {
|
|
return adminstate.MutationResult{}, adminstate.ErrConflict
|
|
}
|
|
routing.CurrentUpstream = command.Target
|
|
state.switches++
|
|
return adminstate.MutationResult{RequestID: command.RequestID, Changed: true, Revision: uint64(state.switches)}, nil
|
|
}
|
|
|
|
func (state *routingState) DisableRouting(_ context.Context, command adminstate.DisableRoutingCommand) (adminstate.MutationResult, error) {
|
|
state.mu.Lock()
|
|
defer state.mu.Unlock()
|
|
routing := &state.snapshot.Routings[0]
|
|
if command.Name != routing.Name || command.ExpectedCurrent != routing.CurrentUpstream {
|
|
return adminstate.MutationResult{}, adminstate.ErrConflict
|
|
}
|
|
if !routing.Enabled {
|
|
return adminstate.MutationResult{RequestID: command.RequestID}, nil
|
|
}
|
|
routing.Enabled = false
|
|
state.stops++
|
|
return adminstate.MutationResult{RequestID: command.RequestID, Changed: true, Revision: uint64(state.stops)}, nil
|
|
}
|
|
|
|
func (state *routingState) current() string {
|
|
state.mu.Lock()
|
|
defer state.mu.Unlock()
|
|
return state.snapshot.Routings[0].CurrentUpstream
|
|
}
|
|
|
|
func (state *routingState) enabled() bool {
|
|
state.mu.Lock()
|
|
defer state.mu.Unlock()
|
|
return state.snapshot.Routings[0].Enabled
|
|
}
|
|
|
|
type routingStats struct {
|
|
mu sync.Mutex
|
|
byUpstream map[string]provider.Stats
|
|
reads int
|
|
}
|
|
|
|
type emptyStats struct{}
|
|
|
|
func (emptyStats) ReadProviderStats([]string) []provider.Stats { return nil }
|
|
|
|
func (stats *routingStats) ReadProviderStats(upstreams []string) []provider.Stats {
|
|
stats.mu.Lock()
|
|
defer stats.mu.Unlock()
|
|
stats.reads++
|
|
result := make([]provider.Stats, len(upstreams))
|
|
for index, upstream := range upstreams {
|
|
result[index] = stats.byUpstream[upstream]
|
|
result[index].UpstreamID = upstream
|
|
}
|
|
return result
|
|
}
|
|
|
|
func (stats *routingStats) readCount() int {
|
|
stats.mu.Lock()
|
|
defer stats.mu.Unlock()
|
|
return stats.reads
|
|
}
|
|
|
|
func (stats *routingStats) set(upstream string, value provider.Stats) {
|
|
stats.mu.Lock()
|
|
defer stats.mu.Unlock()
|
|
stats.byUpstream[upstream] = value
|
|
}
|
|
|
|
type refreshRecorder struct{ count int }
|
|
|
|
func (recorder *refreshRecorder) NotifySnapshotRefresh() { recorder.count++ }
|