feat: automate sequential routing switches
This commit is contained in:
parent
707aff26ea
commit
589cd4e454
@ -115,7 +115,8 @@ flowchart LR
|
|||||||
Redis 会话栅栏,以及 Gateway Outcome 上报的有界队列、序列确认与重试;
|
Redis 会话栅栏,以及 Gateway Outcome 上报的有界队列、序列确认与重试;
|
||||||
Controller 的 Redis 共享 BASIC/EGRESS/TARGET 检查任务、按上游的有界轮转调度、HTTP/HTTPS/SOCKS5
|
Controller 的 Redis 共享 BASIC/EGRESS/TARGET 检查任务、按上游的有界轮转调度、HTTP/HTTPS/SOCKS5
|
||||||
Checker 探测和
|
Checker 探测和
|
||||||
Observation 状态归并。
|
Observation 状态归并;Provider 连续空结果的代次化自动 Sequential 切换、禁用候选过滤和
|
||||||
|
Snapshot 即时刷新。
|
||||||
- **部分完成**:Docker Compose/Kubernetes 运行时 mTLS Overlay。
|
- **部分完成**:Docker Compose/Kubernetes 运行时 mTLS Overlay。
|
||||||
- **待完成**:CONNECT 长连接/Extract 压测场景、故障演练和代表性集群压测。
|
- **待完成**:CONNECT 长连接/Extract 压测场景、故障演练和代表性集群压测。
|
||||||
|
|
||||||
|
|||||||
@ -104,9 +104,13 @@ Worker 维度;该测试纳入 Linux CI 的 race 范围,本机因 `CGO_ENABLE
|
|||||||
- [ ] Cover four-empty-then-success, five-empty, A-to-B-only, disabled references,
|
- [ ] Cover four-empty-then-success, five-empty, A-to-B-only, disabled references,
|
||||||
end behavior, and explicit onUnavailable.
|
end behavior, and explicit onUnavailable.
|
||||||
|
|
||||||
当前进度(2026-07-29):领域构造器与严格配置已统一 Sequential 至少两个
|
当前进度(2026-08-02):领域构造器与严格配置已统一 Sequential 至少两个
|
||||||
Upstream、`endBehavior` 默认 `stop`,并覆盖列表末端停止;disabled candidate、
|
Upstream、`endBehavior` 默认 `stop`,并覆盖列表末端停止。Provider Stats 现为每次
|
||||||
跨实例恢复和 `onUnavailable` 运行链仍待完成。
|
连续空结果分配单调代次;Controller 的公共 Sequential 协调器只接收有界通知,在独立
|
||||||
|
循环中读取权威配置、管理快照和 Stats,并以既有 `ExpectedCurrent` CAS 自动切换。
|
||||||
|
它会跳过禁用 Upstream,支持 `loop`/`stayLast`,同一空结果代次不会在循环后重复切换,
|
||||||
|
成功后立即广播完整 Snapshot。Gateway 已执行 reject/wait/direct;控制面将末端 `stop`
|
||||||
|
持久化为禁用路由的原子管理态变更仍待实现。
|
||||||
|
|
||||||
## Task 5: Provider Fetch Classification and Scheduling
|
## Task 5: Provider Fetch Classification and Scheduling
|
||||||
|
|
||||||
|
|||||||
@ -124,8 +124,8 @@ Controller/Gateway 入口,完整 mTLS 运行时拓扑仍只有静态验证。
|
|||||||
生产任务调度与 REMOVE 编排仍待实现。
|
生产任务调度与 REMOVE 编排仍待实现。
|
||||||
8. Admin/Distribution 细粒度授权和审计查询;Distribution 分布式限流已完成。
|
8. Admin/Distribution 细粒度授权和审计查询;Distribution 分布式限流已完成。
|
||||||
9. 真实 Compose/Kubernetes 集成、故障演练和代表性集群负载测试。
|
9. 真实 Compose/Kubernetes 集成、故障演练和代表性集群负载测试。
|
||||||
10. 将 reject/wait/direct 接入 Distribution 运行链,补齐 Sequential 持久化恢复、跨实例 CAS
|
10. 将 reject/wait/direct 接入 Distribution 运行链;Sequential 已接入 Provider 空结果
|
||||||
和 disabled candidate 语义。
|
代次、管理库 CAS 和禁用候选过滤,仍需补齐末端 `stop` 的原子路由停用管理态。
|
||||||
## 4. 容量结论
|
## 4. 容量结论
|
||||||
|
|
||||||
100,000 QPS 是集群设计输入,不是本次验证结果。只有实现上述运行时,并在
|
100,000 QPS 是集群设计输入,不是本次验证结果。只有实现上述运行时,并在
|
||||||
|
|||||||
@ -20,7 +20,7 @@
|
|||||||
| ROUTE-001 | Routing 自上而下匹配,首条命中停止 | 3534-3798, 5825-6467 | `rule.go` 与不可变/首命中单测 |
|
| ROUTE-001 | Routing 自上而下匹配,首条命中停止 | 3534-3798, 5825-6467 | `rule.go` 与不可变/首命中单测 |
|
||||||
| ROUTE-002 | Routing 与 Upstream 生命周期解耦 | 3534-3798 | 包依赖与配置模型 |
|
| ROUTE-002 | Routing 与 Upstream 生命周期解耦 | 3534-3798 | 包依赖与配置模型 |
|
||||||
| ROUTE-003 | 支持 sequential、random、roundRobin、weighted、leastConnections | 5825-6467 | 五种领域策略、同版本 Gateway Snapshot 派发和定向测试已完成;Distribution 接线待完成 |
|
| ROUTE-003 | 支持 sequential、random、roundRobin、weighted、leastConnections | 5825-6467 | 五种领域策略、同版本 Gateway Snapshot 派发和定向测试已完成;Distribution 接线待完成 |
|
||||||
| ROUTE-004 | Sequential 连续空结果达到阈值后原子切换一次 | 5295-5824, 6520-6617 | 进程内 `RoutingCursor` 版本 CAS 与 100 并发测试已完成;持久化恢复和跨实例 CAS 待完成 |
|
| ROUTE-004 | Sequential 连续空结果达到阈值后原子切换一次 | 5295-5824, 6520-6617 | 进程内 `RoutingCursor` 版本 CAS 与 100 并发测试已完成;Provider Stats 对连续空结果分配单调代次,Controller `SequentialCoordinator` 在独立有界循环中读取权威配置/管理快照,并通过管理库 `ExpectedCurrent` CAS 自动切换。禁用候选会跳过,重复 Tick、循环后旧代次和并发 Tick 均不会再次切换;成功即广播完整 Snapshot |
|
||||||
| ROUTE-005 | 空计数属于 Upstream,当前选择属于 Routing | 8442-8529 | 共享 `UpstreamEmptyState` 双 Routing 测试 |
|
| ROUTE-005 | 空计数属于 Upstream,当前选择属于 Routing | 8442-8529 | 共享 `UpstreamEmptyState` 双 Routing 测试 |
|
||||||
| ROUTE-006 | 旧 Upstream 已有 Proxy 继续耗尽,不因切换直接丢弃 | 6618-6641 | Routing 成功切换后立即发布完整快照;Sequential 仅将新分配切到新的 CurrentUpstream,旧 Proxy 仍保留在快照,既有 Active/Reserved 由本地运行态自然归零。共享 Upstream 不按单 Routing 强制 Drain,避免影响其他 Routing |
|
| ROUTE-006 | 旧 Upstream 已有 Proxy 继续耗尽,不因切换直接丢弃 | 6618-6641 | Routing 成功切换后立即发布完整快照;Sequential 仅将新分配切到新的 CurrentUpstream,旧 Proxy 仍保留在快照,既有 Active/Reserved 由本地运行态自然归零。共享 Upstream 不按单 Routing 强制 Drain,避免影响其他 Routing |
|
||||||
| ROUTE-007 | 无可用 Upstream 时显式 reject、wait 或 direct,默认 reject | 5075-5294, 6743-6760 | Gateway 已实现 reject、带 `wait_timeout` 的本地容量等待与经 TargetPolicy 的 direct;Distribution 接线和默认化策略待完成 |
|
| ROUTE-007 | 无可用 Upstream 时显式 reject、wait 或 direct,默认 reject | 5075-5294, 6743-6760 | Gateway 已实现 reject、带 `wait_timeout` 的本地容量等待与经 TargetPolicy 的 direct;Distribution 接线和默认化策略待完成 |
|
||||||
|
|||||||
@ -19,6 +19,7 @@ import (
|
|||||||
"proxy-pool/internal/controller/operations"
|
"proxy-pool/internal/controller/operations"
|
||||||
"proxy-pool/internal/controller/pool"
|
"proxy-pool/internal/controller/pool"
|
||||||
"proxy-pool/internal/controller/provider"
|
"proxy-pool/internal/controller/provider"
|
||||||
|
controllerRouting "proxy-pool/internal/controller/routing"
|
||||||
controllerRuntime "proxy-pool/internal/controller/runtime"
|
controllerRuntime "proxy-pool/internal/controller/runtime"
|
||||||
"proxy-pool/internal/controller/worker"
|
"proxy-pool/internal/controller/worker"
|
||||||
"proxy-pool/internal/domain/activitypool"
|
"proxy-pool/internal/domain/activitypool"
|
||||||
@ -178,6 +179,7 @@ func runWithWorkerFactory(
|
|||||||
return fmt.Errorf("%w: build Provider supervisor: %w", ErrStartup, err)
|
return fmt.Errorf("%w: build Provider supervisor: %w", ErrStartup, err)
|
||||||
}
|
}
|
||||||
snapshotRefresh := worker.NewSnapshotRefreshBroker()
|
snapshotRefresh := worker.NewSnapshotRefreshBroker()
|
||||||
|
var sequentialCoordinator lifecycle.Runner
|
||||||
|
|
||||||
dependencies := controllerRuntime.Dependencies{}
|
dependencies := controllerRuntime.Dependencies{}
|
||||||
var checkerMetrics healthDomain.TaskMetricsObserver
|
var checkerMetrics healthDomain.TaskMetricsObserver
|
||||||
@ -269,6 +271,22 @@ func runWithWorkerFactory(
|
|||||||
return fmt.Errorf("%w: commit startup configuration: %w", ErrStartup, applyErr)
|
return fmt.Errorf("%w: commit startup configuration: %w", ErrStartup, applyErr)
|
||||||
}
|
}
|
||||||
dependencies.AdminService = service
|
dependencies.AdminService = service
|
||||||
|
if stats, ok := opened.providerResults.(provider.StatsReader); ok && !nilInterface(stats) {
|
||||||
|
coordinator, coordinatorErr := controllerRouting.NewSequentialCoordinator(
|
||||||
|
configurationStore,
|
||||||
|
opened.state,
|
||||||
|
stats,
|
||||||
|
snapshotRefresh,
|
||||||
|
controllerRouting.Options{Now: options.Now},
|
||||||
|
)
|
||||||
|
if coordinatorErr != nil {
|
||||||
|
return fmt.Errorf("%w: build Sequential routing coordinator: %w", ErrStartup, coordinatorErr)
|
||||||
|
}
|
||||||
|
if registrar, ok := opened.providerResults.(provider.ResultObserverRegistrar); ok && !nilInterface(registrar) {
|
||||||
|
registrar.AddResultObserver(coordinator)
|
||||||
|
}
|
||||||
|
sequentialCoordinator = coordinator
|
||||||
|
}
|
||||||
}
|
}
|
||||||
runners := make([]lifecycle.Runner, 0, 3+len(loaded.Value.Upstreams))
|
runners := make([]lifecycle.Runner, 0, 3+len(loaded.Value.Upstreams))
|
||||||
if hasHTTPRuntime(loaded.Value) {
|
if hasHTTPRuntime(loaded.Value) {
|
||||||
@ -396,6 +414,9 @@ func runWithWorkerFactory(
|
|||||||
runners = append(runners, reaper)
|
runners = append(runners, reaper)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
if sequentialCoordinator != nil {
|
||||||
|
runners = append(runners, sequentialCoordinator)
|
||||||
|
}
|
||||||
runners = append(runners, supervisor)
|
runners = append(runners, supervisor)
|
||||||
group, err := lifecycle.NewGroup(runners...)
|
group, err := lifecycle.NewGroup(runners...)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|||||||
@ -14,6 +14,7 @@ var ErrInvalidStatsRecorder = errors.New("invalid Provider stats recorder")
|
|||||||
type Stats struct {
|
type Stats struct {
|
||||||
UpstreamID string
|
UpstreamID string
|
||||||
ConsecutiveEmptyFetch int64
|
ConsecutiveEmptyFetch int64
|
||||||
|
EmptyGeneration uint64
|
||||||
FetchErrorCount int64
|
FetchErrorCount int64
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -30,6 +31,7 @@ type StatsRecorder struct {
|
|||||||
maximum int
|
maximum int
|
||||||
byID map[string]Stats
|
byID map[string]Stats
|
||||||
observers []ResultObserver
|
observers []ResultObserver
|
||||||
|
nextEmptyGeneration uint64
|
||||||
}
|
}
|
||||||
|
|
||||||
func NewStatsRecorder(maximum int) (*StatsRecorder, error) {
|
func NewStatsRecorder(maximum int) (*StatsRecorder, error) {
|
||||||
@ -56,6 +58,10 @@ func (recorder *StatsRecorder) Record(result Result) {
|
|||||||
}
|
}
|
||||||
switch result.Class {
|
switch result.Class {
|
||||||
case upstream.FetchEmpty:
|
case upstream.FetchEmpty:
|
||||||
|
if stats.ConsecutiveEmptyFetch == 0 && recorder.nextEmptyGeneration < math.MaxUint64 {
|
||||||
|
recorder.nextEmptyGeneration++
|
||||||
|
stats.EmptyGeneration = recorder.nextEmptyGeneration
|
||||||
|
}
|
||||||
if stats.ConsecutiveEmptyFetch < math.MaxInt64 {
|
if stats.ConsecutiveEmptyFetch < math.MaxInt64 {
|
||||||
stats.ConsecutiveEmptyFetch++
|
stats.ConsecutiveEmptyFetch++
|
||||||
}
|
}
|
||||||
|
|||||||
@ -29,6 +29,26 @@ func TestStatsRecorderTracksEmptyResetAndErrors(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestStatsRecorderAssignsOneGenerationToEachEmptyEpisode(t *testing.T) {
|
||||||
|
recorder, err := NewStatsRecorder(1)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("NewStatsRecorder() = %v", err)
|
||||||
|
}
|
||||||
|
recorder.Record(Result{UpstreamID: "provider-a", Class: upstream.FetchEmpty})
|
||||||
|
recorder.Record(Result{UpstreamID: "provider-a", Class: upstream.FetchEmpty})
|
||||||
|
first := recorder.ReadProviderStats([]string{"provider-a"})[0]
|
||||||
|
if first.ConsecutiveEmptyFetch != 2 || first.EmptyGeneration == 0 {
|
||||||
|
t.Fatalf("first empty episode = %+v, want count=2 and generation", first)
|
||||||
|
}
|
||||||
|
|
||||||
|
recorder.Record(Result{UpstreamID: "provider-a", Class: upstream.FetchValid})
|
||||||
|
recorder.Record(Result{UpstreamID: "provider-a", Class: upstream.FetchEmpty})
|
||||||
|
second := recorder.ReadProviderStats([]string{"provider-a"})[0]
|
||||||
|
if second.ConsecutiveEmptyFetch != 1 || second.EmptyGeneration <= first.EmptyGeneration {
|
||||||
|
t.Fatalf("second empty episode = %+v after %+v, want reset count and newer generation", second, first)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestStatsRecorderIsBoundedAndConcurrent(t *testing.T) {
|
func TestStatsRecorderIsBoundedAndConcurrent(t *testing.T) {
|
||||||
recorder, err := NewStatsRecorder(1)
|
recorder, err := NewStatsRecorder(1)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|||||||
302
internal/controller/routing/sequential.go
Normal file
302
internal/controller/routing/sequential.go
Normal file
@ -0,0 +1,302 @@
|
|||||||
|
// Package routing coordinates Controller-side automatic Sequential switching.
|
||||||
|
// It consumes bounded Provider result notifications and keeps the actual state
|
||||||
|
// transition in the existing management-store compare-and-swap boundary.
|
||||||
|
package routing
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"crypto/sha256"
|
||||||
|
"encoding/hex"
|
||||||
|
"errors"
|
||||||
|
"reflect"
|
||||||
|
"strconv"
|
||||||
|
"sync"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"proxy-pool/internal/config"
|
||||||
|
"proxy-pool/internal/controller/provider"
|
||||||
|
"proxy-pool/internal/domain/adminstate"
|
||||||
|
"proxy-pool/internal/domain/upstream"
|
||||||
|
)
|
||||||
|
|
||||||
|
var ErrInvalidSequentialCoordinator = errors.New("invalid sequential routing coordinator")
|
||||||
|
|
||||||
|
const (
|
||||||
|
defaultPollInterval = time.Second
|
||||||
|
autoSwitchActor = "proxy-controller"
|
||||||
|
autoSwitchReason = "consecutive empty provider fetches reached routing threshold"
|
||||||
|
)
|
||||||
|
|
||||||
|
// ConfigurationSource supplies one immutable configuration and its matching
|
||||||
|
// authoritative management revision.
|
||||||
|
type ConfigurationSource interface {
|
||||||
|
Snapshot() (*config.Config, uint64)
|
||||||
|
}
|
||||||
|
|
||||||
|
// StateStore contains the only mutation boundary for routing current-upstream
|
||||||
|
// state. Implementations enforce ExpectedCurrent as a compare-and-swap fence.
|
||||||
|
type StateStore interface {
|
||||||
|
Snapshot(context.Context) (adminstate.Snapshot, error)
|
||||||
|
SwitchRouting(context.Context, adminstate.SwitchRoutingCommand) (adminstate.MutationResult, error)
|
||||||
|
}
|
||||||
|
|
||||||
|
type SnapshotRefreshNotifier interface {
|
||||||
|
NotifySnapshotRefresh()
|
||||||
|
}
|
||||||
|
|
||||||
|
type Options struct {
|
||||||
|
Now func() time.Time
|
||||||
|
PollInterval time.Duration
|
||||||
|
}
|
||||||
|
|
||||||
|
type TickResult struct {
|
||||||
|
Switched int
|
||||||
|
}
|
||||||
|
|
||||||
|
// SequentialCoordinator observes Provider fetch outcomes without blocking the
|
||||||
|
// fetch path. Tick serializes bounded scans and only records a generation as
|
||||||
|
// processed after it has made, or intentionally declined, a decision.
|
||||||
|
type SequentialCoordinator struct {
|
||||||
|
configuration ConfigurationSource
|
||||||
|
state StateStore
|
||||||
|
stats provider.StatsReader
|
||||||
|
refresh SnapshotRefreshNotifier
|
||||||
|
now func() time.Time
|
||||||
|
pollInterval time.Duration
|
||||||
|
notify chan struct{}
|
||||||
|
|
||||||
|
mu sync.Mutex
|
||||||
|
processed map[string]uint64
|
||||||
|
}
|
||||||
|
|
||||||
|
var _ provider.ResultObserver = (*SequentialCoordinator)(nil)
|
||||||
|
|
||||||
|
func NewSequentialCoordinator(
|
||||||
|
configuration ConfigurationSource,
|
||||||
|
state StateStore,
|
||||||
|
stats provider.StatsReader,
|
||||||
|
refresh SnapshotRefreshNotifier,
|
||||||
|
options Options,
|
||||||
|
) (*SequentialCoordinator, error) {
|
||||||
|
if nilInterface(configuration) || nilInterface(state) || nilInterface(stats) || options.Now == nil {
|
||||||
|
return nil, ErrInvalidSequentialCoordinator
|
||||||
|
}
|
||||||
|
if options.PollInterval == 0 {
|
||||||
|
options.PollInterval = defaultPollInterval
|
||||||
|
}
|
||||||
|
if options.PollInterval <= 0 {
|
||||||
|
return nil, ErrInvalidSequentialCoordinator
|
||||||
|
}
|
||||||
|
return &SequentialCoordinator{
|
||||||
|
configuration: configuration,
|
||||||
|
state: state,
|
||||||
|
stats: stats,
|
||||||
|
refresh: refresh,
|
||||||
|
now: options.Now,
|
||||||
|
pollInterval: options.PollInterval,
|
||||||
|
notify: make(chan struct{}, 1),
|
||||||
|
processed: make(map[string]uint64),
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// ObserveProviderResult deliberately performs no store calls. Every validated
|
||||||
|
// result can affect an empty episode, but one buffered notification is enough
|
||||||
|
// because Tick reads the latest bounded Stats snapshot.
|
||||||
|
func (coordinator *SequentialCoordinator) ObserveProviderResult(result provider.Result) {
|
||||||
|
if coordinator == nil || !validResultClass(result.Class) || result.UpstreamID == "" {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
select {
|
||||||
|
case coordinator.notify <- struct{}{}:
|
||||||
|
default:
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (coordinator *SequentialCoordinator) Run(ctx context.Context) error {
|
||||||
|
if coordinator == nil || ctx == nil || coordinator.pollInterval <= 0 || coordinator.now == nil || coordinator.notify == nil {
|
||||||
|
return ErrInvalidSequentialCoordinator
|
||||||
|
}
|
||||||
|
ticker := time.NewTicker(coordinator.pollInterval)
|
||||||
|
defer ticker.Stop()
|
||||||
|
for {
|
||||||
|
// A temporary management-store failure must not terminate Provider
|
||||||
|
// reconciliation. The retained empty generation is evaluated on retry.
|
||||||
|
_, _ = coordinator.Tick(ctx)
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return ctx.Err()
|
||||||
|
case <-coordinator.notify:
|
||||||
|
case <-ticker.C:
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (coordinator *SequentialCoordinator) Tick(ctx context.Context) (TickResult, error) {
|
||||||
|
if coordinator == nil || ctx == nil || nilInterface(coordinator.configuration) || nilInterface(coordinator.state) ||
|
||||||
|
nilInterface(coordinator.stats) || coordinator.now == nil {
|
||||||
|
return TickResult{}, ErrInvalidSequentialCoordinator
|
||||||
|
}
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return TickResult{}, err
|
||||||
|
}
|
||||||
|
coordinator.mu.Lock()
|
||||||
|
defer coordinator.mu.Unlock()
|
||||||
|
|
||||||
|
configuration, revision := coordinator.configuration.Snapshot()
|
||||||
|
if configuration == nil || revision == 0 {
|
||||||
|
return TickResult{}, ErrInvalidSequentialCoordinator
|
||||||
|
}
|
||||||
|
snapshot, err := coordinator.state.Snapshot(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return TickResult{}, err
|
||||||
|
}
|
||||||
|
if snapshot.Config == nil || snapshot.Config.Revision != revision {
|
||||||
|
return TickResult{}, ErrInvalidSequentialCoordinator
|
||||||
|
}
|
||||||
|
|
||||||
|
upstreams := upstreamStates(snapshot.Upstreams)
|
||||||
|
routings := routingStates(snapshot.Routings)
|
||||||
|
result := TickResult{}
|
||||||
|
for _, route := range configuration.Routing {
|
||||||
|
if !route.Enabled || route.Strategy.Type != "sequential" || route.Strategy.SwitchAfterEmptyFetch <= 0 {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
state, exists := routings[route.Name]
|
||||||
|
if !exists || !state.Enabled || !upstreamEnabled(configuration, upstreams, state.CurrentUpstream) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
read := coordinator.stats.ReadProviderStats([]string{state.CurrentUpstream})
|
||||||
|
if len(read) != 1 {
|
||||||
|
return result, ErrInvalidSequentialCoordinator
|
||||||
|
}
|
||||||
|
stats := read[0]
|
||||||
|
if stats.UpstreamID != state.CurrentUpstream || stats.ConsecutiveEmptyFetch < int64(route.Strategy.SwitchAfterEmptyFetch) ||
|
||||||
|
stats.EmptyGeneration == 0 {
|
||||||
|
coordinator.clearProcessed(route.Name, state.CurrentUpstream)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
key := processedKey(route.Name, state.CurrentUpstream)
|
||||||
|
if coordinator.processed[key] == stats.EmptyGeneration {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
target, switchable := nextUpstream(route, state.CurrentUpstream, upstreams, configuration)
|
||||||
|
if !switchable {
|
||||||
|
coordinator.processed[key] = stats.EmptyGeneration
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
mutation, mutateErr := coordinator.state.SwitchRouting(ctx, adminstate.SwitchRoutingCommand{
|
||||||
|
RequestID: requestID(route.Name, state.CurrentUpstream, stats.EmptyGeneration),
|
||||||
|
Actor: adminstate.Actor{ID: autoSwitchActor}, OccurredAt: coordinator.now().UTC(),
|
||||||
|
Name: route.Name, ExpectedCurrent: state.CurrentUpstream, Target: target, Reason: autoSwitchReason,
|
||||||
|
})
|
||||||
|
if mutateErr != nil {
|
||||||
|
if errors.Is(mutateErr, adminstate.ErrConflict) {
|
||||||
|
coordinator.processed[key] = stats.EmptyGeneration
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
return result, mutateErr
|
||||||
|
}
|
||||||
|
coordinator.processed[key] = stats.EmptyGeneration
|
||||||
|
if mutation.Changed {
|
||||||
|
result.Switched++
|
||||||
|
if coordinator.refresh != nil {
|
||||||
|
coordinator.refresh.NotifySnapshotRefresh()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return result, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func nextUpstream(
|
||||||
|
route config.Routing,
|
||||||
|
current string,
|
||||||
|
states map[string]adminstate.UpstreamState,
|
||||||
|
configuration *config.Config,
|
||||||
|
) (string, bool) {
|
||||||
|
eligible := make([]string, 0, len(route.Upstreams))
|
||||||
|
currentIndex := -1
|
||||||
|
for _, upstream := range route.Upstreams {
|
||||||
|
if !upstreamEnabled(configuration, states, upstream) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if upstream == current {
|
||||||
|
currentIndex = len(eligible)
|
||||||
|
}
|
||||||
|
eligible = append(eligible, upstream)
|
||||||
|
}
|
||||||
|
if currentIndex < 0 || len(eligible) < 2 {
|
||||||
|
return "", false
|
||||||
|
}
|
||||||
|
if currentIndex+1 < len(eligible) {
|
||||||
|
return eligible[currentIndex+1], true
|
||||||
|
}
|
||||||
|
switch route.Strategy.EndBehavior {
|
||||||
|
case "loop":
|
||||||
|
return eligible[0], eligible[0] != current
|
||||||
|
case "stayLast", "", "stop":
|
||||||
|
return "", false
|
||||||
|
default:
|
||||||
|
return "", false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func upstreamStates(values []adminstate.UpstreamState) map[string]adminstate.UpstreamState {
|
||||||
|
result := make(map[string]adminstate.UpstreamState, len(values))
|
||||||
|
for _, value := range values {
|
||||||
|
result[value.Name] = value
|
||||||
|
}
|
||||||
|
return result
|
||||||
|
}
|
||||||
|
|
||||||
|
func routingStates(values []adminstate.RoutingState) map[string]adminstate.RoutingState {
|
||||||
|
result := make(map[string]adminstate.RoutingState, len(values))
|
||||||
|
for _, value := range values {
|
||||||
|
result[value.Name] = value
|
||||||
|
}
|
||||||
|
return result
|
||||||
|
}
|
||||||
|
|
||||||
|
func upstreamEnabled(configuration *config.Config, states map[string]adminstate.UpstreamState, name string) bool {
|
||||||
|
configured, exists := configuration.Upstreams[name]
|
||||||
|
if !exists || !configured.Enabled {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
state, exists := states[name]
|
||||||
|
return exists && state.Enabled
|
||||||
|
}
|
||||||
|
|
||||||
|
func (coordinator *SequentialCoordinator) clearProcessed(routeName, upstream string) {
|
||||||
|
delete(coordinator.processed, processedKey(routeName, upstream))
|
||||||
|
}
|
||||||
|
|
||||||
|
func processedKey(routeName, upstream string) string {
|
||||||
|
return routeName + "\x00" + upstream
|
||||||
|
}
|
||||||
|
|
||||||
|
func requestID(routeName, upstream string, generation uint64) string {
|
||||||
|
payload := routeName + "\x00" + upstream + "\x00" + strconv.FormatUint(generation, 10)
|
||||||
|
digest := sha256.Sum256([]byte(payload))
|
||||||
|
return "auto-sequential-" + hex.EncodeToString(digest[:16])
|
||||||
|
}
|
||||||
|
|
||||||
|
func validResultClass(class upstream.FetchClass) bool {
|
||||||
|
switch class {
|
||||||
|
case upstream.FetchValid, upstream.FetchEmpty, upstream.FetchDuplicateOnly, upstream.FetchError:
|
||||||
|
return true
|
||||||
|
default:
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func nilInterface(value any) bool {
|
||||||
|
if value == nil {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
reflected := reflect.ValueOf(value)
|
||||||
|
switch reflected.Kind() {
|
||||||
|
case reflect.Chan, reflect.Func, reflect.Interface, reflect.Map, reflect.Pointer, reflect.Slice:
|
||||||
|
return reflected.IsNil()
|
||||||
|
default:
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
}
|
||||||
274
internal/controller/routing/sequential_test.go
Normal file
274
internal/controller/routing/sequential_test.go
Normal file
@ -0,0 +1,274 @@
|
|||||||
|
package routing
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"sync"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"proxy-pool/internal/config"
|
||||||
|
"proxy-pool/internal/controller/provider"
|
||||||
|
"proxy-pool/internal/domain/adminstate"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestSequentialCoordinatorSwitchesOnlyAtConfiguredThreshold(t *testing.T) {
|
||||||
|
state := newRoutingState("provider-a", map[string]bool{"provider-a": true, "provider-b": true})
|
||||||
|
stats := &routingStats{byUpstream: map[string]provider.Stats{
|
||||||
|
"provider-a": {UpstreamID: "provider-a", ConsecutiveEmptyFetch: 4, EmptyGeneration: 1},
|
||||||
|
}}
|
||||||
|
refresh := &refreshRecorder{}
|
||||||
|
coordinator := newCoordinator(t, routingConfiguration("stop", []string{"provider-a", "provider-b"}), state, stats, refresh)
|
||||||
|
|
||||||
|
if result, err := coordinator.Tick(context.Background()); err != nil || result.Switched != 0 || state.current() != "provider-a" {
|
||||||
|
t.Fatalf("Tick(before threshold) = (%+v, %v), current=%q", result, err, state.current())
|
||||||
|
}
|
||||||
|
stats.set("provider-a", provider.Stats{UpstreamID: "provider-a", ConsecutiveEmptyFetch: 5, EmptyGeneration: 1})
|
||||||
|
if result, err := coordinator.Tick(context.Background()); err != nil || result.Switched != 1 || state.current() != "provider-b" {
|
||||||
|
t.Fatalf("Tick(at threshold) = (%+v, %v), current=%q", result, err, state.current())
|
||||||
|
}
|
||||||
|
if refresh.count != 1 || state.switches != 1 {
|
||||||
|
t.Fatalf("refresh=%d switches=%d, want 1", refresh.count, state.switches)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSequentialCoordinatorConsumesStatsRecorderNotifications(t *testing.T) {
|
||||||
|
state := newRoutingState("provider-a", map[string]bool{"provider-a": true, "provider-b": true})
|
||||||
|
stats, err := provider.NewStatsRecorder(2)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("NewStatsRecorder() = %v", err)
|
||||||
|
}
|
||||||
|
refresh := &refreshRecorder{}
|
||||||
|
coordinator, err := NewSequentialCoordinator(
|
||||||
|
staticConfiguration{configuration: routingConfiguration("stop", []string{"provider-a", "provider-b"}), revision: 7},
|
||||||
|
state,
|
||||||
|
stats,
|
||||||
|
refresh,
|
||||||
|
Options{Now: time.Now},
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("NewSequentialCoordinator() = %v", err)
|
||||||
|
}
|
||||||
|
stats.AddResultObserver(coordinator)
|
||||||
|
for range 5 {
|
||||||
|
stats.Record(provider.Result{UpstreamID: "provider-a", Class: "empty"})
|
||||||
|
}
|
||||||
|
select {
|
||||||
|
case <-coordinator.notify:
|
||||||
|
case <-time.After(time.Second):
|
||||||
|
t.Fatal("StatsRecorder did not notify SequentialCoordinator")
|
||||||
|
}
|
||||||
|
if result, err := coordinator.Tick(context.Background()); err != nil || result.Switched != 1 || state.current() != "provider-b" || refresh.count != 1 {
|
||||||
|
t.Fatalf("Tick() = (%+v, %v), current=%q refresh=%d", result, err, state.current(), refresh.count)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSequentialCoordinatorSkipsDisabledCandidatesAndStaysAtConfiguredEnd(t *testing.T) {
|
||||||
|
state := newRoutingState("provider-a", map[string]bool{
|
||||||
|
"provider-a": true, "provider-b": false, "provider-c": true,
|
||||||
|
})
|
||||||
|
stats := &routingStats{byUpstream: map[string]provider.Stats{
|
||||||
|
"provider-a": {UpstreamID: "provider-a", ConsecutiveEmptyFetch: 5, EmptyGeneration: 1},
|
||||||
|
"provider-c": {UpstreamID: "provider-c", ConsecutiveEmptyFetch: 5, EmptyGeneration: 2},
|
||||||
|
}}
|
||||||
|
coordinator := newCoordinator(t, routingConfiguration("stayLast", []string{"provider-a", "provider-b", "provider-c"}), state, stats, &refreshRecorder{})
|
||||||
|
|
||||||
|
if result, err := coordinator.Tick(context.Background()); err != nil || result.Switched != 1 || state.current() != "provider-c" {
|
||||||
|
t.Fatalf("Tick(skip disabled) = (%+v, %v), current=%q", result, err, state.current())
|
||||||
|
}
|
||||||
|
if result, err := coordinator.Tick(context.Background()); err != nil || result.Switched != 0 || state.current() != "provider-c" {
|
||||||
|
t.Fatalf("Tick(stay last) = (%+v, %v), current=%q", result, err, state.current())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSequentialCoordinatorDoesNotReuseAnEmptyGenerationAfterLoop(t *testing.T) {
|
||||||
|
state := newRoutingState("provider-a", map[string]bool{"provider-a": true, "provider-b": true})
|
||||||
|
stats := &routingStats{byUpstream: map[string]provider.Stats{
|
||||||
|
"provider-a": {UpstreamID: "provider-a", ConsecutiveEmptyFetch: 5, EmptyGeneration: 1},
|
||||||
|
"provider-b": {UpstreamID: "provider-b", ConsecutiveEmptyFetch: 5, EmptyGeneration: 2},
|
||||||
|
}}
|
||||||
|
coordinator := newCoordinator(t, routingConfiguration("loop", []string{"provider-a", "provider-b"}), state, stats, &refreshRecorder{})
|
||||||
|
|
||||||
|
for _, want := range []string{"provider-b", "provider-a"} {
|
||||||
|
if result, err := coordinator.Tick(context.Background()); err != nil || result.Switched != 1 || state.current() != want {
|
||||||
|
t.Fatalf("Tick() = (%+v, %v), current=%q, want %q", result, err, state.current(), want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if result, err := coordinator.Tick(context.Background()); err != nil || result.Switched != 0 || state.current() != "provider-a" {
|
||||||
|
t.Fatalf("Tick(reused generation) = (%+v, %v), current=%q", result, err, state.current())
|
||||||
|
}
|
||||||
|
stats.set("provider-a", provider.Stats{UpstreamID: "provider-a", ConsecutiveEmptyFetch: 5, EmptyGeneration: 3})
|
||||||
|
if result, err := coordinator.Tick(context.Background()); err != nil || result.Switched != 1 || state.current() != "provider-b" {
|
||||||
|
t.Fatalf("Tick(new generation) = (%+v, %v), current=%q", result, err, state.current())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSequentialCoordinatorUsesCompareAndSwapUnderConcurrentTicks(t *testing.T) {
|
||||||
|
state := newRoutingState("provider-a", map[string]bool{"provider-a": true, "provider-b": true})
|
||||||
|
stats := &routingStats{byUpstream: map[string]provider.Stats{
|
||||||
|
"provider-a": {UpstreamID: "provider-a", ConsecutiveEmptyFetch: 5, EmptyGeneration: 1},
|
||||||
|
}}
|
||||||
|
coordinator := newCoordinator(t, routingConfiguration("stop", []string{"provider-a", "provider-b"}), state, stats, &refreshRecorder{})
|
||||||
|
|
||||||
|
var wait sync.WaitGroup
|
||||||
|
for range 32 {
|
||||||
|
wait.Add(1)
|
||||||
|
go func() {
|
||||||
|
defer wait.Done()
|
||||||
|
if _, err := coordinator.Tick(context.Background()); err != nil {
|
||||||
|
t.Errorf("Tick() = %v", err)
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
}
|
||||||
|
wait.Wait()
|
||||||
|
if state.switches != 1 || state.current() != "provider-b" {
|
||||||
|
t.Fatalf("switches=%d current=%q, want one switch to provider-b", state.switches, state.current())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSequentialCoordinatorFailsClosedWhenConfigurationRevisionIsStale(t *testing.T) {
|
||||||
|
state := newRoutingState("provider-a", map[string]bool{"provider-a": true, "provider-b": true})
|
||||||
|
state.snapshot.Config.Revision = 8
|
||||||
|
stats := &routingStats{byUpstream: map[string]provider.Stats{
|
||||||
|
"provider-a": {UpstreamID: "provider-a", ConsecutiveEmptyFetch: 5, EmptyGeneration: 1},
|
||||||
|
}}
|
||||||
|
coordinator := newCoordinator(t, routingConfiguration("stop", []string{"provider-a", "provider-b"}), state, stats, &refreshRecorder{})
|
||||||
|
if _, err := coordinator.Tick(context.Background()); err == nil || state.current() != "provider-a" {
|
||||||
|
t.Fatalf("Tick(stale revision) error=%v current=%q, want error and unchanged state", err, state.current())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSequentialCoordinatorRejectsMalformedStatsAndCoalescesNotifications(t *testing.T) {
|
||||||
|
state := newRoutingState("provider-a", map[string]bool{"provider-a": true, "provider-b": true})
|
||||||
|
configuration := staticConfiguration{configuration: routingConfiguration("stop", []string{"provider-a", "provider-b"}), revision: 7}
|
||||||
|
coordinator, err := NewSequentialCoordinator(configuration, state, emptyStats{}, &refreshRecorder{}, Options{Now: time.Now})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("NewSequentialCoordinator() = %v", err)
|
||||||
|
}
|
||||||
|
if _, err := coordinator.Tick(context.Background()); err == nil {
|
||||||
|
t.Fatal("Tick() with malformed StatsReader response error = nil")
|
||||||
|
}
|
||||||
|
for range 4 {
|
||||||
|
coordinator.ObserveProviderResult(provider.Result{UpstreamID: "provider-a", Class: "empty"})
|
||||||
|
}
|
||||||
|
select {
|
||||||
|
case <-coordinator.notify:
|
||||||
|
default:
|
||||||
|
t.Fatal("ObserveProviderResult() did not notify")
|
||||||
|
}
|
||||||
|
select {
|
||||||
|
case <-coordinator.notify:
|
||||||
|
t.Fatal("ObserveProviderResult() did not coalesce notifications")
|
||||||
|
default:
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func newCoordinator(
|
||||||
|
t *testing.T,
|
||||||
|
configuration *config.Config,
|
||||||
|
state *routingState,
|
||||||
|
stats *routingStats,
|
||||||
|
refresh *refreshRecorder,
|
||||||
|
) *SequentialCoordinator {
|
||||||
|
t.Helper()
|
||||||
|
coordinator, err := NewSequentialCoordinator(staticConfiguration{configuration: configuration, revision: 7}, state, stats, refresh, Options{
|
||||||
|
Now: func() time.Time { return time.Date(2026, 8, 2, 12, 0, 0, 0, time.UTC) },
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("NewSequentialCoordinator() = %v", err)
|
||||||
|
}
|
||||||
|
return coordinator
|
||||||
|
}
|
||||||
|
|
||||||
|
func routingConfiguration(endBehavior string, upstreams []string) *config.Config {
|
||||||
|
configured := make(map[string]config.Upstream, len(upstreams))
|
||||||
|
for _, upstream := range upstreams {
|
||||||
|
configured[upstream] = config.Upstream{Enabled: true}
|
||||||
|
}
|
||||||
|
return &config.Config{
|
||||||
|
Routing: []config.Routing{{
|
||||||
|
Name: "checkout", Enabled: true, Purpose: "gateway", Upstreams: append([]string(nil), upstreams...),
|
||||||
|
Strategy: config.Strategy{Type: "sequential", SwitchAfterEmptyFetch: 5, EndBehavior: endBehavior},
|
||||||
|
}},
|
||||||
|
Upstreams: configured,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
type staticConfiguration struct {
|
||||||
|
configuration *config.Config
|
||||||
|
revision uint64
|
||||||
|
}
|
||||||
|
|
||||||
|
func (source staticConfiguration) Snapshot() (*config.Config, uint64) {
|
||||||
|
return source.configuration, source.revision
|
||||||
|
}
|
||||||
|
|
||||||
|
type routingState struct {
|
||||||
|
mu sync.Mutex
|
||||||
|
snapshot adminstate.Snapshot
|
||||||
|
switches int
|
||||||
|
}
|
||||||
|
|
||||||
|
func newRoutingState(current string, enabled map[string]bool) *routingState {
|
||||||
|
upstreams := make([]adminstate.UpstreamState, 0, len(enabled))
|
||||||
|
for name, isEnabled := range enabled {
|
||||||
|
upstreams = append(upstreams, adminstate.UpstreamState{Name: name, Enabled: isEnabled})
|
||||||
|
}
|
||||||
|
return &routingState{snapshot: adminstate.Snapshot{
|
||||||
|
Config: &adminstate.ConfigRevision{Revision: 7}, Upstreams: upstreams,
|
||||||
|
Routings: []adminstate.RoutingState{{Name: "checkout", Enabled: true, CurrentUpstream: current}},
|
||||||
|
}}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (state *routingState) Snapshot(context.Context) (adminstate.Snapshot, error) {
|
||||||
|
state.mu.Lock()
|
||||||
|
defer state.mu.Unlock()
|
||||||
|
return state.snapshot, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (state *routingState) SwitchRouting(_ context.Context, command adminstate.SwitchRoutingCommand) (adminstate.MutationResult, error) {
|
||||||
|
state.mu.Lock()
|
||||||
|
defer state.mu.Unlock()
|
||||||
|
routing := &state.snapshot.Routings[0]
|
||||||
|
if command.Name != routing.Name || command.ExpectedCurrent != routing.CurrentUpstream {
|
||||||
|
return adminstate.MutationResult{}, adminstate.ErrConflict
|
||||||
|
}
|
||||||
|
routing.CurrentUpstream = command.Target
|
||||||
|
state.switches++
|
||||||
|
return adminstate.MutationResult{RequestID: command.RequestID, Changed: true, Revision: uint64(state.switches)}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (state *routingState) current() string {
|
||||||
|
state.mu.Lock()
|
||||||
|
defer state.mu.Unlock()
|
||||||
|
return state.snapshot.Routings[0].CurrentUpstream
|
||||||
|
}
|
||||||
|
|
||||||
|
type routingStats struct {
|
||||||
|
mu sync.Mutex
|
||||||
|
byUpstream map[string]provider.Stats
|
||||||
|
}
|
||||||
|
|
||||||
|
type emptyStats struct{}
|
||||||
|
|
||||||
|
func (emptyStats) ReadProviderStats([]string) []provider.Stats { return nil }
|
||||||
|
|
||||||
|
func (stats *routingStats) ReadProviderStats(upstreams []string) []provider.Stats {
|
||||||
|
stats.mu.Lock()
|
||||||
|
defer stats.mu.Unlock()
|
||||||
|
result := make([]provider.Stats, len(upstreams))
|
||||||
|
for index, upstream := range upstreams {
|
||||||
|
result[index] = stats.byUpstream[upstream]
|
||||||
|
result[index].UpstreamID = upstream
|
||||||
|
}
|
||||||
|
return result
|
||||||
|
}
|
||||||
|
|
||||||
|
func (stats *routingStats) set(upstream string, value provider.Stats) {
|
||||||
|
stats.mu.Lock()
|
||||||
|
defer stats.mu.Unlock()
|
||||||
|
stats.byUpstream[upstream] = value
|
||||||
|
}
|
||||||
|
|
||||||
|
type refreshRecorder struct{ count int }
|
||||||
|
|
||||||
|
func (recorder *refreshRecorder) NotifySnapshotRefresh() { recorder.count++ }
|
||||||
Loading…
Reference in New Issue
Block a user