proxy-pool/internal/controller/health/scheduler_runner_test.go
youfak 6f5a2faea6
Some checks are pending
ci / proto (push) Waiting to run
ci / test (ubuntu-latest) (push) Waiting to run
ci / test (windows-latest) (push) Waiting to run
ci / race (push) Waiting to run
ci / integration (push) Waiting to run
fix: materialize egress checks before basic tasks
2026-08-02 10:39:20 +08:00

438 lines
18 KiB
Go

package health
import (
"context"
"errors"
"testing"
"time"
"proxy-pool/internal/config"
healthDomain "proxy-pool/internal/domain/health"
proxyDomain "proxy-pool/internal/domain/proxy"
)
func TestSchedulerRunnerUsesBoundedDueBatchAndSharedInFlightCount(t *testing.T) {
planner, err := NewPlanner(SchedulePolicy{
Interval: time.Minute, MaxInFlight: 3, Timeout: time.Second, MaxAttempts: 1,
})
if err != nil {
t.Fatalf("NewPlanner(): %v", err)
}
now := time.Date(2026, 7, 31, 18, 0, 0, 0, time.UTC)
source := &dueSourceStub{inFlight: 1, candidates: []Candidate{
{ProxyID: "fetched", State: proxyDomain.StateFetched, Level: healthDomain.LevelBasic, DueAt: now},
{ProxyID: "available", State: proxyDomain.StateAvailable, Level: healthDomain.LevelBasic, DueAt: now},
}}
sink := &taskSinkStub{}
runner, err := NewSchedulerRunner(planner, source, sink, SchedulerRunnerOptions{
PollInterval: time.Second, BatchSize: 2, Now: func() time.Time { return now },
})
if err != nil {
t.Fatalf("NewSchedulerRunner(): %v", err)
}
result, err := runner.Tick(context.Background())
if err != nil || result.Planned != 2 || result.Offered != 2 || source.limit != 2 || len(sink.tasks) != 2 ||
sink.tasks[0].Priority != PriorityFetched {
t.Fatalf("Tick() = (%+v, %v); source=%+v sink=%+v", result, err, source, sink.tasks)
}
}
func TestSchedulerRunnerSkipsDueReadAtSharedCapacity(t *testing.T) {
planner, err := NewPlanner(SchedulePolicy{
Interval: time.Minute, MaxInFlight: 1, Timeout: time.Second, MaxAttempts: 1,
})
if err != nil {
t.Fatalf("NewPlanner(): %v", err)
}
source := &dueSourceStub{inFlight: 1}
runner, err := NewSchedulerRunner(planner, source, &taskSinkStub{}, SchedulerRunnerOptions{
PollInterval: time.Second, BatchSize: 1, Now: time.Now,
})
if err != nil {
t.Fatalf("NewSchedulerRunner(): %v", err)
}
result, err := runner.Tick(context.Background())
if err != nil || result != (TickResult{}) || source.limit != 0 {
t.Fatalf("Tick(at capacity) = (%+v, %v); source=%+v", result, err, source)
}
}
func TestSchedulerRunnerRejectsOversizedSourceAndSinkResponses(t *testing.T) {
planner, err := NewPlanner(SchedulePolicy{
Interval: time.Minute, MaxInFlight: 2, Timeout: time.Second, MaxAttempts: 1,
})
if err != nil {
t.Fatalf("NewPlanner(): %v", err)
}
now := time.Now()
runner, err := NewSchedulerRunner(planner, &dueSourceStub{candidates: []Candidate{
{ProxyID: "one", State: proxyDomain.StateFetched, Level: healthDomain.LevelBasic, DueAt: now},
{ProxyID: "two", State: proxyDomain.StateFetched, Level: healthDomain.LevelBasic, DueAt: now},
}}, &taskSinkStub{}, SchedulerRunnerOptions{PollInterval: time.Second, BatchSize: 1, Now: func() time.Time { return now }})
if err != nil {
t.Fatalf("NewSchedulerRunner(): %v", err)
}
if _, err := runner.Tick(context.Background()); !errors.Is(err, ErrInvalidDueSource) {
t.Fatalf("Tick(oversized source) error = %v, want ErrInvalidDueSource", err)
}
}
func TestConfiguredSchedulerRunnerFollowsUpstreamConfigChanges(t *testing.T) {
now := time.Date(2026, 8, 2, 10, 0, 0, 0, time.UTC)
configuration := &config.Config{
Defaults: config.Defaults{Check: config.Check{
Interval: config.Duration(time.Minute),
MaxInFlight: 2,
Timeout: config.Duration(time.Second),
MaxAttempts: 1,
}},
Upstreams: map[string]config.Upstream{
"provider-a": {Enabled: true},
},
}
source := &dueSourceStub{candidates: []Candidate{{
ProxyID: "proxy-a", State: proxyDomain.StateFetched, Level: healthDomain.LevelBasic, DueAt: now,
}}}
sink := &taskSinkStub{}
runner, err := NewConfiguredSchedulerRunner(&configurationSourceStub{configuration: configuration}, "provider-a", source, sink,
SchedulerRunnerOptions{PollInterval: time.Second, BatchSize: 16, Now: func() time.Time { return now }})
if err != nil {
t.Fatalf("NewConfiguredSchedulerRunner(): %v", err)
}
result, err := runner.Tick(context.Background())
if err != nil || result != (TickResult{Planned: 1, Offered: 1}) || len(sink.tasks) != 1 || sink.tasks[0].Attempts != 1 ||
source.limit != 2 {
t.Fatalf("initial Tick() = (%+v, %v); source=%+v sink=%+v", result, err, source, sink.tasks)
}
updated := configuration.Upstreams["provider-a"]
updated.Check.MaxAttempts = 3
configuration.Upstreams["provider-a"] = updated
result, err = runner.Tick(context.Background())
if err != nil || result != (TickResult{Planned: 1, Offered: 1}) || len(sink.tasks) != 1 || sink.tasks[0].Attempts != 3 {
t.Fatalf("updated Tick() = (%+v, %v); sink=%+v", result, err, sink.tasks)
}
updated.Enabled = false
configuration.Upstreams["provider-a"] = updated
inFlightCalls, dueCalls := source.inFlightCalls, source.dueCalls
result, err = runner.Tick(context.Background())
if err != nil || result != (TickResult{}) || source.inFlightCalls != inFlightCalls || source.dueCalls != dueCalls {
t.Fatalf("disabled Tick() = (%+v, %v); source=%+v", result, err, source)
}
}
func TestConfiguredSchedulerSupervisorDiscoversNewEnabledUpstream(t *testing.T) {
now := time.Date(2026, 8, 2, 10, 0, 0, 0, time.UTC)
configuration := &config.Config{
Defaults: config.Defaults{Check: config.Check{
Interval: config.Duration(time.Minute),
MaxInFlight: 2,
Timeout: config.Duration(time.Second),
MaxAttempts: 1,
}},
Upstreams: map[string]config.Upstream{
"provider-a": {Enabled: true},
},
}
source := &upstreamTaskSourceStub{sources: map[string]*dueSourceStub{
"provider-a": {candidates: []Candidate{{
ProxyID: "proxy-a", State: proxyDomain.StateFetched, Level: healthDomain.LevelBasic, DueAt: now,
}}},
"provider-b": {candidates: []Candidate{{
ProxyID: "proxy-b", State: proxyDomain.StateFetched, Level: healthDomain.LevelBasic, DueAt: now,
}}},
}}
sink := &taskSinkStub{}
supervisor, err := NewConfiguredSchedulerSupervisor(&configurationSourceStub{configuration: configuration}, source, sink,
SchedulerRunnerOptions{PollInterval: time.Second, BatchSize: 16, Now: func() time.Time { return now }})
if err != nil {
t.Fatalf("NewConfiguredSchedulerSupervisor(): %v", err)
}
if result, err := supervisor.Tick(context.Background()); err != nil || result != (TickResult{Planned: 1, Offered: 1}) ||
source.sources["provider-a"].dueCalls != 1 || source.sources["provider-b"].dueCalls != 0 {
t.Fatalf("initial Tick() = (%+v, %v); source=%+v", result, err, source)
}
configuration.Upstreams["provider-a"] = config.Upstream{Enabled: false}
configuration.Upstreams["provider-b"] = config.Upstream{Enabled: true, Check: config.Check{MaxAttempts: 3, MaxInFlight: 1}}
result, err := supervisor.Tick(context.Background())
if err != nil || result != (TickResult{Planned: 1, Offered: 1}) || source.sources["provider-b"].dueCalls != 1 ||
len(sink.tasks) != 1 || sink.tasks[0].Candidate.ProxyID != "proxy-b" || sink.tasks[0].Attempts != 3 || source.sources["provider-b"].limit != 1 {
t.Fatalf("updated Tick() = (%+v, %v); source=%+v sink=%+v", result, err, source, sink.tasks)
}
}
func TestConfiguredSchedulerSupervisorSchedulesBoundedEgressGroups(t *testing.T) {
now := time.Date(2026, 8, 2, 11, 0, 0, 0, time.UTC)
configuration := &config.Config{
Defaults: config.Defaults{Check: config.Check{
Interval: config.Duration(time.Minute), MaxInFlight: 6, Timeout: config.Duration(time.Second), MaxAttempts: 1,
URLs: []string{"https://egress-one.example/identity", "https://egress-two.example/identity"},
}},
Upstreams: map[string]config.Upstream{"provider-a": {Enabled: true}},
}
source := &upstreamTaskSourceStub{sources: map[string]*dueSourceStub{
"provider-a": {candidates: []Candidate{{
ProxyID: "basic", State: proxyDomain.StateFetched, Level: healthDomain.LevelBasic, DueAt: now,
}}, egressCandidates: map[string][]Candidate{
"https://egress-one.example/identity": {{ProxyID: "egress-one", State: proxyDomain.StateFetched, Level: healthDomain.LevelEgress, TargetURL: "https://egress-one.example/identity", DueAt: now}},
"https://egress-two.example/identity": {{ProxyID: "egress-two", State: proxyDomain.StateFetched, Level: healthDomain.LevelEgress, TargetURL: "https://egress-two.example/identity", DueAt: now}},
}},
}}
sink := &taskSinkStub{}
supervisor, err := NewConfiguredSchedulerSupervisor(&configurationSourceStub{configuration: configuration}, source, sink,
SchedulerRunnerOptions{PollInterval: time.Second, BatchSize: 6, Now: func() time.Time { return now }})
if err != nil {
t.Fatalf("NewConfiguredSchedulerSupervisor() = %v", err)
}
result, err := supervisor.Tick(context.Background())
if err != nil || result != (TickResult{Planned: 3, Offered: 3}) || !sink.offeredLevel(healthDomain.LevelEgress) ||
source.sources["provider-a"].egressCalls != 2 {
t.Fatalf("Tick() = (%+v, %v); source=%+v sink=%+v", result, err, source, sink.batches)
}
}
func TestConfiguredSchedulerSupervisorMaterializesEveryEgressGroupBeforeBasic(t *testing.T) {
now := time.Date(2026, 8, 2, 11, 15, 0, 0, time.UTC)
firstURL := "https://egress-one.example/identity"
secondURL := "https://egress-two.example/identity"
configuration := &config.Config{
Defaults: config.Defaults{Check: config.Check{
Interval: config.Duration(time.Minute), MaxInFlight: 1, Timeout: config.Duration(time.Second), MaxAttempts: 1,
URLs: []string{firstURL, secondURL},
}},
Upstreams: map[string]config.Upstream{"provider-a": {Enabled: true}},
}
source := &upstreamTaskSourceStub{sources: map[string]*dueSourceStub{
"provider-a": {candidates: []Candidate{{
ProxyID: "basic", State: proxyDomain.StateFetched, Level: healthDomain.LevelBasic, DueAt: now,
}}, egressCandidates: map[string][]Candidate{
firstURL: {{ProxyID: "egress-one", State: proxyDomain.StateFetched, Level: healthDomain.LevelEgress, TargetURL: firstURL, DueAt: now}},
secondURL: {{ProxyID: "egress-two", State: proxyDomain.StateFetched, Level: healthDomain.LevelEgress, TargetURL: secondURL, DueAt: now}},
}},
}}
sink := &taskSinkStub{}
supervisor, err := NewConfiguredSchedulerSupervisor(&configurationSourceStub{configuration: configuration}, source, sink,
SchedulerRunnerOptions{PollInterval: time.Second, BatchSize: 1, Now: func() time.Time { return now }})
if err != nil {
t.Fatalf("NewConfiguredSchedulerSupervisor() = %v", err)
}
for tick := 0; tick < 2; tick++ {
if result, err := supervisor.Tick(context.Background()); err != nil || result != (TickResult{Planned: 1, Offered: 1}) {
t.Fatalf("Tick(%d) = (%+v, %v)", tick, result, err)
}
}
item := source.sources["provider-a"]
if item.egressCalls != 2 || item.dueCalls != 0 || len(sink.batches) != 2 ||
sink.batches[0][0].Candidate.Level != healthDomain.LevelEgress || sink.batches[1][0].Candidate.Level != healthDomain.LevelEgress {
t.Fatalf("first two ticks did not establish egress groups: source=%+v batches=%+v", item, sink.batches)
}
if result, err := supervisor.Tick(context.Background()); err != nil || result != (TickResult{Planned: 1, Offered: 1}) ||
item.dueCalls != 1 || len(sink.batches) != 3 || sink.batches[2][0].Candidate.Level != healthDomain.LevelBasic {
t.Fatalf("basic Tick() = (%+v, %v), source=%+v batches=%+v", result, err, item, sink.batches)
}
}
func TestConfiguredSchedulerSupervisorSchedulesRoutingTargetProfiles(t *testing.T) {
now := time.Date(2026, 8, 2, 11, 30, 0, 0, time.UTC)
const routingName = "checkout"
const targetURL = "https://checkout.example/health"
configuration := &config.Config{
Defaults: config.Defaults{Check: config.Check{
Interval: config.Duration(time.Minute), MaxInFlight: 4, Timeout: config.Duration(time.Second), MaxAttempts: 1,
}},
Routing: []config.Routing{{
Name: routingName, Enabled: true, Upstreams: []string{"provider-a"},
Check: config.RoutingCheck{Targets: []string{targetURL}},
}},
Upstreams: map[string]config.Upstream{"provider-a": {Enabled: true}},
}
source := &upstreamTaskSourceStub{sources: map[string]*dueSourceStub{
"provider-a": {candidates: []Candidate{{
ProxyID: "basic", State: proxyDomain.StateFetched, Level: healthDomain.LevelBasic, DueAt: now,
}}, targetCandidates: map[string][]Candidate{
targetCandidateKey(routingName, targetURL): {{
ProxyID: "target", State: proxyDomain.StateFetched, Level: healthDomain.LevelTarget,
RoutingName: routingName, TargetURL: targetURL, DueAt: now,
}},
}},
}}
sink := &taskSinkStub{}
supervisor, err := NewConfiguredSchedulerSupervisor(&configurationSourceStub{configuration: configuration}, source, sink,
SchedulerRunnerOptions{PollInterval: time.Second, BatchSize: 4, Now: func() time.Time { return now }})
if err != nil {
t.Fatalf("NewConfiguredSchedulerSupervisor() = %v", err)
}
result, err := supervisor.Tick(context.Background())
if err != nil || result != (TickResult{Planned: 2, Offered: 2}) || !sink.offeredTarget(routingName, targetURL) ||
source.sources["provider-a"].targetCalls != 1 {
t.Fatalf("Tick() = (%+v, %v); source=%+v sink=%+v", result, err, source, sink.batches)
}
}
func TestConfiguredSchedulerSupervisorNeverExceedsTotalBatchAcrossGroups(t *testing.T) {
now := time.Date(2026, 8, 2, 12, 0, 0, 0, time.UTC)
configuration := &config.Config{
Defaults: config.Defaults{Check: config.Check{
Interval: config.Duration(time.Minute), MaxInFlight: 10, Timeout: config.Duration(time.Second), MaxAttempts: 1,
URLs: []string{"https://egress.example/identity"},
}},
Routing: []config.Routing{{
Name: "checkout", Enabled: true, Upstreams: []string{"provider-a"},
Check: config.RoutingCheck{Targets: []string{"https://checkout.example/health"}},
}},
Upstreams: map[string]config.Upstream{"provider-a": {Enabled: true}},
}
source := &upstreamTaskSourceStub{sources: map[string]*dueSourceStub{
"provider-a": {
candidates: []Candidate{
{ProxyID: "basic-a", State: proxyDomain.StateFetched, Level: healthDomain.LevelBasic, DueAt: now},
{ProxyID: "basic-b", State: proxyDomain.StateFetched, Level: healthDomain.LevelBasic, DueAt: now},
},
egressCandidates: map[string][]Candidate{"https://egress.example/identity": {
{ProxyID: "egress-a", State: proxyDomain.StateFetched, Level: healthDomain.LevelEgress, TargetURL: "https://egress.example/identity", DueAt: now},
}},
targetCandidates: map[string][]Candidate{targetCandidateKey("checkout", "https://checkout.example/health"): {
{ProxyID: "target-a", State: proxyDomain.StateFetched, Level: healthDomain.LevelTarget, RoutingName: "checkout", TargetURL: "https://checkout.example/health", DueAt: now},
}},
},
}}
supervisor, err := NewConfiguredSchedulerSupervisor(&configurationSourceStub{configuration: configuration}, source, &taskSinkStub{},
SchedulerRunnerOptions{PollInterval: time.Second, BatchSize: 4, Now: func() time.Time { return now }})
if err != nil {
t.Fatalf("NewConfiguredSchedulerSupervisor() = %v", err)
}
result, err := supervisor.Tick(context.Background())
if err != nil || result.Planned != 4 || result.Offered != 4 {
t.Fatalf("Tick() = (%+v, %v), want exactly four planned/offered tasks", result, err)
}
}
type dueSourceStub struct {
inFlight int
candidates []Candidate
limit int
inFlightCalls int
dueCalls int
egressCandidates map[string][]Candidate
egressCalls int
targetCandidates map[string][]Candidate
targetCalls int
}
func (source *dueSourceStub) InFlight(context.Context, time.Time) (int, error) {
source.inFlightCalls++
return source.inFlight, nil
}
func (source *dueSourceStub) DueCandidates(_ context.Context, _ time.Time, limit int) ([]Candidate, error) {
source.dueCalls++
source.limit = limit
return source.candidates, nil
}
func (source *dueSourceStub) DueEgressCandidates(_ context.Context, targetURL string, _ time.Time, limit int) ([]Candidate, error) {
source.egressCalls++
source.limit = limit
return source.egressCandidates[targetURL], nil
}
func (source *dueSourceStub) DueTargetCandidates(
_ context.Context,
routingName string,
targetURL string,
_ time.Time,
limit int,
) ([]Candidate, error) {
source.targetCalls++
source.limit = limit
return source.targetCandidates[targetCandidateKey(routingName, targetURL)], nil
}
type upstreamTaskSourceStub struct {
sources map[string]*dueSourceStub
}
func (source *upstreamTaskSourceStub) InFlightForUpstream(ctx context.Context, upstreamID string, now time.Time) (int, error) {
item, exists := source.sources[upstreamID]
if !exists {
return 0, errors.New("missing upstream source")
}
return item.InFlight(ctx, now)
}
func (source *upstreamTaskSourceStub) DueCandidatesForUpstream(ctx context.Context, upstreamID string, now time.Time, limit int) ([]Candidate, error) {
item, exists := source.sources[upstreamID]
if !exists {
return nil, errors.New("missing upstream source")
}
return item.DueCandidates(ctx, now, limit)
}
func (source *upstreamTaskSourceStub) DueEgressCandidatesForUpstream(ctx context.Context, upstreamID, targetURL string, now time.Time, limit int) ([]Candidate, error) {
item, exists := source.sources[upstreamID]
if !exists {
return nil, errors.New("missing upstream source")
}
return item.DueEgressCandidates(ctx, targetURL, now, limit)
}
func (source *upstreamTaskSourceStub) DueTargetCandidatesForUpstream(
ctx context.Context,
upstreamID string,
routingName string,
targetURL string,
now time.Time,
limit int,
) ([]Candidate, error) {
item, exists := source.sources[upstreamID]
if !exists {
return nil, errors.New("missing upstream source")
}
return item.DueTargetCandidates(ctx, routingName, targetURL, now, limit)
}
func targetCandidateKey(routingName, targetURL string) string {
return routingName + "\x00" + targetURL
}
type taskSinkStub struct {
tasks []PlannedTask
batches [][]PlannedTask
offer int
}
func (sink *taskSinkStub) Offer(_ context.Context, tasks []PlannedTask) (int, error) {
sink.tasks = append([]PlannedTask(nil), tasks...)
sink.batches = append(sink.batches, append([]PlannedTask(nil), tasks...))
if sink.offer != 0 {
return sink.offer, nil
}
return len(tasks), nil
}
func (sink *taskSinkStub) offeredLevel(level healthDomain.Level) bool {
for _, batch := range sink.batches {
for _, task := range batch {
if task.Candidate.Level == level {
return true
}
}
}
return false
}
func (sink *taskSinkStub) offeredTarget(routingName, targetURL string) bool {
for _, batch := range sink.batches {
for _, task := range batch {
if task.Candidate.Level == healthDomain.LevelTarget && task.Candidate.RoutingName == routingName &&
task.Candidate.TargetURL == targetURL {
return true
}
}
}
return false
}