feat: plan bounded health checks
This commit is contained in:
parent
65aee12d51
commit
0c3fe2c1a1
186
internal/controller/health/scheduler.go
Normal file
186
internal/controller/health/scheduler.go
Normal file
@ -0,0 +1,186 @@
|
|||||||
|
package health
|
||||||
|
|
||||||
|
import (
|
||||||
|
"crypto/sha256"
|
||||||
|
"encoding/binary"
|
||||||
|
"errors"
|
||||||
|
"sort"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
healthDomain "proxy-pool/internal/domain/health"
|
||||||
|
proxyDomain "proxy-pool/internal/domain/proxy"
|
||||||
|
)
|
||||||
|
|
||||||
|
var (
|
||||||
|
ErrInvalidSchedulePolicy = errors.New("invalid health schedule policy")
|
||||||
|
ErrInvalidScheduleRequest = errors.New("invalid health schedule request")
|
||||||
|
)
|
||||||
|
|
||||||
|
// SchedulePolicy is scoped to one effective upstream Check configuration.
|
||||||
|
// MaxInFlight is enforced by the eventual shared task store; Planner uses the
|
||||||
|
// observed count only to ensure one planning pass never exceeds that ceiling.
|
||||||
|
type SchedulePolicy struct {
|
||||||
|
Interval time.Duration
|
||||||
|
Jitter int
|
||||||
|
MaxInFlight int
|
||||||
|
Timeout time.Duration
|
||||||
|
MaxAttempts int
|
||||||
|
}
|
||||||
|
|
||||||
|
// Candidate is a bounded record supplied by a due-index query. The Planner
|
||||||
|
// never scans proxies and never starts a goroutine for a candidate.
|
||||||
|
type Candidate struct {
|
||||||
|
ProxyID string
|
||||||
|
State proxyDomain.State
|
||||||
|
Level healthDomain.Level
|
||||||
|
RoutingName string
|
||||||
|
TargetURL string
|
||||||
|
DueAt time.Time
|
||||||
|
}
|
||||||
|
|
||||||
|
type Priority uint8
|
||||||
|
|
||||||
|
const (
|
||||||
|
PriorityFetched Priority = iota + 1
|
||||||
|
PrioritySuspect
|
||||||
|
PriorityUnhealthy
|
||||||
|
PriorityAvailable
|
||||||
|
)
|
||||||
|
|
||||||
|
// PlannedTask is transport-neutral work ready for a leased broker to assign
|
||||||
|
// to a Checker. The task ID and proxy credential material are added only by
|
||||||
|
// the Controller's broker after it atomically claims the task.
|
||||||
|
type PlannedTask struct {
|
||||||
|
Candidate Candidate
|
||||||
|
Priority Priority
|
||||||
|
Deadline time.Time
|
||||||
|
Attempts int
|
||||||
|
}
|
||||||
|
|
||||||
|
// Planner is stateless and safe for concurrent callers. Its lack of internal
|
||||||
|
// queues makes maxInFlight and batch bounds explicit at the storage boundary.
|
||||||
|
type Planner struct {
|
||||||
|
policy SchedulePolicy
|
||||||
|
}
|
||||||
|
|
||||||
|
func NewPlanner(policy SchedulePolicy) (*Planner, error) {
|
||||||
|
if policy.Interval <= 0 || policy.Jitter < 0 || policy.Jitter > 100 || policy.MaxInFlight <= 0 ||
|
||||||
|
policy.Timeout <= 0 || policy.MaxAttempts <= 0 {
|
||||||
|
return nil, ErrInvalidSchedulePolicy
|
||||||
|
}
|
||||||
|
return &Planner{policy: policy}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// NextDue derives a stable, symmetric jitter from the check identity. A
|
||||||
|
// process restart therefore does not synchronize all proxy checks into one
|
||||||
|
// burst, while the same proxy/profile remains predictably distributed.
|
||||||
|
func (planner *Planner) NextDue(checkedAt time.Time, identity string) (time.Time, error) {
|
||||||
|
if planner == nil || checkedAt.IsZero() || strings.TrimSpace(identity) != identity || identity == "" {
|
||||||
|
return time.Time{}, ErrInvalidScheduleRequest
|
||||||
|
}
|
||||||
|
base := checkedAt.UTC().Add(planner.policy.Interval)
|
||||||
|
if planner.policy.Jitter == 0 {
|
||||||
|
return base, nil
|
||||||
|
}
|
||||||
|
interval := int64(planner.policy.Interval)
|
||||||
|
span := (interval/100)*int64(planner.policy.Jitter) + (interval%100)*int64(planner.policy.Jitter)/100
|
||||||
|
if span <= 0 {
|
||||||
|
return base, nil
|
||||||
|
}
|
||||||
|
digest := sha256.Sum256([]byte(identity))
|
||||||
|
value := binary.BigEndian.Uint64(digest[:8])
|
||||||
|
offset := time.Duration(value % uint64(span+1))
|
||||||
|
if value&1 == 0 {
|
||||||
|
return base.Add(offset), nil
|
||||||
|
}
|
||||||
|
return base.Add(-offset), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Plan returns at most min(maxTasks, MaxInFlight-inFlight) overdue tasks.
|
||||||
|
// Fetched inventory is prioritized over suspect inventory, followed by normal
|
||||||
|
// available inventory, so recovery work does not starve first-use validation.
|
||||||
|
func (planner *Planner) Plan(now time.Time, inFlight, maxTasks int, candidates []Candidate) ([]PlannedTask, error) {
|
||||||
|
if planner == nil || now.IsZero() || inFlight < 0 || maxTasks <= 0 {
|
||||||
|
return nil, ErrInvalidScheduleRequest
|
||||||
|
}
|
||||||
|
if inFlight >= planner.policy.MaxInFlight || len(candidates) == 0 {
|
||||||
|
return []PlannedTask{}, nil
|
||||||
|
}
|
||||||
|
limit := planner.policy.MaxInFlight - inFlight
|
||||||
|
if maxTasks < limit {
|
||||||
|
limit = maxTasks
|
||||||
|
}
|
||||||
|
eligible := make([]plannedCandidate, 0, min(limit, len(candidates)))
|
||||||
|
for _, candidate := range candidates {
|
||||||
|
priority, include, err := planner.validateCandidate(candidate)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if !include || candidate.DueAt.After(now) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
eligible = append(eligible, plannedCandidate{candidate: candidate, priority: priority})
|
||||||
|
}
|
||||||
|
sort.Slice(eligible, func(left, right int) bool {
|
||||||
|
if eligible[left].priority != eligible[right].priority {
|
||||||
|
return eligible[left].priority < eligible[right].priority
|
||||||
|
}
|
||||||
|
if !eligible[left].candidate.DueAt.Equal(eligible[right].candidate.DueAt) {
|
||||||
|
return eligible[left].candidate.DueAt.Before(eligible[right].candidate.DueAt)
|
||||||
|
}
|
||||||
|
return candidateIdentity(eligible[left].candidate) < candidateIdentity(eligible[right].candidate)
|
||||||
|
})
|
||||||
|
if len(eligible) > limit {
|
||||||
|
eligible = eligible[:limit]
|
||||||
|
}
|
||||||
|
deadline := now.UTC().Add(planner.policy.Timeout)
|
||||||
|
result := make([]PlannedTask, len(eligible))
|
||||||
|
for index, item := range eligible {
|
||||||
|
result[index] = PlannedTask{
|
||||||
|
Candidate: item.candidate, Priority: item.priority, Deadline: deadline, Attempts: planner.policy.MaxAttempts,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return result, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
type plannedCandidate struct {
|
||||||
|
candidate Candidate
|
||||||
|
priority Priority
|
||||||
|
}
|
||||||
|
|
||||||
|
func (planner *Planner) validateCandidate(candidate Candidate) (Priority, bool, error) {
|
||||||
|
if strings.TrimSpace(candidate.ProxyID) != candidate.ProxyID || candidate.ProxyID == "" || candidate.DueAt.IsZero() {
|
||||||
|
return 0, false, ErrInvalidScheduleRequest
|
||||||
|
}
|
||||||
|
switch candidate.Level {
|
||||||
|
case healthDomain.LevelBasic, healthDomain.LevelEgress:
|
||||||
|
if candidate.RoutingName != "" || candidate.TargetURL != "" {
|
||||||
|
return 0, false, ErrInvalidScheduleRequest
|
||||||
|
}
|
||||||
|
case healthDomain.LevelTarget:
|
||||||
|
if _, err := healthDomain.NormalizeTargetProfile(healthDomain.TargetProfile{
|
||||||
|
RoutingName: candidate.RoutingName, TargetURL: candidate.TargetURL,
|
||||||
|
}); err != nil {
|
||||||
|
return 0, false, ErrInvalidScheduleRequest
|
||||||
|
}
|
||||||
|
default:
|
||||||
|
return 0, false, ErrInvalidScheduleRequest
|
||||||
|
}
|
||||||
|
switch candidate.State {
|
||||||
|
case proxyDomain.StateFetched:
|
||||||
|
return PriorityFetched, true, nil
|
||||||
|
case proxyDomain.StateSuspect:
|
||||||
|
return PrioritySuspect, true, nil
|
||||||
|
case proxyDomain.StateUnhealthy:
|
||||||
|
return PriorityUnhealthy, true, nil
|
||||||
|
case proxyDomain.StateAvailable:
|
||||||
|
return PriorityAvailable, true, nil
|
||||||
|
default:
|
||||||
|
return 0, false, nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func candidateIdentity(candidate Candidate) string {
|
||||||
|
return candidate.ProxyID + "\x00" + string(candidate.Level) + "\x00" + candidate.RoutingName + "\x00" + candidate.TargetURL
|
||||||
|
}
|
||||||
88
internal/controller/health/scheduler_test.go
Normal file
88
internal/controller/health/scheduler_test.go
Normal file
@ -0,0 +1,88 @@
|
|||||||
|
package health
|
||||||
|
|
||||||
|
import (
|
||||||
|
"errors"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
healthDomain "proxy-pool/internal/domain/health"
|
||||||
|
proxyDomain "proxy-pool/internal/domain/proxy"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestPlannerPrioritizesAndBoundsOverdueCandidates(t *testing.T) {
|
||||||
|
planner, err := NewPlanner(SchedulePolicy{
|
||||||
|
Interval: time.Minute, Jitter: 20, MaxInFlight: 3, Timeout: 4 * time.Second, MaxAttempts: 2,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("NewPlanner(): %v", err)
|
||||||
|
}
|
||||||
|
now := time.Date(2026, 7, 31, 17, 0, 0, 0, time.UTC)
|
||||||
|
plans, err := planner.Plan(now, 1, 10, []Candidate{
|
||||||
|
{ProxyID: "available", State: proxyDomain.StateAvailable, Level: healthDomain.LevelBasic, DueAt: now.Add(-time.Second)},
|
||||||
|
{ProxyID: "suspect", State: proxyDomain.StateSuspect, Level: healthDomain.LevelEgress, DueAt: now.Add(-2 * time.Second)},
|
||||||
|
{ProxyID: "fetched", State: proxyDomain.StateFetched, Level: healthDomain.LevelBasic, DueAt: now.Add(-3 * time.Second)},
|
||||||
|
{ProxyID: "future", State: proxyDomain.StateFetched, Level: healthDomain.LevelBasic, DueAt: now.Add(time.Second)},
|
||||||
|
})
|
||||||
|
if err != nil || len(plans) != 2 {
|
||||||
|
t.Fatalf("Plan() = (%+v, %v)", plans, err)
|
||||||
|
}
|
||||||
|
if plans[0].Candidate.ProxyID != "fetched" || plans[0].Priority != PriorityFetched ||
|
||||||
|
plans[1].Candidate.ProxyID != "suspect" || plans[1].Priority != PrioritySuspect ||
|
||||||
|
plans[0].Deadline != now.Add(4*time.Second) || plans[0].Attempts != 2 {
|
||||||
|
t.Fatalf("plans = %+v", plans)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPlannerSchedulesUnhealthyForRecovery(t *testing.T) {
|
||||||
|
planner, err := NewPlanner(SchedulePolicy{
|
||||||
|
Interval: time.Minute, MaxInFlight: 1, Timeout: time.Second, MaxAttempts: 1,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("NewPlanner(): %v", err)
|
||||||
|
}
|
||||||
|
now := time.Date(2026, 7, 31, 17, 0, 0, 0, time.UTC)
|
||||||
|
plans, err := planner.Plan(now, 0, 1, []Candidate{{
|
||||||
|
ProxyID: "unhealthy", State: proxyDomain.StateUnhealthy, Level: healthDomain.LevelBasic, DueAt: now,
|
||||||
|
}})
|
||||||
|
if err != nil || len(plans) != 1 || plans[0].Priority != PriorityUnhealthy {
|
||||||
|
t.Fatalf("Plan(unhealthy) = (%+v, %v)", plans, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPlannerDerivesStableBoundedJitter(t *testing.T) {
|
||||||
|
planner, err := NewPlanner(SchedulePolicy{
|
||||||
|
Interval: time.Minute, Jitter: 20, MaxInFlight: 1, Timeout: time.Second, MaxAttempts: 1,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("NewPlanner(): %v", err)
|
||||||
|
}
|
||||||
|
checkedAt := time.Date(2026, 7, 31, 17, 0, 0, 0, time.UTC)
|
||||||
|
first, err := planner.NextDue(checkedAt, "proxy-a\x00BASIC")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("NextDue(first): %v", err)
|
||||||
|
}
|
||||||
|
second, err := planner.NextDue(checkedAt, "proxy-a\x00BASIC")
|
||||||
|
if err != nil || second != first {
|
||||||
|
t.Fatalf("NextDue(stable) = (%v, %v), want %v", second, err, first)
|
||||||
|
}
|
||||||
|
base := checkedAt.Add(time.Minute)
|
||||||
|
if delta := first.Sub(base); delta < -12*time.Second || delta > 12*time.Second {
|
||||||
|
t.Fatalf("NextDue jitter delta = %s, want [-12s, 12s]", delta)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPlannerRejectsInvalidCandidatesWithoutPartialPlan(t *testing.T) {
|
||||||
|
planner, err := NewPlanner(SchedulePolicy{
|
||||||
|
Interval: time.Minute, MaxInFlight: 1, Timeout: time.Second, MaxAttempts: 1,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("NewPlanner(): %v", err)
|
||||||
|
}
|
||||||
|
_, err = planner.Plan(time.Now(), 0, 1, []Candidate{{
|
||||||
|
ProxyID: "proxy-a", State: proxyDomain.StateFetched, Level: healthDomain.LevelTarget,
|
||||||
|
RoutingName: "route-a", TargetURL: "not-a-url", DueAt: time.Now(),
|
||||||
|
}})
|
||||||
|
if !errors.Is(err, ErrInvalidScheduleRequest) {
|
||||||
|
t.Fatalf("Plan(invalid target) error = %v, want ErrInvalidScheduleRequest", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
Loading…
Reference in New Issue
Block a user