365 lines
11 KiB
Go
365 lines
11 KiB
Go
// Package health defines transport-free facts and deterministic health
|
|
// reduction rules shared by the Controller and Checker integration layers.
|
|
package health
|
|
|
|
import (
|
|
"crypto/sha256"
|
|
"errors"
|
|
"fmt"
|
|
"net/netip"
|
|
"net/url"
|
|
"strings"
|
|
"time"
|
|
|
|
proxyDomain "proxy-pool/internal/domain/proxy"
|
|
)
|
|
|
|
var (
|
|
ErrInvalidObservation = errors.New("invalid health observation")
|
|
ErrInvalidGlobalState = errors.New("invalid global health state")
|
|
ErrInvalidTargetState = errors.New("invalid target health state")
|
|
ErrNonGlobalObservation = errors.New("target health observation cannot change global proxy health")
|
|
ErrNonTargetObservation = errors.New("global health observation cannot change target profile health")
|
|
ErrStaleObservation = errors.New("stale health observation")
|
|
ErrConflictingObservation = errors.New("conflicting health observation replay")
|
|
ErrInvalidFailureThreshold = errors.New("invalid health failure threshold")
|
|
ErrInvalidCheckPreparation = errors.New("invalid health check preparation")
|
|
)
|
|
|
|
type Level string
|
|
|
|
const (
|
|
LevelBasic Level = "BASIC"
|
|
LevelEgress Level = "EGRESS"
|
|
LevelTarget Level = "TARGET"
|
|
)
|
|
|
|
type TargetStatus string
|
|
|
|
const (
|
|
TargetUnknown TargetStatus = "UNKNOWN"
|
|
TargetAvailable TargetStatus = "AVAILABLE"
|
|
TargetSuspect TargetStatus = "SUSPECT"
|
|
TargetUnhealthy TargetStatus = "UNHEALTHY"
|
|
)
|
|
|
|
// Observation is an immutable result produced by a Checker. It has no
|
|
// authority to modify a Proxy until the Controller reduces and commits it.
|
|
type Observation struct {
|
|
TaskID string
|
|
ProxyID string
|
|
Level Level
|
|
RoutingName string
|
|
TargetURL string
|
|
Success bool
|
|
FailureClass string
|
|
Latency time.Duration
|
|
ObservedEgressIP string
|
|
ObservedAt time.Time
|
|
}
|
|
|
|
type TargetProfile struct {
|
|
RoutingName string
|
|
TargetURL string
|
|
}
|
|
|
|
// GlobalState is the small authoritative state needed to reduce BASIC and
|
|
// EGRESS observations. The persistence adapter owns storing it atomically.
|
|
type GlobalState struct {
|
|
State proxyDomain.State
|
|
ConsecutiveFailures int
|
|
LastTaskID string
|
|
LastObservedAt time.Time
|
|
LastObservationDigest [sha256.Size]byte
|
|
}
|
|
|
|
// TargetState is isolated by (proxy, routing, target URL). A target failure
|
|
// never changes a Proxy's GlobalState.
|
|
type TargetState struct {
|
|
Status TargetStatus
|
|
ConsecutiveFailures int
|
|
LastTaskID string
|
|
LastObservedAt time.Time
|
|
LastSuccessAt time.Time
|
|
Latency time.Duration
|
|
LastObservationDigest [sha256.Size]byte
|
|
}
|
|
|
|
// NormalizeObservation validates and canonicalizes an observation before it
|
|
// crosses an idempotency or persistence boundary.
|
|
func NormalizeObservation(value Observation) (Observation, error) {
|
|
value.TaskID = strings.TrimSpace(value.TaskID)
|
|
value.ProxyID = strings.TrimSpace(value.ProxyID)
|
|
value.RoutingName = strings.TrimSpace(value.RoutingName)
|
|
value.TargetURL = strings.TrimSpace(value.TargetURL)
|
|
value.FailureClass = strings.TrimSpace(value.FailureClass)
|
|
value.ObservedEgressIP = strings.TrimSpace(value.ObservedEgressIP)
|
|
if !validIdentifier(value.TaskID) || !validIdentifier(value.ProxyID) || value.ObservedAt.IsZero() || value.Latency < 0 {
|
|
return Observation{}, ErrInvalidObservation
|
|
}
|
|
if value.Success && value.FailureClass != "" {
|
|
return Observation{}, ErrInvalidObservation
|
|
}
|
|
if value.ObservedEgressIP != "" {
|
|
if _, err := netip.ParseAddr(value.ObservedEgressIP); err != nil {
|
|
return Observation{}, ErrInvalidObservation
|
|
}
|
|
}
|
|
switch value.Level {
|
|
case LevelBasic, LevelEgress:
|
|
if value.RoutingName != "" || value.TargetURL != "" {
|
|
return Observation{}, ErrInvalidObservation
|
|
}
|
|
case LevelTarget:
|
|
profile, err := NormalizeTargetProfile(TargetProfile{RoutingName: value.RoutingName, TargetURL: value.TargetURL})
|
|
if err != nil {
|
|
return Observation{}, err
|
|
}
|
|
value.RoutingName = profile.RoutingName
|
|
value.TargetURL = profile.TargetURL
|
|
default:
|
|
return Observation{}, ErrInvalidObservation
|
|
}
|
|
value.ObservedAt = value.ObservedAt.UTC()
|
|
return value, nil
|
|
}
|
|
|
|
func NormalizeTargetProfile(value TargetProfile) (TargetProfile, error) {
|
|
value.RoutingName = strings.TrimSpace(value.RoutingName)
|
|
value.TargetURL = strings.TrimSpace(value.TargetURL)
|
|
if !validIdentifier(value.RoutingName) || value.TargetURL == "" {
|
|
return TargetProfile{}, ErrInvalidObservation
|
|
}
|
|
parsed, err := url.Parse(value.TargetURL)
|
|
if err != nil || parsed.Scheme == "" || parsed.Host == "" || parsed.User != nil || parsed.Fragment != "" {
|
|
return TargetProfile{}, ErrInvalidObservation
|
|
}
|
|
value.TargetURL = parsed.String()
|
|
return value, nil
|
|
}
|
|
|
|
func (value TargetProfile) Key() string {
|
|
return value.RoutingName + "\x00" + value.TargetURL
|
|
}
|
|
|
|
func ObservationDigest(value Observation) ([sha256.Size]byte, error) {
|
|
normalized, err := NormalizeObservation(value)
|
|
if err != nil {
|
|
return [sha256.Size]byte{}, err
|
|
}
|
|
payload := strings.Join([]string{
|
|
normalized.TaskID, normalized.ProxyID, string(normalized.Level), normalized.RoutingName,
|
|
normalized.TargetURL, fmt.Sprintf("%t", normalized.Success), normalized.FailureClass,
|
|
normalized.Latency.String(), normalized.ObservedEgressIP, normalized.ObservedAt.Format(time.RFC3339Nano),
|
|
}, "\x00")
|
|
return sha256.Sum256([]byte(payload)), nil
|
|
}
|
|
|
|
// BeginGlobalCheck moves only states that are deliberately unavailable during
|
|
// probing into CHECKING. AVAILABLE and SUSPECT remain serving states while a
|
|
// periodic probe is in flight.
|
|
func BeginGlobalCheck(current GlobalState) (proxyDomain.State, error) {
|
|
if err := validateGlobalState(current); err != nil {
|
|
return "", err
|
|
}
|
|
switch current.State {
|
|
case proxyDomain.StateFetched, proxyDomain.StateUnhealthy:
|
|
return proxyDomain.StateChecking, nil
|
|
case proxyDomain.StateChecking, proxyDomain.StateAvailable, proxyDomain.StateSuspect:
|
|
return current.State, nil
|
|
default:
|
|
return "", ErrInvalidCheckPreparation
|
|
}
|
|
}
|
|
|
|
// ReduceGlobal reduces one BASIC or EGRESS fact. It is intentionally pure so
|
|
// Memory and Redis adapters can apply the same result atomically.
|
|
func ReduceGlobal(current GlobalState, observation Observation, maxConsecutiveFailures int) (GlobalState, error) {
|
|
if maxConsecutiveFailures <= 0 {
|
|
return GlobalState{}, ErrInvalidFailureThreshold
|
|
}
|
|
if err := validateGlobalState(current); err != nil {
|
|
return GlobalState{}, err
|
|
}
|
|
normalized, err := NormalizeObservation(observation)
|
|
if err != nil {
|
|
return GlobalState{}, err
|
|
}
|
|
if normalized.Level == LevelTarget {
|
|
return GlobalState{}, ErrNonGlobalObservation
|
|
}
|
|
digest, err := ObservationDigest(normalized)
|
|
if err != nil {
|
|
return GlobalState{}, err
|
|
}
|
|
if err := checkObservationOrder(current.LastTaskID, current.LastObservedAt, current.LastObservationDigest, normalized, digest); err != nil {
|
|
if errors.Is(err, errExactReplay) {
|
|
return current, nil
|
|
}
|
|
return GlobalState{}, err
|
|
}
|
|
next := current
|
|
next.LastTaskID = normalized.TaskID
|
|
next.LastObservedAt = normalized.ObservedAt
|
|
next.LastObservationDigest = digest
|
|
if normalized.Success {
|
|
switch current.State {
|
|
case proxyDomain.StateChecking, proxyDomain.StateAvailable, proxyDomain.StateSuspect:
|
|
next.State = proxyDomain.StateAvailable
|
|
next.ConsecutiveFailures = 0
|
|
return next, nil
|
|
default:
|
|
return GlobalState{}, ErrInvalidGlobalState
|
|
}
|
|
}
|
|
switch current.State {
|
|
case proxyDomain.StateChecking:
|
|
next.ConsecutiveFailures++
|
|
next.State = proxyDomain.StateUnhealthy
|
|
return next, nil
|
|
case proxyDomain.StateAvailable, proxyDomain.StateSuspect:
|
|
next.ConsecutiveFailures++
|
|
if next.ConsecutiveFailures >= maxConsecutiveFailures {
|
|
next.State = proxyDomain.StateUnhealthy
|
|
} else {
|
|
next.State = proxyDomain.StateSuspect
|
|
}
|
|
return next, nil
|
|
default:
|
|
return GlobalState{}, ErrInvalidGlobalState
|
|
}
|
|
}
|
|
|
|
// ReduceTarget reduces one TARGET fact into only its target profile state.
|
|
// Callers must keep this result separate from activity-pool global state.
|
|
func ReduceTarget(current TargetState, observation Observation, maxConsecutiveFailures int) (TargetState, error) {
|
|
if maxConsecutiveFailures <= 0 {
|
|
return TargetState{}, ErrInvalidFailureThreshold
|
|
}
|
|
if current.Status == "" {
|
|
current.Status = TargetUnknown
|
|
}
|
|
if err := validateTargetState(current); err != nil {
|
|
return TargetState{}, err
|
|
}
|
|
normalized, err := NormalizeObservation(observation)
|
|
if err != nil {
|
|
return TargetState{}, err
|
|
}
|
|
if normalized.Level != LevelTarget {
|
|
return TargetState{}, ErrNonTargetObservation
|
|
}
|
|
digest, err := ObservationDigest(normalized)
|
|
if err != nil {
|
|
return TargetState{}, err
|
|
}
|
|
if err := checkObservationOrder(current.LastTaskID, current.LastObservedAt, current.LastObservationDigest, normalized, digest); err != nil {
|
|
if errors.Is(err, errExactReplay) {
|
|
return current, nil
|
|
}
|
|
return TargetState{}, err
|
|
}
|
|
next := current
|
|
next.LastTaskID = normalized.TaskID
|
|
next.LastObservedAt = normalized.ObservedAt
|
|
next.LastObservationDigest = digest
|
|
next.Latency = normalized.Latency
|
|
if normalized.Success {
|
|
next.Status = TargetAvailable
|
|
next.ConsecutiveFailures = 0
|
|
next.LastSuccessAt = normalized.ObservedAt
|
|
return next, nil
|
|
}
|
|
next.ConsecutiveFailures++
|
|
if next.ConsecutiveFailures >= maxConsecutiveFailures {
|
|
next.Status = TargetUnhealthy
|
|
} else {
|
|
next.Status = TargetSuspect
|
|
}
|
|
return next, nil
|
|
}
|
|
|
|
var errExactReplay = errors.New("exact health observation replay")
|
|
|
|
func checkObservationOrder(
|
|
lastTaskID string,
|
|
lastObservedAt time.Time,
|
|
lastDigest [sha256.Size]byte,
|
|
observation Observation,
|
|
digest [sha256.Size]byte,
|
|
) error {
|
|
if lastObservedAt.IsZero() {
|
|
return nil
|
|
}
|
|
if observation.ObservedAt.Before(lastObservedAt) {
|
|
return ErrStaleObservation
|
|
}
|
|
if observation.TaskID == lastTaskID {
|
|
if digest == lastDigest {
|
|
return errExactReplay
|
|
}
|
|
return ErrConflictingObservation
|
|
}
|
|
if observation.ObservedAt.Equal(lastObservedAt) {
|
|
return ErrStaleObservation
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func validateGlobalState(value GlobalState) error {
|
|
if value.ConsecutiveFailures < 0 {
|
|
return ErrInvalidGlobalState
|
|
}
|
|
switch value.State {
|
|
case proxyDomain.StateFetched, proxyDomain.StateChecking, proxyDomain.StateAvailable,
|
|
proxyDomain.StateSuspect, proxyDomain.StateUnhealthy:
|
|
default:
|
|
return ErrInvalidGlobalState
|
|
}
|
|
if value.LastObservedAt.IsZero() {
|
|
if value.LastTaskID != "" || value.ConsecutiveFailures != 0 || value.LastObservationDigest != ([sha256.Size]byte{}) {
|
|
return ErrInvalidGlobalState
|
|
}
|
|
return nil
|
|
}
|
|
if !validIdentifier(value.LastTaskID) || value.LastObservationDigest == ([sha256.Size]byte{}) {
|
|
return ErrInvalidGlobalState
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func validateTargetState(value TargetState) error {
|
|
if value.ConsecutiveFailures < 0 || value.Latency < 0 {
|
|
return ErrInvalidTargetState
|
|
}
|
|
switch value.Status {
|
|
case TargetUnknown, TargetAvailable, TargetSuspect, TargetUnhealthy:
|
|
default:
|
|
return ErrInvalidTargetState
|
|
}
|
|
if value.LastObservedAt.IsZero() {
|
|
if value.Status != TargetUnknown || value.LastTaskID != "" || value.ConsecutiveFailures != 0 ||
|
|
!value.LastSuccessAt.IsZero() || value.LastObservationDigest != ([sha256.Size]byte{}) {
|
|
return ErrInvalidTargetState
|
|
}
|
|
return nil
|
|
}
|
|
if !validIdentifier(value.LastTaskID) || value.LastObservationDigest == ([sha256.Size]byte{}) ||
|
|
(!value.LastSuccessAt.IsZero() && value.LastSuccessAt.After(value.LastObservedAt)) {
|
|
return ErrInvalidTargetState
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func validIdentifier(value string) bool {
|
|
if value == "" || len(value) > 256 {
|
|
return false
|
|
}
|
|
for _, character := range value {
|
|
if character <= ' ' || character == '\x7f' {
|
|
return false
|
|
}
|
|
}
|
|
return true
|
|
}
|