264 lines
9.8 KiB
Go
264 lines
9.8 KiB
Go
package redisactivity
|
|
|
|
import (
|
|
"context"
|
|
"crypto/sha256"
|
|
"encoding/hex"
|
|
"errors"
|
|
"time"
|
|
|
|
"proxy-pool/internal/domain/activitypool"
|
|
healthDomain "proxy-pool/internal/domain/health"
|
|
)
|
|
|
|
var _ activitypool.HealthStore = (*Adapter)(nil)
|
|
var _ activitypool.GlobalHealthStore = (*Adapter)(nil)
|
|
var _ activitypool.TargetHealthStore = (*Adapter)(nil)
|
|
|
|
type targetHealthRecord struct {
|
|
Version int `json:"version"`
|
|
Status string `json:"status"`
|
|
ConsecutiveFailures int `json:"consecutiveFailures"`
|
|
LastHealthTaskID string `json:"lastHealthTaskId,omitempty"`
|
|
LastHealthDigest string `json:"lastHealthDigest,omitempty"`
|
|
LastHealthObservedAtMS int64 `json:"lastHealthObservedAtMs,omitempty"`
|
|
LastSuccessAtMS int64 `json:"lastSuccessAtMs,omitempty"`
|
|
LatencyNS int64 `json:"latencyNs"`
|
|
}
|
|
|
|
func (a *Adapter) ApplyHealth(ctx context.Context, update activitypool.HealthUpdate) (activitypool.Entry, error) {
|
|
if ctx == nil {
|
|
return activitypool.Entry{}, activitypool.ErrInvalidHealthUpdate
|
|
}
|
|
if err := ctx.Err(); err != nil {
|
|
return activitypool.Entry{}, err
|
|
}
|
|
if a == nil || update.ProxyID == "" || update.CheckedAt.IsZero() || update.Latency < 0 ||
|
|
!validProxyState(string(update.NextState)) {
|
|
return activitypool.Entry{}, activitypool.ErrInvalidHealthUpdate
|
|
}
|
|
operationID, err := newOperationID()
|
|
if err != nil {
|
|
return activitypool.Entry{}, err
|
|
}
|
|
result, err := runScript(ctx, a.client, healthScript, []string{
|
|
a.keys.records, a.keys.unique, a.keys.idkeys, a.keys.expiry, a.keys.available,
|
|
a.keys.inventory, a.keys.stateInventory, a.keys.owners, a.keys.ownerExpiry,
|
|
a.keys.operation(operationID),
|
|
}, update.CheckedAt.UnixMilli(), string(update.NextState), int64(update.Latency),
|
|
a.options.CleanupLimit, operationTTLMillis(a.options.OperationTTL), update.ProxyID)
|
|
if err != nil {
|
|
return activitypool.Entry{}, err
|
|
}
|
|
var reply healthScriptReply
|
|
if err := decodeScriptResult(result, &reply); err != nil {
|
|
return activitypool.Entry{}, err
|
|
}
|
|
switch reply.Status {
|
|
case scriptNotFound:
|
|
return activitypool.Entry{}, activitypool.ErrActivityNotFound
|
|
case scriptStale:
|
|
return activitypool.Entry{}, activitypool.ErrStaleHealthUpdate
|
|
case scriptInvalid:
|
|
return activitypool.Entry{}, activitypool.ErrInvalidHealthUpdate
|
|
case scriptOK:
|
|
if reply.Record == "" {
|
|
return activitypool.Entry{}, invalidScriptReply("health reply omitted record")
|
|
}
|
|
record, err := decodeProxyRecord(reply.Record)
|
|
if err != nil {
|
|
return activitypool.Entry{}, invalidScriptReply("health reply contained an invalid record")
|
|
}
|
|
return proxyRecordEntry(record), nil
|
|
default:
|
|
return activitypool.Entry{}, invalidScriptReply("unexpected health status")
|
|
}
|
|
}
|
|
|
|
// ApplyGlobalObservation reduces one BASIC or EGRESS observation inside the
|
|
// same Redis Lua transaction that updates state indexes and inventory.
|
|
func (a *Adapter) ApplyGlobalObservation(
|
|
ctx context.Context,
|
|
command activitypool.GlobalHealthCommand,
|
|
) (activitypool.Entry, error) {
|
|
if ctx == nil {
|
|
return activitypool.Entry{}, activitypool.ErrInvalidHealthUpdate
|
|
}
|
|
if err := ctx.Err(); err != nil {
|
|
return activitypool.Entry{}, err
|
|
}
|
|
normalized, err := healthDomain.NormalizeObservation(command.Observation)
|
|
if err != nil {
|
|
return activitypool.Entry{}, err
|
|
}
|
|
if a == nil || normalized.Level == healthDomain.LevelTarget || command.MaxConsecutiveFailures <= 0 {
|
|
return activitypool.Entry{}, activitypool.ErrInvalidHealthUpdate
|
|
}
|
|
digest, err := healthDomain.ObservationDigest(normalized)
|
|
if err != nil {
|
|
return activitypool.Entry{}, err
|
|
}
|
|
operationID, err := newOperationID()
|
|
if err != nil {
|
|
return activitypool.Entry{}, err
|
|
}
|
|
success := "0"
|
|
if normalized.Success {
|
|
success = "1"
|
|
}
|
|
result, err := runScript(ctx, a.client, healthScript, []string{
|
|
a.keys.records, a.keys.unique, a.keys.idkeys, a.keys.expiry, a.keys.available,
|
|
a.keys.inventory, a.keys.stateInventory, a.keys.owners, a.keys.ownerExpiry,
|
|
a.keys.operation(operationID),
|
|
}, normalized.ObservedAt.UnixMilli(), "", int64(normalized.Latency),
|
|
a.options.CleanupLimit, operationTTLMillis(a.options.OperationTTL), normalized.ProxyID,
|
|
"global", success, command.MaxConsecutiveFailures, normalized.TaskID, hex.EncodeToString(digest[:]))
|
|
if err != nil {
|
|
return activitypool.Entry{}, err
|
|
}
|
|
var reply healthScriptReply
|
|
if err := decodeScriptResult(result, &reply); err != nil {
|
|
return activitypool.Entry{}, err
|
|
}
|
|
switch reply.Status {
|
|
case scriptNotFound:
|
|
return activitypool.Entry{}, activitypool.ErrActivityNotFound
|
|
case scriptStale:
|
|
return activitypool.Entry{}, healthDomain.ErrStaleObservation
|
|
case scriptConflict:
|
|
return activitypool.Entry{}, healthDomain.ErrConflictingObservation
|
|
case scriptInvalid:
|
|
return activitypool.Entry{}, activitypool.ErrInvalidHealthUpdate
|
|
case scriptOK:
|
|
if reply.Record == "" {
|
|
return activitypool.Entry{}, invalidScriptReply("global health reply omitted record")
|
|
}
|
|
record, err := decodeProxyRecord(reply.Record)
|
|
if err != nil {
|
|
return activitypool.Entry{}, invalidScriptReply("global health reply contained an invalid record")
|
|
}
|
|
return proxyRecordEntry(record), nil
|
|
default:
|
|
return activitypool.Entry{}, invalidScriptReply("unexpected global health status")
|
|
}
|
|
}
|
|
|
|
// ApplyTargetObservation reduces a TARGET check into a short-lived Profile
|
|
// record keyed by (proxy, routing, target URL). It intentionally does not
|
|
// update the proxy record, global health, or any proxy selection index.
|
|
func (a *Adapter) ApplyTargetObservation(
|
|
ctx context.Context,
|
|
command activitypool.TargetHealthCommand,
|
|
) (healthDomain.TargetState, error) {
|
|
if ctx == nil {
|
|
return healthDomain.TargetState{}, activitypool.ErrInvalidHealthUpdate
|
|
}
|
|
if err := ctx.Err(); err != nil {
|
|
return healthDomain.TargetState{}, err
|
|
}
|
|
normalized, err := healthDomain.NormalizeObservation(command.Observation)
|
|
if err != nil {
|
|
return healthDomain.TargetState{}, err
|
|
}
|
|
if a == nil || normalized.Level != healthDomain.LevelTarget || command.MaxConsecutiveFailures <= 0 {
|
|
return healthDomain.TargetState{}, activitypool.ErrInvalidHealthUpdate
|
|
}
|
|
profile, err := healthDomain.NormalizeTargetProfile(healthDomain.TargetProfile{
|
|
RoutingName: normalized.RoutingName,
|
|
TargetURL: normalized.TargetURL,
|
|
})
|
|
if err != nil {
|
|
return healthDomain.TargetState{}, err
|
|
}
|
|
digest, err := healthDomain.ObservationDigest(normalized)
|
|
if err != nil {
|
|
return healthDomain.TargetState{}, err
|
|
}
|
|
operationID, err := newOperationID()
|
|
if err != nil {
|
|
return healthDomain.TargetState{}, err
|
|
}
|
|
success := "0"
|
|
if normalized.Success {
|
|
success = "1"
|
|
}
|
|
result, err := runScript(ctx, a.client, targetHealthScript, []string{
|
|
a.keys.records, a.keys.targetHealth(normalized.ProxyID), a.keys.operation(operationID),
|
|
}, normalized.ObservedAt.UnixMilli(), int64(normalized.Latency), normalized.ProxyID,
|
|
digestToken(profile.Key()), success, command.MaxConsecutiveFailures, normalized.TaskID,
|
|
hex.EncodeToString(digest[:]), operationTTLMillis(a.options.OperationTTL))
|
|
if err != nil {
|
|
return healthDomain.TargetState{}, err
|
|
}
|
|
var reply targetHealthScriptReply
|
|
if err := decodeScriptResult(result, &reply); err != nil {
|
|
return healthDomain.TargetState{}, err
|
|
}
|
|
switch reply.Status {
|
|
case scriptNotFound:
|
|
return healthDomain.TargetState{}, activitypool.ErrActivityNotFound
|
|
case scriptStale:
|
|
return healthDomain.TargetState{}, healthDomain.ErrStaleObservation
|
|
case scriptConflict:
|
|
return healthDomain.TargetState{}, healthDomain.ErrConflictingObservation
|
|
case scriptInvalid:
|
|
return healthDomain.TargetState{}, activitypool.ErrInvalidHealthUpdate
|
|
case scriptOK:
|
|
if reply.Target == "" {
|
|
return healthDomain.TargetState{}, invalidScriptReply("target health reply omitted target state")
|
|
}
|
|
state, err := decodeTargetHealthState(reply.Target)
|
|
if err != nil {
|
|
return healthDomain.TargetState{}, invalidScriptReply("target health reply contained an invalid target state")
|
|
}
|
|
return state, nil
|
|
default:
|
|
return healthDomain.TargetState{}, invalidScriptReply("unexpected target health status")
|
|
}
|
|
}
|
|
|
|
func decodeTargetHealthState(payload string) (healthDomain.TargetState, error) {
|
|
var record targetHealthRecord
|
|
if err := decodeJSON(payload, &record); err != nil {
|
|
return healthDomain.TargetState{}, err
|
|
}
|
|
if record.Version != recordVersion || record.ConsecutiveFailures < 0 || record.LatencyNS < 0 {
|
|
return healthDomain.TargetState{}, ErrInvalidRecord
|
|
}
|
|
state := healthDomain.TargetState{
|
|
Status: healthDomain.TargetStatus(record.Status),
|
|
ConsecutiveFailures: record.ConsecutiveFailures,
|
|
LastTaskID: record.LastHealthTaskID,
|
|
Latency: time.Duration(record.LatencyNS),
|
|
}
|
|
switch state.Status {
|
|
case healthDomain.TargetUnknown, healthDomain.TargetAvailable, healthDomain.TargetSuspect, healthDomain.TargetUnhealthy:
|
|
default:
|
|
return healthDomain.TargetState{}, ErrInvalidRecord
|
|
}
|
|
if record.LastHealthObservedAtMS == 0 {
|
|
if state.Status != healthDomain.TargetUnknown || state.LastTaskID != "" || state.ConsecutiveFailures != 0 ||
|
|
record.LastHealthDigest != "" || record.LastSuccessAtMS != 0 || state.Latency != 0 {
|
|
return healthDomain.TargetState{}, ErrInvalidRecord
|
|
}
|
|
return state, nil
|
|
}
|
|
if record.LastHealthObservedAtMS < 0 || record.LastSuccessAtMS < 0 || state.LastTaskID == "" ||
|
|
len(record.LastHealthDigest) != sha256.Size*2 {
|
|
return healthDomain.TargetState{}, ErrInvalidRecord
|
|
}
|
|
digest, err := hex.DecodeString(record.LastHealthDigest)
|
|
if err != nil || len(digest) != sha256.Size {
|
|
return healthDomain.TargetState{}, errors.Join(ErrInvalidRecord, err)
|
|
}
|
|
copy(state.LastObservationDigest[:], digest)
|
|
state.LastObservedAt = time.UnixMilli(record.LastHealthObservedAtMS).UTC()
|
|
if record.LastSuccessAtMS > 0 {
|
|
state.LastSuccessAt = time.UnixMilli(record.LastSuccessAtMS).UTC()
|
|
if state.LastSuccessAt.After(state.LastObservedAt) {
|
|
return healthDomain.TargetState{}, ErrInvalidRecord
|
|
}
|
|
}
|
|
return state, nil
|
|
}
|