364 lines
13 KiB
Go
364 lines
13 KiB
Go
package redisactivity
|
|
|
|
import (
|
|
"context"
|
|
"crypto/rand"
|
|
"encoding/hex"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"strings"
|
|
"time"
|
|
|
|
healthDomain "proxy-pool/internal/domain/health"
|
|
proxyDomain "proxy-pool/internal/domain/proxy"
|
|
)
|
|
|
|
const healthTaskRecordVersion = 1
|
|
|
|
var _ healthDomain.TaskBroker = (*Adapter)(nil)
|
|
|
|
type healthTaskRecord struct {
|
|
Version int `json:"version"`
|
|
TaskID string `json:"taskId"`
|
|
ProxyID string `json:"proxyId"`
|
|
Level string `json:"level"`
|
|
Priority int `json:"priority"`
|
|
DeadlineMS int64 `json:"deadlineMs"`
|
|
NextDueMS int64 `json:"nextDueMs"`
|
|
Attempts int `json:"attempts"`
|
|
State string `json:"state"`
|
|
RoutingName string `json:"routingName,omitempty"`
|
|
TargetURL string `json:"targetUrl,omitempty"`
|
|
LeaseCheckerID string `json:"leaseCheckerId,omitempty"`
|
|
LeaseInstanceID string `json:"leaseInstanceId,omitempty"`
|
|
LeaseToken string `json:"leaseToken,omitempty"`
|
|
LeaseExpiresAtMS int64 `json:"leaseExpiresAtMs,omitempty"`
|
|
CheckerLeaseKey string `json:"checkerLeaseKey,omitempty"`
|
|
}
|
|
|
|
type healthTaskRequest struct {
|
|
Limit int `json:"limit"`
|
|
Tasks []healthTaskRecord `json:"tasks,omitempty"`
|
|
CheckerID string `json:"checkerId,omitempty"`
|
|
InstanceID string `json:"instanceId,omitempty"`
|
|
MaxInFlight int `json:"maxInFlight,omitempty"`
|
|
LeaseTTLMS int64 `json:"leaseTTLMS,omitempty"`
|
|
Levels []healthDomain.Level `json:"levels,omitempty"`
|
|
Tokens []string `json:"tokens,omitempty"`
|
|
Fact *healthTaskFact `json:"fact,omitempty"`
|
|
}
|
|
|
|
type healthTaskFact struct {
|
|
TaskID string `json:"taskId"`
|
|
ProxyID string `json:"proxyId"`
|
|
Level string `json:"level"`
|
|
RoutingName string `json:"routingName,omitempty"`
|
|
TargetURL string `json:"targetUrl,omitempty"`
|
|
CheckerID string `json:"checkerId"`
|
|
LeaseToken string `json:"leaseToken"`
|
|
}
|
|
|
|
// InFlight returns the total queued and leased BASIC tasks after bounded
|
|
// maintenance. The Scheduler uses it only outside the Gateway hot path.
|
|
func (a *Adapter) InFlight(ctx context.Context, now time.Time) (int, error) {
|
|
if a == nil || ctx == nil || now.IsZero() {
|
|
return 0, healthDomain.ErrInvalidTaskBroker
|
|
}
|
|
reply, err := a.runHealthTaskScript(ctx, "inflight", now, healthTaskRequest{Limit: a.options.MaxCheckTasks}, "runtime")
|
|
if err != nil {
|
|
return 0, err
|
|
}
|
|
if reply.Status != scriptOK || reply.Count < 0 {
|
|
return 0, invalidScriptReply("invalid health task inflight reply")
|
|
}
|
|
return reply.Count, nil
|
|
}
|
|
|
|
// DueCandidates returns a bounded BASIC due batch. Stale references are
|
|
// discarded inside the Lua script before they can reach the Controller.
|
|
func (a *Adapter) DueCandidates(ctx context.Context, now time.Time, limit int) ([]healthDomain.Candidate, error) {
|
|
if a == nil || ctx == nil || now.IsZero() || limit <= 0 || limit > a.options.MaxCheckTasks {
|
|
return nil, healthDomain.ErrInvalidTaskBroker
|
|
}
|
|
reply, err := a.runHealthTaskScript(ctx, "due", now, healthTaskRequest{Limit: limit}, "runtime")
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if reply.Status != scriptOK || len(reply.Candidates) > limit {
|
|
return nil, invalidScriptReply("invalid health task due reply")
|
|
}
|
|
result := make([]healthDomain.Candidate, 0, len(reply.Candidates))
|
|
for _, candidate := range reply.Candidates {
|
|
state := proxyDomain.State(candidate.State)
|
|
switch state {
|
|
case proxyDomain.StateFetched, proxyDomain.StateChecking, proxyDomain.StateAvailable, proxyDomain.StateSuspect, proxyDomain.StateUnhealthy:
|
|
default:
|
|
return nil, invalidScriptReply("health task due reply contains invalid state")
|
|
}
|
|
if !validHealthTaskIdentifier(candidate.ProxyID) || candidate.DueAtMS <= 0 {
|
|
return nil, invalidScriptReply("health task due reply contains invalid candidate")
|
|
}
|
|
result = append(result, healthDomain.Candidate{
|
|
ProxyID: candidate.ProxyID, State: state, Level: healthDomain.LevelBasic,
|
|
DueAt: time.UnixMilli(candidate.DueAtMS).UTC(),
|
|
})
|
|
}
|
|
return result, nil
|
|
}
|
|
|
|
// Offer atomically turns due BASIC plans into shared queued tasks. A plan that
|
|
// loses the race leaves no duplicate task and no in-memory queue behind.
|
|
func (a *Adapter) Offer(ctx context.Context, plans []healthDomain.PlannedTask) (int, error) {
|
|
if a == nil || ctx == nil {
|
|
return 0, healthDomain.ErrInvalidTaskBroker
|
|
}
|
|
if err := ctx.Err(); err != nil {
|
|
return 0, err
|
|
}
|
|
if len(plans) == 0 {
|
|
return 0, nil
|
|
}
|
|
if len(plans) > a.options.MaxCheckTasks {
|
|
return 0, healthDomain.ErrInvalidLeasedTask
|
|
}
|
|
now := time.Now().UTC()
|
|
tasks := make([]healthTaskRecord, len(plans))
|
|
for index, plan := range plans {
|
|
if err := validateRedisPlannedTask(plan, now); err != nil {
|
|
return 0, err
|
|
}
|
|
tasks[index] = healthTaskRecord{
|
|
Version: healthTaskRecordVersion, TaskID: healthDomain.TaskIDFor(plan), ProxyID: plan.Candidate.ProxyID,
|
|
Level: string(plan.Candidate.Level), Priority: int(plan.Priority), DeadlineMS: plan.Deadline.UnixMilli(),
|
|
NextDueMS: plan.NextDue.UnixMilli(), Attempts: plan.Attempts, State: "QUEUED",
|
|
}
|
|
}
|
|
reply, err := a.runHealthTaskScript(ctx, "offer", now, healthTaskRequest{Limit: a.options.MaxCheckTasks, Tasks: tasks}, "runtime")
|
|
if err != nil {
|
|
return 0, err
|
|
}
|
|
if reply.Status != scriptOK || reply.Count < 0 || reply.Count > len(plans) {
|
|
return 0, invalidScriptReply("invalid health task offer reply")
|
|
}
|
|
return reply.Count, nil
|
|
}
|
|
|
|
// Claim leases a bounded task batch. Proxy credentials are read only from the
|
|
// live record at lease time and are never copied into Redis task state.
|
|
func (a *Adapter) Claim(ctx context.Context, claim healthDomain.TaskClaim) ([]healthDomain.LeasedTask, error) {
|
|
if a == nil || ctx == nil {
|
|
return nil, healthDomain.ErrInvalidTaskBroker
|
|
}
|
|
if err := ctx.Err(); err != nil {
|
|
return nil, err
|
|
}
|
|
if err := validateRedisTaskClaim(claim, a.options.MaxCheckTasks); err != nil {
|
|
return nil, err
|
|
}
|
|
tokens := make([]string, claim.MaxInFlight)
|
|
for index := range tokens {
|
|
token, err := newRedisLeaseToken()
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
tokens[index] = token
|
|
}
|
|
now := time.Now().UTC()
|
|
reply, err := a.runHealthTaskScript(ctx, "claim", now, healthTaskRequest{
|
|
Limit: a.options.MaxCheckTasks, CheckerID: claim.CheckerID, InstanceID: claim.InstanceID,
|
|
MaxInFlight: claim.MaxInFlight, LeaseTTLMS: a.options.CheckLeaseTTL.Milliseconds(),
|
|
Levels: claim.SupportedLevels, Tokens: tokens,
|
|
}, claim.CheckerID)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if reply.Status != scriptOK || len(reply.Tasks) > claim.MaxInFlight {
|
|
return nil, invalidScriptReply("invalid health task claim reply")
|
|
}
|
|
result := make([]healthDomain.LeasedTask, 0, len(reply.Tasks))
|
|
for _, item := range reply.Tasks {
|
|
task, err := decodeHealthTaskRecord(item.Task)
|
|
if err != nil {
|
|
return nil, invalidScriptReply("health task claim reply contains invalid task")
|
|
}
|
|
record, err := decodeProxyRecord(item.Record)
|
|
if err != nil {
|
|
return nil, invalidScriptReply("health task claim reply contains invalid record")
|
|
}
|
|
if task.State != "LEASED" || task.LeaseCheckerID != claim.CheckerID || task.LeaseToken == "" ||
|
|
task.Level != string(healthDomain.LevelBasic) || task.ProxyID != record.ID || task.DeadlineMS <= now.UnixMilli() {
|
|
return nil, invalidScriptReply("health task claim reply violates lease contract")
|
|
}
|
|
result = append(result, healthDomain.LeasedTask{
|
|
TaskID: task.TaskID, LeaseToken: task.LeaseToken, ProxyID: task.ProxyID,
|
|
Protocol: proxyDomain.Scheme(record.Scheme), Host: record.Host, Port: uint16(record.Port),
|
|
Username: record.Username, Password: record.Password, Level: healthDomain.Level(task.Level),
|
|
Deadline: time.UnixMilli(task.DeadlineMS).UTC(), Attempts: task.Attempts,
|
|
})
|
|
}
|
|
return result, nil
|
|
}
|
|
|
|
func (a *Adapter) AuthorizeObservation(
|
|
ctx context.Context,
|
|
checkerID string,
|
|
leaseToken string,
|
|
observation healthDomain.Observation,
|
|
now time.Time,
|
|
) error {
|
|
return a.authorizeHealthTask(ctx, "authorize", checkerID, leaseToken, observation, now)
|
|
}
|
|
|
|
func (a *Adapter) CompleteObservation(
|
|
ctx context.Context,
|
|
checkerID string,
|
|
leaseToken string,
|
|
observation healthDomain.Observation,
|
|
now time.Time,
|
|
) error {
|
|
return a.authorizeHealthTask(ctx, "complete", checkerID, leaseToken, observation, now)
|
|
}
|
|
|
|
func (a *Adapter) authorizeHealthTask(
|
|
ctx context.Context,
|
|
operation string,
|
|
checkerID string,
|
|
leaseToken string,
|
|
observation healthDomain.Observation,
|
|
now time.Time,
|
|
) error {
|
|
if a == nil || ctx == nil || !validHealthTaskIdentifier(checkerID) || !validHealthTaskIdentifier(leaseToken) || now.IsZero() {
|
|
return healthDomain.ErrInvalidTaskBroker
|
|
}
|
|
if err := ctx.Err(); err != nil {
|
|
return err
|
|
}
|
|
normalized, err := healthDomain.NormalizeObservation(observation)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
reply, err := a.runHealthTaskScript(ctx, operation, now, healthTaskRequest{
|
|
Limit: a.options.MaxCheckTasks,
|
|
Fact: &healthTaskFact{TaskID: normalized.TaskID, ProxyID: normalized.ProxyID, Level: string(normalized.Level),
|
|
RoutingName: normalized.RoutingName, TargetURL: normalized.TargetURL, CheckerID: checkerID, LeaseToken: leaseToken},
|
|
}, checkerID)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
switch reply.Status {
|
|
case scriptOK:
|
|
return nil
|
|
case scriptNotFound:
|
|
return healthDomain.ErrTaskNotFound
|
|
case scriptInvalid:
|
|
return healthDomain.ErrInvalidLeasedTask
|
|
case scriptStatus("lease_expired"):
|
|
return healthDomain.ErrTaskLeaseExpired
|
|
case scriptStatus("not_owned"):
|
|
return healthDomain.ErrTaskLeaseNotOwned
|
|
case scriptStatus("observation"):
|
|
return healthDomain.ErrTaskObservation
|
|
default:
|
|
return invalidScriptReply("unexpected health task authorization status")
|
|
}
|
|
}
|
|
|
|
func (a *Adapter) runHealthTaskScript(
|
|
ctx context.Context,
|
|
operation string,
|
|
now time.Time,
|
|
request healthTaskRequest,
|
|
checkerID string,
|
|
) (healthTaskScriptReply, error) {
|
|
if a == nil || ctx == nil || now.IsZero() || request.Limit <= 0 || strings.TrimSpace(operation) != operation || operation == "" {
|
|
return healthTaskScriptReply{}, healthDomain.ErrInvalidTaskBroker
|
|
}
|
|
payload, err := json.Marshal(request)
|
|
if err != nil {
|
|
return healthTaskScriptReply{}, fmt.Errorf("encode health task request: %w", err)
|
|
}
|
|
result, err := runScript(ctx, a.client, healthTasksScript, []string{
|
|
a.keys.records, a.keys.expiry, a.keys.healthDue, a.keys.healthQueued, a.keys.healthLeases,
|
|
a.keys.healthTasks, a.keys.healthTaskExpiry, a.keys.healthRefTask, a.keys.checkerLeases(checkerID),
|
|
a.keys.stateInventory,
|
|
}, operation, now.UTC().UnixMilli(), string(payload))
|
|
if err != nil {
|
|
return healthTaskScriptReply{}, err
|
|
}
|
|
var reply healthTaskScriptReply
|
|
if err := decodeScriptResult(result, &reply); err != nil {
|
|
return healthTaskScriptReply{}, err
|
|
}
|
|
return reply, nil
|
|
}
|
|
|
|
func validateRedisPlannedTask(task healthDomain.PlannedTask, now time.Time) error {
|
|
if !validHealthTaskIdentifier(task.Candidate.ProxyID) || task.Candidate.Level != healthDomain.LevelBasic ||
|
|
task.Candidate.RoutingName != "" || task.Candidate.TargetURL != "" || task.Candidate.DueAt.IsZero() ||
|
|
task.Deadline.IsZero() || !task.Deadline.After(now) || task.NextDue.IsZero() || !task.NextDue.After(now) || task.Attempts <= 0 {
|
|
return healthDomain.ErrInvalidLeasedTask
|
|
}
|
|
switch task.Candidate.State {
|
|
case proxyDomain.StateFetched, proxyDomain.StateAvailable, proxyDomain.StateSuspect, proxyDomain.StateUnhealthy:
|
|
return nil
|
|
default:
|
|
return healthDomain.ErrInvalidLeasedTask
|
|
}
|
|
}
|
|
|
|
func validateRedisTaskClaim(claim healthDomain.TaskClaim, maximum int) error {
|
|
if !validHealthTaskIdentifier(claim.CheckerID) || !validHealthTaskIdentifier(claim.InstanceID) || claim.MaxInFlight <= 0 ||
|
|
claim.MaxInFlight > maximum || len(claim.SupportedLevels) == 0 {
|
|
return healthDomain.ErrInvalidTaskClaim
|
|
}
|
|
levels := make(map[healthDomain.Level]struct{}, len(claim.SupportedLevels))
|
|
for _, level := range claim.SupportedLevels {
|
|
switch level {
|
|
case healthDomain.LevelBasic, healthDomain.LevelEgress, healthDomain.LevelTarget:
|
|
default:
|
|
return healthDomain.ErrInvalidTaskClaim
|
|
}
|
|
if _, exists := levels[level]; exists {
|
|
return healthDomain.ErrInvalidTaskClaim
|
|
}
|
|
levels[level] = struct{}{}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func decodeHealthTaskRecord(payload string) (healthTaskRecord, error) {
|
|
var record healthTaskRecord
|
|
if err := decodeJSON(payload, &record); err != nil {
|
|
return healthTaskRecord{}, err
|
|
}
|
|
if record.Version != healthTaskRecordVersion || !validHealthTaskIdentifier(record.TaskID) ||
|
|
!validHealthTaskIdentifier(record.ProxyID) || record.Level != string(healthDomain.LevelBasic) ||
|
|
record.Priority < int(healthDomain.PriorityFetched) || record.Priority > int(healthDomain.PriorityAvailable) ||
|
|
record.DeadlineMS <= 0 || record.NextDueMS <= 0 || record.Attempts <= 0 ||
|
|
(record.State != "QUEUED" && record.State != "LEASED" && record.State != "DONE") {
|
|
return healthTaskRecord{}, errors.New("invalid health task record")
|
|
}
|
|
return record, nil
|
|
}
|
|
|
|
func newRedisLeaseToken() (string, error) {
|
|
var entropy [24]byte
|
|
if _, err := rand.Read(entropy[:]); err != nil {
|
|
return "", err
|
|
}
|
|
return "lease_" + hex.EncodeToString(entropy[:]), nil
|
|
}
|
|
|
|
func validHealthTaskIdentifier(value string) bool {
|
|
if value == "" || len(value) > 256 || strings.TrimSpace(value) != value {
|
|
return false
|
|
}
|
|
for _, character := range value {
|
|
if character <= ' ' || character == '\x7f' {
|
|
return false
|
|
}
|
|
}
|
|
return true
|
|
}
|