407 lines
15 KiB
Go
407 lines
15 KiB
Go
package redisactivity
|
|
|
|
import (
|
|
"encoding/hex"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"strings"
|
|
"time"
|
|
|
|
extractionDomain "proxy-pool/internal/domain/extraction"
|
|
proxyDomain "proxy-pool/internal/domain/proxy"
|
|
)
|
|
|
|
var ErrInvalidRecord = errors.New("invalid redis activity record")
|
|
|
|
const recordVersion = 1
|
|
|
|
type proxyRecord struct {
|
|
Version int `json:"version"`
|
|
ID string `json:"id"`
|
|
Scheme string `json:"scheme"`
|
|
Host string `json:"host"`
|
|
Port int64 `json:"port"`
|
|
Username string `json:"username,omitempty"`
|
|
Password string `json:"password,omitempty"`
|
|
CredentialVersion string `json:"credentialVersion,omitempty"`
|
|
SourceUpstream string `json:"sourceUpstream"`
|
|
CreatedAtMS int64 `json:"createdAtMs"`
|
|
ExpiresAtMS int64 `json:"expiresAtMs"`
|
|
UsableUntilMS int64 `json:"usableUntilMs"`
|
|
LastCheckedAtMS int64 `json:"lastCheckedAtMs,omitempty"`
|
|
LastSuccessAtMS int64 `json:"lastSuccessAtMs,omitempty"`
|
|
ConsecutiveFailures int `json:"consecutiveFailures,omitempty"`
|
|
UnhealthySinceMS int64 `json:"unhealthySinceMs,omitempty"`
|
|
LastHealthTaskID string `json:"lastHealthTaskId,omitempty"`
|
|
LastHealthDigest string `json:"lastHealthDigest,omitempty"`
|
|
LastHealthObservedAtMS int64 `json:"lastHealthObservedAtMs,omitempty"`
|
|
LatencyNS int64 `json:"latencyNs"`
|
|
MaxConcurrency int64 `json:"maxConcurrency"`
|
|
State string `json:"state"`
|
|
Tags map[string]string `json:"tags,omitempty"`
|
|
OwnerWorkerID string `json:"ownerWorkerId,omitempty"`
|
|
OwnerIndexKey string `json:"ownerIndexKey"`
|
|
IndexKeys []string `json:"indexKeys,omitempty"`
|
|
}
|
|
|
|
type ownershipRecord struct {
|
|
Version int `json:"version"`
|
|
ProxyID string `json:"proxyId"`
|
|
WorkerID string `json:"workerId"`
|
|
WorkerIndexKey string `json:"workerIndexKey,omitempty"`
|
|
Epoch uint64 `json:"epoch"`
|
|
AssignmentVersion uint64 `json:"assignmentVersion"`
|
|
ExpiresAtMS int64 `json:"expiresAtMs"`
|
|
Draining bool `json:"draining"`
|
|
}
|
|
|
|
type drainTicketRecord struct {
|
|
Version int `json:"version"`
|
|
ProxyID string `json:"proxyId"`
|
|
WorkerID string `json:"workerId"`
|
|
WorkerIndexKey string `json:"workerIndexKey"`
|
|
AssignmentEpoch uint64 `json:"assignmentEpoch"`
|
|
RequiredSnapshotEpoch uint64 `json:"requiredSnapshotEpoch"`
|
|
SessionID string `json:"sessionId,omitempty"`
|
|
SnapshotVersion uint64 `json:"snapshotVersion,omitempty"`
|
|
SnapshotOwnershipEpoch uint64 `json:"snapshotOwnershipEpoch,omitempty"`
|
|
SnapshotChecksum string `json:"snapshotChecksum,omitempty"`
|
|
}
|
|
|
|
type idempotencyRecord struct {
|
|
Version int
|
|
RequestDigest string
|
|
ExpiresAtMS int64
|
|
Result extractionDomain.Result
|
|
}
|
|
|
|
type idempotencyWire struct {
|
|
Version int `json:"version"`
|
|
RequestDigest string `json:"requestDigest"`
|
|
ExpiresAtMS int64 `json:"expiresAtMs"`
|
|
Result extractionWire `json:"result"`
|
|
}
|
|
|
|
type extractionWire struct {
|
|
Requested int `json:"requested"`
|
|
Returned int `json:"returned"`
|
|
ExtractedAtMS int64 `json:"extractedAtMs,omitempty"`
|
|
Items []candidateWire `json:"items"`
|
|
}
|
|
|
|
type candidateWire struct {
|
|
ID string `json:"id"`
|
|
Protocol string `json:"protocol"`
|
|
Host string `json:"host"`
|
|
Port int64 `json:"port"`
|
|
Username string `json:"username,omitempty"`
|
|
Password string `json:"password,omitempty"`
|
|
Region string `json:"region,omitempty"`
|
|
Carrier string `json:"carrier,omitempty"`
|
|
Upstream string `json:"upstream"`
|
|
OwnerWorkerID string `json:"ownerWorkerId,omitempty"`
|
|
URL string `json:"url,omitempty"`
|
|
State string `json:"state"`
|
|
ExpiresAtMS int64 `json:"expiresAtMs"`
|
|
CheckedAtMS int64 `json:"lastCheckedAtMs,omitempty"`
|
|
}
|
|
|
|
func (record proxyRecord) Format(state fmt.State, _ rune) {
|
|
_, _ = fmt.Fprintf(state,
|
|
"redisactivity.proxyRecord{ID:%q, Scheme:%q, Host:%q, Port:%d, Username:%q, Password:<redacted>, State:%q}",
|
|
record.ID, record.Scheme, record.Host, record.Port, record.Username, record.State,
|
|
)
|
|
}
|
|
|
|
func (record ownershipRecord) Format(state fmt.State, _ rune) {
|
|
_, _ = fmt.Fprintf(state,
|
|
"redisactivity.ownershipRecord{ProxyID:%q, WorkerID:%q, Epoch:%d, Version:%d, Draining:%t}",
|
|
record.ProxyID, record.WorkerID, record.Epoch, record.AssignmentVersion, record.Draining,
|
|
)
|
|
}
|
|
|
|
func (record idempotencyRecord) Format(state fmt.State, _ rune) {
|
|
_, _ = fmt.Fprintf(state,
|
|
"redisactivity.idempotencyRecord{RequestDigest:%q, Returned:%d, Credentials:<redacted>}",
|
|
record.RequestDigest, record.Result.Returned,
|
|
)
|
|
}
|
|
|
|
func encodeProxyRecord(record proxyRecord) (string, error) {
|
|
if err := validateProxyRecord(record); err != nil {
|
|
return "", err
|
|
}
|
|
return encodeJSON(record)
|
|
}
|
|
|
|
func decodeProxyRecord(payload string) (proxyRecord, error) {
|
|
var record proxyRecord
|
|
if err := decodeJSON(payload, &record); err != nil {
|
|
return proxyRecord{}, err
|
|
}
|
|
if err := validateProxyRecord(record); err != nil {
|
|
return proxyRecord{}, err
|
|
}
|
|
return record, nil
|
|
}
|
|
|
|
func encodeOwnershipRecord(record ownershipRecord) (string, error) {
|
|
if err := validateOwnershipRecord(record); err != nil {
|
|
return "", err
|
|
}
|
|
return encodeJSON(record)
|
|
}
|
|
|
|
func decodeOwnershipRecord(payload string) (ownershipRecord, error) {
|
|
var record ownershipRecord
|
|
if err := decodeJSON(payload, &record); err != nil {
|
|
return ownershipRecord{}, err
|
|
}
|
|
if err := validateOwnershipRecord(record); err != nil {
|
|
return ownershipRecord{}, err
|
|
}
|
|
return record, nil
|
|
}
|
|
|
|
func encodeIdempotencyRecord(record idempotencyRecord) (string, error) {
|
|
if err := validateIdempotencyRecord(record); err != nil {
|
|
return "", err
|
|
}
|
|
wire := idempotencyWire{
|
|
Version: record.Version, RequestDigest: record.RequestDigest, ExpiresAtMS: record.ExpiresAtMS,
|
|
Result: extractionToWire(record.Result),
|
|
}
|
|
return encodeJSON(wire)
|
|
}
|
|
|
|
func decodeIdempotencyRecord(payload string) (idempotencyRecord, error) {
|
|
var wire idempotencyWire
|
|
if err := decodeJSON(payload, &wire); err != nil {
|
|
return idempotencyRecord{}, err
|
|
}
|
|
if err := validateIdempotencyWire(wire); err != nil {
|
|
return idempotencyRecord{}, err
|
|
}
|
|
record := idempotencyRecord{
|
|
Version: wire.Version, RequestDigest: wire.RequestDigest, ExpiresAtMS: wire.ExpiresAtMS,
|
|
Result: extractionFromWire(wire.Result),
|
|
}
|
|
if err := validateIdempotencyRecord(record); err != nil {
|
|
return idempotencyRecord{}, err
|
|
}
|
|
return record, nil
|
|
}
|
|
|
|
func encodeJSON(value any) (string, error) {
|
|
payload, err := json.Marshal(value)
|
|
if err != nil {
|
|
return "", errors.Join(ErrInvalidRecord, err)
|
|
}
|
|
return string(payload), nil
|
|
}
|
|
|
|
func decodeJSON(payload string, destination any) error {
|
|
decoder := json.NewDecoder(strings.NewReader(payload))
|
|
decoder.DisallowUnknownFields()
|
|
if err := decoder.Decode(destination); err != nil {
|
|
return errors.Join(ErrInvalidRecord, err)
|
|
}
|
|
if err := decoder.Decode(&struct{}{}); !errors.Is(err, io.EOF) {
|
|
if err == nil {
|
|
err = errors.New("multiple JSON values")
|
|
}
|
|
return errors.Join(ErrInvalidRecord, err)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func validateProxyRecord(record proxyRecord) error {
|
|
if record.Version != recordVersion || record.ID == "" || record.Host == "" ||
|
|
record.Port <= 0 || record.Port > 65_535 || record.SourceUpstream == "" ||
|
|
record.CreatedAtMS <= 0 || record.ExpiresAtMS <= 0 || record.UsableUntilMS <= 0 ||
|
|
record.UsableUntilMS > record.ExpiresAtMS || record.LastCheckedAtMS < 0 ||
|
|
record.LastSuccessAtMS < 0 || record.ConsecutiveFailures < 0 || record.LatencyNS < 0 || record.MaxConcurrency < 0 ||
|
|
!validScheme(record.Scheme) || !validProxyState(record.State) ||
|
|
record.OwnerIndexKey == "" || !strings.Contains(record.OwnerIndexKey, "{activity}") {
|
|
return ErrInvalidRecord
|
|
}
|
|
if record.UnhealthySinceMS < 0 || (record.UnhealthySinceMS > 0 && record.State != "UNHEALTHY" && record.State != "CHECKING") ||
|
|
(record.State == "UNHEALTHY" && record.UnhealthySinceMS == 0) ||
|
|
(record.LastHealthTaskID == "") != (record.LastHealthDigest == "") ||
|
|
(record.LastHealthDigest != "" && (!validDigest(record.LastHealthDigest) || record.LastHealthObservedAtMS <= 0)) ||
|
|
record.LastHealthObservedAtMS < 0 {
|
|
return ErrInvalidRecord
|
|
}
|
|
for _, key := range record.IndexKeys {
|
|
if key == "" || !strings.Contains(key, "{activity}") {
|
|
return ErrInvalidRecord
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func validateOwnershipRecord(record ownershipRecord) error {
|
|
if record.Version != recordVersion || record.ProxyID == "" || record.WorkerID == "" ||
|
|
record.Epoch == 0 || record.AssignmentVersion == 0 || record.ExpiresAtMS <= 0 {
|
|
return ErrInvalidRecord
|
|
}
|
|
if record.WorkerIndexKey != "" && !strings.Contains(record.WorkerIndexKey, "{activity}") {
|
|
return ErrInvalidRecord
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func validateDrainTicketRecord(record drainTicketRecord) error {
|
|
if record.Version != recordVersion || record.ProxyID == "" || record.WorkerID == "" ||
|
|
record.AssignmentEpoch == 0 || record.RequiredSnapshotEpoch == 0 ||
|
|
record.RequiredSnapshotEpoch <= record.AssignmentEpoch || record.WorkerIndexKey == "" ||
|
|
!strings.Contains(record.WorkerIndexKey, "{activity}") {
|
|
return ErrInvalidRecord
|
|
}
|
|
bound := record.SnapshotVersion != 0 || record.SnapshotOwnershipEpoch != 0 || record.SessionID != "" || record.SnapshotChecksum != ""
|
|
if !bound {
|
|
return nil
|
|
}
|
|
if record.SessionID == "" || record.SnapshotVersion == 0 || record.SnapshotOwnershipEpoch < record.RequiredSnapshotEpoch ||
|
|
!validDigest(record.SnapshotChecksum) {
|
|
return ErrInvalidRecord
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func validateIdempotencyRecord(record idempotencyRecord) error {
|
|
if record.Version != recordVersion {
|
|
return invalidRecord("unsupported idempotency version")
|
|
}
|
|
if !validDigest(record.RequestDigest) {
|
|
return invalidRecord("invalid request digest")
|
|
}
|
|
if record.ExpiresAtMS <= 0 {
|
|
return invalidRecord("invalid idempotency expiry")
|
|
}
|
|
if record.Result.Requested < 0 || record.Result.Returned < 0 ||
|
|
record.Result.Returned != len(record.Result.Items) || record.Result.Returned > record.Result.Requested {
|
|
return invalidRecord("invalid extraction counters")
|
|
}
|
|
if record.Result.Returned > 0 && record.Result.ExtractedAt.IsZero() {
|
|
return invalidRecord("missing extraction time")
|
|
}
|
|
for _, candidate := range record.Result.Items {
|
|
if candidate.ID == "" || candidate.Host == "" || candidate.Port == 0 || candidate.Upstream == "" {
|
|
return invalidRecord("incomplete extraction candidate identity")
|
|
}
|
|
if !validScheme(candidate.Protocol) || candidate.State != extractionDomain.Extracted {
|
|
return invalidRecord("invalid extraction candidate state")
|
|
}
|
|
if candidate.ExpiresAt.IsZero() || candidate.ExpiresAt.UnixMilli() <= 0 ||
|
|
(!candidate.LastCheckedAt.IsZero() && candidate.LastCheckedAt.UnixMilli() <= 0) {
|
|
return invalidRecord("invalid extraction candidate time")
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func validateIdempotencyWire(wire idempotencyWire) error {
|
|
if wire.Version != recordVersion || !validDigest(wire.RequestDigest) || wire.ExpiresAtMS <= 0 {
|
|
return invalidRecord("invalid idempotency wire header")
|
|
}
|
|
if wire.Result.Requested < 0 || wire.Result.Returned < 0 ||
|
|
wire.Result.Returned != len(wire.Result.Items) || wire.Result.Returned > wire.Result.Requested {
|
|
return invalidRecord("invalid extraction wire counters")
|
|
}
|
|
if wire.Result.Returned > 0 && wire.Result.ExtractedAtMS <= 0 {
|
|
return invalidRecord("invalid extraction wire time")
|
|
}
|
|
for _, candidate := range wire.Result.Items {
|
|
if candidate.ID == "" || candidate.Host == "" || candidate.Port <= 0 || candidate.Port > 65_535 ||
|
|
candidate.Upstream == "" || !validScheme(candidate.Protocol) ||
|
|
extractionDomain.State(candidate.State) != extractionDomain.Extracted ||
|
|
candidate.ExpiresAtMS <= 0 || candidate.CheckedAtMS < 0 {
|
|
return invalidRecord("invalid extraction wire candidate")
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func invalidRecord(reason string) error {
|
|
return fmt.Errorf("%w: %s", ErrInvalidRecord, reason)
|
|
}
|
|
|
|
func extractionToWire(result extractionDomain.Result) extractionWire {
|
|
wire := extractionWire{
|
|
Requested: result.Requested, Returned: result.Returned,
|
|
Items: make([]candidateWire, 0, len(result.Items)),
|
|
}
|
|
if !result.ExtractedAt.IsZero() {
|
|
wire.ExtractedAtMS = result.ExtractedAt.UnixMilli()
|
|
}
|
|
for _, candidate := range result.Items {
|
|
item := candidateWire{
|
|
ID: candidate.ID, Protocol: candidate.Protocol, Host: candidate.Host, Port: int64(candidate.Port),
|
|
Username: candidate.Username, Password: candidate.Password, Region: candidate.Region,
|
|
Carrier: candidate.Carrier, Upstream: candidate.Upstream, OwnerWorkerID: candidate.OwnerWorkerID,
|
|
URL: candidate.URL, State: string(candidate.State), ExpiresAtMS: candidate.ExpiresAt.UnixMilli(),
|
|
}
|
|
if !candidate.LastCheckedAt.IsZero() {
|
|
item.CheckedAtMS = candidate.LastCheckedAt.UnixMilli()
|
|
}
|
|
wire.Items = append(wire.Items, item)
|
|
}
|
|
return wire
|
|
}
|
|
|
|
func extractionFromWire(wire extractionWire) extractionDomain.Result {
|
|
result := extractionDomain.Result{
|
|
Requested: wire.Requested, Returned: wire.Returned,
|
|
Items: make([]extractionDomain.Candidate, 0, len(wire.Items)),
|
|
}
|
|
if wire.ExtractedAtMS > 0 {
|
|
result.ExtractedAt = time.UnixMilli(wire.ExtractedAtMS).UTC()
|
|
}
|
|
for _, item := range wire.Items {
|
|
candidate := extractionDomain.Candidate{
|
|
ID: item.ID, Protocol: item.Protocol, Host: item.Host, Port: uint16(item.Port),
|
|
Username: item.Username, Password: item.Password, Region: item.Region,
|
|
Carrier: item.Carrier, Upstream: item.Upstream, OwnerWorkerID: item.OwnerWorkerID,
|
|
URL: item.URL, State: extractionDomain.State(item.State), ExpiresAt: time.UnixMilli(item.ExpiresAtMS).UTC(),
|
|
}
|
|
if item.CheckedAtMS > 0 {
|
|
candidate.LastCheckedAt = time.UnixMilli(item.CheckedAtMS).UTC()
|
|
}
|
|
result.Items = append(result.Items, candidate)
|
|
}
|
|
return result
|
|
}
|
|
|
|
func validDigest(value string) bool {
|
|
if len(value) != sha256HexSize {
|
|
return false
|
|
}
|
|
_, err := hex.DecodeString(value)
|
|
return err == nil
|
|
}
|
|
|
|
func validScheme(value string) bool {
|
|
switch proxyDomain.Scheme(value) {
|
|
case proxyDomain.SchemeHTTP, proxyDomain.SchemeHTTPS, proxyDomain.SchemeSOCKS5:
|
|
return true
|
|
default:
|
|
return false
|
|
}
|
|
}
|
|
|
|
func validProxyState(value string) bool {
|
|
switch proxyDomain.State(value) {
|
|
case proxyDomain.StateFetched, proxyDomain.StateChecking, proxyDomain.StateAvailable,
|
|
proxyDomain.StateSuspect, proxyDomain.StateDraining, proxyDomain.StateUnhealthy,
|
|
proxyDomain.StateExtracted, proxyDomain.StateExpired, proxyDomain.StateRemoved:
|
|
return true
|
|
default:
|
|
return false
|
|
}
|
|
}
|
|
|
|
const sha256HexSize = 64
|