package redisactivity import ( "encoding/hex" "encoding/json" "errors" "fmt" "io" "strings" "time" extractionDomain "proxy-pool/internal/domain/extraction" proxyDomain "proxy-pool/internal/domain/proxy" ) var ErrInvalidRecord = errors.New("invalid redis activity record") const recordVersion = 1 type proxyRecord struct { Version int `json:"version"` ID string `json:"id"` Scheme string `json:"scheme"` Host string `json:"host"` Port int64 `json:"port"` Username string `json:"username,omitempty"` Password string `json:"password,omitempty"` CredentialVersion string `json:"credentialVersion,omitempty"` SourceUpstream string `json:"sourceUpstream"` CreatedAtMS int64 `json:"createdAtMs"` ExpiresAtMS int64 `json:"expiresAtMs"` UsableUntilMS int64 `json:"usableUntilMs"` LastCheckedAtMS int64 `json:"lastCheckedAtMs,omitempty"` LastSuccessAtMS int64 `json:"lastSuccessAtMs,omitempty"` LatencyNS int64 `json:"latencyNs"` MaxConcurrency int64 `json:"maxConcurrency"` State string `json:"state"` Tags map[string]string `json:"tags,omitempty"` OwnerWorkerID string `json:"ownerWorkerId,omitempty"` OwnerIndexKey string `json:"ownerIndexKey"` IndexKeys []string `json:"indexKeys,omitempty"` } type ownershipRecord struct { Version int `json:"version"` ProxyID string `json:"proxyId"` WorkerID string `json:"workerId"` Epoch uint64 `json:"epoch"` AssignmentVersion uint64 `json:"assignmentVersion"` ExpiresAtMS int64 `json:"expiresAtMs"` Draining bool `json:"draining"` } type idempotencyRecord struct { Version int RequestDigest string ExpiresAtMS int64 Result extractionDomain.Result } type idempotencyWire struct { Version int `json:"version"` RequestDigest string `json:"requestDigest"` ExpiresAtMS int64 `json:"expiresAtMs"` Result extractionWire `json:"result"` } type extractionWire struct { Requested int `json:"requested"` Returned int `json:"returned"` ExtractedAtMS int64 `json:"extractedAtMs,omitempty"` Items []candidateWire `json:"items"` } type candidateWire struct { ID string `json:"id"` Protocol string `json:"protocol"` Host string `json:"host"` Port int64 `json:"port"` Username string `json:"username,omitempty"` Password string `json:"password,omitempty"` Region string `json:"region,omitempty"` Carrier string `json:"carrier,omitempty"` Upstream string `json:"upstream"` OwnerWorkerID string `json:"ownerWorkerId,omitempty"` URL string `json:"url,omitempty"` State string `json:"state"` ExpiresAtMS int64 `json:"expiresAtMs"` CheckedAtMS int64 `json:"lastCheckedAtMs,omitempty"` } func (record proxyRecord) Format(state fmt.State, _ rune) { _, _ = fmt.Fprintf(state, "redisactivity.proxyRecord{ID:%q, Scheme:%q, Host:%q, Port:%d, Username:%q, Password:, State:%q}", record.ID, record.Scheme, record.Host, record.Port, record.Username, record.State, ) } func (record ownershipRecord) Format(state fmt.State, _ rune) { _, _ = fmt.Fprintf(state, "redisactivity.ownershipRecord{ProxyID:%q, WorkerID:%q, Epoch:%d, Version:%d, Draining:%t}", record.ProxyID, record.WorkerID, record.Epoch, record.AssignmentVersion, record.Draining, ) } func (record idempotencyRecord) Format(state fmt.State, _ rune) { _, _ = fmt.Fprintf(state, "redisactivity.idempotencyRecord{RequestDigest:%q, Returned:%d, Credentials:}", record.RequestDigest, record.Result.Returned, ) } func encodeProxyRecord(record proxyRecord) (string, error) { if err := validateProxyRecord(record); err != nil { return "", err } return encodeJSON(record) } func decodeProxyRecord(payload string) (proxyRecord, error) { var record proxyRecord if err := decodeJSON(payload, &record); err != nil { return proxyRecord{}, err } if err := validateProxyRecord(record); err != nil { return proxyRecord{}, err } return record, nil } func encodeOwnershipRecord(record ownershipRecord) (string, error) { if err := validateOwnershipRecord(record); err != nil { return "", err } return encodeJSON(record) } func decodeOwnershipRecord(payload string) (ownershipRecord, error) { var record ownershipRecord if err := decodeJSON(payload, &record); err != nil { return ownershipRecord{}, err } if err := validateOwnershipRecord(record); err != nil { return ownershipRecord{}, err } return record, nil } func encodeIdempotencyRecord(record idempotencyRecord) (string, error) { if err := validateIdempotencyRecord(record); err != nil { return "", err } wire := idempotencyWire{ Version: record.Version, RequestDigest: record.RequestDigest, ExpiresAtMS: record.ExpiresAtMS, Result: extractionToWire(record.Result), } return encodeJSON(wire) } func decodeIdempotencyRecord(payload string) (idempotencyRecord, error) { var wire idempotencyWire if err := decodeJSON(payload, &wire); err != nil { return idempotencyRecord{}, err } if err := validateIdempotencyWire(wire); err != nil { return idempotencyRecord{}, err } record := idempotencyRecord{ Version: wire.Version, RequestDigest: wire.RequestDigest, ExpiresAtMS: wire.ExpiresAtMS, Result: extractionFromWire(wire.Result), } if err := validateIdempotencyRecord(record); err != nil { return idempotencyRecord{}, err } return record, nil } func encodeJSON(value any) (string, error) { payload, err := json.Marshal(value) if err != nil { return "", errors.Join(ErrInvalidRecord, err) } return string(payload), nil } func decodeJSON(payload string, destination any) error { decoder := json.NewDecoder(strings.NewReader(payload)) decoder.DisallowUnknownFields() if err := decoder.Decode(destination); err != nil { return errors.Join(ErrInvalidRecord, err) } if err := decoder.Decode(&struct{}{}); !errors.Is(err, io.EOF) { if err == nil { err = errors.New("multiple JSON values") } return errors.Join(ErrInvalidRecord, err) } return nil } func validateProxyRecord(record proxyRecord) error { if record.Version != recordVersion || record.ID == "" || record.Host == "" || record.Port <= 0 || record.Port > 65_535 || record.SourceUpstream == "" || record.CreatedAtMS <= 0 || record.ExpiresAtMS <= 0 || record.UsableUntilMS <= 0 || record.UsableUntilMS > record.ExpiresAtMS || record.LastCheckedAtMS < 0 || record.LastSuccessAtMS < 0 || record.LatencyNS < 0 || record.MaxConcurrency < 0 || !validScheme(record.Scheme) || !validProxyState(record.State) || record.OwnerIndexKey == "" || !strings.Contains(record.OwnerIndexKey, "{activity}") { return ErrInvalidRecord } for _, key := range record.IndexKeys { if key == "" || !strings.Contains(key, "{activity}") { return ErrInvalidRecord } } return nil } func validateOwnershipRecord(record ownershipRecord) error { if record.Version != recordVersion || record.ProxyID == "" || record.WorkerID == "" || record.Epoch == 0 || record.AssignmentVersion == 0 || record.ExpiresAtMS <= 0 { return ErrInvalidRecord } return nil } func validateIdempotencyRecord(record idempotencyRecord) error { if record.Version != recordVersion { return invalidRecord("unsupported idempotency version") } if !validDigest(record.RequestDigest) { return invalidRecord("invalid request digest") } if record.ExpiresAtMS <= 0 { return invalidRecord("invalid idempotency expiry") } if record.Result.Requested < 0 || record.Result.Returned < 0 || record.Result.Returned != len(record.Result.Items) || record.Result.Returned > record.Result.Requested { return invalidRecord("invalid extraction counters") } if record.Result.Returned > 0 && record.Result.ExtractedAt.IsZero() { return invalidRecord("missing extraction time") } for _, candidate := range record.Result.Items { if candidate.ID == "" || candidate.Host == "" || candidate.Port == 0 || candidate.Upstream == "" { return invalidRecord("incomplete extraction candidate identity") } if !validScheme(candidate.Protocol) || candidate.State != extractionDomain.Extracted { return invalidRecord("invalid extraction candidate state") } if candidate.ExpiresAt.IsZero() || candidate.ExpiresAt.UnixMilli() <= 0 || (!candidate.LastCheckedAt.IsZero() && candidate.LastCheckedAt.UnixMilli() <= 0) { return invalidRecord("invalid extraction candidate time") } } return nil } func validateIdempotencyWire(wire idempotencyWire) error { if wire.Version != recordVersion || !validDigest(wire.RequestDigest) || wire.ExpiresAtMS <= 0 { return invalidRecord("invalid idempotency wire header") } if wire.Result.Requested < 0 || wire.Result.Returned < 0 || wire.Result.Returned != len(wire.Result.Items) || wire.Result.Returned > wire.Result.Requested { return invalidRecord("invalid extraction wire counters") } if wire.Result.Returned > 0 && wire.Result.ExtractedAtMS <= 0 { return invalidRecord("invalid extraction wire time") } for _, candidate := range wire.Result.Items { if candidate.ID == "" || candidate.Host == "" || candidate.Port <= 0 || candidate.Port > 65_535 || candidate.Upstream == "" || !validScheme(candidate.Protocol) || extractionDomain.State(candidate.State) != extractionDomain.Extracted || candidate.ExpiresAtMS <= 0 || candidate.CheckedAtMS < 0 { return invalidRecord("invalid extraction wire candidate") } } return nil } func invalidRecord(reason string) error { return fmt.Errorf("%w: %s", ErrInvalidRecord, reason) } func extractionToWire(result extractionDomain.Result) extractionWire { wire := extractionWire{ Requested: result.Requested, Returned: result.Returned, Items: make([]candidateWire, 0, len(result.Items)), } if !result.ExtractedAt.IsZero() { wire.ExtractedAtMS = result.ExtractedAt.UnixMilli() } for _, candidate := range result.Items { item := candidateWire{ ID: candidate.ID, Protocol: candidate.Protocol, Host: candidate.Host, Port: int64(candidate.Port), Username: candidate.Username, Password: candidate.Password, Region: candidate.Region, Carrier: candidate.Carrier, Upstream: candidate.Upstream, OwnerWorkerID: candidate.OwnerWorkerID, URL: candidate.URL, State: string(candidate.State), ExpiresAtMS: candidate.ExpiresAt.UnixMilli(), } if !candidate.LastCheckedAt.IsZero() { item.CheckedAtMS = candidate.LastCheckedAt.UnixMilli() } wire.Items = append(wire.Items, item) } return wire } func extractionFromWire(wire extractionWire) extractionDomain.Result { result := extractionDomain.Result{ Requested: wire.Requested, Returned: wire.Returned, Items: make([]extractionDomain.Candidate, 0, len(wire.Items)), } if wire.ExtractedAtMS > 0 { result.ExtractedAt = time.UnixMilli(wire.ExtractedAtMS).UTC() } for _, item := range wire.Items { candidate := extractionDomain.Candidate{ ID: item.ID, Protocol: item.Protocol, Host: item.Host, Port: uint16(item.Port), Username: item.Username, Password: item.Password, Region: item.Region, Carrier: item.Carrier, Upstream: item.Upstream, OwnerWorkerID: item.OwnerWorkerID, URL: item.URL, State: extractionDomain.State(item.State), ExpiresAt: time.UnixMilli(item.ExpiresAtMS).UTC(), } if item.CheckedAtMS > 0 { candidate.LastCheckedAt = time.UnixMilli(item.CheckedAtMS).UTC() } result.Items = append(result.Items, candidate) } return result } func validDigest(value string) bool { if len(value) != sha256HexSize { return false } _, err := hex.DecodeString(value) return err == nil } func validScheme(value string) bool { switch proxyDomain.Scheme(value) { case proxyDomain.SchemeHTTP, proxyDomain.SchemeHTTPS, proxyDomain.SchemeSOCKS5: return true default: return false } } func validProxyState(value string) bool { switch proxyDomain.State(value) { case proxyDomain.StateFetched, proxyDomain.StateChecking, proxyDomain.StateAvailable, proxyDomain.StateSuspect, proxyDomain.StateDraining, proxyDomain.StateUnhealthy, proxyDomain.StateExtracted, proxyDomain.StateExpired, proxyDomain.StateRemoved: return true default: return false } } const sha256HexSize = 64