410 lines
15 KiB
Go
410 lines
15 KiB
Go
package distribution
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"net/http"
|
|
"strings"
|
|
"time"
|
|
|
|
controllerExtraction "proxy-pool/internal/controller/extraction"
|
|
"proxy-pool/internal/domain/authorization"
|
|
"proxy-pool/internal/domain/clientpolicy"
|
|
domainExtraction "proxy-pool/internal/domain/extraction"
|
|
"proxy-pool/internal/platform/httpapi"
|
|
"proxy-pool/internal/platform/httpsecurity"
|
|
)
|
|
|
|
const (
|
|
pathExtract = "/api/v1/proxies/extract"
|
|
pathLive = "/health/live"
|
|
pathReady = "/health/ready"
|
|
|
|
maxExtractCount = 1000
|
|
maxFilterValues = 64
|
|
minIdempotencyKeySize = 8
|
|
maxIdempotencyKeySize = 128
|
|
headerIdempotencyKey = "Idempotency-Key"
|
|
)
|
|
|
|
type Config struct {
|
|
BodyLimitBytes int64
|
|
}
|
|
|
|
type Dependencies struct {
|
|
Extractor Extractor
|
|
Identity IdentityResolver
|
|
Readiness ReadinessChecker
|
|
}
|
|
|
|
type Extractor interface {
|
|
Extract(context.Context, controllerExtraction.Request) (controllerExtraction.Response, error)
|
|
}
|
|
|
|
type Identity = httpsecurity.Identity
|
|
|
|
type IdentityResolver interface {
|
|
Resolve(*http.Request) (Identity, error)
|
|
}
|
|
|
|
var _ IdentityResolver = (*httpsecurity.Protection)(nil)
|
|
|
|
type ReadinessChecker interface {
|
|
Ready(context.Context) error
|
|
}
|
|
|
|
type Handler struct {
|
|
bodyLimitBytes int64
|
|
extractor Extractor
|
|
identity IdentityResolver
|
|
readiness ReadinessChecker
|
|
}
|
|
|
|
type extractRequestDTO struct {
|
|
Count int `json:"count"`
|
|
Fulfillment string `json:"fulfillment,omitempty"`
|
|
Filters *extractFiltersDTO `json:"filters,omitempty"`
|
|
}
|
|
|
|
type extractFiltersDTO struct {
|
|
Protocols []string `json:"protocols,omitempty"`
|
|
Regions []string `json:"regions,omitempty"`
|
|
Carriers []string `json:"carriers,omitempty"`
|
|
AllowedUpstreams []string `json:"allowedUpstreams,omitempty"`
|
|
}
|
|
|
|
type extractResponseDTO struct {
|
|
RequestID string `json:"requestId"`
|
|
Requested int `json:"requested"`
|
|
Returned int `json:"returned"`
|
|
Proxies []extractedProxyDTO `json:"proxies"`
|
|
}
|
|
|
|
type extractedProxyDTO struct {
|
|
ID string `json:"id"`
|
|
Protocol string `json:"protocol"`
|
|
Host string `json:"host"`
|
|
Port uint16 `json:"port"`
|
|
Username string `json:"username,omitempty"`
|
|
Password string `json:"password,omitempty"`
|
|
URL string `json:"url"`
|
|
Region string `json:"region,omitempty"`
|
|
Carrier string `json:"carrier,omitempty"`
|
|
Upstream string `json:"upstream"`
|
|
ExpiresAt string `json:"expiresAt"`
|
|
RemainingTTLSeconds int64 `json:"remainingTtlSeconds"`
|
|
ExtractedAt string `json:"extractedAt"`
|
|
}
|
|
|
|
type healthDTO struct {
|
|
Status string `json:"status"`
|
|
}
|
|
|
|
func NewHandler(config Config, deps Dependencies) (*Handler, error) {
|
|
switch {
|
|
case deps.Extractor == nil:
|
|
return nil, errors.New("create distribution handler: extractor is required")
|
|
case deps.Identity == nil:
|
|
return nil, errors.New("create distribution handler: identity resolver is required")
|
|
case deps.Readiness == nil:
|
|
return nil, errors.New("create distribution handler: readiness checker is required")
|
|
case config.BodyLimitBytes <= 0:
|
|
return nil, errors.New("create distribution handler: body limit must be greater than zero")
|
|
}
|
|
return &Handler{
|
|
bodyLimitBytes: config.BodyLimitBytes,
|
|
extractor: deps.Extractor,
|
|
identity: deps.Identity,
|
|
readiness: deps.Readiness,
|
|
}, nil
|
|
}
|
|
|
|
func (h *Handler) ServeHTTP(writer http.ResponseWriter, request *http.Request) {
|
|
requestID, requestIDErr := httpapi.ResolveRequestID(request)
|
|
if requestIDErr != nil {
|
|
h.writeProblem(writer, problemBadRequest(requestID, "INVALID_HEADER", "Invalid request header", "", nil))
|
|
return
|
|
}
|
|
|
|
switch request.URL.Path {
|
|
case pathExtract:
|
|
if request.Method != http.MethodPost {
|
|
writer.Header().Set("Allow", http.MethodPost)
|
|
h.writeProblem(writer, httpapi.NewProblem(http.StatusMethodNotAllowed, "METHOD_NOT_ALLOWED", "Method not allowed", "", requestID))
|
|
return
|
|
}
|
|
h.handleExtract(writer, request, requestID)
|
|
case pathLive:
|
|
if request.Method != http.MethodGet {
|
|
writer.Header().Set("Allow", http.MethodGet)
|
|
h.writeProblem(writer, httpapi.NewProblem(http.StatusMethodNotAllowed, "METHOD_NOT_ALLOWED", "Method not allowed", "", requestID))
|
|
return
|
|
}
|
|
h.writeJSON(writer, requestID, http.StatusOK, healthDTO{Status: "ok"})
|
|
case pathReady:
|
|
if request.Method != http.MethodGet {
|
|
writer.Header().Set("Allow", http.MethodGet)
|
|
h.writeProblem(writer, httpapi.NewProblem(http.StatusMethodNotAllowed, "METHOD_NOT_ALLOWED", "Method not allowed", "", requestID))
|
|
return
|
|
}
|
|
if err := h.readiness.Ready(request.Context()); err != nil {
|
|
h.writeProblem(writer, httpapi.NewProblem(http.StatusServiceUnavailable, "SERVICE_UNAVAILABLE", "Service unavailable", "", requestID))
|
|
return
|
|
}
|
|
h.writeJSON(writer, requestID, http.StatusOK, healthDTO{Status: "ok"})
|
|
default:
|
|
h.writeProblem(writer, httpapi.NewProblem(http.StatusNotFound, "NOT_FOUND", "Not found", "", requestID))
|
|
}
|
|
}
|
|
|
|
func (h *Handler) handleExtract(writer http.ResponseWriter, request *http.Request, requestID string) {
|
|
identity, err := h.identity.Resolve(request)
|
|
if err != nil {
|
|
if !httpsecurity.WriteProblem(writer, requestID, err) {
|
|
h.writeProblem(writer, httpapi.NewProblem(
|
|
http.StatusInternalServerError,
|
|
"INTERNAL_ERROR",
|
|
"Internal server error",
|
|
"",
|
|
requestID,
|
|
))
|
|
}
|
|
return
|
|
}
|
|
if strings.TrimSpace(identity.ClientID) == "" && strings.TrimSpace(identity.SourceIP) == "" {
|
|
h.writeProblem(writer, problemBadRequest(requestID, "INVALID_REQUEST", "Invalid request", "", nil))
|
|
return
|
|
}
|
|
if !identity.Allows(authorization.DistributionExtract) {
|
|
h.writeProblem(writer, httpapi.NewProblem(http.StatusForbidden, "FORBIDDEN", "Forbidden", "", requestID))
|
|
return
|
|
}
|
|
|
|
idempotencyKey, err := validateIdempotencyKey(request.Header.Values(headerIdempotencyKey))
|
|
if err != nil {
|
|
h.writeProblem(writer, problemBadRequest(requestID, "INVALID_HEADER", "Invalid request header", "", []httpapi.InvalidParam{{
|
|
Name: "Idempotency-Key", Reason: "must be 8..128 characters when present",
|
|
}}))
|
|
return
|
|
}
|
|
|
|
var payload extractRequestDTO
|
|
if err := httpapi.DecodeJSON(writer, request, h.bodyLimitBytes, &payload); err != nil {
|
|
h.writeProblem(writer, problemFromDecodeError(requestID, err))
|
|
return
|
|
}
|
|
|
|
invalidParams := validateExtractRequest(payload)
|
|
if len(invalidParams) > 0 {
|
|
h.writeProblem(writer, httpapi.Problem{
|
|
Type: "https://proxy-pool.local/problems/invalid-request",
|
|
Title: "Invalid request",
|
|
Status: http.StatusUnprocessableEntity,
|
|
Code: "INVALID_REQUEST",
|
|
RequestID: requestID,
|
|
InvalidParams: invalidParams,
|
|
})
|
|
return
|
|
}
|
|
|
|
if !identity.ClientPolicy.AllowsExtractCount(payload.Count) {
|
|
h.writeProblem(writer, httpapi.NewProblem(http.StatusForbidden, "FORBIDDEN", "Forbidden", "", requestID))
|
|
return
|
|
}
|
|
filters, allowed := restrictExtractFilters(identity.ClientPolicy, payload.filtersOrZero())
|
|
if !allowed {
|
|
h.writeProblem(writer, httpapi.NewProblem(http.StatusForbidden, "FORBIDDEN", "Forbidden", "", requestID))
|
|
return
|
|
}
|
|
serviceResponse, err := h.extractor.Extract(request.Context(), controllerExtraction.Request{
|
|
RequestID: requestID,
|
|
ClientID: identity.ClientID,
|
|
SourceIP: identity.SourceIP,
|
|
IdempotencyKey: idempotencyKey,
|
|
Count: payload.Count,
|
|
Fulfillment: domainExtraction.Fulfillment(payload.Fulfillment),
|
|
Filters: controllerExtraction.Filters{
|
|
Protocols: cloneStrings(filters.Protocols),
|
|
Regions: cloneStrings(filters.Regions),
|
|
Carriers: cloneStrings(filters.Carriers),
|
|
Upstreams: cloneStrings(filters.AllowedUpstreams),
|
|
},
|
|
})
|
|
if err != nil {
|
|
h.writeProblem(writer, problemFromExtractError(requestID, err))
|
|
return
|
|
}
|
|
|
|
response := extractResponseDTO{
|
|
RequestID: requestID,
|
|
Requested: serviceResponse.Requested,
|
|
Returned: serviceResponse.Returned,
|
|
Proxies: make([]extractedProxyDTO, 0, len(serviceResponse.Proxies)),
|
|
}
|
|
for _, extracted := range serviceResponse.Proxies {
|
|
response.Proxies = append(response.Proxies, extractedProxyDTO{
|
|
ID: extracted.ID,
|
|
Protocol: extracted.Protocol,
|
|
Host: extracted.Host,
|
|
Port: extracted.Port,
|
|
Username: extracted.Username,
|
|
Password: extracted.Password,
|
|
URL: extracted.URL,
|
|
Region: extracted.Region,
|
|
Carrier: extracted.Carrier,
|
|
Upstream: extracted.Upstream,
|
|
ExpiresAt: extracted.ExpiresAt.UTC().Format(time.RFC3339),
|
|
RemainingTTLSeconds: extracted.RemainingTTLSeconds,
|
|
ExtractedAt: extracted.ExtractedAt.UTC().Format(time.RFC3339),
|
|
})
|
|
}
|
|
h.writeJSON(writer, requestID, http.StatusOK, response)
|
|
}
|
|
|
|
func restrictExtractFilters(policy clientpolicy.Policy, filters extractFiltersDTO) (extractFiltersDTO, bool) {
|
|
upstreams, allowed := policy.RestrictUpstreams(filters.AllowedUpstreams)
|
|
if !allowed {
|
|
return extractFiltersDTO{}, false
|
|
}
|
|
regions, allowed := policy.RestrictRegions(filters.Regions)
|
|
if !allowed {
|
|
return extractFiltersDTO{}, false
|
|
}
|
|
filters.AllowedUpstreams = upstreams
|
|
filters.Regions = regions
|
|
return filters, true
|
|
}
|
|
|
|
func validateIdempotencyKey(values []string) (string, error) {
|
|
if len(values) == 0 || (len(values) == 1 && values[0] == "") {
|
|
return "", nil
|
|
}
|
|
if len(values) != 1 {
|
|
return "", errors.New("invalid idempotency key")
|
|
}
|
|
value := values[0]
|
|
if len(value) < minIdempotencyKeySize || len(value) > maxIdempotencyKeySize || strings.TrimSpace(value) != value {
|
|
return "", errors.New("invalid idempotency key")
|
|
}
|
|
for _, character := range value {
|
|
if character < 0x20 || character == 0x7f {
|
|
return "", errors.New("invalid idempotency key")
|
|
}
|
|
}
|
|
return value, nil
|
|
}
|
|
|
|
func validateExtractRequest(payload extractRequestDTO) []httpapi.InvalidParam {
|
|
var invalid []httpapi.InvalidParam
|
|
if payload.Count < 1 || payload.Count > maxExtractCount {
|
|
invalid = append(invalid, httpapi.InvalidParam{Name: "count", Reason: "must be between 1 and 1000"})
|
|
}
|
|
if payload.Fulfillment != "" && payload.Fulfillment != string(domainExtraction.Partial) &&
|
|
payload.Fulfillment != string(domainExtraction.AllOrNothing) {
|
|
invalid = append(invalid, httpapi.InvalidParam{Name: "fulfillment", Reason: "must be partial or allOrNothing"})
|
|
}
|
|
if payload.Filters == nil {
|
|
return invalid
|
|
}
|
|
invalid = append(invalid, validateUniqueStrings("filters.protocols", payload.Filters.Protocols, validProtocol)...)
|
|
invalid = append(invalid, validateUniqueStrings("filters.regions", payload.Filters.Regions, nil)...)
|
|
invalid = append(invalid, validateUniqueStrings("filters.carriers", payload.Filters.Carriers, nil)...)
|
|
invalid = append(invalid, validateUniqueStrings("filters.allowedUpstreams", payload.Filters.AllowedUpstreams, nil)...)
|
|
return invalid
|
|
}
|
|
|
|
func validateUniqueStrings(name string, values []string, allowed map[string]struct{}) []httpapi.InvalidParam {
|
|
if len(values) > maxFilterValues {
|
|
return []httpapi.InvalidParam{{Name: name, Reason: "must contain at most 64 values"}}
|
|
}
|
|
seen := make(map[string]struct{}, len(values))
|
|
invalid := make([]httpapi.InvalidParam, 0, 1)
|
|
for _, value := range values {
|
|
if _, exists := seen[value]; exists {
|
|
invalid = append(invalid, httpapi.InvalidParam{Name: name, Reason: "must not contain duplicates"})
|
|
return invalid
|
|
}
|
|
seen[value] = struct{}{}
|
|
if allowed != nil {
|
|
if _, ok := allowed[value]; !ok {
|
|
invalid = append(invalid, httpapi.InvalidParam{Name: name, Reason: "contains unsupported value"})
|
|
return invalid
|
|
}
|
|
}
|
|
}
|
|
return invalid
|
|
}
|
|
|
|
var validProtocol = map[string]struct{}{
|
|
"http": {},
|
|
"https": {},
|
|
"socks5": {},
|
|
}
|
|
|
|
func problemFromDecodeError(requestID string, err error) httpapi.Problem {
|
|
switch {
|
|
case errors.Is(err, httpapi.ErrUnsupportedMediaType):
|
|
return httpapi.NewProblem(http.StatusUnsupportedMediaType, "UNSUPPORTED_MEDIA_TYPE", "Unsupported media type", "Content-Type must be application/json", requestID)
|
|
case errors.Is(err, httpapi.ErrBodyTooLarge):
|
|
return httpapi.NewProblem(http.StatusRequestEntityTooLarge, "REQUEST_BODY_TOO_LARGE", "Request body too large", "request body exceeds the configured limit", requestID)
|
|
default:
|
|
return problemBadRequest(requestID, "INVALID_JSON", "Invalid JSON request body", "", nil)
|
|
}
|
|
}
|
|
|
|
func problemFromExtractError(requestID string, err error) httpapi.Problem {
|
|
switch {
|
|
case errors.Is(err, domainExtraction.ErrInsufficientProxies):
|
|
return httpapi.NewProblem(http.StatusConflict, "INSUFFICIENT_PROXIES", "Insufficient proxies", "", requestID)
|
|
case errors.Is(err, domainExtraction.ErrIdempotencyConflict):
|
|
return httpapi.NewProblem(http.StatusConflict, "IDEMPOTENCY_CONFLICT", "Idempotency conflict", "", requestID)
|
|
case errors.Is(err, controllerExtraction.ErrCountExceeded):
|
|
return httpapi.NewProblem(http.StatusUnprocessableEntity, "COUNT_EXCEEDED", "Invalid request", "", requestID)
|
|
case errors.Is(err, controllerExtraction.ErrInvalidFulfillment):
|
|
return httpapi.NewProblem(http.StatusUnprocessableEntity, "INVALID_FULFILLMENT", "Invalid request", "", requestID)
|
|
case errors.Is(err, controllerExtraction.ErrInvalidRequest):
|
|
return httpapi.NewProblem(http.StatusUnprocessableEntity, "INVALID_REQUEST", "Invalid request", "", requestID)
|
|
case errors.Is(err, controllerExtraction.ErrAdmissionRejected):
|
|
return httpapi.NewProblem(http.StatusTooManyRequests, "RATE_LIMITED", "Too many requests", "", requestID)
|
|
case errors.Is(err, controllerExtraction.ErrUnavailable):
|
|
return httpapi.NewProblem(http.StatusServiceUnavailable, "SERVICE_UNAVAILABLE", "Service unavailable", "", requestID)
|
|
default:
|
|
return httpapi.NewProblem(http.StatusInternalServerError, "INTERNAL_ERROR", "Internal server error", "", requestID)
|
|
}
|
|
}
|
|
|
|
func problemBadRequest(requestID, code, title, detail string, invalid []httpapi.InvalidParam) httpapi.Problem {
|
|
return httpapi.Problem{
|
|
Type: "https://proxy-pool.local/problems/invalid-request",
|
|
Title: title,
|
|
Status: http.StatusBadRequest,
|
|
Code: code,
|
|
Detail: detail,
|
|
RequestID: requestID,
|
|
InvalidParams: invalid,
|
|
}
|
|
}
|
|
|
|
func (h *Handler) writeJSON(writer http.ResponseWriter, requestID string, status int, value any) {
|
|
writer.Header().Set(httpapi.HeaderRequestID, requestID)
|
|
_ = httpapi.WriteJSON(writer, status, value)
|
|
}
|
|
|
|
func (h *Handler) writeProblem(writer http.ResponseWriter, problem httpapi.Problem) {
|
|
if problem.RequestID == "" {
|
|
problem.RequestID = writer.Header().Get(httpapi.HeaderRequestID)
|
|
}
|
|
httpapi.WriteProblem(writer, problem)
|
|
}
|
|
|
|
func cloneStrings(values []string) []string {
|
|
return append([]string(nil), values...)
|
|
}
|
|
|
|
func (payload extractRequestDTO) filtersOrZero() extractFiltersDTO {
|
|
if payload.Filters == nil {
|
|
return extractFiltersDTO{}
|
|
}
|
|
return *payload.Filters
|
|
}
|