diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..e0829a1 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,34 @@ +name: ci + +on: + push: + pull_request: + +permissions: + contents: read + +jobs: + test: + strategy: + matrix: + os: [ubuntu-latest, windows-latest] + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 + with: + go-version-file: go.mod + cache: true + - run: go vet ./... + - run: go test -timeout 60s ./... + - run: go build ./... + + race: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 + with: + go-version-file: go.mod + cache: true + - run: go test -race -timeout 60s ./internal/... diff --git a/.gitignore b/.gitignore index d1fd622..0d61e3d 100644 --- a/.gitignore +++ b/.gitignore @@ -9,6 +9,7 @@ coverage/ *.out *.test *.prof +.tmp-proto/ # Local configuration and secrets .env diff --git a/README.md b/README.md new file mode 100644 index 0000000..d45d812 --- /dev/null +++ b/README.md @@ -0,0 +1,57 @@ +# Proxy Pool + +面向多供应商代理资源的集中管理与高并发转发平台。系统同时提供： + +- **Gateway**：系统选择上游代理并代转发 HTTP 与 HTTPS CONNECT。 +- **Distribution API**：把真实代理一次性、独占地发放给调用方。 +- **Admin API**：查询、启停、切换和配置重载。 +- **Controller / Checker**：管理供应商获取、健康、容量、状态与数据面快照。 + +> 当前仓库交付的是从 `对话内容.md` 全量重建的设计基线、机器契约、 +> 项目骨架和关键并发领域实现。100,000 QPS 是集群设计目标，尚需在目标 +> 网络和代理规模下完成压测证明。 + +## 快速导航 + +- [产品设计](docs/design/product-design.md) +- [总体架构](docs/design/architecture.md) +- [项目结构](docs/design/project-structure.md) +- [需求追踪](docs/requirements/traceability.md) +- [交付完成度审计](docs/requirements/completion-audit.md) +- [开发指南](docs/development/guide.md) +- [实施计划](docs/development/implementation-plan.md) +- [配置参考](docs/configuration/reference.md) +- [Distribution API](docs/api/distribution.md) +- [控制面协议](docs/api/control-plane.md) +- [安全模型](docs/security/security-model.md) +- [测试策略](docs/testing/strategy.md) +- [运维手册](docs/operations/runbook.md) +- [生产就绪检查](docs/operations/production-readiness.md) + +## 本地验证 + +```powershell +go mod tidy +go test ./... +go vet ./... +go build ./... +``` + +Windows PowerShell 可运行： + +```powershell +./scripts/verify.ps1 +``` + +竞态检测需要启用 CGO 并提供可用的 C 编译器；CI 的 Linux race job 负责 +执行该质量门禁。 + +## 不变量 + +1. Gateway 热路径不访问 PostgreSQL、Redis 或 Provider API。 +2. Proxy 容量使用 `Reserved -> Active` 原子转换，禁止超卖。 +3. Distribution 成功时原子执行 `AVAILABLE -> EXTRACTED`，不提供 Lease、 + Release 或 Renewal。 +4. `pool.maxSize` 是当前未提取库存上限；`fetch.maxTotal` 是累计获取额度。 +5. CONNECT 向客户端提交 200 后不透明重放。 +6. 公开监听必须有认证或 CIDR 访问保护。 diff --git a/api/openapi/admin.yaml b/api/openapi/admin.yaml new file mode 100644 index 0000000..283215b --- /dev/null +++ b/api/openapi/admin.yaml @@ -0,0 +1,202 @@ +openapi: 3.1.0 +info: + title: Proxy Pool Admin API + version: 1.0.0 + description: 运维状态与受控变更接口。该入口必须与 Distribution 分端口和权限。 +servers: + - url: http://127.0.0.1:8082 +tags: + - name: Status + - name: Upstreams + - name: Routing + - name: Configuration +security: + - AdminApiKey: [] + - BasicAuth: [] + - BearerAuth: [] +paths: + /api/v1/status: + get: + tags: [Status] + operationId: getStatus + summary: 获取控制面摘要状态 + responses: + '200': + description: 不含 Proxy 地址、Client 标识或 Secret 的聚合状态 + content: + application/json: + schema: + $ref: '#/components/schemas/Status' + '401': {$ref: '#/components/responses/Unauthorized'} + '403': {$ref: '#/components/responses/Forbidden'} + /api/v1/upstreams/{name}/enable: + post: + tags: [Upstreams] + operationId: enableUpstream + summary: 启用 Upstream + parameters: + - $ref: '#/components/parameters/UpstreamName' + - $ref: '#/components/parameters/RequestID' + responses: + '200': {$ref: '#/components/responses/MutationResult'} + '401': {$ref: '#/components/responses/Unauthorized'} + '403': {$ref: '#/components/responses/Forbidden'} + '404': {$ref: '#/components/responses/NotFound'} + '409': {$ref: '#/components/responses/Conflict'} + /api/v1/upstreams/{name}/disable: + post: + tags: [Upstreams] + operationId: disableUpstream + summary: 禁用 Upstream 并使已有资源自然 Drain + parameters: + - $ref: '#/components/parameters/UpstreamName' + - $ref: '#/components/parameters/RequestID' + responses: + '200': {$ref: '#/components/responses/MutationResult'} + '401': {$ref: '#/components/responses/Unauthorized'} + '403': {$ref: '#/components/responses/Forbidden'} + '404': {$ref: '#/components/responses/NotFound'} + '409': {$ref: '#/components/responses/Conflict'} + /api/v1/routing/{name}/switch: + post: + tags: [Routing] + operationId: switchRouting + summary: 原子切换 Sequential Routing 当前 Upstream + parameters: + - name: name + in: path + required: true + schema: {type: string, minLength: 1, maxLength: 128} + - $ref: '#/components/parameters/RequestID' + requestBody: + required: true + content: + application/json: + schema: + type: object + additionalProperties: false + required: [expectedCurrent, target] + properties: + expectedCurrent: {type: string} + target: {type: string} + reason: {type: string, maxLength: 512} + responses: + '200': {$ref: '#/components/responses/MutationResult'} + '401': {$ref: '#/components/responses/Unauthorized'} + '403': {$ref: '#/components/responses/Forbidden'} + '404': {$ref: '#/components/responses/NotFound'} + '409': {$ref: '#/components/responses/Conflict'} + /api/v1/config/reload: + post: + tags: [Configuration] + operationId: reloadConfiguration + summary: 严格校验并原子发布新配置快照 + parameters: + - $ref: '#/components/parameters/RequestID' + responses: + '200': {$ref: '#/components/responses/MutationResult'} + '401': {$ref: '#/components/responses/Unauthorized'} + '403': {$ref: '#/components/responses/Forbidden'} + '409': {$ref: '#/components/responses/Conflict'} + '422': + description: 新配置无效，旧配置继续运行 + content: + application/problem+json: + schema: {$ref: '#/components/schemas/Problem'} +components: + securitySchemes: + AdminApiKey: {type: apiKey, in: header, name: X-Admin-Key} + BasicAuth: {type: http, scheme: basic} + BearerAuth: {type: http, scheme: bearer} + parameters: + UpstreamName: + name: name + in: path + required: true + schema: {type: string, minLength: 1, maxLength: 128} + RequestID: + name: X-Request-ID + in: header + required: false + schema: {type: string, maxLength: 128} + schemas: + Status: + type: object + additionalProperties: false + required: [configVersion, snapshotVersion, upstreams, workers] + properties: + configVersion: {type: string} + snapshotVersion: {type: integer, minimum: 0} + upstreams: + type: array + items: + type: object + additionalProperties: false + required: [name, enabled, available, checking, suspect, draining, extracted] + properties: + name: {type: string} + enabled: {type: boolean} + available: {type: integer, minimum: 0} + checking: {type: integer, minimum: 0} + suspect: {type: integer, minimum: 0} + draining: {type: integer, minimum: 0} + extracted: {type: integer, minimum: 0} + consecutiveEmptyFetch: {type: integer, minimum: 0} + fetchErrorCount: {type: integer, minimum: 0} + workers: + type: array + items: + type: object + additionalProperties: false + required: [id, zone, connected, snapshotVersion] + properties: + id: {type: string} + zone: {type: string} + connected: {type: boolean} + snapshotVersion: {type: integer, minimum: 0} + staleSeconds: {type: integer, minimum: 0} + MutationResult: + type: object + additionalProperties: false + required: [requestId, changed, version] + properties: + requestId: {type: string} + changed: {type: boolean} + version: {type: integer, minimum: 0} + message: {type: string} + Problem: + type: object + required: [type, title, status, code] + properties: + type: {type: string, format: uri} + title: {type: string} + status: {type: integer} + code: {type: string} + detail: {type: string} + requestId: {type: string} + responses: + MutationResult: + description: 操作已提交或目标状态原本已满足 + content: + application/json: + schema: {$ref: '#/components/schemas/MutationResult'} + Unauthorized: + description: 管理入口认证失败 + content: + application/problem+json: + schema: {$ref: '#/components/schemas/Problem'} + Forbidden: + description: 调用主体无该管理权限 + content: + application/problem+json: + schema: {$ref: '#/components/schemas/Problem'} + NotFound: + description: Upstream 或 Routing 不存在 + content: + application/problem+json: + schema: {$ref: '#/components/schemas/Problem'} + Conflict: + description: 预期版本或 expectedCurrent 与权威状态不一致 + content: + application/problem+json: + schema: {$ref: '#/components/schemas/Problem'} diff --git a/api/openapi/openapi_test.go b/api/openapi/openapi_test.go new file mode 100644 index 0000000..d558bdb --- /dev/null +++ b/api/openapi/openapi_test.go @@ -0,0 +1,63 @@ +package openapi + +import ( + "os" + "testing" + + "go.yaml.in/yaml/v4" +) + +type document struct { + OpenAPI string `yaml:"openapi"` + Paths map[string]map[string]any `yaml:"paths"` +} + +func TestDistributionContract(t *testing.T) { + spec := readDocument(t, "proxy-pool.yaml") + if spec.OpenAPI != "3.1.0" { + t.Fatalf("openapi version = %q, want 3.1.0", spec.OpenAPI) + } + extraction, ok := spec.Paths["/api/v1/proxies/extract"] + if !ok { + t.Fatal("exclusive extraction path is missing") + } + if _, ok := extraction["post"]; !ok { + t.Fatal("exclusive extraction must use POST") + } + for path := range spec.Paths { + if path == "/api/v1/leases" || path == "/api/v1/proxies/release" || path == "/api/v1/proxies/renew" { + t.Fatalf("lease/release path is forbidden: %s", path) + } + } +} + +func TestAdminContract(t *testing.T) { + spec := readDocument(t, "admin.yaml") + for _, path := range []string{ + "/api/v1/status", + "/api/v1/upstreams/{name}/enable", + "/api/v1/upstreams/{name}/disable", + "/api/v1/routing/{name}/switch", + "/api/v1/config/reload", + } { + if _, ok := spec.Paths[path]; !ok { + t.Errorf("admin path is missing: %s", path) + } + } +} + +func readDocument(t *testing.T, path string) document { + t.Helper() + content, err := os.ReadFile(path) + if err != nil { + t.Fatalf("read %s: %v", path, err) + } + var spec document + if err := yaml.Unmarshal(content, &spec); err != nil { + t.Fatalf("parse %s: %v", path, err) + } + if spec.OpenAPI != "3.1.0" { + t.Fatalf("%s openapi version = %q, want 3.1.0", path, spec.OpenAPI) + } + return spec +} diff --git a/api/openapi/proxy-pool.yaml b/api/openapi/proxy-pool.yaml new file mode 100644 index 0000000..1553645 --- /dev/null +++ b/api/openapi/proxy-pool.yaml @@ -0,0 +1,351 @@ +openapi: 3.1.0 +info: + title: Proxy Pool HTTP API + version: 1.0.0 + description: | + Distribution API performs one-time exclusive extraction. A successful + operation atomically transitions every returned proxy from AVAILABLE to + EXTRACTED. Extracted proxies are never allocated again and there is no + release, renew, or lease API. +servers: + - url: http://127.0.0.1:8081 + description: Distribution API +tags: + - name: Distribution + - name: Health +paths: + /api/v1/proxies/extract: + post: + tags: [Distribution] + operationId: extractProxies + summary: 一次性独占提取代理 + description: | + 服务端先完成筛选、行锁定、AVAILABLE -> EXTRACTED 状态更新和审计记录 + 写入，再返回代理。相同代理不会返回给两个成功请求。 + + `partial` 允许实际返回数量小于请求数量；`allOrNothing` 数量不足时不 + 提取任何代理并返回 409。未传 `fulfillment` 时使用服务端配置，默认 + 为 `partial`。`Idempotency-Key` 可避免客户端因响应丢失重试而再次消耗 + 库存。 + security: + - ApiKeyAuth: [] + - BasicAuth: [] + - BearerAuth: [] + - {} + parameters: + - $ref: '#/components/parameters/RequestID' + - $ref: '#/components/parameters/IdempotencyKey' + requestBody: + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/ExtractRequest' + examples: + partial: + value: + count: 5 + fulfillment: partial + filters: + protocols: [http] + regions: [shanghai] + allOrNothing: + value: + count: 10 + fulfillment: allOrNothing + filters: + allowedUpstreams: [provider-a, provider-b] + responses: + '200': + description: 提取事务已提交；返回的代理已永久退出可分配池 + headers: + X-Request-ID: + $ref: '#/components/headers/RequestID' + content: + application/json: + schema: + $ref: '#/components/schemas/ExtractResponse' + '400': + $ref: '#/components/responses/BadRequest' + '401': + $ref: '#/components/responses/Unauthorized' + '403': + $ref: '#/components/responses/Forbidden' + '409': + description: allOrNothing 模式下符合条件的库存不足，未提取任何代理 + content: + application/problem+json: + schema: + $ref: '#/components/schemas/Problem' + example: + type: https://proxy-pool.local/problems/insufficient-proxies + title: Insufficient proxies + status: 409 + code: INSUFFICIENT_PROXIES + detail: requested 10 proxies but only 6 are currently eligible + requestId: req_01J4EXAMPLE + '422': + $ref: '#/components/responses/UnprocessableEntity' + '429': + $ref: '#/components/responses/TooManyRequests' + '503': + $ref: '#/components/responses/ServiceUnavailable' + /health/live: + get: + tags: [Health] + operationId: getLiveness + summary: 进程存活探针 + security: [] + responses: + '200': + description: 进程存活 + content: + application/json: + schema: + $ref: '#/components/schemas/Health' + /health/ready: + get: + tags: [Health] + operationId: getReadiness + summary: Distribution 就绪探针 + description: PostgreSQL 不可用或权威状态不可写时返回 503。 + security: [] + responses: + '200': + description: 可接受提取请求 + content: + application/json: + schema: + $ref: '#/components/schemas/Health' + '503': + $ref: '#/components/responses/ServiceUnavailable' +components: + securitySchemes: + ApiKeyAuth: + type: apiKey + in: header + name: X-API-Key + BasicAuth: + type: http + scheme: basic + BearerAuth: + type: http + scheme: bearer + parameters: + RequestID: + name: X-Request-ID + in: header + required: false + description: 调用方请求标识；缺省时由服务端生成。 + schema: + type: string + maxLength: 128 + IdempotencyKey: + name: Idempotency-Key + in: header + required: false + description: | + 同一客户端在幂等记录保留期内重用该键会得到首次提交结果，不会再次 + 提取。建议所有会自动重试的客户端提供。 + schema: + type: string + minLength: 8 + maxLength: 128 + headers: + RequestID: + description: 服务端最终使用的请求标识。 + schema: + type: string + schemas: + ExtractRequest: + type: object + additionalProperties: false + required: [count] + properties: + count: + type: integer + minimum: 1 + maximum: 1000 + description: 仍受服务端 maxCountPerRequest 限制。 + fulfillment: + type: string + enum: [partial, allOrNothing] + description: 缺省时使用服务端配置；默认 partial。 + filters: + $ref: '#/components/schemas/ExtractFilters' + ExtractFilters: + type: object + additionalProperties: false + properties: + protocols: + type: array + uniqueItems: true + items: + type: string + enum: [http, https, socks5] + regions: + type: array + uniqueItems: true + items: + type: string + carriers: + type: array + uniqueItems: true + items: + type: string + allowedUpstreams: + type: array + uniqueItems: true + items: + type: string + ExtractResponse: + type: object + additionalProperties: false + required: [requestId, requested, returned, proxies] + properties: + requestId: + type: string + requested: + type: integer + minimum: 1 + returned: + type: integer + minimum: 0 + proxies: + type: array + items: + $ref: '#/components/schemas/ExtractedProxy' + ExtractedProxy: + type: object + additionalProperties: false + required: + - id + - protocol + - host + - port + - url + - upstream + - expiresAt + - remainingTtlSeconds + - extractedAt + properties: + id: + type: string + example: px_01J4EXAMPLE + protocol: + type: string + enum: [http, https, socks5] + host: + type: string + example: 192.0.2.10 + port: + type: integer + minimum: 1 + maximum: 65535 + username: + type: string + password: + type: string + format: password + description: 真实代理凭据，只出现在提取成功响应中。 + url: + type: string + format: uri + description: 含真实代理凭据的连接 URL，必须按敏感数据处理。 + example: http://USER:PASSWORD@192.0.2.10:8080 + region: + type: string + carrier: + type: string + upstream: + type: string + expiresAt: + type: string + format: date-time + remainingTtlSeconds: + type: integer + minimum: 0 + extractedAt: + type: string + format: date-time + Health: + type: object + additionalProperties: false + required: [status] + properties: + status: + type: string + enum: [ok, degraded] + version: + type: string + configVersion: + type: string + Problem: + type: object + additionalProperties: true + required: [type, title, status, code] + properties: + type: + type: string + format: uri + title: + type: string + status: + type: integer + code: + type: string + detail: + type: string + requestId: + type: string + invalidParams: + type: array + items: + type: object + required: [name, reason] + properties: + name: + type: string + reason: + type: string + responses: + BadRequest: + description: 请求体、Header 或 JSON 格式无效 + content: + application/problem+json: + schema: + $ref: '#/components/schemas/Problem' + Unauthorized: + description: 所配置的认证方法未通过 + content: + application/problem+json: + schema: + $ref: '#/components/schemas/Problem' + Forbidden: + description: 来源访问控制或客户端权限拒绝 + content: + application/problem+json: + schema: + $ref: '#/components/schemas/Problem' + UnprocessableEntity: + description: 参数语法有效但违反业务约束 + content: + application/problem+json: + schema: + $ref: '#/components/schemas/Problem' + TooManyRequests: + description: 超过全局或客户端速率限制 + headers: + Retry-After: + schema: + type: integer + content: + application/problem+json: + schema: + $ref: '#/components/schemas/Problem' + ServiceUnavailable: + description: 权威存储不可用或服务正在排空 + content: + application/problem+json: + schema: + $ref: '#/components/schemas/Problem' diff --git a/api/proto/controlplane/v1/controlplane.proto b/api/proto/controlplane/v1/controlplane.proto new file mode 100644 index 0000000..e9be4bf --- /dev/null +++ b/api/proto/controlplane/v1/controlplane.proto @@ -0,0 +1,249 @@ +syntax = "proto3"; + +package proxy_pool.controlplane.v1; + +option go_package = "github.com/proxy-pool/proxy-pool/gen/controlplane/v1;controlplanev1"; + +import "google/protobuf/duration.proto"; +import "google/protobuf/empty.proto"; +import "google/protobuf/timestamp.proto"; + +// WorkerControlPlane distributes immutable, worker-specific snapshots. The +// gateway hot path does not call this service for individual requests. +service WorkerControlPlane { + rpc RegisterWorker(RegisterWorkerRequest) returns (RegisterWorkerResponse); + rpc WatchSnapshots(WatchSnapshotsRequest) returns (stream SnapshotEnvelope); + rpc AcknowledgeSnapshot(AcknowledgeSnapshotRequest) returns (google.protobuf.Empty); + rpc ReportOutcomes(stream OutcomeBatch) returns (ReportOutcomesResponse); + rpc ReportRuntime(ReportRuntimeRequest) returns (ReportRuntimeResponse); +} + +// CheckerControlPlane hands bounded check work to independently scalable +// checker processes. Observations are facts; only the Controller reducer may +// change authoritative proxy state. +service CheckerControlPlane { + rpc StreamCheckTasks(StreamCheckTasksRequest) returns (stream CheckTask); + rpc ReportObservations(ObservationBatch) returns (ReportObservationsResponse); +} + +message RegisterWorkerRequest { + string worker_id = 1; + string instance_id = 2; + string zone = 3; + uint32 supported_protocol_version = 4; + map labels = 5; +} + +message RegisterWorkerResponse { + string worker_id = 1; + string session_id = 2; + uint64 ownership_epoch = 3; + google.protobuf.Duration heartbeat_interval = 4; + google.protobuf.Duration max_stale_age = 5; +} + +message WatchSnapshotsRequest { + string worker_id = 1; + string session_id = 2; + uint64 last_applied_version = 3; + bytes last_checksum = 4; +} + +message SnapshotEnvelope { + oneof payload { + WorkerSnapshot full = 1; + SnapshotDelta delta = 2; + } +} + +message WorkerSnapshot { + uint64 version = 1; + uint64 ownership_epoch = 2; + google.protobuf.Timestamp generated_at = 3; + google.protobuf.Timestamp valid_until = 4; + bytes checksum = 5; + repeated RoutingRule routing = 6; + repeated OwnedProxy proxies = 7; +} + +message SnapshotDelta { + uint64 base_version = 1; + uint64 version = 2; + uint64 ownership_epoch = 3; + google.protobuf.Timestamp generated_at = 4; + bytes checksum = 5; + repeated RoutingRule upserted_routing = 6; + repeated string removed_routing_names = 7; + repeated OwnedProxy upserted_proxies = 8; + repeated string removed_proxy_ids = 9; +} + +message RoutingRule { + string name = 1; + bool enabled = 2; + string host_regex = 3; + repeated string methods = 4; + string path_regex = 5; + map headers = 6; + repeated string upstreams = 7; + RoutingStrategy strategy = 8; + UnavailableAction on_unavailable = 9; +} + +message RoutingStrategy { + StrategyType type = 1; + string current_upstream = 2; + map weights = 3; +} + +enum StrategyType { + STRATEGY_TYPE_UNSPECIFIED = 0; + STRATEGY_TYPE_SEQUENTIAL = 1; + STRATEGY_TYPE_RANDOM = 2; + STRATEGY_TYPE_ROUND_ROBIN = 3; + STRATEGY_TYPE_WEIGHTED = 4; + STRATEGY_TYPE_LEAST_CONNECTIONS = 5; +} + +enum UnavailableAction { + UNAVAILABLE_ACTION_UNSPECIFIED = 0; + UNAVAILABLE_ACTION_REJECT = 1; + UNAVAILABLE_ACTION_WAIT = 2; + UNAVAILABLE_ACTION_DIRECT = 3; +} + +message OwnedProxy { + string id = 1; + string upstream = 2; + ProxyProtocol protocol = 3; + string host = 4; + uint32 port = 5; + string username = 6; + string credential_version = 7; + string secret_ref = 8; + google.protobuf.Timestamp expires_at = 9; + uint32 max_concurrency = 10; + map tags = 11; + uint64 ownership_epoch = 12; +} + +enum ProxyProtocol { + PROXY_PROTOCOL_UNSPECIFIED = 0; + PROXY_PROTOCOL_HTTP = 1; + PROXY_PROTOCOL_HTTPS = 2; + PROXY_PROTOCOL_SOCKS5 = 3; +} + +message AcknowledgeSnapshotRequest { + string worker_id = 1; + string session_id = 2; + uint64 version = 3; + uint64 ownership_epoch = 4; + bytes checksum = 5; + bool applied = 6; + string error_code = 7; + string error_message = 8; +} + +message OutcomeBatch { + string worker_id = 1; + string session_id = 2; + uint64 sequence = 3; + repeated ProxyOutcome outcomes = 4; +} + +message ProxyOutcome { + string proxy_id = 1; + string routing_name = 2; + OutcomeStage stage = 3; + bool success = 4; + google.protobuf.Duration latency = 5; + string error_class = 6; + google.protobuf.Timestamp observed_at = 7; +} + +enum OutcomeStage { + OUTCOME_STAGE_UNSPECIFIED = 0; + OUTCOME_STAGE_DIAL = 1; + OUTCOME_STAGE_PROXY_HANDSHAKE = 2; + OUTCOME_STAGE_RESPONSE_HEADERS = 3; + OUTCOME_STAGE_TUNNEL = 4; +} + +message ReportOutcomesResponse { + uint64 accepted_through_sequence = 1; +} + +message ReportRuntimeRequest { + string worker_id = 1; + string session_id = 2; + uint64 snapshot_version = 3; + uint64 ownership_epoch = 4; + repeated ProxyRuntime counters = 5; + google.protobuf.Timestamp observed_at = 6; +} + +message ProxyRuntime { + string proxy_id = 1; + uint32 reserved = 2; + uint32 active = 3; + bool draining = 4; +} + +message ReportRuntimeResponse { + uint64 accepted_ownership_epoch = 1; + repeated string revoke_proxy_ids = 2; + bool require_full_snapshot = 3; +} + +message StreamCheckTasksRequest { + string checker_id = 1; + string instance_id = 2; + uint32 max_in_flight = 3; + repeated CheckLevel supported_levels = 4; +} + +message CheckTask { + string task_id = 1; + string proxy_id = 2; + ProxyProtocol protocol = 3; + string host = 4; + uint32 port = 5; + string secret_ref = 6; + CheckLevel level = 7; + string routing_name = 8; + string target_url = 9; + google.protobuf.Duration timeout = 10; + uint32 attempt = 11; + google.protobuf.Timestamp deadline = 12; +} + +enum CheckLevel { + CHECK_LEVEL_UNSPECIFIED = 0; + CHECK_LEVEL_BASIC = 1; + CHECK_LEVEL_EGRESS = 2; + CHECK_LEVEL_TARGET = 3; +} + +message ObservationBatch { + string checker_id = 1; + repeated HealthObservation observations = 2; +} + +message HealthObservation { + string task_id = 1; + string proxy_id = 2; + CheckLevel level = 3; + string routing_name = 4; + string target_url = 5; + bool success = 6; + string failure_class = 7; + google.protobuf.Duration latency = 8; + string observed_egress_ip = 9; + google.protobuf.Timestamp observed_at = 10; +} + +message ReportObservationsResponse { + uint32 accepted = 1; + uint32 rejected = 2; +} diff --git a/configs/proxy-pool.yaml b/configs/proxy-pool.yaml new file mode 100644 index 0000000..c8d9600 --- /dev/null +++ b/configs/proxy-pool.yaml @@ -0,0 +1,217 @@ +version: 1 + +security: + requireProtectionOnPublicListen: true + +defaults: + fetch: + requestInterval: 1s + timeout: 5s + maxAttempts: 3 + maxInFlight: 1 + maxResponseBytes: 1048576 + templateTimeout: 100ms + retry: + initial: 500ms + max: 30s + jitter: 20 + check: + interval: 30s + jitter: 20 + maxInFlight: 200 + timeout: 3s + maxAttempts: 2 + maxConsecutiveFailures: 3 + urls: + - http://connect.rom.miui.com/generate_204 + +gateway: + enabled: true + listen: 0.0.0.0:8080 + access: + allowCIDRs: + - 10.0.0.0/8 + - 172.16.0.0/12 + trustedProxies: [] + auth: + mode: usernamePassword + username: "${GATEWAY_USER}" + password: "${GATEWAY_PASSWORD}" + limits: + maxConcurrentConnections: 200000 + retry: + maxAttempts: 2 + retryMethods: [GET, HEAD] + destinationPolicy: + denyPrivateNetworks: true + denyLoopback: true + denyLinkLocal: true + denyCIDRs: + - 169.254.169.254/32 + +distribution: + enabled: true + listen: 0.0.0.0:8081 + access: + allowCIDRs: + - 10.0.0.0/8 + - 172.16.0.0/12 + trustedProxies: [] + auth: + mode: apiKey + header: X-API-Key + token: "${DISTRIBUTION_API_KEY}" + limits: + requestsPerMinute: 6000 + requestsPerMinutePerClient: 600 + clientIdentification: + mode: authenticatedClientOrSourceIP + extraction: + fulfillment: partial + maxCountPerRequest: 100 + minRemainingTTL: 30s + maxHealthCheckAge: 15s + reserveForGateway: 100 + +admin: + enabled: true + listen: 127.0.0.1:8082 + auth: + mode: none + +metrics: + enabled: true + listen: 127.0.0.1:9090 + +storage: + postgresURL: "${POSTGRES_URL}" + redisURL: "${REDIS_URL}" + +routing: + - name: gateway-default + enabled: true + purpose: gateway + match: + hostRegex: '.*' + upstreams: [provider-a, provider-b] + strategy: + type: sequential + switchAfterEmptyFetch: 5 + endBehavior: stayLast + onUnavailable: + action: reject + - name: extract-default + enabled: true + purpose: extract + match: + hostRegex: '.*' + upstreams: [provider-a, provider-b] + strategy: + type: sequential + switchAfterEmptyFetch: 5 + endBehavior: stayLast + onUnavailable: + action: reject + +upstreams: + provider-a: + enabled: true + exposure: [gateway, extract] + provider: + billingMode: fetch + protocols: [http, https] + api: + url: https://provider-a.example/api/proxies + method: GET + auth: + type: apiKey + location: header + name: X-Provider-Key + value: "${PROVIDER_A_TOKEN}" + query: + count: '100' + template: '{{.}}' + proxyAuth: + type: response + pool: + maxSize: 10000 + shrinkDelay: 30s + capacity: + maxConcurrencyPerProxy: 20 + lifecycle: + ttl: 5m + allocationSafetyMargin: 30s + fetch: + requestInterval: 1s + timeout: 5s + maxAttempts: 3 + maxInFlight: 1 + maxTotal: 1000000 + maxResponseBytes: 1048576 + templateTimeout: 100ms + retry: + initial: 500ms + max: 30s + jitter: 20 + check: + interval: 30s + jitter: 20 + maxInFlight: 200 + timeout: 3s + maxAttempts: 2 + maxConsecutiveFailures: 3 + urls: + - http://connect.rom.miui.com/generate_204 + provider-b: + enabled: true + exposure: [gateway, extract] + provider: + billingMode: subscription + protocols: [http] + api: + url: https://provider-b.example/api/proxies + method: POST + auth: + type: basic + username: "${PROVIDER_B_USER}" + password: "${PROVIDER_B_PASSWORD}" + headers: + Content-Type: application/json + body: + type: json + value: + count: '100' + template: '{{.}}' + proxyAuth: + type: static + username: "${PROXY_USER}" + password: "${PROXY_PASSWORD}" + pool: + maxSize: 5000 + shrinkDelay: 30s + capacity: + maxConcurrencyPerProxy: 10 + lifecycle: + ttl: 2m + allocationSafetyMargin: 20s + fetch: + requestInterval: 2s + timeout: 5s + maxAttempts: 3 + maxInFlight: 1 + maxTotal: 0 + maxResponseBytes: 1048576 + templateTimeout: 100ms + retry: + initial: 1s + max: 30s + jitter: 20 + check: + interval: 20s + jitter: 20 + maxInFlight: 100 + timeout: 3s + maxAttempts: 2 + maxConsecutiveFailures: 3 + urls: + - http://connect.rom.miui.com/generate_204 diff --git a/deploy/README.md b/deploy/README.md new file mode 100644 index 0000000..f7d0687 --- /dev/null +++ b/deploy/README.md @@ -0,0 +1,17 @@ +# 部署拓扑模板 + +本目录描述 Proxy Pool 目标运行拓扑，覆盖 Compose、HAProxy、Prometheus、 +Grafana 与 Kubernetes。配置已通过静态展开，但当前仓库的 `cmd/proxy-*` +运行时装配仍在实施计划中，因此不要把这些清单视为当前可部署发行版。 + +当前可执行验证： + +```powershell +docker compose -f deploy/docker-compose.yml config --quiet +kubectl kustomize deploy/kubernetes/base | Out-Null +go run ./deploy/tools/configcheck deploy/config/local.yaml +``` + +运行时完成后，还必须通过 `production-readiness.md` 中的一致性、安全、恢复、 +竞态与容量门禁，才能构建镜像并发布。 + diff --git a/deploy/config/local.yaml b/deploy/config/local.yaml new file mode 100644 index 0000000..da97f0d --- /dev/null +++ b/deploy/config/local.yaml @@ -0,0 +1,179 @@ +version: 1 + +security: + requireProtectionOnPublicListen: true + +gateway: + enabled: true + listen: 0.0.0.0:8080 + access: + allowCIDRs: [172.16.0.0/12] + trustedProxies: [172.16.0.0/12] + auth: + mode: usernamePassword + username: local-gateway + password: env:PROXY_POOL_GATEWAY_PASSWORD + limits: + maxConcurrentConnections: 20000 + requestsPerMinutePerClient: 60000 + retry: + maxAttempts: 2 + retryMethods: [GET, HEAD] + destinationPolicy: + denyPrivateNetworks: true + denyLoopback: true + denyLinkLocal: true + denyCIDRs: [169.254.169.254/32] + +distribution: + enabled: true + listen: 0.0.0.0:8081 + access: + allowCIDRs: [172.16.0.0/12] + trustedProxies: [172.16.0.0/12] + auth: + mode: apiKey + header: X-API-Key + token: env:PROXY_POOL_EXTRACT_TOKEN + limits: + requestsPerMinute: 6000 + requestsPerMinutePerClient: 600 + clientIdentification: + mode: trustedProxyOrRemoteIP + extraction: + fulfillment: partial + maxCountPerRequest: 100 + minRemainingTTL: 30s + maxHealthCheckAge: 30s + reserveForGateway: 1000 + +admin: + enabled: true + listen: 0.0.0.0:8082 + access: + allowCIDRs: [172.16.0.0/12] + auth: + mode: apiKey + header: X-Admin-Token + token: env:PROXY_POOL_ADMIN_TOKEN + +metrics: + enabled: true + listen: 0.0.0.0:9090 + +storage: + postgresURL: postgres://proxy_pool:local-only-change-me@postgres:5432/proxy_pool?sslmode=disable + redisURL: redis://redis:6379/0 + +routing: + - name: gateway-default + enabled: true + purpose: gateway + upstreams: [provider-a, provider-b] + strategy: + type: sequential + switchAfterEmptyFetch: 5 + endBehavior: stayLast + onUnavailable: + action: reject + - name: extract-default + enabled: true + purpose: extract + upstreams: [provider-a, provider-b] + strategy: + type: sequential + switchAfterEmptyFetch: 5 + endBehavior: stayLast + onUnavailable: + action: reject + +upstreams: + provider-a: + enabled: true + exposure: [gateway, extract] + provider: + billingMode: fetch + protocols: [http] + api: + url: https://provider-a.invalid/api/proxies + method: GET + auth: + mode: apiKey + header: Authorization + token: env:PROVIDER_A_TOKEN + template: '{{ . }}' + proxyAuth: + mode: response + pool: + maxSize: 5000 + shrinkDelay: 30s + capacity: + maxConcurrencyPerProxy: 20 + lifecycle: + ttl: 5m + allocationSafetyMargin: 20s + fetch: + requestInterval: 1s + timeout: 3s + maxAttempts: 3 + maxInFlight: 1 + maxTotal: 100000 + maxResponseBytes: 4194304 + templateTimeout: 100ms + retry: + initial: 500ms + max: 30s + jitter: 20 + check: + interval: 30s + jitter: 20 + maxInFlight: 200 + timeout: 3s + maxAttempts: 2 + maxConsecutiveFailures: 3 + urls: [https://example.com/] + provider-b: + enabled: true + exposure: [gateway, extract] + provider: + billingMode: fetch + protocols: [http] + api: + url: https://provider-b.invalid/api/proxies + method: GET + auth: + mode: apiKey + header: Authorization + token: env:PROVIDER_B_TOKEN + template: '{{ . }}' + proxyAuth: + mode: response + pool: + maxSize: 5000 + shrinkDelay: 30s + capacity: + maxConcurrencyPerProxy: 20 + lifecycle: + ttl: 5m + allocationSafetyMargin: 20s + fetch: + requestInterval: 1s + timeout: 3s + maxAttempts: 3 + maxInFlight: 1 + maxTotal: 100000 + maxResponseBytes: 4194304 + templateTimeout: 100ms + retry: + initial: 500ms + max: 30s + jitter: 20 + check: + interval: 30s + jitter: 20 + maxInFlight: 200 + timeout: 3s + maxAttempts: 2 + maxConsecutiveFailures: 3 + urls: [https://example.com/] + diff --git a/deploy/docker-compose.yml b/deploy/docker-compose.yml new file mode 100644 index 0000000..5d575dc --- /dev/null +++ b/deploy/docker-compose.yml @@ -0,0 +1,165 @@ +name: proxy-pool + +x-app: &app + build: + context: .. + dockerfile: deploy/docker/Dockerfile + image: proxy-pool:local + restart: unless-stopped + networks: [frontend, backend] + volumes: + - ./config/local.yaml:/etc/proxy-pool/config.yaml:ro + environment: + PROXY_POOL_CONFIG: /etc/proxy-pool/config.yaml + PROXY_POOL_GATEWAY_PASSWORD: ${PROXY_POOL_GATEWAY_PASSWORD:?set PROXY_POOL_GATEWAY_PASSWORD} + PROXY_POOL_EXTRACT_TOKEN: ${PROXY_POOL_EXTRACT_TOKEN:?set PROXY_POOL_EXTRACT_TOKEN} + PROXY_POOL_ADMIN_TOKEN: ${PROXY_POOL_ADMIN_TOKEN:?set PROXY_POOL_ADMIN_TOKEN} + PROVIDER_A_TOKEN: ${PROVIDER_A_TOKEN:?set PROVIDER_A_TOKEN} + PROVIDER_B_TOKEN: ${PROVIDER_B_TOKEN:?set PROVIDER_B_TOKEN} + stop_grace_period: 45s + depends_on: + postgres: + condition: service_healthy + redis: + condition: service_healthy + +services: + gateway-a: + <<: *app + command: ["proxy-gateway"] + expose: ["8080", "9090"] + healthcheck: + test: ["CMD", "curl", "--fail", "--silent", "http://127.0.0.1:9090/readyz"] + interval: 5s + timeout: 2s + retries: 12 + start_period: 10s + + gateway-b: + <<: *app + command: ["proxy-gateway"] + expose: ["8080", "9090"] + healthcheck: + test: ["CMD", "curl", "--fail", "--silent", "http://127.0.0.1:9090/readyz"] + interval: 5s + timeout: 2s + retries: 12 + start_period: 10s + + controller: + <<: *app + command: ["proxy-controller"] + expose: ["8081", "8082", "9090"] + ports: + - "127.0.0.1:8081:8081" + - "127.0.0.1:8082:8082" + healthcheck: + test: ["CMD", "curl", "--fail", "--silent", "http://127.0.0.1:9090/readyz"] + interval: 5s + timeout: 2s + retries: 12 + start_period: 15s + + checker: + <<: *app + command: ["proxy-checker"] + expose: ["9090"] + healthcheck: + test: ["CMD", "curl", "--fail", "--silent", "http://127.0.0.1:9090/readyz"] + interval: 10s + timeout: 2s + retries: 12 + start_period: 15s + + + haproxy: + image: haproxy:3.2-alpine + restart: unless-stopped + networks: [frontend] + ports: + - "127.0.0.1:8080:8080" + - "127.0.0.1:8404:8404" + volumes: + - ./haproxy/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro + depends_on: + gateway-a: + condition: service_healthy + gateway-b: + condition: service_healthy + healthcheck: + test: ["CMD", "wget", "-q", "-O", "/dev/null", "http://127.0.0.1:8404/healthz"] + interval: 5s + timeout: 2s + retries: 6 + + postgres: + image: postgres:18-alpine + restart: unless-stopped + networks: [backend] + environment: + POSTGRES_DB: proxy_pool + POSTGRES_USER: proxy_pool + POSTGRES_PASSWORD: local-only-change-me + volumes: + - postgres-data:/var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -U proxy_pool -d proxy_pool"] + interval: 5s + timeout: 3s + retries: 12 + + redis: + image: redis:8.2-alpine + restart: unless-stopped + command: ["redis-server", "--appendonly", "yes", "--save", "60", "1"] + networks: [backend] + volumes: + - redis-data:/data + healthcheck: + test: ["CMD", "redis-cli", "ping"] + interval: 5s + timeout: 3s + retries: 12 + + prometheus: + image: prom/prometheus:v3.5.0 + restart: unless-stopped + networks: [frontend, backend] + ports: + - "127.0.0.1:9091:9090" + volumes: + - ./prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro + - ./prometheus/rules:/etc/prometheus/rules:ro + - prometheus-data:/prometheus + command: + - --config.file=/etc/prometheus/prometheus.yml + - --storage.tsdb.path=/prometheus + - --storage.tsdb.retention.time=7d + - --web.enable-lifecycle + + grafana: + image: grafana/grafana:12.1.0 + restart: unless-stopped + networks: [backend] + ports: + - "127.0.0.1:3000:3000" + environment: + GF_SECURITY_ADMIN_USER: admin + GF_SECURITY_ADMIN_PASSWORD: local-only-change-me + GF_USERS_ALLOW_SIGN_UP: "false" + volumes: + - ./grafana/provisioning:/etc/grafana/provisioning:ro + - ./grafana/dashboards:/var/lib/grafana/dashboards:ro + - grafana-data:/var/lib/grafana + depends_on: [prometheus] + +networks: + frontend: {} + backend: + internal: true + +volumes: + postgres-data: {} + redis-data: {} + prometheus-data: {} + grafana-data: {} diff --git a/deploy/docker/Dockerfile b/deploy/docker/Dockerfile new file mode 100644 index 0000000..3837f30 --- /dev/null +++ b/deploy/docker/Dockerfile @@ -0,0 +1,30 @@ +# syntax=docker/dockerfile:1.7 +FROM golang:1.26-bookworm AS build + +WORKDIR /src +COPY go.mod go.sum ./ +RUN go mod download +COPY . . + +ARG TARGETOS=linux +ARG TARGETARCH=amd64 +RUN CGO_ENABLED=0 GOOS=${TARGETOS} GOARCH=${TARGETARCH} \ + go build -trimpath -ldflags="-s -w" -o /out/proxy-gateway ./cmd/proxy-gateway && \ + CGO_ENABLED=0 GOOS=${TARGETOS} GOARCH=${TARGETARCH} \ + go build -trimpath -ldflags="-s -w" -o /out/proxy-controller ./cmd/proxy-controller && \ + CGO_ENABLED=0 GOOS=${TARGETOS} GOARCH=${TARGETARCH} \ + go build -trimpath -ldflags="-s -w" -o /out/proxy-checker ./cmd/proxy-checker && \ + CGO_ENABLED=0 GOOS=${TARGETOS} GOARCH=${TARGETARCH} \ + go build -trimpath -ldflags="-s -w" -o /out/proxy-loadgen ./cmd/proxy-loadgen + +FROM debian:bookworm-slim + +RUN apt-get update && \ + apt-get install -y --no-install-recommends ca-certificates curl tini && \ + rm -rf /var/lib/apt/lists/* && \ + useradd --uid 10001 --create-home --shell /usr/sbin/nologin proxy-pool + +COPY --from=build /out/ /usr/local/bin/ +USER 10001:10001 +ENTRYPOINT ["/usr/bin/tini", "--"] + diff --git a/deploy/grafana/dashboards/proxy-pool-overview.json b/deploy/grafana/dashboards/proxy-pool-overview.json new file mode 100644 index 0000000..95992d2 --- /dev/null +++ b/deploy/grafana/dashboards/proxy-pool-overview.json @@ -0,0 +1,21 @@ +{ + "annotations": {"list": []}, + "editable": true, + "graphTooltip": 1, + "panels": [ + {"type":"timeseries","title":"Gateway QPS","gridPos":{"h":8,"w":8,"x":0,"y":0},"targets":[{"expr":"sum(rate(proxy_pool_gateway_requests_total[1m]))","legendFormat":"QPS"}]}, + {"type":"timeseries","title":"Gateway p99","gridPos":{"h":8,"w":8,"x":8,"y":0},"targets":[{"expr":"histogram_quantile(0.99, sum by (le) (rate(proxy_pool_gateway_request_duration_seconds_bucket[5m])))","legendFormat":"p99"}]}, + {"type":"timeseries","title":"Available Slots","gridPos":{"h":8,"w":8,"x":16,"y":0},"targets":[{"expr":"sum(proxy_pool_available_slots)","legendFormat":"slots"}]}, + {"type":"timeseries","title":"Provider Fetch","gridPos":{"h":8,"w":12,"x":0,"y":8},"targets":[{"expr":"sum by (result) (rate(proxy_pool_provider_fetch_total[5m]))","legendFormat":"{{result}}"}]}, + {"type":"timeseries","title":"Extraction","gridPos":{"h":8,"w":12,"x":12,"y":8},"targets":[{"expr":"sum by (result) (rate(proxy_pool_extraction_total[5m]))","legendFormat":"{{result}}"}]}, + {"type":"timeseries","title":"Snapshot Age","gridPos":{"h":8,"w":12,"x":0,"y":16},"targets":[{"expr":"max by (worker) (proxy_pool_snapshot_age_seconds)","legendFormat":"{{worker}}"}]}, + {"type":"timeseries","title":"Checker Queue","gridPos":{"h":8,"w":12,"x":12,"y":16},"targets":[{"expr":"sum(proxy_pool_checker_queue_depth)","legendFormat":"depth"}]} + ], + "schemaVersion": 41, + "tags": ["proxy-pool"], + "templating": {"list": []}, + "time": {"from":"now-6h","to":"now"}, + "title": "Proxy Pool Overview", + "uid": "proxy-pool-overview", + "version": 1 +} diff --git a/deploy/grafana/provisioning/dashboards/dashboards.yml b/deploy/grafana/provisioning/dashboards/dashboards.yml new file mode 100644 index 0000000..be51dff --- /dev/null +++ b/deploy/grafana/provisioning/dashboards/dashboards.yml @@ -0,0 +1,11 @@ +apiVersion: 1 +providers: + - name: proxy-pool + orgId: 1 + folder: Proxy Pool + type: file + disableDeletion: true + updateIntervalSeconds: 30 + options: + path: /var/lib/grafana/dashboards + diff --git a/deploy/grafana/provisioning/datasources/prometheus.yml b/deploy/grafana/provisioning/datasources/prometheus.yml new file mode 100644 index 0000000..f8dccdb --- /dev/null +++ b/deploy/grafana/provisioning/datasources/prometheus.yml @@ -0,0 +1,10 @@ +apiVersion: 1 +datasources: + - name: Prometheus + uid: prometheus + type: prometheus + access: proxy + url: http://prometheus:9090 + isDefault: true + editable: false + diff --git a/deploy/haproxy/haproxy.cfg b/deploy/haproxy/haproxy.cfg new file mode 100644 index 0000000..5b36e10 --- /dev/null +++ b/deploy/haproxy/haproxy.cfg @@ -0,0 +1,39 @@ +global + log stdout format raw local0 + maxconn 100000 + hard-stop-after 45s + +defaults + log global + mode tcp + option tcplog + timeout connect 3s + timeout client 2m + timeout server 2m + timeout tunnel 1h + +frontend proxy_gateway + bind :8080 + default_backend gateway_workers + +backend gateway_workers + balance leastconn + option tcp-check + default-server inter 2s fall 3 rise 2 slowstart 10s + server gateway-a gateway-a:8080 check resolvers docker init-addr libc,none + server gateway-b gateway-b:8080 check resolvers docker init-addr libc,none + +resolvers docker + nameserver dns 127.0.0.11:53 + resolve_retries 3 + timeout resolve 1s + timeout retry 1s + hold valid 10s + +frontend stats + mode http + bind :8404 + http-request use-service prometheus-exporter if { path /metrics } + http-request return status 200 content-type text/plain string ok if { path /healthz } + stats enable + stats uri /stats diff --git a/deploy/kubernetes/base/autoscaling.yaml b/deploy/kubernetes/base/autoscaling.yaml new file mode 100644 index 0000000..760c6f5 --- /dev/null +++ b/deploy/kubernetes/base/autoscaling.yaml @@ -0,0 +1,47 @@ +apiVersion: autoscaling/v2 +kind: HorizontalPodAutoscaler +metadata: {name: proxy-gateway, namespace: proxy-pool} +spec: + scaleTargetRef: {apiVersion: apps/v1, kind: Deployment, name: proxy-gateway} + minReplicas: 6 + maxReplicas: 60 + behavior: + scaleUp: + stabilizationWindowSeconds: 0 + policies: + - {type: Percent, value: 100, periodSeconds: 30} + - {type: Pods, value: 8, periodSeconds: 30} + selectPolicy: Max + scaleDown: + stabilizationWindowSeconds: 600 + policies: [{type: Percent, value: 10, periodSeconds: 60}] + metrics: + - type: Resource + resource: + name: cpu + target: {type: Utilization, averageUtilization: 55} + - type: Resource + resource: + name: memory + target: {type: Utilization, averageUtilization: 65} +--- +apiVersion: autoscaling/v2 +kind: HorizontalPodAutoscaler +metadata: {name: proxy-checker, namespace: proxy-pool} +spec: + scaleTargetRef: {apiVersion: apps/v1, kind: Deployment, name: proxy-checker} + minReplicas: 3 + maxReplicas: 30 + behavior: + scaleUp: + stabilizationWindowSeconds: 0 + policies: [{type: Percent, value: 100, periodSeconds: 30}] + scaleDown: + stabilizationWindowSeconds: 300 + policies: [{type: Percent, value: 20, periodSeconds: 60}] + metrics: + - type: Resource + resource: + name: cpu + target: {type: Utilization, averageUtilization: 60} + diff --git a/deploy/kubernetes/base/availability.yaml b/deploy/kubernetes/base/availability.yaml new file mode 100644 index 0000000..7f71f40 --- /dev/null +++ b/deploy/kubernetes/base/availability.yaml @@ -0,0 +1,21 @@ +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: {name: proxy-gateway, namespace: proxy-pool} +spec: + maxUnavailable: 1 + selector: {matchLabels: {app.kubernetes.io/name: proxy-gateway}} +--- +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: {name: proxy-controller, namespace: proxy-pool} +spec: + minAvailable: 2 + selector: {matchLabels: {app.kubernetes.io/name: proxy-controller}} +--- +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: {name: proxy-checker, namespace: proxy-pool} +spec: + minAvailable: 2 + selector: {matchLabels: {app.kubernetes.io/name: proxy-checker}} + diff --git a/deploy/kubernetes/base/checker.yaml b/deploy/kubernetes/base/checker.yaml new file mode 100644 index 0000000..bc0a001 --- /dev/null +++ b/deploy/kubernetes/base/checker.yaml @@ -0,0 +1,66 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: proxy-checker + namespace: proxy-pool + labels: {app.kubernetes.io/name: proxy-checker, app.kubernetes.io/part-of: proxy-pool} +spec: + replicas: 3 + minReadySeconds: 5 + revisionHistoryLimit: 3 + strategy: + type: RollingUpdate + rollingUpdate: {maxUnavailable: 1, maxSurge: 1} + selector: + matchLabels: {app.kubernetes.io/name: proxy-checker} + template: + metadata: + labels: {app.kubernetes.io/name: proxy-checker, app.kubernetes.io/part-of: proxy-pool} + annotations: {prometheus.io/scrape: "true", prometheus.io/port: "9090", prometheus.io/path: /metrics} + spec: + serviceAccountName: proxy-pool + automountServiceAccountToken: false + terminationGracePeriodSeconds: 45 + securityContext: {runAsNonRoot: true, seccompProfile: {type: RuntimeDefault}} + topologySpreadConstraints: + - maxSkew: 1 + topologyKey: kubernetes.io/hostname + whenUnsatisfiable: ScheduleAnyway + labelSelector: {matchLabels: {app.kubernetes.io/name: proxy-checker}} + containers: + - name: checker + image: REGISTRY/proxy-pool:VERSION + imagePullPolicy: IfNotPresent + command: [proxy-checker] + env: + - {name: PROXY_POOL_CONFIG, value: /etc/proxy-pool/config.yaml} + envFrom: + - secretRef: {name: proxy-pool-secrets} + ports: + - {name: metrics, containerPort: 9090} + readinessProbe: + httpGet: {path: /readyz, port: metrics} + periodSeconds: 5 + timeoutSeconds: 2 + livenessProbe: + httpGet: {path: /livez, port: metrics} + periodSeconds: 10 + timeoutSeconds: 2 + lifecycle: + preStop: {exec: {command: [sh, -c, "sleep 3"]}} + resources: + requests: {cpu: "1", memory: 512Mi} + limits: {cpu: "2", memory: 1Gi} + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: {drop: [ALL]} + volumeMounts: + - {name: config, mountPath: /etc/proxy-pool, readOnly: true} + - {name: tmp, mountPath: /tmp} + volumes: + - name: config + configMap: {name: proxy-pool-config} + - name: tmp + emptyDir: {sizeLimit: 64Mi} + diff --git a/deploy/kubernetes/base/configmap.yaml b/deploy/kubernetes/base/configmap.yaml new file mode 100644 index 0000000..a472e27 --- /dev/null +++ b/deploy/kubernetes/base/configmap.yaml @@ -0,0 +1,172 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: proxy-pool-config + namespace: proxy-pool +data: + config.yaml: | + version: 1 + security: + requireProtectionOnPublicListen: true + gateway: + enabled: true + listen: 0.0.0.0:8080 + access: + allowCIDRs: [0.0.0.0/0] + trustedProxies: [] + auth: + mode: usernamePassword + username: env:PROXY_POOL_GATEWAY_USERNAME + password: env:PROXY_POOL_GATEWAY_PASSWORD + limits: + maxConcurrentConnections: 100000 + requestsPerMinutePerClient: 60000 + retry: + maxAttempts: 2 + retryMethods: [GET, HEAD] + destinationPolicy: + denyPrivateNetworks: true + denyLoopback: true + denyLinkLocal: true + denyCIDRs: [169.254.169.254/32] + distribution: + enabled: true + listen: 0.0.0.0:8081 + access: + allowCIDRs: [10.0.0.0/8] + trustedProxies: [10.0.0.0/8] + auth: + mode: apiKey + header: X-API-Key + token: env:PROXY_POOL_EXTRACT_TOKEN + limits: + requestsPerMinute: 30000 + requestsPerMinutePerClient: 3000 + clientIdentification: + mode: trustedProxyOrRemoteIP + extraction: + fulfillment: partial + maxCountPerRequest: 100 + minRemainingTTL: 30s + maxHealthCheckAge: 30s + reserveForGateway: 5000 + admin: + enabled: true + listen: 0.0.0.0:8082 + access: + allowCIDRs: [10.0.0.0/8] + auth: + mode: apiKey + header: X-Admin-Token + token: env:PROXY_POOL_ADMIN_TOKEN + metrics: + enabled: true + listen: 0.0.0.0:9090 + storage: + postgresURL: env:PROXY_POOL_POSTGRES_URL + redisURL: env:PROXY_POOL_REDIS_URL + routing: + - name: gateway-default + enabled: true + purpose: gateway + upstreams: [provider-a, provider-b] + strategy: + type: sequential + switchAfterEmptyFetch: 5 + endBehavior: stayLast + onUnavailable: + action: reject + - name: extract-default + enabled: true + purpose: extract + upstreams: [provider-a, provider-b] + strategy: + type: sequential + switchAfterEmptyFetch: 5 + endBehavior: stayLast + onUnavailable: + action: reject + upstreams: + provider-a: + enabled: true + exposure: [gateway, extract] + provider: + billingMode: fetch + protocols: [http] + api: + url: https://PROVIDER_A_HOST/api/proxies + method: GET + auth: + mode: apiKey + header: Authorization + token: env:PROVIDER_A_TOKEN + template: '{{ . }}' + proxyAuth: + mode: response + pool: + maxSize: 25000 + shrinkDelay: 30s + capacity: + maxConcurrencyPerProxy: 20 + lifecycle: + ttl: 5m + allocationSafetyMargin: 20s + fetch: + requestInterval: 1s + timeout: 3s + maxAttempts: 3 + maxInFlight: 1 + maxTotal: 1000000 + maxResponseBytes: 4194304 + templateTimeout: 100ms + retry: {initial: 500ms, max: 30s, jitter: 20} + check: + interval: 30s + jitter: 20 + maxInFlight: 500 + timeout: 3s + maxAttempts: 2 + maxConsecutiveFailures: 3 + urls: [https://example.com/] + provider-b: + enabled: true + exposure: [gateway, extract] + provider: + billingMode: fetch + protocols: [http] + api: + url: https://PROVIDER_B_HOST/api/proxies + method: GET + auth: + mode: apiKey + header: Authorization + token: env:PROVIDER_B_TOKEN + template: '{{ . }}' + proxyAuth: + mode: response + pool: + maxSize: 25000 + shrinkDelay: 30s + capacity: + maxConcurrencyPerProxy: 20 + lifecycle: + ttl: 5m + allocationSafetyMargin: 20s + fetch: + requestInterval: 1s + timeout: 3s + maxAttempts: 3 + maxInFlight: 1 + maxTotal: 1000000 + maxResponseBytes: 4194304 + templateTimeout: 100ms + retry: {initial: 500ms, max: 30s, jitter: 20} + check: + interval: 30s + jitter: 20 + maxInFlight: 500 + timeout: 3s + maxAttempts: 2 + maxConsecutiveFailures: 3 + urls: [https://example.com/] + diff --git a/deploy/kubernetes/base/controller.yaml b/deploy/kubernetes/base/controller.yaml new file mode 100644 index 0000000..824dc2d --- /dev/null +++ b/deploy/kubernetes/base/controller.yaml @@ -0,0 +1,92 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: proxy-controller + namespace: proxy-pool + labels: {app.kubernetes.io/name: proxy-controller, app.kubernetes.io/part-of: proxy-pool} +spec: + replicas: 3 + minReadySeconds: 10 + revisionHistoryLimit: 3 + strategy: + type: RollingUpdate + rollingUpdate: {maxUnavailable: 1, maxSurge: 1} + selector: + matchLabels: {app.kubernetes.io/name: proxy-controller} + template: + metadata: + labels: {app.kubernetes.io/name: proxy-controller, app.kubernetes.io/part-of: proxy-pool} + annotations: {prometheus.io/scrape: "true", prometheus.io/port: "9090", prometheus.io/path: /metrics} + spec: + serviceAccountName: proxy-pool + automountServiceAccountToken: false + terminationGracePeriodSeconds: 60 + securityContext: {runAsNonRoot: true, seccompProfile: {type: RuntimeDefault}} + affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + topologyKey: kubernetes.io/hostname + labelSelector: {matchLabels: {app.kubernetes.io/name: proxy-controller}} + containers: + - name: controller + image: REGISTRY/proxy-pool:VERSION + imagePullPolicy: IfNotPresent + command: [proxy-controller] + env: + - {name: PROXY_POOL_CONFIG, value: /etc/proxy-pool/config.yaml} + envFrom: + - secretRef: {name: proxy-pool-secrets} + ports: + - {name: distribution, containerPort: 8081} + - {name: admin, containerPort: 8082} + - {name: control, containerPort: 8443} + - {name: metrics, containerPort: 9090} + readinessProbe: + httpGet: {path: /readyz, port: metrics} + periodSeconds: 5 + timeoutSeconds: 2 + failureThreshold: 3 + livenessProbe: + httpGet: {path: /livez, port: metrics} + periodSeconds: 10 + timeoutSeconds: 2 + failureThreshold: 3 + startupProbe: + httpGet: {path: /livez, port: metrics} + periodSeconds: 2 + failureThreshold: 45 + lifecycle: + preStop: {exec: {command: [sh, -c, "sleep 5"]}} + resources: + requests: {cpu: "1", memory: 1Gi} + limits: {cpu: "2", memory: 2Gi} + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: {drop: [ALL]} + volumeMounts: + - {name: config, mountPath: /etc/proxy-pool, readOnly: true} + - {name: tmp, mountPath: /tmp} + volumes: + - name: config + configMap: {name: proxy-pool-config} + - name: tmp + emptyDir: {sizeLimit: 64Mi} +--- +apiVersion: v1 +kind: Service +metadata: + name: proxy-controller + namespace: proxy-pool + labels: {app.kubernetes.io/name: proxy-controller} +spec: + type: ClusterIP + selector: {app.kubernetes.io/name: proxy-controller} + ports: + - {name: distribution, port: 8081, targetPort: distribution} + - {name: admin, port: 8082, targetPort: admin} + - {name: control, port: 8443, targetPort: control} + - {name: metrics, port: 9090, targetPort: metrics} + diff --git a/deploy/kubernetes/base/gateway.yaml b/deploy/kubernetes/base/gateway.yaml new file mode 100644 index 0000000..14e4af6 --- /dev/null +++ b/deploy/kubernetes/base/gateway.yaml @@ -0,0 +1,92 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: proxy-gateway + namespace: proxy-pool + labels: {app.kubernetes.io/name: proxy-gateway, app.kubernetes.io/part-of: proxy-pool} +spec: + replicas: 6 + minReadySeconds: 10 + revisionHistoryLimit: 3 + strategy: + type: RollingUpdate + rollingUpdate: {maxUnavailable: 1, maxSurge: 2} + selector: + matchLabels: {app.kubernetes.io/name: proxy-gateway} + template: + metadata: + labels: {app.kubernetes.io/name: proxy-gateway, app.kubernetes.io/part-of: proxy-pool} + annotations: {prometheus.io/scrape: "true", prometheus.io/port: "9090", prometheus.io/path: /metrics} + spec: + serviceAccountName: proxy-pool + automountServiceAccountToken: false + terminationGracePeriodSeconds: 60 + securityContext: {runAsNonRoot: true, seccompProfile: {type: RuntimeDefault}} + topologySpreadConstraints: + - maxSkew: 1 + topologyKey: topology.kubernetes.io/zone + whenUnsatisfiable: DoNotSchedule + labelSelector: {matchLabels: {app.kubernetes.io/name: proxy-gateway}} + - maxSkew: 1 + topologyKey: kubernetes.io/hostname + whenUnsatisfiable: ScheduleAnyway + labelSelector: {matchLabels: {app.kubernetes.io/name: proxy-gateway}} + containers: + - name: gateway + image: REGISTRY/proxy-pool:VERSION + imagePullPolicy: IfNotPresent + command: [proxy-gateway] + env: + - {name: PROXY_POOL_CONFIG, value: /etc/proxy-pool/config.yaml} + envFrom: + - secretRef: {name: proxy-pool-secrets} + ports: + - {name: proxy, containerPort: 8080, protocol: TCP} + - {name: metrics, containerPort: 9090, protocol: TCP} + readinessProbe: + httpGet: {path: /readyz, port: metrics} + periodSeconds: 3 + timeoutSeconds: 1 + failureThreshold: 3 + livenessProbe: + httpGet: {path: /livez, port: metrics} + periodSeconds: 10 + timeoutSeconds: 2 + failureThreshold: 3 + startupProbe: + httpGet: {path: /livez, port: metrics} + periodSeconds: 2 + failureThreshold: 30 + lifecycle: + preStop: {exec: {command: [sh, -c, "sleep 5"]}} + resources: + requests: {cpu: "2", memory: 1Gi} + limits: {cpu: "4", memory: 2Gi} + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: {drop: [ALL]} + volumeMounts: + - {name: config, mountPath: /etc/proxy-pool, readOnly: true} + - {name: tmp, mountPath: /tmp} + volumes: + - name: config + configMap: {name: proxy-pool-config} + - name: tmp + emptyDir: {sizeLimit: 64Mi} +--- +apiVersion: v1 +kind: Service +metadata: + name: proxy-gateway + namespace: proxy-pool + labels: {app.kubernetes.io/name: proxy-gateway} + annotations: + service.beta.kubernetes.io/aws-load-balancer-type: nlb +spec: + type: LoadBalancer + externalTrafficPolicy: Local + selector: {app.kubernetes.io/name: proxy-gateway} + ports: + - {name: proxy, port: 8080, targetPort: proxy, protocol: TCP} + diff --git a/deploy/kubernetes/base/kustomization.yaml b/deploy/kubernetes/base/kustomization.yaml new file mode 100644 index 0000000..a26498e --- /dev/null +++ b/deploy/kubernetes/base/kustomization.yaml @@ -0,0 +1,17 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - namespace.yaml + - serviceaccount.yaml + - configmap.yaml + - gateway.yaml + - controller.yaml + - checker.yaml + - availability.yaml + - autoscaling.yaml + - networkpolicy.yaml +images: + - name: REGISTRY/proxy-pool + newName: REGISTRY/proxy-pool + newTag: VERSION + diff --git a/deploy/kubernetes/base/namespace.yaml b/deploy/kubernetes/base/namespace.yaml new file mode 100644 index 0000000..66a093e --- /dev/null +++ b/deploy/kubernetes/base/namespace.yaml @@ -0,0 +1,10 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: proxy-pool + labels: + app.kubernetes.io/part-of: proxy-pool + pod-security.kubernetes.io/enforce: restricted + pod-security.kubernetes.io/audit: restricted + pod-security.kubernetes.io/warn: restricted + diff --git a/deploy/kubernetes/base/networkpolicy.yaml b/deploy/kubernetes/base/networkpolicy.yaml new file mode 100644 index 0000000..67f3627 --- /dev/null +++ b/deploy/kubernetes/base/networkpolicy.yaml @@ -0,0 +1,52 @@ +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: {name: default-deny, namespace: proxy-pool} +spec: + podSelector: {} + policyTypes: [Ingress, Egress] +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: {name: gateway-traffic, namespace: proxy-pool} +spec: + podSelector: {matchLabels: {app.kubernetes.io/name: proxy-gateway}} + policyTypes: [Ingress, Egress] + ingress: + - ports: [{port: 8080, protocol: TCP}] + - from: + - namespaceSelector: {matchLabels: {kubernetes.io/metadata.name: monitoring}} + ports: [{port: 9090, protocol: TCP}] + egress: + - to: + - namespaceSelector: {matchLabels: {kubernetes.io/metadata.name: kube-system}} + ports: [{port: 53, protocol: UDP}, {port: 53, protocol: TCP}] + - to: + - podSelector: {matchLabels: {app.kubernetes.io/name: proxy-controller}} + ports: [{port: 8443, protocol: TCP}] + # Gateway 需要连接任意公网目标；应用层 DestinationPolicy 仍拒绝私网、回环和元数据地址。 + - to: + - ipBlock: + cidr: 0.0.0.0/0 + except: [10.0.0.0/8, 100.64.0.0/10, 127.0.0.0/8, 169.254.0.0/16, 172.16.0.0/12, 192.168.0.0/16] +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: {name: control-plane-traffic, namespace: proxy-pool} +spec: + podSelector: + matchExpressions: + - {key: app.kubernetes.io/name, operator: In, values: [proxy-controller, proxy-checker]} + policyTypes: [Ingress, Egress] + ingress: + - from: + - namespaceSelector: {matchLabels: {kubernetes.io/metadata.name: proxy-pool}} + - from: + - namespaceSelector: {matchLabels: {kubernetes.io/metadata.name: monitoring}} + ports: [{port: 9090, protocol: TCP}] + egress: + - to: + - namespaceSelector: {matchLabels: {kubernetes.io/metadata.name: kube-system}} + ports: [{port: 53, protocol: UDP}, {port: 53, protocol: TCP}] + # Provider、健康目标及外部托管 PostgreSQL/Redis 的精确网段应在环境 Overlay 收紧。 + - to: [{ipBlock: {cidr: 0.0.0.0/0}}] + diff --git a/deploy/kubernetes/base/secret.example.yaml b/deploy/kubernetes/base/secret.example.yaml new file mode 100644 index 0000000..ead02fc --- /dev/null +++ b/deploy/kubernetes/base/secret.example.yaml @@ -0,0 +1,16 @@ +apiVersion: v1 +kind: Secret +metadata: + name: proxy-pool-secrets + namespace: proxy-pool +type: Opaque +stringData: + PROXY_POOL_GATEWAY_USERNAME: GATEWAY_USER + PROXY_POOL_GATEWAY_PASSWORD: GATEWAY_PASSWORD + PROXY_POOL_EXTRACT_TOKEN: EXTRACT_TOKEN + PROXY_POOL_ADMIN_TOKEN: ADMIN_TOKEN + PROXY_POOL_POSTGRES_URL: postgres://USER:PASSWORD@POSTGRES_HOST:5432/proxy_pool?sslmode=verify-full + PROXY_POOL_REDIS_URL: rediss://:PASSWORD@REDIS_HOST:6379/0 + PROVIDER_A_TOKEN: PROVIDER_A_TOKEN + PROVIDER_B_TOKEN: PROVIDER_B_TOKEN + diff --git a/deploy/kubernetes/base/serviceaccount.yaml b/deploy/kubernetes/base/serviceaccount.yaml new file mode 100644 index 0000000..e7a0386 --- /dev/null +++ b/deploy/kubernetes/base/serviceaccount.yaml @@ -0,0 +1,7 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: proxy-pool + namespace: proxy-pool +automountServiceAccountToken: false + diff --git a/deploy/prometheus/prometheus.yml b/deploy/prometheus/prometheus.yml new file mode 100644 index 0000000..325e204 --- /dev/null +++ b/deploy/prometheus/prometheus.yml @@ -0,0 +1,24 @@ +global: + scrape_interval: 15s + evaluation_interval: 15s + external_labels: + environment: local + +rule_files: + - /etc/prometheus/rules/*.yml + +scrape_configs: + - job_name: proxy-gateway + static_configs: + - targets: [gateway-a:9090, gateway-b:9090] + - job_name: proxy-controller + static_configs: + - targets: [controller:9090] + - job_name: proxy-checker + static_configs: + - targets: [checker:9090] + - job_name: haproxy + metrics_path: /metrics + static_configs: + - targets: [haproxy:8404] + diff --git a/deploy/prometheus/rules/proxy-pool.yml b/deploy/prometheus/rules/proxy-pool.yml new file mode 100644 index 0000000..c68ead8 --- /dev/null +++ b/deploy/prometheus/rules/proxy-pool.yml @@ -0,0 +1,48 @@ +groups: + - name: proxy-pool + rules: + - alert: ProxyPoolGatewayHighErrorRate + expr: | + sum(rate(proxy_pool_gateway_requests_total{result="error"}[5m])) + / clamp_min(sum(rate(proxy_pool_gateway_requests_total[5m])), 1) > 0.02 + for: 10m + labels: + severity: warning + annotations: + summary: Gateway 错误率持续高于 2% + - alert: ProxyPoolGatewaySnapshotStale + expr: proxy_pool_snapshot_age_seconds > 60 + for: 2m + labels: + severity: critical + annotations: + summary: Gateway Snapshot 已超过安全陈旧时间 + - alert: ProxyPoolNoAvailableSlots + expr: sum(proxy_pool_available_slots) == 0 + for: 1m + labels: + severity: critical + annotations: + summary: Gateway 可分配容量耗尽 + - alert: ProxyPoolProviderFetchErrors + expr: sum by (upstream) (rate(proxy_pool_provider_fetch_total{result="error"}[10m])) > 0.2 + for: 10m + labels: + severity: warning + annotations: + summary: Provider Fetch 错误持续发生 + - alert: ProxyPoolCheckerBacklog + expr: proxy_pool_checker_queue_depth > 10000 + for: 5m + labels: + severity: warning + annotations: + summary: Checker 队列积压 + - alert: ProxyPoolExtractionConflict + expr: sum(rate(proxy_pool_extraction_total{result="conflict"}[5m])) > 0 + for: 5m + labels: + severity: warning + annotations: + summary: 独占提取发生持续事务冲突 + diff --git a/deploy/tools/configcheck/main.go b/deploy/tools/configcheck/main.go new file mode 100644 index 0000000..1585282 --- /dev/null +++ b/deploy/tools/configcheck/main.go @@ -0,0 +1,29 @@ +// configcheck 使用与进程启动相同的严格加载器校验部署配置。 +package main + +import ( + "fmt" + "os" + + "github.com/proxy-pool/proxy-pool/internal/config" +) + +func main() { + if len(os.Args) != 2 { + fmt.Fprintln(os.Stderr, "usage: configcheck CONFIG_FILE") + os.Exit(2) + } + + file, err := os.Open(os.Args[1]) + if err != nil { + fmt.Fprintf(os.Stderr, "open config: %v\n", err) + os.Exit(1) + } + defer file.Close() + + if _, err := config.Load(file); err != nil { + fmt.Fprintf(os.Stderr, "invalid config: %v\n", err) + os.Exit(1) + } + fmt.Printf("valid config: %s\n", os.Args[1]) +} diff --git a/diagrams/README.md b/diagrams/README.md new file mode 100644 index 0000000..ee21838 --- /dev/null +++ b/diagrams/README.md @@ -0,0 +1,536 @@ +# Proxy Pool Mermaid 图集 + +本图集依据最终需求语义绘制。图中的 100k QPS 是待压测验证的集群目标；Extract +均表示一次性独占发放，不存在 Lease、Renew 或 Release。 + +## 01 系统上下文 + +```mermaid +flowchart LR + Client[Gateway Client] --> LB[Layer 4 Load Balancer] + ExtractClient[Extract Client] --> Dist[Distribution API] + Operator[Operator] --> Admin[Admin API] + LB --> Gateway[Gateway Cluster] + Gateway --> Internet[Target via Proxy] + Dist --> Controller[Controller Cluster] + Admin --> Controller + Controller --> Provider[Provider APIs] + Controller --> Checker[Checker Cluster] + Controller --> PG[(PostgreSQL)] + Controller --> Redis[(Redis)] +``` + +## 02 进程职责边界 + +```mermaid +flowchart TB + subgraph DataPlane[Data Plane] + G[proxy-gateway] + Snap[Immutable Snapshot] + G --> Snap + end + subgraph ControlPlane[Control Plane] + C[proxy-controller] + K[proxy-checker] + C <--> K + end + subgraph Tools[Tools] + L[proxy-loadgen] + end + C -->|Snapshot and ownership| G + G -->|batched outcomes| C + L --> G +``` + +## 03 领域模块依赖 + +```mermaid +flowchart TD + Cmd[cmd assembly] --> Gateway[gateway modules] + Cmd --> Controller[controller modules] + Cmd --> Adapters[adapters] + Gateway --> Domain[domain] + Controller --> Domain + Adapters --> Domain + Domain -. no import .-> HTTP[(HTTP)] + Domain -. no import .-> SQL[(SQL)] + Domain -. no import .-> Redis[(Redis)] +``` + +## 04 Gateway 请求路径 + +```mermaid +sequenceDiagram + participant C as Client + participant G as Gateway + participant D as Dispatcher + participant T as Transport + participant P as Proxy + C->>G: HTTP request + G->>G: auth and admission + G->>D: acquire route + D->>D: reserve capacity with CAS + D-->>G: allocation + G->>T: execute + T->>P: dial and handshake + P-->>T: response + T-->>C: stream response + T->>D: release active and report +``` + +## 05 CONNECT 提交点 + +```mermaid +stateDiagram-v2 + [*] --> Accepted + Accepted --> Reserved: Acquire + Reserved --> Dialing: dial proxy + Dialing --> Cancelled: fail before commit + Dialing --> Active: proxy CONNECT succeeds + Active --> TunnelCommitted: send 200 to client + TunnelCommitted --> Closed: stream ends + Cancelled --> [*] + Closed --> [*] + note right of TunnelCommitted: transparent replay forbidden +``` + +## 06 HTTP 安全重试决策 + +```mermaid +flowchart TD + F[Attempt failed] --> H{Headers sent to client?} + H -->|yes| Stop[Do not retry] + H -->|no| M{Method allowed?} + M -->|no| Stop + M -->|GET or HEAD| A{Attempts remain?} + A -->|no| Stop + A -->|yes| X[Exclude failed Proxy] + X --> N[Acquire another Proxy] +``` + +## 07 Routing 首条命中 + +```mermaid +flowchart TD + Req[RouteRequest] --> R1{Rule 1 matches?} + R1 -->|yes| U1[Use Rule 1 upstream strategy] + R1 -->|no| R2{Rule 2 matches?} + R2 -->|yes| U2[Use Rule 2 upstream strategy] + R2 -->|no| RN{Default rule matches?} + RN -->|yes| UN[Use default strategy] + RN -->|no| Reject[Apply onUnavailable] +``` + +## 08 Sequential 原子切换 + +```mermaid +sequenceDiagram + participant F1 as Fetch goroutine 1 + participant F2 as Fetch goroutine 2 + participant U as Upstream A counter + participant R as Routing state + F1->>U: empty reaches threshold + F2->>U: concurrent empty + U->>R: depleted generation 7 + U->>R: depleted generation 7 + R->>R: CAS A to B succeeds once + R-->>F1: current B + R-->>F2: current B +``` + +## 09 Provider Fetch 调度 + +```mermaid +flowchart TD + Signal[Capacity signal] --> SF{Fetch already running?} + SF -->|yes| Merge[Merge into bounded signal] + SF -->|no| Leader[Acquire logical leader] + Leader --> Demand[Recompute slot demand] + Demand --> Limit{Below maxSize and maxTotal?} + Limit -->|no| Done[Stop] + Limit -->|yes| Rate[Wait requestInterval] + Rate --> Call[Call Provider under maxInFlight] + Call --> Classify[Classify result] +``` + +## 10 Fetch 结果分类 + +```mermaid +flowchart LR + Response[Provider response] --> Transport{Transport and auth valid?} + Transport -->|no| Error[Error and backoff] + Transport -->|yes| Parse{Template and parse valid?} + Parse -->|no| Error + Parse -->|yes| Legal{Legal candidates count} + Legal -->|zero| Empty[Empty plus one] + Legal -->|positive| New{New after dedupe?} + New -->|none| Duplicate[Duplicate-only and reset Empty] + New -->|some| Success[Success and reset Empty] +``` + +## 11 退避与 Retry-After + +```mermaid +stateDiagram-v2 + [*] --> Ready + Ready --> Calling: rate token acquired + Calling --> Ready: success or empty + Calling --> RetryAfter: HTTP 429 + Calling --> Backoff: timeout or server error + RetryAfter --> Ready: provider deadline reached + Backoff --> Ready: exponential delay plus jitter + Backoff --> Open: max attempts exhausted + Open --> Ready: next scheduled cycle +``` + +## 12 Pool Reconcile + +```mermaid +flowchart TD + Inventory[Managed inventory] --> Count[Count states and pending expected] + Capacity[Available slots] --> Need[Compute demand] + Count --> Bound{pool maxSize reached?} + Need --> Bound + Bound -->|yes| NoFetch[Do not fetch] + Bound -->|no| Quota{fetch maxTotal reached?} + Quota -->|yes| NoFetch + Quota -->|no| Fetch[Schedule bounded fetch] +``` + +## 13 Proxy 生命周期 + +```mermaid +stateDiagram-v2 + [*] --> FETCHED + FETCHED --> CHECKING + CHECKING --> AVAILABLE: passed + CHECKING --> UNHEALTHY: exhausted + AVAILABLE --> SUSPECT: meaningful failure + SUSPECT --> AVAILABLE: recheck passed + SUSPECT --> UNHEALTHY: failures reached + AVAILABLE --> DRAINING: expiry or revoke + AVAILABLE --> EXTRACTED: exclusive transaction + DRAINING --> EXPIRED: capacity zero + UNHEALTHY --> REMOVED + EXTRACTED --> EXPIRED: TTL reached + EXPIRED --> REMOVED +``` + +## 14 原子容量转换 + +```mermaid +flowchart LR + C0[active A reserved R] --> Check{A plus R below limit?} + Check -->|no| Full[Reject candidate] + Check -->|yes CAS| Reserved[active A reserved R plus 1] + Reserved -->|Commit| Active[active A plus 1 reserved R] + Reserved -->|Cancel| C0 + Active -->|Release| C0 +``` + +## 15 Worker 所有权 + +```mermaid +flowchart TB + Controller[Controller allocator] -->|epoch 12| W1[Worker 1] + Controller -->|epoch 12| W2[Worker 2] + P1[Proxy shard A] --> W1 + P2[Proxy shard B] --> W2 + U[Unowned inventory] --> Controller + W1 -. cannot allocate .-> P2 + W2 -. cannot allocate .-> P1 +``` + +## 16 Snapshot 发布与 ACK + +```mermaid +sequenceDiagram + participant DB as PostgreSQL + participant C as Controller + participant W as Worker + DB->>C: outbox revision 42 + C->>C: build worker snapshot + C->>W: epoch 8 version 42 checksum + W->>W: validate and build indexes + W->>W: atomic swap + W-->>C: ACK epoch 8 version 42 + C->>DB: mark outbox delivered +``` + +## 17 Snapshot 缺口恢复 + +```mermaid +flowchart TD + Delta[Receive delta version 45] --> Current{Current version is 44?} + Current -->|yes| Check[Verify checksum and epoch] + Current -->|no current 42| Reject[Reject delta] + Reject --> Full[Request full snapshot] + Full --> Build[Build indexes in background] + Check --> Apply[Apply delta atomically] + Build --> Apply +``` + +## 18 Snapshot 陈旧状态 + +```mermaid +stateDiagram-v2 + [*] --> Fresh + Fresh --> StaleAllowed: controller disconnected + StaleAllowed --> Fresh: valid snapshot received + StaleAllowed --> DrainOnly: maxStaleAge exceeded + DrainOnly --> Fresh: full snapshot and new epoch + DrainOnly --> Stopped: existing traffic drained +``` + +## 19 健康任务调度 + +```mermaid +flowchart TD + Proxies[Proxy inventory] --> Priority{State priority} + Priority -->|new| New[Immediate basic check] + Priority -->|SUSPECT| Fast[Fast recheck] + Priority -->|stable AVAILABLE| Normal[Normal interval] + New --> Jitter[Stable hash plus jitter] + Fast --> Jitter + Normal --> Jitter + Jitter --> Bound[maxInFlight semaphore] + Bound --> Checker[Checker workers] +``` + +## 20 健康 Observation Reducer + +```mermaid +flowchart LR + Obs[Health Observation] --> Scope{Scope} + Scope -->|global| Global[Global health reducer] + Scope -->|route target| Target[Target profile reducer] + Global --> Consecutive[Consecutive outcome state] + Consecutive --> Transition[AVAILABLE SUSPECT UNHEALTHY] + Target --> RouteHealth[Only affected route health] + RouteHealth -. no direct global delete .-> Transition +``` + +## 21 Extract partial + +```mermaid +sequenceDiagram + participant C as Client + participant API as Distribution + participant DB as PostgreSQL + C->>API: count 10 fulfillment partial + API->>DB: lock eligible rows + DB-->>API: 6 rows + API->>DB: update 6 to EXTRACTED and audit + DB-->>API: commit + API-->>C: requested 10 returned 6 +``` + +## 22 Extract allOrNothing + +```mermaid +sequenceDiagram + participant C as Client + participant API as Distribution + participant DB as PostgreSQL + C->>API: count 10 fulfillment allOrNothing + API->>DB: lock eligible rows + DB-->>API: only 6 rows + API->>DB: rollback entire transaction + API-->>C: insufficient inventory and returned 0 +``` + +## 23 Gateway 所有权回收后提取 + +```mermaid +sequenceDiagram + participant C as Controller + participant W as Gateway Worker + participant DB as PostgreSQL + C->>W: mark Proxy DRAINING at epoch 13 + W->>W: stop new allocations + W-->>C: ACK active 0 reserved 0 + C->>DB: clear worker ownership + C->>DB: AVAILABLE to EXTRACTED plus audit + DB-->>C: committed exclusive result +``` + +## 24 reserveForGateway 不变量 + +```mermaid +flowchart TD + Eligible[Eligible unowned count] --> Formula[extractable equals eligible minus reserve] + Reserve[reserveForGateway] --> Formula + Requested[requested count] --> Min[return min requested and extractable] + Formula --> Min + Min --> Result{fulfillment} + Result -->|partial| Commit[Commit available quantity] + Result -->|allOrNothing insufficient| Rollback[Return zero] +``` + +## 25 PostgreSQL 独占事务 + +```mermaid +flowchart TD + Begin[BEGIN] --> Select[SELECT eligible FOR UPDATE SKIP LOCKED] + Select --> Enough{Quantity satisfies mode?} + Enough -->|no allOrNothing| Rollback[ROLLBACK] + Enough -->|yes or partial| Update[UPDATE AVAILABLE to EXTRACTED] + Update --> Audit[INSERT extraction records] + Audit --> Outbox[INSERT outbox] + Outbox --> Commit[COMMIT] + Commit --> Return[Return proxies with expiry] +``` + +## 26 并发提取互斥 + +```mermaid +sequenceDiagram + participant A as Extract request A + participant DB as PostgreSQL + participant B as Extract request B + A->>DB: lock rows 1 to 10 + B->>DB: skip locked rows 1 to 10 + B->>DB: lock rows 11 to 20 + A->>DB: commit EXTRACTED 1 to 10 + B->>DB: commit EXTRACTED 11 to 20 + Note over A,B: no Proxy returned twice +``` + +## 27 Outbox 一致性 + +```mermaid +sequenceDiagram + participant C as Controller transaction + participant DB as PostgreSQL + participant P as Publisher + participant W as Worker + C->>DB: state change plus outbox + DB-->>C: atomic commit + P->>DB: read undelivered event + P->>W: publish versioned event + W-->>P: idempotent ACK + P->>DB: mark delivered +``` + +## 28 配置热更新 + +```mermaid +flowchart LR + File[Read revision] --> Parse[Strict parse] + Parse --> Validate[References regex and security] + Validate --> Build[Build immutable config] + Build --> Diff[Diff tasks and resources] + Diff --> Swap[Atomic swap] + Swap --> Drain[Drain removed resources] + Validate -->|error| Keep[Keep old revision] +``` + +## 29 Gateway 优雅停机 + +```mermaid +sequenceDiagram + participant K as Kubernetes + participant G as Gateway + participant LB as Load Balancer + K->>G: SIGTERM + G->>G: readiness false + LB->>LB: remove endpoint + G->>G: reject new connections + G->>G: drain requests and tunnels + G->>G: release active capacities + G-->>K: exit before grace timeout +``` + +## 30 Controller 优雅停机 + +```mermaid +flowchart TD + Term[SIGTERM] --> NotReady[Readiness false] + NotReady --> StopWrites[Stop new Extract and Admin writes] + StopWrites --> StopFetch[Stop new Fetch] + StopFetch --> FinishTx[Commit or rollback current transactions] + FinishTx --> Flush[Flush outbox and reports] + Flush --> Lease[Release Provider leader lease] + Lease --> Exit[Close pools and exit] +``` + +## 31 Kubernetes 故障域拓扑 + +```mermaid +flowchart TB + LB[Load Balancer] --> ZA + LB --> ZB + LB --> ZC + subgraph ZA[Zone A] + GA1[Gateway] + CA[Controller] + KA[Checker] + end + subgraph ZB[Zone B] + GB1[Gateway] + CB[Controller] + KB[Checker] + end + subgraph ZC[Zone C] + GC1[Gateway] + CC[Controller] + KC[Checker] + end +``` + +## 32 Gateway 扩缩决策 + +```mermaid +flowchart TD + Metrics[QPS CPU memory connections latency] --> HPA[HPA decision] + HPA --> Up{Above target?} + Up -->|yes| ScaleUp[Scale up quickly] + Up -->|no| Stable{Stable below target for 10m?} + Stable -->|no| Hold[Hold replicas] + Stable -->|yes| Capacity{Failure-domain headroom remains?} + Capacity -->|no| Hold + Capacity -->|yes| ScaleDown[Scale down at most 10 percent] +``` + +## 33 网络信任边界 + +```mermaid +flowchart LR + Internet --> NLB[Public NLB] + NLB --> Gateway[Gateway 8080] + Ingress[Private Ingress] --> Distribution[Distribution 8081] + Operator[Operator VPN] --> Admin[Admin 8082] + Monitor[Monitoring namespace] --> Metrics[Metrics 9090] + Gateway --> PublicTargets[Public targets only] + Controller[Controller] --> Stores[External PG and Redis] + Controller --> Providers[Provider APIs] +``` + +## 34 可观测信号流 + +```mermaid +flowchart LR + Gateway -->|bounded metrics| Prom[Prometheus] + Controller -->|bounded metrics| Prom + Checker -->|bounded metrics| Prom + Prom --> Grafana[Grafana dashboards] + Prom --> Alerts[Alert rules] + Gateway -->|sampled redacted logs| Logs[Log backend] + Controller -->|audit events| Audit[(Audit storage)] + Alerts --> OnCall[On-call] +``` + +## 35 100k QPS 验证流程 + +```mermaid +flowchart TD + Baseline[Measure one Worker on production shape] --> Formula[Compute replicas at 60 percent target] + Formula --> Warmup[Warm Snapshot and connections] + Warmup --> Steady[Run 10k steady] + Steady --> Ramp[Step ramp to 100k] + Ramp --> Peak[Hold 100k peak window] + Peak --> Failure[Remove largest failure domain] + Failure --> Verify[Verify SLO and all invariants] + Verify --> Evidence[Archive raw metrics config and image digest] +``` + diff --git a/docs/adr/README.md b/docs/adr/README.md new file mode 100644 index 0000000..9df4793 --- /dev/null +++ b/docs/adr/README.md @@ -0,0 +1,34 @@ +# 架构决策记录 + +## ADR-001：控制面与数据面分离 + +**状态：** 接受。 + +Gateway 只依赖本地不可变 Snapshot；Provider、数据库、配置重载和健康聚合 +位于 Controller/Checker。该选择隔离外部 I/O 抖动，并允许数据面按 QPS、 +控制面按 Provider/库存规模独立扩容。 + +## ADR-002：Distribution 使用独占提取 + +**状态：** 接受并覆盖早期 Lease 方案。 + +成功响应前在一个事务中执行 `AVAILABLE -> EXTRACTED`。系统保存审计事实， +但不提供 release、renew 或使用跟踪。这样契合“拿走真实代理后平台不再管理” +的最终产品语义，并消除重复发放。 + +## ADR-003：单 Worker 所有权 + +**状态：** 接受。 + +一个 Proxy 同一时刻至多归属一个 Worker，Gateway 在本地维护 Active/Reserved。 +Distribution 只提取无所有权 Proxy；回收时执行 drain/ACK/归零/解除所有权。 +该选择避免每请求访问 Redis 做全局并发计数。 + +## ADR-004：PostgreSQL 权威、Redis 可重建 + +**状态：** 接受。 + +Proxy 状态、Extraction、配置版本和 Outbox 由 PostgreSQL 持久化。Redis 只 +承担 Leader、短期速率与心跳等协调；Redis 丢失后可从权威状态恢复，避免 +双写状态成为不可判定的事实源。 + diff --git a/docs/api/admin.md b/docs/api/admin.md new file mode 100644 index 0000000..3e990aa --- /dev/null +++ b/docs/api/admin.md @@ -0,0 +1,18 @@ +# Admin API + +Admin API 使用独立监听器与权限，契约位于 `api/openapi/admin.yaml`。公网部署 +不得与 Distribution 复用认证 Token；推荐只绑定管理网段或回环地址。 + +## 端点 + +- `GET /api/v1/status`：返回配置/快照版本、Upstream 聚合计数和 Worker 状态。 +- `POST /api/v1/upstreams/{name}/enable`：启用 Upstream。 +- `POST /api/v1/upstreams/{name}/disable`：停止新 Fetch/分配并自然 Drain。 +- `POST /api/v1/routing/{name}/switch`：用 expectedCurrent 做 CAS 手工切换。 +- `POST /api/v1/config/reload`：严格解析并原子发布新配置。 + +所有写操作写审计记录并返回最终 Request ID 与版本。Enable/Disable 对目标状态 +幂等；Routing Switch 必须携带 `expectedCurrent`，避免并发操作跳过多个供应商。 + +配置重载校验失败返回 422，旧配置继续运行。Status 只返回低基数聚合信息， +不得返回 Proxy 地址、凭据、Client 标识或完整 Provider URL。 diff --git a/docs/api/control-plane.md b/docs/api/control-plane.md new file mode 100644 index 0000000..c02d5f3 --- /dev/null +++ b/docs/api/control-plane.md @@ -0,0 +1,103 @@ +# Control Plane gRPC API + +## 1. 契约范围 + +Proto 源文件位于 `api/proto/controlplane/v1/controlplane.proto`，包含两项 +内部服务： + +- `WorkerControlPlane`：Worker 注册、Snapshot/Delta 分发、ACK、运行态与结果 + 批量上报。 +- `CheckerControlPlane`：健康检查任务流和 Observation 批量上报。 + +该协议不承载 Client 的独占提取，也没有 extraction lease/release。Proxy 的 +`AVAILABLE -> EXTRACTED` 只在 Controller 权威事务中完成。 + +## 2. Worker 会话 + +```mermaid +sequenceDiagram + participant W as Gateway Worker + participant C as Controller + W->>C: RegisterWorker(worker, instance, zone) + C-->>W: session + ownershipEpoch + maxStaleAge + W->>C: WatchSnapshots(lastVersion, checksum) + C-->>W: full WorkerSnapshot + W->>W: validate + build immutable snapshot + W->>W: atomic swap + W->>C: AcknowledgeSnapshot(version, epoch, checksum) + loop bounded interval + W->>C: ReportRuntime(active, reserved, draining) + W->>C: ReportOutcomes(batch sequence) + end +``` + +`worker_id` 是逻辑节点，`instance_id` 区分进程重启，`session_id` 防止旧进程 +继续上报。所有权 `epoch` 小于 Controller 当前值的数据必须拒绝。 + +## 3. Snapshot 与 Delta + +完整 Snapshot 包含： + +- 单调 `version`、`ownership_epoch`、生成时间和有效期。 +- 对该 Worker 可见的有序 Routing。 +- 仅归该 Worker 所有的 Proxy 与每个 Proxy 的容量。 +- 内容 `checksum`。 + +Delta 声明 `base_version`。Worker 只有在本地版本恰好等于 base 且 checksum +验证成功时才能应用；否则丢弃 Delta 并请求完整 Snapshot。构建在后台完成， +热路径只读取一次原子指针。 + +超过 `max_stale_age` 仍未取得有效快照时，Worker 停止接收新流量并排空已有 +请求。控制面中断不能让 Worker 查询 PostgreSQL 或 Redis 补偿热路径。 + +## 4. 所有权与 Drain + +同一 Proxy 同时只归一个 Worker。Controller 回收用于独占提取的 Proxy 时： + +1. 新 Snapshot 标记或移除该 Proxy，使 Worker 停止新预留。 +2. Worker 上报 `draining=true` 以及 Active/Reserved。 +3. 两个计数都归零后 Controller 清除所有权。 +4. 无所有权 Proxy 才能进入 Distribution 提取事务。 + +Worker 崩溃时必须等待所有权 epoch/有效期失效后再转移，避免双主。Proto 中 +`ReportRuntimeResponse.revoke_proxy_ids` 是加速 Drain 的控制信号，不绕过 +Snapshot 版本和权威持久化。 + +## 5. Outcome 上报 + +Outcome 按 Worker 单调 `sequence` 批量上报。Controller 返回已接受的最大序号， +从而支持有限重试和去重。阶段区分： + +- `DIAL`：连接 Proxy 地址失败。 +- `PROXY_HANDSHAKE`：HTTP CONNECT 或 SOCKS 握手失败。 +- `RESPONSE_HEADERS`：目标响应头前失败。 +- `TUNNEL`：隧道建立后结束或失败。 + +Outcome 是 Observation，不直接让 Worker 修改 PostgreSQL 状态。异步上报队列 +必须有界；队列满时丢弃低价值样本并计指标，不能反压 Gateway 热路径。 + +## 6. Checker 任务 + +Checker 注册自身最大并发与支持层级，Controller 发送有 deadline 的任务： + +- `BASIC`：基础连通和协议握手。 +- `EGRESS`：出口身份与匿名性。 +- `TARGET`：针对 Routing/目标组的可达性。 + +Checker 只返回 `HealthObservation`。Controller reducer 按 Proxy、检查层级和 +Routing 决定 AVAILABLE、SUSPECT 或 UNHEALTHY，避免多个 Checker 并发写状态。 + +## 7. 兼容与演进 + +- Proto 字段号一旦发布不得复用。 +- 删除字段使用 `reserved` 保留名称和编号。 +- 新枚举值必须让旧接收方按 UNSPECIFIED/拒绝策略处理。 +- Worker 注册携带 `supported_protocol_version`，不兼容时注册失败而不是静默 + 降级。 +- Stream 断开后使用带 jitter 的有界指数退避，禁止紧密重连。 + +## 8. 传输安全 + +集群环境使用 mTLS，证书身份绑定 Worker/Checker 类型和环境。服务端校验 +消息中的逻辑 ID 与证书授权一致，设置单消息大小、流持续时间、并发 Stream +和上报批次上限。`secret_ref` 是受控引用，不在 Proto 中传播真实密码。 diff --git a/docs/api/distribution.md b/docs/api/distribution.md new file mode 100644 index 0000000..09f8649 --- /dev/null +++ b/docs/api/distribution.md @@ -0,0 +1,200 @@ +# Distribution API + +## 1. 行为契约 + +Distribution API 只提供一次性独占提取： + +```text +筛选 AVAILABLE + -> 锁定候选 + -> 校验 Gateway 预留、TTL、健康与过滤条件 + -> 原子 AVAILABLE -> EXTRACTED + -> 写 Extraction Record + -> 提交事务 + -> 返回真实代理地址 +``` + +事务提交前不得把地址写给 Client。返回成功后，该 Proxy 不再参与 Gateway、 +再次提取或可用库存统计。系统不跟踪 Client 是否使用、使用并发或何时停止， +也不提供 release、renew、status 或租约端点。 + +HTTP 契约源文件：`api/openapi/proxy-pool.yaml`。 + +## 2. 提取请求 + +```http +POST /api/v1/proxies/extract HTTP/1.1 +Host: 127.0.0.1:8081 +Content-Type: application/json +X-API-Key: TOKEN +X-Request-ID: req_01J4EXAMPLE +Idempotency-Key: extract-01J4EXAMPLE + +{ + "count": 5, + "fulfillment": "partial", + "filters": { + "protocols": ["http"], + "regions": ["shanghai"], + "carriers": ["telecom"], + "allowedUpstreams": ["provider-a"] + } +} +``` + +PowerShell 调用示例： + +```powershell +$body = @{ + count = 5 + fulfillment = "partial" + filters = @{ protocols = @("http"); regions = @("shanghai") } +} | ConvertTo-Json -Depth 4 + +Invoke-RestMethod ` + -Method Post ` + -Uri http://127.0.0.1:8081/api/v1/proxies/extract ` + -Headers @{ "X-API-Key" = "TOKEN"; "Idempotency-Key" = "extract-SERIAL" } ` + -ContentType application/json ` + -Body $body +``` + +请求约束： + +- `count` 至少为 1，且不超过服务端 `maxCountPerRequest`。 +- `fulfillment` 省略时使用服务端配置，默认 `partial`。 +- 所有过滤数组执行“数组内 OR、不同维度 AND”。空数组等同不限制。 +- `allowedUpstreams` 只能缩小 Client 可访问的 Upstream 集，不能扩大权限。 + +## 3. 成功响应 + +```json +{ + "requestId": "req_01J4EXAMPLE", + "requested": 5, + "returned": 2, + "proxies": [ + { + "id": "px_01J4A", + "protocol": "http", + "host": "192.0.2.10", + "port": 8080, + "username": "USER", + "password": "PASSWORD", + "url": "http://USER:PASSWORD@192.0.2.10:8080", + "region": "shanghai", + "carrier": "telecom", + "upstream": "provider-a", + "expiresAt": "2026-07-28T10:30:00Z", + "remainingTtlSeconds": 83, + "extractedAt": "2026-07-28T10:28:37Z" + } + ] +} +``` + +`returned` 必须等于 `proxies` 数组长度。`partial` 模式中 `returned` 可以为 +零；这仍表示请求语法有效，只是当前没有可提取库存。 + +返回的 `password` 和 `url` 含真实凭据。Client 必须限制日志、追踪和错误上报 +对响应体的采集。服务端访问日志只记录数量、过滤摘要、Client、Upstream 和 +Request ID，不记录地址或凭据。 + +## 4. 数量语义 + +### 4.1 partial + +锁定的符合条件数量少于 `count` 时，提交实际数量： + +```text +requested=10, eligible=6, reserve=0 -> returned=6 +``` + +### 4.2 allOrNothing + +锁定数量不足时整个事务回滚： + +```json +{ + "type": "https://proxy-pool.local/problems/insufficient-proxies", + "title": "Insufficient proxies", + "status": 409, + "code": "INSUFFICIENT_PROXIES", + "detail": "requested 10 proxies but only 6 are currently eligible", + "requestId": "req_01J4EXAMPLE" +} +``` + +冲突响应后，之前被该请求临时锁定的 Proxy 仍为 AVAILABLE。 + +## 5. 资格过滤与 Gateway 预留 + +候选必须同时满足： + +1. 权威状态是 AVAILABLE。 +2. 没有 Worker 所有权，或已完成 Drain 且 Active/Reserved 均为零。 +3. 剩余 TTL 不低于 `minRemainingTTL`。 +4. 最近健康检查不早于 `maxHealthCheckAge`。 +5. protocol、region、carrier、Upstream 满足过滤与 Client 权限。 +6. 提取后符合条件的共享库存不低于 `reserveForGateway`。 + +Worker-owned Proxy 不得直接提取。Controller 需要先发布 DRAINING，等待 Worker +确认没有 Active/Reserved，再清除所有权并进入提取事务。快照延迟时仍禁止 +Gateway 与 Client 同时获得同一 Proxy。 + +## 6. 并发与事务 + +PostgreSQL Adapter 应在一个事务内使用 `FOR UPDATE SKIP LOCKED` 获取候选， +更新状态并写审计记录。所有状态更新必须包含 `state = 'AVAILABLE'` 前置条件。 + +关键不变量： + +- 两个并发成功响应的 Proxy ID 集合交集为空。 +- 状态更新或审计写入任一步失败，整个批次不返回。 +- `allOrNothing` 不足时零行变为 EXTRACTED。 +- PostgreSQL 不可写时返回 503，不以内存结果冒充成功。 + +## 7. 幂等 + +提取是消耗库存的写操作。客户端收到超时后盲目重试可能再次提取一批不同 +Proxy，因此自动重试应提供稳定的 `Idempotency-Key`。 + +服务端幂等记录至少包含：Client ID、Key、请求体摘要、提交结果和过期时间。 +同一 Client、同一 Key、相同摘要返回首次结果；摘要不同返回 409。幂等记录和 +Extraction Record 必须与状态更新处在相同事务边界或由同一权威恢复流程保证。 + +## 8. 认证、识别与限制 + +认证由部署配置决定，OpenAPI 同时声明 API Key、Basic、Bearer 和无认证场景。 +无认证并不关闭来源 CIDR、Client 识别和限流： + +- 直连请求使用来源 IP 形成匿名 Client。 +- 只有来源属于 `trustedProxies` 时才接受转发头。 +- 全局和每 Client 限流在查询库存前执行。 +- 过滤条件、数量、请求体和 Header 都有长度/数量上限。 + +## 9. 错误模型 + +所有非 2xx 响应使用 `application/problem+json`： + +- `400`：JSON、Header 或基本格式无效。 +- `401`：认证失败。 +- `403`：来源控制、权限或 Upstream 访问被拒绝。 +- `409`：allOrNothing 库存不足，或幂等 Key 冲突。 +- `422`：数量、枚举或过滤组合违反业务约束。 +- `429`：全局或 Client 速率限制，响应 `Retry-After`。 +- `503`：PostgreSQL 不可写、服务排空或权威状态不可用。 + +错误响应不得包含 Provider Secret、Proxy 凭据、SQL 或内部拓扑。 + +## 10. 审计记录 + +每个被提交的 Proxy 对应一条 Extraction Record： + +```text +proxyId, clientId, sourceIP, requestId, upstream, extractedAt, expiresAt +``` + +无认证时 `clientId` 使用 `anonymous` 或稳定匿名标识并保留 `sourceIP`。记录只 +用于审计、排错和计费事实，不承担资源归还语义。Proxy 到期后可以清理运行 +记录，但 Extraction Record 按审计保留策略归档。 diff --git a/docs/configuration/examples.md b/docs/configuration/examples.md new file mode 100644 index 0000000..dc2aa39 --- /dev/null +++ b/docs/configuration/examples.md @@ -0,0 +1,35 @@ +# 配置样例索引 + +`examples/config` 下每个文件都是可独立加载的 Version 1 完整配置，不是 YAML +片段。示例域名使用保留的 `.example` 后缀，Secret 使用带引号的 +`${ENVIRONMENT_VARIABLE}` 占位符。 + +1. `01-local-all.yaml`：本机同时启用 Gateway 与独占提取。 +2. `02-gateway-only.yaml`：仅 Gateway，GET/HEAD 安全重试。 +3. `03-extract-only.yaml`：仅一次性独占提取。 +4. `04-public-gateway-basic-auth.yaml`：公网 Gateway 基础认证。 +5. `05-public-extract-api-key.yaml`：公网提取 API Key 认证。 +6. `06-internal-cidr-no-auth.yaml`：内网无认证，CIDR 访问控制。 +7. `07-auth-any.yaml`：IP 白名单或 API Key 任一通过。 +8. `08-sequential-failover.yaml`：连续 Empty 后从 A 切到 B。 +9. `09-weighted-routing.yaml`：70/30 Upstream 权重。 +10. `10-round-robin-routing.yaml`：Upstream 轮询。 +11. `11-random-routing.yaml`：Upstream 随机选择。 +12. `12-least-connections-routing.yaml`：按本地连接容量选择。 +13. `13-extract-all-or-nothing.yaml`：数量不足时整批回滚。 +14. `14-gateway-reserve.yaml`：共享池为 Gateway 保留库存。 +15. `15-strict-ttl-health.yaml`：严格 TTL 与健康新鲜度过滤。 +16. `16-provider-basic-auth.yaml`：Provider Basic Auth。 +17. `17-provider-api-key.yaml`：Provider Header API Key。 +18. `18-provider-post-json.yaml`：Provider JSON POST 与有界解析。 +19. `19-socks5-upstream.yaml`：SOCKS5 Upstream 模型。 +20. `20-fetch-billing-quota.yaml`：当前库存与累计计费额度分离。 + +批量验证： + +```powershell +go test -count=1 ./examples/config +``` + +项目的配置测试应遍历该目录，以严格加载器解析并校验每个文件。示例中的 +Provider URL 不是连通性测试目标；配置验证只校验语法、引用和不变量。 diff --git a/docs/configuration/reference.md b/docs/configuration/reference.md new file mode 100644 index 0000000..0ce23bd --- /dev/null +++ b/docs/configuration/reference.md @@ -0,0 +1,329 @@ +# 配置参考 + +## 1. 加载规则 + +主配置格式为 YAML，根字段 `version` 当前固定为 `1`。加载器启用严格字段 +检查，拼写错误或未来版本字段不会被静默忽略。推荐启动命令显式传入配置路径： + +```powershell +go run ./cmd/proxy-controller -config configs/proxy-pool.yaml +``` + +所有时间值使用 Go duration，例如 `500ms`、`30s`、`5m`。示例中的 +`${TOKEN}`、`${PASSWORD}`、`${POSTGRES_URL}` 等由加载器从同名环境变量 +展开；这些值不得写入日志、指标、配置转储或错误响应。生产配置优先使用 +环境变量或 Secret 文件，不提交明文值。 + +加载与热更新必须遵循同一顺序： + +```text +读取文件 -> 严格解析 -> 字段校验 -> 引用/正则校验 + -> 构建不可变快照 -> 计算差异 -> 原子替换 +``` + +新配置任何一步失败时保留旧快照。删除或禁用 Upstream 只停止新 Fetch 和新 +分配，已有连接进入 Drain，不强制中断。 + +## 2. 根结构 + +```yaml +version: 1 +defaults: {} +security: {} +gateway: {} +distribution: {} +admin: {} +metrics: {} +storage: {} +routing: [] +upstreams: {} +``` + +- `defaults`：Fetch 与 Check 的公共建议值。生产配置仍建议在每个启用的 + Upstream 显式写出关键限制，避免继承关系不清。 +- `security`：跨入口启动保护。 +- `gateway`：HTTP/HTTPS CONNECT 数据面入口。 +- `distribution`：一次性独占提取入口。 +- `admin`：运维管理入口，必须与 Distribution 分端口。 +- `metrics`：Prometheus 入口。 +- `storage`：Controller 使用的 PostgreSQL 与 Redis 地址。 +- `routing`：有序 Routing 列表，自上而下首条命中停止。 +- `upstreams`：全局共享的 Upstream 运行时定义。 + +## 3. 安全与监听器 + +```yaml +security: + requireProtectionOnPublicListen: true +``` + +启用严格保护时，只要 Gateway、Distribution 或 Admin 满足以下全部条件， +启动即失败： + +1. 入口已启用且监听地址不是回环地址。 +2. `auth.mode: none`。 +3. `access.allowCIDRs` 为空。 + +监听器公共字段： + +```yaml +enabled: true +listen: 0.0.0.0:8081 +access: + allowCIDRs: [10.0.0.0/8] + trustedProxies: [10.10.0.10/32] +auth: + mode: apiKey + header: X-API-Key + token: "${DISTRIBUTION_API_KEY}" +limits: + maxConcurrentConnections: 100000 + requestsPerMinute: 6000 + requestsPerMinutePerClient: 600 +``` + +`trustedProxies` 只决定何时接受 `Forwarded` 或 `X-Forwarded-For`，不能替代 +`allowCIDRs`。来自非可信代理的转发头必须忽略。 + +### 3.1 认证模式 + +- `none`：无身份认证，访问控制与限流仍生效。 +- `usernamePassword`：使用 `username` 和 `password`。 +- `apiKey`：使用 `header` 和 `token`。 +- `ipWhitelist`：使用 `cidrs`。 +- `any`：`methods` 中任一方法成功即可；方法字段名仍是 `mode`。 + +Gateway、Distribution 与 Provider API 的认证是三套独立边界。改变其中一套 +不得连带改变另外两套。 + +## 4. Gateway + +```yaml +gateway: + enabled: true + listen: 127.0.0.1:8080 + auth: {mode: none} + limits: + maxConcurrentConnections: 50000 + retry: + maxAttempts: 2 + retryMethods: [GET, HEAD] + destinationPolicy: + denyPrivateNetworks: true + denyLoopback: true + denyLinkLocal: true + denyCIDRs: [169.254.169.254/32] +``` + +- `retryMethods` 默认只应包含幂等方法。POST、PUT、PATCH、DELETE 不自动重试。 +- CONNECT 向 Client 写出 `200 Connection Established` 后不透明重放。 +- 目的地址策略必须在 DNS 解析前后都执行，防止 DNS Rebinding。 +- `maxConcurrentConnections` 是入口准入上限，不是 Proxy 容量上限。 + +## 5. Distribution + +```yaml +distribution: + enabled: true + listen: 127.0.0.1:8081 + auth: {mode: none} + clientIdentification: {mode: sourceIP} + limits: + requestsPerMinute: 60 + requestsPerMinutePerClient: 30 + extraction: + fulfillment: partial + maxCountPerRequest: 20 + minRemainingTTL: 30s + maxHealthCheckAge: 15s + reserveForGateway: 5 +``` + +Extraction 是固定的一次性独占行为，**没有** `mode`、`leaseDuration`、 +`release` 或 `renew` 配置。事务提交后执行 `AVAILABLE -> EXTRACTED`，该 Proxy +不再由 Gateway 或 Distribution 分配。 + +- `fulfillment`：`partial` 或 `allOrNothing`，默认语义为 `partial`。 +- `maxCountPerRequest`：单次请求硬上限。 +- `minRemainingTTL`：剩余寿命低于此值时不参与提取。 +- `maxHealthCheckAge`：最近检查早于此窗口时不参与提取。 +- `reserveForGateway`：提取后必须留给 Gateway 的最低符合条件库存数量。 + +`partial` 会提交实际可得数量；`allOrNothing` 数量不足时事务回滚，一个也不 +提取。认证关闭时仍应使用 `sourceIP` 识别匿名 Client 并执行全局/来源限流。 + +## 6. Routing + +```yaml +routing: + - name: api-post + enabled: true + purpose: gateway + match: + hostRegex: '^api\\.example\\.com$' + methods: [POST] + pathRegex: '^/v1/' + headers: {X-Tenant: premium} + upstreams: [provider-a, provider-b] + strategy: + type: sequential + switchAfterEmptyFetch: 5 + endBehavior: stayLast + onUnavailable: + action: reject + waitTimeout: 0s +``` + +- Routing 列表有序，首条匹配后停止。 +- `purpose` 为 `gateway` 或 `extract`。 +- `strategy.type` 支持 `sequential`、`random`、`roundRobin`、`weighted`、 + `leastConnections`。 +- `weighted` 使用 `weights` 映射，键必须引用本 Routing 的 Upstream。 +- `sequential` 必须设置大于零的 `switchAfterEmptyFetch`。 +- `onUnavailable.action` 为 `reject`、`wait` 或 `direct`；默认建议 `reject`。 + +Sequential 的空计数属于 Upstream，当前索引属于 Routing。只有 Provider 响应 +成功、模板成功且合法候选为零时才增加空计数。错误不改变空计数；重复候选 +会重置空计数但增加独立 duplicate 指标。 + +## 7. Upstream + +```yaml +upstreams: + provider-a: + enabled: true + exposure: [gateway, extract] + provider: + billingMode: fetch + protocols: [http, https] + api: + url: https://provider-a.example/proxies + method: GET + auth: + type: apiKey + location: header + name: X-Provider-Key + value: "${PROVIDER_API_KEY}" + headers: {} + query: {count: '100'} + body: {type: json, value: {}} + template: '{{.}}' + proxyAuth: {type: response} + pool: {maxSize: 1000, shrinkDelay: 30s} + capacity: {maxConcurrencyPerProxy: 10} + lifecycle: {ttl: 2m, allocationSafetyMargin: 15s} + fetch: + requestInterval: 1s + timeout: 3s + maxAttempts: 3 + maxInFlight: 1 + maxTotal: 100000 + maxResponseBytes: 1048576 + templateTimeout: 100ms + retry: {initial: 500ms, max: 30s, jitter: 20} + check: + interval: 30s + jitter: 20 + maxInFlight: 100 + timeout: 2s + maxAttempts: 2 + maxConsecutiveFailures: 3 + urls: [http://connect.rom.miui.com/generate_204] +``` + +### 7.1 Provider 与代理认证 + +- `api.auth` 用于系统访问 Provider API。 +- `proxyAuth` 用于最终连接被获取的 Proxy。 +- Provider `api.auth.type` 支持 `none`、`basic`、`bearer`、`apiKey`。 +- `apiKey` 使用 `location: header|query` 与 `name`、`value`，程序负责 Header + 设置或 Query URL 编码。 +- `basic` 使用 `username`、`password`；`bearer` 使用 `token`。 +- `proxyAuth.type: response` 表示凭据来自 Provider 响应。 +- `proxyAuth.type: static` 使用配置中的 `username`、`password`。 +- `proxyAuth.type: ipWhitelist` 表示 Provider 按出口 IP 放行，不配置账号密码。 + +Provider API 认证**不使用**入口的 `auth.mode`，代理连接认证也不使用 +`mode`。三者的字段和凭据不可互相回退： + +```yaml +# 对外 Distribution/Gateway/Admin +auth: + mode: apiKey + header: X-API-Key + token: "${CLIENT_API_KEY}" + +# 请求 Provider API +auth: + type: bearer + token: "${PROVIDER_API_TOKEN}" + +# 连接 Provider 返回的 Proxy +proxyAuth: + type: static + username: "${PROVIDER_PROXY_USER}" + password: "${PROVIDER_PROXY_PASSWORD}" +``` + +### 7.2 Pool 与累计额度 + +- `pool.maxSize`：当前系统维护且尚未 EXTRACTED 的 Proxy 硬上限，包括 + FETCHED、CHECKING、AVAILABLE、SUSPECT、DRAINING 和 pending expected。 +- `fetch.maxTotal`：当前运行或计费周期内，从 Provider 成功获取的累计上限； + `0` 表示不设置累计上限。 + +`fetch.maxTotal` 不得小于 `pool.maxSize`。提取一个 Proxy 会释放当前库存位置， +但不会恢复累计获取额度。 + +### 7.3 Fetch 限制 + +- `requestInterval`：同一 Provider 请求间隔。 +- `timeout`：单次调用超时。 +- `maxAttempts`：单次补池动作最大尝试次数。 +- `maxInFlight`：同一 Provider 同时在途请求数。 +- `maxResponseBytes`：读取响应的硬上限。 +- `templateTimeout`：模板解析执行上限。 +- `retry`：错误退避；HTTP 429 还必须尊重 `Retry-After`。 + +大量缺池信号必须合并成 singleflight 或容量为 1 的通知，不能按 Gateway 请求 +数量线性触发 Provider API。 + +### 7.4 生命周期与健康 + +- 明确绝对过期时间优先于响应 TTL，响应 TTL 优先于配置 `lifecycle.ttl`。 +- 距离过期不足 `allocationSafetyMargin` 时停止新分配。 +- `check.jitter` 为调度抖动百分比，避免所有 Proxy 同时探测。 +- 第一次有意义失败进入 SUSPECT；达到 `maxConsecutiveFailures` 后才进入 + UNHEALTHY。 + +## 8. 存储、Admin 与 Metrics + +```yaml +admin: + enabled: true + listen: 127.0.0.1:8082 + auth: {mode: none} +metrics: + enabled: true + listen: 127.0.0.1:9090 +storage: + postgresURL: "${POSTGRES_URL}" + redisURL: "${REDIS_URL}" +``` + +PostgreSQL 是 Proxy 生命周期、Routing 选择、Worker 所有权和 Extraction Record +的权威存储。Redis 只承载可重建的短期协调状态，不能成为独占提取的唯一事实 +来源。Metrics 标签禁止 Proxy IP、Client ID、Session、完整 URL 和 Request ID。 + +## 9. 启动前校验清单 + +1. `version` 必须为 `1`，未知字段拒绝。 +2. 所有启用监听器具有合法 `host:port`。 +3. 非回环监听器满足认证或来源 CIDR 保护。 +4. Routing 名称唯一，正则可编译，引用的 Upstream 存在。 +5. Sequential 阈值大于零，`onUnavailable.action` 明确。 +6. 启用的 Upstream 有正数 `pool.maxSize`、并发和 Fetch 限制。 +7. `allocationSafetyMargin < ttl`。 +8. `fetch.maxTotal == 0` 或 `fetch.maxTotal >= pool.maxSize`。 +9. Distribution 的 fulfillment 合法，单次数量大于零。 +10. Secret 未写入日志可见配置转储。 diff --git a/docs/design/product-design.md b/docs/design/product-design.md new file mode 100644 index 0000000..d89ad64 --- /dev/null +++ b/docs/design/product-design.md @@ -0,0 +1,93 @@ +# Proxy Pool 产品设计文案 + +## 1. 产品定位 + +Proxy Pool 把多个供应商的动态代理统一成一个可运营的资源系统。平台面对 +两类不同使用方式：需要平台代转发流量的应用使用 Gateway；需要拿到真实 +代理并自行建立连接的应用使用 Distribution API。 + +两类入口共享 Provider、健康、TTL 和路由配置，但资源分配语义不同： + +- Gateway 只在一次请求或隧道生命周期内占用代理并发，结束后释放容量。 +- Distribution 一旦返回代理，该代理即永久离开系统可分配池。 + +## 2. 目标用户 + +- **业务调用方**：通过稳定入口使用代理，不感知供应商差异。 +- **代理直提调用方**：按协议、区域、运营商或 Upstream 条件独占提取。 +- **平台管理员**：管理 Provider、Routing、容量、健康和故障切换。 +- **SRE**：依据低基数指标、审计记录和运行手册进行容量与故障管理。 + +## 3. 核心价值 + +### 3.1 供应商差异收敛 + +Provider Adapter 负责请求格式、认证和响应解析。标准化后，Routing、健康、 +容量和业务入口只依赖统一 Proxy 模型，不把供应商字段传入核心域。 + +### 3.2 高并发热路径隔离 + +Gateway Worker 只读取本地不可变快照并维护本地容量计数。供应商延迟、 +数据库抖动和控制面重载不会成为每请求依赖。 + +### 3.3 明确且可审计的资源语义 + +系统区分“短时使用容量”和“一次性独占提取”。每次 Extraction 保存请求、 +调用方、来源、Upstream、提取时间和过期时间审计事实，但不追踪提取后的 +实际使用，也不存在归还接口。 + +## 4. 主要流程 + +### 4.1 Gateway 请求 + +1. 接入层完成认证、来源识别、限流和目标地址检查。 +2. Routing 按配置顺序首条命中。 +3. Dispatcher 从本地快照筛选 Upstream、协议、标签、TTL 和健康条件。 +4. 原子预留 Proxy 容量，建立到上游代理的连接。 +5. 建连成功后转为 Active，传输结束后释放；失败则取消预留。 +6. GET/HEAD 仅在响应提交前按策略重试；CONNECT 建立后不重放。 + +### 4.2 独占提取 + +1. 调用方提交数量、过滤条件和 fulfillment。 +2. Controller 校验调用方限额、TTL、健康新鲜度及 Gateway 保留量。 +3. 在一个数据库事务中锁定候选并执行 `AVAILABLE -> EXTRACTED`。 +4. `partial` 尽量返回；`allOrNothing` 数量不足时零提取。 +5. 响应返回代理 URL、`expiresAt` 和 `remainingTtlSeconds`。 + +### 4.3 Sequential 切换 + +1. Provider 响应成功且解析成功，但合法候选为零，才累计 Empty。 +2. 网络、认证、HTTP、模板或解析失败只计 Error。 +3. 全部候选均重复时计 DuplicateOnly，并重置连续 Empty。 +4. 达到阈值后，引用该 Upstream 的 Routing 原子前进一次。 +5. 旧 Upstream 已有 Proxy 继续耗尽，不因切换被直接删除。 + +## 5. 失败体验 + +- 无候选时严格执行 Routing 的 `reject`、`wait` 或 `direct`，默认拒绝。 +- Distribution 部分满足用 200 返回实际数量；全有或全无不足时返回冲突状态。 +- Provider 故障进入退避，不让调用请求触发同步 Provider 获取。 +- 快照版本断档时 Worker 保留最后一份完整快照并请求全量重同步。 +- 控制面不可用时，现有 Worker 可在快照有效期内继续服务，停止接收新配置。 + +## 6. 容量目标与服务指标 + +- 集群峰值目标：100,000 QPS。 +- Worker 副本数： + +```text +required_workers = ceil(peak_qps / (measured_worker_qps * target_utilization)) + + failure_domain_spares +``` + +- `measured_worker_qps` 必须来自目标协议占比、代理 RTT、连接复用率和安全策略 + 均接近生产的压测。 +- 设计阶段不承诺单 Worker QPS，也不以平均值替代 P95/P99 和错误率。 + +## 7. 首版范围 + +首版包含 HTTP 正向代理、HTTPS CONNECT、REST Distribution/Admin、Provider +适配、健康与生命周期、Sequential 等路由策略、PostgreSQL 权威状态、Redis +可重建协调和集群快照。SOCKS5、跨地域主动主动和高级成本优化保留扩展边界。 + diff --git a/docs/design/project-structure.md b/docs/design/project-structure.md new file mode 100644 index 0000000..88e53ad --- /dev/null +++ b/docs/design/project-structure.md @@ -0,0 +1,96 @@ +# Proxy Pool 项目架构 + +## 1. 仓库结构 + +```text +proxy-pool/ +├── cmd/ +│ ├── proxy-gateway/ # 数据面进程 +│ ├── proxy-controller/ # 控制面与 HTTP API +│ ├── proxy-checker/ # 健康检查执行器 +│ └── proxy-loadgen/ # 可复现容量测试 +├── internal/ +│ ├── config/ # 严格配置解析和校验 +│ ├── domain/ # 无传输、无存储依赖的领域模型 +│ ├── gateway/ # snapshot、dispatch、server、transport +│ ├── controller/ # provider、pool、routing、extraction、health +│ ├── adapters/ # PostgreSQL、Redis、Provider API、内存适配 +│ └── platform/ # 日志、指标、停机和进程装配 +├── api/ # OpenAPI 与 Protobuf 契约 +├── configs/ # 默认配置 +├── examples/ # 可校验配置场景 +├── deploy/ # Compose 与 Kubernetes +├── docs/ # 设计、开发、API、测试、运维 +├── diagrams/ # Mermaid 图集 +├── scripts/ # 验证和生成脚本 +└── test/ # fixture、集成、端到端和负载测试 +``` + +## 2. 进程边界 + +### proxy-gateway + +`gateway -> dispatch -> transport` 是数据面主调用链。`dispatch` 包含筛选、 +选择、session 与重试资格等热路径决策；`transport` 独占连接池、上游握手和 +隧道生命周期。任何包都不得从热路径反向调用 Controller 存储。 + +### proxy-controller + +Controller 是首版模块化单体。Provider、Pool、Routing 和 Extraction 共享 +事务边界和状态演进，避免过早拆成分布式事务。对外端口定义在领域或控制器 +模块，具体 PostgreSQL/Redis/HTTP 实现在 `adapters`。 + +### proxy-checker + +Checker 只产生 Observation。最终状态迁移由 Controller 的确定性 reducer +完成，避免多个检查实例同时写 Proxy 状态。 + +### proxy-loadgen + +负载工具生成 HTTP、CONNECT、连接复用与故障注入场景，输出延迟分位数、 +错误类别、连接数、CPU、RSS、GC 与吞吐。它是 100k QPS 结论的证据工具， +不是业务进程。 + +## 3. 依赖方向 + +```text +cmd -> controller/gateway/checker -> domain + | + +------------> port interfaces +adapters -------------------------> port interfaces +platform -------------------------> standard library / observability SDK +``` + +硬性规则： + +1. `domain` 不导入 HTTP、SQL、Redis、配置或平台包。 +2. `gateway/dispatch` 只依赖本地 Snapshot 与领域类型。 +3. `adapters` 实现端口，不被领域层反向引用。 +4. 配置先解析、校验、编译为运行时对象，再原子发布。 +5. Secret 仅通过引用进入运行时，不进入唯一键、指标或日志字段。 + +## 4. 数据所有权 + +- PostgreSQL：Proxy 生命周期、Extraction 审计、配置版本和 Outbox 的权威源。 +- Redis：Leader、分布式速率、Worker 心跳等可丢失且可重建状态。 +- Worker：仅拥有分配给自己的 Proxy 本地容量计数和不可变快照。 +- Controller：拥有 Provider 调度、Routing 运行态与 Worker 所有权编排。 +- Checker：不拥有 Proxy 状态，只拥有执行中的检查任务。 + +## 5. 一致性边界 + +- Extraction 使用 PostgreSQL 单事务和行锁跳过锁定候选，提交后才响应。 +- Worker 所有权采用 `worker + epoch + version + expiry`，同一 Proxy 至多归属 + 一个 Worker。 +- 从 Worker 回收 Proxy 时先 Drain，等待 ACK 且 Active/Reserved 为零，再 + 解除所有权；只有无所有权 Proxy 能被 Distribution 提取。 +- Snapshot 为整代不可变对象，通过校验和和严格版本序列原子替换。 + +## 6. 扩展规则 + +- 新 Provider：增加 Adapter，不修改 Proxy/Pool/Routing 领域语义。 +- 新入口协议：在 Gateway 增加 ingress adapter；只有存在第二种上游协议 + 执行方式时再抽象 egress adapter。 +- 新路由策略：实现同一策略端口，并提供确定性单测和并发不变量测试。 +- 新存储：实现已有 repository port，不把驱动类型泄漏到控制器。 + diff --git a/docs/development/guide.md b/docs/development/guide.md new file mode 100644 index 0000000..ad1719e --- /dev/null +++ b/docs/development/guide.md @@ -0,0 +1,65 @@ +# 开发指南 + +## 1. 环境 + +- Go 1.26 或 `go.mod` 指定版本。 +- PostgreSQL 与 Redis 仅用于适配器集成测试，领域单测不依赖外部服务。 +- 运行 `go test -race` 需要 CGO 和 C 编译器。 +- Docker Compose 用于本地完整拓扑，Docker 不应成为普通单测前置条件。 + +## 2. 开发循环 + +1. 在 `traceability.md` 中找到需求 ID。 +2. 先写会失败的单测或契约测试，确认失败原因与需求一致。 +3. 实现最小完整行为，不增加空接口或预留目录。 +4. 运行目标包测试，再运行全仓库测试和静态检查。 +5. 更新需求证据、相关文档和配置示例。 + +```powershell +go test ./internal/domain/proxy +go test ./... +go vet ./... +go build ./... +``` + +## 3. 包设计规则 + +- 领域包使用业务语言，不使用 Controller、HTTP 或数据库 DTO。 +- 模块接口应隐藏内部策略步骤，避免把 filter/scorer/picker 拆成浅接口链。 +- 时间逻辑注入 `now` 或 Clock，测试禁止依赖真实睡眠。 +- Provider 调度使用有界信号、singleflight、超时和抖动退避。 +- 后台队列必须有容量、溢出策略、关闭语义和指标。 +- 并发计数必须以不变量测试证明，不只检查最终值。 + +## 4. 配置变更 + +新增字段时同时修改： + +1. `internal/config` 类型、默认值与校验。 +2. `configs/default.yaml`。 +3. `docs/configuration/reference.md`。 +4. 至少一个有效示例和一个无效测试。 +5. 配置版本兼容说明；不静默忽略未知字段。 + +## 5. API 变更 + +- OpenAPI 是 REST 契约源，Protobuf 是 Controller/Worker 契约源。 +- 先更新契约和兼容性测试，再修改 handler。 +- Distribution 不得出现 lease、release、renew、return 等资源归还语义。 +- 错误响应包含稳定 code 和 requestId，不向调用方暴露 Secret 或内部栈。 + +## 6. 并发与性能 + +- Gateway 请求路径不得出现远程存储访问和无界 goroutine 创建。 +- Snapshot 构建在后台完成，发布后只读；请求只做一次原子指针读取。 +- Proxy 容量由同一个打包原子值保存 Active/Reserved，避免分开检查再写入。 +- 性能优化必须附基准；100k QPS 结论必须附完整环境和负载模型。 + +## 7. 提交前检查 + +```powershell +./scripts/verify.ps1 +``` + +审查还要确认：无 Secret 日志、无 Proxy IP 高基数标签、无默认 direct、无 +Extraction Lease API、无把重复结果误计为 Empty 的逻辑。 diff --git a/docs/operations/production-readiness.md b/docs/operations/production-readiness.md new file mode 100644 index 0000000..6c61677 --- /dev/null +++ b/docs/operations/production-readiness.md @@ -0,0 +1,61 @@ +# 生产就绪检查表 + +## 架构与一致性 + +- [ ] Gateway 热路径依赖审计确认无 PostgreSQL、Redis、Provider 或模板执行。 +- [ ] 每个 Proxy 同一时刻最多归属一个 Worker，ownership epoch 单调。 +- [ ] Reserved -> Active 使用单个原子转换，无超卖与负计数。 +- [ ] Sequential 并发 Empty 只切换一次，旧 Upstream Proxy 自然耗尽。 +- [ ] `pool.maxSize` 与 `fetch.maxTotal` 分别按当前库存和累计获取计数。 +- [ ] Extract 只有 `AVAILABLE -> EXTRACTED`，OpenAPI 不存在 release/renew。 +- [ ] Extract 状态更新和审计记录位于同一数据库事务。 +- [ ] `partial` 和 `allOrNothing` 均通过并发事务测试。 +- [ ] `reserveForGateway` 在所有提取路径上统一执行。 + +## 安全 + +- [ ] 非回环监听均配置 Auth 或 allowCIDRs，严格模式已开启。 +- [ ] Gateway、Distribution、Admin 凭据和权限相互独立。 +- [ ] trusted proxy 只包含受控 LoadBalancer/Ingress 网段。 +- [ ] 解析前后均拦截私网、回环、链路本地、元数据地址与 DNS Rebinding。 +- [ ] Secret 由外部密钥系统注入，镜像、ConfigMap、日志没有明文。 +- [ ] Pod 以非 root、只读根文件系统、无 Linux capabilities 运行。 +- [ ] NetworkPolicy 默认拒绝，外部数据库/Redis/Provider 网段已收紧。 +- [ ] Provider 模板有响应大小、执行时间、函数与外部访问限制。 + +## 可用性 + +- [ ] PostgreSQL 和 Redis 跨可用区，有监控、备份和恢复演练证据。 +- [ ] Gateway、Controller、Checker 均跨主机/可用区分散。 +- [ ] PDB、优雅终止与最大连接时长的组合经过驱逐测试。 +- [ ] Controller 断线时 Gateway 在 `maxStaleAge` 内继续，超限拒绝新请求。 +- [ ] Worker 崩溃后 ownership 只在租约过期后再分配。 +- [ ] Snapshot 版本缺口触发全量同步，旧 Delta 被拒绝。 + +## 性能 + +- [ ] 单 Worker 基准使用生产同规格硬件、网络、TLS 和 Snapshot 规模。 +- [ ] 完成 10k 稳态、100k 峰值、CONNECT 活跃连接和建连速率独立测试。 +- [ ] 在最大可用区失效时仍满足容量和延迟 SLO。 +- [ ] 目标利用率不高于 60%，HPA 缩容稳定窗口不低于 10 分钟。 +- [ ] 队列、buffer、日志和结果上报全部有界。 +- [ ] p99 Dispatch 预算、端到端延迟、错误率、CPU、RSS、FD 和网络有原始证据。 + +## 观测与值班 + +- [ ] 仪表盘覆盖 Gateway、Routing、Upstream、Provider、Extract、Snapshot、Checker。 +- [ ] 告警有负责人、严重级别、Runbook 链接和演练记录。 +- [ ] 指标无 Proxy IP、Client ID、Session、完整 URL 或 request ID 高基数标签。 +- [ ] 日志脱敏已用真实 Secret fixture 验证。 +- [ ] 值班人员完成 PostgreSQL、Redis、Snapshot、容量与 Extract 故障演练。 + +## 发布门禁 + +- [ ] `go test ./...`、race、vet、build 全部通过。 +- [ ] OpenAPI/Proto 兼容检查通过。 +- [ ] Compose、Kustomize、Prometheus、HAProxy 配置静态校验通过。 +- [ ] 数据库迁移已在生产数据量副本上演练，并有回退或前向修复方案。 +- [ ] 100k QPS 验证报告包含环境、命令、版本、场景、原始指标和结论。 + +任一关键一致性、安全或恢复项未完成时，不标记生产就绪。 + diff --git a/docs/operations/runbook.md b/docs/operations/runbook.md new file mode 100644 index 0000000..c1f9b58 --- /dev/null +++ b/docs/operations/runbook.md @@ -0,0 +1,249 @@ +# Proxy Pool 运维手册 + +## 1. 运行边界 + +- Gateway 是数据面，正常请求热路径不访问 PostgreSQL、Redis 或 Provider。 +- Controller 是权威控制面，负责 Fetch、生命周期、所有权、Snapshot、Extract + 和审计；多个副本只有一个 Provider 逻辑 Leader。 +- Checker 执行有界健康探测，只上报 Observation，最终状态由 Controller + reducer 决定。 +- Extract 是一次性独占发放。提交后状态为 `EXTRACTED`，没有 Lease、续租或 + Release 接口。 +- `reserveForGateway` 是共享池硬约束，Extract 不得把 Gateway 库存清空。 +- 集群峰值 100,000 QPS 是设计目标，只有完成本文容量验收后才能作为已验证 + 能力对外承诺。 + +## 2. 本地拓扑模板 + +当前仓库交付设计、契约、部署拓扑和关键领域实现；`cmd/proxy-*` 的完整运行时 +装配属于 `implementation-plan.md` 后续任务。此处 Compose/Kubernetes 资产用于 +评审网络、资源、探针和依赖关系，当前只执行静态渲染，不把模板写成可运行服务。 + +### 2.1 前置条件 + +- Docker Engine 25+，Compose v2.30+。 +- 至少 8 CPU、16 GiB 内存和 20 GiB 可用磁盘。 +- 本地端口 `3000`、`8080`、`8081`、`8082`、`8404`、`9091` 未占用。 + +### 2.2 配置凭据 + +`deploy/config/local.yaml` 只用于本机拓扑验证。进入运行时实施阶段后，再通过 +密钥系统提供下列变量，并把 Provider 地址替换为测试 fixture： + +```powershell +$env:PROXY_POOL_GATEWAY_PASSWORD = "LOCAL_GATEWAY_PASSWORD" +$env:PROXY_POOL_EXTRACT_TOKEN = "LOCAL_EXTRACT_TOKEN" +$env:PROXY_POOL_ADMIN_TOKEN = "LOCAL_ADMIN_TOKEN" +$env:PROVIDER_A_TOKEN = "PROVIDER_A_TOKEN" +$env:PROVIDER_B_TOKEN = "PROVIDER_B_TOKEN" +``` + +### 2.3 静态检查 + +```powershell +docker compose -f deploy/docker-compose.yml config +kubectl kustomize deploy/kubernetes/base > rendered.yaml +``` + +目标拓扑入口： + +- Gateway：`127.0.0.1:8080` +- Distribution：`http://127.0.0.1:8081` +- Admin：`http://127.0.0.1:8082` +- HAProxy 状态：`http://127.0.0.1:8404/stats` +- Prometheus：`http://127.0.0.1:9091` +- Grafana：`http://127.0.0.1:3000` + +`deploy/config/local.yaml` 中的 `.invalid` Provider 是故障演示占位。未替换时 +Fetch 应表现为 Error 与退避，不应增加 Empty 计数，也不影响已有 Proxy。 + +## 3. Kubernetes 发布 + +本节是运行时实施完成后的发布规格，不代表当前代码已达到生产就绪。 + +### 3.1 准备 + +1. 使用托管 PostgreSQL 和 Redis，分别配置 TLS、备份、监控和多可用区。 +2. 复制 `secret.example.yaml` 到环境私密配置系统，由 External Secrets、SOPS + 或密钥管理平台生成 `proxy-pool-secrets`，不要提交真实 Secret。 +3. 在环境 Overlay 替换镜像、Provider 地址、允许网段、外部存储地址、资源量 + 和 LoadBalancer 注解。 +4. 根据集群 CNI 能力收紧 NetworkPolicy 的外部网段。 +5. 在预发布环境完成数据库向前兼容迁移，再发布 Controller。 + +### 3.2 服务端应用顺序 + +```bash +kubectl apply -f deploy/kubernetes/base/namespace.yaml +kubectl -n proxy-pool apply -f ENVIRONMENT_SECRET.yaml +kubectl apply -k deploy/kubernetes/base +kubectl -n proxy-pool rollout status deployment/proxy-controller --timeout=5m +kubectl -n proxy-pool rollout status deployment/proxy-checker --timeout=5m +kubectl -n proxy-pool rollout status deployment/proxy-gateway --timeout=10m +``` + +### 3.3 探针语义 + +- `/livez`：进程事件循环仍可运行。数据库或 Redis 短暂失败不得导致 Gateway + liveness 失败。 +- `/readyz`：进程可以接收新工作。Gateway 只有在持有未超过 `maxStaleAge` + 的完整 Snapshot 且仍有准入能力时才 Ready。 +- Controller 只有在配置有效、存储可用、迁移兼容且控制接口已监听时才 Ready。 +- Checker 在任务消费与结果上报通道可用时 Ready。 +- `/metrics`：独立于业务入口，NetworkPolicy 仅允许监控命名空间访问。 + +探针不得执行 Provider 请求或完整数据库扫描。 + +### 3.4 发布顺序与兼容性 + +1. 先做向前兼容数据库迁移。 +2. 发布 Controller，确认旧 Worker 仍能消费旧 Snapshot 协议版本。 +3. 发布 Checker。 +4. 逐批发布 Gateway；每次至少保留 PDB 要求的健康副本。 +5. 观察 30 分钟，再清理已经无人读取的旧字段或旧迁移。 + +回滚只能回到仍兼容当前 Schema 和 Snapshot 版本的镜像。涉及不可逆数据迁移时， +必须使用前向修复。 + +## 4. 容量规划 + +```text +worker_replicas = + ceil(peak_qps / (tested_worker_qps * target_utilization)) + + largest_failure_domain_replicas +``` + +- `peak_qps`：当前目标为 100,000。 +- `tested_worker_qps`：在同 CPU、内存、网络、Go 版本、Snapshot 规模、TLS 与 + 上游响应模型下测出的单 Pod 持续能力。 +- `target_utilization`：不高于 0.60，给突发、GC 和故障转移留空间。 +- `largest_failure_domain_replicas`：最大单可用区失效时丢失的副本数。 + +禁止用 CPU 核数直接推算 QPS，也禁止把短时峰值当持续容量。Kubernetes 基线的 +6 个 Gateway 副本只是初始值，必须由压测结果调整。 + +HPA 使用 CPU/内存作为保护性信号；生产环境建议通过 Prometheus Adapter 加入： + +- 每 Pod Gateway QPS。 +- 活跃连接数和建连速率。 +- p99 Dispatch 延迟。 +- 拒绝率和 Available Slots。 + +连接型工作负载缩容至少稳定 10 分钟，终止前先 NotReady，再等待现有隧道排空。 + +## 5. 日常检查 + +每班次检查： + +1. Gateway QPS、错误率、p95/p99 和活跃连接。 +2. Snapshot age、epoch/version、重同步与 ACK 延迟。 +3. Available Slots、Reserved、Active、各 Proxy 状态数量。 +4. Provider Success、Empty、Duplicate-only、Error、429 与退避。 +5. Checker 队列、SUSPECT 数、检查延迟和目标级失败。 +6. Extract requested/returned、insufficient、冲突和审计写入。 +7. PostgreSQL 连接、锁等待、事务失败、WAL 与备份。 +8. Redis 延迟、内存、主从状态和 Leader 租约抖动。 + +Prometheus 标签禁止包含 Proxy IP、Client ID、Session、完整 URL、request ID。 +需要逐请求调查时使用受控、脱敏且采样的结构化日志。 + +## 6. 优雅停机 + +### Gateway + +1. readiness 立即失败，停止新连接。 +2. 停止应用新 Snapshot，但保留当前不可变版本。 +3. 等待 HTTP 请求和 CONNECT 隧道排空。 +4. 到达 60 秒上限后关闭残余连接，保证容量 reservation 被释放。 + +### Controller + +1. 停止接收新的 Extract/Admin 写请求。 +2. 停止发起 Fetch，释放 Provider Leader 租约。 +3. 完成已进入数据库事务的 Extract 或回滚。 +4. 刷新 outbox、审计和 Worker ACK，再关闭连接池。 + +### Checker + +1. 停止领取新任务。 +2. 在 45 秒内完成或取消现有探测。 +3. 批量上报已完成 Observation，未完成任务由队列重新投递。 + +## 7. 故障处置 + +### 7.1 Snapshot 陈旧 + +症状：`ProxyPoolGatewaySnapshotStale`、Worker 重同步增加、Gateway Ready 下降。 + +1. 检查 Controller、控制流和 outbox 延迟。 +2. 确认 Worker epoch 与 Controller epoch，禁止手工降低 epoch。 +3. 缺版本时强制完整 Snapshot，不要继续应用 Delta。 +4. `maxStaleAge` 内允许旧 Snapshot 服务；超限自动拒绝新流量并排空。 +5. 不得通过无限增大 `maxStaleAge` 隐藏控制面故障。 + +### 7.2 PostgreSQL 不可用 + +1. Gateway 继续使用最后有效 Snapshot。 +2. Controller 将 Distribution 和权威写操作置为不可用，避免返回未提交代理。 +3. Provider Fetch 停止写入；已有流量不受影响。 +4. 恢复后核对迁移、事务回滚、outbox backlog 与 Extract 审计连续性。 + +### 7.3 Redis 不可用 + +1. Gateway 不受影响。 +2. Controller 停止需要分布式互斥的高风险工作，防止多个 Fetch Leader。 +3. 本地限流只作为临时降级，不能声称满足全局额度。 +4. 恢复后确认 Leader 唯一、租约 epoch 单调和重复 Fetch 去重。 + +### 7.4 Provider 故障 + +1. 超时、DNS、认证、非预期 HTTP、响应超限与模板错误全部计 Error。 +2. 429 尊重 `Retry-After`，其余 Error 使用指数退避和 jitter。 +3. Error 不增加 Empty；合法候选为零才增加 Empty。 +4. Duplicate-only 重置 Empty 并记录独立指标。 +5. 达到 Empty 阈值后每条受影响 Routing 只原子切换一次。 + +### 7.5 Gateway 容量耗尽 + +1. 检查 Available Slots，而不是只看 Proxy 数量。 +2. 确认是否大量容量停留在 Reserved，排查 Commit/Cancel 泄漏。 +3. 检查 TTL safety margin、健康状态和 Worker 所有权是否导致候选被过滤。 +4. 快速拒绝新请求，禁止无界等待或把压力转移到 Controller。 +5. 扩容 Gateway 前确认存在可分配 Proxy 所有权切片。 + +### 7.6 Extract 库存不足 + +1. `partial` 返回实际数量；`allOrNothing` 不足时整批回滚。 +2. 检查 TTL、health age、filter、Worker ownership 和 `reserveForGateway`。 +3. 不得降低 `reserveForGateway` 到导致 Gateway 容量告警的水平。 +4. 回收 Worker-owned Proxy 必须先 DRAINING、等待 active/reserved 为零、清除 + ownership，再执行 `AVAILABLE -> EXTRACTED` 事务。 +5. 已提取代理没有 Release；客户端归还请求只记录为无效调用，不恢复库存。 + +### 7.7 Checker 积压 + +1. 优先新 Proxy 与 SUSPECT 复检。 +2. 降低稳定 AVAILABLE 的普通复检频率。 +3. 检查目标超时、DNS 与出口网络，再按任务延迟扩容 Checker。 +4. 队列必须有上限；不得无限积压耗尽内存或 Redis。 + +## 8. 备份与恢复 + +- PostgreSQL：每日全量、连续 WAL/PITR，至少每季度做恢复演练。 +- Redis：仅保存可重建协调状态；不得把 Redis 备份当权威业务备份。 +- 配置：版本化保存校验通过的不可变 Revision 与校验和。 +- Secret：由密钥平台版本化，日志和备份中不得出现明文。 + +恢复顺序：PostgreSQL -> Redis -> Controller -> Checker -> Gateway。恢复后验证 +Proxy 状态、Extraction Record、Worker ownership epoch、outbox 和配置 Revision +单调一致，再开放 Gateway 与 Distribution。 + +## 9. Secret 轮换 + +1. 创建新 Secret 版本，不覆盖旧值。 +2. Provider/API 凭据支持双版本重叠时先发布新版本。 +3. 更新配置 Revision，确认 Controller 成功重建 Adapter。 +4. 观察 Fetch Error、认证失败与 Snapshot ACK。 +5. 所有副本应用后撤销旧 Secret。 + +Proxy 凭据轮换必须增加 `credentialVersion`，确保唯一键不会把新旧凭据错误合并。 diff --git a/docs/requirements/completion-audit.md b/docs/requirements/completion-audit.md new file mode 100644 index 0000000..47bc77f --- /dev/null +++ b/docs/requirements/completion-audit.md @@ -0,0 +1,72 @@ +# 交付完成度审计 + +本文区分设计证据、机器契约、已运行验证和后续实施，防止把架构目标描述成 +已完成产品。 + +## 1. 本次已交付 + +### 设计与开发文档 + +- 全量需求追踪、覆盖关系和统一领域语言。 +- 产品设计、总体架构、项目结构、四项 ADR。 +- 开发、配置、Distribution/Admin API、控制面协议、安全、测试、运维文档。 +- 20 个配置场景和 35 张 Mermaid 架构/流程/状态/故障图。 +- 版本化文档包 `proxy-pool-docs-v1.0.zip`，包含 50 个条目。 + +### 机器契约 + +- Distribution OpenAPI：一次性独占提取、partial/allOrNothing、幂等键、 + TTL/健康过滤结果与标准错误。 +- Admin OpenAPI：状态、Upstream 启停、Routing 切换和配置重载。 +- Protobuf：Worker 注册、全量/增量 Snapshot、ACK、运行态/结果上报、Checker + 任务与 Observation。 + +### 核心参考实现 + +- `CFG-*`：YAML v4 未知字段拒绝、监听保护、引用/上限/认证边界校验，21 份 + 配置持续测试。 +- `PROXY-* / CAP-*`：唯一键、TTL 优先级、状态迁移与 Active/Reserved 打包 + 原子计数；1,000 goroutine 不超卖测试。 +- `ROUTE-001 / ROUTE-004`：首条命中规则与 Concurrent Sequential 单次切换。 +- `FETCH-005 / FETCH-006`：Valid、Empty、DuplicateOnly、Error 分类。 +- `DIST-001..003 / DIST-006..007`：内存事务模型验证独占提取、满足模式、TTL、 + 健康时效与 Gateway 保留量；1,000 并发不重复。 +- `OPS-001`：完整 Snapshot 目标、epoch/version、校验和验证及原子替换。 +- `CAP-001 / GW 热路径边界`：本地 Dispatch 条件过滤与原子容量预留。 + +## 2. 已执行验证 + +```text +go test ./... PASS +go vet ./... PASS +go build ./... PASS +protoc descriptor compilation PASS +docker compose config PASS +kubectl kustomize PASS +configuration examples 21/21 PASS +Mermaid blocks 35 +``` + +Windows 环境为 `CGO_ENABLED=0` 且没有 C 编译器，`go test -race` 在本机未执行； +CI 已配置 Linux race job。Docker/Kubernetes 仅完成静态验证，没有把目标拓扑 +作为已运行系统。 + +## 3. 后续实现范围 + +以下已有设计、接口或部署位置，但尚无端到端生产实现： + +1. `cmd/proxy-gateway/controller/checker/loadgen` 进程装配。 +2. HTTP 正向代理、HTTPS CONNECT、连接池、安全重试与隧道转发。 +3. Provider Adapter、模板沙箱、singleflight、Leader、退避和累计额度执行器。 +4. PostgreSQL repository、Extraction 行锁事务、Outbox 和迁移。 +5. Redis Leader、速率限制、心跳与可重建协调适配器。 +6. Worker ownership drain/ACK/过期回收和网络快照流。 +7. Checker 调度、探测器和健康 reducer。 +8. Admin/Distribution handler、鉴权、限流和审计查询。 +9. 真实 Compose/Kubernetes 集成、故障演练和代表性集群负载测试。 + +## 4. 容量结论 + +100,000 QPS 是集群设计输入，不是本次验证结果。只有实现上述运行时，并在 +记录协议比例、代理 RTT、连接复用、Worker 规格、故障域、CPU/RSS/FD、延迟 +分位数和错误率的环境中通过持续压测后，才能声明已验证容量。 diff --git a/docs/security/security-model.md b/docs/security/security-model.md new file mode 100644 index 0000000..a44ba80 --- /dev/null +++ b/docs/security/security-model.md @@ -0,0 +1,45 @@ +# 安全模型 + +## 1. 信任边界 + +- Gateway、Distribution、Admin 和 Metrics 为四个独立监听边界。 +- Provider API 和上游 Proxy 属于外部不可信网络。 +- Worker 与 Controller 通道必须进行双向身份校验并绑定 cluster/worker。 +- PostgreSQL 保存权威状态；Redis 数据默认按可重建缓存与协调信息处理。 + +## 2. 入口控制 + +- 严格模式下，非回环监听必须配置认证或 CIDR allowlist。 +- 认证关闭不代表匿名状态消失：仍按可信代理链解析来源并形成 Client ID。 +- Access、Auth、Rate Limit 和 Client Identification 相互独立。 +- Admin 使用独立凭据，不能复用普通 Gateway 或 Distribution 凭据。 + +## 3. 目标地址策略 + +在解析和每次连接前同时检查： + +- 私网、回环、链路本地、组播、保留地址和云元数据地址。 +- 域名解析出的全部 A/AAAA 地址，而不是只检查原始 Host。 +- 重定向或重试后的新目标，防止 DNS Rebinding 和策略绕过。 +- CONNECT 的端口 allowlist 与规范化 host:port。 + +## 4. Secret 处理 + +- 配置只保存环境变量或文件引用，不在日志中输出解析后的值。 +- Proxy 唯一键包含 username 与 credentialVersion，不包含密码或 SecretRef 内容。 +- 指标标签不得使用 token、Proxy URL、Client ID、session 或完整目标 URL。 +- Provider 响应和模板错误只记录分类、Upstream 和 requestId。 + +## 5. Provider 模板 + +- 限制响应体大小、模板执行时间和输出候选数量。 +- 模板函数采用白名单，禁止文件、网络、进程和环境变量访问。 +- URL、Header、Query 和 Body 分别结构化编码，不拼接未转义字符串。 +- 认证失败与限流响应分类处理，429 尊重有上限的 Retry-After。 + +## 6. 审计 + +Extraction 审计记录至少包含 requestId、Client、来源、Proxy ID、Upstream、 +提取时间和到期时间。日志脱敏不影响审计关联，但审计接口自身必须受 Admin +权限保护并具备保留期限。 + diff --git a/docs/testing/failure-injection.md b/docs/testing/failure-injection.md new file mode 100644 index 0000000..09439d8 --- /dev/null +++ b/docs/testing/failure-injection.md @@ -0,0 +1,69 @@ +# 故障注入矩阵 + +## 执行规则 + +- 先在隔离环境建立 15 分钟稳定基线，再注入单一故障。 +- 每次只改变一个变量，记录开始/恢复时间与所有指标。 +- 任何数据库写故障后都核对 Proxy 状态、审计和 outbox，而非只看 HTTP 状态码。 +- 故障恢复后至少观察两个健康检查周期和一个 Snapshot 完整发布周期。 + +## 场景 + +### Provider + +- DNS NXDOMAIN、连接超时、TLS 失败、401、429、500。 +- 响应超过 `maxResponseBytes`、模板超过 `templateTimeout`、非法 Proxy。 +- 合法空列表、全部重复、部分重复加部分新增。 + +预期：只有合法空列表增加 Empty；429 尊重 Retry-After；其余 Error 退避； +duplicate-only 不触发 Sequential 切换。 + +### PostgreSQL + +- 断开 60 秒、连接池耗尽、锁等待、事务提交失败、只读切换。 + +预期：Gateway 继续使用最后 Snapshot；Extract 不返回未提交记录；恢复后 outbox +补发且不重复应用。 + +### Redis + +- 断开、延迟 2 秒、Leader key 丢失、主从切换。 + +预期：Gateway 无影响；Provider Fetch 不出现多个有效 Leader；全局限流明确降级； +恢复后 epoch 单调。 + +### Controller + +- 杀死 Leader、滚动重启全部副本、阻断 Worker 控制流。 + +预期：`maxStaleAge` 内 Gateway 继续，之后停止新流量；Leader 切换不重复计费 +Fetch；Delta 缺口触发完整 Snapshot。 + +### Gateway + +- 杀死一个 Pod、驱逐一个节点、丢失一个可用区、耗尽 FD。 + +预期：LoadBalancer 摘除 NotReady Pod；ownership 租约过期前不分给新 Worker； +剩余容量满足已验证故障域目标。 + +### Checker + +- 慢目标、DNS 延迟、队列积压、杀死一半 Pod。 + +预期：优先新 Proxy 和 SUSPECT；稳定 Proxy 降频；队列有界；Observation 重投 +不导致非法状态回退。 + +### Distribution + +- 100 个并发请求争用相同 Proxy；数据库在事务提交时断开；Gateway 同时满载。 + +预期：每个 Proxy 最多返回一次；提交失败不返回代理；allOrNothing 整批回滚； +池中始终剩余 `reserveForGateway`；没有 Release 恢复路径。 + +### 配置与 Secret + +- 未知字段、错误正则、缺失 Upstream、公开监听无保护、凭据轮换失败。 + +预期：新 Revision 整体拒绝，旧不可变 Snapshot 继续；认证错误计 Error 而不是 +Empty；日志不出现 Secret。 + diff --git a/docs/testing/strategy.md b/docs/testing/strategy.md new file mode 100644 index 0000000..bea241d --- /dev/null +++ b/docs/testing/strategy.md @@ -0,0 +1,49 @@ +# 测试策略 + +## 1. 分层 + +- **领域单测**：状态机、TTL、路由、容量、Fetch 分类和 Extraction 原子性。 +- **契约测试**：配置、OpenAPI、Protobuf 和 Provider Adapter fixture。 +- **集成测试**：PostgreSQL 事务、Redis Leader/限流、Outbox 与重建。 +- **端到端测试**：HTTP、CONNECT、Admin、Distribution 和优雅停机。 +- **负载测试**：Worker 调度微基准、50k 隧道 soak、集群 100k QPS 场景。 + +## 2. 必测不变量 + +1. 1,000 个并发预留不突破单 Proxy 最大并发。 +2. 同一 Proxy 在并发 Extraction 中最多出现一次。 +3. `allOrNothing` 不足时不消耗任何候选。 +4. 连续 4 次 Empty 后 Valid 不切换；连续 5 次只从 A 切到 B。 +5. Error 和 DuplicateOnly 不累计 Empty。 +6. 100 个缺池信号只形成一个合并 Provider reconcile。 +7. 并发 Fetch 不突破 `pool.maxSize` 与 `fetch.maxTotal`。 +8. TTL safety margin 内不再分配。 +9. Snapshot 版本断档、目标错误或校验和错误不替换当前视图。 +10. 非幂等 HTTP 和已建立 CONNECT 不自动重放。 +11. 所有 Upstream 不可用时严格执行显式策略。 + +## 3. 基础质量门禁 + +```powershell +gofmt -l . +go vet ./... +go test ./... +go test -race ./internal/... +go build ./... +``` + +单条测试命令超时 60 秒。依赖真实等待的用例必须改为 fake clock；集成和 +soak 测试单独标记，不混入快速单测。 + +## 4. 100k QPS 验收 + +测试报告必须记录： + +- CPU、内存、内核、网卡、文件描述符和 conntrack 配置。 +- Worker 数量、故障域、目标利用率和负载均衡算法。 +- HTTP/CONNECT 比例、keep-alive、请求/响应大小、上游 RTT 与失败率。 +- Proxy 总数、每 Proxy 容量、Routing 数和 Snapshot 更新频率。 +- 持续时间、P50/P95/P99、成功率、重试率、GC、RSS 和 goroutine 数。 + +只有在代表性环境持续达到目标且满足错误率和延迟门槛后，才能把“设计目标” +改为“已验证容量”。 diff --git a/docs/testing/test-strategy.md b/docs/testing/test-strategy.md new file mode 100644 index 0000000..50f138b --- /dev/null +++ b/docs/testing/test-strategy.md @@ -0,0 +1,130 @@ +# 测试与容量验证策略 + +## 1. 原则 + +- 先证明领域不变量，再证明 Adapter 契约，最后证明跨进程行为。 +- 并发测试必须在 race detector 下运行，不能只依赖单线程示例。 +- 时间、随机、网络、Provider 和存储均通过可替换接口或 fixture 控制。 +- 100,000 QPS 是待验证的集群目标，不以架构图、副本数或短时峰值替代证据。 +- 性能通过与正确性通过相互独立；高 QPS 下出现超卖、重复 Extract 或状态 + 回退时，结果一律失败。 + +## 2. 测试分层 + +### 单元测试 + +- Proxy 唯一键、TTL 优先级、状态机和 safety margin。 +- Atomic Capacity 的 Reserve、Commit、Cancel、Release 与幂等错误。 +- Routing first-match 与五种策略。 +- Fetch Success、Empty、Duplicate-only、Error 分类。 +- Backoff、jitter、Retry-After、requestInterval、maxInFlight。 +- Extraction eligibility、partial、allOrNothing、health age、TTL、reserve。 +- 配置严格字段、交叉引用、正则、监听保护和独立计数语义。 + +### 契约测试 + +- Provider 响应模板的大小、超时、函数白名单和解析边界。 +- PostgreSQL `FOR UPDATE SKIP LOCKED` 并发批量提取。 +- Outbox 状态更新、发布与幂等重放。 +- Redis Leader 租约、限流和失联恢复。 +- Snapshot/Delta/ACK/Report 的版本与校验和兼容性。 +- OpenAPI 错误模型、认证矩阵、批量 fulfillment。 + +### 集成与端到端 + +- HTTP 正向代理成功、上游连接前失败和安全重试。 +- HTTPS CONNECT 建立后不透明重放。 +- Controller Fetch -> Check -> AVAILABLE -> Worker Snapshot -> Gateway 转发。 +- Distribution 原子提取后 Gateway 不再分配同一 Proxy。 +- 配置热更新失败保留旧 Revision，成功后新请求使用新 Snapshot。 + +## 3. 必测的 11 类场景 + +1. **并发容量**：1000 协程争用同一 Proxy，始终满足 + `active + reserved <= effectiveMaxConcurrency`。 +2. **Reservation 生命周期**：Dial 成功/失败、超时、取消和重复 Release 均不 + 泄漏或产生负计数。 +3. **singleflight**：100 个缺池信号只产生一个有效 Fetch 调度。 +4. **Provider 限流**：requestInterval、maxInFlight、timeout、重试和 429 + `Retry-After` 在虚拟时钟下准确。 +5. **Fetch 分类**：Error 不动 Empty；合法空响应 Empty++；duplicate-only + 重置 Empty；Success 重置 Empty。 +6. **Sequential 竞态**：达到阈值时多协程只能将 A 切到 B 一次，不能越过 B。 +7. **Drain**：切换或禁用 Upstream 后停止新分配，已有连接完成后才回收。 +8. **Extract 竞态**：多个请求并发提取同一候选集合，每个 Proxy 最多返回一次。 +9. **Extract 批量语义**：partial 提交实际数量；allOrNothing 不足时状态和审计 + 全回滚；始终保留 `reserveForGateway`。 +10. **Worker ownership**：回收过程严格经过 DRAINING、ACK、active/reserved=0、 + unowned，旧 Snapshot 不可再分配。 +11. **控制面故障**：Redis、PostgreSQL、Controller、Checker 与 Provider 分别 + 失效时，行为与 Runbook 一致，Gateway 热路径不被同步依赖拖垮。 + +## 4. 测试命令 + +所有后台测试都设置 60 秒上限： + +```powershell +go test -timeout 60s ./... +go test -timeout 60s -race ./internal/... +go vet ./... +go build ./... +``` + +需要 PostgreSQL/Redis 的测试使用独立数据库和短生命周期容器，不复用开发数据。 +测试结束后验证没有残留 Worker ownership、Leader 租约或未提交 Extraction Record。 + +## 5. 负载模型 + +必须分开运行，避免不同瓶颈互相掩盖： + +### HTTP QPS + +- GET/HEAD 占比、响应体大小、Keep-Alive 复用率与生产预测一致。 +- 依次运行 10k 稳态、阶梯升压和 100k 峰值。 +- 同时记录端到端与 Gateway 内部 Dispatch 延迟。 + +### CONNECT + +- 分开测试活跃隧道数和每秒新建隧道数。 +- 包含短连接、长连接、半关闭、Client 取消和上游主动断开。 +- 验证 200 已发送后不发生透明重放。 + +### Snapshot + +- 1k、10k、100k Proxy Snapshot，测构建、校验、原子切换、内存峰值和 GC。 +- 在满负载下发布 Snapshot，确认请求线程不参与索引构建。 + +### Extract + +- 小批 partial、大批 allOrNothing、高冲突 filter 和库存不足。 +- 与 Gateway 同时运行，持续检查 `reserveForGateway` 和无重复返回。 + +### 故障负载 + +- 负载运行中断开 Controller、Redis、PostgreSQL、一个 Worker 和一个可用区。 +- Provider 注入 DNS、超时、500、429、超大响应、模板错误和合法空响应。 +- Checker 注入慢目标与队列积压。 + +## 6. 通过条件 + +业务 SLO 由产品最终确认，但至少满足以下工程门槛： + +- 无容量超卖、负计数、重复 Extract、审计缺失或状态非法回退。 +- 100k 峰值期间无进程 OOM、FD 耗尽、无界队列或全局锁热点。 +- Gateway 热路径在 PostgreSQL、Redis、Provider 失效时不发起同步访问。 +- p99 Dispatch 小于 100 微秒的设计预算需要在 100k Proxy Snapshot 下单独证明。 +- 最大故障域丢失后，剩余容量仍满足约定 SLO；否则增加副本或降低承诺容量。 +- Snapshot 超过 `maxStaleAge` 后 Gateway 拒绝新流量，已有连接按时排空。 +- 所有数据、命令、Git SHA、镜像 digest、环境和原始指标可以复现。 + +## 7. 测试报告模板 + +```text +版本：Git SHA / image digest / Go version +环境：节点、CPU、内存、NIC、内核、Kubernetes/CNI +配置：Snapshot 规模、Proxy 容量、路由、重试、日志级别 +场景：协议、连接复用、响应体、持续时间、升压曲线、故障注入 +结果：QPS、建连速率、active、p50/p95/p99、错误、CPU、RSS、GC、FD、网络 +不变量：capacity、ownership、extraction、audit、reserve 检查结果 +结论：通过/失败，以及适用边界 +``` diff --git a/examples/config/01-local-all.yaml b/examples/config/01-local-all.yaml new file mode 100644 index 0000000..aa8cc9a --- /dev/null +++ b/examples/config/01-local-all.yaml @@ -0,0 +1,53 @@ +# 本机同时启用 Gateway 与一次性独占提取 API。 +version: 1 +security: + requireProtectionOnPublicListen: true +gateway: + enabled: true + listen: 127.0.0.1:8080 + auth: {mode: none} + retry: + maxAttempts: 2 + retryMethods: [GET, HEAD] +distribution: + enabled: true + listen: 127.0.0.1:8081 + auth: {mode: none} + clientIdentification: {mode: sourceIP} + extraction: + fulfillment: partial + maxCountPerRequest: 20 + minRemainingTTL: 30s + maxHealthCheckAge: 15s + reserveForGateway: 5 +routing: + - name: gateway-default + enabled: true + purpose: gateway + match: {hostRegex: '.*'} + upstreams: [provider-a] + strategy: {type: leastConnections} + onUnavailable: {action: reject} + - name: extract-default + enabled: true + purpose: extract + match: {hostRegex: '.*'} + upstreams: [provider-a] + strategy: {type: random} + onUnavailable: {action: reject} +upstreams: + provider-a: + enabled: true + exposure: [gateway, extract] + provider: {billingMode: fetch, protocols: [http]} + api: + url: https://provider-a.example/proxies + method: GET + auth: {type: none} + template: '{{.}}' + proxyAuth: {type: response} + pool: {maxSize: 100} + capacity: {maxConcurrencyPerProxy: 10} + lifecycle: {ttl: 2m, allocationSafetyMargin: 10s} + fetch: {requestInterval: 1s, timeout: 3s, maxAttempts: 3, maxInFlight: 1, maxTotal: 1000} + check: {interval: 30s, jitter: 20, maxInFlight: 50, timeout: 2s, maxAttempts: 2, maxConsecutiveFailures: 3} diff --git a/examples/config/02-gateway-only.yaml b/examples/config/02-gateway-only.yaml new file mode 100644 index 0000000..bb87463 --- /dev/null +++ b/examples/config/02-gateway-only.yaml @@ -0,0 +1,33 @@ +# 仅提供本机 HTTP/CONNECT 网关。 +version: 1 +security: {requireProtectionOnPublicListen: true} +gateway: + enabled: true + listen: 127.0.0.1:8080 + auth: {mode: none} + limits: {maxConcurrentConnections: 20000} + retry: {maxAttempts: 2, retryMethods: [GET, HEAD]} + destinationPolicy: + denyPrivateNetworks: true + denyLoopback: true + denyLinkLocal: true +routing: + - name: gateway + enabled: true + purpose: gateway + match: {hostRegex: '.*'} + upstreams: [provider-a] + strategy: {type: leastConnections} + onUnavailable: {action: reject} +upstreams: + provider-a: + enabled: true + exposure: [gateway] + provider: {billingMode: subscription, protocols: [http, https]} + api: {url: https://provider-a.example/proxies, method: GET, auth: {type: none}, template: '{{.}}'} + proxyAuth: {type: response} + pool: {maxSize: 2000} + capacity: {maxConcurrencyPerProxy: 20} + lifecycle: {ttl: 5m, allocationSafetyMargin: 30s} + fetch: {requestInterval: 1s, timeout: 3s, maxAttempts: 3, maxInFlight: 1} + check: {interval: 30s, jitter: 20, maxInFlight: 100, timeout: 2s, maxAttempts: 2, maxConsecutiveFailures: 3} diff --git a/examples/config/03-extract-only.yaml b/examples/config/03-extract-only.yaml new file mode 100644 index 0000000..bdd13d8 --- /dev/null +++ b/examples/config/03-extract-only.yaml @@ -0,0 +1,35 @@ +# 仅提供一次性独占提取；没有 lease/release 配置。 +version: 1 +security: {requireProtectionOnPublicListen: true} +distribution: + enabled: true + listen: 127.0.0.1:8081 + auth: {mode: none} + clientIdentification: {mode: sourceIP} + limits: {requestsPerMinute: 60, requestsPerMinutePerClient: 30} + extraction: + fulfillment: partial + maxCountPerRequest: 50 + minRemainingTTL: 30s + maxHealthCheckAge: 15s + reserveForGateway: 0 +routing: + - name: extract + enabled: true + purpose: extract + match: {hostRegex: '.*'} + upstreams: [provider-a] + strategy: {type: random} + onUnavailable: {action: reject} +upstreams: + provider-a: + enabled: true + exposure: [extract] + provider: {billingMode: fetch, protocols: [http]} + api: {url: https://provider-a.example/proxies, method: GET, auth: {type: none}, template: '{{.}}'} + proxyAuth: {type: response} + pool: {maxSize: 500} + capacity: {maxConcurrencyPerProxy: 1} + lifecycle: {ttl: 2m, allocationSafetyMargin: 10s} + fetch: {requestInterval: 1s, timeout: 3s, maxAttempts: 3, maxInFlight: 1, maxTotal: 10000} + check: {interval: 15s, jitter: 20, maxInFlight: 100, timeout: 2s, maxAttempts: 2, maxConsecutiveFailures: 3} diff --git a/examples/config/04-public-gateway-basic-auth.yaml b/examples/config/04-public-gateway-basic-auth.yaml new file mode 100644 index 0000000..e13093f --- /dev/null +++ b/examples/config/04-public-gateway-basic-auth.yaml @@ -0,0 +1,37 @@ +# 公网 Gateway 使用用户名密码，并拒绝内网/回环目标。 +version: 1 +security: {requireProtectionOnPublicListen: true} +gateway: + enabled: true + listen: 0.0.0.0:8080 + auth: + mode: usernamePassword + username: "${GATEWAY_USER}" + password: "${GATEWAY_PASSWORD}" + limits: {maxConcurrentConnections: 50000} + retry: {maxAttempts: 2, retryMethods: [GET, HEAD]} + destinationPolicy: + denyPrivateNetworks: true + denyLoopback: true + denyLinkLocal: true + denyCIDRs: [169.254.169.254/32] +routing: + - name: public-gateway + enabled: true + purpose: gateway + match: {hostRegex: '.*'} + upstreams: [provider-a] + strategy: {type: leastConnections} + onUnavailable: {action: reject} +upstreams: + provider-a: + enabled: true + exposure: [gateway] + provider: {billingMode: subscription, protocols: [http]} + api: {url: https://provider-a.example/proxies, method: GET, auth: {type: none}, template: '{{.}}'} + proxyAuth: {type: response} + pool: {maxSize: 3000} + capacity: {maxConcurrencyPerProxy: 20} + lifecycle: {ttl: 5m, allocationSafetyMargin: 30s} + fetch: {requestInterval: 1s, timeout: 3s, maxAttempts: 3, maxInFlight: 1} + check: {interval: 30s, jitter: 20, maxInFlight: 200, timeout: 2s, maxAttempts: 2, maxConsecutiveFailures: 3} diff --git a/examples/config/05-public-extract-api-key.yaml b/examples/config/05-public-extract-api-key.yaml new file mode 100644 index 0000000..b9eaa4f --- /dev/null +++ b/examples/config/05-public-extract-api-key.yaml @@ -0,0 +1,38 @@ +# 公网提取 API 使用 X-API-Key；成功后代理立即 EXTRACTED。 +version: 1 +security: {requireProtectionOnPublicListen: true} +distribution: + enabled: true + listen: 0.0.0.0:8081 + auth: + mode: apiKey + header: X-API-Key + token: "${DISTRIBUTION_API_KEY}" + clientIdentification: {mode: authenticatedClient} + limits: {requestsPerMinute: 6000, requestsPerMinutePerClient: 300} + extraction: + fulfillment: partial + maxCountPerRequest: 100 + minRemainingTTL: 45s + maxHealthCheckAge: 10s + reserveForGateway: 0 +routing: + - name: public-extract + enabled: true + purpose: extract + match: {hostRegex: '.*'} + upstreams: [provider-a] + strategy: {type: random} + onUnavailable: {action: reject} +upstreams: + provider-a: + enabled: true + exposure: [extract] + provider: {billingMode: fetch, protocols: [http]} + api: {url: https://provider-a.example/proxies, method: GET, auth: {type: none}, template: '{{.}}'} + proxyAuth: {type: response} + pool: {maxSize: 1000} + capacity: {maxConcurrencyPerProxy: 1} + lifecycle: {ttl: 3m, allocationSafetyMargin: 30s} + fetch: {requestInterval: 1s, timeout: 3s, maxAttempts: 3, maxInFlight: 1, maxTotal: 100000} + check: {interval: 15s, jitter: 20, maxInFlight: 200, timeout: 2s, maxAttempts: 2, maxConsecutiveFailures: 3} diff --git a/examples/config/06-internal-cidr-no-auth.yaml b/examples/config/06-internal-cidr-no-auth.yaml new file mode 100644 index 0000000..b2910f9 --- /dev/null +++ b/examples/config/06-internal-cidr-no-auth.yaml @@ -0,0 +1,45 @@ +# 内网关闭认证，但用来源 CIDR 独立保护两个入口。 +version: 1 +security: {requireProtectionOnPublicListen: true} +gateway: + enabled: true + listen: 0.0.0.0:8080 + access: {allowCIDRs: [10.0.0.0/8, 192.168.0.0/16]} + auth: {mode: none} +distribution: + enabled: true + listen: 0.0.0.0:8081 + access: + allowCIDRs: [10.0.0.0/8, 192.168.0.0/16] + trustedProxies: [10.10.0.10/32] + auth: {mode: none} + clientIdentification: {mode: sourceIP} + limits: {requestsPerMinutePerClient: 30} + extraction: {fulfillment: partial, maxCountPerRequest: 10, minRemainingTTL: 30s, maxHealthCheckAge: 15s, reserveForGateway: 10} +routing: + - name: shared-gateway + enabled: true + purpose: gateway + match: {hostRegex: '.*'} + upstreams: [provider-a] + strategy: {type: leastConnections} + onUnavailable: {action: reject} + - name: shared-extract + enabled: true + purpose: extract + match: {hostRegex: '.*'} + upstreams: [provider-a] + strategy: {type: random} + onUnavailable: {action: reject} +upstreams: + provider-a: + enabled: true + exposure: [gateway, extract] + provider: {billingMode: subscription, protocols: [http]} + api: {url: https://provider-a.example/proxies, method: GET, auth: {type: none}, template: '{{.}}'} + proxyAuth: {type: response} + pool: {maxSize: 1000} + capacity: {maxConcurrencyPerProxy: 10} + lifecycle: {ttl: 2m, allocationSafetyMargin: 15s} + fetch: {requestInterval: 1s, timeout: 3s, maxAttempts: 3, maxInFlight: 1} + check: {interval: 20s, jitter: 20, maxInFlight: 100, timeout: 2s, maxAttempts: 2, maxConsecutiveFailures: 3} diff --git a/examples/config/07-auth-any.yaml b/examples/config/07-auth-any.yaml new file mode 100644 index 0000000..41bb4e3 --- /dev/null +++ b/examples/config/07-auth-any.yaml @@ -0,0 +1,37 @@ +# 来源在可信网段或提供 API Key，任一方法通过即可。 +version: 1 +security: {requireProtectionOnPublicListen: true} +distribution: + enabled: true + listen: 0.0.0.0:8081 + auth: + mode: any + methods: + - mode: ipWhitelist + cidrs: [10.0.0.0/8] + - mode: apiKey + header: X-API-Key + value: "${DISTRIBUTION_API_KEY}" + clientIdentification: {mode: authenticatedClientOrSourceIP} + limits: {requestsPerMinute: 600, requestsPerMinutePerClient: 60} + extraction: {fulfillment: partial, maxCountPerRequest: 20, minRemainingTTL: 30s, maxHealthCheckAge: 15s, reserveForGateway: 0} +routing: + - name: extract + enabled: true + purpose: extract + match: {hostRegex: '.*'} + upstreams: [provider-a] + strategy: {type: random} + onUnavailable: {action: reject} +upstreams: + provider-a: + enabled: true + exposure: [extract] + provider: {billingMode: fetch, protocols: [http]} + api: {url: https://provider-a.example/proxies, method: GET, auth: {type: none}, template: '{{.}}'} + proxyAuth: {type: response} + pool: {maxSize: 500} + capacity: {maxConcurrencyPerProxy: 1} + lifecycle: {ttl: 2m, allocationSafetyMargin: 15s} + fetch: {requestInterval: 1s, timeout: 3s, maxAttempts: 3, maxInFlight: 1, maxTotal: 10000} + check: {interval: 15s, jitter: 20, maxInFlight: 100, timeout: 2s, maxAttempts: 2, maxConsecutiveFailures: 3} diff --git a/examples/config/08-sequential-failover.yaml b/examples/config/08-sequential-failover.yaml new file mode 100644 index 0000000..b139554 --- /dev/null +++ b/examples/config/08-sequential-failover.yaml @@ -0,0 +1,38 @@ +# Provider 返回连续 5 次合法空结果后，Routing 从 A 原子前进到 B。 +version: 1 +security: {requireProtectionOnPublicListen: true} +gateway: {enabled: true, listen: '127.0.0.1:8080', auth: {mode: none}} +routing: + - name: sequential-gateway + enabled: true + purpose: gateway + match: {hostRegex: '.*'} + upstreams: [provider-a, provider-b] + strategy: + type: sequential + switchAfterEmptyFetch: 5 + endBehavior: stayLast + onUnavailable: {action: reject} +upstreams: + provider-a: + enabled: true + exposure: [gateway] + provider: {billingMode: fetch, protocols: [http]} + api: {url: https://provider-a.example/proxies, method: GET, auth: {type: none}, template: '{{.}}'} + proxyAuth: {type: response} + pool: {maxSize: 500} + capacity: {maxConcurrencyPerProxy: 10} + lifecycle: {ttl: 2m, allocationSafetyMargin: 15s} + fetch: {requestInterval: 1s, timeout: 3s, maxAttempts: 3, maxInFlight: 1, maxTotal: 10000} + check: {interval: 30s, jitter: 20, maxInFlight: 100, timeout: 2s, maxAttempts: 2, maxConsecutiveFailures: 3} + provider-b: + enabled: true + exposure: [gateway] + provider: {billingMode: subscription, protocols: [http]} + api: {url: https://provider-b.example/proxies, method: GET, auth: {type: none}, template: '{{.}}'} + proxyAuth: {type: response} + pool: {maxSize: 500} + capacity: {maxConcurrencyPerProxy: 10} + lifecycle: {ttl: 3m, allocationSafetyMargin: 15s} + fetch: {requestInterval: 2s, timeout: 3s, maxAttempts: 3, maxInFlight: 1} + check: {interval: 30s, jitter: 20, maxInFlight: 100, timeout: 2s, maxAttempts: 2, maxConsecutiveFailures: 3} diff --git a/examples/config/09-weighted-routing.yaml b/examples/config/09-weighted-routing.yaml new file mode 100644 index 0000000..2159d6a --- /dev/null +++ b/examples/config/09-weighted-routing.yaml @@ -0,0 +1,37 @@ +# 70/30 权重只决定 Upstream 选择，不复制 Upstream 运行态。 +version: 1 +security: {requireProtectionOnPublicListen: true} +gateway: {enabled: true, listen: '127.0.0.1:8080', auth: {mode: none}} +routing: + - name: weighted-gateway + enabled: true + purpose: gateway + match: {hostRegex: '.*'} + upstreams: [provider-a, provider-b] + strategy: + type: weighted + weights: {provider-a: 70, provider-b: 30} + onUnavailable: {action: reject} +upstreams: + provider-a: + enabled: true + exposure: [gateway] + provider: {billingMode: subscription, protocols: [http]} + api: {url: https://provider-a.example/proxies, method: GET, auth: {type: none}, template: '{{.}}'} + proxyAuth: {type: response} + pool: {maxSize: 1000} + capacity: {maxConcurrencyPerProxy: 10} + lifecycle: {ttl: 2m, allocationSafetyMargin: 15s} + fetch: {requestInterval: 1s, timeout: 3s, maxAttempts: 3, maxInFlight: 1} + check: {interval: 30s, jitter: 20, maxInFlight: 100, timeout: 2s, maxAttempts: 2, maxConsecutiveFailures: 3} + provider-b: + enabled: true + exposure: [gateway] + provider: {billingMode: subscription, protocols: [http]} + api: {url: https://provider-b.example/proxies, method: GET, auth: {type: none}, template: '{{.}}'} + proxyAuth: {type: response} + pool: {maxSize: 1000} + capacity: {maxConcurrencyPerProxy: 10} + lifecycle: {ttl: 2m, allocationSafetyMargin: 15s} + fetch: {requestInterval: 1s, timeout: 3s, maxAttempts: 3, maxInFlight: 1} + check: {interval: 30s, jitter: 20, maxInFlight: 100, timeout: 2s, maxAttempts: 2, maxConsecutiveFailures: 3} diff --git a/examples/config/10-round-robin-routing.yaml b/examples/config/10-round-robin-routing.yaml new file mode 100644 index 0000000..7eb9e45 --- /dev/null +++ b/examples/config/10-round-robin-routing.yaml @@ -0,0 +1,31 @@ +# 在可用 Upstream 间轮询。 +version: 1 +security: {requireProtectionOnPublicListen: true} +gateway: {enabled: true, listen: '127.0.0.1:8080', auth: {mode: none}} +routing: + - name: round-robin + enabled: true + purpose: gateway + match: {hostRegex: '.*'} + upstreams: [provider-a, provider-b] + strategy: {type: roundRobin} + onUnavailable: {action: reject} +upstreams: + provider-a: + enabled: true + exposure: [gateway] + provider: {billingMode: subscription, protocols: [http]} + api: {url: https://provider-a.example/proxies, method: GET, auth: {type: none}, template: '{{.}}'} + proxyAuth: {type: response} + pool: {maxSize: 1000} + capacity: {maxConcurrencyPerProxy: 10} + fetch: {requestInterval: 1s, timeout: 3s, maxAttempts: 3, maxInFlight: 1} + provider-b: + enabled: true + exposure: [gateway] + provider: {billingMode: subscription, protocols: [http]} + api: {url: https://provider-b.example/proxies, method: GET, auth: {type: none}, template: '{{.}}'} + proxyAuth: {type: response} + pool: {maxSize: 1000} + capacity: {maxConcurrencyPerProxy: 10} + fetch: {requestInterval: 1s, timeout: 3s, maxAttempts: 3, maxInFlight: 1} diff --git a/examples/config/11-random-routing.yaml b/examples/config/11-random-routing.yaml new file mode 100644 index 0000000..2777134 --- /dev/null +++ b/examples/config/11-random-routing.yaml @@ -0,0 +1,31 @@ +# 每次从可用 Upstream 集合随机选择。 +version: 1 +security: {requireProtectionOnPublicListen: true} +gateway: {enabled: true, listen: '127.0.0.1:8080', auth: {mode: none}} +routing: + - name: random-gateway + enabled: true + purpose: gateway + match: {hostRegex: '.*'} + upstreams: [provider-a, provider-b] + strategy: {type: random} + onUnavailable: {action: reject} +upstreams: + provider-a: + enabled: true + exposure: [gateway] + provider: {billingMode: subscription, protocols: [http]} + api: {url: https://provider-a.example/proxies, method: GET, auth: {type: none}, template: '{{.}}'} + proxyAuth: {type: response} + pool: {maxSize: 1000} + capacity: {maxConcurrencyPerProxy: 10} + fetch: {requestInterval: 1s, timeout: 3s, maxAttempts: 3, maxInFlight: 1} + provider-b: + enabled: true + exposure: [gateway] + provider: {billingMode: subscription, protocols: [http]} + api: {url: https://provider-b.example/proxies, method: GET, auth: {type: none}, template: '{{.}}'} + proxyAuth: {type: response} + pool: {maxSize: 1000} + capacity: {maxConcurrencyPerProxy: 10} + fetch: {requestInterval: 1s, timeout: 3s, maxAttempts: 3, maxInFlight: 1} diff --git a/examples/config/12-least-connections-routing.yaml b/examples/config/12-least-connections-routing.yaml new file mode 100644 index 0000000..3a2fd7d --- /dev/null +++ b/examples/config/12-least-connections-routing.yaml @@ -0,0 +1,28 @@ +# Gateway 按本地 Active + Reserved 选择剩余容量最高的 Proxy。 +version: 1 +security: {requireProtectionOnPublicListen: true} +gateway: + enabled: true + listen: 127.0.0.1:8080 + auth: {mode: none} + limits: {maxConcurrentConnections: 100000} +routing: + - name: least-connections + enabled: true + purpose: gateway + match: {hostRegex: '.*'} + upstreams: [provider-a] + strategy: {type: leastConnections} + onUnavailable: {action: reject} +upstreams: + provider-a: + enabled: true + exposure: [gateway] + provider: {billingMode: subscription, protocols: [http, https]} + api: {url: https://provider-a.example/proxies, method: GET, auth: {type: none}, template: '{{.}}'} + proxyAuth: {type: response} + pool: {maxSize: 10000} + capacity: {maxConcurrencyPerProxy: 50} + lifecycle: {ttl: 5m, allocationSafetyMargin: 30s} + fetch: {requestInterval: 1s, timeout: 3s, maxAttempts: 3, maxInFlight: 1} + check: {interval: 30s, jitter: 20, maxInFlight: 500, timeout: 2s, maxAttempts: 2, maxConsecutiveFailures: 3} diff --git a/examples/config/13-extract-all-or-nothing.yaml b/examples/config/13-extract-all-or-nothing.yaml new file mode 100644 index 0000000..1a75dd5 --- /dev/null +++ b/examples/config/13-extract-all-or-nothing.yaml @@ -0,0 +1,34 @@ +# 数量不足时返回 409，事务不改变任何 Proxy 状态。 +version: 1 +security: {requireProtectionOnPublicListen: true} +distribution: + enabled: true + listen: 127.0.0.1:8081 + auth: {mode: none} + clientIdentification: {mode: sourceIP} + extraction: + fulfillment: allOrNothing + maxCountPerRequest: 100 + minRemainingTTL: 30s + maxHealthCheckAge: 15s + reserveForGateway: 0 +routing: + - name: extract + enabled: true + purpose: extract + match: {hostRegex: '.*'} + upstreams: [provider-a] + strategy: {type: random} + onUnavailable: {action: reject} +upstreams: + provider-a: + enabled: true + exposure: [extract] + provider: {billingMode: fetch, protocols: [http]} + api: {url: https://provider-a.example/proxies, method: GET, auth: {type: none}, template: '{{.}}'} + proxyAuth: {type: response} + pool: {maxSize: 500} + capacity: {maxConcurrencyPerProxy: 1} + lifecycle: {ttl: 2m, allocationSafetyMargin: 15s} + fetch: {requestInterval: 1s, timeout: 3s, maxAttempts: 3, maxInFlight: 1, maxTotal: 50000} + check: {interval: 15s, jitter: 20, maxInFlight: 100, timeout: 2s, maxAttempts: 2, maxConsecutiveFailures: 3} diff --git a/examples/config/14-gateway-reserve.yaml b/examples/config/14-gateway-reserve.yaml new file mode 100644 index 0000000..c381c35 --- /dev/null +++ b/examples/config/14-gateway-reserve.yaml @@ -0,0 +1,42 @@ +# 共享池至少为 Gateway 保留 100 个符合提取条件的可用 Proxy。 +version: 1 +security: {requireProtectionOnPublicListen: true} +gateway: {enabled: true, listen: '127.0.0.1:8080', auth: {mode: none}} +distribution: + enabled: true + listen: 127.0.0.1:8081 + auth: {mode: none} + clientIdentification: {mode: sourceIP} + extraction: + fulfillment: partial + maxCountPerRequest: 100 + minRemainingTTL: 30s + maxHealthCheckAge: 15s + reserveForGateway: 100 +routing: + - name: gateway + enabled: true + purpose: gateway + match: {hostRegex: '.*'} + upstreams: [provider-a] + strategy: {type: leastConnections} + onUnavailable: {action: reject} + - name: extract + enabled: true + purpose: extract + match: {hostRegex: '.*'} + upstreams: [provider-a] + strategy: {type: random} + onUnavailable: {action: reject} +upstreams: + provider-a: + enabled: true + exposure: [gateway, extract] + provider: {billingMode: subscription, protocols: [http]} + api: {url: https://provider-a.example/proxies, method: GET, auth: {type: none}, template: '{{.}}'} + proxyAuth: {type: response} + pool: {maxSize: 2000} + capacity: {maxConcurrencyPerProxy: 10} + lifecycle: {ttl: 3m, allocationSafetyMargin: 20s} + fetch: {requestInterval: 1s, timeout: 3s, maxAttempts: 3, maxInFlight: 1} + check: {interval: 15s, jitter: 20, maxInFlight: 200, timeout: 2s, maxAttempts: 2, maxConsecutiveFailures: 3} diff --git a/examples/config/15-strict-ttl-health.yaml b/examples/config/15-strict-ttl-health.yaml new file mode 100644 index 0000000..b0f823e --- /dev/null +++ b/examples/config/15-strict-ttl-health.yaml @@ -0,0 +1,34 @@ +# 只发放剩余 TTL 至少 60 秒且 5 秒内检查过的 Proxy。 +version: 1 +security: {requireProtectionOnPublicListen: true} +distribution: + enabled: true + listen: 127.0.0.1:8081 + auth: {mode: none} + clientIdentification: {mode: sourceIP} + extraction: + fulfillment: partial + maxCountPerRequest: 20 + minRemainingTTL: 60s + maxHealthCheckAge: 5s + reserveForGateway: 0 +routing: + - name: fresh-extract + enabled: true + purpose: extract + match: {hostRegex: '.*'} + upstreams: [provider-a] + strategy: {type: random} + onUnavailable: {action: reject} +upstreams: + provider-a: + enabled: true + exposure: [extract] + provider: {billingMode: fetch, protocols: [http]} + api: {url: https://provider-a.example/proxies, method: GET, auth: {type: none}, template: '{{.}}'} + proxyAuth: {type: response} + pool: {maxSize: 1000} + capacity: {maxConcurrencyPerProxy: 1} + lifecycle: {ttl: 5m, allocationSafetyMargin: 60s} + fetch: {requestInterval: 1s, timeout: 3s, maxAttempts: 3, maxInFlight: 1, maxTotal: 100000} + check: {interval: 5s, jitter: 20, maxInFlight: 500, timeout: 2s, maxAttempts: 2, maxConsecutiveFailures: 3} diff --git a/examples/config/16-provider-basic-auth.yaml b/examples/config/16-provider-basic-auth.yaml new file mode 100644 index 0000000..44815ff --- /dev/null +++ b/examples/config/16-provider-basic-auth.yaml @@ -0,0 +1,31 @@ +# Provider API 认证与对外 Client 认证相互独立。 +version: 1 +security: {requireProtectionOnPublicListen: true} +gateway: {enabled: true, listen: '127.0.0.1:8080', auth: {mode: none}} +routing: + - name: gateway + enabled: true + purpose: gateway + match: {hostRegex: '.*'} + upstreams: [provider-a] + strategy: {type: leastConnections} + onUnavailable: {action: reject} +upstreams: + provider-a: + enabled: true + exposure: [gateway] + provider: {billingMode: subscription, protocols: [http]} + api: + url: https://provider-a.example/proxies + method: GET + auth: + type: basic + username: "${PROVIDER_API_USER}" + password: "${PROVIDER_API_PASSWORD}" + template: '{{.}}' + proxyAuth: {type: response} + pool: {maxSize: 1000} + capacity: {maxConcurrencyPerProxy: 10} + lifecycle: {ttl: 2m, allocationSafetyMargin: 15s} + fetch: {requestInterval: 1s, timeout: 3s, maxAttempts: 3, maxInFlight: 1} + check: {interval: 30s, jitter: 20, maxInFlight: 100, timeout: 2s, maxAttempts: 2, maxConsecutiveFailures: 3} diff --git a/examples/config/17-provider-api-key.yaml b/examples/config/17-provider-api-key.yaml new file mode 100644 index 0000000..b853e0a --- /dev/null +++ b/examples/config/17-provider-api-key.yaml @@ -0,0 +1,32 @@ +# Provider API Key 放在专用 Header；日志必须统一脱敏。 +version: 1 +security: {requireProtectionOnPublicListen: true} +gateway: {enabled: true, listen: '127.0.0.1:8080', auth: {mode: none}} +routing: + - name: gateway + enabled: true + purpose: gateway + match: {hostRegex: '.*'} + upstreams: [provider-a] + strategy: {type: leastConnections} + onUnavailable: {action: reject} +upstreams: + provider-a: + enabled: true + exposure: [gateway] + provider: {billingMode: fetch, protocols: [http]} + api: + url: https://provider-a.example/proxies + method: GET + auth: + type: apiKey + location: header + name: X-Provider-Key + value: "${PROVIDER_API_KEY}" + template: '{{.}}' + proxyAuth: {type: response} + pool: {maxSize: 500} + capacity: {maxConcurrencyPerProxy: 10} + lifecycle: {ttl: 2m, allocationSafetyMargin: 15s} + fetch: {requestInterval: 1s, timeout: 3s, maxAttempts: 3, maxInFlight: 1, maxTotal: 10000} + check: {interval: 30s, jitter: 20, maxInFlight: 100, timeout: 2s, maxAttempts: 2, maxConsecutiveFailures: 3} diff --git a/examples/config/18-provider-post-json.yaml b/examples/config/18-provider-post-json.yaml new file mode 100644 index 0000000..0427f48 --- /dev/null +++ b/examples/config/18-provider-post-json.yaml @@ -0,0 +1,40 @@ +# Provider 使用 JSON POST；供应商重试由独立 Fetch 策略控制。 +version: 1 +security: {requireProtectionOnPublicListen: true} +gateway: {enabled: true, listen: '127.0.0.1:8080', auth: {mode: none}} +routing: + - name: gateway + enabled: true + purpose: gateway + match: {hostRegex: '.*'} + upstreams: [provider-a] + strategy: {type: leastConnections} + onUnavailable: {action: reject} +upstreams: + provider-a: + enabled: true + exposure: [gateway] + provider: {billingMode: fetch, protocols: [http]} + api: + url: https://provider-a.example/v1/orders + method: POST + auth: {type: apiKey, location: header, name: X-Provider-Key, value: "${PROVIDER_API_KEY}"} + headers: {Content-Type: application/json} + body: + type: json + value: {count: '100', protocol: http} + template: '{{.}}' + proxyAuth: {type: response} + pool: {maxSize: 1000} + capacity: {maxConcurrencyPerProxy: 10} + lifecycle: {ttl: 3m, allocationSafetyMargin: 20s} + fetch: + requestInterval: 2s + timeout: 5s + maxAttempts: 3 + maxInFlight: 1 + maxTotal: 100000 + maxResponseBytes: 1048576 + templateTimeout: 100ms + retry: {initial: 1s, max: 30s, jitter: 20} + check: {interval: 30s, jitter: 20, maxInFlight: 100, timeout: 2s, maxAttempts: 2, maxConsecutiveFailures: 3} diff --git a/examples/config/19-socks5-upstream.yaml b/examples/config/19-socks5-upstream.yaml new file mode 100644 index 0000000..c3c4895 --- /dev/null +++ b/examples/config/19-socks5-upstream.yaml @@ -0,0 +1,27 @@ +# SOCKS5 上游保留在统一 Proxy 模型和策略契约内。 +version: 1 +security: {requireProtectionOnPublicListen: true} +gateway: {enabled: true, listen: '127.0.0.1:8080', auth: {mode: none}} +routing: + - name: socks-upstream + enabled: true + purpose: gateway + match: {hostRegex: '.*'} + upstreams: [provider-socks] + strategy: {type: leastConnections} + onUnavailable: {action: reject} +upstreams: + provider-socks: + enabled: true + exposure: [gateway] + provider: {billingMode: subscription, protocols: [socks5]} + api: {url: https://provider-socks.example/proxies, method: GET, auth: {type: none}, template: '{{.}}'} + proxyAuth: + type: static + username: "${SOCKS_USER}" + password: "${SOCKS_PASSWORD}" + pool: {maxSize: 1000} + capacity: {maxConcurrencyPerProxy: 20} + lifecycle: {ttl: 10m, allocationSafetyMargin: 60s} + fetch: {requestInterval: 5s, timeout: 5s, maxAttempts: 3, maxInFlight: 1} + check: {interval: 30s, jitter: 20, maxInFlight: 100, timeout: 3s, maxAttempts: 2, maxConsecutiveFailures: 3} diff --git a/examples/config/20-fetch-billing-quota.yaml b/examples/config/20-fetch-billing-quota.yaml new file mode 100644 index 0000000..b730204 --- /dev/null +++ b/examples/config/20-fetch-billing-quota.yaml @@ -0,0 +1,35 @@ +# pool.maxSize 限当前未提取库存；fetch.maxTotal 限计费周期累计获取数。 +version: 1 +security: {requireProtectionOnPublicListen: true} +distribution: + enabled: true + listen: 127.0.0.1:8081 + auth: {mode: none} + clientIdentification: {mode: sourceIP} + extraction: {fulfillment: partial, maxCountPerRequest: 20, minRemainingTTL: 30s, maxHealthCheckAge: 15s, reserveForGateway: 0} +routing: + - name: billed-extract + enabled: true + purpose: extract + match: {hostRegex: '.*'} + upstreams: [provider-metered] + strategy: {type: random} + onUnavailable: {action: reject} +upstreams: + provider-metered: + enabled: true + exposure: [extract] + provider: {billingMode: fetch, protocols: [http]} + api: {url: https://provider-metered.example/proxies, method: GET, auth: {type: none}, template: '{{.}}'} + proxyAuth: {type: response} + pool: {maxSize: 100} + capacity: {maxConcurrencyPerProxy: 1} + lifecycle: {ttl: 2m, allocationSafetyMargin: 15s} + fetch: + requestInterval: 1s + timeout: 3s + maxAttempts: 3 + maxInFlight: 1 + maxTotal: 1000 + retry: {initial: 500ms, max: 30s, jitter: 20} + check: {interval: 15s, jitter: 20, maxInFlight: 100, timeout: 2s, maxAttempts: 2, maxConsecutiveFailures: 3} diff --git a/examples/config/examples_test.go b/examples/config/examples_test.go new file mode 100644 index 0000000..e7c3c0a --- /dev/null +++ b/examples/config/examples_test.go @@ -0,0 +1,64 @@ +package configexamples + +import ( + "os" + "path/filepath" + "strings" + "testing" + + projectconfig "github.com/proxy-pool/proxy-pool/internal/config" +) + +func TestAllExamplesLoadStrictly(t *testing.T) { + entries, err := filepath.Glob("*.yaml") + if err != nil { + t.Fatalf("glob examples: %v", err) + } + if len(entries) < 20 { + t.Fatalf("configuration examples = %d, want at least 20", len(entries)) + } + for _, path := range entries { + path := path + t.Run(strings.TrimSuffix(filepath.Base(path), filepath.Ext(path)), func(t *testing.T) { + file, err := os.Open(path) + if err != nil { + t.Fatalf("open %s: %v", path, err) + } + defer file.Close() + cfg, err := projectconfig.Load(file) + if err != nil { + t.Fatalf("load %s: %v", path, err) + } + assertExplicitProviderAuth(t, cfg) + }) + } +} + +func TestMainConfigurationLoadsStrictly(t *testing.T) { + path := filepath.Join("..", "..", "configs", "proxy-pool.yaml") + file, err := os.Open(path) + if err != nil { + t.Fatalf("open %s: %v", path, err) + } + defer file.Close() + cfg, err := projectconfig.Load(file) + if err != nil { + t.Fatalf("load %s: %v", path, err) + } + assertExplicitProviderAuth(t, cfg) +} + +func assertExplicitProviderAuth(t *testing.T, cfg *projectconfig.Config) { + t.Helper() + for name, upstream := range cfg.Upstreams { + if !upstream.Enabled { + continue + } + if upstream.API.Auth.Type == "" { + t.Errorf("upstream %s must explicitly set api.auth.type", name) + } + if upstream.ProxyAuth.Type == "" { + t.Errorf("upstream %s must explicitly set proxyAuth.type", name) + } + } +} diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..68ad726 --- /dev/null +++ b/go.mod @@ -0,0 +1,5 @@ +module github.com/proxy-pool/proxy-pool + +go 1.26.0 + +require go.yaml.in/yaml/v4 v4.0.0-rc.3 diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..1b5b7f5 --- /dev/null +++ b/go.sum @@ -0,0 +1,2 @@ +go.yaml.in/yaml/v4 v4.0.0-rc.3 h1:3h1fjsh1CTAPjW7q/EMe+C8shx5d8ctzZTrLcs/j8Go= +go.yaml.in/yaml/v4 v4.0.0-rc.3/go.mod h1:aZqd9kCMsGL7AuUv/m/PvWLdg5sjJsZ4oHDEnfPPfY0= diff --git a/internal/config/config.go b/internal/config/config.go new file mode 100644 index 0000000..f78ee7e --- /dev/null +++ b/internal/config/config.go @@ -0,0 +1,244 @@ +package config + +import ( + "fmt" + "time" +) + +type Duration time.Duration + +func (d *Duration) UnmarshalText(text []byte) error { + value, err := time.ParseDuration(string(text)) + if err != nil { + return fmt.Errorf("parse duration %q: %w", text, err) + } + *d = Duration(value) + return nil +} + +func (d Duration) Value() time.Duration { return time.Duration(d) } + +type Config struct { + Version int `yaml:"version"` + Defaults Defaults `yaml:"defaults"` + Security Security `yaml:"security"` + Gateway Listener `yaml:"gateway"` + Distribution Distribution `yaml:"distribution"` + Admin Listener `yaml:"admin"` + Metrics Metrics `yaml:"metrics"` + Storage Storage `yaml:"storage"` + Routing []Routing `yaml:"routing"` + Upstreams map[string]Upstream `yaml:"upstreams"` +} + +type Defaults struct { + Fetch Fetch `yaml:"fetch"` + Check Check `yaml:"check"` +} + +type Security struct { + RequireProtectionOnPublicListen bool `yaml:"requireProtectionOnPublicListen"` +} + +type Listener struct { + Enabled bool `yaml:"enabled"` + Listen string `yaml:"listen"` + Access Access `yaml:"access"` + Auth Auth `yaml:"auth"` + Limits Limits `yaml:"limits"` + Retry Retry `yaml:"retry"` + DestinationPolicy DestinationPolicy `yaml:"destinationPolicy"` +} + +type Distribution struct { + Listener `yaml:",inline"` + ClientIdentification ClientIdentification `yaml:"clientIdentification"` + Extraction Extraction `yaml:"extraction"` +} + +type Access struct { + AllowCIDRs []string `yaml:"allowCIDRs"` + TrustedProxies []string `yaml:"trustedProxies"` +} + +type Auth struct { + Mode string `yaml:"mode"` + Username string `yaml:"username"` + Password string `yaml:"password"` + Token string `yaml:"token"` + Header string `yaml:"header"` + CIDRs []string `yaml:"cidrs"` + Methods []AuthMethod `yaml:"methods"` +} + +type AuthMethod struct { + Mode string `yaml:"mode"` + Username string `yaml:"username"` + Password string `yaml:"password"` + Header string `yaml:"header"` + Value string `yaml:"value"` + CIDRs []string `yaml:"cidrs"` +} + +type Limits struct { + MaxConcurrentConnections int `yaml:"maxConcurrentConnections"` + RequestsPerMinute int `yaml:"requestsPerMinute"` + RequestsPerMinutePerClient int `yaml:"requestsPerMinutePerClient"` +} + +type Retry struct { + MaxAttempts int `yaml:"maxAttempts"` + RetryMethods []string `yaml:"retryMethods"` +} + +type DestinationPolicy struct { + DenyPrivateNetworks bool `yaml:"denyPrivateNetworks"` + DenyLoopback bool `yaml:"denyLoopback"` + DenyLinkLocal bool `yaml:"denyLinkLocal"` + DenyCIDRs []string `yaml:"denyCIDRs"` +} + +type ClientIdentification struct { + Mode string `yaml:"mode"` +} + +type Extraction struct { + Fulfillment string `yaml:"fulfillment"` + MaxCountPerRequest int `yaml:"maxCountPerRequest"` + MinRemainingTTL Duration `yaml:"minRemainingTTL"` + MaxHealthCheckAge Duration `yaml:"maxHealthCheckAge"` + ReserveForGateway int `yaml:"reserveForGateway"` +} + +type Metrics struct { + Enabled bool `yaml:"enabled"` + Listen string `yaml:"listen"` +} + +type Storage struct { + PostgresURL string `yaml:"postgresURL"` + RedisURL string `yaml:"redisURL"` +} + +type Routing struct { + Name string `yaml:"name"` + Enabled bool `yaml:"enabled"` + Purpose string `yaml:"purpose"` + Match RoutingMatch `yaml:"match"` + Upstreams []string `yaml:"upstreams"` + Strategy Strategy `yaml:"strategy"` + OnUnavailable OnUnavailable `yaml:"onUnavailable"` +} + +type RoutingMatch struct { + HostRegex string `yaml:"hostRegex"` + Methods []string `yaml:"methods"` + PathRegex string `yaml:"pathRegex"` + Headers map[string]string `yaml:"headers"` +} + +type Strategy struct { + Type string `yaml:"type"` + SwitchAfterEmptyFetch int `yaml:"switchAfterEmptyFetch"` + EndBehavior string `yaml:"endBehavior"` + Weights map[string]int `yaml:"weights"` +} + +type OnUnavailable struct { + Action string `yaml:"action"` + WaitTimeout Duration `yaml:"waitTimeout"` +} + +type Upstream struct { + Enabled bool `yaml:"enabled"` + Exposure []string `yaml:"exposure"` + Provider Provider `yaml:"provider"` + API ProviderAPI `yaml:"api"` + ProxyAuth ProxyAuth `yaml:"proxyAuth"` + Pool Pool `yaml:"pool"` + Capacity Capacity `yaml:"capacity"` + Lifecycle Lifecycle `yaml:"lifecycle"` + Fetch Fetch `yaml:"fetch"` + Check Check `yaml:"check"` +} + +type Provider struct { + BillingMode string `yaml:"billingMode"` + Protocols []string `yaml:"protocols"` +} + +type ProviderAPI struct { + URL string `yaml:"url"` + Method string `yaml:"method"` + Auth ProviderAuth `yaml:"auth"` + Headers map[string]string `yaml:"headers"` + Query map[string]string `yaml:"query"` + Body APIBody `yaml:"body"` + Template string `yaml:"template"` +} + +type ProviderAuth struct { + Type string `yaml:"type"` + Username string `yaml:"username"` + Password string `yaml:"password"` + PasswordFile string `yaml:"passwordFile"` + Token string `yaml:"token"` + TokenFile string `yaml:"tokenFile"` + Location string `yaml:"location"` + Name string `yaml:"name"` + Value string `yaml:"value"` + ValueFile string `yaml:"valueFile"` +} + +type APIBody struct { + Type string `yaml:"type"` + Value map[string]string `yaml:"value"` +} + +type ProxyAuth struct { + Type string `yaml:"type"` + Username string `yaml:"username"` + Password string `yaml:"password"` + PasswordFile string `yaml:"passwordFile"` +} + +type Pool struct { + MaxSize int `yaml:"maxSize"` + ShrinkDelay Duration `yaml:"shrinkDelay"` +} + +type Capacity struct { + MaxConcurrencyPerProxy int `yaml:"maxConcurrencyPerProxy"` +} + +type Lifecycle struct { + TTL Duration `yaml:"ttl"` + AllocationSafetyMargin Duration `yaml:"allocationSafetyMargin"` +} + +type Fetch struct { + RequestInterval Duration `yaml:"requestInterval"` + Timeout Duration `yaml:"timeout"` + MaxAttempts int `yaml:"maxAttempts"` + MaxInFlight int `yaml:"maxInFlight"` + MaxTotal int `yaml:"maxTotal"` + MaxResponseBytes int64 `yaml:"maxResponseBytes"` + TemplateTimeout Duration `yaml:"templateTimeout"` + Retry Backoff `yaml:"retry"` +} + +type Backoff struct { + Initial Duration `yaml:"initial"` + Max Duration `yaml:"max"` + Jitter int `yaml:"jitter"` +} + +type Check struct { + Interval Duration `yaml:"interval"` + Jitter int `yaml:"jitter"` + MaxInFlight int `yaml:"maxInFlight"` + Timeout Duration `yaml:"timeout"` + MaxAttempts int `yaml:"maxAttempts"` + MaxConsecutiveFailures int `yaml:"maxConsecutiveFailures"` + URLs []string `yaml:"urls"` +} diff --git a/internal/config/config_test.go b/internal/config/config_test.go new file mode 100644 index 0000000..dfbb1bb --- /dev/null +++ b/internal/config/config_test.go @@ -0,0 +1,145 @@ +package config + +import ( + "os" + "path/filepath" + "strings" + "testing" +) + +const validConfig = ` +version: 1 +security: + requireProtectionOnPublicListen: true +gateway: + enabled: true + listen: 127.0.0.1:8080 + auth: + mode: none +distribution: + enabled: true + listen: 127.0.0.1:8081 + auth: + mode: none + extraction: + fulfillment: partial + maxCountPerRequest: 20 + minRemainingTTL: 30s + maxHealthCheckAge: 15s + reserveForGateway: 5 +routing: + - name: extract + enabled: true + purpose: extract + upstreams: [provider-a] + strategy: + type: sequential + switchAfterEmptyFetch: 5 + onUnavailable: + action: reject +upstreams: + provider-a: + enabled: true + exposure: [gateway, extract] + provider: + billingMode: fetch + protocols: [http] + api: + url: https://provider.example/proxies + method: GET + template: '{{.}}' + auth: + type: none + proxyAuth: + type: response + pool: + maxSize: 100 + capacity: + maxConcurrencyPerProxy: 10 + lifecycle: + ttl: 120s + allocationSafetyMargin: 10s + fetch: + requestInterval: 1s + timeout: 3s + maxAttempts: 5 + maxInFlight: 1 + maxTotal: 1000 + check: + interval: 30s + jitter: 20 + maxInFlight: 100 + timeout: 2s + maxAttempts: 2 + maxConsecutiveFailures: 3 + urls: [http://connect.rom.miui.com/generate_204] +` + +func TestLoadStrictValidConfiguration(t *testing.T) { + cfg, err := Load(strings.NewReader(validConfig)) + if err != nil { + t.Fatalf("Load(): %v", err) + } + if cfg.Version != 1 || cfg.Upstreams["provider-a"].Pool.MaxSize != 100 { + t.Fatalf("unexpected config: %+v", cfg) + } +} + +func TestLoadRejectsUnknownFields(t *testing.T) { + _, err := Load(strings.NewReader(validConfig + "unknownField: true\n")) + if err == nil || !strings.Contains(err.Error(), "unknownField") { + t.Fatalf("Load() error = %v, want unknown field error", err) + } +} + +func TestValidateRejectsUnprotectedPublicListener(t *testing.T) { + cfg, err := Load(strings.NewReader(strings.Replace(validConfig, + "listen: 127.0.0.1:8080", "listen: 0.0.0.0:8080", 1))) + if err == nil || !strings.Contains(err.Error(), "gateway") || !strings.Contains(err.Error(), "public") { + t.Fatalf("Load() error = %v, want unprotected public listener error", err) + } + if cfg != nil { + t.Fatal("invalid config must not be returned") + } +} + +func TestValidateRejectsMissingUpstreamReference(t *testing.T) { + broken := strings.Replace(validConfig, "upstreams: [provider-a]", "upstreams: [missing]", 1) + _, err := Load(strings.NewReader(broken)) + if err == nil || !strings.Contains(err.Error(), "missing") { + t.Fatalf("Load() error = %v, want missing upstream error", err) + } +} + +func TestValidateSeparatesPoolAndFetchLimits(t *testing.T) { + broken := strings.Replace(validConfig, "maxTotal: 1000", "maxTotal: 50", 1) + _, err := Load(strings.NewReader(broken)) + if err == nil || !strings.Contains(err.Error(), "maxTotal") { + t.Fatalf("Load() error = %v, want maxTotal validation error", err) + } +} + +func TestShippedConfigurationsAreValid(t *testing.T) { + paths, err := filepath.Glob(filepath.Join("..", "..", "examples", "config", "*.yaml")) + if err != nil { + t.Fatalf("Glob(): %v", err) + } + paths = append(paths, filepath.Join("..", "..", "configs", "proxy-pool.yaml")) + if len(paths) != 21 { + t.Fatalf("configuration count = %d, want 21", len(paths)) + } + + for _, path := range paths { + path := path + t.Run(filepath.Base(path), func(t *testing.T) { + file, err := os.Open(path) + if err != nil { + t.Fatalf("Open(): %v", err) + } + defer file.Close() + if _, err := Load(file); err != nil { + t.Fatalf("Load(): %v", err) + } + }) + } +} diff --git a/internal/config/load.go b/internal/config/load.go new file mode 100644 index 0000000..d282e93 --- /dev/null +++ b/internal/config/load.go @@ -0,0 +1,22 @@ +package config + +import ( + "fmt" + "io" + + "go.yaml.in/yaml/v4" +) + +func Load(reader io.Reader) (*Config, error) { + decoder := yaml.NewDecoder(reader) + decoder.KnownFields(true) + + var cfg Config + if err := decoder.Decode(&cfg); err != nil { + return nil, fmt.Errorf("decode configuration: %w", err) + } + if err := Validate(&cfg); err != nil { + return nil, err + } + return &cfg, nil +} diff --git a/internal/config/validate.go b/internal/config/validate.go new file mode 100644 index 0000000..bc47681 --- /dev/null +++ b/internal/config/validate.go @@ -0,0 +1,193 @@ +package config + +import ( + "fmt" + "net" + "net/url" + "regexp" + "strings" +) + +func Validate(cfg *Config) error { + if cfg == nil { + return fmt.Errorf("validate configuration: nil config") + } + if cfg.Version != 1 { + return fmt.Errorf("validate configuration: version must be 1") + } + listeners := []struct { + name string + item Listener + }{ + {name: "gateway", item: cfg.Gateway}, + {name: "distribution", item: cfg.Distribution.Listener}, + {name: "admin", item: cfg.Admin}, + } + for _, listener := range listeners { + if err := validateListener(listener.name, listener.item, cfg.Security); err != nil { + return err + } + } + for name, upstream := range cfg.Upstreams { + if err := validateUpstream(name, upstream); err != nil { + return err + } + } + seen := make(map[string]struct{}, len(cfg.Routing)) + for index, route := range cfg.Routing { + if route.Name == "" { + return fmt.Errorf("validate routing[%d]: name is required", index) + } + if _, ok := seen[route.Name]; ok { + return fmt.Errorf("validate routing %q: duplicate name", route.Name) + } + seen[route.Name] = struct{}{} + if route.Match.HostRegex != "" { + if _, err := regexp.Compile(route.Match.HostRegex); err != nil { + return fmt.Errorf("validate routing %q hostRegex: %w", route.Name, err) + } + } + if route.Match.PathRegex != "" { + if _, err := regexp.Compile(route.Match.PathRegex); err != nil { + return fmt.Errorf("validate routing %q pathRegex: %w", route.Name, err) + } + } + for _, upstream := range route.Upstreams { + if _, ok := cfg.Upstreams[upstream]; !ok { + return fmt.Errorf("validate routing %q: upstream %q does not exist", route.Name, upstream) + } + } + if route.Strategy.Type == "sequential" && route.Strategy.SwitchAfterEmptyFetch <= 0 { + return fmt.Errorf("validate routing %q: switchAfterEmptyFetch must be greater than zero", route.Name) + } + if route.OnUnavailable.Action == "" { + return fmt.Errorf("validate routing %q: onUnavailable.action is required", route.Name) + } + } + if cfg.Distribution.Enabled { + if cfg.Distribution.Extraction.MaxCountPerRequest <= 0 { + return fmt.Errorf("validate distribution: maxCountPerRequest must be greater than zero") + } + if cfg.Distribution.Extraction.Fulfillment != "partial" && cfg.Distribution.Extraction.Fulfillment != "allOrNothing" { + return fmt.Errorf("validate distribution: fulfillment must be partial or allOrNothing") + } + } + return nil +} + +func validateListener(name string, listener Listener, security Security) error { + if !listener.Enabled { + return nil + } + if listener.Listen == "" { + return fmt.Errorf("validate %s: listen is required", name) + } + host, _, err := net.SplitHostPort(listener.Listen) + if err != nil { + return fmt.Errorf("validate %s listen: %w", name, err) + } + if security.RequireProtectionOnPublicListen && isPublicHost(host) && listener.Auth.Mode == "none" && len(listener.Access.AllowCIDRs) == 0 { + return fmt.Errorf("validate %s: unprotected public listener is forbidden", name) + } + for _, cidr := range listener.Access.AllowCIDRs { + if _, _, err := net.ParseCIDR(cidr); err != nil { + return fmt.Errorf("validate %s allowCIDRs %q: %w", name, cidr, err) + } + } + return nil +} + +func validateUpstream(name string, upstream Upstream) error { + if !upstream.Enabled { + return nil + } + if upstream.Pool.MaxSize <= 0 { + return fmt.Errorf("validate upstream %q: pool.maxSize must be greater than zero", name) + } + if upstream.Fetch.MaxTotal > 0 && upstream.Fetch.MaxTotal < upstream.Pool.MaxSize { + return fmt.Errorf("validate upstream %q: fetch.maxTotal cannot be lower than pool.maxSize", name) + } + if upstream.Capacity.MaxConcurrencyPerProxy <= 0 { + return fmt.Errorf("validate upstream %q: maxConcurrencyPerProxy must be greater than zero", name) + } + if upstream.Lifecycle.TTL > 0 && upstream.Lifecycle.AllocationSafetyMargin >= upstream.Lifecycle.TTL { + return fmt.Errorf("validate upstream %q: allocationSafetyMargin must be lower than ttl", name) + } + if upstream.Fetch.RequestInterval < 0 || upstream.Fetch.MaxInFlight <= 0 || upstream.Fetch.MaxAttempts <= 0 { + return fmt.Errorf("validate upstream %q: fetch limits must be positive", name) + } + if upstream.API.URL != "" { + parsed, err := url.Parse(upstream.API.URL) + if err != nil || parsed.Scheme == "" || parsed.Host == "" { + return fmt.Errorf("validate upstream %q: api.url is invalid", name) + } + } + if err := validateProviderAuth(name, upstream.API.Auth); err != nil { + return err + } + if err := validateProxyAuth(name, upstream.ProxyAuth); err != nil { + return err + } + if len(upstream.Exposure) == 0 { + return fmt.Errorf("validate upstream %q: exposure is required", name) + } + for _, exposure := range upstream.Exposure { + if exposure != "gateway" && exposure != "extract" { + return fmt.Errorf("validate upstream %q: unsupported exposure %q", name, exposure) + } + } + return nil +} + +func validateProviderAuth(upstream string, auth ProviderAuth) error { + switch auth.Type { + case "", "none": + return nil + case "basic": + if auth.Username == "" || (auth.Password == "" && auth.PasswordFile == "") { + return fmt.Errorf("validate upstream %q: basic api.auth requires username and password", upstream) + } + case "bearer": + if auth.Token == "" && auth.TokenFile == "" { + return fmt.Errorf("validate upstream %q: bearer api.auth requires token", upstream) + } + case "apiKey": + if auth.Location != "header" && auth.Location != "query" { + return fmt.Errorf("validate upstream %q: apiKey location must be header or query", upstream) + } + if auth.Name == "" || (auth.Value == "" && auth.ValueFile == "") { + return fmt.Errorf("validate upstream %q: apiKey requires name and value", upstream) + } + default: + return fmt.Errorf("validate upstream %q: unsupported api.auth type %q", upstream, auth.Type) + } + return nil +} + +func validateProxyAuth(upstream string, auth ProxyAuth) error { + switch auth.Type { + case "response", "ipWhitelist": + return nil + case "static": + if auth.Username == "" || (auth.Password == "" && auth.PasswordFile == "") { + return fmt.Errorf("validate upstream %q: static proxyAuth requires username and password", upstream) + } + case "": + return fmt.Errorf("validate upstream %q: proxyAuth.type is required", upstream) + default: + return fmt.Errorf("validate upstream %q: unsupported proxyAuth type %q", upstream, auth.Type) + } + return nil +} + +func isPublicHost(host string) bool { + host = strings.Trim(host, "[]") + if host == "localhost" { + return false + } + ip := net.ParseIP(host) + if ip == nil { + return true + } + return !ip.IsLoopback() +} diff --git a/internal/domain/extraction/extraction.go b/internal/domain/extraction/extraction.go new file mode 100644 index 0000000..033790c --- /dev/null +++ b/internal/domain/extraction/extraction.go @@ -0,0 +1,149 @@ +package extraction + +import ( + "context" + "errors" + "sort" + "sync" + "time" +) + +type Fulfillment string + +const ( + Partial Fulfillment = "partial" + AllOrNothing Fulfillment = "allOrNothing" +) + +type State string + +const ( + Available State = "AVAILABLE" + Extracted State = "EXTRACTED" +) + +var ErrInsufficientProxies = errors.New("insufficient proxies") + +type Candidate struct { + ID string + Protocol string + Region string + Carrier string + Upstream string + URL string + State State + ExpiresAt time.Time + LastCheckedAt time.Time +} + +type Command struct { + Requested int + Fulfillment Fulfillment + Now time.Time + MinRemainingTTL time.Duration + MaxHealthCheckAge time.Duration + ReserveForGateway int + Protocols []string + Regions []string + Carriers []string + Upstreams []string +} + +type Record struct { + ProxyID string + ClientID string + SourceIP string + RequestID string + Upstream string + ExtractedAt time.Time + ExpiresAt time.Time +} + +type Result struct { + Requested int + Returned int + Items []Candidate +} + +type Store interface { + Extract(context.Context, Command) (Result, error) +} + +type MemoryStore struct { + mu sync.Mutex + candidates map[string]Candidate +} + +func NewMemoryStore(candidates []Candidate) *MemoryStore { + items := make(map[string]Candidate, len(candidates)) + for _, candidate := range candidates { + items[candidate.ID] = candidate + } + return &MemoryStore{candidates: items} +} + +func (s *MemoryStore) Extract(_ context.Context, command Command) (Result, error) { + s.mu.Lock() + defer s.mu.Unlock() + + result := Result{Requested: command.Requested} + if command.Requested <= 0 { + return result, nil + } + eligible := make([]Candidate, 0, len(s.candidates)) + for _, candidate := range s.candidates { + if eligibleForExtraction(candidate, command) { + eligible = append(eligible, candidate) + } + } + sort.Slice(eligible, func(i, j int) bool { + return eligible[i].ExpiresAt.After(eligible[j].ExpiresAt) + }) + available := len(eligible) - command.ReserveForGateway + if available < 0 { + available = 0 + } + if command.Fulfillment == AllOrNothing && available < command.Requested { + return result, ErrInsufficientProxies + } + count := command.Requested + if count > available { + count = available + } + for i := 0; i < count; i++ { + candidate := eligible[i] + candidate.State = Extracted + s.candidates[candidate.ID] = candidate + result.Items = append(result.Items, candidate) + } + result.Returned = len(result.Items) + return result, nil +} + +func eligibleForExtraction(candidate Candidate, command Command) bool { + if candidate.State != Available { + return false + } + if !candidate.ExpiresAt.IsZero() && candidate.ExpiresAt.Sub(command.Now) < command.MinRemainingTTL { + return false + } + if command.MaxHealthCheckAge > 0 && command.Now.Sub(candidate.LastCheckedAt) > command.MaxHealthCheckAge { + return false + } + return matches(command.Protocols, candidate.Protocol) && + matches(command.Regions, candidate.Region) && + matches(command.Carriers, candidate.Carrier) && + matches(command.Upstreams, candidate.Upstream) +} + +func matches(allowed []string, value string) bool { + if len(allowed) == 0 { + return true + } + for _, candidate := range allowed { + if candidate == value { + return true + } + } + return false +} diff --git a/internal/domain/extraction/extraction_test.go b/internal/domain/extraction/extraction_test.go new file mode 100644 index 0000000..8791d3d --- /dev/null +++ b/internal/domain/extraction/extraction_test.go @@ -0,0 +1,83 @@ +package extraction + +import ( + "context" + "sync" + "testing" + "time" +) + +func TestMemoryStoreNeverExtractsProxyTwice(t *testing.T) { + now := time.Date(2026, 7, 28, 10, 0, 0, 0, time.UTC) + store := NewMemoryStore([]Candidate{ + {ID: "p1", State: Available, ExpiresAt: now.Add(time.Minute), LastCheckedAt: now}, + }) + + var wg sync.WaitGroup + results := make(chan string, 1000) + for range 1000 { + wg.Add(1) + go func() { + defer wg.Done() + result, err := store.Extract(context.Background(), Command{ + Requested: 1, + Fulfillment: Partial, + Now: now, + MinRemainingTTL: 30 * time.Second, + MaxHealthCheckAge: 10 * time.Second, + }) + if err != nil { + t.Errorf("Extract(): %v", err) + return + } + for _, item := range result.Items { + results <- item.ID + } + }() + } + wg.Wait() + close(results) + + count := 0 + for id := range results { + if id != "p1" { + t.Fatalf("unexpected proxy %q", id) + } + count++ + } + if count != 1 { + t.Fatalf("proxy extracted %d times, want exactly once", count) + } +} + +func TestAllOrNothingDoesNotConsumePartialInventory(t *testing.T) { + now := time.Date(2026, 7, 28, 10, 0, 0, 0, time.UTC) + store := NewMemoryStore([]Candidate{ + {ID: "p1", State: Available, ExpiresAt: now.Add(time.Minute), LastCheckedAt: now}, + }) + + result, err := store.Extract(context.Background(), Command{ + Requested: 2, + Fulfillment: AllOrNothing, + Now: now, + MinRemainingTTL: 30 * time.Second, + MaxHealthCheckAge: 10 * time.Second, + }) + if err != ErrInsufficientProxies { + t.Fatalf("Extract() error = %v, want ErrInsufficientProxies", err) + } + if len(result.Items) != 0 { + t.Fatalf("Extract() returned %d items, want 0", len(result.Items)) + } + + partial, err := store.Extract(context.Background(), Command{ + Requested: 1, + Fulfillment: Partial, + Now: now, + MinRemainingTTL: 30 * time.Second, + MaxHealthCheckAge: 10 * time.Second, + }) + if err != nil || len(partial.Items) != 1 { + t.Fatalf("inventory was consumed by failed all-or-nothing: result=%+v err=%v", partial, err) + } +} diff --git a/internal/domain/proxy/capacity.go b/internal/domain/proxy/capacity.go new file mode 100644 index 0000000..3d93c70 --- /dev/null +++ b/internal/domain/proxy/capacity.go @@ -0,0 +1,134 @@ +package proxy + +import ( + "errors" + "sync/atomic" +) + +const counterMask = uint64(1<<32 - 1) + +var ( + ErrReservationCommitted = errors.New("reservation is already committed") + ErrReservationFinished = errors.New("reservation is already finished") +) + +type Capacity struct { + max atomic.Uint32 + counters atomic.Uint64 +} + +func NewCapacity(max int64) *Capacity { + capacity := &Capacity{} + if max < 0 || max > int64(counterMask) { + max = 0 + } + capacity.max.Store(uint32(max)) + return capacity +} + +func (c *Capacity) SetMax(max int64) bool { + if max < 0 || max > int64(counterMask) { + return false + } + c.max.Store(uint32(max)) + return true +} + +func (c *Capacity) Max() int64 { return int64(c.max.Load()) } + +func (c *Capacity) Reserve() (*Reservation, bool) { + for { + current := c.counters.Load() + active, reserved := unpack(current) + if active+reserved >= c.max.Load() { + return nil, false + } + next := pack(active, reserved+1) + if c.counters.CompareAndSwap(current, next) { + return &Reservation{capacity: c}, true + } + } +} + +func (c *Capacity) Active() int64 { + active, _ := unpack(c.counters.Load()) + return int64(active) +} + +func (c *Capacity) Reserved() int64 { + _, reserved := unpack(c.counters.Load()) + return int64(reserved) +} + +func (c *Capacity) commit() { + for { + current := c.counters.Load() + active, reserved := unpack(current) + if reserved == 0 { + return + } + if c.counters.CompareAndSwap(current, pack(active+1, reserved-1)) { + return + } + } +} + +func (c *Capacity) cancel() { + for { + current := c.counters.Load() + active, reserved := unpack(current) + if reserved == 0 || c.counters.CompareAndSwap(current, pack(active, reserved-1)) { + return + } + } +} + +func (c *Capacity) release() { + for { + current := c.counters.Load() + active, reserved := unpack(current) + if active == 0 || c.counters.CompareAndSwap(current, pack(active-1, reserved)) { + return + } + } +} + +func pack(active, reserved uint32) uint64 { + return uint64(reserved)<<32 | uint64(active) +} + +func unpack(value uint64) (active, reserved uint32) { + return uint32(value & counterMask), uint32(value >> 32) +} + +type Reservation struct { + capacity *Capacity + state atomic.Uint32 +} + +func (r *Reservation) Commit() error { + if !r.state.CompareAndSwap(0, 1) { + if r.state.Load() == 1 { + return ErrReservationCommitted + } + return ErrReservationFinished + } + r.capacity.commit() + return nil +} + +func (r *Reservation) Cancel() error { + if !r.state.CompareAndSwap(0, 2) { + return ErrReservationFinished + } + r.capacity.cancel() + return nil +} + +func (r *Reservation) Release() error { + if !r.state.CompareAndSwap(1, 2) { + return ErrReservationFinished + } + r.capacity.release() + return nil +} diff --git a/internal/domain/proxy/proxy.go b/internal/domain/proxy/proxy.go new file mode 100644 index 0000000..63d8454 --- /dev/null +++ b/internal/domain/proxy/proxy.go @@ -0,0 +1,78 @@ +package proxy + +import ( + "fmt" + "net" + "strconv" + "strings" + "time" +) + +type Scheme string + +const ( + SchemeHTTP Scheme = "http" + SchemeHTTPS Scheme = "https" + SchemeSOCKS5 Scheme = "socks5" +) + +type Proxy struct { + ID string + Scheme Scheme + Host string + Port uint16 + Username string + CredentialVersion string + SecretRef string + SourceUpstream string + CreatedAt time.Time + ExpiresAt *time.Time + LastCheckedAt *time.Time + LastSuccessAt *time.Time + Latency time.Duration + MaxConcurrency int64 + State State + Tags map[string]string +} + +func (p Proxy) UniqueKey() string { + host := strings.TrimSuffix(strings.ToLower(strings.TrimSpace(p.Host)), ".") + return strings.Join([]string{ + string(p.Scheme), + host, + strconv.FormatUint(uint64(p.Port), 10), + p.Username, + p.CredentialVersion, + }, "|") +} + +func (p Proxy) Address() string { + return net.JoinHostPort(p.Host, strconv.FormatUint(uint64(p.Port), 10)) +} + +func (p *Proxy) Transition(next State) error { + if p == nil { + return fmt.Errorf("transition proxy: nil proxy") + } + if !CanTransition(p.State, next) { + return fmt.Errorf("transition proxy: %s -> %s is not allowed", p.State, next) + } + p.State = next + return nil +} + +func EffectiveExpiry(now time.Time, expiresAt *time.Time, responseTTL, configuredTTL time.Duration) *time.Time { + if expiresAt != nil { + value := expiresAt.UTC() + return &value + } + if responseTTL > 0 { + value := now.UTC().Add(responseTTL) + return &value + } + if configuredTTL > 0 { + value := now.UTC().Add(configuredTTL) + return &value + } + return nil +} diff --git a/internal/domain/proxy/proxy_test.go b/internal/domain/proxy/proxy_test.go new file mode 100644 index 0000000..b0bee39 --- /dev/null +++ b/internal/domain/proxy/proxy_test.go @@ -0,0 +1,129 @@ +package proxy + +import ( + "sync" + "sync/atomic" + "testing" + "time" +) + +func TestUniqueKeyIncludesCredentialVersionButNotPassword(t *testing.T) { + p := Proxy{ + Scheme: SchemeHTTP, + Host: "EXAMPLE.COM", + Port: 8080, + Username: "alice", + CredentialVersion: "v2", + SecretRef: "secret-password", + } + + got := p.UniqueKey() + want := "http|example.com|8080|alice|v2" + if got != want { + t.Fatalf("UniqueKey() = %q, want %q", got, want) + } + if contains(got, p.SecretRef) { + t.Fatal("unique key leaked the proxy password reference") + } +} + +func TestEffectiveExpiryPrecedence(t *testing.T) { + now := time.Date(2026, 7, 28, 10, 0, 0, 0, time.UTC) + explicit := now.Add(90 * time.Second) + + tests := []struct { + name string + expiresAt *time.Time + response time.Duration + configured time.Duration + want *time.Time + }{ + {name: "explicit timestamp", expiresAt: &explicit, response: 2 * time.Minute, configured: 3 * time.Minute, want: &explicit}, + {name: "response ttl", response: 2 * time.Minute, configured: 3 * time.Minute, want: timePtr(now.Add(2 * time.Minute))}, + {name: "configured ttl", configured: 3 * time.Minute, want: timePtr(now.Add(3 * time.Minute))}, + {name: "non expiring"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := EffectiveExpiry(now, tt.expiresAt, tt.response, tt.configured) + if !equalTimePtr(got, tt.want) { + t.Fatalf("EffectiveExpiry() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestStateMachineRejectsIllegalTransition(t *testing.T) { + p := Proxy{State: StateFetched} + if err := p.Transition(StateChecking); err != nil { + t.Fatalf("FETCHED -> CHECKING: %v", err) + } + if err := p.Transition(StateAvailable); err != nil { + t.Fatalf("CHECKING -> AVAILABLE: %v", err) + } + if err := p.Transition(StateFetched); err == nil { + t.Fatal("AVAILABLE -> FETCHED must be rejected") + } +} + +func TestCapacityNeverOversubscribes(t *testing.T) { + capacity := NewCapacity(8) + var acquired atomic.Int64 + var peak atomic.Int64 + var wg sync.WaitGroup + + for range 1000 { + wg.Add(1) + go func() { + defer wg.Done() + reservation, ok := capacity.Reserve() + if !ok { + return + } + active := acquired.Add(1) + for { + old := peak.Load() + if active <= old || peak.CompareAndSwap(old, active) { + break + } + } + if err := reservation.Commit(); err != nil { + t.Errorf("Commit(): %v", err) + } + acquired.Add(-1) + if err := reservation.Release(); err != nil { + t.Errorf("Release(): %v", err) + } + }() + } + wg.Wait() + + if peak.Load() > 8 { + t.Fatalf("peak reservations = %d, exceeds 8", peak.Load()) + } + if got := capacity.Active(); got != 0 { + t.Fatalf("active = %d, want 0", got) + } + if got := capacity.Reserved(); got != 0 { + t.Fatalf("reserved = %d, want 0", got) + } +} + +func contains(s, part string) bool { + for i := 0; i+len(part) <= len(s); i++ { + if s[i:i+len(part)] == part { + return true + } + } + return false +} + +func timePtr(value time.Time) *time.Time { return &value } + +func equalTimePtr(a, b *time.Time) bool { + if a == nil || b == nil { + return a == b + } + return a.Equal(*b) +} diff --git a/internal/domain/proxy/state.go b/internal/domain/proxy/state.go new file mode 100644 index 0000000..3a1a694 --- /dev/null +++ b/internal/domain/proxy/state.go @@ -0,0 +1,40 @@ +package proxy + +type State string + +const ( + StateFetched State = "FETCHED" + StateChecking State = "CHECKING" + StateAvailable State = "AVAILABLE" + StateSuspect State = "SUSPECT" + StateDraining State = "DRAINING" + StateUnhealthy State = "UNHEALTHY" + StateExtracted State = "EXTRACTED" + StateExpired State = "EXPIRED" + StateRemoved State = "REMOVED" +) + +var transitions = map[State]map[State]struct{}{ + StateFetched: set(StateChecking, StateExpired, StateRemoved), + StateChecking: set(StateAvailable, StateUnhealthy, StateExpired, StateRemoved), + StateAvailable: set(StateSuspect, StateDraining, StateExtracted, StateExpired), + StateSuspect: set(StateAvailable, StateUnhealthy, StateDraining, StateExpired), + StateDraining: set(StateExpired, StateUnhealthy, StateRemoved), + StateUnhealthy: set(StateChecking, StateRemoved, StateExpired), + StateExtracted: set(StateExpired, StateRemoved), + StateExpired: set(StateRemoved), + StateRemoved: {}, +} + +func CanTransition(current, next State) bool { + _, ok := transitions[current][next] + return ok +} + +func set(states ...State) map[State]struct{} { + result := make(map[State]struct{}, len(states)) + for _, state := range states { + result[state] = struct{}{} + } + return result +} diff --git a/internal/domain/routing/routing_test.go b/internal/domain/routing/routing_test.go new file mode 100644 index 0000000..9459757 --- /dev/null +++ b/internal/domain/routing/routing_test.go @@ -0,0 +1,62 @@ +package routing + +import ( + "sync" + "testing" +) + +func TestRuleSetUsesFirstMatchingRule(t *testing.T) { + rules, err := Compile([]Rule{ + {Name: "specific", Match: Match{HostRegex: `(^|\.)jd\.com$`}, Upstreams: []string{"jd"}}, + {Name: "default", Match: Match{HostRegex: `.*`}, Action: ActionReject}, + }) + if err != nil { + t.Fatalf("Compile(): %v", err) + } + + got, ok := rules.Match(Request{Host: "api.jd.com", Method: "GET", Path: "/"}) + if !ok || got.Name != "specific" { + t.Fatalf("Match() = %q, %v; want specific, true", got.Name, ok) + } +} + +func TestSequentialSwitchesOnceAtThreshold(t *testing.T) { + sequence, err := NewSequential([]string{"a", "b", "c"}, 5) + if err != nil { + t.Fatalf("NewSequential(): %v", err) + } + for range 4 { + sequence.ObserveEmpty("a") + } + if got := sequence.Current(); got != "a" { + t.Fatalf("Current() = %q before threshold, want a", got) + } + + var wg sync.WaitGroup + for range 100 { + wg.Add(1) + go func() { + defer wg.Done() + sequence.ObserveEmpty("a") + }() + } + wg.Wait() + + if got := sequence.Current(); got != "b" { + t.Fatalf("Current() = %q after concurrent threshold, want b", got) + } +} + +func TestSequentialValidFetchResetsEmptyCount(t *testing.T) { + sequence, _ := NewSequential([]string{"a", "b"}, 5) + for range 4 { + sequence.ObserveEmpty("a") + } + sequence.ObserveValid("a") + for range 4 { + sequence.ObserveEmpty("a") + } + if got := sequence.Current(); got != "a" { + t.Fatalf("Current() = %q, want a after reset", got) + } +} diff --git a/internal/domain/routing/rule.go b/internal/domain/routing/rule.go new file mode 100644 index 0000000..5561739 --- /dev/null +++ b/internal/domain/routing/rule.go @@ -0,0 +1,118 @@ +package routing + +import ( + "fmt" + "regexp" + "slices" + "strings" +) + +type Action string + +const ( + ActionProxy Action = "proxy" + ActionDirect Action = "direct" + ActionReject Action = "reject" +) + +type Match struct { + HostRegex string + Methods []string + PathRegex string + Headers map[string]string +} + +type Rule struct { + Name string + Match Match + Upstreams []string + Action Action +} + +type Request struct { + Host string + Method string + Path string + Headers map[string]string +} + +type compiledRule struct { + rule Rule + host *regexp.Regexp + path *regexp.Regexp +} + +type RuleSet struct { + rules []compiledRule +} + +func Compile(rules []Rule) (*RuleSet, error) { + compiled := make([]compiledRule, 0, len(rules)) + for _, rule := range rules { + if rule.Name == "" { + return nil, fmt.Errorf("compile routing: rule name is required") + } + host, err := regexp.Compile(rule.Match.HostRegex) + if err != nil { + return nil, fmt.Errorf("compile routing %q host: %w", rule.Name, err) + } + var path *regexp.Regexp + if rule.Match.PathRegex != "" { + path, err = regexp.Compile(rule.Match.PathRegex) + if err != nil { + return nil, fmt.Errorf("compile routing %q path: %w", rule.Name, err) + } + } + if rule.Action == "" && len(rule.Upstreams) > 0 { + rule.Action = ActionProxy + } + compiled = append(compiled, compiledRule{rule: rule, host: host, path: path}) + } + return &RuleSet{rules: compiled}, nil +} + +func (r *RuleSet) Match(request Request) (Rule, bool) { + if r == nil { + return Rule{}, false + } + host := strings.ToLower(strings.TrimSuffix(request.Host, ".")) + method := strings.ToUpper(request.Method) + for _, candidate := range r.rules { + if !candidate.host.MatchString(host) { + continue + } + if len(candidate.rule.Match.Methods) > 0 && !containsFold(candidate.rule.Match.Methods, method) { + continue + } + if candidate.path != nil && !candidate.path.MatchString(request.Path) { + continue + } + if !headersMatch(candidate.rule.Match.Headers, request.Headers) { + continue + } + return candidate.rule, true + } + return Rule{}, false +} + +func containsFold(values []string, target string) bool { + return slices.ContainsFunc(values, func(value string) bool { + return strings.EqualFold(value, target) + }) +} + +func headersMatch(expected, actual map[string]string) bool { + for name, value := range expected { + matched := false + for actualName, actualValue := range actual { + if strings.EqualFold(name, actualName) && actualValue == value { + matched = true + break + } + } + if !matched { + return false + } + } + return true +} diff --git a/internal/domain/routing/sequential.go b/internal/domain/routing/sequential.go new file mode 100644 index 0000000..64ed648 --- /dev/null +++ b/internal/domain/routing/sequential.go @@ -0,0 +1,61 @@ +package routing + +import ( + "fmt" + "sync" +) + +type Sequential struct { + mu sync.RWMutex + upstreams []string + threshold int + current int + empty map[string]int +} + +func NewSequential(upstreams []string, threshold int) (*Sequential, error) { + if len(upstreams) == 0 { + return nil, fmt.Errorf("sequential strategy requires at least one upstream") + } + if threshold <= 0 { + return nil, fmt.Errorf("sequential threshold must be greater than zero") + } + return &Sequential{ + upstreams: append([]string(nil), upstreams...), + threshold: threshold, + empty: make(map[string]int, len(upstreams)), + }, nil +} + +func (s *Sequential) Current() string { + s.mu.RLock() + defer s.mu.RUnlock() + return s.upstreams[s.current] +} + +func (s *Sequential) ObserveEmpty(upstream string) bool { + s.mu.Lock() + defer s.mu.Unlock() + + s.empty[upstream]++ + if s.upstreams[s.current] != upstream || s.empty[upstream] < s.threshold { + return false + } + if s.current+1 >= len(s.upstreams) { + return false + } + s.current++ + return true +} + +func (s *Sequential) ObserveValid(upstream string) { + s.mu.Lock() + defer s.mu.Unlock() + s.empty[upstream] = 0 +} + +func (s *Sequential) EmptyCount(upstream string) int { + s.mu.RLock() + defer s.mu.RUnlock() + return s.empty[upstream] +} diff --git a/internal/domain/upstream/fetch_result.go b/internal/domain/upstream/fetch_result.go new file mode 100644 index 0000000..0187f54 --- /dev/null +++ b/internal/domain/upstream/fetch_result.go @@ -0,0 +1,23 @@ +package upstream + +type FetchClass string + +const ( + FetchValid FetchClass = "valid" + FetchEmpty FetchClass = "empty" + FetchDuplicateOnly FetchClass = "duplicate_only" + FetchError FetchClass = "error" +) + +func ClassifyFetchResult(callErr, parseErr error, validCount, newCount int) FetchClass { + if callErr != nil || parseErr != nil { + return FetchError + } + if validCount <= 0 { + return FetchEmpty + } + if newCount <= 0 { + return FetchDuplicateOnly + } + return FetchValid +} diff --git a/internal/domain/upstream/fetch_result_test.go b/internal/domain/upstream/fetch_result_test.go new file mode 100644 index 0000000..7488607 --- /dev/null +++ b/internal/domain/upstream/fetch_result_test.go @@ -0,0 +1,35 @@ +package upstream + +import "testing" + +func TestClassifyFetchResult(t *testing.T) { + tests := []struct { + name string + callErr error + parseErr error + valid int + newCount int + want FetchClass + }{ + {name: "network error", callErr: errFixture, want: FetchError}, + {name: "parse error", parseErr: errFixture, want: FetchError}, + {name: "empty response", want: FetchEmpty}, + {name: "duplicates are not empty", valid: 3, want: FetchDuplicateOnly}, + {name: "new proxy", valid: 3, newCount: 1, want: FetchValid}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := ClassifyFetchResult(tt.callErr, tt.parseErr, tt.valid, tt.newCount) + if got != tt.want { + t.Fatalf("ClassifyFetchResult() = %q, want %q", got, tt.want) + } + }) + } +} + +type fixtureError string + +func (e fixtureError) Error() string { return string(e) } + +const errFixture = fixtureError("fixture") diff --git a/internal/gateway/dispatch/dispatcher.go b/internal/gateway/dispatch/dispatcher.go new file mode 100644 index 0000000..af0674f --- /dev/null +++ b/internal/gateway/dispatch/dispatcher.go @@ -0,0 +1,109 @@ +package dispatch + +import ( + "errors" + "sync/atomic" + "time" + + proxyDomain "github.com/proxy-pool/proxy-pool/internal/domain/proxy" + "github.com/proxy-pool/proxy-pool/internal/gateway/snapshot" +) + +var ErrNoCandidate = errors.New("no local proxy candidate is available") + +type Request struct { + Now time.Time + Scheme proxyDomain.Scheme + Upstreams []string + RequiredTags map[string]string + Exclude map[string]struct{} + SafetyMargin time.Duration +} + +type Lease struct { + Proxy proxyDomain.Proxy + Epoch uint64 + Version uint64 + reserved *proxyDomain.Reservation +} + +func (l *Lease) Commit() error { return l.reserved.Commit() } +func (l *Lease) Cancel() error { return l.reserved.Cancel() } +func (l *Lease) Release() error { return l.reserved.Release() } + +type Dispatcher struct { + store *snapshot.Store + cursor atomic.Uint64 +} + +func New(store *snapshot.Store) *Dispatcher { + return &Dispatcher{store: store} +} + +func (d *Dispatcher) Acquire(request Request) (*Lease, error) { + if d == nil || d.store == nil { + return nil, ErrNoCandidate + } + view := d.store.Current() + if view == nil || len(view.Entries) == 0 { + return nil, ErrNoCandidate + } + if request.Now.IsZero() { + request.Now = time.Now().UTC() + } + + start := int((d.cursor.Add(1) - 1) % uint64(len(view.Entries))) + for offset := 0; offset < len(view.Entries); offset++ { + entry := view.Entries[(start+offset)%len(view.Entries)] + if !eligible(entry.Proxy, request) { + continue + } + reservation, ok := entry.Runtime.Reserve() + if !ok { + continue + } + return &Lease{ + Proxy: entry.Proxy, + Epoch: view.Epoch, + Version: view.Version, + reserved: reservation, + }, nil + } + return nil, ErrNoCandidate +} + +func eligible(candidate proxyDomain.Proxy, request Request) bool { + if candidate.State != proxyDomain.StateAvailable { + return false + } + if request.Scheme != "" && candidate.Scheme != request.Scheme { + return false + } + if _, excluded := request.Exclude[candidate.ID]; excluded { + return false + } + if candidate.ExpiresAt != nil && !candidate.ExpiresAt.After(request.Now.Add(request.SafetyMargin)) { + return false + } + if !contains(request.Upstreams, candidate.SourceUpstream) { + return false + } + for key, value := range request.RequiredTags { + if candidate.Tags[key] != value { + return false + } + } + return true +} + +func contains(allowed []string, value string) bool { + if len(allowed) == 0 { + return true + } + for _, candidate := range allowed { + if candidate == value { + return true + } + } + return false +} diff --git a/internal/gateway/dispatch/dispatcher_test.go b/internal/gateway/dispatch/dispatcher_test.go new file mode 100644 index 0000000..dd05377 --- /dev/null +++ b/internal/gateway/dispatch/dispatcher_test.go @@ -0,0 +1,90 @@ +package dispatch + +import ( + "errors" + "sync" + "testing" + "time" + + proxyDomain "github.com/proxy-pool/proxy-pool/internal/domain/proxy" + "github.com/proxy-pool/proxy-pool/internal/gateway/snapshot" +) + +func TestAcquireFiltersAndReservesLocalCapacity(t *testing.T) { + now := time.Date(2026, 7, 28, 12, 0, 0, 0, time.UTC) + expiresSoon := now.Add(5 * time.Second) + expiresLater := now.Add(time.Minute) + store := snapshot.NewStore("cluster-a", "worker-a") + proxies := []proxyDomain.Proxy{ + {ID: "wrong-upstream", Scheme: proxyDomain.SchemeHTTP, SourceUpstream: "b", State: proxyDomain.StateAvailable, MaxConcurrency: 1, ExpiresAt: &expiresLater}, + {ID: "expiring", Scheme: proxyDomain.SchemeHTTP, SourceUpstream: "a", State: proxyDomain.StateAvailable, MaxConcurrency: 1, ExpiresAt: &expiresSoon}, + {ID: "selected", Scheme: proxyDomain.SchemeHTTP, SourceUpstream: "a", State: proxyDomain.StateAvailable, MaxConcurrency: 1, ExpiresAt: &expiresLater, Tags: map[string]string{"region": "cn-east"}}, + } + envelope := snapshot.Envelope{ClusterID: "cluster-a", WorkerID: "worker-a", Epoch: 1, Version: 1, Full: true, Proxies: proxies} + envelope.Checksum = snapshot.Checksum(proxies) + if err := store.Apply(envelope); err != nil { + t.Fatalf("Apply(): %v", err) + } + + dispatcher := New(store) + lease, err := dispatcher.Acquire(Request{ + Now: now, + Scheme: proxyDomain.SchemeHTTP, + Upstreams: []string{"a"}, + RequiredTags: map[string]string{"region": "cn-east"}, + SafetyMargin: 10 * time.Second, + }) + if err != nil { + t.Fatalf("Acquire(): %v", err) + } + if lease.Proxy.ID != "selected" { + t.Fatalf("selected proxy = %q, want selected", lease.Proxy.ID) + } + if err := lease.Commit(); err != nil { + t.Fatalf("Commit(): %v", err) + } + if err := lease.Release(); err != nil { + t.Fatalf("Release(): %v", err) + } +} + +func TestAcquireNeverOversubscribesSnapshotProxy(t *testing.T) { + store := snapshot.NewStore("cluster-a", "worker-a") + proxies := []proxyDomain.Proxy{{ID: "p1", Scheme: proxyDomain.SchemeHTTP, State: proxyDomain.StateAvailable, MaxConcurrency: 8}} + envelope := snapshot.Envelope{ClusterID: "cluster-a", WorkerID: "worker-a", Epoch: 1, Version: 1, Full: true, Proxies: proxies} + envelope.Checksum = snapshot.Checksum(proxies) + if err := store.Apply(envelope); err != nil { + t.Fatalf("Apply(): %v", err) + } + dispatcher := New(store) + + var wg sync.WaitGroup + leases := make(chan *Lease, 1000) + for range 1000 { + wg.Add(1) + go func() { + defer wg.Done() + lease, err := dispatcher.Acquire(Request{Now: time.Now(), Scheme: proxyDomain.SchemeHTTP}) + if err == nil { + leases <- lease + return + } + if !errors.Is(err, ErrNoCandidate) { + t.Errorf("Acquire(): %v", err) + } + }() + } + wg.Wait() + close(leases) + + count := 0 + for lease := range leases { + count++ + if err := lease.Cancel(); err != nil { + t.Errorf("Cancel(): %v", err) + } + } + if count != 8 { + t.Fatalf("reserved = %d, want 8", count) + } +} diff --git a/internal/gateway/snapshot/store.go b/internal/gateway/snapshot/store.go new file mode 100644 index 0000000..54dc94b --- /dev/null +++ b/internal/gateway/snapshot/store.go @@ -0,0 +1,149 @@ +package snapshot + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "sort" + "sync" + "sync/atomic" + + proxyDomain "github.com/proxy-pool/proxy-pool/internal/domain/proxy" +) + +var ( + ErrWrongTarget = errors.New("snapshot targets another cluster or worker") + ErrResyncRequired = errors.New("snapshot sequence requires a full resync") + ErrChecksumMismatch = errors.New("snapshot checksum mismatch") +) + +type Envelope struct { + ClusterID string + WorkerID string + Epoch uint64 + Version uint64 + Full bool + Checksum string + Proxies []proxyDomain.Proxy +} + +type Entry struct { + Proxy proxyDomain.Proxy + Runtime *proxyDomain.Capacity +} + +type View struct { + ClusterID string + WorkerID string + Epoch uint64 + Version uint64 + Checksum string + Entries []Entry +} + +type Store struct { + clusterID string + workerID string + current atomic.Pointer[View] + + mu sync.Mutex + runtimes map[string]*proxyDomain.Capacity +} + +func NewStore(clusterID, workerID string) *Store { + return &Store{ + clusterID: clusterID, + workerID: workerID, + runtimes: make(map[string]*proxyDomain.Capacity), + } +} + +func (s *Store) Current() *View { + if s == nil { + return nil + } + return s.current.Load() +} + +func (s *Store) Apply(envelope Envelope) error { + if s == nil { + return fmt.Errorf("apply snapshot: nil store") + } + if envelope.ClusterID != s.clusterID || envelope.WorkerID != s.workerID { + return ErrWrongTarget + } + if !envelope.Full || envelope.Epoch == 0 || envelope.Version == 0 { + return ErrResyncRequired + } + if envelope.Checksum != Checksum(envelope.Proxies) { + return ErrChecksumMismatch + } + + s.mu.Lock() + defer s.mu.Unlock() + + current := s.current.Load() + if current != nil { + switch { + case envelope.Epoch < current.Epoch: + return ErrResyncRequired + case envelope.Epoch == current.Epoch && envelope.Version != current.Version+1: + return ErrResyncRequired + case envelope.Epoch > current.Epoch && envelope.Version != 1: + return ErrResyncRequired + } + } + + proxies := cloneAndSort(envelope.Proxies) + entries := make([]Entry, 0, len(proxies)) + for _, descriptor := range proxies { + runtime := s.runtimes[descriptor.ID] + if runtime == nil { + runtime = proxyDomain.NewCapacity(descriptor.MaxConcurrency) + s.runtimes[descriptor.ID] = runtime + } else { + runtime.SetMax(descriptor.MaxConcurrency) + } + entries = append(entries, Entry{Proxy: descriptor, Runtime: runtime}) + } + + next := &View{ + ClusterID: envelope.ClusterID, + WorkerID: envelope.WorkerID, + Epoch: envelope.Epoch, + Version: envelope.Version, + Checksum: envelope.Checksum, + Entries: entries, + } + s.current.Store(next) + return nil +} + +func Checksum(proxies []proxyDomain.Proxy) string { + canonical := cloneAndSort(proxies) + encoded, err := json.Marshal(canonical) + if err != nil { + panic(fmt.Sprintf("encode snapshot checksum: %v", err)) + } + digest := sha256.Sum256(encoded) + return hex.EncodeToString(digest[:]) +} + +func cloneAndSort(source []proxyDomain.Proxy) []proxyDomain.Proxy { + cloned := make([]proxyDomain.Proxy, len(source)) + for index, descriptor := range source { + cloned[index] = descriptor + if descriptor.Tags != nil { + cloned[index].Tags = make(map[string]string, len(descriptor.Tags)) + for key, value := range descriptor.Tags { + cloned[index].Tags[key] = value + } + } + } + sort.Slice(cloned, func(i, j int) bool { + return cloned[i].ID < cloned[j].ID + }) + return cloned +} diff --git a/internal/gateway/snapshot/store_test.go b/internal/gateway/snapshot/store_test.go new file mode 100644 index 0000000..cdd120f --- /dev/null +++ b/internal/gateway/snapshot/store_test.go @@ -0,0 +1,80 @@ +package snapshot + +import ( + "errors" + "testing" + + proxyDomain "github.com/proxy-pool/proxy-pool/internal/domain/proxy" +) + +func TestStoreAppliesCompleteSnapshotsInOrder(t *testing.T) { + store := NewStore("cluster-a", "worker-a") + first := Envelope{ + ClusterID: "cluster-a", + WorkerID: "worker-a", + Epoch: 1, + Version: 1, + Full: true, + Proxies: []proxyDomain.Proxy{{ + ID: "p1", + Scheme: proxyDomain.SchemeHTTP, + Host: "127.0.0.1", + Port: 18080, + MaxConcurrency: 2, + State: proxyDomain.StateAvailable, + }}, + } + first.Checksum = Checksum(first.Proxies) + if err := store.Apply(first); err != nil { + t.Fatalf("Apply(first): %v", err) + } + + view := store.Current() + if view == nil || view.Version != 1 || len(view.Entries) != 1 { + t.Fatalf("Current() = %+v", view) + } + if view.Entries[0].Runtime == nil { + t.Fatal("snapshot entry has no local runtime capacity") + } + + second := first + second.Version = 2 + second.Proxies = append([]proxyDomain.Proxy(nil), first.Proxies...) + second.Proxies[0].Host = "localhost" + second.Checksum = Checksum(second.Proxies) + if err := store.Apply(second); err != nil { + t.Fatalf("Apply(second): %v", err) + } + if got := store.Current().Entries[0].Proxy.Host; got != "localhost" { + t.Fatalf("host = %q, want localhost", got) + } +} + +func TestStoreRejectsWrongWorkerVersionGapAndChecksum(t *testing.T) { + store := NewStore("cluster-a", "worker-a") + base := Envelope{ClusterID: "cluster-a", WorkerID: "worker-a", Epoch: 1, Version: 1, Full: true} + base.Checksum = Checksum(base.Proxies) + if err := store.Apply(base); err != nil { + t.Fatalf("Apply(base): %v", err) + } + + wrongWorker := base + wrongWorker.Version = 2 + wrongWorker.WorkerID = "worker-b" + if err := store.Apply(wrongWorker); !errors.Is(err, ErrWrongTarget) { + t.Fatalf("wrong worker error = %v, want ErrWrongTarget", err) + } + + gap := base + gap.Version = 3 + if err := store.Apply(gap); !errors.Is(err, ErrResyncRequired) { + t.Fatalf("version gap error = %v, want ErrResyncRequired", err) + } + + badChecksum := base + badChecksum.Version = 2 + badChecksum.Checksum = "bad" + if err := store.Apply(badChecksum); !errors.Is(err, ErrChecksumMismatch) { + t.Fatalf("checksum error = %v, want ErrChecksumMismatch", err) + } +} diff --git a/progress.md b/progress.md index 618fad4..3aa707b 100644 --- a/progress.md +++ b/progress.md @@ -7,5 +7,17 @@ - 已按主题定位配置定稿、实施方案、Distribution API、认证、安全、并发、 故障语义和最终 Exclusive Extraction 修订。 - 已建立新的任务计划与事实记录，旧网页摘要不再作为需求证据。 -- 尚未创建实现代码。 - +- 已建立需求追踪矩阵、统一领域语言、产品设计、总体架构、项目结构、ADR、 + 开发、配置、API、安全、测试与运维文档。 +- 已提供 20 个严格校验的配置示例、Distribution/Admin OpenAPI、Controller/ + Worker/Checker Protobuf 契约和 35 张 Mermaid 图。 +- 已实现并测试 Proxy 状态/TTL/唯一键、打包原子容量、首条路由、Sequential + 并发切换、Fetch 分类、一次性独占提取、严格配置、Snapshot 与本地 Dispatch。 +- `go test ./...`、`go vet ./...`、`go build ./...` 通过。 +- Protobuf 描述符编译、Compose 静态展开、Kustomize 渲染、Grafana JSON 与 + 配置示例校验通过。 +- Windows 当前 `CGO_ENABLED=0` 且无 C 编译器，race 测试由 Linux CI 承担。 +- 100,000 QPS 仍是未验证设计目标；运行进程、存储适配器、完整网络转发与 + 代表性集群压测尚未实施，已在完成审计中明确列出。 +- 已生成 `proxy-pool-docs-v1.0.zip`，包含 50 个条目，SHA-256 为 + `A6882B71196210CE3594A10992C2A0A73EA3A1CF31D8A570709CA999133CE104`。 diff --git a/proxy-pool-docs-v1.0.zip b/proxy-pool-docs-v1.0.zip new file mode 100644 index 0000000..2e0c95f Binary files /dev/null and b/proxy-pool-docs-v1.0.zip differ diff --git a/scripts/verify.ps1 b/scripts/verify.ps1 new file mode 100644 index 0000000..f342890 --- /dev/null +++ b/scripts/verify.ps1 @@ -0,0 +1,31 @@ +$ErrorActionPreference = "Stop" + +function Invoke-Step { + param( + [string]$Name, + [scriptblock]$Command + ) + + Write-Host "==> $Name" + & $Command + if ($LASTEXITCODE -ne 0) { + throw "$Name failed with exit code $LASTEXITCODE" + } +} + +$unformatted = gofmt -l . +if ($unformatted) { + $unformatted | ForEach-Object { Write-Host $_ } + throw "gofmt check failed" +} + +Invoke-Step "go vet" { go vet ./... } +Invoke-Step "unit tests" { go test -timeout 60s ./... } + +if ((go env CGO_ENABLED) -eq "1") { + Invoke-Step "race tests" { go test -race -timeout 60s ./internal/... } +} else { + Write-Host "==> race tests skipped: CGO_ENABLED is not 1" +} + +Invoke-Step "build" { go build ./... } diff --git a/task_plan.md b/task_plan.md index e831991..95924a6 100644 --- a/task_plan.md +++ b/task_plan.md @@ -15,13 +15,15 @@ ## 阶段 1. [已完成] 完整读取对话并识别覆盖关系 -2. [进行中] 建立需求追踪矩阵与统一领域模型 -3. [待开始] 编写总体设计、详细设计和 ADR -4. [待开始] 编写开发、配置、API、测试和运维文档 -5. [待开始] 搭建 Go 模块、命令、核心包、契约和部署目录 -6. [待开始] 实现核心状态机、路由、容量、提取与配置校验 -7. [待开始] 执行单元测试、竞态检查、静态检查和构建 -8. [待开始] 按需求矩阵逐项审计并打包交付 +2. [已完成] 建立需求追踪矩阵与统一领域模型 +3. [已完成] 编写总体设计、产品设计、项目结构和 ADR +4. [已完成] 编写开发、配置、API、测试、安全和运维文档 +5. [已完成] 搭建 Go 模块、核心包、契约和部署拓扑目录 +6. [已完成] 实现状态机、首条路由、Sequential、原子容量、独占提取、 + Fetch 分类、严格配置、不可变快照和本地调度参考实现 +7. [已完成] 执行单元测试、静态检查、构建和静态部署/契约验证；本机因 + `CGO_ENABLED=0` 且无 C 编译器未运行 race，保留给 Linux CI +8. [已完成] 按需求矩阵逐项审计并生成版本化文档包 ## 串并行关系 @@ -40,6 +42,7 @@ ## 已知环境限制 -- Docker CLI 已安装，但 Linux daemon 状态需在集成验证前再次确认。 -- 当前仓库尚无提交；`对话内容.md` 和 `.gitignore` 为现有文件。 - +- Docker Compose 配置与 Kubernetes Kustomize 已完成静态渲染验证；未启动 + 目标运行拓扑。 +- `cmd/proxy-*`、Provider 调度、PostgreSQL/Redis 适配器、Gateway Transport + 与 Checker 运行时属于后续实施范围，见完成审计。