feat: add proxy pool design and core architecture
This commit is contained in:
parent
dab16fda12
commit
7ce9778bdf
34
.github/workflows/ci.yml
vendored
Normal file
34
.github/workflows/ci.yml
vendored
Normal file
@ -0,0 +1,34 @@
|
||||
name: ci
|
||||
|
||||
on:
|
||||
push:
|
||||
pull_request:
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
test:
|
||||
strategy:
|
||||
matrix:
|
||||
os: [ubuntu-latest, windows-latest]
|
||||
runs-on: ${{ matrix.os }}
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/setup-go@v5
|
||||
with:
|
||||
go-version-file: go.mod
|
||||
cache: true
|
||||
- run: go vet ./...
|
||||
- run: go test -timeout 60s ./...
|
||||
- run: go build ./...
|
||||
|
||||
race:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/setup-go@v5
|
||||
with:
|
||||
go-version-file: go.mod
|
||||
cache: true
|
||||
- run: go test -race -timeout 60s ./internal/...
|
||||
1
.gitignore
vendored
1
.gitignore
vendored
@ -9,6 +9,7 @@ coverage/
|
||||
*.out
|
||||
*.test
|
||||
*.prof
|
||||
.tmp-proto/
|
||||
|
||||
# Local configuration and secrets
|
||||
.env
|
||||
|
||||
57
README.md
Normal file
57
README.md
Normal file
@ -0,0 +1,57 @@
|
||||
# Proxy Pool
|
||||
|
||||
面向多供应商代理资源的集中管理与高并发转发平台。系统同时提供:
|
||||
|
||||
- **Gateway**:系统选择上游代理并代转发 HTTP 与 HTTPS CONNECT。
|
||||
- **Distribution API**:把真实代理一次性、独占地发放给调用方。
|
||||
- **Admin API**:查询、启停、切换和配置重载。
|
||||
- **Controller / Checker**:管理供应商获取、健康、容量、状态与数据面快照。
|
||||
|
||||
> 当前仓库交付的是从 `对话内容.md` 全量重建的设计基线、机器契约、
|
||||
> 项目骨架和关键并发领域实现。100,000 QPS 是集群设计目标,尚需在目标
|
||||
> 网络和代理规模下完成压测证明。
|
||||
|
||||
## 快速导航
|
||||
|
||||
- [产品设计](docs/design/product-design.md)
|
||||
- [总体架构](docs/design/architecture.md)
|
||||
- [项目结构](docs/design/project-structure.md)
|
||||
- [需求追踪](docs/requirements/traceability.md)
|
||||
- [交付完成度审计](docs/requirements/completion-audit.md)
|
||||
- [开发指南](docs/development/guide.md)
|
||||
- [实施计划](docs/development/implementation-plan.md)
|
||||
- [配置参考](docs/configuration/reference.md)
|
||||
- [Distribution API](docs/api/distribution.md)
|
||||
- [控制面协议](docs/api/control-plane.md)
|
||||
- [安全模型](docs/security/security-model.md)
|
||||
- [测试策略](docs/testing/strategy.md)
|
||||
- [运维手册](docs/operations/runbook.md)
|
||||
- [生产就绪检查](docs/operations/production-readiness.md)
|
||||
|
||||
## 本地验证
|
||||
|
||||
```powershell
|
||||
go mod tidy
|
||||
go test ./...
|
||||
go vet ./...
|
||||
go build ./...
|
||||
```
|
||||
|
||||
Windows PowerShell 可运行:
|
||||
|
||||
```powershell
|
||||
./scripts/verify.ps1
|
||||
```
|
||||
|
||||
竞态检测需要启用 CGO 并提供可用的 C 编译器;CI 的 Linux race job 负责
|
||||
执行该质量门禁。
|
||||
|
||||
## 不变量
|
||||
|
||||
1. Gateway 热路径不访问 PostgreSQL、Redis 或 Provider API。
|
||||
2. Proxy 容量使用 `Reserved -> Active` 原子转换,禁止超卖。
|
||||
3. Distribution 成功时原子执行 `AVAILABLE -> EXTRACTED`,不提供 Lease、
|
||||
Release 或 Renewal。
|
||||
4. `pool.maxSize` 是当前未提取库存上限;`fetch.maxTotal` 是累计获取额度。
|
||||
5. CONNECT 向客户端提交 200 后不透明重放。
|
||||
6. 公开监听必须有认证或 CIDR 访问保护。
|
||||
202
api/openapi/admin.yaml
Normal file
202
api/openapi/admin.yaml
Normal file
@ -0,0 +1,202 @@
|
||||
openapi: 3.1.0
|
||||
info:
|
||||
title: Proxy Pool Admin API
|
||||
version: 1.0.0
|
||||
description: 运维状态与受控变更接口。该入口必须与 Distribution 分端口和权限。
|
||||
servers:
|
||||
- url: http://127.0.0.1:8082
|
||||
tags:
|
||||
- name: Status
|
||||
- name: Upstreams
|
||||
- name: Routing
|
||||
- name: Configuration
|
||||
security:
|
||||
- AdminApiKey: []
|
||||
- BasicAuth: []
|
||||
- BearerAuth: []
|
||||
paths:
|
||||
/api/v1/status:
|
||||
get:
|
||||
tags: [Status]
|
||||
operationId: getStatus
|
||||
summary: 获取控制面摘要状态
|
||||
responses:
|
||||
'200':
|
||||
description: 不含 Proxy 地址、Client 标识或 Secret 的聚合状态
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
$ref: '#/components/schemas/Status'
|
||||
'401': {$ref: '#/components/responses/Unauthorized'}
|
||||
'403': {$ref: '#/components/responses/Forbidden'}
|
||||
/api/v1/upstreams/{name}/enable:
|
||||
post:
|
||||
tags: [Upstreams]
|
||||
operationId: enableUpstream
|
||||
summary: 启用 Upstream
|
||||
parameters:
|
||||
- $ref: '#/components/parameters/UpstreamName'
|
||||
- $ref: '#/components/parameters/RequestID'
|
||||
responses:
|
||||
'200': {$ref: '#/components/responses/MutationResult'}
|
||||
'401': {$ref: '#/components/responses/Unauthorized'}
|
||||
'403': {$ref: '#/components/responses/Forbidden'}
|
||||
'404': {$ref: '#/components/responses/NotFound'}
|
||||
'409': {$ref: '#/components/responses/Conflict'}
|
||||
/api/v1/upstreams/{name}/disable:
|
||||
post:
|
||||
tags: [Upstreams]
|
||||
operationId: disableUpstream
|
||||
summary: 禁用 Upstream 并使已有资源自然 Drain
|
||||
parameters:
|
||||
- $ref: '#/components/parameters/UpstreamName'
|
||||
- $ref: '#/components/parameters/RequestID'
|
||||
responses:
|
||||
'200': {$ref: '#/components/responses/MutationResult'}
|
||||
'401': {$ref: '#/components/responses/Unauthorized'}
|
||||
'403': {$ref: '#/components/responses/Forbidden'}
|
||||
'404': {$ref: '#/components/responses/NotFound'}
|
||||
'409': {$ref: '#/components/responses/Conflict'}
|
||||
/api/v1/routing/{name}/switch:
|
||||
post:
|
||||
tags: [Routing]
|
||||
operationId: switchRouting
|
||||
summary: 原子切换 Sequential Routing 当前 Upstream
|
||||
parameters:
|
||||
- name: name
|
||||
in: path
|
||||
required: true
|
||||
schema: {type: string, minLength: 1, maxLength: 128}
|
||||
- $ref: '#/components/parameters/RequestID'
|
||||
requestBody:
|
||||
required: true
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
type: object
|
||||
additionalProperties: false
|
||||
required: [expectedCurrent, target]
|
||||
properties:
|
||||
expectedCurrent: {type: string}
|
||||
target: {type: string}
|
||||
reason: {type: string, maxLength: 512}
|
||||
responses:
|
||||
'200': {$ref: '#/components/responses/MutationResult'}
|
||||
'401': {$ref: '#/components/responses/Unauthorized'}
|
||||
'403': {$ref: '#/components/responses/Forbidden'}
|
||||
'404': {$ref: '#/components/responses/NotFound'}
|
||||
'409': {$ref: '#/components/responses/Conflict'}
|
||||
/api/v1/config/reload:
|
||||
post:
|
||||
tags: [Configuration]
|
||||
operationId: reloadConfiguration
|
||||
summary: 严格校验并原子发布新配置快照
|
||||
parameters:
|
||||
- $ref: '#/components/parameters/RequestID'
|
||||
responses:
|
||||
'200': {$ref: '#/components/responses/MutationResult'}
|
||||
'401': {$ref: '#/components/responses/Unauthorized'}
|
||||
'403': {$ref: '#/components/responses/Forbidden'}
|
||||
'409': {$ref: '#/components/responses/Conflict'}
|
||||
'422':
|
||||
description: 新配置无效,旧配置继续运行
|
||||
content:
|
||||
application/problem+json:
|
||||
schema: {$ref: '#/components/schemas/Problem'}
|
||||
components:
|
||||
securitySchemes:
|
||||
AdminApiKey: {type: apiKey, in: header, name: X-Admin-Key}
|
||||
BasicAuth: {type: http, scheme: basic}
|
||||
BearerAuth: {type: http, scheme: bearer}
|
||||
parameters:
|
||||
UpstreamName:
|
||||
name: name
|
||||
in: path
|
||||
required: true
|
||||
schema: {type: string, minLength: 1, maxLength: 128}
|
||||
RequestID:
|
||||
name: X-Request-ID
|
||||
in: header
|
||||
required: false
|
||||
schema: {type: string, maxLength: 128}
|
||||
schemas:
|
||||
Status:
|
||||
type: object
|
||||
additionalProperties: false
|
||||
required: [configVersion, snapshotVersion, upstreams, workers]
|
||||
properties:
|
||||
configVersion: {type: string}
|
||||
snapshotVersion: {type: integer, minimum: 0}
|
||||
upstreams:
|
||||
type: array
|
||||
items:
|
||||
type: object
|
||||
additionalProperties: false
|
||||
required: [name, enabled, available, checking, suspect, draining, extracted]
|
||||
properties:
|
||||
name: {type: string}
|
||||
enabled: {type: boolean}
|
||||
available: {type: integer, minimum: 0}
|
||||
checking: {type: integer, minimum: 0}
|
||||
suspect: {type: integer, minimum: 0}
|
||||
draining: {type: integer, minimum: 0}
|
||||
extracted: {type: integer, minimum: 0}
|
||||
consecutiveEmptyFetch: {type: integer, minimum: 0}
|
||||
fetchErrorCount: {type: integer, minimum: 0}
|
||||
workers:
|
||||
type: array
|
||||
items:
|
||||
type: object
|
||||
additionalProperties: false
|
||||
required: [id, zone, connected, snapshotVersion]
|
||||
properties:
|
||||
id: {type: string}
|
||||
zone: {type: string}
|
||||
connected: {type: boolean}
|
||||
snapshotVersion: {type: integer, minimum: 0}
|
||||
staleSeconds: {type: integer, minimum: 0}
|
||||
MutationResult:
|
||||
type: object
|
||||
additionalProperties: false
|
||||
required: [requestId, changed, version]
|
||||
properties:
|
||||
requestId: {type: string}
|
||||
changed: {type: boolean}
|
||||
version: {type: integer, minimum: 0}
|
||||
message: {type: string}
|
||||
Problem:
|
||||
type: object
|
||||
required: [type, title, status, code]
|
||||
properties:
|
||||
type: {type: string, format: uri}
|
||||
title: {type: string}
|
||||
status: {type: integer}
|
||||
code: {type: string}
|
||||
detail: {type: string}
|
||||
requestId: {type: string}
|
||||
responses:
|
||||
MutationResult:
|
||||
description: 操作已提交或目标状态原本已满足
|
||||
content:
|
||||
application/json:
|
||||
schema: {$ref: '#/components/schemas/MutationResult'}
|
||||
Unauthorized:
|
||||
description: 管理入口认证失败
|
||||
content:
|
||||
application/problem+json:
|
||||
schema: {$ref: '#/components/schemas/Problem'}
|
||||
Forbidden:
|
||||
description: 调用主体无该管理权限
|
||||
content:
|
||||
application/problem+json:
|
||||
schema: {$ref: '#/components/schemas/Problem'}
|
||||
NotFound:
|
||||
description: Upstream 或 Routing 不存在
|
||||
content:
|
||||
application/problem+json:
|
||||
schema: {$ref: '#/components/schemas/Problem'}
|
||||
Conflict:
|
||||
description: 预期版本或 expectedCurrent 与权威状态不一致
|
||||
content:
|
||||
application/problem+json:
|
||||
schema: {$ref: '#/components/schemas/Problem'}
|
||||
63
api/openapi/openapi_test.go
Normal file
63
api/openapi/openapi_test.go
Normal file
@ -0,0 +1,63 @@
|
||||
package openapi
|
||||
|
||||
import (
|
||||
"os"
|
||||
"testing"
|
||||
|
||||
"go.yaml.in/yaml/v4"
|
||||
)
|
||||
|
||||
type document struct {
|
||||
OpenAPI string `yaml:"openapi"`
|
||||
Paths map[string]map[string]any `yaml:"paths"`
|
||||
}
|
||||
|
||||
func TestDistributionContract(t *testing.T) {
|
||||
spec := readDocument(t, "proxy-pool.yaml")
|
||||
if spec.OpenAPI != "3.1.0" {
|
||||
t.Fatalf("openapi version = %q, want 3.1.0", spec.OpenAPI)
|
||||
}
|
||||
extraction, ok := spec.Paths["/api/v1/proxies/extract"]
|
||||
if !ok {
|
||||
t.Fatal("exclusive extraction path is missing")
|
||||
}
|
||||
if _, ok := extraction["post"]; !ok {
|
||||
t.Fatal("exclusive extraction must use POST")
|
||||
}
|
||||
for path := range spec.Paths {
|
||||
if path == "/api/v1/leases" || path == "/api/v1/proxies/release" || path == "/api/v1/proxies/renew" {
|
||||
t.Fatalf("lease/release path is forbidden: %s", path)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestAdminContract(t *testing.T) {
|
||||
spec := readDocument(t, "admin.yaml")
|
||||
for _, path := range []string{
|
||||
"/api/v1/status",
|
||||
"/api/v1/upstreams/{name}/enable",
|
||||
"/api/v1/upstreams/{name}/disable",
|
||||
"/api/v1/routing/{name}/switch",
|
||||
"/api/v1/config/reload",
|
||||
} {
|
||||
if _, ok := spec.Paths[path]; !ok {
|
||||
t.Errorf("admin path is missing: %s", path)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func readDocument(t *testing.T, path string) document {
|
||||
t.Helper()
|
||||
content, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
t.Fatalf("read %s: %v", path, err)
|
||||
}
|
||||
var spec document
|
||||
if err := yaml.Unmarshal(content, &spec); err != nil {
|
||||
t.Fatalf("parse %s: %v", path, err)
|
||||
}
|
||||
if spec.OpenAPI != "3.1.0" {
|
||||
t.Fatalf("%s openapi version = %q, want 3.1.0", path, spec.OpenAPI)
|
||||
}
|
||||
return spec
|
||||
}
|
||||
351
api/openapi/proxy-pool.yaml
Normal file
351
api/openapi/proxy-pool.yaml
Normal file
@ -0,0 +1,351 @@
|
||||
openapi: 3.1.0
|
||||
info:
|
||||
title: Proxy Pool HTTP API
|
||||
version: 1.0.0
|
||||
description: |
|
||||
Distribution API performs one-time exclusive extraction. A successful
|
||||
operation atomically transitions every returned proxy from AVAILABLE to
|
||||
EXTRACTED. Extracted proxies are never allocated again and there is no
|
||||
release, renew, or lease API.
|
||||
servers:
|
||||
- url: http://127.0.0.1:8081
|
||||
description: Distribution API
|
||||
tags:
|
||||
- name: Distribution
|
||||
- name: Health
|
||||
paths:
|
||||
/api/v1/proxies/extract:
|
||||
post:
|
||||
tags: [Distribution]
|
||||
operationId: extractProxies
|
||||
summary: 一次性独占提取代理
|
||||
description: |
|
||||
服务端先完成筛选、行锁定、AVAILABLE -> EXTRACTED 状态更新和审计记录
|
||||
写入,再返回代理。相同代理不会返回给两个成功请求。
|
||||
|
||||
`partial` 允许实际返回数量小于请求数量;`allOrNothing` 数量不足时不
|
||||
提取任何代理并返回 409。未传 `fulfillment` 时使用服务端配置,默认
|
||||
为 `partial`。`Idempotency-Key` 可避免客户端因响应丢失重试而再次消耗
|
||||
库存。
|
||||
security:
|
||||
- ApiKeyAuth: []
|
||||
- BasicAuth: []
|
||||
- BearerAuth: []
|
||||
- {}
|
||||
parameters:
|
||||
- $ref: '#/components/parameters/RequestID'
|
||||
- $ref: '#/components/parameters/IdempotencyKey'
|
||||
requestBody:
|
||||
required: true
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
$ref: '#/components/schemas/ExtractRequest'
|
||||
examples:
|
||||
partial:
|
||||
value:
|
||||
count: 5
|
||||
fulfillment: partial
|
||||
filters:
|
||||
protocols: [http]
|
||||
regions: [shanghai]
|
||||
allOrNothing:
|
||||
value:
|
||||
count: 10
|
||||
fulfillment: allOrNothing
|
||||
filters:
|
||||
allowedUpstreams: [provider-a, provider-b]
|
||||
responses:
|
||||
'200':
|
||||
description: 提取事务已提交;返回的代理已永久退出可分配池
|
||||
headers:
|
||||
X-Request-ID:
|
||||
$ref: '#/components/headers/RequestID'
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
$ref: '#/components/schemas/ExtractResponse'
|
||||
'400':
|
||||
$ref: '#/components/responses/BadRequest'
|
||||
'401':
|
||||
$ref: '#/components/responses/Unauthorized'
|
||||
'403':
|
||||
$ref: '#/components/responses/Forbidden'
|
||||
'409':
|
||||
description: allOrNothing 模式下符合条件的库存不足,未提取任何代理
|
||||
content:
|
||||
application/problem+json:
|
||||
schema:
|
||||
$ref: '#/components/schemas/Problem'
|
||||
example:
|
||||
type: https://proxy-pool.local/problems/insufficient-proxies
|
||||
title: Insufficient proxies
|
||||
status: 409
|
||||
code: INSUFFICIENT_PROXIES
|
||||
detail: requested 10 proxies but only 6 are currently eligible
|
||||
requestId: req_01J4EXAMPLE
|
||||
'422':
|
||||
$ref: '#/components/responses/UnprocessableEntity'
|
||||
'429':
|
||||
$ref: '#/components/responses/TooManyRequests'
|
||||
'503':
|
||||
$ref: '#/components/responses/ServiceUnavailable'
|
||||
/health/live:
|
||||
get:
|
||||
tags: [Health]
|
||||
operationId: getLiveness
|
||||
summary: 进程存活探针
|
||||
security: []
|
||||
responses:
|
||||
'200':
|
||||
description: 进程存活
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
$ref: '#/components/schemas/Health'
|
||||
/health/ready:
|
||||
get:
|
||||
tags: [Health]
|
||||
operationId: getReadiness
|
||||
summary: Distribution 就绪探针
|
||||
description: PostgreSQL 不可用或权威状态不可写时返回 503。
|
||||
security: []
|
||||
responses:
|
||||
'200':
|
||||
description: 可接受提取请求
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
$ref: '#/components/schemas/Health'
|
||||
'503':
|
||||
$ref: '#/components/responses/ServiceUnavailable'
|
||||
components:
|
||||
securitySchemes:
|
||||
ApiKeyAuth:
|
||||
type: apiKey
|
||||
in: header
|
||||
name: X-API-Key
|
||||
BasicAuth:
|
||||
type: http
|
||||
scheme: basic
|
||||
BearerAuth:
|
||||
type: http
|
||||
scheme: bearer
|
||||
parameters:
|
||||
RequestID:
|
||||
name: X-Request-ID
|
||||
in: header
|
||||
required: false
|
||||
description: 调用方请求标识;缺省时由服务端生成。
|
||||
schema:
|
||||
type: string
|
||||
maxLength: 128
|
||||
IdempotencyKey:
|
||||
name: Idempotency-Key
|
||||
in: header
|
||||
required: false
|
||||
description: |
|
||||
同一客户端在幂等记录保留期内重用该键会得到首次提交结果,不会再次
|
||||
提取。建议所有会自动重试的客户端提供。
|
||||
schema:
|
||||
type: string
|
||||
minLength: 8
|
||||
maxLength: 128
|
||||
headers:
|
||||
RequestID:
|
||||
description: 服务端最终使用的请求标识。
|
||||
schema:
|
||||
type: string
|
||||
schemas:
|
||||
ExtractRequest:
|
||||
type: object
|
||||
additionalProperties: false
|
||||
required: [count]
|
||||
properties:
|
||||
count:
|
||||
type: integer
|
||||
minimum: 1
|
||||
maximum: 1000
|
||||
description: 仍受服务端 maxCountPerRequest 限制。
|
||||
fulfillment:
|
||||
type: string
|
||||
enum: [partial, allOrNothing]
|
||||
description: 缺省时使用服务端配置;默认 partial。
|
||||
filters:
|
||||
$ref: '#/components/schemas/ExtractFilters'
|
||||
ExtractFilters:
|
||||
type: object
|
||||
additionalProperties: false
|
||||
properties:
|
||||
protocols:
|
||||
type: array
|
||||
uniqueItems: true
|
||||
items:
|
||||
type: string
|
||||
enum: [http, https, socks5]
|
||||
regions:
|
||||
type: array
|
||||
uniqueItems: true
|
||||
items:
|
||||
type: string
|
||||
carriers:
|
||||
type: array
|
||||
uniqueItems: true
|
||||
items:
|
||||
type: string
|
||||
allowedUpstreams:
|
||||
type: array
|
||||
uniqueItems: true
|
||||
items:
|
||||
type: string
|
||||
ExtractResponse:
|
||||
type: object
|
||||
additionalProperties: false
|
||||
required: [requestId, requested, returned, proxies]
|
||||
properties:
|
||||
requestId:
|
||||
type: string
|
||||
requested:
|
||||
type: integer
|
||||
minimum: 1
|
||||
returned:
|
||||
type: integer
|
||||
minimum: 0
|
||||
proxies:
|
||||
type: array
|
||||
items:
|
||||
$ref: '#/components/schemas/ExtractedProxy'
|
||||
ExtractedProxy:
|
||||
type: object
|
||||
additionalProperties: false
|
||||
required:
|
||||
- id
|
||||
- protocol
|
||||
- host
|
||||
- port
|
||||
- url
|
||||
- upstream
|
||||
- expiresAt
|
||||
- remainingTtlSeconds
|
||||
- extractedAt
|
||||
properties:
|
||||
id:
|
||||
type: string
|
||||
example: px_01J4EXAMPLE
|
||||
protocol:
|
||||
type: string
|
||||
enum: [http, https, socks5]
|
||||
host:
|
||||
type: string
|
||||
example: 192.0.2.10
|
||||
port:
|
||||
type: integer
|
||||
minimum: 1
|
||||
maximum: 65535
|
||||
username:
|
||||
type: string
|
||||
password:
|
||||
type: string
|
||||
format: password
|
||||
description: 真实代理凭据,只出现在提取成功响应中。
|
||||
url:
|
||||
type: string
|
||||
format: uri
|
||||
description: 含真实代理凭据的连接 URL,必须按敏感数据处理。
|
||||
example: http://USER:PASSWORD@192.0.2.10:8080
|
||||
region:
|
||||
type: string
|
||||
carrier:
|
||||
type: string
|
||||
upstream:
|
||||
type: string
|
||||
expiresAt:
|
||||
type: string
|
||||
format: date-time
|
||||
remainingTtlSeconds:
|
||||
type: integer
|
||||
minimum: 0
|
||||
extractedAt:
|
||||
type: string
|
||||
format: date-time
|
||||
Health:
|
||||
type: object
|
||||
additionalProperties: false
|
||||
required: [status]
|
||||
properties:
|
||||
status:
|
||||
type: string
|
||||
enum: [ok, degraded]
|
||||
version:
|
||||
type: string
|
||||
configVersion:
|
||||
type: string
|
||||
Problem:
|
||||
type: object
|
||||
additionalProperties: true
|
||||
required: [type, title, status, code]
|
||||
properties:
|
||||
type:
|
||||
type: string
|
||||
format: uri
|
||||
title:
|
||||
type: string
|
||||
status:
|
||||
type: integer
|
||||
code:
|
||||
type: string
|
||||
detail:
|
||||
type: string
|
||||
requestId:
|
||||
type: string
|
||||
invalidParams:
|
||||
type: array
|
||||
items:
|
||||
type: object
|
||||
required: [name, reason]
|
||||
properties:
|
||||
name:
|
||||
type: string
|
||||
reason:
|
||||
type: string
|
||||
responses:
|
||||
BadRequest:
|
||||
description: 请求体、Header 或 JSON 格式无效
|
||||
content:
|
||||
application/problem+json:
|
||||
schema:
|
||||
$ref: '#/components/schemas/Problem'
|
||||
Unauthorized:
|
||||
description: 所配置的认证方法未通过
|
||||
content:
|
||||
application/problem+json:
|
||||
schema:
|
||||
$ref: '#/components/schemas/Problem'
|
||||
Forbidden:
|
||||
description: 来源访问控制或客户端权限拒绝
|
||||
content:
|
||||
application/problem+json:
|
||||
schema:
|
||||
$ref: '#/components/schemas/Problem'
|
||||
UnprocessableEntity:
|
||||
description: 参数语法有效但违反业务约束
|
||||
content:
|
||||
application/problem+json:
|
||||
schema:
|
||||
$ref: '#/components/schemas/Problem'
|
||||
TooManyRequests:
|
||||
description: 超过全局或客户端速率限制
|
||||
headers:
|
||||
Retry-After:
|
||||
schema:
|
||||
type: integer
|
||||
content:
|
||||
application/problem+json:
|
||||
schema:
|
||||
$ref: '#/components/schemas/Problem'
|
||||
ServiceUnavailable:
|
||||
description: 权威存储不可用或服务正在排空
|
||||
content:
|
||||
application/problem+json:
|
||||
schema:
|
||||
$ref: '#/components/schemas/Problem'
|
||||
249
api/proto/controlplane/v1/controlplane.proto
Normal file
249
api/proto/controlplane/v1/controlplane.proto
Normal file
@ -0,0 +1,249 @@
|
||||
syntax = "proto3";
|
||||
|
||||
package proxy_pool.controlplane.v1;
|
||||
|
||||
option go_package = "github.com/proxy-pool/proxy-pool/gen/controlplane/v1;controlplanev1";
|
||||
|
||||
import "google/protobuf/duration.proto";
|
||||
import "google/protobuf/empty.proto";
|
||||
import "google/protobuf/timestamp.proto";
|
||||
|
||||
// WorkerControlPlane distributes immutable, worker-specific snapshots. The
|
||||
// gateway hot path does not call this service for individual requests.
|
||||
service WorkerControlPlane {
|
||||
rpc RegisterWorker(RegisterWorkerRequest) returns (RegisterWorkerResponse);
|
||||
rpc WatchSnapshots(WatchSnapshotsRequest) returns (stream SnapshotEnvelope);
|
||||
rpc AcknowledgeSnapshot(AcknowledgeSnapshotRequest) returns (google.protobuf.Empty);
|
||||
rpc ReportOutcomes(stream OutcomeBatch) returns (ReportOutcomesResponse);
|
||||
rpc ReportRuntime(ReportRuntimeRequest) returns (ReportRuntimeResponse);
|
||||
}
|
||||
|
||||
// CheckerControlPlane hands bounded check work to independently scalable
|
||||
// checker processes. Observations are facts; only the Controller reducer may
|
||||
// change authoritative proxy state.
|
||||
service CheckerControlPlane {
|
||||
rpc StreamCheckTasks(StreamCheckTasksRequest) returns (stream CheckTask);
|
||||
rpc ReportObservations(ObservationBatch) returns (ReportObservationsResponse);
|
||||
}
|
||||
|
||||
message RegisterWorkerRequest {
|
||||
string worker_id = 1;
|
||||
string instance_id = 2;
|
||||
string zone = 3;
|
||||
uint32 supported_protocol_version = 4;
|
||||
map<string, string> labels = 5;
|
||||
}
|
||||
|
||||
message RegisterWorkerResponse {
|
||||
string worker_id = 1;
|
||||
string session_id = 2;
|
||||
uint64 ownership_epoch = 3;
|
||||
google.protobuf.Duration heartbeat_interval = 4;
|
||||
google.protobuf.Duration max_stale_age = 5;
|
||||
}
|
||||
|
||||
message WatchSnapshotsRequest {
|
||||
string worker_id = 1;
|
||||
string session_id = 2;
|
||||
uint64 last_applied_version = 3;
|
||||
bytes last_checksum = 4;
|
||||
}
|
||||
|
||||
message SnapshotEnvelope {
|
||||
oneof payload {
|
||||
WorkerSnapshot full = 1;
|
||||
SnapshotDelta delta = 2;
|
||||
}
|
||||
}
|
||||
|
||||
message WorkerSnapshot {
|
||||
uint64 version = 1;
|
||||
uint64 ownership_epoch = 2;
|
||||
google.protobuf.Timestamp generated_at = 3;
|
||||
google.protobuf.Timestamp valid_until = 4;
|
||||
bytes checksum = 5;
|
||||
repeated RoutingRule routing = 6;
|
||||
repeated OwnedProxy proxies = 7;
|
||||
}
|
||||
|
||||
message SnapshotDelta {
|
||||
uint64 base_version = 1;
|
||||
uint64 version = 2;
|
||||
uint64 ownership_epoch = 3;
|
||||
google.protobuf.Timestamp generated_at = 4;
|
||||
bytes checksum = 5;
|
||||
repeated RoutingRule upserted_routing = 6;
|
||||
repeated string removed_routing_names = 7;
|
||||
repeated OwnedProxy upserted_proxies = 8;
|
||||
repeated string removed_proxy_ids = 9;
|
||||
}
|
||||
|
||||
message RoutingRule {
|
||||
string name = 1;
|
||||
bool enabled = 2;
|
||||
string host_regex = 3;
|
||||
repeated string methods = 4;
|
||||
string path_regex = 5;
|
||||
map<string, string> headers = 6;
|
||||
repeated string upstreams = 7;
|
||||
RoutingStrategy strategy = 8;
|
||||
UnavailableAction on_unavailable = 9;
|
||||
}
|
||||
|
||||
message RoutingStrategy {
|
||||
StrategyType type = 1;
|
||||
string current_upstream = 2;
|
||||
map<string, uint32> weights = 3;
|
||||
}
|
||||
|
||||
enum StrategyType {
|
||||
STRATEGY_TYPE_UNSPECIFIED = 0;
|
||||
STRATEGY_TYPE_SEQUENTIAL = 1;
|
||||
STRATEGY_TYPE_RANDOM = 2;
|
||||
STRATEGY_TYPE_ROUND_ROBIN = 3;
|
||||
STRATEGY_TYPE_WEIGHTED = 4;
|
||||
STRATEGY_TYPE_LEAST_CONNECTIONS = 5;
|
||||
}
|
||||
|
||||
enum UnavailableAction {
|
||||
UNAVAILABLE_ACTION_UNSPECIFIED = 0;
|
||||
UNAVAILABLE_ACTION_REJECT = 1;
|
||||
UNAVAILABLE_ACTION_WAIT = 2;
|
||||
UNAVAILABLE_ACTION_DIRECT = 3;
|
||||
}
|
||||
|
||||
message OwnedProxy {
|
||||
string id = 1;
|
||||
string upstream = 2;
|
||||
ProxyProtocol protocol = 3;
|
||||
string host = 4;
|
||||
uint32 port = 5;
|
||||
string username = 6;
|
||||
string credential_version = 7;
|
||||
string secret_ref = 8;
|
||||
google.protobuf.Timestamp expires_at = 9;
|
||||
uint32 max_concurrency = 10;
|
||||
map<string, string> tags = 11;
|
||||
uint64 ownership_epoch = 12;
|
||||
}
|
||||
|
||||
enum ProxyProtocol {
|
||||
PROXY_PROTOCOL_UNSPECIFIED = 0;
|
||||
PROXY_PROTOCOL_HTTP = 1;
|
||||
PROXY_PROTOCOL_HTTPS = 2;
|
||||
PROXY_PROTOCOL_SOCKS5 = 3;
|
||||
}
|
||||
|
||||
message AcknowledgeSnapshotRequest {
|
||||
string worker_id = 1;
|
||||
string session_id = 2;
|
||||
uint64 version = 3;
|
||||
uint64 ownership_epoch = 4;
|
||||
bytes checksum = 5;
|
||||
bool applied = 6;
|
||||
string error_code = 7;
|
||||
string error_message = 8;
|
||||
}
|
||||
|
||||
message OutcomeBatch {
|
||||
string worker_id = 1;
|
||||
string session_id = 2;
|
||||
uint64 sequence = 3;
|
||||
repeated ProxyOutcome outcomes = 4;
|
||||
}
|
||||
|
||||
message ProxyOutcome {
|
||||
string proxy_id = 1;
|
||||
string routing_name = 2;
|
||||
OutcomeStage stage = 3;
|
||||
bool success = 4;
|
||||
google.protobuf.Duration latency = 5;
|
||||
string error_class = 6;
|
||||
google.protobuf.Timestamp observed_at = 7;
|
||||
}
|
||||
|
||||
enum OutcomeStage {
|
||||
OUTCOME_STAGE_UNSPECIFIED = 0;
|
||||
OUTCOME_STAGE_DIAL = 1;
|
||||
OUTCOME_STAGE_PROXY_HANDSHAKE = 2;
|
||||
OUTCOME_STAGE_RESPONSE_HEADERS = 3;
|
||||
OUTCOME_STAGE_TUNNEL = 4;
|
||||
}
|
||||
|
||||
message ReportOutcomesResponse {
|
||||
uint64 accepted_through_sequence = 1;
|
||||
}
|
||||
|
||||
message ReportRuntimeRequest {
|
||||
string worker_id = 1;
|
||||
string session_id = 2;
|
||||
uint64 snapshot_version = 3;
|
||||
uint64 ownership_epoch = 4;
|
||||
repeated ProxyRuntime counters = 5;
|
||||
google.protobuf.Timestamp observed_at = 6;
|
||||
}
|
||||
|
||||
message ProxyRuntime {
|
||||
string proxy_id = 1;
|
||||
uint32 reserved = 2;
|
||||
uint32 active = 3;
|
||||
bool draining = 4;
|
||||
}
|
||||
|
||||
message ReportRuntimeResponse {
|
||||
uint64 accepted_ownership_epoch = 1;
|
||||
repeated string revoke_proxy_ids = 2;
|
||||
bool require_full_snapshot = 3;
|
||||
}
|
||||
|
||||
message StreamCheckTasksRequest {
|
||||
string checker_id = 1;
|
||||
string instance_id = 2;
|
||||
uint32 max_in_flight = 3;
|
||||
repeated CheckLevel supported_levels = 4;
|
||||
}
|
||||
|
||||
message CheckTask {
|
||||
string task_id = 1;
|
||||
string proxy_id = 2;
|
||||
ProxyProtocol protocol = 3;
|
||||
string host = 4;
|
||||
uint32 port = 5;
|
||||
string secret_ref = 6;
|
||||
CheckLevel level = 7;
|
||||
string routing_name = 8;
|
||||
string target_url = 9;
|
||||
google.protobuf.Duration timeout = 10;
|
||||
uint32 attempt = 11;
|
||||
google.protobuf.Timestamp deadline = 12;
|
||||
}
|
||||
|
||||
enum CheckLevel {
|
||||
CHECK_LEVEL_UNSPECIFIED = 0;
|
||||
CHECK_LEVEL_BASIC = 1;
|
||||
CHECK_LEVEL_EGRESS = 2;
|
||||
CHECK_LEVEL_TARGET = 3;
|
||||
}
|
||||
|
||||
message ObservationBatch {
|
||||
string checker_id = 1;
|
||||
repeated HealthObservation observations = 2;
|
||||
}
|
||||
|
||||
message HealthObservation {
|
||||
string task_id = 1;
|
||||
string proxy_id = 2;
|
||||
CheckLevel level = 3;
|
||||
string routing_name = 4;
|
||||
string target_url = 5;
|
||||
bool success = 6;
|
||||
string failure_class = 7;
|
||||
google.protobuf.Duration latency = 8;
|
||||
string observed_egress_ip = 9;
|
||||
google.protobuf.Timestamp observed_at = 10;
|
||||
}
|
||||
|
||||
message ReportObservationsResponse {
|
||||
uint32 accepted = 1;
|
||||
uint32 rejected = 2;
|
||||
}
|
||||
217
configs/proxy-pool.yaml
Normal file
217
configs/proxy-pool.yaml
Normal file
@ -0,0 +1,217 @@
|
||||
version: 1
|
||||
|
||||
security:
|
||||
requireProtectionOnPublicListen: true
|
||||
|
||||
defaults:
|
||||
fetch:
|
||||
requestInterval: 1s
|
||||
timeout: 5s
|
||||
maxAttempts: 3
|
||||
maxInFlight: 1
|
||||
maxResponseBytes: 1048576
|
||||
templateTimeout: 100ms
|
||||
retry:
|
||||
initial: 500ms
|
||||
max: 30s
|
||||
jitter: 20
|
||||
check:
|
||||
interval: 30s
|
||||
jitter: 20
|
||||
maxInFlight: 200
|
||||
timeout: 3s
|
||||
maxAttempts: 2
|
||||
maxConsecutiveFailures: 3
|
||||
urls:
|
||||
- http://connect.rom.miui.com/generate_204
|
||||
|
||||
gateway:
|
||||
enabled: true
|
||||
listen: 0.0.0.0:8080
|
||||
access:
|
||||
allowCIDRs:
|
||||
- 10.0.0.0/8
|
||||
- 172.16.0.0/12
|
||||
trustedProxies: []
|
||||
auth:
|
||||
mode: usernamePassword
|
||||
username: "${GATEWAY_USER}"
|
||||
password: "${GATEWAY_PASSWORD}"
|
||||
limits:
|
||||
maxConcurrentConnections: 200000
|
||||
retry:
|
||||
maxAttempts: 2
|
||||
retryMethods: [GET, HEAD]
|
||||
destinationPolicy:
|
||||
denyPrivateNetworks: true
|
||||
denyLoopback: true
|
||||
denyLinkLocal: true
|
||||
denyCIDRs:
|
||||
- 169.254.169.254/32
|
||||
|
||||
distribution:
|
||||
enabled: true
|
||||
listen: 0.0.0.0:8081
|
||||
access:
|
||||
allowCIDRs:
|
||||
- 10.0.0.0/8
|
||||
- 172.16.0.0/12
|
||||
trustedProxies: []
|
||||
auth:
|
||||
mode: apiKey
|
||||
header: X-API-Key
|
||||
token: "${DISTRIBUTION_API_KEY}"
|
||||
limits:
|
||||
requestsPerMinute: 6000
|
||||
requestsPerMinutePerClient: 600
|
||||
clientIdentification:
|
||||
mode: authenticatedClientOrSourceIP
|
||||
extraction:
|
||||
fulfillment: partial
|
||||
maxCountPerRequest: 100
|
||||
minRemainingTTL: 30s
|
||||
maxHealthCheckAge: 15s
|
||||
reserveForGateway: 100
|
||||
|
||||
admin:
|
||||
enabled: true
|
||||
listen: 127.0.0.1:8082
|
||||
auth:
|
||||
mode: none
|
||||
|
||||
metrics:
|
||||
enabled: true
|
||||
listen: 127.0.0.1:9090
|
||||
|
||||
storage:
|
||||
postgresURL: "${POSTGRES_URL}"
|
||||
redisURL: "${REDIS_URL}"
|
||||
|
||||
routing:
|
||||
- name: gateway-default
|
||||
enabled: true
|
||||
purpose: gateway
|
||||
match:
|
||||
hostRegex: '.*'
|
||||
upstreams: [provider-a, provider-b]
|
||||
strategy:
|
||||
type: sequential
|
||||
switchAfterEmptyFetch: 5
|
||||
endBehavior: stayLast
|
||||
onUnavailable:
|
||||
action: reject
|
||||
- name: extract-default
|
||||
enabled: true
|
||||
purpose: extract
|
||||
match:
|
||||
hostRegex: '.*'
|
||||
upstreams: [provider-a, provider-b]
|
||||
strategy:
|
||||
type: sequential
|
||||
switchAfterEmptyFetch: 5
|
||||
endBehavior: stayLast
|
||||
onUnavailable:
|
||||
action: reject
|
||||
|
||||
upstreams:
|
||||
provider-a:
|
||||
enabled: true
|
||||
exposure: [gateway, extract]
|
||||
provider:
|
||||
billingMode: fetch
|
||||
protocols: [http, https]
|
||||
api:
|
||||
url: https://provider-a.example/api/proxies
|
||||
method: GET
|
||||
auth:
|
||||
type: apiKey
|
||||
location: header
|
||||
name: X-Provider-Key
|
||||
value: "${PROVIDER_A_TOKEN}"
|
||||
query:
|
||||
count: '100'
|
||||
template: '{{.}}'
|
||||
proxyAuth:
|
||||
type: response
|
||||
pool:
|
||||
maxSize: 10000
|
||||
shrinkDelay: 30s
|
||||
capacity:
|
||||
maxConcurrencyPerProxy: 20
|
||||
lifecycle:
|
||||
ttl: 5m
|
||||
allocationSafetyMargin: 30s
|
||||
fetch:
|
||||
requestInterval: 1s
|
||||
timeout: 5s
|
||||
maxAttempts: 3
|
||||
maxInFlight: 1
|
||||
maxTotal: 1000000
|
||||
maxResponseBytes: 1048576
|
||||
templateTimeout: 100ms
|
||||
retry:
|
||||
initial: 500ms
|
||||
max: 30s
|
||||
jitter: 20
|
||||
check:
|
||||
interval: 30s
|
||||
jitter: 20
|
||||
maxInFlight: 200
|
||||
timeout: 3s
|
||||
maxAttempts: 2
|
||||
maxConsecutiveFailures: 3
|
||||
urls:
|
||||
- http://connect.rom.miui.com/generate_204
|
||||
provider-b:
|
||||
enabled: true
|
||||
exposure: [gateway, extract]
|
||||
provider:
|
||||
billingMode: subscription
|
||||
protocols: [http]
|
||||
api:
|
||||
url: https://provider-b.example/api/proxies
|
||||
method: POST
|
||||
auth:
|
||||
type: basic
|
||||
username: "${PROVIDER_B_USER}"
|
||||
password: "${PROVIDER_B_PASSWORD}"
|
||||
headers:
|
||||
Content-Type: application/json
|
||||
body:
|
||||
type: json
|
||||
value:
|
||||
count: '100'
|
||||
template: '{{.}}'
|
||||
proxyAuth:
|
||||
type: static
|
||||
username: "${PROXY_USER}"
|
||||
password: "${PROXY_PASSWORD}"
|
||||
pool:
|
||||
maxSize: 5000
|
||||
shrinkDelay: 30s
|
||||
capacity:
|
||||
maxConcurrencyPerProxy: 10
|
||||
lifecycle:
|
||||
ttl: 2m
|
||||
allocationSafetyMargin: 20s
|
||||
fetch:
|
||||
requestInterval: 2s
|
||||
timeout: 5s
|
||||
maxAttempts: 3
|
||||
maxInFlight: 1
|
||||
maxTotal: 0
|
||||
maxResponseBytes: 1048576
|
||||
templateTimeout: 100ms
|
||||
retry:
|
||||
initial: 1s
|
||||
max: 30s
|
||||
jitter: 20
|
||||
check:
|
||||
interval: 20s
|
||||
jitter: 20
|
||||
maxInFlight: 100
|
||||
timeout: 3s
|
||||
maxAttempts: 2
|
||||
maxConsecutiveFailures: 3
|
||||
urls:
|
||||
- http://connect.rom.miui.com/generate_204
|
||||
17
deploy/README.md
Normal file
17
deploy/README.md
Normal file
@ -0,0 +1,17 @@
|
||||
# 部署拓扑模板
|
||||
|
||||
本目录描述 Proxy Pool 目标运行拓扑,覆盖 Compose、HAProxy、Prometheus、
|
||||
Grafana 与 Kubernetes。配置已通过静态展开,但当前仓库的 `cmd/proxy-*`
|
||||
运行时装配仍在实施计划中,因此不要把这些清单视为当前可部署发行版。
|
||||
|
||||
当前可执行验证:
|
||||
|
||||
```powershell
|
||||
docker compose -f deploy/docker-compose.yml config --quiet
|
||||
kubectl kustomize deploy/kubernetes/base | Out-Null
|
||||
go run ./deploy/tools/configcheck deploy/config/local.yaml
|
||||
```
|
||||
|
||||
运行时完成后,还必须通过 `production-readiness.md` 中的一致性、安全、恢复、
|
||||
竞态与容量门禁,才能构建镜像并发布。
|
||||
|
||||
179
deploy/config/local.yaml
Normal file
179
deploy/config/local.yaml
Normal file
@ -0,0 +1,179 @@
|
||||
version: 1
|
||||
|
||||
security:
|
||||
requireProtectionOnPublicListen: true
|
||||
|
||||
gateway:
|
||||
enabled: true
|
||||
listen: 0.0.0.0:8080
|
||||
access:
|
||||
allowCIDRs: [172.16.0.0/12]
|
||||
trustedProxies: [172.16.0.0/12]
|
||||
auth:
|
||||
mode: usernamePassword
|
||||
username: local-gateway
|
||||
password: env:PROXY_POOL_GATEWAY_PASSWORD
|
||||
limits:
|
||||
maxConcurrentConnections: 20000
|
||||
requestsPerMinutePerClient: 60000
|
||||
retry:
|
||||
maxAttempts: 2
|
||||
retryMethods: [GET, HEAD]
|
||||
destinationPolicy:
|
||||
denyPrivateNetworks: true
|
||||
denyLoopback: true
|
||||
denyLinkLocal: true
|
||||
denyCIDRs: [169.254.169.254/32]
|
||||
|
||||
distribution:
|
||||
enabled: true
|
||||
listen: 0.0.0.0:8081
|
||||
access:
|
||||
allowCIDRs: [172.16.0.0/12]
|
||||
trustedProxies: [172.16.0.0/12]
|
||||
auth:
|
||||
mode: apiKey
|
||||
header: X-API-Key
|
||||
token: env:PROXY_POOL_EXTRACT_TOKEN
|
||||
limits:
|
||||
requestsPerMinute: 6000
|
||||
requestsPerMinutePerClient: 600
|
||||
clientIdentification:
|
||||
mode: trustedProxyOrRemoteIP
|
||||
extraction:
|
||||
fulfillment: partial
|
||||
maxCountPerRequest: 100
|
||||
minRemainingTTL: 30s
|
||||
maxHealthCheckAge: 30s
|
||||
reserveForGateway: 1000
|
||||
|
||||
admin:
|
||||
enabled: true
|
||||
listen: 0.0.0.0:8082
|
||||
access:
|
||||
allowCIDRs: [172.16.0.0/12]
|
||||
auth:
|
||||
mode: apiKey
|
||||
header: X-Admin-Token
|
||||
token: env:PROXY_POOL_ADMIN_TOKEN
|
||||
|
||||
metrics:
|
||||
enabled: true
|
||||
listen: 0.0.0.0:9090
|
||||
|
||||
storage:
|
||||
postgresURL: postgres://proxy_pool:local-only-change-me@postgres:5432/proxy_pool?sslmode=disable
|
||||
redisURL: redis://redis:6379/0
|
||||
|
||||
routing:
|
||||
- name: gateway-default
|
||||
enabled: true
|
||||
purpose: gateway
|
||||
upstreams: [provider-a, provider-b]
|
||||
strategy:
|
||||
type: sequential
|
||||
switchAfterEmptyFetch: 5
|
||||
endBehavior: stayLast
|
||||
onUnavailable:
|
||||
action: reject
|
||||
- name: extract-default
|
||||
enabled: true
|
||||
purpose: extract
|
||||
upstreams: [provider-a, provider-b]
|
||||
strategy:
|
||||
type: sequential
|
||||
switchAfterEmptyFetch: 5
|
||||
endBehavior: stayLast
|
||||
onUnavailable:
|
||||
action: reject
|
||||
|
||||
upstreams:
|
||||
provider-a:
|
||||
enabled: true
|
||||
exposure: [gateway, extract]
|
||||
provider:
|
||||
billingMode: fetch
|
||||
protocols: [http]
|
||||
api:
|
||||
url: https://provider-a.invalid/api/proxies
|
||||
method: GET
|
||||
auth:
|
||||
mode: apiKey
|
||||
header: Authorization
|
||||
token: env:PROVIDER_A_TOKEN
|
||||
template: '{{ . }}'
|
||||
proxyAuth:
|
||||
mode: response
|
||||
pool:
|
||||
maxSize: 5000
|
||||
shrinkDelay: 30s
|
||||
capacity:
|
||||
maxConcurrencyPerProxy: 20
|
||||
lifecycle:
|
||||
ttl: 5m
|
||||
allocationSafetyMargin: 20s
|
||||
fetch:
|
||||
requestInterval: 1s
|
||||
timeout: 3s
|
||||
maxAttempts: 3
|
||||
maxInFlight: 1
|
||||
maxTotal: 100000
|
||||
maxResponseBytes: 4194304
|
||||
templateTimeout: 100ms
|
||||
retry:
|
||||
initial: 500ms
|
||||
max: 30s
|
||||
jitter: 20
|
||||
check:
|
||||
interval: 30s
|
||||
jitter: 20
|
||||
maxInFlight: 200
|
||||
timeout: 3s
|
||||
maxAttempts: 2
|
||||
maxConsecutiveFailures: 3
|
||||
urls: [https://example.com/]
|
||||
provider-b:
|
||||
enabled: true
|
||||
exposure: [gateway, extract]
|
||||
provider:
|
||||
billingMode: fetch
|
||||
protocols: [http]
|
||||
api:
|
||||
url: https://provider-b.invalid/api/proxies
|
||||
method: GET
|
||||
auth:
|
||||
mode: apiKey
|
||||
header: Authorization
|
||||
token: env:PROVIDER_B_TOKEN
|
||||
template: '{{ . }}'
|
||||
proxyAuth:
|
||||
mode: response
|
||||
pool:
|
||||
maxSize: 5000
|
||||
shrinkDelay: 30s
|
||||
capacity:
|
||||
maxConcurrencyPerProxy: 20
|
||||
lifecycle:
|
||||
ttl: 5m
|
||||
allocationSafetyMargin: 20s
|
||||
fetch:
|
||||
requestInterval: 1s
|
||||
timeout: 3s
|
||||
maxAttempts: 3
|
||||
maxInFlight: 1
|
||||
maxTotal: 100000
|
||||
maxResponseBytes: 4194304
|
||||
templateTimeout: 100ms
|
||||
retry:
|
||||
initial: 500ms
|
||||
max: 30s
|
||||
jitter: 20
|
||||
check:
|
||||
interval: 30s
|
||||
jitter: 20
|
||||
maxInFlight: 200
|
||||
timeout: 3s
|
||||
maxAttempts: 2
|
||||
maxConsecutiveFailures: 3
|
||||
urls: [https://example.com/]
|
||||
|
||||
165
deploy/docker-compose.yml
Normal file
165
deploy/docker-compose.yml
Normal file
@ -0,0 +1,165 @@
|
||||
name: proxy-pool
|
||||
|
||||
x-app: &app
|
||||
build:
|
||||
context: ..
|
||||
dockerfile: deploy/docker/Dockerfile
|
||||
image: proxy-pool:local
|
||||
restart: unless-stopped
|
||||
networks: [frontend, backend]
|
||||
volumes:
|
||||
- ./config/local.yaml:/etc/proxy-pool/config.yaml:ro
|
||||
environment:
|
||||
PROXY_POOL_CONFIG: /etc/proxy-pool/config.yaml
|
||||
PROXY_POOL_GATEWAY_PASSWORD: ${PROXY_POOL_GATEWAY_PASSWORD:?set PROXY_POOL_GATEWAY_PASSWORD}
|
||||
PROXY_POOL_EXTRACT_TOKEN: ${PROXY_POOL_EXTRACT_TOKEN:?set PROXY_POOL_EXTRACT_TOKEN}
|
||||
PROXY_POOL_ADMIN_TOKEN: ${PROXY_POOL_ADMIN_TOKEN:?set PROXY_POOL_ADMIN_TOKEN}
|
||||
PROVIDER_A_TOKEN: ${PROVIDER_A_TOKEN:?set PROVIDER_A_TOKEN}
|
||||
PROVIDER_B_TOKEN: ${PROVIDER_B_TOKEN:?set PROVIDER_B_TOKEN}
|
||||
stop_grace_period: 45s
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
redis:
|
||||
condition: service_healthy
|
||||
|
||||
services:
|
||||
gateway-a:
|
||||
<<: *app
|
||||
command: ["proxy-gateway"]
|
||||
expose: ["8080", "9090"]
|
||||
healthcheck:
|
||||
test: ["CMD", "curl", "--fail", "--silent", "http://127.0.0.1:9090/readyz"]
|
||||
interval: 5s
|
||||
timeout: 2s
|
||||
retries: 12
|
||||
start_period: 10s
|
||||
|
||||
gateway-b:
|
||||
<<: *app
|
||||
command: ["proxy-gateway"]
|
||||
expose: ["8080", "9090"]
|
||||
healthcheck:
|
||||
test: ["CMD", "curl", "--fail", "--silent", "http://127.0.0.1:9090/readyz"]
|
||||
interval: 5s
|
||||
timeout: 2s
|
||||
retries: 12
|
||||
start_period: 10s
|
||||
|
||||
controller:
|
||||
<<: *app
|
||||
command: ["proxy-controller"]
|
||||
expose: ["8081", "8082", "9090"]
|
||||
ports:
|
||||
- "127.0.0.1:8081:8081"
|
||||
- "127.0.0.1:8082:8082"
|
||||
healthcheck:
|
||||
test: ["CMD", "curl", "--fail", "--silent", "http://127.0.0.1:9090/readyz"]
|
||||
interval: 5s
|
||||
timeout: 2s
|
||||
retries: 12
|
||||
start_period: 15s
|
||||
|
||||
checker:
|
||||
<<: *app
|
||||
command: ["proxy-checker"]
|
||||
expose: ["9090"]
|
||||
healthcheck:
|
||||
test: ["CMD", "curl", "--fail", "--silent", "http://127.0.0.1:9090/readyz"]
|
||||
interval: 10s
|
||||
timeout: 2s
|
||||
retries: 12
|
||||
start_period: 15s
|
||||
|
||||
|
||||
haproxy:
|
||||
image: haproxy:3.2-alpine
|
||||
restart: unless-stopped
|
||||
networks: [frontend]
|
||||
ports:
|
||||
- "127.0.0.1:8080:8080"
|
||||
- "127.0.0.1:8404:8404"
|
||||
volumes:
|
||||
- ./haproxy/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro
|
||||
depends_on:
|
||||
gateway-a:
|
||||
condition: service_healthy
|
||||
gateway-b:
|
||||
condition: service_healthy
|
||||
healthcheck:
|
||||
test: ["CMD", "wget", "-q", "-O", "/dev/null", "http://127.0.0.1:8404/healthz"]
|
||||
interval: 5s
|
||||
timeout: 2s
|
||||
retries: 6
|
||||
|
||||
postgres:
|
||||
image: postgres:18-alpine
|
||||
restart: unless-stopped
|
||||
networks: [backend]
|
||||
environment:
|
||||
POSTGRES_DB: proxy_pool
|
||||
POSTGRES_USER: proxy_pool
|
||||
POSTGRES_PASSWORD: local-only-change-me
|
||||
volumes:
|
||||
- postgres-data:/var/lib/postgresql/data
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "pg_isready -U proxy_pool -d proxy_pool"]
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 12
|
||||
|
||||
redis:
|
||||
image: redis:8.2-alpine
|
||||
restart: unless-stopped
|
||||
command: ["redis-server", "--appendonly", "yes", "--save", "60", "1"]
|
||||
networks: [backend]
|
||||
volumes:
|
||||
- redis-data:/data
|
||||
healthcheck:
|
||||
test: ["CMD", "redis-cli", "ping"]
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 12
|
||||
|
||||
prometheus:
|
||||
image: prom/prometheus:v3.5.0
|
||||
restart: unless-stopped
|
||||
networks: [frontend, backend]
|
||||
ports:
|
||||
- "127.0.0.1:9091:9090"
|
||||
volumes:
|
||||
- ./prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro
|
||||
- ./prometheus/rules:/etc/prometheus/rules:ro
|
||||
- prometheus-data:/prometheus
|
||||
command:
|
||||
- --config.file=/etc/prometheus/prometheus.yml
|
||||
- --storage.tsdb.path=/prometheus
|
||||
- --storage.tsdb.retention.time=7d
|
||||
- --web.enable-lifecycle
|
||||
|
||||
grafana:
|
||||
image: grafana/grafana:12.1.0
|
||||
restart: unless-stopped
|
||||
networks: [backend]
|
||||
ports:
|
||||
- "127.0.0.1:3000:3000"
|
||||
environment:
|
||||
GF_SECURITY_ADMIN_USER: admin
|
||||
GF_SECURITY_ADMIN_PASSWORD: local-only-change-me
|
||||
GF_USERS_ALLOW_SIGN_UP: "false"
|
||||
volumes:
|
||||
- ./grafana/provisioning:/etc/grafana/provisioning:ro
|
||||
- ./grafana/dashboards:/var/lib/grafana/dashboards:ro
|
||||
- grafana-data:/var/lib/grafana
|
||||
depends_on: [prometheus]
|
||||
|
||||
networks:
|
||||
frontend: {}
|
||||
backend:
|
||||
internal: true
|
||||
|
||||
volumes:
|
||||
postgres-data: {}
|
||||
redis-data: {}
|
||||
prometheus-data: {}
|
||||
grafana-data: {}
|
||||
30
deploy/docker/Dockerfile
Normal file
30
deploy/docker/Dockerfile
Normal file
@ -0,0 +1,30 @@
|
||||
# syntax=docker/dockerfile:1.7
|
||||
FROM golang:1.26-bookworm AS build
|
||||
|
||||
WORKDIR /src
|
||||
COPY go.mod go.sum ./
|
||||
RUN go mod download
|
||||
COPY . .
|
||||
|
||||
ARG TARGETOS=linux
|
||||
ARG TARGETARCH=amd64
|
||||
RUN CGO_ENABLED=0 GOOS=${TARGETOS} GOARCH=${TARGETARCH} \
|
||||
go build -trimpath -ldflags="-s -w" -o /out/proxy-gateway ./cmd/proxy-gateway && \
|
||||
CGO_ENABLED=0 GOOS=${TARGETOS} GOARCH=${TARGETARCH} \
|
||||
go build -trimpath -ldflags="-s -w" -o /out/proxy-controller ./cmd/proxy-controller && \
|
||||
CGO_ENABLED=0 GOOS=${TARGETOS} GOARCH=${TARGETARCH} \
|
||||
go build -trimpath -ldflags="-s -w" -o /out/proxy-checker ./cmd/proxy-checker && \
|
||||
CGO_ENABLED=0 GOOS=${TARGETOS} GOARCH=${TARGETARCH} \
|
||||
go build -trimpath -ldflags="-s -w" -o /out/proxy-loadgen ./cmd/proxy-loadgen
|
||||
|
||||
FROM debian:bookworm-slim
|
||||
|
||||
RUN apt-get update && \
|
||||
apt-get install -y --no-install-recommends ca-certificates curl tini && \
|
||||
rm -rf /var/lib/apt/lists/* && \
|
||||
useradd --uid 10001 --create-home --shell /usr/sbin/nologin proxy-pool
|
||||
|
||||
COPY --from=build /out/ /usr/local/bin/
|
||||
USER 10001:10001
|
||||
ENTRYPOINT ["/usr/bin/tini", "--"]
|
||||
|
||||
21
deploy/grafana/dashboards/proxy-pool-overview.json
Normal file
21
deploy/grafana/dashboards/proxy-pool-overview.json
Normal file
@ -0,0 +1,21 @@
|
||||
{
|
||||
"annotations": {"list": []},
|
||||
"editable": true,
|
||||
"graphTooltip": 1,
|
||||
"panels": [
|
||||
{"type":"timeseries","title":"Gateway QPS","gridPos":{"h":8,"w":8,"x":0,"y":0},"targets":[{"expr":"sum(rate(proxy_pool_gateway_requests_total[1m]))","legendFormat":"QPS"}]},
|
||||
{"type":"timeseries","title":"Gateway p99","gridPos":{"h":8,"w":8,"x":8,"y":0},"targets":[{"expr":"histogram_quantile(0.99, sum by (le) (rate(proxy_pool_gateway_request_duration_seconds_bucket[5m])))","legendFormat":"p99"}]},
|
||||
{"type":"timeseries","title":"Available Slots","gridPos":{"h":8,"w":8,"x":16,"y":0},"targets":[{"expr":"sum(proxy_pool_available_slots)","legendFormat":"slots"}]},
|
||||
{"type":"timeseries","title":"Provider Fetch","gridPos":{"h":8,"w":12,"x":0,"y":8},"targets":[{"expr":"sum by (result) (rate(proxy_pool_provider_fetch_total[5m]))","legendFormat":"{{result}}"}]},
|
||||
{"type":"timeseries","title":"Extraction","gridPos":{"h":8,"w":12,"x":12,"y":8},"targets":[{"expr":"sum by (result) (rate(proxy_pool_extraction_total[5m]))","legendFormat":"{{result}}"}]},
|
||||
{"type":"timeseries","title":"Snapshot Age","gridPos":{"h":8,"w":12,"x":0,"y":16},"targets":[{"expr":"max by (worker) (proxy_pool_snapshot_age_seconds)","legendFormat":"{{worker}}"}]},
|
||||
{"type":"timeseries","title":"Checker Queue","gridPos":{"h":8,"w":12,"x":12,"y":16},"targets":[{"expr":"sum(proxy_pool_checker_queue_depth)","legendFormat":"depth"}]}
|
||||
],
|
||||
"schemaVersion": 41,
|
||||
"tags": ["proxy-pool"],
|
||||
"templating": {"list": []},
|
||||
"time": {"from":"now-6h","to":"now"},
|
||||
"title": "Proxy Pool Overview",
|
||||
"uid": "proxy-pool-overview",
|
||||
"version": 1
|
||||
}
|
||||
11
deploy/grafana/provisioning/dashboards/dashboards.yml
Normal file
11
deploy/grafana/provisioning/dashboards/dashboards.yml
Normal file
@ -0,0 +1,11 @@
|
||||
apiVersion: 1
|
||||
providers:
|
||||
- name: proxy-pool
|
||||
orgId: 1
|
||||
folder: Proxy Pool
|
||||
type: file
|
||||
disableDeletion: true
|
||||
updateIntervalSeconds: 30
|
||||
options:
|
||||
path: /var/lib/grafana/dashboards
|
||||
|
||||
10
deploy/grafana/provisioning/datasources/prometheus.yml
Normal file
10
deploy/grafana/provisioning/datasources/prometheus.yml
Normal file
@ -0,0 +1,10 @@
|
||||
apiVersion: 1
|
||||
datasources:
|
||||
- name: Prometheus
|
||||
uid: prometheus
|
||||
type: prometheus
|
||||
access: proxy
|
||||
url: http://prometheus:9090
|
||||
isDefault: true
|
||||
editable: false
|
||||
|
||||
39
deploy/haproxy/haproxy.cfg
Normal file
39
deploy/haproxy/haproxy.cfg
Normal file
@ -0,0 +1,39 @@
|
||||
global
|
||||
log stdout format raw local0
|
||||
maxconn 100000
|
||||
hard-stop-after 45s
|
||||
|
||||
defaults
|
||||
log global
|
||||
mode tcp
|
||||
option tcplog
|
||||
timeout connect 3s
|
||||
timeout client 2m
|
||||
timeout server 2m
|
||||
timeout tunnel 1h
|
||||
|
||||
frontend proxy_gateway
|
||||
bind :8080
|
||||
default_backend gateway_workers
|
||||
|
||||
backend gateway_workers
|
||||
balance leastconn
|
||||
option tcp-check
|
||||
default-server inter 2s fall 3 rise 2 slowstart 10s
|
||||
server gateway-a gateway-a:8080 check resolvers docker init-addr libc,none
|
||||
server gateway-b gateway-b:8080 check resolvers docker init-addr libc,none
|
||||
|
||||
resolvers docker
|
||||
nameserver dns 127.0.0.11:53
|
||||
resolve_retries 3
|
||||
timeout resolve 1s
|
||||
timeout retry 1s
|
||||
hold valid 10s
|
||||
|
||||
frontend stats
|
||||
mode http
|
||||
bind :8404
|
||||
http-request use-service prometheus-exporter if { path /metrics }
|
||||
http-request return status 200 content-type text/plain string ok if { path /healthz }
|
||||
stats enable
|
||||
stats uri /stats
|
||||
47
deploy/kubernetes/base/autoscaling.yaml
Normal file
47
deploy/kubernetes/base/autoscaling.yaml
Normal file
@ -0,0 +1,47 @@
|
||||
apiVersion: autoscaling/v2
|
||||
kind: HorizontalPodAutoscaler
|
||||
metadata: {name: proxy-gateway, namespace: proxy-pool}
|
||||
spec:
|
||||
scaleTargetRef: {apiVersion: apps/v1, kind: Deployment, name: proxy-gateway}
|
||||
minReplicas: 6
|
||||
maxReplicas: 60
|
||||
behavior:
|
||||
scaleUp:
|
||||
stabilizationWindowSeconds: 0
|
||||
policies:
|
||||
- {type: Percent, value: 100, periodSeconds: 30}
|
||||
- {type: Pods, value: 8, periodSeconds: 30}
|
||||
selectPolicy: Max
|
||||
scaleDown:
|
||||
stabilizationWindowSeconds: 600
|
||||
policies: [{type: Percent, value: 10, periodSeconds: 60}]
|
||||
metrics:
|
||||
- type: Resource
|
||||
resource:
|
||||
name: cpu
|
||||
target: {type: Utilization, averageUtilization: 55}
|
||||
- type: Resource
|
||||
resource:
|
||||
name: memory
|
||||
target: {type: Utilization, averageUtilization: 65}
|
||||
---
|
||||
apiVersion: autoscaling/v2
|
||||
kind: HorizontalPodAutoscaler
|
||||
metadata: {name: proxy-checker, namespace: proxy-pool}
|
||||
spec:
|
||||
scaleTargetRef: {apiVersion: apps/v1, kind: Deployment, name: proxy-checker}
|
||||
minReplicas: 3
|
||||
maxReplicas: 30
|
||||
behavior:
|
||||
scaleUp:
|
||||
stabilizationWindowSeconds: 0
|
||||
policies: [{type: Percent, value: 100, periodSeconds: 30}]
|
||||
scaleDown:
|
||||
stabilizationWindowSeconds: 300
|
||||
policies: [{type: Percent, value: 20, periodSeconds: 60}]
|
||||
metrics:
|
||||
- type: Resource
|
||||
resource:
|
||||
name: cpu
|
||||
target: {type: Utilization, averageUtilization: 60}
|
||||
|
||||
21
deploy/kubernetes/base/availability.yaml
Normal file
21
deploy/kubernetes/base/availability.yaml
Normal file
@ -0,0 +1,21 @@
|
||||
apiVersion: policy/v1
|
||||
kind: PodDisruptionBudget
|
||||
metadata: {name: proxy-gateway, namespace: proxy-pool}
|
||||
spec:
|
||||
maxUnavailable: 1
|
||||
selector: {matchLabels: {app.kubernetes.io/name: proxy-gateway}}
|
||||
---
|
||||
apiVersion: policy/v1
|
||||
kind: PodDisruptionBudget
|
||||
metadata: {name: proxy-controller, namespace: proxy-pool}
|
||||
spec:
|
||||
minAvailable: 2
|
||||
selector: {matchLabels: {app.kubernetes.io/name: proxy-controller}}
|
||||
---
|
||||
apiVersion: policy/v1
|
||||
kind: PodDisruptionBudget
|
||||
metadata: {name: proxy-checker, namespace: proxy-pool}
|
||||
spec:
|
||||
minAvailable: 2
|
||||
selector: {matchLabels: {app.kubernetes.io/name: proxy-checker}}
|
||||
|
||||
66
deploy/kubernetes/base/checker.yaml
Normal file
66
deploy/kubernetes/base/checker.yaml
Normal file
@ -0,0 +1,66 @@
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: proxy-checker
|
||||
namespace: proxy-pool
|
||||
labels: {app.kubernetes.io/name: proxy-checker, app.kubernetes.io/part-of: proxy-pool}
|
||||
spec:
|
||||
replicas: 3
|
||||
minReadySeconds: 5
|
||||
revisionHistoryLimit: 3
|
||||
strategy:
|
||||
type: RollingUpdate
|
||||
rollingUpdate: {maxUnavailable: 1, maxSurge: 1}
|
||||
selector:
|
||||
matchLabels: {app.kubernetes.io/name: proxy-checker}
|
||||
template:
|
||||
metadata:
|
||||
labels: {app.kubernetes.io/name: proxy-checker, app.kubernetes.io/part-of: proxy-pool}
|
||||
annotations: {prometheus.io/scrape: "true", prometheus.io/port: "9090", prometheus.io/path: /metrics}
|
||||
spec:
|
||||
serviceAccountName: proxy-pool
|
||||
automountServiceAccountToken: false
|
||||
terminationGracePeriodSeconds: 45
|
||||
securityContext: {runAsNonRoot: true, seccompProfile: {type: RuntimeDefault}}
|
||||
topologySpreadConstraints:
|
||||
- maxSkew: 1
|
||||
topologyKey: kubernetes.io/hostname
|
||||
whenUnsatisfiable: ScheduleAnyway
|
||||
labelSelector: {matchLabels: {app.kubernetes.io/name: proxy-checker}}
|
||||
containers:
|
||||
- name: checker
|
||||
image: REGISTRY/proxy-pool:VERSION
|
||||
imagePullPolicy: IfNotPresent
|
||||
command: [proxy-checker]
|
||||
env:
|
||||
- {name: PROXY_POOL_CONFIG, value: /etc/proxy-pool/config.yaml}
|
||||
envFrom:
|
||||
- secretRef: {name: proxy-pool-secrets}
|
||||
ports:
|
||||
- {name: metrics, containerPort: 9090}
|
||||
readinessProbe:
|
||||
httpGet: {path: /readyz, port: metrics}
|
||||
periodSeconds: 5
|
||||
timeoutSeconds: 2
|
||||
livenessProbe:
|
||||
httpGet: {path: /livez, port: metrics}
|
||||
periodSeconds: 10
|
||||
timeoutSeconds: 2
|
||||
lifecycle:
|
||||
preStop: {exec: {command: [sh, -c, "sleep 3"]}}
|
||||
resources:
|
||||
requests: {cpu: "1", memory: 512Mi}
|
||||
limits: {cpu: "2", memory: 1Gi}
|
||||
securityContext:
|
||||
allowPrivilegeEscalation: false
|
||||
readOnlyRootFilesystem: true
|
||||
capabilities: {drop: [ALL]}
|
||||
volumeMounts:
|
||||
- {name: config, mountPath: /etc/proxy-pool, readOnly: true}
|
||||
- {name: tmp, mountPath: /tmp}
|
||||
volumes:
|
||||
- name: config
|
||||
configMap: {name: proxy-pool-config}
|
||||
- name: tmp
|
||||
emptyDir: {sizeLimit: 64Mi}
|
||||
|
||||
172
deploy/kubernetes/base/configmap.yaml
Normal file
172
deploy/kubernetes/base/configmap.yaml
Normal file
@ -0,0 +1,172 @@
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: proxy-pool-config
|
||||
namespace: proxy-pool
|
||||
data:
|
||||
config.yaml: |
|
||||
version: 1
|
||||
security:
|
||||
requireProtectionOnPublicListen: true
|
||||
gateway:
|
||||
enabled: true
|
||||
listen: 0.0.0.0:8080
|
||||
access:
|
||||
allowCIDRs: [0.0.0.0/0]
|
||||
trustedProxies: []
|
||||
auth:
|
||||
mode: usernamePassword
|
||||
username: env:PROXY_POOL_GATEWAY_USERNAME
|
||||
password: env:PROXY_POOL_GATEWAY_PASSWORD
|
||||
limits:
|
||||
maxConcurrentConnections: 100000
|
||||
requestsPerMinutePerClient: 60000
|
||||
retry:
|
||||
maxAttempts: 2
|
||||
retryMethods: [GET, HEAD]
|
||||
destinationPolicy:
|
||||
denyPrivateNetworks: true
|
||||
denyLoopback: true
|
||||
denyLinkLocal: true
|
||||
denyCIDRs: [169.254.169.254/32]
|
||||
distribution:
|
||||
enabled: true
|
||||
listen: 0.0.0.0:8081
|
||||
access:
|
||||
allowCIDRs: [10.0.0.0/8]
|
||||
trustedProxies: [10.0.0.0/8]
|
||||
auth:
|
||||
mode: apiKey
|
||||
header: X-API-Key
|
||||
token: env:PROXY_POOL_EXTRACT_TOKEN
|
||||
limits:
|
||||
requestsPerMinute: 30000
|
||||
requestsPerMinutePerClient: 3000
|
||||
clientIdentification:
|
||||
mode: trustedProxyOrRemoteIP
|
||||
extraction:
|
||||
fulfillment: partial
|
||||
maxCountPerRequest: 100
|
||||
minRemainingTTL: 30s
|
||||
maxHealthCheckAge: 30s
|
||||
reserveForGateway: 5000
|
||||
admin:
|
||||
enabled: true
|
||||
listen: 0.0.0.0:8082
|
||||
access:
|
||||
allowCIDRs: [10.0.0.0/8]
|
||||
auth:
|
||||
mode: apiKey
|
||||
header: X-Admin-Token
|
||||
token: env:PROXY_POOL_ADMIN_TOKEN
|
||||
metrics:
|
||||
enabled: true
|
||||
listen: 0.0.0.0:9090
|
||||
storage:
|
||||
postgresURL: env:PROXY_POOL_POSTGRES_URL
|
||||
redisURL: env:PROXY_POOL_REDIS_URL
|
||||
routing:
|
||||
- name: gateway-default
|
||||
enabled: true
|
||||
purpose: gateway
|
||||
upstreams: [provider-a, provider-b]
|
||||
strategy:
|
||||
type: sequential
|
||||
switchAfterEmptyFetch: 5
|
||||
endBehavior: stayLast
|
||||
onUnavailable:
|
||||
action: reject
|
||||
- name: extract-default
|
||||
enabled: true
|
||||
purpose: extract
|
||||
upstreams: [provider-a, provider-b]
|
||||
strategy:
|
||||
type: sequential
|
||||
switchAfterEmptyFetch: 5
|
||||
endBehavior: stayLast
|
||||
onUnavailable:
|
||||
action: reject
|
||||
upstreams:
|
||||
provider-a:
|
||||
enabled: true
|
||||
exposure: [gateway, extract]
|
||||
provider:
|
||||
billingMode: fetch
|
||||
protocols: [http]
|
||||
api:
|
||||
url: https://PROVIDER_A_HOST/api/proxies
|
||||
method: GET
|
||||
auth:
|
||||
mode: apiKey
|
||||
header: Authorization
|
||||
token: env:PROVIDER_A_TOKEN
|
||||
template: '{{ . }}'
|
||||
proxyAuth:
|
||||
mode: response
|
||||
pool:
|
||||
maxSize: 25000
|
||||
shrinkDelay: 30s
|
||||
capacity:
|
||||
maxConcurrencyPerProxy: 20
|
||||
lifecycle:
|
||||
ttl: 5m
|
||||
allocationSafetyMargin: 20s
|
||||
fetch:
|
||||
requestInterval: 1s
|
||||
timeout: 3s
|
||||
maxAttempts: 3
|
||||
maxInFlight: 1
|
||||
maxTotal: 1000000
|
||||
maxResponseBytes: 4194304
|
||||
templateTimeout: 100ms
|
||||
retry: {initial: 500ms, max: 30s, jitter: 20}
|
||||
check:
|
||||
interval: 30s
|
||||
jitter: 20
|
||||
maxInFlight: 500
|
||||
timeout: 3s
|
||||
maxAttempts: 2
|
||||
maxConsecutiveFailures: 3
|
||||
urls: [https://example.com/]
|
||||
provider-b:
|
||||
enabled: true
|
||||
exposure: [gateway, extract]
|
||||
provider:
|
||||
billingMode: fetch
|
||||
protocols: [http]
|
||||
api:
|
||||
url: https://PROVIDER_B_HOST/api/proxies
|
||||
method: GET
|
||||
auth:
|
||||
mode: apiKey
|
||||
header: Authorization
|
||||
token: env:PROVIDER_B_TOKEN
|
||||
template: '{{ . }}'
|
||||
proxyAuth:
|
||||
mode: response
|
||||
pool:
|
||||
maxSize: 25000
|
||||
shrinkDelay: 30s
|
||||
capacity:
|
||||
maxConcurrencyPerProxy: 20
|
||||
lifecycle:
|
||||
ttl: 5m
|
||||
allocationSafetyMargin: 20s
|
||||
fetch:
|
||||
requestInterval: 1s
|
||||
timeout: 3s
|
||||
maxAttempts: 3
|
||||
maxInFlight: 1
|
||||
maxTotal: 1000000
|
||||
maxResponseBytes: 4194304
|
||||
templateTimeout: 100ms
|
||||
retry: {initial: 500ms, max: 30s, jitter: 20}
|
||||
check:
|
||||
interval: 30s
|
||||
jitter: 20
|
||||
maxInFlight: 500
|
||||
timeout: 3s
|
||||
maxAttempts: 2
|
||||
maxConsecutiveFailures: 3
|
||||
urls: [https://example.com/]
|
||||
|
||||
92
deploy/kubernetes/base/controller.yaml
Normal file
92
deploy/kubernetes/base/controller.yaml
Normal file
@ -0,0 +1,92 @@
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: proxy-controller
|
||||
namespace: proxy-pool
|
||||
labels: {app.kubernetes.io/name: proxy-controller, app.kubernetes.io/part-of: proxy-pool}
|
||||
spec:
|
||||
replicas: 3
|
||||
minReadySeconds: 10
|
||||
revisionHistoryLimit: 3
|
||||
strategy:
|
||||
type: RollingUpdate
|
||||
rollingUpdate: {maxUnavailable: 1, maxSurge: 1}
|
||||
selector:
|
||||
matchLabels: {app.kubernetes.io/name: proxy-controller}
|
||||
template:
|
||||
metadata:
|
||||
labels: {app.kubernetes.io/name: proxy-controller, app.kubernetes.io/part-of: proxy-pool}
|
||||
annotations: {prometheus.io/scrape: "true", prometheus.io/port: "9090", prometheus.io/path: /metrics}
|
||||
spec:
|
||||
serviceAccountName: proxy-pool
|
||||
automountServiceAccountToken: false
|
||||
terminationGracePeriodSeconds: 60
|
||||
securityContext: {runAsNonRoot: true, seccompProfile: {type: RuntimeDefault}}
|
||||
affinity:
|
||||
podAntiAffinity:
|
||||
preferredDuringSchedulingIgnoredDuringExecution:
|
||||
- weight: 100
|
||||
podAffinityTerm:
|
||||
topologyKey: kubernetes.io/hostname
|
||||
labelSelector: {matchLabels: {app.kubernetes.io/name: proxy-controller}}
|
||||
containers:
|
||||
- name: controller
|
||||
image: REGISTRY/proxy-pool:VERSION
|
||||
imagePullPolicy: IfNotPresent
|
||||
command: [proxy-controller]
|
||||
env:
|
||||
- {name: PROXY_POOL_CONFIG, value: /etc/proxy-pool/config.yaml}
|
||||
envFrom:
|
||||
- secretRef: {name: proxy-pool-secrets}
|
||||
ports:
|
||||
- {name: distribution, containerPort: 8081}
|
||||
- {name: admin, containerPort: 8082}
|
||||
- {name: control, containerPort: 8443}
|
||||
- {name: metrics, containerPort: 9090}
|
||||
readinessProbe:
|
||||
httpGet: {path: /readyz, port: metrics}
|
||||
periodSeconds: 5
|
||||
timeoutSeconds: 2
|
||||
failureThreshold: 3
|
||||
livenessProbe:
|
||||
httpGet: {path: /livez, port: metrics}
|
||||
periodSeconds: 10
|
||||
timeoutSeconds: 2
|
||||
failureThreshold: 3
|
||||
startupProbe:
|
||||
httpGet: {path: /livez, port: metrics}
|
||||
periodSeconds: 2
|
||||
failureThreshold: 45
|
||||
lifecycle:
|
||||
preStop: {exec: {command: [sh, -c, "sleep 5"]}}
|
||||
resources:
|
||||
requests: {cpu: "1", memory: 1Gi}
|
||||
limits: {cpu: "2", memory: 2Gi}
|
||||
securityContext:
|
||||
allowPrivilegeEscalation: false
|
||||
readOnlyRootFilesystem: true
|
||||
capabilities: {drop: [ALL]}
|
||||
volumeMounts:
|
||||
- {name: config, mountPath: /etc/proxy-pool, readOnly: true}
|
||||
- {name: tmp, mountPath: /tmp}
|
||||
volumes:
|
||||
- name: config
|
||||
configMap: {name: proxy-pool-config}
|
||||
- name: tmp
|
||||
emptyDir: {sizeLimit: 64Mi}
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: proxy-controller
|
||||
namespace: proxy-pool
|
||||
labels: {app.kubernetes.io/name: proxy-controller}
|
||||
spec:
|
||||
type: ClusterIP
|
||||
selector: {app.kubernetes.io/name: proxy-controller}
|
||||
ports:
|
||||
- {name: distribution, port: 8081, targetPort: distribution}
|
||||
- {name: admin, port: 8082, targetPort: admin}
|
||||
- {name: control, port: 8443, targetPort: control}
|
||||
- {name: metrics, port: 9090, targetPort: metrics}
|
||||
|
||||
92
deploy/kubernetes/base/gateway.yaml
Normal file
92
deploy/kubernetes/base/gateway.yaml
Normal file
@ -0,0 +1,92 @@
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: proxy-gateway
|
||||
namespace: proxy-pool
|
||||
labels: {app.kubernetes.io/name: proxy-gateway, app.kubernetes.io/part-of: proxy-pool}
|
||||
spec:
|
||||
replicas: 6
|
||||
minReadySeconds: 10
|
||||
revisionHistoryLimit: 3
|
||||
strategy:
|
||||
type: RollingUpdate
|
||||
rollingUpdate: {maxUnavailable: 1, maxSurge: 2}
|
||||
selector:
|
||||
matchLabels: {app.kubernetes.io/name: proxy-gateway}
|
||||
template:
|
||||
metadata:
|
||||
labels: {app.kubernetes.io/name: proxy-gateway, app.kubernetes.io/part-of: proxy-pool}
|
||||
annotations: {prometheus.io/scrape: "true", prometheus.io/port: "9090", prometheus.io/path: /metrics}
|
||||
spec:
|
||||
serviceAccountName: proxy-pool
|
||||
automountServiceAccountToken: false
|
||||
terminationGracePeriodSeconds: 60
|
||||
securityContext: {runAsNonRoot: true, seccompProfile: {type: RuntimeDefault}}
|
||||
topologySpreadConstraints:
|
||||
- maxSkew: 1
|
||||
topologyKey: topology.kubernetes.io/zone
|
||||
whenUnsatisfiable: DoNotSchedule
|
||||
labelSelector: {matchLabels: {app.kubernetes.io/name: proxy-gateway}}
|
||||
- maxSkew: 1
|
||||
topologyKey: kubernetes.io/hostname
|
||||
whenUnsatisfiable: ScheduleAnyway
|
||||
labelSelector: {matchLabels: {app.kubernetes.io/name: proxy-gateway}}
|
||||
containers:
|
||||
- name: gateway
|
||||
image: REGISTRY/proxy-pool:VERSION
|
||||
imagePullPolicy: IfNotPresent
|
||||
command: [proxy-gateway]
|
||||
env:
|
||||
- {name: PROXY_POOL_CONFIG, value: /etc/proxy-pool/config.yaml}
|
||||
envFrom:
|
||||
- secretRef: {name: proxy-pool-secrets}
|
||||
ports:
|
||||
- {name: proxy, containerPort: 8080, protocol: TCP}
|
||||
- {name: metrics, containerPort: 9090, protocol: TCP}
|
||||
readinessProbe:
|
||||
httpGet: {path: /readyz, port: metrics}
|
||||
periodSeconds: 3
|
||||
timeoutSeconds: 1
|
||||
failureThreshold: 3
|
||||
livenessProbe:
|
||||
httpGet: {path: /livez, port: metrics}
|
||||
periodSeconds: 10
|
||||
timeoutSeconds: 2
|
||||
failureThreshold: 3
|
||||
startupProbe:
|
||||
httpGet: {path: /livez, port: metrics}
|
||||
periodSeconds: 2
|
||||
failureThreshold: 30
|
||||
lifecycle:
|
||||
preStop: {exec: {command: [sh, -c, "sleep 5"]}}
|
||||
resources:
|
||||
requests: {cpu: "2", memory: 1Gi}
|
||||
limits: {cpu: "4", memory: 2Gi}
|
||||
securityContext:
|
||||
allowPrivilegeEscalation: false
|
||||
readOnlyRootFilesystem: true
|
||||
capabilities: {drop: [ALL]}
|
||||
volumeMounts:
|
||||
- {name: config, mountPath: /etc/proxy-pool, readOnly: true}
|
||||
- {name: tmp, mountPath: /tmp}
|
||||
volumes:
|
||||
- name: config
|
||||
configMap: {name: proxy-pool-config}
|
||||
- name: tmp
|
||||
emptyDir: {sizeLimit: 64Mi}
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: proxy-gateway
|
||||
namespace: proxy-pool
|
||||
labels: {app.kubernetes.io/name: proxy-gateway}
|
||||
annotations:
|
||||
service.beta.kubernetes.io/aws-load-balancer-type: nlb
|
||||
spec:
|
||||
type: LoadBalancer
|
||||
externalTrafficPolicy: Local
|
||||
selector: {app.kubernetes.io/name: proxy-gateway}
|
||||
ports:
|
||||
- {name: proxy, port: 8080, targetPort: proxy, protocol: TCP}
|
||||
|
||||
17
deploy/kubernetes/base/kustomization.yaml
Normal file
17
deploy/kubernetes/base/kustomization.yaml
Normal file
@ -0,0 +1,17 @@
|
||||
apiVersion: kustomize.config.k8s.io/v1beta1
|
||||
kind: Kustomization
|
||||
resources:
|
||||
- namespace.yaml
|
||||
- serviceaccount.yaml
|
||||
- configmap.yaml
|
||||
- gateway.yaml
|
||||
- controller.yaml
|
||||
- checker.yaml
|
||||
- availability.yaml
|
||||
- autoscaling.yaml
|
||||
- networkpolicy.yaml
|
||||
images:
|
||||
- name: REGISTRY/proxy-pool
|
||||
newName: REGISTRY/proxy-pool
|
||||
newTag: VERSION
|
||||
|
||||
10
deploy/kubernetes/base/namespace.yaml
Normal file
10
deploy/kubernetes/base/namespace.yaml
Normal file
@ -0,0 +1,10 @@
|
||||
apiVersion: v1
|
||||
kind: Namespace
|
||||
metadata:
|
||||
name: proxy-pool
|
||||
labels:
|
||||
app.kubernetes.io/part-of: proxy-pool
|
||||
pod-security.kubernetes.io/enforce: restricted
|
||||
pod-security.kubernetes.io/audit: restricted
|
||||
pod-security.kubernetes.io/warn: restricted
|
||||
|
||||
52
deploy/kubernetes/base/networkpolicy.yaml
Normal file
52
deploy/kubernetes/base/networkpolicy.yaml
Normal file
@ -0,0 +1,52 @@
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: NetworkPolicy
|
||||
metadata: {name: default-deny, namespace: proxy-pool}
|
||||
spec:
|
||||
podSelector: {}
|
||||
policyTypes: [Ingress, Egress]
|
||||
---
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: NetworkPolicy
|
||||
metadata: {name: gateway-traffic, namespace: proxy-pool}
|
||||
spec:
|
||||
podSelector: {matchLabels: {app.kubernetes.io/name: proxy-gateway}}
|
||||
policyTypes: [Ingress, Egress]
|
||||
ingress:
|
||||
- ports: [{port: 8080, protocol: TCP}]
|
||||
- from:
|
||||
- namespaceSelector: {matchLabels: {kubernetes.io/metadata.name: monitoring}}
|
||||
ports: [{port: 9090, protocol: TCP}]
|
||||
egress:
|
||||
- to:
|
||||
- namespaceSelector: {matchLabels: {kubernetes.io/metadata.name: kube-system}}
|
||||
ports: [{port: 53, protocol: UDP}, {port: 53, protocol: TCP}]
|
||||
- to:
|
||||
- podSelector: {matchLabels: {app.kubernetes.io/name: proxy-controller}}
|
||||
ports: [{port: 8443, protocol: TCP}]
|
||||
# Gateway 需要连接任意公网目标;应用层 DestinationPolicy 仍拒绝私网、回环和元数据地址。
|
||||
- to:
|
||||
- ipBlock:
|
||||
cidr: 0.0.0.0/0
|
||||
except: [10.0.0.0/8, 100.64.0.0/10, 127.0.0.0/8, 169.254.0.0/16, 172.16.0.0/12, 192.168.0.0/16]
|
||||
---
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: NetworkPolicy
|
||||
metadata: {name: control-plane-traffic, namespace: proxy-pool}
|
||||
spec:
|
||||
podSelector:
|
||||
matchExpressions:
|
||||
- {key: app.kubernetes.io/name, operator: In, values: [proxy-controller, proxy-checker]}
|
||||
policyTypes: [Ingress, Egress]
|
||||
ingress:
|
||||
- from:
|
||||
- namespaceSelector: {matchLabels: {kubernetes.io/metadata.name: proxy-pool}}
|
||||
- from:
|
||||
- namespaceSelector: {matchLabels: {kubernetes.io/metadata.name: monitoring}}
|
||||
ports: [{port: 9090, protocol: TCP}]
|
||||
egress:
|
||||
- to:
|
||||
- namespaceSelector: {matchLabels: {kubernetes.io/metadata.name: kube-system}}
|
||||
ports: [{port: 53, protocol: UDP}, {port: 53, protocol: TCP}]
|
||||
# Provider、健康目标及外部托管 PostgreSQL/Redis 的精确网段应在环境 Overlay 收紧。
|
||||
- to: [{ipBlock: {cidr: 0.0.0.0/0}}]
|
||||
|
||||
16
deploy/kubernetes/base/secret.example.yaml
Normal file
16
deploy/kubernetes/base/secret.example.yaml
Normal file
@ -0,0 +1,16 @@
|
||||
apiVersion: v1
|
||||
kind: Secret
|
||||
metadata:
|
||||
name: proxy-pool-secrets
|
||||
namespace: proxy-pool
|
||||
type: Opaque
|
||||
stringData:
|
||||
PROXY_POOL_GATEWAY_USERNAME: GATEWAY_USER
|
||||
PROXY_POOL_GATEWAY_PASSWORD: GATEWAY_PASSWORD
|
||||
PROXY_POOL_EXTRACT_TOKEN: EXTRACT_TOKEN
|
||||
PROXY_POOL_ADMIN_TOKEN: ADMIN_TOKEN
|
||||
PROXY_POOL_POSTGRES_URL: postgres://USER:PASSWORD@POSTGRES_HOST:5432/proxy_pool?sslmode=verify-full
|
||||
PROXY_POOL_REDIS_URL: rediss://:PASSWORD@REDIS_HOST:6379/0
|
||||
PROVIDER_A_TOKEN: PROVIDER_A_TOKEN
|
||||
PROVIDER_B_TOKEN: PROVIDER_B_TOKEN
|
||||
|
||||
7
deploy/kubernetes/base/serviceaccount.yaml
Normal file
7
deploy/kubernetes/base/serviceaccount.yaml
Normal file
@ -0,0 +1,7 @@
|
||||
apiVersion: v1
|
||||
kind: ServiceAccount
|
||||
metadata:
|
||||
name: proxy-pool
|
||||
namespace: proxy-pool
|
||||
automountServiceAccountToken: false
|
||||
|
||||
24
deploy/prometheus/prometheus.yml
Normal file
24
deploy/prometheus/prometheus.yml
Normal file
@ -0,0 +1,24 @@
|
||||
global:
|
||||
scrape_interval: 15s
|
||||
evaluation_interval: 15s
|
||||
external_labels:
|
||||
environment: local
|
||||
|
||||
rule_files:
|
||||
- /etc/prometheus/rules/*.yml
|
||||
|
||||
scrape_configs:
|
||||
- job_name: proxy-gateway
|
||||
static_configs:
|
||||
- targets: [gateway-a:9090, gateway-b:9090]
|
||||
- job_name: proxy-controller
|
||||
static_configs:
|
||||
- targets: [controller:9090]
|
||||
- job_name: proxy-checker
|
||||
static_configs:
|
||||
- targets: [checker:9090]
|
||||
- job_name: haproxy
|
||||
metrics_path: /metrics
|
||||
static_configs:
|
||||
- targets: [haproxy:8404]
|
||||
|
||||
48
deploy/prometheus/rules/proxy-pool.yml
Normal file
48
deploy/prometheus/rules/proxy-pool.yml
Normal file
@ -0,0 +1,48 @@
|
||||
groups:
|
||||
- name: proxy-pool
|
||||
rules:
|
||||
- alert: ProxyPoolGatewayHighErrorRate
|
||||
expr: |
|
||||
sum(rate(proxy_pool_gateway_requests_total{result="error"}[5m]))
|
||||
/ clamp_min(sum(rate(proxy_pool_gateway_requests_total[5m])), 1) > 0.02
|
||||
for: 10m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: Gateway 错误率持续高于 2%
|
||||
- alert: ProxyPoolGatewaySnapshotStale
|
||||
expr: proxy_pool_snapshot_age_seconds > 60
|
||||
for: 2m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: Gateway Snapshot 已超过安全陈旧时间
|
||||
- alert: ProxyPoolNoAvailableSlots
|
||||
expr: sum(proxy_pool_available_slots) == 0
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: Gateway 可分配容量耗尽
|
||||
- alert: ProxyPoolProviderFetchErrors
|
||||
expr: sum by (upstream) (rate(proxy_pool_provider_fetch_total{result="error"}[10m])) > 0.2
|
||||
for: 10m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: Provider Fetch 错误持续发生
|
||||
- alert: ProxyPoolCheckerBacklog
|
||||
expr: proxy_pool_checker_queue_depth > 10000
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: Checker 队列积压
|
||||
- alert: ProxyPoolExtractionConflict
|
||||
expr: sum(rate(proxy_pool_extraction_total{result="conflict"}[5m])) > 0
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: 独占提取发生持续事务冲突
|
||||
|
||||
29
deploy/tools/configcheck/main.go
Normal file
29
deploy/tools/configcheck/main.go
Normal file
@ -0,0 +1,29 @@
|
||||
// configcheck 使用与进程启动相同的严格加载器校验部署配置。
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
|
||||
"github.com/proxy-pool/proxy-pool/internal/config"
|
||||
)
|
||||
|
||||
func main() {
|
||||
if len(os.Args) != 2 {
|
||||
fmt.Fprintln(os.Stderr, "usage: configcheck CONFIG_FILE")
|
||||
os.Exit(2)
|
||||
}
|
||||
|
||||
file, err := os.Open(os.Args[1])
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "open config: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
if _, err := config.Load(file); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "invalid config: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
fmt.Printf("valid config: %s\n", os.Args[1])
|
||||
}
|
||||
536
diagrams/README.md
Normal file
536
diagrams/README.md
Normal file
@ -0,0 +1,536 @@
|
||||
# Proxy Pool Mermaid 图集
|
||||
|
||||
本图集依据最终需求语义绘制。图中的 100k QPS 是待压测验证的集群目标;Extract
|
||||
均表示一次性独占发放,不存在 Lease、Renew 或 Release。
|
||||
|
||||
## 01 系统上下文
|
||||
|
||||
```mermaid
|
||||
flowchart LR
|
||||
Client[Gateway Client] --> LB[Layer 4 Load Balancer]
|
||||
ExtractClient[Extract Client] --> Dist[Distribution API]
|
||||
Operator[Operator] --> Admin[Admin API]
|
||||
LB --> Gateway[Gateway Cluster]
|
||||
Gateway --> Internet[Target via Proxy]
|
||||
Dist --> Controller[Controller Cluster]
|
||||
Admin --> Controller
|
||||
Controller --> Provider[Provider APIs]
|
||||
Controller --> Checker[Checker Cluster]
|
||||
Controller --> PG[(PostgreSQL)]
|
||||
Controller --> Redis[(Redis)]
|
||||
```
|
||||
|
||||
## 02 进程职责边界
|
||||
|
||||
```mermaid
|
||||
flowchart TB
|
||||
subgraph DataPlane[Data Plane]
|
||||
G[proxy-gateway]
|
||||
Snap[Immutable Snapshot]
|
||||
G --> Snap
|
||||
end
|
||||
subgraph ControlPlane[Control Plane]
|
||||
C[proxy-controller]
|
||||
K[proxy-checker]
|
||||
C <--> K
|
||||
end
|
||||
subgraph Tools[Tools]
|
||||
L[proxy-loadgen]
|
||||
end
|
||||
C -->|Snapshot and ownership| G
|
||||
G -->|batched outcomes| C
|
||||
L --> G
|
||||
```
|
||||
|
||||
## 03 领域模块依赖
|
||||
|
||||
```mermaid
|
||||
flowchart TD
|
||||
Cmd[cmd assembly] --> Gateway[gateway modules]
|
||||
Cmd --> Controller[controller modules]
|
||||
Cmd --> Adapters[adapters]
|
||||
Gateway --> Domain[domain]
|
||||
Controller --> Domain
|
||||
Adapters --> Domain
|
||||
Domain -. no import .-> HTTP[(HTTP)]
|
||||
Domain -. no import .-> SQL[(SQL)]
|
||||
Domain -. no import .-> Redis[(Redis)]
|
||||
```
|
||||
|
||||
## 04 Gateway 请求路径
|
||||
|
||||
```mermaid
|
||||
sequenceDiagram
|
||||
participant C as Client
|
||||
participant G as Gateway
|
||||
participant D as Dispatcher
|
||||
participant T as Transport
|
||||
participant P as Proxy
|
||||
C->>G: HTTP request
|
||||
G->>G: auth and admission
|
||||
G->>D: acquire route
|
||||
D->>D: reserve capacity with CAS
|
||||
D-->>G: allocation
|
||||
G->>T: execute
|
||||
T->>P: dial and handshake
|
||||
P-->>T: response
|
||||
T-->>C: stream response
|
||||
T->>D: release active and report
|
||||
```
|
||||
|
||||
## 05 CONNECT 提交点
|
||||
|
||||
```mermaid
|
||||
stateDiagram-v2
|
||||
[*] --> Accepted
|
||||
Accepted --> Reserved: Acquire
|
||||
Reserved --> Dialing: dial proxy
|
||||
Dialing --> Cancelled: fail before commit
|
||||
Dialing --> Active: proxy CONNECT succeeds
|
||||
Active --> TunnelCommitted: send 200 to client
|
||||
TunnelCommitted --> Closed: stream ends
|
||||
Cancelled --> [*]
|
||||
Closed --> [*]
|
||||
note right of TunnelCommitted: transparent replay forbidden
|
||||
```
|
||||
|
||||
## 06 HTTP 安全重试决策
|
||||
|
||||
```mermaid
|
||||
flowchart TD
|
||||
F[Attempt failed] --> H{Headers sent to client?}
|
||||
H -->|yes| Stop[Do not retry]
|
||||
H -->|no| M{Method allowed?}
|
||||
M -->|no| Stop
|
||||
M -->|GET or HEAD| A{Attempts remain?}
|
||||
A -->|no| Stop
|
||||
A -->|yes| X[Exclude failed Proxy]
|
||||
X --> N[Acquire another Proxy]
|
||||
```
|
||||
|
||||
## 07 Routing 首条命中
|
||||
|
||||
```mermaid
|
||||
flowchart TD
|
||||
Req[RouteRequest] --> R1{Rule 1 matches?}
|
||||
R1 -->|yes| U1[Use Rule 1 upstream strategy]
|
||||
R1 -->|no| R2{Rule 2 matches?}
|
||||
R2 -->|yes| U2[Use Rule 2 upstream strategy]
|
||||
R2 -->|no| RN{Default rule matches?}
|
||||
RN -->|yes| UN[Use default strategy]
|
||||
RN -->|no| Reject[Apply onUnavailable]
|
||||
```
|
||||
|
||||
## 08 Sequential 原子切换
|
||||
|
||||
```mermaid
|
||||
sequenceDiagram
|
||||
participant F1 as Fetch goroutine 1
|
||||
participant F2 as Fetch goroutine 2
|
||||
participant U as Upstream A counter
|
||||
participant R as Routing state
|
||||
F1->>U: empty reaches threshold
|
||||
F2->>U: concurrent empty
|
||||
U->>R: depleted generation 7
|
||||
U->>R: depleted generation 7
|
||||
R->>R: CAS A to B succeeds once
|
||||
R-->>F1: current B
|
||||
R-->>F2: current B
|
||||
```
|
||||
|
||||
## 09 Provider Fetch 调度
|
||||
|
||||
```mermaid
|
||||
flowchart TD
|
||||
Signal[Capacity signal] --> SF{Fetch already running?}
|
||||
SF -->|yes| Merge[Merge into bounded signal]
|
||||
SF -->|no| Leader[Acquire logical leader]
|
||||
Leader --> Demand[Recompute slot demand]
|
||||
Demand --> Limit{Below maxSize and maxTotal?}
|
||||
Limit -->|no| Done[Stop]
|
||||
Limit -->|yes| Rate[Wait requestInterval]
|
||||
Rate --> Call[Call Provider under maxInFlight]
|
||||
Call --> Classify[Classify result]
|
||||
```
|
||||
|
||||
## 10 Fetch 结果分类
|
||||
|
||||
```mermaid
|
||||
flowchart LR
|
||||
Response[Provider response] --> Transport{Transport and auth valid?}
|
||||
Transport -->|no| Error[Error and backoff]
|
||||
Transport -->|yes| Parse{Template and parse valid?}
|
||||
Parse -->|no| Error
|
||||
Parse -->|yes| Legal{Legal candidates count}
|
||||
Legal -->|zero| Empty[Empty plus one]
|
||||
Legal -->|positive| New{New after dedupe?}
|
||||
New -->|none| Duplicate[Duplicate-only and reset Empty]
|
||||
New -->|some| Success[Success and reset Empty]
|
||||
```
|
||||
|
||||
## 11 退避与 Retry-After
|
||||
|
||||
```mermaid
|
||||
stateDiagram-v2
|
||||
[*] --> Ready
|
||||
Ready --> Calling: rate token acquired
|
||||
Calling --> Ready: success or empty
|
||||
Calling --> RetryAfter: HTTP 429
|
||||
Calling --> Backoff: timeout or server error
|
||||
RetryAfter --> Ready: provider deadline reached
|
||||
Backoff --> Ready: exponential delay plus jitter
|
||||
Backoff --> Open: max attempts exhausted
|
||||
Open --> Ready: next scheduled cycle
|
||||
```
|
||||
|
||||
## 12 Pool Reconcile
|
||||
|
||||
```mermaid
|
||||
flowchart TD
|
||||
Inventory[Managed inventory] --> Count[Count states and pending expected]
|
||||
Capacity[Available slots] --> Need[Compute demand]
|
||||
Count --> Bound{pool maxSize reached?}
|
||||
Need --> Bound
|
||||
Bound -->|yes| NoFetch[Do not fetch]
|
||||
Bound -->|no| Quota{fetch maxTotal reached?}
|
||||
Quota -->|yes| NoFetch
|
||||
Quota -->|no| Fetch[Schedule bounded fetch]
|
||||
```
|
||||
|
||||
## 13 Proxy 生命周期
|
||||
|
||||
```mermaid
|
||||
stateDiagram-v2
|
||||
[*] --> FETCHED
|
||||
FETCHED --> CHECKING
|
||||
CHECKING --> AVAILABLE: passed
|
||||
CHECKING --> UNHEALTHY: exhausted
|
||||
AVAILABLE --> SUSPECT: meaningful failure
|
||||
SUSPECT --> AVAILABLE: recheck passed
|
||||
SUSPECT --> UNHEALTHY: failures reached
|
||||
AVAILABLE --> DRAINING: expiry or revoke
|
||||
AVAILABLE --> EXTRACTED: exclusive transaction
|
||||
DRAINING --> EXPIRED: capacity zero
|
||||
UNHEALTHY --> REMOVED
|
||||
EXTRACTED --> EXPIRED: TTL reached
|
||||
EXPIRED --> REMOVED
|
||||
```
|
||||
|
||||
## 14 原子容量转换
|
||||
|
||||
```mermaid
|
||||
flowchart LR
|
||||
C0[active A reserved R] --> Check{A plus R below limit?}
|
||||
Check -->|no| Full[Reject candidate]
|
||||
Check -->|yes CAS| Reserved[active A reserved R plus 1]
|
||||
Reserved -->|Commit| Active[active A plus 1 reserved R]
|
||||
Reserved -->|Cancel| C0
|
||||
Active -->|Release| C0
|
||||
```
|
||||
|
||||
## 15 Worker 所有权
|
||||
|
||||
```mermaid
|
||||
flowchart TB
|
||||
Controller[Controller allocator] -->|epoch 12| W1[Worker 1]
|
||||
Controller -->|epoch 12| W2[Worker 2]
|
||||
P1[Proxy shard A] --> W1
|
||||
P2[Proxy shard B] --> W2
|
||||
U[Unowned inventory] --> Controller
|
||||
W1 -. cannot allocate .-> P2
|
||||
W2 -. cannot allocate .-> P1
|
||||
```
|
||||
|
||||
## 16 Snapshot 发布与 ACK
|
||||
|
||||
```mermaid
|
||||
sequenceDiagram
|
||||
participant DB as PostgreSQL
|
||||
participant C as Controller
|
||||
participant W as Worker
|
||||
DB->>C: outbox revision 42
|
||||
C->>C: build worker snapshot
|
||||
C->>W: epoch 8 version 42 checksum
|
||||
W->>W: validate and build indexes
|
||||
W->>W: atomic swap
|
||||
W-->>C: ACK epoch 8 version 42
|
||||
C->>DB: mark outbox delivered
|
||||
```
|
||||
|
||||
## 17 Snapshot 缺口恢复
|
||||
|
||||
```mermaid
|
||||
flowchart TD
|
||||
Delta[Receive delta version 45] --> Current{Current version is 44?}
|
||||
Current -->|yes| Check[Verify checksum and epoch]
|
||||
Current -->|no current 42| Reject[Reject delta]
|
||||
Reject --> Full[Request full snapshot]
|
||||
Full --> Build[Build indexes in background]
|
||||
Check --> Apply[Apply delta atomically]
|
||||
Build --> Apply
|
||||
```
|
||||
|
||||
## 18 Snapshot 陈旧状态
|
||||
|
||||
```mermaid
|
||||
stateDiagram-v2
|
||||
[*] --> Fresh
|
||||
Fresh --> StaleAllowed: controller disconnected
|
||||
StaleAllowed --> Fresh: valid snapshot received
|
||||
StaleAllowed --> DrainOnly: maxStaleAge exceeded
|
||||
DrainOnly --> Fresh: full snapshot and new epoch
|
||||
DrainOnly --> Stopped: existing traffic drained
|
||||
```
|
||||
|
||||
## 19 健康任务调度
|
||||
|
||||
```mermaid
|
||||
flowchart TD
|
||||
Proxies[Proxy inventory] --> Priority{State priority}
|
||||
Priority -->|new| New[Immediate basic check]
|
||||
Priority -->|SUSPECT| Fast[Fast recheck]
|
||||
Priority -->|stable AVAILABLE| Normal[Normal interval]
|
||||
New --> Jitter[Stable hash plus jitter]
|
||||
Fast --> Jitter
|
||||
Normal --> Jitter
|
||||
Jitter --> Bound[maxInFlight semaphore]
|
||||
Bound --> Checker[Checker workers]
|
||||
```
|
||||
|
||||
## 20 健康 Observation Reducer
|
||||
|
||||
```mermaid
|
||||
flowchart LR
|
||||
Obs[Health Observation] --> Scope{Scope}
|
||||
Scope -->|global| Global[Global health reducer]
|
||||
Scope -->|route target| Target[Target profile reducer]
|
||||
Global --> Consecutive[Consecutive outcome state]
|
||||
Consecutive --> Transition[AVAILABLE SUSPECT UNHEALTHY]
|
||||
Target --> RouteHealth[Only affected route health]
|
||||
RouteHealth -. no direct global delete .-> Transition
|
||||
```
|
||||
|
||||
## 21 Extract partial
|
||||
|
||||
```mermaid
|
||||
sequenceDiagram
|
||||
participant C as Client
|
||||
participant API as Distribution
|
||||
participant DB as PostgreSQL
|
||||
C->>API: count 10 fulfillment partial
|
||||
API->>DB: lock eligible rows
|
||||
DB-->>API: 6 rows
|
||||
API->>DB: update 6 to EXTRACTED and audit
|
||||
DB-->>API: commit
|
||||
API-->>C: requested 10 returned 6
|
||||
```
|
||||
|
||||
## 22 Extract allOrNothing
|
||||
|
||||
```mermaid
|
||||
sequenceDiagram
|
||||
participant C as Client
|
||||
participant API as Distribution
|
||||
participant DB as PostgreSQL
|
||||
C->>API: count 10 fulfillment allOrNothing
|
||||
API->>DB: lock eligible rows
|
||||
DB-->>API: only 6 rows
|
||||
API->>DB: rollback entire transaction
|
||||
API-->>C: insufficient inventory and returned 0
|
||||
```
|
||||
|
||||
## 23 Gateway 所有权回收后提取
|
||||
|
||||
```mermaid
|
||||
sequenceDiagram
|
||||
participant C as Controller
|
||||
participant W as Gateway Worker
|
||||
participant DB as PostgreSQL
|
||||
C->>W: mark Proxy DRAINING at epoch 13
|
||||
W->>W: stop new allocations
|
||||
W-->>C: ACK active 0 reserved 0
|
||||
C->>DB: clear worker ownership
|
||||
C->>DB: AVAILABLE to EXTRACTED plus audit
|
||||
DB-->>C: committed exclusive result
|
||||
```
|
||||
|
||||
## 24 reserveForGateway 不变量
|
||||
|
||||
```mermaid
|
||||
flowchart TD
|
||||
Eligible[Eligible unowned count] --> Formula[extractable equals eligible minus reserve]
|
||||
Reserve[reserveForGateway] --> Formula
|
||||
Requested[requested count] --> Min[return min requested and extractable]
|
||||
Formula --> Min
|
||||
Min --> Result{fulfillment}
|
||||
Result -->|partial| Commit[Commit available quantity]
|
||||
Result -->|allOrNothing insufficient| Rollback[Return zero]
|
||||
```
|
||||
|
||||
## 25 PostgreSQL 独占事务
|
||||
|
||||
```mermaid
|
||||
flowchart TD
|
||||
Begin[BEGIN] --> Select[SELECT eligible FOR UPDATE SKIP LOCKED]
|
||||
Select --> Enough{Quantity satisfies mode?}
|
||||
Enough -->|no allOrNothing| Rollback[ROLLBACK]
|
||||
Enough -->|yes or partial| Update[UPDATE AVAILABLE to EXTRACTED]
|
||||
Update --> Audit[INSERT extraction records]
|
||||
Audit --> Outbox[INSERT outbox]
|
||||
Outbox --> Commit[COMMIT]
|
||||
Commit --> Return[Return proxies with expiry]
|
||||
```
|
||||
|
||||
## 26 并发提取互斥
|
||||
|
||||
```mermaid
|
||||
sequenceDiagram
|
||||
participant A as Extract request A
|
||||
participant DB as PostgreSQL
|
||||
participant B as Extract request B
|
||||
A->>DB: lock rows 1 to 10
|
||||
B->>DB: skip locked rows 1 to 10
|
||||
B->>DB: lock rows 11 to 20
|
||||
A->>DB: commit EXTRACTED 1 to 10
|
||||
B->>DB: commit EXTRACTED 11 to 20
|
||||
Note over A,B: no Proxy returned twice
|
||||
```
|
||||
|
||||
## 27 Outbox 一致性
|
||||
|
||||
```mermaid
|
||||
sequenceDiagram
|
||||
participant C as Controller transaction
|
||||
participant DB as PostgreSQL
|
||||
participant P as Publisher
|
||||
participant W as Worker
|
||||
C->>DB: state change plus outbox
|
||||
DB-->>C: atomic commit
|
||||
P->>DB: read undelivered event
|
||||
P->>W: publish versioned event
|
||||
W-->>P: idempotent ACK
|
||||
P->>DB: mark delivered
|
||||
```
|
||||
|
||||
## 28 配置热更新
|
||||
|
||||
```mermaid
|
||||
flowchart LR
|
||||
File[Read revision] --> Parse[Strict parse]
|
||||
Parse --> Validate[References regex and security]
|
||||
Validate --> Build[Build immutable config]
|
||||
Build --> Diff[Diff tasks and resources]
|
||||
Diff --> Swap[Atomic swap]
|
||||
Swap --> Drain[Drain removed resources]
|
||||
Validate -->|error| Keep[Keep old revision]
|
||||
```
|
||||
|
||||
## 29 Gateway 优雅停机
|
||||
|
||||
```mermaid
|
||||
sequenceDiagram
|
||||
participant K as Kubernetes
|
||||
participant G as Gateway
|
||||
participant LB as Load Balancer
|
||||
K->>G: SIGTERM
|
||||
G->>G: readiness false
|
||||
LB->>LB: remove endpoint
|
||||
G->>G: reject new connections
|
||||
G->>G: drain requests and tunnels
|
||||
G->>G: release active capacities
|
||||
G-->>K: exit before grace timeout
|
||||
```
|
||||
|
||||
## 30 Controller 优雅停机
|
||||
|
||||
```mermaid
|
||||
flowchart TD
|
||||
Term[SIGTERM] --> NotReady[Readiness false]
|
||||
NotReady --> StopWrites[Stop new Extract and Admin writes]
|
||||
StopWrites --> StopFetch[Stop new Fetch]
|
||||
StopFetch --> FinishTx[Commit or rollback current transactions]
|
||||
FinishTx --> Flush[Flush outbox and reports]
|
||||
Flush --> Lease[Release Provider leader lease]
|
||||
Lease --> Exit[Close pools and exit]
|
||||
```
|
||||
|
||||
## 31 Kubernetes 故障域拓扑
|
||||
|
||||
```mermaid
|
||||
flowchart TB
|
||||
LB[Load Balancer] --> ZA
|
||||
LB --> ZB
|
||||
LB --> ZC
|
||||
subgraph ZA[Zone A]
|
||||
GA1[Gateway]
|
||||
CA[Controller]
|
||||
KA[Checker]
|
||||
end
|
||||
subgraph ZB[Zone B]
|
||||
GB1[Gateway]
|
||||
CB[Controller]
|
||||
KB[Checker]
|
||||
end
|
||||
subgraph ZC[Zone C]
|
||||
GC1[Gateway]
|
||||
CC[Controller]
|
||||
KC[Checker]
|
||||
end
|
||||
```
|
||||
|
||||
## 32 Gateway 扩缩决策
|
||||
|
||||
```mermaid
|
||||
flowchart TD
|
||||
Metrics[QPS CPU memory connections latency] --> HPA[HPA decision]
|
||||
HPA --> Up{Above target?}
|
||||
Up -->|yes| ScaleUp[Scale up quickly]
|
||||
Up -->|no| Stable{Stable below target for 10m?}
|
||||
Stable -->|no| Hold[Hold replicas]
|
||||
Stable -->|yes| Capacity{Failure-domain headroom remains?}
|
||||
Capacity -->|no| Hold
|
||||
Capacity -->|yes| ScaleDown[Scale down at most 10 percent]
|
||||
```
|
||||
|
||||
## 33 网络信任边界
|
||||
|
||||
```mermaid
|
||||
flowchart LR
|
||||
Internet --> NLB[Public NLB]
|
||||
NLB --> Gateway[Gateway 8080]
|
||||
Ingress[Private Ingress] --> Distribution[Distribution 8081]
|
||||
Operator[Operator VPN] --> Admin[Admin 8082]
|
||||
Monitor[Monitoring namespace] --> Metrics[Metrics 9090]
|
||||
Gateway --> PublicTargets[Public targets only]
|
||||
Controller[Controller] --> Stores[External PG and Redis]
|
||||
Controller --> Providers[Provider APIs]
|
||||
```
|
||||
|
||||
## 34 可观测信号流
|
||||
|
||||
```mermaid
|
||||
flowchart LR
|
||||
Gateway -->|bounded metrics| Prom[Prometheus]
|
||||
Controller -->|bounded metrics| Prom
|
||||
Checker -->|bounded metrics| Prom
|
||||
Prom --> Grafana[Grafana dashboards]
|
||||
Prom --> Alerts[Alert rules]
|
||||
Gateway -->|sampled redacted logs| Logs[Log backend]
|
||||
Controller -->|audit events| Audit[(Audit storage)]
|
||||
Alerts --> OnCall[On-call]
|
||||
```
|
||||
|
||||
## 35 100k QPS 验证流程
|
||||
|
||||
```mermaid
|
||||
flowchart TD
|
||||
Baseline[Measure one Worker on production shape] --> Formula[Compute replicas at 60 percent target]
|
||||
Formula --> Warmup[Warm Snapshot and connections]
|
||||
Warmup --> Steady[Run 10k steady]
|
||||
Steady --> Ramp[Step ramp to 100k]
|
||||
Ramp --> Peak[Hold 100k peak window]
|
||||
Peak --> Failure[Remove largest failure domain]
|
||||
Failure --> Verify[Verify SLO and all invariants]
|
||||
Verify --> Evidence[Archive raw metrics config and image digest]
|
||||
```
|
||||
|
||||
34
docs/adr/README.md
Normal file
34
docs/adr/README.md
Normal file
@ -0,0 +1,34 @@
|
||||
# 架构决策记录
|
||||
|
||||
## ADR-001:控制面与数据面分离
|
||||
|
||||
**状态:** 接受。
|
||||
|
||||
Gateway 只依赖本地不可变 Snapshot;Provider、数据库、配置重载和健康聚合
|
||||
位于 Controller/Checker。该选择隔离外部 I/O 抖动,并允许数据面按 QPS、
|
||||
控制面按 Provider/库存规模独立扩容。
|
||||
|
||||
## ADR-002:Distribution 使用独占提取
|
||||
|
||||
**状态:** 接受并覆盖早期 Lease 方案。
|
||||
|
||||
成功响应前在一个事务中执行 `AVAILABLE -> EXTRACTED`。系统保存审计事实,
|
||||
但不提供 release、renew 或使用跟踪。这样契合“拿走真实代理后平台不再管理”
|
||||
的最终产品语义,并消除重复发放。
|
||||
|
||||
## ADR-003:单 Worker 所有权
|
||||
|
||||
**状态:** 接受。
|
||||
|
||||
一个 Proxy 同一时刻至多归属一个 Worker,Gateway 在本地维护 Active/Reserved。
|
||||
Distribution 只提取无所有权 Proxy;回收时执行 drain/ACK/归零/解除所有权。
|
||||
该选择避免每请求访问 Redis 做全局并发计数。
|
||||
|
||||
## ADR-004:PostgreSQL 权威、Redis 可重建
|
||||
|
||||
**状态:** 接受。
|
||||
|
||||
Proxy 状态、Extraction、配置版本和 Outbox 由 PostgreSQL 持久化。Redis 只
|
||||
承担 Leader、短期速率与心跳等协调;Redis 丢失后可从权威状态恢复,避免
|
||||
双写状态成为不可判定的事实源。
|
||||
|
||||
18
docs/api/admin.md
Normal file
18
docs/api/admin.md
Normal file
@ -0,0 +1,18 @@
|
||||
# Admin API
|
||||
|
||||
Admin API 使用独立监听器与权限,契约位于 `api/openapi/admin.yaml`。公网部署
|
||||
不得与 Distribution 复用认证 Token;推荐只绑定管理网段或回环地址。
|
||||
|
||||
## 端点
|
||||
|
||||
- `GET /api/v1/status`:返回配置/快照版本、Upstream 聚合计数和 Worker 状态。
|
||||
- `POST /api/v1/upstreams/{name}/enable`:启用 Upstream。
|
||||
- `POST /api/v1/upstreams/{name}/disable`:停止新 Fetch/分配并自然 Drain。
|
||||
- `POST /api/v1/routing/{name}/switch`:用 expectedCurrent 做 CAS 手工切换。
|
||||
- `POST /api/v1/config/reload`:严格解析并原子发布新配置。
|
||||
|
||||
所有写操作写审计记录并返回最终 Request ID 与版本。Enable/Disable 对目标状态
|
||||
幂等;Routing Switch 必须携带 `expectedCurrent`,避免并发操作跳过多个供应商。
|
||||
|
||||
配置重载校验失败返回 422,旧配置继续运行。Status 只返回低基数聚合信息,
|
||||
不得返回 Proxy 地址、凭据、Client 标识或完整 Provider URL。
|
||||
103
docs/api/control-plane.md
Normal file
103
docs/api/control-plane.md
Normal file
@ -0,0 +1,103 @@
|
||||
# Control Plane gRPC API
|
||||
|
||||
## 1. 契约范围
|
||||
|
||||
Proto 源文件位于 `api/proto/controlplane/v1/controlplane.proto`,包含两项
|
||||
内部服务:
|
||||
|
||||
- `WorkerControlPlane`:Worker 注册、Snapshot/Delta 分发、ACK、运行态与结果
|
||||
批量上报。
|
||||
- `CheckerControlPlane`:健康检查任务流和 Observation 批量上报。
|
||||
|
||||
该协议不承载 Client 的独占提取,也没有 extraction lease/release。Proxy 的
|
||||
`AVAILABLE -> EXTRACTED` 只在 Controller 权威事务中完成。
|
||||
|
||||
## 2. Worker 会话
|
||||
|
||||
```mermaid
|
||||
sequenceDiagram
|
||||
participant W as Gateway Worker
|
||||
participant C as Controller
|
||||
W->>C: RegisterWorker(worker, instance, zone)
|
||||
C-->>W: session + ownershipEpoch + maxStaleAge
|
||||
W->>C: WatchSnapshots(lastVersion, checksum)
|
||||
C-->>W: full WorkerSnapshot
|
||||
W->>W: validate + build immutable snapshot
|
||||
W->>W: atomic swap
|
||||
W->>C: AcknowledgeSnapshot(version, epoch, checksum)
|
||||
loop bounded interval
|
||||
W->>C: ReportRuntime(active, reserved, draining)
|
||||
W->>C: ReportOutcomes(batch sequence)
|
||||
end
|
||||
```
|
||||
|
||||
`worker_id` 是逻辑节点,`instance_id` 区分进程重启,`session_id` 防止旧进程
|
||||
继续上报。所有权 `epoch` 小于 Controller 当前值的数据必须拒绝。
|
||||
|
||||
## 3. Snapshot 与 Delta
|
||||
|
||||
完整 Snapshot 包含:
|
||||
|
||||
- 单调 `version`、`ownership_epoch`、生成时间和有效期。
|
||||
- 对该 Worker 可见的有序 Routing。
|
||||
- 仅归该 Worker 所有的 Proxy 与每个 Proxy 的容量。
|
||||
- 内容 `checksum`。
|
||||
|
||||
Delta 声明 `base_version`。Worker 只有在本地版本恰好等于 base 且 checksum
|
||||
验证成功时才能应用;否则丢弃 Delta 并请求完整 Snapshot。构建在后台完成,
|
||||
热路径只读取一次原子指针。
|
||||
|
||||
超过 `max_stale_age` 仍未取得有效快照时,Worker 停止接收新流量并排空已有
|
||||
请求。控制面中断不能让 Worker 查询 PostgreSQL 或 Redis 补偿热路径。
|
||||
|
||||
## 4. 所有权与 Drain
|
||||
|
||||
同一 Proxy 同时只归一个 Worker。Controller 回收用于独占提取的 Proxy 时:
|
||||
|
||||
1. 新 Snapshot 标记或移除该 Proxy,使 Worker 停止新预留。
|
||||
2. Worker 上报 `draining=true` 以及 Active/Reserved。
|
||||
3. 两个计数都归零后 Controller 清除所有权。
|
||||
4. 无所有权 Proxy 才能进入 Distribution 提取事务。
|
||||
|
||||
Worker 崩溃时必须等待所有权 epoch/有效期失效后再转移,避免双主。Proto 中
|
||||
`ReportRuntimeResponse.revoke_proxy_ids` 是加速 Drain 的控制信号,不绕过
|
||||
Snapshot 版本和权威持久化。
|
||||
|
||||
## 5. Outcome 上报
|
||||
|
||||
Outcome 按 Worker 单调 `sequence` 批量上报。Controller 返回已接受的最大序号,
|
||||
从而支持有限重试和去重。阶段区分:
|
||||
|
||||
- `DIAL`:连接 Proxy 地址失败。
|
||||
- `PROXY_HANDSHAKE`:HTTP CONNECT 或 SOCKS 握手失败。
|
||||
- `RESPONSE_HEADERS`:目标响应头前失败。
|
||||
- `TUNNEL`:隧道建立后结束或失败。
|
||||
|
||||
Outcome 是 Observation,不直接让 Worker 修改 PostgreSQL 状态。异步上报队列
|
||||
必须有界;队列满时丢弃低价值样本并计指标,不能反压 Gateway 热路径。
|
||||
|
||||
## 6. Checker 任务
|
||||
|
||||
Checker 注册自身最大并发与支持层级,Controller 发送有 deadline 的任务:
|
||||
|
||||
- `BASIC`:基础连通和协议握手。
|
||||
- `EGRESS`:出口身份与匿名性。
|
||||
- `TARGET`:针对 Routing/目标组的可达性。
|
||||
|
||||
Checker 只返回 `HealthObservation`。Controller reducer 按 Proxy、检查层级和
|
||||
Routing 决定 AVAILABLE、SUSPECT 或 UNHEALTHY,避免多个 Checker 并发写状态。
|
||||
|
||||
## 7. 兼容与演进
|
||||
|
||||
- Proto 字段号一旦发布不得复用。
|
||||
- 删除字段使用 `reserved` 保留名称和编号。
|
||||
- 新枚举值必须让旧接收方按 UNSPECIFIED/拒绝策略处理。
|
||||
- Worker 注册携带 `supported_protocol_version`,不兼容时注册失败而不是静默
|
||||
降级。
|
||||
- Stream 断开后使用带 jitter 的有界指数退避,禁止紧密重连。
|
||||
|
||||
## 8. 传输安全
|
||||
|
||||
集群环境使用 mTLS,证书身份绑定 Worker/Checker 类型和环境。服务端校验
|
||||
消息中的逻辑 ID 与证书授权一致,设置单消息大小、流持续时间、并发 Stream
|
||||
和上报批次上限。`secret_ref` 是受控引用,不在 Proto 中传播真实密码。
|
||||
200
docs/api/distribution.md
Normal file
200
docs/api/distribution.md
Normal file
@ -0,0 +1,200 @@
|
||||
# Distribution API
|
||||
|
||||
## 1. 行为契约
|
||||
|
||||
Distribution API 只提供一次性独占提取:
|
||||
|
||||
```text
|
||||
筛选 AVAILABLE
|
||||
-> 锁定候选
|
||||
-> 校验 Gateway 预留、TTL、健康与过滤条件
|
||||
-> 原子 AVAILABLE -> EXTRACTED
|
||||
-> 写 Extraction Record
|
||||
-> 提交事务
|
||||
-> 返回真实代理地址
|
||||
```
|
||||
|
||||
事务提交前不得把地址写给 Client。返回成功后,该 Proxy 不再参与 Gateway、
|
||||
再次提取或可用库存统计。系统不跟踪 Client 是否使用、使用并发或何时停止,
|
||||
也不提供 release、renew、status 或租约端点。
|
||||
|
||||
HTTP 契约源文件:`api/openapi/proxy-pool.yaml`。
|
||||
|
||||
## 2. 提取请求
|
||||
|
||||
```http
|
||||
POST /api/v1/proxies/extract HTTP/1.1
|
||||
Host: 127.0.0.1:8081
|
||||
Content-Type: application/json
|
||||
X-API-Key: TOKEN
|
||||
X-Request-ID: req_01J4EXAMPLE
|
||||
Idempotency-Key: extract-01J4EXAMPLE
|
||||
|
||||
{
|
||||
"count": 5,
|
||||
"fulfillment": "partial",
|
||||
"filters": {
|
||||
"protocols": ["http"],
|
||||
"regions": ["shanghai"],
|
||||
"carriers": ["telecom"],
|
||||
"allowedUpstreams": ["provider-a"]
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
PowerShell 调用示例:
|
||||
|
||||
```powershell
|
||||
$body = @{
|
||||
count = 5
|
||||
fulfillment = "partial"
|
||||
filters = @{ protocols = @("http"); regions = @("shanghai") }
|
||||
} | ConvertTo-Json -Depth 4
|
||||
|
||||
Invoke-RestMethod `
|
||||
-Method Post `
|
||||
-Uri http://127.0.0.1:8081/api/v1/proxies/extract `
|
||||
-Headers @{ "X-API-Key" = "TOKEN"; "Idempotency-Key" = "extract-SERIAL" } `
|
||||
-ContentType application/json `
|
||||
-Body $body
|
||||
```
|
||||
|
||||
请求约束:
|
||||
|
||||
- `count` 至少为 1,且不超过服务端 `maxCountPerRequest`。
|
||||
- `fulfillment` 省略时使用服务端配置,默认 `partial`。
|
||||
- 所有过滤数组执行“数组内 OR、不同维度 AND”。空数组等同不限制。
|
||||
- `allowedUpstreams` 只能缩小 Client 可访问的 Upstream 集,不能扩大权限。
|
||||
|
||||
## 3. 成功响应
|
||||
|
||||
```json
|
||||
{
|
||||
"requestId": "req_01J4EXAMPLE",
|
||||
"requested": 5,
|
||||
"returned": 2,
|
||||
"proxies": [
|
||||
{
|
||||
"id": "px_01J4A",
|
||||
"protocol": "http",
|
||||
"host": "192.0.2.10",
|
||||
"port": 8080,
|
||||
"username": "USER",
|
||||
"password": "PASSWORD",
|
||||
"url": "http://USER:PASSWORD@192.0.2.10:8080",
|
||||
"region": "shanghai",
|
||||
"carrier": "telecom",
|
||||
"upstream": "provider-a",
|
||||
"expiresAt": "2026-07-28T10:30:00Z",
|
||||
"remainingTtlSeconds": 83,
|
||||
"extractedAt": "2026-07-28T10:28:37Z"
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
`returned` 必须等于 `proxies` 数组长度。`partial` 模式中 `returned` 可以为
|
||||
零;这仍表示请求语法有效,只是当前没有可提取库存。
|
||||
|
||||
返回的 `password` 和 `url` 含真实凭据。Client 必须限制日志、追踪和错误上报
|
||||
对响应体的采集。服务端访问日志只记录数量、过滤摘要、Client、Upstream 和
|
||||
Request ID,不记录地址或凭据。
|
||||
|
||||
## 4. 数量语义
|
||||
|
||||
### 4.1 partial
|
||||
|
||||
锁定的符合条件数量少于 `count` 时,提交实际数量:
|
||||
|
||||
```text
|
||||
requested=10, eligible=6, reserve=0 -> returned=6
|
||||
```
|
||||
|
||||
### 4.2 allOrNothing
|
||||
|
||||
锁定数量不足时整个事务回滚:
|
||||
|
||||
```json
|
||||
{
|
||||
"type": "https://proxy-pool.local/problems/insufficient-proxies",
|
||||
"title": "Insufficient proxies",
|
||||
"status": 409,
|
||||
"code": "INSUFFICIENT_PROXIES",
|
||||
"detail": "requested 10 proxies but only 6 are currently eligible",
|
||||
"requestId": "req_01J4EXAMPLE"
|
||||
}
|
||||
```
|
||||
|
||||
冲突响应后,之前被该请求临时锁定的 Proxy 仍为 AVAILABLE。
|
||||
|
||||
## 5. 资格过滤与 Gateway 预留
|
||||
|
||||
候选必须同时满足:
|
||||
|
||||
1. 权威状态是 AVAILABLE。
|
||||
2. 没有 Worker 所有权,或已完成 Drain 且 Active/Reserved 均为零。
|
||||
3. 剩余 TTL 不低于 `minRemainingTTL`。
|
||||
4. 最近健康检查不早于 `maxHealthCheckAge`。
|
||||
5. protocol、region、carrier、Upstream 满足过滤与 Client 权限。
|
||||
6. 提取后符合条件的共享库存不低于 `reserveForGateway`。
|
||||
|
||||
Worker-owned Proxy 不得直接提取。Controller 需要先发布 DRAINING,等待 Worker
|
||||
确认没有 Active/Reserved,再清除所有权并进入提取事务。快照延迟时仍禁止
|
||||
Gateway 与 Client 同时获得同一 Proxy。
|
||||
|
||||
## 6. 并发与事务
|
||||
|
||||
PostgreSQL Adapter 应在一个事务内使用 `FOR UPDATE SKIP LOCKED` 获取候选,
|
||||
更新状态并写审计记录。所有状态更新必须包含 `state = 'AVAILABLE'` 前置条件。
|
||||
|
||||
关键不变量:
|
||||
|
||||
- 两个并发成功响应的 Proxy ID 集合交集为空。
|
||||
- 状态更新或审计写入任一步失败,整个批次不返回。
|
||||
- `allOrNothing` 不足时零行变为 EXTRACTED。
|
||||
- PostgreSQL 不可写时返回 503,不以内存结果冒充成功。
|
||||
|
||||
## 7. 幂等
|
||||
|
||||
提取是消耗库存的写操作。客户端收到超时后盲目重试可能再次提取一批不同
|
||||
Proxy,因此自动重试应提供稳定的 `Idempotency-Key`。
|
||||
|
||||
服务端幂等记录至少包含:Client ID、Key、请求体摘要、提交结果和过期时间。
|
||||
同一 Client、同一 Key、相同摘要返回首次结果;摘要不同返回 409。幂等记录和
|
||||
Extraction Record 必须与状态更新处在相同事务边界或由同一权威恢复流程保证。
|
||||
|
||||
## 8. 认证、识别与限制
|
||||
|
||||
认证由部署配置决定,OpenAPI 同时声明 API Key、Basic、Bearer 和无认证场景。
|
||||
无认证并不关闭来源 CIDR、Client 识别和限流:
|
||||
|
||||
- 直连请求使用来源 IP 形成匿名 Client。
|
||||
- 只有来源属于 `trustedProxies` 时才接受转发头。
|
||||
- 全局和每 Client 限流在查询库存前执行。
|
||||
- 过滤条件、数量、请求体和 Header 都有长度/数量上限。
|
||||
|
||||
## 9. 错误模型
|
||||
|
||||
所有非 2xx 响应使用 `application/problem+json`:
|
||||
|
||||
- `400`:JSON、Header 或基本格式无效。
|
||||
- `401`:认证失败。
|
||||
- `403`:来源控制、权限或 Upstream 访问被拒绝。
|
||||
- `409`:allOrNothing 库存不足,或幂等 Key 冲突。
|
||||
- `422`:数量、枚举或过滤组合违反业务约束。
|
||||
- `429`:全局或 Client 速率限制,响应 `Retry-After`。
|
||||
- `503`:PostgreSQL 不可写、服务排空或权威状态不可用。
|
||||
|
||||
错误响应不得包含 Provider Secret、Proxy 凭据、SQL 或内部拓扑。
|
||||
|
||||
## 10. 审计记录
|
||||
|
||||
每个被提交的 Proxy 对应一条 Extraction Record:
|
||||
|
||||
```text
|
||||
proxyId, clientId, sourceIP, requestId, upstream, extractedAt, expiresAt
|
||||
```
|
||||
|
||||
无认证时 `clientId` 使用 `anonymous` 或稳定匿名标识并保留 `sourceIP`。记录只
|
||||
用于审计、排错和计费事实,不承担资源归还语义。Proxy 到期后可以清理运行
|
||||
记录,但 Extraction Record 按审计保留策略归档。
|
||||
35
docs/configuration/examples.md
Normal file
35
docs/configuration/examples.md
Normal file
@ -0,0 +1,35 @@
|
||||
# 配置样例索引
|
||||
|
||||
`examples/config` 下每个文件都是可独立加载的 Version 1 完整配置,不是 YAML
|
||||
片段。示例域名使用保留的 `.example` 后缀,Secret 使用带引号的
|
||||
`${ENVIRONMENT_VARIABLE}` 占位符。
|
||||
|
||||
1. `01-local-all.yaml`:本机同时启用 Gateway 与独占提取。
|
||||
2. `02-gateway-only.yaml`:仅 Gateway,GET/HEAD 安全重试。
|
||||
3. `03-extract-only.yaml`:仅一次性独占提取。
|
||||
4. `04-public-gateway-basic-auth.yaml`:公网 Gateway 基础认证。
|
||||
5. `05-public-extract-api-key.yaml`:公网提取 API Key 认证。
|
||||
6. `06-internal-cidr-no-auth.yaml`:内网无认证,CIDR 访问控制。
|
||||
7. `07-auth-any.yaml`:IP 白名单或 API Key 任一通过。
|
||||
8. `08-sequential-failover.yaml`:连续 Empty 后从 A 切到 B。
|
||||
9. `09-weighted-routing.yaml`:70/30 Upstream 权重。
|
||||
10. `10-round-robin-routing.yaml`:Upstream 轮询。
|
||||
11. `11-random-routing.yaml`:Upstream 随机选择。
|
||||
12. `12-least-connections-routing.yaml`:按本地连接容量选择。
|
||||
13. `13-extract-all-or-nothing.yaml`:数量不足时整批回滚。
|
||||
14. `14-gateway-reserve.yaml`:共享池为 Gateway 保留库存。
|
||||
15. `15-strict-ttl-health.yaml`:严格 TTL 与健康新鲜度过滤。
|
||||
16. `16-provider-basic-auth.yaml`:Provider Basic Auth。
|
||||
17. `17-provider-api-key.yaml`:Provider Header API Key。
|
||||
18. `18-provider-post-json.yaml`:Provider JSON POST 与有界解析。
|
||||
19. `19-socks5-upstream.yaml`:SOCKS5 Upstream 模型。
|
||||
20. `20-fetch-billing-quota.yaml`:当前库存与累计计费额度分离。
|
||||
|
||||
批量验证:
|
||||
|
||||
```powershell
|
||||
go test -count=1 ./examples/config
|
||||
```
|
||||
|
||||
项目的配置测试应遍历该目录,以严格加载器解析并校验每个文件。示例中的
|
||||
Provider URL 不是连通性测试目标;配置验证只校验语法、引用和不变量。
|
||||
329
docs/configuration/reference.md
Normal file
329
docs/configuration/reference.md
Normal file
@ -0,0 +1,329 @@
|
||||
# 配置参考
|
||||
|
||||
## 1. 加载规则
|
||||
|
||||
主配置格式为 YAML,根字段 `version` 当前固定为 `1`。加载器启用严格字段
|
||||
检查,拼写错误或未来版本字段不会被静默忽略。推荐启动命令显式传入配置路径:
|
||||
|
||||
```powershell
|
||||
go run ./cmd/proxy-controller -config configs/proxy-pool.yaml
|
||||
```
|
||||
|
||||
所有时间值使用 Go duration,例如 `500ms`、`30s`、`5m`。示例中的
|
||||
`${TOKEN}`、`${PASSWORD}`、`${POSTGRES_URL}` 等由加载器从同名环境变量
|
||||
展开;这些值不得写入日志、指标、配置转储或错误响应。生产配置优先使用
|
||||
环境变量或 Secret 文件,不提交明文值。
|
||||
|
||||
加载与热更新必须遵循同一顺序:
|
||||
|
||||
```text
|
||||
读取文件 -> 严格解析 -> 字段校验 -> 引用/正则校验
|
||||
-> 构建不可变快照 -> 计算差异 -> 原子替换
|
||||
```
|
||||
|
||||
新配置任何一步失败时保留旧快照。删除或禁用 Upstream 只停止新 Fetch 和新
|
||||
分配,已有连接进入 Drain,不强制中断。
|
||||
|
||||
## 2. 根结构
|
||||
|
||||
```yaml
|
||||
version: 1
|
||||
defaults: {}
|
||||
security: {}
|
||||
gateway: {}
|
||||
distribution: {}
|
||||
admin: {}
|
||||
metrics: {}
|
||||
storage: {}
|
||||
routing: []
|
||||
upstreams: {}
|
||||
```
|
||||
|
||||
- `defaults`:Fetch 与 Check 的公共建议值。生产配置仍建议在每个启用的
|
||||
Upstream 显式写出关键限制,避免继承关系不清。
|
||||
- `security`:跨入口启动保护。
|
||||
- `gateway`:HTTP/HTTPS CONNECT 数据面入口。
|
||||
- `distribution`:一次性独占提取入口。
|
||||
- `admin`:运维管理入口,必须与 Distribution 分端口。
|
||||
- `metrics`:Prometheus 入口。
|
||||
- `storage`:Controller 使用的 PostgreSQL 与 Redis 地址。
|
||||
- `routing`:有序 Routing 列表,自上而下首条命中停止。
|
||||
- `upstreams`:全局共享的 Upstream 运行时定义。
|
||||
|
||||
## 3. 安全与监听器
|
||||
|
||||
```yaml
|
||||
security:
|
||||
requireProtectionOnPublicListen: true
|
||||
```
|
||||
|
||||
启用严格保护时,只要 Gateway、Distribution 或 Admin 满足以下全部条件,
|
||||
启动即失败:
|
||||
|
||||
1. 入口已启用且监听地址不是回环地址。
|
||||
2. `auth.mode: none`。
|
||||
3. `access.allowCIDRs` 为空。
|
||||
|
||||
监听器公共字段:
|
||||
|
||||
```yaml
|
||||
enabled: true
|
||||
listen: 0.0.0.0:8081
|
||||
access:
|
||||
allowCIDRs: [10.0.0.0/8]
|
||||
trustedProxies: [10.10.0.10/32]
|
||||
auth:
|
||||
mode: apiKey
|
||||
header: X-API-Key
|
||||
token: "${DISTRIBUTION_API_KEY}"
|
||||
limits:
|
||||
maxConcurrentConnections: 100000
|
||||
requestsPerMinute: 6000
|
||||
requestsPerMinutePerClient: 600
|
||||
```
|
||||
|
||||
`trustedProxies` 只决定何时接受 `Forwarded` 或 `X-Forwarded-For`,不能替代
|
||||
`allowCIDRs`。来自非可信代理的转发头必须忽略。
|
||||
|
||||
### 3.1 认证模式
|
||||
|
||||
- `none`:无身份认证,访问控制与限流仍生效。
|
||||
- `usernamePassword`:使用 `username` 和 `password`。
|
||||
- `apiKey`:使用 `header` 和 `token`。
|
||||
- `ipWhitelist`:使用 `cidrs`。
|
||||
- `any`:`methods` 中任一方法成功即可;方法字段名仍是 `mode`。
|
||||
|
||||
Gateway、Distribution 与 Provider API 的认证是三套独立边界。改变其中一套
|
||||
不得连带改变另外两套。
|
||||
|
||||
## 4. Gateway
|
||||
|
||||
```yaml
|
||||
gateway:
|
||||
enabled: true
|
||||
listen: 127.0.0.1:8080
|
||||
auth: {mode: none}
|
||||
limits:
|
||||
maxConcurrentConnections: 50000
|
||||
retry:
|
||||
maxAttempts: 2
|
||||
retryMethods: [GET, HEAD]
|
||||
destinationPolicy:
|
||||
denyPrivateNetworks: true
|
||||
denyLoopback: true
|
||||
denyLinkLocal: true
|
||||
denyCIDRs: [169.254.169.254/32]
|
||||
```
|
||||
|
||||
- `retryMethods` 默认只应包含幂等方法。POST、PUT、PATCH、DELETE 不自动重试。
|
||||
- CONNECT 向 Client 写出 `200 Connection Established` 后不透明重放。
|
||||
- 目的地址策略必须在 DNS 解析前后都执行,防止 DNS Rebinding。
|
||||
- `maxConcurrentConnections` 是入口准入上限,不是 Proxy 容量上限。
|
||||
|
||||
## 5. Distribution
|
||||
|
||||
```yaml
|
||||
distribution:
|
||||
enabled: true
|
||||
listen: 127.0.0.1:8081
|
||||
auth: {mode: none}
|
||||
clientIdentification: {mode: sourceIP}
|
||||
limits:
|
||||
requestsPerMinute: 60
|
||||
requestsPerMinutePerClient: 30
|
||||
extraction:
|
||||
fulfillment: partial
|
||||
maxCountPerRequest: 20
|
||||
minRemainingTTL: 30s
|
||||
maxHealthCheckAge: 15s
|
||||
reserveForGateway: 5
|
||||
```
|
||||
|
||||
Extraction 是固定的一次性独占行为,**没有** `mode`、`leaseDuration`、
|
||||
`release` 或 `renew` 配置。事务提交后执行 `AVAILABLE -> EXTRACTED`,该 Proxy
|
||||
不再由 Gateway 或 Distribution 分配。
|
||||
|
||||
- `fulfillment`:`partial` 或 `allOrNothing`,默认语义为 `partial`。
|
||||
- `maxCountPerRequest`:单次请求硬上限。
|
||||
- `minRemainingTTL`:剩余寿命低于此值时不参与提取。
|
||||
- `maxHealthCheckAge`:最近检查早于此窗口时不参与提取。
|
||||
- `reserveForGateway`:提取后必须留给 Gateway 的最低符合条件库存数量。
|
||||
|
||||
`partial` 会提交实际可得数量;`allOrNothing` 数量不足时事务回滚,一个也不
|
||||
提取。认证关闭时仍应使用 `sourceIP` 识别匿名 Client 并执行全局/来源限流。
|
||||
|
||||
## 6. Routing
|
||||
|
||||
```yaml
|
||||
routing:
|
||||
- name: api-post
|
||||
enabled: true
|
||||
purpose: gateway
|
||||
match:
|
||||
hostRegex: '^api\\.example\\.com$'
|
||||
methods: [POST]
|
||||
pathRegex: '^/v1/'
|
||||
headers: {X-Tenant: premium}
|
||||
upstreams: [provider-a, provider-b]
|
||||
strategy:
|
||||
type: sequential
|
||||
switchAfterEmptyFetch: 5
|
||||
endBehavior: stayLast
|
||||
onUnavailable:
|
||||
action: reject
|
||||
waitTimeout: 0s
|
||||
```
|
||||
|
||||
- Routing 列表有序,首条匹配后停止。
|
||||
- `purpose` 为 `gateway` 或 `extract`。
|
||||
- `strategy.type` 支持 `sequential`、`random`、`roundRobin`、`weighted`、
|
||||
`leastConnections`。
|
||||
- `weighted` 使用 `weights` 映射,键必须引用本 Routing 的 Upstream。
|
||||
- `sequential` 必须设置大于零的 `switchAfterEmptyFetch`。
|
||||
- `onUnavailable.action` 为 `reject`、`wait` 或 `direct`;默认建议 `reject`。
|
||||
|
||||
Sequential 的空计数属于 Upstream,当前索引属于 Routing。只有 Provider 响应
|
||||
成功、模板成功且合法候选为零时才增加空计数。错误不改变空计数;重复候选
|
||||
会重置空计数但增加独立 duplicate 指标。
|
||||
|
||||
## 7. Upstream
|
||||
|
||||
```yaml
|
||||
upstreams:
|
||||
provider-a:
|
||||
enabled: true
|
||||
exposure: [gateway, extract]
|
||||
provider:
|
||||
billingMode: fetch
|
||||
protocols: [http, https]
|
||||
api:
|
||||
url: https://provider-a.example/proxies
|
||||
method: GET
|
||||
auth:
|
||||
type: apiKey
|
||||
location: header
|
||||
name: X-Provider-Key
|
||||
value: "${PROVIDER_API_KEY}"
|
||||
headers: {}
|
||||
query: {count: '100'}
|
||||
body: {type: json, value: {}}
|
||||
template: '{{.}}'
|
||||
proxyAuth: {type: response}
|
||||
pool: {maxSize: 1000, shrinkDelay: 30s}
|
||||
capacity: {maxConcurrencyPerProxy: 10}
|
||||
lifecycle: {ttl: 2m, allocationSafetyMargin: 15s}
|
||||
fetch:
|
||||
requestInterval: 1s
|
||||
timeout: 3s
|
||||
maxAttempts: 3
|
||||
maxInFlight: 1
|
||||
maxTotal: 100000
|
||||
maxResponseBytes: 1048576
|
||||
templateTimeout: 100ms
|
||||
retry: {initial: 500ms, max: 30s, jitter: 20}
|
||||
check:
|
||||
interval: 30s
|
||||
jitter: 20
|
||||
maxInFlight: 100
|
||||
timeout: 2s
|
||||
maxAttempts: 2
|
||||
maxConsecutiveFailures: 3
|
||||
urls: [http://connect.rom.miui.com/generate_204]
|
||||
```
|
||||
|
||||
### 7.1 Provider 与代理认证
|
||||
|
||||
- `api.auth` 用于系统访问 Provider API。
|
||||
- `proxyAuth` 用于最终连接被获取的 Proxy。
|
||||
- Provider `api.auth.type` 支持 `none`、`basic`、`bearer`、`apiKey`。
|
||||
- `apiKey` 使用 `location: header|query` 与 `name`、`value`,程序负责 Header
|
||||
设置或 Query URL 编码。
|
||||
- `basic` 使用 `username`、`password`;`bearer` 使用 `token`。
|
||||
- `proxyAuth.type: response` 表示凭据来自 Provider 响应。
|
||||
- `proxyAuth.type: static` 使用配置中的 `username`、`password`。
|
||||
- `proxyAuth.type: ipWhitelist` 表示 Provider 按出口 IP 放行,不配置账号密码。
|
||||
|
||||
Provider API 认证**不使用**入口的 `auth.mode`,代理连接认证也不使用
|
||||
`mode`。三者的字段和凭据不可互相回退:
|
||||
|
||||
```yaml
|
||||
# 对外 Distribution/Gateway/Admin
|
||||
auth:
|
||||
mode: apiKey
|
||||
header: X-API-Key
|
||||
token: "${CLIENT_API_KEY}"
|
||||
|
||||
# 请求 Provider API
|
||||
auth:
|
||||
type: bearer
|
||||
token: "${PROVIDER_API_TOKEN}"
|
||||
|
||||
# 连接 Provider 返回的 Proxy
|
||||
proxyAuth:
|
||||
type: static
|
||||
username: "${PROVIDER_PROXY_USER}"
|
||||
password: "${PROVIDER_PROXY_PASSWORD}"
|
||||
```
|
||||
|
||||
### 7.2 Pool 与累计额度
|
||||
|
||||
- `pool.maxSize`:当前系统维护且尚未 EXTRACTED 的 Proxy 硬上限,包括
|
||||
FETCHED、CHECKING、AVAILABLE、SUSPECT、DRAINING 和 pending expected。
|
||||
- `fetch.maxTotal`:当前运行或计费周期内,从 Provider 成功获取的累计上限;
|
||||
`0` 表示不设置累计上限。
|
||||
|
||||
`fetch.maxTotal` 不得小于 `pool.maxSize`。提取一个 Proxy 会释放当前库存位置,
|
||||
但不会恢复累计获取额度。
|
||||
|
||||
### 7.3 Fetch 限制
|
||||
|
||||
- `requestInterval`:同一 Provider 请求间隔。
|
||||
- `timeout`:单次调用超时。
|
||||
- `maxAttempts`:单次补池动作最大尝试次数。
|
||||
- `maxInFlight`:同一 Provider 同时在途请求数。
|
||||
- `maxResponseBytes`:读取响应的硬上限。
|
||||
- `templateTimeout`:模板解析执行上限。
|
||||
- `retry`:错误退避;HTTP 429 还必须尊重 `Retry-After`。
|
||||
|
||||
大量缺池信号必须合并成 singleflight 或容量为 1 的通知,不能按 Gateway 请求
|
||||
数量线性触发 Provider API。
|
||||
|
||||
### 7.4 生命周期与健康
|
||||
|
||||
- 明确绝对过期时间优先于响应 TTL,响应 TTL 优先于配置 `lifecycle.ttl`。
|
||||
- 距离过期不足 `allocationSafetyMargin` 时停止新分配。
|
||||
- `check.jitter` 为调度抖动百分比,避免所有 Proxy 同时探测。
|
||||
- 第一次有意义失败进入 SUSPECT;达到 `maxConsecutiveFailures` 后才进入
|
||||
UNHEALTHY。
|
||||
|
||||
## 8. 存储、Admin 与 Metrics
|
||||
|
||||
```yaml
|
||||
admin:
|
||||
enabled: true
|
||||
listen: 127.0.0.1:8082
|
||||
auth: {mode: none}
|
||||
metrics:
|
||||
enabled: true
|
||||
listen: 127.0.0.1:9090
|
||||
storage:
|
||||
postgresURL: "${POSTGRES_URL}"
|
||||
redisURL: "${REDIS_URL}"
|
||||
```
|
||||
|
||||
PostgreSQL 是 Proxy 生命周期、Routing 选择、Worker 所有权和 Extraction Record
|
||||
的权威存储。Redis 只承载可重建的短期协调状态,不能成为独占提取的唯一事实
|
||||
来源。Metrics 标签禁止 Proxy IP、Client ID、Session、完整 URL 和 Request ID。
|
||||
|
||||
## 9. 启动前校验清单
|
||||
|
||||
1. `version` 必须为 `1`,未知字段拒绝。
|
||||
2. 所有启用监听器具有合法 `host:port`。
|
||||
3. 非回环监听器满足认证或来源 CIDR 保护。
|
||||
4. Routing 名称唯一,正则可编译,引用的 Upstream 存在。
|
||||
5. Sequential 阈值大于零,`onUnavailable.action` 明确。
|
||||
6. 启用的 Upstream 有正数 `pool.maxSize`、并发和 Fetch 限制。
|
||||
7. `allocationSafetyMargin < ttl`。
|
||||
8. `fetch.maxTotal == 0` 或 `fetch.maxTotal >= pool.maxSize`。
|
||||
9. Distribution 的 fulfillment 合法,单次数量大于零。
|
||||
10. Secret 未写入日志可见配置转储。
|
||||
93
docs/design/product-design.md
Normal file
93
docs/design/product-design.md
Normal file
@ -0,0 +1,93 @@
|
||||
# Proxy Pool 产品设计文案
|
||||
|
||||
## 1. 产品定位
|
||||
|
||||
Proxy Pool 把多个供应商的动态代理统一成一个可运营的资源系统。平台面对
|
||||
两类不同使用方式:需要平台代转发流量的应用使用 Gateway;需要拿到真实
|
||||
代理并自行建立连接的应用使用 Distribution API。
|
||||
|
||||
两类入口共享 Provider、健康、TTL 和路由配置,但资源分配语义不同:
|
||||
|
||||
- Gateway 只在一次请求或隧道生命周期内占用代理并发,结束后释放容量。
|
||||
- Distribution 一旦返回代理,该代理即永久离开系统可分配池。
|
||||
|
||||
## 2. 目标用户
|
||||
|
||||
- **业务调用方**:通过稳定入口使用代理,不感知供应商差异。
|
||||
- **代理直提调用方**:按协议、区域、运营商或 Upstream 条件独占提取。
|
||||
- **平台管理员**:管理 Provider、Routing、容量、健康和故障切换。
|
||||
- **SRE**:依据低基数指标、审计记录和运行手册进行容量与故障管理。
|
||||
|
||||
## 3. 核心价值
|
||||
|
||||
### 3.1 供应商差异收敛
|
||||
|
||||
Provider Adapter 负责请求格式、认证和响应解析。标准化后,Routing、健康、
|
||||
容量和业务入口只依赖统一 Proxy 模型,不把供应商字段传入核心域。
|
||||
|
||||
### 3.2 高并发热路径隔离
|
||||
|
||||
Gateway Worker 只读取本地不可变快照并维护本地容量计数。供应商延迟、
|
||||
数据库抖动和控制面重载不会成为每请求依赖。
|
||||
|
||||
### 3.3 明确且可审计的资源语义
|
||||
|
||||
系统区分“短时使用容量”和“一次性独占提取”。每次 Extraction 保存请求、
|
||||
调用方、来源、Upstream、提取时间和过期时间审计事实,但不追踪提取后的
|
||||
实际使用,也不存在归还接口。
|
||||
|
||||
## 4. 主要流程
|
||||
|
||||
### 4.1 Gateway 请求
|
||||
|
||||
1. 接入层完成认证、来源识别、限流和目标地址检查。
|
||||
2. Routing 按配置顺序首条命中。
|
||||
3. Dispatcher 从本地快照筛选 Upstream、协议、标签、TTL 和健康条件。
|
||||
4. 原子预留 Proxy 容量,建立到上游代理的连接。
|
||||
5. 建连成功后转为 Active,传输结束后释放;失败则取消预留。
|
||||
6. GET/HEAD 仅在响应提交前按策略重试;CONNECT 建立后不重放。
|
||||
|
||||
### 4.2 独占提取
|
||||
|
||||
1. 调用方提交数量、过滤条件和 fulfillment。
|
||||
2. Controller 校验调用方限额、TTL、健康新鲜度及 Gateway 保留量。
|
||||
3. 在一个数据库事务中锁定候选并执行 `AVAILABLE -> EXTRACTED`。
|
||||
4. `partial` 尽量返回;`allOrNothing` 数量不足时零提取。
|
||||
5. 响应返回代理 URL、`expiresAt` 和 `remainingTtlSeconds`。
|
||||
|
||||
### 4.3 Sequential 切换
|
||||
|
||||
1. Provider 响应成功且解析成功,但合法候选为零,才累计 Empty。
|
||||
2. 网络、认证、HTTP、模板或解析失败只计 Error。
|
||||
3. 全部候选均重复时计 DuplicateOnly,并重置连续 Empty。
|
||||
4. 达到阈值后,引用该 Upstream 的 Routing 原子前进一次。
|
||||
5. 旧 Upstream 已有 Proxy 继续耗尽,不因切换被直接删除。
|
||||
|
||||
## 5. 失败体验
|
||||
|
||||
- 无候选时严格执行 Routing 的 `reject`、`wait` 或 `direct`,默认拒绝。
|
||||
- Distribution 部分满足用 200 返回实际数量;全有或全无不足时返回冲突状态。
|
||||
- Provider 故障进入退避,不让调用请求触发同步 Provider 获取。
|
||||
- 快照版本断档时 Worker 保留最后一份完整快照并请求全量重同步。
|
||||
- 控制面不可用时,现有 Worker 可在快照有效期内继续服务,停止接收新配置。
|
||||
|
||||
## 6. 容量目标与服务指标
|
||||
|
||||
- 集群峰值目标:100,000 QPS。
|
||||
- Worker 副本数:
|
||||
|
||||
```text
|
||||
required_workers = ceil(peak_qps / (measured_worker_qps * target_utilization))
|
||||
+ failure_domain_spares
|
||||
```
|
||||
|
||||
- `measured_worker_qps` 必须来自目标协议占比、代理 RTT、连接复用率和安全策略
|
||||
均接近生产的压测。
|
||||
- 设计阶段不承诺单 Worker QPS,也不以平均值替代 P95/P99 和错误率。
|
||||
|
||||
## 7. 首版范围
|
||||
|
||||
首版包含 HTTP 正向代理、HTTPS CONNECT、REST Distribution/Admin、Provider
|
||||
适配、健康与生命周期、Sequential 等路由策略、PostgreSQL 权威状态、Redis
|
||||
可重建协调和集群快照。SOCKS5、跨地域主动主动和高级成本优化保留扩展边界。
|
||||
|
||||
96
docs/design/project-structure.md
Normal file
96
docs/design/project-structure.md
Normal file
@ -0,0 +1,96 @@
|
||||
# Proxy Pool 项目架构
|
||||
|
||||
## 1. 仓库结构
|
||||
|
||||
```text
|
||||
proxy-pool/
|
||||
├── cmd/
|
||||
│ ├── proxy-gateway/ # 数据面进程
|
||||
│ ├── proxy-controller/ # 控制面与 HTTP API
|
||||
│ ├── proxy-checker/ # 健康检查执行器
|
||||
│ └── proxy-loadgen/ # 可复现容量测试
|
||||
├── internal/
|
||||
│ ├── config/ # 严格配置解析和校验
|
||||
│ ├── domain/ # 无传输、无存储依赖的领域模型
|
||||
│ ├── gateway/ # snapshot、dispatch、server、transport
|
||||
│ ├── controller/ # provider、pool、routing、extraction、health
|
||||
│ ├── adapters/ # PostgreSQL、Redis、Provider API、内存适配
|
||||
│ └── platform/ # 日志、指标、停机和进程装配
|
||||
├── api/ # OpenAPI 与 Protobuf 契约
|
||||
├── configs/ # 默认配置
|
||||
├── examples/ # 可校验配置场景
|
||||
├── deploy/ # Compose 与 Kubernetes
|
||||
├── docs/ # 设计、开发、API、测试、运维
|
||||
├── diagrams/ # Mermaid 图集
|
||||
├── scripts/ # 验证和生成脚本
|
||||
└── test/ # fixture、集成、端到端和负载测试
|
||||
```
|
||||
|
||||
## 2. 进程边界
|
||||
|
||||
### proxy-gateway
|
||||
|
||||
`gateway -> dispatch -> transport` 是数据面主调用链。`dispatch` 包含筛选、
|
||||
选择、session 与重试资格等热路径决策;`transport` 独占连接池、上游握手和
|
||||
隧道生命周期。任何包都不得从热路径反向调用 Controller 存储。
|
||||
|
||||
### proxy-controller
|
||||
|
||||
Controller 是首版模块化单体。Provider、Pool、Routing 和 Extraction 共享
|
||||
事务边界和状态演进,避免过早拆成分布式事务。对外端口定义在领域或控制器
|
||||
模块,具体 PostgreSQL/Redis/HTTP 实现在 `adapters`。
|
||||
|
||||
### proxy-checker
|
||||
|
||||
Checker 只产生 Observation。最终状态迁移由 Controller 的确定性 reducer
|
||||
完成,避免多个检查实例同时写 Proxy 状态。
|
||||
|
||||
### proxy-loadgen
|
||||
|
||||
负载工具生成 HTTP、CONNECT、连接复用与故障注入场景,输出延迟分位数、
|
||||
错误类别、连接数、CPU、RSS、GC 与吞吐。它是 100k QPS 结论的证据工具,
|
||||
不是业务进程。
|
||||
|
||||
## 3. 依赖方向
|
||||
|
||||
```text
|
||||
cmd -> controller/gateway/checker -> domain
|
||||
|
|
||||
+------------> port interfaces
|
||||
adapters -------------------------> port interfaces
|
||||
platform -------------------------> standard library / observability SDK
|
||||
```
|
||||
|
||||
硬性规则:
|
||||
|
||||
1. `domain` 不导入 HTTP、SQL、Redis、配置或平台包。
|
||||
2. `gateway/dispatch` 只依赖本地 Snapshot 与领域类型。
|
||||
3. `adapters` 实现端口,不被领域层反向引用。
|
||||
4. 配置先解析、校验、编译为运行时对象,再原子发布。
|
||||
5. Secret 仅通过引用进入运行时,不进入唯一键、指标或日志字段。
|
||||
|
||||
## 4. 数据所有权
|
||||
|
||||
- PostgreSQL:Proxy 生命周期、Extraction 审计、配置版本和 Outbox 的权威源。
|
||||
- Redis:Leader、分布式速率、Worker 心跳等可丢失且可重建状态。
|
||||
- Worker:仅拥有分配给自己的 Proxy 本地容量计数和不可变快照。
|
||||
- Controller:拥有 Provider 调度、Routing 运行态与 Worker 所有权编排。
|
||||
- Checker:不拥有 Proxy 状态,只拥有执行中的检查任务。
|
||||
|
||||
## 5. 一致性边界
|
||||
|
||||
- Extraction 使用 PostgreSQL 单事务和行锁跳过锁定候选,提交后才响应。
|
||||
- Worker 所有权采用 `worker + epoch + version + expiry`,同一 Proxy 至多归属
|
||||
一个 Worker。
|
||||
- 从 Worker 回收 Proxy 时先 Drain,等待 ACK 且 Active/Reserved 为零,再
|
||||
解除所有权;只有无所有权 Proxy 能被 Distribution 提取。
|
||||
- Snapshot 为整代不可变对象,通过校验和和严格版本序列原子替换。
|
||||
|
||||
## 6. 扩展规则
|
||||
|
||||
- 新 Provider:增加 Adapter,不修改 Proxy/Pool/Routing 领域语义。
|
||||
- 新入口协议:在 Gateway 增加 ingress adapter;只有存在第二种上游协议
|
||||
执行方式时再抽象 egress adapter。
|
||||
- 新路由策略:实现同一策略端口,并提供确定性单测和并发不变量测试。
|
||||
- 新存储:实现已有 repository port,不把驱动类型泄漏到控制器。
|
||||
|
||||
65
docs/development/guide.md
Normal file
65
docs/development/guide.md
Normal file
@ -0,0 +1,65 @@
|
||||
# 开发指南
|
||||
|
||||
## 1. 环境
|
||||
|
||||
- Go 1.26 或 `go.mod` 指定版本。
|
||||
- PostgreSQL 与 Redis 仅用于适配器集成测试,领域单测不依赖外部服务。
|
||||
- 运行 `go test -race` 需要 CGO 和 C 编译器。
|
||||
- Docker Compose 用于本地完整拓扑,Docker 不应成为普通单测前置条件。
|
||||
|
||||
## 2. 开发循环
|
||||
|
||||
1. 在 `traceability.md` 中找到需求 ID。
|
||||
2. 先写会失败的单测或契约测试,确认失败原因与需求一致。
|
||||
3. 实现最小完整行为,不增加空接口或预留目录。
|
||||
4. 运行目标包测试,再运行全仓库测试和静态检查。
|
||||
5. 更新需求证据、相关文档和配置示例。
|
||||
|
||||
```powershell
|
||||
go test ./internal/domain/proxy
|
||||
go test ./...
|
||||
go vet ./...
|
||||
go build ./...
|
||||
```
|
||||
|
||||
## 3. 包设计规则
|
||||
|
||||
- 领域包使用业务语言,不使用 Controller、HTTP 或数据库 DTO。
|
||||
- 模块接口应隐藏内部策略步骤,避免把 filter/scorer/picker 拆成浅接口链。
|
||||
- 时间逻辑注入 `now` 或 Clock,测试禁止依赖真实睡眠。
|
||||
- Provider 调度使用有界信号、singleflight、超时和抖动退避。
|
||||
- 后台队列必须有容量、溢出策略、关闭语义和指标。
|
||||
- 并发计数必须以不变量测试证明,不只检查最终值。
|
||||
|
||||
## 4. 配置变更
|
||||
|
||||
新增字段时同时修改:
|
||||
|
||||
1. `internal/config` 类型、默认值与校验。
|
||||
2. `configs/default.yaml`。
|
||||
3. `docs/configuration/reference.md`。
|
||||
4. 至少一个有效示例和一个无效测试。
|
||||
5. 配置版本兼容说明;不静默忽略未知字段。
|
||||
|
||||
## 5. API 变更
|
||||
|
||||
- OpenAPI 是 REST 契约源,Protobuf 是 Controller/Worker 契约源。
|
||||
- 先更新契约和兼容性测试,再修改 handler。
|
||||
- Distribution 不得出现 lease、release、renew、return 等资源归还语义。
|
||||
- 错误响应包含稳定 code 和 requestId,不向调用方暴露 Secret 或内部栈。
|
||||
|
||||
## 6. 并发与性能
|
||||
|
||||
- Gateway 请求路径不得出现远程存储访问和无界 goroutine 创建。
|
||||
- Snapshot 构建在后台完成,发布后只读;请求只做一次原子指针读取。
|
||||
- Proxy 容量由同一个打包原子值保存 Active/Reserved,避免分开检查再写入。
|
||||
- 性能优化必须附基准;100k QPS 结论必须附完整环境和负载模型。
|
||||
|
||||
## 7. 提交前检查
|
||||
|
||||
```powershell
|
||||
./scripts/verify.ps1
|
||||
```
|
||||
|
||||
审查还要确认:无 Secret 日志、无 Proxy IP 高基数标签、无默认 direct、无
|
||||
Extraction Lease API、无把重复结果误计为 Empty 的逻辑。
|
||||
61
docs/operations/production-readiness.md
Normal file
61
docs/operations/production-readiness.md
Normal file
@ -0,0 +1,61 @@
|
||||
# 生产就绪检查表
|
||||
|
||||
## 架构与一致性
|
||||
|
||||
- [ ] Gateway 热路径依赖审计确认无 PostgreSQL、Redis、Provider 或模板执行。
|
||||
- [ ] 每个 Proxy 同一时刻最多归属一个 Worker,ownership epoch 单调。
|
||||
- [ ] Reserved -> Active 使用单个原子转换,无超卖与负计数。
|
||||
- [ ] Sequential 并发 Empty 只切换一次,旧 Upstream Proxy 自然耗尽。
|
||||
- [ ] `pool.maxSize` 与 `fetch.maxTotal` 分别按当前库存和累计获取计数。
|
||||
- [ ] Extract 只有 `AVAILABLE -> EXTRACTED`,OpenAPI 不存在 release/renew。
|
||||
- [ ] Extract 状态更新和审计记录位于同一数据库事务。
|
||||
- [ ] `partial` 和 `allOrNothing` 均通过并发事务测试。
|
||||
- [ ] `reserveForGateway` 在所有提取路径上统一执行。
|
||||
|
||||
## 安全
|
||||
|
||||
- [ ] 非回环监听均配置 Auth 或 allowCIDRs,严格模式已开启。
|
||||
- [ ] Gateway、Distribution、Admin 凭据和权限相互独立。
|
||||
- [ ] trusted proxy 只包含受控 LoadBalancer/Ingress 网段。
|
||||
- [ ] 解析前后均拦截私网、回环、链路本地、元数据地址与 DNS Rebinding。
|
||||
- [ ] Secret 由外部密钥系统注入,镜像、ConfigMap、日志没有明文。
|
||||
- [ ] Pod 以非 root、只读根文件系统、无 Linux capabilities 运行。
|
||||
- [ ] NetworkPolicy 默认拒绝,外部数据库/Redis/Provider 网段已收紧。
|
||||
- [ ] Provider 模板有响应大小、执行时间、函数与外部访问限制。
|
||||
|
||||
## 可用性
|
||||
|
||||
- [ ] PostgreSQL 和 Redis 跨可用区,有监控、备份和恢复演练证据。
|
||||
- [ ] Gateway、Controller、Checker 均跨主机/可用区分散。
|
||||
- [ ] PDB、优雅终止与最大连接时长的组合经过驱逐测试。
|
||||
- [ ] Controller 断线时 Gateway 在 `maxStaleAge` 内继续,超限拒绝新请求。
|
||||
- [ ] Worker 崩溃后 ownership 只在租约过期后再分配。
|
||||
- [ ] Snapshot 版本缺口触发全量同步,旧 Delta 被拒绝。
|
||||
|
||||
## 性能
|
||||
|
||||
- [ ] 单 Worker 基准使用生产同规格硬件、网络、TLS 和 Snapshot 规模。
|
||||
- [ ] 完成 10k 稳态、100k 峰值、CONNECT 活跃连接和建连速率独立测试。
|
||||
- [ ] 在最大可用区失效时仍满足容量和延迟 SLO。
|
||||
- [ ] 目标利用率不高于 60%,HPA 缩容稳定窗口不低于 10 分钟。
|
||||
- [ ] 队列、buffer、日志和结果上报全部有界。
|
||||
- [ ] p99 Dispatch 预算、端到端延迟、错误率、CPU、RSS、FD 和网络有原始证据。
|
||||
|
||||
## 观测与值班
|
||||
|
||||
- [ ] 仪表盘覆盖 Gateway、Routing、Upstream、Provider、Extract、Snapshot、Checker。
|
||||
- [ ] 告警有负责人、严重级别、Runbook 链接和演练记录。
|
||||
- [ ] 指标无 Proxy IP、Client ID、Session、完整 URL 或 request ID 高基数标签。
|
||||
- [ ] 日志脱敏已用真实 Secret fixture 验证。
|
||||
- [ ] 值班人员完成 PostgreSQL、Redis、Snapshot、容量与 Extract 故障演练。
|
||||
|
||||
## 发布门禁
|
||||
|
||||
- [ ] `go test ./...`、race、vet、build 全部通过。
|
||||
- [ ] OpenAPI/Proto 兼容检查通过。
|
||||
- [ ] Compose、Kustomize、Prometheus、HAProxy 配置静态校验通过。
|
||||
- [ ] 数据库迁移已在生产数据量副本上演练,并有回退或前向修复方案。
|
||||
- [ ] 100k QPS 验证报告包含环境、命令、版本、场景、原始指标和结论。
|
||||
|
||||
任一关键一致性、安全或恢复项未完成时,不标记生产就绪。
|
||||
|
||||
249
docs/operations/runbook.md
Normal file
249
docs/operations/runbook.md
Normal file
@ -0,0 +1,249 @@
|
||||
# Proxy Pool 运维手册
|
||||
|
||||
## 1. 运行边界
|
||||
|
||||
- Gateway 是数据面,正常请求热路径不访问 PostgreSQL、Redis 或 Provider。
|
||||
- Controller 是权威控制面,负责 Fetch、生命周期、所有权、Snapshot、Extract
|
||||
和审计;多个副本只有一个 Provider 逻辑 Leader。
|
||||
- Checker 执行有界健康探测,只上报 Observation,最终状态由 Controller
|
||||
reducer 决定。
|
||||
- Extract 是一次性独占发放。提交后状态为 `EXTRACTED`,没有 Lease、续租或
|
||||
Release 接口。
|
||||
- `reserveForGateway` 是共享池硬约束,Extract 不得把 Gateway 库存清空。
|
||||
- 集群峰值 100,000 QPS 是设计目标,只有完成本文容量验收后才能作为已验证
|
||||
能力对外承诺。
|
||||
|
||||
## 2. 本地拓扑模板
|
||||
|
||||
当前仓库交付设计、契约、部署拓扑和关键领域实现;`cmd/proxy-*` 的完整运行时
|
||||
装配属于 `implementation-plan.md` 后续任务。此处 Compose/Kubernetes 资产用于
|
||||
评审网络、资源、探针和依赖关系,当前只执行静态渲染,不把模板写成可运行服务。
|
||||
|
||||
### 2.1 前置条件
|
||||
|
||||
- Docker Engine 25+,Compose v2.30+。
|
||||
- 至少 8 CPU、16 GiB 内存和 20 GiB 可用磁盘。
|
||||
- 本地端口 `3000`、`8080`、`8081`、`8082`、`8404`、`9091` 未占用。
|
||||
|
||||
### 2.2 配置凭据
|
||||
|
||||
`deploy/config/local.yaml` 只用于本机拓扑验证。进入运行时实施阶段后,再通过
|
||||
密钥系统提供下列变量,并把 Provider 地址替换为测试 fixture:
|
||||
|
||||
```powershell
|
||||
$env:PROXY_POOL_GATEWAY_PASSWORD = "LOCAL_GATEWAY_PASSWORD"
|
||||
$env:PROXY_POOL_EXTRACT_TOKEN = "LOCAL_EXTRACT_TOKEN"
|
||||
$env:PROXY_POOL_ADMIN_TOKEN = "LOCAL_ADMIN_TOKEN"
|
||||
$env:PROVIDER_A_TOKEN = "PROVIDER_A_TOKEN"
|
||||
$env:PROVIDER_B_TOKEN = "PROVIDER_B_TOKEN"
|
||||
```
|
||||
|
||||
### 2.3 静态检查
|
||||
|
||||
```powershell
|
||||
docker compose -f deploy/docker-compose.yml config
|
||||
kubectl kustomize deploy/kubernetes/base > rendered.yaml
|
||||
```
|
||||
|
||||
目标拓扑入口:
|
||||
|
||||
- Gateway:`127.0.0.1:8080`
|
||||
- Distribution:`http://127.0.0.1:8081`
|
||||
- Admin:`http://127.0.0.1:8082`
|
||||
- HAProxy 状态:`http://127.0.0.1:8404/stats`
|
||||
- Prometheus:`http://127.0.0.1:9091`
|
||||
- Grafana:`http://127.0.0.1:3000`
|
||||
|
||||
`deploy/config/local.yaml` 中的 `.invalid` Provider 是故障演示占位。未替换时
|
||||
Fetch 应表现为 Error 与退避,不应增加 Empty 计数,也不影响已有 Proxy。
|
||||
|
||||
## 3. Kubernetes 发布
|
||||
|
||||
本节是运行时实施完成后的发布规格,不代表当前代码已达到生产就绪。
|
||||
|
||||
### 3.1 准备
|
||||
|
||||
1. 使用托管 PostgreSQL 和 Redis,分别配置 TLS、备份、监控和多可用区。
|
||||
2. 复制 `secret.example.yaml` 到环境私密配置系统,由 External Secrets、SOPS
|
||||
或密钥管理平台生成 `proxy-pool-secrets`,不要提交真实 Secret。
|
||||
3. 在环境 Overlay 替换镜像、Provider 地址、允许网段、外部存储地址、资源量
|
||||
和 LoadBalancer 注解。
|
||||
4. 根据集群 CNI 能力收紧 NetworkPolicy 的外部网段。
|
||||
5. 在预发布环境完成数据库向前兼容迁移,再发布 Controller。
|
||||
|
||||
### 3.2 服务端应用顺序
|
||||
|
||||
```bash
|
||||
kubectl apply -f deploy/kubernetes/base/namespace.yaml
|
||||
kubectl -n proxy-pool apply -f ENVIRONMENT_SECRET.yaml
|
||||
kubectl apply -k deploy/kubernetes/base
|
||||
kubectl -n proxy-pool rollout status deployment/proxy-controller --timeout=5m
|
||||
kubectl -n proxy-pool rollout status deployment/proxy-checker --timeout=5m
|
||||
kubectl -n proxy-pool rollout status deployment/proxy-gateway --timeout=10m
|
||||
```
|
||||
|
||||
### 3.3 探针语义
|
||||
|
||||
- `/livez`:进程事件循环仍可运行。数据库或 Redis 短暂失败不得导致 Gateway
|
||||
liveness 失败。
|
||||
- `/readyz`:进程可以接收新工作。Gateway 只有在持有未超过 `maxStaleAge`
|
||||
的完整 Snapshot 且仍有准入能力时才 Ready。
|
||||
- Controller 只有在配置有效、存储可用、迁移兼容且控制接口已监听时才 Ready。
|
||||
- Checker 在任务消费与结果上报通道可用时 Ready。
|
||||
- `/metrics`:独立于业务入口,NetworkPolicy 仅允许监控命名空间访问。
|
||||
|
||||
探针不得执行 Provider 请求或完整数据库扫描。
|
||||
|
||||
### 3.4 发布顺序与兼容性
|
||||
|
||||
1. 先做向前兼容数据库迁移。
|
||||
2. 发布 Controller,确认旧 Worker 仍能消费旧 Snapshot 协议版本。
|
||||
3. 发布 Checker。
|
||||
4. 逐批发布 Gateway;每次至少保留 PDB 要求的健康副本。
|
||||
5. 观察 30 分钟,再清理已经无人读取的旧字段或旧迁移。
|
||||
|
||||
回滚只能回到仍兼容当前 Schema 和 Snapshot 版本的镜像。涉及不可逆数据迁移时,
|
||||
必须使用前向修复。
|
||||
|
||||
## 4. 容量规划
|
||||
|
||||
```text
|
||||
worker_replicas =
|
||||
ceil(peak_qps / (tested_worker_qps * target_utilization))
|
||||
+ largest_failure_domain_replicas
|
||||
```
|
||||
|
||||
- `peak_qps`:当前目标为 100,000。
|
||||
- `tested_worker_qps`:在同 CPU、内存、网络、Go 版本、Snapshot 规模、TLS 与
|
||||
上游响应模型下测出的单 Pod 持续能力。
|
||||
- `target_utilization`:不高于 0.60,给突发、GC 和故障转移留空间。
|
||||
- `largest_failure_domain_replicas`:最大单可用区失效时丢失的副本数。
|
||||
|
||||
禁止用 CPU 核数直接推算 QPS,也禁止把短时峰值当持续容量。Kubernetes 基线的
|
||||
6 个 Gateway 副本只是初始值,必须由压测结果调整。
|
||||
|
||||
HPA 使用 CPU/内存作为保护性信号;生产环境建议通过 Prometheus Adapter 加入:
|
||||
|
||||
- 每 Pod Gateway QPS。
|
||||
- 活跃连接数和建连速率。
|
||||
- p99 Dispatch 延迟。
|
||||
- 拒绝率和 Available Slots。
|
||||
|
||||
连接型工作负载缩容至少稳定 10 分钟,终止前先 NotReady,再等待现有隧道排空。
|
||||
|
||||
## 5. 日常检查
|
||||
|
||||
每班次检查:
|
||||
|
||||
1. Gateway QPS、错误率、p95/p99 和活跃连接。
|
||||
2. Snapshot age、epoch/version、重同步与 ACK 延迟。
|
||||
3. Available Slots、Reserved、Active、各 Proxy 状态数量。
|
||||
4. Provider Success、Empty、Duplicate-only、Error、429 与退避。
|
||||
5. Checker 队列、SUSPECT 数、检查延迟和目标级失败。
|
||||
6. Extract requested/returned、insufficient、冲突和审计写入。
|
||||
7. PostgreSQL 连接、锁等待、事务失败、WAL 与备份。
|
||||
8. Redis 延迟、内存、主从状态和 Leader 租约抖动。
|
||||
|
||||
Prometheus 标签禁止包含 Proxy IP、Client ID、Session、完整 URL、request ID。
|
||||
需要逐请求调查时使用受控、脱敏且采样的结构化日志。
|
||||
|
||||
## 6. 优雅停机
|
||||
|
||||
### Gateway
|
||||
|
||||
1. readiness 立即失败,停止新连接。
|
||||
2. 停止应用新 Snapshot,但保留当前不可变版本。
|
||||
3. 等待 HTTP 请求和 CONNECT 隧道排空。
|
||||
4. 到达 60 秒上限后关闭残余连接,保证容量 reservation 被释放。
|
||||
|
||||
### Controller
|
||||
|
||||
1. 停止接收新的 Extract/Admin 写请求。
|
||||
2. 停止发起 Fetch,释放 Provider Leader 租约。
|
||||
3. 完成已进入数据库事务的 Extract 或回滚。
|
||||
4. 刷新 outbox、审计和 Worker ACK,再关闭连接池。
|
||||
|
||||
### Checker
|
||||
|
||||
1. 停止领取新任务。
|
||||
2. 在 45 秒内完成或取消现有探测。
|
||||
3. 批量上报已完成 Observation,未完成任务由队列重新投递。
|
||||
|
||||
## 7. 故障处置
|
||||
|
||||
### 7.1 Snapshot 陈旧
|
||||
|
||||
症状:`ProxyPoolGatewaySnapshotStale`、Worker 重同步增加、Gateway Ready 下降。
|
||||
|
||||
1. 检查 Controller、控制流和 outbox 延迟。
|
||||
2. 确认 Worker epoch 与 Controller epoch,禁止手工降低 epoch。
|
||||
3. 缺版本时强制完整 Snapshot,不要继续应用 Delta。
|
||||
4. `maxStaleAge` 内允许旧 Snapshot 服务;超限自动拒绝新流量并排空。
|
||||
5. 不得通过无限增大 `maxStaleAge` 隐藏控制面故障。
|
||||
|
||||
### 7.2 PostgreSQL 不可用
|
||||
|
||||
1. Gateway 继续使用最后有效 Snapshot。
|
||||
2. Controller 将 Distribution 和权威写操作置为不可用,避免返回未提交代理。
|
||||
3. Provider Fetch 停止写入;已有流量不受影响。
|
||||
4. 恢复后核对迁移、事务回滚、outbox backlog 与 Extract 审计连续性。
|
||||
|
||||
### 7.3 Redis 不可用
|
||||
|
||||
1. Gateway 不受影响。
|
||||
2. Controller 停止需要分布式互斥的高风险工作,防止多个 Fetch Leader。
|
||||
3. 本地限流只作为临时降级,不能声称满足全局额度。
|
||||
4. 恢复后确认 Leader 唯一、租约 epoch 单调和重复 Fetch 去重。
|
||||
|
||||
### 7.4 Provider 故障
|
||||
|
||||
1. 超时、DNS、认证、非预期 HTTP、响应超限与模板错误全部计 Error。
|
||||
2. 429 尊重 `Retry-After`,其余 Error 使用指数退避和 jitter。
|
||||
3. Error 不增加 Empty;合法候选为零才增加 Empty。
|
||||
4. Duplicate-only 重置 Empty 并记录独立指标。
|
||||
5. 达到 Empty 阈值后每条受影响 Routing 只原子切换一次。
|
||||
|
||||
### 7.5 Gateway 容量耗尽
|
||||
|
||||
1. 检查 Available Slots,而不是只看 Proxy 数量。
|
||||
2. 确认是否大量容量停留在 Reserved,排查 Commit/Cancel 泄漏。
|
||||
3. 检查 TTL safety margin、健康状态和 Worker 所有权是否导致候选被过滤。
|
||||
4. 快速拒绝新请求,禁止无界等待或把压力转移到 Controller。
|
||||
5. 扩容 Gateway 前确认存在可分配 Proxy 所有权切片。
|
||||
|
||||
### 7.6 Extract 库存不足
|
||||
|
||||
1. `partial` 返回实际数量;`allOrNothing` 不足时整批回滚。
|
||||
2. 检查 TTL、health age、filter、Worker ownership 和 `reserveForGateway`。
|
||||
3. 不得降低 `reserveForGateway` 到导致 Gateway 容量告警的水平。
|
||||
4. 回收 Worker-owned Proxy 必须先 DRAINING、等待 active/reserved 为零、清除
|
||||
ownership,再执行 `AVAILABLE -> EXTRACTED` 事务。
|
||||
5. 已提取代理没有 Release;客户端归还请求只记录为无效调用,不恢复库存。
|
||||
|
||||
### 7.7 Checker 积压
|
||||
|
||||
1. 优先新 Proxy 与 SUSPECT 复检。
|
||||
2. 降低稳定 AVAILABLE 的普通复检频率。
|
||||
3. 检查目标超时、DNS 与出口网络,再按任务延迟扩容 Checker。
|
||||
4. 队列必须有上限;不得无限积压耗尽内存或 Redis。
|
||||
|
||||
## 8. 备份与恢复
|
||||
|
||||
- PostgreSQL:每日全量、连续 WAL/PITR,至少每季度做恢复演练。
|
||||
- Redis:仅保存可重建协调状态;不得把 Redis 备份当权威业务备份。
|
||||
- 配置:版本化保存校验通过的不可变 Revision 与校验和。
|
||||
- Secret:由密钥平台版本化,日志和备份中不得出现明文。
|
||||
|
||||
恢复顺序:PostgreSQL -> Redis -> Controller -> Checker -> Gateway。恢复后验证
|
||||
Proxy 状态、Extraction Record、Worker ownership epoch、outbox 和配置 Revision
|
||||
单调一致,再开放 Gateway 与 Distribution。
|
||||
|
||||
## 9. Secret 轮换
|
||||
|
||||
1. 创建新 Secret 版本,不覆盖旧值。
|
||||
2. Provider/API 凭据支持双版本重叠时先发布新版本。
|
||||
3. 更新配置 Revision,确认 Controller 成功重建 Adapter。
|
||||
4. 观察 Fetch Error、认证失败与 Snapshot ACK。
|
||||
5. 所有副本应用后撤销旧 Secret。
|
||||
|
||||
Proxy 凭据轮换必须增加 `credentialVersion`,确保唯一键不会把新旧凭据错误合并。
|
||||
72
docs/requirements/completion-audit.md
Normal file
72
docs/requirements/completion-audit.md
Normal file
@ -0,0 +1,72 @@
|
||||
# 交付完成度审计
|
||||
|
||||
本文区分设计证据、机器契约、已运行验证和后续实施,防止把架构目标描述成
|
||||
已完成产品。
|
||||
|
||||
## 1. 本次已交付
|
||||
|
||||
### 设计与开发文档
|
||||
|
||||
- 全量需求追踪、覆盖关系和统一领域语言。
|
||||
- 产品设计、总体架构、项目结构、四项 ADR。
|
||||
- 开发、配置、Distribution/Admin API、控制面协议、安全、测试、运维文档。
|
||||
- 20 个配置场景和 35 张 Mermaid 架构/流程/状态/故障图。
|
||||
- 版本化文档包 `proxy-pool-docs-v1.0.zip`,包含 50 个条目。
|
||||
|
||||
### 机器契约
|
||||
|
||||
- Distribution OpenAPI:一次性独占提取、partial/allOrNothing、幂等键、
|
||||
TTL/健康过滤结果与标准错误。
|
||||
- Admin OpenAPI:状态、Upstream 启停、Routing 切换和配置重载。
|
||||
- Protobuf:Worker 注册、全量/增量 Snapshot、ACK、运行态/结果上报、Checker
|
||||
任务与 Observation。
|
||||
|
||||
### 核心参考实现
|
||||
|
||||
- `CFG-*`:YAML v4 未知字段拒绝、监听保护、引用/上限/认证边界校验,21 份
|
||||
配置持续测试。
|
||||
- `PROXY-* / CAP-*`:唯一键、TTL 优先级、状态迁移与 Active/Reserved 打包
|
||||
原子计数;1,000 goroutine 不超卖测试。
|
||||
- `ROUTE-001 / ROUTE-004`:首条命中规则与 Concurrent Sequential 单次切换。
|
||||
- `FETCH-005 / FETCH-006`:Valid、Empty、DuplicateOnly、Error 分类。
|
||||
- `DIST-001..003 / DIST-006..007`:内存事务模型验证独占提取、满足模式、TTL、
|
||||
健康时效与 Gateway 保留量;1,000 并发不重复。
|
||||
- `OPS-001`:完整 Snapshot 目标、epoch/version、校验和验证及原子替换。
|
||||
- `CAP-001 / GW 热路径边界`:本地 Dispatch 条件过滤与原子容量预留。
|
||||
|
||||
## 2. 已执行验证
|
||||
|
||||
```text
|
||||
go test ./... PASS
|
||||
go vet ./... PASS
|
||||
go build ./... PASS
|
||||
protoc descriptor compilation PASS
|
||||
docker compose config PASS
|
||||
kubectl kustomize PASS
|
||||
configuration examples 21/21 PASS
|
||||
Mermaid blocks 35
|
||||
```
|
||||
|
||||
Windows 环境为 `CGO_ENABLED=0` 且没有 C 编译器,`go test -race` 在本机未执行;
|
||||
CI 已配置 Linux race job。Docker/Kubernetes 仅完成静态验证,没有把目标拓扑
|
||||
作为已运行系统。
|
||||
|
||||
## 3. 后续实现范围
|
||||
|
||||
以下已有设计、接口或部署位置,但尚无端到端生产实现:
|
||||
|
||||
1. `cmd/proxy-gateway/controller/checker/loadgen` 进程装配。
|
||||
2. HTTP 正向代理、HTTPS CONNECT、连接池、安全重试与隧道转发。
|
||||
3. Provider Adapter、模板沙箱、singleflight、Leader、退避和累计额度执行器。
|
||||
4. PostgreSQL repository、Extraction 行锁事务、Outbox 和迁移。
|
||||
5. Redis Leader、速率限制、心跳与可重建协调适配器。
|
||||
6. Worker ownership drain/ACK/过期回收和网络快照流。
|
||||
7. Checker 调度、探测器和健康 reducer。
|
||||
8. Admin/Distribution handler、鉴权、限流和审计查询。
|
||||
9. 真实 Compose/Kubernetes 集成、故障演练和代表性集群负载测试。
|
||||
|
||||
## 4. 容量结论
|
||||
|
||||
100,000 QPS 是集群设计输入,不是本次验证结果。只有实现上述运行时,并在
|
||||
记录协议比例、代理 RTT、连接复用、Worker 规格、故障域、CPU/RSS/FD、延迟
|
||||
分位数和错误率的环境中通过持续压测后,才能声明已验证容量。
|
||||
45
docs/security/security-model.md
Normal file
45
docs/security/security-model.md
Normal file
@ -0,0 +1,45 @@
|
||||
# 安全模型
|
||||
|
||||
## 1. 信任边界
|
||||
|
||||
- Gateway、Distribution、Admin 和 Metrics 为四个独立监听边界。
|
||||
- Provider API 和上游 Proxy 属于外部不可信网络。
|
||||
- Worker 与 Controller 通道必须进行双向身份校验并绑定 cluster/worker。
|
||||
- PostgreSQL 保存权威状态;Redis 数据默认按可重建缓存与协调信息处理。
|
||||
|
||||
## 2. 入口控制
|
||||
|
||||
- 严格模式下,非回环监听必须配置认证或 CIDR allowlist。
|
||||
- 认证关闭不代表匿名状态消失:仍按可信代理链解析来源并形成 Client ID。
|
||||
- Access、Auth、Rate Limit 和 Client Identification 相互独立。
|
||||
- Admin 使用独立凭据,不能复用普通 Gateway 或 Distribution 凭据。
|
||||
|
||||
## 3. 目标地址策略
|
||||
|
||||
在解析和每次连接前同时检查:
|
||||
|
||||
- 私网、回环、链路本地、组播、保留地址和云元数据地址。
|
||||
- 域名解析出的全部 A/AAAA 地址,而不是只检查原始 Host。
|
||||
- 重定向或重试后的新目标,防止 DNS Rebinding 和策略绕过。
|
||||
- CONNECT 的端口 allowlist 与规范化 host:port。
|
||||
|
||||
## 4. Secret 处理
|
||||
|
||||
- 配置只保存环境变量或文件引用,不在日志中输出解析后的值。
|
||||
- Proxy 唯一键包含 username 与 credentialVersion,不包含密码或 SecretRef 内容。
|
||||
- 指标标签不得使用 token、Proxy URL、Client ID、session 或完整目标 URL。
|
||||
- Provider 响应和模板错误只记录分类、Upstream 和 requestId。
|
||||
|
||||
## 5. Provider 模板
|
||||
|
||||
- 限制响应体大小、模板执行时间和输出候选数量。
|
||||
- 模板函数采用白名单,禁止文件、网络、进程和环境变量访问。
|
||||
- URL、Header、Query 和 Body 分别结构化编码,不拼接未转义字符串。
|
||||
- 认证失败与限流响应分类处理,429 尊重有上限的 Retry-After。
|
||||
|
||||
## 6. 审计
|
||||
|
||||
Extraction 审计记录至少包含 requestId、Client、来源、Proxy ID、Upstream、
|
||||
提取时间和到期时间。日志脱敏不影响审计关联,但审计接口自身必须受 Admin
|
||||
权限保护并具备保留期限。
|
||||
|
||||
69
docs/testing/failure-injection.md
Normal file
69
docs/testing/failure-injection.md
Normal file
@ -0,0 +1,69 @@
|
||||
# 故障注入矩阵
|
||||
|
||||
## 执行规则
|
||||
|
||||
- 先在隔离环境建立 15 分钟稳定基线,再注入单一故障。
|
||||
- 每次只改变一个变量,记录开始/恢复时间与所有指标。
|
||||
- 任何数据库写故障后都核对 Proxy 状态、审计和 outbox,而非只看 HTTP 状态码。
|
||||
- 故障恢复后至少观察两个健康检查周期和一个 Snapshot 完整发布周期。
|
||||
|
||||
## 场景
|
||||
|
||||
### Provider
|
||||
|
||||
- DNS NXDOMAIN、连接超时、TLS 失败、401、429、500。
|
||||
- 响应超过 `maxResponseBytes`、模板超过 `templateTimeout`、非法 Proxy。
|
||||
- 合法空列表、全部重复、部分重复加部分新增。
|
||||
|
||||
预期:只有合法空列表增加 Empty;429 尊重 Retry-After;其余 Error 退避;
|
||||
duplicate-only 不触发 Sequential 切换。
|
||||
|
||||
### PostgreSQL
|
||||
|
||||
- 断开 60 秒、连接池耗尽、锁等待、事务提交失败、只读切换。
|
||||
|
||||
预期:Gateway 继续使用最后 Snapshot;Extract 不返回未提交记录;恢复后 outbox
|
||||
补发且不重复应用。
|
||||
|
||||
### Redis
|
||||
|
||||
- 断开、延迟 2 秒、Leader key 丢失、主从切换。
|
||||
|
||||
预期:Gateway 无影响;Provider Fetch 不出现多个有效 Leader;全局限流明确降级;
|
||||
恢复后 epoch 单调。
|
||||
|
||||
### Controller
|
||||
|
||||
- 杀死 Leader、滚动重启全部副本、阻断 Worker 控制流。
|
||||
|
||||
预期:`maxStaleAge` 内 Gateway 继续,之后停止新流量;Leader 切换不重复计费
|
||||
Fetch;Delta 缺口触发完整 Snapshot。
|
||||
|
||||
### Gateway
|
||||
|
||||
- 杀死一个 Pod、驱逐一个节点、丢失一个可用区、耗尽 FD。
|
||||
|
||||
预期:LoadBalancer 摘除 NotReady Pod;ownership 租约过期前不分给新 Worker;
|
||||
剩余容量满足已验证故障域目标。
|
||||
|
||||
### Checker
|
||||
|
||||
- 慢目标、DNS 延迟、队列积压、杀死一半 Pod。
|
||||
|
||||
预期:优先新 Proxy 和 SUSPECT;稳定 Proxy 降频;队列有界;Observation 重投
|
||||
不导致非法状态回退。
|
||||
|
||||
### Distribution
|
||||
|
||||
- 100 个并发请求争用相同 Proxy;数据库在事务提交时断开;Gateway 同时满载。
|
||||
|
||||
预期:每个 Proxy 最多返回一次;提交失败不返回代理;allOrNothing 整批回滚;
|
||||
池中始终剩余 `reserveForGateway`;没有 Release 恢复路径。
|
||||
|
||||
### 配置与 Secret
|
||||
|
||||
- 未知字段、错误正则、缺失 Upstream、公开监听无保护、凭据轮换失败。
|
||||
|
||||
预期:新 Revision 整体拒绝,旧不可变 Snapshot 继续;认证错误计 Error 而不是
|
||||
Empty;日志不出现 Secret。
|
||||
|
||||
49
docs/testing/strategy.md
Normal file
49
docs/testing/strategy.md
Normal file
@ -0,0 +1,49 @@
|
||||
# 测试策略
|
||||
|
||||
## 1. 分层
|
||||
|
||||
- **领域单测**:状态机、TTL、路由、容量、Fetch 分类和 Extraction 原子性。
|
||||
- **契约测试**:配置、OpenAPI、Protobuf 和 Provider Adapter fixture。
|
||||
- **集成测试**:PostgreSQL 事务、Redis Leader/限流、Outbox 与重建。
|
||||
- **端到端测试**:HTTP、CONNECT、Admin、Distribution 和优雅停机。
|
||||
- **负载测试**:Worker 调度微基准、50k 隧道 soak、集群 100k QPS 场景。
|
||||
|
||||
## 2. 必测不变量
|
||||
|
||||
1. 1,000 个并发预留不突破单 Proxy 最大并发。
|
||||
2. 同一 Proxy 在并发 Extraction 中最多出现一次。
|
||||
3. `allOrNothing` 不足时不消耗任何候选。
|
||||
4. 连续 4 次 Empty 后 Valid 不切换;连续 5 次只从 A 切到 B。
|
||||
5. Error 和 DuplicateOnly 不累计 Empty。
|
||||
6. 100 个缺池信号只形成一个合并 Provider reconcile。
|
||||
7. 并发 Fetch 不突破 `pool.maxSize` 与 `fetch.maxTotal`。
|
||||
8. TTL safety margin 内不再分配。
|
||||
9. Snapshot 版本断档、目标错误或校验和错误不替换当前视图。
|
||||
10. 非幂等 HTTP 和已建立 CONNECT 不自动重放。
|
||||
11. 所有 Upstream 不可用时严格执行显式策略。
|
||||
|
||||
## 3. 基础质量门禁
|
||||
|
||||
```powershell
|
||||
gofmt -l .
|
||||
go vet ./...
|
||||
go test ./...
|
||||
go test -race ./internal/...
|
||||
go build ./...
|
||||
```
|
||||
|
||||
单条测试命令超时 60 秒。依赖真实等待的用例必须改为 fake clock;集成和
|
||||
soak 测试单独标记,不混入快速单测。
|
||||
|
||||
## 4. 100k QPS 验收
|
||||
|
||||
测试报告必须记录:
|
||||
|
||||
- CPU、内存、内核、网卡、文件描述符和 conntrack 配置。
|
||||
- Worker 数量、故障域、目标利用率和负载均衡算法。
|
||||
- HTTP/CONNECT 比例、keep-alive、请求/响应大小、上游 RTT 与失败率。
|
||||
- Proxy 总数、每 Proxy 容量、Routing 数和 Snapshot 更新频率。
|
||||
- 持续时间、P50/P95/P99、成功率、重试率、GC、RSS 和 goroutine 数。
|
||||
|
||||
只有在代表性环境持续达到目标且满足错误率和延迟门槛后,才能把“设计目标”
|
||||
改为“已验证容量”。
|
||||
130
docs/testing/test-strategy.md
Normal file
130
docs/testing/test-strategy.md
Normal file
@ -0,0 +1,130 @@
|
||||
# 测试与容量验证策略
|
||||
|
||||
## 1. 原则
|
||||
|
||||
- 先证明领域不变量,再证明 Adapter 契约,最后证明跨进程行为。
|
||||
- 并发测试必须在 race detector 下运行,不能只依赖单线程示例。
|
||||
- 时间、随机、网络、Provider 和存储均通过可替换接口或 fixture 控制。
|
||||
- 100,000 QPS 是待验证的集群目标,不以架构图、副本数或短时峰值替代证据。
|
||||
- 性能通过与正确性通过相互独立;高 QPS 下出现超卖、重复 Extract 或状态
|
||||
回退时,结果一律失败。
|
||||
|
||||
## 2. 测试分层
|
||||
|
||||
### 单元测试
|
||||
|
||||
- Proxy 唯一键、TTL 优先级、状态机和 safety margin。
|
||||
- Atomic Capacity 的 Reserve、Commit、Cancel、Release 与幂等错误。
|
||||
- Routing first-match 与五种策略。
|
||||
- Fetch Success、Empty、Duplicate-only、Error 分类。
|
||||
- Backoff、jitter、Retry-After、requestInterval、maxInFlight。
|
||||
- Extraction eligibility、partial、allOrNothing、health age、TTL、reserve。
|
||||
- 配置严格字段、交叉引用、正则、监听保护和独立计数语义。
|
||||
|
||||
### 契约测试
|
||||
|
||||
- Provider 响应模板的大小、超时、函数白名单和解析边界。
|
||||
- PostgreSQL `FOR UPDATE SKIP LOCKED` 并发批量提取。
|
||||
- Outbox 状态更新、发布与幂等重放。
|
||||
- Redis Leader 租约、限流和失联恢复。
|
||||
- Snapshot/Delta/ACK/Report 的版本与校验和兼容性。
|
||||
- OpenAPI 错误模型、认证矩阵、批量 fulfillment。
|
||||
|
||||
### 集成与端到端
|
||||
|
||||
- HTTP 正向代理成功、上游连接前失败和安全重试。
|
||||
- HTTPS CONNECT 建立后不透明重放。
|
||||
- Controller Fetch -> Check -> AVAILABLE -> Worker Snapshot -> Gateway 转发。
|
||||
- Distribution 原子提取后 Gateway 不再分配同一 Proxy。
|
||||
- 配置热更新失败保留旧 Revision,成功后新请求使用新 Snapshot。
|
||||
|
||||
## 3. 必测的 11 类场景
|
||||
|
||||
1. **并发容量**:1000 协程争用同一 Proxy,始终满足
|
||||
`active + reserved <= effectiveMaxConcurrency`。
|
||||
2. **Reservation 生命周期**:Dial 成功/失败、超时、取消和重复 Release 均不
|
||||
泄漏或产生负计数。
|
||||
3. **singleflight**:100 个缺池信号只产生一个有效 Fetch 调度。
|
||||
4. **Provider 限流**:requestInterval、maxInFlight、timeout、重试和 429
|
||||
`Retry-After` 在虚拟时钟下准确。
|
||||
5. **Fetch 分类**:Error 不动 Empty;合法空响应 Empty++;duplicate-only
|
||||
重置 Empty;Success 重置 Empty。
|
||||
6. **Sequential 竞态**:达到阈值时多协程只能将 A 切到 B 一次,不能越过 B。
|
||||
7. **Drain**:切换或禁用 Upstream 后停止新分配,已有连接完成后才回收。
|
||||
8. **Extract 竞态**:多个请求并发提取同一候选集合,每个 Proxy 最多返回一次。
|
||||
9. **Extract 批量语义**:partial 提交实际数量;allOrNothing 不足时状态和审计
|
||||
全回滚;始终保留 `reserveForGateway`。
|
||||
10. **Worker ownership**:回收过程严格经过 DRAINING、ACK、active/reserved=0、
|
||||
unowned,旧 Snapshot 不可再分配。
|
||||
11. **控制面故障**:Redis、PostgreSQL、Controller、Checker 与 Provider 分别
|
||||
失效时,行为与 Runbook 一致,Gateway 热路径不被同步依赖拖垮。
|
||||
|
||||
## 4. 测试命令
|
||||
|
||||
所有后台测试都设置 60 秒上限:
|
||||
|
||||
```powershell
|
||||
go test -timeout 60s ./...
|
||||
go test -timeout 60s -race ./internal/...
|
||||
go vet ./...
|
||||
go build ./...
|
||||
```
|
||||
|
||||
需要 PostgreSQL/Redis 的测试使用独立数据库和短生命周期容器,不复用开发数据。
|
||||
测试结束后验证没有残留 Worker ownership、Leader 租约或未提交 Extraction Record。
|
||||
|
||||
## 5. 负载模型
|
||||
|
||||
必须分开运行,避免不同瓶颈互相掩盖:
|
||||
|
||||
### HTTP QPS
|
||||
|
||||
- GET/HEAD 占比、响应体大小、Keep-Alive 复用率与生产预测一致。
|
||||
- 依次运行 10k 稳态、阶梯升压和 100k 峰值。
|
||||
- 同时记录端到端与 Gateway 内部 Dispatch 延迟。
|
||||
|
||||
### CONNECT
|
||||
|
||||
- 分开测试活跃隧道数和每秒新建隧道数。
|
||||
- 包含短连接、长连接、半关闭、Client 取消和上游主动断开。
|
||||
- 验证 200 已发送后不发生透明重放。
|
||||
|
||||
### Snapshot
|
||||
|
||||
- 1k、10k、100k Proxy Snapshot,测构建、校验、原子切换、内存峰值和 GC。
|
||||
- 在满负载下发布 Snapshot,确认请求线程不参与索引构建。
|
||||
|
||||
### Extract
|
||||
|
||||
- 小批 partial、大批 allOrNothing、高冲突 filter 和库存不足。
|
||||
- 与 Gateway 同时运行,持续检查 `reserveForGateway` 和无重复返回。
|
||||
|
||||
### 故障负载
|
||||
|
||||
- 负载运行中断开 Controller、Redis、PostgreSQL、一个 Worker 和一个可用区。
|
||||
- Provider 注入 DNS、超时、500、429、超大响应、模板错误和合法空响应。
|
||||
- Checker 注入慢目标与队列积压。
|
||||
|
||||
## 6. 通过条件
|
||||
|
||||
业务 SLO 由产品最终确认,但至少满足以下工程门槛:
|
||||
|
||||
- 无容量超卖、负计数、重复 Extract、审计缺失或状态非法回退。
|
||||
- 100k 峰值期间无进程 OOM、FD 耗尽、无界队列或全局锁热点。
|
||||
- Gateway 热路径在 PostgreSQL、Redis、Provider 失效时不发起同步访问。
|
||||
- p99 Dispatch 小于 100 微秒的设计预算需要在 100k Proxy Snapshot 下单独证明。
|
||||
- 最大故障域丢失后,剩余容量仍满足约定 SLO;否则增加副本或降低承诺容量。
|
||||
- Snapshot 超过 `maxStaleAge` 后 Gateway 拒绝新流量,已有连接按时排空。
|
||||
- 所有数据、命令、Git SHA、镜像 digest、环境和原始指标可以复现。
|
||||
|
||||
## 7. 测试报告模板
|
||||
|
||||
```text
|
||||
版本:Git SHA / image digest / Go version
|
||||
环境:节点、CPU、内存、NIC、内核、Kubernetes/CNI
|
||||
配置:Snapshot 规模、Proxy 容量、路由、重试、日志级别
|
||||
场景:协议、连接复用、响应体、持续时间、升压曲线、故障注入
|
||||
结果:QPS、建连速率、active、p50/p95/p99、错误、CPU、RSS、GC、FD、网络
|
||||
不变量:capacity、ownership、extraction、audit、reserve 检查结果
|
||||
结论:通过/失败,以及适用边界
|
||||
```
|
||||
53
examples/config/01-local-all.yaml
Normal file
53
examples/config/01-local-all.yaml
Normal file
@ -0,0 +1,53 @@
|
||||
# 本机同时启用 Gateway 与一次性独占提取 API。
|
||||
version: 1
|
||||
security:
|
||||
requireProtectionOnPublicListen: true
|
||||
gateway:
|
||||
enabled: true
|
||||
listen: 127.0.0.1:8080
|
||||
auth: {mode: none}
|
||||
retry:
|
||||
maxAttempts: 2
|
||||
retryMethods: [GET, HEAD]
|
||||
distribution:
|
||||
enabled: true
|
||||
listen: 127.0.0.1:8081
|
||||
auth: {mode: none}
|
||||
clientIdentification: {mode: sourceIP}
|
||||
extraction:
|
||||
fulfillment: partial
|
||||
maxCountPerRequest: 20
|
||||
minRemainingTTL: 30s
|
||||
maxHealthCheckAge: 15s
|
||||
reserveForGateway: 5
|
||||
routing:
|
||||
- name: gateway-default
|
||||
enabled: true
|
||||
purpose: gateway
|
||||
match: {hostRegex: '.*'}
|
||||
upstreams: [provider-a]
|
||||
strategy: {type: leastConnections}
|
||||
onUnavailable: {action: reject}
|
||||
- name: extract-default
|
||||
enabled: true
|
||||
purpose: extract
|
||||
match: {hostRegex: '.*'}
|
||||
upstreams: [provider-a]
|
||||
strategy: {type: random}
|
||||
onUnavailable: {action: reject}
|
||||
upstreams:
|
||||
provider-a:
|
||||
enabled: true
|
||||
exposure: [gateway, extract]
|
||||
provider: {billingMode: fetch, protocols: [http]}
|
||||
api:
|
||||
url: https://provider-a.example/proxies
|
||||
method: GET
|
||||
auth: {type: none}
|
||||
template: '{{.}}'
|
||||
proxyAuth: {type: response}
|
||||
pool: {maxSize: 100}
|
||||
capacity: {maxConcurrencyPerProxy: 10}
|
||||
lifecycle: {ttl: 2m, allocationSafetyMargin: 10s}
|
||||
fetch: {requestInterval: 1s, timeout: 3s, maxAttempts: 3, maxInFlight: 1, maxTotal: 1000}
|
||||
check: {interval: 30s, jitter: 20, maxInFlight: 50, timeout: 2s, maxAttempts: 2, maxConsecutiveFailures: 3}
|
||||
33
examples/config/02-gateway-only.yaml
Normal file
33
examples/config/02-gateway-only.yaml
Normal file
@ -0,0 +1,33 @@
|
||||
# 仅提供本机 HTTP/CONNECT 网关。
|
||||
version: 1
|
||||
security: {requireProtectionOnPublicListen: true}
|
||||
gateway:
|
||||
enabled: true
|
||||
listen: 127.0.0.1:8080
|
||||
auth: {mode: none}
|
||||
limits: {maxConcurrentConnections: 20000}
|
||||
retry: {maxAttempts: 2, retryMethods: [GET, HEAD]}
|
||||
destinationPolicy:
|
||||
denyPrivateNetworks: true
|
||||
denyLoopback: true
|
||||
denyLinkLocal: true
|
||||
routing:
|
||||
- name: gateway
|
||||
enabled: true
|
||||
purpose: gateway
|
||||
match: {hostRegex: '.*'}
|
||||
upstreams: [provider-a]
|
||||
strategy: {type: leastConnections}
|
||||
onUnavailable: {action: reject}
|
||||
upstreams:
|
||||
provider-a:
|
||||
enabled: true
|
||||
exposure: [gateway]
|
||||
provider: {billingMode: subscription, protocols: [http, https]}
|
||||
api: {url: https://provider-a.example/proxies, method: GET, auth: {type: none}, template: '{{.}}'}
|
||||
proxyAuth: {type: response}
|
||||
pool: {maxSize: 2000}
|
||||
capacity: {maxConcurrencyPerProxy: 20}
|
||||
lifecycle: {ttl: 5m, allocationSafetyMargin: 30s}
|
||||
fetch: {requestInterval: 1s, timeout: 3s, maxAttempts: 3, maxInFlight: 1}
|
||||
check: {interval: 30s, jitter: 20, maxInFlight: 100, timeout: 2s, maxAttempts: 2, maxConsecutiveFailures: 3}
|
||||
35
examples/config/03-extract-only.yaml
Normal file
35
examples/config/03-extract-only.yaml
Normal file
@ -0,0 +1,35 @@
|
||||
# 仅提供一次性独占提取;没有 lease/release 配置。
|
||||
version: 1
|
||||
security: {requireProtectionOnPublicListen: true}
|
||||
distribution:
|
||||
enabled: true
|
||||
listen: 127.0.0.1:8081
|
||||
auth: {mode: none}
|
||||
clientIdentification: {mode: sourceIP}
|
||||
limits: {requestsPerMinute: 60, requestsPerMinutePerClient: 30}
|
||||
extraction:
|
||||
fulfillment: partial
|
||||
maxCountPerRequest: 50
|
||||
minRemainingTTL: 30s
|
||||
maxHealthCheckAge: 15s
|
||||
reserveForGateway: 0
|
||||
routing:
|
||||
- name: extract
|
||||
enabled: true
|
||||
purpose: extract
|
||||
match: {hostRegex: '.*'}
|
||||
upstreams: [provider-a]
|
||||
strategy: {type: random}
|
||||
onUnavailable: {action: reject}
|
||||
upstreams:
|
||||
provider-a:
|
||||
enabled: true
|
||||
exposure: [extract]
|
||||
provider: {billingMode: fetch, protocols: [http]}
|
||||
api: {url: https://provider-a.example/proxies, method: GET, auth: {type: none}, template: '{{.}}'}
|
||||
proxyAuth: {type: response}
|
||||
pool: {maxSize: 500}
|
||||
capacity: {maxConcurrencyPerProxy: 1}
|
||||
lifecycle: {ttl: 2m, allocationSafetyMargin: 10s}
|
||||
fetch: {requestInterval: 1s, timeout: 3s, maxAttempts: 3, maxInFlight: 1, maxTotal: 10000}
|
||||
check: {interval: 15s, jitter: 20, maxInFlight: 100, timeout: 2s, maxAttempts: 2, maxConsecutiveFailures: 3}
|
||||
37
examples/config/04-public-gateway-basic-auth.yaml
Normal file
37
examples/config/04-public-gateway-basic-auth.yaml
Normal file
@ -0,0 +1,37 @@
|
||||
# 公网 Gateway 使用用户名密码,并拒绝内网/回环目标。
|
||||
version: 1
|
||||
security: {requireProtectionOnPublicListen: true}
|
||||
gateway:
|
||||
enabled: true
|
||||
listen: 0.0.0.0:8080
|
||||
auth:
|
||||
mode: usernamePassword
|
||||
username: "${GATEWAY_USER}"
|
||||
password: "${GATEWAY_PASSWORD}"
|
||||
limits: {maxConcurrentConnections: 50000}
|
||||
retry: {maxAttempts: 2, retryMethods: [GET, HEAD]}
|
||||
destinationPolicy:
|
||||
denyPrivateNetworks: true
|
||||
denyLoopback: true
|
||||
denyLinkLocal: true
|
||||
denyCIDRs: [169.254.169.254/32]
|
||||
routing:
|
||||
- name: public-gateway
|
||||
enabled: true
|
||||
purpose: gateway
|
||||
match: {hostRegex: '.*'}
|
||||
upstreams: [provider-a]
|
||||
strategy: {type: leastConnections}
|
||||
onUnavailable: {action: reject}
|
||||
upstreams:
|
||||
provider-a:
|
||||
enabled: true
|
||||
exposure: [gateway]
|
||||
provider: {billingMode: subscription, protocols: [http]}
|
||||
api: {url: https://provider-a.example/proxies, method: GET, auth: {type: none}, template: '{{.}}'}
|
||||
proxyAuth: {type: response}
|
||||
pool: {maxSize: 3000}
|
||||
capacity: {maxConcurrencyPerProxy: 20}
|
||||
lifecycle: {ttl: 5m, allocationSafetyMargin: 30s}
|
||||
fetch: {requestInterval: 1s, timeout: 3s, maxAttempts: 3, maxInFlight: 1}
|
||||
check: {interval: 30s, jitter: 20, maxInFlight: 200, timeout: 2s, maxAttempts: 2, maxConsecutiveFailures: 3}
|
||||
38
examples/config/05-public-extract-api-key.yaml
Normal file
38
examples/config/05-public-extract-api-key.yaml
Normal file
@ -0,0 +1,38 @@
|
||||
# 公网提取 API 使用 X-API-Key;成功后代理立即 EXTRACTED。
|
||||
version: 1
|
||||
security: {requireProtectionOnPublicListen: true}
|
||||
distribution:
|
||||
enabled: true
|
||||
listen: 0.0.0.0:8081
|
||||
auth:
|
||||
mode: apiKey
|
||||
header: X-API-Key
|
||||
token: "${DISTRIBUTION_API_KEY}"
|
||||
clientIdentification: {mode: authenticatedClient}
|
||||
limits: {requestsPerMinute: 6000, requestsPerMinutePerClient: 300}
|
||||
extraction:
|
||||
fulfillment: partial
|
||||
maxCountPerRequest: 100
|
||||
minRemainingTTL: 45s
|
||||
maxHealthCheckAge: 10s
|
||||
reserveForGateway: 0
|
||||
routing:
|
||||
- name: public-extract
|
||||
enabled: true
|
||||
purpose: extract
|
||||
match: {hostRegex: '.*'}
|
||||
upstreams: [provider-a]
|
||||
strategy: {type: random}
|
||||
onUnavailable: {action: reject}
|
||||
upstreams:
|
||||
provider-a:
|
||||
enabled: true
|
||||
exposure: [extract]
|
||||
provider: {billingMode: fetch, protocols: [http]}
|
||||
api: {url: https://provider-a.example/proxies, method: GET, auth: {type: none}, template: '{{.}}'}
|
||||
proxyAuth: {type: response}
|
||||
pool: {maxSize: 1000}
|
||||
capacity: {maxConcurrencyPerProxy: 1}
|
||||
lifecycle: {ttl: 3m, allocationSafetyMargin: 30s}
|
||||
fetch: {requestInterval: 1s, timeout: 3s, maxAttempts: 3, maxInFlight: 1, maxTotal: 100000}
|
||||
check: {interval: 15s, jitter: 20, maxInFlight: 200, timeout: 2s, maxAttempts: 2, maxConsecutiveFailures: 3}
|
||||
45
examples/config/06-internal-cidr-no-auth.yaml
Normal file
45
examples/config/06-internal-cidr-no-auth.yaml
Normal file
@ -0,0 +1,45 @@
|
||||
# 内网关闭认证,但用来源 CIDR 独立保护两个入口。
|
||||
version: 1
|
||||
security: {requireProtectionOnPublicListen: true}
|
||||
gateway:
|
||||
enabled: true
|
||||
listen: 0.0.0.0:8080
|
||||
access: {allowCIDRs: [10.0.0.0/8, 192.168.0.0/16]}
|
||||
auth: {mode: none}
|
||||
distribution:
|
||||
enabled: true
|
||||
listen: 0.0.0.0:8081
|
||||
access:
|
||||
allowCIDRs: [10.0.0.0/8, 192.168.0.0/16]
|
||||
trustedProxies: [10.10.0.10/32]
|
||||
auth: {mode: none}
|
||||
clientIdentification: {mode: sourceIP}
|
||||
limits: {requestsPerMinutePerClient: 30}
|
||||
extraction: {fulfillment: partial, maxCountPerRequest: 10, minRemainingTTL: 30s, maxHealthCheckAge: 15s, reserveForGateway: 10}
|
||||
routing:
|
||||
- name: shared-gateway
|
||||
enabled: true
|
||||
purpose: gateway
|
||||
match: {hostRegex: '.*'}
|
||||
upstreams: [provider-a]
|
||||
strategy: {type: leastConnections}
|
||||
onUnavailable: {action: reject}
|
||||
- name: shared-extract
|
||||
enabled: true
|
||||
purpose: extract
|
||||
match: {hostRegex: '.*'}
|
||||
upstreams: [provider-a]
|
||||
strategy: {type: random}
|
||||
onUnavailable: {action: reject}
|
||||
upstreams:
|
||||
provider-a:
|
||||
enabled: true
|
||||
exposure: [gateway, extract]
|
||||
provider: {billingMode: subscription, protocols: [http]}
|
||||
api: {url: https://provider-a.example/proxies, method: GET, auth: {type: none}, template: '{{.}}'}
|
||||
proxyAuth: {type: response}
|
||||
pool: {maxSize: 1000}
|
||||
capacity: {maxConcurrencyPerProxy: 10}
|
||||
lifecycle: {ttl: 2m, allocationSafetyMargin: 15s}
|
||||
fetch: {requestInterval: 1s, timeout: 3s, maxAttempts: 3, maxInFlight: 1}
|
||||
check: {interval: 20s, jitter: 20, maxInFlight: 100, timeout: 2s, maxAttempts: 2, maxConsecutiveFailures: 3}
|
||||
37
examples/config/07-auth-any.yaml
Normal file
37
examples/config/07-auth-any.yaml
Normal file
@ -0,0 +1,37 @@
|
||||
# 来源在可信网段或提供 API Key,任一方法通过即可。
|
||||
version: 1
|
||||
security: {requireProtectionOnPublicListen: true}
|
||||
distribution:
|
||||
enabled: true
|
||||
listen: 0.0.0.0:8081
|
||||
auth:
|
||||
mode: any
|
||||
methods:
|
||||
- mode: ipWhitelist
|
||||
cidrs: [10.0.0.0/8]
|
||||
- mode: apiKey
|
||||
header: X-API-Key
|
||||
value: "${DISTRIBUTION_API_KEY}"
|
||||
clientIdentification: {mode: authenticatedClientOrSourceIP}
|
||||
limits: {requestsPerMinute: 600, requestsPerMinutePerClient: 60}
|
||||
extraction: {fulfillment: partial, maxCountPerRequest: 20, minRemainingTTL: 30s, maxHealthCheckAge: 15s, reserveForGateway: 0}
|
||||
routing:
|
||||
- name: extract
|
||||
enabled: true
|
||||
purpose: extract
|
||||
match: {hostRegex: '.*'}
|
||||
upstreams: [provider-a]
|
||||
strategy: {type: random}
|
||||
onUnavailable: {action: reject}
|
||||
upstreams:
|
||||
provider-a:
|
||||
enabled: true
|
||||
exposure: [extract]
|
||||
provider: {billingMode: fetch, protocols: [http]}
|
||||
api: {url: https://provider-a.example/proxies, method: GET, auth: {type: none}, template: '{{.}}'}
|
||||
proxyAuth: {type: response}
|
||||
pool: {maxSize: 500}
|
||||
capacity: {maxConcurrencyPerProxy: 1}
|
||||
lifecycle: {ttl: 2m, allocationSafetyMargin: 15s}
|
||||
fetch: {requestInterval: 1s, timeout: 3s, maxAttempts: 3, maxInFlight: 1, maxTotal: 10000}
|
||||
check: {interval: 15s, jitter: 20, maxInFlight: 100, timeout: 2s, maxAttempts: 2, maxConsecutiveFailures: 3}
|
||||
38
examples/config/08-sequential-failover.yaml
Normal file
38
examples/config/08-sequential-failover.yaml
Normal file
@ -0,0 +1,38 @@
|
||||
# Provider 返回连续 5 次合法空结果后,Routing 从 A 原子前进到 B。
|
||||
version: 1
|
||||
security: {requireProtectionOnPublicListen: true}
|
||||
gateway: {enabled: true, listen: '127.0.0.1:8080', auth: {mode: none}}
|
||||
routing:
|
||||
- name: sequential-gateway
|
||||
enabled: true
|
||||
purpose: gateway
|
||||
match: {hostRegex: '.*'}
|
||||
upstreams: [provider-a, provider-b]
|
||||
strategy:
|
||||
type: sequential
|
||||
switchAfterEmptyFetch: 5
|
||||
endBehavior: stayLast
|
||||
onUnavailable: {action: reject}
|
||||
upstreams:
|
||||
provider-a:
|
||||
enabled: true
|
||||
exposure: [gateway]
|
||||
provider: {billingMode: fetch, protocols: [http]}
|
||||
api: {url: https://provider-a.example/proxies, method: GET, auth: {type: none}, template: '{{.}}'}
|
||||
proxyAuth: {type: response}
|
||||
pool: {maxSize: 500}
|
||||
capacity: {maxConcurrencyPerProxy: 10}
|
||||
lifecycle: {ttl: 2m, allocationSafetyMargin: 15s}
|
||||
fetch: {requestInterval: 1s, timeout: 3s, maxAttempts: 3, maxInFlight: 1, maxTotal: 10000}
|
||||
check: {interval: 30s, jitter: 20, maxInFlight: 100, timeout: 2s, maxAttempts: 2, maxConsecutiveFailures: 3}
|
||||
provider-b:
|
||||
enabled: true
|
||||
exposure: [gateway]
|
||||
provider: {billingMode: subscription, protocols: [http]}
|
||||
api: {url: https://provider-b.example/proxies, method: GET, auth: {type: none}, template: '{{.}}'}
|
||||
proxyAuth: {type: response}
|
||||
pool: {maxSize: 500}
|
||||
capacity: {maxConcurrencyPerProxy: 10}
|
||||
lifecycle: {ttl: 3m, allocationSafetyMargin: 15s}
|
||||
fetch: {requestInterval: 2s, timeout: 3s, maxAttempts: 3, maxInFlight: 1}
|
||||
check: {interval: 30s, jitter: 20, maxInFlight: 100, timeout: 2s, maxAttempts: 2, maxConsecutiveFailures: 3}
|
||||
37
examples/config/09-weighted-routing.yaml
Normal file
37
examples/config/09-weighted-routing.yaml
Normal file
@ -0,0 +1,37 @@
|
||||
# 70/30 权重只决定 Upstream 选择,不复制 Upstream 运行态。
|
||||
version: 1
|
||||
security: {requireProtectionOnPublicListen: true}
|
||||
gateway: {enabled: true, listen: '127.0.0.1:8080', auth: {mode: none}}
|
||||
routing:
|
||||
- name: weighted-gateway
|
||||
enabled: true
|
||||
purpose: gateway
|
||||
match: {hostRegex: '.*'}
|
||||
upstreams: [provider-a, provider-b]
|
||||
strategy:
|
||||
type: weighted
|
||||
weights: {provider-a: 70, provider-b: 30}
|
||||
onUnavailable: {action: reject}
|
||||
upstreams:
|
||||
provider-a:
|
||||
enabled: true
|
||||
exposure: [gateway]
|
||||
provider: {billingMode: subscription, protocols: [http]}
|
||||
api: {url: https://provider-a.example/proxies, method: GET, auth: {type: none}, template: '{{.}}'}
|
||||
proxyAuth: {type: response}
|
||||
pool: {maxSize: 1000}
|
||||
capacity: {maxConcurrencyPerProxy: 10}
|
||||
lifecycle: {ttl: 2m, allocationSafetyMargin: 15s}
|
||||
fetch: {requestInterval: 1s, timeout: 3s, maxAttempts: 3, maxInFlight: 1}
|
||||
check: {interval: 30s, jitter: 20, maxInFlight: 100, timeout: 2s, maxAttempts: 2, maxConsecutiveFailures: 3}
|
||||
provider-b:
|
||||
enabled: true
|
||||
exposure: [gateway]
|
||||
provider: {billingMode: subscription, protocols: [http]}
|
||||
api: {url: https://provider-b.example/proxies, method: GET, auth: {type: none}, template: '{{.}}'}
|
||||
proxyAuth: {type: response}
|
||||
pool: {maxSize: 1000}
|
||||
capacity: {maxConcurrencyPerProxy: 10}
|
||||
lifecycle: {ttl: 2m, allocationSafetyMargin: 15s}
|
||||
fetch: {requestInterval: 1s, timeout: 3s, maxAttempts: 3, maxInFlight: 1}
|
||||
check: {interval: 30s, jitter: 20, maxInFlight: 100, timeout: 2s, maxAttempts: 2, maxConsecutiveFailures: 3}
|
||||
31
examples/config/10-round-robin-routing.yaml
Normal file
31
examples/config/10-round-robin-routing.yaml
Normal file
@ -0,0 +1,31 @@
|
||||
# 在可用 Upstream 间轮询。
|
||||
version: 1
|
||||
security: {requireProtectionOnPublicListen: true}
|
||||
gateway: {enabled: true, listen: '127.0.0.1:8080', auth: {mode: none}}
|
||||
routing:
|
||||
- name: round-robin
|
||||
enabled: true
|
||||
purpose: gateway
|
||||
match: {hostRegex: '.*'}
|
||||
upstreams: [provider-a, provider-b]
|
||||
strategy: {type: roundRobin}
|
||||
onUnavailable: {action: reject}
|
||||
upstreams:
|
||||
provider-a:
|
||||
enabled: true
|
||||
exposure: [gateway]
|
||||
provider: {billingMode: subscription, protocols: [http]}
|
||||
api: {url: https://provider-a.example/proxies, method: GET, auth: {type: none}, template: '{{.}}'}
|
||||
proxyAuth: {type: response}
|
||||
pool: {maxSize: 1000}
|
||||
capacity: {maxConcurrencyPerProxy: 10}
|
||||
fetch: {requestInterval: 1s, timeout: 3s, maxAttempts: 3, maxInFlight: 1}
|
||||
provider-b:
|
||||
enabled: true
|
||||
exposure: [gateway]
|
||||
provider: {billingMode: subscription, protocols: [http]}
|
||||
api: {url: https://provider-b.example/proxies, method: GET, auth: {type: none}, template: '{{.}}'}
|
||||
proxyAuth: {type: response}
|
||||
pool: {maxSize: 1000}
|
||||
capacity: {maxConcurrencyPerProxy: 10}
|
||||
fetch: {requestInterval: 1s, timeout: 3s, maxAttempts: 3, maxInFlight: 1}
|
||||
31
examples/config/11-random-routing.yaml
Normal file
31
examples/config/11-random-routing.yaml
Normal file
@ -0,0 +1,31 @@
|
||||
# 每次从可用 Upstream 集合随机选择。
|
||||
version: 1
|
||||
security: {requireProtectionOnPublicListen: true}
|
||||
gateway: {enabled: true, listen: '127.0.0.1:8080', auth: {mode: none}}
|
||||
routing:
|
||||
- name: random-gateway
|
||||
enabled: true
|
||||
purpose: gateway
|
||||
match: {hostRegex: '.*'}
|
||||
upstreams: [provider-a, provider-b]
|
||||
strategy: {type: random}
|
||||
onUnavailable: {action: reject}
|
||||
upstreams:
|
||||
provider-a:
|
||||
enabled: true
|
||||
exposure: [gateway]
|
||||
provider: {billingMode: subscription, protocols: [http]}
|
||||
api: {url: https://provider-a.example/proxies, method: GET, auth: {type: none}, template: '{{.}}'}
|
||||
proxyAuth: {type: response}
|
||||
pool: {maxSize: 1000}
|
||||
capacity: {maxConcurrencyPerProxy: 10}
|
||||
fetch: {requestInterval: 1s, timeout: 3s, maxAttempts: 3, maxInFlight: 1}
|
||||
provider-b:
|
||||
enabled: true
|
||||
exposure: [gateway]
|
||||
provider: {billingMode: subscription, protocols: [http]}
|
||||
api: {url: https://provider-b.example/proxies, method: GET, auth: {type: none}, template: '{{.}}'}
|
||||
proxyAuth: {type: response}
|
||||
pool: {maxSize: 1000}
|
||||
capacity: {maxConcurrencyPerProxy: 10}
|
||||
fetch: {requestInterval: 1s, timeout: 3s, maxAttempts: 3, maxInFlight: 1}
|
||||
28
examples/config/12-least-connections-routing.yaml
Normal file
28
examples/config/12-least-connections-routing.yaml
Normal file
@ -0,0 +1,28 @@
|
||||
# Gateway 按本地 Active + Reserved 选择剩余容量最高的 Proxy。
|
||||
version: 1
|
||||
security: {requireProtectionOnPublicListen: true}
|
||||
gateway:
|
||||
enabled: true
|
||||
listen: 127.0.0.1:8080
|
||||
auth: {mode: none}
|
||||
limits: {maxConcurrentConnections: 100000}
|
||||
routing:
|
||||
- name: least-connections
|
||||
enabled: true
|
||||
purpose: gateway
|
||||
match: {hostRegex: '.*'}
|
||||
upstreams: [provider-a]
|
||||
strategy: {type: leastConnections}
|
||||
onUnavailable: {action: reject}
|
||||
upstreams:
|
||||
provider-a:
|
||||
enabled: true
|
||||
exposure: [gateway]
|
||||
provider: {billingMode: subscription, protocols: [http, https]}
|
||||
api: {url: https://provider-a.example/proxies, method: GET, auth: {type: none}, template: '{{.}}'}
|
||||
proxyAuth: {type: response}
|
||||
pool: {maxSize: 10000}
|
||||
capacity: {maxConcurrencyPerProxy: 50}
|
||||
lifecycle: {ttl: 5m, allocationSafetyMargin: 30s}
|
||||
fetch: {requestInterval: 1s, timeout: 3s, maxAttempts: 3, maxInFlight: 1}
|
||||
check: {interval: 30s, jitter: 20, maxInFlight: 500, timeout: 2s, maxAttempts: 2, maxConsecutiveFailures: 3}
|
||||
34
examples/config/13-extract-all-or-nothing.yaml
Normal file
34
examples/config/13-extract-all-or-nothing.yaml
Normal file
@ -0,0 +1,34 @@
|
||||
# 数量不足时返回 409,事务不改变任何 Proxy 状态。
|
||||
version: 1
|
||||
security: {requireProtectionOnPublicListen: true}
|
||||
distribution:
|
||||
enabled: true
|
||||
listen: 127.0.0.1:8081
|
||||
auth: {mode: none}
|
||||
clientIdentification: {mode: sourceIP}
|
||||
extraction:
|
||||
fulfillment: allOrNothing
|
||||
maxCountPerRequest: 100
|
||||
minRemainingTTL: 30s
|
||||
maxHealthCheckAge: 15s
|
||||
reserveForGateway: 0
|
||||
routing:
|
||||
- name: extract
|
||||
enabled: true
|
||||
purpose: extract
|
||||
match: {hostRegex: '.*'}
|
||||
upstreams: [provider-a]
|
||||
strategy: {type: random}
|
||||
onUnavailable: {action: reject}
|
||||
upstreams:
|
||||
provider-a:
|
||||
enabled: true
|
||||
exposure: [extract]
|
||||
provider: {billingMode: fetch, protocols: [http]}
|
||||
api: {url: https://provider-a.example/proxies, method: GET, auth: {type: none}, template: '{{.}}'}
|
||||
proxyAuth: {type: response}
|
||||
pool: {maxSize: 500}
|
||||
capacity: {maxConcurrencyPerProxy: 1}
|
||||
lifecycle: {ttl: 2m, allocationSafetyMargin: 15s}
|
||||
fetch: {requestInterval: 1s, timeout: 3s, maxAttempts: 3, maxInFlight: 1, maxTotal: 50000}
|
||||
check: {interval: 15s, jitter: 20, maxInFlight: 100, timeout: 2s, maxAttempts: 2, maxConsecutiveFailures: 3}
|
||||
42
examples/config/14-gateway-reserve.yaml
Normal file
42
examples/config/14-gateway-reserve.yaml
Normal file
@ -0,0 +1,42 @@
|
||||
# 共享池至少为 Gateway 保留 100 个符合提取条件的可用 Proxy。
|
||||
version: 1
|
||||
security: {requireProtectionOnPublicListen: true}
|
||||
gateway: {enabled: true, listen: '127.0.0.1:8080', auth: {mode: none}}
|
||||
distribution:
|
||||
enabled: true
|
||||
listen: 127.0.0.1:8081
|
||||
auth: {mode: none}
|
||||
clientIdentification: {mode: sourceIP}
|
||||
extraction:
|
||||
fulfillment: partial
|
||||
maxCountPerRequest: 100
|
||||
minRemainingTTL: 30s
|
||||
maxHealthCheckAge: 15s
|
||||
reserveForGateway: 100
|
||||
routing:
|
||||
- name: gateway
|
||||
enabled: true
|
||||
purpose: gateway
|
||||
match: {hostRegex: '.*'}
|
||||
upstreams: [provider-a]
|
||||
strategy: {type: leastConnections}
|
||||
onUnavailable: {action: reject}
|
||||
- name: extract
|
||||
enabled: true
|
||||
purpose: extract
|
||||
match: {hostRegex: '.*'}
|
||||
upstreams: [provider-a]
|
||||
strategy: {type: random}
|
||||
onUnavailable: {action: reject}
|
||||
upstreams:
|
||||
provider-a:
|
||||
enabled: true
|
||||
exposure: [gateway, extract]
|
||||
provider: {billingMode: subscription, protocols: [http]}
|
||||
api: {url: https://provider-a.example/proxies, method: GET, auth: {type: none}, template: '{{.}}'}
|
||||
proxyAuth: {type: response}
|
||||
pool: {maxSize: 2000}
|
||||
capacity: {maxConcurrencyPerProxy: 10}
|
||||
lifecycle: {ttl: 3m, allocationSafetyMargin: 20s}
|
||||
fetch: {requestInterval: 1s, timeout: 3s, maxAttempts: 3, maxInFlight: 1}
|
||||
check: {interval: 15s, jitter: 20, maxInFlight: 200, timeout: 2s, maxAttempts: 2, maxConsecutiveFailures: 3}
|
||||
34
examples/config/15-strict-ttl-health.yaml
Normal file
34
examples/config/15-strict-ttl-health.yaml
Normal file
@ -0,0 +1,34 @@
|
||||
# 只发放剩余 TTL 至少 60 秒且 5 秒内检查过的 Proxy。
|
||||
version: 1
|
||||
security: {requireProtectionOnPublicListen: true}
|
||||
distribution:
|
||||
enabled: true
|
||||
listen: 127.0.0.1:8081
|
||||
auth: {mode: none}
|
||||
clientIdentification: {mode: sourceIP}
|
||||
extraction:
|
||||
fulfillment: partial
|
||||
maxCountPerRequest: 20
|
||||
minRemainingTTL: 60s
|
||||
maxHealthCheckAge: 5s
|
||||
reserveForGateway: 0
|
||||
routing:
|
||||
- name: fresh-extract
|
||||
enabled: true
|
||||
purpose: extract
|
||||
match: {hostRegex: '.*'}
|
||||
upstreams: [provider-a]
|
||||
strategy: {type: random}
|
||||
onUnavailable: {action: reject}
|
||||
upstreams:
|
||||
provider-a:
|
||||
enabled: true
|
||||
exposure: [extract]
|
||||
provider: {billingMode: fetch, protocols: [http]}
|
||||
api: {url: https://provider-a.example/proxies, method: GET, auth: {type: none}, template: '{{.}}'}
|
||||
proxyAuth: {type: response}
|
||||
pool: {maxSize: 1000}
|
||||
capacity: {maxConcurrencyPerProxy: 1}
|
||||
lifecycle: {ttl: 5m, allocationSafetyMargin: 60s}
|
||||
fetch: {requestInterval: 1s, timeout: 3s, maxAttempts: 3, maxInFlight: 1, maxTotal: 100000}
|
||||
check: {interval: 5s, jitter: 20, maxInFlight: 500, timeout: 2s, maxAttempts: 2, maxConsecutiveFailures: 3}
|
||||
31
examples/config/16-provider-basic-auth.yaml
Normal file
31
examples/config/16-provider-basic-auth.yaml
Normal file
@ -0,0 +1,31 @@
|
||||
# Provider API 认证与对外 Client 认证相互独立。
|
||||
version: 1
|
||||
security: {requireProtectionOnPublicListen: true}
|
||||
gateway: {enabled: true, listen: '127.0.0.1:8080', auth: {mode: none}}
|
||||
routing:
|
||||
- name: gateway
|
||||
enabled: true
|
||||
purpose: gateway
|
||||
match: {hostRegex: '.*'}
|
||||
upstreams: [provider-a]
|
||||
strategy: {type: leastConnections}
|
||||
onUnavailable: {action: reject}
|
||||
upstreams:
|
||||
provider-a:
|
||||
enabled: true
|
||||
exposure: [gateway]
|
||||
provider: {billingMode: subscription, protocols: [http]}
|
||||
api:
|
||||
url: https://provider-a.example/proxies
|
||||
method: GET
|
||||
auth:
|
||||
type: basic
|
||||
username: "${PROVIDER_API_USER}"
|
||||
password: "${PROVIDER_API_PASSWORD}"
|
||||
template: '{{.}}'
|
||||
proxyAuth: {type: response}
|
||||
pool: {maxSize: 1000}
|
||||
capacity: {maxConcurrencyPerProxy: 10}
|
||||
lifecycle: {ttl: 2m, allocationSafetyMargin: 15s}
|
||||
fetch: {requestInterval: 1s, timeout: 3s, maxAttempts: 3, maxInFlight: 1}
|
||||
check: {interval: 30s, jitter: 20, maxInFlight: 100, timeout: 2s, maxAttempts: 2, maxConsecutiveFailures: 3}
|
||||
32
examples/config/17-provider-api-key.yaml
Normal file
32
examples/config/17-provider-api-key.yaml
Normal file
@ -0,0 +1,32 @@
|
||||
# Provider API Key 放在专用 Header;日志必须统一脱敏。
|
||||
version: 1
|
||||
security: {requireProtectionOnPublicListen: true}
|
||||
gateway: {enabled: true, listen: '127.0.0.1:8080', auth: {mode: none}}
|
||||
routing:
|
||||
- name: gateway
|
||||
enabled: true
|
||||
purpose: gateway
|
||||
match: {hostRegex: '.*'}
|
||||
upstreams: [provider-a]
|
||||
strategy: {type: leastConnections}
|
||||
onUnavailable: {action: reject}
|
||||
upstreams:
|
||||
provider-a:
|
||||
enabled: true
|
||||
exposure: [gateway]
|
||||
provider: {billingMode: fetch, protocols: [http]}
|
||||
api:
|
||||
url: https://provider-a.example/proxies
|
||||
method: GET
|
||||
auth:
|
||||
type: apiKey
|
||||
location: header
|
||||
name: X-Provider-Key
|
||||
value: "${PROVIDER_API_KEY}"
|
||||
template: '{{.}}'
|
||||
proxyAuth: {type: response}
|
||||
pool: {maxSize: 500}
|
||||
capacity: {maxConcurrencyPerProxy: 10}
|
||||
lifecycle: {ttl: 2m, allocationSafetyMargin: 15s}
|
||||
fetch: {requestInterval: 1s, timeout: 3s, maxAttempts: 3, maxInFlight: 1, maxTotal: 10000}
|
||||
check: {interval: 30s, jitter: 20, maxInFlight: 100, timeout: 2s, maxAttempts: 2, maxConsecutiveFailures: 3}
|
||||
40
examples/config/18-provider-post-json.yaml
Normal file
40
examples/config/18-provider-post-json.yaml
Normal file
@ -0,0 +1,40 @@
|
||||
# Provider 使用 JSON POST;供应商重试由独立 Fetch 策略控制。
|
||||
version: 1
|
||||
security: {requireProtectionOnPublicListen: true}
|
||||
gateway: {enabled: true, listen: '127.0.0.1:8080', auth: {mode: none}}
|
||||
routing:
|
||||
- name: gateway
|
||||
enabled: true
|
||||
purpose: gateway
|
||||
match: {hostRegex: '.*'}
|
||||
upstreams: [provider-a]
|
||||
strategy: {type: leastConnections}
|
||||
onUnavailable: {action: reject}
|
||||
upstreams:
|
||||
provider-a:
|
||||
enabled: true
|
||||
exposure: [gateway]
|
||||
provider: {billingMode: fetch, protocols: [http]}
|
||||
api:
|
||||
url: https://provider-a.example/v1/orders
|
||||
method: POST
|
||||
auth: {type: apiKey, location: header, name: X-Provider-Key, value: "${PROVIDER_API_KEY}"}
|
||||
headers: {Content-Type: application/json}
|
||||
body:
|
||||
type: json
|
||||
value: {count: '100', protocol: http}
|
||||
template: '{{.}}'
|
||||
proxyAuth: {type: response}
|
||||
pool: {maxSize: 1000}
|
||||
capacity: {maxConcurrencyPerProxy: 10}
|
||||
lifecycle: {ttl: 3m, allocationSafetyMargin: 20s}
|
||||
fetch:
|
||||
requestInterval: 2s
|
||||
timeout: 5s
|
||||
maxAttempts: 3
|
||||
maxInFlight: 1
|
||||
maxTotal: 100000
|
||||
maxResponseBytes: 1048576
|
||||
templateTimeout: 100ms
|
||||
retry: {initial: 1s, max: 30s, jitter: 20}
|
||||
check: {interval: 30s, jitter: 20, maxInFlight: 100, timeout: 2s, maxAttempts: 2, maxConsecutiveFailures: 3}
|
||||
27
examples/config/19-socks5-upstream.yaml
Normal file
27
examples/config/19-socks5-upstream.yaml
Normal file
@ -0,0 +1,27 @@
|
||||
# SOCKS5 上游保留在统一 Proxy 模型和策略契约内。
|
||||
version: 1
|
||||
security: {requireProtectionOnPublicListen: true}
|
||||
gateway: {enabled: true, listen: '127.0.0.1:8080', auth: {mode: none}}
|
||||
routing:
|
||||
- name: socks-upstream
|
||||
enabled: true
|
||||
purpose: gateway
|
||||
match: {hostRegex: '.*'}
|
||||
upstreams: [provider-socks]
|
||||
strategy: {type: leastConnections}
|
||||
onUnavailable: {action: reject}
|
||||
upstreams:
|
||||
provider-socks:
|
||||
enabled: true
|
||||
exposure: [gateway]
|
||||
provider: {billingMode: subscription, protocols: [socks5]}
|
||||
api: {url: https://provider-socks.example/proxies, method: GET, auth: {type: none}, template: '{{.}}'}
|
||||
proxyAuth:
|
||||
type: static
|
||||
username: "${SOCKS_USER}"
|
||||
password: "${SOCKS_PASSWORD}"
|
||||
pool: {maxSize: 1000}
|
||||
capacity: {maxConcurrencyPerProxy: 20}
|
||||
lifecycle: {ttl: 10m, allocationSafetyMargin: 60s}
|
||||
fetch: {requestInterval: 5s, timeout: 5s, maxAttempts: 3, maxInFlight: 1}
|
||||
check: {interval: 30s, jitter: 20, maxInFlight: 100, timeout: 3s, maxAttempts: 2, maxConsecutiveFailures: 3}
|
||||
35
examples/config/20-fetch-billing-quota.yaml
Normal file
35
examples/config/20-fetch-billing-quota.yaml
Normal file
@ -0,0 +1,35 @@
|
||||
# pool.maxSize 限当前未提取库存;fetch.maxTotal 限计费周期累计获取数。
|
||||
version: 1
|
||||
security: {requireProtectionOnPublicListen: true}
|
||||
distribution:
|
||||
enabled: true
|
||||
listen: 127.0.0.1:8081
|
||||
auth: {mode: none}
|
||||
clientIdentification: {mode: sourceIP}
|
||||
extraction: {fulfillment: partial, maxCountPerRequest: 20, minRemainingTTL: 30s, maxHealthCheckAge: 15s, reserveForGateway: 0}
|
||||
routing:
|
||||
- name: billed-extract
|
||||
enabled: true
|
||||
purpose: extract
|
||||
match: {hostRegex: '.*'}
|
||||
upstreams: [provider-metered]
|
||||
strategy: {type: random}
|
||||
onUnavailable: {action: reject}
|
||||
upstreams:
|
||||
provider-metered:
|
||||
enabled: true
|
||||
exposure: [extract]
|
||||
provider: {billingMode: fetch, protocols: [http]}
|
||||
api: {url: https://provider-metered.example/proxies, method: GET, auth: {type: none}, template: '{{.}}'}
|
||||
proxyAuth: {type: response}
|
||||
pool: {maxSize: 100}
|
||||
capacity: {maxConcurrencyPerProxy: 1}
|
||||
lifecycle: {ttl: 2m, allocationSafetyMargin: 15s}
|
||||
fetch:
|
||||
requestInterval: 1s
|
||||
timeout: 3s
|
||||
maxAttempts: 3
|
||||
maxInFlight: 1
|
||||
maxTotal: 1000
|
||||
retry: {initial: 500ms, max: 30s, jitter: 20}
|
||||
check: {interval: 15s, jitter: 20, maxInFlight: 100, timeout: 2s, maxAttempts: 2, maxConsecutiveFailures: 3}
|
||||
64
examples/config/examples_test.go
Normal file
64
examples/config/examples_test.go
Normal file
@ -0,0 +1,64 @@
|
||||
package configexamples
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
projectconfig "github.com/proxy-pool/proxy-pool/internal/config"
|
||||
)
|
||||
|
||||
func TestAllExamplesLoadStrictly(t *testing.T) {
|
||||
entries, err := filepath.Glob("*.yaml")
|
||||
if err != nil {
|
||||
t.Fatalf("glob examples: %v", err)
|
||||
}
|
||||
if len(entries) < 20 {
|
||||
t.Fatalf("configuration examples = %d, want at least 20", len(entries))
|
||||
}
|
||||
for _, path := range entries {
|
||||
path := path
|
||||
t.Run(strings.TrimSuffix(filepath.Base(path), filepath.Ext(path)), func(t *testing.T) {
|
||||
file, err := os.Open(path)
|
||||
if err != nil {
|
||||
t.Fatalf("open %s: %v", path, err)
|
||||
}
|
||||
defer file.Close()
|
||||
cfg, err := projectconfig.Load(file)
|
||||
if err != nil {
|
||||
t.Fatalf("load %s: %v", path, err)
|
||||
}
|
||||
assertExplicitProviderAuth(t, cfg)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestMainConfigurationLoadsStrictly(t *testing.T) {
|
||||
path := filepath.Join("..", "..", "configs", "proxy-pool.yaml")
|
||||
file, err := os.Open(path)
|
||||
if err != nil {
|
||||
t.Fatalf("open %s: %v", path, err)
|
||||
}
|
||||
defer file.Close()
|
||||
cfg, err := projectconfig.Load(file)
|
||||
if err != nil {
|
||||
t.Fatalf("load %s: %v", path, err)
|
||||
}
|
||||
assertExplicitProviderAuth(t, cfg)
|
||||
}
|
||||
|
||||
func assertExplicitProviderAuth(t *testing.T, cfg *projectconfig.Config) {
|
||||
t.Helper()
|
||||
for name, upstream := range cfg.Upstreams {
|
||||
if !upstream.Enabled {
|
||||
continue
|
||||
}
|
||||
if upstream.API.Auth.Type == "" {
|
||||
t.Errorf("upstream %s must explicitly set api.auth.type", name)
|
||||
}
|
||||
if upstream.ProxyAuth.Type == "" {
|
||||
t.Errorf("upstream %s must explicitly set proxyAuth.type", name)
|
||||
}
|
||||
}
|
||||
}
|
||||
5
go.mod
Normal file
5
go.mod
Normal file
@ -0,0 +1,5 @@
|
||||
module github.com/proxy-pool/proxy-pool
|
||||
|
||||
go 1.26.0
|
||||
|
||||
require go.yaml.in/yaml/v4 v4.0.0-rc.3
|
||||
2
go.sum
Normal file
2
go.sum
Normal file
@ -0,0 +1,2 @@
|
||||
go.yaml.in/yaml/v4 v4.0.0-rc.3 h1:3h1fjsh1CTAPjW7q/EMe+C8shx5d8ctzZTrLcs/j8Go=
|
||||
go.yaml.in/yaml/v4 v4.0.0-rc.3/go.mod h1:aZqd9kCMsGL7AuUv/m/PvWLdg5sjJsZ4oHDEnfPPfY0=
|
||||
244
internal/config/config.go
Normal file
244
internal/config/config.go
Normal file
@ -0,0 +1,244 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"time"
|
||||
)
|
||||
|
||||
type Duration time.Duration
|
||||
|
||||
func (d *Duration) UnmarshalText(text []byte) error {
|
||||
value, err := time.ParseDuration(string(text))
|
||||
if err != nil {
|
||||
return fmt.Errorf("parse duration %q: %w", text, err)
|
||||
}
|
||||
*d = Duration(value)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (d Duration) Value() time.Duration { return time.Duration(d) }
|
||||
|
||||
type Config struct {
|
||||
Version int `yaml:"version"`
|
||||
Defaults Defaults `yaml:"defaults"`
|
||||
Security Security `yaml:"security"`
|
||||
Gateway Listener `yaml:"gateway"`
|
||||
Distribution Distribution `yaml:"distribution"`
|
||||
Admin Listener `yaml:"admin"`
|
||||
Metrics Metrics `yaml:"metrics"`
|
||||
Storage Storage `yaml:"storage"`
|
||||
Routing []Routing `yaml:"routing"`
|
||||
Upstreams map[string]Upstream `yaml:"upstreams"`
|
||||
}
|
||||
|
||||
type Defaults struct {
|
||||
Fetch Fetch `yaml:"fetch"`
|
||||
Check Check `yaml:"check"`
|
||||
}
|
||||
|
||||
type Security struct {
|
||||
RequireProtectionOnPublicListen bool `yaml:"requireProtectionOnPublicListen"`
|
||||
}
|
||||
|
||||
type Listener struct {
|
||||
Enabled bool `yaml:"enabled"`
|
||||
Listen string `yaml:"listen"`
|
||||
Access Access `yaml:"access"`
|
||||
Auth Auth `yaml:"auth"`
|
||||
Limits Limits `yaml:"limits"`
|
||||
Retry Retry `yaml:"retry"`
|
||||
DestinationPolicy DestinationPolicy `yaml:"destinationPolicy"`
|
||||
}
|
||||
|
||||
type Distribution struct {
|
||||
Listener `yaml:",inline"`
|
||||
ClientIdentification ClientIdentification `yaml:"clientIdentification"`
|
||||
Extraction Extraction `yaml:"extraction"`
|
||||
}
|
||||
|
||||
type Access struct {
|
||||
AllowCIDRs []string `yaml:"allowCIDRs"`
|
||||
TrustedProxies []string `yaml:"trustedProxies"`
|
||||
}
|
||||
|
||||
type Auth struct {
|
||||
Mode string `yaml:"mode"`
|
||||
Username string `yaml:"username"`
|
||||
Password string `yaml:"password"`
|
||||
Token string `yaml:"token"`
|
||||
Header string `yaml:"header"`
|
||||
CIDRs []string `yaml:"cidrs"`
|
||||
Methods []AuthMethod `yaml:"methods"`
|
||||
}
|
||||
|
||||
type AuthMethod struct {
|
||||
Mode string `yaml:"mode"`
|
||||
Username string `yaml:"username"`
|
||||
Password string `yaml:"password"`
|
||||
Header string `yaml:"header"`
|
||||
Value string `yaml:"value"`
|
||||
CIDRs []string `yaml:"cidrs"`
|
||||
}
|
||||
|
||||
type Limits struct {
|
||||
MaxConcurrentConnections int `yaml:"maxConcurrentConnections"`
|
||||
RequestsPerMinute int `yaml:"requestsPerMinute"`
|
||||
RequestsPerMinutePerClient int `yaml:"requestsPerMinutePerClient"`
|
||||
}
|
||||
|
||||
type Retry struct {
|
||||
MaxAttempts int `yaml:"maxAttempts"`
|
||||
RetryMethods []string `yaml:"retryMethods"`
|
||||
}
|
||||
|
||||
type DestinationPolicy struct {
|
||||
DenyPrivateNetworks bool `yaml:"denyPrivateNetworks"`
|
||||
DenyLoopback bool `yaml:"denyLoopback"`
|
||||
DenyLinkLocal bool `yaml:"denyLinkLocal"`
|
||||
DenyCIDRs []string `yaml:"denyCIDRs"`
|
||||
}
|
||||
|
||||
type ClientIdentification struct {
|
||||
Mode string `yaml:"mode"`
|
||||
}
|
||||
|
||||
type Extraction struct {
|
||||
Fulfillment string `yaml:"fulfillment"`
|
||||
MaxCountPerRequest int `yaml:"maxCountPerRequest"`
|
||||
MinRemainingTTL Duration `yaml:"minRemainingTTL"`
|
||||
MaxHealthCheckAge Duration `yaml:"maxHealthCheckAge"`
|
||||
ReserveForGateway int `yaml:"reserveForGateway"`
|
||||
}
|
||||
|
||||
type Metrics struct {
|
||||
Enabled bool `yaml:"enabled"`
|
||||
Listen string `yaml:"listen"`
|
||||
}
|
||||
|
||||
type Storage struct {
|
||||
PostgresURL string `yaml:"postgresURL"`
|
||||
RedisURL string `yaml:"redisURL"`
|
||||
}
|
||||
|
||||
type Routing struct {
|
||||
Name string `yaml:"name"`
|
||||
Enabled bool `yaml:"enabled"`
|
||||
Purpose string `yaml:"purpose"`
|
||||
Match RoutingMatch `yaml:"match"`
|
||||
Upstreams []string `yaml:"upstreams"`
|
||||
Strategy Strategy `yaml:"strategy"`
|
||||
OnUnavailable OnUnavailable `yaml:"onUnavailable"`
|
||||
}
|
||||
|
||||
type RoutingMatch struct {
|
||||
HostRegex string `yaml:"hostRegex"`
|
||||
Methods []string `yaml:"methods"`
|
||||
PathRegex string `yaml:"pathRegex"`
|
||||
Headers map[string]string `yaml:"headers"`
|
||||
}
|
||||
|
||||
type Strategy struct {
|
||||
Type string `yaml:"type"`
|
||||
SwitchAfterEmptyFetch int `yaml:"switchAfterEmptyFetch"`
|
||||
EndBehavior string `yaml:"endBehavior"`
|
||||
Weights map[string]int `yaml:"weights"`
|
||||
}
|
||||
|
||||
type OnUnavailable struct {
|
||||
Action string `yaml:"action"`
|
||||
WaitTimeout Duration `yaml:"waitTimeout"`
|
||||
}
|
||||
|
||||
type Upstream struct {
|
||||
Enabled bool `yaml:"enabled"`
|
||||
Exposure []string `yaml:"exposure"`
|
||||
Provider Provider `yaml:"provider"`
|
||||
API ProviderAPI `yaml:"api"`
|
||||
ProxyAuth ProxyAuth `yaml:"proxyAuth"`
|
||||
Pool Pool `yaml:"pool"`
|
||||
Capacity Capacity `yaml:"capacity"`
|
||||
Lifecycle Lifecycle `yaml:"lifecycle"`
|
||||
Fetch Fetch `yaml:"fetch"`
|
||||
Check Check `yaml:"check"`
|
||||
}
|
||||
|
||||
type Provider struct {
|
||||
BillingMode string `yaml:"billingMode"`
|
||||
Protocols []string `yaml:"protocols"`
|
||||
}
|
||||
|
||||
type ProviderAPI struct {
|
||||
URL string `yaml:"url"`
|
||||
Method string `yaml:"method"`
|
||||
Auth ProviderAuth `yaml:"auth"`
|
||||
Headers map[string]string `yaml:"headers"`
|
||||
Query map[string]string `yaml:"query"`
|
||||
Body APIBody `yaml:"body"`
|
||||
Template string `yaml:"template"`
|
||||
}
|
||||
|
||||
type ProviderAuth struct {
|
||||
Type string `yaml:"type"`
|
||||
Username string `yaml:"username"`
|
||||
Password string `yaml:"password"`
|
||||
PasswordFile string `yaml:"passwordFile"`
|
||||
Token string `yaml:"token"`
|
||||
TokenFile string `yaml:"tokenFile"`
|
||||
Location string `yaml:"location"`
|
||||
Name string `yaml:"name"`
|
||||
Value string `yaml:"value"`
|
||||
ValueFile string `yaml:"valueFile"`
|
||||
}
|
||||
|
||||
type APIBody struct {
|
||||
Type string `yaml:"type"`
|
||||
Value map[string]string `yaml:"value"`
|
||||
}
|
||||
|
||||
type ProxyAuth struct {
|
||||
Type string `yaml:"type"`
|
||||
Username string `yaml:"username"`
|
||||
Password string `yaml:"password"`
|
||||
PasswordFile string `yaml:"passwordFile"`
|
||||
}
|
||||
|
||||
type Pool struct {
|
||||
MaxSize int `yaml:"maxSize"`
|
||||
ShrinkDelay Duration `yaml:"shrinkDelay"`
|
||||
}
|
||||
|
||||
type Capacity struct {
|
||||
MaxConcurrencyPerProxy int `yaml:"maxConcurrencyPerProxy"`
|
||||
}
|
||||
|
||||
type Lifecycle struct {
|
||||
TTL Duration `yaml:"ttl"`
|
||||
AllocationSafetyMargin Duration `yaml:"allocationSafetyMargin"`
|
||||
}
|
||||
|
||||
type Fetch struct {
|
||||
RequestInterval Duration `yaml:"requestInterval"`
|
||||
Timeout Duration `yaml:"timeout"`
|
||||
MaxAttempts int `yaml:"maxAttempts"`
|
||||
MaxInFlight int `yaml:"maxInFlight"`
|
||||
MaxTotal int `yaml:"maxTotal"`
|
||||
MaxResponseBytes int64 `yaml:"maxResponseBytes"`
|
||||
TemplateTimeout Duration `yaml:"templateTimeout"`
|
||||
Retry Backoff `yaml:"retry"`
|
||||
}
|
||||
|
||||
type Backoff struct {
|
||||
Initial Duration `yaml:"initial"`
|
||||
Max Duration `yaml:"max"`
|
||||
Jitter int `yaml:"jitter"`
|
||||
}
|
||||
|
||||
type Check struct {
|
||||
Interval Duration `yaml:"interval"`
|
||||
Jitter int `yaml:"jitter"`
|
||||
MaxInFlight int `yaml:"maxInFlight"`
|
||||
Timeout Duration `yaml:"timeout"`
|
||||
MaxAttempts int `yaml:"maxAttempts"`
|
||||
MaxConsecutiveFailures int `yaml:"maxConsecutiveFailures"`
|
||||
URLs []string `yaml:"urls"`
|
||||
}
|
||||
145
internal/config/config_test.go
Normal file
145
internal/config/config_test.go
Normal file
@ -0,0 +1,145 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
const validConfig = `
|
||||
version: 1
|
||||
security:
|
||||
requireProtectionOnPublicListen: true
|
||||
gateway:
|
||||
enabled: true
|
||||
listen: 127.0.0.1:8080
|
||||
auth:
|
||||
mode: none
|
||||
distribution:
|
||||
enabled: true
|
||||
listen: 127.0.0.1:8081
|
||||
auth:
|
||||
mode: none
|
||||
extraction:
|
||||
fulfillment: partial
|
||||
maxCountPerRequest: 20
|
||||
minRemainingTTL: 30s
|
||||
maxHealthCheckAge: 15s
|
||||
reserveForGateway: 5
|
||||
routing:
|
||||
- name: extract
|
||||
enabled: true
|
||||
purpose: extract
|
||||
upstreams: [provider-a]
|
||||
strategy:
|
||||
type: sequential
|
||||
switchAfterEmptyFetch: 5
|
||||
onUnavailable:
|
||||
action: reject
|
||||
upstreams:
|
||||
provider-a:
|
||||
enabled: true
|
||||
exposure: [gateway, extract]
|
||||
provider:
|
||||
billingMode: fetch
|
||||
protocols: [http]
|
||||
api:
|
||||
url: https://provider.example/proxies
|
||||
method: GET
|
||||
template: '{{.}}'
|
||||
auth:
|
||||
type: none
|
||||
proxyAuth:
|
||||
type: response
|
||||
pool:
|
||||
maxSize: 100
|
||||
capacity:
|
||||
maxConcurrencyPerProxy: 10
|
||||
lifecycle:
|
||||
ttl: 120s
|
||||
allocationSafetyMargin: 10s
|
||||
fetch:
|
||||
requestInterval: 1s
|
||||
timeout: 3s
|
||||
maxAttempts: 5
|
||||
maxInFlight: 1
|
||||
maxTotal: 1000
|
||||
check:
|
||||
interval: 30s
|
||||
jitter: 20
|
||||
maxInFlight: 100
|
||||
timeout: 2s
|
||||
maxAttempts: 2
|
||||
maxConsecutiveFailures: 3
|
||||
urls: [http://connect.rom.miui.com/generate_204]
|
||||
`
|
||||
|
||||
func TestLoadStrictValidConfiguration(t *testing.T) {
|
||||
cfg, err := Load(strings.NewReader(validConfig))
|
||||
if err != nil {
|
||||
t.Fatalf("Load(): %v", err)
|
||||
}
|
||||
if cfg.Version != 1 || cfg.Upstreams["provider-a"].Pool.MaxSize != 100 {
|
||||
t.Fatalf("unexpected config: %+v", cfg)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadRejectsUnknownFields(t *testing.T) {
|
||||
_, err := Load(strings.NewReader(validConfig + "unknownField: true\n"))
|
||||
if err == nil || !strings.Contains(err.Error(), "unknownField") {
|
||||
t.Fatalf("Load() error = %v, want unknown field error", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateRejectsUnprotectedPublicListener(t *testing.T) {
|
||||
cfg, err := Load(strings.NewReader(strings.Replace(validConfig,
|
||||
"listen: 127.0.0.1:8080", "listen: 0.0.0.0:8080", 1)))
|
||||
if err == nil || !strings.Contains(err.Error(), "gateway") || !strings.Contains(err.Error(), "public") {
|
||||
t.Fatalf("Load() error = %v, want unprotected public listener error", err)
|
||||
}
|
||||
if cfg != nil {
|
||||
t.Fatal("invalid config must not be returned")
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateRejectsMissingUpstreamReference(t *testing.T) {
|
||||
broken := strings.Replace(validConfig, "upstreams: [provider-a]", "upstreams: [missing]", 1)
|
||||
_, err := Load(strings.NewReader(broken))
|
||||
if err == nil || !strings.Contains(err.Error(), "missing") {
|
||||
t.Fatalf("Load() error = %v, want missing upstream error", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateSeparatesPoolAndFetchLimits(t *testing.T) {
|
||||
broken := strings.Replace(validConfig, "maxTotal: 1000", "maxTotal: 50", 1)
|
||||
_, err := Load(strings.NewReader(broken))
|
||||
if err == nil || !strings.Contains(err.Error(), "maxTotal") {
|
||||
t.Fatalf("Load() error = %v, want maxTotal validation error", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestShippedConfigurationsAreValid(t *testing.T) {
|
||||
paths, err := filepath.Glob(filepath.Join("..", "..", "examples", "config", "*.yaml"))
|
||||
if err != nil {
|
||||
t.Fatalf("Glob(): %v", err)
|
||||
}
|
||||
paths = append(paths, filepath.Join("..", "..", "configs", "proxy-pool.yaml"))
|
||||
if len(paths) != 21 {
|
||||
t.Fatalf("configuration count = %d, want 21", len(paths))
|
||||
}
|
||||
|
||||
for _, path := range paths {
|
||||
path := path
|
||||
t.Run(filepath.Base(path), func(t *testing.T) {
|
||||
file, err := os.Open(path)
|
||||
if err != nil {
|
||||
t.Fatalf("Open(): %v", err)
|
||||
}
|
||||
defer file.Close()
|
||||
if _, err := Load(file); err != nil {
|
||||
t.Fatalf("Load(): %v", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
22
internal/config/load.go
Normal file
22
internal/config/load.go
Normal file
@ -0,0 +1,22 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
|
||||
"go.yaml.in/yaml/v4"
|
||||
)
|
||||
|
||||
func Load(reader io.Reader) (*Config, error) {
|
||||
decoder := yaml.NewDecoder(reader)
|
||||
decoder.KnownFields(true)
|
||||
|
||||
var cfg Config
|
||||
if err := decoder.Decode(&cfg); err != nil {
|
||||
return nil, fmt.Errorf("decode configuration: %w", err)
|
||||
}
|
||||
if err := Validate(&cfg); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &cfg, nil
|
||||
}
|
||||
193
internal/config/validate.go
Normal file
193
internal/config/validate.go
Normal file
@ -0,0 +1,193 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net"
|
||||
"net/url"
|
||||
"regexp"
|
||||
"strings"
|
||||
)
|
||||
|
||||
func Validate(cfg *Config) error {
|
||||
if cfg == nil {
|
||||
return fmt.Errorf("validate configuration: nil config")
|
||||
}
|
||||
if cfg.Version != 1 {
|
||||
return fmt.Errorf("validate configuration: version must be 1")
|
||||
}
|
||||
listeners := []struct {
|
||||
name string
|
||||
item Listener
|
||||
}{
|
||||
{name: "gateway", item: cfg.Gateway},
|
||||
{name: "distribution", item: cfg.Distribution.Listener},
|
||||
{name: "admin", item: cfg.Admin},
|
||||
}
|
||||
for _, listener := range listeners {
|
||||
if err := validateListener(listener.name, listener.item, cfg.Security); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
for name, upstream := range cfg.Upstreams {
|
||||
if err := validateUpstream(name, upstream); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
seen := make(map[string]struct{}, len(cfg.Routing))
|
||||
for index, route := range cfg.Routing {
|
||||
if route.Name == "" {
|
||||
return fmt.Errorf("validate routing[%d]: name is required", index)
|
||||
}
|
||||
if _, ok := seen[route.Name]; ok {
|
||||
return fmt.Errorf("validate routing %q: duplicate name", route.Name)
|
||||
}
|
||||
seen[route.Name] = struct{}{}
|
||||
if route.Match.HostRegex != "" {
|
||||
if _, err := regexp.Compile(route.Match.HostRegex); err != nil {
|
||||
return fmt.Errorf("validate routing %q hostRegex: %w", route.Name, err)
|
||||
}
|
||||
}
|
||||
if route.Match.PathRegex != "" {
|
||||
if _, err := regexp.Compile(route.Match.PathRegex); err != nil {
|
||||
return fmt.Errorf("validate routing %q pathRegex: %w", route.Name, err)
|
||||
}
|
||||
}
|
||||
for _, upstream := range route.Upstreams {
|
||||
if _, ok := cfg.Upstreams[upstream]; !ok {
|
||||
return fmt.Errorf("validate routing %q: upstream %q does not exist", route.Name, upstream)
|
||||
}
|
||||
}
|
||||
if route.Strategy.Type == "sequential" && route.Strategy.SwitchAfterEmptyFetch <= 0 {
|
||||
return fmt.Errorf("validate routing %q: switchAfterEmptyFetch must be greater than zero", route.Name)
|
||||
}
|
||||
if route.OnUnavailable.Action == "" {
|
||||
return fmt.Errorf("validate routing %q: onUnavailable.action is required", route.Name)
|
||||
}
|
||||
}
|
||||
if cfg.Distribution.Enabled {
|
||||
if cfg.Distribution.Extraction.MaxCountPerRequest <= 0 {
|
||||
return fmt.Errorf("validate distribution: maxCountPerRequest must be greater than zero")
|
||||
}
|
||||
if cfg.Distribution.Extraction.Fulfillment != "partial" && cfg.Distribution.Extraction.Fulfillment != "allOrNothing" {
|
||||
return fmt.Errorf("validate distribution: fulfillment must be partial or allOrNothing")
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func validateListener(name string, listener Listener, security Security) error {
|
||||
if !listener.Enabled {
|
||||
return nil
|
||||
}
|
||||
if listener.Listen == "" {
|
||||
return fmt.Errorf("validate %s: listen is required", name)
|
||||
}
|
||||
host, _, err := net.SplitHostPort(listener.Listen)
|
||||
if err != nil {
|
||||
return fmt.Errorf("validate %s listen: %w", name, err)
|
||||
}
|
||||
if security.RequireProtectionOnPublicListen && isPublicHost(host) && listener.Auth.Mode == "none" && len(listener.Access.AllowCIDRs) == 0 {
|
||||
return fmt.Errorf("validate %s: unprotected public listener is forbidden", name)
|
||||
}
|
||||
for _, cidr := range listener.Access.AllowCIDRs {
|
||||
if _, _, err := net.ParseCIDR(cidr); err != nil {
|
||||
return fmt.Errorf("validate %s allowCIDRs %q: %w", name, cidr, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func validateUpstream(name string, upstream Upstream) error {
|
||||
if !upstream.Enabled {
|
||||
return nil
|
||||
}
|
||||
if upstream.Pool.MaxSize <= 0 {
|
||||
return fmt.Errorf("validate upstream %q: pool.maxSize must be greater than zero", name)
|
||||
}
|
||||
if upstream.Fetch.MaxTotal > 0 && upstream.Fetch.MaxTotal < upstream.Pool.MaxSize {
|
||||
return fmt.Errorf("validate upstream %q: fetch.maxTotal cannot be lower than pool.maxSize", name)
|
||||
}
|
||||
if upstream.Capacity.MaxConcurrencyPerProxy <= 0 {
|
||||
return fmt.Errorf("validate upstream %q: maxConcurrencyPerProxy must be greater than zero", name)
|
||||
}
|
||||
if upstream.Lifecycle.TTL > 0 && upstream.Lifecycle.AllocationSafetyMargin >= upstream.Lifecycle.TTL {
|
||||
return fmt.Errorf("validate upstream %q: allocationSafetyMargin must be lower than ttl", name)
|
||||
}
|
||||
if upstream.Fetch.RequestInterval < 0 || upstream.Fetch.MaxInFlight <= 0 || upstream.Fetch.MaxAttempts <= 0 {
|
||||
return fmt.Errorf("validate upstream %q: fetch limits must be positive", name)
|
||||
}
|
||||
if upstream.API.URL != "" {
|
||||
parsed, err := url.Parse(upstream.API.URL)
|
||||
if err != nil || parsed.Scheme == "" || parsed.Host == "" {
|
||||
return fmt.Errorf("validate upstream %q: api.url is invalid", name)
|
||||
}
|
||||
}
|
||||
if err := validateProviderAuth(name, upstream.API.Auth); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := validateProxyAuth(name, upstream.ProxyAuth); err != nil {
|
||||
return err
|
||||
}
|
||||
if len(upstream.Exposure) == 0 {
|
||||
return fmt.Errorf("validate upstream %q: exposure is required", name)
|
||||
}
|
||||
for _, exposure := range upstream.Exposure {
|
||||
if exposure != "gateway" && exposure != "extract" {
|
||||
return fmt.Errorf("validate upstream %q: unsupported exposure %q", name, exposure)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func validateProviderAuth(upstream string, auth ProviderAuth) error {
|
||||
switch auth.Type {
|
||||
case "", "none":
|
||||
return nil
|
||||
case "basic":
|
||||
if auth.Username == "" || (auth.Password == "" && auth.PasswordFile == "") {
|
||||
return fmt.Errorf("validate upstream %q: basic api.auth requires username and password", upstream)
|
||||
}
|
||||
case "bearer":
|
||||
if auth.Token == "" && auth.TokenFile == "" {
|
||||
return fmt.Errorf("validate upstream %q: bearer api.auth requires token", upstream)
|
||||
}
|
||||
case "apiKey":
|
||||
if auth.Location != "header" && auth.Location != "query" {
|
||||
return fmt.Errorf("validate upstream %q: apiKey location must be header or query", upstream)
|
||||
}
|
||||
if auth.Name == "" || (auth.Value == "" && auth.ValueFile == "") {
|
||||
return fmt.Errorf("validate upstream %q: apiKey requires name and value", upstream)
|
||||
}
|
||||
default:
|
||||
return fmt.Errorf("validate upstream %q: unsupported api.auth type %q", upstream, auth.Type)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func validateProxyAuth(upstream string, auth ProxyAuth) error {
|
||||
switch auth.Type {
|
||||
case "response", "ipWhitelist":
|
||||
return nil
|
||||
case "static":
|
||||
if auth.Username == "" || (auth.Password == "" && auth.PasswordFile == "") {
|
||||
return fmt.Errorf("validate upstream %q: static proxyAuth requires username and password", upstream)
|
||||
}
|
||||
case "":
|
||||
return fmt.Errorf("validate upstream %q: proxyAuth.type is required", upstream)
|
||||
default:
|
||||
return fmt.Errorf("validate upstream %q: unsupported proxyAuth type %q", upstream, auth.Type)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func isPublicHost(host string) bool {
|
||||
host = strings.Trim(host, "[]")
|
||||
if host == "localhost" {
|
||||
return false
|
||||
}
|
||||
ip := net.ParseIP(host)
|
||||
if ip == nil {
|
||||
return true
|
||||
}
|
||||
return !ip.IsLoopback()
|
||||
}
|
||||
149
internal/domain/extraction/extraction.go
Normal file
149
internal/domain/extraction/extraction.go
Normal file
@ -0,0 +1,149 @@
|
||||
package extraction
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"sort"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
type Fulfillment string
|
||||
|
||||
const (
|
||||
Partial Fulfillment = "partial"
|
||||
AllOrNothing Fulfillment = "allOrNothing"
|
||||
)
|
||||
|
||||
type State string
|
||||
|
||||
const (
|
||||
Available State = "AVAILABLE"
|
||||
Extracted State = "EXTRACTED"
|
||||
)
|
||||
|
||||
var ErrInsufficientProxies = errors.New("insufficient proxies")
|
||||
|
||||
type Candidate struct {
|
||||
ID string
|
||||
Protocol string
|
||||
Region string
|
||||
Carrier string
|
||||
Upstream string
|
||||
URL string
|
||||
State State
|
||||
ExpiresAt time.Time
|
||||
LastCheckedAt time.Time
|
||||
}
|
||||
|
||||
type Command struct {
|
||||
Requested int
|
||||
Fulfillment Fulfillment
|
||||
Now time.Time
|
||||
MinRemainingTTL time.Duration
|
||||
MaxHealthCheckAge time.Duration
|
||||
ReserveForGateway int
|
||||
Protocols []string
|
||||
Regions []string
|
||||
Carriers []string
|
||||
Upstreams []string
|
||||
}
|
||||
|
||||
type Record struct {
|
||||
ProxyID string
|
||||
ClientID string
|
||||
SourceIP string
|
||||
RequestID string
|
||||
Upstream string
|
||||
ExtractedAt time.Time
|
||||
ExpiresAt time.Time
|
||||
}
|
||||
|
||||
type Result struct {
|
||||
Requested int
|
||||
Returned int
|
||||
Items []Candidate
|
||||
}
|
||||
|
||||
type Store interface {
|
||||
Extract(context.Context, Command) (Result, error)
|
||||
}
|
||||
|
||||
type MemoryStore struct {
|
||||
mu sync.Mutex
|
||||
candidates map[string]Candidate
|
||||
}
|
||||
|
||||
func NewMemoryStore(candidates []Candidate) *MemoryStore {
|
||||
items := make(map[string]Candidate, len(candidates))
|
||||
for _, candidate := range candidates {
|
||||
items[candidate.ID] = candidate
|
||||
}
|
||||
return &MemoryStore{candidates: items}
|
||||
}
|
||||
|
||||
func (s *MemoryStore) Extract(_ context.Context, command Command) (Result, error) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
|
||||
result := Result{Requested: command.Requested}
|
||||
if command.Requested <= 0 {
|
||||
return result, nil
|
||||
}
|
||||
eligible := make([]Candidate, 0, len(s.candidates))
|
||||
for _, candidate := range s.candidates {
|
||||
if eligibleForExtraction(candidate, command) {
|
||||
eligible = append(eligible, candidate)
|
||||
}
|
||||
}
|
||||
sort.Slice(eligible, func(i, j int) bool {
|
||||
return eligible[i].ExpiresAt.After(eligible[j].ExpiresAt)
|
||||
})
|
||||
available := len(eligible) - command.ReserveForGateway
|
||||
if available < 0 {
|
||||
available = 0
|
||||
}
|
||||
if command.Fulfillment == AllOrNothing && available < command.Requested {
|
||||
return result, ErrInsufficientProxies
|
||||
}
|
||||
count := command.Requested
|
||||
if count > available {
|
||||
count = available
|
||||
}
|
||||
for i := 0; i < count; i++ {
|
||||
candidate := eligible[i]
|
||||
candidate.State = Extracted
|
||||
s.candidates[candidate.ID] = candidate
|
||||
result.Items = append(result.Items, candidate)
|
||||
}
|
||||
result.Returned = len(result.Items)
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func eligibleForExtraction(candidate Candidate, command Command) bool {
|
||||
if candidate.State != Available {
|
||||
return false
|
||||
}
|
||||
if !candidate.ExpiresAt.IsZero() && candidate.ExpiresAt.Sub(command.Now) < command.MinRemainingTTL {
|
||||
return false
|
||||
}
|
||||
if command.MaxHealthCheckAge > 0 && command.Now.Sub(candidate.LastCheckedAt) > command.MaxHealthCheckAge {
|
||||
return false
|
||||
}
|
||||
return matches(command.Protocols, candidate.Protocol) &&
|
||||
matches(command.Regions, candidate.Region) &&
|
||||
matches(command.Carriers, candidate.Carrier) &&
|
||||
matches(command.Upstreams, candidate.Upstream)
|
||||
}
|
||||
|
||||
func matches(allowed []string, value string) bool {
|
||||
if len(allowed) == 0 {
|
||||
return true
|
||||
}
|
||||
for _, candidate := range allowed {
|
||||
if candidate == value {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
83
internal/domain/extraction/extraction_test.go
Normal file
83
internal/domain/extraction/extraction_test.go
Normal file
@ -0,0 +1,83 @@
|
||||
package extraction
|
||||
|
||||
import (
|
||||
"context"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestMemoryStoreNeverExtractsProxyTwice(t *testing.T) {
|
||||
now := time.Date(2026, 7, 28, 10, 0, 0, 0, time.UTC)
|
||||
store := NewMemoryStore([]Candidate{
|
||||
{ID: "p1", State: Available, ExpiresAt: now.Add(time.Minute), LastCheckedAt: now},
|
||||
})
|
||||
|
||||
var wg sync.WaitGroup
|
||||
results := make(chan string, 1000)
|
||||
for range 1000 {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
result, err := store.Extract(context.Background(), Command{
|
||||
Requested: 1,
|
||||
Fulfillment: Partial,
|
||||
Now: now,
|
||||
MinRemainingTTL: 30 * time.Second,
|
||||
MaxHealthCheckAge: 10 * time.Second,
|
||||
})
|
||||
if err != nil {
|
||||
t.Errorf("Extract(): %v", err)
|
||||
return
|
||||
}
|
||||
for _, item := range result.Items {
|
||||
results <- item.ID
|
||||
}
|
||||
}()
|
||||
}
|
||||
wg.Wait()
|
||||
close(results)
|
||||
|
||||
count := 0
|
||||
for id := range results {
|
||||
if id != "p1" {
|
||||
t.Fatalf("unexpected proxy %q", id)
|
||||
}
|
||||
count++
|
||||
}
|
||||
if count != 1 {
|
||||
t.Fatalf("proxy extracted %d times, want exactly once", count)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAllOrNothingDoesNotConsumePartialInventory(t *testing.T) {
|
||||
now := time.Date(2026, 7, 28, 10, 0, 0, 0, time.UTC)
|
||||
store := NewMemoryStore([]Candidate{
|
||||
{ID: "p1", State: Available, ExpiresAt: now.Add(time.Minute), LastCheckedAt: now},
|
||||
})
|
||||
|
||||
result, err := store.Extract(context.Background(), Command{
|
||||
Requested: 2,
|
||||
Fulfillment: AllOrNothing,
|
||||
Now: now,
|
||||
MinRemainingTTL: 30 * time.Second,
|
||||
MaxHealthCheckAge: 10 * time.Second,
|
||||
})
|
||||
if err != ErrInsufficientProxies {
|
||||
t.Fatalf("Extract() error = %v, want ErrInsufficientProxies", err)
|
||||
}
|
||||
if len(result.Items) != 0 {
|
||||
t.Fatalf("Extract() returned %d items, want 0", len(result.Items))
|
||||
}
|
||||
|
||||
partial, err := store.Extract(context.Background(), Command{
|
||||
Requested: 1,
|
||||
Fulfillment: Partial,
|
||||
Now: now,
|
||||
MinRemainingTTL: 30 * time.Second,
|
||||
MaxHealthCheckAge: 10 * time.Second,
|
||||
})
|
||||
if err != nil || len(partial.Items) != 1 {
|
||||
t.Fatalf("inventory was consumed by failed all-or-nothing: result=%+v err=%v", partial, err)
|
||||
}
|
||||
}
|
||||
134
internal/domain/proxy/capacity.go
Normal file
134
internal/domain/proxy/capacity.go
Normal file
@ -0,0 +1,134 @@
|
||||
package proxy
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"sync/atomic"
|
||||
)
|
||||
|
||||
const counterMask = uint64(1<<32 - 1)
|
||||
|
||||
var (
|
||||
ErrReservationCommitted = errors.New("reservation is already committed")
|
||||
ErrReservationFinished = errors.New("reservation is already finished")
|
||||
)
|
||||
|
||||
type Capacity struct {
|
||||
max atomic.Uint32
|
||||
counters atomic.Uint64
|
||||
}
|
||||
|
||||
func NewCapacity(max int64) *Capacity {
|
||||
capacity := &Capacity{}
|
||||
if max < 0 || max > int64(counterMask) {
|
||||
max = 0
|
||||
}
|
||||
capacity.max.Store(uint32(max))
|
||||
return capacity
|
||||
}
|
||||
|
||||
func (c *Capacity) SetMax(max int64) bool {
|
||||
if max < 0 || max > int64(counterMask) {
|
||||
return false
|
||||
}
|
||||
c.max.Store(uint32(max))
|
||||
return true
|
||||
}
|
||||
|
||||
func (c *Capacity) Max() int64 { return int64(c.max.Load()) }
|
||||
|
||||
func (c *Capacity) Reserve() (*Reservation, bool) {
|
||||
for {
|
||||
current := c.counters.Load()
|
||||
active, reserved := unpack(current)
|
||||
if active+reserved >= c.max.Load() {
|
||||
return nil, false
|
||||
}
|
||||
next := pack(active, reserved+1)
|
||||
if c.counters.CompareAndSwap(current, next) {
|
||||
return &Reservation{capacity: c}, true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (c *Capacity) Active() int64 {
|
||||
active, _ := unpack(c.counters.Load())
|
||||
return int64(active)
|
||||
}
|
||||
|
||||
func (c *Capacity) Reserved() int64 {
|
||||
_, reserved := unpack(c.counters.Load())
|
||||
return int64(reserved)
|
||||
}
|
||||
|
||||
func (c *Capacity) commit() {
|
||||
for {
|
||||
current := c.counters.Load()
|
||||
active, reserved := unpack(current)
|
||||
if reserved == 0 {
|
||||
return
|
||||
}
|
||||
if c.counters.CompareAndSwap(current, pack(active+1, reserved-1)) {
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (c *Capacity) cancel() {
|
||||
for {
|
||||
current := c.counters.Load()
|
||||
active, reserved := unpack(current)
|
||||
if reserved == 0 || c.counters.CompareAndSwap(current, pack(active, reserved-1)) {
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (c *Capacity) release() {
|
||||
for {
|
||||
current := c.counters.Load()
|
||||
active, reserved := unpack(current)
|
||||
if active == 0 || c.counters.CompareAndSwap(current, pack(active-1, reserved)) {
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func pack(active, reserved uint32) uint64 {
|
||||
return uint64(reserved)<<32 | uint64(active)
|
||||
}
|
||||
|
||||
func unpack(value uint64) (active, reserved uint32) {
|
||||
return uint32(value & counterMask), uint32(value >> 32)
|
||||
}
|
||||
|
||||
type Reservation struct {
|
||||
capacity *Capacity
|
||||
state atomic.Uint32
|
||||
}
|
||||
|
||||
func (r *Reservation) Commit() error {
|
||||
if !r.state.CompareAndSwap(0, 1) {
|
||||
if r.state.Load() == 1 {
|
||||
return ErrReservationCommitted
|
||||
}
|
||||
return ErrReservationFinished
|
||||
}
|
||||
r.capacity.commit()
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *Reservation) Cancel() error {
|
||||
if !r.state.CompareAndSwap(0, 2) {
|
||||
return ErrReservationFinished
|
||||
}
|
||||
r.capacity.cancel()
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *Reservation) Release() error {
|
||||
if !r.state.CompareAndSwap(1, 2) {
|
||||
return ErrReservationFinished
|
||||
}
|
||||
r.capacity.release()
|
||||
return nil
|
||||
}
|
||||
78
internal/domain/proxy/proxy.go
Normal file
78
internal/domain/proxy/proxy.go
Normal file
@ -0,0 +1,78 @@
|
||||
package proxy
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
type Scheme string
|
||||
|
||||
const (
|
||||
SchemeHTTP Scheme = "http"
|
||||
SchemeHTTPS Scheme = "https"
|
||||
SchemeSOCKS5 Scheme = "socks5"
|
||||
)
|
||||
|
||||
type Proxy struct {
|
||||
ID string
|
||||
Scheme Scheme
|
||||
Host string
|
||||
Port uint16
|
||||
Username string
|
||||
CredentialVersion string
|
||||
SecretRef string
|
||||
SourceUpstream string
|
||||
CreatedAt time.Time
|
||||
ExpiresAt *time.Time
|
||||
LastCheckedAt *time.Time
|
||||
LastSuccessAt *time.Time
|
||||
Latency time.Duration
|
||||
MaxConcurrency int64
|
||||
State State
|
||||
Tags map[string]string
|
||||
}
|
||||
|
||||
func (p Proxy) UniqueKey() string {
|
||||
host := strings.TrimSuffix(strings.ToLower(strings.TrimSpace(p.Host)), ".")
|
||||
return strings.Join([]string{
|
||||
string(p.Scheme),
|
||||
host,
|
||||
strconv.FormatUint(uint64(p.Port), 10),
|
||||
p.Username,
|
||||
p.CredentialVersion,
|
||||
}, "|")
|
||||
}
|
||||
|
||||
func (p Proxy) Address() string {
|
||||
return net.JoinHostPort(p.Host, strconv.FormatUint(uint64(p.Port), 10))
|
||||
}
|
||||
|
||||
func (p *Proxy) Transition(next State) error {
|
||||
if p == nil {
|
||||
return fmt.Errorf("transition proxy: nil proxy")
|
||||
}
|
||||
if !CanTransition(p.State, next) {
|
||||
return fmt.Errorf("transition proxy: %s -> %s is not allowed", p.State, next)
|
||||
}
|
||||
p.State = next
|
||||
return nil
|
||||
}
|
||||
|
||||
func EffectiveExpiry(now time.Time, expiresAt *time.Time, responseTTL, configuredTTL time.Duration) *time.Time {
|
||||
if expiresAt != nil {
|
||||
value := expiresAt.UTC()
|
||||
return &value
|
||||
}
|
||||
if responseTTL > 0 {
|
||||
value := now.UTC().Add(responseTTL)
|
||||
return &value
|
||||
}
|
||||
if configuredTTL > 0 {
|
||||
value := now.UTC().Add(configuredTTL)
|
||||
return &value
|
||||
}
|
||||
return nil
|
||||
}
|
||||
129
internal/domain/proxy/proxy_test.go
Normal file
129
internal/domain/proxy/proxy_test.go
Normal file
@ -0,0 +1,129 @@
|
||||
package proxy
|
||||
|
||||
import (
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestUniqueKeyIncludesCredentialVersionButNotPassword(t *testing.T) {
|
||||
p := Proxy{
|
||||
Scheme: SchemeHTTP,
|
||||
Host: "EXAMPLE.COM",
|
||||
Port: 8080,
|
||||
Username: "alice",
|
||||
CredentialVersion: "v2",
|
||||
SecretRef: "secret-password",
|
||||
}
|
||||
|
||||
got := p.UniqueKey()
|
||||
want := "http|example.com|8080|alice|v2"
|
||||
if got != want {
|
||||
t.Fatalf("UniqueKey() = %q, want %q", got, want)
|
||||
}
|
||||
if contains(got, p.SecretRef) {
|
||||
t.Fatal("unique key leaked the proxy password reference")
|
||||
}
|
||||
}
|
||||
|
||||
func TestEffectiveExpiryPrecedence(t *testing.T) {
|
||||
now := time.Date(2026, 7, 28, 10, 0, 0, 0, time.UTC)
|
||||
explicit := now.Add(90 * time.Second)
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
expiresAt *time.Time
|
||||
response time.Duration
|
||||
configured time.Duration
|
||||
want *time.Time
|
||||
}{
|
||||
{name: "explicit timestamp", expiresAt: &explicit, response: 2 * time.Minute, configured: 3 * time.Minute, want: &explicit},
|
||||
{name: "response ttl", response: 2 * time.Minute, configured: 3 * time.Minute, want: timePtr(now.Add(2 * time.Minute))},
|
||||
{name: "configured ttl", configured: 3 * time.Minute, want: timePtr(now.Add(3 * time.Minute))},
|
||||
{name: "non expiring"},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := EffectiveExpiry(now, tt.expiresAt, tt.response, tt.configured)
|
||||
if !equalTimePtr(got, tt.want) {
|
||||
t.Fatalf("EffectiveExpiry() = %v, want %v", got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestStateMachineRejectsIllegalTransition(t *testing.T) {
|
||||
p := Proxy{State: StateFetched}
|
||||
if err := p.Transition(StateChecking); err != nil {
|
||||
t.Fatalf("FETCHED -> CHECKING: %v", err)
|
||||
}
|
||||
if err := p.Transition(StateAvailable); err != nil {
|
||||
t.Fatalf("CHECKING -> AVAILABLE: %v", err)
|
||||
}
|
||||
if err := p.Transition(StateFetched); err == nil {
|
||||
t.Fatal("AVAILABLE -> FETCHED must be rejected")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCapacityNeverOversubscribes(t *testing.T) {
|
||||
capacity := NewCapacity(8)
|
||||
var acquired atomic.Int64
|
||||
var peak atomic.Int64
|
||||
var wg sync.WaitGroup
|
||||
|
||||
for range 1000 {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
reservation, ok := capacity.Reserve()
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
active := acquired.Add(1)
|
||||
for {
|
||||
old := peak.Load()
|
||||
if active <= old || peak.CompareAndSwap(old, active) {
|
||||
break
|
||||
}
|
||||
}
|
||||
if err := reservation.Commit(); err != nil {
|
||||
t.Errorf("Commit(): %v", err)
|
||||
}
|
||||
acquired.Add(-1)
|
||||
if err := reservation.Release(); err != nil {
|
||||
t.Errorf("Release(): %v", err)
|
||||
}
|
||||
}()
|
||||
}
|
||||
wg.Wait()
|
||||
|
||||
if peak.Load() > 8 {
|
||||
t.Fatalf("peak reservations = %d, exceeds 8", peak.Load())
|
||||
}
|
||||
if got := capacity.Active(); got != 0 {
|
||||
t.Fatalf("active = %d, want 0", got)
|
||||
}
|
||||
if got := capacity.Reserved(); got != 0 {
|
||||
t.Fatalf("reserved = %d, want 0", got)
|
||||
}
|
||||
}
|
||||
|
||||
func contains(s, part string) bool {
|
||||
for i := 0; i+len(part) <= len(s); i++ {
|
||||
if s[i:i+len(part)] == part {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func timePtr(value time.Time) *time.Time { return &value }
|
||||
|
||||
func equalTimePtr(a, b *time.Time) bool {
|
||||
if a == nil || b == nil {
|
||||
return a == b
|
||||
}
|
||||
return a.Equal(*b)
|
||||
}
|
||||
40
internal/domain/proxy/state.go
Normal file
40
internal/domain/proxy/state.go
Normal file
@ -0,0 +1,40 @@
|
||||
package proxy
|
||||
|
||||
type State string
|
||||
|
||||
const (
|
||||
StateFetched State = "FETCHED"
|
||||
StateChecking State = "CHECKING"
|
||||
StateAvailable State = "AVAILABLE"
|
||||
StateSuspect State = "SUSPECT"
|
||||
StateDraining State = "DRAINING"
|
||||
StateUnhealthy State = "UNHEALTHY"
|
||||
StateExtracted State = "EXTRACTED"
|
||||
StateExpired State = "EXPIRED"
|
||||
StateRemoved State = "REMOVED"
|
||||
)
|
||||
|
||||
var transitions = map[State]map[State]struct{}{
|
||||
StateFetched: set(StateChecking, StateExpired, StateRemoved),
|
||||
StateChecking: set(StateAvailable, StateUnhealthy, StateExpired, StateRemoved),
|
||||
StateAvailable: set(StateSuspect, StateDraining, StateExtracted, StateExpired),
|
||||
StateSuspect: set(StateAvailable, StateUnhealthy, StateDraining, StateExpired),
|
||||
StateDraining: set(StateExpired, StateUnhealthy, StateRemoved),
|
||||
StateUnhealthy: set(StateChecking, StateRemoved, StateExpired),
|
||||
StateExtracted: set(StateExpired, StateRemoved),
|
||||
StateExpired: set(StateRemoved),
|
||||
StateRemoved: {},
|
||||
}
|
||||
|
||||
func CanTransition(current, next State) bool {
|
||||
_, ok := transitions[current][next]
|
||||
return ok
|
||||
}
|
||||
|
||||
func set(states ...State) map[State]struct{} {
|
||||
result := make(map[State]struct{}, len(states))
|
||||
for _, state := range states {
|
||||
result[state] = struct{}{}
|
||||
}
|
||||
return result
|
||||
}
|
||||
62
internal/domain/routing/routing_test.go
Normal file
62
internal/domain/routing/routing_test.go
Normal file
@ -0,0 +1,62 @@
|
||||
package routing
|
||||
|
||||
import (
|
||||
"sync"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestRuleSetUsesFirstMatchingRule(t *testing.T) {
|
||||
rules, err := Compile([]Rule{
|
||||
{Name: "specific", Match: Match{HostRegex: `(^|\.)jd\.com$`}, Upstreams: []string{"jd"}},
|
||||
{Name: "default", Match: Match{HostRegex: `.*`}, Action: ActionReject},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Compile(): %v", err)
|
||||
}
|
||||
|
||||
got, ok := rules.Match(Request{Host: "api.jd.com", Method: "GET", Path: "/"})
|
||||
if !ok || got.Name != "specific" {
|
||||
t.Fatalf("Match() = %q, %v; want specific, true", got.Name, ok)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSequentialSwitchesOnceAtThreshold(t *testing.T) {
|
||||
sequence, err := NewSequential([]string{"a", "b", "c"}, 5)
|
||||
if err != nil {
|
||||
t.Fatalf("NewSequential(): %v", err)
|
||||
}
|
||||
for range 4 {
|
||||
sequence.ObserveEmpty("a")
|
||||
}
|
||||
if got := sequence.Current(); got != "a" {
|
||||
t.Fatalf("Current() = %q before threshold, want a", got)
|
||||
}
|
||||
|
||||
var wg sync.WaitGroup
|
||||
for range 100 {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
sequence.ObserveEmpty("a")
|
||||
}()
|
||||
}
|
||||
wg.Wait()
|
||||
|
||||
if got := sequence.Current(); got != "b" {
|
||||
t.Fatalf("Current() = %q after concurrent threshold, want b", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSequentialValidFetchResetsEmptyCount(t *testing.T) {
|
||||
sequence, _ := NewSequential([]string{"a", "b"}, 5)
|
||||
for range 4 {
|
||||
sequence.ObserveEmpty("a")
|
||||
}
|
||||
sequence.ObserveValid("a")
|
||||
for range 4 {
|
||||
sequence.ObserveEmpty("a")
|
||||
}
|
||||
if got := sequence.Current(); got != "a" {
|
||||
t.Fatalf("Current() = %q, want a after reset", got)
|
||||
}
|
||||
}
|
||||
118
internal/domain/routing/rule.go
Normal file
118
internal/domain/routing/rule.go
Normal file
@ -0,0 +1,118 @@
|
||||
package routing
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"regexp"
|
||||
"slices"
|
||||
"strings"
|
||||
)
|
||||
|
||||
type Action string
|
||||
|
||||
const (
|
||||
ActionProxy Action = "proxy"
|
||||
ActionDirect Action = "direct"
|
||||
ActionReject Action = "reject"
|
||||
)
|
||||
|
||||
type Match struct {
|
||||
HostRegex string
|
||||
Methods []string
|
||||
PathRegex string
|
||||
Headers map[string]string
|
||||
}
|
||||
|
||||
type Rule struct {
|
||||
Name string
|
||||
Match Match
|
||||
Upstreams []string
|
||||
Action Action
|
||||
}
|
||||
|
||||
type Request struct {
|
||||
Host string
|
||||
Method string
|
||||
Path string
|
||||
Headers map[string]string
|
||||
}
|
||||
|
||||
type compiledRule struct {
|
||||
rule Rule
|
||||
host *regexp.Regexp
|
||||
path *regexp.Regexp
|
||||
}
|
||||
|
||||
type RuleSet struct {
|
||||
rules []compiledRule
|
||||
}
|
||||
|
||||
func Compile(rules []Rule) (*RuleSet, error) {
|
||||
compiled := make([]compiledRule, 0, len(rules))
|
||||
for _, rule := range rules {
|
||||
if rule.Name == "" {
|
||||
return nil, fmt.Errorf("compile routing: rule name is required")
|
||||
}
|
||||
host, err := regexp.Compile(rule.Match.HostRegex)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("compile routing %q host: %w", rule.Name, err)
|
||||
}
|
||||
var path *regexp.Regexp
|
||||
if rule.Match.PathRegex != "" {
|
||||
path, err = regexp.Compile(rule.Match.PathRegex)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("compile routing %q path: %w", rule.Name, err)
|
||||
}
|
||||
}
|
||||
if rule.Action == "" && len(rule.Upstreams) > 0 {
|
||||
rule.Action = ActionProxy
|
||||
}
|
||||
compiled = append(compiled, compiledRule{rule: rule, host: host, path: path})
|
||||
}
|
||||
return &RuleSet{rules: compiled}, nil
|
||||
}
|
||||
|
||||
func (r *RuleSet) Match(request Request) (Rule, bool) {
|
||||
if r == nil {
|
||||
return Rule{}, false
|
||||
}
|
||||
host := strings.ToLower(strings.TrimSuffix(request.Host, "."))
|
||||
method := strings.ToUpper(request.Method)
|
||||
for _, candidate := range r.rules {
|
||||
if !candidate.host.MatchString(host) {
|
||||
continue
|
||||
}
|
||||
if len(candidate.rule.Match.Methods) > 0 && !containsFold(candidate.rule.Match.Methods, method) {
|
||||
continue
|
||||
}
|
||||
if candidate.path != nil && !candidate.path.MatchString(request.Path) {
|
||||
continue
|
||||
}
|
||||
if !headersMatch(candidate.rule.Match.Headers, request.Headers) {
|
||||
continue
|
||||
}
|
||||
return candidate.rule, true
|
||||
}
|
||||
return Rule{}, false
|
||||
}
|
||||
|
||||
func containsFold(values []string, target string) bool {
|
||||
return slices.ContainsFunc(values, func(value string) bool {
|
||||
return strings.EqualFold(value, target)
|
||||
})
|
||||
}
|
||||
|
||||
func headersMatch(expected, actual map[string]string) bool {
|
||||
for name, value := range expected {
|
||||
matched := false
|
||||
for actualName, actualValue := range actual {
|
||||
if strings.EqualFold(name, actualName) && actualValue == value {
|
||||
matched = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !matched {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
61
internal/domain/routing/sequential.go
Normal file
61
internal/domain/routing/sequential.go
Normal file
@ -0,0 +1,61 @@
|
||||
package routing
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"sync"
|
||||
)
|
||||
|
||||
type Sequential struct {
|
||||
mu sync.RWMutex
|
||||
upstreams []string
|
||||
threshold int
|
||||
current int
|
||||
empty map[string]int
|
||||
}
|
||||
|
||||
func NewSequential(upstreams []string, threshold int) (*Sequential, error) {
|
||||
if len(upstreams) == 0 {
|
||||
return nil, fmt.Errorf("sequential strategy requires at least one upstream")
|
||||
}
|
||||
if threshold <= 0 {
|
||||
return nil, fmt.Errorf("sequential threshold must be greater than zero")
|
||||
}
|
||||
return &Sequential{
|
||||
upstreams: append([]string(nil), upstreams...),
|
||||
threshold: threshold,
|
||||
empty: make(map[string]int, len(upstreams)),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (s *Sequential) Current() string {
|
||||
s.mu.RLock()
|
||||
defer s.mu.RUnlock()
|
||||
return s.upstreams[s.current]
|
||||
}
|
||||
|
||||
func (s *Sequential) ObserveEmpty(upstream string) bool {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
|
||||
s.empty[upstream]++
|
||||
if s.upstreams[s.current] != upstream || s.empty[upstream] < s.threshold {
|
||||
return false
|
||||
}
|
||||
if s.current+1 >= len(s.upstreams) {
|
||||
return false
|
||||
}
|
||||
s.current++
|
||||
return true
|
||||
}
|
||||
|
||||
func (s *Sequential) ObserveValid(upstream string) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
s.empty[upstream] = 0
|
||||
}
|
||||
|
||||
func (s *Sequential) EmptyCount(upstream string) int {
|
||||
s.mu.RLock()
|
||||
defer s.mu.RUnlock()
|
||||
return s.empty[upstream]
|
||||
}
|
||||
23
internal/domain/upstream/fetch_result.go
Normal file
23
internal/domain/upstream/fetch_result.go
Normal file
@ -0,0 +1,23 @@
|
||||
package upstream
|
||||
|
||||
type FetchClass string
|
||||
|
||||
const (
|
||||
FetchValid FetchClass = "valid"
|
||||
FetchEmpty FetchClass = "empty"
|
||||
FetchDuplicateOnly FetchClass = "duplicate_only"
|
||||
FetchError FetchClass = "error"
|
||||
)
|
||||
|
||||
func ClassifyFetchResult(callErr, parseErr error, validCount, newCount int) FetchClass {
|
||||
if callErr != nil || parseErr != nil {
|
||||
return FetchError
|
||||
}
|
||||
if validCount <= 0 {
|
||||
return FetchEmpty
|
||||
}
|
||||
if newCount <= 0 {
|
||||
return FetchDuplicateOnly
|
||||
}
|
||||
return FetchValid
|
||||
}
|
||||
35
internal/domain/upstream/fetch_result_test.go
Normal file
35
internal/domain/upstream/fetch_result_test.go
Normal file
@ -0,0 +1,35 @@
|
||||
package upstream
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestClassifyFetchResult(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
callErr error
|
||||
parseErr error
|
||||
valid int
|
||||
newCount int
|
||||
want FetchClass
|
||||
}{
|
||||
{name: "network error", callErr: errFixture, want: FetchError},
|
||||
{name: "parse error", parseErr: errFixture, want: FetchError},
|
||||
{name: "empty response", want: FetchEmpty},
|
||||
{name: "duplicates are not empty", valid: 3, want: FetchDuplicateOnly},
|
||||
{name: "new proxy", valid: 3, newCount: 1, want: FetchValid},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := ClassifyFetchResult(tt.callErr, tt.parseErr, tt.valid, tt.newCount)
|
||||
if got != tt.want {
|
||||
t.Fatalf("ClassifyFetchResult() = %q, want %q", got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
type fixtureError string
|
||||
|
||||
func (e fixtureError) Error() string { return string(e) }
|
||||
|
||||
const errFixture = fixtureError("fixture")
|
||||
109
internal/gateway/dispatch/dispatcher.go
Normal file
109
internal/gateway/dispatch/dispatcher.go
Normal file
@ -0,0 +1,109 @@
|
||||
package dispatch
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
proxyDomain "github.com/proxy-pool/proxy-pool/internal/domain/proxy"
|
||||
"github.com/proxy-pool/proxy-pool/internal/gateway/snapshot"
|
||||
)
|
||||
|
||||
var ErrNoCandidate = errors.New("no local proxy candidate is available")
|
||||
|
||||
type Request struct {
|
||||
Now time.Time
|
||||
Scheme proxyDomain.Scheme
|
||||
Upstreams []string
|
||||
RequiredTags map[string]string
|
||||
Exclude map[string]struct{}
|
||||
SafetyMargin time.Duration
|
||||
}
|
||||
|
||||
type Lease struct {
|
||||
Proxy proxyDomain.Proxy
|
||||
Epoch uint64
|
||||
Version uint64
|
||||
reserved *proxyDomain.Reservation
|
||||
}
|
||||
|
||||
func (l *Lease) Commit() error { return l.reserved.Commit() }
|
||||
func (l *Lease) Cancel() error { return l.reserved.Cancel() }
|
||||
func (l *Lease) Release() error { return l.reserved.Release() }
|
||||
|
||||
type Dispatcher struct {
|
||||
store *snapshot.Store
|
||||
cursor atomic.Uint64
|
||||
}
|
||||
|
||||
func New(store *snapshot.Store) *Dispatcher {
|
||||
return &Dispatcher{store: store}
|
||||
}
|
||||
|
||||
func (d *Dispatcher) Acquire(request Request) (*Lease, error) {
|
||||
if d == nil || d.store == nil {
|
||||
return nil, ErrNoCandidate
|
||||
}
|
||||
view := d.store.Current()
|
||||
if view == nil || len(view.Entries) == 0 {
|
||||
return nil, ErrNoCandidate
|
||||
}
|
||||
if request.Now.IsZero() {
|
||||
request.Now = time.Now().UTC()
|
||||
}
|
||||
|
||||
start := int((d.cursor.Add(1) - 1) % uint64(len(view.Entries)))
|
||||
for offset := 0; offset < len(view.Entries); offset++ {
|
||||
entry := view.Entries[(start+offset)%len(view.Entries)]
|
||||
if !eligible(entry.Proxy, request) {
|
||||
continue
|
||||
}
|
||||
reservation, ok := entry.Runtime.Reserve()
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
return &Lease{
|
||||
Proxy: entry.Proxy,
|
||||
Epoch: view.Epoch,
|
||||
Version: view.Version,
|
||||
reserved: reservation,
|
||||
}, nil
|
||||
}
|
||||
return nil, ErrNoCandidate
|
||||
}
|
||||
|
||||
func eligible(candidate proxyDomain.Proxy, request Request) bool {
|
||||
if candidate.State != proxyDomain.StateAvailable {
|
||||
return false
|
||||
}
|
||||
if request.Scheme != "" && candidate.Scheme != request.Scheme {
|
||||
return false
|
||||
}
|
||||
if _, excluded := request.Exclude[candidate.ID]; excluded {
|
||||
return false
|
||||
}
|
||||
if candidate.ExpiresAt != nil && !candidate.ExpiresAt.After(request.Now.Add(request.SafetyMargin)) {
|
||||
return false
|
||||
}
|
||||
if !contains(request.Upstreams, candidate.SourceUpstream) {
|
||||
return false
|
||||
}
|
||||
for key, value := range request.RequiredTags {
|
||||
if candidate.Tags[key] != value {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func contains(allowed []string, value string) bool {
|
||||
if len(allowed) == 0 {
|
||||
return true
|
||||
}
|
||||
for _, candidate := range allowed {
|
||||
if candidate == value {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
90
internal/gateway/dispatch/dispatcher_test.go
Normal file
90
internal/gateway/dispatch/dispatcher_test.go
Normal file
@ -0,0 +1,90 @@
|
||||
package dispatch
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
proxyDomain "github.com/proxy-pool/proxy-pool/internal/domain/proxy"
|
||||
"github.com/proxy-pool/proxy-pool/internal/gateway/snapshot"
|
||||
)
|
||||
|
||||
func TestAcquireFiltersAndReservesLocalCapacity(t *testing.T) {
|
||||
now := time.Date(2026, 7, 28, 12, 0, 0, 0, time.UTC)
|
||||
expiresSoon := now.Add(5 * time.Second)
|
||||
expiresLater := now.Add(time.Minute)
|
||||
store := snapshot.NewStore("cluster-a", "worker-a")
|
||||
proxies := []proxyDomain.Proxy{
|
||||
{ID: "wrong-upstream", Scheme: proxyDomain.SchemeHTTP, SourceUpstream: "b", State: proxyDomain.StateAvailable, MaxConcurrency: 1, ExpiresAt: &expiresLater},
|
||||
{ID: "expiring", Scheme: proxyDomain.SchemeHTTP, SourceUpstream: "a", State: proxyDomain.StateAvailable, MaxConcurrency: 1, ExpiresAt: &expiresSoon},
|
||||
{ID: "selected", Scheme: proxyDomain.SchemeHTTP, SourceUpstream: "a", State: proxyDomain.StateAvailable, MaxConcurrency: 1, ExpiresAt: &expiresLater, Tags: map[string]string{"region": "cn-east"}},
|
||||
}
|
||||
envelope := snapshot.Envelope{ClusterID: "cluster-a", WorkerID: "worker-a", Epoch: 1, Version: 1, Full: true, Proxies: proxies}
|
||||
envelope.Checksum = snapshot.Checksum(proxies)
|
||||
if err := store.Apply(envelope); err != nil {
|
||||
t.Fatalf("Apply(): %v", err)
|
||||
}
|
||||
|
||||
dispatcher := New(store)
|
||||
lease, err := dispatcher.Acquire(Request{
|
||||
Now: now,
|
||||
Scheme: proxyDomain.SchemeHTTP,
|
||||
Upstreams: []string{"a"},
|
||||
RequiredTags: map[string]string{"region": "cn-east"},
|
||||
SafetyMargin: 10 * time.Second,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Acquire(): %v", err)
|
||||
}
|
||||
if lease.Proxy.ID != "selected" {
|
||||
t.Fatalf("selected proxy = %q, want selected", lease.Proxy.ID)
|
||||
}
|
||||
if err := lease.Commit(); err != nil {
|
||||
t.Fatalf("Commit(): %v", err)
|
||||
}
|
||||
if err := lease.Release(); err != nil {
|
||||
t.Fatalf("Release(): %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAcquireNeverOversubscribesSnapshotProxy(t *testing.T) {
|
||||
store := snapshot.NewStore("cluster-a", "worker-a")
|
||||
proxies := []proxyDomain.Proxy{{ID: "p1", Scheme: proxyDomain.SchemeHTTP, State: proxyDomain.StateAvailable, MaxConcurrency: 8}}
|
||||
envelope := snapshot.Envelope{ClusterID: "cluster-a", WorkerID: "worker-a", Epoch: 1, Version: 1, Full: true, Proxies: proxies}
|
||||
envelope.Checksum = snapshot.Checksum(proxies)
|
||||
if err := store.Apply(envelope); err != nil {
|
||||
t.Fatalf("Apply(): %v", err)
|
||||
}
|
||||
dispatcher := New(store)
|
||||
|
||||
var wg sync.WaitGroup
|
||||
leases := make(chan *Lease, 1000)
|
||||
for range 1000 {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
lease, err := dispatcher.Acquire(Request{Now: time.Now(), Scheme: proxyDomain.SchemeHTTP})
|
||||
if err == nil {
|
||||
leases <- lease
|
||||
return
|
||||
}
|
||||
if !errors.Is(err, ErrNoCandidate) {
|
||||
t.Errorf("Acquire(): %v", err)
|
||||
}
|
||||
}()
|
||||
}
|
||||
wg.Wait()
|
||||
close(leases)
|
||||
|
||||
count := 0
|
||||
for lease := range leases {
|
||||
count++
|
||||
if err := lease.Cancel(); err != nil {
|
||||
t.Errorf("Cancel(): %v", err)
|
||||
}
|
||||
}
|
||||
if count != 8 {
|
||||
t.Fatalf("reserved = %d, want 8", count)
|
||||
}
|
||||
}
|
||||
149
internal/gateway/snapshot/store.go
Normal file
149
internal/gateway/snapshot/store.go
Normal file
@ -0,0 +1,149 @@
|
||||
package snapshot
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sort"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
|
||||
proxyDomain "github.com/proxy-pool/proxy-pool/internal/domain/proxy"
|
||||
)
|
||||
|
||||
var (
|
||||
ErrWrongTarget = errors.New("snapshot targets another cluster or worker")
|
||||
ErrResyncRequired = errors.New("snapshot sequence requires a full resync")
|
||||
ErrChecksumMismatch = errors.New("snapshot checksum mismatch")
|
||||
)
|
||||
|
||||
type Envelope struct {
|
||||
ClusterID string
|
||||
WorkerID string
|
||||
Epoch uint64
|
||||
Version uint64
|
||||
Full bool
|
||||
Checksum string
|
||||
Proxies []proxyDomain.Proxy
|
||||
}
|
||||
|
||||
type Entry struct {
|
||||
Proxy proxyDomain.Proxy
|
||||
Runtime *proxyDomain.Capacity
|
||||
}
|
||||
|
||||
type View struct {
|
||||
ClusterID string
|
||||
WorkerID string
|
||||
Epoch uint64
|
||||
Version uint64
|
||||
Checksum string
|
||||
Entries []Entry
|
||||
}
|
||||
|
||||
type Store struct {
|
||||
clusterID string
|
||||
workerID string
|
||||
current atomic.Pointer[View]
|
||||
|
||||
mu sync.Mutex
|
||||
runtimes map[string]*proxyDomain.Capacity
|
||||
}
|
||||
|
||||
func NewStore(clusterID, workerID string) *Store {
|
||||
return &Store{
|
||||
clusterID: clusterID,
|
||||
workerID: workerID,
|
||||
runtimes: make(map[string]*proxyDomain.Capacity),
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Store) Current() *View {
|
||||
if s == nil {
|
||||
return nil
|
||||
}
|
||||
return s.current.Load()
|
||||
}
|
||||
|
||||
func (s *Store) Apply(envelope Envelope) error {
|
||||
if s == nil {
|
||||
return fmt.Errorf("apply snapshot: nil store")
|
||||
}
|
||||
if envelope.ClusterID != s.clusterID || envelope.WorkerID != s.workerID {
|
||||
return ErrWrongTarget
|
||||
}
|
||||
if !envelope.Full || envelope.Epoch == 0 || envelope.Version == 0 {
|
||||
return ErrResyncRequired
|
||||
}
|
||||
if envelope.Checksum != Checksum(envelope.Proxies) {
|
||||
return ErrChecksumMismatch
|
||||
}
|
||||
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
|
||||
current := s.current.Load()
|
||||
if current != nil {
|
||||
switch {
|
||||
case envelope.Epoch < current.Epoch:
|
||||
return ErrResyncRequired
|
||||
case envelope.Epoch == current.Epoch && envelope.Version != current.Version+1:
|
||||
return ErrResyncRequired
|
||||
case envelope.Epoch > current.Epoch && envelope.Version != 1:
|
||||
return ErrResyncRequired
|
||||
}
|
||||
}
|
||||
|
||||
proxies := cloneAndSort(envelope.Proxies)
|
||||
entries := make([]Entry, 0, len(proxies))
|
||||
for _, descriptor := range proxies {
|
||||
runtime := s.runtimes[descriptor.ID]
|
||||
if runtime == nil {
|
||||
runtime = proxyDomain.NewCapacity(descriptor.MaxConcurrency)
|
||||
s.runtimes[descriptor.ID] = runtime
|
||||
} else {
|
||||
runtime.SetMax(descriptor.MaxConcurrency)
|
||||
}
|
||||
entries = append(entries, Entry{Proxy: descriptor, Runtime: runtime})
|
||||
}
|
||||
|
||||
next := &View{
|
||||
ClusterID: envelope.ClusterID,
|
||||
WorkerID: envelope.WorkerID,
|
||||
Epoch: envelope.Epoch,
|
||||
Version: envelope.Version,
|
||||
Checksum: envelope.Checksum,
|
||||
Entries: entries,
|
||||
}
|
||||
s.current.Store(next)
|
||||
return nil
|
||||
}
|
||||
|
||||
func Checksum(proxies []proxyDomain.Proxy) string {
|
||||
canonical := cloneAndSort(proxies)
|
||||
encoded, err := json.Marshal(canonical)
|
||||
if err != nil {
|
||||
panic(fmt.Sprintf("encode snapshot checksum: %v", err))
|
||||
}
|
||||
digest := sha256.Sum256(encoded)
|
||||
return hex.EncodeToString(digest[:])
|
||||
}
|
||||
|
||||
func cloneAndSort(source []proxyDomain.Proxy) []proxyDomain.Proxy {
|
||||
cloned := make([]proxyDomain.Proxy, len(source))
|
||||
for index, descriptor := range source {
|
||||
cloned[index] = descriptor
|
||||
if descriptor.Tags != nil {
|
||||
cloned[index].Tags = make(map[string]string, len(descriptor.Tags))
|
||||
for key, value := range descriptor.Tags {
|
||||
cloned[index].Tags[key] = value
|
||||
}
|
||||
}
|
||||
}
|
||||
sort.Slice(cloned, func(i, j int) bool {
|
||||
return cloned[i].ID < cloned[j].ID
|
||||
})
|
||||
return cloned
|
||||
}
|
||||
80
internal/gateway/snapshot/store_test.go
Normal file
80
internal/gateway/snapshot/store_test.go
Normal file
@ -0,0 +1,80 @@
|
||||
package snapshot
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"testing"
|
||||
|
||||
proxyDomain "github.com/proxy-pool/proxy-pool/internal/domain/proxy"
|
||||
)
|
||||
|
||||
func TestStoreAppliesCompleteSnapshotsInOrder(t *testing.T) {
|
||||
store := NewStore("cluster-a", "worker-a")
|
||||
first := Envelope{
|
||||
ClusterID: "cluster-a",
|
||||
WorkerID: "worker-a",
|
||||
Epoch: 1,
|
||||
Version: 1,
|
||||
Full: true,
|
||||
Proxies: []proxyDomain.Proxy{{
|
||||
ID: "p1",
|
||||
Scheme: proxyDomain.SchemeHTTP,
|
||||
Host: "127.0.0.1",
|
||||
Port: 18080,
|
||||
MaxConcurrency: 2,
|
||||
State: proxyDomain.StateAvailable,
|
||||
}},
|
||||
}
|
||||
first.Checksum = Checksum(first.Proxies)
|
||||
if err := store.Apply(first); err != nil {
|
||||
t.Fatalf("Apply(first): %v", err)
|
||||
}
|
||||
|
||||
view := store.Current()
|
||||
if view == nil || view.Version != 1 || len(view.Entries) != 1 {
|
||||
t.Fatalf("Current() = %+v", view)
|
||||
}
|
||||
if view.Entries[0].Runtime == nil {
|
||||
t.Fatal("snapshot entry has no local runtime capacity")
|
||||
}
|
||||
|
||||
second := first
|
||||
second.Version = 2
|
||||
second.Proxies = append([]proxyDomain.Proxy(nil), first.Proxies...)
|
||||
second.Proxies[0].Host = "localhost"
|
||||
second.Checksum = Checksum(second.Proxies)
|
||||
if err := store.Apply(second); err != nil {
|
||||
t.Fatalf("Apply(second): %v", err)
|
||||
}
|
||||
if got := store.Current().Entries[0].Proxy.Host; got != "localhost" {
|
||||
t.Fatalf("host = %q, want localhost", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStoreRejectsWrongWorkerVersionGapAndChecksum(t *testing.T) {
|
||||
store := NewStore("cluster-a", "worker-a")
|
||||
base := Envelope{ClusterID: "cluster-a", WorkerID: "worker-a", Epoch: 1, Version: 1, Full: true}
|
||||
base.Checksum = Checksum(base.Proxies)
|
||||
if err := store.Apply(base); err != nil {
|
||||
t.Fatalf("Apply(base): %v", err)
|
||||
}
|
||||
|
||||
wrongWorker := base
|
||||
wrongWorker.Version = 2
|
||||
wrongWorker.WorkerID = "worker-b"
|
||||
if err := store.Apply(wrongWorker); !errors.Is(err, ErrWrongTarget) {
|
||||
t.Fatalf("wrong worker error = %v, want ErrWrongTarget", err)
|
||||
}
|
||||
|
||||
gap := base
|
||||
gap.Version = 3
|
||||
if err := store.Apply(gap); !errors.Is(err, ErrResyncRequired) {
|
||||
t.Fatalf("version gap error = %v, want ErrResyncRequired", err)
|
||||
}
|
||||
|
||||
badChecksum := base
|
||||
badChecksum.Version = 2
|
||||
badChecksum.Checksum = "bad"
|
||||
if err := store.Apply(badChecksum); !errors.Is(err, ErrChecksumMismatch) {
|
||||
t.Fatalf("checksum error = %v, want ErrChecksumMismatch", err)
|
||||
}
|
||||
}
|
||||
16
progress.md
16
progress.md
@ -7,5 +7,17 @@
|
||||
- 已按主题定位配置定稿、实施方案、Distribution API、认证、安全、并发、
|
||||
故障语义和最终 Exclusive Extraction 修订。
|
||||
- 已建立新的任务计划与事实记录,旧网页摘要不再作为需求证据。
|
||||
- 尚未创建实现代码。
|
||||
|
||||
- 已建立需求追踪矩阵、统一领域语言、产品设计、总体架构、项目结构、ADR、
|
||||
开发、配置、API、安全、测试与运维文档。
|
||||
- 已提供 20 个严格校验的配置示例、Distribution/Admin OpenAPI、Controller/
|
||||
Worker/Checker Protobuf 契约和 35 张 Mermaid 图。
|
||||
- 已实现并测试 Proxy 状态/TTL/唯一键、打包原子容量、首条路由、Sequential
|
||||
并发切换、Fetch 分类、一次性独占提取、严格配置、Snapshot 与本地 Dispatch。
|
||||
- `go test ./...`、`go vet ./...`、`go build ./...` 通过。
|
||||
- Protobuf 描述符编译、Compose 静态展开、Kustomize 渲染、Grafana JSON 与
|
||||
配置示例校验通过。
|
||||
- Windows 当前 `CGO_ENABLED=0` 且无 C 编译器,race 测试由 Linux CI 承担。
|
||||
- 100,000 QPS 仍是未验证设计目标;运行进程、存储适配器、完整网络转发与
|
||||
代表性集群压测尚未实施,已在完成审计中明确列出。
|
||||
- 已生成 `proxy-pool-docs-v1.0.zip`,包含 50 个条目,SHA-256 为
|
||||
`A6882B71196210CE3594A10992C2A0A73EA3A1CF31D8A570709CA999133CE104`。
|
||||
|
||||
BIN
proxy-pool-docs-v1.0.zip
Normal file
BIN
proxy-pool-docs-v1.0.zip
Normal file
Binary file not shown.
31
scripts/verify.ps1
Normal file
31
scripts/verify.ps1
Normal file
@ -0,0 +1,31 @@
|
||||
$ErrorActionPreference = "Stop"
|
||||
|
||||
function Invoke-Step {
|
||||
param(
|
||||
[string]$Name,
|
||||
[scriptblock]$Command
|
||||
)
|
||||
|
||||
Write-Host "==> $Name"
|
||||
& $Command
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
throw "$Name failed with exit code $LASTEXITCODE"
|
||||
}
|
||||
}
|
||||
|
||||
$unformatted = gofmt -l .
|
||||
if ($unformatted) {
|
||||
$unformatted | ForEach-Object { Write-Host $_ }
|
||||
throw "gofmt check failed"
|
||||
}
|
||||
|
||||
Invoke-Step "go vet" { go vet ./... }
|
||||
Invoke-Step "unit tests" { go test -timeout 60s ./... }
|
||||
|
||||
if ((go env CGO_ENABLED) -eq "1") {
|
||||
Invoke-Step "race tests" { go test -race -timeout 60s ./internal/... }
|
||||
} else {
|
||||
Write-Host "==> race tests skipped: CGO_ENABLED is not 1"
|
||||
}
|
||||
|
||||
Invoke-Step "build" { go build ./... }
|
||||
23
task_plan.md
23
task_plan.md
@ -15,13 +15,15 @@
|
||||
## 阶段
|
||||
|
||||
1. [已完成] 完整读取对话并识别覆盖关系
|
||||
2. [进行中] 建立需求追踪矩阵与统一领域模型
|
||||
3. [待开始] 编写总体设计、详细设计和 ADR
|
||||
4. [待开始] 编写开发、配置、API、测试和运维文档
|
||||
5. [待开始] 搭建 Go 模块、命令、核心包、契约和部署目录
|
||||
6. [待开始] 实现核心状态机、路由、容量、提取与配置校验
|
||||
7. [待开始] 执行单元测试、竞态检查、静态检查和构建
|
||||
8. [待开始] 按需求矩阵逐项审计并打包交付
|
||||
2. [已完成] 建立需求追踪矩阵与统一领域模型
|
||||
3. [已完成] 编写总体设计、产品设计、项目结构和 ADR
|
||||
4. [已完成] 编写开发、配置、API、测试、安全和运维文档
|
||||
5. [已完成] 搭建 Go 模块、核心包、契约和部署拓扑目录
|
||||
6. [已完成] 实现状态机、首条路由、Sequential、原子容量、独占提取、
|
||||
Fetch 分类、严格配置、不可变快照和本地调度参考实现
|
||||
7. [已完成] 执行单元测试、静态检查、构建和静态部署/契约验证;本机因
|
||||
`CGO_ENABLED=0` 且无 C 编译器未运行 race,保留给 Linux CI
|
||||
8. [已完成] 按需求矩阵逐项审计并生成版本化文档包
|
||||
|
||||
## 串并行关系
|
||||
|
||||
@ -40,6 +42,7 @@
|
||||
|
||||
## 已知环境限制
|
||||
|
||||
- Docker CLI 已安装,但 Linux daemon 状态需在集成验证前再次确认。
|
||||
- 当前仓库尚无提交;`对话内容.md` 和 `.gitignore` 为现有文件。
|
||||
|
||||
- Docker Compose 配置与 Kubernetes Kustomize 已完成静态渲染验证;未启动
|
||||
目标运行拓扑。
|
||||
- `cmd/proxy-*`、Provider 调度、PostgreSQL/Redis 适配器、Gateway Transport
|
||||
与 Checker 运行时属于后续实施范围,见完成审计。
|
||||
|
||||
Loading…
Reference in New Issue
Block a user