Compare commits

..

50 Commits

Author SHA1 Message Date
youfak
d6ac783c1d ci: build linux application image
Some checks failed
ci / openapi (push) Has been cancelled
ci / proto (push) Has been cancelled
ci / image (push) Has been cancelled
ci / deployment (push) Has been cancelled
ci / test (ubuntu-latest) (push) Has been cancelled
ci / test (windows-latest) (push) Has been cancelled
ci / race (push) Has been cancelled
ci / integration (push) Has been cancelled
2026-08-07 21:11:36 +08:00
youfak
4f0245a7ec docs: align loadgen capability descriptions 2026-08-07 21:09:59 +08:00
youfak
3ae04d8a62 fix: roll development pods on config changes 2026-08-07 21:09:08 +08:00
youfak
51158007b4 ci: validate local deployment configuration 2026-08-07 21:04:08 +08:00
youfak
5c3c893c71 fix: exclude local secrets from docker context 2026-08-07 21:02:28 +08:00
youfak
5e9e8c0d49 ci: validate openapi contracts with redocly 2026-08-07 20:59:37 +08:00
youfak
a92dcab333 fix: require single uri san for spiffe identity 2026-08-07 20:48:09 +08:00
youfak
57013dd6b2 fix: unify strict spiffe identity validation 2026-08-07 20:16:46 +08:00
youfak
e318ddbb54 docs: align development identity delivery evidence 2026-08-07 20:12:18 +08:00
youfak
25218e8d27 feat: derive worker identity from spiffe certificates 2026-08-07 20:07:59 +08:00
youfak
cc30399535 feat: reload control plane trust bundles 2026-08-07 18:26:09 +08:00
youfak
72d0517eae feat: reload control plane leaf certificates 2026-08-07 18:18:04 +08:00
youfak
3196667d54 test: enforce repository delivery audit 2026-08-07 18:12:15 +08:00
youfak
e4b728f72e ci: pin deployment config validation toolchain 2026-08-07 18:08:02 +08:00
youfak
0ca06beb5d docs: complete capacity operations runbook 2026-08-07 18:05:17 +08:00
youfak
4f49473cdd feat: add development kubernetes mtls overlay 2026-08-07 18:02:27 +08:00
youfak
2124523cef test: record bounded gateway benchmark evidence 2026-08-07 17:45:15 +08:00
youfak
bb4b205aa0 docs: package versioned delivery artifacts 2026-08-07 17:39:23 +08:00
youfak
a384fce1b0 ci: validate deployment manifests 2026-08-07 17:34:25 +08:00
youfak
1a8eda65c0 feat: gate loadgen reports by acceptance thresholds 2026-08-07 17:30:38 +08:00
youfak
a4aa245b6b feat: alert on gateway p99 latency 2026-08-07 17:14:54 +08:00
youfak
04a9fa89a0 feat: observe gateway request durations 2026-08-07 17:12:57 +08:00
youfak
7f51e333c5 fix: align observability assets with registered metrics 2026-08-07 17:04:46 +08:00
youfak
a191ca5837 feat: honor admin routing state in target scheduling 2026-08-07 16:57:30 +08:00
youfak
351836b5e4 feat: honor admin upstream state in health scheduling 2026-08-07 16:51:25 +08:00
youfak
5e748c6325 feat: advance sequential routes past disabled upstreams 2026-08-07 16:41:00 +08:00
youfak
7e73b5ad1c feat: run compose control plane with mtls 2026-08-07 16:35:11 +08:00
youfak
3bcc7298a2 feat: support static gateway direct routes 2026-08-07 16:16:00 +08:00
youfak
d9d3a3d6c0 fix: account for queued loadgen rate starts 2026-08-07 16:00:09 +08:00
youfak
5ffd87f56b feat: persist loadgen reports 2026-08-07 15:46:10 +08:00
youfak
1367f1b11f feat: report loadgen rate dispatch loss 2026-08-07 15:42:28 +08:00
youfak
26572da97e feat: observe gateway connection lifecycles 2026-08-07 15:37:35 +08:00
youfak
bda9cc03df feat: configure gateway transport pools 2026-08-07 15:27:17 +08:00
youfak
d4539da9c5 feat: cap gateway client concurrency 2026-08-07 15:18:52 +08:00
youfak
2b18ad8dd4 feat: limit gateway clients by credential 2026-08-07 15:10:39 +08:00
youfak
6194e3e673 feat: add gateway sticky sessions 2026-08-02 16:47:51 +08:00
youfak
5389d6f886 feat: restrict gateway routes by credential 2026-08-02 16:21:10 +08:00
youfak
f644737435 feat: enforce credential extraction policies 2026-08-02 16:07:27 +08:00
youfak
aaed288211 feat: enforce credential endpoint permissions 2026-08-02 15:50:23 +08:00
youfak
ec3ceb2c9a feat: expose admin audit pages 2026-08-02 15:29:15 +08:00
youfak
e220d5368c feat: persist terminal sequential stops 2026-08-02 15:14:20 +08:00
youfak
589cd4e454 feat: automate sequential routing switches 2026-08-02 15:00:08 +08:00
youfak
707aff26ea feat: observe gateway capacity invariants 2026-08-02 14:45:47 +08:00
youfak
af1cd70986 feat: add secret-safe process logging 2026-08-02 14:29:10 +08:00
youfak
b24d71fb90 feat: expose capacity metrics 2026-08-02 14:21:18 +08:00
youfak
4691b9150c feat: expose extraction metrics 2026-08-02 14:08:45 +08:00
youfak
162fd4b7b8 feat: expose provider fetch metrics 2026-08-02 13:55:23 +08:00
youfak
38f4504aa8 feat: refresh worker snapshots after routing changes 2026-08-02 13:33:53 +08:00
youfak
92c8079fd7 feat: expose controller drain metrics 2026-08-02 13:20:42 +08:00
youfak
10d8f2e3d4 feat: drain worker assignments for disabled upstreams 2026-08-02 13:15:14 +08:00
189 changed files with 12362 additions and 765 deletions

23
.dockerignore Normal file
View File

@ -0,0 +1,23 @@
# Source control metadata
.git
# Local configuration and credential material
.env
.env.*
configs/local.yaml
deploy/.control-plane-tls/
*.crt
*.key
*.pem
# Local build, test, and tooling outputs
.playwright-mcp/
temp/
bin/
dist/
coverage/
*.out
*.test
*.prof
*.exe
.tmp-proto/

View File

@ -8,6 +8,16 @@ permissions:
contents: read
jobs:
openapi:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: actions/setup-node@v4
with:
node-version: 24
- shell: pwsh
run: ./scripts/verify-openapi.ps1
proto:
runs-on: ubuntu-latest
steps:
@ -19,6 +29,44 @@ jobs:
- shell: pwsh
run: ./scripts/verify-proto.ps1
image:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- name: Build Linux application image
run: docker build --file deploy/docker/Dockerfile --tag proxy-pool:ci .
deployment:
runs-on: ubuntu-latest
env:
PROXY_POOL_GATEWAY_USERNAME: ci-gateway-user
PROXY_POOL_GATEWAY_PASSWORD: ci-gateway-password
PROXY_POOL_EXTRACT_TOKEN: ci-extract-token
PROXY_POOL_ADMIN_TOKEN: ci-admin-token
PROXY_POOL_CONFIG_FINGERPRINT_KEY: ci-config-fingerprint-key-at-least-32-bytes
PROXY_POOL_POSTGRES_URL: postgres://ci:ci@postgres.example:5432/proxy_pool?sslmode=verify-full
PROXY_POOL_REDIS_URL: rediss://:ci@redis.example:6379/0
PROVIDER_A_TOKEN: ci-provider-a-token
PROVIDER_B_TOKEN: ci-provider-b-token
steps:
- uses: actions/checkout@v4
- uses: actions/setup-go@v5
with:
go-version-file: go.mod
cache: true
- name: Render Compose configuration
run: docker compose -f deploy/docker-compose.yml config --quiet
- name: Render integration Compose configuration
run: docker compose -f deploy/docker-compose.test.yml config --quiet
- name: Validate local Compose configuration
run: go run ./deploy/tools/configcheck deploy/config/local.yaml
- name: Render Kubernetes base
run: kubectl kustomize deploy/kubernetes/base > /dev/null
- name: Render Kubernetes development mTLS overlay
run: kubectl kustomize deploy/kubernetes/overlays/development-mtls > /dev/null
- name: Validate development mTLS configuration
run: go run ./deploy/tools/configcheck deploy/kubernetes/overlays/development-mtls/config.yaml
test:
strategy:
matrix:

1
.gitignore vendored
View File

@ -17,6 +17,7 @@ coverage/
.env.*
!.env.example
configs/local.yaml
deploy/.control-plane-tls/
*.pem
*.key

123
README.md
View File

@ -41,8 +41,28 @@ Proxy Pool 用 Controller 协调这些变化,并让 Gateway 数据面只消费
PostgreSQL 迁移与 Redis 活动池,并支持联动优雅停机。
- **Checker 指标**Metrics 启用时暴露 Checker 任务下发与 Observation 接受/拒绝计数;
标签仅使用固定检查级别和结果,不记录 Proxy、IP、URL 或凭据。
- **Gateway 指标**Metrics 启用时暴露代理尝试的固定阶段成功/失败计数,以及本地
Outcome 队列满后的丢弃计数;不记录 Proxy、路由、目标、客户端或凭据。
- **Gateway 指标**Metrics 启用时暴露代理尝试的固定阶段成功/失败计数、本地
Outcome 队列满后的丢弃计数、HTTP/CONNECT 已接收请求数与在途数,以及活跃
CONNECT 隧道数和请求生命周期 p50/p95/p99 直方图;协议标签仅有 HTTP 与 CONNECT不记录 Proxy、路由、目标、
客户端或凭据。
- **Drain 指标**Controller 暴露 `proxy_pool_controller_drain_candidates_total`
`proxy_pool_controller_drains_started_total``reason` 仅有 `unhealthy`
`upstream_disabled`,不包含 Proxy、Worker、Upstream、会话或地址。
- **Provider 指标**Controller 暴露
`proxy_pool_controller_provider_fetch_results_total{class}`
`proxy_pool_controller_provider_valid_candidates_total`
`proxy_pool_controller_provider_new_proxies_total``class` 仅有 `valid`
`empty`、`duplicate_only`、`error`,不包含 Upstream、Proxy 或错误文本标签。
- **提取指标**Controller 暴露
`proxy_pool_controller_extraction_requests_total{result}`
`proxy_pool_controller_extraction_requested_proxies_total`
`proxy_pool_controller_extraction_returned_proxies_total``result` 仅有完成、
部分、空、库存不足、幂等冲突、限流、不可用、无效与内部错误等固定枚举,
不包含 Client、请求、过滤条件、Upstream、Proxy 或错误文本标签。
- **容量指标**Controller 在既有 Provider 库存对账周期聚合托管 Proxy、可用/有效
Slot、待拉取数量和活跃上游数并暴露
`proxy_pool_controller_capacity_inventory_reads_total{result}``result` 仅有
`success``error`,不按 Upstream、Worker、Proxy、会话或地址拆分。
- **PostgreSQL 管理面**持久化配置版本、Upstream/Routing 管理状态、Admin
审计与 Outbox不保存 Proxy 明细或逐次提取记录。
- **Gateway 组件**HTTP 正向代理、HTTPS CONNECT、双向 Tunnel、重试、超时、
@ -56,9 +76,20 @@ Proxy Pool 用 Controller 协调这些变化,并让 Gateway 数据面只消费
Connections 选择上游。无候选时支持 reject、受 `waitTimeout` 限制的本地容量等待,
以及仍经过目标地址策略的 direct`proxy-gateway` 通过独立控制面拨号地址维护
Session并在每份 Snapshot 有效期的一半前接收版本递增的完整刷新;仅在持有未过期
Snapshot 时 Ready凭据材料只保留在当前节点内存 View。
Snapshot 时 Ready凭据材料只保留在当前节点内存 View。Controller 内成功提交的
Upstream 启停、Routing 切换和配置发布会向本进程全部在线 Worker 快照流广播刷新;
定时刷新仍作为跨进程收敛与失效保护。
- **Gateway 粘性会话**:可为已认证请求启用 X-Proxy-Session 等配置 Header
Gateway 以 Client、Routing 和会话值派生本地有界绑定,成功建连后固定到同一
Proxy并在代理失效、快照移除或转发失败时自动重绑。原始会话值不会写入日志、
指标、PostgreSQL 或 Redis也不会向目标站点转发。
- **安全边界**Gateway、Distribution 与 Admin 使用各自的认证语义,并支持
CIDR、可信代理、严格请求解析和敏感信息最小化。
CIDR、可信代理、严格请求解析和敏感信息最小化Admin 的读写权限与
Distribution 提取权限可按命中凭据分别收敛。Distribution 凭据还可限制单次
提取数量、可访问 Upstream 与地区Gateway 凭据可限制可访问 Routing未配置时
保持既有全范围行为。Gateway 凭据还可在每个 Worker 内限制每分钟请求数,
同时限制 HTTP 请求和 CONNECT 隧道的并发数;热路径计数不写入 Redis 或
PostgreSQL。
## 架构概览
@ -85,21 +116,25 @@ flowchart LR
## 当前完成度
截至 **2026-08-02**,实施计划中可直接勾选的检查项为 **57 / 7477.0%**。详情见
截至 **2026-08-07**,实施计划中可直接勾选的检查项为 **71 / 7594.7%**。详情见
[实施计划](docs/development/implementation-plan.md)和
[交付完成度审计](docs/requirements/completion-audit.md)。
- **已完成**严格配置、Provider 获取与协调、Redis 活动池、Distribution 原子
提取与限流、Controller 的 Admin/Distribution/Metrics 监听,以及 PostgreSQL
管理状态WorkerControlPlane 的 Register、Snapshot ACK、Runtime 心跳接收和
管理状态与有界审计查询WorkerControlPlane 的 Register、Snapshot ACK、Runtime 心跳接收和
Redis 会话栅栏,以及 Gateway Outcome 上报的有界队列、序列确认与重试;
Controller 的 Redis 共享 BASIC/EGRESS/TARGET 检查任务、按上游的有界轮转调度、HTTP/HTTPS/SOCKS5
Checker 探测和
Observation 状态归并。
- **部分完成**Docker Compose/Kubernetes 运行时 mTLS Overlay。
- **待完成**CONNECT 长连接/Extract 压测场景、故障演练和代表性集群压测。
Observation 状态归并Provider 连续空结果与管理态禁用当前项触发的 Sequential CAS
自动推进、禁用候选过滤、末端路由停用和 Snapshot 即时刷新。
- **部分完成**Kubernetes 运行时 mTLS OverlayCompose 已具备本地的
Controller/Gateway/Checker mTLS 运行链路。
- **待完成**:故障演练和代表性集群压测;现有 HTTP、CONNECT 长连接和 Extract
场景只提供可复现的负载工具,不构成容量验证结论。
检查项数量不等于生产就绪度。静态部署清单与 protobuf descriptor 验证也不代表
检查项数量不等于生产就绪度。CI 会验证 OpenAPI 契约、固定版本的 protobuf descriptor/
生成代码漂移,以及 Compose/Kustomize 的静态渲染;这些门禁也不代表
端到端拓扑已经完成;`100,000 QPS` 仍只是待验证的集群设计目标。
## 快速开始
@ -124,8 +159,17 @@ Secret 分离管理。
```powershell
go run ./deploy/tools/configcheck deploy/config/local.yaml
./scripts/verify.ps1
./scripts/package-docs.ps1
./scripts/benchmark-gateway.ps1
```
`package-docs.ps1` 默认生成被忽略的 `dist/proxy-pool-docs-v1.0.zip`;包内包含 README、
`docs/`、图表、OpenAPI、Proto 契约和部署手册,并以 `manifest.json` 记录 Git revision、文件大小和
SHA-256。可使用 `-Version vMAJOR.MINOR[.PATCH]``-OutputPath OUTPUT.zip` 生成指定交付物。
`benchmark-gateway.ps1` 固定执行一次 100k 索引调度、Routing Round Robin 和 Snapshot Apply
微基准,并把版本与原始输出写入 `dist/gateway-benchmarks.txt`;它用于回归比较,不构成
网络转发或 100k QPS 集群容量证明。
Redis、PostgreSQL 和 Controller fixture 脚本会使用 Docker 启动隔离依赖:
```powershell
@ -134,6 +178,19 @@ Redis、PostgreSQL 和 Controller fixture 脚本会使用 Docker 启动隔离依
./scripts/test-controller.ps1
```
本地 Compose 还会启动 Controller、两个从开发证书派生身份的 Gateway 和一个 Checker。
首次启动前生成仅用于本机的 7 天 mTLS 证书;输出目录受 `.gitignore` 保护,脚本拒绝写入非空目录:
```powershell
./scripts/generate-local-controlplane-certs.ps1
docker compose -f deploy/docker-compose.yml up -d --build
docker compose -f deploy/docker-compose.yml ps
```
Gateway 只有取得有效 Snapshot 后才会通过 `/readyz`Checker 在首次成功领取控制面任务
批次后才会通过 `/readyz`,空批次也代表连接和身份验证已经成功。后续任务领取失败会立即
撤销 Checker 的就绪状态。
PostgreSQL 与 Redis 是 Controller 的启动依赖。当前可运行的 Controller 入口如下,
`CONFIG_FILE` 替换为实际配置路径,并确保其中的 PostgreSQL 与 Redis 地址可从
进程所在网络访问:
@ -147,7 +204,7 @@ go run ./cmd/proxy-controller -config CONFIG_FILE
改用宿主机可达的存储地址。
本地配置中的 `.invalid` Provider URL 是故障演示占位,不会提供真实代理。
Gateway 已提供启动命令;需要先启用 Controller `controlPlane` 并配置匹配的 mTLS
Gateway 也可独立启动;需要启用 Controller `controlPlane` 并配置匹配的 mTLS
证书(回环 fixture 可使用明文),再提供独立的拨号地址和 Worker 身份:
```powershell
@ -159,6 +216,10 @@ go run ./cmd/proxy-gateway -config CONFIG_FILE `
以上参数也可通过 `PROXY_POOL_CONTROL_PLANE_ADDRESS`、`PROXY_POOL_CLUSTER_ID`、
`PROXY_POOL_WORKER_ID`、`PROXY_POOL_INSTANCE_ID` 与 `PROXY_POOL_ZONE` 提供。
生产工作负载可设置 `PROXY_POOL_AUTO_IDENTITY=true`,省略 Worker 和 Instance ID
Gateway 会从挂载的 `gatewayTLS` 证书解析
`spiffe://<trust-domain>/<environment>/worker/<worker-id>`,并将 `<worker-id>`
作为默认实例身份。Controller 仍会将请求 ID 与证书 URI 严格比对。
Gateway 的 `/livez`、`/readyz`、`/metrics` 使用配置中的 `metrics.listen`;无有效
Snapshot 时 `/readyz` 返回 `503`。Checker 使用独立的逻辑/实例身份拉取有界任务:
@ -171,12 +232,17 @@ go run ./cmd/proxy-checker -config CONFIG_FILE `
Checker 的参数也可通过 `PROXY_POOL_CONTROL_PLANE_ADDRESS`
`PROXY_POOL_CHECKER_ID`、`PROXY_POOL_CHECKER_INSTANCE_ID` 与
`PROXY_POOL_CHECKER_MAX_IN_FLIGHT` 提供。它不会访问 Redis/PostgreSQL生产
`PROXY_POOL_CHECKER_MAX_IN_FLIGHT` 提供。`PROXY_POOL_AUTO_IDENTITY=true` 会从
`checkerTLS``.../checker/<checker-id>` URI 自动派生 Checker 与缺省实例身份。
它不会访问 Redis/PostgreSQL生产
Controller 在启用控制面时装配 Redis 共享任务 broker并按启用的 Upstream 调度
HTTP/HTTPS/SOCKS5 BASIC 检查、按每个 `check.urls` 创建 EGRESS 任务,并按启用 Routing 的
`check.targets` 创建 TARGET 任务。调度监督器每轮读取已发布配置,因此 reload 后的上游/路由启停、
检查间隔、抖动、超时、重试次数、`maxInFlight`、EGRESS URL 和 TARGET Profile 都会在下一轮生效;
BASIC、EGRESS 与 TARGET 以有界轮转组共享上游并发上限。新启用的上游无需重启 Controller。
`check.targets` 创建 TARGET 任务。调度监督器每轮读取已发布配置;启用 Admin 时只调度配置与
PostgreSQL 管理态同 revision 且均启用的 Upstream 和 Routing。管理态停用 Upstream 会在下一轮阻止新的
BASIC、EGRESS、TARGET 任务;管理态停用 Routing 则停止该 Routing 的新 TARGET 任务。revision 不一致或
状态不完整时按失败关闭。因此 reload 后的上游/路由启停、检查间隔、
抖动、超时、重试次数、`maxInFlight`、EGRESS URL 和 TARGET Profile 都会在下一轮生效BASIC、
EGRESS 与 TARGET 以有界轮转组共享上游并发上限。新启用的上游无需重启 Controller。
EGRESS 对成功响应提取纯文本 IP 或常见 JSON IP 字段并将其作为全局健康事实回传TARGET 事实
仅归并到对应的 `(routing_name, target_url)` Profile不改变 Proxy 全局健康。
@ -225,9 +291,27 @@ go run ./cmd/proxy-loadgen `
```
命令输出 JSON 报告,包含成功/失败分类、`TunnelsEstablished`、
`ExtractResponsesValidated`、`ExtractReturned`、`ExtractValidationFailures`、固定内存的
`ExtractResponsesValidated`、`ExtractReturned`、`ExtractValidationFailures`、`RateStartsGenerated`、
`RateStartsDropped`、固定内存的
连接握手延迟分位上界、吞吐和 Go 运行时内存/GC 快照。它不会输出提取响应中的地址或
凭据,也不构成 100,000 QPS 证明。
凭据,也不构成 100,000 QPS 证明。限速场景以 `Requests` 表示实际发起数;当
`RateStartsDropped` 非零时,目标速率受压测端并发容量或时间窗限制,报告吞吐不得
标注为已达到配置的 `-rate`。每个生成的限速令牌均会计入实际发起或丢弃,因此
`RateStartsGenerated = Requests + RateStartsDropped`
需要保留单次容量证据时,使用 `-output REPORT_FILE` 同时写出 JSON 文件;文件在
同目录完整写入后才替换目标,标准输出仍保留相同报告,便于交给日志或指标系统。
`-max-error-rate` 接受 `0``1``-max-p99` 接受正的 Go duration任一阈值违反时
报告会附带 `acceptance` 结果,命令仍写出标准输出和 `-output` 文件,然后以退出码 `3`
结束,适合在 CI 中保留失败压测证据:
```powershell
go run ./cmd/proxy-loadgen `
-target https://TARGET_URL/health `
-proxy http://GATEWAY_HOST:8080 `
-duration 30s -rate 5000 -concurrency 128 `
-max-error-rate 0.01 -max-p99 500ms `
-output REPORT_FILE
```
## 关键配置与入口
@ -263,7 +347,10 @@ go run ./cmd/proxy-loadgen `
回收会自动为已分配项发起带健康/归属栅栏的 Drain待 Snapshot ACK 与运行态归零后再删除;
TARGET 事实按路由目标 Profile 独立归并。
- **P1 - Gateway 与 Routing**Gateway 进程、快照凭据分发、五种 Routing 策略与
`onUnavailable` 已接入;动态容量调整和配置停用触发的 Drain 编排待完成。
`onUnavailable` 已接入;上游停用会从后续完整 Snapshot 排除,并对现有 Worker
ownership 发起带策略 revision 栅栏的 Drain。Routing 切换立即刷新在线 Worker 的
完整快照Sequential 的新分配切到新上游,旧 Proxy 与既有连接自然排空。动态容量
调整仍待完成。
- **P1 - 可观测与部署**:低基数业务指标、完整 Compose/Kubernetes 进程拓扑,
以及故障转移和恢复演练。
- **P2 - 容量证明**`proxy-loadgen`、HTTP/CONNECT/Extract 分场景压测,以及

View File

@ -2,14 +2,23 @@ openapi: 3.1.0
info:
title: Proxy Pool Admin API
version: 1.0.0
description: 运维状态与受控变更接口。该入口必须与 Distribution 分端口和权限。
description: |
运维状态与受控变更接口。该入口必须与 Distribution 分端口和权限。
已认证凭据可配置 `admin:read`Status、审计和 `admin:write`(变更);
未配置权限的旧凭据保留全权限行为。
servers:
- url: http://127.0.0.1:8082
tags:
- name: Status
description: 不含代理明细与凭据的控制面聚合状态。
- name: Audit
description: 权威管理状态变更的游标审计查询。
- name: Upstreams
description: 上游的启停与受控状态变更。
- name: Routing
description: Routing 当前上游的受控切换。
- name: Configuration
description: 经校验后原子发布的配置重载。
security:
- AdminApiKey: []
- BasicAuth: []
@ -21,6 +30,7 @@ paths:
tags: [Status]
operationId: getStatus
summary: 获取控制面摘要状态
description: 需要 `admin:read`。
responses:
'200':
description: 不含 Proxy 地址、Client 标识或 Secret 的聚合状态
@ -36,11 +46,50 @@ paths:
'405': {$ref: '#/components/responses/MethodNotAllowed'}
'500': {$ref: '#/components/responses/InternalServerError'}
'503': {$ref: '#/components/responses/ServiceUnavailable'}
/api/v1/audit:
get:
tags: [Audit]
operationId: listAuditRecords
summary: 按 ID 游标读取管理面审计记录
description: |
需要 `admin:read`。
仅返回权威管理面的变更审计记录,按 `id` 升序排列。`afterId` 是排他游标:
后续页面只包含 `id` 大于该值的记录。未传 `limit` 时返回 100 条,单页最多
1000 条。
审计接口不返回 Proxy、Upstream URL、凭据或活动池内容。
parameters:
- name: afterId
in: query
required: false
description: 排他游标;仅返回 ID 大于此值的记录。
schema: {type: integer, minimum: 0, default: 0}
- name: limit
in: query
required: false
description: 单页记录数;默认 100最大 1000。
schema: {type: integer, minimum: 1, maximum: 1000, default: 100}
responses:
'200':
description: 管理面审计记录页
headers:
X-Request-ID: {$ref: '#/components/headers/RequestID'}
content:
application/json:
schema: {$ref: '#/components/schemas/AuditPage'}
'401': {$ref: '#/components/responses/Unauthorized'}
'403': {$ref: '#/components/responses/Forbidden'}
'400': {$ref: '#/components/responses/BadRequest'}
'405': {$ref: '#/components/responses/MethodNotAllowed'}
'500': {$ref: '#/components/responses/InternalServerError'}
'503': {$ref: '#/components/responses/ServiceUnavailable'}
/api/v1/upstreams/{name}/enable:
post:
tags: [Upstreams]
operationId: enableUpstream
summary: 启用 Upstream
description: 需要 `admin:write`。
parameters:
- $ref: '#/components/parameters/UpstreamName'
- $ref: '#/components/parameters/RequestID'
@ -59,6 +108,7 @@ paths:
tags: [Upstreams]
operationId: disableUpstream
summary: 禁用 Upstream 并使已有资源自然 Drain
description: 需要 `admin:write`。
parameters:
- $ref: '#/components/parameters/UpstreamName'
- $ref: '#/components/parameters/RequestID'
@ -77,6 +127,7 @@ paths:
tags: [Routing]
operationId: switchRouting
summary: 原子切换 Sequential Routing 当前 Upstream
description: 需要 `admin:write`。
parameters:
- name: name
in: path
@ -113,6 +164,7 @@ paths:
tags: [Configuration]
operationId: reloadConfiguration
summary: 严格校验并原子发布新配置快照
description: 需要 `admin:write`。
parameters:
- $ref: '#/components/parameters/RequestID'
responses:
@ -152,6 +204,31 @@ components:
description: 服务端最终使用的请求标识。
schema: {type: string, maxLength: 128}
schemas:
AuditPage:
type: object
additionalProperties: false
required: [records]
properties:
records:
type: array
description: 按 ID 升序的审计记录;没有更多记录时为空数组。
items: {$ref: '#/components/schemas/AuditRecord'}
AuditRecord:
type: object
additionalProperties: false
required: [id, requestId, actorId, action, resourceType, resourceName, changed, version, occurredAt]
properties:
id: {type: integer, minimum: 0}
requestId: {type: string}
actorId: {type: string}
sourceIp: {type: string}
action: {type: string}
resourceType: {type: string}
resourceName: {type: string}
changed: {type: boolean}
version: {type: integer, minimum: 0}
reason: {type: string}
occurredAt: {type: string, format: date-time}
Status:
type: object
additionalProperties: false

View File

@ -10,6 +10,7 @@ import (
type document struct {
OpenAPI string `yaml:"openapi"`
Paths map[string]map[string]any `yaml:"paths"`
Components map[string]any `yaml:"components"`
}
func TestDistributionContract(t *testing.T) {
@ -37,6 +38,7 @@ func TestAdminContract(t *testing.T) {
spec := readDocument(t, "admin.yaml")
for _, path := range []string{
"/api/v1/status",
"/api/v1/audit",
"/api/v1/upstreams/{name}/enable",
"/api/v1/upstreams/{name}/disable",
"/api/v1/routing/{name}/switch",
@ -48,6 +50,67 @@ func TestAdminContract(t *testing.T) {
}
requireResponses(t, spec.Paths["/api/v1/routing/{name}/switch"]["post"],
"200", "400", "401", "403", "404", "405", "409", "413", "415", "422", "500", "503")
requireResponses(t, spec.Paths["/api/v1/audit"]["get"],
"200", "400", "401", "403", "405", "500", "503")
requireAuditPaginationContract(t, spec.Paths["/api/v1/audit"]["get"])
requireAuditSchemas(t, spec)
}
func requireAuditPaginationContract(t *testing.T, operation any) {
t.Helper()
operationMap, ok := operation.(map[string]any)
if !ok {
t.Fatalf("audit operation has type %T, want map", operation)
}
parameters, ok := operationMap["parameters"].([]any)
if !ok {
t.Fatalf("audit parameters has type %T, want array", operationMap["parameters"])
}
afterID := parameterSchema(t, parameters, "afterId")
if afterID["type"] != "integer" || afterID["minimum"] != 0 || afterID["default"] != 0 {
t.Errorf("afterId schema = %#v, want integer with minimum/default 0", afterID)
}
limit := parameterSchema(t, parameters, "limit")
if limit["type"] != "integer" || limit["minimum"] != 1 || limit["maximum"] != 1000 || limit["default"] != 100 {
t.Errorf("limit schema = %#v, want integer [1, 1000] with default 100", limit)
}
}
func parameterSchema(t *testing.T, parameters []any, name string) map[string]any {
t.Helper()
for _, raw := range parameters {
parameter, ok := raw.(map[string]any)
if !ok || parameter["name"] != name || parameter["in"] != "query" {
continue
}
schema, ok := parameter["schema"].(map[string]any)
if !ok {
t.Fatalf("%s parameter schema has type %T, want map", name, parameter["schema"])
}
return schema
}
t.Fatalf("query parameter %q is missing", name)
return nil
}
func requireAuditSchemas(t *testing.T, spec document) {
t.Helper()
schemas, ok := spec.Components["schemas"].(map[string]any)
if !ok {
t.Fatal("components.schemas is missing")
}
for _, name := range []string{"AuditPage", "AuditRecord"} {
if _, ok := schemas[name].(map[string]any); !ok {
t.Errorf("schema %s is missing", name)
}
}
record, _ := schemas["AuditRecord"].(map[string]any)
properties, _ := record["properties"].(map[string]any)
for _, forbidden := range []string{"proxy", "upstreamUrl", "credential", "activePool"} {
if _, exists := properties[forbidden]; exists {
t.Errorf("AuditRecord must not expose %q", forbidden)
}
}
}
func requireResponses(t *testing.T, operation any, codes ...string) {

View File

@ -7,13 +7,18 @@ info:
Redis operation atomically removes or marks every returned proxy as
extracted in the TTL activity pool. Proxy details and per-extraction
records are not persisted in PostgreSQL. There is no release, renew, or
lease API.
lease API. An authenticated credential may be limited to the fixed
`distribution:extract` permission; legacy credentials without configured
permissions retain full access. Credential client policy may further bound
a request's count, upstreams, and regions before Redis extraction.
servers:
- url: http://127.0.0.1:8081
description: Distribution API
tags:
- name: Distribution
description: 一次性独占提取上游代理的受保护接口。
- name: Health
description: Distribution 监听器的存活与就绪探针。
paths:
/api/v1/proxies/extract:
post:
@ -21,6 +26,8 @@ paths:
operationId: extractProxies
summary: 一次性独占提取代理
description: |
需要 `distribution:extract`。
服务端通过单个 Redis 原子操作完成筛选、TTL/所有权复核、Gateway 预留、
从可分配池移除所选条目和短期幂等结果写入,再返回代理。相同代理不会
返回给两个成功请求,提取路径不访问 PostgreSQL。
@ -28,7 +35,8 @@ paths:
`partial` 允许实际返回数量小于请求数量;`allOrNothing` 数量不足时不
提取任何代理并返回 409。未传 `fulfillment` 时使用服务端配置,默认
为 `partial`。`Idempotency-Key` 在 Redis 的有界 TTL 窗口内避免客户端因
响应丢失重试而再次消耗库存。
响应丢失重试而再次消耗库存。命中凭据的 Client policy 可以额外限制
count、allowedUpstreams 和 regions越过该边界返回 403未调用 Redis。
security:
- ApiKeyAuth: []
- BasicAuth: []

5
api/openapi/redocly.yaml Normal file
View File

@ -0,0 +1,5 @@
extends:
- minimal
rules:
tag-description: error

View File

@ -26,6 +26,31 @@ func TestOpenAPIDocumentsHaveClosedContracts(t *testing.T) {
}
}
func TestOpenAPITagsHaveDescriptions(t *testing.T) {
for _, name := range []string{"proxy-pool.yaml", "admin.yaml"} {
name := name
t.Run(name, func(t *testing.T) {
root := readOpenAPIRoot(t, name)
tags, ok := root["tags"].([]any)
if !ok || len(tags) == 0 {
t.Fatal("tags must be a non-empty array")
}
for _, rawTag := range tags {
tag, ok := rawTag.(map[string]any)
if !ok {
t.Errorf("tag has type %T, want object", rawTag)
continue
}
name, _ := tag["name"].(string)
description, _ := tag["description"].(string)
if strings.TrimSpace(description) == "" {
t.Errorf("tag %q has no description", name)
}
}
})
}
}
func validateLocalReferences(t *testing.T, root map[string]any) {
t.Helper()
walkOpenAPI(root, func(path string, value any) {

View File

@ -99,6 +99,7 @@ message RoutingRule {
RoutingStrategy strategy = 8;
UnavailableAction on_unavailable = 9;
google.protobuf.Duration wait_timeout = 10;
RoutingAction action = 11;
}
message RoutingStrategy {
@ -123,6 +124,13 @@ enum UnavailableAction {
UNAVAILABLE_ACTION_DIRECT = 3;
}
enum RoutingAction {
ROUTING_ACTION_UNSPECIFIED = 0;
ROUTING_ACTION_PROXY = 1;
ROUTING_ACTION_DIRECT = 2;
ROUTING_ACTION_REJECT = 3;
}
message OwnedProxy {
string id = 1;
string upstream = 2;

View File

@ -15,6 +15,7 @@ import (
controlplanev1 "proxy-pool/gen/controlplane/v1"
"proxy-pool/internal/checker/bootstrap"
"proxy-pool/internal/config"
"proxy-pool/internal/platform/logging"
)
const (
@ -22,6 +23,7 @@ const (
controlPlaneAddressEnvironment = "PROXY_POOL_CONTROL_PLANE_ADDRESS"
checkerIDEnvironment = "PROXY_POOL_CHECKER_ID"
instanceIDEnvironment = "PROXY_POOL_CHECKER_INSTANCE_ID"
autoIdentityEnvironment = "PROXY_POOL_AUTO_IDENTITY"
maxInFlightEnvironment = "PROXY_POOL_CHECKER_MAX_IN_FLIGHT"
)
@ -41,6 +43,7 @@ func execute(ctx context.Context, args []string, getenv environmentLookup, run c
controlPlaneAddress := flags.String("control-plane", "", "remote Controller control-plane address")
checkerID := flags.String("checker-id", "", "unique Checker identifier")
instanceID := flags.String("instance-id", "", "unique Checker process instance identifier")
autoIdentity := flags.Bool("auto-identity", false, "derive Checker and missing instance IDs from the mTLS SPIFFE certificate")
maxInFlight := flags.Int("max-in-flight", 0, "maximum concurrent tasks")
levels := flags.String("levels", "basic,egress,target", "supported levels: basic,egress,target")
if err := flags.Parse(args); err != nil {
@ -63,28 +66,45 @@ func execute(ctx context.Context, args []string, getenv environmentLookup, run c
*maxInFlight = value
}
}
if !flagWasSet(flags, "auto-identity") {
if value, err := strconv.ParseBool(getenv(autoIdentityEnvironment)); err == nil {
*autoIdentity = value
}
}
}
supportedLevels, err := parseLevels(*levels)
missingIdentity := !*autoIdentity && (!validValue(*checkerID) || !validValue(*instanceID))
invalidProvidedIdentity := (*checkerID != "" && !validValue(*checkerID)) || (*instanceID != "" && !validValue(*instanceID))
if ctx == nil || run == nil || err != nil || !validValue(*configPath) || !validValue(*controlPlaneAddress) ||
!validValue(*checkerID) || !validValue(*instanceID) || *maxInFlight <= 0 {
missingIdentity || invalidProvidedIdentity || *maxInFlight <= 0 {
_, _ = fmt.Fprintf(stderr,
"proxy-checker: -config, -control-plane, -checker-id, -instance-id and positive -max-in-flight are required; "+
"environment fallbacks: %s, %s, %s, %s, %s\n",
configEnvironment, controlPlaneAddressEnvironment, checkerIDEnvironment, instanceIDEnvironment, maxInFlightEnvironment,
"proxy-checker: -config, -control-plane and positive -max-in-flight are required; -checker-id/-instance-id are required unless -auto-identity is enabled; "+
"environment fallbacks: %s, %s, %s, %s, %s, %s\n",
configEnvironment, controlPlaneAddressEnvironment, checkerIDEnvironment, instanceIDEnvironment, maxInFlightEnvironment, autoIdentityEnvironment,
)
return 2
}
err = run(ctx, bootstrap.Options{
ConfigPath: *configPath, Resolver: config.OSResolver{}, ControlPlaneAddress: *controlPlaneAddress,
CheckerID: *checkerID, InstanceID: *instanceID, MaxInFlight: *maxInFlight, SupportedLevels: supportedLevels,
CheckerID: *checkerID, InstanceID: *instanceID, AutoIdentity: *autoIdentity, MaxInFlight: *maxInFlight, SupportedLevels: supportedLevels,
})
if err == nil || (errors.Is(err, context.Canceled) && ctx.Err() != nil) {
return 0
}
_, _ = fmt.Fprintf(stderr, "proxy-checker: %v\n", err)
logging.WriteProcessError(stderr, "proxy-checker", err)
return 1
}
func flagWasSet(flags *flag.FlagSet, name string) bool {
set := false
flags.Visit(func(item *flag.Flag) {
if item.Name == name {
set = true
}
})
return set
}
func parseLevels(value string) ([]controlplanev1.CheckLevel, error) {
levels := make([]controlplanev1.CheckLevel, 0, 3)
seen := make(map[controlplanev1.CheckLevel]struct{})

View File

@ -24,6 +24,21 @@ func TestExecuteUsesFlagsAndPassesCheckerIdentity(t *testing.T) {
}
}
func TestExecutePassesAutoCheckerIdentityFromEnvironment(t *testing.T) {
received := bootstrap.Options{}
values := map[string]string{
configEnvironment: "checker.yaml", controlPlaneAddressEnvironment: "127.0.0.1:8443",
maxInFlightEnvironment: "2", autoIdentityEnvironment: "true",
}
code := execute(context.Background(), nil, func(name string) string { return values[name] }, func(_ context.Context, options bootstrap.Options) error {
received = options
return nil
}, io.Discard)
if code != 0 || !received.AutoIdentity || received.CheckerID != "" || received.InstanceID != "" {
t.Fatalf("execute(auto identity) = (%d, %+v)", code, received)
}
}
func TestExecuteRejectsInvalidLevelSet(t *testing.T) {
code := execute(context.Background(), []string{
"-config", "config.yaml", "-control-plane", "127.0.0.1:8443", "-checker-id", "checker-a", "-instance-id", "instance-a",

View File

@ -13,6 +13,7 @@ import (
"proxy-pool/internal/config"
"proxy-pool/internal/controller/bootstrap"
"proxy-pool/internal/platform/logging"
)
const (
@ -67,6 +68,6 @@ func execute(
if err == nil || (errors.Is(err, context.Canceled) && ctx.Err() != nil) {
return 0
}
_, _ = fmt.Fprintf(stderr, "proxy-controller: %v\n", err)
logging.WriteProcessError(stderr, "proxy-controller", err)
return 1
}

View File

@ -75,7 +75,9 @@ func TestExecuteMapsStartupFailureAndSignalCancellation(t *testing.T) {
) error {
return want
}, &stderr)
if code != 1 || !bytes.Contains(stderr.Bytes(), []byte(want.Error())) {
if code != 1 || !bytes.Contains(stderr.Bytes(), []byte(`"msg":"process failed"`)) ||
!bytes.Contains(stderr.Bytes(), []byte(`"component":"proxy-controller"`)) ||
bytes.Contains(stderr.Bytes(), []byte(want.Error())) {
t.Fatalf("execute(startup failure) = %d, stderr = %q", code, stderr.String())
}

View File

@ -8,11 +8,13 @@ import (
"io"
"os"
"os/signal"
"strconv"
"strings"
"syscall"
"proxy-pool/internal/config"
"proxy-pool/internal/gateway/bootstrap"
"proxy-pool/internal/platform/logging"
)
const (
@ -21,6 +23,7 @@ const (
clusterIDEnvironment = "PROXY_POOL_CLUSTER_ID"
workerIDEnvironment = "PROXY_POOL_WORKER_ID"
instanceIDEnvironment = "PROXY_POOL_INSTANCE_ID"
autoIdentityEnvironment = "PROXY_POOL_AUTO_IDENTITY"
zoneEnvironment = "PROXY_POOL_ZONE"
)
@ -47,6 +50,7 @@ func execute(
clusterID := flags.String("cluster-id", "", "cluster identifier")
workerID := flags.String("worker-id", "", "unique Worker identifier")
instanceID := flags.String("instance-id", "", "unique process instance identifier")
autoIdentity := flags.Bool("auto-identity", false, "derive Worker and missing instance IDs from the mTLS SPIFFE certificate")
zone := flags.String("zone", "", "availability zone identifier")
if err := flags.Parse(args); err != nil {
if errors.Is(err, flag.ErrHelp) {
@ -65,27 +69,44 @@ func execute(
setIfEmpty(workerID, getenv(workerIDEnvironment))
setIfEmpty(instanceID, getenv(instanceIDEnvironment))
setIfEmpty(zone, getenv(zoneEnvironment))
if !flagWasSet(flags, "auto-identity") {
if value, err := strconv.ParseBool(getenv(autoIdentityEnvironment)); err == nil {
*autoIdentity = value
}
}
}
missingIdentity := !*autoIdentity && (!validValue(*workerID) || !validValue(*instanceID))
invalidProvidedIdentity := (*workerID != "" && !validValue(*workerID)) || (*instanceID != "" && !validValue(*instanceID))
if ctx == nil || run == nil || !validValue(*configPath) || !validValue(*controlPlaneAddress) ||
!validValue(*clusterID) || !validValue(*workerID) || !validValue(*instanceID) || !validValue(*zone) {
!validValue(*clusterID) || missingIdentity || invalidProvidedIdentity || !validValue(*zone) {
_, _ = fmt.Fprintf(stderr,
"proxy-gateway: -config, -control-plane, -cluster-id, -worker-id, -instance-id and -zone are required; "+
"environment fallbacks: %s, %s, %s, %s, %s, %s\n",
configEnvironment, controlPlaneAddressEnvironment, clusterIDEnvironment, workerIDEnvironment, instanceIDEnvironment, zoneEnvironment,
"proxy-gateway: -config, -control-plane, -cluster-id and -zone are required; -worker-id/-instance-id are required unless -auto-identity is enabled; "+
"environment fallbacks: %s, %s, %s, %s, %s, %s, %s\n",
configEnvironment, controlPlaneAddressEnvironment, clusterIDEnvironment, workerIDEnvironment, instanceIDEnvironment, zoneEnvironment, autoIdentityEnvironment,
)
return 2
}
err := run(ctx, bootstrap.Options{
ConfigPath: *configPath, Resolver: config.OSResolver{}, ControlPlaneAddress: *controlPlaneAddress,
ClusterID: *clusterID, WorkerID: *workerID, InstanceID: *instanceID, Zone: *zone,
ClusterID: *clusterID, WorkerID: *workerID, InstanceID: *instanceID, AutoIdentity: *autoIdentity, Zone: *zone,
})
if err == nil || (errors.Is(err, context.Canceled) && ctx.Err() != nil) {
return 0
}
_, _ = fmt.Fprintf(stderr, "proxy-gateway: %v\n", err)
logging.WriteProcessError(stderr, "proxy-gateway", err)
return 1
}
func flagWasSet(flags *flag.FlagSet, name string) bool {
set := false
flags.Visit(func(item *flag.Flag) {
if item.Name == name {
set = true
}
})
return set
}
func setIfEmpty(target *string, value string) {
if target != nil && *target == "" {
*target = value

View File

@ -55,6 +55,21 @@ func TestExecuteReadsGatewayOptionsFromEnvironment(t *testing.T) {
}
}
func TestExecutePassesAutoGatewayIdentityFromEnvironment(t *testing.T) {
captured := bootstrap.Options{}
values := map[string]string{
configEnvironment: "gateway.yaml", controlPlaneAddressEnvironment: "127.0.0.1:8443",
clusterIDEnvironment: "cluster-a", zoneEnvironment: "zone-a", autoIdentityEnvironment: "true",
}
code := execute(context.Background(), nil, func(name string) string { return values[name] }, func(_ context.Context, options bootstrap.Options) error {
captured = options
return nil
}, io.Discard)
if code != 0 || !captured.AutoIdentity || captured.WorkerID != "" || captured.InstanceID != "" {
t.Fatalf("execute(auto identity) = (%d, %+v)", code, captured)
}
}
func TestExecuteRejectsIncompleteGatewayOptions(t *testing.T) {
t.Parallel()

View File

@ -7,9 +7,11 @@ import (
"flag"
"fmt"
"io"
"math"
"net/http"
"os"
"os/signal"
"path/filepath"
"strings"
"syscall"
"time"
@ -41,6 +43,9 @@ func execute(ctx context.Context, args []string, run loadRun, stdout, stderr io.
hold := flags.Duration("hold", 0, "CONNECT tunnel hold duration; required by -scenario connect")
extractCount := flags.Int("extract-count", 1, "proxies requested by each extract scenario request")
extractFulfillment := flags.String("extract-fulfillment", "partial", "extract fulfillment: partial or allOrNothing")
maxErrorRate := flags.Float64("max-error-rate", -1, "optional maximum failed/completed rate from 0 to 1")
maxP99 := flags.Duration("max-p99", 0, "optional maximum p99 latency")
outputPath := flags.String("output", "", "optional JSON report file; replaces only after a complete write")
var headers headerValues
flags.Var(&headers, "header", "repeatable HTTP header in Name: Value form")
if err := flags.Parse(args); err != nil {
@ -53,6 +58,15 @@ func execute(ctx context.Context, args []string, run loadRun, stdout, stderr io.
_, _ = fmt.Fprintln(stderr, "proxy-loadgen: target and a bounded workload are required")
return 2
}
if strings.TrimSpace(*outputPath) != *outputPath {
_, _ = fmt.Fprintln(stderr, "proxy-loadgen: invalid output path")
return 2
}
criteria, configured, err := acceptanceCriteria(*maxErrorRate, *maxP99)
if err != nil {
_, _ = fmt.Fprintf(stderr, "proxy-loadgen: %v\n", err)
return 2
}
parsedHeaders, err := headers.Header()
if err != nil {
_, _ = fmt.Fprintf(stderr, "proxy-loadgen: %v\n", err)
@ -72,13 +86,80 @@ func execute(ctx context.Context, args []string, run loadRun, stdout, stderr io.
_, _ = fmt.Fprintf(stderr, "proxy-loadgen: %v\n", err)
return 1
}
if configured {
acceptance, acceptanceErr := loadgen.EvaluateAcceptance(report, criteria)
if acceptanceErr != nil {
_, _ = fmt.Fprintf(stderr, "proxy-loadgen: %v\n", acceptanceErr)
return 2
}
report.Acceptance = &acceptance
}
if err := writeReportFile(*outputPath, report); err != nil {
_, _ = fmt.Fprintf(stderr, "proxy-loadgen: write report: %v\n", err)
return 1
}
if err := json.NewEncoder(stdout).Encode(report); err != nil {
_, _ = fmt.Fprintf(stderr, "proxy-loadgen: encode report: %v\n", err)
return 1
}
if report.Acceptance != nil && !report.Acceptance.Passed {
_, _ = fmt.Fprintln(stderr, "proxy-loadgen: workload acceptance thresholds were not met")
return 3
}
return 0
}
func acceptanceCriteria(maxErrorRate float64, maxP99 time.Duration) (loadgen.AcceptanceCriteria, bool, error) {
if math.IsNaN(maxErrorRate) || math.IsInf(maxErrorRate, 0) || maxErrorRate < -1 || maxErrorRate > 1 || maxP99 < 0 {
return loadgen.AcceptanceCriteria{}, false, loadgen.ErrInvalidAcceptanceCriteria
}
criteria := loadgen.AcceptanceCriteria{
EnforceFailureRate: maxErrorRate >= 0,
MaxFailureRate: maxErrorRate,
EnforceP99Latency: maxP99 > 0,
MaxP99Latency: maxP99,
}
return criteria, criteria.EnforceFailureRate || criteria.EnforceP99Latency, nil
}
func writeReportFile(path string, report loadgen.Report) error {
if path == "" {
return nil
}
content, err := json.Marshal(report)
if err != nil {
return err
}
content = append(content, '\n')
directory := filepath.Dir(path)
temporary, err := os.CreateTemp(directory, "."+filepath.Base(path)+".tmp-*")
if err != nil {
return err
}
temporaryPath := temporary.Name()
defer os.Remove(temporaryPath)
if err := temporary.Chmod(0o600); err != nil {
_ = temporary.Close()
return err
}
if _, err := temporary.Write(content); err != nil {
_ = temporary.Close()
return err
}
if err := temporary.Close(); err != nil {
return err
}
// Prefer same-directory rename so a completed report replaces the old one
// atomically on platforms that support replacement.
if err := os.Rename(temporaryPath, path); err == nil {
return nil
}
if err := os.Remove(path); err != nil && !os.IsNotExist(err) {
return err
}
return os.Rename(temporaryPath, path)
}
type headerValues []string
func (values *headerValues) String() string {

View File

@ -4,6 +4,9 @@ import (
"bytes"
"context"
"encoding/json"
"math"
"os"
"path/filepath"
"testing"
"time"
@ -18,7 +21,10 @@ func TestExecutePassesBoundedWorkloadAndWritesJSON(t *testing.T) {
"-requests", "3", "-concurrency", "2", "-timeout", "2s", "-header", "X-Run: fixed", "-body", `{"count":1}`,
}, func(_ context.Context, options loadgen.Options) (loadgen.Report, error) {
received = options
return loadgen.Report{Requests: 3, Completed: 3, Succeeded: 3, Duration: time.Second}, nil
return loadgen.Report{
Requests: 3, Completed: 3, Succeeded: 3, Duration: time.Second,
RateStartsGenerated: 3, RateStartsDropped: 1,
}, nil
}, &stdout, &stderr)
if code != 0 || received.TargetURL != "https://target.example/path" || received.ProxyURL != "http://gateway.example:8080" ||
received.Method != "post" || received.Requests != 3 || received.Duration != 0 || received.Concurrency != 2 ||
@ -26,7 +32,8 @@ func TestExecutePassesBoundedWorkloadAndWritesJSON(t *testing.T) {
t.Fatalf("execute() = %d; options=%+v stderr=%q", code, received, stderr.String())
}
var report loadgen.Report
if err := json.Unmarshal(stdout.Bytes(), &report); err != nil || report.Succeeded != 3 {
if err := json.Unmarshal(stdout.Bytes(), &report); err != nil || report.Succeeded != 3 ||
report.RateStartsGenerated != 3 || report.RateStartsDropped != 1 {
t.Fatalf("JSON report = (%+v, %v)", report, err)
}
}
@ -70,3 +77,85 @@ func TestExecutePassesExtractScenario(t *testing.T) {
t.Fatalf("execute() = %d; options=%+v stderr=%q", code, received, stderr.String())
}
}
func TestExecuteWritesReportFile(t *testing.T) {
t.Parallel()
path := filepath.Join(t.TempDir(), "load-report.json")
if err := os.WriteFile(path, []byte("incomplete"), 0o600); err != nil {
t.Fatalf("WriteFile(): %v", err)
}
var stdout, stderr bytes.Buffer
code := execute(context.Background(), []string{
"-target", "https://target.example/health", "-requests", "1", "-concurrency", "1", "-output", path,
}, func(_ context.Context, _ loadgen.Options) (loadgen.Report, error) {
return loadgen.Report{Requests: 1, Completed: 1, Succeeded: 1, Duration: time.Second}, nil
}, &stdout, &stderr)
if code != 0 || stderr.Len() != 0 {
t.Fatalf("execute() = %d, stderr=%q", code, stderr.String())
}
content, err := os.ReadFile(path)
if err != nil {
t.Fatalf("ReadFile(): %v", err)
}
var report loadgen.Report
if err := json.Unmarshal(content, &report); err != nil || report.Succeeded != 1 || string(content) == "incomplete" {
t.Fatalf("report file = (%q, %+v, %v)", content, report, err)
}
}
func TestExecuteWritesReportBeforeReturningAcceptanceFailure(t *testing.T) {
path := filepath.Join(t.TempDir(), "load-report.json")
var stdout, stderr bytes.Buffer
code := execute(context.Background(), []string{
"-target", "https://target.example/health", "-requests", "10", "-concurrency", "1",
"-max-error-rate", "0.1", "-max-p99", "100ms", "-output", path,
}, func(_ context.Context, _ loadgen.Options) (loadgen.Report, error) {
return loadgen.Report{
Requests: 10, Completed: 10, Failed: 2, Duration: time.Second,
Latency: loadgen.LatencyReport{Samples: 10, P99UpperBound: 250 * time.Millisecond},
}, nil
}, &stdout, &stderr)
if code != 3 || stderr.Len() == 0 {
t.Fatalf("execute() = %d, stderr=%q", code, stderr.String())
}
for _, content := range [][]byte{stdout.Bytes(), mustReadFile(t, path)} {
var report loadgen.Report
if err := json.Unmarshal(content, &report); err != nil || report.Acceptance == nil || report.Acceptance.Passed ||
len(report.Acceptance.Violations) != 2 {
t.Fatalf("acceptance report = (%+v, %v)", report, err)
}
}
}
func TestExecuteRejectsInvalidAcceptanceThresholdBeforeRunningWorkload(t *testing.T) {
var stdout, stderr bytes.Buffer
called := false
code := execute(context.Background(), []string{
"-target", "https://target.example/health", "-requests", "1", "-max-error-rate", "1.1",
}, func(_ context.Context, _ loadgen.Options) (loadgen.Report, error) {
called = true
return loadgen.Report{}, nil
}, &stdout, &stderr)
if code != 2 || called || stdout.Len() != 0 || stderr.Len() == 0 {
t.Fatalf("execute() = %d, called=%t stdout=%q stderr=%q", code, called, stdout.String(), stderr.String())
}
}
func TestAcceptanceCriteriaRejectsNonFiniteErrorRate(t *testing.T) {
for _, value := range []float64{math.NaN(), math.Inf(1)} {
if _, _, err := acceptanceCriteria(value, 0); err == nil {
t.Fatalf("acceptanceCriteria(%v) accepted a non-finite error rate", value)
}
}
}
func mustReadFile(t *testing.T, path string) []byte {
t.Helper()
content, err := os.ReadFile(path)
if err != nil {
t.Fatalf("ReadFile(%q): %v", path, err)
}
return content
}

View File

@ -5,10 +5,24 @@ Grafana 与 Kubernetes。镜像会构建 `proxy-controller`、`proxy-gateway`
`proxy-checker`Controller 已装配 Redis BASIC 检查任务Checker 可通过认证
控制面执行 HTTP/HTTPS BASIC 探测。
默认 Compose 和 Kubernetes Base 仍将 `controlPlane.enabled` 保持为 `false`
跨节点控制面必须由环境 Overlay 提供 Controller 可拨号地址、独立 Worker/Checker
身份及 mTLS 证书,因此这些默认清单不能视为完整的生产发布配置。`checker.yaml`
保留为 Overlay 模板,尚未加入 Base。
Compose 使用 `config/local.yaml` 启用 Controller、两个 Gateway 和一个 Checker 的
mTLS 控制面。`generate-local-controlplane-certs.ps1` 为三个固定工作负载生成不同的
SPIFFE URI 证书,私钥只写入被忽略的 `deploy/.control-plane-tls/`。运行前先执行:
```powershell
./scripts/generate-local-controlplane-certs.ps1
docker compose -f deploy/docker-compose.yml up -d --build
```
Kubernetes Base 仍保持 `controlPlane.enabled: false`。`kubernetes/overlays/development-mtls`
启用单副本 Controller/Gateway/Checker mTLS 拓扑,挂载三组外部 TLS Secret并从
`gateway-a`、`checker-a` 的证书 URI 自动派生身份。具体创建 Secret 和部署步骤见
[development mTLS overlay](kubernetes/overlays/development-mtls/README.md)。生产 Overlay
必须为每个弹性 Gateway/Checker 工作负载配置唯一身份与证书轮换,不能复用固定开发证书。
身份系统只需把每 Pod 的叶证书、私钥和信任包写入配置指定路径,并签发唯一
`.../worker/<worker-id>``.../checker/<checker-id>` URI启动命令设置
`-auto-identity` 后由进程派生逻辑/缺省实例 ID不依赖手工环境变量。开发 Compose 和
此 Overlay 已启用该模式;其中证书只适用于本地开发,不代表生产身份签发与挂载实现。
当前可执行验证:
@ -16,6 +30,7 @@ Grafana 与 Kubernetes。镜像会构建 `proxy-controller`、`proxy-gateway`
docker compose -f deploy/docker-compose.yml config --quiet
docker compose -f deploy/docker-compose.test.yml config --quiet
kubectl kustomize deploy/kubernetes/base | Out-Null
kubectl kustomize deploy/kubernetes/overlays/development-mtls | Out-Null
go run ./deploy/tools/configcheck deploy/config/local.yaml
```
@ -26,3 +41,8 @@ PostgreSQL Adapter 和对应执行脚本完成前,不把数据库契约记为
发布前仍须通过 `production-readiness.md` 中的一致性、安全、恢复、竞态与容量门禁,
并完成代表性集群容量验证。
Grafana Overview 与 Prometheus 规则只引用代码已注册的低基数指标。它们覆盖 Gateway
请求/端到端时延/Outcome、Controller 容量、Provider、Extraction、Checker 和 Drain不按 Proxy、IP、
Client、Upstream、Worker、Session 或完整 URL 聚合。`go test ./deploy` 会解析两类资产并拒绝
不存在的指标与禁止标签,指标改名或新增面板时必须同步更新该契约。

66
deploy/ci_test.go Normal file
View File

@ -0,0 +1,66 @@
package deploy
import (
"os"
"strings"
"testing"
"go.yaml.in/yaml/v4"
)
func TestCIValidatesProtocolAndDeploymentAssets(t *testing.T) {
payload, err := os.ReadFile("../.github/workflows/ci.yml")
if err != nil {
t.Fatalf("read ci workflow: %v", err)
}
workflow := string(payload)
var document map[string]any
if err := yaml.Unmarshal(payload, &document); err != nil {
t.Fatalf("parse ci workflow: %v", err)
}
jobs, ok := document["jobs"].(map[string]any)
if !ok {
t.Fatal("ci workflow has no jobs")
}
for _, name := range []string{"openapi", "proto", "image", "deployment", "test", "race", "integration"} {
if _, exists := jobs[name]; !exists {
t.Errorf("ci workflow has no %s job", name)
}
}
for _, required := range []string{
"./scripts/verify-openapi.ps1",
"./scripts/verify-proto.ps1",
"docker build --file deploy/docker/Dockerfile --tag proxy-pool:ci .",
"docker compose -f deploy/docker-compose.yml config --quiet",
"docker compose -f deploy/docker-compose.test.yml config --quiet",
"go run ./deploy/tools/configcheck deploy/config/local.yaml",
"kubectl kustomize deploy/kubernetes/base",
"kubectl kustomize deploy/kubernetes/overlays/development-mtls",
"go run ./deploy/tools/configcheck deploy/kubernetes/overlays/development-mtls/config.yaml",
} {
if !strings.Contains(workflow, required) {
t.Errorf("ci workflow does not validate %q", required)
}
}
deployment, ok := jobs["deployment"].(map[string]any)
if !ok {
t.Fatal("ci deployment job has invalid definition")
}
steps, ok := deployment["steps"].([]any)
if !ok {
t.Fatal("ci deployment job has no steps")
}
for _, step := range steps {
definition, ok := step.(map[string]any)
if !ok || definition["uses"] != "actions/setup-go@v5" {
continue
}
options, ok := definition["with"].(map[string]any)
if !ok || options["go-version-file"] != "go.mod" {
t.Error("ci deployment job must pin actions/setup-go to go.mod")
return
}
return
}
t.Error("ci deployment job must install the Go version declared by go.mod before running configcheck")
}

View File

@ -1,6 +1,7 @@
package deploy
import (
"encoding/json"
"os"
"path/filepath"
"regexp"
@ -11,6 +12,49 @@ import (
"go.yaml.in/yaml/v4"
)
var observableMetricNames = map[string]struct{}{
"proxy_pool_checker_observations_total": {},
"proxy_pool_checker_tasks_dispatched_total": {},
"proxy_pool_controller_capacity_active_upstreams": {},
"proxy_pool_controller_capacity_available_slots": {},
"proxy_pool_controller_capacity_effective_slots": {},
"proxy_pool_controller_capacity_inventory_reads_total": {},
"proxy_pool_controller_capacity_managed_proxies": {},
"proxy_pool_controller_capacity_pending_expected_proxies": {},
"proxy_pool_controller_drain_candidates_total": {},
"proxy_pool_controller_drains_started_total": {},
"proxy_pool_controller_extraction_requested_proxies_total": {},
"proxy_pool_controller_extraction_requests_total": {},
"proxy_pool_controller_extraction_returned_proxies_total": {},
"proxy_pool_controller_provider_fetch_results_total": {},
"proxy_pool_controller_provider_new_proxies_total": {},
"proxy_pool_controller_provider_valid_candidates_total": {},
"proxy_pool_gateway_active_tunnels": {},
"proxy_pool_gateway_capacity_invariant_violations_total": {},
"proxy_pool_gateway_outcome_queue_dropped_total": {},
"proxy_pool_gateway_outcomes_total": {},
"proxy_pool_gateway_request_duration_seconds": {},
"proxy_pool_gateway_requests_in_flight": {},
"proxy_pool_gateway_requests_total": {},
}
type dashboardDocument struct {
Panels []struct {
Targets []struct {
Expression string `json:"expr"`
} `json:"targets"`
} `json:"panels"`
}
type alertRulesDocument struct {
Groups []struct {
Rules []struct {
Expression string `yaml:"expr"`
Alert string `yaml:"alert"`
} `yaml:"rules"`
} `yaml:"groups"`
}
type composeDocument struct {
Services map[string]composeService `yaml:"services"`
Volumes map[string]any `yaml:"volumes"`
@ -20,8 +64,10 @@ type composeService struct {
Image string `yaml:"image"`
Command []string `yaml:"command"`
Ports []string `yaml:"ports"`
Expose []string `yaml:"expose"`
Volumes []string `yaml:"volumes"`
Tmpfs []string `yaml:"tmpfs"`
Environment map[string]string `yaml:"environment"`
DependsOn any `yaml:"depends_on"`
}
@ -133,6 +179,65 @@ func TestLocalGatewaysDoNotDependOnControlPlaneStorage(t *testing.T) {
}
}
func TestLocalComposeProvidesAuthenticatedControlPlaneForEveryWorker(t *testing.T) {
document := loadComposeDocument(t)
controller := document.Services["controller"]
if !slices.Contains(controller.Expose, "8443") || !slices.Contains(controller.Volumes, "./.control-plane-tls/controller:/run/proxy-pool-tls/server:ro") {
t.Fatalf("controller does not expose and mount control-plane TLS: expose=%v volumes=%v", controller.Expose, controller.Volumes)
}
for _, expected := range []struct {
name string
certificate string
}{
{name: "gateway-a", certificate: "gateway-a"},
{name: "gateway-b", certificate: "gateway-b"},
{name: "checker", certificate: "checker-a"},
} {
service, ok := document.Services[expected.name]
if !ok {
t.Fatalf("docker-compose.yml has no %s", expected.name)
}
if service.Environment["PROXY_POOL_CONTROL_PLANE_ADDRESS"] != "controller:8443" ||
!slices.Contains(service.Volumes, "./.control-plane-tls/"+expected.certificate+":/run/proxy-pool-tls/client:ro") ||
!composeDependsOn(service.DependsOn, "controller") {
t.Errorf("%s control-plane wiring = env=%v volumes=%v dependsOn=%v", expected.name, service.Environment, service.Volumes, service.DependsOn)
}
if expected.name == "checker" {
if service.Environment["PROXY_POOL_AUTO_IDENTITY"] != "true" ||
service.Environment["PROXY_POOL_CHECKER_ID"] != "" ||
service.Environment["PROXY_POOL_CHECKER_INSTANCE_ID"] != "" ||
service.Environment["PROXY_POOL_CHECKER_MAX_IN_FLIGHT"] != "200" || !slices.Contains(service.Expose, "9090") {
t.Errorf("checker identity or metrics wiring = env=%v expose=%v", service.Environment, service.Expose)
}
continue
}
if service.Environment["PROXY_POOL_AUTO_IDENTITY"] != "true" ||
service.Environment["PROXY_POOL_WORKER_ID"] != "" ||
service.Environment["PROXY_POOL_INSTANCE_ID"] != "" ||
service.Environment["PROXY_POOL_CLUSTER_ID"] != "compose-local" || service.Environment["PROXY_POOL_ZONE"] != "compose-local" {
t.Errorf("%s identity wiring = %v", expected.name, service.Environment)
}
}
}
func TestLocalControlPlaneCertificateGeneratorUsesDistinctSPIFFERoles(t *testing.T) {
payload, err := os.ReadFile("../scripts/generate-local-controlplane-certs.ps1")
if err != nil {
t.Fatalf("read generate-local-controlplane-certs.ps1: %v", err)
}
script := string(payload)
for _, required := range []string{
"OutputDirectory must be empty", "DNS:controller",
"spiffe://proxy-pool.local/development/worker/gateway-a",
"spiffe://proxy-pool.local/development/worker/gateway-b",
"spiffe://proxy-pool.local/development/checker/checker-a",
} {
if !strings.Contains(script, required) {
t.Errorf("certificate generator missing %q", required)
}
}
}
func TestDeploymentEntrypointsExistInSource(t *testing.T) {
dockerfile, err := os.ReadFile("docker/Dockerfile")
if err != nil {
@ -160,13 +265,93 @@ func TestDeploymentEntrypointsExistInSource(t *testing.T) {
}
}
func TestKubernetesBaseExcludesUnimplementedProcessManifests(t *testing.T) {
func TestDockerBuildContextExcludesLocalSecretsAndArtifacts(t *testing.T) {
payload, err := os.ReadFile("../.dockerignore")
if err != nil {
t.Fatalf("read .dockerignore: %v", err)
}
rules := make(map[string]struct{})
for _, raw := range strings.Split(string(payload), "\n") {
rule := strings.TrimSpace(raw)
if rule == "" || strings.HasPrefix(rule, "#") {
continue
}
rules[rule] = struct{}{}
}
for _, required := range []string{
".git", ".env", ".env.*", "*.key", "*.pem", "*.crt",
"deploy/.control-plane-tls/", "bin/", "dist/", "coverage/", ".tmp-proto/",
} {
if _, exists := rules[required]; !exists {
t.Errorf(".dockerignore does not exclude %q", required)
}
}
}
func TestKubernetesBaseLeavesControlPlaneClientsForMTLSOverlay(t *testing.T) {
payload, err := os.ReadFile("kubernetes/base/kustomization.yaml")
if err != nil {
t.Fatalf("read kustomization.yaml: %v", err)
}
if strings.Contains(string(payload), "checker.yaml") {
t.Fatal("kubernetes base includes checker.yaml before proxy-checker exists")
t.Fatal("kubernetes base includes checker.yaml without per-workload mTLS identity overlay")
}
}
func TestObservabilityAssetsUseRegisteredLowCardinalityMetrics(t *testing.T) {
dashboardPayload, err := os.ReadFile("grafana/dashboards/proxy-pool-overview.json")
if err != nil {
t.Fatalf("read overview dashboard: %v", err)
}
var dashboard dashboardDocument
if err := json.Unmarshal(dashboardPayload, &dashboard); err != nil {
t.Fatalf("parse overview dashboard: %v", err)
}
expressions := make([]string, 0, len(dashboard.Panels))
for _, panel := range dashboard.Panels {
for _, target := range panel.Targets {
expressions = append(expressions, target.Expression)
}
}
rulesPayload, err := os.ReadFile("prometheus/rules/proxy-pool.yml")
if err != nil {
t.Fatalf("read Prometheus rules: %v", err)
}
var rules alertRulesDocument
if err := yaml.Unmarshal(rulesPayload, &rules); err != nil {
t.Fatalf("parse Prometheus rules: %v", err)
}
alerts := make([]string, 0)
for _, group := range rules.Groups {
for _, rule := range group.Rules {
expressions = append(expressions, rule.Expression)
alerts = append(alerts, rule.Alert)
}
}
metricPattern := regexp.MustCompile(`proxy_pool_[a-z0-9_]+`)
forbiddenLabel := regexp.MustCompile(`(?:by\s*\([^)]*\b(?:upstream|worker)\b|\{[^}]*\b(?:upstream|worker)\s*=)`)
for _, expression := range expressions {
for _, name := range metricPattern.FindAllString(expression, -1) {
baseName := strings.TrimSuffix(strings.TrimSuffix(strings.TrimSuffix(name, "_bucket"), "_count"), "_sum")
if _, exists := observableMetricNames[baseName]; !exists {
t.Errorf("observability expression references unregistered metric %q: %s", name, expression)
}
}
if forbiddenLabel.MatchString(expression) {
t.Errorf("observability expression uses a forbidden high-cardinality label: %s", expression)
}
}
for _, required := range []string{
"proxy_pool_gateway_requests_total", "proxy_pool_gateway_request_duration_seconds", "proxy_pool_controller_capacity_available_slots",
"proxy_pool_controller_provider_fetch_results_total", "proxy_pool_controller_extraction_requests_total",
"proxy_pool_checker_observations_total",
} {
if !strings.Contains(strings.Join(expressions, "\n"), required) {
t.Errorf("observability assets do not cover %s", required)
}
}
if !slices.Contains(alerts, "ProxyPoolGatewayHighLatency") {
t.Error("Prometheus rules do not alert on Gateway p99 latency")
}
}

View File

@ -16,6 +16,11 @@ gateway:
limits:
maxConcurrentConnections: 20000
requestsPerMinutePerClient: 60000
transport:
maxIdleConns: 20000
maxIdleConnsPerHost: 32
maxConnsPerHost: 32
tunnelBufferBytes: 32768
retry:
maxAttempts: 2
retryMethods: [GET, HEAD]
@ -59,7 +64,30 @@ admin:
token: "${PROXY_POOL_ADMIN_TOKEN}"
controlPlane:
enabled: false
enabled: true
listen: 0.0.0.0:8443
protocolVersion: 1
heartbeatInterval: 10s
sessionTTL: 30s
maxStaleAge: 30s
maxMessageBytes: 4194304
maxRuntimeCounters: 100000
maxConcurrentStreams: 1000
tls:
mode: mtls
certFile: /run/proxy-pool-tls/server/tls.crt
keyFile: /run/proxy-pool-tls/server/tls.key
clientCAFile: /run/proxy-pool-tls/server/ca.crt
trustDomain: proxy-pool.local
environment: development
gatewayTLS:
certFile: /run/proxy-pool-tls/client/tls.crt
keyFile: /run/proxy-pool-tls/client/tls.key
serverCAFile: /run/proxy-pool-tls/client/ca.crt
checkerTLS:
certFile: /run/proxy-pool-tls/client/tls.crt
keyFile: /run/proxy-pool-tls/client/tls.key
serverCAFile: /run/proxy-pool-tls/client/ca.crt
metrics:
enabled: true

View File

@ -15,8 +15,6 @@ x-app: &app
image: proxy-pool:local
restart: unless-stopped
networks: [frontend, backend]
volumes:
- ./config/local.yaml:/etc/proxy-pool/config.yaml:ro
environment: *app-environment
stop_grace_period: 45s
@ -24,6 +22,18 @@ services:
gateway-a:
<<: *app
command: ["proxy-gateway"]
environment:
<<: *app-environment
PROXY_POOL_CONTROL_PLANE_ADDRESS: controller:8443
PROXY_POOL_CLUSTER_ID: compose-local
PROXY_POOL_AUTO_IDENTITY: "true"
PROXY_POOL_ZONE: compose-local
volumes:
- ./config/local.yaml:/etc/proxy-pool/config.yaml:ro
- ./.control-plane-tls/gateway-a:/run/proxy-pool-tls/client:ro
depends_on:
controller:
condition: service_healthy
expose: ["8080", "9090"]
healthcheck:
test: ["CMD", "curl", "--fail", "--silent", "http://127.0.0.1:9090/readyz"]
@ -35,6 +45,18 @@ services:
gateway-b:
<<: *app
command: ["proxy-gateway"]
environment:
<<: *app-environment
PROXY_POOL_CONTROL_PLANE_ADDRESS: controller:8443
PROXY_POOL_CLUSTER_ID: compose-local
PROXY_POOL_AUTO_IDENTITY: "true"
PROXY_POOL_ZONE: compose-local
volumes:
- ./config/local.yaml:/etc/proxy-pool/config.yaml:ro
- ./.control-plane-tls/gateway-b:/run/proxy-pool-tls/client:ro
depends_on:
controller:
condition: service_healthy
expose: ["8080", "9090"]
healthcheck:
test: ["CMD", "curl", "--fail", "--silent", "http://127.0.0.1:9090/readyz"]
@ -49,12 +71,15 @@ services:
environment:
<<: *app-environment
PROXY_POOL_CONFIG_FINGERPRINT_KEY: ${PROXY_POOL_CONFIG_FINGERPRINT_KEY:?set PROXY_POOL_CONFIG_FINGERPRINT_KEY}
volumes:
- ./config/local.yaml:/etc/proxy-pool/config.yaml:ro
- ./.control-plane-tls/controller:/run/proxy-pool-tls/server:ro
depends_on:
postgres:
condition: service_healthy
redis:
condition: service_healthy
expose: ["8081", "8082", "9090"]
expose: ["8081", "8082", "8443", "9090"]
ports:
- "127.0.0.1:8081:8081"
- "127.0.0.1:8082:8082"
@ -65,6 +90,28 @@ services:
retries: 12
start_period: 15s
checker:
<<: *app
command: ["proxy-checker"]
environment:
<<: *app-environment
PROXY_POOL_CONTROL_PLANE_ADDRESS: controller:8443
PROXY_POOL_AUTO_IDENTITY: "true"
PROXY_POOL_CHECKER_MAX_IN_FLIGHT: "200"
volumes:
- ./config/local.yaml:/etc/proxy-pool/config.yaml:ro
- ./.control-plane-tls/checker-a:/run/proxy-pool-tls/client:ro
depends_on:
controller:
condition: service_healthy
expose: ["9090"]
healthcheck:
test: ["CMD", "curl", "--fail", "--silent", "http://127.0.0.1:9090/readyz"]
interval: 5s
timeout: 2s
retries: 12
start_period: 10s
haproxy:
image: haproxy:3.2-alpine
restart: unless-stopped

View File

@ -3,13 +3,19 @@
"editable": true,
"graphTooltip": 1,
"panels": [
{"type":"timeseries","title":"Gateway QPS","gridPos":{"h":8,"w":8,"x":0,"y":0},"targets":[{"expr":"sum(rate(proxy_pool_gateway_requests_total[1m]))","legendFormat":"QPS"}]},
{"type":"timeseries","title":"Gateway p99","gridPos":{"h":8,"w":8,"x":8,"y":0},"targets":[{"expr":"histogram_quantile(0.99, sum by (le) (rate(proxy_pool_gateway_request_duration_seconds_bucket[5m])))","legendFormat":"p99"}]},
{"type":"timeseries","title":"Available Slots","gridPos":{"h":8,"w":8,"x":16,"y":0},"targets":[{"expr":"sum(proxy_pool_available_slots)","legendFormat":"slots"}]},
{"type":"timeseries","title":"Provider Fetch","gridPos":{"h":8,"w":12,"x":0,"y":8},"targets":[{"expr":"sum by (result) (rate(proxy_pool_provider_fetch_total[5m]))","legendFormat":"{{result}}"}]},
{"type":"timeseries","title":"Extraction","gridPos":{"h":8,"w":12,"x":12,"y":8},"targets":[{"expr":"sum by (result) (rate(proxy_pool_extraction_total[5m]))","legendFormat":"{{result}}"}]},
{"type":"timeseries","title":"Snapshot Age","gridPos":{"h":8,"w":12,"x":0,"y":16},"targets":[{"expr":"max by (worker) (proxy_pool_snapshot_age_seconds)","legendFormat":"{{worker}}"}]},
{"type":"timeseries","title":"Checker Queue","gridPos":{"h":8,"w":12,"x":12,"y":16},"targets":[{"expr":"sum(proxy_pool_checker_queue_depth)","legendFormat":"depth"}]}
{"type":"timeseries","title":"Gateway QPS","gridPos":{"h":8,"w":8,"x":0,"y":0},"targets":[{"expr":"sum by (protocol) (rate(proxy_pool_gateway_requests_total[1m]))","legendFormat":"{{protocol}}"}]},
{"type":"timeseries","title":"Gateway p99","gridPos":{"h":8,"w":8,"x":8,"y":0},"targets":[{"expr":"histogram_quantile(0.99, sum by (le, protocol) (rate(proxy_pool_gateway_request_duration_seconds_bucket[5m])))","legendFormat":"{{protocol}}"}]},
{"type":"timeseries","title":"Gateway In Flight and Tunnels","gridPos":{"h":8,"w":8,"x":16,"y":0},"targets":[{"expr":"sum by (protocol) (proxy_pool_gateway_requests_in_flight)","legendFormat":"in-flight {{protocol}}"},{"expr":"sum(proxy_pool_gateway_active_tunnels)","legendFormat":"tunnels"}]},
{"type":"timeseries","title":"Gateway Outcome Failures","gridPos":{"h":8,"w":12,"x":0,"y":8},"targets":[{"expr":"sum by (stage) (rate(proxy_pool_gateway_outcomes_total{result=\"failure\"}[5m]))","legendFormat":"{{stage}}"}]},
{"type":"timeseries","title":"Gateway Outcome Queue Drops","gridPos":{"h":8,"w":12,"x":12,"y":8},"targets":[{"expr":"sum(rate(proxy_pool_gateway_outcome_queue_dropped_total[5m]))","legendFormat":"drops/s"}]},
{"type":"timeseries","title":"Controller Capacity","gridPos":{"h":8,"w":12,"x":0,"y":16},"targets":[{"expr":"sum(proxy_pool_controller_capacity_available_slots)","legendFormat":"available slots"},{"expr":"sum(proxy_pool_controller_capacity_effective_slots)","legendFormat":"effective slots"},{"expr":"sum(proxy_pool_controller_capacity_pending_expected_proxies)","legendFormat":"pending proxies"}]},
{"type":"timeseries","title":"Managed Proxies and Active Upstreams","gridPos":{"h":8,"w":12,"x":12,"y":16},"targets":[{"expr":"sum(proxy_pool_controller_capacity_managed_proxies)","legendFormat":"managed proxies"},{"expr":"sum(proxy_pool_controller_capacity_active_upstreams)","legendFormat":"active upstreams"}]},
{"type":"timeseries","title":"Provider Fetch Results","gridPos":{"h":8,"w":12,"x":0,"y":24},"targets":[{"expr":"sum by (class) (rate(proxy_pool_controller_provider_fetch_results_total[5m]))","legendFormat":"{{class}}"},{"expr":"sum(rate(proxy_pool_controller_provider_new_proxies_total[5m]))","legendFormat":"new proxies/s"}]},
{"type":"timeseries","title":"Extraction Results","gridPos":{"h":8,"w":12,"x":12,"y":24},"targets":[{"expr":"sum by (result) (rate(proxy_pool_controller_extraction_requests_total[5m]))","legendFormat":"{{result}}"},{"expr":"sum(rate(proxy_pool_controller_extraction_returned_proxies_total[5m]))","legendFormat":"returned proxies/s"}]},
{"type":"timeseries","title":"Checker Task Dispatch","gridPos":{"h":8,"w":12,"x":0,"y":32},"targets":[{"expr":"sum by (level) (rate(proxy_pool_checker_tasks_dispatched_total[5m]))","legendFormat":"{{level}}"}]},
{"type":"timeseries","title":"Checker Observations","gridPos":{"h":8,"w":12,"x":12,"y":32},"targets":[{"expr":"sum by (level, result) (rate(proxy_pool_checker_observations_total[5m]))","legendFormat":"{{level}} {{result}}"}]},
{"type":"timeseries","title":"Drain Tickets Started","gridPos":{"h":8,"w":12,"x":0,"y":40},"targets":[{"expr":"sum by (reason) (rate(proxy_pool_controller_drains_started_total[5m]))","legendFormat":"{{reason}}"}]},
{"type":"timeseries","title":"Capacity Inventory Reads","gridPos":{"h":8,"w":12,"x":12,"y":40},"targets":[{"expr":"sum by (result) (rate(proxy_pool_controller_capacity_inventory_reads_total[5m]))","legendFormat":"{{result}}"}]}
],
"schemaVersion": 41,
"tags": ["proxy-pool"],
@ -17,5 +23,5 @@
"time": {"from":"now-6h","to":"now"},
"title": "Proxy Pool Overview",
"uid": "proxy-pool-overview",
"version": 1
"version": 2
}

View File

@ -21,6 +21,16 @@ data:
limits:
maxConcurrentConnections: 100000
requestsPerMinutePerClient: 60000
transport:
dialTimeout: 10s
handshakeTimeout: 15s
responseHeaderTimeout: 30s
idleConnTimeout: 90s
maxIdleConns: 20000
maxIdleConnsPerHost: 32
maxConnsPerHost: 32
tunnelBufferBytes: 32768
tunnelIdleTimeout: 5m
retry:
maxAttempts: 2
retryMethods: [GET, HEAD]

View File

@ -0,0 +1,63 @@
# Kubernetes development mTLS overlay
该 Overlay 在 base 资源之上启用 Controller 控制面 mTLS并启动一个 Gateway 和
Checker。进程通过 `PROXY_POOL_AUTO_IDENTITY=true` 从各自证书的 URI SAN 派生身份;
它只用于开发或预发布的单副本控制面验证,不是生产弹性身份方案。
## Prepare certificates
先生成未提交的开发证书。输出目录必须为空;生成器的 `controller` 证书包含
`DNS:controller`Overlay 中的客户端也只拨号 `controller:8443`
```powershell
./scripts/generate-local-controlplane-certs.ps1
kubectl apply -f deploy/kubernetes/base/namespace.yaml
```
将生成的三组证书写入 Kubernetes Secret。下面的命令可重复执行只更新对应 Secret
```powershell
kubectl -n proxy-pool create secret generic proxy-pool-controlplane-server-tls `
--from-file=tls.crt=deploy/.control-plane-tls/controller/tls.crt `
--from-file=tls.key=deploy/.control-plane-tls/controller/tls.key `
--from-file=ca.crt=deploy/.control-plane-tls/controller/ca.crt `
--dry-run=client -o yaml | kubectl apply -f -
kubectl -n proxy-pool create secret generic proxy-pool-controlplane-gateway-tls `
--from-file=tls.crt=deploy/.control-plane-tls/gateway-a/tls.crt `
--from-file=tls.key=deploy/.control-plane-tls/gateway-a/tls.key `
--from-file=ca.crt=deploy/.control-plane-tls/gateway-a/ca.crt `
--dry-run=client -o yaml | kubectl apply -f -
kubectl -n proxy-pool create secret generic proxy-pool-controlplane-checker-tls `
--from-file=tls.crt=deploy/.control-plane-tls/checker-a/tls.crt `
--from-file=tls.key=deploy/.control-plane-tls/checker-a/tls.key `
--from-file=ca.crt=deploy/.control-plane-tls/checker-a/ca.crt `
--dry-run=client -o yaml | kubectl apply -f -
```
另行从私密配置系统创建 `proxy-pool-secrets`。可参考 base 目录的
`secret.example.yaml`但不要把实际数据库、Redis、Provider 或 API 凭据写入 Git。
## Deploy
```powershell
kubectl apply -k deploy/kubernetes/overlays/development-mtls
kubectl -n proxy-pool rollout status deployment/proxy-controller --timeout=5m
kubectl -n proxy-pool rollout status deployment/proxy-gateway --timeout=5m
kubectl -n proxy-pool rollout status deployment/proxy-checker --timeout=5m
```
Overlay 的 Gateway 从 `gateway-a` 证书派生 `worker_id=gateway-a`Checker 从
`checker-a` 证书派生 `checker_id=checker-a`,并把 Gateway HPA 限制为
`minReplicas=maxReplicas=1`。因此不得在此 Overlay 上增加副本数或放宽 HPA重复使用
同一证书和 Worker 身份会破坏 session、ownership 和证书角色边界。
Overlay 的 `proxy-pool-runtime-config` 带内容哈希;修改 `config.yaml`Kustomize 会重写三个
Deployment 的 ConfigMap volume 引用,从而触发 Kubernetes 滚动更新。控制面 TLS 与
`proxy-pool-secrets` 使用外部创建的固定名称 Secret轮换后仍需按运行手册显式滚动对应工作负载。
生产环境需要由工作负载身份系统为每个副本签发独立、可轮换的证书,并将该副本的
身份注入 Gateway Client TLS 或 Checker Client TLS并启用自动身份派生。应用会在每次新
控制面 TLS 握手读取更新后的叶证书和信任根;已有 gRPC 流仍按原会话保留,根轮换时先投放
新旧根的重叠信任包,再按 PDB 滚动排空旧连接。

View File

@ -0,0 +1,24 @@
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
name: checker-control-plane
namespace: proxy-pool
spec:
podSelector:
matchLabels: {app.kubernetes.io/name: proxy-checker}
policyTypes: [Ingress, Egress]
ingress:
- from:
- namespaceSelector: {matchLabels: {kubernetes.io/metadata.name: monitoring}}
ports: [{port: 9090, protocol: TCP}]
egress:
- to:
- namespaceSelector: {matchLabels: {kubernetes.io/metadata.name: kube-system}}
ports: [{port: 53, protocol: UDP}, {port: 53, protocol: TCP}]
- to:
- podSelector: {matchLabels: {app.kubernetes.io/name: proxy-controller}}
ports: [{port: 8443, protocol: TCP}]
- to:
- ipBlock:
cidr: 0.0.0.0/0
except: [10.0.0.0/8, 100.64.0.0/10, 127.0.0.0/8, 169.254.0.0/16, 172.16.0.0/12, 192.168.0.0/16]

View File

@ -0,0 +1,26 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: proxy-checker
namespace: proxy-pool
spec:
replicas: 1
template:
spec:
containers:
- name: checker
env:
- {name: PROXY_POOL_CONTROL_PLANE_ADDRESS, value: controller:8443}
- {name: PROXY_POOL_AUTO_IDENTITY, value: "true"}
- {name: PROXY_POOL_CHECKER_MAX_IN_FLIGHT, value: "200"}
volumeMounts:
- name: controlplane-checker-tls
mountPath: /run/proxy-pool-tls/client
readOnly: true
volumes:
- name: config
configMap:
name: proxy-pool-runtime-config
- name: controlplane-checker-tls
secret:
secretName: proxy-pool-controlplane-checker-tls

View File

@ -0,0 +1,65 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: proxy-checker
namespace: proxy-pool
labels: {app.kubernetes.io/name: proxy-checker, app.kubernetes.io/part-of: proxy-pool}
spec:
replicas: 1
minReadySeconds: 5
revisionHistoryLimit: 3
strategy:
type: RollingUpdate
rollingUpdate: {maxUnavailable: 1, maxSurge: 1}
selector:
matchLabels: {app.kubernetes.io/name: proxy-checker}
template:
metadata:
labels: {app.kubernetes.io/name: proxy-checker, app.kubernetes.io/part-of: proxy-pool}
annotations: {prometheus.io/scrape: "true", prometheus.io/port: "9090", prometheus.io/path: /metrics}
spec:
serviceAccountName: proxy-pool
automountServiceAccountToken: false
terminationGracePeriodSeconds: 45
securityContext: {runAsNonRoot: true, seccompProfile: {type: RuntimeDefault}}
topologySpreadConstraints:
- maxSkew: 1
topologyKey: kubernetes.io/hostname
whenUnsatisfiable: ScheduleAnyway
labelSelector: {matchLabels: {app.kubernetes.io/name: proxy-checker}}
containers:
- name: checker
image: REGISTRY/proxy-pool:VERSION
imagePullPolicy: IfNotPresent
command: [proxy-checker]
env:
- {name: PROXY_POOL_CONFIG, value: /etc/proxy-pool/config.yaml}
envFrom:
- secretRef: {name: proxy-pool-secrets}
ports:
- {name: metrics, containerPort: 9090}
readinessProbe:
httpGet: {path: /readyz, port: metrics}
periodSeconds: 5
timeoutSeconds: 2
livenessProbe:
httpGet: {path: /livez, port: metrics}
periodSeconds: 10
timeoutSeconds: 2
lifecycle:
preStop: {exec: {command: [sh, -c, "sleep 3"]}}
resources:
requests: {cpu: "1", memory: 512Mi}
limits: {cpu: "2", memory: 1Gi}
securityContext:
allowPrivilegeEscalation: false
readOnlyRootFilesystem: true
capabilities: {drop: [ALL]}
volumeMounts:
- {name: config, mountPath: /etc/proxy-pool, readOnly: true}
- {name: tmp, mountPath: /tmp}
volumes:
- name: config
configMap: {name: proxy-pool-config}
- name: tmp
emptyDir: {sizeLimit: 64Mi}

View File

@ -0,0 +1,216 @@
version: 1
security:
requireProtectionOnPublicListen: true
gateway:
enabled: true
listen: 0.0.0.0:8080
access:
allowCIDRs: [0.0.0.0/0]
trustedProxies: []
auth:
mode: usernamePassword
username: "${PROXY_POOL_GATEWAY_USERNAME}"
password: "${PROXY_POOL_GATEWAY_PASSWORD}"
limits:
maxConcurrentConnections: 20000
requestsPerMinutePerClient: 60000
transport:
maxIdleConns: 20000
maxIdleConnsPerHost: 32
maxConnsPerHost: 32
tunnelBufferBytes: 32768
retry:
maxAttempts: 2
retryMethods: [GET, HEAD]
destinationPolicy:
denyPrivateNetworks: true
denyLoopback: true
denyLinkLocal: true
denyCIDRs: [169.254.169.254/32]
distribution:
enabled: true
listen: 0.0.0.0:8081
access:
allowCIDRs: [10.0.0.0/8]
trustedProxies: [10.0.0.0/8]
auth:
mode: apiKey
header: X-API-Key
token: "${PROXY_POOL_EXTRACT_TOKEN}"
limits:
requestsPerMinute: 6000
requestsPerMinutePerClient: 600
clientIdentification:
mode: sourceIP
extraction:
fulfillment: partial
maxCountPerRequest: 100
minRemainingTTL: 30s
maxHealthCheckAge: 30s
reserveForGateway: 1000
idempotencyTTL: 5m
admin:
enabled: true
listen: 0.0.0.0:8082
access:
allowCIDRs: [10.0.0.0/8]
auth:
mode: apiKey
header: X-Admin-Key
token: "${PROXY_POOL_ADMIN_TOKEN}"
controlPlane:
enabled: true
listen: 0.0.0.0:8443
protocolVersion: 1
heartbeatInterval: 10s
sessionTTL: 30s
maxStaleAge: 30s
maxMessageBytes: 4194304
maxRuntimeCounters: 100000
maxConcurrentStreams: 1000
tls:
mode: mtls
certFile: /run/proxy-pool-tls/server/tls.crt
keyFile: /run/proxy-pool-tls/server/tls.key
clientCAFile: /run/proxy-pool-tls/server/ca.crt
trustDomain: proxy-pool.local
environment: development
gatewayTLS:
certFile: /run/proxy-pool-tls/client/tls.crt
keyFile: /run/proxy-pool-tls/client/tls.key
serverCAFile: /run/proxy-pool-tls/client/ca.crt
checkerTLS:
certFile: /run/proxy-pool-tls/client/tls.crt
keyFile: /run/proxy-pool-tls/client/tls.key
serverCAFile: /run/proxy-pool-tls/client/ca.crt
metrics:
enabled: true
listen: 0.0.0.0:9090
storage:
postgresURL: "${PROXY_POOL_POSTGRES_URL}"
redisURL: "${PROXY_POOL_REDIS_URL}"
routing:
- name: gateway-default
enabled: true
purpose: gateway
upstreams: [provider-a, provider-b]
strategy:
type: sequential
switchAfterEmptyFetch: 5
endBehavior: stayLast
onUnavailable:
action: reject
- name: extract-default
enabled: true
purpose: extract
upstreams: [provider-a, provider-b]
strategy:
type: sequential
switchAfterEmptyFetch: 5
endBehavior: stayLast
onUnavailable:
action: reject
upstreams:
provider-a:
enabled: true
exposure: [gateway, extract]
provider:
billingMode: fetch
protocols: [http]
api:
url: https://provider-a.invalid/api/proxies
method: GET
auth:
type: apiKey
location: header
name: Authorization
value: "${PROVIDER_A_TOKEN}"
template: '{{ . }}'
proxyAuth:
type: response
pool:
maxSize: 5000
shrinkDelay: 30s
capacity:
maxConcurrencyPerProxy: 20
refill:
reconcileInterval: 1s
minimumAvailableSlots: 8000
targetAvailableSlots: 12000
lifecycle:
ttl: 5m
allocationSafetyMargin: 20s
fetch:
estimatedIPsPerCall: 100
requestInterval: 1s
timeout: 3s
maxAttempts: 3
maxInFlight: 1
maxTotal: 100000
maxResponseBytes: 4194304
templateTimeout: 100ms
retry: {initial: 500ms, max: 30s, jitter: 20}
check:
interval: 30s
jitter: 20
maxInFlight: 200
timeout: 3s
maxAttempts: 2
maxConsecutiveFailures: 3
urls: [https://example.com/]
provider-b:
enabled: true
exposure: [gateway, extract]
provider:
billingMode: fetch
protocols: [http]
api:
url: https://provider-b.invalid/api/proxies
method: GET
auth:
type: apiKey
location: header
name: Authorization
value: "${PROVIDER_B_TOKEN}"
template: '{{ . }}'
proxyAuth:
type: response
pool:
maxSize: 5000
shrinkDelay: 30s
capacity:
maxConcurrencyPerProxy: 20
refill:
reconcileInterval: 1s
minimumAvailableSlots: 8000
targetAvailableSlots: 12000
lifecycle:
ttl: 5m
allocationSafetyMargin: 20s
fetch:
estimatedIPsPerCall: 100
requestInterval: 1s
timeout: 3s
maxAttempts: 3
maxInFlight: 1
maxTotal: 100000
maxResponseBytes: 4194304
templateTimeout: 100ms
retry: {initial: 500ms, max: 30s, jitter: 20}
check:
interval: 30s
jitter: 20
maxInFlight: 200
timeout: 3s
maxAttempts: 2
maxConsecutiveFailures: 3
urls: [https://example.com/]

View File

@ -0,0 +1,6 @@
apiVersion: v1
kind: ConfigMap
metadata:
name: proxy-pool-config
namespace: proxy-pool
$patch: delete

View File

@ -0,0 +1,22 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: proxy-controller
namespace: proxy-pool
spec:
replicas: 1
template:
spec:
containers:
- name: controller
volumeMounts:
- name: controlplane-server-tls
mountPath: /run/proxy-pool-tls/server
readOnly: true
volumes:
- name: config
configMap:
name: proxy-pool-runtime-config
- name: controlplane-server-tls
secret:
secretName: proxy-pool-controlplane-server-tls

View File

@ -0,0 +1,11 @@
apiVersion: v1
kind: Service
metadata:
name: controller
namespace: proxy-pool
labels: {app.kubernetes.io/name: proxy-controller}
spec:
type: ClusterIP
selector: {app.kubernetes.io/name: proxy-controller}
ports:
- {name: control, port: 8443, targetPort: control}

View File

@ -0,0 +1,8 @@
apiVersion: autoscaling/v2
kind: HorizontalPodAutoscaler
metadata:
name: proxy-gateway
namespace: proxy-pool
spec:
minReplicas: 1
maxReplicas: 1

View File

@ -0,0 +1,27 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: proxy-gateway
namespace: proxy-pool
spec:
replicas: 1
template:
spec:
containers:
- name: gateway
env:
- {name: PROXY_POOL_CONTROL_PLANE_ADDRESS, value: controller:8443}
- {name: PROXY_POOL_CLUSTER_ID, value: kubernetes-development}
- {name: PROXY_POOL_AUTO_IDENTITY, value: "true"}
- {name: PROXY_POOL_ZONE, value: kubernetes-development}
volumeMounts:
- name: controlplane-gateway-tls
mountPath: /run/proxy-pool-tls/client
readOnly: true
volumes:
- name: config
configMap:
name: proxy-pool-runtime-config
- name: controlplane-gateway-tls
secret:
secretName: proxy-pool-controlplane-gateway-tls

View File

@ -0,0 +1,18 @@
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
resources:
- ../../base
- checker.yaml
- controller-service.yaml
- checker-networkpolicy.yaml
configMapGenerator:
- name: proxy-pool-runtime-config
namespace: proxy-pool
files:
- config.yaml
patches:
- path: configmap-delete.yaml
- path: controller-patch.yaml
- path: gateway-patch.yaml
- path: checker-patch.yaml
- path: gateway-hpa-patch.yaml

View File

@ -0,0 +1,73 @@
package deploy
import (
"os"
"os/exec"
"regexp"
"strings"
"testing"
)
func TestKubernetesDevelopmentMTLSOverlayRendersCertificateDerivedIdentities(t *testing.T) {
kubectl, err := exec.LookPath("kubectl")
if err != nil {
t.Skip("kubectl is required to render the Kubernetes development mTLS overlay")
}
command := exec.Command(kubectl, "kustomize", "kubernetes/overlays/development-mtls")
command.Dir = "."
payload, err := command.CombinedOutput()
if err != nil {
t.Fatalf("render development mTLS overlay: %v\n%s", err, payload)
}
rendered := string(payload)
for _, required := range []string{
"mode: mtls",
"trustDomain: proxy-pool.local",
"value: controller:8443",
"name: PROXY_POOL_AUTO_IDENTITY",
"value: \"true\"",
"secretName: proxy-pool-controlplane-server-tls",
"secretName: proxy-pool-controlplane-gateway-tls",
"secretName: proxy-pool-controlplane-checker-tls",
"name: proxy-checker",
"name: checker-control-plane",
"minReplicas: 1",
"maxReplicas: 1",
} {
if !strings.Contains(rendered, required) {
t.Errorf("development mTLS overlay does not render %q", required)
}
}
for _, forbidden := range []string{"name: PROXY_POOL_WORKER_ID", "name: PROXY_POOL_INSTANCE_ID", "name: PROXY_POOL_CHECKER_ID", "name: PROXY_POOL_CHECKER_INSTANCE_ID"} {
if strings.Contains(rendered, forbidden) {
t.Errorf("development mTLS overlay still renders manual identity %q", forbidden)
}
}
}
func TestKubernetesDevelopmentMTLSOverlayUsesVersionedConfigMap(t *testing.T) {
payload, err := os.ReadFile("kubernetes/overlays/development-mtls/kustomization.yaml")
if err != nil {
t.Fatalf("read development mTLS kustomization: %v", err)
}
if strings.Contains(string(payload), "disableNameSuffixHash: true") {
t.Fatal("development mTLS overlay disables ConfigMap versioning")
}
kubectl, err := exec.LookPath("kubectl")
if err != nil {
t.Skip("kubectl is required to render the Kubernetes development mTLS overlay")
}
command := exec.Command(kubectl, "kustomize", "kubernetes/overlays/development-mtls")
command.Dir = "."
renderedPayload, err := command.CombinedOutput()
if err != nil {
t.Fatalf("render development mTLS overlay: %v\n%s", err, renderedPayload)
}
matched := regexp.MustCompile(`(?m)^ name: (proxy-pool-runtime-config-[a-z0-9]+)$`).FindStringSubmatch(string(renderedPayload))
if len(matched) != 2 {
t.Fatalf("rendered overlay has no versioned config map: %s", renderedPayload)
}
if occurrences := strings.Count(string(renderedPayload), "name: "+matched[1]); occurrences != 4 {
t.Fatalf("versioned config map reference count = %d, want config map plus three workload volumes", occurrences)
}
}

View File

@ -14,6 +14,9 @@ scrape_configs:
- job_name: proxy-controller
static_configs:
- targets: [controller:9090]
- job_name: proxy-checker
static_configs:
- targets: [checker:9090]
- job_name: haproxy
metrics_path: /metrics
static_configs:

View File

@ -1,40 +1,74 @@
groups:
- name: proxy-pool
rules:
- alert: ProxyPoolGatewayHighErrorRate
- alert: ProxyPoolGatewayOutcomeFailureRate
expr: |
sum(rate(proxy_pool_gateway_requests_total{result="error"}[5m]))
/ clamp_min(sum(rate(proxy_pool_gateway_requests_total[5m])), 1) > 0.02
sum(rate(proxy_pool_gateway_outcomes_total{result="failure"}[5m]))
/ clamp_min(sum(rate(proxy_pool_gateway_outcomes_total[5m])), 1) > 0.1
for: 10m
labels:
severity: warning
annotations:
summary: Gateway 错误率持续高于 2%
- alert: ProxyPoolGatewaySnapshotStale
expr: proxy_pool_snapshot_age_seconds > 60
for: 2m
labels:
severity: critical
annotations:
summary: Gateway Snapshot 已超过安全陈旧时间
- alert: ProxyPoolNoAvailableSlots
expr: sum(proxy_pool_available_slots) == 0
for: 1m
labels:
severity: critical
annotations:
summary: Gateway 可分配容量耗尽
- alert: ProxyPoolProviderFetchErrors
expr: sum by (upstream) (rate(proxy_pool_provider_fetch_total{result="error"}[10m])) > 0.2
summary: Gateway 代理尝试失败比例持续高于 10%
- alert: ProxyPoolGatewayHighLatency
expr: |
histogram_quantile(0.99,
sum by (le, protocol) (rate(proxy_pool_gateway_request_duration_seconds_bucket[5m]))) > 5
for: 10m
labels:
severity: warning
annotations:
summary: Provider Fetch 错误持续发生
- alert: ProxyPoolExtractionConflict
expr: sum(rate(proxy_pool_extraction_total{result="conflict"}[5m])) > 0
summary: Gateway p99 请求时延持续高于 5 秒
- alert: ProxyPoolGatewayOutcomeQueueDrops
expr: sum(increase(proxy_pool_gateway_outcome_queue_dropped_total[5m])) > 0
for: 5m
labels:
severity: warning
annotations:
summary: 独占提取发生持续事务冲突
summary: Gateway Outcome 本地队列发生丢弃
- alert: ProxyPoolNoAvailableSlots
expr: |
sum(proxy_pool_controller_capacity_active_upstreams) > 0
and sum(proxy_pool_controller_capacity_available_slots) == 0
for: 1m
labels:
severity: critical
annotations:
summary: Controller 可分配容量耗尽
- alert: ProxyPoolCapacityInventoryErrors
expr: sum(rate(proxy_pool_controller_capacity_inventory_reads_total{result="error"}[10m])) > 0.2
for: 10m
labels:
severity: warning
annotations:
summary: Controller 容量库存读取持续失败
- alert: ProxyPoolProviderFetchErrors
expr: sum(rate(proxy_pool_controller_provider_fetch_results_total{class="error"}[10m])) > 0.2
for: 10m
labels:
severity: warning
annotations:
summary: Provider 获取持续失败
- alert: ProxyPoolCheckerObservationRejections
expr: |
sum(rate(proxy_pool_checker_observations_total{result="rejected"}[5m]))
/ clamp_min(sum(rate(proxy_pool_checker_observations_total[5m])), 1) > 0.05
for: 10m
labels:
severity: warning
annotations:
summary: Checker Observation 拒绝比例持续高于 5%
- alert: ProxyPoolExtractionIdempotencyConflict
expr: sum(rate(proxy_pool_controller_extraction_requests_total{result="idempotency_conflict"}[5m])) > 0
for: 5m
labels:
severity: warning
annotations:
summary: Distribution 幂等键冲突持续发生
- alert: ProxyPoolControlPlaneTargetDown
expr: up{job=~"proxy-gateway|proxy-controller|proxy-checker"} == 0
for: 2m
labels:
severity: critical
annotations:
summary: Proxy Pool 控制面目标不可抓取

View File

@ -26,6 +26,7 @@ PostgreSQL 事务内提交。若把它们暴露成多个 Repository调用方
type Mutator interface {
SetUpstreamEnabled(context.Context, SetUpstreamCommand) (MutationResult, error)
SwitchRouting(context.Context, SwitchRoutingCommand) (MutationResult, error)
DisableRouting(context.Context, DisableRoutingCommand) (MutationResult, error)
CommitConfig(context.Context, CommitConfigCommand) (MutationResult, error)
}
@ -43,7 +44,7 @@ type Outbox interface {
}
```
三个 mutation 方法保留现有 Admin 语义事务、修订号、CAS、审计和 Outbox 都
四个 mutation 方法保留现有 Admin/自动切换语义事务、修订号、CAS、审计和 Outbox 都
隐藏在模块实现内部。不会暴露 `BeginTx`、SQL executor 或五个可被错误组合的浅
Repository。
@ -95,6 +96,12 @@ revision因此并发提交或 Supervisor 同步的迟到旧版本不能覆盖
并发使用同一 expected 值时最多一个请求成功。目标等于当前值且 expected 匹配时
返回 `changed=false`,仍写审计但不写 Outbox。
`DisableRouting` 用于 Sequential 的末端 `stop`。它同样锁定目标 Routing并要求
`expectedCurrent` 仍等于权威当前值;满足条件时保留 `current_upstream`、仅将
`enabled` 置为 `false`。这样延迟的 Provider Empty 观察不会覆盖人工切换或配置重载。
已停用且 expected 匹配时返回 `changed=false` 并审计,不再产生 Outbox真实停用写入
`routing.disabled` 事件。该行为与 `SwitchRouting` 共享同一个全局 revision 分配边界。
### Outbox 消费
Outbox 使用有界 claim/ack而不是无界全表扫描
@ -129,7 +136,8 @@ MemoryStore 与 PostgreSQL Adapter 运行相同契约,至少覆盖:
- 配置首次提交、相同重放、校验和冲突和非法引用零写入。
- Upstream enable/disable 幂等、修订单调、审计必写、Outbox 仅在变更时写。
- Routing CAS、目标校验和 100 个并发请求最多一个成功。
- Routing CAS、目标校验和 100 个并发请求最多一个成功terminal stop 的 100 个
并发请求仅一个真实停用,其余为幂等审计 no-op。
- 任一审计/Outbox 写故障导致状态完全回滚。
- Outbox 有界 claim、租约到期重试、错误 consumer ACK 拒绝和顺序稳定。
- 审计按 ID 稳定分页并保留 Actor、资源、动作、修订和 UTC 时间Routing no-op

View File

@ -6,6 +6,7 @@ Admin API 使用独立监听器与权限,契约位于 `api/openapi/admin.yaml`
## 端点
- `GET /api/v1/status`:返回配置/快照版本、Upstream 聚合计数和 Worker 状态。
- `GET /api/v1/audit`:按审计记录 ID 升序读取权威管理面的变更记录。
- `POST /api/v1/upstreams/{name}/enable`:启用 Upstream。
- `POST /api/v1/upstreams/{name}/disable`:停止新 Fetch/分配并自然 Drain。
- `POST /api/v1/routing/{name}/switch`:用 expectedCurrent 做 CAS 手工切换。
@ -14,9 +15,30 @@ Admin API 使用独立监听器与权限,契约位于 `api/openapi/admin.yaml`
所有写操作写审计记录并返回最终 Request ID 与版本。Enable/Disable 对目标状态
幂等Routing Switch 必须携带 `expectedCurrent`,避免并发操作跳过多个供应商。
## 授权
认证成功后的权限由 Listener `auth.permissions``auth.methods[].permissions` 提供。
`admin:read` 允许 Status 与审计查询;`admin:write` 允许 Upstream 启停、Routing
切换和配置重载。每个权限独立匹配,写权限不隐含读权限;未配置权限的旧凭据保持
全权限兼容。`mode: any` 使用实际命中凭据的方法级权限。权限不足返回 `403`,不会
调用 Service、写审计或触发配置发布。
配置重载校验失败返回 422旧配置继续运行。Status 只返回低基数聚合信息,
不得返回 Proxy 地址、凭据、Client 标识或完整 Provider URL。
## 审计分页与数据范围
`GET /api/v1/audit` 使用 `afterId` 排他游标分页:响应只包含 `id` 大于
`afterId` 的记录,并按 `id` 升序排列;未传时 `afterId``0`。未传 `limit`
时服务端使用 `100`
`limit` 必须为 `1..1000`,其中 `1000` 等于 `adminstate.MaxPageSize`。客户端应将
本页最后一条记录的 `id` 作为下一次请求的 `afterId`;空 `records` 表示当前游标后
没有记录。
该接口仅暴露管理面变更记录及操作者标识、可信来源 IP、操作类型、资源标识、变更
状态、控制面版本、原因和发生时间。它不返回 Proxy、任何 Upstream/Provider URL、
凭据或 Secret、Provider 响应载荷,也不读取或暴露 Redis 活动池及其提取状态。
## 运行时实现边界
`admin.Handler` 只依赖 `Service` 控制面接口,不直接操作数据库、路由游标或配置

View File

@ -27,7 +27,9 @@ Controller 已实现并验证 `RegisterWorker`、`AcknowledgeSnapshot` 和
`WatchSnapshots` 已在 Register 后发送与当前 ownership epoch 对应的基础完整快照,并在
每份快照有效期的一半前重新构建、下发版本递增的完整快照;这样 ownership、Routing 和
凭据变化会在同一长连接内收敛,而无需等待有效期到达后重新注册。
凭据变化会在同一长连接内收敛而无需等待有效期到达后重新注册。Controller 内已提交的
Upstream 启停、Routing 切换和配置发布还会向全部本地 Worker 流广播一次合并后的立即刷新;
多 Controller 副本仍由该定时机制完成跨进程收敛。
Gateway 校验后 ACK 并开始 Runtime 心跳。Controller 会从 Redis 的有界 Worker ownership
索引构建已归属 Proxy 内容,并将租约到期收紧到 Proxy 的 `usable_until`。Proxy 引用的
凭据材料按 `secret_ref + credential_version` 去重,随完整 Snapshot 经 mTLS 下发,仅保留在
@ -234,10 +236,15 @@ Gateway 不复用 `controlPlane.listen` 作为客户端地址。`listen` 是 Con
- `-worker-id` / `PROXY_POOL_WORKER_ID`:唯一逻辑 Worker。
- `-instance-id` / `PROXY_POOL_INSTANCE_ID`:唯一进程实例。
- `-zone` / `PROXY_POOL_ZONE`:实例可用区。
- `-auto-identity` / `PROXY_POOL_AUTO_IDENTITY=true`:仅 mTLS 模式可用;从客户端
证书中唯一的 `.../worker/<worker-id>` SPIFFE URI 派生 Worker ID并在未显式配置时
使用同一值作为 Instance ID。
`controlPlane.tls.mode=mtls`Gateway 使用 `controlPlane.gatewayTLS` 中独立的
客户端证书、私钥和 Controller CA 发起 TLS 1.3 连接;证书必须符合 Controller 的
SPIFFE Worker 身份校验。
SPIFFE Worker 身份校验。客户端 X.509-SVID 叶证书必须且只能包含一个 URI SAN且该 URI
必须是对应角色的身份。Gateway 自动解析和 Controller 授权共用严格 URI 规则:身份 URI
不得包含用户信息、端口、查询、片段或转义路径。
`disabled` 仅接受回环控制面地址,供本地 fixture 使用。
## 10. Checker 启动参数
@ -247,6 +254,8 @@ SPIFFE Worker 身份校验。
- `-instance-id` / `PROXY_POOL_CHECKER_INSTANCE_ID`:唯一进程实例。
- `-max-in-flight` / `PROXY_POOL_CHECKER_MAX_IN_FLIGHT`:本进程任务上限。
- `-levels`:逗号分隔的 `basic,egress,target` 能力集合。
- `-auto-identity` / `PROXY_POOL_AUTO_IDENTITY=true`:仅 mTLS 模式可用;从客户端
证书中唯一的 `.../checker/<checker-id>` SPIFFE URI 派生 Checker ID 和缺省 Instance ID。
`controlPlane.tls.mode=mtls`Checker 使用 `controlPlane.checkerTLS` 的独立
客户端证书、私钥和 Controller CA 建立 TLS 1.3 连接;证书必须符合 Controller 的

View File

@ -197,6 +197,16 @@ TTL 到期或 Redis 数据丢失后不再保证旧 Key 去重,系统不回退
错误响应不得包含 Provider Secret、Proxy 凭据、SQL 或内部拓扑。
`POST /api/v1/proxies/extract` 需要 `distribution:extract`。权限从 Listener 的
`auth.permissions``mode: any` 的命中 `auth.methods[].permissions` 取得;缺少权限
返回 `403`,不会读取请求体、消耗幂等键或调用提取服务。未声明权限的旧凭据保持全
权限兼容。
Distribution 认证凭据还可声明 client 提取约束maxExtractCount、
allowedUpstreams 与 allowedRegions。该约束随实际命中的凭据返回早于幂等键和
提取服务执行。未传地区或 Upstream 过滤时,服务端自动写入凭据允许集合;传入
集合外的值、或数量超过凭据上限时返回 403并且不会消耗库存。
## 10. 短期运行记录与数据最小化
Redis 幂等结果只在配置的 TTL 窗口内保留重放响应所需的数据:

View File

@ -122,9 +122,122 @@ Lua 可精确表示的整数范围内。
- `any``methods` 中任一方法成功即可Bearer 方法的 Secret 使用
`value`/`valueFile`。
### 3.2 端点权限
`auth.permissions` 是普通认证方式Basic、API Key、Bearer、IP 白名单)的固定权限
集合。未配置时保留历史全权限行为;显式配置后只允许下列有限值:
- `admin:read`:读取 Admin Status 与审计页。
- `admin:write`:执行 Upstream/Routing/配置变更。
- `distribution:extract`:调用 Distribution 独占提取。
- `*`:所有权限,且必须单独出现。
`mode: any` 必须将 `permissions` 写在各 `methods` 项中,权限随实际命中的凭据
返回,顶层 `auth.permissions` 不允许同时配置。权限值不接受空白、重复或未知项。
缺少所需权限的已认证请求返回 `403`,并在进入 JSON 解码、提取或管理 mutation 前
结束。`mode: none` 不允许配置权限,避免把匿名访问误配置为受控授权。
```yaml
admin:
auth:
mode: any
methods:
- mode: bearer
valueFile: /run/secrets/admin-reader-token
permissions: [admin:read]
- mode: bearer
valueFile: /run/secrets/admin-writer-token
permissions: [admin:read, admin:write]
```
### 3.3 Distribution Client 提取约束
认证凭据可在 auth.client 中设置固定的 Client 约束mode: any 则必须在实际
命中的 auth.methods 项下配置。当前 Client 约束按入口明确分工,避免出现只声明
不执行的访问控制。
- maxExtractCountDistribution 单次提取的额外上限0 表示不追加上限。
- allowedUpstreams允许访问的 Upstream 名称集合。省略请求过滤条件时,服务端
自动使用这个集合;请求携带集合外名称会返回 403。
- allowedRegions允许访问的地区集合行为与 allowedUpstreams 相同。
空 client 配置保留旧版兼容语义不会限制已认证凭据。Upstream 名称必须引用
当前配置中已有的 Upstream。
~~~yaml
distribution:
auth:
mode: any
methods:
- mode: apiKey
header: X-API-Key
valueFile: /run/secrets/tenant-a-key
permissions: [distribution:extract]
client:
maxExtractCount: 20
allowedUpstreams: [provider-a, provider-b]
allowedRegions: [shanghai, beijing]
~~~
### 3.4 Gateway Client 路由约束
Gateway 的 auth.client 支持 allowedRoutings、requestsPerMinute 和
maxConcurrentConnections。认证成功后
Gateway 先完成目标地址策略和本地 Routing 匹配,再于本地 Dispatcher 前检查
Routing 集合;不匹配返回 403不会尝试选择或预留 Proxy。速率和并发限制均以
认证 Client 为计数键,在每个 Gateway Worker 内执行,超过限制返回 429。三项
检查都只读取请求上下文和当前本地状态,不访问 Redis、PostgreSQL 或 Provider。
allowedRoutings 中的每个名称必须对应一个启用的 Gateway Routing。Gateway 上的
maxExtractCount、allowedUpstreams 和 allowedRegions以及 Distribution 上的
allowedRoutings 和 requestsPerMinute 均会被配置校验拒绝。Admin 不支持任何
client 约束。
~~~yaml
gateway:
auth:
mode: any
methods:
- mode: bearer
valueFile: /run/secrets/checkout-gateway-token
client:
allowedRoutings: [checkout]
requestsPerMinute: 600
maxConcurrentConnections: 20
~~~
Gateway、Distribution、Admin 与 Provider API 是独立认证边界。改变其中一套
不得连带改变其他入口。
### 3.5 Gateway 粘性会话
gateway.stickySession 让调用方使用一个会话头尽量固定出口 Proxy。绑定键是认证
Client、已匹配 Routing 和会话标识的组合,因此不同 Client 或不同 Routing 使用
相同会话值也不会共享出口。该功能要求 Gateway 认证开启。
~~~yaml
gateway:
auth:
mode: bearer
tokenFile: /run/secrets/gateway-token
stickySession:
enabled: true
header: X-Proxy-Session
ttl: 20s
maxEntries: 100000
~~~
- header 必须是规范 HTTP Header 名;一个请求只能携带一个值,值由字母、数字、
点、下划线、连字符组成且最长 128 字节。
- ttl 是绑定的最长时长;实际到期时间还会被 Proxy 的 usableUntil、代理过期时间
和 allocation safety margin 截短。
- maxEntries 是当前 Gateway Worker 的硬上限。缓存只保存 Proxy ID 和过期时间,
不保存 Proxy 地址、凭据或原始会话标识。
- 会话头仅用于本地选路Gateway 在转发 HTTP 请求前删除它。Proxy 不再符合当前
Snapshot 或一次代理尝试失败时,绑定会被清除并按正常 Routing 重选。
- 该版本不把 Redis、PostgreSQL 或 gRPC 引入 Gateway 热路径;跨 Worker 粘性需要
上游负载均衡保持 Worker 亲和,后续由控制面快照协议扩展共享恢复能力。
## 4. Worker 控制面
控制面默认关闭;默认配置中的 `8443` 端口预留不表示服务已监听。控制面 Session、
@ -187,15 +300,21 @@ controlPlane:
- `gatewayTLS` 是 Gateway 的客户端证书、私钥和 Controller CA`tls` 的服务端
证书和 Worker CA 分离。三项可以同时省略(未运行 Gateway配置任一项时必须完整提供。
- `checkerTLS``gatewayTLS` 有相同字段和完整性校验,但必须使用独立的 Checker
证书。Controller 分别验证 `.../worker/<worker-id>`
`.../checker/<checker-id>` SPIFFE URI不能跨角色复用证书。
证书。每个客户端 X.509-SVID 叶证书必须且只能包含一个 URI SANController 分别验证
`.../worker/<worker-id>``.../checker/<checker-id>` SPIFFE URI不能跨角色复用证书。
- Controller、Gateway 与 Checker 会在每个新的控制面 TLS 握手读取最新叶证书、私钥和
信任根;文件暂时不可读时继续使用最后一次有效材料。既有 gRPC 流不强制重握手,根 CA
轮换仍需先发布包含新旧根的重叠信任包,并在宽限期后滚动排空旧连接。
Gateway 连接 Controller 时使用独立启动参数而非 `controlPlane.listen`。至少设置
`PROXY_POOL_CONTROL_PLANE_ADDRESS`、`PROXY_POOL_CLUSTER_ID`、
`PROXY_POOL_WORKER_ID`、`PROXY_POOL_INSTANCE_ID` 和 `PROXY_POOL_ZONE`,详见
[控制面协议](../api/control-plane.md#9-gateway-启动参数)。基础 Compose/Kubernetes
模板保持 `controlPlane.enabled: false`,环境 Overlay 挂载 mTLS 证书并启用后才可启动
Gateway。
[控制面协议](../api/control-plane.md#9-gateway-启动参数)。Compose 本地模板从被忽略的
开发证书目录派生 Worker/Checker 身份Kubernetes Base 保持
`controlPlane.enabled: false`,必须由具备每工作负载唯一身份的 mTLS Overlay 启用。
在 mTLS 生产工作负载中,`PROXY_POOL_AUTO_IDENTITY=true` 可省略 `WORKER_ID`
`INSTANCE_ID`:进程仅接受其 `gatewayTLS` 叶证书中唯一的
`spiffe://<trust-domain>/<environment>/worker/<worker-id>` URI并以该 ID 作为缺省实例。
Checker 同样使用独立的可拨号地址:`proxy-checker` 的 `-control-plane`
`-checker-id`、`-instance-id` 和 `-max-in-flight` 可由对应的
@ -203,9 +322,16 @@ Checker 同样使用独立的可拨号地址:`proxy-checker` 的 `-control-pla
模式只接受回环 Controller 地址。Checker 只从 gRPC 领取任务并批量上报事实,不读取
Redis/PostgreSQLController 在生产启动拓扑中装配 Redis 共享任务队列,当前调度
HTTP/HTTPS/SOCKS5 BASIC、EGRESS 和 TARGET 检查。调度监督器在每轮从已发布配置读取启用的
上游与 RoutingAdmin reload 发布后,上游/路由启停、有效 `check` 策略和目标列表会在下一轮生效,
上游与 Routing启用 Admin 时还要求 PostgreSQL 管理态与配置 revision 一致,并取两者均启用的
上游和 Routing。管理态禁用的上游不会再产生新的 BASIC、EGRESS 或 TARGET 任务;管理态禁用的
Routing 也不会再产生其 TARGET 任务revision 不一致或状态不完整时本轮失败关闭。Admin reload 发布后,
上游/路由启停、有效 `check` 策略和目标列表会在下一轮生效,
新启用的上游无需重启 Controller。
`PROXY_POOL_AUTO_IDENTITY=true` 对 Checker 使用同样规则:仅接受 `checkerTLS` 中唯一且仅有的
`spiffe://<trust-domain>/<environment>/checker/<checker-id>` URI并在未显式设置时复用其
`checker-id` 作为实例 ID。
`maxRuntimeCounters` 同时限制单个 Runtime 报告和单个 Outcome 批次的条目数。Gateway
在本地维护容量为 `65536` 的非阻塞 Outcome 队列,默认微批上限为 `512`,实际取二者中
较小值;该队列与其序列确认状态仅存在于 Gateway 进程内。Controller 的 Redis 状态只保存
@ -220,9 +346,24 @@ gateway:
auth: {mode: none}
limits:
maxConcurrentConnections: 50000
transport:
dialTimeout: 10s
handshakeTimeout: 15s
responseHeaderTimeout: 30s
idleConnTimeout: 90s
maxIdleConns: 20000
maxIdleConnsPerHost: 32
maxConnsPerHost: 32
tunnelBufferBytes: 32768
tunnelIdleTimeout: 5m
retry:
maxAttempts: 2
retryMethods: [GET, HEAD]
stickySession:
enabled: true
header: X-Proxy-Session
ttl: 20s
maxEntries: 100000
destinationPolicy:
denyPrivateNetworks: true
denyLoopback: true
@ -240,6 +381,31 @@ gateway:
- 保留地址、CGNAT 与云元数据端点始终拒绝,不能通过私网/链路本地开关放行。
- `maxConcurrentConnections` 是入口准入上限,不是 Proxy 容量上限。
### 5.1 Gateway 本地传输层
**gateway.transport** 只配置每个 Gateway Worker 的本地 HTTP Transport 和 CONNECT
隧道 I/O不读取或写入 Redis、PostgreSQL、Provider也不参与热路径分配。修改
这些字段需要滚动重启对应 Gateway Worker 后生效。
- **dialTimeout**、**handshakeTimeout**、**responseHeaderTimeout**、
**idleConnTimeout**:分别限制拨号、上游 TLS/CONNECT 握手、HTTP 响应头和空闲
HTTP 连接的时长。
- **maxIdleConns**:单个本地 HTTP Transport 的总空闲连接上限。Gateway 分别为代理
转发和 direct fallback 建立 Transport最坏情况下两者都可能保有空闲连接。
- **maxIdleConnsPerHost**:单个上游 Host 的空闲 HTTP 连接上限。
- **maxConnsPerHost**:单个上游 Host 的 HTTP 活跃加空闲连接上限。它不替代
**limits.maxConcurrentConnections**,也不限制已经建立的 CONNECT 隧道。
- **tunnelBufferBytes**:每个活跃 CONNECT 隧道的每个复制方向使用一个缓冲区,内存
预算至少按 活跃隧道数 * 2 * tunnelBufferBytes 计算。
- **tunnelIdleTimeout**CONNECT 隧道双向没有数据活动时的最长存活时间。
所有字段均为可选项;数值 0 表示沿用进程默认值:拨号 10s、握手 15s、
响应头 30s、空闲 HTTP 连接 90s、总空闲连接 1024、每 Host 空闲连接
64、隧道缓冲 32768 字节、隧道空闲 5m。计数和时长不得为负数
maxIdleConnsPerHost 不得大于显式配置的 maxIdleConns。示例中的 20000/32
是高并发 HTTP 转发的起点,不是每秒请求量的换算公式,应同时按可用文件描述符、
上游限额和观测到的复用率调节。
## 6. Distribution
```yaml
@ -291,6 +457,7 @@ routing:
- name: api-post
enabled: true
purpose: gateway
action: proxy
match:
hostRegex: '^api\\.example\\.com$'
methods: [POST]
@ -310,6 +477,10 @@ routing:
- Routing 列表有序,首条匹配后停止。
- `purpose``gateway``extract`
- `action` 省略时为 `proxy`。`proxy` 使用 Upstream 与 Strategy`direct` 直接连接已通过
目标地址策略验证的目标;`reject` 拒绝匹配请求。`direct` 与 `reject` 仅允许
`gateway` Routing且不得同时配置 `upstreams`、`strategy`、`onUnavailable` 或
`check.targets`
- `strategy.type` 支持 `sequential`、`random`、`roundRobin`、`weighted`、
`leastConnections`
- `weighted` 使用 `weights` 映射,键必须引用本 Routing 的 Upstream。
@ -317,6 +488,10 @@ routing:
`switchAfterEmptyFetch``endBehavior` 省略时默认为 `stop`,也可显式设置
`loop``stayLast`
- `onUnavailable.action``reject`、`wait` 或 `direct`;默认建议 `reject`
- `action: direct` 在 Dispatcher 前执行,不申请 Proxy 容量、不写 Proxy Outcome也不建立
粘性 Proxy 绑定它仍执行入口认证、Client 路由约束、速率/并发限制和目标地址策略。
`onUnavailable.action: direct` 仅是 `action: proxy` 在本地没有候选时的回退,二者
不可互换。
- `check.targets` 是 Routing 级 HTTP/HTTPS 探测目标。每个 Routing 最多 16 个 URL且一个
启用 Upstream 被其引用的 TARGET Profile 总数最多 64 个。每个 Profile 的身份是
`(routing.name, target URL)`;失败只影响该 Profile不改变 Proxy 的全局健康状态。
@ -325,6 +500,16 @@ Sequential 的空计数属于 Upstream当前索引属于 Routing。只有 Pro
成功、模板成功且合法候选为零时才增加空计数。错误不改变空计数;重复候选
会重置空计数但增加独立 duplicate 指标。
Controller 在连续空结果达到阈值时按顺序切换当前 Upstream到达最后一个可用
Upstream 且 `endBehavior: stop` 时,会以当前 Upstream 的 CAS 条件原子停用该
Routing。停用后的 Gateway 规则按既有 `onUnavailable.action` 执行 `reject`、`wait`
`direct`,直到一次配置重载提交新的管理快照。
管理态禁用当前 Sequential Upstream 不等待新的 Provider 空结果Controller 会按
配置顺序以相同的 `ExpectedCurrent` CAS 推进到下一个启用 Upstream并立即刷新完整
Snapshot。若当前项之后没有启用候选只有 `endBehavior: loop` 会回绕搜索更早的候选;
`stop``stayLast` 都会原子停用 Routing因为已禁用的当前项不能继续承接新分配。
## 8. Upstream
```yaml
@ -476,6 +661,11 @@ proxyAuth:
- 启用 Routing 的 `check.targets` 会为其引用的每个启用 Upstream 创建 TARGET 检查组。BASIC、
EGRESS 与 TARGET 使用固定批次和轮转顺序,并共享该 Upstream 的 `check.maxInFlight`,避免
配置多个目标后产生无界检查流量。
- 启用 Admin 时,调度仅使用配置和管理态均启用、且 revision 一致的 Upstream管理态停用会在
下一调度轮阻止新的 BASIC、EGRESS、TARGET 任务。revision 不一致、重复或缺失的上游状态按失败
关闭处理,不读取 Redis due-index。
- 启用 Admin 时TARGET Profile 还要求其 Routing 在配置和管理态中均启用;管理态停用 Routing
仅停止该 Routing 的新 TARGET 任务,不影响同一 Upstream 的 BASIC/EGRESS 或其他启用 Routing。
- 第一次有意义失败进入 SUSPECT达到 `maxConsecutiveFailures` 后才进入
UNHEALTHY。
- `unhealthyRemoveAfter` 控制 UNHEALTHY 持续多久后可由 Controller 回收;`0s`
@ -526,10 +716,20 @@ Metrics 启用时 `listen` 必须是合法 `host:port`。该入口固定提供 `
`result` 固定为 accepted、rejected。Gateway 另暴露
`proxy_pool_gateway_outcomes_total{stage,result}`
`proxy_pool_gateway_outcome_queue_dropped_total``stage` 固定为 DIAL、PROXY_HANDSHAKE、
RESPONSE_HEADERS、TUNNEL`result` 固定为 success、failure。绝不包含 Proxy、IP、Checker、
Client、路由、目标 URL 或凭据标签。Provider、提取和容量等业务指标仍在后续实施范围。Distribution 启用时 `/readyz` 只以 Redis 活动池为
RESPONSE_HEADERS、TUNNEL`result` 固定为 success、failure。Provider、Extraction、
Capacity 和 Drain 业务指标也已接入,所有标签均为固定枚举;绝不包含 Proxy、IP、Checker、
Client、路由、目标 URL 或凭据标签。Capacity 从既有 Provider 库存对账循环聚合,
不进入 Gateway 热路径。Distribution 启用时 `/readyz` 只以 Redis 活动池为
服务流量门槛PostgreSQL 故障由 Admin 接口独立报告。Metrics 开关或监听地址
变更需要重启 Controller。
变更需要重启 Controller。Controller、Gateway 和 Checker 的进程级致命错误以 JSON
结构化日志输出包含组件和稳定错误类型不输出错误原文敏感属性、URL 用户信息和
查询 Secret 在写出前统一替换为 `[REDACTED]`
Gateway 还按每个 Worker 暴露 HTTP/CONNECT 请求总数、在途请求数和活跃 CONNECT
隧道数;协议标签固定为 HTTP、CONNECT。请求数在 Handler 接受请求时增加,在途数在
全部拒绝、转发或隧道关闭后归零;活跃隧道只覆盖成功建立并开始 relay 的连接。这些
指标是连接池、入口并发、文件描述符和长连接排空的本地观测,不携带 Proxy、路由、
目标、Client 或凭据。
## 10. 启动前校验清单

View File

@ -80,10 +80,10 @@ Provider、Pool、Routing、Distribution 在首版需要共享事务和一致性
### 3.4 proxy-loadgen
- 当前实现 HTTP 请求场景:固定请求数或固定时长,受限并发与可选目标 QPS可配置方法、
重复请求头和请求体。HTTPS 目标经 HTTP Gateway 时由 Transport 走 CONNECT。
- 输出场景、延迟分位上界、错误分类、吞吐和 Go 内存/GC 快照。CONNECT 长连接、
Extract 并发、进程 CPU/RSS/句柄与网络采样仍需补齐。
- 当前实现 HTTP、CONNECT 长连接和 Extract 场景:固定请求数或固定时长,受限并发与可选
目标 QPS可配置方法、重复请求头和请求体。HTTPS 目标经 HTTP Gateway 时由 Transport 走 CONNECT。
- 输出场景、延迟分位上界、错误分类、吞吐和 Go 内存/GC 快照。故障注入、进程 CPU/RSS/句柄
与网络采样以及代表性集群报告仍需补齐。
## 4. 模块边界
@ -319,7 +319,8 @@ Upstream 的 Empty 事实全局共享;每条 Routing 独立 CAS 当前索引
协程只能有一个成功从 A 切到 B其他协程读取新版本不会再切到 C。
Sequential 至少配置两个 Upstream列表耗尽后的默认行为是 `stop``loop` 和
`stayLast` 必须显式配置。disabled Upstream 不参与新分配,其运行时跳过与权威
游标持久化仍由后续 Routing Runtime 完成。
游标由 Controller 的权威管理状态持久化。达到末端 `stop`Controller 以观察到的
当前 Upstream 作为 CAS 栅栏禁用 Routing并让完整 Snapshot 显式关闭该规则。
## 10. Exclusive Extraction

View File

@ -42,10 +42,14 @@ Gateway Worker 只读取本地不可变快照并维护本地容量计数。供
1. 接入层完成认证、来源识别、限流和目标地址检查。
2. Routing 按配置顺序首条命中。
3. Dispatcher 从本地快照筛选 Upstream、协议、标签、TTL 和健康条件。
4. 原子预留 Proxy 容量,建立到上游代理的连接。
5. 建连成功后转为 Active传输结束后释放失败则取消预留。
6. GET/HEAD 仅在响应提交前按策略重试CONNECT 建立后不重放。
3. 若启用并携带粘性会话头,先在本 Worker 的有界缓存中按认证 Client、Routing
和会话标识查找仍符合当前快照的 Proxy。
4. Dispatcher 从本地快照筛选 Upstream、协议、标签、TTL 和健康条件;失效会话
自动清除并按常规策略重新选择。
5. 原子预留 Proxy 容量,建立到上游代理的连接。
6. 建连成功后转为 Active并在需要时写入不超过 Proxy 可用期的会话绑定;
传输结束后释放,失败则取消预留并清除该绑定。
7. GET/HEAD 仅在响应提交前按策略重试CONNECT 建立后不重放。
### 4.2 独占提取

View File

@ -53,10 +53,10 @@ deadline 内执行 HTTP/HTTPS/SOCKS5 BASIC、EGRESS 和 TARGET 探测并微批
### proxy-loadgen
负载工具当前生成有界 HTTP 请求,支持经 Gateway 请求 HTTPS 目标、固定请求数或时长、
目标 QPS、可重复请求头和请求体、连接复用和 JSON 指标输出。延迟统计使用固定大小直方图,不会因长时间高 QPS
运行积压样本。CONNECT 长连接、Extract 和故障注入场景仍待补齐;它是 100k QPS 结论的
证据工具,不是业务进程。
负载工具当前生成有界 HTTP、CONNECT 长连接和 Extract 请求,支持经 Gateway 请求 HTTPS 目标、
固定请求数或时长、目标 QPS、可重复请求头和请求体、连接复用和 JSON 指标输出。延迟统计使用
固定大小直方图,不会因长时间高 QPS 运行积压样本。故障注入和代表性集群报告仍待补齐;它是
100k QPS 结论的证据工具,不是业务进程。
## 3. 依赖方向

View File

@ -54,12 +54,12 @@ test/{fixtures,integration,e2e,load}/
**Files:** `go.mod`, `.golangci.yml`, `README.md`, `scripts/verify.ps1`,
`.github/workflows/ci.yml`
- [ ] Create module `github.com/proxy-pool/proxy-pool` with Go 1.26.
- [ ] Pin YAML v4, pgx/v5, go-redis/v9, gRPC, protobuf, Prometheus, and x/sync.
- [x] Create Go module `proxy-pool` with Go 1.26.
- [x] Pin YAML v4, pgx/v5, go-redis/v9, gRPC, protobuf, Prometheus, and x/sync.
- [x] Add `scripts/verify.ps1` that runs format check, `go vet`, unit tests, race tests,
and builds all commands, each test command bounded to 60 seconds.
- [x] Add CI for Windows and Linux with unit/race/build jobs.
- [ ] Verify `go mod tidy`, `go test ./...`, and `go build ./cmd/...` succeed.
- [x] Verify `go mod tidy`, `go test ./...`, and `go build ./cmd/...` succeed.
## Task 2: Strict Configuration
@ -80,14 +80,18 @@ test/{fixtures,integration,e2e,load}/
- [x] Implement Proxy fields, UTC TTL precedence, canonical host/port, and unique key.
- [x] Implement state transitions and reject illegal transitions.
- [ ] Implement sharded runtime counters with CAS Reserve, Commit, Cancel, Release.
- [x] Implement packed per-Proxy runtime counters with CAS Reserve, Commit, Cancel, Release.
- [x] Prove with 1,000 concurrent goroutines that effective capacity is never exceeded.
- [ ] Add race coverage and duplicate-release invariant metrics hook.
- [x] Add race coverage and duplicate-release invariant metrics hook.
当前进度2026-07-29):固定 Max 下的每 Proxy 打包 CAS、Cancel/Commit/Release
当前进度2026-08-02):固定 Max 下的每 Proxy 打包 CAS、Cancel/Commit/Release
生命周期、重复终结、错误顺序和同一 Reservation 并发终结已通过领域测试;已退出
Proxy 会停止新预留、保留非零 Drain 计数,并在归零后由后续 Snapshot Apply 回收。
动态降容契约、低基数不变量指标和 Linux race 证据仍待完成。
Snapshot 降低 `maxConcurrency` 时会保留既有 Active 工作、禁止新预留,直到运行计数
低于新上限;运行态复用、降容和恢复预留均有回归测试。领域的
`CapacityInvariantObserver` 只发出固定生命周期违规枚举Gateway 指标
`proxy_pool_gateway_capacity_invariant_violations_total{operation}` 不含 Proxy、请求或
Worker 维度;该测试纳入 Linux CI 的 race 范围,本机因 `CGO_ENABLED=0` 未执行 race。
## Task 4: Routing and Sequential Switching
@ -97,12 +101,18 @@ Proxy 会停止新预留、保留非零 Drain 计数,并在归零后由后续
- [x] Implement random, round-robin, weighted, least-connections, and sequential.
- [x] Model upstream empty counters separately from per-routing current indexes.
- [x] Implement versioned CAS switch so simultaneous threshold observers advance once.
- [ ] Cover four-empty-then-success, five-empty, A-to-B-only, disabled references,
- [x] Cover four-empty-then-success, five-empty, A-to-B-only, disabled references,
end behavior, and explicit onUnavailable.
当前进度2026-07-29领域构造器与严格配置已统一 Sequential 至少两个
Upstream、`endBehavior` 默认 `stop`并覆盖列表末端停止disabled candidate、
跨实例恢复和 `onUnavailable` 运行链仍待完成。
当前进度2026-08-02领域构造器与严格配置已统一 Sequential 至少两个
Upstream、`endBehavior` 默认 `stop`并覆盖列表末端停止。Provider Stats 现为每次
连续空结果分配单调代次Controller 的公共 Sequential 协调器只接收有界通知,在独立
循环中读取权威配置、管理快照和 Stats并以既有 `ExpectedCurrent` CAS 自动切换。
它会跳过禁用 Upstream支持 `loop`/`stayLast`,同一空结果代次不会在循环后重复切换,
成功后立即广播完整 Snapshot。末端 `stop` 使用独立 `DisableRouting` 公用命令,
`ExpectedCurrent` CAS 原子停用 Routing并在同一权威事务写入审计与 Outbox
内存与 PostgreSQL Adapter 都运行相同并发、幂等和失败回滚契约。Gateway 已执行
reject/wait/direct并会在快照刷新后看到停用状态。
## Task 5: Provider Fetch Classification and Scheduling
@ -120,7 +130,7 @@ Upstream、`endBehavior` 默认 `stop`并覆盖列表末端停止disabled
**Files:** `internal/controller/pool/*.go`, `internal/domain/upstream/pool.go`, tests
- [ ] Compute Available Slots from eligible Proxy capacity, Active, Reserved, TTL,
- [x] Compute Available Slots from eligible Proxy capacity, Active, Reserved, TTL,
health, ownership, pending expected fetch, and gateway reserve.
- [x] Implement pool.maxSize and fetch.maxTotal as distinct counters.
- [x] Allocate each Proxy to one Worker with epoch/version/expiry ownership.
@ -180,8 +190,10 @@ Upstream、`endBehavior` 默认 `stop`并覆盖列表末端停止disabled
- [x] Implement the Worker heartbeat receiving path and session lifecycle.
- [x] Keep Provider output in Redis TTL activity state and node memory only; keep the
Gateway request path on immutable local snapshots with no Redis/PostgreSQL calls.
- [x] Expose Distribution extraction/status and Admin status/enable/disable/switch/reload
HTTP handlers and contracts.
- [x] Expose Distribution extraction/status and Admin status/audit/enable/disable/switch/reload
HTTP handlers and contracts with credential-level endpoint permissions and
Distribution credential-level extraction boundaries; enforce matched
Gateway credential routing boundaries before local dispatch.
- [x] Add Compose-backed Redis 8.2 integration and shared Adapter contract tests.
- [x] Add PostgreSQL management Adapter and Compose-backed integration tests.
@ -205,8 +217,8 @@ Distribution/Admin 服务构造、错误合并和资源关闭。生产 Provider
停机Admin disable 会取消 Runtimereload 在提交前预检并在发布后替换运行实例。
组合 fixture 已验证隔离 Redis namespace 下的选主、Provider HTTP 调用、模板解析
和活动池写入。Controller Metrics 独立入口现已提供 `/livez`、`/readyz` 与基础
Prometheus 运行时指标,三监听器隔离已通过测试;业务指标仍待
实现。双存储 bootstrap 已通过 PostgreSQL 18 + Redis 8.2 组合 fixture覆盖
Prometheus 运行时指标,三监听器隔离已通过测试;Checker、Gateway、Drain、
Provider、Extraction 与容量业务指标已接入;进程级密钥安全结构化日志也已落地。双存储 bootstrap 已通过 PostgreSQL 18 + Redis 8.2 组合 fixture覆盖
迁移、启动配置提交、Readiness、Admin Status 和 Metrics 探针。
WorkerControlPlane 现已接入 Controller 生命周期Register、ACK 和 Runtime
@ -302,12 +314,20 @@ Ticket/排除快照/Runtime 零计数闭环。
Controller 即可生效。Redis 任务存储现已扩展 BASIC/EGRESS/TARGET 的独立有界索引;路由目标
Profile 在启用 Routing 与 Upstream 的组合上才进入调度。
补充进度2026-08-02配置停用已进入 ownership Drain 编排。Controller 以静态配置
与 Admin 管理态的交集生成完整、版本化的 Upstream 策略视图Redis 只从对应 `owned` 索引
有界读取仍可用的已归属 Proxy。候选携带 Upstream revision`ownership.lua` 在复用既有
Ticket/排除 Snapshot/Runtime 零计数闭环前原子复核策略仍为停用、Proxy source、Worker、
assignment epoch 和未过期租约。重启用后的策略 revision 会使旧候选返回无操作。Provider
Supervisor 也改为同时服从静态配置与管理态,消除两条启停消费链的不一致。
补充进度2026-08-02已新增 `proxy-loadgen` HTTP 与 CONNECT 长连接场景。固定请求数
和固定时长两种模式均通过固定 worker 数与有界派发通道执行,可选 QPS 限速;报告使用固定大小
延迟直方图输出状态分类、CONNECT 建立数、Extract 校验数、吞吐和 Go 内存/GC 快照。CONNECT
以原始 TCP 握手连接 HTTP Gateway建连成功后按 `hold` 保持,且不透明读取隧道内容。`extract`
场景会自动生成独立 Request/Idempotency 标识,校验返回数量与单响应 ID 唯一性,且不记录地址或
凭据。故障注入以及代表性集群报告仍未实现。
凭据。报告可选按最大错误率和 p99 延迟门禁;违反阈值时仍完整写出 JSON 证据并以
专用退出码失败。故障注入以及代表性集群报告仍未实现。
## Task 12: Machine-readable Contracts
@ -318,24 +338,27 @@ Profile 在启用 Routing 与 Upstream 的组合上才进入调度。
and idempotency behavior.
- [x] Specify Worker register, snapshot, delta, ACK, report, heartbeat, ownership drain,
and resync messages.
- [ ] Validate OpenAPI and compile protobuf descriptors in CI.
- [x] Validate OpenAPI contracts and compile protobuf descriptors/generated-code drift in CI.
当前进度2026-07-29Distribution/Admin OpenAPI 已由 Go 测试在双平台 CI
校验本地 `$ref` 闭合、operationId 唯一、响应存在及 security scheme 引用;
`scripts/verify-proto.ps1` 已可复现编译包含 imports/source info 的 descriptor
并在本地存在 `protoc` 时进入完整验证;完整 OAS 工具验证与 CI 强制安装/执行
`protoc` 仍待完成。
并使用 SHA-256 固定的 `protoc` 35.0 安装器在 CI 完整验证 descriptor 与生成代码漂移;
OpenAPI 结构契约由 Go 测试在双平台 CI 执行;`scripts/verify-openapi.ps1` 固定
`@redocly/cli@2.25.4`,按 OpenAPI 3.1 最小规则集验证两份文档并将 Tag 描述作为错误。
## Task 13: Deployment and Observability
**Files:** `deploy/**`, `internal/platform/**`, `docs/operations/**`
- [ ] Add Compose for local Controller/Gateway/Checker/PostgreSQL/Redis/Prometheus/
- [x] Add Compose for local Controller/Gateway/Checker/PostgreSQL/Redis/Prometheus/
Grafana/HAProxy.
- [ ] Add Kubernetes Deployments, Services, PDBs, HPA, NetworkPolicy, Secrets examples,
- [x] Add Kubernetes Deployments, Services, PDBs, HPA, NetworkPolicy, Secrets examples,
probes, resource limits, topology spread, and graceful termination.
- [ ] Add low-cardinality Prometheus metrics and structured secret-safe logs.
- [ ] Document backup, recovery, rollout, rollback, capacity, kernel, file descriptor,
- [ ] Add the production Kubernetes mTLS identity overlay, certificate rotation and unique
elastic Worker identity wiring.
- [x] Add low-cardinality Prometheus metrics and structured secret-safe logs.
- [x] Document backup, recovery, rollout, rollback, capacity, kernel, file descriptor,
NAT/conntrack, and incident runbooks.
当前进度2026-08-02已接入 Checker 任务流指标
@ -344,7 +367,22 @@ Profile 在启用 Routing 与 Upstream 的组合上才进入调度。
`BASIC`、`EGRESS`、`TARGET` 级别及 `accepted`、`rejected` 结果。Gateway 还暴露
`proxy_pool_gateway_outcomes_total{stage,result}`
`proxy_pool_gateway_outcome_queue_dropped_total`其中阶段和结果均为固定枚举。Provider、
提取、容量指标与密钥安全的结构化日志仍待补齐。
拉取已暴露 `proxy_pool_controller_provider_fetch_results_total{class}`
`proxy_pool_controller_provider_valid_candidates_total`
`proxy_pool_controller_provider_new_proxies_total``class` 仅允许 `valid`、`empty`、
`duplicate_only`、`error`。Extraction 还暴露固定 `result` 的请求次数、请求数与响应
交付数,幂等重放按响应交付统计。容量指标由既有 Provider 库存对账周期聚合,不进入
Gateway 热路径。`platform/logging` 以 JSON `slog` 输出进程级致命错误字段、URL
用户信息、查询 Secret 和错误对象均经过脱敏,且不在请求热路径逐条写日志。
CI 另有独立 Deployment job在占位凭据下渲染 Compose并使用 `kubectl kustomize`
渲染 Kubernetes base 和单副本 development mTLS Overlay该 job 同时展开测试 Compose、
校验本地 Compose 配置。它不启动容器、不访问真实存储或密钥。
独立 Image job 会构建 Linux 多阶段应用镜像,覆盖 Dockerfile、Linux 交叉编译与运行时层;
容器服务编排和业务运行链路仍需在具备镜像网络的环境中验证。
development Overlay 已挂载独立 Controller/Gateway/Checker TLS Secret、启用 Checker 和最小
网络策略,并从固定开发证书派生 `gateway-a`、`checker-a` 身份且 HPA 锁为单副本,不能代替生产证书轮换。
控制面 TLS 叶证书和信任根已在每次新握手时从文件加载,短暂文件不一致时保留最后一次有效材料;
生产 Overlay 仍必须由工作负载身份系统提供每副本唯一 SPIFFE URI 和弹性身份注入。
## Task 14: Documentation, Examples, and Diagrams
@ -354,14 +392,27 @@ Profile 在启用 Routing 与 Upstream 的组合上才进入调度。
reference, API guide, deployment guide, security model, testing guide, and roadmap.
- [x] Provide at least 20 validated configuration examples.
- [x] Provide at least 30 Mermaid architecture, flow, sequence, state, and failure diagrams.
- [ ] Generate `proxy-pool-docs-v1.0.zip` from versioned documentation assets.
- [x] Generate `proxy-pool-docs-v1.0.zip` from versioned documentation assets.
`scripts/package-docs.ps1` 会将 README、`docs/`、图表、OpenAPI、Proto 契约与部署手册复制至临时目录,
生成包含 Git revision、文件大小和 SHA-256 的 `manifest.json`,再以临时 ZIP 原子替换目标。
默认输出为被 Git 忽略的 `dist/proxy-pool-docs-v1.0.zip`Go 回归测试实际执行脚本并校验归档内容。
## Task 15: Completion Audit
- [ ] Map every requirement ID to code, test, contract, document, or verified runtime evidence.
- [x] Map every requirement ID to code, test, contract, document, or verified runtime evidence.
- [ ] Run `gofmt`, `go vet`, unit tests, race tests, builds, contract validation, and
documentation link/example validation.
- [ ] Run bounded local performance benchmarks; label 100k QPS as unverified until a
- [x] Run bounded local performance benchmarks; label 100k QPS as unverified until a
representative cluster load run exists.
- [ ] Confirm no TODO/TBD/placeholders, secrets, unbounded queues, high-cardinality metric
`scripts/benchmark-gateway.ps1` 固定使用 `-count=1 -benchtime=1x` 分别执行 100k 索引调度、
Round Robin 和 Snapshot Apply 基准,并将 Git revision 与原始输出保存到 `dist/`。该基线不包含
网络、TLS、上游 RTT、真实连接或多 Worker 协调,不能替代代表性集群报告。
- [x] Confirm no TODO/TBD/placeholders, secrets, unbounded queues, high-cardinality metric
labels, extraction Lease APIs, or conflicting maxSize semantics remain.
`docs` 回归测试扫描生产源与部署清单中的未完成标记和私钥材料OpenAPI 测试拒绝
Extract Lease/Release/Renew 路径Gateway Outcome Queue 构造器和配置校验拒绝无界容量,
指标/部署契约拒绝高基数标签,`pool.maxSize` 与累计 `fetch.maxTotal` 的边界由配置和
活动池契约测试共同锁定。示例、CI fixture 与环境变量引用不是未实现占位或生产 Secret。

View File

@ -77,6 +77,61 @@ func TestDocumentedGoCommandsReferenceExistingTargets(t *testing.T) {
}
}
func TestRunbookCoversHostAndNetworkCapacityBaseline(t *testing.T) {
content := readDocument(t, filepath.Join(repositoryRoot(t), "docs", "operations", "runbook.md"))
for _, required := range []string{
"## 8. 备份与恢复",
"## 10. 主机与网络容量基线",
"ulimit -n",
"conntrack",
"NAT",
"回滚",
} {
if !strings.Contains(content, required) {
t.Errorf("runbook does not document %q", required)
}
}
}
func TestRepositoryAuditRejectsUnfinishedSourceAndPrivateKeyMaterial(t *testing.T) {
repositoryRoot := repositoryRoot(t)
unfinished := regexp.MustCompile(`(?i)\b(?:TODO|TBD|FIXME)\b`)
privateKey := regexp.MustCompile(`-----BEGIN (?:[A-Z ]+ )?PRIVATE KEY-----`)
for _, directory := range []string{".github", "api", "cmd", "deploy", "internal", "scripts"} {
root := filepath.Join(repositoryRoot, directory)
err := filepath.WalkDir(root, func(path string, entry os.DirEntry, err error) error {
if err != nil {
return err
}
if entry.IsDir() {
if entry.Name() == ".control-plane-tls" || strings.HasPrefix(entry.Name(), ".tmp") || entry.Name() == "dist" {
return filepath.SkipDir
}
return nil
}
switch strings.ToLower(filepath.Ext(path)) {
case ".go", ".yaml", ".yml", ".ps1":
default:
return nil
}
content, readErr := os.ReadFile(path)
if readErr != nil {
return readErr
}
if unfinished.Match(content) {
t.Errorf("%s contains an unfinished implementation marker", relativeTestName(repositoryRoot, path))
}
if privateKey.Match(content) {
t.Errorf("%s contains private key material", relativeTestName(repositoryRoot, path))
}
return nil
})
if err != nil {
t.Fatalf("audit %s: %v", directory, err)
}
}
}
func markdownDocuments(t *testing.T, repositoryRoot string) []string {
t.Helper()
documents := []string{filepath.Join(repositoryRoot, "README.md")}

View File

@ -2,28 +2,29 @@
## 架构与一致性
- [ ] Gateway 热路径依赖审计确认无 PostgreSQL、Redis、Provider 或模板执行。
- [ ] 每个 Proxy 同一时刻最多归属一个 Workerownership epoch 单调。
- [ ] Reserved -> Active 使用单个原子转换,无超卖与负计数。
- [ ] Sequential 并发 Empty 只切换一次,旧 Upstream Proxy 自然耗尽。
- [x] Gateway 热路径依赖审计确认无 PostgreSQL、Redis、Provider 或模板执行。
- [x] 每个 Proxy 同一时刻最多归属一个 Workerownership epoch 单调。
- [x] Reserved -> Active 使用单个原子转换,无超卖与负计数。
- [x] Sequential 并发 Empty 只切换一次,旧 Upstream Proxy 自然耗尽;末端 `stop`
使用权威 CAS 停用 Routing 并刷新 Gateway 快照。
- [x] `pool.maxSize``fetch.maxTotal` 分别按当前库存和累计获取计数。
- [ ] Extract 只有 `AVAILABLE -> EXTRACTED`OpenAPI 不存在 release/renew。
- [ ] Extract 状态更新和短期幂等结果位于同一个 Redis 原子操作。
- [ ] PostgreSQL 中不存在 Proxy 明细或逐次提取记录。
- [ ] `partial``allOrNothing` 均通过 Redis 原子并发测试。
- [ ] `reserveForGateway` 在所有提取路径上统一执行。
- [x] Extract 只有 `AVAILABLE -> EXTRACTED`OpenAPI 不存在 release/renew。
- [x] Extract 状态更新和短期幂等结果位于同一个 Redis 原子操作。
- [x] PostgreSQL 中不存在 Proxy 明细或逐次提取记录。
- [x] `partial``allOrNothing` 均通过 Redis 原子并发测试。
- [x] `reserveForGateway` 在所有提取路径上统一执行。
## 安全
- [ ] 非回环监听均配置 Auth 或 allowCIDRs严格模式已开启。
- [ ] Gateway、Distribution、Admin 凭据和权限相互独立。
- [x] 非回环监听均配置 Auth 或 allowCIDRs严格模式已开启。
- [x] Gateway、Distribution、Admin 凭据和权限相互独立。
- [ ] trusted proxy 只包含受控 LoadBalancer/Ingress 网段。
- [ ] 解析前后均拦截私网、回环、链路本地、元数据地址与 DNS Rebinding。
- [x] 解析前后均拦截私网、回环、链路本地、元数据地址与 DNS Rebinding。
- [ ] Secret 由外部密钥系统注入镜像、ConfigMap、日志没有明文。
- [ ] 所有 Controller 使用同一枚至少 32 字节的独立配置 HMAC 密钥,并完成轮换演练。
- [ ] Pod 以非 root、只读根文件系统、无 Linux capabilities 运行。
- [ ] NetworkPolicy 默认拒绝,外部数据库/Redis/Provider 网段已收紧。
- [ ] Provider 模板有响应大小、执行时间、函数与外部访问限制。
- [x] Provider 模板有响应大小、执行时间、函数与外部访问限制。
## 可用性
@ -31,9 +32,9 @@
- [ ] Redis 跨可用区并受监控,完成丢失后由 Provider 重建活动池的演练。
- [ ] Gateway、Controller、Checker 均跨主机/可用区分散。
- [ ] PDB、优雅终止与最大连接时长的组合经过驱逐测试。
- [ ] Controller 断线时 Gateway 在 `maxStaleAge` 内继续,超限拒绝新请求。
- [ ] Worker 崩溃后 ownership 只在租约过期后再分配。
- [ ] Snapshot 版本缺口触发全量同步,旧 Delta 被拒绝。
- [x] Controller 断线时 Gateway 在 `maxStaleAge` 内继续,超限拒绝新请求。
- [x] Worker 崩溃后 ownership 只在租约过期后再分配。
- [x] Snapshot 版本缺口触发全量同步,旧 Delta 被拒绝。
## 性能
@ -41,15 +42,15 @@
- [ ] 完成 10k 稳态、100k 峰值、CONNECT 活跃连接和建连速率独立测试。
- [ ] 在最大可用区失效时仍满足容量和延迟 SLO。
- [ ] 目标利用率不高于 60%HPA 缩容稳定窗口不低于 10 分钟。
- [ ] 队列、buffer、日志和结果上报全部有界。
- [x] 队列、buffer、日志和结果上报全部有界。
- [ ] p99 Dispatch 预算、端到端延迟、错误率、CPU、RSS、FD 和网络有原始证据。
## 观测与值班
- [ ] 仪表盘覆盖 Gateway、Routing、Upstream、Provider、Extract、Snapshot、Checker
- [x] 仪表盘覆盖 Gateway、库存、Provider、Extract、Snapshot、Checker 和 Drain
- [ ] 告警有负责人、严重级别、Runbook 链接和演练记录。
- [ ] 指标无 Proxy IP、Client ID、Session、完整 URL 或 request ID 高基数标签。
- [ ] 日志脱敏已用真实 Secret fixture 验证。
- [x] 指标无 Proxy IP、Client ID、Session、完整 URL 或 request ID 高基数标签。
- [x] 日志脱敏已用真实 Secret fixture 验证。
- [ ] 值班人员完成 PostgreSQL、Redis、Snapshot、容量与 Extract 故障演练。
## 发布门禁

View File

@ -24,11 +24,10 @@
`cmd/proxy-controller` 已完成配置单次加载、PostgreSQL 迁移、Redis 活动池、
Distribution/Admin/Metrics 独立监听和有界停机装配。Provider 自动补池、分布式
配额、动态重载和 Admin 低基数统计已装配Controller 已装配 Redis BASIC/EGRESS/TARGET 任务 broker
`proxy-checker` 可执行 HTTP/HTTPS/SOCKS5 BASIC/EGRESS/TARGET 探测。`proxy-loadgen` 已提供有界 HTTP
请求场景CONNECT 长连接/Extract 压测与完整 mTLS 环境 Overlay 仍属于
`implementation-plan.md` 后续任务。
因此 Compose/Kubernetes 资产当前仍用于评审网络、资源、探针和依赖关系,不能
视为完整可运行拓扑。
`proxy-checker` 可执行 HTTP/HTTPS/SOCKS5 BASIC/EGRESS/TARGET 探测。`proxy-loadgen` 已提供有界 HTTP、
CONNECT 长连接和 Extract 场景,并可按最大错误率和 p99 延迟输出机器可判定的验收结果;
它不替代代表性集群压测。Compose 已声明本地 Controller/Gateway/Checker mTLS 拓扑,
但本机尚未完成容器端到端启动Kubernetes 仍需要环境专属的工作负载身份 Overlay。
### 2.1 前置条件
@ -50,6 +49,13 @@ $env:PROVIDER_A_TOKEN = "PROVIDER_A_TOKEN"
$env:PROVIDER_B_TOKEN = "PROVIDER_B_TOKEN"
```
首次启动 Compose 前,生成不入库的本地控制面证书。生成器要求目标目录为空,避免覆写
已有密钥:
```powershell
.\scripts\generate-local-controlplane-certs.ps1
```
### 2.3 静态检查
```powershell
@ -57,6 +63,10 @@ docker compose -f deploy/docker-compose.yml config
kubectl kustomize deploy/kubernetes/base > rendered.yaml
```
Compose 启动后Gateway 的 `/readyz` 需要先接收有效 SnapshotChecker 的 `/readyz`
需要成功建立一次任务领取流,并会在后续领取失败时回到未就绪。两者的 `/livez` 只表示
进程仍在运行。
本地 Compose 的 Redis 只作为可重建短效状态 fixture固定使用
`--appendonly no --save ""`,且不挂载 `/data` 或命名卷。真实 Redis 8.2 契约可
通过 `.\scripts\test-redis.ps1` 执行;脚本使用唯一命名空间并在结束时定向清理,
@ -103,6 +113,10 @@ Fetch 应表现为 Error 与退避,不应增加 Empty 计数,也不影响已
4. 根据集群 CNI 能力收紧 NetworkPolicy 的外部网段。
5. 在预发布环境完成数据库向前兼容迁移,再发布 Controller。
环境 Overlay 应保持 ConfigMap 内容哈希,令配置变化进入 Deployment Pod 模板并触发滚动更新。
由外部系统以固定名称注入的 Secret 不会天然改变 Pod 模板;完成 Secret 轮换后,必须按受影响
工作负载执行受控滚动,并先验证新旧信任根的重叠窗口。
### 3.2 服务端应用顺序
```bash
@ -138,6 +152,12 @@ Proxy ID、IP、Checker ID、目标 URL、Client ID 或凭据加入指标标签
Gateway 指标同样使用固定标签集:
每个 Gateway Worker 还暴露 requests_total、requests_in_flight 与 active_tunnels
三类本地连接指标。protocol 标签只允许 HTTP、CONNECT请求指标覆盖从 Handler
接受请求到全部转发、拒绝或隧道关闭的生命周期。活跃隧道只在成功写出 CONNECT 200
并开始 relay 后增加,关闭后立即减少。这些指标用于核对入口准入、连接池调优、
文件描述符预算和长连接排空,不得按 Proxy、路由、目标、Client 或凭据拆分。
- `proxy_pool_gateway_outcomes_total{stage,result}`:每次代理尝试在最远完成阶段的成功/失败数量。
- `proxy_pool_gateway_outcome_queue_dropped_total`Outcome 本地有界队列已满后丢弃的观测数量。
@ -270,7 +290,9 @@ Outbox 发布器必须以稳定 consumer ID 有界领取;发布成功后原子
2. 429 尊重 `Retry-After`,其余 Error 使用指数退避和 jitter。
3. Error 不增加 Empty合法候选为零才增加 Empty。
4. Duplicate-only 重置 Empty 并记录独立指标。
5. 达到 Empty 阈值后每条受影响 Routing 只原子切换一次。
5. 达到 Empty 阈值后每条受影响 Routing 只原子切换一次;最后一个可用 Upstream
仍持续 Empty 且配置为 `stop` 时,确认审计中存在 `disable_routing`
`routing.disabled` Outbox 事件,并检查 Gateway 已收到关闭该 Routing 的完整快照。
### 7.5 Gateway 容量耗尽
@ -331,3 +353,46 @@ ownership epoch、活动池 TTL、短期幂等窗口、outbox 和配置 Revision
5. 所有副本应用后撤销旧 Secret。
Proxy 凭据轮换必须增加 `credentialVersion`,确保唯一键不会把新旧凭据错误合并。
控制面 mTLS 的叶证书、私钥和信任根会在每次新的 TLS 握手时重新读取Kubernetes 投影
Secret 或 CSI 写入短暂不完整时保留最后一次有效材料。既有 gRPC 长连接不会被强制替换,
因此根 CA 轮换必须先发布包含新旧根的重叠信任包、确认新连接成功,再按 PDB 约束滚动
重启或排空旧 Worker最后才撤销旧根并使旧流重新认证。
## 10. 主机与网络容量基线
在预发布压测和每次生产扩容前,记录 Gateway 节点、容器运行时和 Pod 内的下列只读基线;
不要在故障处理中临时提高内核限制,任何变更都必须经过压测和变更评审。
```bash
ulimit -n
cat /proc/self/limits | grep 'open files'
sysctl fs.file-max net.core.somaxconn net.ipv4.ip_local_port_range
test -r /proc/sys/net/netfilter/nf_conntrack_max && cat /proc/sys/net/netfilter/nf_conntrack_max
ss -s
```
同时从节点或受监控的 NAT 网关采集 conntrack 使用率、丢包、重传、SYN backlog、TIME_WAIT、
端口分配失败与 SNAT 端口耗尽事件。`conntrack -L` 会遍历整个表,不得在高峰时将它作为
常规排障命令;优先使用节点监控或 `conntrack -S` 的聚合计数。
Gateway 的文件描述符预算必须按实际连接模型计算,而不是仅按 QPS每条活跃 CONNECT
隧道通常占用客户端和上游两个 socketHTTP 上游并发、idle 连接池、监听 socket、日志和
运行时也会占用描述符。部署前确认 Pod 内 `ulimit -n`、容器运行时 `LimitNOFILE` 与节点
`fs.file-max` 均高于下式结果并保留至少 20% 余量:
```text
fd_budget = 2 * max_active_connect_tunnels
+ max_concurrent_http_upstreams
+ max_idle_transport_connections
+ process_reserve
```
如果节点通过 NAT 访问上游,源端口和 conntrack 表同样构成硬上限。压测报告必须记录每个
节点的 egress IP 数、可用临时端口范围、NAT/SNAT 设备限制与单目标连接分布;单一 egress IP
不足时应在发布前增加 egress IP 或拆分节点池,不能依靠无限重试掩盖端口耗尽。
建议将 FD、conntrack、NAT 端口利用率、SYN overflow、TCP retransmit、TIME_WAIT、CPU、RSS
和 Gateway p99 作为同一份容量证据采集。任何一项超过预设预警线时停止扩容或发布,先降低
新连接速率并保留现有 CONNECT 隧道排空;回滚到兼容镜像或配置 revision 后,再根据原始
指标分析根因。

114
docs/package_test.go Normal file
View File

@ -0,0 +1,114 @@
package docs_test
import (
"archive/zip"
"encoding/json"
"os"
"os/exec"
"path/filepath"
"strings"
"testing"
)
type docsPackageManifest struct {
Version string `json:"version"`
Revision string `json:"revision"`
Files []struct {
Path string `json:"path"`
SHA256 string `json:"sha256"`
} `json:"files"`
}
func TestGatewayBenchmarkScriptIsBoundedAndWritesEvidence(t *testing.T) {
pwsh, err := exec.LookPath("pwsh")
if err != nil {
t.Skip("pwsh is required to exercise the gateway benchmark script")
}
repositoryRoot := repositoryRoot(t)
outputPath := filepath.Join(t.TempDir(), "gateway-benchmarks.txt")
command := exec.Command(pwsh, "-NoProfile", "-NonInteractive", "-File", filepath.Join(repositoryRoot, "scripts", "benchmark-gateway.ps1"), "-OutputPath", outputPath)
command.Dir = repositoryRoot
if output, err := command.CombinedOutput(); err != nil {
t.Fatalf("run gateway benchmarks: %v\n%s", err, output)
}
payload, err := os.ReadFile(outputPath)
if err != nil {
t.Fatalf("read benchmark evidence: %v", err)
}
evidence := string(payload)
for _, required := range []string{
"BenchmarkAcquire100k(Indexed|RoutingRoundRobin)",
"BenchmarkStoreApply100k",
"-benchtime=1x",
"-count=1",
"revision:",
} {
if !strings.Contains(evidence, required) {
t.Errorf("benchmark evidence does not contain %q", required)
}
}
}
func TestPackageDocsCreatesTraceableArchive(t *testing.T) {
pwsh, err := exec.LookPath("pwsh")
if err != nil {
t.Skip("pwsh is required to exercise the documentation packaging script")
}
repositoryRoot := repositoryRoot(t)
outputPath := filepath.Join(t.TempDir(), "proxy-pool-docs-v1.0.zip")
if err := os.WriteFile(outputPath, []byte("incomplete"), 0o600); err != nil {
t.Fatalf("seed old documentation archive: %v", err)
}
command := exec.Command(pwsh, "-NoProfile", "-NonInteractive", "-File", filepath.Join(repositoryRoot, "scripts", "package-docs.ps1"), "-OutputPath", outputPath)
command.Dir = repositoryRoot
if output, err := command.CombinedOutput(); err != nil {
t.Fatalf("package docs: %v\n%s", err, output)
}
archive, err := zip.OpenReader(outputPath)
if err != nil {
t.Fatalf("open documentation archive: %v", err)
}
defer archive.Close()
entries := make(map[string]*zip.File, len(archive.File))
for _, entry := range archive.File {
entries[entry.Name] = entry
}
for _, required := range []string{
"README.md",
"deploy/README.md",
"deploy/kubernetes/overlays/development-mtls/README.md",
"docs/design/architecture.md",
"docs/operations/runbook.md",
"diagrams/README.md",
"api/openapi/proxy-pool.yaml",
"api/proto/controlplane/v1/controlplane.proto",
"manifest.json",
} {
if _, exists := entries[required]; !exists {
t.Errorf("documentation archive does not contain %s", required)
}
}
manifestEntry, exists := entries["manifest.json"]
if !exists {
return
}
reader, err := manifestEntry.Open()
if err != nil {
t.Fatalf("open manifest: %v", err)
}
defer reader.Close()
var manifest docsPackageManifest
if err := json.NewDecoder(reader).Decode(&manifest); err != nil {
t.Fatalf("decode manifest: %v", err)
}
if manifest.Version != "v1.0" || manifest.Revision == "" || len(manifest.Files) < 20 {
t.Fatalf("manifest = %+v", manifest)
}
for _, file := range manifest.Files {
if file.Path == "" || file.SHA256 == "" {
t.Fatalf("invalid manifest file = %+v", file)
}
}
}

View File

@ -11,14 +11,18 @@
- 产品设计、总体架构、项目结构、四项 ADR。
- 开发、配置、Distribution/Admin API、控制面协议、安全、测试、运维文档。
- 20 个配置场景和 35 张 Mermaid 架构/流程/状态/故障图。
- `scripts/package-docs.ps1` 可生成版本化 ZIP包含 README、文档、图表、OpenAPI、Proto
契约,以及带 Git revision 和 SHA-256 的文件清单。
### 机器契约
- Distribution OpenAPI一次性独占提取、partial/allOrNothing、幂等键、
Redis TTL 活动池原子语义、TTL/健康过滤结果与标准错误。
- Admin OpenAPI状态、Upstream 启停、Routing 切换和配置重载。
- Admin OpenAPI状态、按 ID 游标分页的权威审计查询、Upstream 启停、Routing
切换和配置重载。
- 两份 OpenAPI 已进入 Go/CI 结构门禁覆盖本地引用闭合、operationId、响应和
security scheme完整标准工具验证仍待补齐。
security scheme固定版本的 Redocly CLI 还会按 OpenAPI 3.1 最小规则集验证两份文档,
并把 Tag 描述提升为错误。
- ProtobufWorker 注册、全量/增量 Snapshot、`usable_until`、ACK、运行态/
结果上报、Checker 任务与 Observation。
@ -29,8 +33,9 @@
- `PROXY-* / CAP-*`唯一键、TTL 优先级、状态迁移与 Active/Reserved 打包
原子计数1,000 goroutine 不超卖,以及 Cancel、重复终结、错误顺序、并发
Commit/Cancel/Release 计数守恒测试。
- `ROUTE-001 / ROUTE-004`:首条命中规则与进程内 Concurrent Sequential 单次
切换;策略运行时接线、持久化恢复和跨实例 CAS 尚未完成。
- `ROUTE-001 / ROUTE-004`首条命中规则、Provider 空结果驱动的 Controller
Sequential 切换及末端 stop 已接入权威管理状态;跨 Controller 通过 PostgreSQL
`ExpectedCurrent` CAS 收敛,切换/停用均刷新本进程 Worker Snapshot。
- `FETCH-005 / FETCH-006`Valid、Empty、DuplicateOnly、Error 分类。
- `DIST-001..003 / DIST-006..007`:内存活动池参考实现验证独占提取、满足模式、
TTL、健康时效与 Gateway 保留量1,000 并发不重复。
@ -43,8 +48,11 @@
- `PROVIDER-*`Provider HTTP Client、严格响应上限、模板解析安全边界、凭据
引用 Store 与 Reconciler Adapter 已实现。
- `DIST/Admin HTTP`:严格 JSON、Request ID、Problem 响应及 Distribution/Admin
Handler 已实现共享认证、CIDR、可信代理、Client ID 与本地准入保护链已接入,
Controller Runtime 已将二者装配到独立监听器并支持联动优雅停机。
Handler 已实现Admin 审计查询以有界 `afterId` 游标读取 PostgreSQL 权威记录,
不读取 Proxy 或 Redis 活动池。共享认证、CIDR、可信代理、Client ID、凭据级
固定权限、Distribution 凭据级提取数量/Upstream/地区限制、Gateway 凭据级
Routing 限制与本地准入保护链已接入Controller Runtime 已将二者装配到独立
监听器并支持联动优雅停机。
- `Redis Activity Adapter`:真实 Redis 8.2 已覆盖 Provider Upsert、健康更新、
原子独占提取、短期幂等、Worker ownership、库存和有界过期清理Memory/Redis
运行同一公用契约。
@ -89,44 +97,51 @@ docker compose config PASS
kubectl kustomize PASS
configuration examples 21/21 PASS
Mermaid blocks 35
bounded gateway microbenchmarks PASS (local baseline only)
```
Windows 环境为 `CGO_ENABLED=0` 且没有 C 编译器,`go test -race` 在本机未执行;
CI 已配置 Linux race job。PostgreSQL 18 和 Redis 8.2 的隔离 Adapter fixture
已经运行;当前 Docker 镜像和激活的 Compose/Kubernetes 清单只包含已实现的
Controller/Gateway 入口,完整 mTLS 运行时拓扑仍只有静态验证。
已经运行;本地 Compose 已声明 Controller、两个独立 Gateway 身份和一个 Checker 的
mTLS 运行拓扑并由配置、证书、命令装配和部署契约测试覆盖。Kubernetes base 刻意不直接
启用 Checker`development-mtls` Overlay 已为单副本 Controller、Gateway 和 Checker
提供固定开发证书、运行时身份派生、mTLS Secret 挂载、最小出站 NetworkPolicy 与渲染/配置契约验证。
其 ConfigMap 使用内容哈希以在配置变化时触发三个工作负载滚动;固定名称的外部 Secret 仍要求
受控轮换和显式重启。
Controller、Gateway 与 Checker 的叶证书和信任根已在每个新 TLS 握手时重载,文件短暂
不一致时保留最后一次有效材料。Gateway、Checker 启动派生和 Controller 授权共用同一
SPIFFE URI 解析器X.509-SVID 叶证书必须且只能包含一个 URI SAN并精确匹配其角色。
生产环境仍必须通过独立 Overlay 为每个弹性 Worker 注入唯一 SPIFFE URI且容器端到端
启动仍待具备镜像网络的环境验证。
GitHub Actions 另行执行 Dockerfile 的 Linux 多阶段镜像构建;该门禁证明镜像可以构建,
不替代 Controller/Gateway/Checker 与真实 Redis/PostgreSQL 的容器运行验证。
运维手册已覆盖管理面/活动池恢复顺序、发布与兼容回滚、Secret 轮换、容量计算,以及
主机内核、文件描述符、NAT/conntrack 的只读基线和证据采集要求;这些文档不替代真实环境
的备份恢复、驱逐或容量演练。
仓库审计会拒绝生产源和部署清单中的 `TODO`、`TBD`、`FIXME` 与私钥材料;原有 OpenAPI、
Queue、配置、指标和部署契约分别锁定一次性 Extract、队列容量、库存/累计获取边界及
低基数标签。根 `.dockerignore` 还会从镜像构建上下文排除本地控制面证书、私钥、环境文件和
构建产物。开发示例、环境变量引用和 CI fixture 不被视为生产 Secret 或未实现占位。
## 3. 后续实现范围
以下已有设计、接口或部署位置,但尚无端到端生产实现:
1. `proxy-loadgen` 已提供有界 HTTP、HTTP Gateway CONNECT 长连接和 Distribution Extract
响应校验场景;`proxy-checker` 的 BASIC 任务进程已经完成,
`proxy-controller` 已完成
Admin/Distribution/Metrics 与 PostgreSQL/Redis 启动装配,`proxy-gateway` 已完成
HTTP/Metrics 与控制面 Session 装配Checker 与 Gateway Outcome 的低基数业务指标
已接入,但 Provider、提取和容量指标链未闭环。
2. Gateway 的生产连接池调优与代表性流量压测。
3. Provider 分布式 singleflight/Leader、长期凭据回收和累计额度执行器。
4. Controller 的 PostgreSQL 连接池、迁移和 pgx Adapter 启动装配已完成;
公用 bootstrap 已通过 PostgreSQL 18 + Redis 8.2 双存储集成Controller
三监听器与探针集成已完成;可选聚合指标和完整容器进程部署验证仍待实现。
5. Worker Register/ACK/Runtime gRPC 接收路径、Redis 服务端 TTL 会话生命周期、
SPIFFE mTLS 校验和 Controller 生命周期接线已完成Redis Provider Leader、
分布式请求额度、Distribution Client 限制和 Provider 状态丢失重建已完成。
6. Worker 基础网络快照流、Proxy/Gateway Routing/凭据 Snapshot payload、Gateway Snapshot
客户端和进程装配、同版本 Routing 编译/动态匹配、五种策略上游选择与 reject/wait/direct 已完成;
Outcome 上报已完成基础观测链Redis ownership drain/ACK/过期回收及按 Worker 的可下发索引已完成。
7. BASIC Checker 调度与 HTTP/HTTPS/SOCKS5 探测器、全局与 TARGET Profile 的 Memory/Redis
原子归并、Controller Reducer 和 Observation 上报 RPC 已完成EGRESS、TARGET
生产任务调度与 REMOVE 编排仍待实现。
8. Admin/Distribution 细粒度授权和审计查询Distribution 分布式限流已完成。
9. 真实 Compose/Kubernetes 集成、故障演练和代表性集群负载测试。
10. 将 reject/wait/direct 接入 Distribution 运行链,补齐 Sequential 持久化恢复、跨实例 CAS
和 disabled candidate 语义。
11. 补齐 Proxy Capacity 动态降容契约、Reservation 全生命周期观测;短 TTL
Proxy 的零计数运行态已在后续 Snapshot Apply 中回收。
1. 已实现的 Controller、Gateway、Checker 和 Loadgen 进程仍需在实际容器网络中完成
mTLS、DNS、Redis/PostgreSQL 连通性、健康探针和优雅停机的端到端验证。
2. Gateway 已支持连接池、每 Host 连接上限、握手/空闲超时、隧道缓冲、请求生命周期指标
与 p99 告警;仍需代表性环境的流量压测和容量报告。
3. Provider 长期凭据回收与累计额度的生产运营策略仍需结合实际供应商合同、失效时间和
密钥平台完成演练。
4. Kubernetes 的开发 mTLS Overlay 已覆盖单副本开发证书身份派生和 Checker 出站
NetworkPolicy生产环境仍需要按工作负载身份系统提供证书轮换、每副本唯一 Worker ID
与可弹性扩缩容的身份注入base 清单不假定或伪造这些能力。
5. Redis/PostgreSQL 故障转移、控制面断线恢复、Snapshot 回滚、Drain 回收和 Extract
库存不足的完整故障演练。
6. 10k 稳态、100k 峰值、CONNECT 活跃连接和建连速率的代表性集群负载测试。
## 4. 容量结论
100,000 QPS 是集群设计输入,不是本次验证结果。只有实现上述运行时,并在

View File

@ -9,8 +9,8 @@
|---|---|---|---|
| ARCH-001 | 数据面 Worker 与控制面 Controller 分离 | 1-70 | 包、协议和部署拓扑已分离Controller 已运行 Worker Register/Watch/ACK/Runtime/Outcome 与 Checker Observation gRPC并发布 Proxy/Gateway Routing/按引用去重凭据完整快照Checker 任务流已具备有界领取、租约栅栏和任务期凭据契约。Gateway 已将快照编译为同版本动态 View并由独立进程维护控制面会话。`proxy-checker` 与 Redis BASIC/EGRESS/TARGET 共享任务运行态、`proxy-loadgen` 有界 HTTP/CONNECT/Extract 校验场景均已实现 |
| ARCH-002 | 热路径只做认证、本地路由和网络转发 | 1-70, 380-430 | Gateway bootstrap 集成测试验证启动期控制面会话与快照就绪HTTP 请求只走本地 Snapshot/DispatchOutcome 仅写入有界非阻塞本地队列,代表性性能剖析待完成 |
| ARCH-003 | Gateway、Distribution、Admin、Metrics 独立入口 | 8904-8958 | Controller 命令已装配 Distribution/Admin/Metrics 三个独立监听及联动停机Gateway 命令已装配代理与 Metrics 监听,运行时 mTLS 部署 Overlay 待完成 |
| ARCH-004 | Controller 集中 Provider 获取与切换 | 1403-1580 | Redis Leader、动态 Provider Supervisor 与 Bootstrap 生产装配已完成;Admin disable/reload 驱动取消替换,多副本按权威 HMAC 指纹和 revision 栅栏收敛并拒绝旧配置换主Routing 切换到 Drain 的编排待完成 |
| ARCH-003 | Gateway、Distribution、Admin、Metrics 独立入口 | 8904-8958 | Controller 命令已装配 Distribution/Admin/Metrics 三个独立监听及联动停机Gateway 命令已装配代理与 Metrics 监听。本地 Compose 已声明 Controller/Gateway/Checker mTLS 拓扑Kubernetes `development-mtls` Overlay 已通过渲染与配置契约验证单副本开发证书身份派生链路,生产工作负载身份仍待环境实现 |
| ARCH-004 | Controller 集中 Provider 获取与切换 | 1403-1580 | Redis Leader、动态 Provider Supervisor 与 Bootstrap 生产装配已完成;静态配置与 Admin 管理态共同决定 Provider 启停,停用策略以 Upstream revision 栅栏驱动既有 ownership DrainRouting 切换会向本进程 Worker 快照流广播完整刷新,多副本按定时刷新与权威 HMAC 指纹/revision 栅栏收敛并拒绝旧配置换主 |
| ARCH-005 | 100k QPS 峰值使用多 Worker 集群 | 当前会话 | 未验证设计目标;待代表性集群负载报告 |
## Routing 与 Upstream
@ -19,11 +19,11 @@
|---|---|---|---|
| ROUTE-001 | Routing 自上而下匹配,首条命中停止 | 3534-3798, 5825-6467 | `rule.go` 与不可变/首命中单测 |
| ROUTE-002 | Routing 与 Upstream 生命周期解耦 | 3534-3798 | 包依赖与配置模型 |
| ROUTE-003 | 支持 sequential、random、roundRobin、weighted、leastConnections | 5825-6467 | 五种领域策略、同版本 Gateway Snapshot 派发和定向测试已完成Distribution 接线待完成 |
| ROUTE-004 | Sequential 连续空结果达到阈值后原子切换一次 | 5295-5824, 6520-6617 | 进程内 `RoutingCursor` 版本 CAS 与 100 并发测试已完成;持久化恢复和跨实例 CAS 待完成 |
| ROUTE-003 | 支持 sequential、random、roundRobin、weighted、leastConnections | 5825-6467 | 五种领域策略、同版本 Gateway Snapshot 派发和定向测试已完成。Distribution 按独占 Extract 契约分配 Proxy不参与目标请求的 Routing 策略选择 |
| ROUTE-004 | Sequential 连续空结果达到阈值后原子切换一次 | 5295-5824, 6520-6617 | Provider Stats 对连续空结果分配单调代次Controller `SequentialCoordinator` 在独立有界循环中读取权威配置/管理快照,并通过 `ExpectedCurrent` CAS 自动切换。禁用候选会跳过;若当前项被管理态禁用,协调器不读取 Provider Stats按配置顺序推进到下一启用项只有 `loop` 可回绕,`stop`/`stayLast` 无后继时通过 `DisableRouting` 原子停用路由。重复 Tick、旧代次及跨 Controller 并发 Tick 均不会重复变更;成功切换或停用都会广播完整 Snapshot并与审计/Outbox 同事务提交。|
| ROUTE-005 | 空计数属于 Upstream当前选择属于 Routing | 8442-8529 | 共享 `UpstreamEmptyState` 双 Routing 测试 |
| ROUTE-006 | 旧 Upstream 已有 Proxy 继续耗尽,不因切换直接丢弃 | 6618-6641 | 通用 ownership Drain/ACK 原语已测试Routing 切换到 Drain 的编排待完成 |
| ROUTE-007 | 无可用 Upstream 时显式 reject、wait 或 direct默认 reject | 5075-5294, 6743-6760 | Gateway 已实现 reject、带 `wait_timeout` 的本地容量等待与经 TargetPolicy 的 directDistribution 接线和默认化策略待完成 |
| ROUTE-006 | 旧 Upstream 已有 Proxy 继续耗尽,不因切换直接丢弃 | 6618-6641 | Routing 成功切换后立即发布完整快照Sequential 仅将新分配切到新的 CurrentUpstream旧 Proxy 仍保留在快照,既有 Active/Reserved 由本地运行态自然归零。共享 Upstream 不按单 Routing 强制 Drain避免影响其他 Routing |
| ROUTE-007 | 无可用 Upstream 时显式 reject、wait 或 direct默认 reject | 5075-5294, 6743-6760 | Gateway 已实现默认 reject、带 `wait_timeout` 的本地容量等待,以及经 TargetPolicy 的 direct。静态 `routing.action: direct` 在认证、限流和目标策略后绕过 Proxy 调度Distribution 不承载目标请求路由 |
## Provider 与补池
@ -45,8 +45,8 @@
| PROXY-001 | Proxy 保存协议、地址、凭据引用、来源、TTL、健康、容量和标签 | 71-105, 8605-8678 | Domain 类型与序列化测试 |
| PROXY-002 | 唯一键包含 scheme、host、port、username、credentialVersion | 6655-6727, 8605-8678 | 去重单测 |
| PROXY-003 | TTL 来源优先级明确并统一 UTC | 681-747, 8655-8678 | TTL 表驱动测试 |
| CAP-001 | Gateway 分配使用 Reserved -> Active 原子转换 | 1203-1467, 8530-8597 | 固定 Max 下打包 CAS、1,000 并发不超卖、退役 Proxy 停止新预留及零计数历史运行态回收已完成;动态降容和完整生命周期证据待完成 |
| CAP-002 | 补池依据 Available Slots不只看 Proxy 数量 | 1203-1402, 8530-8597 | `AvailableSlots`、显式 minimum/target 水位、pending 槽位和迟滞 Reconciler 已测试Worker session/ACK/sequence/TTL/ownership fence、基础 Snapshot 流及 Gateway 会话组件已通过内存、Redis 8.2 和 gRPC 测试;权威 payload、目标健康和 Gateway reserve 策略仍待完成 |
| CAP-001 | Gateway 分配使用 Reserved -> Active 原子转换 | 1203-1467, 8530-8597 | 固定 Max 下打包 CAS、1,000 并发不超卖、退役 Proxy 停止新预留及零计数历史运行态回收已完成;Snapshot 动态降容会保留既有 Active、在计数低于新上限前拒绝新预留运行态复用与恢复预留均有回归测试。非法 Commit/Cancel/Release 经固定枚举观察器进入低基数指标 |
| CAP-002 | 补池依据 Available Slots不只看 Proxy 数量 | 1203-1402, 8530-8597 | `AvailableSlots`、显式 minimum/target 水位、pending 槽位和迟滞 Reconciler 已测试Worker session/ACK/sequence/TTL/ownership fence、权威 Proxy/Routing/凭据 payload、目标健康和 Gateway reserve 策略已通过内存、Redis 8.2、gRPC 与 Gateway 回归测试 |
| CAP-003 | pool.maxSize 包括 FETCHED/CHECKING/AVAILABLE/SUSPECT/DRAINING 与 pending expected | 3001-3533, 6642-6680 | `FetchBudget` 100 并发额度预占测试 |
| CAP-004 | TTL safety margin 内禁止新分配 | 173-220, 6728-6741 | 时钟测试 |
| CAP-005 | 多 Worker 不在热路径访问 Redis 计数 | 1403-1467 | Gateway 包依赖审计、Snapshot/Dispatch 测试 |
@ -86,6 +86,6 @@
| SEC-002 | 非回环监听无保护时严格模式启动失败 | 8112-8441 | 配置校验测试 |
| OPS-001 | 配置校验后构建不可变快照并原子替换 | 8959-8999 | 100k 索引、版本/epoch 与并发 Apply/Acquire 测试 |
| OPS-002 | 优雅停机停止新请求/Fetch等待现有流量后超时关闭 | 8981-9000 | Provider Run 收敛与 `Handler.Shutdown` HTTP 排空、Hijacked CONNECT 超时关闭测试 |
| OPS-003 | PostgreSQL 只保存管理修订、Upstream/Routing 状态、Admin 审计与 Outbox | 当前会话 | ADR-006、`adminstate` 公用契约和六表 Schema 边界测试;真实 PostgreSQL 契约待完成 |
| OBS-001 | 指标禁止 Proxy IP、session、Client、完整 URL 高基数标签 | 9001-9029 | Controller Prometheus/探针模块已实现;Checker Collector 暴露固定等级和结果Gateway Collector 暴露 `proxy_pool_gateway_outcomes_total{stage,result}``proxy_pool_gateway_outcome_queue_dropped_total`均以注册表测试固定标签集。Provider、提取和容量业务指标待实现 |
| OPS-003 | PostgreSQL 只保存管理修订、Upstream/Routing 状态、Admin 审计与 Outbox | 当前会话 | ADR-006、`adminstate` 公用契约和六表 Schema 边界测试;真实 PostgreSQL 18 契约、迁移幂等和 `information_schema` 数据边界测试已完成 |
| OBS-001 | 指标禁止 Proxy IP、session、Client、完整 URL 高基数标签 | 9001-9029 | Controller Prometheus/探针模块已实现;Gateway Collector 暴露固定 `protocol` 的请求总数、并发数、隧道数和生命周期直方图,以及固定 `stage,result` 的 Outcome 指标。Checker、Drain、Provider、Extraction 与 Capacity Collector 仅使用固定枚举标签Grafana 和 Prometheus 规则由部署契约测试校验只引用已注册指标且不按 Upstream/Worker 聚合。`platform/logging` 使用 JSON `slog`敏感字段、URL 用户信息、查询 Secret 和错误对象写出前均被脱敏;回归测试锁定标签集与 Secret 不泄露 |
| TEST-001 | 覆盖对话中列出的 11 个关键并发与故障场景 | 9030-9082 | 测试清单Redis 活动池由 Memory/Redis 公用契约覆盖,跨进程故障场景仍按清单推进 |

View File

@ -17,8 +17,19 @@
- `httpsecurity.Protection` 统一实现 Basic、API Key、Bearer、CIDR、可信代理链、
Client ID 与入口准入API 使用 401/`WWW-Authenticate`Gateway 使用
407/`Proxy-Authenticate`。
- Listener 可将固定权限绑定到单个认证方式,或在 `mode: any` 中绑定到每个凭据;
`admin:read`、`admin:write` 与 `distribution:extract` 在 Handler 入口完成匹配,
不把无权请求传入管理 mutation 或 Proxy 提取流程。
- Token Client ID 使用 SHA-256 的 128 位摘要前缀,不把 Token 本身写入领域、
日志或审计键。
- 启用 Gateway 粘性会话时,绑定键由认证 Client、Routing 和会话头共同派生;
原始会话标识不写入 PostgreSQL、Redis、日志或指标也不会转发给目标站点。
绑定仅保留在当前 Worker 的有界内存中,并在代理不再符合当前 Snapshot、到达
代理可用期或转发失败时失效。
- Gateway 凭据的 requestsPerMinute 限制按认证 Client 在本 Worker 内执行,不把
认证身份、会话或请求计数写入 PostgreSQL、Redis 或 Prometheus 标签。
- Gateway 凭据的 maxConcurrentConnections 在请求或 CONNECT 隧道的完整生命周期
内保留一个本地租约;无论转发成功、失败还是服务排空,租约都会释放。
## 3. 目标地址策略
@ -37,6 +48,8 @@ CGNAT、协议转换保留段和已知云元数据端点属于硬拒绝项。私
## 4. Secret 处理
- 配置只保存环境变量或文件引用,不在日志中输出解析后的值。
- Docker 镜像构建上下文通过根目录 `.dockerignore` 排除本地环境文件、控制面证书/私钥、
凭据文件、Git 元数据和测试/构建产物;运行时证书只能通过受控挂载提供。
- Proxy 唯一键包含 username 与 credentialVersion不包含密码或 SecretRef 内容。
- Provider Parser 将静态/响应密码写入 `credentials.Store`Proxy Snapshot 只携带
SecretRef 与 credentialVersionGateway 通过统一 Resolver 按版本读取。
@ -52,8 +65,9 @@ CGNAT、协议转换保留段和已知云元数据端点属于硬拒绝项。私
## 6. 审计
Extraction 审计记录至少包含 requestId、Client、来源、Proxy ID、Upstream、
提取时间和到期时间。日志脱敏不影响审计关联,但审计接口自身必须受 Admin
PostgreSQL 审计只记录 Admin 管理 mutation 的 actor、来源、资源、变更结果和
时间;它不保存 Proxy ID、代理地址、凭据、逐次 Extraction 或 Gateway 请求。
Extraction 的短期幂等结果仅位于 Redis TTL 活动池;审计查询接口必须受 Admin
权限保护并具备保留期限。
## 7. 优雅停机

View File

@ -83,13 +83,19 @@ soak 测试单独标记,不混入快速单测。
## 5. 当前本地微基准
2026-07-30Windows/amd64、Intel Core Ultra 7 155H
2026-08-07Windows/amd64、Go 1.26.4、Intel Core Ultra 7 155H以固定
`-count=1 -benchtime=1x` 运行:
```text
BenchmarkAcquire100kIndexed-22 1000000-1867125 893.9-1047 ns/op 256 B/op 2 allocs/op
BenchmarkStoreApply100k-22 1 518.7 ms/op 540 MB/op 3000887 allocs/op
BenchmarkAcquire100kIndexed-22 1 13100 ns/op 256 B/op 2 allocs/op
BenchmarkAcquire100kRoutingRoundRobin-22 1 45200 ns/op 13680 B/op 81 allocs/op
BenchmarkStoreApply100k-22 1 539017100 ns/op 684677920 B/op 3001735 allocs/op
```
使用 `./scripts/benchmark-gateway.ps1` 可复现该固定单次基线,输出写入被 Git 忽略的
`dist/gateway-benchmarks.txt` 并附带 revision。单次基准存在运行噪声应只用于发现明显回归
性能比较需要在同规格环境增加多次迭代后另行保存原始证据。
`Acquire` 已使用 scheme/upstream/tag 索引,结果只代表本地选择和容量预留。
`Store.Apply` 属于冷路径且当前内存开销较高;运行态为防止旧快照在途连接超配,
暂不自动回收曾出现过的 Proxy ID但注册表有 1,000,000 项硬上限,达到上限时

View File

@ -182,6 +182,7 @@ Admin 应用层测试覆盖 typed-nil 依赖、Actor/SourceIP 映射、Routing C
配置Snapshot 规模、Proxy 容量、路由、重试、日志级别
场景:协议、连接复用、响应体、持续时间、升压曲线、故障注入
结果QPS、建连速率、active、p50/p95/p99、错误、CPU、RSS、GC、FD、网络
限速:配置 rate、实际 Requests、RateStartsGenerated、RateStartsDropped
不变量capacity、ownership、extraction、idempotency、admin audit、reserve 检查结果
结论:通过/失败,以及适用边界
```

View File

@ -49,6 +49,27 @@ Extract 请求使用哪些 Upstream。
- Gateway、Distribution、Admin、Metrics 使用独立监听和认证/访问控制。
- 非回环监听且无认证、无 CIDR 保护时,严格模式必须拒绝启动。
## 可观测性边界
- Provider 拉取结果只按固定 `class` 聚合:`valid`、`empty`、`duplicate_only`、`error`。
指标分别记录拉取结果次数、合法候选数和新入池代理数,不按 Upstream、Proxy、IP、错误
文本或凭据拆分。
- Provider 指标已完成。
- Extraction 只按固定 `result` 聚合请求:`complete`、`partial`、`empty`、
`insufficient`、`idempotency_conflict`、`rate_limited`、`unavailable`、`invalid`、
`error`。计数只记录请求数量和响应交付数量;幂等重放不被误记为新的 Proxy 消费。
- Extraction 指标已完成。
- 容量指标由 Provider 库存对账循环采样,聚合 managed、available、effective、pending
与 active Upstream Gauge。Upstream ID 仅保留在 Controller 进程内用于聚合替换和
任期结束清理,不输出为 Prometheus 标签;读取结果只允许 `success``error`
- 容量指标已完成。
- 进程级致命错误统一经 JSON `slog` 输出敏感属性、错误对象、URL 用户信息和查询
Secret 在写出前脱敏。该出口只用于生命周期边界,不在 Gateway/Distribution 请求热路径
逐条写日志。
## 集群与性能
- 用户补充:高峰可能达到 100,000 请求/秒。
@ -124,10 +145,17 @@ Routing 自上而下匹配,首条命中停止;支持 Gateway 与 Extract 两
Gateway 使用既有 `SessionSupervisor` 退避重连。Gateway 本地 Store 对从完整 Snapshot
消失且仍有 Active/Reserved 的 Proxy 已按 draining 继续上报。Redis Runtime 替换已在当前
session ACK、Ticket 屏障和完整零计数同时成立时自动释放 ownerReaper 已对持续
`UNHEALTHY` 的已归属代理调用条件式 Drain。配置停用尚未接入同一触发编排。
`UNHEALTHY` 的已归属代理调用条件式 Drain。配置停用也会发布有效的上游策略视图
从上游 owned 索引有界选择候选;候选携带上游 revisionRedis 在创建 Drain Ticket 前
再次核验策略仍停用、Proxy source、Worker 和 assignment epoch避免旧候选跨启停使用。
- Snapshot 的 `version` 是同一 Worker 流的连续序列,`ownership_epoch` 是独立且只能前进
的权威栅栏epoch 变化不重置 version。此前 Gateway 错把 epoch 前进要求为 version=1
与 Controller 的 `lastAppliedVersion+1` 生成规则冲突,现已用连续版本规则统一。
- `SnapshotRefreshBroker` 是 Controller 进程内的公共扇出接口:已提交的 Upstream 启停、
Routing 切换和配置发布各向每条本地 Worker 流发送一个可合并刷新信号。Routing 不记录
单代理归属,不能按单条 Routing 对共享 Upstream 强制 DrainSequential 切换只改变后续
新分配,旧 Proxy/连接由原有 Snapshot 与运行态自然排空。其他 Controller 副本继续在
固定刷新周期内读取 PostgreSQL 权威管理态并收敛。
## Drain Ticket2026-08-02
@ -222,3 +250,45 @@ Routing 自上而下匹配,首条命中停止;支持 Gateway 与 Extract 两
- 生产 `docker-compose.yml` 仍把 PostgreSQL 18 命名卷挂在旧路径
`/var/lib/postgresql/data`;直接修改可能影响已有本地数据,必须配套迁移步骤后
单独处理,当前不能把生产持久化拓扑视为已验证。
## 控制面 mTLS 身份审计2026-08-07
- X.509-SVID 叶证书承载单一工作负载身份Gateway/Checker 启动时的自动身份派生与
Controller 授权必须都拒绝包含多个 URI SAN 的证书,即使其中只有一个 URI 与当前角色匹配。
- 解析边界已统一到 `workerruntime.SingleSPIFFEIdentity`:精确匹配信任域、环境、角色和
标识符,拒绝用户信息、端口、查询、片段、转义路径、额外 URI SAN 与跨角色复用。
- TLS 校验可能形成多条等价验证链;授权只检查同一叶证书一次,不能将链路数误判为多个身份。
- Windows 本地 Go 运行环境为 `CGO_ENABLED=0` 且没有 C 编译器Docker Engine 可用,但
`golang:1.26-bookworm``debian:bookworm-slim` 未缓存Docker Desktop HTTPS 代理也不可用。
因此已完成 Go 全量、Compose 静态和 Kustomize 静态验证,容器端到端验证仍待具备镜像网络的环境。
## OpenAPI 契约审计2026-08-07
- Go 结构契约继续锁定本地引用、operationId、响应与认证引用额外以固定
`@redocly/cli@2.25.4` 对两份 OpenAPI 3.1 文档执行标准验证,避免只依赖自定义遍历器。
- 标准最小规则集中的 Tag 描述已提升为错误Distribution、Health、Status、Audit、Upstreams、
Routing 和 Configuration 标签均有面向 API 使用者的稳定说明。
## Docker 构建上下文审计2026-08-07
- Dockerfile 构建阶段需要 `COPY . .`,因此根 `.dockerignore` 是隔离本地工作区与镜像上下文的
必要边界。它显式排除 Git 元数据、环境文件、本地配置、控制面证书/私钥、PEM/CRT/KEY 文件及
构建和测试产物;部署测试会在清单被删除或放宽时失败。
## 部署静态门禁审计2026-08-07
- GitHub Actions 的 Deployment job 在同一组无敏感值 fixture 下渲染生产 Compose、测试 Compose
以及 Kubernetes base/development mTLS Overlay并对两份配置运行严格校验它仍不替代
需要镜像网络和真实服务的容器端到端测试。
## Kubernetes 配置发布审计2026-08-07
- development mTLS Overlay 的 ConfigMap 采用 Kustomize 内容哈希,渲染契约要求三个工作负载
都引用同一版本化名称,因此配置变更进入 Pod 模板并触发滚动。外部固定名称 Secret 不含内容哈希,
它们的轮换需要平台侧 reloader 或受控显式滚动;这一限制已写入运行手册。
## Linux 镜像构建审计2026-08-07
- GitHub Actions 现在在 Linux Runner 执行 Dockerfile 多阶段构建,覆盖目标平台 Go 编译和最小
运行时层。该 job 依赖 Runner 镜像网络;它不启动 Compose 服务,因此不能作为 mTLS、存储、
探针或优雅停机端到端证据。

View File

@ -134,6 +134,58 @@ func (UnavailableAction) EnumDescriptor() ([]byte, []int) {
return file_controlplane_v1_controlplane_proto_rawDescGZIP(), []int{1}
}
type RoutingAction int32
const (
RoutingAction_ROUTING_ACTION_UNSPECIFIED RoutingAction = 0
RoutingAction_ROUTING_ACTION_PROXY RoutingAction = 1
RoutingAction_ROUTING_ACTION_DIRECT RoutingAction = 2
RoutingAction_ROUTING_ACTION_REJECT RoutingAction = 3
)
// Enum value maps for RoutingAction.
var (
RoutingAction_name = map[int32]string{
0: "ROUTING_ACTION_UNSPECIFIED",
1: "ROUTING_ACTION_PROXY",
2: "ROUTING_ACTION_DIRECT",
3: "ROUTING_ACTION_REJECT",
}
RoutingAction_value = map[string]int32{
"ROUTING_ACTION_UNSPECIFIED": 0,
"ROUTING_ACTION_PROXY": 1,
"ROUTING_ACTION_DIRECT": 2,
"ROUTING_ACTION_REJECT": 3,
}
)
func (x RoutingAction) Enum() *RoutingAction {
p := new(RoutingAction)
*p = x
return p
}
func (x RoutingAction) String() string {
return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x))
}
func (RoutingAction) Descriptor() protoreflect.EnumDescriptor {
return file_controlplane_v1_controlplane_proto_enumTypes[2].Descriptor()
}
func (RoutingAction) Type() protoreflect.EnumType {
return &file_controlplane_v1_controlplane_proto_enumTypes[2]
}
func (x RoutingAction) Number() protoreflect.EnumNumber {
return protoreflect.EnumNumber(x)
}
// Deprecated: Use RoutingAction.Descriptor instead.
func (RoutingAction) EnumDescriptor() ([]byte, []int) {
return file_controlplane_v1_controlplane_proto_rawDescGZIP(), []int{2}
}
type ProxyProtocol int32
const (
@ -170,11 +222,11 @@ func (x ProxyProtocol) String() string {
}
func (ProxyProtocol) Descriptor() protoreflect.EnumDescriptor {
return file_controlplane_v1_controlplane_proto_enumTypes[2].Descriptor()
return file_controlplane_v1_controlplane_proto_enumTypes[3].Descriptor()
}
func (ProxyProtocol) Type() protoreflect.EnumType {
return &file_controlplane_v1_controlplane_proto_enumTypes[2]
return &file_controlplane_v1_controlplane_proto_enumTypes[3]
}
func (x ProxyProtocol) Number() protoreflect.EnumNumber {
@ -183,7 +235,7 @@ func (x ProxyProtocol) Number() protoreflect.EnumNumber {
// Deprecated: Use ProxyProtocol.Descriptor instead.
func (ProxyProtocol) EnumDescriptor() ([]byte, []int) {
return file_controlplane_v1_controlplane_proto_rawDescGZIP(), []int{2}
return file_controlplane_v1_controlplane_proto_rawDescGZIP(), []int{3}
}
type OutcomeStage int32
@ -225,11 +277,11 @@ func (x OutcomeStage) String() string {
}
func (OutcomeStage) Descriptor() protoreflect.EnumDescriptor {
return file_controlplane_v1_controlplane_proto_enumTypes[3].Descriptor()
return file_controlplane_v1_controlplane_proto_enumTypes[4].Descriptor()
}
func (OutcomeStage) Type() protoreflect.EnumType {
return &file_controlplane_v1_controlplane_proto_enumTypes[3]
return &file_controlplane_v1_controlplane_proto_enumTypes[4]
}
func (x OutcomeStage) Number() protoreflect.EnumNumber {
@ -238,7 +290,7 @@ func (x OutcomeStage) Number() protoreflect.EnumNumber {
// Deprecated: Use OutcomeStage.Descriptor instead.
func (OutcomeStage) EnumDescriptor() ([]byte, []int) {
return file_controlplane_v1_controlplane_proto_rawDescGZIP(), []int{3}
return file_controlplane_v1_controlplane_proto_rawDescGZIP(), []int{4}
}
type CheckLevel int32
@ -277,11 +329,11 @@ func (x CheckLevel) String() string {
}
func (CheckLevel) Descriptor() protoreflect.EnumDescriptor {
return file_controlplane_v1_controlplane_proto_enumTypes[4].Descriptor()
return file_controlplane_v1_controlplane_proto_enumTypes[5].Descriptor()
}
func (CheckLevel) Type() protoreflect.EnumType {
return &file_controlplane_v1_controlplane_proto_enumTypes[4]
return &file_controlplane_v1_controlplane_proto_enumTypes[5]
}
func (x CheckLevel) Number() protoreflect.EnumNumber {
@ -290,7 +342,7 @@ func (x CheckLevel) Number() protoreflect.EnumNumber {
// Deprecated: Use CheckLevel.Descriptor instead.
func (CheckLevel) EnumDescriptor() ([]byte, []int) {
return file_controlplane_v1_controlplane_proto_rawDescGZIP(), []int{4}
return file_controlplane_v1_controlplane_proto_rawDescGZIP(), []int{5}
}
type RegisterWorkerRequest struct {
@ -885,6 +937,7 @@ type RoutingRule struct {
Strategy *RoutingStrategy `protobuf:"bytes,8,opt,name=strategy,proto3" json:"strategy,omitempty"`
OnUnavailable UnavailableAction `protobuf:"varint,9,opt,name=on_unavailable,json=onUnavailable,proto3,enum=proxy_pool.controlplane.v1.UnavailableAction" json:"on_unavailable,omitempty"`
WaitTimeout *durationpb.Duration `protobuf:"bytes,10,opt,name=wait_timeout,json=waitTimeout,proto3" json:"wait_timeout,omitempty"`
Action RoutingAction `protobuf:"varint,11,opt,name=action,proto3,enum=proxy_pool.controlplane.v1.RoutingAction" json:"action,omitempty"`
unknownFields protoimpl.UnknownFields
sizeCache protoimpl.SizeCache
}
@ -989,6 +1042,13 @@ func (x *RoutingRule) GetWaitTimeout() *durationpb.Duration {
return nil
}
func (x *RoutingRule) GetAction() RoutingAction {
if x != nil {
return x.Action
}
return RoutingAction_ROUTING_ACTION_UNSPECIFIED
}
type RoutingStrategy struct {
state protoimpl.MessageState `protogen:"open.v1"`
Type StrategyType `protobuf:"varint,1,opt,name=type,proto3,enum=proxy_pool.controlplane.v1.StrategyType" json:"type,omitempty"`
@ -2240,7 +2300,7 @@ const file_controlplane_v1_controlplane_proto_rawDesc = "" +
"\x10upserted_routing\x18\x06 \x03(\v2'.proxy_pool.controlplane.v1.RoutingRuleR\x0fupsertedRouting\x122\n" +
"\x15removed_routing_names\x18\a \x03(\tR\x13removedRoutingNames\x12Q\n" +
"\x10upserted_proxies\x18\b \x03(\v2&.proxy_pool.controlplane.v1.OwnedProxyR\x0fupsertedProxies\x12*\n" +
"\x11removed_proxy_ids\x18\t \x03(\tR\x0fremovedProxyIds\"\x9a\x04\n" +
"\x11removed_proxy_ids\x18\t \x03(\tR\x0fremovedProxyIds\"\xdd\x04\n" +
"\vRoutingRule\x12\x12\n" +
"\x04name\x18\x01 \x01(\tR\x04name\x12\x18\n" +
"\aenabled\x18\x02 \x01(\bR\aenabled\x12\x1d\n" +
@ -2254,7 +2314,8 @@ const file_controlplane_v1_controlplane_proto_rawDesc = "" +
"\bstrategy\x18\b \x01(\v2+.proxy_pool.controlplane.v1.RoutingStrategyR\bstrategy\x12T\n" +
"\x0eon_unavailable\x18\t \x01(\x0e2-.proxy_pool.controlplane.v1.UnavailableActionR\ronUnavailable\x12<\n" +
"\fwait_timeout\x18\n" +
" \x01(\v2\x19.google.protobuf.DurationR\vwaitTimeout\x1a:\n" +
" \x01(\v2\x19.google.protobuf.DurationR\vwaitTimeout\x12A\n" +
"\x06action\x18\v \x01(\x0e2).proxy_pool.controlplane.v1.RoutingActionR\x06action\x1a:\n" +
"\fHeadersEntry\x12\x10\n" +
"\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" +
"\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\"\x8a\x02\n" +
@ -2397,7 +2458,12 @@ const file_controlplane_v1_controlplane_proto_rawDesc = "" +
"\x1eUNAVAILABLE_ACTION_UNSPECIFIED\x10\x00\x12\x1d\n" +
"\x19UNAVAILABLE_ACTION_REJECT\x10\x01\x12\x1b\n" +
"\x17UNAVAILABLE_ACTION_WAIT\x10\x02\x12\x1d\n" +
"\x19UNAVAILABLE_ACTION_DIRECT\x10\x03*}\n" +
"\x19UNAVAILABLE_ACTION_DIRECT\x10\x03*\x7f\n" +
"\rRoutingAction\x12\x1e\n" +
"\x1aROUTING_ACTION_UNSPECIFIED\x10\x00\x12\x18\n" +
"\x14ROUTING_ACTION_PROXY\x10\x01\x12\x19\n" +
"\x15ROUTING_ACTION_DIRECT\x10\x02\x12\x19\n" +
"\x15ROUTING_ACTION_REJECT\x10\x03*}\n" +
"\rProxyProtocol\x12\x1e\n" +
"\x1aPROXY_PROTOCOL_UNSPECIFIED\x10\x00\x12\x17\n" +
"\x13PROXY_PROTOCOL_HTTP\x10\x01\x12\x18\n" +
@ -2437,102 +2503,104 @@ func file_controlplane_v1_controlplane_proto_rawDescGZIP() []byte {
return file_controlplane_v1_controlplane_proto_rawDescData
}
var file_controlplane_v1_controlplane_proto_enumTypes = make([]protoimpl.EnumInfo, 5)
var file_controlplane_v1_controlplane_proto_enumTypes = make([]protoimpl.EnumInfo, 6)
var file_controlplane_v1_controlplane_proto_msgTypes = make([]protoimpl.MessageInfo, 26)
var file_controlplane_v1_controlplane_proto_goTypes = []any{
(StrategyType)(0), // 0: proxy_pool.controlplane.v1.StrategyType
(UnavailableAction)(0), // 1: proxy_pool.controlplane.v1.UnavailableAction
(ProxyProtocol)(0), // 2: proxy_pool.controlplane.v1.ProxyProtocol
(OutcomeStage)(0), // 3: proxy_pool.controlplane.v1.OutcomeStage
(CheckLevel)(0), // 4: proxy_pool.controlplane.v1.CheckLevel
(*RegisterWorkerRequest)(nil), // 5: proxy_pool.controlplane.v1.RegisterWorkerRequest
(*RegisterWorkerResponse)(nil), // 6: proxy_pool.controlplane.v1.RegisterWorkerResponse
(*WatchSnapshotsRequest)(nil), // 7: proxy_pool.controlplane.v1.WatchSnapshotsRequest
(*SnapshotEnvelope)(nil), // 8: proxy_pool.controlplane.v1.SnapshotEnvelope
(*WorkerSnapshot)(nil), // 9: proxy_pool.controlplane.v1.WorkerSnapshot
(*SnapshotCredential)(nil), // 10: proxy_pool.controlplane.v1.SnapshotCredential
(*SnapshotDelta)(nil), // 11: proxy_pool.controlplane.v1.SnapshotDelta
(*RoutingRule)(nil), // 12: proxy_pool.controlplane.v1.RoutingRule
(*RoutingStrategy)(nil), // 13: proxy_pool.controlplane.v1.RoutingStrategy
(*OwnedProxy)(nil), // 14: proxy_pool.controlplane.v1.OwnedProxy
(*AcknowledgeSnapshotRequest)(nil), // 15: proxy_pool.controlplane.v1.AcknowledgeSnapshotRequest
(*OutcomeBatch)(nil), // 16: proxy_pool.controlplane.v1.OutcomeBatch
(*ProxyOutcome)(nil), // 17: proxy_pool.controlplane.v1.ProxyOutcome
(*ReportOutcomesResponse)(nil), // 18: proxy_pool.controlplane.v1.ReportOutcomesResponse
(*ReportRuntimeRequest)(nil), // 19: proxy_pool.controlplane.v1.ReportRuntimeRequest
(*ProxyRuntime)(nil), // 20: proxy_pool.controlplane.v1.ProxyRuntime
(*ReportRuntimeResponse)(nil), // 21: proxy_pool.controlplane.v1.ReportRuntimeResponse
(*StreamCheckTasksRequest)(nil), // 22: proxy_pool.controlplane.v1.StreamCheckTasksRequest
(*CheckTask)(nil), // 23: proxy_pool.controlplane.v1.CheckTask
(*ObservationBatch)(nil), // 24: proxy_pool.controlplane.v1.ObservationBatch
(*HealthObservation)(nil), // 25: proxy_pool.controlplane.v1.HealthObservation
(*ReportObservationsResponse)(nil), // 26: proxy_pool.controlplane.v1.ReportObservationsResponse
nil, // 27: proxy_pool.controlplane.v1.RegisterWorkerRequest.LabelsEntry
nil, // 28: proxy_pool.controlplane.v1.RoutingRule.HeadersEntry
nil, // 29: proxy_pool.controlplane.v1.RoutingStrategy.WeightsEntry
nil, // 30: proxy_pool.controlplane.v1.OwnedProxy.TagsEntry
(*durationpb.Duration)(nil), // 31: google.protobuf.Duration
(*timestamppb.Timestamp)(nil), // 32: google.protobuf.Timestamp
(*emptypb.Empty)(nil), // 33: google.protobuf.Empty
(RoutingAction)(0), // 2: proxy_pool.controlplane.v1.RoutingAction
(ProxyProtocol)(0), // 3: proxy_pool.controlplane.v1.ProxyProtocol
(OutcomeStage)(0), // 4: proxy_pool.controlplane.v1.OutcomeStage
(CheckLevel)(0), // 5: proxy_pool.controlplane.v1.CheckLevel
(*RegisterWorkerRequest)(nil), // 6: proxy_pool.controlplane.v1.RegisterWorkerRequest
(*RegisterWorkerResponse)(nil), // 7: proxy_pool.controlplane.v1.RegisterWorkerResponse
(*WatchSnapshotsRequest)(nil), // 8: proxy_pool.controlplane.v1.WatchSnapshotsRequest
(*SnapshotEnvelope)(nil), // 9: proxy_pool.controlplane.v1.SnapshotEnvelope
(*WorkerSnapshot)(nil), // 10: proxy_pool.controlplane.v1.WorkerSnapshot
(*SnapshotCredential)(nil), // 11: proxy_pool.controlplane.v1.SnapshotCredential
(*SnapshotDelta)(nil), // 12: proxy_pool.controlplane.v1.SnapshotDelta
(*RoutingRule)(nil), // 13: proxy_pool.controlplane.v1.RoutingRule
(*RoutingStrategy)(nil), // 14: proxy_pool.controlplane.v1.RoutingStrategy
(*OwnedProxy)(nil), // 15: proxy_pool.controlplane.v1.OwnedProxy
(*AcknowledgeSnapshotRequest)(nil), // 16: proxy_pool.controlplane.v1.AcknowledgeSnapshotRequest
(*OutcomeBatch)(nil), // 17: proxy_pool.controlplane.v1.OutcomeBatch
(*ProxyOutcome)(nil), // 18: proxy_pool.controlplane.v1.ProxyOutcome
(*ReportOutcomesResponse)(nil), // 19: proxy_pool.controlplane.v1.ReportOutcomesResponse
(*ReportRuntimeRequest)(nil), // 20: proxy_pool.controlplane.v1.ReportRuntimeRequest
(*ProxyRuntime)(nil), // 21: proxy_pool.controlplane.v1.ProxyRuntime
(*ReportRuntimeResponse)(nil), // 22: proxy_pool.controlplane.v1.ReportRuntimeResponse
(*StreamCheckTasksRequest)(nil), // 23: proxy_pool.controlplane.v1.StreamCheckTasksRequest
(*CheckTask)(nil), // 24: proxy_pool.controlplane.v1.CheckTask
(*ObservationBatch)(nil), // 25: proxy_pool.controlplane.v1.ObservationBatch
(*HealthObservation)(nil), // 26: proxy_pool.controlplane.v1.HealthObservation
(*ReportObservationsResponse)(nil), // 27: proxy_pool.controlplane.v1.ReportObservationsResponse
nil, // 28: proxy_pool.controlplane.v1.RegisterWorkerRequest.LabelsEntry
nil, // 29: proxy_pool.controlplane.v1.RoutingRule.HeadersEntry
nil, // 30: proxy_pool.controlplane.v1.RoutingStrategy.WeightsEntry
nil, // 31: proxy_pool.controlplane.v1.OwnedProxy.TagsEntry
(*durationpb.Duration)(nil), // 32: google.protobuf.Duration
(*timestamppb.Timestamp)(nil), // 33: google.protobuf.Timestamp
(*emptypb.Empty)(nil), // 34: google.protobuf.Empty
}
var file_controlplane_v1_controlplane_proto_depIdxs = []int32{
27, // 0: proxy_pool.controlplane.v1.RegisterWorkerRequest.labels:type_name -> proxy_pool.controlplane.v1.RegisterWorkerRequest.LabelsEntry
31, // 1: proxy_pool.controlplane.v1.RegisterWorkerResponse.heartbeat_interval:type_name -> google.protobuf.Duration
31, // 2: proxy_pool.controlplane.v1.RegisterWorkerResponse.max_stale_age:type_name -> google.protobuf.Duration
9, // 3: proxy_pool.controlplane.v1.SnapshotEnvelope.full:type_name -> proxy_pool.controlplane.v1.WorkerSnapshot
11, // 4: proxy_pool.controlplane.v1.SnapshotEnvelope.delta:type_name -> proxy_pool.controlplane.v1.SnapshotDelta
32, // 5: proxy_pool.controlplane.v1.WorkerSnapshot.generated_at:type_name -> google.protobuf.Timestamp
32, // 6: proxy_pool.controlplane.v1.WorkerSnapshot.valid_until:type_name -> google.protobuf.Timestamp
12, // 7: proxy_pool.controlplane.v1.WorkerSnapshot.routing:type_name -> proxy_pool.controlplane.v1.RoutingRule
14, // 8: proxy_pool.controlplane.v1.WorkerSnapshot.proxies:type_name -> proxy_pool.controlplane.v1.OwnedProxy
10, // 9: proxy_pool.controlplane.v1.WorkerSnapshot.credentials:type_name -> proxy_pool.controlplane.v1.SnapshotCredential
32, // 10: proxy_pool.controlplane.v1.SnapshotDelta.generated_at:type_name -> google.protobuf.Timestamp
12, // 11: proxy_pool.controlplane.v1.SnapshotDelta.upserted_routing:type_name -> proxy_pool.controlplane.v1.RoutingRule
14, // 12: proxy_pool.controlplane.v1.SnapshotDelta.upserted_proxies:type_name -> proxy_pool.controlplane.v1.OwnedProxy
28, // 13: proxy_pool.controlplane.v1.RoutingRule.headers:type_name -> proxy_pool.controlplane.v1.RoutingRule.HeadersEntry
13, // 14: proxy_pool.controlplane.v1.RoutingRule.strategy:type_name -> proxy_pool.controlplane.v1.RoutingStrategy
28, // 0: proxy_pool.controlplane.v1.RegisterWorkerRequest.labels:type_name -> proxy_pool.controlplane.v1.RegisterWorkerRequest.LabelsEntry
32, // 1: proxy_pool.controlplane.v1.RegisterWorkerResponse.heartbeat_interval:type_name -> google.protobuf.Duration
32, // 2: proxy_pool.controlplane.v1.RegisterWorkerResponse.max_stale_age:type_name -> google.protobuf.Duration
10, // 3: proxy_pool.controlplane.v1.SnapshotEnvelope.full:type_name -> proxy_pool.controlplane.v1.WorkerSnapshot
12, // 4: proxy_pool.controlplane.v1.SnapshotEnvelope.delta:type_name -> proxy_pool.controlplane.v1.SnapshotDelta
33, // 5: proxy_pool.controlplane.v1.WorkerSnapshot.generated_at:type_name -> google.protobuf.Timestamp
33, // 6: proxy_pool.controlplane.v1.WorkerSnapshot.valid_until:type_name -> google.protobuf.Timestamp
13, // 7: proxy_pool.controlplane.v1.WorkerSnapshot.routing:type_name -> proxy_pool.controlplane.v1.RoutingRule
15, // 8: proxy_pool.controlplane.v1.WorkerSnapshot.proxies:type_name -> proxy_pool.controlplane.v1.OwnedProxy
11, // 9: proxy_pool.controlplane.v1.WorkerSnapshot.credentials:type_name -> proxy_pool.controlplane.v1.SnapshotCredential
33, // 10: proxy_pool.controlplane.v1.SnapshotDelta.generated_at:type_name -> google.protobuf.Timestamp
13, // 11: proxy_pool.controlplane.v1.SnapshotDelta.upserted_routing:type_name -> proxy_pool.controlplane.v1.RoutingRule
15, // 12: proxy_pool.controlplane.v1.SnapshotDelta.upserted_proxies:type_name -> proxy_pool.controlplane.v1.OwnedProxy
29, // 13: proxy_pool.controlplane.v1.RoutingRule.headers:type_name -> proxy_pool.controlplane.v1.RoutingRule.HeadersEntry
14, // 14: proxy_pool.controlplane.v1.RoutingRule.strategy:type_name -> proxy_pool.controlplane.v1.RoutingStrategy
1, // 15: proxy_pool.controlplane.v1.RoutingRule.on_unavailable:type_name -> proxy_pool.controlplane.v1.UnavailableAction
31, // 16: proxy_pool.controlplane.v1.RoutingRule.wait_timeout:type_name -> google.protobuf.Duration
0, // 17: proxy_pool.controlplane.v1.RoutingStrategy.type:type_name -> proxy_pool.controlplane.v1.StrategyType
29, // 18: proxy_pool.controlplane.v1.RoutingStrategy.weights:type_name -> proxy_pool.controlplane.v1.RoutingStrategy.WeightsEntry
2, // 19: proxy_pool.controlplane.v1.OwnedProxy.protocol:type_name -> proxy_pool.controlplane.v1.ProxyProtocol
32, // 20: proxy_pool.controlplane.v1.OwnedProxy.expires_at:type_name -> google.protobuf.Timestamp
30, // 21: proxy_pool.controlplane.v1.OwnedProxy.tags:type_name -> proxy_pool.controlplane.v1.OwnedProxy.TagsEntry
32, // 22: proxy_pool.controlplane.v1.OwnedProxy.usable_until:type_name -> google.protobuf.Timestamp
17, // 23: proxy_pool.controlplane.v1.OutcomeBatch.outcomes:type_name -> proxy_pool.controlplane.v1.ProxyOutcome
3, // 24: proxy_pool.controlplane.v1.ProxyOutcome.stage:type_name -> proxy_pool.controlplane.v1.OutcomeStage
31, // 25: proxy_pool.controlplane.v1.ProxyOutcome.latency:type_name -> google.protobuf.Duration
32, // 26: proxy_pool.controlplane.v1.ProxyOutcome.observed_at:type_name -> google.protobuf.Timestamp
20, // 27: proxy_pool.controlplane.v1.ReportRuntimeRequest.counters:type_name -> proxy_pool.controlplane.v1.ProxyRuntime
32, // 28: proxy_pool.controlplane.v1.ReportRuntimeRequest.observed_at:type_name -> google.protobuf.Timestamp
4, // 29: proxy_pool.controlplane.v1.StreamCheckTasksRequest.supported_levels:type_name -> proxy_pool.controlplane.v1.CheckLevel
2, // 30: proxy_pool.controlplane.v1.CheckTask.protocol:type_name -> proxy_pool.controlplane.v1.ProxyProtocol
4, // 31: proxy_pool.controlplane.v1.CheckTask.level:type_name -> proxy_pool.controlplane.v1.CheckLevel
31, // 32: proxy_pool.controlplane.v1.CheckTask.timeout:type_name -> google.protobuf.Duration
32, // 33: proxy_pool.controlplane.v1.CheckTask.deadline:type_name -> google.protobuf.Timestamp
25, // 34: proxy_pool.controlplane.v1.ObservationBatch.observations:type_name -> proxy_pool.controlplane.v1.HealthObservation
4, // 35: proxy_pool.controlplane.v1.HealthObservation.level:type_name -> proxy_pool.controlplane.v1.CheckLevel
31, // 36: proxy_pool.controlplane.v1.HealthObservation.latency:type_name -> google.protobuf.Duration
32, // 37: proxy_pool.controlplane.v1.HealthObservation.observed_at:type_name -> google.protobuf.Timestamp
5, // 38: proxy_pool.controlplane.v1.WorkerControlPlane.RegisterWorker:input_type -> proxy_pool.controlplane.v1.RegisterWorkerRequest
7, // 39: proxy_pool.controlplane.v1.WorkerControlPlane.WatchSnapshots:input_type -> proxy_pool.controlplane.v1.WatchSnapshotsRequest
15, // 40: proxy_pool.controlplane.v1.WorkerControlPlane.AcknowledgeSnapshot:input_type -> proxy_pool.controlplane.v1.AcknowledgeSnapshotRequest
16, // 41: proxy_pool.controlplane.v1.WorkerControlPlane.ReportOutcomes:input_type -> proxy_pool.controlplane.v1.OutcomeBatch
19, // 42: proxy_pool.controlplane.v1.WorkerControlPlane.ReportRuntime:input_type -> proxy_pool.controlplane.v1.ReportRuntimeRequest
22, // 43: proxy_pool.controlplane.v1.CheckerControlPlane.StreamCheckTasks:input_type -> proxy_pool.controlplane.v1.StreamCheckTasksRequest
24, // 44: proxy_pool.controlplane.v1.CheckerControlPlane.ReportObservations:input_type -> proxy_pool.controlplane.v1.ObservationBatch
6, // 45: proxy_pool.controlplane.v1.WorkerControlPlane.RegisterWorker:output_type -> proxy_pool.controlplane.v1.RegisterWorkerResponse
8, // 46: proxy_pool.controlplane.v1.WorkerControlPlane.WatchSnapshots:output_type -> proxy_pool.controlplane.v1.SnapshotEnvelope
33, // 47: proxy_pool.controlplane.v1.WorkerControlPlane.AcknowledgeSnapshot:output_type -> google.protobuf.Empty
18, // 48: proxy_pool.controlplane.v1.WorkerControlPlane.ReportOutcomes:output_type -> proxy_pool.controlplane.v1.ReportOutcomesResponse
21, // 49: proxy_pool.controlplane.v1.WorkerControlPlane.ReportRuntime:output_type -> proxy_pool.controlplane.v1.ReportRuntimeResponse
23, // 50: proxy_pool.controlplane.v1.CheckerControlPlane.StreamCheckTasks:output_type -> proxy_pool.controlplane.v1.CheckTask
26, // 51: proxy_pool.controlplane.v1.CheckerControlPlane.ReportObservations:output_type -> proxy_pool.controlplane.v1.ReportObservationsResponse
45, // [45:52] is the sub-list for method output_type
38, // [38:45] is the sub-list for method input_type
38, // [38:38] is the sub-list for extension type_name
38, // [38:38] is the sub-list for extension extendee
0, // [0:38] is the sub-list for field type_name
32, // 16: proxy_pool.controlplane.v1.RoutingRule.wait_timeout:type_name -> google.protobuf.Duration
2, // 17: proxy_pool.controlplane.v1.RoutingRule.action:type_name -> proxy_pool.controlplane.v1.RoutingAction
0, // 18: proxy_pool.controlplane.v1.RoutingStrategy.type:type_name -> proxy_pool.controlplane.v1.StrategyType
30, // 19: proxy_pool.controlplane.v1.RoutingStrategy.weights:type_name -> proxy_pool.controlplane.v1.RoutingStrategy.WeightsEntry
3, // 20: proxy_pool.controlplane.v1.OwnedProxy.protocol:type_name -> proxy_pool.controlplane.v1.ProxyProtocol
33, // 21: proxy_pool.controlplane.v1.OwnedProxy.expires_at:type_name -> google.protobuf.Timestamp
31, // 22: proxy_pool.controlplane.v1.OwnedProxy.tags:type_name -> proxy_pool.controlplane.v1.OwnedProxy.TagsEntry
33, // 23: proxy_pool.controlplane.v1.OwnedProxy.usable_until:type_name -> google.protobuf.Timestamp
18, // 24: proxy_pool.controlplane.v1.OutcomeBatch.outcomes:type_name -> proxy_pool.controlplane.v1.ProxyOutcome
4, // 25: proxy_pool.controlplane.v1.ProxyOutcome.stage:type_name -> proxy_pool.controlplane.v1.OutcomeStage
32, // 26: proxy_pool.controlplane.v1.ProxyOutcome.latency:type_name -> google.protobuf.Duration
33, // 27: proxy_pool.controlplane.v1.ProxyOutcome.observed_at:type_name -> google.protobuf.Timestamp
21, // 28: proxy_pool.controlplane.v1.ReportRuntimeRequest.counters:type_name -> proxy_pool.controlplane.v1.ProxyRuntime
33, // 29: proxy_pool.controlplane.v1.ReportRuntimeRequest.observed_at:type_name -> google.protobuf.Timestamp
5, // 30: proxy_pool.controlplane.v1.StreamCheckTasksRequest.supported_levels:type_name -> proxy_pool.controlplane.v1.CheckLevel
3, // 31: proxy_pool.controlplane.v1.CheckTask.protocol:type_name -> proxy_pool.controlplane.v1.ProxyProtocol
5, // 32: proxy_pool.controlplane.v1.CheckTask.level:type_name -> proxy_pool.controlplane.v1.CheckLevel
32, // 33: proxy_pool.controlplane.v1.CheckTask.timeout:type_name -> google.protobuf.Duration
33, // 34: proxy_pool.controlplane.v1.CheckTask.deadline:type_name -> google.protobuf.Timestamp
26, // 35: proxy_pool.controlplane.v1.ObservationBatch.observations:type_name -> proxy_pool.controlplane.v1.HealthObservation
5, // 36: proxy_pool.controlplane.v1.HealthObservation.level:type_name -> proxy_pool.controlplane.v1.CheckLevel
32, // 37: proxy_pool.controlplane.v1.HealthObservation.latency:type_name -> google.protobuf.Duration
33, // 38: proxy_pool.controlplane.v1.HealthObservation.observed_at:type_name -> google.protobuf.Timestamp
6, // 39: proxy_pool.controlplane.v1.WorkerControlPlane.RegisterWorker:input_type -> proxy_pool.controlplane.v1.RegisterWorkerRequest
8, // 40: proxy_pool.controlplane.v1.WorkerControlPlane.WatchSnapshots:input_type -> proxy_pool.controlplane.v1.WatchSnapshotsRequest
16, // 41: proxy_pool.controlplane.v1.WorkerControlPlane.AcknowledgeSnapshot:input_type -> proxy_pool.controlplane.v1.AcknowledgeSnapshotRequest
17, // 42: proxy_pool.controlplane.v1.WorkerControlPlane.ReportOutcomes:input_type -> proxy_pool.controlplane.v1.OutcomeBatch
20, // 43: proxy_pool.controlplane.v1.WorkerControlPlane.ReportRuntime:input_type -> proxy_pool.controlplane.v1.ReportRuntimeRequest
23, // 44: proxy_pool.controlplane.v1.CheckerControlPlane.StreamCheckTasks:input_type -> proxy_pool.controlplane.v1.StreamCheckTasksRequest
25, // 45: proxy_pool.controlplane.v1.CheckerControlPlane.ReportObservations:input_type -> proxy_pool.controlplane.v1.ObservationBatch
7, // 46: proxy_pool.controlplane.v1.WorkerControlPlane.RegisterWorker:output_type -> proxy_pool.controlplane.v1.RegisterWorkerResponse
9, // 47: proxy_pool.controlplane.v1.WorkerControlPlane.WatchSnapshots:output_type -> proxy_pool.controlplane.v1.SnapshotEnvelope
34, // 48: proxy_pool.controlplane.v1.WorkerControlPlane.AcknowledgeSnapshot:output_type -> google.protobuf.Empty
19, // 49: proxy_pool.controlplane.v1.WorkerControlPlane.ReportOutcomes:output_type -> proxy_pool.controlplane.v1.ReportOutcomesResponse
22, // 50: proxy_pool.controlplane.v1.WorkerControlPlane.ReportRuntime:output_type -> proxy_pool.controlplane.v1.ReportRuntimeResponse
24, // 51: proxy_pool.controlplane.v1.CheckerControlPlane.StreamCheckTasks:output_type -> proxy_pool.controlplane.v1.CheckTask
27, // 52: proxy_pool.controlplane.v1.CheckerControlPlane.ReportObservations:output_type -> proxy_pool.controlplane.v1.ReportObservationsResponse
46, // [46:53] is the sub-list for method output_type
39, // [39:46] is the sub-list for method input_type
39, // [39:39] is the sub-list for extension type_name
39, // [39:39] is the sub-list for extension extendee
0, // [0:39] is the sub-list for field type_name
}
func init() { file_controlplane_v1_controlplane_proto_init() }
@ -2549,7 +2617,7 @@ func file_controlplane_v1_controlplane_proto_init() {
File: protoimpl.DescBuilder{
GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
RawDescriptor: unsafe.Slice(unsafe.StringData(file_controlplane_v1_controlplane_proto_rawDesc), len(file_controlplane_v1_controlplane_proto_rawDesc)),
NumEnums: 5,
NumEnums: 6,
NumMessages: 26,
NumExtensions: 0,
NumServices: 2,

View File

@ -65,6 +65,10 @@ func TestOperationsPrioritizeCanceledContext(t *testing.T) {
_, operationErr := store.SwitchRouting(ctx, adminstate.SwitchRoutingCommand{})
return operationErr
}},
{name: "disable routing", run: func() error {
_, operationErr := store.DisableRouting(ctx, adminstate.DisableRoutingCommand{})
return operationErr
}},
{name: "snapshot", run: func() error {
_, operationErr := store.Snapshot(ctx)
return operationErr
@ -119,6 +123,10 @@ func TestInvalidOperationsDoNotBeginTransaction(t *testing.T) {
_, operationErr := store.SwitchRouting(context.Background(), adminstate.SwitchRoutingCommand{})
return operationErr
}},
{name: "disable routing", run: func() error {
_, operationErr := store.DisableRouting(context.Background(), adminstate.DisableRoutingCommand{})
return operationErr
}},
{name: "read audit", run: func() error {
_, operationErr := store.ReadAudit(context.Background(), adminstate.AuditQuery{})
return operationErr

View File

@ -0,0 +1,15 @@
BEGIN;
ALTER TABLE admin_audit_log
DROP CONSTRAINT IF EXISTS admin_audit_log_action_check;
ALTER TABLE admin_audit_log
ADD CONSTRAINT admin_audit_log_action_check
CHECK (action IN (
'commit_config',
'set_upstream_enabled',
'switch_routing',
'disable_routing'
));
COMMIT;

View File

@ -27,15 +27,18 @@ func TestApplyMigrationsValidatesContextAndPool(t *testing.T) {
func TestMigrationsAreOrderedTransactionalAndImmutable(t *testing.T) {
t.Parallel()
migrations := Migrations()
if len(migrations) != 1 {
t.Fatalf("len(Migrations()) = %d, want 1", len(migrations))
if len(migrations) != 2 {
t.Fatalf("len(Migrations()) = %d, want 2", len(migrations))
}
if migrations[0].Version != 1 || migrations[0].Name != "admin_state" {
t.Fatalf("migration metadata = %+v", migrations[0])
if migrations[0].Version != 1 || migrations[0].Name != "admin_state" ||
migrations[1].Version != 2 || migrations[1].Name != "disable_routing" {
t.Fatalf("migration metadata = %+v", migrations)
}
normalized := strings.TrimSpace(migrations[0].SQL)
for _, migration := range migrations {
normalized := strings.TrimSpace(migration.SQL)
if !strings.HasPrefix(normalized, "BEGIN;") || !strings.HasSuffix(normalized, "COMMIT;") {
t.Fatalf("migration is not transaction wrapped: %q", normalized)
t.Fatalf("migration %d is not transaction wrapped: %q", migration.Version, normalized)
}
}
for index := 1; index < len(migrations); index++ {
if migrations[index-1].Version >= migrations[index].Version {
@ -49,6 +52,25 @@ func TestMigrationsAreOrderedTransactionalAndImmutable(t *testing.T) {
}
}
func TestRoutingDisableMigrationExtendsOnlyAuditActionConstraint(t *testing.T) {
t.Parallel()
lower := strings.ToLower(Migrations()[1].SQL)
for _, required := range []string{
"alter table admin_audit_log",
"drop constraint if exists admin_audit_log_action_check",
"disable_routing",
} {
if !strings.Contains(lower, required) {
t.Errorf("routing disable migration missing %q", required)
}
}
for _, forbidden := range []string{"create table", "proxy", "credential", "extraction", "ownership", "idempotency"} {
if strings.Contains(lower, forbidden) {
t.Errorf("routing disable migration contains forbidden identifier %q", forbidden)
}
}
}
func TestAdminSchemaContainsOnlyManagementTables(t *testing.T) {
t.Parallel()
sql := Migrations()[0].SQL

View File

@ -301,6 +301,96 @@ func (adapter *adapter) SwitchRouting(
}, nil
}
func (adapter *adapter) DisableRouting(
ctx context.Context,
command adminstate.DisableRoutingCommand,
) (adminstate.MutationResult, error) {
result := adminstate.MutationResult{RequestID: command.RequestID}
if err := contextError(ctx); err != nil {
return result, err
}
if command.Validate() != nil || !adapter.valid() {
return result, adminstate.ErrInvalidCommand
}
tx, currentRevision, err := adapter.beginMutation(ctx, "disable routing")
if err != nil {
return result, err
}
defer rollback(tx)
var enabled bool
var current string
if err := tx.QueryRow(ctx, `
SELECT enabled, current_upstream
FROM routing_admin_state
WHERE name = $1
FOR UPDATE`, command.Name).Scan(&enabled, &current); err != nil {
if errors.Is(err, pgx.ErrNoRows) {
return result, adminstate.ErrNotFound
}
return result, databaseError(ctx, "read routing state", err)
}
if current != command.ExpectedCurrent {
return result, adminstate.ErrConflict
}
if !enabled {
if err := insertAudit(ctx, tx, command.RequestID, command.Actor,
adminstate.ActionDisableRoute, "routing", command.Name, false,
currentRevision, command.Reason, command.OccurredAt, "audit routing disable no-op"); err != nil {
return result, err
}
if err := commit(ctx, tx, "commit routing disable no-op"); err != nil {
return result, err
}
result.Revision = uint64(currentRevision)
return result, nil
}
nextRevision, err := allocateRevision(ctx, tx, currentRevision, "routing", command.OccurredAt)
if err != nil {
return result, err
}
commandTag, err := tx.Exec(ctx, `
UPDATE routing_admin_state
SET enabled = FALSE, revision = $1, updated_at = $2
WHERE name = $3 AND enabled = TRUE AND current_upstream = $4`,
nextRevision, utc(command.OccurredAt), command.Name, command.ExpectedCurrent)
if err != nil {
return result, databaseError(ctx, "disable routing state", err)
}
if commandTag.RowsAffected() != 1 {
return result, adminstate.ErrConflict
}
if err := insertAudit(ctx, tx, command.RequestID, command.Actor,
adminstate.ActionDisableRoute, "routing", command.Name, true,
nextRevision, command.Reason, command.OccurredAt, "audit routing disable"); err != nil {
return result, err
}
payload, err := encodePayload(map[string]any{
"current": command.ExpectedCurrent,
"enabled": false,
"name": command.Name,
"reason": command.Reason,
"revision": nextRevision,
})
if err != nil {
return result, unavailable("encode routing disable event")
}
if err := insertOutbox(ctx, tx, nextRevision, "routing.disabled", "routing",
command.Name, payload, command.OccurredAt, "insert routing disable event"); err != nil {
return result, err
}
if err := commit(ctx, tx, "commit routing disable"); err != nil {
return result, err
}
return adminstate.MutationResult{
RequestID: command.RequestID,
Changed: true,
Revision: uint64(nextRevision),
}, nil
}
func (adapter *adapter) beginMutation(ctx context.Context, operation string) (pgx.Tx, int64, error) {
tx, err := adapter.begin(ctx, pgx.TxOptions{AccessMode: pgx.ReadWrite}, operation)
if err != nil {

View File

@ -62,6 +62,41 @@ func TestMutationRollsBackWhenOutboxInsertFails(t *testing.T) {
}
}
func TestDisableRoutingRollsBackWhenAuditOrOutboxInsertFails(t *testing.T) {
for _, target := range []struct {
name string
table string
}{
{name: "audit", table: "admin_audit_log"},
{name: "outbox", table: "admin_outbox"},
} {
t.Run(target.name, func(t *testing.T) {
fixture := newPostgresTestFixture(t)
now := integrationNow()
commitIntegrationConfig(t, fixture.Store, now)
installRejectInsertTrigger(t, fixture, target.table, "reject_routing_disable")
_, err := fixture.Store.DisableRouting(context.Background(), adminstate.DisableRoutingCommand{
RequestID: "req-routing-disable-failure", Actor: integrationActor(), OccurredAt: now.Add(time.Second),
Name: "checkout", ExpectedCurrent: "provider-a", Reason: "terminal empty results",
})
if !errors.Is(err, adminstate.ErrUnavailable) {
t.Fatalf("DisableRouting(%s failure) error = %v, want ErrUnavailable", target.name, err)
}
assertFailedRoutingDisableLeftBaseline(t, fixture)
dropRejectInsertTrigger(t, fixture, target.table, "reject_routing_disable")
result, err := fixture.Store.DisableRouting(context.Background(), adminstate.DisableRoutingCommand{
RequestID: "req-routing-disable-after-failure", Actor: integrationActor(), OccurredAt: now.Add(2 * time.Second),
Name: "checkout", ExpectedCurrent: "provider-a", Reason: "terminal empty results",
})
if err != nil || !result.Changed || result.Revision != 2 {
t.Fatalf("DisableRouting(after rollback) = %+v, %v", result, err)
}
})
}
}
func assertFailedMutationLeftBaseline(t *testing.T, fixture postgresTestFixture) {
t.Helper()
snapshot, err := fixture.Store.Snapshot(context.Background())
@ -83,6 +118,28 @@ func assertFailedMutationLeftBaseline(t *testing.T, fixture postgresTestFixture)
}
}
func assertFailedRoutingDisableLeftBaseline(t *testing.T, fixture postgresTestFixture) {
t.Helper()
snapshot, err := fixture.Store.Snapshot(context.Background())
if err != nil || snapshot.Revision != 1 || len(snapshot.Routings) != 1 || !snapshot.Routings[0].Enabled ||
snapshot.Routings[0].CurrentUpstream != "provider-a" {
t.Fatalf("Snapshot(after failed DisableRouting) = %+v, %v", snapshot, err)
}
audits, err := fixture.Store.ReadAudit(context.Background(), adminstate.AuditQuery{Limit: 10})
if err != nil || len(audits) != 1 {
t.Fatalf("ReadAudit(after failed DisableRouting) = %+v, %v", audits, err)
}
for table, want := range map[string]int{"control_revisions": 1, "admin_outbox": 1} {
var count int
if err := fixture.Pool.QueryRow(context.Background(), "SELECT COUNT(*) FROM "+table).Scan(&count); err != nil {
t.Fatalf("count %s: %v", table, err)
}
if count != want {
t.Fatalf("%s row count = %d, want %d", table, count, want)
}
}
}
func installRejectInsertTrigger(t *testing.T, fixture postgresTestFixture, table, trigger string) {
t.Helper()
function := trigger + "_fn"

View File

@ -224,6 +224,53 @@ func TestRedisRejectsUnhealthyDrainCandidateAfterRecovery(t *testing.T) {
}
}
func TestRedisDisabledUpstreamDrainFencesPolicyRevision(t *testing.T) {
fixture := newRedisTestFixture(t)
now := redisTestNow()
seedRedisAvailable(t, fixture.Adapter, "provider-a", now, now.Add(time.Second), time.Minute,
testProxy("proxy-a", "192.0.2.10"))
assignment, err := fixture.Adapter.Assign(context.Background(), now.Add(2*time.Second), "proxy-a", "worker-a", time.Minute)
if err != nil {
t.Fatalf("Assign() error = %v", err)
}
disabled := activitypool.UpstreamDrainPolicy{UpstreamID: "provider-a", Revision: 31}
if err := fixture.Adapter.ReplaceUpstreamDrainPolicies(context.Background(), []activitypool.UpstreamDrainPolicy{disabled}); err != nil {
t.Fatalf("ReplaceUpstreamDrainPolicies(disabled) error = %v", err)
}
candidates, err := fixture.Adapter.ListDisabledUpstreamDrainCandidates(context.Background(), now.Add(3*time.Second), disabled, 1)
if err != nil || len(candidates) != 1 || candidates[0] != (activitypool.DisabledUpstreamDrainCandidate{
UpstreamID: "provider-a", PolicyRevision: 31, ProxyID: "proxy-a", WorkerID: "worker-a", AssignmentEpoch: assignment.Epoch,
}) {
t.Fatalf("ListDisabledUpstreamDrainCandidates() = %+v, %v", candidates, err)
}
if err := fixture.Adapter.ReplaceUpstreamDrainPolicies(context.Background(), []activitypool.UpstreamDrainPolicy{{
UpstreamID: "provider-a", Revision: 32, Enabled: true,
}}); err != nil {
t.Fatalf("ReplaceUpstreamDrainPolicies(enabled) error = %v", err)
}
if started, err := fixture.Adapter.BeginDisabledUpstreamDrain(context.Background(), now.Add(3*time.Second), candidates[0]); err != nil || started {
t.Fatalf("BeginDisabledUpstreamDrain(stale policy) = %t, %v", started, err)
}
if current, ok, err := fixture.Adapter.Get(context.Background(), "proxy-a"); err != nil || !ok || current.Draining {
t.Fatalf("Get(after stale candidate) = %+v, %t, %v", current, ok, err)
}
disabled.Revision = 33
if err := fixture.Adapter.ReplaceUpstreamDrainPolicies(context.Background(), []activitypool.UpstreamDrainPolicy{disabled}); err != nil {
t.Fatalf("ReplaceUpstreamDrainPolicies(disabled again) error = %v", err)
}
candidates, err = fixture.Adapter.ListDisabledUpstreamDrainCandidates(context.Background(), now.Add(4*time.Second), disabled, 1)
if err != nil || len(candidates) != 1 {
t.Fatalf("ListDisabledUpstreamDrainCandidates(disabled again) = %+v, %v", candidates, err)
}
if started, err := fixture.Adapter.BeginDisabledUpstreamDrain(context.Background(), now.Add(4*time.Second), candidates[0]); err != nil || !started {
t.Fatalf("BeginDisabledUpstreamDrain() = %t, %v", started, err)
}
if current, ok, err := fixture.Adapter.Get(context.Background(), "proxy-a"); err != nil || !ok || !current.Draining {
t.Fatalf("Get(after drain) = %+v, %t, %v", current, ok, err)
}
}
func redisGlobalObservation(taskID string, success bool, observedAt time.Time) healthDomain.Observation {
observation := healthDomain.Observation{
TaskID: taskID, ProxyID: "proxy-a", Level: healthDomain.LevelBasic,

View File

@ -18,6 +18,7 @@ type keyspace struct {
owners string
ownerExpiry string
drainTickets string
upstreamDrainPolicies string
epoch string
inventory string
stateInventory string
@ -51,6 +52,7 @@ func newKeyspace(namespace string) keyspace {
owners: prefix + ":owners",
ownerExpiry: prefix + ":owner-expiry",
drainTickets: prefix + ":drain-tickets",
upstreamDrainPolicies: prefix + ":upstream-drain-policies",
epoch: prefix + ":epoch",
inventory: prefix + ":inventory",
stateInventory: prefix + ":state-inventory",

View File

@ -3,6 +3,7 @@ package redisactivity
import (
"context"
"encoding/json"
"strconv"
"time"
"proxy-pool/internal/domain/activitypool"
@ -12,6 +13,7 @@ const (
maintenanceInventory = "inventory"
maintenanceSweep = "sweep"
maintenanceUnhealthy = "unhealthy"
maintenanceDisabledUpstream = "disabled_upstream"
maximumUnhealthySweepScan = 1024
)
@ -19,6 +21,7 @@ const (
var (
_ activitypool.InventoryReader = (*Adapter)(nil)
_ activitypool.Maintainer = (*Adapter)(nil)
_ activitypool.DisabledUpstreamDrainLister = (*Adapter)(nil)
)
func (a *Adapter) Inventory(ctx context.Context, upstreamID string, now time.Time) (activitypool.Inventory, error) {
@ -124,6 +127,52 @@ func (a *Adapter) SweepUnhealthy(
return result, nil
}
func (a *Adapter) ListDisabledUpstreamDrainCandidates(
ctx context.Context,
now time.Time,
policy activitypool.UpstreamDrainPolicy,
limit int,
) ([]activitypool.DisabledUpstreamDrainCandidate, error) {
if ctx == nil {
return nil, activitypool.ErrInvalidMaintenance
}
if err := ctx.Err(); err != nil {
return nil, err
}
if a == nil || now.IsZero() || policy.UpstreamID == "" || policy.Revision == 0 || policy.Enabled || limit <= 0 {
return nil, activitypool.ErrInvalidMaintenance
}
if limit > maximumUnhealthySweepScan/4 {
limit = maximumUnhealthySweepScan / 4
}
reply, err := a.runMaintenance(ctx, maintenanceDisabledUpstream, now, limit, policy.UpstreamID, "")
if err != nil {
return nil, err
}
if reply.Status == scriptStale {
return nil, nil
}
if reply.Status == scriptInvalid {
return nil, activitypool.ErrInvalidMaintenance
}
if reply.Status != scriptOK || len(reply.DisabledDrainCandidates) > limit {
return nil, invalidScriptReply("unexpected disabled upstream sweep reply")
}
wantRevision := strconv.FormatUint(policy.Revision, 10)
result := make([]activitypool.DisabledUpstreamDrainCandidate, 0, len(reply.DisabledDrainCandidates))
for _, candidate := range reply.DisabledDrainCandidates {
if candidate.UpstreamID != policy.UpstreamID || candidate.PolicyRevision != wantRevision ||
candidate.ProxyID == "" || candidate.WorkerID == "" || candidate.AssignmentEpoch == 0 {
return nil, invalidScriptReply("disabled upstream sweep reply contained an invalid drain candidate")
}
result = append(result, activitypool.DisabledUpstreamDrainCandidate{
UpstreamID: candidate.UpstreamID, PolicyRevision: policy.Revision, ProxyID: candidate.ProxyID,
WorkerID: candidate.WorkerID, AssignmentEpoch: candidate.AssignmentEpoch,
})
}
return result, nil
}
func (a *Adapter) runMaintenance(
ctx context.Context,
operation string,
@ -141,6 +190,7 @@ func (a *Adapter) runMaintenance(
a.keys.inventory, a.keys.stateInventory, a.keys.owners, a.keys.ownerExpiry,
a.keys.operation(operationID), a.keys.healthDue, a.keys.healthQueued, a.keys.healthLeases,
a.keys.healthTasks, a.keys.healthTaskExpiry, a.keys.healthRefTask, a.keys.healthUnhealthy,
a.keys.owned(upstreamID), a.keys.upstreamDrainPolicies,
}, operation, now.UnixMilli(), limit, upstreamID, operationTTLMillis(a.options.OperationTTL), policies)
if err != nil {
return maintenanceScriptReply{}, err

View File

@ -6,6 +6,7 @@ import (
"encoding/hex"
"errors"
"fmt"
"strconv"
"time"
"proxy-pool/internal/domain/activitypool"
@ -17,6 +18,7 @@ const (
ownershipRenew = "renew"
ownershipBeginDrain = "begin_drain"
ownershipBeginUnhealthyDrain = "begin_unhealthy_drain"
ownershipBeginDisabledDrain = "begin_disabled_upstream_drain"
ownershipAcknowledgeDrain = "acknowledge_drain"
ownershipGet = "get"
ownershipExpire = "expire"
@ -25,6 +27,7 @@ const (
var _ ownershipDomain.Repository = (*Adapter)(nil)
var _ ownershipDomain.DrainTicketStore = (*Adapter)(nil)
var _ activitypool.UnhealthyDrainStarter = (*Adapter)(nil)
var _ activitypool.DisabledUpstreamDrainStarter = (*Adapter)(nil)
func (a *Adapter) Assign(
ctx context.Context,
@ -39,7 +42,7 @@ func (a *Adapter) Assign(
if now.IsZero() || proxyID == "" || workerID == "" || ttl <= 0 {
return ownershipDomain.Assignment{}, ownershipDomain.ErrInvalidOwnership
}
reply, err := a.runOwnership(ctx, ownershipAssign, true, now.UnixMilli(), proxyID, workerID, 0, durationMillis(ttl), 0, 0)
reply, err := a.runOwnership(ctx, ownershipAssign, true, now.UnixMilli(), proxyID, workerID, 0, durationMillis(ttl), 0, 0, "", "")
if err != nil {
return ownershipDomain.Assignment{}, err
}
@ -71,7 +74,7 @@ func (a *Adapter) Renew(
if now.IsZero() || proxyID == "" || workerID == "" || epoch == 0 || ttl <= 0 {
return ownershipDomain.Assignment{}, ownershipDomain.ErrInvalidOwnership
}
reply, err := a.runOwnership(ctx, ownershipRenew, true, now.UnixMilli(), proxyID, workerID, epoch, durationMillis(ttl), 0, 0)
reply, err := a.runOwnership(ctx, ownershipRenew, true, now.UnixMilli(), proxyID, workerID, epoch, durationMillis(ttl), 0, 0, "", "")
if err != nil {
return ownershipDomain.Assignment{}, err
}
@ -99,7 +102,7 @@ func (a *Adapter) BeginDrain(
if proxyID == "" || workerID == "" || epoch == 0 {
return ownershipDomain.Assignment{}, ownershipDomain.ErrInvalidOwnership
}
reply, err := a.runOwnership(ctx, ownershipBeginDrain, true, 0, proxyID, workerID, epoch, 0, 0, 0)
reply, err := a.runOwnership(ctx, ownershipBeginDrain, true, 0, proxyID, workerID, epoch, 0, 0, 0, "", "")
if err != nil {
return ownershipDomain.Assignment{}, err
}
@ -128,7 +131,7 @@ func (a *Adapter) BeginUnhealthyDrain(
}
reply, err := a.runOwnership(
ctx, ownershipBeginUnhealthyDrain, true, now.UTC().UnixMilli(), candidate.ProxyID, candidate.WorkerID,
candidate.AssignmentEpoch, candidate.UnhealthySince.UTC().UnixMilli(), 0, 0,
candidate.AssignmentEpoch, candidate.UnhealthySince.UTC().UnixMilli(), 0, 0, "", "",
)
if err != nil {
return false, err
@ -148,6 +151,40 @@ func (a *Adapter) BeginUnhealthyDrain(
}
}
func (a *Adapter) BeginDisabledUpstreamDrain(
ctx context.Context,
now time.Time,
candidate activitypool.DisabledUpstreamDrainCandidate,
) (bool, error) {
if err := validateOwnershipCall(ctx, a); err != nil {
return false, err
}
if now.IsZero() || candidate.UpstreamID == "" || candidate.PolicyRevision == 0 || candidate.ProxyID == "" ||
candidate.WorkerID == "" || candidate.AssignmentEpoch == 0 {
return false, activitypool.ErrInvalidMaintenance
}
reply, err := a.runOwnership(
ctx, ownershipBeginDisabledDrain, true, now.UTC().UnixMilli(), candidate.ProxyID, candidate.WorkerID,
candidate.AssignmentEpoch, 0, 0, 0, candidate.UpstreamID, strconv.FormatUint(candidate.PolicyRevision, 10),
)
if err != nil {
return false, err
}
switch reply.Status {
case scriptOK:
if _, err := decodeAssignmentReply(reply); err != nil {
return false, err
}
return reply.Started, nil
case scriptStale, scriptNotFound:
return false, nil
case scriptInvalid:
return false, activitypool.ErrInvalidMaintenance
default:
return false, invalidScriptReply("unexpected disabled upstream ownership drain status")
}
}
func (a *Adapter) PendingDrains(ctx context.Context, workerID string, limit int) ([]ownershipDomain.DrainTicket, error) {
if err := validateOwnershipCall(ctx, a); err != nil {
return nil, err
@ -242,7 +279,7 @@ func (a *Adapter) AcknowledgeDrain(
if proxyID == "" || workerID == "" || epoch == 0 || active < 0 || reserved < 0 {
return ownershipDomain.ErrInvalidOwnership
}
reply, err := a.runOwnership(ctx, ownershipAcknowledgeDrain, true, 0, proxyID, workerID, epoch, 0, active, reserved)
reply, err := a.runOwnership(ctx, ownershipAcknowledgeDrain, true, 0, proxyID, workerID, epoch, 0, active, reserved, "", "")
if err != nil {
return err
}
@ -269,7 +306,7 @@ func (a *Adapter) Get(ctx context.Context, proxyID string) (ownershipDomain.Assi
if proxyID == "" {
return ownershipDomain.Assignment{}, false, ownershipDomain.ErrInvalidOwnership
}
reply, err := a.runOwnership(ctx, ownershipGet, false, 0, proxyID, "", 0, 0, 0, 0)
reply, err := a.runOwnership(ctx, ownershipGet, false, 0, proxyID, "", 0, 0, 0, 0, "", "")
if err != nil {
return ownershipDomain.Assignment{}, false, err
}
@ -293,7 +330,7 @@ func (a *Adapter) Expire(ctx context.Context, now time.Time, limit int) ([]owner
if now.IsZero() || limit <= 0 {
return nil, ownershipDomain.ErrInvalidOwnership
}
reply, err := a.runOwnership(ctx, ownershipExpire, true, now.UnixMilli(), "", "", 0, int64(limit), 0, 0)
reply, err := a.runOwnership(ctx, ownershipExpire, true, now.UnixMilli(), "", "", 0, int64(limit), 0, 0, "", "")
if err != nil {
return nil, err
}
@ -331,6 +368,8 @@ func (a *Adapter) runOwnership(
value int64,
active int64,
reserved int64,
upstreamID string,
policyRevision string,
) (ownershipScriptReply, error) {
operationKey := a.keys.epoch
if mutating {
@ -345,9 +384,9 @@ func (a *Adapter) runOwnership(
a.keys.inventory, a.keys.stateInventory, a.keys.owners, a.keys.ownerExpiry,
a.keys.epoch, operationKey, a.keys.healthDue, a.keys.healthQueued, a.keys.healthLeases,
a.keys.healthTasks, a.keys.healthTaskExpiry, a.keys.healthRefTask, a.keys.healthUnhealthy,
a.keys.drainTickets,
a.keys.drainTickets, a.keys.upstreamDrainPolicies,
}, operation, operationTTLMillis(a.options.OperationTTL), a.options.CleanupLimit,
nowMS, proxyID, workerID, epoch, value, active, reserved, a.keys.workerOwned(workerID), a.keys.workerDraining(workerID))
nowMS, proxyID, workerID, epoch, value, active, reserved, a.keys.workerOwned(workerID), a.keys.workerDraining(workerID), upstreamID, policyRevision)
if err != nil {
return ownershipScriptReply{}, err
}

View File

@ -88,6 +88,10 @@ type ownershipScriptReply struct {
Started bool `json:"started"`
}
type upstreamDrainPolicyScriptReply struct {
Status scriptStatus `json:"status"`
}
type drainTicketsScriptReply struct {
Status scriptStatus `json:"status"`
Tickets []string `json:"tickets"`
@ -98,6 +102,7 @@ type maintenanceScriptReply struct {
Count int `json:"count"`
DeferredOwned int `json:"deferredOwned"`
DrainCandidates []maintenanceDrainCandidateScriptRow `json:"drainCandidates"`
DisabledDrainCandidates []disabledDrainCandidateScriptRow `json:"disabledDrainCandidates"`
}
type maintenanceDrainCandidateScriptRow struct {
@ -107,6 +112,14 @@ type maintenanceDrainCandidateScriptRow struct {
UnhealthySinceMS int64 `json:"unhealthySinceMs"`
}
type disabledDrainCandidateScriptRow struct {
UpstreamID string `json:"upstreamId"`
PolicyRevision string `json:"policyRevision"`
ProxyID string `json:"proxyId"`
WorkerID string `json:"workerId"`
AssignmentEpoch uint64 `json:"assignmentEpoch"`
}
type statusScriptReply struct {
Status scriptStatus `json:"status"`
Inventories []statusScriptInventory `json:"inventories"`
@ -188,6 +201,9 @@ var capacitySource string
//go:embed scripts/worker_snapshot.lua
var workerSnapshotSource string
//go:embed scripts/upstream_drain_policy.lua
var upstreamDrainPolicySource string
var (
upsertScript = redis.NewScript(upsertSource)
healthScript = redis.NewScript(healthSource)
@ -203,6 +219,7 @@ var (
runtimeScript = redis.NewScript(runtimeSource)
capacityScript = redis.NewScript(capacitySource)
workerSnapshotScript = redis.NewScript(workerSnapshotSource)
upstreamDrainPolicyScript = redis.NewScript(upstreamDrainPolicySource)
)
func runScript(ctx context.Context, client redis.Scripter, script *redis.Script, keys []string, args ...any) (any, error) {

View File

@ -17,6 +17,7 @@ local health_task_expiry_key = KEYS[16]
local health_ref_task_key = KEYS[17]
local health_unhealthy_key = KEYS[18]
local drain_tickets_key = KEYS[19]
local upstream_drain_policies_key = KEYS[20]
local operation = ARGV[1]
local operation_ttl_ms = tonumber(ARGV[2])
@ -30,6 +31,8 @@ local active = tonumber(ARGV[9])
local reserved = tonumber(ARGV[10])
local worker_index_key = ARGV[11]
local worker_draining_index_key = ARGV[12]
local upstream_id = ARGV[13]
local policy_revision = ARGV[14]
local mutating = operation ~= 'get'
local function finish(reply)
@ -212,6 +215,11 @@ local function decode_table(raw)
return value
end
local function valid_uint(value)
return type(value) == 'string' and string.match(value, '^[0-9]+$') and
value ~= '0' and (string.len(value) == 1 or string.sub(value, 1, 1) ~= '0')
end
local function valid_assignment(assignment)
return assignment and assignment.version == 1 and type(assignment.proxyId) == 'string' and
assignment.proxyId ~= '' and type(assignment.workerId) == 'string' and assignment.workerId ~= '' and
@ -396,6 +404,33 @@ if operation == 'begin_unhealthy_drain' then
return finish({status = 'ok', record = encoded, started = started})
end
if operation == 'begin_disabled_upstream_drain' then
cleanup_hard_expired(now_ms)
if type(upstream_id) ~= 'string' or upstream_id == '' or not valid_uint(policy_revision) then
return finish({status = 'invalid'})
end
local policy = decode_table(redis.call('HGET', upstream_drain_policies_key, upstream_id))
if not policy or policy.version ~= 1 or policy.upstreamId ~= upstream_id or policy.enabled ~= false or
policy.revision ~= policy_revision then
return finish({status = 'stale'})
end
local current = decode_table(redis.call('HGET', owners_key, proxy_id))
if not valid_assignment(current) or current.workerId ~= worker_id or tonumber(current.epoch) ~= epoch or
tonumber(current.expiresAtMs) <= now_ms then
return finish({status = 'stale'})
end
local record = decode_table(redis.call('HGET', records_key, proxy_id))
if not record or record.sourceUpstream ~= upstream_id or record.ownerWorkerId ~= worker_id or
record.state ~= 'AVAILABLE' then
return finish({status = 'stale'})
end
local encoded, started, failure = begin_drain(current)
if failure then
return finish({status = failure})
end
return finish({status = 'ok', record = encoded, started = started})
end
if operation == 'acknowledge_drain' then
local current = decode_table(redis.call('HGET', owners_key, proxy_id))
if not valid_assignment(current) or current.workerId ~= worker_id or tonumber(current.epoch) ~= epoch then

View File

@ -15,6 +15,8 @@ local health_tasks_key = KEYS[14]
local health_task_expiry_key = KEYS[15]
local health_ref_task_key = KEYS[16]
local health_unhealthy_key = KEYS[17]
local owned_upstream_key = KEYS[18]
local upstream_drain_policies_key = KEYS[19]
local operation = ARGV[1]
local now_ms = tonumber(ARGV[2])
@ -220,6 +222,40 @@ if operation == 'unhealthy' then
end
return finish({status = 'ok', count = removed, deferredOwned = deferred_owned, drainCandidates = drain_candidates})
end
if operation == 'disabled_upstream' then
local raw_policy = redis.call('HGET', upstream_drain_policies_key, upstream_id)
local policy = raw_policy and decode_table(raw_policy) or nil
if not policy or policy.version ~= 1 or policy.upstreamId ~= upstream_id or policy.enabled ~= false or
type(policy.revision) ~= 'string' or not string.match(policy.revision, '^[0-9]+$') or policy.revision == '0' then
return finish({status = 'stale', count = 0, disabledDrainCandidates = {}})
end
local scan_limit = limit * 4
if scan_limit > 1024 then
scan_limit = 1024
end
local proxy_ids = redis.call('ZRANGEBYSCORE', owned_upstream_key, '(' .. now_ms, '+inf', 'LIMIT', 0, scan_limit)
local candidates = {}
for _, proxy_id in ipairs(proxy_ids) do
local record = decode_table(redis.call('HGET', records_key, proxy_id))
local owner = decode_table(redis.call('HGET', owners_key, proxy_id))
if record and owner and record.sourceUpstream == upstream_id and record.state == 'AVAILABLE' and
record.ownerWorkerId == owner.workerId and owner.draining == false and
type(owner.workerId) == 'string' and owner.workerId ~= '' and tonumber(owner.epoch) and
tonumber(owner.epoch) > 0 and tonumber(owner.expiresAtMs) and tonumber(owner.expiresAtMs) > now_ms then
table.insert(candidates, {
upstreamId = upstream_id,
policyRevision = policy.revision,
proxyId = proxy_id,
workerId = owner.workerId,
assignmentEpoch = tonumber(owner.epoch),
})
if #candidates == limit then
break
end
end
end
return finish({status = 'ok', count = 0, disabledDrainCandidates = candidates})
end
if operation == 'inventory' then
local count = tonumber(redis.call('HGET', inventory_key, upstream_id) or '0')
if count < 0 then

View File

@ -0,0 +1,23 @@
local policies_key = KEYS[1]
local policies_json = ARGV[1]
local decoded, policies = pcall(cjson.decode, policies_json or '')
if not decoded or type(policies) ~= 'table' then
return cjson.encode({status = 'invalid'})
end
local seen = {}
for _, policy in ipairs(policies) do
if type(policy) ~= 'table' or type(policy.upstreamId) ~= 'string' or policy.upstreamId == '' or
type(policy.revision) ~= 'string' or not string.match(policy.revision, '^[0-9]+$') or
policy.revision == '0' or type(policy.enabled) ~= 'boolean' or seen[policy.upstreamId] then
return cjson.encode({status = 'invalid'})
end
seen[policy.upstreamId] = true
end
redis.call('DEL', policies_key)
for _, policy in ipairs(policies) do
redis.call('HSET', policies_key, policy.upstreamId, cjson.encode(policy))
end
return cjson.encode({status = 'ok'})

View File

@ -0,0 +1,63 @@
package redisactivity
import (
"context"
"encoding/json"
"strconv"
"proxy-pool/internal/domain/activitypool"
)
var _ activitypool.UpstreamDrainPolicyWriter = (*Adapter)(nil)
type upstreamDrainPolicyRecord struct {
Version int `json:"version"`
UpstreamID string `json:"upstreamId"`
Revision string `json:"revision"`
Enabled bool `json:"enabled"`
}
func (a *Adapter) ReplaceUpstreamDrainPolicies(ctx context.Context, policies []activitypool.UpstreamDrainPolicy) error {
if ctx == nil {
return activitypool.ErrInvalidMaintenance
}
if err := ctx.Err(); err != nil {
return err
}
if a == nil {
return activitypool.ErrInvalidMaintenance
}
records := make([]upstreamDrainPolicyRecord, 0, len(policies))
seen := make(map[string]struct{}, len(policies))
for _, policy := range policies {
if policy.UpstreamID == "" || policy.Revision == 0 {
return activitypool.ErrInvalidMaintenance
}
if _, duplicate := seen[policy.UpstreamID]; duplicate {
return activitypool.ErrInvalidMaintenance
}
seen[policy.UpstreamID] = struct{}{}
records = append(records, upstreamDrainPolicyRecord{
Version: 1, UpstreamID: policy.UpstreamID, Revision: strconv.FormatUint(policy.Revision, 10), Enabled: policy.Enabled,
})
}
payload, err := json.Marshal(records)
if err != nil {
return err
}
result, err := runScript(ctx, a.client, upstreamDrainPolicyScript, []string{a.keys.upstreamDrainPolicies}, string(payload))
if err != nil {
return err
}
var reply upstreamDrainPolicyScriptReply
if err := decodeScriptResult(result, &reply); err != nil {
return err
}
if reply.Status == scriptInvalid {
return activitypool.ErrInvalidMaintenance
}
if reply.Status != scriptOK {
return invalidScriptReply("unexpected upstream drain policy reply")
}
return nil
}

View File

@ -8,9 +8,13 @@ import (
"context"
"errors"
"fmt"
"net"
"net/http"
"reflect"
"strings"
"sync/atomic"
"github.com/prometheus/client_golang/prometheus"
"google.golang.org/grpc"
"google.golang.org/grpc/credentials"
@ -19,12 +23,17 @@ import (
"proxy-pool/internal/checker/probe"
"proxy-pool/internal/config"
"proxy-pool/internal/controlplane/clienttransport"
"proxy-pool/internal/controlplane/tlsreload"
"proxy-pool/internal/domain/workerruntime"
"proxy-pool/internal/platform/httpserver"
"proxy-pool/internal/platform/lifecycle"
platformMetrics "proxy-pool/internal/platform/metrics"
)
var (
ErrInvalidOptions = errors.New("invalid checker bootstrap options")
ErrStartup = errors.New("checker startup failed")
ErrNotReady = errors.New("checker control plane is not ready")
)
type Options struct {
@ -33,9 +42,12 @@ type Options struct {
ControlPlaneAddress string
CheckerID string
InstanceID string
AutoIdentity bool
MaxInFlight int
SupportedLevels []controlplanev1.CheckLevel
GRPCTransport credentials.TransportCredentials
MetricsListener net.Listener
HTTP httpserver.Options
}
func Run(ctx context.Context, options Options) error {
@ -49,11 +61,27 @@ func Run(ctx context.Context, options Options) error {
if !configuration.ControlPlane.Enabled {
return errors.Join(ErrInvalidOptions, errors.New("controlPlane must be enabled"))
}
options, err = resolveIdentity(configuration, options)
if err != nil {
return fmt.Errorf("%w: %w", ErrStartup, err)
}
runtime, err := newRuntime(ctx, configuration, options)
if err != nil {
return fmt.Errorf("%w: %w", ErrStartup, err)
}
defer runtime.Close()
return runtime.Run(ctx)
}
func newRuntime(ctx context.Context, configuration *config.Config, options Options) (*runtime, error) {
if ctx == nil || configuration == nil {
return nil, ErrInvalidOptions
}
transport, err := clienttransport.New(
configuration.ControlPlane, options.ControlPlaneAddress, configuration.ControlPlane.CheckerTLS, options.GRPCTransport, "checkerTLS",
)
if err != nil {
return fmt.Errorf("%w: %w", ErrInvalidOptions, err)
return nil, fmt.Errorf("%w: %w", ErrInvalidOptions, err)
}
connection, err := grpc.NewClient(options.ControlPlaneAddress,
grpc.WithTransportCredentials(transport),
@ -63,31 +91,164 @@ func Run(ctx context.Context, options Options) error {
),
)
if err != nil {
return fmt.Errorf("%w: dial control plane: %w", ErrStartup, err)
return nil, fmt.Errorf("dial control plane: %w", err)
}
defer connection.Close()
closeConnection := true
defer func() {
if closeConnection {
_ = connection.Close()
}
}()
client := controlplane.NewGeneratedClient(controlplanev1.NewCheckerControlPlaneClient(connection))
readiness := &controlPlaneReadiness{}
runner, err := controlplane.NewRunner(client, probe.NewExecutor(), controlplane.Options{
CheckerID: options.CheckerID, InstanceID: options.InstanceID, MaxInFlight: options.MaxInFlight,
SupportedLevels: append([]controlplanev1.CheckLevel(nil), options.SupportedLevels...),
ReportBatchSize: options.MaxInFlight,
ReportBatchSize: options.MaxInFlight, OnSuccessfulPull: readiness.MarkReady, OnFailedPull: readiness.MarkUnavailable,
})
if err != nil {
return fmt.Errorf("%w: build checker runner: %w", ErrStartup, err)
return nil, fmt.Errorf("build checker runner: %w", err)
}
return runner.Run(ctx)
runners := []lifecycle.Runner{runner}
if configuration.Metrics.Enabled {
metrics, metricsErr := newMetricsRuntime(ctx, configuration.Metrics, options, readiness)
if metricsErr != nil {
return nil, metricsErr
}
runners = append(runners, metrics)
} else if options.MetricsListener != nil {
return nil, ErrInvalidOptions
}
group, err := lifecycle.NewGroup(runners...)
if err != nil {
return nil, err
}
closeConnection = false
return &runtime{connection: connection, group: group}, nil
}
func validateOptions(ctx context.Context, options Options) error {
if ctx == nil || strings.TrimSpace(options.ConfigPath) != options.ConfigPath || options.ConfigPath == "" ||
nilInterface(options.Resolver) || !clienttransport.ValidDialAddress(options.ControlPlaneAddress) ||
!workerruntime.ValidIdentifier(options.CheckerID) || !workerruntime.ValidIdentifier(options.InstanceID) ||
(options.CheckerID != "" && !workerruntime.ValidIdentifier(options.CheckerID)) ||
(options.InstanceID != "" && !workerruntime.ValidIdentifier(options.InstanceID)) ||
(!options.AutoIdentity && (!workerruntime.ValidIdentifier(options.CheckerID) || !workerruntime.ValidIdentifier(options.InstanceID))) ||
options.MaxInFlight <= 0 || len(options.SupportedLevels) == 0 {
return ErrInvalidOptions
}
return nil
}
func resolveIdentity(configuration *config.Config, options Options) (Options, error) {
if configuration == nil {
return Options{}, ErrInvalidOptions
}
if !options.AutoIdentity {
return options, nil
}
if configuration.ControlPlane.TLS.Mode != "mtls" {
return Options{}, errors.Join(ErrInvalidOptions, errors.New("auto identity requires controlPlane mTLS"))
}
identity, err := tlsreload.ResolveSPIFFEIdentity(
configuration.ControlPlane.CheckerTLS.CertFile, configuration.ControlPlane.CheckerTLS.KeyFile,
configuration.ControlPlane.TLS.TrustDomain, configuration.ControlPlane.TLS.Environment, "checker",
)
if err != nil {
return Options{}, fmt.Errorf("resolve checker SPIFFE identity: %w", err)
}
if options.CheckerID == "" {
options.CheckerID = identity
}
if options.InstanceID == "" {
options.InstanceID = identity
}
if !workerruntime.ValidIdentifier(options.CheckerID) || !workerruntime.ValidIdentifier(options.InstanceID) || options.CheckerID != identity {
return Options{}, errors.Join(ErrInvalidOptions, errors.New("checker identity does not match SPIFFE certificate"))
}
return options, nil
}
type runtime struct {
connection *grpc.ClientConn
group *lifecycle.Group
}
func (runtime *runtime) Run(ctx context.Context) error {
if runtime == nil || runtime.connection == nil || runtime.group == nil || ctx == nil {
return ErrInvalidOptions
}
return runtime.group.Run(ctx)
}
func (runtime *runtime) Close() {
if runtime != nil && runtime.connection != nil {
_ = runtime.connection.Close()
}
}
type controlPlaneReadiness struct {
ready atomic.Bool
}
func (readiness *controlPlaneReadiness) MarkReady() {
if readiness != nil {
readiness.ready.Store(true)
}
}
func (readiness *controlPlaneReadiness) MarkUnavailable() {
if readiness != nil {
readiness.ready.Store(false)
}
}
func (readiness *controlPlaneReadiness) Ready(ctx context.Context) error {
if ctx == nil || readiness == nil || !readiness.ready.Load() {
return ErrNotReady
}
return ctx.Err()
}
type metricsRuntime struct {
listener net.Listener
handler http.Handler
options httpserver.Options
}
func newMetricsRuntime(
ctx context.Context,
configuration config.Metrics,
options Options,
readiness *controlPlaneReadiness,
) (*metricsRuntime, error) {
if ctx == nil || !configuration.Enabled || readiness == nil {
return nil, ErrInvalidOptions
}
handler, err := platformMetrics.NewHandler(platformMetrics.Dependencies{
Gatherer: prometheus.DefaultGatherer, Readiness: readiness,
})
if err != nil {
return nil, fmt.Errorf("build checker metrics handler: %w", err)
}
listener := options.MetricsListener
if listener == nil {
listener, err = (&net.ListenConfig{}).Listen(ctx, "tcp", configuration.Listen)
if err != nil {
return nil, fmt.Errorf("listen checker metrics: %w", err)
}
}
return &metricsRuntime{listener: listener, handler: handler, options: options.HTTP}, nil
}
func (runtime *metricsRuntime) Run(ctx context.Context) error {
if runtime == nil || runtime.listener == nil || runtime.handler == nil || ctx == nil {
return ErrInvalidOptions
}
return httpserver.Serve(ctx, runtime.options, httpserver.Endpoint{
Name: "metrics", Listener: runtime.listener, Handler: runtime.handler,
})
}
func loadConfiguration(ctx context.Context, path string, resolver config.Resolver) (*config.Config, error) {
if ctx == nil || strings.TrimSpace(path) != path || path == "" || nilInterface(resolver) {
return nil, ErrInvalidOptions

View File

@ -0,0 +1,55 @@
package bootstrap
import (
"context"
"errors"
"net"
"net/http"
"net/http/httptest"
"testing"
"proxy-pool/internal/config"
)
func TestControlPlaneReadinessTransitionsAfterSuccessfulPull(t *testing.T) {
readiness := &controlPlaneReadiness{}
if err := readiness.Ready(context.Background()); !errors.Is(err, ErrNotReady) {
t.Fatalf("Ready() before pull = %v, want ErrNotReady", err)
}
readiness.MarkReady()
if err := readiness.Ready(context.Background()); err != nil {
t.Fatalf("Ready() after successful pull = %v", err)
}
readiness.MarkUnavailable()
if err := readiness.Ready(context.Background()); !errors.Is(err, ErrNotReady) {
t.Fatalf("Ready() after failed pull = %v, want ErrNotReady", err)
}
}
func TestMetricsRuntimeUsesControlPlaneReadiness(t *testing.T) {
listener, err := net.Listen("tcp", "127.0.0.1:0")
if err != nil {
t.Fatalf("Listen(): %v", err)
}
defer listener.Close()
readiness := &controlPlaneReadiness{}
runtime, err := newMetricsRuntime(context.Background(), config.Metrics{Enabled: true, Listen: listener.Addr().String()}, Options{
MetricsListener: listener,
}, readiness)
if err != nil {
t.Fatalf("newMetricsRuntime(): %v", err)
}
assertCheckerProbe(t, runtime.handler, "/livez", http.StatusOK, "live\n")
assertCheckerProbe(t, runtime.handler, "/readyz", http.StatusServiceUnavailable, "unavailable\n")
readiness.MarkReady()
assertCheckerProbe(t, runtime.handler, "/readyz", http.StatusOK, "ready\n")
}
func assertCheckerProbe(t *testing.T, handler http.Handler, path string, wantStatus int, wantBody string) {
t.Helper()
recorder := httptest.NewRecorder()
handler.ServeHTTP(recorder, httptest.NewRequest(http.MethodGet, path, nil))
if recorder.Code != wantStatus || recorder.Body.String() != wantBody {
t.Fatalf("GET %s = %d %q, want %d %q", path, recorder.Code, recorder.Body.String(), wantStatus, wantBody)
}
}

View File

@ -52,6 +52,8 @@ type Options struct {
PollInterval time.Duration
RetryDelay time.Duration
Now func() time.Time
OnSuccessfulPull func()
OnFailedPull func()
}
// Runner consumes at most one server-bounded pull per RunOnce. Run adds a
@ -93,6 +95,9 @@ func (runner *Runner) Run(ctx context.Context) error {
if err != nil && ctx.Err() != nil {
return ctx.Err()
}
if err != nil && runner.options.OnFailedPull != nil {
runner.options.OnFailedPull()
}
wait := runner.options.PollInterval
if err != nil {
wait = runner.options.RetryDelay
@ -118,9 +123,13 @@ func (runner *Runner) RunOnce(ctx context.Context) error {
return err
}
tasks, err := collectTasks(stream, runner.options.MaxInFlight)
if err != nil || len(tasks) == 0 {
if err != nil {
return err
}
if len(tasks) == 0 {
runner.observeSuccessfulPull()
return nil
}
observations := runner.executeTasks(ctx, tasks)
for start := 0; start < len(observations); start += runner.options.ReportBatchSize {
end := start + runner.options.ReportBatchSize
@ -137,9 +146,16 @@ func (runner *Runner) RunOnce(ctx context.Context) error {
return ErrObservationsRejected
}
}
runner.observeSuccessfulPull()
return nil
}
func (runner *Runner) observeSuccessfulPull() {
if runner != nil && runner.options.OnSuccessfulPull != nil {
runner.options.OnSuccessfulPull()
}
}
func collectTasks(stream TaskStream, maximum int) ([]*controlplanev1.CheckTask, error) {
if stream == nil || maximum <= 0 {
return nil, ErrInvalidRunner

View File

@ -2,8 +2,10 @@ package controlplane
import (
"context"
"errors"
"io"
"sync"
"sync/atomic"
"testing"
"time"
@ -67,6 +69,51 @@ func TestRunnerOmitsEgressProbeURLFromGlobalObservation(t *testing.T) {
}
}
func TestRunnerMarksReadyAfterSuccessfulEmptyPull(t *testing.T) {
var successfulPulls atomic.Int64
runner, err := NewRunner(&clientStub{stream: &taskStreamStub{}}, &executorStub{}, Options{
CheckerID: "checker-a", InstanceID: "instance-a", MaxInFlight: 1,
SupportedLevels: []controlplanev1.CheckLevel{controlplanev1.CheckLevel_CHECK_LEVEL_BASIC},
Now: time.Now, OnSuccessfulPull: func() { successfulPulls.Add(1) },
})
if err != nil {
t.Fatalf("NewRunner(): %v", err)
}
if err := runner.RunOnce(context.Background()); err != nil {
t.Fatalf("RunOnce(): %v", err)
}
if successfulPulls.Load() != 1 {
t.Fatalf("successful pulls = %d, want 1", successfulPulls.Load())
}
}
func TestRunnerMarksUnavailableAfterPullFailure(t *testing.T) {
failed := make(chan struct{})
var markOnce sync.Once
runner, err := NewRunner(&clientStub{streamErr: errors.New("control plane unavailable")}, &executorStub{}, Options{
CheckerID: "checker-a", InstanceID: "instance-a", MaxInFlight: 1,
SupportedLevels: []controlplanev1.CheckLevel{controlplanev1.CheckLevel_CHECK_LEVEL_BASIC},
Now: time.Now, RetryDelay: time.Millisecond,
OnFailedPull: func() { markOnce.Do(func() { close(failed) }) },
})
if err != nil {
t.Fatalf("NewRunner(): %v", err)
}
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
completed := make(chan error, 1)
go func() { completed <- runner.Run(ctx) }()
select {
case <-failed:
cancel()
case <-time.After(time.Second):
t.Fatal("Runner did not mark failed pull")
}
if err := <-completed; !errors.Is(err, context.Canceled) {
t.Fatalf("Run() = %v, want context.Canceled", err)
}
}
func checkerTask(id string, now time.Time) *controlplanev1.CheckTask {
return &controlplanev1.CheckTask{
TaskId: id, ProxyId: id + "-proxy", Protocol: controlplanev1.ProxyProtocol_PROXY_PROTOCOL_HTTP,
@ -77,11 +124,12 @@ func checkerTask(id string, now time.Time) *controlplanev1.CheckTask {
type clientStub struct {
stream TaskStream
streamErr error
batches []*controlplanev1.ObservationBatch
}
func (stub *clientStub) StreamCheckTasks(context.Context, *controlplanev1.StreamCheckTasksRequest) (TaskStream, error) {
return stub.stream, nil
return stub.stream, stub.streamErr
}
func (stub *clientStub) ReportObservations(_ context.Context, batch *controlplanev1.ObservationBatch) (*controlplanev1.ReportObservationsResponse, error) {

View File

@ -3,6 +3,8 @@ package config
import (
"fmt"
"time"
"proxy-pool/internal/domain/clientpolicy"
)
// MaximumCheckURLs bounds per-upstream EGRESS references. The bound keeps
@ -63,6 +65,8 @@ type Listener struct {
Auth Auth `yaml:"auth"`
Limits Limits `yaml:"limits"`
Retry Retry `yaml:"retry"`
StickySession StickySession `yaml:"stickySession"`
Transport GatewayTransport `yaml:"transport"`
DestinationPolicy DestinationPolicy `yaml:"destinationPolicy"`
}
@ -79,6 +83,8 @@ type Access struct {
type Auth struct {
Mode string `yaml:"mode"`
Permissions []string `yaml:"permissions"`
ClientPolicy clientpolicy.Policy `yaml:"client"`
Username string `yaml:"username"`
Password string `yaml:"password"`
PasswordFile string `yaml:"passwordFile"`
@ -91,6 +97,8 @@ type Auth struct {
type AuthMethod struct {
Mode string `yaml:"mode"`
Permissions []string `yaml:"permissions"`
ClientPolicy clientpolicy.Policy `yaml:"client"`
Username string `yaml:"username"`
Password string `yaml:"password"`
PasswordFile string `yaml:"passwordFile"`
@ -111,6 +119,29 @@ type Retry struct {
RetryMethods []string `yaml:"retryMethods"`
}
// StickySession configures per-Worker, bounded Gateway affinity. It does not
// store Proxy details or session identifiers in PostgreSQL or Redis.
type StickySession struct {
Enabled bool `yaml:"enabled"`
Header string `yaml:"header"`
TTL Duration `yaml:"ttl"`
MaxEntries int `yaml:"maxEntries"`
}
// GatewayTransport configures the local Gateway process connection pools and
// tunnel I/O. It intentionally contains no remote state or provider settings.
type GatewayTransport struct {
DialTimeout Duration `yaml:"dialTimeout"`
HandshakeTimeout Duration `yaml:"handshakeTimeout"`
ResponseHeaderTimeout Duration `yaml:"responseHeaderTimeout"`
IdleConnTimeout Duration `yaml:"idleConnTimeout"`
MaxIdleConns int `yaml:"maxIdleConns"`
MaxIdleConnsPerHost int `yaml:"maxIdleConnsPerHost"`
MaxConnsPerHost int `yaml:"maxConnsPerHost"`
TunnelBufferBytes int `yaml:"tunnelBufferBytes"`
TunnelIdleTimeout Duration `yaml:"tunnelIdleTimeout"`
}
type DestinationPolicy struct {
DenyPrivateNetworks *bool `yaml:"denyPrivateNetworks"`
DenyLoopback *bool `yaml:"denyLoopback"`
@ -183,6 +214,7 @@ type Routing struct {
Name string `yaml:"name"`
Enabled bool `yaml:"enabled"`
Purpose string `yaml:"purpose"`
Action string `yaml:"action"`
Match RoutingMatch `yaml:"match"`
Upstreams []string `yaml:"upstreams"`
Strategy Strategy `yaml:"strategy"`

View File

@ -10,6 +10,9 @@ import (
"time"
"go.yaml.in/yaml/v4"
"proxy-pool/internal/domain/authorization"
"proxy-pool/internal/domain/clientpolicy"
)
const validConfig = `
@ -201,6 +204,7 @@ func TestShippedDeploymentConfigurationsResolveEnvironment(t *testing.T) {
wantGatewayUser string
wantPostgresURL string
wantRedisURL string
wantTransport GatewayTransport
}{
{
name: "local",
@ -208,6 +212,12 @@ func TestShippedDeploymentConfigurationsResolveEnvironment(t *testing.T) {
wantGatewayUser: "local-gateway",
wantPostgresURL: "postgres://proxy_pool:local-only-change-me@postgres:5432/proxy_pool?sslmode=disable",
wantRedisURL: "redis://redis:6379/0",
wantTransport: GatewayTransport{
MaxIdleConns: 20000,
MaxIdleConnsPerHost: 32,
MaxConnsPerHost: 32,
TunnelBufferBytes: 32768,
},
},
{
name: "kubernetes",
@ -216,6 +226,17 @@ func TestShippedDeploymentConfigurationsResolveEnvironment(t *testing.T) {
wantGatewayUser: "resolved-gateway-user",
wantPostgresURL: "postgres://resolved",
wantRedisURL: "redis://resolved",
wantTransport: GatewayTransport{
DialTimeout: Duration(10 * time.Second),
HandshakeTimeout: Duration(15 * time.Second),
ResponseHeaderTimeout: Duration(30 * time.Second),
IdleConnTimeout: Duration(90 * time.Second),
MaxIdleConns: 20000,
MaxIdleConnsPerHost: 32,
MaxConnsPerHost: 32,
TunnelBufferBytes: 32768,
TunnelIdleTimeout: Duration(5 * time.Minute),
},
},
}
for _, test := range tests {
@ -245,6 +266,7 @@ func TestShippedDeploymentConfigurationsResolveEnvironment(t *testing.T) {
cfg.Admin.Auth.Token != "resolved-admin-token" ||
cfg.Storage.PostgresURL != test.wantPostgresURL ||
cfg.Storage.RedisURL != test.wantRedisURL ||
cfg.Gateway.Transport != test.wantTransport ||
cfg.Upstreams["provider-a"].API.Auth.Value != "resolved-provider-a-token" ||
cfg.Upstreams["provider-b"].API.Auth.Value != "resolved-provider-b-token" {
t.Fatalf("deployment values were not resolved: %+v", cfg.Redacted())
@ -327,6 +349,206 @@ func TestValidateAcceptsBearerListenerAuthentication(t *testing.T) {
}
}
func TestValidateListenerPermissionConfiguration(t *testing.T) {
t.Parallel()
cfg := mustLoadValidConfig(t)
cfg.Admin.Enabled = true
cfg.Admin.Listen = "127.0.0.1:8082"
cfg.Admin.Auth = Auth{
Mode: "bearer",
Token: "admin-token",
Permissions: []string{authorization.AdminRead},
}
if err := Validate(cfg); err != nil {
t.Fatalf("Validate(admin read permission) error = %v", err)
}
cfg.Admin.Auth.Permissions = []string{"unknown"}
if err := Validate(cfg); err == nil || !strings.Contains(err.Error(), "permissions") {
t.Fatalf("Validate(unknown permission) error = %v", err)
}
cfg.Admin.Auth = Auth{
Mode: "any",
Permissions: []string{authorization.AdminRead},
Methods: []AuthMethod{{
Mode: "bearer", Value: "read-token", Permissions: []string{authorization.AdminRead},
}},
}
if err := Validate(cfg); err == nil || !strings.Contains(err.Error(), "auth.permissions") {
t.Fatalf("Validate(any top-level permission) error = %v", err)
}
cfg.Admin.Auth.Permissions = nil
if err := Validate(cfg); err != nil {
t.Fatalf("Validate(any method permission) error = %v", err)
}
}
func TestValidateDistributionCredentialClientPolicy(t *testing.T) {
t.Parallel()
cfg, err := Load(strings.NewReader(validConfig))
if err != nil {
t.Fatalf("Load() error = %v", err)
}
cfg.Distribution.Auth = Auth{
Mode: "bearer", Token: "distribution-token",
ClientPolicy: clientpolicy.Policy{
MaxExtractCount: 5,
AllowedUpstreams: []string{"provider-a"},
AllowedRegions: []string{"shanghai"},
},
}
if err := Validate(cfg); err != nil {
t.Fatalf("Validate(distribution client policy) error = %v", err)
}
cfg.Distribution.Auth.ClientPolicy.AllowedUpstreams = []string{"missing"}
if err := Validate(cfg); err == nil || !strings.Contains(err.Error(), "missing") {
t.Fatalf("Validate(unknown allowed upstream) error = %v", err)
}
cfg.Distribution.Auth.ClientPolicy = clientpolicy.Policy{}
cfg.Gateway.Auth = Auth{
Mode: "bearer", Token: "gateway-token",
ClientPolicy: clientpolicy.Policy{AllowedRoutings: []string{"extract"}},
}
cfg.Routing[0].Purpose = "gateway"
if err := Validate(cfg); err != nil {
t.Fatalf("Validate(gateway routing policy) error = %v", err)
}
cfg.Gateway.Auth.ClientPolicy.AllowedRoutings = []string{"missing"}
if err := Validate(cfg); err == nil || !strings.Contains(err.Error(), "missing") {
t.Fatalf("Validate(unknown gateway routing) error = %v", err)
}
cfg.Gateway.Auth.ClientPolicy = clientpolicy.Policy{MaxExtractCount: 1}
if err := Validate(cfg); err == nil || !strings.Contains(err.Error(), "only supported on distribution") {
t.Fatalf("Validate(gateway extraction policy) error = %v", err)
}
cfg.Gateway.Auth.ClientPolicy = clientpolicy.Policy{RequestsPerMinute: 60}
if err := Validate(cfg); err != nil {
t.Fatalf("Validate(gateway request rate policy) error = %v", err)
}
cfg.Gateway.Auth.ClientPolicy = clientpolicy.Policy{MaxConcurrentConnections: 3}
if err := Validate(cfg); err != nil {
t.Fatalf("Validate(gateway concurrency policy) error = %v", err)
}
cfg.Gateway.Auth.ClientPolicy = clientpolicy.Policy{}
cfg.Distribution.Auth.ClientPolicy = clientpolicy.Policy{MaxConcurrentConnections: 3}
if err := Validate(cfg); err == nil || !strings.Contains(err.Error(), "only supported on gateway") {
t.Fatalf("Validate(distribution gateway limit policy) error = %v", err)
}
}
func TestValidateGatewayStickySession(t *testing.T) {
t.Parallel()
cfg := mustLoadValidConfig(t)
cfg.Gateway.Auth = Auth{Mode: "bearer", Token: "gateway-token"}
cfg.Gateway.StickySession = StickySession{
Enabled: true, Header: "X-Proxy-Session", TTL: Duration(30 * time.Second), MaxEntries: 10_000,
}
cfg.Routing[0].Purpose = "gateway"
if err := Validate(cfg); err != nil {
t.Fatalf("Validate(valid gateway stickySession) error = %v", err)
}
cfg.Gateway.StickySession.Header = ""
if err := Validate(cfg); err == nil || !strings.Contains(err.Error(), "stickySession.header") {
t.Fatalf("Validate(missing stickySession header) error = %v", err)
}
cfg.Gateway.StickySession.Header = "X Proxy Session"
if err := Validate(cfg); err == nil || !strings.Contains(err.Error(), "stickySession.header") {
t.Fatalf("Validate(invalid stickySession header) error = %v", err)
}
cfg.Gateway.StickySession.Header = "X-Proxy-Session"
cfg.Gateway.StickySession.TTL = 0
if err := Validate(cfg); err == nil || !strings.Contains(err.Error(), "stickySession.ttl") {
t.Fatalf("Validate(zero stickySession ttl) error = %v", err)
}
cfg.Gateway.StickySession.TTL = Duration(30 * time.Second)
cfg.Gateway.Auth = Auth{Mode: "none"}
if err := Validate(cfg); err == nil || !strings.Contains(err.Error(), "requires authentication") {
t.Fatalf("Validate(unauthenticated stickySession) error = %v", err)
}
cfg.Gateway.Auth = Auth{Mode: "bearer", Token: "gateway-token"}
cfg.Distribution.StickySession = StickySession{
Enabled: true, Header: "X-Proxy-Session", TTL: Duration(30 * time.Second), MaxEntries: 10,
}
if err := Validate(cfg); err == nil || !strings.Contains(err.Error(), "only supported on gateway") {
t.Fatalf("Validate(distribution stickySession) error = %v", err)
}
}
func TestValidateStaticGatewayRoutingAction(t *testing.T) {
cfg := mustLoadValidConfig(t)
cfg.Routing[0] = Routing{
Name: "direct-api", Enabled: true, Purpose: "gateway", Action: "direct",
Match: RoutingMatch{HostRegex: "^api\\.example\\.test$"},
}
if err := Validate(cfg); err != nil {
t.Fatalf("Validate(static direct routing) = %v", err)
}
cfg.Routing[0].Purpose = "extract"
if err := Validate(cfg); err == nil || !strings.Contains(err.Error(), "only supported on gateway") {
t.Fatalf("Validate(extract direct routing) error = %v", err)
}
}
func TestValidateGatewayTransport(t *testing.T) {
t.Parallel()
cfg := mustLoadValidConfig(t)
cfg.Gateway.Transport = GatewayTransport{
DialTimeout: Duration(time.Second),
HandshakeTimeout: Duration(2 * time.Second),
ResponseHeaderTimeout: Duration(3 * time.Second),
IdleConnTimeout: Duration(time.Minute),
MaxIdleConns: 500,
MaxIdleConnsPerHost: 20,
MaxConnsPerHost: 10,
TunnelBufferBytes: 32 << 10,
TunnelIdleTimeout: Duration(5 * time.Minute),
}
if err := Validate(cfg); err != nil {
t.Fatalf("Validate(valid gateway transport) error = %v", err)
}
cfg.Gateway.Transport.DialTimeout = Duration(-time.Second)
if err := Validate(cfg); err == nil || !strings.Contains(err.Error(), "transport.dialTimeout") {
t.Fatalf("Validate(negative dial timeout) error = %v", err)
}
cfg.Gateway.Transport.DialTimeout = Duration(time.Second)
cfg.Gateway.Transport.MaxConnsPerHost = -1
if err := Validate(cfg); err == nil || !strings.Contains(err.Error(), "transport.maxConnsPerHost") {
t.Fatalf("Validate(negative max connections) error = %v", err)
}
cfg.Gateway.Transport.MaxConnsPerHost = 10
cfg.Gateway.Transport.MaxIdleConns = 19
cfg.Gateway.Transport.MaxIdleConnsPerHost = 20
if err := Validate(cfg); err == nil || !strings.Contains(err.Error(), "maxIdleConnsPerHost") {
t.Fatalf("Validate(inconsistent idle pool) error = %v", err)
}
cfg.Gateway.Transport.MaxIdleConns = 500
cfg.Gateway.Transport.MaxIdleConnsPerHost = 20
cfg.Distribution.Transport = GatewayTransport{MaxConnsPerHost: 10}
if err := Validate(cfg); err == nil || !strings.Contains(err.Error(), "distribution transport") {
t.Fatalf("Validate(distribution transport) error = %v", err)
}
}
func TestValidateMetricsListener(t *testing.T) {
t.Parallel()
cfg := mustLoadValidConfig(t)

View File

@ -80,9 +80,13 @@ func cloneListener(source Listener) Listener {
cloned.Access.AllowCIDRs = cloneStrings(source.Access.AllowCIDRs)
cloned.Access.TrustedProxies = cloneStrings(source.Access.TrustedProxies)
cloned.Auth.CIDRs = cloneStrings(source.Auth.CIDRs)
cloned.Auth.Permissions = cloneStrings(source.Auth.Permissions)
cloned.Auth.ClientPolicy = source.Auth.ClientPolicy.Clone()
cloned.Auth.Methods = append([]AuthMethod(nil), source.Auth.Methods...)
for index := range cloned.Auth.Methods {
cloned.Auth.Methods[index].CIDRs = cloneStrings(source.Auth.Methods[index].CIDRs)
cloned.Auth.Methods[index].Permissions = cloneStrings(source.Auth.Methods[index].Permissions)
cloned.Auth.Methods[index].ClientPolicy = source.Auth.Methods[index].ClientPolicy.Clone()
}
cloned.Retry.RetryMethods = cloneStrings(source.Retry.RetryMethods)
cloned.DestinationPolicy.DenyPrivateNetworks = cloneBool(source.DestinationPolicy.DenyPrivateNetworks)

View File

@ -4,10 +4,14 @@ import (
"fmt"
"math"
"net"
"net/http"
"net/url"
"regexp"
"strconv"
"strings"
"proxy-pool/internal/domain/authorization"
"proxy-pool/internal/domain/clientpolicy"
)
var (
@ -35,6 +39,11 @@ func Validate(cfg *Config) error {
return err
}
}
if cfg.Admin.Enabled {
if err := validateNoClientPolicy("admin", cfg.Admin.Auth); err != nil {
return err
}
}
if cfg.Metrics.Enabled {
if cfg.Metrics.Listen == "" {
return fmt.Errorf("validate metrics listen: address is required")
@ -74,11 +83,15 @@ func Validate(cfg *Config) error {
return fmt.Errorf("validate configuration: at least one enabled upstream is required")
}
seen := make(map[string]struct{}, len(cfg.Routing))
gatewayRoutings := make(map[string]struct{}, len(cfg.Routing))
targetProfilesByUpstream := make(map[string]int, len(cfg.Upstreams))
for index, route := range cfg.Routing {
if err := validateRouting(index, route, cfg.Upstreams, seen); err != nil {
return err
}
if route.Enabled && route.Purpose == "gateway" {
gatewayRoutings[route.Name] = struct{}{}
}
if !route.Enabled || len(route.Check.Targets) == 0 {
continue
}
@ -93,7 +106,27 @@ func Validate(cfg *Config) error {
}
}
}
if cfg.Gateway.Enabled {
if err := validateGatewayClientPolicies(cfg.Gateway.Auth, gatewayRoutings); err != nil {
return err
}
if err := validateGatewayStickySession(cfg.Gateway); err != nil {
return err
}
if err := validateGatewayTransport(cfg.Gateway); err != nil {
return err
}
}
if cfg.Distribution.Enabled {
if err := validateDistributionClientPolicies(cfg.Distribution.Auth, cfg.Upstreams); err != nil {
return err
}
if cfg.Distribution.StickySession.Enabled {
return fmt.Errorf("validate distribution stickySession: only supported on gateway")
}
if gatewayTransportConfigured(cfg.Distribution.Transport) {
return fmt.Errorf("validate distribution transport: only supported on gateway")
}
clientIdentificationMode := cfg.Distribution.ClientIdentification.Mode
if clientIdentificationMode == "" {
clientIdentificationMode = "sourceIP"
@ -124,9 +157,97 @@ func Validate(cfg *Config) error {
return err
}
}
if cfg.Admin.Enabled && cfg.Admin.StickySession.Enabled {
return fmt.Errorf("validate admin stickySession: only supported on gateway")
}
if cfg.Admin.Enabled && gatewayTransportConfigured(cfg.Admin.Transport) {
return fmt.Errorf("validate admin transport: only supported on gateway")
}
return nil
}
func validateGatewayStickySession(listener Listener) error {
item := listener.StickySession
if !item.Enabled {
return nil
}
if listener.Auth.Mode == "" || listener.Auth.Mode == "none" {
return fmt.Errorf("validate gateway stickySession: requires authentication")
}
if !validCanonicalHeaderName(item.Header) {
return fmt.Errorf("validate gateway stickySession.header: must be a canonical HTTP header name")
}
if item.TTL.Value() <= 0 {
return fmt.Errorf("validate gateway stickySession.ttl: must be greater than zero")
}
if item.MaxEntries <= 0 || item.MaxEntries > MaximumPoolSize {
return fmt.Errorf("validate gateway stickySession.maxEntries: must be in [1, %d]", MaximumPoolSize)
}
return nil
}
func validateGatewayTransport(listener Listener) error {
item := listener.Transport
for _, timeout := range []struct {
name string
value Duration
}{
{name: "dialTimeout", value: item.DialTimeout},
{name: "handshakeTimeout", value: item.HandshakeTimeout},
{name: "responseHeaderTimeout", value: item.ResponseHeaderTimeout},
{name: "idleConnTimeout", value: item.IdleConnTimeout},
{name: "tunnelIdleTimeout", value: item.TunnelIdleTimeout},
} {
if timeout.value.Value() < 0 {
return fmt.Errorf("validate gateway transport.%s: must be non-negative", timeout.name)
}
}
for _, limit := range []struct {
name string
value int
}{
{name: "maxIdleConns", value: item.MaxIdleConns},
{name: "maxIdleConnsPerHost", value: item.MaxIdleConnsPerHost},
{name: "maxConnsPerHost", value: item.MaxConnsPerHost},
{name: "tunnelBufferBytes", value: item.TunnelBufferBytes},
} {
if limit.value < 0 || limit.value > MaximumPoolSize {
return fmt.Errorf("validate gateway transport.%s: must be in [0, %d]", limit.name, MaximumPoolSize)
}
}
if item.MaxIdleConns > 0 && item.MaxIdleConnsPerHost > item.MaxIdleConns {
return fmt.Errorf("validate gateway transport.maxIdleConnsPerHost: must not exceed maxIdleConns")
}
return nil
}
func gatewayTransportConfigured(item GatewayTransport) bool {
return item.DialTimeout != 0 ||
item.HandshakeTimeout != 0 ||
item.ResponseHeaderTimeout != 0 ||
item.IdleConnTimeout != 0 ||
item.MaxIdleConns != 0 ||
item.MaxIdleConnsPerHost != 0 ||
item.MaxConnsPerHost != 0 ||
item.TunnelBufferBytes != 0 ||
item.TunnelIdleTimeout != 0
}
func validCanonicalHeaderName(value string) bool {
if value == "" || http.CanonicalHeaderKey(value) != value {
return false
}
for _, character := range []byte(value) {
if character >= 'a' && character <= 'z' || character >= 'A' && character <= 'Z' ||
character >= '0' && character <= '9' || strings.ContainsRune("!#$%&'*+-.^_|~", rune(character)) ||
character == 96 {
continue
}
return false
}
return true
}
func validateControlPlane(item ControlPlane) error {
if err := validateClientTLS("gatewayTLS", item.GatewayTLS); err != nil {
return err
@ -302,6 +423,28 @@ func validateRouting(index int, route Routing, upstreams map[string]Upstream, se
return fmt.Errorf("validate %s pathRegex: %w", scope, err)
}
}
action := routingAction(route.Action)
if err := validateEnum(scope+" action", action, "proxy", "direct", "reject"); err != nil {
return err
}
if action != "proxy" {
if route.Purpose != "gateway" {
return fmt.Errorf("validate %s action: only supported on gateway", scope)
}
if len(route.Upstreams) != 0 {
return fmt.Errorf("validate %s action: direct and reject routes must not reference upstreams", scope)
}
if routingStrategyConfigured(route.Strategy) {
return fmt.Errorf("validate %s action: direct and reject routes must not configure strategy", scope)
}
if route.OnUnavailable.Action != "" || route.OnUnavailable.WaitTimeout != 0 {
return fmt.Errorf("validate %s action: direct and reject routes must not configure onUnavailable", scope)
}
if len(route.Check.Targets) != 0 {
return fmt.Errorf("validate %s action: direct and reject routes must not configure check targets", scope)
}
return nil
}
for _, upstream := range route.Upstreams {
if _, ok := upstreams[upstream]; !ok {
return fmt.Errorf("validate %s: upstream %q does not exist", scope, upstream)
@ -325,6 +468,20 @@ func validateRouting(index int, route Routing, upstreams map[string]Upstream, se
return nil
}
func routingAction(value string) string {
if value == "" {
return "proxy"
}
return value
}
func routingStrategyConfigured(strategy Strategy) bool {
return strategy.Type != "" ||
strategy.SwitchAfterEmptyFetch != 0 ||
strategy.EndBehavior != "" ||
len(strategy.Weights) != 0
}
func validateStrategy(scope string, upstreams []string, strategy Strategy) error {
if err := validateEnum(scope+" strategy.type", strategy.Type,
"sequential", "random", "roundRobin", "weighted", "leastConnections"); err != nil {
@ -372,8 +529,20 @@ func validateStrategy(scope string, upstreams []string, strategy Strategy) error
}
func validateListenerAuth(listener string, auth Auth) error {
if err := authorization.Validate(auth.Permissions); err != nil {
return fmt.Errorf("validate %s auth.permissions: %w", listener, err)
}
if err := auth.ClientPolicy.Validate(); err != nil {
return fmt.Errorf("validate %s auth.client: %w", listener, err)
}
switch auth.Mode {
case "none":
if len(auth.Permissions) != 0 {
return fmt.Errorf("validate %s auth.permissions: requires an authentication mode", listener)
}
if !auth.ClientPolicy.IsZero() {
return fmt.Errorf("validate %s auth.client: requires an authentication mode", listener)
}
return nil
case "usernamePassword":
if auth.Username == "" || (auth.Password == "" && auth.PasswordFile == "") {
@ -395,6 +564,12 @@ func validateListenerAuth(listener string, auth Auth) error {
return err
}
case "any":
if len(auth.Permissions) != 0 {
return fmt.Errorf("validate %s auth.permissions: configure permissions on auth.methods", listener)
}
if !auth.ClientPolicy.IsZero() {
return fmt.Errorf("validate %s auth.client: configure client policy on auth.methods", listener)
}
if len(auth.Methods) == 0 {
return fmt.Errorf("validate %s auth.mode any: methods are required", listener)
}
@ -410,6 +585,12 @@ func validateListenerAuth(listener string, auth Auth) error {
}
func validateAuthMethod(listener string, index int, method AuthMethod) error {
if err := authorization.Validate(method.Permissions); err != nil {
return fmt.Errorf("validate %s auth.methods[%d].permissions: %w", listener, index, err)
}
if err := method.ClientPolicy.Validate(); err != nil {
return fmt.Errorf("validate %s auth.methods[%d].client: %w", listener, index, err)
}
switch method.Mode {
case "usernamePassword":
if method.Username == "" || (method.Password == "" && method.PasswordFile == "") {
@ -436,6 +617,76 @@ func validateAuthMethod(listener string, index int, method AuthMethod) error {
return nil
}
func validateNoClientPolicy(listener string, auth Auth) error {
if !auth.ClientPolicy.IsZero() {
return fmt.Errorf("validate %s auth.client: client extraction policy is only supported on distribution", listener)
}
for index, method := range auth.Methods {
if !method.ClientPolicy.IsZero() {
return fmt.Errorf("validate %s auth.methods[%d].client: client extraction policy is only supported on distribution", listener, index)
}
}
return nil
}
func validateDistributionClientPolicies(auth Auth, upstreams map[string]Upstream) error {
if err := validateDistributionClientPolicy("distribution auth.client", auth.ClientPolicy, upstreams); err != nil {
return err
}
for index, method := range auth.Methods {
if err := validateDistributionClientPolicy(
fmt.Sprintf("distribution auth.methods[%d].client", index), method.ClientPolicy, upstreams,
); err != nil {
return err
}
}
return nil
}
func validateDistributionClientPolicy(scope string, policy clientpolicy.Policy, upstreams map[string]Upstream) error {
if len(policy.AllowedRoutings) != 0 || policy.RequestsPerMinute != 0 || policy.MaxConcurrentConnections != 0 {
return fmt.Errorf("validate %s: routing access and Gateway limits are only supported on gateway", scope)
}
for _, upstream := range policy.AllowedUpstreams {
if _, exists := upstreams[upstream]; !exists {
return fmt.Errorf("validate %s.allowedUpstreams: upstream %q does not exist", scope, upstream)
}
}
return nil
}
func validateGatewayClientPolicies(auth Auth, routings map[string]struct{}) error {
if err := validateGatewayClientPolicy("gateway auth.client", auth.ClientPolicy, routings); err != nil {
return err
}
for index, method := range auth.Methods {
if err := validateGatewayClientPolicy(
fmt.Sprintf("gateway auth.methods[%d].client", index), method.ClientPolicy, routings,
); err != nil {
return err
}
}
return nil
}
func validateGatewayClientPolicy(scope string, policy clientpolicy.Policy, routings map[string]struct{}) error {
if policy.MaxExtractCount != 0 || len(policy.AllowedUpstreams) != 0 || len(policy.AllowedRegions) != 0 {
return fmt.Errorf("validate %s: extraction constraints are only supported on distribution", scope)
}
if int64(policy.RequestsPerMinute) > MaximumExactCounter {
return fmt.Errorf("validate %s.requestsPerMinute: exceeds exact counter range", scope)
}
if int64(policy.MaxConcurrentConnections) > MaximumExactCounter {
return fmt.Errorf("validate %s.maxConcurrentConnections: exceeds exact counter range", scope)
}
for _, routing := range policy.AllowedRoutings {
if _, exists := routings[routing]; !exists {
return fmt.Errorf("validate %s.allowedRoutings: routing %q does not exist or is not an enabled gateway routing", scope, routing)
}
}
return nil
}
func validateCIDRs(name string, cidrs []string) error {
for _, cidr := range cidrs {
if _, _, err := net.ParseCIDR(cidr); err != nil {

View File

@ -5,18 +5,24 @@ import (
"errors"
"fmt"
"net/http"
"strconv"
"strings"
"time"
"proxy-pool/internal/domain/adminstate"
"proxy-pool/internal/domain/authorization"
"proxy-pool/internal/platform/httpapi"
"proxy-pool/internal/platform/httpsecurity"
)
const (
statusPath = "/api/v1/status"
auditPath = "/api/v1/audit"
reloadPath = "/api/v1/config/reload"
upstreamPrefix = "/api/v1/upstreams/"
routingPrefix = "/api/v1/routing/"
maxResourceNameBytes = 128
defaultAuditPageSize = 100
)
var (
@ -29,6 +35,7 @@ var (
type Service interface {
Status(context.Context) (Status, error)
ReadAudit(context.Context, AuditQuery) (AuditPage, error)
SetUpstreamEnabled(context.Context, SetUpstreamCommand) (MutationResult, error)
SwitchRouting(context.Context, SwitchCommand) (MutationResult, error)
ReloadConfiguration(context.Context, ReloadCommand) (MutationResult, error)
@ -78,6 +85,36 @@ type MutationResult struct {
Message string `json:"message,omitempty"`
}
type AuditQuery struct {
AfterID uint64
Limit int
}
func (query AuditQuery) Validate() error {
if query.Limit <= 0 || query.Limit > adminstate.MaxPageSize {
return ErrInvalidConfiguration
}
return nil
}
type AuditRecord struct {
ID uint64 `json:"id"`
RequestID string `json:"requestId"`
ActorID string `json:"actorId"`
SourceIP string `json:"sourceIp,omitempty"`
Action string `json:"action"`
ResourceType string `json:"resourceType"`
ResourceName string `json:"resourceName"`
Changed bool `json:"changed"`
Version uint64 `json:"version"`
Reason string `json:"reason,omitempty"`
OccurredAt time.Time `json:"occurredAt"`
}
type AuditPage struct {
Records []AuditRecord `json:"records"`
}
type SetUpstreamCommand struct {
RequestID string
ActorID string
@ -134,12 +171,27 @@ func (handler *Handler) ServeHTTP(writer http.ResponseWriter, request *http.Requ
if !requireMethod(writer, request, http.MethodGet, requestID) {
return
}
if !requirePermission(writer, identity, authorization.AdminRead, requestID) {
return
}
handler.getStatus(writer, request, requestID)
return
case auditPath:
if !requireMethod(writer, request, http.MethodGet, requestID) {
return
}
if !requirePermission(writer, identity, authorization.AdminRead, requestID) {
return
}
handler.readAudit(writer, request, requestID)
return
case reloadPath:
if !requireMethod(writer, request, http.MethodPost, requestID) {
return
}
if !requirePermission(writer, identity, authorization.AdminWrite, requestID) {
return
}
handler.reload(writer, request, requestID, identity)
return
}
@ -148,6 +200,9 @@ func (handler *Handler) ServeHTTP(writer http.ResponseWriter, request *http.Requ
if !requireMethod(writer, request, http.MethodPost, requestID) {
return
}
if !requirePermission(writer, identity, authorization.AdminWrite, requestID) {
return
}
handler.setUpstreamEnabled(writer, request, name, action == "enable", requestID, identity)
return
}
@ -155,6 +210,9 @@ func (handler *Handler) ServeHTTP(writer http.ResponseWriter, request *http.Requ
if !requireMethod(writer, request, http.MethodPost, requestID) {
return
}
if !requirePermission(writer, identity, authorization.AdminWrite, requestID) {
return
}
handler.switchRouting(writer, request, name, requestID, identity)
return
}
@ -162,6 +220,19 @@ func (handler *Handler) ServeHTTP(writer http.ResponseWriter, request *http.Requ
writeTransportProblem(writer, http.StatusNotFound, "NOT_FOUND", "Not found", "the requested endpoint does not exist", requestID)
}
func requirePermission(
writer http.ResponseWriter,
identity httpsecurity.Identity,
required string,
requestID string,
) bool {
if identity.Allows(required) {
return true
}
writeTransportProblem(writer, http.StatusForbidden, "FORBIDDEN", "Forbidden", "the authenticated identity lacks endpoint permission", requestID)
return false
}
func (handler *Handler) getStatus(writer http.ResponseWriter, request *http.Request, requestID string) {
status, err := handler.service.Status(request.Context())
if err != nil {
@ -178,6 +249,57 @@ func (handler *Handler) getStatus(writer http.ResponseWriter, request *http.Requ
_ = httpapi.WriteJSON(writer, http.StatusOK, status)
}
func (handler *Handler) readAudit(writer http.ResponseWriter, request *http.Request, requestID string) {
query, err := parseAuditQuery(request)
if err != nil {
writeTransportProblem(writer, http.StatusBadRequest, "INVALID_AUDIT_QUERY", "Invalid audit query", "audit pagination fields violate the API contract", requestID)
return
}
page, err := handler.service.ReadAudit(request.Context(), query)
if err != nil {
writeServiceProblem(writer, err, requestID)
return
}
if page.Records == nil {
page.Records = []AuditRecord{}
}
writer.Header().Set(httpapi.HeaderRequestID, requestID)
_ = httpapi.WriteJSON(writer, http.StatusOK, page)
}
func parseAuditQuery(request *http.Request) (AuditQuery, error) {
if request == nil || request.URL == nil {
return AuditQuery{}, ErrInvalidConfiguration
}
values := request.URL.Query()
query := AuditQuery{Limit: defaultAuditPageSize}
for name, value := range values {
if len(value) != 1 {
return AuditQuery{}, ErrInvalidConfiguration
}
switch name {
case "afterId":
parsed, err := strconv.ParseUint(value[0], 10, 64)
if err != nil {
return AuditQuery{}, ErrInvalidConfiguration
}
query.AfterID = parsed
case "limit":
parsed, err := strconv.ParseUint(value[0], 10, 32)
if err != nil || parsed == 0 || parsed > adminstate.MaxPageSize {
return AuditQuery{}, ErrInvalidConfiguration
}
query.Limit = int(parsed)
default:
return AuditQuery{}, ErrInvalidConfiguration
}
}
if err := query.Validate(); err != nil {
return AuditQuery{}, err
}
return query, nil
}
func (handler *Handler) setUpstreamEnabled(
writer http.ResponseWriter,
request *http.Request,

View File

@ -8,7 +8,9 @@ import (
"net/http/httptest"
"strings"
"testing"
"time"
"proxy-pool/internal/domain/authorization"
"proxy-pool/internal/platform/httpapi"
"proxy-pool/internal/platform/httpsecurity"
)
@ -38,6 +40,109 @@ func TestHandlerReturnsStatusWithoutSensitiveDetails(t *testing.T) {
}
}
func TestHandlerEnforcesReadAndWritePermissions(t *testing.T) {
t.Parallel()
service := &stubService{}
readHandler, err := NewHandler(service, scopedAuthorizer{permissions: []string{authorization.AdminRead}}, Options{MaxBodyBytes: 1024})
if err != nil {
t.Fatalf("NewHandler(read) error = %v", err)
}
status := httptest.NewRecorder()
readHandler.ServeHTTP(status, httptest.NewRequest(http.MethodGet, "/api/v1/status", nil))
if status.Code != http.StatusOK || service.statusCalls != 1 {
t.Fatalf("read status = %d calls=%d", status.Code, service.statusCalls)
}
mutation := httptest.NewRecorder()
readHandler.ServeHTTP(mutation, httptest.NewRequest(http.MethodPost, "/api/v1/config/reload", nil))
if mutation.Code != http.StatusForbidden || service.lastReload != (ReloadCommand{}) || !strings.Contains(mutation.Body.String(), `"code":"FORBIDDEN"`) {
t.Fatalf("read mutation response = status %d command %+v body=%s", mutation.Code, service.lastReload, mutation.Body.String())
}
writeHandler, err := NewHandler(service, scopedAuthorizer{permissions: []string{authorization.AdminWrite}}, Options{MaxBodyBytes: 1024})
if err != nil {
t.Fatalf("NewHandler(write) error = %v", err)
}
audit := httptest.NewRecorder()
writeHandler.ServeHTTP(audit, httptest.NewRequest(http.MethodGet, "/api/v1/audit", nil))
if audit.Code != http.StatusForbidden || service.auditCalls != 0 || !strings.Contains(audit.Body.String(), `"code":"FORBIDDEN"`) {
t.Fatalf("write audit response = status %d calls=%d body=%s", audit.Code, service.auditCalls, audit.Body.String())
}
}
func TestHandlerReadsBoundedAuditPage(t *testing.T) {
t.Parallel()
occurredAt := time.Date(2026, 8, 2, 9, 30, 0, 0, time.UTC)
service := &stubService{audit: AuditPage{Records: []AuditRecord{{
ID: 8, RequestID: "req-8", ActorID: "admin:alice", Action: "switch_routing",
ResourceType: "routing", ResourceName: "checkout", Changed: true, Version: 12,
Reason: "capacity", OccurredAt: occurredAt,
}}}}
handler := mustHandler(t, service)
recorder := httptest.NewRecorder()
request := httptest.NewRequest(http.MethodGet, "/api/v1/audit?afterId=7&limit=2", nil)
request.Header.Set(httpapi.HeaderRequestID, "req-audit-page")
handler.ServeHTTP(recorder, request)
if recorder.Code != http.StatusOK {
t.Fatalf("status = %d, want %d; body=%s", recorder.Code, http.StatusOK, recorder.Body.String())
}
if service.auditCalls != 1 || service.lastAudit != (AuditQuery{AfterID: 7, Limit: 2}) {
t.Fatalf("ReadAudit() calls=%d query=%+v", service.auditCalls, service.lastAudit)
}
if requestID := recorder.Header().Get(httpapi.HeaderRequestID); requestID != "req-audit-page" {
t.Fatalf("response request ID = %q, want req-audit-page", requestID)
}
var response AuditPage
if err := json.Unmarshal(recorder.Body.Bytes(), &response); err != nil {
t.Fatalf("decode response: %v", err)
}
if len(response.Records) != 1 || response.Records[0].ID != 8 || !response.Records[0].OccurredAt.Equal(occurredAt) {
t.Fatalf("unexpected audit response: %+v", response)
}
}
func TestHandlerUsesDefaultAuditLimitAndRejectsInvalidAuditQueries(t *testing.T) {
t.Parallel()
tests := []struct {
name string
path string
wantStatus int
wantQuery AuditQuery
}{
{name: "default", path: "/api/v1/audit", wantStatus: http.StatusOK, wantQuery: AuditQuery{Limit: defaultAuditPageSize}},
{name: "unknown field", path: "/api/v1/audit?beforeId=1", wantStatus: http.StatusBadRequest},
{name: "duplicate field", path: "/api/v1/audit?limit=1&limit=2", wantStatus: http.StatusBadRequest},
{name: "zero limit", path: "/api/v1/audit?limit=0", wantStatus: http.StatusBadRequest},
{name: "oversized limit", path: "/api/v1/audit?limit=1001", wantStatus: http.StatusBadRequest},
{name: "invalid cursor", path: "/api/v1/audit?afterId=nope", wantStatus: http.StatusBadRequest},
}
for _, test := range tests {
test := test
t.Run(test.name, func(t *testing.T) {
t.Parallel()
service := &stubService{}
handler := mustHandler(t, service)
recorder := httptest.NewRecorder()
handler.ServeHTTP(recorder, httptest.NewRequest(http.MethodGet, test.path, nil))
if recorder.Code != test.wantStatus {
t.Fatalf("status = %d, want %d; body=%s", recorder.Code, test.wantStatus, recorder.Body.String())
}
if test.wantStatus == http.StatusOK {
if service.auditCalls != 1 || service.lastAudit != test.wantQuery {
t.Fatalf("ReadAudit() calls=%d query=%+v, want %+v", service.auditCalls, service.lastAudit, test.wantQuery)
}
return
}
if service.auditCalls != 0 || !strings.Contains(recorder.Body.String(), `"code":"INVALID_AUDIT_QUERY"`) {
t.Fatalf("invalid audit query reached service or returned wrong problem: calls=%d body=%s", service.auditCalls, recorder.Body.String())
}
})
}
}
func TestNewHandlerRejectsMissingDependenciesAndInvalidLimit(t *testing.T) {
t.Parallel()
if _, err := NewHandler(nil, allowAuthorizer{}, Options{MaxBodyBytes: 1024}); !errors.Is(err, ErrInvalidHandler) {
@ -232,6 +337,7 @@ func TestHandlerRejectsInvalidTransportRequests(t *testing.T) {
wantStatus int
}{
{name: "method", method: http.MethodPut, path: "/api/v1/config/reload", wantStatus: http.StatusMethodNotAllowed},
{name: "audit method", method: http.MethodPost, path: "/api/v1/audit", wantStatus: http.StatusMethodNotAllowed},
{name: "unknown route", method: http.MethodGet, path: "/missing", wantStatus: http.StatusNotFound},
{name: "invalid name", method: http.MethodPost, path: "/api/v1/upstreams//enable", wantStatus: http.StatusNotFound},
{name: "unknown JSON field", method: http.MethodPost, path: "/api/v1/routing/r/switch", body: `{"expectedCurrent":"a","target":"b","extra":1}`, content: httpapi.JSONContentType, wantStatus: http.StatusBadRequest},
@ -277,12 +383,15 @@ func mustHandler(t *testing.T, service Service) *Handler {
type stubService struct {
status Status
audit AuditPage
mutation MutationResult
err error
lastAudit AuditQuery
lastUpstream SetUpstreamCommand
lastSwitch SwitchCommand
lastReload ReloadCommand
statusCalls int
auditCalls int
}
func (service *stubService) Status(context.Context) (Status, error) {
@ -290,6 +399,12 @@ func (service *stubService) Status(context.Context) (Status, error) {
return service.status, service.err
}
func (service *stubService) ReadAudit(_ context.Context, query AuditQuery) (AuditPage, error) {
service.auditCalls++
service.lastAudit = query
return service.audit, service.err
}
func (service *stubService) SetUpstreamEnabled(_ context.Context, command SetUpstreamCommand) (MutationResult, error) {
service.lastUpstream = command
return service.mutation, service.err
@ -311,6 +426,12 @@ func (allowAuthorizer) Resolve(*http.Request) (httpsecurity.Identity, error) {
return httpsecurity.Identity{ClientID: "admin:test", SourceIP: "192.0.2.10"}, nil
}
type scopedAuthorizer struct{ permissions []string }
func (authorizer scopedAuthorizer) Resolve(*http.Request) (httpsecurity.Identity, error) {
return httpsecurity.Identity{ClientID: "admin:scoped", SourceIP: "192.0.2.20", Permissions: authorizer.permissions}, nil
}
type rejectAuthorizer struct{}
func (rejectAuthorizer) Resolve(*http.Request) (httpsecurity.Identity, error) {

View File

@ -16,6 +16,7 @@ var ErrInvalidApplicationService = errors.New("invalid admin application service
type StateRepository interface {
adminstate.Mutator
adminstate.SnapshotReader
adminstate.AuditReader
}
type OperationalStatusReader interface {
@ -37,6 +38,12 @@ type RuntimeController interface {
ValidateUpstream(context.Context, string) error
}
// SnapshotRefreshNotifier requests immediate full Worker snapshots after a
// committed management change. Implementations must coalesce notifications.
type SnapshotRefreshNotifier interface {
NotifySnapshotRefresh()
}
var _ ConfigurationPublisher = (*config.Store)(nil)
type ApplicationDependencies struct {
@ -45,6 +52,7 @@ type ApplicationDependencies struct {
Configuration ConfigurationLoader
Publisher ConfigurationPublisher
Runtime RuntimeController
SnapshotRefresh SnapshotRefreshNotifier
}
type ApplicationOptions struct {
@ -80,6 +88,7 @@ type ApplicationService struct {
configuration ConfigurationLoader
publisher ConfigurationPublisher
runtime RuntimeController
snapshotRefresh SnapshotRefreshNotifier
now func() time.Time
fingerprintKey []byte
}
@ -98,6 +107,7 @@ func NewApplicationService(dependencies ApplicationDependencies, options Applica
configuration: dependencies.Configuration,
publisher: dependencies.Publisher,
runtime: dependencies.Runtime,
snapshotRefresh: dependencies.SnapshotRefresh,
now: options.Now,
fingerprintKey: append([]byte(nil), options.FingerprintKey...),
}, nil
@ -119,6 +129,9 @@ func (service *ApplicationService) SetUpstreamEnabled(ctx context.Context, comma
if err == nil && service.runtime != nil {
service.runtime.Notify()
}
if err == nil && result.Changed && service.snapshotRefresh != nil {
service.snapshotRefresh.NotifySnapshotRefresh()
}
return mutationResult(result), mapAdminStateError(err)
}
@ -132,9 +145,42 @@ func (service *ApplicationService) SwitchRouting(ctx context.Context, command Sw
Target: command.Target,
Reason: command.Reason,
})
if err == nil && result.Changed && service.snapshotRefresh != nil {
service.snapshotRefresh.NotifySnapshotRefresh()
}
return mutationResult(result), mapAdminStateError(err)
}
// ReadAudit returns one bounded, stable page from the authoritative management
// audit log. It deliberately exposes only management-plane records and never
// reads Proxy activity, credentials, or Provider payloads.
func (service *ApplicationService) ReadAudit(ctx context.Context, query AuditQuery) (AuditPage, error) {
if service == nil || nilInterface(service.state) || query.Validate() != nil {
return AuditPage{}, ErrInvalidConfiguration
}
records, err := service.state.ReadAudit(ctx, adminstate.AuditQuery{AfterID: query.AfterID, Limit: query.Limit})
if err != nil {
return AuditPage{}, mapAdminStateError(err)
}
page := AuditPage{Records: make([]AuditRecord, 0, len(records))}
for _, record := range records {
page.Records = append(page.Records, AuditRecord{
ID: record.ID,
RequestID: record.RequestID,
ActorID: record.Actor.ID,
SourceIP: record.Actor.SourceIP,
Action: string(record.Action),
ResourceType: record.ResourceType,
ResourceName: record.ResourceName,
Changed: record.Changed,
Version: record.Revision,
Reason: record.Reason,
OccurredAt: record.OccurredAt.UTC(),
})
}
return page, nil
}
func (service *ApplicationService) Status(ctx context.Context) (Status, error) {
snapshot, err := service.state.Snapshot(ctx)
if err != nil {
@ -256,6 +302,9 @@ func (service *ApplicationService) ApplyConfiguration(
if published && service.runtime != nil {
service.runtime.Notify()
}
if published && service.snapshotRefresh != nil {
service.snapshotRefresh.NotifySnapshotRefresh()
}
return mutationResult(result), nil
}

View File

@ -29,12 +29,14 @@ func TestApplicationServiceMapsUpstreamMutationToAdminState(t *testing.T) {
},
}
runtime := &recordingRuntimeNotifier{}
refresh := &recordingSnapshotRefreshNotifier{}
service, err := NewApplicationService(ApplicationDependencies{
State: state,
Operations: staticOperationalStatusReader{},
Configuration: staticConfigurationLoader{},
Publisher: &recordingConfigurationPublisher{},
Runtime: runtime,
SnapshotRefresh: refresh,
}, applicationTestOptions(func() time.Time { return now }))
if err != nil {
t.Fatalf("NewApplicationService() error = %v", err)
@ -65,6 +67,9 @@ func TestApplicationServiceMapsUpstreamMutationToAdminState(t *testing.T) {
if runtime.notifications != 1 {
t.Fatalf("runtime notifications = %d, want 1", runtime.notifications)
}
if refresh.notifications != 1 {
t.Fatalf("snapshot refresh notifications = %d, want 1", refresh.notifications)
}
}
func TestApplicationServicePreflightsProviderRuntimeBeforeMutation(t *testing.T) {
@ -147,7 +152,14 @@ func TestApplicationServiceMapsRoutingSwitchAndDomainErrors(t *testing.T) {
t.Parallel()
now := time.Date(2026, 7, 29, 11, 0, 0, 0, time.FixedZone("test", 8*60*60))
state := &recordingAdminState{mutation: adminstate.MutationResult{RequestID: "req-switch", Changed: true, Revision: 21}}
service := mustApplicationService(t, state, applicationTestOptions(func() time.Time { return now }))
refresh := &recordingSnapshotRefreshNotifier{}
service, serviceErr := NewApplicationService(ApplicationDependencies{
State: state, Operations: staticOperationalStatusReader{}, Configuration: staticConfigurationLoader{},
Publisher: &recordingConfigurationPublisher{}, SnapshotRefresh: refresh,
}, applicationTestOptions(func() time.Time { return now }))
if serviceErr != nil {
t.Fatalf("NewApplicationService() = %v", serviceErr)
}
result, err := service.SwitchRouting(context.Background(), SwitchCommand{
RequestID: "req-switch", ActorID: "admin:bob", SourceIP: "198.51.100.7",
@ -166,6 +178,9 @@ func TestApplicationServiceMapsRoutingSwitchAndDomainErrors(t *testing.T) {
if state.lastSwitch != wantCommand {
t.Fatalf("admin state command = %+v, want %+v", state.lastSwitch, wantCommand)
}
if refresh.notifications != 1 {
t.Fatalf("snapshot refresh notifications = %d, want 1", refresh.notifications)
}
tests := []struct {
domain error
@ -186,6 +201,45 @@ func TestApplicationServiceMapsRoutingSwitchAndDomainErrors(t *testing.T) {
t.Fatalf("SwitchRouting(%v) error = %v, want mapped %v preserving cause", test.domain, err, test.want)
}
}
if refresh.notifications != 1 {
t.Fatalf("snapshot refresh notifications after failed switches = %d, want 1", refresh.notifications)
}
}
func TestApplicationServiceReadsAuthoritativeAuditPage(t *testing.T) {
t.Parallel()
occurredAt := time.Date(2026, 8, 2, 10, 0, 0, 0, time.FixedZone("test", 8*60*60))
state := &recordingAdminState{audits: []adminstate.AuditRecord{{
ID: 42, RequestID: "req-switch", Actor: adminstate.Actor{ID: "admin:alice", SourceIP: "192.0.2.10"},
Action: adminstate.ActionSwitchRoute, ResourceType: "routing", ResourceName: "checkout",
Changed: true, Revision: 9, Reason: "capacity", OccurredAt: occurredAt,
}}}
service := mustApplicationService(t, state, applicationTestOptions(time.Now))
page, err := service.ReadAudit(context.Background(), AuditQuery{AfterID: 41, Limit: 2})
if err != nil {
t.Fatalf("ReadAudit() error = %v", err)
}
if state.lastAuditQuery != (adminstate.AuditQuery{AfterID: 41, Limit: 2}) {
t.Fatalf("audit query = %+v", state.lastAuditQuery)
}
if len(page.Records) != 1 {
t.Fatalf("audit records = %+v", page.Records)
}
record := page.Records[0]
if record.ID != 42 || record.Action != string(adminstate.ActionSwitchRoute) || record.Version != 9 ||
record.ActorID != "admin:alice" || record.SourceIP != "192.0.2.10" || !record.OccurredAt.Equal(occurredAt.UTC()) {
t.Fatalf("mapped audit record = %+v", record)
}
state.err = adminstate.ErrUnavailable
if _, err := service.ReadAudit(context.Background(), AuditQuery{Limit: 1}); !errors.Is(err, ErrUnavailable) || !errors.Is(err, adminstate.ErrUnavailable) {
t.Fatalf("ReadAudit() unavailable error = %v", err)
}
if _, err := service.ReadAudit(context.Background(), AuditQuery{}); !errors.Is(err, ErrInvalidConfiguration) {
t.Fatalf("ReadAudit() invalid query error = %v", err)
}
}
func TestApplicationServiceBuildsStatusFromAuthoritativeAndOperationalSnapshots(t *testing.T) {
@ -281,6 +335,7 @@ func TestApplicationServiceReloadPersistsManagementViewBeforePublishing(t *testi
configuration := validReloadConfiguration()
publisher := &recordingConfigurationPublisher{}
runtime := &recordingRuntimeNotifier{}
refresh := &recordingSnapshotRefreshNotifier{}
state := &recordingAdminState{
mutation: adminstate.MutationResult{RequestID: "req-reload", Changed: true, Revision: 42},
snapshot: adminstate.Snapshot{Routings: []adminstate.RoutingState{
@ -297,7 +352,7 @@ func TestApplicationServiceReloadPersistsManagementViewBeforePublishing(t *testi
Configuration: staticConfigurationLoader{loaded: LoadedConfiguration{
Value: configuration, Source: "configs/proxy-pool.yaml",
}},
Publisher: publisher, Runtime: runtime,
Publisher: publisher, Runtime: runtime, SnapshotRefresh: refresh,
}, applicationTestOptions(func() time.Time { return now }))
if err != nil {
t.Fatalf("NewApplicationService() error = %v", err)
@ -318,6 +373,9 @@ func TestApplicationServiceReloadPersistsManagementViewBeforePublishing(t *testi
if runtime.notifications != 1 {
t.Fatalf("runtime notifications = %d, want 1", runtime.notifications)
}
if refresh.notifications != 1 {
t.Fatalf("snapshot refresh notifications = %d, want 1", refresh.notifications)
}
command := state.lastConfig
if command.RequestID != "req-reload" || command.Actor != (adminstate.Actor{ID: "admin:alice", SourceIP: "192.0.2.10"}) ||
!command.OccurredAt.Equal(now) || command.Source != "configs/proxy-pool.yaml" {
@ -623,8 +681,11 @@ type recordingAdminState struct {
mutation adminstate.MutationResult
err error
snapshot adminstate.Snapshot
audits []adminstate.AuditRecord
lastAuditQuery adminstate.AuditQuery
lastUpstream adminstate.SetUpstreamCommand
lastSwitch adminstate.SwitchRoutingCommand
lastDisable adminstate.DisableRoutingCommand
lastConfig adminstate.CommitConfigCommand
onCommit func(adminstate.CommitConfigCommand)
}
@ -643,6 +704,10 @@ func (state *orderedCommitState) SwitchRouting(context.Context, adminstate.Switc
return adminstate.MutationResult{}, nil
}
func (state *orderedCommitState) DisableRouting(context.Context, adminstate.DisableRoutingCommand) (adminstate.MutationResult, error) {
return adminstate.MutationResult{}, nil
}
func (state *orderedCommitState) CommitConfig(_ context.Context, command adminstate.CommitConfigCommand) (adminstate.MutationResult, error) {
revision := state.next.Add(1)
if revision == 1 {
@ -656,6 +721,10 @@ func (*orderedCommitState) Snapshot(context.Context) (adminstate.Snapshot, error
return adminstate.Snapshot{}, nil
}
func (*orderedCommitState) ReadAudit(context.Context, adminstate.AuditQuery) ([]adminstate.AuditRecord, error) {
return nil, nil
}
func (state *recordingAdminState) SetUpstreamEnabled(_ context.Context, command adminstate.SetUpstreamCommand) (adminstate.MutationResult, error) {
state.lastUpstream = command
return state.mutation, state.err
@ -666,6 +735,11 @@ func (state *recordingAdminState) SwitchRouting(_ context.Context, command admin
return state.mutation, state.err
}
func (state *recordingAdminState) DisableRouting(_ context.Context, command adminstate.DisableRoutingCommand) (adminstate.MutationResult, error) {
state.lastDisable = command
return state.mutation, state.err
}
func (state *recordingAdminState) CommitConfig(_ context.Context, command adminstate.CommitConfigCommand) (adminstate.MutationResult, error) {
state.lastConfig = command
if state.onCommit != nil {
@ -678,6 +752,11 @@ func (state *recordingAdminState) Snapshot(context.Context) (adminstate.Snapshot
return state.snapshot, state.err
}
func (state *recordingAdminState) ReadAudit(_ context.Context, query adminstate.AuditQuery) ([]adminstate.AuditRecord, error) {
state.lastAuditQuery = query
return append([]adminstate.AuditRecord(nil), state.audits...), state.err
}
type staticOperationalStatusReader struct {
status OperationalStatus
err error
@ -712,6 +791,10 @@ type recordingRuntimeNotifier struct {
validationErr error
}
type recordingSnapshotRefreshNotifier struct{ notifications int }
func (notifier *recordingSnapshotRefreshNotifier) NotifySnapshotRefresh() { notifier.notifications++ }
func (notifier *recordingRuntimeNotifier) Notify() { notifier.notifications++ }
func (notifier *recordingRuntimeNotifier) ValidateConfiguration(context.Context, *config.Config) error {

View File

@ -19,6 +19,7 @@ import (
"proxy-pool/internal/controller/operations"
"proxy-pool/internal/controller/pool"
"proxy-pool/internal/controller/provider"
controllerRouting "proxy-pool/internal/controller/routing"
controllerRuntime "proxy-pool/internal/controller/runtime"
"proxy-pool/internal/controller/worker"
"proxy-pool/internal/domain/activitypool"
@ -77,6 +78,7 @@ type ports struct {
coordinator provider.Coordinator
credentials credentials.Store
providerResults provider.ResultRecorder
capacityMetrics pool.CapacityObserver
workerStore workerruntime.ControlStore
close func() error
}
@ -176,13 +178,63 @@ func runWithWorkerFactory(
if err != nil {
return fmt.Errorf("%w: build Provider supervisor: %w", ErrStartup, err)
}
snapshotRefresh := worker.NewSnapshotRefreshBroker()
var sequentialCoordinator lifecycle.Runner
dependencies := controllerRuntime.Dependencies{}
var checkerMetrics healthDomain.TaskMetricsObserver
var drainMetrics healthDomain.DrainMetricsObserver
var extractionMetrics extraction.MetricsObserver
if loaded.Value.Metrics.Enabled {
if nilInterface(opened.metricsReadiness) {
return errors.Join(ErrStartup, ErrInvalidOptions)
}
collector, collectorErr := platformMetrics.NewCheckerCollector(prometheus.DefaultRegisterer)
if collectorErr != nil {
return fmt.Errorf("%w: build Checker metrics: %w", ErrStartup, collectorErr)
}
checkerMetrics = collector
drainCollector, drainCollectorErr := platformMetrics.NewDrainCollector(prometheus.DefaultRegisterer)
if drainCollectorErr != nil {
return fmt.Errorf("%w: build Drain metrics: %w", ErrStartup, drainCollectorErr)
}
drainMetrics = drainCollector
providerCollector, providerCollectorErr := platformMetrics.NewProviderCollector(prometheus.DefaultRegisterer)
if providerCollectorErr != nil {
return fmt.Errorf("%w: build Provider metrics: %w", ErrStartup, providerCollectorErr)
}
if registrar, ok := opened.providerResults.(provider.ResultObserverRegistrar); ok && !nilInterface(registrar) {
registrar.AddResultObserver(providerCollector)
}
capacityCollector, capacityCollectorErr := platformMetrics.NewCapacityCollector(prometheus.DefaultRegisterer)
if capacityCollectorErr != nil {
return fmt.Errorf("%w: build capacity metrics: %w", ErrStartup, capacityCollectorErr)
}
opened.capacityMetrics = capacityCollector
extractionCollector, extractionCollectorErr := platformMetrics.NewExtractionCollector(prometheus.DefaultRegisterer)
if extractionCollectorErr != nil {
return fmt.Errorf("%w: build extraction metrics: %w", ErrStartup, extractionCollectorErr)
}
extractionMetrics = extractionCollector
handler, handlerErr := platformMetrics.NewHandler(platformMetrics.Dependencies{
Gatherer: prometheus.DefaultGatherer, Readiness: opened.metricsReadiness,
})
if handlerErr != nil {
return fmt.Errorf("%w: build metrics handler: %w", ErrStartup, handlerErr)
}
dependencies.MetricsHandler = handler
}
if loaded.Value.Distribution.Enabled {
if nilInterface(opened.activity) || nilInterface(opened.readiness) || nilInterface(opened.admission) {
return errors.Join(ErrStartup, ErrInvalidOptions)
}
service, serviceErr := extraction.NewService(opened.activity, extractionPolicy(loaded.Value), opened.admission, options.Now)
service, serviceErr := extraction.NewService(
opened.activity,
extractionPolicy(loaded.Value),
opened.admission,
options.Now,
extractionMetrics,
)
if serviceErr != nil {
return fmt.Errorf("%w: build extraction service: %w", ErrStartup, serviceErr)
}
@ -208,7 +260,7 @@ func runWithWorkerFactory(
}
service, serviceErr := admin.NewApplicationService(admin.ApplicationDependencies{
State: opened.state, Operations: statusReader, Configuration: loader, Publisher: configurationStore,
Runtime: supervisor,
Runtime: supervisor, SnapshotRefresh: snapshotRefresh,
}, admin.ApplicationOptions{Now: options.Now, FingerprintKey: options.FingerprintKey})
if serviceErr != nil {
return fmt.Errorf("%w: build admin service: %w", ErrStartup, serviceErr)
@ -219,26 +271,23 @@ func runWithWorkerFactory(
return fmt.Errorf("%w: commit startup configuration: %w", ErrStartup, applyErr)
}
dependencies.AdminService = service
if stats, ok := opened.providerResults.(provider.StatsReader); ok && !nilInterface(stats) {
coordinator, coordinatorErr := controllerRouting.NewSequentialCoordinator(
configurationStore,
opened.state,
stats,
snapshotRefresh,
controllerRouting.Options{Now: options.Now},
)
if coordinatorErr != nil {
return fmt.Errorf("%w: build Sequential routing coordinator: %w", ErrStartup, coordinatorErr)
}
var checkerMetrics healthDomain.TaskMetricsObserver
if loaded.Value.Metrics.Enabled {
if nilInterface(opened.metricsReadiness) {
return errors.Join(ErrStartup, ErrInvalidOptions)
if registrar, ok := opened.providerResults.(provider.ResultObserverRegistrar); ok && !nilInterface(registrar) {
registrar.AddResultObserver(coordinator)
}
collector, collectorErr := platformMetrics.NewCheckerCollector(prometheus.DefaultRegisterer)
if collectorErr != nil {
return fmt.Errorf("%w: build Checker metrics: %w", ErrStartup, collectorErr)
sequentialCoordinator = coordinator
}
checkerMetrics = collector
handler, handlerErr := platformMetrics.NewHandler(platformMetrics.Dependencies{
Gatherer: prometheus.DefaultGatherer, Readiness: opened.metricsReadiness,
})
if handlerErr != nil {
return fmt.Errorf("%w: build metrics handler: %w", ErrStartup, handlerErr)
}
dependencies.MetricsHandler = handler
}
runners := make([]lifecycle.Runner, 0, 3+len(loaded.Value.Upstreams))
if hasHTTPRuntime(loaded.Value) {
runner, err := factory.New(configurationStore.Current(), dependencies, controllerRuntime.Options{HTTP: options.HTTP})
@ -284,6 +333,7 @@ func runWithWorkerFactory(
DrainTickets: drainTickets,
RoutingSource: routingSource,
Credentials: opened.credentials,
SnapshotRefresh: snapshotRefresh,
})
if serviceErr != nil {
return fmt.Errorf("%w: build Worker control service: %w", ErrStartup, serviceErr)
@ -320,7 +370,14 @@ func runWithWorkerFactory(
}
runners = append(runners, runner)
if tasks, ok := opened.activity.(healthTaskRuntime); ok && !nilInterface(tasks) {
schedulers, schedulerErr := newHealthSchedulers(configurationStore, tasks, options.Now)
var schedulerStates []controllerHealth.SchedulerStateSource
if loaded.Value.Admin.Enabled {
if nilInterface(opened.state) {
return errors.Join(ErrStartup, ErrInvalidOptions)
}
schedulerStates = append(schedulerStates, opened.state)
}
schedulers, schedulerErr := newHealthSchedulers(configurationStore, tasks, options.Now, schedulerStates...)
if schedulerErr != nil {
return fmt.Errorf("%w: build Checker health schedulers: %w", ErrStartup, schedulerErr)
}
@ -331,7 +388,7 @@ func runWithWorkerFactory(
configurationStore,
unhealthy,
controllerHealth.UnhealthyReaperOptions{
PollInterval: checkSchedulerPollInterval, BatchSize: checkSchedulerBatchSize, Now: options.Now,
PollInterval: checkSchedulerPollInterval, BatchSize: checkSchedulerBatchSize, Now: options.Now, Metrics: drainMetrics,
},
)
if reaperErr != nil {
@ -339,6 +396,33 @@ func runWithWorkerFactory(
}
runners = append(runners, reaper)
}
if drainStore, ok := opened.activity.(controllerHealth.UpstreamDrainStore); ok && !nilInterface(drainStore) {
var policyStates []controllerHealth.UpstreamDrainStateSource
if loaded.Value.Admin.Enabled {
if nilInterface(opened.state) {
return errors.Join(ErrStartup, ErrInvalidOptions)
}
policyStates = append(policyStates, opened.state)
}
policySource, policyErr := controllerHealth.NewEffectiveUpstreamDrainPolicySource(configurationStore, policyStates...)
if policyErr != nil {
return fmt.Errorf("%w: build upstream Drain policy source: %w", ErrStartup, policyErr)
}
reaper, reaperErr := controllerHealth.NewConfiguredUpstreamDrainReaper(
policySource,
drainStore,
controllerHealth.UpstreamDrainReaperOptions{
PollInterval: checkSchedulerPollInterval, BatchSize: checkSchedulerBatchSize, Now: options.Now, Metrics: drainMetrics,
},
)
if reaperErr != nil {
return fmt.Errorf("%w: build upstream Drain reaper: %w", ErrStartup, reaperErr)
}
runners = append(runners, reaper)
}
}
if sequentialCoordinator != nil {
runners = append(runners, sequentialCoordinator)
}
runners = append(runners, supervisor)
group, err := lifecycle.NewGroup(runners...)
@ -352,8 +436,10 @@ func newHealthSchedulers(
configuration controllerHealth.ConfigurationSource,
tasks healthTaskRuntime,
now func() time.Time,
states ...controllerHealth.SchedulerStateSource,
) ([]lifecycle.Runner, error) {
if nilInterface(configuration) || nilInterface(tasks) || now == nil {
if nilInterface(configuration) || nilInterface(tasks) || now == nil || len(states) > 1 ||
(len(states) == 1 && nilInterface(states[0])) {
return nil, ErrInvalidOptions
}
current := configuration.Current()
@ -372,7 +458,7 @@ func newHealthSchedulers(
}
supervisor, err := controllerHealth.NewConfiguredSchedulerSupervisor(configuration, tasks, tasks, controllerHealth.SchedulerRunnerOptions{
PollInterval: checkSchedulerPollInterval, BatchSize: checkSchedulerBatchSize, Now: now,
})
}, states...)
if err != nil {
return nil, err
}

View File

@ -75,6 +75,7 @@ func providerRuntimeBuilder(opened ports) (buildUpstreamRuntime, error) {
Parser: parser,
Activity: opened.activity,
Results: results,
Capacity: opened.capacityMetrics,
})
}, nil
}

View File

@ -276,7 +276,7 @@ func (supervisor *providerSupervisor) desired(ctx context.Context) (map[string]c
for name, upstream := range configuration.Upstreams {
isEnabled := upstream.Enabled
if !nilInterface(supervisor.state) {
isEnabled = enabled[name]
isEnabled = isEnabled && enabled[name]
}
if isEnabled {
upstream.Enabled = true

View File

@ -71,6 +71,38 @@ func TestProviderSupervisorAppliesDisableAndConfigurationReplacement(t *testing.
}
}
func TestProviderSupervisorKeepsStaticallyDisabledUpstreamStopped(t *testing.T) {
configuration, err := config.Load(strings.NewReader(bootstrapTestConfig))
if err != nil {
t.Fatalf("config.Load(): %v", err)
}
disabled := configuration.Upstreams["provider-b"]
disabled.Enabled = false
configuration.Upstreams["provider-b"] = disabled
store, err := config.NewStore(configuration)
if err != nil {
t.Fatalf("config.NewStore(): %v", err)
}
supervisor, err := newProviderSupervisor(
store,
&mutableProviderState{enabled: map[string]bool{"provider-a": true, "provider-b": true}},
func(string, config.Upstream) (lifecycle.Runner, error) {
return supervisorRunnerFunc(func(context.Context) error { return nil }), nil
},
nil, nil, nil, bootstrapTestFingerprintKey, time.Hour,
)
if err != nil {
t.Fatalf("newProviderSupervisor() error = %v", err)
}
desired, err := supervisor.desired(context.Background())
if err != nil || len(desired) != 1 {
t.Fatalf("desired() = %+v, %v", desired, err)
}
if _, exists := desired["provider-a"]; !exists {
t.Fatalf("desired() = %+v, provider-a missing", desired)
}
}
func TestProviderSupervisorPropagatesUnexpectedRuntimeFailure(t *testing.T) {
configuration, err := config.Load(strings.NewReader(bootstrapTestConfig))
if err != nil {

View File

@ -8,6 +8,8 @@ import (
"time"
controllerExtraction "proxy-pool/internal/controller/extraction"
"proxy-pool/internal/domain/authorization"
"proxy-pool/internal/domain/clientpolicy"
domainExtraction "proxy-pool/internal/domain/extraction"
"proxy-pool/internal/platform/httpapi"
"proxy-pool/internal/platform/httpsecurity"
@ -173,6 +175,10 @@ func (h *Handler) handleExtract(writer http.ResponseWriter, request *http.Reques
h.writeProblem(writer, problemBadRequest(requestID, "INVALID_REQUEST", "Invalid request", "", nil))
return
}
if !identity.Allows(authorization.DistributionExtract) {
h.writeProblem(writer, httpapi.NewProblem(http.StatusForbidden, "FORBIDDEN", "Forbidden", "", requestID))
return
}
idempotencyKey, err := validateIdempotencyKey(request.Header.Values(headerIdempotencyKey))
if err != nil {
@ -201,7 +207,15 @@ func (h *Handler) handleExtract(writer http.ResponseWriter, request *http.Reques
return
}
filters := payload.filtersOrZero()
if !identity.ClientPolicy.AllowsExtractCount(payload.Count) {
h.writeProblem(writer, httpapi.NewProblem(http.StatusForbidden, "FORBIDDEN", "Forbidden", "", requestID))
return
}
filters, allowed := restrictExtractFilters(identity.ClientPolicy, payload.filtersOrZero())
if !allowed {
h.writeProblem(writer, httpapi.NewProblem(http.StatusForbidden, "FORBIDDEN", "Forbidden", "", requestID))
return
}
serviceResponse, err := h.extractor.Extract(request.Context(), controllerExtraction.Request{
RequestID: requestID,
ClientID: identity.ClientID,
@ -247,6 +261,20 @@ func (h *Handler) handleExtract(writer http.ResponseWriter, request *http.Reques
h.writeJSON(writer, requestID, http.StatusOK, response)
}
func restrictExtractFilters(policy clientpolicy.Policy, filters extractFiltersDTO) (extractFiltersDTO, bool) {
upstreams, allowed := policy.RestrictUpstreams(filters.AllowedUpstreams)
if !allowed {
return extractFiltersDTO{}, false
}
regions, allowed := policy.RestrictRegions(filters.Regions)
if !allowed {
return extractFiltersDTO{}, false
}
filters.AllowedUpstreams = upstreams
filters.Regions = regions
return filters, true
}
func validateIdempotencyKey(values []string) (string, error) {
if len(values) == 0 || (len(values) == 1 && values[0] == "") {
return "", nil

View File

@ -7,11 +7,14 @@ import (
"fmt"
"net/http"
"net/http/httptest"
"slices"
"strings"
"testing"
"time"
controllerExtraction "proxy-pool/internal/controller/extraction"
"proxy-pool/internal/domain/authorization"
"proxy-pool/internal/domain/clientpolicy"
domainExtraction "proxy-pool/internal/domain/extraction"
"proxy-pool/internal/platform/httpapi"
"proxy-pool/internal/platform/httpsecurity"
@ -157,6 +160,108 @@ func TestHandlerExtractSuccessMapsOpenAPIDTOAndReturnsRequestID(t *testing.T) {
}
}
func TestHandlerRejectsIdentityWithoutExtractPermission(t *testing.T) {
t.Parallel()
extractor := &fakeExtractor{}
handler := mustNewHandler(t, Config{BodyLimitBytes: 1024}, Dependencies{
Extractor: extractor,
Identity: fakeIdentityResolver{identity: Identity{
ClientID: "tenant-read-only", SourceIP: "198.51.100.8", Permissions: []string{authorization.AdminRead},
}},
Readiness: fakeReadinessChecker{},
})
request := httptest.NewRequest(http.MethodPost, "/api/v1/proxies/extract", strings.NewReader(`{"count":1}`))
request.Header.Set("Content-Type", httpapi.JSONContentType)
request.Header.Set(headerIdempotencyKey, "idem-12345678")
recorder := httptest.NewRecorder()
handler.ServeHTTP(recorder, request)
if recorder.Code != http.StatusForbidden || extractor.calls != 0 || !strings.Contains(recorder.Body.String(), `"code":"FORBIDDEN"`) {
t.Fatalf("response = status %d calls=%d body=%s", recorder.Code, extractor.calls, recorder.Body.String())
}
}
func TestHandlerEnforcesCredentialExtractionPolicy(t *testing.T) {
t.Parallel()
tests := []struct {
name string
body string
wantStatus int
wantUpstreams []string
wantRegions []string
wantExtractorCall int
}{
{
name: "omitted filters are bound to credential policy",
body: "{\"count\":2}",
wantStatus: http.StatusOK,
wantUpstreams: []string{"provider-a"},
wantRegions: []string{"shanghai"},
wantExtractorCall: 1,
},
{
name: "count exceeds credential maximum",
body: "{\"count\":3}",
wantStatus: http.StatusForbidden,
wantExtractorCall: 0,
},
{
name: "upstream exceeds credential boundary",
body: "{\"count\":1,\"filters\":{\"allowedUpstreams\":[\"provider-b\"]}}",
wantStatus: http.StatusForbidden,
wantExtractorCall: 0,
},
{
name: "region exceeds credential boundary",
body: "{\"count\":1,\"filters\":{\"regions\":[\"beijing\"]}}",
wantStatus: http.StatusForbidden,
wantExtractorCall: 0,
},
}
for _, test := range tests {
test := test
t.Run(test.name, func(t *testing.T) {
t.Parallel()
extractor := &fakeExtractor{}
handler := mustNewHandler(t, Config{BodyLimitBytes: 1024}, Dependencies{
Extractor: extractor,
Identity: fakeIdentityResolver{identity: Identity{
ClientID: "tenant-restricted",
Permissions: []string{authorization.DistributionExtract},
ClientPolicy: clientpolicy.Policy{
MaxExtractCount: 2,
AllowedUpstreams: []string{"provider-a"},
AllowedRegions: []string{"shanghai"},
},
}},
Readiness: fakeReadinessChecker{},
})
request := httptest.NewRequest(http.MethodPost, pathExtract, strings.NewReader(test.body))
request.Header.Set("Content-Type", httpapi.JSONContentType)
response := httptest.NewRecorder()
handler.ServeHTTP(response, request)
if response.Code != test.wantStatus || extractor.calls != test.wantExtractorCall {
t.Fatalf("response = status %d calls=%d, want status %d calls=%d",
response.Code, extractor.calls, test.wantStatus, test.wantExtractorCall)
}
if response.Code == http.StatusForbidden && !strings.Contains(response.Body.String(), "\"code\":\"FORBIDDEN\"") {
t.Fatalf("forbidden response = %s", response.Body.String())
}
if got := extractor.request.Filters.Upstreams; !slices.Equal(got, test.wantUpstreams) {
t.Fatalf("upstreams = %v, want %v", got, test.wantUpstreams)
}
if got := extractor.request.Filters.Regions; !slices.Equal(got, test.wantRegions) {
t.Fatalf("regions = %v, want %v", got, test.wantRegions)
}
})
}
}
func TestHandlerRejectsEmptyResolvedIdentity(t *testing.T) {
t.Parallel()
extractor := &fakeExtractor{}

View File

@ -0,0 +1,80 @@
package extraction
import (
"errors"
domain "proxy-pool/internal/domain/extraction"
)
// MetricResult is a fixed, low-cardinality classification of one extraction
// request after application-level validation, admission, and storage handling.
type MetricResult string
const (
MetricComplete MetricResult = "complete"
MetricPartial MetricResult = "partial"
MetricEmpty MetricResult = "empty"
MetricInsufficient MetricResult = "insufficient"
MetricIdempotencyConflict MetricResult = "idempotency_conflict"
MetricRateLimited MetricResult = "rate_limited"
MetricUnavailable MetricResult = "unavailable"
MetricInvalid MetricResult = "invalid"
MetricError MetricResult = "error"
)
// MetricsEvent contains only aggregate request and response counts. It must
// never contain a Client, Proxy, Upstream, filter, identifier, or error text.
type MetricsEvent struct {
Result MetricResult
Requested int
Returned int
}
// MetricsObserver receives one completed extraction request. Implementations
// must return promptly and must not perform network or persistent storage I/O.
type MetricsObserver interface {
ObserveExtraction(MetricsEvent)
}
func classifyMetricsEvent(response Response, resultErr error) MetricsEvent {
event := MetricsEvent{Requested: response.Requested, Returned: response.Returned}
if resultErr == nil {
switch {
case response.Returned == response.Requested:
event.Result = MetricComplete
case response.Returned > 0 && response.Returned < response.Requested:
event.Result = MetricPartial
case response.Returned == 0:
event.Result = MetricEmpty
default:
event.Result = MetricError
}
return event
}
switch {
case errors.Is(resultErr, domain.ErrInsufficientProxies):
event.Result = MetricInsufficient
case errors.Is(resultErr, domain.ErrIdempotencyConflict):
event.Result = MetricIdempotencyConflict
case errors.Is(resultErr, ErrAdmissionRejected):
event.Result = MetricRateLimited
case errors.Is(resultErr, ErrUnavailable):
event.Result = MetricUnavailable
case errors.Is(resultErr, ErrInvalidRequest), errors.Is(resultErr, ErrCountExceeded),
errors.Is(resultErr, ErrInvalidFulfillment), errors.Is(resultErr, domain.ErrInvalidCommand):
event.Result = MetricInvalid
default:
event.Result = MetricError
}
return event
}
func validMetricResult(result MetricResult) bool {
switch result {
case MetricComplete, MetricPartial, MetricEmpty, MetricInsufficient, MetricIdempotencyConflict,
MetricRateLimited, MetricUnavailable, MetricInvalid, MetricError:
return true
default:
return false
}
}

View File

@ -73,9 +73,16 @@ type Service struct {
policy Policy
admission admission.Admitter
now func() time.Time
metrics MetricsObserver
}
func NewService(store domain.Store, policy Policy, admitter admission.Admitter, now func() time.Time) (*Service, error) {
func NewService(
store domain.Store,
policy Policy,
admitter admission.Admitter,
now func() time.Time,
metrics ...MetricsObserver,
) (*Service, error) {
if admitter == nil {
return nil, fmt.Errorf("%w: admission is required", ErrInvalidServicePolicy)
}
@ -92,11 +99,19 @@ func NewService(store domain.Store, policy Policy, admitter admission.Admitter,
if now == nil {
now = time.Now
}
return &Service{store: store, policy: policy, admission: admitter, now: now}, nil
if len(metrics) > 1 {
return nil, ErrInvalidServicePolicy
}
var observer MetricsObserver
if len(metrics) == 1 {
observer = metrics[0]
}
return &Service{store: store, policy: policy, admission: admitter, now: now, metrics: observer}, nil
}
func (s *Service) Extract(ctx context.Context, request Request) (Response, error) {
response := Response{RequestID: request.RequestID, Requested: request.Count}
func (s *Service) Extract(ctx context.Context, request Request) (response Response, resultErr error) {
response = Response{RequestID: request.RequestID, Requested: request.Count}
defer func() { s.observeMetrics(classifyMetricsEvent(response, resultErr)) }()
if request.RequestID == "" || (request.ClientID == "" && request.SourceIP == "") || request.Count <= 0 {
return response, ErrInvalidRequest
}
@ -176,6 +191,13 @@ func (s *Service) Extract(ctx context.Context, request Request) (Response, error
return response, nil
}
func (s *Service) observeMetrics(event MetricsEvent) {
if s == nil || s.metrics == nil || !validMetricResult(event.Result) {
return
}
s.metrics.ObserveExtraction(event)
}
func admissionKey(request Request) string {
if request.ClientID != "" {
return "client:" + request.ClientID

View File

@ -244,6 +244,86 @@ func TestServiceIdempotentReplayKeepsOriginalExtractionTime(t *testing.T) {
}
}
func TestServiceEmitsFixedMetricsForTerminalResults(t *testing.T) {
observer := &recordingMetricsObserver{}
policy := Policy{MaxCountPerRequest: 2, DefaultFulfillment: domain.Partial}
tests := []struct {
name string
request Request
result domain.Result
storeErr error
admitErr error
wantEvent MetricsEvent
}{
{
name: "complete",
request: Request{RequestID: "req-complete", ClientID: "client-a", Count: 2},
result: domain.Result{Requested: 2, Returned: 2, Items: []domain.Candidate{{}, {}}},
wantEvent: MetricsEvent{Result: MetricComplete, Requested: 2, Returned: 2},
},
{
name: "partial",
request: Request{RequestID: "req-partial", ClientID: "client-a", Count: 2},
result: domain.Result{Requested: 2, Returned: 1, Items: []domain.Candidate{{}}},
wantEvent: MetricsEvent{Result: MetricPartial, Requested: 2, Returned: 1},
},
{
name: "empty",
request: Request{RequestID: "req-empty", ClientID: "client-a", Count: 1},
result: domain.Result{Requested: 1},
wantEvent: MetricsEvent{Result: MetricEmpty, Requested: 1, Returned: 0},
},
{
name: "unexpected over delivery",
request: Request{RequestID: "req-over", ClientID: "client-a", Count: 1},
result: domain.Result{Requested: 1, Returned: 2, Items: []domain.Candidate{{}, {}}},
wantEvent: MetricsEvent{Result: MetricError, Requested: 1, Returned: 2},
},
{
name: "insufficient",
request: Request{RequestID: "req-insufficient", ClientID: "client-a", Count: 1},
storeErr: domain.ErrInsufficientProxies,
wantEvent: MetricsEvent{Result: MetricInsufficient, Requested: 1},
},
{
name: "idempotency conflict",
request: Request{RequestID: "req-conflict", ClientID: "client-a", Count: 1},
storeErr: domain.ErrIdempotencyConflict,
wantEvent: MetricsEvent{Result: MetricIdempotencyConflict, Requested: 1},
},
{
name: "rate limited",
request: Request{RequestID: "req-rate", ClientID: "client-a", Count: 1},
admitErr: errors.New("rate limited"),
wantEvent: MetricsEvent{Result: MetricRateLimited, Requested: 1},
},
{
name: "unavailable",
request: Request{RequestID: "req-unavailable", ClientID: "client-a", Count: 1},
storeErr: domain.ErrStoreUnavailable,
wantEvent: MetricsEvent{Result: MetricUnavailable, Requested: 1},
},
{
name: "invalid",
request: Request{RequestID: "req-invalid", Count: 1},
wantEvent: MetricsEvent{Result: MetricInvalid, Requested: 1},
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
service, err := NewService(&recordingStore{result: test.result, err: test.storeErr}, policy,
&recordingAdmission{err: test.admitErr}, time.Now, observer)
if err != nil {
t.Fatalf("NewService() error = %v", err)
}
_, _ = service.Extract(context.Background(), test.request)
if got := observer.events[len(observer.events)-1]; got != test.wantEvent {
t.Fatalf("metrics event = %+v, want %+v", got, test.wantEvent)
}
})
}
}
type recordingStore struct {
command domain.Command
result domain.Result
@ -257,6 +337,8 @@ type recordingAdmission struct {
calls int
}
type recordingMetricsObserver struct{ events []MetricsEvent }
type allowAllAdmission struct{}
func (allowAllAdmission) Admit(context.Context, string) error { return nil }
@ -267,6 +349,10 @@ func (a *recordingAdmission) Admit(_ context.Context, key string) error {
return a.err
}
func (observer *recordingMetricsObserver) ObserveExtraction(event MetricsEvent) {
observer.events = append(observer.events, event)
}
func (s *recordingStore) Extract(_ context.Context, command domain.Command) (domain.Result, error) {
s.calls++
s.command = command

View File

@ -9,6 +9,7 @@ import (
"time"
"proxy-pool/internal/config"
"proxy-pool/internal/domain/adminstate"
healthDomain "proxy-pool/internal/domain/health"
)
@ -169,12 +170,26 @@ type ConfiguredSchedulerRunner struct {
// bounded BASIC scheduling without a Controller restart.
type ConfiguredSchedulerSupervisor struct {
configuration ConfigurationSource
state SchedulerStateSource
source UpstreamTaskSource
sink TaskSink
options SchedulerRunnerOptions
cursor atomic.Uint64
}
// SchedulerConfigurationSource returns a configuration and its Admin revision
// from one atomic publication. config.Store implements this interface.
type SchedulerConfigurationSource interface {
ConfigurationSource
Snapshot() (*config.Config, uint64)
}
// SchedulerStateSource provides the persisted Admin view that fences a
// scheduler from combining two different configuration revisions.
type SchedulerStateSource interface {
Snapshot(context.Context) (adminstate.Snapshot, error)
}
func NewConfiguredSchedulerRunner(
configuration ConfigurationSource,
upstreamID string,
@ -196,14 +211,22 @@ func NewConfiguredSchedulerSupervisor(
source UpstreamTaskSource,
sink TaskSink,
options SchedulerRunnerOptions,
states ...SchedulerStateSource,
) (*ConfiguredSchedulerSupervisor, error) {
if nilInterface(configuration) || nilInterface(source) || nilInterface(sink) || options.PollInterval <= 0 ||
options.BatchSize <= 0 || options.Now == nil {
options.BatchSize <= 0 || options.Now == nil || len(states) > 1 || (len(states) == 1 && nilInterface(states[0])) {
return nil, ErrInvalidSchedulerRunner
}
return &ConfiguredSchedulerSupervisor{
supervisor := &ConfiguredSchedulerSupervisor{
configuration: configuration, source: source, sink: sink, options: options,
}, nil
}
if len(states) == 1 {
if _, ok := configuration.(SchedulerConfigurationSource); !ok {
return nil, ErrInvalidSchedulerRunner
}
supervisor.state = states[0]
}
return supervisor, nil
}
type TickResult struct {
@ -245,11 +268,10 @@ func (supervisor *ConfiguredSchedulerSupervisor) Tick(ctx context.Context) (Tick
if err := ctx.Err(); err != nil {
return TickResult{}, err
}
configuration := supervisor.configuration.Current()
if configuration == nil {
return TickResult{}, ErrInvalidSchedulerRunner
configuration, names, routings, err := supervisor.effectiveConfiguration(ctx)
if err != nil {
return TickResult{}, err
}
names := enabledUpstreamNames(configuration)
result := TickResult{}
for _, name := range names {
planner, configured, err := configuredPlannerForConfig(configuration, name)
@ -264,7 +286,7 @@ func (supervisor *ConfiguredSchedulerSupervisor) Tick(ctx context.Context) (Tick
return TickResult{}, err
}
check := config.EffectiveCheck(configuration.Defaults.Check, configuration.Upstreams[name].Check)
groups, err := supervisor.upstreamDueGroups(basicSource, name, check.URLs, configuredTargetProfiles(configuration, name))
groups, err := supervisor.upstreamDueGroups(basicSource, name, check.URLs, configuredTargetProfiles(configuration, name, routings))
if err != nil {
return TickResult{}, err
}
@ -289,6 +311,91 @@ func (supervisor *ConfiguredSchedulerSupervisor) Tick(ctx context.Context) (Tick
return result, nil
}
// effectiveConfiguration returns the effective Upstream and Routing state for
// one scheduler tick. When Admin state is present, it requires the
// configuration and state snapshots to share a revision; an incomplete or
// torn view schedules nothing.
func (supervisor *ConfiguredSchedulerSupervisor) effectiveConfiguration(
ctx context.Context,
) (*config.Config, []string, map[string]adminstate.RoutingState, error) {
if supervisor == nil || nilInterface(supervisor.configuration) {
return nil, nil, nil, ErrInvalidSchedulerRunner
}
if supervisor.state == nil {
configuration := supervisor.configuration.Current()
if configuration == nil {
return nil, nil, nil, ErrInvalidSchedulerRunner
}
return configuration, enabledUpstreamNames(configuration), nil, nil
}
configurationSource, ok := supervisor.configuration.(SchedulerConfigurationSource)
if !ok || nilInterface(configurationSource) || nilInterface(supervisor.state) {
return nil, nil, nil, ErrInvalidSchedulerRunner
}
configuration, revision := configurationSource.Snapshot()
if configuration == nil || revision == 0 {
return nil, nil, nil, ErrInvalidSchedulerRunner
}
snapshot, err := supervisor.state.Snapshot(ctx)
if err != nil {
return nil, nil, nil, err
}
if snapshot.Config == nil || snapshot.Config.Revision != revision {
return nil, nil, nil, ErrInvalidSchedulerRunner
}
states, err := newSchedulerAdminStates(configuration, snapshot)
if err != nil {
return nil, nil, nil, err
}
names, err := enabledUpstreamNamesForState(configuration, states.upstreams)
if err != nil {
return nil, nil, nil, err
}
return configuration, names, states.routings, nil
}
type schedulerAdminStates struct {
upstreams map[string]adminstate.UpstreamState
routings map[string]adminstate.RoutingState
}
func newSchedulerAdminStates(
configuration *config.Config,
snapshot adminstate.Snapshot,
) (schedulerAdminStates, error) {
if configuration == nil {
return schedulerAdminStates{}, ErrInvalidSchedulerRunner
}
states := schedulerAdminStates{
upstreams: make(map[string]adminstate.UpstreamState, len(snapshot.Upstreams)),
routings: make(map[string]adminstate.RoutingState, len(snapshot.Routings)),
}
for _, upstream := range snapshot.Upstreams {
if upstream.Name == "" || upstream.Revision == 0 {
return schedulerAdminStates{}, ErrInvalidSchedulerRunner
}
if _, duplicate := states.upstreams[upstream.Name]; duplicate {
return schedulerAdminStates{}, ErrInvalidSchedulerRunner
}
states.upstreams[upstream.Name] = upstream
}
for _, routing := range snapshot.Routings {
if routing.Name == "" || routing.Revision == 0 {
return schedulerAdminStates{}, ErrInvalidSchedulerRunner
}
if _, duplicate := states.routings[routing.Name]; duplicate {
return schedulerAdminStates{}, ErrInvalidSchedulerRunner
}
states.routings[routing.Name] = routing
}
for _, routing := range configuration.Routing {
if _, exists := states.routings[routing.Name]; !exists {
return schedulerAdminStates{}, ErrInvalidSchedulerRunner
}
}
return states, nil
}
func (supervisor *ConfiguredSchedulerSupervisor) upstreamDueGroups(
basic DueSource,
upstreamID string,
@ -326,13 +433,17 @@ type configuredTargetProfile struct {
TargetURL string
}
func configuredTargetProfiles(configuration *config.Config, upstreamID string) []configuredTargetProfile {
func configuredTargetProfiles(
configuration *config.Config,
upstreamID string,
routings map[string]adminstate.RoutingState,
) []configuredTargetProfile {
if configuration == nil || upstreamID == "" {
return nil
}
profiles := make([]configuredTargetProfile, 0)
for _, route := range configuration.Routing {
if !route.Enabled || !containsString(route.Upstreams, upstreamID) {
if !route.Enabled || !containsString(route.Upstreams, upstreamID) || (routings != nil && !routings[route.Name].Enabled) {
continue
}
for _, targetURL := range route.Check.Targets {
@ -491,6 +602,27 @@ func enabledUpstreamNames(configuration *config.Config) []string {
return names
}
func enabledUpstreamNamesForState(
configuration *config.Config,
states map[string]adminstate.UpstreamState,
) ([]string, error) {
if configuration == nil {
return nil, ErrInvalidSchedulerRunner
}
names := make([]string, 0, len(configuration.Upstreams))
for name, upstream := range configuration.Upstreams {
state, exists := states[name]
if !exists {
return nil, ErrInvalidSchedulerRunner
}
if upstream.Enabled && state.Enabled {
names = append(names, name)
}
}
sort.Strings(names)
return names, nil
}
func runScheduler(ctx context.Context, pollInterval time.Duration, tick func(context.Context) (TickResult, error)) error {
if ctx == nil || pollInterval <= 0 || tick == nil {
return ErrInvalidSchedulerRunner

View File

@ -7,6 +7,7 @@ import (
"time"
"proxy-pool/internal/config"
"proxy-pool/internal/domain/adminstate"
healthDomain "proxy-pool/internal/domain/health"
proxyDomain "proxy-pool/internal/domain/proxy"
)
@ -164,6 +165,160 @@ func TestConfiguredSchedulerSupervisorDiscoversNewEnabledUpstream(t *testing.T)
}
}
func TestConfiguredSchedulerSupervisorSkipsAdminDisabledUpstream(t *testing.T) {
now := time.Date(2026, 8, 7, 10, 0, 0, 0, time.UTC)
configuration := schedulerConfiguration(now, "provider-a", "provider-b")
configuration.Defaults.Check.URLs = []string{"https://egress.example/identity"}
configuration.Routing = []config.Routing{{
Name: "checkout", Enabled: true, Upstreams: []string{"provider-a"},
Check: config.RoutingCheck{Targets: []string{"https://checkout.example/health"}},
}}
source := &upstreamTaskSourceStub{sources: map[string]*dueSourceStub{
"provider-a": {
candidates: []Candidate{{ProxyID: "proxy-a", State: proxyDomain.StateFetched, Level: healthDomain.LevelBasic, DueAt: now}},
egressCandidates: map[string][]Candidate{"https://egress.example/identity": {
{ProxyID: "egress-a", State: proxyDomain.StateFetched, Level: healthDomain.LevelEgress, TargetURL: "https://egress.example/identity", DueAt: now},
}},
targetCandidates: map[string][]Candidate{targetCandidateKey("checkout", "https://checkout.example/health"): {
{ProxyID: "target-a", State: proxyDomain.StateFetched, Level: healthDomain.LevelTarget, RoutingName: "checkout", TargetURL: "https://checkout.example/health", DueAt: now},
}},
},
"provider-b": {candidates: []Candidate{{ProxyID: "proxy-b", State: proxyDomain.StateFetched, Level: healthDomain.LevelBasic, DueAt: now}}},
}}
state := schedulerStateStub{snapshot: adminstate.Snapshot{
Config: &adminstate.ConfigRevision{Revision: 42},
Upstreams: []adminstate.UpstreamState{
{Name: "provider-a", Enabled: false, Revision: 11},
{Name: "provider-b", Enabled: true, Revision: 12},
},
Routings: []adminstate.RoutingState{{Name: "checkout", Enabled: true, Revision: 13}},
}}
sink := &taskSinkStub{}
supervisor, err := NewConfiguredSchedulerSupervisor(
&versionedConfigurationSourceStub{configuration: configuration, revision: 42}, source, sink,
SchedulerRunnerOptions{PollInterval: time.Second, BatchSize: 16, Now: func() time.Time { return now }}, state,
)
if err != nil {
t.Fatalf("NewConfiguredSchedulerSupervisor() = %v", err)
}
result, err := supervisor.Tick(context.Background())
if err != nil || result != (TickResult{Planned: 1, Offered: 1}) || source.sources["provider-a"].inFlightCalls != 0 ||
source.sources["provider-a"].dueCalls != 0 || source.sources["provider-a"].egressCalls != 0 ||
source.sources["provider-a"].targetCalls != 0 || source.sources["provider-b"].dueCalls != 1 ||
len(sink.tasks) != 1 || sink.tasks[0].Candidate.ProxyID != "proxy-b" {
t.Fatalf("Tick() = (%+v, %v); source=%+v sink=%+v", result, err, source, sink.tasks)
}
}
func TestConfiguredSchedulerSupervisorSkipsAdminDisabledRoutingTarget(t *testing.T) {
now := time.Date(2026, 8, 7, 10, 15, 0, 0, time.UTC)
configuration := schedulerConfiguration(now, "provider-a")
configuration.Routing = []config.Routing{{
Name: "checkout", Enabled: true, Upstreams: []string{"provider-a"},
Check: config.RoutingCheck{Targets: []string{"https://checkout.example/health"}},
}}
source := &upstreamTaskSourceStub{sources: map[string]*dueSourceStub{
"provider-a": {
candidates: []Candidate{{ProxyID: "proxy-a", State: proxyDomain.StateFetched, Level: healthDomain.LevelBasic, DueAt: now}},
targetCandidates: map[string][]Candidate{targetCandidateKey("checkout", "https://checkout.example/health"): {
{ProxyID: "target-a", State: proxyDomain.StateFetched, Level: healthDomain.LevelTarget, RoutingName: "checkout", TargetURL: "https://checkout.example/health", DueAt: now},
}},
},
}}
supervisor, err := NewConfiguredSchedulerSupervisor(
&versionedConfigurationSourceStub{configuration: configuration, revision: 42}, source, &taskSinkStub{},
SchedulerRunnerOptions{PollInterval: time.Second, BatchSize: 16, Now: func() time.Time { return now }},
schedulerStateStub{snapshot: adminstate.Snapshot{
Config: &adminstate.ConfigRevision{Revision: 42},
Upstreams: []adminstate.UpstreamState{{Name: "provider-a", Enabled: true, Revision: 11}},
Routings: []adminstate.RoutingState{{Name: "checkout", Enabled: false, Revision: 12}},
}},
)
if err != nil {
t.Fatalf("NewConfiguredSchedulerSupervisor() = %v", err)
}
if result, err := supervisor.Tick(context.Background()); err != nil || result != (TickResult{Planned: 1, Offered: 1}) ||
source.sources["provider-a"].dueCalls != 1 || source.sources["provider-a"].targetCalls != 0 {
t.Fatalf("Tick() = (%+v, %v); source=%+v", result, err, source.sources["provider-a"])
}
}
func TestConfiguredSchedulerSupervisorFailsClosedOnAdminRevisionMismatch(t *testing.T) {
now := time.Date(2026, 8, 7, 10, 30, 0, 0, time.UTC)
configuration := schedulerConfiguration(now, "provider-a")
source := &upstreamTaskSourceStub{sources: map[string]*dueSourceStub{
"provider-a": {candidates: []Candidate{{ProxyID: "proxy-a", State: proxyDomain.StateFetched, Level: healthDomain.LevelBasic, DueAt: now}}},
}}
supervisor, err := NewConfiguredSchedulerSupervisor(
&versionedConfigurationSourceStub{configuration: configuration, revision: 42}, source, &taskSinkStub{},
SchedulerRunnerOptions{PollInterval: time.Second, BatchSize: 16, Now: func() time.Time { return now }},
schedulerStateStub{snapshot: adminstate.Snapshot{Config: &adminstate.ConfigRevision{Revision: 41}}},
)
if err != nil {
t.Fatalf("NewConfiguredSchedulerSupervisor() = %v", err)
}
if result, err := supervisor.Tick(context.Background()); !errors.Is(err, ErrInvalidSchedulerRunner) || result != (TickResult{}) ||
source.sources["provider-a"].inFlightCalls != 0 || source.sources["provider-a"].dueCalls != 0 {
t.Fatalf("Tick() = (%+v, %v); source=%+v", result, err, source.sources["provider-a"])
}
}
func TestConfiguredSchedulerSupervisorFailsClosedOnIncompleteAdminState(t *testing.T) {
now := time.Date(2026, 8, 7, 10, 45, 0, 0, time.UTC)
configuration := schedulerConfiguration(now, "provider-a")
source := &upstreamTaskSourceStub{sources: map[string]*dueSourceStub{
"provider-a": {candidates: []Candidate{{ProxyID: "proxy-a", State: proxyDomain.StateFetched, Level: healthDomain.LevelBasic, DueAt: now}}},
}}
supervisor, err := NewConfiguredSchedulerSupervisor(
&versionedConfigurationSourceStub{configuration: configuration, revision: 42}, source, &taskSinkStub{},
SchedulerRunnerOptions{PollInterval: time.Second, BatchSize: 16, Now: func() time.Time { return now }},
schedulerStateStub{snapshot: adminstate.Snapshot{Config: &adminstate.ConfigRevision{Revision: 42}}},
)
if err != nil {
t.Fatalf("NewConfiguredSchedulerSupervisor() = %v", err)
}
if result, err := supervisor.Tick(context.Background()); !errors.Is(err, ErrInvalidSchedulerRunner) || result != (TickResult{}) ||
source.sources["provider-a"].inFlightCalls != 0 || source.sources["provider-a"].dueCalls != 0 {
t.Fatalf("Tick() = (%+v, %v); source=%+v", result, err, source.sources["provider-a"])
}
}
func TestConfiguredSchedulerSupervisorFailsClosedOnMissingAdminRoutingState(t *testing.T) {
now := time.Date(2026, 8, 7, 10, 50, 0, 0, time.UTC)
configuration := schedulerConfiguration(now, "provider-a")
configuration.Routing = []config.Routing{{Name: "checkout", Enabled: true, Upstreams: []string{"provider-a"}}}
source := &upstreamTaskSourceStub{sources: map[string]*dueSourceStub{
"provider-a": {candidates: []Candidate{{ProxyID: "proxy-a", State: proxyDomain.StateFetched, Level: healthDomain.LevelBasic, DueAt: now}}},
}}
supervisor, err := NewConfiguredSchedulerSupervisor(
&versionedConfigurationSourceStub{configuration: configuration, revision: 42}, source, &taskSinkStub{},
SchedulerRunnerOptions{PollInterval: time.Second, BatchSize: 16, Now: func() time.Time { return now }},
schedulerStateStub{snapshot: adminstate.Snapshot{
Config: &adminstate.ConfigRevision{Revision: 42},
Upstreams: []adminstate.UpstreamState{{Name: "provider-a", Enabled: true, Revision: 11}},
}},
)
if err != nil {
t.Fatalf("NewConfiguredSchedulerSupervisor() = %v", err)
}
if result, err := supervisor.Tick(context.Background()); !errors.Is(err, ErrInvalidSchedulerRunner) || result != (TickResult{}) ||
source.sources["provider-a"].inFlightCalls != 0 || source.sources["provider-a"].dueCalls != 0 {
t.Fatalf("Tick() = (%+v, %v); source=%+v", result, err, source.sources["provider-a"])
}
}
func TestConfiguredSchedulerSupervisorRequiresVersionedConfigurationWithAdminState(t *testing.T) {
now := time.Date(2026, 8, 7, 11, 0, 0, 0, time.UTC)
supervisor, err := NewConfiguredSchedulerSupervisor(
&configurationSourceStub{configuration: schedulerConfiguration(now, "provider-a")},
&upstreamTaskSourceStub{sources: map[string]*dueSourceStub{"provider-a": {}}}, &taskSinkStub{},
SchedulerRunnerOptions{PollInterval: time.Second, BatchSize: 1, Now: func() time.Time { return now }}, schedulerStateStub{},
)
if !errors.Is(err, ErrInvalidSchedulerRunner) || supervisor != nil {
t.Fatalf("NewConfiguredSchedulerSupervisor(non-versioned configuration) = (%v, %v)", supervisor, err)
}
}
func TestConfiguredSchedulerSupervisorSchedulesBoundedEgressGroups(t *testing.T) {
now := time.Date(2026, 8, 2, 11, 0, 0, 0, time.UTC)
configuration := &config.Config{
@ -310,6 +465,38 @@ func TestConfiguredSchedulerSupervisorNeverExceedsTotalBatchAcrossGroups(t *test
}
}
func schedulerConfiguration(now time.Time, upstreams ...string) *config.Config {
configured := &config.Config{
Defaults: config.Defaults{Check: config.Check{
Interval: config.Duration(time.Minute), MaxInFlight: 2, Timeout: config.Duration(time.Second), MaxAttempts: 1,
}},
Upstreams: make(map[string]config.Upstream, len(upstreams)),
}
for _, upstream := range upstreams {
configured.Upstreams[upstream] = config.Upstream{Enabled: true}
}
return configured
}
type versionedConfigurationSourceStub struct {
configuration *config.Config
revision uint64
}
func (source *versionedConfigurationSourceStub) Current() *config.Config {
return source.configuration
}
func (source *versionedConfigurationSourceStub) Snapshot() (*config.Config, uint64) {
return source.configuration, source.revision
}
type schedulerStateStub struct{ snapshot adminstate.Snapshot }
func (source schedulerStateStub) Snapshot(context.Context) (adminstate.Snapshot, error) {
return source.snapshot, nil
}
type dueSourceStub struct {
inFlight int
candidates []Candidate

View File

@ -7,6 +7,7 @@ import (
"proxy-pool/internal/config"
"proxy-pool/internal/domain/activitypool"
healthDomain "proxy-pool/internal/domain/health"
)
var ErrInvalidUnhealthyReaper = errors.New("invalid unhealthy reaper")
@ -15,6 +16,7 @@ type UnhealthyReaperOptions struct {
PollInterval time.Duration
BatchSize int
Now func() time.Time
Metrics healthDomain.DrainMetricsObserver
}
// ConfiguredUnhealthyReaper applies the current upstream grace policies to a
@ -62,12 +64,23 @@ func (reaper *ConfiguredUnhealthyReaper) Tick(ctx context.Context) (activitypool
}
drainer, supported := reaper.store.(activitypool.UnhealthyDrainStarter)
if !supported || nilInterface(drainer) {
if reaper.options.Metrics != nil {
reaper.options.Metrics.ObserveDrain(healthDomain.DrainReasonUnhealthy, len(result.DrainCandidates), 0)
}
return result, nil
}
started := 0
for _, candidate := range result.DrainCandidates {
if _, err := drainer.BeginUnhealthyDrain(ctx, utcNow, candidate); err != nil {
didStart, err := drainer.BeginUnhealthyDrain(ctx, utcNow, candidate)
if err != nil {
return result, err
}
if didStart {
started++
}
}
if reaper.options.Metrics != nil {
reaper.options.Metrics.ObserveDrain(healthDomain.DrainReasonUnhealthy, len(result.DrainCandidates), started)
}
return result, nil
}

View File

@ -7,6 +7,7 @@ import (
"proxy-pool/internal/config"
"proxy-pool/internal/domain/activitypool"
healthDomain "proxy-pool/internal/domain/health"
)
type unhealthyRemoverStub struct {
@ -85,11 +86,12 @@ func TestConfiguredUnhealthyReaperStartsBoundedDrainCandidates(t *testing.T) {
store := &unhealthyRemoverStub{result: activitypool.UnhealthySweepResult{
DeferredOwned: 1, DrainCandidates: []activitypool.UnhealthyDrainCandidate{candidate},
}}
metrics := &recordingDrainMetrics{}
reaper, err := NewConfiguredUnhealthyReaper(&configurationSourceStub{configuration: &config.Config{
Upstreams: map[string]config.Upstream{
"provider-a": {Enabled: true, Check: config.Check{UnhealthyRemoveAfter: config.Duration(time.Minute)}},
},
}}, store, UnhealthyReaperOptions{PollInterval: time.Second, BatchSize: 4, Now: func() time.Time { return now }})
}}, store, UnhealthyReaperOptions{PollInterval: time.Second, BatchSize: 4, Now: func() time.Time { return now }, Metrics: metrics})
if err != nil {
t.Fatalf("NewConfiguredUnhealthyReaper() = %v", err)
}
@ -100,4 +102,9 @@ func TestConfiguredUnhealthyReaperStartsBoundedDrainCandidates(t *testing.T) {
if store.drainCalls != 1 || !store.drainNow.Equal(now) || len(store.candidates) != 1 || store.candidates[0] != candidate {
t.Fatalf("BeginUnhealthyDrain() calls=%d now=%v candidates=%+v", store.drainCalls, store.drainNow, store.candidates)
}
if len(metrics.records) != 1 || metrics.records[0] != (drainMetricRecord{
reason: healthDomain.DrainReasonUnhealthy, candidates: 1, started: 1,
}) {
t.Fatalf("Drain metrics = %+v", metrics.records)
}
}

View File

@ -0,0 +1,211 @@
package health
import (
"context"
"errors"
"sort"
"time"
"proxy-pool/internal/config"
"proxy-pool/internal/domain/activitypool"
"proxy-pool/internal/domain/adminstate"
healthDomain "proxy-pool/internal/domain/health"
)
var ErrInvalidUpstreamDrainReaper = errors.New("invalid upstream drain reaper")
// UpstreamDrainConfigurationSource returns one configuration and revision from
// the same atomic publication. config.Store implements this interface.
type UpstreamDrainConfigurationSource interface {
Snapshot() (*config.Config, uint64)
}
type UpstreamDrainStateSource interface {
Snapshot(context.Context) (adminstate.Snapshot, error)
}
// EffectiveUpstreamDrainPolicySource merges static configuration with optional
// Admin state. With Admin enabled it uses the same revision equality rule as
// Worker routing, so a torn config/state view produces no policy publication.
type EffectiveUpstreamDrainPolicySource struct {
configuration UpstreamDrainConfigurationSource
state UpstreamDrainStateSource
}
func NewEffectiveUpstreamDrainPolicySource(
configuration UpstreamDrainConfigurationSource,
states ...UpstreamDrainStateSource,
) (*EffectiveUpstreamDrainPolicySource, error) {
if nilInterface(configuration) || len(states) > 1 || (len(states) == 1 && nilInterface(states[0])) {
return nil, ErrInvalidUpstreamDrainReaper
}
source := &EffectiveUpstreamDrainPolicySource{configuration: configuration}
if len(states) == 1 {
source.state = states[0]
}
return source, nil
}
func (source *EffectiveUpstreamDrainPolicySource) ReadUpstreamDrainPolicies(
ctx context.Context,
) ([]activitypool.UpstreamDrainPolicy, error) {
if ctx == nil || source == nil || nilInterface(source.configuration) {
return nil, ErrInvalidUpstreamDrainReaper
}
if err := ctx.Err(); err != nil {
return nil, err
}
configuration, revision := source.configuration.Snapshot()
if configuration == nil {
return nil, ErrInvalidUpstreamDrainReaper
}
states := make(map[string]adminstate.UpstreamState, len(configuration.Upstreams))
if source.state != nil {
snapshot, err := source.state.Snapshot(ctx)
if err != nil {
return nil, err
}
if snapshot.Config == nil || snapshot.Config.Revision != revision {
return nil, ErrInvalidUpstreamDrainReaper
}
for _, upstream := range snapshot.Upstreams {
if upstream.Name == "" || upstream.Revision == 0 {
return nil, ErrInvalidUpstreamDrainReaper
}
if _, duplicate := states[upstream.Name]; duplicate {
return nil, ErrInvalidUpstreamDrainReaper
}
states[upstream.Name] = upstream
}
}
if revision == 0 {
// A config-only Controller has no persisted management revision. Its
// configuration cannot be hot-reloaded, so one stable non-zero fence is enough.
revision = 1
}
names := make([]string, 0, len(configuration.Upstreams))
for name := range configuration.Upstreams {
names = append(names, name)
}
sort.Strings(names)
policies := make([]activitypool.UpstreamDrainPolicy, 0, len(names))
for _, name := range names {
upstream := configuration.Upstreams[name]
policy := activitypool.UpstreamDrainPolicy{UpstreamID: name, Revision: revision, Enabled: upstream.Enabled}
if source.state != nil {
admin, exists := states[name]
if !exists {
return nil, ErrInvalidUpstreamDrainReaper
}
policy.Enabled = policy.Enabled && admin.Enabled
policy.Revision = admin.Revision
}
policies = append(policies, policy)
}
return policies, nil
}
type UpstreamDrainPolicySource interface {
ReadUpstreamDrainPolicies(context.Context) ([]activitypool.UpstreamDrainPolicy, error)
}
// UpstreamDrainStore is the narrow activity-pool capability required by the
// configuration-driven Drain reaper.
type UpstreamDrainStore interface {
activitypool.UpstreamDrainPolicyWriter
activitypool.DisabledUpstreamDrainLister
activitypool.DisabledUpstreamDrainStarter
}
type UpstreamDrainReaperOptions struct {
PollInterval time.Duration
BatchSize int
Now func() time.Time
Metrics healthDomain.DrainMetricsObserver
}
type UpstreamDrainResult struct {
Candidates int
Started int
}
// ConfiguredUpstreamDrainReaper continually publishes an effective, complete
// policy view before reading one bounded batch of disabled Upstream ownership.
// The Redis ownership operation performs the final policy-revision fence.
type ConfiguredUpstreamDrainReaper struct {
policies UpstreamDrainPolicySource
store UpstreamDrainStore
options UpstreamDrainReaperOptions
}
func NewConfiguredUpstreamDrainReaper(
policies UpstreamDrainPolicySource,
store UpstreamDrainStore,
options UpstreamDrainReaperOptions,
) (*ConfiguredUpstreamDrainReaper, error) {
if nilInterface(policies) || nilInterface(store) || options.PollInterval <= 0 || options.BatchSize <= 0 || options.Now == nil {
return nil, ErrInvalidUpstreamDrainReaper
}
return &ConfiguredUpstreamDrainReaper{policies: policies, store: store, options: options}, nil
}
func (reaper *ConfiguredUpstreamDrainReaper) Tick(ctx context.Context) (UpstreamDrainResult, error) {
if ctx == nil || reaper == nil || nilInterface(reaper.policies) || nilInterface(reaper.store) ||
reaper.options.PollInterval <= 0 || reaper.options.BatchSize <= 0 || reaper.options.Now == nil {
return UpstreamDrainResult{}, ErrInvalidUpstreamDrainReaper
}
if err := ctx.Err(); err != nil {
return UpstreamDrainResult{}, err
}
now := reaper.options.Now()
if now.IsZero() {
return UpstreamDrainResult{}, ErrInvalidUpstreamDrainReaper
}
policies, err := reaper.policies.ReadUpstreamDrainPolicies(ctx)
if err != nil {
return UpstreamDrainResult{}, err
}
if err := reaper.store.ReplaceUpstreamDrainPolicies(ctx, policies); err != nil {
return UpstreamDrainResult{}, err
}
result := UpstreamDrainResult{}
remaining := reaper.options.BatchSize
utcNow := now.UTC()
for _, policy := range policies {
if policy.Enabled || remaining == 0 {
continue
}
candidates, err := reaper.store.ListDisabledUpstreamDrainCandidates(ctx, utcNow, policy, remaining)
if err != nil {
return result, err
}
if len(candidates) > remaining {
return result, ErrInvalidUpstreamDrainReaper
}
for _, candidate := range candidates {
started, err := reaper.store.BeginDisabledUpstreamDrain(ctx, utcNow, candidate)
if err != nil {
return result, err
}
result.Candidates++
if started {
result.Started++
}
}
remaining -= len(candidates)
}
if reaper.options.Metrics != nil {
reaper.options.Metrics.ObserveDrain(healthDomain.DrainReasonUpstreamDisabled, result.Candidates, result.Started)
}
return result, nil
}
func (reaper *ConfiguredUpstreamDrainReaper) Run(ctx context.Context) error {
if reaper == nil {
return ErrInvalidUpstreamDrainReaper
}
return runScheduler(ctx, reaper.options.PollInterval, func(tickCtx context.Context) (TickResult, error) {
_, err := reaper.Tick(tickCtx)
return TickResult{}, err
})
}

View File

@ -0,0 +1,133 @@
package health
import (
"context"
"testing"
"time"
"proxy-pool/internal/config"
"proxy-pool/internal/domain/activitypool"
"proxy-pool/internal/domain/adminstate"
healthDomain "proxy-pool/internal/domain/health"
proxyDomain "proxy-pool/internal/domain/proxy"
)
type upstreamDrainPolicySourceStub struct {
policies []activitypool.UpstreamDrainPolicy
err error
}
func (source upstreamDrainPolicySourceStub) ReadUpstreamDrainPolicies(context.Context) ([]activitypool.UpstreamDrainPolicy, error) {
return append([]activitypool.UpstreamDrainPolicy(nil), source.policies...), source.err
}
type upstreamDrainConfigurationStub struct {
configuration *config.Config
revision uint64
}
type upstreamDrainStateStub struct{ snapshot adminstate.Snapshot }
func (source upstreamDrainStateStub) Snapshot(context.Context) (adminstate.Snapshot, error) {
return source.snapshot, nil
}
type drainMetricRecord struct {
reason healthDomain.DrainReason
candidates int
started int
}
type recordingDrainMetrics struct{ records []drainMetricRecord }
func (metrics *recordingDrainMetrics) ObserveDrain(reason healthDomain.DrainReason, candidates, started int) {
metrics.records = append(metrics.records, drainMetricRecord{reason: reason, candidates: candidates, started: started})
}
func (source upstreamDrainConfigurationStub) Snapshot() (*config.Config, uint64) {
return source.configuration, source.revision
}
func TestConfiguredUpstreamDrainReaperStartsBoundedDisabledAssignments(t *testing.T) {
now := time.Date(2026, 8, 2, 16, 0, 0, 0, time.UTC)
pool := activitypool.NewMemoryPool()
if _, err := pool.UpsertFetched(context.Background(), "provider-a", activitypool.FetchedBatch{
ObservedAt: now, ConfiguredTTL: time.Minute, MaxSize: 1,
Proxies: []proxyDomain.Proxy{{
ID: "proxy-a", Scheme: proxyDomain.SchemeHTTP, Host: "192.0.2.10", Port: 8080, State: proxyDomain.StateAvailable,
}},
}); err != nil {
t.Fatalf("UpsertFetched() error = %v", err)
}
assignment, err := pool.Assign(context.Background(), now, "proxy-a", "worker-a", time.Minute)
if err != nil {
t.Fatalf("Assign() error = %v", err)
}
policy := activitypool.UpstreamDrainPolicy{UpstreamID: "provider-a", Revision: 21}
metrics := &recordingDrainMetrics{}
reaper, err := NewConfiguredUpstreamDrainReaper(upstreamDrainPolicySourceStub{policies: []activitypool.UpstreamDrainPolicy{policy}}, pool,
UpstreamDrainReaperOptions{PollInterval: time.Second, BatchSize: 4, Now: func() time.Time { return now }, Metrics: metrics})
if err != nil {
t.Fatalf("NewConfiguredUpstreamDrainReaper() error = %v", err)
}
result, err := reaper.Tick(context.Background())
if err != nil || result != (UpstreamDrainResult{Candidates: 1, Started: 1}) {
t.Fatalf("Tick() = %+v, %v", result, err)
}
if current, found, err := pool.Get(context.Background(), "proxy-a"); err != nil || !found ||
current.Epoch != assignment.Epoch || !current.Draining {
t.Fatalf("Get(after drain) = %+v, %t, %v", current, found, err)
}
if len(metrics.records) != 1 || metrics.records[0] != (drainMetricRecord{
reason: healthDomain.DrainReasonUpstreamDisabled, candidates: 1, started: 1,
}) {
t.Fatalf("Drain metrics = %+v", metrics.records)
}
}
func TestEffectiveUpstreamDrainPolicySourceUsesStableConfigOnlyRevision(t *testing.T) {
configuration := &config.Config{Upstreams: map[string]config.Upstream{
"provider-b": {Enabled: true},
"provider-a": {Enabled: false},
}}
source, err := NewEffectiveUpstreamDrainPolicySource(upstreamDrainConfigurationStub{configuration: configuration})
if err != nil {
t.Fatalf("NewEffectiveUpstreamDrainPolicySource() error = %v", err)
}
policies, err := source.ReadUpstreamDrainPolicies(context.Background())
if err != nil || len(policies) != 2 || policies[0] != (activitypool.UpstreamDrainPolicy{
UpstreamID: "provider-a", Revision: 1,
}) || policies[1] != (activitypool.UpstreamDrainPolicy{
UpstreamID: "provider-b", Revision: 1, Enabled: true,
}) {
t.Fatalf("ReadUpstreamDrainPolicies() = %+v, %v", policies, err)
}
}
func TestEffectiveUpstreamDrainPolicySourceCombinesConfigurationAndAdminState(t *testing.T) {
configuration := &config.Config{Upstreams: map[string]config.Upstream{
"provider-a": {Enabled: false},
"provider-b": {Enabled: true},
}}
source, err := NewEffectiveUpstreamDrainPolicySource(
upstreamDrainConfigurationStub{configuration: configuration, revision: 9},
upstreamDrainStateStub{snapshot: adminstate.Snapshot{
Config: &adminstate.ConfigRevision{Revision: 9},
Upstreams: []adminstate.UpstreamState{
{Name: "provider-a", Enabled: true, Revision: 7},
{Name: "provider-b", Enabled: false, Revision: 8},
},
}},
)
if err != nil {
t.Fatalf("NewEffectiveUpstreamDrainPolicySource() error = %v", err)
}
policies, err := source.ReadUpstreamDrainPolicies(context.Background())
want := []activitypool.UpstreamDrainPolicy{
{UpstreamID: "provider-a", Revision: 7},
{UpstreamID: "provider-b", Revision: 8},
}
if err != nil || len(policies) != len(want) || policies[0] != want[0] || policies[1] != want[1] {
t.Fatalf("ReadUpstreamDrainPolicies() = %+v, %v; want %+v", policies, err, want)
}
}

Some files were not shown because too many files have changed in this diff Show More