Compare commits
No commits in common. "d6ac783c1d6906cabc2462a4be460bdb374c88bd" and "5ad1d10957c11ba28a91a7c709ee45714c9d8fa3" have entirely different histories.
d6ac783c1d
...
5ad1d10957
@ -1,23 +0,0 @@
|
|||||||
# Source control metadata
|
|
||||||
.git
|
|
||||||
|
|
||||||
# Local configuration and credential material
|
|
||||||
.env
|
|
||||||
.env.*
|
|
||||||
configs/local.yaml
|
|
||||||
deploy/.control-plane-tls/
|
|
||||||
*.crt
|
|
||||||
*.key
|
|
||||||
*.pem
|
|
||||||
|
|
||||||
# Local build, test, and tooling outputs
|
|
||||||
.playwright-mcp/
|
|
||||||
temp/
|
|
||||||
bin/
|
|
||||||
dist/
|
|
||||||
coverage/
|
|
||||||
*.out
|
|
||||||
*.test
|
|
||||||
*.prof
|
|
||||||
*.exe
|
|
||||||
.tmp-proto/
|
|
||||||
48
.github/workflows/ci.yml
vendored
48
.github/workflows/ci.yml
vendored
@ -8,16 +8,6 @@ permissions:
|
|||||||
contents: read
|
contents: read
|
||||||
|
|
||||||
jobs:
|
jobs:
|
||||||
openapi:
|
|
||||||
runs-on: ubuntu-latest
|
|
||||||
steps:
|
|
||||||
- uses: actions/checkout@v4
|
|
||||||
- uses: actions/setup-node@v4
|
|
||||||
with:
|
|
||||||
node-version: 24
|
|
||||||
- shell: pwsh
|
|
||||||
run: ./scripts/verify-openapi.ps1
|
|
||||||
|
|
||||||
proto:
|
proto:
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
steps:
|
steps:
|
||||||
@ -29,44 +19,6 @@ jobs:
|
|||||||
- shell: pwsh
|
- shell: pwsh
|
||||||
run: ./scripts/verify-proto.ps1
|
run: ./scripts/verify-proto.ps1
|
||||||
|
|
||||||
image:
|
|
||||||
runs-on: ubuntu-latest
|
|
||||||
steps:
|
|
||||||
- uses: actions/checkout@v4
|
|
||||||
- name: Build Linux application image
|
|
||||||
run: docker build --file deploy/docker/Dockerfile --tag proxy-pool:ci .
|
|
||||||
|
|
||||||
deployment:
|
|
||||||
runs-on: ubuntu-latest
|
|
||||||
env:
|
|
||||||
PROXY_POOL_GATEWAY_USERNAME: ci-gateway-user
|
|
||||||
PROXY_POOL_GATEWAY_PASSWORD: ci-gateway-password
|
|
||||||
PROXY_POOL_EXTRACT_TOKEN: ci-extract-token
|
|
||||||
PROXY_POOL_ADMIN_TOKEN: ci-admin-token
|
|
||||||
PROXY_POOL_CONFIG_FINGERPRINT_KEY: ci-config-fingerprint-key-at-least-32-bytes
|
|
||||||
PROXY_POOL_POSTGRES_URL: postgres://ci:ci@postgres.example:5432/proxy_pool?sslmode=verify-full
|
|
||||||
PROXY_POOL_REDIS_URL: rediss://:ci@redis.example:6379/0
|
|
||||||
PROVIDER_A_TOKEN: ci-provider-a-token
|
|
||||||
PROVIDER_B_TOKEN: ci-provider-b-token
|
|
||||||
steps:
|
|
||||||
- uses: actions/checkout@v4
|
|
||||||
- uses: actions/setup-go@v5
|
|
||||||
with:
|
|
||||||
go-version-file: go.mod
|
|
||||||
cache: true
|
|
||||||
- name: Render Compose configuration
|
|
||||||
run: docker compose -f deploy/docker-compose.yml config --quiet
|
|
||||||
- name: Render integration Compose configuration
|
|
||||||
run: docker compose -f deploy/docker-compose.test.yml config --quiet
|
|
||||||
- name: Validate local Compose configuration
|
|
||||||
run: go run ./deploy/tools/configcheck deploy/config/local.yaml
|
|
||||||
- name: Render Kubernetes base
|
|
||||||
run: kubectl kustomize deploy/kubernetes/base > /dev/null
|
|
||||||
- name: Render Kubernetes development mTLS overlay
|
|
||||||
run: kubectl kustomize deploy/kubernetes/overlays/development-mtls > /dev/null
|
|
||||||
- name: Validate development mTLS configuration
|
|
||||||
run: go run ./deploy/tools/configcheck deploy/kubernetes/overlays/development-mtls/config.yaml
|
|
||||||
|
|
||||||
test:
|
test:
|
||||||
strategy:
|
strategy:
|
||||||
matrix:
|
matrix:
|
||||||
|
|||||||
1
.gitignore
vendored
1
.gitignore
vendored
@ -17,7 +17,6 @@ coverage/
|
|||||||
.env.*
|
.env.*
|
||||||
!.env.example
|
!.env.example
|
||||||
configs/local.yaml
|
configs/local.yaml
|
||||||
deploy/.control-plane-tls/
|
|
||||||
*.pem
|
*.pem
|
||||||
*.key
|
*.key
|
||||||
|
|
||||||
|
|||||||
123
README.md
123
README.md
@ -41,28 +41,8 @@ Proxy Pool 用 Controller 协调这些变化,并让 Gateway 数据面只消费
|
|||||||
PostgreSQL 迁移与 Redis 活动池,并支持联动优雅停机。
|
PostgreSQL 迁移与 Redis 活动池,并支持联动优雅停机。
|
||||||
- **Checker 指标**:Metrics 启用时暴露 Checker 任务下发与 Observation 接受/拒绝计数;
|
- **Checker 指标**:Metrics 启用时暴露 Checker 任务下发与 Observation 接受/拒绝计数;
|
||||||
标签仅使用固定检查级别和结果,不记录 Proxy、IP、URL 或凭据。
|
标签仅使用固定检查级别和结果,不记录 Proxy、IP、URL 或凭据。
|
||||||
- **Gateway 指标**:Metrics 启用时暴露代理尝试的固定阶段成功/失败计数、本地
|
- **Gateway 指标**:Metrics 启用时暴露代理尝试的固定阶段成功/失败计数,以及本地
|
||||||
Outcome 队列满后的丢弃计数、HTTP/CONNECT 已接收请求数与在途数,以及活跃
|
Outcome 队列满后的丢弃计数;不记录 Proxy、路由、目标、客户端或凭据。
|
||||||
CONNECT 隧道数和请求生命周期 p50/p95/p99 直方图;协议标签仅有 HTTP 与 CONNECT,不记录 Proxy、路由、目标、
|
|
||||||
客户端或凭据。
|
|
||||||
- **Drain 指标**:Controller 暴露 `proxy_pool_controller_drain_candidates_total` 与
|
|
||||||
`proxy_pool_controller_drains_started_total`;`reason` 仅有 `unhealthy` 与
|
|
||||||
`upstream_disabled`,不包含 Proxy、Worker、Upstream、会话或地址。
|
|
||||||
- **Provider 指标**:Controller 暴露
|
|
||||||
`proxy_pool_controller_provider_fetch_results_total{class}`、
|
|
||||||
`proxy_pool_controller_provider_valid_candidates_total` 与
|
|
||||||
`proxy_pool_controller_provider_new_proxies_total`;`class` 仅有 `valid`、
|
|
||||||
`empty`、`duplicate_only`、`error`,不包含 Upstream、Proxy 或错误文本标签。
|
|
||||||
- **提取指标**:Controller 暴露
|
|
||||||
`proxy_pool_controller_extraction_requests_total{result}`、
|
|
||||||
`proxy_pool_controller_extraction_requested_proxies_total` 与
|
|
||||||
`proxy_pool_controller_extraction_returned_proxies_total`;`result` 仅有完成、
|
|
||||||
部分、空、库存不足、幂等冲突、限流、不可用、无效与内部错误等固定枚举,
|
|
||||||
不包含 Client、请求、过滤条件、Upstream、Proxy 或错误文本标签。
|
|
||||||
- **容量指标**:Controller 在既有 Provider 库存对账周期聚合托管 Proxy、可用/有效
|
|
||||||
Slot、待拉取数量和活跃上游数,并暴露
|
|
||||||
`proxy_pool_controller_capacity_inventory_reads_total{result}`;`result` 仅有
|
|
||||||
`success` 和 `error`,不按 Upstream、Worker、Proxy、会话或地址拆分。
|
|
||||||
- **PostgreSQL 管理面**:持久化配置版本、Upstream/Routing 管理状态、Admin
|
- **PostgreSQL 管理面**:持久化配置版本、Upstream/Routing 管理状态、Admin
|
||||||
审计与 Outbox;不保存 Proxy 明细或逐次提取记录。
|
审计与 Outbox;不保存 Proxy 明细或逐次提取记录。
|
||||||
- **Gateway 组件**:HTTP 正向代理、HTTPS CONNECT、双向 Tunnel、重试、超时、
|
- **Gateway 组件**:HTTP 正向代理、HTTPS CONNECT、双向 Tunnel、重试、超时、
|
||||||
@ -76,20 +56,9 @@ Proxy Pool 用 Controller 协调这些变化,并让 Gateway 数据面只消费
|
|||||||
Connections 选择上游。无候选时支持 reject、受 `waitTimeout` 限制的本地容量等待,
|
Connections 选择上游。无候选时支持 reject、受 `waitTimeout` 限制的本地容量等待,
|
||||||
以及仍经过目标地址策略的 direct;`proxy-gateway` 通过独立控制面拨号地址维护
|
以及仍经过目标地址策略的 direct;`proxy-gateway` 通过独立控制面拨号地址维护
|
||||||
Session,并在每份 Snapshot 有效期的一半前接收版本递增的完整刷新;仅在持有未过期
|
Session,并在每份 Snapshot 有效期的一半前接收版本递增的完整刷新;仅在持有未过期
|
||||||
Snapshot 时 Ready,凭据材料只保留在当前节点内存 View。Controller 内成功提交的
|
Snapshot 时 Ready,凭据材料只保留在当前节点内存 View。
|
||||||
Upstream 启停、Routing 切换和配置发布会向本进程全部在线 Worker 快照流广播刷新;
|
|
||||||
定时刷新仍作为跨进程收敛与失效保护。
|
|
||||||
- **Gateway 粘性会话**:可为已认证请求启用 X-Proxy-Session 等配置 Header;
|
|
||||||
Gateway 以 Client、Routing 和会话值派生本地有界绑定,成功建连后固定到同一
|
|
||||||
Proxy,并在代理失效、快照移除或转发失败时自动重绑。原始会话值不会写入日志、
|
|
||||||
指标、PostgreSQL 或 Redis,也不会向目标站点转发。
|
|
||||||
- **安全边界**:Gateway、Distribution 与 Admin 使用各自的认证语义,并支持
|
- **安全边界**:Gateway、Distribution 与 Admin 使用各自的认证语义,并支持
|
||||||
CIDR、可信代理、严格请求解析和敏感信息最小化;Admin 的读写权限与
|
CIDR、可信代理、严格请求解析和敏感信息最小化。
|
||||||
Distribution 提取权限可按命中凭据分别收敛。Distribution 凭据还可限制单次
|
|
||||||
提取数量、可访问 Upstream 与地区,Gateway 凭据可限制可访问 Routing;未配置时
|
|
||||||
保持既有全范围行为。Gateway 凭据还可在每个 Worker 内限制每分钟请求数,
|
|
||||||
同时限制 HTTP 请求和 CONNECT 隧道的并发数;热路径计数不写入 Redis 或
|
|
||||||
PostgreSQL。
|
|
||||||
|
|
||||||
## 架构概览
|
## 架构概览
|
||||||
|
|
||||||
@ -116,25 +85,21 @@ flowchart LR
|
|||||||
|
|
||||||
## 当前完成度
|
## 当前完成度
|
||||||
|
|
||||||
截至 **2026-08-07**,实施计划中可直接勾选的检查项为 **71 / 75(94.7%)**。详情见
|
截至 **2026-08-02**,实施计划中可直接勾选的检查项为 **57 / 74(77.0%)**。详情见
|
||||||
[实施计划](docs/development/implementation-plan.md)和
|
[实施计划](docs/development/implementation-plan.md)和
|
||||||
[交付完成度审计](docs/requirements/completion-audit.md)。
|
[交付完成度审计](docs/requirements/completion-audit.md)。
|
||||||
|
|
||||||
- **已完成**:严格配置、Provider 获取与协调、Redis 活动池、Distribution 原子
|
- **已完成**:严格配置、Provider 获取与协调、Redis 活动池、Distribution 原子
|
||||||
提取与限流、Controller 的 Admin/Distribution/Metrics 监听,以及 PostgreSQL
|
提取与限流、Controller 的 Admin/Distribution/Metrics 监听,以及 PostgreSQL
|
||||||
管理状态与有界审计查询;WorkerControlPlane 的 Register、Snapshot ACK、Runtime 心跳接收和
|
管理状态;WorkerControlPlane 的 Register、Snapshot ACK、Runtime 心跳接收和
|
||||||
Redis 会话栅栏,以及 Gateway Outcome 上报的有界队列、序列确认与重试;
|
Redis 会话栅栏,以及 Gateway Outcome 上报的有界队列、序列确认与重试;
|
||||||
Controller 的 Redis 共享 BASIC/EGRESS/TARGET 检查任务、按上游的有界轮转调度、HTTP/HTTPS/SOCKS5
|
Controller 的 Redis 共享 BASIC/EGRESS/TARGET 检查任务、按上游的有界轮转调度、HTTP/HTTPS/SOCKS5
|
||||||
Checker 探测和
|
Checker 探测和
|
||||||
Observation 状态归并;Provider 连续空结果与管理态禁用当前项触发的 Sequential CAS
|
Observation 状态归并。
|
||||||
自动推进、禁用候选过滤、末端路由停用和 Snapshot 即时刷新。
|
- **部分完成**:Docker Compose/Kubernetes 运行时 mTLS Overlay。
|
||||||
- **部分完成**:Kubernetes 运行时 mTLS Overlay;Compose 已具备本地的
|
- **待完成**:CONNECT 长连接/Extract 压测场景、故障演练和代表性集群压测。
|
||||||
Controller/Gateway/Checker mTLS 运行链路。
|
|
||||||
- **待完成**:故障演练和代表性集群压测;现有 HTTP、CONNECT 长连接和 Extract
|
|
||||||
场景只提供可复现的负载工具,不构成容量验证结论。
|
|
||||||
|
|
||||||
检查项数量不等于生产就绪度。CI 会验证 OpenAPI 契约、固定版本的 protobuf descriptor/
|
检查项数量不等于生产就绪度。静态部署清单与 protobuf descriptor 验证也不代表
|
||||||
生成代码漂移,以及 Compose/Kustomize 的静态渲染;这些门禁也不代表
|
|
||||||
端到端拓扑已经完成;`100,000 QPS` 仍只是待验证的集群设计目标。
|
端到端拓扑已经完成;`100,000 QPS` 仍只是待验证的集群设计目标。
|
||||||
|
|
||||||
## 快速开始
|
## 快速开始
|
||||||
@ -159,17 +124,8 @@ Secret 分离管理。
|
|||||||
```powershell
|
```powershell
|
||||||
go run ./deploy/tools/configcheck deploy/config/local.yaml
|
go run ./deploy/tools/configcheck deploy/config/local.yaml
|
||||||
./scripts/verify.ps1
|
./scripts/verify.ps1
|
||||||
./scripts/package-docs.ps1
|
|
||||||
./scripts/benchmark-gateway.ps1
|
|
||||||
```
|
```
|
||||||
|
|
||||||
`package-docs.ps1` 默认生成被忽略的 `dist/proxy-pool-docs-v1.0.zip`;包内包含 README、
|
|
||||||
`docs/`、图表、OpenAPI、Proto 契约和部署手册,并以 `manifest.json` 记录 Git revision、文件大小和
|
|
||||||
SHA-256。可使用 `-Version vMAJOR.MINOR[.PATCH]` 与 `-OutputPath OUTPUT.zip` 生成指定交付物。
|
|
||||||
`benchmark-gateway.ps1` 固定执行一次 100k 索引调度、Routing Round Robin 和 Snapshot Apply
|
|
||||||
微基准,并把版本与原始输出写入 `dist/gateway-benchmarks.txt`;它用于回归比较,不构成
|
|
||||||
网络转发或 100k QPS 集群容量证明。
|
|
||||||
|
|
||||||
Redis、PostgreSQL 和 Controller fixture 脚本会使用 Docker 启动隔离依赖:
|
Redis、PostgreSQL 和 Controller fixture 脚本会使用 Docker 启动隔离依赖:
|
||||||
|
|
||||||
```powershell
|
```powershell
|
||||||
@ -178,19 +134,6 @@ Redis、PostgreSQL 和 Controller fixture 脚本会使用 Docker 启动隔离依
|
|||||||
./scripts/test-controller.ps1
|
./scripts/test-controller.ps1
|
||||||
```
|
```
|
||||||
|
|
||||||
本地 Compose 还会启动 Controller、两个从开发证书派生身份的 Gateway 和一个 Checker。
|
|
||||||
首次启动前生成仅用于本机的 7 天 mTLS 证书;输出目录受 `.gitignore` 保护,脚本拒绝写入非空目录:
|
|
||||||
|
|
||||||
```powershell
|
|
||||||
./scripts/generate-local-controlplane-certs.ps1
|
|
||||||
docker compose -f deploy/docker-compose.yml up -d --build
|
|
||||||
docker compose -f deploy/docker-compose.yml ps
|
|
||||||
```
|
|
||||||
|
|
||||||
Gateway 只有取得有效 Snapshot 后才会通过 `/readyz`;Checker 在首次成功领取控制面任务
|
|
||||||
批次后才会通过 `/readyz`,空批次也代表连接和身份验证已经成功。后续任务领取失败会立即
|
|
||||||
撤销 Checker 的就绪状态。
|
|
||||||
|
|
||||||
PostgreSQL 与 Redis 是 Controller 的启动依赖。当前可运行的 Controller 入口如下,
|
PostgreSQL 与 Redis 是 Controller 的启动依赖。当前可运行的 Controller 入口如下,
|
||||||
将 `CONFIG_FILE` 替换为实际配置路径,并确保其中的 PostgreSQL 与 Redis 地址可从
|
将 `CONFIG_FILE` 替换为实际配置路径,并确保其中的 PostgreSQL 与 Redis 地址可从
|
||||||
进程所在网络访问:
|
进程所在网络访问:
|
||||||
@ -204,7 +147,7 @@ go run ./cmd/proxy-controller -config CONFIG_FILE
|
|||||||
改用宿主机可达的存储地址。
|
改用宿主机可达的存储地址。
|
||||||
|
|
||||||
本地配置中的 `.invalid` Provider URL 是故障演示占位,不会提供真实代理。
|
本地配置中的 `.invalid` Provider URL 是故障演示占位,不会提供真实代理。
|
||||||
Gateway 也可独立启动;需要启用 Controller `controlPlane` 并配置匹配的 mTLS
|
Gateway 已提供启动命令;需要先启用 Controller `controlPlane` 并配置匹配的 mTLS
|
||||||
证书(回环 fixture 可使用明文),再提供独立的拨号地址和 Worker 身份:
|
证书(回环 fixture 可使用明文),再提供独立的拨号地址和 Worker 身份:
|
||||||
|
|
||||||
```powershell
|
```powershell
|
||||||
@ -216,10 +159,6 @@ go run ./cmd/proxy-gateway -config CONFIG_FILE `
|
|||||||
|
|
||||||
以上参数也可通过 `PROXY_POOL_CONTROL_PLANE_ADDRESS`、`PROXY_POOL_CLUSTER_ID`、
|
以上参数也可通过 `PROXY_POOL_CONTROL_PLANE_ADDRESS`、`PROXY_POOL_CLUSTER_ID`、
|
||||||
`PROXY_POOL_WORKER_ID`、`PROXY_POOL_INSTANCE_ID` 与 `PROXY_POOL_ZONE` 提供。
|
`PROXY_POOL_WORKER_ID`、`PROXY_POOL_INSTANCE_ID` 与 `PROXY_POOL_ZONE` 提供。
|
||||||
生产工作负载可设置 `PROXY_POOL_AUTO_IDENTITY=true`,省略 Worker 和 Instance ID;
|
|
||||||
Gateway 会从挂载的 `gatewayTLS` 证书解析
|
|
||||||
`spiffe://<trust-domain>/<environment>/worker/<worker-id>`,并将 `<worker-id>`
|
|
||||||
作为默认实例身份。Controller 仍会将请求 ID 与证书 URI 严格比对。
|
|
||||||
Gateway 的 `/livez`、`/readyz`、`/metrics` 使用配置中的 `metrics.listen`;无有效
|
Gateway 的 `/livez`、`/readyz`、`/metrics` 使用配置中的 `metrics.listen`;无有效
|
||||||
Snapshot 时 `/readyz` 返回 `503`。Checker 使用独立的逻辑/实例身份拉取有界任务:
|
Snapshot 时 `/readyz` 返回 `503`。Checker 使用独立的逻辑/实例身份拉取有界任务:
|
||||||
|
|
||||||
@ -232,17 +171,12 @@ go run ./cmd/proxy-checker -config CONFIG_FILE `
|
|||||||
|
|
||||||
Checker 的参数也可通过 `PROXY_POOL_CONTROL_PLANE_ADDRESS`、
|
Checker 的参数也可通过 `PROXY_POOL_CONTROL_PLANE_ADDRESS`、
|
||||||
`PROXY_POOL_CHECKER_ID`、`PROXY_POOL_CHECKER_INSTANCE_ID` 与
|
`PROXY_POOL_CHECKER_ID`、`PROXY_POOL_CHECKER_INSTANCE_ID` 与
|
||||||
`PROXY_POOL_CHECKER_MAX_IN_FLIGHT` 提供。`PROXY_POOL_AUTO_IDENTITY=true` 会从
|
`PROXY_POOL_CHECKER_MAX_IN_FLIGHT` 提供。它不会访问 Redis/PostgreSQL;生产
|
||||||
`checkerTLS` 的 `.../checker/<checker-id>` URI 自动派生 Checker 与缺省实例身份。
|
|
||||||
它不会访问 Redis/PostgreSQL;生产
|
|
||||||
Controller 在启用控制面时装配 Redis 共享任务 broker,并按启用的 Upstream 调度
|
Controller 在启用控制面时装配 Redis 共享任务 broker,并按启用的 Upstream 调度
|
||||||
HTTP/HTTPS/SOCKS5 BASIC 检查、按每个 `check.urls` 创建 EGRESS 任务,并按启用 Routing 的
|
HTTP/HTTPS/SOCKS5 BASIC 检查、按每个 `check.urls` 创建 EGRESS 任务,并按启用 Routing 的
|
||||||
`check.targets` 创建 TARGET 任务。调度监督器每轮读取已发布配置;启用 Admin 时只调度配置与
|
`check.targets` 创建 TARGET 任务。调度监督器每轮读取已发布配置,因此 reload 后的上游/路由启停、
|
||||||
PostgreSQL 管理态同 revision 且均启用的 Upstream 和 Routing。管理态停用 Upstream 会在下一轮阻止新的
|
检查间隔、抖动、超时、重试次数、`maxInFlight`、EGRESS URL 和 TARGET Profile 都会在下一轮生效;
|
||||||
BASIC、EGRESS、TARGET 任务;管理态停用 Routing 则停止该 Routing 的新 TARGET 任务。revision 不一致或
|
BASIC、EGRESS 与 TARGET 以有界轮转组共享上游并发上限。新启用的上游无需重启 Controller。
|
||||||
状态不完整时按失败关闭。因此 reload 后的上游/路由启停、检查间隔、
|
|
||||||
抖动、超时、重试次数、`maxInFlight`、EGRESS URL 和 TARGET Profile 都会在下一轮生效;BASIC、
|
|
||||||
EGRESS 与 TARGET 以有界轮转组共享上游并发上限。新启用的上游无需重启 Controller。
|
|
||||||
EGRESS 对成功响应提取纯文本 IP 或常见 JSON IP 字段,并将其作为全局健康事实回传;TARGET 事实
|
EGRESS 对成功响应提取纯文本 IP 或常见 JSON IP 字段,并将其作为全局健康事实回传;TARGET 事实
|
||||||
仅归并到对应的 `(routing_name, target_url)` Profile,不改变 Proxy 全局健康。
|
仅归并到对应的 `(routing_name, target_url)` Profile,不改变 Proxy 全局健康。
|
||||||
|
|
||||||
@ -291,27 +225,9 @@ go run ./cmd/proxy-loadgen `
|
|||||||
```
|
```
|
||||||
|
|
||||||
命令输出 JSON 报告,包含成功/失败分类、`TunnelsEstablished`、
|
命令输出 JSON 报告,包含成功/失败分类、`TunnelsEstablished`、
|
||||||
`ExtractResponsesValidated`、`ExtractReturned`、`ExtractValidationFailures`、`RateStartsGenerated`、
|
`ExtractResponsesValidated`、`ExtractReturned`、`ExtractValidationFailures`、固定内存的
|
||||||
`RateStartsDropped`、固定内存的
|
|
||||||
连接握手延迟分位上界、吞吐和 Go 运行时内存/GC 快照。它不会输出提取响应中的地址或
|
连接握手延迟分位上界、吞吐和 Go 运行时内存/GC 快照。它不会输出提取响应中的地址或
|
||||||
凭据,也不构成 100,000 QPS 证明。限速场景以 `Requests` 表示实际发起数;当
|
凭据,也不构成 100,000 QPS 证明。
|
||||||
`RateStartsDropped` 非零时,目标速率受压测端并发容量或时间窗限制,报告吞吐不得
|
|
||||||
标注为已达到配置的 `-rate`。每个生成的限速令牌均会计入实际发起或丢弃,因此
|
|
||||||
`RateStartsGenerated = Requests + RateStartsDropped`。
|
|
||||||
需要保留单次容量证据时,使用 `-output REPORT_FILE` 同时写出 JSON 文件;文件在
|
|
||||||
同目录完整写入后才替换目标,标准输出仍保留相同报告,便于交给日志或指标系统。
|
|
||||||
`-max-error-rate` 接受 `0` 到 `1`,`-max-p99` 接受正的 Go duration;任一阈值违反时,
|
|
||||||
报告会附带 `acceptance` 结果,命令仍写出标准输出和 `-output` 文件,然后以退出码 `3`
|
|
||||||
结束,适合在 CI 中保留失败压测证据:
|
|
||||||
|
|
||||||
```powershell
|
|
||||||
go run ./cmd/proxy-loadgen `
|
|
||||||
-target https://TARGET_URL/health `
|
|
||||||
-proxy http://GATEWAY_HOST:8080 `
|
|
||||||
-duration 30s -rate 5000 -concurrency 128 `
|
|
||||||
-max-error-rate 0.01 -max-p99 500ms `
|
|
||||||
-output REPORT_FILE
|
|
||||||
```
|
|
||||||
|
|
||||||
## 关键配置与入口
|
## 关键配置与入口
|
||||||
|
|
||||||
@ -347,10 +263,7 @@ go run ./cmd/proxy-loadgen `
|
|||||||
回收会自动为已分配项发起带健康/归属栅栏的 Drain,待 Snapshot ACK 与运行态归零后再删除;
|
回收会自动为已分配项发起带健康/归属栅栏的 Drain,待 Snapshot ACK 与运行态归零后再删除;
|
||||||
TARGET 事实按路由目标 Profile 独立归并。
|
TARGET 事实按路由目标 Profile 独立归并。
|
||||||
- **P1 - Gateway 与 Routing**:Gateway 进程、快照凭据分发、五种 Routing 策略与
|
- **P1 - Gateway 与 Routing**:Gateway 进程、快照凭据分发、五种 Routing 策略与
|
||||||
`onUnavailable` 已接入;上游停用会从后续完整 Snapshot 排除,并对现有 Worker
|
`onUnavailable` 已接入;动态容量调整和配置停用触发的 Drain 编排待完成。
|
||||||
ownership 发起带策略 revision 栅栏的 Drain。Routing 切换立即刷新在线 Worker 的
|
|
||||||
完整快照;Sequential 的新分配切到新上游,旧 Proxy 与既有连接自然排空。动态容量
|
|
||||||
调整仍待完成。
|
|
||||||
- **P1 - 可观测与部署**:低基数业务指标、完整 Compose/Kubernetes 进程拓扑,
|
- **P1 - 可观测与部署**:低基数业务指标、完整 Compose/Kubernetes 进程拓扑,
|
||||||
以及故障转移和恢复演练。
|
以及故障转移和恢复演练。
|
||||||
- **P2 - 容量证明**:`proxy-loadgen`、HTTP/CONNECT/Extract 分场景压测,以及
|
- **P2 - 容量证明**:`proxy-loadgen`、HTTP/CONNECT/Extract 分场景压测,以及
|
||||||
|
|||||||
@ -2,23 +2,14 @@ openapi: 3.1.0
|
|||||||
info:
|
info:
|
||||||
title: Proxy Pool Admin API
|
title: Proxy Pool Admin API
|
||||||
version: 1.0.0
|
version: 1.0.0
|
||||||
description: |
|
description: 运维状态与受控变更接口。该入口必须与 Distribution 分端口和权限。
|
||||||
运维状态与受控变更接口。该入口必须与 Distribution 分端口和权限。
|
|
||||||
已认证凭据可配置 `admin:read`(Status、审计)和 `admin:write`(变更);
|
|
||||||
未配置权限的旧凭据保留全权限行为。
|
|
||||||
servers:
|
servers:
|
||||||
- url: http://127.0.0.1:8082
|
- url: http://127.0.0.1:8082
|
||||||
tags:
|
tags:
|
||||||
- name: Status
|
- name: Status
|
||||||
description: 不含代理明细与凭据的控制面聚合状态。
|
|
||||||
- name: Audit
|
|
||||||
description: 权威管理状态变更的游标审计查询。
|
|
||||||
- name: Upstreams
|
- name: Upstreams
|
||||||
description: 上游的启停与受控状态变更。
|
|
||||||
- name: Routing
|
- name: Routing
|
||||||
description: Routing 当前上游的受控切换。
|
|
||||||
- name: Configuration
|
- name: Configuration
|
||||||
description: 经校验后原子发布的配置重载。
|
|
||||||
security:
|
security:
|
||||||
- AdminApiKey: []
|
- AdminApiKey: []
|
||||||
- BasicAuth: []
|
- BasicAuth: []
|
||||||
@ -30,7 +21,6 @@ paths:
|
|||||||
tags: [Status]
|
tags: [Status]
|
||||||
operationId: getStatus
|
operationId: getStatus
|
||||||
summary: 获取控制面摘要状态
|
summary: 获取控制面摘要状态
|
||||||
description: 需要 `admin:read`。
|
|
||||||
responses:
|
responses:
|
||||||
'200':
|
'200':
|
||||||
description: 不含 Proxy 地址、Client 标识或 Secret 的聚合状态
|
description: 不含 Proxy 地址、Client 标识或 Secret 的聚合状态
|
||||||
@ -46,50 +36,11 @@ paths:
|
|||||||
'405': {$ref: '#/components/responses/MethodNotAllowed'}
|
'405': {$ref: '#/components/responses/MethodNotAllowed'}
|
||||||
'500': {$ref: '#/components/responses/InternalServerError'}
|
'500': {$ref: '#/components/responses/InternalServerError'}
|
||||||
'503': {$ref: '#/components/responses/ServiceUnavailable'}
|
'503': {$ref: '#/components/responses/ServiceUnavailable'}
|
||||||
/api/v1/audit:
|
|
||||||
get:
|
|
||||||
tags: [Audit]
|
|
||||||
operationId: listAuditRecords
|
|
||||||
summary: 按 ID 游标读取管理面审计记录
|
|
||||||
description: |
|
|
||||||
需要 `admin:read`。
|
|
||||||
|
|
||||||
仅返回权威管理面的变更审计记录,按 `id` 升序排列。`afterId` 是排他游标:
|
|
||||||
后续页面只包含 `id` 大于该值的记录。未传 `limit` 时返回 100 条,单页最多
|
|
||||||
1000 条。
|
|
||||||
|
|
||||||
审计接口不返回 Proxy、Upstream URL、凭据或活动池内容。
|
|
||||||
parameters:
|
|
||||||
- name: afterId
|
|
||||||
in: query
|
|
||||||
required: false
|
|
||||||
description: 排他游标;仅返回 ID 大于此值的记录。
|
|
||||||
schema: {type: integer, minimum: 0, default: 0}
|
|
||||||
- name: limit
|
|
||||||
in: query
|
|
||||||
required: false
|
|
||||||
description: 单页记录数;默认 100,最大 1000。
|
|
||||||
schema: {type: integer, minimum: 1, maximum: 1000, default: 100}
|
|
||||||
responses:
|
|
||||||
'200':
|
|
||||||
description: 管理面审计记录页
|
|
||||||
headers:
|
|
||||||
X-Request-ID: {$ref: '#/components/headers/RequestID'}
|
|
||||||
content:
|
|
||||||
application/json:
|
|
||||||
schema: {$ref: '#/components/schemas/AuditPage'}
|
|
||||||
'401': {$ref: '#/components/responses/Unauthorized'}
|
|
||||||
'403': {$ref: '#/components/responses/Forbidden'}
|
|
||||||
'400': {$ref: '#/components/responses/BadRequest'}
|
|
||||||
'405': {$ref: '#/components/responses/MethodNotAllowed'}
|
|
||||||
'500': {$ref: '#/components/responses/InternalServerError'}
|
|
||||||
'503': {$ref: '#/components/responses/ServiceUnavailable'}
|
|
||||||
/api/v1/upstreams/{name}/enable:
|
/api/v1/upstreams/{name}/enable:
|
||||||
post:
|
post:
|
||||||
tags: [Upstreams]
|
tags: [Upstreams]
|
||||||
operationId: enableUpstream
|
operationId: enableUpstream
|
||||||
summary: 启用 Upstream
|
summary: 启用 Upstream
|
||||||
description: 需要 `admin:write`。
|
|
||||||
parameters:
|
parameters:
|
||||||
- $ref: '#/components/parameters/UpstreamName'
|
- $ref: '#/components/parameters/UpstreamName'
|
||||||
- $ref: '#/components/parameters/RequestID'
|
- $ref: '#/components/parameters/RequestID'
|
||||||
@ -108,7 +59,6 @@ paths:
|
|||||||
tags: [Upstreams]
|
tags: [Upstreams]
|
||||||
operationId: disableUpstream
|
operationId: disableUpstream
|
||||||
summary: 禁用 Upstream 并使已有资源自然 Drain
|
summary: 禁用 Upstream 并使已有资源自然 Drain
|
||||||
description: 需要 `admin:write`。
|
|
||||||
parameters:
|
parameters:
|
||||||
- $ref: '#/components/parameters/UpstreamName'
|
- $ref: '#/components/parameters/UpstreamName'
|
||||||
- $ref: '#/components/parameters/RequestID'
|
- $ref: '#/components/parameters/RequestID'
|
||||||
@ -127,7 +77,6 @@ paths:
|
|||||||
tags: [Routing]
|
tags: [Routing]
|
||||||
operationId: switchRouting
|
operationId: switchRouting
|
||||||
summary: 原子切换 Sequential Routing 当前 Upstream
|
summary: 原子切换 Sequential Routing 当前 Upstream
|
||||||
description: 需要 `admin:write`。
|
|
||||||
parameters:
|
parameters:
|
||||||
- name: name
|
- name: name
|
||||||
in: path
|
in: path
|
||||||
@ -164,7 +113,6 @@ paths:
|
|||||||
tags: [Configuration]
|
tags: [Configuration]
|
||||||
operationId: reloadConfiguration
|
operationId: reloadConfiguration
|
||||||
summary: 严格校验并原子发布新配置快照
|
summary: 严格校验并原子发布新配置快照
|
||||||
description: 需要 `admin:write`。
|
|
||||||
parameters:
|
parameters:
|
||||||
- $ref: '#/components/parameters/RequestID'
|
- $ref: '#/components/parameters/RequestID'
|
||||||
responses:
|
responses:
|
||||||
@ -204,31 +152,6 @@ components:
|
|||||||
description: 服务端最终使用的请求标识。
|
description: 服务端最终使用的请求标识。
|
||||||
schema: {type: string, maxLength: 128}
|
schema: {type: string, maxLength: 128}
|
||||||
schemas:
|
schemas:
|
||||||
AuditPage:
|
|
||||||
type: object
|
|
||||||
additionalProperties: false
|
|
||||||
required: [records]
|
|
||||||
properties:
|
|
||||||
records:
|
|
||||||
type: array
|
|
||||||
description: 按 ID 升序的审计记录;没有更多记录时为空数组。
|
|
||||||
items: {$ref: '#/components/schemas/AuditRecord'}
|
|
||||||
AuditRecord:
|
|
||||||
type: object
|
|
||||||
additionalProperties: false
|
|
||||||
required: [id, requestId, actorId, action, resourceType, resourceName, changed, version, occurredAt]
|
|
||||||
properties:
|
|
||||||
id: {type: integer, minimum: 0}
|
|
||||||
requestId: {type: string}
|
|
||||||
actorId: {type: string}
|
|
||||||
sourceIp: {type: string}
|
|
||||||
action: {type: string}
|
|
||||||
resourceType: {type: string}
|
|
||||||
resourceName: {type: string}
|
|
||||||
changed: {type: boolean}
|
|
||||||
version: {type: integer, minimum: 0}
|
|
||||||
reason: {type: string}
|
|
||||||
occurredAt: {type: string, format: date-time}
|
|
||||||
Status:
|
Status:
|
||||||
type: object
|
type: object
|
||||||
additionalProperties: false
|
additionalProperties: false
|
||||||
|
|||||||
@ -8,9 +8,8 @@ import (
|
|||||||
)
|
)
|
||||||
|
|
||||||
type document struct {
|
type document struct {
|
||||||
OpenAPI string `yaml:"openapi"`
|
OpenAPI string `yaml:"openapi"`
|
||||||
Paths map[string]map[string]any `yaml:"paths"`
|
Paths map[string]map[string]any `yaml:"paths"`
|
||||||
Components map[string]any `yaml:"components"`
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestDistributionContract(t *testing.T) {
|
func TestDistributionContract(t *testing.T) {
|
||||||
@ -38,7 +37,6 @@ func TestAdminContract(t *testing.T) {
|
|||||||
spec := readDocument(t, "admin.yaml")
|
spec := readDocument(t, "admin.yaml")
|
||||||
for _, path := range []string{
|
for _, path := range []string{
|
||||||
"/api/v1/status",
|
"/api/v1/status",
|
||||||
"/api/v1/audit",
|
|
||||||
"/api/v1/upstreams/{name}/enable",
|
"/api/v1/upstreams/{name}/enable",
|
||||||
"/api/v1/upstreams/{name}/disable",
|
"/api/v1/upstreams/{name}/disable",
|
||||||
"/api/v1/routing/{name}/switch",
|
"/api/v1/routing/{name}/switch",
|
||||||
@ -50,67 +48,6 @@ func TestAdminContract(t *testing.T) {
|
|||||||
}
|
}
|
||||||
requireResponses(t, spec.Paths["/api/v1/routing/{name}/switch"]["post"],
|
requireResponses(t, spec.Paths["/api/v1/routing/{name}/switch"]["post"],
|
||||||
"200", "400", "401", "403", "404", "405", "409", "413", "415", "422", "500", "503")
|
"200", "400", "401", "403", "404", "405", "409", "413", "415", "422", "500", "503")
|
||||||
requireResponses(t, spec.Paths["/api/v1/audit"]["get"],
|
|
||||||
"200", "400", "401", "403", "405", "500", "503")
|
|
||||||
requireAuditPaginationContract(t, spec.Paths["/api/v1/audit"]["get"])
|
|
||||||
requireAuditSchemas(t, spec)
|
|
||||||
}
|
|
||||||
|
|
||||||
func requireAuditPaginationContract(t *testing.T, operation any) {
|
|
||||||
t.Helper()
|
|
||||||
operationMap, ok := operation.(map[string]any)
|
|
||||||
if !ok {
|
|
||||||
t.Fatalf("audit operation has type %T, want map", operation)
|
|
||||||
}
|
|
||||||
parameters, ok := operationMap["parameters"].([]any)
|
|
||||||
if !ok {
|
|
||||||
t.Fatalf("audit parameters has type %T, want array", operationMap["parameters"])
|
|
||||||
}
|
|
||||||
afterID := parameterSchema(t, parameters, "afterId")
|
|
||||||
if afterID["type"] != "integer" || afterID["minimum"] != 0 || afterID["default"] != 0 {
|
|
||||||
t.Errorf("afterId schema = %#v, want integer with minimum/default 0", afterID)
|
|
||||||
}
|
|
||||||
limit := parameterSchema(t, parameters, "limit")
|
|
||||||
if limit["type"] != "integer" || limit["minimum"] != 1 || limit["maximum"] != 1000 || limit["default"] != 100 {
|
|
||||||
t.Errorf("limit schema = %#v, want integer [1, 1000] with default 100", limit)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func parameterSchema(t *testing.T, parameters []any, name string) map[string]any {
|
|
||||||
t.Helper()
|
|
||||||
for _, raw := range parameters {
|
|
||||||
parameter, ok := raw.(map[string]any)
|
|
||||||
if !ok || parameter["name"] != name || parameter["in"] != "query" {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
schema, ok := parameter["schema"].(map[string]any)
|
|
||||||
if !ok {
|
|
||||||
t.Fatalf("%s parameter schema has type %T, want map", name, parameter["schema"])
|
|
||||||
}
|
|
||||||
return schema
|
|
||||||
}
|
|
||||||
t.Fatalf("query parameter %q is missing", name)
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func requireAuditSchemas(t *testing.T, spec document) {
|
|
||||||
t.Helper()
|
|
||||||
schemas, ok := spec.Components["schemas"].(map[string]any)
|
|
||||||
if !ok {
|
|
||||||
t.Fatal("components.schemas is missing")
|
|
||||||
}
|
|
||||||
for _, name := range []string{"AuditPage", "AuditRecord"} {
|
|
||||||
if _, ok := schemas[name].(map[string]any); !ok {
|
|
||||||
t.Errorf("schema %s is missing", name)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
record, _ := schemas["AuditRecord"].(map[string]any)
|
|
||||||
properties, _ := record["properties"].(map[string]any)
|
|
||||||
for _, forbidden := range []string{"proxy", "upstreamUrl", "credential", "activePool"} {
|
|
||||||
if _, exists := properties[forbidden]; exists {
|
|
||||||
t.Errorf("AuditRecord must not expose %q", forbidden)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func requireResponses(t *testing.T, operation any, codes ...string) {
|
func requireResponses(t *testing.T, operation any, codes ...string) {
|
||||||
|
|||||||
@ -7,18 +7,13 @@ info:
|
|||||||
Redis operation atomically removes or marks every returned proxy as
|
Redis operation atomically removes or marks every returned proxy as
|
||||||
extracted in the TTL activity pool. Proxy details and per-extraction
|
extracted in the TTL activity pool. Proxy details and per-extraction
|
||||||
records are not persisted in PostgreSQL. There is no release, renew, or
|
records are not persisted in PostgreSQL. There is no release, renew, or
|
||||||
lease API. An authenticated credential may be limited to the fixed
|
lease API.
|
||||||
`distribution:extract` permission; legacy credentials without configured
|
|
||||||
permissions retain full access. Credential client policy may further bound
|
|
||||||
a request's count, upstreams, and regions before Redis extraction.
|
|
||||||
servers:
|
servers:
|
||||||
- url: http://127.0.0.1:8081
|
- url: http://127.0.0.1:8081
|
||||||
description: Distribution API
|
description: Distribution API
|
||||||
tags:
|
tags:
|
||||||
- name: Distribution
|
- name: Distribution
|
||||||
description: 一次性独占提取上游代理的受保护接口。
|
|
||||||
- name: Health
|
- name: Health
|
||||||
description: Distribution 监听器的存活与就绪探针。
|
|
||||||
paths:
|
paths:
|
||||||
/api/v1/proxies/extract:
|
/api/v1/proxies/extract:
|
||||||
post:
|
post:
|
||||||
@ -26,8 +21,6 @@ paths:
|
|||||||
operationId: extractProxies
|
operationId: extractProxies
|
||||||
summary: 一次性独占提取代理
|
summary: 一次性独占提取代理
|
||||||
description: |
|
description: |
|
||||||
需要 `distribution:extract`。
|
|
||||||
|
|
||||||
服务端通过单个 Redis 原子操作完成筛选、TTL/所有权复核、Gateway 预留、
|
服务端通过单个 Redis 原子操作完成筛选、TTL/所有权复核、Gateway 预留、
|
||||||
从可分配池移除所选条目和短期幂等结果写入,再返回代理。相同代理不会
|
从可分配池移除所选条目和短期幂等结果写入,再返回代理。相同代理不会
|
||||||
返回给两个成功请求,提取路径不访问 PostgreSQL。
|
返回给两个成功请求,提取路径不访问 PostgreSQL。
|
||||||
@ -35,8 +28,7 @@ paths:
|
|||||||
`partial` 允许实际返回数量小于请求数量;`allOrNothing` 数量不足时不
|
`partial` 允许实际返回数量小于请求数量;`allOrNothing` 数量不足时不
|
||||||
提取任何代理并返回 409。未传 `fulfillment` 时使用服务端配置,默认
|
提取任何代理并返回 409。未传 `fulfillment` 时使用服务端配置,默认
|
||||||
为 `partial`。`Idempotency-Key` 在 Redis 的有界 TTL 窗口内避免客户端因
|
为 `partial`。`Idempotency-Key` 在 Redis 的有界 TTL 窗口内避免客户端因
|
||||||
响应丢失重试而再次消耗库存。命中凭据的 Client policy 可以额外限制
|
响应丢失重试而再次消耗库存。
|
||||||
count、allowedUpstreams 和 regions;越过该边界返回 403,未调用 Redis。
|
|
||||||
security:
|
security:
|
||||||
- ApiKeyAuth: []
|
- ApiKeyAuth: []
|
||||||
- BasicAuth: []
|
- BasicAuth: []
|
||||||
|
|||||||
@ -1,5 +0,0 @@
|
|||||||
extends:
|
|
||||||
- minimal
|
|
||||||
|
|
||||||
rules:
|
|
||||||
tag-description: error
|
|
||||||
@ -26,31 +26,6 @@ func TestOpenAPIDocumentsHaveClosedContracts(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestOpenAPITagsHaveDescriptions(t *testing.T) {
|
|
||||||
for _, name := range []string{"proxy-pool.yaml", "admin.yaml"} {
|
|
||||||
name := name
|
|
||||||
t.Run(name, func(t *testing.T) {
|
|
||||||
root := readOpenAPIRoot(t, name)
|
|
||||||
tags, ok := root["tags"].([]any)
|
|
||||||
if !ok || len(tags) == 0 {
|
|
||||||
t.Fatal("tags must be a non-empty array")
|
|
||||||
}
|
|
||||||
for _, rawTag := range tags {
|
|
||||||
tag, ok := rawTag.(map[string]any)
|
|
||||||
if !ok {
|
|
||||||
t.Errorf("tag has type %T, want object", rawTag)
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
name, _ := tag["name"].(string)
|
|
||||||
description, _ := tag["description"].(string)
|
|
||||||
if strings.TrimSpace(description) == "" {
|
|
||||||
t.Errorf("tag %q has no description", name)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func validateLocalReferences(t *testing.T, root map[string]any) {
|
func validateLocalReferences(t *testing.T, root map[string]any) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
walkOpenAPI(root, func(path string, value any) {
|
walkOpenAPI(root, func(path string, value any) {
|
||||||
|
|||||||
@ -99,7 +99,6 @@ message RoutingRule {
|
|||||||
RoutingStrategy strategy = 8;
|
RoutingStrategy strategy = 8;
|
||||||
UnavailableAction on_unavailable = 9;
|
UnavailableAction on_unavailable = 9;
|
||||||
google.protobuf.Duration wait_timeout = 10;
|
google.protobuf.Duration wait_timeout = 10;
|
||||||
RoutingAction action = 11;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
message RoutingStrategy {
|
message RoutingStrategy {
|
||||||
@ -124,13 +123,6 @@ enum UnavailableAction {
|
|||||||
UNAVAILABLE_ACTION_DIRECT = 3;
|
UNAVAILABLE_ACTION_DIRECT = 3;
|
||||||
}
|
}
|
||||||
|
|
||||||
enum RoutingAction {
|
|
||||||
ROUTING_ACTION_UNSPECIFIED = 0;
|
|
||||||
ROUTING_ACTION_PROXY = 1;
|
|
||||||
ROUTING_ACTION_DIRECT = 2;
|
|
||||||
ROUTING_ACTION_REJECT = 3;
|
|
||||||
}
|
|
||||||
|
|
||||||
message OwnedProxy {
|
message OwnedProxy {
|
||||||
string id = 1;
|
string id = 1;
|
||||||
string upstream = 2;
|
string upstream = 2;
|
||||||
|
|||||||
@ -15,7 +15,6 @@ import (
|
|||||||
controlplanev1 "proxy-pool/gen/controlplane/v1"
|
controlplanev1 "proxy-pool/gen/controlplane/v1"
|
||||||
"proxy-pool/internal/checker/bootstrap"
|
"proxy-pool/internal/checker/bootstrap"
|
||||||
"proxy-pool/internal/config"
|
"proxy-pool/internal/config"
|
||||||
"proxy-pool/internal/platform/logging"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
const (
|
const (
|
||||||
@ -23,7 +22,6 @@ const (
|
|||||||
controlPlaneAddressEnvironment = "PROXY_POOL_CONTROL_PLANE_ADDRESS"
|
controlPlaneAddressEnvironment = "PROXY_POOL_CONTROL_PLANE_ADDRESS"
|
||||||
checkerIDEnvironment = "PROXY_POOL_CHECKER_ID"
|
checkerIDEnvironment = "PROXY_POOL_CHECKER_ID"
|
||||||
instanceIDEnvironment = "PROXY_POOL_CHECKER_INSTANCE_ID"
|
instanceIDEnvironment = "PROXY_POOL_CHECKER_INSTANCE_ID"
|
||||||
autoIdentityEnvironment = "PROXY_POOL_AUTO_IDENTITY"
|
|
||||||
maxInFlightEnvironment = "PROXY_POOL_CHECKER_MAX_IN_FLIGHT"
|
maxInFlightEnvironment = "PROXY_POOL_CHECKER_MAX_IN_FLIGHT"
|
||||||
)
|
)
|
||||||
|
|
||||||
@ -43,7 +41,6 @@ func execute(ctx context.Context, args []string, getenv environmentLookup, run c
|
|||||||
controlPlaneAddress := flags.String("control-plane", "", "remote Controller control-plane address")
|
controlPlaneAddress := flags.String("control-plane", "", "remote Controller control-plane address")
|
||||||
checkerID := flags.String("checker-id", "", "unique Checker identifier")
|
checkerID := flags.String("checker-id", "", "unique Checker identifier")
|
||||||
instanceID := flags.String("instance-id", "", "unique Checker process instance identifier")
|
instanceID := flags.String("instance-id", "", "unique Checker process instance identifier")
|
||||||
autoIdentity := flags.Bool("auto-identity", false, "derive Checker and missing instance IDs from the mTLS SPIFFE certificate")
|
|
||||||
maxInFlight := flags.Int("max-in-flight", 0, "maximum concurrent tasks")
|
maxInFlight := flags.Int("max-in-flight", 0, "maximum concurrent tasks")
|
||||||
levels := flags.String("levels", "basic,egress,target", "supported levels: basic,egress,target")
|
levels := flags.String("levels", "basic,egress,target", "supported levels: basic,egress,target")
|
||||||
if err := flags.Parse(args); err != nil {
|
if err := flags.Parse(args); err != nil {
|
||||||
@ -66,45 +63,28 @@ func execute(ctx context.Context, args []string, getenv environmentLookup, run c
|
|||||||
*maxInFlight = value
|
*maxInFlight = value
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if !flagWasSet(flags, "auto-identity") {
|
|
||||||
if value, err := strconv.ParseBool(getenv(autoIdentityEnvironment)); err == nil {
|
|
||||||
*autoIdentity = value
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
supportedLevels, err := parseLevels(*levels)
|
supportedLevels, err := parseLevels(*levels)
|
||||||
missingIdentity := !*autoIdentity && (!validValue(*checkerID) || !validValue(*instanceID))
|
|
||||||
invalidProvidedIdentity := (*checkerID != "" && !validValue(*checkerID)) || (*instanceID != "" && !validValue(*instanceID))
|
|
||||||
if ctx == nil || run == nil || err != nil || !validValue(*configPath) || !validValue(*controlPlaneAddress) ||
|
if ctx == nil || run == nil || err != nil || !validValue(*configPath) || !validValue(*controlPlaneAddress) ||
|
||||||
missingIdentity || invalidProvidedIdentity || *maxInFlight <= 0 {
|
!validValue(*checkerID) || !validValue(*instanceID) || *maxInFlight <= 0 {
|
||||||
_, _ = fmt.Fprintf(stderr,
|
_, _ = fmt.Fprintf(stderr,
|
||||||
"proxy-checker: -config, -control-plane and positive -max-in-flight are required; -checker-id/-instance-id are required unless -auto-identity is enabled; "+
|
"proxy-checker: -config, -control-plane, -checker-id, -instance-id and positive -max-in-flight are required; "+
|
||||||
"environment fallbacks: %s, %s, %s, %s, %s, %s\n",
|
"environment fallbacks: %s, %s, %s, %s, %s\n",
|
||||||
configEnvironment, controlPlaneAddressEnvironment, checkerIDEnvironment, instanceIDEnvironment, maxInFlightEnvironment, autoIdentityEnvironment,
|
configEnvironment, controlPlaneAddressEnvironment, checkerIDEnvironment, instanceIDEnvironment, maxInFlightEnvironment,
|
||||||
)
|
)
|
||||||
return 2
|
return 2
|
||||||
}
|
}
|
||||||
err = run(ctx, bootstrap.Options{
|
err = run(ctx, bootstrap.Options{
|
||||||
ConfigPath: *configPath, Resolver: config.OSResolver{}, ControlPlaneAddress: *controlPlaneAddress,
|
ConfigPath: *configPath, Resolver: config.OSResolver{}, ControlPlaneAddress: *controlPlaneAddress,
|
||||||
CheckerID: *checkerID, InstanceID: *instanceID, AutoIdentity: *autoIdentity, MaxInFlight: *maxInFlight, SupportedLevels: supportedLevels,
|
CheckerID: *checkerID, InstanceID: *instanceID, MaxInFlight: *maxInFlight, SupportedLevels: supportedLevels,
|
||||||
})
|
})
|
||||||
if err == nil || (errors.Is(err, context.Canceled) && ctx.Err() != nil) {
|
if err == nil || (errors.Is(err, context.Canceled) && ctx.Err() != nil) {
|
||||||
return 0
|
return 0
|
||||||
}
|
}
|
||||||
logging.WriteProcessError(stderr, "proxy-checker", err)
|
_, _ = fmt.Fprintf(stderr, "proxy-checker: %v\n", err)
|
||||||
return 1
|
return 1
|
||||||
}
|
}
|
||||||
|
|
||||||
func flagWasSet(flags *flag.FlagSet, name string) bool {
|
|
||||||
set := false
|
|
||||||
flags.Visit(func(item *flag.Flag) {
|
|
||||||
if item.Name == name {
|
|
||||||
set = true
|
|
||||||
}
|
|
||||||
})
|
|
||||||
return set
|
|
||||||
}
|
|
||||||
|
|
||||||
func parseLevels(value string) ([]controlplanev1.CheckLevel, error) {
|
func parseLevels(value string) ([]controlplanev1.CheckLevel, error) {
|
||||||
levels := make([]controlplanev1.CheckLevel, 0, 3)
|
levels := make([]controlplanev1.CheckLevel, 0, 3)
|
||||||
seen := make(map[controlplanev1.CheckLevel]struct{})
|
seen := make(map[controlplanev1.CheckLevel]struct{})
|
||||||
|
|||||||
@ -24,21 +24,6 @@ func TestExecuteUsesFlagsAndPassesCheckerIdentity(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestExecutePassesAutoCheckerIdentityFromEnvironment(t *testing.T) {
|
|
||||||
received := bootstrap.Options{}
|
|
||||||
values := map[string]string{
|
|
||||||
configEnvironment: "checker.yaml", controlPlaneAddressEnvironment: "127.0.0.1:8443",
|
|
||||||
maxInFlightEnvironment: "2", autoIdentityEnvironment: "true",
|
|
||||||
}
|
|
||||||
code := execute(context.Background(), nil, func(name string) string { return values[name] }, func(_ context.Context, options bootstrap.Options) error {
|
|
||||||
received = options
|
|
||||||
return nil
|
|
||||||
}, io.Discard)
|
|
||||||
if code != 0 || !received.AutoIdentity || received.CheckerID != "" || received.InstanceID != "" {
|
|
||||||
t.Fatalf("execute(auto identity) = (%d, %+v)", code, received)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestExecuteRejectsInvalidLevelSet(t *testing.T) {
|
func TestExecuteRejectsInvalidLevelSet(t *testing.T) {
|
||||||
code := execute(context.Background(), []string{
|
code := execute(context.Background(), []string{
|
||||||
"-config", "config.yaml", "-control-plane", "127.0.0.1:8443", "-checker-id", "checker-a", "-instance-id", "instance-a",
|
"-config", "config.yaml", "-control-plane", "127.0.0.1:8443", "-checker-id", "checker-a", "-instance-id", "instance-a",
|
||||||
|
|||||||
@ -13,7 +13,6 @@ import (
|
|||||||
|
|
||||||
"proxy-pool/internal/config"
|
"proxy-pool/internal/config"
|
||||||
"proxy-pool/internal/controller/bootstrap"
|
"proxy-pool/internal/controller/bootstrap"
|
||||||
"proxy-pool/internal/platform/logging"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
const (
|
const (
|
||||||
@ -68,6 +67,6 @@ func execute(
|
|||||||
if err == nil || (errors.Is(err, context.Canceled) && ctx.Err() != nil) {
|
if err == nil || (errors.Is(err, context.Canceled) && ctx.Err() != nil) {
|
||||||
return 0
|
return 0
|
||||||
}
|
}
|
||||||
logging.WriteProcessError(stderr, "proxy-controller", err)
|
_, _ = fmt.Fprintf(stderr, "proxy-controller: %v\n", err)
|
||||||
return 1
|
return 1
|
||||||
}
|
}
|
||||||
|
|||||||
@ -75,9 +75,7 @@ func TestExecuteMapsStartupFailureAndSignalCancellation(t *testing.T) {
|
|||||||
) error {
|
) error {
|
||||||
return want
|
return want
|
||||||
}, &stderr)
|
}, &stderr)
|
||||||
if code != 1 || !bytes.Contains(stderr.Bytes(), []byte(`"msg":"process failed"`)) ||
|
if code != 1 || !bytes.Contains(stderr.Bytes(), []byte(want.Error())) {
|
||||||
!bytes.Contains(stderr.Bytes(), []byte(`"component":"proxy-controller"`)) ||
|
|
||||||
bytes.Contains(stderr.Bytes(), []byte(want.Error())) {
|
|
||||||
t.Fatalf("execute(startup failure) = %d, stderr = %q", code, stderr.String())
|
t.Fatalf("execute(startup failure) = %d, stderr = %q", code, stderr.String())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@ -8,13 +8,11 @@ import (
|
|||||||
"io"
|
"io"
|
||||||
"os"
|
"os"
|
||||||
"os/signal"
|
"os/signal"
|
||||||
"strconv"
|
|
||||||
"strings"
|
"strings"
|
||||||
"syscall"
|
"syscall"
|
||||||
|
|
||||||
"proxy-pool/internal/config"
|
"proxy-pool/internal/config"
|
||||||
"proxy-pool/internal/gateway/bootstrap"
|
"proxy-pool/internal/gateway/bootstrap"
|
||||||
"proxy-pool/internal/platform/logging"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
const (
|
const (
|
||||||
@ -23,7 +21,6 @@ const (
|
|||||||
clusterIDEnvironment = "PROXY_POOL_CLUSTER_ID"
|
clusterIDEnvironment = "PROXY_POOL_CLUSTER_ID"
|
||||||
workerIDEnvironment = "PROXY_POOL_WORKER_ID"
|
workerIDEnvironment = "PROXY_POOL_WORKER_ID"
|
||||||
instanceIDEnvironment = "PROXY_POOL_INSTANCE_ID"
|
instanceIDEnvironment = "PROXY_POOL_INSTANCE_ID"
|
||||||
autoIdentityEnvironment = "PROXY_POOL_AUTO_IDENTITY"
|
|
||||||
zoneEnvironment = "PROXY_POOL_ZONE"
|
zoneEnvironment = "PROXY_POOL_ZONE"
|
||||||
)
|
)
|
||||||
|
|
||||||
@ -50,7 +47,6 @@ func execute(
|
|||||||
clusterID := flags.String("cluster-id", "", "cluster identifier")
|
clusterID := flags.String("cluster-id", "", "cluster identifier")
|
||||||
workerID := flags.String("worker-id", "", "unique Worker identifier")
|
workerID := flags.String("worker-id", "", "unique Worker identifier")
|
||||||
instanceID := flags.String("instance-id", "", "unique process instance identifier")
|
instanceID := flags.String("instance-id", "", "unique process instance identifier")
|
||||||
autoIdentity := flags.Bool("auto-identity", false, "derive Worker and missing instance IDs from the mTLS SPIFFE certificate")
|
|
||||||
zone := flags.String("zone", "", "availability zone identifier")
|
zone := flags.String("zone", "", "availability zone identifier")
|
||||||
if err := flags.Parse(args); err != nil {
|
if err := flags.Parse(args); err != nil {
|
||||||
if errors.Is(err, flag.ErrHelp) {
|
if errors.Is(err, flag.ErrHelp) {
|
||||||
@ -69,44 +65,27 @@ func execute(
|
|||||||
setIfEmpty(workerID, getenv(workerIDEnvironment))
|
setIfEmpty(workerID, getenv(workerIDEnvironment))
|
||||||
setIfEmpty(instanceID, getenv(instanceIDEnvironment))
|
setIfEmpty(instanceID, getenv(instanceIDEnvironment))
|
||||||
setIfEmpty(zone, getenv(zoneEnvironment))
|
setIfEmpty(zone, getenv(zoneEnvironment))
|
||||||
if !flagWasSet(flags, "auto-identity") {
|
|
||||||
if value, err := strconv.ParseBool(getenv(autoIdentityEnvironment)); err == nil {
|
|
||||||
*autoIdentity = value
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
missingIdentity := !*autoIdentity && (!validValue(*workerID) || !validValue(*instanceID))
|
|
||||||
invalidProvidedIdentity := (*workerID != "" && !validValue(*workerID)) || (*instanceID != "" && !validValue(*instanceID))
|
|
||||||
if ctx == nil || run == nil || !validValue(*configPath) || !validValue(*controlPlaneAddress) ||
|
if ctx == nil || run == nil || !validValue(*configPath) || !validValue(*controlPlaneAddress) ||
|
||||||
!validValue(*clusterID) || missingIdentity || invalidProvidedIdentity || !validValue(*zone) {
|
!validValue(*clusterID) || !validValue(*workerID) || !validValue(*instanceID) || !validValue(*zone) {
|
||||||
_, _ = fmt.Fprintf(stderr,
|
_, _ = fmt.Fprintf(stderr,
|
||||||
"proxy-gateway: -config, -control-plane, -cluster-id and -zone are required; -worker-id/-instance-id are required unless -auto-identity is enabled; "+
|
"proxy-gateway: -config, -control-plane, -cluster-id, -worker-id, -instance-id and -zone are required; "+
|
||||||
"environment fallbacks: %s, %s, %s, %s, %s, %s, %s\n",
|
"environment fallbacks: %s, %s, %s, %s, %s, %s\n",
|
||||||
configEnvironment, controlPlaneAddressEnvironment, clusterIDEnvironment, workerIDEnvironment, instanceIDEnvironment, zoneEnvironment, autoIdentityEnvironment,
|
configEnvironment, controlPlaneAddressEnvironment, clusterIDEnvironment, workerIDEnvironment, instanceIDEnvironment, zoneEnvironment,
|
||||||
)
|
)
|
||||||
return 2
|
return 2
|
||||||
}
|
}
|
||||||
err := run(ctx, bootstrap.Options{
|
err := run(ctx, bootstrap.Options{
|
||||||
ConfigPath: *configPath, Resolver: config.OSResolver{}, ControlPlaneAddress: *controlPlaneAddress,
|
ConfigPath: *configPath, Resolver: config.OSResolver{}, ControlPlaneAddress: *controlPlaneAddress,
|
||||||
ClusterID: *clusterID, WorkerID: *workerID, InstanceID: *instanceID, AutoIdentity: *autoIdentity, Zone: *zone,
|
ClusterID: *clusterID, WorkerID: *workerID, InstanceID: *instanceID, Zone: *zone,
|
||||||
})
|
})
|
||||||
if err == nil || (errors.Is(err, context.Canceled) && ctx.Err() != nil) {
|
if err == nil || (errors.Is(err, context.Canceled) && ctx.Err() != nil) {
|
||||||
return 0
|
return 0
|
||||||
}
|
}
|
||||||
logging.WriteProcessError(stderr, "proxy-gateway", err)
|
_, _ = fmt.Fprintf(stderr, "proxy-gateway: %v\n", err)
|
||||||
return 1
|
return 1
|
||||||
}
|
}
|
||||||
|
|
||||||
func flagWasSet(flags *flag.FlagSet, name string) bool {
|
|
||||||
set := false
|
|
||||||
flags.Visit(func(item *flag.Flag) {
|
|
||||||
if item.Name == name {
|
|
||||||
set = true
|
|
||||||
}
|
|
||||||
})
|
|
||||||
return set
|
|
||||||
}
|
|
||||||
|
|
||||||
func setIfEmpty(target *string, value string) {
|
func setIfEmpty(target *string, value string) {
|
||||||
if target != nil && *target == "" {
|
if target != nil && *target == "" {
|
||||||
*target = value
|
*target = value
|
||||||
|
|||||||
@ -55,21 +55,6 @@ func TestExecuteReadsGatewayOptionsFromEnvironment(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestExecutePassesAutoGatewayIdentityFromEnvironment(t *testing.T) {
|
|
||||||
captured := bootstrap.Options{}
|
|
||||||
values := map[string]string{
|
|
||||||
configEnvironment: "gateway.yaml", controlPlaneAddressEnvironment: "127.0.0.1:8443",
|
|
||||||
clusterIDEnvironment: "cluster-a", zoneEnvironment: "zone-a", autoIdentityEnvironment: "true",
|
|
||||||
}
|
|
||||||
code := execute(context.Background(), nil, func(name string) string { return values[name] }, func(_ context.Context, options bootstrap.Options) error {
|
|
||||||
captured = options
|
|
||||||
return nil
|
|
||||||
}, io.Discard)
|
|
||||||
if code != 0 || !captured.AutoIdentity || captured.WorkerID != "" || captured.InstanceID != "" {
|
|
||||||
t.Fatalf("execute(auto identity) = (%d, %+v)", code, captured)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestExecuteRejectsIncompleteGatewayOptions(t *testing.T) {
|
func TestExecuteRejectsIncompleteGatewayOptions(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
|
|||||||
@ -7,11 +7,9 @@ import (
|
|||||||
"flag"
|
"flag"
|
||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
"math"
|
|
||||||
"net/http"
|
"net/http"
|
||||||
"os"
|
"os"
|
||||||
"os/signal"
|
"os/signal"
|
||||||
"path/filepath"
|
|
||||||
"strings"
|
"strings"
|
||||||
"syscall"
|
"syscall"
|
||||||
"time"
|
"time"
|
||||||
@ -43,9 +41,6 @@ func execute(ctx context.Context, args []string, run loadRun, stdout, stderr io.
|
|||||||
hold := flags.Duration("hold", 0, "CONNECT tunnel hold duration; required by -scenario connect")
|
hold := flags.Duration("hold", 0, "CONNECT tunnel hold duration; required by -scenario connect")
|
||||||
extractCount := flags.Int("extract-count", 1, "proxies requested by each extract scenario request")
|
extractCount := flags.Int("extract-count", 1, "proxies requested by each extract scenario request")
|
||||||
extractFulfillment := flags.String("extract-fulfillment", "partial", "extract fulfillment: partial or allOrNothing")
|
extractFulfillment := flags.String("extract-fulfillment", "partial", "extract fulfillment: partial or allOrNothing")
|
||||||
maxErrorRate := flags.Float64("max-error-rate", -1, "optional maximum failed/completed rate from 0 to 1")
|
|
||||||
maxP99 := flags.Duration("max-p99", 0, "optional maximum p99 latency")
|
|
||||||
outputPath := flags.String("output", "", "optional JSON report file; replaces only after a complete write")
|
|
||||||
var headers headerValues
|
var headers headerValues
|
||||||
flags.Var(&headers, "header", "repeatable HTTP header in Name: Value form")
|
flags.Var(&headers, "header", "repeatable HTTP header in Name: Value form")
|
||||||
if err := flags.Parse(args); err != nil {
|
if err := flags.Parse(args); err != nil {
|
||||||
@ -58,15 +53,6 @@ func execute(ctx context.Context, args []string, run loadRun, stdout, stderr io.
|
|||||||
_, _ = fmt.Fprintln(stderr, "proxy-loadgen: target and a bounded workload are required")
|
_, _ = fmt.Fprintln(stderr, "proxy-loadgen: target and a bounded workload are required")
|
||||||
return 2
|
return 2
|
||||||
}
|
}
|
||||||
if strings.TrimSpace(*outputPath) != *outputPath {
|
|
||||||
_, _ = fmt.Fprintln(stderr, "proxy-loadgen: invalid output path")
|
|
||||||
return 2
|
|
||||||
}
|
|
||||||
criteria, configured, err := acceptanceCriteria(*maxErrorRate, *maxP99)
|
|
||||||
if err != nil {
|
|
||||||
_, _ = fmt.Fprintf(stderr, "proxy-loadgen: %v\n", err)
|
|
||||||
return 2
|
|
||||||
}
|
|
||||||
parsedHeaders, err := headers.Header()
|
parsedHeaders, err := headers.Header()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
_, _ = fmt.Fprintf(stderr, "proxy-loadgen: %v\n", err)
|
_, _ = fmt.Fprintf(stderr, "proxy-loadgen: %v\n", err)
|
||||||
@ -86,80 +72,13 @@ func execute(ctx context.Context, args []string, run loadRun, stdout, stderr io.
|
|||||||
_, _ = fmt.Fprintf(stderr, "proxy-loadgen: %v\n", err)
|
_, _ = fmt.Fprintf(stderr, "proxy-loadgen: %v\n", err)
|
||||||
return 1
|
return 1
|
||||||
}
|
}
|
||||||
if configured {
|
|
||||||
acceptance, acceptanceErr := loadgen.EvaluateAcceptance(report, criteria)
|
|
||||||
if acceptanceErr != nil {
|
|
||||||
_, _ = fmt.Fprintf(stderr, "proxy-loadgen: %v\n", acceptanceErr)
|
|
||||||
return 2
|
|
||||||
}
|
|
||||||
report.Acceptance = &acceptance
|
|
||||||
}
|
|
||||||
if err := writeReportFile(*outputPath, report); err != nil {
|
|
||||||
_, _ = fmt.Fprintf(stderr, "proxy-loadgen: write report: %v\n", err)
|
|
||||||
return 1
|
|
||||||
}
|
|
||||||
if err := json.NewEncoder(stdout).Encode(report); err != nil {
|
if err := json.NewEncoder(stdout).Encode(report); err != nil {
|
||||||
_, _ = fmt.Fprintf(stderr, "proxy-loadgen: encode report: %v\n", err)
|
_, _ = fmt.Fprintf(stderr, "proxy-loadgen: encode report: %v\n", err)
|
||||||
return 1
|
return 1
|
||||||
}
|
}
|
||||||
if report.Acceptance != nil && !report.Acceptance.Passed {
|
|
||||||
_, _ = fmt.Fprintln(stderr, "proxy-loadgen: workload acceptance thresholds were not met")
|
|
||||||
return 3
|
|
||||||
}
|
|
||||||
return 0
|
return 0
|
||||||
}
|
}
|
||||||
|
|
||||||
func acceptanceCriteria(maxErrorRate float64, maxP99 time.Duration) (loadgen.AcceptanceCriteria, bool, error) {
|
|
||||||
if math.IsNaN(maxErrorRate) || math.IsInf(maxErrorRate, 0) || maxErrorRate < -1 || maxErrorRate > 1 || maxP99 < 0 {
|
|
||||||
return loadgen.AcceptanceCriteria{}, false, loadgen.ErrInvalidAcceptanceCriteria
|
|
||||||
}
|
|
||||||
criteria := loadgen.AcceptanceCriteria{
|
|
||||||
EnforceFailureRate: maxErrorRate >= 0,
|
|
||||||
MaxFailureRate: maxErrorRate,
|
|
||||||
EnforceP99Latency: maxP99 > 0,
|
|
||||||
MaxP99Latency: maxP99,
|
|
||||||
}
|
|
||||||
return criteria, criteria.EnforceFailureRate || criteria.EnforceP99Latency, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func writeReportFile(path string, report loadgen.Report) error {
|
|
||||||
if path == "" {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
content, err := json.Marshal(report)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
content = append(content, '\n')
|
|
||||||
directory := filepath.Dir(path)
|
|
||||||
temporary, err := os.CreateTemp(directory, "."+filepath.Base(path)+".tmp-*")
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
temporaryPath := temporary.Name()
|
|
||||||
defer os.Remove(temporaryPath)
|
|
||||||
if err := temporary.Chmod(0o600); err != nil {
|
|
||||||
_ = temporary.Close()
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if _, err := temporary.Write(content); err != nil {
|
|
||||||
_ = temporary.Close()
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if err := temporary.Close(); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
// Prefer same-directory rename so a completed report replaces the old one
|
|
||||||
// atomically on platforms that support replacement.
|
|
||||||
if err := os.Rename(temporaryPath, path); err == nil {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
if err := os.Remove(path); err != nil && !os.IsNotExist(err) {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
return os.Rename(temporaryPath, path)
|
|
||||||
}
|
|
||||||
|
|
||||||
type headerValues []string
|
type headerValues []string
|
||||||
|
|
||||||
func (values *headerValues) String() string {
|
func (values *headerValues) String() string {
|
||||||
|
|||||||
@ -4,9 +4,6 @@ import (
|
|||||||
"bytes"
|
"bytes"
|
||||||
"context"
|
"context"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"math"
|
|
||||||
"os"
|
|
||||||
"path/filepath"
|
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
@ -21,10 +18,7 @@ func TestExecutePassesBoundedWorkloadAndWritesJSON(t *testing.T) {
|
|||||||
"-requests", "3", "-concurrency", "2", "-timeout", "2s", "-header", "X-Run: fixed", "-body", `{"count":1}`,
|
"-requests", "3", "-concurrency", "2", "-timeout", "2s", "-header", "X-Run: fixed", "-body", `{"count":1}`,
|
||||||
}, func(_ context.Context, options loadgen.Options) (loadgen.Report, error) {
|
}, func(_ context.Context, options loadgen.Options) (loadgen.Report, error) {
|
||||||
received = options
|
received = options
|
||||||
return loadgen.Report{
|
return loadgen.Report{Requests: 3, Completed: 3, Succeeded: 3, Duration: time.Second}, nil
|
||||||
Requests: 3, Completed: 3, Succeeded: 3, Duration: time.Second,
|
|
||||||
RateStartsGenerated: 3, RateStartsDropped: 1,
|
|
||||||
}, nil
|
|
||||||
}, &stdout, &stderr)
|
}, &stdout, &stderr)
|
||||||
if code != 0 || received.TargetURL != "https://target.example/path" || received.ProxyURL != "http://gateway.example:8080" ||
|
if code != 0 || received.TargetURL != "https://target.example/path" || received.ProxyURL != "http://gateway.example:8080" ||
|
||||||
received.Method != "post" || received.Requests != 3 || received.Duration != 0 || received.Concurrency != 2 ||
|
received.Method != "post" || received.Requests != 3 || received.Duration != 0 || received.Concurrency != 2 ||
|
||||||
@ -32,8 +26,7 @@ func TestExecutePassesBoundedWorkloadAndWritesJSON(t *testing.T) {
|
|||||||
t.Fatalf("execute() = %d; options=%+v stderr=%q", code, received, stderr.String())
|
t.Fatalf("execute() = %d; options=%+v stderr=%q", code, received, stderr.String())
|
||||||
}
|
}
|
||||||
var report loadgen.Report
|
var report loadgen.Report
|
||||||
if err := json.Unmarshal(stdout.Bytes(), &report); err != nil || report.Succeeded != 3 ||
|
if err := json.Unmarshal(stdout.Bytes(), &report); err != nil || report.Succeeded != 3 {
|
||||||
report.RateStartsGenerated != 3 || report.RateStartsDropped != 1 {
|
|
||||||
t.Fatalf("JSON report = (%+v, %v)", report, err)
|
t.Fatalf("JSON report = (%+v, %v)", report, err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -77,85 +70,3 @@ func TestExecutePassesExtractScenario(t *testing.T) {
|
|||||||
t.Fatalf("execute() = %d; options=%+v stderr=%q", code, received, stderr.String())
|
t.Fatalf("execute() = %d; options=%+v stderr=%q", code, received, stderr.String())
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestExecuteWritesReportFile(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
path := filepath.Join(t.TempDir(), "load-report.json")
|
|
||||||
if err := os.WriteFile(path, []byte("incomplete"), 0o600); err != nil {
|
|
||||||
t.Fatalf("WriteFile(): %v", err)
|
|
||||||
}
|
|
||||||
var stdout, stderr bytes.Buffer
|
|
||||||
code := execute(context.Background(), []string{
|
|
||||||
"-target", "https://target.example/health", "-requests", "1", "-concurrency", "1", "-output", path,
|
|
||||||
}, func(_ context.Context, _ loadgen.Options) (loadgen.Report, error) {
|
|
||||||
return loadgen.Report{Requests: 1, Completed: 1, Succeeded: 1, Duration: time.Second}, nil
|
|
||||||
}, &stdout, &stderr)
|
|
||||||
if code != 0 || stderr.Len() != 0 {
|
|
||||||
t.Fatalf("execute() = %d, stderr=%q", code, stderr.String())
|
|
||||||
}
|
|
||||||
content, err := os.ReadFile(path)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("ReadFile(): %v", err)
|
|
||||||
}
|
|
||||||
var report loadgen.Report
|
|
||||||
if err := json.Unmarshal(content, &report); err != nil || report.Succeeded != 1 || string(content) == "incomplete" {
|
|
||||||
t.Fatalf("report file = (%q, %+v, %v)", content, report, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestExecuteWritesReportBeforeReturningAcceptanceFailure(t *testing.T) {
|
|
||||||
path := filepath.Join(t.TempDir(), "load-report.json")
|
|
||||||
var stdout, stderr bytes.Buffer
|
|
||||||
code := execute(context.Background(), []string{
|
|
||||||
"-target", "https://target.example/health", "-requests", "10", "-concurrency", "1",
|
|
||||||
"-max-error-rate", "0.1", "-max-p99", "100ms", "-output", path,
|
|
||||||
}, func(_ context.Context, _ loadgen.Options) (loadgen.Report, error) {
|
|
||||||
return loadgen.Report{
|
|
||||||
Requests: 10, Completed: 10, Failed: 2, Duration: time.Second,
|
|
||||||
Latency: loadgen.LatencyReport{Samples: 10, P99UpperBound: 250 * time.Millisecond},
|
|
||||||
}, nil
|
|
||||||
}, &stdout, &stderr)
|
|
||||||
if code != 3 || stderr.Len() == 0 {
|
|
||||||
t.Fatalf("execute() = %d, stderr=%q", code, stderr.String())
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, content := range [][]byte{stdout.Bytes(), mustReadFile(t, path)} {
|
|
||||||
var report loadgen.Report
|
|
||||||
if err := json.Unmarshal(content, &report); err != nil || report.Acceptance == nil || report.Acceptance.Passed ||
|
|
||||||
len(report.Acceptance.Violations) != 2 {
|
|
||||||
t.Fatalf("acceptance report = (%+v, %v)", report, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestExecuteRejectsInvalidAcceptanceThresholdBeforeRunningWorkload(t *testing.T) {
|
|
||||||
var stdout, stderr bytes.Buffer
|
|
||||||
called := false
|
|
||||||
code := execute(context.Background(), []string{
|
|
||||||
"-target", "https://target.example/health", "-requests", "1", "-max-error-rate", "1.1",
|
|
||||||
}, func(_ context.Context, _ loadgen.Options) (loadgen.Report, error) {
|
|
||||||
called = true
|
|
||||||
return loadgen.Report{}, nil
|
|
||||||
}, &stdout, &stderr)
|
|
||||||
if code != 2 || called || stdout.Len() != 0 || stderr.Len() == 0 {
|
|
||||||
t.Fatalf("execute() = %d, called=%t stdout=%q stderr=%q", code, called, stdout.String(), stderr.String())
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestAcceptanceCriteriaRejectsNonFiniteErrorRate(t *testing.T) {
|
|
||||||
for _, value := range []float64{math.NaN(), math.Inf(1)} {
|
|
||||||
if _, _, err := acceptanceCriteria(value, 0); err == nil {
|
|
||||||
t.Fatalf("acceptanceCriteria(%v) accepted a non-finite error rate", value)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func mustReadFile(t *testing.T, path string) []byte {
|
|
||||||
t.Helper()
|
|
||||||
content, err := os.ReadFile(path)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("ReadFile(%q): %v", path, err)
|
|
||||||
}
|
|
||||||
return content
|
|
||||||
}
|
|
||||||
|
|||||||
@ -5,24 +5,10 @@ Grafana 与 Kubernetes。镜像会构建 `proxy-controller`、`proxy-gateway`
|
|||||||
`proxy-checker`;Controller 已装配 Redis BASIC 检查任务,Checker 可通过认证
|
`proxy-checker`;Controller 已装配 Redis BASIC 检查任务,Checker 可通过认证
|
||||||
控制面执行 HTTP/HTTPS BASIC 探测。
|
控制面执行 HTTP/HTTPS BASIC 探测。
|
||||||
|
|
||||||
Compose 使用 `config/local.yaml` 启用 Controller、两个 Gateway 和一个 Checker 的
|
默认 Compose 和 Kubernetes Base 仍将 `controlPlane.enabled` 保持为 `false`。
|
||||||
mTLS 控制面。`generate-local-controlplane-certs.ps1` 为三个固定工作负载生成不同的
|
跨节点控制面必须由环境 Overlay 提供 Controller 可拨号地址、独立 Worker/Checker
|
||||||
SPIFFE URI 证书,私钥只写入被忽略的 `deploy/.control-plane-tls/`。运行前先执行:
|
身份及 mTLS 证书,因此这些默认清单不能视为完整的生产发布配置。`checker.yaml`
|
||||||
|
保留为 Overlay 模板,尚未加入 Base。
|
||||||
```powershell
|
|
||||||
./scripts/generate-local-controlplane-certs.ps1
|
|
||||||
docker compose -f deploy/docker-compose.yml up -d --build
|
|
||||||
```
|
|
||||||
|
|
||||||
Kubernetes Base 仍保持 `controlPlane.enabled: false`。`kubernetes/overlays/development-mtls`
|
|
||||||
启用单副本 Controller/Gateway/Checker mTLS 拓扑,挂载三组外部 TLS Secret,并从
|
|
||||||
`gateway-a`、`checker-a` 的证书 URI 自动派生身份。具体创建 Secret 和部署步骤见
|
|
||||||
[development mTLS overlay](kubernetes/overlays/development-mtls/README.md)。生产 Overlay
|
|
||||||
必须为每个弹性 Gateway/Checker 工作负载配置唯一身份与证书轮换,不能复用固定开发证书。
|
|
||||||
身份系统只需把每 Pod 的叶证书、私钥和信任包写入配置指定路径,并签发唯一
|
|
||||||
`.../worker/<worker-id>` 或 `.../checker/<checker-id>` URI;启动命令设置
|
|
||||||
`-auto-identity` 后由进程派生逻辑/缺省实例 ID,不依赖手工环境变量。开发 Compose 和
|
|
||||||
此 Overlay 已启用该模式;其中证书只适用于本地开发,不代表生产身份签发与挂载实现。
|
|
||||||
|
|
||||||
当前可执行验证:
|
当前可执行验证:
|
||||||
|
|
||||||
@ -30,7 +16,6 @@ Kubernetes Base 仍保持 `controlPlane.enabled: false`。`kubernetes/overlays/d
|
|||||||
docker compose -f deploy/docker-compose.yml config --quiet
|
docker compose -f deploy/docker-compose.yml config --quiet
|
||||||
docker compose -f deploy/docker-compose.test.yml config --quiet
|
docker compose -f deploy/docker-compose.test.yml config --quiet
|
||||||
kubectl kustomize deploy/kubernetes/base | Out-Null
|
kubectl kustomize deploy/kubernetes/base | Out-Null
|
||||||
kubectl kustomize deploy/kubernetes/overlays/development-mtls | Out-Null
|
|
||||||
go run ./deploy/tools/configcheck deploy/config/local.yaml
|
go run ./deploy/tools/configcheck deploy/config/local.yaml
|
||||||
```
|
```
|
||||||
|
|
||||||
@ -41,8 +26,3 @@ PostgreSQL Adapter 和对应执行脚本完成前,不把数据库契约记为
|
|||||||
|
|
||||||
发布前仍须通过 `production-readiness.md` 中的一致性、安全、恢复、竞态与容量门禁,
|
发布前仍须通过 `production-readiness.md` 中的一致性、安全、恢复、竞态与容量门禁,
|
||||||
并完成代表性集群容量验证。
|
并完成代表性集群容量验证。
|
||||||
|
|
||||||
Grafana Overview 与 Prometheus 规则只引用代码已注册的低基数指标。它们覆盖 Gateway
|
|
||||||
请求/端到端时延/Outcome、Controller 容量、Provider、Extraction、Checker 和 Drain;不按 Proxy、IP、
|
|
||||||
Client、Upstream、Worker、Session 或完整 URL 聚合。`go test ./deploy` 会解析两类资产并拒绝
|
|
||||||
不存在的指标与禁止标签,指标改名或新增面板时必须同步更新该契约。
|
|
||||||
|
|||||||
@ -1,66 +0,0 @@
|
|||||||
package deploy
|
|
||||||
|
|
||||||
import (
|
|
||||||
"os"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"go.yaml.in/yaml/v4"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestCIValidatesProtocolAndDeploymentAssets(t *testing.T) {
|
|
||||||
payload, err := os.ReadFile("../.github/workflows/ci.yml")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("read ci workflow: %v", err)
|
|
||||||
}
|
|
||||||
workflow := string(payload)
|
|
||||||
var document map[string]any
|
|
||||||
if err := yaml.Unmarshal(payload, &document); err != nil {
|
|
||||||
t.Fatalf("parse ci workflow: %v", err)
|
|
||||||
}
|
|
||||||
jobs, ok := document["jobs"].(map[string]any)
|
|
||||||
if !ok {
|
|
||||||
t.Fatal("ci workflow has no jobs")
|
|
||||||
}
|
|
||||||
for _, name := range []string{"openapi", "proto", "image", "deployment", "test", "race", "integration"} {
|
|
||||||
if _, exists := jobs[name]; !exists {
|
|
||||||
t.Errorf("ci workflow has no %s job", name)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
for _, required := range []string{
|
|
||||||
"./scripts/verify-openapi.ps1",
|
|
||||||
"./scripts/verify-proto.ps1",
|
|
||||||
"docker build --file deploy/docker/Dockerfile --tag proxy-pool:ci .",
|
|
||||||
"docker compose -f deploy/docker-compose.yml config --quiet",
|
|
||||||
"docker compose -f deploy/docker-compose.test.yml config --quiet",
|
|
||||||
"go run ./deploy/tools/configcheck deploy/config/local.yaml",
|
|
||||||
"kubectl kustomize deploy/kubernetes/base",
|
|
||||||
"kubectl kustomize deploy/kubernetes/overlays/development-mtls",
|
|
||||||
"go run ./deploy/tools/configcheck deploy/kubernetes/overlays/development-mtls/config.yaml",
|
|
||||||
} {
|
|
||||||
if !strings.Contains(workflow, required) {
|
|
||||||
t.Errorf("ci workflow does not validate %q", required)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
deployment, ok := jobs["deployment"].(map[string]any)
|
|
||||||
if !ok {
|
|
||||||
t.Fatal("ci deployment job has invalid definition")
|
|
||||||
}
|
|
||||||
steps, ok := deployment["steps"].([]any)
|
|
||||||
if !ok {
|
|
||||||
t.Fatal("ci deployment job has no steps")
|
|
||||||
}
|
|
||||||
for _, step := range steps {
|
|
||||||
definition, ok := step.(map[string]any)
|
|
||||||
if !ok || definition["uses"] != "actions/setup-go@v5" {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
options, ok := definition["with"].(map[string]any)
|
|
||||||
if !ok || options["go-version-file"] != "go.mod" {
|
|
||||||
t.Error("ci deployment job must pin actions/setup-go to go.mod")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
return
|
|
||||||
}
|
|
||||||
t.Error("ci deployment job must install the Go version declared by go.mod before running configcheck")
|
|
||||||
}
|
|
||||||
@ -1,7 +1,6 @@
|
|||||||
package deploy
|
package deploy
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"encoding/json"
|
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"regexp"
|
"regexp"
|
||||||
@ -12,63 +11,18 @@ import (
|
|||||||
"go.yaml.in/yaml/v4"
|
"go.yaml.in/yaml/v4"
|
||||||
)
|
)
|
||||||
|
|
||||||
var observableMetricNames = map[string]struct{}{
|
|
||||||
"proxy_pool_checker_observations_total": {},
|
|
||||||
"proxy_pool_checker_tasks_dispatched_total": {},
|
|
||||||
"proxy_pool_controller_capacity_active_upstreams": {},
|
|
||||||
"proxy_pool_controller_capacity_available_slots": {},
|
|
||||||
"proxy_pool_controller_capacity_effective_slots": {},
|
|
||||||
"proxy_pool_controller_capacity_inventory_reads_total": {},
|
|
||||||
"proxy_pool_controller_capacity_managed_proxies": {},
|
|
||||||
"proxy_pool_controller_capacity_pending_expected_proxies": {},
|
|
||||||
"proxy_pool_controller_drain_candidates_total": {},
|
|
||||||
"proxy_pool_controller_drains_started_total": {},
|
|
||||||
"proxy_pool_controller_extraction_requested_proxies_total": {},
|
|
||||||
"proxy_pool_controller_extraction_requests_total": {},
|
|
||||||
"proxy_pool_controller_extraction_returned_proxies_total": {},
|
|
||||||
"proxy_pool_controller_provider_fetch_results_total": {},
|
|
||||||
"proxy_pool_controller_provider_new_proxies_total": {},
|
|
||||||
"proxy_pool_controller_provider_valid_candidates_total": {},
|
|
||||||
"proxy_pool_gateway_active_tunnels": {},
|
|
||||||
"proxy_pool_gateway_capacity_invariant_violations_total": {},
|
|
||||||
"proxy_pool_gateway_outcome_queue_dropped_total": {},
|
|
||||||
"proxy_pool_gateway_outcomes_total": {},
|
|
||||||
"proxy_pool_gateway_request_duration_seconds": {},
|
|
||||||
"proxy_pool_gateway_requests_in_flight": {},
|
|
||||||
"proxy_pool_gateway_requests_total": {},
|
|
||||||
}
|
|
||||||
|
|
||||||
type dashboardDocument struct {
|
|
||||||
Panels []struct {
|
|
||||||
Targets []struct {
|
|
||||||
Expression string `json:"expr"`
|
|
||||||
} `json:"targets"`
|
|
||||||
} `json:"panels"`
|
|
||||||
}
|
|
||||||
|
|
||||||
type alertRulesDocument struct {
|
|
||||||
Groups []struct {
|
|
||||||
Rules []struct {
|
|
||||||
Expression string `yaml:"expr"`
|
|
||||||
Alert string `yaml:"alert"`
|
|
||||||
} `yaml:"rules"`
|
|
||||||
} `yaml:"groups"`
|
|
||||||
}
|
|
||||||
|
|
||||||
type composeDocument struct {
|
type composeDocument struct {
|
||||||
Services map[string]composeService `yaml:"services"`
|
Services map[string]composeService `yaml:"services"`
|
||||||
Volumes map[string]any `yaml:"volumes"`
|
Volumes map[string]any `yaml:"volumes"`
|
||||||
}
|
}
|
||||||
|
|
||||||
type composeService struct {
|
type composeService struct {
|
||||||
Image string `yaml:"image"`
|
Image string `yaml:"image"`
|
||||||
Command []string `yaml:"command"`
|
Command []string `yaml:"command"`
|
||||||
Ports []string `yaml:"ports"`
|
Ports []string `yaml:"ports"`
|
||||||
Expose []string `yaml:"expose"`
|
Volumes []string `yaml:"volumes"`
|
||||||
Volumes []string `yaml:"volumes"`
|
Tmpfs []string `yaml:"tmpfs"`
|
||||||
Tmpfs []string `yaml:"tmpfs"`
|
DependsOn any `yaml:"depends_on"`
|
||||||
Environment map[string]string `yaml:"environment"`
|
|
||||||
DependsOn any `yaml:"depends_on"`
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestPostgresIntegrationFixtureIsIsolatedAndEphemeral(t *testing.T) {
|
func TestPostgresIntegrationFixtureIsIsolatedAndEphemeral(t *testing.T) {
|
||||||
@ -179,65 +133,6 @@ func TestLocalGatewaysDoNotDependOnControlPlaneStorage(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestLocalComposeProvidesAuthenticatedControlPlaneForEveryWorker(t *testing.T) {
|
|
||||||
document := loadComposeDocument(t)
|
|
||||||
controller := document.Services["controller"]
|
|
||||||
if !slices.Contains(controller.Expose, "8443") || !slices.Contains(controller.Volumes, "./.control-plane-tls/controller:/run/proxy-pool-tls/server:ro") {
|
|
||||||
t.Fatalf("controller does not expose and mount control-plane TLS: expose=%v volumes=%v", controller.Expose, controller.Volumes)
|
|
||||||
}
|
|
||||||
for _, expected := range []struct {
|
|
||||||
name string
|
|
||||||
certificate string
|
|
||||||
}{
|
|
||||||
{name: "gateway-a", certificate: "gateway-a"},
|
|
||||||
{name: "gateway-b", certificate: "gateway-b"},
|
|
||||||
{name: "checker", certificate: "checker-a"},
|
|
||||||
} {
|
|
||||||
service, ok := document.Services[expected.name]
|
|
||||||
if !ok {
|
|
||||||
t.Fatalf("docker-compose.yml has no %s", expected.name)
|
|
||||||
}
|
|
||||||
if service.Environment["PROXY_POOL_CONTROL_PLANE_ADDRESS"] != "controller:8443" ||
|
|
||||||
!slices.Contains(service.Volumes, "./.control-plane-tls/"+expected.certificate+":/run/proxy-pool-tls/client:ro") ||
|
|
||||||
!composeDependsOn(service.DependsOn, "controller") {
|
|
||||||
t.Errorf("%s control-plane wiring = env=%v volumes=%v dependsOn=%v", expected.name, service.Environment, service.Volumes, service.DependsOn)
|
|
||||||
}
|
|
||||||
if expected.name == "checker" {
|
|
||||||
if service.Environment["PROXY_POOL_AUTO_IDENTITY"] != "true" ||
|
|
||||||
service.Environment["PROXY_POOL_CHECKER_ID"] != "" ||
|
|
||||||
service.Environment["PROXY_POOL_CHECKER_INSTANCE_ID"] != "" ||
|
|
||||||
service.Environment["PROXY_POOL_CHECKER_MAX_IN_FLIGHT"] != "200" || !slices.Contains(service.Expose, "9090") {
|
|
||||||
t.Errorf("checker identity or metrics wiring = env=%v expose=%v", service.Environment, service.Expose)
|
|
||||||
}
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
if service.Environment["PROXY_POOL_AUTO_IDENTITY"] != "true" ||
|
|
||||||
service.Environment["PROXY_POOL_WORKER_ID"] != "" ||
|
|
||||||
service.Environment["PROXY_POOL_INSTANCE_ID"] != "" ||
|
|
||||||
service.Environment["PROXY_POOL_CLUSTER_ID"] != "compose-local" || service.Environment["PROXY_POOL_ZONE"] != "compose-local" {
|
|
||||||
t.Errorf("%s identity wiring = %v", expected.name, service.Environment)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestLocalControlPlaneCertificateGeneratorUsesDistinctSPIFFERoles(t *testing.T) {
|
|
||||||
payload, err := os.ReadFile("../scripts/generate-local-controlplane-certs.ps1")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("read generate-local-controlplane-certs.ps1: %v", err)
|
|
||||||
}
|
|
||||||
script := string(payload)
|
|
||||||
for _, required := range []string{
|
|
||||||
"OutputDirectory must be empty", "DNS:controller",
|
|
||||||
"spiffe://proxy-pool.local/development/worker/gateway-a",
|
|
||||||
"spiffe://proxy-pool.local/development/worker/gateway-b",
|
|
||||||
"spiffe://proxy-pool.local/development/checker/checker-a",
|
|
||||||
} {
|
|
||||||
if !strings.Contains(script, required) {
|
|
||||||
t.Errorf("certificate generator missing %q", required)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestDeploymentEntrypointsExistInSource(t *testing.T) {
|
func TestDeploymentEntrypointsExistInSource(t *testing.T) {
|
||||||
dockerfile, err := os.ReadFile("docker/Dockerfile")
|
dockerfile, err := os.ReadFile("docker/Dockerfile")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@ -265,93 +160,13 @@ func TestDeploymentEntrypointsExistInSource(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestDockerBuildContextExcludesLocalSecretsAndArtifacts(t *testing.T) {
|
func TestKubernetesBaseExcludesUnimplementedProcessManifests(t *testing.T) {
|
||||||
payload, err := os.ReadFile("../.dockerignore")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("read .dockerignore: %v", err)
|
|
||||||
}
|
|
||||||
rules := make(map[string]struct{})
|
|
||||||
for _, raw := range strings.Split(string(payload), "\n") {
|
|
||||||
rule := strings.TrimSpace(raw)
|
|
||||||
if rule == "" || strings.HasPrefix(rule, "#") {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
rules[rule] = struct{}{}
|
|
||||||
}
|
|
||||||
for _, required := range []string{
|
|
||||||
".git", ".env", ".env.*", "*.key", "*.pem", "*.crt",
|
|
||||||
"deploy/.control-plane-tls/", "bin/", "dist/", "coverage/", ".tmp-proto/",
|
|
||||||
} {
|
|
||||||
if _, exists := rules[required]; !exists {
|
|
||||||
t.Errorf(".dockerignore does not exclude %q", required)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestKubernetesBaseLeavesControlPlaneClientsForMTLSOverlay(t *testing.T) {
|
|
||||||
payload, err := os.ReadFile("kubernetes/base/kustomization.yaml")
|
payload, err := os.ReadFile("kubernetes/base/kustomization.yaml")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("read kustomization.yaml: %v", err)
|
t.Fatalf("read kustomization.yaml: %v", err)
|
||||||
}
|
}
|
||||||
if strings.Contains(string(payload), "checker.yaml") {
|
if strings.Contains(string(payload), "checker.yaml") {
|
||||||
t.Fatal("kubernetes base includes checker.yaml without per-workload mTLS identity overlay")
|
t.Fatal("kubernetes base includes checker.yaml before proxy-checker exists")
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestObservabilityAssetsUseRegisteredLowCardinalityMetrics(t *testing.T) {
|
|
||||||
dashboardPayload, err := os.ReadFile("grafana/dashboards/proxy-pool-overview.json")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("read overview dashboard: %v", err)
|
|
||||||
}
|
|
||||||
var dashboard dashboardDocument
|
|
||||||
if err := json.Unmarshal(dashboardPayload, &dashboard); err != nil {
|
|
||||||
t.Fatalf("parse overview dashboard: %v", err)
|
|
||||||
}
|
|
||||||
expressions := make([]string, 0, len(dashboard.Panels))
|
|
||||||
for _, panel := range dashboard.Panels {
|
|
||||||
for _, target := range panel.Targets {
|
|
||||||
expressions = append(expressions, target.Expression)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
rulesPayload, err := os.ReadFile("prometheus/rules/proxy-pool.yml")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("read Prometheus rules: %v", err)
|
|
||||||
}
|
|
||||||
var rules alertRulesDocument
|
|
||||||
if err := yaml.Unmarshal(rulesPayload, &rules); err != nil {
|
|
||||||
t.Fatalf("parse Prometheus rules: %v", err)
|
|
||||||
}
|
|
||||||
alerts := make([]string, 0)
|
|
||||||
for _, group := range rules.Groups {
|
|
||||||
for _, rule := range group.Rules {
|
|
||||||
expressions = append(expressions, rule.Expression)
|
|
||||||
alerts = append(alerts, rule.Alert)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
metricPattern := regexp.MustCompile(`proxy_pool_[a-z0-9_]+`)
|
|
||||||
forbiddenLabel := regexp.MustCompile(`(?:by\s*\([^)]*\b(?:upstream|worker)\b|\{[^}]*\b(?:upstream|worker)\s*=)`)
|
|
||||||
for _, expression := range expressions {
|
|
||||||
for _, name := range metricPattern.FindAllString(expression, -1) {
|
|
||||||
baseName := strings.TrimSuffix(strings.TrimSuffix(strings.TrimSuffix(name, "_bucket"), "_count"), "_sum")
|
|
||||||
if _, exists := observableMetricNames[baseName]; !exists {
|
|
||||||
t.Errorf("observability expression references unregistered metric %q: %s", name, expression)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if forbiddenLabel.MatchString(expression) {
|
|
||||||
t.Errorf("observability expression uses a forbidden high-cardinality label: %s", expression)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
for _, required := range []string{
|
|
||||||
"proxy_pool_gateway_requests_total", "proxy_pool_gateway_request_duration_seconds", "proxy_pool_controller_capacity_available_slots",
|
|
||||||
"proxy_pool_controller_provider_fetch_results_total", "proxy_pool_controller_extraction_requests_total",
|
|
||||||
"proxy_pool_checker_observations_total",
|
|
||||||
} {
|
|
||||||
if !strings.Contains(strings.Join(expressions, "\n"), required) {
|
|
||||||
t.Errorf("observability assets do not cover %s", required)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if !slices.Contains(alerts, "ProxyPoolGatewayHighLatency") {
|
|
||||||
t.Error("Prometheus rules do not alert on Gateway p99 latency")
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@ -16,11 +16,6 @@ gateway:
|
|||||||
limits:
|
limits:
|
||||||
maxConcurrentConnections: 20000
|
maxConcurrentConnections: 20000
|
||||||
requestsPerMinutePerClient: 60000
|
requestsPerMinutePerClient: 60000
|
||||||
transport:
|
|
||||||
maxIdleConns: 20000
|
|
||||||
maxIdleConnsPerHost: 32
|
|
||||||
maxConnsPerHost: 32
|
|
||||||
tunnelBufferBytes: 32768
|
|
||||||
retry:
|
retry:
|
||||||
maxAttempts: 2
|
maxAttempts: 2
|
||||||
retryMethods: [GET, HEAD]
|
retryMethods: [GET, HEAD]
|
||||||
@ -64,30 +59,7 @@ admin:
|
|||||||
token: "${PROXY_POOL_ADMIN_TOKEN}"
|
token: "${PROXY_POOL_ADMIN_TOKEN}"
|
||||||
|
|
||||||
controlPlane:
|
controlPlane:
|
||||||
enabled: true
|
enabled: false
|
||||||
listen: 0.0.0.0:8443
|
|
||||||
protocolVersion: 1
|
|
||||||
heartbeatInterval: 10s
|
|
||||||
sessionTTL: 30s
|
|
||||||
maxStaleAge: 30s
|
|
||||||
maxMessageBytes: 4194304
|
|
||||||
maxRuntimeCounters: 100000
|
|
||||||
maxConcurrentStreams: 1000
|
|
||||||
tls:
|
|
||||||
mode: mtls
|
|
||||||
certFile: /run/proxy-pool-tls/server/tls.crt
|
|
||||||
keyFile: /run/proxy-pool-tls/server/tls.key
|
|
||||||
clientCAFile: /run/proxy-pool-tls/server/ca.crt
|
|
||||||
trustDomain: proxy-pool.local
|
|
||||||
environment: development
|
|
||||||
gatewayTLS:
|
|
||||||
certFile: /run/proxy-pool-tls/client/tls.crt
|
|
||||||
keyFile: /run/proxy-pool-tls/client/tls.key
|
|
||||||
serverCAFile: /run/proxy-pool-tls/client/ca.crt
|
|
||||||
checkerTLS:
|
|
||||||
certFile: /run/proxy-pool-tls/client/tls.crt
|
|
||||||
keyFile: /run/proxy-pool-tls/client/tls.key
|
|
||||||
serverCAFile: /run/proxy-pool-tls/client/ca.crt
|
|
||||||
|
|
||||||
metrics:
|
metrics:
|
||||||
enabled: true
|
enabled: true
|
||||||
|
|||||||
@ -15,6 +15,8 @@ x-app: &app
|
|||||||
image: proxy-pool:local
|
image: proxy-pool:local
|
||||||
restart: unless-stopped
|
restart: unless-stopped
|
||||||
networks: [frontend, backend]
|
networks: [frontend, backend]
|
||||||
|
volumes:
|
||||||
|
- ./config/local.yaml:/etc/proxy-pool/config.yaml:ro
|
||||||
environment: *app-environment
|
environment: *app-environment
|
||||||
stop_grace_period: 45s
|
stop_grace_period: 45s
|
||||||
|
|
||||||
@ -22,18 +24,6 @@ services:
|
|||||||
gateway-a:
|
gateway-a:
|
||||||
<<: *app
|
<<: *app
|
||||||
command: ["proxy-gateway"]
|
command: ["proxy-gateway"]
|
||||||
environment:
|
|
||||||
<<: *app-environment
|
|
||||||
PROXY_POOL_CONTROL_PLANE_ADDRESS: controller:8443
|
|
||||||
PROXY_POOL_CLUSTER_ID: compose-local
|
|
||||||
PROXY_POOL_AUTO_IDENTITY: "true"
|
|
||||||
PROXY_POOL_ZONE: compose-local
|
|
||||||
volumes:
|
|
||||||
- ./config/local.yaml:/etc/proxy-pool/config.yaml:ro
|
|
||||||
- ./.control-plane-tls/gateway-a:/run/proxy-pool-tls/client:ro
|
|
||||||
depends_on:
|
|
||||||
controller:
|
|
||||||
condition: service_healthy
|
|
||||||
expose: ["8080", "9090"]
|
expose: ["8080", "9090"]
|
||||||
healthcheck:
|
healthcheck:
|
||||||
test: ["CMD", "curl", "--fail", "--silent", "http://127.0.0.1:9090/readyz"]
|
test: ["CMD", "curl", "--fail", "--silent", "http://127.0.0.1:9090/readyz"]
|
||||||
@ -45,18 +35,6 @@ services:
|
|||||||
gateway-b:
|
gateway-b:
|
||||||
<<: *app
|
<<: *app
|
||||||
command: ["proxy-gateway"]
|
command: ["proxy-gateway"]
|
||||||
environment:
|
|
||||||
<<: *app-environment
|
|
||||||
PROXY_POOL_CONTROL_PLANE_ADDRESS: controller:8443
|
|
||||||
PROXY_POOL_CLUSTER_ID: compose-local
|
|
||||||
PROXY_POOL_AUTO_IDENTITY: "true"
|
|
||||||
PROXY_POOL_ZONE: compose-local
|
|
||||||
volumes:
|
|
||||||
- ./config/local.yaml:/etc/proxy-pool/config.yaml:ro
|
|
||||||
- ./.control-plane-tls/gateway-b:/run/proxy-pool-tls/client:ro
|
|
||||||
depends_on:
|
|
||||||
controller:
|
|
||||||
condition: service_healthy
|
|
||||||
expose: ["8080", "9090"]
|
expose: ["8080", "9090"]
|
||||||
healthcheck:
|
healthcheck:
|
||||||
test: ["CMD", "curl", "--fail", "--silent", "http://127.0.0.1:9090/readyz"]
|
test: ["CMD", "curl", "--fail", "--silent", "http://127.0.0.1:9090/readyz"]
|
||||||
@ -71,15 +49,12 @@ services:
|
|||||||
environment:
|
environment:
|
||||||
<<: *app-environment
|
<<: *app-environment
|
||||||
PROXY_POOL_CONFIG_FINGERPRINT_KEY: ${PROXY_POOL_CONFIG_FINGERPRINT_KEY:?set PROXY_POOL_CONFIG_FINGERPRINT_KEY}
|
PROXY_POOL_CONFIG_FINGERPRINT_KEY: ${PROXY_POOL_CONFIG_FINGERPRINT_KEY:?set PROXY_POOL_CONFIG_FINGERPRINT_KEY}
|
||||||
volumes:
|
|
||||||
- ./config/local.yaml:/etc/proxy-pool/config.yaml:ro
|
|
||||||
- ./.control-plane-tls/controller:/run/proxy-pool-tls/server:ro
|
|
||||||
depends_on:
|
depends_on:
|
||||||
postgres:
|
postgres:
|
||||||
condition: service_healthy
|
condition: service_healthy
|
||||||
redis:
|
redis:
|
||||||
condition: service_healthy
|
condition: service_healthy
|
||||||
expose: ["8081", "8082", "8443", "9090"]
|
expose: ["8081", "8082", "9090"]
|
||||||
ports:
|
ports:
|
||||||
- "127.0.0.1:8081:8081"
|
- "127.0.0.1:8081:8081"
|
||||||
- "127.0.0.1:8082:8082"
|
- "127.0.0.1:8082:8082"
|
||||||
@ -90,28 +65,6 @@ services:
|
|||||||
retries: 12
|
retries: 12
|
||||||
start_period: 15s
|
start_period: 15s
|
||||||
|
|
||||||
checker:
|
|
||||||
<<: *app
|
|
||||||
command: ["proxy-checker"]
|
|
||||||
environment:
|
|
||||||
<<: *app-environment
|
|
||||||
PROXY_POOL_CONTROL_PLANE_ADDRESS: controller:8443
|
|
||||||
PROXY_POOL_AUTO_IDENTITY: "true"
|
|
||||||
PROXY_POOL_CHECKER_MAX_IN_FLIGHT: "200"
|
|
||||||
volumes:
|
|
||||||
- ./config/local.yaml:/etc/proxy-pool/config.yaml:ro
|
|
||||||
- ./.control-plane-tls/checker-a:/run/proxy-pool-tls/client:ro
|
|
||||||
depends_on:
|
|
||||||
controller:
|
|
||||||
condition: service_healthy
|
|
||||||
expose: ["9090"]
|
|
||||||
healthcheck:
|
|
||||||
test: ["CMD", "curl", "--fail", "--silent", "http://127.0.0.1:9090/readyz"]
|
|
||||||
interval: 5s
|
|
||||||
timeout: 2s
|
|
||||||
retries: 12
|
|
||||||
start_period: 10s
|
|
||||||
|
|
||||||
haproxy:
|
haproxy:
|
||||||
image: haproxy:3.2-alpine
|
image: haproxy:3.2-alpine
|
||||||
restart: unless-stopped
|
restart: unless-stopped
|
||||||
|
|||||||
@ -3,19 +3,13 @@
|
|||||||
"editable": true,
|
"editable": true,
|
||||||
"graphTooltip": 1,
|
"graphTooltip": 1,
|
||||||
"panels": [
|
"panels": [
|
||||||
{"type":"timeseries","title":"Gateway QPS","gridPos":{"h":8,"w":8,"x":0,"y":0},"targets":[{"expr":"sum by (protocol) (rate(proxy_pool_gateway_requests_total[1m]))","legendFormat":"{{protocol}}"}]},
|
{"type":"timeseries","title":"Gateway QPS","gridPos":{"h":8,"w":8,"x":0,"y":0},"targets":[{"expr":"sum(rate(proxy_pool_gateway_requests_total[1m]))","legendFormat":"QPS"}]},
|
||||||
{"type":"timeseries","title":"Gateway p99","gridPos":{"h":8,"w":8,"x":8,"y":0},"targets":[{"expr":"histogram_quantile(0.99, sum by (le, protocol) (rate(proxy_pool_gateway_request_duration_seconds_bucket[5m])))","legendFormat":"{{protocol}}"}]},
|
{"type":"timeseries","title":"Gateway p99","gridPos":{"h":8,"w":8,"x":8,"y":0},"targets":[{"expr":"histogram_quantile(0.99, sum by (le) (rate(proxy_pool_gateway_request_duration_seconds_bucket[5m])))","legendFormat":"p99"}]},
|
||||||
{"type":"timeseries","title":"Gateway In Flight and Tunnels","gridPos":{"h":8,"w":8,"x":16,"y":0},"targets":[{"expr":"sum by (protocol) (proxy_pool_gateway_requests_in_flight)","legendFormat":"in-flight {{protocol}}"},{"expr":"sum(proxy_pool_gateway_active_tunnels)","legendFormat":"tunnels"}]},
|
{"type":"timeseries","title":"Available Slots","gridPos":{"h":8,"w":8,"x":16,"y":0},"targets":[{"expr":"sum(proxy_pool_available_slots)","legendFormat":"slots"}]},
|
||||||
{"type":"timeseries","title":"Gateway Outcome Failures","gridPos":{"h":8,"w":12,"x":0,"y":8},"targets":[{"expr":"sum by (stage) (rate(proxy_pool_gateway_outcomes_total{result=\"failure\"}[5m]))","legendFormat":"{{stage}}"}]},
|
{"type":"timeseries","title":"Provider Fetch","gridPos":{"h":8,"w":12,"x":0,"y":8},"targets":[{"expr":"sum by (result) (rate(proxy_pool_provider_fetch_total[5m]))","legendFormat":"{{result}}"}]},
|
||||||
{"type":"timeseries","title":"Gateway Outcome Queue Drops","gridPos":{"h":8,"w":12,"x":12,"y":8},"targets":[{"expr":"sum(rate(proxy_pool_gateway_outcome_queue_dropped_total[5m]))","legendFormat":"drops/s"}]},
|
{"type":"timeseries","title":"Extraction","gridPos":{"h":8,"w":12,"x":12,"y":8},"targets":[{"expr":"sum by (result) (rate(proxy_pool_extraction_total[5m]))","legendFormat":"{{result}}"}]},
|
||||||
{"type":"timeseries","title":"Controller Capacity","gridPos":{"h":8,"w":12,"x":0,"y":16},"targets":[{"expr":"sum(proxy_pool_controller_capacity_available_slots)","legendFormat":"available slots"},{"expr":"sum(proxy_pool_controller_capacity_effective_slots)","legendFormat":"effective slots"},{"expr":"sum(proxy_pool_controller_capacity_pending_expected_proxies)","legendFormat":"pending proxies"}]},
|
{"type":"timeseries","title":"Snapshot Age","gridPos":{"h":8,"w":12,"x":0,"y":16},"targets":[{"expr":"max by (worker) (proxy_pool_snapshot_age_seconds)","legendFormat":"{{worker}}"}]},
|
||||||
{"type":"timeseries","title":"Managed Proxies and Active Upstreams","gridPos":{"h":8,"w":12,"x":12,"y":16},"targets":[{"expr":"sum(proxy_pool_controller_capacity_managed_proxies)","legendFormat":"managed proxies"},{"expr":"sum(proxy_pool_controller_capacity_active_upstreams)","legendFormat":"active upstreams"}]},
|
{"type":"timeseries","title":"Checker Queue","gridPos":{"h":8,"w":12,"x":12,"y":16},"targets":[{"expr":"sum(proxy_pool_checker_queue_depth)","legendFormat":"depth"}]}
|
||||||
{"type":"timeseries","title":"Provider Fetch Results","gridPos":{"h":8,"w":12,"x":0,"y":24},"targets":[{"expr":"sum by (class) (rate(proxy_pool_controller_provider_fetch_results_total[5m]))","legendFormat":"{{class}}"},{"expr":"sum(rate(proxy_pool_controller_provider_new_proxies_total[5m]))","legendFormat":"new proxies/s"}]},
|
|
||||||
{"type":"timeseries","title":"Extraction Results","gridPos":{"h":8,"w":12,"x":12,"y":24},"targets":[{"expr":"sum by (result) (rate(proxy_pool_controller_extraction_requests_total[5m]))","legendFormat":"{{result}}"},{"expr":"sum(rate(proxy_pool_controller_extraction_returned_proxies_total[5m]))","legendFormat":"returned proxies/s"}]},
|
|
||||||
{"type":"timeseries","title":"Checker Task Dispatch","gridPos":{"h":8,"w":12,"x":0,"y":32},"targets":[{"expr":"sum by (level) (rate(proxy_pool_checker_tasks_dispatched_total[5m]))","legendFormat":"{{level}}"}]},
|
|
||||||
{"type":"timeseries","title":"Checker Observations","gridPos":{"h":8,"w":12,"x":12,"y":32},"targets":[{"expr":"sum by (level, result) (rate(proxy_pool_checker_observations_total[5m]))","legendFormat":"{{level}} {{result}}"}]},
|
|
||||||
{"type":"timeseries","title":"Drain Tickets Started","gridPos":{"h":8,"w":12,"x":0,"y":40},"targets":[{"expr":"sum by (reason) (rate(proxy_pool_controller_drains_started_total[5m]))","legendFormat":"{{reason}}"}]},
|
|
||||||
{"type":"timeseries","title":"Capacity Inventory Reads","gridPos":{"h":8,"w":12,"x":12,"y":40},"targets":[{"expr":"sum by (result) (rate(proxy_pool_controller_capacity_inventory_reads_total[5m]))","legendFormat":"{{result}}"}]}
|
|
||||||
],
|
],
|
||||||
"schemaVersion": 41,
|
"schemaVersion": 41,
|
||||||
"tags": ["proxy-pool"],
|
"tags": ["proxy-pool"],
|
||||||
@ -23,5 +17,5 @@
|
|||||||
"time": {"from":"now-6h","to":"now"},
|
"time": {"from":"now-6h","to":"now"},
|
||||||
"title": "Proxy Pool Overview",
|
"title": "Proxy Pool Overview",
|
||||||
"uid": "proxy-pool-overview",
|
"uid": "proxy-pool-overview",
|
||||||
"version": 2
|
"version": 1
|
||||||
}
|
}
|
||||||
|
|||||||
@ -21,16 +21,6 @@ data:
|
|||||||
limits:
|
limits:
|
||||||
maxConcurrentConnections: 100000
|
maxConcurrentConnections: 100000
|
||||||
requestsPerMinutePerClient: 60000
|
requestsPerMinutePerClient: 60000
|
||||||
transport:
|
|
||||||
dialTimeout: 10s
|
|
||||||
handshakeTimeout: 15s
|
|
||||||
responseHeaderTimeout: 30s
|
|
||||||
idleConnTimeout: 90s
|
|
||||||
maxIdleConns: 20000
|
|
||||||
maxIdleConnsPerHost: 32
|
|
||||||
maxConnsPerHost: 32
|
|
||||||
tunnelBufferBytes: 32768
|
|
||||||
tunnelIdleTimeout: 5m
|
|
||||||
retry:
|
retry:
|
||||||
maxAttempts: 2
|
maxAttempts: 2
|
||||||
retryMethods: [GET, HEAD]
|
retryMethods: [GET, HEAD]
|
||||||
|
|||||||
@ -1,63 +0,0 @@
|
|||||||
# Kubernetes development mTLS overlay
|
|
||||||
|
|
||||||
该 Overlay 在 base 资源之上启用 Controller 控制面 mTLS,并启动一个 Gateway 和
|
|
||||||
Checker。进程通过 `PROXY_POOL_AUTO_IDENTITY=true` 从各自证书的 URI SAN 派生身份;
|
|
||||||
它只用于开发或预发布的单副本控制面验证,不是生产弹性身份方案。
|
|
||||||
|
|
||||||
## Prepare certificates
|
|
||||||
|
|
||||||
先生成未提交的开发证书。输出目录必须为空;生成器的 `controller` 证书包含
|
|
||||||
`DNS:controller`,Overlay 中的客户端也只拨号 `controller:8443`。
|
|
||||||
|
|
||||||
```powershell
|
|
||||||
./scripts/generate-local-controlplane-certs.ps1
|
|
||||||
kubectl apply -f deploy/kubernetes/base/namespace.yaml
|
|
||||||
```
|
|
||||||
|
|
||||||
将生成的三组证书写入 Kubernetes Secret。下面的命令可重复执行,只更新对应 Secret:
|
|
||||||
|
|
||||||
```powershell
|
|
||||||
kubectl -n proxy-pool create secret generic proxy-pool-controlplane-server-tls `
|
|
||||||
--from-file=tls.crt=deploy/.control-plane-tls/controller/tls.crt `
|
|
||||||
--from-file=tls.key=deploy/.control-plane-tls/controller/tls.key `
|
|
||||||
--from-file=ca.crt=deploy/.control-plane-tls/controller/ca.crt `
|
|
||||||
--dry-run=client -o yaml | kubectl apply -f -
|
|
||||||
|
|
||||||
kubectl -n proxy-pool create secret generic proxy-pool-controlplane-gateway-tls `
|
|
||||||
--from-file=tls.crt=deploy/.control-plane-tls/gateway-a/tls.crt `
|
|
||||||
--from-file=tls.key=deploy/.control-plane-tls/gateway-a/tls.key `
|
|
||||||
--from-file=ca.crt=deploy/.control-plane-tls/gateway-a/ca.crt `
|
|
||||||
--dry-run=client -o yaml | kubectl apply -f -
|
|
||||||
|
|
||||||
kubectl -n proxy-pool create secret generic proxy-pool-controlplane-checker-tls `
|
|
||||||
--from-file=tls.crt=deploy/.control-plane-tls/checker-a/tls.crt `
|
|
||||||
--from-file=tls.key=deploy/.control-plane-tls/checker-a/tls.key `
|
|
||||||
--from-file=ca.crt=deploy/.control-plane-tls/checker-a/ca.crt `
|
|
||||||
--dry-run=client -o yaml | kubectl apply -f -
|
|
||||||
```
|
|
||||||
|
|
||||||
另行从私密配置系统创建 `proxy-pool-secrets`。可参考 base 目录的
|
|
||||||
`secret.example.yaml`,但不要把实际数据库、Redis、Provider 或 API 凭据写入 Git。
|
|
||||||
|
|
||||||
## Deploy
|
|
||||||
|
|
||||||
```powershell
|
|
||||||
kubectl apply -k deploy/kubernetes/overlays/development-mtls
|
|
||||||
kubectl -n proxy-pool rollout status deployment/proxy-controller --timeout=5m
|
|
||||||
kubectl -n proxy-pool rollout status deployment/proxy-gateway --timeout=5m
|
|
||||||
kubectl -n proxy-pool rollout status deployment/proxy-checker --timeout=5m
|
|
||||||
```
|
|
||||||
|
|
||||||
Overlay 的 Gateway 从 `gateway-a` 证书派生 `worker_id=gateway-a`,Checker 从
|
|
||||||
`checker-a` 证书派生 `checker_id=checker-a`,并把 Gateway HPA 限制为
|
|
||||||
`minReplicas=maxReplicas=1`。因此不得在此 Overlay 上增加副本数或放宽 HPA;重复使用
|
|
||||||
同一证书和 Worker 身份会破坏 session、ownership 和证书角色边界。
|
|
||||||
|
|
||||||
Overlay 的 `proxy-pool-runtime-config` 带内容哈希;修改 `config.yaml` 后,Kustomize 会重写三个
|
|
||||||
Deployment 的 ConfigMap volume 引用,从而触发 Kubernetes 滚动更新。控制面 TLS 与
|
|
||||||
`proxy-pool-secrets` 使用外部创建的固定名称 Secret,轮换后仍需按运行手册显式滚动对应工作负载。
|
|
||||||
|
|
||||||
生产环境需要由工作负载身份系统为每个副本签发独立、可轮换的证书,并将该副本的
|
|
||||||
身份注入 Gateway Client TLS 或 Checker Client TLS,并启用自动身份派生。应用会在每次新
|
|
||||||
控制面 TLS 握手读取更新后的叶证书和信任根;已有 gRPC 流仍按原会话保留,根轮换时先投放
|
|
||||||
新旧根的重叠信任包,再按 PDB 滚动排空旧连接。
|
|
||||||
@ -1,24 +0,0 @@
|
|||||||
apiVersion: networking.k8s.io/v1
|
|
||||||
kind: NetworkPolicy
|
|
||||||
metadata:
|
|
||||||
name: checker-control-plane
|
|
||||||
namespace: proxy-pool
|
|
||||||
spec:
|
|
||||||
podSelector:
|
|
||||||
matchLabels: {app.kubernetes.io/name: proxy-checker}
|
|
||||||
policyTypes: [Ingress, Egress]
|
|
||||||
ingress:
|
|
||||||
- from:
|
|
||||||
- namespaceSelector: {matchLabels: {kubernetes.io/metadata.name: monitoring}}
|
|
||||||
ports: [{port: 9090, protocol: TCP}]
|
|
||||||
egress:
|
|
||||||
- to:
|
|
||||||
- namespaceSelector: {matchLabels: {kubernetes.io/metadata.name: kube-system}}
|
|
||||||
ports: [{port: 53, protocol: UDP}, {port: 53, protocol: TCP}]
|
|
||||||
- to:
|
|
||||||
- podSelector: {matchLabels: {app.kubernetes.io/name: proxy-controller}}
|
|
||||||
ports: [{port: 8443, protocol: TCP}]
|
|
||||||
- to:
|
|
||||||
- ipBlock:
|
|
||||||
cidr: 0.0.0.0/0
|
|
||||||
except: [10.0.0.0/8, 100.64.0.0/10, 127.0.0.0/8, 169.254.0.0/16, 172.16.0.0/12, 192.168.0.0/16]
|
|
||||||
@ -1,26 +0,0 @@
|
|||||||
apiVersion: apps/v1
|
|
||||||
kind: Deployment
|
|
||||||
metadata:
|
|
||||||
name: proxy-checker
|
|
||||||
namespace: proxy-pool
|
|
||||||
spec:
|
|
||||||
replicas: 1
|
|
||||||
template:
|
|
||||||
spec:
|
|
||||||
containers:
|
|
||||||
- name: checker
|
|
||||||
env:
|
|
||||||
- {name: PROXY_POOL_CONTROL_PLANE_ADDRESS, value: controller:8443}
|
|
||||||
- {name: PROXY_POOL_AUTO_IDENTITY, value: "true"}
|
|
||||||
- {name: PROXY_POOL_CHECKER_MAX_IN_FLIGHT, value: "200"}
|
|
||||||
volumeMounts:
|
|
||||||
- name: controlplane-checker-tls
|
|
||||||
mountPath: /run/proxy-pool-tls/client
|
|
||||||
readOnly: true
|
|
||||||
volumes:
|
|
||||||
- name: config
|
|
||||||
configMap:
|
|
||||||
name: proxy-pool-runtime-config
|
|
||||||
- name: controlplane-checker-tls
|
|
||||||
secret:
|
|
||||||
secretName: proxy-pool-controlplane-checker-tls
|
|
||||||
@ -1,65 +0,0 @@
|
|||||||
apiVersion: apps/v1
|
|
||||||
kind: Deployment
|
|
||||||
metadata:
|
|
||||||
name: proxy-checker
|
|
||||||
namespace: proxy-pool
|
|
||||||
labels: {app.kubernetes.io/name: proxy-checker, app.kubernetes.io/part-of: proxy-pool}
|
|
||||||
spec:
|
|
||||||
replicas: 1
|
|
||||||
minReadySeconds: 5
|
|
||||||
revisionHistoryLimit: 3
|
|
||||||
strategy:
|
|
||||||
type: RollingUpdate
|
|
||||||
rollingUpdate: {maxUnavailable: 1, maxSurge: 1}
|
|
||||||
selector:
|
|
||||||
matchLabels: {app.kubernetes.io/name: proxy-checker}
|
|
||||||
template:
|
|
||||||
metadata:
|
|
||||||
labels: {app.kubernetes.io/name: proxy-checker, app.kubernetes.io/part-of: proxy-pool}
|
|
||||||
annotations: {prometheus.io/scrape: "true", prometheus.io/port: "9090", prometheus.io/path: /metrics}
|
|
||||||
spec:
|
|
||||||
serviceAccountName: proxy-pool
|
|
||||||
automountServiceAccountToken: false
|
|
||||||
terminationGracePeriodSeconds: 45
|
|
||||||
securityContext: {runAsNonRoot: true, seccompProfile: {type: RuntimeDefault}}
|
|
||||||
topologySpreadConstraints:
|
|
||||||
- maxSkew: 1
|
|
||||||
topologyKey: kubernetes.io/hostname
|
|
||||||
whenUnsatisfiable: ScheduleAnyway
|
|
||||||
labelSelector: {matchLabels: {app.kubernetes.io/name: proxy-checker}}
|
|
||||||
containers:
|
|
||||||
- name: checker
|
|
||||||
image: REGISTRY/proxy-pool:VERSION
|
|
||||||
imagePullPolicy: IfNotPresent
|
|
||||||
command: [proxy-checker]
|
|
||||||
env:
|
|
||||||
- {name: PROXY_POOL_CONFIG, value: /etc/proxy-pool/config.yaml}
|
|
||||||
envFrom:
|
|
||||||
- secretRef: {name: proxy-pool-secrets}
|
|
||||||
ports:
|
|
||||||
- {name: metrics, containerPort: 9090}
|
|
||||||
readinessProbe:
|
|
||||||
httpGet: {path: /readyz, port: metrics}
|
|
||||||
periodSeconds: 5
|
|
||||||
timeoutSeconds: 2
|
|
||||||
livenessProbe:
|
|
||||||
httpGet: {path: /livez, port: metrics}
|
|
||||||
periodSeconds: 10
|
|
||||||
timeoutSeconds: 2
|
|
||||||
lifecycle:
|
|
||||||
preStop: {exec: {command: [sh, -c, "sleep 3"]}}
|
|
||||||
resources:
|
|
||||||
requests: {cpu: "1", memory: 512Mi}
|
|
||||||
limits: {cpu: "2", memory: 1Gi}
|
|
||||||
securityContext:
|
|
||||||
allowPrivilegeEscalation: false
|
|
||||||
readOnlyRootFilesystem: true
|
|
||||||
capabilities: {drop: [ALL]}
|
|
||||||
volumeMounts:
|
|
||||||
- {name: config, mountPath: /etc/proxy-pool, readOnly: true}
|
|
||||||
- {name: tmp, mountPath: /tmp}
|
|
||||||
volumes:
|
|
||||||
- name: config
|
|
||||||
configMap: {name: proxy-pool-config}
|
|
||||||
- name: tmp
|
|
||||||
emptyDir: {sizeLimit: 64Mi}
|
|
||||||
@ -1,216 +0,0 @@
|
|||||||
version: 1
|
|
||||||
|
|
||||||
security:
|
|
||||||
requireProtectionOnPublicListen: true
|
|
||||||
|
|
||||||
gateway:
|
|
||||||
enabled: true
|
|
||||||
listen: 0.0.0.0:8080
|
|
||||||
access:
|
|
||||||
allowCIDRs: [0.0.0.0/0]
|
|
||||||
trustedProxies: []
|
|
||||||
auth:
|
|
||||||
mode: usernamePassword
|
|
||||||
username: "${PROXY_POOL_GATEWAY_USERNAME}"
|
|
||||||
password: "${PROXY_POOL_GATEWAY_PASSWORD}"
|
|
||||||
limits:
|
|
||||||
maxConcurrentConnections: 20000
|
|
||||||
requestsPerMinutePerClient: 60000
|
|
||||||
transport:
|
|
||||||
maxIdleConns: 20000
|
|
||||||
maxIdleConnsPerHost: 32
|
|
||||||
maxConnsPerHost: 32
|
|
||||||
tunnelBufferBytes: 32768
|
|
||||||
retry:
|
|
||||||
maxAttempts: 2
|
|
||||||
retryMethods: [GET, HEAD]
|
|
||||||
destinationPolicy:
|
|
||||||
denyPrivateNetworks: true
|
|
||||||
denyLoopback: true
|
|
||||||
denyLinkLocal: true
|
|
||||||
denyCIDRs: [169.254.169.254/32]
|
|
||||||
|
|
||||||
distribution:
|
|
||||||
enabled: true
|
|
||||||
listen: 0.0.0.0:8081
|
|
||||||
access:
|
|
||||||
allowCIDRs: [10.0.0.0/8]
|
|
||||||
trustedProxies: [10.0.0.0/8]
|
|
||||||
auth:
|
|
||||||
mode: apiKey
|
|
||||||
header: X-API-Key
|
|
||||||
token: "${PROXY_POOL_EXTRACT_TOKEN}"
|
|
||||||
limits:
|
|
||||||
requestsPerMinute: 6000
|
|
||||||
requestsPerMinutePerClient: 600
|
|
||||||
clientIdentification:
|
|
||||||
mode: sourceIP
|
|
||||||
extraction:
|
|
||||||
fulfillment: partial
|
|
||||||
maxCountPerRequest: 100
|
|
||||||
minRemainingTTL: 30s
|
|
||||||
maxHealthCheckAge: 30s
|
|
||||||
reserveForGateway: 1000
|
|
||||||
idempotencyTTL: 5m
|
|
||||||
|
|
||||||
admin:
|
|
||||||
enabled: true
|
|
||||||
listen: 0.0.0.0:8082
|
|
||||||
access:
|
|
||||||
allowCIDRs: [10.0.0.0/8]
|
|
||||||
auth:
|
|
||||||
mode: apiKey
|
|
||||||
header: X-Admin-Key
|
|
||||||
token: "${PROXY_POOL_ADMIN_TOKEN}"
|
|
||||||
|
|
||||||
controlPlane:
|
|
||||||
enabled: true
|
|
||||||
listen: 0.0.0.0:8443
|
|
||||||
protocolVersion: 1
|
|
||||||
heartbeatInterval: 10s
|
|
||||||
sessionTTL: 30s
|
|
||||||
maxStaleAge: 30s
|
|
||||||
maxMessageBytes: 4194304
|
|
||||||
maxRuntimeCounters: 100000
|
|
||||||
maxConcurrentStreams: 1000
|
|
||||||
tls:
|
|
||||||
mode: mtls
|
|
||||||
certFile: /run/proxy-pool-tls/server/tls.crt
|
|
||||||
keyFile: /run/proxy-pool-tls/server/tls.key
|
|
||||||
clientCAFile: /run/proxy-pool-tls/server/ca.crt
|
|
||||||
trustDomain: proxy-pool.local
|
|
||||||
environment: development
|
|
||||||
gatewayTLS:
|
|
||||||
certFile: /run/proxy-pool-tls/client/tls.crt
|
|
||||||
keyFile: /run/proxy-pool-tls/client/tls.key
|
|
||||||
serverCAFile: /run/proxy-pool-tls/client/ca.crt
|
|
||||||
checkerTLS:
|
|
||||||
certFile: /run/proxy-pool-tls/client/tls.crt
|
|
||||||
keyFile: /run/proxy-pool-tls/client/tls.key
|
|
||||||
serverCAFile: /run/proxy-pool-tls/client/ca.crt
|
|
||||||
|
|
||||||
metrics:
|
|
||||||
enabled: true
|
|
||||||
listen: 0.0.0.0:9090
|
|
||||||
|
|
||||||
storage:
|
|
||||||
postgresURL: "${PROXY_POOL_POSTGRES_URL}"
|
|
||||||
redisURL: "${PROXY_POOL_REDIS_URL}"
|
|
||||||
|
|
||||||
routing:
|
|
||||||
- name: gateway-default
|
|
||||||
enabled: true
|
|
||||||
purpose: gateway
|
|
||||||
upstreams: [provider-a, provider-b]
|
|
||||||
strategy:
|
|
||||||
type: sequential
|
|
||||||
switchAfterEmptyFetch: 5
|
|
||||||
endBehavior: stayLast
|
|
||||||
onUnavailable:
|
|
||||||
action: reject
|
|
||||||
- name: extract-default
|
|
||||||
enabled: true
|
|
||||||
purpose: extract
|
|
||||||
upstreams: [provider-a, provider-b]
|
|
||||||
strategy:
|
|
||||||
type: sequential
|
|
||||||
switchAfterEmptyFetch: 5
|
|
||||||
endBehavior: stayLast
|
|
||||||
onUnavailable:
|
|
||||||
action: reject
|
|
||||||
|
|
||||||
upstreams:
|
|
||||||
provider-a:
|
|
||||||
enabled: true
|
|
||||||
exposure: [gateway, extract]
|
|
||||||
provider:
|
|
||||||
billingMode: fetch
|
|
||||||
protocols: [http]
|
|
||||||
api:
|
|
||||||
url: https://provider-a.invalid/api/proxies
|
|
||||||
method: GET
|
|
||||||
auth:
|
|
||||||
type: apiKey
|
|
||||||
location: header
|
|
||||||
name: Authorization
|
|
||||||
value: "${PROVIDER_A_TOKEN}"
|
|
||||||
template: '{{ . }}'
|
|
||||||
proxyAuth:
|
|
||||||
type: response
|
|
||||||
pool:
|
|
||||||
maxSize: 5000
|
|
||||||
shrinkDelay: 30s
|
|
||||||
capacity:
|
|
||||||
maxConcurrencyPerProxy: 20
|
|
||||||
refill:
|
|
||||||
reconcileInterval: 1s
|
|
||||||
minimumAvailableSlots: 8000
|
|
||||||
targetAvailableSlots: 12000
|
|
||||||
lifecycle:
|
|
||||||
ttl: 5m
|
|
||||||
allocationSafetyMargin: 20s
|
|
||||||
fetch:
|
|
||||||
estimatedIPsPerCall: 100
|
|
||||||
requestInterval: 1s
|
|
||||||
timeout: 3s
|
|
||||||
maxAttempts: 3
|
|
||||||
maxInFlight: 1
|
|
||||||
maxTotal: 100000
|
|
||||||
maxResponseBytes: 4194304
|
|
||||||
templateTimeout: 100ms
|
|
||||||
retry: {initial: 500ms, max: 30s, jitter: 20}
|
|
||||||
check:
|
|
||||||
interval: 30s
|
|
||||||
jitter: 20
|
|
||||||
maxInFlight: 200
|
|
||||||
timeout: 3s
|
|
||||||
maxAttempts: 2
|
|
||||||
maxConsecutiveFailures: 3
|
|
||||||
urls: [https://example.com/]
|
|
||||||
provider-b:
|
|
||||||
enabled: true
|
|
||||||
exposure: [gateway, extract]
|
|
||||||
provider:
|
|
||||||
billingMode: fetch
|
|
||||||
protocols: [http]
|
|
||||||
api:
|
|
||||||
url: https://provider-b.invalid/api/proxies
|
|
||||||
method: GET
|
|
||||||
auth:
|
|
||||||
type: apiKey
|
|
||||||
location: header
|
|
||||||
name: Authorization
|
|
||||||
value: "${PROVIDER_B_TOKEN}"
|
|
||||||
template: '{{ . }}'
|
|
||||||
proxyAuth:
|
|
||||||
type: response
|
|
||||||
pool:
|
|
||||||
maxSize: 5000
|
|
||||||
shrinkDelay: 30s
|
|
||||||
capacity:
|
|
||||||
maxConcurrencyPerProxy: 20
|
|
||||||
refill:
|
|
||||||
reconcileInterval: 1s
|
|
||||||
minimumAvailableSlots: 8000
|
|
||||||
targetAvailableSlots: 12000
|
|
||||||
lifecycle:
|
|
||||||
ttl: 5m
|
|
||||||
allocationSafetyMargin: 20s
|
|
||||||
fetch:
|
|
||||||
estimatedIPsPerCall: 100
|
|
||||||
requestInterval: 1s
|
|
||||||
timeout: 3s
|
|
||||||
maxAttempts: 3
|
|
||||||
maxInFlight: 1
|
|
||||||
maxTotal: 100000
|
|
||||||
maxResponseBytes: 4194304
|
|
||||||
templateTimeout: 100ms
|
|
||||||
retry: {initial: 500ms, max: 30s, jitter: 20}
|
|
||||||
check:
|
|
||||||
interval: 30s
|
|
||||||
jitter: 20
|
|
||||||
maxInFlight: 200
|
|
||||||
timeout: 3s
|
|
||||||
maxAttempts: 2
|
|
||||||
maxConsecutiveFailures: 3
|
|
||||||
urls: [https://example.com/]
|
|
||||||
@ -1,6 +0,0 @@
|
|||||||
apiVersion: v1
|
|
||||||
kind: ConfigMap
|
|
||||||
metadata:
|
|
||||||
name: proxy-pool-config
|
|
||||||
namespace: proxy-pool
|
|
||||||
$patch: delete
|
|
||||||
@ -1,22 +0,0 @@
|
|||||||
apiVersion: apps/v1
|
|
||||||
kind: Deployment
|
|
||||||
metadata:
|
|
||||||
name: proxy-controller
|
|
||||||
namespace: proxy-pool
|
|
||||||
spec:
|
|
||||||
replicas: 1
|
|
||||||
template:
|
|
||||||
spec:
|
|
||||||
containers:
|
|
||||||
- name: controller
|
|
||||||
volumeMounts:
|
|
||||||
- name: controlplane-server-tls
|
|
||||||
mountPath: /run/proxy-pool-tls/server
|
|
||||||
readOnly: true
|
|
||||||
volumes:
|
|
||||||
- name: config
|
|
||||||
configMap:
|
|
||||||
name: proxy-pool-runtime-config
|
|
||||||
- name: controlplane-server-tls
|
|
||||||
secret:
|
|
||||||
secretName: proxy-pool-controlplane-server-tls
|
|
||||||
@ -1,11 +0,0 @@
|
|||||||
apiVersion: v1
|
|
||||||
kind: Service
|
|
||||||
metadata:
|
|
||||||
name: controller
|
|
||||||
namespace: proxy-pool
|
|
||||||
labels: {app.kubernetes.io/name: proxy-controller}
|
|
||||||
spec:
|
|
||||||
type: ClusterIP
|
|
||||||
selector: {app.kubernetes.io/name: proxy-controller}
|
|
||||||
ports:
|
|
||||||
- {name: control, port: 8443, targetPort: control}
|
|
||||||
@ -1,8 +0,0 @@
|
|||||||
apiVersion: autoscaling/v2
|
|
||||||
kind: HorizontalPodAutoscaler
|
|
||||||
metadata:
|
|
||||||
name: proxy-gateway
|
|
||||||
namespace: proxy-pool
|
|
||||||
spec:
|
|
||||||
minReplicas: 1
|
|
||||||
maxReplicas: 1
|
|
||||||
@ -1,27 +0,0 @@
|
|||||||
apiVersion: apps/v1
|
|
||||||
kind: Deployment
|
|
||||||
metadata:
|
|
||||||
name: proxy-gateway
|
|
||||||
namespace: proxy-pool
|
|
||||||
spec:
|
|
||||||
replicas: 1
|
|
||||||
template:
|
|
||||||
spec:
|
|
||||||
containers:
|
|
||||||
- name: gateway
|
|
||||||
env:
|
|
||||||
- {name: PROXY_POOL_CONTROL_PLANE_ADDRESS, value: controller:8443}
|
|
||||||
- {name: PROXY_POOL_CLUSTER_ID, value: kubernetes-development}
|
|
||||||
- {name: PROXY_POOL_AUTO_IDENTITY, value: "true"}
|
|
||||||
- {name: PROXY_POOL_ZONE, value: kubernetes-development}
|
|
||||||
volumeMounts:
|
|
||||||
- name: controlplane-gateway-tls
|
|
||||||
mountPath: /run/proxy-pool-tls/client
|
|
||||||
readOnly: true
|
|
||||||
volumes:
|
|
||||||
- name: config
|
|
||||||
configMap:
|
|
||||||
name: proxy-pool-runtime-config
|
|
||||||
- name: controlplane-gateway-tls
|
|
||||||
secret:
|
|
||||||
secretName: proxy-pool-controlplane-gateway-tls
|
|
||||||
@ -1,18 +0,0 @@
|
|||||||
apiVersion: kustomize.config.k8s.io/v1beta1
|
|
||||||
kind: Kustomization
|
|
||||||
resources:
|
|
||||||
- ../../base
|
|
||||||
- checker.yaml
|
|
||||||
- controller-service.yaml
|
|
||||||
- checker-networkpolicy.yaml
|
|
||||||
configMapGenerator:
|
|
||||||
- name: proxy-pool-runtime-config
|
|
||||||
namespace: proxy-pool
|
|
||||||
files:
|
|
||||||
- config.yaml
|
|
||||||
patches:
|
|
||||||
- path: configmap-delete.yaml
|
|
||||||
- path: controller-patch.yaml
|
|
||||||
- path: gateway-patch.yaml
|
|
||||||
- path: checker-patch.yaml
|
|
||||||
- path: gateway-hpa-patch.yaml
|
|
||||||
@ -1,73 +0,0 @@
|
|||||||
package deploy
|
|
||||||
|
|
||||||
import (
|
|
||||||
"os"
|
|
||||||
"os/exec"
|
|
||||||
"regexp"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestKubernetesDevelopmentMTLSOverlayRendersCertificateDerivedIdentities(t *testing.T) {
|
|
||||||
kubectl, err := exec.LookPath("kubectl")
|
|
||||||
if err != nil {
|
|
||||||
t.Skip("kubectl is required to render the Kubernetes development mTLS overlay")
|
|
||||||
}
|
|
||||||
command := exec.Command(kubectl, "kustomize", "kubernetes/overlays/development-mtls")
|
|
||||||
command.Dir = "."
|
|
||||||
payload, err := command.CombinedOutput()
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("render development mTLS overlay: %v\n%s", err, payload)
|
|
||||||
}
|
|
||||||
rendered := string(payload)
|
|
||||||
for _, required := range []string{
|
|
||||||
"mode: mtls",
|
|
||||||
"trustDomain: proxy-pool.local",
|
|
||||||
"value: controller:8443",
|
|
||||||
"name: PROXY_POOL_AUTO_IDENTITY",
|
|
||||||
"value: \"true\"",
|
|
||||||
"secretName: proxy-pool-controlplane-server-tls",
|
|
||||||
"secretName: proxy-pool-controlplane-gateway-tls",
|
|
||||||
"secretName: proxy-pool-controlplane-checker-tls",
|
|
||||||
"name: proxy-checker",
|
|
||||||
"name: checker-control-plane",
|
|
||||||
"minReplicas: 1",
|
|
||||||
"maxReplicas: 1",
|
|
||||||
} {
|
|
||||||
if !strings.Contains(rendered, required) {
|
|
||||||
t.Errorf("development mTLS overlay does not render %q", required)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
for _, forbidden := range []string{"name: PROXY_POOL_WORKER_ID", "name: PROXY_POOL_INSTANCE_ID", "name: PROXY_POOL_CHECKER_ID", "name: PROXY_POOL_CHECKER_INSTANCE_ID"} {
|
|
||||||
if strings.Contains(rendered, forbidden) {
|
|
||||||
t.Errorf("development mTLS overlay still renders manual identity %q", forbidden)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestKubernetesDevelopmentMTLSOverlayUsesVersionedConfigMap(t *testing.T) {
|
|
||||||
payload, err := os.ReadFile("kubernetes/overlays/development-mtls/kustomization.yaml")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("read development mTLS kustomization: %v", err)
|
|
||||||
}
|
|
||||||
if strings.Contains(string(payload), "disableNameSuffixHash: true") {
|
|
||||||
t.Fatal("development mTLS overlay disables ConfigMap versioning")
|
|
||||||
}
|
|
||||||
kubectl, err := exec.LookPath("kubectl")
|
|
||||||
if err != nil {
|
|
||||||
t.Skip("kubectl is required to render the Kubernetes development mTLS overlay")
|
|
||||||
}
|
|
||||||
command := exec.Command(kubectl, "kustomize", "kubernetes/overlays/development-mtls")
|
|
||||||
command.Dir = "."
|
|
||||||
renderedPayload, err := command.CombinedOutput()
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("render development mTLS overlay: %v\n%s", err, renderedPayload)
|
|
||||||
}
|
|
||||||
matched := regexp.MustCompile(`(?m)^ name: (proxy-pool-runtime-config-[a-z0-9]+)$`).FindStringSubmatch(string(renderedPayload))
|
|
||||||
if len(matched) != 2 {
|
|
||||||
t.Fatalf("rendered overlay has no versioned config map: %s", renderedPayload)
|
|
||||||
}
|
|
||||||
if occurrences := strings.Count(string(renderedPayload), "name: "+matched[1]); occurrences != 4 {
|
|
||||||
t.Fatalf("versioned config map reference count = %d, want config map plus three workload volumes", occurrences)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@ -14,9 +14,6 @@ scrape_configs:
|
|||||||
- job_name: proxy-controller
|
- job_name: proxy-controller
|
||||||
static_configs:
|
static_configs:
|
||||||
- targets: [controller:9090]
|
- targets: [controller:9090]
|
||||||
- job_name: proxy-checker
|
|
||||||
static_configs:
|
|
||||||
- targets: [checker:9090]
|
|
||||||
- job_name: haproxy
|
- job_name: haproxy
|
||||||
metrics_path: /metrics
|
metrics_path: /metrics
|
||||||
static_configs:
|
static_configs:
|
||||||
|
|||||||
@ -1,74 +1,40 @@
|
|||||||
groups:
|
groups:
|
||||||
- name: proxy-pool
|
- name: proxy-pool
|
||||||
rules:
|
rules:
|
||||||
- alert: ProxyPoolGatewayOutcomeFailureRate
|
- alert: ProxyPoolGatewayHighErrorRate
|
||||||
expr: |
|
expr: |
|
||||||
sum(rate(proxy_pool_gateway_outcomes_total{result="failure"}[5m]))
|
sum(rate(proxy_pool_gateway_requests_total{result="error"}[5m]))
|
||||||
/ clamp_min(sum(rate(proxy_pool_gateway_outcomes_total[5m])), 1) > 0.1
|
/ clamp_min(sum(rate(proxy_pool_gateway_requests_total[5m])), 1) > 0.02
|
||||||
for: 10m
|
for: 10m
|
||||||
labels:
|
labels:
|
||||||
severity: warning
|
severity: warning
|
||||||
annotations:
|
annotations:
|
||||||
summary: Gateway 代理尝试失败比例持续高于 10%
|
summary: Gateway 错误率持续高于 2%
|
||||||
- alert: ProxyPoolGatewayHighLatency
|
- alert: ProxyPoolGatewaySnapshotStale
|
||||||
expr: |
|
expr: proxy_pool_snapshot_age_seconds > 60
|
||||||
histogram_quantile(0.99,
|
|
||||||
sum by (le, protocol) (rate(proxy_pool_gateway_request_duration_seconds_bucket[5m]))) > 5
|
|
||||||
for: 10m
|
|
||||||
labels:
|
|
||||||
severity: warning
|
|
||||||
annotations:
|
|
||||||
summary: Gateway p99 请求时延持续高于 5 秒
|
|
||||||
- alert: ProxyPoolGatewayOutcomeQueueDrops
|
|
||||||
expr: sum(increase(proxy_pool_gateway_outcome_queue_dropped_total[5m])) > 0
|
|
||||||
for: 5m
|
|
||||||
labels:
|
|
||||||
severity: warning
|
|
||||||
annotations:
|
|
||||||
summary: Gateway Outcome 本地队列发生丢弃
|
|
||||||
- alert: ProxyPoolNoAvailableSlots
|
|
||||||
expr: |
|
|
||||||
sum(proxy_pool_controller_capacity_active_upstreams) > 0
|
|
||||||
and sum(proxy_pool_controller_capacity_available_slots) == 0
|
|
||||||
for: 1m
|
|
||||||
labels:
|
|
||||||
severity: critical
|
|
||||||
annotations:
|
|
||||||
summary: Controller 可分配容量耗尽
|
|
||||||
- alert: ProxyPoolCapacityInventoryErrors
|
|
||||||
expr: sum(rate(proxy_pool_controller_capacity_inventory_reads_total{result="error"}[10m])) > 0.2
|
|
||||||
for: 10m
|
|
||||||
labels:
|
|
||||||
severity: warning
|
|
||||||
annotations:
|
|
||||||
summary: Controller 容量库存读取持续失败
|
|
||||||
- alert: ProxyPoolProviderFetchErrors
|
|
||||||
expr: sum(rate(proxy_pool_controller_provider_fetch_results_total{class="error"}[10m])) > 0.2
|
|
||||||
for: 10m
|
|
||||||
labels:
|
|
||||||
severity: warning
|
|
||||||
annotations:
|
|
||||||
summary: Provider 获取持续失败
|
|
||||||
- alert: ProxyPoolCheckerObservationRejections
|
|
||||||
expr: |
|
|
||||||
sum(rate(proxy_pool_checker_observations_total{result="rejected"}[5m]))
|
|
||||||
/ clamp_min(sum(rate(proxy_pool_checker_observations_total[5m])), 1) > 0.05
|
|
||||||
for: 10m
|
|
||||||
labels:
|
|
||||||
severity: warning
|
|
||||||
annotations:
|
|
||||||
summary: Checker Observation 拒绝比例持续高于 5%
|
|
||||||
- alert: ProxyPoolExtractionIdempotencyConflict
|
|
||||||
expr: sum(rate(proxy_pool_controller_extraction_requests_total{result="idempotency_conflict"}[5m])) > 0
|
|
||||||
for: 5m
|
|
||||||
labels:
|
|
||||||
severity: warning
|
|
||||||
annotations:
|
|
||||||
summary: Distribution 幂等键冲突持续发生
|
|
||||||
- alert: ProxyPoolControlPlaneTargetDown
|
|
||||||
expr: up{job=~"proxy-gateway|proxy-controller|proxy-checker"} == 0
|
|
||||||
for: 2m
|
for: 2m
|
||||||
labels:
|
labels:
|
||||||
severity: critical
|
severity: critical
|
||||||
annotations:
|
annotations:
|
||||||
summary: Proxy Pool 控制面目标不可抓取
|
summary: Gateway Snapshot 已超过安全陈旧时间
|
||||||
|
- alert: ProxyPoolNoAvailableSlots
|
||||||
|
expr: sum(proxy_pool_available_slots) == 0
|
||||||
|
for: 1m
|
||||||
|
labels:
|
||||||
|
severity: critical
|
||||||
|
annotations:
|
||||||
|
summary: Gateway 可分配容量耗尽
|
||||||
|
- alert: ProxyPoolProviderFetchErrors
|
||||||
|
expr: sum by (upstream) (rate(proxy_pool_provider_fetch_total{result="error"}[10m])) > 0.2
|
||||||
|
for: 10m
|
||||||
|
labels:
|
||||||
|
severity: warning
|
||||||
|
annotations:
|
||||||
|
summary: Provider Fetch 错误持续发生
|
||||||
|
- alert: ProxyPoolExtractionConflict
|
||||||
|
expr: sum(rate(proxy_pool_extraction_total{result="conflict"}[5m])) > 0
|
||||||
|
for: 5m
|
||||||
|
labels:
|
||||||
|
severity: warning
|
||||||
|
annotations:
|
||||||
|
summary: 独占提取发生持续事务冲突
|
||||||
|
|||||||
@ -26,7 +26,6 @@ PostgreSQL 事务内提交。若把它们暴露成多个 Repository,调用方
|
|||||||
type Mutator interface {
|
type Mutator interface {
|
||||||
SetUpstreamEnabled(context.Context, SetUpstreamCommand) (MutationResult, error)
|
SetUpstreamEnabled(context.Context, SetUpstreamCommand) (MutationResult, error)
|
||||||
SwitchRouting(context.Context, SwitchRoutingCommand) (MutationResult, error)
|
SwitchRouting(context.Context, SwitchRoutingCommand) (MutationResult, error)
|
||||||
DisableRouting(context.Context, DisableRoutingCommand) (MutationResult, error)
|
|
||||||
CommitConfig(context.Context, CommitConfigCommand) (MutationResult, error)
|
CommitConfig(context.Context, CommitConfigCommand) (MutationResult, error)
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -44,7 +43,7 @@ type Outbox interface {
|
|||||||
}
|
}
|
||||||
```
|
```
|
||||||
|
|
||||||
四个 mutation 方法保留现有 Admin/自动切换语义,事务、修订号、CAS、审计和 Outbox 都
|
三个 mutation 方法保留现有 Admin 语义,事务、修订号、CAS、审计和 Outbox 都
|
||||||
隐藏在模块实现内部。不会暴露 `BeginTx`、SQL executor 或五个可被错误组合的浅
|
隐藏在模块实现内部。不会暴露 `BeginTx`、SQL executor 或五个可被错误组合的浅
|
||||||
Repository。
|
Repository。
|
||||||
|
|
||||||
@ -96,12 +95,6 @@ revision,因此并发提交或 Supervisor 同步的迟到旧版本不能覆盖
|
|||||||
并发使用同一 expected 值时最多一个请求成功。目标等于当前值且 expected 匹配时
|
并发使用同一 expected 值时最多一个请求成功。目标等于当前值且 expected 匹配时
|
||||||
返回 `changed=false`,仍写审计但不写 Outbox。
|
返回 `changed=false`,仍写审计但不写 Outbox。
|
||||||
|
|
||||||
`DisableRouting` 用于 Sequential 的末端 `stop`。它同样锁定目标 Routing,并要求
|
|
||||||
`expectedCurrent` 仍等于权威当前值;满足条件时保留 `current_upstream`、仅将
|
|
||||||
`enabled` 置为 `false`。这样延迟的 Provider Empty 观察不会覆盖人工切换或配置重载。
|
|
||||||
已停用且 expected 匹配时返回 `changed=false` 并审计,不再产生 Outbox;真实停用写入
|
|
||||||
`routing.disabled` 事件。该行为与 `SwitchRouting` 共享同一个全局 revision 分配边界。
|
|
||||||
|
|
||||||
### Outbox 消费
|
### Outbox 消费
|
||||||
|
|
||||||
Outbox 使用有界 claim/ack,而不是无界全表扫描:
|
Outbox 使用有界 claim/ack,而不是无界全表扫描:
|
||||||
@ -136,8 +129,7 @@ MemoryStore 与 PostgreSQL Adapter 运行相同契约,至少覆盖:
|
|||||||
|
|
||||||
- 配置首次提交、相同重放、校验和冲突和非法引用零写入。
|
- 配置首次提交、相同重放、校验和冲突和非法引用零写入。
|
||||||
- Upstream enable/disable 幂等、修订单调、审计必写、Outbox 仅在变更时写。
|
- Upstream enable/disable 幂等、修订单调、审计必写、Outbox 仅在变更时写。
|
||||||
- Routing CAS、目标校验和 100 个并发请求最多一个成功;terminal stop 的 100 个
|
- Routing CAS、目标校验和 100 个并发请求最多一个成功。
|
||||||
并发请求仅一个真实停用,其余为幂等审计 no-op。
|
|
||||||
- 任一审计/Outbox 写故障导致状态完全回滚。
|
- 任一审计/Outbox 写故障导致状态完全回滚。
|
||||||
- Outbox 有界 claim、租约到期重试、错误 consumer ACK 拒绝和顺序稳定。
|
- Outbox 有界 claim、租约到期重试、错误 consumer ACK 拒绝和顺序稳定。
|
||||||
- 审计按 ID 稳定分页并保留 Actor、资源、动作、修订和 UTC 时间;Routing no-op
|
- 审计按 ID 稳定分页并保留 Actor、资源、动作、修订和 UTC 时间;Routing no-op
|
||||||
|
|||||||
@ -6,7 +6,6 @@ Admin API 使用独立监听器与权限,契约位于 `api/openapi/admin.yaml`
|
|||||||
## 端点
|
## 端点
|
||||||
|
|
||||||
- `GET /api/v1/status`:返回配置/快照版本、Upstream 聚合计数和 Worker 状态。
|
- `GET /api/v1/status`:返回配置/快照版本、Upstream 聚合计数和 Worker 状态。
|
||||||
- `GET /api/v1/audit`:按审计记录 ID 升序读取权威管理面的变更记录。
|
|
||||||
- `POST /api/v1/upstreams/{name}/enable`:启用 Upstream。
|
- `POST /api/v1/upstreams/{name}/enable`:启用 Upstream。
|
||||||
- `POST /api/v1/upstreams/{name}/disable`:停止新 Fetch/分配并自然 Drain。
|
- `POST /api/v1/upstreams/{name}/disable`:停止新 Fetch/分配并自然 Drain。
|
||||||
- `POST /api/v1/routing/{name}/switch`:用 expectedCurrent 做 CAS 手工切换。
|
- `POST /api/v1/routing/{name}/switch`:用 expectedCurrent 做 CAS 手工切换。
|
||||||
@ -15,30 +14,9 @@ Admin API 使用独立监听器与权限,契约位于 `api/openapi/admin.yaml`
|
|||||||
所有写操作写审计记录并返回最终 Request ID 与版本。Enable/Disable 对目标状态
|
所有写操作写审计记录并返回最终 Request ID 与版本。Enable/Disable 对目标状态
|
||||||
幂等;Routing Switch 必须携带 `expectedCurrent`,避免并发操作跳过多个供应商。
|
幂等;Routing Switch 必须携带 `expectedCurrent`,避免并发操作跳过多个供应商。
|
||||||
|
|
||||||
## 授权
|
|
||||||
|
|
||||||
认证成功后的权限由 Listener `auth.permissions` 或 `auth.methods[].permissions` 提供。
|
|
||||||
`admin:read` 允许 Status 与审计查询;`admin:write` 允许 Upstream 启停、Routing
|
|
||||||
切换和配置重载。每个权限独立匹配,写权限不隐含读权限;未配置权限的旧凭据保持
|
|
||||||
全权限兼容。`mode: any` 使用实际命中凭据的方法级权限。权限不足返回 `403`,不会
|
|
||||||
调用 Service、写审计或触发配置发布。
|
|
||||||
|
|
||||||
配置重载校验失败返回 422,旧配置继续运行。Status 只返回低基数聚合信息,
|
配置重载校验失败返回 422,旧配置继续运行。Status 只返回低基数聚合信息,
|
||||||
不得返回 Proxy 地址、凭据、Client 标识或完整 Provider URL。
|
不得返回 Proxy 地址、凭据、Client 标识或完整 Provider URL。
|
||||||
|
|
||||||
## 审计分页与数据范围
|
|
||||||
|
|
||||||
`GET /api/v1/audit` 使用 `afterId` 排他游标分页:响应只包含 `id` 大于
|
|
||||||
`afterId` 的记录,并按 `id` 升序排列;未传时 `afterId` 为 `0`。未传 `limit`
|
|
||||||
时服务端使用 `100`;
|
|
||||||
`limit` 必须为 `1..1000`,其中 `1000` 等于 `adminstate.MaxPageSize`。客户端应将
|
|
||||||
本页最后一条记录的 `id` 作为下一次请求的 `afterId`;空 `records` 表示当前游标后
|
|
||||||
没有记录。
|
|
||||||
|
|
||||||
该接口仅暴露管理面变更记录及操作者标识、可信来源 IP、操作类型、资源标识、变更
|
|
||||||
状态、控制面版本、原因和发生时间。它不返回 Proxy、任何 Upstream/Provider URL、
|
|
||||||
凭据或 Secret、Provider 响应载荷,也不读取或暴露 Redis 活动池及其提取状态。
|
|
||||||
|
|
||||||
## 运行时实现边界
|
## 运行时实现边界
|
||||||
|
|
||||||
`admin.Handler` 只依赖 `Service` 控制面接口,不直接操作数据库、路由游标或配置
|
`admin.Handler` 只依赖 `Service` 控制面接口,不直接操作数据库、路由游标或配置
|
||||||
|
|||||||
@ -27,9 +27,7 @@ Controller 已实现并验证 `RegisterWorker`、`AcknowledgeSnapshot` 和
|
|||||||
|
|
||||||
`WatchSnapshots` 已在 Register 后发送与当前 ownership epoch 对应的基础完整快照,并在
|
`WatchSnapshots` 已在 Register 后发送与当前 ownership epoch 对应的基础完整快照,并在
|
||||||
每份快照有效期的一半前重新构建、下发版本递增的完整快照;这样 ownership、Routing 和
|
每份快照有效期的一半前重新构建、下发版本递增的完整快照;这样 ownership、Routing 和
|
||||||
凭据变化会在同一长连接内收敛,而无需等待有效期到达后重新注册。Controller 内已提交的
|
凭据变化会在同一长连接内收敛,而无需等待有效期到达后重新注册。
|
||||||
Upstream 启停、Routing 切换和配置发布还会向全部本地 Worker 流广播一次合并后的立即刷新;
|
|
||||||
多 Controller 副本仍由该定时机制完成跨进程收敛。
|
|
||||||
Gateway 校验后 ACK 并开始 Runtime 心跳。Controller 会从 Redis 的有界 Worker ownership
|
Gateway 校验后 ACK 并开始 Runtime 心跳。Controller 会从 Redis 的有界 Worker ownership
|
||||||
索引构建已归属 Proxy 内容,并将租约到期收紧到 Proxy 的 `usable_until`。Proxy 引用的
|
索引构建已归属 Proxy 内容,并将租约到期收紧到 Proxy 的 `usable_until`。Proxy 引用的
|
||||||
凭据材料按 `secret_ref + credential_version` 去重,随完整 Snapshot 经 mTLS 下发,仅保留在
|
凭据材料按 `secret_ref + credential_version` 去重,随完整 Snapshot 经 mTLS 下发,仅保留在
|
||||||
@ -236,15 +234,10 @@ Gateway 不复用 `controlPlane.listen` 作为客户端地址。`listen` 是 Con
|
|||||||
- `-worker-id` / `PROXY_POOL_WORKER_ID`:唯一逻辑 Worker。
|
- `-worker-id` / `PROXY_POOL_WORKER_ID`:唯一逻辑 Worker。
|
||||||
- `-instance-id` / `PROXY_POOL_INSTANCE_ID`:唯一进程实例。
|
- `-instance-id` / `PROXY_POOL_INSTANCE_ID`:唯一进程实例。
|
||||||
- `-zone` / `PROXY_POOL_ZONE`:实例可用区。
|
- `-zone` / `PROXY_POOL_ZONE`:实例可用区。
|
||||||
- `-auto-identity` / `PROXY_POOL_AUTO_IDENTITY=true`:仅 mTLS 模式可用;从客户端
|
|
||||||
证书中唯一的 `.../worker/<worker-id>` SPIFFE URI 派生 Worker ID,并在未显式配置时
|
|
||||||
使用同一值作为 Instance ID。
|
|
||||||
|
|
||||||
当 `controlPlane.tls.mode=mtls` 时,Gateway 使用 `controlPlane.gatewayTLS` 中独立的
|
当 `controlPlane.tls.mode=mtls` 时,Gateway 使用 `controlPlane.gatewayTLS` 中独立的
|
||||||
客户端证书、私钥和 Controller CA 发起 TLS 1.3 连接;证书必须符合 Controller 的
|
客户端证书、私钥和 Controller CA 发起 TLS 1.3 连接;证书必须符合 Controller 的
|
||||||
SPIFFE Worker 身份校验。客户端 X.509-SVID 叶证书必须且只能包含一个 URI SAN,且该 URI
|
SPIFFE Worker 身份校验。
|
||||||
必须是对应角色的身份。Gateway 自动解析和 Controller 授权共用严格 URI 规则:身份 URI
|
|
||||||
不得包含用户信息、端口、查询、片段或转义路径。
|
|
||||||
`disabled` 仅接受回环控制面地址,供本地 fixture 使用。
|
`disabled` 仅接受回环控制面地址,供本地 fixture 使用。
|
||||||
|
|
||||||
## 10. Checker 启动参数
|
## 10. Checker 启动参数
|
||||||
@ -254,8 +247,6 @@ SPIFFE Worker 身份校验。客户端 X.509-SVID 叶证书必须且只能包含
|
|||||||
- `-instance-id` / `PROXY_POOL_CHECKER_INSTANCE_ID`:唯一进程实例。
|
- `-instance-id` / `PROXY_POOL_CHECKER_INSTANCE_ID`:唯一进程实例。
|
||||||
- `-max-in-flight` / `PROXY_POOL_CHECKER_MAX_IN_FLIGHT`:本进程任务上限。
|
- `-max-in-flight` / `PROXY_POOL_CHECKER_MAX_IN_FLIGHT`:本进程任务上限。
|
||||||
- `-levels`:逗号分隔的 `basic,egress,target` 能力集合。
|
- `-levels`:逗号分隔的 `basic,egress,target` 能力集合。
|
||||||
- `-auto-identity` / `PROXY_POOL_AUTO_IDENTITY=true`:仅 mTLS 模式可用;从客户端
|
|
||||||
证书中唯一的 `.../checker/<checker-id>` SPIFFE URI 派生 Checker ID 和缺省 Instance ID。
|
|
||||||
|
|
||||||
当 `controlPlane.tls.mode=mtls` 时,Checker 使用 `controlPlane.checkerTLS` 的独立
|
当 `controlPlane.tls.mode=mtls` 时,Checker 使用 `controlPlane.checkerTLS` 的独立
|
||||||
客户端证书、私钥和 Controller CA 建立 TLS 1.3 连接;证书必须符合 Controller 的
|
客户端证书、私钥和 Controller CA 建立 TLS 1.3 连接;证书必须符合 Controller 的
|
||||||
|
|||||||
@ -197,16 +197,6 @@ TTL 到期或 Redis 数据丢失后不再保证旧 Key 去重,系统不回退
|
|||||||
|
|
||||||
错误响应不得包含 Provider Secret、Proxy 凭据、SQL 或内部拓扑。
|
错误响应不得包含 Provider Secret、Proxy 凭据、SQL 或内部拓扑。
|
||||||
|
|
||||||
`POST /api/v1/proxies/extract` 需要 `distribution:extract`。权限从 Listener 的
|
|
||||||
`auth.permissions` 或 `mode: any` 的命中 `auth.methods[].permissions` 取得;缺少权限
|
|
||||||
返回 `403`,不会读取请求体、消耗幂等键或调用提取服务。未声明权限的旧凭据保持全
|
|
||||||
权限兼容。
|
|
||||||
|
|
||||||
Distribution 认证凭据还可声明 client 提取约束:maxExtractCount、
|
|
||||||
allowedUpstreams 与 allowedRegions。该约束随实际命中的凭据返回,早于幂等键和
|
|
||||||
提取服务执行。未传地区或 Upstream 过滤时,服务端自动写入凭据允许集合;传入
|
|
||||||
集合外的值、或数量超过凭据上限时返回 403,并且不会消耗库存。
|
|
||||||
|
|
||||||
## 10. 短期运行记录与数据最小化
|
## 10. 短期运行记录与数据最小化
|
||||||
|
|
||||||
Redis 幂等结果只在配置的 TTL 窗口内保留重放响应所需的数据:
|
Redis 幂等结果只在配置的 TTL 窗口内保留重放响应所需的数据:
|
||||||
|
|||||||
@ -122,122 +122,9 @@ Lua 可精确表示的整数范围内。
|
|||||||
- `any`:`methods` 中任一方法成功即可;Bearer 方法的 Secret 使用
|
- `any`:`methods` 中任一方法成功即可;Bearer 方法的 Secret 使用
|
||||||
`value`/`valueFile`。
|
`value`/`valueFile`。
|
||||||
|
|
||||||
### 3.2 端点权限
|
|
||||||
|
|
||||||
`auth.permissions` 是普通认证方式(Basic、API Key、Bearer、IP 白名单)的固定权限
|
|
||||||
集合。未配置时保留历史全权限行为;显式配置后只允许下列有限值:
|
|
||||||
|
|
||||||
- `admin:read`:读取 Admin Status 与审计页。
|
|
||||||
- `admin:write`:执行 Upstream/Routing/配置变更。
|
|
||||||
- `distribution:extract`:调用 Distribution 独占提取。
|
|
||||||
- `*`:所有权限,且必须单独出现。
|
|
||||||
|
|
||||||
`mode: any` 必须将 `permissions` 写在各 `methods` 项中,权限随实际命中的凭据
|
|
||||||
返回,顶层 `auth.permissions` 不允许同时配置。权限值不接受空白、重复或未知项。
|
|
||||||
缺少所需权限的已认证请求返回 `403`,并在进入 JSON 解码、提取或管理 mutation 前
|
|
||||||
结束。`mode: none` 不允许配置权限,避免把匿名访问误配置为受控授权。
|
|
||||||
|
|
||||||
```yaml
|
|
||||||
admin:
|
|
||||||
auth:
|
|
||||||
mode: any
|
|
||||||
methods:
|
|
||||||
- mode: bearer
|
|
||||||
valueFile: /run/secrets/admin-reader-token
|
|
||||||
permissions: [admin:read]
|
|
||||||
- mode: bearer
|
|
||||||
valueFile: /run/secrets/admin-writer-token
|
|
||||||
permissions: [admin:read, admin:write]
|
|
||||||
```
|
|
||||||
|
|
||||||
### 3.3 Distribution Client 提取约束
|
|
||||||
|
|
||||||
认证凭据可在 auth.client 中设置固定的 Client 约束;mode: any 则必须在实际
|
|
||||||
命中的 auth.methods 项下配置。当前 Client 约束按入口明确分工,避免出现只声明
|
|
||||||
不执行的访问控制。
|
|
||||||
|
|
||||||
- maxExtractCount:Distribution 单次提取的额外上限;0 表示不追加上限。
|
|
||||||
- allowedUpstreams:允许访问的 Upstream 名称集合。省略请求过滤条件时,服务端
|
|
||||||
自动使用这个集合;请求携带集合外名称会返回 403。
|
|
||||||
- allowedRegions:允许访问的地区集合,行为与 allowedUpstreams 相同。
|
|
||||||
|
|
||||||
空 client 配置保留旧版兼容语义,不会限制已认证凭据。Upstream 名称必须引用
|
|
||||||
当前配置中已有的 Upstream。
|
|
||||||
|
|
||||||
~~~yaml
|
|
||||||
distribution:
|
|
||||||
auth:
|
|
||||||
mode: any
|
|
||||||
methods:
|
|
||||||
- mode: apiKey
|
|
||||||
header: X-API-Key
|
|
||||||
valueFile: /run/secrets/tenant-a-key
|
|
||||||
permissions: [distribution:extract]
|
|
||||||
client:
|
|
||||||
maxExtractCount: 20
|
|
||||||
allowedUpstreams: [provider-a, provider-b]
|
|
||||||
allowedRegions: [shanghai, beijing]
|
|
||||||
~~~
|
|
||||||
|
|
||||||
### 3.4 Gateway Client 路由约束
|
|
||||||
|
|
||||||
Gateway 的 auth.client 支持 allowedRoutings、requestsPerMinute 和
|
|
||||||
maxConcurrentConnections。认证成功后,
|
|
||||||
Gateway 先完成目标地址策略和本地 Routing 匹配,再于本地 Dispatcher 前检查
|
|
||||||
Routing 集合;不匹配返回 403,不会尝试选择或预留 Proxy。速率和并发限制均以
|
|
||||||
认证 Client 为计数键,在每个 Gateway Worker 内执行,超过限制返回 429。三项
|
|
||||||
检查都只读取请求上下文和当前本地状态,不访问 Redis、PostgreSQL 或 Provider。
|
|
||||||
|
|
||||||
allowedRoutings 中的每个名称必须对应一个启用的 Gateway Routing。Gateway 上的
|
|
||||||
maxExtractCount、allowedUpstreams 和 allowedRegions,以及 Distribution 上的
|
|
||||||
allowedRoutings 和 requestsPerMinute 均会被配置校验拒绝。Admin 不支持任何
|
|
||||||
client 约束。
|
|
||||||
|
|
||||||
~~~yaml
|
|
||||||
gateway:
|
|
||||||
auth:
|
|
||||||
mode: any
|
|
||||||
methods:
|
|
||||||
- mode: bearer
|
|
||||||
valueFile: /run/secrets/checkout-gateway-token
|
|
||||||
client:
|
|
||||||
allowedRoutings: [checkout]
|
|
||||||
requestsPerMinute: 600
|
|
||||||
maxConcurrentConnections: 20
|
|
||||||
~~~
|
|
||||||
|
|
||||||
Gateway、Distribution、Admin 与 Provider API 是独立认证边界。改变其中一套
|
Gateway、Distribution、Admin 与 Provider API 是独立认证边界。改变其中一套
|
||||||
不得连带改变其他入口。
|
不得连带改变其他入口。
|
||||||
|
|
||||||
### 3.5 Gateway 粘性会话
|
|
||||||
|
|
||||||
gateway.stickySession 让调用方使用一个会话头尽量固定出口 Proxy。绑定键是认证
|
|
||||||
Client、已匹配 Routing 和会话标识的组合,因此不同 Client 或不同 Routing 使用
|
|
||||||
相同会话值也不会共享出口。该功能要求 Gateway 认证开启。
|
|
||||||
|
|
||||||
~~~yaml
|
|
||||||
gateway:
|
|
||||||
auth:
|
|
||||||
mode: bearer
|
|
||||||
tokenFile: /run/secrets/gateway-token
|
|
||||||
stickySession:
|
|
||||||
enabled: true
|
|
||||||
header: X-Proxy-Session
|
|
||||||
ttl: 20s
|
|
||||||
maxEntries: 100000
|
|
||||||
~~~
|
|
||||||
|
|
||||||
- header 必须是规范 HTTP Header 名;一个请求只能携带一个值,值由字母、数字、
|
|
||||||
点、下划线、连字符组成且最长 128 字节。
|
|
||||||
- ttl 是绑定的最长时长;实际到期时间还会被 Proxy 的 usableUntil、代理过期时间
|
|
||||||
和 allocation safety margin 截短。
|
|
||||||
- maxEntries 是当前 Gateway Worker 的硬上限。缓存只保存 Proxy ID 和过期时间,
|
|
||||||
不保存 Proxy 地址、凭据或原始会话标识。
|
|
||||||
- 会话头仅用于本地选路,Gateway 在转发 HTTP 请求前删除它。Proxy 不再符合当前
|
|
||||||
Snapshot 或一次代理尝试失败时,绑定会被清除并按正常 Routing 重选。
|
|
||||||
- 该版本不把 Redis、PostgreSQL 或 gRPC 引入 Gateway 热路径;跨 Worker 粘性需要
|
|
||||||
上游负载均衡保持 Worker 亲和,后续由控制面快照协议扩展共享恢复能力。
|
|
||||||
|
|
||||||
## 4. Worker 控制面
|
## 4. Worker 控制面
|
||||||
|
|
||||||
控制面默认关闭;默认配置中的 `8443` 端口预留不表示服务已监听。控制面 Session、
|
控制面默认关闭;默认配置中的 `8443` 端口预留不表示服务已监听。控制面 Session、
|
||||||
@ -300,21 +187,15 @@ controlPlane:
|
|||||||
- `gatewayTLS` 是 Gateway 的客户端证书、私钥和 Controller CA,与 `tls` 的服务端
|
- `gatewayTLS` 是 Gateway 的客户端证书、私钥和 Controller CA,与 `tls` 的服务端
|
||||||
证书和 Worker CA 分离。三项可以同时省略(未运行 Gateway),配置任一项时必须完整提供。
|
证书和 Worker CA 分离。三项可以同时省略(未运行 Gateway),配置任一项时必须完整提供。
|
||||||
- `checkerTLS` 与 `gatewayTLS` 有相同字段和完整性校验,但必须使用独立的 Checker
|
- `checkerTLS` 与 `gatewayTLS` 有相同字段和完整性校验,但必须使用独立的 Checker
|
||||||
证书。每个客户端 X.509-SVID 叶证书必须且只能包含一个 URI SAN;Controller 分别验证
|
证书。Controller 分别验证 `.../worker/<worker-id>` 与
|
||||||
`.../worker/<worker-id>` 与 `.../checker/<checker-id>` SPIFFE URI,不能跨角色复用证书。
|
`.../checker/<checker-id>` SPIFFE URI,不能跨角色复用证书。
|
||||||
- Controller、Gateway 与 Checker 会在每个新的控制面 TLS 握手读取最新叶证书、私钥和
|
|
||||||
信任根;文件暂时不可读时继续使用最后一次有效材料。既有 gRPC 流不强制重握手,根 CA
|
|
||||||
轮换仍需先发布包含新旧根的重叠信任包,并在宽限期后滚动排空旧连接。
|
|
||||||
|
|
||||||
Gateway 连接 Controller 时使用独立启动参数而非 `controlPlane.listen`。至少设置
|
Gateway 连接 Controller 时使用独立启动参数而非 `controlPlane.listen`。至少设置
|
||||||
`PROXY_POOL_CONTROL_PLANE_ADDRESS`、`PROXY_POOL_CLUSTER_ID`、
|
`PROXY_POOL_CONTROL_PLANE_ADDRESS`、`PROXY_POOL_CLUSTER_ID`、
|
||||||
`PROXY_POOL_WORKER_ID`、`PROXY_POOL_INSTANCE_ID` 和 `PROXY_POOL_ZONE`,详见
|
`PROXY_POOL_WORKER_ID`、`PROXY_POOL_INSTANCE_ID` 和 `PROXY_POOL_ZONE`,详见
|
||||||
[控制面协议](../api/control-plane.md#9-gateway-启动参数)。Compose 本地模板从被忽略的
|
[控制面协议](../api/control-plane.md#9-gateway-启动参数)。基础 Compose/Kubernetes
|
||||||
开发证书目录派生 Worker/Checker 身份;Kubernetes Base 保持
|
模板保持 `controlPlane.enabled: false`,环境 Overlay 挂载 mTLS 证书并启用后才可启动
|
||||||
`controlPlane.enabled: false`,必须由具备每工作负载唯一身份的 mTLS Overlay 启用。
|
Gateway。
|
||||||
在 mTLS 生产工作负载中,`PROXY_POOL_AUTO_IDENTITY=true` 可省略 `WORKER_ID` 和
|
|
||||||
`INSTANCE_ID`:进程仅接受其 `gatewayTLS` 叶证书中唯一的
|
|
||||||
`spiffe://<trust-domain>/<environment>/worker/<worker-id>` URI,并以该 ID 作为缺省实例。
|
|
||||||
|
|
||||||
Checker 同样使用独立的可拨号地址:`proxy-checker` 的 `-control-plane`、
|
Checker 同样使用独立的可拨号地址:`proxy-checker` 的 `-control-plane`、
|
||||||
`-checker-id`、`-instance-id` 和 `-max-in-flight` 可由对应的
|
`-checker-id`、`-instance-id` 和 `-max-in-flight` 可由对应的
|
||||||
@ -322,16 +203,9 @@ Checker 同样使用独立的可拨号地址:`proxy-checker` 的 `-control-pla
|
|||||||
模式只接受回环 Controller 地址。Checker 只从 gRPC 领取任务并批量上报事实,不读取
|
模式只接受回环 Controller 地址。Checker 只从 gRPC 领取任务并批量上报事实,不读取
|
||||||
Redis/PostgreSQL;Controller 在生产启动拓扑中装配 Redis 共享任务队列,当前调度
|
Redis/PostgreSQL;Controller 在生产启动拓扑中装配 Redis 共享任务队列,当前调度
|
||||||
HTTP/HTTPS/SOCKS5 BASIC、EGRESS 和 TARGET 检查。调度监督器在每轮从已发布配置读取启用的
|
HTTP/HTTPS/SOCKS5 BASIC、EGRESS 和 TARGET 检查。调度监督器在每轮从已发布配置读取启用的
|
||||||
上游与 Routing;启用 Admin 时还要求 PostgreSQL 管理态与配置 revision 一致,并取两者均启用的
|
上游与 Routing;Admin reload 发布后,上游/路由启停、有效 `check` 策略和目标列表会在下一轮生效,
|
||||||
上游和 Routing。管理态禁用的上游不会再产生新的 BASIC、EGRESS 或 TARGET 任务;管理态禁用的
|
|
||||||
Routing 也不会再产生其 TARGET 任务;revision 不一致或状态不完整时本轮失败关闭。Admin reload 发布后,
|
|
||||||
上游/路由启停、有效 `check` 策略和目标列表会在下一轮生效,
|
|
||||||
新启用的上游无需重启 Controller。
|
新启用的上游无需重启 Controller。
|
||||||
|
|
||||||
`PROXY_POOL_AUTO_IDENTITY=true` 对 Checker 使用同样规则:仅接受 `checkerTLS` 中唯一且仅有的
|
|
||||||
`spiffe://<trust-domain>/<environment>/checker/<checker-id>` URI,并在未显式设置时复用其
|
|
||||||
`checker-id` 作为实例 ID。
|
|
||||||
|
|
||||||
`maxRuntimeCounters` 同时限制单个 Runtime 报告和单个 Outcome 批次的条目数。Gateway
|
`maxRuntimeCounters` 同时限制单个 Runtime 报告和单个 Outcome 批次的条目数。Gateway
|
||||||
在本地维护容量为 `65536` 的非阻塞 Outcome 队列,默认微批上限为 `512`,实际取二者中
|
在本地维护容量为 `65536` 的非阻塞 Outcome 队列,默认微批上限为 `512`,实际取二者中
|
||||||
较小值;该队列与其序列确认状态仅存在于 Gateway 进程内。Controller 的 Redis 状态只保存
|
较小值;该队列与其序列确认状态仅存在于 Gateway 进程内。Controller 的 Redis 状态只保存
|
||||||
@ -346,24 +220,9 @@ gateway:
|
|||||||
auth: {mode: none}
|
auth: {mode: none}
|
||||||
limits:
|
limits:
|
||||||
maxConcurrentConnections: 50000
|
maxConcurrentConnections: 50000
|
||||||
transport:
|
|
||||||
dialTimeout: 10s
|
|
||||||
handshakeTimeout: 15s
|
|
||||||
responseHeaderTimeout: 30s
|
|
||||||
idleConnTimeout: 90s
|
|
||||||
maxIdleConns: 20000
|
|
||||||
maxIdleConnsPerHost: 32
|
|
||||||
maxConnsPerHost: 32
|
|
||||||
tunnelBufferBytes: 32768
|
|
||||||
tunnelIdleTimeout: 5m
|
|
||||||
retry:
|
retry:
|
||||||
maxAttempts: 2
|
maxAttempts: 2
|
||||||
retryMethods: [GET, HEAD]
|
retryMethods: [GET, HEAD]
|
||||||
stickySession:
|
|
||||||
enabled: true
|
|
||||||
header: X-Proxy-Session
|
|
||||||
ttl: 20s
|
|
||||||
maxEntries: 100000
|
|
||||||
destinationPolicy:
|
destinationPolicy:
|
||||||
denyPrivateNetworks: true
|
denyPrivateNetworks: true
|
||||||
denyLoopback: true
|
denyLoopback: true
|
||||||
@ -381,31 +240,6 @@ gateway:
|
|||||||
- 保留地址、CGNAT 与云元数据端点始终拒绝,不能通过私网/链路本地开关放行。
|
- 保留地址、CGNAT 与云元数据端点始终拒绝,不能通过私网/链路本地开关放行。
|
||||||
- `maxConcurrentConnections` 是入口准入上限,不是 Proxy 容量上限。
|
- `maxConcurrentConnections` 是入口准入上限,不是 Proxy 容量上限。
|
||||||
|
|
||||||
### 5.1 Gateway 本地传输层
|
|
||||||
|
|
||||||
**gateway.transport** 只配置每个 Gateway Worker 的本地 HTTP Transport 和 CONNECT
|
|
||||||
隧道 I/O,不读取或写入 Redis、PostgreSQL、Provider,也不参与热路径分配。修改
|
|
||||||
这些字段需要滚动重启对应 Gateway Worker 后生效。
|
|
||||||
|
|
||||||
- **dialTimeout**、**handshakeTimeout**、**responseHeaderTimeout**、
|
|
||||||
**idleConnTimeout**:分别限制拨号、上游 TLS/CONNECT 握手、HTTP 响应头和空闲
|
|
||||||
HTTP 连接的时长。
|
|
||||||
- **maxIdleConns**:单个本地 HTTP Transport 的总空闲连接上限。Gateway 分别为代理
|
|
||||||
转发和 direct fallback 建立 Transport,最坏情况下两者都可能保有空闲连接。
|
|
||||||
- **maxIdleConnsPerHost**:单个上游 Host 的空闲 HTTP 连接上限。
|
|
||||||
- **maxConnsPerHost**:单个上游 Host 的 HTTP 活跃加空闲连接上限。它不替代
|
|
||||||
**limits.maxConcurrentConnections**,也不限制已经建立的 CONNECT 隧道。
|
|
||||||
- **tunnelBufferBytes**:每个活跃 CONNECT 隧道的每个复制方向使用一个缓冲区,内存
|
|
||||||
预算至少按 活跃隧道数 * 2 * tunnelBufferBytes 计算。
|
|
||||||
- **tunnelIdleTimeout**:CONNECT 隧道双向没有数据活动时的最长存活时间。
|
|
||||||
|
|
||||||
所有字段均为可选项;数值 0 表示沿用进程默认值:拨号 10s、握手 15s、
|
|
||||||
响应头 30s、空闲 HTTP 连接 90s、总空闲连接 1024、每 Host 空闲连接
|
|
||||||
64、隧道缓冲 32768 字节、隧道空闲 5m。计数和时长不得为负数;
|
|
||||||
maxIdleConnsPerHost 不得大于显式配置的 maxIdleConns。示例中的 20000/32
|
|
||||||
是高并发 HTTP 转发的起点,不是每秒请求量的换算公式,应同时按可用文件描述符、
|
|
||||||
上游限额和观测到的复用率调节。
|
|
||||||
|
|
||||||
## 6. Distribution
|
## 6. Distribution
|
||||||
|
|
||||||
```yaml
|
```yaml
|
||||||
@ -457,7 +291,6 @@ routing:
|
|||||||
- name: api-post
|
- name: api-post
|
||||||
enabled: true
|
enabled: true
|
||||||
purpose: gateway
|
purpose: gateway
|
||||||
action: proxy
|
|
||||||
match:
|
match:
|
||||||
hostRegex: '^api\\.example\\.com$'
|
hostRegex: '^api\\.example\\.com$'
|
||||||
methods: [POST]
|
methods: [POST]
|
||||||
@ -477,10 +310,6 @@ routing:
|
|||||||
|
|
||||||
- Routing 列表有序,首条匹配后停止。
|
- Routing 列表有序,首条匹配后停止。
|
||||||
- `purpose` 为 `gateway` 或 `extract`。
|
- `purpose` 为 `gateway` 或 `extract`。
|
||||||
- `action` 省略时为 `proxy`。`proxy` 使用 Upstream 与 Strategy;`direct` 直接连接已通过
|
|
||||||
目标地址策略验证的目标;`reject` 拒绝匹配请求。`direct` 与 `reject` 仅允许
|
|
||||||
`gateway` Routing,且不得同时配置 `upstreams`、`strategy`、`onUnavailable` 或
|
|
||||||
`check.targets`。
|
|
||||||
- `strategy.type` 支持 `sequential`、`random`、`roundRobin`、`weighted`、
|
- `strategy.type` 支持 `sequential`、`random`、`roundRobin`、`weighted`、
|
||||||
`leastConnections`。
|
`leastConnections`。
|
||||||
- `weighted` 使用 `weights` 映射,键必须引用本 Routing 的 Upstream。
|
- `weighted` 使用 `weights` 映射,键必须引用本 Routing 的 Upstream。
|
||||||
@ -488,10 +317,6 @@ routing:
|
|||||||
`switchAfterEmptyFetch`;`endBehavior` 省略时默认为 `stop`,也可显式设置
|
`switchAfterEmptyFetch`;`endBehavior` 省略时默认为 `stop`,也可显式设置
|
||||||
`loop` 或 `stayLast`。
|
`loop` 或 `stayLast`。
|
||||||
- `onUnavailable.action` 为 `reject`、`wait` 或 `direct`;默认建议 `reject`。
|
- `onUnavailable.action` 为 `reject`、`wait` 或 `direct`;默认建议 `reject`。
|
||||||
- `action: direct` 在 Dispatcher 前执行,不申请 Proxy 容量、不写 Proxy Outcome,也不建立
|
|
||||||
粘性 Proxy 绑定;它仍执行入口认证、Client 路由约束、速率/并发限制和目标地址策略。
|
|
||||||
`onUnavailable.action: direct` 仅是 `action: proxy` 在本地没有候选时的回退,二者
|
|
||||||
不可互换。
|
|
||||||
- `check.targets` 是 Routing 级 HTTP/HTTPS 探测目标。每个 Routing 最多 16 个 URL,且一个
|
- `check.targets` 是 Routing 级 HTTP/HTTPS 探测目标。每个 Routing 最多 16 个 URL,且一个
|
||||||
启用 Upstream 被其引用的 TARGET Profile 总数最多 64 个。每个 Profile 的身份是
|
启用 Upstream 被其引用的 TARGET Profile 总数最多 64 个。每个 Profile 的身份是
|
||||||
`(routing.name, target URL)`;失败只影响该 Profile,不改变 Proxy 的全局健康状态。
|
`(routing.name, target URL)`;失败只影响该 Profile,不改变 Proxy 的全局健康状态。
|
||||||
@ -500,16 +325,6 @@ Sequential 的空计数属于 Upstream,当前索引属于 Routing。只有 Pro
|
|||||||
成功、模板成功且合法候选为零时才增加空计数。错误不改变空计数;重复候选
|
成功、模板成功且合法候选为零时才增加空计数。错误不改变空计数;重复候选
|
||||||
会重置空计数但增加独立 duplicate 指标。
|
会重置空计数但增加独立 duplicate 指标。
|
||||||
|
|
||||||
Controller 在连续空结果达到阈值时按顺序切换当前 Upstream;到达最后一个可用
|
|
||||||
Upstream 且 `endBehavior: stop` 时,会以当前 Upstream 的 CAS 条件原子停用该
|
|
||||||
Routing。停用后的 Gateway 规则按既有 `onUnavailable.action` 执行 `reject`、`wait`
|
|
||||||
或 `direct`,直到一次配置重载提交新的管理快照。
|
|
||||||
|
|
||||||
管理态禁用当前 Sequential Upstream 不等待新的 Provider 空结果:Controller 会按
|
|
||||||
配置顺序以相同的 `ExpectedCurrent` CAS 推进到下一个启用 Upstream,并立即刷新完整
|
|
||||||
Snapshot。若当前项之后没有启用候选,只有 `endBehavior: loop` 会回绕搜索更早的候选;
|
|
||||||
`stop` 与 `stayLast` 都会原子停用 Routing,因为已禁用的当前项不能继续承接新分配。
|
|
||||||
|
|
||||||
## 8. Upstream
|
## 8. Upstream
|
||||||
|
|
||||||
```yaml
|
```yaml
|
||||||
@ -661,11 +476,6 @@ proxyAuth:
|
|||||||
- 启用 Routing 的 `check.targets` 会为其引用的每个启用 Upstream 创建 TARGET 检查组。BASIC、
|
- 启用 Routing 的 `check.targets` 会为其引用的每个启用 Upstream 创建 TARGET 检查组。BASIC、
|
||||||
EGRESS 与 TARGET 使用固定批次和轮转顺序,并共享该 Upstream 的 `check.maxInFlight`,避免
|
EGRESS 与 TARGET 使用固定批次和轮转顺序,并共享该 Upstream 的 `check.maxInFlight`,避免
|
||||||
配置多个目标后产生无界检查流量。
|
配置多个目标后产生无界检查流量。
|
||||||
- 启用 Admin 时,调度仅使用配置和管理态均启用、且 revision 一致的 Upstream;管理态停用会在
|
|
||||||
下一调度轮阻止新的 BASIC、EGRESS、TARGET 任务。revision 不一致、重复或缺失的上游状态按失败
|
|
||||||
关闭处理,不读取 Redis due-index。
|
|
||||||
- 启用 Admin 时,TARGET Profile 还要求其 Routing 在配置和管理态中均启用;管理态停用 Routing
|
|
||||||
仅停止该 Routing 的新 TARGET 任务,不影响同一 Upstream 的 BASIC/EGRESS 或其他启用 Routing。
|
|
||||||
- 第一次有意义失败进入 SUSPECT;达到 `maxConsecutiveFailures` 后才进入
|
- 第一次有意义失败进入 SUSPECT;达到 `maxConsecutiveFailures` 后才进入
|
||||||
UNHEALTHY。
|
UNHEALTHY。
|
||||||
- `unhealthyRemoveAfter` 控制 UNHEALTHY 持续多久后可由 Controller 回收;`0s`
|
- `unhealthyRemoveAfter` 控制 UNHEALTHY 持续多久后可由 Controller 回收;`0s`
|
||||||
@ -716,20 +526,10 @@ Metrics 启用时 `listen` 必须是合法 `host:port`。该入口固定提供 `
|
|||||||
`result` 固定为 accepted、rejected。Gateway 另暴露
|
`result` 固定为 accepted、rejected。Gateway 另暴露
|
||||||
`proxy_pool_gateway_outcomes_total{stage,result}` 与
|
`proxy_pool_gateway_outcomes_total{stage,result}` 与
|
||||||
`proxy_pool_gateway_outcome_queue_dropped_total`;`stage` 固定为 DIAL、PROXY_HANDSHAKE、
|
`proxy_pool_gateway_outcome_queue_dropped_total`;`stage` 固定为 DIAL、PROXY_HANDSHAKE、
|
||||||
RESPONSE_HEADERS、TUNNEL,`result` 固定为 success、failure。Provider、Extraction、
|
RESPONSE_HEADERS、TUNNEL,`result` 固定为 success、failure。绝不包含 Proxy、IP、Checker、
|
||||||
Capacity 和 Drain 业务指标也已接入,所有标签均为固定枚举;绝不包含 Proxy、IP、Checker、
|
Client、路由、目标 URL 或凭据标签。Provider、提取和容量等业务指标仍在后续实施范围。Distribution 启用时 `/readyz` 只以 Redis 活动池为
|
||||||
Client、路由、目标 URL 或凭据标签。Capacity 从既有 Provider 库存对账循环聚合,
|
|
||||||
不进入 Gateway 热路径。Distribution 启用时 `/readyz` 只以 Redis 活动池为
|
|
||||||
服务流量门槛,PostgreSQL 故障由 Admin 接口独立报告。Metrics 开关或监听地址
|
服务流量门槛,PostgreSQL 故障由 Admin 接口独立报告。Metrics 开关或监听地址
|
||||||
变更需要重启 Controller。Controller、Gateway 和 Checker 的进程级致命错误以 JSON
|
变更需要重启 Controller。
|
||||||
结构化日志输出,包含组件和稳定错误类型,不输出错误原文;敏感属性、URL 用户信息和
|
|
||||||
查询 Secret 在写出前统一替换为 `[REDACTED]`。
|
|
||||||
|
|
||||||
Gateway 还按每个 Worker 暴露 HTTP/CONNECT 请求总数、在途请求数和活跃 CONNECT
|
|
||||||
隧道数;协议标签固定为 HTTP、CONNECT。请求数在 Handler 接受请求时增加,在途数在
|
|
||||||
全部拒绝、转发或隧道关闭后归零;活跃隧道只覆盖成功建立并开始 relay 的连接。这些
|
|
||||||
指标是连接池、入口并发、文件描述符和长连接排空的本地观测,不携带 Proxy、路由、
|
|
||||||
目标、Client 或凭据。
|
|
||||||
|
|
||||||
## 10. 启动前校验清单
|
## 10. 启动前校验清单
|
||||||
|
|
||||||
|
|||||||
@ -80,10 +80,10 @@ Provider、Pool、Routing、Distribution 在首版需要共享事务和一致性
|
|||||||
|
|
||||||
### 3.4 proxy-loadgen
|
### 3.4 proxy-loadgen
|
||||||
|
|
||||||
- 当前实现 HTTP、CONNECT 长连接和 Extract 场景:固定请求数或固定时长,受限并发与可选
|
- 当前实现 HTTP 请求场景:固定请求数或固定时长,受限并发与可选目标 QPS;可配置方法、
|
||||||
目标 QPS;可配置方法、重复请求头和请求体。HTTPS 目标经 HTTP Gateway 时由 Transport 走 CONNECT。
|
重复请求头和请求体。HTTPS 目标经 HTTP Gateway 时由 Transport 走 CONNECT。
|
||||||
- 输出场景、延迟分位上界、错误分类、吞吐和 Go 内存/GC 快照。故障注入、进程 CPU/RSS/句柄
|
- 输出场景、延迟分位上界、错误分类、吞吐和 Go 内存/GC 快照。CONNECT 长连接、
|
||||||
与网络采样以及代表性集群报告仍需补齐。
|
Extract 并发、进程 CPU/RSS/句柄与网络采样仍需补齐。
|
||||||
|
|
||||||
## 4. 模块边界
|
## 4. 模块边界
|
||||||
|
|
||||||
@ -319,8 +319,7 @@ Upstream 的 Empty 事实全局共享;每条 Routing 独立 CAS 当前索引
|
|||||||
协程只能有一个成功从 A 切到 B,其他协程读取新版本,不会再切到 C。
|
协程只能有一个成功从 A 切到 B,其他协程读取新版本,不会再切到 C。
|
||||||
Sequential 至少配置两个 Upstream;列表耗尽后的默认行为是 `stop`,`loop` 和
|
Sequential 至少配置两个 Upstream;列表耗尽后的默认行为是 `stop`,`loop` 和
|
||||||
`stayLast` 必须显式配置。disabled Upstream 不参与新分配,其运行时跳过与权威
|
`stayLast` 必须显式配置。disabled Upstream 不参与新分配,其运行时跳过与权威
|
||||||
游标由 Controller 的权威管理状态持久化。达到末端 `stop` 时,Controller 以观察到的
|
游标持久化仍由后续 Routing Runtime 完成。
|
||||||
当前 Upstream 作为 CAS 栅栏禁用 Routing,并让完整 Snapshot 显式关闭该规则。
|
|
||||||
|
|
||||||
## 10. Exclusive Extraction
|
## 10. Exclusive Extraction
|
||||||
|
|
||||||
|
|||||||
@ -42,14 +42,10 @@ Gateway Worker 只读取本地不可变快照并维护本地容量计数。供
|
|||||||
|
|
||||||
1. 接入层完成认证、来源识别、限流和目标地址检查。
|
1. 接入层完成认证、来源识别、限流和目标地址检查。
|
||||||
2. Routing 按配置顺序首条命中。
|
2. Routing 按配置顺序首条命中。
|
||||||
3. 若启用并携带粘性会话头,先在本 Worker 的有界缓存中按认证 Client、Routing
|
3. Dispatcher 从本地快照筛选 Upstream、协议、标签、TTL 和健康条件。
|
||||||
和会话标识查找仍符合当前快照的 Proxy。
|
4. 原子预留 Proxy 容量,建立到上游代理的连接。
|
||||||
4. Dispatcher 从本地快照筛选 Upstream、协议、标签、TTL 和健康条件;失效会话
|
5. 建连成功后转为 Active,传输结束后释放;失败则取消预留。
|
||||||
自动清除并按常规策略重新选择。
|
6. GET/HEAD 仅在响应提交前按策略重试;CONNECT 建立后不重放。
|
||||||
5. 原子预留 Proxy 容量,建立到上游代理的连接。
|
|
||||||
6. 建连成功后转为 Active,并在需要时写入不超过 Proxy 可用期的会话绑定;
|
|
||||||
传输结束后释放,失败则取消预留并清除该绑定。
|
|
||||||
7. GET/HEAD 仅在响应提交前按策略重试;CONNECT 建立后不重放。
|
|
||||||
|
|
||||||
### 4.2 独占提取
|
### 4.2 独占提取
|
||||||
|
|
||||||
|
|||||||
@ -53,10 +53,10 @@ deadline 内执行 HTTP/HTTPS/SOCKS5 BASIC、EGRESS 和 TARGET 探测并微批
|
|||||||
|
|
||||||
### proxy-loadgen
|
### proxy-loadgen
|
||||||
|
|
||||||
负载工具当前生成有界 HTTP、CONNECT 长连接和 Extract 请求,支持经 Gateway 请求 HTTPS 目标、
|
负载工具当前生成有界 HTTP 请求,支持经 Gateway 请求 HTTPS 目标、固定请求数或时长、
|
||||||
固定请求数或时长、目标 QPS、可重复请求头和请求体、连接复用和 JSON 指标输出。延迟统计使用
|
目标 QPS、可重复请求头和请求体、连接复用和 JSON 指标输出。延迟统计使用固定大小直方图,不会因长时间高 QPS
|
||||||
固定大小直方图,不会因长时间高 QPS 运行积压样本。故障注入和代表性集群报告仍待补齐;它是
|
运行积压样本。CONNECT 长连接、Extract 和故障注入场景仍待补齐;它是 100k QPS 结论的
|
||||||
100k QPS 结论的证据工具,不是业务进程。
|
证据工具,不是业务进程。
|
||||||
|
|
||||||
## 3. 依赖方向
|
## 3. 依赖方向
|
||||||
|
|
||||||
|
|||||||
@ -54,12 +54,12 @@ test/{fixtures,integration,e2e,load}/
|
|||||||
**Files:** `go.mod`, `.golangci.yml`, `README.md`, `scripts/verify.ps1`,
|
**Files:** `go.mod`, `.golangci.yml`, `README.md`, `scripts/verify.ps1`,
|
||||||
`.github/workflows/ci.yml`
|
`.github/workflows/ci.yml`
|
||||||
|
|
||||||
- [x] Create Go module `proxy-pool` with Go 1.26.
|
- [ ] Create module `github.com/proxy-pool/proxy-pool` with Go 1.26.
|
||||||
- [x] Pin YAML v4, pgx/v5, go-redis/v9, gRPC, protobuf, Prometheus, and x/sync.
|
- [ ] Pin YAML v4, pgx/v5, go-redis/v9, gRPC, protobuf, Prometheus, and x/sync.
|
||||||
- [x] Add `scripts/verify.ps1` that runs format check, `go vet`, unit tests, race tests,
|
- [x] Add `scripts/verify.ps1` that runs format check, `go vet`, unit tests, race tests,
|
||||||
and builds all commands, each test command bounded to 60 seconds.
|
and builds all commands, each test command bounded to 60 seconds.
|
||||||
- [x] Add CI for Windows and Linux with unit/race/build jobs.
|
- [x] Add CI for Windows and Linux with unit/race/build jobs.
|
||||||
- [x] Verify `go mod tidy`, `go test ./...`, and `go build ./cmd/...` succeed.
|
- [ ] Verify `go mod tidy`, `go test ./...`, and `go build ./cmd/...` succeed.
|
||||||
|
|
||||||
## Task 2: Strict Configuration
|
## Task 2: Strict Configuration
|
||||||
|
|
||||||
@ -80,18 +80,14 @@ test/{fixtures,integration,e2e,load}/
|
|||||||
|
|
||||||
- [x] Implement Proxy fields, UTC TTL precedence, canonical host/port, and unique key.
|
- [x] Implement Proxy fields, UTC TTL precedence, canonical host/port, and unique key.
|
||||||
- [x] Implement state transitions and reject illegal transitions.
|
- [x] Implement state transitions and reject illegal transitions.
|
||||||
- [x] Implement packed per-Proxy runtime counters with CAS Reserve, Commit, Cancel, Release.
|
- [ ] Implement sharded runtime counters with CAS Reserve, Commit, Cancel, Release.
|
||||||
- [x] Prove with 1,000 concurrent goroutines that effective capacity is never exceeded.
|
- [x] Prove with 1,000 concurrent goroutines that effective capacity is never exceeded.
|
||||||
- [x] Add race coverage and duplicate-release invariant metrics hook.
|
- [ ] Add race coverage and duplicate-release invariant metrics hook.
|
||||||
|
|
||||||
当前进度(2026-08-02):固定 Max 下的每 Proxy 打包 CAS、Cancel/Commit/Release
|
当前进度(2026-07-29):固定 Max 下的每 Proxy 打包 CAS、Cancel/Commit/Release
|
||||||
生命周期、重复终结、错误顺序和同一 Reservation 并发终结已通过领域测试;已退出
|
生命周期、重复终结、错误顺序和同一 Reservation 并发终结已通过领域测试;已退出
|
||||||
Proxy 会停止新预留、保留非零 Drain 计数,并在归零后由后续 Snapshot Apply 回收。
|
Proxy 会停止新预留、保留非零 Drain 计数,并在归零后由后续 Snapshot Apply 回收。
|
||||||
Snapshot 降低 `maxConcurrency` 时会保留既有 Active 工作、禁止新预留,直到运行计数
|
动态降容契约、低基数不变量指标和 Linux race 证据仍待完成。
|
||||||
低于新上限;运行态复用、降容和恢复预留均有回归测试。领域的
|
|
||||||
`CapacityInvariantObserver` 只发出固定生命周期违规枚举,Gateway 指标
|
|
||||||
`proxy_pool_gateway_capacity_invariant_violations_total{operation}` 不含 Proxy、请求或
|
|
||||||
Worker 维度;该测试纳入 Linux CI 的 race 范围,本机因 `CGO_ENABLED=0` 未执行 race。
|
|
||||||
|
|
||||||
## Task 4: Routing and Sequential Switching
|
## Task 4: Routing and Sequential Switching
|
||||||
|
|
||||||
@ -101,18 +97,12 @@ Worker 维度;该测试纳入 Linux CI 的 race 范围,本机因 `CGO_ENABLE
|
|||||||
- [x] Implement random, round-robin, weighted, least-connections, and sequential.
|
- [x] Implement random, round-robin, weighted, least-connections, and sequential.
|
||||||
- [x] Model upstream empty counters separately from per-routing current indexes.
|
- [x] Model upstream empty counters separately from per-routing current indexes.
|
||||||
- [x] Implement versioned CAS switch so simultaneous threshold observers advance once.
|
- [x] Implement versioned CAS switch so simultaneous threshold observers advance once.
|
||||||
- [x] Cover four-empty-then-success, five-empty, A-to-B-only, disabled references,
|
- [ ] Cover four-empty-then-success, five-empty, A-to-B-only, disabled references,
|
||||||
end behavior, and explicit onUnavailable.
|
end behavior, and explicit onUnavailable.
|
||||||
|
|
||||||
当前进度(2026-08-02):领域构造器与严格配置已统一 Sequential 至少两个
|
当前进度(2026-07-29):领域构造器与严格配置已统一 Sequential 至少两个
|
||||||
Upstream、`endBehavior` 默认 `stop`,并覆盖列表末端停止。Provider Stats 现为每次
|
Upstream、`endBehavior` 默认 `stop`,并覆盖列表末端停止;disabled candidate、
|
||||||
连续空结果分配单调代次;Controller 的公共 Sequential 协调器只接收有界通知,在独立
|
跨实例恢复和 `onUnavailable` 运行链仍待完成。
|
||||||
循环中读取权威配置、管理快照和 Stats,并以既有 `ExpectedCurrent` CAS 自动切换。
|
|
||||||
它会跳过禁用 Upstream,支持 `loop`/`stayLast`,同一空结果代次不会在循环后重复切换,
|
|
||||||
成功后立即广播完整 Snapshot。末端 `stop` 使用独立 `DisableRouting` 公用命令,
|
|
||||||
以 `ExpectedCurrent` CAS 原子停用 Routing,并在同一权威事务写入审计与 Outbox;
|
|
||||||
内存与 PostgreSQL Adapter 都运行相同并发、幂等和失败回滚契约。Gateway 已执行
|
|
||||||
reject/wait/direct,并会在快照刷新后看到停用状态。
|
|
||||||
|
|
||||||
## Task 5: Provider Fetch Classification and Scheduling
|
## Task 5: Provider Fetch Classification and Scheduling
|
||||||
|
|
||||||
@ -130,7 +120,7 @@ reject/wait/direct,并会在快照刷新后看到停用状态。
|
|||||||
|
|
||||||
**Files:** `internal/controller/pool/*.go`, `internal/domain/upstream/pool.go`, tests
|
**Files:** `internal/controller/pool/*.go`, `internal/domain/upstream/pool.go`, tests
|
||||||
|
|
||||||
- [x] Compute Available Slots from eligible Proxy capacity, Active, Reserved, TTL,
|
- [ ] Compute Available Slots from eligible Proxy capacity, Active, Reserved, TTL,
|
||||||
health, ownership, pending expected fetch, and gateway reserve.
|
health, ownership, pending expected fetch, and gateway reserve.
|
||||||
- [x] Implement pool.maxSize and fetch.maxTotal as distinct counters.
|
- [x] Implement pool.maxSize and fetch.maxTotal as distinct counters.
|
||||||
- [x] Allocate each Proxy to one Worker with epoch/version/expiry ownership.
|
- [x] Allocate each Proxy to one Worker with epoch/version/expiry ownership.
|
||||||
@ -190,10 +180,8 @@ reject/wait/direct,并会在快照刷新后看到停用状态。
|
|||||||
- [x] Implement the Worker heartbeat receiving path and session lifecycle.
|
- [x] Implement the Worker heartbeat receiving path and session lifecycle.
|
||||||
- [x] Keep Provider output in Redis TTL activity state and node memory only; keep the
|
- [x] Keep Provider output in Redis TTL activity state and node memory only; keep the
|
||||||
Gateway request path on immutable local snapshots with no Redis/PostgreSQL calls.
|
Gateway request path on immutable local snapshots with no Redis/PostgreSQL calls.
|
||||||
- [x] Expose Distribution extraction/status and Admin status/audit/enable/disable/switch/reload
|
- [x] Expose Distribution extraction/status and Admin status/enable/disable/switch/reload
|
||||||
HTTP handlers and contracts with credential-level endpoint permissions and
|
HTTP handlers and contracts.
|
||||||
Distribution credential-level extraction boundaries; enforce matched
|
|
||||||
Gateway credential routing boundaries before local dispatch.
|
|
||||||
- [x] Add Compose-backed Redis 8.2 integration and shared Adapter contract tests.
|
- [x] Add Compose-backed Redis 8.2 integration and shared Adapter contract tests.
|
||||||
- [x] Add PostgreSQL management Adapter and Compose-backed integration tests.
|
- [x] Add PostgreSQL management Adapter and Compose-backed integration tests.
|
||||||
|
|
||||||
@ -217,8 +205,8 @@ Distribution/Admin 服务构造、错误合并和资源关闭。生产 Provider
|
|||||||
停机;Admin disable 会取消 Runtime,reload 在提交前预检并在发布后替换运行实例。
|
停机;Admin disable 会取消 Runtime,reload 在提交前预检并在发布后替换运行实例。
|
||||||
组合 fixture 已验证隔离 Redis namespace 下的选主、Provider HTTP 调用、模板解析
|
组合 fixture 已验证隔离 Redis namespace 下的选主、Provider HTTP 调用、模板解析
|
||||||
和活动池写入。Controller Metrics 独立入口现已提供 `/livez`、`/readyz` 与基础
|
和活动池写入。Controller Metrics 独立入口现已提供 `/livez`、`/readyz` 与基础
|
||||||
Prometheus 运行时指标,三监听器隔离已通过测试;Checker、Gateway、Drain、
|
Prometheus 运行时指标,三监听器隔离已通过测试;业务指标仍待
|
||||||
Provider、Extraction 与容量业务指标已接入;进程级密钥安全结构化日志也已落地。双存储 bootstrap 已通过 PostgreSQL 18 + Redis 8.2 组合 fixture,覆盖
|
实现。双存储 bootstrap 已通过 PostgreSQL 18 + Redis 8.2 组合 fixture,覆盖
|
||||||
迁移、启动配置提交、Readiness、Admin Status 和 Metrics 探针。
|
迁移、启动配置提交、Readiness、Admin Status 和 Metrics 探针。
|
||||||
|
|
||||||
WorkerControlPlane 现已接入 Controller 生命周期:Register、ACK 和 Runtime
|
WorkerControlPlane 现已接入 Controller 生命周期:Register、ACK 和 Runtime
|
||||||
@ -314,20 +302,12 @@ Ticket/排除快照/Runtime 零计数闭环。
|
|||||||
Controller 即可生效。Redis 任务存储现已扩展 BASIC/EGRESS/TARGET 的独立有界索引;路由目标
|
Controller 即可生效。Redis 任务存储现已扩展 BASIC/EGRESS/TARGET 的独立有界索引;路由目标
|
||||||
Profile 在启用 Routing 与 Upstream 的组合上才进入调度。
|
Profile 在启用 Routing 与 Upstream 的组合上才进入调度。
|
||||||
|
|
||||||
补充进度(2026-08-02):配置停用已进入 ownership Drain 编排。Controller 以静态配置
|
|
||||||
与 Admin 管理态的交集生成完整、版本化的 Upstream 策略视图;Redis 只从对应 `owned` 索引
|
|
||||||
有界读取仍可用的已归属 Proxy。候选携带 Upstream revision,`ownership.lua` 在复用既有
|
|
||||||
Ticket/排除 Snapshot/Runtime 零计数闭环前,原子复核策略仍为停用、Proxy source、Worker、
|
|
||||||
assignment epoch 和未过期租约。重启用后的策略 revision 会使旧候选返回无操作。Provider
|
|
||||||
Supervisor 也改为同时服从静态配置与管理态,消除两条启停消费链的不一致。
|
|
||||||
|
|
||||||
补充进度(2026-08-02):已新增 `proxy-loadgen` HTTP 与 CONNECT 长连接场景。固定请求数
|
补充进度(2026-08-02):已新增 `proxy-loadgen` HTTP 与 CONNECT 长连接场景。固定请求数
|
||||||
和固定时长两种模式均通过固定 worker 数与有界派发通道执行,可选 QPS 限速;报告使用固定大小
|
和固定时长两种模式均通过固定 worker 数与有界派发通道执行,可选 QPS 限速;报告使用固定大小
|
||||||
延迟直方图,输出状态分类、CONNECT 建立数、Extract 校验数、吞吐和 Go 内存/GC 快照。CONNECT
|
延迟直方图,输出状态分类、CONNECT 建立数、Extract 校验数、吞吐和 Go 内存/GC 快照。CONNECT
|
||||||
以原始 TCP 握手连接 HTTP Gateway,建连成功后按 `hold` 保持,且不透明读取隧道内容。`extract`
|
以原始 TCP 握手连接 HTTP Gateway,建连成功后按 `hold` 保持,且不透明读取隧道内容。`extract`
|
||||||
场景会自动生成独立 Request/Idempotency 标识,校验返回数量与单响应 ID 唯一性,且不记录地址或
|
场景会自动生成独立 Request/Idempotency 标识,校验返回数量与单响应 ID 唯一性,且不记录地址或
|
||||||
凭据。报告可选按最大错误率和 p99 延迟门禁;违反阈值时仍完整写出 JSON 证据并以
|
凭据。故障注入以及代表性集群报告仍未实现。
|
||||||
专用退出码失败。故障注入以及代表性集群报告仍未实现。
|
|
||||||
|
|
||||||
## Task 12: Machine-readable Contracts
|
## Task 12: Machine-readable Contracts
|
||||||
|
|
||||||
@ -338,27 +318,24 @@ Supervisor 也改为同时服从静态配置与管理态,消除两条启停消
|
|||||||
and idempotency behavior.
|
and idempotency behavior.
|
||||||
- [x] Specify Worker register, snapshot, delta, ACK, report, heartbeat, ownership drain,
|
- [x] Specify Worker register, snapshot, delta, ACK, report, heartbeat, ownership drain,
|
||||||
and resync messages.
|
and resync messages.
|
||||||
- [x] Validate OpenAPI contracts and compile protobuf descriptors/generated-code drift in CI.
|
- [ ] Validate OpenAPI and compile protobuf descriptors in CI.
|
||||||
|
|
||||||
当前进度(2026-07-29):Distribution/Admin OpenAPI 已由 Go 测试在双平台 CI
|
当前进度(2026-07-29):Distribution/Admin OpenAPI 已由 Go 测试在双平台 CI
|
||||||
校验本地 `$ref` 闭合、operationId 唯一、响应存在及 security scheme 引用;
|
校验本地 `$ref` 闭合、operationId 唯一、响应存在及 security scheme 引用;
|
||||||
`scripts/verify-proto.ps1` 已可复现编译包含 imports/source info 的 descriptor,
|
`scripts/verify-proto.ps1` 已可复现编译包含 imports/source info 的 descriptor,
|
||||||
并使用 SHA-256 固定的 `protoc` 35.0 安装器在 CI 完整验证 descriptor 与生成代码漂移;
|
并在本地存在 `protoc` 时进入完整验证;完整 OAS 工具验证与 CI 强制安装/执行
|
||||||
OpenAPI 结构契约由 Go 测试在双平台 CI 执行;`scripts/verify-openapi.ps1` 固定
|
`protoc` 仍待完成。
|
||||||
`@redocly/cli@2.25.4`,按 OpenAPI 3.1 最小规则集验证两份文档并将 Tag 描述作为错误。
|
|
||||||
|
|
||||||
## Task 13: Deployment and Observability
|
## Task 13: Deployment and Observability
|
||||||
|
|
||||||
**Files:** `deploy/**`, `internal/platform/**`, `docs/operations/**`
|
**Files:** `deploy/**`, `internal/platform/**`, `docs/operations/**`
|
||||||
|
|
||||||
- [x] Add Compose for local Controller/Gateway/Checker/PostgreSQL/Redis/Prometheus/
|
- [ ] Add Compose for local Controller/Gateway/Checker/PostgreSQL/Redis/Prometheus/
|
||||||
Grafana/HAProxy.
|
Grafana/HAProxy.
|
||||||
- [x] Add Kubernetes Deployments, Services, PDBs, HPA, NetworkPolicy, Secrets examples,
|
- [ ] Add Kubernetes Deployments, Services, PDBs, HPA, NetworkPolicy, Secrets examples,
|
||||||
probes, resource limits, topology spread, and graceful termination.
|
probes, resource limits, topology spread, and graceful termination.
|
||||||
- [ ] Add the production Kubernetes mTLS identity overlay, certificate rotation and unique
|
- [ ] Add low-cardinality Prometheus metrics and structured secret-safe logs.
|
||||||
elastic Worker identity wiring.
|
- [ ] Document backup, recovery, rollout, rollback, capacity, kernel, file descriptor,
|
||||||
- [x] Add low-cardinality Prometheus metrics and structured secret-safe logs.
|
|
||||||
- [x] Document backup, recovery, rollout, rollback, capacity, kernel, file descriptor,
|
|
||||||
NAT/conntrack, and incident runbooks.
|
NAT/conntrack, and incident runbooks.
|
||||||
|
|
||||||
当前进度(2026-08-02):已接入 Checker 任务流指标
|
当前进度(2026-08-02):已接入 Checker 任务流指标
|
||||||
@ -367,22 +344,7 @@ OpenAPI 结构契约由 Go 测试在双平台 CI 执行;`scripts/verify-openap
|
|||||||
`BASIC`、`EGRESS`、`TARGET` 级别及 `accepted`、`rejected` 结果。Gateway 还暴露
|
`BASIC`、`EGRESS`、`TARGET` 级别及 `accepted`、`rejected` 结果。Gateway 还暴露
|
||||||
`proxy_pool_gateway_outcomes_total{stage,result}` 和
|
`proxy_pool_gateway_outcomes_total{stage,result}` 和
|
||||||
`proxy_pool_gateway_outcome_queue_dropped_total`,其中阶段和结果均为固定枚举。Provider、
|
`proxy_pool_gateway_outcome_queue_dropped_total`,其中阶段和结果均为固定枚举。Provider、
|
||||||
拉取已暴露 `proxy_pool_controller_provider_fetch_results_total{class}`、
|
提取、容量指标与密钥安全的结构化日志仍待补齐。
|
||||||
`proxy_pool_controller_provider_valid_candidates_total` 与
|
|
||||||
`proxy_pool_controller_provider_new_proxies_total`;`class` 仅允许 `valid`、`empty`、
|
|
||||||
`duplicate_only`、`error`。Extraction 还暴露固定 `result` 的请求次数、请求数与响应
|
|
||||||
交付数,幂等重放按响应交付统计。容量指标由既有 Provider 库存对账周期聚合,不进入
|
|
||||||
Gateway 热路径。`platform/logging` 以 JSON `slog` 输出进程级致命错误,字段、URL
|
|
||||||
用户信息、查询 Secret 和错误对象均经过脱敏,且不在请求热路径逐条写日志。
|
|
||||||
CI 另有独立 Deployment job,在占位凭据下渲染 Compose,并使用 `kubectl kustomize`
|
|
||||||
渲染 Kubernetes base 和单副本 development mTLS Overlay;该 job 同时展开测试 Compose、
|
|
||||||
校验本地 Compose 配置。它不启动容器、不访问真实存储或密钥。
|
|
||||||
独立 Image job 会构建 Linux 多阶段应用镜像,覆盖 Dockerfile、Linux 交叉编译与运行时层;
|
|
||||||
容器服务编排和业务运行链路仍需在具备镜像网络的环境中验证。
|
|
||||||
development Overlay 已挂载独立 Controller/Gateway/Checker TLS Secret、启用 Checker 和最小
|
|
||||||
网络策略,并从固定开发证书派生 `gateway-a`、`checker-a` 身份且 HPA 锁为单副本,不能代替生产证书轮换。
|
|
||||||
控制面 TLS 叶证书和信任根已在每次新握手时从文件加载,短暂文件不一致时保留最后一次有效材料;
|
|
||||||
生产 Overlay 仍必须由工作负载身份系统提供每副本唯一 SPIFFE URI 和弹性身份注入。
|
|
||||||
|
|
||||||
## Task 14: Documentation, Examples, and Diagrams
|
## Task 14: Documentation, Examples, and Diagrams
|
||||||
|
|
||||||
@ -392,27 +354,14 @@ development Overlay 已挂载独立 Controller/Gateway/Checker TLS Secret、启
|
|||||||
reference, API guide, deployment guide, security model, testing guide, and roadmap.
|
reference, API guide, deployment guide, security model, testing guide, and roadmap.
|
||||||
- [x] Provide at least 20 validated configuration examples.
|
- [x] Provide at least 20 validated configuration examples.
|
||||||
- [x] Provide at least 30 Mermaid architecture, flow, sequence, state, and failure diagrams.
|
- [x] Provide at least 30 Mermaid architecture, flow, sequence, state, and failure diagrams.
|
||||||
- [x] Generate `proxy-pool-docs-v1.0.zip` from versioned documentation assets.
|
- [ ] Generate `proxy-pool-docs-v1.0.zip` from versioned documentation assets.
|
||||||
|
|
||||||
`scripts/package-docs.ps1` 会将 README、`docs/`、图表、OpenAPI、Proto 契约与部署手册复制至临时目录,
|
|
||||||
生成包含 Git revision、文件大小和 SHA-256 的 `manifest.json`,再以临时 ZIP 原子替换目标。
|
|
||||||
默认输出为被 Git 忽略的 `dist/proxy-pool-docs-v1.0.zip`;Go 回归测试实际执行脚本并校验归档内容。
|
|
||||||
|
|
||||||
## Task 15: Completion Audit
|
## Task 15: Completion Audit
|
||||||
|
|
||||||
- [x] Map every requirement ID to code, test, contract, document, or verified runtime evidence.
|
- [ ] Map every requirement ID to code, test, contract, document, or verified runtime evidence.
|
||||||
- [ ] Run `gofmt`, `go vet`, unit tests, race tests, builds, contract validation, and
|
- [ ] Run `gofmt`, `go vet`, unit tests, race tests, builds, contract validation, and
|
||||||
documentation link/example validation.
|
documentation link/example validation.
|
||||||
- [x] Run bounded local performance benchmarks; label 100k QPS as unverified until a
|
- [ ] Run bounded local performance benchmarks; label 100k QPS as unverified until a
|
||||||
representative cluster load run exists.
|
representative cluster load run exists.
|
||||||
|
- [ ] Confirm no TODO/TBD/placeholders, secrets, unbounded queues, high-cardinality metric
|
||||||
`scripts/benchmark-gateway.ps1` 固定使用 `-count=1 -benchtime=1x` 分别执行 100k 索引调度、
|
|
||||||
Round Robin 和 Snapshot Apply 基准,并将 Git revision 与原始输出保存到 `dist/`。该基线不包含
|
|
||||||
网络、TLS、上游 RTT、真实连接或多 Worker 协调,不能替代代表性集群报告。
|
|
||||||
- [x] Confirm no TODO/TBD/placeholders, secrets, unbounded queues, high-cardinality metric
|
|
||||||
labels, extraction Lease APIs, or conflicting maxSize semantics remain.
|
labels, extraction Lease APIs, or conflicting maxSize semantics remain.
|
||||||
|
|
||||||
`docs` 回归测试扫描生产源与部署清单中的未完成标记和私钥材料;OpenAPI 测试拒绝
|
|
||||||
Extract Lease/Release/Renew 路径,Gateway Outcome Queue 构造器和配置校验拒绝无界容量,
|
|
||||||
指标/部署契约拒绝高基数标签,`pool.maxSize` 与累计 `fetch.maxTotal` 的边界由配置和
|
|
||||||
活动池契约测试共同锁定。示例、CI fixture 与环境变量引用不是未实现占位或生产 Secret。
|
|
||||||
|
|||||||
@ -77,61 +77,6 @@ func TestDocumentedGoCommandsReferenceExistingTargets(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestRunbookCoversHostAndNetworkCapacityBaseline(t *testing.T) {
|
|
||||||
content := readDocument(t, filepath.Join(repositoryRoot(t), "docs", "operations", "runbook.md"))
|
|
||||||
for _, required := range []string{
|
|
||||||
"## 8. 备份与恢复",
|
|
||||||
"## 10. 主机与网络容量基线",
|
|
||||||
"ulimit -n",
|
|
||||||
"conntrack",
|
|
||||||
"NAT",
|
|
||||||
"回滚",
|
|
||||||
} {
|
|
||||||
if !strings.Contains(content, required) {
|
|
||||||
t.Errorf("runbook does not document %q", required)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestRepositoryAuditRejectsUnfinishedSourceAndPrivateKeyMaterial(t *testing.T) {
|
|
||||||
repositoryRoot := repositoryRoot(t)
|
|
||||||
unfinished := regexp.MustCompile(`(?i)\b(?:TODO|TBD|FIXME)\b`)
|
|
||||||
privateKey := regexp.MustCompile(`-----BEGIN (?:[A-Z ]+ )?PRIVATE KEY-----`)
|
|
||||||
for _, directory := range []string{".github", "api", "cmd", "deploy", "internal", "scripts"} {
|
|
||||||
root := filepath.Join(repositoryRoot, directory)
|
|
||||||
err := filepath.WalkDir(root, func(path string, entry os.DirEntry, err error) error {
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if entry.IsDir() {
|
|
||||||
if entry.Name() == ".control-plane-tls" || strings.HasPrefix(entry.Name(), ".tmp") || entry.Name() == "dist" {
|
|
||||||
return filepath.SkipDir
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
switch strings.ToLower(filepath.Ext(path)) {
|
|
||||||
case ".go", ".yaml", ".yml", ".ps1":
|
|
||||||
default:
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
content, readErr := os.ReadFile(path)
|
|
||||||
if readErr != nil {
|
|
||||||
return readErr
|
|
||||||
}
|
|
||||||
if unfinished.Match(content) {
|
|
||||||
t.Errorf("%s contains an unfinished implementation marker", relativeTestName(repositoryRoot, path))
|
|
||||||
}
|
|
||||||
if privateKey.Match(content) {
|
|
||||||
t.Errorf("%s contains private key material", relativeTestName(repositoryRoot, path))
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("audit %s: %v", directory, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func markdownDocuments(t *testing.T, repositoryRoot string) []string {
|
func markdownDocuments(t *testing.T, repositoryRoot string) []string {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
documents := []string{filepath.Join(repositoryRoot, "README.md")}
|
documents := []string{filepath.Join(repositoryRoot, "README.md")}
|
||||||
|
|||||||
@ -2,29 +2,28 @@
|
|||||||
|
|
||||||
## 架构与一致性
|
## 架构与一致性
|
||||||
|
|
||||||
- [x] Gateway 热路径依赖审计确认无 PostgreSQL、Redis、Provider 或模板执行。
|
- [ ] Gateway 热路径依赖审计确认无 PostgreSQL、Redis、Provider 或模板执行。
|
||||||
- [x] 每个 Proxy 同一时刻最多归属一个 Worker,ownership epoch 单调。
|
- [ ] 每个 Proxy 同一时刻最多归属一个 Worker,ownership epoch 单调。
|
||||||
- [x] Reserved -> Active 使用单个原子转换,无超卖与负计数。
|
- [ ] Reserved -> Active 使用单个原子转换,无超卖与负计数。
|
||||||
- [x] Sequential 并发 Empty 只切换一次,旧 Upstream Proxy 自然耗尽;末端 `stop`
|
- [ ] Sequential 并发 Empty 只切换一次,旧 Upstream Proxy 自然耗尽。
|
||||||
使用权威 CAS 停用 Routing 并刷新 Gateway 快照。
|
|
||||||
- [x] `pool.maxSize` 与 `fetch.maxTotal` 分别按当前库存和累计获取计数。
|
- [x] `pool.maxSize` 与 `fetch.maxTotal` 分别按当前库存和累计获取计数。
|
||||||
- [x] Extract 只有 `AVAILABLE -> EXTRACTED`,OpenAPI 不存在 release/renew。
|
- [ ] Extract 只有 `AVAILABLE -> EXTRACTED`,OpenAPI 不存在 release/renew。
|
||||||
- [x] Extract 状态更新和短期幂等结果位于同一个 Redis 原子操作。
|
- [ ] Extract 状态更新和短期幂等结果位于同一个 Redis 原子操作。
|
||||||
- [x] PostgreSQL 中不存在 Proxy 明细或逐次提取记录。
|
- [ ] PostgreSQL 中不存在 Proxy 明细或逐次提取记录。
|
||||||
- [x] `partial` 和 `allOrNothing` 均通过 Redis 原子并发测试。
|
- [ ] `partial` 和 `allOrNothing` 均通过 Redis 原子并发测试。
|
||||||
- [x] `reserveForGateway` 在所有提取路径上统一执行。
|
- [ ] `reserveForGateway` 在所有提取路径上统一执行。
|
||||||
|
|
||||||
## 安全
|
## 安全
|
||||||
|
|
||||||
- [x] 非回环监听均配置 Auth 或 allowCIDRs,严格模式已开启。
|
- [ ] 非回环监听均配置 Auth 或 allowCIDRs,严格模式已开启。
|
||||||
- [x] Gateway、Distribution、Admin 凭据和权限相互独立。
|
- [ ] Gateway、Distribution、Admin 凭据和权限相互独立。
|
||||||
- [ ] trusted proxy 只包含受控 LoadBalancer/Ingress 网段。
|
- [ ] trusted proxy 只包含受控 LoadBalancer/Ingress 网段。
|
||||||
- [x] 解析前后均拦截私网、回环、链路本地、元数据地址与 DNS Rebinding。
|
- [ ] 解析前后均拦截私网、回环、链路本地、元数据地址与 DNS Rebinding。
|
||||||
- [ ] Secret 由外部密钥系统注入,镜像、ConfigMap、日志没有明文。
|
- [ ] Secret 由外部密钥系统注入,镜像、ConfigMap、日志没有明文。
|
||||||
- [ ] 所有 Controller 使用同一枚至少 32 字节的独立配置 HMAC 密钥,并完成轮换演练。
|
- [ ] 所有 Controller 使用同一枚至少 32 字节的独立配置 HMAC 密钥,并完成轮换演练。
|
||||||
- [ ] Pod 以非 root、只读根文件系统、无 Linux capabilities 运行。
|
- [ ] Pod 以非 root、只读根文件系统、无 Linux capabilities 运行。
|
||||||
- [ ] NetworkPolicy 默认拒绝,外部数据库/Redis/Provider 网段已收紧。
|
- [ ] NetworkPolicy 默认拒绝,外部数据库/Redis/Provider 网段已收紧。
|
||||||
- [x] Provider 模板有响应大小、执行时间、函数与外部访问限制。
|
- [ ] Provider 模板有响应大小、执行时间、函数与外部访问限制。
|
||||||
|
|
||||||
## 可用性
|
## 可用性
|
||||||
|
|
||||||
@ -32,9 +31,9 @@
|
|||||||
- [ ] Redis 跨可用区并受监控,完成丢失后由 Provider 重建活动池的演练。
|
- [ ] Redis 跨可用区并受监控,完成丢失后由 Provider 重建活动池的演练。
|
||||||
- [ ] Gateway、Controller、Checker 均跨主机/可用区分散。
|
- [ ] Gateway、Controller、Checker 均跨主机/可用区分散。
|
||||||
- [ ] PDB、优雅终止与最大连接时长的组合经过驱逐测试。
|
- [ ] PDB、优雅终止与最大连接时长的组合经过驱逐测试。
|
||||||
- [x] Controller 断线时 Gateway 在 `maxStaleAge` 内继续,超限拒绝新请求。
|
- [ ] Controller 断线时 Gateway 在 `maxStaleAge` 内继续,超限拒绝新请求。
|
||||||
- [x] Worker 崩溃后 ownership 只在租约过期后再分配。
|
- [ ] Worker 崩溃后 ownership 只在租约过期后再分配。
|
||||||
- [x] Snapshot 版本缺口触发全量同步,旧 Delta 被拒绝。
|
- [ ] Snapshot 版本缺口触发全量同步,旧 Delta 被拒绝。
|
||||||
|
|
||||||
## 性能
|
## 性能
|
||||||
|
|
||||||
@ -42,15 +41,15 @@
|
|||||||
- [ ] 完成 10k 稳态、100k 峰值、CONNECT 活跃连接和建连速率独立测试。
|
- [ ] 完成 10k 稳态、100k 峰值、CONNECT 活跃连接和建连速率独立测试。
|
||||||
- [ ] 在最大可用区失效时仍满足容量和延迟 SLO。
|
- [ ] 在最大可用区失效时仍满足容量和延迟 SLO。
|
||||||
- [ ] 目标利用率不高于 60%,HPA 缩容稳定窗口不低于 10 分钟。
|
- [ ] 目标利用率不高于 60%,HPA 缩容稳定窗口不低于 10 分钟。
|
||||||
- [x] 队列、buffer、日志和结果上报全部有界。
|
- [ ] 队列、buffer、日志和结果上报全部有界。
|
||||||
- [ ] p99 Dispatch 预算、端到端延迟、错误率、CPU、RSS、FD 和网络有原始证据。
|
- [ ] p99 Dispatch 预算、端到端延迟、错误率、CPU、RSS、FD 和网络有原始证据。
|
||||||
|
|
||||||
## 观测与值班
|
## 观测与值班
|
||||||
|
|
||||||
- [x] 仪表盘覆盖 Gateway、库存、Provider、Extract、Snapshot、Checker 和 Drain。
|
- [ ] 仪表盘覆盖 Gateway、Routing、Upstream、Provider、Extract、Snapshot、Checker。
|
||||||
- [ ] 告警有负责人、严重级别、Runbook 链接和演练记录。
|
- [ ] 告警有负责人、严重级别、Runbook 链接和演练记录。
|
||||||
- [x] 指标无 Proxy IP、Client ID、Session、完整 URL 或 request ID 高基数标签。
|
- [ ] 指标无 Proxy IP、Client ID、Session、完整 URL 或 request ID 高基数标签。
|
||||||
- [x] 日志脱敏已用真实 Secret fixture 验证。
|
- [ ] 日志脱敏已用真实 Secret fixture 验证。
|
||||||
- [ ] 值班人员完成 PostgreSQL、Redis、Snapshot、容量与 Extract 故障演练。
|
- [ ] 值班人员完成 PostgreSQL、Redis、Snapshot、容量与 Extract 故障演练。
|
||||||
|
|
||||||
## 发布门禁
|
## 发布门禁
|
||||||
|
|||||||
@ -24,10 +24,11 @@
|
|||||||
`cmd/proxy-controller` 已完成配置单次加载、PostgreSQL 迁移、Redis 活动池、
|
`cmd/proxy-controller` 已完成配置单次加载、PostgreSQL 迁移、Redis 活动池、
|
||||||
Distribution/Admin/Metrics 独立监听和有界停机装配。Provider 自动补池、分布式
|
Distribution/Admin/Metrics 独立监听和有界停机装配。Provider 自动补池、分布式
|
||||||
配额、动态重载和 Admin 低基数统计已装配;Controller 已装配 Redis BASIC/EGRESS/TARGET 任务 broker,
|
配额、动态重载和 Admin 低基数统计已装配;Controller 已装配 Redis BASIC/EGRESS/TARGET 任务 broker,
|
||||||
`proxy-checker` 可执行 HTTP/HTTPS/SOCKS5 BASIC/EGRESS/TARGET 探测。`proxy-loadgen` 已提供有界 HTTP、
|
`proxy-checker` 可执行 HTTP/HTTPS/SOCKS5 BASIC/EGRESS/TARGET 探测。`proxy-loadgen` 已提供有界 HTTP
|
||||||
CONNECT 长连接和 Extract 场景,并可按最大错误率和 p99 延迟输出机器可判定的验收结果;
|
请求场景;CONNECT 长连接/Extract 压测与完整 mTLS 环境 Overlay 仍属于
|
||||||
它不替代代表性集群压测。Compose 已声明本地 Controller/Gateway/Checker mTLS 拓扑,
|
`implementation-plan.md` 后续任务。
|
||||||
但本机尚未完成容器端到端启动;Kubernetes 仍需要环境专属的工作负载身份 Overlay。
|
因此 Compose/Kubernetes 资产当前仍用于评审网络、资源、探针和依赖关系,不能
|
||||||
|
视为完整可运行拓扑。
|
||||||
|
|
||||||
### 2.1 前置条件
|
### 2.1 前置条件
|
||||||
|
|
||||||
@ -49,13 +50,6 @@ $env:PROVIDER_A_TOKEN = "PROVIDER_A_TOKEN"
|
|||||||
$env:PROVIDER_B_TOKEN = "PROVIDER_B_TOKEN"
|
$env:PROVIDER_B_TOKEN = "PROVIDER_B_TOKEN"
|
||||||
```
|
```
|
||||||
|
|
||||||
首次启动 Compose 前,生成不入库的本地控制面证书。生成器要求目标目录为空,避免覆写
|
|
||||||
已有密钥:
|
|
||||||
|
|
||||||
```powershell
|
|
||||||
.\scripts\generate-local-controlplane-certs.ps1
|
|
||||||
```
|
|
||||||
|
|
||||||
### 2.3 静态检查
|
### 2.3 静态检查
|
||||||
|
|
||||||
```powershell
|
```powershell
|
||||||
@ -63,10 +57,6 @@ docker compose -f deploy/docker-compose.yml config
|
|||||||
kubectl kustomize deploy/kubernetes/base > rendered.yaml
|
kubectl kustomize deploy/kubernetes/base > rendered.yaml
|
||||||
```
|
```
|
||||||
|
|
||||||
Compose 启动后,Gateway 的 `/readyz` 需要先接收有效 Snapshot;Checker 的 `/readyz`
|
|
||||||
需要成功建立一次任务领取流,并会在后续领取失败时回到未就绪。两者的 `/livez` 只表示
|
|
||||||
进程仍在运行。
|
|
||||||
|
|
||||||
本地 Compose 的 Redis 只作为可重建短效状态 fixture,固定使用
|
本地 Compose 的 Redis 只作为可重建短效状态 fixture,固定使用
|
||||||
`--appendonly no --save ""`,且不挂载 `/data` 或命名卷。真实 Redis 8.2 契约可
|
`--appendonly no --save ""`,且不挂载 `/data` 或命名卷。真实 Redis 8.2 契约可
|
||||||
通过 `.\scripts\test-redis.ps1` 执行;脚本使用唯一命名空间并在结束时定向清理,
|
通过 `.\scripts\test-redis.ps1` 执行;脚本使用唯一命名空间并在结束时定向清理,
|
||||||
@ -113,10 +103,6 @@ Fetch 应表现为 Error 与退避,不应增加 Empty 计数,也不影响已
|
|||||||
4. 根据集群 CNI 能力收紧 NetworkPolicy 的外部网段。
|
4. 根据集群 CNI 能力收紧 NetworkPolicy 的外部网段。
|
||||||
5. 在预发布环境完成数据库向前兼容迁移,再发布 Controller。
|
5. 在预发布环境完成数据库向前兼容迁移,再发布 Controller。
|
||||||
|
|
||||||
环境 Overlay 应保持 ConfigMap 内容哈希,令配置变化进入 Deployment Pod 模板并触发滚动更新。
|
|
||||||
由外部系统以固定名称注入的 Secret 不会天然改变 Pod 模板;完成 Secret 轮换后,必须按受影响
|
|
||||||
工作负载执行受控滚动,并先验证新旧信任根的重叠窗口。
|
|
||||||
|
|
||||||
### 3.2 服务端应用顺序
|
### 3.2 服务端应用顺序
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
@ -152,12 +138,6 @@ Proxy ID、IP、Checker ID、目标 URL、Client ID 或凭据加入指标标签
|
|||||||
|
|
||||||
Gateway 指标同样使用固定标签集:
|
Gateway 指标同样使用固定标签集:
|
||||||
|
|
||||||
每个 Gateway Worker 还暴露 requests_total、requests_in_flight 与 active_tunnels
|
|
||||||
三类本地连接指标。protocol 标签只允许 HTTP、CONNECT;请求指标覆盖从 Handler
|
|
||||||
接受请求到全部转发、拒绝或隧道关闭的生命周期。活跃隧道只在成功写出 CONNECT 200
|
|
||||||
并开始 relay 后增加,关闭后立即减少。这些指标用于核对入口准入、连接池调优、
|
|
||||||
文件描述符预算和长连接排空,不得按 Proxy、路由、目标、Client 或凭据拆分。
|
|
||||||
|
|
||||||
- `proxy_pool_gateway_outcomes_total{stage,result}`:每次代理尝试在最远完成阶段的成功/失败数量。
|
- `proxy_pool_gateway_outcomes_total{stage,result}`:每次代理尝试在最远完成阶段的成功/失败数量。
|
||||||
- `proxy_pool_gateway_outcome_queue_dropped_total`:Outcome 本地有界队列已满后丢弃的观测数量。
|
- `proxy_pool_gateway_outcome_queue_dropped_total`:Outcome 本地有界队列已满后丢弃的观测数量。
|
||||||
|
|
||||||
@ -290,9 +270,7 @@ Outbox 发布器必须以稳定 consumer ID 有界领取;发布成功后原子
|
|||||||
2. 429 尊重 `Retry-After`,其余 Error 使用指数退避和 jitter。
|
2. 429 尊重 `Retry-After`,其余 Error 使用指数退避和 jitter。
|
||||||
3. Error 不增加 Empty;合法候选为零才增加 Empty。
|
3. Error 不增加 Empty;合法候选为零才增加 Empty。
|
||||||
4. Duplicate-only 重置 Empty 并记录独立指标。
|
4. Duplicate-only 重置 Empty 并记录独立指标。
|
||||||
5. 达到 Empty 阈值后每条受影响 Routing 只原子切换一次;最后一个可用 Upstream
|
5. 达到 Empty 阈值后每条受影响 Routing 只原子切换一次。
|
||||||
仍持续 Empty 且配置为 `stop` 时,确认审计中存在 `disable_routing` 和
|
|
||||||
`routing.disabled` Outbox 事件,并检查 Gateway 已收到关闭该 Routing 的完整快照。
|
|
||||||
|
|
||||||
### 7.5 Gateway 容量耗尽
|
### 7.5 Gateway 容量耗尽
|
||||||
|
|
||||||
@ -353,46 +331,3 @@ ownership epoch、活动池 TTL、短期幂等窗口、outbox 和配置 Revision
|
|||||||
5. 所有副本应用后撤销旧 Secret。
|
5. 所有副本应用后撤销旧 Secret。
|
||||||
|
|
||||||
Proxy 凭据轮换必须增加 `credentialVersion`,确保唯一键不会把新旧凭据错误合并。
|
Proxy 凭据轮换必须增加 `credentialVersion`,确保唯一键不会把新旧凭据错误合并。
|
||||||
|
|
||||||
控制面 mTLS 的叶证书、私钥和信任根会在每次新的 TLS 握手时重新读取;Kubernetes 投影
|
|
||||||
Secret 或 CSI 写入短暂不完整时保留最后一次有效材料。既有 gRPC 长连接不会被强制替换,
|
|
||||||
因此根 CA 轮换必须先发布包含新旧根的重叠信任包、确认新连接成功,再按 PDB 约束滚动
|
|
||||||
重启或排空旧 Worker,最后才撤销旧根并使旧流重新认证。
|
|
||||||
|
|
||||||
## 10. 主机与网络容量基线
|
|
||||||
|
|
||||||
在预发布压测和每次生产扩容前,记录 Gateway 节点、容器运行时和 Pod 内的下列只读基线;
|
|
||||||
不要在故障处理中临时提高内核限制,任何变更都必须经过压测和变更评审。
|
|
||||||
|
|
||||||
```bash
|
|
||||||
ulimit -n
|
|
||||||
cat /proc/self/limits | grep 'open files'
|
|
||||||
sysctl fs.file-max net.core.somaxconn net.ipv4.ip_local_port_range
|
|
||||||
test -r /proc/sys/net/netfilter/nf_conntrack_max && cat /proc/sys/net/netfilter/nf_conntrack_max
|
|
||||||
ss -s
|
|
||||||
```
|
|
||||||
|
|
||||||
同时从节点或受监控的 NAT 网关采集 conntrack 使用率、丢包、重传、SYN backlog、TIME_WAIT、
|
|
||||||
端口分配失败与 SNAT 端口耗尽事件。`conntrack -L` 会遍历整个表,不得在高峰时将它作为
|
|
||||||
常规排障命令;优先使用节点监控或 `conntrack -S` 的聚合计数。
|
|
||||||
|
|
||||||
Gateway 的文件描述符预算必须按实际连接模型计算,而不是仅按 QPS:每条活跃 CONNECT
|
|
||||||
隧道通常占用客户端和上游两个 socket;HTTP 上游并发、idle 连接池、监听 socket、日志和
|
|
||||||
运行时也会占用描述符。部署前确认 Pod 内 `ulimit -n`、容器运行时 `LimitNOFILE` 与节点
|
|
||||||
`fs.file-max` 均高于下式结果并保留至少 20% 余量:
|
|
||||||
|
|
||||||
```text
|
|
||||||
fd_budget = 2 * max_active_connect_tunnels
|
|
||||||
+ max_concurrent_http_upstreams
|
|
||||||
+ max_idle_transport_connections
|
|
||||||
+ process_reserve
|
|
||||||
```
|
|
||||||
|
|
||||||
如果节点通过 NAT 访问上游,源端口和 conntrack 表同样构成硬上限。压测报告必须记录每个
|
|
||||||
节点的 egress IP 数、可用临时端口范围、NAT/SNAT 设备限制与单目标连接分布;单一 egress IP
|
|
||||||
不足时应在发布前增加 egress IP 或拆分节点池,不能依靠无限重试掩盖端口耗尽。
|
|
||||||
|
|
||||||
建议将 FD、conntrack、NAT 端口利用率、SYN overflow、TCP retransmit、TIME_WAIT、CPU、RSS
|
|
||||||
和 Gateway p99 作为同一份容量证据采集。任何一项超过预设预警线时停止扩容或发布,先降低
|
|
||||||
新连接速率并保留现有 CONNECT 隧道排空;回滚到兼容镜像或配置 revision 后,再根据原始
|
|
||||||
指标分析根因。
|
|
||||||
|
|||||||
@ -1,114 +0,0 @@
|
|||||||
package docs_test
|
|
||||||
|
|
||||||
import (
|
|
||||||
"archive/zip"
|
|
||||||
"encoding/json"
|
|
||||||
"os"
|
|
||||||
"os/exec"
|
|
||||||
"path/filepath"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
)
|
|
||||||
|
|
||||||
type docsPackageManifest struct {
|
|
||||||
Version string `json:"version"`
|
|
||||||
Revision string `json:"revision"`
|
|
||||||
Files []struct {
|
|
||||||
Path string `json:"path"`
|
|
||||||
SHA256 string `json:"sha256"`
|
|
||||||
} `json:"files"`
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestGatewayBenchmarkScriptIsBoundedAndWritesEvidence(t *testing.T) {
|
|
||||||
pwsh, err := exec.LookPath("pwsh")
|
|
||||||
if err != nil {
|
|
||||||
t.Skip("pwsh is required to exercise the gateway benchmark script")
|
|
||||||
}
|
|
||||||
repositoryRoot := repositoryRoot(t)
|
|
||||||
outputPath := filepath.Join(t.TempDir(), "gateway-benchmarks.txt")
|
|
||||||
command := exec.Command(pwsh, "-NoProfile", "-NonInteractive", "-File", filepath.Join(repositoryRoot, "scripts", "benchmark-gateway.ps1"), "-OutputPath", outputPath)
|
|
||||||
command.Dir = repositoryRoot
|
|
||||||
if output, err := command.CombinedOutput(); err != nil {
|
|
||||||
t.Fatalf("run gateway benchmarks: %v\n%s", err, output)
|
|
||||||
}
|
|
||||||
payload, err := os.ReadFile(outputPath)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("read benchmark evidence: %v", err)
|
|
||||||
}
|
|
||||||
evidence := string(payload)
|
|
||||||
for _, required := range []string{
|
|
||||||
"BenchmarkAcquire100k(Indexed|RoutingRoundRobin)",
|
|
||||||
"BenchmarkStoreApply100k",
|
|
||||||
"-benchtime=1x",
|
|
||||||
"-count=1",
|
|
||||||
"revision:",
|
|
||||||
} {
|
|
||||||
if !strings.Contains(evidence, required) {
|
|
||||||
t.Errorf("benchmark evidence does not contain %q", required)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestPackageDocsCreatesTraceableArchive(t *testing.T) {
|
|
||||||
pwsh, err := exec.LookPath("pwsh")
|
|
||||||
if err != nil {
|
|
||||||
t.Skip("pwsh is required to exercise the documentation packaging script")
|
|
||||||
}
|
|
||||||
repositoryRoot := repositoryRoot(t)
|
|
||||||
outputPath := filepath.Join(t.TempDir(), "proxy-pool-docs-v1.0.zip")
|
|
||||||
if err := os.WriteFile(outputPath, []byte("incomplete"), 0o600); err != nil {
|
|
||||||
t.Fatalf("seed old documentation archive: %v", err)
|
|
||||||
}
|
|
||||||
command := exec.Command(pwsh, "-NoProfile", "-NonInteractive", "-File", filepath.Join(repositoryRoot, "scripts", "package-docs.ps1"), "-OutputPath", outputPath)
|
|
||||||
command.Dir = repositoryRoot
|
|
||||||
if output, err := command.CombinedOutput(); err != nil {
|
|
||||||
t.Fatalf("package docs: %v\n%s", err, output)
|
|
||||||
}
|
|
||||||
|
|
||||||
archive, err := zip.OpenReader(outputPath)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("open documentation archive: %v", err)
|
|
||||||
}
|
|
||||||
defer archive.Close()
|
|
||||||
entries := make(map[string]*zip.File, len(archive.File))
|
|
||||||
for _, entry := range archive.File {
|
|
||||||
entries[entry.Name] = entry
|
|
||||||
}
|
|
||||||
for _, required := range []string{
|
|
||||||
"README.md",
|
|
||||||
"deploy/README.md",
|
|
||||||
"deploy/kubernetes/overlays/development-mtls/README.md",
|
|
||||||
"docs/design/architecture.md",
|
|
||||||
"docs/operations/runbook.md",
|
|
||||||
"diagrams/README.md",
|
|
||||||
"api/openapi/proxy-pool.yaml",
|
|
||||||
"api/proto/controlplane/v1/controlplane.proto",
|
|
||||||
"manifest.json",
|
|
||||||
} {
|
|
||||||
if _, exists := entries[required]; !exists {
|
|
||||||
t.Errorf("documentation archive does not contain %s", required)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
manifestEntry, exists := entries["manifest.json"]
|
|
||||||
if !exists {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
reader, err := manifestEntry.Open()
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("open manifest: %v", err)
|
|
||||||
}
|
|
||||||
defer reader.Close()
|
|
||||||
var manifest docsPackageManifest
|
|
||||||
if err := json.NewDecoder(reader).Decode(&manifest); err != nil {
|
|
||||||
t.Fatalf("decode manifest: %v", err)
|
|
||||||
}
|
|
||||||
if manifest.Version != "v1.0" || manifest.Revision == "" || len(manifest.Files) < 20 {
|
|
||||||
t.Fatalf("manifest = %+v", manifest)
|
|
||||||
}
|
|
||||||
for _, file := range manifest.Files {
|
|
||||||
if file.Path == "" || file.SHA256 == "" {
|
|
||||||
t.Fatalf("invalid manifest file = %+v", file)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@ -11,18 +11,14 @@
|
|||||||
- 产品设计、总体架构、项目结构、四项 ADR。
|
- 产品设计、总体架构、项目结构、四项 ADR。
|
||||||
- 开发、配置、Distribution/Admin API、控制面协议、安全、测试、运维文档。
|
- 开发、配置、Distribution/Admin API、控制面协议、安全、测试、运维文档。
|
||||||
- 20 个配置场景和 35 张 Mermaid 架构/流程/状态/故障图。
|
- 20 个配置场景和 35 张 Mermaid 架构/流程/状态/故障图。
|
||||||
- `scripts/package-docs.ps1` 可生成版本化 ZIP,包含 README、文档、图表、OpenAPI、Proto
|
|
||||||
契约,以及带 Git revision 和 SHA-256 的文件清单。
|
|
||||||
|
|
||||||
### 机器契约
|
### 机器契约
|
||||||
|
|
||||||
- Distribution OpenAPI:一次性独占提取、partial/allOrNothing、幂等键、
|
- Distribution OpenAPI:一次性独占提取、partial/allOrNothing、幂等键、
|
||||||
Redis TTL 活动池原子语义、TTL/健康过滤结果与标准错误。
|
Redis TTL 活动池原子语义、TTL/健康过滤结果与标准错误。
|
||||||
- Admin OpenAPI:状态、按 ID 游标分页的权威审计查询、Upstream 启停、Routing
|
- Admin OpenAPI:状态、Upstream 启停、Routing 切换和配置重载。
|
||||||
切换和配置重载。
|
|
||||||
- 两份 OpenAPI 已进入 Go/CI 结构门禁,覆盖本地引用闭合、operationId、响应和
|
- 两份 OpenAPI 已进入 Go/CI 结构门禁,覆盖本地引用闭合、operationId、响应和
|
||||||
security scheme;固定版本的 Redocly CLI 还会按 OpenAPI 3.1 最小规则集验证两份文档,
|
security scheme;完整标准工具验证仍待补齐。
|
||||||
并把 Tag 描述提升为错误。
|
|
||||||
- Protobuf:Worker 注册、全量/增量 Snapshot、`usable_until`、ACK、运行态/
|
- Protobuf:Worker 注册、全量/增量 Snapshot、`usable_until`、ACK、运行态/
|
||||||
结果上报、Checker 任务与 Observation。
|
结果上报、Checker 任务与 Observation。
|
||||||
|
|
||||||
@ -33,9 +29,8 @@
|
|||||||
- `PROXY-* / CAP-*`:唯一键、TTL 优先级、状态迁移与 Active/Reserved 打包
|
- `PROXY-* / CAP-*`:唯一键、TTL 优先级、状态迁移与 Active/Reserved 打包
|
||||||
原子计数;1,000 goroutine 不超卖,以及 Cancel、重复终结、错误顺序、并发
|
原子计数;1,000 goroutine 不超卖,以及 Cancel、重复终结、错误顺序、并发
|
||||||
Commit/Cancel/Release 计数守恒测试。
|
Commit/Cancel/Release 计数守恒测试。
|
||||||
- `ROUTE-001 / ROUTE-004`:首条命中规则、Provider 空结果驱动的 Controller
|
- `ROUTE-001 / ROUTE-004`:首条命中规则与进程内 Concurrent Sequential 单次
|
||||||
Sequential 切换及末端 stop 已接入权威管理状态;跨 Controller 通过 PostgreSQL
|
切换;策略运行时接线、持久化恢复和跨实例 CAS 尚未完成。
|
||||||
`ExpectedCurrent` CAS 收敛,切换/停用均刷新本进程 Worker Snapshot。
|
|
||||||
- `FETCH-005 / FETCH-006`:Valid、Empty、DuplicateOnly、Error 分类。
|
- `FETCH-005 / FETCH-006`:Valid、Empty、DuplicateOnly、Error 分类。
|
||||||
- `DIST-001..003 / DIST-006..007`:内存活动池参考实现验证独占提取、满足模式、
|
- `DIST-001..003 / DIST-006..007`:内存活动池参考实现验证独占提取、满足模式、
|
||||||
TTL、健康时效与 Gateway 保留量;1,000 并发不重复。
|
TTL、健康时效与 Gateway 保留量;1,000 并发不重复。
|
||||||
@ -48,11 +43,8 @@
|
|||||||
- `PROVIDER-*`:Provider HTTP Client、严格响应上限、模板解析安全边界、凭据
|
- `PROVIDER-*`:Provider HTTP Client、严格响应上限、模板解析安全边界、凭据
|
||||||
引用 Store 与 Reconciler Adapter 已实现。
|
引用 Store 与 Reconciler Adapter 已实现。
|
||||||
- `DIST/Admin HTTP`:严格 JSON、Request ID、Problem 响应及 Distribution/Admin
|
- `DIST/Admin HTTP`:严格 JSON、Request ID、Problem 响应及 Distribution/Admin
|
||||||
Handler 已实现;Admin 审计查询以有界 `afterId` 游标读取 PostgreSQL 权威记录,
|
Handler 已实现;共享认证、CIDR、可信代理、Client ID 与本地准入保护链已接入,
|
||||||
不读取 Proxy 或 Redis 活动池。共享认证、CIDR、可信代理、Client ID、凭据级
|
Controller Runtime 已将二者装配到独立监听器并支持联动优雅停机。
|
||||||
固定权限、Distribution 凭据级提取数量/Upstream/地区限制、Gateway 凭据级
|
|
||||||
Routing 限制与本地准入保护链已接入,Controller Runtime 已将二者装配到独立
|
|
||||||
监听器并支持联动优雅停机。
|
|
||||||
- `Redis Activity Adapter`:真实 Redis 8.2 已覆盖 Provider Upsert、健康更新、
|
- `Redis Activity Adapter`:真实 Redis 8.2 已覆盖 Provider Upsert、健康更新、
|
||||||
原子独占提取、短期幂等、Worker ownership、库存和有界过期清理,Memory/Redis
|
原子独占提取、短期幂等、Worker ownership、库存和有界过期清理,Memory/Redis
|
||||||
运行同一公用契约。
|
运行同一公用契约。
|
||||||
@ -97,51 +89,44 @@ docker compose config PASS
|
|||||||
kubectl kustomize PASS
|
kubectl kustomize PASS
|
||||||
configuration examples 21/21 PASS
|
configuration examples 21/21 PASS
|
||||||
Mermaid blocks 35
|
Mermaid blocks 35
|
||||||
bounded gateway microbenchmarks PASS (local baseline only)
|
|
||||||
```
|
```
|
||||||
|
|
||||||
Windows 环境为 `CGO_ENABLED=0` 且没有 C 编译器,`go test -race` 在本机未执行;
|
Windows 环境为 `CGO_ENABLED=0` 且没有 C 编译器,`go test -race` 在本机未执行;
|
||||||
CI 已配置 Linux race job。PostgreSQL 18 和 Redis 8.2 的隔离 Adapter fixture
|
CI 已配置 Linux race job。PostgreSQL 18 和 Redis 8.2 的隔离 Adapter fixture
|
||||||
已经运行;本地 Compose 已声明 Controller、两个独立 Gateway 身份和一个 Checker 的
|
已经运行;当前 Docker 镜像和激活的 Compose/Kubernetes 清单只包含已实现的
|
||||||
mTLS 运行拓扑,并由配置、证书、命令装配和部署契约测试覆盖。Kubernetes base 刻意不直接
|
Controller/Gateway 入口,完整 mTLS 运行时拓扑仍只有静态验证。
|
||||||
启用 Checker;`development-mtls` Overlay 已为单副本 Controller、Gateway 和 Checker
|
|
||||||
提供固定开发证书、运行时身份派生、mTLS Secret 挂载、最小出站 NetworkPolicy 与渲染/配置契约验证。
|
|
||||||
其 ConfigMap 使用内容哈希以在配置变化时触发三个工作负载滚动;固定名称的外部 Secret 仍要求
|
|
||||||
受控轮换和显式重启。
|
|
||||||
Controller、Gateway 与 Checker 的叶证书和信任根已在每个新 TLS 握手时重载,文件短暂
|
|
||||||
不一致时保留最后一次有效材料。Gateway、Checker 启动派生和 Controller 授权共用同一
|
|
||||||
SPIFFE URI 解析器:X.509-SVID 叶证书必须且只能包含一个 URI SAN,并精确匹配其角色。
|
|
||||||
生产环境仍必须通过独立 Overlay 为每个弹性 Worker 注入唯一 SPIFFE URI,且容器端到端
|
|
||||||
启动仍待具备镜像网络的环境验证。
|
|
||||||
|
|
||||||
GitHub Actions 另行执行 Dockerfile 的 Linux 多阶段镜像构建;该门禁证明镜像可以构建,
|
|
||||||
不替代 Controller/Gateway/Checker 与真实 Redis/PostgreSQL 的容器运行验证。
|
|
||||||
|
|
||||||
运维手册已覆盖管理面/活动池恢复顺序、发布与兼容回滚、Secret 轮换、容量计算,以及
|
|
||||||
主机内核、文件描述符、NAT/conntrack 的只读基线和证据采集要求;这些文档不替代真实环境
|
|
||||||
的备份恢复、驱逐或容量演练。
|
|
||||||
|
|
||||||
仓库审计会拒绝生产源和部署清单中的 `TODO`、`TBD`、`FIXME` 与私钥材料;原有 OpenAPI、
|
|
||||||
Queue、配置、指标和部署契约分别锁定一次性 Extract、队列容量、库存/累计获取边界及
|
|
||||||
低基数标签。根 `.dockerignore` 还会从镜像构建上下文排除本地控制面证书、私钥、环境文件和
|
|
||||||
构建产物。开发示例、环境变量引用和 CI fixture 不被视为生产 Secret 或未实现占位。
|
|
||||||
|
|
||||||
## 3. 后续实现范围
|
## 3. 后续实现范围
|
||||||
|
|
||||||
以下已有设计、接口或部署位置,但尚无端到端生产实现:
|
以下已有设计、接口或部署位置,但尚无端到端生产实现:
|
||||||
|
|
||||||
1. 已实现的 Controller、Gateway、Checker 和 Loadgen 进程仍需在实际容器网络中完成
|
1. `proxy-loadgen` 已提供有界 HTTP、HTTP Gateway CONNECT 长连接和 Distribution Extract
|
||||||
mTLS、DNS、Redis/PostgreSQL 连通性、健康探针和优雅停机的端到端验证。
|
响应校验场景;`proxy-checker` 的 BASIC 任务进程已经完成,
|
||||||
2. Gateway 已支持连接池、每 Host 连接上限、握手/空闲超时、隧道缓冲、请求生命周期指标
|
`proxy-controller` 已完成
|
||||||
与 p99 告警;仍需代表性环境的流量压测和容量报告。
|
Admin/Distribution/Metrics 与 PostgreSQL/Redis 启动装配,`proxy-gateway` 已完成
|
||||||
3. Provider 长期凭据回收与累计额度的生产运营策略仍需结合实际供应商合同、失效时间和
|
HTTP/Metrics 与控制面 Session 装配;Checker 与 Gateway Outcome 的低基数业务指标
|
||||||
密钥平台完成演练。
|
已接入,但 Provider、提取和容量指标链未闭环。
|
||||||
4. Kubernetes 的开发 mTLS Overlay 已覆盖单副本开发证书身份派生和 Checker 出站
|
2. Gateway 的生产连接池调优与代表性流量压测。
|
||||||
NetworkPolicy;生产环境仍需要按工作负载身份系统提供证书轮换、每副本唯一 Worker ID
|
3. Provider 分布式 singleflight/Leader、长期凭据回收和累计额度执行器。
|
||||||
与可弹性扩缩容的身份注入,base 清单不假定或伪造这些能力。
|
4. Controller 的 PostgreSQL 连接池、迁移和 pgx Adapter 启动装配已完成;
|
||||||
5. Redis/PostgreSQL 故障转移、控制面断线恢复、Snapshot 回滚、Drain 回收和 Extract
|
公用 bootstrap 已通过 PostgreSQL 18 + Redis 8.2 双存储集成,Controller
|
||||||
库存不足的完整故障演练。
|
三监听器与探针集成已完成;可选聚合指标和完整容器进程部署验证仍待实现。
|
||||||
6. 10k 稳态、100k 峰值、CONNECT 活跃连接和建连速率的代表性集群负载测试。
|
5. Worker Register/ACK/Runtime gRPC 接收路径、Redis 服务端 TTL 会话生命周期、
|
||||||
|
SPIFFE mTLS 校验和 Controller 生命周期接线已完成;Redis Provider Leader、
|
||||||
|
分布式请求额度、Distribution Client 限制和 Provider 状态丢失重建已完成。
|
||||||
|
6. Worker 基础网络快照流、Proxy/Gateway Routing/凭据 Snapshot payload、Gateway Snapshot
|
||||||
|
客户端和进程装配、同版本 Routing 编译/动态匹配、五种策略上游选择与 reject/wait/direct 已完成;
|
||||||
|
Outcome 上报已完成基础观测链;Redis ownership drain/ACK/过期回收及按 Worker 的可下发索引已完成。
|
||||||
|
7. BASIC Checker 调度与 HTTP/HTTPS/SOCKS5 探测器、全局与 TARGET Profile 的 Memory/Redis
|
||||||
|
原子归并、Controller Reducer 和 Observation 上报 RPC 已完成;EGRESS、TARGET
|
||||||
|
生产任务调度与 REMOVE 编排仍待实现。
|
||||||
|
8. Admin/Distribution 细粒度授权和审计查询;Distribution 分布式限流已完成。
|
||||||
|
9. 真实 Compose/Kubernetes 集成、故障演练和代表性集群负载测试。
|
||||||
|
10. 将 reject/wait/direct 接入 Distribution 运行链,补齐 Sequential 持久化恢复、跨实例 CAS
|
||||||
|
和 disabled candidate 语义。
|
||||||
|
11. 补齐 Proxy Capacity 动态降容契约、Reservation 全生命周期观测;短 TTL
|
||||||
|
Proxy 的零计数运行态已在后续 Snapshot Apply 中回收。
|
||||||
|
|
||||||
## 4. 容量结论
|
## 4. 容量结论
|
||||||
|
|
||||||
100,000 QPS 是集群设计输入,不是本次验证结果。只有实现上述运行时,并在
|
100,000 QPS 是集群设计输入,不是本次验证结果。只有实现上述运行时,并在
|
||||||
|
|||||||
@ -9,8 +9,8 @@
|
|||||||
|---|---|---|---|
|
|---|---|---|---|
|
||||||
| ARCH-001 | 数据面 Worker 与控制面 Controller 分离 | 1-70 | 包、协议和部署拓扑已分离;Controller 已运行 Worker Register/Watch/ACK/Runtime/Outcome 与 Checker Observation gRPC,并发布 Proxy/Gateway Routing/按引用去重凭据完整快照;Checker 任务流已具备有界领取、租约栅栏和任务期凭据契约。Gateway 已将快照编译为同版本动态 View,并由独立进程维护控制面会话。`proxy-checker` 与 Redis BASIC/EGRESS/TARGET 共享任务运行态、`proxy-loadgen` 有界 HTTP/CONNECT/Extract 校验场景均已实现 |
|
| ARCH-001 | 数据面 Worker 与控制面 Controller 分离 | 1-70 | 包、协议和部署拓扑已分离;Controller 已运行 Worker Register/Watch/ACK/Runtime/Outcome 与 Checker Observation gRPC,并发布 Proxy/Gateway Routing/按引用去重凭据完整快照;Checker 任务流已具备有界领取、租约栅栏和任务期凭据契约。Gateway 已将快照编译为同版本动态 View,并由独立进程维护控制面会话。`proxy-checker` 与 Redis BASIC/EGRESS/TARGET 共享任务运行态、`proxy-loadgen` 有界 HTTP/CONNECT/Extract 校验场景均已实现 |
|
||||||
| ARCH-002 | 热路径只做认证、本地路由和网络转发 | 1-70, 380-430 | Gateway bootstrap 集成测试验证启动期控制面会话与快照就绪,HTTP 请求只走本地 Snapshot/Dispatch;Outcome 仅写入有界非阻塞本地队列,代表性性能剖析待完成 |
|
| ARCH-002 | 热路径只做认证、本地路由和网络转发 | 1-70, 380-430 | Gateway bootstrap 集成测试验证启动期控制面会话与快照就绪,HTTP 请求只走本地 Snapshot/Dispatch;Outcome 仅写入有界非阻塞本地队列,代表性性能剖析待完成 |
|
||||||
| ARCH-003 | Gateway、Distribution、Admin、Metrics 独立入口 | 8904-8958 | Controller 命令已装配 Distribution/Admin/Metrics 三个独立监听及联动停机;Gateway 命令已装配代理与 Metrics 监听。本地 Compose 已声明 Controller/Gateway/Checker mTLS 拓扑;Kubernetes `development-mtls` Overlay 已通过渲染与配置契约验证单副本开发证书身份派生链路,生产工作负载身份仍待环境实现 |
|
| ARCH-003 | Gateway、Distribution、Admin、Metrics 独立入口 | 8904-8958 | Controller 命令已装配 Distribution/Admin/Metrics 三个独立监听及联动停机;Gateway 命令已装配代理与 Metrics 监听,运行时 mTLS 部署 Overlay 待完成 |
|
||||||
| ARCH-004 | Controller 集中 Provider 获取与切换 | 1403-1580 | Redis Leader、动态 Provider Supervisor 与 Bootstrap 生产装配已完成;静态配置与 Admin 管理态共同决定 Provider 启停,停用策略以 Upstream revision 栅栏驱动既有 ownership Drain;Routing 切换会向本进程 Worker 快照流广播完整刷新,多副本按定时刷新与权威 HMAC 指纹/revision 栅栏收敛并拒绝旧配置换主 |
|
| ARCH-004 | Controller 集中 Provider 获取与切换 | 1403-1580 | Redis Leader、动态 Provider Supervisor 与 Bootstrap 生产装配已完成;Admin disable/reload 驱动取消替换,多副本按权威 HMAC 指纹和 revision 栅栏收敛并拒绝旧配置换主;Routing 切换到 Drain 的编排待完成 |
|
||||||
| ARCH-005 | 100k QPS 峰值使用多 Worker 集群 | 当前会话 | 未验证设计目标;待代表性集群负载报告 |
|
| ARCH-005 | 100k QPS 峰值使用多 Worker 集群 | 当前会话 | 未验证设计目标;待代表性集群负载报告 |
|
||||||
|
|
||||||
## Routing 与 Upstream
|
## Routing 与 Upstream
|
||||||
@ -19,11 +19,11 @@
|
|||||||
|---|---|---|---|
|
|---|---|---|---|
|
||||||
| ROUTE-001 | Routing 自上而下匹配,首条命中停止 | 3534-3798, 5825-6467 | `rule.go` 与不可变/首命中单测 |
|
| ROUTE-001 | Routing 自上而下匹配,首条命中停止 | 3534-3798, 5825-6467 | `rule.go` 与不可变/首命中单测 |
|
||||||
| ROUTE-002 | Routing 与 Upstream 生命周期解耦 | 3534-3798 | 包依赖与配置模型 |
|
| ROUTE-002 | Routing 与 Upstream 生命周期解耦 | 3534-3798 | 包依赖与配置模型 |
|
||||||
| ROUTE-003 | 支持 sequential、random、roundRobin、weighted、leastConnections | 5825-6467 | 五种领域策略、同版本 Gateway Snapshot 派发和定向测试已完成。Distribution 按独占 Extract 契约分配 Proxy,不参与目标请求的 Routing 策略选择 |
|
| ROUTE-003 | 支持 sequential、random、roundRobin、weighted、leastConnections | 5825-6467 | 五种领域策略、同版本 Gateway Snapshot 派发和定向测试已完成;Distribution 接线待完成 |
|
||||||
| ROUTE-004 | Sequential 连续空结果达到阈值后原子切换一次 | 5295-5824, 6520-6617 | Provider Stats 对连续空结果分配单调代次,Controller `SequentialCoordinator` 在独立有界循环中读取权威配置/管理快照,并通过 `ExpectedCurrent` CAS 自动切换。禁用候选会跳过;若当前项被管理态禁用,协调器不读取 Provider Stats,按配置顺序推进到下一启用项,只有 `loop` 可回绕,`stop`/`stayLast` 无后继时通过 `DisableRouting` 原子停用路由。重复 Tick、旧代次及跨 Controller 并发 Tick 均不会重复变更;成功切换或停用都会广播完整 Snapshot,并与审计/Outbox 同事务提交。|
|
| ROUTE-004 | Sequential 连续空结果达到阈值后原子切换一次 | 5295-5824, 6520-6617 | 进程内 `RoutingCursor` 版本 CAS 与 100 并发测试已完成;持久化恢复和跨实例 CAS 待完成 |
|
||||||
| ROUTE-005 | 空计数属于 Upstream,当前选择属于 Routing | 8442-8529 | 共享 `UpstreamEmptyState` 双 Routing 测试 |
|
| ROUTE-005 | 空计数属于 Upstream,当前选择属于 Routing | 8442-8529 | 共享 `UpstreamEmptyState` 双 Routing 测试 |
|
||||||
| ROUTE-006 | 旧 Upstream 已有 Proxy 继续耗尽,不因切换直接丢弃 | 6618-6641 | Routing 成功切换后立即发布完整快照;Sequential 仅将新分配切到新的 CurrentUpstream,旧 Proxy 仍保留在快照,既有 Active/Reserved 由本地运行态自然归零。共享 Upstream 不按单 Routing 强制 Drain,避免影响其他 Routing |
|
| ROUTE-006 | 旧 Upstream 已有 Proxy 继续耗尽,不因切换直接丢弃 | 6618-6641 | 通用 ownership Drain/ACK 原语已测试;Routing 切换到 Drain 的编排待完成 |
|
||||||
| ROUTE-007 | 无可用 Upstream 时显式 reject、wait 或 direct,默认 reject | 5075-5294, 6743-6760 | Gateway 已实现默认 reject、带 `wait_timeout` 的本地容量等待,以及经 TargetPolicy 的 direct。静态 `routing.action: direct` 在认证、限流和目标策略后绕过 Proxy 调度;Distribution 不承载目标请求路由 |
|
| ROUTE-007 | 无可用 Upstream 时显式 reject、wait 或 direct,默认 reject | 5075-5294, 6743-6760 | Gateway 已实现 reject、带 `wait_timeout` 的本地容量等待与经 TargetPolicy 的 direct;Distribution 接线和默认化策略待完成 |
|
||||||
|
|
||||||
## Provider 与补池
|
## Provider 与补池
|
||||||
|
|
||||||
@ -45,8 +45,8 @@
|
|||||||
| PROXY-001 | Proxy 保存协议、地址、凭据引用、来源、TTL、健康、容量和标签 | 71-105, 8605-8678 | Domain 类型与序列化测试 |
|
| PROXY-001 | Proxy 保存协议、地址、凭据引用、来源、TTL、健康、容量和标签 | 71-105, 8605-8678 | Domain 类型与序列化测试 |
|
||||||
| PROXY-002 | 唯一键包含 scheme、host、port、username、credentialVersion | 6655-6727, 8605-8678 | 去重单测 |
|
| PROXY-002 | 唯一键包含 scheme、host、port、username、credentialVersion | 6655-6727, 8605-8678 | 去重单测 |
|
||||||
| PROXY-003 | TTL 来源优先级明确并统一 UTC | 681-747, 8655-8678 | TTL 表驱动测试 |
|
| PROXY-003 | TTL 来源优先级明确并统一 UTC | 681-747, 8655-8678 | TTL 表驱动测试 |
|
||||||
| CAP-001 | Gateway 分配使用 Reserved -> Active 原子转换 | 1203-1467, 8530-8597 | 固定 Max 下打包 CAS、1,000 并发不超卖、退役 Proxy 停止新预留及零计数历史运行态回收已完成;Snapshot 动态降容会保留既有 Active、在计数低于新上限前拒绝新预留,运行态复用与恢复预留均有回归测试。非法 Commit/Cancel/Release 经固定枚举观察器进入低基数指标 |
|
| CAP-001 | Gateway 分配使用 Reserved -> Active 原子转换 | 1203-1467, 8530-8597 | 固定 Max 下打包 CAS、1,000 并发不超卖、退役 Proxy 停止新预留及零计数历史运行态回收已完成;动态降容和完整生命周期证据待完成 |
|
||||||
| CAP-002 | 补池依据 Available Slots,不只看 Proxy 数量 | 1203-1402, 8530-8597 | `AvailableSlots`、显式 minimum/target 水位、pending 槽位和迟滞 Reconciler 已测试;Worker session/ACK/sequence/TTL/ownership fence、权威 Proxy/Routing/凭据 payload、目标健康和 Gateway reserve 策略已通过内存、Redis 8.2、gRPC 与 Gateway 回归测试 |
|
| CAP-002 | 补池依据 Available Slots,不只看 Proxy 数量 | 1203-1402, 8530-8597 | `AvailableSlots`、显式 minimum/target 水位、pending 槽位和迟滞 Reconciler 已测试;Worker session/ACK/sequence/TTL/ownership fence、基础 Snapshot 流及 Gateway 会话组件已通过内存、Redis 8.2 和 gRPC 测试;权威 payload、目标健康和 Gateway reserve 策略仍待完成 |
|
||||||
| CAP-003 | pool.maxSize 包括 FETCHED/CHECKING/AVAILABLE/SUSPECT/DRAINING 与 pending expected | 3001-3533, 6642-6680 | `FetchBudget` 100 并发额度预占测试 |
|
| CAP-003 | pool.maxSize 包括 FETCHED/CHECKING/AVAILABLE/SUSPECT/DRAINING 与 pending expected | 3001-3533, 6642-6680 | `FetchBudget` 100 并发额度预占测试 |
|
||||||
| CAP-004 | TTL safety margin 内禁止新分配 | 173-220, 6728-6741 | 时钟测试 |
|
| CAP-004 | TTL safety margin 内禁止新分配 | 173-220, 6728-6741 | 时钟测试 |
|
||||||
| CAP-005 | 多 Worker 不在热路径访问 Redis 计数 | 1403-1467 | Gateway 包依赖审计、Snapshot/Dispatch 测试 |
|
| CAP-005 | 多 Worker 不在热路径访问 Redis 计数 | 1403-1467 | Gateway 包依赖审计、Snapshot/Dispatch 测试 |
|
||||||
@ -86,6 +86,6 @@
|
|||||||
| SEC-002 | 非回环监听无保护时严格模式启动失败 | 8112-8441 | 配置校验测试 |
|
| SEC-002 | 非回环监听无保护时严格模式启动失败 | 8112-8441 | 配置校验测试 |
|
||||||
| OPS-001 | 配置校验后构建不可变快照并原子替换 | 8959-8999 | 100k 索引、版本/epoch 与并发 Apply/Acquire 测试 |
|
| OPS-001 | 配置校验后构建不可变快照并原子替换 | 8959-8999 | 100k 索引、版本/epoch 与并发 Apply/Acquire 测试 |
|
||||||
| OPS-002 | 优雅停机停止新请求/Fetch,等待现有流量后超时关闭 | 8981-9000 | Provider Run 收敛与 `Handler.Shutdown` HTTP 排空、Hijacked CONNECT 超时关闭测试 |
|
| OPS-002 | 优雅停机停止新请求/Fetch,等待现有流量后超时关闭 | 8981-9000 | Provider Run 收敛与 `Handler.Shutdown` HTTP 排空、Hijacked CONNECT 超时关闭测试 |
|
||||||
| OPS-003 | PostgreSQL 只保存管理修订、Upstream/Routing 状态、Admin 审计与 Outbox | 当前会话 | ADR-006、`adminstate` 公用契约和六表 Schema 边界测试;真实 PostgreSQL 18 契约、迁移幂等和 `information_schema` 数据边界测试已完成 |
|
| OPS-003 | PostgreSQL 只保存管理修订、Upstream/Routing 状态、Admin 审计与 Outbox | 当前会话 | ADR-006、`adminstate` 公用契约和六表 Schema 边界测试;真实 PostgreSQL 契约待完成 |
|
||||||
| OBS-001 | 指标禁止 Proxy IP、session、Client、完整 URL 高基数标签 | 9001-9029 | Controller Prometheus/探针模块已实现;Gateway Collector 暴露固定 `protocol` 的请求总数、并发数、隧道数和生命周期直方图,以及固定 `stage,result` 的 Outcome 指标。Checker、Drain、Provider、Extraction 与 Capacity Collector 仅使用固定枚举标签;Grafana 和 Prometheus 规则由部署契约测试校验只引用已注册指标且不按 Upstream/Worker 聚合。`platform/logging` 使用 JSON `slog`,敏感字段、URL 用户信息、查询 Secret 和错误对象写出前均被脱敏;回归测试锁定标签集与 Secret 不泄露 |
|
| OBS-001 | 指标禁止 Proxy IP、session、Client、完整 URL 高基数标签 | 9001-9029 | Controller Prometheus/探针模块已实现;Checker Collector 暴露固定等级和结果,Gateway Collector 暴露 `proxy_pool_gateway_outcomes_total{stage,result}` 与 `proxy_pool_gateway_outcome_queue_dropped_total`,均以注册表测试固定标签集。Provider、提取和容量业务指标待实现 |
|
||||||
| TEST-001 | 覆盖对话中列出的 11 个关键并发与故障场景 | 9030-9082 | 测试清单;Redis 活动池由 Memory/Redis 公用契约覆盖,跨进程故障场景仍按清单推进 |
|
| TEST-001 | 覆盖对话中列出的 11 个关键并发与故障场景 | 9030-9082 | 测试清单;Redis 活动池由 Memory/Redis 公用契约覆盖,跨进程故障场景仍按清单推进 |
|
||||||
|
|||||||
@ -17,19 +17,8 @@
|
|||||||
- `httpsecurity.Protection` 统一实现 Basic、API Key、Bearer、CIDR、可信代理链、
|
- `httpsecurity.Protection` 统一实现 Basic、API Key、Bearer、CIDR、可信代理链、
|
||||||
Client ID 与入口准入;API 使用 401/`WWW-Authenticate`,Gateway 使用
|
Client ID 与入口准入;API 使用 401/`WWW-Authenticate`,Gateway 使用
|
||||||
407/`Proxy-Authenticate`。
|
407/`Proxy-Authenticate`。
|
||||||
- Listener 可将固定权限绑定到单个认证方式,或在 `mode: any` 中绑定到每个凭据;
|
|
||||||
`admin:read`、`admin:write` 与 `distribution:extract` 在 Handler 入口完成匹配,
|
|
||||||
不把无权请求传入管理 mutation 或 Proxy 提取流程。
|
|
||||||
- Token Client ID 使用 SHA-256 的 128 位摘要前缀,不把 Token 本身写入领域、
|
- Token Client ID 使用 SHA-256 的 128 位摘要前缀,不把 Token 本身写入领域、
|
||||||
日志或审计键。
|
日志或审计键。
|
||||||
- 启用 Gateway 粘性会话时,绑定键由认证 Client、Routing 和会话头共同派生;
|
|
||||||
原始会话标识不写入 PostgreSQL、Redis、日志或指标,也不会转发给目标站点。
|
|
||||||
绑定仅保留在当前 Worker 的有界内存中,并在代理不再符合当前 Snapshot、到达
|
|
||||||
代理可用期或转发失败时失效。
|
|
||||||
- Gateway 凭据的 requestsPerMinute 限制按认证 Client 在本 Worker 内执行,不把
|
|
||||||
认证身份、会话或请求计数写入 PostgreSQL、Redis 或 Prometheus 标签。
|
|
||||||
- Gateway 凭据的 maxConcurrentConnections 在请求或 CONNECT 隧道的完整生命周期
|
|
||||||
内保留一个本地租约;无论转发成功、失败还是服务排空,租约都会释放。
|
|
||||||
|
|
||||||
## 3. 目标地址策略
|
## 3. 目标地址策略
|
||||||
|
|
||||||
@ -48,8 +37,6 @@ CGNAT、协议转换保留段和已知云元数据端点属于硬拒绝项。私
|
|||||||
## 4. Secret 处理
|
## 4. Secret 处理
|
||||||
|
|
||||||
- 配置只保存环境变量或文件引用,不在日志中输出解析后的值。
|
- 配置只保存环境变量或文件引用,不在日志中输出解析后的值。
|
||||||
- Docker 镜像构建上下文通过根目录 `.dockerignore` 排除本地环境文件、控制面证书/私钥、
|
|
||||||
凭据文件、Git 元数据和测试/构建产物;运行时证书只能通过受控挂载提供。
|
|
||||||
- Proxy 唯一键包含 username 与 credentialVersion,不包含密码或 SecretRef 内容。
|
- Proxy 唯一键包含 username 与 credentialVersion,不包含密码或 SecretRef 内容。
|
||||||
- Provider Parser 将静态/响应密码写入 `credentials.Store`,Proxy Snapshot 只携带
|
- Provider Parser 将静态/响应密码写入 `credentials.Store`,Proxy Snapshot 只携带
|
||||||
SecretRef 与 credentialVersion;Gateway 通过统一 Resolver 按版本读取。
|
SecretRef 与 credentialVersion;Gateway 通过统一 Resolver 按版本读取。
|
||||||
@ -65,9 +52,8 @@ CGNAT、协议转换保留段和已知云元数据端点属于硬拒绝项。私
|
|||||||
|
|
||||||
## 6. 审计
|
## 6. 审计
|
||||||
|
|
||||||
PostgreSQL 审计只记录 Admin 管理 mutation 的 actor、来源、资源、变更结果和
|
Extraction 审计记录至少包含 requestId、Client、来源、Proxy ID、Upstream、
|
||||||
时间;它不保存 Proxy ID、代理地址、凭据、逐次 Extraction 或 Gateway 请求。
|
提取时间和到期时间。日志脱敏不影响审计关联,但审计接口自身必须受 Admin
|
||||||
Extraction 的短期幂等结果仅位于 Redis TTL 活动池;审计查询接口必须受 Admin
|
|
||||||
权限保护并具备保留期限。
|
权限保护并具备保留期限。
|
||||||
|
|
||||||
## 7. 优雅停机
|
## 7. 优雅停机
|
||||||
|
|||||||
@ -83,19 +83,13 @@ soak 测试单独标记,不混入快速单测。
|
|||||||
|
|
||||||
## 5. 当前本地微基准
|
## 5. 当前本地微基准
|
||||||
|
|
||||||
2026-08-07,Windows/amd64、Go 1.26.4、Intel Core Ultra 7 155H,以固定
|
2026-07-30,Windows/amd64、Intel Core Ultra 7 155H:
|
||||||
`-count=1 -benchtime=1x` 运行:
|
|
||||||
|
|
||||||
```text
|
```text
|
||||||
BenchmarkAcquire100kIndexed-22 1 13100 ns/op 256 B/op 2 allocs/op
|
BenchmarkAcquire100kIndexed-22 1000000-1867125 893.9-1047 ns/op 256 B/op 2 allocs/op
|
||||||
BenchmarkAcquire100kRoutingRoundRobin-22 1 45200 ns/op 13680 B/op 81 allocs/op
|
BenchmarkStoreApply100k-22 1 518.7 ms/op 540 MB/op 3000887 allocs/op
|
||||||
BenchmarkStoreApply100k-22 1 539017100 ns/op 684677920 B/op 3001735 allocs/op
|
|
||||||
```
|
```
|
||||||
|
|
||||||
使用 `./scripts/benchmark-gateway.ps1` 可复现该固定单次基线,输出写入被 Git 忽略的
|
|
||||||
`dist/gateway-benchmarks.txt` 并附带 revision。单次基准存在运行噪声,应只用于发现明显回归;
|
|
||||||
性能比较需要在同规格环境增加多次迭代后另行保存原始证据。
|
|
||||||
|
|
||||||
`Acquire` 已使用 scheme/upstream/tag 索引,结果只代表本地选择和容量预留。
|
`Acquire` 已使用 scheme/upstream/tag 索引,结果只代表本地选择和容量预留。
|
||||||
`Store.Apply` 属于冷路径且当前内存开销较高;运行态为防止旧快照在途连接超配,
|
`Store.Apply` 属于冷路径且当前内存开销较高;运行态为防止旧快照在途连接超配,
|
||||||
暂不自动回收曾出现过的 Proxy ID,但注册表有 1,000,000 项硬上限,达到上限时
|
暂不自动回收曾出现过的 Proxy ID,但注册表有 1,000,000 项硬上限,达到上限时
|
||||||
|
|||||||
@ -182,7 +182,6 @@ Admin 应用层测试覆盖 typed-nil 依赖、Actor/SourceIP 映射、Routing C
|
|||||||
配置:Snapshot 规模、Proxy 容量、路由、重试、日志级别
|
配置:Snapshot 规模、Proxy 容量、路由、重试、日志级别
|
||||||
场景:协议、连接复用、响应体、持续时间、升压曲线、故障注入
|
场景:协议、连接复用、响应体、持续时间、升压曲线、故障注入
|
||||||
结果:QPS、建连速率、active、p50/p95/p99、错误、CPU、RSS、GC、FD、网络
|
结果:QPS、建连速率、active、p50/p95/p99、错误、CPU、RSS、GC、FD、网络
|
||||||
限速:配置 rate、实际 Requests、RateStartsGenerated、RateStartsDropped
|
|
||||||
不变量:capacity、ownership、extraction、idempotency、admin audit、reserve 检查结果
|
不变量:capacity、ownership、extraction、idempotency、admin audit、reserve 检查结果
|
||||||
结论:通过/失败,以及适用边界
|
结论:通过/失败,以及适用边界
|
||||||
```
|
```
|
||||||
|
|||||||
72
findings.md
72
findings.md
@ -49,27 +49,6 @@ Extract 请求使用哪些 Upstream。
|
|||||||
- Gateway、Distribution、Admin、Metrics 使用独立监听和认证/访问控制。
|
- Gateway、Distribution、Admin、Metrics 使用独立监听和认证/访问控制。
|
||||||
- 非回环监听且无认证、无 CIDR 保护时,严格模式必须拒绝启动。
|
- 非回环监听且无认证、无 CIDR 保护时,严格模式必须拒绝启动。
|
||||||
|
|
||||||
## 可观测性边界
|
|
||||||
|
|
||||||
- Provider 拉取结果只按固定 `class` 聚合:`valid`、`empty`、`duplicate_only`、`error`。
|
|
||||||
指标分别记录拉取结果次数、合法候选数和新入池代理数,不按 Upstream、Proxy、IP、错误
|
|
||||||
文本或凭据拆分。
|
|
||||||
- Provider 指标已完成。
|
|
||||||
|
|
||||||
- Extraction 只按固定 `result` 聚合请求:`complete`、`partial`、`empty`、
|
|
||||||
`insufficient`、`idempotency_conflict`、`rate_limited`、`unavailable`、`invalid`、
|
|
||||||
`error`。计数只记录请求数量和响应交付数量;幂等重放不被误记为新的 Proxy 消费。
|
|
||||||
- Extraction 指标已完成。
|
|
||||||
|
|
||||||
- 容量指标由 Provider 库存对账循环采样,聚合 managed、available、effective、pending
|
|
||||||
与 active Upstream Gauge。Upstream ID 仅保留在 Controller 进程内用于聚合替换和
|
|
||||||
任期结束清理,不输出为 Prometheus 标签;读取结果只允许 `success` 或 `error`。
|
|
||||||
- 容量指标已完成。
|
|
||||||
|
|
||||||
- 进程级致命错误统一经 JSON `slog` 输出;敏感属性、错误对象、URL 用户信息和查询
|
|
||||||
Secret 在写出前脱敏。该出口只用于生命周期边界,不在 Gateway/Distribution 请求热路径
|
|
||||||
逐条写日志。
|
|
||||||
|
|
||||||
## 集群与性能
|
## 集群与性能
|
||||||
|
|
||||||
- 用户补充:高峰可能达到 100,000 请求/秒。
|
- 用户补充:高峰可能达到 100,000 请求/秒。
|
||||||
@ -145,17 +124,10 @@ Routing 自上而下匹配,首条命中停止;支持 Gateway 与 Extract 两
|
|||||||
Gateway 使用既有 `SessionSupervisor` 退避重连。Gateway 本地 Store 对从完整 Snapshot
|
Gateway 使用既有 `SessionSupervisor` 退避重连。Gateway 本地 Store 对从完整 Snapshot
|
||||||
消失且仍有 Active/Reserved 的 Proxy 已按 draining 继续上报。Redis Runtime 替换已在当前
|
消失且仍有 Active/Reserved 的 Proxy 已按 draining 继续上报。Redis Runtime 替换已在当前
|
||||||
session ACK、Ticket 屏障和完整零计数同时成立时自动释放 owner;Reaper 已对持续
|
session ACK、Ticket 屏障和完整零计数同时成立时自动释放 owner;Reaper 已对持续
|
||||||
`UNHEALTHY` 的已归属代理调用条件式 Drain。配置停用也会发布有效的上游策略视图,
|
`UNHEALTHY` 的已归属代理调用条件式 Drain。配置停用尚未接入同一触发编排。
|
||||||
从上游 owned 索引有界选择候选;候选携带上游 revision,Redis 在创建 Drain Ticket 前
|
|
||||||
再次核验策略仍停用、Proxy source、Worker 和 assignment epoch,避免旧候选跨启停使用。
|
|
||||||
- Snapshot 的 `version` 是同一 Worker 流的连续序列,`ownership_epoch` 是独立且只能前进
|
- Snapshot 的 `version` 是同一 Worker 流的连续序列,`ownership_epoch` 是独立且只能前进
|
||||||
的权威栅栏;epoch 变化不重置 version。此前 Gateway 错把 epoch 前进要求为 version=1,
|
的权威栅栏;epoch 变化不重置 version。此前 Gateway 错把 epoch 前进要求为 version=1,
|
||||||
与 Controller 的 `lastAppliedVersion+1` 生成规则冲突,现已用连续版本规则统一。
|
与 Controller 的 `lastAppliedVersion+1` 生成规则冲突,现已用连续版本规则统一。
|
||||||
- `SnapshotRefreshBroker` 是 Controller 进程内的公共扇出接口:已提交的 Upstream 启停、
|
|
||||||
Routing 切换和配置发布各向每条本地 Worker 流发送一个可合并刷新信号。Routing 不记录
|
|
||||||
单代理归属,不能按单条 Routing 对共享 Upstream 强制 Drain;Sequential 切换只改变后续
|
|
||||||
新分配,旧 Proxy/连接由原有 Snapshot 与运行态自然排空。其他 Controller 副本继续在
|
|
||||||
固定刷新周期内读取 PostgreSQL 权威管理态并收敛。
|
|
||||||
|
|
||||||
## Drain Ticket(2026-08-02)
|
## Drain Ticket(2026-08-02)
|
||||||
|
|
||||||
@ -250,45 +222,3 @@ Routing 自上而下匹配,首条命中停止;支持 Gateway 与 Extract 两
|
|||||||
- 生产 `docker-compose.yml` 仍把 PostgreSQL 18 命名卷挂在旧路径
|
- 生产 `docker-compose.yml` 仍把 PostgreSQL 18 命名卷挂在旧路径
|
||||||
`/var/lib/postgresql/data`;直接修改可能影响已有本地数据,必须配套迁移步骤后
|
`/var/lib/postgresql/data`;直接修改可能影响已有本地数据,必须配套迁移步骤后
|
||||||
单独处理,当前不能把生产持久化拓扑视为已验证。
|
单独处理,当前不能把生产持久化拓扑视为已验证。
|
||||||
|
|
||||||
## 控制面 mTLS 身份审计(2026-08-07)
|
|
||||||
|
|
||||||
- X.509-SVID 叶证书承载单一工作负载身份;Gateway/Checker 启动时的自动身份派生与
|
|
||||||
Controller 授权必须都拒绝包含多个 URI SAN 的证书,即使其中只有一个 URI 与当前角色匹配。
|
|
||||||
- 解析边界已统一到 `workerruntime.SingleSPIFFEIdentity`:精确匹配信任域、环境、角色和
|
|
||||||
标识符,拒绝用户信息、端口、查询、片段、转义路径、额外 URI SAN 与跨角色复用。
|
|
||||||
- TLS 校验可能形成多条等价验证链;授权只检查同一叶证书一次,不能将链路数误判为多个身份。
|
|
||||||
- Windows 本地 Go 运行环境为 `CGO_ENABLED=0` 且没有 C 编译器;Docker Engine 可用,但
|
|
||||||
`golang:1.26-bookworm` 与 `debian:bookworm-slim` 未缓存,Docker Desktop HTTPS 代理也不可用。
|
|
||||||
因此已完成 Go 全量、Compose 静态和 Kustomize 静态验证,容器端到端验证仍待具备镜像网络的环境。
|
|
||||||
|
|
||||||
## OpenAPI 契约审计(2026-08-07)
|
|
||||||
|
|
||||||
- Go 结构契约继续锁定本地引用、operationId、响应与认证引用;额外以固定
|
|
||||||
`@redocly/cli@2.25.4` 对两份 OpenAPI 3.1 文档执行标准验证,避免只依赖自定义遍历器。
|
|
||||||
- 标准最小规则集中的 Tag 描述已提升为错误;Distribution、Health、Status、Audit、Upstreams、
|
|
||||||
Routing 和 Configuration 标签均有面向 API 使用者的稳定说明。
|
|
||||||
|
|
||||||
## Docker 构建上下文审计(2026-08-07)
|
|
||||||
|
|
||||||
- Dockerfile 构建阶段需要 `COPY . .`,因此根 `.dockerignore` 是隔离本地工作区与镜像上下文的
|
|
||||||
必要边界。它显式排除 Git 元数据、环境文件、本地配置、控制面证书/私钥、PEM/CRT/KEY 文件及
|
|
||||||
构建和测试产物;部署测试会在清单被删除或放宽时失败。
|
|
||||||
|
|
||||||
## 部署静态门禁审计(2026-08-07)
|
|
||||||
|
|
||||||
- GitHub Actions 的 Deployment job 在同一组无敏感值 fixture 下渲染生产 Compose、测试 Compose
|
|
||||||
以及 Kubernetes base/development mTLS Overlay,并对两份配置运行严格校验;它仍不替代
|
|
||||||
需要镜像网络和真实服务的容器端到端测试。
|
|
||||||
|
|
||||||
## Kubernetes 配置发布审计(2026-08-07)
|
|
||||||
|
|
||||||
- development mTLS Overlay 的 ConfigMap 采用 Kustomize 内容哈希,渲染契约要求三个工作负载
|
|
||||||
都引用同一版本化名称,因此配置变更进入 Pod 模板并触发滚动。外部固定名称 Secret 不含内容哈希,
|
|
||||||
它们的轮换需要平台侧 reloader 或受控显式滚动;这一限制已写入运行手册。
|
|
||||||
|
|
||||||
## Linux 镜像构建审计(2026-08-07)
|
|
||||||
|
|
||||||
- GitHub Actions 现在在 Linux Runner 执行 Dockerfile 多阶段构建,覆盖目标平台 Go 编译和最小
|
|
||||||
运行时层。该 job 依赖 Runner 镜像网络;它不启动 Compose 服务,因此不能作为 mTLS、存储、
|
|
||||||
探针或优雅停机端到端证据。
|
|
||||||
|
|||||||
@ -134,58 +134,6 @@ func (UnavailableAction) EnumDescriptor() ([]byte, []int) {
|
|||||||
return file_controlplane_v1_controlplane_proto_rawDescGZIP(), []int{1}
|
return file_controlplane_v1_controlplane_proto_rawDescGZIP(), []int{1}
|
||||||
}
|
}
|
||||||
|
|
||||||
type RoutingAction int32
|
|
||||||
|
|
||||||
const (
|
|
||||||
RoutingAction_ROUTING_ACTION_UNSPECIFIED RoutingAction = 0
|
|
||||||
RoutingAction_ROUTING_ACTION_PROXY RoutingAction = 1
|
|
||||||
RoutingAction_ROUTING_ACTION_DIRECT RoutingAction = 2
|
|
||||||
RoutingAction_ROUTING_ACTION_REJECT RoutingAction = 3
|
|
||||||
)
|
|
||||||
|
|
||||||
// Enum value maps for RoutingAction.
|
|
||||||
var (
|
|
||||||
RoutingAction_name = map[int32]string{
|
|
||||||
0: "ROUTING_ACTION_UNSPECIFIED",
|
|
||||||
1: "ROUTING_ACTION_PROXY",
|
|
||||||
2: "ROUTING_ACTION_DIRECT",
|
|
||||||
3: "ROUTING_ACTION_REJECT",
|
|
||||||
}
|
|
||||||
RoutingAction_value = map[string]int32{
|
|
||||||
"ROUTING_ACTION_UNSPECIFIED": 0,
|
|
||||||
"ROUTING_ACTION_PROXY": 1,
|
|
||||||
"ROUTING_ACTION_DIRECT": 2,
|
|
||||||
"ROUTING_ACTION_REJECT": 3,
|
|
||||||
}
|
|
||||||
)
|
|
||||||
|
|
||||||
func (x RoutingAction) Enum() *RoutingAction {
|
|
||||||
p := new(RoutingAction)
|
|
||||||
*p = x
|
|
||||||
return p
|
|
||||||
}
|
|
||||||
|
|
||||||
func (x RoutingAction) String() string {
|
|
||||||
return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x))
|
|
||||||
}
|
|
||||||
|
|
||||||
func (RoutingAction) Descriptor() protoreflect.EnumDescriptor {
|
|
||||||
return file_controlplane_v1_controlplane_proto_enumTypes[2].Descriptor()
|
|
||||||
}
|
|
||||||
|
|
||||||
func (RoutingAction) Type() protoreflect.EnumType {
|
|
||||||
return &file_controlplane_v1_controlplane_proto_enumTypes[2]
|
|
||||||
}
|
|
||||||
|
|
||||||
func (x RoutingAction) Number() protoreflect.EnumNumber {
|
|
||||||
return protoreflect.EnumNumber(x)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Deprecated: Use RoutingAction.Descriptor instead.
|
|
||||||
func (RoutingAction) EnumDescriptor() ([]byte, []int) {
|
|
||||||
return file_controlplane_v1_controlplane_proto_rawDescGZIP(), []int{2}
|
|
||||||
}
|
|
||||||
|
|
||||||
type ProxyProtocol int32
|
type ProxyProtocol int32
|
||||||
|
|
||||||
const (
|
const (
|
||||||
@ -222,11 +170,11 @@ func (x ProxyProtocol) String() string {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (ProxyProtocol) Descriptor() protoreflect.EnumDescriptor {
|
func (ProxyProtocol) Descriptor() protoreflect.EnumDescriptor {
|
||||||
return file_controlplane_v1_controlplane_proto_enumTypes[3].Descriptor()
|
return file_controlplane_v1_controlplane_proto_enumTypes[2].Descriptor()
|
||||||
}
|
}
|
||||||
|
|
||||||
func (ProxyProtocol) Type() protoreflect.EnumType {
|
func (ProxyProtocol) Type() protoreflect.EnumType {
|
||||||
return &file_controlplane_v1_controlplane_proto_enumTypes[3]
|
return &file_controlplane_v1_controlplane_proto_enumTypes[2]
|
||||||
}
|
}
|
||||||
|
|
||||||
func (x ProxyProtocol) Number() protoreflect.EnumNumber {
|
func (x ProxyProtocol) Number() protoreflect.EnumNumber {
|
||||||
@ -235,7 +183,7 @@ func (x ProxyProtocol) Number() protoreflect.EnumNumber {
|
|||||||
|
|
||||||
// Deprecated: Use ProxyProtocol.Descriptor instead.
|
// Deprecated: Use ProxyProtocol.Descriptor instead.
|
||||||
func (ProxyProtocol) EnumDescriptor() ([]byte, []int) {
|
func (ProxyProtocol) EnumDescriptor() ([]byte, []int) {
|
||||||
return file_controlplane_v1_controlplane_proto_rawDescGZIP(), []int{3}
|
return file_controlplane_v1_controlplane_proto_rawDescGZIP(), []int{2}
|
||||||
}
|
}
|
||||||
|
|
||||||
type OutcomeStage int32
|
type OutcomeStage int32
|
||||||
@ -277,11 +225,11 @@ func (x OutcomeStage) String() string {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (OutcomeStage) Descriptor() protoreflect.EnumDescriptor {
|
func (OutcomeStage) Descriptor() protoreflect.EnumDescriptor {
|
||||||
return file_controlplane_v1_controlplane_proto_enumTypes[4].Descriptor()
|
return file_controlplane_v1_controlplane_proto_enumTypes[3].Descriptor()
|
||||||
}
|
}
|
||||||
|
|
||||||
func (OutcomeStage) Type() protoreflect.EnumType {
|
func (OutcomeStage) Type() protoreflect.EnumType {
|
||||||
return &file_controlplane_v1_controlplane_proto_enumTypes[4]
|
return &file_controlplane_v1_controlplane_proto_enumTypes[3]
|
||||||
}
|
}
|
||||||
|
|
||||||
func (x OutcomeStage) Number() protoreflect.EnumNumber {
|
func (x OutcomeStage) Number() protoreflect.EnumNumber {
|
||||||
@ -290,7 +238,7 @@ func (x OutcomeStage) Number() protoreflect.EnumNumber {
|
|||||||
|
|
||||||
// Deprecated: Use OutcomeStage.Descriptor instead.
|
// Deprecated: Use OutcomeStage.Descriptor instead.
|
||||||
func (OutcomeStage) EnumDescriptor() ([]byte, []int) {
|
func (OutcomeStage) EnumDescriptor() ([]byte, []int) {
|
||||||
return file_controlplane_v1_controlplane_proto_rawDescGZIP(), []int{4}
|
return file_controlplane_v1_controlplane_proto_rawDescGZIP(), []int{3}
|
||||||
}
|
}
|
||||||
|
|
||||||
type CheckLevel int32
|
type CheckLevel int32
|
||||||
@ -329,11 +277,11 @@ func (x CheckLevel) String() string {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (CheckLevel) Descriptor() protoreflect.EnumDescriptor {
|
func (CheckLevel) Descriptor() protoreflect.EnumDescriptor {
|
||||||
return file_controlplane_v1_controlplane_proto_enumTypes[5].Descriptor()
|
return file_controlplane_v1_controlplane_proto_enumTypes[4].Descriptor()
|
||||||
}
|
}
|
||||||
|
|
||||||
func (CheckLevel) Type() protoreflect.EnumType {
|
func (CheckLevel) Type() protoreflect.EnumType {
|
||||||
return &file_controlplane_v1_controlplane_proto_enumTypes[5]
|
return &file_controlplane_v1_controlplane_proto_enumTypes[4]
|
||||||
}
|
}
|
||||||
|
|
||||||
func (x CheckLevel) Number() protoreflect.EnumNumber {
|
func (x CheckLevel) Number() protoreflect.EnumNumber {
|
||||||
@ -342,7 +290,7 @@ func (x CheckLevel) Number() protoreflect.EnumNumber {
|
|||||||
|
|
||||||
// Deprecated: Use CheckLevel.Descriptor instead.
|
// Deprecated: Use CheckLevel.Descriptor instead.
|
||||||
func (CheckLevel) EnumDescriptor() ([]byte, []int) {
|
func (CheckLevel) EnumDescriptor() ([]byte, []int) {
|
||||||
return file_controlplane_v1_controlplane_proto_rawDescGZIP(), []int{5}
|
return file_controlplane_v1_controlplane_proto_rawDescGZIP(), []int{4}
|
||||||
}
|
}
|
||||||
|
|
||||||
type RegisterWorkerRequest struct {
|
type RegisterWorkerRequest struct {
|
||||||
@ -937,7 +885,6 @@ type RoutingRule struct {
|
|||||||
Strategy *RoutingStrategy `protobuf:"bytes,8,opt,name=strategy,proto3" json:"strategy,omitempty"`
|
Strategy *RoutingStrategy `protobuf:"bytes,8,opt,name=strategy,proto3" json:"strategy,omitempty"`
|
||||||
OnUnavailable UnavailableAction `protobuf:"varint,9,opt,name=on_unavailable,json=onUnavailable,proto3,enum=proxy_pool.controlplane.v1.UnavailableAction" json:"on_unavailable,omitempty"`
|
OnUnavailable UnavailableAction `protobuf:"varint,9,opt,name=on_unavailable,json=onUnavailable,proto3,enum=proxy_pool.controlplane.v1.UnavailableAction" json:"on_unavailable,omitempty"`
|
||||||
WaitTimeout *durationpb.Duration `protobuf:"bytes,10,opt,name=wait_timeout,json=waitTimeout,proto3" json:"wait_timeout,omitempty"`
|
WaitTimeout *durationpb.Duration `protobuf:"bytes,10,opt,name=wait_timeout,json=waitTimeout,proto3" json:"wait_timeout,omitempty"`
|
||||||
Action RoutingAction `protobuf:"varint,11,opt,name=action,proto3,enum=proxy_pool.controlplane.v1.RoutingAction" json:"action,omitempty"`
|
|
||||||
unknownFields protoimpl.UnknownFields
|
unknownFields protoimpl.UnknownFields
|
||||||
sizeCache protoimpl.SizeCache
|
sizeCache protoimpl.SizeCache
|
||||||
}
|
}
|
||||||
@ -1042,13 +989,6 @@ func (x *RoutingRule) GetWaitTimeout() *durationpb.Duration {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (x *RoutingRule) GetAction() RoutingAction {
|
|
||||||
if x != nil {
|
|
||||||
return x.Action
|
|
||||||
}
|
|
||||||
return RoutingAction_ROUTING_ACTION_UNSPECIFIED
|
|
||||||
}
|
|
||||||
|
|
||||||
type RoutingStrategy struct {
|
type RoutingStrategy struct {
|
||||||
state protoimpl.MessageState `protogen:"open.v1"`
|
state protoimpl.MessageState `protogen:"open.v1"`
|
||||||
Type StrategyType `protobuf:"varint,1,opt,name=type,proto3,enum=proxy_pool.controlplane.v1.StrategyType" json:"type,omitempty"`
|
Type StrategyType `protobuf:"varint,1,opt,name=type,proto3,enum=proxy_pool.controlplane.v1.StrategyType" json:"type,omitempty"`
|
||||||
@ -2300,7 +2240,7 @@ const file_controlplane_v1_controlplane_proto_rawDesc = "" +
|
|||||||
"\x10upserted_routing\x18\x06 \x03(\v2'.proxy_pool.controlplane.v1.RoutingRuleR\x0fupsertedRouting\x122\n" +
|
"\x10upserted_routing\x18\x06 \x03(\v2'.proxy_pool.controlplane.v1.RoutingRuleR\x0fupsertedRouting\x122\n" +
|
||||||
"\x15removed_routing_names\x18\a \x03(\tR\x13removedRoutingNames\x12Q\n" +
|
"\x15removed_routing_names\x18\a \x03(\tR\x13removedRoutingNames\x12Q\n" +
|
||||||
"\x10upserted_proxies\x18\b \x03(\v2&.proxy_pool.controlplane.v1.OwnedProxyR\x0fupsertedProxies\x12*\n" +
|
"\x10upserted_proxies\x18\b \x03(\v2&.proxy_pool.controlplane.v1.OwnedProxyR\x0fupsertedProxies\x12*\n" +
|
||||||
"\x11removed_proxy_ids\x18\t \x03(\tR\x0fremovedProxyIds\"\xdd\x04\n" +
|
"\x11removed_proxy_ids\x18\t \x03(\tR\x0fremovedProxyIds\"\x9a\x04\n" +
|
||||||
"\vRoutingRule\x12\x12\n" +
|
"\vRoutingRule\x12\x12\n" +
|
||||||
"\x04name\x18\x01 \x01(\tR\x04name\x12\x18\n" +
|
"\x04name\x18\x01 \x01(\tR\x04name\x12\x18\n" +
|
||||||
"\aenabled\x18\x02 \x01(\bR\aenabled\x12\x1d\n" +
|
"\aenabled\x18\x02 \x01(\bR\aenabled\x12\x1d\n" +
|
||||||
@ -2314,8 +2254,7 @@ const file_controlplane_v1_controlplane_proto_rawDesc = "" +
|
|||||||
"\bstrategy\x18\b \x01(\v2+.proxy_pool.controlplane.v1.RoutingStrategyR\bstrategy\x12T\n" +
|
"\bstrategy\x18\b \x01(\v2+.proxy_pool.controlplane.v1.RoutingStrategyR\bstrategy\x12T\n" +
|
||||||
"\x0eon_unavailable\x18\t \x01(\x0e2-.proxy_pool.controlplane.v1.UnavailableActionR\ronUnavailable\x12<\n" +
|
"\x0eon_unavailable\x18\t \x01(\x0e2-.proxy_pool.controlplane.v1.UnavailableActionR\ronUnavailable\x12<\n" +
|
||||||
"\fwait_timeout\x18\n" +
|
"\fwait_timeout\x18\n" +
|
||||||
" \x01(\v2\x19.google.protobuf.DurationR\vwaitTimeout\x12A\n" +
|
" \x01(\v2\x19.google.protobuf.DurationR\vwaitTimeout\x1a:\n" +
|
||||||
"\x06action\x18\v \x01(\x0e2).proxy_pool.controlplane.v1.RoutingActionR\x06action\x1a:\n" +
|
|
||||||
"\fHeadersEntry\x12\x10\n" +
|
"\fHeadersEntry\x12\x10\n" +
|
||||||
"\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" +
|
"\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" +
|
||||||
"\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\"\x8a\x02\n" +
|
"\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\"\x8a\x02\n" +
|
||||||
@ -2458,12 +2397,7 @@ const file_controlplane_v1_controlplane_proto_rawDesc = "" +
|
|||||||
"\x1eUNAVAILABLE_ACTION_UNSPECIFIED\x10\x00\x12\x1d\n" +
|
"\x1eUNAVAILABLE_ACTION_UNSPECIFIED\x10\x00\x12\x1d\n" +
|
||||||
"\x19UNAVAILABLE_ACTION_REJECT\x10\x01\x12\x1b\n" +
|
"\x19UNAVAILABLE_ACTION_REJECT\x10\x01\x12\x1b\n" +
|
||||||
"\x17UNAVAILABLE_ACTION_WAIT\x10\x02\x12\x1d\n" +
|
"\x17UNAVAILABLE_ACTION_WAIT\x10\x02\x12\x1d\n" +
|
||||||
"\x19UNAVAILABLE_ACTION_DIRECT\x10\x03*\x7f\n" +
|
"\x19UNAVAILABLE_ACTION_DIRECT\x10\x03*}\n" +
|
||||||
"\rRoutingAction\x12\x1e\n" +
|
|
||||||
"\x1aROUTING_ACTION_UNSPECIFIED\x10\x00\x12\x18\n" +
|
|
||||||
"\x14ROUTING_ACTION_PROXY\x10\x01\x12\x19\n" +
|
|
||||||
"\x15ROUTING_ACTION_DIRECT\x10\x02\x12\x19\n" +
|
|
||||||
"\x15ROUTING_ACTION_REJECT\x10\x03*}\n" +
|
|
||||||
"\rProxyProtocol\x12\x1e\n" +
|
"\rProxyProtocol\x12\x1e\n" +
|
||||||
"\x1aPROXY_PROTOCOL_UNSPECIFIED\x10\x00\x12\x17\n" +
|
"\x1aPROXY_PROTOCOL_UNSPECIFIED\x10\x00\x12\x17\n" +
|
||||||
"\x13PROXY_PROTOCOL_HTTP\x10\x01\x12\x18\n" +
|
"\x13PROXY_PROTOCOL_HTTP\x10\x01\x12\x18\n" +
|
||||||
@ -2503,104 +2437,102 @@ func file_controlplane_v1_controlplane_proto_rawDescGZIP() []byte {
|
|||||||
return file_controlplane_v1_controlplane_proto_rawDescData
|
return file_controlplane_v1_controlplane_proto_rawDescData
|
||||||
}
|
}
|
||||||
|
|
||||||
var file_controlplane_v1_controlplane_proto_enumTypes = make([]protoimpl.EnumInfo, 6)
|
var file_controlplane_v1_controlplane_proto_enumTypes = make([]protoimpl.EnumInfo, 5)
|
||||||
var file_controlplane_v1_controlplane_proto_msgTypes = make([]protoimpl.MessageInfo, 26)
|
var file_controlplane_v1_controlplane_proto_msgTypes = make([]protoimpl.MessageInfo, 26)
|
||||||
var file_controlplane_v1_controlplane_proto_goTypes = []any{
|
var file_controlplane_v1_controlplane_proto_goTypes = []any{
|
||||||
(StrategyType)(0), // 0: proxy_pool.controlplane.v1.StrategyType
|
(StrategyType)(0), // 0: proxy_pool.controlplane.v1.StrategyType
|
||||||
(UnavailableAction)(0), // 1: proxy_pool.controlplane.v1.UnavailableAction
|
(UnavailableAction)(0), // 1: proxy_pool.controlplane.v1.UnavailableAction
|
||||||
(RoutingAction)(0), // 2: proxy_pool.controlplane.v1.RoutingAction
|
(ProxyProtocol)(0), // 2: proxy_pool.controlplane.v1.ProxyProtocol
|
||||||
(ProxyProtocol)(0), // 3: proxy_pool.controlplane.v1.ProxyProtocol
|
(OutcomeStage)(0), // 3: proxy_pool.controlplane.v1.OutcomeStage
|
||||||
(OutcomeStage)(0), // 4: proxy_pool.controlplane.v1.OutcomeStage
|
(CheckLevel)(0), // 4: proxy_pool.controlplane.v1.CheckLevel
|
||||||
(CheckLevel)(0), // 5: proxy_pool.controlplane.v1.CheckLevel
|
(*RegisterWorkerRequest)(nil), // 5: proxy_pool.controlplane.v1.RegisterWorkerRequest
|
||||||
(*RegisterWorkerRequest)(nil), // 6: proxy_pool.controlplane.v1.RegisterWorkerRequest
|
(*RegisterWorkerResponse)(nil), // 6: proxy_pool.controlplane.v1.RegisterWorkerResponse
|
||||||
(*RegisterWorkerResponse)(nil), // 7: proxy_pool.controlplane.v1.RegisterWorkerResponse
|
(*WatchSnapshotsRequest)(nil), // 7: proxy_pool.controlplane.v1.WatchSnapshotsRequest
|
||||||
(*WatchSnapshotsRequest)(nil), // 8: proxy_pool.controlplane.v1.WatchSnapshotsRequest
|
(*SnapshotEnvelope)(nil), // 8: proxy_pool.controlplane.v1.SnapshotEnvelope
|
||||||
(*SnapshotEnvelope)(nil), // 9: proxy_pool.controlplane.v1.SnapshotEnvelope
|
(*WorkerSnapshot)(nil), // 9: proxy_pool.controlplane.v1.WorkerSnapshot
|
||||||
(*WorkerSnapshot)(nil), // 10: proxy_pool.controlplane.v1.WorkerSnapshot
|
(*SnapshotCredential)(nil), // 10: proxy_pool.controlplane.v1.SnapshotCredential
|
||||||
(*SnapshotCredential)(nil), // 11: proxy_pool.controlplane.v1.SnapshotCredential
|
(*SnapshotDelta)(nil), // 11: proxy_pool.controlplane.v1.SnapshotDelta
|
||||||
(*SnapshotDelta)(nil), // 12: proxy_pool.controlplane.v1.SnapshotDelta
|
(*RoutingRule)(nil), // 12: proxy_pool.controlplane.v1.RoutingRule
|
||||||
(*RoutingRule)(nil), // 13: proxy_pool.controlplane.v1.RoutingRule
|
(*RoutingStrategy)(nil), // 13: proxy_pool.controlplane.v1.RoutingStrategy
|
||||||
(*RoutingStrategy)(nil), // 14: proxy_pool.controlplane.v1.RoutingStrategy
|
(*OwnedProxy)(nil), // 14: proxy_pool.controlplane.v1.OwnedProxy
|
||||||
(*OwnedProxy)(nil), // 15: proxy_pool.controlplane.v1.OwnedProxy
|
(*AcknowledgeSnapshotRequest)(nil), // 15: proxy_pool.controlplane.v1.AcknowledgeSnapshotRequest
|
||||||
(*AcknowledgeSnapshotRequest)(nil), // 16: proxy_pool.controlplane.v1.AcknowledgeSnapshotRequest
|
(*OutcomeBatch)(nil), // 16: proxy_pool.controlplane.v1.OutcomeBatch
|
||||||
(*OutcomeBatch)(nil), // 17: proxy_pool.controlplane.v1.OutcomeBatch
|
(*ProxyOutcome)(nil), // 17: proxy_pool.controlplane.v1.ProxyOutcome
|
||||||
(*ProxyOutcome)(nil), // 18: proxy_pool.controlplane.v1.ProxyOutcome
|
(*ReportOutcomesResponse)(nil), // 18: proxy_pool.controlplane.v1.ReportOutcomesResponse
|
||||||
(*ReportOutcomesResponse)(nil), // 19: proxy_pool.controlplane.v1.ReportOutcomesResponse
|
(*ReportRuntimeRequest)(nil), // 19: proxy_pool.controlplane.v1.ReportRuntimeRequest
|
||||||
(*ReportRuntimeRequest)(nil), // 20: proxy_pool.controlplane.v1.ReportRuntimeRequest
|
(*ProxyRuntime)(nil), // 20: proxy_pool.controlplane.v1.ProxyRuntime
|
||||||
(*ProxyRuntime)(nil), // 21: proxy_pool.controlplane.v1.ProxyRuntime
|
(*ReportRuntimeResponse)(nil), // 21: proxy_pool.controlplane.v1.ReportRuntimeResponse
|
||||||
(*ReportRuntimeResponse)(nil), // 22: proxy_pool.controlplane.v1.ReportRuntimeResponse
|
(*StreamCheckTasksRequest)(nil), // 22: proxy_pool.controlplane.v1.StreamCheckTasksRequest
|
||||||
(*StreamCheckTasksRequest)(nil), // 23: proxy_pool.controlplane.v1.StreamCheckTasksRequest
|
(*CheckTask)(nil), // 23: proxy_pool.controlplane.v1.CheckTask
|
||||||
(*CheckTask)(nil), // 24: proxy_pool.controlplane.v1.CheckTask
|
(*ObservationBatch)(nil), // 24: proxy_pool.controlplane.v1.ObservationBatch
|
||||||
(*ObservationBatch)(nil), // 25: proxy_pool.controlplane.v1.ObservationBatch
|
(*HealthObservation)(nil), // 25: proxy_pool.controlplane.v1.HealthObservation
|
||||||
(*HealthObservation)(nil), // 26: proxy_pool.controlplane.v1.HealthObservation
|
(*ReportObservationsResponse)(nil), // 26: proxy_pool.controlplane.v1.ReportObservationsResponse
|
||||||
(*ReportObservationsResponse)(nil), // 27: proxy_pool.controlplane.v1.ReportObservationsResponse
|
nil, // 27: proxy_pool.controlplane.v1.RegisterWorkerRequest.LabelsEntry
|
||||||
nil, // 28: proxy_pool.controlplane.v1.RegisterWorkerRequest.LabelsEntry
|
nil, // 28: proxy_pool.controlplane.v1.RoutingRule.HeadersEntry
|
||||||
nil, // 29: proxy_pool.controlplane.v1.RoutingRule.HeadersEntry
|
nil, // 29: proxy_pool.controlplane.v1.RoutingStrategy.WeightsEntry
|
||||||
nil, // 30: proxy_pool.controlplane.v1.RoutingStrategy.WeightsEntry
|
nil, // 30: proxy_pool.controlplane.v1.OwnedProxy.TagsEntry
|
||||||
nil, // 31: proxy_pool.controlplane.v1.OwnedProxy.TagsEntry
|
(*durationpb.Duration)(nil), // 31: google.protobuf.Duration
|
||||||
(*durationpb.Duration)(nil), // 32: google.protobuf.Duration
|
(*timestamppb.Timestamp)(nil), // 32: google.protobuf.Timestamp
|
||||||
(*timestamppb.Timestamp)(nil), // 33: google.protobuf.Timestamp
|
(*emptypb.Empty)(nil), // 33: google.protobuf.Empty
|
||||||
(*emptypb.Empty)(nil), // 34: google.protobuf.Empty
|
|
||||||
}
|
}
|
||||||
var file_controlplane_v1_controlplane_proto_depIdxs = []int32{
|
var file_controlplane_v1_controlplane_proto_depIdxs = []int32{
|
||||||
28, // 0: proxy_pool.controlplane.v1.RegisterWorkerRequest.labels:type_name -> proxy_pool.controlplane.v1.RegisterWorkerRequest.LabelsEntry
|
27, // 0: proxy_pool.controlplane.v1.RegisterWorkerRequest.labels:type_name -> proxy_pool.controlplane.v1.RegisterWorkerRequest.LabelsEntry
|
||||||
32, // 1: proxy_pool.controlplane.v1.RegisterWorkerResponse.heartbeat_interval:type_name -> google.protobuf.Duration
|
31, // 1: proxy_pool.controlplane.v1.RegisterWorkerResponse.heartbeat_interval:type_name -> google.protobuf.Duration
|
||||||
32, // 2: proxy_pool.controlplane.v1.RegisterWorkerResponse.max_stale_age:type_name -> google.protobuf.Duration
|
31, // 2: proxy_pool.controlplane.v1.RegisterWorkerResponse.max_stale_age:type_name -> google.protobuf.Duration
|
||||||
10, // 3: proxy_pool.controlplane.v1.SnapshotEnvelope.full:type_name -> proxy_pool.controlplane.v1.WorkerSnapshot
|
9, // 3: proxy_pool.controlplane.v1.SnapshotEnvelope.full:type_name -> proxy_pool.controlplane.v1.WorkerSnapshot
|
||||||
12, // 4: proxy_pool.controlplane.v1.SnapshotEnvelope.delta:type_name -> proxy_pool.controlplane.v1.SnapshotDelta
|
11, // 4: proxy_pool.controlplane.v1.SnapshotEnvelope.delta:type_name -> proxy_pool.controlplane.v1.SnapshotDelta
|
||||||
33, // 5: proxy_pool.controlplane.v1.WorkerSnapshot.generated_at:type_name -> google.protobuf.Timestamp
|
32, // 5: proxy_pool.controlplane.v1.WorkerSnapshot.generated_at:type_name -> google.protobuf.Timestamp
|
||||||
33, // 6: proxy_pool.controlplane.v1.WorkerSnapshot.valid_until:type_name -> google.protobuf.Timestamp
|
32, // 6: proxy_pool.controlplane.v1.WorkerSnapshot.valid_until:type_name -> google.protobuf.Timestamp
|
||||||
13, // 7: proxy_pool.controlplane.v1.WorkerSnapshot.routing:type_name -> proxy_pool.controlplane.v1.RoutingRule
|
12, // 7: proxy_pool.controlplane.v1.WorkerSnapshot.routing:type_name -> proxy_pool.controlplane.v1.RoutingRule
|
||||||
15, // 8: proxy_pool.controlplane.v1.WorkerSnapshot.proxies:type_name -> proxy_pool.controlplane.v1.OwnedProxy
|
14, // 8: proxy_pool.controlplane.v1.WorkerSnapshot.proxies:type_name -> proxy_pool.controlplane.v1.OwnedProxy
|
||||||
11, // 9: proxy_pool.controlplane.v1.WorkerSnapshot.credentials:type_name -> proxy_pool.controlplane.v1.SnapshotCredential
|
10, // 9: proxy_pool.controlplane.v1.WorkerSnapshot.credentials:type_name -> proxy_pool.controlplane.v1.SnapshotCredential
|
||||||
33, // 10: proxy_pool.controlplane.v1.SnapshotDelta.generated_at:type_name -> google.protobuf.Timestamp
|
32, // 10: proxy_pool.controlplane.v1.SnapshotDelta.generated_at:type_name -> google.protobuf.Timestamp
|
||||||
13, // 11: proxy_pool.controlplane.v1.SnapshotDelta.upserted_routing:type_name -> proxy_pool.controlplane.v1.RoutingRule
|
12, // 11: proxy_pool.controlplane.v1.SnapshotDelta.upserted_routing:type_name -> proxy_pool.controlplane.v1.RoutingRule
|
||||||
15, // 12: proxy_pool.controlplane.v1.SnapshotDelta.upserted_proxies:type_name -> proxy_pool.controlplane.v1.OwnedProxy
|
14, // 12: proxy_pool.controlplane.v1.SnapshotDelta.upserted_proxies:type_name -> proxy_pool.controlplane.v1.OwnedProxy
|
||||||
29, // 13: proxy_pool.controlplane.v1.RoutingRule.headers:type_name -> proxy_pool.controlplane.v1.RoutingRule.HeadersEntry
|
28, // 13: proxy_pool.controlplane.v1.RoutingRule.headers:type_name -> proxy_pool.controlplane.v1.RoutingRule.HeadersEntry
|
||||||
14, // 14: proxy_pool.controlplane.v1.RoutingRule.strategy:type_name -> proxy_pool.controlplane.v1.RoutingStrategy
|
13, // 14: proxy_pool.controlplane.v1.RoutingRule.strategy:type_name -> proxy_pool.controlplane.v1.RoutingStrategy
|
||||||
1, // 15: proxy_pool.controlplane.v1.RoutingRule.on_unavailable:type_name -> proxy_pool.controlplane.v1.UnavailableAction
|
1, // 15: proxy_pool.controlplane.v1.RoutingRule.on_unavailable:type_name -> proxy_pool.controlplane.v1.UnavailableAction
|
||||||
32, // 16: proxy_pool.controlplane.v1.RoutingRule.wait_timeout:type_name -> google.protobuf.Duration
|
31, // 16: proxy_pool.controlplane.v1.RoutingRule.wait_timeout:type_name -> google.protobuf.Duration
|
||||||
2, // 17: proxy_pool.controlplane.v1.RoutingRule.action:type_name -> proxy_pool.controlplane.v1.RoutingAction
|
0, // 17: proxy_pool.controlplane.v1.RoutingStrategy.type:type_name -> proxy_pool.controlplane.v1.StrategyType
|
||||||
0, // 18: proxy_pool.controlplane.v1.RoutingStrategy.type:type_name -> proxy_pool.controlplane.v1.StrategyType
|
29, // 18: proxy_pool.controlplane.v1.RoutingStrategy.weights:type_name -> proxy_pool.controlplane.v1.RoutingStrategy.WeightsEntry
|
||||||
30, // 19: proxy_pool.controlplane.v1.RoutingStrategy.weights:type_name -> proxy_pool.controlplane.v1.RoutingStrategy.WeightsEntry
|
2, // 19: proxy_pool.controlplane.v1.OwnedProxy.protocol:type_name -> proxy_pool.controlplane.v1.ProxyProtocol
|
||||||
3, // 20: proxy_pool.controlplane.v1.OwnedProxy.protocol:type_name -> proxy_pool.controlplane.v1.ProxyProtocol
|
32, // 20: proxy_pool.controlplane.v1.OwnedProxy.expires_at:type_name -> google.protobuf.Timestamp
|
||||||
33, // 21: proxy_pool.controlplane.v1.OwnedProxy.expires_at:type_name -> google.protobuf.Timestamp
|
30, // 21: proxy_pool.controlplane.v1.OwnedProxy.tags:type_name -> proxy_pool.controlplane.v1.OwnedProxy.TagsEntry
|
||||||
31, // 22: proxy_pool.controlplane.v1.OwnedProxy.tags:type_name -> proxy_pool.controlplane.v1.OwnedProxy.TagsEntry
|
32, // 22: proxy_pool.controlplane.v1.OwnedProxy.usable_until:type_name -> google.protobuf.Timestamp
|
||||||
33, // 23: proxy_pool.controlplane.v1.OwnedProxy.usable_until:type_name -> google.protobuf.Timestamp
|
17, // 23: proxy_pool.controlplane.v1.OutcomeBatch.outcomes:type_name -> proxy_pool.controlplane.v1.ProxyOutcome
|
||||||
18, // 24: proxy_pool.controlplane.v1.OutcomeBatch.outcomes:type_name -> proxy_pool.controlplane.v1.ProxyOutcome
|
3, // 24: proxy_pool.controlplane.v1.ProxyOutcome.stage:type_name -> proxy_pool.controlplane.v1.OutcomeStage
|
||||||
4, // 25: proxy_pool.controlplane.v1.ProxyOutcome.stage:type_name -> proxy_pool.controlplane.v1.OutcomeStage
|
31, // 25: proxy_pool.controlplane.v1.ProxyOutcome.latency:type_name -> google.protobuf.Duration
|
||||||
32, // 26: proxy_pool.controlplane.v1.ProxyOutcome.latency:type_name -> google.protobuf.Duration
|
32, // 26: proxy_pool.controlplane.v1.ProxyOutcome.observed_at:type_name -> google.protobuf.Timestamp
|
||||||
33, // 27: proxy_pool.controlplane.v1.ProxyOutcome.observed_at:type_name -> google.protobuf.Timestamp
|
20, // 27: proxy_pool.controlplane.v1.ReportRuntimeRequest.counters:type_name -> proxy_pool.controlplane.v1.ProxyRuntime
|
||||||
21, // 28: proxy_pool.controlplane.v1.ReportRuntimeRequest.counters:type_name -> proxy_pool.controlplane.v1.ProxyRuntime
|
32, // 28: proxy_pool.controlplane.v1.ReportRuntimeRequest.observed_at:type_name -> google.protobuf.Timestamp
|
||||||
33, // 29: proxy_pool.controlplane.v1.ReportRuntimeRequest.observed_at:type_name -> google.protobuf.Timestamp
|
4, // 29: proxy_pool.controlplane.v1.StreamCheckTasksRequest.supported_levels:type_name -> proxy_pool.controlplane.v1.CheckLevel
|
||||||
5, // 30: proxy_pool.controlplane.v1.StreamCheckTasksRequest.supported_levels:type_name -> proxy_pool.controlplane.v1.CheckLevel
|
2, // 30: proxy_pool.controlplane.v1.CheckTask.protocol:type_name -> proxy_pool.controlplane.v1.ProxyProtocol
|
||||||
3, // 31: proxy_pool.controlplane.v1.CheckTask.protocol:type_name -> proxy_pool.controlplane.v1.ProxyProtocol
|
4, // 31: proxy_pool.controlplane.v1.CheckTask.level:type_name -> proxy_pool.controlplane.v1.CheckLevel
|
||||||
5, // 32: proxy_pool.controlplane.v1.CheckTask.level:type_name -> proxy_pool.controlplane.v1.CheckLevel
|
31, // 32: proxy_pool.controlplane.v1.CheckTask.timeout:type_name -> google.protobuf.Duration
|
||||||
32, // 33: proxy_pool.controlplane.v1.CheckTask.timeout:type_name -> google.protobuf.Duration
|
32, // 33: proxy_pool.controlplane.v1.CheckTask.deadline:type_name -> google.protobuf.Timestamp
|
||||||
33, // 34: proxy_pool.controlplane.v1.CheckTask.deadline:type_name -> google.protobuf.Timestamp
|
25, // 34: proxy_pool.controlplane.v1.ObservationBatch.observations:type_name -> proxy_pool.controlplane.v1.HealthObservation
|
||||||
26, // 35: proxy_pool.controlplane.v1.ObservationBatch.observations:type_name -> proxy_pool.controlplane.v1.HealthObservation
|
4, // 35: proxy_pool.controlplane.v1.HealthObservation.level:type_name -> proxy_pool.controlplane.v1.CheckLevel
|
||||||
5, // 36: proxy_pool.controlplane.v1.HealthObservation.level:type_name -> proxy_pool.controlplane.v1.CheckLevel
|
31, // 36: proxy_pool.controlplane.v1.HealthObservation.latency:type_name -> google.protobuf.Duration
|
||||||
32, // 37: proxy_pool.controlplane.v1.HealthObservation.latency:type_name -> google.protobuf.Duration
|
32, // 37: proxy_pool.controlplane.v1.HealthObservation.observed_at:type_name -> google.protobuf.Timestamp
|
||||||
33, // 38: proxy_pool.controlplane.v1.HealthObservation.observed_at:type_name -> google.protobuf.Timestamp
|
5, // 38: proxy_pool.controlplane.v1.WorkerControlPlane.RegisterWorker:input_type -> proxy_pool.controlplane.v1.RegisterWorkerRequest
|
||||||
6, // 39: proxy_pool.controlplane.v1.WorkerControlPlane.RegisterWorker:input_type -> proxy_pool.controlplane.v1.RegisterWorkerRequest
|
7, // 39: proxy_pool.controlplane.v1.WorkerControlPlane.WatchSnapshots:input_type -> proxy_pool.controlplane.v1.WatchSnapshotsRequest
|
||||||
8, // 40: proxy_pool.controlplane.v1.WorkerControlPlane.WatchSnapshots:input_type -> proxy_pool.controlplane.v1.WatchSnapshotsRequest
|
15, // 40: proxy_pool.controlplane.v1.WorkerControlPlane.AcknowledgeSnapshot:input_type -> proxy_pool.controlplane.v1.AcknowledgeSnapshotRequest
|
||||||
16, // 41: proxy_pool.controlplane.v1.WorkerControlPlane.AcknowledgeSnapshot:input_type -> proxy_pool.controlplane.v1.AcknowledgeSnapshotRequest
|
16, // 41: proxy_pool.controlplane.v1.WorkerControlPlane.ReportOutcomes:input_type -> proxy_pool.controlplane.v1.OutcomeBatch
|
||||||
17, // 42: proxy_pool.controlplane.v1.WorkerControlPlane.ReportOutcomes:input_type -> proxy_pool.controlplane.v1.OutcomeBatch
|
19, // 42: proxy_pool.controlplane.v1.WorkerControlPlane.ReportRuntime:input_type -> proxy_pool.controlplane.v1.ReportRuntimeRequest
|
||||||
20, // 43: proxy_pool.controlplane.v1.WorkerControlPlane.ReportRuntime:input_type -> proxy_pool.controlplane.v1.ReportRuntimeRequest
|
22, // 43: proxy_pool.controlplane.v1.CheckerControlPlane.StreamCheckTasks:input_type -> proxy_pool.controlplane.v1.StreamCheckTasksRequest
|
||||||
23, // 44: proxy_pool.controlplane.v1.CheckerControlPlane.StreamCheckTasks:input_type -> proxy_pool.controlplane.v1.StreamCheckTasksRequest
|
24, // 44: proxy_pool.controlplane.v1.CheckerControlPlane.ReportObservations:input_type -> proxy_pool.controlplane.v1.ObservationBatch
|
||||||
25, // 45: proxy_pool.controlplane.v1.CheckerControlPlane.ReportObservations:input_type -> proxy_pool.controlplane.v1.ObservationBatch
|
6, // 45: proxy_pool.controlplane.v1.WorkerControlPlane.RegisterWorker:output_type -> proxy_pool.controlplane.v1.RegisterWorkerResponse
|
||||||
7, // 46: proxy_pool.controlplane.v1.WorkerControlPlane.RegisterWorker:output_type -> proxy_pool.controlplane.v1.RegisterWorkerResponse
|
8, // 46: proxy_pool.controlplane.v1.WorkerControlPlane.WatchSnapshots:output_type -> proxy_pool.controlplane.v1.SnapshotEnvelope
|
||||||
9, // 47: proxy_pool.controlplane.v1.WorkerControlPlane.WatchSnapshots:output_type -> proxy_pool.controlplane.v1.SnapshotEnvelope
|
33, // 47: proxy_pool.controlplane.v1.WorkerControlPlane.AcknowledgeSnapshot:output_type -> google.protobuf.Empty
|
||||||
34, // 48: proxy_pool.controlplane.v1.WorkerControlPlane.AcknowledgeSnapshot:output_type -> google.protobuf.Empty
|
18, // 48: proxy_pool.controlplane.v1.WorkerControlPlane.ReportOutcomes:output_type -> proxy_pool.controlplane.v1.ReportOutcomesResponse
|
||||||
19, // 49: proxy_pool.controlplane.v1.WorkerControlPlane.ReportOutcomes:output_type -> proxy_pool.controlplane.v1.ReportOutcomesResponse
|
21, // 49: proxy_pool.controlplane.v1.WorkerControlPlane.ReportRuntime:output_type -> proxy_pool.controlplane.v1.ReportRuntimeResponse
|
||||||
22, // 50: proxy_pool.controlplane.v1.WorkerControlPlane.ReportRuntime:output_type -> proxy_pool.controlplane.v1.ReportRuntimeResponse
|
23, // 50: proxy_pool.controlplane.v1.CheckerControlPlane.StreamCheckTasks:output_type -> proxy_pool.controlplane.v1.CheckTask
|
||||||
24, // 51: proxy_pool.controlplane.v1.CheckerControlPlane.StreamCheckTasks:output_type -> proxy_pool.controlplane.v1.CheckTask
|
26, // 51: proxy_pool.controlplane.v1.CheckerControlPlane.ReportObservations:output_type -> proxy_pool.controlplane.v1.ReportObservationsResponse
|
||||||
27, // 52: proxy_pool.controlplane.v1.CheckerControlPlane.ReportObservations:output_type -> proxy_pool.controlplane.v1.ReportObservationsResponse
|
45, // [45:52] is the sub-list for method output_type
|
||||||
46, // [46:53] is the sub-list for method output_type
|
38, // [38:45] is the sub-list for method input_type
|
||||||
39, // [39:46] is the sub-list for method input_type
|
38, // [38:38] is the sub-list for extension type_name
|
||||||
39, // [39:39] is the sub-list for extension type_name
|
38, // [38:38] is the sub-list for extension extendee
|
||||||
39, // [39:39] is the sub-list for extension extendee
|
0, // [0:38] is the sub-list for field type_name
|
||||||
0, // [0:39] is the sub-list for field type_name
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func init() { file_controlplane_v1_controlplane_proto_init() }
|
func init() { file_controlplane_v1_controlplane_proto_init() }
|
||||||
@ -2617,7 +2549,7 @@ func file_controlplane_v1_controlplane_proto_init() {
|
|||||||
File: protoimpl.DescBuilder{
|
File: protoimpl.DescBuilder{
|
||||||
GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
|
GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
|
||||||
RawDescriptor: unsafe.Slice(unsafe.StringData(file_controlplane_v1_controlplane_proto_rawDesc), len(file_controlplane_v1_controlplane_proto_rawDesc)),
|
RawDescriptor: unsafe.Slice(unsafe.StringData(file_controlplane_v1_controlplane_proto_rawDesc), len(file_controlplane_v1_controlplane_proto_rawDesc)),
|
||||||
NumEnums: 6,
|
NumEnums: 5,
|
||||||
NumMessages: 26,
|
NumMessages: 26,
|
||||||
NumExtensions: 0,
|
NumExtensions: 0,
|
||||||
NumServices: 2,
|
NumServices: 2,
|
||||||
|
|||||||
@ -65,10 +65,6 @@ func TestOperationsPrioritizeCanceledContext(t *testing.T) {
|
|||||||
_, operationErr := store.SwitchRouting(ctx, adminstate.SwitchRoutingCommand{})
|
_, operationErr := store.SwitchRouting(ctx, adminstate.SwitchRoutingCommand{})
|
||||||
return operationErr
|
return operationErr
|
||||||
}},
|
}},
|
||||||
{name: "disable routing", run: func() error {
|
|
||||||
_, operationErr := store.DisableRouting(ctx, adminstate.DisableRoutingCommand{})
|
|
||||||
return operationErr
|
|
||||||
}},
|
|
||||||
{name: "snapshot", run: func() error {
|
{name: "snapshot", run: func() error {
|
||||||
_, operationErr := store.Snapshot(ctx)
|
_, operationErr := store.Snapshot(ctx)
|
||||||
return operationErr
|
return operationErr
|
||||||
@ -123,10 +119,6 @@ func TestInvalidOperationsDoNotBeginTransaction(t *testing.T) {
|
|||||||
_, operationErr := store.SwitchRouting(context.Background(), adminstate.SwitchRoutingCommand{})
|
_, operationErr := store.SwitchRouting(context.Background(), adminstate.SwitchRoutingCommand{})
|
||||||
return operationErr
|
return operationErr
|
||||||
}},
|
}},
|
||||||
{name: "disable routing", run: func() error {
|
|
||||||
_, operationErr := store.DisableRouting(context.Background(), adminstate.DisableRoutingCommand{})
|
|
||||||
return operationErr
|
|
||||||
}},
|
|
||||||
{name: "read audit", run: func() error {
|
{name: "read audit", run: func() error {
|
||||||
_, operationErr := store.ReadAudit(context.Background(), adminstate.AuditQuery{})
|
_, operationErr := store.ReadAudit(context.Background(), adminstate.AuditQuery{})
|
||||||
return operationErr
|
return operationErr
|
||||||
|
|||||||
@ -1,15 +0,0 @@
|
|||||||
BEGIN;
|
|
||||||
|
|
||||||
ALTER TABLE admin_audit_log
|
|
||||||
DROP CONSTRAINT IF EXISTS admin_audit_log_action_check;
|
|
||||||
|
|
||||||
ALTER TABLE admin_audit_log
|
|
||||||
ADD CONSTRAINT admin_audit_log_action_check
|
|
||||||
CHECK (action IN (
|
|
||||||
'commit_config',
|
|
||||||
'set_upstream_enabled',
|
|
||||||
'switch_routing',
|
|
||||||
'disable_routing'
|
|
||||||
));
|
|
||||||
|
|
||||||
COMMIT;
|
|
||||||
@ -27,18 +27,15 @@ func TestApplyMigrationsValidatesContextAndPool(t *testing.T) {
|
|||||||
func TestMigrationsAreOrderedTransactionalAndImmutable(t *testing.T) {
|
func TestMigrationsAreOrderedTransactionalAndImmutable(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
migrations := Migrations()
|
migrations := Migrations()
|
||||||
if len(migrations) != 2 {
|
if len(migrations) != 1 {
|
||||||
t.Fatalf("len(Migrations()) = %d, want 2", len(migrations))
|
t.Fatalf("len(Migrations()) = %d, want 1", len(migrations))
|
||||||
}
|
}
|
||||||
if migrations[0].Version != 1 || migrations[0].Name != "admin_state" ||
|
if migrations[0].Version != 1 || migrations[0].Name != "admin_state" {
|
||||||
migrations[1].Version != 2 || migrations[1].Name != "disable_routing" {
|
t.Fatalf("migration metadata = %+v", migrations[0])
|
||||||
t.Fatalf("migration metadata = %+v", migrations)
|
|
||||||
}
|
}
|
||||||
for _, migration := range migrations {
|
normalized := strings.TrimSpace(migrations[0].SQL)
|
||||||
normalized := strings.TrimSpace(migration.SQL)
|
if !strings.HasPrefix(normalized, "BEGIN;") || !strings.HasSuffix(normalized, "COMMIT;") {
|
||||||
if !strings.HasPrefix(normalized, "BEGIN;") || !strings.HasSuffix(normalized, "COMMIT;") {
|
t.Fatalf("migration is not transaction wrapped: %q", normalized)
|
||||||
t.Fatalf("migration %d is not transaction wrapped: %q", migration.Version, normalized)
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
for index := 1; index < len(migrations); index++ {
|
for index := 1; index < len(migrations); index++ {
|
||||||
if migrations[index-1].Version >= migrations[index].Version {
|
if migrations[index-1].Version >= migrations[index].Version {
|
||||||
@ -52,25 +49,6 @@ func TestMigrationsAreOrderedTransactionalAndImmutable(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestRoutingDisableMigrationExtendsOnlyAuditActionConstraint(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
lower := strings.ToLower(Migrations()[1].SQL)
|
|
||||||
for _, required := range []string{
|
|
||||||
"alter table admin_audit_log",
|
|
||||||
"drop constraint if exists admin_audit_log_action_check",
|
|
||||||
"disable_routing",
|
|
||||||
} {
|
|
||||||
if !strings.Contains(lower, required) {
|
|
||||||
t.Errorf("routing disable migration missing %q", required)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
for _, forbidden := range []string{"create table", "proxy", "credential", "extraction", "ownership", "idempotency"} {
|
|
||||||
if strings.Contains(lower, forbidden) {
|
|
||||||
t.Errorf("routing disable migration contains forbidden identifier %q", forbidden)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestAdminSchemaContainsOnlyManagementTables(t *testing.T) {
|
func TestAdminSchemaContainsOnlyManagementTables(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
sql := Migrations()[0].SQL
|
sql := Migrations()[0].SQL
|
||||||
|
|||||||
@ -301,96 +301,6 @@ func (adapter *adapter) SwitchRouting(
|
|||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (adapter *adapter) DisableRouting(
|
|
||||||
ctx context.Context,
|
|
||||||
command adminstate.DisableRoutingCommand,
|
|
||||||
) (adminstate.MutationResult, error) {
|
|
||||||
result := adminstate.MutationResult{RequestID: command.RequestID}
|
|
||||||
if err := contextError(ctx); err != nil {
|
|
||||||
return result, err
|
|
||||||
}
|
|
||||||
if command.Validate() != nil || !adapter.valid() {
|
|
||||||
return result, adminstate.ErrInvalidCommand
|
|
||||||
}
|
|
||||||
|
|
||||||
tx, currentRevision, err := adapter.beginMutation(ctx, "disable routing")
|
|
||||||
if err != nil {
|
|
||||||
return result, err
|
|
||||||
}
|
|
||||||
defer rollback(tx)
|
|
||||||
|
|
||||||
var enabled bool
|
|
||||||
var current string
|
|
||||||
if err := tx.QueryRow(ctx, `
|
|
||||||
SELECT enabled, current_upstream
|
|
||||||
FROM routing_admin_state
|
|
||||||
WHERE name = $1
|
|
||||||
FOR UPDATE`, command.Name).Scan(&enabled, ¤t); err != nil {
|
|
||||||
if errors.Is(err, pgx.ErrNoRows) {
|
|
||||||
return result, adminstate.ErrNotFound
|
|
||||||
}
|
|
||||||
return result, databaseError(ctx, "read routing state", err)
|
|
||||||
}
|
|
||||||
if current != command.ExpectedCurrent {
|
|
||||||
return result, adminstate.ErrConflict
|
|
||||||
}
|
|
||||||
if !enabled {
|
|
||||||
if err := insertAudit(ctx, tx, command.RequestID, command.Actor,
|
|
||||||
adminstate.ActionDisableRoute, "routing", command.Name, false,
|
|
||||||
currentRevision, command.Reason, command.OccurredAt, "audit routing disable no-op"); err != nil {
|
|
||||||
return result, err
|
|
||||||
}
|
|
||||||
if err := commit(ctx, tx, "commit routing disable no-op"); err != nil {
|
|
||||||
return result, err
|
|
||||||
}
|
|
||||||
result.Revision = uint64(currentRevision)
|
|
||||||
return result, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
nextRevision, err := allocateRevision(ctx, tx, currentRevision, "routing", command.OccurredAt)
|
|
||||||
if err != nil {
|
|
||||||
return result, err
|
|
||||||
}
|
|
||||||
commandTag, err := tx.Exec(ctx, `
|
|
||||||
UPDATE routing_admin_state
|
|
||||||
SET enabled = FALSE, revision = $1, updated_at = $2
|
|
||||||
WHERE name = $3 AND enabled = TRUE AND current_upstream = $4`,
|
|
||||||
nextRevision, utc(command.OccurredAt), command.Name, command.ExpectedCurrent)
|
|
||||||
if err != nil {
|
|
||||||
return result, databaseError(ctx, "disable routing state", err)
|
|
||||||
}
|
|
||||||
if commandTag.RowsAffected() != 1 {
|
|
||||||
return result, adminstate.ErrConflict
|
|
||||||
}
|
|
||||||
if err := insertAudit(ctx, tx, command.RequestID, command.Actor,
|
|
||||||
adminstate.ActionDisableRoute, "routing", command.Name, true,
|
|
||||||
nextRevision, command.Reason, command.OccurredAt, "audit routing disable"); err != nil {
|
|
||||||
return result, err
|
|
||||||
}
|
|
||||||
payload, err := encodePayload(map[string]any{
|
|
||||||
"current": command.ExpectedCurrent,
|
|
||||||
"enabled": false,
|
|
||||||
"name": command.Name,
|
|
||||||
"reason": command.Reason,
|
|
||||||
"revision": nextRevision,
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
return result, unavailable("encode routing disable event")
|
|
||||||
}
|
|
||||||
if err := insertOutbox(ctx, tx, nextRevision, "routing.disabled", "routing",
|
|
||||||
command.Name, payload, command.OccurredAt, "insert routing disable event"); err != nil {
|
|
||||||
return result, err
|
|
||||||
}
|
|
||||||
if err := commit(ctx, tx, "commit routing disable"); err != nil {
|
|
||||||
return result, err
|
|
||||||
}
|
|
||||||
return adminstate.MutationResult{
|
|
||||||
RequestID: command.RequestID,
|
|
||||||
Changed: true,
|
|
||||||
Revision: uint64(nextRevision),
|
|
||||||
}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (adapter *adapter) beginMutation(ctx context.Context, operation string) (pgx.Tx, int64, error) {
|
func (adapter *adapter) beginMutation(ctx context.Context, operation string) (pgx.Tx, int64, error) {
|
||||||
tx, err := adapter.begin(ctx, pgx.TxOptions{AccessMode: pgx.ReadWrite}, operation)
|
tx, err := adapter.begin(ctx, pgx.TxOptions{AccessMode: pgx.ReadWrite}, operation)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|||||||
@ -62,41 +62,6 @@ func TestMutationRollsBackWhenOutboxInsertFails(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestDisableRoutingRollsBackWhenAuditOrOutboxInsertFails(t *testing.T) {
|
|
||||||
for _, target := range []struct {
|
|
||||||
name string
|
|
||||||
table string
|
|
||||||
}{
|
|
||||||
{name: "audit", table: "admin_audit_log"},
|
|
||||||
{name: "outbox", table: "admin_outbox"},
|
|
||||||
} {
|
|
||||||
t.Run(target.name, func(t *testing.T) {
|
|
||||||
fixture := newPostgresTestFixture(t)
|
|
||||||
now := integrationNow()
|
|
||||||
commitIntegrationConfig(t, fixture.Store, now)
|
|
||||||
installRejectInsertTrigger(t, fixture, target.table, "reject_routing_disable")
|
|
||||||
|
|
||||||
_, err := fixture.Store.DisableRouting(context.Background(), adminstate.DisableRoutingCommand{
|
|
||||||
RequestID: "req-routing-disable-failure", Actor: integrationActor(), OccurredAt: now.Add(time.Second),
|
|
||||||
Name: "checkout", ExpectedCurrent: "provider-a", Reason: "terminal empty results",
|
|
||||||
})
|
|
||||||
if !errors.Is(err, adminstate.ErrUnavailable) {
|
|
||||||
t.Fatalf("DisableRouting(%s failure) error = %v, want ErrUnavailable", target.name, err)
|
|
||||||
}
|
|
||||||
assertFailedRoutingDisableLeftBaseline(t, fixture)
|
|
||||||
|
|
||||||
dropRejectInsertTrigger(t, fixture, target.table, "reject_routing_disable")
|
|
||||||
result, err := fixture.Store.DisableRouting(context.Background(), adminstate.DisableRoutingCommand{
|
|
||||||
RequestID: "req-routing-disable-after-failure", Actor: integrationActor(), OccurredAt: now.Add(2 * time.Second),
|
|
||||||
Name: "checkout", ExpectedCurrent: "provider-a", Reason: "terminal empty results",
|
|
||||||
})
|
|
||||||
if err != nil || !result.Changed || result.Revision != 2 {
|
|
||||||
t.Fatalf("DisableRouting(after rollback) = %+v, %v", result, err)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func assertFailedMutationLeftBaseline(t *testing.T, fixture postgresTestFixture) {
|
func assertFailedMutationLeftBaseline(t *testing.T, fixture postgresTestFixture) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
snapshot, err := fixture.Store.Snapshot(context.Background())
|
snapshot, err := fixture.Store.Snapshot(context.Background())
|
||||||
@ -118,28 +83,6 @@ func assertFailedMutationLeftBaseline(t *testing.T, fixture postgresTestFixture)
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func assertFailedRoutingDisableLeftBaseline(t *testing.T, fixture postgresTestFixture) {
|
|
||||||
t.Helper()
|
|
||||||
snapshot, err := fixture.Store.Snapshot(context.Background())
|
|
||||||
if err != nil || snapshot.Revision != 1 || len(snapshot.Routings) != 1 || !snapshot.Routings[0].Enabled ||
|
|
||||||
snapshot.Routings[0].CurrentUpstream != "provider-a" {
|
|
||||||
t.Fatalf("Snapshot(after failed DisableRouting) = %+v, %v", snapshot, err)
|
|
||||||
}
|
|
||||||
audits, err := fixture.Store.ReadAudit(context.Background(), adminstate.AuditQuery{Limit: 10})
|
|
||||||
if err != nil || len(audits) != 1 {
|
|
||||||
t.Fatalf("ReadAudit(after failed DisableRouting) = %+v, %v", audits, err)
|
|
||||||
}
|
|
||||||
for table, want := range map[string]int{"control_revisions": 1, "admin_outbox": 1} {
|
|
||||||
var count int
|
|
||||||
if err := fixture.Pool.QueryRow(context.Background(), "SELECT COUNT(*) FROM "+table).Scan(&count); err != nil {
|
|
||||||
t.Fatalf("count %s: %v", table, err)
|
|
||||||
}
|
|
||||||
if count != want {
|
|
||||||
t.Fatalf("%s row count = %d, want %d", table, count, want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func installRejectInsertTrigger(t *testing.T, fixture postgresTestFixture, table, trigger string) {
|
func installRejectInsertTrigger(t *testing.T, fixture postgresTestFixture, table, trigger string) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
function := trigger + "_fn"
|
function := trigger + "_fn"
|
||||||
|
|||||||
@ -224,53 +224,6 @@ func TestRedisRejectsUnhealthyDrainCandidateAfterRecovery(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestRedisDisabledUpstreamDrainFencesPolicyRevision(t *testing.T) {
|
|
||||||
fixture := newRedisTestFixture(t)
|
|
||||||
now := redisTestNow()
|
|
||||||
seedRedisAvailable(t, fixture.Adapter, "provider-a", now, now.Add(time.Second), time.Minute,
|
|
||||||
testProxy("proxy-a", "192.0.2.10"))
|
|
||||||
assignment, err := fixture.Adapter.Assign(context.Background(), now.Add(2*time.Second), "proxy-a", "worker-a", time.Minute)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("Assign() error = %v", err)
|
|
||||||
}
|
|
||||||
disabled := activitypool.UpstreamDrainPolicy{UpstreamID: "provider-a", Revision: 31}
|
|
||||||
if err := fixture.Adapter.ReplaceUpstreamDrainPolicies(context.Background(), []activitypool.UpstreamDrainPolicy{disabled}); err != nil {
|
|
||||||
t.Fatalf("ReplaceUpstreamDrainPolicies(disabled) error = %v", err)
|
|
||||||
}
|
|
||||||
candidates, err := fixture.Adapter.ListDisabledUpstreamDrainCandidates(context.Background(), now.Add(3*time.Second), disabled, 1)
|
|
||||||
if err != nil || len(candidates) != 1 || candidates[0] != (activitypool.DisabledUpstreamDrainCandidate{
|
|
||||||
UpstreamID: "provider-a", PolicyRevision: 31, ProxyID: "proxy-a", WorkerID: "worker-a", AssignmentEpoch: assignment.Epoch,
|
|
||||||
}) {
|
|
||||||
t.Fatalf("ListDisabledUpstreamDrainCandidates() = %+v, %v", candidates, err)
|
|
||||||
}
|
|
||||||
if err := fixture.Adapter.ReplaceUpstreamDrainPolicies(context.Background(), []activitypool.UpstreamDrainPolicy{{
|
|
||||||
UpstreamID: "provider-a", Revision: 32, Enabled: true,
|
|
||||||
}}); err != nil {
|
|
||||||
t.Fatalf("ReplaceUpstreamDrainPolicies(enabled) error = %v", err)
|
|
||||||
}
|
|
||||||
if started, err := fixture.Adapter.BeginDisabledUpstreamDrain(context.Background(), now.Add(3*time.Second), candidates[0]); err != nil || started {
|
|
||||||
t.Fatalf("BeginDisabledUpstreamDrain(stale policy) = %t, %v", started, err)
|
|
||||||
}
|
|
||||||
if current, ok, err := fixture.Adapter.Get(context.Background(), "proxy-a"); err != nil || !ok || current.Draining {
|
|
||||||
t.Fatalf("Get(after stale candidate) = %+v, %t, %v", current, ok, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
disabled.Revision = 33
|
|
||||||
if err := fixture.Adapter.ReplaceUpstreamDrainPolicies(context.Background(), []activitypool.UpstreamDrainPolicy{disabled}); err != nil {
|
|
||||||
t.Fatalf("ReplaceUpstreamDrainPolicies(disabled again) error = %v", err)
|
|
||||||
}
|
|
||||||
candidates, err = fixture.Adapter.ListDisabledUpstreamDrainCandidates(context.Background(), now.Add(4*time.Second), disabled, 1)
|
|
||||||
if err != nil || len(candidates) != 1 {
|
|
||||||
t.Fatalf("ListDisabledUpstreamDrainCandidates(disabled again) = %+v, %v", candidates, err)
|
|
||||||
}
|
|
||||||
if started, err := fixture.Adapter.BeginDisabledUpstreamDrain(context.Background(), now.Add(4*time.Second), candidates[0]); err != nil || !started {
|
|
||||||
t.Fatalf("BeginDisabledUpstreamDrain() = %t, %v", started, err)
|
|
||||||
}
|
|
||||||
if current, ok, err := fixture.Adapter.Get(context.Background(), "proxy-a"); err != nil || !ok || !current.Draining {
|
|
||||||
t.Fatalf("Get(after drain) = %+v, %t, %v", current, ok, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func redisGlobalObservation(taskID string, success bool, observedAt time.Time) healthDomain.Observation {
|
func redisGlobalObservation(taskID string, success bool, observedAt time.Time) healthDomain.Observation {
|
||||||
observation := healthDomain.Observation{
|
observation := healthDomain.Observation{
|
||||||
TaskID: taskID, ProxyID: "proxy-a", Level: healthDomain.LevelBasic,
|
TaskID: taskID, ProxyID: "proxy-a", Level: healthDomain.LevelBasic,
|
||||||
|
|||||||
@ -9,69 +9,67 @@ import (
|
|||||||
const redisKeyPrefix = "pp:{activity}:"
|
const redisKeyPrefix = "pp:{activity}:"
|
||||||
|
|
||||||
type keyspace struct {
|
type keyspace struct {
|
||||||
prefix string
|
prefix string
|
||||||
records string
|
records string
|
||||||
unique string
|
unique string
|
||||||
idkeys string
|
idkeys string
|
||||||
expiry string
|
expiry string
|
||||||
available string
|
available string
|
||||||
owners string
|
owners string
|
||||||
ownerExpiry string
|
ownerExpiry string
|
||||||
drainTickets string
|
drainTickets string
|
||||||
upstreamDrainPolicies string
|
epoch string
|
||||||
epoch string
|
inventory string
|
||||||
inventory string
|
stateInventory string
|
||||||
stateInventory string
|
workerSessions string
|
||||||
workerSessions string
|
workerSessionExpiry string
|
||||||
workerSessionExpiry string
|
workerSnapshots string
|
||||||
workerSnapshots string
|
workerSnapshotExpiry string
|
||||||
workerSnapshotExpiry string
|
workerRuntime string
|
||||||
workerRuntime string
|
workerRuntimeExpiry string
|
||||||
workerRuntimeExpiry string
|
workerOutcomes string
|
||||||
workerOutcomes string
|
healthDue string
|
||||||
healthDue string
|
healthUnhealthy string
|
||||||
healthUnhealthy string
|
healthEgressDue string
|
||||||
healthEgressDue string
|
healthTargetDue string
|
||||||
healthTargetDue string
|
healthQueued string
|
||||||
healthQueued string
|
healthLeases string
|
||||||
healthLeases string
|
healthTasks string
|
||||||
healthTasks string
|
healthTaskExpiry string
|
||||||
healthTaskExpiry string
|
healthRefTask string
|
||||||
healthRefTask string
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func newKeyspace(namespace string) keyspace {
|
func newKeyspace(namespace string) keyspace {
|
||||||
prefix := redisKeyPrefix + namespace
|
prefix := redisKeyPrefix + namespace
|
||||||
return keyspace{
|
return keyspace{
|
||||||
prefix: prefix,
|
prefix: prefix,
|
||||||
records: prefix + ":records",
|
records: prefix + ":records",
|
||||||
unique: prefix + ":unique",
|
unique: prefix + ":unique",
|
||||||
idkeys: prefix + ":idkeys",
|
idkeys: prefix + ":idkeys",
|
||||||
expiry: prefix + ":expiry",
|
expiry: prefix + ":expiry",
|
||||||
available: prefix + ":available",
|
available: prefix + ":available",
|
||||||
owners: prefix + ":owners",
|
owners: prefix + ":owners",
|
||||||
ownerExpiry: prefix + ":owner-expiry",
|
ownerExpiry: prefix + ":owner-expiry",
|
||||||
drainTickets: prefix + ":drain-tickets",
|
drainTickets: prefix + ":drain-tickets",
|
||||||
upstreamDrainPolicies: prefix + ":upstream-drain-policies",
|
epoch: prefix + ":epoch",
|
||||||
epoch: prefix + ":epoch",
|
inventory: prefix + ":inventory",
|
||||||
inventory: prefix + ":inventory",
|
stateInventory: prefix + ":state-inventory",
|
||||||
stateInventory: prefix + ":state-inventory",
|
workerSessions: prefix + ":worker-sessions",
|
||||||
workerSessions: prefix + ":worker-sessions",
|
workerSessionExpiry: prefix + ":worker-session-expiry",
|
||||||
workerSessionExpiry: prefix + ":worker-session-expiry",
|
workerSnapshots: prefix + ":worker-snapshots",
|
||||||
workerSnapshots: prefix + ":worker-snapshots",
|
workerSnapshotExpiry: prefix + ":worker-snapshot-expiry",
|
||||||
workerSnapshotExpiry: prefix + ":worker-snapshot-expiry",
|
workerRuntime: prefix + ":worker-runtime",
|
||||||
workerRuntime: prefix + ":worker-runtime",
|
workerRuntimeExpiry: prefix + ":worker-runtime-expiry",
|
||||||
workerRuntimeExpiry: prefix + ":worker-runtime-expiry",
|
workerOutcomes: prefix + ":worker-outcomes",
|
||||||
workerOutcomes: prefix + ":worker-outcomes",
|
healthDue: prefix + ":health-due",
|
||||||
healthDue: prefix + ":health-due",
|
healthUnhealthy: prefix + ":health-unhealthy",
|
||||||
healthUnhealthy: prefix + ":health-unhealthy",
|
healthEgressDue: prefix + ":health-egress-due",
|
||||||
healthEgressDue: prefix + ":health-egress-due",
|
healthTargetDue: prefix + ":health-target-due",
|
||||||
healthTargetDue: prefix + ":health-target-due",
|
healthQueued: prefix + ":health-queued",
|
||||||
healthQueued: prefix + ":health-queued",
|
healthLeases: prefix + ":health-leases",
|
||||||
healthLeases: prefix + ":health-leases",
|
healthTasks: prefix + ":health-tasks",
|
||||||
healthTasks: prefix + ":health-tasks",
|
healthTaskExpiry: prefix + ":health-task-expiry",
|
||||||
healthTaskExpiry: prefix + ":health-task-expiry",
|
healthRefTask: prefix + ":health-ref-task",
|
||||||
healthRefTask: prefix + ":health-ref-task",
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@ -3,25 +3,22 @@ package redisactivity
|
|||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"strconv"
|
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"proxy-pool/internal/domain/activitypool"
|
"proxy-pool/internal/domain/activitypool"
|
||||||
)
|
)
|
||||||
|
|
||||||
const (
|
const (
|
||||||
maintenanceInventory = "inventory"
|
maintenanceInventory = "inventory"
|
||||||
maintenanceSweep = "sweep"
|
maintenanceSweep = "sweep"
|
||||||
maintenanceUnhealthy = "unhealthy"
|
maintenanceUnhealthy = "unhealthy"
|
||||||
maintenanceDisabledUpstream = "disabled_upstream"
|
|
||||||
|
|
||||||
maximumUnhealthySweepScan = 1024
|
maximumUnhealthySweepScan = 1024
|
||||||
)
|
)
|
||||||
|
|
||||||
var (
|
var (
|
||||||
_ activitypool.InventoryReader = (*Adapter)(nil)
|
_ activitypool.InventoryReader = (*Adapter)(nil)
|
||||||
_ activitypool.Maintainer = (*Adapter)(nil)
|
_ activitypool.Maintainer = (*Adapter)(nil)
|
||||||
_ activitypool.DisabledUpstreamDrainLister = (*Adapter)(nil)
|
|
||||||
)
|
)
|
||||||
|
|
||||||
func (a *Adapter) Inventory(ctx context.Context, upstreamID string, now time.Time) (activitypool.Inventory, error) {
|
func (a *Adapter) Inventory(ctx context.Context, upstreamID string, now time.Time) (activitypool.Inventory, error) {
|
||||||
@ -127,52 +124,6 @@ func (a *Adapter) SweepUnhealthy(
|
|||||||
return result, nil
|
return result, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (a *Adapter) ListDisabledUpstreamDrainCandidates(
|
|
||||||
ctx context.Context,
|
|
||||||
now time.Time,
|
|
||||||
policy activitypool.UpstreamDrainPolicy,
|
|
||||||
limit int,
|
|
||||||
) ([]activitypool.DisabledUpstreamDrainCandidate, error) {
|
|
||||||
if ctx == nil {
|
|
||||||
return nil, activitypool.ErrInvalidMaintenance
|
|
||||||
}
|
|
||||||
if err := ctx.Err(); err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
if a == nil || now.IsZero() || policy.UpstreamID == "" || policy.Revision == 0 || policy.Enabled || limit <= 0 {
|
|
||||||
return nil, activitypool.ErrInvalidMaintenance
|
|
||||||
}
|
|
||||||
if limit > maximumUnhealthySweepScan/4 {
|
|
||||||
limit = maximumUnhealthySweepScan / 4
|
|
||||||
}
|
|
||||||
reply, err := a.runMaintenance(ctx, maintenanceDisabledUpstream, now, limit, policy.UpstreamID, "")
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
if reply.Status == scriptStale {
|
|
||||||
return nil, nil
|
|
||||||
}
|
|
||||||
if reply.Status == scriptInvalid {
|
|
||||||
return nil, activitypool.ErrInvalidMaintenance
|
|
||||||
}
|
|
||||||
if reply.Status != scriptOK || len(reply.DisabledDrainCandidates) > limit {
|
|
||||||
return nil, invalidScriptReply("unexpected disabled upstream sweep reply")
|
|
||||||
}
|
|
||||||
wantRevision := strconv.FormatUint(policy.Revision, 10)
|
|
||||||
result := make([]activitypool.DisabledUpstreamDrainCandidate, 0, len(reply.DisabledDrainCandidates))
|
|
||||||
for _, candidate := range reply.DisabledDrainCandidates {
|
|
||||||
if candidate.UpstreamID != policy.UpstreamID || candidate.PolicyRevision != wantRevision ||
|
|
||||||
candidate.ProxyID == "" || candidate.WorkerID == "" || candidate.AssignmentEpoch == 0 {
|
|
||||||
return nil, invalidScriptReply("disabled upstream sweep reply contained an invalid drain candidate")
|
|
||||||
}
|
|
||||||
result = append(result, activitypool.DisabledUpstreamDrainCandidate{
|
|
||||||
UpstreamID: candidate.UpstreamID, PolicyRevision: policy.Revision, ProxyID: candidate.ProxyID,
|
|
||||||
WorkerID: candidate.WorkerID, AssignmentEpoch: candidate.AssignmentEpoch,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
return result, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (a *Adapter) runMaintenance(
|
func (a *Adapter) runMaintenance(
|
||||||
ctx context.Context,
|
ctx context.Context,
|
||||||
operation string,
|
operation string,
|
||||||
@ -190,7 +141,6 @@ func (a *Adapter) runMaintenance(
|
|||||||
a.keys.inventory, a.keys.stateInventory, a.keys.owners, a.keys.ownerExpiry,
|
a.keys.inventory, a.keys.stateInventory, a.keys.owners, a.keys.ownerExpiry,
|
||||||
a.keys.operation(operationID), a.keys.healthDue, a.keys.healthQueued, a.keys.healthLeases,
|
a.keys.operation(operationID), a.keys.healthDue, a.keys.healthQueued, a.keys.healthLeases,
|
||||||
a.keys.healthTasks, a.keys.healthTaskExpiry, a.keys.healthRefTask, a.keys.healthUnhealthy,
|
a.keys.healthTasks, a.keys.healthTaskExpiry, a.keys.healthRefTask, a.keys.healthUnhealthy,
|
||||||
a.keys.owned(upstreamID), a.keys.upstreamDrainPolicies,
|
|
||||||
}, operation, now.UnixMilli(), limit, upstreamID, operationTTLMillis(a.options.OperationTTL), policies)
|
}, operation, now.UnixMilli(), limit, upstreamID, operationTTLMillis(a.options.OperationTTL), policies)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return maintenanceScriptReply{}, err
|
return maintenanceScriptReply{}, err
|
||||||
|
|||||||
@ -6,7 +6,6 @@ import (
|
|||||||
"encoding/hex"
|
"encoding/hex"
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"strconv"
|
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"proxy-pool/internal/domain/activitypool"
|
"proxy-pool/internal/domain/activitypool"
|
||||||
@ -18,7 +17,6 @@ const (
|
|||||||
ownershipRenew = "renew"
|
ownershipRenew = "renew"
|
||||||
ownershipBeginDrain = "begin_drain"
|
ownershipBeginDrain = "begin_drain"
|
||||||
ownershipBeginUnhealthyDrain = "begin_unhealthy_drain"
|
ownershipBeginUnhealthyDrain = "begin_unhealthy_drain"
|
||||||
ownershipBeginDisabledDrain = "begin_disabled_upstream_drain"
|
|
||||||
ownershipAcknowledgeDrain = "acknowledge_drain"
|
ownershipAcknowledgeDrain = "acknowledge_drain"
|
||||||
ownershipGet = "get"
|
ownershipGet = "get"
|
||||||
ownershipExpire = "expire"
|
ownershipExpire = "expire"
|
||||||
@ -27,7 +25,6 @@ const (
|
|||||||
var _ ownershipDomain.Repository = (*Adapter)(nil)
|
var _ ownershipDomain.Repository = (*Adapter)(nil)
|
||||||
var _ ownershipDomain.DrainTicketStore = (*Adapter)(nil)
|
var _ ownershipDomain.DrainTicketStore = (*Adapter)(nil)
|
||||||
var _ activitypool.UnhealthyDrainStarter = (*Adapter)(nil)
|
var _ activitypool.UnhealthyDrainStarter = (*Adapter)(nil)
|
||||||
var _ activitypool.DisabledUpstreamDrainStarter = (*Adapter)(nil)
|
|
||||||
|
|
||||||
func (a *Adapter) Assign(
|
func (a *Adapter) Assign(
|
||||||
ctx context.Context,
|
ctx context.Context,
|
||||||
@ -42,7 +39,7 @@ func (a *Adapter) Assign(
|
|||||||
if now.IsZero() || proxyID == "" || workerID == "" || ttl <= 0 {
|
if now.IsZero() || proxyID == "" || workerID == "" || ttl <= 0 {
|
||||||
return ownershipDomain.Assignment{}, ownershipDomain.ErrInvalidOwnership
|
return ownershipDomain.Assignment{}, ownershipDomain.ErrInvalidOwnership
|
||||||
}
|
}
|
||||||
reply, err := a.runOwnership(ctx, ownershipAssign, true, now.UnixMilli(), proxyID, workerID, 0, durationMillis(ttl), 0, 0, "", "")
|
reply, err := a.runOwnership(ctx, ownershipAssign, true, now.UnixMilli(), proxyID, workerID, 0, durationMillis(ttl), 0, 0)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return ownershipDomain.Assignment{}, err
|
return ownershipDomain.Assignment{}, err
|
||||||
}
|
}
|
||||||
@ -74,7 +71,7 @@ func (a *Adapter) Renew(
|
|||||||
if now.IsZero() || proxyID == "" || workerID == "" || epoch == 0 || ttl <= 0 {
|
if now.IsZero() || proxyID == "" || workerID == "" || epoch == 0 || ttl <= 0 {
|
||||||
return ownershipDomain.Assignment{}, ownershipDomain.ErrInvalidOwnership
|
return ownershipDomain.Assignment{}, ownershipDomain.ErrInvalidOwnership
|
||||||
}
|
}
|
||||||
reply, err := a.runOwnership(ctx, ownershipRenew, true, now.UnixMilli(), proxyID, workerID, epoch, durationMillis(ttl), 0, 0, "", "")
|
reply, err := a.runOwnership(ctx, ownershipRenew, true, now.UnixMilli(), proxyID, workerID, epoch, durationMillis(ttl), 0, 0)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return ownershipDomain.Assignment{}, err
|
return ownershipDomain.Assignment{}, err
|
||||||
}
|
}
|
||||||
@ -102,7 +99,7 @@ func (a *Adapter) BeginDrain(
|
|||||||
if proxyID == "" || workerID == "" || epoch == 0 {
|
if proxyID == "" || workerID == "" || epoch == 0 {
|
||||||
return ownershipDomain.Assignment{}, ownershipDomain.ErrInvalidOwnership
|
return ownershipDomain.Assignment{}, ownershipDomain.ErrInvalidOwnership
|
||||||
}
|
}
|
||||||
reply, err := a.runOwnership(ctx, ownershipBeginDrain, true, 0, proxyID, workerID, epoch, 0, 0, 0, "", "")
|
reply, err := a.runOwnership(ctx, ownershipBeginDrain, true, 0, proxyID, workerID, epoch, 0, 0, 0)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return ownershipDomain.Assignment{}, err
|
return ownershipDomain.Assignment{}, err
|
||||||
}
|
}
|
||||||
@ -131,7 +128,7 @@ func (a *Adapter) BeginUnhealthyDrain(
|
|||||||
}
|
}
|
||||||
reply, err := a.runOwnership(
|
reply, err := a.runOwnership(
|
||||||
ctx, ownershipBeginUnhealthyDrain, true, now.UTC().UnixMilli(), candidate.ProxyID, candidate.WorkerID,
|
ctx, ownershipBeginUnhealthyDrain, true, now.UTC().UnixMilli(), candidate.ProxyID, candidate.WorkerID,
|
||||||
candidate.AssignmentEpoch, candidate.UnhealthySince.UTC().UnixMilli(), 0, 0, "", "",
|
candidate.AssignmentEpoch, candidate.UnhealthySince.UTC().UnixMilli(), 0, 0,
|
||||||
)
|
)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return false, err
|
return false, err
|
||||||
@ -151,40 +148,6 @@ func (a *Adapter) BeginUnhealthyDrain(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (a *Adapter) BeginDisabledUpstreamDrain(
|
|
||||||
ctx context.Context,
|
|
||||||
now time.Time,
|
|
||||||
candidate activitypool.DisabledUpstreamDrainCandidate,
|
|
||||||
) (bool, error) {
|
|
||||||
if err := validateOwnershipCall(ctx, a); err != nil {
|
|
||||||
return false, err
|
|
||||||
}
|
|
||||||
if now.IsZero() || candidate.UpstreamID == "" || candidate.PolicyRevision == 0 || candidate.ProxyID == "" ||
|
|
||||||
candidate.WorkerID == "" || candidate.AssignmentEpoch == 0 {
|
|
||||||
return false, activitypool.ErrInvalidMaintenance
|
|
||||||
}
|
|
||||||
reply, err := a.runOwnership(
|
|
||||||
ctx, ownershipBeginDisabledDrain, true, now.UTC().UnixMilli(), candidate.ProxyID, candidate.WorkerID,
|
|
||||||
candidate.AssignmentEpoch, 0, 0, 0, candidate.UpstreamID, strconv.FormatUint(candidate.PolicyRevision, 10),
|
|
||||||
)
|
|
||||||
if err != nil {
|
|
||||||
return false, err
|
|
||||||
}
|
|
||||||
switch reply.Status {
|
|
||||||
case scriptOK:
|
|
||||||
if _, err := decodeAssignmentReply(reply); err != nil {
|
|
||||||
return false, err
|
|
||||||
}
|
|
||||||
return reply.Started, nil
|
|
||||||
case scriptStale, scriptNotFound:
|
|
||||||
return false, nil
|
|
||||||
case scriptInvalid:
|
|
||||||
return false, activitypool.ErrInvalidMaintenance
|
|
||||||
default:
|
|
||||||
return false, invalidScriptReply("unexpected disabled upstream ownership drain status")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (a *Adapter) PendingDrains(ctx context.Context, workerID string, limit int) ([]ownershipDomain.DrainTicket, error) {
|
func (a *Adapter) PendingDrains(ctx context.Context, workerID string, limit int) ([]ownershipDomain.DrainTicket, error) {
|
||||||
if err := validateOwnershipCall(ctx, a); err != nil {
|
if err := validateOwnershipCall(ctx, a); err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
@ -279,7 +242,7 @@ func (a *Adapter) AcknowledgeDrain(
|
|||||||
if proxyID == "" || workerID == "" || epoch == 0 || active < 0 || reserved < 0 {
|
if proxyID == "" || workerID == "" || epoch == 0 || active < 0 || reserved < 0 {
|
||||||
return ownershipDomain.ErrInvalidOwnership
|
return ownershipDomain.ErrInvalidOwnership
|
||||||
}
|
}
|
||||||
reply, err := a.runOwnership(ctx, ownershipAcknowledgeDrain, true, 0, proxyID, workerID, epoch, 0, active, reserved, "", "")
|
reply, err := a.runOwnership(ctx, ownershipAcknowledgeDrain, true, 0, proxyID, workerID, epoch, 0, active, reserved)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
@ -306,7 +269,7 @@ func (a *Adapter) Get(ctx context.Context, proxyID string) (ownershipDomain.Assi
|
|||||||
if proxyID == "" {
|
if proxyID == "" {
|
||||||
return ownershipDomain.Assignment{}, false, ownershipDomain.ErrInvalidOwnership
|
return ownershipDomain.Assignment{}, false, ownershipDomain.ErrInvalidOwnership
|
||||||
}
|
}
|
||||||
reply, err := a.runOwnership(ctx, ownershipGet, false, 0, proxyID, "", 0, 0, 0, 0, "", "")
|
reply, err := a.runOwnership(ctx, ownershipGet, false, 0, proxyID, "", 0, 0, 0, 0)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return ownershipDomain.Assignment{}, false, err
|
return ownershipDomain.Assignment{}, false, err
|
||||||
}
|
}
|
||||||
@ -330,7 +293,7 @@ func (a *Adapter) Expire(ctx context.Context, now time.Time, limit int) ([]owner
|
|||||||
if now.IsZero() || limit <= 0 {
|
if now.IsZero() || limit <= 0 {
|
||||||
return nil, ownershipDomain.ErrInvalidOwnership
|
return nil, ownershipDomain.ErrInvalidOwnership
|
||||||
}
|
}
|
||||||
reply, err := a.runOwnership(ctx, ownershipExpire, true, now.UnixMilli(), "", "", 0, int64(limit), 0, 0, "", "")
|
reply, err := a.runOwnership(ctx, ownershipExpire, true, now.UnixMilli(), "", "", 0, int64(limit), 0, 0)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
@ -368,8 +331,6 @@ func (a *Adapter) runOwnership(
|
|||||||
value int64,
|
value int64,
|
||||||
active int64,
|
active int64,
|
||||||
reserved int64,
|
reserved int64,
|
||||||
upstreamID string,
|
|
||||||
policyRevision string,
|
|
||||||
) (ownershipScriptReply, error) {
|
) (ownershipScriptReply, error) {
|
||||||
operationKey := a.keys.epoch
|
operationKey := a.keys.epoch
|
||||||
if mutating {
|
if mutating {
|
||||||
@ -384,9 +345,9 @@ func (a *Adapter) runOwnership(
|
|||||||
a.keys.inventory, a.keys.stateInventory, a.keys.owners, a.keys.ownerExpiry,
|
a.keys.inventory, a.keys.stateInventory, a.keys.owners, a.keys.ownerExpiry,
|
||||||
a.keys.epoch, operationKey, a.keys.healthDue, a.keys.healthQueued, a.keys.healthLeases,
|
a.keys.epoch, operationKey, a.keys.healthDue, a.keys.healthQueued, a.keys.healthLeases,
|
||||||
a.keys.healthTasks, a.keys.healthTaskExpiry, a.keys.healthRefTask, a.keys.healthUnhealthy,
|
a.keys.healthTasks, a.keys.healthTaskExpiry, a.keys.healthRefTask, a.keys.healthUnhealthy,
|
||||||
a.keys.drainTickets, a.keys.upstreamDrainPolicies,
|
a.keys.drainTickets,
|
||||||
}, operation, operationTTLMillis(a.options.OperationTTL), a.options.CleanupLimit,
|
}, operation, operationTTLMillis(a.options.OperationTTL), a.options.CleanupLimit,
|
||||||
nowMS, proxyID, workerID, epoch, value, active, reserved, a.keys.workerOwned(workerID), a.keys.workerDraining(workerID), upstreamID, policyRevision)
|
nowMS, proxyID, workerID, epoch, value, active, reserved, a.keys.workerOwned(workerID), a.keys.workerDraining(workerID))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return ownershipScriptReply{}, err
|
return ownershipScriptReply{}, err
|
||||||
}
|
}
|
||||||
|
|||||||
@ -88,21 +88,16 @@ type ownershipScriptReply struct {
|
|||||||
Started bool `json:"started"`
|
Started bool `json:"started"`
|
||||||
}
|
}
|
||||||
|
|
||||||
type upstreamDrainPolicyScriptReply struct {
|
|
||||||
Status scriptStatus `json:"status"`
|
|
||||||
}
|
|
||||||
|
|
||||||
type drainTicketsScriptReply struct {
|
type drainTicketsScriptReply struct {
|
||||||
Status scriptStatus `json:"status"`
|
Status scriptStatus `json:"status"`
|
||||||
Tickets []string `json:"tickets"`
|
Tickets []string `json:"tickets"`
|
||||||
}
|
}
|
||||||
|
|
||||||
type maintenanceScriptReply struct {
|
type maintenanceScriptReply struct {
|
||||||
Status scriptStatus `json:"status"`
|
Status scriptStatus `json:"status"`
|
||||||
Count int `json:"count"`
|
Count int `json:"count"`
|
||||||
DeferredOwned int `json:"deferredOwned"`
|
DeferredOwned int `json:"deferredOwned"`
|
||||||
DrainCandidates []maintenanceDrainCandidateScriptRow `json:"drainCandidates"`
|
DrainCandidates []maintenanceDrainCandidateScriptRow `json:"drainCandidates"`
|
||||||
DisabledDrainCandidates []disabledDrainCandidateScriptRow `json:"disabledDrainCandidates"`
|
|
||||||
}
|
}
|
||||||
|
|
||||||
type maintenanceDrainCandidateScriptRow struct {
|
type maintenanceDrainCandidateScriptRow struct {
|
||||||
@ -112,14 +107,6 @@ type maintenanceDrainCandidateScriptRow struct {
|
|||||||
UnhealthySinceMS int64 `json:"unhealthySinceMs"`
|
UnhealthySinceMS int64 `json:"unhealthySinceMs"`
|
||||||
}
|
}
|
||||||
|
|
||||||
type disabledDrainCandidateScriptRow struct {
|
|
||||||
UpstreamID string `json:"upstreamId"`
|
|
||||||
PolicyRevision string `json:"policyRevision"`
|
|
||||||
ProxyID string `json:"proxyId"`
|
|
||||||
WorkerID string `json:"workerId"`
|
|
||||||
AssignmentEpoch uint64 `json:"assignmentEpoch"`
|
|
||||||
}
|
|
||||||
|
|
||||||
type statusScriptReply struct {
|
type statusScriptReply struct {
|
||||||
Status scriptStatus `json:"status"`
|
Status scriptStatus `json:"status"`
|
||||||
Inventories []statusScriptInventory `json:"inventories"`
|
Inventories []statusScriptInventory `json:"inventories"`
|
||||||
@ -201,25 +188,21 @@ var capacitySource string
|
|||||||
//go:embed scripts/worker_snapshot.lua
|
//go:embed scripts/worker_snapshot.lua
|
||||||
var workerSnapshotSource string
|
var workerSnapshotSource string
|
||||||
|
|
||||||
//go:embed scripts/upstream_drain_policy.lua
|
|
||||||
var upstreamDrainPolicySource string
|
|
||||||
|
|
||||||
var (
|
var (
|
||||||
upsertScript = redis.NewScript(upsertSource)
|
upsertScript = redis.NewScript(upsertSource)
|
||||||
healthScript = redis.NewScript(healthSource)
|
healthScript = redis.NewScript(healthSource)
|
||||||
healthTasksScript = redis.NewScript(healthTasksSource)
|
healthTasksScript = redis.NewScript(healthTasksSource)
|
||||||
targetHealthScript = redis.NewScript(targetHealthSource)
|
targetHealthScript = redis.NewScript(targetHealthSource)
|
||||||
upstreamLookupScript = redis.NewScript(upstreamLookupSource)
|
upstreamLookupScript = redis.NewScript(upstreamLookupSource)
|
||||||
extractScript = redis.NewScript(extractSource)
|
extractScript = redis.NewScript(extractSource)
|
||||||
ownershipScript = redis.NewScript(ownershipSource)
|
ownershipScript = redis.NewScript(ownershipSource)
|
||||||
drainTicketsScript = redis.NewScript(drainTicketsSource)
|
drainTicketsScript = redis.NewScript(drainTicketsSource)
|
||||||
bindDrainTicketScript = redis.NewScript(bindDrainTicketSource)
|
bindDrainTicketScript = redis.NewScript(bindDrainTicketSource)
|
||||||
sweepScript = redis.NewScript(sweepSource)
|
sweepScript = redis.NewScript(sweepSource)
|
||||||
statusScript = redis.NewScript(statusSource)
|
statusScript = redis.NewScript(statusSource)
|
||||||
runtimeScript = redis.NewScript(runtimeSource)
|
runtimeScript = redis.NewScript(runtimeSource)
|
||||||
capacityScript = redis.NewScript(capacitySource)
|
capacityScript = redis.NewScript(capacitySource)
|
||||||
workerSnapshotScript = redis.NewScript(workerSnapshotSource)
|
workerSnapshotScript = redis.NewScript(workerSnapshotSource)
|
||||||
upstreamDrainPolicyScript = redis.NewScript(upstreamDrainPolicySource)
|
|
||||||
)
|
)
|
||||||
|
|
||||||
func runScript(ctx context.Context, client redis.Scripter, script *redis.Script, keys []string, args ...any) (any, error) {
|
func runScript(ctx context.Context, client redis.Scripter, script *redis.Script, keys []string, args ...any) (any, error) {
|
||||||
|
|||||||
@ -17,7 +17,6 @@ local health_task_expiry_key = KEYS[16]
|
|||||||
local health_ref_task_key = KEYS[17]
|
local health_ref_task_key = KEYS[17]
|
||||||
local health_unhealthy_key = KEYS[18]
|
local health_unhealthy_key = KEYS[18]
|
||||||
local drain_tickets_key = KEYS[19]
|
local drain_tickets_key = KEYS[19]
|
||||||
local upstream_drain_policies_key = KEYS[20]
|
|
||||||
|
|
||||||
local operation = ARGV[1]
|
local operation = ARGV[1]
|
||||||
local operation_ttl_ms = tonumber(ARGV[2])
|
local operation_ttl_ms = tonumber(ARGV[2])
|
||||||
@ -31,8 +30,6 @@ local active = tonumber(ARGV[9])
|
|||||||
local reserved = tonumber(ARGV[10])
|
local reserved = tonumber(ARGV[10])
|
||||||
local worker_index_key = ARGV[11]
|
local worker_index_key = ARGV[11]
|
||||||
local worker_draining_index_key = ARGV[12]
|
local worker_draining_index_key = ARGV[12]
|
||||||
local upstream_id = ARGV[13]
|
|
||||||
local policy_revision = ARGV[14]
|
|
||||||
local mutating = operation ~= 'get'
|
local mutating = operation ~= 'get'
|
||||||
|
|
||||||
local function finish(reply)
|
local function finish(reply)
|
||||||
@ -215,11 +212,6 @@ local function decode_table(raw)
|
|||||||
return value
|
return value
|
||||||
end
|
end
|
||||||
|
|
||||||
local function valid_uint(value)
|
|
||||||
return type(value) == 'string' and string.match(value, '^[0-9]+$') and
|
|
||||||
value ~= '0' and (string.len(value) == 1 or string.sub(value, 1, 1) ~= '0')
|
|
||||||
end
|
|
||||||
|
|
||||||
local function valid_assignment(assignment)
|
local function valid_assignment(assignment)
|
||||||
return assignment and assignment.version == 1 and type(assignment.proxyId) == 'string' and
|
return assignment and assignment.version == 1 and type(assignment.proxyId) == 'string' and
|
||||||
assignment.proxyId ~= '' and type(assignment.workerId) == 'string' and assignment.workerId ~= '' and
|
assignment.proxyId ~= '' and type(assignment.workerId) == 'string' and assignment.workerId ~= '' and
|
||||||
@ -404,33 +396,6 @@ if operation == 'begin_unhealthy_drain' then
|
|||||||
return finish({status = 'ok', record = encoded, started = started})
|
return finish({status = 'ok', record = encoded, started = started})
|
||||||
end
|
end
|
||||||
|
|
||||||
if operation == 'begin_disabled_upstream_drain' then
|
|
||||||
cleanup_hard_expired(now_ms)
|
|
||||||
if type(upstream_id) ~= 'string' or upstream_id == '' or not valid_uint(policy_revision) then
|
|
||||||
return finish({status = 'invalid'})
|
|
||||||
end
|
|
||||||
local policy = decode_table(redis.call('HGET', upstream_drain_policies_key, upstream_id))
|
|
||||||
if not policy or policy.version ~= 1 or policy.upstreamId ~= upstream_id or policy.enabled ~= false or
|
|
||||||
policy.revision ~= policy_revision then
|
|
||||||
return finish({status = 'stale'})
|
|
||||||
end
|
|
||||||
local current = decode_table(redis.call('HGET', owners_key, proxy_id))
|
|
||||||
if not valid_assignment(current) or current.workerId ~= worker_id or tonumber(current.epoch) ~= epoch or
|
|
||||||
tonumber(current.expiresAtMs) <= now_ms then
|
|
||||||
return finish({status = 'stale'})
|
|
||||||
end
|
|
||||||
local record = decode_table(redis.call('HGET', records_key, proxy_id))
|
|
||||||
if not record or record.sourceUpstream ~= upstream_id or record.ownerWorkerId ~= worker_id or
|
|
||||||
record.state ~= 'AVAILABLE' then
|
|
||||||
return finish({status = 'stale'})
|
|
||||||
end
|
|
||||||
local encoded, started, failure = begin_drain(current)
|
|
||||||
if failure then
|
|
||||||
return finish({status = failure})
|
|
||||||
end
|
|
||||||
return finish({status = 'ok', record = encoded, started = started})
|
|
||||||
end
|
|
||||||
|
|
||||||
if operation == 'acknowledge_drain' then
|
if operation == 'acknowledge_drain' then
|
||||||
local current = decode_table(redis.call('HGET', owners_key, proxy_id))
|
local current = decode_table(redis.call('HGET', owners_key, proxy_id))
|
||||||
if not valid_assignment(current) or current.workerId ~= worker_id or tonumber(current.epoch) ~= epoch then
|
if not valid_assignment(current) or current.workerId ~= worker_id or tonumber(current.epoch) ~= epoch then
|
||||||
|
|||||||
@ -15,8 +15,6 @@ local health_tasks_key = KEYS[14]
|
|||||||
local health_task_expiry_key = KEYS[15]
|
local health_task_expiry_key = KEYS[15]
|
||||||
local health_ref_task_key = KEYS[16]
|
local health_ref_task_key = KEYS[16]
|
||||||
local health_unhealthy_key = KEYS[17]
|
local health_unhealthy_key = KEYS[17]
|
||||||
local owned_upstream_key = KEYS[18]
|
|
||||||
local upstream_drain_policies_key = KEYS[19]
|
|
||||||
|
|
||||||
local operation = ARGV[1]
|
local operation = ARGV[1]
|
||||||
local now_ms = tonumber(ARGV[2])
|
local now_ms = tonumber(ARGV[2])
|
||||||
@ -222,40 +220,6 @@ if operation == 'unhealthy' then
|
|||||||
end
|
end
|
||||||
return finish({status = 'ok', count = removed, deferredOwned = deferred_owned, drainCandidates = drain_candidates})
|
return finish({status = 'ok', count = removed, deferredOwned = deferred_owned, drainCandidates = drain_candidates})
|
||||||
end
|
end
|
||||||
if operation == 'disabled_upstream' then
|
|
||||||
local raw_policy = redis.call('HGET', upstream_drain_policies_key, upstream_id)
|
|
||||||
local policy = raw_policy and decode_table(raw_policy) or nil
|
|
||||||
if not policy or policy.version ~= 1 or policy.upstreamId ~= upstream_id or policy.enabled ~= false or
|
|
||||||
type(policy.revision) ~= 'string' or not string.match(policy.revision, '^[0-9]+$') or policy.revision == '0' then
|
|
||||||
return finish({status = 'stale', count = 0, disabledDrainCandidates = {}})
|
|
||||||
end
|
|
||||||
local scan_limit = limit * 4
|
|
||||||
if scan_limit > 1024 then
|
|
||||||
scan_limit = 1024
|
|
||||||
end
|
|
||||||
local proxy_ids = redis.call('ZRANGEBYSCORE', owned_upstream_key, '(' .. now_ms, '+inf', 'LIMIT', 0, scan_limit)
|
|
||||||
local candidates = {}
|
|
||||||
for _, proxy_id in ipairs(proxy_ids) do
|
|
||||||
local record = decode_table(redis.call('HGET', records_key, proxy_id))
|
|
||||||
local owner = decode_table(redis.call('HGET', owners_key, proxy_id))
|
|
||||||
if record and owner and record.sourceUpstream == upstream_id and record.state == 'AVAILABLE' and
|
|
||||||
record.ownerWorkerId == owner.workerId and owner.draining == false and
|
|
||||||
type(owner.workerId) == 'string' and owner.workerId ~= '' and tonumber(owner.epoch) and
|
|
||||||
tonumber(owner.epoch) > 0 and tonumber(owner.expiresAtMs) and tonumber(owner.expiresAtMs) > now_ms then
|
|
||||||
table.insert(candidates, {
|
|
||||||
upstreamId = upstream_id,
|
|
||||||
policyRevision = policy.revision,
|
|
||||||
proxyId = proxy_id,
|
|
||||||
workerId = owner.workerId,
|
|
||||||
assignmentEpoch = tonumber(owner.epoch),
|
|
||||||
})
|
|
||||||
if #candidates == limit then
|
|
||||||
break
|
|
||||||
end
|
|
||||||
end
|
|
||||||
end
|
|
||||||
return finish({status = 'ok', count = 0, disabledDrainCandidates = candidates})
|
|
||||||
end
|
|
||||||
if operation == 'inventory' then
|
if operation == 'inventory' then
|
||||||
local count = tonumber(redis.call('HGET', inventory_key, upstream_id) or '0')
|
local count = tonumber(redis.call('HGET', inventory_key, upstream_id) or '0')
|
||||||
if count < 0 then
|
if count < 0 then
|
||||||
|
|||||||
@ -1,23 +0,0 @@
|
|||||||
local policies_key = KEYS[1]
|
|
||||||
local policies_json = ARGV[1]
|
|
||||||
|
|
||||||
local decoded, policies = pcall(cjson.decode, policies_json or '')
|
|
||||||
if not decoded or type(policies) ~= 'table' then
|
|
||||||
return cjson.encode({status = 'invalid'})
|
|
||||||
end
|
|
||||||
|
|
||||||
local seen = {}
|
|
||||||
for _, policy in ipairs(policies) do
|
|
||||||
if type(policy) ~= 'table' or type(policy.upstreamId) ~= 'string' or policy.upstreamId == '' or
|
|
||||||
type(policy.revision) ~= 'string' or not string.match(policy.revision, '^[0-9]+$') or
|
|
||||||
policy.revision == '0' or type(policy.enabled) ~= 'boolean' or seen[policy.upstreamId] then
|
|
||||||
return cjson.encode({status = 'invalid'})
|
|
||||||
end
|
|
||||||
seen[policy.upstreamId] = true
|
|
||||||
end
|
|
||||||
|
|
||||||
redis.call('DEL', policies_key)
|
|
||||||
for _, policy in ipairs(policies) do
|
|
||||||
redis.call('HSET', policies_key, policy.upstreamId, cjson.encode(policy))
|
|
||||||
end
|
|
||||||
return cjson.encode({status = 'ok'})
|
|
||||||
@ -1,63 +0,0 @@
|
|||||||
package redisactivity
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"encoding/json"
|
|
||||||
"strconv"
|
|
||||||
|
|
||||||
"proxy-pool/internal/domain/activitypool"
|
|
||||||
)
|
|
||||||
|
|
||||||
var _ activitypool.UpstreamDrainPolicyWriter = (*Adapter)(nil)
|
|
||||||
|
|
||||||
type upstreamDrainPolicyRecord struct {
|
|
||||||
Version int `json:"version"`
|
|
||||||
UpstreamID string `json:"upstreamId"`
|
|
||||||
Revision string `json:"revision"`
|
|
||||||
Enabled bool `json:"enabled"`
|
|
||||||
}
|
|
||||||
|
|
||||||
func (a *Adapter) ReplaceUpstreamDrainPolicies(ctx context.Context, policies []activitypool.UpstreamDrainPolicy) error {
|
|
||||||
if ctx == nil {
|
|
||||||
return activitypool.ErrInvalidMaintenance
|
|
||||||
}
|
|
||||||
if err := ctx.Err(); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if a == nil {
|
|
||||||
return activitypool.ErrInvalidMaintenance
|
|
||||||
}
|
|
||||||
records := make([]upstreamDrainPolicyRecord, 0, len(policies))
|
|
||||||
seen := make(map[string]struct{}, len(policies))
|
|
||||||
for _, policy := range policies {
|
|
||||||
if policy.UpstreamID == "" || policy.Revision == 0 {
|
|
||||||
return activitypool.ErrInvalidMaintenance
|
|
||||||
}
|
|
||||||
if _, duplicate := seen[policy.UpstreamID]; duplicate {
|
|
||||||
return activitypool.ErrInvalidMaintenance
|
|
||||||
}
|
|
||||||
seen[policy.UpstreamID] = struct{}{}
|
|
||||||
records = append(records, upstreamDrainPolicyRecord{
|
|
||||||
Version: 1, UpstreamID: policy.UpstreamID, Revision: strconv.FormatUint(policy.Revision, 10), Enabled: policy.Enabled,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
payload, err := json.Marshal(records)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
result, err := runScript(ctx, a.client, upstreamDrainPolicyScript, []string{a.keys.upstreamDrainPolicies}, string(payload))
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
var reply upstreamDrainPolicyScriptReply
|
|
||||||
if err := decodeScriptResult(result, &reply); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if reply.Status == scriptInvalid {
|
|
||||||
return activitypool.ErrInvalidMaintenance
|
|
||||||
}
|
|
||||||
if reply.Status != scriptOK {
|
|
||||||
return invalidScriptReply("unexpected upstream drain policy reply")
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
@ -8,13 +8,9 @@ import (
|
|||||||
"context"
|
"context"
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"net"
|
|
||||||
"net/http"
|
|
||||||
"reflect"
|
"reflect"
|
||||||
"strings"
|
"strings"
|
||||||
"sync/atomic"
|
|
||||||
|
|
||||||
"github.com/prometheus/client_golang/prometheus"
|
|
||||||
"google.golang.org/grpc"
|
"google.golang.org/grpc"
|
||||||
"google.golang.org/grpc/credentials"
|
"google.golang.org/grpc/credentials"
|
||||||
|
|
||||||
@ -23,17 +19,12 @@ import (
|
|||||||
"proxy-pool/internal/checker/probe"
|
"proxy-pool/internal/checker/probe"
|
||||||
"proxy-pool/internal/config"
|
"proxy-pool/internal/config"
|
||||||
"proxy-pool/internal/controlplane/clienttransport"
|
"proxy-pool/internal/controlplane/clienttransport"
|
||||||
"proxy-pool/internal/controlplane/tlsreload"
|
|
||||||
"proxy-pool/internal/domain/workerruntime"
|
"proxy-pool/internal/domain/workerruntime"
|
||||||
"proxy-pool/internal/platform/httpserver"
|
|
||||||
"proxy-pool/internal/platform/lifecycle"
|
|
||||||
platformMetrics "proxy-pool/internal/platform/metrics"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
var (
|
var (
|
||||||
ErrInvalidOptions = errors.New("invalid checker bootstrap options")
|
ErrInvalidOptions = errors.New("invalid checker bootstrap options")
|
||||||
ErrStartup = errors.New("checker startup failed")
|
ErrStartup = errors.New("checker startup failed")
|
||||||
ErrNotReady = errors.New("checker control plane is not ready")
|
|
||||||
)
|
)
|
||||||
|
|
||||||
type Options struct {
|
type Options struct {
|
||||||
@ -42,12 +33,9 @@ type Options struct {
|
|||||||
ControlPlaneAddress string
|
ControlPlaneAddress string
|
||||||
CheckerID string
|
CheckerID string
|
||||||
InstanceID string
|
InstanceID string
|
||||||
AutoIdentity bool
|
|
||||||
MaxInFlight int
|
MaxInFlight int
|
||||||
SupportedLevels []controlplanev1.CheckLevel
|
SupportedLevels []controlplanev1.CheckLevel
|
||||||
GRPCTransport credentials.TransportCredentials
|
GRPCTransport credentials.TransportCredentials
|
||||||
MetricsListener net.Listener
|
|
||||||
HTTP httpserver.Options
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func Run(ctx context.Context, options Options) error {
|
func Run(ctx context.Context, options Options) error {
|
||||||
@ -61,27 +49,11 @@ func Run(ctx context.Context, options Options) error {
|
|||||||
if !configuration.ControlPlane.Enabled {
|
if !configuration.ControlPlane.Enabled {
|
||||||
return errors.Join(ErrInvalidOptions, errors.New("controlPlane must be enabled"))
|
return errors.Join(ErrInvalidOptions, errors.New("controlPlane must be enabled"))
|
||||||
}
|
}
|
||||||
options, err = resolveIdentity(configuration, options)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("%w: %w", ErrStartup, err)
|
|
||||||
}
|
|
||||||
runtime, err := newRuntime(ctx, configuration, options)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("%w: %w", ErrStartup, err)
|
|
||||||
}
|
|
||||||
defer runtime.Close()
|
|
||||||
return runtime.Run(ctx)
|
|
||||||
}
|
|
||||||
|
|
||||||
func newRuntime(ctx context.Context, configuration *config.Config, options Options) (*runtime, error) {
|
|
||||||
if ctx == nil || configuration == nil {
|
|
||||||
return nil, ErrInvalidOptions
|
|
||||||
}
|
|
||||||
transport, err := clienttransport.New(
|
transport, err := clienttransport.New(
|
||||||
configuration.ControlPlane, options.ControlPlaneAddress, configuration.ControlPlane.CheckerTLS, options.GRPCTransport, "checkerTLS",
|
configuration.ControlPlane, options.ControlPlaneAddress, configuration.ControlPlane.CheckerTLS, options.GRPCTransport, "checkerTLS",
|
||||||
)
|
)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("%w: %w", ErrInvalidOptions, err)
|
return fmt.Errorf("%w: %w", ErrInvalidOptions, err)
|
||||||
}
|
}
|
||||||
connection, err := grpc.NewClient(options.ControlPlaneAddress,
|
connection, err := grpc.NewClient(options.ControlPlaneAddress,
|
||||||
grpc.WithTransportCredentials(transport),
|
grpc.WithTransportCredentials(transport),
|
||||||
@ -91,164 +63,31 @@ func newRuntime(ctx context.Context, configuration *config.Config, options Optio
|
|||||||
),
|
),
|
||||||
)
|
)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("dial control plane: %w", err)
|
return fmt.Errorf("%w: dial control plane: %w", ErrStartup, err)
|
||||||
}
|
}
|
||||||
closeConnection := true
|
defer connection.Close()
|
||||||
defer func() {
|
|
||||||
if closeConnection {
|
|
||||||
_ = connection.Close()
|
|
||||||
}
|
|
||||||
}()
|
|
||||||
client := controlplane.NewGeneratedClient(controlplanev1.NewCheckerControlPlaneClient(connection))
|
client := controlplane.NewGeneratedClient(controlplanev1.NewCheckerControlPlaneClient(connection))
|
||||||
readiness := &controlPlaneReadiness{}
|
|
||||||
runner, err := controlplane.NewRunner(client, probe.NewExecutor(), controlplane.Options{
|
runner, err := controlplane.NewRunner(client, probe.NewExecutor(), controlplane.Options{
|
||||||
CheckerID: options.CheckerID, InstanceID: options.InstanceID, MaxInFlight: options.MaxInFlight,
|
CheckerID: options.CheckerID, InstanceID: options.InstanceID, MaxInFlight: options.MaxInFlight,
|
||||||
SupportedLevels: append([]controlplanev1.CheckLevel(nil), options.SupportedLevels...),
|
SupportedLevels: append([]controlplanev1.CheckLevel(nil), options.SupportedLevels...),
|
||||||
ReportBatchSize: options.MaxInFlight, OnSuccessfulPull: readiness.MarkReady, OnFailedPull: readiness.MarkUnavailable,
|
ReportBatchSize: options.MaxInFlight,
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("build checker runner: %w", err)
|
return fmt.Errorf("%w: build checker runner: %w", ErrStartup, err)
|
||||||
}
|
}
|
||||||
runners := []lifecycle.Runner{runner}
|
return runner.Run(ctx)
|
||||||
if configuration.Metrics.Enabled {
|
|
||||||
metrics, metricsErr := newMetricsRuntime(ctx, configuration.Metrics, options, readiness)
|
|
||||||
if metricsErr != nil {
|
|
||||||
return nil, metricsErr
|
|
||||||
}
|
|
||||||
runners = append(runners, metrics)
|
|
||||||
} else if options.MetricsListener != nil {
|
|
||||||
return nil, ErrInvalidOptions
|
|
||||||
}
|
|
||||||
group, err := lifecycle.NewGroup(runners...)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
closeConnection = false
|
|
||||||
return &runtime{connection: connection, group: group}, nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func validateOptions(ctx context.Context, options Options) error {
|
func validateOptions(ctx context.Context, options Options) error {
|
||||||
if ctx == nil || strings.TrimSpace(options.ConfigPath) != options.ConfigPath || options.ConfigPath == "" ||
|
if ctx == nil || strings.TrimSpace(options.ConfigPath) != options.ConfigPath || options.ConfigPath == "" ||
|
||||||
nilInterface(options.Resolver) || !clienttransport.ValidDialAddress(options.ControlPlaneAddress) ||
|
nilInterface(options.Resolver) || !clienttransport.ValidDialAddress(options.ControlPlaneAddress) ||
|
||||||
(options.CheckerID != "" && !workerruntime.ValidIdentifier(options.CheckerID)) ||
|
!workerruntime.ValidIdentifier(options.CheckerID) || !workerruntime.ValidIdentifier(options.InstanceID) ||
|
||||||
(options.InstanceID != "" && !workerruntime.ValidIdentifier(options.InstanceID)) ||
|
|
||||||
(!options.AutoIdentity && (!workerruntime.ValidIdentifier(options.CheckerID) || !workerruntime.ValidIdentifier(options.InstanceID))) ||
|
|
||||||
options.MaxInFlight <= 0 || len(options.SupportedLevels) == 0 {
|
options.MaxInFlight <= 0 || len(options.SupportedLevels) == 0 {
|
||||||
return ErrInvalidOptions
|
return ErrInvalidOptions
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func resolveIdentity(configuration *config.Config, options Options) (Options, error) {
|
|
||||||
if configuration == nil {
|
|
||||||
return Options{}, ErrInvalidOptions
|
|
||||||
}
|
|
||||||
if !options.AutoIdentity {
|
|
||||||
return options, nil
|
|
||||||
}
|
|
||||||
if configuration.ControlPlane.TLS.Mode != "mtls" {
|
|
||||||
return Options{}, errors.Join(ErrInvalidOptions, errors.New("auto identity requires controlPlane mTLS"))
|
|
||||||
}
|
|
||||||
identity, err := tlsreload.ResolveSPIFFEIdentity(
|
|
||||||
configuration.ControlPlane.CheckerTLS.CertFile, configuration.ControlPlane.CheckerTLS.KeyFile,
|
|
||||||
configuration.ControlPlane.TLS.TrustDomain, configuration.ControlPlane.TLS.Environment, "checker",
|
|
||||||
)
|
|
||||||
if err != nil {
|
|
||||||
return Options{}, fmt.Errorf("resolve checker SPIFFE identity: %w", err)
|
|
||||||
}
|
|
||||||
if options.CheckerID == "" {
|
|
||||||
options.CheckerID = identity
|
|
||||||
}
|
|
||||||
if options.InstanceID == "" {
|
|
||||||
options.InstanceID = identity
|
|
||||||
}
|
|
||||||
if !workerruntime.ValidIdentifier(options.CheckerID) || !workerruntime.ValidIdentifier(options.InstanceID) || options.CheckerID != identity {
|
|
||||||
return Options{}, errors.Join(ErrInvalidOptions, errors.New("checker identity does not match SPIFFE certificate"))
|
|
||||||
}
|
|
||||||
return options, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
type runtime struct {
|
|
||||||
connection *grpc.ClientConn
|
|
||||||
group *lifecycle.Group
|
|
||||||
}
|
|
||||||
|
|
||||||
func (runtime *runtime) Run(ctx context.Context) error {
|
|
||||||
if runtime == nil || runtime.connection == nil || runtime.group == nil || ctx == nil {
|
|
||||||
return ErrInvalidOptions
|
|
||||||
}
|
|
||||||
return runtime.group.Run(ctx)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (runtime *runtime) Close() {
|
|
||||||
if runtime != nil && runtime.connection != nil {
|
|
||||||
_ = runtime.connection.Close()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
type controlPlaneReadiness struct {
|
|
||||||
ready atomic.Bool
|
|
||||||
}
|
|
||||||
|
|
||||||
func (readiness *controlPlaneReadiness) MarkReady() {
|
|
||||||
if readiness != nil {
|
|
||||||
readiness.ready.Store(true)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (readiness *controlPlaneReadiness) MarkUnavailable() {
|
|
||||||
if readiness != nil {
|
|
||||||
readiness.ready.Store(false)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (readiness *controlPlaneReadiness) Ready(ctx context.Context) error {
|
|
||||||
if ctx == nil || readiness == nil || !readiness.ready.Load() {
|
|
||||||
return ErrNotReady
|
|
||||||
}
|
|
||||||
return ctx.Err()
|
|
||||||
}
|
|
||||||
|
|
||||||
type metricsRuntime struct {
|
|
||||||
listener net.Listener
|
|
||||||
handler http.Handler
|
|
||||||
options httpserver.Options
|
|
||||||
}
|
|
||||||
|
|
||||||
func newMetricsRuntime(
|
|
||||||
ctx context.Context,
|
|
||||||
configuration config.Metrics,
|
|
||||||
options Options,
|
|
||||||
readiness *controlPlaneReadiness,
|
|
||||||
) (*metricsRuntime, error) {
|
|
||||||
if ctx == nil || !configuration.Enabled || readiness == nil {
|
|
||||||
return nil, ErrInvalidOptions
|
|
||||||
}
|
|
||||||
handler, err := platformMetrics.NewHandler(platformMetrics.Dependencies{
|
|
||||||
Gatherer: prometheus.DefaultGatherer, Readiness: readiness,
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("build checker metrics handler: %w", err)
|
|
||||||
}
|
|
||||||
listener := options.MetricsListener
|
|
||||||
if listener == nil {
|
|
||||||
listener, err = (&net.ListenConfig{}).Listen(ctx, "tcp", configuration.Listen)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("listen checker metrics: %w", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return &metricsRuntime{listener: listener, handler: handler, options: options.HTTP}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (runtime *metricsRuntime) Run(ctx context.Context) error {
|
|
||||||
if runtime == nil || runtime.listener == nil || runtime.handler == nil || ctx == nil {
|
|
||||||
return ErrInvalidOptions
|
|
||||||
}
|
|
||||||
return httpserver.Serve(ctx, runtime.options, httpserver.Endpoint{
|
|
||||||
Name: "metrics", Listener: runtime.listener, Handler: runtime.handler,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
func loadConfiguration(ctx context.Context, path string, resolver config.Resolver) (*config.Config, error) {
|
func loadConfiguration(ctx context.Context, path string, resolver config.Resolver) (*config.Config, error) {
|
||||||
if ctx == nil || strings.TrimSpace(path) != path || path == "" || nilInterface(resolver) {
|
if ctx == nil || strings.TrimSpace(path) != path || path == "" || nilInterface(resolver) {
|
||||||
return nil, ErrInvalidOptions
|
return nil, ErrInvalidOptions
|
||||||
|
|||||||
@ -1,55 +0,0 @@
|
|||||||
package bootstrap
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"errors"
|
|
||||||
"net"
|
|
||||||
"net/http"
|
|
||||||
"net/http/httptest"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"proxy-pool/internal/config"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestControlPlaneReadinessTransitionsAfterSuccessfulPull(t *testing.T) {
|
|
||||||
readiness := &controlPlaneReadiness{}
|
|
||||||
if err := readiness.Ready(context.Background()); !errors.Is(err, ErrNotReady) {
|
|
||||||
t.Fatalf("Ready() before pull = %v, want ErrNotReady", err)
|
|
||||||
}
|
|
||||||
readiness.MarkReady()
|
|
||||||
if err := readiness.Ready(context.Background()); err != nil {
|
|
||||||
t.Fatalf("Ready() after successful pull = %v", err)
|
|
||||||
}
|
|
||||||
readiness.MarkUnavailable()
|
|
||||||
if err := readiness.Ready(context.Background()); !errors.Is(err, ErrNotReady) {
|
|
||||||
t.Fatalf("Ready() after failed pull = %v, want ErrNotReady", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestMetricsRuntimeUsesControlPlaneReadiness(t *testing.T) {
|
|
||||||
listener, err := net.Listen("tcp", "127.0.0.1:0")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("Listen(): %v", err)
|
|
||||||
}
|
|
||||||
defer listener.Close()
|
|
||||||
readiness := &controlPlaneReadiness{}
|
|
||||||
runtime, err := newMetricsRuntime(context.Background(), config.Metrics{Enabled: true, Listen: listener.Addr().String()}, Options{
|
|
||||||
MetricsListener: listener,
|
|
||||||
}, readiness)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("newMetricsRuntime(): %v", err)
|
|
||||||
}
|
|
||||||
assertCheckerProbe(t, runtime.handler, "/livez", http.StatusOK, "live\n")
|
|
||||||
assertCheckerProbe(t, runtime.handler, "/readyz", http.StatusServiceUnavailable, "unavailable\n")
|
|
||||||
readiness.MarkReady()
|
|
||||||
assertCheckerProbe(t, runtime.handler, "/readyz", http.StatusOK, "ready\n")
|
|
||||||
}
|
|
||||||
|
|
||||||
func assertCheckerProbe(t *testing.T, handler http.Handler, path string, wantStatus int, wantBody string) {
|
|
||||||
t.Helper()
|
|
||||||
recorder := httptest.NewRecorder()
|
|
||||||
handler.ServeHTTP(recorder, httptest.NewRequest(http.MethodGet, path, nil))
|
|
||||||
if recorder.Code != wantStatus || recorder.Body.String() != wantBody {
|
|
||||||
t.Fatalf("GET %s = %d %q, want %d %q", path, recorder.Code, recorder.Body.String(), wantStatus, wantBody)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@ -44,16 +44,14 @@ type Executor interface {
|
|||||||
}
|
}
|
||||||
|
|
||||||
type Options struct {
|
type Options struct {
|
||||||
CheckerID string
|
CheckerID string
|
||||||
InstanceID string
|
InstanceID string
|
||||||
MaxInFlight int
|
MaxInFlight int
|
||||||
SupportedLevels []controlplanev1.CheckLevel
|
SupportedLevels []controlplanev1.CheckLevel
|
||||||
ReportBatchSize int
|
ReportBatchSize int
|
||||||
PollInterval time.Duration
|
PollInterval time.Duration
|
||||||
RetryDelay time.Duration
|
RetryDelay time.Duration
|
||||||
Now func() time.Time
|
Now func() time.Time
|
||||||
OnSuccessfulPull func()
|
|
||||||
OnFailedPull func()
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Runner consumes at most one server-bounded pull per RunOnce. Run adds a
|
// Runner consumes at most one server-bounded pull per RunOnce. Run adds a
|
||||||
@ -95,9 +93,6 @@ func (runner *Runner) Run(ctx context.Context) error {
|
|||||||
if err != nil && ctx.Err() != nil {
|
if err != nil && ctx.Err() != nil {
|
||||||
return ctx.Err()
|
return ctx.Err()
|
||||||
}
|
}
|
||||||
if err != nil && runner.options.OnFailedPull != nil {
|
|
||||||
runner.options.OnFailedPull()
|
|
||||||
}
|
|
||||||
wait := runner.options.PollInterval
|
wait := runner.options.PollInterval
|
||||||
if err != nil {
|
if err != nil {
|
||||||
wait = runner.options.RetryDelay
|
wait = runner.options.RetryDelay
|
||||||
@ -123,13 +118,9 @@ func (runner *Runner) RunOnce(ctx context.Context) error {
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
tasks, err := collectTasks(stream, runner.options.MaxInFlight)
|
tasks, err := collectTasks(stream, runner.options.MaxInFlight)
|
||||||
if err != nil {
|
if err != nil || len(tasks) == 0 {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
if len(tasks) == 0 {
|
|
||||||
runner.observeSuccessfulPull()
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
observations := runner.executeTasks(ctx, tasks)
|
observations := runner.executeTasks(ctx, tasks)
|
||||||
for start := 0; start < len(observations); start += runner.options.ReportBatchSize {
|
for start := 0; start < len(observations); start += runner.options.ReportBatchSize {
|
||||||
end := start + runner.options.ReportBatchSize
|
end := start + runner.options.ReportBatchSize
|
||||||
@ -146,16 +137,9 @@ func (runner *Runner) RunOnce(ctx context.Context) error {
|
|||||||
return ErrObservationsRejected
|
return ErrObservationsRejected
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
runner.observeSuccessfulPull()
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (runner *Runner) observeSuccessfulPull() {
|
|
||||||
if runner != nil && runner.options.OnSuccessfulPull != nil {
|
|
||||||
runner.options.OnSuccessfulPull()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func collectTasks(stream TaskStream, maximum int) ([]*controlplanev1.CheckTask, error) {
|
func collectTasks(stream TaskStream, maximum int) ([]*controlplanev1.CheckTask, error) {
|
||||||
if stream == nil || maximum <= 0 {
|
if stream == nil || maximum <= 0 {
|
||||||
return nil, ErrInvalidRunner
|
return nil, ErrInvalidRunner
|
||||||
|
|||||||
@ -2,10 +2,8 @@ package controlplane
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"errors"
|
|
||||||
"io"
|
"io"
|
||||||
"sync"
|
"sync"
|
||||||
"sync/atomic"
|
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
@ -69,51 +67,6 @@ func TestRunnerOmitsEgressProbeURLFromGlobalObservation(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestRunnerMarksReadyAfterSuccessfulEmptyPull(t *testing.T) {
|
|
||||||
var successfulPulls atomic.Int64
|
|
||||||
runner, err := NewRunner(&clientStub{stream: &taskStreamStub{}}, &executorStub{}, Options{
|
|
||||||
CheckerID: "checker-a", InstanceID: "instance-a", MaxInFlight: 1,
|
|
||||||
SupportedLevels: []controlplanev1.CheckLevel{controlplanev1.CheckLevel_CHECK_LEVEL_BASIC},
|
|
||||||
Now: time.Now, OnSuccessfulPull: func() { successfulPulls.Add(1) },
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("NewRunner(): %v", err)
|
|
||||||
}
|
|
||||||
if err := runner.RunOnce(context.Background()); err != nil {
|
|
||||||
t.Fatalf("RunOnce(): %v", err)
|
|
||||||
}
|
|
||||||
if successfulPulls.Load() != 1 {
|
|
||||||
t.Fatalf("successful pulls = %d, want 1", successfulPulls.Load())
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestRunnerMarksUnavailableAfterPullFailure(t *testing.T) {
|
|
||||||
failed := make(chan struct{})
|
|
||||||
var markOnce sync.Once
|
|
||||||
runner, err := NewRunner(&clientStub{streamErr: errors.New("control plane unavailable")}, &executorStub{}, Options{
|
|
||||||
CheckerID: "checker-a", InstanceID: "instance-a", MaxInFlight: 1,
|
|
||||||
SupportedLevels: []controlplanev1.CheckLevel{controlplanev1.CheckLevel_CHECK_LEVEL_BASIC},
|
|
||||||
Now: time.Now, RetryDelay: time.Millisecond,
|
|
||||||
OnFailedPull: func() { markOnce.Do(func() { close(failed) }) },
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("NewRunner(): %v", err)
|
|
||||||
}
|
|
||||||
ctx, cancel := context.WithCancel(context.Background())
|
|
||||||
defer cancel()
|
|
||||||
completed := make(chan error, 1)
|
|
||||||
go func() { completed <- runner.Run(ctx) }()
|
|
||||||
select {
|
|
||||||
case <-failed:
|
|
||||||
cancel()
|
|
||||||
case <-time.After(time.Second):
|
|
||||||
t.Fatal("Runner did not mark failed pull")
|
|
||||||
}
|
|
||||||
if err := <-completed; !errors.Is(err, context.Canceled) {
|
|
||||||
t.Fatalf("Run() = %v, want context.Canceled", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func checkerTask(id string, now time.Time) *controlplanev1.CheckTask {
|
func checkerTask(id string, now time.Time) *controlplanev1.CheckTask {
|
||||||
return &controlplanev1.CheckTask{
|
return &controlplanev1.CheckTask{
|
||||||
TaskId: id, ProxyId: id + "-proxy", Protocol: controlplanev1.ProxyProtocol_PROXY_PROTOCOL_HTTP,
|
TaskId: id, ProxyId: id + "-proxy", Protocol: controlplanev1.ProxyProtocol_PROXY_PROTOCOL_HTTP,
|
||||||
@ -123,13 +76,12 @@ func checkerTask(id string, now time.Time) *controlplanev1.CheckTask {
|
|||||||
}
|
}
|
||||||
|
|
||||||
type clientStub struct {
|
type clientStub struct {
|
||||||
stream TaskStream
|
stream TaskStream
|
||||||
streamErr error
|
batches []*controlplanev1.ObservationBatch
|
||||||
batches []*controlplanev1.ObservationBatch
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func (stub *clientStub) StreamCheckTasks(context.Context, *controlplanev1.StreamCheckTasksRequest) (TaskStream, error) {
|
func (stub *clientStub) StreamCheckTasks(context.Context, *controlplanev1.StreamCheckTasksRequest) (TaskStream, error) {
|
||||||
return stub.stream, stub.streamErr
|
return stub.stream, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (stub *clientStub) ReportObservations(_ context.Context, batch *controlplanev1.ObservationBatch) (*controlplanev1.ReportObservationsResponse, error) {
|
func (stub *clientStub) ReportObservations(_ context.Context, batch *controlplanev1.ObservationBatch) (*controlplanev1.ReportObservationsResponse, error) {
|
||||||
|
|||||||
@ -3,8 +3,6 @@ package config
|
|||||||
import (
|
import (
|
||||||
"fmt"
|
"fmt"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"proxy-pool/internal/domain/clientpolicy"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
// MaximumCheckURLs bounds per-upstream EGRESS references. The bound keeps
|
// MaximumCheckURLs bounds per-upstream EGRESS references. The bound keeps
|
||||||
@ -65,8 +63,6 @@ type Listener struct {
|
|||||||
Auth Auth `yaml:"auth"`
|
Auth Auth `yaml:"auth"`
|
||||||
Limits Limits `yaml:"limits"`
|
Limits Limits `yaml:"limits"`
|
||||||
Retry Retry `yaml:"retry"`
|
Retry Retry `yaml:"retry"`
|
||||||
StickySession StickySession `yaml:"stickySession"`
|
|
||||||
Transport GatewayTransport `yaml:"transport"`
|
|
||||||
DestinationPolicy DestinationPolicy `yaml:"destinationPolicy"`
|
DestinationPolicy DestinationPolicy `yaml:"destinationPolicy"`
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -82,30 +78,26 @@ type Access struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
type Auth struct {
|
type Auth struct {
|
||||||
Mode string `yaml:"mode"`
|
Mode string `yaml:"mode"`
|
||||||
Permissions []string `yaml:"permissions"`
|
Username string `yaml:"username"`
|
||||||
ClientPolicy clientpolicy.Policy `yaml:"client"`
|
Password string `yaml:"password"`
|
||||||
Username string `yaml:"username"`
|
PasswordFile string `yaml:"passwordFile"`
|
||||||
Password string `yaml:"password"`
|
Token string `yaml:"token"`
|
||||||
PasswordFile string `yaml:"passwordFile"`
|
TokenFile string `yaml:"tokenFile"`
|
||||||
Token string `yaml:"token"`
|
Header string `yaml:"header"`
|
||||||
TokenFile string `yaml:"tokenFile"`
|
CIDRs []string `yaml:"cidrs"`
|
||||||
Header string `yaml:"header"`
|
Methods []AuthMethod `yaml:"methods"`
|
||||||
CIDRs []string `yaml:"cidrs"`
|
|
||||||
Methods []AuthMethod `yaml:"methods"`
|
|
||||||
}
|
}
|
||||||
|
|
||||||
type AuthMethod struct {
|
type AuthMethod struct {
|
||||||
Mode string `yaml:"mode"`
|
Mode string `yaml:"mode"`
|
||||||
Permissions []string `yaml:"permissions"`
|
Username string `yaml:"username"`
|
||||||
ClientPolicy clientpolicy.Policy `yaml:"client"`
|
Password string `yaml:"password"`
|
||||||
Username string `yaml:"username"`
|
PasswordFile string `yaml:"passwordFile"`
|
||||||
Password string `yaml:"password"`
|
Header string `yaml:"header"`
|
||||||
PasswordFile string `yaml:"passwordFile"`
|
Value string `yaml:"value"`
|
||||||
Header string `yaml:"header"`
|
ValueFile string `yaml:"valueFile"`
|
||||||
Value string `yaml:"value"`
|
CIDRs []string `yaml:"cidrs"`
|
||||||
ValueFile string `yaml:"valueFile"`
|
|
||||||
CIDRs []string `yaml:"cidrs"`
|
|
||||||
}
|
}
|
||||||
|
|
||||||
type Limits struct {
|
type Limits struct {
|
||||||
@ -119,29 +111,6 @@ type Retry struct {
|
|||||||
RetryMethods []string `yaml:"retryMethods"`
|
RetryMethods []string `yaml:"retryMethods"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// StickySession configures per-Worker, bounded Gateway affinity. It does not
|
|
||||||
// store Proxy details or session identifiers in PostgreSQL or Redis.
|
|
||||||
type StickySession struct {
|
|
||||||
Enabled bool `yaml:"enabled"`
|
|
||||||
Header string `yaml:"header"`
|
|
||||||
TTL Duration `yaml:"ttl"`
|
|
||||||
MaxEntries int `yaml:"maxEntries"`
|
|
||||||
}
|
|
||||||
|
|
||||||
// GatewayTransport configures the local Gateway process connection pools and
|
|
||||||
// tunnel I/O. It intentionally contains no remote state or provider settings.
|
|
||||||
type GatewayTransport struct {
|
|
||||||
DialTimeout Duration `yaml:"dialTimeout"`
|
|
||||||
HandshakeTimeout Duration `yaml:"handshakeTimeout"`
|
|
||||||
ResponseHeaderTimeout Duration `yaml:"responseHeaderTimeout"`
|
|
||||||
IdleConnTimeout Duration `yaml:"idleConnTimeout"`
|
|
||||||
MaxIdleConns int `yaml:"maxIdleConns"`
|
|
||||||
MaxIdleConnsPerHost int `yaml:"maxIdleConnsPerHost"`
|
|
||||||
MaxConnsPerHost int `yaml:"maxConnsPerHost"`
|
|
||||||
TunnelBufferBytes int `yaml:"tunnelBufferBytes"`
|
|
||||||
TunnelIdleTimeout Duration `yaml:"tunnelIdleTimeout"`
|
|
||||||
}
|
|
||||||
|
|
||||||
type DestinationPolicy struct {
|
type DestinationPolicy struct {
|
||||||
DenyPrivateNetworks *bool `yaml:"denyPrivateNetworks"`
|
DenyPrivateNetworks *bool `yaml:"denyPrivateNetworks"`
|
||||||
DenyLoopback *bool `yaml:"denyLoopback"`
|
DenyLoopback *bool `yaml:"denyLoopback"`
|
||||||
@ -214,7 +183,6 @@ type Routing struct {
|
|||||||
Name string `yaml:"name"`
|
Name string `yaml:"name"`
|
||||||
Enabled bool `yaml:"enabled"`
|
Enabled bool `yaml:"enabled"`
|
||||||
Purpose string `yaml:"purpose"`
|
Purpose string `yaml:"purpose"`
|
||||||
Action string `yaml:"action"`
|
|
||||||
Match RoutingMatch `yaml:"match"`
|
Match RoutingMatch `yaml:"match"`
|
||||||
Upstreams []string `yaml:"upstreams"`
|
Upstreams []string `yaml:"upstreams"`
|
||||||
Strategy Strategy `yaml:"strategy"`
|
Strategy Strategy `yaml:"strategy"`
|
||||||
|
|||||||
@ -10,9 +10,6 @@ import (
|
|||||||
"time"
|
"time"
|
||||||
|
|
||||||
"go.yaml.in/yaml/v4"
|
"go.yaml.in/yaml/v4"
|
||||||
|
|
||||||
"proxy-pool/internal/domain/authorization"
|
|
||||||
"proxy-pool/internal/domain/clientpolicy"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
const validConfig = `
|
const validConfig = `
|
||||||
@ -204,7 +201,6 @@ func TestShippedDeploymentConfigurationsResolveEnvironment(t *testing.T) {
|
|||||||
wantGatewayUser string
|
wantGatewayUser string
|
||||||
wantPostgresURL string
|
wantPostgresURL string
|
||||||
wantRedisURL string
|
wantRedisURL string
|
||||||
wantTransport GatewayTransport
|
|
||||||
}{
|
}{
|
||||||
{
|
{
|
||||||
name: "local",
|
name: "local",
|
||||||
@ -212,12 +208,6 @@ func TestShippedDeploymentConfigurationsResolveEnvironment(t *testing.T) {
|
|||||||
wantGatewayUser: "local-gateway",
|
wantGatewayUser: "local-gateway",
|
||||||
wantPostgresURL: "postgres://proxy_pool:local-only-change-me@postgres:5432/proxy_pool?sslmode=disable",
|
wantPostgresURL: "postgres://proxy_pool:local-only-change-me@postgres:5432/proxy_pool?sslmode=disable",
|
||||||
wantRedisURL: "redis://redis:6379/0",
|
wantRedisURL: "redis://redis:6379/0",
|
||||||
wantTransport: GatewayTransport{
|
|
||||||
MaxIdleConns: 20000,
|
|
||||||
MaxIdleConnsPerHost: 32,
|
|
||||||
MaxConnsPerHost: 32,
|
|
||||||
TunnelBufferBytes: 32768,
|
|
||||||
},
|
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "kubernetes",
|
name: "kubernetes",
|
||||||
@ -226,17 +216,6 @@ func TestShippedDeploymentConfigurationsResolveEnvironment(t *testing.T) {
|
|||||||
wantGatewayUser: "resolved-gateway-user",
|
wantGatewayUser: "resolved-gateway-user",
|
||||||
wantPostgresURL: "postgres://resolved",
|
wantPostgresURL: "postgres://resolved",
|
||||||
wantRedisURL: "redis://resolved",
|
wantRedisURL: "redis://resolved",
|
||||||
wantTransport: GatewayTransport{
|
|
||||||
DialTimeout: Duration(10 * time.Second),
|
|
||||||
HandshakeTimeout: Duration(15 * time.Second),
|
|
||||||
ResponseHeaderTimeout: Duration(30 * time.Second),
|
|
||||||
IdleConnTimeout: Duration(90 * time.Second),
|
|
||||||
MaxIdleConns: 20000,
|
|
||||||
MaxIdleConnsPerHost: 32,
|
|
||||||
MaxConnsPerHost: 32,
|
|
||||||
TunnelBufferBytes: 32768,
|
|
||||||
TunnelIdleTimeout: Duration(5 * time.Minute),
|
|
||||||
},
|
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
for _, test := range tests {
|
for _, test := range tests {
|
||||||
@ -266,7 +245,6 @@ func TestShippedDeploymentConfigurationsResolveEnvironment(t *testing.T) {
|
|||||||
cfg.Admin.Auth.Token != "resolved-admin-token" ||
|
cfg.Admin.Auth.Token != "resolved-admin-token" ||
|
||||||
cfg.Storage.PostgresURL != test.wantPostgresURL ||
|
cfg.Storage.PostgresURL != test.wantPostgresURL ||
|
||||||
cfg.Storage.RedisURL != test.wantRedisURL ||
|
cfg.Storage.RedisURL != test.wantRedisURL ||
|
||||||
cfg.Gateway.Transport != test.wantTransport ||
|
|
||||||
cfg.Upstreams["provider-a"].API.Auth.Value != "resolved-provider-a-token" ||
|
cfg.Upstreams["provider-a"].API.Auth.Value != "resolved-provider-a-token" ||
|
||||||
cfg.Upstreams["provider-b"].API.Auth.Value != "resolved-provider-b-token" {
|
cfg.Upstreams["provider-b"].API.Auth.Value != "resolved-provider-b-token" {
|
||||||
t.Fatalf("deployment values were not resolved: %+v", cfg.Redacted())
|
t.Fatalf("deployment values were not resolved: %+v", cfg.Redacted())
|
||||||
@ -349,206 +327,6 @@ func TestValidateAcceptsBearerListenerAuthentication(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestValidateListenerPermissionConfiguration(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
cfg := mustLoadValidConfig(t)
|
|
||||||
cfg.Admin.Enabled = true
|
|
||||||
cfg.Admin.Listen = "127.0.0.1:8082"
|
|
||||||
cfg.Admin.Auth = Auth{
|
|
||||||
Mode: "bearer",
|
|
||||||
Token: "admin-token",
|
|
||||||
Permissions: []string{authorization.AdminRead},
|
|
||||||
}
|
|
||||||
if err := Validate(cfg); err != nil {
|
|
||||||
t.Fatalf("Validate(admin read permission) error = %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
cfg.Admin.Auth.Permissions = []string{"unknown"}
|
|
||||||
if err := Validate(cfg); err == nil || !strings.Contains(err.Error(), "permissions") {
|
|
||||||
t.Fatalf("Validate(unknown permission) error = %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
cfg.Admin.Auth = Auth{
|
|
||||||
Mode: "any",
|
|
||||||
Permissions: []string{authorization.AdminRead},
|
|
||||||
Methods: []AuthMethod{{
|
|
||||||
Mode: "bearer", Value: "read-token", Permissions: []string{authorization.AdminRead},
|
|
||||||
}},
|
|
||||||
}
|
|
||||||
if err := Validate(cfg); err == nil || !strings.Contains(err.Error(), "auth.permissions") {
|
|
||||||
t.Fatalf("Validate(any top-level permission) error = %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
cfg.Admin.Auth.Permissions = nil
|
|
||||||
if err := Validate(cfg); err != nil {
|
|
||||||
t.Fatalf("Validate(any method permission) error = %v", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestValidateDistributionCredentialClientPolicy(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
cfg, err := Load(strings.NewReader(validConfig))
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("Load() error = %v", err)
|
|
||||||
}
|
|
||||||
cfg.Distribution.Auth = Auth{
|
|
||||||
Mode: "bearer", Token: "distribution-token",
|
|
||||||
ClientPolicy: clientpolicy.Policy{
|
|
||||||
MaxExtractCount: 5,
|
|
||||||
AllowedUpstreams: []string{"provider-a"},
|
|
||||||
AllowedRegions: []string{"shanghai"},
|
|
||||||
},
|
|
||||||
}
|
|
||||||
if err := Validate(cfg); err != nil {
|
|
||||||
t.Fatalf("Validate(distribution client policy) error = %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
cfg.Distribution.Auth.ClientPolicy.AllowedUpstreams = []string{"missing"}
|
|
||||||
if err := Validate(cfg); err == nil || !strings.Contains(err.Error(), "missing") {
|
|
||||||
t.Fatalf("Validate(unknown allowed upstream) error = %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
cfg.Distribution.Auth.ClientPolicy = clientpolicy.Policy{}
|
|
||||||
cfg.Gateway.Auth = Auth{
|
|
||||||
Mode: "bearer", Token: "gateway-token",
|
|
||||||
ClientPolicy: clientpolicy.Policy{AllowedRoutings: []string{"extract"}},
|
|
||||||
}
|
|
||||||
cfg.Routing[0].Purpose = "gateway"
|
|
||||||
if err := Validate(cfg); err != nil {
|
|
||||||
t.Fatalf("Validate(gateway routing policy) error = %v", err)
|
|
||||||
}
|
|
||||||
cfg.Gateway.Auth.ClientPolicy.AllowedRoutings = []string{"missing"}
|
|
||||||
if err := Validate(cfg); err == nil || !strings.Contains(err.Error(), "missing") {
|
|
||||||
t.Fatalf("Validate(unknown gateway routing) error = %v", err)
|
|
||||||
}
|
|
||||||
cfg.Gateway.Auth.ClientPolicy = clientpolicy.Policy{MaxExtractCount: 1}
|
|
||||||
if err := Validate(cfg); err == nil || !strings.Contains(err.Error(), "only supported on distribution") {
|
|
||||||
t.Fatalf("Validate(gateway extraction policy) error = %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
cfg.Gateway.Auth.ClientPolicy = clientpolicy.Policy{RequestsPerMinute: 60}
|
|
||||||
if err := Validate(cfg); err != nil {
|
|
||||||
t.Fatalf("Validate(gateway request rate policy) error = %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
cfg.Gateway.Auth.ClientPolicy = clientpolicy.Policy{MaxConcurrentConnections: 3}
|
|
||||||
if err := Validate(cfg); err != nil {
|
|
||||||
t.Fatalf("Validate(gateway concurrency policy) error = %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
cfg.Gateway.Auth.ClientPolicy = clientpolicy.Policy{}
|
|
||||||
cfg.Distribution.Auth.ClientPolicy = clientpolicy.Policy{MaxConcurrentConnections: 3}
|
|
||||||
if err := Validate(cfg); err == nil || !strings.Contains(err.Error(), "only supported on gateway") {
|
|
||||||
t.Fatalf("Validate(distribution gateway limit policy) error = %v", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestValidateGatewayStickySession(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
cfg := mustLoadValidConfig(t)
|
|
||||||
cfg.Gateway.Auth = Auth{Mode: "bearer", Token: "gateway-token"}
|
|
||||||
cfg.Gateway.StickySession = StickySession{
|
|
||||||
Enabled: true, Header: "X-Proxy-Session", TTL: Duration(30 * time.Second), MaxEntries: 10_000,
|
|
||||||
}
|
|
||||||
cfg.Routing[0].Purpose = "gateway"
|
|
||||||
if err := Validate(cfg); err != nil {
|
|
||||||
t.Fatalf("Validate(valid gateway stickySession) error = %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
cfg.Gateway.StickySession.Header = ""
|
|
||||||
if err := Validate(cfg); err == nil || !strings.Contains(err.Error(), "stickySession.header") {
|
|
||||||
t.Fatalf("Validate(missing stickySession header) error = %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
cfg.Gateway.StickySession.Header = "X Proxy Session"
|
|
||||||
if err := Validate(cfg); err == nil || !strings.Contains(err.Error(), "stickySession.header") {
|
|
||||||
t.Fatalf("Validate(invalid stickySession header) error = %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
cfg.Gateway.StickySession.Header = "X-Proxy-Session"
|
|
||||||
cfg.Gateway.StickySession.TTL = 0
|
|
||||||
if err := Validate(cfg); err == nil || !strings.Contains(err.Error(), "stickySession.ttl") {
|
|
||||||
t.Fatalf("Validate(zero stickySession ttl) error = %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
cfg.Gateway.StickySession.TTL = Duration(30 * time.Second)
|
|
||||||
cfg.Gateway.Auth = Auth{Mode: "none"}
|
|
||||||
if err := Validate(cfg); err == nil || !strings.Contains(err.Error(), "requires authentication") {
|
|
||||||
t.Fatalf("Validate(unauthenticated stickySession) error = %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
cfg.Gateway.Auth = Auth{Mode: "bearer", Token: "gateway-token"}
|
|
||||||
cfg.Distribution.StickySession = StickySession{
|
|
||||||
Enabled: true, Header: "X-Proxy-Session", TTL: Duration(30 * time.Second), MaxEntries: 10,
|
|
||||||
}
|
|
||||||
if err := Validate(cfg); err == nil || !strings.Contains(err.Error(), "only supported on gateway") {
|
|
||||||
t.Fatalf("Validate(distribution stickySession) error = %v", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestValidateStaticGatewayRoutingAction(t *testing.T) {
|
|
||||||
cfg := mustLoadValidConfig(t)
|
|
||||||
cfg.Routing[0] = Routing{
|
|
||||||
Name: "direct-api", Enabled: true, Purpose: "gateway", Action: "direct",
|
|
||||||
Match: RoutingMatch{HostRegex: "^api\\.example\\.test$"},
|
|
||||||
}
|
|
||||||
if err := Validate(cfg); err != nil {
|
|
||||||
t.Fatalf("Validate(static direct routing) = %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
cfg.Routing[0].Purpose = "extract"
|
|
||||||
if err := Validate(cfg); err == nil || !strings.Contains(err.Error(), "only supported on gateway") {
|
|
||||||
t.Fatalf("Validate(extract direct routing) error = %v", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestValidateGatewayTransport(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
cfg := mustLoadValidConfig(t)
|
|
||||||
cfg.Gateway.Transport = GatewayTransport{
|
|
||||||
DialTimeout: Duration(time.Second),
|
|
||||||
HandshakeTimeout: Duration(2 * time.Second),
|
|
||||||
ResponseHeaderTimeout: Duration(3 * time.Second),
|
|
||||||
IdleConnTimeout: Duration(time.Minute),
|
|
||||||
MaxIdleConns: 500,
|
|
||||||
MaxIdleConnsPerHost: 20,
|
|
||||||
MaxConnsPerHost: 10,
|
|
||||||
TunnelBufferBytes: 32 << 10,
|
|
||||||
TunnelIdleTimeout: Duration(5 * time.Minute),
|
|
||||||
}
|
|
||||||
if err := Validate(cfg); err != nil {
|
|
||||||
t.Fatalf("Validate(valid gateway transport) error = %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
cfg.Gateway.Transport.DialTimeout = Duration(-time.Second)
|
|
||||||
if err := Validate(cfg); err == nil || !strings.Contains(err.Error(), "transport.dialTimeout") {
|
|
||||||
t.Fatalf("Validate(negative dial timeout) error = %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
cfg.Gateway.Transport.DialTimeout = Duration(time.Second)
|
|
||||||
cfg.Gateway.Transport.MaxConnsPerHost = -1
|
|
||||||
if err := Validate(cfg); err == nil || !strings.Contains(err.Error(), "transport.maxConnsPerHost") {
|
|
||||||
t.Fatalf("Validate(negative max connections) error = %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
cfg.Gateway.Transport.MaxConnsPerHost = 10
|
|
||||||
cfg.Gateway.Transport.MaxIdleConns = 19
|
|
||||||
cfg.Gateway.Transport.MaxIdleConnsPerHost = 20
|
|
||||||
if err := Validate(cfg); err == nil || !strings.Contains(err.Error(), "maxIdleConnsPerHost") {
|
|
||||||
t.Fatalf("Validate(inconsistent idle pool) error = %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
cfg.Gateway.Transport.MaxIdleConns = 500
|
|
||||||
cfg.Gateway.Transport.MaxIdleConnsPerHost = 20
|
|
||||||
cfg.Distribution.Transport = GatewayTransport{MaxConnsPerHost: 10}
|
|
||||||
if err := Validate(cfg); err == nil || !strings.Contains(err.Error(), "distribution transport") {
|
|
||||||
t.Fatalf("Validate(distribution transport) error = %v", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestValidateMetricsListener(t *testing.T) {
|
func TestValidateMetricsListener(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
cfg := mustLoadValidConfig(t)
|
cfg := mustLoadValidConfig(t)
|
||||||
|
|||||||
@ -80,13 +80,9 @@ func cloneListener(source Listener) Listener {
|
|||||||
cloned.Access.AllowCIDRs = cloneStrings(source.Access.AllowCIDRs)
|
cloned.Access.AllowCIDRs = cloneStrings(source.Access.AllowCIDRs)
|
||||||
cloned.Access.TrustedProxies = cloneStrings(source.Access.TrustedProxies)
|
cloned.Access.TrustedProxies = cloneStrings(source.Access.TrustedProxies)
|
||||||
cloned.Auth.CIDRs = cloneStrings(source.Auth.CIDRs)
|
cloned.Auth.CIDRs = cloneStrings(source.Auth.CIDRs)
|
||||||
cloned.Auth.Permissions = cloneStrings(source.Auth.Permissions)
|
|
||||||
cloned.Auth.ClientPolicy = source.Auth.ClientPolicy.Clone()
|
|
||||||
cloned.Auth.Methods = append([]AuthMethod(nil), source.Auth.Methods...)
|
cloned.Auth.Methods = append([]AuthMethod(nil), source.Auth.Methods...)
|
||||||
for index := range cloned.Auth.Methods {
|
for index := range cloned.Auth.Methods {
|
||||||
cloned.Auth.Methods[index].CIDRs = cloneStrings(source.Auth.Methods[index].CIDRs)
|
cloned.Auth.Methods[index].CIDRs = cloneStrings(source.Auth.Methods[index].CIDRs)
|
||||||
cloned.Auth.Methods[index].Permissions = cloneStrings(source.Auth.Methods[index].Permissions)
|
|
||||||
cloned.Auth.Methods[index].ClientPolicy = source.Auth.Methods[index].ClientPolicy.Clone()
|
|
||||||
}
|
}
|
||||||
cloned.Retry.RetryMethods = cloneStrings(source.Retry.RetryMethods)
|
cloned.Retry.RetryMethods = cloneStrings(source.Retry.RetryMethods)
|
||||||
cloned.DestinationPolicy.DenyPrivateNetworks = cloneBool(source.DestinationPolicy.DenyPrivateNetworks)
|
cloned.DestinationPolicy.DenyPrivateNetworks = cloneBool(source.DestinationPolicy.DenyPrivateNetworks)
|
||||||
|
|||||||
@ -4,14 +4,10 @@ import (
|
|||||||
"fmt"
|
"fmt"
|
||||||
"math"
|
"math"
|
||||||
"net"
|
"net"
|
||||||
"net/http"
|
|
||||||
"net/url"
|
"net/url"
|
||||||
"regexp"
|
"regexp"
|
||||||
"strconv"
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
"proxy-pool/internal/domain/authorization"
|
|
||||||
"proxy-pool/internal/domain/clientpolicy"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
var (
|
var (
|
||||||
@ -39,11 +35,6 @@ func Validate(cfg *Config) error {
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if cfg.Admin.Enabled {
|
|
||||||
if err := validateNoClientPolicy("admin", cfg.Admin.Auth); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if cfg.Metrics.Enabled {
|
if cfg.Metrics.Enabled {
|
||||||
if cfg.Metrics.Listen == "" {
|
if cfg.Metrics.Listen == "" {
|
||||||
return fmt.Errorf("validate metrics listen: address is required")
|
return fmt.Errorf("validate metrics listen: address is required")
|
||||||
@ -83,15 +74,11 @@ func Validate(cfg *Config) error {
|
|||||||
return fmt.Errorf("validate configuration: at least one enabled upstream is required")
|
return fmt.Errorf("validate configuration: at least one enabled upstream is required")
|
||||||
}
|
}
|
||||||
seen := make(map[string]struct{}, len(cfg.Routing))
|
seen := make(map[string]struct{}, len(cfg.Routing))
|
||||||
gatewayRoutings := make(map[string]struct{}, len(cfg.Routing))
|
|
||||||
targetProfilesByUpstream := make(map[string]int, len(cfg.Upstreams))
|
targetProfilesByUpstream := make(map[string]int, len(cfg.Upstreams))
|
||||||
for index, route := range cfg.Routing {
|
for index, route := range cfg.Routing {
|
||||||
if err := validateRouting(index, route, cfg.Upstreams, seen); err != nil {
|
if err := validateRouting(index, route, cfg.Upstreams, seen); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
if route.Enabled && route.Purpose == "gateway" {
|
|
||||||
gatewayRoutings[route.Name] = struct{}{}
|
|
||||||
}
|
|
||||||
if !route.Enabled || len(route.Check.Targets) == 0 {
|
if !route.Enabled || len(route.Check.Targets) == 0 {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
@ -106,27 +93,7 @@ func Validate(cfg *Config) error {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if cfg.Gateway.Enabled {
|
|
||||||
if err := validateGatewayClientPolicies(cfg.Gateway.Auth, gatewayRoutings); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if err := validateGatewayStickySession(cfg.Gateway); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if err := validateGatewayTransport(cfg.Gateway); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if cfg.Distribution.Enabled {
|
if cfg.Distribution.Enabled {
|
||||||
if err := validateDistributionClientPolicies(cfg.Distribution.Auth, cfg.Upstreams); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if cfg.Distribution.StickySession.Enabled {
|
|
||||||
return fmt.Errorf("validate distribution stickySession: only supported on gateway")
|
|
||||||
}
|
|
||||||
if gatewayTransportConfigured(cfg.Distribution.Transport) {
|
|
||||||
return fmt.Errorf("validate distribution transport: only supported on gateway")
|
|
||||||
}
|
|
||||||
clientIdentificationMode := cfg.Distribution.ClientIdentification.Mode
|
clientIdentificationMode := cfg.Distribution.ClientIdentification.Mode
|
||||||
if clientIdentificationMode == "" {
|
if clientIdentificationMode == "" {
|
||||||
clientIdentificationMode = "sourceIP"
|
clientIdentificationMode = "sourceIP"
|
||||||
@ -157,97 +124,9 @@ func Validate(cfg *Config) error {
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if cfg.Admin.Enabled && cfg.Admin.StickySession.Enabled {
|
|
||||||
return fmt.Errorf("validate admin stickySession: only supported on gateway")
|
|
||||||
}
|
|
||||||
if cfg.Admin.Enabled && gatewayTransportConfigured(cfg.Admin.Transport) {
|
|
||||||
return fmt.Errorf("validate admin transport: only supported on gateway")
|
|
||||||
}
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func validateGatewayStickySession(listener Listener) error {
|
|
||||||
item := listener.StickySession
|
|
||||||
if !item.Enabled {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
if listener.Auth.Mode == "" || listener.Auth.Mode == "none" {
|
|
||||||
return fmt.Errorf("validate gateway stickySession: requires authentication")
|
|
||||||
}
|
|
||||||
if !validCanonicalHeaderName(item.Header) {
|
|
||||||
return fmt.Errorf("validate gateway stickySession.header: must be a canonical HTTP header name")
|
|
||||||
}
|
|
||||||
if item.TTL.Value() <= 0 {
|
|
||||||
return fmt.Errorf("validate gateway stickySession.ttl: must be greater than zero")
|
|
||||||
}
|
|
||||||
if item.MaxEntries <= 0 || item.MaxEntries > MaximumPoolSize {
|
|
||||||
return fmt.Errorf("validate gateway stickySession.maxEntries: must be in [1, %d]", MaximumPoolSize)
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func validateGatewayTransport(listener Listener) error {
|
|
||||||
item := listener.Transport
|
|
||||||
for _, timeout := range []struct {
|
|
||||||
name string
|
|
||||||
value Duration
|
|
||||||
}{
|
|
||||||
{name: "dialTimeout", value: item.DialTimeout},
|
|
||||||
{name: "handshakeTimeout", value: item.HandshakeTimeout},
|
|
||||||
{name: "responseHeaderTimeout", value: item.ResponseHeaderTimeout},
|
|
||||||
{name: "idleConnTimeout", value: item.IdleConnTimeout},
|
|
||||||
{name: "tunnelIdleTimeout", value: item.TunnelIdleTimeout},
|
|
||||||
} {
|
|
||||||
if timeout.value.Value() < 0 {
|
|
||||||
return fmt.Errorf("validate gateway transport.%s: must be non-negative", timeout.name)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
for _, limit := range []struct {
|
|
||||||
name string
|
|
||||||
value int
|
|
||||||
}{
|
|
||||||
{name: "maxIdleConns", value: item.MaxIdleConns},
|
|
||||||
{name: "maxIdleConnsPerHost", value: item.MaxIdleConnsPerHost},
|
|
||||||
{name: "maxConnsPerHost", value: item.MaxConnsPerHost},
|
|
||||||
{name: "tunnelBufferBytes", value: item.TunnelBufferBytes},
|
|
||||||
} {
|
|
||||||
if limit.value < 0 || limit.value > MaximumPoolSize {
|
|
||||||
return fmt.Errorf("validate gateway transport.%s: must be in [0, %d]", limit.name, MaximumPoolSize)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if item.MaxIdleConns > 0 && item.MaxIdleConnsPerHost > item.MaxIdleConns {
|
|
||||||
return fmt.Errorf("validate gateway transport.maxIdleConnsPerHost: must not exceed maxIdleConns")
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func gatewayTransportConfigured(item GatewayTransport) bool {
|
|
||||||
return item.DialTimeout != 0 ||
|
|
||||||
item.HandshakeTimeout != 0 ||
|
|
||||||
item.ResponseHeaderTimeout != 0 ||
|
|
||||||
item.IdleConnTimeout != 0 ||
|
|
||||||
item.MaxIdleConns != 0 ||
|
|
||||||
item.MaxIdleConnsPerHost != 0 ||
|
|
||||||
item.MaxConnsPerHost != 0 ||
|
|
||||||
item.TunnelBufferBytes != 0 ||
|
|
||||||
item.TunnelIdleTimeout != 0
|
|
||||||
}
|
|
||||||
|
|
||||||
func validCanonicalHeaderName(value string) bool {
|
|
||||||
if value == "" || http.CanonicalHeaderKey(value) != value {
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
for _, character := range []byte(value) {
|
|
||||||
if character >= 'a' && character <= 'z' || character >= 'A' && character <= 'Z' ||
|
|
||||||
character >= '0' && character <= '9' || strings.ContainsRune("!#$%&'*+-.^_|~", rune(character)) ||
|
|
||||||
character == 96 {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
|
|
||||||
func validateControlPlane(item ControlPlane) error {
|
func validateControlPlane(item ControlPlane) error {
|
||||||
if err := validateClientTLS("gatewayTLS", item.GatewayTLS); err != nil {
|
if err := validateClientTLS("gatewayTLS", item.GatewayTLS); err != nil {
|
||||||
return err
|
return err
|
||||||
@ -423,28 +302,6 @@ func validateRouting(index int, route Routing, upstreams map[string]Upstream, se
|
|||||||
return fmt.Errorf("validate %s pathRegex: %w", scope, err)
|
return fmt.Errorf("validate %s pathRegex: %w", scope, err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
action := routingAction(route.Action)
|
|
||||||
if err := validateEnum(scope+" action", action, "proxy", "direct", "reject"); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if action != "proxy" {
|
|
||||||
if route.Purpose != "gateway" {
|
|
||||||
return fmt.Errorf("validate %s action: only supported on gateway", scope)
|
|
||||||
}
|
|
||||||
if len(route.Upstreams) != 0 {
|
|
||||||
return fmt.Errorf("validate %s action: direct and reject routes must not reference upstreams", scope)
|
|
||||||
}
|
|
||||||
if routingStrategyConfigured(route.Strategy) {
|
|
||||||
return fmt.Errorf("validate %s action: direct and reject routes must not configure strategy", scope)
|
|
||||||
}
|
|
||||||
if route.OnUnavailable.Action != "" || route.OnUnavailable.WaitTimeout != 0 {
|
|
||||||
return fmt.Errorf("validate %s action: direct and reject routes must not configure onUnavailable", scope)
|
|
||||||
}
|
|
||||||
if len(route.Check.Targets) != 0 {
|
|
||||||
return fmt.Errorf("validate %s action: direct and reject routes must not configure check targets", scope)
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
for _, upstream := range route.Upstreams {
|
for _, upstream := range route.Upstreams {
|
||||||
if _, ok := upstreams[upstream]; !ok {
|
if _, ok := upstreams[upstream]; !ok {
|
||||||
return fmt.Errorf("validate %s: upstream %q does not exist", scope, upstream)
|
return fmt.Errorf("validate %s: upstream %q does not exist", scope, upstream)
|
||||||
@ -468,20 +325,6 @@ func validateRouting(index int, route Routing, upstreams map[string]Upstream, se
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func routingAction(value string) string {
|
|
||||||
if value == "" {
|
|
||||||
return "proxy"
|
|
||||||
}
|
|
||||||
return value
|
|
||||||
}
|
|
||||||
|
|
||||||
func routingStrategyConfigured(strategy Strategy) bool {
|
|
||||||
return strategy.Type != "" ||
|
|
||||||
strategy.SwitchAfterEmptyFetch != 0 ||
|
|
||||||
strategy.EndBehavior != "" ||
|
|
||||||
len(strategy.Weights) != 0
|
|
||||||
}
|
|
||||||
|
|
||||||
func validateStrategy(scope string, upstreams []string, strategy Strategy) error {
|
func validateStrategy(scope string, upstreams []string, strategy Strategy) error {
|
||||||
if err := validateEnum(scope+" strategy.type", strategy.Type,
|
if err := validateEnum(scope+" strategy.type", strategy.Type,
|
||||||
"sequential", "random", "roundRobin", "weighted", "leastConnections"); err != nil {
|
"sequential", "random", "roundRobin", "weighted", "leastConnections"); err != nil {
|
||||||
@ -529,20 +372,8 @@ func validateStrategy(scope string, upstreams []string, strategy Strategy) error
|
|||||||
}
|
}
|
||||||
|
|
||||||
func validateListenerAuth(listener string, auth Auth) error {
|
func validateListenerAuth(listener string, auth Auth) error {
|
||||||
if err := authorization.Validate(auth.Permissions); err != nil {
|
|
||||||
return fmt.Errorf("validate %s auth.permissions: %w", listener, err)
|
|
||||||
}
|
|
||||||
if err := auth.ClientPolicy.Validate(); err != nil {
|
|
||||||
return fmt.Errorf("validate %s auth.client: %w", listener, err)
|
|
||||||
}
|
|
||||||
switch auth.Mode {
|
switch auth.Mode {
|
||||||
case "none":
|
case "none":
|
||||||
if len(auth.Permissions) != 0 {
|
|
||||||
return fmt.Errorf("validate %s auth.permissions: requires an authentication mode", listener)
|
|
||||||
}
|
|
||||||
if !auth.ClientPolicy.IsZero() {
|
|
||||||
return fmt.Errorf("validate %s auth.client: requires an authentication mode", listener)
|
|
||||||
}
|
|
||||||
return nil
|
return nil
|
||||||
case "usernamePassword":
|
case "usernamePassword":
|
||||||
if auth.Username == "" || (auth.Password == "" && auth.PasswordFile == "") {
|
if auth.Username == "" || (auth.Password == "" && auth.PasswordFile == "") {
|
||||||
@ -564,12 +395,6 @@ func validateListenerAuth(listener string, auth Auth) error {
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
case "any":
|
case "any":
|
||||||
if len(auth.Permissions) != 0 {
|
|
||||||
return fmt.Errorf("validate %s auth.permissions: configure permissions on auth.methods", listener)
|
|
||||||
}
|
|
||||||
if !auth.ClientPolicy.IsZero() {
|
|
||||||
return fmt.Errorf("validate %s auth.client: configure client policy on auth.methods", listener)
|
|
||||||
}
|
|
||||||
if len(auth.Methods) == 0 {
|
if len(auth.Methods) == 0 {
|
||||||
return fmt.Errorf("validate %s auth.mode any: methods are required", listener)
|
return fmt.Errorf("validate %s auth.mode any: methods are required", listener)
|
||||||
}
|
}
|
||||||
@ -585,12 +410,6 @@ func validateListenerAuth(listener string, auth Auth) error {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func validateAuthMethod(listener string, index int, method AuthMethod) error {
|
func validateAuthMethod(listener string, index int, method AuthMethod) error {
|
||||||
if err := authorization.Validate(method.Permissions); err != nil {
|
|
||||||
return fmt.Errorf("validate %s auth.methods[%d].permissions: %w", listener, index, err)
|
|
||||||
}
|
|
||||||
if err := method.ClientPolicy.Validate(); err != nil {
|
|
||||||
return fmt.Errorf("validate %s auth.methods[%d].client: %w", listener, index, err)
|
|
||||||
}
|
|
||||||
switch method.Mode {
|
switch method.Mode {
|
||||||
case "usernamePassword":
|
case "usernamePassword":
|
||||||
if method.Username == "" || (method.Password == "" && method.PasswordFile == "") {
|
if method.Username == "" || (method.Password == "" && method.PasswordFile == "") {
|
||||||
@ -617,76 +436,6 @@ func validateAuthMethod(listener string, index int, method AuthMethod) error {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func validateNoClientPolicy(listener string, auth Auth) error {
|
|
||||||
if !auth.ClientPolicy.IsZero() {
|
|
||||||
return fmt.Errorf("validate %s auth.client: client extraction policy is only supported on distribution", listener)
|
|
||||||
}
|
|
||||||
for index, method := range auth.Methods {
|
|
||||||
if !method.ClientPolicy.IsZero() {
|
|
||||||
return fmt.Errorf("validate %s auth.methods[%d].client: client extraction policy is only supported on distribution", listener, index)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func validateDistributionClientPolicies(auth Auth, upstreams map[string]Upstream) error {
|
|
||||||
if err := validateDistributionClientPolicy("distribution auth.client", auth.ClientPolicy, upstreams); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
for index, method := range auth.Methods {
|
|
||||||
if err := validateDistributionClientPolicy(
|
|
||||||
fmt.Sprintf("distribution auth.methods[%d].client", index), method.ClientPolicy, upstreams,
|
|
||||||
); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func validateDistributionClientPolicy(scope string, policy clientpolicy.Policy, upstreams map[string]Upstream) error {
|
|
||||||
if len(policy.AllowedRoutings) != 0 || policy.RequestsPerMinute != 0 || policy.MaxConcurrentConnections != 0 {
|
|
||||||
return fmt.Errorf("validate %s: routing access and Gateway limits are only supported on gateway", scope)
|
|
||||||
}
|
|
||||||
for _, upstream := range policy.AllowedUpstreams {
|
|
||||||
if _, exists := upstreams[upstream]; !exists {
|
|
||||||
return fmt.Errorf("validate %s.allowedUpstreams: upstream %q does not exist", scope, upstream)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func validateGatewayClientPolicies(auth Auth, routings map[string]struct{}) error {
|
|
||||||
if err := validateGatewayClientPolicy("gateway auth.client", auth.ClientPolicy, routings); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
for index, method := range auth.Methods {
|
|
||||||
if err := validateGatewayClientPolicy(
|
|
||||||
fmt.Sprintf("gateway auth.methods[%d].client", index), method.ClientPolicy, routings,
|
|
||||||
); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func validateGatewayClientPolicy(scope string, policy clientpolicy.Policy, routings map[string]struct{}) error {
|
|
||||||
if policy.MaxExtractCount != 0 || len(policy.AllowedUpstreams) != 0 || len(policy.AllowedRegions) != 0 {
|
|
||||||
return fmt.Errorf("validate %s: extraction constraints are only supported on distribution", scope)
|
|
||||||
}
|
|
||||||
if int64(policy.RequestsPerMinute) > MaximumExactCounter {
|
|
||||||
return fmt.Errorf("validate %s.requestsPerMinute: exceeds exact counter range", scope)
|
|
||||||
}
|
|
||||||
if int64(policy.MaxConcurrentConnections) > MaximumExactCounter {
|
|
||||||
return fmt.Errorf("validate %s.maxConcurrentConnections: exceeds exact counter range", scope)
|
|
||||||
}
|
|
||||||
for _, routing := range policy.AllowedRoutings {
|
|
||||||
if _, exists := routings[routing]; !exists {
|
|
||||||
return fmt.Errorf("validate %s.allowedRoutings: routing %q does not exist or is not an enabled gateway routing", scope, routing)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func validateCIDRs(name string, cidrs []string) error {
|
func validateCIDRs(name string, cidrs []string) error {
|
||||||
for _, cidr := range cidrs {
|
for _, cidr := range cidrs {
|
||||||
if _, _, err := net.ParseCIDR(cidr); err != nil {
|
if _, _, err := net.ParseCIDR(cidr); err != nil {
|
||||||
|
|||||||
@ -5,24 +5,18 @@ import (
|
|||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"net/http"
|
"net/http"
|
||||||
"strconv"
|
|
||||||
"strings"
|
"strings"
|
||||||
"time"
|
|
||||||
|
|
||||||
"proxy-pool/internal/domain/adminstate"
|
|
||||||
"proxy-pool/internal/domain/authorization"
|
|
||||||
"proxy-pool/internal/platform/httpapi"
|
"proxy-pool/internal/platform/httpapi"
|
||||||
"proxy-pool/internal/platform/httpsecurity"
|
"proxy-pool/internal/platform/httpsecurity"
|
||||||
)
|
)
|
||||||
|
|
||||||
const (
|
const (
|
||||||
statusPath = "/api/v1/status"
|
statusPath = "/api/v1/status"
|
||||||
auditPath = "/api/v1/audit"
|
|
||||||
reloadPath = "/api/v1/config/reload"
|
reloadPath = "/api/v1/config/reload"
|
||||||
upstreamPrefix = "/api/v1/upstreams/"
|
upstreamPrefix = "/api/v1/upstreams/"
|
||||||
routingPrefix = "/api/v1/routing/"
|
routingPrefix = "/api/v1/routing/"
|
||||||
maxResourceNameBytes = 128
|
maxResourceNameBytes = 128
|
||||||
defaultAuditPageSize = 100
|
|
||||||
)
|
)
|
||||||
|
|
||||||
var (
|
var (
|
||||||
@ -35,7 +29,6 @@ var (
|
|||||||
|
|
||||||
type Service interface {
|
type Service interface {
|
||||||
Status(context.Context) (Status, error)
|
Status(context.Context) (Status, error)
|
||||||
ReadAudit(context.Context, AuditQuery) (AuditPage, error)
|
|
||||||
SetUpstreamEnabled(context.Context, SetUpstreamCommand) (MutationResult, error)
|
SetUpstreamEnabled(context.Context, SetUpstreamCommand) (MutationResult, error)
|
||||||
SwitchRouting(context.Context, SwitchCommand) (MutationResult, error)
|
SwitchRouting(context.Context, SwitchCommand) (MutationResult, error)
|
||||||
ReloadConfiguration(context.Context, ReloadCommand) (MutationResult, error)
|
ReloadConfiguration(context.Context, ReloadCommand) (MutationResult, error)
|
||||||
@ -85,36 +78,6 @@ type MutationResult struct {
|
|||||||
Message string `json:"message,omitempty"`
|
Message string `json:"message,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
type AuditQuery struct {
|
|
||||||
AfterID uint64
|
|
||||||
Limit int
|
|
||||||
}
|
|
||||||
|
|
||||||
func (query AuditQuery) Validate() error {
|
|
||||||
if query.Limit <= 0 || query.Limit > adminstate.MaxPageSize {
|
|
||||||
return ErrInvalidConfiguration
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
type AuditRecord struct {
|
|
||||||
ID uint64 `json:"id"`
|
|
||||||
RequestID string `json:"requestId"`
|
|
||||||
ActorID string `json:"actorId"`
|
|
||||||
SourceIP string `json:"sourceIp,omitempty"`
|
|
||||||
Action string `json:"action"`
|
|
||||||
ResourceType string `json:"resourceType"`
|
|
||||||
ResourceName string `json:"resourceName"`
|
|
||||||
Changed bool `json:"changed"`
|
|
||||||
Version uint64 `json:"version"`
|
|
||||||
Reason string `json:"reason,omitempty"`
|
|
||||||
OccurredAt time.Time `json:"occurredAt"`
|
|
||||||
}
|
|
||||||
|
|
||||||
type AuditPage struct {
|
|
||||||
Records []AuditRecord `json:"records"`
|
|
||||||
}
|
|
||||||
|
|
||||||
type SetUpstreamCommand struct {
|
type SetUpstreamCommand struct {
|
||||||
RequestID string
|
RequestID string
|
||||||
ActorID string
|
ActorID string
|
||||||
@ -171,27 +134,12 @@ func (handler *Handler) ServeHTTP(writer http.ResponseWriter, request *http.Requ
|
|||||||
if !requireMethod(writer, request, http.MethodGet, requestID) {
|
if !requireMethod(writer, request, http.MethodGet, requestID) {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
if !requirePermission(writer, identity, authorization.AdminRead, requestID) {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
handler.getStatus(writer, request, requestID)
|
handler.getStatus(writer, request, requestID)
|
||||||
return
|
return
|
||||||
case auditPath:
|
|
||||||
if !requireMethod(writer, request, http.MethodGet, requestID) {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if !requirePermission(writer, identity, authorization.AdminRead, requestID) {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
handler.readAudit(writer, request, requestID)
|
|
||||||
return
|
|
||||||
case reloadPath:
|
case reloadPath:
|
||||||
if !requireMethod(writer, request, http.MethodPost, requestID) {
|
if !requireMethod(writer, request, http.MethodPost, requestID) {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
if !requirePermission(writer, identity, authorization.AdminWrite, requestID) {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
handler.reload(writer, request, requestID, identity)
|
handler.reload(writer, request, requestID, identity)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
@ -200,9 +148,6 @@ func (handler *Handler) ServeHTTP(writer http.ResponseWriter, request *http.Requ
|
|||||||
if !requireMethod(writer, request, http.MethodPost, requestID) {
|
if !requireMethod(writer, request, http.MethodPost, requestID) {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
if !requirePermission(writer, identity, authorization.AdminWrite, requestID) {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
handler.setUpstreamEnabled(writer, request, name, action == "enable", requestID, identity)
|
handler.setUpstreamEnabled(writer, request, name, action == "enable", requestID, identity)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
@ -210,9 +155,6 @@ func (handler *Handler) ServeHTTP(writer http.ResponseWriter, request *http.Requ
|
|||||||
if !requireMethod(writer, request, http.MethodPost, requestID) {
|
if !requireMethod(writer, request, http.MethodPost, requestID) {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
if !requirePermission(writer, identity, authorization.AdminWrite, requestID) {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
handler.switchRouting(writer, request, name, requestID, identity)
|
handler.switchRouting(writer, request, name, requestID, identity)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
@ -220,19 +162,6 @@ func (handler *Handler) ServeHTTP(writer http.ResponseWriter, request *http.Requ
|
|||||||
writeTransportProblem(writer, http.StatusNotFound, "NOT_FOUND", "Not found", "the requested endpoint does not exist", requestID)
|
writeTransportProblem(writer, http.StatusNotFound, "NOT_FOUND", "Not found", "the requested endpoint does not exist", requestID)
|
||||||
}
|
}
|
||||||
|
|
||||||
func requirePermission(
|
|
||||||
writer http.ResponseWriter,
|
|
||||||
identity httpsecurity.Identity,
|
|
||||||
required string,
|
|
||||||
requestID string,
|
|
||||||
) bool {
|
|
||||||
if identity.Allows(required) {
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
writeTransportProblem(writer, http.StatusForbidden, "FORBIDDEN", "Forbidden", "the authenticated identity lacks endpoint permission", requestID)
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
|
|
||||||
func (handler *Handler) getStatus(writer http.ResponseWriter, request *http.Request, requestID string) {
|
func (handler *Handler) getStatus(writer http.ResponseWriter, request *http.Request, requestID string) {
|
||||||
status, err := handler.service.Status(request.Context())
|
status, err := handler.service.Status(request.Context())
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@ -249,57 +178,6 @@ func (handler *Handler) getStatus(writer http.ResponseWriter, request *http.Requ
|
|||||||
_ = httpapi.WriteJSON(writer, http.StatusOK, status)
|
_ = httpapi.WriteJSON(writer, http.StatusOK, status)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (handler *Handler) readAudit(writer http.ResponseWriter, request *http.Request, requestID string) {
|
|
||||||
query, err := parseAuditQuery(request)
|
|
||||||
if err != nil {
|
|
||||||
writeTransportProblem(writer, http.StatusBadRequest, "INVALID_AUDIT_QUERY", "Invalid audit query", "audit pagination fields violate the API contract", requestID)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
page, err := handler.service.ReadAudit(request.Context(), query)
|
|
||||||
if err != nil {
|
|
||||||
writeServiceProblem(writer, err, requestID)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if page.Records == nil {
|
|
||||||
page.Records = []AuditRecord{}
|
|
||||||
}
|
|
||||||
writer.Header().Set(httpapi.HeaderRequestID, requestID)
|
|
||||||
_ = httpapi.WriteJSON(writer, http.StatusOK, page)
|
|
||||||
}
|
|
||||||
|
|
||||||
func parseAuditQuery(request *http.Request) (AuditQuery, error) {
|
|
||||||
if request == nil || request.URL == nil {
|
|
||||||
return AuditQuery{}, ErrInvalidConfiguration
|
|
||||||
}
|
|
||||||
values := request.URL.Query()
|
|
||||||
query := AuditQuery{Limit: defaultAuditPageSize}
|
|
||||||
for name, value := range values {
|
|
||||||
if len(value) != 1 {
|
|
||||||
return AuditQuery{}, ErrInvalidConfiguration
|
|
||||||
}
|
|
||||||
switch name {
|
|
||||||
case "afterId":
|
|
||||||
parsed, err := strconv.ParseUint(value[0], 10, 64)
|
|
||||||
if err != nil {
|
|
||||||
return AuditQuery{}, ErrInvalidConfiguration
|
|
||||||
}
|
|
||||||
query.AfterID = parsed
|
|
||||||
case "limit":
|
|
||||||
parsed, err := strconv.ParseUint(value[0], 10, 32)
|
|
||||||
if err != nil || parsed == 0 || parsed > adminstate.MaxPageSize {
|
|
||||||
return AuditQuery{}, ErrInvalidConfiguration
|
|
||||||
}
|
|
||||||
query.Limit = int(parsed)
|
|
||||||
default:
|
|
||||||
return AuditQuery{}, ErrInvalidConfiguration
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if err := query.Validate(); err != nil {
|
|
||||||
return AuditQuery{}, err
|
|
||||||
}
|
|
||||||
return query, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (handler *Handler) setUpstreamEnabled(
|
func (handler *Handler) setUpstreamEnabled(
|
||||||
writer http.ResponseWriter,
|
writer http.ResponseWriter,
|
||||||
request *http.Request,
|
request *http.Request,
|
||||||
|
|||||||
@ -8,9 +8,7 @@ import (
|
|||||||
"net/http/httptest"
|
"net/http/httptest"
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
|
||||||
|
|
||||||
"proxy-pool/internal/domain/authorization"
|
|
||||||
"proxy-pool/internal/platform/httpapi"
|
"proxy-pool/internal/platform/httpapi"
|
||||||
"proxy-pool/internal/platform/httpsecurity"
|
"proxy-pool/internal/platform/httpsecurity"
|
||||||
)
|
)
|
||||||
@ -40,109 +38,6 @@ func TestHandlerReturnsStatusWithoutSensitiveDetails(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestHandlerEnforcesReadAndWritePermissions(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
service := &stubService{}
|
|
||||||
readHandler, err := NewHandler(service, scopedAuthorizer{permissions: []string{authorization.AdminRead}}, Options{MaxBodyBytes: 1024})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("NewHandler(read) error = %v", err)
|
|
||||||
}
|
|
||||||
status := httptest.NewRecorder()
|
|
||||||
readHandler.ServeHTTP(status, httptest.NewRequest(http.MethodGet, "/api/v1/status", nil))
|
|
||||||
if status.Code != http.StatusOK || service.statusCalls != 1 {
|
|
||||||
t.Fatalf("read status = %d calls=%d", status.Code, service.statusCalls)
|
|
||||||
}
|
|
||||||
mutation := httptest.NewRecorder()
|
|
||||||
readHandler.ServeHTTP(mutation, httptest.NewRequest(http.MethodPost, "/api/v1/config/reload", nil))
|
|
||||||
if mutation.Code != http.StatusForbidden || service.lastReload != (ReloadCommand{}) || !strings.Contains(mutation.Body.String(), `"code":"FORBIDDEN"`) {
|
|
||||||
t.Fatalf("read mutation response = status %d command %+v body=%s", mutation.Code, service.lastReload, mutation.Body.String())
|
|
||||||
}
|
|
||||||
|
|
||||||
writeHandler, err := NewHandler(service, scopedAuthorizer{permissions: []string{authorization.AdminWrite}}, Options{MaxBodyBytes: 1024})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("NewHandler(write) error = %v", err)
|
|
||||||
}
|
|
||||||
audit := httptest.NewRecorder()
|
|
||||||
writeHandler.ServeHTTP(audit, httptest.NewRequest(http.MethodGet, "/api/v1/audit", nil))
|
|
||||||
if audit.Code != http.StatusForbidden || service.auditCalls != 0 || !strings.Contains(audit.Body.String(), `"code":"FORBIDDEN"`) {
|
|
||||||
t.Fatalf("write audit response = status %d calls=%d body=%s", audit.Code, service.auditCalls, audit.Body.String())
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestHandlerReadsBoundedAuditPage(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
occurredAt := time.Date(2026, 8, 2, 9, 30, 0, 0, time.UTC)
|
|
||||||
service := &stubService{audit: AuditPage{Records: []AuditRecord{{
|
|
||||||
ID: 8, RequestID: "req-8", ActorID: "admin:alice", Action: "switch_routing",
|
|
||||||
ResourceType: "routing", ResourceName: "checkout", Changed: true, Version: 12,
|
|
||||||
Reason: "capacity", OccurredAt: occurredAt,
|
|
||||||
}}}}
|
|
||||||
handler := mustHandler(t, service)
|
|
||||||
recorder := httptest.NewRecorder()
|
|
||||||
request := httptest.NewRequest(http.MethodGet, "/api/v1/audit?afterId=7&limit=2", nil)
|
|
||||||
request.Header.Set(httpapi.HeaderRequestID, "req-audit-page")
|
|
||||||
|
|
||||||
handler.ServeHTTP(recorder, request)
|
|
||||||
|
|
||||||
if recorder.Code != http.StatusOK {
|
|
||||||
t.Fatalf("status = %d, want %d; body=%s", recorder.Code, http.StatusOK, recorder.Body.String())
|
|
||||||
}
|
|
||||||
if service.auditCalls != 1 || service.lastAudit != (AuditQuery{AfterID: 7, Limit: 2}) {
|
|
||||||
t.Fatalf("ReadAudit() calls=%d query=%+v", service.auditCalls, service.lastAudit)
|
|
||||||
}
|
|
||||||
if requestID := recorder.Header().Get(httpapi.HeaderRequestID); requestID != "req-audit-page" {
|
|
||||||
t.Fatalf("response request ID = %q, want req-audit-page", requestID)
|
|
||||||
}
|
|
||||||
var response AuditPage
|
|
||||||
if err := json.Unmarshal(recorder.Body.Bytes(), &response); err != nil {
|
|
||||||
t.Fatalf("decode response: %v", err)
|
|
||||||
}
|
|
||||||
if len(response.Records) != 1 || response.Records[0].ID != 8 || !response.Records[0].OccurredAt.Equal(occurredAt) {
|
|
||||||
t.Fatalf("unexpected audit response: %+v", response)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestHandlerUsesDefaultAuditLimitAndRejectsInvalidAuditQueries(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
tests := []struct {
|
|
||||||
name string
|
|
||||||
path string
|
|
||||||
wantStatus int
|
|
||||||
wantQuery AuditQuery
|
|
||||||
}{
|
|
||||||
{name: "default", path: "/api/v1/audit", wantStatus: http.StatusOK, wantQuery: AuditQuery{Limit: defaultAuditPageSize}},
|
|
||||||
{name: "unknown field", path: "/api/v1/audit?beforeId=1", wantStatus: http.StatusBadRequest},
|
|
||||||
{name: "duplicate field", path: "/api/v1/audit?limit=1&limit=2", wantStatus: http.StatusBadRequest},
|
|
||||||
{name: "zero limit", path: "/api/v1/audit?limit=0", wantStatus: http.StatusBadRequest},
|
|
||||||
{name: "oversized limit", path: "/api/v1/audit?limit=1001", wantStatus: http.StatusBadRequest},
|
|
||||||
{name: "invalid cursor", path: "/api/v1/audit?afterId=nope", wantStatus: http.StatusBadRequest},
|
|
||||||
}
|
|
||||||
for _, test := range tests {
|
|
||||||
test := test
|
|
||||||
t.Run(test.name, func(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
service := &stubService{}
|
|
||||||
handler := mustHandler(t, service)
|
|
||||||
recorder := httptest.NewRecorder()
|
|
||||||
|
|
||||||
handler.ServeHTTP(recorder, httptest.NewRequest(http.MethodGet, test.path, nil))
|
|
||||||
|
|
||||||
if recorder.Code != test.wantStatus {
|
|
||||||
t.Fatalf("status = %d, want %d; body=%s", recorder.Code, test.wantStatus, recorder.Body.String())
|
|
||||||
}
|
|
||||||
if test.wantStatus == http.StatusOK {
|
|
||||||
if service.auditCalls != 1 || service.lastAudit != test.wantQuery {
|
|
||||||
t.Fatalf("ReadAudit() calls=%d query=%+v, want %+v", service.auditCalls, service.lastAudit, test.wantQuery)
|
|
||||||
}
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if service.auditCalls != 0 || !strings.Contains(recorder.Body.String(), `"code":"INVALID_AUDIT_QUERY"`) {
|
|
||||||
t.Fatalf("invalid audit query reached service or returned wrong problem: calls=%d body=%s", service.auditCalls, recorder.Body.String())
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestNewHandlerRejectsMissingDependenciesAndInvalidLimit(t *testing.T) {
|
func TestNewHandlerRejectsMissingDependenciesAndInvalidLimit(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
if _, err := NewHandler(nil, allowAuthorizer{}, Options{MaxBodyBytes: 1024}); !errors.Is(err, ErrInvalidHandler) {
|
if _, err := NewHandler(nil, allowAuthorizer{}, Options{MaxBodyBytes: 1024}); !errors.Is(err, ErrInvalidHandler) {
|
||||||
@ -337,7 +232,6 @@ func TestHandlerRejectsInvalidTransportRequests(t *testing.T) {
|
|||||||
wantStatus int
|
wantStatus int
|
||||||
}{
|
}{
|
||||||
{name: "method", method: http.MethodPut, path: "/api/v1/config/reload", wantStatus: http.StatusMethodNotAllowed},
|
{name: "method", method: http.MethodPut, path: "/api/v1/config/reload", wantStatus: http.StatusMethodNotAllowed},
|
||||||
{name: "audit method", method: http.MethodPost, path: "/api/v1/audit", wantStatus: http.StatusMethodNotAllowed},
|
|
||||||
{name: "unknown route", method: http.MethodGet, path: "/missing", wantStatus: http.StatusNotFound},
|
{name: "unknown route", method: http.MethodGet, path: "/missing", wantStatus: http.StatusNotFound},
|
||||||
{name: "invalid name", method: http.MethodPost, path: "/api/v1/upstreams//enable", wantStatus: http.StatusNotFound},
|
{name: "invalid name", method: http.MethodPost, path: "/api/v1/upstreams//enable", wantStatus: http.StatusNotFound},
|
||||||
{name: "unknown JSON field", method: http.MethodPost, path: "/api/v1/routing/r/switch", body: `{"expectedCurrent":"a","target":"b","extra":1}`, content: httpapi.JSONContentType, wantStatus: http.StatusBadRequest},
|
{name: "unknown JSON field", method: http.MethodPost, path: "/api/v1/routing/r/switch", body: `{"expectedCurrent":"a","target":"b","extra":1}`, content: httpapi.JSONContentType, wantStatus: http.StatusBadRequest},
|
||||||
@ -383,15 +277,12 @@ func mustHandler(t *testing.T, service Service) *Handler {
|
|||||||
|
|
||||||
type stubService struct {
|
type stubService struct {
|
||||||
status Status
|
status Status
|
||||||
audit AuditPage
|
|
||||||
mutation MutationResult
|
mutation MutationResult
|
||||||
err error
|
err error
|
||||||
lastAudit AuditQuery
|
|
||||||
lastUpstream SetUpstreamCommand
|
lastUpstream SetUpstreamCommand
|
||||||
lastSwitch SwitchCommand
|
lastSwitch SwitchCommand
|
||||||
lastReload ReloadCommand
|
lastReload ReloadCommand
|
||||||
statusCalls int
|
statusCalls int
|
||||||
auditCalls int
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func (service *stubService) Status(context.Context) (Status, error) {
|
func (service *stubService) Status(context.Context) (Status, error) {
|
||||||
@ -399,12 +290,6 @@ func (service *stubService) Status(context.Context) (Status, error) {
|
|||||||
return service.status, service.err
|
return service.status, service.err
|
||||||
}
|
}
|
||||||
|
|
||||||
func (service *stubService) ReadAudit(_ context.Context, query AuditQuery) (AuditPage, error) {
|
|
||||||
service.auditCalls++
|
|
||||||
service.lastAudit = query
|
|
||||||
return service.audit, service.err
|
|
||||||
}
|
|
||||||
|
|
||||||
func (service *stubService) SetUpstreamEnabled(_ context.Context, command SetUpstreamCommand) (MutationResult, error) {
|
func (service *stubService) SetUpstreamEnabled(_ context.Context, command SetUpstreamCommand) (MutationResult, error) {
|
||||||
service.lastUpstream = command
|
service.lastUpstream = command
|
||||||
return service.mutation, service.err
|
return service.mutation, service.err
|
||||||
@ -426,12 +311,6 @@ func (allowAuthorizer) Resolve(*http.Request) (httpsecurity.Identity, error) {
|
|||||||
return httpsecurity.Identity{ClientID: "admin:test", SourceIP: "192.0.2.10"}, nil
|
return httpsecurity.Identity{ClientID: "admin:test", SourceIP: "192.0.2.10"}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
type scopedAuthorizer struct{ permissions []string }
|
|
||||||
|
|
||||||
func (authorizer scopedAuthorizer) Resolve(*http.Request) (httpsecurity.Identity, error) {
|
|
||||||
return httpsecurity.Identity{ClientID: "admin:scoped", SourceIP: "192.0.2.20", Permissions: authorizer.permissions}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
type rejectAuthorizer struct{}
|
type rejectAuthorizer struct{}
|
||||||
|
|
||||||
func (rejectAuthorizer) Resolve(*http.Request) (httpsecurity.Identity, error) {
|
func (rejectAuthorizer) Resolve(*http.Request) (httpsecurity.Identity, error) {
|
||||||
|
|||||||
@ -16,7 +16,6 @@ var ErrInvalidApplicationService = errors.New("invalid admin application service
|
|||||||
type StateRepository interface {
|
type StateRepository interface {
|
||||||
adminstate.Mutator
|
adminstate.Mutator
|
||||||
adminstate.SnapshotReader
|
adminstate.SnapshotReader
|
||||||
adminstate.AuditReader
|
|
||||||
}
|
}
|
||||||
|
|
||||||
type OperationalStatusReader interface {
|
type OperationalStatusReader interface {
|
||||||
@ -38,21 +37,14 @@ type RuntimeController interface {
|
|||||||
ValidateUpstream(context.Context, string) error
|
ValidateUpstream(context.Context, string) error
|
||||||
}
|
}
|
||||||
|
|
||||||
// SnapshotRefreshNotifier requests immediate full Worker snapshots after a
|
|
||||||
// committed management change. Implementations must coalesce notifications.
|
|
||||||
type SnapshotRefreshNotifier interface {
|
|
||||||
NotifySnapshotRefresh()
|
|
||||||
}
|
|
||||||
|
|
||||||
var _ ConfigurationPublisher = (*config.Store)(nil)
|
var _ ConfigurationPublisher = (*config.Store)(nil)
|
||||||
|
|
||||||
type ApplicationDependencies struct {
|
type ApplicationDependencies struct {
|
||||||
State StateRepository
|
State StateRepository
|
||||||
Operations OperationalStatusReader
|
Operations OperationalStatusReader
|
||||||
Configuration ConfigurationLoader
|
Configuration ConfigurationLoader
|
||||||
Publisher ConfigurationPublisher
|
Publisher ConfigurationPublisher
|
||||||
Runtime RuntimeController
|
Runtime RuntimeController
|
||||||
SnapshotRefresh SnapshotRefreshNotifier
|
|
||||||
}
|
}
|
||||||
|
|
||||||
type ApplicationOptions struct {
|
type ApplicationOptions struct {
|
||||||
@ -83,14 +75,13 @@ type LoadedConfiguration struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
type ApplicationService struct {
|
type ApplicationService struct {
|
||||||
state StateRepository
|
state StateRepository
|
||||||
operations OperationalStatusReader
|
operations OperationalStatusReader
|
||||||
configuration ConfigurationLoader
|
configuration ConfigurationLoader
|
||||||
publisher ConfigurationPublisher
|
publisher ConfigurationPublisher
|
||||||
runtime RuntimeController
|
runtime RuntimeController
|
||||||
snapshotRefresh SnapshotRefreshNotifier
|
now func() time.Time
|
||||||
now func() time.Time
|
fingerprintKey []byte
|
||||||
fingerprintKey []byte
|
|
||||||
}
|
}
|
||||||
|
|
||||||
var _ Service = (*ApplicationService)(nil)
|
var _ Service = (*ApplicationService)(nil)
|
||||||
@ -102,14 +93,13 @@ func NewApplicationService(dependencies ApplicationDependencies, options Applica
|
|||||||
return nil, ErrInvalidApplicationService
|
return nil, ErrInvalidApplicationService
|
||||||
}
|
}
|
||||||
return &ApplicationService{
|
return &ApplicationService{
|
||||||
state: dependencies.State,
|
state: dependencies.State,
|
||||||
operations: dependencies.Operations,
|
operations: dependencies.Operations,
|
||||||
configuration: dependencies.Configuration,
|
configuration: dependencies.Configuration,
|
||||||
publisher: dependencies.Publisher,
|
publisher: dependencies.Publisher,
|
||||||
runtime: dependencies.Runtime,
|
runtime: dependencies.Runtime,
|
||||||
snapshotRefresh: dependencies.SnapshotRefresh,
|
now: options.Now,
|
||||||
now: options.Now,
|
fingerprintKey: append([]byte(nil), options.FingerprintKey...),
|
||||||
fingerprintKey: append([]byte(nil), options.FingerprintKey...),
|
|
||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -129,9 +119,6 @@ func (service *ApplicationService) SetUpstreamEnabled(ctx context.Context, comma
|
|||||||
if err == nil && service.runtime != nil {
|
if err == nil && service.runtime != nil {
|
||||||
service.runtime.Notify()
|
service.runtime.Notify()
|
||||||
}
|
}
|
||||||
if err == nil && result.Changed && service.snapshotRefresh != nil {
|
|
||||||
service.snapshotRefresh.NotifySnapshotRefresh()
|
|
||||||
}
|
|
||||||
return mutationResult(result), mapAdminStateError(err)
|
return mutationResult(result), mapAdminStateError(err)
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -145,42 +132,9 @@ func (service *ApplicationService) SwitchRouting(ctx context.Context, command Sw
|
|||||||
Target: command.Target,
|
Target: command.Target,
|
||||||
Reason: command.Reason,
|
Reason: command.Reason,
|
||||||
})
|
})
|
||||||
if err == nil && result.Changed && service.snapshotRefresh != nil {
|
|
||||||
service.snapshotRefresh.NotifySnapshotRefresh()
|
|
||||||
}
|
|
||||||
return mutationResult(result), mapAdminStateError(err)
|
return mutationResult(result), mapAdminStateError(err)
|
||||||
}
|
}
|
||||||
|
|
||||||
// ReadAudit returns one bounded, stable page from the authoritative management
|
|
||||||
// audit log. It deliberately exposes only management-plane records and never
|
|
||||||
// reads Proxy activity, credentials, or Provider payloads.
|
|
||||||
func (service *ApplicationService) ReadAudit(ctx context.Context, query AuditQuery) (AuditPage, error) {
|
|
||||||
if service == nil || nilInterface(service.state) || query.Validate() != nil {
|
|
||||||
return AuditPage{}, ErrInvalidConfiguration
|
|
||||||
}
|
|
||||||
records, err := service.state.ReadAudit(ctx, adminstate.AuditQuery{AfterID: query.AfterID, Limit: query.Limit})
|
|
||||||
if err != nil {
|
|
||||||
return AuditPage{}, mapAdminStateError(err)
|
|
||||||
}
|
|
||||||
page := AuditPage{Records: make([]AuditRecord, 0, len(records))}
|
|
||||||
for _, record := range records {
|
|
||||||
page.Records = append(page.Records, AuditRecord{
|
|
||||||
ID: record.ID,
|
|
||||||
RequestID: record.RequestID,
|
|
||||||
ActorID: record.Actor.ID,
|
|
||||||
SourceIP: record.Actor.SourceIP,
|
|
||||||
Action: string(record.Action),
|
|
||||||
ResourceType: record.ResourceType,
|
|
||||||
ResourceName: record.ResourceName,
|
|
||||||
Changed: record.Changed,
|
|
||||||
Version: record.Revision,
|
|
||||||
Reason: record.Reason,
|
|
||||||
OccurredAt: record.OccurredAt.UTC(),
|
|
||||||
})
|
|
||||||
}
|
|
||||||
return page, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (service *ApplicationService) Status(ctx context.Context) (Status, error) {
|
func (service *ApplicationService) Status(ctx context.Context) (Status, error) {
|
||||||
snapshot, err := service.state.Snapshot(ctx)
|
snapshot, err := service.state.Snapshot(ctx)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@ -302,9 +256,6 @@ func (service *ApplicationService) ApplyConfiguration(
|
|||||||
if published && service.runtime != nil {
|
if published && service.runtime != nil {
|
||||||
service.runtime.Notify()
|
service.runtime.Notify()
|
||||||
}
|
}
|
||||||
if published && service.snapshotRefresh != nil {
|
|
||||||
service.snapshotRefresh.NotifySnapshotRefresh()
|
|
||||||
}
|
|
||||||
return mutationResult(result), nil
|
return mutationResult(result), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@ -29,14 +29,12 @@ func TestApplicationServiceMapsUpstreamMutationToAdminState(t *testing.T) {
|
|||||||
},
|
},
|
||||||
}
|
}
|
||||||
runtime := &recordingRuntimeNotifier{}
|
runtime := &recordingRuntimeNotifier{}
|
||||||
refresh := &recordingSnapshotRefreshNotifier{}
|
|
||||||
service, err := NewApplicationService(ApplicationDependencies{
|
service, err := NewApplicationService(ApplicationDependencies{
|
||||||
State: state,
|
State: state,
|
||||||
Operations: staticOperationalStatusReader{},
|
Operations: staticOperationalStatusReader{},
|
||||||
Configuration: staticConfigurationLoader{},
|
Configuration: staticConfigurationLoader{},
|
||||||
Publisher: &recordingConfigurationPublisher{},
|
Publisher: &recordingConfigurationPublisher{},
|
||||||
Runtime: runtime,
|
Runtime: runtime,
|
||||||
SnapshotRefresh: refresh,
|
|
||||||
}, applicationTestOptions(func() time.Time { return now }))
|
}, applicationTestOptions(func() time.Time { return now }))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("NewApplicationService() error = %v", err)
|
t.Fatalf("NewApplicationService() error = %v", err)
|
||||||
@ -67,9 +65,6 @@ func TestApplicationServiceMapsUpstreamMutationToAdminState(t *testing.T) {
|
|||||||
if runtime.notifications != 1 {
|
if runtime.notifications != 1 {
|
||||||
t.Fatalf("runtime notifications = %d, want 1", runtime.notifications)
|
t.Fatalf("runtime notifications = %d, want 1", runtime.notifications)
|
||||||
}
|
}
|
||||||
if refresh.notifications != 1 {
|
|
||||||
t.Fatalf("snapshot refresh notifications = %d, want 1", refresh.notifications)
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestApplicationServicePreflightsProviderRuntimeBeforeMutation(t *testing.T) {
|
func TestApplicationServicePreflightsProviderRuntimeBeforeMutation(t *testing.T) {
|
||||||
@ -152,14 +147,7 @@ func TestApplicationServiceMapsRoutingSwitchAndDomainErrors(t *testing.T) {
|
|||||||
t.Parallel()
|
t.Parallel()
|
||||||
now := time.Date(2026, 7, 29, 11, 0, 0, 0, time.FixedZone("test", 8*60*60))
|
now := time.Date(2026, 7, 29, 11, 0, 0, 0, time.FixedZone("test", 8*60*60))
|
||||||
state := &recordingAdminState{mutation: adminstate.MutationResult{RequestID: "req-switch", Changed: true, Revision: 21}}
|
state := &recordingAdminState{mutation: adminstate.MutationResult{RequestID: "req-switch", Changed: true, Revision: 21}}
|
||||||
refresh := &recordingSnapshotRefreshNotifier{}
|
service := mustApplicationService(t, state, applicationTestOptions(func() time.Time { return now }))
|
||||||
service, serviceErr := NewApplicationService(ApplicationDependencies{
|
|
||||||
State: state, Operations: staticOperationalStatusReader{}, Configuration: staticConfigurationLoader{},
|
|
||||||
Publisher: &recordingConfigurationPublisher{}, SnapshotRefresh: refresh,
|
|
||||||
}, applicationTestOptions(func() time.Time { return now }))
|
|
||||||
if serviceErr != nil {
|
|
||||||
t.Fatalf("NewApplicationService() = %v", serviceErr)
|
|
||||||
}
|
|
||||||
|
|
||||||
result, err := service.SwitchRouting(context.Background(), SwitchCommand{
|
result, err := service.SwitchRouting(context.Background(), SwitchCommand{
|
||||||
RequestID: "req-switch", ActorID: "admin:bob", SourceIP: "198.51.100.7",
|
RequestID: "req-switch", ActorID: "admin:bob", SourceIP: "198.51.100.7",
|
||||||
@ -178,9 +166,6 @@ func TestApplicationServiceMapsRoutingSwitchAndDomainErrors(t *testing.T) {
|
|||||||
if state.lastSwitch != wantCommand {
|
if state.lastSwitch != wantCommand {
|
||||||
t.Fatalf("admin state command = %+v, want %+v", state.lastSwitch, wantCommand)
|
t.Fatalf("admin state command = %+v, want %+v", state.lastSwitch, wantCommand)
|
||||||
}
|
}
|
||||||
if refresh.notifications != 1 {
|
|
||||||
t.Fatalf("snapshot refresh notifications = %d, want 1", refresh.notifications)
|
|
||||||
}
|
|
||||||
|
|
||||||
tests := []struct {
|
tests := []struct {
|
||||||
domain error
|
domain error
|
||||||
@ -201,45 +186,6 @@ func TestApplicationServiceMapsRoutingSwitchAndDomainErrors(t *testing.T) {
|
|||||||
t.Fatalf("SwitchRouting(%v) error = %v, want mapped %v preserving cause", test.domain, err, test.want)
|
t.Fatalf("SwitchRouting(%v) error = %v, want mapped %v preserving cause", test.domain, err, test.want)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if refresh.notifications != 1 {
|
|
||||||
t.Fatalf("snapshot refresh notifications after failed switches = %d, want 1", refresh.notifications)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestApplicationServiceReadsAuthoritativeAuditPage(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
occurredAt := time.Date(2026, 8, 2, 10, 0, 0, 0, time.FixedZone("test", 8*60*60))
|
|
||||||
state := &recordingAdminState{audits: []adminstate.AuditRecord{{
|
|
||||||
ID: 42, RequestID: "req-switch", Actor: adminstate.Actor{ID: "admin:alice", SourceIP: "192.0.2.10"},
|
|
||||||
Action: adminstate.ActionSwitchRoute, ResourceType: "routing", ResourceName: "checkout",
|
|
||||||
Changed: true, Revision: 9, Reason: "capacity", OccurredAt: occurredAt,
|
|
||||||
}}}
|
|
||||||
service := mustApplicationService(t, state, applicationTestOptions(time.Now))
|
|
||||||
|
|
||||||
page, err := service.ReadAudit(context.Background(), AuditQuery{AfterID: 41, Limit: 2})
|
|
||||||
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("ReadAudit() error = %v", err)
|
|
||||||
}
|
|
||||||
if state.lastAuditQuery != (adminstate.AuditQuery{AfterID: 41, Limit: 2}) {
|
|
||||||
t.Fatalf("audit query = %+v", state.lastAuditQuery)
|
|
||||||
}
|
|
||||||
if len(page.Records) != 1 {
|
|
||||||
t.Fatalf("audit records = %+v", page.Records)
|
|
||||||
}
|
|
||||||
record := page.Records[0]
|
|
||||||
if record.ID != 42 || record.Action != string(adminstate.ActionSwitchRoute) || record.Version != 9 ||
|
|
||||||
record.ActorID != "admin:alice" || record.SourceIP != "192.0.2.10" || !record.OccurredAt.Equal(occurredAt.UTC()) {
|
|
||||||
t.Fatalf("mapped audit record = %+v", record)
|
|
||||||
}
|
|
||||||
|
|
||||||
state.err = adminstate.ErrUnavailable
|
|
||||||
if _, err := service.ReadAudit(context.Background(), AuditQuery{Limit: 1}); !errors.Is(err, ErrUnavailable) || !errors.Is(err, adminstate.ErrUnavailable) {
|
|
||||||
t.Fatalf("ReadAudit() unavailable error = %v", err)
|
|
||||||
}
|
|
||||||
if _, err := service.ReadAudit(context.Background(), AuditQuery{}); !errors.Is(err, ErrInvalidConfiguration) {
|
|
||||||
t.Fatalf("ReadAudit() invalid query error = %v", err)
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestApplicationServiceBuildsStatusFromAuthoritativeAndOperationalSnapshots(t *testing.T) {
|
func TestApplicationServiceBuildsStatusFromAuthoritativeAndOperationalSnapshots(t *testing.T) {
|
||||||
@ -335,7 +281,6 @@ func TestApplicationServiceReloadPersistsManagementViewBeforePublishing(t *testi
|
|||||||
configuration := validReloadConfiguration()
|
configuration := validReloadConfiguration()
|
||||||
publisher := &recordingConfigurationPublisher{}
|
publisher := &recordingConfigurationPublisher{}
|
||||||
runtime := &recordingRuntimeNotifier{}
|
runtime := &recordingRuntimeNotifier{}
|
||||||
refresh := &recordingSnapshotRefreshNotifier{}
|
|
||||||
state := &recordingAdminState{
|
state := &recordingAdminState{
|
||||||
mutation: adminstate.MutationResult{RequestID: "req-reload", Changed: true, Revision: 42},
|
mutation: adminstate.MutationResult{RequestID: "req-reload", Changed: true, Revision: 42},
|
||||||
snapshot: adminstate.Snapshot{Routings: []adminstate.RoutingState{
|
snapshot: adminstate.Snapshot{Routings: []adminstate.RoutingState{
|
||||||
@ -352,7 +297,7 @@ func TestApplicationServiceReloadPersistsManagementViewBeforePublishing(t *testi
|
|||||||
Configuration: staticConfigurationLoader{loaded: LoadedConfiguration{
|
Configuration: staticConfigurationLoader{loaded: LoadedConfiguration{
|
||||||
Value: configuration, Source: "configs/proxy-pool.yaml",
|
Value: configuration, Source: "configs/proxy-pool.yaml",
|
||||||
}},
|
}},
|
||||||
Publisher: publisher, Runtime: runtime, SnapshotRefresh: refresh,
|
Publisher: publisher, Runtime: runtime,
|
||||||
}, applicationTestOptions(func() time.Time { return now }))
|
}, applicationTestOptions(func() time.Time { return now }))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("NewApplicationService() error = %v", err)
|
t.Fatalf("NewApplicationService() error = %v", err)
|
||||||
@ -373,9 +318,6 @@ func TestApplicationServiceReloadPersistsManagementViewBeforePublishing(t *testi
|
|||||||
if runtime.notifications != 1 {
|
if runtime.notifications != 1 {
|
||||||
t.Fatalf("runtime notifications = %d, want 1", runtime.notifications)
|
t.Fatalf("runtime notifications = %d, want 1", runtime.notifications)
|
||||||
}
|
}
|
||||||
if refresh.notifications != 1 {
|
|
||||||
t.Fatalf("snapshot refresh notifications = %d, want 1", refresh.notifications)
|
|
||||||
}
|
|
||||||
command := state.lastConfig
|
command := state.lastConfig
|
||||||
if command.RequestID != "req-reload" || command.Actor != (adminstate.Actor{ID: "admin:alice", SourceIP: "192.0.2.10"}) ||
|
if command.RequestID != "req-reload" || command.Actor != (adminstate.Actor{ID: "admin:alice", SourceIP: "192.0.2.10"}) ||
|
||||||
!command.OccurredAt.Equal(now) || command.Source != "configs/proxy-pool.yaml" {
|
!command.OccurredAt.Equal(now) || command.Source != "configs/proxy-pool.yaml" {
|
||||||
@ -678,16 +620,13 @@ func mustApplicationService(t *testing.T, state StateRepository, options Applica
|
|||||||
}
|
}
|
||||||
|
|
||||||
type recordingAdminState struct {
|
type recordingAdminState struct {
|
||||||
mutation adminstate.MutationResult
|
mutation adminstate.MutationResult
|
||||||
err error
|
err error
|
||||||
snapshot adminstate.Snapshot
|
snapshot adminstate.Snapshot
|
||||||
audits []adminstate.AuditRecord
|
lastUpstream adminstate.SetUpstreamCommand
|
||||||
lastAuditQuery adminstate.AuditQuery
|
lastSwitch adminstate.SwitchRoutingCommand
|
||||||
lastUpstream adminstate.SetUpstreamCommand
|
lastConfig adminstate.CommitConfigCommand
|
||||||
lastSwitch adminstate.SwitchRoutingCommand
|
onCommit func(adminstate.CommitConfigCommand)
|
||||||
lastDisable adminstate.DisableRoutingCommand
|
|
||||||
lastConfig adminstate.CommitConfigCommand
|
|
||||||
onCommit func(adminstate.CommitConfigCommand)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
type orderedCommitState struct {
|
type orderedCommitState struct {
|
||||||
@ -704,10 +643,6 @@ func (state *orderedCommitState) SwitchRouting(context.Context, adminstate.Switc
|
|||||||
return adminstate.MutationResult{}, nil
|
return adminstate.MutationResult{}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (state *orderedCommitState) DisableRouting(context.Context, adminstate.DisableRoutingCommand) (adminstate.MutationResult, error) {
|
|
||||||
return adminstate.MutationResult{}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (state *orderedCommitState) CommitConfig(_ context.Context, command adminstate.CommitConfigCommand) (adminstate.MutationResult, error) {
|
func (state *orderedCommitState) CommitConfig(_ context.Context, command adminstate.CommitConfigCommand) (adminstate.MutationResult, error) {
|
||||||
revision := state.next.Add(1)
|
revision := state.next.Add(1)
|
||||||
if revision == 1 {
|
if revision == 1 {
|
||||||
@ -721,10 +656,6 @@ func (*orderedCommitState) Snapshot(context.Context) (adminstate.Snapshot, error
|
|||||||
return adminstate.Snapshot{}, nil
|
return adminstate.Snapshot{}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (*orderedCommitState) ReadAudit(context.Context, adminstate.AuditQuery) ([]adminstate.AuditRecord, error) {
|
|
||||||
return nil, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (state *recordingAdminState) SetUpstreamEnabled(_ context.Context, command adminstate.SetUpstreamCommand) (adminstate.MutationResult, error) {
|
func (state *recordingAdminState) SetUpstreamEnabled(_ context.Context, command adminstate.SetUpstreamCommand) (adminstate.MutationResult, error) {
|
||||||
state.lastUpstream = command
|
state.lastUpstream = command
|
||||||
return state.mutation, state.err
|
return state.mutation, state.err
|
||||||
@ -735,11 +666,6 @@ func (state *recordingAdminState) SwitchRouting(_ context.Context, command admin
|
|||||||
return state.mutation, state.err
|
return state.mutation, state.err
|
||||||
}
|
}
|
||||||
|
|
||||||
func (state *recordingAdminState) DisableRouting(_ context.Context, command adminstate.DisableRoutingCommand) (adminstate.MutationResult, error) {
|
|
||||||
state.lastDisable = command
|
|
||||||
return state.mutation, state.err
|
|
||||||
}
|
|
||||||
|
|
||||||
func (state *recordingAdminState) CommitConfig(_ context.Context, command adminstate.CommitConfigCommand) (adminstate.MutationResult, error) {
|
func (state *recordingAdminState) CommitConfig(_ context.Context, command adminstate.CommitConfigCommand) (adminstate.MutationResult, error) {
|
||||||
state.lastConfig = command
|
state.lastConfig = command
|
||||||
if state.onCommit != nil {
|
if state.onCommit != nil {
|
||||||
@ -752,11 +678,6 @@ func (state *recordingAdminState) Snapshot(context.Context) (adminstate.Snapshot
|
|||||||
return state.snapshot, state.err
|
return state.snapshot, state.err
|
||||||
}
|
}
|
||||||
|
|
||||||
func (state *recordingAdminState) ReadAudit(_ context.Context, query adminstate.AuditQuery) ([]adminstate.AuditRecord, error) {
|
|
||||||
state.lastAuditQuery = query
|
|
||||||
return append([]adminstate.AuditRecord(nil), state.audits...), state.err
|
|
||||||
}
|
|
||||||
|
|
||||||
type staticOperationalStatusReader struct {
|
type staticOperationalStatusReader struct {
|
||||||
status OperationalStatus
|
status OperationalStatus
|
||||||
err error
|
err error
|
||||||
@ -791,10 +712,6 @@ type recordingRuntimeNotifier struct {
|
|||||||
validationErr error
|
validationErr error
|
||||||
}
|
}
|
||||||
|
|
||||||
type recordingSnapshotRefreshNotifier struct{ notifications int }
|
|
||||||
|
|
||||||
func (notifier *recordingSnapshotRefreshNotifier) NotifySnapshotRefresh() { notifier.notifications++ }
|
|
||||||
|
|
||||||
func (notifier *recordingRuntimeNotifier) Notify() { notifier.notifications++ }
|
func (notifier *recordingRuntimeNotifier) Notify() { notifier.notifications++ }
|
||||||
|
|
||||||
func (notifier *recordingRuntimeNotifier) ValidateConfiguration(context.Context, *config.Config) error {
|
func (notifier *recordingRuntimeNotifier) ValidateConfiguration(context.Context, *config.Config) error {
|
||||||
|
|||||||
@ -19,7 +19,6 @@ import (
|
|||||||
"proxy-pool/internal/controller/operations"
|
"proxy-pool/internal/controller/operations"
|
||||||
"proxy-pool/internal/controller/pool"
|
"proxy-pool/internal/controller/pool"
|
||||||
"proxy-pool/internal/controller/provider"
|
"proxy-pool/internal/controller/provider"
|
||||||
controllerRouting "proxy-pool/internal/controller/routing"
|
|
||||||
controllerRuntime "proxy-pool/internal/controller/runtime"
|
controllerRuntime "proxy-pool/internal/controller/runtime"
|
||||||
"proxy-pool/internal/controller/worker"
|
"proxy-pool/internal/controller/worker"
|
||||||
"proxy-pool/internal/domain/activitypool"
|
"proxy-pool/internal/domain/activitypool"
|
||||||
@ -78,7 +77,6 @@ type ports struct {
|
|||||||
coordinator provider.Coordinator
|
coordinator provider.Coordinator
|
||||||
credentials credentials.Store
|
credentials credentials.Store
|
||||||
providerResults provider.ResultRecorder
|
providerResults provider.ResultRecorder
|
||||||
capacityMetrics pool.CapacityObserver
|
|
||||||
workerStore workerruntime.ControlStore
|
workerStore workerruntime.ControlStore
|
||||||
close func() error
|
close func() error
|
||||||
}
|
}
|
||||||
@ -178,63 +176,13 @@ func runWithWorkerFactory(
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("%w: build Provider supervisor: %w", ErrStartup, err)
|
return fmt.Errorf("%w: build Provider supervisor: %w", ErrStartup, err)
|
||||||
}
|
}
|
||||||
snapshotRefresh := worker.NewSnapshotRefreshBroker()
|
|
||||||
var sequentialCoordinator lifecycle.Runner
|
|
||||||
|
|
||||||
dependencies := controllerRuntime.Dependencies{}
|
dependencies := controllerRuntime.Dependencies{}
|
||||||
var checkerMetrics healthDomain.TaskMetricsObserver
|
|
||||||
var drainMetrics healthDomain.DrainMetricsObserver
|
|
||||||
var extractionMetrics extraction.MetricsObserver
|
|
||||||
if loaded.Value.Metrics.Enabled {
|
|
||||||
if nilInterface(opened.metricsReadiness) {
|
|
||||||
return errors.Join(ErrStartup, ErrInvalidOptions)
|
|
||||||
}
|
|
||||||
collector, collectorErr := platformMetrics.NewCheckerCollector(prometheus.DefaultRegisterer)
|
|
||||||
if collectorErr != nil {
|
|
||||||
return fmt.Errorf("%w: build Checker metrics: %w", ErrStartup, collectorErr)
|
|
||||||
}
|
|
||||||
checkerMetrics = collector
|
|
||||||
drainCollector, drainCollectorErr := platformMetrics.NewDrainCollector(prometheus.DefaultRegisterer)
|
|
||||||
if drainCollectorErr != nil {
|
|
||||||
return fmt.Errorf("%w: build Drain metrics: %w", ErrStartup, drainCollectorErr)
|
|
||||||
}
|
|
||||||
drainMetrics = drainCollector
|
|
||||||
providerCollector, providerCollectorErr := platformMetrics.NewProviderCollector(prometheus.DefaultRegisterer)
|
|
||||||
if providerCollectorErr != nil {
|
|
||||||
return fmt.Errorf("%w: build Provider metrics: %w", ErrStartup, providerCollectorErr)
|
|
||||||
}
|
|
||||||
if registrar, ok := opened.providerResults.(provider.ResultObserverRegistrar); ok && !nilInterface(registrar) {
|
|
||||||
registrar.AddResultObserver(providerCollector)
|
|
||||||
}
|
|
||||||
capacityCollector, capacityCollectorErr := platformMetrics.NewCapacityCollector(prometheus.DefaultRegisterer)
|
|
||||||
if capacityCollectorErr != nil {
|
|
||||||
return fmt.Errorf("%w: build capacity metrics: %w", ErrStartup, capacityCollectorErr)
|
|
||||||
}
|
|
||||||
opened.capacityMetrics = capacityCollector
|
|
||||||
extractionCollector, extractionCollectorErr := platformMetrics.NewExtractionCollector(prometheus.DefaultRegisterer)
|
|
||||||
if extractionCollectorErr != nil {
|
|
||||||
return fmt.Errorf("%w: build extraction metrics: %w", ErrStartup, extractionCollectorErr)
|
|
||||||
}
|
|
||||||
extractionMetrics = extractionCollector
|
|
||||||
handler, handlerErr := platformMetrics.NewHandler(platformMetrics.Dependencies{
|
|
||||||
Gatherer: prometheus.DefaultGatherer, Readiness: opened.metricsReadiness,
|
|
||||||
})
|
|
||||||
if handlerErr != nil {
|
|
||||||
return fmt.Errorf("%w: build metrics handler: %w", ErrStartup, handlerErr)
|
|
||||||
}
|
|
||||||
dependencies.MetricsHandler = handler
|
|
||||||
}
|
|
||||||
if loaded.Value.Distribution.Enabled {
|
if loaded.Value.Distribution.Enabled {
|
||||||
if nilInterface(opened.activity) || nilInterface(opened.readiness) || nilInterface(opened.admission) {
|
if nilInterface(opened.activity) || nilInterface(opened.readiness) || nilInterface(opened.admission) {
|
||||||
return errors.Join(ErrStartup, ErrInvalidOptions)
|
return errors.Join(ErrStartup, ErrInvalidOptions)
|
||||||
}
|
}
|
||||||
service, serviceErr := extraction.NewService(
|
service, serviceErr := extraction.NewService(opened.activity, extractionPolicy(loaded.Value), opened.admission, options.Now)
|
||||||
opened.activity,
|
|
||||||
extractionPolicy(loaded.Value),
|
|
||||||
opened.admission,
|
|
||||||
options.Now,
|
|
||||||
extractionMetrics,
|
|
||||||
)
|
|
||||||
if serviceErr != nil {
|
if serviceErr != nil {
|
||||||
return fmt.Errorf("%w: build extraction service: %w", ErrStartup, serviceErr)
|
return fmt.Errorf("%w: build extraction service: %w", ErrStartup, serviceErr)
|
||||||
}
|
}
|
||||||
@ -260,7 +208,7 @@ func runWithWorkerFactory(
|
|||||||
}
|
}
|
||||||
service, serviceErr := admin.NewApplicationService(admin.ApplicationDependencies{
|
service, serviceErr := admin.NewApplicationService(admin.ApplicationDependencies{
|
||||||
State: opened.state, Operations: statusReader, Configuration: loader, Publisher: configurationStore,
|
State: opened.state, Operations: statusReader, Configuration: loader, Publisher: configurationStore,
|
||||||
Runtime: supervisor, SnapshotRefresh: snapshotRefresh,
|
Runtime: supervisor,
|
||||||
}, admin.ApplicationOptions{Now: options.Now, FingerprintKey: options.FingerprintKey})
|
}, admin.ApplicationOptions{Now: options.Now, FingerprintKey: options.FingerprintKey})
|
||||||
if serviceErr != nil {
|
if serviceErr != nil {
|
||||||
return fmt.Errorf("%w: build admin service: %w", ErrStartup, serviceErr)
|
return fmt.Errorf("%w: build admin service: %w", ErrStartup, serviceErr)
|
||||||
@ -271,23 +219,26 @@ func runWithWorkerFactory(
|
|||||||
return fmt.Errorf("%w: commit startup configuration: %w", ErrStartup, applyErr)
|
return fmt.Errorf("%w: commit startup configuration: %w", ErrStartup, applyErr)
|
||||||
}
|
}
|
||||||
dependencies.AdminService = service
|
dependencies.AdminService = service
|
||||||
if stats, ok := opened.providerResults.(provider.StatsReader); ok && !nilInterface(stats) {
|
|
||||||
coordinator, coordinatorErr := controllerRouting.NewSequentialCoordinator(
|
|
||||||
configurationStore,
|
|
||||||
opened.state,
|
|
||||||
stats,
|
|
||||||
snapshotRefresh,
|
|
||||||
controllerRouting.Options{Now: options.Now},
|
|
||||||
)
|
|
||||||
if coordinatorErr != nil {
|
|
||||||
return fmt.Errorf("%w: build Sequential routing coordinator: %w", ErrStartup, coordinatorErr)
|
|
||||||
}
|
|
||||||
if registrar, ok := opened.providerResults.(provider.ResultObserverRegistrar); ok && !nilInterface(registrar) {
|
|
||||||
registrar.AddResultObserver(coordinator)
|
|
||||||
}
|
|
||||||
sequentialCoordinator = coordinator
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
var checkerMetrics healthDomain.TaskMetricsObserver
|
||||||
|
if loaded.Value.Metrics.Enabled {
|
||||||
|
if nilInterface(opened.metricsReadiness) {
|
||||||
|
return errors.Join(ErrStartup, ErrInvalidOptions)
|
||||||
|
}
|
||||||
|
collector, collectorErr := platformMetrics.NewCheckerCollector(prometheus.DefaultRegisterer)
|
||||||
|
if collectorErr != nil {
|
||||||
|
return fmt.Errorf("%w: build Checker metrics: %w", ErrStartup, collectorErr)
|
||||||
|
}
|
||||||
|
checkerMetrics = collector
|
||||||
|
handler, handlerErr := platformMetrics.NewHandler(platformMetrics.Dependencies{
|
||||||
|
Gatherer: prometheus.DefaultGatherer, Readiness: opened.metricsReadiness,
|
||||||
|
})
|
||||||
|
if handlerErr != nil {
|
||||||
|
return fmt.Errorf("%w: build metrics handler: %w", ErrStartup, handlerErr)
|
||||||
|
}
|
||||||
|
dependencies.MetricsHandler = handler
|
||||||
|
}
|
||||||
|
|
||||||
runners := make([]lifecycle.Runner, 0, 3+len(loaded.Value.Upstreams))
|
runners := make([]lifecycle.Runner, 0, 3+len(loaded.Value.Upstreams))
|
||||||
if hasHTTPRuntime(loaded.Value) {
|
if hasHTTPRuntime(loaded.Value) {
|
||||||
runner, err := factory.New(configurationStore.Current(), dependencies, controllerRuntime.Options{HTTP: options.HTTP})
|
runner, err := factory.New(configurationStore.Current(), dependencies, controllerRuntime.Options{HTTP: options.HTTP})
|
||||||
@ -333,7 +284,6 @@ func runWithWorkerFactory(
|
|||||||
DrainTickets: drainTickets,
|
DrainTickets: drainTickets,
|
||||||
RoutingSource: routingSource,
|
RoutingSource: routingSource,
|
||||||
Credentials: opened.credentials,
|
Credentials: opened.credentials,
|
||||||
SnapshotRefresh: snapshotRefresh,
|
|
||||||
})
|
})
|
||||||
if serviceErr != nil {
|
if serviceErr != nil {
|
||||||
return fmt.Errorf("%w: build Worker control service: %w", ErrStartup, serviceErr)
|
return fmt.Errorf("%w: build Worker control service: %w", ErrStartup, serviceErr)
|
||||||
@ -370,14 +320,7 @@ func runWithWorkerFactory(
|
|||||||
}
|
}
|
||||||
runners = append(runners, runner)
|
runners = append(runners, runner)
|
||||||
if tasks, ok := opened.activity.(healthTaskRuntime); ok && !nilInterface(tasks) {
|
if tasks, ok := opened.activity.(healthTaskRuntime); ok && !nilInterface(tasks) {
|
||||||
var schedulerStates []controllerHealth.SchedulerStateSource
|
schedulers, schedulerErr := newHealthSchedulers(configurationStore, tasks, options.Now)
|
||||||
if loaded.Value.Admin.Enabled {
|
|
||||||
if nilInterface(opened.state) {
|
|
||||||
return errors.Join(ErrStartup, ErrInvalidOptions)
|
|
||||||
}
|
|
||||||
schedulerStates = append(schedulerStates, opened.state)
|
|
||||||
}
|
|
||||||
schedulers, schedulerErr := newHealthSchedulers(configurationStore, tasks, options.Now, schedulerStates...)
|
|
||||||
if schedulerErr != nil {
|
if schedulerErr != nil {
|
||||||
return fmt.Errorf("%w: build Checker health schedulers: %w", ErrStartup, schedulerErr)
|
return fmt.Errorf("%w: build Checker health schedulers: %w", ErrStartup, schedulerErr)
|
||||||
}
|
}
|
||||||
@ -388,7 +331,7 @@ func runWithWorkerFactory(
|
|||||||
configurationStore,
|
configurationStore,
|
||||||
unhealthy,
|
unhealthy,
|
||||||
controllerHealth.UnhealthyReaperOptions{
|
controllerHealth.UnhealthyReaperOptions{
|
||||||
PollInterval: checkSchedulerPollInterval, BatchSize: checkSchedulerBatchSize, Now: options.Now, Metrics: drainMetrics,
|
PollInterval: checkSchedulerPollInterval, BatchSize: checkSchedulerBatchSize, Now: options.Now,
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
if reaperErr != nil {
|
if reaperErr != nil {
|
||||||
@ -396,33 +339,6 @@ func runWithWorkerFactory(
|
|||||||
}
|
}
|
||||||
runners = append(runners, reaper)
|
runners = append(runners, reaper)
|
||||||
}
|
}
|
||||||
if drainStore, ok := opened.activity.(controllerHealth.UpstreamDrainStore); ok && !nilInterface(drainStore) {
|
|
||||||
var policyStates []controllerHealth.UpstreamDrainStateSource
|
|
||||||
if loaded.Value.Admin.Enabled {
|
|
||||||
if nilInterface(opened.state) {
|
|
||||||
return errors.Join(ErrStartup, ErrInvalidOptions)
|
|
||||||
}
|
|
||||||
policyStates = append(policyStates, opened.state)
|
|
||||||
}
|
|
||||||
policySource, policyErr := controllerHealth.NewEffectiveUpstreamDrainPolicySource(configurationStore, policyStates...)
|
|
||||||
if policyErr != nil {
|
|
||||||
return fmt.Errorf("%w: build upstream Drain policy source: %w", ErrStartup, policyErr)
|
|
||||||
}
|
|
||||||
reaper, reaperErr := controllerHealth.NewConfiguredUpstreamDrainReaper(
|
|
||||||
policySource,
|
|
||||||
drainStore,
|
|
||||||
controllerHealth.UpstreamDrainReaperOptions{
|
|
||||||
PollInterval: checkSchedulerPollInterval, BatchSize: checkSchedulerBatchSize, Now: options.Now, Metrics: drainMetrics,
|
|
||||||
},
|
|
||||||
)
|
|
||||||
if reaperErr != nil {
|
|
||||||
return fmt.Errorf("%w: build upstream Drain reaper: %w", ErrStartup, reaperErr)
|
|
||||||
}
|
|
||||||
runners = append(runners, reaper)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if sequentialCoordinator != nil {
|
|
||||||
runners = append(runners, sequentialCoordinator)
|
|
||||||
}
|
}
|
||||||
runners = append(runners, supervisor)
|
runners = append(runners, supervisor)
|
||||||
group, err := lifecycle.NewGroup(runners...)
|
group, err := lifecycle.NewGroup(runners...)
|
||||||
@ -436,10 +352,8 @@ func newHealthSchedulers(
|
|||||||
configuration controllerHealth.ConfigurationSource,
|
configuration controllerHealth.ConfigurationSource,
|
||||||
tasks healthTaskRuntime,
|
tasks healthTaskRuntime,
|
||||||
now func() time.Time,
|
now func() time.Time,
|
||||||
states ...controllerHealth.SchedulerStateSource,
|
|
||||||
) ([]lifecycle.Runner, error) {
|
) ([]lifecycle.Runner, error) {
|
||||||
if nilInterface(configuration) || nilInterface(tasks) || now == nil || len(states) > 1 ||
|
if nilInterface(configuration) || nilInterface(tasks) || now == nil {
|
||||||
(len(states) == 1 && nilInterface(states[0])) {
|
|
||||||
return nil, ErrInvalidOptions
|
return nil, ErrInvalidOptions
|
||||||
}
|
}
|
||||||
current := configuration.Current()
|
current := configuration.Current()
|
||||||
@ -458,7 +372,7 @@ func newHealthSchedulers(
|
|||||||
}
|
}
|
||||||
supervisor, err := controllerHealth.NewConfiguredSchedulerSupervisor(configuration, tasks, tasks, controllerHealth.SchedulerRunnerOptions{
|
supervisor, err := controllerHealth.NewConfiguredSchedulerSupervisor(configuration, tasks, tasks, controllerHealth.SchedulerRunnerOptions{
|
||||||
PollInterval: checkSchedulerPollInterval, BatchSize: checkSchedulerBatchSize, Now: now,
|
PollInterval: checkSchedulerPollInterval, BatchSize: checkSchedulerBatchSize, Now: now,
|
||||||
}, states...)
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|||||||
@ -75,7 +75,6 @@ func providerRuntimeBuilder(opened ports) (buildUpstreamRuntime, error) {
|
|||||||
Parser: parser,
|
Parser: parser,
|
||||||
Activity: opened.activity,
|
Activity: opened.activity,
|
||||||
Results: results,
|
Results: results,
|
||||||
Capacity: opened.capacityMetrics,
|
|
||||||
})
|
})
|
||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|||||||
@ -276,7 +276,7 @@ func (supervisor *providerSupervisor) desired(ctx context.Context) (map[string]c
|
|||||||
for name, upstream := range configuration.Upstreams {
|
for name, upstream := range configuration.Upstreams {
|
||||||
isEnabled := upstream.Enabled
|
isEnabled := upstream.Enabled
|
||||||
if !nilInterface(supervisor.state) {
|
if !nilInterface(supervisor.state) {
|
||||||
isEnabled = isEnabled && enabled[name]
|
isEnabled = enabled[name]
|
||||||
}
|
}
|
||||||
if isEnabled {
|
if isEnabled {
|
||||||
upstream.Enabled = true
|
upstream.Enabled = true
|
||||||
|
|||||||
@ -71,38 +71,6 @@ func TestProviderSupervisorAppliesDisableAndConfigurationReplacement(t *testing.
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestProviderSupervisorKeepsStaticallyDisabledUpstreamStopped(t *testing.T) {
|
|
||||||
configuration, err := config.Load(strings.NewReader(bootstrapTestConfig))
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("config.Load(): %v", err)
|
|
||||||
}
|
|
||||||
disabled := configuration.Upstreams["provider-b"]
|
|
||||||
disabled.Enabled = false
|
|
||||||
configuration.Upstreams["provider-b"] = disabled
|
|
||||||
store, err := config.NewStore(configuration)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("config.NewStore(): %v", err)
|
|
||||||
}
|
|
||||||
supervisor, err := newProviderSupervisor(
|
|
||||||
store,
|
|
||||||
&mutableProviderState{enabled: map[string]bool{"provider-a": true, "provider-b": true}},
|
|
||||||
func(string, config.Upstream) (lifecycle.Runner, error) {
|
|
||||||
return supervisorRunnerFunc(func(context.Context) error { return nil }), nil
|
|
||||||
},
|
|
||||||
nil, nil, nil, bootstrapTestFingerprintKey, time.Hour,
|
|
||||||
)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("newProviderSupervisor() error = %v", err)
|
|
||||||
}
|
|
||||||
desired, err := supervisor.desired(context.Background())
|
|
||||||
if err != nil || len(desired) != 1 {
|
|
||||||
t.Fatalf("desired() = %+v, %v", desired, err)
|
|
||||||
}
|
|
||||||
if _, exists := desired["provider-a"]; !exists {
|
|
||||||
t.Fatalf("desired() = %+v, provider-a missing", desired)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestProviderSupervisorPropagatesUnexpectedRuntimeFailure(t *testing.T) {
|
func TestProviderSupervisorPropagatesUnexpectedRuntimeFailure(t *testing.T) {
|
||||||
configuration, err := config.Load(strings.NewReader(bootstrapTestConfig))
|
configuration, err := config.Load(strings.NewReader(bootstrapTestConfig))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|||||||
@ -8,8 +8,6 @@ import (
|
|||||||
"time"
|
"time"
|
||||||
|
|
||||||
controllerExtraction "proxy-pool/internal/controller/extraction"
|
controllerExtraction "proxy-pool/internal/controller/extraction"
|
||||||
"proxy-pool/internal/domain/authorization"
|
|
||||||
"proxy-pool/internal/domain/clientpolicy"
|
|
||||||
domainExtraction "proxy-pool/internal/domain/extraction"
|
domainExtraction "proxy-pool/internal/domain/extraction"
|
||||||
"proxy-pool/internal/platform/httpapi"
|
"proxy-pool/internal/platform/httpapi"
|
||||||
"proxy-pool/internal/platform/httpsecurity"
|
"proxy-pool/internal/platform/httpsecurity"
|
||||||
@ -175,10 +173,6 @@ func (h *Handler) handleExtract(writer http.ResponseWriter, request *http.Reques
|
|||||||
h.writeProblem(writer, problemBadRequest(requestID, "INVALID_REQUEST", "Invalid request", "", nil))
|
h.writeProblem(writer, problemBadRequest(requestID, "INVALID_REQUEST", "Invalid request", "", nil))
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
if !identity.Allows(authorization.DistributionExtract) {
|
|
||||||
h.writeProblem(writer, httpapi.NewProblem(http.StatusForbidden, "FORBIDDEN", "Forbidden", "", requestID))
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
idempotencyKey, err := validateIdempotencyKey(request.Header.Values(headerIdempotencyKey))
|
idempotencyKey, err := validateIdempotencyKey(request.Header.Values(headerIdempotencyKey))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@ -207,15 +201,7 @@ func (h *Handler) handleExtract(writer http.ResponseWriter, request *http.Reques
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
if !identity.ClientPolicy.AllowsExtractCount(payload.Count) {
|
filters := payload.filtersOrZero()
|
||||||
h.writeProblem(writer, httpapi.NewProblem(http.StatusForbidden, "FORBIDDEN", "Forbidden", "", requestID))
|
|
||||||
return
|
|
||||||
}
|
|
||||||
filters, allowed := restrictExtractFilters(identity.ClientPolicy, payload.filtersOrZero())
|
|
||||||
if !allowed {
|
|
||||||
h.writeProblem(writer, httpapi.NewProblem(http.StatusForbidden, "FORBIDDEN", "Forbidden", "", requestID))
|
|
||||||
return
|
|
||||||
}
|
|
||||||
serviceResponse, err := h.extractor.Extract(request.Context(), controllerExtraction.Request{
|
serviceResponse, err := h.extractor.Extract(request.Context(), controllerExtraction.Request{
|
||||||
RequestID: requestID,
|
RequestID: requestID,
|
||||||
ClientID: identity.ClientID,
|
ClientID: identity.ClientID,
|
||||||
@ -261,20 +247,6 @@ func (h *Handler) handleExtract(writer http.ResponseWriter, request *http.Reques
|
|||||||
h.writeJSON(writer, requestID, http.StatusOK, response)
|
h.writeJSON(writer, requestID, http.StatusOK, response)
|
||||||
}
|
}
|
||||||
|
|
||||||
func restrictExtractFilters(policy clientpolicy.Policy, filters extractFiltersDTO) (extractFiltersDTO, bool) {
|
|
||||||
upstreams, allowed := policy.RestrictUpstreams(filters.AllowedUpstreams)
|
|
||||||
if !allowed {
|
|
||||||
return extractFiltersDTO{}, false
|
|
||||||
}
|
|
||||||
regions, allowed := policy.RestrictRegions(filters.Regions)
|
|
||||||
if !allowed {
|
|
||||||
return extractFiltersDTO{}, false
|
|
||||||
}
|
|
||||||
filters.AllowedUpstreams = upstreams
|
|
||||||
filters.Regions = regions
|
|
||||||
return filters, true
|
|
||||||
}
|
|
||||||
|
|
||||||
func validateIdempotencyKey(values []string) (string, error) {
|
func validateIdempotencyKey(values []string) (string, error) {
|
||||||
if len(values) == 0 || (len(values) == 1 && values[0] == "") {
|
if len(values) == 0 || (len(values) == 1 && values[0] == "") {
|
||||||
return "", nil
|
return "", nil
|
||||||
|
|||||||
@ -7,14 +7,11 @@ import (
|
|||||||
"fmt"
|
"fmt"
|
||||||
"net/http"
|
"net/http"
|
||||||
"net/http/httptest"
|
"net/http/httptest"
|
||||||
"slices"
|
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
controllerExtraction "proxy-pool/internal/controller/extraction"
|
controllerExtraction "proxy-pool/internal/controller/extraction"
|
||||||
"proxy-pool/internal/domain/authorization"
|
|
||||||
"proxy-pool/internal/domain/clientpolicy"
|
|
||||||
domainExtraction "proxy-pool/internal/domain/extraction"
|
domainExtraction "proxy-pool/internal/domain/extraction"
|
||||||
"proxy-pool/internal/platform/httpapi"
|
"proxy-pool/internal/platform/httpapi"
|
||||||
"proxy-pool/internal/platform/httpsecurity"
|
"proxy-pool/internal/platform/httpsecurity"
|
||||||
@ -160,108 +157,6 @@ func TestHandlerExtractSuccessMapsOpenAPIDTOAndReturnsRequestID(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestHandlerRejectsIdentityWithoutExtractPermission(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
extractor := &fakeExtractor{}
|
|
||||||
handler := mustNewHandler(t, Config{BodyLimitBytes: 1024}, Dependencies{
|
|
||||||
Extractor: extractor,
|
|
||||||
Identity: fakeIdentityResolver{identity: Identity{
|
|
||||||
ClientID: "tenant-read-only", SourceIP: "198.51.100.8", Permissions: []string{authorization.AdminRead},
|
|
||||||
}},
|
|
||||||
Readiness: fakeReadinessChecker{},
|
|
||||||
})
|
|
||||||
request := httptest.NewRequest(http.MethodPost, "/api/v1/proxies/extract", strings.NewReader(`{"count":1}`))
|
|
||||||
request.Header.Set("Content-Type", httpapi.JSONContentType)
|
|
||||||
request.Header.Set(headerIdempotencyKey, "idem-12345678")
|
|
||||||
recorder := httptest.NewRecorder()
|
|
||||||
|
|
||||||
handler.ServeHTTP(recorder, request)
|
|
||||||
|
|
||||||
if recorder.Code != http.StatusForbidden || extractor.calls != 0 || !strings.Contains(recorder.Body.String(), `"code":"FORBIDDEN"`) {
|
|
||||||
t.Fatalf("response = status %d calls=%d body=%s", recorder.Code, extractor.calls, recorder.Body.String())
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestHandlerEnforcesCredentialExtractionPolicy(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
tests := []struct {
|
|
||||||
name string
|
|
||||||
body string
|
|
||||||
wantStatus int
|
|
||||||
wantUpstreams []string
|
|
||||||
wantRegions []string
|
|
||||||
wantExtractorCall int
|
|
||||||
}{
|
|
||||||
{
|
|
||||||
name: "omitted filters are bound to credential policy",
|
|
||||||
body: "{\"count\":2}",
|
|
||||||
wantStatus: http.StatusOK,
|
|
||||||
wantUpstreams: []string{"provider-a"},
|
|
||||||
wantRegions: []string{"shanghai"},
|
|
||||||
wantExtractorCall: 1,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "count exceeds credential maximum",
|
|
||||||
body: "{\"count\":3}",
|
|
||||||
wantStatus: http.StatusForbidden,
|
|
||||||
wantExtractorCall: 0,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "upstream exceeds credential boundary",
|
|
||||||
body: "{\"count\":1,\"filters\":{\"allowedUpstreams\":[\"provider-b\"]}}",
|
|
||||||
wantStatus: http.StatusForbidden,
|
|
||||||
wantExtractorCall: 0,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "region exceeds credential boundary",
|
|
||||||
body: "{\"count\":1,\"filters\":{\"regions\":[\"beijing\"]}}",
|
|
||||||
wantStatus: http.StatusForbidden,
|
|
||||||
wantExtractorCall: 0,
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, test := range tests {
|
|
||||||
test := test
|
|
||||||
t.Run(test.name, func(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
extractor := &fakeExtractor{}
|
|
||||||
handler := mustNewHandler(t, Config{BodyLimitBytes: 1024}, Dependencies{
|
|
||||||
Extractor: extractor,
|
|
||||||
Identity: fakeIdentityResolver{identity: Identity{
|
|
||||||
ClientID: "tenant-restricted",
|
|
||||||
Permissions: []string{authorization.DistributionExtract},
|
|
||||||
ClientPolicy: clientpolicy.Policy{
|
|
||||||
MaxExtractCount: 2,
|
|
||||||
AllowedUpstreams: []string{"provider-a"},
|
|
||||||
AllowedRegions: []string{"shanghai"},
|
|
||||||
},
|
|
||||||
}},
|
|
||||||
Readiness: fakeReadinessChecker{},
|
|
||||||
})
|
|
||||||
request := httptest.NewRequest(http.MethodPost, pathExtract, strings.NewReader(test.body))
|
|
||||||
request.Header.Set("Content-Type", httpapi.JSONContentType)
|
|
||||||
response := httptest.NewRecorder()
|
|
||||||
|
|
||||||
handler.ServeHTTP(response, request)
|
|
||||||
|
|
||||||
if response.Code != test.wantStatus || extractor.calls != test.wantExtractorCall {
|
|
||||||
t.Fatalf("response = status %d calls=%d, want status %d calls=%d",
|
|
||||||
response.Code, extractor.calls, test.wantStatus, test.wantExtractorCall)
|
|
||||||
}
|
|
||||||
if response.Code == http.StatusForbidden && !strings.Contains(response.Body.String(), "\"code\":\"FORBIDDEN\"") {
|
|
||||||
t.Fatalf("forbidden response = %s", response.Body.String())
|
|
||||||
}
|
|
||||||
if got := extractor.request.Filters.Upstreams; !slices.Equal(got, test.wantUpstreams) {
|
|
||||||
t.Fatalf("upstreams = %v, want %v", got, test.wantUpstreams)
|
|
||||||
}
|
|
||||||
if got := extractor.request.Filters.Regions; !slices.Equal(got, test.wantRegions) {
|
|
||||||
t.Fatalf("regions = %v, want %v", got, test.wantRegions)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestHandlerRejectsEmptyResolvedIdentity(t *testing.T) {
|
func TestHandlerRejectsEmptyResolvedIdentity(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
extractor := &fakeExtractor{}
|
extractor := &fakeExtractor{}
|
||||||
|
|||||||
@ -1,80 +0,0 @@
|
|||||||
package extraction
|
|
||||||
|
|
||||||
import (
|
|
||||||
"errors"
|
|
||||||
|
|
||||||
domain "proxy-pool/internal/domain/extraction"
|
|
||||||
)
|
|
||||||
|
|
||||||
// MetricResult is a fixed, low-cardinality classification of one extraction
|
|
||||||
// request after application-level validation, admission, and storage handling.
|
|
||||||
type MetricResult string
|
|
||||||
|
|
||||||
const (
|
|
||||||
MetricComplete MetricResult = "complete"
|
|
||||||
MetricPartial MetricResult = "partial"
|
|
||||||
MetricEmpty MetricResult = "empty"
|
|
||||||
MetricInsufficient MetricResult = "insufficient"
|
|
||||||
MetricIdempotencyConflict MetricResult = "idempotency_conflict"
|
|
||||||
MetricRateLimited MetricResult = "rate_limited"
|
|
||||||
MetricUnavailable MetricResult = "unavailable"
|
|
||||||
MetricInvalid MetricResult = "invalid"
|
|
||||||
MetricError MetricResult = "error"
|
|
||||||
)
|
|
||||||
|
|
||||||
// MetricsEvent contains only aggregate request and response counts. It must
|
|
||||||
// never contain a Client, Proxy, Upstream, filter, identifier, or error text.
|
|
||||||
type MetricsEvent struct {
|
|
||||||
Result MetricResult
|
|
||||||
Requested int
|
|
||||||
Returned int
|
|
||||||
}
|
|
||||||
|
|
||||||
// MetricsObserver receives one completed extraction request. Implementations
|
|
||||||
// must return promptly and must not perform network or persistent storage I/O.
|
|
||||||
type MetricsObserver interface {
|
|
||||||
ObserveExtraction(MetricsEvent)
|
|
||||||
}
|
|
||||||
|
|
||||||
func classifyMetricsEvent(response Response, resultErr error) MetricsEvent {
|
|
||||||
event := MetricsEvent{Requested: response.Requested, Returned: response.Returned}
|
|
||||||
if resultErr == nil {
|
|
||||||
switch {
|
|
||||||
case response.Returned == response.Requested:
|
|
||||||
event.Result = MetricComplete
|
|
||||||
case response.Returned > 0 && response.Returned < response.Requested:
|
|
||||||
event.Result = MetricPartial
|
|
||||||
case response.Returned == 0:
|
|
||||||
event.Result = MetricEmpty
|
|
||||||
default:
|
|
||||||
event.Result = MetricError
|
|
||||||
}
|
|
||||||
return event
|
|
||||||
}
|
|
||||||
switch {
|
|
||||||
case errors.Is(resultErr, domain.ErrInsufficientProxies):
|
|
||||||
event.Result = MetricInsufficient
|
|
||||||
case errors.Is(resultErr, domain.ErrIdempotencyConflict):
|
|
||||||
event.Result = MetricIdempotencyConflict
|
|
||||||
case errors.Is(resultErr, ErrAdmissionRejected):
|
|
||||||
event.Result = MetricRateLimited
|
|
||||||
case errors.Is(resultErr, ErrUnavailable):
|
|
||||||
event.Result = MetricUnavailable
|
|
||||||
case errors.Is(resultErr, ErrInvalidRequest), errors.Is(resultErr, ErrCountExceeded),
|
|
||||||
errors.Is(resultErr, ErrInvalidFulfillment), errors.Is(resultErr, domain.ErrInvalidCommand):
|
|
||||||
event.Result = MetricInvalid
|
|
||||||
default:
|
|
||||||
event.Result = MetricError
|
|
||||||
}
|
|
||||||
return event
|
|
||||||
}
|
|
||||||
|
|
||||||
func validMetricResult(result MetricResult) bool {
|
|
||||||
switch result {
|
|
||||||
case MetricComplete, MetricPartial, MetricEmpty, MetricInsufficient, MetricIdempotencyConflict,
|
|
||||||
MetricRateLimited, MetricUnavailable, MetricInvalid, MetricError:
|
|
||||||
return true
|
|
||||||
default:
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@ -73,16 +73,9 @@ type Service struct {
|
|||||||
policy Policy
|
policy Policy
|
||||||
admission admission.Admitter
|
admission admission.Admitter
|
||||||
now func() time.Time
|
now func() time.Time
|
||||||
metrics MetricsObserver
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func NewService(
|
func NewService(store domain.Store, policy Policy, admitter admission.Admitter, now func() time.Time) (*Service, error) {
|
||||||
store domain.Store,
|
|
||||||
policy Policy,
|
|
||||||
admitter admission.Admitter,
|
|
||||||
now func() time.Time,
|
|
||||||
metrics ...MetricsObserver,
|
|
||||||
) (*Service, error) {
|
|
||||||
if admitter == nil {
|
if admitter == nil {
|
||||||
return nil, fmt.Errorf("%w: admission is required", ErrInvalidServicePolicy)
|
return nil, fmt.Errorf("%w: admission is required", ErrInvalidServicePolicy)
|
||||||
}
|
}
|
||||||
@ -99,19 +92,11 @@ func NewService(
|
|||||||
if now == nil {
|
if now == nil {
|
||||||
now = time.Now
|
now = time.Now
|
||||||
}
|
}
|
||||||
if len(metrics) > 1 {
|
return &Service{store: store, policy: policy, admission: admitter, now: now}, nil
|
||||||
return nil, ErrInvalidServicePolicy
|
|
||||||
}
|
|
||||||
var observer MetricsObserver
|
|
||||||
if len(metrics) == 1 {
|
|
||||||
observer = metrics[0]
|
|
||||||
}
|
|
||||||
return &Service{store: store, policy: policy, admission: admitter, now: now, metrics: observer}, nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *Service) Extract(ctx context.Context, request Request) (response Response, resultErr error) {
|
func (s *Service) Extract(ctx context.Context, request Request) (Response, error) {
|
||||||
response = Response{RequestID: request.RequestID, Requested: request.Count}
|
response := Response{RequestID: request.RequestID, Requested: request.Count}
|
||||||
defer func() { s.observeMetrics(classifyMetricsEvent(response, resultErr)) }()
|
|
||||||
if request.RequestID == "" || (request.ClientID == "" && request.SourceIP == "") || request.Count <= 0 {
|
if request.RequestID == "" || (request.ClientID == "" && request.SourceIP == "") || request.Count <= 0 {
|
||||||
return response, ErrInvalidRequest
|
return response, ErrInvalidRequest
|
||||||
}
|
}
|
||||||
@ -191,13 +176,6 @@ func (s *Service) Extract(ctx context.Context, request Request) (response Respon
|
|||||||
return response, nil
|
return response, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *Service) observeMetrics(event MetricsEvent) {
|
|
||||||
if s == nil || s.metrics == nil || !validMetricResult(event.Result) {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
s.metrics.ObserveExtraction(event)
|
|
||||||
}
|
|
||||||
|
|
||||||
func admissionKey(request Request) string {
|
func admissionKey(request Request) string {
|
||||||
if request.ClientID != "" {
|
if request.ClientID != "" {
|
||||||
return "client:" + request.ClientID
|
return "client:" + request.ClientID
|
||||||
|
|||||||
@ -244,86 +244,6 @@ func TestServiceIdempotentReplayKeepsOriginalExtractionTime(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestServiceEmitsFixedMetricsForTerminalResults(t *testing.T) {
|
|
||||||
observer := &recordingMetricsObserver{}
|
|
||||||
policy := Policy{MaxCountPerRequest: 2, DefaultFulfillment: domain.Partial}
|
|
||||||
tests := []struct {
|
|
||||||
name string
|
|
||||||
request Request
|
|
||||||
result domain.Result
|
|
||||||
storeErr error
|
|
||||||
admitErr error
|
|
||||||
wantEvent MetricsEvent
|
|
||||||
}{
|
|
||||||
{
|
|
||||||
name: "complete",
|
|
||||||
request: Request{RequestID: "req-complete", ClientID: "client-a", Count: 2},
|
|
||||||
result: domain.Result{Requested: 2, Returned: 2, Items: []domain.Candidate{{}, {}}},
|
|
||||||
wantEvent: MetricsEvent{Result: MetricComplete, Requested: 2, Returned: 2},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "partial",
|
|
||||||
request: Request{RequestID: "req-partial", ClientID: "client-a", Count: 2},
|
|
||||||
result: domain.Result{Requested: 2, Returned: 1, Items: []domain.Candidate{{}}},
|
|
||||||
wantEvent: MetricsEvent{Result: MetricPartial, Requested: 2, Returned: 1},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "empty",
|
|
||||||
request: Request{RequestID: "req-empty", ClientID: "client-a", Count: 1},
|
|
||||||
result: domain.Result{Requested: 1},
|
|
||||||
wantEvent: MetricsEvent{Result: MetricEmpty, Requested: 1, Returned: 0},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "unexpected over delivery",
|
|
||||||
request: Request{RequestID: "req-over", ClientID: "client-a", Count: 1},
|
|
||||||
result: domain.Result{Requested: 1, Returned: 2, Items: []domain.Candidate{{}, {}}},
|
|
||||||
wantEvent: MetricsEvent{Result: MetricError, Requested: 1, Returned: 2},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "insufficient",
|
|
||||||
request: Request{RequestID: "req-insufficient", ClientID: "client-a", Count: 1},
|
|
||||||
storeErr: domain.ErrInsufficientProxies,
|
|
||||||
wantEvent: MetricsEvent{Result: MetricInsufficient, Requested: 1},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "idempotency conflict",
|
|
||||||
request: Request{RequestID: "req-conflict", ClientID: "client-a", Count: 1},
|
|
||||||
storeErr: domain.ErrIdempotencyConflict,
|
|
||||||
wantEvent: MetricsEvent{Result: MetricIdempotencyConflict, Requested: 1},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "rate limited",
|
|
||||||
request: Request{RequestID: "req-rate", ClientID: "client-a", Count: 1},
|
|
||||||
admitErr: errors.New("rate limited"),
|
|
||||||
wantEvent: MetricsEvent{Result: MetricRateLimited, Requested: 1},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "unavailable",
|
|
||||||
request: Request{RequestID: "req-unavailable", ClientID: "client-a", Count: 1},
|
|
||||||
storeErr: domain.ErrStoreUnavailable,
|
|
||||||
wantEvent: MetricsEvent{Result: MetricUnavailable, Requested: 1},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "invalid",
|
|
||||||
request: Request{RequestID: "req-invalid", Count: 1},
|
|
||||||
wantEvent: MetricsEvent{Result: MetricInvalid, Requested: 1},
|
|
||||||
},
|
|
||||||
}
|
|
||||||
for _, test := range tests {
|
|
||||||
t.Run(test.name, func(t *testing.T) {
|
|
||||||
service, err := NewService(&recordingStore{result: test.result, err: test.storeErr}, policy,
|
|
||||||
&recordingAdmission{err: test.admitErr}, time.Now, observer)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("NewService() error = %v", err)
|
|
||||||
}
|
|
||||||
_, _ = service.Extract(context.Background(), test.request)
|
|
||||||
if got := observer.events[len(observer.events)-1]; got != test.wantEvent {
|
|
||||||
t.Fatalf("metrics event = %+v, want %+v", got, test.wantEvent)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
type recordingStore struct {
|
type recordingStore struct {
|
||||||
command domain.Command
|
command domain.Command
|
||||||
result domain.Result
|
result domain.Result
|
||||||
@ -337,8 +257,6 @@ type recordingAdmission struct {
|
|||||||
calls int
|
calls int
|
||||||
}
|
}
|
||||||
|
|
||||||
type recordingMetricsObserver struct{ events []MetricsEvent }
|
|
||||||
|
|
||||||
type allowAllAdmission struct{}
|
type allowAllAdmission struct{}
|
||||||
|
|
||||||
func (allowAllAdmission) Admit(context.Context, string) error { return nil }
|
func (allowAllAdmission) Admit(context.Context, string) error { return nil }
|
||||||
@ -349,10 +267,6 @@ func (a *recordingAdmission) Admit(_ context.Context, key string) error {
|
|||||||
return a.err
|
return a.err
|
||||||
}
|
}
|
||||||
|
|
||||||
func (observer *recordingMetricsObserver) ObserveExtraction(event MetricsEvent) {
|
|
||||||
observer.events = append(observer.events, event)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *recordingStore) Extract(_ context.Context, command domain.Command) (domain.Result, error) {
|
func (s *recordingStore) Extract(_ context.Context, command domain.Command) (domain.Result, error) {
|
||||||
s.calls++
|
s.calls++
|
||||||
s.command = command
|
s.command = command
|
||||||
|
|||||||
@ -9,7 +9,6 @@ import (
|
|||||||
"time"
|
"time"
|
||||||
|
|
||||||
"proxy-pool/internal/config"
|
"proxy-pool/internal/config"
|
||||||
"proxy-pool/internal/domain/adminstate"
|
|
||||||
healthDomain "proxy-pool/internal/domain/health"
|
healthDomain "proxy-pool/internal/domain/health"
|
||||||
)
|
)
|
||||||
|
|
||||||
@ -170,26 +169,12 @@ type ConfiguredSchedulerRunner struct {
|
|||||||
// bounded BASIC scheduling without a Controller restart.
|
// bounded BASIC scheduling without a Controller restart.
|
||||||
type ConfiguredSchedulerSupervisor struct {
|
type ConfiguredSchedulerSupervisor struct {
|
||||||
configuration ConfigurationSource
|
configuration ConfigurationSource
|
||||||
state SchedulerStateSource
|
|
||||||
source UpstreamTaskSource
|
source UpstreamTaskSource
|
||||||
sink TaskSink
|
sink TaskSink
|
||||||
options SchedulerRunnerOptions
|
options SchedulerRunnerOptions
|
||||||
cursor atomic.Uint64
|
cursor atomic.Uint64
|
||||||
}
|
}
|
||||||
|
|
||||||
// SchedulerConfigurationSource returns a configuration and its Admin revision
|
|
||||||
// from one atomic publication. config.Store implements this interface.
|
|
||||||
type SchedulerConfigurationSource interface {
|
|
||||||
ConfigurationSource
|
|
||||||
Snapshot() (*config.Config, uint64)
|
|
||||||
}
|
|
||||||
|
|
||||||
// SchedulerStateSource provides the persisted Admin view that fences a
|
|
||||||
// scheduler from combining two different configuration revisions.
|
|
||||||
type SchedulerStateSource interface {
|
|
||||||
Snapshot(context.Context) (adminstate.Snapshot, error)
|
|
||||||
}
|
|
||||||
|
|
||||||
func NewConfiguredSchedulerRunner(
|
func NewConfiguredSchedulerRunner(
|
||||||
configuration ConfigurationSource,
|
configuration ConfigurationSource,
|
||||||
upstreamID string,
|
upstreamID string,
|
||||||
@ -211,22 +196,14 @@ func NewConfiguredSchedulerSupervisor(
|
|||||||
source UpstreamTaskSource,
|
source UpstreamTaskSource,
|
||||||
sink TaskSink,
|
sink TaskSink,
|
||||||
options SchedulerRunnerOptions,
|
options SchedulerRunnerOptions,
|
||||||
states ...SchedulerStateSource,
|
|
||||||
) (*ConfiguredSchedulerSupervisor, error) {
|
) (*ConfiguredSchedulerSupervisor, error) {
|
||||||
if nilInterface(configuration) || nilInterface(source) || nilInterface(sink) || options.PollInterval <= 0 ||
|
if nilInterface(configuration) || nilInterface(source) || nilInterface(sink) || options.PollInterval <= 0 ||
|
||||||
options.BatchSize <= 0 || options.Now == nil || len(states) > 1 || (len(states) == 1 && nilInterface(states[0])) {
|
options.BatchSize <= 0 || options.Now == nil {
|
||||||
return nil, ErrInvalidSchedulerRunner
|
return nil, ErrInvalidSchedulerRunner
|
||||||
}
|
}
|
||||||
supervisor := &ConfiguredSchedulerSupervisor{
|
return &ConfiguredSchedulerSupervisor{
|
||||||
configuration: configuration, source: source, sink: sink, options: options,
|
configuration: configuration, source: source, sink: sink, options: options,
|
||||||
}
|
}, nil
|
||||||
if len(states) == 1 {
|
|
||||||
if _, ok := configuration.(SchedulerConfigurationSource); !ok {
|
|
||||||
return nil, ErrInvalidSchedulerRunner
|
|
||||||
}
|
|
||||||
supervisor.state = states[0]
|
|
||||||
}
|
|
||||||
return supervisor, nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
type TickResult struct {
|
type TickResult struct {
|
||||||
@ -268,10 +245,11 @@ func (supervisor *ConfiguredSchedulerSupervisor) Tick(ctx context.Context) (Tick
|
|||||||
if err := ctx.Err(); err != nil {
|
if err := ctx.Err(); err != nil {
|
||||||
return TickResult{}, err
|
return TickResult{}, err
|
||||||
}
|
}
|
||||||
configuration, names, routings, err := supervisor.effectiveConfiguration(ctx)
|
configuration := supervisor.configuration.Current()
|
||||||
if err != nil {
|
if configuration == nil {
|
||||||
return TickResult{}, err
|
return TickResult{}, ErrInvalidSchedulerRunner
|
||||||
}
|
}
|
||||||
|
names := enabledUpstreamNames(configuration)
|
||||||
result := TickResult{}
|
result := TickResult{}
|
||||||
for _, name := range names {
|
for _, name := range names {
|
||||||
planner, configured, err := configuredPlannerForConfig(configuration, name)
|
planner, configured, err := configuredPlannerForConfig(configuration, name)
|
||||||
@ -286,7 +264,7 @@ func (supervisor *ConfiguredSchedulerSupervisor) Tick(ctx context.Context) (Tick
|
|||||||
return TickResult{}, err
|
return TickResult{}, err
|
||||||
}
|
}
|
||||||
check := config.EffectiveCheck(configuration.Defaults.Check, configuration.Upstreams[name].Check)
|
check := config.EffectiveCheck(configuration.Defaults.Check, configuration.Upstreams[name].Check)
|
||||||
groups, err := supervisor.upstreamDueGroups(basicSource, name, check.URLs, configuredTargetProfiles(configuration, name, routings))
|
groups, err := supervisor.upstreamDueGroups(basicSource, name, check.URLs, configuredTargetProfiles(configuration, name))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return TickResult{}, err
|
return TickResult{}, err
|
||||||
}
|
}
|
||||||
@ -311,91 +289,6 @@ func (supervisor *ConfiguredSchedulerSupervisor) Tick(ctx context.Context) (Tick
|
|||||||
return result, nil
|
return result, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// effectiveConfiguration returns the effective Upstream and Routing state for
|
|
||||||
// one scheduler tick. When Admin state is present, it requires the
|
|
||||||
// configuration and state snapshots to share a revision; an incomplete or
|
|
||||||
// torn view schedules nothing.
|
|
||||||
func (supervisor *ConfiguredSchedulerSupervisor) effectiveConfiguration(
|
|
||||||
ctx context.Context,
|
|
||||||
) (*config.Config, []string, map[string]adminstate.RoutingState, error) {
|
|
||||||
if supervisor == nil || nilInterface(supervisor.configuration) {
|
|
||||||
return nil, nil, nil, ErrInvalidSchedulerRunner
|
|
||||||
}
|
|
||||||
if supervisor.state == nil {
|
|
||||||
configuration := supervisor.configuration.Current()
|
|
||||||
if configuration == nil {
|
|
||||||
return nil, nil, nil, ErrInvalidSchedulerRunner
|
|
||||||
}
|
|
||||||
return configuration, enabledUpstreamNames(configuration), nil, nil
|
|
||||||
}
|
|
||||||
configurationSource, ok := supervisor.configuration.(SchedulerConfigurationSource)
|
|
||||||
if !ok || nilInterface(configurationSource) || nilInterface(supervisor.state) {
|
|
||||||
return nil, nil, nil, ErrInvalidSchedulerRunner
|
|
||||||
}
|
|
||||||
configuration, revision := configurationSource.Snapshot()
|
|
||||||
if configuration == nil || revision == 0 {
|
|
||||||
return nil, nil, nil, ErrInvalidSchedulerRunner
|
|
||||||
}
|
|
||||||
snapshot, err := supervisor.state.Snapshot(ctx)
|
|
||||||
if err != nil {
|
|
||||||
return nil, nil, nil, err
|
|
||||||
}
|
|
||||||
if snapshot.Config == nil || snapshot.Config.Revision != revision {
|
|
||||||
return nil, nil, nil, ErrInvalidSchedulerRunner
|
|
||||||
}
|
|
||||||
states, err := newSchedulerAdminStates(configuration, snapshot)
|
|
||||||
if err != nil {
|
|
||||||
return nil, nil, nil, err
|
|
||||||
}
|
|
||||||
names, err := enabledUpstreamNamesForState(configuration, states.upstreams)
|
|
||||||
if err != nil {
|
|
||||||
return nil, nil, nil, err
|
|
||||||
}
|
|
||||||
return configuration, names, states.routings, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
type schedulerAdminStates struct {
|
|
||||||
upstreams map[string]adminstate.UpstreamState
|
|
||||||
routings map[string]adminstate.RoutingState
|
|
||||||
}
|
|
||||||
|
|
||||||
func newSchedulerAdminStates(
|
|
||||||
configuration *config.Config,
|
|
||||||
snapshot adminstate.Snapshot,
|
|
||||||
) (schedulerAdminStates, error) {
|
|
||||||
if configuration == nil {
|
|
||||||
return schedulerAdminStates{}, ErrInvalidSchedulerRunner
|
|
||||||
}
|
|
||||||
states := schedulerAdminStates{
|
|
||||||
upstreams: make(map[string]adminstate.UpstreamState, len(snapshot.Upstreams)),
|
|
||||||
routings: make(map[string]adminstate.RoutingState, len(snapshot.Routings)),
|
|
||||||
}
|
|
||||||
for _, upstream := range snapshot.Upstreams {
|
|
||||||
if upstream.Name == "" || upstream.Revision == 0 {
|
|
||||||
return schedulerAdminStates{}, ErrInvalidSchedulerRunner
|
|
||||||
}
|
|
||||||
if _, duplicate := states.upstreams[upstream.Name]; duplicate {
|
|
||||||
return schedulerAdminStates{}, ErrInvalidSchedulerRunner
|
|
||||||
}
|
|
||||||
states.upstreams[upstream.Name] = upstream
|
|
||||||
}
|
|
||||||
for _, routing := range snapshot.Routings {
|
|
||||||
if routing.Name == "" || routing.Revision == 0 {
|
|
||||||
return schedulerAdminStates{}, ErrInvalidSchedulerRunner
|
|
||||||
}
|
|
||||||
if _, duplicate := states.routings[routing.Name]; duplicate {
|
|
||||||
return schedulerAdminStates{}, ErrInvalidSchedulerRunner
|
|
||||||
}
|
|
||||||
states.routings[routing.Name] = routing
|
|
||||||
}
|
|
||||||
for _, routing := range configuration.Routing {
|
|
||||||
if _, exists := states.routings[routing.Name]; !exists {
|
|
||||||
return schedulerAdminStates{}, ErrInvalidSchedulerRunner
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return states, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (supervisor *ConfiguredSchedulerSupervisor) upstreamDueGroups(
|
func (supervisor *ConfiguredSchedulerSupervisor) upstreamDueGroups(
|
||||||
basic DueSource,
|
basic DueSource,
|
||||||
upstreamID string,
|
upstreamID string,
|
||||||
@ -433,17 +326,13 @@ type configuredTargetProfile struct {
|
|||||||
TargetURL string
|
TargetURL string
|
||||||
}
|
}
|
||||||
|
|
||||||
func configuredTargetProfiles(
|
func configuredTargetProfiles(configuration *config.Config, upstreamID string) []configuredTargetProfile {
|
||||||
configuration *config.Config,
|
|
||||||
upstreamID string,
|
|
||||||
routings map[string]adminstate.RoutingState,
|
|
||||||
) []configuredTargetProfile {
|
|
||||||
if configuration == nil || upstreamID == "" {
|
if configuration == nil || upstreamID == "" {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
profiles := make([]configuredTargetProfile, 0)
|
profiles := make([]configuredTargetProfile, 0)
|
||||||
for _, route := range configuration.Routing {
|
for _, route := range configuration.Routing {
|
||||||
if !route.Enabled || !containsString(route.Upstreams, upstreamID) || (routings != nil && !routings[route.Name].Enabled) {
|
if !route.Enabled || !containsString(route.Upstreams, upstreamID) {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
for _, targetURL := range route.Check.Targets {
|
for _, targetURL := range route.Check.Targets {
|
||||||
@ -602,27 +491,6 @@ func enabledUpstreamNames(configuration *config.Config) []string {
|
|||||||
return names
|
return names
|
||||||
}
|
}
|
||||||
|
|
||||||
func enabledUpstreamNamesForState(
|
|
||||||
configuration *config.Config,
|
|
||||||
states map[string]adminstate.UpstreamState,
|
|
||||||
) ([]string, error) {
|
|
||||||
if configuration == nil {
|
|
||||||
return nil, ErrInvalidSchedulerRunner
|
|
||||||
}
|
|
||||||
names := make([]string, 0, len(configuration.Upstreams))
|
|
||||||
for name, upstream := range configuration.Upstreams {
|
|
||||||
state, exists := states[name]
|
|
||||||
if !exists {
|
|
||||||
return nil, ErrInvalidSchedulerRunner
|
|
||||||
}
|
|
||||||
if upstream.Enabled && state.Enabled {
|
|
||||||
names = append(names, name)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
sort.Strings(names)
|
|
||||||
return names, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func runScheduler(ctx context.Context, pollInterval time.Duration, tick func(context.Context) (TickResult, error)) error {
|
func runScheduler(ctx context.Context, pollInterval time.Duration, tick func(context.Context) (TickResult, error)) error {
|
||||||
if ctx == nil || pollInterval <= 0 || tick == nil {
|
if ctx == nil || pollInterval <= 0 || tick == nil {
|
||||||
return ErrInvalidSchedulerRunner
|
return ErrInvalidSchedulerRunner
|
||||||
|
|||||||
@ -7,7 +7,6 @@ import (
|
|||||||
"time"
|
"time"
|
||||||
|
|
||||||
"proxy-pool/internal/config"
|
"proxy-pool/internal/config"
|
||||||
"proxy-pool/internal/domain/adminstate"
|
|
||||||
healthDomain "proxy-pool/internal/domain/health"
|
healthDomain "proxy-pool/internal/domain/health"
|
||||||
proxyDomain "proxy-pool/internal/domain/proxy"
|
proxyDomain "proxy-pool/internal/domain/proxy"
|
||||||
)
|
)
|
||||||
@ -165,160 +164,6 @@ func TestConfiguredSchedulerSupervisorDiscoversNewEnabledUpstream(t *testing.T)
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestConfiguredSchedulerSupervisorSkipsAdminDisabledUpstream(t *testing.T) {
|
|
||||||
now := time.Date(2026, 8, 7, 10, 0, 0, 0, time.UTC)
|
|
||||||
configuration := schedulerConfiguration(now, "provider-a", "provider-b")
|
|
||||||
configuration.Defaults.Check.URLs = []string{"https://egress.example/identity"}
|
|
||||||
configuration.Routing = []config.Routing{{
|
|
||||||
Name: "checkout", Enabled: true, Upstreams: []string{"provider-a"},
|
|
||||||
Check: config.RoutingCheck{Targets: []string{"https://checkout.example/health"}},
|
|
||||||
}}
|
|
||||||
source := &upstreamTaskSourceStub{sources: map[string]*dueSourceStub{
|
|
||||||
"provider-a": {
|
|
||||||
candidates: []Candidate{{ProxyID: "proxy-a", State: proxyDomain.StateFetched, Level: healthDomain.LevelBasic, DueAt: now}},
|
|
||||||
egressCandidates: map[string][]Candidate{"https://egress.example/identity": {
|
|
||||||
{ProxyID: "egress-a", State: proxyDomain.StateFetched, Level: healthDomain.LevelEgress, TargetURL: "https://egress.example/identity", DueAt: now},
|
|
||||||
}},
|
|
||||||
targetCandidates: map[string][]Candidate{targetCandidateKey("checkout", "https://checkout.example/health"): {
|
|
||||||
{ProxyID: "target-a", State: proxyDomain.StateFetched, Level: healthDomain.LevelTarget, RoutingName: "checkout", TargetURL: "https://checkout.example/health", DueAt: now},
|
|
||||||
}},
|
|
||||||
},
|
|
||||||
"provider-b": {candidates: []Candidate{{ProxyID: "proxy-b", State: proxyDomain.StateFetched, Level: healthDomain.LevelBasic, DueAt: now}}},
|
|
||||||
}}
|
|
||||||
state := schedulerStateStub{snapshot: adminstate.Snapshot{
|
|
||||||
Config: &adminstate.ConfigRevision{Revision: 42},
|
|
||||||
Upstreams: []adminstate.UpstreamState{
|
|
||||||
{Name: "provider-a", Enabled: false, Revision: 11},
|
|
||||||
{Name: "provider-b", Enabled: true, Revision: 12},
|
|
||||||
},
|
|
||||||
Routings: []adminstate.RoutingState{{Name: "checkout", Enabled: true, Revision: 13}},
|
|
||||||
}}
|
|
||||||
sink := &taskSinkStub{}
|
|
||||||
supervisor, err := NewConfiguredSchedulerSupervisor(
|
|
||||||
&versionedConfigurationSourceStub{configuration: configuration, revision: 42}, source, sink,
|
|
||||||
SchedulerRunnerOptions{PollInterval: time.Second, BatchSize: 16, Now: func() time.Time { return now }}, state,
|
|
||||||
)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("NewConfiguredSchedulerSupervisor() = %v", err)
|
|
||||||
}
|
|
||||||
result, err := supervisor.Tick(context.Background())
|
|
||||||
if err != nil || result != (TickResult{Planned: 1, Offered: 1}) || source.sources["provider-a"].inFlightCalls != 0 ||
|
|
||||||
source.sources["provider-a"].dueCalls != 0 || source.sources["provider-a"].egressCalls != 0 ||
|
|
||||||
source.sources["provider-a"].targetCalls != 0 || source.sources["provider-b"].dueCalls != 1 ||
|
|
||||||
len(sink.tasks) != 1 || sink.tasks[0].Candidate.ProxyID != "proxy-b" {
|
|
||||||
t.Fatalf("Tick() = (%+v, %v); source=%+v sink=%+v", result, err, source, sink.tasks)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestConfiguredSchedulerSupervisorSkipsAdminDisabledRoutingTarget(t *testing.T) {
|
|
||||||
now := time.Date(2026, 8, 7, 10, 15, 0, 0, time.UTC)
|
|
||||||
configuration := schedulerConfiguration(now, "provider-a")
|
|
||||||
configuration.Routing = []config.Routing{{
|
|
||||||
Name: "checkout", Enabled: true, Upstreams: []string{"provider-a"},
|
|
||||||
Check: config.RoutingCheck{Targets: []string{"https://checkout.example/health"}},
|
|
||||||
}}
|
|
||||||
source := &upstreamTaskSourceStub{sources: map[string]*dueSourceStub{
|
|
||||||
"provider-a": {
|
|
||||||
candidates: []Candidate{{ProxyID: "proxy-a", State: proxyDomain.StateFetched, Level: healthDomain.LevelBasic, DueAt: now}},
|
|
||||||
targetCandidates: map[string][]Candidate{targetCandidateKey("checkout", "https://checkout.example/health"): {
|
|
||||||
{ProxyID: "target-a", State: proxyDomain.StateFetched, Level: healthDomain.LevelTarget, RoutingName: "checkout", TargetURL: "https://checkout.example/health", DueAt: now},
|
|
||||||
}},
|
|
||||||
},
|
|
||||||
}}
|
|
||||||
supervisor, err := NewConfiguredSchedulerSupervisor(
|
|
||||||
&versionedConfigurationSourceStub{configuration: configuration, revision: 42}, source, &taskSinkStub{},
|
|
||||||
SchedulerRunnerOptions{PollInterval: time.Second, BatchSize: 16, Now: func() time.Time { return now }},
|
|
||||||
schedulerStateStub{snapshot: adminstate.Snapshot{
|
|
||||||
Config: &adminstate.ConfigRevision{Revision: 42},
|
|
||||||
Upstreams: []adminstate.UpstreamState{{Name: "provider-a", Enabled: true, Revision: 11}},
|
|
||||||
Routings: []adminstate.RoutingState{{Name: "checkout", Enabled: false, Revision: 12}},
|
|
||||||
}},
|
|
||||||
)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("NewConfiguredSchedulerSupervisor() = %v", err)
|
|
||||||
}
|
|
||||||
if result, err := supervisor.Tick(context.Background()); err != nil || result != (TickResult{Planned: 1, Offered: 1}) ||
|
|
||||||
source.sources["provider-a"].dueCalls != 1 || source.sources["provider-a"].targetCalls != 0 {
|
|
||||||
t.Fatalf("Tick() = (%+v, %v); source=%+v", result, err, source.sources["provider-a"])
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestConfiguredSchedulerSupervisorFailsClosedOnAdminRevisionMismatch(t *testing.T) {
|
|
||||||
now := time.Date(2026, 8, 7, 10, 30, 0, 0, time.UTC)
|
|
||||||
configuration := schedulerConfiguration(now, "provider-a")
|
|
||||||
source := &upstreamTaskSourceStub{sources: map[string]*dueSourceStub{
|
|
||||||
"provider-a": {candidates: []Candidate{{ProxyID: "proxy-a", State: proxyDomain.StateFetched, Level: healthDomain.LevelBasic, DueAt: now}}},
|
|
||||||
}}
|
|
||||||
supervisor, err := NewConfiguredSchedulerSupervisor(
|
|
||||||
&versionedConfigurationSourceStub{configuration: configuration, revision: 42}, source, &taskSinkStub{},
|
|
||||||
SchedulerRunnerOptions{PollInterval: time.Second, BatchSize: 16, Now: func() time.Time { return now }},
|
|
||||||
schedulerStateStub{snapshot: adminstate.Snapshot{Config: &adminstate.ConfigRevision{Revision: 41}}},
|
|
||||||
)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("NewConfiguredSchedulerSupervisor() = %v", err)
|
|
||||||
}
|
|
||||||
if result, err := supervisor.Tick(context.Background()); !errors.Is(err, ErrInvalidSchedulerRunner) || result != (TickResult{}) ||
|
|
||||||
source.sources["provider-a"].inFlightCalls != 0 || source.sources["provider-a"].dueCalls != 0 {
|
|
||||||
t.Fatalf("Tick() = (%+v, %v); source=%+v", result, err, source.sources["provider-a"])
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestConfiguredSchedulerSupervisorFailsClosedOnIncompleteAdminState(t *testing.T) {
|
|
||||||
now := time.Date(2026, 8, 7, 10, 45, 0, 0, time.UTC)
|
|
||||||
configuration := schedulerConfiguration(now, "provider-a")
|
|
||||||
source := &upstreamTaskSourceStub{sources: map[string]*dueSourceStub{
|
|
||||||
"provider-a": {candidates: []Candidate{{ProxyID: "proxy-a", State: proxyDomain.StateFetched, Level: healthDomain.LevelBasic, DueAt: now}}},
|
|
||||||
}}
|
|
||||||
supervisor, err := NewConfiguredSchedulerSupervisor(
|
|
||||||
&versionedConfigurationSourceStub{configuration: configuration, revision: 42}, source, &taskSinkStub{},
|
|
||||||
SchedulerRunnerOptions{PollInterval: time.Second, BatchSize: 16, Now: func() time.Time { return now }},
|
|
||||||
schedulerStateStub{snapshot: adminstate.Snapshot{Config: &adminstate.ConfigRevision{Revision: 42}}},
|
|
||||||
)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("NewConfiguredSchedulerSupervisor() = %v", err)
|
|
||||||
}
|
|
||||||
if result, err := supervisor.Tick(context.Background()); !errors.Is(err, ErrInvalidSchedulerRunner) || result != (TickResult{}) ||
|
|
||||||
source.sources["provider-a"].inFlightCalls != 0 || source.sources["provider-a"].dueCalls != 0 {
|
|
||||||
t.Fatalf("Tick() = (%+v, %v); source=%+v", result, err, source.sources["provider-a"])
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestConfiguredSchedulerSupervisorFailsClosedOnMissingAdminRoutingState(t *testing.T) {
|
|
||||||
now := time.Date(2026, 8, 7, 10, 50, 0, 0, time.UTC)
|
|
||||||
configuration := schedulerConfiguration(now, "provider-a")
|
|
||||||
configuration.Routing = []config.Routing{{Name: "checkout", Enabled: true, Upstreams: []string{"provider-a"}}}
|
|
||||||
source := &upstreamTaskSourceStub{sources: map[string]*dueSourceStub{
|
|
||||||
"provider-a": {candidates: []Candidate{{ProxyID: "proxy-a", State: proxyDomain.StateFetched, Level: healthDomain.LevelBasic, DueAt: now}}},
|
|
||||||
}}
|
|
||||||
supervisor, err := NewConfiguredSchedulerSupervisor(
|
|
||||||
&versionedConfigurationSourceStub{configuration: configuration, revision: 42}, source, &taskSinkStub{},
|
|
||||||
SchedulerRunnerOptions{PollInterval: time.Second, BatchSize: 16, Now: func() time.Time { return now }},
|
|
||||||
schedulerStateStub{snapshot: adminstate.Snapshot{
|
|
||||||
Config: &adminstate.ConfigRevision{Revision: 42},
|
|
||||||
Upstreams: []adminstate.UpstreamState{{Name: "provider-a", Enabled: true, Revision: 11}},
|
|
||||||
}},
|
|
||||||
)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("NewConfiguredSchedulerSupervisor() = %v", err)
|
|
||||||
}
|
|
||||||
if result, err := supervisor.Tick(context.Background()); !errors.Is(err, ErrInvalidSchedulerRunner) || result != (TickResult{}) ||
|
|
||||||
source.sources["provider-a"].inFlightCalls != 0 || source.sources["provider-a"].dueCalls != 0 {
|
|
||||||
t.Fatalf("Tick() = (%+v, %v); source=%+v", result, err, source.sources["provider-a"])
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestConfiguredSchedulerSupervisorRequiresVersionedConfigurationWithAdminState(t *testing.T) {
|
|
||||||
now := time.Date(2026, 8, 7, 11, 0, 0, 0, time.UTC)
|
|
||||||
supervisor, err := NewConfiguredSchedulerSupervisor(
|
|
||||||
&configurationSourceStub{configuration: schedulerConfiguration(now, "provider-a")},
|
|
||||||
&upstreamTaskSourceStub{sources: map[string]*dueSourceStub{"provider-a": {}}}, &taskSinkStub{},
|
|
||||||
SchedulerRunnerOptions{PollInterval: time.Second, BatchSize: 1, Now: func() time.Time { return now }}, schedulerStateStub{},
|
|
||||||
)
|
|
||||||
if !errors.Is(err, ErrInvalidSchedulerRunner) || supervisor != nil {
|
|
||||||
t.Fatalf("NewConfiguredSchedulerSupervisor(non-versioned configuration) = (%v, %v)", supervisor, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestConfiguredSchedulerSupervisorSchedulesBoundedEgressGroups(t *testing.T) {
|
func TestConfiguredSchedulerSupervisorSchedulesBoundedEgressGroups(t *testing.T) {
|
||||||
now := time.Date(2026, 8, 2, 11, 0, 0, 0, time.UTC)
|
now := time.Date(2026, 8, 2, 11, 0, 0, 0, time.UTC)
|
||||||
configuration := &config.Config{
|
configuration := &config.Config{
|
||||||
@ -465,38 +310,6 @@ func TestConfiguredSchedulerSupervisorNeverExceedsTotalBatchAcrossGroups(t *test
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func schedulerConfiguration(now time.Time, upstreams ...string) *config.Config {
|
|
||||||
configured := &config.Config{
|
|
||||||
Defaults: config.Defaults{Check: config.Check{
|
|
||||||
Interval: config.Duration(time.Minute), MaxInFlight: 2, Timeout: config.Duration(time.Second), MaxAttempts: 1,
|
|
||||||
}},
|
|
||||||
Upstreams: make(map[string]config.Upstream, len(upstreams)),
|
|
||||||
}
|
|
||||||
for _, upstream := range upstreams {
|
|
||||||
configured.Upstreams[upstream] = config.Upstream{Enabled: true}
|
|
||||||
}
|
|
||||||
return configured
|
|
||||||
}
|
|
||||||
|
|
||||||
type versionedConfigurationSourceStub struct {
|
|
||||||
configuration *config.Config
|
|
||||||
revision uint64
|
|
||||||
}
|
|
||||||
|
|
||||||
func (source *versionedConfigurationSourceStub) Current() *config.Config {
|
|
||||||
return source.configuration
|
|
||||||
}
|
|
||||||
|
|
||||||
func (source *versionedConfigurationSourceStub) Snapshot() (*config.Config, uint64) {
|
|
||||||
return source.configuration, source.revision
|
|
||||||
}
|
|
||||||
|
|
||||||
type schedulerStateStub struct{ snapshot adminstate.Snapshot }
|
|
||||||
|
|
||||||
func (source schedulerStateStub) Snapshot(context.Context) (adminstate.Snapshot, error) {
|
|
||||||
return source.snapshot, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
type dueSourceStub struct {
|
type dueSourceStub struct {
|
||||||
inFlight int
|
inFlight int
|
||||||
candidates []Candidate
|
candidates []Candidate
|
||||||
|
|||||||
@ -7,7 +7,6 @@ import (
|
|||||||
|
|
||||||
"proxy-pool/internal/config"
|
"proxy-pool/internal/config"
|
||||||
"proxy-pool/internal/domain/activitypool"
|
"proxy-pool/internal/domain/activitypool"
|
||||||
healthDomain "proxy-pool/internal/domain/health"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
var ErrInvalidUnhealthyReaper = errors.New("invalid unhealthy reaper")
|
var ErrInvalidUnhealthyReaper = errors.New("invalid unhealthy reaper")
|
||||||
@ -16,7 +15,6 @@ type UnhealthyReaperOptions struct {
|
|||||||
PollInterval time.Duration
|
PollInterval time.Duration
|
||||||
BatchSize int
|
BatchSize int
|
||||||
Now func() time.Time
|
Now func() time.Time
|
||||||
Metrics healthDomain.DrainMetricsObserver
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// ConfiguredUnhealthyReaper applies the current upstream grace policies to a
|
// ConfiguredUnhealthyReaper applies the current upstream grace policies to a
|
||||||
@ -64,23 +62,12 @@ func (reaper *ConfiguredUnhealthyReaper) Tick(ctx context.Context) (activitypool
|
|||||||
}
|
}
|
||||||
drainer, supported := reaper.store.(activitypool.UnhealthyDrainStarter)
|
drainer, supported := reaper.store.(activitypool.UnhealthyDrainStarter)
|
||||||
if !supported || nilInterface(drainer) {
|
if !supported || nilInterface(drainer) {
|
||||||
if reaper.options.Metrics != nil {
|
|
||||||
reaper.options.Metrics.ObserveDrain(healthDomain.DrainReasonUnhealthy, len(result.DrainCandidates), 0)
|
|
||||||
}
|
|
||||||
return result, nil
|
return result, nil
|
||||||
}
|
}
|
||||||
started := 0
|
|
||||||
for _, candidate := range result.DrainCandidates {
|
for _, candidate := range result.DrainCandidates {
|
||||||
didStart, err := drainer.BeginUnhealthyDrain(ctx, utcNow, candidate)
|
if _, err := drainer.BeginUnhealthyDrain(ctx, utcNow, candidate); err != nil {
|
||||||
if err != nil {
|
|
||||||
return result, err
|
return result, err
|
||||||
}
|
}
|
||||||
if didStart {
|
|
||||||
started++
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if reaper.options.Metrics != nil {
|
|
||||||
reaper.options.Metrics.ObserveDrain(healthDomain.DrainReasonUnhealthy, len(result.DrainCandidates), started)
|
|
||||||
}
|
}
|
||||||
return result, nil
|
return result, nil
|
||||||
}
|
}
|
||||||
|
|||||||
@ -7,7 +7,6 @@ import (
|
|||||||
|
|
||||||
"proxy-pool/internal/config"
|
"proxy-pool/internal/config"
|
||||||
"proxy-pool/internal/domain/activitypool"
|
"proxy-pool/internal/domain/activitypool"
|
||||||
healthDomain "proxy-pool/internal/domain/health"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
type unhealthyRemoverStub struct {
|
type unhealthyRemoverStub struct {
|
||||||
@ -86,12 +85,11 @@ func TestConfiguredUnhealthyReaperStartsBoundedDrainCandidates(t *testing.T) {
|
|||||||
store := &unhealthyRemoverStub{result: activitypool.UnhealthySweepResult{
|
store := &unhealthyRemoverStub{result: activitypool.UnhealthySweepResult{
|
||||||
DeferredOwned: 1, DrainCandidates: []activitypool.UnhealthyDrainCandidate{candidate},
|
DeferredOwned: 1, DrainCandidates: []activitypool.UnhealthyDrainCandidate{candidate},
|
||||||
}}
|
}}
|
||||||
metrics := &recordingDrainMetrics{}
|
|
||||||
reaper, err := NewConfiguredUnhealthyReaper(&configurationSourceStub{configuration: &config.Config{
|
reaper, err := NewConfiguredUnhealthyReaper(&configurationSourceStub{configuration: &config.Config{
|
||||||
Upstreams: map[string]config.Upstream{
|
Upstreams: map[string]config.Upstream{
|
||||||
"provider-a": {Enabled: true, Check: config.Check{UnhealthyRemoveAfter: config.Duration(time.Minute)}},
|
"provider-a": {Enabled: true, Check: config.Check{UnhealthyRemoveAfter: config.Duration(time.Minute)}},
|
||||||
},
|
},
|
||||||
}}, store, UnhealthyReaperOptions{PollInterval: time.Second, BatchSize: 4, Now: func() time.Time { return now }, Metrics: metrics})
|
}}, store, UnhealthyReaperOptions{PollInterval: time.Second, BatchSize: 4, Now: func() time.Time { return now }})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("NewConfiguredUnhealthyReaper() = %v", err)
|
t.Fatalf("NewConfiguredUnhealthyReaper() = %v", err)
|
||||||
}
|
}
|
||||||
@ -102,9 +100,4 @@ func TestConfiguredUnhealthyReaperStartsBoundedDrainCandidates(t *testing.T) {
|
|||||||
if store.drainCalls != 1 || !store.drainNow.Equal(now) || len(store.candidates) != 1 || store.candidates[0] != candidate {
|
if store.drainCalls != 1 || !store.drainNow.Equal(now) || len(store.candidates) != 1 || store.candidates[0] != candidate {
|
||||||
t.Fatalf("BeginUnhealthyDrain() calls=%d now=%v candidates=%+v", store.drainCalls, store.drainNow, store.candidates)
|
t.Fatalf("BeginUnhealthyDrain() calls=%d now=%v candidates=%+v", store.drainCalls, store.drainNow, store.candidates)
|
||||||
}
|
}
|
||||||
if len(metrics.records) != 1 || metrics.records[0] != (drainMetricRecord{
|
|
||||||
reason: healthDomain.DrainReasonUnhealthy, candidates: 1, started: 1,
|
|
||||||
}) {
|
|
||||||
t.Fatalf("Drain metrics = %+v", metrics.records)
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|||||||
@ -1,211 +0,0 @@
|
|||||||
package health
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"errors"
|
|
||||||
"sort"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"proxy-pool/internal/config"
|
|
||||||
"proxy-pool/internal/domain/activitypool"
|
|
||||||
"proxy-pool/internal/domain/adminstate"
|
|
||||||
healthDomain "proxy-pool/internal/domain/health"
|
|
||||||
)
|
|
||||||
|
|
||||||
var ErrInvalidUpstreamDrainReaper = errors.New("invalid upstream drain reaper")
|
|
||||||
|
|
||||||
// UpstreamDrainConfigurationSource returns one configuration and revision from
|
|
||||||
// the same atomic publication. config.Store implements this interface.
|
|
||||||
type UpstreamDrainConfigurationSource interface {
|
|
||||||
Snapshot() (*config.Config, uint64)
|
|
||||||
}
|
|
||||||
|
|
||||||
type UpstreamDrainStateSource interface {
|
|
||||||
Snapshot(context.Context) (adminstate.Snapshot, error)
|
|
||||||
}
|
|
||||||
|
|
||||||
// EffectiveUpstreamDrainPolicySource merges static configuration with optional
|
|
||||||
// Admin state. With Admin enabled it uses the same revision equality rule as
|
|
||||||
// Worker routing, so a torn config/state view produces no policy publication.
|
|
||||||
type EffectiveUpstreamDrainPolicySource struct {
|
|
||||||
configuration UpstreamDrainConfigurationSource
|
|
||||||
state UpstreamDrainStateSource
|
|
||||||
}
|
|
||||||
|
|
||||||
func NewEffectiveUpstreamDrainPolicySource(
|
|
||||||
configuration UpstreamDrainConfigurationSource,
|
|
||||||
states ...UpstreamDrainStateSource,
|
|
||||||
) (*EffectiveUpstreamDrainPolicySource, error) {
|
|
||||||
if nilInterface(configuration) || len(states) > 1 || (len(states) == 1 && nilInterface(states[0])) {
|
|
||||||
return nil, ErrInvalidUpstreamDrainReaper
|
|
||||||
}
|
|
||||||
source := &EffectiveUpstreamDrainPolicySource{configuration: configuration}
|
|
||||||
if len(states) == 1 {
|
|
||||||
source.state = states[0]
|
|
||||||
}
|
|
||||||
return source, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (source *EffectiveUpstreamDrainPolicySource) ReadUpstreamDrainPolicies(
|
|
||||||
ctx context.Context,
|
|
||||||
) ([]activitypool.UpstreamDrainPolicy, error) {
|
|
||||||
if ctx == nil || source == nil || nilInterface(source.configuration) {
|
|
||||||
return nil, ErrInvalidUpstreamDrainReaper
|
|
||||||
}
|
|
||||||
if err := ctx.Err(); err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
configuration, revision := source.configuration.Snapshot()
|
|
||||||
if configuration == nil {
|
|
||||||
return nil, ErrInvalidUpstreamDrainReaper
|
|
||||||
}
|
|
||||||
states := make(map[string]adminstate.UpstreamState, len(configuration.Upstreams))
|
|
||||||
if source.state != nil {
|
|
||||||
snapshot, err := source.state.Snapshot(ctx)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
if snapshot.Config == nil || snapshot.Config.Revision != revision {
|
|
||||||
return nil, ErrInvalidUpstreamDrainReaper
|
|
||||||
}
|
|
||||||
for _, upstream := range snapshot.Upstreams {
|
|
||||||
if upstream.Name == "" || upstream.Revision == 0 {
|
|
||||||
return nil, ErrInvalidUpstreamDrainReaper
|
|
||||||
}
|
|
||||||
if _, duplicate := states[upstream.Name]; duplicate {
|
|
||||||
return nil, ErrInvalidUpstreamDrainReaper
|
|
||||||
}
|
|
||||||
states[upstream.Name] = upstream
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if revision == 0 {
|
|
||||||
// A config-only Controller has no persisted management revision. Its
|
|
||||||
// configuration cannot be hot-reloaded, so one stable non-zero fence is enough.
|
|
||||||
revision = 1
|
|
||||||
}
|
|
||||||
names := make([]string, 0, len(configuration.Upstreams))
|
|
||||||
for name := range configuration.Upstreams {
|
|
||||||
names = append(names, name)
|
|
||||||
}
|
|
||||||
sort.Strings(names)
|
|
||||||
policies := make([]activitypool.UpstreamDrainPolicy, 0, len(names))
|
|
||||||
for _, name := range names {
|
|
||||||
upstream := configuration.Upstreams[name]
|
|
||||||
policy := activitypool.UpstreamDrainPolicy{UpstreamID: name, Revision: revision, Enabled: upstream.Enabled}
|
|
||||||
if source.state != nil {
|
|
||||||
admin, exists := states[name]
|
|
||||||
if !exists {
|
|
||||||
return nil, ErrInvalidUpstreamDrainReaper
|
|
||||||
}
|
|
||||||
policy.Enabled = policy.Enabled && admin.Enabled
|
|
||||||
policy.Revision = admin.Revision
|
|
||||||
}
|
|
||||||
policies = append(policies, policy)
|
|
||||||
}
|
|
||||||
return policies, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
type UpstreamDrainPolicySource interface {
|
|
||||||
ReadUpstreamDrainPolicies(context.Context) ([]activitypool.UpstreamDrainPolicy, error)
|
|
||||||
}
|
|
||||||
|
|
||||||
// UpstreamDrainStore is the narrow activity-pool capability required by the
|
|
||||||
// configuration-driven Drain reaper.
|
|
||||||
type UpstreamDrainStore interface {
|
|
||||||
activitypool.UpstreamDrainPolicyWriter
|
|
||||||
activitypool.DisabledUpstreamDrainLister
|
|
||||||
activitypool.DisabledUpstreamDrainStarter
|
|
||||||
}
|
|
||||||
|
|
||||||
type UpstreamDrainReaperOptions struct {
|
|
||||||
PollInterval time.Duration
|
|
||||||
BatchSize int
|
|
||||||
Now func() time.Time
|
|
||||||
Metrics healthDomain.DrainMetricsObserver
|
|
||||||
}
|
|
||||||
|
|
||||||
type UpstreamDrainResult struct {
|
|
||||||
Candidates int
|
|
||||||
Started int
|
|
||||||
}
|
|
||||||
|
|
||||||
// ConfiguredUpstreamDrainReaper continually publishes an effective, complete
|
|
||||||
// policy view before reading one bounded batch of disabled Upstream ownership.
|
|
||||||
// The Redis ownership operation performs the final policy-revision fence.
|
|
||||||
type ConfiguredUpstreamDrainReaper struct {
|
|
||||||
policies UpstreamDrainPolicySource
|
|
||||||
store UpstreamDrainStore
|
|
||||||
options UpstreamDrainReaperOptions
|
|
||||||
}
|
|
||||||
|
|
||||||
func NewConfiguredUpstreamDrainReaper(
|
|
||||||
policies UpstreamDrainPolicySource,
|
|
||||||
store UpstreamDrainStore,
|
|
||||||
options UpstreamDrainReaperOptions,
|
|
||||||
) (*ConfiguredUpstreamDrainReaper, error) {
|
|
||||||
if nilInterface(policies) || nilInterface(store) || options.PollInterval <= 0 || options.BatchSize <= 0 || options.Now == nil {
|
|
||||||
return nil, ErrInvalidUpstreamDrainReaper
|
|
||||||
}
|
|
||||||
return &ConfiguredUpstreamDrainReaper{policies: policies, store: store, options: options}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (reaper *ConfiguredUpstreamDrainReaper) Tick(ctx context.Context) (UpstreamDrainResult, error) {
|
|
||||||
if ctx == nil || reaper == nil || nilInterface(reaper.policies) || nilInterface(reaper.store) ||
|
|
||||||
reaper.options.PollInterval <= 0 || reaper.options.BatchSize <= 0 || reaper.options.Now == nil {
|
|
||||||
return UpstreamDrainResult{}, ErrInvalidUpstreamDrainReaper
|
|
||||||
}
|
|
||||||
if err := ctx.Err(); err != nil {
|
|
||||||
return UpstreamDrainResult{}, err
|
|
||||||
}
|
|
||||||
now := reaper.options.Now()
|
|
||||||
if now.IsZero() {
|
|
||||||
return UpstreamDrainResult{}, ErrInvalidUpstreamDrainReaper
|
|
||||||
}
|
|
||||||
policies, err := reaper.policies.ReadUpstreamDrainPolicies(ctx)
|
|
||||||
if err != nil {
|
|
||||||
return UpstreamDrainResult{}, err
|
|
||||||
}
|
|
||||||
if err := reaper.store.ReplaceUpstreamDrainPolicies(ctx, policies); err != nil {
|
|
||||||
return UpstreamDrainResult{}, err
|
|
||||||
}
|
|
||||||
result := UpstreamDrainResult{}
|
|
||||||
remaining := reaper.options.BatchSize
|
|
||||||
utcNow := now.UTC()
|
|
||||||
for _, policy := range policies {
|
|
||||||
if policy.Enabled || remaining == 0 {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
candidates, err := reaper.store.ListDisabledUpstreamDrainCandidates(ctx, utcNow, policy, remaining)
|
|
||||||
if err != nil {
|
|
||||||
return result, err
|
|
||||||
}
|
|
||||||
if len(candidates) > remaining {
|
|
||||||
return result, ErrInvalidUpstreamDrainReaper
|
|
||||||
}
|
|
||||||
for _, candidate := range candidates {
|
|
||||||
started, err := reaper.store.BeginDisabledUpstreamDrain(ctx, utcNow, candidate)
|
|
||||||
if err != nil {
|
|
||||||
return result, err
|
|
||||||
}
|
|
||||||
result.Candidates++
|
|
||||||
if started {
|
|
||||||
result.Started++
|
|
||||||
}
|
|
||||||
}
|
|
||||||
remaining -= len(candidates)
|
|
||||||
}
|
|
||||||
if reaper.options.Metrics != nil {
|
|
||||||
reaper.options.Metrics.ObserveDrain(healthDomain.DrainReasonUpstreamDisabled, result.Candidates, result.Started)
|
|
||||||
}
|
|
||||||
return result, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (reaper *ConfiguredUpstreamDrainReaper) Run(ctx context.Context) error {
|
|
||||||
if reaper == nil {
|
|
||||||
return ErrInvalidUpstreamDrainReaper
|
|
||||||
}
|
|
||||||
return runScheduler(ctx, reaper.options.PollInterval, func(tickCtx context.Context) (TickResult, error) {
|
|
||||||
_, err := reaper.Tick(tickCtx)
|
|
||||||
return TickResult{}, err
|
|
||||||
})
|
|
||||||
}
|
|
||||||
@ -1,133 +0,0 @@
|
|||||||
package health
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"proxy-pool/internal/config"
|
|
||||||
"proxy-pool/internal/domain/activitypool"
|
|
||||||
"proxy-pool/internal/domain/adminstate"
|
|
||||||
healthDomain "proxy-pool/internal/domain/health"
|
|
||||||
proxyDomain "proxy-pool/internal/domain/proxy"
|
|
||||||
)
|
|
||||||
|
|
||||||
type upstreamDrainPolicySourceStub struct {
|
|
||||||
policies []activitypool.UpstreamDrainPolicy
|
|
||||||
err error
|
|
||||||
}
|
|
||||||
|
|
||||||
func (source upstreamDrainPolicySourceStub) ReadUpstreamDrainPolicies(context.Context) ([]activitypool.UpstreamDrainPolicy, error) {
|
|
||||||
return append([]activitypool.UpstreamDrainPolicy(nil), source.policies...), source.err
|
|
||||||
}
|
|
||||||
|
|
||||||
type upstreamDrainConfigurationStub struct {
|
|
||||||
configuration *config.Config
|
|
||||||
revision uint64
|
|
||||||
}
|
|
||||||
|
|
||||||
type upstreamDrainStateStub struct{ snapshot adminstate.Snapshot }
|
|
||||||
|
|
||||||
func (source upstreamDrainStateStub) Snapshot(context.Context) (adminstate.Snapshot, error) {
|
|
||||||
return source.snapshot, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
type drainMetricRecord struct {
|
|
||||||
reason healthDomain.DrainReason
|
|
||||||
candidates int
|
|
||||||
started int
|
|
||||||
}
|
|
||||||
|
|
||||||
type recordingDrainMetrics struct{ records []drainMetricRecord }
|
|
||||||
|
|
||||||
func (metrics *recordingDrainMetrics) ObserveDrain(reason healthDomain.DrainReason, candidates, started int) {
|
|
||||||
metrics.records = append(metrics.records, drainMetricRecord{reason: reason, candidates: candidates, started: started})
|
|
||||||
}
|
|
||||||
|
|
||||||
func (source upstreamDrainConfigurationStub) Snapshot() (*config.Config, uint64) {
|
|
||||||
return source.configuration, source.revision
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestConfiguredUpstreamDrainReaperStartsBoundedDisabledAssignments(t *testing.T) {
|
|
||||||
now := time.Date(2026, 8, 2, 16, 0, 0, 0, time.UTC)
|
|
||||||
pool := activitypool.NewMemoryPool()
|
|
||||||
if _, err := pool.UpsertFetched(context.Background(), "provider-a", activitypool.FetchedBatch{
|
|
||||||
ObservedAt: now, ConfiguredTTL: time.Minute, MaxSize: 1,
|
|
||||||
Proxies: []proxyDomain.Proxy{{
|
|
||||||
ID: "proxy-a", Scheme: proxyDomain.SchemeHTTP, Host: "192.0.2.10", Port: 8080, State: proxyDomain.StateAvailable,
|
|
||||||
}},
|
|
||||||
}); err != nil {
|
|
||||||
t.Fatalf("UpsertFetched() error = %v", err)
|
|
||||||
}
|
|
||||||
assignment, err := pool.Assign(context.Background(), now, "proxy-a", "worker-a", time.Minute)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("Assign() error = %v", err)
|
|
||||||
}
|
|
||||||
policy := activitypool.UpstreamDrainPolicy{UpstreamID: "provider-a", Revision: 21}
|
|
||||||
metrics := &recordingDrainMetrics{}
|
|
||||||
reaper, err := NewConfiguredUpstreamDrainReaper(upstreamDrainPolicySourceStub{policies: []activitypool.UpstreamDrainPolicy{policy}}, pool,
|
|
||||||
UpstreamDrainReaperOptions{PollInterval: time.Second, BatchSize: 4, Now: func() time.Time { return now }, Metrics: metrics})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("NewConfiguredUpstreamDrainReaper() error = %v", err)
|
|
||||||
}
|
|
||||||
result, err := reaper.Tick(context.Background())
|
|
||||||
if err != nil || result != (UpstreamDrainResult{Candidates: 1, Started: 1}) {
|
|
||||||
t.Fatalf("Tick() = %+v, %v", result, err)
|
|
||||||
}
|
|
||||||
if current, found, err := pool.Get(context.Background(), "proxy-a"); err != nil || !found ||
|
|
||||||
current.Epoch != assignment.Epoch || !current.Draining {
|
|
||||||
t.Fatalf("Get(after drain) = %+v, %t, %v", current, found, err)
|
|
||||||
}
|
|
||||||
if len(metrics.records) != 1 || metrics.records[0] != (drainMetricRecord{
|
|
||||||
reason: healthDomain.DrainReasonUpstreamDisabled, candidates: 1, started: 1,
|
|
||||||
}) {
|
|
||||||
t.Fatalf("Drain metrics = %+v", metrics.records)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestEffectiveUpstreamDrainPolicySourceUsesStableConfigOnlyRevision(t *testing.T) {
|
|
||||||
configuration := &config.Config{Upstreams: map[string]config.Upstream{
|
|
||||||
"provider-b": {Enabled: true},
|
|
||||||
"provider-a": {Enabled: false},
|
|
||||||
}}
|
|
||||||
source, err := NewEffectiveUpstreamDrainPolicySource(upstreamDrainConfigurationStub{configuration: configuration})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("NewEffectiveUpstreamDrainPolicySource() error = %v", err)
|
|
||||||
}
|
|
||||||
policies, err := source.ReadUpstreamDrainPolicies(context.Background())
|
|
||||||
if err != nil || len(policies) != 2 || policies[0] != (activitypool.UpstreamDrainPolicy{
|
|
||||||
UpstreamID: "provider-a", Revision: 1,
|
|
||||||
}) || policies[1] != (activitypool.UpstreamDrainPolicy{
|
|
||||||
UpstreamID: "provider-b", Revision: 1, Enabled: true,
|
|
||||||
}) {
|
|
||||||
t.Fatalf("ReadUpstreamDrainPolicies() = %+v, %v", policies, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestEffectiveUpstreamDrainPolicySourceCombinesConfigurationAndAdminState(t *testing.T) {
|
|
||||||
configuration := &config.Config{Upstreams: map[string]config.Upstream{
|
|
||||||
"provider-a": {Enabled: false},
|
|
||||||
"provider-b": {Enabled: true},
|
|
||||||
}}
|
|
||||||
source, err := NewEffectiveUpstreamDrainPolicySource(
|
|
||||||
upstreamDrainConfigurationStub{configuration: configuration, revision: 9},
|
|
||||||
upstreamDrainStateStub{snapshot: adminstate.Snapshot{
|
|
||||||
Config: &adminstate.ConfigRevision{Revision: 9},
|
|
||||||
Upstreams: []adminstate.UpstreamState{
|
|
||||||
{Name: "provider-a", Enabled: true, Revision: 7},
|
|
||||||
{Name: "provider-b", Enabled: false, Revision: 8},
|
|
||||||
},
|
|
||||||
}},
|
|
||||||
)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("NewEffectiveUpstreamDrainPolicySource() error = %v", err)
|
|
||||||
}
|
|
||||||
policies, err := source.ReadUpstreamDrainPolicies(context.Background())
|
|
||||||
want := []activitypool.UpstreamDrainPolicy{
|
|
||||||
{UpstreamID: "provider-a", Revision: 7},
|
|
||||||
{UpstreamID: "provider-b", Revision: 8},
|
|
||||||
}
|
|
||||||
if err != nil || len(policies) != len(want) || policies[0] != want[0] || policies[1] != want[1] {
|
|
||||||
t.Fatalf("ReadUpstreamDrainPolicies() = %+v, %v; want %+v", policies, err, want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue
Block a user