Compare commits
No commits in common. "d6ac783c1d6906cabc2462a4be460bdb374c88bd" and "5ad1d10957c11ba28a91a7c709ee45714c9d8fa3" have entirely different histories.
d6ac783c1d
...
5ad1d10957
@ -1,23 +0,0 @@
|
||||
# Source control metadata
|
||||
.git
|
||||
|
||||
# Local configuration and credential material
|
||||
.env
|
||||
.env.*
|
||||
configs/local.yaml
|
||||
deploy/.control-plane-tls/
|
||||
*.crt
|
||||
*.key
|
||||
*.pem
|
||||
|
||||
# Local build, test, and tooling outputs
|
||||
.playwright-mcp/
|
||||
temp/
|
||||
bin/
|
||||
dist/
|
||||
coverage/
|
||||
*.out
|
||||
*.test
|
||||
*.prof
|
||||
*.exe
|
||||
.tmp-proto/
|
||||
48
.github/workflows/ci.yml
vendored
48
.github/workflows/ci.yml
vendored
@ -8,16 +8,6 @@ permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
openapi:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: 24
|
||||
- shell: pwsh
|
||||
run: ./scripts/verify-openapi.ps1
|
||||
|
||||
proto:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
@ -29,44 +19,6 @@ jobs:
|
||||
- shell: pwsh
|
||||
run: ./scripts/verify-proto.ps1
|
||||
|
||||
image:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- name: Build Linux application image
|
||||
run: docker build --file deploy/docker/Dockerfile --tag proxy-pool:ci .
|
||||
|
||||
deployment:
|
||||
runs-on: ubuntu-latest
|
||||
env:
|
||||
PROXY_POOL_GATEWAY_USERNAME: ci-gateway-user
|
||||
PROXY_POOL_GATEWAY_PASSWORD: ci-gateway-password
|
||||
PROXY_POOL_EXTRACT_TOKEN: ci-extract-token
|
||||
PROXY_POOL_ADMIN_TOKEN: ci-admin-token
|
||||
PROXY_POOL_CONFIG_FINGERPRINT_KEY: ci-config-fingerprint-key-at-least-32-bytes
|
||||
PROXY_POOL_POSTGRES_URL: postgres://ci:ci@postgres.example:5432/proxy_pool?sslmode=verify-full
|
||||
PROXY_POOL_REDIS_URL: rediss://:ci@redis.example:6379/0
|
||||
PROVIDER_A_TOKEN: ci-provider-a-token
|
||||
PROVIDER_B_TOKEN: ci-provider-b-token
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/setup-go@v5
|
||||
with:
|
||||
go-version-file: go.mod
|
||||
cache: true
|
||||
- name: Render Compose configuration
|
||||
run: docker compose -f deploy/docker-compose.yml config --quiet
|
||||
- name: Render integration Compose configuration
|
||||
run: docker compose -f deploy/docker-compose.test.yml config --quiet
|
||||
- name: Validate local Compose configuration
|
||||
run: go run ./deploy/tools/configcheck deploy/config/local.yaml
|
||||
- name: Render Kubernetes base
|
||||
run: kubectl kustomize deploy/kubernetes/base > /dev/null
|
||||
- name: Render Kubernetes development mTLS overlay
|
||||
run: kubectl kustomize deploy/kubernetes/overlays/development-mtls > /dev/null
|
||||
- name: Validate development mTLS configuration
|
||||
run: go run ./deploy/tools/configcheck deploy/kubernetes/overlays/development-mtls/config.yaml
|
||||
|
||||
test:
|
||||
strategy:
|
||||
matrix:
|
||||
|
||||
1
.gitignore
vendored
1
.gitignore
vendored
@ -17,7 +17,6 @@ coverage/
|
||||
.env.*
|
||||
!.env.example
|
||||
configs/local.yaml
|
||||
deploy/.control-plane-tls/
|
||||
*.pem
|
||||
*.key
|
||||
|
||||
|
||||
123
README.md
123
README.md
@ -41,28 +41,8 @@ Proxy Pool 用 Controller 协调这些变化,并让 Gateway 数据面只消费
|
||||
PostgreSQL 迁移与 Redis 活动池,并支持联动优雅停机。
|
||||
- **Checker 指标**:Metrics 启用时暴露 Checker 任务下发与 Observation 接受/拒绝计数;
|
||||
标签仅使用固定检查级别和结果,不记录 Proxy、IP、URL 或凭据。
|
||||
- **Gateway 指标**:Metrics 启用时暴露代理尝试的固定阶段成功/失败计数、本地
|
||||
Outcome 队列满后的丢弃计数、HTTP/CONNECT 已接收请求数与在途数,以及活跃
|
||||
CONNECT 隧道数和请求生命周期 p50/p95/p99 直方图;协议标签仅有 HTTP 与 CONNECT,不记录 Proxy、路由、目标、
|
||||
客户端或凭据。
|
||||
- **Drain 指标**:Controller 暴露 `proxy_pool_controller_drain_candidates_total` 与
|
||||
`proxy_pool_controller_drains_started_total`;`reason` 仅有 `unhealthy` 与
|
||||
`upstream_disabled`,不包含 Proxy、Worker、Upstream、会话或地址。
|
||||
- **Provider 指标**:Controller 暴露
|
||||
`proxy_pool_controller_provider_fetch_results_total{class}`、
|
||||
`proxy_pool_controller_provider_valid_candidates_total` 与
|
||||
`proxy_pool_controller_provider_new_proxies_total`;`class` 仅有 `valid`、
|
||||
`empty`、`duplicate_only`、`error`,不包含 Upstream、Proxy 或错误文本标签。
|
||||
- **提取指标**:Controller 暴露
|
||||
`proxy_pool_controller_extraction_requests_total{result}`、
|
||||
`proxy_pool_controller_extraction_requested_proxies_total` 与
|
||||
`proxy_pool_controller_extraction_returned_proxies_total`;`result` 仅有完成、
|
||||
部分、空、库存不足、幂等冲突、限流、不可用、无效与内部错误等固定枚举,
|
||||
不包含 Client、请求、过滤条件、Upstream、Proxy 或错误文本标签。
|
||||
- **容量指标**:Controller 在既有 Provider 库存对账周期聚合托管 Proxy、可用/有效
|
||||
Slot、待拉取数量和活跃上游数,并暴露
|
||||
`proxy_pool_controller_capacity_inventory_reads_total{result}`;`result` 仅有
|
||||
`success` 和 `error`,不按 Upstream、Worker、Proxy、会话或地址拆分。
|
||||
- **Gateway 指标**:Metrics 启用时暴露代理尝试的固定阶段成功/失败计数,以及本地
|
||||
Outcome 队列满后的丢弃计数;不记录 Proxy、路由、目标、客户端或凭据。
|
||||
- **PostgreSQL 管理面**:持久化配置版本、Upstream/Routing 管理状态、Admin
|
||||
审计与 Outbox;不保存 Proxy 明细或逐次提取记录。
|
||||
- **Gateway 组件**:HTTP 正向代理、HTTPS CONNECT、双向 Tunnel、重试、超时、
|
||||
@ -76,20 +56,9 @@ Proxy Pool 用 Controller 协调这些变化,并让 Gateway 数据面只消费
|
||||
Connections 选择上游。无候选时支持 reject、受 `waitTimeout` 限制的本地容量等待,
|
||||
以及仍经过目标地址策略的 direct;`proxy-gateway` 通过独立控制面拨号地址维护
|
||||
Session,并在每份 Snapshot 有效期的一半前接收版本递增的完整刷新;仅在持有未过期
|
||||
Snapshot 时 Ready,凭据材料只保留在当前节点内存 View。Controller 内成功提交的
|
||||
Upstream 启停、Routing 切换和配置发布会向本进程全部在线 Worker 快照流广播刷新;
|
||||
定时刷新仍作为跨进程收敛与失效保护。
|
||||
- **Gateway 粘性会话**:可为已认证请求启用 X-Proxy-Session 等配置 Header;
|
||||
Gateway 以 Client、Routing 和会话值派生本地有界绑定,成功建连后固定到同一
|
||||
Proxy,并在代理失效、快照移除或转发失败时自动重绑。原始会话值不会写入日志、
|
||||
指标、PostgreSQL 或 Redis,也不会向目标站点转发。
|
||||
Snapshot 时 Ready,凭据材料只保留在当前节点内存 View。
|
||||
- **安全边界**:Gateway、Distribution 与 Admin 使用各自的认证语义,并支持
|
||||
CIDR、可信代理、严格请求解析和敏感信息最小化;Admin 的读写权限与
|
||||
Distribution 提取权限可按命中凭据分别收敛。Distribution 凭据还可限制单次
|
||||
提取数量、可访问 Upstream 与地区,Gateway 凭据可限制可访问 Routing;未配置时
|
||||
保持既有全范围行为。Gateway 凭据还可在每个 Worker 内限制每分钟请求数,
|
||||
同时限制 HTTP 请求和 CONNECT 隧道的并发数;热路径计数不写入 Redis 或
|
||||
PostgreSQL。
|
||||
CIDR、可信代理、严格请求解析和敏感信息最小化。
|
||||
|
||||
## 架构概览
|
||||
|
||||
@ -116,25 +85,21 @@ flowchart LR
|
||||
|
||||
## 当前完成度
|
||||
|
||||
截至 **2026-08-07**,实施计划中可直接勾选的检查项为 **71 / 75(94.7%)**。详情见
|
||||
截至 **2026-08-02**,实施计划中可直接勾选的检查项为 **57 / 74(77.0%)**。详情见
|
||||
[实施计划](docs/development/implementation-plan.md)和
|
||||
[交付完成度审计](docs/requirements/completion-audit.md)。
|
||||
|
||||
- **已完成**:严格配置、Provider 获取与协调、Redis 活动池、Distribution 原子
|
||||
提取与限流、Controller 的 Admin/Distribution/Metrics 监听,以及 PostgreSQL
|
||||
管理状态与有界审计查询;WorkerControlPlane 的 Register、Snapshot ACK、Runtime 心跳接收和
|
||||
管理状态;WorkerControlPlane 的 Register、Snapshot ACK、Runtime 心跳接收和
|
||||
Redis 会话栅栏,以及 Gateway Outcome 上报的有界队列、序列确认与重试;
|
||||
Controller 的 Redis 共享 BASIC/EGRESS/TARGET 检查任务、按上游的有界轮转调度、HTTP/HTTPS/SOCKS5
|
||||
Checker 探测和
|
||||
Observation 状态归并;Provider 连续空结果与管理态禁用当前项触发的 Sequential CAS
|
||||
自动推进、禁用候选过滤、末端路由停用和 Snapshot 即时刷新。
|
||||
- **部分完成**:Kubernetes 运行时 mTLS Overlay;Compose 已具备本地的
|
||||
Controller/Gateway/Checker mTLS 运行链路。
|
||||
- **待完成**:故障演练和代表性集群压测;现有 HTTP、CONNECT 长连接和 Extract
|
||||
场景只提供可复现的负载工具,不构成容量验证结论。
|
||||
Observation 状态归并。
|
||||
- **部分完成**:Docker Compose/Kubernetes 运行时 mTLS Overlay。
|
||||
- **待完成**:CONNECT 长连接/Extract 压测场景、故障演练和代表性集群压测。
|
||||
|
||||
检查项数量不等于生产就绪度。CI 会验证 OpenAPI 契约、固定版本的 protobuf descriptor/
|
||||
生成代码漂移,以及 Compose/Kustomize 的静态渲染;这些门禁也不代表
|
||||
检查项数量不等于生产就绪度。静态部署清单与 protobuf descriptor 验证也不代表
|
||||
端到端拓扑已经完成;`100,000 QPS` 仍只是待验证的集群设计目标。
|
||||
|
||||
## 快速开始
|
||||
@ -159,17 +124,8 @@ Secret 分离管理。
|
||||
```powershell
|
||||
go run ./deploy/tools/configcheck deploy/config/local.yaml
|
||||
./scripts/verify.ps1
|
||||
./scripts/package-docs.ps1
|
||||
./scripts/benchmark-gateway.ps1
|
||||
```
|
||||
|
||||
`package-docs.ps1` 默认生成被忽略的 `dist/proxy-pool-docs-v1.0.zip`;包内包含 README、
|
||||
`docs/`、图表、OpenAPI、Proto 契约和部署手册,并以 `manifest.json` 记录 Git revision、文件大小和
|
||||
SHA-256。可使用 `-Version vMAJOR.MINOR[.PATCH]` 与 `-OutputPath OUTPUT.zip` 生成指定交付物。
|
||||
`benchmark-gateway.ps1` 固定执行一次 100k 索引调度、Routing Round Robin 和 Snapshot Apply
|
||||
微基准,并把版本与原始输出写入 `dist/gateway-benchmarks.txt`;它用于回归比较,不构成
|
||||
网络转发或 100k QPS 集群容量证明。
|
||||
|
||||
Redis、PostgreSQL 和 Controller fixture 脚本会使用 Docker 启动隔离依赖:
|
||||
|
||||
```powershell
|
||||
@ -178,19 +134,6 @@ Redis、PostgreSQL 和 Controller fixture 脚本会使用 Docker 启动隔离依
|
||||
./scripts/test-controller.ps1
|
||||
```
|
||||
|
||||
本地 Compose 还会启动 Controller、两个从开发证书派生身份的 Gateway 和一个 Checker。
|
||||
首次启动前生成仅用于本机的 7 天 mTLS 证书;输出目录受 `.gitignore` 保护,脚本拒绝写入非空目录:
|
||||
|
||||
```powershell
|
||||
./scripts/generate-local-controlplane-certs.ps1
|
||||
docker compose -f deploy/docker-compose.yml up -d --build
|
||||
docker compose -f deploy/docker-compose.yml ps
|
||||
```
|
||||
|
||||
Gateway 只有取得有效 Snapshot 后才会通过 `/readyz`;Checker 在首次成功领取控制面任务
|
||||
批次后才会通过 `/readyz`,空批次也代表连接和身份验证已经成功。后续任务领取失败会立即
|
||||
撤销 Checker 的就绪状态。
|
||||
|
||||
PostgreSQL 与 Redis 是 Controller 的启动依赖。当前可运行的 Controller 入口如下,
|
||||
将 `CONFIG_FILE` 替换为实际配置路径,并确保其中的 PostgreSQL 与 Redis 地址可从
|
||||
进程所在网络访问:
|
||||
@ -204,7 +147,7 @@ go run ./cmd/proxy-controller -config CONFIG_FILE
|
||||
改用宿主机可达的存储地址。
|
||||
|
||||
本地配置中的 `.invalid` Provider URL 是故障演示占位,不会提供真实代理。
|
||||
Gateway 也可独立启动;需要启用 Controller `controlPlane` 并配置匹配的 mTLS
|
||||
Gateway 已提供启动命令;需要先启用 Controller `controlPlane` 并配置匹配的 mTLS
|
||||
证书(回环 fixture 可使用明文),再提供独立的拨号地址和 Worker 身份:
|
||||
|
||||
```powershell
|
||||
@ -216,10 +159,6 @@ go run ./cmd/proxy-gateway -config CONFIG_FILE `
|
||||
|
||||
以上参数也可通过 `PROXY_POOL_CONTROL_PLANE_ADDRESS`、`PROXY_POOL_CLUSTER_ID`、
|
||||
`PROXY_POOL_WORKER_ID`、`PROXY_POOL_INSTANCE_ID` 与 `PROXY_POOL_ZONE` 提供。
|
||||
生产工作负载可设置 `PROXY_POOL_AUTO_IDENTITY=true`,省略 Worker 和 Instance ID;
|
||||
Gateway 会从挂载的 `gatewayTLS` 证书解析
|
||||
`spiffe://<trust-domain>/<environment>/worker/<worker-id>`,并将 `<worker-id>`
|
||||
作为默认实例身份。Controller 仍会将请求 ID 与证书 URI 严格比对。
|
||||
Gateway 的 `/livez`、`/readyz`、`/metrics` 使用配置中的 `metrics.listen`;无有效
|
||||
Snapshot 时 `/readyz` 返回 `503`。Checker 使用独立的逻辑/实例身份拉取有界任务:
|
||||
|
||||
@ -232,17 +171,12 @@ go run ./cmd/proxy-checker -config CONFIG_FILE `
|
||||
|
||||
Checker 的参数也可通过 `PROXY_POOL_CONTROL_PLANE_ADDRESS`、
|
||||
`PROXY_POOL_CHECKER_ID`、`PROXY_POOL_CHECKER_INSTANCE_ID` 与
|
||||
`PROXY_POOL_CHECKER_MAX_IN_FLIGHT` 提供。`PROXY_POOL_AUTO_IDENTITY=true` 会从
|
||||
`checkerTLS` 的 `.../checker/<checker-id>` URI 自动派生 Checker 与缺省实例身份。
|
||||
它不会访问 Redis/PostgreSQL;生产
|
||||
`PROXY_POOL_CHECKER_MAX_IN_FLIGHT` 提供。它不会访问 Redis/PostgreSQL;生产
|
||||
Controller 在启用控制面时装配 Redis 共享任务 broker,并按启用的 Upstream 调度
|
||||
HTTP/HTTPS/SOCKS5 BASIC 检查、按每个 `check.urls` 创建 EGRESS 任务,并按启用 Routing 的
|
||||
`check.targets` 创建 TARGET 任务。调度监督器每轮读取已发布配置;启用 Admin 时只调度配置与
|
||||
PostgreSQL 管理态同 revision 且均启用的 Upstream 和 Routing。管理态停用 Upstream 会在下一轮阻止新的
|
||||
BASIC、EGRESS、TARGET 任务;管理态停用 Routing 则停止该 Routing 的新 TARGET 任务。revision 不一致或
|
||||
状态不完整时按失败关闭。因此 reload 后的上游/路由启停、检查间隔、
|
||||
抖动、超时、重试次数、`maxInFlight`、EGRESS URL 和 TARGET Profile 都会在下一轮生效;BASIC、
|
||||
EGRESS 与 TARGET 以有界轮转组共享上游并发上限。新启用的上游无需重启 Controller。
|
||||
`check.targets` 创建 TARGET 任务。调度监督器每轮读取已发布配置,因此 reload 后的上游/路由启停、
|
||||
检查间隔、抖动、超时、重试次数、`maxInFlight`、EGRESS URL 和 TARGET Profile 都会在下一轮生效;
|
||||
BASIC、EGRESS 与 TARGET 以有界轮转组共享上游并发上限。新启用的上游无需重启 Controller。
|
||||
EGRESS 对成功响应提取纯文本 IP 或常见 JSON IP 字段,并将其作为全局健康事实回传;TARGET 事实
|
||||
仅归并到对应的 `(routing_name, target_url)` Profile,不改变 Proxy 全局健康。
|
||||
|
||||
@ -291,27 +225,9 @@ go run ./cmd/proxy-loadgen `
|
||||
```
|
||||
|
||||
命令输出 JSON 报告,包含成功/失败分类、`TunnelsEstablished`、
|
||||
`ExtractResponsesValidated`、`ExtractReturned`、`ExtractValidationFailures`、`RateStartsGenerated`、
|
||||
`RateStartsDropped`、固定内存的
|
||||
`ExtractResponsesValidated`、`ExtractReturned`、`ExtractValidationFailures`、固定内存的
|
||||
连接握手延迟分位上界、吞吐和 Go 运行时内存/GC 快照。它不会输出提取响应中的地址或
|
||||
凭据,也不构成 100,000 QPS 证明。限速场景以 `Requests` 表示实际发起数;当
|
||||
`RateStartsDropped` 非零时,目标速率受压测端并发容量或时间窗限制,报告吞吐不得
|
||||
标注为已达到配置的 `-rate`。每个生成的限速令牌均会计入实际发起或丢弃,因此
|
||||
`RateStartsGenerated = Requests + RateStartsDropped`。
|
||||
需要保留单次容量证据时,使用 `-output REPORT_FILE` 同时写出 JSON 文件;文件在
|
||||
同目录完整写入后才替换目标,标准输出仍保留相同报告,便于交给日志或指标系统。
|
||||
`-max-error-rate` 接受 `0` 到 `1`,`-max-p99` 接受正的 Go duration;任一阈值违反时,
|
||||
报告会附带 `acceptance` 结果,命令仍写出标准输出和 `-output` 文件,然后以退出码 `3`
|
||||
结束,适合在 CI 中保留失败压测证据:
|
||||
|
||||
```powershell
|
||||
go run ./cmd/proxy-loadgen `
|
||||
-target https://TARGET_URL/health `
|
||||
-proxy http://GATEWAY_HOST:8080 `
|
||||
-duration 30s -rate 5000 -concurrency 128 `
|
||||
-max-error-rate 0.01 -max-p99 500ms `
|
||||
-output REPORT_FILE
|
||||
```
|
||||
凭据,也不构成 100,000 QPS 证明。
|
||||
|
||||
## 关键配置与入口
|
||||
|
||||
@ -347,10 +263,7 @@ go run ./cmd/proxy-loadgen `
|
||||
回收会自动为已分配项发起带健康/归属栅栏的 Drain,待 Snapshot ACK 与运行态归零后再删除;
|
||||
TARGET 事实按路由目标 Profile 独立归并。
|
||||
- **P1 - Gateway 与 Routing**:Gateway 进程、快照凭据分发、五种 Routing 策略与
|
||||
`onUnavailable` 已接入;上游停用会从后续完整 Snapshot 排除,并对现有 Worker
|
||||
ownership 发起带策略 revision 栅栏的 Drain。Routing 切换立即刷新在线 Worker 的
|
||||
完整快照;Sequential 的新分配切到新上游,旧 Proxy 与既有连接自然排空。动态容量
|
||||
调整仍待完成。
|
||||
`onUnavailable` 已接入;动态容量调整和配置停用触发的 Drain 编排待完成。
|
||||
- **P1 - 可观测与部署**:低基数业务指标、完整 Compose/Kubernetes 进程拓扑,
|
||||
以及故障转移和恢复演练。
|
||||
- **P2 - 容量证明**:`proxy-loadgen`、HTTP/CONNECT/Extract 分场景压测,以及
|
||||
|
||||
@ -2,23 +2,14 @@ openapi: 3.1.0
|
||||
info:
|
||||
title: Proxy Pool Admin API
|
||||
version: 1.0.0
|
||||
description: |
|
||||
运维状态与受控变更接口。该入口必须与 Distribution 分端口和权限。
|
||||
已认证凭据可配置 `admin:read`(Status、审计)和 `admin:write`(变更);
|
||||
未配置权限的旧凭据保留全权限行为。
|
||||
description: 运维状态与受控变更接口。该入口必须与 Distribution 分端口和权限。
|
||||
servers:
|
||||
- url: http://127.0.0.1:8082
|
||||
tags:
|
||||
- name: Status
|
||||
description: 不含代理明细与凭据的控制面聚合状态。
|
||||
- name: Audit
|
||||
description: 权威管理状态变更的游标审计查询。
|
||||
- name: Upstreams
|
||||
description: 上游的启停与受控状态变更。
|
||||
- name: Routing
|
||||
description: Routing 当前上游的受控切换。
|
||||
- name: Configuration
|
||||
description: 经校验后原子发布的配置重载。
|
||||
security:
|
||||
- AdminApiKey: []
|
||||
- BasicAuth: []
|
||||
@ -30,7 +21,6 @@ paths:
|
||||
tags: [Status]
|
||||
operationId: getStatus
|
||||
summary: 获取控制面摘要状态
|
||||
description: 需要 `admin:read`。
|
||||
responses:
|
||||
'200':
|
||||
description: 不含 Proxy 地址、Client 标识或 Secret 的聚合状态
|
||||
@ -46,50 +36,11 @@ paths:
|
||||
'405': {$ref: '#/components/responses/MethodNotAllowed'}
|
||||
'500': {$ref: '#/components/responses/InternalServerError'}
|
||||
'503': {$ref: '#/components/responses/ServiceUnavailable'}
|
||||
/api/v1/audit:
|
||||
get:
|
||||
tags: [Audit]
|
||||
operationId: listAuditRecords
|
||||
summary: 按 ID 游标读取管理面审计记录
|
||||
description: |
|
||||
需要 `admin:read`。
|
||||
|
||||
仅返回权威管理面的变更审计记录,按 `id` 升序排列。`afterId` 是排他游标:
|
||||
后续页面只包含 `id` 大于该值的记录。未传 `limit` 时返回 100 条,单页最多
|
||||
1000 条。
|
||||
|
||||
审计接口不返回 Proxy、Upstream URL、凭据或活动池内容。
|
||||
parameters:
|
||||
- name: afterId
|
||||
in: query
|
||||
required: false
|
||||
description: 排他游标;仅返回 ID 大于此值的记录。
|
||||
schema: {type: integer, minimum: 0, default: 0}
|
||||
- name: limit
|
||||
in: query
|
||||
required: false
|
||||
description: 单页记录数;默认 100,最大 1000。
|
||||
schema: {type: integer, minimum: 1, maximum: 1000, default: 100}
|
||||
responses:
|
||||
'200':
|
||||
description: 管理面审计记录页
|
||||
headers:
|
||||
X-Request-ID: {$ref: '#/components/headers/RequestID'}
|
||||
content:
|
||||
application/json:
|
||||
schema: {$ref: '#/components/schemas/AuditPage'}
|
||||
'401': {$ref: '#/components/responses/Unauthorized'}
|
||||
'403': {$ref: '#/components/responses/Forbidden'}
|
||||
'400': {$ref: '#/components/responses/BadRequest'}
|
||||
'405': {$ref: '#/components/responses/MethodNotAllowed'}
|
||||
'500': {$ref: '#/components/responses/InternalServerError'}
|
||||
'503': {$ref: '#/components/responses/ServiceUnavailable'}
|
||||
/api/v1/upstreams/{name}/enable:
|
||||
post:
|
||||
tags: [Upstreams]
|
||||
operationId: enableUpstream
|
||||
summary: 启用 Upstream
|
||||
description: 需要 `admin:write`。
|
||||
parameters:
|
||||
- $ref: '#/components/parameters/UpstreamName'
|
||||
- $ref: '#/components/parameters/RequestID'
|
||||
@ -108,7 +59,6 @@ paths:
|
||||
tags: [Upstreams]
|
||||
operationId: disableUpstream
|
||||
summary: 禁用 Upstream 并使已有资源自然 Drain
|
||||
description: 需要 `admin:write`。
|
||||
parameters:
|
||||
- $ref: '#/components/parameters/UpstreamName'
|
||||
- $ref: '#/components/parameters/RequestID'
|
||||
@ -127,7 +77,6 @@ paths:
|
||||
tags: [Routing]
|
||||
operationId: switchRouting
|
||||
summary: 原子切换 Sequential Routing 当前 Upstream
|
||||
description: 需要 `admin:write`。
|
||||
parameters:
|
||||
- name: name
|
||||
in: path
|
||||
@ -164,7 +113,6 @@ paths:
|
||||
tags: [Configuration]
|
||||
operationId: reloadConfiguration
|
||||
summary: 严格校验并原子发布新配置快照
|
||||
description: 需要 `admin:write`。
|
||||
parameters:
|
||||
- $ref: '#/components/parameters/RequestID'
|
||||
responses:
|
||||
@ -204,31 +152,6 @@ components:
|
||||
description: 服务端最终使用的请求标识。
|
||||
schema: {type: string, maxLength: 128}
|
||||
schemas:
|
||||
AuditPage:
|
||||
type: object
|
||||
additionalProperties: false
|
||||
required: [records]
|
||||
properties:
|
||||
records:
|
||||
type: array
|
||||
description: 按 ID 升序的审计记录;没有更多记录时为空数组。
|
||||
items: {$ref: '#/components/schemas/AuditRecord'}
|
||||
AuditRecord:
|
||||
type: object
|
||||
additionalProperties: false
|
||||
required: [id, requestId, actorId, action, resourceType, resourceName, changed, version, occurredAt]
|
||||
properties:
|
||||
id: {type: integer, minimum: 0}
|
||||
requestId: {type: string}
|
||||
actorId: {type: string}
|
||||
sourceIp: {type: string}
|
||||
action: {type: string}
|
||||
resourceType: {type: string}
|
||||
resourceName: {type: string}
|
||||
changed: {type: boolean}
|
||||
version: {type: integer, minimum: 0}
|
||||
reason: {type: string}
|
||||
occurredAt: {type: string, format: date-time}
|
||||
Status:
|
||||
type: object
|
||||
additionalProperties: false
|
||||
|
||||
@ -10,7 +10,6 @@ import (
|
||||
type document struct {
|
||||
OpenAPI string `yaml:"openapi"`
|
||||
Paths map[string]map[string]any `yaml:"paths"`
|
||||
Components map[string]any `yaml:"components"`
|
||||
}
|
||||
|
||||
func TestDistributionContract(t *testing.T) {
|
||||
@ -38,7 +37,6 @@ func TestAdminContract(t *testing.T) {
|
||||
spec := readDocument(t, "admin.yaml")
|
||||
for _, path := range []string{
|
||||
"/api/v1/status",
|
||||
"/api/v1/audit",
|
||||
"/api/v1/upstreams/{name}/enable",
|
||||
"/api/v1/upstreams/{name}/disable",
|
||||
"/api/v1/routing/{name}/switch",
|
||||
@ -50,67 +48,6 @@ func TestAdminContract(t *testing.T) {
|
||||
}
|
||||
requireResponses(t, spec.Paths["/api/v1/routing/{name}/switch"]["post"],
|
||||
"200", "400", "401", "403", "404", "405", "409", "413", "415", "422", "500", "503")
|
||||
requireResponses(t, spec.Paths["/api/v1/audit"]["get"],
|
||||
"200", "400", "401", "403", "405", "500", "503")
|
||||
requireAuditPaginationContract(t, spec.Paths["/api/v1/audit"]["get"])
|
||||
requireAuditSchemas(t, spec)
|
||||
}
|
||||
|
||||
func requireAuditPaginationContract(t *testing.T, operation any) {
|
||||
t.Helper()
|
||||
operationMap, ok := operation.(map[string]any)
|
||||
if !ok {
|
||||
t.Fatalf("audit operation has type %T, want map", operation)
|
||||
}
|
||||
parameters, ok := operationMap["parameters"].([]any)
|
||||
if !ok {
|
||||
t.Fatalf("audit parameters has type %T, want array", operationMap["parameters"])
|
||||
}
|
||||
afterID := parameterSchema(t, parameters, "afterId")
|
||||
if afterID["type"] != "integer" || afterID["minimum"] != 0 || afterID["default"] != 0 {
|
||||
t.Errorf("afterId schema = %#v, want integer with minimum/default 0", afterID)
|
||||
}
|
||||
limit := parameterSchema(t, parameters, "limit")
|
||||
if limit["type"] != "integer" || limit["minimum"] != 1 || limit["maximum"] != 1000 || limit["default"] != 100 {
|
||||
t.Errorf("limit schema = %#v, want integer [1, 1000] with default 100", limit)
|
||||
}
|
||||
}
|
||||
|
||||
func parameterSchema(t *testing.T, parameters []any, name string) map[string]any {
|
||||
t.Helper()
|
||||
for _, raw := range parameters {
|
||||
parameter, ok := raw.(map[string]any)
|
||||
if !ok || parameter["name"] != name || parameter["in"] != "query" {
|
||||
continue
|
||||
}
|
||||
schema, ok := parameter["schema"].(map[string]any)
|
||||
if !ok {
|
||||
t.Fatalf("%s parameter schema has type %T, want map", name, parameter["schema"])
|
||||
}
|
||||
return schema
|
||||
}
|
||||
t.Fatalf("query parameter %q is missing", name)
|
||||
return nil
|
||||
}
|
||||
|
||||
func requireAuditSchemas(t *testing.T, spec document) {
|
||||
t.Helper()
|
||||
schemas, ok := spec.Components["schemas"].(map[string]any)
|
||||
if !ok {
|
||||
t.Fatal("components.schemas is missing")
|
||||
}
|
||||
for _, name := range []string{"AuditPage", "AuditRecord"} {
|
||||
if _, ok := schemas[name].(map[string]any); !ok {
|
||||
t.Errorf("schema %s is missing", name)
|
||||
}
|
||||
}
|
||||
record, _ := schemas["AuditRecord"].(map[string]any)
|
||||
properties, _ := record["properties"].(map[string]any)
|
||||
for _, forbidden := range []string{"proxy", "upstreamUrl", "credential", "activePool"} {
|
||||
if _, exists := properties[forbidden]; exists {
|
||||
t.Errorf("AuditRecord must not expose %q", forbidden)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func requireResponses(t *testing.T, operation any, codes ...string) {
|
||||
|
||||
@ -7,18 +7,13 @@ info:
|
||||
Redis operation atomically removes or marks every returned proxy as
|
||||
extracted in the TTL activity pool. Proxy details and per-extraction
|
||||
records are not persisted in PostgreSQL. There is no release, renew, or
|
||||
lease API. An authenticated credential may be limited to the fixed
|
||||
`distribution:extract` permission; legacy credentials without configured
|
||||
permissions retain full access. Credential client policy may further bound
|
||||
a request's count, upstreams, and regions before Redis extraction.
|
||||
lease API.
|
||||
servers:
|
||||
- url: http://127.0.0.1:8081
|
||||
description: Distribution API
|
||||
tags:
|
||||
- name: Distribution
|
||||
description: 一次性独占提取上游代理的受保护接口。
|
||||
- name: Health
|
||||
description: Distribution 监听器的存活与就绪探针。
|
||||
paths:
|
||||
/api/v1/proxies/extract:
|
||||
post:
|
||||
@ -26,8 +21,6 @@ paths:
|
||||
operationId: extractProxies
|
||||
summary: 一次性独占提取代理
|
||||
description: |
|
||||
需要 `distribution:extract`。
|
||||
|
||||
服务端通过单个 Redis 原子操作完成筛选、TTL/所有权复核、Gateway 预留、
|
||||
从可分配池移除所选条目和短期幂等结果写入,再返回代理。相同代理不会
|
||||
返回给两个成功请求,提取路径不访问 PostgreSQL。
|
||||
@ -35,8 +28,7 @@ paths:
|
||||
`partial` 允许实际返回数量小于请求数量;`allOrNothing` 数量不足时不
|
||||
提取任何代理并返回 409。未传 `fulfillment` 时使用服务端配置,默认
|
||||
为 `partial`。`Idempotency-Key` 在 Redis 的有界 TTL 窗口内避免客户端因
|
||||
响应丢失重试而再次消耗库存。命中凭据的 Client policy 可以额外限制
|
||||
count、allowedUpstreams 和 regions;越过该边界返回 403,未调用 Redis。
|
||||
响应丢失重试而再次消耗库存。
|
||||
security:
|
||||
- ApiKeyAuth: []
|
||||
- BasicAuth: []
|
||||
|
||||
@ -1,5 +0,0 @@
|
||||
extends:
|
||||
- minimal
|
||||
|
||||
rules:
|
||||
tag-description: error
|
||||
@ -26,31 +26,6 @@ func TestOpenAPIDocumentsHaveClosedContracts(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestOpenAPITagsHaveDescriptions(t *testing.T) {
|
||||
for _, name := range []string{"proxy-pool.yaml", "admin.yaml"} {
|
||||
name := name
|
||||
t.Run(name, func(t *testing.T) {
|
||||
root := readOpenAPIRoot(t, name)
|
||||
tags, ok := root["tags"].([]any)
|
||||
if !ok || len(tags) == 0 {
|
||||
t.Fatal("tags must be a non-empty array")
|
||||
}
|
||||
for _, rawTag := range tags {
|
||||
tag, ok := rawTag.(map[string]any)
|
||||
if !ok {
|
||||
t.Errorf("tag has type %T, want object", rawTag)
|
||||
continue
|
||||
}
|
||||
name, _ := tag["name"].(string)
|
||||
description, _ := tag["description"].(string)
|
||||
if strings.TrimSpace(description) == "" {
|
||||
t.Errorf("tag %q has no description", name)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func validateLocalReferences(t *testing.T, root map[string]any) {
|
||||
t.Helper()
|
||||
walkOpenAPI(root, func(path string, value any) {
|
||||
|
||||
@ -99,7 +99,6 @@ message RoutingRule {
|
||||
RoutingStrategy strategy = 8;
|
||||
UnavailableAction on_unavailable = 9;
|
||||
google.protobuf.Duration wait_timeout = 10;
|
||||
RoutingAction action = 11;
|
||||
}
|
||||
|
||||
message RoutingStrategy {
|
||||
@ -124,13 +123,6 @@ enum UnavailableAction {
|
||||
UNAVAILABLE_ACTION_DIRECT = 3;
|
||||
}
|
||||
|
||||
enum RoutingAction {
|
||||
ROUTING_ACTION_UNSPECIFIED = 0;
|
||||
ROUTING_ACTION_PROXY = 1;
|
||||
ROUTING_ACTION_DIRECT = 2;
|
||||
ROUTING_ACTION_REJECT = 3;
|
||||
}
|
||||
|
||||
message OwnedProxy {
|
||||
string id = 1;
|
||||
string upstream = 2;
|
||||
|
||||
@ -15,7 +15,6 @@ import (
|
||||
controlplanev1 "proxy-pool/gen/controlplane/v1"
|
||||
"proxy-pool/internal/checker/bootstrap"
|
||||
"proxy-pool/internal/config"
|
||||
"proxy-pool/internal/platform/logging"
|
||||
)
|
||||
|
||||
const (
|
||||
@ -23,7 +22,6 @@ const (
|
||||
controlPlaneAddressEnvironment = "PROXY_POOL_CONTROL_PLANE_ADDRESS"
|
||||
checkerIDEnvironment = "PROXY_POOL_CHECKER_ID"
|
||||
instanceIDEnvironment = "PROXY_POOL_CHECKER_INSTANCE_ID"
|
||||
autoIdentityEnvironment = "PROXY_POOL_AUTO_IDENTITY"
|
||||
maxInFlightEnvironment = "PROXY_POOL_CHECKER_MAX_IN_FLIGHT"
|
||||
)
|
||||
|
||||
@ -43,7 +41,6 @@ func execute(ctx context.Context, args []string, getenv environmentLookup, run c
|
||||
controlPlaneAddress := flags.String("control-plane", "", "remote Controller control-plane address")
|
||||
checkerID := flags.String("checker-id", "", "unique Checker identifier")
|
||||
instanceID := flags.String("instance-id", "", "unique Checker process instance identifier")
|
||||
autoIdentity := flags.Bool("auto-identity", false, "derive Checker and missing instance IDs from the mTLS SPIFFE certificate")
|
||||
maxInFlight := flags.Int("max-in-flight", 0, "maximum concurrent tasks")
|
||||
levels := flags.String("levels", "basic,egress,target", "supported levels: basic,egress,target")
|
||||
if err := flags.Parse(args); err != nil {
|
||||
@ -66,45 +63,28 @@ func execute(ctx context.Context, args []string, getenv environmentLookup, run c
|
||||
*maxInFlight = value
|
||||
}
|
||||
}
|
||||
if !flagWasSet(flags, "auto-identity") {
|
||||
if value, err := strconv.ParseBool(getenv(autoIdentityEnvironment)); err == nil {
|
||||
*autoIdentity = value
|
||||
}
|
||||
}
|
||||
}
|
||||
supportedLevels, err := parseLevels(*levels)
|
||||
missingIdentity := !*autoIdentity && (!validValue(*checkerID) || !validValue(*instanceID))
|
||||
invalidProvidedIdentity := (*checkerID != "" && !validValue(*checkerID)) || (*instanceID != "" && !validValue(*instanceID))
|
||||
if ctx == nil || run == nil || err != nil || !validValue(*configPath) || !validValue(*controlPlaneAddress) ||
|
||||
missingIdentity || invalidProvidedIdentity || *maxInFlight <= 0 {
|
||||
!validValue(*checkerID) || !validValue(*instanceID) || *maxInFlight <= 0 {
|
||||
_, _ = fmt.Fprintf(stderr,
|
||||
"proxy-checker: -config, -control-plane and positive -max-in-flight are required; -checker-id/-instance-id are required unless -auto-identity is enabled; "+
|
||||
"environment fallbacks: %s, %s, %s, %s, %s, %s\n",
|
||||
configEnvironment, controlPlaneAddressEnvironment, checkerIDEnvironment, instanceIDEnvironment, maxInFlightEnvironment, autoIdentityEnvironment,
|
||||
"proxy-checker: -config, -control-plane, -checker-id, -instance-id and positive -max-in-flight are required; "+
|
||||
"environment fallbacks: %s, %s, %s, %s, %s\n",
|
||||
configEnvironment, controlPlaneAddressEnvironment, checkerIDEnvironment, instanceIDEnvironment, maxInFlightEnvironment,
|
||||
)
|
||||
return 2
|
||||
}
|
||||
err = run(ctx, bootstrap.Options{
|
||||
ConfigPath: *configPath, Resolver: config.OSResolver{}, ControlPlaneAddress: *controlPlaneAddress,
|
||||
CheckerID: *checkerID, InstanceID: *instanceID, AutoIdentity: *autoIdentity, MaxInFlight: *maxInFlight, SupportedLevels: supportedLevels,
|
||||
CheckerID: *checkerID, InstanceID: *instanceID, MaxInFlight: *maxInFlight, SupportedLevels: supportedLevels,
|
||||
})
|
||||
if err == nil || (errors.Is(err, context.Canceled) && ctx.Err() != nil) {
|
||||
return 0
|
||||
}
|
||||
logging.WriteProcessError(stderr, "proxy-checker", err)
|
||||
_, _ = fmt.Fprintf(stderr, "proxy-checker: %v\n", err)
|
||||
return 1
|
||||
}
|
||||
|
||||
func flagWasSet(flags *flag.FlagSet, name string) bool {
|
||||
set := false
|
||||
flags.Visit(func(item *flag.Flag) {
|
||||
if item.Name == name {
|
||||
set = true
|
||||
}
|
||||
})
|
||||
return set
|
||||
}
|
||||
|
||||
func parseLevels(value string) ([]controlplanev1.CheckLevel, error) {
|
||||
levels := make([]controlplanev1.CheckLevel, 0, 3)
|
||||
seen := make(map[controlplanev1.CheckLevel]struct{})
|
||||
|
||||
@ -24,21 +24,6 @@ func TestExecuteUsesFlagsAndPassesCheckerIdentity(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecutePassesAutoCheckerIdentityFromEnvironment(t *testing.T) {
|
||||
received := bootstrap.Options{}
|
||||
values := map[string]string{
|
||||
configEnvironment: "checker.yaml", controlPlaneAddressEnvironment: "127.0.0.1:8443",
|
||||
maxInFlightEnvironment: "2", autoIdentityEnvironment: "true",
|
||||
}
|
||||
code := execute(context.Background(), nil, func(name string) string { return values[name] }, func(_ context.Context, options bootstrap.Options) error {
|
||||
received = options
|
||||
return nil
|
||||
}, io.Discard)
|
||||
if code != 0 || !received.AutoIdentity || received.CheckerID != "" || received.InstanceID != "" {
|
||||
t.Fatalf("execute(auto identity) = (%d, %+v)", code, received)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecuteRejectsInvalidLevelSet(t *testing.T) {
|
||||
code := execute(context.Background(), []string{
|
||||
"-config", "config.yaml", "-control-plane", "127.0.0.1:8443", "-checker-id", "checker-a", "-instance-id", "instance-a",
|
||||
|
||||
@ -13,7 +13,6 @@ import (
|
||||
|
||||
"proxy-pool/internal/config"
|
||||
"proxy-pool/internal/controller/bootstrap"
|
||||
"proxy-pool/internal/platform/logging"
|
||||
)
|
||||
|
||||
const (
|
||||
@ -68,6 +67,6 @@ func execute(
|
||||
if err == nil || (errors.Is(err, context.Canceled) && ctx.Err() != nil) {
|
||||
return 0
|
||||
}
|
||||
logging.WriteProcessError(stderr, "proxy-controller", err)
|
||||
_, _ = fmt.Fprintf(stderr, "proxy-controller: %v\n", err)
|
||||
return 1
|
||||
}
|
||||
|
||||
@ -75,9 +75,7 @@ func TestExecuteMapsStartupFailureAndSignalCancellation(t *testing.T) {
|
||||
) error {
|
||||
return want
|
||||
}, &stderr)
|
||||
if code != 1 || !bytes.Contains(stderr.Bytes(), []byte(`"msg":"process failed"`)) ||
|
||||
!bytes.Contains(stderr.Bytes(), []byte(`"component":"proxy-controller"`)) ||
|
||||
bytes.Contains(stderr.Bytes(), []byte(want.Error())) {
|
||||
if code != 1 || !bytes.Contains(stderr.Bytes(), []byte(want.Error())) {
|
||||
t.Fatalf("execute(startup failure) = %d, stderr = %q", code, stderr.String())
|
||||
}
|
||||
|
||||
|
||||
@ -8,13 +8,11 @@ import (
|
||||
"io"
|
||||
"os"
|
||||
"os/signal"
|
||||
"strconv"
|
||||
"strings"
|
||||
"syscall"
|
||||
|
||||
"proxy-pool/internal/config"
|
||||
"proxy-pool/internal/gateway/bootstrap"
|
||||
"proxy-pool/internal/platform/logging"
|
||||
)
|
||||
|
||||
const (
|
||||
@ -23,7 +21,6 @@ const (
|
||||
clusterIDEnvironment = "PROXY_POOL_CLUSTER_ID"
|
||||
workerIDEnvironment = "PROXY_POOL_WORKER_ID"
|
||||
instanceIDEnvironment = "PROXY_POOL_INSTANCE_ID"
|
||||
autoIdentityEnvironment = "PROXY_POOL_AUTO_IDENTITY"
|
||||
zoneEnvironment = "PROXY_POOL_ZONE"
|
||||
)
|
||||
|
||||
@ -50,7 +47,6 @@ func execute(
|
||||
clusterID := flags.String("cluster-id", "", "cluster identifier")
|
||||
workerID := flags.String("worker-id", "", "unique Worker identifier")
|
||||
instanceID := flags.String("instance-id", "", "unique process instance identifier")
|
||||
autoIdentity := flags.Bool("auto-identity", false, "derive Worker and missing instance IDs from the mTLS SPIFFE certificate")
|
||||
zone := flags.String("zone", "", "availability zone identifier")
|
||||
if err := flags.Parse(args); err != nil {
|
||||
if errors.Is(err, flag.ErrHelp) {
|
||||
@ -69,44 +65,27 @@ func execute(
|
||||
setIfEmpty(workerID, getenv(workerIDEnvironment))
|
||||
setIfEmpty(instanceID, getenv(instanceIDEnvironment))
|
||||
setIfEmpty(zone, getenv(zoneEnvironment))
|
||||
if !flagWasSet(flags, "auto-identity") {
|
||||
if value, err := strconv.ParseBool(getenv(autoIdentityEnvironment)); err == nil {
|
||||
*autoIdentity = value
|
||||
}
|
||||
}
|
||||
}
|
||||
missingIdentity := !*autoIdentity && (!validValue(*workerID) || !validValue(*instanceID))
|
||||
invalidProvidedIdentity := (*workerID != "" && !validValue(*workerID)) || (*instanceID != "" && !validValue(*instanceID))
|
||||
if ctx == nil || run == nil || !validValue(*configPath) || !validValue(*controlPlaneAddress) ||
|
||||
!validValue(*clusterID) || missingIdentity || invalidProvidedIdentity || !validValue(*zone) {
|
||||
!validValue(*clusterID) || !validValue(*workerID) || !validValue(*instanceID) || !validValue(*zone) {
|
||||
_, _ = fmt.Fprintf(stderr,
|
||||
"proxy-gateway: -config, -control-plane, -cluster-id and -zone are required; -worker-id/-instance-id are required unless -auto-identity is enabled; "+
|
||||
"environment fallbacks: %s, %s, %s, %s, %s, %s, %s\n",
|
||||
configEnvironment, controlPlaneAddressEnvironment, clusterIDEnvironment, workerIDEnvironment, instanceIDEnvironment, zoneEnvironment, autoIdentityEnvironment,
|
||||
"proxy-gateway: -config, -control-plane, -cluster-id, -worker-id, -instance-id and -zone are required; "+
|
||||
"environment fallbacks: %s, %s, %s, %s, %s, %s\n",
|
||||
configEnvironment, controlPlaneAddressEnvironment, clusterIDEnvironment, workerIDEnvironment, instanceIDEnvironment, zoneEnvironment,
|
||||
)
|
||||
return 2
|
||||
}
|
||||
err := run(ctx, bootstrap.Options{
|
||||
ConfigPath: *configPath, Resolver: config.OSResolver{}, ControlPlaneAddress: *controlPlaneAddress,
|
||||
ClusterID: *clusterID, WorkerID: *workerID, InstanceID: *instanceID, AutoIdentity: *autoIdentity, Zone: *zone,
|
||||
ClusterID: *clusterID, WorkerID: *workerID, InstanceID: *instanceID, Zone: *zone,
|
||||
})
|
||||
if err == nil || (errors.Is(err, context.Canceled) && ctx.Err() != nil) {
|
||||
return 0
|
||||
}
|
||||
logging.WriteProcessError(stderr, "proxy-gateway", err)
|
||||
_, _ = fmt.Fprintf(stderr, "proxy-gateway: %v\n", err)
|
||||
return 1
|
||||
}
|
||||
|
||||
func flagWasSet(flags *flag.FlagSet, name string) bool {
|
||||
set := false
|
||||
flags.Visit(func(item *flag.Flag) {
|
||||
if item.Name == name {
|
||||
set = true
|
||||
}
|
||||
})
|
||||
return set
|
||||
}
|
||||
|
||||
func setIfEmpty(target *string, value string) {
|
||||
if target != nil && *target == "" {
|
||||
*target = value
|
||||
|
||||
@ -55,21 +55,6 @@ func TestExecuteReadsGatewayOptionsFromEnvironment(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecutePassesAutoGatewayIdentityFromEnvironment(t *testing.T) {
|
||||
captured := bootstrap.Options{}
|
||||
values := map[string]string{
|
||||
configEnvironment: "gateway.yaml", controlPlaneAddressEnvironment: "127.0.0.1:8443",
|
||||
clusterIDEnvironment: "cluster-a", zoneEnvironment: "zone-a", autoIdentityEnvironment: "true",
|
||||
}
|
||||
code := execute(context.Background(), nil, func(name string) string { return values[name] }, func(_ context.Context, options bootstrap.Options) error {
|
||||
captured = options
|
||||
return nil
|
||||
}, io.Discard)
|
||||
if code != 0 || !captured.AutoIdentity || captured.WorkerID != "" || captured.InstanceID != "" {
|
||||
t.Fatalf("execute(auto identity) = (%d, %+v)", code, captured)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecuteRejectsIncompleteGatewayOptions(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
|
||||
@ -7,11 +7,9 @@ import (
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"math"
|
||||
"net/http"
|
||||
"os"
|
||||
"os/signal"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"syscall"
|
||||
"time"
|
||||
@ -43,9 +41,6 @@ func execute(ctx context.Context, args []string, run loadRun, stdout, stderr io.
|
||||
hold := flags.Duration("hold", 0, "CONNECT tunnel hold duration; required by -scenario connect")
|
||||
extractCount := flags.Int("extract-count", 1, "proxies requested by each extract scenario request")
|
||||
extractFulfillment := flags.String("extract-fulfillment", "partial", "extract fulfillment: partial or allOrNothing")
|
||||
maxErrorRate := flags.Float64("max-error-rate", -1, "optional maximum failed/completed rate from 0 to 1")
|
||||
maxP99 := flags.Duration("max-p99", 0, "optional maximum p99 latency")
|
||||
outputPath := flags.String("output", "", "optional JSON report file; replaces only after a complete write")
|
||||
var headers headerValues
|
||||
flags.Var(&headers, "header", "repeatable HTTP header in Name: Value form")
|
||||
if err := flags.Parse(args); err != nil {
|
||||
@ -58,15 +53,6 @@ func execute(ctx context.Context, args []string, run loadRun, stdout, stderr io.
|
||||
_, _ = fmt.Fprintln(stderr, "proxy-loadgen: target and a bounded workload are required")
|
||||
return 2
|
||||
}
|
||||
if strings.TrimSpace(*outputPath) != *outputPath {
|
||||
_, _ = fmt.Fprintln(stderr, "proxy-loadgen: invalid output path")
|
||||
return 2
|
||||
}
|
||||
criteria, configured, err := acceptanceCriteria(*maxErrorRate, *maxP99)
|
||||
if err != nil {
|
||||
_, _ = fmt.Fprintf(stderr, "proxy-loadgen: %v\n", err)
|
||||
return 2
|
||||
}
|
||||
parsedHeaders, err := headers.Header()
|
||||
if err != nil {
|
||||
_, _ = fmt.Fprintf(stderr, "proxy-loadgen: %v\n", err)
|
||||
@ -86,80 +72,13 @@ func execute(ctx context.Context, args []string, run loadRun, stdout, stderr io.
|
||||
_, _ = fmt.Fprintf(stderr, "proxy-loadgen: %v\n", err)
|
||||
return 1
|
||||
}
|
||||
if configured {
|
||||
acceptance, acceptanceErr := loadgen.EvaluateAcceptance(report, criteria)
|
||||
if acceptanceErr != nil {
|
||||
_, _ = fmt.Fprintf(stderr, "proxy-loadgen: %v\n", acceptanceErr)
|
||||
return 2
|
||||
}
|
||||
report.Acceptance = &acceptance
|
||||
}
|
||||
if err := writeReportFile(*outputPath, report); err != nil {
|
||||
_, _ = fmt.Fprintf(stderr, "proxy-loadgen: write report: %v\n", err)
|
||||
return 1
|
||||
}
|
||||
if err := json.NewEncoder(stdout).Encode(report); err != nil {
|
||||
_, _ = fmt.Fprintf(stderr, "proxy-loadgen: encode report: %v\n", err)
|
||||
return 1
|
||||
}
|
||||
if report.Acceptance != nil && !report.Acceptance.Passed {
|
||||
_, _ = fmt.Fprintln(stderr, "proxy-loadgen: workload acceptance thresholds were not met")
|
||||
return 3
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
func acceptanceCriteria(maxErrorRate float64, maxP99 time.Duration) (loadgen.AcceptanceCriteria, bool, error) {
|
||||
if math.IsNaN(maxErrorRate) || math.IsInf(maxErrorRate, 0) || maxErrorRate < -1 || maxErrorRate > 1 || maxP99 < 0 {
|
||||
return loadgen.AcceptanceCriteria{}, false, loadgen.ErrInvalidAcceptanceCriteria
|
||||
}
|
||||
criteria := loadgen.AcceptanceCriteria{
|
||||
EnforceFailureRate: maxErrorRate >= 0,
|
||||
MaxFailureRate: maxErrorRate,
|
||||
EnforceP99Latency: maxP99 > 0,
|
||||
MaxP99Latency: maxP99,
|
||||
}
|
||||
return criteria, criteria.EnforceFailureRate || criteria.EnforceP99Latency, nil
|
||||
}
|
||||
|
||||
func writeReportFile(path string, report loadgen.Report) error {
|
||||
if path == "" {
|
||||
return nil
|
||||
}
|
||||
content, err := json.Marshal(report)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
content = append(content, '\n')
|
||||
directory := filepath.Dir(path)
|
||||
temporary, err := os.CreateTemp(directory, "."+filepath.Base(path)+".tmp-*")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
temporaryPath := temporary.Name()
|
||||
defer os.Remove(temporaryPath)
|
||||
if err := temporary.Chmod(0o600); err != nil {
|
||||
_ = temporary.Close()
|
||||
return err
|
||||
}
|
||||
if _, err := temporary.Write(content); err != nil {
|
||||
_ = temporary.Close()
|
||||
return err
|
||||
}
|
||||
if err := temporary.Close(); err != nil {
|
||||
return err
|
||||
}
|
||||
// Prefer same-directory rename so a completed report replaces the old one
|
||||
// atomically on platforms that support replacement.
|
||||
if err := os.Rename(temporaryPath, path); err == nil {
|
||||
return nil
|
||||
}
|
||||
if err := os.Remove(path); err != nil && !os.IsNotExist(err) {
|
||||
return err
|
||||
}
|
||||
return os.Rename(temporaryPath, path)
|
||||
}
|
||||
|
||||
type headerValues []string
|
||||
|
||||
func (values *headerValues) String() string {
|
||||
|
||||
@ -4,9 +4,6 @@ import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"math"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
@ -21,10 +18,7 @@ func TestExecutePassesBoundedWorkloadAndWritesJSON(t *testing.T) {
|
||||
"-requests", "3", "-concurrency", "2", "-timeout", "2s", "-header", "X-Run: fixed", "-body", `{"count":1}`,
|
||||
}, func(_ context.Context, options loadgen.Options) (loadgen.Report, error) {
|
||||
received = options
|
||||
return loadgen.Report{
|
||||
Requests: 3, Completed: 3, Succeeded: 3, Duration: time.Second,
|
||||
RateStartsGenerated: 3, RateStartsDropped: 1,
|
||||
}, nil
|
||||
return loadgen.Report{Requests: 3, Completed: 3, Succeeded: 3, Duration: time.Second}, nil
|
||||
}, &stdout, &stderr)
|
||||
if code != 0 || received.TargetURL != "https://target.example/path" || received.ProxyURL != "http://gateway.example:8080" ||
|
||||
received.Method != "post" || received.Requests != 3 || received.Duration != 0 || received.Concurrency != 2 ||
|
||||
@ -32,8 +26,7 @@ func TestExecutePassesBoundedWorkloadAndWritesJSON(t *testing.T) {
|
||||
t.Fatalf("execute() = %d; options=%+v stderr=%q", code, received, stderr.String())
|
||||
}
|
||||
var report loadgen.Report
|
||||
if err := json.Unmarshal(stdout.Bytes(), &report); err != nil || report.Succeeded != 3 ||
|
||||
report.RateStartsGenerated != 3 || report.RateStartsDropped != 1 {
|
||||
if err := json.Unmarshal(stdout.Bytes(), &report); err != nil || report.Succeeded != 3 {
|
||||
t.Fatalf("JSON report = (%+v, %v)", report, err)
|
||||
}
|
||||
}
|
||||
@ -77,85 +70,3 @@ func TestExecutePassesExtractScenario(t *testing.T) {
|
||||
t.Fatalf("execute() = %d; options=%+v stderr=%q", code, received, stderr.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecuteWritesReportFile(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
path := filepath.Join(t.TempDir(), "load-report.json")
|
||||
if err := os.WriteFile(path, []byte("incomplete"), 0o600); err != nil {
|
||||
t.Fatalf("WriteFile(): %v", err)
|
||||
}
|
||||
var stdout, stderr bytes.Buffer
|
||||
code := execute(context.Background(), []string{
|
||||
"-target", "https://target.example/health", "-requests", "1", "-concurrency", "1", "-output", path,
|
||||
}, func(_ context.Context, _ loadgen.Options) (loadgen.Report, error) {
|
||||
return loadgen.Report{Requests: 1, Completed: 1, Succeeded: 1, Duration: time.Second}, nil
|
||||
}, &stdout, &stderr)
|
||||
if code != 0 || stderr.Len() != 0 {
|
||||
t.Fatalf("execute() = %d, stderr=%q", code, stderr.String())
|
||||
}
|
||||
content, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
t.Fatalf("ReadFile(): %v", err)
|
||||
}
|
||||
var report loadgen.Report
|
||||
if err := json.Unmarshal(content, &report); err != nil || report.Succeeded != 1 || string(content) == "incomplete" {
|
||||
t.Fatalf("report file = (%q, %+v, %v)", content, report, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecuteWritesReportBeforeReturningAcceptanceFailure(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "load-report.json")
|
||||
var stdout, stderr bytes.Buffer
|
||||
code := execute(context.Background(), []string{
|
||||
"-target", "https://target.example/health", "-requests", "10", "-concurrency", "1",
|
||||
"-max-error-rate", "0.1", "-max-p99", "100ms", "-output", path,
|
||||
}, func(_ context.Context, _ loadgen.Options) (loadgen.Report, error) {
|
||||
return loadgen.Report{
|
||||
Requests: 10, Completed: 10, Failed: 2, Duration: time.Second,
|
||||
Latency: loadgen.LatencyReport{Samples: 10, P99UpperBound: 250 * time.Millisecond},
|
||||
}, nil
|
||||
}, &stdout, &stderr)
|
||||
if code != 3 || stderr.Len() == 0 {
|
||||
t.Fatalf("execute() = %d, stderr=%q", code, stderr.String())
|
||||
}
|
||||
|
||||
for _, content := range [][]byte{stdout.Bytes(), mustReadFile(t, path)} {
|
||||
var report loadgen.Report
|
||||
if err := json.Unmarshal(content, &report); err != nil || report.Acceptance == nil || report.Acceptance.Passed ||
|
||||
len(report.Acceptance.Violations) != 2 {
|
||||
t.Fatalf("acceptance report = (%+v, %v)", report, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecuteRejectsInvalidAcceptanceThresholdBeforeRunningWorkload(t *testing.T) {
|
||||
var stdout, stderr bytes.Buffer
|
||||
called := false
|
||||
code := execute(context.Background(), []string{
|
||||
"-target", "https://target.example/health", "-requests", "1", "-max-error-rate", "1.1",
|
||||
}, func(_ context.Context, _ loadgen.Options) (loadgen.Report, error) {
|
||||
called = true
|
||||
return loadgen.Report{}, nil
|
||||
}, &stdout, &stderr)
|
||||
if code != 2 || called || stdout.Len() != 0 || stderr.Len() == 0 {
|
||||
t.Fatalf("execute() = %d, called=%t stdout=%q stderr=%q", code, called, stdout.String(), stderr.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestAcceptanceCriteriaRejectsNonFiniteErrorRate(t *testing.T) {
|
||||
for _, value := range []float64{math.NaN(), math.Inf(1)} {
|
||||
if _, _, err := acceptanceCriteria(value, 0); err == nil {
|
||||
t.Fatalf("acceptanceCriteria(%v) accepted a non-finite error rate", value)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func mustReadFile(t *testing.T, path string) []byte {
|
||||
t.Helper()
|
||||
content, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
t.Fatalf("ReadFile(%q): %v", path, err)
|
||||
}
|
||||
return content
|
||||
}
|
||||
|
||||
@ -5,24 +5,10 @@ Grafana 与 Kubernetes。镜像会构建 `proxy-controller`、`proxy-gateway`
|
||||
`proxy-checker`;Controller 已装配 Redis BASIC 检查任务,Checker 可通过认证
|
||||
控制面执行 HTTP/HTTPS BASIC 探测。
|
||||
|
||||
Compose 使用 `config/local.yaml` 启用 Controller、两个 Gateway 和一个 Checker 的
|
||||
mTLS 控制面。`generate-local-controlplane-certs.ps1` 为三个固定工作负载生成不同的
|
||||
SPIFFE URI 证书,私钥只写入被忽略的 `deploy/.control-plane-tls/`。运行前先执行:
|
||||
|
||||
```powershell
|
||||
./scripts/generate-local-controlplane-certs.ps1
|
||||
docker compose -f deploy/docker-compose.yml up -d --build
|
||||
```
|
||||
|
||||
Kubernetes Base 仍保持 `controlPlane.enabled: false`。`kubernetes/overlays/development-mtls`
|
||||
启用单副本 Controller/Gateway/Checker mTLS 拓扑,挂载三组外部 TLS Secret,并从
|
||||
`gateway-a`、`checker-a` 的证书 URI 自动派生身份。具体创建 Secret 和部署步骤见
|
||||
[development mTLS overlay](kubernetes/overlays/development-mtls/README.md)。生产 Overlay
|
||||
必须为每个弹性 Gateway/Checker 工作负载配置唯一身份与证书轮换,不能复用固定开发证书。
|
||||
身份系统只需把每 Pod 的叶证书、私钥和信任包写入配置指定路径,并签发唯一
|
||||
`.../worker/<worker-id>` 或 `.../checker/<checker-id>` URI;启动命令设置
|
||||
`-auto-identity` 后由进程派生逻辑/缺省实例 ID,不依赖手工环境变量。开发 Compose 和
|
||||
此 Overlay 已启用该模式;其中证书只适用于本地开发,不代表生产身份签发与挂载实现。
|
||||
默认 Compose 和 Kubernetes Base 仍将 `controlPlane.enabled` 保持为 `false`。
|
||||
跨节点控制面必须由环境 Overlay 提供 Controller 可拨号地址、独立 Worker/Checker
|
||||
身份及 mTLS 证书,因此这些默认清单不能视为完整的生产发布配置。`checker.yaml`
|
||||
保留为 Overlay 模板,尚未加入 Base。
|
||||
|
||||
当前可执行验证:
|
||||
|
||||
@ -30,7 +16,6 @@ Kubernetes Base 仍保持 `controlPlane.enabled: false`。`kubernetes/overlays/d
|
||||
docker compose -f deploy/docker-compose.yml config --quiet
|
||||
docker compose -f deploy/docker-compose.test.yml config --quiet
|
||||
kubectl kustomize deploy/kubernetes/base | Out-Null
|
||||
kubectl kustomize deploy/kubernetes/overlays/development-mtls | Out-Null
|
||||
go run ./deploy/tools/configcheck deploy/config/local.yaml
|
||||
```
|
||||
|
||||
@ -41,8 +26,3 @@ PostgreSQL Adapter 和对应执行脚本完成前,不把数据库契约记为
|
||||
|
||||
发布前仍须通过 `production-readiness.md` 中的一致性、安全、恢复、竞态与容量门禁,
|
||||
并完成代表性集群容量验证。
|
||||
|
||||
Grafana Overview 与 Prometheus 规则只引用代码已注册的低基数指标。它们覆盖 Gateway
|
||||
请求/端到端时延/Outcome、Controller 容量、Provider、Extraction、Checker 和 Drain;不按 Proxy、IP、
|
||||
Client、Upstream、Worker、Session 或完整 URL 聚合。`go test ./deploy` 会解析两类资产并拒绝
|
||||
不存在的指标与禁止标签,指标改名或新增面板时必须同步更新该契约。
|
||||
|
||||
@ -1,66 +0,0 @@
|
||||
package deploy
|
||||
|
||||
import (
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"go.yaml.in/yaml/v4"
|
||||
)
|
||||
|
||||
func TestCIValidatesProtocolAndDeploymentAssets(t *testing.T) {
|
||||
payload, err := os.ReadFile("../.github/workflows/ci.yml")
|
||||
if err != nil {
|
||||
t.Fatalf("read ci workflow: %v", err)
|
||||
}
|
||||
workflow := string(payload)
|
||||
var document map[string]any
|
||||
if err := yaml.Unmarshal(payload, &document); err != nil {
|
||||
t.Fatalf("parse ci workflow: %v", err)
|
||||
}
|
||||
jobs, ok := document["jobs"].(map[string]any)
|
||||
if !ok {
|
||||
t.Fatal("ci workflow has no jobs")
|
||||
}
|
||||
for _, name := range []string{"openapi", "proto", "image", "deployment", "test", "race", "integration"} {
|
||||
if _, exists := jobs[name]; !exists {
|
||||
t.Errorf("ci workflow has no %s job", name)
|
||||
}
|
||||
}
|
||||
for _, required := range []string{
|
||||
"./scripts/verify-openapi.ps1",
|
||||
"./scripts/verify-proto.ps1",
|
||||
"docker build --file deploy/docker/Dockerfile --tag proxy-pool:ci .",
|
||||
"docker compose -f deploy/docker-compose.yml config --quiet",
|
||||
"docker compose -f deploy/docker-compose.test.yml config --quiet",
|
||||
"go run ./deploy/tools/configcheck deploy/config/local.yaml",
|
||||
"kubectl kustomize deploy/kubernetes/base",
|
||||
"kubectl kustomize deploy/kubernetes/overlays/development-mtls",
|
||||
"go run ./deploy/tools/configcheck deploy/kubernetes/overlays/development-mtls/config.yaml",
|
||||
} {
|
||||
if !strings.Contains(workflow, required) {
|
||||
t.Errorf("ci workflow does not validate %q", required)
|
||||
}
|
||||
}
|
||||
deployment, ok := jobs["deployment"].(map[string]any)
|
||||
if !ok {
|
||||
t.Fatal("ci deployment job has invalid definition")
|
||||
}
|
||||
steps, ok := deployment["steps"].([]any)
|
||||
if !ok {
|
||||
t.Fatal("ci deployment job has no steps")
|
||||
}
|
||||
for _, step := range steps {
|
||||
definition, ok := step.(map[string]any)
|
||||
if !ok || definition["uses"] != "actions/setup-go@v5" {
|
||||
continue
|
||||
}
|
||||
options, ok := definition["with"].(map[string]any)
|
||||
if !ok || options["go-version-file"] != "go.mod" {
|
||||
t.Error("ci deployment job must pin actions/setup-go to go.mod")
|
||||
return
|
||||
}
|
||||
return
|
||||
}
|
||||
t.Error("ci deployment job must install the Go version declared by go.mod before running configcheck")
|
||||
}
|
||||
@ -1,7 +1,6 @@
|
||||
package deploy
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
@ -12,49 +11,6 @@ import (
|
||||
"go.yaml.in/yaml/v4"
|
||||
)
|
||||
|
||||
var observableMetricNames = map[string]struct{}{
|
||||
"proxy_pool_checker_observations_total": {},
|
||||
"proxy_pool_checker_tasks_dispatched_total": {},
|
||||
"proxy_pool_controller_capacity_active_upstreams": {},
|
||||
"proxy_pool_controller_capacity_available_slots": {},
|
||||
"proxy_pool_controller_capacity_effective_slots": {},
|
||||
"proxy_pool_controller_capacity_inventory_reads_total": {},
|
||||
"proxy_pool_controller_capacity_managed_proxies": {},
|
||||
"proxy_pool_controller_capacity_pending_expected_proxies": {},
|
||||
"proxy_pool_controller_drain_candidates_total": {},
|
||||
"proxy_pool_controller_drains_started_total": {},
|
||||
"proxy_pool_controller_extraction_requested_proxies_total": {},
|
||||
"proxy_pool_controller_extraction_requests_total": {},
|
||||
"proxy_pool_controller_extraction_returned_proxies_total": {},
|
||||
"proxy_pool_controller_provider_fetch_results_total": {},
|
||||
"proxy_pool_controller_provider_new_proxies_total": {},
|
||||
"proxy_pool_controller_provider_valid_candidates_total": {},
|
||||
"proxy_pool_gateway_active_tunnels": {},
|
||||
"proxy_pool_gateway_capacity_invariant_violations_total": {},
|
||||
"proxy_pool_gateway_outcome_queue_dropped_total": {},
|
||||
"proxy_pool_gateway_outcomes_total": {},
|
||||
"proxy_pool_gateway_request_duration_seconds": {},
|
||||
"proxy_pool_gateway_requests_in_flight": {},
|
||||
"proxy_pool_gateway_requests_total": {},
|
||||
}
|
||||
|
||||
type dashboardDocument struct {
|
||||
Panels []struct {
|
||||
Targets []struct {
|
||||
Expression string `json:"expr"`
|
||||
} `json:"targets"`
|
||||
} `json:"panels"`
|
||||
}
|
||||
|
||||
type alertRulesDocument struct {
|
||||
Groups []struct {
|
||||
Rules []struct {
|
||||
Expression string `yaml:"expr"`
|
||||
Alert string `yaml:"alert"`
|
||||
} `yaml:"rules"`
|
||||
} `yaml:"groups"`
|
||||
}
|
||||
|
||||
type composeDocument struct {
|
||||
Services map[string]composeService `yaml:"services"`
|
||||
Volumes map[string]any `yaml:"volumes"`
|
||||
@ -64,10 +20,8 @@ type composeService struct {
|
||||
Image string `yaml:"image"`
|
||||
Command []string `yaml:"command"`
|
||||
Ports []string `yaml:"ports"`
|
||||
Expose []string `yaml:"expose"`
|
||||
Volumes []string `yaml:"volumes"`
|
||||
Tmpfs []string `yaml:"tmpfs"`
|
||||
Environment map[string]string `yaml:"environment"`
|
||||
DependsOn any `yaml:"depends_on"`
|
||||
}
|
||||
|
||||
@ -179,65 +133,6 @@ func TestLocalGatewaysDoNotDependOnControlPlaneStorage(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestLocalComposeProvidesAuthenticatedControlPlaneForEveryWorker(t *testing.T) {
|
||||
document := loadComposeDocument(t)
|
||||
controller := document.Services["controller"]
|
||||
if !slices.Contains(controller.Expose, "8443") || !slices.Contains(controller.Volumes, "./.control-plane-tls/controller:/run/proxy-pool-tls/server:ro") {
|
||||
t.Fatalf("controller does not expose and mount control-plane TLS: expose=%v volumes=%v", controller.Expose, controller.Volumes)
|
||||
}
|
||||
for _, expected := range []struct {
|
||||
name string
|
||||
certificate string
|
||||
}{
|
||||
{name: "gateway-a", certificate: "gateway-a"},
|
||||
{name: "gateway-b", certificate: "gateway-b"},
|
||||
{name: "checker", certificate: "checker-a"},
|
||||
} {
|
||||
service, ok := document.Services[expected.name]
|
||||
if !ok {
|
||||
t.Fatalf("docker-compose.yml has no %s", expected.name)
|
||||
}
|
||||
if service.Environment["PROXY_POOL_CONTROL_PLANE_ADDRESS"] != "controller:8443" ||
|
||||
!slices.Contains(service.Volumes, "./.control-plane-tls/"+expected.certificate+":/run/proxy-pool-tls/client:ro") ||
|
||||
!composeDependsOn(service.DependsOn, "controller") {
|
||||
t.Errorf("%s control-plane wiring = env=%v volumes=%v dependsOn=%v", expected.name, service.Environment, service.Volumes, service.DependsOn)
|
||||
}
|
||||
if expected.name == "checker" {
|
||||
if service.Environment["PROXY_POOL_AUTO_IDENTITY"] != "true" ||
|
||||
service.Environment["PROXY_POOL_CHECKER_ID"] != "" ||
|
||||
service.Environment["PROXY_POOL_CHECKER_INSTANCE_ID"] != "" ||
|
||||
service.Environment["PROXY_POOL_CHECKER_MAX_IN_FLIGHT"] != "200" || !slices.Contains(service.Expose, "9090") {
|
||||
t.Errorf("checker identity or metrics wiring = env=%v expose=%v", service.Environment, service.Expose)
|
||||
}
|
||||
continue
|
||||
}
|
||||
if service.Environment["PROXY_POOL_AUTO_IDENTITY"] != "true" ||
|
||||
service.Environment["PROXY_POOL_WORKER_ID"] != "" ||
|
||||
service.Environment["PROXY_POOL_INSTANCE_ID"] != "" ||
|
||||
service.Environment["PROXY_POOL_CLUSTER_ID"] != "compose-local" || service.Environment["PROXY_POOL_ZONE"] != "compose-local" {
|
||||
t.Errorf("%s identity wiring = %v", expected.name, service.Environment)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestLocalControlPlaneCertificateGeneratorUsesDistinctSPIFFERoles(t *testing.T) {
|
||||
payload, err := os.ReadFile("../scripts/generate-local-controlplane-certs.ps1")
|
||||
if err != nil {
|
||||
t.Fatalf("read generate-local-controlplane-certs.ps1: %v", err)
|
||||
}
|
||||
script := string(payload)
|
||||
for _, required := range []string{
|
||||
"OutputDirectory must be empty", "DNS:controller",
|
||||
"spiffe://proxy-pool.local/development/worker/gateway-a",
|
||||
"spiffe://proxy-pool.local/development/worker/gateway-b",
|
||||
"spiffe://proxy-pool.local/development/checker/checker-a",
|
||||
} {
|
||||
if !strings.Contains(script, required) {
|
||||
t.Errorf("certificate generator missing %q", required)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestDeploymentEntrypointsExistInSource(t *testing.T) {
|
||||
dockerfile, err := os.ReadFile("docker/Dockerfile")
|
||||
if err != nil {
|
||||
@ -265,93 +160,13 @@ func TestDeploymentEntrypointsExistInSource(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestDockerBuildContextExcludesLocalSecretsAndArtifacts(t *testing.T) {
|
||||
payload, err := os.ReadFile("../.dockerignore")
|
||||
if err != nil {
|
||||
t.Fatalf("read .dockerignore: %v", err)
|
||||
}
|
||||
rules := make(map[string]struct{})
|
||||
for _, raw := range strings.Split(string(payload), "\n") {
|
||||
rule := strings.TrimSpace(raw)
|
||||
if rule == "" || strings.HasPrefix(rule, "#") {
|
||||
continue
|
||||
}
|
||||
rules[rule] = struct{}{}
|
||||
}
|
||||
for _, required := range []string{
|
||||
".git", ".env", ".env.*", "*.key", "*.pem", "*.crt",
|
||||
"deploy/.control-plane-tls/", "bin/", "dist/", "coverage/", ".tmp-proto/",
|
||||
} {
|
||||
if _, exists := rules[required]; !exists {
|
||||
t.Errorf(".dockerignore does not exclude %q", required)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestKubernetesBaseLeavesControlPlaneClientsForMTLSOverlay(t *testing.T) {
|
||||
func TestKubernetesBaseExcludesUnimplementedProcessManifests(t *testing.T) {
|
||||
payload, err := os.ReadFile("kubernetes/base/kustomization.yaml")
|
||||
if err != nil {
|
||||
t.Fatalf("read kustomization.yaml: %v", err)
|
||||
}
|
||||
if strings.Contains(string(payload), "checker.yaml") {
|
||||
t.Fatal("kubernetes base includes checker.yaml without per-workload mTLS identity overlay")
|
||||
}
|
||||
}
|
||||
|
||||
func TestObservabilityAssetsUseRegisteredLowCardinalityMetrics(t *testing.T) {
|
||||
dashboardPayload, err := os.ReadFile("grafana/dashboards/proxy-pool-overview.json")
|
||||
if err != nil {
|
||||
t.Fatalf("read overview dashboard: %v", err)
|
||||
}
|
||||
var dashboard dashboardDocument
|
||||
if err := json.Unmarshal(dashboardPayload, &dashboard); err != nil {
|
||||
t.Fatalf("parse overview dashboard: %v", err)
|
||||
}
|
||||
expressions := make([]string, 0, len(dashboard.Panels))
|
||||
for _, panel := range dashboard.Panels {
|
||||
for _, target := range panel.Targets {
|
||||
expressions = append(expressions, target.Expression)
|
||||
}
|
||||
}
|
||||
rulesPayload, err := os.ReadFile("prometheus/rules/proxy-pool.yml")
|
||||
if err != nil {
|
||||
t.Fatalf("read Prometheus rules: %v", err)
|
||||
}
|
||||
var rules alertRulesDocument
|
||||
if err := yaml.Unmarshal(rulesPayload, &rules); err != nil {
|
||||
t.Fatalf("parse Prometheus rules: %v", err)
|
||||
}
|
||||
alerts := make([]string, 0)
|
||||
for _, group := range rules.Groups {
|
||||
for _, rule := range group.Rules {
|
||||
expressions = append(expressions, rule.Expression)
|
||||
alerts = append(alerts, rule.Alert)
|
||||
}
|
||||
}
|
||||
metricPattern := regexp.MustCompile(`proxy_pool_[a-z0-9_]+`)
|
||||
forbiddenLabel := regexp.MustCompile(`(?:by\s*\([^)]*\b(?:upstream|worker)\b|\{[^}]*\b(?:upstream|worker)\s*=)`)
|
||||
for _, expression := range expressions {
|
||||
for _, name := range metricPattern.FindAllString(expression, -1) {
|
||||
baseName := strings.TrimSuffix(strings.TrimSuffix(strings.TrimSuffix(name, "_bucket"), "_count"), "_sum")
|
||||
if _, exists := observableMetricNames[baseName]; !exists {
|
||||
t.Errorf("observability expression references unregistered metric %q: %s", name, expression)
|
||||
}
|
||||
}
|
||||
if forbiddenLabel.MatchString(expression) {
|
||||
t.Errorf("observability expression uses a forbidden high-cardinality label: %s", expression)
|
||||
}
|
||||
}
|
||||
for _, required := range []string{
|
||||
"proxy_pool_gateway_requests_total", "proxy_pool_gateway_request_duration_seconds", "proxy_pool_controller_capacity_available_slots",
|
||||
"proxy_pool_controller_provider_fetch_results_total", "proxy_pool_controller_extraction_requests_total",
|
||||
"proxy_pool_checker_observations_total",
|
||||
} {
|
||||
if !strings.Contains(strings.Join(expressions, "\n"), required) {
|
||||
t.Errorf("observability assets do not cover %s", required)
|
||||
}
|
||||
}
|
||||
if !slices.Contains(alerts, "ProxyPoolGatewayHighLatency") {
|
||||
t.Error("Prometheus rules do not alert on Gateway p99 latency")
|
||||
t.Fatal("kubernetes base includes checker.yaml before proxy-checker exists")
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@ -16,11 +16,6 @@ gateway:
|
||||
limits:
|
||||
maxConcurrentConnections: 20000
|
||||
requestsPerMinutePerClient: 60000
|
||||
transport:
|
||||
maxIdleConns: 20000
|
||||
maxIdleConnsPerHost: 32
|
||||
maxConnsPerHost: 32
|
||||
tunnelBufferBytes: 32768
|
||||
retry:
|
||||
maxAttempts: 2
|
||||
retryMethods: [GET, HEAD]
|
||||
@ -64,30 +59,7 @@ admin:
|
||||
token: "${PROXY_POOL_ADMIN_TOKEN}"
|
||||
|
||||
controlPlane:
|
||||
enabled: true
|
||||
listen: 0.0.0.0:8443
|
||||
protocolVersion: 1
|
||||
heartbeatInterval: 10s
|
||||
sessionTTL: 30s
|
||||
maxStaleAge: 30s
|
||||
maxMessageBytes: 4194304
|
||||
maxRuntimeCounters: 100000
|
||||
maxConcurrentStreams: 1000
|
||||
tls:
|
||||
mode: mtls
|
||||
certFile: /run/proxy-pool-tls/server/tls.crt
|
||||
keyFile: /run/proxy-pool-tls/server/tls.key
|
||||
clientCAFile: /run/proxy-pool-tls/server/ca.crt
|
||||
trustDomain: proxy-pool.local
|
||||
environment: development
|
||||
gatewayTLS:
|
||||
certFile: /run/proxy-pool-tls/client/tls.crt
|
||||
keyFile: /run/proxy-pool-tls/client/tls.key
|
||||
serverCAFile: /run/proxy-pool-tls/client/ca.crt
|
||||
checkerTLS:
|
||||
certFile: /run/proxy-pool-tls/client/tls.crt
|
||||
keyFile: /run/proxy-pool-tls/client/tls.key
|
||||
serverCAFile: /run/proxy-pool-tls/client/ca.crt
|
||||
enabled: false
|
||||
|
||||
metrics:
|
||||
enabled: true
|
||||
|
||||
@ -15,6 +15,8 @@ x-app: &app
|
||||
image: proxy-pool:local
|
||||
restart: unless-stopped
|
||||
networks: [frontend, backend]
|
||||
volumes:
|
||||
- ./config/local.yaml:/etc/proxy-pool/config.yaml:ro
|
||||
environment: *app-environment
|
||||
stop_grace_period: 45s
|
||||
|
||||
@ -22,18 +24,6 @@ services:
|
||||
gateway-a:
|
||||
<<: *app
|
||||
command: ["proxy-gateway"]
|
||||
environment:
|
||||
<<: *app-environment
|
||||
PROXY_POOL_CONTROL_PLANE_ADDRESS: controller:8443
|
||||
PROXY_POOL_CLUSTER_ID: compose-local
|
||||
PROXY_POOL_AUTO_IDENTITY: "true"
|
||||
PROXY_POOL_ZONE: compose-local
|
||||
volumes:
|
||||
- ./config/local.yaml:/etc/proxy-pool/config.yaml:ro
|
||||
- ./.control-plane-tls/gateway-a:/run/proxy-pool-tls/client:ro
|
||||
depends_on:
|
||||
controller:
|
||||
condition: service_healthy
|
||||
expose: ["8080", "9090"]
|
||||
healthcheck:
|
||||
test: ["CMD", "curl", "--fail", "--silent", "http://127.0.0.1:9090/readyz"]
|
||||
@ -45,18 +35,6 @@ services:
|
||||
gateway-b:
|
||||
<<: *app
|
||||
command: ["proxy-gateway"]
|
||||
environment:
|
||||
<<: *app-environment
|
||||
PROXY_POOL_CONTROL_PLANE_ADDRESS: controller:8443
|
||||
PROXY_POOL_CLUSTER_ID: compose-local
|
||||
PROXY_POOL_AUTO_IDENTITY: "true"
|
||||
PROXY_POOL_ZONE: compose-local
|
||||
volumes:
|
||||
- ./config/local.yaml:/etc/proxy-pool/config.yaml:ro
|
||||
- ./.control-plane-tls/gateway-b:/run/proxy-pool-tls/client:ro
|
||||
depends_on:
|
||||
controller:
|
||||
condition: service_healthy
|
||||
expose: ["8080", "9090"]
|
||||
healthcheck:
|
||||
test: ["CMD", "curl", "--fail", "--silent", "http://127.0.0.1:9090/readyz"]
|
||||
@ -71,15 +49,12 @@ services:
|
||||
environment:
|
||||
<<: *app-environment
|
||||
PROXY_POOL_CONFIG_FINGERPRINT_KEY: ${PROXY_POOL_CONFIG_FINGERPRINT_KEY:?set PROXY_POOL_CONFIG_FINGERPRINT_KEY}
|
||||
volumes:
|
||||
- ./config/local.yaml:/etc/proxy-pool/config.yaml:ro
|
||||
- ./.control-plane-tls/controller:/run/proxy-pool-tls/server:ro
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
redis:
|
||||
condition: service_healthy
|
||||
expose: ["8081", "8082", "8443", "9090"]
|
||||
expose: ["8081", "8082", "9090"]
|
||||
ports:
|
||||
- "127.0.0.1:8081:8081"
|
||||
- "127.0.0.1:8082:8082"
|
||||
@ -90,28 +65,6 @@ services:
|
||||
retries: 12
|
||||
start_period: 15s
|
||||
|
||||
checker:
|
||||
<<: *app
|
||||
command: ["proxy-checker"]
|
||||
environment:
|
||||
<<: *app-environment
|
||||
PROXY_POOL_CONTROL_PLANE_ADDRESS: controller:8443
|
||||
PROXY_POOL_AUTO_IDENTITY: "true"
|
||||
PROXY_POOL_CHECKER_MAX_IN_FLIGHT: "200"
|
||||
volumes:
|
||||
- ./config/local.yaml:/etc/proxy-pool/config.yaml:ro
|
||||
- ./.control-plane-tls/checker-a:/run/proxy-pool-tls/client:ro
|
||||
depends_on:
|
||||
controller:
|
||||
condition: service_healthy
|
||||
expose: ["9090"]
|
||||
healthcheck:
|
||||
test: ["CMD", "curl", "--fail", "--silent", "http://127.0.0.1:9090/readyz"]
|
||||
interval: 5s
|
||||
timeout: 2s
|
||||
retries: 12
|
||||
start_period: 10s
|
||||
|
||||
haproxy:
|
||||
image: haproxy:3.2-alpine
|
||||
restart: unless-stopped
|
||||
|
||||
@ -3,19 +3,13 @@
|
||||
"editable": true,
|
||||
"graphTooltip": 1,
|
||||
"panels": [
|
||||
{"type":"timeseries","title":"Gateway QPS","gridPos":{"h":8,"w":8,"x":0,"y":0},"targets":[{"expr":"sum by (protocol) (rate(proxy_pool_gateway_requests_total[1m]))","legendFormat":"{{protocol}}"}]},
|
||||
{"type":"timeseries","title":"Gateway p99","gridPos":{"h":8,"w":8,"x":8,"y":0},"targets":[{"expr":"histogram_quantile(0.99, sum by (le, protocol) (rate(proxy_pool_gateway_request_duration_seconds_bucket[5m])))","legendFormat":"{{protocol}}"}]},
|
||||
{"type":"timeseries","title":"Gateway In Flight and Tunnels","gridPos":{"h":8,"w":8,"x":16,"y":0},"targets":[{"expr":"sum by (protocol) (proxy_pool_gateway_requests_in_flight)","legendFormat":"in-flight {{protocol}}"},{"expr":"sum(proxy_pool_gateway_active_tunnels)","legendFormat":"tunnels"}]},
|
||||
{"type":"timeseries","title":"Gateway Outcome Failures","gridPos":{"h":8,"w":12,"x":0,"y":8},"targets":[{"expr":"sum by (stage) (rate(proxy_pool_gateway_outcomes_total{result=\"failure\"}[5m]))","legendFormat":"{{stage}}"}]},
|
||||
{"type":"timeseries","title":"Gateway Outcome Queue Drops","gridPos":{"h":8,"w":12,"x":12,"y":8},"targets":[{"expr":"sum(rate(proxy_pool_gateway_outcome_queue_dropped_total[5m]))","legendFormat":"drops/s"}]},
|
||||
{"type":"timeseries","title":"Controller Capacity","gridPos":{"h":8,"w":12,"x":0,"y":16},"targets":[{"expr":"sum(proxy_pool_controller_capacity_available_slots)","legendFormat":"available slots"},{"expr":"sum(proxy_pool_controller_capacity_effective_slots)","legendFormat":"effective slots"},{"expr":"sum(proxy_pool_controller_capacity_pending_expected_proxies)","legendFormat":"pending proxies"}]},
|
||||
{"type":"timeseries","title":"Managed Proxies and Active Upstreams","gridPos":{"h":8,"w":12,"x":12,"y":16},"targets":[{"expr":"sum(proxy_pool_controller_capacity_managed_proxies)","legendFormat":"managed proxies"},{"expr":"sum(proxy_pool_controller_capacity_active_upstreams)","legendFormat":"active upstreams"}]},
|
||||
{"type":"timeseries","title":"Provider Fetch Results","gridPos":{"h":8,"w":12,"x":0,"y":24},"targets":[{"expr":"sum by (class) (rate(proxy_pool_controller_provider_fetch_results_total[5m]))","legendFormat":"{{class}}"},{"expr":"sum(rate(proxy_pool_controller_provider_new_proxies_total[5m]))","legendFormat":"new proxies/s"}]},
|
||||
{"type":"timeseries","title":"Extraction Results","gridPos":{"h":8,"w":12,"x":12,"y":24},"targets":[{"expr":"sum by (result) (rate(proxy_pool_controller_extraction_requests_total[5m]))","legendFormat":"{{result}}"},{"expr":"sum(rate(proxy_pool_controller_extraction_returned_proxies_total[5m]))","legendFormat":"returned proxies/s"}]},
|
||||
{"type":"timeseries","title":"Checker Task Dispatch","gridPos":{"h":8,"w":12,"x":0,"y":32},"targets":[{"expr":"sum by (level) (rate(proxy_pool_checker_tasks_dispatched_total[5m]))","legendFormat":"{{level}}"}]},
|
||||
{"type":"timeseries","title":"Checker Observations","gridPos":{"h":8,"w":12,"x":12,"y":32},"targets":[{"expr":"sum by (level, result) (rate(proxy_pool_checker_observations_total[5m]))","legendFormat":"{{level}} {{result}}"}]},
|
||||
{"type":"timeseries","title":"Drain Tickets Started","gridPos":{"h":8,"w":12,"x":0,"y":40},"targets":[{"expr":"sum by (reason) (rate(proxy_pool_controller_drains_started_total[5m]))","legendFormat":"{{reason}}"}]},
|
||||
{"type":"timeseries","title":"Capacity Inventory Reads","gridPos":{"h":8,"w":12,"x":12,"y":40},"targets":[{"expr":"sum by (result) (rate(proxy_pool_controller_capacity_inventory_reads_total[5m]))","legendFormat":"{{result}}"}]}
|
||||
{"type":"timeseries","title":"Gateway QPS","gridPos":{"h":8,"w":8,"x":0,"y":0},"targets":[{"expr":"sum(rate(proxy_pool_gateway_requests_total[1m]))","legendFormat":"QPS"}]},
|
||||
{"type":"timeseries","title":"Gateway p99","gridPos":{"h":8,"w":8,"x":8,"y":0},"targets":[{"expr":"histogram_quantile(0.99, sum by (le) (rate(proxy_pool_gateway_request_duration_seconds_bucket[5m])))","legendFormat":"p99"}]},
|
||||
{"type":"timeseries","title":"Available Slots","gridPos":{"h":8,"w":8,"x":16,"y":0},"targets":[{"expr":"sum(proxy_pool_available_slots)","legendFormat":"slots"}]},
|
||||
{"type":"timeseries","title":"Provider Fetch","gridPos":{"h":8,"w":12,"x":0,"y":8},"targets":[{"expr":"sum by (result) (rate(proxy_pool_provider_fetch_total[5m]))","legendFormat":"{{result}}"}]},
|
||||
{"type":"timeseries","title":"Extraction","gridPos":{"h":8,"w":12,"x":12,"y":8},"targets":[{"expr":"sum by (result) (rate(proxy_pool_extraction_total[5m]))","legendFormat":"{{result}}"}]},
|
||||
{"type":"timeseries","title":"Snapshot Age","gridPos":{"h":8,"w":12,"x":0,"y":16},"targets":[{"expr":"max by (worker) (proxy_pool_snapshot_age_seconds)","legendFormat":"{{worker}}"}]},
|
||||
{"type":"timeseries","title":"Checker Queue","gridPos":{"h":8,"w":12,"x":12,"y":16},"targets":[{"expr":"sum(proxy_pool_checker_queue_depth)","legendFormat":"depth"}]}
|
||||
],
|
||||
"schemaVersion": 41,
|
||||
"tags": ["proxy-pool"],
|
||||
@ -23,5 +17,5 @@
|
||||
"time": {"from":"now-6h","to":"now"},
|
||||
"title": "Proxy Pool Overview",
|
||||
"uid": "proxy-pool-overview",
|
||||
"version": 2
|
||||
"version": 1
|
||||
}
|
||||
|
||||
@ -21,16 +21,6 @@ data:
|
||||
limits:
|
||||
maxConcurrentConnections: 100000
|
||||
requestsPerMinutePerClient: 60000
|
||||
transport:
|
||||
dialTimeout: 10s
|
||||
handshakeTimeout: 15s
|
||||
responseHeaderTimeout: 30s
|
||||
idleConnTimeout: 90s
|
||||
maxIdleConns: 20000
|
||||
maxIdleConnsPerHost: 32
|
||||
maxConnsPerHost: 32
|
||||
tunnelBufferBytes: 32768
|
||||
tunnelIdleTimeout: 5m
|
||||
retry:
|
||||
maxAttempts: 2
|
||||
retryMethods: [GET, HEAD]
|
||||
|
||||
@ -1,63 +0,0 @@
|
||||
# Kubernetes development mTLS overlay
|
||||
|
||||
该 Overlay 在 base 资源之上启用 Controller 控制面 mTLS,并启动一个 Gateway 和
|
||||
Checker。进程通过 `PROXY_POOL_AUTO_IDENTITY=true` 从各自证书的 URI SAN 派生身份;
|
||||
它只用于开发或预发布的单副本控制面验证,不是生产弹性身份方案。
|
||||
|
||||
## Prepare certificates
|
||||
|
||||
先生成未提交的开发证书。输出目录必须为空;生成器的 `controller` 证书包含
|
||||
`DNS:controller`,Overlay 中的客户端也只拨号 `controller:8443`。
|
||||
|
||||
```powershell
|
||||
./scripts/generate-local-controlplane-certs.ps1
|
||||
kubectl apply -f deploy/kubernetes/base/namespace.yaml
|
||||
```
|
||||
|
||||
将生成的三组证书写入 Kubernetes Secret。下面的命令可重复执行,只更新对应 Secret:
|
||||
|
||||
```powershell
|
||||
kubectl -n proxy-pool create secret generic proxy-pool-controlplane-server-tls `
|
||||
--from-file=tls.crt=deploy/.control-plane-tls/controller/tls.crt `
|
||||
--from-file=tls.key=deploy/.control-plane-tls/controller/tls.key `
|
||||
--from-file=ca.crt=deploy/.control-plane-tls/controller/ca.crt `
|
||||
--dry-run=client -o yaml | kubectl apply -f -
|
||||
|
||||
kubectl -n proxy-pool create secret generic proxy-pool-controlplane-gateway-tls `
|
||||
--from-file=tls.crt=deploy/.control-plane-tls/gateway-a/tls.crt `
|
||||
--from-file=tls.key=deploy/.control-plane-tls/gateway-a/tls.key `
|
||||
--from-file=ca.crt=deploy/.control-plane-tls/gateway-a/ca.crt `
|
||||
--dry-run=client -o yaml | kubectl apply -f -
|
||||
|
||||
kubectl -n proxy-pool create secret generic proxy-pool-controlplane-checker-tls `
|
||||
--from-file=tls.crt=deploy/.control-plane-tls/checker-a/tls.crt `
|
||||
--from-file=tls.key=deploy/.control-plane-tls/checker-a/tls.key `
|
||||
--from-file=ca.crt=deploy/.control-plane-tls/checker-a/ca.crt `
|
||||
--dry-run=client -o yaml | kubectl apply -f -
|
||||
```
|
||||
|
||||
另行从私密配置系统创建 `proxy-pool-secrets`。可参考 base 目录的
|
||||
`secret.example.yaml`,但不要把实际数据库、Redis、Provider 或 API 凭据写入 Git。
|
||||
|
||||
## Deploy
|
||||
|
||||
```powershell
|
||||
kubectl apply -k deploy/kubernetes/overlays/development-mtls
|
||||
kubectl -n proxy-pool rollout status deployment/proxy-controller --timeout=5m
|
||||
kubectl -n proxy-pool rollout status deployment/proxy-gateway --timeout=5m
|
||||
kubectl -n proxy-pool rollout status deployment/proxy-checker --timeout=5m
|
||||
```
|
||||
|
||||
Overlay 的 Gateway 从 `gateway-a` 证书派生 `worker_id=gateway-a`,Checker 从
|
||||
`checker-a` 证书派生 `checker_id=checker-a`,并把 Gateway HPA 限制为
|
||||
`minReplicas=maxReplicas=1`。因此不得在此 Overlay 上增加副本数或放宽 HPA;重复使用
|
||||
同一证书和 Worker 身份会破坏 session、ownership 和证书角色边界。
|
||||
|
||||
Overlay 的 `proxy-pool-runtime-config` 带内容哈希;修改 `config.yaml` 后,Kustomize 会重写三个
|
||||
Deployment 的 ConfigMap volume 引用,从而触发 Kubernetes 滚动更新。控制面 TLS 与
|
||||
`proxy-pool-secrets` 使用外部创建的固定名称 Secret,轮换后仍需按运行手册显式滚动对应工作负载。
|
||||
|
||||
生产环境需要由工作负载身份系统为每个副本签发独立、可轮换的证书,并将该副本的
|
||||
身份注入 Gateway Client TLS 或 Checker Client TLS,并启用自动身份派生。应用会在每次新
|
||||
控制面 TLS 握手读取更新后的叶证书和信任根;已有 gRPC 流仍按原会话保留,根轮换时先投放
|
||||
新旧根的重叠信任包,再按 PDB 滚动排空旧连接。
|
||||
@ -1,24 +0,0 @@
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: NetworkPolicy
|
||||
metadata:
|
||||
name: checker-control-plane
|
||||
namespace: proxy-pool
|
||||
spec:
|
||||
podSelector:
|
||||
matchLabels: {app.kubernetes.io/name: proxy-checker}
|
||||
policyTypes: [Ingress, Egress]
|
||||
ingress:
|
||||
- from:
|
||||
- namespaceSelector: {matchLabels: {kubernetes.io/metadata.name: monitoring}}
|
||||
ports: [{port: 9090, protocol: TCP}]
|
||||
egress:
|
||||
- to:
|
||||
- namespaceSelector: {matchLabels: {kubernetes.io/metadata.name: kube-system}}
|
||||
ports: [{port: 53, protocol: UDP}, {port: 53, protocol: TCP}]
|
||||
- to:
|
||||
- podSelector: {matchLabels: {app.kubernetes.io/name: proxy-controller}}
|
||||
ports: [{port: 8443, protocol: TCP}]
|
||||
- to:
|
||||
- ipBlock:
|
||||
cidr: 0.0.0.0/0
|
||||
except: [10.0.0.0/8, 100.64.0.0/10, 127.0.0.0/8, 169.254.0.0/16, 172.16.0.0/12, 192.168.0.0/16]
|
||||
@ -1,26 +0,0 @@
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: proxy-checker
|
||||
namespace: proxy-pool
|
||||
spec:
|
||||
replicas: 1
|
||||
template:
|
||||
spec:
|
||||
containers:
|
||||
- name: checker
|
||||
env:
|
||||
- {name: PROXY_POOL_CONTROL_PLANE_ADDRESS, value: controller:8443}
|
||||
- {name: PROXY_POOL_AUTO_IDENTITY, value: "true"}
|
||||
- {name: PROXY_POOL_CHECKER_MAX_IN_FLIGHT, value: "200"}
|
||||
volumeMounts:
|
||||
- name: controlplane-checker-tls
|
||||
mountPath: /run/proxy-pool-tls/client
|
||||
readOnly: true
|
||||
volumes:
|
||||
- name: config
|
||||
configMap:
|
||||
name: proxy-pool-runtime-config
|
||||
- name: controlplane-checker-tls
|
||||
secret:
|
||||
secretName: proxy-pool-controlplane-checker-tls
|
||||
@ -1,65 +0,0 @@
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: proxy-checker
|
||||
namespace: proxy-pool
|
||||
labels: {app.kubernetes.io/name: proxy-checker, app.kubernetes.io/part-of: proxy-pool}
|
||||
spec:
|
||||
replicas: 1
|
||||
minReadySeconds: 5
|
||||
revisionHistoryLimit: 3
|
||||
strategy:
|
||||
type: RollingUpdate
|
||||
rollingUpdate: {maxUnavailable: 1, maxSurge: 1}
|
||||
selector:
|
||||
matchLabels: {app.kubernetes.io/name: proxy-checker}
|
||||
template:
|
||||
metadata:
|
||||
labels: {app.kubernetes.io/name: proxy-checker, app.kubernetes.io/part-of: proxy-pool}
|
||||
annotations: {prometheus.io/scrape: "true", prometheus.io/port: "9090", prometheus.io/path: /metrics}
|
||||
spec:
|
||||
serviceAccountName: proxy-pool
|
||||
automountServiceAccountToken: false
|
||||
terminationGracePeriodSeconds: 45
|
||||
securityContext: {runAsNonRoot: true, seccompProfile: {type: RuntimeDefault}}
|
||||
topologySpreadConstraints:
|
||||
- maxSkew: 1
|
||||
topologyKey: kubernetes.io/hostname
|
||||
whenUnsatisfiable: ScheduleAnyway
|
||||
labelSelector: {matchLabels: {app.kubernetes.io/name: proxy-checker}}
|
||||
containers:
|
||||
- name: checker
|
||||
image: REGISTRY/proxy-pool:VERSION
|
||||
imagePullPolicy: IfNotPresent
|
||||
command: [proxy-checker]
|
||||
env:
|
||||
- {name: PROXY_POOL_CONFIG, value: /etc/proxy-pool/config.yaml}
|
||||
envFrom:
|
||||
- secretRef: {name: proxy-pool-secrets}
|
||||
ports:
|
||||
- {name: metrics, containerPort: 9090}
|
||||
readinessProbe:
|
||||
httpGet: {path: /readyz, port: metrics}
|
||||
periodSeconds: 5
|
||||
timeoutSeconds: 2
|
||||
livenessProbe:
|
||||
httpGet: {path: /livez, port: metrics}
|
||||
periodSeconds: 10
|
||||
timeoutSeconds: 2
|
||||
lifecycle:
|
||||
preStop: {exec: {command: [sh, -c, "sleep 3"]}}
|
||||
resources:
|
||||
requests: {cpu: "1", memory: 512Mi}
|
||||
limits: {cpu: "2", memory: 1Gi}
|
||||
securityContext:
|
||||
allowPrivilegeEscalation: false
|
||||
readOnlyRootFilesystem: true
|
||||
capabilities: {drop: [ALL]}
|
||||
volumeMounts:
|
||||
- {name: config, mountPath: /etc/proxy-pool, readOnly: true}
|
||||
- {name: tmp, mountPath: /tmp}
|
||||
volumes:
|
||||
- name: config
|
||||
configMap: {name: proxy-pool-config}
|
||||
- name: tmp
|
||||
emptyDir: {sizeLimit: 64Mi}
|
||||
@ -1,216 +0,0 @@
|
||||
version: 1
|
||||
|
||||
security:
|
||||
requireProtectionOnPublicListen: true
|
||||
|
||||
gateway:
|
||||
enabled: true
|
||||
listen: 0.0.0.0:8080
|
||||
access:
|
||||
allowCIDRs: [0.0.0.0/0]
|
||||
trustedProxies: []
|
||||
auth:
|
||||
mode: usernamePassword
|
||||
username: "${PROXY_POOL_GATEWAY_USERNAME}"
|
||||
password: "${PROXY_POOL_GATEWAY_PASSWORD}"
|
||||
limits:
|
||||
maxConcurrentConnections: 20000
|
||||
requestsPerMinutePerClient: 60000
|
||||
transport:
|
||||
maxIdleConns: 20000
|
||||
maxIdleConnsPerHost: 32
|
||||
maxConnsPerHost: 32
|
||||
tunnelBufferBytes: 32768
|
||||
retry:
|
||||
maxAttempts: 2
|
||||
retryMethods: [GET, HEAD]
|
||||
destinationPolicy:
|
||||
denyPrivateNetworks: true
|
||||
denyLoopback: true
|
||||
denyLinkLocal: true
|
||||
denyCIDRs: [169.254.169.254/32]
|
||||
|
||||
distribution:
|
||||
enabled: true
|
||||
listen: 0.0.0.0:8081
|
||||
access:
|
||||
allowCIDRs: [10.0.0.0/8]
|
||||
trustedProxies: [10.0.0.0/8]
|
||||
auth:
|
||||
mode: apiKey
|
||||
header: X-API-Key
|
||||
token: "${PROXY_POOL_EXTRACT_TOKEN}"
|
||||
limits:
|
||||
requestsPerMinute: 6000
|
||||
requestsPerMinutePerClient: 600
|
||||
clientIdentification:
|
||||
mode: sourceIP
|
||||
extraction:
|
||||
fulfillment: partial
|
||||
maxCountPerRequest: 100
|
||||
minRemainingTTL: 30s
|
||||
maxHealthCheckAge: 30s
|
||||
reserveForGateway: 1000
|
||||
idempotencyTTL: 5m
|
||||
|
||||
admin:
|
||||
enabled: true
|
||||
listen: 0.0.0.0:8082
|
||||
access:
|
||||
allowCIDRs: [10.0.0.0/8]
|
||||
auth:
|
||||
mode: apiKey
|
||||
header: X-Admin-Key
|
||||
token: "${PROXY_POOL_ADMIN_TOKEN}"
|
||||
|
||||
controlPlane:
|
||||
enabled: true
|
||||
listen: 0.0.0.0:8443
|
||||
protocolVersion: 1
|
||||
heartbeatInterval: 10s
|
||||
sessionTTL: 30s
|
||||
maxStaleAge: 30s
|
||||
maxMessageBytes: 4194304
|
||||
maxRuntimeCounters: 100000
|
||||
maxConcurrentStreams: 1000
|
||||
tls:
|
||||
mode: mtls
|
||||
certFile: /run/proxy-pool-tls/server/tls.crt
|
||||
keyFile: /run/proxy-pool-tls/server/tls.key
|
||||
clientCAFile: /run/proxy-pool-tls/server/ca.crt
|
||||
trustDomain: proxy-pool.local
|
||||
environment: development
|
||||
gatewayTLS:
|
||||
certFile: /run/proxy-pool-tls/client/tls.crt
|
||||
keyFile: /run/proxy-pool-tls/client/tls.key
|
||||
serverCAFile: /run/proxy-pool-tls/client/ca.crt
|
||||
checkerTLS:
|
||||
certFile: /run/proxy-pool-tls/client/tls.crt
|
||||
keyFile: /run/proxy-pool-tls/client/tls.key
|
||||
serverCAFile: /run/proxy-pool-tls/client/ca.crt
|
||||
|
||||
metrics:
|
||||
enabled: true
|
||||
listen: 0.0.0.0:9090
|
||||
|
||||
storage:
|
||||
postgresURL: "${PROXY_POOL_POSTGRES_URL}"
|
||||
redisURL: "${PROXY_POOL_REDIS_URL}"
|
||||
|
||||
routing:
|
||||
- name: gateway-default
|
||||
enabled: true
|
||||
purpose: gateway
|
||||
upstreams: [provider-a, provider-b]
|
||||
strategy:
|
||||
type: sequential
|
||||
switchAfterEmptyFetch: 5
|
||||
endBehavior: stayLast
|
||||
onUnavailable:
|
||||
action: reject
|
||||
- name: extract-default
|
||||
enabled: true
|
||||
purpose: extract
|
||||
upstreams: [provider-a, provider-b]
|
||||
strategy:
|
||||
type: sequential
|
||||
switchAfterEmptyFetch: 5
|
||||
endBehavior: stayLast
|
||||
onUnavailable:
|
||||
action: reject
|
||||
|
||||
upstreams:
|
||||
provider-a:
|
||||
enabled: true
|
||||
exposure: [gateway, extract]
|
||||
provider:
|
||||
billingMode: fetch
|
||||
protocols: [http]
|
||||
api:
|
||||
url: https://provider-a.invalid/api/proxies
|
||||
method: GET
|
||||
auth:
|
||||
type: apiKey
|
||||
location: header
|
||||
name: Authorization
|
||||
value: "${PROVIDER_A_TOKEN}"
|
||||
template: '{{ . }}'
|
||||
proxyAuth:
|
||||
type: response
|
||||
pool:
|
||||
maxSize: 5000
|
||||
shrinkDelay: 30s
|
||||
capacity:
|
||||
maxConcurrencyPerProxy: 20
|
||||
refill:
|
||||
reconcileInterval: 1s
|
||||
minimumAvailableSlots: 8000
|
||||
targetAvailableSlots: 12000
|
||||
lifecycle:
|
||||
ttl: 5m
|
||||
allocationSafetyMargin: 20s
|
||||
fetch:
|
||||
estimatedIPsPerCall: 100
|
||||
requestInterval: 1s
|
||||
timeout: 3s
|
||||
maxAttempts: 3
|
||||
maxInFlight: 1
|
||||
maxTotal: 100000
|
||||
maxResponseBytes: 4194304
|
||||
templateTimeout: 100ms
|
||||
retry: {initial: 500ms, max: 30s, jitter: 20}
|
||||
check:
|
||||
interval: 30s
|
||||
jitter: 20
|
||||
maxInFlight: 200
|
||||
timeout: 3s
|
||||
maxAttempts: 2
|
||||
maxConsecutiveFailures: 3
|
||||
urls: [https://example.com/]
|
||||
provider-b:
|
||||
enabled: true
|
||||
exposure: [gateway, extract]
|
||||
provider:
|
||||
billingMode: fetch
|
||||
protocols: [http]
|
||||
api:
|
||||
url: https://provider-b.invalid/api/proxies
|
||||
method: GET
|
||||
auth:
|
||||
type: apiKey
|
||||
location: header
|
||||
name: Authorization
|
||||
value: "${PROVIDER_B_TOKEN}"
|
||||
template: '{{ . }}'
|
||||
proxyAuth:
|
||||
type: response
|
||||
pool:
|
||||
maxSize: 5000
|
||||
shrinkDelay: 30s
|
||||
capacity:
|
||||
maxConcurrencyPerProxy: 20
|
||||
refill:
|
||||
reconcileInterval: 1s
|
||||
minimumAvailableSlots: 8000
|
||||
targetAvailableSlots: 12000
|
||||
lifecycle:
|
||||
ttl: 5m
|
||||
allocationSafetyMargin: 20s
|
||||
fetch:
|
||||
estimatedIPsPerCall: 100
|
||||
requestInterval: 1s
|
||||
timeout: 3s
|
||||
maxAttempts: 3
|
||||
maxInFlight: 1
|
||||
maxTotal: 100000
|
||||
maxResponseBytes: 4194304
|
||||
templateTimeout: 100ms
|
||||
retry: {initial: 500ms, max: 30s, jitter: 20}
|
||||
check:
|
||||
interval: 30s
|
||||
jitter: 20
|
||||
maxInFlight: 200
|
||||
timeout: 3s
|
||||
maxAttempts: 2
|
||||
maxConsecutiveFailures: 3
|
||||
urls: [https://example.com/]
|
||||
@ -1,6 +0,0 @@
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: proxy-pool-config
|
||||
namespace: proxy-pool
|
||||
$patch: delete
|
||||
@ -1,22 +0,0 @@
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: proxy-controller
|
||||
namespace: proxy-pool
|
||||
spec:
|
||||
replicas: 1
|
||||
template:
|
||||
spec:
|
||||
containers:
|
||||
- name: controller
|
||||
volumeMounts:
|
||||
- name: controlplane-server-tls
|
||||
mountPath: /run/proxy-pool-tls/server
|
||||
readOnly: true
|
||||
volumes:
|
||||
- name: config
|
||||
configMap:
|
||||
name: proxy-pool-runtime-config
|
||||
- name: controlplane-server-tls
|
||||
secret:
|
||||
secretName: proxy-pool-controlplane-server-tls
|
||||
@ -1,11 +0,0 @@
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: controller
|
||||
namespace: proxy-pool
|
||||
labels: {app.kubernetes.io/name: proxy-controller}
|
||||
spec:
|
||||
type: ClusterIP
|
||||
selector: {app.kubernetes.io/name: proxy-controller}
|
||||
ports:
|
||||
- {name: control, port: 8443, targetPort: control}
|
||||
@ -1,8 +0,0 @@
|
||||
apiVersion: autoscaling/v2
|
||||
kind: HorizontalPodAutoscaler
|
||||
metadata:
|
||||
name: proxy-gateway
|
||||
namespace: proxy-pool
|
||||
spec:
|
||||
minReplicas: 1
|
||||
maxReplicas: 1
|
||||
@ -1,27 +0,0 @@
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: proxy-gateway
|
||||
namespace: proxy-pool
|
||||
spec:
|
||||
replicas: 1
|
||||
template:
|
||||
spec:
|
||||
containers:
|
||||
- name: gateway
|
||||
env:
|
||||
- {name: PROXY_POOL_CONTROL_PLANE_ADDRESS, value: controller:8443}
|
||||
- {name: PROXY_POOL_CLUSTER_ID, value: kubernetes-development}
|
||||
- {name: PROXY_POOL_AUTO_IDENTITY, value: "true"}
|
||||
- {name: PROXY_POOL_ZONE, value: kubernetes-development}
|
||||
volumeMounts:
|
||||
- name: controlplane-gateway-tls
|
||||
mountPath: /run/proxy-pool-tls/client
|
||||
readOnly: true
|
||||
volumes:
|
||||
- name: config
|
||||
configMap:
|
||||
name: proxy-pool-runtime-config
|
||||
- name: controlplane-gateway-tls
|
||||
secret:
|
||||
secretName: proxy-pool-controlplane-gateway-tls
|
||||
@ -1,18 +0,0 @@
|
||||
apiVersion: kustomize.config.k8s.io/v1beta1
|
||||
kind: Kustomization
|
||||
resources:
|
||||
- ../../base
|
||||
- checker.yaml
|
||||
- controller-service.yaml
|
||||
- checker-networkpolicy.yaml
|
||||
configMapGenerator:
|
||||
- name: proxy-pool-runtime-config
|
||||
namespace: proxy-pool
|
||||
files:
|
||||
- config.yaml
|
||||
patches:
|
||||
- path: configmap-delete.yaml
|
||||
- path: controller-patch.yaml
|
||||
- path: gateway-patch.yaml
|
||||
- path: checker-patch.yaml
|
||||
- path: gateway-hpa-patch.yaml
|
||||
@ -1,73 +0,0 @@
|
||||
package deploy
|
||||
|
||||
import (
|
||||
"os"
|
||||
"os/exec"
|
||||
"regexp"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestKubernetesDevelopmentMTLSOverlayRendersCertificateDerivedIdentities(t *testing.T) {
|
||||
kubectl, err := exec.LookPath("kubectl")
|
||||
if err != nil {
|
||||
t.Skip("kubectl is required to render the Kubernetes development mTLS overlay")
|
||||
}
|
||||
command := exec.Command(kubectl, "kustomize", "kubernetes/overlays/development-mtls")
|
||||
command.Dir = "."
|
||||
payload, err := command.CombinedOutput()
|
||||
if err != nil {
|
||||
t.Fatalf("render development mTLS overlay: %v\n%s", err, payload)
|
||||
}
|
||||
rendered := string(payload)
|
||||
for _, required := range []string{
|
||||
"mode: mtls",
|
||||
"trustDomain: proxy-pool.local",
|
||||
"value: controller:8443",
|
||||
"name: PROXY_POOL_AUTO_IDENTITY",
|
||||
"value: \"true\"",
|
||||
"secretName: proxy-pool-controlplane-server-tls",
|
||||
"secretName: proxy-pool-controlplane-gateway-tls",
|
||||
"secretName: proxy-pool-controlplane-checker-tls",
|
||||
"name: proxy-checker",
|
||||
"name: checker-control-plane",
|
||||
"minReplicas: 1",
|
||||
"maxReplicas: 1",
|
||||
} {
|
||||
if !strings.Contains(rendered, required) {
|
||||
t.Errorf("development mTLS overlay does not render %q", required)
|
||||
}
|
||||
}
|
||||
for _, forbidden := range []string{"name: PROXY_POOL_WORKER_ID", "name: PROXY_POOL_INSTANCE_ID", "name: PROXY_POOL_CHECKER_ID", "name: PROXY_POOL_CHECKER_INSTANCE_ID"} {
|
||||
if strings.Contains(rendered, forbidden) {
|
||||
t.Errorf("development mTLS overlay still renders manual identity %q", forbidden)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestKubernetesDevelopmentMTLSOverlayUsesVersionedConfigMap(t *testing.T) {
|
||||
payload, err := os.ReadFile("kubernetes/overlays/development-mtls/kustomization.yaml")
|
||||
if err != nil {
|
||||
t.Fatalf("read development mTLS kustomization: %v", err)
|
||||
}
|
||||
if strings.Contains(string(payload), "disableNameSuffixHash: true") {
|
||||
t.Fatal("development mTLS overlay disables ConfigMap versioning")
|
||||
}
|
||||
kubectl, err := exec.LookPath("kubectl")
|
||||
if err != nil {
|
||||
t.Skip("kubectl is required to render the Kubernetes development mTLS overlay")
|
||||
}
|
||||
command := exec.Command(kubectl, "kustomize", "kubernetes/overlays/development-mtls")
|
||||
command.Dir = "."
|
||||
renderedPayload, err := command.CombinedOutput()
|
||||
if err != nil {
|
||||
t.Fatalf("render development mTLS overlay: %v\n%s", err, renderedPayload)
|
||||
}
|
||||
matched := regexp.MustCompile(`(?m)^ name: (proxy-pool-runtime-config-[a-z0-9]+)$`).FindStringSubmatch(string(renderedPayload))
|
||||
if len(matched) != 2 {
|
||||
t.Fatalf("rendered overlay has no versioned config map: %s", renderedPayload)
|
||||
}
|
||||
if occurrences := strings.Count(string(renderedPayload), "name: "+matched[1]); occurrences != 4 {
|
||||
t.Fatalf("versioned config map reference count = %d, want config map plus three workload volumes", occurrences)
|
||||
}
|
||||
}
|
||||
@ -14,9 +14,6 @@ scrape_configs:
|
||||
- job_name: proxy-controller
|
||||
static_configs:
|
||||
- targets: [controller:9090]
|
||||
- job_name: proxy-checker
|
||||
static_configs:
|
||||
- targets: [checker:9090]
|
||||
- job_name: haproxy
|
||||
metrics_path: /metrics
|
||||
static_configs:
|
||||
|
||||
@ -1,74 +1,40 @@
|
||||
groups:
|
||||
- name: proxy-pool
|
||||
rules:
|
||||
- alert: ProxyPoolGatewayOutcomeFailureRate
|
||||
- alert: ProxyPoolGatewayHighErrorRate
|
||||
expr: |
|
||||
sum(rate(proxy_pool_gateway_outcomes_total{result="failure"}[5m]))
|
||||
/ clamp_min(sum(rate(proxy_pool_gateway_outcomes_total[5m])), 1) > 0.1
|
||||
sum(rate(proxy_pool_gateway_requests_total{result="error"}[5m]))
|
||||
/ clamp_min(sum(rate(proxy_pool_gateway_requests_total[5m])), 1) > 0.02
|
||||
for: 10m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: Gateway 代理尝试失败比例持续高于 10%
|
||||
- alert: ProxyPoolGatewayHighLatency
|
||||
expr: |
|
||||
histogram_quantile(0.99,
|
||||
sum by (le, protocol) (rate(proxy_pool_gateway_request_duration_seconds_bucket[5m]))) > 5
|
||||
for: 10m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: Gateway p99 请求时延持续高于 5 秒
|
||||
- alert: ProxyPoolGatewayOutcomeQueueDrops
|
||||
expr: sum(increase(proxy_pool_gateway_outcome_queue_dropped_total[5m])) > 0
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: Gateway Outcome 本地队列发生丢弃
|
||||
- alert: ProxyPoolNoAvailableSlots
|
||||
expr: |
|
||||
sum(proxy_pool_controller_capacity_active_upstreams) > 0
|
||||
and sum(proxy_pool_controller_capacity_available_slots) == 0
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: Controller 可分配容量耗尽
|
||||
- alert: ProxyPoolCapacityInventoryErrors
|
||||
expr: sum(rate(proxy_pool_controller_capacity_inventory_reads_total{result="error"}[10m])) > 0.2
|
||||
for: 10m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: Controller 容量库存读取持续失败
|
||||
- alert: ProxyPoolProviderFetchErrors
|
||||
expr: sum(rate(proxy_pool_controller_provider_fetch_results_total{class="error"}[10m])) > 0.2
|
||||
for: 10m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: Provider 获取持续失败
|
||||
- alert: ProxyPoolCheckerObservationRejections
|
||||
expr: |
|
||||
sum(rate(proxy_pool_checker_observations_total{result="rejected"}[5m]))
|
||||
/ clamp_min(sum(rate(proxy_pool_checker_observations_total[5m])), 1) > 0.05
|
||||
for: 10m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: Checker Observation 拒绝比例持续高于 5%
|
||||
- alert: ProxyPoolExtractionIdempotencyConflict
|
||||
expr: sum(rate(proxy_pool_controller_extraction_requests_total{result="idempotency_conflict"}[5m])) > 0
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: Distribution 幂等键冲突持续发生
|
||||
- alert: ProxyPoolControlPlaneTargetDown
|
||||
expr: up{job=~"proxy-gateway|proxy-controller|proxy-checker"} == 0
|
||||
summary: Gateway 错误率持续高于 2%
|
||||
- alert: ProxyPoolGatewaySnapshotStale
|
||||
expr: proxy_pool_snapshot_age_seconds > 60
|
||||
for: 2m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: Proxy Pool 控制面目标不可抓取
|
||||
summary: Gateway Snapshot 已超过安全陈旧时间
|
||||
- alert: ProxyPoolNoAvailableSlots
|
||||
expr: sum(proxy_pool_available_slots) == 0
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: Gateway 可分配容量耗尽
|
||||
- alert: ProxyPoolProviderFetchErrors
|
||||
expr: sum by (upstream) (rate(proxy_pool_provider_fetch_total{result="error"}[10m])) > 0.2
|
||||
for: 10m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: Provider Fetch 错误持续发生
|
||||
- alert: ProxyPoolExtractionConflict
|
||||
expr: sum(rate(proxy_pool_extraction_total{result="conflict"}[5m])) > 0
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: 独占提取发生持续事务冲突
|
||||
|
||||
@ -26,7 +26,6 @@ PostgreSQL 事务内提交。若把它们暴露成多个 Repository,调用方
|
||||
type Mutator interface {
|
||||
SetUpstreamEnabled(context.Context, SetUpstreamCommand) (MutationResult, error)
|
||||
SwitchRouting(context.Context, SwitchRoutingCommand) (MutationResult, error)
|
||||
DisableRouting(context.Context, DisableRoutingCommand) (MutationResult, error)
|
||||
CommitConfig(context.Context, CommitConfigCommand) (MutationResult, error)
|
||||
}
|
||||
|
||||
@ -44,7 +43,7 @@ type Outbox interface {
|
||||
}
|
||||
```
|
||||
|
||||
四个 mutation 方法保留现有 Admin/自动切换语义,事务、修订号、CAS、审计和 Outbox 都
|
||||
三个 mutation 方法保留现有 Admin 语义,事务、修订号、CAS、审计和 Outbox 都
|
||||
隐藏在模块实现内部。不会暴露 `BeginTx`、SQL executor 或五个可被错误组合的浅
|
||||
Repository。
|
||||
|
||||
@ -96,12 +95,6 @@ revision,因此并发提交或 Supervisor 同步的迟到旧版本不能覆盖
|
||||
并发使用同一 expected 值时最多一个请求成功。目标等于当前值且 expected 匹配时
|
||||
返回 `changed=false`,仍写审计但不写 Outbox。
|
||||
|
||||
`DisableRouting` 用于 Sequential 的末端 `stop`。它同样锁定目标 Routing,并要求
|
||||
`expectedCurrent` 仍等于权威当前值;满足条件时保留 `current_upstream`、仅将
|
||||
`enabled` 置为 `false`。这样延迟的 Provider Empty 观察不会覆盖人工切换或配置重载。
|
||||
已停用且 expected 匹配时返回 `changed=false` 并审计,不再产生 Outbox;真实停用写入
|
||||
`routing.disabled` 事件。该行为与 `SwitchRouting` 共享同一个全局 revision 分配边界。
|
||||
|
||||
### Outbox 消费
|
||||
|
||||
Outbox 使用有界 claim/ack,而不是无界全表扫描:
|
||||
@ -136,8 +129,7 @@ MemoryStore 与 PostgreSQL Adapter 运行相同契约,至少覆盖:
|
||||
|
||||
- 配置首次提交、相同重放、校验和冲突和非法引用零写入。
|
||||
- Upstream enable/disable 幂等、修订单调、审计必写、Outbox 仅在变更时写。
|
||||
- Routing CAS、目标校验和 100 个并发请求最多一个成功;terminal stop 的 100 个
|
||||
并发请求仅一个真实停用,其余为幂等审计 no-op。
|
||||
- Routing CAS、目标校验和 100 个并发请求最多一个成功。
|
||||
- 任一审计/Outbox 写故障导致状态完全回滚。
|
||||
- Outbox 有界 claim、租约到期重试、错误 consumer ACK 拒绝和顺序稳定。
|
||||
- 审计按 ID 稳定分页并保留 Actor、资源、动作、修订和 UTC 时间;Routing no-op
|
||||
|
||||
@ -6,7 +6,6 @@ Admin API 使用独立监听器与权限,契约位于 `api/openapi/admin.yaml`
|
||||
## 端点
|
||||
|
||||
- `GET /api/v1/status`:返回配置/快照版本、Upstream 聚合计数和 Worker 状态。
|
||||
- `GET /api/v1/audit`:按审计记录 ID 升序读取权威管理面的变更记录。
|
||||
- `POST /api/v1/upstreams/{name}/enable`:启用 Upstream。
|
||||
- `POST /api/v1/upstreams/{name}/disable`:停止新 Fetch/分配并自然 Drain。
|
||||
- `POST /api/v1/routing/{name}/switch`:用 expectedCurrent 做 CAS 手工切换。
|
||||
@ -15,30 +14,9 @@ Admin API 使用独立监听器与权限,契约位于 `api/openapi/admin.yaml`
|
||||
所有写操作写审计记录并返回最终 Request ID 与版本。Enable/Disable 对目标状态
|
||||
幂等;Routing Switch 必须携带 `expectedCurrent`,避免并发操作跳过多个供应商。
|
||||
|
||||
## 授权
|
||||
|
||||
认证成功后的权限由 Listener `auth.permissions` 或 `auth.methods[].permissions` 提供。
|
||||
`admin:read` 允许 Status 与审计查询;`admin:write` 允许 Upstream 启停、Routing
|
||||
切换和配置重载。每个权限独立匹配,写权限不隐含读权限;未配置权限的旧凭据保持
|
||||
全权限兼容。`mode: any` 使用实际命中凭据的方法级权限。权限不足返回 `403`,不会
|
||||
调用 Service、写审计或触发配置发布。
|
||||
|
||||
配置重载校验失败返回 422,旧配置继续运行。Status 只返回低基数聚合信息,
|
||||
不得返回 Proxy 地址、凭据、Client 标识或完整 Provider URL。
|
||||
|
||||
## 审计分页与数据范围
|
||||
|
||||
`GET /api/v1/audit` 使用 `afterId` 排他游标分页:响应只包含 `id` 大于
|
||||
`afterId` 的记录,并按 `id` 升序排列;未传时 `afterId` 为 `0`。未传 `limit`
|
||||
时服务端使用 `100`;
|
||||
`limit` 必须为 `1..1000`,其中 `1000` 等于 `adminstate.MaxPageSize`。客户端应将
|
||||
本页最后一条记录的 `id` 作为下一次请求的 `afterId`;空 `records` 表示当前游标后
|
||||
没有记录。
|
||||
|
||||
该接口仅暴露管理面变更记录及操作者标识、可信来源 IP、操作类型、资源标识、变更
|
||||
状态、控制面版本、原因和发生时间。它不返回 Proxy、任何 Upstream/Provider URL、
|
||||
凭据或 Secret、Provider 响应载荷,也不读取或暴露 Redis 活动池及其提取状态。
|
||||
|
||||
## 运行时实现边界
|
||||
|
||||
`admin.Handler` 只依赖 `Service` 控制面接口,不直接操作数据库、路由游标或配置
|
||||
|
||||
@ -27,9 +27,7 @@ Controller 已实现并验证 `RegisterWorker`、`AcknowledgeSnapshot` 和
|
||||
|
||||
`WatchSnapshots` 已在 Register 后发送与当前 ownership epoch 对应的基础完整快照,并在
|
||||
每份快照有效期的一半前重新构建、下发版本递增的完整快照;这样 ownership、Routing 和
|
||||
凭据变化会在同一长连接内收敛,而无需等待有效期到达后重新注册。Controller 内已提交的
|
||||
Upstream 启停、Routing 切换和配置发布还会向全部本地 Worker 流广播一次合并后的立即刷新;
|
||||
多 Controller 副本仍由该定时机制完成跨进程收敛。
|
||||
凭据变化会在同一长连接内收敛,而无需等待有效期到达后重新注册。
|
||||
Gateway 校验后 ACK 并开始 Runtime 心跳。Controller 会从 Redis 的有界 Worker ownership
|
||||
索引构建已归属 Proxy 内容,并将租约到期收紧到 Proxy 的 `usable_until`。Proxy 引用的
|
||||
凭据材料按 `secret_ref + credential_version` 去重,随完整 Snapshot 经 mTLS 下发,仅保留在
|
||||
@ -236,15 +234,10 @@ Gateway 不复用 `controlPlane.listen` 作为客户端地址。`listen` 是 Con
|
||||
- `-worker-id` / `PROXY_POOL_WORKER_ID`:唯一逻辑 Worker。
|
||||
- `-instance-id` / `PROXY_POOL_INSTANCE_ID`:唯一进程实例。
|
||||
- `-zone` / `PROXY_POOL_ZONE`:实例可用区。
|
||||
- `-auto-identity` / `PROXY_POOL_AUTO_IDENTITY=true`:仅 mTLS 模式可用;从客户端
|
||||
证书中唯一的 `.../worker/<worker-id>` SPIFFE URI 派生 Worker ID,并在未显式配置时
|
||||
使用同一值作为 Instance ID。
|
||||
|
||||
当 `controlPlane.tls.mode=mtls` 时,Gateway 使用 `controlPlane.gatewayTLS` 中独立的
|
||||
客户端证书、私钥和 Controller CA 发起 TLS 1.3 连接;证书必须符合 Controller 的
|
||||
SPIFFE Worker 身份校验。客户端 X.509-SVID 叶证书必须且只能包含一个 URI SAN,且该 URI
|
||||
必须是对应角色的身份。Gateway 自动解析和 Controller 授权共用严格 URI 规则:身份 URI
|
||||
不得包含用户信息、端口、查询、片段或转义路径。
|
||||
SPIFFE Worker 身份校验。
|
||||
`disabled` 仅接受回环控制面地址,供本地 fixture 使用。
|
||||
|
||||
## 10. Checker 启动参数
|
||||
@ -254,8 +247,6 @@ SPIFFE Worker 身份校验。客户端 X.509-SVID 叶证书必须且只能包含
|
||||
- `-instance-id` / `PROXY_POOL_CHECKER_INSTANCE_ID`:唯一进程实例。
|
||||
- `-max-in-flight` / `PROXY_POOL_CHECKER_MAX_IN_FLIGHT`:本进程任务上限。
|
||||
- `-levels`:逗号分隔的 `basic,egress,target` 能力集合。
|
||||
- `-auto-identity` / `PROXY_POOL_AUTO_IDENTITY=true`:仅 mTLS 模式可用;从客户端
|
||||
证书中唯一的 `.../checker/<checker-id>` SPIFFE URI 派生 Checker ID 和缺省 Instance ID。
|
||||
|
||||
当 `controlPlane.tls.mode=mtls` 时,Checker 使用 `controlPlane.checkerTLS` 的独立
|
||||
客户端证书、私钥和 Controller CA 建立 TLS 1.3 连接;证书必须符合 Controller 的
|
||||
|
||||
@ -197,16 +197,6 @@ TTL 到期或 Redis 数据丢失后不再保证旧 Key 去重,系统不回退
|
||||
|
||||
错误响应不得包含 Provider Secret、Proxy 凭据、SQL 或内部拓扑。
|
||||
|
||||
`POST /api/v1/proxies/extract` 需要 `distribution:extract`。权限从 Listener 的
|
||||
`auth.permissions` 或 `mode: any` 的命中 `auth.methods[].permissions` 取得;缺少权限
|
||||
返回 `403`,不会读取请求体、消耗幂等键或调用提取服务。未声明权限的旧凭据保持全
|
||||
权限兼容。
|
||||
|
||||
Distribution 认证凭据还可声明 client 提取约束:maxExtractCount、
|
||||
allowedUpstreams 与 allowedRegions。该约束随实际命中的凭据返回,早于幂等键和
|
||||
提取服务执行。未传地区或 Upstream 过滤时,服务端自动写入凭据允许集合;传入
|
||||
集合外的值、或数量超过凭据上限时返回 403,并且不会消耗库存。
|
||||
|
||||
## 10. 短期运行记录与数据最小化
|
||||
|
||||
Redis 幂等结果只在配置的 TTL 窗口内保留重放响应所需的数据:
|
||||
|
||||
@ -122,122 +122,9 @@ Lua 可精确表示的整数范围内。
|
||||
- `any`:`methods` 中任一方法成功即可;Bearer 方法的 Secret 使用
|
||||
`value`/`valueFile`。
|
||||
|
||||
### 3.2 端点权限
|
||||
|
||||
`auth.permissions` 是普通认证方式(Basic、API Key、Bearer、IP 白名单)的固定权限
|
||||
集合。未配置时保留历史全权限行为;显式配置后只允许下列有限值:
|
||||
|
||||
- `admin:read`:读取 Admin Status 与审计页。
|
||||
- `admin:write`:执行 Upstream/Routing/配置变更。
|
||||
- `distribution:extract`:调用 Distribution 独占提取。
|
||||
- `*`:所有权限,且必须单独出现。
|
||||
|
||||
`mode: any` 必须将 `permissions` 写在各 `methods` 项中,权限随实际命中的凭据
|
||||
返回,顶层 `auth.permissions` 不允许同时配置。权限值不接受空白、重复或未知项。
|
||||
缺少所需权限的已认证请求返回 `403`,并在进入 JSON 解码、提取或管理 mutation 前
|
||||
结束。`mode: none` 不允许配置权限,避免把匿名访问误配置为受控授权。
|
||||
|
||||
```yaml
|
||||
admin:
|
||||
auth:
|
||||
mode: any
|
||||
methods:
|
||||
- mode: bearer
|
||||
valueFile: /run/secrets/admin-reader-token
|
||||
permissions: [admin:read]
|
||||
- mode: bearer
|
||||
valueFile: /run/secrets/admin-writer-token
|
||||
permissions: [admin:read, admin:write]
|
||||
```
|
||||
|
||||
### 3.3 Distribution Client 提取约束
|
||||
|
||||
认证凭据可在 auth.client 中设置固定的 Client 约束;mode: any 则必须在实际
|
||||
命中的 auth.methods 项下配置。当前 Client 约束按入口明确分工,避免出现只声明
|
||||
不执行的访问控制。
|
||||
|
||||
- maxExtractCount:Distribution 单次提取的额外上限;0 表示不追加上限。
|
||||
- allowedUpstreams:允许访问的 Upstream 名称集合。省略请求过滤条件时,服务端
|
||||
自动使用这个集合;请求携带集合外名称会返回 403。
|
||||
- allowedRegions:允许访问的地区集合,行为与 allowedUpstreams 相同。
|
||||
|
||||
空 client 配置保留旧版兼容语义,不会限制已认证凭据。Upstream 名称必须引用
|
||||
当前配置中已有的 Upstream。
|
||||
|
||||
~~~yaml
|
||||
distribution:
|
||||
auth:
|
||||
mode: any
|
||||
methods:
|
||||
- mode: apiKey
|
||||
header: X-API-Key
|
||||
valueFile: /run/secrets/tenant-a-key
|
||||
permissions: [distribution:extract]
|
||||
client:
|
||||
maxExtractCount: 20
|
||||
allowedUpstreams: [provider-a, provider-b]
|
||||
allowedRegions: [shanghai, beijing]
|
||||
~~~
|
||||
|
||||
### 3.4 Gateway Client 路由约束
|
||||
|
||||
Gateway 的 auth.client 支持 allowedRoutings、requestsPerMinute 和
|
||||
maxConcurrentConnections。认证成功后,
|
||||
Gateway 先完成目标地址策略和本地 Routing 匹配,再于本地 Dispatcher 前检查
|
||||
Routing 集合;不匹配返回 403,不会尝试选择或预留 Proxy。速率和并发限制均以
|
||||
认证 Client 为计数键,在每个 Gateway Worker 内执行,超过限制返回 429。三项
|
||||
检查都只读取请求上下文和当前本地状态,不访问 Redis、PostgreSQL 或 Provider。
|
||||
|
||||
allowedRoutings 中的每个名称必须对应一个启用的 Gateway Routing。Gateway 上的
|
||||
maxExtractCount、allowedUpstreams 和 allowedRegions,以及 Distribution 上的
|
||||
allowedRoutings 和 requestsPerMinute 均会被配置校验拒绝。Admin 不支持任何
|
||||
client 约束。
|
||||
|
||||
~~~yaml
|
||||
gateway:
|
||||
auth:
|
||||
mode: any
|
||||
methods:
|
||||
- mode: bearer
|
||||
valueFile: /run/secrets/checkout-gateway-token
|
||||
client:
|
||||
allowedRoutings: [checkout]
|
||||
requestsPerMinute: 600
|
||||
maxConcurrentConnections: 20
|
||||
~~~
|
||||
|
||||
Gateway、Distribution、Admin 与 Provider API 是独立认证边界。改变其中一套
|
||||
不得连带改变其他入口。
|
||||
|
||||
### 3.5 Gateway 粘性会话
|
||||
|
||||
gateway.stickySession 让调用方使用一个会话头尽量固定出口 Proxy。绑定键是认证
|
||||
Client、已匹配 Routing 和会话标识的组合,因此不同 Client 或不同 Routing 使用
|
||||
相同会话值也不会共享出口。该功能要求 Gateway 认证开启。
|
||||
|
||||
~~~yaml
|
||||
gateway:
|
||||
auth:
|
||||
mode: bearer
|
||||
tokenFile: /run/secrets/gateway-token
|
||||
stickySession:
|
||||
enabled: true
|
||||
header: X-Proxy-Session
|
||||
ttl: 20s
|
||||
maxEntries: 100000
|
||||
~~~
|
||||
|
||||
- header 必须是规范 HTTP Header 名;一个请求只能携带一个值,值由字母、数字、
|
||||
点、下划线、连字符组成且最长 128 字节。
|
||||
- ttl 是绑定的最长时长;实际到期时间还会被 Proxy 的 usableUntil、代理过期时间
|
||||
和 allocation safety margin 截短。
|
||||
- maxEntries 是当前 Gateway Worker 的硬上限。缓存只保存 Proxy ID 和过期时间,
|
||||
不保存 Proxy 地址、凭据或原始会话标识。
|
||||
- 会话头仅用于本地选路,Gateway 在转发 HTTP 请求前删除它。Proxy 不再符合当前
|
||||
Snapshot 或一次代理尝试失败时,绑定会被清除并按正常 Routing 重选。
|
||||
- 该版本不把 Redis、PostgreSQL 或 gRPC 引入 Gateway 热路径;跨 Worker 粘性需要
|
||||
上游负载均衡保持 Worker 亲和,后续由控制面快照协议扩展共享恢复能力。
|
||||
|
||||
## 4. Worker 控制面
|
||||
|
||||
控制面默认关闭;默认配置中的 `8443` 端口预留不表示服务已监听。控制面 Session、
|
||||
@ -300,21 +187,15 @@ controlPlane:
|
||||
- `gatewayTLS` 是 Gateway 的客户端证书、私钥和 Controller CA,与 `tls` 的服务端
|
||||
证书和 Worker CA 分离。三项可以同时省略(未运行 Gateway),配置任一项时必须完整提供。
|
||||
- `checkerTLS` 与 `gatewayTLS` 有相同字段和完整性校验,但必须使用独立的 Checker
|
||||
证书。每个客户端 X.509-SVID 叶证书必须且只能包含一个 URI SAN;Controller 分别验证
|
||||
`.../worker/<worker-id>` 与 `.../checker/<checker-id>` SPIFFE URI,不能跨角色复用证书。
|
||||
- Controller、Gateway 与 Checker 会在每个新的控制面 TLS 握手读取最新叶证书、私钥和
|
||||
信任根;文件暂时不可读时继续使用最后一次有效材料。既有 gRPC 流不强制重握手,根 CA
|
||||
轮换仍需先发布包含新旧根的重叠信任包,并在宽限期后滚动排空旧连接。
|
||||
证书。Controller 分别验证 `.../worker/<worker-id>` 与
|
||||
`.../checker/<checker-id>` SPIFFE URI,不能跨角色复用证书。
|
||||
|
||||
Gateway 连接 Controller 时使用独立启动参数而非 `controlPlane.listen`。至少设置
|
||||
`PROXY_POOL_CONTROL_PLANE_ADDRESS`、`PROXY_POOL_CLUSTER_ID`、
|
||||
`PROXY_POOL_WORKER_ID`、`PROXY_POOL_INSTANCE_ID` 和 `PROXY_POOL_ZONE`,详见
|
||||
[控制面协议](../api/control-plane.md#9-gateway-启动参数)。Compose 本地模板从被忽略的
|
||||
开发证书目录派生 Worker/Checker 身份;Kubernetes Base 保持
|
||||
`controlPlane.enabled: false`,必须由具备每工作负载唯一身份的 mTLS Overlay 启用。
|
||||
在 mTLS 生产工作负载中,`PROXY_POOL_AUTO_IDENTITY=true` 可省略 `WORKER_ID` 和
|
||||
`INSTANCE_ID`:进程仅接受其 `gatewayTLS` 叶证书中唯一的
|
||||
`spiffe://<trust-domain>/<environment>/worker/<worker-id>` URI,并以该 ID 作为缺省实例。
|
||||
[控制面协议](../api/control-plane.md#9-gateway-启动参数)。基础 Compose/Kubernetes
|
||||
模板保持 `controlPlane.enabled: false`,环境 Overlay 挂载 mTLS 证书并启用后才可启动
|
||||
Gateway。
|
||||
|
||||
Checker 同样使用独立的可拨号地址:`proxy-checker` 的 `-control-plane`、
|
||||
`-checker-id`、`-instance-id` 和 `-max-in-flight` 可由对应的
|
||||
@ -322,16 +203,9 @@ Checker 同样使用独立的可拨号地址:`proxy-checker` 的 `-control-pla
|
||||
模式只接受回环 Controller 地址。Checker 只从 gRPC 领取任务并批量上报事实,不读取
|
||||
Redis/PostgreSQL;Controller 在生产启动拓扑中装配 Redis 共享任务队列,当前调度
|
||||
HTTP/HTTPS/SOCKS5 BASIC、EGRESS 和 TARGET 检查。调度监督器在每轮从已发布配置读取启用的
|
||||
上游与 Routing;启用 Admin 时还要求 PostgreSQL 管理态与配置 revision 一致,并取两者均启用的
|
||||
上游和 Routing。管理态禁用的上游不会再产生新的 BASIC、EGRESS 或 TARGET 任务;管理态禁用的
|
||||
Routing 也不会再产生其 TARGET 任务;revision 不一致或状态不完整时本轮失败关闭。Admin reload 发布后,
|
||||
上游/路由启停、有效 `check` 策略和目标列表会在下一轮生效,
|
||||
上游与 Routing;Admin reload 发布后,上游/路由启停、有效 `check` 策略和目标列表会在下一轮生效,
|
||||
新启用的上游无需重启 Controller。
|
||||
|
||||
`PROXY_POOL_AUTO_IDENTITY=true` 对 Checker 使用同样规则:仅接受 `checkerTLS` 中唯一且仅有的
|
||||
`spiffe://<trust-domain>/<environment>/checker/<checker-id>` URI,并在未显式设置时复用其
|
||||
`checker-id` 作为实例 ID。
|
||||
|
||||
`maxRuntimeCounters` 同时限制单个 Runtime 报告和单个 Outcome 批次的条目数。Gateway
|
||||
在本地维护容量为 `65536` 的非阻塞 Outcome 队列,默认微批上限为 `512`,实际取二者中
|
||||
较小值;该队列与其序列确认状态仅存在于 Gateway 进程内。Controller 的 Redis 状态只保存
|
||||
@ -346,24 +220,9 @@ gateway:
|
||||
auth: {mode: none}
|
||||
limits:
|
||||
maxConcurrentConnections: 50000
|
||||
transport:
|
||||
dialTimeout: 10s
|
||||
handshakeTimeout: 15s
|
||||
responseHeaderTimeout: 30s
|
||||
idleConnTimeout: 90s
|
||||
maxIdleConns: 20000
|
||||
maxIdleConnsPerHost: 32
|
||||
maxConnsPerHost: 32
|
||||
tunnelBufferBytes: 32768
|
||||
tunnelIdleTimeout: 5m
|
||||
retry:
|
||||
maxAttempts: 2
|
||||
retryMethods: [GET, HEAD]
|
||||
stickySession:
|
||||
enabled: true
|
||||
header: X-Proxy-Session
|
||||
ttl: 20s
|
||||
maxEntries: 100000
|
||||
destinationPolicy:
|
||||
denyPrivateNetworks: true
|
||||
denyLoopback: true
|
||||
@ -381,31 +240,6 @@ gateway:
|
||||
- 保留地址、CGNAT 与云元数据端点始终拒绝,不能通过私网/链路本地开关放行。
|
||||
- `maxConcurrentConnections` 是入口准入上限,不是 Proxy 容量上限。
|
||||
|
||||
### 5.1 Gateway 本地传输层
|
||||
|
||||
**gateway.transport** 只配置每个 Gateway Worker 的本地 HTTP Transport 和 CONNECT
|
||||
隧道 I/O,不读取或写入 Redis、PostgreSQL、Provider,也不参与热路径分配。修改
|
||||
这些字段需要滚动重启对应 Gateway Worker 后生效。
|
||||
|
||||
- **dialTimeout**、**handshakeTimeout**、**responseHeaderTimeout**、
|
||||
**idleConnTimeout**:分别限制拨号、上游 TLS/CONNECT 握手、HTTP 响应头和空闲
|
||||
HTTP 连接的时长。
|
||||
- **maxIdleConns**:单个本地 HTTP Transport 的总空闲连接上限。Gateway 分别为代理
|
||||
转发和 direct fallback 建立 Transport,最坏情况下两者都可能保有空闲连接。
|
||||
- **maxIdleConnsPerHost**:单个上游 Host 的空闲 HTTP 连接上限。
|
||||
- **maxConnsPerHost**:单个上游 Host 的 HTTP 活跃加空闲连接上限。它不替代
|
||||
**limits.maxConcurrentConnections**,也不限制已经建立的 CONNECT 隧道。
|
||||
- **tunnelBufferBytes**:每个活跃 CONNECT 隧道的每个复制方向使用一个缓冲区,内存
|
||||
预算至少按 活跃隧道数 * 2 * tunnelBufferBytes 计算。
|
||||
- **tunnelIdleTimeout**:CONNECT 隧道双向没有数据活动时的最长存活时间。
|
||||
|
||||
所有字段均为可选项;数值 0 表示沿用进程默认值:拨号 10s、握手 15s、
|
||||
响应头 30s、空闲 HTTP 连接 90s、总空闲连接 1024、每 Host 空闲连接
|
||||
64、隧道缓冲 32768 字节、隧道空闲 5m。计数和时长不得为负数;
|
||||
maxIdleConnsPerHost 不得大于显式配置的 maxIdleConns。示例中的 20000/32
|
||||
是高并发 HTTP 转发的起点,不是每秒请求量的换算公式,应同时按可用文件描述符、
|
||||
上游限额和观测到的复用率调节。
|
||||
|
||||
## 6. Distribution
|
||||
|
||||
```yaml
|
||||
@ -457,7 +291,6 @@ routing:
|
||||
- name: api-post
|
||||
enabled: true
|
||||
purpose: gateway
|
||||
action: proxy
|
||||
match:
|
||||
hostRegex: '^api\\.example\\.com$'
|
||||
methods: [POST]
|
||||
@ -477,10 +310,6 @@ routing:
|
||||
|
||||
- Routing 列表有序,首条匹配后停止。
|
||||
- `purpose` 为 `gateway` 或 `extract`。
|
||||
- `action` 省略时为 `proxy`。`proxy` 使用 Upstream 与 Strategy;`direct` 直接连接已通过
|
||||
目标地址策略验证的目标;`reject` 拒绝匹配请求。`direct` 与 `reject` 仅允许
|
||||
`gateway` Routing,且不得同时配置 `upstreams`、`strategy`、`onUnavailable` 或
|
||||
`check.targets`。
|
||||
- `strategy.type` 支持 `sequential`、`random`、`roundRobin`、`weighted`、
|
||||
`leastConnections`。
|
||||
- `weighted` 使用 `weights` 映射,键必须引用本 Routing 的 Upstream。
|
||||
@ -488,10 +317,6 @@ routing:
|
||||
`switchAfterEmptyFetch`;`endBehavior` 省略时默认为 `stop`,也可显式设置
|
||||
`loop` 或 `stayLast`。
|
||||
- `onUnavailable.action` 为 `reject`、`wait` 或 `direct`;默认建议 `reject`。
|
||||
- `action: direct` 在 Dispatcher 前执行,不申请 Proxy 容量、不写 Proxy Outcome,也不建立
|
||||
粘性 Proxy 绑定;它仍执行入口认证、Client 路由约束、速率/并发限制和目标地址策略。
|
||||
`onUnavailable.action: direct` 仅是 `action: proxy` 在本地没有候选时的回退,二者
|
||||
不可互换。
|
||||
- `check.targets` 是 Routing 级 HTTP/HTTPS 探测目标。每个 Routing 最多 16 个 URL,且一个
|
||||
启用 Upstream 被其引用的 TARGET Profile 总数最多 64 个。每个 Profile 的身份是
|
||||
`(routing.name, target URL)`;失败只影响该 Profile,不改变 Proxy 的全局健康状态。
|
||||
@ -500,16 +325,6 @@ Sequential 的空计数属于 Upstream,当前索引属于 Routing。只有 Pro
|
||||
成功、模板成功且合法候选为零时才增加空计数。错误不改变空计数;重复候选
|
||||
会重置空计数但增加独立 duplicate 指标。
|
||||
|
||||
Controller 在连续空结果达到阈值时按顺序切换当前 Upstream;到达最后一个可用
|
||||
Upstream 且 `endBehavior: stop` 时,会以当前 Upstream 的 CAS 条件原子停用该
|
||||
Routing。停用后的 Gateway 规则按既有 `onUnavailable.action` 执行 `reject`、`wait`
|
||||
或 `direct`,直到一次配置重载提交新的管理快照。
|
||||
|
||||
管理态禁用当前 Sequential Upstream 不等待新的 Provider 空结果:Controller 会按
|
||||
配置顺序以相同的 `ExpectedCurrent` CAS 推进到下一个启用 Upstream,并立即刷新完整
|
||||
Snapshot。若当前项之后没有启用候选,只有 `endBehavior: loop` 会回绕搜索更早的候选;
|
||||
`stop` 与 `stayLast` 都会原子停用 Routing,因为已禁用的当前项不能继续承接新分配。
|
||||
|
||||
## 8. Upstream
|
||||
|
||||
```yaml
|
||||
@ -661,11 +476,6 @@ proxyAuth:
|
||||
- 启用 Routing 的 `check.targets` 会为其引用的每个启用 Upstream 创建 TARGET 检查组。BASIC、
|
||||
EGRESS 与 TARGET 使用固定批次和轮转顺序,并共享该 Upstream 的 `check.maxInFlight`,避免
|
||||
配置多个目标后产生无界检查流量。
|
||||
- 启用 Admin 时,调度仅使用配置和管理态均启用、且 revision 一致的 Upstream;管理态停用会在
|
||||
下一调度轮阻止新的 BASIC、EGRESS、TARGET 任务。revision 不一致、重复或缺失的上游状态按失败
|
||||
关闭处理,不读取 Redis due-index。
|
||||
- 启用 Admin 时,TARGET Profile 还要求其 Routing 在配置和管理态中均启用;管理态停用 Routing
|
||||
仅停止该 Routing 的新 TARGET 任务,不影响同一 Upstream 的 BASIC/EGRESS 或其他启用 Routing。
|
||||
- 第一次有意义失败进入 SUSPECT;达到 `maxConsecutiveFailures` 后才进入
|
||||
UNHEALTHY。
|
||||
- `unhealthyRemoveAfter` 控制 UNHEALTHY 持续多久后可由 Controller 回收;`0s`
|
||||
@ -716,20 +526,10 @@ Metrics 启用时 `listen` 必须是合法 `host:port`。该入口固定提供 `
|
||||
`result` 固定为 accepted、rejected。Gateway 另暴露
|
||||
`proxy_pool_gateway_outcomes_total{stage,result}` 与
|
||||
`proxy_pool_gateway_outcome_queue_dropped_total`;`stage` 固定为 DIAL、PROXY_HANDSHAKE、
|
||||
RESPONSE_HEADERS、TUNNEL,`result` 固定为 success、failure。Provider、Extraction、
|
||||
Capacity 和 Drain 业务指标也已接入,所有标签均为固定枚举;绝不包含 Proxy、IP、Checker、
|
||||
Client、路由、目标 URL 或凭据标签。Capacity 从既有 Provider 库存对账循环聚合,
|
||||
不进入 Gateway 热路径。Distribution 启用时 `/readyz` 只以 Redis 活动池为
|
||||
RESPONSE_HEADERS、TUNNEL,`result` 固定为 success、failure。绝不包含 Proxy、IP、Checker、
|
||||
Client、路由、目标 URL 或凭据标签。Provider、提取和容量等业务指标仍在后续实施范围。Distribution 启用时 `/readyz` 只以 Redis 活动池为
|
||||
服务流量门槛,PostgreSQL 故障由 Admin 接口独立报告。Metrics 开关或监听地址
|
||||
变更需要重启 Controller。Controller、Gateway 和 Checker 的进程级致命错误以 JSON
|
||||
结构化日志输出,包含组件和稳定错误类型,不输出错误原文;敏感属性、URL 用户信息和
|
||||
查询 Secret 在写出前统一替换为 `[REDACTED]`。
|
||||
|
||||
Gateway 还按每个 Worker 暴露 HTTP/CONNECT 请求总数、在途请求数和活跃 CONNECT
|
||||
隧道数;协议标签固定为 HTTP、CONNECT。请求数在 Handler 接受请求时增加,在途数在
|
||||
全部拒绝、转发或隧道关闭后归零;活跃隧道只覆盖成功建立并开始 relay 的连接。这些
|
||||
指标是连接池、入口并发、文件描述符和长连接排空的本地观测,不携带 Proxy、路由、
|
||||
目标、Client 或凭据。
|
||||
变更需要重启 Controller。
|
||||
|
||||
## 10. 启动前校验清单
|
||||
|
||||
|
||||
@ -80,10 +80,10 @@ Provider、Pool、Routing、Distribution 在首版需要共享事务和一致性
|
||||
|
||||
### 3.4 proxy-loadgen
|
||||
|
||||
- 当前实现 HTTP、CONNECT 长连接和 Extract 场景:固定请求数或固定时长,受限并发与可选
|
||||
目标 QPS;可配置方法、重复请求头和请求体。HTTPS 目标经 HTTP Gateway 时由 Transport 走 CONNECT。
|
||||
- 输出场景、延迟分位上界、错误分类、吞吐和 Go 内存/GC 快照。故障注入、进程 CPU/RSS/句柄
|
||||
与网络采样以及代表性集群报告仍需补齐。
|
||||
- 当前实现 HTTP 请求场景:固定请求数或固定时长,受限并发与可选目标 QPS;可配置方法、
|
||||
重复请求头和请求体。HTTPS 目标经 HTTP Gateway 时由 Transport 走 CONNECT。
|
||||
- 输出场景、延迟分位上界、错误分类、吞吐和 Go 内存/GC 快照。CONNECT 长连接、
|
||||
Extract 并发、进程 CPU/RSS/句柄与网络采样仍需补齐。
|
||||
|
||||
## 4. 模块边界
|
||||
|
||||
@ -319,8 +319,7 @@ Upstream 的 Empty 事实全局共享;每条 Routing 独立 CAS 当前索引
|
||||
协程只能有一个成功从 A 切到 B,其他协程读取新版本,不会再切到 C。
|
||||
Sequential 至少配置两个 Upstream;列表耗尽后的默认行为是 `stop`,`loop` 和
|
||||
`stayLast` 必须显式配置。disabled Upstream 不参与新分配,其运行时跳过与权威
|
||||
游标由 Controller 的权威管理状态持久化。达到末端 `stop` 时,Controller 以观察到的
|
||||
当前 Upstream 作为 CAS 栅栏禁用 Routing,并让完整 Snapshot 显式关闭该规则。
|
||||
游标持久化仍由后续 Routing Runtime 完成。
|
||||
|
||||
## 10. Exclusive Extraction
|
||||
|
||||
|
||||
@ -42,14 +42,10 @@ Gateway Worker 只读取本地不可变快照并维护本地容量计数。供
|
||||
|
||||
1. 接入层完成认证、来源识别、限流和目标地址检查。
|
||||
2. Routing 按配置顺序首条命中。
|
||||
3. 若启用并携带粘性会话头,先在本 Worker 的有界缓存中按认证 Client、Routing
|
||||
和会话标识查找仍符合当前快照的 Proxy。
|
||||
4. Dispatcher 从本地快照筛选 Upstream、协议、标签、TTL 和健康条件;失效会话
|
||||
自动清除并按常规策略重新选择。
|
||||
5. 原子预留 Proxy 容量,建立到上游代理的连接。
|
||||
6. 建连成功后转为 Active,并在需要时写入不超过 Proxy 可用期的会话绑定;
|
||||
传输结束后释放,失败则取消预留并清除该绑定。
|
||||
7. GET/HEAD 仅在响应提交前按策略重试;CONNECT 建立后不重放。
|
||||
3. Dispatcher 从本地快照筛选 Upstream、协议、标签、TTL 和健康条件。
|
||||
4. 原子预留 Proxy 容量,建立到上游代理的连接。
|
||||
5. 建连成功后转为 Active,传输结束后释放;失败则取消预留。
|
||||
6. GET/HEAD 仅在响应提交前按策略重试;CONNECT 建立后不重放。
|
||||
|
||||
### 4.2 独占提取
|
||||
|
||||
|
||||
@ -53,10 +53,10 @@ deadline 内执行 HTTP/HTTPS/SOCKS5 BASIC、EGRESS 和 TARGET 探测并微批
|
||||
|
||||
### proxy-loadgen
|
||||
|
||||
负载工具当前生成有界 HTTP、CONNECT 长连接和 Extract 请求,支持经 Gateway 请求 HTTPS 目标、
|
||||
固定请求数或时长、目标 QPS、可重复请求头和请求体、连接复用和 JSON 指标输出。延迟统计使用
|
||||
固定大小直方图,不会因长时间高 QPS 运行积压样本。故障注入和代表性集群报告仍待补齐;它是
|
||||
100k QPS 结论的证据工具,不是业务进程。
|
||||
负载工具当前生成有界 HTTP 请求,支持经 Gateway 请求 HTTPS 目标、固定请求数或时长、
|
||||
目标 QPS、可重复请求头和请求体、连接复用和 JSON 指标输出。延迟统计使用固定大小直方图,不会因长时间高 QPS
|
||||
运行积压样本。CONNECT 长连接、Extract 和故障注入场景仍待补齐;它是 100k QPS 结论的
|
||||
证据工具,不是业务进程。
|
||||
|
||||
## 3. 依赖方向
|
||||
|
||||
|
||||
@ -54,12 +54,12 @@ test/{fixtures,integration,e2e,load}/
|
||||
**Files:** `go.mod`, `.golangci.yml`, `README.md`, `scripts/verify.ps1`,
|
||||
`.github/workflows/ci.yml`
|
||||
|
||||
- [x] Create Go module `proxy-pool` with Go 1.26.
|
||||
- [x] Pin YAML v4, pgx/v5, go-redis/v9, gRPC, protobuf, Prometheus, and x/sync.
|
||||
- [ ] Create module `github.com/proxy-pool/proxy-pool` with Go 1.26.
|
||||
- [ ] Pin YAML v4, pgx/v5, go-redis/v9, gRPC, protobuf, Prometheus, and x/sync.
|
||||
- [x] Add `scripts/verify.ps1` that runs format check, `go vet`, unit tests, race tests,
|
||||
and builds all commands, each test command bounded to 60 seconds.
|
||||
- [x] Add CI for Windows and Linux with unit/race/build jobs.
|
||||
- [x] Verify `go mod tidy`, `go test ./...`, and `go build ./cmd/...` succeed.
|
||||
- [ ] Verify `go mod tidy`, `go test ./...`, and `go build ./cmd/...` succeed.
|
||||
|
||||
## Task 2: Strict Configuration
|
||||
|
||||
@ -80,18 +80,14 @@ test/{fixtures,integration,e2e,load}/
|
||||
|
||||
- [x] Implement Proxy fields, UTC TTL precedence, canonical host/port, and unique key.
|
||||
- [x] Implement state transitions and reject illegal transitions.
|
||||
- [x] Implement packed per-Proxy runtime counters with CAS Reserve, Commit, Cancel, Release.
|
||||
- [ ] Implement sharded runtime counters with CAS Reserve, Commit, Cancel, Release.
|
||||
- [x] Prove with 1,000 concurrent goroutines that effective capacity is never exceeded.
|
||||
- [x] Add race coverage and duplicate-release invariant metrics hook.
|
||||
- [ ] Add race coverage and duplicate-release invariant metrics hook.
|
||||
|
||||
当前进度(2026-08-02):固定 Max 下的每 Proxy 打包 CAS、Cancel/Commit/Release
|
||||
当前进度(2026-07-29):固定 Max 下的每 Proxy 打包 CAS、Cancel/Commit/Release
|
||||
生命周期、重复终结、错误顺序和同一 Reservation 并发终结已通过领域测试;已退出
|
||||
Proxy 会停止新预留、保留非零 Drain 计数,并在归零后由后续 Snapshot Apply 回收。
|
||||
Snapshot 降低 `maxConcurrency` 时会保留既有 Active 工作、禁止新预留,直到运行计数
|
||||
低于新上限;运行态复用、降容和恢复预留均有回归测试。领域的
|
||||
`CapacityInvariantObserver` 只发出固定生命周期违规枚举,Gateway 指标
|
||||
`proxy_pool_gateway_capacity_invariant_violations_total{operation}` 不含 Proxy、请求或
|
||||
Worker 维度;该测试纳入 Linux CI 的 race 范围,本机因 `CGO_ENABLED=0` 未执行 race。
|
||||
动态降容契约、低基数不变量指标和 Linux race 证据仍待完成。
|
||||
|
||||
## Task 4: Routing and Sequential Switching
|
||||
|
||||
@ -101,18 +97,12 @@ Worker 维度;该测试纳入 Linux CI 的 race 范围,本机因 `CGO_ENABLE
|
||||
- [x] Implement random, round-robin, weighted, least-connections, and sequential.
|
||||
- [x] Model upstream empty counters separately from per-routing current indexes.
|
||||
- [x] Implement versioned CAS switch so simultaneous threshold observers advance once.
|
||||
- [x] Cover four-empty-then-success, five-empty, A-to-B-only, disabled references,
|
||||
- [ ] Cover four-empty-then-success, five-empty, A-to-B-only, disabled references,
|
||||
end behavior, and explicit onUnavailable.
|
||||
|
||||
当前进度(2026-08-02):领域构造器与严格配置已统一 Sequential 至少两个
|
||||
Upstream、`endBehavior` 默认 `stop`,并覆盖列表末端停止。Provider Stats 现为每次
|
||||
连续空结果分配单调代次;Controller 的公共 Sequential 协调器只接收有界通知,在独立
|
||||
循环中读取权威配置、管理快照和 Stats,并以既有 `ExpectedCurrent` CAS 自动切换。
|
||||
它会跳过禁用 Upstream,支持 `loop`/`stayLast`,同一空结果代次不会在循环后重复切换,
|
||||
成功后立即广播完整 Snapshot。末端 `stop` 使用独立 `DisableRouting` 公用命令,
|
||||
以 `ExpectedCurrent` CAS 原子停用 Routing,并在同一权威事务写入审计与 Outbox;
|
||||
内存与 PostgreSQL Adapter 都运行相同并发、幂等和失败回滚契约。Gateway 已执行
|
||||
reject/wait/direct,并会在快照刷新后看到停用状态。
|
||||
当前进度(2026-07-29):领域构造器与严格配置已统一 Sequential 至少两个
|
||||
Upstream、`endBehavior` 默认 `stop`,并覆盖列表末端停止;disabled candidate、
|
||||
跨实例恢复和 `onUnavailable` 运行链仍待完成。
|
||||
|
||||
## Task 5: Provider Fetch Classification and Scheduling
|
||||
|
||||
@ -130,7 +120,7 @@ reject/wait/direct,并会在快照刷新后看到停用状态。
|
||||
|
||||
**Files:** `internal/controller/pool/*.go`, `internal/domain/upstream/pool.go`, tests
|
||||
|
||||
- [x] Compute Available Slots from eligible Proxy capacity, Active, Reserved, TTL,
|
||||
- [ ] Compute Available Slots from eligible Proxy capacity, Active, Reserved, TTL,
|
||||
health, ownership, pending expected fetch, and gateway reserve.
|
||||
- [x] Implement pool.maxSize and fetch.maxTotal as distinct counters.
|
||||
- [x] Allocate each Proxy to one Worker with epoch/version/expiry ownership.
|
||||
@ -190,10 +180,8 @@ reject/wait/direct,并会在快照刷新后看到停用状态。
|
||||
- [x] Implement the Worker heartbeat receiving path and session lifecycle.
|
||||
- [x] Keep Provider output in Redis TTL activity state and node memory only; keep the
|
||||
Gateway request path on immutable local snapshots with no Redis/PostgreSQL calls.
|
||||
- [x] Expose Distribution extraction/status and Admin status/audit/enable/disable/switch/reload
|
||||
HTTP handlers and contracts with credential-level endpoint permissions and
|
||||
Distribution credential-level extraction boundaries; enforce matched
|
||||
Gateway credential routing boundaries before local dispatch.
|
||||
- [x] Expose Distribution extraction/status and Admin status/enable/disable/switch/reload
|
||||
HTTP handlers and contracts.
|
||||
- [x] Add Compose-backed Redis 8.2 integration and shared Adapter contract tests.
|
||||
- [x] Add PostgreSQL management Adapter and Compose-backed integration tests.
|
||||
|
||||
@ -217,8 +205,8 @@ Distribution/Admin 服务构造、错误合并和资源关闭。生产 Provider
|
||||
停机;Admin disable 会取消 Runtime,reload 在提交前预检并在发布后替换运行实例。
|
||||
组合 fixture 已验证隔离 Redis namespace 下的选主、Provider HTTP 调用、模板解析
|
||||
和活动池写入。Controller Metrics 独立入口现已提供 `/livez`、`/readyz` 与基础
|
||||
Prometheus 运行时指标,三监听器隔离已通过测试;Checker、Gateway、Drain、
|
||||
Provider、Extraction 与容量业务指标已接入;进程级密钥安全结构化日志也已落地。双存储 bootstrap 已通过 PostgreSQL 18 + Redis 8.2 组合 fixture,覆盖
|
||||
Prometheus 运行时指标,三监听器隔离已通过测试;业务指标仍待
|
||||
实现。双存储 bootstrap 已通过 PostgreSQL 18 + Redis 8.2 组合 fixture,覆盖
|
||||
迁移、启动配置提交、Readiness、Admin Status 和 Metrics 探针。
|
||||
|
||||
WorkerControlPlane 现已接入 Controller 生命周期:Register、ACK 和 Runtime
|
||||
@ -314,20 +302,12 @@ Ticket/排除快照/Runtime 零计数闭环。
|
||||
Controller 即可生效。Redis 任务存储现已扩展 BASIC/EGRESS/TARGET 的独立有界索引;路由目标
|
||||
Profile 在启用 Routing 与 Upstream 的组合上才进入调度。
|
||||
|
||||
补充进度(2026-08-02):配置停用已进入 ownership Drain 编排。Controller 以静态配置
|
||||
与 Admin 管理态的交集生成完整、版本化的 Upstream 策略视图;Redis 只从对应 `owned` 索引
|
||||
有界读取仍可用的已归属 Proxy。候选携带 Upstream revision,`ownership.lua` 在复用既有
|
||||
Ticket/排除 Snapshot/Runtime 零计数闭环前,原子复核策略仍为停用、Proxy source、Worker、
|
||||
assignment epoch 和未过期租约。重启用后的策略 revision 会使旧候选返回无操作。Provider
|
||||
Supervisor 也改为同时服从静态配置与管理态,消除两条启停消费链的不一致。
|
||||
|
||||
补充进度(2026-08-02):已新增 `proxy-loadgen` HTTP 与 CONNECT 长连接场景。固定请求数
|
||||
和固定时长两种模式均通过固定 worker 数与有界派发通道执行,可选 QPS 限速;报告使用固定大小
|
||||
延迟直方图,输出状态分类、CONNECT 建立数、Extract 校验数、吞吐和 Go 内存/GC 快照。CONNECT
|
||||
以原始 TCP 握手连接 HTTP Gateway,建连成功后按 `hold` 保持,且不透明读取隧道内容。`extract`
|
||||
场景会自动生成独立 Request/Idempotency 标识,校验返回数量与单响应 ID 唯一性,且不记录地址或
|
||||
凭据。报告可选按最大错误率和 p99 延迟门禁;违反阈值时仍完整写出 JSON 证据并以
|
||||
专用退出码失败。故障注入以及代表性集群报告仍未实现。
|
||||
凭据。故障注入以及代表性集群报告仍未实现。
|
||||
|
||||
## Task 12: Machine-readable Contracts
|
||||
|
||||
@ -338,27 +318,24 @@ Supervisor 也改为同时服从静态配置与管理态,消除两条启停消
|
||||
and idempotency behavior.
|
||||
- [x] Specify Worker register, snapshot, delta, ACK, report, heartbeat, ownership drain,
|
||||
and resync messages.
|
||||
- [x] Validate OpenAPI contracts and compile protobuf descriptors/generated-code drift in CI.
|
||||
- [ ] Validate OpenAPI and compile protobuf descriptors in CI.
|
||||
|
||||
当前进度(2026-07-29):Distribution/Admin OpenAPI 已由 Go 测试在双平台 CI
|
||||
校验本地 `$ref` 闭合、operationId 唯一、响应存在及 security scheme 引用;
|
||||
`scripts/verify-proto.ps1` 已可复现编译包含 imports/source info 的 descriptor,
|
||||
并使用 SHA-256 固定的 `protoc` 35.0 安装器在 CI 完整验证 descriptor 与生成代码漂移;
|
||||
OpenAPI 结构契约由 Go 测试在双平台 CI 执行;`scripts/verify-openapi.ps1` 固定
|
||||
`@redocly/cli@2.25.4`,按 OpenAPI 3.1 最小规则集验证两份文档并将 Tag 描述作为错误。
|
||||
并在本地存在 `protoc` 时进入完整验证;完整 OAS 工具验证与 CI 强制安装/执行
|
||||
`protoc` 仍待完成。
|
||||
|
||||
## Task 13: Deployment and Observability
|
||||
|
||||
**Files:** `deploy/**`, `internal/platform/**`, `docs/operations/**`
|
||||
|
||||
- [x] Add Compose for local Controller/Gateway/Checker/PostgreSQL/Redis/Prometheus/
|
||||
- [ ] Add Compose for local Controller/Gateway/Checker/PostgreSQL/Redis/Prometheus/
|
||||
Grafana/HAProxy.
|
||||
- [x] Add Kubernetes Deployments, Services, PDBs, HPA, NetworkPolicy, Secrets examples,
|
||||
- [ ] Add Kubernetes Deployments, Services, PDBs, HPA, NetworkPolicy, Secrets examples,
|
||||
probes, resource limits, topology spread, and graceful termination.
|
||||
- [ ] Add the production Kubernetes mTLS identity overlay, certificate rotation and unique
|
||||
elastic Worker identity wiring.
|
||||
- [x] Add low-cardinality Prometheus metrics and structured secret-safe logs.
|
||||
- [x] Document backup, recovery, rollout, rollback, capacity, kernel, file descriptor,
|
||||
- [ ] Add low-cardinality Prometheus metrics and structured secret-safe logs.
|
||||
- [ ] Document backup, recovery, rollout, rollback, capacity, kernel, file descriptor,
|
||||
NAT/conntrack, and incident runbooks.
|
||||
|
||||
当前进度(2026-08-02):已接入 Checker 任务流指标
|
||||
@ -367,22 +344,7 @@ OpenAPI 结构契约由 Go 测试在双平台 CI 执行;`scripts/verify-openap
|
||||
`BASIC`、`EGRESS`、`TARGET` 级别及 `accepted`、`rejected` 结果。Gateway 还暴露
|
||||
`proxy_pool_gateway_outcomes_total{stage,result}` 和
|
||||
`proxy_pool_gateway_outcome_queue_dropped_total`,其中阶段和结果均为固定枚举。Provider、
|
||||
拉取已暴露 `proxy_pool_controller_provider_fetch_results_total{class}`、
|
||||
`proxy_pool_controller_provider_valid_candidates_total` 与
|
||||
`proxy_pool_controller_provider_new_proxies_total`;`class` 仅允许 `valid`、`empty`、
|
||||
`duplicate_only`、`error`。Extraction 还暴露固定 `result` 的请求次数、请求数与响应
|
||||
交付数,幂等重放按响应交付统计。容量指标由既有 Provider 库存对账周期聚合,不进入
|
||||
Gateway 热路径。`platform/logging` 以 JSON `slog` 输出进程级致命错误,字段、URL
|
||||
用户信息、查询 Secret 和错误对象均经过脱敏,且不在请求热路径逐条写日志。
|
||||
CI 另有独立 Deployment job,在占位凭据下渲染 Compose,并使用 `kubectl kustomize`
|
||||
渲染 Kubernetes base 和单副本 development mTLS Overlay;该 job 同时展开测试 Compose、
|
||||
校验本地 Compose 配置。它不启动容器、不访问真实存储或密钥。
|
||||
独立 Image job 会构建 Linux 多阶段应用镜像,覆盖 Dockerfile、Linux 交叉编译与运行时层;
|
||||
容器服务编排和业务运行链路仍需在具备镜像网络的环境中验证。
|
||||
development Overlay 已挂载独立 Controller/Gateway/Checker TLS Secret、启用 Checker 和最小
|
||||
网络策略,并从固定开发证书派生 `gateway-a`、`checker-a` 身份且 HPA 锁为单副本,不能代替生产证书轮换。
|
||||
控制面 TLS 叶证书和信任根已在每次新握手时从文件加载,短暂文件不一致时保留最后一次有效材料;
|
||||
生产 Overlay 仍必须由工作负载身份系统提供每副本唯一 SPIFFE URI 和弹性身份注入。
|
||||
提取、容量指标与密钥安全的结构化日志仍待补齐。
|
||||
|
||||
## Task 14: Documentation, Examples, and Diagrams
|
||||
|
||||
@ -392,27 +354,14 @@ development Overlay 已挂载独立 Controller/Gateway/Checker TLS Secret、启
|
||||
reference, API guide, deployment guide, security model, testing guide, and roadmap.
|
||||
- [x] Provide at least 20 validated configuration examples.
|
||||
- [x] Provide at least 30 Mermaid architecture, flow, sequence, state, and failure diagrams.
|
||||
- [x] Generate `proxy-pool-docs-v1.0.zip` from versioned documentation assets.
|
||||
|
||||
`scripts/package-docs.ps1` 会将 README、`docs/`、图表、OpenAPI、Proto 契约与部署手册复制至临时目录,
|
||||
生成包含 Git revision、文件大小和 SHA-256 的 `manifest.json`,再以临时 ZIP 原子替换目标。
|
||||
默认输出为被 Git 忽略的 `dist/proxy-pool-docs-v1.0.zip`;Go 回归测试实际执行脚本并校验归档内容。
|
||||
- [ ] Generate `proxy-pool-docs-v1.0.zip` from versioned documentation assets.
|
||||
|
||||
## Task 15: Completion Audit
|
||||
|
||||
- [x] Map every requirement ID to code, test, contract, document, or verified runtime evidence.
|
||||
- [ ] Map every requirement ID to code, test, contract, document, or verified runtime evidence.
|
||||
- [ ] Run `gofmt`, `go vet`, unit tests, race tests, builds, contract validation, and
|
||||
documentation link/example validation.
|
||||
- [x] Run bounded local performance benchmarks; label 100k QPS as unverified until a
|
||||
- [ ] Run bounded local performance benchmarks; label 100k QPS as unverified until a
|
||||
representative cluster load run exists.
|
||||
|
||||
`scripts/benchmark-gateway.ps1` 固定使用 `-count=1 -benchtime=1x` 分别执行 100k 索引调度、
|
||||
Round Robin 和 Snapshot Apply 基准,并将 Git revision 与原始输出保存到 `dist/`。该基线不包含
|
||||
网络、TLS、上游 RTT、真实连接或多 Worker 协调,不能替代代表性集群报告。
|
||||
- [x] Confirm no TODO/TBD/placeholders, secrets, unbounded queues, high-cardinality metric
|
||||
- [ ] Confirm no TODO/TBD/placeholders, secrets, unbounded queues, high-cardinality metric
|
||||
labels, extraction Lease APIs, or conflicting maxSize semantics remain.
|
||||
|
||||
`docs` 回归测试扫描生产源与部署清单中的未完成标记和私钥材料;OpenAPI 测试拒绝
|
||||
Extract Lease/Release/Renew 路径,Gateway Outcome Queue 构造器和配置校验拒绝无界容量,
|
||||
指标/部署契约拒绝高基数标签,`pool.maxSize` 与累计 `fetch.maxTotal` 的边界由配置和
|
||||
活动池契约测试共同锁定。示例、CI fixture 与环境变量引用不是未实现占位或生产 Secret。
|
||||
|
||||
@ -77,61 +77,6 @@ func TestDocumentedGoCommandsReferenceExistingTargets(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunbookCoversHostAndNetworkCapacityBaseline(t *testing.T) {
|
||||
content := readDocument(t, filepath.Join(repositoryRoot(t), "docs", "operations", "runbook.md"))
|
||||
for _, required := range []string{
|
||||
"## 8. 备份与恢复",
|
||||
"## 10. 主机与网络容量基线",
|
||||
"ulimit -n",
|
||||
"conntrack",
|
||||
"NAT",
|
||||
"回滚",
|
||||
} {
|
||||
if !strings.Contains(content, required) {
|
||||
t.Errorf("runbook does not document %q", required)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRepositoryAuditRejectsUnfinishedSourceAndPrivateKeyMaterial(t *testing.T) {
|
||||
repositoryRoot := repositoryRoot(t)
|
||||
unfinished := regexp.MustCompile(`(?i)\b(?:TODO|TBD|FIXME)\b`)
|
||||
privateKey := regexp.MustCompile(`-----BEGIN (?:[A-Z ]+ )?PRIVATE KEY-----`)
|
||||
for _, directory := range []string{".github", "api", "cmd", "deploy", "internal", "scripts"} {
|
||||
root := filepath.Join(repositoryRoot, directory)
|
||||
err := filepath.WalkDir(root, func(path string, entry os.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if entry.IsDir() {
|
||||
if entry.Name() == ".control-plane-tls" || strings.HasPrefix(entry.Name(), ".tmp") || entry.Name() == "dist" {
|
||||
return filepath.SkipDir
|
||||
}
|
||||
return nil
|
||||
}
|
||||
switch strings.ToLower(filepath.Ext(path)) {
|
||||
case ".go", ".yaml", ".yml", ".ps1":
|
||||
default:
|
||||
return nil
|
||||
}
|
||||
content, readErr := os.ReadFile(path)
|
||||
if readErr != nil {
|
||||
return readErr
|
||||
}
|
||||
if unfinished.Match(content) {
|
||||
t.Errorf("%s contains an unfinished implementation marker", relativeTestName(repositoryRoot, path))
|
||||
}
|
||||
if privateKey.Match(content) {
|
||||
t.Errorf("%s contains private key material", relativeTestName(repositoryRoot, path))
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("audit %s: %v", directory, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func markdownDocuments(t *testing.T, repositoryRoot string) []string {
|
||||
t.Helper()
|
||||
documents := []string{filepath.Join(repositoryRoot, "README.md")}
|
||||
|
||||
@ -2,29 +2,28 @@
|
||||
|
||||
## 架构与一致性
|
||||
|
||||
- [x] Gateway 热路径依赖审计确认无 PostgreSQL、Redis、Provider 或模板执行。
|
||||
- [x] 每个 Proxy 同一时刻最多归属一个 Worker,ownership epoch 单调。
|
||||
- [x] Reserved -> Active 使用单个原子转换,无超卖与负计数。
|
||||
- [x] Sequential 并发 Empty 只切换一次,旧 Upstream Proxy 自然耗尽;末端 `stop`
|
||||
使用权威 CAS 停用 Routing 并刷新 Gateway 快照。
|
||||
- [ ] Gateway 热路径依赖审计确认无 PostgreSQL、Redis、Provider 或模板执行。
|
||||
- [ ] 每个 Proxy 同一时刻最多归属一个 Worker,ownership epoch 单调。
|
||||
- [ ] Reserved -> Active 使用单个原子转换,无超卖与负计数。
|
||||
- [ ] Sequential 并发 Empty 只切换一次,旧 Upstream Proxy 自然耗尽。
|
||||
- [x] `pool.maxSize` 与 `fetch.maxTotal` 分别按当前库存和累计获取计数。
|
||||
- [x] Extract 只有 `AVAILABLE -> EXTRACTED`,OpenAPI 不存在 release/renew。
|
||||
- [x] Extract 状态更新和短期幂等结果位于同一个 Redis 原子操作。
|
||||
- [x] PostgreSQL 中不存在 Proxy 明细或逐次提取记录。
|
||||
- [x] `partial` 和 `allOrNothing` 均通过 Redis 原子并发测试。
|
||||
- [x] `reserveForGateway` 在所有提取路径上统一执行。
|
||||
- [ ] Extract 只有 `AVAILABLE -> EXTRACTED`,OpenAPI 不存在 release/renew。
|
||||
- [ ] Extract 状态更新和短期幂等结果位于同一个 Redis 原子操作。
|
||||
- [ ] PostgreSQL 中不存在 Proxy 明细或逐次提取记录。
|
||||
- [ ] `partial` 和 `allOrNothing` 均通过 Redis 原子并发测试。
|
||||
- [ ] `reserveForGateway` 在所有提取路径上统一执行。
|
||||
|
||||
## 安全
|
||||
|
||||
- [x] 非回环监听均配置 Auth 或 allowCIDRs,严格模式已开启。
|
||||
- [x] Gateway、Distribution、Admin 凭据和权限相互独立。
|
||||
- [ ] 非回环监听均配置 Auth 或 allowCIDRs,严格模式已开启。
|
||||
- [ ] Gateway、Distribution、Admin 凭据和权限相互独立。
|
||||
- [ ] trusted proxy 只包含受控 LoadBalancer/Ingress 网段。
|
||||
- [x] 解析前后均拦截私网、回环、链路本地、元数据地址与 DNS Rebinding。
|
||||
- [ ] 解析前后均拦截私网、回环、链路本地、元数据地址与 DNS Rebinding。
|
||||
- [ ] Secret 由外部密钥系统注入,镜像、ConfigMap、日志没有明文。
|
||||
- [ ] 所有 Controller 使用同一枚至少 32 字节的独立配置 HMAC 密钥,并完成轮换演练。
|
||||
- [ ] Pod 以非 root、只读根文件系统、无 Linux capabilities 运行。
|
||||
- [ ] NetworkPolicy 默认拒绝,外部数据库/Redis/Provider 网段已收紧。
|
||||
- [x] Provider 模板有响应大小、执行时间、函数与外部访问限制。
|
||||
- [ ] Provider 模板有响应大小、执行时间、函数与外部访问限制。
|
||||
|
||||
## 可用性
|
||||
|
||||
@ -32,9 +31,9 @@
|
||||
- [ ] Redis 跨可用区并受监控,完成丢失后由 Provider 重建活动池的演练。
|
||||
- [ ] Gateway、Controller、Checker 均跨主机/可用区分散。
|
||||
- [ ] PDB、优雅终止与最大连接时长的组合经过驱逐测试。
|
||||
- [x] Controller 断线时 Gateway 在 `maxStaleAge` 内继续,超限拒绝新请求。
|
||||
- [x] Worker 崩溃后 ownership 只在租约过期后再分配。
|
||||
- [x] Snapshot 版本缺口触发全量同步,旧 Delta 被拒绝。
|
||||
- [ ] Controller 断线时 Gateway 在 `maxStaleAge` 内继续,超限拒绝新请求。
|
||||
- [ ] Worker 崩溃后 ownership 只在租约过期后再分配。
|
||||
- [ ] Snapshot 版本缺口触发全量同步,旧 Delta 被拒绝。
|
||||
|
||||
## 性能
|
||||
|
||||
@ -42,15 +41,15 @@
|
||||
- [ ] 完成 10k 稳态、100k 峰值、CONNECT 活跃连接和建连速率独立测试。
|
||||
- [ ] 在最大可用区失效时仍满足容量和延迟 SLO。
|
||||
- [ ] 目标利用率不高于 60%,HPA 缩容稳定窗口不低于 10 分钟。
|
||||
- [x] 队列、buffer、日志和结果上报全部有界。
|
||||
- [ ] 队列、buffer、日志和结果上报全部有界。
|
||||
- [ ] p99 Dispatch 预算、端到端延迟、错误率、CPU、RSS、FD 和网络有原始证据。
|
||||
|
||||
## 观测与值班
|
||||
|
||||
- [x] 仪表盘覆盖 Gateway、库存、Provider、Extract、Snapshot、Checker 和 Drain。
|
||||
- [ ] 仪表盘覆盖 Gateway、Routing、Upstream、Provider、Extract、Snapshot、Checker。
|
||||
- [ ] 告警有负责人、严重级别、Runbook 链接和演练记录。
|
||||
- [x] 指标无 Proxy IP、Client ID、Session、完整 URL 或 request ID 高基数标签。
|
||||
- [x] 日志脱敏已用真实 Secret fixture 验证。
|
||||
- [ ] 指标无 Proxy IP、Client ID、Session、完整 URL 或 request ID 高基数标签。
|
||||
- [ ] 日志脱敏已用真实 Secret fixture 验证。
|
||||
- [ ] 值班人员完成 PostgreSQL、Redis、Snapshot、容量与 Extract 故障演练。
|
||||
|
||||
## 发布门禁
|
||||
|
||||
@ -24,10 +24,11 @@
|
||||
`cmd/proxy-controller` 已完成配置单次加载、PostgreSQL 迁移、Redis 活动池、
|
||||
Distribution/Admin/Metrics 独立监听和有界停机装配。Provider 自动补池、分布式
|
||||
配额、动态重载和 Admin 低基数统计已装配;Controller 已装配 Redis BASIC/EGRESS/TARGET 任务 broker,
|
||||
`proxy-checker` 可执行 HTTP/HTTPS/SOCKS5 BASIC/EGRESS/TARGET 探测。`proxy-loadgen` 已提供有界 HTTP、
|
||||
CONNECT 长连接和 Extract 场景,并可按最大错误率和 p99 延迟输出机器可判定的验收结果;
|
||||
它不替代代表性集群压测。Compose 已声明本地 Controller/Gateway/Checker mTLS 拓扑,
|
||||
但本机尚未完成容器端到端启动;Kubernetes 仍需要环境专属的工作负载身份 Overlay。
|
||||
`proxy-checker` 可执行 HTTP/HTTPS/SOCKS5 BASIC/EGRESS/TARGET 探测。`proxy-loadgen` 已提供有界 HTTP
|
||||
请求场景;CONNECT 长连接/Extract 压测与完整 mTLS 环境 Overlay 仍属于
|
||||
`implementation-plan.md` 后续任务。
|
||||
因此 Compose/Kubernetes 资产当前仍用于评审网络、资源、探针和依赖关系,不能
|
||||
视为完整可运行拓扑。
|
||||
|
||||
### 2.1 前置条件
|
||||
|
||||
@ -49,13 +50,6 @@ $env:PROVIDER_A_TOKEN = "PROVIDER_A_TOKEN"
|
||||
$env:PROVIDER_B_TOKEN = "PROVIDER_B_TOKEN"
|
||||
```
|
||||
|
||||
首次启动 Compose 前,生成不入库的本地控制面证书。生成器要求目标目录为空,避免覆写
|
||||
已有密钥:
|
||||
|
||||
```powershell
|
||||
.\scripts\generate-local-controlplane-certs.ps1
|
||||
```
|
||||
|
||||
### 2.3 静态检查
|
||||
|
||||
```powershell
|
||||
@ -63,10 +57,6 @@ docker compose -f deploy/docker-compose.yml config
|
||||
kubectl kustomize deploy/kubernetes/base > rendered.yaml
|
||||
```
|
||||
|
||||
Compose 启动后,Gateway 的 `/readyz` 需要先接收有效 Snapshot;Checker 的 `/readyz`
|
||||
需要成功建立一次任务领取流,并会在后续领取失败时回到未就绪。两者的 `/livez` 只表示
|
||||
进程仍在运行。
|
||||
|
||||
本地 Compose 的 Redis 只作为可重建短效状态 fixture,固定使用
|
||||
`--appendonly no --save ""`,且不挂载 `/data` 或命名卷。真实 Redis 8.2 契约可
|
||||
通过 `.\scripts\test-redis.ps1` 执行;脚本使用唯一命名空间并在结束时定向清理,
|
||||
@ -113,10 +103,6 @@ Fetch 应表现为 Error 与退避,不应增加 Empty 计数,也不影响已
|
||||
4. 根据集群 CNI 能力收紧 NetworkPolicy 的外部网段。
|
||||
5. 在预发布环境完成数据库向前兼容迁移,再发布 Controller。
|
||||
|
||||
环境 Overlay 应保持 ConfigMap 内容哈希,令配置变化进入 Deployment Pod 模板并触发滚动更新。
|
||||
由外部系统以固定名称注入的 Secret 不会天然改变 Pod 模板;完成 Secret 轮换后,必须按受影响
|
||||
工作负载执行受控滚动,并先验证新旧信任根的重叠窗口。
|
||||
|
||||
### 3.2 服务端应用顺序
|
||||
|
||||
```bash
|
||||
@ -152,12 +138,6 @@ Proxy ID、IP、Checker ID、目标 URL、Client ID 或凭据加入指标标签
|
||||
|
||||
Gateway 指标同样使用固定标签集:
|
||||
|
||||
每个 Gateway Worker 还暴露 requests_total、requests_in_flight 与 active_tunnels
|
||||
三类本地连接指标。protocol 标签只允许 HTTP、CONNECT;请求指标覆盖从 Handler
|
||||
接受请求到全部转发、拒绝或隧道关闭的生命周期。活跃隧道只在成功写出 CONNECT 200
|
||||
并开始 relay 后增加,关闭后立即减少。这些指标用于核对入口准入、连接池调优、
|
||||
文件描述符预算和长连接排空,不得按 Proxy、路由、目标、Client 或凭据拆分。
|
||||
|
||||
- `proxy_pool_gateway_outcomes_total{stage,result}`:每次代理尝试在最远完成阶段的成功/失败数量。
|
||||
- `proxy_pool_gateway_outcome_queue_dropped_total`:Outcome 本地有界队列已满后丢弃的观测数量。
|
||||
|
||||
@ -290,9 +270,7 @@ Outbox 发布器必须以稳定 consumer ID 有界领取;发布成功后原子
|
||||
2. 429 尊重 `Retry-After`,其余 Error 使用指数退避和 jitter。
|
||||
3. Error 不增加 Empty;合法候选为零才增加 Empty。
|
||||
4. Duplicate-only 重置 Empty 并记录独立指标。
|
||||
5. 达到 Empty 阈值后每条受影响 Routing 只原子切换一次;最后一个可用 Upstream
|
||||
仍持续 Empty 且配置为 `stop` 时,确认审计中存在 `disable_routing` 和
|
||||
`routing.disabled` Outbox 事件,并检查 Gateway 已收到关闭该 Routing 的完整快照。
|
||||
5. 达到 Empty 阈值后每条受影响 Routing 只原子切换一次。
|
||||
|
||||
### 7.5 Gateway 容量耗尽
|
||||
|
||||
@ -353,46 +331,3 @@ ownership epoch、活动池 TTL、短期幂等窗口、outbox 和配置 Revision
|
||||
5. 所有副本应用后撤销旧 Secret。
|
||||
|
||||
Proxy 凭据轮换必须增加 `credentialVersion`,确保唯一键不会把新旧凭据错误合并。
|
||||
|
||||
控制面 mTLS 的叶证书、私钥和信任根会在每次新的 TLS 握手时重新读取;Kubernetes 投影
|
||||
Secret 或 CSI 写入短暂不完整时保留最后一次有效材料。既有 gRPC 长连接不会被强制替换,
|
||||
因此根 CA 轮换必须先发布包含新旧根的重叠信任包、确认新连接成功,再按 PDB 约束滚动
|
||||
重启或排空旧 Worker,最后才撤销旧根并使旧流重新认证。
|
||||
|
||||
## 10. 主机与网络容量基线
|
||||
|
||||
在预发布压测和每次生产扩容前,记录 Gateway 节点、容器运行时和 Pod 内的下列只读基线;
|
||||
不要在故障处理中临时提高内核限制,任何变更都必须经过压测和变更评审。
|
||||
|
||||
```bash
|
||||
ulimit -n
|
||||
cat /proc/self/limits | grep 'open files'
|
||||
sysctl fs.file-max net.core.somaxconn net.ipv4.ip_local_port_range
|
||||
test -r /proc/sys/net/netfilter/nf_conntrack_max && cat /proc/sys/net/netfilter/nf_conntrack_max
|
||||
ss -s
|
||||
```
|
||||
|
||||
同时从节点或受监控的 NAT 网关采集 conntrack 使用率、丢包、重传、SYN backlog、TIME_WAIT、
|
||||
端口分配失败与 SNAT 端口耗尽事件。`conntrack -L` 会遍历整个表,不得在高峰时将它作为
|
||||
常规排障命令;优先使用节点监控或 `conntrack -S` 的聚合计数。
|
||||
|
||||
Gateway 的文件描述符预算必须按实际连接模型计算,而不是仅按 QPS:每条活跃 CONNECT
|
||||
隧道通常占用客户端和上游两个 socket;HTTP 上游并发、idle 连接池、监听 socket、日志和
|
||||
运行时也会占用描述符。部署前确认 Pod 内 `ulimit -n`、容器运行时 `LimitNOFILE` 与节点
|
||||
`fs.file-max` 均高于下式结果并保留至少 20% 余量:
|
||||
|
||||
```text
|
||||
fd_budget = 2 * max_active_connect_tunnels
|
||||
+ max_concurrent_http_upstreams
|
||||
+ max_idle_transport_connections
|
||||
+ process_reserve
|
||||
```
|
||||
|
||||
如果节点通过 NAT 访问上游,源端口和 conntrack 表同样构成硬上限。压测报告必须记录每个
|
||||
节点的 egress IP 数、可用临时端口范围、NAT/SNAT 设备限制与单目标连接分布;单一 egress IP
|
||||
不足时应在发布前增加 egress IP 或拆分节点池,不能依靠无限重试掩盖端口耗尽。
|
||||
|
||||
建议将 FD、conntrack、NAT 端口利用率、SYN overflow、TCP retransmit、TIME_WAIT、CPU、RSS
|
||||
和 Gateway p99 作为同一份容量证据采集。任何一项超过预设预警线时停止扩容或发布,先降低
|
||||
新连接速率并保留现有 CONNECT 隧道排空;回滚到兼容镜像或配置 revision 后,再根据原始
|
||||
指标分析根因。
|
||||
|
||||
@ -1,114 +0,0 @@
|
||||
package docs_test
|
||||
|
||||
import (
|
||||
"archive/zip"
|
||||
"encoding/json"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
type docsPackageManifest struct {
|
||||
Version string `json:"version"`
|
||||
Revision string `json:"revision"`
|
||||
Files []struct {
|
||||
Path string `json:"path"`
|
||||
SHA256 string `json:"sha256"`
|
||||
} `json:"files"`
|
||||
}
|
||||
|
||||
func TestGatewayBenchmarkScriptIsBoundedAndWritesEvidence(t *testing.T) {
|
||||
pwsh, err := exec.LookPath("pwsh")
|
||||
if err != nil {
|
||||
t.Skip("pwsh is required to exercise the gateway benchmark script")
|
||||
}
|
||||
repositoryRoot := repositoryRoot(t)
|
||||
outputPath := filepath.Join(t.TempDir(), "gateway-benchmarks.txt")
|
||||
command := exec.Command(pwsh, "-NoProfile", "-NonInteractive", "-File", filepath.Join(repositoryRoot, "scripts", "benchmark-gateway.ps1"), "-OutputPath", outputPath)
|
||||
command.Dir = repositoryRoot
|
||||
if output, err := command.CombinedOutput(); err != nil {
|
||||
t.Fatalf("run gateway benchmarks: %v\n%s", err, output)
|
||||
}
|
||||
payload, err := os.ReadFile(outputPath)
|
||||
if err != nil {
|
||||
t.Fatalf("read benchmark evidence: %v", err)
|
||||
}
|
||||
evidence := string(payload)
|
||||
for _, required := range []string{
|
||||
"BenchmarkAcquire100k(Indexed|RoutingRoundRobin)",
|
||||
"BenchmarkStoreApply100k",
|
||||
"-benchtime=1x",
|
||||
"-count=1",
|
||||
"revision:",
|
||||
} {
|
||||
if !strings.Contains(evidence, required) {
|
||||
t.Errorf("benchmark evidence does not contain %q", required)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestPackageDocsCreatesTraceableArchive(t *testing.T) {
|
||||
pwsh, err := exec.LookPath("pwsh")
|
||||
if err != nil {
|
||||
t.Skip("pwsh is required to exercise the documentation packaging script")
|
||||
}
|
||||
repositoryRoot := repositoryRoot(t)
|
||||
outputPath := filepath.Join(t.TempDir(), "proxy-pool-docs-v1.0.zip")
|
||||
if err := os.WriteFile(outputPath, []byte("incomplete"), 0o600); err != nil {
|
||||
t.Fatalf("seed old documentation archive: %v", err)
|
||||
}
|
||||
command := exec.Command(pwsh, "-NoProfile", "-NonInteractive", "-File", filepath.Join(repositoryRoot, "scripts", "package-docs.ps1"), "-OutputPath", outputPath)
|
||||
command.Dir = repositoryRoot
|
||||
if output, err := command.CombinedOutput(); err != nil {
|
||||
t.Fatalf("package docs: %v\n%s", err, output)
|
||||
}
|
||||
|
||||
archive, err := zip.OpenReader(outputPath)
|
||||
if err != nil {
|
||||
t.Fatalf("open documentation archive: %v", err)
|
||||
}
|
||||
defer archive.Close()
|
||||
entries := make(map[string]*zip.File, len(archive.File))
|
||||
for _, entry := range archive.File {
|
||||
entries[entry.Name] = entry
|
||||
}
|
||||
for _, required := range []string{
|
||||
"README.md",
|
||||
"deploy/README.md",
|
||||
"deploy/kubernetes/overlays/development-mtls/README.md",
|
||||
"docs/design/architecture.md",
|
||||
"docs/operations/runbook.md",
|
||||
"diagrams/README.md",
|
||||
"api/openapi/proxy-pool.yaml",
|
||||
"api/proto/controlplane/v1/controlplane.proto",
|
||||
"manifest.json",
|
||||
} {
|
||||
if _, exists := entries[required]; !exists {
|
||||
t.Errorf("documentation archive does not contain %s", required)
|
||||
}
|
||||
}
|
||||
|
||||
manifestEntry, exists := entries["manifest.json"]
|
||||
if !exists {
|
||||
return
|
||||
}
|
||||
reader, err := manifestEntry.Open()
|
||||
if err != nil {
|
||||
t.Fatalf("open manifest: %v", err)
|
||||
}
|
||||
defer reader.Close()
|
||||
var manifest docsPackageManifest
|
||||
if err := json.NewDecoder(reader).Decode(&manifest); err != nil {
|
||||
t.Fatalf("decode manifest: %v", err)
|
||||
}
|
||||
if manifest.Version != "v1.0" || manifest.Revision == "" || len(manifest.Files) < 20 {
|
||||
t.Fatalf("manifest = %+v", manifest)
|
||||
}
|
||||
for _, file := range manifest.Files {
|
||||
if file.Path == "" || file.SHA256 == "" {
|
||||
t.Fatalf("invalid manifest file = %+v", file)
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -11,18 +11,14 @@
|
||||
- 产品设计、总体架构、项目结构、四项 ADR。
|
||||
- 开发、配置、Distribution/Admin API、控制面协议、安全、测试、运维文档。
|
||||
- 20 个配置场景和 35 张 Mermaid 架构/流程/状态/故障图。
|
||||
- `scripts/package-docs.ps1` 可生成版本化 ZIP,包含 README、文档、图表、OpenAPI、Proto
|
||||
契约,以及带 Git revision 和 SHA-256 的文件清单。
|
||||
|
||||
### 机器契约
|
||||
|
||||
- Distribution OpenAPI:一次性独占提取、partial/allOrNothing、幂等键、
|
||||
Redis TTL 活动池原子语义、TTL/健康过滤结果与标准错误。
|
||||
- Admin OpenAPI:状态、按 ID 游标分页的权威审计查询、Upstream 启停、Routing
|
||||
切换和配置重载。
|
||||
- Admin OpenAPI:状态、Upstream 启停、Routing 切换和配置重载。
|
||||
- 两份 OpenAPI 已进入 Go/CI 结构门禁,覆盖本地引用闭合、operationId、响应和
|
||||
security scheme;固定版本的 Redocly CLI 还会按 OpenAPI 3.1 最小规则集验证两份文档,
|
||||
并把 Tag 描述提升为错误。
|
||||
security scheme;完整标准工具验证仍待补齐。
|
||||
- Protobuf:Worker 注册、全量/增量 Snapshot、`usable_until`、ACK、运行态/
|
||||
结果上报、Checker 任务与 Observation。
|
||||
|
||||
@ -33,9 +29,8 @@
|
||||
- `PROXY-* / CAP-*`:唯一键、TTL 优先级、状态迁移与 Active/Reserved 打包
|
||||
原子计数;1,000 goroutine 不超卖,以及 Cancel、重复终结、错误顺序、并发
|
||||
Commit/Cancel/Release 计数守恒测试。
|
||||
- `ROUTE-001 / ROUTE-004`:首条命中规则、Provider 空结果驱动的 Controller
|
||||
Sequential 切换及末端 stop 已接入权威管理状态;跨 Controller 通过 PostgreSQL
|
||||
`ExpectedCurrent` CAS 收敛,切换/停用均刷新本进程 Worker Snapshot。
|
||||
- `ROUTE-001 / ROUTE-004`:首条命中规则与进程内 Concurrent Sequential 单次
|
||||
切换;策略运行时接线、持久化恢复和跨实例 CAS 尚未完成。
|
||||
- `FETCH-005 / FETCH-006`:Valid、Empty、DuplicateOnly、Error 分类。
|
||||
- `DIST-001..003 / DIST-006..007`:内存活动池参考实现验证独占提取、满足模式、
|
||||
TTL、健康时效与 Gateway 保留量;1,000 并发不重复。
|
||||
@ -48,11 +43,8 @@
|
||||
- `PROVIDER-*`:Provider HTTP Client、严格响应上限、模板解析安全边界、凭据
|
||||
引用 Store 与 Reconciler Adapter 已实现。
|
||||
- `DIST/Admin HTTP`:严格 JSON、Request ID、Problem 响应及 Distribution/Admin
|
||||
Handler 已实现;Admin 审计查询以有界 `afterId` 游标读取 PostgreSQL 权威记录,
|
||||
不读取 Proxy 或 Redis 活动池。共享认证、CIDR、可信代理、Client ID、凭据级
|
||||
固定权限、Distribution 凭据级提取数量/Upstream/地区限制、Gateway 凭据级
|
||||
Routing 限制与本地准入保护链已接入,Controller Runtime 已将二者装配到独立
|
||||
监听器并支持联动优雅停机。
|
||||
Handler 已实现;共享认证、CIDR、可信代理、Client ID 与本地准入保护链已接入,
|
||||
Controller Runtime 已将二者装配到独立监听器并支持联动优雅停机。
|
||||
- `Redis Activity Adapter`:真实 Redis 8.2 已覆盖 Provider Upsert、健康更新、
|
||||
原子独占提取、短期幂等、Worker ownership、库存和有界过期清理,Memory/Redis
|
||||
运行同一公用契约。
|
||||
@ -97,51 +89,44 @@ docker compose config PASS
|
||||
kubectl kustomize PASS
|
||||
configuration examples 21/21 PASS
|
||||
Mermaid blocks 35
|
||||
bounded gateway microbenchmarks PASS (local baseline only)
|
||||
```
|
||||
|
||||
Windows 环境为 `CGO_ENABLED=0` 且没有 C 编译器,`go test -race` 在本机未执行;
|
||||
CI 已配置 Linux race job。PostgreSQL 18 和 Redis 8.2 的隔离 Adapter fixture
|
||||
已经运行;本地 Compose 已声明 Controller、两个独立 Gateway 身份和一个 Checker 的
|
||||
mTLS 运行拓扑,并由配置、证书、命令装配和部署契约测试覆盖。Kubernetes base 刻意不直接
|
||||
启用 Checker;`development-mtls` Overlay 已为单副本 Controller、Gateway 和 Checker
|
||||
提供固定开发证书、运行时身份派生、mTLS Secret 挂载、最小出站 NetworkPolicy 与渲染/配置契约验证。
|
||||
其 ConfigMap 使用内容哈希以在配置变化时触发三个工作负载滚动;固定名称的外部 Secret 仍要求
|
||||
受控轮换和显式重启。
|
||||
Controller、Gateway 与 Checker 的叶证书和信任根已在每个新 TLS 握手时重载,文件短暂
|
||||
不一致时保留最后一次有效材料。Gateway、Checker 启动派生和 Controller 授权共用同一
|
||||
SPIFFE URI 解析器:X.509-SVID 叶证书必须且只能包含一个 URI SAN,并精确匹配其角色。
|
||||
生产环境仍必须通过独立 Overlay 为每个弹性 Worker 注入唯一 SPIFFE URI,且容器端到端
|
||||
启动仍待具备镜像网络的环境验证。
|
||||
|
||||
GitHub Actions 另行执行 Dockerfile 的 Linux 多阶段镜像构建;该门禁证明镜像可以构建,
|
||||
不替代 Controller/Gateway/Checker 与真实 Redis/PostgreSQL 的容器运行验证。
|
||||
|
||||
运维手册已覆盖管理面/活动池恢复顺序、发布与兼容回滚、Secret 轮换、容量计算,以及
|
||||
主机内核、文件描述符、NAT/conntrack 的只读基线和证据采集要求;这些文档不替代真实环境
|
||||
的备份恢复、驱逐或容量演练。
|
||||
|
||||
仓库审计会拒绝生产源和部署清单中的 `TODO`、`TBD`、`FIXME` 与私钥材料;原有 OpenAPI、
|
||||
Queue、配置、指标和部署契约分别锁定一次性 Extract、队列容量、库存/累计获取边界及
|
||||
低基数标签。根 `.dockerignore` 还会从镜像构建上下文排除本地控制面证书、私钥、环境文件和
|
||||
构建产物。开发示例、环境变量引用和 CI fixture 不被视为生产 Secret 或未实现占位。
|
||||
已经运行;当前 Docker 镜像和激活的 Compose/Kubernetes 清单只包含已实现的
|
||||
Controller/Gateway 入口,完整 mTLS 运行时拓扑仍只有静态验证。
|
||||
|
||||
## 3. 后续实现范围
|
||||
|
||||
以下已有设计、接口或部署位置,但尚无端到端生产实现:
|
||||
|
||||
1. 已实现的 Controller、Gateway、Checker 和 Loadgen 进程仍需在实际容器网络中完成
|
||||
mTLS、DNS、Redis/PostgreSQL 连通性、健康探针和优雅停机的端到端验证。
|
||||
2. Gateway 已支持连接池、每 Host 连接上限、握手/空闲超时、隧道缓冲、请求生命周期指标
|
||||
与 p99 告警;仍需代表性环境的流量压测和容量报告。
|
||||
3. Provider 长期凭据回收与累计额度的生产运营策略仍需结合实际供应商合同、失效时间和
|
||||
密钥平台完成演练。
|
||||
4. Kubernetes 的开发 mTLS Overlay 已覆盖单副本开发证书身份派生和 Checker 出站
|
||||
NetworkPolicy;生产环境仍需要按工作负载身份系统提供证书轮换、每副本唯一 Worker ID
|
||||
与可弹性扩缩容的身份注入,base 清单不假定或伪造这些能力。
|
||||
5. Redis/PostgreSQL 故障转移、控制面断线恢复、Snapshot 回滚、Drain 回收和 Extract
|
||||
库存不足的完整故障演练。
|
||||
6. 10k 稳态、100k 峰值、CONNECT 活跃连接和建连速率的代表性集群负载测试。
|
||||
1. `proxy-loadgen` 已提供有界 HTTP、HTTP Gateway CONNECT 长连接和 Distribution Extract
|
||||
响应校验场景;`proxy-checker` 的 BASIC 任务进程已经完成,
|
||||
`proxy-controller` 已完成
|
||||
Admin/Distribution/Metrics 与 PostgreSQL/Redis 启动装配,`proxy-gateway` 已完成
|
||||
HTTP/Metrics 与控制面 Session 装配;Checker 与 Gateway Outcome 的低基数业务指标
|
||||
已接入,但 Provider、提取和容量指标链未闭环。
|
||||
2. Gateway 的生产连接池调优与代表性流量压测。
|
||||
3. Provider 分布式 singleflight/Leader、长期凭据回收和累计额度执行器。
|
||||
4. Controller 的 PostgreSQL 连接池、迁移和 pgx Adapter 启动装配已完成;
|
||||
公用 bootstrap 已通过 PostgreSQL 18 + Redis 8.2 双存储集成,Controller
|
||||
三监听器与探针集成已完成;可选聚合指标和完整容器进程部署验证仍待实现。
|
||||
5. Worker Register/ACK/Runtime gRPC 接收路径、Redis 服务端 TTL 会话生命周期、
|
||||
SPIFFE mTLS 校验和 Controller 生命周期接线已完成;Redis Provider Leader、
|
||||
分布式请求额度、Distribution Client 限制和 Provider 状态丢失重建已完成。
|
||||
6. Worker 基础网络快照流、Proxy/Gateway Routing/凭据 Snapshot payload、Gateway Snapshot
|
||||
客户端和进程装配、同版本 Routing 编译/动态匹配、五种策略上游选择与 reject/wait/direct 已完成;
|
||||
Outcome 上报已完成基础观测链;Redis ownership drain/ACK/过期回收及按 Worker 的可下发索引已完成。
|
||||
7. BASIC Checker 调度与 HTTP/HTTPS/SOCKS5 探测器、全局与 TARGET Profile 的 Memory/Redis
|
||||
原子归并、Controller Reducer 和 Observation 上报 RPC 已完成;EGRESS、TARGET
|
||||
生产任务调度与 REMOVE 编排仍待实现。
|
||||
8. Admin/Distribution 细粒度授权和审计查询;Distribution 分布式限流已完成。
|
||||
9. 真实 Compose/Kubernetes 集成、故障演练和代表性集群负载测试。
|
||||
10. 将 reject/wait/direct 接入 Distribution 运行链,补齐 Sequential 持久化恢复、跨实例 CAS
|
||||
和 disabled candidate 语义。
|
||||
11. 补齐 Proxy Capacity 动态降容契约、Reservation 全生命周期观测;短 TTL
|
||||
Proxy 的零计数运行态已在后续 Snapshot Apply 中回收。
|
||||
|
||||
## 4. 容量结论
|
||||
|
||||
100,000 QPS 是集群设计输入,不是本次验证结果。只有实现上述运行时,并在
|
||||
|
||||
@ -9,8 +9,8 @@
|
||||
|---|---|---|---|
|
||||
| ARCH-001 | 数据面 Worker 与控制面 Controller 分离 | 1-70 | 包、协议和部署拓扑已分离;Controller 已运行 Worker Register/Watch/ACK/Runtime/Outcome 与 Checker Observation gRPC,并发布 Proxy/Gateway Routing/按引用去重凭据完整快照;Checker 任务流已具备有界领取、租约栅栏和任务期凭据契约。Gateway 已将快照编译为同版本动态 View,并由独立进程维护控制面会话。`proxy-checker` 与 Redis BASIC/EGRESS/TARGET 共享任务运行态、`proxy-loadgen` 有界 HTTP/CONNECT/Extract 校验场景均已实现 |
|
||||
| ARCH-002 | 热路径只做认证、本地路由和网络转发 | 1-70, 380-430 | Gateway bootstrap 集成测试验证启动期控制面会话与快照就绪,HTTP 请求只走本地 Snapshot/Dispatch;Outcome 仅写入有界非阻塞本地队列,代表性性能剖析待完成 |
|
||||
| ARCH-003 | Gateway、Distribution、Admin、Metrics 独立入口 | 8904-8958 | Controller 命令已装配 Distribution/Admin/Metrics 三个独立监听及联动停机;Gateway 命令已装配代理与 Metrics 监听。本地 Compose 已声明 Controller/Gateway/Checker mTLS 拓扑;Kubernetes `development-mtls` Overlay 已通过渲染与配置契约验证单副本开发证书身份派生链路,生产工作负载身份仍待环境实现 |
|
||||
| ARCH-004 | Controller 集中 Provider 获取与切换 | 1403-1580 | Redis Leader、动态 Provider Supervisor 与 Bootstrap 生产装配已完成;静态配置与 Admin 管理态共同决定 Provider 启停,停用策略以 Upstream revision 栅栏驱动既有 ownership Drain;Routing 切换会向本进程 Worker 快照流广播完整刷新,多副本按定时刷新与权威 HMAC 指纹/revision 栅栏收敛并拒绝旧配置换主 |
|
||||
| ARCH-003 | Gateway、Distribution、Admin、Metrics 独立入口 | 8904-8958 | Controller 命令已装配 Distribution/Admin/Metrics 三个独立监听及联动停机;Gateway 命令已装配代理与 Metrics 监听,运行时 mTLS 部署 Overlay 待完成 |
|
||||
| ARCH-004 | Controller 集中 Provider 获取与切换 | 1403-1580 | Redis Leader、动态 Provider Supervisor 与 Bootstrap 生产装配已完成;Admin disable/reload 驱动取消替换,多副本按权威 HMAC 指纹和 revision 栅栏收敛并拒绝旧配置换主;Routing 切换到 Drain 的编排待完成 |
|
||||
| ARCH-005 | 100k QPS 峰值使用多 Worker 集群 | 当前会话 | 未验证设计目标;待代表性集群负载报告 |
|
||||
|
||||
## Routing 与 Upstream
|
||||
@ -19,11 +19,11 @@
|
||||
|---|---|---|---|
|
||||
| ROUTE-001 | Routing 自上而下匹配,首条命中停止 | 3534-3798, 5825-6467 | `rule.go` 与不可变/首命中单测 |
|
||||
| ROUTE-002 | Routing 与 Upstream 生命周期解耦 | 3534-3798 | 包依赖与配置模型 |
|
||||
| ROUTE-003 | 支持 sequential、random、roundRobin、weighted、leastConnections | 5825-6467 | 五种领域策略、同版本 Gateway Snapshot 派发和定向测试已完成。Distribution 按独占 Extract 契约分配 Proxy,不参与目标请求的 Routing 策略选择 |
|
||||
| ROUTE-004 | Sequential 连续空结果达到阈值后原子切换一次 | 5295-5824, 6520-6617 | Provider Stats 对连续空结果分配单调代次,Controller `SequentialCoordinator` 在独立有界循环中读取权威配置/管理快照,并通过 `ExpectedCurrent` CAS 自动切换。禁用候选会跳过;若当前项被管理态禁用,协调器不读取 Provider Stats,按配置顺序推进到下一启用项,只有 `loop` 可回绕,`stop`/`stayLast` 无后继时通过 `DisableRouting` 原子停用路由。重复 Tick、旧代次及跨 Controller 并发 Tick 均不会重复变更;成功切换或停用都会广播完整 Snapshot,并与审计/Outbox 同事务提交。|
|
||||
| ROUTE-003 | 支持 sequential、random、roundRobin、weighted、leastConnections | 5825-6467 | 五种领域策略、同版本 Gateway Snapshot 派发和定向测试已完成;Distribution 接线待完成 |
|
||||
| ROUTE-004 | Sequential 连续空结果达到阈值后原子切换一次 | 5295-5824, 6520-6617 | 进程内 `RoutingCursor` 版本 CAS 与 100 并发测试已完成;持久化恢复和跨实例 CAS 待完成 |
|
||||
| ROUTE-005 | 空计数属于 Upstream,当前选择属于 Routing | 8442-8529 | 共享 `UpstreamEmptyState` 双 Routing 测试 |
|
||||
| ROUTE-006 | 旧 Upstream 已有 Proxy 继续耗尽,不因切换直接丢弃 | 6618-6641 | Routing 成功切换后立即发布完整快照;Sequential 仅将新分配切到新的 CurrentUpstream,旧 Proxy 仍保留在快照,既有 Active/Reserved 由本地运行态自然归零。共享 Upstream 不按单 Routing 强制 Drain,避免影响其他 Routing |
|
||||
| ROUTE-007 | 无可用 Upstream 时显式 reject、wait 或 direct,默认 reject | 5075-5294, 6743-6760 | Gateway 已实现默认 reject、带 `wait_timeout` 的本地容量等待,以及经 TargetPolicy 的 direct。静态 `routing.action: direct` 在认证、限流和目标策略后绕过 Proxy 调度;Distribution 不承载目标请求路由 |
|
||||
| ROUTE-006 | 旧 Upstream 已有 Proxy 继续耗尽,不因切换直接丢弃 | 6618-6641 | 通用 ownership Drain/ACK 原语已测试;Routing 切换到 Drain 的编排待完成 |
|
||||
| ROUTE-007 | 无可用 Upstream 时显式 reject、wait 或 direct,默认 reject | 5075-5294, 6743-6760 | Gateway 已实现 reject、带 `wait_timeout` 的本地容量等待与经 TargetPolicy 的 direct;Distribution 接线和默认化策略待完成 |
|
||||
|
||||
## Provider 与补池
|
||||
|
||||
@ -45,8 +45,8 @@
|
||||
| PROXY-001 | Proxy 保存协议、地址、凭据引用、来源、TTL、健康、容量和标签 | 71-105, 8605-8678 | Domain 类型与序列化测试 |
|
||||
| PROXY-002 | 唯一键包含 scheme、host、port、username、credentialVersion | 6655-6727, 8605-8678 | 去重单测 |
|
||||
| PROXY-003 | TTL 来源优先级明确并统一 UTC | 681-747, 8655-8678 | TTL 表驱动测试 |
|
||||
| CAP-001 | Gateway 分配使用 Reserved -> Active 原子转换 | 1203-1467, 8530-8597 | 固定 Max 下打包 CAS、1,000 并发不超卖、退役 Proxy 停止新预留及零计数历史运行态回收已完成;Snapshot 动态降容会保留既有 Active、在计数低于新上限前拒绝新预留,运行态复用与恢复预留均有回归测试。非法 Commit/Cancel/Release 经固定枚举观察器进入低基数指标 |
|
||||
| CAP-002 | 补池依据 Available Slots,不只看 Proxy 数量 | 1203-1402, 8530-8597 | `AvailableSlots`、显式 minimum/target 水位、pending 槽位和迟滞 Reconciler 已测试;Worker session/ACK/sequence/TTL/ownership fence、权威 Proxy/Routing/凭据 payload、目标健康和 Gateway reserve 策略已通过内存、Redis 8.2、gRPC 与 Gateway 回归测试 |
|
||||
| CAP-001 | Gateway 分配使用 Reserved -> Active 原子转换 | 1203-1467, 8530-8597 | 固定 Max 下打包 CAS、1,000 并发不超卖、退役 Proxy 停止新预留及零计数历史运行态回收已完成;动态降容和完整生命周期证据待完成 |
|
||||
| CAP-002 | 补池依据 Available Slots,不只看 Proxy 数量 | 1203-1402, 8530-8597 | `AvailableSlots`、显式 minimum/target 水位、pending 槽位和迟滞 Reconciler 已测试;Worker session/ACK/sequence/TTL/ownership fence、基础 Snapshot 流及 Gateway 会话组件已通过内存、Redis 8.2 和 gRPC 测试;权威 payload、目标健康和 Gateway reserve 策略仍待完成 |
|
||||
| CAP-003 | pool.maxSize 包括 FETCHED/CHECKING/AVAILABLE/SUSPECT/DRAINING 与 pending expected | 3001-3533, 6642-6680 | `FetchBudget` 100 并发额度预占测试 |
|
||||
| CAP-004 | TTL safety margin 内禁止新分配 | 173-220, 6728-6741 | 时钟测试 |
|
||||
| CAP-005 | 多 Worker 不在热路径访问 Redis 计数 | 1403-1467 | Gateway 包依赖审计、Snapshot/Dispatch 测试 |
|
||||
@ -86,6 +86,6 @@
|
||||
| SEC-002 | 非回环监听无保护时严格模式启动失败 | 8112-8441 | 配置校验测试 |
|
||||
| OPS-001 | 配置校验后构建不可变快照并原子替换 | 8959-8999 | 100k 索引、版本/epoch 与并发 Apply/Acquire 测试 |
|
||||
| OPS-002 | 优雅停机停止新请求/Fetch,等待现有流量后超时关闭 | 8981-9000 | Provider Run 收敛与 `Handler.Shutdown` HTTP 排空、Hijacked CONNECT 超时关闭测试 |
|
||||
| OPS-003 | PostgreSQL 只保存管理修订、Upstream/Routing 状态、Admin 审计与 Outbox | 当前会话 | ADR-006、`adminstate` 公用契约和六表 Schema 边界测试;真实 PostgreSQL 18 契约、迁移幂等和 `information_schema` 数据边界测试已完成 |
|
||||
| OBS-001 | 指标禁止 Proxy IP、session、Client、完整 URL 高基数标签 | 9001-9029 | Controller Prometheus/探针模块已实现;Gateway Collector 暴露固定 `protocol` 的请求总数、并发数、隧道数和生命周期直方图,以及固定 `stage,result` 的 Outcome 指标。Checker、Drain、Provider、Extraction 与 Capacity Collector 仅使用固定枚举标签;Grafana 和 Prometheus 规则由部署契约测试校验只引用已注册指标且不按 Upstream/Worker 聚合。`platform/logging` 使用 JSON `slog`,敏感字段、URL 用户信息、查询 Secret 和错误对象写出前均被脱敏;回归测试锁定标签集与 Secret 不泄露 |
|
||||
| OPS-003 | PostgreSQL 只保存管理修订、Upstream/Routing 状态、Admin 审计与 Outbox | 当前会话 | ADR-006、`adminstate` 公用契约和六表 Schema 边界测试;真实 PostgreSQL 契约待完成 |
|
||||
| OBS-001 | 指标禁止 Proxy IP、session、Client、完整 URL 高基数标签 | 9001-9029 | Controller Prometheus/探针模块已实现;Checker Collector 暴露固定等级和结果,Gateway Collector 暴露 `proxy_pool_gateway_outcomes_total{stage,result}` 与 `proxy_pool_gateway_outcome_queue_dropped_total`,均以注册表测试固定标签集。Provider、提取和容量业务指标待实现 |
|
||||
| TEST-001 | 覆盖对话中列出的 11 个关键并发与故障场景 | 9030-9082 | 测试清单;Redis 活动池由 Memory/Redis 公用契约覆盖,跨进程故障场景仍按清单推进 |
|
||||
|
||||
@ -17,19 +17,8 @@
|
||||
- `httpsecurity.Protection` 统一实现 Basic、API Key、Bearer、CIDR、可信代理链、
|
||||
Client ID 与入口准入;API 使用 401/`WWW-Authenticate`,Gateway 使用
|
||||
407/`Proxy-Authenticate`。
|
||||
- Listener 可将固定权限绑定到单个认证方式,或在 `mode: any` 中绑定到每个凭据;
|
||||
`admin:read`、`admin:write` 与 `distribution:extract` 在 Handler 入口完成匹配,
|
||||
不把无权请求传入管理 mutation 或 Proxy 提取流程。
|
||||
- Token Client ID 使用 SHA-256 的 128 位摘要前缀,不把 Token 本身写入领域、
|
||||
日志或审计键。
|
||||
- 启用 Gateway 粘性会话时,绑定键由认证 Client、Routing 和会话头共同派生;
|
||||
原始会话标识不写入 PostgreSQL、Redis、日志或指标,也不会转发给目标站点。
|
||||
绑定仅保留在当前 Worker 的有界内存中,并在代理不再符合当前 Snapshot、到达
|
||||
代理可用期或转发失败时失效。
|
||||
- Gateway 凭据的 requestsPerMinute 限制按认证 Client 在本 Worker 内执行,不把
|
||||
认证身份、会话或请求计数写入 PostgreSQL、Redis 或 Prometheus 标签。
|
||||
- Gateway 凭据的 maxConcurrentConnections 在请求或 CONNECT 隧道的完整生命周期
|
||||
内保留一个本地租约;无论转发成功、失败还是服务排空,租约都会释放。
|
||||
|
||||
## 3. 目标地址策略
|
||||
|
||||
@ -48,8 +37,6 @@ CGNAT、协议转换保留段和已知云元数据端点属于硬拒绝项。私
|
||||
## 4. Secret 处理
|
||||
|
||||
- 配置只保存环境变量或文件引用,不在日志中输出解析后的值。
|
||||
- Docker 镜像构建上下文通过根目录 `.dockerignore` 排除本地环境文件、控制面证书/私钥、
|
||||
凭据文件、Git 元数据和测试/构建产物;运行时证书只能通过受控挂载提供。
|
||||
- Proxy 唯一键包含 username 与 credentialVersion,不包含密码或 SecretRef 内容。
|
||||
- Provider Parser 将静态/响应密码写入 `credentials.Store`,Proxy Snapshot 只携带
|
||||
SecretRef 与 credentialVersion;Gateway 通过统一 Resolver 按版本读取。
|
||||
@ -65,9 +52,8 @@ CGNAT、协议转换保留段和已知云元数据端点属于硬拒绝项。私
|
||||
|
||||
## 6. 审计
|
||||
|
||||
PostgreSQL 审计只记录 Admin 管理 mutation 的 actor、来源、资源、变更结果和
|
||||
时间;它不保存 Proxy ID、代理地址、凭据、逐次 Extraction 或 Gateway 请求。
|
||||
Extraction 的短期幂等结果仅位于 Redis TTL 活动池;审计查询接口必须受 Admin
|
||||
Extraction 审计记录至少包含 requestId、Client、来源、Proxy ID、Upstream、
|
||||
提取时间和到期时间。日志脱敏不影响审计关联,但审计接口自身必须受 Admin
|
||||
权限保护并具备保留期限。
|
||||
|
||||
## 7. 优雅停机
|
||||
|
||||
@ -83,19 +83,13 @@ soak 测试单独标记,不混入快速单测。
|
||||
|
||||
## 5. 当前本地微基准
|
||||
|
||||
2026-08-07,Windows/amd64、Go 1.26.4、Intel Core Ultra 7 155H,以固定
|
||||
`-count=1 -benchtime=1x` 运行:
|
||||
2026-07-30,Windows/amd64、Intel Core Ultra 7 155H:
|
||||
|
||||
```text
|
||||
BenchmarkAcquire100kIndexed-22 1 13100 ns/op 256 B/op 2 allocs/op
|
||||
BenchmarkAcquire100kRoutingRoundRobin-22 1 45200 ns/op 13680 B/op 81 allocs/op
|
||||
BenchmarkStoreApply100k-22 1 539017100 ns/op 684677920 B/op 3001735 allocs/op
|
||||
BenchmarkAcquire100kIndexed-22 1000000-1867125 893.9-1047 ns/op 256 B/op 2 allocs/op
|
||||
BenchmarkStoreApply100k-22 1 518.7 ms/op 540 MB/op 3000887 allocs/op
|
||||
```
|
||||
|
||||
使用 `./scripts/benchmark-gateway.ps1` 可复现该固定单次基线,输出写入被 Git 忽略的
|
||||
`dist/gateway-benchmarks.txt` 并附带 revision。单次基准存在运行噪声,应只用于发现明显回归;
|
||||
性能比较需要在同规格环境增加多次迭代后另行保存原始证据。
|
||||
|
||||
`Acquire` 已使用 scheme/upstream/tag 索引,结果只代表本地选择和容量预留。
|
||||
`Store.Apply` 属于冷路径且当前内存开销较高;运行态为防止旧快照在途连接超配,
|
||||
暂不自动回收曾出现过的 Proxy ID,但注册表有 1,000,000 项硬上限,达到上限时
|
||||
|
||||
@ -182,7 +182,6 @@ Admin 应用层测试覆盖 typed-nil 依赖、Actor/SourceIP 映射、Routing C
|
||||
配置:Snapshot 规模、Proxy 容量、路由、重试、日志级别
|
||||
场景:协议、连接复用、响应体、持续时间、升压曲线、故障注入
|
||||
结果:QPS、建连速率、active、p50/p95/p99、错误、CPU、RSS、GC、FD、网络
|
||||
限速:配置 rate、实际 Requests、RateStartsGenerated、RateStartsDropped
|
||||
不变量:capacity、ownership、extraction、idempotency、admin audit、reserve 检查结果
|
||||
结论:通过/失败,以及适用边界
|
||||
```
|
||||
|
||||
72
findings.md
72
findings.md
@ -49,27 +49,6 @@ Extract 请求使用哪些 Upstream。
|
||||
- Gateway、Distribution、Admin、Metrics 使用独立监听和认证/访问控制。
|
||||
- 非回环监听且无认证、无 CIDR 保护时,严格模式必须拒绝启动。
|
||||
|
||||
## 可观测性边界
|
||||
|
||||
- Provider 拉取结果只按固定 `class` 聚合:`valid`、`empty`、`duplicate_only`、`error`。
|
||||
指标分别记录拉取结果次数、合法候选数和新入池代理数,不按 Upstream、Proxy、IP、错误
|
||||
文本或凭据拆分。
|
||||
- Provider 指标已完成。
|
||||
|
||||
- Extraction 只按固定 `result` 聚合请求:`complete`、`partial`、`empty`、
|
||||
`insufficient`、`idempotency_conflict`、`rate_limited`、`unavailable`、`invalid`、
|
||||
`error`。计数只记录请求数量和响应交付数量;幂等重放不被误记为新的 Proxy 消费。
|
||||
- Extraction 指标已完成。
|
||||
|
||||
- 容量指标由 Provider 库存对账循环采样,聚合 managed、available、effective、pending
|
||||
与 active Upstream Gauge。Upstream ID 仅保留在 Controller 进程内用于聚合替换和
|
||||
任期结束清理,不输出为 Prometheus 标签;读取结果只允许 `success` 或 `error`。
|
||||
- 容量指标已完成。
|
||||
|
||||
- 进程级致命错误统一经 JSON `slog` 输出;敏感属性、错误对象、URL 用户信息和查询
|
||||
Secret 在写出前脱敏。该出口只用于生命周期边界,不在 Gateway/Distribution 请求热路径
|
||||
逐条写日志。
|
||||
|
||||
## 集群与性能
|
||||
|
||||
- 用户补充:高峰可能达到 100,000 请求/秒。
|
||||
@ -145,17 +124,10 @@ Routing 自上而下匹配,首条命中停止;支持 Gateway 与 Extract 两
|
||||
Gateway 使用既有 `SessionSupervisor` 退避重连。Gateway 本地 Store 对从完整 Snapshot
|
||||
消失且仍有 Active/Reserved 的 Proxy 已按 draining 继续上报。Redis Runtime 替换已在当前
|
||||
session ACK、Ticket 屏障和完整零计数同时成立时自动释放 owner;Reaper 已对持续
|
||||
`UNHEALTHY` 的已归属代理调用条件式 Drain。配置停用也会发布有效的上游策略视图,
|
||||
从上游 owned 索引有界选择候选;候选携带上游 revision,Redis 在创建 Drain Ticket 前
|
||||
再次核验策略仍停用、Proxy source、Worker 和 assignment epoch,避免旧候选跨启停使用。
|
||||
`UNHEALTHY` 的已归属代理调用条件式 Drain。配置停用尚未接入同一触发编排。
|
||||
- Snapshot 的 `version` 是同一 Worker 流的连续序列,`ownership_epoch` 是独立且只能前进
|
||||
的权威栅栏;epoch 变化不重置 version。此前 Gateway 错把 epoch 前进要求为 version=1,
|
||||
与 Controller 的 `lastAppliedVersion+1` 生成规则冲突,现已用连续版本规则统一。
|
||||
- `SnapshotRefreshBroker` 是 Controller 进程内的公共扇出接口:已提交的 Upstream 启停、
|
||||
Routing 切换和配置发布各向每条本地 Worker 流发送一个可合并刷新信号。Routing 不记录
|
||||
单代理归属,不能按单条 Routing 对共享 Upstream 强制 Drain;Sequential 切换只改变后续
|
||||
新分配,旧 Proxy/连接由原有 Snapshot 与运行态自然排空。其他 Controller 副本继续在
|
||||
固定刷新周期内读取 PostgreSQL 权威管理态并收敛。
|
||||
|
||||
## Drain Ticket(2026-08-02)
|
||||
|
||||
@ -250,45 +222,3 @@ Routing 自上而下匹配,首条命中停止;支持 Gateway 与 Extract 两
|
||||
- 生产 `docker-compose.yml` 仍把 PostgreSQL 18 命名卷挂在旧路径
|
||||
`/var/lib/postgresql/data`;直接修改可能影响已有本地数据,必须配套迁移步骤后
|
||||
单独处理,当前不能把生产持久化拓扑视为已验证。
|
||||
|
||||
## 控制面 mTLS 身份审计(2026-08-07)
|
||||
|
||||
- X.509-SVID 叶证书承载单一工作负载身份;Gateway/Checker 启动时的自动身份派生与
|
||||
Controller 授权必须都拒绝包含多个 URI SAN 的证书,即使其中只有一个 URI 与当前角色匹配。
|
||||
- 解析边界已统一到 `workerruntime.SingleSPIFFEIdentity`:精确匹配信任域、环境、角色和
|
||||
标识符,拒绝用户信息、端口、查询、片段、转义路径、额外 URI SAN 与跨角色复用。
|
||||
- TLS 校验可能形成多条等价验证链;授权只检查同一叶证书一次,不能将链路数误判为多个身份。
|
||||
- Windows 本地 Go 运行环境为 `CGO_ENABLED=0` 且没有 C 编译器;Docker Engine 可用,但
|
||||
`golang:1.26-bookworm` 与 `debian:bookworm-slim` 未缓存,Docker Desktop HTTPS 代理也不可用。
|
||||
因此已完成 Go 全量、Compose 静态和 Kustomize 静态验证,容器端到端验证仍待具备镜像网络的环境。
|
||||
|
||||
## OpenAPI 契约审计(2026-08-07)
|
||||
|
||||
- Go 结构契约继续锁定本地引用、operationId、响应与认证引用;额外以固定
|
||||
`@redocly/cli@2.25.4` 对两份 OpenAPI 3.1 文档执行标准验证,避免只依赖自定义遍历器。
|
||||
- 标准最小规则集中的 Tag 描述已提升为错误;Distribution、Health、Status、Audit、Upstreams、
|
||||
Routing 和 Configuration 标签均有面向 API 使用者的稳定说明。
|
||||
|
||||
## Docker 构建上下文审计(2026-08-07)
|
||||
|
||||
- Dockerfile 构建阶段需要 `COPY . .`,因此根 `.dockerignore` 是隔离本地工作区与镜像上下文的
|
||||
必要边界。它显式排除 Git 元数据、环境文件、本地配置、控制面证书/私钥、PEM/CRT/KEY 文件及
|
||||
构建和测试产物;部署测试会在清单被删除或放宽时失败。
|
||||
|
||||
## 部署静态门禁审计(2026-08-07)
|
||||
|
||||
- GitHub Actions 的 Deployment job 在同一组无敏感值 fixture 下渲染生产 Compose、测试 Compose
|
||||
以及 Kubernetes base/development mTLS Overlay,并对两份配置运行严格校验;它仍不替代
|
||||
需要镜像网络和真实服务的容器端到端测试。
|
||||
|
||||
## Kubernetes 配置发布审计(2026-08-07)
|
||||
|
||||
- development mTLS Overlay 的 ConfigMap 采用 Kustomize 内容哈希,渲染契约要求三个工作负载
|
||||
都引用同一版本化名称,因此配置变更进入 Pod 模板并触发滚动。外部固定名称 Secret 不含内容哈希,
|
||||
它们的轮换需要平台侧 reloader 或受控显式滚动;这一限制已写入运行手册。
|
||||
|
||||
## Linux 镜像构建审计(2026-08-07)
|
||||
|
||||
- GitHub Actions 现在在 Linux Runner 执行 Dockerfile 多阶段构建,覆盖目标平台 Go 编译和最小
|
||||
运行时层。该 job 依赖 Runner 镜像网络;它不启动 Compose 服务,因此不能作为 mTLS、存储、
|
||||
探针或优雅停机端到端证据。
|
||||
|
||||
@ -134,58 +134,6 @@ func (UnavailableAction) EnumDescriptor() ([]byte, []int) {
|
||||
return file_controlplane_v1_controlplane_proto_rawDescGZIP(), []int{1}
|
||||
}
|
||||
|
||||
type RoutingAction int32
|
||||
|
||||
const (
|
||||
RoutingAction_ROUTING_ACTION_UNSPECIFIED RoutingAction = 0
|
||||
RoutingAction_ROUTING_ACTION_PROXY RoutingAction = 1
|
||||
RoutingAction_ROUTING_ACTION_DIRECT RoutingAction = 2
|
||||
RoutingAction_ROUTING_ACTION_REJECT RoutingAction = 3
|
||||
)
|
||||
|
||||
// Enum value maps for RoutingAction.
|
||||
var (
|
||||
RoutingAction_name = map[int32]string{
|
||||
0: "ROUTING_ACTION_UNSPECIFIED",
|
||||
1: "ROUTING_ACTION_PROXY",
|
||||
2: "ROUTING_ACTION_DIRECT",
|
||||
3: "ROUTING_ACTION_REJECT",
|
||||
}
|
||||
RoutingAction_value = map[string]int32{
|
||||
"ROUTING_ACTION_UNSPECIFIED": 0,
|
||||
"ROUTING_ACTION_PROXY": 1,
|
||||
"ROUTING_ACTION_DIRECT": 2,
|
||||
"ROUTING_ACTION_REJECT": 3,
|
||||
}
|
||||
)
|
||||
|
||||
func (x RoutingAction) Enum() *RoutingAction {
|
||||
p := new(RoutingAction)
|
||||
*p = x
|
||||
return p
|
||||
}
|
||||
|
||||
func (x RoutingAction) String() string {
|
||||
return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x))
|
||||
}
|
||||
|
||||
func (RoutingAction) Descriptor() protoreflect.EnumDescriptor {
|
||||
return file_controlplane_v1_controlplane_proto_enumTypes[2].Descriptor()
|
||||
}
|
||||
|
||||
func (RoutingAction) Type() protoreflect.EnumType {
|
||||
return &file_controlplane_v1_controlplane_proto_enumTypes[2]
|
||||
}
|
||||
|
||||
func (x RoutingAction) Number() protoreflect.EnumNumber {
|
||||
return protoreflect.EnumNumber(x)
|
||||
}
|
||||
|
||||
// Deprecated: Use RoutingAction.Descriptor instead.
|
||||
func (RoutingAction) EnumDescriptor() ([]byte, []int) {
|
||||
return file_controlplane_v1_controlplane_proto_rawDescGZIP(), []int{2}
|
||||
}
|
||||
|
||||
type ProxyProtocol int32
|
||||
|
||||
const (
|
||||
@ -222,11 +170,11 @@ func (x ProxyProtocol) String() string {
|
||||
}
|
||||
|
||||
func (ProxyProtocol) Descriptor() protoreflect.EnumDescriptor {
|
||||
return file_controlplane_v1_controlplane_proto_enumTypes[3].Descriptor()
|
||||
return file_controlplane_v1_controlplane_proto_enumTypes[2].Descriptor()
|
||||
}
|
||||
|
||||
func (ProxyProtocol) Type() protoreflect.EnumType {
|
||||
return &file_controlplane_v1_controlplane_proto_enumTypes[3]
|
||||
return &file_controlplane_v1_controlplane_proto_enumTypes[2]
|
||||
}
|
||||
|
||||
func (x ProxyProtocol) Number() protoreflect.EnumNumber {
|
||||
@ -235,7 +183,7 @@ func (x ProxyProtocol) Number() protoreflect.EnumNumber {
|
||||
|
||||
// Deprecated: Use ProxyProtocol.Descriptor instead.
|
||||
func (ProxyProtocol) EnumDescriptor() ([]byte, []int) {
|
||||
return file_controlplane_v1_controlplane_proto_rawDescGZIP(), []int{3}
|
||||
return file_controlplane_v1_controlplane_proto_rawDescGZIP(), []int{2}
|
||||
}
|
||||
|
||||
type OutcomeStage int32
|
||||
@ -277,11 +225,11 @@ func (x OutcomeStage) String() string {
|
||||
}
|
||||
|
||||
func (OutcomeStage) Descriptor() protoreflect.EnumDescriptor {
|
||||
return file_controlplane_v1_controlplane_proto_enumTypes[4].Descriptor()
|
||||
return file_controlplane_v1_controlplane_proto_enumTypes[3].Descriptor()
|
||||
}
|
||||
|
||||
func (OutcomeStage) Type() protoreflect.EnumType {
|
||||
return &file_controlplane_v1_controlplane_proto_enumTypes[4]
|
||||
return &file_controlplane_v1_controlplane_proto_enumTypes[3]
|
||||
}
|
||||
|
||||
func (x OutcomeStage) Number() protoreflect.EnumNumber {
|
||||
@ -290,7 +238,7 @@ func (x OutcomeStage) Number() protoreflect.EnumNumber {
|
||||
|
||||
// Deprecated: Use OutcomeStage.Descriptor instead.
|
||||
func (OutcomeStage) EnumDescriptor() ([]byte, []int) {
|
||||
return file_controlplane_v1_controlplane_proto_rawDescGZIP(), []int{4}
|
||||
return file_controlplane_v1_controlplane_proto_rawDescGZIP(), []int{3}
|
||||
}
|
||||
|
||||
type CheckLevel int32
|
||||
@ -329,11 +277,11 @@ func (x CheckLevel) String() string {
|
||||
}
|
||||
|
||||
func (CheckLevel) Descriptor() protoreflect.EnumDescriptor {
|
||||
return file_controlplane_v1_controlplane_proto_enumTypes[5].Descriptor()
|
||||
return file_controlplane_v1_controlplane_proto_enumTypes[4].Descriptor()
|
||||
}
|
||||
|
||||
func (CheckLevel) Type() protoreflect.EnumType {
|
||||
return &file_controlplane_v1_controlplane_proto_enumTypes[5]
|
||||
return &file_controlplane_v1_controlplane_proto_enumTypes[4]
|
||||
}
|
||||
|
||||
func (x CheckLevel) Number() protoreflect.EnumNumber {
|
||||
@ -342,7 +290,7 @@ func (x CheckLevel) Number() protoreflect.EnumNumber {
|
||||
|
||||
// Deprecated: Use CheckLevel.Descriptor instead.
|
||||
func (CheckLevel) EnumDescriptor() ([]byte, []int) {
|
||||
return file_controlplane_v1_controlplane_proto_rawDescGZIP(), []int{5}
|
||||
return file_controlplane_v1_controlplane_proto_rawDescGZIP(), []int{4}
|
||||
}
|
||||
|
||||
type RegisterWorkerRequest struct {
|
||||
@ -937,7 +885,6 @@ type RoutingRule struct {
|
||||
Strategy *RoutingStrategy `protobuf:"bytes,8,opt,name=strategy,proto3" json:"strategy,omitempty"`
|
||||
OnUnavailable UnavailableAction `protobuf:"varint,9,opt,name=on_unavailable,json=onUnavailable,proto3,enum=proxy_pool.controlplane.v1.UnavailableAction" json:"on_unavailable,omitempty"`
|
||||
WaitTimeout *durationpb.Duration `protobuf:"bytes,10,opt,name=wait_timeout,json=waitTimeout,proto3" json:"wait_timeout,omitempty"`
|
||||
Action RoutingAction `protobuf:"varint,11,opt,name=action,proto3,enum=proxy_pool.controlplane.v1.RoutingAction" json:"action,omitempty"`
|
||||
unknownFields protoimpl.UnknownFields
|
||||
sizeCache protoimpl.SizeCache
|
||||
}
|
||||
@ -1042,13 +989,6 @@ func (x *RoutingRule) GetWaitTimeout() *durationpb.Duration {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (x *RoutingRule) GetAction() RoutingAction {
|
||||
if x != nil {
|
||||
return x.Action
|
||||
}
|
||||
return RoutingAction_ROUTING_ACTION_UNSPECIFIED
|
||||
}
|
||||
|
||||
type RoutingStrategy struct {
|
||||
state protoimpl.MessageState `protogen:"open.v1"`
|
||||
Type StrategyType `protobuf:"varint,1,opt,name=type,proto3,enum=proxy_pool.controlplane.v1.StrategyType" json:"type,omitempty"`
|
||||
@ -2300,7 +2240,7 @@ const file_controlplane_v1_controlplane_proto_rawDesc = "" +
|
||||
"\x10upserted_routing\x18\x06 \x03(\v2'.proxy_pool.controlplane.v1.RoutingRuleR\x0fupsertedRouting\x122\n" +
|
||||
"\x15removed_routing_names\x18\a \x03(\tR\x13removedRoutingNames\x12Q\n" +
|
||||
"\x10upserted_proxies\x18\b \x03(\v2&.proxy_pool.controlplane.v1.OwnedProxyR\x0fupsertedProxies\x12*\n" +
|
||||
"\x11removed_proxy_ids\x18\t \x03(\tR\x0fremovedProxyIds\"\xdd\x04\n" +
|
||||
"\x11removed_proxy_ids\x18\t \x03(\tR\x0fremovedProxyIds\"\x9a\x04\n" +
|
||||
"\vRoutingRule\x12\x12\n" +
|
||||
"\x04name\x18\x01 \x01(\tR\x04name\x12\x18\n" +
|
||||
"\aenabled\x18\x02 \x01(\bR\aenabled\x12\x1d\n" +
|
||||
@ -2314,8 +2254,7 @@ const file_controlplane_v1_controlplane_proto_rawDesc = "" +
|
||||
"\bstrategy\x18\b \x01(\v2+.proxy_pool.controlplane.v1.RoutingStrategyR\bstrategy\x12T\n" +
|
||||
"\x0eon_unavailable\x18\t \x01(\x0e2-.proxy_pool.controlplane.v1.UnavailableActionR\ronUnavailable\x12<\n" +
|
||||
"\fwait_timeout\x18\n" +
|
||||
" \x01(\v2\x19.google.protobuf.DurationR\vwaitTimeout\x12A\n" +
|
||||
"\x06action\x18\v \x01(\x0e2).proxy_pool.controlplane.v1.RoutingActionR\x06action\x1a:\n" +
|
||||
" \x01(\v2\x19.google.protobuf.DurationR\vwaitTimeout\x1a:\n" +
|
||||
"\fHeadersEntry\x12\x10\n" +
|
||||
"\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" +
|
||||
"\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\"\x8a\x02\n" +
|
||||
@ -2458,12 +2397,7 @@ const file_controlplane_v1_controlplane_proto_rawDesc = "" +
|
||||
"\x1eUNAVAILABLE_ACTION_UNSPECIFIED\x10\x00\x12\x1d\n" +
|
||||
"\x19UNAVAILABLE_ACTION_REJECT\x10\x01\x12\x1b\n" +
|
||||
"\x17UNAVAILABLE_ACTION_WAIT\x10\x02\x12\x1d\n" +
|
||||
"\x19UNAVAILABLE_ACTION_DIRECT\x10\x03*\x7f\n" +
|
||||
"\rRoutingAction\x12\x1e\n" +
|
||||
"\x1aROUTING_ACTION_UNSPECIFIED\x10\x00\x12\x18\n" +
|
||||
"\x14ROUTING_ACTION_PROXY\x10\x01\x12\x19\n" +
|
||||
"\x15ROUTING_ACTION_DIRECT\x10\x02\x12\x19\n" +
|
||||
"\x15ROUTING_ACTION_REJECT\x10\x03*}\n" +
|
||||
"\x19UNAVAILABLE_ACTION_DIRECT\x10\x03*}\n" +
|
||||
"\rProxyProtocol\x12\x1e\n" +
|
||||
"\x1aPROXY_PROTOCOL_UNSPECIFIED\x10\x00\x12\x17\n" +
|
||||
"\x13PROXY_PROTOCOL_HTTP\x10\x01\x12\x18\n" +
|
||||
@ -2503,104 +2437,102 @@ func file_controlplane_v1_controlplane_proto_rawDescGZIP() []byte {
|
||||
return file_controlplane_v1_controlplane_proto_rawDescData
|
||||
}
|
||||
|
||||
var file_controlplane_v1_controlplane_proto_enumTypes = make([]protoimpl.EnumInfo, 6)
|
||||
var file_controlplane_v1_controlplane_proto_enumTypes = make([]protoimpl.EnumInfo, 5)
|
||||
var file_controlplane_v1_controlplane_proto_msgTypes = make([]protoimpl.MessageInfo, 26)
|
||||
var file_controlplane_v1_controlplane_proto_goTypes = []any{
|
||||
(StrategyType)(0), // 0: proxy_pool.controlplane.v1.StrategyType
|
||||
(UnavailableAction)(0), // 1: proxy_pool.controlplane.v1.UnavailableAction
|
||||
(RoutingAction)(0), // 2: proxy_pool.controlplane.v1.RoutingAction
|
||||
(ProxyProtocol)(0), // 3: proxy_pool.controlplane.v1.ProxyProtocol
|
||||
(OutcomeStage)(0), // 4: proxy_pool.controlplane.v1.OutcomeStage
|
||||
(CheckLevel)(0), // 5: proxy_pool.controlplane.v1.CheckLevel
|
||||
(*RegisterWorkerRequest)(nil), // 6: proxy_pool.controlplane.v1.RegisterWorkerRequest
|
||||
(*RegisterWorkerResponse)(nil), // 7: proxy_pool.controlplane.v1.RegisterWorkerResponse
|
||||
(*WatchSnapshotsRequest)(nil), // 8: proxy_pool.controlplane.v1.WatchSnapshotsRequest
|
||||
(*SnapshotEnvelope)(nil), // 9: proxy_pool.controlplane.v1.SnapshotEnvelope
|
||||
(*WorkerSnapshot)(nil), // 10: proxy_pool.controlplane.v1.WorkerSnapshot
|
||||
(*SnapshotCredential)(nil), // 11: proxy_pool.controlplane.v1.SnapshotCredential
|
||||
(*SnapshotDelta)(nil), // 12: proxy_pool.controlplane.v1.SnapshotDelta
|
||||
(*RoutingRule)(nil), // 13: proxy_pool.controlplane.v1.RoutingRule
|
||||
(*RoutingStrategy)(nil), // 14: proxy_pool.controlplane.v1.RoutingStrategy
|
||||
(*OwnedProxy)(nil), // 15: proxy_pool.controlplane.v1.OwnedProxy
|
||||
(*AcknowledgeSnapshotRequest)(nil), // 16: proxy_pool.controlplane.v1.AcknowledgeSnapshotRequest
|
||||
(*OutcomeBatch)(nil), // 17: proxy_pool.controlplane.v1.OutcomeBatch
|
||||
(*ProxyOutcome)(nil), // 18: proxy_pool.controlplane.v1.ProxyOutcome
|
||||
(*ReportOutcomesResponse)(nil), // 19: proxy_pool.controlplane.v1.ReportOutcomesResponse
|
||||
(*ReportRuntimeRequest)(nil), // 20: proxy_pool.controlplane.v1.ReportRuntimeRequest
|
||||
(*ProxyRuntime)(nil), // 21: proxy_pool.controlplane.v1.ProxyRuntime
|
||||
(*ReportRuntimeResponse)(nil), // 22: proxy_pool.controlplane.v1.ReportRuntimeResponse
|
||||
(*StreamCheckTasksRequest)(nil), // 23: proxy_pool.controlplane.v1.StreamCheckTasksRequest
|
||||
(*CheckTask)(nil), // 24: proxy_pool.controlplane.v1.CheckTask
|
||||
(*ObservationBatch)(nil), // 25: proxy_pool.controlplane.v1.ObservationBatch
|
||||
(*HealthObservation)(nil), // 26: proxy_pool.controlplane.v1.HealthObservation
|
||||
(*ReportObservationsResponse)(nil), // 27: proxy_pool.controlplane.v1.ReportObservationsResponse
|
||||
nil, // 28: proxy_pool.controlplane.v1.RegisterWorkerRequest.LabelsEntry
|
||||
nil, // 29: proxy_pool.controlplane.v1.RoutingRule.HeadersEntry
|
||||
nil, // 30: proxy_pool.controlplane.v1.RoutingStrategy.WeightsEntry
|
||||
nil, // 31: proxy_pool.controlplane.v1.OwnedProxy.TagsEntry
|
||||
(*durationpb.Duration)(nil), // 32: google.protobuf.Duration
|
||||
(*timestamppb.Timestamp)(nil), // 33: google.protobuf.Timestamp
|
||||
(*emptypb.Empty)(nil), // 34: google.protobuf.Empty
|
||||
(ProxyProtocol)(0), // 2: proxy_pool.controlplane.v1.ProxyProtocol
|
||||
(OutcomeStage)(0), // 3: proxy_pool.controlplane.v1.OutcomeStage
|
||||
(CheckLevel)(0), // 4: proxy_pool.controlplane.v1.CheckLevel
|
||||
(*RegisterWorkerRequest)(nil), // 5: proxy_pool.controlplane.v1.RegisterWorkerRequest
|
||||
(*RegisterWorkerResponse)(nil), // 6: proxy_pool.controlplane.v1.RegisterWorkerResponse
|
||||
(*WatchSnapshotsRequest)(nil), // 7: proxy_pool.controlplane.v1.WatchSnapshotsRequest
|
||||
(*SnapshotEnvelope)(nil), // 8: proxy_pool.controlplane.v1.SnapshotEnvelope
|
||||
(*WorkerSnapshot)(nil), // 9: proxy_pool.controlplane.v1.WorkerSnapshot
|
||||
(*SnapshotCredential)(nil), // 10: proxy_pool.controlplane.v1.SnapshotCredential
|
||||
(*SnapshotDelta)(nil), // 11: proxy_pool.controlplane.v1.SnapshotDelta
|
||||
(*RoutingRule)(nil), // 12: proxy_pool.controlplane.v1.RoutingRule
|
||||
(*RoutingStrategy)(nil), // 13: proxy_pool.controlplane.v1.RoutingStrategy
|
||||
(*OwnedProxy)(nil), // 14: proxy_pool.controlplane.v1.OwnedProxy
|
||||
(*AcknowledgeSnapshotRequest)(nil), // 15: proxy_pool.controlplane.v1.AcknowledgeSnapshotRequest
|
||||
(*OutcomeBatch)(nil), // 16: proxy_pool.controlplane.v1.OutcomeBatch
|
||||
(*ProxyOutcome)(nil), // 17: proxy_pool.controlplane.v1.ProxyOutcome
|
||||
(*ReportOutcomesResponse)(nil), // 18: proxy_pool.controlplane.v1.ReportOutcomesResponse
|
||||
(*ReportRuntimeRequest)(nil), // 19: proxy_pool.controlplane.v1.ReportRuntimeRequest
|
||||
(*ProxyRuntime)(nil), // 20: proxy_pool.controlplane.v1.ProxyRuntime
|
||||
(*ReportRuntimeResponse)(nil), // 21: proxy_pool.controlplane.v1.ReportRuntimeResponse
|
||||
(*StreamCheckTasksRequest)(nil), // 22: proxy_pool.controlplane.v1.StreamCheckTasksRequest
|
||||
(*CheckTask)(nil), // 23: proxy_pool.controlplane.v1.CheckTask
|
||||
(*ObservationBatch)(nil), // 24: proxy_pool.controlplane.v1.ObservationBatch
|
||||
(*HealthObservation)(nil), // 25: proxy_pool.controlplane.v1.HealthObservation
|
||||
(*ReportObservationsResponse)(nil), // 26: proxy_pool.controlplane.v1.ReportObservationsResponse
|
||||
nil, // 27: proxy_pool.controlplane.v1.RegisterWorkerRequest.LabelsEntry
|
||||
nil, // 28: proxy_pool.controlplane.v1.RoutingRule.HeadersEntry
|
||||
nil, // 29: proxy_pool.controlplane.v1.RoutingStrategy.WeightsEntry
|
||||
nil, // 30: proxy_pool.controlplane.v1.OwnedProxy.TagsEntry
|
||||
(*durationpb.Duration)(nil), // 31: google.protobuf.Duration
|
||||
(*timestamppb.Timestamp)(nil), // 32: google.protobuf.Timestamp
|
||||
(*emptypb.Empty)(nil), // 33: google.protobuf.Empty
|
||||
}
|
||||
var file_controlplane_v1_controlplane_proto_depIdxs = []int32{
|
||||
28, // 0: proxy_pool.controlplane.v1.RegisterWorkerRequest.labels:type_name -> proxy_pool.controlplane.v1.RegisterWorkerRequest.LabelsEntry
|
||||
32, // 1: proxy_pool.controlplane.v1.RegisterWorkerResponse.heartbeat_interval:type_name -> google.protobuf.Duration
|
||||
32, // 2: proxy_pool.controlplane.v1.RegisterWorkerResponse.max_stale_age:type_name -> google.protobuf.Duration
|
||||
10, // 3: proxy_pool.controlplane.v1.SnapshotEnvelope.full:type_name -> proxy_pool.controlplane.v1.WorkerSnapshot
|
||||
12, // 4: proxy_pool.controlplane.v1.SnapshotEnvelope.delta:type_name -> proxy_pool.controlplane.v1.SnapshotDelta
|
||||
33, // 5: proxy_pool.controlplane.v1.WorkerSnapshot.generated_at:type_name -> google.protobuf.Timestamp
|
||||
33, // 6: proxy_pool.controlplane.v1.WorkerSnapshot.valid_until:type_name -> google.protobuf.Timestamp
|
||||
13, // 7: proxy_pool.controlplane.v1.WorkerSnapshot.routing:type_name -> proxy_pool.controlplane.v1.RoutingRule
|
||||
15, // 8: proxy_pool.controlplane.v1.WorkerSnapshot.proxies:type_name -> proxy_pool.controlplane.v1.OwnedProxy
|
||||
11, // 9: proxy_pool.controlplane.v1.WorkerSnapshot.credentials:type_name -> proxy_pool.controlplane.v1.SnapshotCredential
|
||||
33, // 10: proxy_pool.controlplane.v1.SnapshotDelta.generated_at:type_name -> google.protobuf.Timestamp
|
||||
13, // 11: proxy_pool.controlplane.v1.SnapshotDelta.upserted_routing:type_name -> proxy_pool.controlplane.v1.RoutingRule
|
||||
15, // 12: proxy_pool.controlplane.v1.SnapshotDelta.upserted_proxies:type_name -> proxy_pool.controlplane.v1.OwnedProxy
|
||||
29, // 13: proxy_pool.controlplane.v1.RoutingRule.headers:type_name -> proxy_pool.controlplane.v1.RoutingRule.HeadersEntry
|
||||
14, // 14: proxy_pool.controlplane.v1.RoutingRule.strategy:type_name -> proxy_pool.controlplane.v1.RoutingStrategy
|
||||
27, // 0: proxy_pool.controlplane.v1.RegisterWorkerRequest.labels:type_name -> proxy_pool.controlplane.v1.RegisterWorkerRequest.LabelsEntry
|
||||
31, // 1: proxy_pool.controlplane.v1.RegisterWorkerResponse.heartbeat_interval:type_name -> google.protobuf.Duration
|
||||
31, // 2: proxy_pool.controlplane.v1.RegisterWorkerResponse.max_stale_age:type_name -> google.protobuf.Duration
|
||||
9, // 3: proxy_pool.controlplane.v1.SnapshotEnvelope.full:type_name -> proxy_pool.controlplane.v1.WorkerSnapshot
|
||||
11, // 4: proxy_pool.controlplane.v1.SnapshotEnvelope.delta:type_name -> proxy_pool.controlplane.v1.SnapshotDelta
|
||||
32, // 5: proxy_pool.controlplane.v1.WorkerSnapshot.generated_at:type_name -> google.protobuf.Timestamp
|
||||
32, // 6: proxy_pool.controlplane.v1.WorkerSnapshot.valid_until:type_name -> google.protobuf.Timestamp
|
||||
12, // 7: proxy_pool.controlplane.v1.WorkerSnapshot.routing:type_name -> proxy_pool.controlplane.v1.RoutingRule
|
||||
14, // 8: proxy_pool.controlplane.v1.WorkerSnapshot.proxies:type_name -> proxy_pool.controlplane.v1.OwnedProxy
|
||||
10, // 9: proxy_pool.controlplane.v1.WorkerSnapshot.credentials:type_name -> proxy_pool.controlplane.v1.SnapshotCredential
|
||||
32, // 10: proxy_pool.controlplane.v1.SnapshotDelta.generated_at:type_name -> google.protobuf.Timestamp
|
||||
12, // 11: proxy_pool.controlplane.v1.SnapshotDelta.upserted_routing:type_name -> proxy_pool.controlplane.v1.RoutingRule
|
||||
14, // 12: proxy_pool.controlplane.v1.SnapshotDelta.upserted_proxies:type_name -> proxy_pool.controlplane.v1.OwnedProxy
|
||||
28, // 13: proxy_pool.controlplane.v1.RoutingRule.headers:type_name -> proxy_pool.controlplane.v1.RoutingRule.HeadersEntry
|
||||
13, // 14: proxy_pool.controlplane.v1.RoutingRule.strategy:type_name -> proxy_pool.controlplane.v1.RoutingStrategy
|
||||
1, // 15: proxy_pool.controlplane.v1.RoutingRule.on_unavailable:type_name -> proxy_pool.controlplane.v1.UnavailableAction
|
||||
32, // 16: proxy_pool.controlplane.v1.RoutingRule.wait_timeout:type_name -> google.protobuf.Duration
|
||||
2, // 17: proxy_pool.controlplane.v1.RoutingRule.action:type_name -> proxy_pool.controlplane.v1.RoutingAction
|
||||
0, // 18: proxy_pool.controlplane.v1.RoutingStrategy.type:type_name -> proxy_pool.controlplane.v1.StrategyType
|
||||
30, // 19: proxy_pool.controlplane.v1.RoutingStrategy.weights:type_name -> proxy_pool.controlplane.v1.RoutingStrategy.WeightsEntry
|
||||
3, // 20: proxy_pool.controlplane.v1.OwnedProxy.protocol:type_name -> proxy_pool.controlplane.v1.ProxyProtocol
|
||||
33, // 21: proxy_pool.controlplane.v1.OwnedProxy.expires_at:type_name -> google.protobuf.Timestamp
|
||||
31, // 22: proxy_pool.controlplane.v1.OwnedProxy.tags:type_name -> proxy_pool.controlplane.v1.OwnedProxy.TagsEntry
|
||||
33, // 23: proxy_pool.controlplane.v1.OwnedProxy.usable_until:type_name -> google.protobuf.Timestamp
|
||||
18, // 24: proxy_pool.controlplane.v1.OutcomeBatch.outcomes:type_name -> proxy_pool.controlplane.v1.ProxyOutcome
|
||||
4, // 25: proxy_pool.controlplane.v1.ProxyOutcome.stage:type_name -> proxy_pool.controlplane.v1.OutcomeStage
|
||||
32, // 26: proxy_pool.controlplane.v1.ProxyOutcome.latency:type_name -> google.protobuf.Duration
|
||||
33, // 27: proxy_pool.controlplane.v1.ProxyOutcome.observed_at:type_name -> google.protobuf.Timestamp
|
||||
21, // 28: proxy_pool.controlplane.v1.ReportRuntimeRequest.counters:type_name -> proxy_pool.controlplane.v1.ProxyRuntime
|
||||
33, // 29: proxy_pool.controlplane.v1.ReportRuntimeRequest.observed_at:type_name -> google.protobuf.Timestamp
|
||||
5, // 30: proxy_pool.controlplane.v1.StreamCheckTasksRequest.supported_levels:type_name -> proxy_pool.controlplane.v1.CheckLevel
|
||||
3, // 31: proxy_pool.controlplane.v1.CheckTask.protocol:type_name -> proxy_pool.controlplane.v1.ProxyProtocol
|
||||
5, // 32: proxy_pool.controlplane.v1.CheckTask.level:type_name -> proxy_pool.controlplane.v1.CheckLevel
|
||||
32, // 33: proxy_pool.controlplane.v1.CheckTask.timeout:type_name -> google.protobuf.Duration
|
||||
33, // 34: proxy_pool.controlplane.v1.CheckTask.deadline:type_name -> google.protobuf.Timestamp
|
||||
26, // 35: proxy_pool.controlplane.v1.ObservationBatch.observations:type_name -> proxy_pool.controlplane.v1.HealthObservation
|
||||
5, // 36: proxy_pool.controlplane.v1.HealthObservation.level:type_name -> proxy_pool.controlplane.v1.CheckLevel
|
||||
32, // 37: proxy_pool.controlplane.v1.HealthObservation.latency:type_name -> google.protobuf.Duration
|
||||
33, // 38: proxy_pool.controlplane.v1.HealthObservation.observed_at:type_name -> google.protobuf.Timestamp
|
||||
6, // 39: proxy_pool.controlplane.v1.WorkerControlPlane.RegisterWorker:input_type -> proxy_pool.controlplane.v1.RegisterWorkerRequest
|
||||
8, // 40: proxy_pool.controlplane.v1.WorkerControlPlane.WatchSnapshots:input_type -> proxy_pool.controlplane.v1.WatchSnapshotsRequest
|
||||
16, // 41: proxy_pool.controlplane.v1.WorkerControlPlane.AcknowledgeSnapshot:input_type -> proxy_pool.controlplane.v1.AcknowledgeSnapshotRequest
|
||||
17, // 42: proxy_pool.controlplane.v1.WorkerControlPlane.ReportOutcomes:input_type -> proxy_pool.controlplane.v1.OutcomeBatch
|
||||
20, // 43: proxy_pool.controlplane.v1.WorkerControlPlane.ReportRuntime:input_type -> proxy_pool.controlplane.v1.ReportRuntimeRequest
|
||||
23, // 44: proxy_pool.controlplane.v1.CheckerControlPlane.StreamCheckTasks:input_type -> proxy_pool.controlplane.v1.StreamCheckTasksRequest
|
||||
25, // 45: proxy_pool.controlplane.v1.CheckerControlPlane.ReportObservations:input_type -> proxy_pool.controlplane.v1.ObservationBatch
|
||||
7, // 46: proxy_pool.controlplane.v1.WorkerControlPlane.RegisterWorker:output_type -> proxy_pool.controlplane.v1.RegisterWorkerResponse
|
||||
9, // 47: proxy_pool.controlplane.v1.WorkerControlPlane.WatchSnapshots:output_type -> proxy_pool.controlplane.v1.SnapshotEnvelope
|
||||
34, // 48: proxy_pool.controlplane.v1.WorkerControlPlane.AcknowledgeSnapshot:output_type -> google.protobuf.Empty
|
||||
19, // 49: proxy_pool.controlplane.v1.WorkerControlPlane.ReportOutcomes:output_type -> proxy_pool.controlplane.v1.ReportOutcomesResponse
|
||||
22, // 50: proxy_pool.controlplane.v1.WorkerControlPlane.ReportRuntime:output_type -> proxy_pool.controlplane.v1.ReportRuntimeResponse
|
||||
24, // 51: proxy_pool.controlplane.v1.CheckerControlPlane.StreamCheckTasks:output_type -> proxy_pool.controlplane.v1.CheckTask
|
||||
27, // 52: proxy_pool.controlplane.v1.CheckerControlPlane.ReportObservations:output_type -> proxy_pool.controlplane.v1.ReportObservationsResponse
|
||||
46, // [46:53] is the sub-list for method output_type
|
||||
39, // [39:46] is the sub-list for method input_type
|
||||
39, // [39:39] is the sub-list for extension type_name
|
||||
39, // [39:39] is the sub-list for extension extendee
|
||||
0, // [0:39] is the sub-list for field type_name
|
||||
31, // 16: proxy_pool.controlplane.v1.RoutingRule.wait_timeout:type_name -> google.protobuf.Duration
|
||||
0, // 17: proxy_pool.controlplane.v1.RoutingStrategy.type:type_name -> proxy_pool.controlplane.v1.StrategyType
|
||||
29, // 18: proxy_pool.controlplane.v1.RoutingStrategy.weights:type_name -> proxy_pool.controlplane.v1.RoutingStrategy.WeightsEntry
|
||||
2, // 19: proxy_pool.controlplane.v1.OwnedProxy.protocol:type_name -> proxy_pool.controlplane.v1.ProxyProtocol
|
||||
32, // 20: proxy_pool.controlplane.v1.OwnedProxy.expires_at:type_name -> google.protobuf.Timestamp
|
||||
30, // 21: proxy_pool.controlplane.v1.OwnedProxy.tags:type_name -> proxy_pool.controlplane.v1.OwnedProxy.TagsEntry
|
||||
32, // 22: proxy_pool.controlplane.v1.OwnedProxy.usable_until:type_name -> google.protobuf.Timestamp
|
||||
17, // 23: proxy_pool.controlplane.v1.OutcomeBatch.outcomes:type_name -> proxy_pool.controlplane.v1.ProxyOutcome
|
||||
3, // 24: proxy_pool.controlplane.v1.ProxyOutcome.stage:type_name -> proxy_pool.controlplane.v1.OutcomeStage
|
||||
31, // 25: proxy_pool.controlplane.v1.ProxyOutcome.latency:type_name -> google.protobuf.Duration
|
||||
32, // 26: proxy_pool.controlplane.v1.ProxyOutcome.observed_at:type_name -> google.protobuf.Timestamp
|
||||
20, // 27: proxy_pool.controlplane.v1.ReportRuntimeRequest.counters:type_name -> proxy_pool.controlplane.v1.ProxyRuntime
|
||||
32, // 28: proxy_pool.controlplane.v1.ReportRuntimeRequest.observed_at:type_name -> google.protobuf.Timestamp
|
||||
4, // 29: proxy_pool.controlplane.v1.StreamCheckTasksRequest.supported_levels:type_name -> proxy_pool.controlplane.v1.CheckLevel
|
||||
2, // 30: proxy_pool.controlplane.v1.CheckTask.protocol:type_name -> proxy_pool.controlplane.v1.ProxyProtocol
|
||||
4, // 31: proxy_pool.controlplane.v1.CheckTask.level:type_name -> proxy_pool.controlplane.v1.CheckLevel
|
||||
31, // 32: proxy_pool.controlplane.v1.CheckTask.timeout:type_name -> google.protobuf.Duration
|
||||
32, // 33: proxy_pool.controlplane.v1.CheckTask.deadline:type_name -> google.protobuf.Timestamp
|
||||
25, // 34: proxy_pool.controlplane.v1.ObservationBatch.observations:type_name -> proxy_pool.controlplane.v1.HealthObservation
|
||||
4, // 35: proxy_pool.controlplane.v1.HealthObservation.level:type_name -> proxy_pool.controlplane.v1.CheckLevel
|
||||
31, // 36: proxy_pool.controlplane.v1.HealthObservation.latency:type_name -> google.protobuf.Duration
|
||||
32, // 37: proxy_pool.controlplane.v1.HealthObservation.observed_at:type_name -> google.protobuf.Timestamp
|
||||
5, // 38: proxy_pool.controlplane.v1.WorkerControlPlane.RegisterWorker:input_type -> proxy_pool.controlplane.v1.RegisterWorkerRequest
|
||||
7, // 39: proxy_pool.controlplane.v1.WorkerControlPlane.WatchSnapshots:input_type -> proxy_pool.controlplane.v1.WatchSnapshotsRequest
|
||||
15, // 40: proxy_pool.controlplane.v1.WorkerControlPlane.AcknowledgeSnapshot:input_type -> proxy_pool.controlplane.v1.AcknowledgeSnapshotRequest
|
||||
16, // 41: proxy_pool.controlplane.v1.WorkerControlPlane.ReportOutcomes:input_type -> proxy_pool.controlplane.v1.OutcomeBatch
|
||||
19, // 42: proxy_pool.controlplane.v1.WorkerControlPlane.ReportRuntime:input_type -> proxy_pool.controlplane.v1.ReportRuntimeRequest
|
||||
22, // 43: proxy_pool.controlplane.v1.CheckerControlPlane.StreamCheckTasks:input_type -> proxy_pool.controlplane.v1.StreamCheckTasksRequest
|
||||
24, // 44: proxy_pool.controlplane.v1.CheckerControlPlane.ReportObservations:input_type -> proxy_pool.controlplane.v1.ObservationBatch
|
||||
6, // 45: proxy_pool.controlplane.v1.WorkerControlPlane.RegisterWorker:output_type -> proxy_pool.controlplane.v1.RegisterWorkerResponse
|
||||
8, // 46: proxy_pool.controlplane.v1.WorkerControlPlane.WatchSnapshots:output_type -> proxy_pool.controlplane.v1.SnapshotEnvelope
|
||||
33, // 47: proxy_pool.controlplane.v1.WorkerControlPlane.AcknowledgeSnapshot:output_type -> google.protobuf.Empty
|
||||
18, // 48: proxy_pool.controlplane.v1.WorkerControlPlane.ReportOutcomes:output_type -> proxy_pool.controlplane.v1.ReportOutcomesResponse
|
||||
21, // 49: proxy_pool.controlplane.v1.WorkerControlPlane.ReportRuntime:output_type -> proxy_pool.controlplane.v1.ReportRuntimeResponse
|
||||
23, // 50: proxy_pool.controlplane.v1.CheckerControlPlane.StreamCheckTasks:output_type -> proxy_pool.controlplane.v1.CheckTask
|
||||
26, // 51: proxy_pool.controlplane.v1.CheckerControlPlane.ReportObservations:output_type -> proxy_pool.controlplane.v1.ReportObservationsResponse
|
||||
45, // [45:52] is the sub-list for method output_type
|
||||
38, // [38:45] is the sub-list for method input_type
|
||||
38, // [38:38] is the sub-list for extension type_name
|
||||
38, // [38:38] is the sub-list for extension extendee
|
||||
0, // [0:38] is the sub-list for field type_name
|
||||
}
|
||||
|
||||
func init() { file_controlplane_v1_controlplane_proto_init() }
|
||||
@ -2617,7 +2549,7 @@ func file_controlplane_v1_controlplane_proto_init() {
|
||||
File: protoimpl.DescBuilder{
|
||||
GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
|
||||
RawDescriptor: unsafe.Slice(unsafe.StringData(file_controlplane_v1_controlplane_proto_rawDesc), len(file_controlplane_v1_controlplane_proto_rawDesc)),
|
||||
NumEnums: 6,
|
||||
NumEnums: 5,
|
||||
NumMessages: 26,
|
||||
NumExtensions: 0,
|
||||
NumServices: 2,
|
||||
|
||||
@ -65,10 +65,6 @@ func TestOperationsPrioritizeCanceledContext(t *testing.T) {
|
||||
_, operationErr := store.SwitchRouting(ctx, adminstate.SwitchRoutingCommand{})
|
||||
return operationErr
|
||||
}},
|
||||
{name: "disable routing", run: func() error {
|
||||
_, operationErr := store.DisableRouting(ctx, adminstate.DisableRoutingCommand{})
|
||||
return operationErr
|
||||
}},
|
||||
{name: "snapshot", run: func() error {
|
||||
_, operationErr := store.Snapshot(ctx)
|
||||
return operationErr
|
||||
@ -123,10 +119,6 @@ func TestInvalidOperationsDoNotBeginTransaction(t *testing.T) {
|
||||
_, operationErr := store.SwitchRouting(context.Background(), adminstate.SwitchRoutingCommand{})
|
||||
return operationErr
|
||||
}},
|
||||
{name: "disable routing", run: func() error {
|
||||
_, operationErr := store.DisableRouting(context.Background(), adminstate.DisableRoutingCommand{})
|
||||
return operationErr
|
||||
}},
|
||||
{name: "read audit", run: func() error {
|
||||
_, operationErr := store.ReadAudit(context.Background(), adminstate.AuditQuery{})
|
||||
return operationErr
|
||||
|
||||
@ -1,15 +0,0 @@
|
||||
BEGIN;
|
||||
|
||||
ALTER TABLE admin_audit_log
|
||||
DROP CONSTRAINT IF EXISTS admin_audit_log_action_check;
|
||||
|
||||
ALTER TABLE admin_audit_log
|
||||
ADD CONSTRAINT admin_audit_log_action_check
|
||||
CHECK (action IN (
|
||||
'commit_config',
|
||||
'set_upstream_enabled',
|
||||
'switch_routing',
|
||||
'disable_routing'
|
||||
));
|
||||
|
||||
COMMIT;
|
||||
@ -27,18 +27,15 @@ func TestApplyMigrationsValidatesContextAndPool(t *testing.T) {
|
||||
func TestMigrationsAreOrderedTransactionalAndImmutable(t *testing.T) {
|
||||
t.Parallel()
|
||||
migrations := Migrations()
|
||||
if len(migrations) != 2 {
|
||||
t.Fatalf("len(Migrations()) = %d, want 2", len(migrations))
|
||||
if len(migrations) != 1 {
|
||||
t.Fatalf("len(Migrations()) = %d, want 1", len(migrations))
|
||||
}
|
||||
if migrations[0].Version != 1 || migrations[0].Name != "admin_state" ||
|
||||
migrations[1].Version != 2 || migrations[1].Name != "disable_routing" {
|
||||
t.Fatalf("migration metadata = %+v", migrations)
|
||||
if migrations[0].Version != 1 || migrations[0].Name != "admin_state" {
|
||||
t.Fatalf("migration metadata = %+v", migrations[0])
|
||||
}
|
||||
for _, migration := range migrations {
|
||||
normalized := strings.TrimSpace(migration.SQL)
|
||||
normalized := strings.TrimSpace(migrations[0].SQL)
|
||||
if !strings.HasPrefix(normalized, "BEGIN;") || !strings.HasSuffix(normalized, "COMMIT;") {
|
||||
t.Fatalf("migration %d is not transaction wrapped: %q", migration.Version, normalized)
|
||||
}
|
||||
t.Fatalf("migration is not transaction wrapped: %q", normalized)
|
||||
}
|
||||
for index := 1; index < len(migrations); index++ {
|
||||
if migrations[index-1].Version >= migrations[index].Version {
|
||||
@ -52,25 +49,6 @@ func TestMigrationsAreOrderedTransactionalAndImmutable(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestRoutingDisableMigrationExtendsOnlyAuditActionConstraint(t *testing.T) {
|
||||
t.Parallel()
|
||||
lower := strings.ToLower(Migrations()[1].SQL)
|
||||
for _, required := range []string{
|
||||
"alter table admin_audit_log",
|
||||
"drop constraint if exists admin_audit_log_action_check",
|
||||
"disable_routing",
|
||||
} {
|
||||
if !strings.Contains(lower, required) {
|
||||
t.Errorf("routing disable migration missing %q", required)
|
||||
}
|
||||
}
|
||||
for _, forbidden := range []string{"create table", "proxy", "credential", "extraction", "ownership", "idempotency"} {
|
||||
if strings.Contains(lower, forbidden) {
|
||||
t.Errorf("routing disable migration contains forbidden identifier %q", forbidden)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestAdminSchemaContainsOnlyManagementTables(t *testing.T) {
|
||||
t.Parallel()
|
||||
sql := Migrations()[0].SQL
|
||||
|
||||
@ -301,96 +301,6 @@ func (adapter *adapter) SwitchRouting(
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (adapter *adapter) DisableRouting(
|
||||
ctx context.Context,
|
||||
command adminstate.DisableRoutingCommand,
|
||||
) (adminstate.MutationResult, error) {
|
||||
result := adminstate.MutationResult{RequestID: command.RequestID}
|
||||
if err := contextError(ctx); err != nil {
|
||||
return result, err
|
||||
}
|
||||
if command.Validate() != nil || !adapter.valid() {
|
||||
return result, adminstate.ErrInvalidCommand
|
||||
}
|
||||
|
||||
tx, currentRevision, err := adapter.beginMutation(ctx, "disable routing")
|
||||
if err != nil {
|
||||
return result, err
|
||||
}
|
||||
defer rollback(tx)
|
||||
|
||||
var enabled bool
|
||||
var current string
|
||||
if err := tx.QueryRow(ctx, `
|
||||
SELECT enabled, current_upstream
|
||||
FROM routing_admin_state
|
||||
WHERE name = $1
|
||||
FOR UPDATE`, command.Name).Scan(&enabled, ¤t); err != nil {
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return result, adminstate.ErrNotFound
|
||||
}
|
||||
return result, databaseError(ctx, "read routing state", err)
|
||||
}
|
||||
if current != command.ExpectedCurrent {
|
||||
return result, adminstate.ErrConflict
|
||||
}
|
||||
if !enabled {
|
||||
if err := insertAudit(ctx, tx, command.RequestID, command.Actor,
|
||||
adminstate.ActionDisableRoute, "routing", command.Name, false,
|
||||
currentRevision, command.Reason, command.OccurredAt, "audit routing disable no-op"); err != nil {
|
||||
return result, err
|
||||
}
|
||||
if err := commit(ctx, tx, "commit routing disable no-op"); err != nil {
|
||||
return result, err
|
||||
}
|
||||
result.Revision = uint64(currentRevision)
|
||||
return result, nil
|
||||
}
|
||||
|
||||
nextRevision, err := allocateRevision(ctx, tx, currentRevision, "routing", command.OccurredAt)
|
||||
if err != nil {
|
||||
return result, err
|
||||
}
|
||||
commandTag, err := tx.Exec(ctx, `
|
||||
UPDATE routing_admin_state
|
||||
SET enabled = FALSE, revision = $1, updated_at = $2
|
||||
WHERE name = $3 AND enabled = TRUE AND current_upstream = $4`,
|
||||
nextRevision, utc(command.OccurredAt), command.Name, command.ExpectedCurrent)
|
||||
if err != nil {
|
||||
return result, databaseError(ctx, "disable routing state", err)
|
||||
}
|
||||
if commandTag.RowsAffected() != 1 {
|
||||
return result, adminstate.ErrConflict
|
||||
}
|
||||
if err := insertAudit(ctx, tx, command.RequestID, command.Actor,
|
||||
adminstate.ActionDisableRoute, "routing", command.Name, true,
|
||||
nextRevision, command.Reason, command.OccurredAt, "audit routing disable"); err != nil {
|
||||
return result, err
|
||||
}
|
||||
payload, err := encodePayload(map[string]any{
|
||||
"current": command.ExpectedCurrent,
|
||||
"enabled": false,
|
||||
"name": command.Name,
|
||||
"reason": command.Reason,
|
||||
"revision": nextRevision,
|
||||
})
|
||||
if err != nil {
|
||||
return result, unavailable("encode routing disable event")
|
||||
}
|
||||
if err := insertOutbox(ctx, tx, nextRevision, "routing.disabled", "routing",
|
||||
command.Name, payload, command.OccurredAt, "insert routing disable event"); err != nil {
|
||||
return result, err
|
||||
}
|
||||
if err := commit(ctx, tx, "commit routing disable"); err != nil {
|
||||
return result, err
|
||||
}
|
||||
return adminstate.MutationResult{
|
||||
RequestID: command.RequestID,
|
||||
Changed: true,
|
||||
Revision: uint64(nextRevision),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (adapter *adapter) beginMutation(ctx context.Context, operation string) (pgx.Tx, int64, error) {
|
||||
tx, err := adapter.begin(ctx, pgx.TxOptions{AccessMode: pgx.ReadWrite}, operation)
|
||||
if err != nil {
|
||||
|
||||
@ -62,41 +62,6 @@ func TestMutationRollsBackWhenOutboxInsertFails(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestDisableRoutingRollsBackWhenAuditOrOutboxInsertFails(t *testing.T) {
|
||||
for _, target := range []struct {
|
||||
name string
|
||||
table string
|
||||
}{
|
||||
{name: "audit", table: "admin_audit_log"},
|
||||
{name: "outbox", table: "admin_outbox"},
|
||||
} {
|
||||
t.Run(target.name, func(t *testing.T) {
|
||||
fixture := newPostgresTestFixture(t)
|
||||
now := integrationNow()
|
||||
commitIntegrationConfig(t, fixture.Store, now)
|
||||
installRejectInsertTrigger(t, fixture, target.table, "reject_routing_disable")
|
||||
|
||||
_, err := fixture.Store.DisableRouting(context.Background(), adminstate.DisableRoutingCommand{
|
||||
RequestID: "req-routing-disable-failure", Actor: integrationActor(), OccurredAt: now.Add(time.Second),
|
||||
Name: "checkout", ExpectedCurrent: "provider-a", Reason: "terminal empty results",
|
||||
})
|
||||
if !errors.Is(err, adminstate.ErrUnavailable) {
|
||||
t.Fatalf("DisableRouting(%s failure) error = %v, want ErrUnavailable", target.name, err)
|
||||
}
|
||||
assertFailedRoutingDisableLeftBaseline(t, fixture)
|
||||
|
||||
dropRejectInsertTrigger(t, fixture, target.table, "reject_routing_disable")
|
||||
result, err := fixture.Store.DisableRouting(context.Background(), adminstate.DisableRoutingCommand{
|
||||
RequestID: "req-routing-disable-after-failure", Actor: integrationActor(), OccurredAt: now.Add(2 * time.Second),
|
||||
Name: "checkout", ExpectedCurrent: "provider-a", Reason: "terminal empty results",
|
||||
})
|
||||
if err != nil || !result.Changed || result.Revision != 2 {
|
||||
t.Fatalf("DisableRouting(after rollback) = %+v, %v", result, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func assertFailedMutationLeftBaseline(t *testing.T, fixture postgresTestFixture) {
|
||||
t.Helper()
|
||||
snapshot, err := fixture.Store.Snapshot(context.Background())
|
||||
@ -118,28 +83,6 @@ func assertFailedMutationLeftBaseline(t *testing.T, fixture postgresTestFixture)
|
||||
}
|
||||
}
|
||||
|
||||
func assertFailedRoutingDisableLeftBaseline(t *testing.T, fixture postgresTestFixture) {
|
||||
t.Helper()
|
||||
snapshot, err := fixture.Store.Snapshot(context.Background())
|
||||
if err != nil || snapshot.Revision != 1 || len(snapshot.Routings) != 1 || !snapshot.Routings[0].Enabled ||
|
||||
snapshot.Routings[0].CurrentUpstream != "provider-a" {
|
||||
t.Fatalf("Snapshot(after failed DisableRouting) = %+v, %v", snapshot, err)
|
||||
}
|
||||
audits, err := fixture.Store.ReadAudit(context.Background(), adminstate.AuditQuery{Limit: 10})
|
||||
if err != nil || len(audits) != 1 {
|
||||
t.Fatalf("ReadAudit(after failed DisableRouting) = %+v, %v", audits, err)
|
||||
}
|
||||
for table, want := range map[string]int{"control_revisions": 1, "admin_outbox": 1} {
|
||||
var count int
|
||||
if err := fixture.Pool.QueryRow(context.Background(), "SELECT COUNT(*) FROM "+table).Scan(&count); err != nil {
|
||||
t.Fatalf("count %s: %v", table, err)
|
||||
}
|
||||
if count != want {
|
||||
t.Fatalf("%s row count = %d, want %d", table, count, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func installRejectInsertTrigger(t *testing.T, fixture postgresTestFixture, table, trigger string) {
|
||||
t.Helper()
|
||||
function := trigger + "_fn"
|
||||
|
||||
@ -224,53 +224,6 @@ func TestRedisRejectsUnhealthyDrainCandidateAfterRecovery(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestRedisDisabledUpstreamDrainFencesPolicyRevision(t *testing.T) {
|
||||
fixture := newRedisTestFixture(t)
|
||||
now := redisTestNow()
|
||||
seedRedisAvailable(t, fixture.Adapter, "provider-a", now, now.Add(time.Second), time.Minute,
|
||||
testProxy("proxy-a", "192.0.2.10"))
|
||||
assignment, err := fixture.Adapter.Assign(context.Background(), now.Add(2*time.Second), "proxy-a", "worker-a", time.Minute)
|
||||
if err != nil {
|
||||
t.Fatalf("Assign() error = %v", err)
|
||||
}
|
||||
disabled := activitypool.UpstreamDrainPolicy{UpstreamID: "provider-a", Revision: 31}
|
||||
if err := fixture.Adapter.ReplaceUpstreamDrainPolicies(context.Background(), []activitypool.UpstreamDrainPolicy{disabled}); err != nil {
|
||||
t.Fatalf("ReplaceUpstreamDrainPolicies(disabled) error = %v", err)
|
||||
}
|
||||
candidates, err := fixture.Adapter.ListDisabledUpstreamDrainCandidates(context.Background(), now.Add(3*time.Second), disabled, 1)
|
||||
if err != nil || len(candidates) != 1 || candidates[0] != (activitypool.DisabledUpstreamDrainCandidate{
|
||||
UpstreamID: "provider-a", PolicyRevision: 31, ProxyID: "proxy-a", WorkerID: "worker-a", AssignmentEpoch: assignment.Epoch,
|
||||
}) {
|
||||
t.Fatalf("ListDisabledUpstreamDrainCandidates() = %+v, %v", candidates, err)
|
||||
}
|
||||
if err := fixture.Adapter.ReplaceUpstreamDrainPolicies(context.Background(), []activitypool.UpstreamDrainPolicy{{
|
||||
UpstreamID: "provider-a", Revision: 32, Enabled: true,
|
||||
}}); err != nil {
|
||||
t.Fatalf("ReplaceUpstreamDrainPolicies(enabled) error = %v", err)
|
||||
}
|
||||
if started, err := fixture.Adapter.BeginDisabledUpstreamDrain(context.Background(), now.Add(3*time.Second), candidates[0]); err != nil || started {
|
||||
t.Fatalf("BeginDisabledUpstreamDrain(stale policy) = %t, %v", started, err)
|
||||
}
|
||||
if current, ok, err := fixture.Adapter.Get(context.Background(), "proxy-a"); err != nil || !ok || current.Draining {
|
||||
t.Fatalf("Get(after stale candidate) = %+v, %t, %v", current, ok, err)
|
||||
}
|
||||
|
||||
disabled.Revision = 33
|
||||
if err := fixture.Adapter.ReplaceUpstreamDrainPolicies(context.Background(), []activitypool.UpstreamDrainPolicy{disabled}); err != nil {
|
||||
t.Fatalf("ReplaceUpstreamDrainPolicies(disabled again) error = %v", err)
|
||||
}
|
||||
candidates, err = fixture.Adapter.ListDisabledUpstreamDrainCandidates(context.Background(), now.Add(4*time.Second), disabled, 1)
|
||||
if err != nil || len(candidates) != 1 {
|
||||
t.Fatalf("ListDisabledUpstreamDrainCandidates(disabled again) = %+v, %v", candidates, err)
|
||||
}
|
||||
if started, err := fixture.Adapter.BeginDisabledUpstreamDrain(context.Background(), now.Add(4*time.Second), candidates[0]); err != nil || !started {
|
||||
t.Fatalf("BeginDisabledUpstreamDrain() = %t, %v", started, err)
|
||||
}
|
||||
if current, ok, err := fixture.Adapter.Get(context.Background(), "proxy-a"); err != nil || !ok || !current.Draining {
|
||||
t.Fatalf("Get(after drain) = %+v, %t, %v", current, ok, err)
|
||||
}
|
||||
}
|
||||
|
||||
func redisGlobalObservation(taskID string, success bool, observedAt time.Time) healthDomain.Observation {
|
||||
observation := healthDomain.Observation{
|
||||
TaskID: taskID, ProxyID: "proxy-a", Level: healthDomain.LevelBasic,
|
||||
|
||||
@ -18,7 +18,6 @@ type keyspace struct {
|
||||
owners string
|
||||
ownerExpiry string
|
||||
drainTickets string
|
||||
upstreamDrainPolicies string
|
||||
epoch string
|
||||
inventory string
|
||||
stateInventory string
|
||||
@ -52,7 +51,6 @@ func newKeyspace(namespace string) keyspace {
|
||||
owners: prefix + ":owners",
|
||||
ownerExpiry: prefix + ":owner-expiry",
|
||||
drainTickets: prefix + ":drain-tickets",
|
||||
upstreamDrainPolicies: prefix + ":upstream-drain-policies",
|
||||
epoch: prefix + ":epoch",
|
||||
inventory: prefix + ":inventory",
|
||||
stateInventory: prefix + ":state-inventory",
|
||||
|
||||
@ -3,7 +3,6 @@ package redisactivity
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"proxy-pool/internal/domain/activitypool"
|
||||
@ -13,7 +12,6 @@ const (
|
||||
maintenanceInventory = "inventory"
|
||||
maintenanceSweep = "sweep"
|
||||
maintenanceUnhealthy = "unhealthy"
|
||||
maintenanceDisabledUpstream = "disabled_upstream"
|
||||
|
||||
maximumUnhealthySweepScan = 1024
|
||||
)
|
||||
@ -21,7 +19,6 @@ const (
|
||||
var (
|
||||
_ activitypool.InventoryReader = (*Adapter)(nil)
|
||||
_ activitypool.Maintainer = (*Adapter)(nil)
|
||||
_ activitypool.DisabledUpstreamDrainLister = (*Adapter)(nil)
|
||||
)
|
||||
|
||||
func (a *Adapter) Inventory(ctx context.Context, upstreamID string, now time.Time) (activitypool.Inventory, error) {
|
||||
@ -127,52 +124,6 @@ func (a *Adapter) SweepUnhealthy(
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func (a *Adapter) ListDisabledUpstreamDrainCandidates(
|
||||
ctx context.Context,
|
||||
now time.Time,
|
||||
policy activitypool.UpstreamDrainPolicy,
|
||||
limit int,
|
||||
) ([]activitypool.DisabledUpstreamDrainCandidate, error) {
|
||||
if ctx == nil {
|
||||
return nil, activitypool.ErrInvalidMaintenance
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if a == nil || now.IsZero() || policy.UpstreamID == "" || policy.Revision == 0 || policy.Enabled || limit <= 0 {
|
||||
return nil, activitypool.ErrInvalidMaintenance
|
||||
}
|
||||
if limit > maximumUnhealthySweepScan/4 {
|
||||
limit = maximumUnhealthySweepScan / 4
|
||||
}
|
||||
reply, err := a.runMaintenance(ctx, maintenanceDisabledUpstream, now, limit, policy.UpstreamID, "")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if reply.Status == scriptStale {
|
||||
return nil, nil
|
||||
}
|
||||
if reply.Status == scriptInvalid {
|
||||
return nil, activitypool.ErrInvalidMaintenance
|
||||
}
|
||||
if reply.Status != scriptOK || len(reply.DisabledDrainCandidates) > limit {
|
||||
return nil, invalidScriptReply("unexpected disabled upstream sweep reply")
|
||||
}
|
||||
wantRevision := strconv.FormatUint(policy.Revision, 10)
|
||||
result := make([]activitypool.DisabledUpstreamDrainCandidate, 0, len(reply.DisabledDrainCandidates))
|
||||
for _, candidate := range reply.DisabledDrainCandidates {
|
||||
if candidate.UpstreamID != policy.UpstreamID || candidate.PolicyRevision != wantRevision ||
|
||||
candidate.ProxyID == "" || candidate.WorkerID == "" || candidate.AssignmentEpoch == 0 {
|
||||
return nil, invalidScriptReply("disabled upstream sweep reply contained an invalid drain candidate")
|
||||
}
|
||||
result = append(result, activitypool.DisabledUpstreamDrainCandidate{
|
||||
UpstreamID: candidate.UpstreamID, PolicyRevision: policy.Revision, ProxyID: candidate.ProxyID,
|
||||
WorkerID: candidate.WorkerID, AssignmentEpoch: candidate.AssignmentEpoch,
|
||||
})
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func (a *Adapter) runMaintenance(
|
||||
ctx context.Context,
|
||||
operation string,
|
||||
@ -190,7 +141,6 @@ func (a *Adapter) runMaintenance(
|
||||
a.keys.inventory, a.keys.stateInventory, a.keys.owners, a.keys.ownerExpiry,
|
||||
a.keys.operation(operationID), a.keys.healthDue, a.keys.healthQueued, a.keys.healthLeases,
|
||||
a.keys.healthTasks, a.keys.healthTaskExpiry, a.keys.healthRefTask, a.keys.healthUnhealthy,
|
||||
a.keys.owned(upstreamID), a.keys.upstreamDrainPolicies,
|
||||
}, operation, now.UnixMilli(), limit, upstreamID, operationTTLMillis(a.options.OperationTTL), policies)
|
||||
if err != nil {
|
||||
return maintenanceScriptReply{}, err
|
||||
|
||||
@ -6,7 +6,6 @@ import (
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"proxy-pool/internal/domain/activitypool"
|
||||
@ -18,7 +17,6 @@ const (
|
||||
ownershipRenew = "renew"
|
||||
ownershipBeginDrain = "begin_drain"
|
||||
ownershipBeginUnhealthyDrain = "begin_unhealthy_drain"
|
||||
ownershipBeginDisabledDrain = "begin_disabled_upstream_drain"
|
||||
ownershipAcknowledgeDrain = "acknowledge_drain"
|
||||
ownershipGet = "get"
|
||||
ownershipExpire = "expire"
|
||||
@ -27,7 +25,6 @@ const (
|
||||
var _ ownershipDomain.Repository = (*Adapter)(nil)
|
||||
var _ ownershipDomain.DrainTicketStore = (*Adapter)(nil)
|
||||
var _ activitypool.UnhealthyDrainStarter = (*Adapter)(nil)
|
||||
var _ activitypool.DisabledUpstreamDrainStarter = (*Adapter)(nil)
|
||||
|
||||
func (a *Adapter) Assign(
|
||||
ctx context.Context,
|
||||
@ -42,7 +39,7 @@ func (a *Adapter) Assign(
|
||||
if now.IsZero() || proxyID == "" || workerID == "" || ttl <= 0 {
|
||||
return ownershipDomain.Assignment{}, ownershipDomain.ErrInvalidOwnership
|
||||
}
|
||||
reply, err := a.runOwnership(ctx, ownershipAssign, true, now.UnixMilli(), proxyID, workerID, 0, durationMillis(ttl), 0, 0, "", "")
|
||||
reply, err := a.runOwnership(ctx, ownershipAssign, true, now.UnixMilli(), proxyID, workerID, 0, durationMillis(ttl), 0, 0)
|
||||
if err != nil {
|
||||
return ownershipDomain.Assignment{}, err
|
||||
}
|
||||
@ -74,7 +71,7 @@ func (a *Adapter) Renew(
|
||||
if now.IsZero() || proxyID == "" || workerID == "" || epoch == 0 || ttl <= 0 {
|
||||
return ownershipDomain.Assignment{}, ownershipDomain.ErrInvalidOwnership
|
||||
}
|
||||
reply, err := a.runOwnership(ctx, ownershipRenew, true, now.UnixMilli(), proxyID, workerID, epoch, durationMillis(ttl), 0, 0, "", "")
|
||||
reply, err := a.runOwnership(ctx, ownershipRenew, true, now.UnixMilli(), proxyID, workerID, epoch, durationMillis(ttl), 0, 0)
|
||||
if err != nil {
|
||||
return ownershipDomain.Assignment{}, err
|
||||
}
|
||||
@ -102,7 +99,7 @@ func (a *Adapter) BeginDrain(
|
||||
if proxyID == "" || workerID == "" || epoch == 0 {
|
||||
return ownershipDomain.Assignment{}, ownershipDomain.ErrInvalidOwnership
|
||||
}
|
||||
reply, err := a.runOwnership(ctx, ownershipBeginDrain, true, 0, proxyID, workerID, epoch, 0, 0, 0, "", "")
|
||||
reply, err := a.runOwnership(ctx, ownershipBeginDrain, true, 0, proxyID, workerID, epoch, 0, 0, 0)
|
||||
if err != nil {
|
||||
return ownershipDomain.Assignment{}, err
|
||||
}
|
||||
@ -131,7 +128,7 @@ func (a *Adapter) BeginUnhealthyDrain(
|
||||
}
|
||||
reply, err := a.runOwnership(
|
||||
ctx, ownershipBeginUnhealthyDrain, true, now.UTC().UnixMilli(), candidate.ProxyID, candidate.WorkerID,
|
||||
candidate.AssignmentEpoch, candidate.UnhealthySince.UTC().UnixMilli(), 0, 0, "", "",
|
||||
candidate.AssignmentEpoch, candidate.UnhealthySince.UTC().UnixMilli(), 0, 0,
|
||||
)
|
||||
if err != nil {
|
||||
return false, err
|
||||
@ -151,40 +148,6 @@ func (a *Adapter) BeginUnhealthyDrain(
|
||||
}
|
||||
}
|
||||
|
||||
func (a *Adapter) BeginDisabledUpstreamDrain(
|
||||
ctx context.Context,
|
||||
now time.Time,
|
||||
candidate activitypool.DisabledUpstreamDrainCandidate,
|
||||
) (bool, error) {
|
||||
if err := validateOwnershipCall(ctx, a); err != nil {
|
||||
return false, err
|
||||
}
|
||||
if now.IsZero() || candidate.UpstreamID == "" || candidate.PolicyRevision == 0 || candidate.ProxyID == "" ||
|
||||
candidate.WorkerID == "" || candidate.AssignmentEpoch == 0 {
|
||||
return false, activitypool.ErrInvalidMaintenance
|
||||
}
|
||||
reply, err := a.runOwnership(
|
||||
ctx, ownershipBeginDisabledDrain, true, now.UTC().UnixMilli(), candidate.ProxyID, candidate.WorkerID,
|
||||
candidate.AssignmentEpoch, 0, 0, 0, candidate.UpstreamID, strconv.FormatUint(candidate.PolicyRevision, 10),
|
||||
)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
switch reply.Status {
|
||||
case scriptOK:
|
||||
if _, err := decodeAssignmentReply(reply); err != nil {
|
||||
return false, err
|
||||
}
|
||||
return reply.Started, nil
|
||||
case scriptStale, scriptNotFound:
|
||||
return false, nil
|
||||
case scriptInvalid:
|
||||
return false, activitypool.ErrInvalidMaintenance
|
||||
default:
|
||||
return false, invalidScriptReply("unexpected disabled upstream ownership drain status")
|
||||
}
|
||||
}
|
||||
|
||||
func (a *Adapter) PendingDrains(ctx context.Context, workerID string, limit int) ([]ownershipDomain.DrainTicket, error) {
|
||||
if err := validateOwnershipCall(ctx, a); err != nil {
|
||||
return nil, err
|
||||
@ -279,7 +242,7 @@ func (a *Adapter) AcknowledgeDrain(
|
||||
if proxyID == "" || workerID == "" || epoch == 0 || active < 0 || reserved < 0 {
|
||||
return ownershipDomain.ErrInvalidOwnership
|
||||
}
|
||||
reply, err := a.runOwnership(ctx, ownershipAcknowledgeDrain, true, 0, proxyID, workerID, epoch, 0, active, reserved, "", "")
|
||||
reply, err := a.runOwnership(ctx, ownershipAcknowledgeDrain, true, 0, proxyID, workerID, epoch, 0, active, reserved)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@ -306,7 +269,7 @@ func (a *Adapter) Get(ctx context.Context, proxyID string) (ownershipDomain.Assi
|
||||
if proxyID == "" {
|
||||
return ownershipDomain.Assignment{}, false, ownershipDomain.ErrInvalidOwnership
|
||||
}
|
||||
reply, err := a.runOwnership(ctx, ownershipGet, false, 0, proxyID, "", 0, 0, 0, 0, "", "")
|
||||
reply, err := a.runOwnership(ctx, ownershipGet, false, 0, proxyID, "", 0, 0, 0, 0)
|
||||
if err != nil {
|
||||
return ownershipDomain.Assignment{}, false, err
|
||||
}
|
||||
@ -330,7 +293,7 @@ func (a *Adapter) Expire(ctx context.Context, now time.Time, limit int) ([]owner
|
||||
if now.IsZero() || limit <= 0 {
|
||||
return nil, ownershipDomain.ErrInvalidOwnership
|
||||
}
|
||||
reply, err := a.runOwnership(ctx, ownershipExpire, true, now.UnixMilli(), "", "", 0, int64(limit), 0, 0, "", "")
|
||||
reply, err := a.runOwnership(ctx, ownershipExpire, true, now.UnixMilli(), "", "", 0, int64(limit), 0, 0)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@ -368,8 +331,6 @@ func (a *Adapter) runOwnership(
|
||||
value int64,
|
||||
active int64,
|
||||
reserved int64,
|
||||
upstreamID string,
|
||||
policyRevision string,
|
||||
) (ownershipScriptReply, error) {
|
||||
operationKey := a.keys.epoch
|
||||
if mutating {
|
||||
@ -384,9 +345,9 @@ func (a *Adapter) runOwnership(
|
||||
a.keys.inventory, a.keys.stateInventory, a.keys.owners, a.keys.ownerExpiry,
|
||||
a.keys.epoch, operationKey, a.keys.healthDue, a.keys.healthQueued, a.keys.healthLeases,
|
||||
a.keys.healthTasks, a.keys.healthTaskExpiry, a.keys.healthRefTask, a.keys.healthUnhealthy,
|
||||
a.keys.drainTickets, a.keys.upstreamDrainPolicies,
|
||||
a.keys.drainTickets,
|
||||
}, operation, operationTTLMillis(a.options.OperationTTL), a.options.CleanupLimit,
|
||||
nowMS, proxyID, workerID, epoch, value, active, reserved, a.keys.workerOwned(workerID), a.keys.workerDraining(workerID), upstreamID, policyRevision)
|
||||
nowMS, proxyID, workerID, epoch, value, active, reserved, a.keys.workerOwned(workerID), a.keys.workerDraining(workerID))
|
||||
if err != nil {
|
||||
return ownershipScriptReply{}, err
|
||||
}
|
||||
|
||||
@ -88,10 +88,6 @@ type ownershipScriptReply struct {
|
||||
Started bool `json:"started"`
|
||||
}
|
||||
|
||||
type upstreamDrainPolicyScriptReply struct {
|
||||
Status scriptStatus `json:"status"`
|
||||
}
|
||||
|
||||
type drainTicketsScriptReply struct {
|
||||
Status scriptStatus `json:"status"`
|
||||
Tickets []string `json:"tickets"`
|
||||
@ -102,7 +98,6 @@ type maintenanceScriptReply struct {
|
||||
Count int `json:"count"`
|
||||
DeferredOwned int `json:"deferredOwned"`
|
||||
DrainCandidates []maintenanceDrainCandidateScriptRow `json:"drainCandidates"`
|
||||
DisabledDrainCandidates []disabledDrainCandidateScriptRow `json:"disabledDrainCandidates"`
|
||||
}
|
||||
|
||||
type maintenanceDrainCandidateScriptRow struct {
|
||||
@ -112,14 +107,6 @@ type maintenanceDrainCandidateScriptRow struct {
|
||||
UnhealthySinceMS int64 `json:"unhealthySinceMs"`
|
||||
}
|
||||
|
||||
type disabledDrainCandidateScriptRow struct {
|
||||
UpstreamID string `json:"upstreamId"`
|
||||
PolicyRevision string `json:"policyRevision"`
|
||||
ProxyID string `json:"proxyId"`
|
||||
WorkerID string `json:"workerId"`
|
||||
AssignmentEpoch uint64 `json:"assignmentEpoch"`
|
||||
}
|
||||
|
||||
type statusScriptReply struct {
|
||||
Status scriptStatus `json:"status"`
|
||||
Inventories []statusScriptInventory `json:"inventories"`
|
||||
@ -201,9 +188,6 @@ var capacitySource string
|
||||
//go:embed scripts/worker_snapshot.lua
|
||||
var workerSnapshotSource string
|
||||
|
||||
//go:embed scripts/upstream_drain_policy.lua
|
||||
var upstreamDrainPolicySource string
|
||||
|
||||
var (
|
||||
upsertScript = redis.NewScript(upsertSource)
|
||||
healthScript = redis.NewScript(healthSource)
|
||||
@ -219,7 +203,6 @@ var (
|
||||
runtimeScript = redis.NewScript(runtimeSource)
|
||||
capacityScript = redis.NewScript(capacitySource)
|
||||
workerSnapshotScript = redis.NewScript(workerSnapshotSource)
|
||||
upstreamDrainPolicyScript = redis.NewScript(upstreamDrainPolicySource)
|
||||
)
|
||||
|
||||
func runScript(ctx context.Context, client redis.Scripter, script *redis.Script, keys []string, args ...any) (any, error) {
|
||||
|
||||
@ -17,7 +17,6 @@ local health_task_expiry_key = KEYS[16]
|
||||
local health_ref_task_key = KEYS[17]
|
||||
local health_unhealthy_key = KEYS[18]
|
||||
local drain_tickets_key = KEYS[19]
|
||||
local upstream_drain_policies_key = KEYS[20]
|
||||
|
||||
local operation = ARGV[1]
|
||||
local operation_ttl_ms = tonumber(ARGV[2])
|
||||
@ -31,8 +30,6 @@ local active = tonumber(ARGV[9])
|
||||
local reserved = tonumber(ARGV[10])
|
||||
local worker_index_key = ARGV[11]
|
||||
local worker_draining_index_key = ARGV[12]
|
||||
local upstream_id = ARGV[13]
|
||||
local policy_revision = ARGV[14]
|
||||
local mutating = operation ~= 'get'
|
||||
|
||||
local function finish(reply)
|
||||
@ -215,11 +212,6 @@ local function decode_table(raw)
|
||||
return value
|
||||
end
|
||||
|
||||
local function valid_uint(value)
|
||||
return type(value) == 'string' and string.match(value, '^[0-9]+$') and
|
||||
value ~= '0' and (string.len(value) == 1 or string.sub(value, 1, 1) ~= '0')
|
||||
end
|
||||
|
||||
local function valid_assignment(assignment)
|
||||
return assignment and assignment.version == 1 and type(assignment.proxyId) == 'string' and
|
||||
assignment.proxyId ~= '' and type(assignment.workerId) == 'string' and assignment.workerId ~= '' and
|
||||
@ -404,33 +396,6 @@ if operation == 'begin_unhealthy_drain' then
|
||||
return finish({status = 'ok', record = encoded, started = started})
|
||||
end
|
||||
|
||||
if operation == 'begin_disabled_upstream_drain' then
|
||||
cleanup_hard_expired(now_ms)
|
||||
if type(upstream_id) ~= 'string' or upstream_id == '' or not valid_uint(policy_revision) then
|
||||
return finish({status = 'invalid'})
|
||||
end
|
||||
local policy = decode_table(redis.call('HGET', upstream_drain_policies_key, upstream_id))
|
||||
if not policy or policy.version ~= 1 or policy.upstreamId ~= upstream_id or policy.enabled ~= false or
|
||||
policy.revision ~= policy_revision then
|
||||
return finish({status = 'stale'})
|
||||
end
|
||||
local current = decode_table(redis.call('HGET', owners_key, proxy_id))
|
||||
if not valid_assignment(current) or current.workerId ~= worker_id or tonumber(current.epoch) ~= epoch or
|
||||
tonumber(current.expiresAtMs) <= now_ms then
|
||||
return finish({status = 'stale'})
|
||||
end
|
||||
local record = decode_table(redis.call('HGET', records_key, proxy_id))
|
||||
if not record or record.sourceUpstream ~= upstream_id or record.ownerWorkerId ~= worker_id or
|
||||
record.state ~= 'AVAILABLE' then
|
||||
return finish({status = 'stale'})
|
||||
end
|
||||
local encoded, started, failure = begin_drain(current)
|
||||
if failure then
|
||||
return finish({status = failure})
|
||||
end
|
||||
return finish({status = 'ok', record = encoded, started = started})
|
||||
end
|
||||
|
||||
if operation == 'acknowledge_drain' then
|
||||
local current = decode_table(redis.call('HGET', owners_key, proxy_id))
|
||||
if not valid_assignment(current) or current.workerId ~= worker_id or tonumber(current.epoch) ~= epoch then
|
||||
|
||||
@ -15,8 +15,6 @@ local health_tasks_key = KEYS[14]
|
||||
local health_task_expiry_key = KEYS[15]
|
||||
local health_ref_task_key = KEYS[16]
|
||||
local health_unhealthy_key = KEYS[17]
|
||||
local owned_upstream_key = KEYS[18]
|
||||
local upstream_drain_policies_key = KEYS[19]
|
||||
|
||||
local operation = ARGV[1]
|
||||
local now_ms = tonumber(ARGV[2])
|
||||
@ -222,40 +220,6 @@ if operation == 'unhealthy' then
|
||||
end
|
||||
return finish({status = 'ok', count = removed, deferredOwned = deferred_owned, drainCandidates = drain_candidates})
|
||||
end
|
||||
if operation == 'disabled_upstream' then
|
||||
local raw_policy = redis.call('HGET', upstream_drain_policies_key, upstream_id)
|
||||
local policy = raw_policy and decode_table(raw_policy) or nil
|
||||
if not policy or policy.version ~= 1 or policy.upstreamId ~= upstream_id or policy.enabled ~= false or
|
||||
type(policy.revision) ~= 'string' or not string.match(policy.revision, '^[0-9]+$') or policy.revision == '0' then
|
||||
return finish({status = 'stale', count = 0, disabledDrainCandidates = {}})
|
||||
end
|
||||
local scan_limit = limit * 4
|
||||
if scan_limit > 1024 then
|
||||
scan_limit = 1024
|
||||
end
|
||||
local proxy_ids = redis.call('ZRANGEBYSCORE', owned_upstream_key, '(' .. now_ms, '+inf', 'LIMIT', 0, scan_limit)
|
||||
local candidates = {}
|
||||
for _, proxy_id in ipairs(proxy_ids) do
|
||||
local record = decode_table(redis.call('HGET', records_key, proxy_id))
|
||||
local owner = decode_table(redis.call('HGET', owners_key, proxy_id))
|
||||
if record and owner and record.sourceUpstream == upstream_id and record.state == 'AVAILABLE' and
|
||||
record.ownerWorkerId == owner.workerId and owner.draining == false and
|
||||
type(owner.workerId) == 'string' and owner.workerId ~= '' and tonumber(owner.epoch) and
|
||||
tonumber(owner.epoch) > 0 and tonumber(owner.expiresAtMs) and tonumber(owner.expiresAtMs) > now_ms then
|
||||
table.insert(candidates, {
|
||||
upstreamId = upstream_id,
|
||||
policyRevision = policy.revision,
|
||||
proxyId = proxy_id,
|
||||
workerId = owner.workerId,
|
||||
assignmentEpoch = tonumber(owner.epoch),
|
||||
})
|
||||
if #candidates == limit then
|
||||
break
|
||||
end
|
||||
end
|
||||
end
|
||||
return finish({status = 'ok', count = 0, disabledDrainCandidates = candidates})
|
||||
end
|
||||
if operation == 'inventory' then
|
||||
local count = tonumber(redis.call('HGET', inventory_key, upstream_id) or '0')
|
||||
if count < 0 then
|
||||
|
||||
@ -1,23 +0,0 @@
|
||||
local policies_key = KEYS[1]
|
||||
local policies_json = ARGV[1]
|
||||
|
||||
local decoded, policies = pcall(cjson.decode, policies_json or '')
|
||||
if not decoded or type(policies) ~= 'table' then
|
||||
return cjson.encode({status = 'invalid'})
|
||||
end
|
||||
|
||||
local seen = {}
|
||||
for _, policy in ipairs(policies) do
|
||||
if type(policy) ~= 'table' or type(policy.upstreamId) ~= 'string' or policy.upstreamId == '' or
|
||||
type(policy.revision) ~= 'string' or not string.match(policy.revision, '^[0-9]+$') or
|
||||
policy.revision == '0' or type(policy.enabled) ~= 'boolean' or seen[policy.upstreamId] then
|
||||
return cjson.encode({status = 'invalid'})
|
||||
end
|
||||
seen[policy.upstreamId] = true
|
||||
end
|
||||
|
||||
redis.call('DEL', policies_key)
|
||||
for _, policy in ipairs(policies) do
|
||||
redis.call('HSET', policies_key, policy.upstreamId, cjson.encode(policy))
|
||||
end
|
||||
return cjson.encode({status = 'ok'})
|
||||
@ -1,63 +0,0 @@
|
||||
package redisactivity
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"strconv"
|
||||
|
||||
"proxy-pool/internal/domain/activitypool"
|
||||
)
|
||||
|
||||
var _ activitypool.UpstreamDrainPolicyWriter = (*Adapter)(nil)
|
||||
|
||||
type upstreamDrainPolicyRecord struct {
|
||||
Version int `json:"version"`
|
||||
UpstreamID string `json:"upstreamId"`
|
||||
Revision string `json:"revision"`
|
||||
Enabled bool `json:"enabled"`
|
||||
}
|
||||
|
||||
func (a *Adapter) ReplaceUpstreamDrainPolicies(ctx context.Context, policies []activitypool.UpstreamDrainPolicy) error {
|
||||
if ctx == nil {
|
||||
return activitypool.ErrInvalidMaintenance
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
if a == nil {
|
||||
return activitypool.ErrInvalidMaintenance
|
||||
}
|
||||
records := make([]upstreamDrainPolicyRecord, 0, len(policies))
|
||||
seen := make(map[string]struct{}, len(policies))
|
||||
for _, policy := range policies {
|
||||
if policy.UpstreamID == "" || policy.Revision == 0 {
|
||||
return activitypool.ErrInvalidMaintenance
|
||||
}
|
||||
if _, duplicate := seen[policy.UpstreamID]; duplicate {
|
||||
return activitypool.ErrInvalidMaintenance
|
||||
}
|
||||
seen[policy.UpstreamID] = struct{}{}
|
||||
records = append(records, upstreamDrainPolicyRecord{
|
||||
Version: 1, UpstreamID: policy.UpstreamID, Revision: strconv.FormatUint(policy.Revision, 10), Enabled: policy.Enabled,
|
||||
})
|
||||
}
|
||||
payload, err := json.Marshal(records)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
result, err := runScript(ctx, a.client, upstreamDrainPolicyScript, []string{a.keys.upstreamDrainPolicies}, string(payload))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var reply upstreamDrainPolicyScriptReply
|
||||
if err := decodeScriptResult(result, &reply); err != nil {
|
||||
return err
|
||||
}
|
||||
if reply.Status == scriptInvalid {
|
||||
return activitypool.ErrInvalidMaintenance
|
||||
}
|
||||
if reply.Status != scriptOK {
|
||||
return invalidScriptReply("unexpected upstream drain policy reply")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@ -8,13 +8,9 @@ import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net"
|
||||
"net/http"
|
||||
"reflect"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
"google.golang.org/grpc"
|
||||
"google.golang.org/grpc/credentials"
|
||||
|
||||
@ -23,17 +19,12 @@ import (
|
||||
"proxy-pool/internal/checker/probe"
|
||||
"proxy-pool/internal/config"
|
||||
"proxy-pool/internal/controlplane/clienttransport"
|
||||
"proxy-pool/internal/controlplane/tlsreload"
|
||||
"proxy-pool/internal/domain/workerruntime"
|
||||
"proxy-pool/internal/platform/httpserver"
|
||||
"proxy-pool/internal/platform/lifecycle"
|
||||
platformMetrics "proxy-pool/internal/platform/metrics"
|
||||
)
|
||||
|
||||
var (
|
||||
ErrInvalidOptions = errors.New("invalid checker bootstrap options")
|
||||
ErrStartup = errors.New("checker startup failed")
|
||||
ErrNotReady = errors.New("checker control plane is not ready")
|
||||
)
|
||||
|
||||
type Options struct {
|
||||
@ -42,12 +33,9 @@ type Options struct {
|
||||
ControlPlaneAddress string
|
||||
CheckerID string
|
||||
InstanceID string
|
||||
AutoIdentity bool
|
||||
MaxInFlight int
|
||||
SupportedLevels []controlplanev1.CheckLevel
|
||||
GRPCTransport credentials.TransportCredentials
|
||||
MetricsListener net.Listener
|
||||
HTTP httpserver.Options
|
||||
}
|
||||
|
||||
func Run(ctx context.Context, options Options) error {
|
||||
@ -61,27 +49,11 @@ func Run(ctx context.Context, options Options) error {
|
||||
if !configuration.ControlPlane.Enabled {
|
||||
return errors.Join(ErrInvalidOptions, errors.New("controlPlane must be enabled"))
|
||||
}
|
||||
options, err = resolveIdentity(configuration, options)
|
||||
if err != nil {
|
||||
return fmt.Errorf("%w: %w", ErrStartup, err)
|
||||
}
|
||||
runtime, err := newRuntime(ctx, configuration, options)
|
||||
if err != nil {
|
||||
return fmt.Errorf("%w: %w", ErrStartup, err)
|
||||
}
|
||||
defer runtime.Close()
|
||||
return runtime.Run(ctx)
|
||||
}
|
||||
|
||||
func newRuntime(ctx context.Context, configuration *config.Config, options Options) (*runtime, error) {
|
||||
if ctx == nil || configuration == nil {
|
||||
return nil, ErrInvalidOptions
|
||||
}
|
||||
transport, err := clienttransport.New(
|
||||
configuration.ControlPlane, options.ControlPlaneAddress, configuration.ControlPlane.CheckerTLS, options.GRPCTransport, "checkerTLS",
|
||||
)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%w: %w", ErrInvalidOptions, err)
|
||||
return fmt.Errorf("%w: %w", ErrInvalidOptions, err)
|
||||
}
|
||||
connection, err := grpc.NewClient(options.ControlPlaneAddress,
|
||||
grpc.WithTransportCredentials(transport),
|
||||
@ -91,164 +63,31 @@ func newRuntime(ctx context.Context, configuration *config.Config, options Optio
|
||||
),
|
||||
)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("dial control plane: %w", err)
|
||||
return fmt.Errorf("%w: dial control plane: %w", ErrStartup, err)
|
||||
}
|
||||
closeConnection := true
|
||||
defer func() {
|
||||
if closeConnection {
|
||||
_ = connection.Close()
|
||||
}
|
||||
}()
|
||||
defer connection.Close()
|
||||
client := controlplane.NewGeneratedClient(controlplanev1.NewCheckerControlPlaneClient(connection))
|
||||
readiness := &controlPlaneReadiness{}
|
||||
runner, err := controlplane.NewRunner(client, probe.NewExecutor(), controlplane.Options{
|
||||
CheckerID: options.CheckerID, InstanceID: options.InstanceID, MaxInFlight: options.MaxInFlight,
|
||||
SupportedLevels: append([]controlplanev1.CheckLevel(nil), options.SupportedLevels...),
|
||||
ReportBatchSize: options.MaxInFlight, OnSuccessfulPull: readiness.MarkReady, OnFailedPull: readiness.MarkUnavailable,
|
||||
ReportBatchSize: options.MaxInFlight,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("build checker runner: %w", err)
|
||||
return fmt.Errorf("%w: build checker runner: %w", ErrStartup, err)
|
||||
}
|
||||
runners := []lifecycle.Runner{runner}
|
||||
if configuration.Metrics.Enabled {
|
||||
metrics, metricsErr := newMetricsRuntime(ctx, configuration.Metrics, options, readiness)
|
||||
if metricsErr != nil {
|
||||
return nil, metricsErr
|
||||
}
|
||||
runners = append(runners, metrics)
|
||||
} else if options.MetricsListener != nil {
|
||||
return nil, ErrInvalidOptions
|
||||
}
|
||||
group, err := lifecycle.NewGroup(runners...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
closeConnection = false
|
||||
return &runtime{connection: connection, group: group}, nil
|
||||
return runner.Run(ctx)
|
||||
}
|
||||
|
||||
func validateOptions(ctx context.Context, options Options) error {
|
||||
if ctx == nil || strings.TrimSpace(options.ConfigPath) != options.ConfigPath || options.ConfigPath == "" ||
|
||||
nilInterface(options.Resolver) || !clienttransport.ValidDialAddress(options.ControlPlaneAddress) ||
|
||||
(options.CheckerID != "" && !workerruntime.ValidIdentifier(options.CheckerID)) ||
|
||||
(options.InstanceID != "" && !workerruntime.ValidIdentifier(options.InstanceID)) ||
|
||||
(!options.AutoIdentity && (!workerruntime.ValidIdentifier(options.CheckerID) || !workerruntime.ValidIdentifier(options.InstanceID))) ||
|
||||
!workerruntime.ValidIdentifier(options.CheckerID) || !workerruntime.ValidIdentifier(options.InstanceID) ||
|
||||
options.MaxInFlight <= 0 || len(options.SupportedLevels) == 0 {
|
||||
return ErrInvalidOptions
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func resolveIdentity(configuration *config.Config, options Options) (Options, error) {
|
||||
if configuration == nil {
|
||||
return Options{}, ErrInvalidOptions
|
||||
}
|
||||
if !options.AutoIdentity {
|
||||
return options, nil
|
||||
}
|
||||
if configuration.ControlPlane.TLS.Mode != "mtls" {
|
||||
return Options{}, errors.Join(ErrInvalidOptions, errors.New("auto identity requires controlPlane mTLS"))
|
||||
}
|
||||
identity, err := tlsreload.ResolveSPIFFEIdentity(
|
||||
configuration.ControlPlane.CheckerTLS.CertFile, configuration.ControlPlane.CheckerTLS.KeyFile,
|
||||
configuration.ControlPlane.TLS.TrustDomain, configuration.ControlPlane.TLS.Environment, "checker",
|
||||
)
|
||||
if err != nil {
|
||||
return Options{}, fmt.Errorf("resolve checker SPIFFE identity: %w", err)
|
||||
}
|
||||
if options.CheckerID == "" {
|
||||
options.CheckerID = identity
|
||||
}
|
||||
if options.InstanceID == "" {
|
||||
options.InstanceID = identity
|
||||
}
|
||||
if !workerruntime.ValidIdentifier(options.CheckerID) || !workerruntime.ValidIdentifier(options.InstanceID) || options.CheckerID != identity {
|
||||
return Options{}, errors.Join(ErrInvalidOptions, errors.New("checker identity does not match SPIFFE certificate"))
|
||||
}
|
||||
return options, nil
|
||||
}
|
||||
|
||||
type runtime struct {
|
||||
connection *grpc.ClientConn
|
||||
group *lifecycle.Group
|
||||
}
|
||||
|
||||
func (runtime *runtime) Run(ctx context.Context) error {
|
||||
if runtime == nil || runtime.connection == nil || runtime.group == nil || ctx == nil {
|
||||
return ErrInvalidOptions
|
||||
}
|
||||
return runtime.group.Run(ctx)
|
||||
}
|
||||
|
||||
func (runtime *runtime) Close() {
|
||||
if runtime != nil && runtime.connection != nil {
|
||||
_ = runtime.connection.Close()
|
||||
}
|
||||
}
|
||||
|
||||
type controlPlaneReadiness struct {
|
||||
ready atomic.Bool
|
||||
}
|
||||
|
||||
func (readiness *controlPlaneReadiness) MarkReady() {
|
||||
if readiness != nil {
|
||||
readiness.ready.Store(true)
|
||||
}
|
||||
}
|
||||
|
||||
func (readiness *controlPlaneReadiness) MarkUnavailable() {
|
||||
if readiness != nil {
|
||||
readiness.ready.Store(false)
|
||||
}
|
||||
}
|
||||
|
||||
func (readiness *controlPlaneReadiness) Ready(ctx context.Context) error {
|
||||
if ctx == nil || readiness == nil || !readiness.ready.Load() {
|
||||
return ErrNotReady
|
||||
}
|
||||
return ctx.Err()
|
||||
}
|
||||
|
||||
type metricsRuntime struct {
|
||||
listener net.Listener
|
||||
handler http.Handler
|
||||
options httpserver.Options
|
||||
}
|
||||
|
||||
func newMetricsRuntime(
|
||||
ctx context.Context,
|
||||
configuration config.Metrics,
|
||||
options Options,
|
||||
readiness *controlPlaneReadiness,
|
||||
) (*metricsRuntime, error) {
|
||||
if ctx == nil || !configuration.Enabled || readiness == nil {
|
||||
return nil, ErrInvalidOptions
|
||||
}
|
||||
handler, err := platformMetrics.NewHandler(platformMetrics.Dependencies{
|
||||
Gatherer: prometheus.DefaultGatherer, Readiness: readiness,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("build checker metrics handler: %w", err)
|
||||
}
|
||||
listener := options.MetricsListener
|
||||
if listener == nil {
|
||||
listener, err = (&net.ListenConfig{}).Listen(ctx, "tcp", configuration.Listen)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("listen checker metrics: %w", err)
|
||||
}
|
||||
}
|
||||
return &metricsRuntime{listener: listener, handler: handler, options: options.HTTP}, nil
|
||||
}
|
||||
|
||||
func (runtime *metricsRuntime) Run(ctx context.Context) error {
|
||||
if runtime == nil || runtime.listener == nil || runtime.handler == nil || ctx == nil {
|
||||
return ErrInvalidOptions
|
||||
}
|
||||
return httpserver.Serve(ctx, runtime.options, httpserver.Endpoint{
|
||||
Name: "metrics", Listener: runtime.listener, Handler: runtime.handler,
|
||||
})
|
||||
}
|
||||
|
||||
func loadConfiguration(ctx context.Context, path string, resolver config.Resolver) (*config.Config, error) {
|
||||
if ctx == nil || strings.TrimSpace(path) != path || path == "" || nilInterface(resolver) {
|
||||
return nil, ErrInvalidOptions
|
||||
|
||||
@ -1,55 +0,0 @@
|
||||
package bootstrap
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
|
||||
"proxy-pool/internal/config"
|
||||
)
|
||||
|
||||
func TestControlPlaneReadinessTransitionsAfterSuccessfulPull(t *testing.T) {
|
||||
readiness := &controlPlaneReadiness{}
|
||||
if err := readiness.Ready(context.Background()); !errors.Is(err, ErrNotReady) {
|
||||
t.Fatalf("Ready() before pull = %v, want ErrNotReady", err)
|
||||
}
|
||||
readiness.MarkReady()
|
||||
if err := readiness.Ready(context.Background()); err != nil {
|
||||
t.Fatalf("Ready() after successful pull = %v", err)
|
||||
}
|
||||
readiness.MarkUnavailable()
|
||||
if err := readiness.Ready(context.Background()); !errors.Is(err, ErrNotReady) {
|
||||
t.Fatalf("Ready() after failed pull = %v, want ErrNotReady", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMetricsRuntimeUsesControlPlaneReadiness(t *testing.T) {
|
||||
listener, err := net.Listen("tcp", "127.0.0.1:0")
|
||||
if err != nil {
|
||||
t.Fatalf("Listen(): %v", err)
|
||||
}
|
||||
defer listener.Close()
|
||||
readiness := &controlPlaneReadiness{}
|
||||
runtime, err := newMetricsRuntime(context.Background(), config.Metrics{Enabled: true, Listen: listener.Addr().String()}, Options{
|
||||
MetricsListener: listener,
|
||||
}, readiness)
|
||||
if err != nil {
|
||||
t.Fatalf("newMetricsRuntime(): %v", err)
|
||||
}
|
||||
assertCheckerProbe(t, runtime.handler, "/livez", http.StatusOK, "live\n")
|
||||
assertCheckerProbe(t, runtime.handler, "/readyz", http.StatusServiceUnavailable, "unavailable\n")
|
||||
readiness.MarkReady()
|
||||
assertCheckerProbe(t, runtime.handler, "/readyz", http.StatusOK, "ready\n")
|
||||
}
|
||||
|
||||
func assertCheckerProbe(t *testing.T, handler http.Handler, path string, wantStatus int, wantBody string) {
|
||||
t.Helper()
|
||||
recorder := httptest.NewRecorder()
|
||||
handler.ServeHTTP(recorder, httptest.NewRequest(http.MethodGet, path, nil))
|
||||
if recorder.Code != wantStatus || recorder.Body.String() != wantBody {
|
||||
t.Fatalf("GET %s = %d %q, want %d %q", path, recorder.Code, recorder.Body.String(), wantStatus, wantBody)
|
||||
}
|
||||
}
|
||||
@ -52,8 +52,6 @@ type Options struct {
|
||||
PollInterval time.Duration
|
||||
RetryDelay time.Duration
|
||||
Now func() time.Time
|
||||
OnSuccessfulPull func()
|
||||
OnFailedPull func()
|
||||
}
|
||||
|
||||
// Runner consumes at most one server-bounded pull per RunOnce. Run adds a
|
||||
@ -95,9 +93,6 @@ func (runner *Runner) Run(ctx context.Context) error {
|
||||
if err != nil && ctx.Err() != nil {
|
||||
return ctx.Err()
|
||||
}
|
||||
if err != nil && runner.options.OnFailedPull != nil {
|
||||
runner.options.OnFailedPull()
|
||||
}
|
||||
wait := runner.options.PollInterval
|
||||
if err != nil {
|
||||
wait = runner.options.RetryDelay
|
||||
@ -123,13 +118,9 @@ func (runner *Runner) RunOnce(ctx context.Context) error {
|
||||
return err
|
||||
}
|
||||
tasks, err := collectTasks(stream, runner.options.MaxInFlight)
|
||||
if err != nil {
|
||||
if err != nil || len(tasks) == 0 {
|
||||
return err
|
||||
}
|
||||
if len(tasks) == 0 {
|
||||
runner.observeSuccessfulPull()
|
||||
return nil
|
||||
}
|
||||
observations := runner.executeTasks(ctx, tasks)
|
||||
for start := 0; start < len(observations); start += runner.options.ReportBatchSize {
|
||||
end := start + runner.options.ReportBatchSize
|
||||
@ -146,16 +137,9 @@ func (runner *Runner) RunOnce(ctx context.Context) error {
|
||||
return ErrObservationsRejected
|
||||
}
|
||||
}
|
||||
runner.observeSuccessfulPull()
|
||||
return nil
|
||||
}
|
||||
|
||||
func (runner *Runner) observeSuccessfulPull() {
|
||||
if runner != nil && runner.options.OnSuccessfulPull != nil {
|
||||
runner.options.OnSuccessfulPull()
|
||||
}
|
||||
}
|
||||
|
||||
func collectTasks(stream TaskStream, maximum int) ([]*controlplanev1.CheckTask, error) {
|
||||
if stream == nil || maximum <= 0 {
|
||||
return nil, ErrInvalidRunner
|
||||
|
||||
@ -2,10 +2,8 @@ package controlplane
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"io"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
@ -69,51 +67,6 @@ func TestRunnerOmitsEgressProbeURLFromGlobalObservation(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunnerMarksReadyAfterSuccessfulEmptyPull(t *testing.T) {
|
||||
var successfulPulls atomic.Int64
|
||||
runner, err := NewRunner(&clientStub{stream: &taskStreamStub{}}, &executorStub{}, Options{
|
||||
CheckerID: "checker-a", InstanceID: "instance-a", MaxInFlight: 1,
|
||||
SupportedLevels: []controlplanev1.CheckLevel{controlplanev1.CheckLevel_CHECK_LEVEL_BASIC},
|
||||
Now: time.Now, OnSuccessfulPull: func() { successfulPulls.Add(1) },
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("NewRunner(): %v", err)
|
||||
}
|
||||
if err := runner.RunOnce(context.Background()); err != nil {
|
||||
t.Fatalf("RunOnce(): %v", err)
|
||||
}
|
||||
if successfulPulls.Load() != 1 {
|
||||
t.Fatalf("successful pulls = %d, want 1", successfulPulls.Load())
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunnerMarksUnavailableAfterPullFailure(t *testing.T) {
|
||||
failed := make(chan struct{})
|
||||
var markOnce sync.Once
|
||||
runner, err := NewRunner(&clientStub{streamErr: errors.New("control plane unavailable")}, &executorStub{}, Options{
|
||||
CheckerID: "checker-a", InstanceID: "instance-a", MaxInFlight: 1,
|
||||
SupportedLevels: []controlplanev1.CheckLevel{controlplanev1.CheckLevel_CHECK_LEVEL_BASIC},
|
||||
Now: time.Now, RetryDelay: time.Millisecond,
|
||||
OnFailedPull: func() { markOnce.Do(func() { close(failed) }) },
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("NewRunner(): %v", err)
|
||||
}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
completed := make(chan error, 1)
|
||||
go func() { completed <- runner.Run(ctx) }()
|
||||
select {
|
||||
case <-failed:
|
||||
cancel()
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("Runner did not mark failed pull")
|
||||
}
|
||||
if err := <-completed; !errors.Is(err, context.Canceled) {
|
||||
t.Fatalf("Run() = %v, want context.Canceled", err)
|
||||
}
|
||||
}
|
||||
|
||||
func checkerTask(id string, now time.Time) *controlplanev1.CheckTask {
|
||||
return &controlplanev1.CheckTask{
|
||||
TaskId: id, ProxyId: id + "-proxy", Protocol: controlplanev1.ProxyProtocol_PROXY_PROTOCOL_HTTP,
|
||||
@ -124,12 +77,11 @@ func checkerTask(id string, now time.Time) *controlplanev1.CheckTask {
|
||||
|
||||
type clientStub struct {
|
||||
stream TaskStream
|
||||
streamErr error
|
||||
batches []*controlplanev1.ObservationBatch
|
||||
}
|
||||
|
||||
func (stub *clientStub) StreamCheckTasks(context.Context, *controlplanev1.StreamCheckTasksRequest) (TaskStream, error) {
|
||||
return stub.stream, stub.streamErr
|
||||
return stub.stream, nil
|
||||
}
|
||||
|
||||
func (stub *clientStub) ReportObservations(_ context.Context, batch *controlplanev1.ObservationBatch) (*controlplanev1.ReportObservationsResponse, error) {
|
||||
|
||||
@ -3,8 +3,6 @@ package config
|
||||
import (
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"proxy-pool/internal/domain/clientpolicy"
|
||||
)
|
||||
|
||||
// MaximumCheckURLs bounds per-upstream EGRESS references. The bound keeps
|
||||
@ -65,8 +63,6 @@ type Listener struct {
|
||||
Auth Auth `yaml:"auth"`
|
||||
Limits Limits `yaml:"limits"`
|
||||
Retry Retry `yaml:"retry"`
|
||||
StickySession StickySession `yaml:"stickySession"`
|
||||
Transport GatewayTransport `yaml:"transport"`
|
||||
DestinationPolicy DestinationPolicy `yaml:"destinationPolicy"`
|
||||
}
|
||||
|
||||
@ -83,8 +79,6 @@ type Access struct {
|
||||
|
||||
type Auth struct {
|
||||
Mode string `yaml:"mode"`
|
||||
Permissions []string `yaml:"permissions"`
|
||||
ClientPolicy clientpolicy.Policy `yaml:"client"`
|
||||
Username string `yaml:"username"`
|
||||
Password string `yaml:"password"`
|
||||
PasswordFile string `yaml:"passwordFile"`
|
||||
@ -97,8 +91,6 @@ type Auth struct {
|
||||
|
||||
type AuthMethod struct {
|
||||
Mode string `yaml:"mode"`
|
||||
Permissions []string `yaml:"permissions"`
|
||||
ClientPolicy clientpolicy.Policy `yaml:"client"`
|
||||
Username string `yaml:"username"`
|
||||
Password string `yaml:"password"`
|
||||
PasswordFile string `yaml:"passwordFile"`
|
||||
@ -119,29 +111,6 @@ type Retry struct {
|
||||
RetryMethods []string `yaml:"retryMethods"`
|
||||
}
|
||||
|
||||
// StickySession configures per-Worker, bounded Gateway affinity. It does not
|
||||
// store Proxy details or session identifiers in PostgreSQL or Redis.
|
||||
type StickySession struct {
|
||||
Enabled bool `yaml:"enabled"`
|
||||
Header string `yaml:"header"`
|
||||
TTL Duration `yaml:"ttl"`
|
||||
MaxEntries int `yaml:"maxEntries"`
|
||||
}
|
||||
|
||||
// GatewayTransport configures the local Gateway process connection pools and
|
||||
// tunnel I/O. It intentionally contains no remote state or provider settings.
|
||||
type GatewayTransport struct {
|
||||
DialTimeout Duration `yaml:"dialTimeout"`
|
||||
HandshakeTimeout Duration `yaml:"handshakeTimeout"`
|
||||
ResponseHeaderTimeout Duration `yaml:"responseHeaderTimeout"`
|
||||
IdleConnTimeout Duration `yaml:"idleConnTimeout"`
|
||||
MaxIdleConns int `yaml:"maxIdleConns"`
|
||||
MaxIdleConnsPerHost int `yaml:"maxIdleConnsPerHost"`
|
||||
MaxConnsPerHost int `yaml:"maxConnsPerHost"`
|
||||
TunnelBufferBytes int `yaml:"tunnelBufferBytes"`
|
||||
TunnelIdleTimeout Duration `yaml:"tunnelIdleTimeout"`
|
||||
}
|
||||
|
||||
type DestinationPolicy struct {
|
||||
DenyPrivateNetworks *bool `yaml:"denyPrivateNetworks"`
|
||||
DenyLoopback *bool `yaml:"denyLoopback"`
|
||||
@ -214,7 +183,6 @@ type Routing struct {
|
||||
Name string `yaml:"name"`
|
||||
Enabled bool `yaml:"enabled"`
|
||||
Purpose string `yaml:"purpose"`
|
||||
Action string `yaml:"action"`
|
||||
Match RoutingMatch `yaml:"match"`
|
||||
Upstreams []string `yaml:"upstreams"`
|
||||
Strategy Strategy `yaml:"strategy"`
|
||||
|
||||
@ -10,9 +10,6 @@ import (
|
||||
"time"
|
||||
|
||||
"go.yaml.in/yaml/v4"
|
||||
|
||||
"proxy-pool/internal/domain/authorization"
|
||||
"proxy-pool/internal/domain/clientpolicy"
|
||||
)
|
||||
|
||||
const validConfig = `
|
||||
@ -204,7 +201,6 @@ func TestShippedDeploymentConfigurationsResolveEnvironment(t *testing.T) {
|
||||
wantGatewayUser string
|
||||
wantPostgresURL string
|
||||
wantRedisURL string
|
||||
wantTransport GatewayTransport
|
||||
}{
|
||||
{
|
||||
name: "local",
|
||||
@ -212,12 +208,6 @@ func TestShippedDeploymentConfigurationsResolveEnvironment(t *testing.T) {
|
||||
wantGatewayUser: "local-gateway",
|
||||
wantPostgresURL: "postgres://proxy_pool:local-only-change-me@postgres:5432/proxy_pool?sslmode=disable",
|
||||
wantRedisURL: "redis://redis:6379/0",
|
||||
wantTransport: GatewayTransport{
|
||||
MaxIdleConns: 20000,
|
||||
MaxIdleConnsPerHost: 32,
|
||||
MaxConnsPerHost: 32,
|
||||
TunnelBufferBytes: 32768,
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "kubernetes",
|
||||
@ -226,17 +216,6 @@ func TestShippedDeploymentConfigurationsResolveEnvironment(t *testing.T) {
|
||||
wantGatewayUser: "resolved-gateway-user",
|
||||
wantPostgresURL: "postgres://resolved",
|
||||
wantRedisURL: "redis://resolved",
|
||||
wantTransport: GatewayTransport{
|
||||
DialTimeout: Duration(10 * time.Second),
|
||||
HandshakeTimeout: Duration(15 * time.Second),
|
||||
ResponseHeaderTimeout: Duration(30 * time.Second),
|
||||
IdleConnTimeout: Duration(90 * time.Second),
|
||||
MaxIdleConns: 20000,
|
||||
MaxIdleConnsPerHost: 32,
|
||||
MaxConnsPerHost: 32,
|
||||
TunnelBufferBytes: 32768,
|
||||
TunnelIdleTimeout: Duration(5 * time.Minute),
|
||||
},
|
||||
},
|
||||
}
|
||||
for _, test := range tests {
|
||||
@ -266,7 +245,6 @@ func TestShippedDeploymentConfigurationsResolveEnvironment(t *testing.T) {
|
||||
cfg.Admin.Auth.Token != "resolved-admin-token" ||
|
||||
cfg.Storage.PostgresURL != test.wantPostgresURL ||
|
||||
cfg.Storage.RedisURL != test.wantRedisURL ||
|
||||
cfg.Gateway.Transport != test.wantTransport ||
|
||||
cfg.Upstreams["provider-a"].API.Auth.Value != "resolved-provider-a-token" ||
|
||||
cfg.Upstreams["provider-b"].API.Auth.Value != "resolved-provider-b-token" {
|
||||
t.Fatalf("deployment values were not resolved: %+v", cfg.Redacted())
|
||||
@ -349,206 +327,6 @@ func TestValidateAcceptsBearerListenerAuthentication(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateListenerPermissionConfiguration(t *testing.T) {
|
||||
t.Parallel()
|
||||
cfg := mustLoadValidConfig(t)
|
||||
cfg.Admin.Enabled = true
|
||||
cfg.Admin.Listen = "127.0.0.1:8082"
|
||||
cfg.Admin.Auth = Auth{
|
||||
Mode: "bearer",
|
||||
Token: "admin-token",
|
||||
Permissions: []string{authorization.AdminRead},
|
||||
}
|
||||
if err := Validate(cfg); err != nil {
|
||||
t.Fatalf("Validate(admin read permission) error = %v", err)
|
||||
}
|
||||
|
||||
cfg.Admin.Auth.Permissions = []string{"unknown"}
|
||||
if err := Validate(cfg); err == nil || !strings.Contains(err.Error(), "permissions") {
|
||||
t.Fatalf("Validate(unknown permission) error = %v", err)
|
||||
}
|
||||
|
||||
cfg.Admin.Auth = Auth{
|
||||
Mode: "any",
|
||||
Permissions: []string{authorization.AdminRead},
|
||||
Methods: []AuthMethod{{
|
||||
Mode: "bearer", Value: "read-token", Permissions: []string{authorization.AdminRead},
|
||||
}},
|
||||
}
|
||||
if err := Validate(cfg); err == nil || !strings.Contains(err.Error(), "auth.permissions") {
|
||||
t.Fatalf("Validate(any top-level permission) error = %v", err)
|
||||
}
|
||||
|
||||
cfg.Admin.Auth.Permissions = nil
|
||||
if err := Validate(cfg); err != nil {
|
||||
t.Fatalf("Validate(any method permission) error = %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateDistributionCredentialClientPolicy(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
cfg, err := Load(strings.NewReader(validConfig))
|
||||
if err != nil {
|
||||
t.Fatalf("Load() error = %v", err)
|
||||
}
|
||||
cfg.Distribution.Auth = Auth{
|
||||
Mode: "bearer", Token: "distribution-token",
|
||||
ClientPolicy: clientpolicy.Policy{
|
||||
MaxExtractCount: 5,
|
||||
AllowedUpstreams: []string{"provider-a"},
|
||||
AllowedRegions: []string{"shanghai"},
|
||||
},
|
||||
}
|
||||
if err := Validate(cfg); err != nil {
|
||||
t.Fatalf("Validate(distribution client policy) error = %v", err)
|
||||
}
|
||||
|
||||
cfg.Distribution.Auth.ClientPolicy.AllowedUpstreams = []string{"missing"}
|
||||
if err := Validate(cfg); err == nil || !strings.Contains(err.Error(), "missing") {
|
||||
t.Fatalf("Validate(unknown allowed upstream) error = %v", err)
|
||||
}
|
||||
|
||||
cfg.Distribution.Auth.ClientPolicy = clientpolicy.Policy{}
|
||||
cfg.Gateway.Auth = Auth{
|
||||
Mode: "bearer", Token: "gateway-token",
|
||||
ClientPolicy: clientpolicy.Policy{AllowedRoutings: []string{"extract"}},
|
||||
}
|
||||
cfg.Routing[0].Purpose = "gateway"
|
||||
if err := Validate(cfg); err != nil {
|
||||
t.Fatalf("Validate(gateway routing policy) error = %v", err)
|
||||
}
|
||||
cfg.Gateway.Auth.ClientPolicy.AllowedRoutings = []string{"missing"}
|
||||
if err := Validate(cfg); err == nil || !strings.Contains(err.Error(), "missing") {
|
||||
t.Fatalf("Validate(unknown gateway routing) error = %v", err)
|
||||
}
|
||||
cfg.Gateway.Auth.ClientPolicy = clientpolicy.Policy{MaxExtractCount: 1}
|
||||
if err := Validate(cfg); err == nil || !strings.Contains(err.Error(), "only supported on distribution") {
|
||||
t.Fatalf("Validate(gateway extraction policy) error = %v", err)
|
||||
}
|
||||
|
||||
cfg.Gateway.Auth.ClientPolicy = clientpolicy.Policy{RequestsPerMinute: 60}
|
||||
if err := Validate(cfg); err != nil {
|
||||
t.Fatalf("Validate(gateway request rate policy) error = %v", err)
|
||||
}
|
||||
|
||||
cfg.Gateway.Auth.ClientPolicy = clientpolicy.Policy{MaxConcurrentConnections: 3}
|
||||
if err := Validate(cfg); err != nil {
|
||||
t.Fatalf("Validate(gateway concurrency policy) error = %v", err)
|
||||
}
|
||||
|
||||
cfg.Gateway.Auth.ClientPolicy = clientpolicy.Policy{}
|
||||
cfg.Distribution.Auth.ClientPolicy = clientpolicy.Policy{MaxConcurrentConnections: 3}
|
||||
if err := Validate(cfg); err == nil || !strings.Contains(err.Error(), "only supported on gateway") {
|
||||
t.Fatalf("Validate(distribution gateway limit policy) error = %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateGatewayStickySession(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
cfg := mustLoadValidConfig(t)
|
||||
cfg.Gateway.Auth = Auth{Mode: "bearer", Token: "gateway-token"}
|
||||
cfg.Gateway.StickySession = StickySession{
|
||||
Enabled: true, Header: "X-Proxy-Session", TTL: Duration(30 * time.Second), MaxEntries: 10_000,
|
||||
}
|
||||
cfg.Routing[0].Purpose = "gateway"
|
||||
if err := Validate(cfg); err != nil {
|
||||
t.Fatalf("Validate(valid gateway stickySession) error = %v", err)
|
||||
}
|
||||
|
||||
cfg.Gateway.StickySession.Header = ""
|
||||
if err := Validate(cfg); err == nil || !strings.Contains(err.Error(), "stickySession.header") {
|
||||
t.Fatalf("Validate(missing stickySession header) error = %v", err)
|
||||
}
|
||||
|
||||
cfg.Gateway.StickySession.Header = "X Proxy Session"
|
||||
if err := Validate(cfg); err == nil || !strings.Contains(err.Error(), "stickySession.header") {
|
||||
t.Fatalf("Validate(invalid stickySession header) error = %v", err)
|
||||
}
|
||||
|
||||
cfg.Gateway.StickySession.Header = "X-Proxy-Session"
|
||||
cfg.Gateway.StickySession.TTL = 0
|
||||
if err := Validate(cfg); err == nil || !strings.Contains(err.Error(), "stickySession.ttl") {
|
||||
t.Fatalf("Validate(zero stickySession ttl) error = %v", err)
|
||||
}
|
||||
|
||||
cfg.Gateway.StickySession.TTL = Duration(30 * time.Second)
|
||||
cfg.Gateway.Auth = Auth{Mode: "none"}
|
||||
if err := Validate(cfg); err == nil || !strings.Contains(err.Error(), "requires authentication") {
|
||||
t.Fatalf("Validate(unauthenticated stickySession) error = %v", err)
|
||||
}
|
||||
|
||||
cfg.Gateway.Auth = Auth{Mode: "bearer", Token: "gateway-token"}
|
||||
cfg.Distribution.StickySession = StickySession{
|
||||
Enabled: true, Header: "X-Proxy-Session", TTL: Duration(30 * time.Second), MaxEntries: 10,
|
||||
}
|
||||
if err := Validate(cfg); err == nil || !strings.Contains(err.Error(), "only supported on gateway") {
|
||||
t.Fatalf("Validate(distribution stickySession) error = %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateStaticGatewayRoutingAction(t *testing.T) {
|
||||
cfg := mustLoadValidConfig(t)
|
||||
cfg.Routing[0] = Routing{
|
||||
Name: "direct-api", Enabled: true, Purpose: "gateway", Action: "direct",
|
||||
Match: RoutingMatch{HostRegex: "^api\\.example\\.test$"},
|
||||
}
|
||||
if err := Validate(cfg); err != nil {
|
||||
t.Fatalf("Validate(static direct routing) = %v", err)
|
||||
}
|
||||
|
||||
cfg.Routing[0].Purpose = "extract"
|
||||
if err := Validate(cfg); err == nil || !strings.Contains(err.Error(), "only supported on gateway") {
|
||||
t.Fatalf("Validate(extract direct routing) error = %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateGatewayTransport(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
cfg := mustLoadValidConfig(t)
|
||||
cfg.Gateway.Transport = GatewayTransport{
|
||||
DialTimeout: Duration(time.Second),
|
||||
HandshakeTimeout: Duration(2 * time.Second),
|
||||
ResponseHeaderTimeout: Duration(3 * time.Second),
|
||||
IdleConnTimeout: Duration(time.Minute),
|
||||
MaxIdleConns: 500,
|
||||
MaxIdleConnsPerHost: 20,
|
||||
MaxConnsPerHost: 10,
|
||||
TunnelBufferBytes: 32 << 10,
|
||||
TunnelIdleTimeout: Duration(5 * time.Minute),
|
||||
}
|
||||
if err := Validate(cfg); err != nil {
|
||||
t.Fatalf("Validate(valid gateway transport) error = %v", err)
|
||||
}
|
||||
|
||||
cfg.Gateway.Transport.DialTimeout = Duration(-time.Second)
|
||||
if err := Validate(cfg); err == nil || !strings.Contains(err.Error(), "transport.dialTimeout") {
|
||||
t.Fatalf("Validate(negative dial timeout) error = %v", err)
|
||||
}
|
||||
|
||||
cfg.Gateway.Transport.DialTimeout = Duration(time.Second)
|
||||
cfg.Gateway.Transport.MaxConnsPerHost = -1
|
||||
if err := Validate(cfg); err == nil || !strings.Contains(err.Error(), "transport.maxConnsPerHost") {
|
||||
t.Fatalf("Validate(negative max connections) error = %v", err)
|
||||
}
|
||||
|
||||
cfg.Gateway.Transport.MaxConnsPerHost = 10
|
||||
cfg.Gateway.Transport.MaxIdleConns = 19
|
||||
cfg.Gateway.Transport.MaxIdleConnsPerHost = 20
|
||||
if err := Validate(cfg); err == nil || !strings.Contains(err.Error(), "maxIdleConnsPerHost") {
|
||||
t.Fatalf("Validate(inconsistent idle pool) error = %v", err)
|
||||
}
|
||||
|
||||
cfg.Gateway.Transport.MaxIdleConns = 500
|
||||
cfg.Gateway.Transport.MaxIdleConnsPerHost = 20
|
||||
cfg.Distribution.Transport = GatewayTransport{MaxConnsPerHost: 10}
|
||||
if err := Validate(cfg); err == nil || !strings.Contains(err.Error(), "distribution transport") {
|
||||
t.Fatalf("Validate(distribution transport) error = %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateMetricsListener(t *testing.T) {
|
||||
t.Parallel()
|
||||
cfg := mustLoadValidConfig(t)
|
||||
|
||||
@ -80,13 +80,9 @@ func cloneListener(source Listener) Listener {
|
||||
cloned.Access.AllowCIDRs = cloneStrings(source.Access.AllowCIDRs)
|
||||
cloned.Access.TrustedProxies = cloneStrings(source.Access.TrustedProxies)
|
||||
cloned.Auth.CIDRs = cloneStrings(source.Auth.CIDRs)
|
||||
cloned.Auth.Permissions = cloneStrings(source.Auth.Permissions)
|
||||
cloned.Auth.ClientPolicy = source.Auth.ClientPolicy.Clone()
|
||||
cloned.Auth.Methods = append([]AuthMethod(nil), source.Auth.Methods...)
|
||||
for index := range cloned.Auth.Methods {
|
||||
cloned.Auth.Methods[index].CIDRs = cloneStrings(source.Auth.Methods[index].CIDRs)
|
||||
cloned.Auth.Methods[index].Permissions = cloneStrings(source.Auth.Methods[index].Permissions)
|
||||
cloned.Auth.Methods[index].ClientPolicy = source.Auth.Methods[index].ClientPolicy.Clone()
|
||||
}
|
||||
cloned.Retry.RetryMethods = cloneStrings(source.Retry.RetryMethods)
|
||||
cloned.DestinationPolicy.DenyPrivateNetworks = cloneBool(source.DestinationPolicy.DenyPrivateNetworks)
|
||||
|
||||
@ -4,14 +4,10 @@ import (
|
||||
"fmt"
|
||||
"math"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"proxy-pool/internal/domain/authorization"
|
||||
"proxy-pool/internal/domain/clientpolicy"
|
||||
)
|
||||
|
||||
var (
|
||||
@ -39,11 +35,6 @@ func Validate(cfg *Config) error {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if cfg.Admin.Enabled {
|
||||
if err := validateNoClientPolicy("admin", cfg.Admin.Auth); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if cfg.Metrics.Enabled {
|
||||
if cfg.Metrics.Listen == "" {
|
||||
return fmt.Errorf("validate metrics listen: address is required")
|
||||
@ -83,15 +74,11 @@ func Validate(cfg *Config) error {
|
||||
return fmt.Errorf("validate configuration: at least one enabled upstream is required")
|
||||
}
|
||||
seen := make(map[string]struct{}, len(cfg.Routing))
|
||||
gatewayRoutings := make(map[string]struct{}, len(cfg.Routing))
|
||||
targetProfilesByUpstream := make(map[string]int, len(cfg.Upstreams))
|
||||
for index, route := range cfg.Routing {
|
||||
if err := validateRouting(index, route, cfg.Upstreams, seen); err != nil {
|
||||
return err
|
||||
}
|
||||
if route.Enabled && route.Purpose == "gateway" {
|
||||
gatewayRoutings[route.Name] = struct{}{}
|
||||
}
|
||||
if !route.Enabled || len(route.Check.Targets) == 0 {
|
||||
continue
|
||||
}
|
||||
@ -106,27 +93,7 @@ func Validate(cfg *Config) error {
|
||||
}
|
||||
}
|
||||
}
|
||||
if cfg.Gateway.Enabled {
|
||||
if err := validateGatewayClientPolicies(cfg.Gateway.Auth, gatewayRoutings); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := validateGatewayStickySession(cfg.Gateway); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := validateGatewayTransport(cfg.Gateway); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if cfg.Distribution.Enabled {
|
||||
if err := validateDistributionClientPolicies(cfg.Distribution.Auth, cfg.Upstreams); err != nil {
|
||||
return err
|
||||
}
|
||||
if cfg.Distribution.StickySession.Enabled {
|
||||
return fmt.Errorf("validate distribution stickySession: only supported on gateway")
|
||||
}
|
||||
if gatewayTransportConfigured(cfg.Distribution.Transport) {
|
||||
return fmt.Errorf("validate distribution transport: only supported on gateway")
|
||||
}
|
||||
clientIdentificationMode := cfg.Distribution.ClientIdentification.Mode
|
||||
if clientIdentificationMode == "" {
|
||||
clientIdentificationMode = "sourceIP"
|
||||
@ -157,97 +124,9 @@ func Validate(cfg *Config) error {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if cfg.Admin.Enabled && cfg.Admin.StickySession.Enabled {
|
||||
return fmt.Errorf("validate admin stickySession: only supported on gateway")
|
||||
}
|
||||
if cfg.Admin.Enabled && gatewayTransportConfigured(cfg.Admin.Transport) {
|
||||
return fmt.Errorf("validate admin transport: only supported on gateway")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func validateGatewayStickySession(listener Listener) error {
|
||||
item := listener.StickySession
|
||||
if !item.Enabled {
|
||||
return nil
|
||||
}
|
||||
if listener.Auth.Mode == "" || listener.Auth.Mode == "none" {
|
||||
return fmt.Errorf("validate gateway stickySession: requires authentication")
|
||||
}
|
||||
if !validCanonicalHeaderName(item.Header) {
|
||||
return fmt.Errorf("validate gateway stickySession.header: must be a canonical HTTP header name")
|
||||
}
|
||||
if item.TTL.Value() <= 0 {
|
||||
return fmt.Errorf("validate gateway stickySession.ttl: must be greater than zero")
|
||||
}
|
||||
if item.MaxEntries <= 0 || item.MaxEntries > MaximumPoolSize {
|
||||
return fmt.Errorf("validate gateway stickySession.maxEntries: must be in [1, %d]", MaximumPoolSize)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func validateGatewayTransport(listener Listener) error {
|
||||
item := listener.Transport
|
||||
for _, timeout := range []struct {
|
||||
name string
|
||||
value Duration
|
||||
}{
|
||||
{name: "dialTimeout", value: item.DialTimeout},
|
||||
{name: "handshakeTimeout", value: item.HandshakeTimeout},
|
||||
{name: "responseHeaderTimeout", value: item.ResponseHeaderTimeout},
|
||||
{name: "idleConnTimeout", value: item.IdleConnTimeout},
|
||||
{name: "tunnelIdleTimeout", value: item.TunnelIdleTimeout},
|
||||
} {
|
||||
if timeout.value.Value() < 0 {
|
||||
return fmt.Errorf("validate gateway transport.%s: must be non-negative", timeout.name)
|
||||
}
|
||||
}
|
||||
for _, limit := range []struct {
|
||||
name string
|
||||
value int
|
||||
}{
|
||||
{name: "maxIdleConns", value: item.MaxIdleConns},
|
||||
{name: "maxIdleConnsPerHost", value: item.MaxIdleConnsPerHost},
|
||||
{name: "maxConnsPerHost", value: item.MaxConnsPerHost},
|
||||
{name: "tunnelBufferBytes", value: item.TunnelBufferBytes},
|
||||
} {
|
||||
if limit.value < 0 || limit.value > MaximumPoolSize {
|
||||
return fmt.Errorf("validate gateway transport.%s: must be in [0, %d]", limit.name, MaximumPoolSize)
|
||||
}
|
||||
}
|
||||
if item.MaxIdleConns > 0 && item.MaxIdleConnsPerHost > item.MaxIdleConns {
|
||||
return fmt.Errorf("validate gateway transport.maxIdleConnsPerHost: must not exceed maxIdleConns")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func gatewayTransportConfigured(item GatewayTransport) bool {
|
||||
return item.DialTimeout != 0 ||
|
||||
item.HandshakeTimeout != 0 ||
|
||||
item.ResponseHeaderTimeout != 0 ||
|
||||
item.IdleConnTimeout != 0 ||
|
||||
item.MaxIdleConns != 0 ||
|
||||
item.MaxIdleConnsPerHost != 0 ||
|
||||
item.MaxConnsPerHost != 0 ||
|
||||
item.TunnelBufferBytes != 0 ||
|
||||
item.TunnelIdleTimeout != 0
|
||||
}
|
||||
|
||||
func validCanonicalHeaderName(value string) bool {
|
||||
if value == "" || http.CanonicalHeaderKey(value) != value {
|
||||
return false
|
||||
}
|
||||
for _, character := range []byte(value) {
|
||||
if character >= 'a' && character <= 'z' || character >= 'A' && character <= 'Z' ||
|
||||
character >= '0' && character <= '9' || strings.ContainsRune("!#$%&'*+-.^_|~", rune(character)) ||
|
||||
character == 96 {
|
||||
continue
|
||||
}
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func validateControlPlane(item ControlPlane) error {
|
||||
if err := validateClientTLS("gatewayTLS", item.GatewayTLS); err != nil {
|
||||
return err
|
||||
@ -423,28 +302,6 @@ func validateRouting(index int, route Routing, upstreams map[string]Upstream, se
|
||||
return fmt.Errorf("validate %s pathRegex: %w", scope, err)
|
||||
}
|
||||
}
|
||||
action := routingAction(route.Action)
|
||||
if err := validateEnum(scope+" action", action, "proxy", "direct", "reject"); err != nil {
|
||||
return err
|
||||
}
|
||||
if action != "proxy" {
|
||||
if route.Purpose != "gateway" {
|
||||
return fmt.Errorf("validate %s action: only supported on gateway", scope)
|
||||
}
|
||||
if len(route.Upstreams) != 0 {
|
||||
return fmt.Errorf("validate %s action: direct and reject routes must not reference upstreams", scope)
|
||||
}
|
||||
if routingStrategyConfigured(route.Strategy) {
|
||||
return fmt.Errorf("validate %s action: direct and reject routes must not configure strategy", scope)
|
||||
}
|
||||
if route.OnUnavailable.Action != "" || route.OnUnavailable.WaitTimeout != 0 {
|
||||
return fmt.Errorf("validate %s action: direct and reject routes must not configure onUnavailable", scope)
|
||||
}
|
||||
if len(route.Check.Targets) != 0 {
|
||||
return fmt.Errorf("validate %s action: direct and reject routes must not configure check targets", scope)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
for _, upstream := range route.Upstreams {
|
||||
if _, ok := upstreams[upstream]; !ok {
|
||||
return fmt.Errorf("validate %s: upstream %q does not exist", scope, upstream)
|
||||
@ -468,20 +325,6 @@ func validateRouting(index int, route Routing, upstreams map[string]Upstream, se
|
||||
return nil
|
||||
}
|
||||
|
||||
func routingAction(value string) string {
|
||||
if value == "" {
|
||||
return "proxy"
|
||||
}
|
||||
return value
|
||||
}
|
||||
|
||||
func routingStrategyConfigured(strategy Strategy) bool {
|
||||
return strategy.Type != "" ||
|
||||
strategy.SwitchAfterEmptyFetch != 0 ||
|
||||
strategy.EndBehavior != "" ||
|
||||
len(strategy.Weights) != 0
|
||||
}
|
||||
|
||||
func validateStrategy(scope string, upstreams []string, strategy Strategy) error {
|
||||
if err := validateEnum(scope+" strategy.type", strategy.Type,
|
||||
"sequential", "random", "roundRobin", "weighted", "leastConnections"); err != nil {
|
||||
@ -529,20 +372,8 @@ func validateStrategy(scope string, upstreams []string, strategy Strategy) error
|
||||
}
|
||||
|
||||
func validateListenerAuth(listener string, auth Auth) error {
|
||||
if err := authorization.Validate(auth.Permissions); err != nil {
|
||||
return fmt.Errorf("validate %s auth.permissions: %w", listener, err)
|
||||
}
|
||||
if err := auth.ClientPolicy.Validate(); err != nil {
|
||||
return fmt.Errorf("validate %s auth.client: %w", listener, err)
|
||||
}
|
||||
switch auth.Mode {
|
||||
case "none":
|
||||
if len(auth.Permissions) != 0 {
|
||||
return fmt.Errorf("validate %s auth.permissions: requires an authentication mode", listener)
|
||||
}
|
||||
if !auth.ClientPolicy.IsZero() {
|
||||
return fmt.Errorf("validate %s auth.client: requires an authentication mode", listener)
|
||||
}
|
||||
return nil
|
||||
case "usernamePassword":
|
||||
if auth.Username == "" || (auth.Password == "" && auth.PasswordFile == "") {
|
||||
@ -564,12 +395,6 @@ func validateListenerAuth(listener string, auth Auth) error {
|
||||
return err
|
||||
}
|
||||
case "any":
|
||||
if len(auth.Permissions) != 0 {
|
||||
return fmt.Errorf("validate %s auth.permissions: configure permissions on auth.methods", listener)
|
||||
}
|
||||
if !auth.ClientPolicy.IsZero() {
|
||||
return fmt.Errorf("validate %s auth.client: configure client policy on auth.methods", listener)
|
||||
}
|
||||
if len(auth.Methods) == 0 {
|
||||
return fmt.Errorf("validate %s auth.mode any: methods are required", listener)
|
||||
}
|
||||
@ -585,12 +410,6 @@ func validateListenerAuth(listener string, auth Auth) error {
|
||||
}
|
||||
|
||||
func validateAuthMethod(listener string, index int, method AuthMethod) error {
|
||||
if err := authorization.Validate(method.Permissions); err != nil {
|
||||
return fmt.Errorf("validate %s auth.methods[%d].permissions: %w", listener, index, err)
|
||||
}
|
||||
if err := method.ClientPolicy.Validate(); err != nil {
|
||||
return fmt.Errorf("validate %s auth.methods[%d].client: %w", listener, index, err)
|
||||
}
|
||||
switch method.Mode {
|
||||
case "usernamePassword":
|
||||
if method.Username == "" || (method.Password == "" && method.PasswordFile == "") {
|
||||
@ -617,76 +436,6 @@ func validateAuthMethod(listener string, index int, method AuthMethod) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func validateNoClientPolicy(listener string, auth Auth) error {
|
||||
if !auth.ClientPolicy.IsZero() {
|
||||
return fmt.Errorf("validate %s auth.client: client extraction policy is only supported on distribution", listener)
|
||||
}
|
||||
for index, method := range auth.Methods {
|
||||
if !method.ClientPolicy.IsZero() {
|
||||
return fmt.Errorf("validate %s auth.methods[%d].client: client extraction policy is only supported on distribution", listener, index)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func validateDistributionClientPolicies(auth Auth, upstreams map[string]Upstream) error {
|
||||
if err := validateDistributionClientPolicy("distribution auth.client", auth.ClientPolicy, upstreams); err != nil {
|
||||
return err
|
||||
}
|
||||
for index, method := range auth.Methods {
|
||||
if err := validateDistributionClientPolicy(
|
||||
fmt.Sprintf("distribution auth.methods[%d].client", index), method.ClientPolicy, upstreams,
|
||||
); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func validateDistributionClientPolicy(scope string, policy clientpolicy.Policy, upstreams map[string]Upstream) error {
|
||||
if len(policy.AllowedRoutings) != 0 || policy.RequestsPerMinute != 0 || policy.MaxConcurrentConnections != 0 {
|
||||
return fmt.Errorf("validate %s: routing access and Gateway limits are only supported on gateway", scope)
|
||||
}
|
||||
for _, upstream := range policy.AllowedUpstreams {
|
||||
if _, exists := upstreams[upstream]; !exists {
|
||||
return fmt.Errorf("validate %s.allowedUpstreams: upstream %q does not exist", scope, upstream)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func validateGatewayClientPolicies(auth Auth, routings map[string]struct{}) error {
|
||||
if err := validateGatewayClientPolicy("gateway auth.client", auth.ClientPolicy, routings); err != nil {
|
||||
return err
|
||||
}
|
||||
for index, method := range auth.Methods {
|
||||
if err := validateGatewayClientPolicy(
|
||||
fmt.Sprintf("gateway auth.methods[%d].client", index), method.ClientPolicy, routings,
|
||||
); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func validateGatewayClientPolicy(scope string, policy clientpolicy.Policy, routings map[string]struct{}) error {
|
||||
if policy.MaxExtractCount != 0 || len(policy.AllowedUpstreams) != 0 || len(policy.AllowedRegions) != 0 {
|
||||
return fmt.Errorf("validate %s: extraction constraints are only supported on distribution", scope)
|
||||
}
|
||||
if int64(policy.RequestsPerMinute) > MaximumExactCounter {
|
||||
return fmt.Errorf("validate %s.requestsPerMinute: exceeds exact counter range", scope)
|
||||
}
|
||||
if int64(policy.MaxConcurrentConnections) > MaximumExactCounter {
|
||||
return fmt.Errorf("validate %s.maxConcurrentConnections: exceeds exact counter range", scope)
|
||||
}
|
||||
for _, routing := range policy.AllowedRoutings {
|
||||
if _, exists := routings[routing]; !exists {
|
||||
return fmt.Errorf("validate %s.allowedRoutings: routing %q does not exist or is not an enabled gateway routing", scope, routing)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func validateCIDRs(name string, cidrs []string) error {
|
||||
for _, cidr := range cidrs {
|
||||
if _, _, err := net.ParseCIDR(cidr); err != nil {
|
||||
|
||||
@ -5,24 +5,18 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"proxy-pool/internal/domain/adminstate"
|
||||
"proxy-pool/internal/domain/authorization"
|
||||
"proxy-pool/internal/platform/httpapi"
|
||||
"proxy-pool/internal/platform/httpsecurity"
|
||||
)
|
||||
|
||||
const (
|
||||
statusPath = "/api/v1/status"
|
||||
auditPath = "/api/v1/audit"
|
||||
reloadPath = "/api/v1/config/reload"
|
||||
upstreamPrefix = "/api/v1/upstreams/"
|
||||
routingPrefix = "/api/v1/routing/"
|
||||
maxResourceNameBytes = 128
|
||||
defaultAuditPageSize = 100
|
||||
)
|
||||
|
||||
var (
|
||||
@ -35,7 +29,6 @@ var (
|
||||
|
||||
type Service interface {
|
||||
Status(context.Context) (Status, error)
|
||||
ReadAudit(context.Context, AuditQuery) (AuditPage, error)
|
||||
SetUpstreamEnabled(context.Context, SetUpstreamCommand) (MutationResult, error)
|
||||
SwitchRouting(context.Context, SwitchCommand) (MutationResult, error)
|
||||
ReloadConfiguration(context.Context, ReloadCommand) (MutationResult, error)
|
||||
@ -85,36 +78,6 @@ type MutationResult struct {
|
||||
Message string `json:"message,omitempty"`
|
||||
}
|
||||
|
||||
type AuditQuery struct {
|
||||
AfterID uint64
|
||||
Limit int
|
||||
}
|
||||
|
||||
func (query AuditQuery) Validate() error {
|
||||
if query.Limit <= 0 || query.Limit > adminstate.MaxPageSize {
|
||||
return ErrInvalidConfiguration
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
type AuditRecord struct {
|
||||
ID uint64 `json:"id"`
|
||||
RequestID string `json:"requestId"`
|
||||
ActorID string `json:"actorId"`
|
||||
SourceIP string `json:"sourceIp,omitempty"`
|
||||
Action string `json:"action"`
|
||||
ResourceType string `json:"resourceType"`
|
||||
ResourceName string `json:"resourceName"`
|
||||
Changed bool `json:"changed"`
|
||||
Version uint64 `json:"version"`
|
||||
Reason string `json:"reason,omitempty"`
|
||||
OccurredAt time.Time `json:"occurredAt"`
|
||||
}
|
||||
|
||||
type AuditPage struct {
|
||||
Records []AuditRecord `json:"records"`
|
||||
}
|
||||
|
||||
type SetUpstreamCommand struct {
|
||||
RequestID string
|
||||
ActorID string
|
||||
@ -171,27 +134,12 @@ func (handler *Handler) ServeHTTP(writer http.ResponseWriter, request *http.Requ
|
||||
if !requireMethod(writer, request, http.MethodGet, requestID) {
|
||||
return
|
||||
}
|
||||
if !requirePermission(writer, identity, authorization.AdminRead, requestID) {
|
||||
return
|
||||
}
|
||||
handler.getStatus(writer, request, requestID)
|
||||
return
|
||||
case auditPath:
|
||||
if !requireMethod(writer, request, http.MethodGet, requestID) {
|
||||
return
|
||||
}
|
||||
if !requirePermission(writer, identity, authorization.AdminRead, requestID) {
|
||||
return
|
||||
}
|
||||
handler.readAudit(writer, request, requestID)
|
||||
return
|
||||
case reloadPath:
|
||||
if !requireMethod(writer, request, http.MethodPost, requestID) {
|
||||
return
|
||||
}
|
||||
if !requirePermission(writer, identity, authorization.AdminWrite, requestID) {
|
||||
return
|
||||
}
|
||||
handler.reload(writer, request, requestID, identity)
|
||||
return
|
||||
}
|
||||
@ -200,9 +148,6 @@ func (handler *Handler) ServeHTTP(writer http.ResponseWriter, request *http.Requ
|
||||
if !requireMethod(writer, request, http.MethodPost, requestID) {
|
||||
return
|
||||
}
|
||||
if !requirePermission(writer, identity, authorization.AdminWrite, requestID) {
|
||||
return
|
||||
}
|
||||
handler.setUpstreamEnabled(writer, request, name, action == "enable", requestID, identity)
|
||||
return
|
||||
}
|
||||
@ -210,9 +155,6 @@ func (handler *Handler) ServeHTTP(writer http.ResponseWriter, request *http.Requ
|
||||
if !requireMethod(writer, request, http.MethodPost, requestID) {
|
||||
return
|
||||
}
|
||||
if !requirePermission(writer, identity, authorization.AdminWrite, requestID) {
|
||||
return
|
||||
}
|
||||
handler.switchRouting(writer, request, name, requestID, identity)
|
||||
return
|
||||
}
|
||||
@ -220,19 +162,6 @@ func (handler *Handler) ServeHTTP(writer http.ResponseWriter, request *http.Requ
|
||||
writeTransportProblem(writer, http.StatusNotFound, "NOT_FOUND", "Not found", "the requested endpoint does not exist", requestID)
|
||||
}
|
||||
|
||||
func requirePermission(
|
||||
writer http.ResponseWriter,
|
||||
identity httpsecurity.Identity,
|
||||
required string,
|
||||
requestID string,
|
||||
) bool {
|
||||
if identity.Allows(required) {
|
||||
return true
|
||||
}
|
||||
writeTransportProblem(writer, http.StatusForbidden, "FORBIDDEN", "Forbidden", "the authenticated identity lacks endpoint permission", requestID)
|
||||
return false
|
||||
}
|
||||
|
||||
func (handler *Handler) getStatus(writer http.ResponseWriter, request *http.Request, requestID string) {
|
||||
status, err := handler.service.Status(request.Context())
|
||||
if err != nil {
|
||||
@ -249,57 +178,6 @@ func (handler *Handler) getStatus(writer http.ResponseWriter, request *http.Requ
|
||||
_ = httpapi.WriteJSON(writer, http.StatusOK, status)
|
||||
}
|
||||
|
||||
func (handler *Handler) readAudit(writer http.ResponseWriter, request *http.Request, requestID string) {
|
||||
query, err := parseAuditQuery(request)
|
||||
if err != nil {
|
||||
writeTransportProblem(writer, http.StatusBadRequest, "INVALID_AUDIT_QUERY", "Invalid audit query", "audit pagination fields violate the API contract", requestID)
|
||||
return
|
||||
}
|
||||
page, err := handler.service.ReadAudit(request.Context(), query)
|
||||
if err != nil {
|
||||
writeServiceProblem(writer, err, requestID)
|
||||
return
|
||||
}
|
||||
if page.Records == nil {
|
||||
page.Records = []AuditRecord{}
|
||||
}
|
||||
writer.Header().Set(httpapi.HeaderRequestID, requestID)
|
||||
_ = httpapi.WriteJSON(writer, http.StatusOK, page)
|
||||
}
|
||||
|
||||
func parseAuditQuery(request *http.Request) (AuditQuery, error) {
|
||||
if request == nil || request.URL == nil {
|
||||
return AuditQuery{}, ErrInvalidConfiguration
|
||||
}
|
||||
values := request.URL.Query()
|
||||
query := AuditQuery{Limit: defaultAuditPageSize}
|
||||
for name, value := range values {
|
||||
if len(value) != 1 {
|
||||
return AuditQuery{}, ErrInvalidConfiguration
|
||||
}
|
||||
switch name {
|
||||
case "afterId":
|
||||
parsed, err := strconv.ParseUint(value[0], 10, 64)
|
||||
if err != nil {
|
||||
return AuditQuery{}, ErrInvalidConfiguration
|
||||
}
|
||||
query.AfterID = parsed
|
||||
case "limit":
|
||||
parsed, err := strconv.ParseUint(value[0], 10, 32)
|
||||
if err != nil || parsed == 0 || parsed > adminstate.MaxPageSize {
|
||||
return AuditQuery{}, ErrInvalidConfiguration
|
||||
}
|
||||
query.Limit = int(parsed)
|
||||
default:
|
||||
return AuditQuery{}, ErrInvalidConfiguration
|
||||
}
|
||||
}
|
||||
if err := query.Validate(); err != nil {
|
||||
return AuditQuery{}, err
|
||||
}
|
||||
return query, nil
|
||||
}
|
||||
|
||||
func (handler *Handler) setUpstreamEnabled(
|
||||
writer http.ResponseWriter,
|
||||
request *http.Request,
|
||||
|
||||
@ -8,9 +8,7 @@ import (
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"proxy-pool/internal/domain/authorization"
|
||||
"proxy-pool/internal/platform/httpapi"
|
||||
"proxy-pool/internal/platform/httpsecurity"
|
||||
)
|
||||
@ -40,109 +38,6 @@ func TestHandlerReturnsStatusWithoutSensitiveDetails(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandlerEnforcesReadAndWritePermissions(t *testing.T) {
|
||||
t.Parallel()
|
||||
service := &stubService{}
|
||||
readHandler, err := NewHandler(service, scopedAuthorizer{permissions: []string{authorization.AdminRead}}, Options{MaxBodyBytes: 1024})
|
||||
if err != nil {
|
||||
t.Fatalf("NewHandler(read) error = %v", err)
|
||||
}
|
||||
status := httptest.NewRecorder()
|
||||
readHandler.ServeHTTP(status, httptest.NewRequest(http.MethodGet, "/api/v1/status", nil))
|
||||
if status.Code != http.StatusOK || service.statusCalls != 1 {
|
||||
t.Fatalf("read status = %d calls=%d", status.Code, service.statusCalls)
|
||||
}
|
||||
mutation := httptest.NewRecorder()
|
||||
readHandler.ServeHTTP(mutation, httptest.NewRequest(http.MethodPost, "/api/v1/config/reload", nil))
|
||||
if mutation.Code != http.StatusForbidden || service.lastReload != (ReloadCommand{}) || !strings.Contains(mutation.Body.String(), `"code":"FORBIDDEN"`) {
|
||||
t.Fatalf("read mutation response = status %d command %+v body=%s", mutation.Code, service.lastReload, mutation.Body.String())
|
||||
}
|
||||
|
||||
writeHandler, err := NewHandler(service, scopedAuthorizer{permissions: []string{authorization.AdminWrite}}, Options{MaxBodyBytes: 1024})
|
||||
if err != nil {
|
||||
t.Fatalf("NewHandler(write) error = %v", err)
|
||||
}
|
||||
audit := httptest.NewRecorder()
|
||||
writeHandler.ServeHTTP(audit, httptest.NewRequest(http.MethodGet, "/api/v1/audit", nil))
|
||||
if audit.Code != http.StatusForbidden || service.auditCalls != 0 || !strings.Contains(audit.Body.String(), `"code":"FORBIDDEN"`) {
|
||||
t.Fatalf("write audit response = status %d calls=%d body=%s", audit.Code, service.auditCalls, audit.Body.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandlerReadsBoundedAuditPage(t *testing.T) {
|
||||
t.Parallel()
|
||||
occurredAt := time.Date(2026, 8, 2, 9, 30, 0, 0, time.UTC)
|
||||
service := &stubService{audit: AuditPage{Records: []AuditRecord{{
|
||||
ID: 8, RequestID: "req-8", ActorID: "admin:alice", Action: "switch_routing",
|
||||
ResourceType: "routing", ResourceName: "checkout", Changed: true, Version: 12,
|
||||
Reason: "capacity", OccurredAt: occurredAt,
|
||||
}}}}
|
||||
handler := mustHandler(t, service)
|
||||
recorder := httptest.NewRecorder()
|
||||
request := httptest.NewRequest(http.MethodGet, "/api/v1/audit?afterId=7&limit=2", nil)
|
||||
request.Header.Set(httpapi.HeaderRequestID, "req-audit-page")
|
||||
|
||||
handler.ServeHTTP(recorder, request)
|
||||
|
||||
if recorder.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, want %d; body=%s", recorder.Code, http.StatusOK, recorder.Body.String())
|
||||
}
|
||||
if service.auditCalls != 1 || service.lastAudit != (AuditQuery{AfterID: 7, Limit: 2}) {
|
||||
t.Fatalf("ReadAudit() calls=%d query=%+v", service.auditCalls, service.lastAudit)
|
||||
}
|
||||
if requestID := recorder.Header().Get(httpapi.HeaderRequestID); requestID != "req-audit-page" {
|
||||
t.Fatalf("response request ID = %q, want req-audit-page", requestID)
|
||||
}
|
||||
var response AuditPage
|
||||
if err := json.Unmarshal(recorder.Body.Bytes(), &response); err != nil {
|
||||
t.Fatalf("decode response: %v", err)
|
||||
}
|
||||
if len(response.Records) != 1 || response.Records[0].ID != 8 || !response.Records[0].OccurredAt.Equal(occurredAt) {
|
||||
t.Fatalf("unexpected audit response: %+v", response)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandlerUsesDefaultAuditLimitAndRejectsInvalidAuditQueries(t *testing.T) {
|
||||
t.Parallel()
|
||||
tests := []struct {
|
||||
name string
|
||||
path string
|
||||
wantStatus int
|
||||
wantQuery AuditQuery
|
||||
}{
|
||||
{name: "default", path: "/api/v1/audit", wantStatus: http.StatusOK, wantQuery: AuditQuery{Limit: defaultAuditPageSize}},
|
||||
{name: "unknown field", path: "/api/v1/audit?beforeId=1", wantStatus: http.StatusBadRequest},
|
||||
{name: "duplicate field", path: "/api/v1/audit?limit=1&limit=2", wantStatus: http.StatusBadRequest},
|
||||
{name: "zero limit", path: "/api/v1/audit?limit=0", wantStatus: http.StatusBadRequest},
|
||||
{name: "oversized limit", path: "/api/v1/audit?limit=1001", wantStatus: http.StatusBadRequest},
|
||||
{name: "invalid cursor", path: "/api/v1/audit?afterId=nope", wantStatus: http.StatusBadRequest},
|
||||
}
|
||||
for _, test := range tests {
|
||||
test := test
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
service := &stubService{}
|
||||
handler := mustHandler(t, service)
|
||||
recorder := httptest.NewRecorder()
|
||||
|
||||
handler.ServeHTTP(recorder, httptest.NewRequest(http.MethodGet, test.path, nil))
|
||||
|
||||
if recorder.Code != test.wantStatus {
|
||||
t.Fatalf("status = %d, want %d; body=%s", recorder.Code, test.wantStatus, recorder.Body.String())
|
||||
}
|
||||
if test.wantStatus == http.StatusOK {
|
||||
if service.auditCalls != 1 || service.lastAudit != test.wantQuery {
|
||||
t.Fatalf("ReadAudit() calls=%d query=%+v, want %+v", service.auditCalls, service.lastAudit, test.wantQuery)
|
||||
}
|
||||
return
|
||||
}
|
||||
if service.auditCalls != 0 || !strings.Contains(recorder.Body.String(), `"code":"INVALID_AUDIT_QUERY"`) {
|
||||
t.Fatalf("invalid audit query reached service or returned wrong problem: calls=%d body=%s", service.auditCalls, recorder.Body.String())
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewHandlerRejectsMissingDependenciesAndInvalidLimit(t *testing.T) {
|
||||
t.Parallel()
|
||||
if _, err := NewHandler(nil, allowAuthorizer{}, Options{MaxBodyBytes: 1024}); !errors.Is(err, ErrInvalidHandler) {
|
||||
@ -337,7 +232,6 @@ func TestHandlerRejectsInvalidTransportRequests(t *testing.T) {
|
||||
wantStatus int
|
||||
}{
|
||||
{name: "method", method: http.MethodPut, path: "/api/v1/config/reload", wantStatus: http.StatusMethodNotAllowed},
|
||||
{name: "audit method", method: http.MethodPost, path: "/api/v1/audit", wantStatus: http.StatusMethodNotAllowed},
|
||||
{name: "unknown route", method: http.MethodGet, path: "/missing", wantStatus: http.StatusNotFound},
|
||||
{name: "invalid name", method: http.MethodPost, path: "/api/v1/upstreams//enable", wantStatus: http.StatusNotFound},
|
||||
{name: "unknown JSON field", method: http.MethodPost, path: "/api/v1/routing/r/switch", body: `{"expectedCurrent":"a","target":"b","extra":1}`, content: httpapi.JSONContentType, wantStatus: http.StatusBadRequest},
|
||||
@ -383,15 +277,12 @@ func mustHandler(t *testing.T, service Service) *Handler {
|
||||
|
||||
type stubService struct {
|
||||
status Status
|
||||
audit AuditPage
|
||||
mutation MutationResult
|
||||
err error
|
||||
lastAudit AuditQuery
|
||||
lastUpstream SetUpstreamCommand
|
||||
lastSwitch SwitchCommand
|
||||
lastReload ReloadCommand
|
||||
statusCalls int
|
||||
auditCalls int
|
||||
}
|
||||
|
||||
func (service *stubService) Status(context.Context) (Status, error) {
|
||||
@ -399,12 +290,6 @@ func (service *stubService) Status(context.Context) (Status, error) {
|
||||
return service.status, service.err
|
||||
}
|
||||
|
||||
func (service *stubService) ReadAudit(_ context.Context, query AuditQuery) (AuditPage, error) {
|
||||
service.auditCalls++
|
||||
service.lastAudit = query
|
||||
return service.audit, service.err
|
||||
}
|
||||
|
||||
func (service *stubService) SetUpstreamEnabled(_ context.Context, command SetUpstreamCommand) (MutationResult, error) {
|
||||
service.lastUpstream = command
|
||||
return service.mutation, service.err
|
||||
@ -426,12 +311,6 @@ func (allowAuthorizer) Resolve(*http.Request) (httpsecurity.Identity, error) {
|
||||
return httpsecurity.Identity{ClientID: "admin:test", SourceIP: "192.0.2.10"}, nil
|
||||
}
|
||||
|
||||
type scopedAuthorizer struct{ permissions []string }
|
||||
|
||||
func (authorizer scopedAuthorizer) Resolve(*http.Request) (httpsecurity.Identity, error) {
|
||||
return httpsecurity.Identity{ClientID: "admin:scoped", SourceIP: "192.0.2.20", Permissions: authorizer.permissions}, nil
|
||||
}
|
||||
|
||||
type rejectAuthorizer struct{}
|
||||
|
||||
func (rejectAuthorizer) Resolve(*http.Request) (httpsecurity.Identity, error) {
|
||||
|
||||
@ -16,7 +16,6 @@ var ErrInvalidApplicationService = errors.New("invalid admin application service
|
||||
type StateRepository interface {
|
||||
adminstate.Mutator
|
||||
adminstate.SnapshotReader
|
||||
adminstate.AuditReader
|
||||
}
|
||||
|
||||
type OperationalStatusReader interface {
|
||||
@ -38,12 +37,6 @@ type RuntimeController interface {
|
||||
ValidateUpstream(context.Context, string) error
|
||||
}
|
||||
|
||||
// SnapshotRefreshNotifier requests immediate full Worker snapshots after a
|
||||
// committed management change. Implementations must coalesce notifications.
|
||||
type SnapshotRefreshNotifier interface {
|
||||
NotifySnapshotRefresh()
|
||||
}
|
||||
|
||||
var _ ConfigurationPublisher = (*config.Store)(nil)
|
||||
|
||||
type ApplicationDependencies struct {
|
||||
@ -52,7 +45,6 @@ type ApplicationDependencies struct {
|
||||
Configuration ConfigurationLoader
|
||||
Publisher ConfigurationPublisher
|
||||
Runtime RuntimeController
|
||||
SnapshotRefresh SnapshotRefreshNotifier
|
||||
}
|
||||
|
||||
type ApplicationOptions struct {
|
||||
@ -88,7 +80,6 @@ type ApplicationService struct {
|
||||
configuration ConfigurationLoader
|
||||
publisher ConfigurationPublisher
|
||||
runtime RuntimeController
|
||||
snapshotRefresh SnapshotRefreshNotifier
|
||||
now func() time.Time
|
||||
fingerprintKey []byte
|
||||
}
|
||||
@ -107,7 +98,6 @@ func NewApplicationService(dependencies ApplicationDependencies, options Applica
|
||||
configuration: dependencies.Configuration,
|
||||
publisher: dependencies.Publisher,
|
||||
runtime: dependencies.Runtime,
|
||||
snapshotRefresh: dependencies.SnapshotRefresh,
|
||||
now: options.Now,
|
||||
fingerprintKey: append([]byte(nil), options.FingerprintKey...),
|
||||
}, nil
|
||||
@ -129,9 +119,6 @@ func (service *ApplicationService) SetUpstreamEnabled(ctx context.Context, comma
|
||||
if err == nil && service.runtime != nil {
|
||||
service.runtime.Notify()
|
||||
}
|
||||
if err == nil && result.Changed && service.snapshotRefresh != nil {
|
||||
service.snapshotRefresh.NotifySnapshotRefresh()
|
||||
}
|
||||
return mutationResult(result), mapAdminStateError(err)
|
||||
}
|
||||
|
||||
@ -145,42 +132,9 @@ func (service *ApplicationService) SwitchRouting(ctx context.Context, command Sw
|
||||
Target: command.Target,
|
||||
Reason: command.Reason,
|
||||
})
|
||||
if err == nil && result.Changed && service.snapshotRefresh != nil {
|
||||
service.snapshotRefresh.NotifySnapshotRefresh()
|
||||
}
|
||||
return mutationResult(result), mapAdminStateError(err)
|
||||
}
|
||||
|
||||
// ReadAudit returns one bounded, stable page from the authoritative management
|
||||
// audit log. It deliberately exposes only management-plane records and never
|
||||
// reads Proxy activity, credentials, or Provider payloads.
|
||||
func (service *ApplicationService) ReadAudit(ctx context.Context, query AuditQuery) (AuditPage, error) {
|
||||
if service == nil || nilInterface(service.state) || query.Validate() != nil {
|
||||
return AuditPage{}, ErrInvalidConfiguration
|
||||
}
|
||||
records, err := service.state.ReadAudit(ctx, adminstate.AuditQuery{AfterID: query.AfterID, Limit: query.Limit})
|
||||
if err != nil {
|
||||
return AuditPage{}, mapAdminStateError(err)
|
||||
}
|
||||
page := AuditPage{Records: make([]AuditRecord, 0, len(records))}
|
||||
for _, record := range records {
|
||||
page.Records = append(page.Records, AuditRecord{
|
||||
ID: record.ID,
|
||||
RequestID: record.RequestID,
|
||||
ActorID: record.Actor.ID,
|
||||
SourceIP: record.Actor.SourceIP,
|
||||
Action: string(record.Action),
|
||||
ResourceType: record.ResourceType,
|
||||
ResourceName: record.ResourceName,
|
||||
Changed: record.Changed,
|
||||
Version: record.Revision,
|
||||
Reason: record.Reason,
|
||||
OccurredAt: record.OccurredAt.UTC(),
|
||||
})
|
||||
}
|
||||
return page, nil
|
||||
}
|
||||
|
||||
func (service *ApplicationService) Status(ctx context.Context) (Status, error) {
|
||||
snapshot, err := service.state.Snapshot(ctx)
|
||||
if err != nil {
|
||||
@ -302,9 +256,6 @@ func (service *ApplicationService) ApplyConfiguration(
|
||||
if published && service.runtime != nil {
|
||||
service.runtime.Notify()
|
||||
}
|
||||
if published && service.snapshotRefresh != nil {
|
||||
service.snapshotRefresh.NotifySnapshotRefresh()
|
||||
}
|
||||
return mutationResult(result), nil
|
||||
}
|
||||
|
||||
|
||||
@ -29,14 +29,12 @@ func TestApplicationServiceMapsUpstreamMutationToAdminState(t *testing.T) {
|
||||
},
|
||||
}
|
||||
runtime := &recordingRuntimeNotifier{}
|
||||
refresh := &recordingSnapshotRefreshNotifier{}
|
||||
service, err := NewApplicationService(ApplicationDependencies{
|
||||
State: state,
|
||||
Operations: staticOperationalStatusReader{},
|
||||
Configuration: staticConfigurationLoader{},
|
||||
Publisher: &recordingConfigurationPublisher{},
|
||||
Runtime: runtime,
|
||||
SnapshotRefresh: refresh,
|
||||
}, applicationTestOptions(func() time.Time { return now }))
|
||||
if err != nil {
|
||||
t.Fatalf("NewApplicationService() error = %v", err)
|
||||
@ -67,9 +65,6 @@ func TestApplicationServiceMapsUpstreamMutationToAdminState(t *testing.T) {
|
||||
if runtime.notifications != 1 {
|
||||
t.Fatalf("runtime notifications = %d, want 1", runtime.notifications)
|
||||
}
|
||||
if refresh.notifications != 1 {
|
||||
t.Fatalf("snapshot refresh notifications = %d, want 1", refresh.notifications)
|
||||
}
|
||||
}
|
||||
|
||||
func TestApplicationServicePreflightsProviderRuntimeBeforeMutation(t *testing.T) {
|
||||
@ -152,14 +147,7 @@ func TestApplicationServiceMapsRoutingSwitchAndDomainErrors(t *testing.T) {
|
||||
t.Parallel()
|
||||
now := time.Date(2026, 7, 29, 11, 0, 0, 0, time.FixedZone("test", 8*60*60))
|
||||
state := &recordingAdminState{mutation: adminstate.MutationResult{RequestID: "req-switch", Changed: true, Revision: 21}}
|
||||
refresh := &recordingSnapshotRefreshNotifier{}
|
||||
service, serviceErr := NewApplicationService(ApplicationDependencies{
|
||||
State: state, Operations: staticOperationalStatusReader{}, Configuration: staticConfigurationLoader{},
|
||||
Publisher: &recordingConfigurationPublisher{}, SnapshotRefresh: refresh,
|
||||
}, applicationTestOptions(func() time.Time { return now }))
|
||||
if serviceErr != nil {
|
||||
t.Fatalf("NewApplicationService() = %v", serviceErr)
|
||||
}
|
||||
service := mustApplicationService(t, state, applicationTestOptions(func() time.Time { return now }))
|
||||
|
||||
result, err := service.SwitchRouting(context.Background(), SwitchCommand{
|
||||
RequestID: "req-switch", ActorID: "admin:bob", SourceIP: "198.51.100.7",
|
||||
@ -178,9 +166,6 @@ func TestApplicationServiceMapsRoutingSwitchAndDomainErrors(t *testing.T) {
|
||||
if state.lastSwitch != wantCommand {
|
||||
t.Fatalf("admin state command = %+v, want %+v", state.lastSwitch, wantCommand)
|
||||
}
|
||||
if refresh.notifications != 1 {
|
||||
t.Fatalf("snapshot refresh notifications = %d, want 1", refresh.notifications)
|
||||
}
|
||||
|
||||
tests := []struct {
|
||||
domain error
|
||||
@ -201,45 +186,6 @@ func TestApplicationServiceMapsRoutingSwitchAndDomainErrors(t *testing.T) {
|
||||
t.Fatalf("SwitchRouting(%v) error = %v, want mapped %v preserving cause", test.domain, err, test.want)
|
||||
}
|
||||
}
|
||||
if refresh.notifications != 1 {
|
||||
t.Fatalf("snapshot refresh notifications after failed switches = %d, want 1", refresh.notifications)
|
||||
}
|
||||
}
|
||||
|
||||
func TestApplicationServiceReadsAuthoritativeAuditPage(t *testing.T) {
|
||||
t.Parallel()
|
||||
occurredAt := time.Date(2026, 8, 2, 10, 0, 0, 0, time.FixedZone("test", 8*60*60))
|
||||
state := &recordingAdminState{audits: []adminstate.AuditRecord{{
|
||||
ID: 42, RequestID: "req-switch", Actor: adminstate.Actor{ID: "admin:alice", SourceIP: "192.0.2.10"},
|
||||
Action: adminstate.ActionSwitchRoute, ResourceType: "routing", ResourceName: "checkout",
|
||||
Changed: true, Revision: 9, Reason: "capacity", OccurredAt: occurredAt,
|
||||
}}}
|
||||
service := mustApplicationService(t, state, applicationTestOptions(time.Now))
|
||||
|
||||
page, err := service.ReadAudit(context.Background(), AuditQuery{AfterID: 41, Limit: 2})
|
||||
|
||||
if err != nil {
|
||||
t.Fatalf("ReadAudit() error = %v", err)
|
||||
}
|
||||
if state.lastAuditQuery != (adminstate.AuditQuery{AfterID: 41, Limit: 2}) {
|
||||
t.Fatalf("audit query = %+v", state.lastAuditQuery)
|
||||
}
|
||||
if len(page.Records) != 1 {
|
||||
t.Fatalf("audit records = %+v", page.Records)
|
||||
}
|
||||
record := page.Records[0]
|
||||
if record.ID != 42 || record.Action != string(adminstate.ActionSwitchRoute) || record.Version != 9 ||
|
||||
record.ActorID != "admin:alice" || record.SourceIP != "192.0.2.10" || !record.OccurredAt.Equal(occurredAt.UTC()) {
|
||||
t.Fatalf("mapped audit record = %+v", record)
|
||||
}
|
||||
|
||||
state.err = adminstate.ErrUnavailable
|
||||
if _, err := service.ReadAudit(context.Background(), AuditQuery{Limit: 1}); !errors.Is(err, ErrUnavailable) || !errors.Is(err, adminstate.ErrUnavailable) {
|
||||
t.Fatalf("ReadAudit() unavailable error = %v", err)
|
||||
}
|
||||
if _, err := service.ReadAudit(context.Background(), AuditQuery{}); !errors.Is(err, ErrInvalidConfiguration) {
|
||||
t.Fatalf("ReadAudit() invalid query error = %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestApplicationServiceBuildsStatusFromAuthoritativeAndOperationalSnapshots(t *testing.T) {
|
||||
@ -335,7 +281,6 @@ func TestApplicationServiceReloadPersistsManagementViewBeforePublishing(t *testi
|
||||
configuration := validReloadConfiguration()
|
||||
publisher := &recordingConfigurationPublisher{}
|
||||
runtime := &recordingRuntimeNotifier{}
|
||||
refresh := &recordingSnapshotRefreshNotifier{}
|
||||
state := &recordingAdminState{
|
||||
mutation: adminstate.MutationResult{RequestID: "req-reload", Changed: true, Revision: 42},
|
||||
snapshot: adminstate.Snapshot{Routings: []adminstate.RoutingState{
|
||||
@ -352,7 +297,7 @@ func TestApplicationServiceReloadPersistsManagementViewBeforePublishing(t *testi
|
||||
Configuration: staticConfigurationLoader{loaded: LoadedConfiguration{
|
||||
Value: configuration, Source: "configs/proxy-pool.yaml",
|
||||
}},
|
||||
Publisher: publisher, Runtime: runtime, SnapshotRefresh: refresh,
|
||||
Publisher: publisher, Runtime: runtime,
|
||||
}, applicationTestOptions(func() time.Time { return now }))
|
||||
if err != nil {
|
||||
t.Fatalf("NewApplicationService() error = %v", err)
|
||||
@ -373,9 +318,6 @@ func TestApplicationServiceReloadPersistsManagementViewBeforePublishing(t *testi
|
||||
if runtime.notifications != 1 {
|
||||
t.Fatalf("runtime notifications = %d, want 1", runtime.notifications)
|
||||
}
|
||||
if refresh.notifications != 1 {
|
||||
t.Fatalf("snapshot refresh notifications = %d, want 1", refresh.notifications)
|
||||
}
|
||||
command := state.lastConfig
|
||||
if command.RequestID != "req-reload" || command.Actor != (adminstate.Actor{ID: "admin:alice", SourceIP: "192.0.2.10"}) ||
|
||||
!command.OccurredAt.Equal(now) || command.Source != "configs/proxy-pool.yaml" {
|
||||
@ -681,11 +623,8 @@ type recordingAdminState struct {
|
||||
mutation adminstate.MutationResult
|
||||
err error
|
||||
snapshot adminstate.Snapshot
|
||||
audits []adminstate.AuditRecord
|
||||
lastAuditQuery adminstate.AuditQuery
|
||||
lastUpstream adminstate.SetUpstreamCommand
|
||||
lastSwitch adminstate.SwitchRoutingCommand
|
||||
lastDisable adminstate.DisableRoutingCommand
|
||||
lastConfig adminstate.CommitConfigCommand
|
||||
onCommit func(adminstate.CommitConfigCommand)
|
||||
}
|
||||
@ -704,10 +643,6 @@ func (state *orderedCommitState) SwitchRouting(context.Context, adminstate.Switc
|
||||
return adminstate.MutationResult{}, nil
|
||||
}
|
||||
|
||||
func (state *orderedCommitState) DisableRouting(context.Context, adminstate.DisableRoutingCommand) (adminstate.MutationResult, error) {
|
||||
return adminstate.MutationResult{}, nil
|
||||
}
|
||||
|
||||
func (state *orderedCommitState) CommitConfig(_ context.Context, command adminstate.CommitConfigCommand) (adminstate.MutationResult, error) {
|
||||
revision := state.next.Add(1)
|
||||
if revision == 1 {
|
||||
@ -721,10 +656,6 @@ func (*orderedCommitState) Snapshot(context.Context) (adminstate.Snapshot, error
|
||||
return adminstate.Snapshot{}, nil
|
||||
}
|
||||
|
||||
func (*orderedCommitState) ReadAudit(context.Context, adminstate.AuditQuery) ([]adminstate.AuditRecord, error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func (state *recordingAdminState) SetUpstreamEnabled(_ context.Context, command adminstate.SetUpstreamCommand) (adminstate.MutationResult, error) {
|
||||
state.lastUpstream = command
|
||||
return state.mutation, state.err
|
||||
@ -735,11 +666,6 @@ func (state *recordingAdminState) SwitchRouting(_ context.Context, command admin
|
||||
return state.mutation, state.err
|
||||
}
|
||||
|
||||
func (state *recordingAdminState) DisableRouting(_ context.Context, command adminstate.DisableRoutingCommand) (adminstate.MutationResult, error) {
|
||||
state.lastDisable = command
|
||||
return state.mutation, state.err
|
||||
}
|
||||
|
||||
func (state *recordingAdminState) CommitConfig(_ context.Context, command adminstate.CommitConfigCommand) (adminstate.MutationResult, error) {
|
||||
state.lastConfig = command
|
||||
if state.onCommit != nil {
|
||||
@ -752,11 +678,6 @@ func (state *recordingAdminState) Snapshot(context.Context) (adminstate.Snapshot
|
||||
return state.snapshot, state.err
|
||||
}
|
||||
|
||||
func (state *recordingAdminState) ReadAudit(_ context.Context, query adminstate.AuditQuery) ([]adminstate.AuditRecord, error) {
|
||||
state.lastAuditQuery = query
|
||||
return append([]adminstate.AuditRecord(nil), state.audits...), state.err
|
||||
}
|
||||
|
||||
type staticOperationalStatusReader struct {
|
||||
status OperationalStatus
|
||||
err error
|
||||
@ -791,10 +712,6 @@ type recordingRuntimeNotifier struct {
|
||||
validationErr error
|
||||
}
|
||||
|
||||
type recordingSnapshotRefreshNotifier struct{ notifications int }
|
||||
|
||||
func (notifier *recordingSnapshotRefreshNotifier) NotifySnapshotRefresh() { notifier.notifications++ }
|
||||
|
||||
func (notifier *recordingRuntimeNotifier) Notify() { notifier.notifications++ }
|
||||
|
||||
func (notifier *recordingRuntimeNotifier) ValidateConfiguration(context.Context, *config.Config) error {
|
||||
|
||||
@ -19,7 +19,6 @@ import (
|
||||
"proxy-pool/internal/controller/operations"
|
||||
"proxy-pool/internal/controller/pool"
|
||||
"proxy-pool/internal/controller/provider"
|
||||
controllerRouting "proxy-pool/internal/controller/routing"
|
||||
controllerRuntime "proxy-pool/internal/controller/runtime"
|
||||
"proxy-pool/internal/controller/worker"
|
||||
"proxy-pool/internal/domain/activitypool"
|
||||
@ -78,7 +77,6 @@ type ports struct {
|
||||
coordinator provider.Coordinator
|
||||
credentials credentials.Store
|
||||
providerResults provider.ResultRecorder
|
||||
capacityMetrics pool.CapacityObserver
|
||||
workerStore workerruntime.ControlStore
|
||||
close func() error
|
||||
}
|
||||
@ -178,63 +176,13 @@ func runWithWorkerFactory(
|
||||
if err != nil {
|
||||
return fmt.Errorf("%w: build Provider supervisor: %w", ErrStartup, err)
|
||||
}
|
||||
snapshotRefresh := worker.NewSnapshotRefreshBroker()
|
||||
var sequentialCoordinator lifecycle.Runner
|
||||
|
||||
dependencies := controllerRuntime.Dependencies{}
|
||||
var checkerMetrics healthDomain.TaskMetricsObserver
|
||||
var drainMetrics healthDomain.DrainMetricsObserver
|
||||
var extractionMetrics extraction.MetricsObserver
|
||||
if loaded.Value.Metrics.Enabled {
|
||||
if nilInterface(opened.metricsReadiness) {
|
||||
return errors.Join(ErrStartup, ErrInvalidOptions)
|
||||
}
|
||||
collector, collectorErr := platformMetrics.NewCheckerCollector(prometheus.DefaultRegisterer)
|
||||
if collectorErr != nil {
|
||||
return fmt.Errorf("%w: build Checker metrics: %w", ErrStartup, collectorErr)
|
||||
}
|
||||
checkerMetrics = collector
|
||||
drainCollector, drainCollectorErr := platformMetrics.NewDrainCollector(prometheus.DefaultRegisterer)
|
||||
if drainCollectorErr != nil {
|
||||
return fmt.Errorf("%w: build Drain metrics: %w", ErrStartup, drainCollectorErr)
|
||||
}
|
||||
drainMetrics = drainCollector
|
||||
providerCollector, providerCollectorErr := platformMetrics.NewProviderCollector(prometheus.DefaultRegisterer)
|
||||
if providerCollectorErr != nil {
|
||||
return fmt.Errorf("%w: build Provider metrics: %w", ErrStartup, providerCollectorErr)
|
||||
}
|
||||
if registrar, ok := opened.providerResults.(provider.ResultObserverRegistrar); ok && !nilInterface(registrar) {
|
||||
registrar.AddResultObserver(providerCollector)
|
||||
}
|
||||
capacityCollector, capacityCollectorErr := platformMetrics.NewCapacityCollector(prometheus.DefaultRegisterer)
|
||||
if capacityCollectorErr != nil {
|
||||
return fmt.Errorf("%w: build capacity metrics: %w", ErrStartup, capacityCollectorErr)
|
||||
}
|
||||
opened.capacityMetrics = capacityCollector
|
||||
extractionCollector, extractionCollectorErr := platformMetrics.NewExtractionCollector(prometheus.DefaultRegisterer)
|
||||
if extractionCollectorErr != nil {
|
||||
return fmt.Errorf("%w: build extraction metrics: %w", ErrStartup, extractionCollectorErr)
|
||||
}
|
||||
extractionMetrics = extractionCollector
|
||||
handler, handlerErr := platformMetrics.NewHandler(platformMetrics.Dependencies{
|
||||
Gatherer: prometheus.DefaultGatherer, Readiness: opened.metricsReadiness,
|
||||
})
|
||||
if handlerErr != nil {
|
||||
return fmt.Errorf("%w: build metrics handler: %w", ErrStartup, handlerErr)
|
||||
}
|
||||
dependencies.MetricsHandler = handler
|
||||
}
|
||||
if loaded.Value.Distribution.Enabled {
|
||||
if nilInterface(opened.activity) || nilInterface(opened.readiness) || nilInterface(opened.admission) {
|
||||
return errors.Join(ErrStartup, ErrInvalidOptions)
|
||||
}
|
||||
service, serviceErr := extraction.NewService(
|
||||
opened.activity,
|
||||
extractionPolicy(loaded.Value),
|
||||
opened.admission,
|
||||
options.Now,
|
||||
extractionMetrics,
|
||||
)
|
||||
service, serviceErr := extraction.NewService(opened.activity, extractionPolicy(loaded.Value), opened.admission, options.Now)
|
||||
if serviceErr != nil {
|
||||
return fmt.Errorf("%w: build extraction service: %w", ErrStartup, serviceErr)
|
||||
}
|
||||
@ -260,7 +208,7 @@ func runWithWorkerFactory(
|
||||
}
|
||||
service, serviceErr := admin.NewApplicationService(admin.ApplicationDependencies{
|
||||
State: opened.state, Operations: statusReader, Configuration: loader, Publisher: configurationStore,
|
||||
Runtime: supervisor, SnapshotRefresh: snapshotRefresh,
|
||||
Runtime: supervisor,
|
||||
}, admin.ApplicationOptions{Now: options.Now, FingerprintKey: options.FingerprintKey})
|
||||
if serviceErr != nil {
|
||||
return fmt.Errorf("%w: build admin service: %w", ErrStartup, serviceErr)
|
||||
@ -271,23 +219,26 @@ func runWithWorkerFactory(
|
||||
return fmt.Errorf("%w: commit startup configuration: %w", ErrStartup, applyErr)
|
||||
}
|
||||
dependencies.AdminService = service
|
||||
if stats, ok := opened.providerResults.(provider.StatsReader); ok && !nilInterface(stats) {
|
||||
coordinator, coordinatorErr := controllerRouting.NewSequentialCoordinator(
|
||||
configurationStore,
|
||||
opened.state,
|
||||
stats,
|
||||
snapshotRefresh,
|
||||
controllerRouting.Options{Now: options.Now},
|
||||
)
|
||||
if coordinatorErr != nil {
|
||||
return fmt.Errorf("%w: build Sequential routing coordinator: %w", ErrStartup, coordinatorErr)
|
||||
}
|
||||
if registrar, ok := opened.providerResults.(provider.ResultObserverRegistrar); ok && !nilInterface(registrar) {
|
||||
registrar.AddResultObserver(coordinator)
|
||||
var checkerMetrics healthDomain.TaskMetricsObserver
|
||||
if loaded.Value.Metrics.Enabled {
|
||||
if nilInterface(opened.metricsReadiness) {
|
||||
return errors.Join(ErrStartup, ErrInvalidOptions)
|
||||
}
|
||||
sequentialCoordinator = coordinator
|
||||
collector, collectorErr := platformMetrics.NewCheckerCollector(prometheus.DefaultRegisterer)
|
||||
if collectorErr != nil {
|
||||
return fmt.Errorf("%w: build Checker metrics: %w", ErrStartup, collectorErr)
|
||||
}
|
||||
checkerMetrics = collector
|
||||
handler, handlerErr := platformMetrics.NewHandler(platformMetrics.Dependencies{
|
||||
Gatherer: prometheus.DefaultGatherer, Readiness: opened.metricsReadiness,
|
||||
})
|
||||
if handlerErr != nil {
|
||||
return fmt.Errorf("%w: build metrics handler: %w", ErrStartup, handlerErr)
|
||||
}
|
||||
dependencies.MetricsHandler = handler
|
||||
}
|
||||
|
||||
runners := make([]lifecycle.Runner, 0, 3+len(loaded.Value.Upstreams))
|
||||
if hasHTTPRuntime(loaded.Value) {
|
||||
runner, err := factory.New(configurationStore.Current(), dependencies, controllerRuntime.Options{HTTP: options.HTTP})
|
||||
@ -333,7 +284,6 @@ func runWithWorkerFactory(
|
||||
DrainTickets: drainTickets,
|
||||
RoutingSource: routingSource,
|
||||
Credentials: opened.credentials,
|
||||
SnapshotRefresh: snapshotRefresh,
|
||||
})
|
||||
if serviceErr != nil {
|
||||
return fmt.Errorf("%w: build Worker control service: %w", ErrStartup, serviceErr)
|
||||
@ -370,14 +320,7 @@ func runWithWorkerFactory(
|
||||
}
|
||||
runners = append(runners, runner)
|
||||
if tasks, ok := opened.activity.(healthTaskRuntime); ok && !nilInterface(tasks) {
|
||||
var schedulerStates []controllerHealth.SchedulerStateSource
|
||||
if loaded.Value.Admin.Enabled {
|
||||
if nilInterface(opened.state) {
|
||||
return errors.Join(ErrStartup, ErrInvalidOptions)
|
||||
}
|
||||
schedulerStates = append(schedulerStates, opened.state)
|
||||
}
|
||||
schedulers, schedulerErr := newHealthSchedulers(configurationStore, tasks, options.Now, schedulerStates...)
|
||||
schedulers, schedulerErr := newHealthSchedulers(configurationStore, tasks, options.Now)
|
||||
if schedulerErr != nil {
|
||||
return fmt.Errorf("%w: build Checker health schedulers: %w", ErrStartup, schedulerErr)
|
||||
}
|
||||
@ -388,7 +331,7 @@ func runWithWorkerFactory(
|
||||
configurationStore,
|
||||
unhealthy,
|
||||
controllerHealth.UnhealthyReaperOptions{
|
||||
PollInterval: checkSchedulerPollInterval, BatchSize: checkSchedulerBatchSize, Now: options.Now, Metrics: drainMetrics,
|
||||
PollInterval: checkSchedulerPollInterval, BatchSize: checkSchedulerBatchSize, Now: options.Now,
|
||||
},
|
||||
)
|
||||
if reaperErr != nil {
|
||||
@ -396,33 +339,6 @@ func runWithWorkerFactory(
|
||||
}
|
||||
runners = append(runners, reaper)
|
||||
}
|
||||
if drainStore, ok := opened.activity.(controllerHealth.UpstreamDrainStore); ok && !nilInterface(drainStore) {
|
||||
var policyStates []controllerHealth.UpstreamDrainStateSource
|
||||
if loaded.Value.Admin.Enabled {
|
||||
if nilInterface(opened.state) {
|
||||
return errors.Join(ErrStartup, ErrInvalidOptions)
|
||||
}
|
||||
policyStates = append(policyStates, opened.state)
|
||||
}
|
||||
policySource, policyErr := controllerHealth.NewEffectiveUpstreamDrainPolicySource(configurationStore, policyStates...)
|
||||
if policyErr != nil {
|
||||
return fmt.Errorf("%w: build upstream Drain policy source: %w", ErrStartup, policyErr)
|
||||
}
|
||||
reaper, reaperErr := controllerHealth.NewConfiguredUpstreamDrainReaper(
|
||||
policySource,
|
||||
drainStore,
|
||||
controllerHealth.UpstreamDrainReaperOptions{
|
||||
PollInterval: checkSchedulerPollInterval, BatchSize: checkSchedulerBatchSize, Now: options.Now, Metrics: drainMetrics,
|
||||
},
|
||||
)
|
||||
if reaperErr != nil {
|
||||
return fmt.Errorf("%w: build upstream Drain reaper: %w", ErrStartup, reaperErr)
|
||||
}
|
||||
runners = append(runners, reaper)
|
||||
}
|
||||
}
|
||||
if sequentialCoordinator != nil {
|
||||
runners = append(runners, sequentialCoordinator)
|
||||
}
|
||||
runners = append(runners, supervisor)
|
||||
group, err := lifecycle.NewGroup(runners...)
|
||||
@ -436,10 +352,8 @@ func newHealthSchedulers(
|
||||
configuration controllerHealth.ConfigurationSource,
|
||||
tasks healthTaskRuntime,
|
||||
now func() time.Time,
|
||||
states ...controllerHealth.SchedulerStateSource,
|
||||
) ([]lifecycle.Runner, error) {
|
||||
if nilInterface(configuration) || nilInterface(tasks) || now == nil || len(states) > 1 ||
|
||||
(len(states) == 1 && nilInterface(states[0])) {
|
||||
if nilInterface(configuration) || nilInterface(tasks) || now == nil {
|
||||
return nil, ErrInvalidOptions
|
||||
}
|
||||
current := configuration.Current()
|
||||
@ -458,7 +372,7 @@ func newHealthSchedulers(
|
||||
}
|
||||
supervisor, err := controllerHealth.NewConfiguredSchedulerSupervisor(configuration, tasks, tasks, controllerHealth.SchedulerRunnerOptions{
|
||||
PollInterval: checkSchedulerPollInterval, BatchSize: checkSchedulerBatchSize, Now: now,
|
||||
}, states...)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
@ -75,7 +75,6 @@ func providerRuntimeBuilder(opened ports) (buildUpstreamRuntime, error) {
|
||||
Parser: parser,
|
||||
Activity: opened.activity,
|
||||
Results: results,
|
||||
Capacity: opened.capacityMetrics,
|
||||
})
|
||||
}, nil
|
||||
}
|
||||
|
||||
@ -276,7 +276,7 @@ func (supervisor *providerSupervisor) desired(ctx context.Context) (map[string]c
|
||||
for name, upstream := range configuration.Upstreams {
|
||||
isEnabled := upstream.Enabled
|
||||
if !nilInterface(supervisor.state) {
|
||||
isEnabled = isEnabled && enabled[name]
|
||||
isEnabled = enabled[name]
|
||||
}
|
||||
if isEnabled {
|
||||
upstream.Enabled = true
|
||||
|
||||
@ -71,38 +71,6 @@ func TestProviderSupervisorAppliesDisableAndConfigurationReplacement(t *testing.
|
||||
}
|
||||
}
|
||||
|
||||
func TestProviderSupervisorKeepsStaticallyDisabledUpstreamStopped(t *testing.T) {
|
||||
configuration, err := config.Load(strings.NewReader(bootstrapTestConfig))
|
||||
if err != nil {
|
||||
t.Fatalf("config.Load(): %v", err)
|
||||
}
|
||||
disabled := configuration.Upstreams["provider-b"]
|
||||
disabled.Enabled = false
|
||||
configuration.Upstreams["provider-b"] = disabled
|
||||
store, err := config.NewStore(configuration)
|
||||
if err != nil {
|
||||
t.Fatalf("config.NewStore(): %v", err)
|
||||
}
|
||||
supervisor, err := newProviderSupervisor(
|
||||
store,
|
||||
&mutableProviderState{enabled: map[string]bool{"provider-a": true, "provider-b": true}},
|
||||
func(string, config.Upstream) (lifecycle.Runner, error) {
|
||||
return supervisorRunnerFunc(func(context.Context) error { return nil }), nil
|
||||
},
|
||||
nil, nil, nil, bootstrapTestFingerprintKey, time.Hour,
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("newProviderSupervisor() error = %v", err)
|
||||
}
|
||||
desired, err := supervisor.desired(context.Background())
|
||||
if err != nil || len(desired) != 1 {
|
||||
t.Fatalf("desired() = %+v, %v", desired, err)
|
||||
}
|
||||
if _, exists := desired["provider-a"]; !exists {
|
||||
t.Fatalf("desired() = %+v, provider-a missing", desired)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProviderSupervisorPropagatesUnexpectedRuntimeFailure(t *testing.T) {
|
||||
configuration, err := config.Load(strings.NewReader(bootstrapTestConfig))
|
||||
if err != nil {
|
||||
|
||||
@ -8,8 +8,6 @@ import (
|
||||
"time"
|
||||
|
||||
controllerExtraction "proxy-pool/internal/controller/extraction"
|
||||
"proxy-pool/internal/domain/authorization"
|
||||
"proxy-pool/internal/domain/clientpolicy"
|
||||
domainExtraction "proxy-pool/internal/domain/extraction"
|
||||
"proxy-pool/internal/platform/httpapi"
|
||||
"proxy-pool/internal/platform/httpsecurity"
|
||||
@ -175,10 +173,6 @@ func (h *Handler) handleExtract(writer http.ResponseWriter, request *http.Reques
|
||||
h.writeProblem(writer, problemBadRequest(requestID, "INVALID_REQUEST", "Invalid request", "", nil))
|
||||
return
|
||||
}
|
||||
if !identity.Allows(authorization.DistributionExtract) {
|
||||
h.writeProblem(writer, httpapi.NewProblem(http.StatusForbidden, "FORBIDDEN", "Forbidden", "", requestID))
|
||||
return
|
||||
}
|
||||
|
||||
idempotencyKey, err := validateIdempotencyKey(request.Header.Values(headerIdempotencyKey))
|
||||
if err != nil {
|
||||
@ -207,15 +201,7 @@ func (h *Handler) handleExtract(writer http.ResponseWriter, request *http.Reques
|
||||
return
|
||||
}
|
||||
|
||||
if !identity.ClientPolicy.AllowsExtractCount(payload.Count) {
|
||||
h.writeProblem(writer, httpapi.NewProblem(http.StatusForbidden, "FORBIDDEN", "Forbidden", "", requestID))
|
||||
return
|
||||
}
|
||||
filters, allowed := restrictExtractFilters(identity.ClientPolicy, payload.filtersOrZero())
|
||||
if !allowed {
|
||||
h.writeProblem(writer, httpapi.NewProblem(http.StatusForbidden, "FORBIDDEN", "Forbidden", "", requestID))
|
||||
return
|
||||
}
|
||||
filters := payload.filtersOrZero()
|
||||
serviceResponse, err := h.extractor.Extract(request.Context(), controllerExtraction.Request{
|
||||
RequestID: requestID,
|
||||
ClientID: identity.ClientID,
|
||||
@ -261,20 +247,6 @@ func (h *Handler) handleExtract(writer http.ResponseWriter, request *http.Reques
|
||||
h.writeJSON(writer, requestID, http.StatusOK, response)
|
||||
}
|
||||
|
||||
func restrictExtractFilters(policy clientpolicy.Policy, filters extractFiltersDTO) (extractFiltersDTO, bool) {
|
||||
upstreams, allowed := policy.RestrictUpstreams(filters.AllowedUpstreams)
|
||||
if !allowed {
|
||||
return extractFiltersDTO{}, false
|
||||
}
|
||||
regions, allowed := policy.RestrictRegions(filters.Regions)
|
||||
if !allowed {
|
||||
return extractFiltersDTO{}, false
|
||||
}
|
||||
filters.AllowedUpstreams = upstreams
|
||||
filters.Regions = regions
|
||||
return filters, true
|
||||
}
|
||||
|
||||
func validateIdempotencyKey(values []string) (string, error) {
|
||||
if len(values) == 0 || (len(values) == 1 && values[0] == "") {
|
||||
return "", nil
|
||||
|
||||
@ -7,14 +7,11 @@ import (
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
controllerExtraction "proxy-pool/internal/controller/extraction"
|
||||
"proxy-pool/internal/domain/authorization"
|
||||
"proxy-pool/internal/domain/clientpolicy"
|
||||
domainExtraction "proxy-pool/internal/domain/extraction"
|
||||
"proxy-pool/internal/platform/httpapi"
|
||||
"proxy-pool/internal/platform/httpsecurity"
|
||||
@ -160,108 +157,6 @@ func TestHandlerExtractSuccessMapsOpenAPIDTOAndReturnsRequestID(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandlerRejectsIdentityWithoutExtractPermission(t *testing.T) {
|
||||
t.Parallel()
|
||||
extractor := &fakeExtractor{}
|
||||
handler := mustNewHandler(t, Config{BodyLimitBytes: 1024}, Dependencies{
|
||||
Extractor: extractor,
|
||||
Identity: fakeIdentityResolver{identity: Identity{
|
||||
ClientID: "tenant-read-only", SourceIP: "198.51.100.8", Permissions: []string{authorization.AdminRead},
|
||||
}},
|
||||
Readiness: fakeReadinessChecker{},
|
||||
})
|
||||
request := httptest.NewRequest(http.MethodPost, "/api/v1/proxies/extract", strings.NewReader(`{"count":1}`))
|
||||
request.Header.Set("Content-Type", httpapi.JSONContentType)
|
||||
request.Header.Set(headerIdempotencyKey, "idem-12345678")
|
||||
recorder := httptest.NewRecorder()
|
||||
|
||||
handler.ServeHTTP(recorder, request)
|
||||
|
||||
if recorder.Code != http.StatusForbidden || extractor.calls != 0 || !strings.Contains(recorder.Body.String(), `"code":"FORBIDDEN"`) {
|
||||
t.Fatalf("response = status %d calls=%d body=%s", recorder.Code, extractor.calls, recorder.Body.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandlerEnforcesCredentialExtractionPolicy(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
body string
|
||||
wantStatus int
|
||||
wantUpstreams []string
|
||||
wantRegions []string
|
||||
wantExtractorCall int
|
||||
}{
|
||||
{
|
||||
name: "omitted filters are bound to credential policy",
|
||||
body: "{\"count\":2}",
|
||||
wantStatus: http.StatusOK,
|
||||
wantUpstreams: []string{"provider-a"},
|
||||
wantRegions: []string{"shanghai"},
|
||||
wantExtractorCall: 1,
|
||||
},
|
||||
{
|
||||
name: "count exceeds credential maximum",
|
||||
body: "{\"count\":3}",
|
||||
wantStatus: http.StatusForbidden,
|
||||
wantExtractorCall: 0,
|
||||
},
|
||||
{
|
||||
name: "upstream exceeds credential boundary",
|
||||
body: "{\"count\":1,\"filters\":{\"allowedUpstreams\":[\"provider-b\"]}}",
|
||||
wantStatus: http.StatusForbidden,
|
||||
wantExtractorCall: 0,
|
||||
},
|
||||
{
|
||||
name: "region exceeds credential boundary",
|
||||
body: "{\"count\":1,\"filters\":{\"regions\":[\"beijing\"]}}",
|
||||
wantStatus: http.StatusForbidden,
|
||||
wantExtractorCall: 0,
|
||||
},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
test := test
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
extractor := &fakeExtractor{}
|
||||
handler := mustNewHandler(t, Config{BodyLimitBytes: 1024}, Dependencies{
|
||||
Extractor: extractor,
|
||||
Identity: fakeIdentityResolver{identity: Identity{
|
||||
ClientID: "tenant-restricted",
|
||||
Permissions: []string{authorization.DistributionExtract},
|
||||
ClientPolicy: clientpolicy.Policy{
|
||||
MaxExtractCount: 2,
|
||||
AllowedUpstreams: []string{"provider-a"},
|
||||
AllowedRegions: []string{"shanghai"},
|
||||
},
|
||||
}},
|
||||
Readiness: fakeReadinessChecker{},
|
||||
})
|
||||
request := httptest.NewRequest(http.MethodPost, pathExtract, strings.NewReader(test.body))
|
||||
request.Header.Set("Content-Type", httpapi.JSONContentType)
|
||||
response := httptest.NewRecorder()
|
||||
|
||||
handler.ServeHTTP(response, request)
|
||||
|
||||
if response.Code != test.wantStatus || extractor.calls != test.wantExtractorCall {
|
||||
t.Fatalf("response = status %d calls=%d, want status %d calls=%d",
|
||||
response.Code, extractor.calls, test.wantStatus, test.wantExtractorCall)
|
||||
}
|
||||
if response.Code == http.StatusForbidden && !strings.Contains(response.Body.String(), "\"code\":\"FORBIDDEN\"") {
|
||||
t.Fatalf("forbidden response = %s", response.Body.String())
|
||||
}
|
||||
if got := extractor.request.Filters.Upstreams; !slices.Equal(got, test.wantUpstreams) {
|
||||
t.Fatalf("upstreams = %v, want %v", got, test.wantUpstreams)
|
||||
}
|
||||
if got := extractor.request.Filters.Regions; !slices.Equal(got, test.wantRegions) {
|
||||
t.Fatalf("regions = %v, want %v", got, test.wantRegions)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandlerRejectsEmptyResolvedIdentity(t *testing.T) {
|
||||
t.Parallel()
|
||||
extractor := &fakeExtractor{}
|
||||
|
||||
@ -1,80 +0,0 @@
|
||||
package extraction
|
||||
|
||||
import (
|
||||
"errors"
|
||||
|
||||
domain "proxy-pool/internal/domain/extraction"
|
||||
)
|
||||
|
||||
// MetricResult is a fixed, low-cardinality classification of one extraction
|
||||
// request after application-level validation, admission, and storage handling.
|
||||
type MetricResult string
|
||||
|
||||
const (
|
||||
MetricComplete MetricResult = "complete"
|
||||
MetricPartial MetricResult = "partial"
|
||||
MetricEmpty MetricResult = "empty"
|
||||
MetricInsufficient MetricResult = "insufficient"
|
||||
MetricIdempotencyConflict MetricResult = "idempotency_conflict"
|
||||
MetricRateLimited MetricResult = "rate_limited"
|
||||
MetricUnavailable MetricResult = "unavailable"
|
||||
MetricInvalid MetricResult = "invalid"
|
||||
MetricError MetricResult = "error"
|
||||
)
|
||||
|
||||
// MetricsEvent contains only aggregate request and response counts. It must
|
||||
// never contain a Client, Proxy, Upstream, filter, identifier, or error text.
|
||||
type MetricsEvent struct {
|
||||
Result MetricResult
|
||||
Requested int
|
||||
Returned int
|
||||
}
|
||||
|
||||
// MetricsObserver receives one completed extraction request. Implementations
|
||||
// must return promptly and must not perform network or persistent storage I/O.
|
||||
type MetricsObserver interface {
|
||||
ObserveExtraction(MetricsEvent)
|
||||
}
|
||||
|
||||
func classifyMetricsEvent(response Response, resultErr error) MetricsEvent {
|
||||
event := MetricsEvent{Requested: response.Requested, Returned: response.Returned}
|
||||
if resultErr == nil {
|
||||
switch {
|
||||
case response.Returned == response.Requested:
|
||||
event.Result = MetricComplete
|
||||
case response.Returned > 0 && response.Returned < response.Requested:
|
||||
event.Result = MetricPartial
|
||||
case response.Returned == 0:
|
||||
event.Result = MetricEmpty
|
||||
default:
|
||||
event.Result = MetricError
|
||||
}
|
||||
return event
|
||||
}
|
||||
switch {
|
||||
case errors.Is(resultErr, domain.ErrInsufficientProxies):
|
||||
event.Result = MetricInsufficient
|
||||
case errors.Is(resultErr, domain.ErrIdempotencyConflict):
|
||||
event.Result = MetricIdempotencyConflict
|
||||
case errors.Is(resultErr, ErrAdmissionRejected):
|
||||
event.Result = MetricRateLimited
|
||||
case errors.Is(resultErr, ErrUnavailable):
|
||||
event.Result = MetricUnavailable
|
||||
case errors.Is(resultErr, ErrInvalidRequest), errors.Is(resultErr, ErrCountExceeded),
|
||||
errors.Is(resultErr, ErrInvalidFulfillment), errors.Is(resultErr, domain.ErrInvalidCommand):
|
||||
event.Result = MetricInvalid
|
||||
default:
|
||||
event.Result = MetricError
|
||||
}
|
||||
return event
|
||||
}
|
||||
|
||||
func validMetricResult(result MetricResult) bool {
|
||||
switch result {
|
||||
case MetricComplete, MetricPartial, MetricEmpty, MetricInsufficient, MetricIdempotencyConflict,
|
||||
MetricRateLimited, MetricUnavailable, MetricInvalid, MetricError:
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
@ -73,16 +73,9 @@ type Service struct {
|
||||
policy Policy
|
||||
admission admission.Admitter
|
||||
now func() time.Time
|
||||
metrics MetricsObserver
|
||||
}
|
||||
|
||||
func NewService(
|
||||
store domain.Store,
|
||||
policy Policy,
|
||||
admitter admission.Admitter,
|
||||
now func() time.Time,
|
||||
metrics ...MetricsObserver,
|
||||
) (*Service, error) {
|
||||
func NewService(store domain.Store, policy Policy, admitter admission.Admitter, now func() time.Time) (*Service, error) {
|
||||
if admitter == nil {
|
||||
return nil, fmt.Errorf("%w: admission is required", ErrInvalidServicePolicy)
|
||||
}
|
||||
@ -99,19 +92,11 @@ func NewService(
|
||||
if now == nil {
|
||||
now = time.Now
|
||||
}
|
||||
if len(metrics) > 1 {
|
||||
return nil, ErrInvalidServicePolicy
|
||||
}
|
||||
var observer MetricsObserver
|
||||
if len(metrics) == 1 {
|
||||
observer = metrics[0]
|
||||
}
|
||||
return &Service{store: store, policy: policy, admission: admitter, now: now, metrics: observer}, nil
|
||||
return &Service{store: store, policy: policy, admission: admitter, now: now}, nil
|
||||
}
|
||||
|
||||
func (s *Service) Extract(ctx context.Context, request Request) (response Response, resultErr error) {
|
||||
response = Response{RequestID: request.RequestID, Requested: request.Count}
|
||||
defer func() { s.observeMetrics(classifyMetricsEvent(response, resultErr)) }()
|
||||
func (s *Service) Extract(ctx context.Context, request Request) (Response, error) {
|
||||
response := Response{RequestID: request.RequestID, Requested: request.Count}
|
||||
if request.RequestID == "" || (request.ClientID == "" && request.SourceIP == "") || request.Count <= 0 {
|
||||
return response, ErrInvalidRequest
|
||||
}
|
||||
@ -191,13 +176,6 @@ func (s *Service) Extract(ctx context.Context, request Request) (response Respon
|
||||
return response, nil
|
||||
}
|
||||
|
||||
func (s *Service) observeMetrics(event MetricsEvent) {
|
||||
if s == nil || s.metrics == nil || !validMetricResult(event.Result) {
|
||||
return
|
||||
}
|
||||
s.metrics.ObserveExtraction(event)
|
||||
}
|
||||
|
||||
func admissionKey(request Request) string {
|
||||
if request.ClientID != "" {
|
||||
return "client:" + request.ClientID
|
||||
|
||||
@ -244,86 +244,6 @@ func TestServiceIdempotentReplayKeepsOriginalExtractionTime(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestServiceEmitsFixedMetricsForTerminalResults(t *testing.T) {
|
||||
observer := &recordingMetricsObserver{}
|
||||
policy := Policy{MaxCountPerRequest: 2, DefaultFulfillment: domain.Partial}
|
||||
tests := []struct {
|
||||
name string
|
||||
request Request
|
||||
result domain.Result
|
||||
storeErr error
|
||||
admitErr error
|
||||
wantEvent MetricsEvent
|
||||
}{
|
||||
{
|
||||
name: "complete",
|
||||
request: Request{RequestID: "req-complete", ClientID: "client-a", Count: 2},
|
||||
result: domain.Result{Requested: 2, Returned: 2, Items: []domain.Candidate{{}, {}}},
|
||||
wantEvent: MetricsEvent{Result: MetricComplete, Requested: 2, Returned: 2},
|
||||
},
|
||||
{
|
||||
name: "partial",
|
||||
request: Request{RequestID: "req-partial", ClientID: "client-a", Count: 2},
|
||||
result: domain.Result{Requested: 2, Returned: 1, Items: []domain.Candidate{{}}},
|
||||
wantEvent: MetricsEvent{Result: MetricPartial, Requested: 2, Returned: 1},
|
||||
},
|
||||
{
|
||||
name: "empty",
|
||||
request: Request{RequestID: "req-empty", ClientID: "client-a", Count: 1},
|
||||
result: domain.Result{Requested: 1},
|
||||
wantEvent: MetricsEvent{Result: MetricEmpty, Requested: 1, Returned: 0},
|
||||
},
|
||||
{
|
||||
name: "unexpected over delivery",
|
||||
request: Request{RequestID: "req-over", ClientID: "client-a", Count: 1},
|
||||
result: domain.Result{Requested: 1, Returned: 2, Items: []domain.Candidate{{}, {}}},
|
||||
wantEvent: MetricsEvent{Result: MetricError, Requested: 1, Returned: 2},
|
||||
},
|
||||
{
|
||||
name: "insufficient",
|
||||
request: Request{RequestID: "req-insufficient", ClientID: "client-a", Count: 1},
|
||||
storeErr: domain.ErrInsufficientProxies,
|
||||
wantEvent: MetricsEvent{Result: MetricInsufficient, Requested: 1},
|
||||
},
|
||||
{
|
||||
name: "idempotency conflict",
|
||||
request: Request{RequestID: "req-conflict", ClientID: "client-a", Count: 1},
|
||||
storeErr: domain.ErrIdempotencyConflict,
|
||||
wantEvent: MetricsEvent{Result: MetricIdempotencyConflict, Requested: 1},
|
||||
},
|
||||
{
|
||||
name: "rate limited",
|
||||
request: Request{RequestID: "req-rate", ClientID: "client-a", Count: 1},
|
||||
admitErr: errors.New("rate limited"),
|
||||
wantEvent: MetricsEvent{Result: MetricRateLimited, Requested: 1},
|
||||
},
|
||||
{
|
||||
name: "unavailable",
|
||||
request: Request{RequestID: "req-unavailable", ClientID: "client-a", Count: 1},
|
||||
storeErr: domain.ErrStoreUnavailable,
|
||||
wantEvent: MetricsEvent{Result: MetricUnavailable, Requested: 1},
|
||||
},
|
||||
{
|
||||
name: "invalid",
|
||||
request: Request{RequestID: "req-invalid", Count: 1},
|
||||
wantEvent: MetricsEvent{Result: MetricInvalid, Requested: 1},
|
||||
},
|
||||
}
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
service, err := NewService(&recordingStore{result: test.result, err: test.storeErr}, policy,
|
||||
&recordingAdmission{err: test.admitErr}, time.Now, observer)
|
||||
if err != nil {
|
||||
t.Fatalf("NewService() error = %v", err)
|
||||
}
|
||||
_, _ = service.Extract(context.Background(), test.request)
|
||||
if got := observer.events[len(observer.events)-1]; got != test.wantEvent {
|
||||
t.Fatalf("metrics event = %+v, want %+v", got, test.wantEvent)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
type recordingStore struct {
|
||||
command domain.Command
|
||||
result domain.Result
|
||||
@ -337,8 +257,6 @@ type recordingAdmission struct {
|
||||
calls int
|
||||
}
|
||||
|
||||
type recordingMetricsObserver struct{ events []MetricsEvent }
|
||||
|
||||
type allowAllAdmission struct{}
|
||||
|
||||
func (allowAllAdmission) Admit(context.Context, string) error { return nil }
|
||||
@ -349,10 +267,6 @@ func (a *recordingAdmission) Admit(_ context.Context, key string) error {
|
||||
return a.err
|
||||
}
|
||||
|
||||
func (observer *recordingMetricsObserver) ObserveExtraction(event MetricsEvent) {
|
||||
observer.events = append(observer.events, event)
|
||||
}
|
||||
|
||||
func (s *recordingStore) Extract(_ context.Context, command domain.Command) (domain.Result, error) {
|
||||
s.calls++
|
||||
s.command = command
|
||||
|
||||
@ -9,7 +9,6 @@ import (
|
||||
"time"
|
||||
|
||||
"proxy-pool/internal/config"
|
||||
"proxy-pool/internal/domain/adminstate"
|
||||
healthDomain "proxy-pool/internal/domain/health"
|
||||
)
|
||||
|
||||
@ -170,26 +169,12 @@ type ConfiguredSchedulerRunner struct {
|
||||
// bounded BASIC scheduling without a Controller restart.
|
||||
type ConfiguredSchedulerSupervisor struct {
|
||||
configuration ConfigurationSource
|
||||
state SchedulerStateSource
|
||||
source UpstreamTaskSource
|
||||
sink TaskSink
|
||||
options SchedulerRunnerOptions
|
||||
cursor atomic.Uint64
|
||||
}
|
||||
|
||||
// SchedulerConfigurationSource returns a configuration and its Admin revision
|
||||
// from one atomic publication. config.Store implements this interface.
|
||||
type SchedulerConfigurationSource interface {
|
||||
ConfigurationSource
|
||||
Snapshot() (*config.Config, uint64)
|
||||
}
|
||||
|
||||
// SchedulerStateSource provides the persisted Admin view that fences a
|
||||
// scheduler from combining two different configuration revisions.
|
||||
type SchedulerStateSource interface {
|
||||
Snapshot(context.Context) (adminstate.Snapshot, error)
|
||||
}
|
||||
|
||||
func NewConfiguredSchedulerRunner(
|
||||
configuration ConfigurationSource,
|
||||
upstreamID string,
|
||||
@ -211,22 +196,14 @@ func NewConfiguredSchedulerSupervisor(
|
||||
source UpstreamTaskSource,
|
||||
sink TaskSink,
|
||||
options SchedulerRunnerOptions,
|
||||
states ...SchedulerStateSource,
|
||||
) (*ConfiguredSchedulerSupervisor, error) {
|
||||
if nilInterface(configuration) || nilInterface(source) || nilInterface(sink) || options.PollInterval <= 0 ||
|
||||
options.BatchSize <= 0 || options.Now == nil || len(states) > 1 || (len(states) == 1 && nilInterface(states[0])) {
|
||||
options.BatchSize <= 0 || options.Now == nil {
|
||||
return nil, ErrInvalidSchedulerRunner
|
||||
}
|
||||
supervisor := &ConfiguredSchedulerSupervisor{
|
||||
return &ConfiguredSchedulerSupervisor{
|
||||
configuration: configuration, source: source, sink: sink, options: options,
|
||||
}
|
||||
if len(states) == 1 {
|
||||
if _, ok := configuration.(SchedulerConfigurationSource); !ok {
|
||||
return nil, ErrInvalidSchedulerRunner
|
||||
}
|
||||
supervisor.state = states[0]
|
||||
}
|
||||
return supervisor, nil
|
||||
}, nil
|
||||
}
|
||||
|
||||
type TickResult struct {
|
||||
@ -268,10 +245,11 @@ func (supervisor *ConfiguredSchedulerSupervisor) Tick(ctx context.Context) (Tick
|
||||
if err := ctx.Err(); err != nil {
|
||||
return TickResult{}, err
|
||||
}
|
||||
configuration, names, routings, err := supervisor.effectiveConfiguration(ctx)
|
||||
if err != nil {
|
||||
return TickResult{}, err
|
||||
configuration := supervisor.configuration.Current()
|
||||
if configuration == nil {
|
||||
return TickResult{}, ErrInvalidSchedulerRunner
|
||||
}
|
||||
names := enabledUpstreamNames(configuration)
|
||||
result := TickResult{}
|
||||
for _, name := range names {
|
||||
planner, configured, err := configuredPlannerForConfig(configuration, name)
|
||||
@ -286,7 +264,7 @@ func (supervisor *ConfiguredSchedulerSupervisor) Tick(ctx context.Context) (Tick
|
||||
return TickResult{}, err
|
||||
}
|
||||
check := config.EffectiveCheck(configuration.Defaults.Check, configuration.Upstreams[name].Check)
|
||||
groups, err := supervisor.upstreamDueGroups(basicSource, name, check.URLs, configuredTargetProfiles(configuration, name, routings))
|
||||
groups, err := supervisor.upstreamDueGroups(basicSource, name, check.URLs, configuredTargetProfiles(configuration, name))
|
||||
if err != nil {
|
||||
return TickResult{}, err
|
||||
}
|
||||
@ -311,91 +289,6 @@ func (supervisor *ConfiguredSchedulerSupervisor) Tick(ctx context.Context) (Tick
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// effectiveConfiguration returns the effective Upstream and Routing state for
|
||||
// one scheduler tick. When Admin state is present, it requires the
|
||||
// configuration and state snapshots to share a revision; an incomplete or
|
||||
// torn view schedules nothing.
|
||||
func (supervisor *ConfiguredSchedulerSupervisor) effectiveConfiguration(
|
||||
ctx context.Context,
|
||||
) (*config.Config, []string, map[string]adminstate.RoutingState, error) {
|
||||
if supervisor == nil || nilInterface(supervisor.configuration) {
|
||||
return nil, nil, nil, ErrInvalidSchedulerRunner
|
||||
}
|
||||
if supervisor.state == nil {
|
||||
configuration := supervisor.configuration.Current()
|
||||
if configuration == nil {
|
||||
return nil, nil, nil, ErrInvalidSchedulerRunner
|
||||
}
|
||||
return configuration, enabledUpstreamNames(configuration), nil, nil
|
||||
}
|
||||
configurationSource, ok := supervisor.configuration.(SchedulerConfigurationSource)
|
||||
if !ok || nilInterface(configurationSource) || nilInterface(supervisor.state) {
|
||||
return nil, nil, nil, ErrInvalidSchedulerRunner
|
||||
}
|
||||
configuration, revision := configurationSource.Snapshot()
|
||||
if configuration == nil || revision == 0 {
|
||||
return nil, nil, nil, ErrInvalidSchedulerRunner
|
||||
}
|
||||
snapshot, err := supervisor.state.Snapshot(ctx)
|
||||
if err != nil {
|
||||
return nil, nil, nil, err
|
||||
}
|
||||
if snapshot.Config == nil || snapshot.Config.Revision != revision {
|
||||
return nil, nil, nil, ErrInvalidSchedulerRunner
|
||||
}
|
||||
states, err := newSchedulerAdminStates(configuration, snapshot)
|
||||
if err != nil {
|
||||
return nil, nil, nil, err
|
||||
}
|
||||
names, err := enabledUpstreamNamesForState(configuration, states.upstreams)
|
||||
if err != nil {
|
||||
return nil, nil, nil, err
|
||||
}
|
||||
return configuration, names, states.routings, nil
|
||||
}
|
||||
|
||||
type schedulerAdminStates struct {
|
||||
upstreams map[string]adminstate.UpstreamState
|
||||
routings map[string]adminstate.RoutingState
|
||||
}
|
||||
|
||||
func newSchedulerAdminStates(
|
||||
configuration *config.Config,
|
||||
snapshot adminstate.Snapshot,
|
||||
) (schedulerAdminStates, error) {
|
||||
if configuration == nil {
|
||||
return schedulerAdminStates{}, ErrInvalidSchedulerRunner
|
||||
}
|
||||
states := schedulerAdminStates{
|
||||
upstreams: make(map[string]adminstate.UpstreamState, len(snapshot.Upstreams)),
|
||||
routings: make(map[string]adminstate.RoutingState, len(snapshot.Routings)),
|
||||
}
|
||||
for _, upstream := range snapshot.Upstreams {
|
||||
if upstream.Name == "" || upstream.Revision == 0 {
|
||||
return schedulerAdminStates{}, ErrInvalidSchedulerRunner
|
||||
}
|
||||
if _, duplicate := states.upstreams[upstream.Name]; duplicate {
|
||||
return schedulerAdminStates{}, ErrInvalidSchedulerRunner
|
||||
}
|
||||
states.upstreams[upstream.Name] = upstream
|
||||
}
|
||||
for _, routing := range snapshot.Routings {
|
||||
if routing.Name == "" || routing.Revision == 0 {
|
||||
return schedulerAdminStates{}, ErrInvalidSchedulerRunner
|
||||
}
|
||||
if _, duplicate := states.routings[routing.Name]; duplicate {
|
||||
return schedulerAdminStates{}, ErrInvalidSchedulerRunner
|
||||
}
|
||||
states.routings[routing.Name] = routing
|
||||
}
|
||||
for _, routing := range configuration.Routing {
|
||||
if _, exists := states.routings[routing.Name]; !exists {
|
||||
return schedulerAdminStates{}, ErrInvalidSchedulerRunner
|
||||
}
|
||||
}
|
||||
return states, nil
|
||||
}
|
||||
|
||||
func (supervisor *ConfiguredSchedulerSupervisor) upstreamDueGroups(
|
||||
basic DueSource,
|
||||
upstreamID string,
|
||||
@ -433,17 +326,13 @@ type configuredTargetProfile struct {
|
||||
TargetURL string
|
||||
}
|
||||
|
||||
func configuredTargetProfiles(
|
||||
configuration *config.Config,
|
||||
upstreamID string,
|
||||
routings map[string]adminstate.RoutingState,
|
||||
) []configuredTargetProfile {
|
||||
func configuredTargetProfiles(configuration *config.Config, upstreamID string) []configuredTargetProfile {
|
||||
if configuration == nil || upstreamID == "" {
|
||||
return nil
|
||||
}
|
||||
profiles := make([]configuredTargetProfile, 0)
|
||||
for _, route := range configuration.Routing {
|
||||
if !route.Enabled || !containsString(route.Upstreams, upstreamID) || (routings != nil && !routings[route.Name].Enabled) {
|
||||
if !route.Enabled || !containsString(route.Upstreams, upstreamID) {
|
||||
continue
|
||||
}
|
||||
for _, targetURL := range route.Check.Targets {
|
||||
@ -602,27 +491,6 @@ func enabledUpstreamNames(configuration *config.Config) []string {
|
||||
return names
|
||||
}
|
||||
|
||||
func enabledUpstreamNamesForState(
|
||||
configuration *config.Config,
|
||||
states map[string]adminstate.UpstreamState,
|
||||
) ([]string, error) {
|
||||
if configuration == nil {
|
||||
return nil, ErrInvalidSchedulerRunner
|
||||
}
|
||||
names := make([]string, 0, len(configuration.Upstreams))
|
||||
for name, upstream := range configuration.Upstreams {
|
||||
state, exists := states[name]
|
||||
if !exists {
|
||||
return nil, ErrInvalidSchedulerRunner
|
||||
}
|
||||
if upstream.Enabled && state.Enabled {
|
||||
names = append(names, name)
|
||||
}
|
||||
}
|
||||
sort.Strings(names)
|
||||
return names, nil
|
||||
}
|
||||
|
||||
func runScheduler(ctx context.Context, pollInterval time.Duration, tick func(context.Context) (TickResult, error)) error {
|
||||
if ctx == nil || pollInterval <= 0 || tick == nil {
|
||||
return ErrInvalidSchedulerRunner
|
||||
|
||||
@ -7,7 +7,6 @@ import (
|
||||
"time"
|
||||
|
||||
"proxy-pool/internal/config"
|
||||
"proxy-pool/internal/domain/adminstate"
|
||||
healthDomain "proxy-pool/internal/domain/health"
|
||||
proxyDomain "proxy-pool/internal/domain/proxy"
|
||||
)
|
||||
@ -165,160 +164,6 @@ func TestConfiguredSchedulerSupervisorDiscoversNewEnabledUpstream(t *testing.T)
|
||||
}
|
||||
}
|
||||
|
||||
func TestConfiguredSchedulerSupervisorSkipsAdminDisabledUpstream(t *testing.T) {
|
||||
now := time.Date(2026, 8, 7, 10, 0, 0, 0, time.UTC)
|
||||
configuration := schedulerConfiguration(now, "provider-a", "provider-b")
|
||||
configuration.Defaults.Check.URLs = []string{"https://egress.example/identity"}
|
||||
configuration.Routing = []config.Routing{{
|
||||
Name: "checkout", Enabled: true, Upstreams: []string{"provider-a"},
|
||||
Check: config.RoutingCheck{Targets: []string{"https://checkout.example/health"}},
|
||||
}}
|
||||
source := &upstreamTaskSourceStub{sources: map[string]*dueSourceStub{
|
||||
"provider-a": {
|
||||
candidates: []Candidate{{ProxyID: "proxy-a", State: proxyDomain.StateFetched, Level: healthDomain.LevelBasic, DueAt: now}},
|
||||
egressCandidates: map[string][]Candidate{"https://egress.example/identity": {
|
||||
{ProxyID: "egress-a", State: proxyDomain.StateFetched, Level: healthDomain.LevelEgress, TargetURL: "https://egress.example/identity", DueAt: now},
|
||||
}},
|
||||
targetCandidates: map[string][]Candidate{targetCandidateKey("checkout", "https://checkout.example/health"): {
|
||||
{ProxyID: "target-a", State: proxyDomain.StateFetched, Level: healthDomain.LevelTarget, RoutingName: "checkout", TargetURL: "https://checkout.example/health", DueAt: now},
|
||||
}},
|
||||
},
|
||||
"provider-b": {candidates: []Candidate{{ProxyID: "proxy-b", State: proxyDomain.StateFetched, Level: healthDomain.LevelBasic, DueAt: now}}},
|
||||
}}
|
||||
state := schedulerStateStub{snapshot: adminstate.Snapshot{
|
||||
Config: &adminstate.ConfigRevision{Revision: 42},
|
||||
Upstreams: []adminstate.UpstreamState{
|
||||
{Name: "provider-a", Enabled: false, Revision: 11},
|
||||
{Name: "provider-b", Enabled: true, Revision: 12},
|
||||
},
|
||||
Routings: []adminstate.RoutingState{{Name: "checkout", Enabled: true, Revision: 13}},
|
||||
}}
|
||||
sink := &taskSinkStub{}
|
||||
supervisor, err := NewConfiguredSchedulerSupervisor(
|
||||
&versionedConfigurationSourceStub{configuration: configuration, revision: 42}, source, sink,
|
||||
SchedulerRunnerOptions{PollInterval: time.Second, BatchSize: 16, Now: func() time.Time { return now }}, state,
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("NewConfiguredSchedulerSupervisor() = %v", err)
|
||||
}
|
||||
result, err := supervisor.Tick(context.Background())
|
||||
if err != nil || result != (TickResult{Planned: 1, Offered: 1}) || source.sources["provider-a"].inFlightCalls != 0 ||
|
||||
source.sources["provider-a"].dueCalls != 0 || source.sources["provider-a"].egressCalls != 0 ||
|
||||
source.sources["provider-a"].targetCalls != 0 || source.sources["provider-b"].dueCalls != 1 ||
|
||||
len(sink.tasks) != 1 || sink.tasks[0].Candidate.ProxyID != "proxy-b" {
|
||||
t.Fatalf("Tick() = (%+v, %v); source=%+v sink=%+v", result, err, source, sink.tasks)
|
||||
}
|
||||
}
|
||||
|
||||
func TestConfiguredSchedulerSupervisorSkipsAdminDisabledRoutingTarget(t *testing.T) {
|
||||
now := time.Date(2026, 8, 7, 10, 15, 0, 0, time.UTC)
|
||||
configuration := schedulerConfiguration(now, "provider-a")
|
||||
configuration.Routing = []config.Routing{{
|
||||
Name: "checkout", Enabled: true, Upstreams: []string{"provider-a"},
|
||||
Check: config.RoutingCheck{Targets: []string{"https://checkout.example/health"}},
|
||||
}}
|
||||
source := &upstreamTaskSourceStub{sources: map[string]*dueSourceStub{
|
||||
"provider-a": {
|
||||
candidates: []Candidate{{ProxyID: "proxy-a", State: proxyDomain.StateFetched, Level: healthDomain.LevelBasic, DueAt: now}},
|
||||
targetCandidates: map[string][]Candidate{targetCandidateKey("checkout", "https://checkout.example/health"): {
|
||||
{ProxyID: "target-a", State: proxyDomain.StateFetched, Level: healthDomain.LevelTarget, RoutingName: "checkout", TargetURL: "https://checkout.example/health", DueAt: now},
|
||||
}},
|
||||
},
|
||||
}}
|
||||
supervisor, err := NewConfiguredSchedulerSupervisor(
|
||||
&versionedConfigurationSourceStub{configuration: configuration, revision: 42}, source, &taskSinkStub{},
|
||||
SchedulerRunnerOptions{PollInterval: time.Second, BatchSize: 16, Now: func() time.Time { return now }},
|
||||
schedulerStateStub{snapshot: adminstate.Snapshot{
|
||||
Config: &adminstate.ConfigRevision{Revision: 42},
|
||||
Upstreams: []adminstate.UpstreamState{{Name: "provider-a", Enabled: true, Revision: 11}},
|
||||
Routings: []adminstate.RoutingState{{Name: "checkout", Enabled: false, Revision: 12}},
|
||||
}},
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("NewConfiguredSchedulerSupervisor() = %v", err)
|
||||
}
|
||||
if result, err := supervisor.Tick(context.Background()); err != nil || result != (TickResult{Planned: 1, Offered: 1}) ||
|
||||
source.sources["provider-a"].dueCalls != 1 || source.sources["provider-a"].targetCalls != 0 {
|
||||
t.Fatalf("Tick() = (%+v, %v); source=%+v", result, err, source.sources["provider-a"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestConfiguredSchedulerSupervisorFailsClosedOnAdminRevisionMismatch(t *testing.T) {
|
||||
now := time.Date(2026, 8, 7, 10, 30, 0, 0, time.UTC)
|
||||
configuration := schedulerConfiguration(now, "provider-a")
|
||||
source := &upstreamTaskSourceStub{sources: map[string]*dueSourceStub{
|
||||
"provider-a": {candidates: []Candidate{{ProxyID: "proxy-a", State: proxyDomain.StateFetched, Level: healthDomain.LevelBasic, DueAt: now}}},
|
||||
}}
|
||||
supervisor, err := NewConfiguredSchedulerSupervisor(
|
||||
&versionedConfigurationSourceStub{configuration: configuration, revision: 42}, source, &taskSinkStub{},
|
||||
SchedulerRunnerOptions{PollInterval: time.Second, BatchSize: 16, Now: func() time.Time { return now }},
|
||||
schedulerStateStub{snapshot: adminstate.Snapshot{Config: &adminstate.ConfigRevision{Revision: 41}}},
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("NewConfiguredSchedulerSupervisor() = %v", err)
|
||||
}
|
||||
if result, err := supervisor.Tick(context.Background()); !errors.Is(err, ErrInvalidSchedulerRunner) || result != (TickResult{}) ||
|
||||
source.sources["provider-a"].inFlightCalls != 0 || source.sources["provider-a"].dueCalls != 0 {
|
||||
t.Fatalf("Tick() = (%+v, %v); source=%+v", result, err, source.sources["provider-a"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestConfiguredSchedulerSupervisorFailsClosedOnIncompleteAdminState(t *testing.T) {
|
||||
now := time.Date(2026, 8, 7, 10, 45, 0, 0, time.UTC)
|
||||
configuration := schedulerConfiguration(now, "provider-a")
|
||||
source := &upstreamTaskSourceStub{sources: map[string]*dueSourceStub{
|
||||
"provider-a": {candidates: []Candidate{{ProxyID: "proxy-a", State: proxyDomain.StateFetched, Level: healthDomain.LevelBasic, DueAt: now}}},
|
||||
}}
|
||||
supervisor, err := NewConfiguredSchedulerSupervisor(
|
||||
&versionedConfigurationSourceStub{configuration: configuration, revision: 42}, source, &taskSinkStub{},
|
||||
SchedulerRunnerOptions{PollInterval: time.Second, BatchSize: 16, Now: func() time.Time { return now }},
|
||||
schedulerStateStub{snapshot: adminstate.Snapshot{Config: &adminstate.ConfigRevision{Revision: 42}}},
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("NewConfiguredSchedulerSupervisor() = %v", err)
|
||||
}
|
||||
if result, err := supervisor.Tick(context.Background()); !errors.Is(err, ErrInvalidSchedulerRunner) || result != (TickResult{}) ||
|
||||
source.sources["provider-a"].inFlightCalls != 0 || source.sources["provider-a"].dueCalls != 0 {
|
||||
t.Fatalf("Tick() = (%+v, %v); source=%+v", result, err, source.sources["provider-a"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestConfiguredSchedulerSupervisorFailsClosedOnMissingAdminRoutingState(t *testing.T) {
|
||||
now := time.Date(2026, 8, 7, 10, 50, 0, 0, time.UTC)
|
||||
configuration := schedulerConfiguration(now, "provider-a")
|
||||
configuration.Routing = []config.Routing{{Name: "checkout", Enabled: true, Upstreams: []string{"provider-a"}}}
|
||||
source := &upstreamTaskSourceStub{sources: map[string]*dueSourceStub{
|
||||
"provider-a": {candidates: []Candidate{{ProxyID: "proxy-a", State: proxyDomain.StateFetched, Level: healthDomain.LevelBasic, DueAt: now}}},
|
||||
}}
|
||||
supervisor, err := NewConfiguredSchedulerSupervisor(
|
||||
&versionedConfigurationSourceStub{configuration: configuration, revision: 42}, source, &taskSinkStub{},
|
||||
SchedulerRunnerOptions{PollInterval: time.Second, BatchSize: 16, Now: func() time.Time { return now }},
|
||||
schedulerStateStub{snapshot: adminstate.Snapshot{
|
||||
Config: &adminstate.ConfigRevision{Revision: 42},
|
||||
Upstreams: []adminstate.UpstreamState{{Name: "provider-a", Enabled: true, Revision: 11}},
|
||||
}},
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("NewConfiguredSchedulerSupervisor() = %v", err)
|
||||
}
|
||||
if result, err := supervisor.Tick(context.Background()); !errors.Is(err, ErrInvalidSchedulerRunner) || result != (TickResult{}) ||
|
||||
source.sources["provider-a"].inFlightCalls != 0 || source.sources["provider-a"].dueCalls != 0 {
|
||||
t.Fatalf("Tick() = (%+v, %v); source=%+v", result, err, source.sources["provider-a"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestConfiguredSchedulerSupervisorRequiresVersionedConfigurationWithAdminState(t *testing.T) {
|
||||
now := time.Date(2026, 8, 7, 11, 0, 0, 0, time.UTC)
|
||||
supervisor, err := NewConfiguredSchedulerSupervisor(
|
||||
&configurationSourceStub{configuration: schedulerConfiguration(now, "provider-a")},
|
||||
&upstreamTaskSourceStub{sources: map[string]*dueSourceStub{"provider-a": {}}}, &taskSinkStub{},
|
||||
SchedulerRunnerOptions{PollInterval: time.Second, BatchSize: 1, Now: func() time.Time { return now }}, schedulerStateStub{},
|
||||
)
|
||||
if !errors.Is(err, ErrInvalidSchedulerRunner) || supervisor != nil {
|
||||
t.Fatalf("NewConfiguredSchedulerSupervisor(non-versioned configuration) = (%v, %v)", supervisor, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestConfiguredSchedulerSupervisorSchedulesBoundedEgressGroups(t *testing.T) {
|
||||
now := time.Date(2026, 8, 2, 11, 0, 0, 0, time.UTC)
|
||||
configuration := &config.Config{
|
||||
@ -465,38 +310,6 @@ func TestConfiguredSchedulerSupervisorNeverExceedsTotalBatchAcrossGroups(t *test
|
||||
}
|
||||
}
|
||||
|
||||
func schedulerConfiguration(now time.Time, upstreams ...string) *config.Config {
|
||||
configured := &config.Config{
|
||||
Defaults: config.Defaults{Check: config.Check{
|
||||
Interval: config.Duration(time.Minute), MaxInFlight: 2, Timeout: config.Duration(time.Second), MaxAttempts: 1,
|
||||
}},
|
||||
Upstreams: make(map[string]config.Upstream, len(upstreams)),
|
||||
}
|
||||
for _, upstream := range upstreams {
|
||||
configured.Upstreams[upstream] = config.Upstream{Enabled: true}
|
||||
}
|
||||
return configured
|
||||
}
|
||||
|
||||
type versionedConfigurationSourceStub struct {
|
||||
configuration *config.Config
|
||||
revision uint64
|
||||
}
|
||||
|
||||
func (source *versionedConfigurationSourceStub) Current() *config.Config {
|
||||
return source.configuration
|
||||
}
|
||||
|
||||
func (source *versionedConfigurationSourceStub) Snapshot() (*config.Config, uint64) {
|
||||
return source.configuration, source.revision
|
||||
}
|
||||
|
||||
type schedulerStateStub struct{ snapshot adminstate.Snapshot }
|
||||
|
||||
func (source schedulerStateStub) Snapshot(context.Context) (adminstate.Snapshot, error) {
|
||||
return source.snapshot, nil
|
||||
}
|
||||
|
||||
type dueSourceStub struct {
|
||||
inFlight int
|
||||
candidates []Candidate
|
||||
|
||||
@ -7,7 +7,6 @@ import (
|
||||
|
||||
"proxy-pool/internal/config"
|
||||
"proxy-pool/internal/domain/activitypool"
|
||||
healthDomain "proxy-pool/internal/domain/health"
|
||||
)
|
||||
|
||||
var ErrInvalidUnhealthyReaper = errors.New("invalid unhealthy reaper")
|
||||
@ -16,7 +15,6 @@ type UnhealthyReaperOptions struct {
|
||||
PollInterval time.Duration
|
||||
BatchSize int
|
||||
Now func() time.Time
|
||||
Metrics healthDomain.DrainMetricsObserver
|
||||
}
|
||||
|
||||
// ConfiguredUnhealthyReaper applies the current upstream grace policies to a
|
||||
@ -64,23 +62,12 @@ func (reaper *ConfiguredUnhealthyReaper) Tick(ctx context.Context) (activitypool
|
||||
}
|
||||
drainer, supported := reaper.store.(activitypool.UnhealthyDrainStarter)
|
||||
if !supported || nilInterface(drainer) {
|
||||
if reaper.options.Metrics != nil {
|
||||
reaper.options.Metrics.ObserveDrain(healthDomain.DrainReasonUnhealthy, len(result.DrainCandidates), 0)
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
started := 0
|
||||
for _, candidate := range result.DrainCandidates {
|
||||
didStart, err := drainer.BeginUnhealthyDrain(ctx, utcNow, candidate)
|
||||
if err != nil {
|
||||
if _, err := drainer.BeginUnhealthyDrain(ctx, utcNow, candidate); err != nil {
|
||||
return result, err
|
||||
}
|
||||
if didStart {
|
||||
started++
|
||||
}
|
||||
}
|
||||
if reaper.options.Metrics != nil {
|
||||
reaper.options.Metrics.ObserveDrain(healthDomain.DrainReasonUnhealthy, len(result.DrainCandidates), started)
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
@ -7,7 +7,6 @@ import (
|
||||
|
||||
"proxy-pool/internal/config"
|
||||
"proxy-pool/internal/domain/activitypool"
|
||||
healthDomain "proxy-pool/internal/domain/health"
|
||||
)
|
||||
|
||||
type unhealthyRemoverStub struct {
|
||||
@ -86,12 +85,11 @@ func TestConfiguredUnhealthyReaperStartsBoundedDrainCandidates(t *testing.T) {
|
||||
store := &unhealthyRemoverStub{result: activitypool.UnhealthySweepResult{
|
||||
DeferredOwned: 1, DrainCandidates: []activitypool.UnhealthyDrainCandidate{candidate},
|
||||
}}
|
||||
metrics := &recordingDrainMetrics{}
|
||||
reaper, err := NewConfiguredUnhealthyReaper(&configurationSourceStub{configuration: &config.Config{
|
||||
Upstreams: map[string]config.Upstream{
|
||||
"provider-a": {Enabled: true, Check: config.Check{UnhealthyRemoveAfter: config.Duration(time.Minute)}},
|
||||
},
|
||||
}}, store, UnhealthyReaperOptions{PollInterval: time.Second, BatchSize: 4, Now: func() time.Time { return now }, Metrics: metrics})
|
||||
}}, store, UnhealthyReaperOptions{PollInterval: time.Second, BatchSize: 4, Now: func() time.Time { return now }})
|
||||
if err != nil {
|
||||
t.Fatalf("NewConfiguredUnhealthyReaper() = %v", err)
|
||||
}
|
||||
@ -102,9 +100,4 @@ func TestConfiguredUnhealthyReaperStartsBoundedDrainCandidates(t *testing.T) {
|
||||
if store.drainCalls != 1 || !store.drainNow.Equal(now) || len(store.candidates) != 1 || store.candidates[0] != candidate {
|
||||
t.Fatalf("BeginUnhealthyDrain() calls=%d now=%v candidates=%+v", store.drainCalls, store.drainNow, store.candidates)
|
||||
}
|
||||
if len(metrics.records) != 1 || metrics.records[0] != (drainMetricRecord{
|
||||
reason: healthDomain.DrainReasonUnhealthy, candidates: 1, started: 1,
|
||||
}) {
|
||||
t.Fatalf("Drain metrics = %+v", metrics.records)
|
||||
}
|
||||
}
|
||||
|
||||
@ -1,211 +0,0 @@
|
||||
package health
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"sort"
|
||||
"time"
|
||||
|
||||
"proxy-pool/internal/config"
|
||||
"proxy-pool/internal/domain/activitypool"
|
||||
"proxy-pool/internal/domain/adminstate"
|
||||
healthDomain "proxy-pool/internal/domain/health"
|
||||
)
|
||||
|
||||
var ErrInvalidUpstreamDrainReaper = errors.New("invalid upstream drain reaper")
|
||||
|
||||
// UpstreamDrainConfigurationSource returns one configuration and revision from
|
||||
// the same atomic publication. config.Store implements this interface.
|
||||
type UpstreamDrainConfigurationSource interface {
|
||||
Snapshot() (*config.Config, uint64)
|
||||
}
|
||||
|
||||
type UpstreamDrainStateSource interface {
|
||||
Snapshot(context.Context) (adminstate.Snapshot, error)
|
||||
}
|
||||
|
||||
// EffectiveUpstreamDrainPolicySource merges static configuration with optional
|
||||
// Admin state. With Admin enabled it uses the same revision equality rule as
|
||||
// Worker routing, so a torn config/state view produces no policy publication.
|
||||
type EffectiveUpstreamDrainPolicySource struct {
|
||||
configuration UpstreamDrainConfigurationSource
|
||||
state UpstreamDrainStateSource
|
||||
}
|
||||
|
||||
func NewEffectiveUpstreamDrainPolicySource(
|
||||
configuration UpstreamDrainConfigurationSource,
|
||||
states ...UpstreamDrainStateSource,
|
||||
) (*EffectiveUpstreamDrainPolicySource, error) {
|
||||
if nilInterface(configuration) || len(states) > 1 || (len(states) == 1 && nilInterface(states[0])) {
|
||||
return nil, ErrInvalidUpstreamDrainReaper
|
||||
}
|
||||
source := &EffectiveUpstreamDrainPolicySource{configuration: configuration}
|
||||
if len(states) == 1 {
|
||||
source.state = states[0]
|
||||
}
|
||||
return source, nil
|
||||
}
|
||||
|
||||
func (source *EffectiveUpstreamDrainPolicySource) ReadUpstreamDrainPolicies(
|
||||
ctx context.Context,
|
||||
) ([]activitypool.UpstreamDrainPolicy, error) {
|
||||
if ctx == nil || source == nil || nilInterface(source.configuration) {
|
||||
return nil, ErrInvalidUpstreamDrainReaper
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
configuration, revision := source.configuration.Snapshot()
|
||||
if configuration == nil {
|
||||
return nil, ErrInvalidUpstreamDrainReaper
|
||||
}
|
||||
states := make(map[string]adminstate.UpstreamState, len(configuration.Upstreams))
|
||||
if source.state != nil {
|
||||
snapshot, err := source.state.Snapshot(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if snapshot.Config == nil || snapshot.Config.Revision != revision {
|
||||
return nil, ErrInvalidUpstreamDrainReaper
|
||||
}
|
||||
for _, upstream := range snapshot.Upstreams {
|
||||
if upstream.Name == "" || upstream.Revision == 0 {
|
||||
return nil, ErrInvalidUpstreamDrainReaper
|
||||
}
|
||||
if _, duplicate := states[upstream.Name]; duplicate {
|
||||
return nil, ErrInvalidUpstreamDrainReaper
|
||||
}
|
||||
states[upstream.Name] = upstream
|
||||
}
|
||||
}
|
||||
if revision == 0 {
|
||||
// A config-only Controller has no persisted management revision. Its
|
||||
// configuration cannot be hot-reloaded, so one stable non-zero fence is enough.
|
||||
revision = 1
|
||||
}
|
||||
names := make([]string, 0, len(configuration.Upstreams))
|
||||
for name := range configuration.Upstreams {
|
||||
names = append(names, name)
|
||||
}
|
||||
sort.Strings(names)
|
||||
policies := make([]activitypool.UpstreamDrainPolicy, 0, len(names))
|
||||
for _, name := range names {
|
||||
upstream := configuration.Upstreams[name]
|
||||
policy := activitypool.UpstreamDrainPolicy{UpstreamID: name, Revision: revision, Enabled: upstream.Enabled}
|
||||
if source.state != nil {
|
||||
admin, exists := states[name]
|
||||
if !exists {
|
||||
return nil, ErrInvalidUpstreamDrainReaper
|
||||
}
|
||||
policy.Enabled = policy.Enabled && admin.Enabled
|
||||
policy.Revision = admin.Revision
|
||||
}
|
||||
policies = append(policies, policy)
|
||||
}
|
||||
return policies, nil
|
||||
}
|
||||
|
||||
type UpstreamDrainPolicySource interface {
|
||||
ReadUpstreamDrainPolicies(context.Context) ([]activitypool.UpstreamDrainPolicy, error)
|
||||
}
|
||||
|
||||
// UpstreamDrainStore is the narrow activity-pool capability required by the
|
||||
// configuration-driven Drain reaper.
|
||||
type UpstreamDrainStore interface {
|
||||
activitypool.UpstreamDrainPolicyWriter
|
||||
activitypool.DisabledUpstreamDrainLister
|
||||
activitypool.DisabledUpstreamDrainStarter
|
||||
}
|
||||
|
||||
type UpstreamDrainReaperOptions struct {
|
||||
PollInterval time.Duration
|
||||
BatchSize int
|
||||
Now func() time.Time
|
||||
Metrics healthDomain.DrainMetricsObserver
|
||||
}
|
||||
|
||||
type UpstreamDrainResult struct {
|
||||
Candidates int
|
||||
Started int
|
||||
}
|
||||
|
||||
// ConfiguredUpstreamDrainReaper continually publishes an effective, complete
|
||||
// policy view before reading one bounded batch of disabled Upstream ownership.
|
||||
// The Redis ownership operation performs the final policy-revision fence.
|
||||
type ConfiguredUpstreamDrainReaper struct {
|
||||
policies UpstreamDrainPolicySource
|
||||
store UpstreamDrainStore
|
||||
options UpstreamDrainReaperOptions
|
||||
}
|
||||
|
||||
func NewConfiguredUpstreamDrainReaper(
|
||||
policies UpstreamDrainPolicySource,
|
||||
store UpstreamDrainStore,
|
||||
options UpstreamDrainReaperOptions,
|
||||
) (*ConfiguredUpstreamDrainReaper, error) {
|
||||
if nilInterface(policies) || nilInterface(store) || options.PollInterval <= 0 || options.BatchSize <= 0 || options.Now == nil {
|
||||
return nil, ErrInvalidUpstreamDrainReaper
|
||||
}
|
||||
return &ConfiguredUpstreamDrainReaper{policies: policies, store: store, options: options}, nil
|
||||
}
|
||||
|
||||
func (reaper *ConfiguredUpstreamDrainReaper) Tick(ctx context.Context) (UpstreamDrainResult, error) {
|
||||
if ctx == nil || reaper == nil || nilInterface(reaper.policies) || nilInterface(reaper.store) ||
|
||||
reaper.options.PollInterval <= 0 || reaper.options.BatchSize <= 0 || reaper.options.Now == nil {
|
||||
return UpstreamDrainResult{}, ErrInvalidUpstreamDrainReaper
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return UpstreamDrainResult{}, err
|
||||
}
|
||||
now := reaper.options.Now()
|
||||
if now.IsZero() {
|
||||
return UpstreamDrainResult{}, ErrInvalidUpstreamDrainReaper
|
||||
}
|
||||
policies, err := reaper.policies.ReadUpstreamDrainPolicies(ctx)
|
||||
if err != nil {
|
||||
return UpstreamDrainResult{}, err
|
||||
}
|
||||
if err := reaper.store.ReplaceUpstreamDrainPolicies(ctx, policies); err != nil {
|
||||
return UpstreamDrainResult{}, err
|
||||
}
|
||||
result := UpstreamDrainResult{}
|
||||
remaining := reaper.options.BatchSize
|
||||
utcNow := now.UTC()
|
||||
for _, policy := range policies {
|
||||
if policy.Enabled || remaining == 0 {
|
||||
continue
|
||||
}
|
||||
candidates, err := reaper.store.ListDisabledUpstreamDrainCandidates(ctx, utcNow, policy, remaining)
|
||||
if err != nil {
|
||||
return result, err
|
||||
}
|
||||
if len(candidates) > remaining {
|
||||
return result, ErrInvalidUpstreamDrainReaper
|
||||
}
|
||||
for _, candidate := range candidates {
|
||||
started, err := reaper.store.BeginDisabledUpstreamDrain(ctx, utcNow, candidate)
|
||||
if err != nil {
|
||||
return result, err
|
||||
}
|
||||
result.Candidates++
|
||||
if started {
|
||||
result.Started++
|
||||
}
|
||||
}
|
||||
remaining -= len(candidates)
|
||||
}
|
||||
if reaper.options.Metrics != nil {
|
||||
reaper.options.Metrics.ObserveDrain(healthDomain.DrainReasonUpstreamDisabled, result.Candidates, result.Started)
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func (reaper *ConfiguredUpstreamDrainReaper) Run(ctx context.Context) error {
|
||||
if reaper == nil {
|
||||
return ErrInvalidUpstreamDrainReaper
|
||||
}
|
||||
return runScheduler(ctx, reaper.options.PollInterval, func(tickCtx context.Context) (TickResult, error) {
|
||||
_, err := reaper.Tick(tickCtx)
|
||||
return TickResult{}, err
|
||||
})
|
||||
}
|
||||
@ -1,133 +0,0 @@
|
||||
package health
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"proxy-pool/internal/config"
|
||||
"proxy-pool/internal/domain/activitypool"
|
||||
"proxy-pool/internal/domain/adminstate"
|
||||
healthDomain "proxy-pool/internal/domain/health"
|
||||
proxyDomain "proxy-pool/internal/domain/proxy"
|
||||
)
|
||||
|
||||
type upstreamDrainPolicySourceStub struct {
|
||||
policies []activitypool.UpstreamDrainPolicy
|
||||
err error
|
||||
}
|
||||
|
||||
func (source upstreamDrainPolicySourceStub) ReadUpstreamDrainPolicies(context.Context) ([]activitypool.UpstreamDrainPolicy, error) {
|
||||
return append([]activitypool.UpstreamDrainPolicy(nil), source.policies...), source.err
|
||||
}
|
||||
|
||||
type upstreamDrainConfigurationStub struct {
|
||||
configuration *config.Config
|
||||
revision uint64
|
||||
}
|
||||
|
||||
type upstreamDrainStateStub struct{ snapshot adminstate.Snapshot }
|
||||
|
||||
func (source upstreamDrainStateStub) Snapshot(context.Context) (adminstate.Snapshot, error) {
|
||||
return source.snapshot, nil
|
||||
}
|
||||
|
||||
type drainMetricRecord struct {
|
||||
reason healthDomain.DrainReason
|
||||
candidates int
|
||||
started int
|
||||
}
|
||||
|
||||
type recordingDrainMetrics struct{ records []drainMetricRecord }
|
||||
|
||||
func (metrics *recordingDrainMetrics) ObserveDrain(reason healthDomain.DrainReason, candidates, started int) {
|
||||
metrics.records = append(metrics.records, drainMetricRecord{reason: reason, candidates: candidates, started: started})
|
||||
}
|
||||
|
||||
func (source upstreamDrainConfigurationStub) Snapshot() (*config.Config, uint64) {
|
||||
return source.configuration, source.revision
|
||||
}
|
||||
|
||||
func TestConfiguredUpstreamDrainReaperStartsBoundedDisabledAssignments(t *testing.T) {
|
||||
now := time.Date(2026, 8, 2, 16, 0, 0, 0, time.UTC)
|
||||
pool := activitypool.NewMemoryPool()
|
||||
if _, err := pool.UpsertFetched(context.Background(), "provider-a", activitypool.FetchedBatch{
|
||||
ObservedAt: now, ConfiguredTTL: time.Minute, MaxSize: 1,
|
||||
Proxies: []proxyDomain.Proxy{{
|
||||
ID: "proxy-a", Scheme: proxyDomain.SchemeHTTP, Host: "192.0.2.10", Port: 8080, State: proxyDomain.StateAvailable,
|
||||
}},
|
||||
}); err != nil {
|
||||
t.Fatalf("UpsertFetched() error = %v", err)
|
||||
}
|
||||
assignment, err := pool.Assign(context.Background(), now, "proxy-a", "worker-a", time.Minute)
|
||||
if err != nil {
|
||||
t.Fatalf("Assign() error = %v", err)
|
||||
}
|
||||
policy := activitypool.UpstreamDrainPolicy{UpstreamID: "provider-a", Revision: 21}
|
||||
metrics := &recordingDrainMetrics{}
|
||||
reaper, err := NewConfiguredUpstreamDrainReaper(upstreamDrainPolicySourceStub{policies: []activitypool.UpstreamDrainPolicy{policy}}, pool,
|
||||
UpstreamDrainReaperOptions{PollInterval: time.Second, BatchSize: 4, Now: func() time.Time { return now }, Metrics: metrics})
|
||||
if err != nil {
|
||||
t.Fatalf("NewConfiguredUpstreamDrainReaper() error = %v", err)
|
||||
}
|
||||
result, err := reaper.Tick(context.Background())
|
||||
if err != nil || result != (UpstreamDrainResult{Candidates: 1, Started: 1}) {
|
||||
t.Fatalf("Tick() = %+v, %v", result, err)
|
||||
}
|
||||
if current, found, err := pool.Get(context.Background(), "proxy-a"); err != nil || !found ||
|
||||
current.Epoch != assignment.Epoch || !current.Draining {
|
||||
t.Fatalf("Get(after drain) = %+v, %t, %v", current, found, err)
|
||||
}
|
||||
if len(metrics.records) != 1 || metrics.records[0] != (drainMetricRecord{
|
||||
reason: healthDomain.DrainReasonUpstreamDisabled, candidates: 1, started: 1,
|
||||
}) {
|
||||
t.Fatalf("Drain metrics = %+v", metrics.records)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEffectiveUpstreamDrainPolicySourceUsesStableConfigOnlyRevision(t *testing.T) {
|
||||
configuration := &config.Config{Upstreams: map[string]config.Upstream{
|
||||
"provider-b": {Enabled: true},
|
||||
"provider-a": {Enabled: false},
|
||||
}}
|
||||
source, err := NewEffectiveUpstreamDrainPolicySource(upstreamDrainConfigurationStub{configuration: configuration})
|
||||
if err != nil {
|
||||
t.Fatalf("NewEffectiveUpstreamDrainPolicySource() error = %v", err)
|
||||
}
|
||||
policies, err := source.ReadUpstreamDrainPolicies(context.Background())
|
||||
if err != nil || len(policies) != 2 || policies[0] != (activitypool.UpstreamDrainPolicy{
|
||||
UpstreamID: "provider-a", Revision: 1,
|
||||
}) || policies[1] != (activitypool.UpstreamDrainPolicy{
|
||||
UpstreamID: "provider-b", Revision: 1, Enabled: true,
|
||||
}) {
|
||||
t.Fatalf("ReadUpstreamDrainPolicies() = %+v, %v", policies, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEffectiveUpstreamDrainPolicySourceCombinesConfigurationAndAdminState(t *testing.T) {
|
||||
configuration := &config.Config{Upstreams: map[string]config.Upstream{
|
||||
"provider-a": {Enabled: false},
|
||||
"provider-b": {Enabled: true},
|
||||
}}
|
||||
source, err := NewEffectiveUpstreamDrainPolicySource(
|
||||
upstreamDrainConfigurationStub{configuration: configuration, revision: 9},
|
||||
upstreamDrainStateStub{snapshot: adminstate.Snapshot{
|
||||
Config: &adminstate.ConfigRevision{Revision: 9},
|
||||
Upstreams: []adminstate.UpstreamState{
|
||||
{Name: "provider-a", Enabled: true, Revision: 7},
|
||||
{Name: "provider-b", Enabled: false, Revision: 8},
|
||||
},
|
||||
}},
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("NewEffectiveUpstreamDrainPolicySource() error = %v", err)
|
||||
}
|
||||
policies, err := source.ReadUpstreamDrainPolicies(context.Background())
|
||||
want := []activitypool.UpstreamDrainPolicy{
|
||||
{UpstreamID: "provider-a", Revision: 7},
|
||||
{UpstreamID: "provider-b", Revision: 8},
|
||||
}
|
||||
if err != nil || len(policies) != len(want) || policies[0] != want[0] || policies[1] != want[1] {
|
||||
t.Fatalf("ReadUpstreamDrainPolicies() = %+v, %v; want %+v", policies, err, want)
|
||||
}
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue
Block a user