youfak/proxy-pool
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Compare commits

base: youfak:e9945d933f6c064dd74c47feae048a49a1e7df51
Branches Tags
youfak:build/proxy-pool-architecture
...
compare: youfak:b8f51041672089616f2c5b4cc5e56f36e8125786
Branches Tags
youfak:build/proxy-pool-architecture

5 Commits
e9945d933f ... b8f5104167

Author SHA1 Message Date
youfak
b8f5104167 docs: record provider lifecycle delivery
Some checks are pending
ci / test (ubuntu-latest) (push) Waiting to run
Details
ci / test (windows-latest) (push) Waiting to run
Details
ci / race (push) Waiting to run
Details
ci / integration (push) Waiting to run
Details
2026-07-30 19:47:10 +08:00
youfak
8997e3880e feat: run provider lifecycle in controller 2026-07-30 19:47:02 +08:00
youfak
40f4b3ffab feat: enforce distributed provider fetch quota 2026-07-30 16:11:04 +08:00
youfak
d648ba37e0 docs: record worker runtime capacity delivery 2026-07-30 15:46:27 +08:00
youfak
45ba6fb958 feat: add authoritative worker runtime capacity 2026-07-30 15:46:18 +08:00
95 changed files with 6683 additions and 511 deletions
Show Stats Expand all files Collapse all files

3
README.md
View File

@ -52,6 +52,7 @@ Windows PowerShell 可运行：
2. Proxy 容量使用 `Reserved -> Active` 原子转换，禁止超卖。
3. Distribution 成功时原子执行 `AVAILABLE -> EXTRACTED`，不提供 Lease、
Release 或 Renewal。
4. `pool.maxSize` 是当前未提取库存上限；`fetch.maxTotal` 是累计获取额度。
4. `pool.maxSize` 是当前未提取库存硬上限；`fetch.maxTotal` 是 Redis generation
内的累计获取停止阈值。
5. CONNECT 向客户端提交 200 后不透明重放。
6. 公开监听必须有认证或 CIDR 访问保护。

1
api/proto/controlplane/v1/controlplane.proto
View File

@ -182,6 +182,7 @@ message ReportRuntimeRequest {
uint64 ownership_epoch = 4;
repeated ProxyRuntime counters = 5;
google.protobuf.Timestamp observed_at = 6;
uint64 report_sequence = 7;
}
message ProxyRuntime {

13
cmd/proxy-controller/main.go
View File

@ -15,7 +15,10 @@ import (
"proxy-pool/internal/controller/bootstrap"
)
const configEnvironment = "PROXY_POOL_CONFIG"
const (
configEnvironment = "PROXY_POOL_CONFIG"
fingerprintKeyEnvironment = "PROXY_POOL_CONFIG_FINGERPRINT_KEY"
)
type environmentLookup func(string) string
type controllerRun func(context.Context, bootstrap.Options) error
@ -54,7 +57,13 @@ func execute(
return 2
}
err := run(ctx, bootstrap.Options{ConfigPath: *configPath, Resolver: config.OSResolver{}})
var fingerprintKey []byte
if getenv != nil {
fingerprintKey = []byte(getenv(fingerprintKeyEnvironment))
}
err := run(ctx, bootstrap.Options{
ConfigPath: *configPath, Resolver: config.OSResolver{}, FingerprintKey: fingerprintKey,
})
if err == nil || (errors.Is(err, context.Canceled) && ctx.Err() != nil) {
return 0
}

12
cmd/proxy-controller/main_test.go
View File

@ -16,12 +16,15 @@ func TestExecuteUsesFlagBeforeEnvironment(t *testing.T) {
if name == configEnvironment {
return "environment.yaml"
}
if name == fingerprintKeyEnvironment {
return "0123456789abcdef0123456789abcdef"
}
return ""
}, func(_ context.Context, options bootstrap.Options) error {
received = options
return nil
}, &bytes.Buffer{})
if code != 0 || received.ConfigPath != "flag.yaml" || received.Resolver == nil {
if code != 0 || received.ConfigPath != "flag.yaml" || received.Resolver == nil || len(received.FingerprintKey) != 32 {
t.Fatalf("execute() = %d, options = %+v", code, received)
}
}
@ -29,7 +32,12 @@ func TestExecuteUsesFlagBeforeEnvironment(t *testing.T) {
func TestExecuteFallsBackToEnvironment(t *testing.T) {
t.Parallel()
var received bootstrap.Options
code := execute(context.Background(), nil, func(string) string { return "environment.yaml" }, func(
code := execute(context.Background(), nil, func(name string) string {
if name == configEnvironment {
return "environment.yaml"
}
return "0123456789abcdef0123456789abcdef"
}, func(
_ context.Context,
options bootstrap.Options,
) error {

19
deploy/docker-compose.yml
View File

@ -1,5 +1,13 @@
name: proxy-pool
x-app-environment: &app-environment
PROXY_POOL_CONFIG: /etc/proxy-pool/config.yaml
PROXY_POOL_GATEWAY_PASSWORD: ${PROXY_POOL_GATEWAY_PASSWORD:?set PROXY_POOL_GATEWAY_PASSWORD}
PROXY_POOL_EXTRACT_TOKEN: ${PROXY_POOL_EXTRACT_TOKEN:?set PROXY_POOL_EXTRACT_TOKEN}
PROXY_POOL_ADMIN_TOKEN: ${PROXY_POOL_ADMIN_TOKEN:?set PROXY_POOL_ADMIN_TOKEN}
PROVIDER_A_TOKEN: ${PROVIDER_A_TOKEN:?set PROVIDER_A_TOKEN}
PROVIDER_B_TOKEN: ${PROVIDER_B_TOKEN:?set PROVIDER_B_TOKEN}
x-app: &app
build:
context: ..
@ -9,13 +17,7 @@ x-app: &app
networks: [frontend, backend]
volumes:
- ./config/local.yaml:/etc/proxy-pool/config.yaml:ro
environment:
PROXY_POOL_CONFIG: /etc/proxy-pool/config.yaml
PROXY_POOL_GATEWAY_PASSWORD: ${PROXY_POOL_GATEWAY_PASSWORD:?set PROXY_POOL_GATEWAY_PASSWORD}
PROXY_POOL_EXTRACT_TOKEN: ${PROXY_POOL_EXTRACT_TOKEN:?set PROXY_POOL_EXTRACT_TOKEN}
PROXY_POOL_ADMIN_TOKEN: ${PROXY_POOL_ADMIN_TOKEN:?set PROXY_POOL_ADMIN_TOKEN}
PROVIDER_A_TOKEN: ${PROVIDER_A_TOKEN:?set PROVIDER_A_TOKEN}
PROVIDER_B_TOKEN: ${PROVIDER_B_TOKEN:?set PROVIDER_B_TOKEN}
environment: *app-environment
stop_grace_period: 45s
services:
@ -44,6 +46,9 @@ services:
controller:
<<: *app
command: ["proxy-controller"]
environment:
<<: *app-environment
PROXY_POOL_CONFIG_FINGERPRINT_KEY: ${PROXY_POOL_CONFIG_FINGERPRINT_KEY:?set PROXY_POOL_CONFIG_FINGERPRINT_KEY}
depends_on:
postgres:
condition: service_healthy

2
deploy/kubernetes/base/secret.example.yaml
View File

@ -9,8 +9,8 @@ stringData:
PROXY_POOL_GATEWAY_PASSWORD: GATEWAY_PASSWORD
PROXY_POOL_EXTRACT_TOKEN: EXTRACT_TOKEN
PROXY_POOL_ADMIN_TOKEN: ADMIN_TOKEN
PROXY_POOL_CONFIG_FINGERPRINT_KEY: CONFIG_FINGERPRINT_KEY_MINIMUM_32_BYTES
PROXY_POOL_POSTGRES_URL: postgres://USER:PASSWORD@POSTGRES_HOST:5432/proxy_pool?sslmode=verify-full
PROXY_POOL_REDIS_URL: rediss://:PASSWORD@REDIS_HOST:6379/0
PROVIDER_A_TOKEN: PROVIDER_A_TOKEN
PROVIDER_B_TOKEN: PROVIDER_B_TOKEN

47
docs/adr/005-redis-activity-pool.md
View File

@ -63,10 +63,14 @@ var _ activitypool.HealthStore = (*Adapter)(nil)
var _ activitypool.InventoryReader = (*Adapter)(nil)
var _ extraction.Store = (*Adapter)(nil)
var _ ownership.Repository = (*Adapter)(nil)
var _ workerruntime.SessionWriter = (*Adapter)(nil)
var _ workerruntime.ReportWriter = (*Adapter)(nil)
var _ workerruntime.RuntimeReader = (*Adapter)(nil)
var _ pool.InventoryReader = (*Adapter)(nil)
```
Provider、Checker、Distribution 和 Ownership 只依赖各自需要的端口，不直接
依赖 Redis 客户端、键名、Lua 返回格式或清理策略。
Provider、Checker、Distribution、Ownership 和 Worker Runtime 只依赖各自需要
的端口，不直接依赖 Redis 客户端、键名、Lua 返回格式或清理策略。
`ownership.Repository` 改为适合远程存储的上下文感知接口：
@ -97,6 +101,11 @@ pp:{activity}:owners HASH proxyID -> ownership assignment
pp:{activity}:owner-expiry ZSET proxyID -> ownership expiry milliseconds
pp:{activity}:epoch STRING ownership 全局递增代次
pp:{activity}:inventory HASH upstreamID -> 当前未提取库存
pp:{activity}:worker-sessions HASH workerID -> 当前 Worker session
pp:{activity}:worker-session-expiry ZSET workerID -> session expiry milliseconds
pp:{activity}:worker-runtime HASH workerID -> 完整稀疏运行态报告
pp:{activity}:worker-runtime-expiry ZSET workerID -> report expiry milliseconds
pp:{activity}:owned:<digest> ZSET 单 Upstream 已分配 AVAILABLE Proxy
pp:{activity}:idem:<digest> STRING 带 TTL 的提取幂等结果
pp:{activity}:op:<digest> STRING 带 TTL 的内部操作结果
```
@ -167,6 +176,38 @@ Assign、Renew、BeginDrain 和 AcknowledgeDrain 分别使用有界小脚本，
- Assign 与 Extract 并发竞争同一 Proxy 时，只允许一个操作成功。
- Expire 使用 `limit` 分批回收过期 assignment，禁止无界返回。
### Worker 运行态与容量汇总
Gateway 的 `Capacity` 使用一次打包原子读取取得同一时刻的 Active/Reserved，
`snapshot.Store` 周期生成完整稀疏报告。当前 Snapshot 已移除但仍有活动连接的
Proxy 继续以 `draining=true` 上报，直到 Active/Reserved 同时归零。
Redis Adapter 在单个 `{activity}` 原子边界内维护 Worker session 和运行态报告：
1. 新 Worker session 替换旧 session，并隔离旧实例后续写入。
2. session 保存 Controller 已 ACK 的 snapshot version 与 ownership epoch；报告
必须与 ACK 上界完全一致，不能通过自报超前 epoch 绕过所有权校验。
3. `report_sequence` 严格递增；同序号、同内容可幂等重放，冲突或倒序拒绝。
4. 非零计数必须匹配当前 Proxy owner、Worker ID 和 ownership epoch。
5. session/report TTL 使用 Redis 服务端时间；过期、缺失或损坏时容量 fail-closed。
6. 空报告清除该 Worker 的全部旧计数，稀疏报告中缺失的 Proxy 计数视为零。
`pool.InventoryReader` 低频返回单个 Upstream 的 `Managed` 和
`AvailableSlots`。Managed 统计 FETCHED/CHECKING/AVAILABLE/SUSPECT/DRAINING；
Available Slots 只统计超过 safety margin、状态为 AVAILABLE 且所有权与新鲜
Worker 运行态一致的 `max - active - reserved`。未分配 Proxy 可直接贡献 Max；
已分配但运行态未知的 Proxy 贡献零槽位。PostgreSQL 不保存这些短效报告或容量
明细，Gateway 每次请求也不访问 Redis。
Managed 直接读取现有 Upstream 权威计数；Available Slots 只扫描目标 Upstream
的未分配可用索引与已分配可用索引，不扫描全局 Proxy，也不受其他供应商活记录
数量影响。索引成员数超过单次扫描预算时直接返回不可用，不降级为近似容量。
生产规模验收仍需验证脚本 p95/p99、CPU、内存和过期风暴下的有界行为。
Gateway `snapshot.Store` 扫描有界的当前 Snapshot，并使用分片索引补充已移除但
仍非零的 runtime，不扫描全部历史 Proxy；历史 Capacity 注册表设硬上限，达到
上限时拒绝新 Snapshot 并保持旧视图，避免长期轮换造成无界内存增长。
### 短 TTL 清理
Redis Hash 字段没有独立 TTL，因此使用三层有界清理：
@ -226,6 +267,8 @@ Redis `inventory` 是当前未提取 Proxy 数量的运行时真值：
- Assign 与 Extract 并发互斥，以及 renew/drain/ACK/expire。
- 提交后连接断开、脚本缓存丢失、上下文取消和 Redis 不可用。
- 30 秒 TTL 持续写入下的有界清理与库存一致性。
- Worker session 替换、运行态序号幂等/冲突、报告过期和 ownership epoch 隔离。
- 权威 Managed/Available Slots 聚合及扫描预算耗尽时的 fail-closed 行为。
Lua 语义必须使用真实 Redis 8.2 集成测试验证。单元测试最长 60 秒，并执行
gofmt、go vet、全量测试、构建和 diff whitespace 检查。100,000 QPS 只能由

11
docs/adr/006-postgresql-admin-state.md
View File

@ -70,9 +70,14 @@ Outbox。状态变化、审计和 Outbox 在同一事务提交。
### 配置修订
配置提交只保存管理面恢复所需的非敏感事实：配置版本、SHA-256 校验和、来源、
Upstream 启用状态和 Routing 候选/当前选择。已解析 Secret、Provider Token、
Proxy 凭据和完整运行时对象不进入 PostgreSQL。
配置提交只保存管理面恢复所需的非敏感事实：配置版本、完整已解析配置的
HMAC-SHA-256 指纹、来源、Upstream 启用状态和 Routing 候选/当前选择。HMAC 使用
独立外部高熵密钥，覆盖 Secret 轮换以驱动多副本收敛，同时避免普通摘要成为
低熵 Secret 的离线校验器。HMAC 密钥、已解析 Secret、Provider Token、Proxy
凭据、配置正文和完整运行时对象均不进入 PostgreSQL。
配置发布携带事务返回的全局 revision。本地 `config.Store` 只接受严格递增
revision，因此并发提交或 Supervisor 同步的迟到旧版本不能覆盖较新运行配置。
配置重载以一个事务替换管理快照。新 Routing 的当前 Upstream 必须属于其候选集，
所有引用的 Upstream 必须存在，名称与列表必须非空且唯一。校验失败发生在事务前，

20
docs/api/admin.md
View File

@ -39,10 +39,24 @@ Status 以一个权威管理快照决定 Upstream 集合和 Enabled 状态，只
1. `FileConfigurationLoader` 通过 `config.LoadResolved` 严格解析、解析 Secret 引用
并完成全量校验。
2. 从脱敏管理投影计算版本与校验和；Secret 值及其可验证摘要不进入管理状态。
2. 使用独立外部密钥对完整已解析配置计算 HMAC-SHA-256 指纹；PostgreSQL 只保存
不透明 HMAC，不保存密钥、配置正文或 Secret 明文，因此 URL、模板和 Secret
轮换都会产生新版本，也不能借数据库摘要离线猜测低熵 Secret。
3. 在同一 `adminstate` mutation 中提交配置修订、管理状态、审计和 Outbox。
4. 提交成功后由 `config.Store` 一次原子指针交换发布完整运行配置；提交失败时旧
配置保持不变。幂等重放仍执行发布，以修复进程本地状态。
4. 提交成功后由 `config.Store` 按 PostgreSQL revision 原子发布完整运行配置；
Store 只接受严格递增 revision，提交失败或迟到旧 revision 不覆盖当前配置。
幂等重放仅在本地 revision 落后时修复进程状态。
5. Provider Supervisor 在提交前构造预检所有启用 Upstream；发布后按新配置取消、
替换或新增 Runtime。enable 同样在管理状态 mutation 前预检目标 Runtime。
6. 其他 Controller 每秒比较本地指纹与 PostgreSQL 权威指纹；所有副本必须使用
相同 `PROXY_POOL_CONFIG_FINGERPRINT_KEY`。共享配置源已同步时严格重载、预检
并按 revision 发布，源尚未同步时停止旧 Provider Runtime，禁止旧 URL/Secret
在换主后继续调用。
disable 成功后会立即通知 Supervisor 取消目标 Runtime；每秒一次的权威状态对账
用于修复进程内通知丢失。已取得的分布式 Permit 仍按幂等、保守规则完成结算。
PostgreSQL 瞬时读取失败不会终止 Controller 或取消当前 Provider Runtime；
Supervisor 保留 last-known 状态并在下一周期重试。
主配置或 Secret 文件 I/O 故障归类为 503；语法、未知字段、引用和语义校验失败
归类为 422。请求取消和截止时间保持原始上下文错误，不误报为配置错误。

12
docs/api/control-plane.md
View File

@ -34,6 +34,18 @@ sequenceDiagram
`worker_id` 是逻辑节点，`instance_id` 区分进程重启，`session_id` 防止旧进程
继续上报。所有权 `epoch` 小于 Controller 当前值的数据必须拒绝。
`ReportRuntimeRequest.report_sequence` 在当前 `session_id` 内严格单调递增。
相同序号只允许内容完全相同的幂等重放；较小序号或相同序号的不同内容必须
拒绝。`observed_at` 只用于观测，不作为乱序判定依据，运行态 TTL 统一使用
Controller 侧 Redis 服务端时间。session 同时保存 Controller 已接受的
`snapshot_version/ownership_epoch`；运行态报告必须与该 ACK 上界完全一致，
Worker 自报的超前版本或 epoch 也必须拒绝。
运行态上报是完整稀疏替换：只携带 Active/Reserved 非零的 Proxy，空列表表示
当前会话全部归零。Controller 只有在 Worker session、报告 TTL、Proxy ownership
和 ownership epoch 同时有效时才使用计数；报告缺失、过期或不一致时按零可用
容量 fail-closed，不能把未知计数解释成空闲容量。
## 3. Snapshot 与 Delta
完整 Snapshot 包含：

38
docs/configuration/reference.md
View File

@ -16,10 +16,13 @@ Controller 入口已实现，源码运行方式为：
go run ./cmd/proxy-controller -config CONFIG_FILE
```
配置路径优先使用 `-config`，未提供时读取 `PROXY_POOL_CONFIG`。该入口已装配
配置路径优先使用 `-config`，未提供时读取 `PROXY_POOL_CONFIG`。启用 Admin 时还
必须设置 `PROXY_POOL_CONFIG_FINGERPRINT_KEY`，值为至少 32 字节的独立高熵密钥；
所有 Controller 副本必须一致，且该密钥不得放入 YAML 或 PostgreSQL。该入口已装配
PostgreSQL 管理面迁移、Redis 活动池、Distribution/Admin 独立监听与优雅停机；
Controller Metrics 独立监听、`/livez`、`/readyz` 和基础 Prometheus 运行时指标；
Provider 自动补池、业务指标和完整部署拓扑仍在后续实施范围。
Provider 自动补池、分布式配额、动态重载和 Admin 低基数统计已装配。完整 Gateway、
Checker、Worker 控制面与代表性负载验证仍在后续实施范围。
所有时间值使用 Go duration，例如 `500ms`、`30s`、`5m`。示例中的
`${TOKEN}`、`${PASSWORD}`、`${POSTGRES_URL}` 等由加载器从同名环境变量
@ -312,11 +315,30 @@ proxyAuth:
- `pool.maxSize`：当前系统维护且尚未 EXTRACTED 的 Proxy 硬上限，包括
FETCHED、CHECKING、AVAILABLE、SUSPECT、DRAINING 和 pending expected。
- `fetch.maxTotal`：当前运行或计费周期内，从 Provider 成功获取的累计上限；
`0` 表示不设置累计上限。
- `fetch.maxTotal`：当前 Redis generation 内的累计获取停止阈值；`0` 表示不设置。
Redis 在调用前原子校验 `fetched total + pending expected + expected`，并在达到
阈值后停止发起新调用。
`fetch.maxTotal` 不得小于 `pool.maxSize`。提取一个 Proxy 会释放当前库存位置，
但不会恢复累计获取额度。
但不会恢复累计获取额度。Provider 调用结果不确定、响应无法解析或 Permit 过期时，
系统按 `estimatedIPsPerCall` 保守记账，避免故障或换主造成额度低估。Redis 全量
状态丢失会创建新 generation，因此需要由外部计费系统提供跨 generation 的长期额度。
`estimatedIPsPerCall` 是预留估值，不是通用的 Provider 响应硬限制。如果某次实际
合法返回量超过估值，系统只保留本地池容量允许的数量，但累计账本按实际合法数量
记账并停止后续调用；该次可能越过停止阈值。需要绝对硬上限时，必须同时在 Provider
请求参数中配置供应商支持的批量上限，并保证其不超过剩余额度。
当前实现限制单个 `pool.maxSize <= 1,000,000`、配置内 Upstream 总数不超过
`4,096`，并要求单代理并发、Refill 双水位、理论总槽位及其他传入 Redis Lua 的
累计/并发计数不超过 `2^53-1`。这些边界在配置加载和 Admin reload 提交前校验，
不会等到 Provider Runtime 启动后才失败。
`proxyAuth.type: response` 的用户名/密码是 Parser 到 Redis Activity Adapter 之间的
临时凭据。每次 Parser handoff 使用独立、幂等释放的 lease，避免并发 Fetch 互相
删除或轮换版本覆盖；成功复制到 TTL 活动记录、候选被截断或解析失败后都会释放。
内存 lease 上限按所有配置 Upstream 的 `pool.maxSize * fetch.maxInFlight` 汇总，
配置 reload 只提高上限，不预分配对应内存。
### 7.3 Fetch 限制
@ -376,6 +398,12 @@ PostgreSQL 只保存配置版本、Upstream/Routing 管理状态、Admin 审计
PostgreSQL 故障本身不应使 Redis 中可完成的 Extract 返回 `503`。Metrics 标签
禁止 Proxy IP、Client ID、Session、完整 URL 和 Request ID。
启用 Admin 的多 Controller 部署必须让所有副本读取同一版本化配置源、Secret
版本和 `PROXY_POOL_CONFIG_FINGERPRINT_KEY`。Supervisor 每秒比较本地完整配置的
HMAC-SHA-256 与 PostgreSQL 权威值：管理状态暂时不可读时沿用 last-known
Runtime；指纹已更新但本地源尚未同步时停止旧 Provider，待源匹配并通过预检后
按全局 revision 恢复，避免旧凭据换主或迟到旧配置回写。
Metrics 启用时 `listen` 必须是合法 `host:port`。该入口固定提供 `/livez`、
`/readyz` 和 `/metrics`，不复用 Distribution/Admin 的认证边界；外部访问必须由
网络策略限制。当前 `/metrics` 已包含 Go/进程基础指标，Provider、提取和容量等

42
docs/design/architecture.md
View File

@ -143,6 +143,15 @@ type Proxy struct {
运行态 `active` 与 `reserved` 存在 Worker 本地、按 Proxy ID 分片，不写入
不可变 Snapshot。
Worker 以有界周期批量上报运行态，而不是在每个 Gateway 请求上写 Redis。
报告通过公用 `workerruntime` seam 表达为完整稀疏替换：Gateway 从打包原子计数
读取同一时刻的 Active/Reserved；已从当前 Snapshot 移除但仍有连接的 Proxy
继续以 draining 状态上报。Controller 使用 session、单调 report sequence、
ownership epoch、Controller 已 ACK 的 snapshot/epoch 上界和 Redis 服务端 TTL
共同校验；缺失、过期或超前报告按零可用容量 fail-closed。运行态报告扫描有界
当前 Snapshot，并以分片索引补充已移除但仍非零的 runtime；历史 Capacity
注册表有硬上限，避免短 TTL Proxy 持续轮换导致心跳扫描与内存无界增长。
## 6. Proxy 状态机
```mermaid
@ -244,6 +253,31 @@ flowchart TD
fail-closed，不回退为本地 Leader。补池使用 minimum/target 双水位迟滞，库存
复核期间若仍有 pending Fetch，则等待下一轮再同步 Managed，避免重复计数。
Controller Bootstrap 通过 `Provider Supervisor` 按权威配置和 PostgreSQL 管理状态
维护每个 Upstream 的独立 `UpstreamRuntime`，再与 HTTP Runtime 通过公用 lifecycle
Group 联动启动、取消和等待。Admin disable 会通知 Supervisor 取消对应 Runtime；
reload 在提交前预检新 Runtime，并在发布后逐个取消、替换或新增。低频对账用于修复
丢失通知。每个 Leader 任期重新创建本地补池预算与合并信号；Redis 是
requestInterval、maxInFlight 和 maxTotal 的唯一分布式裁决者。本地预算只负责
pool.maxSize。
maxTotal 使用 Redis generation 内的累计值与 pending expected 原子预留。调用结果
不确定、解析失败或结算丢失时按 expected 保守记账，换主后旧 Permit 仍可幂等结算。
如果 Provider 单次实际返回量超过 `estimatedIPsPerCall`，账本按实际合法数量记账并
停止后续调用，但该次可能越过停止阈值；严格硬封顶需要 Provider API 支持可控批量。
响应中携带的代理凭据先进入有界内存凭据表，Redis Activity Adapter 复制凭据材料
进入 TTL 活动记录后立即按版本 fence 释放临时引用；解析中途失败也会回收已写引用。
因此短 TTL、持续轮换的新代理地址不会耗尽启动时的凭据容量。Provider 结果只记录
每个 Upstream 的低基数 Empty/Error 计数，不保留响应体、代理地址或错误对象。
补池读取 `pool.InventoryReader` 返回的权威 `Managed` 与 `AvailableSlots`，不使用
AVAILABLE Proxy 数量乘固定并发的近似值。Available Slots 同时计入 Proxy 状态、
TTL safety margin、MaxConcurrency、Worker Active/Reserved 和当前 ownership；
未知 Worker 运行态贡献零槽位。Redis 只扫描目标 Upstream 的未分配/已分配
AVAILABLE 索引，Managed 读取已有权威计数；该读取属于 Controller 冷路径，
Gateway 热路径仍只访问本地 Snapshot 和本地原子计数。
### 8.1 Empty、Duplicate 与 Error
- **Empty**：HTTP/认证成功、模板执行成功，解析后合法 Proxy 数为 0。
@ -403,6 +437,12 @@ flowchart LR
- Routing 立即对新请求生效；旧请求持有旧 Snapshot 完成。
- 修改 API 地址或凭据版本会重建 Provider Adapter，但不会把错误计成 Empty。
- 新配置任何校验失败时，保留旧版本并报告完整错误。
- 多 Controller 以 PostgreSQL 中的 HMAC-SHA-256 完整配置指纹检测修订；HMAC
密钥由外部 Secret 注入且所有副本一致。副本从共享配置源读取相同 revision，
预检后通过单调 revision 栅栏原子发布。权威指纹已变化但本地源仍旧时，旧
Provider fail-closed 停止，避免旧 URL/Secret 在 Leader 换主后继续使用。
- PostgreSQL 瞬时读取失败时沿用 last-known Provider Runtime 并重试，不连带终止
Distribution、Admin 或 Metrics。
## 15. 安全
@ -463,7 +503,7 @@ CPU、内存、网络、Go 版本、配置和上游响应模型下测得。
| Provider 超时/500 | 计 Error、退避；不计 Empty，不影响已有 Proxy |
| Provider 合法空响应 | Empty++；达到阈值触发相关 Routing 原子切换 |
| Redis 不可用 | Gateway 暂用未过期快照；停止 Fetch 入池、Extract 和所有权变更 |
| PostgreSQL 不可用 | Gateway 与 Redis Extract 不受影响；停止管理状态变更和 Admin 审计 |
| PostgreSQL 不可用 | Gateway 与 Redis Extract 不受影响；停止管理写入，Provider 沿用 last-known 状态并重试 |
| Controller 断线 | Worker 在 maxStaleAge 内继续；超限拒绝新流量并排空 |
| Worker 崩溃 | 所有权租约过期后重新分配；期间不双重所有 |
| Checker 积压 | 降低普通复检频率，优先新 Proxy 与 SUSPECT，不无限排队 |

30
docs/development/implementation-plan.md
View File

@ -195,11 +195,14 @@ Admin/Distribution 必需依赖。共享 `platform/httpserver` 与
已完成。pgx Adapter 已在真实 PostgreSQL 18 上运行同一公用契约，并验证
Repeatable Read 快照、`SKIP LOCKED`、原子 ACK、审计/Outbox 故障回滚和数据边界。
Admin `ApplicationService` 已将 mutation、权威管理快照、低基数运行态
聚合与配置重载接到同一公用 seam；严格文件加载、脱敏管理摘要及原子配置发布
已通过失败路径和并发测试。`cmd/proxy-controller` 与公用 `controller/bootstrap`
聚合与配置重载接到同一公用 seam；严格文件加载、外部密钥 HMAC 管理指纹及
revision 单调配置发布已通过失败路径和确定性并发测试。`cmd/proxy-controller` 与公用 `controller/bootstrap`
已完成配置单次加载、PostgreSQL 连接/迁移、Redis 活动池、状态聚合、
Distribution/Admin 服务构造、错误合并和资源关闭；Provider 调度及完整 HTTP
进程端到端测试仍待实现。Controller Metrics 独立入口现已提供 `/livez`、
Distribution/Admin 服务构造、错误合并和资源关闭。生产 Provider Supervisor 已按
权威管理状态动态装配 Upstream，并与 HTTP Runtime 通过公用 lifecycle Group 联动
停机；Admin disable 会取消 Runtime，reload 在提交前预检并在发布后替换运行实例。
组合 fixture 已验证隔离 Redis namespace 下的选主、Provider HTTP 调用、模板解析
和活动池写入。Controller Metrics 独立入口现已提供 `/livez`、
`/readyz` 与基础 Prometheus 运行时指标，三监听器隔离已通过测试；业务指标仍待
实现。双存储 bootstrap 已通过 PostgreSQL 18 + Redis 8.2 组合 fixture，覆盖
迁移、启动配置提交、Readiness、Admin Status 和 Metrics 探针。
@ -219,8 +222,23 @@ generation + epoch fence、全局 requestInterval、全局 maxInFlight Permit、
回收及 Redis 状态丢失后的新 generation 自动重建；Redis 异常期间不发放请求。
补池配置新增必填 `refill` 双水位和 `fetch.estimatedIPsPerCall`，Pool Reconciler
已实现迟滞与 pending 槽位折算，FetchBudget 仅在无 pending 时同步 Redis 权威
Managed。Provider Fleet、Worker Active/Reserved 汇总和 bootstrap 接线仍待完成，
因此本轮不勾选 Task 10 的组合验收项。
Managed。Gateway 已增加打包原子 Active/Reserved 读取与完整稀疏运行态快照；
公用 `workerruntime` session/report/read seam 同时提供并发安全内存参考实现和
生产 Redis Adapter。Redis 以服务端时间、Worker session、已 ACK snapshot/epoch、
单调 report sequence 和报告 TTL 原子隔离旧实例，并由 `pool.InventoryReader` 汇总
权威 Managed/Available Slots；真实 Redis 8.2 已覆盖空报告、幂等重放、倒序、
冲突、超前 epoch、过期、单 Upstream 扫描隔离和预算耗尽的 fail-closed 行为。
Redis Provider Permit 现已把 requestInterval、maxInFlight 与 maxTotal 放在同一
原子边界，按 expected 预留、实际合法数量结算，并支持失败保守计费、换主后结算和
过期回收。响应型代理凭据使用独立、有界 lease，在 Redis Upsert、候选截断或解析
失败后按版本释放；Provider Empty/Error 低基数计数已接入 Admin Status，配置删除
时回收历史统计容量。Redis inventory 扫描上限固定覆盖配置允许的最大池，支持小池
启动后动态扩容。Supervisor 以 PostgreSQL 权威 HMAC 指纹和 revision 栅栏协调
多副本 reload：管理库瞬断沿用 last-known 状态，本地共享源落后时停止旧 Provider，
源匹配并预检后自动替换，迟到旧 revision 不覆盖新配置。
WorkerControlPlane gRPC 接收端、session 签发/心跳、Snapshot ACK
账本、Client 分布式限流和健康执行链仍待完成，因此本轮不勾选 Task 10 的组合
验收项。
## Task 11: Checker and Health Reducer

3
docs/operations/production-readiness.md
View File

@ -6,7 +6,7 @@
- [ ] 每个 Proxy 同一时刻最多归属一个 Worker，ownership epoch 单调。
- [ ] Reserved -> Active 使用单个原子转换，无超卖与负计数。
- [ ] Sequential 并发 Empty 只切换一次，旧 Upstream Proxy 自然耗尽。
- [ ] `pool.maxSize` 与 `fetch.maxTotal` 分别按当前库存和累计获取计数。
- [x] `pool.maxSize` 与 `fetch.maxTotal` 分别按当前库存和累计获取计数。
- [ ] Extract 只有 `AVAILABLE -> EXTRACTED`，OpenAPI 不存在 release/renew。
- [ ] Extract 状态更新和短期幂等结果位于同一个 Redis 原子操作。
- [ ] PostgreSQL 中不存在 Proxy 明细或逐次提取记录。
@ -20,6 +20,7 @@
- [ ] trusted proxy 只包含受控 LoadBalancer/Ingress 网段。
- [ ] 解析前后均拦截私网、回环、链路本地、元数据地址与 DNS Rebinding。
- [ ] Secret 由外部密钥系统注入，镜像、ConfigMap、日志没有明文。
- [ ] 所有 Controller 使用同一枚至少 32 字节的独立配置 HMAC 密钥，并完成轮换演练。
- [ ] Pod 以非 root、只读根文件系统、无 Linux capabilities 运行。
- [ ] NetworkPolicy 默认拒绝，外部数据库/Redis/Provider 网段已收紧。
- [ ] Provider 模板有响应大小、执行时间、函数与外部访问限制。

15
docs/operations/runbook.md
View File

@ -19,9 +19,9 @@
## 2. 本地拓扑模板
`cmd/proxy-controller` 已完成配置单次加载、PostgreSQL 迁移、Redis 活动池、
Distribution/Admin/Metrics 独立监听和有界停机装配。Provider 自动补池、业务
指标以及 Gateway/Checker/Loadgen 三个进程仍属于 `implementation-plan.md`
后续任务。
Distribution/Admin/Metrics 独立监听和有界停机装配。Provider 自动补池、分布式
配额、动态重载和 Admin 低基数统计已装配；完整 Gateway/Checker/Loadgen 与 Worker
控制面仍属于 `implementation-plan.md` 后续任务。
因此 Compose/Kubernetes 资产当前仍用于评审网络、资源、探针和依赖关系，不能
视为完整可运行拓扑。
@ -40,6 +40,7 @@ Distribution/Admin/Metrics 独立监听和有界停机装配。Provider 自动
$env:PROXY_POOL_GATEWAY_PASSWORD = "LOCAL_GATEWAY_PASSWORD"
$env:PROXY_POOL_EXTRACT_TOKEN = "LOCAL_EXTRACT_TOKEN"
$env:PROXY_POOL_ADMIN_TOKEN = "LOCAL_ADMIN_TOKEN"
$env:PROXY_POOL_CONFIG_FINGERPRINT_KEY = "LOCAL_HIGH_ENTROPY_KEY_AT_LEAST_32_BYTES"
$env:PROVIDER_A_TOKEN = "PROVIDER_A_TOKEN"
$env:PROVIDER_B_TOKEN = "PROVIDER_B_TOKEN"
```
@ -90,6 +91,8 @@ Fetch 应表现为 Error 与退避，不应增加 Empty 计数，也不影响已
1. 使用托管 PostgreSQL 和 Redis，分别配置 TLS、备份、监控和多可用区。
2. 复制 `secret.example.yaml` 到环境私密配置系统，由 External Secrets、SOPS
或密钥管理平台生成 `proxy-pool-secrets`，不要提交真实 Secret。
`PROXY_POOL_CONFIG_FINGERPRINT_KEY` 必须使用至少 32 字节的高熵随机值，所有
Controller 副本保持一致，且与配置中的业务 Secret 分离管理。
3. 在环境 Overlay 替换镜像、Provider 地址、允许网段、外部存储地址、资源量
和 LoadBalancer 注解。
4. 根据集群 CNI 能力收紧 NetworkPolicy 的外部网段。
@ -221,11 +224,15 @@ Outbox 发布器必须以稳定 consumer ID 有界领取；发布成功后原子
1. Gateway 继续使用最后有效 Snapshot。
2. Redis 健康且运行配置有效时，Distribution 继续执行原子 Extract，Provider
继续刷新 TTL 活动池。
按 last-known 管理状态继续刷新 TTL 活动池；状态读取失败不得触发全进程退出。
3. 拒绝配置版本、Upstream/Routing 管理状态和其他需要 Admin 审计/outbox 的写入；
不得把 Proxy 明细临时落入 PostgreSQL。
4. 恢复后核对迁移、管理事务回滚、Admin 审计与 outbox backlog；不存在 Proxy
明细或逐次提取记录恢复步骤。
5. 若权威配置指纹已变化，确认每个 Controller 的共享配置源和 Secret 版本已同步；
同时确认 `PROXY_POOL_CONFIG_FINGERPRINT_KEY` 一致。指纹不匹配的副本会停止旧
Provider，匹配并预检成功后按 PostgreSQL revision 自动恢复；迟到旧 revision
不会覆盖较新本地配置。
### 7.3 Redis 不可用

8
docs/requirements/traceability.md
View File

@ -10,7 +10,7 @@
| ARCH-001 | 数据面 Worker 与控制面 Controller 分离 | 1-70 | 包、协议和部署拓扑已分离；Controller 命令已实现，Gateway/Checker/Loadgen 构建产物待实现 |
| ARCH-002 | 热路径只做认证、本地路由和网络转发 | 1-70, 380-430 | Snapshot/Dispatch 及依赖边界已验证；完整 Gateway 进程与代表性性能剖析待完成 |
| ARCH-003 | Gateway、Distribution、Admin、Metrics 独立入口 | 8904-8958 | Controller 命令已装配 Distribution/Admin/Metrics 三个独立监听及联动停机；Gateway 生产入口待装配 |
| ARCH-004 | Controller 集中 Provider 获取与切换 | 1403-1580 | 单进程 Reconciler、合并通知和切换领域契约已完成；分布式 Leader 与运行装配待完成 |
| ARCH-004 | Controller 集中 Provider 获取与切换 | 1403-1580 | Redis Leader、动态 Provider Supervisor 与 Bootstrap 生产装配已完成；Admin disable/reload 驱动取消替换，多副本按权威 HMAC 指纹和 revision 栅栏收敛并拒绝旧配置换主；Routing 切换到 Drain 的编排待完成 |
| ARCH-005 | 100k QPS 峰值使用多 Worker 集群 | 当前会话 | 未验证设计目标；待代表性集群负载报告 |
## Routing 与 Upstream
@ -32,11 +32,11 @@
| FETCH-001 | 每个 Provider 有独立 requestInterval、maxInFlight、timeout 和 retry | 968-2394 | `provider/reconciler_test.go` |
| FETCH-002 | 大量缺池信号合并为 singleflight/容量 1 通知 | 2067-2136, 8808-8849 | `coalesce.Signal` 与 100 并发通知测试 |
| FETCH-003 | 错误使用指数退避和抖动，429 尊重 Retry-After | 1601-1831, 8808-8856 | `provider/reconciler_test.go` 与 `providerapi/http_adapter_test.go` |
| FETCH-004 | Provider 获取由单逻辑 Leader 执行 | 1403-1580 | Redis Coordinator 已通过真实 Redis 双实例互斥、epoch 接管、全局间隔/在途 Permit 与 generation 重建测试；Provider Fleet/bootstrap 接线待完成 |
| FETCH-004 | Provider 获取由单逻辑 Leader 执行 | 1403-1580 | Redis Coordinator 已通过真实 Redis 双实例互斥、epoch 接管、全局间隔/在途 Permit 与 generation 重建测试；生产 Supervisor/bootstrap 已通过 Admin disable 和 Provider HTTP 到隔离 Redis 库存的组合 fixture |
| FETCH-005 | Empty 与 Error 分开；只有合法候选为零时 Empty++ | 8442-8529 | `fetch_result_test.go` 分类矩阵 |
| FETCH-006 | 重复候选不当作 Empty，记录独立指标 | 8442-8480 | DuplicateOnly 分类与 Provider 测试 |
| FETCH-007 | 模板限制响应大小、执行时间、函数集和外部访问 | 8808-8856 | `providerapi/template_parser_test.go` 输入、输出、候选、超时、递归与函数白名单测试 |
| FETCH-008 | pool.maxSize 与 fetch.maxTotal 语义分离 | 9190-9280 | `FetchBudget` 并发预占/释放测试 |
| FETCH-008 | pool.maxSize 与 fetch.maxTotal 语义分离 | 9190-9280 | 本地 `FetchBudget` 仅约束当前库存；Redis Permit 原子维护累计与 pending 额度，并通过换主、取消、幂等和过期保守结算测试 |
## Proxy 生命周期与容量
@ -46,7 +46,7 @@
| PROXY-002 | 唯一键包含 scheme、host、port、username、credentialVersion | 6655-6727, 8605-8678 | 去重单测 |
| PROXY-003 | TTL 来源优先级明确并统一 UTC | 681-747, 8655-8678 | TTL 表驱动测试 |
| CAP-001 | Gateway 分配使用 Reserved -> Active 原子转换 | 1203-1467, 8530-8597 | 固定 Max 下打包 CAS 与 1,000 并发不超卖已完成；动态降容和完整生命周期证据待完成 |
| CAP-002 | 补池依据 Available Slots，不只看 Proxy 数量 | 1203-1402, 8530-8597 | `AvailableSlots`、显式 minimum/target 水位、pending 槽位和迟滞 Reconciler 已测试；Worker Active/Reserved、ownership、目标健康及 Gateway reserve 运行时聚合待完成 |
| CAP-002 | 补池依据 Available Slots，不只看 Proxy 数量 | 1203-1402, 8530-8597 | `AvailableSlots`、显式 minimum/target 水位、pending 槽位和迟滞 Reconciler 已测试；Gateway 打包 Active/Reserved 报告、Worker session/ACK/sequence/TTL/ownership fence、单 Upstream 索引及 Redis 权威 Managed/Slots 汇总已通过内存与真实 Redis 测试；WorkerControlPlane 接线、目标健康和 Gateway reserve 策略仍待完成 |
| CAP-003 | pool.maxSize 包括 FETCHED/CHECKING/AVAILABLE/SUSPECT/DRAINING 与 pending expected | 3001-3533, 6642-6680 | `FetchBudget` 100 并发额度预占测试 |
| CAP-004 | TTL safety margin 内禁止新分配 | 173-220, 6728-6741 | 时钟测试 |
| CAP-005 | 多 Worker 不在热路径访问 Redis 计数 | 1403-1467 | Gateway 包依赖审计、Snapshot/Dispatch 测试 |

22
docs/testing/strategy.md
View File

@ -22,11 +22,20 @@
4. 连续 4 次 Empty 后 Valid 不切换；连续 5 次只从 A 切到 B。
5. Error 和 DuplicateOnly 不累计 Empty。
6. 100 个缺池信号只形成一个合并 Provider reconcile。
7. 并发 Fetch 不突破 `pool.maxSize` 与 `fetch.maxTotal`。
7. 并发 Fetch 不突破 `pool.maxSize`，且不会在 Redis `fetch.maxTotal` 停止阈值耗尽后发起新调用。
8. TTL safety margin 内不再分配。
9. Snapshot 版本断档、目标错误或校验和错误不替换当前视图。
10. 非幂等 HTTP 和已建立 CONNECT 不自动重放。
11. 所有 Upstream 不可用时严格执行显式策略。
11. Admin disable 取消目标 Provider Runtime；reload 构造失败不提交，成功时替换运行实例。
12. 短 TTL 响应凭据使用独立 lease；并发 Fetch、截断、解析失败和 Redis Upsert
后均准确释放且不互相撤销。
13. PostgreSQL 状态瞬断保留 last-known Provider；多副本本地指纹落后时停止旧
Runtime，共享源同步后自动预检并恢复；不同 HMAC 密钥不能误判为相同配置。
14. 并发配置提交与 Supervisor 同步按 revision 单调发布，迟到旧 revision 不覆盖
已发布新配置。
15. 配置删除的 Provider 统计项被回收，禁用但仍配置的统计项保留，容量可复用。
16. 小池启动后重载到大池时，Redis inventory 扫描上限仍覆盖配置允许的最大池。
17. 所有 Upstream 不可用时严格执行显式策略。
## 3. 基础质量门禁
@ -74,16 +83,17 @@ soak 测试单独标记，不混入快速单测。
## 5. 当前本地微基准
2026-07-28，Windows/amd64、Intel Core Ultra 7 155H：
2026-07-30，Windows/amd64、Intel Core Ultra 7 155H：
```text
BenchmarkAcquire100kIndexed-22 3553592 640.0 ns/op 256 B/op 2 allocs/op
BenchmarkStoreApply100k-22 1 472.7 ms/op 654 MB/op 2700642 allocs/op
BenchmarkAcquire100kIndexed-22 1000000-1867125 893.9-1047 ns/op 256 B/op 2 allocs/op
BenchmarkStoreApply100k-22 1 518.7 ms/op 540 MB/op 3000887 allocs/op
```
`Acquire` 已使用 scheme/upstream/tag 索引，结果只代表本地选择和容量预留。
`Store.Apply` 属于冷路径且当前内存开销较高；运行态为防止旧快照在途连接超配，
暂不自动回收曾出现过的 Proxy ID。后续需要基于 RCU/引用计数定义安全回收点。
暂不自动回收曾出现过的 Proxy ID，但注册表有 1,000,000 项硬上限，达到上限时
拒绝新 Snapshot 并保留旧视图。后续需要基于 RCU/引用计数定义安全回收点。
这些数据不包含网络、认证、Provider、存储或多 Worker 协调，不能作为
100k QPS 端到端验收结论。

2
docs/testing/test-strategy.md
View File

@ -120,7 +120,7 @@ Redis Readiness 和 Admin Status；HTTP Runner 使用测试 Adapter，避免占
Admin 应用层测试覆盖 typed-nil 依赖、Actor/SourceIP 映射、Routing CAS 错误、
权威管理快照与低基数运行态聚合、未知字段拒绝、主配置/Secret 文件 I/O 分类、
持久化失败不发布、幂等重放发布、脱敏管理摘要和原子配置 Store 并发读写。静态
持久化失败不发布、HMAC 管理指纹、revision 单调发布和原子配置 Store 并发读写。静态
导入边界测试禁止 Admin 引用 Redis Activity/Extract 与 Proxy 明细包。
需要 PostgreSQL/Redis 的测试使用独立实例和短生命周期容器，不复用开发数据。

69
internal/adapters/providerapi/template_parser.go
View File

@ -20,6 +20,7 @@ const (
defaultTemplateTimeout = 100 * time.Millisecond
defaultTemplateMaxBytes = int64(1 << 20)
defaultMaxCandidates = 10_000
credentialReleaseTimeout = time.Second
maxRegexPatterns = 64
maxRegexPatternBytes = 1024
)
@ -137,7 +138,14 @@ func NewTemplateParser(
return parser, nil
}
func (p *TemplateParser) Parse(ctx context.Context, body []byte) ([]proxyDomain.Proxy, error) {
func (p *TemplateParser) Parse(ctx context.Context, body []byte) (proxies []proxyDomain.Proxy, resultErr error) {
storedCredentials := make([]credentials.Reference, 0)
defer func() {
if resultErr == nil {
return
}
p.releaseCredentials(storedCredentials)
}()
if err := ctx.Err(); err != nil {
return nil, err
}
@ -161,7 +169,7 @@ func (p *TemplateParser) Parse(ctx context.Context, body []byte) ([]proxyDomain.
if len(tokens) > p.maxCandidates {
return nil, &limitError{kind: ErrTooManyCandidates, size: int64(len(tokens)), limit: int64(p.maxCandidates)}
}
proxies := make([]proxyDomain.Proxy, 0, len(tokens))
proxies = make([]proxyDomain.Proxy, 0, len(tokens))
credentialIndexes := make(map[string]int)
for _, token := range tokens {
candidate, credential, ok := p.parseCandidate(token)
@ -183,6 +191,7 @@ func (p *TemplateParser) Parse(ctx context.Context, body []byte) ([]proxyDomain.
}
candidate.SecretRef = reference.SecretRef
candidate.CredentialVersion = reference.CredentialVersion
storedCredentials = append(storedCredentials, reference)
credentialKey = candidateCredentialKey(candidate)
if index, exists := credentialIndexes[credentialKey]; exists {
proxies[index] = candidate
@ -201,9 +210,65 @@ func (p *TemplateParser) Parse(ctx context.Context, body []byte) ([]proxyDomain.
if len(tokens) > 0 && len(proxies) == 0 {
return nil, ErrInvalidProxyOutput
}
p.releaseUnusedCredentials(storedCredentials, proxies)
return proxies, nil
}
func (p *TemplateParser) ReleaseCandidates(candidates []proxyDomain.Proxy) {
references := make([]credentials.Reference, 0, len(candidates))
for _, candidate := range candidates {
if candidate.SecretRef == "" || candidate.CredentialVersion == "" {
continue
}
references = append(references, credentials.Reference{
SecretRef: candidate.SecretRef, CredentialVersion: candidate.CredentialVersion,
})
}
p.releaseCredentials(references)
}
func (p *TemplateParser) releaseUnusedCredentials(
stored []credentials.Reference,
candidates []proxyDomain.Proxy,
) {
retained := make(map[credentials.Reference]int, len(candidates))
for _, candidate := range candidates {
if candidate.SecretRef == "" || candidate.CredentialVersion == "" {
continue
}
retained[credentials.Reference{
SecretRef: candidate.SecretRef, CredentialVersion: candidate.CredentialVersion,
}]++
}
unused := make([]credentials.Reference, 0, len(stored))
for _, reference := range stored {
if retained[reference] > 0 {
retained[reference]--
continue
}
unused = append(unused, reference)
}
p.releaseCredentials(unused)
}
func (p *TemplateParser) releaseCredentials(references []credentials.Reference) {
if p == nil {
return
}
releaser, ok := p.credentialStore.(credentials.Releaser)
if !ok {
return
}
releaseCtx, cancel := context.WithTimeout(context.Background(), credentialReleaseTimeout)
defer cancel()
for _, reference := range references {
if releaseCtx.Err() != nil {
return
}
_ = releaser.Release(releaseCtx, reference)
}
}
func (p *TemplateParser) regexFind(pattern, value string) (string, error) {
compiled, err := p.compileRegex(pattern)
if err != nil {

68
internal/adapters/providerapi/template_parser_test.go
View File

@ -318,7 +318,7 @@ func TestTemplateParserDoesNotOverrideStaticProxyAuthFromResponse(t *testing.T)
}
func TestTemplateParserRetainsDistinctEndpointsSharingStaticCredentials(t *testing.T) {
store, err := credentials.NewMemoryStore(1)
store, err := credentials.NewMemoryStore(2)
if err != nil {
t.Fatalf("NewMemoryStore(): %v", err)
}
@ -343,8 +343,11 @@ func TestTemplateParserRetainsDistinctEndpointsSharingStaticCredentials(t *testi
if len(proxies) != 2 {
t.Fatalf("proxy count = %d, want both static-auth endpoints", len(proxies))
}
if proxies[0].SecretRef == "" || proxies[0].SecretRef != proxies[1].SecretRef {
t.Fatalf("static credential references = %q and %q, want same opaque reference", proxies[0].SecretRef, proxies[1].SecretRef)
if proxies[0].SecretRef == "" || proxies[1].SecretRef == "" || proxies[0].SecretRef == proxies[1].SecretRef {
t.Fatalf("static credential references = %q and %q, want independent leases", proxies[0].SecretRef, proxies[1].SecretRef)
}
if proxies[0].CredentialVersion != proxies[1].CredentialVersion {
t.Fatalf("static credential versions = %q and %q, want same value version", proxies[0].CredentialVersion, proxies[1].CredentialVersion)
}
}
@ -391,6 +394,33 @@ func TestTemplateParserStoresResponseCredentialsByOpaqueReference(t *testing.T)
}
}
func TestTemplateParserReleasesPartialCredentialsWhenParseFails(t *testing.T) {
store, err := credentials.NewMemoryStore(1)
if err != nil {
t.Fatalf("NewMemoryStore(): %v", err)
}
parser, err := newTemplateParser("provider-a", config.Upstream{
Provider: config.Provider{Protocols: []string{"http"}},
API: config.ProviderAPI{Template: strings.Join([]string{
"http://alice:first-password@192.0.2.10:8080",
"http://bob:second-password@192.0.2.11:8080",
}, "\n")},
ProxyAuth: config.ProxyAuth{Type: "response"},
Pool: config.Pool{MaxSize: 2},
Fetch: config.Fetch{MaxResponseBytes: 1024, TemplateTimeout: config.Duration(time.Second)},
}, store)
if err != nil {
t.Fatalf("newTemplateParser(): %v", err)
}
if _, err := parser.Parse(context.Background(), nil); !errors.Is(err, credentials.ErrCapacityExceeded) {
t.Fatalf("Parse() error = %v, want ErrCapacityExceeded", err)
}
if _, err := store.Put(context.Background(), "replacement", credentials.Value{Password: "replacement"}); err != nil {
t.Fatalf("Put(after failed parse): %v", err)
}
}
func TestTemplateParserKeepsDistinctAccountsForSameEndpointResolvable(t *testing.T) {
store, err := credentials.NewMemoryStore(2)
if err != nil {
@ -432,7 +462,7 @@ func TestTemplateParserKeepsDistinctAccountsForSameEndpointResolvable(t *testing
}
func TestTemplateParserKeepsLatestCredentialVersionWithinOneResponse(t *testing.T) {
store, err := credentials.NewMemoryStore(1)
store, err := credentials.NewMemoryStore(2)
if err != nil {
t.Fatalf("NewMemoryStore(): %v", err)
}
@ -467,6 +497,36 @@ func TestTemplateParserKeepsLatestCredentialVersionWithinOneResponse(t *testing.
if value.Password != "new-secret" {
t.Fatalf("resolved latest password mismatch")
}
parser.ReleaseCandidates(proxies)
if _, err := store.Put(context.Background(), "replacement", credentials.Value{Password: "replacement"}); err != nil {
t.Fatalf("Put(after releasing latest candidates): %v", err)
}
}
func TestTemplateParserReleaseCandidatesReturnsCredentialCapacity(t *testing.T) {
store, err := credentials.NewMemoryStore(1)
if err != nil {
t.Fatalf("NewMemoryStore(): %v", err)
}
parser, err := newTemplateParser("provider-a", config.Upstream{
Provider: config.Provider{Protocols: []string{"http"}},
API: config.ProviderAPI{Template: "http://alice:secret@192.0.2.10:8080"},
ProxyAuth: config.ProxyAuth{Type: "response"},
Pool: config.Pool{MaxSize: 1},
Fetch: config.Fetch{MaxResponseBytes: 1024, TemplateTimeout: config.Duration(time.Second)},
}, store)
if err != nil {
t.Fatalf("NewTemplateParser(): %v", err)
}
proxies, err := parser.Parse(context.Background(), nil)
if err != nil {
t.Fatalf("Parse(): %v", err)
}
parser.ReleaseCandidates(proxies)
if _, err := store.Put(context.Background(), "replacement", credentials.Value{Password: "replacement"}); err != nil {
t.Fatalf("Put(after ReleaseCandidates): %v", err)
}
}
func TestTemplateParserRedactsCredentialStoreErrors(t *testing.T) {

18
internal/adapters/redisactivity/adapter.go
View File

@ -23,29 +23,41 @@ type Options struct {
Credentials credentials.Store
OperationTTL time.Duration
MaxCandidateScan int
MaxRuntimeCounters int
MaxInventoryScan int
CleanupLimit int
}
type Adapter struct {
client redis.Scripter
credentials credentials.Store
credentialReleaser credentials.Releaser
keys keyspace
options Options
}
func New(client redis.Scripter, options Options) (*Adapter, error) {
options.Namespace = strings.TrimSpace(options.Namespace)
if options.MaxRuntimeCounters == 0 {
options.MaxRuntimeCounters = options.MaxCandidateScan
}
if options.MaxInventoryScan == 0 {
options.MaxInventoryScan = options.MaxCandidateScan
}
if nilInterface(client) || nilInterface(options.Credentials) ||
!namespacePattern.MatchString(options.Namespace) || options.OperationTTL <= 0 ||
options.MaxCandidateScan <= 0 || options.CleanupLimit <= 0 {
options.MaxCandidateScan <= 0 || options.MaxRuntimeCounters <= 0 ||
options.MaxInventoryScan <= 0 || options.CleanupLimit <= 0 {
return nil, ErrInvalidOptions
}
return &Adapter{
adapter := &Adapter{
client: client,
credentials: options.Credentials,
keys: newKeyspace(options.Namespace),
options: options,
}, nil
}
adapter.credentialReleaser, _ = options.Credentials.(credentials.Releaser)
return adapter, nil
}
func (a *Adapter) Format(state fmt.State, _ rune) {

18
internal/adapters/redisactivity/adapter_test.go
View File

@ -58,6 +58,8 @@ func TestNewRejectsInvalidDependenciesAndOptions(t *testing.T) {
{name: "colon in namespace", client: client, options: withNamespace(valid, "tenant:other")},
{name: "zero operation ttl", client: client, options: withOperationTTL(valid, 0)},
{name: "zero candidate scan", client: client, options: withMaxCandidateScan(valid, 0)},
{name: "negative runtime counters", client: client, options: withMaxRuntimeCounters(valid, -1)},
{name: "negative inventory scan", client: client, options: withMaxInventoryScan(valid, -1)},
{name: "negative cleanup limit", client: client, options: withCleanupLimit(valid, -1)},
}
for _, tt := range tests {
@ -93,7 +95,9 @@ func TestNewBuildsClusterSafeKeyspaceAndHashesDynamicTokens(t *testing.T) {
adapter.keys.records, adapter.keys.unique, adapter.keys.idkeys,
adapter.keys.expiry, adapter.keys.available, adapter.keys.owners,
adapter.keys.ownerExpiry, adapter.keys.epoch, adapter.keys.inventory,
adapter.keys.stateInventory,
adapter.keys.stateInventory, adapter.keys.workerSessions,
adapter.keys.workerSessionExpiry, adapter.keys.workerRuntime,
adapter.keys.workerRuntimeExpiry,
}
for _, key := range staticKeys {
if strings.Count(key, "{activity}") != 1 || strings.Count(key, "{") != 1 || strings.Count(key, "}") != 1 {
@ -109,6 +113,7 @@ func TestNewBuildsClusterSafeKeyspaceAndHashesDynamicTokens(t *testing.T) {
adapter.keys.region(raw),
adapter.keys.carrier(raw),
adapter.keys.upstream(raw),
adapter.keys.owned(raw),
}
for _, key := range dynamicKeys {
if strings.Contains(key, raw) || strings.Count(key, "{activity}") != 1 || strings.Count(key, "{") != 1 {
@ -161,6 +166,7 @@ func TestProxyRecordCodecIsDeterministicStrictAndRedacted(t *testing.T) {
UsableUntilMS: 58_000, LastCheckedAtMS: 2_000, LastSuccessAtMS: 2_000,
LatencyNS: int64(25 * time.Millisecond), MaxConcurrency: 8,
State: string(proxyDomain.StateAvailable), Tags: map[string]string{"region": "cn", "carrier": "ct"},
OwnerIndexKey: "pp:{activity}:test:owned:index",
}
first, err := encodeProxyRecord(record)
if err != nil {
@ -260,6 +266,16 @@ func withMaxCandidateScan(options Options, limit int) Options {
return options
}
func withMaxRuntimeCounters(options Options, limit int) Options {
options.MaxRuntimeCounters = limit
return options
}
func withMaxInventoryScan(options Options, limit int) Options {
options.MaxInventoryScan = limit
return options
}
func withCleanupLimit(options Options, limit int) Options {
options.CleanupLimit = limit
return options

45
internal/adapters/redisactivity/capacity.go Normal file
View File

@ -0,0 +1,45 @@
package redisactivity
import (
"context"
"time"
controllerPool "proxy-pool/internal/controller/pool"
"proxy-pool/internal/domain/activitypool"
)
var _ controllerPool.InventoryReader = (*Adapter)(nil)
func (a *Adapter) ReadInventory(
ctx context.Context,
upstreamID string,
safetyMargin time.Duration,
) (controllerPool.InventorySnapshot, error) {
if ctx == nil || a == nil || !runtimeClean(upstreamID) || safetyMargin < 0 {
return controllerPool.InventorySnapshot{}, activitypool.ErrInvalidInventory
}
if err := ctx.Err(); err != nil {
return controllerPool.InventorySnapshot{}, err
}
result, err := runScript(ctx, a.client, capacityScript, []string{
a.keys.records, a.keys.inventory, a.keys.upstream(upstreamID), a.keys.owned(upstreamID), a.keys.owners,
a.keys.workerSessions, a.keys.workerSessionExpiry,
a.keys.workerRuntime, a.keys.workerRuntimeExpiry,
}, upstreamID, durationMillis(safetyMargin), a.options.MaxInventoryScan, a.options.CleanupLimit)
if err != nil {
return controllerPool.InventorySnapshot{}, err
}
var reply capacityScriptReply
if err := decodeScriptResult(result, &reply); err != nil {
return controllerPool.InventorySnapshot{}, err
}
if reply.Status == scriptInvalid {
return controllerPool.InventorySnapshot{}, activitypool.ErrInvalidInventory
}
if reply.Status != scriptOK || reply.Managed < 0 || reply.AvailableSlots < 0 {
return controllerPool.InventorySnapshot{}, invalidScriptReply("capacity inventory is unavailable")
}
return controllerPool.InventorySnapshot{
Managed: reply.Managed, AvailableSlots: reply.AvailableSlots,
}, nil
}

105
internal/adapters/redisactivity/capacity_integration_test.go Normal file
View File

@ -0,0 +1,105 @@
//go:build integration
package redisactivity
import (
"context"
"testing"
"time"
controllerPool "proxy-pool/internal/controller/pool"
"proxy-pool/internal/domain/workerruntime"
)
func TestRedisCapacityInventoryCombinesProxyAndWorkerRuntime(t *testing.T) {
fixture := newRedisTestFixture(t)
now := redisTestNow()
unowned := testProxy("proxy-unowned", "192.0.2.10")
unowned.MaxConcurrency = 10
owned := testProxy("proxy-owned", "192.0.2.11")
owned.MaxConcurrency = 10
seedRedisAvailable(t, fixture.Adapter, "provider-a", now, now.Add(time.Second), 2*time.Minute, unowned)
seedRedisAvailable(t, fixture.Adapter, "provider-a", now, now.Add(time.Second), 2*time.Minute, owned)
assignment, err := fixture.Adapter.Assign(context.Background(), now.Add(2*time.Second),
"proxy-owned", "worker-a", time.Minute)
if err != nil {
t.Fatalf("Assign(): %v", err)
}
if err := fixture.Adapter.ReplaceSession(context.Background(), workerruntime.Session{
WorkerID: "worker-a", InstanceID: "instance-a", SessionID: "session-a",
AckedSnapshotVersion: 1, AckedOwnershipEpoch: assignment.Epoch,
}, time.Minute); err != nil {
t.Fatalf("ReplaceSession(): %v", err)
}
if err := fixture.Adapter.ReplaceRuntime(context.Background(), workerruntime.Report{
WorkerID: "worker-a", SessionID: "session-a", Sequence: 1,
SnapshotVersion: 1, OwnershipEpoch: assignment.Epoch, ObservedAt: now,
Counters: []workerruntime.Counter{{ProxyID: "proxy-owned", Active: 3, Reserved: 2}},
}, time.Minute); err != nil {
t.Fatalf("ReplaceRuntime(): %v", err)
}
inventory, err := fixture.Adapter.ReadInventory(context.Background(), "provider-a", 0)
if err != nil || inventory.Managed != 2 || inventory.AvailableSlots != 15 {
t.Fatalf("ReadInventory() = %+v, %v; want managed=2 slots=15", inventory, err)
}
if _, ok := any(fixture.Adapter).(controllerPool.InventoryReader); !ok {
t.Fatal("Adapter does not implement pool.InventoryReader")
}
inventory, err = fixture.Adapter.ReadInventory(context.Background(), "provider-a", 2*time.Hour)
if err != nil || inventory.Managed != 2 || inventory.AvailableSlots != 0 {
t.Fatalf("ReadInventory(safety margin) = %+v, %v", inventory, err)
}
fixture.Adapter.options.MaxInventoryScan = 1
if _, err := fixture.Adapter.ReadInventory(context.Background(), "provider-a", 0); err == nil {
t.Fatal("ReadInventory(over scan limit) error = nil")
}
}
func TestRedisCapacityInventoryFailsClosedForExpiredRuntime(t *testing.T) {
fixture := newRedisTestFixture(t)
now := redisTestNow()
owned := testProxy("proxy-owned", "192.0.2.11")
owned.MaxConcurrency = 10
seedRedisAvailable(t, fixture.Adapter, "provider-a", now, now.Add(time.Second), 2*time.Minute, owned)
assignment, err := fixture.Adapter.Assign(context.Background(), now.Add(2*time.Second),
"proxy-owned", "worker-a", time.Minute)
if err != nil {
t.Fatalf("Assign(): %v", err)
}
if err := fixture.Adapter.ReplaceSession(context.Background(), workerruntime.Session{
WorkerID: "worker-a", InstanceID: "instance-a", SessionID: "session-a",
AckedSnapshotVersion: 1, AckedOwnershipEpoch: assignment.Epoch,
}, 100*time.Millisecond); err != nil {
t.Fatalf("ReplaceSession(): %v", err)
}
if err := fixture.Adapter.ReplaceRuntime(context.Background(), workerruntime.Report{
WorkerID: "worker-a", SessionID: "session-a", Sequence: 1,
SnapshotVersion: 1, OwnershipEpoch: assignment.Epoch, ObservedAt: now,
}, 100*time.Millisecond); err != nil {
t.Fatalf("ReplaceRuntime(): %v", err)
}
time.Sleep(150 * time.Millisecond)
inventory, err := fixture.Adapter.ReadInventory(context.Background(), "provider-a", 0)
if err != nil || inventory.Managed != 1 || inventory.AvailableSlots != 0 {
t.Fatalf("ReadInventory(expired runtime) = %+v, %v", inventory, err)
}
}
func TestRedisCapacityInventoryScanIsIsolatedPerUpstream(t *testing.T) {
fixture := newRedisTestFixture(t)
fixture.Adapter.options.MaxInventoryScan = 1
now := redisTestNow()
target := testProxy("proxy-target", "192.0.2.10")
target.MaxConcurrency = 4
seedRedisAvailable(t, fixture.Adapter, "provider-a", now, now.Add(time.Second), 2*time.Minute, target)
seedRedisAvailable(t, fixture.Adapter, "provider-b", now, now.Add(time.Second), 2*time.Minute,
testProxy("proxy-other-1", "192.0.2.11"))
seedRedisAvailable(t, fixture.Adapter, "provider-b", now, now.Add(2*time.Second), 2*time.Minute,
testProxy("proxy-other-2", "192.0.2.12"))
inventory, err := fixture.Adapter.ReadInventory(context.Background(), "provider-a", 0)
if err != nil || inventory.Managed != 1 || inventory.AvailableSlots != 4 {
t.Fatalf("ReadInventory(provider-a) = %+v, %v; want isolated managed=1 slots=4", inventory, err)
}
}

4
internal/adapters/redisactivity/codec.go
View File

@ -37,6 +37,7 @@ type proxyRecord struct {
State string `json:"state"`
Tags map[string]string `json:"tags,omitempty"`
OwnerWorkerID string `json:"ownerWorkerId,omitempty"`
OwnerIndexKey string `json:"ownerIndexKey"`
IndexKeys []string `json:"indexKeys,omitempty"`
}
@ -203,7 +204,8 @@ func validateProxyRecord(record proxyRecord) error {
record.CreatedAtMS <= 0 || record.ExpiresAtMS <= 0 || record.UsableUntilMS <= 0 ||
record.UsableUntilMS > record.ExpiresAtMS || record.LastCheckedAtMS < 0 ||
record.LastSuccessAtMS < 0 || record.LatencyNS < 0 || record.MaxConcurrency < 0 ||
!validScheme(record.Scheme) || !validProxyState(record.State) {
!validScheme(record.Scheme) || !validProxyState(record.State) ||
record.OwnerIndexKey == "" || !strings.Contains(record.OwnerIndexKey, "{activity}") {
return ErrInvalidRecord
}
for _, key := range record.IndexKeys {

12
internal/adapters/redisactivity/keys.go
View File

@ -20,6 +20,10 @@ type keyspace struct {
epoch string
inventory string
stateInventory string
workerSessions string
workerSessionExpiry string
workerRuntime string
workerRuntimeExpiry string
}
func newKeyspace(namespace string) keyspace {
@ -36,6 +40,10 @@ func newKeyspace(namespace string) keyspace {
epoch: prefix + ":epoch",
inventory: prefix + ":inventory",
stateInventory: prefix + ":state-inventory",
workerSessions: prefix + ":worker-sessions",
workerSessionExpiry: prefix + ":worker-session-expiry",
workerRuntime: prefix + ":worker-runtime",
workerRuntimeExpiry: prefix + ":worker-runtime-expiry",
}
}
@ -67,6 +75,10 @@ func (keys keyspace) upstream(value string) string {
return keys.facet("upstream", value)
}
func (keys keyspace) owned(value string) string {
return keys.facet("owned", value)
}
func (keys keyspace) facet(name, value string) string {
return keys.prefix + ":" + name + ":" + digestToken(value)
}

248
internal/adapters/redisactivity/runtime.go Normal file
View File

@ -0,0 +1,248 @@
package redisactivity
import (
"context"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"sort"
"strconv"
"strings"
"time"
"proxy-pool/internal/domain/workerruntime"
)
const runtimeWireVersion = 1
const (
runtimeReplaceSession = "replace_session"
runtimeReplaceReport = "replace_report"
runtimeRead = "read"
)
type runtimeSessionWire struct {
Version int `json:"version"`
WorkerID string `json:"workerId"`
InstanceID string `json:"instanceId"`
SessionID string `json:"sessionId"`
AckedSnapshotVersion string `json:"ackedSnapshotVersion"`
AckedOwnershipEpoch string `json:"ackedOwnershipEpoch"`
}
type runtimeCounterWire struct {
ProxyID string `json:"proxyId"`
Active int64 `json:"active"`
Reserved int64 `json:"reserved"`
Draining bool `json:"draining"`
}
type runtimeReportWire struct {
Version int `json:"version"`
WorkerID string `json:"workerId"`
SessionID string `json:"sessionId"`
Sequence string `json:"sequence"`
SnapshotVersion string `json:"snapshotVersion"`
OwnershipEpoch string `json:"ownershipEpoch"`
ObservedAtMS int64 `json:"observedAtMs"`
Counters []runtimeCounterWire `json:"counters"`
}
type runtimeOwnedProxyWire struct {
ProxyID string `json:"proxyId"`
WorkerID string `json:"workerId"`
OwnershipEpoch string `json:"ownershipEpoch"`
}
type runtimeSnapshotWire struct {
ProxyID string `json:"proxyId"`
Active int64 `json:"active"`
Reserved int64 `json:"reserved"`
Draining bool `json:"draining"`
Fresh bool `json:"fresh"`
}
var (
_ workerruntime.SessionWriter = (*Adapter)(nil)
_ workerruntime.ReportWriter = (*Adapter)(nil)
_ workerruntime.RuntimeReader = (*Adapter)(nil)
)
func (a *Adapter) ReplaceSession(ctx context.Context, session workerruntime.Session, ttl time.Duration) error {
if err := validateRuntimeCall(ctx, a); err != nil {
return err
}
if !runtimeClean(session.WorkerID) || !runtimeClean(session.InstanceID) || !runtimeClean(session.SessionID) ||
session.AckedSnapshotVersion == 0 || session.AckedOwnershipEpoch == 0 || ttl <= 0 {
return workerruntime.ErrInvalidSession
}
payload, err := json.Marshal(runtimeSessionWire{
Version: runtimeWireVersion, WorkerID: session.WorkerID,
InstanceID: session.InstanceID, SessionID: session.SessionID,
AckedSnapshotVersion: strconv.FormatUint(session.AckedSnapshotVersion, 10),
AckedOwnershipEpoch: strconv.FormatUint(session.AckedOwnershipEpoch, 10),
})
if err != nil {
return workerruntime.ErrInvalidSession
}
reply, err := a.runRuntime(ctx, runtimeReplaceSession, durationMillis(ttl), payload, "")
if err != nil {
return err
}
switch reply.Status {
case scriptOK:
return nil
case scriptInvalid:
return workerruntime.ErrInvalidSession
case scriptStale:
return workerruntime.ErrStaleSession
default:
return invalidScriptReply("unexpected worker session reply")
}
}
func (a *Adapter) ReplaceRuntime(ctx context.Context, report workerruntime.Report, ttl time.Duration) error {
if err := validateRuntimeCall(ctx, a); err != nil {
return err
}
payload, digest, err := a.encodeRuntimeReport(report, ttl)
if err != nil {
return err
}
reply, err := a.runRuntime(ctx, runtimeReplaceReport, durationMillis(ttl), payload, digest)
if err != nil {
return err
}
switch reply.Status {
case scriptOK:
return nil
case scriptInvalid:
return workerruntime.ErrInvalidReport
case scriptStale:
return workerruntime.ErrStaleReport
case scriptConflict:
return workerruntime.ErrConflictingReport
case scriptUnavailable:
return workerruntime.ErrStaleSession
default:
return invalidScriptReply("unexpected worker runtime reply")
}
}
func (a *Adapter) ReadRuntime(ctx context.Context, proxies []workerruntime.OwnedProxy) ([]workerruntime.Snapshot, error) {
if err := validateRuntimeCall(ctx, a); err != nil {
return nil, err
}
if len(proxies) > a.options.MaxRuntimeCounters {
return nil, workerruntime.ErrInvalidQuery
}
wires := make([]runtimeOwnedProxyWire, len(proxies))
seen := make(map[string]struct{}, len(proxies))
for index, proxy := range proxies {
if !runtimeClean(proxy.ProxyID) || !runtimeClean(proxy.WorkerID) || proxy.OwnershipEpoch == 0 {
return nil, workerruntime.ErrInvalidQuery
}
key := proxy.WorkerID + "\x00" + proxy.ProxyID
if _, exists := seen[key]; exists {
return nil, workerruntime.ErrInvalidQuery
}
seen[key] = struct{}{}
wires[index] = runtimeOwnedProxyWire{
ProxyID: proxy.ProxyID, WorkerID: proxy.WorkerID,
OwnershipEpoch: strconv.FormatUint(proxy.OwnershipEpoch, 10),
}
}
payload, err := json.Marshal(wires)
if err != nil {
return nil, workerruntime.ErrInvalidQuery
}
reply, err := a.runRuntime(ctx, runtimeRead, 0, payload, "")
if err != nil {
return nil, err
}
if reply.Status == scriptInvalid {
return nil, workerruntime.ErrInvalidQuery
}
if reply.Status != scriptOK || len(reply.Snapshots) != len(proxies) {
return nil, invalidScriptReply("unexpected worker runtime read reply")
}
result := make([]workerruntime.Snapshot, len(reply.Snapshots))
for index, snapshot := range reply.Snapshots {
if snapshot.ProxyID != proxies[index].ProxyID || snapshot.Active < 0 || snapshot.Reserved < 0 {
return nil, invalidScriptReply("invalid worker runtime snapshot")
}
result[index] = workerruntime.Snapshot{
ProxyID: snapshot.ProxyID, Active: snapshot.Active, Reserved: snapshot.Reserved,
Draining: snapshot.Draining, Fresh: snapshot.Fresh,
}
}
return result, nil
}
func (a *Adapter) encodeRuntimeReport(report workerruntime.Report, ttl time.Duration) ([]byte, string, error) {
if ttl <= 0 || !runtimeClean(report.WorkerID) || !runtimeClean(report.SessionID) ||
report.Sequence == 0 || report.SnapshotVersion == 0 || report.OwnershipEpoch == 0 || report.ObservedAt.IsZero() ||
len(report.Counters) > a.options.MaxRuntimeCounters {
return nil, "", workerruntime.ErrInvalidReport
}
counters := append([]workerruntime.Counter(nil), report.Counters...)
sort.Slice(counters, func(left, right int) bool { return counters[left].ProxyID < counters[right].ProxyID })
wires := make([]runtimeCounterWire, len(counters))
for index, counter := range counters {
if !runtimeClean(counter.ProxyID) || counter.Active < 0 || counter.Reserved < 0 ||
(index > 0 && counters[index-1].ProxyID == counter.ProxyID) {
return nil, "", workerruntime.ErrInvalidReport
}
wires[index] = runtimeCounterWire{
ProxyID: counter.ProxyID, Active: counter.Active,
Reserved: counter.Reserved, Draining: counter.Draining,
}
}
payload, err := json.Marshal(runtimeReportWire{
Version: runtimeWireVersion, WorkerID: report.WorkerID, SessionID: report.SessionID,
Sequence: strconv.FormatUint(report.Sequence, 10),
SnapshotVersion: strconv.FormatUint(report.SnapshotVersion, 10),
OwnershipEpoch: strconv.FormatUint(report.OwnershipEpoch, 10),
ObservedAtMS: report.ObservedAt.UTC().UnixMilli(), Counters: wires,
})
if err != nil {
return nil, "", workerruntime.ErrInvalidReport
}
digest := sha256.Sum256(payload)
return payload, hex.EncodeToString(digest[:]), nil
}
func (a *Adapter) runRuntime(
ctx context.Context,
operation string,
ttlMS int64,
payload []byte,
digest string,
) (runtimeScriptReply, error) {
result, err := runScript(ctx, a.client, runtimeScript, []string{
a.keys.workerSessions, a.keys.workerSessionExpiry,
a.keys.workerRuntime, a.keys.workerRuntimeExpiry, a.keys.owners,
}, operation, ttlMS, a.options.CleanupLimit, string(payload), digest)
if err != nil {
return runtimeScriptReply{}, err
}
var reply runtimeScriptReply
if err := decodeScriptResult(result, &reply); err != nil {
return runtimeScriptReply{}, err
}
return reply, nil
}
func validateRuntimeCall(ctx context.Context, adapter *Adapter) error {
if ctx == nil || adapter == nil {
return workerruntime.ErrInvalidStore
}
if err := ctx.Err(); err != nil {
return err
}
return nil
}
func runtimeClean(value string) bool {
return value != "" && strings.TrimSpace(value) == value
}

147
internal/adapters/redisactivity/runtime_integration_test.go Normal file
View File

@ -0,0 +1,147 @@
//go:build integration
package redisactivity
import (
"context"
"errors"
"testing"
"time"
"proxy-pool/internal/domain/workerruntime"
)
func TestRedisWorkerRuntimeReplacesSparseCountersAndFencesReports(t *testing.T) {
fixture := newRedisTestFixture(t)
now := redisTestNow()
seedRedisAvailable(t, fixture.Adapter, "provider-a", now, now.Add(time.Second), 2*time.Minute,
testProxy("proxy-a", "192.0.2.10"))
assignment, err := fixture.Adapter.Assign(context.Background(), now.Add(2*time.Second),
"proxy-a", "worker-a", time.Minute)
if err != nil {
t.Fatalf("Assign(): %v", err)
}
session := workerruntime.Session{
WorkerID: "worker-a", InstanceID: "instance-a", SessionID: "session-a",
AckedSnapshotVersion: 3, AckedOwnershipEpoch: assignment.Epoch,
}
if err := fixture.Adapter.ReplaceSession(context.Background(), session, time.Minute); err != nil {
t.Fatalf("ReplaceSession(): %v", err)
}
report := workerruntime.Report{
WorkerID: "worker-a", SessionID: "session-a", Sequence: 2,
SnapshotVersion: 3, OwnershipEpoch: assignment.Epoch, ObservedAt: now,
Counters: []workerruntime.Counter{{ProxyID: "proxy-a", Active: 2, Reserved: 1}},
}
if err := fixture.Adapter.ReplaceRuntime(context.Background(), report, time.Minute); err != nil {
t.Fatalf("ReplaceRuntime(first): %v", err)
}
if err := fixture.Adapter.ReplaceRuntime(context.Background(), report, time.Minute); err != nil {
t.Fatalf("ReplaceRuntime(replay): %v", err)
}
conflict := report
conflict.Counters = []workerruntime.Counter{{ProxyID: "proxy-a", Active: 3}}
if err := fixture.Adapter.ReplaceRuntime(context.Background(), conflict, time.Minute); !errors.Is(err, workerruntime.ErrConflictingReport) {
t.Fatalf("ReplaceRuntime(conflict) error = %v", err)
}
stale := report
stale.Sequence = 1
if err := fixture.Adapter.ReplaceRuntime(context.Background(), stale, time.Minute); !errors.Is(err, workerruntime.ErrStaleReport) {
t.Fatalf("ReplaceRuntime(stale) error = %v", err)
}
query := []workerruntime.OwnedProxy{{
ProxyID: "proxy-a", WorkerID: "worker-a", OwnershipEpoch: assignment.Epoch,
}}
got, err := fixture.Adapter.ReadRuntime(context.Background(), query)
if err != nil || len(got) != 1 || got[0] != (workerruntime.Snapshot{
ProxyID: "proxy-a", Active: 2, Reserved: 1, Fresh: true,
}) {
t.Fatalf("ReadRuntime(first) = %+v, %v", got, err)
}
report.Sequence = 3
report.Counters = nil
if err := fixture.Adapter.ReplaceRuntime(context.Background(), report, time.Minute); err != nil {
t.Fatalf("ReplaceRuntime(empty): %v", err)
}
got, err = fixture.Adapter.ReadRuntime(context.Background(), query)
if err != nil || len(got) != 1 || got[0] != (workerruntime.Snapshot{ProxyID: "proxy-a", Fresh: true}) {
t.Fatalf("ReadRuntime(empty) = %+v, %v", got, err)
}
if err := fixture.Adapter.ReplaceSession(context.Background(), workerruntime.Session{
WorkerID: "worker-a", InstanceID: "instance-b", SessionID: "session-b",
AckedSnapshotVersion: 4, AckedOwnershipEpoch: assignment.Epoch + 1,
}, time.Minute); err != nil {
t.Fatalf("ReplaceSession(new): %v", err)
}
if err := fixture.Adapter.ReplaceRuntime(context.Background(), report, time.Minute); !errors.Is(err, workerruntime.ErrStaleSession) {
t.Fatalf("ReplaceRuntime(old session) error = %v", err)
}
}
func TestRedisWorkerRuntimeExpiresFailClosed(t *testing.T) {
fixture := newRedisTestFixture(t)
now := redisTestNow()
seedRedisAvailable(t, fixture.Adapter, "provider-a", now, now.Add(time.Second), 2*time.Minute,
testProxy("proxy-a", "192.0.2.10"))
assignment, err := fixture.Adapter.Assign(context.Background(), now.Add(2*time.Second),
"proxy-a", "worker-a", time.Minute)
if err != nil {
t.Fatalf("Assign(): %v", err)
}
if err := fixture.Adapter.ReplaceSession(context.Background(), workerruntime.Session{
WorkerID: "worker-a", InstanceID: "instance-a", SessionID: "session-a",
AckedSnapshotVersion: 1, AckedOwnershipEpoch: assignment.Epoch,
}, 100*time.Millisecond); err != nil {
t.Fatalf("ReplaceSession(): %v", err)
}
if err := fixture.Adapter.ReplaceRuntime(context.Background(), workerruntime.Report{
WorkerID: "worker-a", SessionID: "session-a", Sequence: 1,
SnapshotVersion: 1, OwnershipEpoch: assignment.Epoch, ObservedAt: now,
}, 100*time.Millisecond); err != nil {
t.Fatalf("ReplaceRuntime(): %v", err)
}
time.Sleep(150 * time.Millisecond)
got, err := fixture.Adapter.ReadRuntime(context.Background(), []workerruntime.OwnedProxy{{
ProxyID: "proxy-a", WorkerID: "worker-a", OwnershipEpoch: assignment.Epoch,
}})
if err != nil || len(got) != 1 || got[0].Fresh {
t.Fatalf("ReadRuntime(expired) = %+v, %v", got, err)
}
}
func TestRedisWorkerRuntimeRejectsEmptyReportBeyondAcknowledgedSnapshot(t *testing.T) {
fixture := newRedisTestFixture(t)
now := redisTestNow()
if err := fixture.Adapter.ReplaceSession(context.Background(), workerruntime.Session{
WorkerID: "worker-a", InstanceID: "instance-a", SessionID: "session-a",
AckedSnapshotVersion: 3, AckedOwnershipEpoch: 9,
}, time.Minute); err != nil {
t.Fatalf("ReplaceSession(): %v", err)
}
for name, report := range map[string]workerruntime.Report{
"version": {
WorkerID: "worker-a", SessionID: "session-a", Sequence: 1,
SnapshotVersion: 4, OwnershipEpoch: 9, ObservedAt: now,
},
"epoch": {
WorkerID: "worker-a", SessionID: "session-a", Sequence: 1,
SnapshotVersion: 3, OwnershipEpoch: 10, ObservedAt: now,
},
} {
t.Run(name, func(t *testing.T) {
if err := fixture.Adapter.ReplaceRuntime(context.Background(), report, time.Minute); !errors.Is(err, workerruntime.ErrStaleReport) {
t.Fatalf("ReplaceRuntime() error = %v, want ErrStaleReport", err)
}
})
}
}
func TestRedisWorkerRuntimeAcceptsEmptyRead(t *testing.T) {
fixture := newRedisTestFixture(t)
got, err := fixture.Adapter.ReadRuntime(context.Background(), nil)
if err != nil || got == nil || len(got) != 0 {
t.Fatalf("ReadRuntime(empty) = %#v, %v", got, err)
}
}

19
internal/adapters/redisactivity/scripts.go
View File

@ -74,6 +74,17 @@ type statusScriptInventory struct {
Extracted int64 `json:"extracted"`
}
type runtimeScriptReply struct {
Status scriptStatus `json:"status"`
Snapshots []runtimeSnapshotWire `json:"snapshots"`
}
type capacityScriptReply struct {
Status scriptStatus `json:"status"`
Managed int `json:"managed"`
AvailableSlots int64 `json:"availableSlots,string"`
}
//go:embed scripts/upsert.lua
var upsertSource string
@ -92,6 +103,12 @@ var sweepSource string
//go:embed scripts/status.lua
var statusSource string
//go:embed scripts/runtime.lua
var runtimeSource string
//go:embed scripts/capacity.lua
var capacitySource string
var (
upsertScript = redis.NewScript(upsertSource)
healthScript = redis.NewScript(healthSource)
@ -99,6 +116,8 @@ var (
ownershipScript = redis.NewScript(ownershipSource)
sweepScript = redis.NewScript(sweepSource)
statusScript = redis.NewScript(statusSource)
runtimeScript = redis.NewScript(runtimeSource)
capacityScript = redis.NewScript(capacitySource)
)
func runScript(ctx context.Context, client redis.Scripter, script *redis.Script, keys []string, args ...any) (any, error) {

177
internal/adapters/redisactivity/scripts/capacity.lua Normal file
View File

@ -0,0 +1,177 @@
local records_key = KEYS[1]
local inventory_key = KEYS[2]
local available_upstream_key = KEYS[3]
local owned_upstream_key = KEYS[4]
local owners_key = KEYS[5]
local sessions_key = KEYS[6]
local session_expiry_key = KEYS[7]
local runtime_key = KEYS[8]
local runtime_expiry_key = KEYS[9]
local upstream_id = ARGV[1]
local safety_margin_ms = tonumber(ARGV[2])
local scan_limit = tonumber(ARGV[3])
local cleanup_limit = tonumber(ARGV[4])
local function reply(status, managed, available_slots)
return cjson.encode({
status = status,
managed = managed or 0,
availableSlots = tostring(available_slots or 0)
})
end
local function now_ms()
local value = redis.call('TIME')
return tonumber(value[1]) * 1000 + math.floor(tonumber(value[2]) / 1000)
end
local function decode_table(value)
if not value then
return nil
end
local ok, decoded = pcall(cjson.decode, value)
if not ok or type(decoded) ~= 'table' then
return nil
end
return decoded
end
local function valid_uint(value)
return type(value) == 'string' and string.match(value, '^[0-9]+$') and
value ~= '0' and (string.len(value) == 1 or string.sub(value, 1, 1) ~= '0')
end
local function compare_uint(left, right)
if string.len(left) ~= string.len(right) then
return string.len(left) < string.len(right) and -1 or 1
end
if left == right then
return 0
end
return left < right and -1 or 1
end
local function cleanup(now)
local expired_sessions = redis.call('ZRANGEBYSCORE', session_expiry_key, '-inf', now, 'LIMIT', 0, cleanup_limit)
for _, worker_id in ipairs(expired_sessions) do
redis.call('HDEL', sessions_key, worker_id)
redis.call('HDEL', runtime_key, worker_id)
redis.call('ZREM', session_expiry_key, worker_id)
redis.call('ZREM', runtime_expiry_key, worker_id)
end
local expired_reports = redis.call('ZRANGEBYSCORE', runtime_expiry_key, '-inf', now, 'LIMIT', 0, cleanup_limit)
for _, worker_id in ipairs(expired_reports) do
redis.call('HDEL', runtime_key, worker_id)
redis.call('ZREM', runtime_expiry_key, worker_id)
end
end
if type(upstream_id) ~= 'string' or upstream_id == '' or not safety_margin_ms or safety_margin_ms < 0 or
not scan_limit or scan_limit <= 0 or not cleanup_limit or cleanup_limit <= 0 then
return reply('invalid', 0, 0)
end
local now = now_ms()
cleanup(now)
local threshold = now + safety_margin_ms
local available_ids = redis.call('ZRANGEBYSCORE', available_upstream_key, '(' .. threshold, '+inf',
'LIMIT', 0, scan_limit + 1)
if #available_ids > scan_limit then
return reply('unavailable', 0, 0)
end
local remaining = scan_limit - #available_ids
local owned_ids = redis.call('ZRANGEBYSCORE', owned_upstream_key, '(' .. threshold, '+inf',
'LIMIT', 0, remaining + 1)
if #owned_ids > remaining then
return reply('unavailable', 0, 0)
end
local managed = tonumber(redis.call('HGET', inventory_key, upstream_id) or '0')
if not managed or managed < 0 or managed ~= math.floor(managed) then
return reply('unavailable', 0, 0)
end
local available_slots = 0
local worker_cache = {}
local seen = {}
for _, proxy_id in ipairs(available_ids) do
seen[proxy_id] = true
local record = decode_table(redis.call('HGET', records_key, proxy_id))
if not record or type(record.sourceUpstream) ~= 'string' or type(record.state) ~= 'string' or
record.sourceUpstream ~= upstream_id or record.state ~= 'AVAILABLE' or
type(record.usableUntilMs) ~= 'number' or record.usableUntilMs <= threshold or
type(record.maxConcurrency) ~= 'number' or record.maxConcurrency < 0 or
record.maxConcurrency ~= math.floor(record.maxConcurrency) or
(record.ownerWorkerId and record.ownerWorkerId ~= '') or redis.call('HGET', owners_key, proxy_id) then
return reply('unavailable', 0, 0)
end
available_slots = available_slots + record.maxConcurrency
end
for _, proxy_id in ipairs(owned_ids) do
if seen[proxy_id] then
return reply('unavailable', 0, 0)
end
local record = decode_table(redis.call('HGET', records_key, proxy_id))
if not record or record.sourceUpstream ~= upstream_id or record.state ~= 'AVAILABLE' or
type(record.usableUntilMs) ~= 'number' or record.usableUntilMs <= threshold or
type(record.maxConcurrency) ~= 'number' or record.maxConcurrency < 0 or
record.maxConcurrency ~= math.floor(record.maxConcurrency) or
type(record.ownerWorkerId) ~= 'string' or record.ownerWorkerId == '' then
return reply('unavailable', 0, 0)
end
local owner_worker_id = record.ownerWorkerId
local owner = decode_table(redis.call('HGET', owners_key, proxy_id))
if not owner or owner.workerId ~= owner_worker_id or type(owner.epoch) ~= 'number' or
type(owner.expiresAtMs) ~= 'number' or owner.expiresAtMs <= now or
type(owner.draining) ~= 'boolean' then
return reply('unavailable', 0, 0)
end
local cached = worker_cache[owner_worker_id]
if not cached then
local session = decode_table(redis.call('HGET', sessions_key, owner_worker_id))
local report = decode_table(redis.call('HGET', runtime_key, owner_worker_id))
cached = {fresh = false, counters = {}}
if session and report and session.workerId == owner_worker_id and
report.workerId == owner_worker_id and session.sessionId == report.sessionId and
type(session.expiresAtMs) == 'number' and session.expiresAtMs > now and
type(report.expiresAtMs) == 'number' and report.expiresAtMs > now and
valid_uint(session.ackedSnapshotVersion) and valid_uint(session.ackedOwnershipEpoch) and
report.snapshotVersion == session.ackedSnapshotVersion and
report.ownershipEpoch == session.ackedOwnershipEpoch then
cached.fresh = true
cached.ownershipEpoch = report.ownershipEpoch
if type(report.counters) == 'table' then
for _, counter in pairs(report.counters) do
if type(counter) == 'table' and type(counter.proxyId) == 'string' then
cached.counters[counter.proxyId] = counter
end
end
end
end
worker_cache[owner_worker_id] = cached
end
local owner_epoch = tostring(owner.epoch)
if cached.fresh and valid_uint(owner_epoch) and
compare_uint(cached.ownershipEpoch, owner_epoch) >= 0 then
local counter = cached.counters[proxy_id]
local active = 0
local reserved = 0
local draining = false
if counter then
active = counter.active
reserved = counter.reserved
draining = counter.draining
end
if type(active) == 'number' and type(reserved) == 'number' and active >= 0 and reserved >= 0 and
active == math.floor(active) and reserved == math.floor(reserved) and
not draining and not owner.draining then
local slots = record.maxConcurrency - active - reserved
if slots > 0 then
available_slots = available_slots + slots
end
end
end
end
return reply('ok', managed, available_slots)

7
internal/adapters/redisactivity/scripts/extract.lua
View File

@ -113,6 +113,12 @@ local function remove_available(proxy_id, record)
end
end
local function remove_owned(proxy_id, record)
if record and type(record.ownerIndexKey) == 'string' and record.ownerIndexKey ~= '' then
redis.call('ZREM', record.ownerIndexKey, proxy_id)
end
end
local function remove_proxy(proxy_id)
local raw = redis.call('HGET', records_key, proxy_id)
local record = nil
@ -120,6 +126,7 @@ local function remove_proxy(proxy_id)
local decoded
decoded, record = pcall(cjson.decode, raw)
remove_available(proxy_id, decoded and record or nil)
remove_owned(proxy_id, decoded and record or nil)
if decoded and type(record) == 'table' and is_managed(record.state) then
decrement_inventory(record.sourceUpstream)
end

18
internal/adapters/redisactivity/scripts/health.lua
View File

@ -70,12 +70,19 @@ local function remove_available(id, record)
end
end
local function remove_owned(id, record)
if record and type(record.ownerIndexKey) == 'string' and record.ownerIndexKey ~= '' then
redis.call('ZREM', record.ownerIndexKey, id)
end
end
local function remove_proxy(id)
local raw = redis.call('HGET', records_key, id)
local record = nil
if raw then
record = cjson.decode(raw)
remove_available(id, record)
remove_owned(id, record)
if is_managed(record.state) then
decrement_inventory(record.sourceUpstream)
end
@ -142,6 +149,9 @@ if not raw then
return finish({status = 'not_found'})
end
local record = cjson.decode(raw)
if type(record.ownerIndexKey) ~= 'string' or record.ownerIndexKey == '' then
return finish({status = 'invalid'})
end
if tonumber(record.expiresAtMs) <= checked_at_ms then
remove_proxy(proxy_id)
return finish({status = 'not_found'})
@ -184,13 +194,19 @@ end
local encoded = cjson.encode(record)
redis.call('HSET', records_key, proxy_id, encoded)
local owned = (record.ownerWorkerId and record.ownerWorkerId ~= '') or redis.call('HEXISTS', owners_key, proxy_id) == 1
if next_state == 'AVAILABLE' and not owned and tonumber(record.usableUntilMs) > checked_at_ms then
remove_owned(proxy_id, record)
if next_state == 'AVAILABLE' and tonumber(record.usableUntilMs) > checked_at_ms then
if owned then
redis.call('ZADD', record.ownerIndexKey, record.usableUntilMs, proxy_id)
touch(record.ownerIndexKey, tonumber(record.expiresAtMs))
else
redis.call('ZADD', available_key, record.usableUntilMs, proxy_id)
for _, index_key in ipairs(record.indexKeys or {}) do
redis.call('ZADD', index_key, record.usableUntilMs, proxy_id)
touch(index_key, tonumber(record.expiresAtMs))
end
end
end
touch(records_key, tonumber(record.expiresAtMs))
touch(unique_key, tonumber(record.expiresAtMs))
touch(idkeys_key, tonumber(record.expiresAtMs))

13
internal/adapters/redisactivity/scripts/ownership.lua
View File

@ -94,6 +94,12 @@ local function remove_available(id, record)
end
end
local function remove_owned(id, record)
if record and type(record.ownerIndexKey) == 'string' and record.ownerIndexKey ~= '' then
redis.call('ZREM', record.ownerIndexKey, id)
end
end
local function add_available(id, record, at_ms)
local usable_until_ms = record and tonumber(record.usableUntilMs)
if not usable_until_ms or record.state ~= 'AVAILABLE' or usable_until_ms <= at_ms then
@ -118,6 +124,7 @@ local function remove_proxy(id)
local decoded
decoded, record = pcall(cjson.decode, raw)
remove_available(id, decoded and record or nil)
remove_owned(id, decoded and record or nil)
if decoded and type(record) == 'table' and is_managed(record.state) then
decrement_inventory(record.sourceUpstream)
end
@ -169,6 +176,7 @@ local function clear_owner(id, assignment, at_ms, restore)
local raw_record = redis.call('HGET', records_key, id)
local record = decode_table(raw_record)
if record and (not assignment or record.ownerWorkerId == assignment.workerId) then
remove_owned(id, record)
record.ownerWorkerId = nil
redis.call('HSET', records_key, id, cjson.encode(record))
if restore then
@ -195,6 +203,7 @@ if operation == 'assign' then
local record = decode_table(redis.call('HGET', records_key, proxy_id))
if not record or record.state ~= 'AVAILABLE' or
type(record.ownerIndexKey) ~= 'string' or record.ownerIndexKey == '' or
(record.ownerWorkerId and record.ownerWorkerId ~= '') or
redis.call('HEXISTS', owners_key, proxy_id) == 1 or
not tonumber(record.usableUntilMs) or tonumber(record.usableUntilMs) <= now_ms then
@ -223,6 +232,8 @@ if operation == 'assign' then
record.ownerWorkerId = worker_id
redis.call('HSET', records_key, proxy_id, cjson.encode(record))
remove_available(proxy_id, record)
redis.call('ZADD', record.ownerIndexKey, record.usableUntilMs, proxy_id)
touch(record.ownerIndexKey, tonumber(record.expiresAtMs))
return finish({status = 'ok', record = encoded})
end
@ -267,6 +278,8 @@ if operation == 'begin_drain' then
current.assignmentVersion = tonumber(current.assignmentVersion) + 1
local encoded = cjson.encode(current)
redis.call('HSET', owners_key, proxy_id, encoded)
local record = decode_table(redis.call('HGET', records_key, proxy_id))
remove_owned(proxy_id, record)
return finish({status = 'ok', record = encoded})
end
return finish({status = 'ok', record = cjson.encode(current)})

229
internal/adapters/redisactivity/scripts/runtime.lua Normal file
View File

@ -0,0 +1,229 @@
local sessions_key = KEYS[1]
local session_expiry_key = KEYS[2]
local runtime_key = KEYS[3]
local runtime_expiry_key = KEYS[4]
local owners_key = KEYS[5]
local operation = ARGV[1]
local ttl_ms = tonumber(ARGV[2])
local cleanup_limit = tonumber(ARGV[3])
local payload = ARGV[4]
local digest = ARGV[5]
local function reply(status, snapshots)
if snapshots then
return cjson.encode({status = status, snapshots = snapshots})
end
return '{"status":' .. cjson.encode(status) .. ',"snapshots":[]}'
end
local function now_ms()
local value = redis.call('TIME')
return tonumber(value[1]) * 1000 + math.floor(tonumber(value[2]) / 1000)
end
local function decode_table(value)
if not value then
return nil
end
local ok, decoded = pcall(cjson.decode, value)
if not ok or type(decoded) ~= 'table' then
return nil
end
return decoded
end
local function valid_uint(value)
return type(value) == 'string' and string.match(value, '^[0-9]+$') and
value ~= '0' and (string.len(value) == 1 or string.sub(value, 1, 1) ~= '0')
end
local function compare_uint(left, right)
if string.len(left) ~= string.len(right) then
return string.len(left) < string.len(right) and -1 or 1
end
if left == right then
return 0
end
return left < right and -1 or 1
end
local function cleanup(now)
local expired_sessions = redis.call('ZRANGEBYSCORE', session_expiry_key, '-inf', now, 'LIMIT', 0, cleanup_limit)
for _, worker_id in ipairs(expired_sessions) do
redis.call('HDEL', sessions_key, worker_id)
redis.call('HDEL', runtime_key, worker_id)
redis.call('ZREM', session_expiry_key, worker_id)
redis.call('ZREM', runtime_expiry_key, worker_id)
end
local expired_reports = redis.call('ZRANGEBYSCORE', runtime_expiry_key, '-inf', now, 'LIMIT', 0, cleanup_limit)
for _, worker_id in ipairs(expired_reports) do
redis.call('HDEL', runtime_key, worker_id)
redis.call('ZREM', runtime_expiry_key, worker_id)
end
end
local function valid_session(value)
return value and value.version == 1 and type(value.workerId) == 'string' and value.workerId ~= '' and
type(value.instanceId) == 'string' and value.instanceId ~= '' and
type(value.sessionId) == 'string' and value.sessionId ~= '' and
valid_uint(value.ackedSnapshotVersion) and valid_uint(value.ackedOwnershipEpoch)
end
local function valid_owner(value, worker_id, ownership_epoch, now)
return value and type(value.workerId) == 'string' and value.workerId == worker_id and
type(value.epoch) == 'number' and valid_uint(tostring(value.epoch)) and
compare_uint(ownership_epoch, tostring(value.epoch)) >= 0 and
type(value.expiresAtMs) == 'number' and value.expiresAtMs > now
end
local now = now_ms()
cleanup(now)
if operation == 'replace_session' then
if not ttl_ms or ttl_ms <= 0 then
return reply('invalid')
end
local session = decode_table(payload)
if not valid_session(session) then
return reply('invalid')
end
local current = decode_table(redis.call('HGET', sessions_key, session.workerId))
if current and valid_session(current) and current.sessionId == session.sessionId and
current.instanceId == session.instanceId and type(current.expiresAtMs) == 'number' and
current.expiresAtMs > now then
local epoch_order = compare_uint(session.ackedOwnershipEpoch, current.ackedOwnershipEpoch)
local version_order = compare_uint(session.ackedSnapshotVersion, current.ackedSnapshotVersion)
if epoch_order < 0 or (epoch_order == 0 and version_order < 0) then
return reply('stale')
end
if epoch_order > 0 or version_order > 0 then
redis.call('HDEL', runtime_key, session.workerId)
redis.call('ZREM', runtime_expiry_key, session.workerId)
end
else
redis.call('HDEL', runtime_key, session.workerId)
redis.call('ZREM', runtime_expiry_key, session.workerId)
end
session.expiresAtMs = now + ttl_ms
redis.call('HSET', sessions_key, session.workerId, cjson.encode(session))
redis.call('ZADD', session_expiry_key, session.expiresAtMs, session.workerId)
return reply('ok')
end
if operation == 'replace_report' then
if not ttl_ms or ttl_ms <= 0 or type(digest) ~= 'string' or digest == '' then
return reply('invalid')
end
local report = decode_table(payload)
if not report or report.version ~= 1 or type(report.workerId) ~= 'string' or report.workerId == '' or
type(report.sessionId) ~= 'string' or report.sessionId == '' or not valid_uint(report.sequence) or
not valid_uint(report.snapshotVersion) or not valid_uint(report.ownershipEpoch) or
type(report.observedAtMs) ~= 'number' or type(report.counters) ~= 'table' then
return reply('invalid')
end
local session = decode_table(redis.call('HGET', sessions_key, report.workerId))
if not valid_session(session) or session.workerId ~= report.workerId or session.sessionId ~= report.sessionId or
type(session.expiresAtMs) ~= 'number' or session.expiresAtMs <= now then
return reply('unavailable')
end
if report.snapshotVersion ~= session.ackedSnapshotVersion or
report.ownershipEpoch ~= session.ackedOwnershipEpoch then
return reply('stale')
end
local current = decode_table(redis.call('HGET', runtime_key, report.workerId))
if current and current.sessionId == report.sessionId and valid_uint(current.sequence) then
local ordering = compare_uint(report.sequence, current.sequence)
if ordering < 0 then
return reply('stale')
end
if ordering == 0 then
if current.digest == digest then
return reply('ok')
end
return reply('conflict')
end
end
local seen = {}
for _, counter in pairs(report.counters) do
if type(counter) ~= 'table' or type(counter.proxyId) ~= 'string' or counter.proxyId == '' or
type(counter.active) ~= 'number' or counter.active < 0 or counter.active ~= math.floor(counter.active) or
type(counter.reserved) ~= 'number' or counter.reserved < 0 or counter.reserved ~= math.floor(counter.reserved) or
type(counter.draining) ~= 'boolean' or seen[counter.proxyId] then
return reply('invalid')
end
seen[counter.proxyId] = true
local owner = decode_table(redis.call('HGET', owners_key, counter.proxyId))
if not valid_owner(owner, report.workerId, report.ownershipEpoch, now) then
return reply('stale')
end
end
report.digest = digest
report.expiresAtMs = now + ttl_ms
redis.call('HSET', runtime_key, report.workerId, cjson.encode(report))
redis.call('ZADD', runtime_expiry_key, report.expiresAtMs, report.workerId)
session.expiresAtMs = now + ttl_ms
redis.call('HSET', sessions_key, report.workerId, cjson.encode(session))
redis.call('ZADD', session_expiry_key, session.expiresAtMs, report.workerId)
return reply('ok')
end
if operation == 'read' then
local queries = decode_table(payload)
if not queries then
return reply('invalid')
end
if next(queries) == nil then
return '{"status":"ok","snapshots":[]}'
end
local snapshots = cjson.decode('[]')
local cache = {}
for _, query in ipairs(queries) do
if type(query) ~= 'table' or type(query.proxyId) ~= 'string' or query.proxyId == '' or
type(query.workerId) ~= 'string' or query.workerId == '' or not valid_uint(query.ownershipEpoch) then
return reply('invalid')
end
local snapshot = {proxyId = query.proxyId, active = 0, reserved = 0, draining = false, fresh = false}
local owner = decode_table(redis.call('HGET', owners_key, query.proxyId))
if valid_owner(owner, query.workerId, query.ownershipEpoch, now) and
compare_uint(query.ownershipEpoch, tostring(owner.epoch)) == 0 then
local cached = cache[query.workerId]
if not cached then
local session = decode_table(redis.call('HGET', sessions_key, query.workerId))
local report = decode_table(redis.call('HGET', runtime_key, query.workerId))
cached = {fresh = false, counters = {}}
if valid_session(session) and session.workerId == query.workerId and report and
report.workerId == query.workerId and report.sessionId == session.sessionId and
type(session.expiresAtMs) == 'number' and session.expiresAtMs > now and
type(report.expiresAtMs) == 'number' and report.expiresAtMs > now and
valid_uint(report.ownershipEpoch) and
report.snapshotVersion == session.ackedSnapshotVersion and
report.ownershipEpoch == session.ackedOwnershipEpoch then
cached.fresh = true
cached.ownershipEpoch = report.ownershipEpoch
if type(report.counters) == 'table' then
for _, counter in pairs(report.counters) do
if type(counter) == 'table' and type(counter.proxyId) == 'string' then
cached.counters[counter.proxyId] = counter
end
end
end
end
cache[query.workerId] = cached
end
if cached.fresh and compare_uint(cached.ownershipEpoch, query.ownershipEpoch) >= 0 then
snapshot.fresh = true
local counter = cached.counters[query.proxyId]
if counter then
snapshot.active = counter.active
snapshot.reserved = counter.reserved
snapshot.draining = counter.draining
end
end
end
snapshots[#snapshots + 1] = snapshot
end
return reply('ok', snapshots)
end
return reply('invalid')

7
internal/adapters/redisactivity/scripts/sweep.lua
View File

@ -73,6 +73,12 @@ local function remove_available(proxy_id, record)
end
end
local function remove_owned(proxy_id, record)
if record and type(record.ownerIndexKey) == 'string' and record.ownerIndexKey ~= '' then
redis.call('ZREM', record.ownerIndexKey, proxy_id)
end
end
local function remove_proxy(proxy_id)
local raw = redis.call('HGET', records_key, proxy_id)
local record = nil
@ -80,6 +86,7 @@ local function remove_proxy(proxy_id)
local decoded
decoded, record = pcall(cjson.decode, raw)
remove_available(proxy_id, decoded and record or nil)
remove_owned(proxy_id, decoded and record or nil)
if decoded and type(record) == 'table' and is_managed(record.state) then
decrement_inventory(record.sourceUpstream)
end

24
internal/adapters/redisactivity/scripts/upsert.lua
View File

@ -70,12 +70,19 @@ local function remove_available(proxy_id, record)
end
end
local function remove_owned(proxy_id, record)
if record and type(record.ownerIndexKey) == 'string' and record.ownerIndexKey ~= '' then
redis.call('ZREM', record.ownerIndexKey, proxy_id)
end
end
local function remove_proxy(proxy_id)
local raw = redis.call('HGET', records_key, proxy_id)
local record = nil
if raw then
record = cjson.decode(raw)
remove_available(proxy_id, record)
remove_owned(proxy_id, record)
if is_managed(record.state) then
decrement_inventory(record.sourceUpstream)
end
@ -123,6 +130,16 @@ local function add_available(proxy_id, record)
end
end
local function sync_owned(proxy_id, record)
if record.state == 'AVAILABLE' and record.ownerWorkerId and record.ownerWorkerId ~= '' and
tonumber(record.usableUntilMs) > now_ms then
redis.call('ZADD', record.ownerIndexKey, record.usableUntilMs, proxy_id)
touch(record.ownerIndexKey, tonumber(record.expiresAtMs))
else
redis.call('ZREM', record.ownerIndexKey, proxy_id)
end
end
local function finish(reply)
local encoded = cjson.encode(reply)
redis.call('SET', operation_key, encoded, 'PX', operation_ttl_ms)
@ -132,6 +149,11 @@ end
cleanup_expired()
for _, candidate in ipairs(candidates) do
local decoded, incoming = pcall(cjson.decode, candidate.record)
if not decoded or type(incoming) ~= 'table' or
type(incoming.ownerIndexKey) ~= 'string' or incoming.ownerIndexKey == '' then
return finish({status = 'invalid', accepted = 0, inserted = 0, refreshed = 0, dropped = 0})
end
local mapped = redis.call('HGET', idkeys_key, candidate.proxyId)
if mapped and mapped ~= candidate.uniqueDigest then
return finish({status = 'invalid', accepted = 0, inserted = 0, refreshed = 0, dropped = 0})
@ -174,6 +196,7 @@ for _, candidate in ipairs(candidates) do
local encoded = cjson.encode(incoming)
redis.call('HSET', records_key, incumbent_id, encoded)
redis.call('ZADD', expiry_key, incoming.expiresAtMs, incumbent_id)
sync_owned(incumbent_id, incoming)
add_available(incumbent_id, incoming)
if tonumber(incoming.expiresAtMs) > max_expiry_ms then
max_expiry_ms = tonumber(incoming.expiresAtMs)
@ -192,6 +215,7 @@ for _, candidate in ipairs(candidates) do
redis.call('HSET', unique_key, candidate.uniqueDigest, candidate.proxyId)
redis.call('HSET', idkeys_key, candidate.proxyId, candidate.uniqueDigest)
redis.call('ZADD', expiry_key, incoming.expiresAtMs, candidate.proxyId)
sync_owned(candidate.proxyId, incoming)
if is_managed(incoming.state) then
redis.call('HINCRBY', inventory_key, candidate.upstream, 1)
end

6
internal/adapters/redisactivity/testredis_test.go
View File

@ -38,6 +38,10 @@ func TestRedisFixtureUsesIsolatedNamespace(t *testing.T) {
}
func newRedisTestFixture(t *testing.T) redisTestFixture {
return newRedisFixtureWithCredentialCapacity(t, 10_000)
}
func newRedisFixtureWithCredentialCapacity(t *testing.T, credentialCapacity int) redisTestFixture {
t.Helper()
redisURL := os.Getenv("PROXY_POOL_TEST_REDIS_URL")
if redisURL == "" {
@ -54,7 +58,7 @@ func newRedisTestFixture(t *testing.T) redisTestFixture {
_ = client.Close()
t.Fatalf("ping test Redis: %v", err)
}
credentialStore, err := credentials.NewMemoryStore(10_000)
credentialStore, err := credentials.NewMemoryStore(credentialCapacity)
if err != nil {
_ = client.Close()
t.Fatalf("NewMemoryStore(): %v", err)

31
internal/adapters/redisactivity/upsert.go
View File

@ -15,7 +15,10 @@ import (
"proxy-pool/internal/platform/credentials"
)
const maxUpsertScriptBatch = 256
const (
maxUpsertScriptBatch = 256
transientCredentialReleaseTimeout = time.Second
)
type upsertCandidate struct {
ProxyID string `json:"proxyId"`
@ -28,6 +31,9 @@ var _ activitypool.Upserter = (*Adapter)(nil)
func (a *Adapter) UpsertFetched(ctx context.Context, upstreamID string, batch activitypool.FetchedBatch) (activitypool.UpsertResult, error) {
var result activitypool.UpsertResult
if a != nil {
defer a.releaseTransientCredentials(batch.Proxies)
}
if ctx == nil {
return result, activitypool.ErrInvalidBatch
}
@ -84,6 +90,26 @@ func (a *Adapter) UpsertFetched(ctx context.Context, upstreamID string, batch ac
return result, nil
}
func (a *Adapter) releaseTransientCredentials(proxies []proxyDomain.Proxy) {
if a.credentialReleaser == nil {
return
}
releaseCtx, cancel := context.WithTimeout(context.Background(), transientCredentialReleaseTimeout)
defer cancel()
for _, candidate := range proxies {
if releaseCtx.Err() != nil {
return
}
if candidate.SecretRef == "" || candidate.CredentialVersion == "" {
continue
}
_ = a.credentialReleaser.Release(releaseCtx, credentials.Reference{
SecretRef: candidate.SecretRef,
CredentialVersion: candidate.CredentialVersion,
})
}
}
func (a *Adapter) prepareUpsertCandidate(
ctx context.Context,
upstreamID string,
@ -139,7 +165,8 @@ func (a *Adapter) prepareUpsertCandidate(
CreatedAtMS: candidate.CreatedAt.UnixMilli(), ExpiresAtMS: expiresAt.UnixMilli(),
UsableUntilMS: usableUntil.UnixMilli(), LatencyNS: int64(candidate.Latency),
MaxConcurrency: candidate.MaxConcurrency, State: string(candidate.State),
Tags: cloneTags(candidate.Tags), IndexKeys: a.availableIndexKeys(candidate),
Tags: cloneTags(candidate.Tags), OwnerIndexKey: a.keys.owned(upstreamID),
IndexKeys: a.availableIndexKeys(candidate),
}
if candidate.LastCheckedAt != nil {
record.LastCheckedAtMS = candidate.LastCheckedAt.UnixMilli()

25
internal/adapters/redisactivity/upsert_integration_test.go
View File

@ -173,6 +173,31 @@ func TestRedisUpsertResolvesCredentialsBeforeCommit(t *testing.T) {
}
}
func TestRedisUpsertReleasesTransientCredentialCapacity(t *testing.T) {
fixture := newRedisFixtureWithCredentialCapacity(t, 1)
first := testProxy("proxy-a", "192.0.2.10")
firstReference, err := fixture.Credentials.Put(context.Background(), "first", credentials.Value{
Username: "first", Password: "first-password",
})
if err != nil {
t.Fatalf("Put(first credential): %v", err)
}
first.SecretRef = firstReference.SecretRef
first.CredentialVersion = firstReference.CredentialVersion
if _, err := fixture.Adapter.UpsertFetched(context.Background(), "provider-a", activitypool.FetchedBatch{
ObservedAt: time.Now().UTC(), ConfiguredTTL: time.Minute, MaxSize: 2,
Proxies: []proxyDomain.Proxy{first},
}); err != nil {
t.Fatalf("UpsertFetched(first): %v", err)
}
if _, err := fixture.Credentials.Put(context.Background(), "second", credentials.Value{
Username: "second", Password: "second-password",
}); err != nil {
t.Fatalf("Put(second credential after upsert): %v", err)
}
}
func TestRedisHealthTransitionsAreMonotonicAndIdempotent(t *testing.T) {
fixture := newRedisTestFixture(t)
now := redisTestNow()

65
internal/adapters/redisprovider/adapter.go
View File

@ -55,6 +55,8 @@ func (adapter *Adapter) RunLeader(
) error {
if ctx == nil || adapter == nil || work == nil || strings.TrimSpace(upstreamID) != upstreamID || upstreamID == "" ||
limits.RequestInterval < 0 || limits.MaxInFlight <= 0 || limits.MaxAttemptDuration <= 0 ||
limits.MaxTotal < 0 || limits.MaxTotal > controllerProvider.MaximumCoordinationInteger ||
int64(limits.MaxInFlight) > controllerProvider.MaximumCoordinationInteger ||
limits.MaxAttemptDuration > time.Duration(math.MaxInt64)-adapter.options.PermitGrace {
return controllerProvider.ErrInvalidCoordination
}
@ -217,9 +219,10 @@ func (session *leaderSession) Fence() controllerProvider.Fence {
return controllerProvider.Fence{Generation: session.generation, Epoch: session.epoch}
}
func (session *leaderSession) AcquireFetch(ctx context.Context) (controllerProvider.RequestPermit, error) {
if ctx == nil || session == nil || session.adapter == nil || session.ctx == nil {
return nil, controllerProvider.ErrInvalidCoordination
func (session *leaderSession) AcquireFetch(ctx context.Context, expected int) (controllerProvider.RequestPermit, bool, error) {
if ctx == nil || session == nil || session.adapter == nil || session.ctx == nil ||
expected <= 0 || int64(expected) > controllerProvider.MaximumCoordinationInteger {
return nil, false, controllerProvider.ErrInvalidCoordination
}
operationCtx, cancel := context.WithCancel(ctx)
stop := context.AfterFunc(session.ctx, cancel)
@ -229,32 +232,37 @@ func (session *leaderSession) AcquireFetch(ctx context.Context) (controllerProvi
}()
permitToken, err := randomToken()
if err != nil {
return nil, errors.Join(controllerProvider.ErrCoordinationUnavailable, err)
return nil, false, errors.Join(controllerProvider.ErrCoordinationUnavailable, err)
}
permitTTL := session.limits.MaxAttemptDuration + session.adapter.options.PermitGrace
for operationCtx.Err() == nil {
reply, scriptErr := runScript(operationCtx, session.adapter.client, session.keys,
"acquire_fetch", session.generation, session.holderID, session.token, session.epoch,
permitToken, durationMillis(session.limits.RequestInterval), session.limits.MaxInFlight,
durationMillis(permitTTL),
durationMillis(permitTTL), expected, session.limits.MaxTotal,
)
if scriptErr != nil {
if session.ctx.Err() != nil {
return nil, controllerProvider.ErrLeadershipLost
return nil, false, controllerProvider.ErrLeadershipLost
}
if operationCtx.Err() != nil {
return nil, operationCtx.Err()
return nil, false, operationCtx.Err()
}
if err := wait(operationCtx, session.adapter.options.RetryInterval); err != nil {
return nil, err
return nil, false, err
}
continue
}
switch reply.Status {
case "ok":
return &requestPermit{adapter: session.adapter, keys: session.keys, token: permitToken}, nil
return &requestPermit{
adapter: session.adapter, keys: session.keys, token: permitToken,
settlementTTL: permitTTL,
}, true, nil
case "quota_exhausted":
return nil, false, nil
case "stale":
return nil, controllerProvider.ErrLeadershipLost
return nil, false, controllerProvider.ErrLeadershipLost
case "rate_limited", "at_capacity":
delay := time.Duration(reply.WaitMS) * time.Millisecond
if delay <= 0 {
@ -262,30 +270,42 @@ func (session *leaderSession) AcquireFetch(ctx context.Context) (controllerProvi
}
if err := wait(operationCtx, delay); err != nil {
if session.ctx.Err() != nil {
return nil, controllerProvider.ErrLeadershipLost
return nil, false, controllerProvider.ErrLeadershipLost
}
return nil, err
return nil, false, err
}
default:
return nil, controllerProvider.ErrCoordinationUnavailable
return nil, false, controllerProvider.ErrCoordinationUnavailable
}
}
if session.ctx.Err() != nil {
return nil, controllerProvider.ErrLeadershipLost
return nil, false, controllerProvider.ErrLeadershipLost
}
return nil, operationCtx.Err()
return nil, false, operationCtx.Err()
}
type requestPermit struct {
adapter *Adapter
keys upstreamKeys
token string
settlementTTL time.Duration
mu sync.Mutex
done bool
}
func (permit *requestPermit) Release(ctx context.Context) error {
if ctx == nil || permit == nil || permit.adapter == nil || permit.token == "" {
func (permit *requestPermit) Complete(ctx context.Context, fetched int) error {
if fetched < 0 || int64(fetched) > controllerProvider.MaximumCoordinationInteger {
return controllerProvider.ErrInvalidCoordination
}
return permit.finish(ctx, "complete_fetch", fetched)
}
func (permit *requestPermit) Cancel(ctx context.Context) error {
return permit.finish(ctx, "cancel_fetch", 0)
}
func (permit *requestPermit) finish(ctx context.Context, operation string, fetched int) error {
if ctx == nil || permit == nil || permit.adapter == nil || permit.token == "" || permit.settlementTTL <= 0 {
return controllerProvider.ErrInvalidCoordination
}
permit.mu.Lock()
@ -293,9 +313,14 @@ func (permit *requestPermit) Release(ctx context.Context) error {
if permit.done {
return nil
}
reply, err := runScript(ctx, permit.adapter.client, permit.keys, "release_fetch", permit.token)
for ctx.Err() == nil {
reply, err := runScript(ctx, permit.adapter.client, permit.keys,
operation, permit.token, fetched, durationMillis(permit.settlementTTL))
if err != nil {
return err
if waitErr := wait(ctx, permit.adapter.options.RetryInterval); waitErr != nil {
return waitErr
}
continue
}
if reply.Status != "ok" {
return controllerProvider.ErrCoordinationUnavailable
@ -303,6 +328,8 @@ func (permit *requestPermit) Release(ctx context.Context) error {
permit.done = true
return nil
}
return ctx.Err()
}
func randomToken() (string, error) {
var token [16]byte

21
internal/adapters/redisprovider/adapter_test.go
View File

@ -89,6 +89,27 @@ func TestRunLeaderRejectsInvalidCalls(t *testing.T) {
}
}
func TestRunLeaderRejectsNegativeFetchQuota(t *testing.T) {
t.Parallel()
client := redis.NewClient(&redis.Options{Addr: "127.0.0.1:0"})
t.Cleanup(func() { _ = client.Close() })
adapter, err := New(client, Options{
Namespace: "controller", HolderID: "controller-a", LeaseTTL: 3 * time.Second,
RenewEvery: time.Second, RetryInterval: 50 * time.Millisecond,
})
if err != nil {
t.Fatalf("New(): %v", err)
}
limits := controllerProvider.CoordinationLimits{
MaxInFlight: 1, MaxAttemptDuration: time.Second, MaxTotal: -1,
}
err = adapter.RunLeader(context.Background(), "provider-a", limits,
func(context.Context, controllerProvider.LeaderSession) error { return nil })
if !errors.Is(err, controllerProvider.ErrInvalidCoordination) {
t.Fatalf("RunLeader() error = %v, want ErrInvalidCoordination", err)
}
}
func TestLeaderWorkResultPrefersParentCancellation(t *testing.T) {
t.Parallel()
ctx, cancel := context.WithCancel(context.Background())

192
internal/adapters/redisprovider/coordination_integration_test.go
View File

@ -103,36 +103,201 @@ func TestRedisLeaderSessionEnforcesGlobalIntervalAndInFlightLimit(t *testing.T)
})
}()
session := receiveSession(t, sessions)
first, err := session.AcquireFetch(context.Background())
if err != nil {
t.Fatalf("first AcquireFetch(): %v", err)
first, available, err := session.AcquireFetch(context.Background(), 1)
if err != nil || !available {
t.Fatalf("first AcquireFetch() = (%v, %t, %v)", first, available, err)
}
startedAt := time.Now()
secondResult := make(chan permitResultFixture, 1)
go func() {
permit, acquireErr := session.AcquireFetch(context.Background())
secondResult <- permitResultFixture{permit: permit, err: acquireErr}
permit, permitAvailable, acquireErr := session.AcquireFetch(context.Background(), 1)
secondResult <- permitResultFixture{permit: permit, available: permitAvailable, err: acquireErr}
}()
select {
case result := <-secondResult:
t.Fatalf("second AcquireFetch() returned before release: %+v", result)
case <-time.After(100 * time.Millisecond):
}
if err := first.Release(context.Background()); err != nil {
t.Fatalf("first Release(): %v", err)
if err := first.Complete(context.Background(), 1); err != nil {
t.Fatalf("first Complete(): %v", err)
}
result := receivePermit(t, secondResult)
if result.err != nil || result.permit == nil {
t.Fatalf("second AcquireFetch() = (%v, %v)", result.permit, result.err)
if result.err != nil || !result.available || result.permit == nil {
t.Fatalf("second AcquireFetch() = (%v, %t, %v)", result.permit, result.available, result.err)
}
if elapsed := time.Since(startedAt); elapsed < 200*time.Millisecond {
t.Fatalf("global request interval = %s, want at least 200ms", elapsed)
}
if err := result.permit.Release(context.Background()); err != nil {
t.Fatalf("second Release(): %v", err)
if err := result.permit.Cancel(context.Background()); err != nil {
t.Fatalf("second Cancel(): %v", err)
}
if err := result.permit.Release(context.Background()); err != nil {
t.Fatalf("idempotent second Release(): %v", err)
if err := result.permit.Cancel(context.Background()); err != nil {
t.Fatalf("idempotent second Cancel(): %v", err)
}
cancel()
waitRunner(t, done)
}
func TestRedisLeaderSessionPreservesFetchQuotaAcrossFailover(t *testing.T) {
fixture := newRedisFixture(t)
limits := controllerProvider.CoordinationLimits{
MaxInFlight: 1, MaxAttemptDuration: time.Second, MaxTotal: 2,
}
firstCtx, cancelFirst := context.WithCancel(context.Background())
firstSessions := make(chan controllerProvider.LeaderSession, 1)
firstDone := make(chan error, 1)
go func() {
firstDone <- fixture.coordinator(t, "controller-a").RunLeader(
firstCtx, "provider-a", limits,
func(workCtx context.Context, session controllerProvider.LeaderSession) error {
firstSessions <- session
<-workCtx.Done()
return nil
},
)
}()
firstSession := receiveSession(t, firstSessions)
firstPermit, available, err := firstSession.AcquireFetch(context.Background(), 2)
if err != nil || !available || firstPermit == nil {
t.Fatalf("first AcquireFetch() = (%v, %t, %v)", firstPermit, available, err)
}
firstFence := firstSession.Fence()
cancelFirst()
waitRunner(t, firstDone)
if err := firstPermit.Complete(context.Background(), 1); err != nil {
t.Fatalf("Complete() after leadership loss: %v", err)
}
secondCtx, cancelSecond := context.WithCancel(context.Background())
defer cancelSecond()
secondSessions := make(chan controllerProvider.LeaderSession, 1)
secondDone := make(chan error, 1)
go func() {
secondDone <- fixture.coordinator(t, "controller-b").RunLeader(
secondCtx, "provider-a", limits,
func(workCtx context.Context, session controllerProvider.LeaderSession) error {
secondSessions <- session
<-workCtx.Done()
return nil
},
)
}()
secondSession := receiveSession(t, secondSessions)
secondFence := secondSession.Fence()
if secondFence.Generation != firstFence.Generation || secondFence.Epoch <= firstFence.Epoch {
t.Fatalf("second fence = %+v, first = %+v", secondFence, firstFence)
}
secondPermit, available, err := secondSession.AcquireFetch(context.Background(), 1)
if err != nil || !available || secondPermit == nil {
t.Fatalf("second AcquireFetch() = (%v, %t, %v)", secondPermit, available, err)
}
if err := secondPermit.Complete(context.Background(), 1); err != nil {
t.Fatalf("second Complete(): %v", err)
}
exhaustedPermit, available, err := secondSession.AcquireFetch(context.Background(), 1)
if err != nil || available || exhaustedPermit != nil {
t.Fatalf("exhausted AcquireFetch() = (%v, %t, %v), want unavailable", exhaustedPermit, available, err)
}
cancelSecond()
waitRunner(t, secondDone)
}
func TestRedisFetchQuotaSettlementIsIdempotentAndCancellationRefundsReservation(t *testing.T) {
fixture := newRedisFixture(t)
limits := controllerProvider.CoordinationLimits{
MaxInFlight: 2, MaxAttemptDuration: time.Second, MaxTotal: 2,
}
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
sessions := make(chan controllerProvider.LeaderSession, 1)
done := make(chan error, 1)
go func() {
done <- fixture.coordinator(t, "controller-a").RunLeader(
ctx, "provider-a", limits,
func(workCtx context.Context, session controllerProvider.LeaderSession) error {
sessions <- session
<-workCtx.Done()
return nil
},
)
}()
session := receiveSession(t, sessions)
cancelled, available, err := session.AcquireFetch(context.Background(), 2)
if err != nil || !available || cancelled == nil {
t.Fatalf("cancelled AcquireFetch() = (%v, %t, %v)", cancelled, available, err)
}
if err := cancelled.Cancel(context.Background()); err != nil {
t.Fatalf("Cancel(): %v", err)
}
if err := cancelled.Cancel(context.Background()); err != nil {
t.Fatalf("idempotent Cancel(): %v", err)
}
completed, available, err := session.AcquireFetch(context.Background(), 2)
if err != nil || !available || completed == nil {
t.Fatalf("completed AcquireFetch() = (%v, %t, %v)", completed, available, err)
}
if err := completed.Complete(context.Background(), 1); err != nil {
t.Fatalf("Complete(): %v", err)
}
if err := completed.Complete(context.Background(), 1); err != nil {
t.Fatalf("idempotent Complete(): %v", err)
}
if err := completed.Cancel(context.Background()); err != nil {
t.Fatalf("Cancel() after Complete(): %v", err)
}
last, available, err := session.AcquireFetch(context.Background(), 1)
if err != nil || !available || last == nil {
t.Fatalf("last AcquireFetch() = (%v, %t, %v)", last, available, err)
}
if err := last.Complete(context.Background(), 1); err != nil {
t.Fatalf("last Complete(): %v", err)
}
exhausted, available, err := session.AcquireFetch(context.Background(), 1)
if err != nil || available || exhausted != nil {
t.Fatalf("exhausted AcquireFetch() = (%v, %t, %v)", exhausted, available, err)
}
cancel()
waitRunner(t, done)
}
func TestRedisExpiredFetchReservationIsConservativelyCharged(t *testing.T) {
fixture := newRedisFixture(t)
coordinator, err := New(fixture.client, Options{
Namespace: fixture.namespace, HolderID: "controller-a", LeaseTTL: 600 * time.Millisecond,
RenewEvery: 150 * time.Millisecond, RetryInterval: 10 * time.Millisecond, PermitGrace: 10 * time.Millisecond,
})
if err != nil {
t.Fatalf("New(): %v", err)
}
limits := controllerProvider.CoordinationLimits{
MaxInFlight: 1, MaxAttemptDuration: 40 * time.Millisecond, MaxTotal: 1,
}
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
sessions := make(chan controllerProvider.LeaderSession, 1)
done := make(chan error, 1)
go func() {
done <- coordinator.RunLeader(ctx, "provider-a", limits,
func(workCtx context.Context, session controllerProvider.LeaderSession) error {
sessions <- session
<-workCtx.Done()
return nil
})
}()
session := receiveSession(t, sessions)
abandoned, available, err := session.AcquireFetch(context.Background(), 1)
if err != nil || !available || abandoned == nil {
t.Fatalf("abandoned AcquireFetch() = (%v, %t, %v)", abandoned, available, err)
}
time.Sleep(80 * time.Millisecond)
exhausted, available, err := session.AcquireFetch(context.Background(), 1)
if err != nil || available || exhausted != nil {
t.Fatalf("post-expiry AcquireFetch() = (%v, %t, %v), want charged quota", exhausted, available, err)
}
cancel()
waitRunner(t, done)
@ -281,6 +446,7 @@ func receivePermit(t *testing.T, values <-chan permitResultFixture) permitResult
type permitResultFixture struct {
permit controllerProvider.RequestPermit
available bool
err error
}

13
internal/adapters/redisprovider/keys.go
View File

@ -19,6 +19,10 @@ type upstreamKeys struct {
leader string
next string
inflight string
fetchedTotal string
pendingTotal string
permits string
permitExpiry string
}
func (builder keyBuilder) forUpstream(upstreamID string) (upstreamKeys, error) {
@ -33,11 +37,18 @@ func (builder keyBuilder) forUpstream(upstreamID string) (upstreamKeys, error) {
leader: prefix + ":leader",
next: prefix + ":next-request",
inflight: prefix + ":inflight",
fetchedTotal: prefix + ":fetched-total",
pendingTotal: prefix + ":pending-total",
permits: prefix + ":permits",
permitExpiry: prefix + ":permit-expiry",
}, nil
}
func (keys upstreamKeys) all() []string {
return []string{keys.generation, keys.epoch, keys.leader, keys.next, keys.inflight}
return []string{
keys.generation, keys.epoch, keys.leader, keys.next, keys.inflight,
keys.fetchedTotal, keys.pendingTotal, keys.permits, keys.permitExpiry,
}
}
func digestParts(values ...string) string {

139
internal/adapters/redisprovider/scripts/provider.lua
View File

@ -1,4 +1,5 @@
local operation = ARGV[1]
local max_safe_integer = 9007199254740991
local function now_ms()
local value = redis.call('TIME')
@ -33,6 +34,63 @@ local function same_leader(value, generation, holder_id, token, epoch)
value.token == token and value.epoch == tostring(epoch)
end
local function read_counter(key)
local encoded = redis.call('GET', key)
if not encoded then
return 0, nil
end
local value = tonumber(encoded)
if not value or value < 0 or value ~= math.floor(value) then
return nil, 'invalid'
end
return value, nil
end
local function read_permit(token)
local encoded = redis.call('HGET', KEYS[8], token)
if not encoded then
return nil, nil
end
local ok, value = pcall(cjson.decode, encoded)
if not ok or type(value) ~= 'table' or type(value.state) ~= 'string' or
type(value.expected) ~= 'number' or value.expected <= 0 or
value.expected ~= math.floor(value.expected) then
return nil, 'invalid'
end
return value, nil
end
local function cleanup_expired(now)
local expired = redis.call('ZRANGEBYSCORE', KEYS[9], '-inf', now, 'LIMIT', 0, 256)
if #expired == 0 then
return nil
end
local fetched, fetched_error = read_counter(KEYS[6])
local pending, pending_error = read_counter(KEYS[7])
if fetched_error or pending_error then
return 'invalid'
end
for _, permit_token in ipairs(expired) do
local permit, permit_error = read_permit(permit_token)
if permit_error then
return 'invalid'
end
if permit and permit.state == 'reserved' then
if pending < permit.expected or fetched > max_safe_integer - permit.expected then
return 'invalid'
end
pending = pending - permit.expected
fetched = fetched + permit.expected
end
redis.call('HDEL', KEYS[8], permit_token)
redis.call('ZREM', KEYS[5], permit_token)
redis.call('ZREM', KEYS[9], permit_token)
end
redis.call('SET', KEYS[6], fetched)
redis.call('SET', KEYS[7], pending)
return nil
end
if operation == 'acquire_leader' then
local generation_candidate = ARGV[2]
local holder_id = ARGV[3]
@ -41,7 +99,10 @@ if operation == 'acquire_leader' then
if not lease_ttl or lease_ttl <= 0 then
return reply('invalid', '', 0, 0)
end
redis.call('SET', KEYS[1], generation_candidate, 'NX')
local created = redis.call('SET', KEYS[1], generation_candidate, 'NX')
if created then
redis.call('DEL', KEYS[2], KEYS[3], KEYS[4], KEYS[5], KEYS[6], KEYS[7], KEYS[8], KEYS[9])
end
local generation = redis.call('GET', KEYS[1])
local current, current_error = read_leader()
if current_error then
@ -112,6 +173,13 @@ if operation == 'acquire_fetch' then
local request_interval = tonumber(ARGV[7])
local max_in_flight = tonumber(ARGV[8])
local permit_ttl = tonumber(ARGV[9])
local expected = tonumber(ARGV[10])
local max_total = tonumber(ARGV[11])
if not expected or expected <= 0 or expected ~= math.floor(expected) or
expected > max_safe_integer or not max_total or max_total < 0 or
max_total > max_safe_integer or max_total ~= math.floor(max_total) then
return reply('invalid', generation, epoch or 0, 0)
end
local current, current_error = read_leader()
if current_error then
return reply('unavailable', generation, epoch or 0, 0)
@ -120,11 +188,27 @@ if operation == 'acquire_fetch' then
return reply('stale', generation, epoch or 0, 0)
end
local now = now_ms()
redis.call('ZREMRANGEBYSCORE', KEYS[5], '-inf', now)
local existing = redis.call('ZSCORE', KEYS[5], permit_token)
if existing then
if cleanup_expired(now) then
return reply('unavailable', generation, epoch, 0)
end
local existing, existing_error = read_permit(permit_token)
if existing_error then
return reply('unavailable', generation, epoch, 0)
end
if existing and existing.state == 'reserved' then
return reply('ok', generation, epoch, 0)
end
if existing then
return reply('unavailable', generation, epoch, 0)
end
local fetched, fetched_error = read_counter(KEYS[6])
local pending, pending_error = read_counter(KEYS[7])
if fetched_error or pending_error then
return reply('unavailable', generation, epoch, 0)
end
if max_total > 0 and fetched + pending + expected > max_total then
return reply('quota_exhausted', generation, epoch, 0)
end
local next_request = redis.call('GET', KEYS[4])
if next_request and not tonumber(next_request) then
return reply('unavailable', generation, epoch, 0)
@ -141,7 +225,13 @@ if operation == 'acquire_fetch' then
return reply('at_capacity', generation, epoch, wait_ms)
end
redis.call('ZADD', KEYS[5], now + permit_ttl, permit_token)
redis.call('PEXPIRE', KEYS[5], permit_ttl + 1000)
redis.call('ZADD', KEYS[9], now + permit_ttl, permit_token)
redis.call('HSET', KEYS[8], permit_token, cjson.encode({
version = 1,
state = 'reserved',
expected = expected
}))
redis.call('SET', KEYS[7], pending + expected)
if request_interval > 0 then
redis.call('SET', KEYS[4], now + request_interval, 'PX', request_interval)
else
@ -150,8 +240,43 @@ if operation == 'acquire_fetch' then
return reply('ok', generation, epoch, 0)
end
if operation == 'release_fetch' then
redis.call('ZREM', KEYS[5], ARGV[2])
if operation == 'complete_fetch' or operation == 'cancel_fetch' then
local permit_token = ARGV[2]
local fetched_count = tonumber(ARGV[3])
local settlement_ttl = tonumber(ARGV[4])
if not fetched_count or fetched_count < 0 or fetched_count ~= math.floor(fetched_count) or
fetched_count > max_safe_integer or not settlement_ttl or settlement_ttl <= 0 then
return reply('invalid', '', 0, 0)
end
local now = now_ms()
if cleanup_expired(now) then
return reply('unavailable', '', 0, 0)
end
local permit, permit_error = read_permit(permit_token)
if permit_error then
return reply('unavailable', '', 0, 0)
end
if not permit or permit.state ~= 'reserved' then
return reply('ok', '', 0, 0)
end
local fetched, fetched_error = read_counter(KEYS[6])
local pending, pending_error = read_counter(KEYS[7])
if fetched_error or pending_error or pending < permit.expected or
fetched > max_safe_integer - fetched_count then
return reply('unavailable', '', 0, 0)
end
pending = pending - permit.expected
if operation == 'complete_fetch' then
fetched = fetched + fetched_count
permit.state = 'completed'
else
permit.state = 'cancelled'
end
redis.call('SET', KEYS[6], fetched)
redis.call('SET', KEYS[7], pending)
redis.call('HSET', KEYS[8], permit_token, cjson.encode(permit))
redis.call('ZREM', KEYS[5], permit_token)
redis.call('ZADD', KEYS[9], now + settlement_ttl, permit_token)
return reply('ok', '', 0, 0)
end

6
internal/config/config.go
View File

@ -5,6 +5,12 @@ import (
"time"
)
const (
MaximumPoolSize = 1_000_000
MaximumExactCounter = int64(1<<53 - 1)
MaximumUpstreams = 4_096
)
type Duration time.Duration
func (d *Duration) UnmarshalText(text []byte) error {

60
internal/config/config_test.go
View File

@ -4,6 +4,7 @@ import (
"fmt"
"os"
"path/filepath"
"strconv"
"strings"
"testing"
"time"
@ -553,6 +554,13 @@ func TestValidateRejectsInvalidConfigurationMatrix(t *testing.T) {
},
want: "estimatedIPsPerCall",
},
{
name: "pool exceeds runtime scan bound",
mutate: func(cfg *Config) {
updateUpstream(cfg, func(upstream *Upstream) { upstream.Pool.MaxSize = MaximumPoolSize + 1 })
},
want: "pool.maxSize",
},
{
name: "zero refill interval",
mutate: func(cfg *Config) {
@ -655,6 +663,58 @@ func TestValidateRejectsInvalidConfigurationMatrix(t *testing.T) {
}
}
func TestValidateRejectsUnboundedUpstreamCardinality(t *testing.T) {
cfg := mustLoadValidConfig(t)
for index := len(cfg.Upstreams); index <= MaximumUpstreams; index++ {
cfg.Upstreams[fmt.Sprintf("disabled-%d", index)] = Upstream{}
}
if err := Validate(cfg); err == nil || !strings.Contains(err.Error(), "upstream count") {
t.Fatalf("Validate(too many upstreams) error = %v", err)
}
}
func TestValidateRejectsCountersOutsideRedisExactRange(t *testing.T) {
if strconv.IntSize < 64 {
t.Skip("64-bit int is required for values above the Redis exact range")
}
tests := []struct {
name string
mutate func(*Upstream)
}{
{name: "fetch total", mutate: func(upstream *Upstream) {
upstream.Fetch.MaxTotal = int(MaximumExactCounter) + 1
}},
{name: "proxy concurrency", mutate: func(upstream *Upstream) {
upstream.Capacity.MaxConcurrencyPerProxy = int(MaximumExactCounter) + 1
}},
{name: "minimum slots", mutate: func(upstream *Upstream) {
upstream.Refill.MinimumAvailableSlots = MaximumExactCounter + 1
upstream.Refill.TargetAvailableSlots = MaximumExactCounter + 2
}},
{name: "target slots", mutate: func(upstream *Upstream) {
upstream.Capacity.MaxConcurrencyPerProxy = int(MaximumExactCounter)
upstream.Pool.MaxSize = 1
upstream.Refill.MinimumAvailableSlots = MaximumExactCounter
upstream.Refill.TargetAvailableSlots = MaximumExactCounter + 1
}},
{name: "theoretical slots", mutate: func(upstream *Upstream) {
upstream.Pool.MaxSize = 2
upstream.Capacity.MaxConcurrencyPerProxy = int(MaximumExactCounter/2 + 1)
upstream.Refill.MinimumAvailableSlots = 1
upstream.Refill.TargetAvailableSlots = 2
}},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
cfg := mustLoadValidConfig(t)
updateUpstream(cfg, test.mutate)
if err := Validate(cfg); err == nil || !strings.Contains(err.Error(), "exact counter range") {
t.Fatalf("Validate(inexact counter) error = %v", err)
}
})
}
}
func TestResolvedConfigFormattingRedactsSecrets(t *testing.T) {
configured := strings.Replace(validConfig, ` auth:
mode: none`, ` auth:

30
internal/config/fingerprint.go Normal file
View File

@ -0,0 +1,30 @@
package config
import (
"crypto/hmac"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
)
const MinimumFingerprintKeyBytes = 32
var ErrInvalidFingerprint = errors.New("invalid configuration fingerprint")
// Fingerprint returns a keyed digest of one fully resolved configuration.
// Only the digest is persisted; the resolved configuration and key remain local.
func Fingerprint(configuration *Config, key []byte) (string, error) {
if configuration == nil || len(key) < MinimumFingerprintKeyBytes {
return "", ErrInvalidFingerprint
}
encoded, err := json.Marshal(configuration)
if err != nil {
return "", errors.Join(ErrInvalidFingerprint, err)
}
digest := hmac.New(sha256.New, key)
if _, err := digest.Write(encoded); err != nil {
return "", errors.Join(ErrInvalidFingerprint, err)
}
return hex.EncodeToString(digest.Sum(nil)), nil
}

68
internal/config/fingerprint_test.go Normal file
View File

@ -0,0 +1,68 @@
package config
import "testing"
var testFingerprintKey = []byte("0123456789abcdef0123456789abcdef")
func TestFingerprintIsStableAndTracksSecretRotation(t *testing.T) {
first := storeTestConfig("provider-a")
upstream := first.Upstreams["provider-a"]
upstream.ProxyAuth.Password = "secret-a"
first.Upstreams["provider-a"] = upstream
stable, err := Fingerprint(first, testFingerprintKey)
if err != nil {
t.Fatalf("Fingerprint(first): %v", err)
}
again, err := Fingerprint(first, testFingerprintKey)
if err != nil || again != stable {
t.Fatalf("Fingerprint(stable) = %q, %v; want %q", again, err, stable)
}
rotated := storeTestConfig("provider-a")
upstream = rotated.Upstreams["provider-a"]
upstream.ProxyAuth.Password = "secret-b"
rotated.Upstreams["provider-a"] = upstream
changed, err := Fingerprint(rotated, testFingerprintKey)
if err != nil {
t.Fatalf("Fingerprint(rotated): %v", err)
}
if changed == stable {
t.Fatal("Fingerprint did not change after secret rotation")
}
if len(changed) != 64 {
t.Fatalf("Fingerprint length = %d, want 64", len(changed))
}
}
func TestFingerprintChangesWithIndependentKey(t *testing.T) {
configuration := storeTestConfig("provider-a")
first, err := Fingerprint(configuration, testFingerprintKey)
if err != nil {
t.Fatalf("Fingerprint(first key): %v", err)
}
second, err := Fingerprint(configuration, []byte("fedcba9876543210fedcba9876543210"))
if err != nil {
t.Fatalf("Fingerprint(second key): %v", err)
}
if first == second {
t.Fatal("Fingerprint did not change with independent key")
}
}
func TestFingerprintRejectsInvalidInputs(t *testing.T) {
for name, test := range map[string]struct {
configuration *Config
key []byte
}{
"nil configuration": {key: testFingerprintKey},
"missing key": {configuration: storeTestConfig("provider-a")},
"short key": {configuration: storeTestConfig("provider-a"), key: []byte("too-short")},
} {
t.Run(name, func(t *testing.T) {
if _, err := Fingerprint(test.configuration, test.key); err == nil {
t.Fatal("Fingerprint() succeeded")
}
})
}
}

46
internal/config/store.go
View File

@ -9,7 +9,12 @@ var ErrInvalidStore = errors.New("invalid configuration store")
// Store publishes complete validated configurations with one atomic pointer swap.
type Store struct {
current atomic.Pointer[Config]
current atomic.Pointer[publishedConfiguration]
}
type publishedConfiguration struct {
value Config
revision uint64
}
func NewStore(initial *Config) (*Store, error) {
@ -17,7 +22,7 @@ func NewStore(initial *Config) (*Store, error) {
return nil, errors.Join(ErrInvalidStore, err)
}
store := &Store{}
store.Publish(initial)
store.current.Store(&publishedConfiguration{value: cloneConfig(*initial)})
return store, nil
}
@ -25,19 +30,38 @@ func (store *Store) Current() *Config {
if store == nil {
return nil
}
current := store.current.Load()
if current == nil {
published := store.current.Load()
if published == nil {
return nil
}
cloned := cloneConfig(*current)
cloned := cloneConfig(published.value)
return &cloned
}
// Publish accepts a non-nil configuration already validated by the caller.
func (store *Store) Publish(configuration *Config) {
if store == nil || configuration == nil {
return
func (store *Store) Revision() uint64 {
if store == nil {
return 0
}
published := store.current.Load()
if published == nil {
return 0
}
return published.revision
}
// PublishRevision publishes only a strictly newer authoritative revision.
func (store *Store) PublishRevision(configuration *Config, revision uint64) bool {
if store == nil || configuration == nil || revision == 0 {
return false
}
for {
current := store.current.Load()
if current != nil && revision <= current.revision {
return false
}
next := &publishedConfiguration{value: cloneConfig(*configuration), revision: revision}
if store.current.CompareAndSwap(current, next) {
return true
}
}
cloned := cloneConfig(*configuration)
store.current.Store(&cloned)
}

29
internal/config/store_test.go
View File

@ -26,7 +26,9 @@ func TestStorePublishesAndReturnsDetachedConfigurations(t *testing.T) {
}
next := storeTestConfig("provider-b")
store.Publish(next)
if !store.PublishRevision(next, 1) {
t.Fatal("PublishRevision() rejected newer configuration")
}
next.Routing[0].Upstreams[0] = "mutated"
if got := store.Current().Routing[0].Upstreams[0]; got != "provider-b" {
t.Fatalf("Publish() retained caller state: %q", got)
@ -56,7 +58,7 @@ func TestStoreSupportsConcurrentReadersAndPublishers(t *testing.T) {
if index%2 == 1 {
name = "provider-b"
}
store.Publish(storeTestConfig(name))
store.PublishRevision(storeTestConfig(name), uint64(index+1))
}(index)
go func() {
defer wait.Done()
@ -69,6 +71,29 @@ func TestStoreSupportsConcurrentReadersAndPublishers(t *testing.T) {
wait.Wait()
}
func TestStoreRejectsOutOfOrderRevisionPublication(t *testing.T) {
t.Parallel()
store, err := NewStore(storeTestConfig("provider-a"))
if err != nil {
t.Fatalf("NewStore() error = %v", err)
}
if published := store.PublishRevision(storeTestConfig("provider-b"), 2); !published {
t.Fatal("PublishRevision(newer) rejected")
}
if published := store.PublishRevision(storeTestConfig("provider-c"), 1); published {
t.Fatal("PublishRevision(stale) succeeded")
}
if published := store.PublishRevision(storeTestConfig("provider-c"), 2); published {
t.Fatal("PublishRevision(equal) succeeded")
}
if got := store.Current().Routing[0].Upstreams[0]; got != "provider-b" {
t.Fatalf("Current() upstream = %q, want provider-b", got)
}
if got := store.Revision(); got != 2 {
t.Fatalf("Revision() = %d, want 2", got)
}
}
func storeTestConfig(upstreamName string) *Config {
return &Config{
Version: 1,

24
internal/config/validate.go
View File

@ -46,6 +46,9 @@ func Validate(cfg *Config) error {
if err := validateCheck("defaults.check", cfg.Defaults.Check); err != nil {
return err
}
if len(cfg.Upstreams) > MaximumUpstreams {
return fmt.Errorf("validate configuration: upstream count exceeds %d", MaximumUpstreams)
}
enabledUpstreams := 0
for name, upstream := range cfg.Upstreams {
if upstream.Enabled {
@ -330,28 +333,46 @@ func validateUpstream(name string, upstream Upstream) error {
if err := requirePositive(scope+" pool.maxSize", upstream.Pool.MaxSize); err != nil {
return err
}
if upstream.Pool.MaxSize > MaximumPoolSize {
return fmt.Errorf("validate %s pool.maxSize: exceeds %d", scope, MaximumPoolSize)
}
if err := requireNonNegative(scope+" fetch.maxTotal", upstream.Fetch.MaxTotal); err != nil {
return err
}
if int64(upstream.Fetch.MaxTotal) > MaximumExactCounter {
return fmt.Errorf("validate %s fetch.maxTotal: exceeds exact counter range", scope)
}
if upstream.Fetch.MaxTotal > 0 && upstream.Fetch.MaxTotal < upstream.Pool.MaxSize {
return fmt.Errorf("validate %s fetch.maxTotal: cannot be lower than pool.maxSize", scope)
}
if err := requirePositive(scope+" capacity.maxConcurrencyPerProxy", upstream.Capacity.MaxConcurrencyPerProxy); err != nil {
return err
}
if int64(upstream.Capacity.MaxConcurrencyPerProxy) > MaximumExactCounter {
return fmt.Errorf("validate %s capacity.maxConcurrencyPerProxy: exceeds exact counter range", scope)
}
if err := requirePositive(scope+" refill.reconcileInterval", upstream.Refill.ReconcileInterval); err != nil {
return err
}
if err := requirePositive(scope+" refill.minimumAvailableSlots", upstream.Refill.MinimumAvailableSlots); err != nil {
return err
}
if upstream.Refill.MinimumAvailableSlots > MaximumExactCounter {
return fmt.Errorf("validate %s refill.minimumAvailableSlots: exceeds exact counter range", scope)
}
if upstream.Refill.TargetAvailableSlots <= upstream.Refill.MinimumAvailableSlots {
return fmt.Errorf("validate %s refill.targetAvailableSlots: must be greater than minimumAvailableSlots", scope)
}
if upstream.Refill.TargetAvailableSlots > MaximumExactCounter {
return fmt.Errorf("validate %s refill.targetAvailableSlots: exceeds exact counter range", scope)
}
if int64(upstream.Pool.MaxSize) > math.MaxInt64/int64(upstream.Capacity.MaxConcurrencyPerProxy) {
return fmt.Errorf("validate %s refill.targetAvailableSlots: theoretical capacity overflows int64", scope)
}
theoreticalSlots := int64(upstream.Pool.MaxSize) * int64(upstream.Capacity.MaxConcurrencyPerProxy)
if theoreticalSlots > MaximumExactCounter {
return fmt.Errorf("validate %s refill.targetAvailableSlots: theoretical capacity exceeds exact counter range", scope)
}
if upstream.Refill.TargetAvailableSlots > theoreticalSlots {
return fmt.Errorf("validate %s refill.targetAvailableSlots: exceeds theoretical capacity", scope)
}
@ -415,6 +436,9 @@ func validateFetch(scope string, fetch Fetch) error {
if err := requirePositive(scope+".maxInFlight", fetch.MaxInFlight); err != nil {
return err
}
if int64(fetch.MaxInFlight) > MaximumExactCounter {
return fmt.Errorf("validate %s.maxInFlight: exceeds exact counter range", scope)
}
if err := requireNonNegative(scope+".maxTotal", fetch.MaxTotal); err != nil {
return err
}

48
internal/controller/admin/service.go
View File

@ -2,9 +2,6 @@ package admin
import (
"context"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
"sort"
"strings"
@ -29,9 +26,15 @@ type ConfigurationLoader interface {
LoadConfiguration(context.Context) (LoadedConfiguration, error)
}
// ConfigurationPublisher must atomically publish an already validated configuration.
// ConfigurationPublisher publishes only a newer authoritative configuration revision.
type ConfigurationPublisher interface {
Publish(*config.Config)
PublishRevision(*config.Config, uint64) bool
}
type RuntimeController interface {
Notify()
ValidateConfiguration(context.Context, *config.Config) error
ValidateUpstream(context.Context, string) error
}
var _ ConfigurationPublisher = (*config.Store)(nil)
@ -41,10 +44,12 @@ type ApplicationDependencies struct {
Operations OperationalStatusReader
Configuration ConfigurationLoader
Publisher ConfigurationPublisher
Runtime RuntimeController
}
type ApplicationOptions struct {
Now func() time.Time
FingerprintKey []byte
}
type OperationalStatus struct {
@ -74,14 +79,17 @@ type ApplicationService struct {
operations OperationalStatusReader
configuration ConfigurationLoader
publisher ConfigurationPublisher
runtime RuntimeController
now func() time.Time
fingerprintKey []byte
}
var _ Service = (*ApplicationService)(nil)
func NewApplicationService(dependencies ApplicationDependencies, options ApplicationOptions) (*ApplicationService, error) {
if nilInterface(dependencies.State) || nilInterface(dependencies.Operations) || nilInterface(dependencies.Configuration) ||
nilInterface(dependencies.Publisher) || options.Now == nil {
nilInterface(dependencies.Publisher) || options.Now == nil ||
len(options.FingerprintKey) < config.MinimumFingerprintKeyBytes {
return nil, ErrInvalidApplicationService
}
return &ApplicationService{
@ -89,11 +97,18 @@ func NewApplicationService(dependencies ApplicationDependencies, options Applica
operations: dependencies.Operations,
configuration: dependencies.Configuration,
publisher: dependencies.Publisher,
runtime: dependencies.Runtime,
now: options.Now,
fingerprintKey: append([]byte(nil), options.FingerprintKey...),
}, nil
}
func (service *ApplicationService) SetUpstreamEnabled(ctx context.Context, command SetUpstreamCommand) (MutationResult, error) {
if command.Enabled && service.runtime != nil {
if err := service.runtime.ValidateUpstream(ctx, command.Name); err != nil {
return MutationResult{RequestID: command.RequestID}, errors.Join(ErrInvalidConfiguration, err)
}
}
result, err := service.state.SetUpstreamEnabled(ctx, adminstate.SetUpstreamCommand{
RequestID: command.RequestID,
Actor: adminstate.Actor{ID: command.ActorID, SourceIP: command.SourceIP},
@ -101,6 +116,9 @@ func (service *ApplicationService) SetUpstreamEnabled(ctx context.Context, comma
Name: command.Name,
Enabled: command.Enabled,
})
if err == nil && service.runtime != nil {
service.runtime.Notify()
}
return mutationResult(result), mapAdminStateError(err)
}
@ -202,14 +220,16 @@ func (service *ApplicationService) ApplyConfiguration(
if err := config.Validate(loaded.Value); err != nil {
return MutationResult{RequestID: command.RequestID}, errors.Join(ErrInvalidConfiguration, err)
}
if service.runtime != nil {
if err := service.runtime.ValidateConfiguration(ctx, loaded.Value); err != nil {
return MutationResult{RequestID: command.RequestID}, errors.Join(ErrInvalidConfiguration, err)
}
}
managementView := loaded.Value.Redacted()
encoded, err := json.Marshal(managementView)
checksum, err := config.Fingerprint(loaded.Value, service.fingerprintKey)
if err != nil {
return MutationResult{RequestID: command.RequestID}, errors.Join(ErrInvalidConfiguration, err)
}
digest := sha256.Sum256(encoded)
checksum := hex.EncodeToString(digest[:])
current, err := service.state.Snapshot(ctx)
if err != nil {
@ -229,7 +249,13 @@ func (service *ApplicationService) ApplyConfiguration(
if err != nil {
return mutationResult(result), mapAdminStateError(err)
}
service.publisher.Publish(loaded.Value)
if result.Revision == 0 {
return mutationResult(result), errors.Join(ErrUnavailable, ErrInvalidApplicationService)
}
published := service.publisher.PublishRevision(loaded.Value, result.Revision)
if published && service.runtime != nil {
service.runtime.Notify()
}
return mutationResult(result), nil
}

215
internal/controller/admin/service_test.go
View File

@ -3,6 +3,7 @@ package admin
import (
"context"
"errors"
"sync/atomic"
"testing"
"time"
@ -10,6 +11,12 @@ import (
"proxy-pool/internal/domain/adminstate"
)
var applicationTestFingerprintKey = []byte("0123456789abcdef0123456789abcdef")
func applicationTestOptions(now func() time.Time) ApplicationOptions {
return ApplicationOptions{Now: now, FingerprintKey: applicationTestFingerprintKey}
}
func TestApplicationServiceMapsUpstreamMutationToAdminState(t *testing.T) {
t.Parallel()
now := time.Date(2026, 7, 29, 10, 0, 0, 0, time.UTC)
@ -21,12 +28,14 @@ func TestApplicationServiceMapsUpstreamMutationToAdminState(t *testing.T) {
Message: "enabled",
},
}
runtime := &recordingRuntimeNotifier{}
service, err := NewApplicationService(ApplicationDependencies{
State: state,
Operations: staticOperationalStatusReader{},
Configuration: staticConfigurationLoader{},
Publisher: &recordingConfigurationPublisher{},
}, ApplicationOptions{Now: func() time.Time { return now }})
Runtime: runtime,
}, applicationTestOptions(func() time.Time { return now }))
if err != nil {
t.Fatalf("NewApplicationService() error = %v", err)
}
@ -53,6 +62,34 @@ func TestApplicationServiceMapsUpstreamMutationToAdminState(t *testing.T) {
}) {
t.Fatalf("admin state command = %+v", state.lastUpstream)
}
if runtime.notifications != 1 {
t.Fatalf("runtime notifications = %d, want 1", runtime.notifications)
}
}
func TestApplicationServicePreflightsProviderRuntimeBeforeMutation(t *testing.T) {
t.Parallel()
wantErr := errors.New("invalid Provider template")
state := &recordingAdminState{}
runtime := &recordingRuntimeNotifier{validationErr: wantErr}
service, err := NewApplicationService(ApplicationDependencies{
State: state, Operations: staticOperationalStatusReader{},
Configuration: staticConfigurationLoader{}, Publisher: &recordingConfigurationPublisher{},
Runtime: runtime,
}, applicationTestOptions(time.Now))
if err != nil {
t.Fatalf("NewApplicationService(): %v", err)
}
_, err = service.SetUpstreamEnabled(context.Background(), SetUpstreamCommand{
RequestID: "req-enable", Name: "provider-a", Enabled: true,
})
if !errors.Is(err, ErrInvalidConfiguration) || !errors.Is(err, wantErr) {
t.Fatalf("SetUpstreamEnabled() error = %v", err)
}
if state.lastUpstream != (adminstate.SetUpstreamCommand{}) {
t.Fatalf("state mutated before runtime preflight: %+v", state.lastUpstream)
}
}
func TestNewApplicationServiceRejectsMissingDependencies(t *testing.T) {
@ -68,11 +105,12 @@ func TestNewApplicationServiceRejectsMissingDependencies(t *testing.T) {
dependencies ApplicationDependencies
options ApplicationOptions
}{
{name: "state", dependencies: func() ApplicationDependencies { value := valid; value.State = nil; return value }(), options: ApplicationOptions{Now: time.Now}},
{name: "operations", dependencies: func() ApplicationDependencies { value := valid; value.Operations = nil; return value }(), options: ApplicationOptions{Now: time.Now}},
{name: "configuration", dependencies: func() ApplicationDependencies { value := valid; value.Configuration = nil; return value }(), options: ApplicationOptions{Now: time.Now}},
{name: "publisher", dependencies: func() ApplicationDependencies { value := valid; value.Publisher = nil; return value }(), options: ApplicationOptions{Now: time.Now}},
{name: "state", dependencies: func() ApplicationDependencies { value := valid; value.State = nil; return value }(), options: applicationTestOptions(time.Now)},
{name: "operations", dependencies: func() ApplicationDependencies { value := valid; value.Operations = nil; return value }(), options: applicationTestOptions(time.Now)},
{name: "configuration", dependencies: func() ApplicationDependencies { value := valid; value.Configuration = nil; return value }(), options: applicationTestOptions(time.Now)},
{name: "publisher", dependencies: func() ApplicationDependencies { value := valid; value.Publisher = nil; return value }(), options: applicationTestOptions(time.Now)},
{name: "clock", dependencies: valid},
{name: "fingerprint key", dependencies: valid, options: ApplicationOptions{Now: time.Now}},
}
for _, test := range tests {
test := test
@ -99,7 +137,7 @@ func TestNewApplicationServiceRejectsTypedNilDependencies(t *testing.T) {
func() ApplicationDependencies { value := valid; value.State = state; return value }(),
func() ApplicationDependencies { value := valid; value.Publisher = publisher; return value }(),
} {
if _, err := NewApplicationService(dependencies, ApplicationOptions{Now: time.Now}); !errors.Is(err, ErrInvalidApplicationService) {
if _, err := NewApplicationService(dependencies, applicationTestOptions(time.Now)); !errors.Is(err, ErrInvalidApplicationService) {
t.Fatalf("NewApplicationService(typed nil) error = %v, want %v", err, ErrInvalidApplicationService)
}
}
@ -109,7 +147,7 @@ func TestApplicationServiceMapsRoutingSwitchAndDomainErrors(t *testing.T) {
t.Parallel()
now := time.Date(2026, 7, 29, 11, 0, 0, 0, time.FixedZone("test", 8*60*60))
state := &recordingAdminState{mutation: adminstate.MutationResult{RequestID: "req-switch", Changed: true, Revision: 21}}
service := mustApplicationService(t, state, ApplicationOptions{Now: func() time.Time { return now }})
service := mustApplicationService(t, state, applicationTestOptions(func() time.Time { return now }))
result, err := service.SwitchRouting(context.Background(), SwitchCommand{
RequestID: "req-switch", ActorID: "admin:bob", SourceIP: "198.51.100.7",
@ -174,7 +212,7 @@ func TestApplicationServiceBuildsStatusFromAuthoritativeAndOperationalSnapshots(
service, err := NewApplicationService(ApplicationDependencies{
State: state, Operations: operations, Configuration: staticConfigurationLoader{},
Publisher: &recordingConfigurationPublisher{},
}, ApplicationOptions{Now: time.Now})
}, applicationTestOptions(time.Now))
if err != nil {
t.Fatalf("NewApplicationService() error = %v", err)
}
@ -205,7 +243,7 @@ func TestApplicationServiceMapsStatusDependencyFailures(t *testing.T) {
service, err := NewApplicationService(ApplicationDependencies{
State: &recordingAdminState{err: stateFailure}, Operations: staticOperationalStatusReader{},
Configuration: staticConfigurationLoader{}, Publisher: &recordingConfigurationPublisher{},
}, ApplicationOptions{Now: time.Now})
}, applicationTestOptions(time.Now))
if err != nil {
t.Fatalf("NewApplicationService() error = %v", err)
}
@ -217,7 +255,7 @@ func TestApplicationServiceMapsStatusDependencyFailures(t *testing.T) {
service, err = NewApplicationService(ApplicationDependencies{
State: &recordingAdminState{}, Operations: staticOperationalStatusReader{err: operationsFailure},
Configuration: staticConfigurationLoader{}, Publisher: &recordingConfigurationPublisher{},
}, ApplicationOptions{Now: time.Now})
}, applicationTestOptions(time.Now))
if err != nil {
t.Fatalf("NewApplicationService() error = %v", err)
}
@ -228,7 +266,7 @@ func TestApplicationServiceMapsStatusDependencyFailures(t *testing.T) {
service, err = NewApplicationService(ApplicationDependencies{
State: &recordingAdminState{}, Operations: staticOperationalStatusReader{err: context.Canceled},
Configuration: staticConfigurationLoader{}, Publisher: &recordingConfigurationPublisher{},
}, ApplicationOptions{Now: time.Now})
}, applicationTestOptions(time.Now))
if err != nil {
t.Fatalf("NewApplicationService() error = %v", err)
}
@ -242,6 +280,7 @@ func TestApplicationServiceReloadPersistsManagementViewBeforePublishing(t *testi
now := time.Date(2026, 7, 29, 12, 0, 0, 0, time.UTC)
configuration := validReloadConfiguration()
publisher := &recordingConfigurationPublisher{}
runtime := &recordingRuntimeNotifier{}
state := &recordingAdminState{
mutation: adminstate.MutationResult{RequestID: "req-reload", Changed: true, Revision: 42},
snapshot: adminstate.Snapshot{Routings: []adminstate.RoutingState{
@ -258,8 +297,8 @@ func TestApplicationServiceReloadPersistsManagementViewBeforePublishing(t *testi
Configuration: staticConfigurationLoader{loaded: LoadedConfiguration{
Value: configuration, Source: "configs/proxy-pool.yaml",
}},
Publisher: publisher,
}, ApplicationOptions{Now: func() time.Time { return now }})
Publisher: publisher, Runtime: runtime,
}, applicationTestOptions(func() time.Time { return now }))
if err != nil {
t.Fatalf("NewApplicationService() error = %v", err)
}
@ -276,6 +315,9 @@ func TestApplicationServiceReloadPersistsManagementViewBeforePublishing(t *testi
if len(publisher.published) != 1 || publisher.published[0] != configuration {
t.Fatalf("published configurations = %+v", publisher.published)
}
if runtime.notifications != 1 {
t.Fatalf("runtime notifications = %d, want 1", runtime.notifications)
}
command := state.lastConfig
if command.RequestID != "req-reload" || command.Actor != (adminstate.Actor{ID: "admin:alice", SourceIP: "192.0.2.10"}) ||
!command.OccurredAt.Equal(now) || command.Source != "configs/proxy-pool.yaml" {
@ -306,7 +348,7 @@ func TestApplicationServiceApplyConfigurationUsesProvidedSnapshotWithoutReloadin
Operations: staticOperationalStatusReader{},
Configuration: forbiddenConfigurationLoader{},
Publisher: publisher,
}, ApplicationOptions{Now: func() time.Time { return now }})
}, applicationTestOptions(func() time.Time { return now }))
if err != nil {
t.Fatalf("NewApplicationService() error = %v", err)
}
@ -355,7 +397,7 @@ func TestApplicationServiceReloadDoesNotPublishInvalidOrUncommittedConfiguration
service, err := NewApplicationService(ApplicationDependencies{
State: state, Operations: staticOperationalStatusReader{},
Configuration: staticConfigurationLoader{loaded: test.loaded}, Publisher: publisher,
}, ApplicationOptions{Now: time.Now})
}, applicationTestOptions(time.Now))
if err != nil {
t.Fatalf("NewApplicationService() error = %v", err)
}
@ -381,7 +423,7 @@ func TestApplicationServiceReloadPublishesSuccessfulReplay(t *testing.T) {
Configuration: staticConfigurationLoader{loaded: LoadedConfiguration{
Value: validReloadConfiguration(), Source: "config.yaml",
}}, Publisher: publisher,
}, ApplicationOptions{Now: time.Now})
}, applicationTestOptions(time.Now))
if err != nil {
t.Fatalf("NewApplicationService() error = %v", err)
}
@ -393,13 +435,82 @@ func TestApplicationServiceReloadPublishesSuccessfulReplay(t *testing.T) {
}
}
func TestApplicationServiceRejectsSuccessfulCommitWithoutRevision(t *testing.T) {
publisher := &recordingConfigurationPublisher{}
service, err := NewApplicationService(ApplicationDependencies{
State: &recordingAdminState{}, Operations: staticOperationalStatusReader{},
Configuration: staticConfigurationLoader{loaded: LoadedConfiguration{
Value: validReloadConfiguration(), Source: "config.yaml",
}}, Publisher: publisher,
}, applicationTestOptions(time.Now))
if err != nil {
t.Fatalf("NewApplicationService(): %v", err)
}
_, err = service.ReloadConfiguration(context.Background(), ReloadCommand{RequestID: "req-zero-revision"})
if !errors.Is(err, ErrUnavailable) {
t.Fatalf("ReloadConfiguration() error = %v, want unavailable", err)
}
if len(publisher.published) != 0 {
t.Fatalf("published configurations = %d, want 0", len(publisher.published))
}
}
func TestApplicationServiceKeepsNewestConfigurationWhenOlderCommitReturnsLater(t *testing.T) {
store, err := config.NewStore(configWithOnlyUpstream("provider-a"))
if err != nil {
t.Fatalf("config.NewStore(): %v", err)
}
state := &orderedCommitState{
firstCommitted: make(chan struct{}),
releaseFirst: make(chan struct{}),
}
service, err := NewApplicationService(ApplicationDependencies{
State: state, Operations: staticOperationalStatusReader{},
Configuration: staticConfigurationLoader{}, Publisher: store,
}, applicationTestOptions(time.Now))
if err != nil {
t.Fatalf("NewApplicationService(): %v", err)
}
firstDone := make(chan error, 1)
go func() {
_, applyErr := service.ApplyConfiguration(context.Background(), ReloadCommand{RequestID: "req-old"}, LoadedConfiguration{
Value: configWithOnlyUpstream("provider-b"), Source: "old.yaml",
})
firstDone <- applyErr
}()
select {
case <-state.firstCommitted:
case <-time.After(time.Second):
t.Fatal("first commit did not reach delayed return")
}
if _, err := service.ApplyConfiguration(context.Background(), ReloadCommand{RequestID: "req-new"}, LoadedConfiguration{
Value: configWithOnlyUpstream("provider-c"), Source: "new.yaml",
}); err != nil {
t.Fatalf("ApplyConfiguration(new): %v", err)
}
close(state.releaseFirst)
if err := <-firstDone; err != nil {
t.Fatalf("ApplyConfiguration(old): %v", err)
}
if got := store.Current().Routing[0].Upstreams[0]; got != "provider-c" {
t.Fatalf("published upstream = %q, want provider-c", got)
}
if got := store.Revision(); got != 2 {
t.Fatalf("published revision = %d, want 2", got)
}
}
func TestApplicationServiceReloadPreservesCancellation(t *testing.T) {
t.Parallel()
service, err := NewApplicationService(ApplicationDependencies{
State: &recordingAdminState{}, Operations: staticOperationalStatusReader{},
Configuration: staticConfigurationLoader{err: context.Canceled},
Publisher: &recordingConfigurationPublisher{},
}, ApplicationOptions{Now: time.Now})
}, applicationTestOptions(time.Now))
if err != nil {
t.Fatalf("NewApplicationService() error = %v", err)
}
@ -409,13 +520,16 @@ func TestApplicationServiceReloadPreservesCancellation(t *testing.T) {
}
}
func TestApplicationServiceUsesSecretFreeManagementChecksum(t *testing.T) {
func TestApplicationServiceUsesOpaqueChecksumThatTracksSecretRotation(t *testing.T) {
t.Parallel()
state := &recordingAdminState{}
publisher := &recordingConfigurationPublisher{}
var commands []adminstate.CommitConfigCommand
state.onCommit = func(command adminstate.CommitConfigCommand) {
commands = append(commands, command)
state.mutation = adminstate.MutationResult{
RequestID: command.RequestID, Changed: true, Revision: uint64(len(commands)),
}
}
for _, secret := range []string{"secret-a", "secret-b"} {
configuration := validReloadConfiguration()
@ -427,7 +541,7 @@ func TestApplicationServiceUsesSecretFreeManagementChecksum(t *testing.T) {
Configuration: staticConfigurationLoader{loaded: LoadedConfiguration{
Value: configuration, Source: "config.yaml",
}}, Publisher: publisher,
}, ApplicationOptions{Now: time.Now})
}, applicationTestOptions(time.Now))
if err != nil {
t.Fatalf("NewApplicationService() error = %v", err)
}
@ -437,8 +551,8 @@ func TestApplicationServiceUsesSecretFreeManagementChecksum(t *testing.T) {
t.Fatalf("ReloadConfiguration() error = %v", err)
}
}
if len(commands) != 2 || commands[0].Checksum != commands[1].Checksum || commands[0].ConfigVersion != commands[1].ConfigVersion {
t.Fatalf("secret rotation changed public management digest: %+v", commands)
if len(commands) != 2 || commands[0].Checksum == commands[1].Checksum || commands[0].ConfigVersion == commands[1].ConfigVersion {
t.Fatalf("secret rotation did not change opaque configuration digest: %+v", commands)
}
if len(publisher.published) != 2 || publisher.published[1].Upstreams["provider-a"].ProxyAuth.Password != "secret-b" {
t.Fatalf("secret rotation was not published: %+v", publisher.published)
@ -467,6 +581,16 @@ func validReloadConfiguration() *config.Config {
}
}
func configWithOnlyUpstream(name string) *config.Config {
configuration := validReloadConfiguration()
configuration.Upstreams = map[string]config.Upstream{name: validReloadUpstream("secret")}
configuration.Routing = []config.Routing{{
Name: "default", Enabled: true, Purpose: "gateway", Upstreams: []string{name},
Strategy: config.Strategy{Type: "random"}, OnUnavailable: config.OnUnavailable{Action: "reject"},
}}
return configuration
}
func validReloadUpstream(secret string) config.Upstream {
return config.Upstream{
Enabled: true, Exposure: []string{"gateway"},
@ -505,6 +629,33 @@ type recordingAdminState struct {
onCommit func(adminstate.CommitConfigCommand)
}
type orderedCommitState struct {
next atomic.Uint64
firstCommitted chan struct{}
releaseFirst chan struct{}
}
func (state *orderedCommitState) SetUpstreamEnabled(context.Context, adminstate.SetUpstreamCommand) (adminstate.MutationResult, error) {
return adminstate.MutationResult{}, nil
}
func (state *orderedCommitState) SwitchRouting(context.Context, adminstate.SwitchRoutingCommand) (adminstate.MutationResult, error) {
return adminstate.MutationResult{}, nil
}
func (state *orderedCommitState) CommitConfig(_ context.Context, command adminstate.CommitConfigCommand) (adminstate.MutationResult, error) {
revision := state.next.Add(1)
if revision == 1 {
close(state.firstCommitted)
<-state.releaseFirst
}
return adminstate.MutationResult{RequestID: command.RequestID, Changed: true, Revision: revision}, nil
}
func (*orderedCommitState) Snapshot(context.Context) (adminstate.Snapshot, error) {
return adminstate.Snapshot{}, nil
}
func (state *recordingAdminState) SetUpstreamEnabled(_ context.Context, command adminstate.SetUpstreamCommand) (adminstate.MutationResult, error) {
state.lastUpstream = command
return state.mutation, state.err
@ -553,8 +704,26 @@ func (loader staticConfigurationLoader) LoadConfiguration(context.Context) (Load
type recordingConfigurationPublisher struct {
published []*config.Config
revisions []uint64
}
func (publisher *recordingConfigurationPublisher) Publish(configuration *config.Config) {
publisher.published = append(publisher.published, configuration)
type recordingRuntimeNotifier struct {
notifications int
validationErr error
}
func (notifier *recordingRuntimeNotifier) Notify() { notifier.notifications++ }
func (notifier *recordingRuntimeNotifier) ValidateConfiguration(context.Context, *config.Config) error {
return notifier.validationErr
}
func (notifier *recordingRuntimeNotifier) ValidateUpstream(context.Context, string) error {
return notifier.validationErr
}
func (publisher *recordingConfigurationPublisher) PublishRevision(configuration *config.Config, revision uint64) bool {
publisher.published = append(publisher.published, configuration)
publisher.revisions = append(publisher.revisions, revision)
return true
}

115
internal/controller/bootstrap/bootstrap.go
View File

@ -5,6 +5,7 @@ import (
"errors"
"fmt"
"reflect"
"sort"
"strings"
"time"
@ -15,11 +16,15 @@ import (
"proxy-pool/internal/controller/distribution"
"proxy-pool/internal/controller/extraction"
"proxy-pool/internal/controller/operations"
"proxy-pool/internal/controller/pool"
"proxy-pool/internal/controller/provider"
controllerRuntime "proxy-pool/internal/controller/runtime"
"proxy-pool/internal/domain/activitypool"
extractionDomain "proxy-pool/internal/domain/extraction"
"proxy-pool/internal/platform/admission"
"proxy-pool/internal/platform/credentials"
"proxy-pool/internal/platform/httpserver"
"proxy-pool/internal/platform/lifecycle"
platformMetrics "proxy-pool/internal/platform/metrics"
)
@ -33,10 +38,15 @@ type Options struct {
Resolver config.Resolver
Now func() time.Time
HTTP httpserver.Options
HolderID string
RedisNamespace string
FingerprintKey []byte
}
type activityStore interface {
extractionDomain.Store
activitypool.Upserter
pool.InventoryReader
activitypool.StateInventoryReader
}
@ -45,6 +55,9 @@ type ports struct {
activity activityStore
readiness distribution.ReadinessChecker
metricsReadiness platformMetrics.ReadinessChecker
coordinator provider.Coordinator
credentials credentials.Store
providerResults provider.ResultRecorder
close func() error
}
@ -61,7 +74,9 @@ type runtimeFactory interface {
}
func Run(ctx context.Context, options Options) error {
return run(ctx, options, &productionInfrastructure{}, productionRuntimeFactory{})
return run(ctx, options, &productionInfrastructure{
holderID: options.HolderID, namespace: options.RedisNamespace,
}, productionRuntimeFactory{})
}
func run(ctx context.Context, options Options, infrastructure infrastructure, factory runtimeFactory) (resultErr error) {
@ -84,6 +99,9 @@ func run(ctx context.Context, options Options, infrastructure infrastructure, fa
if err != nil {
return fmt.Errorf("%w: load configuration: %w", ErrStartup, err)
}
if loaded.Value.Admin.Enabled && len(options.FingerprintKey) < config.MinimumFingerprintKeyBytes {
return errors.Join(ErrInvalidOptions, config.ErrInvalidFingerprint)
}
configurationStore, err := config.NewStore(loaded.Value)
if err != nil {
return fmt.Errorf("%w: initialize configuration store: %w", ErrStartup, err)
@ -99,6 +117,31 @@ func run(ctx context.Context, options Options, infrastructure infrastructure, fa
defer func() {
resultErr = errors.Join(resultErr, opened.close())
}()
var providerState providerStateReader
if loaded.Value.Admin.Enabled {
providerState = opened.state
}
supervisor, err := newProviderSupervisor(
configurationStore,
providerState,
func(name string, upstream config.Upstream) (lifecycle.Runner, error) {
buildRuntime, err := providerRuntimeBuilder(opened)
if err != nil {
return nil, err
}
return buildRuntime(name, upstream)
},
func(ctx context.Context, configuration *config.Config) error {
return prepareProviderConfiguration(ctx, configuration, opened.credentials)
},
func(configuration *config.Config) { retainProviderStats(configuration, opened.providerResults) },
loader,
options.FingerprintKey,
providerSupervisorInterval,
)
if err != nil {
return fmt.Errorf("%w: build Provider supervisor: %w", ErrStartup, err)
}
dependencies := controllerRuntime.Dependencies{}
if loaded.Value.Distribution.Enabled {
@ -116,13 +159,23 @@ func run(ctx context.Context, options Options, infrastructure infrastructure, fa
if nilInterface(opened.state) || nilInterface(opened.activity) {
return errors.Join(ErrStartup, ErrInvalidOptions)
}
statusReader, statusErr := operations.NewReader(configurationStore, opened.activity, options.Now)
var providerStats []provider.StatsReader
if stats, ok := opened.providerResults.(provider.StatsReader); ok && !nilInterface(stats) {
providerStats = append(providerStats, stats)
}
statusReader, statusErr := operations.NewReader(
configurationStore,
opened.activity,
options.Now,
providerStats...,
)
if statusErr != nil {
return fmt.Errorf("%w: build operational status reader: %w", ErrStartup, statusErr)
}
service, serviceErr := admin.NewApplicationService(admin.ApplicationDependencies{
State: opened.state, Operations: statusReader, Configuration: loader, Publisher: configurationStore,
}, admin.ApplicationOptions{Now: options.Now})
Runtime: supervisor,
}, admin.ApplicationOptions{Now: options.Now, FingerprintKey: options.FingerprintKey})
if serviceErr != nil {
return fmt.Errorf("%w: build admin service: %w", ErrStartup, serviceErr)
}
@ -146,6 +199,8 @@ func run(ctx context.Context, options Options, infrastructure infrastructure, fa
dependencies.MetricsHandler = handler
}
runners := make([]lifecycle.Runner, 0, 2)
if hasHTTPRuntime(loaded.Value) {
runner, err := factory.New(configurationStore.Current(), dependencies, controllerRuntime.Options{HTTP: options.HTTP})
if err != nil {
return fmt.Errorf("%w: build HTTP runtime: %w", ErrStartup, err)
@ -153,7 +208,59 @@ func run(ctx context.Context, options Options, infrastructure infrastructure, fa
if nilInterface(runner) {
return errors.Join(ErrStartup, ErrInvalidOptions)
}
return runner.Run(ctx)
runners = append(runners, runner)
}
runners = append(runners, supervisor)
group, err := lifecycle.NewGroup(runners...)
if err != nil {
return fmt.Errorf("%w: build process lifecycle: %w", ErrStartup, err)
}
return group.Run(ctx)
}
func prepareProviderConfiguration(
ctx context.Context,
configuration *config.Config,
credentialStore credentials.Store,
) error {
if ctx == nil || configuration == nil {
return ErrProviderRuntime
}
if err := ctx.Err(); err != nil {
return err
}
if !configuration.Admin.Enabled && !hasEnabledUpstream(configuration) {
return nil
}
ensurer, ok := credentialStore.(credentials.CapacityEnsurer)
if !ok || nilInterface(credentialStore) {
return ErrProviderRuntime
}
if err := ensurer.EnsureCapacity(ctx, providerCredentialCapacity(configuration)); err != nil {
return err
}
return nil
}
func retainProviderStats(configuration *config.Config, results provider.ResultRecorder) {
if configuration == nil {
return
}
retainer, ok := results.(provider.StatsRetainer)
if !ok || nilInterface(retainer) {
return
}
names := make([]string, 0, len(configuration.Upstreams))
for name := range configuration.Upstreams {
names = append(names, name)
}
sort.Strings(names)
retainer.RetainProviderStats(names)
}
func hasHTTPRuntime(configuration *config.Config) bool {
return configuration != nil &&
(configuration.Distribution.Enabled || configuration.Admin.Enabled || configuration.Metrics.Enabled)
}
func extractionPolicy(configuration *config.Config) extraction.Policy {

134
internal/controller/bootstrap/bootstrap_integration_test.go
View File

@ -4,16 +4,25 @@ package bootstrap
import (
"context"
"errors"
"net/http"
"net/http/httptest"
"os"
"strconv"
"strings"
"sync/atomic"
"testing"
"time"
"github.com/redis/go-redis/v9"
"proxy-pool/internal/adapters/redisactivity"
"proxy-pool/internal/config"
"proxy-pool/internal/controller/admin"
"proxy-pool/internal/controller/pool"
"proxy-pool/internal/controller/provider"
controllerRuntime "proxy-pool/internal/controller/runtime"
"proxy-pool/internal/platform/credentials"
)
func TestProductionBootstrapOpensBothStoresAndCommitsStartupConfiguration(t *testing.T) {
@ -22,18 +31,38 @@ func TestProductionBootstrapOpensBothStoresAndCommitsStartupConfiguration(t *tes
if postgresURL == "" || redisURL == "" {
t.Skip("PROXY_POOL_TEST_POSTGRES_URL and PROXY_POOL_TEST_REDIS_URL are required")
}
var providerCalls atomic.Int64
providerServer := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, _ *http.Request) {
providerCalls.Add(1)
_, _ = writer.Write([]byte("http://192.0.2.10:8080"))
}))
defer providerServer.Close()
namespace := "controller-it-" + strconv.FormatInt(time.Now().UnixNano(), 10)
inventory := newIntegrationInventoryReader(t, redisURL, namespace)
source := strings.ReplaceAll(bootstrapTestConfig, "postgres://fixture", postgresURL)
source = strings.ReplaceAll(source, "redis://fixture", redisURL)
source = strings.ReplaceAll(source, "https://provider.invalid/proxies", providerServer.URL)
resolver := &memoryResolver{files: map[string][]byte{"controller.yaml": []byte(source)}}
factory := &integrationRuntimeFactory{}
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
providerResults := make(chan provider.Result, 8)
factory := &integrationRuntimeFactory{
cancel: cancel, inventory: inventory, providerResults: providerResults,
}
infrastructure := &integrationInfrastructure{
productionInfrastructure: productionInfrastructure{namespace: namespace},
results: providerResultRecorder(func(result provider.Result) { providerResults <- result }),
}
err := run(context.Background(), Options{
err := run(ctx, Options{
ConfigPath: "controller.yaml", Resolver: resolver, Now: func() time.Time {
return time.Date(2026, 7, 30, 13, 0, 0, 0, time.UTC)
},
}, &productionInfrastructure{}, factory)
if err != nil {
t.Fatalf("run() error = %v", err)
FingerprintKey: bootstrapTestFingerprintKey,
}, infrastructure, factory)
if !errors.Is(err, context.Canceled) {
t.Fatalf("run() error = %v, want context cancellation", err)
}
if factory.status.ConfigVersion == "" || len(factory.status.Upstreams) != 2 {
t.Fatalf("Admin Status = %+v", factory.status)
@ -46,6 +75,9 @@ func TestProductionBootstrapOpensBothStoresAndCommitsStartupConfiguration(t *tes
!strings.Contains(factory.metricsBody, "go_") {
t.Fatalf("Metrics probes = ready:%d metrics:%d body:%q", factory.readyStatus, factory.metricsStatus, factory.metricsBody)
}
if providerCalls.Load() == 0 {
t.Fatal("production Provider HTTP adapter was not called")
}
}
type integrationRuntimeFactory struct {
@ -53,6 +85,9 @@ type integrationRuntimeFactory struct {
readyStatus int
metricsStatus int
metricsBody string
cancel context.CancelFunc
inventory pool.InventoryReader
providerResults <-chan provider.Result
}
func (factory *integrationRuntimeFactory) New(
@ -73,10 +108,99 @@ func (factory *integrationRuntimeFactory) New(
factory.metricsBody = metrics.Body.String()
status, err := dependencies.AdminService.Status(ctx)
factory.status = status
if err != nil {
return err
}
if err := waitForProviderInventory(
ctx,
factory.inventory,
factory.providerResults,
[]string{"provider-a", "provider-b"},
); err != nil {
return err
}
factory.cancel()
<-ctx.Done()
return ctx.Err()
}}, nil
}
func newIntegrationInventoryReader(t *testing.T, redisURL, namespace string) pool.InventoryReader {
t.Helper()
options, err := redis.ParseURL(redisURL)
if err != nil {
t.Fatalf("redis.ParseURL(): %v", err)
}
client := redis.NewClient(options)
t.Cleanup(func() { _ = client.Close() })
credentialStore, err := credentials.NewMemoryStore(200)
if err != nil {
t.Fatalf("credentials.NewMemoryStore(): %v", err)
}
reader, err := redisactivity.New(client, redisactivity.Options{
Namespace: namespace, Credentials: credentialStore,
OperationTTL: redisOperationTTL, MaxCandidateScan: redisMinimumScan,
MaxRuntimeCounters: 200, MaxInventoryScan: 200, CleanupLimit: redisCleanupLimit,
})
if err != nil {
t.Fatalf("redisactivity.New(): %v", err)
}
return reader
}
func waitForProviderInventory(
ctx context.Context,
inventory pool.InventoryReader,
results <-chan provider.Result,
upstreamIDs []string,
) error {
deadline, cancel := context.WithTimeout(ctx, 5*time.Second)
defer cancel()
ticker := time.NewTicker(10 * time.Millisecond)
defer ticker.Stop()
var latestResult provider.Result
for {
var inventoryErr error
for _, upstreamID := range upstreamIDs {
snapshot, err := inventory.ReadInventory(deadline, upstreamID, 0)
inventoryErr = errors.Join(inventoryErr, err)
if err == nil && snapshot.Managed > 0 {
return nil
}
}
select {
case <-deadline.Done():
return errors.Join(
errors.New("wait for Provider Redis inventory"),
deadline.Err(),
inventoryErr,
latestResult.Err,
)
case result := <-results:
latestResult = result
case <-ticker.C:
}
}
}
type integrationInfrastructure struct {
productionInfrastructure
results provider.ResultRecorder
}
func (infrastructure *integrationInfrastructure) Open(
ctx context.Context,
configuration *config.Config,
) (ports, error) {
opened, err := infrastructure.productionInfrastructure.Open(ctx, configuration)
opened.providerResults = infrastructure.results
return opened, err
}
type providerResultRecorder func(provider.Result)
func (record providerResultRecorder) Record(result provider.Result) { record(result) }
type integrationRunner struct {
run func(context.Context) error
}

216
internal/controller/bootstrap/bootstrap_test.go
View File

@ -3,32 +3,48 @@ package bootstrap
import (
"context"
"errors"
"strings"
"testing"
"time"
"proxy-pool/internal/config"
"proxy-pool/internal/controller/admin"
"proxy-pool/internal/controller/pool"
"proxy-pool/internal/controller/provider"
controllerRuntime "proxy-pool/internal/controller/runtime"
"proxy-pool/internal/domain/activitypool"
"proxy-pool/internal/domain/adminstate"
extractionDomain "proxy-pool/internal/domain/extraction"
"proxy-pool/internal/domain/upstream"
"proxy-pool/internal/platform/credentials"
)
var bootstrapTestFingerprintKey = []byte("0123456789abcdef0123456789abcdef")
func TestRunLoadsOneSnapshotCommitsItAndClosesInfrastructure(t *testing.T) {
t.Parallel()
resolver := &memoryResolver{files: map[string][]byte{"controller.yaml": []byte(bootstrapTestConfig)}}
state := adminstate.NewMemoryStore()
activity := &stubActivityStore{}
credentialStore, err := credentials.NewMemoryStore(10)
if err != nil {
t.Fatalf("NewMemoryStore(): %v", err)
}
closeErr := errors.New("close failed")
infrastructure := &stubInfrastructure{ports: ports{
state: state, activity: activity, readiness: readyStub{}, metricsReadiness: readyStub{},
coordinator: coordinatorStub{}, credentials: credentialStore,
close: func() error { return closeErr },
}}
runErr := errors.New("runtime failed")
factory := &recordingRuntimeFactory{runner: runnerStub{err: runErr}}
now := time.Date(2026, 7, 30, 11, 0, 0, 0, time.UTC)
err := run(context.Background(), Options{
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
defer cancel()
err = run(ctx, Options{
ConfigPath: "controller.yaml", Resolver: resolver, Now: func() time.Time { return now },
FingerprintKey: bootstrapTestFingerprintKey,
}, infrastructure, factory)
if !errors.Is(err, runErr) || !errors.Is(err, closeErr) {
t.Fatalf("run() error = %v, want runtime and close errors", err)
@ -74,6 +90,147 @@ func TestRunRejectsInvalidOptionsBeforeIO(t *testing.T) {
}
}
func TestRunRejectsMissingAdminFingerprintKeyBeforeOpeningInfrastructure(t *testing.T) {
for name, key := range map[string][]byte{
"missing": nil,
"short": []byte("too-short"),
} {
t.Run(name, func(t *testing.T) {
infrastructure := &stubInfrastructure{}
err := run(context.Background(), Options{
ConfigPath: "controller.yaml",
Resolver: &memoryResolver{files: map[string][]byte{"controller.yaml": []byte(bootstrapTestConfig)}},
Now: time.Now,
FingerprintKey: key,
}, infrastructure, &recordingRuntimeFactory{})
if !errors.Is(err, ErrInvalidOptions) || !errors.Is(err, config.ErrInvalidFingerprint) {
t.Fatalf("run() error = %v", err)
}
if infrastructure.opens != 0 {
t.Fatalf("infrastructure opens = %d, want 0", infrastructure.opens)
}
})
}
}
func TestRunSupportsProviderOnlyConfigurationWithoutHTTPRuntime(t *testing.T) {
source := strings.ReplaceAll(bootstrapTestConfig, "distribution:\n enabled: true", "distribution:\n enabled: false")
source = strings.ReplaceAll(source, "admin:\n enabled: true", "admin:\n enabled: false")
source = strings.ReplaceAll(source, "metrics:\n enabled: true", "metrics:\n enabled: false")
credentialStore, err := credentials.NewMemoryStore(10)
if err != nil {
t.Fatalf("NewMemoryStore(): %v", err)
}
infrastructure := &stubInfrastructure{ports: ports{
activity: &stubActivityStore{}, coordinator: coordinatorStub{}, credentials: credentialStore,
close: func() error { return nil },
}}
factory := &recordingRuntimeFactory{}
ctx, cancel := context.WithCancel(context.Background())
time.AfterFunc(20*time.Millisecond, cancel)
err = run(ctx, Options{
ConfigPath: "controller.yaml",
Resolver: &memoryResolver{files: map[string][]byte{"controller.yaml": []byte(source)}},
Now: time.Now,
}, infrastructure, factory)
if !errors.Is(err, context.Canceled) {
t.Fatalf("run(provider only) error = %v, want context cancellation", err)
}
if factory.configuration != nil {
t.Fatal("HTTP runtime factory was called for Provider-only configuration")
}
}
func TestRunAdminDisableStopsActiveProviderRuntime(t *testing.T) {
state := adminstate.NewMemoryStore()
credentialStore, err := credentials.NewMemoryStore(200)
if err != nil {
t.Fatalf("NewMemoryStore(): %v", err)
}
started := make(chan string, 2)
stopped := make(chan string, 2)
infrastructure := &stubInfrastructure{ports: ports{
state: state, activity: &stubActivityStore{}, readiness: readyStub{}, metricsReadiness: readyStub{},
coordinator: coordinatorFunc(func(ctx context.Context, upstreamID string) error {
started <- upstreamID
<-ctx.Done()
stopped <- upstreamID
return ctx.Err()
}),
credentials: credentialStore,
close: func() error { return nil },
}}
wantErr := errors.New("test HTTP runtime stopped")
factory := runtimeFactoryFunc(func(
_ *config.Config,
dependencies controllerRuntime.Dependencies,
_ controllerRuntime.Options,
) (controllerRunner, error) {
return runnerFunc(func(ctx context.Context) error {
for {
select {
case upstreamID := <-started:
if upstreamID != "provider-a" {
continue
}
if _, err := dependencies.AdminService.SetUpstreamEnabled(ctx, admin.SetUpstreamCommand{
RequestID: "req-disable", ActorID: "admin:test", Name: "provider-a", Enabled: false,
}); err != nil {
return err
}
for {
select {
case stoppedID := <-stopped:
if stoppedID == "provider-a" {
return wantErr
}
case <-ctx.Done():
return ctx.Err()
}
}
case <-ctx.Done():
return ctx.Err()
}
}
}), nil
})
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
defer cancel()
err = run(ctx, Options{
ConfigPath: "controller.yaml",
Resolver: &memoryResolver{files: map[string][]byte{"controller.yaml": []byte(bootstrapTestConfig)}},
Now: time.Now,
FingerprintKey: bootstrapTestFingerprintKey,
}, infrastructure, factory)
if !errors.Is(err, wantErr) {
t.Fatalf("run() error = %v, want %v", err, wantErr)
}
}
func TestRetainProviderStatsKeepsAllConfiguredProviders(t *testing.T) {
configuration, err := config.Load(strings.NewReader(bootstrapTestConfig))
if err != nil {
t.Fatalf("config.Load(): %v", err)
}
disabled := configuration.Upstreams["provider-b"]
disabled.Enabled = false
configuration.Upstreams["provider-b"] = disabled
stats, err := provider.NewStatsRecorder(2)
if err != nil {
t.Fatalf("provider.NewStatsRecorder(): %v", err)
}
stats.Record(provider.Result{UpstreamID: "removed", Class: upstream.FetchError})
stats.Record(provider.Result{UpstreamID: "provider-b", Class: upstream.FetchError})
retainProviderStats(configuration, stats)
stats.Record(provider.Result{UpstreamID: "provider-a", Class: upstream.FetchError})
got := stats.ReadProviderStats([]string{"removed", "provider-a", "provider-b"})
if got[0].FetchErrorCount != 0 || got[1].FetchErrorCount != 1 || got[2].FetchErrorCount != 1 {
t.Fatalf("Provider stats after retention = %+v", got)
}
}
type memoryResolver struct {
files map[string][]byte
reads int
@ -127,6 +284,24 @@ type runnerStub struct{ err error }
func (runner runnerStub) Run(context.Context) error { return runner.err }
type runnerFunc func(context.Context) error
func (run runnerFunc) Run(ctx context.Context) error { return run(ctx) }
type runtimeFactoryFunc func(
*config.Config,
controllerRuntime.Dependencies,
controllerRuntime.Options,
) (controllerRunner, error)
func (factory runtimeFactoryFunc) New(
configuration *config.Config,
dependencies controllerRuntime.Dependencies,
options controllerRuntime.Options,
) (controllerRunner, error) {
return factory(configuration, dependencies, options)
}
type readyStub struct{}
func (readyStub) Ready(context.Context) error { return nil }
@ -149,6 +324,45 @@ func (*stubActivityStore) ReadStateInventory(
return result, nil
}
func (*stubActivityStore) UpsertFetched(
_ context.Context,
_ string,
batch activitypool.FetchedBatch,
) (activitypool.UpsertResult, error) {
return activitypool.UpsertResult{Accepted: len(batch.Proxies), Inserted: len(batch.Proxies)}, nil
}
func (*stubActivityStore) ReadInventory(
context.Context,
string,
time.Duration,
) (pool.InventorySnapshot, error) {
return pool.InventorySnapshot{Managed: 100, AvailableSlots: 1_000}, nil
}
type coordinatorStub struct{}
func (coordinatorStub) RunLeader(
ctx context.Context,
_ string,
_ provider.CoordinationLimits,
_ func(context.Context, provider.LeaderSession) error,
) error {
<-ctx.Done()
return ctx.Err()
}
type coordinatorFunc func(context.Context, string) error
func (run coordinatorFunc) RunLeader(
ctx context.Context,
upstreamID string,
_ provider.CoordinationLimits,
_ func(context.Context, provider.LeaderSession) error,
) error {
return run(ctx, upstreamID)
}
const bootstrapTestConfig = `
version: 1
security:

119
internal/controller/bootstrap/infrastructure.go
View File

@ -2,6 +2,8 @@ package bootstrap
import (
"context"
"crypto/rand"
"encoding/hex"
"errors"
"strings"
"time"
@ -11,7 +13,9 @@ import (
"proxy-pool/internal/adapters/postgresadmin"
"proxy-pool/internal/adapters/redisactivity"
"proxy-pool/internal/adapters/redisprovider"
"proxy-pool/internal/config"
controllerProvider "proxy-pool/internal/controller/provider"
"proxy-pool/internal/platform/credentials"
platformMetrics "proxy-pool/internal/platform/metrics"
)
@ -20,7 +24,12 @@ const (
redisNamespace = "controller"
redisOperationTTL = 30 * time.Second
redisMinimumScan = 4_096
redisMaximumScan = config.MaximumPoolSize
redisCleanupLimit = 1_024
providerLeaseTTL = 15 * time.Second
providerRenewEvery = 3 * time.Second
providerRetryInterval = 100 * time.Millisecond
providerPermitGrace = 5 * time.Second
)
var (
@ -30,15 +39,22 @@ var (
ErrRedisUnavailable = errors.New("Redis unavailable")
)
type productionInfrastructure struct{}
type productionInfrastructure struct {
holderID string
namespace string
}
func (*productionInfrastructure) Open(
func (infrastructure *productionInfrastructure) Open(
ctx context.Context,
configuration *config.Config,
) (_ ports, resultErr error) {
if ctx == nil || configuration == nil {
return ports{}, ErrInvalidOptions
}
namespace, err := resolveRedisNamespace(infrastructure.namespace)
if err != nil {
return ports{}, err
}
var postgresPool *pgxpool.Pool
var redisClient *redis.Client
closeResources := func() error {
@ -83,7 +99,8 @@ func (*productionInfrastructure) Open(
}
}
if configuration.Distribution.Enabled || configuration.Admin.Enabled {
providersEnabled := hasEnabledUpstream(configuration)
if configuration.Distribution.Enabled || configuration.Admin.Enabled || providersEnabled {
if strings.TrimSpace(configuration.Storage.RedisURL) == "" {
return ports{}, ErrRedisConfiguration
}
@ -95,15 +112,17 @@ func (*productionInfrastructure) Open(
if err = redisClient.Ping(ctx).Err(); err != nil {
return ports{}, contextOr(ctx, ErrRedisUnavailable)
}
credentialStore, err := credentials.NewMemoryStore(credentialCapacity(configuration))
credentialStore, err := credentials.NewMemoryStore(providerCredentialCapacity(configuration))
if err != nil {
return ports{}, err
}
adapter, err := redisactivity.New(redisClient, redisactivity.Options{
Namespace: redisNamespace,
Namespace: namespace,
Credentials: credentialStore,
OperationTTL: redisOperationTTL,
MaxCandidateScan: candidateScan(configuration),
MaxRuntimeCounters: credentialCapacity(configuration),
MaxInventoryScan: maxInventoryScan(configuration),
CleanupLimit: redisCleanupLimit,
})
if err != nil {
@ -111,6 +130,26 @@ func (*productionInfrastructure) Open(
}
opened.activity = adapter
opened.readiness = redisReadiness{client: redisClient}
opened.credentials = credentialStore
if providersEnabled {
stats, statsErr := controllerProvider.NewStatsRecorder(config.MaximumUpstreams)
if statsErr != nil {
return ports{}, statsErr
}
opened.providerResults = stats
holderID, holderErr := resolveHolderID(infrastructure.holderID)
if holderErr != nil {
return ports{}, holderErr
}
opened.coordinator, err = redisprovider.New(redisClient, redisprovider.Options{
Namespace: namespace, HolderID: holderID,
LeaseTTL: providerLeaseTTL, RenewEvery: providerRenewEvery,
RetryInterval: providerRetryInterval, PermitGrace: providerPermitGrace,
})
if err != nil {
return ports{}, err
}
}
}
if configuration.Metrics.Enabled {
opened.metricsReadiness = selectMetricsReadiness(
@ -122,11 +161,21 @@ func (*productionInfrastructure) Open(
return opened, nil
}
func resolveRedisNamespace(configured string) (string, error) {
if strings.TrimSpace(configured) != configured {
return "", ErrInvalidOptions
}
if configured == "" {
return redisNamespace, nil
}
return configured, nil
}
func selectMetricsReadiness(
configuration *config.Config,
admin, activity platformMetrics.ReadinessChecker,
) platformMetrics.ReadinessChecker {
if configuration.Distribution.Enabled {
if configuration.Distribution.Enabled || hasEnabledUpstream(configuration) {
return activity
}
if configuration.Admin.Enabled {
@ -184,7 +233,7 @@ func credentialCapacity(configuration *config.Config) int {
capacity := 0
maximum := int(^uint(0) >> 1)
for _, upstream := range configuration.Upstreams {
if upstream.Pool.MaxSize <= 0 {
if !upstream.Enabled || upstream.Pool.MaxSize <= 0 {
continue
}
if capacity > maximum-upstream.Pool.MaxSize {
@ -198,6 +247,62 @@ func credentialCapacity(configuration *config.Config) int {
return capacity
}
func providerCredentialCapacity(configuration *config.Config) int {
capacity := 0
maximum := int(^uint(0) >> 1)
for _, upstream := range configuration.Upstreams {
if upstream.Pool.MaxSize <= 0 {
continue
}
maxInFlight := upstream.Fetch.MaxInFlight
if maxInFlight <= 0 {
maxInFlight = 1
}
if upstream.Pool.MaxSize > maximum/maxInFlight {
return maximum
}
leases := upstream.Pool.MaxSize * maxInFlight
if capacity > maximum-leases {
return maximum
}
capacity += leases
}
if capacity == 0 {
return 1
}
return capacity
}
func maxInventoryScan(_ *config.Config) int {
return config.MaximumPoolSize
}
func hasEnabledUpstream(configuration *config.Config) bool {
if configuration == nil {
return false
}
for _, upstream := range configuration.Upstreams {
if upstream.Enabled {
return true
}
}
return false
}
func resolveHolderID(configured string) (string, error) {
if strings.TrimSpace(configured) != configured {
return "", ErrInvalidOptions
}
if configured != "" {
return configured, nil
}
var entropy [16]byte
if _, err := rand.Read(entropy[:]); err != nil {
return "", errors.Join(ErrStartup, err)
}
return "controller-" + hex.EncodeToString(entropy[:]), nil
}
func candidateScan(configuration *config.Config) int {
configured := configuration.Distribution.Extraction
if configured.MaxCountPerRequest > int(^uint(0)>>1)-configured.ReserveForGateway {

19
internal/controller/bootstrap/infrastructure_test.go
View File

@ -79,13 +79,17 @@ func TestProductionRedisSizingUsesConfigurationBounds(t *testing.T) {
MaxCountPerRequest: 100, ReserveForGateway: 5_000,
}},
Upstreams: map[string]config.Upstream{
"provider-a": {Pool: config.Pool{MaxSize: 3_000}},
"provider-b": {Pool: config.Pool{MaxSize: 2_000}},
"provider-a": {Enabled: true, Pool: config.Pool{MaxSize: 3_000}, Fetch: config.Fetch{MaxInFlight: 2}},
"provider-b": {Enabled: true, Pool: config.Pool{MaxSize: 2_000}, Fetch: config.Fetch{MaxInFlight: 1}},
"disabled": {Pool: config.Pool{MaxSize: 50_000}, Fetch: config.Fetch{MaxInFlight: 3}},
},
}
if got := credentialCapacity(configuration); got != 5_000 {
t.Fatalf("credentialCapacity() = %d, want 5000", got)
}
if got := providerCredentialCapacity(configuration); got != 158_000 {
t.Fatalf("providerCredentialCapacity() = %d, want 158000", got)
}
if got := candidateScan(configuration); got != 5_100 {
t.Fatalf("candidateScan() = %d, want 5100", got)
}
@ -94,3 +98,14 @@ func TestProductionRedisSizingUsesConfigurationBounds(t *testing.T) {
t.Fatalf("candidateScan(minimum) = %d, want %d", got, redisMinimumScan)
}
}
func TestMaxInventoryScanSupportsPoolGrowthAfterReload(t *testing.T) {
t.Parallel()
configuration := &config.Config{Upstreams: map[string]config.Upstream{
"provider-a": {Enabled: true, Pool: config.Pool{MaxSize: 100}},
}}
if got := maxInventoryScan(configuration); got != config.MaximumPoolSize {
t.Fatalf("maxInventoryScan(initial small pool) = %d, want %d", got, config.MaximumPoolSize)
}
}

126
internal/controller/bootstrap/provider.go Normal file
View File

@ -0,0 +1,126 @@
package bootstrap
import (
"errors"
"math"
"sort"
"strings"
"time"
"proxy-pool/internal/adapters/providerapi"
"proxy-pool/internal/config"
"proxy-pool/internal/controller/pool"
controllerProvider "proxy-pool/internal/controller/provider"
)
var ErrProviderRuntime = errors.New("invalid Provider runtime configuration")
func newProviderFleet(configuration *config.Config, opened ports) (*controllerProvider.Fleet, error) {
if configuration == nil {
return nil, ErrProviderRuntime
}
names := make([]string, 0, len(configuration.Upstreams))
for name, upstream := range configuration.Upstreams {
if upstream.Enabled {
names = append(names, name)
}
}
if len(names) == 0 {
return nil, nil
}
builder, err := providerRuntimeBuilder(opened)
if err != nil {
return nil, err
}
sort.Strings(names)
runtimes := make([]*controllerProvider.UpstreamRuntime, 0, len(names))
for _, name := range names {
runtime, err := builder(name, configuration.Upstreams[name])
if err != nil {
return nil, err
}
runtimes = append(runtimes, runtime)
}
return controllerProvider.NewFleet(runtimes...)
}
type buildUpstreamRuntime func(string, config.Upstream) (*controllerProvider.UpstreamRuntime, error)
func providerRuntimeBuilder(opened ports) (buildUpstreamRuntime, error) {
if nilInterface(opened.coordinator) || nilInterface(opened.activity) || nilInterface(opened.credentials) {
return nil, ErrProviderRuntime
}
results := opened.providerResults
if nilInterface(results) {
results = discardProviderResults{}
}
return func(name string, upstream config.Upstream) (*controllerProvider.UpstreamRuntime, error) {
upstream.Enabled = true
mapped, err := providerRuntimeConfig(name, upstream)
if err != nil {
return nil, err
}
adapter, err := providerapi.NewHTTPAdapter(upstream.API, upstream.Fetch, nil)
if err != nil {
return nil, err
}
parser, err := providerapi.NewTemplateParser(name, upstream, opened.credentials)
if err != nil {
return nil, err
}
return controllerProvider.NewUpstreamRuntime(mapped, controllerProvider.UpstreamRuntimeDependencies{
Coordinator: opened.coordinator,
Inventory: opened.activity,
Adapter: adapter,
Parser: parser,
Activity: opened.activity,
Results: results,
})
}, nil
}
func providerRuntimeConfig(
upstreamID string,
upstream config.Upstream,
) (controllerProvider.UpstreamRuntimeConfig, error) {
if strings.TrimSpace(upstreamID) != upstreamID || upstreamID == "" || !upstream.Enabled ||
upstream.Fetch.EstimatedIPsPerCall <= 0 || upstream.Capacity.MaxConcurrencyPerProxy <= 0 ||
upstream.Pool.MaxSize > config.MaximumPoolSize ||
int64(upstream.Fetch.EstimatedIPsPerCall) > controllerProvider.MaximumCoordinationInteger ||
int64(upstream.Fetch.MaxInFlight) > controllerProvider.MaximumCoordinationInteger ||
int64(upstream.Fetch.MaxTotal) > controllerProvider.MaximumCoordinationInteger ||
int64(upstream.Fetch.EstimatedIPsPerCall) > math.MaxInt64/int64(upstream.Capacity.MaxConcurrencyPerProxy) {
return controllerProvider.UpstreamRuntimeConfig{}, ErrProviderRuntime
}
expectedSlots := int64(upstream.Fetch.EstimatedIPsPerCall) * int64(upstream.Capacity.MaxConcurrencyPerProxy)
return controllerProvider.UpstreamRuntimeConfig{
Provider: controllerProvider.Config{
UpstreamID: upstreamID,
RequestInterval: time.Duration(upstream.Fetch.RequestInterval),
Timeout: time.Duration(upstream.Fetch.Timeout),
MaxAttempts: upstream.Fetch.MaxAttempts,
MaxInFlight: upstream.Fetch.MaxInFlight,
MaxTotal: int64(upstream.Fetch.MaxTotal),
MaxSize: upstream.Pool.MaxSize,
TTL: time.Duration(upstream.Lifecycle.TTL),
AllocationSafetyMargin: time.Duration(upstream.Lifecycle.AllocationSafetyMargin),
Retry: controllerProvider.RetryConfig{
Initial: time.Duration(upstream.Fetch.Retry.Initial),
Max: time.Duration(upstream.Fetch.Retry.Max),
Jitter: upstream.Fetch.Retry.Jitter,
},
},
ReconcilePolicy: pool.ReconcilePolicy{
MinimumAvailableSlots: upstream.Refill.MinimumAvailableSlots,
TargetAvailableSlots: upstream.Refill.TargetAvailableSlots,
ExpectedPerFetch: upstream.Fetch.EstimatedIPsPerCall,
ExpectedSlotsPerFetch: expectedSlots,
SafetyMargin: time.Duration(upstream.Lifecycle.AllocationSafetyMargin),
},
ReconcileInterval: time.Duration(upstream.Refill.ReconcileInterval),
}, nil
}
type discardProviderResults struct{}
func (discardProviderResults) Record(controllerProvider.Result) {}

375
internal/controller/bootstrap/provider_supervisor.go Normal file
View File

@ -0,0 +1,375 @@
package bootstrap
import (
"context"
"errors"
"fmt"
"reflect"
"sort"
"sync"
"time"
"proxy-pool/internal/config"
"proxy-pool/internal/controller/admin"
"proxy-pool/internal/domain/adminstate"
"proxy-pool/internal/platform/lifecycle"
)
const providerSupervisorInterval = time.Second
var (
ErrProviderSupervisor = errors.New("invalid Provider supervisor")
errProviderManagementStateUnavailable = errors.New("Provider management state unavailable")
errProviderManagementSnapshotStale = errors.New("Provider management snapshot stale")
errProviderConfigurationPending = errors.New("Provider configuration synchronization pending")
)
type providerConfigurationStore interface {
Current() *config.Config
Revision() uint64
PublishRevision(*config.Config, uint64) bool
}
type providerConfigurationSource interface {
LoadConfiguration(context.Context) (admin.LoadedConfiguration, error)
}
type providerStateReader interface {
Snapshot(context.Context) (adminstate.Snapshot, error)
}
type providerRunnerBuilder func(string, config.Upstream) (lifecycle.Runner, error)
type providerConfigurationPreparer func(context.Context, *config.Config) error
type providerConfigurationObserver func(*config.Config)
type providerSupervisor struct {
configuration providerConfigurationStore
state providerStateReader
build providerRunnerBuilder
prepare providerConfigurationPreparer
observe providerConfigurationObserver
source providerConfigurationSource
fingerprintKey []byte
interval time.Duration
notify chan struct{}
}
type runningProvider struct {
configuration config.Upstream
cancel context.CancelFunc
done chan struct{}
}
func newProviderSupervisor(
configuration providerConfigurationStore,
state providerStateReader,
build providerRunnerBuilder,
prepare providerConfigurationPreparer,
observe providerConfigurationObserver,
source providerConfigurationSource,
fingerprintKey []byte,
interval time.Duration,
) (*providerSupervisor, error) {
if nilInterface(configuration) || build == nil || interval <= 0 ||
(!nilInterface(state) && len(fingerprintKey) < config.MinimumFingerprintKeyBytes) {
return nil, ErrProviderSupervisor
}
return &providerSupervisor{
configuration: configuration,
state: state,
build: build,
prepare: prepare,
observe: observe,
source: source,
fingerprintKey: append([]byte(nil), fingerprintKey...),
interval: interval,
notify: make(chan struct{}, 1),
}, nil
}
func (supervisor *providerSupervisor) Notify() {
if supervisor == nil || supervisor.notify == nil {
return
}
select {
case supervisor.notify <- struct{}{}:
default:
}
}
func (supervisor *providerSupervisor) ValidateConfiguration(ctx context.Context, configuration *config.Config) error {
if supervisor == nil || ctx == nil || configuration == nil || supervisor.build == nil {
return ErrProviderSupervisor
}
if err := supervisor.validateConfiguration(ctx, configuration); err != nil {
return errors.Join(ErrProviderSupervisor, err)
}
return nil
}
func (supervisor *providerSupervisor) validateConfiguration(ctx context.Context, configuration *config.Config) error {
if err := ctx.Err(); err != nil {
return err
}
if err := config.Validate(configuration); err != nil {
return err
}
if supervisor.prepare != nil {
if err := supervisor.prepare(ctx, configuration); err != nil {
return err
}
}
names := make([]string, 0, len(configuration.Upstreams))
for name, upstream := range configuration.Upstreams {
if upstream.Enabled {
names = append(names, name)
}
}
sort.Strings(names)
for _, name := range names {
if err := ctx.Err(); err != nil {
return err
}
if _, err := supervisor.build(name, configuration.Upstreams[name]); err != nil {
return err
}
}
return nil
}
func (supervisor *providerSupervisor) ValidateUpstream(ctx context.Context, name string) error {
if supervisor == nil || ctx == nil || name == "" || supervisor.build == nil {
return ErrProviderSupervisor
}
if err := ctx.Err(); err != nil {
return err
}
configuration := supervisor.configuration.Current()
if configuration == nil {
return ErrProviderSupervisor
}
if supervisor.prepare != nil {
if err := supervisor.prepare(ctx, configuration); err != nil {
return errors.Join(ErrProviderSupervisor, err)
}
}
upstream, exists := configuration.Upstreams[name]
if !exists {
return ErrProviderSupervisor
}
upstream.Enabled = true
if _, err := supervisor.build(name, upstream); err != nil {
return errors.Join(ErrProviderSupervisor, err)
}
return nil
}
func (supervisor *providerSupervisor) Run(ctx context.Context) error {
if supervisor == nil || ctx == nil || nilInterface(supervisor.configuration) ||
supervisor.build == nil || supervisor.interval <= 0 || supervisor.notify == nil {
return ErrProviderSupervisor
}
active := make(map[string]*runningProvider)
failures := make(chan error, 1)
defer stopAllProviders(active)
ticker := time.NewTicker(supervisor.interval)
defer ticker.Stop()
for {
if err := supervisor.reconcile(ctx, active, failures); err != nil {
return err
}
select {
case <-ctx.Done():
return ctx.Err()
case err := <-failures:
return err
case <-supervisor.notify:
case <-ticker.C:
}
}
}
func (supervisor *providerSupervisor) reconcile(
ctx context.Context,
active map[string]*runningProvider,
failures chan<- error,
) error {
desired, err := supervisor.desired(ctx)
if err != nil {
if errors.Is(err, errProviderManagementStateUnavailable) ||
errors.Is(err, errProviderManagementSnapshotStale) {
return nil
}
if errors.Is(err, errProviderConfigurationPending) {
stopAllProviders(active)
clear(active)
return nil
}
return err
}
names := make([]string, 0, len(desired))
prepared := make(map[string]lifecycle.Runner)
for name, upstream := range desired {
names = append(names, name)
current := active[name]
if current != nil && reflect.DeepEqual(current.configuration, upstream) {
continue
}
runner, buildErr := supervisor.build(name, upstream)
if buildErr != nil || nilInterface(runner) {
return errors.Join(ErrProviderSupervisor, buildErr)
}
prepared[name] = runner
}
sort.Strings(names)
for name, current := range active {
if _, keep := desired[name]; !keep {
stopProvider(current)
delete(active, name)
}
}
for _, name := range names {
runner := prepared[name]
if runner == nil {
continue
}
if current := active[name]; current != nil {
stopProvider(current)
}
active[name] = startProvider(ctx, name, desired[name], runner, failures)
}
return nil
}
func (supervisor *providerSupervisor) desired(ctx context.Context) (map[string]config.Upstream, error) {
configuration := supervisor.configuration.Current()
if configuration == nil {
return nil, ErrProviderSupervisor
}
enabled := make(map[string]bool, len(configuration.Upstreams))
var snapshot adminstate.Snapshot
if !nilInterface(supervisor.state) {
var err error
snapshot, err = supervisor.state.Snapshot(ctx)
if err != nil {
return nil, errors.Join(errProviderManagementStateUnavailable, err)
}
configuration, err = supervisor.synchronizeConfiguration(ctx, configuration, snapshot)
if err != nil {
return nil, err
}
for _, upstream := range snapshot.Upstreams {
enabled[upstream.Name] = upstream.Enabled
}
}
if supervisor.prepare != nil {
if err := supervisor.prepare(ctx, configuration); err != nil {
return nil, errors.Join(ErrProviderSupervisor, err)
}
}
if supervisor.observe != nil {
supervisor.observe(configuration)
}
desired := make(map[string]config.Upstream)
for name, upstream := range configuration.Upstreams {
isEnabled := upstream.Enabled
if !nilInterface(supervisor.state) {
isEnabled = enabled[name]
}
if isEnabled {
upstream.Enabled = true
desired[name] = upstream
}
}
return desired, nil
}
func (supervisor *providerSupervisor) synchronizeConfiguration(
ctx context.Context,
current *config.Config,
snapshot adminstate.Snapshot,
) (*config.Config, error) {
if snapshot.Config == nil || snapshot.Config.Checksum == "" {
return current, nil
}
if supervisor.configuration.Revision() > snapshot.Config.Revision {
return nil, errProviderManagementSnapshotStale
}
checksum, err := config.Fingerprint(current, supervisor.fingerprintKey)
if err != nil {
return nil, errors.Join(ErrProviderSupervisor, err)
}
if checksum == snapshot.Config.Checksum {
supervisor.configuration.PublishRevision(current, snapshot.Config.Revision)
return current, nil
}
if nilInterface(supervisor.source) {
return nil, errProviderConfigurationPending
}
loaded, err := supervisor.source.LoadConfiguration(ctx)
if err != nil || loaded.Value == nil {
return nil, errors.Join(errProviderConfigurationPending, err)
}
checksum, err = config.Fingerprint(loaded.Value, supervisor.fingerprintKey)
if err != nil || checksum != snapshot.Config.Checksum {
return nil, errors.Join(errProviderConfigurationPending, err)
}
if err := supervisor.validateConfiguration(ctx, loaded.Value); err != nil {
return nil, errors.Join(errProviderConfigurationPending, err)
}
if supervisor.configuration.PublishRevision(loaded.Value, snapshot.Config.Revision) {
return loaded.Value, nil
}
if supervisor.configuration.Revision() > snapshot.Config.Revision {
return nil, errProviderManagementSnapshotStale
}
return nil, errProviderConfigurationPending
}
func startProvider(
ctx context.Context,
name string,
configuration config.Upstream,
runner lifecycle.Runner,
failures chan<- error,
) *runningProvider {
runCtx, cancel := context.WithCancel(ctx)
running := &runningProvider{configuration: configuration, cancel: cancel, done: make(chan struct{})}
go func() {
defer close(running.done)
err := runner.Run(runCtx)
if runCtx.Err() != nil {
return
}
if err == nil {
err = lifecycle.ErrRunnerStopped
}
select {
case failures <- fmt.Errorf("Provider %s runtime: %w", name, err):
default:
}
}()
return running
}
func stopProvider(running *runningProvider) {
if running == nil {
return
}
running.cancel()
<-running.done
}
func stopAllProviders(active map[string]*runningProvider) {
var wait sync.WaitGroup
for _, running := range active {
wait.Add(1)
go func(current *runningProvider) {
defer wait.Done()
stopProvider(current)
}(running)
}
wait.Wait()
}

376
internal/controller/bootstrap/provider_supervisor_test.go Normal file
View File

@ -0,0 +1,376 @@
package bootstrap
import (
"context"
"errors"
"strings"
"sync"
"testing"
"time"
"proxy-pool/internal/config"
"proxy-pool/internal/controller/admin"
"proxy-pool/internal/domain/adminstate"
"proxy-pool/internal/platform/lifecycle"
)
func TestProviderSupervisorAppliesDisableAndConfigurationReplacement(t *testing.T) {
configuration, err := config.Load(strings.NewReader(bootstrapTestConfig))
if err != nil {
t.Fatalf("config.Load(): %v", err)
}
store, err := config.NewStore(configuration)
if err != nil {
t.Fatalf("config.NewStore(): %v", err)
}
state := &mutableProviderState{enabled: map[string]bool{"provider-a": true, "provider-b": true}}
started := make(chan providerRuntimeEvent, 4)
stopped := make(chan providerRuntimeEvent, 4)
supervisor, err := newProviderSupervisor(store, state, func(name string, upstream config.Upstream) (lifecycle.Runner, error) {
event := providerRuntimeEvent{name: name, url: upstream.API.URL}
return supervisorRunnerFunc(func(ctx context.Context) error {
started <- event
<-ctx.Done()
stopped <- event
return ctx.Err()
}), nil
}, nil, nil, nil, bootstrapTestFingerprintKey, time.Hour)
if err != nil {
t.Fatalf("newProviderSupervisor(): %v", err)
}
ctx, cancel := context.WithCancel(context.Background())
done := make(chan error, 1)
go func() { done <- supervisor.Run(ctx) }()
waitForRuntimeEvents(t, started, 2)
state.set("provider-a", false)
supervisor.Notify()
if event := waitForRuntimeEvent(t, stopped); event.name != "provider-a" {
t.Fatalf("stopped Provider = %s, want provider-a", event.name)
}
updated := store.Current()
providerB := updated.Upstreams["provider-b"]
providerB.API.URL = "https://replacement.invalid/proxies"
updated.Upstreams["provider-b"] = providerB
if !store.PublishRevision(updated, 1) {
t.Fatal("PublishRevision() rejected updated configuration")
}
supervisor.Notify()
if event := waitForRuntimeEvent(t, stopped); event.name != "provider-b" {
t.Fatalf("replaced Provider = %s, want provider-b", event.name)
}
if event := waitForRuntimeEvent(t, started); event.name != "provider-b" || event.url != providerB.API.URL {
t.Fatalf("replacement Provider = %+v", event)
}
cancel()
if err := <-done; !errors.Is(err, context.Canceled) {
t.Fatalf("Run() error = %v, want context cancellation", err)
}
}
func TestProviderSupervisorPropagatesUnexpectedRuntimeFailure(t *testing.T) {
configuration, err := config.Load(strings.NewReader(bootstrapTestConfig))
if err != nil {
t.Fatalf("config.Load(): %v", err)
}
store, err := config.NewStore(configuration)
if err != nil {
t.Fatalf("config.NewStore(): %v", err)
}
wantErr := errors.New("coordination stopped")
supervisor, err := newProviderSupervisor(store, nil, func(string, config.Upstream) (lifecycle.Runner, error) {
return supervisorRunnerFunc(func(context.Context) error { return wantErr }), nil
}, nil, nil, nil, bootstrapTestFingerprintKey, time.Hour)
if err != nil {
t.Fatalf("newProviderSupervisor(): %v", err)
}
if err := supervisor.Run(context.Background()); !errors.Is(err, wantErr) {
t.Fatalf("Run() error = %v, want %v", err, wantErr)
}
}
func TestProviderSupervisorRetainsRuntimesWhileManagementStateIsUnavailable(t *testing.T) {
configuration, err := config.Load(strings.NewReader(bootstrapTestConfig))
if err != nil {
t.Fatalf("config.Load(): %v", err)
}
store, err := config.NewStore(configuration)
if err != nil {
t.Fatalf("config.NewStore(): %v", err)
}
state := &mutableProviderState{enabled: map[string]bool{"provider-a": true, "provider-b": true}}
started := make(chan providerRuntimeEvent, 2)
stopped := make(chan providerRuntimeEvent, 2)
supervisor, err := newProviderSupervisor(store, state, func(name string, upstream config.Upstream) (lifecycle.Runner, error) {
event := providerRuntimeEvent{name: name, url: upstream.API.URL}
return supervisorRunnerFunc(func(ctx context.Context) error {
started <- event
<-ctx.Done()
stopped <- event
return ctx.Err()
}), nil
}, nil, nil, nil, bootstrapTestFingerprintKey, time.Hour)
if err != nil {
t.Fatalf("newProviderSupervisor(): %v", err)
}
ctx, cancel := context.WithCancel(context.Background())
done := make(chan error, 1)
go func() { done <- supervisor.Run(ctx) }()
waitForRuntimeEvents(t, started, 2)
state.setError(errors.New("PostgreSQL temporarily unavailable"))
supervisor.Notify()
select {
case err := <-done:
t.Fatalf("Supervisor stopped during transient state failure: %v", err)
case event := <-stopped:
t.Fatalf("Provider stopped during transient state failure: %+v", event)
case <-time.After(50 * time.Millisecond):
}
state.setError(nil)
state.set("provider-a", false)
supervisor.Notify()
if event := waitForRuntimeEvent(t, stopped); event.name != "provider-a" {
t.Fatalf("stopped Provider = %s, want provider-a", event.name)
}
cancel()
if err := <-done; !errors.Is(err, context.Canceled) {
t.Fatalf("Run() error = %v, want context cancellation", err)
}
}
func TestProviderSupervisorStopsStaleRuntimesAndLoadsAuthoritativeConfiguration(t *testing.T) {
initial, err := config.Load(strings.NewReader(bootstrapTestConfig))
if err != nil {
t.Fatalf("config.Load(): %v", err)
}
store, err := config.NewStore(initial)
if err != nil {
t.Fatalf("config.NewStore(): %v", err)
}
initialChecksum, err := config.Fingerprint(initial, bootstrapTestFingerprintKey)
if err != nil {
t.Fatalf("config.Fingerprint(initial): %v", err)
}
state := &mutableProviderState{
enabled: map[string]bool{"provider-a": true, "provider-b": true},
checksum: initialChecksum,
revision: 1,
}
source := &mutableProviderConfigurationSource{configuration: initial}
started := make(chan providerRuntimeEvent, 4)
stopped := make(chan providerRuntimeEvent, 4)
supervisor, err := newProviderSupervisor(store, state, func(name string, upstream config.Upstream) (lifecycle.Runner, error) {
event := providerRuntimeEvent{name: name, url: upstream.API.URL}
return supervisorRunnerFunc(func(ctx context.Context) error {
started <- event
<-ctx.Done()
stopped <- event
return ctx.Err()
}), nil
}, nil, nil, source, bootstrapTestFingerprintKey, time.Hour)
if err != nil {
t.Fatalf("newProviderSupervisor(): %v", err)
}
ctx, cancel := context.WithCancel(context.Background())
done := make(chan error, 1)
go func() { done <- supervisor.Run(ctx) }()
waitForRuntimeEvents(t, started, 2)
updated := store.Current()
providerA := updated.Upstreams["provider-a"]
providerA.API.URL = "https://replacement.invalid/proxies"
updated.Upstreams["provider-a"] = providerA
updatedChecksum, err := config.Fingerprint(updated, bootstrapTestFingerprintKey)
if err != nil {
t.Fatalf("config.Fingerprint(updated): %v", err)
}
state.setChecksum(updatedChecksum)
supervisor.Notify()
waitForRuntimeEvents(t, stopped, 2)
select {
case err := <-done:
t.Fatalf("Supervisor stopped for stale local configuration: %v", err)
case <-time.After(50 * time.Millisecond):
}
source.set(updated)
supervisor.Notify()
events := []providerRuntimeEvent{waitForRuntimeEvent(t, started), waitForRuntimeEvent(t, started)}
foundReplacement := false
for _, event := range events {
if event.name == "provider-a" && event.url == providerA.API.URL {
foundReplacement = true
}
}
if !foundReplacement {
t.Fatalf("started Provider runtimes = %+v, replacement missing", events)
}
if currentChecksum, err := config.Fingerprint(store.Current(), bootstrapTestFingerprintKey); err != nil || currentChecksum != updatedChecksum {
t.Fatalf("published checksum = %q, %v; want %q", currentChecksum, err, updatedChecksum)
}
cancel()
if err := <-done; !errors.Is(err, context.Canceled) {
t.Fatalf("Run() error = %v, want context cancellation", err)
}
}
func TestProviderSupervisorDoesNotPublishConfigurationOlderThanLocalRevision(t *testing.T) {
initial, err := config.Load(strings.NewReader(bootstrapTestConfig))
if err != nil {
t.Fatalf("config.Load(): %v", err)
}
store, err := config.NewStore(initial)
if err != nil {
t.Fatalf("config.NewStore(): %v", err)
}
store.PublishRevision(initial, 1)
candidate := store.Current()
providerA := candidate.Upstreams["provider-a"]
providerA.API.URL = "https://candidate.invalid/proxies"
candidate.Upstreams["provider-a"] = providerA
candidateChecksum, err := config.Fingerprint(candidate, bootstrapTestFingerprintKey)
if err != nil {
t.Fatalf("config.Fingerprint(candidate): %v", err)
}
newest := store.Current()
providerA = newest.Upstreams["provider-a"]
providerA.API.URL = "https://newest.invalid/proxies"
newest.Upstreams["provider-a"] = providerA
source := providerConfigurationSourceFunc(func(context.Context) (admin.LoadedConfiguration, error) {
if !store.PublishRevision(newest, 3) {
t.Fatal("failed to publish simulated concurrent revision")
}
return admin.LoadedConfiguration{Value: candidate, Source: "controller.yaml"}, nil
})
supervisor, err := newProviderSupervisor(
store,
&mutableProviderState{},
func(string, config.Upstream) (lifecycle.Runner, error) {
return supervisorRunnerFunc(func(context.Context) error { return nil }), nil
},
nil,
nil,
source,
bootstrapTestFingerprintKey,
time.Hour,
)
if err != nil {
t.Fatalf("newProviderSupervisor(): %v", err)
}
_, err = supervisor.synchronizeConfiguration(context.Background(), initial, adminstate.Snapshot{
Config: &adminstate.ConfigRevision{Revision: 2, Checksum: candidateChecksum},
})
if !errors.Is(err, errProviderManagementSnapshotStale) {
t.Fatalf("synchronizeConfiguration() error = %v, want stale snapshot", err)
}
if got := store.Revision(); got != 3 {
t.Fatalf("configuration revision = %d, want 3", got)
}
}
type providerRuntimeEvent struct {
name string
url string
}
type supervisorRunnerFunc func(context.Context) error
func (run supervisorRunnerFunc) Run(ctx context.Context) error { return run(ctx) }
type mutableProviderState struct {
mu sync.Mutex
enabled map[string]bool
err error
checksum string
revision uint64
}
func (state *mutableProviderState) Snapshot(context.Context) (adminstate.Snapshot, error) {
state.mu.Lock()
defer state.mu.Unlock()
if state.err != nil {
return adminstate.Snapshot{}, state.err
}
snapshot := adminstate.Snapshot{Upstreams: make([]adminstate.UpstreamState, 0, len(state.enabled))}
if state.checksum != "" {
snapshot.Config = &adminstate.ConfigRevision{
Revision: state.revision, ConfigVersion: "cfg-" + state.checksum, Checksum: state.checksum,
}
}
for name, enabled := range state.enabled {
snapshot.Upstreams = append(snapshot.Upstreams, adminstate.UpstreamState{Name: name, Enabled: enabled})
}
return snapshot, nil
}
func (state *mutableProviderState) setError(err error) {
state.mu.Lock()
defer state.mu.Unlock()
state.err = err
}
func (state *mutableProviderState) setChecksum(checksum string) {
state.mu.Lock()
defer state.mu.Unlock()
state.checksum = checksum
state.revision++
}
type mutableProviderConfigurationSource struct {
mu sync.Mutex
configuration *config.Config
}
type providerConfigurationSourceFunc func(context.Context) (admin.LoadedConfiguration, error)
func (source providerConfigurationSourceFunc) LoadConfiguration(ctx context.Context) (admin.LoadedConfiguration, error) {
return source(ctx)
}
func (source *mutableProviderConfigurationSource) LoadConfiguration(context.Context) (admin.LoadedConfiguration, error) {
source.mu.Lock()
defer source.mu.Unlock()
return admin.LoadedConfiguration{Value: source.configuration, Source: "controller.yaml"}, nil
}
func (source *mutableProviderConfigurationSource) set(configuration *config.Config) {
source.mu.Lock()
defer source.mu.Unlock()
source.configuration = configuration
}
func (state *mutableProviderState) set(name string, enabled bool) {
state.mu.Lock()
defer state.mu.Unlock()
state.enabled[name] = enabled
}
func waitForRuntimeEvents(t *testing.T, events <-chan providerRuntimeEvent, count int) {
t.Helper()
for range count {
_ = waitForRuntimeEvent(t, events)
}
}
func waitForRuntimeEvent(t *testing.T, events <-chan providerRuntimeEvent) providerRuntimeEvent {
t.Helper()
select {
case event := <-events:
return event
case <-time.After(time.Second):
t.Fatal("timed out waiting for Provider runtime event")
return providerRuntimeEvent{}
}
}

104
internal/controller/bootstrap/provider_test.go Normal file
View File

@ -0,0 +1,104 @@
package bootstrap
import (
"strconv"
"strings"
"testing"
"time"
"proxy-pool/internal/config"
"proxy-pool/internal/controller/provider"
"proxy-pool/internal/platform/credentials"
)
func TestProviderRuntimeConfigMapsValidatedUpstreamOnce(t *testing.T) {
configuration, err := config.Load(strings.NewReader(bootstrapTestConfig))
if err != nil {
t.Fatalf("config.Load(): %v", err)
}
upstream := configuration.Upstreams["provider-a"]
mapped, err := providerRuntimeConfig("provider-a", upstream)
if err != nil {
t.Fatalf("providerRuntimeConfig(): %v", err)
}
if mapped.Provider.UpstreamID != "provider-a" ||
mapped.Provider.RequestInterval != time.Second ||
mapped.Provider.Timeout != 3*time.Second ||
mapped.Provider.MaxAttempts != 3 || mapped.Provider.MaxInFlight != 1 ||
mapped.Provider.MaxTotal != 1_000 || mapped.Provider.MaxSize != 100 ||
mapped.Provider.TTL != 2*time.Minute ||
mapped.Provider.AllocationSafetyMargin != 10*time.Second {
t.Fatalf("Provider config = %+v", mapped.Provider)
}
if mapped.ReconcileInterval != time.Second ||
mapped.ReconcilePolicy.MinimumAvailableSlots != 100 ||
mapped.ReconcilePolicy.TargetAvailableSlots != 200 ||
mapped.ReconcilePolicy.ExpectedPerFetch != 10 ||
mapped.ReconcilePolicy.ExpectedSlotsPerFetch != 100 ||
mapped.ReconcilePolicy.SafetyMargin != 10*time.Second {
t.Fatalf("Reconcile config = %+v interval=%s", mapped.ReconcilePolicy, mapped.ReconcileInterval)
}
}
func TestNewProviderFleetBuildsEnabledUpstreamsInStableOrder(t *testing.T) {
configuration, err := config.Load(strings.NewReader(bootstrapTestConfig))
if err != nil {
t.Fatalf("config.Load(): %v", err)
}
configuration.Upstreams["disabled"] = config.Upstream{}
credentialStore, err := credentials.NewMemoryStore(10)
if err != nil {
t.Fatalf("NewMemoryStore(): %v", err)
}
fleet, err := newProviderFleet(configuration, ports{
activity: &stubActivityStore{}, coordinator: coordinatorStub{}, credentials: credentialStore,
})
if err != nil {
t.Fatalf("newProviderFleet(): %v", err)
}
if got := fleet.IDs(); len(got) != 2 || got[0] != "provider-a" || got[1] != "provider-b" {
t.Fatalf("Fleet IDs = %v, want [provider-a provider-b]", got)
}
configuration.Upstreams["provider-a"] = config.Upstream{}
configuration.Upstreams["provider-b"] = config.Upstream{}
fleet, err = newProviderFleet(configuration, ports{})
if err != nil || fleet != nil {
t.Fatalf("newProviderFleet(no enabled) = (%v, %v), want nil fleet", fleet, err)
}
}
func TestProviderRuntimeConfigRejectsDisabledAndOverflowingInputs(t *testing.T) {
if _, err := providerRuntimeConfig("provider-a", config.Upstream{}); err == nil {
t.Fatal("providerRuntimeConfig(disabled) error = nil")
}
overflowing := config.Upstream{
Enabled: true,
Pool: config.Pool{MaxSize: int(^uint(0) >> 1)},
Capacity: config.Capacity{
MaxConcurrencyPerProxy: int(^uint(0) >> 1),
},
Fetch: config.Fetch{EstimatedIPsPerCall: int(^uint(0) >> 1)},
}
if _, err := providerRuntimeConfig("provider-a", overflowing); err == nil {
t.Fatal("providerRuntimeConfig(overflow) error = nil")
}
configuration, err := config.Load(strings.NewReader(bootstrapTestConfig))
if err != nil {
t.Fatalf("config.Load(): %v", err)
}
tooLarge := configuration.Upstreams["provider-a"]
tooLarge.Pool.MaxSize = redisMaximumScan + 1
if _, err := providerRuntimeConfig("provider-a", tooLarge); err == nil {
t.Fatal("providerRuntimeConfig(oversized inventory) error = nil")
}
tooLarge = configuration.Upstreams["provider-a"]
tooLarge.Fetch.MaxTotal = int(provider.MaximumCoordinationInteger)
if strconv.IntSize == 64 {
tooLarge.Fetch.MaxTotal++
if _, err := providerRuntimeConfig("provider-a", tooLarge); err == nil {
t.Fatal("providerRuntimeConfig(inexact Redis integer) error = nil")
}
}
}

24
internal/controller/operations/reader.go
View File

@ -9,6 +9,7 @@ import (
"proxy-pool/internal/config"
"proxy-pool/internal/controller/admin"
"proxy-pool/internal/controller/provider"
"proxy-pool/internal/domain/activitypool"
)
@ -24,6 +25,7 @@ type ConfigurationReader interface {
type Reader struct {
configuration ConfigurationReader
inventory activitypool.StateInventoryReader
providerStats provider.StatsReader
now func() time.Time
}
@ -33,11 +35,17 @@ func NewReader(
configuration ConfigurationReader,
inventory activitypool.StateInventoryReader,
now func() time.Time,
stats ...provider.StatsReader,
) (*Reader, error) {
if nilInterface(configuration) || nilInterface(inventory) || now == nil {
if nilInterface(configuration) || nilInterface(inventory) || now == nil || len(stats) > 1 ||
(len(stats) == 1 && nilInterface(stats[0])) {
return nil, ErrInvalidReader
}
return &Reader{configuration: configuration, inventory: inventory, now: now}, nil
reader := &Reader{configuration: configuration, inventory: inventory, now: now}
if len(stats) == 1 {
reader.providerStats = stats[0]
}
return reader, nil
}
func (reader *Reader) ReadOperationalStatus(ctx context.Context) (admin.OperationalStatus, error) {
@ -69,8 +77,16 @@ func (reader *Reader) ReadOperationalStatus(ctx context.Context) (admin.Operatio
}
status := admin.OperationalStatus{Upstreams: make([]admin.UpstreamActivity, len(inventories))}
providerStats := make([]provider.Stats, len(upstreamIDs))
if reader.providerStats != nil {
providerStats = reader.providerStats.ReadProviderStats(upstreamIDs)
if len(providerStats) != len(upstreamIDs) {
return admin.OperationalStatus{}, ErrUnavailable
}
}
for index, inventory := range inventories {
if inventory.UpstreamID != upstreamIDs[index] || invalidInventory(inventory) {
if inventory.UpstreamID != upstreamIDs[index] || invalidInventory(inventory) ||
providerStats[index].UpstreamID != "" && providerStats[index].UpstreamID != upstreamIDs[index] {
return admin.OperationalStatus{}, ErrUnavailable
}
status.Upstreams[index] = admin.UpstreamActivity{
@ -80,6 +96,8 @@ func (reader *Reader) ReadOperationalStatus(ctx context.Context) (admin.Operatio
Suspect: inventory.Suspect,
Draining: inventory.Draining,
Extracted: inventory.Extracted,
ConsecutiveEmptyFetch: providerStats[index].ConsecutiveEmptyFetch,
FetchErrorCount: providerStats[index].FetchErrorCount,
}
}
return status, nil

16
internal/controller/operations/reader_test.go
View File

@ -7,6 +7,7 @@ import (
"time"
"proxy-pool/internal/config"
"proxy-pool/internal/controller/provider"
"proxy-pool/internal/domain/activitypool"
)
@ -19,7 +20,10 @@ func TestReaderMapsCurrentUpstreamsToAdminOperationalStatus(t *testing.T) {
}}
reader, err := NewReader(staticConfigurationReader{configuration: &config.Config{
Upstreams: map[string]config.Upstream{"provider-b": {}, "provider-a": {}},
}}, inventory, func() time.Time { return now })
}}, inventory, func() time.Time { return now }, staticProviderStatsReader{result: []provider.Stats{
{UpstreamID: "provider-a", ConsecutiveEmptyFetch: 3, FetchErrorCount: 4},
{UpstreamID: "provider-b", FetchErrorCount: 1},
}})
if err != nil {
t.Fatalf("NewReader() error = %v", err)
}
@ -37,7 +41,7 @@ func TestReaderMapsCurrentUpstreamsToAdminOperationalStatus(t *testing.T) {
}
first := status.Upstreams[0]
if first.Name != "provider-a" || first.Available != 11 || first.Checking != 2 || first.Suspect != 1 ||
first.Draining != 4 || first.Extracted != 8 {
first.Draining != 4 || first.Extracted != 8 || first.ConsecutiveEmptyFetch != 3 || first.FetchErrorCount != 4 {
t.Fatalf("first upstream = %+v", first)
}
if status.Upstreams[1].Name != "provider-b" || status.Upstreams[1].Available != 7 {
@ -107,6 +111,14 @@ type recordingStateInventoryReader struct {
now time.Time
}
type staticProviderStatsReader struct {
result []provider.Stats
}
func (reader staticProviderStatsReader) ReadProviderStats([]string) []provider.Stats {
return append([]provider.Stats(nil), reader.result...)
}
func (reader *recordingStateInventoryReader) ReadStateInventory(
_ context.Context,
upstreamIDs []string,

36
internal/controller/pool/fetch_budget.go
View File

@ -20,48 +20,38 @@ var (
type FetchBudgetConfig struct {
UpstreamID string
MaxSize int
MaxTotal int64
ExpectedPerFetch int
Managed int
FetchedTotal int64
}
type FetchBudgetSnapshot struct {
Managed int
PendingExpected int
FetchedTotal int64
}
// FetchBudget owns both current-inventory and cumulative-fetch accounting for
// one upstream. Reserving the expected response before I/O closes the race
// between concurrent provider calls.
// FetchBudget owns current-inventory accounting for one upstream. Reserving
// the expected response before I/O closes the race between concurrent calls;
// distributed cumulative quota belongs to the Provider coordination permit.
type FetchBudget struct {
mu sync.Mutex
upstreamID string
maxSize int
maxTotal int64
expected int
usage FetchBudgetSnapshot
}
func NewFetchBudget(config FetchBudgetConfig) (*FetchBudget, error) {
if config.UpstreamID == "" || config.MaxSize <= 0 || config.ExpectedPerFetch <= 0 ||
config.ExpectedPerFetch > config.MaxSize || config.MaxTotal < 0 ||
config.Managed < 0 || config.FetchedTotal < 0 {
return nil, ErrInvalidFetchBudget
}
if config.MaxTotal > 0 && int64(config.ExpectedPerFetch) > config.MaxTotal {
config.ExpectedPerFetch > config.MaxSize || config.Managed < 0 {
return nil, ErrInvalidFetchBudget
}
return &FetchBudget{
upstreamID: config.UpstreamID,
maxSize: config.MaxSize,
maxTotal: config.MaxTotal,
expected: config.ExpectedPerFetch,
usage: FetchBudgetSnapshot{
Managed: config.Managed,
FetchedTotal: config.FetchedTotal,
},
}, nil
}
@ -95,14 +85,7 @@ func (b *FetchBudget) FetchAllowance() int {
func (b *FetchBudget) canReserveLocked() bool {
poolRoom := b.maxSize - b.usage.Managed - b.usage.PendingExpected
if poolRoom < b.expected {
return false
}
if b.maxTotal > 0 {
totalRoom := b.maxTotal - b.usage.FetchedTotal - int64(b.usage.PendingExpected)
return totalRoom >= int64(b.expected)
}
return true
return poolRoom >= b.expected
}
func (b *FetchBudget) Snapshot() FetchBudgetSnapshot {
@ -115,8 +98,8 @@ func (b *FetchBudget) Snapshot() FetchBudgetSnapshot {
}
// SynchronizeManaged replaces the local current-inventory count with the
// authoritative activity-store observation. Pending requests and cumulative
// fetch usage remain owned by this budget.
// authoritative activity-store observation. Pending reservations remain owned
// by this budget.
func (b *FetchBudget) SynchronizeManaged(managed int) error {
if b == nil || managed < 0 {
return ErrInvalidManagedSynchronization
@ -158,11 +141,11 @@ func (p *fetchPermit) Expected() int {
return p.expected
}
func (p *fetchPermit) Complete(fetched, retained int) error {
func (p *fetchPermit) Complete(retained int) error {
if p == nil || p.budget == nil {
return ErrFetchPermitFinished
}
if fetched < 0 || retained < 0 || retained > fetched || retained > p.expected {
if retained < 0 || retained > p.expected {
return ErrInvalidFetchCompletion
}
p.budget.mu.Lock()
@ -173,7 +156,6 @@ func (p *fetchPermit) Complete(fetched, retained int) error {
p.finished = true
p.budget.usage.PendingExpected -= p.expected
p.budget.usage.Managed += retained
p.budget.usage.FetchedTotal += int64(fetched)
return nil
}

52
internal/controller/pool/fetch_budget_test.go
View File

@ -6,14 +6,12 @@ import (
"testing"
)
func TestFetchBudgetReservesExpectedCapacityAndSeparatesCounters(t *testing.T) {
func TestFetchBudgetReservesExpectedPoolCapacity(t *testing.T) {
budget, err := NewFetchBudget(FetchBudgetConfig{
UpstreamID: "provider-a",
MaxSize: 10,
MaxTotal: 20,
ExpectedPerFetch: 4,
Managed: 2,
FetchedTotal: 3,
})
if err != nil {
t.Fatalf("NewFetchBudget(): %v", err)
@ -31,21 +29,21 @@ func TestFetchBudgetReservesExpectedCapacityAndSeparatesCounters(t *testing.T) {
t.Fatalf("third ReserveFetch() = (_, %v, %v), want no capacity", ok, err)
}
if err := first.Complete(4, 3); err != nil {
if err := first.Complete(3); err != nil {
t.Fatalf("first.Complete(): %v", err)
}
if err := second.Cancel(); err != nil {
t.Fatalf("second.Cancel(): %v", err)
}
usage := budget.Snapshot()
if usage.Managed != 5 || usage.PendingExpected != 0 || usage.FetchedTotal != 7 {
t.Fatalf("Snapshot() = %+v, want managed=5 pending=0 fetched=7", usage)
if usage.Managed != 5 || usage.PendingExpected != 0 {
t.Fatalf("Snapshot() = %+v, want managed=5 pending=0", usage)
}
if err := budget.ReleaseManaged(2); err != nil {
t.Fatalf("ReleaseManaged(): %v", err)
}
if usage := budget.Snapshot(); usage.Managed != 3 || usage.FetchedTotal != 7 {
t.Fatalf("Snapshot() after release = %+v, want managed=3 fetched=7", usage)
if usage := budget.Snapshot(); usage.Managed != 3 {
t.Fatalf("Snapshot() after release = %+v, want managed=3", usage)
}
}
@ -63,7 +61,7 @@ func TestFetchBudgetRejectsManagedCounterUnderflow(t *testing.T) {
func TestFetchBudgetSynchronizesAuthoritativeManagedInventory(t *testing.T) {
budget, err := NewFetchBudget(FetchBudgetConfig{
UpstreamID: "a", MaxSize: 5, ExpectedPerFetch: 2, Managed: 3, FetchedTotal: 7,
UpstreamID: "a", MaxSize: 5, ExpectedPerFetch: 2, Managed: 3,
})
if err != nil {
t.Fatalf("NewFetchBudget(): %v", err)
@ -77,8 +75,8 @@ func TestFetchBudgetSynchronizesAuthoritativeManagedInventory(t *testing.T) {
t.Fatalf("SynchronizeManaged() error = %v, want ErrManagedSynchronizationInFlight", err)
}
usage := budget.Snapshot()
if usage.Managed != 3 || usage.PendingExpected != 2 || usage.FetchedTotal != 7 {
t.Fatalf("Snapshot() = %+v, want managed=3 pending=2 fetched=7", usage)
if usage.Managed != 3 || usage.PendingExpected != 2 {
t.Fatalf("Snapshot() = %+v, want managed=3 pending=2", usage)
}
if err := permit.Cancel(); err != nil {
t.Fatalf("Cancel(): %v", err)
@ -86,8 +84,8 @@ func TestFetchBudgetSynchronizesAuthoritativeManagedInventory(t *testing.T) {
if err := budget.SynchronizeManaged(1); err != nil {
t.Fatalf("SynchronizeManaged() after cancel: %v", err)
}
if usage := budget.Snapshot(); usage.Managed != 1 || usage.PendingExpected != 0 || usage.FetchedTotal != 7 {
t.Fatalf("Snapshot() after synchronization = %+v, want managed=1 pending=0 fetched=7", usage)
if usage := budget.Snapshot(); usage.Managed != 1 || usage.PendingExpected != 0 {
t.Fatalf("Snapshot() after synchronization = %+v, want managed=1 pending=0", usage)
}
}
@ -103,36 +101,16 @@ func TestFetchBudgetRejectsNegativeManagedSynchronization(t *testing.T) {
}
}
func TestFetchBudgetRequiresWholeExpectedBatchToFitLimits(t *testing.T) {
tests := []struct {
name string
config FetchBudgetConfig
}{
{
name: "pool size",
config: FetchBudgetConfig{
func TestFetchBudgetRequiresWholeExpectedBatchToFitPool(t *testing.T) {
budget, err := NewFetchBudget(FetchBudgetConfig{
UpstreamID: "a", MaxSize: 10, ExpectedPerFetch: 4, Managed: 7,
},
},
{
name: "cumulative total",
config: FetchBudgetConfig{
UpstreamID: "a", MaxSize: 10, MaxTotal: 5, ExpectedPerFetch: 4, FetchedTotal: 2,
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
budget, err := NewFetchBudget(tt.config)
})
if err != nil {
t.Fatalf("NewFetchBudget(): %v", err)
}
if _, ok, err := budget.ReserveFetch("a"); err != nil || ok {
t.Fatalf("ReserveFetch() = (_, %v, %v), want no capacity", ok, err)
}
})
}
}
func TestFetchBudgetConcurrentReservationsNeverExceedMaxSize(t *testing.T) {
@ -185,7 +163,7 @@ func TestFetchPermitRejectsDoubleFinishAndInvalidCounts(t *testing.T) {
if err != nil || !ok {
t.Fatalf("ReserveFetch() = (_, %v, %v), want permit", ok, err)
}
if err := permit.Complete(1, 2); !errors.Is(err, ErrInvalidFetchCompletion) {
if err := permit.Complete(3); !errors.Is(err, ErrInvalidFetchCompletion) {
t.Fatalf("Complete() error = %v, want ErrInvalidFetchCompletion", err)
}
if err := permit.Cancel(); err != nil {

17
internal/controller/pool/inventory.go Normal file
View File

@ -0,0 +1,17 @@
package pool
import (
"context"
"time"
)
type InventorySnapshot struct {
Managed int
AvailableSlots int64
}
type InventoryReader interface {
// ReadInventory returns a bounded, authoritative aggregate. Unknown Worker
// runtime must reduce capacity rather than being treated as idle.
ReadInventory(context.Context, string, time.Duration) (InventorySnapshot, error)
}

22
internal/controller/pool/reconciler.go
View File

@ -26,7 +26,6 @@ type FetchNotifier interface {
type ReconcileDecision struct {
AvailableSlots int64
PendingExpected int
FetchedTotal int64
FetchAllowance int
EffectiveSlots int64
Triggered bool
@ -62,15 +61,28 @@ func NewReconciler(policy ReconcilePolicy, budget *FetchBudget, notifier FetchNo
// Reconcile centralizes the cold-path decision. The notifier may coalesce many
// calls; Provider Reconciler atomically reserves the budget before doing I/O.
func (r *Reconciler) Reconcile(now time.Time, inventory upstream.Inventory) ReconcileDecision {
return r.reconcileSnapshot(
InventorySnapshot{AvailableSlots: inventory.AvailableSlots(now, r.policy.SafetyMargin)},
false,
)
}
func (r *Reconciler) ReconcileSnapshot(inventory InventorySnapshot) ReconcileDecision {
return r.reconcileSnapshot(inventory, true)
}
func (r *Reconciler) reconcileSnapshot(inventory InventorySnapshot, synchronizeManaged bool) ReconcileDecision {
usage := r.budget.Snapshot()
availableSlots := inventory.AvailableSlots(now, r.policy.SafetyMargin)
if synchronizeManaged && inventory.Managed >= 0 && usage.PendingExpected == 0 {
_ = r.budget.SynchronizeManaged(inventory.Managed)
usage = r.budget.Snapshot()
}
pendingSlots := saturatingMultiply(int64(usage.PendingExpected), r.slotsPerProxy)
decision := ReconcileDecision{
AvailableSlots: availableSlots,
AvailableSlots: inventory.AvailableSlots,
PendingExpected: usage.PendingExpected,
FetchedTotal: usage.FetchedTotal,
FetchAllowance: r.budget.FetchAllowance(),
EffectiveSlots: saturatingAdd(availableSlots, pendingSlots),
EffectiveSlots: saturatingAdd(inventory.AvailableSlots, pendingSlots),
}
r.mu.Lock()
if r.refilling {

58
internal/controller/pool/reconciler_test.go
View File

@ -10,7 +10,7 @@ import (
func TestPoolReconcilerSignalsOnlyWhenSlotsAreLowAndWholeFetchFits(t *testing.T) {
budget, err := NewFetchBudget(FetchBudgetConfig{
UpstreamID: "provider-a", MaxSize: 10, MaxTotal: 20, ExpectedPerFetch: 2,
UpstreamID: "provider-a", MaxSize: 10, ExpectedPerFetch: 2,
})
if err != nil {
t.Fatalf("NewFetchBudget(): %v", err)
@ -32,7 +32,6 @@ func TestPoolReconcilerSignalsOnlyWhenSlotsAreLowAndWholeFetchFits(t *testing.T)
State: proxyDomain.StateAvailable, ExpiresAt: now.Add(time.Minute), Max: 4, Active: 3,
}},
MaxSize: 10,
MaxTotal: 20,
}
decision := reconciler.Reconcile(now, inventory)
@ -44,9 +43,9 @@ func TestPoolReconcilerSignalsOnlyWhenSlotsAreLowAndWholeFetchFits(t *testing.T)
}
}
func TestPoolReconcilerUsesBudgetPendingAndCumulativeCounters(t *testing.T) {
func TestPoolReconcilerUsesPendingPoolReservation(t *testing.T) {
budget, err := NewFetchBudget(FetchBudgetConfig{
UpstreamID: "provider-a", MaxSize: 2, MaxTotal: 2, ExpectedPerFetch: 2,
UpstreamID: "provider-a", MaxSize: 2, ExpectedPerFetch: 2,
})
if err != nil {
t.Fatalf("NewFetchBudget(): %v", err)
@ -65,7 +64,7 @@ func TestPoolReconcilerUsesBudgetPendingAndCumulativeCounters(t *testing.T) {
t.Fatalf("NewReconciler(): %v", err)
}
decision := reconciler.Reconcile(time.Now(), upstream.Inventory{MaxSize: 2, MaxTotal: 2})
decision := reconciler.Reconcile(time.Now(), upstream.Inventory{MaxSize: 2})
if decision.Triggered || decision.PendingExpected != 2 || decision.FetchAllowance != 0 {
t.Fatalf("Reconcile() = %+v, want pending fetch to suppress signal", decision)
}
@ -133,7 +132,7 @@ func TestPoolReconcilerPendingEstimatePausesWithoutEndingRefillEpisode(t *testin
if decision := reconciler.Reconcile(now, inventoryWithSlots(now, 2)); decision.Triggered || decision.EffectiveSlots != 8 {
t.Fatalf("pending Reconcile() = %+v, want paused at target estimate", decision)
}
if err := permit.Complete(2, 1); err != nil {
if err := permit.Complete(1); err != nil {
t.Fatalf("Complete(): %v", err)
}
if decision := reconciler.Reconcile(now, inventoryWithSlots(now, 5)); !decision.Triggered {
@ -141,6 +140,53 @@ func TestPoolReconcilerPendingEstimatePausesWithoutEndingRefillEpisode(t *testin
}
}
func TestPoolReconcilerConsumesAuthoritativeInventorySnapshot(t *testing.T) {
budget, err := NewFetchBudget(FetchBudgetConfig{
UpstreamID: "provider-a", MaxSize: 10, ExpectedPerFetch: 2,
})
if err != nil {
t.Fatalf("NewFetchBudget(): %v", err)
}
notifier := &recordingFetchNotifier{}
reconciler, err := NewReconciler(ReconcilePolicy{
MinimumAvailableSlots: 3, TargetAvailableSlots: 8,
ExpectedPerFetch: 2, ExpectedSlotsPerFetch: 2,
}, budget, notifier)
if err != nil {
t.Fatalf("NewReconciler(): %v", err)
}
decision := reconciler.ReconcileSnapshot(InventorySnapshot{Managed: 9, AvailableSlots: 2})
if decision.Triggered || decision.AvailableSlots != 2 || decision.FetchAllowance != 0 {
t.Fatalf("ReconcileSnapshot() = %+v, want authoritative managed inventory to close budget", decision)
}
if usage := budget.Snapshot(); usage.Managed != 9 {
t.Fatalf("FetchBudget.Managed = %d, want 9", usage.Managed)
}
}
func TestPoolReconcilerLegacyInventoryDoesNotClearManagedBudget(t *testing.T) {
budget, err := NewFetchBudget(FetchBudgetConfig{
UpstreamID: "provider-a", MaxSize: 10, ExpectedPerFetch: 2, Managed: 9,
})
if err != nil {
t.Fatalf("NewFetchBudget(): %v", err)
}
reconciler, err := NewReconciler(ReconcilePolicy{
MinimumAvailableSlots: 3, TargetAvailableSlots: 8,
ExpectedPerFetch: 2, ExpectedSlotsPerFetch: 2,
}, budget, &recordingFetchNotifier{})
if err != nil {
t.Fatalf("NewReconciler(): %v", err)
}
decision := reconciler.Reconcile(time.Now(), upstream.Inventory{})
if decision.Triggered || decision.FetchAllowance != 0 {
t.Fatalf("Reconcile(legacy) = %+v, want preserved managed budget", decision)
}
if usage := budget.Snapshot(); usage.Managed != 9 {
t.Fatalf("FetchBudget.Managed = %d, want 9", usage.Managed)
}
}
func inventoryWithSlots(now time.Time, slots int64) upstream.Inventory {
return upstream.Inventory{Proxies: []upstream.ProxyCapacity{{
State: proxyDomain.StateAvailable, ExpiresAt: now.Add(time.Minute), Max: slots,

13
internal/controller/provider/coordination.go
View File

@ -13,10 +13,13 @@ var (
ErrLeaderWorkStopped = errors.New("provider leader work stopped")
)
const MaximumCoordinationInteger = int64(1<<53 - 1)
type CoordinationLimits struct {
RequestInterval time.Duration
MaxInFlight int
MaxAttemptDuration time.Duration
MaxTotal int64
}
type Fence struct {
@ -32,10 +35,14 @@ type Coordinator interface {
type LeaderSession interface {
Fence() Fence
AcquireFetch(context.Context) (RequestPermit, error)
AcquireFetch(context.Context, int) (RequestPermit, bool, error)
}
type RequestPermit interface {
// Release is idempotent. A failed release expires automatically in storage.
Release(context.Context) error
// Complete atomically releases in-flight capacity and charges the actual
// successful candidate count. Repeated calls are idempotent.
Complete(context.Context, int) error
// Cancel releases a reservation that is known not to have consumed Provider
// quota. A crashed or abandoned reservation is conservatively charged.
Cancel(context.Context) error
}

64
internal/controller/provider/fleet.go Normal file
View File

@ -0,0 +1,64 @@
package provider
import (
"context"
"errors"
"sort"
"proxy-pool/internal/platform/lifecycle"
)
var ErrInvalidFleet = errors.New("invalid Provider fleet")
type Fleet struct {
runtimes []*UpstreamRuntime
group *lifecycle.Group
}
func NewFleet(runtimes ...*UpstreamRuntime) (*Fleet, error) {
if len(runtimes) == 0 {
return nil, ErrInvalidFleet
}
seen := make(map[string]struct{}, len(runtimes))
owned := make([]*UpstreamRuntime, len(runtimes))
for index, runtime := range runtimes {
if runtime == nil || runtime.config.Provider.UpstreamID == "" {
return nil, ErrInvalidFleet
}
if _, exists := seen[runtime.config.Provider.UpstreamID]; exists {
return nil, ErrInvalidFleet
}
seen[runtime.config.Provider.UpstreamID] = struct{}{}
owned[index] = runtime
}
sort.Slice(owned, func(left, right int) bool {
return owned[left].ID() < owned[right].ID()
})
runners := make([]lifecycle.Runner, len(owned))
for index, runtime := range owned {
runners[index] = runtime
}
group, err := lifecycle.NewGroup(runners...)
if err != nil {
return nil, errors.Join(ErrInvalidFleet, err)
}
return &Fleet{runtimes: owned, group: group}, nil
}
func (fleet *Fleet) Run(ctx context.Context) error {
if fleet == nil || ctx == nil || len(fleet.runtimes) == 0 || fleet.group == nil {
return ErrInvalidFleet
}
return fleet.group.Run(ctx)
}
func (fleet *Fleet) IDs() []string {
if fleet == nil {
return nil
}
ids := make([]string, len(fleet.runtimes))
for index, runtime := range fleet.runtimes {
ids[index] = runtime.ID()
}
return ids
}

101
internal/controller/provider/fleet_test.go Normal file
View File

@ -0,0 +1,101 @@
package provider
import (
"context"
"errors"
"testing"
"time"
"proxy-pool/internal/controller/pool"
"proxy-pool/internal/domain/activitypool"
proxyDomain "proxy-pool/internal/domain/proxy"
)
func TestFleetRunsAllUpstreamsAndCancelsSiblingsOnFailure(t *testing.T) {
started := make(chan string, 2)
siblingCancelled := make(chan struct{}, 1)
first := newFleetRuntime(t, "provider-a", coordinatorFunc(func(
context.Context, string, CoordinationLimits, func(context.Context, LeaderSession) error,
) error {
started <- "provider-a"
return errors.New("provider-a stopped")
}))
second := newFleetRuntime(t, "provider-b", coordinatorFunc(func(
ctx context.Context, _ string, _ CoordinationLimits, _ func(context.Context, LeaderSession) error,
) error {
started <- "provider-b"
<-ctx.Done()
siblingCancelled <- struct{}{}
return ctx.Err()
}))
fleet, err := NewFleet(first, second)
if err != nil {
t.Fatalf("NewFleet(): %v", err)
}
err = fleet.Run(context.Background())
if err == nil || err.Error() != "provider-a stopped" {
t.Fatalf("Run() error = %v, want provider-a failure", err)
}
seen := map[string]bool{<-started: true, <-started: true}
if !seen["provider-a"] || !seen["provider-b"] {
t.Fatalf("started upstreams = %v", seen)
}
select {
case <-siblingCancelled:
case <-time.After(time.Second):
t.Fatal("sibling runtime was not cancelled")
}
}
func TestNewFleetRejectsEmptyNilAndDuplicateUpstreams(t *testing.T) {
valid := newFleetRuntime(t, "provider-a", coordinatorFunc(func(
ctx context.Context, _ string, _ CoordinationLimits, _ func(context.Context, LeaderSession) error,
) error {
return ctx.Err()
}))
tests := []struct {
name string
runtimes []*UpstreamRuntime
}{
{name: "empty"},
{name: "nil", runtimes: []*UpstreamRuntime{nil}},
{name: "duplicate", runtimes: []*UpstreamRuntime{valid, valid}},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
fleet, err := NewFleet(test.runtimes...)
if err == nil || fleet != nil {
t.Fatalf("NewFleet() = (%v, %v), want invalid fleet", fleet, err)
}
})
}
}
func newFleetRuntime(t *testing.T, upstreamID string, coordinator Coordinator) *UpstreamRuntime {
t.Helper()
runtime, err := NewUpstreamRuntime(UpstreamRuntimeConfig{
Provider: runtimeProviderConfig(upstreamID),
ReconcilePolicy: runtimeReconcilePolicy(),
ReconcileInterval: time.Second,
}, UpstreamRuntimeDependencies{
Coordinator: coordinator,
Inventory: inventoryReaderFunc(func(context.Context, string, time.Duration) (pool.InventorySnapshot, error) {
return pool.InventorySnapshot{}, nil
}),
Adapter: adapterFunc(func(context.Context) (FetchResponse, error) {
return FetchResponse{Body: []byte("fixture")}, nil
}),
Parser: parserFunc(func(context.Context, []byte) ([]proxyDomain.Proxy, error) {
return []proxyDomain.Proxy{{ID: "proxy-1"}}, nil
}),
Activity: activitySinkFunc(func(context.Context, string, activitypool.FetchedBatch) (activitypool.UpsertResult, error) {
return activitypool.UpsertResult{Accepted: 1, Inserted: 1}, nil
}),
Results: resultRecorderFunc(func(Result) {}),
})
if err != nil {
t.Fatalf("NewUpstreamRuntime(%s): %v", upstreamID, err)
}
return runtime
}

7
internal/controller/provider/ports.go
View File

@ -27,9 +27,11 @@ type RetryableError interface {
Retryable() bool
}
// Parser must be safe for concurrent calls and must honor context cancellation.
// Parser owns any transient resources attached to parsed candidates. It must
// be safe for concurrent calls and must honor context cancellation.
type Parser interface {
Parse(context.Context, []byte) ([]proxyDomain.Proxy, error)
ReleaseCandidates([]proxyDomain.Proxy)
}
type Result struct {
@ -42,7 +44,8 @@ type Result struct {
}
type ResultRecorder interface {
// Record may be called concurrently and must not retain mutable result data.
// Record may be called concurrently, must return promptly, and must not
// retain mutable result data.
Record(Result)
}

122
internal/controller/provider/reconciler.go
View File

@ -19,6 +19,7 @@ type Config struct {
Timeout time.Duration
MaxAttempts int
MaxInFlight int
MaxTotal int64
MaxSize int
TTL time.Duration
AllocationSafetyMargin time.Duration
@ -31,6 +32,8 @@ type RetryConfig struct {
Jitter int
}
const requestPermitSettlementTimeout = 5 * time.Second
type Reconciler struct {
config Config
ports Ports
@ -64,7 +67,7 @@ func NewReconciler(config Config, ports Ports, runtimes ...Runtime) (*Reconciler
return nil, fmt.Errorf("new provider reconciler: upstream ID is required")
}
if config.RequestInterval < 0 || config.Timeout <= 0 || config.MaxAttempts <= 0 ||
config.MaxInFlight <= 0 || config.MaxSize <= 0 {
config.MaxInFlight <= 0 || config.MaxTotal < 0 || config.MaxSize <= 0 {
return nil, fmt.Errorf("new provider reconciler: fetch limits must be positive")
}
if config.TTL < 0 || config.AllocationSafetyMargin < 0 ||
@ -112,7 +115,10 @@ func (r *Reconciler) Notify() {
r.signal.Notify()
}
func (r *Reconciler) Run(ctx context.Context) error {
func (r *Reconciler) RunLeader(ctx context.Context, session LeaderSession) error {
if ctx == nil || session == nil {
return ErrInvalidCoordination
}
var workers sync.WaitGroup
defer workers.Wait()
for {
@ -131,17 +137,57 @@ func (r *Reconciler) Run(ctx context.Context) error {
go func() {
defer workers.Done()
defer func() { <-r.inFlight }()
r.reconcile(ctx)
r.reconcile(ctx, session)
}()
}
}
func (r *Reconciler) reconcile(ctx context.Context) {
func (r *Reconciler) reconcile(ctx context.Context, session LeaderSession) {
permit, available, err := r.ports.Capacity.ReserveFetch(r.config.UpstreamID)
if err != nil {
r.ports.Results.Record(Result{
UpstreamID: r.config.UpstreamID,
Class: upstream.FetchError,
Err: fmt.Errorf("reserve fetch capacity: %w", err),
Attempt: 1,
})
return
}
if !available {
return
}
if permit == nil || permit.Expected() <= 0 {
if permit != nil {
_ = permit.Cancel()
}
r.ports.Results.Record(Result{
UpstreamID: r.config.UpstreamID,
Class: upstream.FetchError,
Err: fmt.Errorf("reserve fetch capacity: invalid permit"),
Attempt: 1,
})
return
}
permitFinished := false
defer func() {
if !permitFinished {
_ = permit.Cancel()
}
}()
for attempt := 1; attempt <= r.config.MaxAttempts; attempt++ {
response, result, retryable, ok := r.fetchAttempt(ctx, attempt)
response, result, retryable, ok := r.fetchAttempt(ctx, session, permit, attempt)
if !ok {
return
}
if result.Class != upstream.FetchError {
if capacityErr := permit.Complete(result.NewCount); capacityErr != nil {
result.Class = upstream.FetchError
result.Err = errors.Join(result.Err, capacityErr)
} else {
permitFinished = true
}
}
r.ports.Results.Record(result)
if result.Class != upstream.FetchError || !retryable || attempt == r.config.MaxAttempts {
return
@ -157,38 +203,42 @@ func (r *Reconciler) reconcile(ctx context.Context) {
}
}
func (r *Reconciler) fetchAttempt(ctx context.Context, attempt int) (FetchResponse, Result, bool, bool) {
func (r *Reconciler) fetchAttempt(
ctx context.Context,
session LeaderSession,
permit upstream.FetchPermit,
attempt int,
) (FetchResponse, Result, bool, bool) {
if err := r.waitForRequestSlot(ctx); err != nil {
return FetchResponse{}, Result{}, false, false
}
permit, available, err := r.ports.Capacity.ReserveFetch(r.config.UpstreamID)
requestPermit, available, err := session.AcquireFetch(ctx, permit.Expected())
if err != nil {
resultErr := fmt.Errorf("reserve fetch capacity: %w", err)
if ctx.Err() != nil || errors.Is(err, ErrLeadershipLost) {
return FetchResponse{}, Result{}, false, false
}
return FetchResponse{}, Result{
UpstreamID: r.config.UpstreamID,
Class: upstream.FetchError,
Err: resultErr,
Err: fmt.Errorf("acquire distributed fetch capacity: %w", err),
Attempt: attempt,
}, false, true
}
if !available {
return FetchResponse{}, Result{}, false, false
}
if permit == nil || permit.Expected() <= 0 {
if permit != nil {
_ = permit.Cancel()
}
if requestPermit == nil {
return FetchResponse{}, Result{
UpstreamID: r.config.UpstreamID,
Class: upstream.FetchError,
Err: fmt.Errorf("reserve fetch capacity: invalid permit"),
Err: fmt.Errorf("acquire distributed fetch capacity: invalid permit"),
Attempt: attempt,
}, false, true
}
permitFinished := false
requestSettlementAttempted := false
defer func() {
if !permitFinished {
_ = permit.Cancel()
if !requestSettlementAttempted {
_ = r.settleRequestPermit(requestPermit, false, 0)
}
}()
@ -200,13 +250,23 @@ func (r *Reconciler) fetchAttempt(ctx context.Context, attempt int) (FetchRespon
defer cancel()
response, callErr := r.ports.Adapter.Fetch(callCtx)
var parseErr, candidateErr, capacityErr error
var parseErr, candidateErr, coordinationErr error
var validCount, newCount int
if callErr == nil {
if callErr != nil {
requestSettlementAttempted = true
coordinationErr = r.settleRequestPermit(requestPermit, true, permit.Expected())
} else {
candidates, err := r.ports.Parser.Parse(callCtx, response.Body)
defer r.ports.Parser.ReleaseCandidates(candidates)
parseErr = err
validCount = len(candidates)
if parseErr == nil && validCount > 0 {
charged := validCount
if parseErr != nil {
charged = permit.Expected()
}
requestSettlementAttempted = true
coordinationErr = r.settleRequestPermit(requestPermit, true, charged)
if parseErr == nil && coordinationErr == nil && validCount > 0 {
retained := candidates
if expected := permit.Expected(); expected < len(retained) {
retained = retained[:expected]
@ -222,12 +282,13 @@ func (r *Reconciler) fetchAttempt(ctx context.Context, attempt int) (FetchRespon
newCount = upserted.Inserted
}
}
if callErr == nil && parseErr == nil && candidateErr == nil {
capacityErr = permit.Complete(validCount, newCount)
permitFinished = capacityErr == nil
}
resultErr := errors.Join(callErr, parseErr, candidateErr, capacityErr)
class := upstream.ClassifyFetchResult(callErr, errors.Join(parseErr, candidateErr, capacityErr), validCount, newCount)
resultErr := errors.Join(callErr, parseErr, candidateErr, coordinationErr)
class := upstream.ClassifyFetchResult(
callErr,
errors.Join(parseErr, candidateErr, coordinationErr),
validCount,
newCount,
)
return response, Result{
UpstreamID: r.config.UpstreamID,
Class: class,
@ -238,6 +299,15 @@ func (r *Reconciler) fetchAttempt(ctx context.Context, attempt int) (FetchRespon
}, isRetryable(callErr) || parseErr != nil, true
}
func (r *Reconciler) settleRequestPermit(permit RequestPermit, complete bool, fetched int) error {
ctx, cancel := context.WithTimeout(context.Background(), requestPermitSettlementTimeout)
defer cancel()
if complete {
return permit.Complete(ctx, fetched)
}
return permit.Cancel(ctx)
}
func isRetryable(err error) bool {
if err == nil {
return false

300
internal/controller/provider/reconciler_test.go
View File

@ -8,6 +8,7 @@ import (
"testing"
"time"
controllerPool "proxy-pool/internal/controller/pool"
"proxy-pool/internal/domain/activitypool"
proxyDomain "proxy-pool/internal/domain/proxy"
"proxy-pool/internal/domain/upstream"
@ -89,7 +90,7 @@ func TestReconcilerCoalescesConcurrentNotifications(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
done := make(chan error, 1)
go func() { done <- reconciler.Run(ctx) }()
go func() { done <- reconciler.RunLeader(ctx, unlimitedLeaderSession{}) }()
select {
case got := <-result:
@ -130,7 +131,7 @@ func TestReconcilerEnforcesRequestInterval(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
done := make(chan error, 1)
go func() { done <- reconciler.Run(ctx) }()
go func() { done <- reconciler.RunLeader(ctx, unlimitedLeaderSession{}) }()
reconciler.Notify()
<-results
@ -183,7 +184,7 @@ func TestReconcilerRetriesErrorsWithExponentialBackoffAndJitter(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
done := make(chan error, 1)
go func() { done <- reconciler.Run(ctx) }()
go func() { done <- reconciler.RunLeader(ctx, unlimitedLeaderSession{}) }()
reconciler.Notify()
wantClasses := []upstream.FetchClass{
@ -291,7 +292,7 @@ func TestReconcilerHonorsRetryAfterBeforeBackoff(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
done := make(chan error, 1)
go func() { done <- reconciler.Run(ctx) }()
go func() { done <- reconciler.RunLeader(ctx, unlimitedLeaderSession{}) }()
reconciler.Notify()
<-results
<-results
@ -332,7 +333,7 @@ func TestReconcilerCapsRetryAfter(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
done := make(chan error, 1)
go func() { done <- reconciler.Run(ctx) }()
go func() { done <- reconciler.RunLeader(ctx, unlimitedLeaderSession{}) }()
reconciler.Notify()
<-results
<-results
@ -417,7 +418,7 @@ func TestReconcilerDropsNotificationFanoutWhileFetchIsInFlight(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
done := make(chan error, 1)
go func() { done <- reconciler.Run(ctx) }()
go func() { done <- reconciler.RunLeader(ctx, unlimitedLeaderSession{}) }()
reconciler.Notify()
<-started
@ -473,8 +474,8 @@ func TestReconcilerEnforcesMaxInFlightAcrossRunConsumers(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
done := make(chan error, 2)
go func() { done <- reconciler.Run(ctx) }()
go func() { done <- reconciler.Run(ctx) }()
go func() { done <- reconciler.RunLeader(ctx, unlimitedLeaderSession{}) }()
go func() { done <- reconciler.RunLeader(ctx, unlimitedLeaderSession{}) }()
reconciler.Notify()
<-started
@ -519,7 +520,7 @@ func TestReconcilerUsesConfiguredMaxInFlight(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
done := make(chan error, 1)
go func() { done <- reconciler.Run(ctx) }()
go func() { done <- reconciler.RunLeader(ctx, unlimitedLeaderSession{}) }()
reconciler.Notify()
<-started
reconciler.Notify()
@ -545,6 +546,7 @@ func TestReconcilerDoesNotRefetchWhenActivitySinkFails(t *testing.T) {
clock := newFakeClock(time.Date(2026, 7, 28, 12, 0, 0, 0, time.UTC))
sleeper := &fakeSleeper{clock: clock}
results := make(chan Result, 3)
globalCompleted := make(chan int, 1)
var calls atomic.Int64
ports := successfulPorts(func() { calls.Add(1) }, results)
ports.Activity = activitySinkFunc(func(context.Context, string, activitypool.FetchedBatch) (activitypool.UpsertResult, error) {
@ -562,13 +564,20 @@ func TestReconcilerDoesNotRefetchWhenActivitySinkFails(t *testing.T) {
t.Fatalf("NewReconciler(): %v", err)
}
result := runSingleReconcile(t, reconciler, results)
result := runSingleReconcile(t, reconciler, results, leaderSessionFunc(
func(context.Context, int) (RequestPermit, bool, error) {
return &recordingRequestPermit{completed: globalCompleted}, true, nil
},
))
if result.Class != upstream.FetchError {
t.Fatalf("result class = %q, want %q", result.Class, upstream.FetchError)
}
if got := calls.Load(); got != 1 {
t.Fatalf("ProviderAdapter.Fetch() calls = %d, want 1", got)
}
if got := <-globalCompleted; got != 1 {
t.Fatalf("global charged count = %d, want fetched=1", got)
}
if got := sleeper.Durations(); len(got) != 0 {
t.Fatalf("Sleep durations = %v, want no retry backoff", got)
}
@ -623,7 +632,7 @@ func TestReconcilerDoesNotCallProviderWithoutFetchCapacity(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
done := make(chan error, 1)
go func() { done <- reconciler.Run(ctx) }()
go func() { done <- reconciler.RunLeader(ctx, unlimitedLeaderSession{}) }()
reconciler.Notify()
time.Sleep(20 * time.Millisecond)
cancel()
@ -640,13 +649,116 @@ func TestReconcilerDoesNotCallProviderWithoutFetchCapacity(t *testing.T) {
}
}
func TestReconcilerCompletesFetchPermitWithFetchedAndRetainedCounts(t *testing.T) {
func TestReconcilerDoesNotCallProviderWhenDistributedQuotaIsExhausted(t *testing.T) {
results := make(chan Result, 1)
completed := make(chan fetchCompletion, 1)
var calls atomic.Int64
ports := successfulPorts(func() { calls.Add(1) }, results)
ports.Capacity = fetchCapacityFunc(func(string) (upstream.FetchPermit, bool, error) {
return &recordingFetchPermit{expected: 2}, true, nil
})
reconciler, err := NewReconciler(Config{
UpstreamID: "provider-a", Timeout: time.Second, MaxAttempts: 1, MaxInFlight: 1, MaxSize: 100,
}, ports)
if err != nil {
t.Fatalf("NewReconciler(): %v", err)
}
session := leaderSessionFunc(func(_ context.Context, expected int) (RequestPermit, bool, error) {
if expected != 2 {
t.Errorf("distributed expected = %d, want 2", expected)
}
return nil, false, nil
})
ctx, cancel := context.WithCancel(context.Background())
done := make(chan error, 1)
go func() { done <- reconciler.RunLeader(ctx, session) }()
reconciler.Notify()
time.Sleep(20 * time.Millisecond)
cancel()
if err := <-done; err != nil {
t.Fatalf("RunLeader(): %v", err)
}
if got := calls.Load(); got != 0 {
t.Fatalf("ProviderAdapter.Fetch() calls = %d, want 0", got)
}
select {
case result := <-results:
t.Fatalf("unexpected fetch result: %+v", result)
default:
}
}
func TestReconcilerChargesExpectedWhenSuccessfulResponseCannotBeParsed(t *testing.T) {
results := make(chan Result, 1)
globalCompleted := make(chan int, 1)
localCancelled := make(chan struct{}, 1)
ports := successfulPorts(func() {}, results)
ports.Parser = parserFunc(func(context.Context, []byte) ([]proxyDomain.Proxy, error) {
return []proxyDomain.Proxy{{ID: "p1"}, {ID: "p2"}, {ID: "p3"}}, nil
return nil, errors.New("invalid provider payload")
})
ports.Capacity = fetchCapacityFunc(func(string) (upstream.FetchPermit, bool, error) {
return &recordingFetchPermit{expected: 2, cancelled: localCancelled}, true, nil
})
reconciler, err := NewReconciler(Config{
UpstreamID: "provider-a", Timeout: time.Second, MaxAttempts: 1, MaxInFlight: 1, MaxSize: 100,
}, ports)
if err != nil {
t.Fatalf("NewReconciler(): %v", err)
}
session := leaderSessionFunc(func(context.Context, int) (RequestPermit, bool, error) {
return &recordingRequestPermit{completed: globalCompleted}, true, nil
})
result := runSingleReconcile(t, reconciler, results, session)
if result.Class != upstream.FetchError {
t.Fatalf("result class = %q, want %q", result.Class, upstream.FetchError)
}
if got := <-globalCompleted; got != 2 {
t.Fatalf("global charged count = %d, want expected=2", got)
}
select {
case <-localCancelled:
default:
t.Fatal("local pool reservation was not cancelled")
}
}
func TestReconcilerConservativelyChargesExpectedWhenProviderCallOutcomeIsUnknown(t *testing.T) {
results := make(chan Result, 1)
globalCompleted := make(chan int, 1)
ports := successfulPorts(func() {}, results)
ports.Adapter = adapterFunc(func(context.Context) (FetchResponse, error) {
return FetchResponse{}, errors.New("provider connection failed")
})
reconciler, err := NewReconciler(Config{
UpstreamID: "provider-a", Timeout: time.Second, MaxAttempts: 1, MaxInFlight: 1, MaxSize: 100,
}, ports)
if err != nil {
t.Fatalf("NewReconciler(): %v", err)
}
session := leaderSessionFunc(func(context.Context, int) (RequestPermit, bool, error) {
return &recordingRequestPermit{completed: globalCompleted}, true, nil
})
result := runSingleReconcile(t, reconciler, results, session)
if result.Class != upstream.FetchError {
t.Fatalf("result class = %q, want %q", result.Class, upstream.FetchError)
}
if got := <-globalCompleted; got <= 0 {
t.Fatalf("global charged count = %d, want conservative expected count", got)
}
}
func TestReconcilerChargesFetchedGloballyAndCompletesRetainedLocally(t *testing.T) {
results := make(chan Result, 1)
localCompleted := make(chan fetchCompletion, 1)
globalCompleted := make(chan int, 1)
released := make(chan []proxyDomain.Proxy, 1)
ports := successfulPorts(func() {}, results)
ports.Parser = &recordingCandidateParser{
candidates: []proxyDomain.Proxy{{ID: "p1"}, {ID: "p2"}, {ID: "p3"}},
released: released,
}
ports.Activity = activitySinkFunc(func(_ context.Context, _ string, batch activitypool.FetchedBatch) (activitypool.UpsertResult, error) {
if len(batch.Proxies) != 2 {
t.Errorf("activity batch proxies = %d, want permit limit 2", len(batch.Proxies))
@ -654,7 +766,7 @@ func TestReconcilerCompletesFetchPermitWithFetchedAndRetainedCounts(t *testing.T
return activitypool.UpsertResult{Accepted: 2, Inserted: 1}, nil
})
ports.Capacity = fetchCapacityFunc(func(string) (upstream.FetchPermit, bool, error) {
return &recordingFetchPermit{expected: 2, completed: completed}, true, nil
return &recordingFetchPermit{expected: 2, completed: localCompleted}, true, nil
})
reconciler, err := NewReconciler(Config{
UpstreamID: "provider-a", Timeout: time.Second, MaxAttempts: 1, MaxInFlight: 1, MaxSize: 100,
@ -663,12 +775,30 @@ func TestReconcilerCompletesFetchPermitWithFetchedAndRetainedCounts(t *testing.T
t.Fatalf("NewReconciler(): %v", err)
}
result := runSingleReconcile(t, reconciler, results)
result := runSingleReconcile(t, reconciler, results, leaderSessionFunc(
func(_ context.Context, expected int) (RequestPermit, bool, error) {
if expected != 2 {
t.Errorf("distributed expected = %d, want 2", expected)
}
return &recordingRequestPermit{completed: globalCompleted}, true, nil
},
))
if result.ValidCount != 3 || result.NewCount != 1 {
t.Fatalf("result = %+v, want valid=3 new=1", result)
}
if got := <-completed; got.fetched != 3 || got.retained != 1 {
t.Fatalf("fetch completion = %+v, want fetched=3 retained=1", got)
if got := <-globalCompleted; got != 3 {
t.Fatalf("global fetch completion = %d, want fetched=3", got)
}
if got := <-localCompleted; got.retained != 1 {
t.Fatalf("local fetch completion = %+v, want retained=1", got)
}
select {
case got := <-released:
if len(got) != 3 {
t.Fatalf("released candidates = %d, want all 3 parsed candidates", len(got))
}
case <-time.After(time.Second):
t.Fatal("parsed candidates were not released")
}
}
@ -699,7 +829,7 @@ func TestReconcilerRetriesParserErrorsAsFetchErrors(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
done := make(chan error, 1)
go func() { done <- reconciler.Run(ctx) }()
go func() { done <- reconciler.RunLeader(ctx, unlimitedLeaderSession{}) }()
reconciler.Notify()
first, second := <-results, <-results
cancel()
@ -717,6 +847,53 @@ func TestReconcilerRetriesParserErrorsAsFetchErrors(t *testing.T) {
}
}
func TestReconcilerKeepsLocalPoolReservationAcrossRetryBackoff(t *testing.T) {
results := make(chan Result, 2)
budget, err := controllerPool.NewFetchBudget(controllerPool.FetchBudgetConfig{
UpstreamID: "provider-a", MaxSize: 10, ExpectedPerFetch: 1,
})
if err != nil {
t.Fatalf("NewFetchBudget(): %v", err)
}
var parses atomic.Int64
ports := successfulPorts(func() {}, results)
ports.Capacity = budget
ports.Parser = parserFunc(func(context.Context, []byte) ([]proxyDomain.Proxy, error) {
if parses.Add(1) == 1 {
return nil, errors.New("temporary parser failure")
}
return []proxyDomain.Proxy{{ID: "proxy-1"}}, nil
})
reconciler, err := NewReconciler(Config{
UpstreamID: "provider-a", Timeout: time.Second, MaxAttempts: 2,
MaxInFlight: 1, MaxSize: 10,
Retry: RetryConfig{Initial: time.Millisecond, Max: time.Millisecond},
}, ports, Runtime{Sleeper: sleeperFunc(func(context.Context, time.Duration) error {
if got := budget.Snapshot().PendingExpected; got != 1 {
t.Errorf("PendingExpected during retry backoff = %d, want 1", got)
}
return nil
})})
if err != nil {
t.Fatalf("NewReconciler(): %v", err)
}
ctx, cancel := context.WithCancel(context.Background())
done := make(chan error, 1)
go func() { done <- reconciler.RunLeader(ctx, unlimitedLeaderSession{}) }()
reconciler.Notify()
<-results
<-results
cancel()
if err := <-done; err != nil {
t.Fatalf("RunLeader(): %v", err)
}
usage := budget.Snapshot()
if usage.PendingExpected != 0 || usage.Managed != 1 {
t.Fatalf("FetchBudget snapshot = %+v, want pending=0 managed=1", usage)
}
}
func TestNewReconcilerRejectsInvalidSchedulingConfiguration(t *testing.T) {
results := make(chan Result, 1)
ports := successfulPorts(func() {}, results)
@ -744,11 +921,23 @@ func TestNewReconcilerRejectsInvalidSchedulingConfiguration(t *testing.T) {
}
}
func runSingleReconcile(t *testing.T, reconciler *Reconciler, results <-chan Result) Result {
func runSingleReconcile(
t *testing.T,
reconciler *Reconciler,
results <-chan Result,
sessions ...LeaderSession,
) Result {
t.Helper()
session := LeaderSession(unlimitedLeaderSession{})
if len(sessions) > 1 {
t.Fatal("runSingleReconcile accepts at most one LeaderSession")
}
if len(sessions) == 1 {
session = sessions[0]
}
ctx, cancel := context.WithCancel(context.Background())
done := make(chan error, 1)
go func() { done <- reconciler.Run(ctx) }()
go func() { done <- reconciler.RunLeader(ctx, session) }()
reconciler.Notify()
var result Result
select {
@ -821,6 +1010,12 @@ func (s *errorSleeper) Sleep(context.Context, time.Duration) error {
return errors.New("unexpected sleep")
}
type sleeperFunc func(context.Context, time.Duration) error
func (f sleeperFunc) Sleep(ctx context.Context, duration time.Duration) error {
return f(ctx, duration)
}
func (s *fakeSleeper) Sleep(ctx context.Context, duration time.Duration) error {
if err := ctx.Err(); err != nil {
return err
@ -848,6 +1043,21 @@ func (f parserFunc) Parse(ctx context.Context, body []byte) ([]proxyDomain.Proxy
return f(ctx, body)
}
func (parserFunc) ReleaseCandidates([]proxyDomain.Proxy) {}
type recordingCandidateParser struct {
candidates []proxyDomain.Proxy
released chan<- []proxyDomain.Proxy
}
func (parser *recordingCandidateParser) Parse(context.Context, []byte) ([]proxyDomain.Proxy, error) {
return append([]proxyDomain.Proxy(nil), parser.candidates...), nil
}
func (parser *recordingCandidateParser) ReleaseCandidates(candidates []proxyDomain.Proxy) {
parser.released <- append([]proxyDomain.Proxy(nil), candidates...)
}
type activitySinkFunc func(context.Context, string, activitypool.FetchedBatch) (activitypool.UpsertResult, error)
func (f activitySinkFunc) UpsertFetched(ctx context.Context, upstreamID string, batch activitypool.FetchedBatch) (activitypool.UpsertResult, error) {
@ -867,26 +1077,66 @@ func (unlimitedFetchCapacity) ReserveFetch(string) (upstream.FetchPermit, bool,
}
type fetchCompletion struct {
fetched int
retained int
}
type recordingFetchPermit struct {
expected int
completed chan<- fetchCompletion
cancelled chan<- struct{}
}
func (p *recordingFetchPermit) Expected() int { return p.expected }
func (p *recordingFetchPermit) Complete(fetched, retained int) error {
func (p *recordingFetchPermit) Complete(retained int) error {
if p.completed != nil {
p.completed <- fetchCompletion{fetched: fetched, retained: retained}
p.completed <- fetchCompletion{retained: retained}
}
return nil
}
func (*recordingFetchPermit) Cancel() error { return nil }
func (p *recordingFetchPermit) Cancel() error {
if p.cancelled != nil {
p.cancelled <- struct{}{}
}
return nil
}
type resultRecorderFunc func(Result)
func (f resultRecorderFunc) Record(result Result) { f(result) }
type unlimitedLeaderSession struct{}
func (unlimitedLeaderSession) Fence() Fence { return Fence{Generation: "test", Epoch: 1} }
func (unlimitedLeaderSession) AcquireFetch(context.Context, int) (RequestPermit, bool, error) {
return &recordingRequestPermit{}, true, nil
}
type leaderSessionFunc func(context.Context, int) (RequestPermit, bool, error)
func (leaderSessionFunc) Fence() Fence { return Fence{Generation: "test", Epoch: 1} }
func (f leaderSessionFunc) AcquireFetch(ctx context.Context, expected int) (RequestPermit, bool, error) {
return f(ctx, expected)
}
type recordingRequestPermit struct {
completed chan<- int
cancelled chan<- struct{}
}
func (p *recordingRequestPermit) Complete(_ context.Context, fetched int) error {
if p.completed != nil {
p.completed <- fetched
}
return nil
}
func (p *recordingRequestPermit) Cancel(context.Context) error {
if p.cancelled != nil {
p.cancelled <- struct{}{}
}
return nil
}

103
internal/controller/provider/stats.go Normal file
View File

@ -0,0 +1,103 @@
package provider
import (
"errors"
"math"
"sync"
"proxy-pool/internal/domain/upstream"
)
var ErrInvalidStatsRecorder = errors.New("invalid Provider stats recorder")
type Stats struct {
UpstreamID string
ConsecutiveEmptyFetch int64
FetchErrorCount int64
}
type StatsReader interface {
ReadProviderStats([]string) []Stats
}
type StatsRetainer interface {
RetainProviderStats([]string)
}
type StatsRecorder struct {
mu sync.Mutex
maximum int
byID map[string]Stats
}
func NewStatsRecorder(maximum int) (*StatsRecorder, error) {
if maximum <= 0 {
return nil, ErrInvalidStatsRecorder
}
return &StatsRecorder{maximum: maximum, byID: make(map[string]Stats)}, nil
}
func (recorder *StatsRecorder) Record(result Result) {
if recorder == nil || result.UpstreamID == "" {
return
}
recorder.mu.Lock()
defer recorder.mu.Unlock()
stats, exists := recorder.byID[result.UpstreamID]
if !exists {
if len(recorder.byID) >= recorder.maximum {
return
}
stats.UpstreamID = result.UpstreamID
}
switch result.Class {
case upstream.FetchEmpty:
if stats.ConsecutiveEmptyFetch < math.MaxInt64 {
stats.ConsecutiveEmptyFetch++
}
case upstream.FetchValid, upstream.FetchDuplicateOnly:
stats.ConsecutiveEmptyFetch = 0
case upstream.FetchError:
if stats.FetchErrorCount < math.MaxInt64 {
stats.FetchErrorCount++
}
default:
return
}
recorder.byID[result.UpstreamID] = stats
}
func (recorder *StatsRecorder) ReadProviderStats(upstreamIDs []string) []Stats {
result := make([]Stats, len(upstreamIDs))
if recorder == nil {
return result
}
recorder.mu.Lock()
defer recorder.mu.Unlock()
for index, upstreamID := range upstreamIDs {
result[index] = recorder.byID[upstreamID]
result[index].UpstreamID = upstreamID
}
return result
}
// RetainProviderStats removes observations for upstreams no longer present in
// the complete configuration. Disabled but configured upstreams must be kept.
func (recorder *StatsRecorder) RetainProviderStats(upstreamIDs []string) {
if recorder == nil {
return
}
retained := make(map[string]struct{}, len(upstreamIDs))
for _, upstreamID := range upstreamIDs {
if upstreamID != "" {
retained[upstreamID] = struct{}{}
}
}
recorder.mu.Lock()
defer recorder.mu.Unlock()
for upstreamID := range recorder.byID {
if _, keep := retained[upstreamID]; !keep {
delete(recorder.byID, upstreamID)
}
}
}

83
internal/controller/provider/stats_test.go Normal file
View File

@ -0,0 +1,83 @@
package provider
import (
"sync"
"testing"
"proxy-pool/internal/domain/upstream"
)
func TestStatsRecorderTracksEmptyResetAndErrors(t *testing.T) {
recorder, err := NewStatsRecorder(2)
if err != nil {
t.Fatalf("NewStatsRecorder(): %v", err)
}
for _, class := range []upstream.FetchClass{
upstream.FetchEmpty,
upstream.FetchEmpty,
upstream.FetchError,
} {
recorder.Record(Result{UpstreamID: "provider-a", Class: class})
}
stats := recorder.ReadProviderStats([]string{"provider-a"})[0]
if stats.ConsecutiveEmptyFetch != 2 || stats.FetchErrorCount != 1 {
t.Fatalf("stats = %+v, want empty=2 errors=1", stats)
}
recorder.Record(Result{UpstreamID: "provider-a", Class: upstream.FetchDuplicateOnly})
if got := recorder.ReadProviderStats([]string{"provider-a"})[0].ConsecutiveEmptyFetch; got != 0 {
t.Fatalf("consecutive empty after duplicate = %d, want 0", got)
}
}
func TestStatsRecorderIsBoundedAndConcurrent(t *testing.T) {
recorder, err := NewStatsRecorder(1)
if err != nil {
t.Fatalf("NewStatsRecorder(): %v", err)
}
const workers = 100
var wait sync.WaitGroup
for range workers {
wait.Add(1)
go func() {
defer wait.Done()
recorder.Record(Result{UpstreamID: "provider-a", Class: upstream.FetchError})
}()
}
wait.Wait()
recorder.Record(Result{UpstreamID: "provider-b", Class: upstream.FetchError})
stats := recorder.ReadProviderStats([]string{"provider-a", "provider-b"})
if stats[0].FetchErrorCount != workers || stats[1].FetchErrorCount != 0 {
t.Fatalf("stats = %+v, want bounded provider-a errors", stats)
}
}
func TestStatsRecorderRetainsConfiguredProvidersAndReusesCapacity(t *testing.T) {
recorder, err := NewStatsRecorder(2)
if err != nil {
t.Fatalf("NewStatsRecorder(): %v", err)
}
recorder.Record(Result{UpstreamID: "removed-a", Class: upstream.FetchError})
recorder.Record(Result{UpstreamID: "provider-b", Class: upstream.FetchError})
const workers = 100
var wait sync.WaitGroup
for range workers {
wait.Add(2)
go func() {
defer wait.Done()
recorder.Record(Result{UpstreamID: "provider-b", Class: upstream.FetchError})
}()
go func() {
defer wait.Done()
recorder.RetainProviderStats([]string{"provider-a", "provider-b"})
}()
}
wait.Wait()
recorder.RetainProviderStats([]string{"provider-a", "provider-b"})
recorder.Record(Result{UpstreamID: "provider-a", Class: upstream.FetchError})
stats := recorder.ReadProviderStats([]string{"removed-a", "provider-a", "provider-b"})
if stats[0].FetchErrorCount != 0 || stats[1].FetchErrorCount != 1 || stats[2].FetchErrorCount == 0 {
t.Fatalf("stats after retention = %+v", stats)
}
}

219
internal/controller/provider/upstream_runtime.go Normal file
View File

@ -0,0 +1,219 @@
package provider
import (
"context"
"errors"
"fmt"
"hash/fnv"
"reflect"
"time"
"proxy-pool/internal/controller/pool"
"proxy-pool/internal/domain/activitypool"
)
var (
ErrInvalidUpstreamRuntime = errors.New("invalid Provider upstream runtime")
ErrUpstreamRuntimeStopped = errors.New("Provider upstream runtime stopped")
)
type UpstreamRuntimeConfig struct {
Provider Config
ReconcilePolicy pool.ReconcilePolicy
ReconcileInterval time.Duration
}
type UpstreamRuntimeDependencies struct {
Coordinator Coordinator
Inventory pool.InventoryReader
Adapter ProviderAdapter
Parser Parser
Activity activitypool.Upserter
Results ResultRecorder
}
// UpstreamRuntime owns every leader-scoped object for one Upstream. A fresh
// local budget and coalescing signal are created for each leadership term.
type UpstreamRuntime struct {
config UpstreamRuntimeConfig
dependencies UpstreamRuntimeDependencies
sleeper Sleeper
}
func NewUpstreamRuntime(
config UpstreamRuntimeConfig,
dependencies UpstreamRuntimeDependencies,
) (*UpstreamRuntime, error) {
if config.ReconcileInterval <= 0 || nilRuntimeDependency(dependencies.Coordinator) ||
nilRuntimeDependency(dependencies.Inventory) || nilRuntimeDependency(dependencies.Adapter) ||
nilRuntimeDependency(dependencies.Parser) || nilRuntimeDependency(dependencies.Activity) ||
nilRuntimeDependency(dependencies.Results) {
return nil, ErrInvalidUpstreamRuntime
}
runtime := &UpstreamRuntime{
config: config, dependencies: dependencies, sleeper: timerSleeper{},
}
if _, err := runtime.newLeaderTerm(); err != nil {
return nil, errors.Join(ErrInvalidUpstreamRuntime, err)
}
return runtime, nil
}
func (runtime *UpstreamRuntime) Run(ctx context.Context) error {
if runtime == nil || ctx == nil {
return ErrInvalidUpstreamRuntime
}
limits := CoordinationLimits{
RequestInterval: runtime.config.Provider.RequestInterval,
MaxInFlight: runtime.config.Provider.MaxInFlight,
MaxAttemptDuration: runtime.config.Provider.Timeout,
MaxTotal: runtime.config.Provider.MaxTotal,
}
err := runtime.dependencies.Coordinator.RunLeader(
ctx,
runtime.config.Provider.UpstreamID,
limits,
func(leaderCtx context.Context, session LeaderSession) error {
term, buildErr := runtime.newLeaderTerm()
if buildErr != nil {
return buildErr
}
return term.run(leaderCtx, session)
},
)
if ctx.Err() != nil {
return ctx.Err()
}
if err == nil {
return ErrUpstreamRuntimeStopped
}
return err
}
func (runtime *UpstreamRuntime) ID() string {
if runtime == nil {
return ""
}
return runtime.config.Provider.UpstreamID
}
func (runtime *UpstreamRuntime) newLeaderTerm() (*upstreamLeaderTerm, error) {
budget, err := pool.NewFetchBudget(pool.FetchBudgetConfig{
UpstreamID: runtime.config.Provider.UpstreamID,
MaxSize: runtime.config.Provider.MaxSize,
ExpectedPerFetch: runtime.config.ReconcilePolicy.ExpectedPerFetch,
})
if err != nil {
return nil, err
}
providerConfig := runtime.config.Provider
providerConfig.RequestInterval = 0
providerReconciler, err := NewReconciler(providerConfig, Ports{
Adapter: runtime.dependencies.Adapter, Parser: runtime.dependencies.Parser,
Activity: runtime.dependencies.Activity, Results: runtime.dependencies.Results,
Capacity: budget,
})
if err != nil {
return nil, err
}
poolReconciler, err := pool.NewReconciler(
runtime.config.ReconcilePolicy,
budget,
providerReconciler,
)
if err != nil {
return nil, err
}
return &upstreamLeaderTerm{
upstreamID: runtime.config.Provider.UpstreamID,
interval: runtime.config.ReconcileInterval,
safetyMargin: runtime.config.ReconcilePolicy.SafetyMargin,
inventory: runtime.dependencies.Inventory,
pool: poolReconciler,
provider: providerReconciler,
sleeper: runtime.sleeper,
}, nil
}
type upstreamLeaderTerm struct {
upstreamID string
interval time.Duration
safetyMargin time.Duration
inventory pool.InventoryReader
pool *pool.Reconciler
provider *Reconciler
sleeper Sleeper
}
func (term *upstreamLeaderTerm) run(ctx context.Context, session LeaderSession) error {
termCtx, cancel := context.WithCancel(ctx)
defer cancel()
done := make(chan error, 2)
go func() { done <- term.provider.RunLeader(termCtx, session) }()
go func() { done <- term.reconcileInventory(termCtx) }()
first := <-done
cancel()
second := <-done
if ctx.Err() != nil {
return nil
}
if first == nil {
first = ErrUpstreamRuntimeStopped
}
if second != nil && !errors.Is(second, context.Canceled) {
return errors.Join(first, second)
}
return first
}
func (term *upstreamLeaderTerm) reconcileInventory(ctx context.Context) error {
if delay := initialReconcileDelay(term.upstreamID, term.interval); delay > 0 {
if err := term.sleeper.Sleep(ctx, delay); err != nil {
if ctx.Err() != nil {
return nil
}
return fmt.Errorf("wait for initial Provider inventory reconciliation: %w", err)
}
}
for ctx.Err() == nil {
inventory, err := term.inventory.ReadInventory(
ctx,
term.upstreamID,
term.safetyMargin,
)
if err == nil && inventory.Managed >= 0 && inventory.AvailableSlots >= 0 {
term.pool.ReconcileSnapshot(inventory)
}
if err := term.sleeper.Sleep(ctx, term.interval); err != nil {
if ctx.Err() != nil {
return nil
}
return fmt.Errorf("wait for Provider inventory reconciliation: %w", err)
}
}
return nil
}
func initialReconcileDelay(upstreamID string, interval time.Duration) time.Duration {
window := min(interval/4, 250*time.Millisecond)
if window <= 1 {
return 0
}
digest := fnv.New64a()
_, _ = digest.Write([]byte(upstreamID))
return time.Duration(digest.Sum64() % uint64(window))
}
func nilRuntimeDependency(value any) bool {
if value == nil {
return true
}
reflected := reflect.ValueOf(value)
switch reflected.Kind() {
case reflect.Chan, reflect.Func, reflect.Interface, reflect.Map, reflect.Pointer, reflect.Slice:
return reflected.IsNil()
default:
return false
}
}

247
internal/controller/provider/upstream_runtime_test.go Normal file
View File

@ -0,0 +1,247 @@
package provider
import (
"context"
"errors"
"sync/atomic"
"testing"
"time"
"proxy-pool/internal/controller/pool"
"proxy-pool/internal/domain/activitypool"
proxyDomain "proxy-pool/internal/domain/proxy"
)
func TestUpstreamRuntimeReadsInventoryAndFetchesOnlyInsideLeaderTerm(t *testing.T) {
var inventoryReads atomic.Int64
fetched := make(chan struct{}, 1)
runtime, err := NewUpstreamRuntime(UpstreamRuntimeConfig{
Provider: runtimeProviderConfig("provider-a"),
ReconcilePolicy: runtimeReconcilePolicy(),
ReconcileInterval: 10 * time.Millisecond,
}, UpstreamRuntimeDependencies{
Coordinator: coordinatorFunc(func(
ctx context.Context,
upstreamID string,
limits CoordinationLimits,
work func(context.Context, LeaderSession) error,
) error {
if upstreamID != "provider-a" || limits.MaxTotal != 10 || limits.MaxInFlight != 1 {
t.Errorf("coordination = (%q, %+v)", upstreamID, limits)
}
return work(ctx, unlimitedLeaderSession{})
}),
Inventory: inventoryReaderFunc(func(context.Context, string, time.Duration) (pool.InventorySnapshot, error) {
inventoryReads.Add(1)
return pool.InventorySnapshot{}, nil
}),
Adapter: adapterFunc(func(context.Context) (FetchResponse, error) {
fetched <- struct{}{}
return FetchResponse{Body: []byte("fixture")}, nil
}),
Parser: parserFunc(func(context.Context, []byte) ([]proxyDomain.Proxy, error) {
return []proxyDomain.Proxy{{ID: "proxy-1"}}, nil
}),
Activity: activitySinkFunc(func(context.Context, string, activitypool.FetchedBatch) (activitypool.UpsertResult, error) {
return activitypool.UpsertResult{Accepted: 1, Inserted: 1}, nil
}),
Results: resultRecorderFunc(func(Result) {}),
})
if err != nil {
t.Fatalf("NewUpstreamRuntime(): %v", err)
}
ctx, cancel := context.WithCancel(context.Background())
done := make(chan error, 1)
go func() { done <- runtime.Run(ctx) }()
select {
case <-fetched:
case <-time.After(time.Second):
t.Fatal("timed out waiting for leader fetch")
}
cancel()
if err := <-done; !errors.Is(err, context.Canceled) {
t.Fatalf("Run() error = %v, want context cancellation", err)
}
if got := inventoryReads.Load(); got == 0 {
t.Fatal("inventory was not read inside leader term")
}
}
func TestUpstreamRuntimeFailsClosedUntilInventoryReadRecovers(t *testing.T) {
var inventoryReads atomic.Int64
var providerCalls atomic.Int64
fetched := make(chan struct{}, 1)
runtime, err := NewUpstreamRuntime(UpstreamRuntimeConfig{
Provider: runtimeProviderConfig("provider-a"),
ReconcilePolicy: runtimeReconcilePolicy(),
ReconcileInterval: 10 * time.Millisecond,
}, UpstreamRuntimeDependencies{
Coordinator: coordinatorFunc(func(
ctx context.Context,
_ string,
_ CoordinationLimits,
work func(context.Context, LeaderSession) error,
) error {
return work(ctx, unlimitedLeaderSession{})
}),
Inventory: inventoryReaderFunc(func(context.Context, string, time.Duration) (pool.InventorySnapshot, error) {
if inventoryReads.Add(1) == 1 {
return pool.InventorySnapshot{}, errors.New("inventory unavailable")
}
return pool.InventorySnapshot{}, nil
}),
Adapter: adapterFunc(func(context.Context) (FetchResponse, error) {
providerCalls.Add(1)
fetched <- struct{}{}
return FetchResponse{Body: []byte("fixture")}, nil
}),
Parser: parserFunc(func(context.Context, []byte) ([]proxyDomain.Proxy, error) {
return []proxyDomain.Proxy{{ID: "proxy-1"}}, nil
}),
Activity: activitySinkFunc(func(context.Context, string, activitypool.FetchedBatch) (activitypool.UpsertResult, error) {
return activitypool.UpsertResult{Accepted: 1, Inserted: 1}, nil
}),
Results: resultRecorderFunc(func(Result) {}),
})
if err != nil {
t.Fatalf("NewUpstreamRuntime(): %v", err)
}
ctx, cancel := context.WithCancel(context.Background())
done := make(chan error, 1)
go func() { done <- runtime.Run(ctx) }()
select {
case <-fetched:
case <-time.After(time.Second):
t.Fatal("timed out waiting for recovered inventory fetch")
}
cancel()
<-done
if got := inventoryReads.Load(); got < 2 {
t.Fatalf("inventory reads = %d, want recovery retry", got)
}
if got := providerCalls.Load(); got != 1 {
t.Fatalf("provider calls = %d, want one call after recovery", got)
}
}
func TestUpstreamRuntimeDelegatesRequestIntervalOnlyToCoordinator(t *testing.T) {
providerConfig := runtimeProviderConfig("provider-a")
providerConfig.RequestInterval = 500 * time.Millisecond
fetched := make(chan struct{}, 2)
runtime, err := NewUpstreamRuntime(UpstreamRuntimeConfig{
Provider: providerConfig,
ReconcilePolicy: runtimeReconcilePolicy(),
ReconcileInterval: 5 * time.Millisecond,
}, UpstreamRuntimeDependencies{
Coordinator: coordinatorFunc(func(
ctx context.Context,
_ string,
limits CoordinationLimits,
work func(context.Context, LeaderSession) error,
) error {
if limits.RequestInterval != 500*time.Millisecond {
t.Errorf("distributed RequestInterval = %s, want 500ms", limits.RequestInterval)
}
return work(ctx, unlimitedLeaderSession{})
}),
Inventory: inventoryReaderFunc(func(context.Context, string, time.Duration) (pool.InventorySnapshot, error) {
return pool.InventorySnapshot{}, nil
}),
Adapter: adapterFunc(func(context.Context) (FetchResponse, error) {
fetched <- struct{}{}
return FetchResponse{Body: []byte("fixture")}, nil
}),
Parser: parserFunc(func(context.Context, []byte) ([]proxyDomain.Proxy, error) {
return []proxyDomain.Proxy{{ID: "proxy-1"}}, nil
}),
Activity: activitySinkFunc(func(context.Context, string, activitypool.FetchedBatch) (activitypool.UpsertResult, error) {
return activitypool.UpsertResult{Accepted: 1, Inserted: 1}, nil
}),
Results: resultRecorderFunc(func(Result) {}),
})
if err != nil {
t.Fatalf("NewUpstreamRuntime(): %v", err)
}
ctx, cancel := context.WithCancel(context.Background())
done := make(chan error, 1)
go func() { done <- runtime.Run(ctx) }()
for range 2 {
select {
case <-fetched:
case <-time.After(150 * time.Millisecond):
cancel()
<-done
t.Fatal("local Provider reconciler duplicated the distributed request interval")
}
}
cancel()
<-done
}
func TestNewUpstreamRuntimeRejectsInvalidDependencies(t *testing.T) {
validConfig := UpstreamRuntimeConfig{
Provider: runtimeProviderConfig("provider-a"),
ReconcilePolicy: runtimeReconcilePolicy(),
ReconcileInterval: time.Second,
}
if runtime, err := NewUpstreamRuntime(validConfig, UpstreamRuntimeDependencies{}); err == nil || runtime != nil {
t.Fatalf("NewUpstreamRuntime() = (%v, %v), want invalid dependencies", runtime, err)
}
}
func TestInitialReconcileDelayIsStableAndBounded(t *testing.T) {
const interval = time.Second
first := initialReconcileDelay("provider-a", interval)
if first != initialReconcileDelay("provider-a", interval) {
t.Fatal("initial reconcile delay is not stable")
}
if first < 0 || first >= 250*time.Millisecond {
t.Fatalf("initial reconcile delay = %s, want [0, 250ms)", first)
}
if other := initialReconcileDelay("provider-b", interval); other == first {
t.Fatalf("different upstreams have the same initial delay %s", first)
}
}
func runtimeProviderConfig(upstreamID string) Config {
return Config{
UpstreamID: upstreamID, Timeout: time.Second, MaxAttempts: 1,
MaxInFlight: 1, MaxTotal: 10, MaxSize: 10,
}
}
func runtimeReconcilePolicy() pool.ReconcilePolicy {
return pool.ReconcilePolicy{
MinimumAvailableSlots: 1, TargetAvailableSlots: 2,
ExpectedPerFetch: 1, ExpectedSlotsPerFetch: 1,
}
}
type coordinatorFunc func(
context.Context,
string,
CoordinationLimits,
func(context.Context, LeaderSession) error,
) error
func (f coordinatorFunc) RunLeader(
ctx context.Context,
upstreamID string,
limits CoordinationLimits,
work func(context.Context, LeaderSession) error,
) error {
return f(ctx, upstreamID, limits, work)
}
type inventoryReaderFunc func(context.Context, string, time.Duration) (pool.InventorySnapshot, error)
func (f inventoryReaderFunc) ReadInventory(
ctx context.Context,
upstreamID string,
safetyMargin time.Duration,
) (pool.InventorySnapshot, error) {
return f(ctx, upstreamID, safetyMargin)
}

63
internal/domain/proxy/capacity.go
View File

@ -15,17 +15,44 @@ var (
type Capacity struct {
max atomic.Uint32
counters atomic.Uint64
configuredObserver *activityObserver
observer atomic.Pointer[activityObserver]
}
type activityObserver struct{ notify func(bool) }
func NewCapacity(max int64) *Capacity {
return NewCapacityWithActivityObserver(max, nil)
}
// NewCapacityWithActivityObserver reports successful zero-to-nonzero and
// nonzero-to-zero transitions. The observer must tolerate concurrent calls.
func NewCapacityWithActivityObserver(max int64, observer func(nonzero bool)) *Capacity {
capacity := &Capacity{}
if max < 0 || max > int64(counterMask) {
max = 0
}
capacity.max.Store(uint32(max))
if observer != nil {
capacity.configuredObserver = &activityObserver{notify: observer}
capacity.observer.Store(capacity.configuredObserver)
}
return capacity
}
// SetActivityObservationEnabled lets snapshot ownership disable callbacks for
// current Proxies and enable them only while a runtime is retired and draining.
func (c *Capacity) SetActivityObservationEnabled(enabled bool) {
if c == nil || c.configuredObserver == nil {
return
}
if enabled {
c.observer.Store(c.configuredObserver)
return
}
c.observer.Store(nil)
}
func (c *Capacity) SetMax(max int64) bool {
if max < 0 || max > int64(counterMask) {
return false
@ -36,6 +63,15 @@ func (c *Capacity) SetMax(max int64) bool {
func (c *Capacity) Max() int64 { return int64(c.max.Load()) }
func (c *Capacity) Counters() (active, reserved, maximum int64) {
if c == nil {
return 0, 0, 0
}
packed := c.counters.Load()
activeCounter, reservedCounter := unpack(packed)
return int64(activeCounter), int64(reservedCounter), int64(c.max.Load())
}
func (c *Capacity) Reserve() (*Reservation, bool) {
for {
current := c.counters.Load()
@ -45,6 +81,11 @@ func (c *Capacity) Reserve() (*Reservation, bool) {
}
next := pack(active, reserved+1)
if c.counters.CompareAndSwap(current, next) {
if active+reserved == 0 {
if observer := c.observer.Load(); observer != nil {
observer.notify(true)
}
}
return &Reservation{capacity: c}, true
}
}
@ -77,7 +118,16 @@ func (c *Capacity) cancel() {
for {
current := c.counters.Load()
active, reserved := unpack(current)
if reserved == 0 || c.counters.CompareAndSwap(current, pack(active, reserved-1)) {
if reserved == 0 {
return
}
next := pack(active, reserved-1)
if c.counters.CompareAndSwap(current, next) {
if active+reserved == 1 {
if observer := c.observer.Load(); observer != nil {
observer.notify(false)
}
}
return
}
}
@ -87,7 +137,16 @@ func (c *Capacity) release() {
for {
current := c.counters.Load()
active, reserved := unpack(current)
if active == 0 || c.counters.CompareAndSwap(current, pack(active-1, reserved)) {
if active == 0 {
return
}
next := pack(active-1, reserved)
if c.counters.CompareAndSwap(current, next) {
if active+reserved == 1 {
if observer := c.observer.Load(); observer != nil {
observer.notify(false)
}
}
return
}
}

51
internal/domain/proxy/capacity_test.go
View File

@ -2,6 +2,7 @@ package proxy
import (
"errors"
"reflect"
"sync"
"sync/atomic"
"testing"
@ -38,6 +39,56 @@ func TestReservationCancelReleasesReservedCapacity(t *testing.T) {
assertCapacityCounters(t, capacity, 0, 0)
}
func TestCapacityCountersReadsOnePackedSnapshot(t *testing.T) {
capacity := NewCapacity(3)
first, ok := capacity.Reserve()
if !ok {
t.Fatal("first Reserve() = false")
}
second, ok := capacity.Reserve()
if !ok {
t.Fatal("second Reserve() = false")
}
if err := first.Commit(); err != nil {
t.Fatalf("Commit(): %v", err)
}
active, reserved, maximum := capacity.Counters()
if active != 1 || reserved != 1 || maximum != 3 {
t.Fatalf("Counters() = (%d, %d, %d), want (1, 1, 3)", active, reserved, maximum)
}
if err := first.Release(); err != nil {
t.Fatalf("Release(): %v", err)
}
if err := second.Cancel(); err != nil {
t.Fatalf("Cancel(): %v", err)
}
}
func TestCapacityActivityObserverTracksOnlyNonzeroTransitions(t *testing.T) {
var transitions []bool
capacity := NewCapacityWithActivityObserver(2, func(nonzero bool) {
transitions = append(transitions, nonzero)
})
first, ok := capacity.Reserve()
if !ok {
t.Fatal("Reserve(first) = false")
}
second, ok := capacity.Reserve()
if !ok {
t.Fatal("Reserve(second) = false")
}
if err := first.Cancel(); err != nil {
t.Fatalf("Cancel(first): %v", err)
}
if err := second.Cancel(); err != nil {
t.Fatalf("Cancel(second): %v", err)
}
if !reflect.DeepEqual(transitions, []bool{true, false}) {
t.Fatalf("transitions = %v, want [true false]", transitions)
}
}
func TestReservationCommitAndReleaseAreSingleUse(t *testing.T) {
capacity := NewCapacity(1)
reservation, ok := capacity.Reserve()

2
internal/domain/upstream/fetch_capacity.go
View File

@ -8,6 +8,6 @@ type FetchCapacity interface {
type FetchPermit interface {
Expected() int
Complete(fetched, retained int) error
Complete(retained int) error
Cancel() error
}

11
internal/domain/upstream/pool.go
View File

@ -17,9 +17,7 @@ type ProxyCapacity struct {
type Inventory struct {
Proxies []ProxyCapacity
PendingExpected int
FetchedTotal int64
MaxSize int
MaxTotal int64
}
func (i Inventory) AvailableSlots(now time.Time, safetyMargin time.Duration) int64 {
@ -55,15 +53,6 @@ func (i Inventory) FetchAllowance(requested int) int {
return 0
}
allowed := min(requested, max(i.MaxSize-i.ManagedCount(), 0))
if i.MaxTotal > 0 {
remaining := i.MaxTotal - i.FetchedTotal
if remaining <= 0 {
return 0
}
if int64(allowed) > remaining {
allowed = int(remaining)
}
}
return allowed
}

11
internal/domain/upstream/pool_test.go
View File

@ -21,7 +21,7 @@ func TestInventoryAvailableSlotsUsesOnlyAllocatableCapacity(t *testing.T) {
}
}
func TestInventoryFetchAllowanceSeparatesPoolAndCumulativeLimits(t *testing.T) {
func TestInventoryFetchAllowanceUsesManagedPoolCapacity(t *testing.T) {
inventory := Inventory{
Proxies: []ProxyCapacity{
{State: proxyDomain.StateFetched},
@ -32,20 +32,13 @@ func TestInventoryFetchAllowanceSeparatesPoolAndCumulativeLimits(t *testing.T) {
{State: proxyDomain.StateExtracted},
},
PendingExpected: 2,
FetchedTotal: 98,
MaxSize: 10,
MaxTotal: 100,
}
if got := inventory.ManagedCount(); got != 7 {
t.Fatalf("ManagedCount() = %d, want 7", got)
}
if got := inventory.FetchAllowance(10); got != 2 {
t.Fatalf("FetchAllowance() = %d, want 2 from cumulative quota", got)
}
inventory.MaxTotal = 0
if got := inventory.FetchAllowance(10); got != 3 {
t.Fatalf("FetchAllowance() with unlimited cumulative quota = %d, want 3 from pool size", got)
t.Fatalf("FetchAllowance() = %d, want 3 from pool size", got)
}
}

211
internal/domain/workerruntime/memory.go Normal file
View File

@ -0,0 +1,211 @@
package workerruntime
import (
"context"
"crypto/sha256"
"encoding/json"
"sort"
"strings"
"sync"
"time"
)
type MemoryStore struct {
mu sync.Mutex
now func() time.Time
sessions map[string]memorySession
reports map[string]memoryReport
}
type memorySession struct {
value Session
expiresAt time.Time
}
type memoryReport struct {
value Report
digest [sha256.Size]byte
expiresAt time.Time
counters map[string]Counter
}
var (
_ SessionWriter = (*MemoryStore)(nil)
_ ReportWriter = (*MemoryStore)(nil)
_ RuntimeReader = (*MemoryStore)(nil)
)
func NewMemoryStore(now func() time.Time) (*MemoryStore, error) {
if now == nil {
return nil, ErrInvalidStore
}
return &MemoryStore{
now: now, sessions: make(map[string]memorySession), reports: make(map[string]memoryReport),
}, nil
}
func (store *MemoryStore) ReplaceSession(ctx context.Context, session Session, ttl time.Duration) error {
if ctx == nil || store == nil || !validSession(session) || ttl <= 0 {
return ErrInvalidSession
}
if err := ctx.Err(); err != nil {
return err
}
now := store.now().UTC()
if now.IsZero() {
return ErrInvalidStore
}
store.mu.Lock()
defer store.mu.Unlock()
current, exists := store.sessions[session.WorkerID]
identityChanged := exists && (current.value.SessionID != session.SessionID || current.value.InstanceID != session.InstanceID)
expired := exists && !current.expiresAt.After(now)
if exists && !identityChanged && !expired && sessionBefore(session, current.value) {
return ErrStaleSession
}
ackAdvanced := exists && !identityChanged && !expired && sessionAfter(session, current.value)
if identityChanged || expired || ackAdvanced {
delete(store.reports, session.WorkerID)
}
store.sessions[session.WorkerID] = memorySession{value: session, expiresAt: now.Add(ttl)}
return nil
}
func (store *MemoryStore) ReplaceRuntime(ctx context.Context, report Report, ttl time.Duration) error {
if ctx == nil || store == nil || ttl <= 0 {
return ErrInvalidReport
}
if err := ctx.Err(); err != nil {
return err
}
normalized, counterIndex, err := normalizeReport(report)
if err != nil {
return err
}
payload, err := json.Marshal(normalized)
if err != nil {
return ErrInvalidReport
}
digest := sha256.Sum256(payload)
now := store.now().UTC()
if now.IsZero() {
return ErrInvalidStore
}
store.mu.Lock()
defer store.mu.Unlock()
session, exists := store.sessions[report.WorkerID]
if !exists || !session.expiresAt.After(now) || session.value.SessionID != report.SessionID {
return ErrStaleSession
}
if report.SnapshotVersion != session.value.AckedSnapshotVersion ||
report.OwnershipEpoch != session.value.AckedOwnershipEpoch {
return ErrStaleReport
}
if current, exists := store.reports[report.WorkerID]; exists && current.value.SessionID == report.SessionID {
switch {
case normalized.Sequence < current.value.Sequence:
return ErrStaleReport
case normalized.Sequence == current.value.Sequence && digest != current.digest:
return ErrConflictingReport
case normalized.Sequence == current.value.Sequence:
return nil
}
}
store.reports[report.WorkerID] = memoryReport{
value: normalized, digest: digest, expiresAt: now.Add(ttl), counters: counterIndex,
}
session.expiresAt = now.Add(ttl)
store.sessions[report.WorkerID] = session
return nil
}
func (store *MemoryStore) ReadRuntime(ctx context.Context, proxies []OwnedProxy) ([]Snapshot, error) {
if ctx == nil || store == nil {
return nil, ErrInvalidQuery
}
if err := ctx.Err(); err != nil {
return nil, err
}
seen := make(map[string]struct{}, len(proxies))
for _, proxy := range proxies {
if !clean(proxy.ProxyID) || !clean(proxy.WorkerID) || proxy.OwnershipEpoch == 0 {
return nil, ErrInvalidQuery
}
key := proxy.WorkerID + "\x00" + proxy.ProxyID
if _, exists := seen[key]; exists {
return nil, ErrInvalidQuery
}
seen[key] = struct{}{}
}
now := store.now().UTC()
if now.IsZero() {
return nil, ErrInvalidStore
}
store.mu.Lock()
defer store.mu.Unlock()
result := make([]Snapshot, len(proxies))
for index, proxy := range proxies {
result[index].ProxyID = proxy.ProxyID
session, sessionExists := store.sessions[proxy.WorkerID]
report, reportExists := store.reports[proxy.WorkerID]
if !sessionExists || !reportExists || !session.expiresAt.After(now) || !report.expiresAt.After(now) ||
report.value.SessionID != session.value.SessionID ||
report.value.SnapshotVersion != session.value.AckedSnapshotVersion ||
report.value.OwnershipEpoch != session.value.AckedOwnershipEpoch ||
report.value.OwnershipEpoch < proxy.OwnershipEpoch {
continue
}
result[index].Fresh = true
if counter, exists := report.counters[proxy.ProxyID]; exists {
result[index].Active = counter.Active
result[index].Reserved = counter.Reserved
result[index].Draining = counter.Draining
}
}
return result, nil
}
func normalizeReport(report Report) (Report, map[string]Counter, error) {
if !clean(report.WorkerID) || !clean(report.SessionID) || report.Sequence == 0 ||
report.SnapshotVersion == 0 || report.OwnershipEpoch == 0 || report.ObservedAt.IsZero() {
return Report{}, nil, ErrInvalidReport
}
normalized := report
normalized.ObservedAt = report.ObservedAt.UTC()
normalized.Counters = append([]Counter(nil), report.Counters...)
sort.Slice(normalized.Counters, func(left, right int) bool {
return normalized.Counters[left].ProxyID < normalized.Counters[right].ProxyID
})
index := make(map[string]Counter, len(normalized.Counters))
for _, counter := range normalized.Counters {
if !clean(counter.ProxyID) || counter.Active < 0 || counter.Reserved < 0 {
return Report{}, nil, ErrInvalidReport
}
if _, exists := index[counter.ProxyID]; exists {
return Report{}, nil, ErrInvalidReport
}
index[counter.ProxyID] = counter
}
return normalized, index, nil
}
func validSession(session Session) bool {
return clean(session.WorkerID) && clean(session.InstanceID) && clean(session.SessionID) &&
session.AckedSnapshotVersion > 0 && session.AckedOwnershipEpoch > 0
}
func sessionBefore(left, right Session) bool {
return left.AckedOwnershipEpoch < right.AckedOwnershipEpoch ||
(left.AckedOwnershipEpoch == right.AckedOwnershipEpoch &&
left.AckedSnapshotVersion < right.AckedSnapshotVersion)
}
func sessionAfter(left, right Session) bool {
return left.AckedOwnershipEpoch > right.AckedOwnershipEpoch ||
(left.AckedOwnershipEpoch == right.AckedOwnershipEpoch &&
left.AckedSnapshotVersion > right.AckedSnapshotVersion)
}
func clean(value string) bool {
return value != "" && strings.TrimSpace(value) == value
}

156
internal/domain/workerruntime/memory_test.go Normal file
View File

@ -0,0 +1,156 @@
package workerruntime
import (
"context"
"errors"
"testing"
"time"
)
func TestMemoryStoreReplacesSparseRuntimeAndClearsMissingCounters(t *testing.T) {
now := time.Date(2026, 7, 30, 15, 0, 0, 0, time.UTC)
store := newRuntimeStore(t, &now)
ctx := context.Background()
registerRuntimeSession(t, store, "session-a", time.Minute)
report := Report{
WorkerID: "worker-a", SessionID: "session-a", Sequence: 1,
SnapshotVersion: 3, OwnershipEpoch: 9, ObservedAt: now,
Counters: []Counter{{ProxyID: "proxy-a", Active: 2, Reserved: 1}},
}
if err := store.ReplaceRuntime(ctx, report, time.Minute); err != nil {
t.Fatalf("ReplaceRuntime(first): %v", err)
}
got, err := store.ReadRuntime(ctx, []OwnedProxy{{ProxyID: "proxy-a", WorkerID: "worker-a", OwnershipEpoch: 9}})
if err != nil || len(got) != 1 || got[0] != (Snapshot{ProxyID: "proxy-a", Active: 2, Reserved: 1, Fresh: true}) {
t.Fatalf("ReadRuntime(first) = %+v, %v", got, err)
}
report.Sequence = 2
report.ObservedAt = now.Add(time.Second)
report.Counters = nil
if err := store.ReplaceRuntime(ctx, report, time.Minute); err != nil {
t.Fatalf("ReplaceRuntime(empty): %v", err)
}
got, err = store.ReadRuntime(ctx, []OwnedProxy{{ProxyID: "proxy-a", WorkerID: "worker-a", OwnershipEpoch: 9}})
if err != nil || len(got) != 1 || got[0] != (Snapshot{ProxyID: "proxy-a", Fresh: true}) {
t.Fatalf("ReadRuntime(empty) = %+v, %v", got, err)
}
}
func TestMemoryStoreFencesSessionsAndReportSequence(t *testing.T) {
now := time.Date(2026, 7, 30, 15, 0, 0, 0, time.UTC)
store := newRuntimeStore(t, &now)
ctx := context.Background()
registerRuntimeSession(t, store, "session-a", time.Minute)
report := Report{
WorkerID: "worker-a", SessionID: "session-a", Sequence: 2,
SnapshotVersion: 3, OwnershipEpoch: 9, ObservedAt: now,
Counters: []Counter{{ProxyID: "proxy-a", Active: 1}},
}
if err := store.ReplaceRuntime(ctx, report, time.Minute); err != nil {
t.Fatalf("ReplaceRuntime(first): %v", err)
}
if err := store.ReplaceRuntime(ctx, report, time.Minute); err != nil {
t.Fatalf("ReplaceRuntime(idempotent): %v", err)
}
conflict := report
conflict.Counters = []Counter{{ProxyID: "proxy-a", Active: 2}}
if err := store.ReplaceRuntime(ctx, conflict, time.Minute); !errors.Is(err, ErrConflictingReport) {
t.Fatalf("ReplaceRuntime(conflict) error = %v", err)
}
stale := report
stale.Sequence = 1
if err := store.ReplaceRuntime(ctx, stale, time.Minute); !errors.Is(err, ErrStaleReport) {
t.Fatalf("ReplaceRuntime(stale) error = %v", err)
}
registerRuntimeSession(t, store, "session-b", time.Minute)
if err := store.ReplaceRuntime(ctx, report, time.Minute); !errors.Is(err, ErrStaleSession) {
t.Fatalf("ReplaceRuntime(old session) error = %v", err)
}
}
func TestMemoryStoreFailsClosedForExpiredOrOlderOwnershipReport(t *testing.T) {
now := time.Date(2026, 7, 30, 15, 0, 0, 0, time.UTC)
store := newRuntimeStore(t, &now)
ctx := context.Background()
registerRuntimeSession(t, store, "session-a", time.Minute)
if err := store.ReplaceRuntime(ctx, Report{
WorkerID: "worker-a", SessionID: "session-a", Sequence: 1,
SnapshotVersion: 3, OwnershipEpoch: 9, ObservedAt: now,
}, time.Minute); err != nil {
t.Fatalf("ReplaceRuntime(): %v", err)
}
queries := []OwnedProxy{
{ProxyID: "proxy-a", WorkerID: "worker-a", OwnershipEpoch: 10},
{ProxyID: "proxy-b", WorkerID: "worker-a", OwnershipEpoch: 9},
}
got, err := store.ReadRuntime(ctx, queries)
if err != nil || got[0].Fresh || !got[1].Fresh {
t.Fatalf("ReadRuntime(ownership fence) = %+v, %v", got, err)
}
now = now.Add(time.Minute)
got, err = store.ReadRuntime(ctx, queries[1:])
if err != nil || len(got) != 1 || got[0].Fresh {
t.Fatalf("ReadRuntime(expired) = %+v, %v", got, err)
}
}
func TestMemoryStoreRejectsRuntimeBeyondAcknowledgedSnapshot(t *testing.T) {
now := time.Date(2026, 7, 30, 15, 0, 0, 0, time.UTC)
store := newRuntimeStore(t, &now)
registerRuntimeSession(t, store, "session-a", time.Minute)
report := Report{
WorkerID: "worker-a", SessionID: "session-a", Sequence: 1,
SnapshotVersion: 4, OwnershipEpoch: 9, ObservedAt: now,
}
if err := store.ReplaceRuntime(context.Background(), report, time.Minute); !errors.Is(err, ErrStaleReport) {
t.Fatalf("ReplaceRuntime(ahead snapshot) error = %v, want ErrStaleReport", err)
}
report.SnapshotVersion = 3
report.OwnershipEpoch = 10
if err := store.ReplaceRuntime(context.Background(), report, time.Minute); !errors.Is(err, ErrStaleReport) {
t.Fatalf("ReplaceRuntime(ahead epoch) error = %v, want ErrStaleReport", err)
}
}
func TestMemoryStoreExpiredSameIdentitySessionDoesNotReactivateOldReport(t *testing.T) {
now := time.Date(2026, 7, 30, 15, 0, 0, 0, time.UTC)
store := newRuntimeStore(t, &now)
registerRuntimeSession(t, store, "session-a", time.Second)
if err := store.ReplaceRuntime(context.Background(), Report{
WorkerID: "worker-a", SessionID: "session-a", Sequence: 1,
SnapshotVersion: 3, OwnershipEpoch: 9, ObservedAt: now,
}, time.Minute); err != nil {
t.Fatalf("ReplaceRuntime(): %v", err)
}
session := store.sessions["worker-a"]
session.expiresAt = now.Add(time.Second)
store.sessions["worker-a"] = session
now = now.Add(2 * time.Second)
registerRuntimeSession(t, store, "session-a", time.Minute)
got, err := store.ReadRuntime(context.Background(), []OwnedProxy{{
ProxyID: "proxy-a", WorkerID: "worker-a", OwnershipEpoch: 9,
}})
if err != nil || len(got) != 1 || got[0].Fresh {
t.Fatalf("ReadRuntime(after re-register) = %+v, %v; want stale", got, err)
}
}
func newRuntimeStore(t *testing.T, now *time.Time) *MemoryStore {
t.Helper()
store, err := NewMemoryStore(func() time.Time { return *now })
if err != nil {
t.Fatalf("NewMemoryStore(): %v", err)
}
return store
}
func registerRuntimeSession(t *testing.T, store *MemoryStore, sessionID string, ttl time.Duration) {
t.Helper()
if err := store.ReplaceSession(context.Background(), Session{
WorkerID: "worker-a", InstanceID: "instance-a", SessionID: sessionID,
AckedSnapshotVersion: 3, AckedOwnershipEpoch: 9,
}, ttl); err != nil {
t.Fatalf("ReplaceSession(): %v", err)
}
}

70
internal/domain/workerruntime/runtime.go Normal file
View File

@ -0,0 +1,70 @@
package workerruntime
import (
"context"
"errors"
"time"
)
var (
ErrInvalidStore = errors.New("invalid worker runtime store")
ErrInvalidSession = errors.New("invalid worker runtime session")
ErrInvalidReport = errors.New("invalid worker runtime report")
ErrInvalidQuery = errors.New("invalid worker runtime query")
ErrStaleSession = errors.New("stale worker runtime session")
ErrStaleReport = errors.New("stale worker runtime report")
ErrConflictingReport = errors.New("conflicting worker runtime report")
)
type Session struct {
WorkerID string
InstanceID string
SessionID string
AckedSnapshotVersion uint64
AckedOwnershipEpoch uint64
}
type Counter struct {
ProxyID string
Active int64
Reserved int64
Draining bool
}
// Report is a complete sparse replacement. Missing counters are zero for the
// reported Worker snapshot; callers must increase Sequence for every update.
type Report struct {
WorkerID string
SessionID string
Sequence uint64
SnapshotVersion uint64
OwnershipEpoch uint64
ObservedAt time.Time
Counters []Counter
}
type OwnedProxy struct {
ProxyID string
WorkerID string
OwnershipEpoch uint64
}
type Snapshot struct {
ProxyID string
Active int64
Reserved int64
Draining bool
Fresh bool
}
type SessionWriter interface {
ReplaceSession(context.Context, Session, time.Duration) error
}
type ReportWriter interface {
ReplaceRuntime(context.Context, Report, time.Duration) error
}
type RuntimeReader interface {
ReadRuntime(context.Context, []OwnedProxy) ([]Snapshot, error)
}

172
internal/gateway/snapshot/store.go
View File

@ -7,19 +7,90 @@ import (
"errors"
"fmt"
"sort"
"strings"
"sync"
"sync/atomic"
"time"
proxyDomain "proxy-pool/internal/domain/proxy"
"proxy-pool/internal/domain/workerruntime"
)
var (
ErrWrongTarget = errors.New("snapshot targets another cluster or worker")
ErrResyncRequired = errors.New("snapshot sequence requires a full resync")
ErrChecksumMismatch = errors.New("snapshot checksum mismatch")
ErrInvalidRuntimeReport = errors.New("invalid worker runtime report")
ErrInvalidRuntimeLimit = errors.New("invalid snapshot runtime limit")
ErrRuntimeLimitExceeded = errors.New("snapshot runtime limit exceeded")
)
const defaultRuntimeLimit = 1_000_000
const activeRuntimeShardCount = 64
type activeRuntimeShard struct {
mu sync.Mutex
entries map[string]*proxyDomain.Capacity
}
type activeRuntimeIndex [activeRuntimeShardCount]activeRuntimeShard
func (index *activeRuntimeIndex) track(proxyID string, runtime *proxyDomain.Capacity, nonzero bool) {
shard := &index[activeRuntimeShardIndex(proxyID)]
shard.mu.Lock()
defer shard.mu.Unlock()
if nonzero {
if shard.entries == nil {
shard.entries = make(map[string]*proxyDomain.Capacity)
}
shard.entries[proxyID] = runtime
return
}
active, reserved, _ := runtime.Counters()
if active == 0 && reserved == 0 && shard.entries[proxyID] == runtime {
delete(shard.entries, proxyID)
}
}
func (index *activeRuntimeIndex) remove(proxyID string, runtime *proxyDomain.Capacity) {
shard := &index[activeRuntimeShardIndex(proxyID)]
shard.mu.Lock()
defer shard.mu.Unlock()
if shard.entries[proxyID] == runtime {
delete(shard.entries, proxyID)
}
}
func (index *activeRuntimeIndex) rangeEntries(visit func(string, *proxyDomain.Capacity)) {
for shardIndex := range index {
shard := &index[shardIndex]
shard.mu.Lock()
for proxyID, runtime := range shard.entries {
visit(proxyID, runtime)
}
shard.mu.Unlock()
}
}
func activeRuntimeShardIndex(proxyID string) uint64 {
const (
offset = uint64(14695981039346656037)
prime = uint64(1099511628211)
)
hash := offset
for index := 0; index < len(proxyID); index++ {
hash ^= uint64(proxyID[index])
hash *= prime
}
return hash % activeRuntimeShardCount
}
type runtimeRegistration struct {
capacity *proxyDomain.Capacity
current atomic.Bool
}
type Envelope struct {
ClusterID string
WorkerID string
@ -70,17 +141,29 @@ type Store struct {
current atomic.Pointer[View]
mu sync.Mutex
runtimes map[string]*proxyDomain.Capacity
runtimes map[string]*runtimeRegistration
active activeRuntimeIndex
limit int
}
func NewStore(clusterID, workerID string) *Store {
return &Store{
clusterID: clusterID,
workerID: workerID,
runtimes: make(map[string]*proxyDomain.Capacity),
runtimes: make(map[string]*runtimeRegistration),
limit: defaultRuntimeLimit,
}
}
func NewStoreWithRuntimeLimit(clusterID, workerID string, limit int) (*Store, error) {
if limit <= 0 {
return nil, ErrInvalidRuntimeLimit
}
store := NewStore(clusterID, workerID)
store.limit = limit
return store, nil
}
func (s *Store) Current() *View {
if s == nil {
return nil
@ -88,6 +171,49 @@ func (s *Store) Current() *View {
return s.current.Load()
}
func (s *Store) RuntimeReport(sessionID string, sequence uint64, observedAt time.Time) (workerruntime.Report, error) {
if s == nil || strings.TrimSpace(sessionID) != sessionID || sessionID == "" || sequence == 0 || observedAt.IsZero() {
return workerruntime.Report{}, ErrInvalidRuntimeReport
}
current := s.current.Load()
if current == nil {
return workerruntime.Report{}, ErrInvalidRuntimeReport
}
visible := make(map[string]struct{}, len(current.Entries))
counters := make([]workerruntime.Counter, 0)
for _, entry := range current.Entries {
visible[entry.Proxy.ID] = struct{}{}
active, reserved, _ := entry.Runtime.Counters()
if active == 0 && reserved == 0 {
continue
}
counters = append(counters, workerruntime.Counter{
ProxyID: entry.Proxy.ID, Active: active, Reserved: reserved,
Draining: entry.Proxy.State == proxyDomain.StateDraining,
})
}
s.active.rangeEntries(func(proxyID string, runtime *proxyDomain.Capacity) {
if _, currentProxy := visible[proxyID]; currentProxy {
return
}
active, reserved, _ := runtime.Counters()
if active == 0 && reserved == 0 {
return
}
counters = append(counters, workerruntime.Counter{
ProxyID: proxyID, Active: active, Reserved: reserved, Draining: true,
})
})
sort.Slice(counters, func(left, right int) bool {
return counters[left].ProxyID < counters[right].ProxyID
})
return workerruntime.Report{
WorkerID: s.workerID, SessionID: sessionID, Sequence: sequence,
SnapshotVersion: current.Version, OwnershipEpoch: current.Epoch,
ObservedAt: observedAt.UTC(), Counters: counters,
}, nil
}
func (s *Store) Apply(envelope Envelope) error {
if s == nil {
return fmt.Errorf("apply snapshot: nil store")
@ -118,19 +244,47 @@ func (s *Store) Apply(envelope Envelope) error {
}
proxies := cloneAndSort(envelope.Proxies)
newRuntimeCount := 0
for _, descriptor := range proxies {
if s.runtimes[descriptor.ID] == nil {
newRuntimeCount++
}
}
if len(s.runtimes)+newRuntimeCount > s.limit {
return ErrRuntimeLimitExceeded
}
if current != nil {
for _, entry := range current.Entries {
registration := s.runtimes[entry.Proxy.ID]
registration.capacity.SetActivityObservationEnabled(true)
registration.current.Store(false)
active, reserved, _ := registration.capacity.Counters()
s.active.track(entry.Proxy.ID, registration.capacity, active+reserved > 0)
}
}
entries := make([]Entry, 0, len(proxies))
for _, descriptor := range proxies {
runtime := s.runtimes[descriptor.ID]
if runtime == nil {
runtime = proxyDomain.NewCapacity(descriptor.MaxConcurrency)
} else {
runtime.SetMax(descriptor.MaxConcurrency)
registration := s.runtimes[descriptor.ID]
if registration == nil {
proxyID := descriptor.ID
registration = &runtimeRegistration{}
registration.capacity = proxyDomain.NewCapacityWithActivityObserver(descriptor.MaxConcurrency, func(nonzero bool) {
if registration.current.Load() {
return
}
s.active.track(proxyID, registration.capacity, nonzero)
})
} else {
registration.capacity.SetMax(descriptor.MaxConcurrency)
}
registration.current.Store(true)
registration.capacity.SetActivityObservationEnabled(false)
s.active.remove(descriptor.ID, registration.capacity)
// Keep runtimes for temporarily absent IDs. Old immutable views may still
// hold in-flight leases, so reclaiming here could reset active capacity if
// the same Proxy reappears in a later snapshot.
s.runtimes[descriptor.ID] = runtime
entries = append(entries, Entry{Proxy: descriptor, Runtime: runtime})
s.runtimes[descriptor.ID] = registration
entries = append(entries, Entry{Proxy: descriptor, Runtime: registration.capacity})
}
next := &View{

160
internal/gateway/snapshot/store_test.go
View File

@ -8,6 +8,7 @@ import (
"time"
proxyDomain "proxy-pool/internal/domain/proxy"
"proxy-pool/internal/domain/workerruntime"
)
func TestStoreAppliesCompleteSnapshotsInOrder(t *testing.T) {
@ -266,6 +267,165 @@ func TestStoreReusesRuntimeWhenProxyDisappearsAndReappears(t *testing.T) {
}
}
func TestStoreRuntimeReportKeepsRemovedActiveProxyUntilRelease(t *testing.T) {
store := NewStore("cluster-a", "worker-a")
now := time.Date(2026, 7, 30, 15, 0, 0, 0, time.UTC)
proxy := proxyDomain.Proxy{
ID: "proxy-a", Scheme: proxyDomain.SchemeHTTP, SourceUpstream: "provider-a",
State: proxyDomain.StateAvailable, MaxConcurrency: 1,
}
initial := Envelope{
ClusterID: "cluster-a", WorkerID: "worker-a", Epoch: 1, Version: 1, Full: true,
Proxies: []proxyDomain.Proxy{proxy},
}
initial.Checksum = Checksum(initial.Proxies)
if err := store.Apply(initial); err != nil {
t.Fatalf("Apply(initial): %v", err)
}
runtime := store.Current().Entries[0].Runtime
reservation, ok := runtime.Reserve()
if !ok {
t.Fatal("Reserve() = false")
}
if err := reservation.Commit(); err != nil {
t.Fatalf("Commit(): %v", err)
}
removed := Envelope{ClusterID: "cluster-a", WorkerID: "worker-a", Epoch: 1, Version: 2, Full: true}
removed.Checksum = Checksum(nil)
if err := store.Apply(removed); err != nil {
t.Fatalf("Apply(remove): %v", err)
}
if got := store.activeRuntimeCount(); got != 1 {
t.Fatalf("activeRuntimeCount(removed) = %d, want 1", got)
}
report, err := store.RuntimeReport("session-a", 7, now)
if err != nil {
t.Fatalf("RuntimeReport(): %v", err)
}
if report.WorkerID != "worker-a" || report.SessionID != "session-a" || report.Sequence != 7 ||
report.SnapshotVersion != 2 || report.OwnershipEpoch != 1 || !report.ObservedAt.Equal(now) ||
len(report.Counters) != 1 || report.Counters[0] != (workerruntime.Counter{
ProxyID: "proxy-a", Active: 1, Draining: true,
}) {
t.Fatalf("RuntimeReport() = %+v", report)
}
if err := reservation.Release(); err != nil {
t.Fatalf("Release(): %v", err)
}
if got := store.activeRuntimeCount(); got != 0 {
t.Fatalf("activeRuntimeCount(released) = %d, want 0", got)
}
report, err = store.RuntimeReport("session-a", 8, now.Add(time.Second))
if err != nil || len(report.Counters) != 0 {
t.Fatalf("RuntimeReport(after release) = %+v, %v", report, err)
}
}
func TestStoreRuntimeReportMarksCurrentDrainingProxy(t *testing.T) {
store := NewStore("cluster-a", "worker-a")
now := time.Date(2026, 7, 30, 15, 0, 0, 0, time.UTC)
proxy := proxyDomain.Proxy{
ID: "proxy-a", Scheme: proxyDomain.SchemeHTTP,
State: proxyDomain.StateDraining, MaxConcurrency: 1,
}
envelope := Envelope{
ClusterID: "cluster-a", WorkerID: "worker-a", Epoch: 1, Version: 1, Full: true,
Proxies: []proxyDomain.Proxy{proxy},
}
envelope.Checksum = Checksum(envelope.Proxies)
if err := store.Apply(envelope); err != nil {
t.Fatalf("Apply(): %v", err)
}
reservation, ok := store.Current().Entries[0].Runtime.Reserve()
if !ok {
t.Fatal("Reserve() = false")
}
report, err := store.RuntimeReport("session-a", 1, now)
if err != nil || len(report.Counters) != 1 || !report.Counters[0].Draining {
t.Fatalf("RuntimeReport() = %+v, %v", report, err)
}
if err := reservation.Cancel(); err != nil {
t.Fatalf("Cancel(): %v", err)
}
}
func TestStoreBoundsHistoricalRuntimeRegistryAndKeepsApplyTransactional(t *testing.T) {
store, err := NewStoreWithRuntimeLimit("cluster-a", "worker-a", 1)
if err != nil {
t.Fatalf("NewStoreWithRuntimeLimit(): %v", err)
}
first := proxyDomain.Proxy{
ID: "proxy-a", Scheme: proxyDomain.SchemeHTTP,
State: proxyDomain.StateAvailable, MaxConcurrency: 1,
}
envelope := Envelope{
ClusterID: "cluster-a", WorkerID: "worker-a", Epoch: 1, Version: 1,
Full: true, Proxies: []proxyDomain.Proxy{first},
}
envelope.Checksum = Checksum(envelope.Proxies)
if err := store.Apply(envelope); err != nil {
t.Fatalf("Apply(first): %v", err)
}
removed := Envelope{
ClusterID: "cluster-a", WorkerID: "worker-a", Epoch: 1, Version: 2, Full: true,
}
removed.Checksum = Checksum(nil)
if err := store.Apply(removed); err != nil {
t.Fatalf("Apply(removed): %v", err)
}
second := first
second.ID = "proxy-b"
overLimit := Envelope{
ClusterID: "cluster-a", WorkerID: "worker-a", Epoch: 1, Version: 3,
Full: true, Proxies: []proxyDomain.Proxy{second},
}
overLimit.Checksum = Checksum(overLimit.Proxies)
if err := store.Apply(overLimit); !errors.Is(err, ErrRuntimeLimitExceeded) {
t.Fatalf("Apply(over limit) error = %v, want ErrRuntimeLimitExceeded", err)
}
if current := store.Current(); current.Version != 2 || len(current.Entries) != 0 {
t.Fatalf("Current() after rejected apply = version %d entries %d", current.Version, len(current.Entries))
}
}
func TestStoreRuntimeActiveIndexDropsZeroCounters(t *testing.T) {
store := NewStore("cluster-a", "worker-a")
proxy := proxyDomain.Proxy{
ID: "proxy-a", Scheme: proxyDomain.SchemeHTTP,
State: proxyDomain.StateAvailable, MaxConcurrency: 1,
}
envelope := Envelope{
ClusterID: "cluster-a", WorkerID: "worker-a", Epoch: 1, Version: 1,
Full: true, Proxies: []proxyDomain.Proxy{proxy},
}
envelope.Checksum = Checksum(envelope.Proxies)
if err := store.Apply(envelope); err != nil {
t.Fatalf("Apply(): %v", err)
}
reservation, ok := store.Current().Entries[0].Runtime.Reserve()
if !ok {
t.Fatal("Reserve() = false")
}
if got := store.activeRuntimeCount(); got != 0 {
t.Fatalf("activeRuntimeCount(current) = %d, want 0", got)
}
if err := reservation.Cancel(); err != nil {
t.Fatalf("Cancel(): %v", err)
}
if got := store.activeRuntimeCount(); got != 0 {
t.Fatalf("activeRuntimeCount() = %d, want 0", got)
}
}
func (s *Store) activeRuntimeCount() int {
count := 0
s.active.rangeEntries(func(_ string, _ *proxyDomain.Capacity) {
count++
})
return count
}
func collectSelectionIDs(selection Selection) []string {
ids := make([]string, 0, selection.Len())
for index := 0; index < selection.Len(); index++ {

108
internal/platform/credentials/store.go
View File

@ -48,10 +48,27 @@ type Store interface {
Resolve(context.Context, Reference) (Value, error)
}
// Releaser removes transient credential material after the consumer has copied
// it into its authoritative storage. Release is idempotent and version fenced.
type Releaser interface {
Release(context.Context, Reference) error
}
type CapacityEnsurer interface {
EnsureCapacity(context.Context, int) error
}
type entry struct {
scope *scopeState
value Value
reference Reference
}
type scopeState struct {
name string
value Value
version uint64
reference Reference
leases int
}
// MemoryStore keeps credentials in process memory and serializes access with a
@ -59,7 +76,7 @@ type entry struct {
type MemoryStore struct {
lock chan struct{}
capacity int
byScope map[string]*entry
byScope map[string]*scopeState
byRef map[string]*entry
}
@ -82,7 +99,7 @@ func NewMemoryStore(capacity int) (*MemoryStore, error) {
return &MemoryStore{
lock: lock,
capacity: capacity,
byScope: make(map[string]*entry),
byScope: make(map[string]*scopeState),
byRef: make(map[string]*entry),
}, nil
}
@ -100,36 +117,38 @@ func (s *MemoryStore) Put(ctx context.Context, scope string, value Value) (Refer
if err := s.acquire(ctx); err != nil {
return Reference{}, err
}
defer s.release()
defer s.unlock()
if err := ctx.Err(); err != nil {
return Reference{}, err
}
if len(s.byRef) >= s.capacity {
return Reference{}, ErrCapacityExceeded
}
if current, ok := s.byScope[scope]; ok {
if current.value == value {
return current.reference, nil
}
current.value = value
current.version++
current.reference.CredentialVersion = versionString(current.version)
return current.reference, nil
}
if len(s.byScope) >= s.capacity {
current, exists := s.byScope[scope]
if !exists && len(s.byScope) >= s.capacity {
return Reference{}, ErrCapacityExceeded
}
secretRef, err := s.newUniqueSecretRef()
if err != nil {
return Reference{}, err
}
if !exists {
current = &scopeState{name: scope, value: value, version: 1}
s.byScope[scope] = current
} else if current.value != value {
current.value = value
current.version++
}
created := &entry{
scope: current,
value: value,
version: 1,
reference: Reference{
SecretRef: secretRef,
CredentialVersion: versionString(1),
CredentialVersion: versionString(current.version),
},
}
s.byScope[scope] = created
current.leases++
s.byRef[secretRef] = created
return created.reference, nil
}
@ -147,7 +166,7 @@ func (s *MemoryStore) Resolve(ctx context.Context, reference Reference) (Value,
if err := s.acquire(ctx); err != nil {
return Value{}, err
}
defer s.release()
defer s.unlock()
if err := ctx.Err(); err != nil {
return Value{}, err
}
@ -162,6 +181,57 @@ func (s *MemoryStore) Resolve(ctx context.Context, reference Reference) (Value,
return current.value, nil
}
func (s *MemoryStore) Release(ctx context.Context, reference Reference) error {
if err := contextError(ctx); err != nil {
return err
}
if !s.valid() {
return ErrInvalidStore
}
if reference.SecretRef == "" || !validVersion(reference.CredentialVersion) {
return ErrInvalidReference
}
if err := s.acquire(ctx); err != nil {
return err
}
defer s.unlock()
if err := ctx.Err(); err != nil {
return err
}
current, ok := s.byRef[reference.SecretRef]
if !ok || current.reference != reference {
return nil
}
delete(s.byRef, reference.SecretRef)
current.scope.leases--
if current.scope.leases == 0 {
delete(s.byScope, current.scope.name)
current.scope.value = Value{}
}
current.value = Value{}
return nil
}
func (s *MemoryStore) EnsureCapacity(ctx context.Context, minimum int) error {
if err := contextError(ctx); err != nil {
return err
}
if !s.valid() {
return ErrInvalidStore
}
if minimum <= 0 {
return ErrInvalidCapacity
}
if err := s.acquire(ctx); err != nil {
return err
}
defer s.unlock()
if minimum > s.capacity {
s.capacity = minimum
}
return nil
}
func (s *MemoryStore) acquire(ctx context.Context) error {
if err := contextError(ctx); err != nil {
return err
@ -174,7 +244,7 @@ func (s *MemoryStore) acquire(ctx context.Context) error {
}
}
func (s *MemoryStore) release() {
func (s *MemoryStore) unlock() {
s.lock <- struct{}{}
}

148
internal/platform/credentials/store_test.go
View File

@ -11,7 +11,7 @@ import (
"time"
)
func TestMemoryStorePutIsIdempotentForUnchangedScope(t *testing.T) {
func TestMemoryStorePutCreatesIndependentLeasesForUnchangedScope(t *testing.T) {
store, err := NewMemoryStore(2)
if err != nil {
t.Fatalf("NewMemoryStore(): %v", err)
@ -26,10 +26,10 @@ func TestMemoryStorePutIsIdempotentForUnchangedScope(t *testing.T) {
if err != nil {
t.Fatalf("Put(second): %v", err)
}
if first != second {
t.Fatalf("second reference = %#v, want %#v", second, first)
if first == second || first.SecretRef == second.SecretRef {
t.Fatalf("references = %#v and %#v, want independent leases", first, second)
}
if first.SecretRef == "" || first.CredentialVersion != "v1" {
if first.SecretRef == "" || first.CredentialVersion != "v1" || second.CredentialVersion != "v1" {
t.Fatalf("first reference = %#v, want opaque ref at v1", first)
}
for _, plaintext := range []string{"provider-a", value.Username, value.Password} {
@ -45,6 +45,12 @@ func TestMemoryStorePutIsIdempotentForUnchangedScope(t *testing.T) {
if got != value {
t.Fatalf("Resolve() = %#v, want %#v", got, value)
}
if err := store.Release(context.Background(), first); err != nil {
t.Fatalf("Release(first lease): %v", err)
}
if got, err := store.Resolve(context.Background(), second); err != nil || got != value {
t.Fatalf("Resolve(second lease) = %#v, %v", got, err)
}
}
func TestCredentialFormattingRedactsSensitiveMaterial(t *testing.T) {
@ -89,8 +95,8 @@ func TestCredentialFormattingRedactsSensitiveMaterial(t *testing.T) {
}
}
func TestMemoryStorePutIncrementsVersionAndRejectsStaleReference(t *testing.T) {
store, err := NewMemoryStore(1)
func TestMemoryStorePutIncrementsVersionWithoutRevokingActiveLease(t *testing.T) {
store, err := NewMemoryStore(2)
if err != nil {
t.Fatalf("NewMemoryStore(): %v", err)
}
@ -106,14 +112,15 @@ func TestMemoryStorePutIncrementsVersionAndRejectsStaleReference(t *testing.T) {
if err != nil {
t.Fatalf("Put(new): %v", err)
}
if newReference.SecretRef != oldReference.SecretRef {
t.Fatalf("new SecretRef changed across versions")
if newReference.SecretRef == oldReference.SecretRef {
t.Fatalf("new SecretRef reused an active lease")
}
if newReference.CredentialVersion != "v2" {
t.Fatalf("new CredentialVersion = %q, want v2", newReference.CredentialVersion)
}
if _, err := store.Resolve(context.Background(), oldReference); !errors.Is(err, ErrCredentialVersionMismatch) {
t.Fatalf("Resolve(stale) error = %v, want ErrCredentialVersionMismatch", err)
old, err := store.Resolve(context.Background(), oldReference)
if err != nil || old.Password != "old-password" {
t.Fatalf("Resolve(active old lease) = %#v, %v", old, err)
}
got, err := store.Resolve(context.Background(), newReference)
if err != nil {
@ -148,8 +155,80 @@ func TestMemoryStoreEnforcesCapacityWithoutChangingExistingCredentials(t *testin
if got != want {
t.Fatalf("Resolve(existing) returned changed credentials")
}
if _, err := store.Put(context.Background(), "provider-a", want); !errors.Is(err, ErrCapacityExceeded) {
t.Fatalf("Put(second lease at capacity) error = %v, want ErrCapacityExceeded", err)
}
if err := store.Release(context.Background(), reference); err != nil {
t.Fatalf("Release(first lease): %v", err)
}
if _, err := store.Put(context.Background(), "provider-a", want); err != nil {
t.Fatalf("Put(idempotent at capacity): %v", err)
t.Fatalf("Put(after lease release): %v", err)
}
}
func TestMemoryStoreReleaseMakesCapacityReusable(t *testing.T) {
store, err := NewMemoryStore(1)
if err != nil {
t.Fatalf("NewMemoryStore(): %v", err)
}
first, err := store.Put(context.Background(), "provider-a", Value{Password: "first-password"})
if err != nil {
t.Fatalf("Put(first): %v", err)
}
if err := store.Release(context.Background(), first); err != nil {
t.Fatalf("Release(first): %v", err)
}
if _, err := store.Resolve(context.Background(), first); !errors.Is(err, ErrCredentialMissing) {
t.Fatalf("Resolve(released) error = %v, want ErrCredentialMissing", err)
}
if _, err := store.Put(context.Background(), "provider-b", Value{Password: "second-password"}); err != nil {
t.Fatalf("Put(after release): %v", err)
}
if err := store.Release(context.Background(), first); err != nil {
t.Fatalf("Release(idempotent): %v", err)
}
}
func TestMemoryStoreReleaseOfStaleReferencePreservesCurrentVersion(t *testing.T) {
store, err := NewMemoryStore(2)
if err != nil {
t.Fatalf("NewMemoryStore(): %v", err)
}
stale, err := store.Put(context.Background(), "provider-a", Value{Password: "old-password"})
if err != nil {
t.Fatalf("Put(old): %v", err)
}
current, err := store.Put(context.Background(), "provider-a", Value{Password: "new-password"})
if err != nil {
t.Fatalf("Put(new): %v", err)
}
if err := store.Release(context.Background(), stale); err != nil {
t.Fatalf("Release(stale): %v", err)
}
if _, err := store.Resolve(context.Background(), current); err != nil {
t.Fatalf("Resolve(current): %v", err)
}
}
func TestMemoryStoreEnsureCapacityOnlyGrowsLimit(t *testing.T) {
store, err := NewMemoryStore(1)
if err != nil {
t.Fatalf("NewMemoryStore(): %v", err)
}
if err := store.EnsureCapacity(context.Background(), 2); err != nil {
t.Fatalf("EnsureCapacity(2): %v", err)
}
if _, err := store.Put(context.Background(), "provider-a", Value{}); err != nil {
t.Fatalf("Put(provider-a): %v", err)
}
if _, err := store.Put(context.Background(), "provider-b", Value{}); err != nil {
t.Fatalf("Put(provider-b): %v", err)
}
if err := store.EnsureCapacity(context.Background(), 1); err != nil {
t.Fatalf("EnsureCapacity(shrink request): %v", err)
}
if _, err := store.Put(context.Background(), "provider-c", Value{}); !errors.Is(err, ErrCapacityExceeded) {
t.Fatalf("Put(provider-c) error = %v, want retained capacity 2", err)
}
}
@ -211,13 +290,13 @@ func TestMemoryStoreRejectsNilAndZeroValueStores(t *testing.T) {
}
}
func TestMemoryStoreIsConcurrencySafeAndIdempotent(t *testing.T) {
store, err := NewMemoryStore(1)
func TestMemoryStoreCreatesIndependentConcurrentLeases(t *testing.T) {
const workers = 100
store, err := NewMemoryStore(workers)
if err != nil {
t.Fatalf("NewMemoryStore(): %v", err)
}
want := Value{Username: "alice", Password: "shared-password"}
const workers = 100
references := make(chan Reference, workers)
errorsSeen := make(chan error, workers)
var wait sync.WaitGroup
@ -239,29 +318,29 @@ func TestMemoryStoreIsConcurrencySafeAndIdempotent(t *testing.T) {
for err := range errorsSeen {
t.Errorf("concurrent Put(): %v", err)
}
var first Reference
secretRefs := make(map[string]struct{}, workers)
for reference := range references {
if first == (Reference{}) {
first = reference
if reference.CredentialVersion != "v1" {
t.Errorf("concurrent version = %q, want v1", reference.CredentialVersion)
}
if reference != first {
t.Errorf("concurrent Put() reference differs from first")
secretRefs[reference.SecretRef] = struct{}{}
}
if len(secretRefs) != workers {
t.Fatalf("unique concurrent leases = %d, want %d", len(secretRefs), workers)
}
if first.CredentialVersion != "v1" {
t.Fatalf("concurrent version = %q, want v1", first.CredentialVersion)
for secretRef := range secretRefs {
got, err := store.Resolve(context.Background(), Reference{
SecretRef: secretRef, CredentialVersion: "v1",
})
if err != nil || got != want {
t.Fatalf("Resolve(concurrent lease) = %#v, %v", got, err)
}
got, err := store.Resolve(context.Background(), first)
if err != nil {
t.Fatalf("Resolve(): %v", err)
}
if got != want {
t.Fatalf("Resolve() returned unexpected credentials")
}
}
func TestMemoryStoreSerializesConcurrentCredentialChanges(t *testing.T) {
store, err := NewMemoryStore(1)
const workers = 100
store, err := NewMemoryStore(workers)
if err != nil {
t.Fatalf("NewMemoryStore(): %v", err)
}
@ -270,7 +349,6 @@ func TestMemoryStoreSerializesConcurrentCredentialChanges(t *testing.T) {
reference Reference
err error
}
const workers = 100
results := make(chan result, workers)
var wait sync.WaitGroup
for index := range workers {
@ -286,18 +364,13 @@ func TestMemoryStoreSerializesConcurrentCredentialChanges(t *testing.T) {
close(results)
versions := make(map[string]struct{}, workers)
var secretRef string
secretRefs := make(map[string]struct{}, workers)
var latest result
for current := range results {
if current.err != nil {
t.Fatalf("concurrent Put(): %v", current.err)
}
if secretRef == "" {
secretRef = current.reference.SecretRef
}
if current.reference.SecretRef != secretRef {
t.Fatal("SecretRef changed across concurrent updates")
}
secretRefs[current.reference.SecretRef] = struct{}{}
versions[current.reference.CredentialVersion] = struct{}{}
if current.reference.CredentialVersion == "v100" {
latest = current
@ -306,6 +379,9 @@ func TestMemoryStoreSerializesConcurrentCredentialChanges(t *testing.T) {
if len(versions) != workers {
t.Fatalf("unique versions = %d, want %d", len(versions), workers)
}
if len(secretRefs) != workers {
t.Fatalf("unique leases = %d, want %d", len(secretRefs), workers)
}
if latest.reference == (Reference{}) {
t.Fatal("highest version v100 was not returned")
}

72
internal/platform/lifecycle/group.go Normal file
View File

@ -0,0 +1,72 @@
package lifecycle
import (
"context"
"errors"
"reflect"
)
var (
ErrInvalidGroup = errors.New("invalid lifecycle group")
ErrRunnerStopped = errors.New("lifecycle runner stopped")
)
type Runner interface {
Run(context.Context) error
}
type Group struct {
runners []Runner
}
func NewGroup(runners ...Runner) (*Group, error) {
if len(runners) == 0 {
return nil, ErrInvalidGroup
}
owned := make([]Runner, len(runners))
for index, runner := range runners {
if isNilRunner(runner) {
return nil, ErrInvalidGroup
}
owned[index] = runner
}
return &Group{runners: owned}, nil
}
func (group *Group) Run(ctx context.Context) error {
if group == nil || ctx == nil || len(group.runners) == 0 {
return ErrInvalidGroup
}
groupCtx, cancel := context.WithCancel(ctx)
defer cancel()
done := make(chan error, len(group.runners))
for _, runner := range group.runners {
go func() { done <- runner.Run(groupCtx) }()
}
first := <-done
cancel()
for range len(group.runners) - 1 {
<-done
}
if ctx.Err() != nil {
return ctx.Err()
}
if first == nil {
return ErrRunnerStopped
}
return first
}
func isNilRunner(runner Runner) bool {
if runner == nil {
return true
}
reflected := reflect.ValueOf(runner)
switch reflected.Kind() {
case reflect.Chan, reflect.Func, reflect.Interface, reflect.Map, reflect.Pointer, reflect.Slice:
return reflected.IsNil()
default:
return false
}
}

87
internal/platform/lifecycle/group_test.go Normal file
View File

@ -0,0 +1,87 @@
package lifecycle
import (
"context"
"errors"
"sync/atomic"
"testing"
"time"
)
func TestGroupStartsAllRunnersAndCancelsSiblingsOnFirstError(t *testing.T) {
started := make(chan struct{}, 2)
cancelled := make(chan struct{}, 1)
wantErr := errors.New("runner failed")
group, err := NewGroup(
runnerFunc(func(context.Context) error {
started <- struct{}{}
return wantErr
}),
runnerFunc(func(ctx context.Context) error {
started <- struct{}{}
<-ctx.Done()
cancelled <- struct{}{}
return ctx.Err()
}),
)
if err != nil {
t.Fatalf("NewGroup(): %v", err)
}
if err := group.Run(context.Background()); !errors.Is(err, wantErr) {
t.Fatalf("Run() error = %v, want %v", err, wantErr)
}
<-started
<-started
select {
case <-cancelled:
case <-time.After(time.Second):
t.Fatal("sibling runner was not cancelled")
}
}
func TestGroupWaitsForEveryRunnerBeforeReturning(t *testing.T) {
release := make(chan struct{})
var exited atomic.Bool
group, err := NewGroup(
runnerFunc(func(context.Context) error { return errors.New("failed") }),
runnerFunc(func(ctx context.Context) error {
<-ctx.Done()
<-release
exited.Store(true)
return nil
}),
)
if err != nil {
t.Fatalf("NewGroup(): %v", err)
}
done := make(chan error, 1)
go func() { done <- group.Run(context.Background()) }()
select {
case <-done:
t.Fatal("Run() returned before sibling exited")
case <-time.After(20 * time.Millisecond):
}
close(release)
<-done
if !exited.Load() {
t.Fatal("sibling exit was not observed")
}
}
func TestNewGroupRejectsEmptyAndTypedNilRunners(t *testing.T) {
var typedNil *nilRunner
for _, runners := range [][]Runner{nil, {typedNil}} {
group, err := NewGroup(runners...)
if err == nil || group != nil {
t.Fatalf("NewGroup() = (%v, %v), want invalid group", group, err)
}
}
}
type runnerFunc func(context.Context) error
func (f runnerFunc) Run(ctx context.Context) error { return f(ctx) }
type nilRunner struct{}
func (*nilRunner) Run(context.Context) error { return nil }

20
progress.md
View File

@ -2,6 +2,26 @@
## 2026-07-30
- Gateway `Capacity` 新增一次打包原子读取，`snapshot.Store` 可生成完整稀疏
Active/Reserved 运行态报告；当前快照已移除但仍有连接的 Proxy 会持续以
draining 上报，归零后从后续报告消失。
- 新增公用 `workerruntime` session/report/read seam 与并发安全 MemoryStore；
完整替换、空报告清零、session fencing、单调 sequence、同内容幂等重放、
冲突/倒序拒绝和 TTL fail-closed 均已有单测。
- 生产 `redisactivity` Adapter 新增 Worker session/运行态 Lua 和权威
`pool.InventoryReader`；Managed/Available Slots 原子计入状态、TTL safety、
MaxConcurrency、ownership 及 Active/Reserved，未知或过期运行态贡献零容量。
- 真实 Redis 8.2 已覆盖 ACK snapshot/epoch、旧 session、空报告、超前 epoch、
报告过期、扫描预算耗尽及容量聚合。Managed 使用已有权威计数，Available
Slots 只扫描目标 Upstream 的未分配/已分配可用索引；Gateway 报告扫描当前
Snapshot，并用分片索引补充已移除但仍非零的 runtime，历史注册表设置硬上限。
WorkerControlPlane 接收端、Provider
Fleet/bootstrap、全局 `fetch.maxTotal` 和健康执行链仍待完成，总验收计数保持
51/73。
- Gateway 当前 Proxy 禁用活跃索引回调，只有移出 Snapshot 后才开启分片追踪；
本机 100k Proxy `Acquire` 三轮 1 秒基准为 893.9-1047 ns/op、256 B/op、
2 allocs/op。该数据只证明本地调度微基准，不代表 100k QPS 集群验收。
- 新增公用 Provider `Coordinator.RunLeader` / `LeaderSession` 深 seam 和独立
`redisprovider` Adapter；Redis Lua 原子维护 generation、epoch、Leader 租约、
全局 requestInterval 与带 TTL 的 maxInFlight Permit，异常时 fail-closed。

8
task_plan.md
View File

@ -35,7 +35,8 @@
机器契约和文档类滞后勾选已按仓库证据校正
13. [进行中] 落地 `proxy-controller` 进程装配；配置单次加载、PostgreSQL 迁移、
Redis 活动池、低基数状态聚合、Distribution/Admin/Metrics 启动与关闭已完成，
双存储 bootstrap 和探针集成已通过，Provider、业务指标与完整容器进程链仍待实现
双存储 bootstrap 和探针集成已通过；Worker 运行态存储与权威容量读取原语已
完成，WorkerControlPlane、Provider、业务指标与完整容器进程链仍待实现
## 串并行关系
@ -57,7 +58,8 @@
- Docker Compose 配置与 Kubernetes Kustomize 已完成静态渲染验证；Redis 8.2
与 PostgreSQL 18 的隔离 Adapter fixture 已运行，完整目标运行拓扑尚未启动。
- `cmd/proxy-controller` 已实现 Admin/Distribution/Metrics 与双存储启动装配；
Gateway、Checker、Loadgen、Provider Leader/分布式限流、业务指标、Redis
故障转移验证与代表性集群压测属于后续实施范围。
Provider 分布式协调和 Worker 运行态 Redis 原语已完成，但 WorkerControlPlane
接收端、Provider Fleet、Gateway、Checker、Loadgen、业务指标、Redis 故障
转移验证与代表性集群压测属于后续实施范围。
- `implementation-plan.md` 当前按 73 个验收项统计；已校正为 51 项完成，
验收项完成率约 69.9%，不等同于生产就绪度。