Compare commits

...

5 Commits

Author SHA1 Message Date
youfak
b8f5104167 docs: record provider lifecycle delivery
Some checks are pending
ci / test (ubuntu-latest) (push) Waiting to run
ci / test (windows-latest) (push) Waiting to run
ci / race (push) Waiting to run
ci / integration (push) Waiting to run
2026-07-30 19:47:10 +08:00
youfak
8997e3880e feat: run provider lifecycle in controller 2026-07-30 19:47:02 +08:00
youfak
40f4b3ffab feat: enforce distributed provider fetch quota 2026-07-30 16:11:04 +08:00
youfak
d648ba37e0 docs: record worker runtime capacity delivery 2026-07-30 15:46:27 +08:00
youfak
45ba6fb958 feat: add authoritative worker runtime capacity 2026-07-30 15:46:18 +08:00
95 changed files with 6683 additions and 511 deletions

View File

@ -52,6 +52,7 @@ Windows PowerShell 可运行:
2. Proxy 容量使用 `Reserved -> Active` 原子转换,禁止超卖。
3. Distribution 成功时原子执行 `AVAILABLE -> EXTRACTED`,不提供 Lease、
Release 或 Renewal。
4. `pool.maxSize` 是当前未提取库存上限;`fetch.maxTotal` 是累计获取额度。
4. `pool.maxSize` 是当前未提取库存硬上限;`fetch.maxTotal` 是 Redis generation
内的累计获取停止阈值。
5. CONNECT 向客户端提交 200 后不透明重放。
6. 公开监听必须有认证或 CIDR 访问保护。

View File

@ -182,6 +182,7 @@ message ReportRuntimeRequest {
uint64 ownership_epoch = 4;
repeated ProxyRuntime counters = 5;
google.protobuf.Timestamp observed_at = 6;
uint64 report_sequence = 7;
}
message ProxyRuntime {

View File

@ -15,7 +15,10 @@ import (
"proxy-pool/internal/controller/bootstrap"
)
const configEnvironment = "PROXY_POOL_CONFIG"
const (
configEnvironment = "PROXY_POOL_CONFIG"
fingerprintKeyEnvironment = "PROXY_POOL_CONFIG_FINGERPRINT_KEY"
)
type environmentLookup func(string) string
type controllerRun func(context.Context, bootstrap.Options) error
@ -54,7 +57,13 @@ func execute(
return 2
}
err := run(ctx, bootstrap.Options{ConfigPath: *configPath, Resolver: config.OSResolver{}})
var fingerprintKey []byte
if getenv != nil {
fingerprintKey = []byte(getenv(fingerprintKeyEnvironment))
}
err := run(ctx, bootstrap.Options{
ConfigPath: *configPath, Resolver: config.OSResolver{}, FingerprintKey: fingerprintKey,
})
if err == nil || (errors.Is(err, context.Canceled) && ctx.Err() != nil) {
return 0
}

View File

@ -16,12 +16,15 @@ func TestExecuteUsesFlagBeforeEnvironment(t *testing.T) {
if name == configEnvironment {
return "environment.yaml"
}
if name == fingerprintKeyEnvironment {
return "0123456789abcdef0123456789abcdef"
}
return ""
}, func(_ context.Context, options bootstrap.Options) error {
received = options
return nil
}, &bytes.Buffer{})
if code != 0 || received.ConfigPath != "flag.yaml" || received.Resolver == nil {
if code != 0 || received.ConfigPath != "flag.yaml" || received.Resolver == nil || len(received.FingerprintKey) != 32 {
t.Fatalf("execute() = %d, options = %+v", code, received)
}
}
@ -29,7 +32,12 @@ func TestExecuteUsesFlagBeforeEnvironment(t *testing.T) {
func TestExecuteFallsBackToEnvironment(t *testing.T) {
t.Parallel()
var received bootstrap.Options
code := execute(context.Background(), nil, func(string) string { return "environment.yaml" }, func(
code := execute(context.Background(), nil, func(name string) string {
if name == configEnvironment {
return "environment.yaml"
}
return "0123456789abcdef0123456789abcdef"
}, func(
_ context.Context,
options bootstrap.Options,
) error {

View File

@ -1,5 +1,13 @@
name: proxy-pool
x-app-environment: &app-environment
PROXY_POOL_CONFIG: /etc/proxy-pool/config.yaml
PROXY_POOL_GATEWAY_PASSWORD: ${PROXY_POOL_GATEWAY_PASSWORD:?set PROXY_POOL_GATEWAY_PASSWORD}
PROXY_POOL_EXTRACT_TOKEN: ${PROXY_POOL_EXTRACT_TOKEN:?set PROXY_POOL_EXTRACT_TOKEN}
PROXY_POOL_ADMIN_TOKEN: ${PROXY_POOL_ADMIN_TOKEN:?set PROXY_POOL_ADMIN_TOKEN}
PROVIDER_A_TOKEN: ${PROVIDER_A_TOKEN:?set PROVIDER_A_TOKEN}
PROVIDER_B_TOKEN: ${PROVIDER_B_TOKEN:?set PROVIDER_B_TOKEN}
x-app: &app
build:
context: ..
@ -9,13 +17,7 @@ x-app: &app
networks: [frontend, backend]
volumes:
- ./config/local.yaml:/etc/proxy-pool/config.yaml:ro
environment:
PROXY_POOL_CONFIG: /etc/proxy-pool/config.yaml
PROXY_POOL_GATEWAY_PASSWORD: ${PROXY_POOL_GATEWAY_PASSWORD:?set PROXY_POOL_GATEWAY_PASSWORD}
PROXY_POOL_EXTRACT_TOKEN: ${PROXY_POOL_EXTRACT_TOKEN:?set PROXY_POOL_EXTRACT_TOKEN}
PROXY_POOL_ADMIN_TOKEN: ${PROXY_POOL_ADMIN_TOKEN:?set PROXY_POOL_ADMIN_TOKEN}
PROVIDER_A_TOKEN: ${PROVIDER_A_TOKEN:?set PROVIDER_A_TOKEN}
PROVIDER_B_TOKEN: ${PROVIDER_B_TOKEN:?set PROVIDER_B_TOKEN}
environment: *app-environment
stop_grace_period: 45s
services:
@ -44,6 +46,9 @@ services:
controller:
<<: *app
command: ["proxy-controller"]
environment:
<<: *app-environment
PROXY_POOL_CONFIG_FINGERPRINT_KEY: ${PROXY_POOL_CONFIG_FINGERPRINT_KEY:?set PROXY_POOL_CONFIG_FINGERPRINT_KEY}
depends_on:
postgres:
condition: service_healthy

View File

@ -9,8 +9,8 @@ stringData:
PROXY_POOL_GATEWAY_PASSWORD: GATEWAY_PASSWORD
PROXY_POOL_EXTRACT_TOKEN: EXTRACT_TOKEN
PROXY_POOL_ADMIN_TOKEN: ADMIN_TOKEN
PROXY_POOL_CONFIG_FINGERPRINT_KEY: CONFIG_FINGERPRINT_KEY_MINIMUM_32_BYTES
PROXY_POOL_POSTGRES_URL: postgres://USER:PASSWORD@POSTGRES_HOST:5432/proxy_pool?sslmode=verify-full
PROXY_POOL_REDIS_URL: rediss://:PASSWORD@REDIS_HOST:6379/0
PROVIDER_A_TOKEN: PROVIDER_A_TOKEN
PROVIDER_B_TOKEN: PROVIDER_B_TOKEN

View File

@ -63,10 +63,14 @@ var _ activitypool.HealthStore = (*Adapter)(nil)
var _ activitypool.InventoryReader = (*Adapter)(nil)
var _ extraction.Store = (*Adapter)(nil)
var _ ownership.Repository = (*Adapter)(nil)
var _ workerruntime.SessionWriter = (*Adapter)(nil)
var _ workerruntime.ReportWriter = (*Adapter)(nil)
var _ workerruntime.RuntimeReader = (*Adapter)(nil)
var _ pool.InventoryReader = (*Adapter)(nil)
```
Provider、Checker、Distribution 和 Ownership 只依赖各自需要的端口,不直接
依赖 Redis 客户端、键名、Lua 返回格式或清理策略。
Provider、Checker、Distribution、Ownership 和 Worker Runtime 只依赖各自需要
的端口,不直接依赖 Redis 客户端、键名、Lua 返回格式或清理策略。
`ownership.Repository` 改为适合远程存储的上下文感知接口:
@ -97,6 +101,11 @@ pp:{activity}:owners HASH proxyID -> ownership assignment
pp:{activity}:owner-expiry ZSET proxyID -> ownership expiry milliseconds
pp:{activity}:epoch STRING ownership 全局递增代次
pp:{activity}:inventory HASH upstreamID -> 当前未提取库存
pp:{activity}:worker-sessions HASH workerID -> 当前 Worker session
pp:{activity}:worker-session-expiry ZSET workerID -> session expiry milliseconds
pp:{activity}:worker-runtime HASH workerID -> 完整稀疏运行态报告
pp:{activity}:worker-runtime-expiry ZSET workerID -> report expiry milliseconds
pp:{activity}:owned:<digest> ZSET 单 Upstream 已分配 AVAILABLE Proxy
pp:{activity}:idem:<digest> STRING 带 TTL 的提取幂等结果
pp:{activity}:op:<digest> STRING 带 TTL 的内部操作结果
```
@ -167,6 +176,38 @@ Assign、Renew、BeginDrain 和 AcknowledgeDrain 分别使用有界小脚本,
- Assign 与 Extract 并发竞争同一 Proxy 时,只允许一个操作成功。
- Expire 使用 `limit` 分批回收过期 assignment禁止无界返回。
### Worker 运行态与容量汇总
Gateway 的 `Capacity` 使用一次打包原子读取取得同一时刻的 Active/Reserved
`snapshot.Store` 周期生成完整稀疏报告。当前 Snapshot 已移除但仍有活动连接的
Proxy 继续以 `draining=true` 上报,直到 Active/Reserved 同时归零。
Redis Adapter 在单个 `{activity}` 原子边界内维护 Worker session 和运行态报告:
1. 新 Worker session 替换旧 session并隔离旧实例后续写入。
2. session 保存 Controller 已 ACK 的 snapshot version 与 ownership epoch报告
必须与 ACK 上界完全一致,不能通过自报超前 epoch 绕过所有权校验。
3. `report_sequence` 严格递增;同序号、同内容可幂等重放,冲突或倒序拒绝。
4. 非零计数必须匹配当前 Proxy owner、Worker ID 和 ownership epoch。
5. session/report TTL 使用 Redis 服务端时间;过期、缺失或损坏时容量 fail-closed。
6. 空报告清除该 Worker 的全部旧计数,稀疏报告中缺失的 Proxy 计数视为零。
`pool.InventoryReader` 低频返回单个 Upstream 的 `Managed`
`AvailableSlots`。Managed 统计 FETCHED/CHECKING/AVAILABLE/SUSPECT/DRAINING
Available Slots 只统计超过 safety margin、状态为 AVAILABLE 且所有权与新鲜
Worker 运行态一致的 `max - active - reserved`。未分配 Proxy 可直接贡献 Max
已分配但运行态未知的 Proxy 贡献零槽位。PostgreSQL 不保存这些短效报告或容量
明细Gateway 每次请求也不访问 Redis。
Managed 直接读取现有 Upstream 权威计数Available Slots 只扫描目标 Upstream
的未分配可用索引与已分配可用索引,不扫描全局 Proxy也不受其他供应商活记录
数量影响。索引成员数超过单次扫描预算时直接返回不可用,不降级为近似容量。
生产规模验收仍需验证脚本 p95/p99、CPU、内存和过期风暴下的有界行为。
Gateway `snapshot.Store` 扫描有界的当前 Snapshot并使用分片索引补充已移除但
仍非零的 runtime不扫描全部历史 Proxy历史 Capacity 注册表设硬上限,达到
上限时拒绝新 Snapshot 并保持旧视图,避免长期轮换造成无界内存增长。
### 短 TTL 清理
Redis Hash 字段没有独立 TTL因此使用三层有界清理
@ -226,6 +267,8 @@ Redis `inventory` 是当前未提取 Proxy 数量的运行时真值:
- Assign 与 Extract 并发互斥,以及 renew/drain/ACK/expire。
- 提交后连接断开、脚本缓存丢失、上下文取消和 Redis 不可用。
- 30 秒 TTL 持续写入下的有界清理与库存一致性。
- Worker session 替换、运行态序号幂等/冲突、报告过期和 ownership epoch 隔离。
- 权威 Managed/Available Slots 聚合及扫描预算耗尽时的 fail-closed 行为。
Lua 语义必须使用真实 Redis 8.2 集成测试验证。单元测试最长 60 秒,并执行
gofmt、go vet、全量测试、构建和 diff whitespace 检查。100,000 QPS 只能由

View File

@ -70,9 +70,14 @@ Outbox。状态变化、审计和 Outbox 在同一事务提交。
### 配置修订
配置提交只保存管理面恢复所需的非敏感事实配置版本、SHA-256 校验和、来源、
Upstream 启用状态和 Routing 候选/当前选择。已解析 Secret、Provider Token、
Proxy 凭据和完整运行时对象不进入 PostgreSQL。
配置提交只保存管理面恢复所需的非敏感事实:配置版本、完整已解析配置的
HMAC-SHA-256 指纹、来源、Upstream 启用状态和 Routing 候选/当前选择。HMAC 使用
独立外部高熵密钥,覆盖 Secret 轮换以驱动多副本收敛,同时避免普通摘要成为
低熵 Secret 的离线校验器。HMAC 密钥、已解析 Secret、Provider Token、Proxy
凭据、配置正文和完整运行时对象均不进入 PostgreSQL。
配置发布携带事务返回的全局 revision。本地 `config.Store` 只接受严格递增
revision因此并发提交或 Supervisor 同步的迟到旧版本不能覆盖较新运行配置。
配置重载以一个事务替换管理快照。新 Routing 的当前 Upstream 必须属于其候选集,
所有引用的 Upstream 必须存在,名称与列表必须非空且唯一。校验失败发生在事务前,

View File

@ -39,10 +39,24 @@ Status 以一个权威管理快照决定 Upstream 集合和 Enabled 状态,只
1. `FileConfigurationLoader` 通过 `config.LoadResolved` 严格解析、解析 Secret 引用
并完成全量校验。
2. 从脱敏管理投影计算版本与校验和Secret 值及其可验证摘要不进入管理状态。
2. 使用独立外部密钥对完整已解析配置计算 HMAC-SHA-256 指纹PostgreSQL 只保存
不透明 HMAC不保存密钥、配置正文或 Secret 明文,因此 URL、模板和 Secret
轮换都会产生新版本,也不能借数据库摘要离线猜测低熵 Secret。
3. 在同一 `adminstate` mutation 中提交配置修订、管理状态、审计和 Outbox。
4. 提交成功后由 `config.Store` 一次原子指针交换发布完整运行配置;提交失败时旧
配置保持不变。幂等重放仍执行发布,以修复进程本地状态。
4. 提交成功后由 `config.Store` 按 PostgreSQL revision 原子发布完整运行配置;
Store 只接受严格递增 revision提交失败或迟到旧 revision 不覆盖当前配置。
幂等重放仅在本地 revision 落后时修复进程状态。
5. Provider Supervisor 在提交前构造预检所有启用 Upstream发布后按新配置取消、
替换或新增 Runtime。enable 同样在管理状态 mutation 前预检目标 Runtime。
6. 其他 Controller 每秒比较本地指纹与 PostgreSQL 权威指纹;所有副本必须使用
相同 `PROXY_POOL_CONFIG_FINGERPRINT_KEY`。共享配置源已同步时严格重载、预检
并按 revision 发布,源尚未同步时停止旧 Provider Runtime禁止旧 URL/Secret
在换主后继续调用。
disable 成功后会立即通知 Supervisor 取消目标 Runtime每秒一次的权威状态对账
用于修复进程内通知丢失。已取得的分布式 Permit 仍按幂等、保守规则完成结算。
PostgreSQL 瞬时读取失败不会终止 Controller 或取消当前 Provider Runtime
Supervisor 保留 last-known 状态并在下一周期重试。
主配置或 Secret 文件 I/O 故障归类为 503语法、未知字段、引用和语义校验失败
归类为 422。请求取消和截止时间保持原始上下文错误不误报为配置错误。

View File

@ -34,6 +34,18 @@ sequenceDiagram
`worker_id` 是逻辑节点,`instance_id` 区分进程重启,`session_id` 防止旧进程
继续上报。所有权 `epoch` 小于 Controller 当前值的数据必须拒绝。
`ReportRuntimeRequest.report_sequence` 在当前 `session_id` 内严格单调递增。
相同序号只允许内容完全相同的幂等重放;较小序号或相同序号的不同内容必须
拒绝。`observed_at` 只用于观测,不作为乱序判定依据,运行态 TTL 统一使用
Controller 侧 Redis 服务端时间。session 同时保存 Controller 已接受的
`snapshot_version/ownership_epoch`;运行态报告必须与该 ACK 上界完全一致,
Worker 自报的超前版本或 epoch 也必须拒绝。
运行态上报是完整稀疏替换:只携带 Active/Reserved 非零的 Proxy空列表表示
当前会话全部归零。Controller 只有在 Worker session、报告 TTL、Proxy ownership
和 ownership epoch 同时有效时才使用计数;报告缺失、过期或不一致时按零可用
容量 fail-closed不能把未知计数解释成空闲容量。
## 3. Snapshot 与 Delta
完整 Snapshot 包含:

View File

@ -16,10 +16,13 @@ Controller 入口已实现,源码运行方式为:
go run ./cmd/proxy-controller -config CONFIG_FILE
```
配置路径优先使用 `-config`,未提供时读取 `PROXY_POOL_CONFIG`。该入口已装配
配置路径优先使用 `-config`,未提供时读取 `PROXY_POOL_CONFIG`。启用 Admin 时还
必须设置 `PROXY_POOL_CONFIG_FINGERPRINT_KEY`,值为至少 32 字节的独立高熵密钥;
所有 Controller 副本必须一致,且该密钥不得放入 YAML 或 PostgreSQL。该入口已装配
PostgreSQL 管理面迁移、Redis 活动池、Distribution/Admin 独立监听与优雅停机;
Controller Metrics 独立监听、`/livez`、`/readyz` 和基础 Prometheus 运行时指标;
Provider 自动补池、业务指标和完整部署拓扑仍在后续实施范围。
Provider 自动补池、分布式配额、动态重载和 Admin 低基数统计已装配。完整 Gateway、
Checker、Worker 控制面与代表性负载验证仍在后续实施范围。
所有时间值使用 Go duration例如 `500ms`、`30s`、`5m`。示例中的
`${TOKEN}`、`${PASSWORD}`、`${POSTGRES_URL}` 等由加载器从同名环境变量
@ -312,11 +315,30 @@ proxyAuth:
- `pool.maxSize`:当前系统维护且尚未 EXTRACTED 的 Proxy 硬上限,包括
FETCHED、CHECKING、AVAILABLE、SUSPECT、DRAINING 和 pending expected。
- `fetch.maxTotal`:当前运行或计费周期内,从 Provider 成功获取的累计上限;
`0` 表示不设置累计上限。
- `fetch.maxTotal`:当前 Redis generation 内的累计获取停止阈值;`0` 表示不设置。
Redis 在调用前原子校验 `fetched total + pending expected + expected`,并在达到
阈值后停止发起新调用。
`fetch.maxTotal` 不得小于 `pool.maxSize`。提取一个 Proxy 会释放当前库存位置,
但不会恢复累计获取额度。
但不会恢复累计获取额度。Provider 调用结果不确定、响应无法解析或 Permit 过期时,
系统按 `estimatedIPsPerCall` 保守记账避免故障或换主造成额度低估。Redis 全量
状态丢失会创建新 generation因此需要由外部计费系统提供跨 generation 的长期额度。
`estimatedIPsPerCall` 是预留估值,不是通用的 Provider 响应硬限制。如果某次实际
合法返回量超过估值,系统只保留本地池容量允许的数量,但累计账本按实际合法数量
记账并停止后续调用;该次可能越过停止阈值。需要绝对硬上限时,必须同时在 Provider
请求参数中配置供应商支持的批量上限,并保证其不超过剩余额度。
当前实现限制单个 `pool.maxSize <= 1,000,000`、配置内 Upstream 总数不超过
`4,096`并要求单代理并发、Refill 双水位、理论总槽位及其他传入 Redis Lua 的
累计/并发计数不超过 `2^53-1`。这些边界在配置加载和 Admin reload 提交前校验,
不会等到 Provider Runtime 启动后才失败。
`proxyAuth.type: response` 的用户名/密码是 Parser 到 Redis Activity Adapter 之间的
临时凭据。每次 Parser handoff 使用独立、幂等释放的 lease避免并发 Fetch 互相
删除或轮换版本覆盖;成功复制到 TTL 活动记录、候选被截断或解析失败后都会释放。
内存 lease 上限按所有配置 Upstream 的 `pool.maxSize * fetch.maxInFlight` 汇总,
配置 reload 只提高上限,不预分配对应内存。
### 7.3 Fetch 限制
@ -376,6 +398,12 @@ PostgreSQL 只保存配置版本、Upstream/Routing 管理状态、Admin 审计
PostgreSQL 故障本身不应使 Redis 中可完成的 Extract 返回 `503`。Metrics 标签
禁止 Proxy IP、Client ID、Session、完整 URL 和 Request ID。
启用 Admin 的多 Controller 部署必须让所有副本读取同一版本化配置源、Secret
版本和 `PROXY_POOL_CONFIG_FINGERPRINT_KEY`。Supervisor 每秒比较本地完整配置的
HMAC-SHA-256 与 PostgreSQL 权威值:管理状态暂时不可读时沿用 last-known
Runtime指纹已更新但本地源尚未同步时停止旧 Provider待源匹配并通过预检后
按全局 revision 恢复,避免旧凭据换主或迟到旧配置回写。
Metrics 启用时 `listen` 必须是合法 `host:port`。该入口固定提供 `/livez`
`/readyz``/metrics`,不复用 Distribution/Admin 的认证边界;外部访问必须由
网络策略限制。当前 `/metrics` 已包含 Go/进程基础指标Provider、提取和容量等

View File

@ -143,6 +143,15 @@ type Proxy struct {
运行态 `active``reserved` 存在 Worker 本地、按 Proxy ID 分片,不写入
不可变 Snapshot。
Worker 以有界周期批量上报运行态,而不是在每个 Gateway 请求上写 Redis。
报告通过公用 `workerruntime` seam 表达为完整稀疏替换Gateway 从打包原子计数
读取同一时刻的 Active/Reserved已从当前 Snapshot 移除但仍有连接的 Proxy
继续以 draining 状态上报。Controller 使用 session、单调 report sequence、
ownership epoch、Controller 已 ACK 的 snapshot/epoch 上界和 Redis 服务端 TTL
共同校验;缺失、过期或超前报告按零可用容量 fail-closed。运行态报告扫描有界
当前 Snapshot并以分片索引补充已移除但仍非零的 runtime历史 Capacity
注册表有硬上限,避免短 TTL Proxy 持续轮换导致心跳扫描与内存无界增长。
## 6. Proxy 状态机
```mermaid
@ -244,6 +253,31 @@ flowchart TD
fail-closed不回退为本地 Leader。补池使用 minimum/target 双水位迟滞,库存
复核期间若仍有 pending Fetch则等待下一轮再同步 Managed避免重复计数。
Controller Bootstrap 通过 `Provider Supervisor` 按权威配置和 PostgreSQL 管理状态
维护每个 Upstream 的独立 `UpstreamRuntime`,再与 HTTP Runtime 通过公用 lifecycle
Group 联动启动、取消和等待。Admin disable 会通知 Supervisor 取消对应 Runtime
reload 在提交前预检新 Runtime并在发布后逐个取消、替换或新增。低频对账用于修复
丢失通知。每个 Leader 任期重新创建本地补池预算与合并信号Redis 是
requestInterval、maxInFlight 和 maxTotal 的唯一分布式裁决者。本地预算只负责
pool.maxSize。
maxTotal 使用 Redis generation 内的累计值与 pending expected 原子预留。调用结果
不确定、解析失败或结算丢失时按 expected 保守记账,换主后旧 Permit 仍可幂等结算。
如果 Provider 单次实际返回量超过 `estimatedIPsPerCall`,账本按实际合法数量记账并
停止后续调用,但该次可能越过停止阈值;严格硬封顶需要 Provider API 支持可控批量。
响应中携带的代理凭据先进入有界内存凭据表Redis Activity Adapter 复制凭据材料
进入 TTL 活动记录后立即按版本 fence 释放临时引用;解析中途失败也会回收已写引用。
因此短 TTL、持续轮换的新代理地址不会耗尽启动时的凭据容量。Provider 结果只记录
每个 Upstream 的低基数 Empty/Error 计数,不保留响应体、代理地址或错误对象。
补池读取 `pool.InventoryReader` 返回的权威 `Managed``AvailableSlots`,不使用
AVAILABLE Proxy 数量乘固定并发的近似值。Available Slots 同时计入 Proxy 状态、
TTL safety margin、MaxConcurrency、Worker Active/Reserved 和当前 ownership
未知 Worker 运行态贡献零槽位。Redis 只扫描目标 Upstream 的未分配/已分配
AVAILABLE 索引Managed 读取已有权威计数;该读取属于 Controller 冷路径,
Gateway 热路径仍只访问本地 Snapshot 和本地原子计数。
### 8.1 Empty、Duplicate 与 Error
- **Empty**HTTP/认证成功、模板执行成功,解析后合法 Proxy 数为 0。
@ -403,6 +437,12 @@ flowchart LR
- Routing 立即对新请求生效;旧请求持有旧 Snapshot 完成。
- 修改 API 地址或凭据版本会重建 Provider Adapter但不会把错误计成 Empty。
- 新配置任何校验失败时,保留旧版本并报告完整错误。
- 多 Controller 以 PostgreSQL 中的 HMAC-SHA-256 完整配置指纹检测修订HMAC
密钥由外部 Secret 注入且所有副本一致。副本从共享配置源读取相同 revision
预检后通过单调 revision 栅栏原子发布。权威指纹已变化但本地源仍旧时,旧
Provider fail-closed 停止,避免旧 URL/Secret 在 Leader 换主后继续使用。
- PostgreSQL 瞬时读取失败时沿用 last-known Provider Runtime 并重试,不连带终止
Distribution、Admin 或 Metrics。
## 15. 安全
@ -463,7 +503,7 @@ CPU、内存、网络、Go 版本、配置和上游响应模型下测得。
| Provider 超时/500 | 计 Error、退避不计 Empty不影响已有 Proxy |
| Provider 合法空响应 | Empty++;达到阈值触发相关 Routing 原子切换 |
| Redis 不可用 | Gateway 暂用未过期快照;停止 Fetch 入池、Extract 和所有权变更 |
| PostgreSQL 不可用 | Gateway 与 Redis Extract 不受影响;停止管理状态变更和 Admin 审计 |
| PostgreSQL 不可用 | Gateway 与 Redis Extract 不受影响;停止管理写入Provider 沿用 last-known 状态并重试 |
| Controller 断线 | Worker 在 maxStaleAge 内继续;超限拒绝新流量并排空 |
| Worker 崩溃 | 所有权租约过期后重新分配;期间不双重所有 |
| Checker 积压 | 降低普通复检频率,优先新 Proxy 与 SUSPECT不无限排队 |

View File

@ -195,11 +195,14 @@ Admin/Distribution 必需依赖。共享 `platform/httpserver` 与
已完成。pgx Adapter 已在真实 PostgreSQL 18 上运行同一公用契约,并验证
Repeatable Read 快照、`SKIP LOCKED`、原子 ACK、审计/Outbox 故障回滚和数据边界。
Admin `ApplicationService` 已将 mutation、权威管理快照、低基数运行态
聚合与配置重载接到同一公用 seam严格文件加载、脱敏管理摘要及原子配置发布
已通过失败路径和并发测试。`cmd/proxy-controller` 与公用 `controller/bootstrap`
聚合与配置重载接到同一公用 seam严格文件加载、外部密钥 HMAC 管理指纹及
revision 单调配置发布已通过失败路径和确定性并发测试。`cmd/proxy-controller` 与公用 `controller/bootstrap`
已完成配置单次加载、PostgreSQL 连接/迁移、Redis 活动池、状态聚合、
Distribution/Admin 服务构造、错误合并和资源关闭Provider 调度及完整 HTTP
进程端到端测试仍待实现。Controller Metrics 独立入口现已提供 `/livez`
Distribution/Admin 服务构造、错误合并和资源关闭。生产 Provider Supervisor 已按
权威管理状态动态装配 Upstream并与 HTTP Runtime 通过公用 lifecycle Group 联动
停机Admin disable 会取消 Runtimereload 在提交前预检并在发布后替换运行实例。
组合 fixture 已验证隔离 Redis namespace 下的选主、Provider HTTP 调用、模板解析
和活动池写入。Controller Metrics 独立入口现已提供 `/livez`
`/readyz` 与基础 Prometheus 运行时指标,三监听器隔离已通过测试;业务指标仍待
实现。双存储 bootstrap 已通过 PostgreSQL 18 + Redis 8.2 组合 fixture覆盖
迁移、启动配置提交、Readiness、Admin Status 和 Metrics 探针。
@ -219,8 +222,23 @@ generation + epoch fence、全局 requestInterval、全局 maxInFlight Permit、
回收及 Redis 状态丢失后的新 generation 自动重建Redis 异常期间不发放请求。
补池配置新增必填 `refill` 双水位和 `fetch.estimatedIPsPerCall`Pool Reconciler
已实现迟滞与 pending 槽位折算FetchBudget 仅在无 pending 时同步 Redis 权威
Managed。Provider Fleet、Worker Active/Reserved 汇总和 bootstrap 接线仍待完成,
因此本轮不勾选 Task 10 的组合验收项。
Managed。Gateway 已增加打包原子 Active/Reserved 读取与完整稀疏运行态快照;
公用 `workerruntime` session/report/read seam 同时提供并发安全内存参考实现和
生产 Redis Adapter。Redis 以服务端时间、Worker session、已 ACK snapshot/epoch、
单调 report sequence 和报告 TTL 原子隔离旧实例,并由 `pool.InventoryReader` 汇总
权威 Managed/Available Slots真实 Redis 8.2 已覆盖空报告、幂等重放、倒序、
冲突、超前 epoch、过期、单 Upstream 扫描隔离和预算耗尽的 fail-closed 行为。
Redis Provider Permit 现已把 requestInterval、maxInFlight 与 maxTotal 放在同一
原子边界,按 expected 预留、实际合法数量结算,并支持失败保守计费、换主后结算和
过期回收。响应型代理凭据使用独立、有界 lease在 Redis Upsert、候选截断或解析
失败后按版本释放Provider Empty/Error 低基数计数已接入 Admin Status配置删除
时回收历史统计容量。Redis inventory 扫描上限固定覆盖配置允许的最大池,支持小池
启动后动态扩容。Supervisor 以 PostgreSQL 权威 HMAC 指纹和 revision 栅栏协调
多副本 reload管理库瞬断沿用 last-known 状态,本地共享源落后时停止旧 Provider
源匹配并预检后自动替换,迟到旧 revision 不覆盖新配置。
WorkerControlPlane gRPC 接收端、session 签发/心跳、Snapshot ACK
账本、Client 分布式限流和健康执行链仍待完成,因此本轮不勾选 Task 10 的组合
验收项。
## Task 11: Checker and Health Reducer

View File

@ -6,7 +6,7 @@
- [ ] 每个 Proxy 同一时刻最多归属一个 Workerownership epoch 单调。
- [ ] Reserved -> Active 使用单个原子转换,无超卖与负计数。
- [ ] Sequential 并发 Empty 只切换一次,旧 Upstream Proxy 自然耗尽。
- [ ] `pool.maxSize``fetch.maxTotal` 分别按当前库存和累计获取计数。
- [x] `pool.maxSize``fetch.maxTotal` 分别按当前库存和累计获取计数。
- [ ] Extract 只有 `AVAILABLE -> EXTRACTED`OpenAPI 不存在 release/renew。
- [ ] Extract 状态更新和短期幂等结果位于同一个 Redis 原子操作。
- [ ] PostgreSQL 中不存在 Proxy 明细或逐次提取记录。
@ -20,6 +20,7 @@
- [ ] trusted proxy 只包含受控 LoadBalancer/Ingress 网段。
- [ ] 解析前后均拦截私网、回环、链路本地、元数据地址与 DNS Rebinding。
- [ ] Secret 由外部密钥系统注入镜像、ConfigMap、日志没有明文。
- [ ] 所有 Controller 使用同一枚至少 32 字节的独立配置 HMAC 密钥,并完成轮换演练。
- [ ] Pod 以非 root、只读根文件系统、无 Linux capabilities 运行。
- [ ] NetworkPolicy 默认拒绝,外部数据库/Redis/Provider 网段已收紧。
- [ ] Provider 模板有响应大小、执行时间、函数与外部访问限制。

View File

@ -19,9 +19,9 @@
## 2. 本地拓扑模板
`cmd/proxy-controller` 已完成配置单次加载、PostgreSQL 迁移、Redis 活动池、
Distribution/Admin/Metrics 独立监听和有界停机装配。Provider 自动补池、业务
指标以及 Gateway/Checker/Loadgen 三个进程仍属于 `implementation-plan.md`
后续任务。
Distribution/Admin/Metrics 独立监听和有界停机装配。Provider 自动补池、分布式
配额、动态重载和 Admin 低基数统计已装配;完整 Gateway/Checker/Loadgen 与 Worker
控制面仍属于 `implementation-plan.md` 后续任务。
因此 Compose/Kubernetes 资产当前仍用于评审网络、资源、探针和依赖关系,不能
视为完整可运行拓扑。
@ -40,6 +40,7 @@ Distribution/Admin/Metrics 独立监听和有界停机装配。Provider 自动
$env:PROXY_POOL_GATEWAY_PASSWORD = "LOCAL_GATEWAY_PASSWORD"
$env:PROXY_POOL_EXTRACT_TOKEN = "LOCAL_EXTRACT_TOKEN"
$env:PROXY_POOL_ADMIN_TOKEN = "LOCAL_ADMIN_TOKEN"
$env:PROXY_POOL_CONFIG_FINGERPRINT_KEY = "LOCAL_HIGH_ENTROPY_KEY_AT_LEAST_32_BYTES"
$env:PROVIDER_A_TOKEN = "PROVIDER_A_TOKEN"
$env:PROVIDER_B_TOKEN = "PROVIDER_B_TOKEN"
```
@ -90,6 +91,8 @@ Fetch 应表现为 Error 与退避,不应增加 Empty 计数,也不影响已
1. 使用托管 PostgreSQL 和 Redis分别配置 TLS、备份、监控和多可用区。
2. 复制 `secret.example.yaml` 到环境私密配置系统,由 External Secrets、SOPS
或密钥管理平台生成 `proxy-pool-secrets`,不要提交真实 Secret。
`PROXY_POOL_CONFIG_FINGERPRINT_KEY` 必须使用至少 32 字节的高熵随机值,所有
Controller 副本保持一致,且与配置中的业务 Secret 分离管理。
3. 在环境 Overlay 替换镜像、Provider 地址、允许网段、外部存储地址、资源量
和 LoadBalancer 注解。
4. 根据集群 CNI 能力收紧 NetworkPolicy 的外部网段。
@ -221,11 +224,15 @@ Outbox 发布器必须以稳定 consumer ID 有界领取;发布成功后原子
1. Gateway 继续使用最后有效 Snapshot。
2. Redis 健康且运行配置有效时Distribution 继续执行原子 ExtractProvider
继续刷新 TTL 活动池。
按 last-known 管理状态继续刷新 TTL 活动池;状态读取失败不得触发全进程退出
3. 拒绝配置版本、Upstream/Routing 管理状态和其他需要 Admin 审计/outbox 的写入;
不得把 Proxy 明细临时落入 PostgreSQL。
4. 恢复后核对迁移、管理事务回滚、Admin 审计与 outbox backlog不存在 Proxy
明细或逐次提取记录恢复步骤。
5. 若权威配置指纹已变化,确认每个 Controller 的共享配置源和 Secret 版本已同步;
同时确认 `PROXY_POOL_CONFIG_FINGERPRINT_KEY` 一致。指纹不匹配的副本会停止旧
Provider匹配并预检成功后按 PostgreSQL revision 自动恢复;迟到旧 revision
不会覆盖较新本地配置。
### 7.3 Redis 不可用

View File

@ -10,7 +10,7 @@
| ARCH-001 | 数据面 Worker 与控制面 Controller 分离 | 1-70 | 包、协议和部署拓扑已分离Controller 命令已实现Gateway/Checker/Loadgen 构建产物待实现 |
| ARCH-002 | 热路径只做认证、本地路由和网络转发 | 1-70, 380-430 | Snapshot/Dispatch 及依赖边界已验证;完整 Gateway 进程与代表性性能剖析待完成 |
| ARCH-003 | Gateway、Distribution、Admin、Metrics 独立入口 | 8904-8958 | Controller 命令已装配 Distribution/Admin/Metrics 三个独立监听及联动停机Gateway 生产入口待装配 |
| ARCH-004 | Controller 集中 Provider 获取与切换 | 1403-1580 | 单进程 Reconciler、合并通知和切换领域契约已完成分布式 Leader 与运行装配待完成 |
| ARCH-004 | Controller 集中 Provider 获取与切换 | 1403-1580 | Redis Leader、动态 Provider Supervisor 与 Bootstrap 生产装配已完成Admin disable/reload 驱动取消替换,多副本按权威 HMAC 指纹和 revision 栅栏收敛并拒绝旧配置换主Routing 切换到 Drain 的编排待完成 |
| ARCH-005 | 100k QPS 峰值使用多 Worker 集群 | 当前会话 | 未验证设计目标;待代表性集群负载报告 |
## Routing 与 Upstream
@ -32,11 +32,11 @@
| FETCH-001 | 每个 Provider 有独立 requestInterval、maxInFlight、timeout 和 retry | 968-2394 | `provider/reconciler_test.go` |
| FETCH-002 | 大量缺池信号合并为 singleflight/容量 1 通知 | 2067-2136, 8808-8849 | `coalesce.Signal` 与 100 并发通知测试 |
| FETCH-003 | 错误使用指数退避和抖动429 尊重 Retry-After | 1601-1831, 8808-8856 | `provider/reconciler_test.go``providerapi/http_adapter_test.go` |
| FETCH-004 | Provider 获取由单逻辑 Leader 执行 | 1403-1580 | Redis Coordinator 已通过真实 Redis 双实例互斥、epoch 接管、全局间隔/在途 Permit 与 generation 重建测试;Provider Fleet/bootstrap 接线待完成 |
| FETCH-004 | Provider 获取由单逻辑 Leader 执行 | 1403-1580 | Redis Coordinator 已通过真实 Redis 双实例互斥、epoch 接管、全局间隔/在途 Permit 与 generation 重建测试;生产 Supervisor/bootstrap 已通过 Admin disable 和 Provider HTTP 到隔离 Redis 库存的组合 fixture |
| FETCH-005 | Empty 与 Error 分开;只有合法候选为零时 Empty++ | 8442-8529 | `fetch_result_test.go` 分类矩阵 |
| FETCH-006 | 重复候选不当作 Empty记录独立指标 | 8442-8480 | DuplicateOnly 分类与 Provider 测试 |
| FETCH-007 | 模板限制响应大小、执行时间、函数集和外部访问 | 8808-8856 | `providerapi/template_parser_test.go` 输入、输出、候选、超时、递归与函数白名单测试 |
| FETCH-008 | pool.maxSize 与 fetch.maxTotal 语义分离 | 9190-9280 | `FetchBudget` 并发预占/释放测试 |
| FETCH-008 | pool.maxSize 与 fetch.maxTotal 语义分离 | 9190-9280 | 本地 `FetchBudget` 仅约束当前库存Redis Permit 原子维护累计与 pending 额度,并通过换主、取消、幂等和过期保守结算测试 |
## Proxy 生命周期与容量
@ -46,7 +46,7 @@
| PROXY-002 | 唯一键包含 scheme、host、port、username、credentialVersion | 6655-6727, 8605-8678 | 去重单测 |
| PROXY-003 | TTL 来源优先级明确并统一 UTC | 681-747, 8655-8678 | TTL 表驱动测试 |
| CAP-001 | Gateway 分配使用 Reserved -> Active 原子转换 | 1203-1467, 8530-8597 | 固定 Max 下打包 CAS 与 1,000 并发不超卖已完成;动态降容和完整生命周期证据待完成 |
| CAP-002 | 补池依据 Available Slots不只看 Proxy 数量 | 1203-1402, 8530-8597 | `AvailableSlots`、显式 minimum/target 水位、pending 槽位和迟滞 Reconciler 已测试;Worker Active/Reserved、ownership、目标健康及 Gateway reserve 运行时聚合待完成 |
| CAP-002 | 补池依据 Available Slots不只看 Proxy 数量 | 1203-1402, 8530-8597 | `AvailableSlots`、显式 minimum/target 水位、pending 槽位和迟滞 Reconciler 已测试;Gateway 打包 Active/Reserved 报告、Worker session/ACK/sequence/TTL/ownership fence、单 Upstream 索引及 Redis 权威 Managed/Slots 汇总已通过内存与真实 Redis 测试WorkerControlPlane 接线、目标健康和 Gateway reserve 策略仍待完成 |
| CAP-003 | pool.maxSize 包括 FETCHED/CHECKING/AVAILABLE/SUSPECT/DRAINING 与 pending expected | 3001-3533, 6642-6680 | `FetchBudget` 100 并发额度预占测试 |
| CAP-004 | TTL safety margin 内禁止新分配 | 173-220, 6728-6741 | 时钟测试 |
| CAP-005 | 多 Worker 不在热路径访问 Redis 计数 | 1403-1467 | Gateway 包依赖审计、Snapshot/Dispatch 测试 |

View File

@ -22,11 +22,20 @@
4. 连续 4 次 Empty 后 Valid 不切换;连续 5 次只从 A 切到 B。
5. Error 和 DuplicateOnly 不累计 Empty。
6. 100 个缺池信号只形成一个合并 Provider reconcile。
7. 并发 Fetch 不突破 `pool.maxSize``fetch.maxTotal`
7. 并发 Fetch 不突破 `pool.maxSize`,且不会在 Redis `fetch.maxTotal` 停止阈值耗尽后发起新调用
8. TTL safety margin 内不再分配。
9. Snapshot 版本断档、目标错误或校验和错误不替换当前视图。
10. 非幂等 HTTP 和已建立 CONNECT 不自动重放。
11. 所有 Upstream 不可用时严格执行显式策略。
11. Admin disable 取消目标 Provider Runtimereload 构造失败不提交,成功时替换运行实例。
12. 短 TTL 响应凭据使用独立 lease并发 Fetch、截断、解析失败和 Redis Upsert
后均准确释放且不互相撤销。
13. PostgreSQL 状态瞬断保留 last-known Provider多副本本地指纹落后时停止旧
Runtime共享源同步后自动预检并恢复不同 HMAC 密钥不能误判为相同配置。
14. 并发配置提交与 Supervisor 同步按 revision 单调发布,迟到旧 revision 不覆盖
已发布新配置。
15. 配置删除的 Provider 统计项被回收,禁用但仍配置的统计项保留,容量可复用。
16. 小池启动后重载到大池时Redis inventory 扫描上限仍覆盖配置允许的最大池。
17. 所有 Upstream 不可用时严格执行显式策略。
## 3. 基础质量门禁
@ -74,16 +83,17 @@ soak 测试单独标记,不混入快速单测。
## 5. 当前本地微基准
2026-07-28Windows/amd64、Intel Core Ultra 7 155H
2026-07-30Windows/amd64、Intel Core Ultra 7 155H
```text
BenchmarkAcquire100kIndexed-22 3553592 640.0 ns/op 256 B/op 2 allocs/op
BenchmarkStoreApply100k-22 1 472.7 ms/op 654 MB/op 2700642 allocs/op
BenchmarkAcquire100kIndexed-22 1000000-1867125 893.9-1047 ns/op 256 B/op 2 allocs/op
BenchmarkStoreApply100k-22 1 518.7 ms/op 540 MB/op 3000887 allocs/op
```
`Acquire` 已使用 scheme/upstream/tag 索引,结果只代表本地选择和容量预留。
`Store.Apply` 属于冷路径且当前内存开销较高;运行态为防止旧快照在途连接超配,
暂不自动回收曾出现过的 Proxy ID。后续需要基于 RCU/引用计数定义安全回收点。
暂不自动回收曾出现过的 Proxy ID但注册表有 1,000,000 项硬上限,达到上限时
拒绝新 Snapshot 并保留旧视图。后续需要基于 RCU/引用计数定义安全回收点。
这些数据不包含网络、认证、Provider、存储或多 Worker 协调,不能作为
100k QPS 端到端验收结论。

View File

@ -120,7 +120,7 @@ Redis Readiness 和 Admin StatusHTTP Runner 使用测试 Adapter避免占
Admin 应用层测试覆盖 typed-nil 依赖、Actor/SourceIP 映射、Routing CAS 错误、
权威管理快照与低基数运行态聚合、未知字段拒绝、主配置/Secret 文件 I/O 分类、
持久化失败不发布、幂等重放发布、脱敏管理摘要和原子配置 Store 并发读写。静态
持久化失败不发布、HMAC 管理指纹、revision 单调发布和原子配置 Store 并发读写。静态
导入边界测试禁止 Admin 引用 Redis Activity/Extract 与 Proxy 明细包。
需要 PostgreSQL/Redis 的测试使用独立实例和短生命周期容器,不复用开发数据。

View File

@ -17,11 +17,12 @@ import (
)
const (
defaultTemplateTimeout = 100 * time.Millisecond
defaultTemplateMaxBytes = int64(1 << 20)
defaultMaxCandidates = 10_000
maxRegexPatterns = 64
maxRegexPatternBytes = 1024
defaultTemplateTimeout = 100 * time.Millisecond
defaultTemplateMaxBytes = int64(1 << 20)
defaultMaxCandidates = 10_000
credentialReleaseTimeout = time.Second
maxRegexPatterns = 64
maxRegexPatternBytes = 1024
)
type TemplateParser struct {
@ -137,7 +138,14 @@ func NewTemplateParser(
return parser, nil
}
func (p *TemplateParser) Parse(ctx context.Context, body []byte) ([]proxyDomain.Proxy, error) {
func (p *TemplateParser) Parse(ctx context.Context, body []byte) (proxies []proxyDomain.Proxy, resultErr error) {
storedCredentials := make([]credentials.Reference, 0)
defer func() {
if resultErr == nil {
return
}
p.releaseCredentials(storedCredentials)
}()
if err := ctx.Err(); err != nil {
return nil, err
}
@ -161,7 +169,7 @@ func (p *TemplateParser) Parse(ctx context.Context, body []byte) ([]proxyDomain.
if len(tokens) > p.maxCandidates {
return nil, &limitError{kind: ErrTooManyCandidates, size: int64(len(tokens)), limit: int64(p.maxCandidates)}
}
proxies := make([]proxyDomain.Proxy, 0, len(tokens))
proxies = make([]proxyDomain.Proxy, 0, len(tokens))
credentialIndexes := make(map[string]int)
for _, token := range tokens {
candidate, credential, ok := p.parseCandidate(token)
@ -183,6 +191,7 @@ func (p *TemplateParser) Parse(ctx context.Context, body []byte) ([]proxyDomain.
}
candidate.SecretRef = reference.SecretRef
candidate.CredentialVersion = reference.CredentialVersion
storedCredentials = append(storedCredentials, reference)
credentialKey = candidateCredentialKey(candidate)
if index, exists := credentialIndexes[credentialKey]; exists {
proxies[index] = candidate
@ -201,9 +210,65 @@ func (p *TemplateParser) Parse(ctx context.Context, body []byte) ([]proxyDomain.
if len(tokens) > 0 && len(proxies) == 0 {
return nil, ErrInvalidProxyOutput
}
p.releaseUnusedCredentials(storedCredentials, proxies)
return proxies, nil
}
func (p *TemplateParser) ReleaseCandidates(candidates []proxyDomain.Proxy) {
references := make([]credentials.Reference, 0, len(candidates))
for _, candidate := range candidates {
if candidate.SecretRef == "" || candidate.CredentialVersion == "" {
continue
}
references = append(references, credentials.Reference{
SecretRef: candidate.SecretRef, CredentialVersion: candidate.CredentialVersion,
})
}
p.releaseCredentials(references)
}
func (p *TemplateParser) releaseUnusedCredentials(
stored []credentials.Reference,
candidates []proxyDomain.Proxy,
) {
retained := make(map[credentials.Reference]int, len(candidates))
for _, candidate := range candidates {
if candidate.SecretRef == "" || candidate.CredentialVersion == "" {
continue
}
retained[credentials.Reference{
SecretRef: candidate.SecretRef, CredentialVersion: candidate.CredentialVersion,
}]++
}
unused := make([]credentials.Reference, 0, len(stored))
for _, reference := range stored {
if retained[reference] > 0 {
retained[reference]--
continue
}
unused = append(unused, reference)
}
p.releaseCredentials(unused)
}
func (p *TemplateParser) releaseCredentials(references []credentials.Reference) {
if p == nil {
return
}
releaser, ok := p.credentialStore.(credentials.Releaser)
if !ok {
return
}
releaseCtx, cancel := context.WithTimeout(context.Background(), credentialReleaseTimeout)
defer cancel()
for _, reference := range references {
if releaseCtx.Err() != nil {
return
}
_ = releaser.Release(releaseCtx, reference)
}
}
func (p *TemplateParser) regexFind(pattern, value string) (string, error) {
compiled, err := p.compileRegex(pattern)
if err != nil {

View File

@ -318,7 +318,7 @@ func TestTemplateParserDoesNotOverrideStaticProxyAuthFromResponse(t *testing.T)
}
func TestTemplateParserRetainsDistinctEndpointsSharingStaticCredentials(t *testing.T) {
store, err := credentials.NewMemoryStore(1)
store, err := credentials.NewMemoryStore(2)
if err != nil {
t.Fatalf("NewMemoryStore(): %v", err)
}
@ -343,8 +343,11 @@ func TestTemplateParserRetainsDistinctEndpointsSharingStaticCredentials(t *testi
if len(proxies) != 2 {
t.Fatalf("proxy count = %d, want both static-auth endpoints", len(proxies))
}
if proxies[0].SecretRef == "" || proxies[0].SecretRef != proxies[1].SecretRef {
t.Fatalf("static credential references = %q and %q, want same opaque reference", proxies[0].SecretRef, proxies[1].SecretRef)
if proxies[0].SecretRef == "" || proxies[1].SecretRef == "" || proxies[0].SecretRef == proxies[1].SecretRef {
t.Fatalf("static credential references = %q and %q, want independent leases", proxies[0].SecretRef, proxies[1].SecretRef)
}
if proxies[0].CredentialVersion != proxies[1].CredentialVersion {
t.Fatalf("static credential versions = %q and %q, want same value version", proxies[0].CredentialVersion, proxies[1].CredentialVersion)
}
}
@ -391,6 +394,33 @@ func TestTemplateParserStoresResponseCredentialsByOpaqueReference(t *testing.T)
}
}
func TestTemplateParserReleasesPartialCredentialsWhenParseFails(t *testing.T) {
store, err := credentials.NewMemoryStore(1)
if err != nil {
t.Fatalf("NewMemoryStore(): %v", err)
}
parser, err := newTemplateParser("provider-a", config.Upstream{
Provider: config.Provider{Protocols: []string{"http"}},
API: config.ProviderAPI{Template: strings.Join([]string{
"http://alice:first-password@192.0.2.10:8080",
"http://bob:second-password@192.0.2.11:8080",
}, "\n")},
ProxyAuth: config.ProxyAuth{Type: "response"},
Pool: config.Pool{MaxSize: 2},
Fetch: config.Fetch{MaxResponseBytes: 1024, TemplateTimeout: config.Duration(time.Second)},
}, store)
if err != nil {
t.Fatalf("newTemplateParser(): %v", err)
}
if _, err := parser.Parse(context.Background(), nil); !errors.Is(err, credentials.ErrCapacityExceeded) {
t.Fatalf("Parse() error = %v, want ErrCapacityExceeded", err)
}
if _, err := store.Put(context.Background(), "replacement", credentials.Value{Password: "replacement"}); err != nil {
t.Fatalf("Put(after failed parse): %v", err)
}
}
func TestTemplateParserKeepsDistinctAccountsForSameEndpointResolvable(t *testing.T) {
store, err := credentials.NewMemoryStore(2)
if err != nil {
@ -432,7 +462,7 @@ func TestTemplateParserKeepsDistinctAccountsForSameEndpointResolvable(t *testing
}
func TestTemplateParserKeepsLatestCredentialVersionWithinOneResponse(t *testing.T) {
store, err := credentials.NewMemoryStore(1)
store, err := credentials.NewMemoryStore(2)
if err != nil {
t.Fatalf("NewMemoryStore(): %v", err)
}
@ -467,6 +497,36 @@ func TestTemplateParserKeepsLatestCredentialVersionWithinOneResponse(t *testing.
if value.Password != "new-secret" {
t.Fatalf("resolved latest password mismatch")
}
parser.ReleaseCandidates(proxies)
if _, err := store.Put(context.Background(), "replacement", credentials.Value{Password: "replacement"}); err != nil {
t.Fatalf("Put(after releasing latest candidates): %v", err)
}
}
func TestTemplateParserReleaseCandidatesReturnsCredentialCapacity(t *testing.T) {
store, err := credentials.NewMemoryStore(1)
if err != nil {
t.Fatalf("NewMemoryStore(): %v", err)
}
parser, err := newTemplateParser("provider-a", config.Upstream{
Provider: config.Provider{Protocols: []string{"http"}},
API: config.ProviderAPI{Template: "http://alice:secret@192.0.2.10:8080"},
ProxyAuth: config.ProxyAuth{Type: "response"},
Pool: config.Pool{MaxSize: 1},
Fetch: config.Fetch{MaxResponseBytes: 1024, TemplateTimeout: config.Duration(time.Second)},
}, store)
if err != nil {
t.Fatalf("NewTemplateParser(): %v", err)
}
proxies, err := parser.Parse(context.Background(), nil)
if err != nil {
t.Fatalf("Parse(): %v", err)
}
parser.ReleaseCandidates(proxies)
if _, err := store.Put(context.Background(), "replacement", credentials.Value{Password: "replacement"}); err != nil {
t.Fatalf("Put(after ReleaseCandidates): %v", err)
}
}
func TestTemplateParserRedactsCredentialStoreErrors(t *testing.T) {

View File

@ -19,33 +19,45 @@ var (
)
type Options struct {
Namespace string
Credentials credentials.Store
OperationTTL time.Duration
MaxCandidateScan int
CleanupLimit int
Namespace string
Credentials credentials.Store
OperationTTL time.Duration
MaxCandidateScan int
MaxRuntimeCounters int
MaxInventoryScan int
CleanupLimit int
}
type Adapter struct {
client redis.Scripter
credentials credentials.Store
keys keyspace
options Options
client redis.Scripter
credentials credentials.Store
credentialReleaser credentials.Releaser
keys keyspace
options Options
}
func New(client redis.Scripter, options Options) (*Adapter, error) {
options.Namespace = strings.TrimSpace(options.Namespace)
if options.MaxRuntimeCounters == 0 {
options.MaxRuntimeCounters = options.MaxCandidateScan
}
if options.MaxInventoryScan == 0 {
options.MaxInventoryScan = options.MaxCandidateScan
}
if nilInterface(client) || nilInterface(options.Credentials) ||
!namespacePattern.MatchString(options.Namespace) || options.OperationTTL <= 0 ||
options.MaxCandidateScan <= 0 || options.CleanupLimit <= 0 {
options.MaxCandidateScan <= 0 || options.MaxRuntimeCounters <= 0 ||
options.MaxInventoryScan <= 0 || options.CleanupLimit <= 0 {
return nil, ErrInvalidOptions
}
return &Adapter{
adapter := &Adapter{
client: client,
credentials: options.Credentials,
keys: newKeyspace(options.Namespace),
options: options,
}, nil
}
adapter.credentialReleaser, _ = options.Credentials.(credentials.Releaser)
return adapter, nil
}
func (a *Adapter) Format(state fmt.State, _ rune) {

View File

@ -58,6 +58,8 @@ func TestNewRejectsInvalidDependenciesAndOptions(t *testing.T) {
{name: "colon in namespace", client: client, options: withNamespace(valid, "tenant:other")},
{name: "zero operation ttl", client: client, options: withOperationTTL(valid, 0)},
{name: "zero candidate scan", client: client, options: withMaxCandidateScan(valid, 0)},
{name: "negative runtime counters", client: client, options: withMaxRuntimeCounters(valid, -1)},
{name: "negative inventory scan", client: client, options: withMaxInventoryScan(valid, -1)},
{name: "negative cleanup limit", client: client, options: withCleanupLimit(valid, -1)},
}
for _, tt := range tests {
@ -93,7 +95,9 @@ func TestNewBuildsClusterSafeKeyspaceAndHashesDynamicTokens(t *testing.T) {
adapter.keys.records, adapter.keys.unique, adapter.keys.idkeys,
adapter.keys.expiry, adapter.keys.available, adapter.keys.owners,
adapter.keys.ownerExpiry, adapter.keys.epoch, adapter.keys.inventory,
adapter.keys.stateInventory,
adapter.keys.stateInventory, adapter.keys.workerSessions,
adapter.keys.workerSessionExpiry, adapter.keys.workerRuntime,
adapter.keys.workerRuntimeExpiry,
}
for _, key := range staticKeys {
if strings.Count(key, "{activity}") != 1 || strings.Count(key, "{") != 1 || strings.Count(key, "}") != 1 {
@ -109,6 +113,7 @@ func TestNewBuildsClusterSafeKeyspaceAndHashesDynamicTokens(t *testing.T) {
adapter.keys.region(raw),
adapter.keys.carrier(raw),
adapter.keys.upstream(raw),
adapter.keys.owned(raw),
}
for _, key := range dynamicKeys {
if strings.Contains(key, raw) || strings.Count(key, "{activity}") != 1 || strings.Count(key, "{") != 1 {
@ -161,6 +166,7 @@ func TestProxyRecordCodecIsDeterministicStrictAndRedacted(t *testing.T) {
UsableUntilMS: 58_000, LastCheckedAtMS: 2_000, LastSuccessAtMS: 2_000,
LatencyNS: int64(25 * time.Millisecond), MaxConcurrency: 8,
State: string(proxyDomain.StateAvailable), Tags: map[string]string{"region": "cn", "carrier": "ct"},
OwnerIndexKey: "pp:{activity}:test:owned:index",
}
first, err := encodeProxyRecord(record)
if err != nil {
@ -260,6 +266,16 @@ func withMaxCandidateScan(options Options, limit int) Options {
return options
}
func withMaxRuntimeCounters(options Options, limit int) Options {
options.MaxRuntimeCounters = limit
return options
}
func withMaxInventoryScan(options Options, limit int) Options {
options.MaxInventoryScan = limit
return options
}
func withCleanupLimit(options Options, limit int) Options {
options.CleanupLimit = limit
return options

View File

@ -0,0 +1,45 @@
package redisactivity
import (
"context"
"time"
controllerPool "proxy-pool/internal/controller/pool"
"proxy-pool/internal/domain/activitypool"
)
var _ controllerPool.InventoryReader = (*Adapter)(nil)
func (a *Adapter) ReadInventory(
ctx context.Context,
upstreamID string,
safetyMargin time.Duration,
) (controllerPool.InventorySnapshot, error) {
if ctx == nil || a == nil || !runtimeClean(upstreamID) || safetyMargin < 0 {
return controllerPool.InventorySnapshot{}, activitypool.ErrInvalidInventory
}
if err := ctx.Err(); err != nil {
return controllerPool.InventorySnapshot{}, err
}
result, err := runScript(ctx, a.client, capacityScript, []string{
a.keys.records, a.keys.inventory, a.keys.upstream(upstreamID), a.keys.owned(upstreamID), a.keys.owners,
a.keys.workerSessions, a.keys.workerSessionExpiry,
a.keys.workerRuntime, a.keys.workerRuntimeExpiry,
}, upstreamID, durationMillis(safetyMargin), a.options.MaxInventoryScan, a.options.CleanupLimit)
if err != nil {
return controllerPool.InventorySnapshot{}, err
}
var reply capacityScriptReply
if err := decodeScriptResult(result, &reply); err != nil {
return controllerPool.InventorySnapshot{}, err
}
if reply.Status == scriptInvalid {
return controllerPool.InventorySnapshot{}, activitypool.ErrInvalidInventory
}
if reply.Status != scriptOK || reply.Managed < 0 || reply.AvailableSlots < 0 {
return controllerPool.InventorySnapshot{}, invalidScriptReply("capacity inventory is unavailable")
}
return controllerPool.InventorySnapshot{
Managed: reply.Managed, AvailableSlots: reply.AvailableSlots,
}, nil
}

View File

@ -0,0 +1,105 @@
//go:build integration
package redisactivity
import (
"context"
"testing"
"time"
controllerPool "proxy-pool/internal/controller/pool"
"proxy-pool/internal/domain/workerruntime"
)
func TestRedisCapacityInventoryCombinesProxyAndWorkerRuntime(t *testing.T) {
fixture := newRedisTestFixture(t)
now := redisTestNow()
unowned := testProxy("proxy-unowned", "192.0.2.10")
unowned.MaxConcurrency = 10
owned := testProxy("proxy-owned", "192.0.2.11")
owned.MaxConcurrency = 10
seedRedisAvailable(t, fixture.Adapter, "provider-a", now, now.Add(time.Second), 2*time.Minute, unowned)
seedRedisAvailable(t, fixture.Adapter, "provider-a", now, now.Add(time.Second), 2*time.Minute, owned)
assignment, err := fixture.Adapter.Assign(context.Background(), now.Add(2*time.Second),
"proxy-owned", "worker-a", time.Minute)
if err != nil {
t.Fatalf("Assign(): %v", err)
}
if err := fixture.Adapter.ReplaceSession(context.Background(), workerruntime.Session{
WorkerID: "worker-a", InstanceID: "instance-a", SessionID: "session-a",
AckedSnapshotVersion: 1, AckedOwnershipEpoch: assignment.Epoch,
}, time.Minute); err != nil {
t.Fatalf("ReplaceSession(): %v", err)
}
if err := fixture.Adapter.ReplaceRuntime(context.Background(), workerruntime.Report{
WorkerID: "worker-a", SessionID: "session-a", Sequence: 1,
SnapshotVersion: 1, OwnershipEpoch: assignment.Epoch, ObservedAt: now,
Counters: []workerruntime.Counter{{ProxyID: "proxy-owned", Active: 3, Reserved: 2}},
}, time.Minute); err != nil {
t.Fatalf("ReplaceRuntime(): %v", err)
}
inventory, err := fixture.Adapter.ReadInventory(context.Background(), "provider-a", 0)
if err != nil || inventory.Managed != 2 || inventory.AvailableSlots != 15 {
t.Fatalf("ReadInventory() = %+v, %v; want managed=2 slots=15", inventory, err)
}
if _, ok := any(fixture.Adapter).(controllerPool.InventoryReader); !ok {
t.Fatal("Adapter does not implement pool.InventoryReader")
}
inventory, err = fixture.Adapter.ReadInventory(context.Background(), "provider-a", 2*time.Hour)
if err != nil || inventory.Managed != 2 || inventory.AvailableSlots != 0 {
t.Fatalf("ReadInventory(safety margin) = %+v, %v", inventory, err)
}
fixture.Adapter.options.MaxInventoryScan = 1
if _, err := fixture.Adapter.ReadInventory(context.Background(), "provider-a", 0); err == nil {
t.Fatal("ReadInventory(over scan limit) error = nil")
}
}
func TestRedisCapacityInventoryFailsClosedForExpiredRuntime(t *testing.T) {
fixture := newRedisTestFixture(t)
now := redisTestNow()
owned := testProxy("proxy-owned", "192.0.2.11")
owned.MaxConcurrency = 10
seedRedisAvailable(t, fixture.Adapter, "provider-a", now, now.Add(time.Second), 2*time.Minute, owned)
assignment, err := fixture.Adapter.Assign(context.Background(), now.Add(2*time.Second),
"proxy-owned", "worker-a", time.Minute)
if err != nil {
t.Fatalf("Assign(): %v", err)
}
if err := fixture.Adapter.ReplaceSession(context.Background(), workerruntime.Session{
WorkerID: "worker-a", InstanceID: "instance-a", SessionID: "session-a",
AckedSnapshotVersion: 1, AckedOwnershipEpoch: assignment.Epoch,
}, 100*time.Millisecond); err != nil {
t.Fatalf("ReplaceSession(): %v", err)
}
if err := fixture.Adapter.ReplaceRuntime(context.Background(), workerruntime.Report{
WorkerID: "worker-a", SessionID: "session-a", Sequence: 1,
SnapshotVersion: 1, OwnershipEpoch: assignment.Epoch, ObservedAt: now,
}, 100*time.Millisecond); err != nil {
t.Fatalf("ReplaceRuntime(): %v", err)
}
time.Sleep(150 * time.Millisecond)
inventory, err := fixture.Adapter.ReadInventory(context.Background(), "provider-a", 0)
if err != nil || inventory.Managed != 1 || inventory.AvailableSlots != 0 {
t.Fatalf("ReadInventory(expired runtime) = %+v, %v", inventory, err)
}
}
func TestRedisCapacityInventoryScanIsIsolatedPerUpstream(t *testing.T) {
fixture := newRedisTestFixture(t)
fixture.Adapter.options.MaxInventoryScan = 1
now := redisTestNow()
target := testProxy("proxy-target", "192.0.2.10")
target.MaxConcurrency = 4
seedRedisAvailable(t, fixture.Adapter, "provider-a", now, now.Add(time.Second), 2*time.Minute, target)
seedRedisAvailable(t, fixture.Adapter, "provider-b", now, now.Add(time.Second), 2*time.Minute,
testProxy("proxy-other-1", "192.0.2.11"))
seedRedisAvailable(t, fixture.Adapter, "provider-b", now, now.Add(2*time.Second), 2*time.Minute,
testProxy("proxy-other-2", "192.0.2.12"))
inventory, err := fixture.Adapter.ReadInventory(context.Background(), "provider-a", 0)
if err != nil || inventory.Managed != 1 || inventory.AvailableSlots != 4 {
t.Fatalf("ReadInventory(provider-a) = %+v, %v; want isolated managed=1 slots=4", inventory, err)
}
}

View File

@ -37,6 +37,7 @@ type proxyRecord struct {
State string `json:"state"`
Tags map[string]string `json:"tags,omitempty"`
OwnerWorkerID string `json:"ownerWorkerId,omitempty"`
OwnerIndexKey string `json:"ownerIndexKey"`
IndexKeys []string `json:"indexKeys,omitempty"`
}
@ -203,7 +204,8 @@ func validateProxyRecord(record proxyRecord) error {
record.CreatedAtMS <= 0 || record.ExpiresAtMS <= 0 || record.UsableUntilMS <= 0 ||
record.UsableUntilMS > record.ExpiresAtMS || record.LastCheckedAtMS < 0 ||
record.LastSuccessAtMS < 0 || record.LatencyNS < 0 || record.MaxConcurrency < 0 ||
!validScheme(record.Scheme) || !validProxyState(record.State) {
!validScheme(record.Scheme) || !validProxyState(record.State) ||
record.OwnerIndexKey == "" || !strings.Contains(record.OwnerIndexKey, "{activity}") {
return ErrInvalidRecord
}
for _, key := range record.IndexKeys {

View File

@ -9,33 +9,41 @@ import (
const redisKeyPrefix = "pp:{activity}:"
type keyspace struct {
prefix string
records string
unique string
idkeys string
expiry string
available string
owners string
ownerExpiry string
epoch string
inventory string
stateInventory string
prefix string
records string
unique string
idkeys string
expiry string
available string
owners string
ownerExpiry string
epoch string
inventory string
stateInventory string
workerSessions string
workerSessionExpiry string
workerRuntime string
workerRuntimeExpiry string
}
func newKeyspace(namespace string) keyspace {
prefix := redisKeyPrefix + namespace
return keyspace{
prefix: prefix,
records: prefix + ":records",
unique: prefix + ":unique",
idkeys: prefix + ":idkeys",
expiry: prefix + ":expiry",
available: prefix + ":available",
owners: prefix + ":owners",
ownerExpiry: prefix + ":owner-expiry",
epoch: prefix + ":epoch",
inventory: prefix + ":inventory",
stateInventory: prefix + ":state-inventory",
prefix: prefix,
records: prefix + ":records",
unique: prefix + ":unique",
idkeys: prefix + ":idkeys",
expiry: prefix + ":expiry",
available: prefix + ":available",
owners: prefix + ":owners",
ownerExpiry: prefix + ":owner-expiry",
epoch: prefix + ":epoch",
inventory: prefix + ":inventory",
stateInventory: prefix + ":state-inventory",
workerSessions: prefix + ":worker-sessions",
workerSessionExpiry: prefix + ":worker-session-expiry",
workerRuntime: prefix + ":worker-runtime",
workerRuntimeExpiry: prefix + ":worker-runtime-expiry",
}
}
@ -67,6 +75,10 @@ func (keys keyspace) upstream(value string) string {
return keys.facet("upstream", value)
}
func (keys keyspace) owned(value string) string {
return keys.facet("owned", value)
}
func (keys keyspace) facet(name, value string) string {
return keys.prefix + ":" + name + ":" + digestToken(value)
}

View File

@ -0,0 +1,248 @@
package redisactivity
import (
"context"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"sort"
"strconv"
"strings"
"time"
"proxy-pool/internal/domain/workerruntime"
)
const runtimeWireVersion = 1
const (
runtimeReplaceSession = "replace_session"
runtimeReplaceReport = "replace_report"
runtimeRead = "read"
)
type runtimeSessionWire struct {
Version int `json:"version"`
WorkerID string `json:"workerId"`
InstanceID string `json:"instanceId"`
SessionID string `json:"sessionId"`
AckedSnapshotVersion string `json:"ackedSnapshotVersion"`
AckedOwnershipEpoch string `json:"ackedOwnershipEpoch"`
}
type runtimeCounterWire struct {
ProxyID string `json:"proxyId"`
Active int64 `json:"active"`
Reserved int64 `json:"reserved"`
Draining bool `json:"draining"`
}
type runtimeReportWire struct {
Version int `json:"version"`
WorkerID string `json:"workerId"`
SessionID string `json:"sessionId"`
Sequence string `json:"sequence"`
SnapshotVersion string `json:"snapshotVersion"`
OwnershipEpoch string `json:"ownershipEpoch"`
ObservedAtMS int64 `json:"observedAtMs"`
Counters []runtimeCounterWire `json:"counters"`
}
type runtimeOwnedProxyWire struct {
ProxyID string `json:"proxyId"`
WorkerID string `json:"workerId"`
OwnershipEpoch string `json:"ownershipEpoch"`
}
type runtimeSnapshotWire struct {
ProxyID string `json:"proxyId"`
Active int64 `json:"active"`
Reserved int64 `json:"reserved"`
Draining bool `json:"draining"`
Fresh bool `json:"fresh"`
}
var (
_ workerruntime.SessionWriter = (*Adapter)(nil)
_ workerruntime.ReportWriter = (*Adapter)(nil)
_ workerruntime.RuntimeReader = (*Adapter)(nil)
)
func (a *Adapter) ReplaceSession(ctx context.Context, session workerruntime.Session, ttl time.Duration) error {
if err := validateRuntimeCall(ctx, a); err != nil {
return err
}
if !runtimeClean(session.WorkerID) || !runtimeClean(session.InstanceID) || !runtimeClean(session.SessionID) ||
session.AckedSnapshotVersion == 0 || session.AckedOwnershipEpoch == 0 || ttl <= 0 {
return workerruntime.ErrInvalidSession
}
payload, err := json.Marshal(runtimeSessionWire{
Version: runtimeWireVersion, WorkerID: session.WorkerID,
InstanceID: session.InstanceID, SessionID: session.SessionID,
AckedSnapshotVersion: strconv.FormatUint(session.AckedSnapshotVersion, 10),
AckedOwnershipEpoch: strconv.FormatUint(session.AckedOwnershipEpoch, 10),
})
if err != nil {
return workerruntime.ErrInvalidSession
}
reply, err := a.runRuntime(ctx, runtimeReplaceSession, durationMillis(ttl), payload, "")
if err != nil {
return err
}
switch reply.Status {
case scriptOK:
return nil
case scriptInvalid:
return workerruntime.ErrInvalidSession
case scriptStale:
return workerruntime.ErrStaleSession
default:
return invalidScriptReply("unexpected worker session reply")
}
}
func (a *Adapter) ReplaceRuntime(ctx context.Context, report workerruntime.Report, ttl time.Duration) error {
if err := validateRuntimeCall(ctx, a); err != nil {
return err
}
payload, digest, err := a.encodeRuntimeReport(report, ttl)
if err != nil {
return err
}
reply, err := a.runRuntime(ctx, runtimeReplaceReport, durationMillis(ttl), payload, digest)
if err != nil {
return err
}
switch reply.Status {
case scriptOK:
return nil
case scriptInvalid:
return workerruntime.ErrInvalidReport
case scriptStale:
return workerruntime.ErrStaleReport
case scriptConflict:
return workerruntime.ErrConflictingReport
case scriptUnavailable:
return workerruntime.ErrStaleSession
default:
return invalidScriptReply("unexpected worker runtime reply")
}
}
func (a *Adapter) ReadRuntime(ctx context.Context, proxies []workerruntime.OwnedProxy) ([]workerruntime.Snapshot, error) {
if err := validateRuntimeCall(ctx, a); err != nil {
return nil, err
}
if len(proxies) > a.options.MaxRuntimeCounters {
return nil, workerruntime.ErrInvalidQuery
}
wires := make([]runtimeOwnedProxyWire, len(proxies))
seen := make(map[string]struct{}, len(proxies))
for index, proxy := range proxies {
if !runtimeClean(proxy.ProxyID) || !runtimeClean(proxy.WorkerID) || proxy.OwnershipEpoch == 0 {
return nil, workerruntime.ErrInvalidQuery
}
key := proxy.WorkerID + "\x00" + proxy.ProxyID
if _, exists := seen[key]; exists {
return nil, workerruntime.ErrInvalidQuery
}
seen[key] = struct{}{}
wires[index] = runtimeOwnedProxyWire{
ProxyID: proxy.ProxyID, WorkerID: proxy.WorkerID,
OwnershipEpoch: strconv.FormatUint(proxy.OwnershipEpoch, 10),
}
}
payload, err := json.Marshal(wires)
if err != nil {
return nil, workerruntime.ErrInvalidQuery
}
reply, err := a.runRuntime(ctx, runtimeRead, 0, payload, "")
if err != nil {
return nil, err
}
if reply.Status == scriptInvalid {
return nil, workerruntime.ErrInvalidQuery
}
if reply.Status != scriptOK || len(reply.Snapshots) != len(proxies) {
return nil, invalidScriptReply("unexpected worker runtime read reply")
}
result := make([]workerruntime.Snapshot, len(reply.Snapshots))
for index, snapshot := range reply.Snapshots {
if snapshot.ProxyID != proxies[index].ProxyID || snapshot.Active < 0 || snapshot.Reserved < 0 {
return nil, invalidScriptReply("invalid worker runtime snapshot")
}
result[index] = workerruntime.Snapshot{
ProxyID: snapshot.ProxyID, Active: snapshot.Active, Reserved: snapshot.Reserved,
Draining: snapshot.Draining, Fresh: snapshot.Fresh,
}
}
return result, nil
}
func (a *Adapter) encodeRuntimeReport(report workerruntime.Report, ttl time.Duration) ([]byte, string, error) {
if ttl <= 0 || !runtimeClean(report.WorkerID) || !runtimeClean(report.SessionID) ||
report.Sequence == 0 || report.SnapshotVersion == 0 || report.OwnershipEpoch == 0 || report.ObservedAt.IsZero() ||
len(report.Counters) > a.options.MaxRuntimeCounters {
return nil, "", workerruntime.ErrInvalidReport
}
counters := append([]workerruntime.Counter(nil), report.Counters...)
sort.Slice(counters, func(left, right int) bool { return counters[left].ProxyID < counters[right].ProxyID })
wires := make([]runtimeCounterWire, len(counters))
for index, counter := range counters {
if !runtimeClean(counter.ProxyID) || counter.Active < 0 || counter.Reserved < 0 ||
(index > 0 && counters[index-1].ProxyID == counter.ProxyID) {
return nil, "", workerruntime.ErrInvalidReport
}
wires[index] = runtimeCounterWire{
ProxyID: counter.ProxyID, Active: counter.Active,
Reserved: counter.Reserved, Draining: counter.Draining,
}
}
payload, err := json.Marshal(runtimeReportWire{
Version: runtimeWireVersion, WorkerID: report.WorkerID, SessionID: report.SessionID,
Sequence: strconv.FormatUint(report.Sequence, 10),
SnapshotVersion: strconv.FormatUint(report.SnapshotVersion, 10),
OwnershipEpoch: strconv.FormatUint(report.OwnershipEpoch, 10),
ObservedAtMS: report.ObservedAt.UTC().UnixMilli(), Counters: wires,
})
if err != nil {
return nil, "", workerruntime.ErrInvalidReport
}
digest := sha256.Sum256(payload)
return payload, hex.EncodeToString(digest[:]), nil
}
func (a *Adapter) runRuntime(
ctx context.Context,
operation string,
ttlMS int64,
payload []byte,
digest string,
) (runtimeScriptReply, error) {
result, err := runScript(ctx, a.client, runtimeScript, []string{
a.keys.workerSessions, a.keys.workerSessionExpiry,
a.keys.workerRuntime, a.keys.workerRuntimeExpiry, a.keys.owners,
}, operation, ttlMS, a.options.CleanupLimit, string(payload), digest)
if err != nil {
return runtimeScriptReply{}, err
}
var reply runtimeScriptReply
if err := decodeScriptResult(result, &reply); err != nil {
return runtimeScriptReply{}, err
}
return reply, nil
}
func validateRuntimeCall(ctx context.Context, adapter *Adapter) error {
if ctx == nil || adapter == nil {
return workerruntime.ErrInvalidStore
}
if err := ctx.Err(); err != nil {
return err
}
return nil
}
func runtimeClean(value string) bool {
return value != "" && strings.TrimSpace(value) == value
}

View File

@ -0,0 +1,147 @@
//go:build integration
package redisactivity
import (
"context"
"errors"
"testing"
"time"
"proxy-pool/internal/domain/workerruntime"
)
func TestRedisWorkerRuntimeReplacesSparseCountersAndFencesReports(t *testing.T) {
fixture := newRedisTestFixture(t)
now := redisTestNow()
seedRedisAvailable(t, fixture.Adapter, "provider-a", now, now.Add(time.Second), 2*time.Minute,
testProxy("proxy-a", "192.0.2.10"))
assignment, err := fixture.Adapter.Assign(context.Background(), now.Add(2*time.Second),
"proxy-a", "worker-a", time.Minute)
if err != nil {
t.Fatalf("Assign(): %v", err)
}
session := workerruntime.Session{
WorkerID: "worker-a", InstanceID: "instance-a", SessionID: "session-a",
AckedSnapshotVersion: 3, AckedOwnershipEpoch: assignment.Epoch,
}
if err := fixture.Adapter.ReplaceSession(context.Background(), session, time.Minute); err != nil {
t.Fatalf("ReplaceSession(): %v", err)
}
report := workerruntime.Report{
WorkerID: "worker-a", SessionID: "session-a", Sequence: 2,
SnapshotVersion: 3, OwnershipEpoch: assignment.Epoch, ObservedAt: now,
Counters: []workerruntime.Counter{{ProxyID: "proxy-a", Active: 2, Reserved: 1}},
}
if err := fixture.Adapter.ReplaceRuntime(context.Background(), report, time.Minute); err != nil {
t.Fatalf("ReplaceRuntime(first): %v", err)
}
if err := fixture.Adapter.ReplaceRuntime(context.Background(), report, time.Minute); err != nil {
t.Fatalf("ReplaceRuntime(replay): %v", err)
}
conflict := report
conflict.Counters = []workerruntime.Counter{{ProxyID: "proxy-a", Active: 3}}
if err := fixture.Adapter.ReplaceRuntime(context.Background(), conflict, time.Minute); !errors.Is(err, workerruntime.ErrConflictingReport) {
t.Fatalf("ReplaceRuntime(conflict) error = %v", err)
}
stale := report
stale.Sequence = 1
if err := fixture.Adapter.ReplaceRuntime(context.Background(), stale, time.Minute); !errors.Is(err, workerruntime.ErrStaleReport) {
t.Fatalf("ReplaceRuntime(stale) error = %v", err)
}
query := []workerruntime.OwnedProxy{{
ProxyID: "proxy-a", WorkerID: "worker-a", OwnershipEpoch: assignment.Epoch,
}}
got, err := fixture.Adapter.ReadRuntime(context.Background(), query)
if err != nil || len(got) != 1 || got[0] != (workerruntime.Snapshot{
ProxyID: "proxy-a", Active: 2, Reserved: 1, Fresh: true,
}) {
t.Fatalf("ReadRuntime(first) = %+v, %v", got, err)
}
report.Sequence = 3
report.Counters = nil
if err := fixture.Adapter.ReplaceRuntime(context.Background(), report, time.Minute); err != nil {
t.Fatalf("ReplaceRuntime(empty): %v", err)
}
got, err = fixture.Adapter.ReadRuntime(context.Background(), query)
if err != nil || len(got) != 1 || got[0] != (workerruntime.Snapshot{ProxyID: "proxy-a", Fresh: true}) {
t.Fatalf("ReadRuntime(empty) = %+v, %v", got, err)
}
if err := fixture.Adapter.ReplaceSession(context.Background(), workerruntime.Session{
WorkerID: "worker-a", InstanceID: "instance-b", SessionID: "session-b",
AckedSnapshotVersion: 4, AckedOwnershipEpoch: assignment.Epoch + 1,
}, time.Minute); err != nil {
t.Fatalf("ReplaceSession(new): %v", err)
}
if err := fixture.Adapter.ReplaceRuntime(context.Background(), report, time.Minute); !errors.Is(err, workerruntime.ErrStaleSession) {
t.Fatalf("ReplaceRuntime(old session) error = %v", err)
}
}
func TestRedisWorkerRuntimeExpiresFailClosed(t *testing.T) {
fixture := newRedisTestFixture(t)
now := redisTestNow()
seedRedisAvailable(t, fixture.Adapter, "provider-a", now, now.Add(time.Second), 2*time.Minute,
testProxy("proxy-a", "192.0.2.10"))
assignment, err := fixture.Adapter.Assign(context.Background(), now.Add(2*time.Second),
"proxy-a", "worker-a", time.Minute)
if err != nil {
t.Fatalf("Assign(): %v", err)
}
if err := fixture.Adapter.ReplaceSession(context.Background(), workerruntime.Session{
WorkerID: "worker-a", InstanceID: "instance-a", SessionID: "session-a",
AckedSnapshotVersion: 1, AckedOwnershipEpoch: assignment.Epoch,
}, 100*time.Millisecond); err != nil {
t.Fatalf("ReplaceSession(): %v", err)
}
if err := fixture.Adapter.ReplaceRuntime(context.Background(), workerruntime.Report{
WorkerID: "worker-a", SessionID: "session-a", Sequence: 1,
SnapshotVersion: 1, OwnershipEpoch: assignment.Epoch, ObservedAt: now,
}, 100*time.Millisecond); err != nil {
t.Fatalf("ReplaceRuntime(): %v", err)
}
time.Sleep(150 * time.Millisecond)
got, err := fixture.Adapter.ReadRuntime(context.Background(), []workerruntime.OwnedProxy{{
ProxyID: "proxy-a", WorkerID: "worker-a", OwnershipEpoch: assignment.Epoch,
}})
if err != nil || len(got) != 1 || got[0].Fresh {
t.Fatalf("ReadRuntime(expired) = %+v, %v", got, err)
}
}
func TestRedisWorkerRuntimeRejectsEmptyReportBeyondAcknowledgedSnapshot(t *testing.T) {
fixture := newRedisTestFixture(t)
now := redisTestNow()
if err := fixture.Adapter.ReplaceSession(context.Background(), workerruntime.Session{
WorkerID: "worker-a", InstanceID: "instance-a", SessionID: "session-a",
AckedSnapshotVersion: 3, AckedOwnershipEpoch: 9,
}, time.Minute); err != nil {
t.Fatalf("ReplaceSession(): %v", err)
}
for name, report := range map[string]workerruntime.Report{
"version": {
WorkerID: "worker-a", SessionID: "session-a", Sequence: 1,
SnapshotVersion: 4, OwnershipEpoch: 9, ObservedAt: now,
},
"epoch": {
WorkerID: "worker-a", SessionID: "session-a", Sequence: 1,
SnapshotVersion: 3, OwnershipEpoch: 10, ObservedAt: now,
},
} {
t.Run(name, func(t *testing.T) {
if err := fixture.Adapter.ReplaceRuntime(context.Background(), report, time.Minute); !errors.Is(err, workerruntime.ErrStaleReport) {
t.Fatalf("ReplaceRuntime() error = %v, want ErrStaleReport", err)
}
})
}
}
func TestRedisWorkerRuntimeAcceptsEmptyRead(t *testing.T) {
fixture := newRedisTestFixture(t)
got, err := fixture.Adapter.ReadRuntime(context.Background(), nil)
if err != nil || got == nil || len(got) != 0 {
t.Fatalf("ReadRuntime(empty) = %#v, %v", got, err)
}
}

View File

@ -74,6 +74,17 @@ type statusScriptInventory struct {
Extracted int64 `json:"extracted"`
}
type runtimeScriptReply struct {
Status scriptStatus `json:"status"`
Snapshots []runtimeSnapshotWire `json:"snapshots"`
}
type capacityScriptReply struct {
Status scriptStatus `json:"status"`
Managed int `json:"managed"`
AvailableSlots int64 `json:"availableSlots,string"`
}
//go:embed scripts/upsert.lua
var upsertSource string
@ -92,6 +103,12 @@ var sweepSource string
//go:embed scripts/status.lua
var statusSource string
//go:embed scripts/runtime.lua
var runtimeSource string
//go:embed scripts/capacity.lua
var capacitySource string
var (
upsertScript = redis.NewScript(upsertSource)
healthScript = redis.NewScript(healthSource)
@ -99,6 +116,8 @@ var (
ownershipScript = redis.NewScript(ownershipSource)
sweepScript = redis.NewScript(sweepSource)
statusScript = redis.NewScript(statusSource)
runtimeScript = redis.NewScript(runtimeSource)
capacityScript = redis.NewScript(capacitySource)
)
func runScript(ctx context.Context, client redis.Scripter, script *redis.Script, keys []string, args ...any) (any, error) {

View File

@ -0,0 +1,177 @@
local records_key = KEYS[1]
local inventory_key = KEYS[2]
local available_upstream_key = KEYS[3]
local owned_upstream_key = KEYS[4]
local owners_key = KEYS[5]
local sessions_key = KEYS[6]
local session_expiry_key = KEYS[7]
local runtime_key = KEYS[8]
local runtime_expiry_key = KEYS[9]
local upstream_id = ARGV[1]
local safety_margin_ms = tonumber(ARGV[2])
local scan_limit = tonumber(ARGV[3])
local cleanup_limit = tonumber(ARGV[4])
local function reply(status, managed, available_slots)
return cjson.encode({
status = status,
managed = managed or 0,
availableSlots = tostring(available_slots or 0)
})
end
local function now_ms()
local value = redis.call('TIME')
return tonumber(value[1]) * 1000 + math.floor(tonumber(value[2]) / 1000)
end
local function decode_table(value)
if not value then
return nil
end
local ok, decoded = pcall(cjson.decode, value)
if not ok or type(decoded) ~= 'table' then
return nil
end
return decoded
end
local function valid_uint(value)
return type(value) == 'string' and string.match(value, '^[0-9]+$') and
value ~= '0' and (string.len(value) == 1 or string.sub(value, 1, 1) ~= '0')
end
local function compare_uint(left, right)
if string.len(left) ~= string.len(right) then
return string.len(left) < string.len(right) and -1 or 1
end
if left == right then
return 0
end
return left < right and -1 or 1
end
local function cleanup(now)
local expired_sessions = redis.call('ZRANGEBYSCORE', session_expiry_key, '-inf', now, 'LIMIT', 0, cleanup_limit)
for _, worker_id in ipairs(expired_sessions) do
redis.call('HDEL', sessions_key, worker_id)
redis.call('HDEL', runtime_key, worker_id)
redis.call('ZREM', session_expiry_key, worker_id)
redis.call('ZREM', runtime_expiry_key, worker_id)
end
local expired_reports = redis.call('ZRANGEBYSCORE', runtime_expiry_key, '-inf', now, 'LIMIT', 0, cleanup_limit)
for _, worker_id in ipairs(expired_reports) do
redis.call('HDEL', runtime_key, worker_id)
redis.call('ZREM', runtime_expiry_key, worker_id)
end
end
if type(upstream_id) ~= 'string' or upstream_id == '' or not safety_margin_ms or safety_margin_ms < 0 or
not scan_limit or scan_limit <= 0 or not cleanup_limit or cleanup_limit <= 0 then
return reply('invalid', 0, 0)
end
local now = now_ms()
cleanup(now)
local threshold = now + safety_margin_ms
local available_ids = redis.call('ZRANGEBYSCORE', available_upstream_key, '(' .. threshold, '+inf',
'LIMIT', 0, scan_limit + 1)
if #available_ids > scan_limit then
return reply('unavailable', 0, 0)
end
local remaining = scan_limit - #available_ids
local owned_ids = redis.call('ZRANGEBYSCORE', owned_upstream_key, '(' .. threshold, '+inf',
'LIMIT', 0, remaining + 1)
if #owned_ids > remaining then
return reply('unavailable', 0, 0)
end
local managed = tonumber(redis.call('HGET', inventory_key, upstream_id) or '0')
if not managed or managed < 0 or managed ~= math.floor(managed) then
return reply('unavailable', 0, 0)
end
local available_slots = 0
local worker_cache = {}
local seen = {}
for _, proxy_id in ipairs(available_ids) do
seen[proxy_id] = true
local record = decode_table(redis.call('HGET', records_key, proxy_id))
if not record or type(record.sourceUpstream) ~= 'string' or type(record.state) ~= 'string' or
record.sourceUpstream ~= upstream_id or record.state ~= 'AVAILABLE' or
type(record.usableUntilMs) ~= 'number' or record.usableUntilMs <= threshold or
type(record.maxConcurrency) ~= 'number' or record.maxConcurrency < 0 or
record.maxConcurrency ~= math.floor(record.maxConcurrency) or
(record.ownerWorkerId and record.ownerWorkerId ~= '') or redis.call('HGET', owners_key, proxy_id) then
return reply('unavailable', 0, 0)
end
available_slots = available_slots + record.maxConcurrency
end
for _, proxy_id in ipairs(owned_ids) do
if seen[proxy_id] then
return reply('unavailable', 0, 0)
end
local record = decode_table(redis.call('HGET', records_key, proxy_id))
if not record or record.sourceUpstream ~= upstream_id or record.state ~= 'AVAILABLE' or
type(record.usableUntilMs) ~= 'number' or record.usableUntilMs <= threshold or
type(record.maxConcurrency) ~= 'number' or record.maxConcurrency < 0 or
record.maxConcurrency ~= math.floor(record.maxConcurrency) or
type(record.ownerWorkerId) ~= 'string' or record.ownerWorkerId == '' then
return reply('unavailable', 0, 0)
end
local owner_worker_id = record.ownerWorkerId
local owner = decode_table(redis.call('HGET', owners_key, proxy_id))
if not owner or owner.workerId ~= owner_worker_id or type(owner.epoch) ~= 'number' or
type(owner.expiresAtMs) ~= 'number' or owner.expiresAtMs <= now or
type(owner.draining) ~= 'boolean' then
return reply('unavailable', 0, 0)
end
local cached = worker_cache[owner_worker_id]
if not cached then
local session = decode_table(redis.call('HGET', sessions_key, owner_worker_id))
local report = decode_table(redis.call('HGET', runtime_key, owner_worker_id))
cached = {fresh = false, counters = {}}
if session and report and session.workerId == owner_worker_id and
report.workerId == owner_worker_id and session.sessionId == report.sessionId and
type(session.expiresAtMs) == 'number' and session.expiresAtMs > now and
type(report.expiresAtMs) == 'number' and report.expiresAtMs > now and
valid_uint(session.ackedSnapshotVersion) and valid_uint(session.ackedOwnershipEpoch) and
report.snapshotVersion == session.ackedSnapshotVersion and
report.ownershipEpoch == session.ackedOwnershipEpoch then
cached.fresh = true
cached.ownershipEpoch = report.ownershipEpoch
if type(report.counters) == 'table' then
for _, counter in pairs(report.counters) do
if type(counter) == 'table' and type(counter.proxyId) == 'string' then
cached.counters[counter.proxyId] = counter
end
end
end
end
worker_cache[owner_worker_id] = cached
end
local owner_epoch = tostring(owner.epoch)
if cached.fresh and valid_uint(owner_epoch) and
compare_uint(cached.ownershipEpoch, owner_epoch) >= 0 then
local counter = cached.counters[proxy_id]
local active = 0
local reserved = 0
local draining = false
if counter then
active = counter.active
reserved = counter.reserved
draining = counter.draining
end
if type(active) == 'number' and type(reserved) == 'number' and active >= 0 and reserved >= 0 and
active == math.floor(active) and reserved == math.floor(reserved) and
not draining and not owner.draining then
local slots = record.maxConcurrency - active - reserved
if slots > 0 then
available_slots = available_slots + slots
end
end
end
end
return reply('ok', managed, available_slots)

View File

@ -113,6 +113,12 @@ local function remove_available(proxy_id, record)
end
end
local function remove_owned(proxy_id, record)
if record and type(record.ownerIndexKey) == 'string' and record.ownerIndexKey ~= '' then
redis.call('ZREM', record.ownerIndexKey, proxy_id)
end
end
local function remove_proxy(proxy_id)
local raw = redis.call('HGET', records_key, proxy_id)
local record = nil
@ -120,6 +126,7 @@ local function remove_proxy(proxy_id)
local decoded
decoded, record = pcall(cjson.decode, raw)
remove_available(proxy_id, decoded and record or nil)
remove_owned(proxy_id, decoded and record or nil)
if decoded and type(record) == 'table' and is_managed(record.state) then
decrement_inventory(record.sourceUpstream)
end

View File

@ -70,12 +70,19 @@ local function remove_available(id, record)
end
end
local function remove_owned(id, record)
if record and type(record.ownerIndexKey) == 'string' and record.ownerIndexKey ~= '' then
redis.call('ZREM', record.ownerIndexKey, id)
end
end
local function remove_proxy(id)
local raw = redis.call('HGET', records_key, id)
local record = nil
if raw then
record = cjson.decode(raw)
remove_available(id, record)
remove_owned(id, record)
if is_managed(record.state) then
decrement_inventory(record.sourceUpstream)
end
@ -142,6 +149,9 @@ if not raw then
return finish({status = 'not_found'})
end
local record = cjson.decode(raw)
if type(record.ownerIndexKey) ~= 'string' or record.ownerIndexKey == '' then
return finish({status = 'invalid'})
end
if tonumber(record.expiresAtMs) <= checked_at_ms then
remove_proxy(proxy_id)
return finish({status = 'not_found'})
@ -184,11 +194,17 @@ end
local encoded = cjson.encode(record)
redis.call('HSET', records_key, proxy_id, encoded)
local owned = (record.ownerWorkerId and record.ownerWorkerId ~= '') or redis.call('HEXISTS', owners_key, proxy_id) == 1
if next_state == 'AVAILABLE' and not owned and tonumber(record.usableUntilMs) > checked_at_ms then
redis.call('ZADD', available_key, record.usableUntilMs, proxy_id)
for _, index_key in ipairs(record.indexKeys or {}) do
redis.call('ZADD', index_key, record.usableUntilMs, proxy_id)
touch(index_key, tonumber(record.expiresAtMs))
remove_owned(proxy_id, record)
if next_state == 'AVAILABLE' and tonumber(record.usableUntilMs) > checked_at_ms then
if owned then
redis.call('ZADD', record.ownerIndexKey, record.usableUntilMs, proxy_id)
touch(record.ownerIndexKey, tonumber(record.expiresAtMs))
else
redis.call('ZADD', available_key, record.usableUntilMs, proxy_id)
for _, index_key in ipairs(record.indexKeys or {}) do
redis.call('ZADD', index_key, record.usableUntilMs, proxy_id)
touch(index_key, tonumber(record.expiresAtMs))
end
end
end
touch(records_key, tonumber(record.expiresAtMs))

View File

@ -94,6 +94,12 @@ local function remove_available(id, record)
end
end
local function remove_owned(id, record)
if record and type(record.ownerIndexKey) == 'string' and record.ownerIndexKey ~= '' then
redis.call('ZREM', record.ownerIndexKey, id)
end
end
local function add_available(id, record, at_ms)
local usable_until_ms = record and tonumber(record.usableUntilMs)
if not usable_until_ms or record.state ~= 'AVAILABLE' or usable_until_ms <= at_ms then
@ -118,6 +124,7 @@ local function remove_proxy(id)
local decoded
decoded, record = pcall(cjson.decode, raw)
remove_available(id, decoded and record or nil)
remove_owned(id, decoded and record or nil)
if decoded and type(record) == 'table' and is_managed(record.state) then
decrement_inventory(record.sourceUpstream)
end
@ -169,6 +176,7 @@ local function clear_owner(id, assignment, at_ms, restore)
local raw_record = redis.call('HGET', records_key, id)
local record = decode_table(raw_record)
if record and (not assignment or record.ownerWorkerId == assignment.workerId) then
remove_owned(id, record)
record.ownerWorkerId = nil
redis.call('HSET', records_key, id, cjson.encode(record))
if restore then
@ -195,6 +203,7 @@ if operation == 'assign' then
local record = decode_table(redis.call('HGET', records_key, proxy_id))
if not record or record.state ~= 'AVAILABLE' or
type(record.ownerIndexKey) ~= 'string' or record.ownerIndexKey == '' or
(record.ownerWorkerId and record.ownerWorkerId ~= '') or
redis.call('HEXISTS', owners_key, proxy_id) == 1 or
not tonumber(record.usableUntilMs) or tonumber(record.usableUntilMs) <= now_ms then
@ -223,6 +232,8 @@ if operation == 'assign' then
record.ownerWorkerId = worker_id
redis.call('HSET', records_key, proxy_id, cjson.encode(record))
remove_available(proxy_id, record)
redis.call('ZADD', record.ownerIndexKey, record.usableUntilMs, proxy_id)
touch(record.ownerIndexKey, tonumber(record.expiresAtMs))
return finish({status = 'ok', record = encoded})
end
@ -267,6 +278,8 @@ if operation == 'begin_drain' then
current.assignmentVersion = tonumber(current.assignmentVersion) + 1
local encoded = cjson.encode(current)
redis.call('HSET', owners_key, proxy_id, encoded)
local record = decode_table(redis.call('HGET', records_key, proxy_id))
remove_owned(proxy_id, record)
return finish({status = 'ok', record = encoded})
end
return finish({status = 'ok', record = cjson.encode(current)})

View File

@ -0,0 +1,229 @@
local sessions_key = KEYS[1]
local session_expiry_key = KEYS[2]
local runtime_key = KEYS[3]
local runtime_expiry_key = KEYS[4]
local owners_key = KEYS[5]
local operation = ARGV[1]
local ttl_ms = tonumber(ARGV[2])
local cleanup_limit = tonumber(ARGV[3])
local payload = ARGV[4]
local digest = ARGV[5]
local function reply(status, snapshots)
if snapshots then
return cjson.encode({status = status, snapshots = snapshots})
end
return '{"status":' .. cjson.encode(status) .. ',"snapshots":[]}'
end
local function now_ms()
local value = redis.call('TIME')
return tonumber(value[1]) * 1000 + math.floor(tonumber(value[2]) / 1000)
end
local function decode_table(value)
if not value then
return nil
end
local ok, decoded = pcall(cjson.decode, value)
if not ok or type(decoded) ~= 'table' then
return nil
end
return decoded
end
local function valid_uint(value)
return type(value) == 'string' and string.match(value, '^[0-9]+$') and
value ~= '0' and (string.len(value) == 1 or string.sub(value, 1, 1) ~= '0')
end
local function compare_uint(left, right)
if string.len(left) ~= string.len(right) then
return string.len(left) < string.len(right) and -1 or 1
end
if left == right then
return 0
end
return left < right and -1 or 1
end
local function cleanup(now)
local expired_sessions = redis.call('ZRANGEBYSCORE', session_expiry_key, '-inf', now, 'LIMIT', 0, cleanup_limit)
for _, worker_id in ipairs(expired_sessions) do
redis.call('HDEL', sessions_key, worker_id)
redis.call('HDEL', runtime_key, worker_id)
redis.call('ZREM', session_expiry_key, worker_id)
redis.call('ZREM', runtime_expiry_key, worker_id)
end
local expired_reports = redis.call('ZRANGEBYSCORE', runtime_expiry_key, '-inf', now, 'LIMIT', 0, cleanup_limit)
for _, worker_id in ipairs(expired_reports) do
redis.call('HDEL', runtime_key, worker_id)
redis.call('ZREM', runtime_expiry_key, worker_id)
end
end
local function valid_session(value)
return value and value.version == 1 and type(value.workerId) == 'string' and value.workerId ~= '' and
type(value.instanceId) == 'string' and value.instanceId ~= '' and
type(value.sessionId) == 'string' and value.sessionId ~= '' and
valid_uint(value.ackedSnapshotVersion) and valid_uint(value.ackedOwnershipEpoch)
end
local function valid_owner(value, worker_id, ownership_epoch, now)
return value and type(value.workerId) == 'string' and value.workerId == worker_id and
type(value.epoch) == 'number' and valid_uint(tostring(value.epoch)) and
compare_uint(ownership_epoch, tostring(value.epoch)) >= 0 and
type(value.expiresAtMs) == 'number' and value.expiresAtMs > now
end
local now = now_ms()
cleanup(now)
if operation == 'replace_session' then
if not ttl_ms or ttl_ms <= 0 then
return reply('invalid')
end
local session = decode_table(payload)
if not valid_session(session) then
return reply('invalid')
end
local current = decode_table(redis.call('HGET', sessions_key, session.workerId))
if current and valid_session(current) and current.sessionId == session.sessionId and
current.instanceId == session.instanceId and type(current.expiresAtMs) == 'number' and
current.expiresAtMs > now then
local epoch_order = compare_uint(session.ackedOwnershipEpoch, current.ackedOwnershipEpoch)
local version_order = compare_uint(session.ackedSnapshotVersion, current.ackedSnapshotVersion)
if epoch_order < 0 or (epoch_order == 0 and version_order < 0) then
return reply('stale')
end
if epoch_order > 0 or version_order > 0 then
redis.call('HDEL', runtime_key, session.workerId)
redis.call('ZREM', runtime_expiry_key, session.workerId)
end
else
redis.call('HDEL', runtime_key, session.workerId)
redis.call('ZREM', runtime_expiry_key, session.workerId)
end
session.expiresAtMs = now + ttl_ms
redis.call('HSET', sessions_key, session.workerId, cjson.encode(session))
redis.call('ZADD', session_expiry_key, session.expiresAtMs, session.workerId)
return reply('ok')
end
if operation == 'replace_report' then
if not ttl_ms or ttl_ms <= 0 or type(digest) ~= 'string' or digest == '' then
return reply('invalid')
end
local report = decode_table(payload)
if not report or report.version ~= 1 or type(report.workerId) ~= 'string' or report.workerId == '' or
type(report.sessionId) ~= 'string' or report.sessionId == '' or not valid_uint(report.sequence) or
not valid_uint(report.snapshotVersion) or not valid_uint(report.ownershipEpoch) or
type(report.observedAtMs) ~= 'number' or type(report.counters) ~= 'table' then
return reply('invalid')
end
local session = decode_table(redis.call('HGET', sessions_key, report.workerId))
if not valid_session(session) or session.workerId ~= report.workerId or session.sessionId ~= report.sessionId or
type(session.expiresAtMs) ~= 'number' or session.expiresAtMs <= now then
return reply('unavailable')
end
if report.snapshotVersion ~= session.ackedSnapshotVersion or
report.ownershipEpoch ~= session.ackedOwnershipEpoch then
return reply('stale')
end
local current = decode_table(redis.call('HGET', runtime_key, report.workerId))
if current and current.sessionId == report.sessionId and valid_uint(current.sequence) then
local ordering = compare_uint(report.sequence, current.sequence)
if ordering < 0 then
return reply('stale')
end
if ordering == 0 then
if current.digest == digest then
return reply('ok')
end
return reply('conflict')
end
end
local seen = {}
for _, counter in pairs(report.counters) do
if type(counter) ~= 'table' or type(counter.proxyId) ~= 'string' or counter.proxyId == '' or
type(counter.active) ~= 'number' or counter.active < 0 or counter.active ~= math.floor(counter.active) or
type(counter.reserved) ~= 'number' or counter.reserved < 0 or counter.reserved ~= math.floor(counter.reserved) or
type(counter.draining) ~= 'boolean' or seen[counter.proxyId] then
return reply('invalid')
end
seen[counter.proxyId] = true
local owner = decode_table(redis.call('HGET', owners_key, counter.proxyId))
if not valid_owner(owner, report.workerId, report.ownershipEpoch, now) then
return reply('stale')
end
end
report.digest = digest
report.expiresAtMs = now + ttl_ms
redis.call('HSET', runtime_key, report.workerId, cjson.encode(report))
redis.call('ZADD', runtime_expiry_key, report.expiresAtMs, report.workerId)
session.expiresAtMs = now + ttl_ms
redis.call('HSET', sessions_key, report.workerId, cjson.encode(session))
redis.call('ZADD', session_expiry_key, session.expiresAtMs, report.workerId)
return reply('ok')
end
if operation == 'read' then
local queries = decode_table(payload)
if not queries then
return reply('invalid')
end
if next(queries) == nil then
return '{"status":"ok","snapshots":[]}'
end
local snapshots = cjson.decode('[]')
local cache = {}
for _, query in ipairs(queries) do
if type(query) ~= 'table' or type(query.proxyId) ~= 'string' or query.proxyId == '' or
type(query.workerId) ~= 'string' or query.workerId == '' or not valid_uint(query.ownershipEpoch) then
return reply('invalid')
end
local snapshot = {proxyId = query.proxyId, active = 0, reserved = 0, draining = false, fresh = false}
local owner = decode_table(redis.call('HGET', owners_key, query.proxyId))
if valid_owner(owner, query.workerId, query.ownershipEpoch, now) and
compare_uint(query.ownershipEpoch, tostring(owner.epoch)) == 0 then
local cached = cache[query.workerId]
if not cached then
local session = decode_table(redis.call('HGET', sessions_key, query.workerId))
local report = decode_table(redis.call('HGET', runtime_key, query.workerId))
cached = {fresh = false, counters = {}}
if valid_session(session) and session.workerId == query.workerId and report and
report.workerId == query.workerId and report.sessionId == session.sessionId and
type(session.expiresAtMs) == 'number' and session.expiresAtMs > now and
type(report.expiresAtMs) == 'number' and report.expiresAtMs > now and
valid_uint(report.ownershipEpoch) and
report.snapshotVersion == session.ackedSnapshotVersion and
report.ownershipEpoch == session.ackedOwnershipEpoch then
cached.fresh = true
cached.ownershipEpoch = report.ownershipEpoch
if type(report.counters) == 'table' then
for _, counter in pairs(report.counters) do
if type(counter) == 'table' and type(counter.proxyId) == 'string' then
cached.counters[counter.proxyId] = counter
end
end
end
end
cache[query.workerId] = cached
end
if cached.fresh and compare_uint(cached.ownershipEpoch, query.ownershipEpoch) >= 0 then
snapshot.fresh = true
local counter = cached.counters[query.proxyId]
if counter then
snapshot.active = counter.active
snapshot.reserved = counter.reserved
snapshot.draining = counter.draining
end
end
end
snapshots[#snapshots + 1] = snapshot
end
return reply('ok', snapshots)
end
return reply('invalid')

View File

@ -73,6 +73,12 @@ local function remove_available(proxy_id, record)
end
end
local function remove_owned(proxy_id, record)
if record and type(record.ownerIndexKey) == 'string' and record.ownerIndexKey ~= '' then
redis.call('ZREM', record.ownerIndexKey, proxy_id)
end
end
local function remove_proxy(proxy_id)
local raw = redis.call('HGET', records_key, proxy_id)
local record = nil
@ -80,6 +86,7 @@ local function remove_proxy(proxy_id)
local decoded
decoded, record = pcall(cjson.decode, raw)
remove_available(proxy_id, decoded and record or nil)
remove_owned(proxy_id, decoded and record or nil)
if decoded and type(record) == 'table' and is_managed(record.state) then
decrement_inventory(record.sourceUpstream)
end

View File

@ -70,12 +70,19 @@ local function remove_available(proxy_id, record)
end
end
local function remove_owned(proxy_id, record)
if record and type(record.ownerIndexKey) == 'string' and record.ownerIndexKey ~= '' then
redis.call('ZREM', record.ownerIndexKey, proxy_id)
end
end
local function remove_proxy(proxy_id)
local raw = redis.call('HGET', records_key, proxy_id)
local record = nil
if raw then
record = cjson.decode(raw)
remove_available(proxy_id, record)
remove_owned(proxy_id, record)
if is_managed(record.state) then
decrement_inventory(record.sourceUpstream)
end
@ -123,6 +130,16 @@ local function add_available(proxy_id, record)
end
end
local function sync_owned(proxy_id, record)
if record.state == 'AVAILABLE' and record.ownerWorkerId and record.ownerWorkerId ~= '' and
tonumber(record.usableUntilMs) > now_ms then
redis.call('ZADD', record.ownerIndexKey, record.usableUntilMs, proxy_id)
touch(record.ownerIndexKey, tonumber(record.expiresAtMs))
else
redis.call('ZREM', record.ownerIndexKey, proxy_id)
end
end
local function finish(reply)
local encoded = cjson.encode(reply)
redis.call('SET', operation_key, encoded, 'PX', operation_ttl_ms)
@ -132,6 +149,11 @@ end
cleanup_expired()
for _, candidate in ipairs(candidates) do
local decoded, incoming = pcall(cjson.decode, candidate.record)
if not decoded or type(incoming) ~= 'table' or
type(incoming.ownerIndexKey) ~= 'string' or incoming.ownerIndexKey == '' then
return finish({status = 'invalid', accepted = 0, inserted = 0, refreshed = 0, dropped = 0})
end
local mapped = redis.call('HGET', idkeys_key, candidate.proxyId)
if mapped and mapped ~= candidate.uniqueDigest then
return finish({status = 'invalid', accepted = 0, inserted = 0, refreshed = 0, dropped = 0})
@ -174,6 +196,7 @@ for _, candidate in ipairs(candidates) do
local encoded = cjson.encode(incoming)
redis.call('HSET', records_key, incumbent_id, encoded)
redis.call('ZADD', expiry_key, incoming.expiresAtMs, incumbent_id)
sync_owned(incumbent_id, incoming)
add_available(incumbent_id, incoming)
if tonumber(incoming.expiresAtMs) > max_expiry_ms then
max_expiry_ms = tonumber(incoming.expiresAtMs)
@ -192,6 +215,7 @@ for _, candidate in ipairs(candidates) do
redis.call('HSET', unique_key, candidate.uniqueDigest, candidate.proxyId)
redis.call('HSET', idkeys_key, candidate.proxyId, candidate.uniqueDigest)
redis.call('ZADD', expiry_key, incoming.expiresAtMs, candidate.proxyId)
sync_owned(candidate.proxyId, incoming)
if is_managed(incoming.state) then
redis.call('HINCRBY', inventory_key, candidate.upstream, 1)
end

View File

@ -38,6 +38,10 @@ func TestRedisFixtureUsesIsolatedNamespace(t *testing.T) {
}
func newRedisTestFixture(t *testing.T) redisTestFixture {
return newRedisFixtureWithCredentialCapacity(t, 10_000)
}
func newRedisFixtureWithCredentialCapacity(t *testing.T, credentialCapacity int) redisTestFixture {
t.Helper()
redisURL := os.Getenv("PROXY_POOL_TEST_REDIS_URL")
if redisURL == "" {
@ -54,7 +58,7 @@ func newRedisTestFixture(t *testing.T) redisTestFixture {
_ = client.Close()
t.Fatalf("ping test Redis: %v", err)
}
credentialStore, err := credentials.NewMemoryStore(10_000)
credentialStore, err := credentials.NewMemoryStore(credentialCapacity)
if err != nil {
_ = client.Close()
t.Fatalf("NewMemoryStore(): %v", err)

View File

@ -15,7 +15,10 @@ import (
"proxy-pool/internal/platform/credentials"
)
const maxUpsertScriptBatch = 256
const (
maxUpsertScriptBatch = 256
transientCredentialReleaseTimeout = time.Second
)
type upsertCandidate struct {
ProxyID string `json:"proxyId"`
@ -28,6 +31,9 @@ var _ activitypool.Upserter = (*Adapter)(nil)
func (a *Adapter) UpsertFetched(ctx context.Context, upstreamID string, batch activitypool.FetchedBatch) (activitypool.UpsertResult, error) {
var result activitypool.UpsertResult
if a != nil {
defer a.releaseTransientCredentials(batch.Proxies)
}
if ctx == nil {
return result, activitypool.ErrInvalidBatch
}
@ -84,6 +90,26 @@ func (a *Adapter) UpsertFetched(ctx context.Context, upstreamID string, batch ac
return result, nil
}
func (a *Adapter) releaseTransientCredentials(proxies []proxyDomain.Proxy) {
if a.credentialReleaser == nil {
return
}
releaseCtx, cancel := context.WithTimeout(context.Background(), transientCredentialReleaseTimeout)
defer cancel()
for _, candidate := range proxies {
if releaseCtx.Err() != nil {
return
}
if candidate.SecretRef == "" || candidate.CredentialVersion == "" {
continue
}
_ = a.credentialReleaser.Release(releaseCtx, credentials.Reference{
SecretRef: candidate.SecretRef,
CredentialVersion: candidate.CredentialVersion,
})
}
}
func (a *Adapter) prepareUpsertCandidate(
ctx context.Context,
upstreamID string,
@ -139,7 +165,8 @@ func (a *Adapter) prepareUpsertCandidate(
CreatedAtMS: candidate.CreatedAt.UnixMilli(), ExpiresAtMS: expiresAt.UnixMilli(),
UsableUntilMS: usableUntil.UnixMilli(), LatencyNS: int64(candidate.Latency),
MaxConcurrency: candidate.MaxConcurrency, State: string(candidate.State),
Tags: cloneTags(candidate.Tags), IndexKeys: a.availableIndexKeys(candidate),
Tags: cloneTags(candidate.Tags), OwnerIndexKey: a.keys.owned(upstreamID),
IndexKeys: a.availableIndexKeys(candidate),
}
if candidate.LastCheckedAt != nil {
record.LastCheckedAtMS = candidate.LastCheckedAt.UnixMilli()

View File

@ -173,6 +173,31 @@ func TestRedisUpsertResolvesCredentialsBeforeCommit(t *testing.T) {
}
}
func TestRedisUpsertReleasesTransientCredentialCapacity(t *testing.T) {
fixture := newRedisFixtureWithCredentialCapacity(t, 1)
first := testProxy("proxy-a", "192.0.2.10")
firstReference, err := fixture.Credentials.Put(context.Background(), "first", credentials.Value{
Username: "first", Password: "first-password",
})
if err != nil {
t.Fatalf("Put(first credential): %v", err)
}
first.SecretRef = firstReference.SecretRef
first.CredentialVersion = firstReference.CredentialVersion
if _, err := fixture.Adapter.UpsertFetched(context.Background(), "provider-a", activitypool.FetchedBatch{
ObservedAt: time.Now().UTC(), ConfiguredTTL: time.Minute, MaxSize: 2,
Proxies: []proxyDomain.Proxy{first},
}); err != nil {
t.Fatalf("UpsertFetched(first): %v", err)
}
if _, err := fixture.Credentials.Put(context.Background(), "second", credentials.Value{
Username: "second", Password: "second-password",
}); err != nil {
t.Fatalf("Put(second credential after upsert): %v", err)
}
}
func TestRedisHealthTransitionsAreMonotonicAndIdempotent(t *testing.T) {
fixture := newRedisTestFixture(t)
now := redisTestNow()

View File

@ -55,6 +55,8 @@ func (adapter *Adapter) RunLeader(
) error {
if ctx == nil || adapter == nil || work == nil || strings.TrimSpace(upstreamID) != upstreamID || upstreamID == "" ||
limits.RequestInterval < 0 || limits.MaxInFlight <= 0 || limits.MaxAttemptDuration <= 0 ||
limits.MaxTotal < 0 || limits.MaxTotal > controllerProvider.MaximumCoordinationInteger ||
int64(limits.MaxInFlight) > controllerProvider.MaximumCoordinationInteger ||
limits.MaxAttemptDuration > time.Duration(math.MaxInt64)-adapter.options.PermitGrace {
return controllerProvider.ErrInvalidCoordination
}
@ -217,9 +219,10 @@ func (session *leaderSession) Fence() controllerProvider.Fence {
return controllerProvider.Fence{Generation: session.generation, Epoch: session.epoch}
}
func (session *leaderSession) AcquireFetch(ctx context.Context) (controllerProvider.RequestPermit, error) {
if ctx == nil || session == nil || session.adapter == nil || session.ctx == nil {
return nil, controllerProvider.ErrInvalidCoordination
func (session *leaderSession) AcquireFetch(ctx context.Context, expected int) (controllerProvider.RequestPermit, bool, error) {
if ctx == nil || session == nil || session.adapter == nil || session.ctx == nil ||
expected <= 0 || int64(expected) > controllerProvider.MaximumCoordinationInteger {
return nil, false, controllerProvider.ErrInvalidCoordination
}
operationCtx, cancel := context.WithCancel(ctx)
stop := context.AfterFunc(session.ctx, cancel)
@ -229,32 +232,37 @@ func (session *leaderSession) AcquireFetch(ctx context.Context) (controllerProvi
}()
permitToken, err := randomToken()
if err != nil {
return nil, errors.Join(controllerProvider.ErrCoordinationUnavailable, err)
return nil, false, errors.Join(controllerProvider.ErrCoordinationUnavailable, err)
}
permitTTL := session.limits.MaxAttemptDuration + session.adapter.options.PermitGrace
for operationCtx.Err() == nil {
reply, scriptErr := runScript(operationCtx, session.adapter.client, session.keys,
"acquire_fetch", session.generation, session.holderID, session.token, session.epoch,
permitToken, durationMillis(session.limits.RequestInterval), session.limits.MaxInFlight,
durationMillis(permitTTL),
durationMillis(permitTTL), expected, session.limits.MaxTotal,
)
if scriptErr != nil {
if session.ctx.Err() != nil {
return nil, controllerProvider.ErrLeadershipLost
return nil, false, controllerProvider.ErrLeadershipLost
}
if operationCtx.Err() != nil {
return nil, operationCtx.Err()
return nil, false, operationCtx.Err()
}
if err := wait(operationCtx, session.adapter.options.RetryInterval); err != nil {
return nil, err
return nil, false, err
}
continue
}
switch reply.Status {
case "ok":
return &requestPermit{adapter: session.adapter, keys: session.keys, token: permitToken}, nil
return &requestPermit{
adapter: session.adapter, keys: session.keys, token: permitToken,
settlementTTL: permitTTL,
}, true, nil
case "quota_exhausted":
return nil, false, nil
case "stale":
return nil, controllerProvider.ErrLeadershipLost
return nil, false, controllerProvider.ErrLeadershipLost
case "rate_limited", "at_capacity":
delay := time.Duration(reply.WaitMS) * time.Millisecond
if delay <= 0 {
@ -262,30 +270,42 @@ func (session *leaderSession) AcquireFetch(ctx context.Context) (controllerProvi
}
if err := wait(operationCtx, delay); err != nil {
if session.ctx.Err() != nil {
return nil, controllerProvider.ErrLeadershipLost
return nil, false, controllerProvider.ErrLeadershipLost
}
return nil, err
return nil, false, err
}
default:
return nil, controllerProvider.ErrCoordinationUnavailable
return nil, false, controllerProvider.ErrCoordinationUnavailable
}
}
if session.ctx.Err() != nil {
return nil, controllerProvider.ErrLeadershipLost
return nil, false, controllerProvider.ErrLeadershipLost
}
return nil, operationCtx.Err()
return nil, false, operationCtx.Err()
}
type requestPermit struct {
adapter *Adapter
keys upstreamKeys
token string
mu sync.Mutex
done bool
adapter *Adapter
keys upstreamKeys
token string
settlementTTL time.Duration
mu sync.Mutex
done bool
}
func (permit *requestPermit) Release(ctx context.Context) error {
if ctx == nil || permit == nil || permit.adapter == nil || permit.token == "" {
func (permit *requestPermit) Complete(ctx context.Context, fetched int) error {
if fetched < 0 || int64(fetched) > controllerProvider.MaximumCoordinationInteger {
return controllerProvider.ErrInvalidCoordination
}
return permit.finish(ctx, "complete_fetch", fetched)
}
func (permit *requestPermit) Cancel(ctx context.Context) error {
return permit.finish(ctx, "cancel_fetch", 0)
}
func (permit *requestPermit) finish(ctx context.Context, operation string, fetched int) error {
if ctx == nil || permit == nil || permit.adapter == nil || permit.token == "" || permit.settlementTTL <= 0 {
return controllerProvider.ErrInvalidCoordination
}
permit.mu.Lock()
@ -293,15 +313,22 @@ func (permit *requestPermit) Release(ctx context.Context) error {
if permit.done {
return nil
}
reply, err := runScript(ctx, permit.adapter.client, permit.keys, "release_fetch", permit.token)
if err != nil {
return err
for ctx.Err() == nil {
reply, err := runScript(ctx, permit.adapter.client, permit.keys,
operation, permit.token, fetched, durationMillis(permit.settlementTTL))
if err != nil {
if waitErr := wait(ctx, permit.adapter.options.RetryInterval); waitErr != nil {
return waitErr
}
continue
}
if reply.Status != "ok" {
return controllerProvider.ErrCoordinationUnavailable
}
permit.done = true
return nil
}
if reply.Status != "ok" {
return controllerProvider.ErrCoordinationUnavailable
}
permit.done = true
return nil
return ctx.Err()
}
func randomToken() (string, error) {

View File

@ -89,6 +89,27 @@ func TestRunLeaderRejectsInvalidCalls(t *testing.T) {
}
}
func TestRunLeaderRejectsNegativeFetchQuota(t *testing.T) {
t.Parallel()
client := redis.NewClient(&redis.Options{Addr: "127.0.0.1:0"})
t.Cleanup(func() { _ = client.Close() })
adapter, err := New(client, Options{
Namespace: "controller", HolderID: "controller-a", LeaseTTL: 3 * time.Second,
RenewEvery: time.Second, RetryInterval: 50 * time.Millisecond,
})
if err != nil {
t.Fatalf("New(): %v", err)
}
limits := controllerProvider.CoordinationLimits{
MaxInFlight: 1, MaxAttemptDuration: time.Second, MaxTotal: -1,
}
err = adapter.RunLeader(context.Background(), "provider-a", limits,
func(context.Context, controllerProvider.LeaderSession) error { return nil })
if !errors.Is(err, controllerProvider.ErrInvalidCoordination) {
t.Fatalf("RunLeader() error = %v, want ErrInvalidCoordination", err)
}
}
func TestLeaderWorkResultPrefersParentCancellation(t *testing.T) {
t.Parallel()
ctx, cancel := context.WithCancel(context.Background())

View File

@ -103,36 +103,201 @@ func TestRedisLeaderSessionEnforcesGlobalIntervalAndInFlightLimit(t *testing.T)
})
}()
session := receiveSession(t, sessions)
first, err := session.AcquireFetch(context.Background())
if err != nil {
t.Fatalf("first AcquireFetch(): %v", err)
first, available, err := session.AcquireFetch(context.Background(), 1)
if err != nil || !available {
t.Fatalf("first AcquireFetch() = (%v, %t, %v)", first, available, err)
}
startedAt := time.Now()
secondResult := make(chan permitResultFixture, 1)
go func() {
permit, acquireErr := session.AcquireFetch(context.Background())
secondResult <- permitResultFixture{permit: permit, err: acquireErr}
permit, permitAvailable, acquireErr := session.AcquireFetch(context.Background(), 1)
secondResult <- permitResultFixture{permit: permit, available: permitAvailable, err: acquireErr}
}()
select {
case result := <-secondResult:
t.Fatalf("second AcquireFetch() returned before release: %+v", result)
case <-time.After(100 * time.Millisecond):
}
if err := first.Release(context.Background()); err != nil {
t.Fatalf("first Release(): %v", err)
if err := first.Complete(context.Background(), 1); err != nil {
t.Fatalf("first Complete(): %v", err)
}
result := receivePermit(t, secondResult)
if result.err != nil || result.permit == nil {
t.Fatalf("second AcquireFetch() = (%v, %v)", result.permit, result.err)
if result.err != nil || !result.available || result.permit == nil {
t.Fatalf("second AcquireFetch() = (%v, %t, %v)", result.permit, result.available, result.err)
}
if elapsed := time.Since(startedAt); elapsed < 200*time.Millisecond {
t.Fatalf("global request interval = %s, want at least 200ms", elapsed)
}
if err := result.permit.Release(context.Background()); err != nil {
t.Fatalf("second Release(): %v", err)
if err := result.permit.Cancel(context.Background()); err != nil {
t.Fatalf("second Cancel(): %v", err)
}
if err := result.permit.Release(context.Background()); err != nil {
t.Fatalf("idempotent second Release(): %v", err)
if err := result.permit.Cancel(context.Background()); err != nil {
t.Fatalf("idempotent second Cancel(): %v", err)
}
cancel()
waitRunner(t, done)
}
func TestRedisLeaderSessionPreservesFetchQuotaAcrossFailover(t *testing.T) {
fixture := newRedisFixture(t)
limits := controllerProvider.CoordinationLimits{
MaxInFlight: 1, MaxAttemptDuration: time.Second, MaxTotal: 2,
}
firstCtx, cancelFirst := context.WithCancel(context.Background())
firstSessions := make(chan controllerProvider.LeaderSession, 1)
firstDone := make(chan error, 1)
go func() {
firstDone <- fixture.coordinator(t, "controller-a").RunLeader(
firstCtx, "provider-a", limits,
func(workCtx context.Context, session controllerProvider.LeaderSession) error {
firstSessions <- session
<-workCtx.Done()
return nil
},
)
}()
firstSession := receiveSession(t, firstSessions)
firstPermit, available, err := firstSession.AcquireFetch(context.Background(), 2)
if err != nil || !available || firstPermit == nil {
t.Fatalf("first AcquireFetch() = (%v, %t, %v)", firstPermit, available, err)
}
firstFence := firstSession.Fence()
cancelFirst()
waitRunner(t, firstDone)
if err := firstPermit.Complete(context.Background(), 1); err != nil {
t.Fatalf("Complete() after leadership loss: %v", err)
}
secondCtx, cancelSecond := context.WithCancel(context.Background())
defer cancelSecond()
secondSessions := make(chan controllerProvider.LeaderSession, 1)
secondDone := make(chan error, 1)
go func() {
secondDone <- fixture.coordinator(t, "controller-b").RunLeader(
secondCtx, "provider-a", limits,
func(workCtx context.Context, session controllerProvider.LeaderSession) error {
secondSessions <- session
<-workCtx.Done()
return nil
},
)
}()
secondSession := receiveSession(t, secondSessions)
secondFence := secondSession.Fence()
if secondFence.Generation != firstFence.Generation || secondFence.Epoch <= firstFence.Epoch {
t.Fatalf("second fence = %+v, first = %+v", secondFence, firstFence)
}
secondPermit, available, err := secondSession.AcquireFetch(context.Background(), 1)
if err != nil || !available || secondPermit == nil {
t.Fatalf("second AcquireFetch() = (%v, %t, %v)", secondPermit, available, err)
}
if err := secondPermit.Complete(context.Background(), 1); err != nil {
t.Fatalf("second Complete(): %v", err)
}
exhaustedPermit, available, err := secondSession.AcquireFetch(context.Background(), 1)
if err != nil || available || exhaustedPermit != nil {
t.Fatalf("exhausted AcquireFetch() = (%v, %t, %v), want unavailable", exhaustedPermit, available, err)
}
cancelSecond()
waitRunner(t, secondDone)
}
func TestRedisFetchQuotaSettlementIsIdempotentAndCancellationRefundsReservation(t *testing.T) {
fixture := newRedisFixture(t)
limits := controllerProvider.CoordinationLimits{
MaxInFlight: 2, MaxAttemptDuration: time.Second, MaxTotal: 2,
}
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
sessions := make(chan controllerProvider.LeaderSession, 1)
done := make(chan error, 1)
go func() {
done <- fixture.coordinator(t, "controller-a").RunLeader(
ctx, "provider-a", limits,
func(workCtx context.Context, session controllerProvider.LeaderSession) error {
sessions <- session
<-workCtx.Done()
return nil
},
)
}()
session := receiveSession(t, sessions)
cancelled, available, err := session.AcquireFetch(context.Background(), 2)
if err != nil || !available || cancelled == nil {
t.Fatalf("cancelled AcquireFetch() = (%v, %t, %v)", cancelled, available, err)
}
if err := cancelled.Cancel(context.Background()); err != nil {
t.Fatalf("Cancel(): %v", err)
}
if err := cancelled.Cancel(context.Background()); err != nil {
t.Fatalf("idempotent Cancel(): %v", err)
}
completed, available, err := session.AcquireFetch(context.Background(), 2)
if err != nil || !available || completed == nil {
t.Fatalf("completed AcquireFetch() = (%v, %t, %v)", completed, available, err)
}
if err := completed.Complete(context.Background(), 1); err != nil {
t.Fatalf("Complete(): %v", err)
}
if err := completed.Complete(context.Background(), 1); err != nil {
t.Fatalf("idempotent Complete(): %v", err)
}
if err := completed.Cancel(context.Background()); err != nil {
t.Fatalf("Cancel() after Complete(): %v", err)
}
last, available, err := session.AcquireFetch(context.Background(), 1)
if err != nil || !available || last == nil {
t.Fatalf("last AcquireFetch() = (%v, %t, %v)", last, available, err)
}
if err := last.Complete(context.Background(), 1); err != nil {
t.Fatalf("last Complete(): %v", err)
}
exhausted, available, err := session.AcquireFetch(context.Background(), 1)
if err != nil || available || exhausted != nil {
t.Fatalf("exhausted AcquireFetch() = (%v, %t, %v)", exhausted, available, err)
}
cancel()
waitRunner(t, done)
}
func TestRedisExpiredFetchReservationIsConservativelyCharged(t *testing.T) {
fixture := newRedisFixture(t)
coordinator, err := New(fixture.client, Options{
Namespace: fixture.namespace, HolderID: "controller-a", LeaseTTL: 600 * time.Millisecond,
RenewEvery: 150 * time.Millisecond, RetryInterval: 10 * time.Millisecond, PermitGrace: 10 * time.Millisecond,
})
if err != nil {
t.Fatalf("New(): %v", err)
}
limits := controllerProvider.CoordinationLimits{
MaxInFlight: 1, MaxAttemptDuration: 40 * time.Millisecond, MaxTotal: 1,
}
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
sessions := make(chan controllerProvider.LeaderSession, 1)
done := make(chan error, 1)
go func() {
done <- coordinator.RunLeader(ctx, "provider-a", limits,
func(workCtx context.Context, session controllerProvider.LeaderSession) error {
sessions <- session
<-workCtx.Done()
return nil
})
}()
session := receiveSession(t, sessions)
abandoned, available, err := session.AcquireFetch(context.Background(), 1)
if err != nil || !available || abandoned == nil {
t.Fatalf("abandoned AcquireFetch() = (%v, %t, %v)", abandoned, available, err)
}
time.Sleep(80 * time.Millisecond)
exhausted, available, err := session.AcquireFetch(context.Background(), 1)
if err != nil || available || exhausted != nil {
t.Fatalf("post-expiry AcquireFetch() = (%v, %t, %v), want charged quota", exhausted, available, err)
}
cancel()
waitRunner(t, done)
@ -280,8 +445,9 @@ func receivePermit(t *testing.T, values <-chan permitResultFixture) permitResult
}
type permitResultFixture struct {
permit controllerProvider.RequestPermit
err error
permit controllerProvider.RequestPermit
available bool
err error
}
func waitRunner(t *testing.T, done <-chan error) {

View File

@ -14,11 +14,15 @@ type keyBuilder struct {
}
type upstreamKeys struct {
generation string
epoch string
leader string
next string
inflight string
generation string
epoch string
leader string
next string
inflight string
fetchedTotal string
pendingTotal string
permits string
permitExpiry string
}
func (builder keyBuilder) forUpstream(upstreamID string) (upstreamKeys, error) {
@ -28,16 +32,23 @@ func (builder keyBuilder) forUpstream(upstreamID string) (upstreamKeys, error) {
digest := digestParts(upstreamID)
prefix := "pp:" + builder.namespace + ":{provider:" + digest + "}"
return upstreamKeys{
generation: prefix + ":generation",
epoch: prefix + ":epoch",
leader: prefix + ":leader",
next: prefix + ":next-request",
inflight: prefix + ":inflight",
generation: prefix + ":generation",
epoch: prefix + ":epoch",
leader: prefix + ":leader",
next: prefix + ":next-request",
inflight: prefix + ":inflight",
fetchedTotal: prefix + ":fetched-total",
pendingTotal: prefix + ":pending-total",
permits: prefix + ":permits",
permitExpiry: prefix + ":permit-expiry",
}, nil
}
func (keys upstreamKeys) all() []string {
return []string{keys.generation, keys.epoch, keys.leader, keys.next, keys.inflight}
return []string{
keys.generation, keys.epoch, keys.leader, keys.next, keys.inflight,
keys.fetchedTotal, keys.pendingTotal, keys.permits, keys.permitExpiry,
}
}
func digestParts(values ...string) string {

View File

@ -1,4 +1,5 @@
local operation = ARGV[1]
local max_safe_integer = 9007199254740991
local function now_ms()
local value = redis.call('TIME')
@ -33,6 +34,63 @@ local function same_leader(value, generation, holder_id, token, epoch)
value.token == token and value.epoch == tostring(epoch)
end
local function read_counter(key)
local encoded = redis.call('GET', key)
if not encoded then
return 0, nil
end
local value = tonumber(encoded)
if not value or value < 0 or value ~= math.floor(value) then
return nil, 'invalid'
end
return value, nil
end
local function read_permit(token)
local encoded = redis.call('HGET', KEYS[8], token)
if not encoded then
return nil, nil
end
local ok, value = pcall(cjson.decode, encoded)
if not ok or type(value) ~= 'table' or type(value.state) ~= 'string' or
type(value.expected) ~= 'number' or value.expected <= 0 or
value.expected ~= math.floor(value.expected) then
return nil, 'invalid'
end
return value, nil
end
local function cleanup_expired(now)
local expired = redis.call('ZRANGEBYSCORE', KEYS[9], '-inf', now, 'LIMIT', 0, 256)
if #expired == 0 then
return nil
end
local fetched, fetched_error = read_counter(KEYS[6])
local pending, pending_error = read_counter(KEYS[7])
if fetched_error or pending_error then
return 'invalid'
end
for _, permit_token in ipairs(expired) do
local permit, permit_error = read_permit(permit_token)
if permit_error then
return 'invalid'
end
if permit and permit.state == 'reserved' then
if pending < permit.expected or fetched > max_safe_integer - permit.expected then
return 'invalid'
end
pending = pending - permit.expected
fetched = fetched + permit.expected
end
redis.call('HDEL', KEYS[8], permit_token)
redis.call('ZREM', KEYS[5], permit_token)
redis.call('ZREM', KEYS[9], permit_token)
end
redis.call('SET', KEYS[6], fetched)
redis.call('SET', KEYS[7], pending)
return nil
end
if operation == 'acquire_leader' then
local generation_candidate = ARGV[2]
local holder_id = ARGV[3]
@ -41,7 +99,10 @@ if operation == 'acquire_leader' then
if not lease_ttl or lease_ttl <= 0 then
return reply('invalid', '', 0, 0)
end
redis.call('SET', KEYS[1], generation_candidate, 'NX')
local created = redis.call('SET', KEYS[1], generation_candidate, 'NX')
if created then
redis.call('DEL', KEYS[2], KEYS[3], KEYS[4], KEYS[5], KEYS[6], KEYS[7], KEYS[8], KEYS[9])
end
local generation = redis.call('GET', KEYS[1])
local current, current_error = read_leader()
if current_error then
@ -112,6 +173,13 @@ if operation == 'acquire_fetch' then
local request_interval = tonumber(ARGV[7])
local max_in_flight = tonumber(ARGV[8])
local permit_ttl = tonumber(ARGV[9])
local expected = tonumber(ARGV[10])
local max_total = tonumber(ARGV[11])
if not expected or expected <= 0 or expected ~= math.floor(expected) or
expected > max_safe_integer or not max_total or max_total < 0 or
max_total > max_safe_integer or max_total ~= math.floor(max_total) then
return reply('invalid', generation, epoch or 0, 0)
end
local current, current_error = read_leader()
if current_error then
return reply('unavailable', generation, epoch or 0, 0)
@ -120,11 +188,27 @@ if operation == 'acquire_fetch' then
return reply('stale', generation, epoch or 0, 0)
end
local now = now_ms()
redis.call('ZREMRANGEBYSCORE', KEYS[5], '-inf', now)
local existing = redis.call('ZSCORE', KEYS[5], permit_token)
if existing then
if cleanup_expired(now) then
return reply('unavailable', generation, epoch, 0)
end
local existing, existing_error = read_permit(permit_token)
if existing_error then
return reply('unavailable', generation, epoch, 0)
end
if existing and existing.state == 'reserved' then
return reply('ok', generation, epoch, 0)
end
if existing then
return reply('unavailable', generation, epoch, 0)
end
local fetched, fetched_error = read_counter(KEYS[6])
local pending, pending_error = read_counter(KEYS[7])
if fetched_error or pending_error then
return reply('unavailable', generation, epoch, 0)
end
if max_total > 0 and fetched + pending + expected > max_total then
return reply('quota_exhausted', generation, epoch, 0)
end
local next_request = redis.call('GET', KEYS[4])
if next_request and not tonumber(next_request) then
return reply('unavailable', generation, epoch, 0)
@ -141,7 +225,13 @@ if operation == 'acquire_fetch' then
return reply('at_capacity', generation, epoch, wait_ms)
end
redis.call('ZADD', KEYS[5], now + permit_ttl, permit_token)
redis.call('PEXPIRE', KEYS[5], permit_ttl + 1000)
redis.call('ZADD', KEYS[9], now + permit_ttl, permit_token)
redis.call('HSET', KEYS[8], permit_token, cjson.encode({
version = 1,
state = 'reserved',
expected = expected
}))
redis.call('SET', KEYS[7], pending + expected)
if request_interval > 0 then
redis.call('SET', KEYS[4], now + request_interval, 'PX', request_interval)
else
@ -150,8 +240,43 @@ if operation == 'acquire_fetch' then
return reply('ok', generation, epoch, 0)
end
if operation == 'release_fetch' then
redis.call('ZREM', KEYS[5], ARGV[2])
if operation == 'complete_fetch' or operation == 'cancel_fetch' then
local permit_token = ARGV[2]
local fetched_count = tonumber(ARGV[3])
local settlement_ttl = tonumber(ARGV[4])
if not fetched_count or fetched_count < 0 or fetched_count ~= math.floor(fetched_count) or
fetched_count > max_safe_integer or not settlement_ttl or settlement_ttl <= 0 then
return reply('invalid', '', 0, 0)
end
local now = now_ms()
if cleanup_expired(now) then
return reply('unavailable', '', 0, 0)
end
local permit, permit_error = read_permit(permit_token)
if permit_error then
return reply('unavailable', '', 0, 0)
end
if not permit or permit.state ~= 'reserved' then
return reply('ok', '', 0, 0)
end
local fetched, fetched_error = read_counter(KEYS[6])
local pending, pending_error = read_counter(KEYS[7])
if fetched_error or pending_error or pending < permit.expected or
fetched > max_safe_integer - fetched_count then
return reply('unavailable', '', 0, 0)
end
pending = pending - permit.expected
if operation == 'complete_fetch' then
fetched = fetched + fetched_count
permit.state = 'completed'
else
permit.state = 'cancelled'
end
redis.call('SET', KEYS[6], fetched)
redis.call('SET', KEYS[7], pending)
redis.call('HSET', KEYS[8], permit_token, cjson.encode(permit))
redis.call('ZREM', KEYS[5], permit_token)
redis.call('ZADD', KEYS[9], now + settlement_ttl, permit_token)
return reply('ok', '', 0, 0)
end

View File

@ -5,6 +5,12 @@ import (
"time"
)
const (
MaximumPoolSize = 1_000_000
MaximumExactCounter = int64(1<<53 - 1)
MaximumUpstreams = 4_096
)
type Duration time.Duration
func (d *Duration) UnmarshalText(text []byte) error {

View File

@ -4,6 +4,7 @@ import (
"fmt"
"os"
"path/filepath"
"strconv"
"strings"
"testing"
"time"
@ -553,6 +554,13 @@ func TestValidateRejectsInvalidConfigurationMatrix(t *testing.T) {
},
want: "estimatedIPsPerCall",
},
{
name: "pool exceeds runtime scan bound",
mutate: func(cfg *Config) {
updateUpstream(cfg, func(upstream *Upstream) { upstream.Pool.MaxSize = MaximumPoolSize + 1 })
},
want: "pool.maxSize",
},
{
name: "zero refill interval",
mutate: func(cfg *Config) {
@ -655,6 +663,58 @@ func TestValidateRejectsInvalidConfigurationMatrix(t *testing.T) {
}
}
func TestValidateRejectsUnboundedUpstreamCardinality(t *testing.T) {
cfg := mustLoadValidConfig(t)
for index := len(cfg.Upstreams); index <= MaximumUpstreams; index++ {
cfg.Upstreams[fmt.Sprintf("disabled-%d", index)] = Upstream{}
}
if err := Validate(cfg); err == nil || !strings.Contains(err.Error(), "upstream count") {
t.Fatalf("Validate(too many upstreams) error = %v", err)
}
}
func TestValidateRejectsCountersOutsideRedisExactRange(t *testing.T) {
if strconv.IntSize < 64 {
t.Skip("64-bit int is required for values above the Redis exact range")
}
tests := []struct {
name string
mutate func(*Upstream)
}{
{name: "fetch total", mutate: func(upstream *Upstream) {
upstream.Fetch.MaxTotal = int(MaximumExactCounter) + 1
}},
{name: "proxy concurrency", mutate: func(upstream *Upstream) {
upstream.Capacity.MaxConcurrencyPerProxy = int(MaximumExactCounter) + 1
}},
{name: "minimum slots", mutate: func(upstream *Upstream) {
upstream.Refill.MinimumAvailableSlots = MaximumExactCounter + 1
upstream.Refill.TargetAvailableSlots = MaximumExactCounter + 2
}},
{name: "target slots", mutate: func(upstream *Upstream) {
upstream.Capacity.MaxConcurrencyPerProxy = int(MaximumExactCounter)
upstream.Pool.MaxSize = 1
upstream.Refill.MinimumAvailableSlots = MaximumExactCounter
upstream.Refill.TargetAvailableSlots = MaximumExactCounter + 1
}},
{name: "theoretical slots", mutate: func(upstream *Upstream) {
upstream.Pool.MaxSize = 2
upstream.Capacity.MaxConcurrencyPerProxy = int(MaximumExactCounter/2 + 1)
upstream.Refill.MinimumAvailableSlots = 1
upstream.Refill.TargetAvailableSlots = 2
}},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
cfg := mustLoadValidConfig(t)
updateUpstream(cfg, test.mutate)
if err := Validate(cfg); err == nil || !strings.Contains(err.Error(), "exact counter range") {
t.Fatalf("Validate(inexact counter) error = %v", err)
}
})
}
}
func TestResolvedConfigFormattingRedactsSecrets(t *testing.T) {
configured := strings.Replace(validConfig, ` auth:
mode: none`, ` auth:

View File

@ -0,0 +1,30 @@
package config
import (
"crypto/hmac"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
)
const MinimumFingerprintKeyBytes = 32
var ErrInvalidFingerprint = errors.New("invalid configuration fingerprint")
// Fingerprint returns a keyed digest of one fully resolved configuration.
// Only the digest is persisted; the resolved configuration and key remain local.
func Fingerprint(configuration *Config, key []byte) (string, error) {
if configuration == nil || len(key) < MinimumFingerprintKeyBytes {
return "", ErrInvalidFingerprint
}
encoded, err := json.Marshal(configuration)
if err != nil {
return "", errors.Join(ErrInvalidFingerprint, err)
}
digest := hmac.New(sha256.New, key)
if _, err := digest.Write(encoded); err != nil {
return "", errors.Join(ErrInvalidFingerprint, err)
}
return hex.EncodeToString(digest.Sum(nil)), nil
}

View File

@ -0,0 +1,68 @@
package config
import "testing"
var testFingerprintKey = []byte("0123456789abcdef0123456789abcdef")
func TestFingerprintIsStableAndTracksSecretRotation(t *testing.T) {
first := storeTestConfig("provider-a")
upstream := first.Upstreams["provider-a"]
upstream.ProxyAuth.Password = "secret-a"
first.Upstreams["provider-a"] = upstream
stable, err := Fingerprint(first, testFingerprintKey)
if err != nil {
t.Fatalf("Fingerprint(first): %v", err)
}
again, err := Fingerprint(first, testFingerprintKey)
if err != nil || again != stable {
t.Fatalf("Fingerprint(stable) = %q, %v; want %q", again, err, stable)
}
rotated := storeTestConfig("provider-a")
upstream = rotated.Upstreams["provider-a"]
upstream.ProxyAuth.Password = "secret-b"
rotated.Upstreams["provider-a"] = upstream
changed, err := Fingerprint(rotated, testFingerprintKey)
if err != nil {
t.Fatalf("Fingerprint(rotated): %v", err)
}
if changed == stable {
t.Fatal("Fingerprint did not change after secret rotation")
}
if len(changed) != 64 {
t.Fatalf("Fingerprint length = %d, want 64", len(changed))
}
}
func TestFingerprintChangesWithIndependentKey(t *testing.T) {
configuration := storeTestConfig("provider-a")
first, err := Fingerprint(configuration, testFingerprintKey)
if err != nil {
t.Fatalf("Fingerprint(first key): %v", err)
}
second, err := Fingerprint(configuration, []byte("fedcba9876543210fedcba9876543210"))
if err != nil {
t.Fatalf("Fingerprint(second key): %v", err)
}
if first == second {
t.Fatal("Fingerprint did not change with independent key")
}
}
func TestFingerprintRejectsInvalidInputs(t *testing.T) {
for name, test := range map[string]struct {
configuration *Config
key []byte
}{
"nil configuration": {key: testFingerprintKey},
"missing key": {configuration: storeTestConfig("provider-a")},
"short key": {configuration: storeTestConfig("provider-a"), key: []byte("too-short")},
} {
t.Run(name, func(t *testing.T) {
if _, err := Fingerprint(test.configuration, test.key); err == nil {
t.Fatal("Fingerprint() succeeded")
}
})
}
}

View File

@ -9,7 +9,12 @@ var ErrInvalidStore = errors.New("invalid configuration store")
// Store publishes complete validated configurations with one atomic pointer swap.
type Store struct {
current atomic.Pointer[Config]
current atomic.Pointer[publishedConfiguration]
}
type publishedConfiguration struct {
value Config
revision uint64
}
func NewStore(initial *Config) (*Store, error) {
@ -17,7 +22,7 @@ func NewStore(initial *Config) (*Store, error) {
return nil, errors.Join(ErrInvalidStore, err)
}
store := &Store{}
store.Publish(initial)
store.current.Store(&publishedConfiguration{value: cloneConfig(*initial)})
return store, nil
}
@ -25,19 +30,38 @@ func (store *Store) Current() *Config {
if store == nil {
return nil
}
current := store.current.Load()
if current == nil {
published := store.current.Load()
if published == nil {
return nil
}
cloned := cloneConfig(*current)
cloned := cloneConfig(published.value)
return &cloned
}
// Publish accepts a non-nil configuration already validated by the caller.
func (store *Store) Publish(configuration *Config) {
if store == nil || configuration == nil {
return
func (store *Store) Revision() uint64 {
if store == nil {
return 0
}
published := store.current.Load()
if published == nil {
return 0
}
return published.revision
}
// PublishRevision publishes only a strictly newer authoritative revision.
func (store *Store) PublishRevision(configuration *Config, revision uint64) bool {
if store == nil || configuration == nil || revision == 0 {
return false
}
for {
current := store.current.Load()
if current != nil && revision <= current.revision {
return false
}
next := &publishedConfiguration{value: cloneConfig(*configuration), revision: revision}
if store.current.CompareAndSwap(current, next) {
return true
}
}
cloned := cloneConfig(*configuration)
store.current.Store(&cloned)
}

View File

@ -26,7 +26,9 @@ func TestStorePublishesAndReturnsDetachedConfigurations(t *testing.T) {
}
next := storeTestConfig("provider-b")
store.Publish(next)
if !store.PublishRevision(next, 1) {
t.Fatal("PublishRevision() rejected newer configuration")
}
next.Routing[0].Upstreams[0] = "mutated"
if got := store.Current().Routing[0].Upstreams[0]; got != "provider-b" {
t.Fatalf("Publish() retained caller state: %q", got)
@ -56,7 +58,7 @@ func TestStoreSupportsConcurrentReadersAndPublishers(t *testing.T) {
if index%2 == 1 {
name = "provider-b"
}
store.Publish(storeTestConfig(name))
store.PublishRevision(storeTestConfig(name), uint64(index+1))
}(index)
go func() {
defer wait.Done()
@ -69,6 +71,29 @@ func TestStoreSupportsConcurrentReadersAndPublishers(t *testing.T) {
wait.Wait()
}
func TestStoreRejectsOutOfOrderRevisionPublication(t *testing.T) {
t.Parallel()
store, err := NewStore(storeTestConfig("provider-a"))
if err != nil {
t.Fatalf("NewStore() error = %v", err)
}
if published := store.PublishRevision(storeTestConfig("provider-b"), 2); !published {
t.Fatal("PublishRevision(newer) rejected")
}
if published := store.PublishRevision(storeTestConfig("provider-c"), 1); published {
t.Fatal("PublishRevision(stale) succeeded")
}
if published := store.PublishRevision(storeTestConfig("provider-c"), 2); published {
t.Fatal("PublishRevision(equal) succeeded")
}
if got := store.Current().Routing[0].Upstreams[0]; got != "provider-b" {
t.Fatalf("Current() upstream = %q, want provider-b", got)
}
if got := store.Revision(); got != 2 {
t.Fatalf("Revision() = %d, want 2", got)
}
}
func storeTestConfig(upstreamName string) *Config {
return &Config{
Version: 1,

View File

@ -46,6 +46,9 @@ func Validate(cfg *Config) error {
if err := validateCheck("defaults.check", cfg.Defaults.Check); err != nil {
return err
}
if len(cfg.Upstreams) > MaximumUpstreams {
return fmt.Errorf("validate configuration: upstream count exceeds %d", MaximumUpstreams)
}
enabledUpstreams := 0
for name, upstream := range cfg.Upstreams {
if upstream.Enabled {
@ -330,28 +333,46 @@ func validateUpstream(name string, upstream Upstream) error {
if err := requirePositive(scope+" pool.maxSize", upstream.Pool.MaxSize); err != nil {
return err
}
if upstream.Pool.MaxSize > MaximumPoolSize {
return fmt.Errorf("validate %s pool.maxSize: exceeds %d", scope, MaximumPoolSize)
}
if err := requireNonNegative(scope+" fetch.maxTotal", upstream.Fetch.MaxTotal); err != nil {
return err
}
if int64(upstream.Fetch.MaxTotal) > MaximumExactCounter {
return fmt.Errorf("validate %s fetch.maxTotal: exceeds exact counter range", scope)
}
if upstream.Fetch.MaxTotal > 0 && upstream.Fetch.MaxTotal < upstream.Pool.MaxSize {
return fmt.Errorf("validate %s fetch.maxTotal: cannot be lower than pool.maxSize", scope)
}
if err := requirePositive(scope+" capacity.maxConcurrencyPerProxy", upstream.Capacity.MaxConcurrencyPerProxy); err != nil {
return err
}
if int64(upstream.Capacity.MaxConcurrencyPerProxy) > MaximumExactCounter {
return fmt.Errorf("validate %s capacity.maxConcurrencyPerProxy: exceeds exact counter range", scope)
}
if err := requirePositive(scope+" refill.reconcileInterval", upstream.Refill.ReconcileInterval); err != nil {
return err
}
if err := requirePositive(scope+" refill.minimumAvailableSlots", upstream.Refill.MinimumAvailableSlots); err != nil {
return err
}
if upstream.Refill.MinimumAvailableSlots > MaximumExactCounter {
return fmt.Errorf("validate %s refill.minimumAvailableSlots: exceeds exact counter range", scope)
}
if upstream.Refill.TargetAvailableSlots <= upstream.Refill.MinimumAvailableSlots {
return fmt.Errorf("validate %s refill.targetAvailableSlots: must be greater than minimumAvailableSlots", scope)
}
if upstream.Refill.TargetAvailableSlots > MaximumExactCounter {
return fmt.Errorf("validate %s refill.targetAvailableSlots: exceeds exact counter range", scope)
}
if int64(upstream.Pool.MaxSize) > math.MaxInt64/int64(upstream.Capacity.MaxConcurrencyPerProxy) {
return fmt.Errorf("validate %s refill.targetAvailableSlots: theoretical capacity overflows int64", scope)
}
theoreticalSlots := int64(upstream.Pool.MaxSize) * int64(upstream.Capacity.MaxConcurrencyPerProxy)
if theoreticalSlots > MaximumExactCounter {
return fmt.Errorf("validate %s refill.targetAvailableSlots: theoretical capacity exceeds exact counter range", scope)
}
if upstream.Refill.TargetAvailableSlots > theoreticalSlots {
return fmt.Errorf("validate %s refill.targetAvailableSlots: exceeds theoretical capacity", scope)
}
@ -415,6 +436,9 @@ func validateFetch(scope string, fetch Fetch) error {
if err := requirePositive(scope+".maxInFlight", fetch.MaxInFlight); err != nil {
return err
}
if int64(fetch.MaxInFlight) > MaximumExactCounter {
return fmt.Errorf("validate %s.maxInFlight: exceeds exact counter range", scope)
}
if err := requireNonNegative(scope+".maxTotal", fetch.MaxTotal); err != nil {
return err
}

View File

@ -2,9 +2,6 @@ package admin
import (
"context"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
"sort"
"strings"
@ -29,9 +26,15 @@ type ConfigurationLoader interface {
LoadConfiguration(context.Context) (LoadedConfiguration, error)
}
// ConfigurationPublisher must atomically publish an already validated configuration.
// ConfigurationPublisher publishes only a newer authoritative configuration revision.
type ConfigurationPublisher interface {
Publish(*config.Config)
PublishRevision(*config.Config, uint64) bool
}
type RuntimeController interface {
Notify()
ValidateConfiguration(context.Context, *config.Config) error
ValidateUpstream(context.Context, string) error
}
var _ ConfigurationPublisher = (*config.Store)(nil)
@ -41,10 +44,12 @@ type ApplicationDependencies struct {
Operations OperationalStatusReader
Configuration ConfigurationLoader
Publisher ConfigurationPublisher
Runtime RuntimeController
}
type ApplicationOptions struct {
Now func() time.Time
Now func() time.Time
FingerprintKey []byte
}
type OperationalStatus struct {
@ -70,30 +75,40 @@ type LoadedConfiguration struct {
}
type ApplicationService struct {
state StateRepository
operations OperationalStatusReader
configuration ConfigurationLoader
publisher ConfigurationPublisher
now func() time.Time
state StateRepository
operations OperationalStatusReader
configuration ConfigurationLoader
publisher ConfigurationPublisher
runtime RuntimeController
now func() time.Time
fingerprintKey []byte
}
var _ Service = (*ApplicationService)(nil)
func NewApplicationService(dependencies ApplicationDependencies, options ApplicationOptions) (*ApplicationService, error) {
if nilInterface(dependencies.State) || nilInterface(dependencies.Operations) || nilInterface(dependencies.Configuration) ||
nilInterface(dependencies.Publisher) || options.Now == nil {
nilInterface(dependencies.Publisher) || options.Now == nil ||
len(options.FingerprintKey) < config.MinimumFingerprintKeyBytes {
return nil, ErrInvalidApplicationService
}
return &ApplicationService{
state: dependencies.State,
operations: dependencies.Operations,
configuration: dependencies.Configuration,
publisher: dependencies.Publisher,
now: options.Now,
state: dependencies.State,
operations: dependencies.Operations,
configuration: dependencies.Configuration,
publisher: dependencies.Publisher,
runtime: dependencies.Runtime,
now: options.Now,
fingerprintKey: append([]byte(nil), options.FingerprintKey...),
}, nil
}
func (service *ApplicationService) SetUpstreamEnabled(ctx context.Context, command SetUpstreamCommand) (MutationResult, error) {
if command.Enabled && service.runtime != nil {
if err := service.runtime.ValidateUpstream(ctx, command.Name); err != nil {
return MutationResult{RequestID: command.RequestID}, errors.Join(ErrInvalidConfiguration, err)
}
}
result, err := service.state.SetUpstreamEnabled(ctx, adminstate.SetUpstreamCommand{
RequestID: command.RequestID,
Actor: adminstate.Actor{ID: command.ActorID, SourceIP: command.SourceIP},
@ -101,6 +116,9 @@ func (service *ApplicationService) SetUpstreamEnabled(ctx context.Context, comma
Name: command.Name,
Enabled: command.Enabled,
})
if err == nil && service.runtime != nil {
service.runtime.Notify()
}
return mutationResult(result), mapAdminStateError(err)
}
@ -202,14 +220,16 @@ func (service *ApplicationService) ApplyConfiguration(
if err := config.Validate(loaded.Value); err != nil {
return MutationResult{RequestID: command.RequestID}, errors.Join(ErrInvalidConfiguration, err)
}
if service.runtime != nil {
if err := service.runtime.ValidateConfiguration(ctx, loaded.Value); err != nil {
return MutationResult{RequestID: command.RequestID}, errors.Join(ErrInvalidConfiguration, err)
}
}
managementView := loaded.Value.Redacted()
encoded, err := json.Marshal(managementView)
checksum, err := config.Fingerprint(loaded.Value, service.fingerprintKey)
if err != nil {
return MutationResult{RequestID: command.RequestID}, errors.Join(ErrInvalidConfiguration, err)
}
digest := sha256.Sum256(encoded)
checksum := hex.EncodeToString(digest[:])
current, err := service.state.Snapshot(ctx)
if err != nil {
@ -229,7 +249,13 @@ func (service *ApplicationService) ApplyConfiguration(
if err != nil {
return mutationResult(result), mapAdminStateError(err)
}
service.publisher.Publish(loaded.Value)
if result.Revision == 0 {
return mutationResult(result), errors.Join(ErrUnavailable, ErrInvalidApplicationService)
}
published := service.publisher.PublishRevision(loaded.Value, result.Revision)
if published && service.runtime != nil {
service.runtime.Notify()
}
return mutationResult(result), nil
}

View File

@ -3,6 +3,7 @@ package admin
import (
"context"
"errors"
"sync/atomic"
"testing"
"time"
@ -10,6 +11,12 @@ import (
"proxy-pool/internal/domain/adminstate"
)
var applicationTestFingerprintKey = []byte("0123456789abcdef0123456789abcdef")
func applicationTestOptions(now func() time.Time) ApplicationOptions {
return ApplicationOptions{Now: now, FingerprintKey: applicationTestFingerprintKey}
}
func TestApplicationServiceMapsUpstreamMutationToAdminState(t *testing.T) {
t.Parallel()
now := time.Date(2026, 7, 29, 10, 0, 0, 0, time.UTC)
@ -21,12 +28,14 @@ func TestApplicationServiceMapsUpstreamMutationToAdminState(t *testing.T) {
Message: "enabled",
},
}
runtime := &recordingRuntimeNotifier{}
service, err := NewApplicationService(ApplicationDependencies{
State: state,
Operations: staticOperationalStatusReader{},
Configuration: staticConfigurationLoader{},
Publisher: &recordingConfigurationPublisher{},
}, ApplicationOptions{Now: func() time.Time { return now }})
Runtime: runtime,
}, applicationTestOptions(func() time.Time { return now }))
if err != nil {
t.Fatalf("NewApplicationService() error = %v", err)
}
@ -53,6 +62,34 @@ func TestApplicationServiceMapsUpstreamMutationToAdminState(t *testing.T) {
}) {
t.Fatalf("admin state command = %+v", state.lastUpstream)
}
if runtime.notifications != 1 {
t.Fatalf("runtime notifications = %d, want 1", runtime.notifications)
}
}
func TestApplicationServicePreflightsProviderRuntimeBeforeMutation(t *testing.T) {
t.Parallel()
wantErr := errors.New("invalid Provider template")
state := &recordingAdminState{}
runtime := &recordingRuntimeNotifier{validationErr: wantErr}
service, err := NewApplicationService(ApplicationDependencies{
State: state, Operations: staticOperationalStatusReader{},
Configuration: staticConfigurationLoader{}, Publisher: &recordingConfigurationPublisher{},
Runtime: runtime,
}, applicationTestOptions(time.Now))
if err != nil {
t.Fatalf("NewApplicationService(): %v", err)
}
_, err = service.SetUpstreamEnabled(context.Background(), SetUpstreamCommand{
RequestID: "req-enable", Name: "provider-a", Enabled: true,
})
if !errors.Is(err, ErrInvalidConfiguration) || !errors.Is(err, wantErr) {
t.Fatalf("SetUpstreamEnabled() error = %v", err)
}
if state.lastUpstream != (adminstate.SetUpstreamCommand{}) {
t.Fatalf("state mutated before runtime preflight: %+v", state.lastUpstream)
}
}
func TestNewApplicationServiceRejectsMissingDependencies(t *testing.T) {
@ -68,11 +105,12 @@ func TestNewApplicationServiceRejectsMissingDependencies(t *testing.T) {
dependencies ApplicationDependencies
options ApplicationOptions
}{
{name: "state", dependencies: func() ApplicationDependencies { value := valid; value.State = nil; return value }(), options: ApplicationOptions{Now: time.Now}},
{name: "operations", dependencies: func() ApplicationDependencies { value := valid; value.Operations = nil; return value }(), options: ApplicationOptions{Now: time.Now}},
{name: "configuration", dependencies: func() ApplicationDependencies { value := valid; value.Configuration = nil; return value }(), options: ApplicationOptions{Now: time.Now}},
{name: "publisher", dependencies: func() ApplicationDependencies { value := valid; value.Publisher = nil; return value }(), options: ApplicationOptions{Now: time.Now}},
{name: "state", dependencies: func() ApplicationDependencies { value := valid; value.State = nil; return value }(), options: applicationTestOptions(time.Now)},
{name: "operations", dependencies: func() ApplicationDependencies { value := valid; value.Operations = nil; return value }(), options: applicationTestOptions(time.Now)},
{name: "configuration", dependencies: func() ApplicationDependencies { value := valid; value.Configuration = nil; return value }(), options: applicationTestOptions(time.Now)},
{name: "publisher", dependencies: func() ApplicationDependencies { value := valid; value.Publisher = nil; return value }(), options: applicationTestOptions(time.Now)},
{name: "clock", dependencies: valid},
{name: "fingerprint key", dependencies: valid, options: ApplicationOptions{Now: time.Now}},
}
for _, test := range tests {
test := test
@ -99,7 +137,7 @@ func TestNewApplicationServiceRejectsTypedNilDependencies(t *testing.T) {
func() ApplicationDependencies { value := valid; value.State = state; return value }(),
func() ApplicationDependencies { value := valid; value.Publisher = publisher; return value }(),
} {
if _, err := NewApplicationService(dependencies, ApplicationOptions{Now: time.Now}); !errors.Is(err, ErrInvalidApplicationService) {
if _, err := NewApplicationService(dependencies, applicationTestOptions(time.Now)); !errors.Is(err, ErrInvalidApplicationService) {
t.Fatalf("NewApplicationService(typed nil) error = %v, want %v", err, ErrInvalidApplicationService)
}
}
@ -109,7 +147,7 @@ func TestApplicationServiceMapsRoutingSwitchAndDomainErrors(t *testing.T) {
t.Parallel()
now := time.Date(2026, 7, 29, 11, 0, 0, 0, time.FixedZone("test", 8*60*60))
state := &recordingAdminState{mutation: adminstate.MutationResult{RequestID: "req-switch", Changed: true, Revision: 21}}
service := mustApplicationService(t, state, ApplicationOptions{Now: func() time.Time { return now }})
service := mustApplicationService(t, state, applicationTestOptions(func() time.Time { return now }))
result, err := service.SwitchRouting(context.Background(), SwitchCommand{
RequestID: "req-switch", ActorID: "admin:bob", SourceIP: "198.51.100.7",
@ -174,7 +212,7 @@ func TestApplicationServiceBuildsStatusFromAuthoritativeAndOperationalSnapshots(
service, err := NewApplicationService(ApplicationDependencies{
State: state, Operations: operations, Configuration: staticConfigurationLoader{},
Publisher: &recordingConfigurationPublisher{},
}, ApplicationOptions{Now: time.Now})
}, applicationTestOptions(time.Now))
if err != nil {
t.Fatalf("NewApplicationService() error = %v", err)
}
@ -205,7 +243,7 @@ func TestApplicationServiceMapsStatusDependencyFailures(t *testing.T) {
service, err := NewApplicationService(ApplicationDependencies{
State: &recordingAdminState{err: stateFailure}, Operations: staticOperationalStatusReader{},
Configuration: staticConfigurationLoader{}, Publisher: &recordingConfigurationPublisher{},
}, ApplicationOptions{Now: time.Now})
}, applicationTestOptions(time.Now))
if err != nil {
t.Fatalf("NewApplicationService() error = %v", err)
}
@ -217,7 +255,7 @@ func TestApplicationServiceMapsStatusDependencyFailures(t *testing.T) {
service, err = NewApplicationService(ApplicationDependencies{
State: &recordingAdminState{}, Operations: staticOperationalStatusReader{err: operationsFailure},
Configuration: staticConfigurationLoader{}, Publisher: &recordingConfigurationPublisher{},
}, ApplicationOptions{Now: time.Now})
}, applicationTestOptions(time.Now))
if err != nil {
t.Fatalf("NewApplicationService() error = %v", err)
}
@ -228,7 +266,7 @@ func TestApplicationServiceMapsStatusDependencyFailures(t *testing.T) {
service, err = NewApplicationService(ApplicationDependencies{
State: &recordingAdminState{}, Operations: staticOperationalStatusReader{err: context.Canceled},
Configuration: staticConfigurationLoader{}, Publisher: &recordingConfigurationPublisher{},
}, ApplicationOptions{Now: time.Now})
}, applicationTestOptions(time.Now))
if err != nil {
t.Fatalf("NewApplicationService() error = %v", err)
}
@ -242,6 +280,7 @@ func TestApplicationServiceReloadPersistsManagementViewBeforePublishing(t *testi
now := time.Date(2026, 7, 29, 12, 0, 0, 0, time.UTC)
configuration := validReloadConfiguration()
publisher := &recordingConfigurationPublisher{}
runtime := &recordingRuntimeNotifier{}
state := &recordingAdminState{
mutation: adminstate.MutationResult{RequestID: "req-reload", Changed: true, Revision: 42},
snapshot: adminstate.Snapshot{Routings: []adminstate.RoutingState{
@ -258,8 +297,8 @@ func TestApplicationServiceReloadPersistsManagementViewBeforePublishing(t *testi
Configuration: staticConfigurationLoader{loaded: LoadedConfiguration{
Value: configuration, Source: "configs/proxy-pool.yaml",
}},
Publisher: publisher,
}, ApplicationOptions{Now: func() time.Time { return now }})
Publisher: publisher, Runtime: runtime,
}, applicationTestOptions(func() time.Time { return now }))
if err != nil {
t.Fatalf("NewApplicationService() error = %v", err)
}
@ -276,6 +315,9 @@ func TestApplicationServiceReloadPersistsManagementViewBeforePublishing(t *testi
if len(publisher.published) != 1 || publisher.published[0] != configuration {
t.Fatalf("published configurations = %+v", publisher.published)
}
if runtime.notifications != 1 {
t.Fatalf("runtime notifications = %d, want 1", runtime.notifications)
}
command := state.lastConfig
if command.RequestID != "req-reload" || command.Actor != (adminstate.Actor{ID: "admin:alice", SourceIP: "192.0.2.10"}) ||
!command.OccurredAt.Equal(now) || command.Source != "configs/proxy-pool.yaml" {
@ -306,7 +348,7 @@ func TestApplicationServiceApplyConfigurationUsesProvidedSnapshotWithoutReloadin
Operations: staticOperationalStatusReader{},
Configuration: forbiddenConfigurationLoader{},
Publisher: publisher,
}, ApplicationOptions{Now: func() time.Time { return now }})
}, applicationTestOptions(func() time.Time { return now }))
if err != nil {
t.Fatalf("NewApplicationService() error = %v", err)
}
@ -355,7 +397,7 @@ func TestApplicationServiceReloadDoesNotPublishInvalidOrUncommittedConfiguration
service, err := NewApplicationService(ApplicationDependencies{
State: state, Operations: staticOperationalStatusReader{},
Configuration: staticConfigurationLoader{loaded: test.loaded}, Publisher: publisher,
}, ApplicationOptions{Now: time.Now})
}, applicationTestOptions(time.Now))
if err != nil {
t.Fatalf("NewApplicationService() error = %v", err)
}
@ -381,7 +423,7 @@ func TestApplicationServiceReloadPublishesSuccessfulReplay(t *testing.T) {
Configuration: staticConfigurationLoader{loaded: LoadedConfiguration{
Value: validReloadConfiguration(), Source: "config.yaml",
}}, Publisher: publisher,
}, ApplicationOptions{Now: time.Now})
}, applicationTestOptions(time.Now))
if err != nil {
t.Fatalf("NewApplicationService() error = %v", err)
}
@ -393,13 +435,82 @@ func TestApplicationServiceReloadPublishesSuccessfulReplay(t *testing.T) {
}
}
func TestApplicationServiceRejectsSuccessfulCommitWithoutRevision(t *testing.T) {
publisher := &recordingConfigurationPublisher{}
service, err := NewApplicationService(ApplicationDependencies{
State: &recordingAdminState{}, Operations: staticOperationalStatusReader{},
Configuration: staticConfigurationLoader{loaded: LoadedConfiguration{
Value: validReloadConfiguration(), Source: "config.yaml",
}}, Publisher: publisher,
}, applicationTestOptions(time.Now))
if err != nil {
t.Fatalf("NewApplicationService(): %v", err)
}
_, err = service.ReloadConfiguration(context.Background(), ReloadCommand{RequestID: "req-zero-revision"})
if !errors.Is(err, ErrUnavailable) {
t.Fatalf("ReloadConfiguration() error = %v, want unavailable", err)
}
if len(publisher.published) != 0 {
t.Fatalf("published configurations = %d, want 0", len(publisher.published))
}
}
func TestApplicationServiceKeepsNewestConfigurationWhenOlderCommitReturnsLater(t *testing.T) {
store, err := config.NewStore(configWithOnlyUpstream("provider-a"))
if err != nil {
t.Fatalf("config.NewStore(): %v", err)
}
state := &orderedCommitState{
firstCommitted: make(chan struct{}),
releaseFirst: make(chan struct{}),
}
service, err := NewApplicationService(ApplicationDependencies{
State: state, Operations: staticOperationalStatusReader{},
Configuration: staticConfigurationLoader{}, Publisher: store,
}, applicationTestOptions(time.Now))
if err != nil {
t.Fatalf("NewApplicationService(): %v", err)
}
firstDone := make(chan error, 1)
go func() {
_, applyErr := service.ApplyConfiguration(context.Background(), ReloadCommand{RequestID: "req-old"}, LoadedConfiguration{
Value: configWithOnlyUpstream("provider-b"), Source: "old.yaml",
})
firstDone <- applyErr
}()
select {
case <-state.firstCommitted:
case <-time.After(time.Second):
t.Fatal("first commit did not reach delayed return")
}
if _, err := service.ApplyConfiguration(context.Background(), ReloadCommand{RequestID: "req-new"}, LoadedConfiguration{
Value: configWithOnlyUpstream("provider-c"), Source: "new.yaml",
}); err != nil {
t.Fatalf("ApplyConfiguration(new): %v", err)
}
close(state.releaseFirst)
if err := <-firstDone; err != nil {
t.Fatalf("ApplyConfiguration(old): %v", err)
}
if got := store.Current().Routing[0].Upstreams[0]; got != "provider-c" {
t.Fatalf("published upstream = %q, want provider-c", got)
}
if got := store.Revision(); got != 2 {
t.Fatalf("published revision = %d, want 2", got)
}
}
func TestApplicationServiceReloadPreservesCancellation(t *testing.T) {
t.Parallel()
service, err := NewApplicationService(ApplicationDependencies{
State: &recordingAdminState{}, Operations: staticOperationalStatusReader{},
Configuration: staticConfigurationLoader{err: context.Canceled},
Publisher: &recordingConfigurationPublisher{},
}, ApplicationOptions{Now: time.Now})
}, applicationTestOptions(time.Now))
if err != nil {
t.Fatalf("NewApplicationService() error = %v", err)
}
@ -409,13 +520,16 @@ func TestApplicationServiceReloadPreservesCancellation(t *testing.T) {
}
}
func TestApplicationServiceUsesSecretFreeManagementChecksum(t *testing.T) {
func TestApplicationServiceUsesOpaqueChecksumThatTracksSecretRotation(t *testing.T) {
t.Parallel()
state := &recordingAdminState{}
publisher := &recordingConfigurationPublisher{}
var commands []adminstate.CommitConfigCommand
state.onCommit = func(command adminstate.CommitConfigCommand) {
commands = append(commands, command)
state.mutation = adminstate.MutationResult{
RequestID: command.RequestID, Changed: true, Revision: uint64(len(commands)),
}
}
for _, secret := range []string{"secret-a", "secret-b"} {
configuration := validReloadConfiguration()
@ -427,7 +541,7 @@ func TestApplicationServiceUsesSecretFreeManagementChecksum(t *testing.T) {
Configuration: staticConfigurationLoader{loaded: LoadedConfiguration{
Value: configuration, Source: "config.yaml",
}}, Publisher: publisher,
}, ApplicationOptions{Now: time.Now})
}, applicationTestOptions(time.Now))
if err != nil {
t.Fatalf("NewApplicationService() error = %v", err)
}
@ -437,8 +551,8 @@ func TestApplicationServiceUsesSecretFreeManagementChecksum(t *testing.T) {
t.Fatalf("ReloadConfiguration() error = %v", err)
}
}
if len(commands) != 2 || commands[0].Checksum != commands[1].Checksum || commands[0].ConfigVersion != commands[1].ConfigVersion {
t.Fatalf("secret rotation changed public management digest: %+v", commands)
if len(commands) != 2 || commands[0].Checksum == commands[1].Checksum || commands[0].ConfigVersion == commands[1].ConfigVersion {
t.Fatalf("secret rotation did not change opaque configuration digest: %+v", commands)
}
if len(publisher.published) != 2 || publisher.published[1].Upstreams["provider-a"].ProxyAuth.Password != "secret-b" {
t.Fatalf("secret rotation was not published: %+v", publisher.published)
@ -467,6 +581,16 @@ func validReloadConfiguration() *config.Config {
}
}
func configWithOnlyUpstream(name string) *config.Config {
configuration := validReloadConfiguration()
configuration.Upstreams = map[string]config.Upstream{name: validReloadUpstream("secret")}
configuration.Routing = []config.Routing{{
Name: "default", Enabled: true, Purpose: "gateway", Upstreams: []string{name},
Strategy: config.Strategy{Type: "random"}, OnUnavailable: config.OnUnavailable{Action: "reject"},
}}
return configuration
}
func validReloadUpstream(secret string) config.Upstream {
return config.Upstream{
Enabled: true, Exposure: []string{"gateway"},
@ -505,6 +629,33 @@ type recordingAdminState struct {
onCommit func(adminstate.CommitConfigCommand)
}
type orderedCommitState struct {
next atomic.Uint64
firstCommitted chan struct{}
releaseFirst chan struct{}
}
func (state *orderedCommitState) SetUpstreamEnabled(context.Context, adminstate.SetUpstreamCommand) (adminstate.MutationResult, error) {
return adminstate.MutationResult{}, nil
}
func (state *orderedCommitState) SwitchRouting(context.Context, adminstate.SwitchRoutingCommand) (adminstate.MutationResult, error) {
return adminstate.MutationResult{}, nil
}
func (state *orderedCommitState) CommitConfig(_ context.Context, command adminstate.CommitConfigCommand) (adminstate.MutationResult, error) {
revision := state.next.Add(1)
if revision == 1 {
close(state.firstCommitted)
<-state.releaseFirst
}
return adminstate.MutationResult{RequestID: command.RequestID, Changed: true, Revision: revision}, nil
}
func (*orderedCommitState) Snapshot(context.Context) (adminstate.Snapshot, error) {
return adminstate.Snapshot{}, nil
}
func (state *recordingAdminState) SetUpstreamEnabled(_ context.Context, command adminstate.SetUpstreamCommand) (adminstate.MutationResult, error) {
state.lastUpstream = command
return state.mutation, state.err
@ -553,8 +704,26 @@ func (loader staticConfigurationLoader) LoadConfiguration(context.Context) (Load
type recordingConfigurationPublisher struct {
published []*config.Config
revisions []uint64
}
func (publisher *recordingConfigurationPublisher) Publish(configuration *config.Config) {
publisher.published = append(publisher.published, configuration)
type recordingRuntimeNotifier struct {
notifications int
validationErr error
}
func (notifier *recordingRuntimeNotifier) Notify() { notifier.notifications++ }
func (notifier *recordingRuntimeNotifier) ValidateConfiguration(context.Context, *config.Config) error {
return notifier.validationErr
}
func (notifier *recordingRuntimeNotifier) ValidateUpstream(context.Context, string) error {
return notifier.validationErr
}
func (publisher *recordingConfigurationPublisher) PublishRevision(configuration *config.Config, revision uint64) bool {
publisher.published = append(publisher.published, configuration)
publisher.revisions = append(publisher.revisions, revision)
return true
}

View File

@ -5,6 +5,7 @@ import (
"errors"
"fmt"
"reflect"
"sort"
"strings"
"time"
@ -15,11 +16,15 @@ import (
"proxy-pool/internal/controller/distribution"
"proxy-pool/internal/controller/extraction"
"proxy-pool/internal/controller/operations"
"proxy-pool/internal/controller/pool"
"proxy-pool/internal/controller/provider"
controllerRuntime "proxy-pool/internal/controller/runtime"
"proxy-pool/internal/domain/activitypool"
extractionDomain "proxy-pool/internal/domain/extraction"
"proxy-pool/internal/platform/admission"
"proxy-pool/internal/platform/credentials"
"proxy-pool/internal/platform/httpserver"
"proxy-pool/internal/platform/lifecycle"
platformMetrics "proxy-pool/internal/platform/metrics"
)
@ -29,14 +34,19 @@ var (
)
type Options struct {
ConfigPath string
Resolver config.Resolver
Now func() time.Time
HTTP httpserver.Options
ConfigPath string
Resolver config.Resolver
Now func() time.Time
HTTP httpserver.Options
HolderID string
RedisNamespace string
FingerprintKey []byte
}
type activityStore interface {
extractionDomain.Store
activitypool.Upserter
pool.InventoryReader
activitypool.StateInventoryReader
}
@ -45,6 +55,9 @@ type ports struct {
activity activityStore
readiness distribution.ReadinessChecker
metricsReadiness platformMetrics.ReadinessChecker
coordinator provider.Coordinator
credentials credentials.Store
providerResults provider.ResultRecorder
close func() error
}
@ -61,7 +74,9 @@ type runtimeFactory interface {
}
func Run(ctx context.Context, options Options) error {
return run(ctx, options, &productionInfrastructure{}, productionRuntimeFactory{})
return run(ctx, options, &productionInfrastructure{
holderID: options.HolderID, namespace: options.RedisNamespace,
}, productionRuntimeFactory{})
}
func run(ctx context.Context, options Options, infrastructure infrastructure, factory runtimeFactory) (resultErr error) {
@ -84,6 +99,9 @@ func run(ctx context.Context, options Options, infrastructure infrastructure, fa
if err != nil {
return fmt.Errorf("%w: load configuration: %w", ErrStartup, err)
}
if loaded.Value.Admin.Enabled && len(options.FingerprintKey) < config.MinimumFingerprintKeyBytes {
return errors.Join(ErrInvalidOptions, config.ErrInvalidFingerprint)
}
configurationStore, err := config.NewStore(loaded.Value)
if err != nil {
return fmt.Errorf("%w: initialize configuration store: %w", ErrStartup, err)
@ -99,6 +117,31 @@ func run(ctx context.Context, options Options, infrastructure infrastructure, fa
defer func() {
resultErr = errors.Join(resultErr, opened.close())
}()
var providerState providerStateReader
if loaded.Value.Admin.Enabled {
providerState = opened.state
}
supervisor, err := newProviderSupervisor(
configurationStore,
providerState,
func(name string, upstream config.Upstream) (lifecycle.Runner, error) {
buildRuntime, err := providerRuntimeBuilder(opened)
if err != nil {
return nil, err
}
return buildRuntime(name, upstream)
},
func(ctx context.Context, configuration *config.Config) error {
return prepareProviderConfiguration(ctx, configuration, opened.credentials)
},
func(configuration *config.Config) { retainProviderStats(configuration, opened.providerResults) },
loader,
options.FingerprintKey,
providerSupervisorInterval,
)
if err != nil {
return fmt.Errorf("%w: build Provider supervisor: %w", ErrStartup, err)
}
dependencies := controllerRuntime.Dependencies{}
if loaded.Value.Distribution.Enabled {
@ -116,13 +159,23 @@ func run(ctx context.Context, options Options, infrastructure infrastructure, fa
if nilInterface(opened.state) || nilInterface(opened.activity) {
return errors.Join(ErrStartup, ErrInvalidOptions)
}
statusReader, statusErr := operations.NewReader(configurationStore, opened.activity, options.Now)
var providerStats []provider.StatsReader
if stats, ok := opened.providerResults.(provider.StatsReader); ok && !nilInterface(stats) {
providerStats = append(providerStats, stats)
}
statusReader, statusErr := operations.NewReader(
configurationStore,
opened.activity,
options.Now,
providerStats...,
)
if statusErr != nil {
return fmt.Errorf("%w: build operational status reader: %w", ErrStartup, statusErr)
}
service, serviceErr := admin.NewApplicationService(admin.ApplicationDependencies{
State: opened.state, Operations: statusReader, Configuration: loader, Publisher: configurationStore,
}, admin.ApplicationOptions{Now: options.Now})
Runtime: supervisor,
}, admin.ApplicationOptions{Now: options.Now, FingerprintKey: options.FingerprintKey})
if serviceErr != nil {
return fmt.Errorf("%w: build admin service: %w", ErrStartup, serviceErr)
}
@ -146,14 +199,68 @@ func run(ctx context.Context, options Options, infrastructure infrastructure, fa
dependencies.MetricsHandler = handler
}
runner, err := factory.New(configurationStore.Current(), dependencies, controllerRuntime.Options{HTTP: options.HTTP})
runners := make([]lifecycle.Runner, 0, 2)
if hasHTTPRuntime(loaded.Value) {
runner, err := factory.New(configurationStore.Current(), dependencies, controllerRuntime.Options{HTTP: options.HTTP})
if err != nil {
return fmt.Errorf("%w: build HTTP runtime: %w", ErrStartup, err)
}
if nilInterface(runner) {
return errors.Join(ErrStartup, ErrInvalidOptions)
}
runners = append(runners, runner)
}
runners = append(runners, supervisor)
group, err := lifecycle.NewGroup(runners...)
if err != nil {
return fmt.Errorf("%w: build HTTP runtime: %w", ErrStartup, err)
return fmt.Errorf("%w: build process lifecycle: %w", ErrStartup, err)
}
if nilInterface(runner) {
return errors.Join(ErrStartup, ErrInvalidOptions)
return group.Run(ctx)
}
func prepareProviderConfiguration(
ctx context.Context,
configuration *config.Config,
credentialStore credentials.Store,
) error {
if ctx == nil || configuration == nil {
return ErrProviderRuntime
}
return runner.Run(ctx)
if err := ctx.Err(); err != nil {
return err
}
if !configuration.Admin.Enabled && !hasEnabledUpstream(configuration) {
return nil
}
ensurer, ok := credentialStore.(credentials.CapacityEnsurer)
if !ok || nilInterface(credentialStore) {
return ErrProviderRuntime
}
if err := ensurer.EnsureCapacity(ctx, providerCredentialCapacity(configuration)); err != nil {
return err
}
return nil
}
func retainProviderStats(configuration *config.Config, results provider.ResultRecorder) {
if configuration == nil {
return
}
retainer, ok := results.(provider.StatsRetainer)
if !ok || nilInterface(retainer) {
return
}
names := make([]string, 0, len(configuration.Upstreams))
for name := range configuration.Upstreams {
names = append(names, name)
}
sort.Strings(names)
retainer.RetainProviderStats(names)
}
func hasHTTPRuntime(configuration *config.Config) bool {
return configuration != nil &&
(configuration.Distribution.Enabled || configuration.Admin.Enabled || configuration.Metrics.Enabled)
}
func extractionPolicy(configuration *config.Config) extraction.Policy {

View File

@ -4,16 +4,25 @@ package bootstrap
import (
"context"
"errors"
"net/http"
"net/http/httptest"
"os"
"strconv"
"strings"
"sync/atomic"
"testing"
"time"
"github.com/redis/go-redis/v9"
"proxy-pool/internal/adapters/redisactivity"
"proxy-pool/internal/config"
"proxy-pool/internal/controller/admin"
"proxy-pool/internal/controller/pool"
"proxy-pool/internal/controller/provider"
controllerRuntime "proxy-pool/internal/controller/runtime"
"proxy-pool/internal/platform/credentials"
)
func TestProductionBootstrapOpensBothStoresAndCommitsStartupConfiguration(t *testing.T) {
@ -22,18 +31,38 @@ func TestProductionBootstrapOpensBothStoresAndCommitsStartupConfiguration(t *tes
if postgresURL == "" || redisURL == "" {
t.Skip("PROXY_POOL_TEST_POSTGRES_URL and PROXY_POOL_TEST_REDIS_URL are required")
}
var providerCalls atomic.Int64
providerServer := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, _ *http.Request) {
providerCalls.Add(1)
_, _ = writer.Write([]byte("http://192.0.2.10:8080"))
}))
defer providerServer.Close()
namespace := "controller-it-" + strconv.FormatInt(time.Now().UnixNano(), 10)
inventory := newIntegrationInventoryReader(t, redisURL, namespace)
source := strings.ReplaceAll(bootstrapTestConfig, "postgres://fixture", postgresURL)
source = strings.ReplaceAll(source, "redis://fixture", redisURL)
source = strings.ReplaceAll(source, "https://provider.invalid/proxies", providerServer.URL)
resolver := &memoryResolver{files: map[string][]byte{"controller.yaml": []byte(source)}}
factory := &integrationRuntimeFactory{}
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
providerResults := make(chan provider.Result, 8)
factory := &integrationRuntimeFactory{
cancel: cancel, inventory: inventory, providerResults: providerResults,
}
infrastructure := &integrationInfrastructure{
productionInfrastructure: productionInfrastructure{namespace: namespace},
results: providerResultRecorder(func(result provider.Result) { providerResults <- result }),
}
err := run(context.Background(), Options{
err := run(ctx, Options{
ConfigPath: "controller.yaml", Resolver: resolver, Now: func() time.Time {
return time.Date(2026, 7, 30, 13, 0, 0, 0, time.UTC)
},
}, &productionInfrastructure{}, factory)
if err != nil {
t.Fatalf("run() error = %v", err)
FingerprintKey: bootstrapTestFingerprintKey,
}, infrastructure, factory)
if !errors.Is(err, context.Canceled) {
t.Fatalf("run() error = %v, want context cancellation", err)
}
if factory.status.ConfigVersion == "" || len(factory.status.Upstreams) != 2 {
t.Fatalf("Admin Status = %+v", factory.status)
@ -46,13 +75,19 @@ func TestProductionBootstrapOpensBothStoresAndCommitsStartupConfiguration(t *tes
!strings.Contains(factory.metricsBody, "go_") {
t.Fatalf("Metrics probes = ready:%d metrics:%d body:%q", factory.readyStatus, factory.metricsStatus, factory.metricsBody)
}
if providerCalls.Load() == 0 {
t.Fatal("production Provider HTTP adapter was not called")
}
}
type integrationRuntimeFactory struct {
status admin.Status
readyStatus int
metricsStatus int
metricsBody string
status admin.Status
readyStatus int
metricsStatus int
metricsBody string
cancel context.CancelFunc
inventory pool.InventoryReader
providerResults <-chan provider.Result
}
func (factory *integrationRuntimeFactory) New(
@ -73,10 +108,99 @@ func (factory *integrationRuntimeFactory) New(
factory.metricsBody = metrics.Body.String()
status, err := dependencies.AdminService.Status(ctx)
factory.status = status
return err
if err != nil {
return err
}
if err := waitForProviderInventory(
ctx,
factory.inventory,
factory.providerResults,
[]string{"provider-a", "provider-b"},
); err != nil {
return err
}
factory.cancel()
<-ctx.Done()
return ctx.Err()
}}, nil
}
func newIntegrationInventoryReader(t *testing.T, redisURL, namespace string) pool.InventoryReader {
t.Helper()
options, err := redis.ParseURL(redisURL)
if err != nil {
t.Fatalf("redis.ParseURL(): %v", err)
}
client := redis.NewClient(options)
t.Cleanup(func() { _ = client.Close() })
credentialStore, err := credentials.NewMemoryStore(200)
if err != nil {
t.Fatalf("credentials.NewMemoryStore(): %v", err)
}
reader, err := redisactivity.New(client, redisactivity.Options{
Namespace: namespace, Credentials: credentialStore,
OperationTTL: redisOperationTTL, MaxCandidateScan: redisMinimumScan,
MaxRuntimeCounters: 200, MaxInventoryScan: 200, CleanupLimit: redisCleanupLimit,
})
if err != nil {
t.Fatalf("redisactivity.New(): %v", err)
}
return reader
}
func waitForProviderInventory(
ctx context.Context,
inventory pool.InventoryReader,
results <-chan provider.Result,
upstreamIDs []string,
) error {
deadline, cancel := context.WithTimeout(ctx, 5*time.Second)
defer cancel()
ticker := time.NewTicker(10 * time.Millisecond)
defer ticker.Stop()
var latestResult provider.Result
for {
var inventoryErr error
for _, upstreamID := range upstreamIDs {
snapshot, err := inventory.ReadInventory(deadline, upstreamID, 0)
inventoryErr = errors.Join(inventoryErr, err)
if err == nil && snapshot.Managed > 0 {
return nil
}
}
select {
case <-deadline.Done():
return errors.Join(
errors.New("wait for Provider Redis inventory"),
deadline.Err(),
inventoryErr,
latestResult.Err,
)
case result := <-results:
latestResult = result
case <-ticker.C:
}
}
}
type integrationInfrastructure struct {
productionInfrastructure
results provider.ResultRecorder
}
func (infrastructure *integrationInfrastructure) Open(
ctx context.Context,
configuration *config.Config,
) (ports, error) {
opened, err := infrastructure.productionInfrastructure.Open(ctx, configuration)
opened.providerResults = infrastructure.results
return opened, err
}
type providerResultRecorder func(provider.Result)
func (record providerResultRecorder) Record(result provider.Result) { record(result) }
type integrationRunner struct {
run func(context.Context) error
}

View File

@ -3,32 +3,48 @@ package bootstrap
import (
"context"
"errors"
"strings"
"testing"
"time"
"proxy-pool/internal/config"
"proxy-pool/internal/controller/admin"
"proxy-pool/internal/controller/pool"
"proxy-pool/internal/controller/provider"
controllerRuntime "proxy-pool/internal/controller/runtime"
"proxy-pool/internal/domain/activitypool"
"proxy-pool/internal/domain/adminstate"
extractionDomain "proxy-pool/internal/domain/extraction"
"proxy-pool/internal/domain/upstream"
"proxy-pool/internal/platform/credentials"
)
var bootstrapTestFingerprintKey = []byte("0123456789abcdef0123456789abcdef")
func TestRunLoadsOneSnapshotCommitsItAndClosesInfrastructure(t *testing.T) {
t.Parallel()
resolver := &memoryResolver{files: map[string][]byte{"controller.yaml": []byte(bootstrapTestConfig)}}
state := adminstate.NewMemoryStore()
activity := &stubActivityStore{}
credentialStore, err := credentials.NewMemoryStore(10)
if err != nil {
t.Fatalf("NewMemoryStore(): %v", err)
}
closeErr := errors.New("close failed")
infrastructure := &stubInfrastructure{ports: ports{
state: state, activity: activity, readiness: readyStub{}, metricsReadiness: readyStub{},
coordinator: coordinatorStub{}, credentials: credentialStore,
close: func() error { return closeErr },
}}
runErr := errors.New("runtime failed")
factory := &recordingRuntimeFactory{runner: runnerStub{err: runErr}}
now := time.Date(2026, 7, 30, 11, 0, 0, 0, time.UTC)
err := run(context.Background(), Options{
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
defer cancel()
err = run(ctx, Options{
ConfigPath: "controller.yaml", Resolver: resolver, Now: func() time.Time { return now },
FingerprintKey: bootstrapTestFingerprintKey,
}, infrastructure, factory)
if !errors.Is(err, runErr) || !errors.Is(err, closeErr) {
t.Fatalf("run() error = %v, want runtime and close errors", err)
@ -74,6 +90,147 @@ func TestRunRejectsInvalidOptionsBeforeIO(t *testing.T) {
}
}
func TestRunRejectsMissingAdminFingerprintKeyBeforeOpeningInfrastructure(t *testing.T) {
for name, key := range map[string][]byte{
"missing": nil,
"short": []byte("too-short"),
} {
t.Run(name, func(t *testing.T) {
infrastructure := &stubInfrastructure{}
err := run(context.Background(), Options{
ConfigPath: "controller.yaml",
Resolver: &memoryResolver{files: map[string][]byte{"controller.yaml": []byte(bootstrapTestConfig)}},
Now: time.Now,
FingerprintKey: key,
}, infrastructure, &recordingRuntimeFactory{})
if !errors.Is(err, ErrInvalidOptions) || !errors.Is(err, config.ErrInvalidFingerprint) {
t.Fatalf("run() error = %v", err)
}
if infrastructure.opens != 0 {
t.Fatalf("infrastructure opens = %d, want 0", infrastructure.opens)
}
})
}
}
func TestRunSupportsProviderOnlyConfigurationWithoutHTTPRuntime(t *testing.T) {
source := strings.ReplaceAll(bootstrapTestConfig, "distribution:\n enabled: true", "distribution:\n enabled: false")
source = strings.ReplaceAll(source, "admin:\n enabled: true", "admin:\n enabled: false")
source = strings.ReplaceAll(source, "metrics:\n enabled: true", "metrics:\n enabled: false")
credentialStore, err := credentials.NewMemoryStore(10)
if err != nil {
t.Fatalf("NewMemoryStore(): %v", err)
}
infrastructure := &stubInfrastructure{ports: ports{
activity: &stubActivityStore{}, coordinator: coordinatorStub{}, credentials: credentialStore,
close: func() error { return nil },
}}
factory := &recordingRuntimeFactory{}
ctx, cancel := context.WithCancel(context.Background())
time.AfterFunc(20*time.Millisecond, cancel)
err = run(ctx, Options{
ConfigPath: "controller.yaml",
Resolver: &memoryResolver{files: map[string][]byte{"controller.yaml": []byte(source)}},
Now: time.Now,
}, infrastructure, factory)
if !errors.Is(err, context.Canceled) {
t.Fatalf("run(provider only) error = %v, want context cancellation", err)
}
if factory.configuration != nil {
t.Fatal("HTTP runtime factory was called for Provider-only configuration")
}
}
func TestRunAdminDisableStopsActiveProviderRuntime(t *testing.T) {
state := adminstate.NewMemoryStore()
credentialStore, err := credentials.NewMemoryStore(200)
if err != nil {
t.Fatalf("NewMemoryStore(): %v", err)
}
started := make(chan string, 2)
stopped := make(chan string, 2)
infrastructure := &stubInfrastructure{ports: ports{
state: state, activity: &stubActivityStore{}, readiness: readyStub{}, metricsReadiness: readyStub{},
coordinator: coordinatorFunc(func(ctx context.Context, upstreamID string) error {
started <- upstreamID
<-ctx.Done()
stopped <- upstreamID
return ctx.Err()
}),
credentials: credentialStore,
close: func() error { return nil },
}}
wantErr := errors.New("test HTTP runtime stopped")
factory := runtimeFactoryFunc(func(
_ *config.Config,
dependencies controllerRuntime.Dependencies,
_ controllerRuntime.Options,
) (controllerRunner, error) {
return runnerFunc(func(ctx context.Context) error {
for {
select {
case upstreamID := <-started:
if upstreamID != "provider-a" {
continue
}
if _, err := dependencies.AdminService.SetUpstreamEnabled(ctx, admin.SetUpstreamCommand{
RequestID: "req-disable", ActorID: "admin:test", Name: "provider-a", Enabled: false,
}); err != nil {
return err
}
for {
select {
case stoppedID := <-stopped:
if stoppedID == "provider-a" {
return wantErr
}
case <-ctx.Done():
return ctx.Err()
}
}
case <-ctx.Done():
return ctx.Err()
}
}
}), nil
})
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
defer cancel()
err = run(ctx, Options{
ConfigPath: "controller.yaml",
Resolver: &memoryResolver{files: map[string][]byte{"controller.yaml": []byte(bootstrapTestConfig)}},
Now: time.Now,
FingerprintKey: bootstrapTestFingerprintKey,
}, infrastructure, factory)
if !errors.Is(err, wantErr) {
t.Fatalf("run() error = %v, want %v", err, wantErr)
}
}
func TestRetainProviderStatsKeepsAllConfiguredProviders(t *testing.T) {
configuration, err := config.Load(strings.NewReader(bootstrapTestConfig))
if err != nil {
t.Fatalf("config.Load(): %v", err)
}
disabled := configuration.Upstreams["provider-b"]
disabled.Enabled = false
configuration.Upstreams["provider-b"] = disabled
stats, err := provider.NewStatsRecorder(2)
if err != nil {
t.Fatalf("provider.NewStatsRecorder(): %v", err)
}
stats.Record(provider.Result{UpstreamID: "removed", Class: upstream.FetchError})
stats.Record(provider.Result{UpstreamID: "provider-b", Class: upstream.FetchError})
retainProviderStats(configuration, stats)
stats.Record(provider.Result{UpstreamID: "provider-a", Class: upstream.FetchError})
got := stats.ReadProviderStats([]string{"removed", "provider-a", "provider-b"})
if got[0].FetchErrorCount != 0 || got[1].FetchErrorCount != 1 || got[2].FetchErrorCount != 1 {
t.Fatalf("Provider stats after retention = %+v", got)
}
}
type memoryResolver struct {
files map[string][]byte
reads int
@ -127,6 +284,24 @@ type runnerStub struct{ err error }
func (runner runnerStub) Run(context.Context) error { return runner.err }
type runnerFunc func(context.Context) error
func (run runnerFunc) Run(ctx context.Context) error { return run(ctx) }
type runtimeFactoryFunc func(
*config.Config,
controllerRuntime.Dependencies,
controllerRuntime.Options,
) (controllerRunner, error)
func (factory runtimeFactoryFunc) New(
configuration *config.Config,
dependencies controllerRuntime.Dependencies,
options controllerRuntime.Options,
) (controllerRunner, error) {
return factory(configuration, dependencies, options)
}
type readyStub struct{}
func (readyStub) Ready(context.Context) error { return nil }
@ -149,6 +324,45 @@ func (*stubActivityStore) ReadStateInventory(
return result, nil
}
func (*stubActivityStore) UpsertFetched(
_ context.Context,
_ string,
batch activitypool.FetchedBatch,
) (activitypool.UpsertResult, error) {
return activitypool.UpsertResult{Accepted: len(batch.Proxies), Inserted: len(batch.Proxies)}, nil
}
func (*stubActivityStore) ReadInventory(
context.Context,
string,
time.Duration,
) (pool.InventorySnapshot, error) {
return pool.InventorySnapshot{Managed: 100, AvailableSlots: 1_000}, nil
}
type coordinatorStub struct{}
func (coordinatorStub) RunLeader(
ctx context.Context,
_ string,
_ provider.CoordinationLimits,
_ func(context.Context, provider.LeaderSession) error,
) error {
<-ctx.Done()
return ctx.Err()
}
type coordinatorFunc func(context.Context, string) error
func (run coordinatorFunc) RunLeader(
ctx context.Context,
upstreamID string,
_ provider.CoordinationLimits,
_ func(context.Context, provider.LeaderSession) error,
) error {
return run(ctx, upstreamID)
}
const bootstrapTestConfig = `
version: 1
security:

View File

@ -2,6 +2,8 @@ package bootstrap
import (
"context"
"crypto/rand"
"encoding/hex"
"errors"
"strings"
"time"
@ -11,16 +13,23 @@ import (
"proxy-pool/internal/adapters/postgresadmin"
"proxy-pool/internal/adapters/redisactivity"
"proxy-pool/internal/adapters/redisprovider"
"proxy-pool/internal/config"
controllerProvider "proxy-pool/internal/controller/provider"
"proxy-pool/internal/platform/credentials"
platformMetrics "proxy-pool/internal/platform/metrics"
)
const (
redisNamespace = "controller"
redisOperationTTL = 30 * time.Second
redisMinimumScan = 4_096
redisCleanupLimit = 1_024
redisNamespace = "controller"
redisOperationTTL = 30 * time.Second
redisMinimumScan = 4_096
redisMaximumScan = config.MaximumPoolSize
redisCleanupLimit = 1_024
providerLeaseTTL = 15 * time.Second
providerRenewEvery = 3 * time.Second
providerRetryInterval = 100 * time.Millisecond
providerPermitGrace = 5 * time.Second
)
var (
@ -30,15 +39,22 @@ var (
ErrRedisUnavailable = errors.New("Redis unavailable")
)
type productionInfrastructure struct{}
type productionInfrastructure struct {
holderID string
namespace string
}
func (*productionInfrastructure) Open(
func (infrastructure *productionInfrastructure) Open(
ctx context.Context,
configuration *config.Config,
) (_ ports, resultErr error) {
if ctx == nil || configuration == nil {
return ports{}, ErrInvalidOptions
}
namespace, err := resolveRedisNamespace(infrastructure.namespace)
if err != nil {
return ports{}, err
}
var postgresPool *pgxpool.Pool
var redisClient *redis.Client
closeResources := func() error {
@ -83,7 +99,8 @@ func (*productionInfrastructure) Open(
}
}
if configuration.Distribution.Enabled || configuration.Admin.Enabled {
providersEnabled := hasEnabledUpstream(configuration)
if configuration.Distribution.Enabled || configuration.Admin.Enabled || providersEnabled {
if strings.TrimSpace(configuration.Storage.RedisURL) == "" {
return ports{}, ErrRedisConfiguration
}
@ -95,22 +112,44 @@ func (*productionInfrastructure) Open(
if err = redisClient.Ping(ctx).Err(); err != nil {
return ports{}, contextOr(ctx, ErrRedisUnavailable)
}
credentialStore, err := credentials.NewMemoryStore(credentialCapacity(configuration))
credentialStore, err := credentials.NewMemoryStore(providerCredentialCapacity(configuration))
if err != nil {
return ports{}, err
}
adapter, err := redisactivity.New(redisClient, redisactivity.Options{
Namespace: redisNamespace,
Credentials: credentialStore,
OperationTTL: redisOperationTTL,
MaxCandidateScan: candidateScan(configuration),
CleanupLimit: redisCleanupLimit,
Namespace: namespace,
Credentials: credentialStore,
OperationTTL: redisOperationTTL,
MaxCandidateScan: candidateScan(configuration),
MaxRuntimeCounters: credentialCapacity(configuration),
MaxInventoryScan: maxInventoryScan(configuration),
CleanupLimit: redisCleanupLimit,
})
if err != nil {
return ports{}, err
}
opened.activity = adapter
opened.readiness = redisReadiness{client: redisClient}
opened.credentials = credentialStore
if providersEnabled {
stats, statsErr := controllerProvider.NewStatsRecorder(config.MaximumUpstreams)
if statsErr != nil {
return ports{}, statsErr
}
opened.providerResults = stats
holderID, holderErr := resolveHolderID(infrastructure.holderID)
if holderErr != nil {
return ports{}, holderErr
}
opened.coordinator, err = redisprovider.New(redisClient, redisprovider.Options{
Namespace: namespace, HolderID: holderID,
LeaseTTL: providerLeaseTTL, RenewEvery: providerRenewEvery,
RetryInterval: providerRetryInterval, PermitGrace: providerPermitGrace,
})
if err != nil {
return ports{}, err
}
}
}
if configuration.Metrics.Enabled {
opened.metricsReadiness = selectMetricsReadiness(
@ -122,11 +161,21 @@ func (*productionInfrastructure) Open(
return opened, nil
}
func resolveRedisNamespace(configured string) (string, error) {
if strings.TrimSpace(configured) != configured {
return "", ErrInvalidOptions
}
if configured == "" {
return redisNamespace, nil
}
return configured, nil
}
func selectMetricsReadiness(
configuration *config.Config,
admin, activity platformMetrics.ReadinessChecker,
) platformMetrics.ReadinessChecker {
if configuration.Distribution.Enabled {
if configuration.Distribution.Enabled || hasEnabledUpstream(configuration) {
return activity
}
if configuration.Admin.Enabled {
@ -184,7 +233,7 @@ func credentialCapacity(configuration *config.Config) int {
capacity := 0
maximum := int(^uint(0) >> 1)
for _, upstream := range configuration.Upstreams {
if upstream.Pool.MaxSize <= 0 {
if !upstream.Enabled || upstream.Pool.MaxSize <= 0 {
continue
}
if capacity > maximum-upstream.Pool.MaxSize {
@ -198,6 +247,62 @@ func credentialCapacity(configuration *config.Config) int {
return capacity
}
func providerCredentialCapacity(configuration *config.Config) int {
capacity := 0
maximum := int(^uint(0) >> 1)
for _, upstream := range configuration.Upstreams {
if upstream.Pool.MaxSize <= 0 {
continue
}
maxInFlight := upstream.Fetch.MaxInFlight
if maxInFlight <= 0 {
maxInFlight = 1
}
if upstream.Pool.MaxSize > maximum/maxInFlight {
return maximum
}
leases := upstream.Pool.MaxSize * maxInFlight
if capacity > maximum-leases {
return maximum
}
capacity += leases
}
if capacity == 0 {
return 1
}
return capacity
}
func maxInventoryScan(_ *config.Config) int {
return config.MaximumPoolSize
}
func hasEnabledUpstream(configuration *config.Config) bool {
if configuration == nil {
return false
}
for _, upstream := range configuration.Upstreams {
if upstream.Enabled {
return true
}
}
return false
}
func resolveHolderID(configured string) (string, error) {
if strings.TrimSpace(configured) != configured {
return "", ErrInvalidOptions
}
if configured != "" {
return configured, nil
}
var entropy [16]byte
if _, err := rand.Read(entropy[:]); err != nil {
return "", errors.Join(ErrStartup, err)
}
return "controller-" + hex.EncodeToString(entropy[:]), nil
}
func candidateScan(configuration *config.Config) int {
configured := configuration.Distribution.Extraction
if configured.MaxCountPerRequest > int(^uint(0)>>1)-configured.ReserveForGateway {

View File

@ -79,13 +79,17 @@ func TestProductionRedisSizingUsesConfigurationBounds(t *testing.T) {
MaxCountPerRequest: 100, ReserveForGateway: 5_000,
}},
Upstreams: map[string]config.Upstream{
"provider-a": {Pool: config.Pool{MaxSize: 3_000}},
"provider-b": {Pool: config.Pool{MaxSize: 2_000}},
"provider-a": {Enabled: true, Pool: config.Pool{MaxSize: 3_000}, Fetch: config.Fetch{MaxInFlight: 2}},
"provider-b": {Enabled: true, Pool: config.Pool{MaxSize: 2_000}, Fetch: config.Fetch{MaxInFlight: 1}},
"disabled": {Pool: config.Pool{MaxSize: 50_000}, Fetch: config.Fetch{MaxInFlight: 3}},
},
}
if got := credentialCapacity(configuration); got != 5_000 {
t.Fatalf("credentialCapacity() = %d, want 5000", got)
}
if got := providerCredentialCapacity(configuration); got != 158_000 {
t.Fatalf("providerCredentialCapacity() = %d, want 158000", got)
}
if got := candidateScan(configuration); got != 5_100 {
t.Fatalf("candidateScan() = %d, want 5100", got)
}
@ -94,3 +98,14 @@ func TestProductionRedisSizingUsesConfigurationBounds(t *testing.T) {
t.Fatalf("candidateScan(minimum) = %d, want %d", got, redisMinimumScan)
}
}
func TestMaxInventoryScanSupportsPoolGrowthAfterReload(t *testing.T) {
t.Parallel()
configuration := &config.Config{Upstreams: map[string]config.Upstream{
"provider-a": {Enabled: true, Pool: config.Pool{MaxSize: 100}},
}}
if got := maxInventoryScan(configuration); got != config.MaximumPoolSize {
t.Fatalf("maxInventoryScan(initial small pool) = %d, want %d", got, config.MaximumPoolSize)
}
}

View File

@ -0,0 +1,126 @@
package bootstrap
import (
"errors"
"math"
"sort"
"strings"
"time"
"proxy-pool/internal/adapters/providerapi"
"proxy-pool/internal/config"
"proxy-pool/internal/controller/pool"
controllerProvider "proxy-pool/internal/controller/provider"
)
var ErrProviderRuntime = errors.New("invalid Provider runtime configuration")
func newProviderFleet(configuration *config.Config, opened ports) (*controllerProvider.Fleet, error) {
if configuration == nil {
return nil, ErrProviderRuntime
}
names := make([]string, 0, len(configuration.Upstreams))
for name, upstream := range configuration.Upstreams {
if upstream.Enabled {
names = append(names, name)
}
}
if len(names) == 0 {
return nil, nil
}
builder, err := providerRuntimeBuilder(opened)
if err != nil {
return nil, err
}
sort.Strings(names)
runtimes := make([]*controllerProvider.UpstreamRuntime, 0, len(names))
for _, name := range names {
runtime, err := builder(name, configuration.Upstreams[name])
if err != nil {
return nil, err
}
runtimes = append(runtimes, runtime)
}
return controllerProvider.NewFleet(runtimes...)
}
type buildUpstreamRuntime func(string, config.Upstream) (*controllerProvider.UpstreamRuntime, error)
func providerRuntimeBuilder(opened ports) (buildUpstreamRuntime, error) {
if nilInterface(opened.coordinator) || nilInterface(opened.activity) || nilInterface(opened.credentials) {
return nil, ErrProviderRuntime
}
results := opened.providerResults
if nilInterface(results) {
results = discardProviderResults{}
}
return func(name string, upstream config.Upstream) (*controllerProvider.UpstreamRuntime, error) {
upstream.Enabled = true
mapped, err := providerRuntimeConfig(name, upstream)
if err != nil {
return nil, err
}
adapter, err := providerapi.NewHTTPAdapter(upstream.API, upstream.Fetch, nil)
if err != nil {
return nil, err
}
parser, err := providerapi.NewTemplateParser(name, upstream, opened.credentials)
if err != nil {
return nil, err
}
return controllerProvider.NewUpstreamRuntime(mapped, controllerProvider.UpstreamRuntimeDependencies{
Coordinator: opened.coordinator,
Inventory: opened.activity,
Adapter: adapter,
Parser: parser,
Activity: opened.activity,
Results: results,
})
}, nil
}
func providerRuntimeConfig(
upstreamID string,
upstream config.Upstream,
) (controllerProvider.UpstreamRuntimeConfig, error) {
if strings.TrimSpace(upstreamID) != upstreamID || upstreamID == "" || !upstream.Enabled ||
upstream.Fetch.EstimatedIPsPerCall <= 0 || upstream.Capacity.MaxConcurrencyPerProxy <= 0 ||
upstream.Pool.MaxSize > config.MaximumPoolSize ||
int64(upstream.Fetch.EstimatedIPsPerCall) > controllerProvider.MaximumCoordinationInteger ||
int64(upstream.Fetch.MaxInFlight) > controllerProvider.MaximumCoordinationInteger ||
int64(upstream.Fetch.MaxTotal) > controllerProvider.MaximumCoordinationInteger ||
int64(upstream.Fetch.EstimatedIPsPerCall) > math.MaxInt64/int64(upstream.Capacity.MaxConcurrencyPerProxy) {
return controllerProvider.UpstreamRuntimeConfig{}, ErrProviderRuntime
}
expectedSlots := int64(upstream.Fetch.EstimatedIPsPerCall) * int64(upstream.Capacity.MaxConcurrencyPerProxy)
return controllerProvider.UpstreamRuntimeConfig{
Provider: controllerProvider.Config{
UpstreamID: upstreamID,
RequestInterval: time.Duration(upstream.Fetch.RequestInterval),
Timeout: time.Duration(upstream.Fetch.Timeout),
MaxAttempts: upstream.Fetch.MaxAttempts,
MaxInFlight: upstream.Fetch.MaxInFlight,
MaxTotal: int64(upstream.Fetch.MaxTotal),
MaxSize: upstream.Pool.MaxSize,
TTL: time.Duration(upstream.Lifecycle.TTL),
AllocationSafetyMargin: time.Duration(upstream.Lifecycle.AllocationSafetyMargin),
Retry: controllerProvider.RetryConfig{
Initial: time.Duration(upstream.Fetch.Retry.Initial),
Max: time.Duration(upstream.Fetch.Retry.Max),
Jitter: upstream.Fetch.Retry.Jitter,
},
},
ReconcilePolicy: pool.ReconcilePolicy{
MinimumAvailableSlots: upstream.Refill.MinimumAvailableSlots,
TargetAvailableSlots: upstream.Refill.TargetAvailableSlots,
ExpectedPerFetch: upstream.Fetch.EstimatedIPsPerCall,
ExpectedSlotsPerFetch: expectedSlots,
SafetyMargin: time.Duration(upstream.Lifecycle.AllocationSafetyMargin),
},
ReconcileInterval: time.Duration(upstream.Refill.ReconcileInterval),
}, nil
}
type discardProviderResults struct{}
func (discardProviderResults) Record(controllerProvider.Result) {}

View File

@ -0,0 +1,375 @@
package bootstrap
import (
"context"
"errors"
"fmt"
"reflect"
"sort"
"sync"
"time"
"proxy-pool/internal/config"
"proxy-pool/internal/controller/admin"
"proxy-pool/internal/domain/adminstate"
"proxy-pool/internal/platform/lifecycle"
)
const providerSupervisorInterval = time.Second
var (
ErrProviderSupervisor = errors.New("invalid Provider supervisor")
errProviderManagementStateUnavailable = errors.New("Provider management state unavailable")
errProviderManagementSnapshotStale = errors.New("Provider management snapshot stale")
errProviderConfigurationPending = errors.New("Provider configuration synchronization pending")
)
type providerConfigurationStore interface {
Current() *config.Config
Revision() uint64
PublishRevision(*config.Config, uint64) bool
}
type providerConfigurationSource interface {
LoadConfiguration(context.Context) (admin.LoadedConfiguration, error)
}
type providerStateReader interface {
Snapshot(context.Context) (adminstate.Snapshot, error)
}
type providerRunnerBuilder func(string, config.Upstream) (lifecycle.Runner, error)
type providerConfigurationPreparer func(context.Context, *config.Config) error
type providerConfigurationObserver func(*config.Config)
type providerSupervisor struct {
configuration providerConfigurationStore
state providerStateReader
build providerRunnerBuilder
prepare providerConfigurationPreparer
observe providerConfigurationObserver
source providerConfigurationSource
fingerprintKey []byte
interval time.Duration
notify chan struct{}
}
type runningProvider struct {
configuration config.Upstream
cancel context.CancelFunc
done chan struct{}
}
func newProviderSupervisor(
configuration providerConfigurationStore,
state providerStateReader,
build providerRunnerBuilder,
prepare providerConfigurationPreparer,
observe providerConfigurationObserver,
source providerConfigurationSource,
fingerprintKey []byte,
interval time.Duration,
) (*providerSupervisor, error) {
if nilInterface(configuration) || build == nil || interval <= 0 ||
(!nilInterface(state) && len(fingerprintKey) < config.MinimumFingerprintKeyBytes) {
return nil, ErrProviderSupervisor
}
return &providerSupervisor{
configuration: configuration,
state: state,
build: build,
prepare: prepare,
observe: observe,
source: source,
fingerprintKey: append([]byte(nil), fingerprintKey...),
interval: interval,
notify: make(chan struct{}, 1),
}, nil
}
func (supervisor *providerSupervisor) Notify() {
if supervisor == nil || supervisor.notify == nil {
return
}
select {
case supervisor.notify <- struct{}{}:
default:
}
}
func (supervisor *providerSupervisor) ValidateConfiguration(ctx context.Context, configuration *config.Config) error {
if supervisor == nil || ctx == nil || configuration == nil || supervisor.build == nil {
return ErrProviderSupervisor
}
if err := supervisor.validateConfiguration(ctx, configuration); err != nil {
return errors.Join(ErrProviderSupervisor, err)
}
return nil
}
func (supervisor *providerSupervisor) validateConfiguration(ctx context.Context, configuration *config.Config) error {
if err := ctx.Err(); err != nil {
return err
}
if err := config.Validate(configuration); err != nil {
return err
}
if supervisor.prepare != nil {
if err := supervisor.prepare(ctx, configuration); err != nil {
return err
}
}
names := make([]string, 0, len(configuration.Upstreams))
for name, upstream := range configuration.Upstreams {
if upstream.Enabled {
names = append(names, name)
}
}
sort.Strings(names)
for _, name := range names {
if err := ctx.Err(); err != nil {
return err
}
if _, err := supervisor.build(name, configuration.Upstreams[name]); err != nil {
return err
}
}
return nil
}
func (supervisor *providerSupervisor) ValidateUpstream(ctx context.Context, name string) error {
if supervisor == nil || ctx == nil || name == "" || supervisor.build == nil {
return ErrProviderSupervisor
}
if err := ctx.Err(); err != nil {
return err
}
configuration := supervisor.configuration.Current()
if configuration == nil {
return ErrProviderSupervisor
}
if supervisor.prepare != nil {
if err := supervisor.prepare(ctx, configuration); err != nil {
return errors.Join(ErrProviderSupervisor, err)
}
}
upstream, exists := configuration.Upstreams[name]
if !exists {
return ErrProviderSupervisor
}
upstream.Enabled = true
if _, err := supervisor.build(name, upstream); err != nil {
return errors.Join(ErrProviderSupervisor, err)
}
return nil
}
func (supervisor *providerSupervisor) Run(ctx context.Context) error {
if supervisor == nil || ctx == nil || nilInterface(supervisor.configuration) ||
supervisor.build == nil || supervisor.interval <= 0 || supervisor.notify == nil {
return ErrProviderSupervisor
}
active := make(map[string]*runningProvider)
failures := make(chan error, 1)
defer stopAllProviders(active)
ticker := time.NewTicker(supervisor.interval)
defer ticker.Stop()
for {
if err := supervisor.reconcile(ctx, active, failures); err != nil {
return err
}
select {
case <-ctx.Done():
return ctx.Err()
case err := <-failures:
return err
case <-supervisor.notify:
case <-ticker.C:
}
}
}
func (supervisor *providerSupervisor) reconcile(
ctx context.Context,
active map[string]*runningProvider,
failures chan<- error,
) error {
desired, err := supervisor.desired(ctx)
if err != nil {
if errors.Is(err, errProviderManagementStateUnavailable) ||
errors.Is(err, errProviderManagementSnapshotStale) {
return nil
}
if errors.Is(err, errProviderConfigurationPending) {
stopAllProviders(active)
clear(active)
return nil
}
return err
}
names := make([]string, 0, len(desired))
prepared := make(map[string]lifecycle.Runner)
for name, upstream := range desired {
names = append(names, name)
current := active[name]
if current != nil && reflect.DeepEqual(current.configuration, upstream) {
continue
}
runner, buildErr := supervisor.build(name, upstream)
if buildErr != nil || nilInterface(runner) {
return errors.Join(ErrProviderSupervisor, buildErr)
}
prepared[name] = runner
}
sort.Strings(names)
for name, current := range active {
if _, keep := desired[name]; !keep {
stopProvider(current)
delete(active, name)
}
}
for _, name := range names {
runner := prepared[name]
if runner == nil {
continue
}
if current := active[name]; current != nil {
stopProvider(current)
}
active[name] = startProvider(ctx, name, desired[name], runner, failures)
}
return nil
}
func (supervisor *providerSupervisor) desired(ctx context.Context) (map[string]config.Upstream, error) {
configuration := supervisor.configuration.Current()
if configuration == nil {
return nil, ErrProviderSupervisor
}
enabled := make(map[string]bool, len(configuration.Upstreams))
var snapshot adminstate.Snapshot
if !nilInterface(supervisor.state) {
var err error
snapshot, err = supervisor.state.Snapshot(ctx)
if err != nil {
return nil, errors.Join(errProviderManagementStateUnavailable, err)
}
configuration, err = supervisor.synchronizeConfiguration(ctx, configuration, snapshot)
if err != nil {
return nil, err
}
for _, upstream := range snapshot.Upstreams {
enabled[upstream.Name] = upstream.Enabled
}
}
if supervisor.prepare != nil {
if err := supervisor.prepare(ctx, configuration); err != nil {
return nil, errors.Join(ErrProviderSupervisor, err)
}
}
if supervisor.observe != nil {
supervisor.observe(configuration)
}
desired := make(map[string]config.Upstream)
for name, upstream := range configuration.Upstreams {
isEnabled := upstream.Enabled
if !nilInterface(supervisor.state) {
isEnabled = enabled[name]
}
if isEnabled {
upstream.Enabled = true
desired[name] = upstream
}
}
return desired, nil
}
func (supervisor *providerSupervisor) synchronizeConfiguration(
ctx context.Context,
current *config.Config,
snapshot adminstate.Snapshot,
) (*config.Config, error) {
if snapshot.Config == nil || snapshot.Config.Checksum == "" {
return current, nil
}
if supervisor.configuration.Revision() > snapshot.Config.Revision {
return nil, errProviderManagementSnapshotStale
}
checksum, err := config.Fingerprint(current, supervisor.fingerprintKey)
if err != nil {
return nil, errors.Join(ErrProviderSupervisor, err)
}
if checksum == snapshot.Config.Checksum {
supervisor.configuration.PublishRevision(current, snapshot.Config.Revision)
return current, nil
}
if nilInterface(supervisor.source) {
return nil, errProviderConfigurationPending
}
loaded, err := supervisor.source.LoadConfiguration(ctx)
if err != nil || loaded.Value == nil {
return nil, errors.Join(errProviderConfigurationPending, err)
}
checksum, err = config.Fingerprint(loaded.Value, supervisor.fingerprintKey)
if err != nil || checksum != snapshot.Config.Checksum {
return nil, errors.Join(errProviderConfigurationPending, err)
}
if err := supervisor.validateConfiguration(ctx, loaded.Value); err != nil {
return nil, errors.Join(errProviderConfigurationPending, err)
}
if supervisor.configuration.PublishRevision(loaded.Value, snapshot.Config.Revision) {
return loaded.Value, nil
}
if supervisor.configuration.Revision() > snapshot.Config.Revision {
return nil, errProviderManagementSnapshotStale
}
return nil, errProviderConfigurationPending
}
func startProvider(
ctx context.Context,
name string,
configuration config.Upstream,
runner lifecycle.Runner,
failures chan<- error,
) *runningProvider {
runCtx, cancel := context.WithCancel(ctx)
running := &runningProvider{configuration: configuration, cancel: cancel, done: make(chan struct{})}
go func() {
defer close(running.done)
err := runner.Run(runCtx)
if runCtx.Err() != nil {
return
}
if err == nil {
err = lifecycle.ErrRunnerStopped
}
select {
case failures <- fmt.Errorf("Provider %s runtime: %w", name, err):
default:
}
}()
return running
}
func stopProvider(running *runningProvider) {
if running == nil {
return
}
running.cancel()
<-running.done
}
func stopAllProviders(active map[string]*runningProvider) {
var wait sync.WaitGroup
for _, running := range active {
wait.Add(1)
go func(current *runningProvider) {
defer wait.Done()
stopProvider(current)
}(running)
}
wait.Wait()
}

View File

@ -0,0 +1,376 @@
package bootstrap
import (
"context"
"errors"
"strings"
"sync"
"testing"
"time"
"proxy-pool/internal/config"
"proxy-pool/internal/controller/admin"
"proxy-pool/internal/domain/adminstate"
"proxy-pool/internal/platform/lifecycle"
)
func TestProviderSupervisorAppliesDisableAndConfigurationReplacement(t *testing.T) {
configuration, err := config.Load(strings.NewReader(bootstrapTestConfig))
if err != nil {
t.Fatalf("config.Load(): %v", err)
}
store, err := config.NewStore(configuration)
if err != nil {
t.Fatalf("config.NewStore(): %v", err)
}
state := &mutableProviderState{enabled: map[string]bool{"provider-a": true, "provider-b": true}}
started := make(chan providerRuntimeEvent, 4)
stopped := make(chan providerRuntimeEvent, 4)
supervisor, err := newProviderSupervisor(store, state, func(name string, upstream config.Upstream) (lifecycle.Runner, error) {
event := providerRuntimeEvent{name: name, url: upstream.API.URL}
return supervisorRunnerFunc(func(ctx context.Context) error {
started <- event
<-ctx.Done()
stopped <- event
return ctx.Err()
}), nil
}, nil, nil, nil, bootstrapTestFingerprintKey, time.Hour)
if err != nil {
t.Fatalf("newProviderSupervisor(): %v", err)
}
ctx, cancel := context.WithCancel(context.Background())
done := make(chan error, 1)
go func() { done <- supervisor.Run(ctx) }()
waitForRuntimeEvents(t, started, 2)
state.set("provider-a", false)
supervisor.Notify()
if event := waitForRuntimeEvent(t, stopped); event.name != "provider-a" {
t.Fatalf("stopped Provider = %s, want provider-a", event.name)
}
updated := store.Current()
providerB := updated.Upstreams["provider-b"]
providerB.API.URL = "https://replacement.invalid/proxies"
updated.Upstreams["provider-b"] = providerB
if !store.PublishRevision(updated, 1) {
t.Fatal("PublishRevision() rejected updated configuration")
}
supervisor.Notify()
if event := waitForRuntimeEvent(t, stopped); event.name != "provider-b" {
t.Fatalf("replaced Provider = %s, want provider-b", event.name)
}
if event := waitForRuntimeEvent(t, started); event.name != "provider-b" || event.url != providerB.API.URL {
t.Fatalf("replacement Provider = %+v", event)
}
cancel()
if err := <-done; !errors.Is(err, context.Canceled) {
t.Fatalf("Run() error = %v, want context cancellation", err)
}
}
func TestProviderSupervisorPropagatesUnexpectedRuntimeFailure(t *testing.T) {
configuration, err := config.Load(strings.NewReader(bootstrapTestConfig))
if err != nil {
t.Fatalf("config.Load(): %v", err)
}
store, err := config.NewStore(configuration)
if err != nil {
t.Fatalf("config.NewStore(): %v", err)
}
wantErr := errors.New("coordination stopped")
supervisor, err := newProviderSupervisor(store, nil, func(string, config.Upstream) (lifecycle.Runner, error) {
return supervisorRunnerFunc(func(context.Context) error { return wantErr }), nil
}, nil, nil, nil, bootstrapTestFingerprintKey, time.Hour)
if err != nil {
t.Fatalf("newProviderSupervisor(): %v", err)
}
if err := supervisor.Run(context.Background()); !errors.Is(err, wantErr) {
t.Fatalf("Run() error = %v, want %v", err, wantErr)
}
}
func TestProviderSupervisorRetainsRuntimesWhileManagementStateIsUnavailable(t *testing.T) {
configuration, err := config.Load(strings.NewReader(bootstrapTestConfig))
if err != nil {
t.Fatalf("config.Load(): %v", err)
}
store, err := config.NewStore(configuration)
if err != nil {
t.Fatalf("config.NewStore(): %v", err)
}
state := &mutableProviderState{enabled: map[string]bool{"provider-a": true, "provider-b": true}}
started := make(chan providerRuntimeEvent, 2)
stopped := make(chan providerRuntimeEvent, 2)
supervisor, err := newProviderSupervisor(store, state, func(name string, upstream config.Upstream) (lifecycle.Runner, error) {
event := providerRuntimeEvent{name: name, url: upstream.API.URL}
return supervisorRunnerFunc(func(ctx context.Context) error {
started <- event
<-ctx.Done()
stopped <- event
return ctx.Err()
}), nil
}, nil, nil, nil, bootstrapTestFingerprintKey, time.Hour)
if err != nil {
t.Fatalf("newProviderSupervisor(): %v", err)
}
ctx, cancel := context.WithCancel(context.Background())
done := make(chan error, 1)
go func() { done <- supervisor.Run(ctx) }()
waitForRuntimeEvents(t, started, 2)
state.setError(errors.New("PostgreSQL temporarily unavailable"))
supervisor.Notify()
select {
case err := <-done:
t.Fatalf("Supervisor stopped during transient state failure: %v", err)
case event := <-stopped:
t.Fatalf("Provider stopped during transient state failure: %+v", event)
case <-time.After(50 * time.Millisecond):
}
state.setError(nil)
state.set("provider-a", false)
supervisor.Notify()
if event := waitForRuntimeEvent(t, stopped); event.name != "provider-a" {
t.Fatalf("stopped Provider = %s, want provider-a", event.name)
}
cancel()
if err := <-done; !errors.Is(err, context.Canceled) {
t.Fatalf("Run() error = %v, want context cancellation", err)
}
}
func TestProviderSupervisorStopsStaleRuntimesAndLoadsAuthoritativeConfiguration(t *testing.T) {
initial, err := config.Load(strings.NewReader(bootstrapTestConfig))
if err != nil {
t.Fatalf("config.Load(): %v", err)
}
store, err := config.NewStore(initial)
if err != nil {
t.Fatalf("config.NewStore(): %v", err)
}
initialChecksum, err := config.Fingerprint(initial, bootstrapTestFingerprintKey)
if err != nil {
t.Fatalf("config.Fingerprint(initial): %v", err)
}
state := &mutableProviderState{
enabled: map[string]bool{"provider-a": true, "provider-b": true},
checksum: initialChecksum,
revision: 1,
}
source := &mutableProviderConfigurationSource{configuration: initial}
started := make(chan providerRuntimeEvent, 4)
stopped := make(chan providerRuntimeEvent, 4)
supervisor, err := newProviderSupervisor(store, state, func(name string, upstream config.Upstream) (lifecycle.Runner, error) {
event := providerRuntimeEvent{name: name, url: upstream.API.URL}
return supervisorRunnerFunc(func(ctx context.Context) error {
started <- event
<-ctx.Done()
stopped <- event
return ctx.Err()
}), nil
}, nil, nil, source, bootstrapTestFingerprintKey, time.Hour)
if err != nil {
t.Fatalf("newProviderSupervisor(): %v", err)
}
ctx, cancel := context.WithCancel(context.Background())
done := make(chan error, 1)
go func() { done <- supervisor.Run(ctx) }()
waitForRuntimeEvents(t, started, 2)
updated := store.Current()
providerA := updated.Upstreams["provider-a"]
providerA.API.URL = "https://replacement.invalid/proxies"
updated.Upstreams["provider-a"] = providerA
updatedChecksum, err := config.Fingerprint(updated, bootstrapTestFingerprintKey)
if err != nil {
t.Fatalf("config.Fingerprint(updated): %v", err)
}
state.setChecksum(updatedChecksum)
supervisor.Notify()
waitForRuntimeEvents(t, stopped, 2)
select {
case err := <-done:
t.Fatalf("Supervisor stopped for stale local configuration: %v", err)
case <-time.After(50 * time.Millisecond):
}
source.set(updated)
supervisor.Notify()
events := []providerRuntimeEvent{waitForRuntimeEvent(t, started), waitForRuntimeEvent(t, started)}
foundReplacement := false
for _, event := range events {
if event.name == "provider-a" && event.url == providerA.API.URL {
foundReplacement = true
}
}
if !foundReplacement {
t.Fatalf("started Provider runtimes = %+v, replacement missing", events)
}
if currentChecksum, err := config.Fingerprint(store.Current(), bootstrapTestFingerprintKey); err != nil || currentChecksum != updatedChecksum {
t.Fatalf("published checksum = %q, %v; want %q", currentChecksum, err, updatedChecksum)
}
cancel()
if err := <-done; !errors.Is(err, context.Canceled) {
t.Fatalf("Run() error = %v, want context cancellation", err)
}
}
func TestProviderSupervisorDoesNotPublishConfigurationOlderThanLocalRevision(t *testing.T) {
initial, err := config.Load(strings.NewReader(bootstrapTestConfig))
if err != nil {
t.Fatalf("config.Load(): %v", err)
}
store, err := config.NewStore(initial)
if err != nil {
t.Fatalf("config.NewStore(): %v", err)
}
store.PublishRevision(initial, 1)
candidate := store.Current()
providerA := candidate.Upstreams["provider-a"]
providerA.API.URL = "https://candidate.invalid/proxies"
candidate.Upstreams["provider-a"] = providerA
candidateChecksum, err := config.Fingerprint(candidate, bootstrapTestFingerprintKey)
if err != nil {
t.Fatalf("config.Fingerprint(candidate): %v", err)
}
newest := store.Current()
providerA = newest.Upstreams["provider-a"]
providerA.API.URL = "https://newest.invalid/proxies"
newest.Upstreams["provider-a"] = providerA
source := providerConfigurationSourceFunc(func(context.Context) (admin.LoadedConfiguration, error) {
if !store.PublishRevision(newest, 3) {
t.Fatal("failed to publish simulated concurrent revision")
}
return admin.LoadedConfiguration{Value: candidate, Source: "controller.yaml"}, nil
})
supervisor, err := newProviderSupervisor(
store,
&mutableProviderState{},
func(string, config.Upstream) (lifecycle.Runner, error) {
return supervisorRunnerFunc(func(context.Context) error { return nil }), nil
},
nil,
nil,
source,
bootstrapTestFingerprintKey,
time.Hour,
)
if err != nil {
t.Fatalf("newProviderSupervisor(): %v", err)
}
_, err = supervisor.synchronizeConfiguration(context.Background(), initial, adminstate.Snapshot{
Config: &adminstate.ConfigRevision{Revision: 2, Checksum: candidateChecksum},
})
if !errors.Is(err, errProviderManagementSnapshotStale) {
t.Fatalf("synchronizeConfiguration() error = %v, want stale snapshot", err)
}
if got := store.Revision(); got != 3 {
t.Fatalf("configuration revision = %d, want 3", got)
}
}
type providerRuntimeEvent struct {
name string
url string
}
type supervisorRunnerFunc func(context.Context) error
func (run supervisorRunnerFunc) Run(ctx context.Context) error { return run(ctx) }
type mutableProviderState struct {
mu sync.Mutex
enabled map[string]bool
err error
checksum string
revision uint64
}
func (state *mutableProviderState) Snapshot(context.Context) (adminstate.Snapshot, error) {
state.mu.Lock()
defer state.mu.Unlock()
if state.err != nil {
return adminstate.Snapshot{}, state.err
}
snapshot := adminstate.Snapshot{Upstreams: make([]adminstate.UpstreamState, 0, len(state.enabled))}
if state.checksum != "" {
snapshot.Config = &adminstate.ConfigRevision{
Revision: state.revision, ConfigVersion: "cfg-" + state.checksum, Checksum: state.checksum,
}
}
for name, enabled := range state.enabled {
snapshot.Upstreams = append(snapshot.Upstreams, adminstate.UpstreamState{Name: name, Enabled: enabled})
}
return snapshot, nil
}
func (state *mutableProviderState) setError(err error) {
state.mu.Lock()
defer state.mu.Unlock()
state.err = err
}
func (state *mutableProviderState) setChecksum(checksum string) {
state.mu.Lock()
defer state.mu.Unlock()
state.checksum = checksum
state.revision++
}
type mutableProviderConfigurationSource struct {
mu sync.Mutex
configuration *config.Config
}
type providerConfigurationSourceFunc func(context.Context) (admin.LoadedConfiguration, error)
func (source providerConfigurationSourceFunc) LoadConfiguration(ctx context.Context) (admin.LoadedConfiguration, error) {
return source(ctx)
}
func (source *mutableProviderConfigurationSource) LoadConfiguration(context.Context) (admin.LoadedConfiguration, error) {
source.mu.Lock()
defer source.mu.Unlock()
return admin.LoadedConfiguration{Value: source.configuration, Source: "controller.yaml"}, nil
}
func (source *mutableProviderConfigurationSource) set(configuration *config.Config) {
source.mu.Lock()
defer source.mu.Unlock()
source.configuration = configuration
}
func (state *mutableProviderState) set(name string, enabled bool) {
state.mu.Lock()
defer state.mu.Unlock()
state.enabled[name] = enabled
}
func waitForRuntimeEvents(t *testing.T, events <-chan providerRuntimeEvent, count int) {
t.Helper()
for range count {
_ = waitForRuntimeEvent(t, events)
}
}
func waitForRuntimeEvent(t *testing.T, events <-chan providerRuntimeEvent) providerRuntimeEvent {
t.Helper()
select {
case event := <-events:
return event
case <-time.After(time.Second):
t.Fatal("timed out waiting for Provider runtime event")
return providerRuntimeEvent{}
}
}

View File

@ -0,0 +1,104 @@
package bootstrap
import (
"strconv"
"strings"
"testing"
"time"
"proxy-pool/internal/config"
"proxy-pool/internal/controller/provider"
"proxy-pool/internal/platform/credentials"
)
func TestProviderRuntimeConfigMapsValidatedUpstreamOnce(t *testing.T) {
configuration, err := config.Load(strings.NewReader(bootstrapTestConfig))
if err != nil {
t.Fatalf("config.Load(): %v", err)
}
upstream := configuration.Upstreams["provider-a"]
mapped, err := providerRuntimeConfig("provider-a", upstream)
if err != nil {
t.Fatalf("providerRuntimeConfig(): %v", err)
}
if mapped.Provider.UpstreamID != "provider-a" ||
mapped.Provider.RequestInterval != time.Second ||
mapped.Provider.Timeout != 3*time.Second ||
mapped.Provider.MaxAttempts != 3 || mapped.Provider.MaxInFlight != 1 ||
mapped.Provider.MaxTotal != 1_000 || mapped.Provider.MaxSize != 100 ||
mapped.Provider.TTL != 2*time.Minute ||
mapped.Provider.AllocationSafetyMargin != 10*time.Second {
t.Fatalf("Provider config = %+v", mapped.Provider)
}
if mapped.ReconcileInterval != time.Second ||
mapped.ReconcilePolicy.MinimumAvailableSlots != 100 ||
mapped.ReconcilePolicy.TargetAvailableSlots != 200 ||
mapped.ReconcilePolicy.ExpectedPerFetch != 10 ||
mapped.ReconcilePolicy.ExpectedSlotsPerFetch != 100 ||
mapped.ReconcilePolicy.SafetyMargin != 10*time.Second {
t.Fatalf("Reconcile config = %+v interval=%s", mapped.ReconcilePolicy, mapped.ReconcileInterval)
}
}
func TestNewProviderFleetBuildsEnabledUpstreamsInStableOrder(t *testing.T) {
configuration, err := config.Load(strings.NewReader(bootstrapTestConfig))
if err != nil {
t.Fatalf("config.Load(): %v", err)
}
configuration.Upstreams["disabled"] = config.Upstream{}
credentialStore, err := credentials.NewMemoryStore(10)
if err != nil {
t.Fatalf("NewMemoryStore(): %v", err)
}
fleet, err := newProviderFleet(configuration, ports{
activity: &stubActivityStore{}, coordinator: coordinatorStub{}, credentials: credentialStore,
})
if err != nil {
t.Fatalf("newProviderFleet(): %v", err)
}
if got := fleet.IDs(); len(got) != 2 || got[0] != "provider-a" || got[1] != "provider-b" {
t.Fatalf("Fleet IDs = %v, want [provider-a provider-b]", got)
}
configuration.Upstreams["provider-a"] = config.Upstream{}
configuration.Upstreams["provider-b"] = config.Upstream{}
fleet, err = newProviderFleet(configuration, ports{})
if err != nil || fleet != nil {
t.Fatalf("newProviderFleet(no enabled) = (%v, %v), want nil fleet", fleet, err)
}
}
func TestProviderRuntimeConfigRejectsDisabledAndOverflowingInputs(t *testing.T) {
if _, err := providerRuntimeConfig("provider-a", config.Upstream{}); err == nil {
t.Fatal("providerRuntimeConfig(disabled) error = nil")
}
overflowing := config.Upstream{
Enabled: true,
Pool: config.Pool{MaxSize: int(^uint(0) >> 1)},
Capacity: config.Capacity{
MaxConcurrencyPerProxy: int(^uint(0) >> 1),
},
Fetch: config.Fetch{EstimatedIPsPerCall: int(^uint(0) >> 1)},
}
if _, err := providerRuntimeConfig("provider-a", overflowing); err == nil {
t.Fatal("providerRuntimeConfig(overflow) error = nil")
}
configuration, err := config.Load(strings.NewReader(bootstrapTestConfig))
if err != nil {
t.Fatalf("config.Load(): %v", err)
}
tooLarge := configuration.Upstreams["provider-a"]
tooLarge.Pool.MaxSize = redisMaximumScan + 1
if _, err := providerRuntimeConfig("provider-a", tooLarge); err == nil {
t.Fatal("providerRuntimeConfig(oversized inventory) error = nil")
}
tooLarge = configuration.Upstreams["provider-a"]
tooLarge.Fetch.MaxTotal = int(provider.MaximumCoordinationInteger)
if strconv.IntSize == 64 {
tooLarge.Fetch.MaxTotal++
if _, err := providerRuntimeConfig("provider-a", tooLarge); err == nil {
t.Fatal("providerRuntimeConfig(inexact Redis integer) error = nil")
}
}
}

View File

@ -9,6 +9,7 @@ import (
"proxy-pool/internal/config"
"proxy-pool/internal/controller/admin"
"proxy-pool/internal/controller/provider"
"proxy-pool/internal/domain/activitypool"
)
@ -24,6 +25,7 @@ type ConfigurationReader interface {
type Reader struct {
configuration ConfigurationReader
inventory activitypool.StateInventoryReader
providerStats provider.StatsReader
now func() time.Time
}
@ -33,11 +35,17 @@ func NewReader(
configuration ConfigurationReader,
inventory activitypool.StateInventoryReader,
now func() time.Time,
stats ...provider.StatsReader,
) (*Reader, error) {
if nilInterface(configuration) || nilInterface(inventory) || now == nil {
if nilInterface(configuration) || nilInterface(inventory) || now == nil || len(stats) > 1 ||
(len(stats) == 1 && nilInterface(stats[0])) {
return nil, ErrInvalidReader
}
return &Reader{configuration: configuration, inventory: inventory, now: now}, nil
reader := &Reader{configuration: configuration, inventory: inventory, now: now}
if len(stats) == 1 {
reader.providerStats = stats[0]
}
return reader, nil
}
func (reader *Reader) ReadOperationalStatus(ctx context.Context) (admin.OperationalStatus, error) {
@ -69,17 +77,27 @@ func (reader *Reader) ReadOperationalStatus(ctx context.Context) (admin.Operatio
}
status := admin.OperationalStatus{Upstreams: make([]admin.UpstreamActivity, len(inventories))}
providerStats := make([]provider.Stats, len(upstreamIDs))
if reader.providerStats != nil {
providerStats = reader.providerStats.ReadProviderStats(upstreamIDs)
if len(providerStats) != len(upstreamIDs) {
return admin.OperationalStatus{}, ErrUnavailable
}
}
for index, inventory := range inventories {
if inventory.UpstreamID != upstreamIDs[index] || invalidInventory(inventory) {
if inventory.UpstreamID != upstreamIDs[index] || invalidInventory(inventory) ||
providerStats[index].UpstreamID != "" && providerStats[index].UpstreamID != upstreamIDs[index] {
return admin.OperationalStatus{}, ErrUnavailable
}
status.Upstreams[index] = admin.UpstreamActivity{
Name: inventory.UpstreamID,
Available: inventory.Available,
Checking: inventory.Checking,
Suspect: inventory.Suspect,
Draining: inventory.Draining,
Extracted: inventory.Extracted,
Name: inventory.UpstreamID,
Available: inventory.Available,
Checking: inventory.Checking,
Suspect: inventory.Suspect,
Draining: inventory.Draining,
Extracted: inventory.Extracted,
ConsecutiveEmptyFetch: providerStats[index].ConsecutiveEmptyFetch,
FetchErrorCount: providerStats[index].FetchErrorCount,
}
}
return status, nil

View File

@ -7,6 +7,7 @@ import (
"time"
"proxy-pool/internal/config"
"proxy-pool/internal/controller/provider"
"proxy-pool/internal/domain/activitypool"
)
@ -19,7 +20,10 @@ func TestReaderMapsCurrentUpstreamsToAdminOperationalStatus(t *testing.T) {
}}
reader, err := NewReader(staticConfigurationReader{configuration: &config.Config{
Upstreams: map[string]config.Upstream{"provider-b": {}, "provider-a": {}},
}}, inventory, func() time.Time { return now })
}}, inventory, func() time.Time { return now }, staticProviderStatsReader{result: []provider.Stats{
{UpstreamID: "provider-a", ConsecutiveEmptyFetch: 3, FetchErrorCount: 4},
{UpstreamID: "provider-b", FetchErrorCount: 1},
}})
if err != nil {
t.Fatalf("NewReader() error = %v", err)
}
@ -37,7 +41,7 @@ func TestReaderMapsCurrentUpstreamsToAdminOperationalStatus(t *testing.T) {
}
first := status.Upstreams[0]
if first.Name != "provider-a" || first.Available != 11 || first.Checking != 2 || first.Suspect != 1 ||
first.Draining != 4 || first.Extracted != 8 {
first.Draining != 4 || first.Extracted != 8 || first.ConsecutiveEmptyFetch != 3 || first.FetchErrorCount != 4 {
t.Fatalf("first upstream = %+v", first)
}
if status.Upstreams[1].Name != "provider-b" || status.Upstreams[1].Available != 7 {
@ -107,6 +111,14 @@ type recordingStateInventoryReader struct {
now time.Time
}
type staticProviderStatsReader struct {
result []provider.Stats
}
func (reader staticProviderStatsReader) ReadProviderStats([]string) []provider.Stats {
return append([]provider.Stats(nil), reader.result...)
}
func (reader *recordingStateInventoryReader) ReadStateInventory(
_ context.Context,
upstreamIDs []string,

View File

@ -20,48 +20,38 @@ var (
type FetchBudgetConfig struct {
UpstreamID string
MaxSize int
MaxTotal int64
ExpectedPerFetch int
Managed int
FetchedTotal int64
}
type FetchBudgetSnapshot struct {
Managed int
PendingExpected int
FetchedTotal int64
}
// FetchBudget owns both current-inventory and cumulative-fetch accounting for
// one upstream. Reserving the expected response before I/O closes the race
// between concurrent provider calls.
// FetchBudget owns current-inventory accounting for one upstream. Reserving
// the expected response before I/O closes the race between concurrent calls;
// distributed cumulative quota belongs to the Provider coordination permit.
type FetchBudget struct {
mu sync.Mutex
upstreamID string
maxSize int
maxTotal int64
expected int
usage FetchBudgetSnapshot
}
func NewFetchBudget(config FetchBudgetConfig) (*FetchBudget, error) {
if config.UpstreamID == "" || config.MaxSize <= 0 || config.ExpectedPerFetch <= 0 ||
config.ExpectedPerFetch > config.MaxSize || config.MaxTotal < 0 ||
config.Managed < 0 || config.FetchedTotal < 0 {
return nil, ErrInvalidFetchBudget
}
if config.MaxTotal > 0 && int64(config.ExpectedPerFetch) > config.MaxTotal {
config.ExpectedPerFetch > config.MaxSize || config.Managed < 0 {
return nil, ErrInvalidFetchBudget
}
return &FetchBudget{
upstreamID: config.UpstreamID,
maxSize: config.MaxSize,
maxTotal: config.MaxTotal,
expected: config.ExpectedPerFetch,
usage: FetchBudgetSnapshot{
Managed: config.Managed,
FetchedTotal: config.FetchedTotal,
Managed: config.Managed,
},
}, nil
}
@ -95,14 +85,7 @@ func (b *FetchBudget) FetchAllowance() int {
func (b *FetchBudget) canReserveLocked() bool {
poolRoom := b.maxSize - b.usage.Managed - b.usage.PendingExpected
if poolRoom < b.expected {
return false
}
if b.maxTotal > 0 {
totalRoom := b.maxTotal - b.usage.FetchedTotal - int64(b.usage.PendingExpected)
return totalRoom >= int64(b.expected)
}
return true
return poolRoom >= b.expected
}
func (b *FetchBudget) Snapshot() FetchBudgetSnapshot {
@ -115,8 +98,8 @@ func (b *FetchBudget) Snapshot() FetchBudgetSnapshot {
}
// SynchronizeManaged replaces the local current-inventory count with the
// authoritative activity-store observation. Pending requests and cumulative
// fetch usage remain owned by this budget.
// authoritative activity-store observation. Pending reservations remain owned
// by this budget.
func (b *FetchBudget) SynchronizeManaged(managed int) error {
if b == nil || managed < 0 {
return ErrInvalidManagedSynchronization
@ -158,11 +141,11 @@ func (p *fetchPermit) Expected() int {
return p.expected
}
func (p *fetchPermit) Complete(fetched, retained int) error {
func (p *fetchPermit) Complete(retained int) error {
if p == nil || p.budget == nil {
return ErrFetchPermitFinished
}
if fetched < 0 || retained < 0 || retained > fetched || retained > p.expected {
if retained < 0 || retained > p.expected {
return ErrInvalidFetchCompletion
}
p.budget.mu.Lock()
@ -173,7 +156,6 @@ func (p *fetchPermit) Complete(fetched, retained int) error {
p.finished = true
p.budget.usage.PendingExpected -= p.expected
p.budget.usage.Managed += retained
p.budget.usage.FetchedTotal += int64(fetched)
return nil
}

View File

@ -6,14 +6,12 @@ import (
"testing"
)
func TestFetchBudgetReservesExpectedCapacityAndSeparatesCounters(t *testing.T) {
func TestFetchBudgetReservesExpectedPoolCapacity(t *testing.T) {
budget, err := NewFetchBudget(FetchBudgetConfig{
UpstreamID: "provider-a",
MaxSize: 10,
MaxTotal: 20,
ExpectedPerFetch: 4,
Managed: 2,
FetchedTotal: 3,
})
if err != nil {
t.Fatalf("NewFetchBudget(): %v", err)
@ -31,21 +29,21 @@ func TestFetchBudgetReservesExpectedCapacityAndSeparatesCounters(t *testing.T) {
t.Fatalf("third ReserveFetch() = (_, %v, %v), want no capacity", ok, err)
}
if err := first.Complete(4, 3); err != nil {
if err := first.Complete(3); err != nil {
t.Fatalf("first.Complete(): %v", err)
}
if err := second.Cancel(); err != nil {
t.Fatalf("second.Cancel(): %v", err)
}
usage := budget.Snapshot()
if usage.Managed != 5 || usage.PendingExpected != 0 || usage.FetchedTotal != 7 {
t.Fatalf("Snapshot() = %+v, want managed=5 pending=0 fetched=7", usage)
if usage.Managed != 5 || usage.PendingExpected != 0 {
t.Fatalf("Snapshot() = %+v, want managed=5 pending=0", usage)
}
if err := budget.ReleaseManaged(2); err != nil {
t.Fatalf("ReleaseManaged(): %v", err)
}
if usage := budget.Snapshot(); usage.Managed != 3 || usage.FetchedTotal != 7 {
t.Fatalf("Snapshot() after release = %+v, want managed=3 fetched=7", usage)
if usage := budget.Snapshot(); usage.Managed != 3 {
t.Fatalf("Snapshot() after release = %+v, want managed=3", usage)
}
}
@ -63,7 +61,7 @@ func TestFetchBudgetRejectsManagedCounterUnderflow(t *testing.T) {
func TestFetchBudgetSynchronizesAuthoritativeManagedInventory(t *testing.T) {
budget, err := NewFetchBudget(FetchBudgetConfig{
UpstreamID: "a", MaxSize: 5, ExpectedPerFetch: 2, Managed: 3, FetchedTotal: 7,
UpstreamID: "a", MaxSize: 5, ExpectedPerFetch: 2, Managed: 3,
})
if err != nil {
t.Fatalf("NewFetchBudget(): %v", err)
@ -77,8 +75,8 @@ func TestFetchBudgetSynchronizesAuthoritativeManagedInventory(t *testing.T) {
t.Fatalf("SynchronizeManaged() error = %v, want ErrManagedSynchronizationInFlight", err)
}
usage := budget.Snapshot()
if usage.Managed != 3 || usage.PendingExpected != 2 || usage.FetchedTotal != 7 {
t.Fatalf("Snapshot() = %+v, want managed=3 pending=2 fetched=7", usage)
if usage.Managed != 3 || usage.PendingExpected != 2 {
t.Fatalf("Snapshot() = %+v, want managed=3 pending=2", usage)
}
if err := permit.Cancel(); err != nil {
t.Fatalf("Cancel(): %v", err)
@ -86,8 +84,8 @@ func TestFetchBudgetSynchronizesAuthoritativeManagedInventory(t *testing.T) {
if err := budget.SynchronizeManaged(1); err != nil {
t.Fatalf("SynchronizeManaged() after cancel: %v", err)
}
if usage := budget.Snapshot(); usage.Managed != 1 || usage.PendingExpected != 0 || usage.FetchedTotal != 7 {
t.Fatalf("Snapshot() after synchronization = %+v, want managed=1 pending=0 fetched=7", usage)
if usage := budget.Snapshot(); usage.Managed != 1 || usage.PendingExpected != 0 {
t.Fatalf("Snapshot() after synchronization = %+v, want managed=1 pending=0", usage)
}
}
@ -103,35 +101,15 @@ func TestFetchBudgetRejectsNegativeManagedSynchronization(t *testing.T) {
}
}
func TestFetchBudgetRequiresWholeExpectedBatchToFitLimits(t *testing.T) {
tests := []struct {
name string
config FetchBudgetConfig
}{
{
name: "pool size",
config: FetchBudgetConfig{
UpstreamID: "a", MaxSize: 10, ExpectedPerFetch: 4, Managed: 7,
},
},
{
name: "cumulative total",
config: FetchBudgetConfig{
UpstreamID: "a", MaxSize: 10, MaxTotal: 5, ExpectedPerFetch: 4, FetchedTotal: 2,
},
},
func TestFetchBudgetRequiresWholeExpectedBatchToFitPool(t *testing.T) {
budget, err := NewFetchBudget(FetchBudgetConfig{
UpstreamID: "a", MaxSize: 10, ExpectedPerFetch: 4, Managed: 7,
})
if err != nil {
t.Fatalf("NewFetchBudget(): %v", err)
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
budget, err := NewFetchBudget(tt.config)
if err != nil {
t.Fatalf("NewFetchBudget(): %v", err)
}
if _, ok, err := budget.ReserveFetch("a"); err != nil || ok {
t.Fatalf("ReserveFetch() = (_, %v, %v), want no capacity", ok, err)
}
})
if _, ok, err := budget.ReserveFetch("a"); err != nil || ok {
t.Fatalf("ReserveFetch() = (_, %v, %v), want no capacity", ok, err)
}
}
@ -185,7 +163,7 @@ func TestFetchPermitRejectsDoubleFinishAndInvalidCounts(t *testing.T) {
if err != nil || !ok {
t.Fatalf("ReserveFetch() = (_, %v, %v), want permit", ok, err)
}
if err := permit.Complete(1, 2); !errors.Is(err, ErrInvalidFetchCompletion) {
if err := permit.Complete(3); !errors.Is(err, ErrInvalidFetchCompletion) {
t.Fatalf("Complete() error = %v, want ErrInvalidFetchCompletion", err)
}
if err := permit.Cancel(); err != nil {

View File

@ -0,0 +1,17 @@
package pool
import (
"context"
"time"
)
type InventorySnapshot struct {
Managed int
AvailableSlots int64
}
type InventoryReader interface {
// ReadInventory returns a bounded, authoritative aggregate. Unknown Worker
// runtime must reduce capacity rather than being treated as idle.
ReadInventory(context.Context, string, time.Duration) (InventorySnapshot, error)
}

View File

@ -26,7 +26,6 @@ type FetchNotifier interface {
type ReconcileDecision struct {
AvailableSlots int64
PendingExpected int
FetchedTotal int64
FetchAllowance int
EffectiveSlots int64
Triggered bool
@ -62,15 +61,28 @@ func NewReconciler(policy ReconcilePolicy, budget *FetchBudget, notifier FetchNo
// Reconcile centralizes the cold-path decision. The notifier may coalesce many
// calls; Provider Reconciler atomically reserves the budget before doing I/O.
func (r *Reconciler) Reconcile(now time.Time, inventory upstream.Inventory) ReconcileDecision {
return r.reconcileSnapshot(
InventorySnapshot{AvailableSlots: inventory.AvailableSlots(now, r.policy.SafetyMargin)},
false,
)
}
func (r *Reconciler) ReconcileSnapshot(inventory InventorySnapshot) ReconcileDecision {
return r.reconcileSnapshot(inventory, true)
}
func (r *Reconciler) reconcileSnapshot(inventory InventorySnapshot, synchronizeManaged bool) ReconcileDecision {
usage := r.budget.Snapshot()
availableSlots := inventory.AvailableSlots(now, r.policy.SafetyMargin)
if synchronizeManaged && inventory.Managed >= 0 && usage.PendingExpected == 0 {
_ = r.budget.SynchronizeManaged(inventory.Managed)
usage = r.budget.Snapshot()
}
pendingSlots := saturatingMultiply(int64(usage.PendingExpected), r.slotsPerProxy)
decision := ReconcileDecision{
AvailableSlots: availableSlots,
AvailableSlots: inventory.AvailableSlots,
PendingExpected: usage.PendingExpected,
FetchedTotal: usage.FetchedTotal,
FetchAllowance: r.budget.FetchAllowance(),
EffectiveSlots: saturatingAdd(availableSlots, pendingSlots),
EffectiveSlots: saturatingAdd(inventory.AvailableSlots, pendingSlots),
}
r.mu.Lock()
if r.refilling {

View File

@ -10,7 +10,7 @@ import (
func TestPoolReconcilerSignalsOnlyWhenSlotsAreLowAndWholeFetchFits(t *testing.T) {
budget, err := NewFetchBudget(FetchBudgetConfig{
UpstreamID: "provider-a", MaxSize: 10, MaxTotal: 20, ExpectedPerFetch: 2,
UpstreamID: "provider-a", MaxSize: 10, ExpectedPerFetch: 2,
})
if err != nil {
t.Fatalf("NewFetchBudget(): %v", err)
@ -31,8 +31,7 @@ func TestPoolReconcilerSignalsOnlyWhenSlotsAreLowAndWholeFetchFits(t *testing.T)
Proxies: []upstream.ProxyCapacity{{
State: proxyDomain.StateAvailable, ExpiresAt: now.Add(time.Minute), Max: 4, Active: 3,
}},
MaxSize: 10,
MaxTotal: 20,
MaxSize: 10,
}
decision := reconciler.Reconcile(now, inventory)
@ -44,9 +43,9 @@ func TestPoolReconcilerSignalsOnlyWhenSlotsAreLowAndWholeFetchFits(t *testing.T)
}
}
func TestPoolReconcilerUsesBudgetPendingAndCumulativeCounters(t *testing.T) {
func TestPoolReconcilerUsesPendingPoolReservation(t *testing.T) {
budget, err := NewFetchBudget(FetchBudgetConfig{
UpstreamID: "provider-a", MaxSize: 2, MaxTotal: 2, ExpectedPerFetch: 2,
UpstreamID: "provider-a", MaxSize: 2, ExpectedPerFetch: 2,
})
if err != nil {
t.Fatalf("NewFetchBudget(): %v", err)
@ -65,7 +64,7 @@ func TestPoolReconcilerUsesBudgetPendingAndCumulativeCounters(t *testing.T) {
t.Fatalf("NewReconciler(): %v", err)
}
decision := reconciler.Reconcile(time.Now(), upstream.Inventory{MaxSize: 2, MaxTotal: 2})
decision := reconciler.Reconcile(time.Now(), upstream.Inventory{MaxSize: 2})
if decision.Triggered || decision.PendingExpected != 2 || decision.FetchAllowance != 0 {
t.Fatalf("Reconcile() = %+v, want pending fetch to suppress signal", decision)
}
@ -133,7 +132,7 @@ func TestPoolReconcilerPendingEstimatePausesWithoutEndingRefillEpisode(t *testin
if decision := reconciler.Reconcile(now, inventoryWithSlots(now, 2)); decision.Triggered || decision.EffectiveSlots != 8 {
t.Fatalf("pending Reconcile() = %+v, want paused at target estimate", decision)
}
if err := permit.Complete(2, 1); err != nil {
if err := permit.Complete(1); err != nil {
t.Fatalf("Complete(): %v", err)
}
if decision := reconciler.Reconcile(now, inventoryWithSlots(now, 5)); !decision.Triggered {
@ -141,6 +140,53 @@ func TestPoolReconcilerPendingEstimatePausesWithoutEndingRefillEpisode(t *testin
}
}
func TestPoolReconcilerConsumesAuthoritativeInventorySnapshot(t *testing.T) {
budget, err := NewFetchBudget(FetchBudgetConfig{
UpstreamID: "provider-a", MaxSize: 10, ExpectedPerFetch: 2,
})
if err != nil {
t.Fatalf("NewFetchBudget(): %v", err)
}
notifier := &recordingFetchNotifier{}
reconciler, err := NewReconciler(ReconcilePolicy{
MinimumAvailableSlots: 3, TargetAvailableSlots: 8,
ExpectedPerFetch: 2, ExpectedSlotsPerFetch: 2,
}, budget, notifier)
if err != nil {
t.Fatalf("NewReconciler(): %v", err)
}
decision := reconciler.ReconcileSnapshot(InventorySnapshot{Managed: 9, AvailableSlots: 2})
if decision.Triggered || decision.AvailableSlots != 2 || decision.FetchAllowance != 0 {
t.Fatalf("ReconcileSnapshot() = %+v, want authoritative managed inventory to close budget", decision)
}
if usage := budget.Snapshot(); usage.Managed != 9 {
t.Fatalf("FetchBudget.Managed = %d, want 9", usage.Managed)
}
}
func TestPoolReconcilerLegacyInventoryDoesNotClearManagedBudget(t *testing.T) {
budget, err := NewFetchBudget(FetchBudgetConfig{
UpstreamID: "provider-a", MaxSize: 10, ExpectedPerFetch: 2, Managed: 9,
})
if err != nil {
t.Fatalf("NewFetchBudget(): %v", err)
}
reconciler, err := NewReconciler(ReconcilePolicy{
MinimumAvailableSlots: 3, TargetAvailableSlots: 8,
ExpectedPerFetch: 2, ExpectedSlotsPerFetch: 2,
}, budget, &recordingFetchNotifier{})
if err != nil {
t.Fatalf("NewReconciler(): %v", err)
}
decision := reconciler.Reconcile(time.Now(), upstream.Inventory{})
if decision.Triggered || decision.FetchAllowance != 0 {
t.Fatalf("Reconcile(legacy) = %+v, want preserved managed budget", decision)
}
if usage := budget.Snapshot(); usage.Managed != 9 {
t.Fatalf("FetchBudget.Managed = %d, want 9", usage.Managed)
}
}
func inventoryWithSlots(now time.Time, slots int64) upstream.Inventory {
return upstream.Inventory{Proxies: []upstream.ProxyCapacity{{
State: proxyDomain.StateAvailable, ExpiresAt: now.Add(time.Minute), Max: slots,

View File

@ -13,10 +13,13 @@ var (
ErrLeaderWorkStopped = errors.New("provider leader work stopped")
)
const MaximumCoordinationInteger = int64(1<<53 - 1)
type CoordinationLimits struct {
RequestInterval time.Duration
MaxInFlight int
MaxAttemptDuration time.Duration
MaxTotal int64
}
type Fence struct {
@ -32,10 +35,14 @@ type Coordinator interface {
type LeaderSession interface {
Fence() Fence
AcquireFetch(context.Context) (RequestPermit, error)
AcquireFetch(context.Context, int) (RequestPermit, bool, error)
}
type RequestPermit interface {
// Release is idempotent. A failed release expires automatically in storage.
Release(context.Context) error
// Complete atomically releases in-flight capacity and charges the actual
// successful candidate count. Repeated calls are idempotent.
Complete(context.Context, int) error
// Cancel releases a reservation that is known not to have consumed Provider
// quota. A crashed or abandoned reservation is conservatively charged.
Cancel(context.Context) error
}

View File

@ -0,0 +1,64 @@
package provider
import (
"context"
"errors"
"sort"
"proxy-pool/internal/platform/lifecycle"
)
var ErrInvalidFleet = errors.New("invalid Provider fleet")
type Fleet struct {
runtimes []*UpstreamRuntime
group *lifecycle.Group
}
func NewFleet(runtimes ...*UpstreamRuntime) (*Fleet, error) {
if len(runtimes) == 0 {
return nil, ErrInvalidFleet
}
seen := make(map[string]struct{}, len(runtimes))
owned := make([]*UpstreamRuntime, len(runtimes))
for index, runtime := range runtimes {
if runtime == nil || runtime.config.Provider.UpstreamID == "" {
return nil, ErrInvalidFleet
}
if _, exists := seen[runtime.config.Provider.UpstreamID]; exists {
return nil, ErrInvalidFleet
}
seen[runtime.config.Provider.UpstreamID] = struct{}{}
owned[index] = runtime
}
sort.Slice(owned, func(left, right int) bool {
return owned[left].ID() < owned[right].ID()
})
runners := make([]lifecycle.Runner, len(owned))
for index, runtime := range owned {
runners[index] = runtime
}
group, err := lifecycle.NewGroup(runners...)
if err != nil {
return nil, errors.Join(ErrInvalidFleet, err)
}
return &Fleet{runtimes: owned, group: group}, nil
}
func (fleet *Fleet) Run(ctx context.Context) error {
if fleet == nil || ctx == nil || len(fleet.runtimes) == 0 || fleet.group == nil {
return ErrInvalidFleet
}
return fleet.group.Run(ctx)
}
func (fleet *Fleet) IDs() []string {
if fleet == nil {
return nil
}
ids := make([]string, len(fleet.runtimes))
for index, runtime := range fleet.runtimes {
ids[index] = runtime.ID()
}
return ids
}

View File

@ -0,0 +1,101 @@
package provider
import (
"context"
"errors"
"testing"
"time"
"proxy-pool/internal/controller/pool"
"proxy-pool/internal/domain/activitypool"
proxyDomain "proxy-pool/internal/domain/proxy"
)
func TestFleetRunsAllUpstreamsAndCancelsSiblingsOnFailure(t *testing.T) {
started := make(chan string, 2)
siblingCancelled := make(chan struct{}, 1)
first := newFleetRuntime(t, "provider-a", coordinatorFunc(func(
context.Context, string, CoordinationLimits, func(context.Context, LeaderSession) error,
) error {
started <- "provider-a"
return errors.New("provider-a stopped")
}))
second := newFleetRuntime(t, "provider-b", coordinatorFunc(func(
ctx context.Context, _ string, _ CoordinationLimits, _ func(context.Context, LeaderSession) error,
) error {
started <- "provider-b"
<-ctx.Done()
siblingCancelled <- struct{}{}
return ctx.Err()
}))
fleet, err := NewFleet(first, second)
if err != nil {
t.Fatalf("NewFleet(): %v", err)
}
err = fleet.Run(context.Background())
if err == nil || err.Error() != "provider-a stopped" {
t.Fatalf("Run() error = %v, want provider-a failure", err)
}
seen := map[string]bool{<-started: true, <-started: true}
if !seen["provider-a"] || !seen["provider-b"] {
t.Fatalf("started upstreams = %v", seen)
}
select {
case <-siblingCancelled:
case <-time.After(time.Second):
t.Fatal("sibling runtime was not cancelled")
}
}
func TestNewFleetRejectsEmptyNilAndDuplicateUpstreams(t *testing.T) {
valid := newFleetRuntime(t, "provider-a", coordinatorFunc(func(
ctx context.Context, _ string, _ CoordinationLimits, _ func(context.Context, LeaderSession) error,
) error {
return ctx.Err()
}))
tests := []struct {
name string
runtimes []*UpstreamRuntime
}{
{name: "empty"},
{name: "nil", runtimes: []*UpstreamRuntime{nil}},
{name: "duplicate", runtimes: []*UpstreamRuntime{valid, valid}},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
fleet, err := NewFleet(test.runtimes...)
if err == nil || fleet != nil {
t.Fatalf("NewFleet() = (%v, %v), want invalid fleet", fleet, err)
}
})
}
}
func newFleetRuntime(t *testing.T, upstreamID string, coordinator Coordinator) *UpstreamRuntime {
t.Helper()
runtime, err := NewUpstreamRuntime(UpstreamRuntimeConfig{
Provider: runtimeProviderConfig(upstreamID),
ReconcilePolicy: runtimeReconcilePolicy(),
ReconcileInterval: time.Second,
}, UpstreamRuntimeDependencies{
Coordinator: coordinator,
Inventory: inventoryReaderFunc(func(context.Context, string, time.Duration) (pool.InventorySnapshot, error) {
return pool.InventorySnapshot{}, nil
}),
Adapter: adapterFunc(func(context.Context) (FetchResponse, error) {
return FetchResponse{Body: []byte("fixture")}, nil
}),
Parser: parserFunc(func(context.Context, []byte) ([]proxyDomain.Proxy, error) {
return []proxyDomain.Proxy{{ID: "proxy-1"}}, nil
}),
Activity: activitySinkFunc(func(context.Context, string, activitypool.FetchedBatch) (activitypool.UpsertResult, error) {
return activitypool.UpsertResult{Accepted: 1, Inserted: 1}, nil
}),
Results: resultRecorderFunc(func(Result) {}),
})
if err != nil {
t.Fatalf("NewUpstreamRuntime(%s): %v", upstreamID, err)
}
return runtime
}

View File

@ -27,9 +27,11 @@ type RetryableError interface {
Retryable() bool
}
// Parser must be safe for concurrent calls and must honor context cancellation.
// Parser owns any transient resources attached to parsed candidates. It must
// be safe for concurrent calls and must honor context cancellation.
type Parser interface {
Parse(context.Context, []byte) ([]proxyDomain.Proxy, error)
ReleaseCandidates([]proxyDomain.Proxy)
}
type Result struct {
@ -42,7 +44,8 @@ type Result struct {
}
type ResultRecorder interface {
// Record may be called concurrently and must not retain mutable result data.
// Record may be called concurrently, must return promptly, and must not
// retain mutable result data.
Record(Result)
}

View File

@ -19,6 +19,7 @@ type Config struct {
Timeout time.Duration
MaxAttempts int
MaxInFlight int
MaxTotal int64
MaxSize int
TTL time.Duration
AllocationSafetyMargin time.Duration
@ -31,6 +32,8 @@ type RetryConfig struct {
Jitter int
}
const requestPermitSettlementTimeout = 5 * time.Second
type Reconciler struct {
config Config
ports Ports
@ -64,7 +67,7 @@ func NewReconciler(config Config, ports Ports, runtimes ...Runtime) (*Reconciler
return nil, fmt.Errorf("new provider reconciler: upstream ID is required")
}
if config.RequestInterval < 0 || config.Timeout <= 0 || config.MaxAttempts <= 0 ||
config.MaxInFlight <= 0 || config.MaxSize <= 0 {
config.MaxInFlight <= 0 || config.MaxTotal < 0 || config.MaxSize <= 0 {
return nil, fmt.Errorf("new provider reconciler: fetch limits must be positive")
}
if config.TTL < 0 || config.AllocationSafetyMargin < 0 ||
@ -112,7 +115,10 @@ func (r *Reconciler) Notify() {
r.signal.Notify()
}
func (r *Reconciler) Run(ctx context.Context) error {
func (r *Reconciler) RunLeader(ctx context.Context, session LeaderSession) error {
if ctx == nil || session == nil {
return ErrInvalidCoordination
}
var workers sync.WaitGroup
defer workers.Wait()
for {
@ -131,17 +137,57 @@ func (r *Reconciler) Run(ctx context.Context) error {
go func() {
defer workers.Done()
defer func() { <-r.inFlight }()
r.reconcile(ctx)
r.reconcile(ctx, session)
}()
}
}
func (r *Reconciler) reconcile(ctx context.Context) {
func (r *Reconciler) reconcile(ctx context.Context, session LeaderSession) {
permit, available, err := r.ports.Capacity.ReserveFetch(r.config.UpstreamID)
if err != nil {
r.ports.Results.Record(Result{
UpstreamID: r.config.UpstreamID,
Class: upstream.FetchError,
Err: fmt.Errorf("reserve fetch capacity: %w", err),
Attempt: 1,
})
return
}
if !available {
return
}
if permit == nil || permit.Expected() <= 0 {
if permit != nil {
_ = permit.Cancel()
}
r.ports.Results.Record(Result{
UpstreamID: r.config.UpstreamID,
Class: upstream.FetchError,
Err: fmt.Errorf("reserve fetch capacity: invalid permit"),
Attempt: 1,
})
return
}
permitFinished := false
defer func() {
if !permitFinished {
_ = permit.Cancel()
}
}()
for attempt := 1; attempt <= r.config.MaxAttempts; attempt++ {
response, result, retryable, ok := r.fetchAttempt(ctx, attempt)
response, result, retryable, ok := r.fetchAttempt(ctx, session, permit, attempt)
if !ok {
return
}
if result.Class != upstream.FetchError {
if capacityErr := permit.Complete(result.NewCount); capacityErr != nil {
result.Class = upstream.FetchError
result.Err = errors.Join(result.Err, capacityErr)
} else {
permitFinished = true
}
}
r.ports.Results.Record(result)
if result.Class != upstream.FetchError || !retryable || attempt == r.config.MaxAttempts {
return
@ -157,38 +203,42 @@ func (r *Reconciler) reconcile(ctx context.Context) {
}
}
func (r *Reconciler) fetchAttempt(ctx context.Context, attempt int) (FetchResponse, Result, bool, bool) {
func (r *Reconciler) fetchAttempt(
ctx context.Context,
session LeaderSession,
permit upstream.FetchPermit,
attempt int,
) (FetchResponse, Result, bool, bool) {
if err := r.waitForRequestSlot(ctx); err != nil {
return FetchResponse{}, Result{}, false, false
}
permit, available, err := r.ports.Capacity.ReserveFetch(r.config.UpstreamID)
requestPermit, available, err := session.AcquireFetch(ctx, permit.Expected())
if err != nil {
resultErr := fmt.Errorf("reserve fetch capacity: %w", err)
if ctx.Err() != nil || errors.Is(err, ErrLeadershipLost) {
return FetchResponse{}, Result{}, false, false
}
return FetchResponse{}, Result{
UpstreamID: r.config.UpstreamID,
Class: upstream.FetchError,
Err: resultErr,
Err: fmt.Errorf("acquire distributed fetch capacity: %w", err),
Attempt: attempt,
}, false, true
}
if !available {
return FetchResponse{}, Result{}, false, false
}
if permit == nil || permit.Expected() <= 0 {
if permit != nil {
_ = permit.Cancel()
}
if requestPermit == nil {
return FetchResponse{}, Result{
UpstreamID: r.config.UpstreamID,
Class: upstream.FetchError,
Err: fmt.Errorf("reserve fetch capacity: invalid permit"),
Err: fmt.Errorf("acquire distributed fetch capacity: invalid permit"),
Attempt: attempt,
}, false, true
}
permitFinished := false
requestSettlementAttempted := false
defer func() {
if !permitFinished {
_ = permit.Cancel()
if !requestSettlementAttempted {
_ = r.settleRequestPermit(requestPermit, false, 0)
}
}()
@ -200,13 +250,23 @@ func (r *Reconciler) fetchAttempt(ctx context.Context, attempt int) (FetchRespon
defer cancel()
response, callErr := r.ports.Adapter.Fetch(callCtx)
var parseErr, candidateErr, capacityErr error
var parseErr, candidateErr, coordinationErr error
var validCount, newCount int
if callErr == nil {
if callErr != nil {
requestSettlementAttempted = true
coordinationErr = r.settleRequestPermit(requestPermit, true, permit.Expected())
} else {
candidates, err := r.ports.Parser.Parse(callCtx, response.Body)
defer r.ports.Parser.ReleaseCandidates(candidates)
parseErr = err
validCount = len(candidates)
if parseErr == nil && validCount > 0 {
charged := validCount
if parseErr != nil {
charged = permit.Expected()
}
requestSettlementAttempted = true
coordinationErr = r.settleRequestPermit(requestPermit, true, charged)
if parseErr == nil && coordinationErr == nil && validCount > 0 {
retained := candidates
if expected := permit.Expected(); expected < len(retained) {
retained = retained[:expected]
@ -222,12 +282,13 @@ func (r *Reconciler) fetchAttempt(ctx context.Context, attempt int) (FetchRespon
newCount = upserted.Inserted
}
}
if callErr == nil && parseErr == nil && candidateErr == nil {
capacityErr = permit.Complete(validCount, newCount)
permitFinished = capacityErr == nil
}
resultErr := errors.Join(callErr, parseErr, candidateErr, capacityErr)
class := upstream.ClassifyFetchResult(callErr, errors.Join(parseErr, candidateErr, capacityErr), validCount, newCount)
resultErr := errors.Join(callErr, parseErr, candidateErr, coordinationErr)
class := upstream.ClassifyFetchResult(
callErr,
errors.Join(parseErr, candidateErr, coordinationErr),
validCount,
newCount,
)
return response, Result{
UpstreamID: r.config.UpstreamID,
Class: class,
@ -238,6 +299,15 @@ func (r *Reconciler) fetchAttempt(ctx context.Context, attempt int) (FetchRespon
}, isRetryable(callErr) || parseErr != nil, true
}
func (r *Reconciler) settleRequestPermit(permit RequestPermit, complete bool, fetched int) error {
ctx, cancel := context.WithTimeout(context.Background(), requestPermitSettlementTimeout)
defer cancel()
if complete {
return permit.Complete(ctx, fetched)
}
return permit.Cancel(ctx)
}
func isRetryable(err error) bool {
if err == nil {
return false

View File

@ -8,6 +8,7 @@ import (
"testing"
"time"
controllerPool "proxy-pool/internal/controller/pool"
"proxy-pool/internal/domain/activitypool"
proxyDomain "proxy-pool/internal/domain/proxy"
"proxy-pool/internal/domain/upstream"
@ -89,7 +90,7 @@ func TestReconcilerCoalescesConcurrentNotifications(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
done := make(chan error, 1)
go func() { done <- reconciler.Run(ctx) }()
go func() { done <- reconciler.RunLeader(ctx, unlimitedLeaderSession{}) }()
select {
case got := <-result:
@ -130,7 +131,7 @@ func TestReconcilerEnforcesRequestInterval(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
done := make(chan error, 1)
go func() { done <- reconciler.Run(ctx) }()
go func() { done <- reconciler.RunLeader(ctx, unlimitedLeaderSession{}) }()
reconciler.Notify()
<-results
@ -183,7 +184,7 @@ func TestReconcilerRetriesErrorsWithExponentialBackoffAndJitter(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
done := make(chan error, 1)
go func() { done <- reconciler.Run(ctx) }()
go func() { done <- reconciler.RunLeader(ctx, unlimitedLeaderSession{}) }()
reconciler.Notify()
wantClasses := []upstream.FetchClass{
@ -291,7 +292,7 @@ func TestReconcilerHonorsRetryAfterBeforeBackoff(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
done := make(chan error, 1)
go func() { done <- reconciler.Run(ctx) }()
go func() { done <- reconciler.RunLeader(ctx, unlimitedLeaderSession{}) }()
reconciler.Notify()
<-results
<-results
@ -332,7 +333,7 @@ func TestReconcilerCapsRetryAfter(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
done := make(chan error, 1)
go func() { done <- reconciler.Run(ctx) }()
go func() { done <- reconciler.RunLeader(ctx, unlimitedLeaderSession{}) }()
reconciler.Notify()
<-results
<-results
@ -417,7 +418,7 @@ func TestReconcilerDropsNotificationFanoutWhileFetchIsInFlight(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
done := make(chan error, 1)
go func() { done <- reconciler.Run(ctx) }()
go func() { done <- reconciler.RunLeader(ctx, unlimitedLeaderSession{}) }()
reconciler.Notify()
<-started
@ -473,8 +474,8 @@ func TestReconcilerEnforcesMaxInFlightAcrossRunConsumers(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
done := make(chan error, 2)
go func() { done <- reconciler.Run(ctx) }()
go func() { done <- reconciler.Run(ctx) }()
go func() { done <- reconciler.RunLeader(ctx, unlimitedLeaderSession{}) }()
go func() { done <- reconciler.RunLeader(ctx, unlimitedLeaderSession{}) }()
reconciler.Notify()
<-started
@ -519,7 +520,7 @@ func TestReconcilerUsesConfiguredMaxInFlight(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
done := make(chan error, 1)
go func() { done <- reconciler.Run(ctx) }()
go func() { done <- reconciler.RunLeader(ctx, unlimitedLeaderSession{}) }()
reconciler.Notify()
<-started
reconciler.Notify()
@ -545,6 +546,7 @@ func TestReconcilerDoesNotRefetchWhenActivitySinkFails(t *testing.T) {
clock := newFakeClock(time.Date(2026, 7, 28, 12, 0, 0, 0, time.UTC))
sleeper := &fakeSleeper{clock: clock}
results := make(chan Result, 3)
globalCompleted := make(chan int, 1)
var calls atomic.Int64
ports := successfulPorts(func() { calls.Add(1) }, results)
ports.Activity = activitySinkFunc(func(context.Context, string, activitypool.FetchedBatch) (activitypool.UpsertResult, error) {
@ -562,13 +564,20 @@ func TestReconcilerDoesNotRefetchWhenActivitySinkFails(t *testing.T) {
t.Fatalf("NewReconciler(): %v", err)
}
result := runSingleReconcile(t, reconciler, results)
result := runSingleReconcile(t, reconciler, results, leaderSessionFunc(
func(context.Context, int) (RequestPermit, bool, error) {
return &recordingRequestPermit{completed: globalCompleted}, true, nil
},
))
if result.Class != upstream.FetchError {
t.Fatalf("result class = %q, want %q", result.Class, upstream.FetchError)
}
if got := calls.Load(); got != 1 {
t.Fatalf("ProviderAdapter.Fetch() calls = %d, want 1", got)
}
if got := <-globalCompleted; got != 1 {
t.Fatalf("global charged count = %d, want fetched=1", got)
}
if got := sleeper.Durations(); len(got) != 0 {
t.Fatalf("Sleep durations = %v, want no retry backoff", got)
}
@ -623,7 +632,7 @@ func TestReconcilerDoesNotCallProviderWithoutFetchCapacity(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
done := make(chan error, 1)
go func() { done <- reconciler.Run(ctx) }()
go func() { done <- reconciler.RunLeader(ctx, unlimitedLeaderSession{}) }()
reconciler.Notify()
time.Sleep(20 * time.Millisecond)
cancel()
@ -640,13 +649,116 @@ func TestReconcilerDoesNotCallProviderWithoutFetchCapacity(t *testing.T) {
}
}
func TestReconcilerCompletesFetchPermitWithFetchedAndRetainedCounts(t *testing.T) {
func TestReconcilerDoesNotCallProviderWhenDistributedQuotaIsExhausted(t *testing.T) {
results := make(chan Result, 1)
completed := make(chan fetchCompletion, 1)
var calls atomic.Int64
ports := successfulPorts(func() { calls.Add(1) }, results)
ports.Capacity = fetchCapacityFunc(func(string) (upstream.FetchPermit, bool, error) {
return &recordingFetchPermit{expected: 2}, true, nil
})
reconciler, err := NewReconciler(Config{
UpstreamID: "provider-a", Timeout: time.Second, MaxAttempts: 1, MaxInFlight: 1, MaxSize: 100,
}, ports)
if err != nil {
t.Fatalf("NewReconciler(): %v", err)
}
session := leaderSessionFunc(func(_ context.Context, expected int) (RequestPermit, bool, error) {
if expected != 2 {
t.Errorf("distributed expected = %d, want 2", expected)
}
return nil, false, nil
})
ctx, cancel := context.WithCancel(context.Background())
done := make(chan error, 1)
go func() { done <- reconciler.RunLeader(ctx, session) }()
reconciler.Notify()
time.Sleep(20 * time.Millisecond)
cancel()
if err := <-done; err != nil {
t.Fatalf("RunLeader(): %v", err)
}
if got := calls.Load(); got != 0 {
t.Fatalf("ProviderAdapter.Fetch() calls = %d, want 0", got)
}
select {
case result := <-results:
t.Fatalf("unexpected fetch result: %+v", result)
default:
}
}
func TestReconcilerChargesExpectedWhenSuccessfulResponseCannotBeParsed(t *testing.T) {
results := make(chan Result, 1)
globalCompleted := make(chan int, 1)
localCancelled := make(chan struct{}, 1)
ports := successfulPorts(func() {}, results)
ports.Parser = parserFunc(func(context.Context, []byte) ([]proxyDomain.Proxy, error) {
return []proxyDomain.Proxy{{ID: "p1"}, {ID: "p2"}, {ID: "p3"}}, nil
return nil, errors.New("invalid provider payload")
})
ports.Capacity = fetchCapacityFunc(func(string) (upstream.FetchPermit, bool, error) {
return &recordingFetchPermit{expected: 2, cancelled: localCancelled}, true, nil
})
reconciler, err := NewReconciler(Config{
UpstreamID: "provider-a", Timeout: time.Second, MaxAttempts: 1, MaxInFlight: 1, MaxSize: 100,
}, ports)
if err != nil {
t.Fatalf("NewReconciler(): %v", err)
}
session := leaderSessionFunc(func(context.Context, int) (RequestPermit, bool, error) {
return &recordingRequestPermit{completed: globalCompleted}, true, nil
})
result := runSingleReconcile(t, reconciler, results, session)
if result.Class != upstream.FetchError {
t.Fatalf("result class = %q, want %q", result.Class, upstream.FetchError)
}
if got := <-globalCompleted; got != 2 {
t.Fatalf("global charged count = %d, want expected=2", got)
}
select {
case <-localCancelled:
default:
t.Fatal("local pool reservation was not cancelled")
}
}
func TestReconcilerConservativelyChargesExpectedWhenProviderCallOutcomeIsUnknown(t *testing.T) {
results := make(chan Result, 1)
globalCompleted := make(chan int, 1)
ports := successfulPorts(func() {}, results)
ports.Adapter = adapterFunc(func(context.Context) (FetchResponse, error) {
return FetchResponse{}, errors.New("provider connection failed")
})
reconciler, err := NewReconciler(Config{
UpstreamID: "provider-a", Timeout: time.Second, MaxAttempts: 1, MaxInFlight: 1, MaxSize: 100,
}, ports)
if err != nil {
t.Fatalf("NewReconciler(): %v", err)
}
session := leaderSessionFunc(func(context.Context, int) (RequestPermit, bool, error) {
return &recordingRequestPermit{completed: globalCompleted}, true, nil
})
result := runSingleReconcile(t, reconciler, results, session)
if result.Class != upstream.FetchError {
t.Fatalf("result class = %q, want %q", result.Class, upstream.FetchError)
}
if got := <-globalCompleted; got <= 0 {
t.Fatalf("global charged count = %d, want conservative expected count", got)
}
}
func TestReconcilerChargesFetchedGloballyAndCompletesRetainedLocally(t *testing.T) {
results := make(chan Result, 1)
localCompleted := make(chan fetchCompletion, 1)
globalCompleted := make(chan int, 1)
released := make(chan []proxyDomain.Proxy, 1)
ports := successfulPorts(func() {}, results)
ports.Parser = &recordingCandidateParser{
candidates: []proxyDomain.Proxy{{ID: "p1"}, {ID: "p2"}, {ID: "p3"}},
released: released,
}
ports.Activity = activitySinkFunc(func(_ context.Context, _ string, batch activitypool.FetchedBatch) (activitypool.UpsertResult, error) {
if len(batch.Proxies) != 2 {
t.Errorf("activity batch proxies = %d, want permit limit 2", len(batch.Proxies))
@ -654,7 +766,7 @@ func TestReconcilerCompletesFetchPermitWithFetchedAndRetainedCounts(t *testing.T
return activitypool.UpsertResult{Accepted: 2, Inserted: 1}, nil
})
ports.Capacity = fetchCapacityFunc(func(string) (upstream.FetchPermit, bool, error) {
return &recordingFetchPermit{expected: 2, completed: completed}, true, nil
return &recordingFetchPermit{expected: 2, completed: localCompleted}, true, nil
})
reconciler, err := NewReconciler(Config{
UpstreamID: "provider-a", Timeout: time.Second, MaxAttempts: 1, MaxInFlight: 1, MaxSize: 100,
@ -663,12 +775,30 @@ func TestReconcilerCompletesFetchPermitWithFetchedAndRetainedCounts(t *testing.T
t.Fatalf("NewReconciler(): %v", err)
}
result := runSingleReconcile(t, reconciler, results)
result := runSingleReconcile(t, reconciler, results, leaderSessionFunc(
func(_ context.Context, expected int) (RequestPermit, bool, error) {
if expected != 2 {
t.Errorf("distributed expected = %d, want 2", expected)
}
return &recordingRequestPermit{completed: globalCompleted}, true, nil
},
))
if result.ValidCount != 3 || result.NewCount != 1 {
t.Fatalf("result = %+v, want valid=3 new=1", result)
}
if got := <-completed; got.fetched != 3 || got.retained != 1 {
t.Fatalf("fetch completion = %+v, want fetched=3 retained=1", got)
if got := <-globalCompleted; got != 3 {
t.Fatalf("global fetch completion = %d, want fetched=3", got)
}
if got := <-localCompleted; got.retained != 1 {
t.Fatalf("local fetch completion = %+v, want retained=1", got)
}
select {
case got := <-released:
if len(got) != 3 {
t.Fatalf("released candidates = %d, want all 3 parsed candidates", len(got))
}
case <-time.After(time.Second):
t.Fatal("parsed candidates were not released")
}
}
@ -699,7 +829,7 @@ func TestReconcilerRetriesParserErrorsAsFetchErrors(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
done := make(chan error, 1)
go func() { done <- reconciler.Run(ctx) }()
go func() { done <- reconciler.RunLeader(ctx, unlimitedLeaderSession{}) }()
reconciler.Notify()
first, second := <-results, <-results
cancel()
@ -717,6 +847,53 @@ func TestReconcilerRetriesParserErrorsAsFetchErrors(t *testing.T) {
}
}
func TestReconcilerKeepsLocalPoolReservationAcrossRetryBackoff(t *testing.T) {
results := make(chan Result, 2)
budget, err := controllerPool.NewFetchBudget(controllerPool.FetchBudgetConfig{
UpstreamID: "provider-a", MaxSize: 10, ExpectedPerFetch: 1,
})
if err != nil {
t.Fatalf("NewFetchBudget(): %v", err)
}
var parses atomic.Int64
ports := successfulPorts(func() {}, results)
ports.Capacity = budget
ports.Parser = parserFunc(func(context.Context, []byte) ([]proxyDomain.Proxy, error) {
if parses.Add(1) == 1 {
return nil, errors.New("temporary parser failure")
}
return []proxyDomain.Proxy{{ID: "proxy-1"}}, nil
})
reconciler, err := NewReconciler(Config{
UpstreamID: "provider-a", Timeout: time.Second, MaxAttempts: 2,
MaxInFlight: 1, MaxSize: 10,
Retry: RetryConfig{Initial: time.Millisecond, Max: time.Millisecond},
}, ports, Runtime{Sleeper: sleeperFunc(func(context.Context, time.Duration) error {
if got := budget.Snapshot().PendingExpected; got != 1 {
t.Errorf("PendingExpected during retry backoff = %d, want 1", got)
}
return nil
})})
if err != nil {
t.Fatalf("NewReconciler(): %v", err)
}
ctx, cancel := context.WithCancel(context.Background())
done := make(chan error, 1)
go func() { done <- reconciler.RunLeader(ctx, unlimitedLeaderSession{}) }()
reconciler.Notify()
<-results
<-results
cancel()
if err := <-done; err != nil {
t.Fatalf("RunLeader(): %v", err)
}
usage := budget.Snapshot()
if usage.PendingExpected != 0 || usage.Managed != 1 {
t.Fatalf("FetchBudget snapshot = %+v, want pending=0 managed=1", usage)
}
}
func TestNewReconcilerRejectsInvalidSchedulingConfiguration(t *testing.T) {
results := make(chan Result, 1)
ports := successfulPorts(func() {}, results)
@ -744,11 +921,23 @@ func TestNewReconcilerRejectsInvalidSchedulingConfiguration(t *testing.T) {
}
}
func runSingleReconcile(t *testing.T, reconciler *Reconciler, results <-chan Result) Result {
func runSingleReconcile(
t *testing.T,
reconciler *Reconciler,
results <-chan Result,
sessions ...LeaderSession,
) Result {
t.Helper()
session := LeaderSession(unlimitedLeaderSession{})
if len(sessions) > 1 {
t.Fatal("runSingleReconcile accepts at most one LeaderSession")
}
if len(sessions) == 1 {
session = sessions[0]
}
ctx, cancel := context.WithCancel(context.Background())
done := make(chan error, 1)
go func() { done <- reconciler.Run(ctx) }()
go func() { done <- reconciler.RunLeader(ctx, session) }()
reconciler.Notify()
var result Result
select {
@ -821,6 +1010,12 @@ func (s *errorSleeper) Sleep(context.Context, time.Duration) error {
return errors.New("unexpected sleep")
}
type sleeperFunc func(context.Context, time.Duration) error
func (f sleeperFunc) Sleep(ctx context.Context, duration time.Duration) error {
return f(ctx, duration)
}
func (s *fakeSleeper) Sleep(ctx context.Context, duration time.Duration) error {
if err := ctx.Err(); err != nil {
return err
@ -848,6 +1043,21 @@ func (f parserFunc) Parse(ctx context.Context, body []byte) ([]proxyDomain.Proxy
return f(ctx, body)
}
func (parserFunc) ReleaseCandidates([]proxyDomain.Proxy) {}
type recordingCandidateParser struct {
candidates []proxyDomain.Proxy
released chan<- []proxyDomain.Proxy
}
func (parser *recordingCandidateParser) Parse(context.Context, []byte) ([]proxyDomain.Proxy, error) {
return append([]proxyDomain.Proxy(nil), parser.candidates...), nil
}
func (parser *recordingCandidateParser) ReleaseCandidates(candidates []proxyDomain.Proxy) {
parser.released <- append([]proxyDomain.Proxy(nil), candidates...)
}
type activitySinkFunc func(context.Context, string, activitypool.FetchedBatch) (activitypool.UpsertResult, error)
func (f activitySinkFunc) UpsertFetched(ctx context.Context, upstreamID string, batch activitypool.FetchedBatch) (activitypool.UpsertResult, error) {
@ -867,26 +1077,66 @@ func (unlimitedFetchCapacity) ReserveFetch(string) (upstream.FetchPermit, bool,
}
type fetchCompletion struct {
fetched int
retained int
}
type recordingFetchPermit struct {
expected int
completed chan<- fetchCompletion
cancelled chan<- struct{}
}
func (p *recordingFetchPermit) Expected() int { return p.expected }
func (p *recordingFetchPermit) Complete(fetched, retained int) error {
func (p *recordingFetchPermit) Complete(retained int) error {
if p.completed != nil {
p.completed <- fetchCompletion{fetched: fetched, retained: retained}
p.completed <- fetchCompletion{retained: retained}
}
return nil
}
func (*recordingFetchPermit) Cancel() error { return nil }
func (p *recordingFetchPermit) Cancel() error {
if p.cancelled != nil {
p.cancelled <- struct{}{}
}
return nil
}
type resultRecorderFunc func(Result)
func (f resultRecorderFunc) Record(result Result) { f(result) }
type unlimitedLeaderSession struct{}
func (unlimitedLeaderSession) Fence() Fence { return Fence{Generation: "test", Epoch: 1} }
func (unlimitedLeaderSession) AcquireFetch(context.Context, int) (RequestPermit, bool, error) {
return &recordingRequestPermit{}, true, nil
}
type leaderSessionFunc func(context.Context, int) (RequestPermit, bool, error)
func (leaderSessionFunc) Fence() Fence { return Fence{Generation: "test", Epoch: 1} }
func (f leaderSessionFunc) AcquireFetch(ctx context.Context, expected int) (RequestPermit, bool, error) {
return f(ctx, expected)
}
type recordingRequestPermit struct {
completed chan<- int
cancelled chan<- struct{}
}
func (p *recordingRequestPermit) Complete(_ context.Context, fetched int) error {
if p.completed != nil {
p.completed <- fetched
}
return nil
}
func (p *recordingRequestPermit) Cancel(context.Context) error {
if p.cancelled != nil {
p.cancelled <- struct{}{}
}
return nil
}

View File

@ -0,0 +1,103 @@
package provider
import (
"errors"
"math"
"sync"
"proxy-pool/internal/domain/upstream"
)
var ErrInvalidStatsRecorder = errors.New("invalid Provider stats recorder")
type Stats struct {
UpstreamID string
ConsecutiveEmptyFetch int64
FetchErrorCount int64
}
type StatsReader interface {
ReadProviderStats([]string) []Stats
}
type StatsRetainer interface {
RetainProviderStats([]string)
}
type StatsRecorder struct {
mu sync.Mutex
maximum int
byID map[string]Stats
}
func NewStatsRecorder(maximum int) (*StatsRecorder, error) {
if maximum <= 0 {
return nil, ErrInvalidStatsRecorder
}
return &StatsRecorder{maximum: maximum, byID: make(map[string]Stats)}, nil
}
func (recorder *StatsRecorder) Record(result Result) {
if recorder == nil || result.UpstreamID == "" {
return
}
recorder.mu.Lock()
defer recorder.mu.Unlock()
stats, exists := recorder.byID[result.UpstreamID]
if !exists {
if len(recorder.byID) >= recorder.maximum {
return
}
stats.UpstreamID = result.UpstreamID
}
switch result.Class {
case upstream.FetchEmpty:
if stats.ConsecutiveEmptyFetch < math.MaxInt64 {
stats.ConsecutiveEmptyFetch++
}
case upstream.FetchValid, upstream.FetchDuplicateOnly:
stats.ConsecutiveEmptyFetch = 0
case upstream.FetchError:
if stats.FetchErrorCount < math.MaxInt64 {
stats.FetchErrorCount++
}
default:
return
}
recorder.byID[result.UpstreamID] = stats
}
func (recorder *StatsRecorder) ReadProviderStats(upstreamIDs []string) []Stats {
result := make([]Stats, len(upstreamIDs))
if recorder == nil {
return result
}
recorder.mu.Lock()
defer recorder.mu.Unlock()
for index, upstreamID := range upstreamIDs {
result[index] = recorder.byID[upstreamID]
result[index].UpstreamID = upstreamID
}
return result
}
// RetainProviderStats removes observations for upstreams no longer present in
// the complete configuration. Disabled but configured upstreams must be kept.
func (recorder *StatsRecorder) RetainProviderStats(upstreamIDs []string) {
if recorder == nil {
return
}
retained := make(map[string]struct{}, len(upstreamIDs))
for _, upstreamID := range upstreamIDs {
if upstreamID != "" {
retained[upstreamID] = struct{}{}
}
}
recorder.mu.Lock()
defer recorder.mu.Unlock()
for upstreamID := range recorder.byID {
if _, keep := retained[upstreamID]; !keep {
delete(recorder.byID, upstreamID)
}
}
}

View File

@ -0,0 +1,83 @@
package provider
import (
"sync"
"testing"
"proxy-pool/internal/domain/upstream"
)
func TestStatsRecorderTracksEmptyResetAndErrors(t *testing.T) {
recorder, err := NewStatsRecorder(2)
if err != nil {
t.Fatalf("NewStatsRecorder(): %v", err)
}
for _, class := range []upstream.FetchClass{
upstream.FetchEmpty,
upstream.FetchEmpty,
upstream.FetchError,
} {
recorder.Record(Result{UpstreamID: "provider-a", Class: class})
}
stats := recorder.ReadProviderStats([]string{"provider-a"})[0]
if stats.ConsecutiveEmptyFetch != 2 || stats.FetchErrorCount != 1 {
t.Fatalf("stats = %+v, want empty=2 errors=1", stats)
}
recorder.Record(Result{UpstreamID: "provider-a", Class: upstream.FetchDuplicateOnly})
if got := recorder.ReadProviderStats([]string{"provider-a"})[0].ConsecutiveEmptyFetch; got != 0 {
t.Fatalf("consecutive empty after duplicate = %d, want 0", got)
}
}
func TestStatsRecorderIsBoundedAndConcurrent(t *testing.T) {
recorder, err := NewStatsRecorder(1)
if err != nil {
t.Fatalf("NewStatsRecorder(): %v", err)
}
const workers = 100
var wait sync.WaitGroup
for range workers {
wait.Add(1)
go func() {
defer wait.Done()
recorder.Record(Result{UpstreamID: "provider-a", Class: upstream.FetchError})
}()
}
wait.Wait()
recorder.Record(Result{UpstreamID: "provider-b", Class: upstream.FetchError})
stats := recorder.ReadProviderStats([]string{"provider-a", "provider-b"})
if stats[0].FetchErrorCount != workers || stats[1].FetchErrorCount != 0 {
t.Fatalf("stats = %+v, want bounded provider-a errors", stats)
}
}
func TestStatsRecorderRetainsConfiguredProvidersAndReusesCapacity(t *testing.T) {
recorder, err := NewStatsRecorder(2)
if err != nil {
t.Fatalf("NewStatsRecorder(): %v", err)
}
recorder.Record(Result{UpstreamID: "removed-a", Class: upstream.FetchError})
recorder.Record(Result{UpstreamID: "provider-b", Class: upstream.FetchError})
const workers = 100
var wait sync.WaitGroup
for range workers {
wait.Add(2)
go func() {
defer wait.Done()
recorder.Record(Result{UpstreamID: "provider-b", Class: upstream.FetchError})
}()
go func() {
defer wait.Done()
recorder.RetainProviderStats([]string{"provider-a", "provider-b"})
}()
}
wait.Wait()
recorder.RetainProviderStats([]string{"provider-a", "provider-b"})
recorder.Record(Result{UpstreamID: "provider-a", Class: upstream.FetchError})
stats := recorder.ReadProviderStats([]string{"removed-a", "provider-a", "provider-b"})
if stats[0].FetchErrorCount != 0 || stats[1].FetchErrorCount != 1 || stats[2].FetchErrorCount == 0 {
t.Fatalf("stats after retention = %+v", stats)
}
}

View File

@ -0,0 +1,219 @@
package provider
import (
"context"
"errors"
"fmt"
"hash/fnv"
"reflect"
"time"
"proxy-pool/internal/controller/pool"
"proxy-pool/internal/domain/activitypool"
)
var (
ErrInvalidUpstreamRuntime = errors.New("invalid Provider upstream runtime")
ErrUpstreamRuntimeStopped = errors.New("Provider upstream runtime stopped")
)
type UpstreamRuntimeConfig struct {
Provider Config
ReconcilePolicy pool.ReconcilePolicy
ReconcileInterval time.Duration
}
type UpstreamRuntimeDependencies struct {
Coordinator Coordinator
Inventory pool.InventoryReader
Adapter ProviderAdapter
Parser Parser
Activity activitypool.Upserter
Results ResultRecorder
}
// UpstreamRuntime owns every leader-scoped object for one Upstream. A fresh
// local budget and coalescing signal are created for each leadership term.
type UpstreamRuntime struct {
config UpstreamRuntimeConfig
dependencies UpstreamRuntimeDependencies
sleeper Sleeper
}
func NewUpstreamRuntime(
config UpstreamRuntimeConfig,
dependencies UpstreamRuntimeDependencies,
) (*UpstreamRuntime, error) {
if config.ReconcileInterval <= 0 || nilRuntimeDependency(dependencies.Coordinator) ||
nilRuntimeDependency(dependencies.Inventory) || nilRuntimeDependency(dependencies.Adapter) ||
nilRuntimeDependency(dependencies.Parser) || nilRuntimeDependency(dependencies.Activity) ||
nilRuntimeDependency(dependencies.Results) {
return nil, ErrInvalidUpstreamRuntime
}
runtime := &UpstreamRuntime{
config: config, dependencies: dependencies, sleeper: timerSleeper{},
}
if _, err := runtime.newLeaderTerm(); err != nil {
return nil, errors.Join(ErrInvalidUpstreamRuntime, err)
}
return runtime, nil
}
func (runtime *UpstreamRuntime) Run(ctx context.Context) error {
if runtime == nil || ctx == nil {
return ErrInvalidUpstreamRuntime
}
limits := CoordinationLimits{
RequestInterval: runtime.config.Provider.RequestInterval,
MaxInFlight: runtime.config.Provider.MaxInFlight,
MaxAttemptDuration: runtime.config.Provider.Timeout,
MaxTotal: runtime.config.Provider.MaxTotal,
}
err := runtime.dependencies.Coordinator.RunLeader(
ctx,
runtime.config.Provider.UpstreamID,
limits,
func(leaderCtx context.Context, session LeaderSession) error {
term, buildErr := runtime.newLeaderTerm()
if buildErr != nil {
return buildErr
}
return term.run(leaderCtx, session)
},
)
if ctx.Err() != nil {
return ctx.Err()
}
if err == nil {
return ErrUpstreamRuntimeStopped
}
return err
}
func (runtime *UpstreamRuntime) ID() string {
if runtime == nil {
return ""
}
return runtime.config.Provider.UpstreamID
}
func (runtime *UpstreamRuntime) newLeaderTerm() (*upstreamLeaderTerm, error) {
budget, err := pool.NewFetchBudget(pool.FetchBudgetConfig{
UpstreamID: runtime.config.Provider.UpstreamID,
MaxSize: runtime.config.Provider.MaxSize,
ExpectedPerFetch: runtime.config.ReconcilePolicy.ExpectedPerFetch,
})
if err != nil {
return nil, err
}
providerConfig := runtime.config.Provider
providerConfig.RequestInterval = 0
providerReconciler, err := NewReconciler(providerConfig, Ports{
Adapter: runtime.dependencies.Adapter, Parser: runtime.dependencies.Parser,
Activity: runtime.dependencies.Activity, Results: runtime.dependencies.Results,
Capacity: budget,
})
if err != nil {
return nil, err
}
poolReconciler, err := pool.NewReconciler(
runtime.config.ReconcilePolicy,
budget,
providerReconciler,
)
if err != nil {
return nil, err
}
return &upstreamLeaderTerm{
upstreamID: runtime.config.Provider.UpstreamID,
interval: runtime.config.ReconcileInterval,
safetyMargin: runtime.config.ReconcilePolicy.SafetyMargin,
inventory: runtime.dependencies.Inventory,
pool: poolReconciler,
provider: providerReconciler,
sleeper: runtime.sleeper,
}, nil
}
type upstreamLeaderTerm struct {
upstreamID string
interval time.Duration
safetyMargin time.Duration
inventory pool.InventoryReader
pool *pool.Reconciler
provider *Reconciler
sleeper Sleeper
}
func (term *upstreamLeaderTerm) run(ctx context.Context, session LeaderSession) error {
termCtx, cancel := context.WithCancel(ctx)
defer cancel()
done := make(chan error, 2)
go func() { done <- term.provider.RunLeader(termCtx, session) }()
go func() { done <- term.reconcileInventory(termCtx) }()
first := <-done
cancel()
second := <-done
if ctx.Err() != nil {
return nil
}
if first == nil {
first = ErrUpstreamRuntimeStopped
}
if second != nil && !errors.Is(second, context.Canceled) {
return errors.Join(first, second)
}
return first
}
func (term *upstreamLeaderTerm) reconcileInventory(ctx context.Context) error {
if delay := initialReconcileDelay(term.upstreamID, term.interval); delay > 0 {
if err := term.sleeper.Sleep(ctx, delay); err != nil {
if ctx.Err() != nil {
return nil
}
return fmt.Errorf("wait for initial Provider inventory reconciliation: %w", err)
}
}
for ctx.Err() == nil {
inventory, err := term.inventory.ReadInventory(
ctx,
term.upstreamID,
term.safetyMargin,
)
if err == nil && inventory.Managed >= 0 && inventory.AvailableSlots >= 0 {
term.pool.ReconcileSnapshot(inventory)
}
if err := term.sleeper.Sleep(ctx, term.interval); err != nil {
if ctx.Err() != nil {
return nil
}
return fmt.Errorf("wait for Provider inventory reconciliation: %w", err)
}
}
return nil
}
func initialReconcileDelay(upstreamID string, interval time.Duration) time.Duration {
window := min(interval/4, 250*time.Millisecond)
if window <= 1 {
return 0
}
digest := fnv.New64a()
_, _ = digest.Write([]byte(upstreamID))
return time.Duration(digest.Sum64() % uint64(window))
}
func nilRuntimeDependency(value any) bool {
if value == nil {
return true
}
reflected := reflect.ValueOf(value)
switch reflected.Kind() {
case reflect.Chan, reflect.Func, reflect.Interface, reflect.Map, reflect.Pointer, reflect.Slice:
return reflected.IsNil()
default:
return false
}
}

View File

@ -0,0 +1,247 @@
package provider
import (
"context"
"errors"
"sync/atomic"
"testing"
"time"
"proxy-pool/internal/controller/pool"
"proxy-pool/internal/domain/activitypool"
proxyDomain "proxy-pool/internal/domain/proxy"
)
func TestUpstreamRuntimeReadsInventoryAndFetchesOnlyInsideLeaderTerm(t *testing.T) {
var inventoryReads atomic.Int64
fetched := make(chan struct{}, 1)
runtime, err := NewUpstreamRuntime(UpstreamRuntimeConfig{
Provider: runtimeProviderConfig("provider-a"),
ReconcilePolicy: runtimeReconcilePolicy(),
ReconcileInterval: 10 * time.Millisecond,
}, UpstreamRuntimeDependencies{
Coordinator: coordinatorFunc(func(
ctx context.Context,
upstreamID string,
limits CoordinationLimits,
work func(context.Context, LeaderSession) error,
) error {
if upstreamID != "provider-a" || limits.MaxTotal != 10 || limits.MaxInFlight != 1 {
t.Errorf("coordination = (%q, %+v)", upstreamID, limits)
}
return work(ctx, unlimitedLeaderSession{})
}),
Inventory: inventoryReaderFunc(func(context.Context, string, time.Duration) (pool.InventorySnapshot, error) {
inventoryReads.Add(1)
return pool.InventorySnapshot{}, nil
}),
Adapter: adapterFunc(func(context.Context) (FetchResponse, error) {
fetched <- struct{}{}
return FetchResponse{Body: []byte("fixture")}, nil
}),
Parser: parserFunc(func(context.Context, []byte) ([]proxyDomain.Proxy, error) {
return []proxyDomain.Proxy{{ID: "proxy-1"}}, nil
}),
Activity: activitySinkFunc(func(context.Context, string, activitypool.FetchedBatch) (activitypool.UpsertResult, error) {
return activitypool.UpsertResult{Accepted: 1, Inserted: 1}, nil
}),
Results: resultRecorderFunc(func(Result) {}),
})
if err != nil {
t.Fatalf("NewUpstreamRuntime(): %v", err)
}
ctx, cancel := context.WithCancel(context.Background())
done := make(chan error, 1)
go func() { done <- runtime.Run(ctx) }()
select {
case <-fetched:
case <-time.After(time.Second):
t.Fatal("timed out waiting for leader fetch")
}
cancel()
if err := <-done; !errors.Is(err, context.Canceled) {
t.Fatalf("Run() error = %v, want context cancellation", err)
}
if got := inventoryReads.Load(); got == 0 {
t.Fatal("inventory was not read inside leader term")
}
}
func TestUpstreamRuntimeFailsClosedUntilInventoryReadRecovers(t *testing.T) {
var inventoryReads atomic.Int64
var providerCalls atomic.Int64
fetched := make(chan struct{}, 1)
runtime, err := NewUpstreamRuntime(UpstreamRuntimeConfig{
Provider: runtimeProviderConfig("provider-a"),
ReconcilePolicy: runtimeReconcilePolicy(),
ReconcileInterval: 10 * time.Millisecond,
}, UpstreamRuntimeDependencies{
Coordinator: coordinatorFunc(func(
ctx context.Context,
_ string,
_ CoordinationLimits,
work func(context.Context, LeaderSession) error,
) error {
return work(ctx, unlimitedLeaderSession{})
}),
Inventory: inventoryReaderFunc(func(context.Context, string, time.Duration) (pool.InventorySnapshot, error) {
if inventoryReads.Add(1) == 1 {
return pool.InventorySnapshot{}, errors.New("inventory unavailable")
}
return pool.InventorySnapshot{}, nil
}),
Adapter: adapterFunc(func(context.Context) (FetchResponse, error) {
providerCalls.Add(1)
fetched <- struct{}{}
return FetchResponse{Body: []byte("fixture")}, nil
}),
Parser: parserFunc(func(context.Context, []byte) ([]proxyDomain.Proxy, error) {
return []proxyDomain.Proxy{{ID: "proxy-1"}}, nil
}),
Activity: activitySinkFunc(func(context.Context, string, activitypool.FetchedBatch) (activitypool.UpsertResult, error) {
return activitypool.UpsertResult{Accepted: 1, Inserted: 1}, nil
}),
Results: resultRecorderFunc(func(Result) {}),
})
if err != nil {
t.Fatalf("NewUpstreamRuntime(): %v", err)
}
ctx, cancel := context.WithCancel(context.Background())
done := make(chan error, 1)
go func() { done <- runtime.Run(ctx) }()
select {
case <-fetched:
case <-time.After(time.Second):
t.Fatal("timed out waiting for recovered inventory fetch")
}
cancel()
<-done
if got := inventoryReads.Load(); got < 2 {
t.Fatalf("inventory reads = %d, want recovery retry", got)
}
if got := providerCalls.Load(); got != 1 {
t.Fatalf("provider calls = %d, want one call after recovery", got)
}
}
func TestUpstreamRuntimeDelegatesRequestIntervalOnlyToCoordinator(t *testing.T) {
providerConfig := runtimeProviderConfig("provider-a")
providerConfig.RequestInterval = 500 * time.Millisecond
fetched := make(chan struct{}, 2)
runtime, err := NewUpstreamRuntime(UpstreamRuntimeConfig{
Provider: providerConfig,
ReconcilePolicy: runtimeReconcilePolicy(),
ReconcileInterval: 5 * time.Millisecond,
}, UpstreamRuntimeDependencies{
Coordinator: coordinatorFunc(func(
ctx context.Context,
_ string,
limits CoordinationLimits,
work func(context.Context, LeaderSession) error,
) error {
if limits.RequestInterval != 500*time.Millisecond {
t.Errorf("distributed RequestInterval = %s, want 500ms", limits.RequestInterval)
}
return work(ctx, unlimitedLeaderSession{})
}),
Inventory: inventoryReaderFunc(func(context.Context, string, time.Duration) (pool.InventorySnapshot, error) {
return pool.InventorySnapshot{}, nil
}),
Adapter: adapterFunc(func(context.Context) (FetchResponse, error) {
fetched <- struct{}{}
return FetchResponse{Body: []byte("fixture")}, nil
}),
Parser: parserFunc(func(context.Context, []byte) ([]proxyDomain.Proxy, error) {
return []proxyDomain.Proxy{{ID: "proxy-1"}}, nil
}),
Activity: activitySinkFunc(func(context.Context, string, activitypool.FetchedBatch) (activitypool.UpsertResult, error) {
return activitypool.UpsertResult{Accepted: 1, Inserted: 1}, nil
}),
Results: resultRecorderFunc(func(Result) {}),
})
if err != nil {
t.Fatalf("NewUpstreamRuntime(): %v", err)
}
ctx, cancel := context.WithCancel(context.Background())
done := make(chan error, 1)
go func() { done <- runtime.Run(ctx) }()
for range 2 {
select {
case <-fetched:
case <-time.After(150 * time.Millisecond):
cancel()
<-done
t.Fatal("local Provider reconciler duplicated the distributed request interval")
}
}
cancel()
<-done
}
func TestNewUpstreamRuntimeRejectsInvalidDependencies(t *testing.T) {
validConfig := UpstreamRuntimeConfig{
Provider: runtimeProviderConfig("provider-a"),
ReconcilePolicy: runtimeReconcilePolicy(),
ReconcileInterval: time.Second,
}
if runtime, err := NewUpstreamRuntime(validConfig, UpstreamRuntimeDependencies{}); err == nil || runtime != nil {
t.Fatalf("NewUpstreamRuntime() = (%v, %v), want invalid dependencies", runtime, err)
}
}
func TestInitialReconcileDelayIsStableAndBounded(t *testing.T) {
const interval = time.Second
first := initialReconcileDelay("provider-a", interval)
if first != initialReconcileDelay("provider-a", interval) {
t.Fatal("initial reconcile delay is not stable")
}
if first < 0 || first >= 250*time.Millisecond {
t.Fatalf("initial reconcile delay = %s, want [0, 250ms)", first)
}
if other := initialReconcileDelay("provider-b", interval); other == first {
t.Fatalf("different upstreams have the same initial delay %s", first)
}
}
func runtimeProviderConfig(upstreamID string) Config {
return Config{
UpstreamID: upstreamID, Timeout: time.Second, MaxAttempts: 1,
MaxInFlight: 1, MaxTotal: 10, MaxSize: 10,
}
}
func runtimeReconcilePolicy() pool.ReconcilePolicy {
return pool.ReconcilePolicy{
MinimumAvailableSlots: 1, TargetAvailableSlots: 2,
ExpectedPerFetch: 1, ExpectedSlotsPerFetch: 1,
}
}
type coordinatorFunc func(
context.Context,
string,
CoordinationLimits,
func(context.Context, LeaderSession) error,
) error
func (f coordinatorFunc) RunLeader(
ctx context.Context,
upstreamID string,
limits CoordinationLimits,
work func(context.Context, LeaderSession) error,
) error {
return f(ctx, upstreamID, limits, work)
}
type inventoryReaderFunc func(context.Context, string, time.Duration) (pool.InventorySnapshot, error)
func (f inventoryReaderFunc) ReadInventory(
ctx context.Context,
upstreamID string,
safetyMargin time.Duration,
) (pool.InventorySnapshot, error) {
return f(ctx, upstreamID, safetyMargin)
}

View File

@ -13,19 +13,46 @@ var (
)
type Capacity struct {
max atomic.Uint32
counters atomic.Uint64
max atomic.Uint32
counters atomic.Uint64
configuredObserver *activityObserver
observer atomic.Pointer[activityObserver]
}
type activityObserver struct{ notify func(bool) }
func NewCapacity(max int64) *Capacity {
return NewCapacityWithActivityObserver(max, nil)
}
// NewCapacityWithActivityObserver reports successful zero-to-nonzero and
// nonzero-to-zero transitions. The observer must tolerate concurrent calls.
func NewCapacityWithActivityObserver(max int64, observer func(nonzero bool)) *Capacity {
capacity := &Capacity{}
if max < 0 || max > int64(counterMask) {
max = 0
}
capacity.max.Store(uint32(max))
if observer != nil {
capacity.configuredObserver = &activityObserver{notify: observer}
capacity.observer.Store(capacity.configuredObserver)
}
return capacity
}
// SetActivityObservationEnabled lets snapshot ownership disable callbacks for
// current Proxies and enable them only while a runtime is retired and draining.
func (c *Capacity) SetActivityObservationEnabled(enabled bool) {
if c == nil || c.configuredObserver == nil {
return
}
if enabled {
c.observer.Store(c.configuredObserver)
return
}
c.observer.Store(nil)
}
func (c *Capacity) SetMax(max int64) bool {
if max < 0 || max > int64(counterMask) {
return false
@ -36,6 +63,15 @@ func (c *Capacity) SetMax(max int64) bool {
func (c *Capacity) Max() int64 { return int64(c.max.Load()) }
func (c *Capacity) Counters() (active, reserved, maximum int64) {
if c == nil {
return 0, 0, 0
}
packed := c.counters.Load()
activeCounter, reservedCounter := unpack(packed)
return int64(activeCounter), int64(reservedCounter), int64(c.max.Load())
}
func (c *Capacity) Reserve() (*Reservation, bool) {
for {
current := c.counters.Load()
@ -45,6 +81,11 @@ func (c *Capacity) Reserve() (*Reservation, bool) {
}
next := pack(active, reserved+1)
if c.counters.CompareAndSwap(current, next) {
if active+reserved == 0 {
if observer := c.observer.Load(); observer != nil {
observer.notify(true)
}
}
return &Reservation{capacity: c}, true
}
}
@ -77,7 +118,16 @@ func (c *Capacity) cancel() {
for {
current := c.counters.Load()
active, reserved := unpack(current)
if reserved == 0 || c.counters.CompareAndSwap(current, pack(active, reserved-1)) {
if reserved == 0 {
return
}
next := pack(active, reserved-1)
if c.counters.CompareAndSwap(current, next) {
if active+reserved == 1 {
if observer := c.observer.Load(); observer != nil {
observer.notify(false)
}
}
return
}
}
@ -87,7 +137,16 @@ func (c *Capacity) release() {
for {
current := c.counters.Load()
active, reserved := unpack(current)
if active == 0 || c.counters.CompareAndSwap(current, pack(active-1, reserved)) {
if active == 0 {
return
}
next := pack(active-1, reserved)
if c.counters.CompareAndSwap(current, next) {
if active+reserved == 1 {
if observer := c.observer.Load(); observer != nil {
observer.notify(false)
}
}
return
}
}

View File

@ -2,6 +2,7 @@ package proxy
import (
"errors"
"reflect"
"sync"
"sync/atomic"
"testing"
@ -38,6 +39,56 @@ func TestReservationCancelReleasesReservedCapacity(t *testing.T) {
assertCapacityCounters(t, capacity, 0, 0)
}
func TestCapacityCountersReadsOnePackedSnapshot(t *testing.T) {
capacity := NewCapacity(3)
first, ok := capacity.Reserve()
if !ok {
t.Fatal("first Reserve() = false")
}
second, ok := capacity.Reserve()
if !ok {
t.Fatal("second Reserve() = false")
}
if err := first.Commit(); err != nil {
t.Fatalf("Commit(): %v", err)
}
active, reserved, maximum := capacity.Counters()
if active != 1 || reserved != 1 || maximum != 3 {
t.Fatalf("Counters() = (%d, %d, %d), want (1, 1, 3)", active, reserved, maximum)
}
if err := first.Release(); err != nil {
t.Fatalf("Release(): %v", err)
}
if err := second.Cancel(); err != nil {
t.Fatalf("Cancel(): %v", err)
}
}
func TestCapacityActivityObserverTracksOnlyNonzeroTransitions(t *testing.T) {
var transitions []bool
capacity := NewCapacityWithActivityObserver(2, func(nonzero bool) {
transitions = append(transitions, nonzero)
})
first, ok := capacity.Reserve()
if !ok {
t.Fatal("Reserve(first) = false")
}
second, ok := capacity.Reserve()
if !ok {
t.Fatal("Reserve(second) = false")
}
if err := first.Cancel(); err != nil {
t.Fatalf("Cancel(first): %v", err)
}
if err := second.Cancel(); err != nil {
t.Fatalf("Cancel(second): %v", err)
}
if !reflect.DeepEqual(transitions, []bool{true, false}) {
t.Fatalf("transitions = %v, want [true false]", transitions)
}
}
func TestReservationCommitAndReleaseAreSingleUse(t *testing.T) {
capacity := NewCapacity(1)
reservation, ok := capacity.Reserve()

View File

@ -8,6 +8,6 @@ type FetchCapacity interface {
type FetchPermit interface {
Expected() int
Complete(fetched, retained int) error
Complete(retained int) error
Cancel() error
}

View File

@ -17,9 +17,7 @@ type ProxyCapacity struct {
type Inventory struct {
Proxies []ProxyCapacity
PendingExpected int
FetchedTotal int64
MaxSize int
MaxTotal int64
}
func (i Inventory) AvailableSlots(now time.Time, safetyMargin time.Duration) int64 {
@ -55,15 +53,6 @@ func (i Inventory) FetchAllowance(requested int) int {
return 0
}
allowed := min(requested, max(i.MaxSize-i.ManagedCount(), 0))
if i.MaxTotal > 0 {
remaining := i.MaxTotal - i.FetchedTotal
if remaining <= 0 {
return 0
}
if int64(allowed) > remaining {
allowed = int(remaining)
}
}
return allowed
}

View File

@ -21,7 +21,7 @@ func TestInventoryAvailableSlotsUsesOnlyAllocatableCapacity(t *testing.T) {
}
}
func TestInventoryFetchAllowanceSeparatesPoolAndCumulativeLimits(t *testing.T) {
func TestInventoryFetchAllowanceUsesManagedPoolCapacity(t *testing.T) {
inventory := Inventory{
Proxies: []ProxyCapacity{
{State: proxyDomain.StateFetched},
@ -32,20 +32,13 @@ func TestInventoryFetchAllowanceSeparatesPoolAndCumulativeLimits(t *testing.T) {
{State: proxyDomain.StateExtracted},
},
PendingExpected: 2,
FetchedTotal: 98,
MaxSize: 10,
MaxTotal: 100,
}
if got := inventory.ManagedCount(); got != 7 {
t.Fatalf("ManagedCount() = %d, want 7", got)
}
if got := inventory.FetchAllowance(10); got != 2 {
t.Fatalf("FetchAllowance() = %d, want 2 from cumulative quota", got)
}
inventory.MaxTotal = 0
if got := inventory.FetchAllowance(10); got != 3 {
t.Fatalf("FetchAllowance() with unlimited cumulative quota = %d, want 3 from pool size", got)
t.Fatalf("FetchAllowance() = %d, want 3 from pool size", got)
}
}

View File

@ -0,0 +1,211 @@
package workerruntime
import (
"context"
"crypto/sha256"
"encoding/json"
"sort"
"strings"
"sync"
"time"
)
type MemoryStore struct {
mu sync.Mutex
now func() time.Time
sessions map[string]memorySession
reports map[string]memoryReport
}
type memorySession struct {
value Session
expiresAt time.Time
}
type memoryReport struct {
value Report
digest [sha256.Size]byte
expiresAt time.Time
counters map[string]Counter
}
var (
_ SessionWriter = (*MemoryStore)(nil)
_ ReportWriter = (*MemoryStore)(nil)
_ RuntimeReader = (*MemoryStore)(nil)
)
func NewMemoryStore(now func() time.Time) (*MemoryStore, error) {
if now == nil {
return nil, ErrInvalidStore
}
return &MemoryStore{
now: now, sessions: make(map[string]memorySession), reports: make(map[string]memoryReport),
}, nil
}
func (store *MemoryStore) ReplaceSession(ctx context.Context, session Session, ttl time.Duration) error {
if ctx == nil || store == nil || !validSession(session) || ttl <= 0 {
return ErrInvalidSession
}
if err := ctx.Err(); err != nil {
return err
}
now := store.now().UTC()
if now.IsZero() {
return ErrInvalidStore
}
store.mu.Lock()
defer store.mu.Unlock()
current, exists := store.sessions[session.WorkerID]
identityChanged := exists && (current.value.SessionID != session.SessionID || current.value.InstanceID != session.InstanceID)
expired := exists && !current.expiresAt.After(now)
if exists && !identityChanged && !expired && sessionBefore(session, current.value) {
return ErrStaleSession
}
ackAdvanced := exists && !identityChanged && !expired && sessionAfter(session, current.value)
if identityChanged || expired || ackAdvanced {
delete(store.reports, session.WorkerID)
}
store.sessions[session.WorkerID] = memorySession{value: session, expiresAt: now.Add(ttl)}
return nil
}
func (store *MemoryStore) ReplaceRuntime(ctx context.Context, report Report, ttl time.Duration) error {
if ctx == nil || store == nil || ttl <= 0 {
return ErrInvalidReport
}
if err := ctx.Err(); err != nil {
return err
}
normalized, counterIndex, err := normalizeReport(report)
if err != nil {
return err
}
payload, err := json.Marshal(normalized)
if err != nil {
return ErrInvalidReport
}
digest := sha256.Sum256(payload)
now := store.now().UTC()
if now.IsZero() {
return ErrInvalidStore
}
store.mu.Lock()
defer store.mu.Unlock()
session, exists := store.sessions[report.WorkerID]
if !exists || !session.expiresAt.After(now) || session.value.SessionID != report.SessionID {
return ErrStaleSession
}
if report.SnapshotVersion != session.value.AckedSnapshotVersion ||
report.OwnershipEpoch != session.value.AckedOwnershipEpoch {
return ErrStaleReport
}
if current, exists := store.reports[report.WorkerID]; exists && current.value.SessionID == report.SessionID {
switch {
case normalized.Sequence < current.value.Sequence:
return ErrStaleReport
case normalized.Sequence == current.value.Sequence && digest != current.digest:
return ErrConflictingReport
case normalized.Sequence == current.value.Sequence:
return nil
}
}
store.reports[report.WorkerID] = memoryReport{
value: normalized, digest: digest, expiresAt: now.Add(ttl), counters: counterIndex,
}
session.expiresAt = now.Add(ttl)
store.sessions[report.WorkerID] = session
return nil
}
func (store *MemoryStore) ReadRuntime(ctx context.Context, proxies []OwnedProxy) ([]Snapshot, error) {
if ctx == nil || store == nil {
return nil, ErrInvalidQuery
}
if err := ctx.Err(); err != nil {
return nil, err
}
seen := make(map[string]struct{}, len(proxies))
for _, proxy := range proxies {
if !clean(proxy.ProxyID) || !clean(proxy.WorkerID) || proxy.OwnershipEpoch == 0 {
return nil, ErrInvalidQuery
}
key := proxy.WorkerID + "\x00" + proxy.ProxyID
if _, exists := seen[key]; exists {
return nil, ErrInvalidQuery
}
seen[key] = struct{}{}
}
now := store.now().UTC()
if now.IsZero() {
return nil, ErrInvalidStore
}
store.mu.Lock()
defer store.mu.Unlock()
result := make([]Snapshot, len(proxies))
for index, proxy := range proxies {
result[index].ProxyID = proxy.ProxyID
session, sessionExists := store.sessions[proxy.WorkerID]
report, reportExists := store.reports[proxy.WorkerID]
if !sessionExists || !reportExists || !session.expiresAt.After(now) || !report.expiresAt.After(now) ||
report.value.SessionID != session.value.SessionID ||
report.value.SnapshotVersion != session.value.AckedSnapshotVersion ||
report.value.OwnershipEpoch != session.value.AckedOwnershipEpoch ||
report.value.OwnershipEpoch < proxy.OwnershipEpoch {
continue
}
result[index].Fresh = true
if counter, exists := report.counters[proxy.ProxyID]; exists {
result[index].Active = counter.Active
result[index].Reserved = counter.Reserved
result[index].Draining = counter.Draining
}
}
return result, nil
}
func normalizeReport(report Report) (Report, map[string]Counter, error) {
if !clean(report.WorkerID) || !clean(report.SessionID) || report.Sequence == 0 ||
report.SnapshotVersion == 0 || report.OwnershipEpoch == 0 || report.ObservedAt.IsZero() {
return Report{}, nil, ErrInvalidReport
}
normalized := report
normalized.ObservedAt = report.ObservedAt.UTC()
normalized.Counters = append([]Counter(nil), report.Counters...)
sort.Slice(normalized.Counters, func(left, right int) bool {
return normalized.Counters[left].ProxyID < normalized.Counters[right].ProxyID
})
index := make(map[string]Counter, len(normalized.Counters))
for _, counter := range normalized.Counters {
if !clean(counter.ProxyID) || counter.Active < 0 || counter.Reserved < 0 {
return Report{}, nil, ErrInvalidReport
}
if _, exists := index[counter.ProxyID]; exists {
return Report{}, nil, ErrInvalidReport
}
index[counter.ProxyID] = counter
}
return normalized, index, nil
}
func validSession(session Session) bool {
return clean(session.WorkerID) && clean(session.InstanceID) && clean(session.SessionID) &&
session.AckedSnapshotVersion > 0 && session.AckedOwnershipEpoch > 0
}
func sessionBefore(left, right Session) bool {
return left.AckedOwnershipEpoch < right.AckedOwnershipEpoch ||
(left.AckedOwnershipEpoch == right.AckedOwnershipEpoch &&
left.AckedSnapshotVersion < right.AckedSnapshotVersion)
}
func sessionAfter(left, right Session) bool {
return left.AckedOwnershipEpoch > right.AckedOwnershipEpoch ||
(left.AckedOwnershipEpoch == right.AckedOwnershipEpoch &&
left.AckedSnapshotVersion > right.AckedSnapshotVersion)
}
func clean(value string) bool {
return value != "" && strings.TrimSpace(value) == value
}

View File

@ -0,0 +1,156 @@
package workerruntime
import (
"context"
"errors"
"testing"
"time"
)
func TestMemoryStoreReplacesSparseRuntimeAndClearsMissingCounters(t *testing.T) {
now := time.Date(2026, 7, 30, 15, 0, 0, 0, time.UTC)
store := newRuntimeStore(t, &now)
ctx := context.Background()
registerRuntimeSession(t, store, "session-a", time.Minute)
report := Report{
WorkerID: "worker-a", SessionID: "session-a", Sequence: 1,
SnapshotVersion: 3, OwnershipEpoch: 9, ObservedAt: now,
Counters: []Counter{{ProxyID: "proxy-a", Active: 2, Reserved: 1}},
}
if err := store.ReplaceRuntime(ctx, report, time.Minute); err != nil {
t.Fatalf("ReplaceRuntime(first): %v", err)
}
got, err := store.ReadRuntime(ctx, []OwnedProxy{{ProxyID: "proxy-a", WorkerID: "worker-a", OwnershipEpoch: 9}})
if err != nil || len(got) != 1 || got[0] != (Snapshot{ProxyID: "proxy-a", Active: 2, Reserved: 1, Fresh: true}) {
t.Fatalf("ReadRuntime(first) = %+v, %v", got, err)
}
report.Sequence = 2
report.ObservedAt = now.Add(time.Second)
report.Counters = nil
if err := store.ReplaceRuntime(ctx, report, time.Minute); err != nil {
t.Fatalf("ReplaceRuntime(empty): %v", err)
}
got, err = store.ReadRuntime(ctx, []OwnedProxy{{ProxyID: "proxy-a", WorkerID: "worker-a", OwnershipEpoch: 9}})
if err != nil || len(got) != 1 || got[0] != (Snapshot{ProxyID: "proxy-a", Fresh: true}) {
t.Fatalf("ReadRuntime(empty) = %+v, %v", got, err)
}
}
func TestMemoryStoreFencesSessionsAndReportSequence(t *testing.T) {
now := time.Date(2026, 7, 30, 15, 0, 0, 0, time.UTC)
store := newRuntimeStore(t, &now)
ctx := context.Background()
registerRuntimeSession(t, store, "session-a", time.Minute)
report := Report{
WorkerID: "worker-a", SessionID: "session-a", Sequence: 2,
SnapshotVersion: 3, OwnershipEpoch: 9, ObservedAt: now,
Counters: []Counter{{ProxyID: "proxy-a", Active: 1}},
}
if err := store.ReplaceRuntime(ctx, report, time.Minute); err != nil {
t.Fatalf("ReplaceRuntime(first): %v", err)
}
if err := store.ReplaceRuntime(ctx, report, time.Minute); err != nil {
t.Fatalf("ReplaceRuntime(idempotent): %v", err)
}
conflict := report
conflict.Counters = []Counter{{ProxyID: "proxy-a", Active: 2}}
if err := store.ReplaceRuntime(ctx, conflict, time.Minute); !errors.Is(err, ErrConflictingReport) {
t.Fatalf("ReplaceRuntime(conflict) error = %v", err)
}
stale := report
stale.Sequence = 1
if err := store.ReplaceRuntime(ctx, stale, time.Minute); !errors.Is(err, ErrStaleReport) {
t.Fatalf("ReplaceRuntime(stale) error = %v", err)
}
registerRuntimeSession(t, store, "session-b", time.Minute)
if err := store.ReplaceRuntime(ctx, report, time.Minute); !errors.Is(err, ErrStaleSession) {
t.Fatalf("ReplaceRuntime(old session) error = %v", err)
}
}
func TestMemoryStoreFailsClosedForExpiredOrOlderOwnershipReport(t *testing.T) {
now := time.Date(2026, 7, 30, 15, 0, 0, 0, time.UTC)
store := newRuntimeStore(t, &now)
ctx := context.Background()
registerRuntimeSession(t, store, "session-a", time.Minute)
if err := store.ReplaceRuntime(ctx, Report{
WorkerID: "worker-a", SessionID: "session-a", Sequence: 1,
SnapshotVersion: 3, OwnershipEpoch: 9, ObservedAt: now,
}, time.Minute); err != nil {
t.Fatalf("ReplaceRuntime(): %v", err)
}
queries := []OwnedProxy{
{ProxyID: "proxy-a", WorkerID: "worker-a", OwnershipEpoch: 10},
{ProxyID: "proxy-b", WorkerID: "worker-a", OwnershipEpoch: 9},
}
got, err := store.ReadRuntime(ctx, queries)
if err != nil || got[0].Fresh || !got[1].Fresh {
t.Fatalf("ReadRuntime(ownership fence) = %+v, %v", got, err)
}
now = now.Add(time.Minute)
got, err = store.ReadRuntime(ctx, queries[1:])
if err != nil || len(got) != 1 || got[0].Fresh {
t.Fatalf("ReadRuntime(expired) = %+v, %v", got, err)
}
}
func TestMemoryStoreRejectsRuntimeBeyondAcknowledgedSnapshot(t *testing.T) {
now := time.Date(2026, 7, 30, 15, 0, 0, 0, time.UTC)
store := newRuntimeStore(t, &now)
registerRuntimeSession(t, store, "session-a", time.Minute)
report := Report{
WorkerID: "worker-a", SessionID: "session-a", Sequence: 1,
SnapshotVersion: 4, OwnershipEpoch: 9, ObservedAt: now,
}
if err := store.ReplaceRuntime(context.Background(), report, time.Minute); !errors.Is(err, ErrStaleReport) {
t.Fatalf("ReplaceRuntime(ahead snapshot) error = %v, want ErrStaleReport", err)
}
report.SnapshotVersion = 3
report.OwnershipEpoch = 10
if err := store.ReplaceRuntime(context.Background(), report, time.Minute); !errors.Is(err, ErrStaleReport) {
t.Fatalf("ReplaceRuntime(ahead epoch) error = %v, want ErrStaleReport", err)
}
}
func TestMemoryStoreExpiredSameIdentitySessionDoesNotReactivateOldReport(t *testing.T) {
now := time.Date(2026, 7, 30, 15, 0, 0, 0, time.UTC)
store := newRuntimeStore(t, &now)
registerRuntimeSession(t, store, "session-a", time.Second)
if err := store.ReplaceRuntime(context.Background(), Report{
WorkerID: "worker-a", SessionID: "session-a", Sequence: 1,
SnapshotVersion: 3, OwnershipEpoch: 9, ObservedAt: now,
}, time.Minute); err != nil {
t.Fatalf("ReplaceRuntime(): %v", err)
}
session := store.sessions["worker-a"]
session.expiresAt = now.Add(time.Second)
store.sessions["worker-a"] = session
now = now.Add(2 * time.Second)
registerRuntimeSession(t, store, "session-a", time.Minute)
got, err := store.ReadRuntime(context.Background(), []OwnedProxy{{
ProxyID: "proxy-a", WorkerID: "worker-a", OwnershipEpoch: 9,
}})
if err != nil || len(got) != 1 || got[0].Fresh {
t.Fatalf("ReadRuntime(after re-register) = %+v, %v; want stale", got, err)
}
}
func newRuntimeStore(t *testing.T, now *time.Time) *MemoryStore {
t.Helper()
store, err := NewMemoryStore(func() time.Time { return *now })
if err != nil {
t.Fatalf("NewMemoryStore(): %v", err)
}
return store
}
func registerRuntimeSession(t *testing.T, store *MemoryStore, sessionID string, ttl time.Duration) {
t.Helper()
if err := store.ReplaceSession(context.Background(), Session{
WorkerID: "worker-a", InstanceID: "instance-a", SessionID: sessionID,
AckedSnapshotVersion: 3, AckedOwnershipEpoch: 9,
}, ttl); err != nil {
t.Fatalf("ReplaceSession(): %v", err)
}
}

View File

@ -0,0 +1,70 @@
package workerruntime
import (
"context"
"errors"
"time"
)
var (
ErrInvalidStore = errors.New("invalid worker runtime store")
ErrInvalidSession = errors.New("invalid worker runtime session")
ErrInvalidReport = errors.New("invalid worker runtime report")
ErrInvalidQuery = errors.New("invalid worker runtime query")
ErrStaleSession = errors.New("stale worker runtime session")
ErrStaleReport = errors.New("stale worker runtime report")
ErrConflictingReport = errors.New("conflicting worker runtime report")
)
type Session struct {
WorkerID string
InstanceID string
SessionID string
AckedSnapshotVersion uint64
AckedOwnershipEpoch uint64
}
type Counter struct {
ProxyID string
Active int64
Reserved int64
Draining bool
}
// Report is a complete sparse replacement. Missing counters are zero for the
// reported Worker snapshot; callers must increase Sequence for every update.
type Report struct {
WorkerID string
SessionID string
Sequence uint64
SnapshotVersion uint64
OwnershipEpoch uint64
ObservedAt time.Time
Counters []Counter
}
type OwnedProxy struct {
ProxyID string
WorkerID string
OwnershipEpoch uint64
}
type Snapshot struct {
ProxyID string
Active int64
Reserved int64
Draining bool
Fresh bool
}
type SessionWriter interface {
ReplaceSession(context.Context, Session, time.Duration) error
}
type ReportWriter interface {
ReplaceRuntime(context.Context, Report, time.Duration) error
}
type RuntimeReader interface {
ReadRuntime(context.Context, []OwnedProxy) ([]Snapshot, error)
}

View File

@ -7,19 +7,90 @@ import (
"errors"
"fmt"
"sort"
"strings"
"sync"
"sync/atomic"
"time"
proxyDomain "proxy-pool/internal/domain/proxy"
"proxy-pool/internal/domain/workerruntime"
)
var (
ErrWrongTarget = errors.New("snapshot targets another cluster or worker")
ErrResyncRequired = errors.New("snapshot sequence requires a full resync")
ErrChecksumMismatch = errors.New("snapshot checksum mismatch")
ErrWrongTarget = errors.New("snapshot targets another cluster or worker")
ErrResyncRequired = errors.New("snapshot sequence requires a full resync")
ErrChecksumMismatch = errors.New("snapshot checksum mismatch")
ErrInvalidRuntimeReport = errors.New("invalid worker runtime report")
ErrInvalidRuntimeLimit = errors.New("invalid snapshot runtime limit")
ErrRuntimeLimitExceeded = errors.New("snapshot runtime limit exceeded")
)
const defaultRuntimeLimit = 1_000_000
const activeRuntimeShardCount = 64
type activeRuntimeShard struct {
mu sync.Mutex
entries map[string]*proxyDomain.Capacity
}
type activeRuntimeIndex [activeRuntimeShardCount]activeRuntimeShard
func (index *activeRuntimeIndex) track(proxyID string, runtime *proxyDomain.Capacity, nonzero bool) {
shard := &index[activeRuntimeShardIndex(proxyID)]
shard.mu.Lock()
defer shard.mu.Unlock()
if nonzero {
if shard.entries == nil {
shard.entries = make(map[string]*proxyDomain.Capacity)
}
shard.entries[proxyID] = runtime
return
}
active, reserved, _ := runtime.Counters()
if active == 0 && reserved == 0 && shard.entries[proxyID] == runtime {
delete(shard.entries, proxyID)
}
}
func (index *activeRuntimeIndex) remove(proxyID string, runtime *proxyDomain.Capacity) {
shard := &index[activeRuntimeShardIndex(proxyID)]
shard.mu.Lock()
defer shard.mu.Unlock()
if shard.entries[proxyID] == runtime {
delete(shard.entries, proxyID)
}
}
func (index *activeRuntimeIndex) rangeEntries(visit func(string, *proxyDomain.Capacity)) {
for shardIndex := range index {
shard := &index[shardIndex]
shard.mu.Lock()
for proxyID, runtime := range shard.entries {
visit(proxyID, runtime)
}
shard.mu.Unlock()
}
}
func activeRuntimeShardIndex(proxyID string) uint64 {
const (
offset = uint64(14695981039346656037)
prime = uint64(1099511628211)
)
hash := offset
for index := 0; index < len(proxyID); index++ {
hash ^= uint64(proxyID[index])
hash *= prime
}
return hash % activeRuntimeShardCount
}
type runtimeRegistration struct {
capacity *proxyDomain.Capacity
current atomic.Bool
}
type Envelope struct {
ClusterID string
WorkerID string
@ -70,17 +141,29 @@ type Store struct {
current atomic.Pointer[View]
mu sync.Mutex
runtimes map[string]*proxyDomain.Capacity
runtimes map[string]*runtimeRegistration
active activeRuntimeIndex
limit int
}
func NewStore(clusterID, workerID string) *Store {
return &Store{
clusterID: clusterID,
workerID: workerID,
runtimes: make(map[string]*proxyDomain.Capacity),
runtimes: make(map[string]*runtimeRegistration),
limit: defaultRuntimeLimit,
}
}
func NewStoreWithRuntimeLimit(clusterID, workerID string, limit int) (*Store, error) {
if limit <= 0 {
return nil, ErrInvalidRuntimeLimit
}
store := NewStore(clusterID, workerID)
store.limit = limit
return store, nil
}
func (s *Store) Current() *View {
if s == nil {
return nil
@ -88,6 +171,49 @@ func (s *Store) Current() *View {
return s.current.Load()
}
func (s *Store) RuntimeReport(sessionID string, sequence uint64, observedAt time.Time) (workerruntime.Report, error) {
if s == nil || strings.TrimSpace(sessionID) != sessionID || sessionID == "" || sequence == 0 || observedAt.IsZero() {
return workerruntime.Report{}, ErrInvalidRuntimeReport
}
current := s.current.Load()
if current == nil {
return workerruntime.Report{}, ErrInvalidRuntimeReport
}
visible := make(map[string]struct{}, len(current.Entries))
counters := make([]workerruntime.Counter, 0)
for _, entry := range current.Entries {
visible[entry.Proxy.ID] = struct{}{}
active, reserved, _ := entry.Runtime.Counters()
if active == 0 && reserved == 0 {
continue
}
counters = append(counters, workerruntime.Counter{
ProxyID: entry.Proxy.ID, Active: active, Reserved: reserved,
Draining: entry.Proxy.State == proxyDomain.StateDraining,
})
}
s.active.rangeEntries(func(proxyID string, runtime *proxyDomain.Capacity) {
if _, currentProxy := visible[proxyID]; currentProxy {
return
}
active, reserved, _ := runtime.Counters()
if active == 0 && reserved == 0 {
return
}
counters = append(counters, workerruntime.Counter{
ProxyID: proxyID, Active: active, Reserved: reserved, Draining: true,
})
})
sort.Slice(counters, func(left, right int) bool {
return counters[left].ProxyID < counters[right].ProxyID
})
return workerruntime.Report{
WorkerID: s.workerID, SessionID: sessionID, Sequence: sequence,
SnapshotVersion: current.Version, OwnershipEpoch: current.Epoch,
ObservedAt: observedAt.UTC(), Counters: counters,
}, nil
}
func (s *Store) Apply(envelope Envelope) error {
if s == nil {
return fmt.Errorf("apply snapshot: nil store")
@ -118,19 +244,47 @@ func (s *Store) Apply(envelope Envelope) error {
}
proxies := cloneAndSort(envelope.Proxies)
newRuntimeCount := 0
for _, descriptor := range proxies {
if s.runtimes[descriptor.ID] == nil {
newRuntimeCount++
}
}
if len(s.runtimes)+newRuntimeCount > s.limit {
return ErrRuntimeLimitExceeded
}
if current != nil {
for _, entry := range current.Entries {
registration := s.runtimes[entry.Proxy.ID]
registration.capacity.SetActivityObservationEnabled(true)
registration.current.Store(false)
active, reserved, _ := registration.capacity.Counters()
s.active.track(entry.Proxy.ID, registration.capacity, active+reserved > 0)
}
}
entries := make([]Entry, 0, len(proxies))
for _, descriptor := range proxies {
runtime := s.runtimes[descriptor.ID]
if runtime == nil {
runtime = proxyDomain.NewCapacity(descriptor.MaxConcurrency)
registration := s.runtimes[descriptor.ID]
if registration == nil {
proxyID := descriptor.ID
registration = &runtimeRegistration{}
registration.capacity = proxyDomain.NewCapacityWithActivityObserver(descriptor.MaxConcurrency, func(nonzero bool) {
if registration.current.Load() {
return
}
s.active.track(proxyID, registration.capacity, nonzero)
})
} else {
runtime.SetMax(descriptor.MaxConcurrency)
registration.capacity.SetMax(descriptor.MaxConcurrency)
}
registration.current.Store(true)
registration.capacity.SetActivityObservationEnabled(false)
s.active.remove(descriptor.ID, registration.capacity)
// Keep runtimes for temporarily absent IDs. Old immutable views may still
// hold in-flight leases, so reclaiming here could reset active capacity if
// the same Proxy reappears in a later snapshot.
s.runtimes[descriptor.ID] = runtime
entries = append(entries, Entry{Proxy: descriptor, Runtime: runtime})
s.runtimes[descriptor.ID] = registration
entries = append(entries, Entry{Proxy: descriptor, Runtime: registration.capacity})
}
next := &View{

View File

@ -8,6 +8,7 @@ import (
"time"
proxyDomain "proxy-pool/internal/domain/proxy"
"proxy-pool/internal/domain/workerruntime"
)
func TestStoreAppliesCompleteSnapshotsInOrder(t *testing.T) {
@ -266,6 +267,165 @@ func TestStoreReusesRuntimeWhenProxyDisappearsAndReappears(t *testing.T) {
}
}
func TestStoreRuntimeReportKeepsRemovedActiveProxyUntilRelease(t *testing.T) {
store := NewStore("cluster-a", "worker-a")
now := time.Date(2026, 7, 30, 15, 0, 0, 0, time.UTC)
proxy := proxyDomain.Proxy{
ID: "proxy-a", Scheme: proxyDomain.SchemeHTTP, SourceUpstream: "provider-a",
State: proxyDomain.StateAvailable, MaxConcurrency: 1,
}
initial := Envelope{
ClusterID: "cluster-a", WorkerID: "worker-a", Epoch: 1, Version: 1, Full: true,
Proxies: []proxyDomain.Proxy{proxy},
}
initial.Checksum = Checksum(initial.Proxies)
if err := store.Apply(initial); err != nil {
t.Fatalf("Apply(initial): %v", err)
}
runtime := store.Current().Entries[0].Runtime
reservation, ok := runtime.Reserve()
if !ok {
t.Fatal("Reserve() = false")
}
if err := reservation.Commit(); err != nil {
t.Fatalf("Commit(): %v", err)
}
removed := Envelope{ClusterID: "cluster-a", WorkerID: "worker-a", Epoch: 1, Version: 2, Full: true}
removed.Checksum = Checksum(nil)
if err := store.Apply(removed); err != nil {
t.Fatalf("Apply(remove): %v", err)
}
if got := store.activeRuntimeCount(); got != 1 {
t.Fatalf("activeRuntimeCount(removed) = %d, want 1", got)
}
report, err := store.RuntimeReport("session-a", 7, now)
if err != nil {
t.Fatalf("RuntimeReport(): %v", err)
}
if report.WorkerID != "worker-a" || report.SessionID != "session-a" || report.Sequence != 7 ||
report.SnapshotVersion != 2 || report.OwnershipEpoch != 1 || !report.ObservedAt.Equal(now) ||
len(report.Counters) != 1 || report.Counters[0] != (workerruntime.Counter{
ProxyID: "proxy-a", Active: 1, Draining: true,
}) {
t.Fatalf("RuntimeReport() = %+v", report)
}
if err := reservation.Release(); err != nil {
t.Fatalf("Release(): %v", err)
}
if got := store.activeRuntimeCount(); got != 0 {
t.Fatalf("activeRuntimeCount(released) = %d, want 0", got)
}
report, err = store.RuntimeReport("session-a", 8, now.Add(time.Second))
if err != nil || len(report.Counters) != 0 {
t.Fatalf("RuntimeReport(after release) = %+v, %v", report, err)
}
}
func TestStoreRuntimeReportMarksCurrentDrainingProxy(t *testing.T) {
store := NewStore("cluster-a", "worker-a")
now := time.Date(2026, 7, 30, 15, 0, 0, 0, time.UTC)
proxy := proxyDomain.Proxy{
ID: "proxy-a", Scheme: proxyDomain.SchemeHTTP,
State: proxyDomain.StateDraining, MaxConcurrency: 1,
}
envelope := Envelope{
ClusterID: "cluster-a", WorkerID: "worker-a", Epoch: 1, Version: 1, Full: true,
Proxies: []proxyDomain.Proxy{proxy},
}
envelope.Checksum = Checksum(envelope.Proxies)
if err := store.Apply(envelope); err != nil {
t.Fatalf("Apply(): %v", err)
}
reservation, ok := store.Current().Entries[0].Runtime.Reserve()
if !ok {
t.Fatal("Reserve() = false")
}
report, err := store.RuntimeReport("session-a", 1, now)
if err != nil || len(report.Counters) != 1 || !report.Counters[0].Draining {
t.Fatalf("RuntimeReport() = %+v, %v", report, err)
}
if err := reservation.Cancel(); err != nil {
t.Fatalf("Cancel(): %v", err)
}
}
func TestStoreBoundsHistoricalRuntimeRegistryAndKeepsApplyTransactional(t *testing.T) {
store, err := NewStoreWithRuntimeLimit("cluster-a", "worker-a", 1)
if err != nil {
t.Fatalf("NewStoreWithRuntimeLimit(): %v", err)
}
first := proxyDomain.Proxy{
ID: "proxy-a", Scheme: proxyDomain.SchemeHTTP,
State: proxyDomain.StateAvailable, MaxConcurrency: 1,
}
envelope := Envelope{
ClusterID: "cluster-a", WorkerID: "worker-a", Epoch: 1, Version: 1,
Full: true, Proxies: []proxyDomain.Proxy{first},
}
envelope.Checksum = Checksum(envelope.Proxies)
if err := store.Apply(envelope); err != nil {
t.Fatalf("Apply(first): %v", err)
}
removed := Envelope{
ClusterID: "cluster-a", WorkerID: "worker-a", Epoch: 1, Version: 2, Full: true,
}
removed.Checksum = Checksum(nil)
if err := store.Apply(removed); err != nil {
t.Fatalf("Apply(removed): %v", err)
}
second := first
second.ID = "proxy-b"
overLimit := Envelope{
ClusterID: "cluster-a", WorkerID: "worker-a", Epoch: 1, Version: 3,
Full: true, Proxies: []proxyDomain.Proxy{second},
}
overLimit.Checksum = Checksum(overLimit.Proxies)
if err := store.Apply(overLimit); !errors.Is(err, ErrRuntimeLimitExceeded) {
t.Fatalf("Apply(over limit) error = %v, want ErrRuntimeLimitExceeded", err)
}
if current := store.Current(); current.Version != 2 || len(current.Entries) != 0 {
t.Fatalf("Current() after rejected apply = version %d entries %d", current.Version, len(current.Entries))
}
}
func TestStoreRuntimeActiveIndexDropsZeroCounters(t *testing.T) {
store := NewStore("cluster-a", "worker-a")
proxy := proxyDomain.Proxy{
ID: "proxy-a", Scheme: proxyDomain.SchemeHTTP,
State: proxyDomain.StateAvailable, MaxConcurrency: 1,
}
envelope := Envelope{
ClusterID: "cluster-a", WorkerID: "worker-a", Epoch: 1, Version: 1,
Full: true, Proxies: []proxyDomain.Proxy{proxy},
}
envelope.Checksum = Checksum(envelope.Proxies)
if err := store.Apply(envelope); err != nil {
t.Fatalf("Apply(): %v", err)
}
reservation, ok := store.Current().Entries[0].Runtime.Reserve()
if !ok {
t.Fatal("Reserve() = false")
}
if got := store.activeRuntimeCount(); got != 0 {
t.Fatalf("activeRuntimeCount(current) = %d, want 0", got)
}
if err := reservation.Cancel(); err != nil {
t.Fatalf("Cancel(): %v", err)
}
if got := store.activeRuntimeCount(); got != 0 {
t.Fatalf("activeRuntimeCount() = %d, want 0", got)
}
}
func (s *Store) activeRuntimeCount() int {
count := 0
s.active.rangeEntries(func(_ string, _ *proxyDomain.Capacity) {
count++
})
return count
}
func collectSelectionIDs(selection Selection) []string {
ids := make([]string, 0, selection.Len())
for index := 0; index < selection.Len(); index++ {

View File

@ -48,18 +48,35 @@ type Store interface {
Resolve(context.Context, Reference) (Value, error)
}
// Releaser removes transient credential material after the consumer has copied
// it into its authoritative storage. Release is idempotent and version fenced.
type Releaser interface {
Release(context.Context, Reference) error
}
type CapacityEnsurer interface {
EnsureCapacity(context.Context, int) error
}
type entry struct {
scope *scopeState
value Value
version uint64
reference Reference
}
type scopeState struct {
name string
value Value
version uint64
leases int
}
// MemoryStore keeps credentials in process memory and serializes access with a
// context-aware lock.
type MemoryStore struct {
lock chan struct{}
capacity int
byScope map[string]*entry
byScope map[string]*scopeState
byRef map[string]*entry
}
@ -82,7 +99,7 @@ func NewMemoryStore(capacity int) (*MemoryStore, error) {
return &MemoryStore{
lock: lock,
capacity: capacity,
byScope: make(map[string]*entry),
byScope: make(map[string]*scopeState),
byRef: make(map[string]*entry),
}, nil
}
@ -100,36 +117,38 @@ func (s *MemoryStore) Put(ctx context.Context, scope string, value Value) (Refer
if err := s.acquire(ctx); err != nil {
return Reference{}, err
}
defer s.release()
defer s.unlock()
if err := ctx.Err(); err != nil {
return Reference{}, err
}
if current, ok := s.byScope[scope]; ok {
if current.value == value {
return current.reference, nil
}
current.value = value
current.version++
current.reference.CredentialVersion = versionString(current.version)
return current.reference, nil
if len(s.byRef) >= s.capacity {
return Reference{}, ErrCapacityExceeded
}
if len(s.byScope) >= s.capacity {
current, exists := s.byScope[scope]
if !exists && len(s.byScope) >= s.capacity {
return Reference{}, ErrCapacityExceeded
}
secretRef, err := s.newUniqueSecretRef()
if err != nil {
return Reference{}, err
}
if !exists {
current = &scopeState{name: scope, value: value, version: 1}
s.byScope[scope] = current
} else if current.value != value {
current.value = value
current.version++
}
created := &entry{
value: value,
version: 1,
scope: current,
value: value,
reference: Reference{
SecretRef: secretRef,
CredentialVersion: versionString(1),
CredentialVersion: versionString(current.version),
},
}
s.byScope[scope] = created
current.leases++
s.byRef[secretRef] = created
return created.reference, nil
}
@ -147,7 +166,7 @@ func (s *MemoryStore) Resolve(ctx context.Context, reference Reference) (Value,
if err := s.acquire(ctx); err != nil {
return Value{}, err
}
defer s.release()
defer s.unlock()
if err := ctx.Err(); err != nil {
return Value{}, err
}
@ -162,6 +181,57 @@ func (s *MemoryStore) Resolve(ctx context.Context, reference Reference) (Value,
return current.value, nil
}
func (s *MemoryStore) Release(ctx context.Context, reference Reference) error {
if err := contextError(ctx); err != nil {
return err
}
if !s.valid() {
return ErrInvalidStore
}
if reference.SecretRef == "" || !validVersion(reference.CredentialVersion) {
return ErrInvalidReference
}
if err := s.acquire(ctx); err != nil {
return err
}
defer s.unlock()
if err := ctx.Err(); err != nil {
return err
}
current, ok := s.byRef[reference.SecretRef]
if !ok || current.reference != reference {
return nil
}
delete(s.byRef, reference.SecretRef)
current.scope.leases--
if current.scope.leases == 0 {
delete(s.byScope, current.scope.name)
current.scope.value = Value{}
}
current.value = Value{}
return nil
}
func (s *MemoryStore) EnsureCapacity(ctx context.Context, minimum int) error {
if err := contextError(ctx); err != nil {
return err
}
if !s.valid() {
return ErrInvalidStore
}
if minimum <= 0 {
return ErrInvalidCapacity
}
if err := s.acquire(ctx); err != nil {
return err
}
defer s.unlock()
if minimum > s.capacity {
s.capacity = minimum
}
return nil
}
func (s *MemoryStore) acquire(ctx context.Context) error {
if err := contextError(ctx); err != nil {
return err
@ -174,7 +244,7 @@ func (s *MemoryStore) acquire(ctx context.Context) error {
}
}
func (s *MemoryStore) release() {
func (s *MemoryStore) unlock() {
s.lock <- struct{}{}
}

View File

@ -11,7 +11,7 @@ import (
"time"
)
func TestMemoryStorePutIsIdempotentForUnchangedScope(t *testing.T) {
func TestMemoryStorePutCreatesIndependentLeasesForUnchangedScope(t *testing.T) {
store, err := NewMemoryStore(2)
if err != nil {
t.Fatalf("NewMemoryStore(): %v", err)
@ -26,10 +26,10 @@ func TestMemoryStorePutIsIdempotentForUnchangedScope(t *testing.T) {
if err != nil {
t.Fatalf("Put(second): %v", err)
}
if first != second {
t.Fatalf("second reference = %#v, want %#v", second, first)
if first == second || first.SecretRef == second.SecretRef {
t.Fatalf("references = %#v and %#v, want independent leases", first, second)
}
if first.SecretRef == "" || first.CredentialVersion != "v1" {
if first.SecretRef == "" || first.CredentialVersion != "v1" || second.CredentialVersion != "v1" {
t.Fatalf("first reference = %#v, want opaque ref at v1", first)
}
for _, plaintext := range []string{"provider-a", value.Username, value.Password} {
@ -45,6 +45,12 @@ func TestMemoryStorePutIsIdempotentForUnchangedScope(t *testing.T) {
if got != value {
t.Fatalf("Resolve() = %#v, want %#v", got, value)
}
if err := store.Release(context.Background(), first); err != nil {
t.Fatalf("Release(first lease): %v", err)
}
if got, err := store.Resolve(context.Background(), second); err != nil || got != value {
t.Fatalf("Resolve(second lease) = %#v, %v", got, err)
}
}
func TestCredentialFormattingRedactsSensitiveMaterial(t *testing.T) {
@ -89,8 +95,8 @@ func TestCredentialFormattingRedactsSensitiveMaterial(t *testing.T) {
}
}
func TestMemoryStorePutIncrementsVersionAndRejectsStaleReference(t *testing.T) {
store, err := NewMemoryStore(1)
func TestMemoryStorePutIncrementsVersionWithoutRevokingActiveLease(t *testing.T) {
store, err := NewMemoryStore(2)
if err != nil {
t.Fatalf("NewMemoryStore(): %v", err)
}
@ -106,14 +112,15 @@ func TestMemoryStorePutIncrementsVersionAndRejectsStaleReference(t *testing.T) {
if err != nil {
t.Fatalf("Put(new): %v", err)
}
if newReference.SecretRef != oldReference.SecretRef {
t.Fatalf("new SecretRef changed across versions")
if newReference.SecretRef == oldReference.SecretRef {
t.Fatalf("new SecretRef reused an active lease")
}
if newReference.CredentialVersion != "v2" {
t.Fatalf("new CredentialVersion = %q, want v2", newReference.CredentialVersion)
}
if _, err := store.Resolve(context.Background(), oldReference); !errors.Is(err, ErrCredentialVersionMismatch) {
t.Fatalf("Resolve(stale) error = %v, want ErrCredentialVersionMismatch", err)
old, err := store.Resolve(context.Background(), oldReference)
if err != nil || old.Password != "old-password" {
t.Fatalf("Resolve(active old lease) = %#v, %v", old, err)
}
got, err := store.Resolve(context.Background(), newReference)
if err != nil {
@ -148,8 +155,80 @@ func TestMemoryStoreEnforcesCapacityWithoutChangingExistingCredentials(t *testin
if got != want {
t.Fatalf("Resolve(existing) returned changed credentials")
}
if _, err := store.Put(context.Background(), "provider-a", want); !errors.Is(err, ErrCapacityExceeded) {
t.Fatalf("Put(second lease at capacity) error = %v, want ErrCapacityExceeded", err)
}
if err := store.Release(context.Background(), reference); err != nil {
t.Fatalf("Release(first lease): %v", err)
}
if _, err := store.Put(context.Background(), "provider-a", want); err != nil {
t.Fatalf("Put(idempotent at capacity): %v", err)
t.Fatalf("Put(after lease release): %v", err)
}
}
func TestMemoryStoreReleaseMakesCapacityReusable(t *testing.T) {
store, err := NewMemoryStore(1)
if err != nil {
t.Fatalf("NewMemoryStore(): %v", err)
}
first, err := store.Put(context.Background(), "provider-a", Value{Password: "first-password"})
if err != nil {
t.Fatalf("Put(first): %v", err)
}
if err := store.Release(context.Background(), first); err != nil {
t.Fatalf("Release(first): %v", err)
}
if _, err := store.Resolve(context.Background(), first); !errors.Is(err, ErrCredentialMissing) {
t.Fatalf("Resolve(released) error = %v, want ErrCredentialMissing", err)
}
if _, err := store.Put(context.Background(), "provider-b", Value{Password: "second-password"}); err != nil {
t.Fatalf("Put(after release): %v", err)
}
if err := store.Release(context.Background(), first); err != nil {
t.Fatalf("Release(idempotent): %v", err)
}
}
func TestMemoryStoreReleaseOfStaleReferencePreservesCurrentVersion(t *testing.T) {
store, err := NewMemoryStore(2)
if err != nil {
t.Fatalf("NewMemoryStore(): %v", err)
}
stale, err := store.Put(context.Background(), "provider-a", Value{Password: "old-password"})
if err != nil {
t.Fatalf("Put(old): %v", err)
}
current, err := store.Put(context.Background(), "provider-a", Value{Password: "new-password"})
if err != nil {
t.Fatalf("Put(new): %v", err)
}
if err := store.Release(context.Background(), stale); err != nil {
t.Fatalf("Release(stale): %v", err)
}
if _, err := store.Resolve(context.Background(), current); err != nil {
t.Fatalf("Resolve(current): %v", err)
}
}
func TestMemoryStoreEnsureCapacityOnlyGrowsLimit(t *testing.T) {
store, err := NewMemoryStore(1)
if err != nil {
t.Fatalf("NewMemoryStore(): %v", err)
}
if err := store.EnsureCapacity(context.Background(), 2); err != nil {
t.Fatalf("EnsureCapacity(2): %v", err)
}
if _, err := store.Put(context.Background(), "provider-a", Value{}); err != nil {
t.Fatalf("Put(provider-a): %v", err)
}
if _, err := store.Put(context.Background(), "provider-b", Value{}); err != nil {
t.Fatalf("Put(provider-b): %v", err)
}
if err := store.EnsureCapacity(context.Background(), 1); err != nil {
t.Fatalf("EnsureCapacity(shrink request): %v", err)
}
if _, err := store.Put(context.Background(), "provider-c", Value{}); !errors.Is(err, ErrCapacityExceeded) {
t.Fatalf("Put(provider-c) error = %v, want retained capacity 2", err)
}
}
@ -211,13 +290,13 @@ func TestMemoryStoreRejectsNilAndZeroValueStores(t *testing.T) {
}
}
func TestMemoryStoreIsConcurrencySafeAndIdempotent(t *testing.T) {
store, err := NewMemoryStore(1)
func TestMemoryStoreCreatesIndependentConcurrentLeases(t *testing.T) {
const workers = 100
store, err := NewMemoryStore(workers)
if err != nil {
t.Fatalf("NewMemoryStore(): %v", err)
}
want := Value{Username: "alice", Password: "shared-password"}
const workers = 100
references := make(chan Reference, workers)
errorsSeen := make(chan error, workers)
var wait sync.WaitGroup
@ -239,29 +318,29 @@ func TestMemoryStoreIsConcurrencySafeAndIdempotent(t *testing.T) {
for err := range errorsSeen {
t.Errorf("concurrent Put(): %v", err)
}
var first Reference
secretRefs := make(map[string]struct{}, workers)
for reference := range references {
if first == (Reference{}) {
first = reference
if reference.CredentialVersion != "v1" {
t.Errorf("concurrent version = %q, want v1", reference.CredentialVersion)
}
if reference != first {
t.Errorf("concurrent Put() reference differs from first")
secretRefs[reference.SecretRef] = struct{}{}
}
if len(secretRefs) != workers {
t.Fatalf("unique concurrent leases = %d, want %d", len(secretRefs), workers)
}
for secretRef := range secretRefs {
got, err := store.Resolve(context.Background(), Reference{
SecretRef: secretRef, CredentialVersion: "v1",
})
if err != nil || got != want {
t.Fatalf("Resolve(concurrent lease) = %#v, %v", got, err)
}
}
if first.CredentialVersion != "v1" {
t.Fatalf("concurrent version = %q, want v1", first.CredentialVersion)
}
got, err := store.Resolve(context.Background(), first)
if err != nil {
t.Fatalf("Resolve(): %v", err)
}
if got != want {
t.Fatalf("Resolve() returned unexpected credentials")
}
}
func TestMemoryStoreSerializesConcurrentCredentialChanges(t *testing.T) {
store, err := NewMemoryStore(1)
const workers = 100
store, err := NewMemoryStore(workers)
if err != nil {
t.Fatalf("NewMemoryStore(): %v", err)
}
@ -270,7 +349,6 @@ func TestMemoryStoreSerializesConcurrentCredentialChanges(t *testing.T) {
reference Reference
err error
}
const workers = 100
results := make(chan result, workers)
var wait sync.WaitGroup
for index := range workers {
@ -286,18 +364,13 @@ func TestMemoryStoreSerializesConcurrentCredentialChanges(t *testing.T) {
close(results)
versions := make(map[string]struct{}, workers)
var secretRef string
secretRefs := make(map[string]struct{}, workers)
var latest result
for current := range results {
if current.err != nil {
t.Fatalf("concurrent Put(): %v", current.err)
}
if secretRef == "" {
secretRef = current.reference.SecretRef
}
if current.reference.SecretRef != secretRef {
t.Fatal("SecretRef changed across concurrent updates")
}
secretRefs[current.reference.SecretRef] = struct{}{}
versions[current.reference.CredentialVersion] = struct{}{}
if current.reference.CredentialVersion == "v100" {
latest = current
@ -306,6 +379,9 @@ func TestMemoryStoreSerializesConcurrentCredentialChanges(t *testing.T) {
if len(versions) != workers {
t.Fatalf("unique versions = %d, want %d", len(versions), workers)
}
if len(secretRefs) != workers {
t.Fatalf("unique leases = %d, want %d", len(secretRefs), workers)
}
if latest.reference == (Reference{}) {
t.Fatal("highest version v100 was not returned")
}

View File

@ -0,0 +1,72 @@
package lifecycle
import (
"context"
"errors"
"reflect"
)
var (
ErrInvalidGroup = errors.New("invalid lifecycle group")
ErrRunnerStopped = errors.New("lifecycle runner stopped")
)
type Runner interface {
Run(context.Context) error
}
type Group struct {
runners []Runner
}
func NewGroup(runners ...Runner) (*Group, error) {
if len(runners) == 0 {
return nil, ErrInvalidGroup
}
owned := make([]Runner, len(runners))
for index, runner := range runners {
if isNilRunner(runner) {
return nil, ErrInvalidGroup
}
owned[index] = runner
}
return &Group{runners: owned}, nil
}
func (group *Group) Run(ctx context.Context) error {
if group == nil || ctx == nil || len(group.runners) == 0 {
return ErrInvalidGroup
}
groupCtx, cancel := context.WithCancel(ctx)
defer cancel()
done := make(chan error, len(group.runners))
for _, runner := range group.runners {
go func() { done <- runner.Run(groupCtx) }()
}
first := <-done
cancel()
for range len(group.runners) - 1 {
<-done
}
if ctx.Err() != nil {
return ctx.Err()
}
if first == nil {
return ErrRunnerStopped
}
return first
}
func isNilRunner(runner Runner) bool {
if runner == nil {
return true
}
reflected := reflect.ValueOf(runner)
switch reflected.Kind() {
case reflect.Chan, reflect.Func, reflect.Interface, reflect.Map, reflect.Pointer, reflect.Slice:
return reflected.IsNil()
default:
return false
}
}

View File

@ -0,0 +1,87 @@
package lifecycle
import (
"context"
"errors"
"sync/atomic"
"testing"
"time"
)
func TestGroupStartsAllRunnersAndCancelsSiblingsOnFirstError(t *testing.T) {
started := make(chan struct{}, 2)
cancelled := make(chan struct{}, 1)
wantErr := errors.New("runner failed")
group, err := NewGroup(
runnerFunc(func(context.Context) error {
started <- struct{}{}
return wantErr
}),
runnerFunc(func(ctx context.Context) error {
started <- struct{}{}
<-ctx.Done()
cancelled <- struct{}{}
return ctx.Err()
}),
)
if err != nil {
t.Fatalf("NewGroup(): %v", err)
}
if err := group.Run(context.Background()); !errors.Is(err, wantErr) {
t.Fatalf("Run() error = %v, want %v", err, wantErr)
}
<-started
<-started
select {
case <-cancelled:
case <-time.After(time.Second):
t.Fatal("sibling runner was not cancelled")
}
}
func TestGroupWaitsForEveryRunnerBeforeReturning(t *testing.T) {
release := make(chan struct{})
var exited atomic.Bool
group, err := NewGroup(
runnerFunc(func(context.Context) error { return errors.New("failed") }),
runnerFunc(func(ctx context.Context) error {
<-ctx.Done()
<-release
exited.Store(true)
return nil
}),
)
if err != nil {
t.Fatalf("NewGroup(): %v", err)
}
done := make(chan error, 1)
go func() { done <- group.Run(context.Background()) }()
select {
case <-done:
t.Fatal("Run() returned before sibling exited")
case <-time.After(20 * time.Millisecond):
}
close(release)
<-done
if !exited.Load() {
t.Fatal("sibling exit was not observed")
}
}
func TestNewGroupRejectsEmptyAndTypedNilRunners(t *testing.T) {
var typedNil *nilRunner
for _, runners := range [][]Runner{nil, {typedNil}} {
group, err := NewGroup(runners...)
if err == nil || group != nil {
t.Fatalf("NewGroup() = (%v, %v), want invalid group", group, err)
}
}
}
type runnerFunc func(context.Context) error
func (f runnerFunc) Run(ctx context.Context) error { return f(ctx) }
type nilRunner struct{}
func (*nilRunner) Run(context.Context) error { return nil }

View File

@ -2,6 +2,26 @@
## 2026-07-30
- Gateway `Capacity` 新增一次打包原子读取,`snapshot.Store` 可生成完整稀疏
Active/Reserved 运行态报告;当前快照已移除但仍有连接的 Proxy 会持续以
draining 上报,归零后从后续报告消失。
- 新增公用 `workerruntime` session/report/read seam 与并发安全 MemoryStore
完整替换、空报告清零、session fencing、单调 sequence、同内容幂等重放、
冲突/倒序拒绝和 TTL fail-closed 均已有单测。
- 生产 `redisactivity` Adapter 新增 Worker session/运行态 Lua 和权威
`pool.InventoryReader`Managed/Available Slots 原子计入状态、TTL safety、
MaxConcurrency、ownership 及 Active/Reserved未知或过期运行态贡献零容量。
- 真实 Redis 8.2 已覆盖 ACK snapshot/epoch、旧 session、空报告、超前 epoch、
报告过期、扫描预算耗尽及容量聚合。Managed 使用已有权威计数Available
Slots 只扫描目标 Upstream 的未分配/已分配可用索引Gateway 报告扫描当前
Snapshot并用分片索引补充已移除但仍非零的 runtime历史注册表设置硬上限。
WorkerControlPlane 接收端、Provider
Fleet/bootstrap、全局 `fetch.maxTotal` 和健康执行链仍待完成,总验收计数保持
51/73。
- Gateway 当前 Proxy 禁用活跃索引回调,只有移出 Snapshot 后才开启分片追踪;
本机 100k Proxy `Acquire` 三轮 1 秒基准为 893.9-1047 ns/op、256 B/op、
2 allocs/op。该数据只证明本地调度微基准不代表 100k QPS 集群验收。
- 新增公用 Provider `Coordinator.RunLeader` / `LeaderSession` 深 seam 和独立
`redisprovider` AdapterRedis Lua 原子维护 generation、epoch、Leader 租约、
全局 requestInterval 与带 TTL 的 maxInFlight Permit异常时 fail-closed。

View File

@ -35,7 +35,8 @@
机器契约和文档类滞后勾选已按仓库证据校正
13. [进行中] 落地 `proxy-controller` 进程装配配置单次加载、PostgreSQL 迁移、
Redis 活动池、低基数状态聚合、Distribution/Admin/Metrics 启动与关闭已完成,
双存储 bootstrap 和探针集成已通过Provider、业务指标与完整容器进程链仍待实现
双存储 bootstrap 和探针集成已通过Worker 运行态存储与权威容量读取原语已
完成WorkerControlPlane、Provider、业务指标与完整容器进程链仍待实现
## 串并行关系
@ -57,7 +58,8 @@
- Docker Compose 配置与 Kubernetes Kustomize 已完成静态渲染验证Redis 8.2
与 PostgreSQL 18 的隔离 Adapter fixture 已运行,完整目标运行拓扑尚未启动。
- `cmd/proxy-controller` 已实现 Admin/Distribution/Metrics 与双存储启动装配;
Gateway、Checker、Loadgen、Provider Leader/分布式限流、业务指标、Redis
故障转移验证与代表性集群压测属于后续实施范围。
Provider 分布式协调和 Worker 运行态 Redis 原语已完成,但 WorkerControlPlane
接收端、Provider Fleet、Gateway、Checker、Loadgen、业务指标、Redis 故障
转移验证与代表性集群压测属于后续实施范围。
- `implementation-plan.md` 当前按 73 个验收项统计;已校正为 51 项完成,
验收项完成率约 69.9%,不等同于生产就绪度。