Compare commits
5 Commits
e9945d933f
...
b8f5104167
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
b8f5104167 | ||
|
|
8997e3880e | ||
|
|
40f4b3ffab | ||
|
|
d648ba37e0 | ||
|
|
45ba6fb958 |
@ -52,6 +52,7 @@ Windows PowerShell 可运行:
|
||||
2. Proxy 容量使用 `Reserved -> Active` 原子转换,禁止超卖。
|
||||
3. Distribution 成功时原子执行 `AVAILABLE -> EXTRACTED`,不提供 Lease、
|
||||
Release 或 Renewal。
|
||||
4. `pool.maxSize` 是当前未提取库存上限;`fetch.maxTotal` 是累计获取额度。
|
||||
4. `pool.maxSize` 是当前未提取库存硬上限;`fetch.maxTotal` 是 Redis generation
|
||||
内的累计获取停止阈值。
|
||||
5. CONNECT 向客户端提交 200 后不透明重放。
|
||||
6. 公开监听必须有认证或 CIDR 访问保护。
|
||||
|
||||
@ -182,6 +182,7 @@ message ReportRuntimeRequest {
|
||||
uint64 ownership_epoch = 4;
|
||||
repeated ProxyRuntime counters = 5;
|
||||
google.protobuf.Timestamp observed_at = 6;
|
||||
uint64 report_sequence = 7;
|
||||
}
|
||||
|
||||
message ProxyRuntime {
|
||||
|
||||
@ -15,7 +15,10 @@ import (
|
||||
"proxy-pool/internal/controller/bootstrap"
|
||||
)
|
||||
|
||||
const configEnvironment = "PROXY_POOL_CONFIG"
|
||||
const (
|
||||
configEnvironment = "PROXY_POOL_CONFIG"
|
||||
fingerprintKeyEnvironment = "PROXY_POOL_CONFIG_FINGERPRINT_KEY"
|
||||
)
|
||||
|
||||
type environmentLookup func(string) string
|
||||
type controllerRun func(context.Context, bootstrap.Options) error
|
||||
@ -54,7 +57,13 @@ func execute(
|
||||
return 2
|
||||
}
|
||||
|
||||
err := run(ctx, bootstrap.Options{ConfigPath: *configPath, Resolver: config.OSResolver{}})
|
||||
var fingerprintKey []byte
|
||||
if getenv != nil {
|
||||
fingerprintKey = []byte(getenv(fingerprintKeyEnvironment))
|
||||
}
|
||||
err := run(ctx, bootstrap.Options{
|
||||
ConfigPath: *configPath, Resolver: config.OSResolver{}, FingerprintKey: fingerprintKey,
|
||||
})
|
||||
if err == nil || (errors.Is(err, context.Canceled) && ctx.Err() != nil) {
|
||||
return 0
|
||||
}
|
||||
|
||||
@ -16,12 +16,15 @@ func TestExecuteUsesFlagBeforeEnvironment(t *testing.T) {
|
||||
if name == configEnvironment {
|
||||
return "environment.yaml"
|
||||
}
|
||||
if name == fingerprintKeyEnvironment {
|
||||
return "0123456789abcdef0123456789abcdef"
|
||||
}
|
||||
return ""
|
||||
}, func(_ context.Context, options bootstrap.Options) error {
|
||||
received = options
|
||||
return nil
|
||||
}, &bytes.Buffer{})
|
||||
if code != 0 || received.ConfigPath != "flag.yaml" || received.Resolver == nil {
|
||||
if code != 0 || received.ConfigPath != "flag.yaml" || received.Resolver == nil || len(received.FingerprintKey) != 32 {
|
||||
t.Fatalf("execute() = %d, options = %+v", code, received)
|
||||
}
|
||||
}
|
||||
@ -29,7 +32,12 @@ func TestExecuteUsesFlagBeforeEnvironment(t *testing.T) {
|
||||
func TestExecuteFallsBackToEnvironment(t *testing.T) {
|
||||
t.Parallel()
|
||||
var received bootstrap.Options
|
||||
code := execute(context.Background(), nil, func(string) string { return "environment.yaml" }, func(
|
||||
code := execute(context.Background(), nil, func(name string) string {
|
||||
if name == configEnvironment {
|
||||
return "environment.yaml"
|
||||
}
|
||||
return "0123456789abcdef0123456789abcdef"
|
||||
}, func(
|
||||
_ context.Context,
|
||||
options bootstrap.Options,
|
||||
) error {
|
||||
|
||||
@ -1,5 +1,13 @@
|
||||
name: proxy-pool
|
||||
|
||||
x-app-environment: &app-environment
|
||||
PROXY_POOL_CONFIG: /etc/proxy-pool/config.yaml
|
||||
PROXY_POOL_GATEWAY_PASSWORD: ${PROXY_POOL_GATEWAY_PASSWORD:?set PROXY_POOL_GATEWAY_PASSWORD}
|
||||
PROXY_POOL_EXTRACT_TOKEN: ${PROXY_POOL_EXTRACT_TOKEN:?set PROXY_POOL_EXTRACT_TOKEN}
|
||||
PROXY_POOL_ADMIN_TOKEN: ${PROXY_POOL_ADMIN_TOKEN:?set PROXY_POOL_ADMIN_TOKEN}
|
||||
PROVIDER_A_TOKEN: ${PROVIDER_A_TOKEN:?set PROVIDER_A_TOKEN}
|
||||
PROVIDER_B_TOKEN: ${PROVIDER_B_TOKEN:?set PROVIDER_B_TOKEN}
|
||||
|
||||
x-app: &app
|
||||
build:
|
||||
context: ..
|
||||
@ -9,13 +17,7 @@ x-app: &app
|
||||
networks: [frontend, backend]
|
||||
volumes:
|
||||
- ./config/local.yaml:/etc/proxy-pool/config.yaml:ro
|
||||
environment:
|
||||
PROXY_POOL_CONFIG: /etc/proxy-pool/config.yaml
|
||||
PROXY_POOL_GATEWAY_PASSWORD: ${PROXY_POOL_GATEWAY_PASSWORD:?set PROXY_POOL_GATEWAY_PASSWORD}
|
||||
PROXY_POOL_EXTRACT_TOKEN: ${PROXY_POOL_EXTRACT_TOKEN:?set PROXY_POOL_EXTRACT_TOKEN}
|
||||
PROXY_POOL_ADMIN_TOKEN: ${PROXY_POOL_ADMIN_TOKEN:?set PROXY_POOL_ADMIN_TOKEN}
|
||||
PROVIDER_A_TOKEN: ${PROVIDER_A_TOKEN:?set PROVIDER_A_TOKEN}
|
||||
PROVIDER_B_TOKEN: ${PROVIDER_B_TOKEN:?set PROVIDER_B_TOKEN}
|
||||
environment: *app-environment
|
||||
stop_grace_period: 45s
|
||||
|
||||
services:
|
||||
@ -44,6 +46,9 @@ services:
|
||||
controller:
|
||||
<<: *app
|
||||
command: ["proxy-controller"]
|
||||
environment:
|
||||
<<: *app-environment
|
||||
PROXY_POOL_CONFIG_FINGERPRINT_KEY: ${PROXY_POOL_CONFIG_FINGERPRINT_KEY:?set PROXY_POOL_CONFIG_FINGERPRINT_KEY}
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
|
||||
@ -9,8 +9,8 @@ stringData:
|
||||
PROXY_POOL_GATEWAY_PASSWORD: GATEWAY_PASSWORD
|
||||
PROXY_POOL_EXTRACT_TOKEN: EXTRACT_TOKEN
|
||||
PROXY_POOL_ADMIN_TOKEN: ADMIN_TOKEN
|
||||
PROXY_POOL_CONFIG_FINGERPRINT_KEY: CONFIG_FINGERPRINT_KEY_MINIMUM_32_BYTES
|
||||
PROXY_POOL_POSTGRES_URL: postgres://USER:PASSWORD@POSTGRES_HOST:5432/proxy_pool?sslmode=verify-full
|
||||
PROXY_POOL_REDIS_URL: rediss://:PASSWORD@REDIS_HOST:6379/0
|
||||
PROVIDER_A_TOKEN: PROVIDER_A_TOKEN
|
||||
PROVIDER_B_TOKEN: PROVIDER_B_TOKEN
|
||||
|
||||
|
||||
@ -63,10 +63,14 @@ var _ activitypool.HealthStore = (*Adapter)(nil)
|
||||
var _ activitypool.InventoryReader = (*Adapter)(nil)
|
||||
var _ extraction.Store = (*Adapter)(nil)
|
||||
var _ ownership.Repository = (*Adapter)(nil)
|
||||
var _ workerruntime.SessionWriter = (*Adapter)(nil)
|
||||
var _ workerruntime.ReportWriter = (*Adapter)(nil)
|
||||
var _ workerruntime.RuntimeReader = (*Adapter)(nil)
|
||||
var _ pool.InventoryReader = (*Adapter)(nil)
|
||||
```
|
||||
|
||||
Provider、Checker、Distribution 和 Ownership 只依赖各自需要的端口,不直接
|
||||
依赖 Redis 客户端、键名、Lua 返回格式或清理策略。
|
||||
Provider、Checker、Distribution、Ownership 和 Worker Runtime 只依赖各自需要
|
||||
的端口,不直接依赖 Redis 客户端、键名、Lua 返回格式或清理策略。
|
||||
|
||||
`ownership.Repository` 改为适合远程存储的上下文感知接口:
|
||||
|
||||
@ -97,6 +101,11 @@ pp:{activity}:owners HASH proxyID -> ownership assignment
|
||||
pp:{activity}:owner-expiry ZSET proxyID -> ownership expiry milliseconds
|
||||
pp:{activity}:epoch STRING ownership 全局递增代次
|
||||
pp:{activity}:inventory HASH upstreamID -> 当前未提取库存
|
||||
pp:{activity}:worker-sessions HASH workerID -> 当前 Worker session
|
||||
pp:{activity}:worker-session-expiry ZSET workerID -> session expiry milliseconds
|
||||
pp:{activity}:worker-runtime HASH workerID -> 完整稀疏运行态报告
|
||||
pp:{activity}:worker-runtime-expiry ZSET workerID -> report expiry milliseconds
|
||||
pp:{activity}:owned:<digest> ZSET 单 Upstream 已分配 AVAILABLE Proxy
|
||||
pp:{activity}:idem:<digest> STRING 带 TTL 的提取幂等结果
|
||||
pp:{activity}:op:<digest> STRING 带 TTL 的内部操作结果
|
||||
```
|
||||
@ -167,6 +176,38 @@ Assign、Renew、BeginDrain 和 AcknowledgeDrain 分别使用有界小脚本,
|
||||
- Assign 与 Extract 并发竞争同一 Proxy 时,只允许一个操作成功。
|
||||
- Expire 使用 `limit` 分批回收过期 assignment,禁止无界返回。
|
||||
|
||||
### Worker 运行态与容量汇总
|
||||
|
||||
Gateway 的 `Capacity` 使用一次打包原子读取取得同一时刻的 Active/Reserved,
|
||||
`snapshot.Store` 周期生成完整稀疏报告。当前 Snapshot 已移除但仍有活动连接的
|
||||
Proxy 继续以 `draining=true` 上报,直到 Active/Reserved 同时归零。
|
||||
|
||||
Redis Adapter 在单个 `{activity}` 原子边界内维护 Worker session 和运行态报告:
|
||||
|
||||
1. 新 Worker session 替换旧 session,并隔离旧实例后续写入。
|
||||
2. session 保存 Controller 已 ACK 的 snapshot version 与 ownership epoch;报告
|
||||
必须与 ACK 上界完全一致,不能通过自报超前 epoch 绕过所有权校验。
|
||||
3. `report_sequence` 严格递增;同序号、同内容可幂等重放,冲突或倒序拒绝。
|
||||
4. 非零计数必须匹配当前 Proxy owner、Worker ID 和 ownership epoch。
|
||||
5. session/report TTL 使用 Redis 服务端时间;过期、缺失或损坏时容量 fail-closed。
|
||||
6. 空报告清除该 Worker 的全部旧计数,稀疏报告中缺失的 Proxy 计数视为零。
|
||||
|
||||
`pool.InventoryReader` 低频返回单个 Upstream 的 `Managed` 和
|
||||
`AvailableSlots`。Managed 统计 FETCHED/CHECKING/AVAILABLE/SUSPECT/DRAINING;
|
||||
Available Slots 只统计超过 safety margin、状态为 AVAILABLE 且所有权与新鲜
|
||||
Worker 运行态一致的 `max - active - reserved`。未分配 Proxy 可直接贡献 Max;
|
||||
已分配但运行态未知的 Proxy 贡献零槽位。PostgreSQL 不保存这些短效报告或容量
|
||||
明细,Gateway 每次请求也不访问 Redis。
|
||||
|
||||
Managed 直接读取现有 Upstream 权威计数;Available Slots 只扫描目标 Upstream
|
||||
的未分配可用索引与已分配可用索引,不扫描全局 Proxy,也不受其他供应商活记录
|
||||
数量影响。索引成员数超过单次扫描预算时直接返回不可用,不降级为近似容量。
|
||||
生产规模验收仍需验证脚本 p95/p99、CPU、内存和过期风暴下的有界行为。
|
||||
|
||||
Gateway `snapshot.Store` 扫描有界的当前 Snapshot,并使用分片索引补充已移除但
|
||||
仍非零的 runtime,不扫描全部历史 Proxy;历史 Capacity 注册表设硬上限,达到
|
||||
上限时拒绝新 Snapshot 并保持旧视图,避免长期轮换造成无界内存增长。
|
||||
|
||||
### 短 TTL 清理
|
||||
|
||||
Redis Hash 字段没有独立 TTL,因此使用三层有界清理:
|
||||
@ -226,6 +267,8 @@ Redis `inventory` 是当前未提取 Proxy 数量的运行时真值:
|
||||
- Assign 与 Extract 并发互斥,以及 renew/drain/ACK/expire。
|
||||
- 提交后连接断开、脚本缓存丢失、上下文取消和 Redis 不可用。
|
||||
- 30 秒 TTL 持续写入下的有界清理与库存一致性。
|
||||
- Worker session 替换、运行态序号幂等/冲突、报告过期和 ownership epoch 隔离。
|
||||
- 权威 Managed/Available Slots 聚合及扫描预算耗尽时的 fail-closed 行为。
|
||||
|
||||
Lua 语义必须使用真实 Redis 8.2 集成测试验证。单元测试最长 60 秒,并执行
|
||||
gofmt、go vet、全量测试、构建和 diff whitespace 检查。100,000 QPS 只能由
|
||||
|
||||
@ -70,9 +70,14 @@ Outbox。状态变化、审计和 Outbox 在同一事务提交。
|
||||
|
||||
### 配置修订
|
||||
|
||||
配置提交只保存管理面恢复所需的非敏感事实:配置版本、SHA-256 校验和、来源、
|
||||
Upstream 启用状态和 Routing 候选/当前选择。已解析 Secret、Provider Token、
|
||||
Proxy 凭据和完整运行时对象不进入 PostgreSQL。
|
||||
配置提交只保存管理面恢复所需的非敏感事实:配置版本、完整已解析配置的
|
||||
HMAC-SHA-256 指纹、来源、Upstream 启用状态和 Routing 候选/当前选择。HMAC 使用
|
||||
独立外部高熵密钥,覆盖 Secret 轮换以驱动多副本收敛,同时避免普通摘要成为
|
||||
低熵 Secret 的离线校验器。HMAC 密钥、已解析 Secret、Provider Token、Proxy
|
||||
凭据、配置正文和完整运行时对象均不进入 PostgreSQL。
|
||||
|
||||
配置发布携带事务返回的全局 revision。本地 `config.Store` 只接受严格递增
|
||||
revision,因此并发提交或 Supervisor 同步的迟到旧版本不能覆盖较新运行配置。
|
||||
|
||||
配置重载以一个事务替换管理快照。新 Routing 的当前 Upstream 必须属于其候选集,
|
||||
所有引用的 Upstream 必须存在,名称与列表必须非空且唯一。校验失败发生在事务前,
|
||||
|
||||
@ -39,10 +39,24 @@ Status 以一个权威管理快照决定 Upstream 集合和 Enabled 状态,只
|
||||
|
||||
1. `FileConfigurationLoader` 通过 `config.LoadResolved` 严格解析、解析 Secret 引用
|
||||
并完成全量校验。
|
||||
2. 从脱敏管理投影计算版本与校验和;Secret 值及其可验证摘要不进入管理状态。
|
||||
2. 使用独立外部密钥对完整已解析配置计算 HMAC-SHA-256 指纹;PostgreSQL 只保存
|
||||
不透明 HMAC,不保存密钥、配置正文或 Secret 明文,因此 URL、模板和 Secret
|
||||
轮换都会产生新版本,也不能借数据库摘要离线猜测低熵 Secret。
|
||||
3. 在同一 `adminstate` mutation 中提交配置修订、管理状态、审计和 Outbox。
|
||||
4. 提交成功后由 `config.Store` 一次原子指针交换发布完整运行配置;提交失败时旧
|
||||
配置保持不变。幂等重放仍执行发布,以修复进程本地状态。
|
||||
4. 提交成功后由 `config.Store` 按 PostgreSQL revision 原子发布完整运行配置;
|
||||
Store 只接受严格递增 revision,提交失败或迟到旧 revision 不覆盖当前配置。
|
||||
幂等重放仅在本地 revision 落后时修复进程状态。
|
||||
5. Provider Supervisor 在提交前构造预检所有启用 Upstream;发布后按新配置取消、
|
||||
替换或新增 Runtime。enable 同样在管理状态 mutation 前预检目标 Runtime。
|
||||
6. 其他 Controller 每秒比较本地指纹与 PostgreSQL 权威指纹;所有副本必须使用
|
||||
相同 `PROXY_POOL_CONFIG_FINGERPRINT_KEY`。共享配置源已同步时严格重载、预检
|
||||
并按 revision 发布,源尚未同步时停止旧 Provider Runtime,禁止旧 URL/Secret
|
||||
在换主后继续调用。
|
||||
|
||||
disable 成功后会立即通知 Supervisor 取消目标 Runtime;每秒一次的权威状态对账
|
||||
用于修复进程内通知丢失。已取得的分布式 Permit 仍按幂等、保守规则完成结算。
|
||||
PostgreSQL 瞬时读取失败不会终止 Controller 或取消当前 Provider Runtime;
|
||||
Supervisor 保留 last-known 状态并在下一周期重试。
|
||||
|
||||
主配置或 Secret 文件 I/O 故障归类为 503;语法、未知字段、引用和语义校验失败
|
||||
归类为 422。请求取消和截止时间保持原始上下文错误,不误报为配置错误。
|
||||
|
||||
@ -34,6 +34,18 @@ sequenceDiagram
|
||||
`worker_id` 是逻辑节点,`instance_id` 区分进程重启,`session_id` 防止旧进程
|
||||
继续上报。所有权 `epoch` 小于 Controller 当前值的数据必须拒绝。
|
||||
|
||||
`ReportRuntimeRequest.report_sequence` 在当前 `session_id` 内严格单调递增。
|
||||
相同序号只允许内容完全相同的幂等重放;较小序号或相同序号的不同内容必须
|
||||
拒绝。`observed_at` 只用于观测,不作为乱序判定依据,运行态 TTL 统一使用
|
||||
Controller 侧 Redis 服务端时间。session 同时保存 Controller 已接受的
|
||||
`snapshot_version/ownership_epoch`;运行态报告必须与该 ACK 上界完全一致,
|
||||
Worker 自报的超前版本或 epoch 也必须拒绝。
|
||||
|
||||
运行态上报是完整稀疏替换:只携带 Active/Reserved 非零的 Proxy,空列表表示
|
||||
当前会话全部归零。Controller 只有在 Worker session、报告 TTL、Proxy ownership
|
||||
和 ownership epoch 同时有效时才使用计数;报告缺失、过期或不一致时按零可用
|
||||
容量 fail-closed,不能把未知计数解释成空闲容量。
|
||||
|
||||
## 3. Snapshot 与 Delta
|
||||
|
||||
完整 Snapshot 包含:
|
||||
|
||||
@ -16,10 +16,13 @@ Controller 入口已实现,源码运行方式为:
|
||||
go run ./cmd/proxy-controller -config CONFIG_FILE
|
||||
```
|
||||
|
||||
配置路径优先使用 `-config`,未提供时读取 `PROXY_POOL_CONFIG`。该入口已装配
|
||||
配置路径优先使用 `-config`,未提供时读取 `PROXY_POOL_CONFIG`。启用 Admin 时还
|
||||
必须设置 `PROXY_POOL_CONFIG_FINGERPRINT_KEY`,值为至少 32 字节的独立高熵密钥;
|
||||
所有 Controller 副本必须一致,且该密钥不得放入 YAML 或 PostgreSQL。该入口已装配
|
||||
PostgreSQL 管理面迁移、Redis 活动池、Distribution/Admin 独立监听与优雅停机;
|
||||
Controller Metrics 独立监听、`/livez`、`/readyz` 和基础 Prometheus 运行时指标;
|
||||
Provider 自动补池、业务指标和完整部署拓扑仍在后续实施范围。
|
||||
Provider 自动补池、分布式配额、动态重载和 Admin 低基数统计已装配。完整 Gateway、
|
||||
Checker、Worker 控制面与代表性负载验证仍在后续实施范围。
|
||||
|
||||
所有时间值使用 Go duration,例如 `500ms`、`30s`、`5m`。示例中的
|
||||
`${TOKEN}`、`${PASSWORD}`、`${POSTGRES_URL}` 等由加载器从同名环境变量
|
||||
@ -312,11 +315,30 @@ proxyAuth:
|
||||
|
||||
- `pool.maxSize`:当前系统维护且尚未 EXTRACTED 的 Proxy 硬上限,包括
|
||||
FETCHED、CHECKING、AVAILABLE、SUSPECT、DRAINING 和 pending expected。
|
||||
- `fetch.maxTotal`:当前运行或计费周期内,从 Provider 成功获取的累计上限;
|
||||
`0` 表示不设置累计上限。
|
||||
- `fetch.maxTotal`:当前 Redis generation 内的累计获取停止阈值;`0` 表示不设置。
|
||||
Redis 在调用前原子校验 `fetched total + pending expected + expected`,并在达到
|
||||
阈值后停止发起新调用。
|
||||
|
||||
`fetch.maxTotal` 不得小于 `pool.maxSize`。提取一个 Proxy 会释放当前库存位置,
|
||||
但不会恢复累计获取额度。
|
||||
但不会恢复累计获取额度。Provider 调用结果不确定、响应无法解析或 Permit 过期时,
|
||||
系统按 `estimatedIPsPerCall` 保守记账,避免故障或换主造成额度低估。Redis 全量
|
||||
状态丢失会创建新 generation,因此需要由外部计费系统提供跨 generation 的长期额度。
|
||||
|
||||
`estimatedIPsPerCall` 是预留估值,不是通用的 Provider 响应硬限制。如果某次实际
|
||||
合法返回量超过估值,系统只保留本地池容量允许的数量,但累计账本按实际合法数量
|
||||
记账并停止后续调用;该次可能越过停止阈值。需要绝对硬上限时,必须同时在 Provider
|
||||
请求参数中配置供应商支持的批量上限,并保证其不超过剩余额度。
|
||||
|
||||
当前实现限制单个 `pool.maxSize <= 1,000,000`、配置内 Upstream 总数不超过
|
||||
`4,096`,并要求单代理并发、Refill 双水位、理论总槽位及其他传入 Redis Lua 的
|
||||
累计/并发计数不超过 `2^53-1`。这些边界在配置加载和 Admin reload 提交前校验,
|
||||
不会等到 Provider Runtime 启动后才失败。
|
||||
|
||||
`proxyAuth.type: response` 的用户名/密码是 Parser 到 Redis Activity Adapter 之间的
|
||||
临时凭据。每次 Parser handoff 使用独立、幂等释放的 lease,避免并发 Fetch 互相
|
||||
删除或轮换版本覆盖;成功复制到 TTL 活动记录、候选被截断或解析失败后都会释放。
|
||||
内存 lease 上限按所有配置 Upstream 的 `pool.maxSize * fetch.maxInFlight` 汇总,
|
||||
配置 reload 只提高上限,不预分配对应内存。
|
||||
|
||||
### 7.3 Fetch 限制
|
||||
|
||||
@ -376,6 +398,12 @@ PostgreSQL 只保存配置版本、Upstream/Routing 管理状态、Admin 审计
|
||||
PostgreSQL 故障本身不应使 Redis 中可完成的 Extract 返回 `503`。Metrics 标签
|
||||
禁止 Proxy IP、Client ID、Session、完整 URL 和 Request ID。
|
||||
|
||||
启用 Admin 的多 Controller 部署必须让所有副本读取同一版本化配置源、Secret
|
||||
版本和 `PROXY_POOL_CONFIG_FINGERPRINT_KEY`。Supervisor 每秒比较本地完整配置的
|
||||
HMAC-SHA-256 与 PostgreSQL 权威值:管理状态暂时不可读时沿用 last-known
|
||||
Runtime;指纹已更新但本地源尚未同步时停止旧 Provider,待源匹配并通过预检后
|
||||
按全局 revision 恢复,避免旧凭据换主或迟到旧配置回写。
|
||||
|
||||
Metrics 启用时 `listen` 必须是合法 `host:port`。该入口固定提供 `/livez`、
|
||||
`/readyz` 和 `/metrics`,不复用 Distribution/Admin 的认证边界;外部访问必须由
|
||||
网络策略限制。当前 `/metrics` 已包含 Go/进程基础指标,Provider、提取和容量等
|
||||
|
||||
@ -143,6 +143,15 @@ type Proxy struct {
|
||||
运行态 `active` 与 `reserved` 存在 Worker 本地、按 Proxy ID 分片,不写入
|
||||
不可变 Snapshot。
|
||||
|
||||
Worker 以有界周期批量上报运行态,而不是在每个 Gateway 请求上写 Redis。
|
||||
报告通过公用 `workerruntime` seam 表达为完整稀疏替换:Gateway 从打包原子计数
|
||||
读取同一时刻的 Active/Reserved;已从当前 Snapshot 移除但仍有连接的 Proxy
|
||||
继续以 draining 状态上报。Controller 使用 session、单调 report sequence、
|
||||
ownership epoch、Controller 已 ACK 的 snapshot/epoch 上界和 Redis 服务端 TTL
|
||||
共同校验;缺失、过期或超前报告按零可用容量 fail-closed。运行态报告扫描有界
|
||||
当前 Snapshot,并以分片索引补充已移除但仍非零的 runtime;历史 Capacity
|
||||
注册表有硬上限,避免短 TTL Proxy 持续轮换导致心跳扫描与内存无界增长。
|
||||
|
||||
## 6. Proxy 状态机
|
||||
|
||||
```mermaid
|
||||
@ -244,6 +253,31 @@ flowchart TD
|
||||
fail-closed,不回退为本地 Leader。补池使用 minimum/target 双水位迟滞,库存
|
||||
复核期间若仍有 pending Fetch,则等待下一轮再同步 Managed,避免重复计数。
|
||||
|
||||
Controller Bootstrap 通过 `Provider Supervisor` 按权威配置和 PostgreSQL 管理状态
|
||||
维护每个 Upstream 的独立 `UpstreamRuntime`,再与 HTTP Runtime 通过公用 lifecycle
|
||||
Group 联动启动、取消和等待。Admin disable 会通知 Supervisor 取消对应 Runtime;
|
||||
reload 在提交前预检新 Runtime,并在发布后逐个取消、替换或新增。低频对账用于修复
|
||||
丢失通知。每个 Leader 任期重新创建本地补池预算与合并信号;Redis 是
|
||||
requestInterval、maxInFlight 和 maxTotal 的唯一分布式裁决者。本地预算只负责
|
||||
pool.maxSize。
|
||||
|
||||
maxTotal 使用 Redis generation 内的累计值与 pending expected 原子预留。调用结果
|
||||
不确定、解析失败或结算丢失时按 expected 保守记账,换主后旧 Permit 仍可幂等结算。
|
||||
如果 Provider 单次实际返回量超过 `estimatedIPsPerCall`,账本按实际合法数量记账并
|
||||
停止后续调用,但该次可能越过停止阈值;严格硬封顶需要 Provider API 支持可控批量。
|
||||
|
||||
响应中携带的代理凭据先进入有界内存凭据表,Redis Activity Adapter 复制凭据材料
|
||||
进入 TTL 活动记录后立即按版本 fence 释放临时引用;解析中途失败也会回收已写引用。
|
||||
因此短 TTL、持续轮换的新代理地址不会耗尽启动时的凭据容量。Provider 结果只记录
|
||||
每个 Upstream 的低基数 Empty/Error 计数,不保留响应体、代理地址或错误对象。
|
||||
|
||||
补池读取 `pool.InventoryReader` 返回的权威 `Managed` 与 `AvailableSlots`,不使用
|
||||
AVAILABLE Proxy 数量乘固定并发的近似值。Available Slots 同时计入 Proxy 状态、
|
||||
TTL safety margin、MaxConcurrency、Worker Active/Reserved 和当前 ownership;
|
||||
未知 Worker 运行态贡献零槽位。Redis 只扫描目标 Upstream 的未分配/已分配
|
||||
AVAILABLE 索引,Managed 读取已有权威计数;该读取属于 Controller 冷路径,
|
||||
Gateway 热路径仍只访问本地 Snapshot 和本地原子计数。
|
||||
|
||||
### 8.1 Empty、Duplicate 与 Error
|
||||
|
||||
- **Empty**:HTTP/认证成功、模板执行成功,解析后合法 Proxy 数为 0。
|
||||
@ -403,6 +437,12 @@ flowchart LR
|
||||
- Routing 立即对新请求生效;旧请求持有旧 Snapshot 完成。
|
||||
- 修改 API 地址或凭据版本会重建 Provider Adapter,但不会把错误计成 Empty。
|
||||
- 新配置任何校验失败时,保留旧版本并报告完整错误。
|
||||
- 多 Controller 以 PostgreSQL 中的 HMAC-SHA-256 完整配置指纹检测修订;HMAC
|
||||
密钥由外部 Secret 注入且所有副本一致。副本从共享配置源读取相同 revision,
|
||||
预检后通过单调 revision 栅栏原子发布。权威指纹已变化但本地源仍旧时,旧
|
||||
Provider fail-closed 停止,避免旧 URL/Secret 在 Leader 换主后继续使用。
|
||||
- PostgreSQL 瞬时读取失败时沿用 last-known Provider Runtime 并重试,不连带终止
|
||||
Distribution、Admin 或 Metrics。
|
||||
|
||||
## 15. 安全
|
||||
|
||||
@ -463,7 +503,7 @@ CPU、内存、网络、Go 版本、配置和上游响应模型下测得。
|
||||
| Provider 超时/500 | 计 Error、退避;不计 Empty,不影响已有 Proxy |
|
||||
| Provider 合法空响应 | Empty++;达到阈值触发相关 Routing 原子切换 |
|
||||
| Redis 不可用 | Gateway 暂用未过期快照;停止 Fetch 入池、Extract 和所有权变更 |
|
||||
| PostgreSQL 不可用 | Gateway 与 Redis Extract 不受影响;停止管理状态变更和 Admin 审计 |
|
||||
| PostgreSQL 不可用 | Gateway 与 Redis Extract 不受影响;停止管理写入,Provider 沿用 last-known 状态并重试 |
|
||||
| Controller 断线 | Worker 在 maxStaleAge 内继续;超限拒绝新流量并排空 |
|
||||
| Worker 崩溃 | 所有权租约过期后重新分配;期间不双重所有 |
|
||||
| Checker 积压 | 降低普通复检频率,优先新 Proxy 与 SUSPECT,不无限排队 |
|
||||
|
||||
@ -195,11 +195,14 @@ Admin/Distribution 必需依赖。共享 `platform/httpserver` 与
|
||||
已完成。pgx Adapter 已在真实 PostgreSQL 18 上运行同一公用契约,并验证
|
||||
Repeatable Read 快照、`SKIP LOCKED`、原子 ACK、审计/Outbox 故障回滚和数据边界。
|
||||
Admin `ApplicationService` 已将 mutation、权威管理快照、低基数运行态
|
||||
聚合与配置重载接到同一公用 seam;严格文件加载、脱敏管理摘要及原子配置发布
|
||||
已通过失败路径和并发测试。`cmd/proxy-controller` 与公用 `controller/bootstrap`
|
||||
聚合与配置重载接到同一公用 seam;严格文件加载、外部密钥 HMAC 管理指纹及
|
||||
revision 单调配置发布已通过失败路径和确定性并发测试。`cmd/proxy-controller` 与公用 `controller/bootstrap`
|
||||
已完成配置单次加载、PostgreSQL 连接/迁移、Redis 活动池、状态聚合、
|
||||
Distribution/Admin 服务构造、错误合并和资源关闭;Provider 调度及完整 HTTP
|
||||
进程端到端测试仍待实现。Controller Metrics 独立入口现已提供 `/livez`、
|
||||
Distribution/Admin 服务构造、错误合并和资源关闭。生产 Provider Supervisor 已按
|
||||
权威管理状态动态装配 Upstream,并与 HTTP Runtime 通过公用 lifecycle Group 联动
|
||||
停机;Admin disable 会取消 Runtime,reload 在提交前预检并在发布后替换运行实例。
|
||||
组合 fixture 已验证隔离 Redis namespace 下的选主、Provider HTTP 调用、模板解析
|
||||
和活动池写入。Controller Metrics 独立入口现已提供 `/livez`、
|
||||
`/readyz` 与基础 Prometheus 运行时指标,三监听器隔离已通过测试;业务指标仍待
|
||||
实现。双存储 bootstrap 已通过 PostgreSQL 18 + Redis 8.2 组合 fixture,覆盖
|
||||
迁移、启动配置提交、Readiness、Admin Status 和 Metrics 探针。
|
||||
@ -219,8 +222,23 @@ generation + epoch fence、全局 requestInterval、全局 maxInFlight Permit、
|
||||
回收及 Redis 状态丢失后的新 generation 自动重建;Redis 异常期间不发放请求。
|
||||
补池配置新增必填 `refill` 双水位和 `fetch.estimatedIPsPerCall`,Pool Reconciler
|
||||
已实现迟滞与 pending 槽位折算,FetchBudget 仅在无 pending 时同步 Redis 权威
|
||||
Managed。Provider Fleet、Worker Active/Reserved 汇总和 bootstrap 接线仍待完成,
|
||||
因此本轮不勾选 Task 10 的组合验收项。
|
||||
Managed。Gateway 已增加打包原子 Active/Reserved 读取与完整稀疏运行态快照;
|
||||
公用 `workerruntime` session/report/read seam 同时提供并发安全内存参考实现和
|
||||
生产 Redis Adapter。Redis 以服务端时间、Worker session、已 ACK snapshot/epoch、
|
||||
单调 report sequence 和报告 TTL 原子隔离旧实例,并由 `pool.InventoryReader` 汇总
|
||||
权威 Managed/Available Slots;真实 Redis 8.2 已覆盖空报告、幂等重放、倒序、
|
||||
冲突、超前 epoch、过期、单 Upstream 扫描隔离和预算耗尽的 fail-closed 行为。
|
||||
Redis Provider Permit 现已把 requestInterval、maxInFlight 与 maxTotal 放在同一
|
||||
原子边界,按 expected 预留、实际合法数量结算,并支持失败保守计费、换主后结算和
|
||||
过期回收。响应型代理凭据使用独立、有界 lease,在 Redis Upsert、候选截断或解析
|
||||
失败后按版本释放;Provider Empty/Error 低基数计数已接入 Admin Status,配置删除
|
||||
时回收历史统计容量。Redis inventory 扫描上限固定覆盖配置允许的最大池,支持小池
|
||||
启动后动态扩容。Supervisor 以 PostgreSQL 权威 HMAC 指纹和 revision 栅栏协调
|
||||
多副本 reload:管理库瞬断沿用 last-known 状态,本地共享源落后时停止旧 Provider,
|
||||
源匹配并预检后自动替换,迟到旧 revision 不覆盖新配置。
|
||||
WorkerControlPlane gRPC 接收端、session 签发/心跳、Snapshot ACK
|
||||
账本、Client 分布式限流和健康执行链仍待完成,因此本轮不勾选 Task 10 的组合
|
||||
验收项。
|
||||
|
||||
## Task 11: Checker and Health Reducer
|
||||
|
||||
|
||||
@ -6,7 +6,7 @@
|
||||
- [ ] 每个 Proxy 同一时刻最多归属一个 Worker,ownership epoch 单调。
|
||||
- [ ] Reserved -> Active 使用单个原子转换,无超卖与负计数。
|
||||
- [ ] Sequential 并发 Empty 只切换一次,旧 Upstream Proxy 自然耗尽。
|
||||
- [ ] `pool.maxSize` 与 `fetch.maxTotal` 分别按当前库存和累计获取计数。
|
||||
- [x] `pool.maxSize` 与 `fetch.maxTotal` 分别按当前库存和累计获取计数。
|
||||
- [ ] Extract 只有 `AVAILABLE -> EXTRACTED`,OpenAPI 不存在 release/renew。
|
||||
- [ ] Extract 状态更新和短期幂等结果位于同一个 Redis 原子操作。
|
||||
- [ ] PostgreSQL 中不存在 Proxy 明细或逐次提取记录。
|
||||
@ -20,6 +20,7 @@
|
||||
- [ ] trusted proxy 只包含受控 LoadBalancer/Ingress 网段。
|
||||
- [ ] 解析前后均拦截私网、回环、链路本地、元数据地址与 DNS Rebinding。
|
||||
- [ ] Secret 由外部密钥系统注入,镜像、ConfigMap、日志没有明文。
|
||||
- [ ] 所有 Controller 使用同一枚至少 32 字节的独立配置 HMAC 密钥,并完成轮换演练。
|
||||
- [ ] Pod 以非 root、只读根文件系统、无 Linux capabilities 运行。
|
||||
- [ ] NetworkPolicy 默认拒绝,外部数据库/Redis/Provider 网段已收紧。
|
||||
- [ ] Provider 模板有响应大小、执行时间、函数与外部访问限制。
|
||||
|
||||
@ -19,9 +19,9 @@
|
||||
## 2. 本地拓扑模板
|
||||
|
||||
`cmd/proxy-controller` 已完成配置单次加载、PostgreSQL 迁移、Redis 活动池、
|
||||
Distribution/Admin/Metrics 独立监听和有界停机装配。Provider 自动补池、业务
|
||||
指标以及 Gateway/Checker/Loadgen 三个进程仍属于 `implementation-plan.md`
|
||||
后续任务。
|
||||
Distribution/Admin/Metrics 独立监听和有界停机装配。Provider 自动补池、分布式
|
||||
配额、动态重载和 Admin 低基数统计已装配;完整 Gateway/Checker/Loadgen 与 Worker
|
||||
控制面仍属于 `implementation-plan.md` 后续任务。
|
||||
因此 Compose/Kubernetes 资产当前仍用于评审网络、资源、探针和依赖关系,不能
|
||||
视为完整可运行拓扑。
|
||||
|
||||
@ -40,6 +40,7 @@ Distribution/Admin/Metrics 独立监听和有界停机装配。Provider 自动
|
||||
$env:PROXY_POOL_GATEWAY_PASSWORD = "LOCAL_GATEWAY_PASSWORD"
|
||||
$env:PROXY_POOL_EXTRACT_TOKEN = "LOCAL_EXTRACT_TOKEN"
|
||||
$env:PROXY_POOL_ADMIN_TOKEN = "LOCAL_ADMIN_TOKEN"
|
||||
$env:PROXY_POOL_CONFIG_FINGERPRINT_KEY = "LOCAL_HIGH_ENTROPY_KEY_AT_LEAST_32_BYTES"
|
||||
$env:PROVIDER_A_TOKEN = "PROVIDER_A_TOKEN"
|
||||
$env:PROVIDER_B_TOKEN = "PROVIDER_B_TOKEN"
|
||||
```
|
||||
@ -90,6 +91,8 @@ Fetch 应表现为 Error 与退避,不应增加 Empty 计数,也不影响已
|
||||
1. 使用托管 PostgreSQL 和 Redis,分别配置 TLS、备份、监控和多可用区。
|
||||
2. 复制 `secret.example.yaml` 到环境私密配置系统,由 External Secrets、SOPS
|
||||
或密钥管理平台生成 `proxy-pool-secrets`,不要提交真实 Secret。
|
||||
`PROXY_POOL_CONFIG_FINGERPRINT_KEY` 必须使用至少 32 字节的高熵随机值,所有
|
||||
Controller 副本保持一致,且与配置中的业务 Secret 分离管理。
|
||||
3. 在环境 Overlay 替换镜像、Provider 地址、允许网段、外部存储地址、资源量
|
||||
和 LoadBalancer 注解。
|
||||
4. 根据集群 CNI 能力收紧 NetworkPolicy 的外部网段。
|
||||
@ -221,11 +224,15 @@ Outbox 发布器必须以稳定 consumer ID 有界领取;发布成功后原子
|
||||
|
||||
1. Gateway 继续使用最后有效 Snapshot。
|
||||
2. Redis 健康且运行配置有效时,Distribution 继续执行原子 Extract,Provider
|
||||
继续刷新 TTL 活动池。
|
||||
按 last-known 管理状态继续刷新 TTL 活动池;状态读取失败不得触发全进程退出。
|
||||
3. 拒绝配置版本、Upstream/Routing 管理状态和其他需要 Admin 审计/outbox 的写入;
|
||||
不得把 Proxy 明细临时落入 PostgreSQL。
|
||||
4. 恢复后核对迁移、管理事务回滚、Admin 审计与 outbox backlog;不存在 Proxy
|
||||
明细或逐次提取记录恢复步骤。
|
||||
5. 若权威配置指纹已变化,确认每个 Controller 的共享配置源和 Secret 版本已同步;
|
||||
同时确认 `PROXY_POOL_CONFIG_FINGERPRINT_KEY` 一致。指纹不匹配的副本会停止旧
|
||||
Provider,匹配并预检成功后按 PostgreSQL revision 自动恢复;迟到旧 revision
|
||||
不会覆盖较新本地配置。
|
||||
|
||||
### 7.3 Redis 不可用
|
||||
|
||||
|
||||
@ -10,7 +10,7 @@
|
||||
| ARCH-001 | 数据面 Worker 与控制面 Controller 分离 | 1-70 | 包、协议和部署拓扑已分离;Controller 命令已实现,Gateway/Checker/Loadgen 构建产物待实现 |
|
||||
| ARCH-002 | 热路径只做认证、本地路由和网络转发 | 1-70, 380-430 | Snapshot/Dispatch 及依赖边界已验证;完整 Gateway 进程与代表性性能剖析待完成 |
|
||||
| ARCH-003 | Gateway、Distribution、Admin、Metrics 独立入口 | 8904-8958 | Controller 命令已装配 Distribution/Admin/Metrics 三个独立监听及联动停机;Gateway 生产入口待装配 |
|
||||
| ARCH-004 | Controller 集中 Provider 获取与切换 | 1403-1580 | 单进程 Reconciler、合并通知和切换领域契约已完成;分布式 Leader 与运行装配待完成 |
|
||||
| ARCH-004 | Controller 集中 Provider 获取与切换 | 1403-1580 | Redis Leader、动态 Provider Supervisor 与 Bootstrap 生产装配已完成;Admin disable/reload 驱动取消替换,多副本按权威 HMAC 指纹和 revision 栅栏收敛并拒绝旧配置换主;Routing 切换到 Drain 的编排待完成 |
|
||||
| ARCH-005 | 100k QPS 峰值使用多 Worker 集群 | 当前会话 | 未验证设计目标;待代表性集群负载报告 |
|
||||
|
||||
## Routing 与 Upstream
|
||||
@ -32,11 +32,11 @@
|
||||
| FETCH-001 | 每个 Provider 有独立 requestInterval、maxInFlight、timeout 和 retry | 968-2394 | `provider/reconciler_test.go` |
|
||||
| FETCH-002 | 大量缺池信号合并为 singleflight/容量 1 通知 | 2067-2136, 8808-8849 | `coalesce.Signal` 与 100 并发通知测试 |
|
||||
| FETCH-003 | 错误使用指数退避和抖动,429 尊重 Retry-After | 1601-1831, 8808-8856 | `provider/reconciler_test.go` 与 `providerapi/http_adapter_test.go` |
|
||||
| FETCH-004 | Provider 获取由单逻辑 Leader 执行 | 1403-1580 | Redis Coordinator 已通过真实 Redis 双实例互斥、epoch 接管、全局间隔/在途 Permit 与 generation 重建测试;Provider Fleet/bootstrap 接线待完成 |
|
||||
| FETCH-004 | Provider 获取由单逻辑 Leader 执行 | 1403-1580 | Redis Coordinator 已通过真实 Redis 双实例互斥、epoch 接管、全局间隔/在途 Permit 与 generation 重建测试;生产 Supervisor/bootstrap 已通过 Admin disable 和 Provider HTTP 到隔离 Redis 库存的组合 fixture |
|
||||
| FETCH-005 | Empty 与 Error 分开;只有合法候选为零时 Empty++ | 8442-8529 | `fetch_result_test.go` 分类矩阵 |
|
||||
| FETCH-006 | 重复候选不当作 Empty,记录独立指标 | 8442-8480 | DuplicateOnly 分类与 Provider 测试 |
|
||||
| FETCH-007 | 模板限制响应大小、执行时间、函数集和外部访问 | 8808-8856 | `providerapi/template_parser_test.go` 输入、输出、候选、超时、递归与函数白名单测试 |
|
||||
| FETCH-008 | pool.maxSize 与 fetch.maxTotal 语义分离 | 9190-9280 | `FetchBudget` 并发预占/释放测试 |
|
||||
| FETCH-008 | pool.maxSize 与 fetch.maxTotal 语义分离 | 9190-9280 | 本地 `FetchBudget` 仅约束当前库存;Redis Permit 原子维护累计与 pending 额度,并通过换主、取消、幂等和过期保守结算测试 |
|
||||
|
||||
## Proxy 生命周期与容量
|
||||
|
||||
@ -46,7 +46,7 @@
|
||||
| PROXY-002 | 唯一键包含 scheme、host、port、username、credentialVersion | 6655-6727, 8605-8678 | 去重单测 |
|
||||
| PROXY-003 | TTL 来源优先级明确并统一 UTC | 681-747, 8655-8678 | TTL 表驱动测试 |
|
||||
| CAP-001 | Gateway 分配使用 Reserved -> Active 原子转换 | 1203-1467, 8530-8597 | 固定 Max 下打包 CAS 与 1,000 并发不超卖已完成;动态降容和完整生命周期证据待完成 |
|
||||
| CAP-002 | 补池依据 Available Slots,不只看 Proxy 数量 | 1203-1402, 8530-8597 | `AvailableSlots`、显式 minimum/target 水位、pending 槽位和迟滞 Reconciler 已测试;Worker Active/Reserved、ownership、目标健康及 Gateway reserve 运行时聚合待完成 |
|
||||
| CAP-002 | 补池依据 Available Slots,不只看 Proxy 数量 | 1203-1402, 8530-8597 | `AvailableSlots`、显式 minimum/target 水位、pending 槽位和迟滞 Reconciler 已测试;Gateway 打包 Active/Reserved 报告、Worker session/ACK/sequence/TTL/ownership fence、单 Upstream 索引及 Redis 权威 Managed/Slots 汇总已通过内存与真实 Redis 测试;WorkerControlPlane 接线、目标健康和 Gateway reserve 策略仍待完成 |
|
||||
| CAP-003 | pool.maxSize 包括 FETCHED/CHECKING/AVAILABLE/SUSPECT/DRAINING 与 pending expected | 3001-3533, 6642-6680 | `FetchBudget` 100 并发额度预占测试 |
|
||||
| CAP-004 | TTL safety margin 内禁止新分配 | 173-220, 6728-6741 | 时钟测试 |
|
||||
| CAP-005 | 多 Worker 不在热路径访问 Redis 计数 | 1403-1467 | Gateway 包依赖审计、Snapshot/Dispatch 测试 |
|
||||
|
||||
@ -22,11 +22,20 @@
|
||||
4. 连续 4 次 Empty 后 Valid 不切换;连续 5 次只从 A 切到 B。
|
||||
5. Error 和 DuplicateOnly 不累计 Empty。
|
||||
6. 100 个缺池信号只形成一个合并 Provider reconcile。
|
||||
7. 并发 Fetch 不突破 `pool.maxSize` 与 `fetch.maxTotal`。
|
||||
7. 并发 Fetch 不突破 `pool.maxSize`,且不会在 Redis `fetch.maxTotal` 停止阈值耗尽后发起新调用。
|
||||
8. TTL safety margin 内不再分配。
|
||||
9. Snapshot 版本断档、目标错误或校验和错误不替换当前视图。
|
||||
10. 非幂等 HTTP 和已建立 CONNECT 不自动重放。
|
||||
11. 所有 Upstream 不可用时严格执行显式策略。
|
||||
11. Admin disable 取消目标 Provider Runtime;reload 构造失败不提交,成功时替换运行实例。
|
||||
12. 短 TTL 响应凭据使用独立 lease;并发 Fetch、截断、解析失败和 Redis Upsert
|
||||
后均准确释放且不互相撤销。
|
||||
13. PostgreSQL 状态瞬断保留 last-known Provider;多副本本地指纹落后时停止旧
|
||||
Runtime,共享源同步后自动预检并恢复;不同 HMAC 密钥不能误判为相同配置。
|
||||
14. 并发配置提交与 Supervisor 同步按 revision 单调发布,迟到旧 revision 不覆盖
|
||||
已发布新配置。
|
||||
15. 配置删除的 Provider 统计项被回收,禁用但仍配置的统计项保留,容量可复用。
|
||||
16. 小池启动后重载到大池时,Redis inventory 扫描上限仍覆盖配置允许的最大池。
|
||||
17. 所有 Upstream 不可用时严格执行显式策略。
|
||||
|
||||
## 3. 基础质量门禁
|
||||
|
||||
@ -74,16 +83,17 @@ soak 测试单独标记,不混入快速单测。
|
||||
|
||||
## 5. 当前本地微基准
|
||||
|
||||
2026-07-28,Windows/amd64、Intel Core Ultra 7 155H:
|
||||
2026-07-30,Windows/amd64、Intel Core Ultra 7 155H:
|
||||
|
||||
```text
|
||||
BenchmarkAcquire100kIndexed-22 3553592 640.0 ns/op 256 B/op 2 allocs/op
|
||||
BenchmarkStoreApply100k-22 1 472.7 ms/op 654 MB/op 2700642 allocs/op
|
||||
BenchmarkAcquire100kIndexed-22 1000000-1867125 893.9-1047 ns/op 256 B/op 2 allocs/op
|
||||
BenchmarkStoreApply100k-22 1 518.7 ms/op 540 MB/op 3000887 allocs/op
|
||||
```
|
||||
|
||||
`Acquire` 已使用 scheme/upstream/tag 索引,结果只代表本地选择和容量预留。
|
||||
`Store.Apply` 属于冷路径且当前内存开销较高;运行态为防止旧快照在途连接超配,
|
||||
暂不自动回收曾出现过的 Proxy ID。后续需要基于 RCU/引用计数定义安全回收点。
|
||||
暂不自动回收曾出现过的 Proxy ID,但注册表有 1,000,000 项硬上限,达到上限时
|
||||
拒绝新 Snapshot 并保留旧视图。后续需要基于 RCU/引用计数定义安全回收点。
|
||||
这些数据不包含网络、认证、Provider、存储或多 Worker 协调,不能作为
|
||||
100k QPS 端到端验收结论。
|
||||
|
||||
|
||||
@ -120,7 +120,7 @@ Redis Readiness 和 Admin Status;HTTP Runner 使用测试 Adapter,避免占
|
||||
|
||||
Admin 应用层测试覆盖 typed-nil 依赖、Actor/SourceIP 映射、Routing CAS 错误、
|
||||
权威管理快照与低基数运行态聚合、未知字段拒绝、主配置/Secret 文件 I/O 分类、
|
||||
持久化失败不发布、幂等重放发布、脱敏管理摘要和原子配置 Store 并发读写。静态
|
||||
持久化失败不发布、HMAC 管理指纹、revision 单调发布和原子配置 Store 并发读写。静态
|
||||
导入边界测试禁止 Admin 引用 Redis Activity/Extract 与 Proxy 明细包。
|
||||
|
||||
需要 PostgreSQL/Redis 的测试使用独立实例和短生命周期容器,不复用开发数据。
|
||||
|
||||
@ -17,11 +17,12 @@ import (
|
||||
)
|
||||
|
||||
const (
|
||||
defaultTemplateTimeout = 100 * time.Millisecond
|
||||
defaultTemplateMaxBytes = int64(1 << 20)
|
||||
defaultMaxCandidates = 10_000
|
||||
maxRegexPatterns = 64
|
||||
maxRegexPatternBytes = 1024
|
||||
defaultTemplateTimeout = 100 * time.Millisecond
|
||||
defaultTemplateMaxBytes = int64(1 << 20)
|
||||
defaultMaxCandidates = 10_000
|
||||
credentialReleaseTimeout = time.Second
|
||||
maxRegexPatterns = 64
|
||||
maxRegexPatternBytes = 1024
|
||||
)
|
||||
|
||||
type TemplateParser struct {
|
||||
@ -137,7 +138,14 @@ func NewTemplateParser(
|
||||
return parser, nil
|
||||
}
|
||||
|
||||
func (p *TemplateParser) Parse(ctx context.Context, body []byte) ([]proxyDomain.Proxy, error) {
|
||||
func (p *TemplateParser) Parse(ctx context.Context, body []byte) (proxies []proxyDomain.Proxy, resultErr error) {
|
||||
storedCredentials := make([]credentials.Reference, 0)
|
||||
defer func() {
|
||||
if resultErr == nil {
|
||||
return
|
||||
}
|
||||
p.releaseCredentials(storedCredentials)
|
||||
}()
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@ -161,7 +169,7 @@ func (p *TemplateParser) Parse(ctx context.Context, body []byte) ([]proxyDomain.
|
||||
if len(tokens) > p.maxCandidates {
|
||||
return nil, &limitError{kind: ErrTooManyCandidates, size: int64(len(tokens)), limit: int64(p.maxCandidates)}
|
||||
}
|
||||
proxies := make([]proxyDomain.Proxy, 0, len(tokens))
|
||||
proxies = make([]proxyDomain.Proxy, 0, len(tokens))
|
||||
credentialIndexes := make(map[string]int)
|
||||
for _, token := range tokens {
|
||||
candidate, credential, ok := p.parseCandidate(token)
|
||||
@ -183,6 +191,7 @@ func (p *TemplateParser) Parse(ctx context.Context, body []byte) ([]proxyDomain.
|
||||
}
|
||||
candidate.SecretRef = reference.SecretRef
|
||||
candidate.CredentialVersion = reference.CredentialVersion
|
||||
storedCredentials = append(storedCredentials, reference)
|
||||
credentialKey = candidateCredentialKey(candidate)
|
||||
if index, exists := credentialIndexes[credentialKey]; exists {
|
||||
proxies[index] = candidate
|
||||
@ -201,9 +210,65 @@ func (p *TemplateParser) Parse(ctx context.Context, body []byte) ([]proxyDomain.
|
||||
if len(tokens) > 0 && len(proxies) == 0 {
|
||||
return nil, ErrInvalidProxyOutput
|
||||
}
|
||||
p.releaseUnusedCredentials(storedCredentials, proxies)
|
||||
return proxies, nil
|
||||
}
|
||||
|
||||
func (p *TemplateParser) ReleaseCandidates(candidates []proxyDomain.Proxy) {
|
||||
references := make([]credentials.Reference, 0, len(candidates))
|
||||
for _, candidate := range candidates {
|
||||
if candidate.SecretRef == "" || candidate.CredentialVersion == "" {
|
||||
continue
|
||||
}
|
||||
references = append(references, credentials.Reference{
|
||||
SecretRef: candidate.SecretRef, CredentialVersion: candidate.CredentialVersion,
|
||||
})
|
||||
}
|
||||
p.releaseCredentials(references)
|
||||
}
|
||||
|
||||
func (p *TemplateParser) releaseUnusedCredentials(
|
||||
stored []credentials.Reference,
|
||||
candidates []proxyDomain.Proxy,
|
||||
) {
|
||||
retained := make(map[credentials.Reference]int, len(candidates))
|
||||
for _, candidate := range candidates {
|
||||
if candidate.SecretRef == "" || candidate.CredentialVersion == "" {
|
||||
continue
|
||||
}
|
||||
retained[credentials.Reference{
|
||||
SecretRef: candidate.SecretRef, CredentialVersion: candidate.CredentialVersion,
|
||||
}]++
|
||||
}
|
||||
unused := make([]credentials.Reference, 0, len(stored))
|
||||
for _, reference := range stored {
|
||||
if retained[reference] > 0 {
|
||||
retained[reference]--
|
||||
continue
|
||||
}
|
||||
unused = append(unused, reference)
|
||||
}
|
||||
p.releaseCredentials(unused)
|
||||
}
|
||||
|
||||
func (p *TemplateParser) releaseCredentials(references []credentials.Reference) {
|
||||
if p == nil {
|
||||
return
|
||||
}
|
||||
releaser, ok := p.credentialStore.(credentials.Releaser)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
releaseCtx, cancel := context.WithTimeout(context.Background(), credentialReleaseTimeout)
|
||||
defer cancel()
|
||||
for _, reference := range references {
|
||||
if releaseCtx.Err() != nil {
|
||||
return
|
||||
}
|
||||
_ = releaser.Release(releaseCtx, reference)
|
||||
}
|
||||
}
|
||||
|
||||
func (p *TemplateParser) regexFind(pattern, value string) (string, error) {
|
||||
compiled, err := p.compileRegex(pattern)
|
||||
if err != nil {
|
||||
|
||||
@ -318,7 +318,7 @@ func TestTemplateParserDoesNotOverrideStaticProxyAuthFromResponse(t *testing.T)
|
||||
}
|
||||
|
||||
func TestTemplateParserRetainsDistinctEndpointsSharingStaticCredentials(t *testing.T) {
|
||||
store, err := credentials.NewMemoryStore(1)
|
||||
store, err := credentials.NewMemoryStore(2)
|
||||
if err != nil {
|
||||
t.Fatalf("NewMemoryStore(): %v", err)
|
||||
}
|
||||
@ -343,8 +343,11 @@ func TestTemplateParserRetainsDistinctEndpointsSharingStaticCredentials(t *testi
|
||||
if len(proxies) != 2 {
|
||||
t.Fatalf("proxy count = %d, want both static-auth endpoints", len(proxies))
|
||||
}
|
||||
if proxies[0].SecretRef == "" || proxies[0].SecretRef != proxies[1].SecretRef {
|
||||
t.Fatalf("static credential references = %q and %q, want same opaque reference", proxies[0].SecretRef, proxies[1].SecretRef)
|
||||
if proxies[0].SecretRef == "" || proxies[1].SecretRef == "" || proxies[0].SecretRef == proxies[1].SecretRef {
|
||||
t.Fatalf("static credential references = %q and %q, want independent leases", proxies[0].SecretRef, proxies[1].SecretRef)
|
||||
}
|
||||
if proxies[0].CredentialVersion != proxies[1].CredentialVersion {
|
||||
t.Fatalf("static credential versions = %q and %q, want same value version", proxies[0].CredentialVersion, proxies[1].CredentialVersion)
|
||||
}
|
||||
}
|
||||
|
||||
@ -391,6 +394,33 @@ func TestTemplateParserStoresResponseCredentialsByOpaqueReference(t *testing.T)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTemplateParserReleasesPartialCredentialsWhenParseFails(t *testing.T) {
|
||||
store, err := credentials.NewMemoryStore(1)
|
||||
if err != nil {
|
||||
t.Fatalf("NewMemoryStore(): %v", err)
|
||||
}
|
||||
parser, err := newTemplateParser("provider-a", config.Upstream{
|
||||
Provider: config.Provider{Protocols: []string{"http"}},
|
||||
API: config.ProviderAPI{Template: strings.Join([]string{
|
||||
"http://alice:first-password@192.0.2.10:8080",
|
||||
"http://bob:second-password@192.0.2.11:8080",
|
||||
}, "\n")},
|
||||
ProxyAuth: config.ProxyAuth{Type: "response"},
|
||||
Pool: config.Pool{MaxSize: 2},
|
||||
Fetch: config.Fetch{MaxResponseBytes: 1024, TemplateTimeout: config.Duration(time.Second)},
|
||||
}, store)
|
||||
if err != nil {
|
||||
t.Fatalf("newTemplateParser(): %v", err)
|
||||
}
|
||||
|
||||
if _, err := parser.Parse(context.Background(), nil); !errors.Is(err, credentials.ErrCapacityExceeded) {
|
||||
t.Fatalf("Parse() error = %v, want ErrCapacityExceeded", err)
|
||||
}
|
||||
if _, err := store.Put(context.Background(), "replacement", credentials.Value{Password: "replacement"}); err != nil {
|
||||
t.Fatalf("Put(after failed parse): %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTemplateParserKeepsDistinctAccountsForSameEndpointResolvable(t *testing.T) {
|
||||
store, err := credentials.NewMemoryStore(2)
|
||||
if err != nil {
|
||||
@ -432,7 +462,7 @@ func TestTemplateParserKeepsDistinctAccountsForSameEndpointResolvable(t *testing
|
||||
}
|
||||
|
||||
func TestTemplateParserKeepsLatestCredentialVersionWithinOneResponse(t *testing.T) {
|
||||
store, err := credentials.NewMemoryStore(1)
|
||||
store, err := credentials.NewMemoryStore(2)
|
||||
if err != nil {
|
||||
t.Fatalf("NewMemoryStore(): %v", err)
|
||||
}
|
||||
@ -467,6 +497,36 @@ func TestTemplateParserKeepsLatestCredentialVersionWithinOneResponse(t *testing.
|
||||
if value.Password != "new-secret" {
|
||||
t.Fatalf("resolved latest password mismatch")
|
||||
}
|
||||
parser.ReleaseCandidates(proxies)
|
||||
if _, err := store.Put(context.Background(), "replacement", credentials.Value{Password: "replacement"}); err != nil {
|
||||
t.Fatalf("Put(after releasing latest candidates): %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTemplateParserReleaseCandidatesReturnsCredentialCapacity(t *testing.T) {
|
||||
store, err := credentials.NewMemoryStore(1)
|
||||
if err != nil {
|
||||
t.Fatalf("NewMemoryStore(): %v", err)
|
||||
}
|
||||
parser, err := newTemplateParser("provider-a", config.Upstream{
|
||||
Provider: config.Provider{Protocols: []string{"http"}},
|
||||
API: config.ProviderAPI{Template: "http://alice:secret@192.0.2.10:8080"},
|
||||
ProxyAuth: config.ProxyAuth{Type: "response"},
|
||||
Pool: config.Pool{MaxSize: 1},
|
||||
Fetch: config.Fetch{MaxResponseBytes: 1024, TemplateTimeout: config.Duration(time.Second)},
|
||||
}, store)
|
||||
if err != nil {
|
||||
t.Fatalf("NewTemplateParser(): %v", err)
|
||||
}
|
||||
|
||||
proxies, err := parser.Parse(context.Background(), nil)
|
||||
if err != nil {
|
||||
t.Fatalf("Parse(): %v", err)
|
||||
}
|
||||
parser.ReleaseCandidates(proxies)
|
||||
if _, err := store.Put(context.Background(), "replacement", credentials.Value{Password: "replacement"}); err != nil {
|
||||
t.Fatalf("Put(after ReleaseCandidates): %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTemplateParserRedactsCredentialStoreErrors(t *testing.T) {
|
||||
|
||||
@ -19,33 +19,45 @@ var (
|
||||
)
|
||||
|
||||
type Options struct {
|
||||
Namespace string
|
||||
Credentials credentials.Store
|
||||
OperationTTL time.Duration
|
||||
MaxCandidateScan int
|
||||
CleanupLimit int
|
||||
Namespace string
|
||||
Credentials credentials.Store
|
||||
OperationTTL time.Duration
|
||||
MaxCandidateScan int
|
||||
MaxRuntimeCounters int
|
||||
MaxInventoryScan int
|
||||
CleanupLimit int
|
||||
}
|
||||
|
||||
type Adapter struct {
|
||||
client redis.Scripter
|
||||
credentials credentials.Store
|
||||
keys keyspace
|
||||
options Options
|
||||
client redis.Scripter
|
||||
credentials credentials.Store
|
||||
credentialReleaser credentials.Releaser
|
||||
keys keyspace
|
||||
options Options
|
||||
}
|
||||
|
||||
func New(client redis.Scripter, options Options) (*Adapter, error) {
|
||||
options.Namespace = strings.TrimSpace(options.Namespace)
|
||||
if options.MaxRuntimeCounters == 0 {
|
||||
options.MaxRuntimeCounters = options.MaxCandidateScan
|
||||
}
|
||||
if options.MaxInventoryScan == 0 {
|
||||
options.MaxInventoryScan = options.MaxCandidateScan
|
||||
}
|
||||
if nilInterface(client) || nilInterface(options.Credentials) ||
|
||||
!namespacePattern.MatchString(options.Namespace) || options.OperationTTL <= 0 ||
|
||||
options.MaxCandidateScan <= 0 || options.CleanupLimit <= 0 {
|
||||
options.MaxCandidateScan <= 0 || options.MaxRuntimeCounters <= 0 ||
|
||||
options.MaxInventoryScan <= 0 || options.CleanupLimit <= 0 {
|
||||
return nil, ErrInvalidOptions
|
||||
}
|
||||
return &Adapter{
|
||||
adapter := &Adapter{
|
||||
client: client,
|
||||
credentials: options.Credentials,
|
||||
keys: newKeyspace(options.Namespace),
|
||||
options: options,
|
||||
}, nil
|
||||
}
|
||||
adapter.credentialReleaser, _ = options.Credentials.(credentials.Releaser)
|
||||
return adapter, nil
|
||||
}
|
||||
|
||||
func (a *Adapter) Format(state fmt.State, _ rune) {
|
||||
|
||||
@ -58,6 +58,8 @@ func TestNewRejectsInvalidDependenciesAndOptions(t *testing.T) {
|
||||
{name: "colon in namespace", client: client, options: withNamespace(valid, "tenant:other")},
|
||||
{name: "zero operation ttl", client: client, options: withOperationTTL(valid, 0)},
|
||||
{name: "zero candidate scan", client: client, options: withMaxCandidateScan(valid, 0)},
|
||||
{name: "negative runtime counters", client: client, options: withMaxRuntimeCounters(valid, -1)},
|
||||
{name: "negative inventory scan", client: client, options: withMaxInventoryScan(valid, -1)},
|
||||
{name: "negative cleanup limit", client: client, options: withCleanupLimit(valid, -1)},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
@ -93,7 +95,9 @@ func TestNewBuildsClusterSafeKeyspaceAndHashesDynamicTokens(t *testing.T) {
|
||||
adapter.keys.records, adapter.keys.unique, adapter.keys.idkeys,
|
||||
adapter.keys.expiry, adapter.keys.available, adapter.keys.owners,
|
||||
adapter.keys.ownerExpiry, adapter.keys.epoch, adapter.keys.inventory,
|
||||
adapter.keys.stateInventory,
|
||||
adapter.keys.stateInventory, adapter.keys.workerSessions,
|
||||
adapter.keys.workerSessionExpiry, adapter.keys.workerRuntime,
|
||||
adapter.keys.workerRuntimeExpiry,
|
||||
}
|
||||
for _, key := range staticKeys {
|
||||
if strings.Count(key, "{activity}") != 1 || strings.Count(key, "{") != 1 || strings.Count(key, "}") != 1 {
|
||||
@ -109,6 +113,7 @@ func TestNewBuildsClusterSafeKeyspaceAndHashesDynamicTokens(t *testing.T) {
|
||||
adapter.keys.region(raw),
|
||||
adapter.keys.carrier(raw),
|
||||
adapter.keys.upstream(raw),
|
||||
adapter.keys.owned(raw),
|
||||
}
|
||||
for _, key := range dynamicKeys {
|
||||
if strings.Contains(key, raw) || strings.Count(key, "{activity}") != 1 || strings.Count(key, "{") != 1 {
|
||||
@ -161,6 +166,7 @@ func TestProxyRecordCodecIsDeterministicStrictAndRedacted(t *testing.T) {
|
||||
UsableUntilMS: 58_000, LastCheckedAtMS: 2_000, LastSuccessAtMS: 2_000,
|
||||
LatencyNS: int64(25 * time.Millisecond), MaxConcurrency: 8,
|
||||
State: string(proxyDomain.StateAvailable), Tags: map[string]string{"region": "cn", "carrier": "ct"},
|
||||
OwnerIndexKey: "pp:{activity}:test:owned:index",
|
||||
}
|
||||
first, err := encodeProxyRecord(record)
|
||||
if err != nil {
|
||||
@ -260,6 +266,16 @@ func withMaxCandidateScan(options Options, limit int) Options {
|
||||
return options
|
||||
}
|
||||
|
||||
func withMaxRuntimeCounters(options Options, limit int) Options {
|
||||
options.MaxRuntimeCounters = limit
|
||||
return options
|
||||
}
|
||||
|
||||
func withMaxInventoryScan(options Options, limit int) Options {
|
||||
options.MaxInventoryScan = limit
|
||||
return options
|
||||
}
|
||||
|
||||
func withCleanupLimit(options Options, limit int) Options {
|
||||
options.CleanupLimit = limit
|
||||
return options
|
||||
|
||||
45
internal/adapters/redisactivity/capacity.go
Normal file
45
internal/adapters/redisactivity/capacity.go
Normal file
@ -0,0 +1,45 @@
|
||||
package redisactivity
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
|
||||
controllerPool "proxy-pool/internal/controller/pool"
|
||||
"proxy-pool/internal/domain/activitypool"
|
||||
)
|
||||
|
||||
var _ controllerPool.InventoryReader = (*Adapter)(nil)
|
||||
|
||||
func (a *Adapter) ReadInventory(
|
||||
ctx context.Context,
|
||||
upstreamID string,
|
||||
safetyMargin time.Duration,
|
||||
) (controllerPool.InventorySnapshot, error) {
|
||||
if ctx == nil || a == nil || !runtimeClean(upstreamID) || safetyMargin < 0 {
|
||||
return controllerPool.InventorySnapshot{}, activitypool.ErrInvalidInventory
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return controllerPool.InventorySnapshot{}, err
|
||||
}
|
||||
result, err := runScript(ctx, a.client, capacityScript, []string{
|
||||
a.keys.records, a.keys.inventory, a.keys.upstream(upstreamID), a.keys.owned(upstreamID), a.keys.owners,
|
||||
a.keys.workerSessions, a.keys.workerSessionExpiry,
|
||||
a.keys.workerRuntime, a.keys.workerRuntimeExpiry,
|
||||
}, upstreamID, durationMillis(safetyMargin), a.options.MaxInventoryScan, a.options.CleanupLimit)
|
||||
if err != nil {
|
||||
return controllerPool.InventorySnapshot{}, err
|
||||
}
|
||||
var reply capacityScriptReply
|
||||
if err := decodeScriptResult(result, &reply); err != nil {
|
||||
return controllerPool.InventorySnapshot{}, err
|
||||
}
|
||||
if reply.Status == scriptInvalid {
|
||||
return controllerPool.InventorySnapshot{}, activitypool.ErrInvalidInventory
|
||||
}
|
||||
if reply.Status != scriptOK || reply.Managed < 0 || reply.AvailableSlots < 0 {
|
||||
return controllerPool.InventorySnapshot{}, invalidScriptReply("capacity inventory is unavailable")
|
||||
}
|
||||
return controllerPool.InventorySnapshot{
|
||||
Managed: reply.Managed, AvailableSlots: reply.AvailableSlots,
|
||||
}, nil
|
||||
}
|
||||
105
internal/adapters/redisactivity/capacity_integration_test.go
Normal file
105
internal/adapters/redisactivity/capacity_integration_test.go
Normal file
@ -0,0 +1,105 @@
|
||||
//go:build integration
|
||||
|
||||
package redisactivity
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
controllerPool "proxy-pool/internal/controller/pool"
|
||||
"proxy-pool/internal/domain/workerruntime"
|
||||
)
|
||||
|
||||
func TestRedisCapacityInventoryCombinesProxyAndWorkerRuntime(t *testing.T) {
|
||||
fixture := newRedisTestFixture(t)
|
||||
now := redisTestNow()
|
||||
unowned := testProxy("proxy-unowned", "192.0.2.10")
|
||||
unowned.MaxConcurrency = 10
|
||||
owned := testProxy("proxy-owned", "192.0.2.11")
|
||||
owned.MaxConcurrency = 10
|
||||
seedRedisAvailable(t, fixture.Adapter, "provider-a", now, now.Add(time.Second), 2*time.Minute, unowned)
|
||||
seedRedisAvailable(t, fixture.Adapter, "provider-a", now, now.Add(time.Second), 2*time.Minute, owned)
|
||||
assignment, err := fixture.Adapter.Assign(context.Background(), now.Add(2*time.Second),
|
||||
"proxy-owned", "worker-a", time.Minute)
|
||||
if err != nil {
|
||||
t.Fatalf("Assign(): %v", err)
|
||||
}
|
||||
if err := fixture.Adapter.ReplaceSession(context.Background(), workerruntime.Session{
|
||||
WorkerID: "worker-a", InstanceID: "instance-a", SessionID: "session-a",
|
||||
AckedSnapshotVersion: 1, AckedOwnershipEpoch: assignment.Epoch,
|
||||
}, time.Minute); err != nil {
|
||||
t.Fatalf("ReplaceSession(): %v", err)
|
||||
}
|
||||
if err := fixture.Adapter.ReplaceRuntime(context.Background(), workerruntime.Report{
|
||||
WorkerID: "worker-a", SessionID: "session-a", Sequence: 1,
|
||||
SnapshotVersion: 1, OwnershipEpoch: assignment.Epoch, ObservedAt: now,
|
||||
Counters: []workerruntime.Counter{{ProxyID: "proxy-owned", Active: 3, Reserved: 2}},
|
||||
}, time.Minute); err != nil {
|
||||
t.Fatalf("ReplaceRuntime(): %v", err)
|
||||
}
|
||||
|
||||
inventory, err := fixture.Adapter.ReadInventory(context.Background(), "provider-a", 0)
|
||||
if err != nil || inventory.Managed != 2 || inventory.AvailableSlots != 15 {
|
||||
t.Fatalf("ReadInventory() = %+v, %v; want managed=2 slots=15", inventory, err)
|
||||
}
|
||||
if _, ok := any(fixture.Adapter).(controllerPool.InventoryReader); !ok {
|
||||
t.Fatal("Adapter does not implement pool.InventoryReader")
|
||||
}
|
||||
inventory, err = fixture.Adapter.ReadInventory(context.Background(), "provider-a", 2*time.Hour)
|
||||
if err != nil || inventory.Managed != 2 || inventory.AvailableSlots != 0 {
|
||||
t.Fatalf("ReadInventory(safety margin) = %+v, %v", inventory, err)
|
||||
}
|
||||
fixture.Adapter.options.MaxInventoryScan = 1
|
||||
if _, err := fixture.Adapter.ReadInventory(context.Background(), "provider-a", 0); err == nil {
|
||||
t.Fatal("ReadInventory(over scan limit) error = nil")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRedisCapacityInventoryFailsClosedForExpiredRuntime(t *testing.T) {
|
||||
fixture := newRedisTestFixture(t)
|
||||
now := redisTestNow()
|
||||
owned := testProxy("proxy-owned", "192.0.2.11")
|
||||
owned.MaxConcurrency = 10
|
||||
seedRedisAvailable(t, fixture.Adapter, "provider-a", now, now.Add(time.Second), 2*time.Minute, owned)
|
||||
assignment, err := fixture.Adapter.Assign(context.Background(), now.Add(2*time.Second),
|
||||
"proxy-owned", "worker-a", time.Minute)
|
||||
if err != nil {
|
||||
t.Fatalf("Assign(): %v", err)
|
||||
}
|
||||
if err := fixture.Adapter.ReplaceSession(context.Background(), workerruntime.Session{
|
||||
WorkerID: "worker-a", InstanceID: "instance-a", SessionID: "session-a",
|
||||
AckedSnapshotVersion: 1, AckedOwnershipEpoch: assignment.Epoch,
|
||||
}, 100*time.Millisecond); err != nil {
|
||||
t.Fatalf("ReplaceSession(): %v", err)
|
||||
}
|
||||
if err := fixture.Adapter.ReplaceRuntime(context.Background(), workerruntime.Report{
|
||||
WorkerID: "worker-a", SessionID: "session-a", Sequence: 1,
|
||||
SnapshotVersion: 1, OwnershipEpoch: assignment.Epoch, ObservedAt: now,
|
||||
}, 100*time.Millisecond); err != nil {
|
||||
t.Fatalf("ReplaceRuntime(): %v", err)
|
||||
}
|
||||
time.Sleep(150 * time.Millisecond)
|
||||
inventory, err := fixture.Adapter.ReadInventory(context.Background(), "provider-a", 0)
|
||||
if err != nil || inventory.Managed != 1 || inventory.AvailableSlots != 0 {
|
||||
t.Fatalf("ReadInventory(expired runtime) = %+v, %v", inventory, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRedisCapacityInventoryScanIsIsolatedPerUpstream(t *testing.T) {
|
||||
fixture := newRedisTestFixture(t)
|
||||
fixture.Adapter.options.MaxInventoryScan = 1
|
||||
now := redisTestNow()
|
||||
target := testProxy("proxy-target", "192.0.2.10")
|
||||
target.MaxConcurrency = 4
|
||||
seedRedisAvailable(t, fixture.Adapter, "provider-a", now, now.Add(time.Second), 2*time.Minute, target)
|
||||
seedRedisAvailable(t, fixture.Adapter, "provider-b", now, now.Add(time.Second), 2*time.Minute,
|
||||
testProxy("proxy-other-1", "192.0.2.11"))
|
||||
seedRedisAvailable(t, fixture.Adapter, "provider-b", now, now.Add(2*time.Second), 2*time.Minute,
|
||||
testProxy("proxy-other-2", "192.0.2.12"))
|
||||
|
||||
inventory, err := fixture.Adapter.ReadInventory(context.Background(), "provider-a", 0)
|
||||
if err != nil || inventory.Managed != 1 || inventory.AvailableSlots != 4 {
|
||||
t.Fatalf("ReadInventory(provider-a) = %+v, %v; want isolated managed=1 slots=4", inventory, err)
|
||||
}
|
||||
}
|
||||
@ -37,6 +37,7 @@ type proxyRecord struct {
|
||||
State string `json:"state"`
|
||||
Tags map[string]string `json:"tags,omitempty"`
|
||||
OwnerWorkerID string `json:"ownerWorkerId,omitempty"`
|
||||
OwnerIndexKey string `json:"ownerIndexKey"`
|
||||
IndexKeys []string `json:"indexKeys,omitempty"`
|
||||
}
|
||||
|
||||
@ -203,7 +204,8 @@ func validateProxyRecord(record proxyRecord) error {
|
||||
record.CreatedAtMS <= 0 || record.ExpiresAtMS <= 0 || record.UsableUntilMS <= 0 ||
|
||||
record.UsableUntilMS > record.ExpiresAtMS || record.LastCheckedAtMS < 0 ||
|
||||
record.LastSuccessAtMS < 0 || record.LatencyNS < 0 || record.MaxConcurrency < 0 ||
|
||||
!validScheme(record.Scheme) || !validProxyState(record.State) {
|
||||
!validScheme(record.Scheme) || !validProxyState(record.State) ||
|
||||
record.OwnerIndexKey == "" || !strings.Contains(record.OwnerIndexKey, "{activity}") {
|
||||
return ErrInvalidRecord
|
||||
}
|
||||
for _, key := range record.IndexKeys {
|
||||
|
||||
@ -9,33 +9,41 @@ import (
|
||||
const redisKeyPrefix = "pp:{activity}:"
|
||||
|
||||
type keyspace struct {
|
||||
prefix string
|
||||
records string
|
||||
unique string
|
||||
idkeys string
|
||||
expiry string
|
||||
available string
|
||||
owners string
|
||||
ownerExpiry string
|
||||
epoch string
|
||||
inventory string
|
||||
stateInventory string
|
||||
prefix string
|
||||
records string
|
||||
unique string
|
||||
idkeys string
|
||||
expiry string
|
||||
available string
|
||||
owners string
|
||||
ownerExpiry string
|
||||
epoch string
|
||||
inventory string
|
||||
stateInventory string
|
||||
workerSessions string
|
||||
workerSessionExpiry string
|
||||
workerRuntime string
|
||||
workerRuntimeExpiry string
|
||||
}
|
||||
|
||||
func newKeyspace(namespace string) keyspace {
|
||||
prefix := redisKeyPrefix + namespace
|
||||
return keyspace{
|
||||
prefix: prefix,
|
||||
records: prefix + ":records",
|
||||
unique: prefix + ":unique",
|
||||
idkeys: prefix + ":idkeys",
|
||||
expiry: prefix + ":expiry",
|
||||
available: prefix + ":available",
|
||||
owners: prefix + ":owners",
|
||||
ownerExpiry: prefix + ":owner-expiry",
|
||||
epoch: prefix + ":epoch",
|
||||
inventory: prefix + ":inventory",
|
||||
stateInventory: prefix + ":state-inventory",
|
||||
prefix: prefix,
|
||||
records: prefix + ":records",
|
||||
unique: prefix + ":unique",
|
||||
idkeys: prefix + ":idkeys",
|
||||
expiry: prefix + ":expiry",
|
||||
available: prefix + ":available",
|
||||
owners: prefix + ":owners",
|
||||
ownerExpiry: prefix + ":owner-expiry",
|
||||
epoch: prefix + ":epoch",
|
||||
inventory: prefix + ":inventory",
|
||||
stateInventory: prefix + ":state-inventory",
|
||||
workerSessions: prefix + ":worker-sessions",
|
||||
workerSessionExpiry: prefix + ":worker-session-expiry",
|
||||
workerRuntime: prefix + ":worker-runtime",
|
||||
workerRuntimeExpiry: prefix + ":worker-runtime-expiry",
|
||||
}
|
||||
}
|
||||
|
||||
@ -67,6 +75,10 @@ func (keys keyspace) upstream(value string) string {
|
||||
return keys.facet("upstream", value)
|
||||
}
|
||||
|
||||
func (keys keyspace) owned(value string) string {
|
||||
return keys.facet("owned", value)
|
||||
}
|
||||
|
||||
func (keys keyspace) facet(name, value string) string {
|
||||
return keys.prefix + ":" + name + ":" + digestToken(value)
|
||||
}
|
||||
|
||||
248
internal/adapters/redisactivity/runtime.go
Normal file
248
internal/adapters/redisactivity/runtime.go
Normal file
@ -0,0 +1,248 @@
|
||||
package redisactivity
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"proxy-pool/internal/domain/workerruntime"
|
||||
)
|
||||
|
||||
const runtimeWireVersion = 1
|
||||
|
||||
const (
|
||||
runtimeReplaceSession = "replace_session"
|
||||
runtimeReplaceReport = "replace_report"
|
||||
runtimeRead = "read"
|
||||
)
|
||||
|
||||
type runtimeSessionWire struct {
|
||||
Version int `json:"version"`
|
||||
WorkerID string `json:"workerId"`
|
||||
InstanceID string `json:"instanceId"`
|
||||
SessionID string `json:"sessionId"`
|
||||
AckedSnapshotVersion string `json:"ackedSnapshotVersion"`
|
||||
AckedOwnershipEpoch string `json:"ackedOwnershipEpoch"`
|
||||
}
|
||||
|
||||
type runtimeCounterWire struct {
|
||||
ProxyID string `json:"proxyId"`
|
||||
Active int64 `json:"active"`
|
||||
Reserved int64 `json:"reserved"`
|
||||
Draining bool `json:"draining"`
|
||||
}
|
||||
|
||||
type runtimeReportWire struct {
|
||||
Version int `json:"version"`
|
||||
WorkerID string `json:"workerId"`
|
||||
SessionID string `json:"sessionId"`
|
||||
Sequence string `json:"sequence"`
|
||||
SnapshotVersion string `json:"snapshotVersion"`
|
||||
OwnershipEpoch string `json:"ownershipEpoch"`
|
||||
ObservedAtMS int64 `json:"observedAtMs"`
|
||||
Counters []runtimeCounterWire `json:"counters"`
|
||||
}
|
||||
|
||||
type runtimeOwnedProxyWire struct {
|
||||
ProxyID string `json:"proxyId"`
|
||||
WorkerID string `json:"workerId"`
|
||||
OwnershipEpoch string `json:"ownershipEpoch"`
|
||||
}
|
||||
|
||||
type runtimeSnapshotWire struct {
|
||||
ProxyID string `json:"proxyId"`
|
||||
Active int64 `json:"active"`
|
||||
Reserved int64 `json:"reserved"`
|
||||
Draining bool `json:"draining"`
|
||||
Fresh bool `json:"fresh"`
|
||||
}
|
||||
|
||||
var (
|
||||
_ workerruntime.SessionWriter = (*Adapter)(nil)
|
||||
_ workerruntime.ReportWriter = (*Adapter)(nil)
|
||||
_ workerruntime.RuntimeReader = (*Adapter)(nil)
|
||||
)
|
||||
|
||||
func (a *Adapter) ReplaceSession(ctx context.Context, session workerruntime.Session, ttl time.Duration) error {
|
||||
if err := validateRuntimeCall(ctx, a); err != nil {
|
||||
return err
|
||||
}
|
||||
if !runtimeClean(session.WorkerID) || !runtimeClean(session.InstanceID) || !runtimeClean(session.SessionID) ||
|
||||
session.AckedSnapshotVersion == 0 || session.AckedOwnershipEpoch == 0 || ttl <= 0 {
|
||||
return workerruntime.ErrInvalidSession
|
||||
}
|
||||
payload, err := json.Marshal(runtimeSessionWire{
|
||||
Version: runtimeWireVersion, WorkerID: session.WorkerID,
|
||||
InstanceID: session.InstanceID, SessionID: session.SessionID,
|
||||
AckedSnapshotVersion: strconv.FormatUint(session.AckedSnapshotVersion, 10),
|
||||
AckedOwnershipEpoch: strconv.FormatUint(session.AckedOwnershipEpoch, 10),
|
||||
})
|
||||
if err != nil {
|
||||
return workerruntime.ErrInvalidSession
|
||||
}
|
||||
reply, err := a.runRuntime(ctx, runtimeReplaceSession, durationMillis(ttl), payload, "")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
switch reply.Status {
|
||||
case scriptOK:
|
||||
return nil
|
||||
case scriptInvalid:
|
||||
return workerruntime.ErrInvalidSession
|
||||
case scriptStale:
|
||||
return workerruntime.ErrStaleSession
|
||||
default:
|
||||
return invalidScriptReply("unexpected worker session reply")
|
||||
}
|
||||
}
|
||||
|
||||
func (a *Adapter) ReplaceRuntime(ctx context.Context, report workerruntime.Report, ttl time.Duration) error {
|
||||
if err := validateRuntimeCall(ctx, a); err != nil {
|
||||
return err
|
||||
}
|
||||
payload, digest, err := a.encodeRuntimeReport(report, ttl)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
reply, err := a.runRuntime(ctx, runtimeReplaceReport, durationMillis(ttl), payload, digest)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
switch reply.Status {
|
||||
case scriptOK:
|
||||
return nil
|
||||
case scriptInvalid:
|
||||
return workerruntime.ErrInvalidReport
|
||||
case scriptStale:
|
||||
return workerruntime.ErrStaleReport
|
||||
case scriptConflict:
|
||||
return workerruntime.ErrConflictingReport
|
||||
case scriptUnavailable:
|
||||
return workerruntime.ErrStaleSession
|
||||
default:
|
||||
return invalidScriptReply("unexpected worker runtime reply")
|
||||
}
|
||||
}
|
||||
|
||||
func (a *Adapter) ReadRuntime(ctx context.Context, proxies []workerruntime.OwnedProxy) ([]workerruntime.Snapshot, error) {
|
||||
if err := validateRuntimeCall(ctx, a); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(proxies) > a.options.MaxRuntimeCounters {
|
||||
return nil, workerruntime.ErrInvalidQuery
|
||||
}
|
||||
wires := make([]runtimeOwnedProxyWire, len(proxies))
|
||||
seen := make(map[string]struct{}, len(proxies))
|
||||
for index, proxy := range proxies {
|
||||
if !runtimeClean(proxy.ProxyID) || !runtimeClean(proxy.WorkerID) || proxy.OwnershipEpoch == 0 {
|
||||
return nil, workerruntime.ErrInvalidQuery
|
||||
}
|
||||
key := proxy.WorkerID + "\x00" + proxy.ProxyID
|
||||
if _, exists := seen[key]; exists {
|
||||
return nil, workerruntime.ErrInvalidQuery
|
||||
}
|
||||
seen[key] = struct{}{}
|
||||
wires[index] = runtimeOwnedProxyWire{
|
||||
ProxyID: proxy.ProxyID, WorkerID: proxy.WorkerID,
|
||||
OwnershipEpoch: strconv.FormatUint(proxy.OwnershipEpoch, 10),
|
||||
}
|
||||
}
|
||||
payload, err := json.Marshal(wires)
|
||||
if err != nil {
|
||||
return nil, workerruntime.ErrInvalidQuery
|
||||
}
|
||||
reply, err := a.runRuntime(ctx, runtimeRead, 0, payload, "")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if reply.Status == scriptInvalid {
|
||||
return nil, workerruntime.ErrInvalidQuery
|
||||
}
|
||||
if reply.Status != scriptOK || len(reply.Snapshots) != len(proxies) {
|
||||
return nil, invalidScriptReply("unexpected worker runtime read reply")
|
||||
}
|
||||
result := make([]workerruntime.Snapshot, len(reply.Snapshots))
|
||||
for index, snapshot := range reply.Snapshots {
|
||||
if snapshot.ProxyID != proxies[index].ProxyID || snapshot.Active < 0 || snapshot.Reserved < 0 {
|
||||
return nil, invalidScriptReply("invalid worker runtime snapshot")
|
||||
}
|
||||
result[index] = workerruntime.Snapshot{
|
||||
ProxyID: snapshot.ProxyID, Active: snapshot.Active, Reserved: snapshot.Reserved,
|
||||
Draining: snapshot.Draining, Fresh: snapshot.Fresh,
|
||||
}
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func (a *Adapter) encodeRuntimeReport(report workerruntime.Report, ttl time.Duration) ([]byte, string, error) {
|
||||
if ttl <= 0 || !runtimeClean(report.WorkerID) || !runtimeClean(report.SessionID) ||
|
||||
report.Sequence == 0 || report.SnapshotVersion == 0 || report.OwnershipEpoch == 0 || report.ObservedAt.IsZero() ||
|
||||
len(report.Counters) > a.options.MaxRuntimeCounters {
|
||||
return nil, "", workerruntime.ErrInvalidReport
|
||||
}
|
||||
counters := append([]workerruntime.Counter(nil), report.Counters...)
|
||||
sort.Slice(counters, func(left, right int) bool { return counters[left].ProxyID < counters[right].ProxyID })
|
||||
wires := make([]runtimeCounterWire, len(counters))
|
||||
for index, counter := range counters {
|
||||
if !runtimeClean(counter.ProxyID) || counter.Active < 0 || counter.Reserved < 0 ||
|
||||
(index > 0 && counters[index-1].ProxyID == counter.ProxyID) {
|
||||
return nil, "", workerruntime.ErrInvalidReport
|
||||
}
|
||||
wires[index] = runtimeCounterWire{
|
||||
ProxyID: counter.ProxyID, Active: counter.Active,
|
||||
Reserved: counter.Reserved, Draining: counter.Draining,
|
||||
}
|
||||
}
|
||||
payload, err := json.Marshal(runtimeReportWire{
|
||||
Version: runtimeWireVersion, WorkerID: report.WorkerID, SessionID: report.SessionID,
|
||||
Sequence: strconv.FormatUint(report.Sequence, 10),
|
||||
SnapshotVersion: strconv.FormatUint(report.SnapshotVersion, 10),
|
||||
OwnershipEpoch: strconv.FormatUint(report.OwnershipEpoch, 10),
|
||||
ObservedAtMS: report.ObservedAt.UTC().UnixMilli(), Counters: wires,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, "", workerruntime.ErrInvalidReport
|
||||
}
|
||||
digest := sha256.Sum256(payload)
|
||||
return payload, hex.EncodeToString(digest[:]), nil
|
||||
}
|
||||
|
||||
func (a *Adapter) runRuntime(
|
||||
ctx context.Context,
|
||||
operation string,
|
||||
ttlMS int64,
|
||||
payload []byte,
|
||||
digest string,
|
||||
) (runtimeScriptReply, error) {
|
||||
result, err := runScript(ctx, a.client, runtimeScript, []string{
|
||||
a.keys.workerSessions, a.keys.workerSessionExpiry,
|
||||
a.keys.workerRuntime, a.keys.workerRuntimeExpiry, a.keys.owners,
|
||||
}, operation, ttlMS, a.options.CleanupLimit, string(payload), digest)
|
||||
if err != nil {
|
||||
return runtimeScriptReply{}, err
|
||||
}
|
||||
var reply runtimeScriptReply
|
||||
if err := decodeScriptResult(result, &reply); err != nil {
|
||||
return runtimeScriptReply{}, err
|
||||
}
|
||||
return reply, nil
|
||||
}
|
||||
|
||||
func validateRuntimeCall(ctx context.Context, adapter *Adapter) error {
|
||||
if ctx == nil || adapter == nil {
|
||||
return workerruntime.ErrInvalidStore
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func runtimeClean(value string) bool {
|
||||
return value != "" && strings.TrimSpace(value) == value
|
||||
}
|
||||
147
internal/adapters/redisactivity/runtime_integration_test.go
Normal file
147
internal/adapters/redisactivity/runtime_integration_test.go
Normal file
@ -0,0 +1,147 @@
|
||||
//go:build integration
|
||||
|
||||
package redisactivity
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"proxy-pool/internal/domain/workerruntime"
|
||||
)
|
||||
|
||||
func TestRedisWorkerRuntimeReplacesSparseCountersAndFencesReports(t *testing.T) {
|
||||
fixture := newRedisTestFixture(t)
|
||||
now := redisTestNow()
|
||||
seedRedisAvailable(t, fixture.Adapter, "provider-a", now, now.Add(time.Second), 2*time.Minute,
|
||||
testProxy("proxy-a", "192.0.2.10"))
|
||||
assignment, err := fixture.Adapter.Assign(context.Background(), now.Add(2*time.Second),
|
||||
"proxy-a", "worker-a", time.Minute)
|
||||
if err != nil {
|
||||
t.Fatalf("Assign(): %v", err)
|
||||
}
|
||||
session := workerruntime.Session{
|
||||
WorkerID: "worker-a", InstanceID: "instance-a", SessionID: "session-a",
|
||||
AckedSnapshotVersion: 3, AckedOwnershipEpoch: assignment.Epoch,
|
||||
}
|
||||
if err := fixture.Adapter.ReplaceSession(context.Background(), session, time.Minute); err != nil {
|
||||
t.Fatalf("ReplaceSession(): %v", err)
|
||||
}
|
||||
report := workerruntime.Report{
|
||||
WorkerID: "worker-a", SessionID: "session-a", Sequence: 2,
|
||||
SnapshotVersion: 3, OwnershipEpoch: assignment.Epoch, ObservedAt: now,
|
||||
Counters: []workerruntime.Counter{{ProxyID: "proxy-a", Active: 2, Reserved: 1}},
|
||||
}
|
||||
if err := fixture.Adapter.ReplaceRuntime(context.Background(), report, time.Minute); err != nil {
|
||||
t.Fatalf("ReplaceRuntime(first): %v", err)
|
||||
}
|
||||
if err := fixture.Adapter.ReplaceRuntime(context.Background(), report, time.Minute); err != nil {
|
||||
t.Fatalf("ReplaceRuntime(replay): %v", err)
|
||||
}
|
||||
conflict := report
|
||||
conflict.Counters = []workerruntime.Counter{{ProxyID: "proxy-a", Active: 3}}
|
||||
if err := fixture.Adapter.ReplaceRuntime(context.Background(), conflict, time.Minute); !errors.Is(err, workerruntime.ErrConflictingReport) {
|
||||
t.Fatalf("ReplaceRuntime(conflict) error = %v", err)
|
||||
}
|
||||
stale := report
|
||||
stale.Sequence = 1
|
||||
if err := fixture.Adapter.ReplaceRuntime(context.Background(), stale, time.Minute); !errors.Is(err, workerruntime.ErrStaleReport) {
|
||||
t.Fatalf("ReplaceRuntime(stale) error = %v", err)
|
||||
}
|
||||
|
||||
query := []workerruntime.OwnedProxy{{
|
||||
ProxyID: "proxy-a", WorkerID: "worker-a", OwnershipEpoch: assignment.Epoch,
|
||||
}}
|
||||
got, err := fixture.Adapter.ReadRuntime(context.Background(), query)
|
||||
if err != nil || len(got) != 1 || got[0] != (workerruntime.Snapshot{
|
||||
ProxyID: "proxy-a", Active: 2, Reserved: 1, Fresh: true,
|
||||
}) {
|
||||
t.Fatalf("ReadRuntime(first) = %+v, %v", got, err)
|
||||
}
|
||||
report.Sequence = 3
|
||||
report.Counters = nil
|
||||
if err := fixture.Adapter.ReplaceRuntime(context.Background(), report, time.Minute); err != nil {
|
||||
t.Fatalf("ReplaceRuntime(empty): %v", err)
|
||||
}
|
||||
got, err = fixture.Adapter.ReadRuntime(context.Background(), query)
|
||||
if err != nil || len(got) != 1 || got[0] != (workerruntime.Snapshot{ProxyID: "proxy-a", Fresh: true}) {
|
||||
t.Fatalf("ReadRuntime(empty) = %+v, %v", got, err)
|
||||
}
|
||||
|
||||
if err := fixture.Adapter.ReplaceSession(context.Background(), workerruntime.Session{
|
||||
WorkerID: "worker-a", InstanceID: "instance-b", SessionID: "session-b",
|
||||
AckedSnapshotVersion: 4, AckedOwnershipEpoch: assignment.Epoch + 1,
|
||||
}, time.Minute); err != nil {
|
||||
t.Fatalf("ReplaceSession(new): %v", err)
|
||||
}
|
||||
if err := fixture.Adapter.ReplaceRuntime(context.Background(), report, time.Minute); !errors.Is(err, workerruntime.ErrStaleSession) {
|
||||
t.Fatalf("ReplaceRuntime(old session) error = %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRedisWorkerRuntimeExpiresFailClosed(t *testing.T) {
|
||||
fixture := newRedisTestFixture(t)
|
||||
now := redisTestNow()
|
||||
seedRedisAvailable(t, fixture.Adapter, "provider-a", now, now.Add(time.Second), 2*time.Minute,
|
||||
testProxy("proxy-a", "192.0.2.10"))
|
||||
assignment, err := fixture.Adapter.Assign(context.Background(), now.Add(2*time.Second),
|
||||
"proxy-a", "worker-a", time.Minute)
|
||||
if err != nil {
|
||||
t.Fatalf("Assign(): %v", err)
|
||||
}
|
||||
if err := fixture.Adapter.ReplaceSession(context.Background(), workerruntime.Session{
|
||||
WorkerID: "worker-a", InstanceID: "instance-a", SessionID: "session-a",
|
||||
AckedSnapshotVersion: 1, AckedOwnershipEpoch: assignment.Epoch,
|
||||
}, 100*time.Millisecond); err != nil {
|
||||
t.Fatalf("ReplaceSession(): %v", err)
|
||||
}
|
||||
if err := fixture.Adapter.ReplaceRuntime(context.Background(), workerruntime.Report{
|
||||
WorkerID: "worker-a", SessionID: "session-a", Sequence: 1,
|
||||
SnapshotVersion: 1, OwnershipEpoch: assignment.Epoch, ObservedAt: now,
|
||||
}, 100*time.Millisecond); err != nil {
|
||||
t.Fatalf("ReplaceRuntime(): %v", err)
|
||||
}
|
||||
time.Sleep(150 * time.Millisecond)
|
||||
got, err := fixture.Adapter.ReadRuntime(context.Background(), []workerruntime.OwnedProxy{{
|
||||
ProxyID: "proxy-a", WorkerID: "worker-a", OwnershipEpoch: assignment.Epoch,
|
||||
}})
|
||||
if err != nil || len(got) != 1 || got[0].Fresh {
|
||||
t.Fatalf("ReadRuntime(expired) = %+v, %v", got, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRedisWorkerRuntimeRejectsEmptyReportBeyondAcknowledgedSnapshot(t *testing.T) {
|
||||
fixture := newRedisTestFixture(t)
|
||||
now := redisTestNow()
|
||||
if err := fixture.Adapter.ReplaceSession(context.Background(), workerruntime.Session{
|
||||
WorkerID: "worker-a", InstanceID: "instance-a", SessionID: "session-a",
|
||||
AckedSnapshotVersion: 3, AckedOwnershipEpoch: 9,
|
||||
}, time.Minute); err != nil {
|
||||
t.Fatalf("ReplaceSession(): %v", err)
|
||||
}
|
||||
for name, report := range map[string]workerruntime.Report{
|
||||
"version": {
|
||||
WorkerID: "worker-a", SessionID: "session-a", Sequence: 1,
|
||||
SnapshotVersion: 4, OwnershipEpoch: 9, ObservedAt: now,
|
||||
},
|
||||
"epoch": {
|
||||
WorkerID: "worker-a", SessionID: "session-a", Sequence: 1,
|
||||
SnapshotVersion: 3, OwnershipEpoch: 10, ObservedAt: now,
|
||||
},
|
||||
} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
if err := fixture.Adapter.ReplaceRuntime(context.Background(), report, time.Minute); !errors.Is(err, workerruntime.ErrStaleReport) {
|
||||
t.Fatalf("ReplaceRuntime() error = %v, want ErrStaleReport", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestRedisWorkerRuntimeAcceptsEmptyRead(t *testing.T) {
|
||||
fixture := newRedisTestFixture(t)
|
||||
got, err := fixture.Adapter.ReadRuntime(context.Background(), nil)
|
||||
if err != nil || got == nil || len(got) != 0 {
|
||||
t.Fatalf("ReadRuntime(empty) = %#v, %v", got, err)
|
||||
}
|
||||
}
|
||||
@ -74,6 +74,17 @@ type statusScriptInventory struct {
|
||||
Extracted int64 `json:"extracted"`
|
||||
}
|
||||
|
||||
type runtimeScriptReply struct {
|
||||
Status scriptStatus `json:"status"`
|
||||
Snapshots []runtimeSnapshotWire `json:"snapshots"`
|
||||
}
|
||||
|
||||
type capacityScriptReply struct {
|
||||
Status scriptStatus `json:"status"`
|
||||
Managed int `json:"managed"`
|
||||
AvailableSlots int64 `json:"availableSlots,string"`
|
||||
}
|
||||
|
||||
//go:embed scripts/upsert.lua
|
||||
var upsertSource string
|
||||
|
||||
@ -92,6 +103,12 @@ var sweepSource string
|
||||
//go:embed scripts/status.lua
|
||||
var statusSource string
|
||||
|
||||
//go:embed scripts/runtime.lua
|
||||
var runtimeSource string
|
||||
|
||||
//go:embed scripts/capacity.lua
|
||||
var capacitySource string
|
||||
|
||||
var (
|
||||
upsertScript = redis.NewScript(upsertSource)
|
||||
healthScript = redis.NewScript(healthSource)
|
||||
@ -99,6 +116,8 @@ var (
|
||||
ownershipScript = redis.NewScript(ownershipSource)
|
||||
sweepScript = redis.NewScript(sweepSource)
|
||||
statusScript = redis.NewScript(statusSource)
|
||||
runtimeScript = redis.NewScript(runtimeSource)
|
||||
capacityScript = redis.NewScript(capacitySource)
|
||||
)
|
||||
|
||||
func runScript(ctx context.Context, client redis.Scripter, script *redis.Script, keys []string, args ...any) (any, error) {
|
||||
|
||||
177
internal/adapters/redisactivity/scripts/capacity.lua
Normal file
177
internal/adapters/redisactivity/scripts/capacity.lua
Normal file
@ -0,0 +1,177 @@
|
||||
local records_key = KEYS[1]
|
||||
local inventory_key = KEYS[2]
|
||||
local available_upstream_key = KEYS[3]
|
||||
local owned_upstream_key = KEYS[4]
|
||||
local owners_key = KEYS[5]
|
||||
local sessions_key = KEYS[6]
|
||||
local session_expiry_key = KEYS[7]
|
||||
local runtime_key = KEYS[8]
|
||||
local runtime_expiry_key = KEYS[9]
|
||||
|
||||
local upstream_id = ARGV[1]
|
||||
local safety_margin_ms = tonumber(ARGV[2])
|
||||
local scan_limit = tonumber(ARGV[3])
|
||||
local cleanup_limit = tonumber(ARGV[4])
|
||||
|
||||
local function reply(status, managed, available_slots)
|
||||
return cjson.encode({
|
||||
status = status,
|
||||
managed = managed or 0,
|
||||
availableSlots = tostring(available_slots or 0)
|
||||
})
|
||||
end
|
||||
|
||||
local function now_ms()
|
||||
local value = redis.call('TIME')
|
||||
return tonumber(value[1]) * 1000 + math.floor(tonumber(value[2]) / 1000)
|
||||
end
|
||||
|
||||
local function decode_table(value)
|
||||
if not value then
|
||||
return nil
|
||||
end
|
||||
local ok, decoded = pcall(cjson.decode, value)
|
||||
if not ok or type(decoded) ~= 'table' then
|
||||
return nil
|
||||
end
|
||||
return decoded
|
||||
end
|
||||
|
||||
local function valid_uint(value)
|
||||
return type(value) == 'string' and string.match(value, '^[0-9]+$') and
|
||||
value ~= '0' and (string.len(value) == 1 or string.sub(value, 1, 1) ~= '0')
|
||||
end
|
||||
|
||||
local function compare_uint(left, right)
|
||||
if string.len(left) ~= string.len(right) then
|
||||
return string.len(left) < string.len(right) and -1 or 1
|
||||
end
|
||||
if left == right then
|
||||
return 0
|
||||
end
|
||||
return left < right and -1 or 1
|
||||
end
|
||||
|
||||
local function cleanup(now)
|
||||
local expired_sessions = redis.call('ZRANGEBYSCORE', session_expiry_key, '-inf', now, 'LIMIT', 0, cleanup_limit)
|
||||
for _, worker_id in ipairs(expired_sessions) do
|
||||
redis.call('HDEL', sessions_key, worker_id)
|
||||
redis.call('HDEL', runtime_key, worker_id)
|
||||
redis.call('ZREM', session_expiry_key, worker_id)
|
||||
redis.call('ZREM', runtime_expiry_key, worker_id)
|
||||
end
|
||||
local expired_reports = redis.call('ZRANGEBYSCORE', runtime_expiry_key, '-inf', now, 'LIMIT', 0, cleanup_limit)
|
||||
for _, worker_id in ipairs(expired_reports) do
|
||||
redis.call('HDEL', runtime_key, worker_id)
|
||||
redis.call('ZREM', runtime_expiry_key, worker_id)
|
||||
end
|
||||
end
|
||||
|
||||
if type(upstream_id) ~= 'string' or upstream_id == '' or not safety_margin_ms or safety_margin_ms < 0 or
|
||||
not scan_limit or scan_limit <= 0 or not cleanup_limit or cleanup_limit <= 0 then
|
||||
return reply('invalid', 0, 0)
|
||||
end
|
||||
|
||||
local now = now_ms()
|
||||
cleanup(now)
|
||||
local threshold = now + safety_margin_ms
|
||||
local available_ids = redis.call('ZRANGEBYSCORE', available_upstream_key, '(' .. threshold, '+inf',
|
||||
'LIMIT', 0, scan_limit + 1)
|
||||
if #available_ids > scan_limit then
|
||||
return reply('unavailable', 0, 0)
|
||||
end
|
||||
local remaining = scan_limit - #available_ids
|
||||
local owned_ids = redis.call('ZRANGEBYSCORE', owned_upstream_key, '(' .. threshold, '+inf',
|
||||
'LIMIT', 0, remaining + 1)
|
||||
if #owned_ids > remaining then
|
||||
return reply('unavailable', 0, 0)
|
||||
end
|
||||
|
||||
local managed = tonumber(redis.call('HGET', inventory_key, upstream_id) or '0')
|
||||
if not managed or managed < 0 or managed ~= math.floor(managed) then
|
||||
return reply('unavailable', 0, 0)
|
||||
end
|
||||
local available_slots = 0
|
||||
local worker_cache = {}
|
||||
local seen = {}
|
||||
for _, proxy_id in ipairs(available_ids) do
|
||||
seen[proxy_id] = true
|
||||
local record = decode_table(redis.call('HGET', records_key, proxy_id))
|
||||
if not record or type(record.sourceUpstream) ~= 'string' or type(record.state) ~= 'string' or
|
||||
record.sourceUpstream ~= upstream_id or record.state ~= 'AVAILABLE' or
|
||||
type(record.usableUntilMs) ~= 'number' or record.usableUntilMs <= threshold or
|
||||
type(record.maxConcurrency) ~= 'number' or record.maxConcurrency < 0 or
|
||||
record.maxConcurrency ~= math.floor(record.maxConcurrency) or
|
||||
(record.ownerWorkerId and record.ownerWorkerId ~= '') or redis.call('HGET', owners_key, proxy_id) then
|
||||
return reply('unavailable', 0, 0)
|
||||
end
|
||||
available_slots = available_slots + record.maxConcurrency
|
||||
end
|
||||
|
||||
for _, proxy_id in ipairs(owned_ids) do
|
||||
if seen[proxy_id] then
|
||||
return reply('unavailable', 0, 0)
|
||||
end
|
||||
local record = decode_table(redis.call('HGET', records_key, proxy_id))
|
||||
if not record or record.sourceUpstream ~= upstream_id or record.state ~= 'AVAILABLE' or
|
||||
type(record.usableUntilMs) ~= 'number' or record.usableUntilMs <= threshold or
|
||||
type(record.maxConcurrency) ~= 'number' or record.maxConcurrency < 0 or
|
||||
record.maxConcurrency ~= math.floor(record.maxConcurrency) or
|
||||
type(record.ownerWorkerId) ~= 'string' or record.ownerWorkerId == '' then
|
||||
return reply('unavailable', 0, 0)
|
||||
end
|
||||
local owner_worker_id = record.ownerWorkerId
|
||||
local owner = decode_table(redis.call('HGET', owners_key, proxy_id))
|
||||
if not owner or owner.workerId ~= owner_worker_id or type(owner.epoch) ~= 'number' or
|
||||
type(owner.expiresAtMs) ~= 'number' or owner.expiresAtMs <= now or
|
||||
type(owner.draining) ~= 'boolean' then
|
||||
return reply('unavailable', 0, 0)
|
||||
end
|
||||
local cached = worker_cache[owner_worker_id]
|
||||
if not cached then
|
||||
local session = decode_table(redis.call('HGET', sessions_key, owner_worker_id))
|
||||
local report = decode_table(redis.call('HGET', runtime_key, owner_worker_id))
|
||||
cached = {fresh = false, counters = {}}
|
||||
if session and report and session.workerId == owner_worker_id and
|
||||
report.workerId == owner_worker_id and session.sessionId == report.sessionId and
|
||||
type(session.expiresAtMs) == 'number' and session.expiresAtMs > now and
|
||||
type(report.expiresAtMs) == 'number' and report.expiresAtMs > now and
|
||||
valid_uint(session.ackedSnapshotVersion) and valid_uint(session.ackedOwnershipEpoch) and
|
||||
report.snapshotVersion == session.ackedSnapshotVersion and
|
||||
report.ownershipEpoch == session.ackedOwnershipEpoch then
|
||||
cached.fresh = true
|
||||
cached.ownershipEpoch = report.ownershipEpoch
|
||||
if type(report.counters) == 'table' then
|
||||
for _, counter in pairs(report.counters) do
|
||||
if type(counter) == 'table' and type(counter.proxyId) == 'string' then
|
||||
cached.counters[counter.proxyId] = counter
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
worker_cache[owner_worker_id] = cached
|
||||
end
|
||||
local owner_epoch = tostring(owner.epoch)
|
||||
if cached.fresh and valid_uint(owner_epoch) and
|
||||
compare_uint(cached.ownershipEpoch, owner_epoch) >= 0 then
|
||||
local counter = cached.counters[proxy_id]
|
||||
local active = 0
|
||||
local reserved = 0
|
||||
local draining = false
|
||||
if counter then
|
||||
active = counter.active
|
||||
reserved = counter.reserved
|
||||
draining = counter.draining
|
||||
end
|
||||
if type(active) == 'number' and type(reserved) == 'number' and active >= 0 and reserved >= 0 and
|
||||
active == math.floor(active) and reserved == math.floor(reserved) and
|
||||
not draining and not owner.draining then
|
||||
local slots = record.maxConcurrency - active - reserved
|
||||
if slots > 0 then
|
||||
available_slots = available_slots + slots
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
return reply('ok', managed, available_slots)
|
||||
@ -113,6 +113,12 @@ local function remove_available(proxy_id, record)
|
||||
end
|
||||
end
|
||||
|
||||
local function remove_owned(proxy_id, record)
|
||||
if record and type(record.ownerIndexKey) == 'string' and record.ownerIndexKey ~= '' then
|
||||
redis.call('ZREM', record.ownerIndexKey, proxy_id)
|
||||
end
|
||||
end
|
||||
|
||||
local function remove_proxy(proxy_id)
|
||||
local raw = redis.call('HGET', records_key, proxy_id)
|
||||
local record = nil
|
||||
@ -120,6 +126,7 @@ local function remove_proxy(proxy_id)
|
||||
local decoded
|
||||
decoded, record = pcall(cjson.decode, raw)
|
||||
remove_available(proxy_id, decoded and record or nil)
|
||||
remove_owned(proxy_id, decoded and record or nil)
|
||||
if decoded and type(record) == 'table' and is_managed(record.state) then
|
||||
decrement_inventory(record.sourceUpstream)
|
||||
end
|
||||
|
||||
@ -70,12 +70,19 @@ local function remove_available(id, record)
|
||||
end
|
||||
end
|
||||
|
||||
local function remove_owned(id, record)
|
||||
if record and type(record.ownerIndexKey) == 'string' and record.ownerIndexKey ~= '' then
|
||||
redis.call('ZREM', record.ownerIndexKey, id)
|
||||
end
|
||||
end
|
||||
|
||||
local function remove_proxy(id)
|
||||
local raw = redis.call('HGET', records_key, id)
|
||||
local record = nil
|
||||
if raw then
|
||||
record = cjson.decode(raw)
|
||||
remove_available(id, record)
|
||||
remove_owned(id, record)
|
||||
if is_managed(record.state) then
|
||||
decrement_inventory(record.sourceUpstream)
|
||||
end
|
||||
@ -142,6 +149,9 @@ if not raw then
|
||||
return finish({status = 'not_found'})
|
||||
end
|
||||
local record = cjson.decode(raw)
|
||||
if type(record.ownerIndexKey) ~= 'string' or record.ownerIndexKey == '' then
|
||||
return finish({status = 'invalid'})
|
||||
end
|
||||
if tonumber(record.expiresAtMs) <= checked_at_ms then
|
||||
remove_proxy(proxy_id)
|
||||
return finish({status = 'not_found'})
|
||||
@ -184,11 +194,17 @@ end
|
||||
local encoded = cjson.encode(record)
|
||||
redis.call('HSET', records_key, proxy_id, encoded)
|
||||
local owned = (record.ownerWorkerId and record.ownerWorkerId ~= '') or redis.call('HEXISTS', owners_key, proxy_id) == 1
|
||||
if next_state == 'AVAILABLE' and not owned and tonumber(record.usableUntilMs) > checked_at_ms then
|
||||
redis.call('ZADD', available_key, record.usableUntilMs, proxy_id)
|
||||
for _, index_key in ipairs(record.indexKeys or {}) do
|
||||
redis.call('ZADD', index_key, record.usableUntilMs, proxy_id)
|
||||
touch(index_key, tonumber(record.expiresAtMs))
|
||||
remove_owned(proxy_id, record)
|
||||
if next_state == 'AVAILABLE' and tonumber(record.usableUntilMs) > checked_at_ms then
|
||||
if owned then
|
||||
redis.call('ZADD', record.ownerIndexKey, record.usableUntilMs, proxy_id)
|
||||
touch(record.ownerIndexKey, tonumber(record.expiresAtMs))
|
||||
else
|
||||
redis.call('ZADD', available_key, record.usableUntilMs, proxy_id)
|
||||
for _, index_key in ipairs(record.indexKeys or {}) do
|
||||
redis.call('ZADD', index_key, record.usableUntilMs, proxy_id)
|
||||
touch(index_key, tonumber(record.expiresAtMs))
|
||||
end
|
||||
end
|
||||
end
|
||||
touch(records_key, tonumber(record.expiresAtMs))
|
||||
|
||||
@ -94,6 +94,12 @@ local function remove_available(id, record)
|
||||
end
|
||||
end
|
||||
|
||||
local function remove_owned(id, record)
|
||||
if record and type(record.ownerIndexKey) == 'string' and record.ownerIndexKey ~= '' then
|
||||
redis.call('ZREM', record.ownerIndexKey, id)
|
||||
end
|
||||
end
|
||||
|
||||
local function add_available(id, record, at_ms)
|
||||
local usable_until_ms = record and tonumber(record.usableUntilMs)
|
||||
if not usable_until_ms or record.state ~= 'AVAILABLE' or usable_until_ms <= at_ms then
|
||||
@ -118,6 +124,7 @@ local function remove_proxy(id)
|
||||
local decoded
|
||||
decoded, record = pcall(cjson.decode, raw)
|
||||
remove_available(id, decoded and record or nil)
|
||||
remove_owned(id, decoded and record or nil)
|
||||
if decoded and type(record) == 'table' and is_managed(record.state) then
|
||||
decrement_inventory(record.sourceUpstream)
|
||||
end
|
||||
@ -169,6 +176,7 @@ local function clear_owner(id, assignment, at_ms, restore)
|
||||
local raw_record = redis.call('HGET', records_key, id)
|
||||
local record = decode_table(raw_record)
|
||||
if record and (not assignment or record.ownerWorkerId == assignment.workerId) then
|
||||
remove_owned(id, record)
|
||||
record.ownerWorkerId = nil
|
||||
redis.call('HSET', records_key, id, cjson.encode(record))
|
||||
if restore then
|
||||
@ -195,6 +203,7 @@ if operation == 'assign' then
|
||||
|
||||
local record = decode_table(redis.call('HGET', records_key, proxy_id))
|
||||
if not record or record.state ~= 'AVAILABLE' or
|
||||
type(record.ownerIndexKey) ~= 'string' or record.ownerIndexKey == '' or
|
||||
(record.ownerWorkerId and record.ownerWorkerId ~= '') or
|
||||
redis.call('HEXISTS', owners_key, proxy_id) == 1 or
|
||||
not tonumber(record.usableUntilMs) or tonumber(record.usableUntilMs) <= now_ms then
|
||||
@ -223,6 +232,8 @@ if operation == 'assign' then
|
||||
record.ownerWorkerId = worker_id
|
||||
redis.call('HSET', records_key, proxy_id, cjson.encode(record))
|
||||
remove_available(proxy_id, record)
|
||||
redis.call('ZADD', record.ownerIndexKey, record.usableUntilMs, proxy_id)
|
||||
touch(record.ownerIndexKey, tonumber(record.expiresAtMs))
|
||||
return finish({status = 'ok', record = encoded})
|
||||
end
|
||||
|
||||
@ -267,6 +278,8 @@ if operation == 'begin_drain' then
|
||||
current.assignmentVersion = tonumber(current.assignmentVersion) + 1
|
||||
local encoded = cjson.encode(current)
|
||||
redis.call('HSET', owners_key, proxy_id, encoded)
|
||||
local record = decode_table(redis.call('HGET', records_key, proxy_id))
|
||||
remove_owned(proxy_id, record)
|
||||
return finish({status = 'ok', record = encoded})
|
||||
end
|
||||
return finish({status = 'ok', record = cjson.encode(current)})
|
||||
|
||||
229
internal/adapters/redisactivity/scripts/runtime.lua
Normal file
229
internal/adapters/redisactivity/scripts/runtime.lua
Normal file
@ -0,0 +1,229 @@
|
||||
local sessions_key = KEYS[1]
|
||||
local session_expiry_key = KEYS[2]
|
||||
local runtime_key = KEYS[3]
|
||||
local runtime_expiry_key = KEYS[4]
|
||||
local owners_key = KEYS[5]
|
||||
|
||||
local operation = ARGV[1]
|
||||
local ttl_ms = tonumber(ARGV[2])
|
||||
local cleanup_limit = tonumber(ARGV[3])
|
||||
local payload = ARGV[4]
|
||||
local digest = ARGV[5]
|
||||
|
||||
local function reply(status, snapshots)
|
||||
if snapshots then
|
||||
return cjson.encode({status = status, snapshots = snapshots})
|
||||
end
|
||||
return '{"status":' .. cjson.encode(status) .. ',"snapshots":[]}'
|
||||
end
|
||||
|
||||
local function now_ms()
|
||||
local value = redis.call('TIME')
|
||||
return tonumber(value[1]) * 1000 + math.floor(tonumber(value[2]) / 1000)
|
||||
end
|
||||
|
||||
local function decode_table(value)
|
||||
if not value then
|
||||
return nil
|
||||
end
|
||||
local ok, decoded = pcall(cjson.decode, value)
|
||||
if not ok or type(decoded) ~= 'table' then
|
||||
return nil
|
||||
end
|
||||
return decoded
|
||||
end
|
||||
|
||||
local function valid_uint(value)
|
||||
return type(value) == 'string' and string.match(value, '^[0-9]+$') and
|
||||
value ~= '0' and (string.len(value) == 1 or string.sub(value, 1, 1) ~= '0')
|
||||
end
|
||||
|
||||
local function compare_uint(left, right)
|
||||
if string.len(left) ~= string.len(right) then
|
||||
return string.len(left) < string.len(right) and -1 or 1
|
||||
end
|
||||
if left == right then
|
||||
return 0
|
||||
end
|
||||
return left < right and -1 or 1
|
||||
end
|
||||
|
||||
local function cleanup(now)
|
||||
local expired_sessions = redis.call('ZRANGEBYSCORE', session_expiry_key, '-inf', now, 'LIMIT', 0, cleanup_limit)
|
||||
for _, worker_id in ipairs(expired_sessions) do
|
||||
redis.call('HDEL', sessions_key, worker_id)
|
||||
redis.call('HDEL', runtime_key, worker_id)
|
||||
redis.call('ZREM', session_expiry_key, worker_id)
|
||||
redis.call('ZREM', runtime_expiry_key, worker_id)
|
||||
end
|
||||
local expired_reports = redis.call('ZRANGEBYSCORE', runtime_expiry_key, '-inf', now, 'LIMIT', 0, cleanup_limit)
|
||||
for _, worker_id in ipairs(expired_reports) do
|
||||
redis.call('HDEL', runtime_key, worker_id)
|
||||
redis.call('ZREM', runtime_expiry_key, worker_id)
|
||||
end
|
||||
end
|
||||
|
||||
local function valid_session(value)
|
||||
return value and value.version == 1 and type(value.workerId) == 'string' and value.workerId ~= '' and
|
||||
type(value.instanceId) == 'string' and value.instanceId ~= '' and
|
||||
type(value.sessionId) == 'string' and value.sessionId ~= '' and
|
||||
valid_uint(value.ackedSnapshotVersion) and valid_uint(value.ackedOwnershipEpoch)
|
||||
end
|
||||
|
||||
local function valid_owner(value, worker_id, ownership_epoch, now)
|
||||
return value and type(value.workerId) == 'string' and value.workerId == worker_id and
|
||||
type(value.epoch) == 'number' and valid_uint(tostring(value.epoch)) and
|
||||
compare_uint(ownership_epoch, tostring(value.epoch)) >= 0 and
|
||||
type(value.expiresAtMs) == 'number' and value.expiresAtMs > now
|
||||
end
|
||||
|
||||
local now = now_ms()
|
||||
cleanup(now)
|
||||
|
||||
if operation == 'replace_session' then
|
||||
if not ttl_ms or ttl_ms <= 0 then
|
||||
return reply('invalid')
|
||||
end
|
||||
local session = decode_table(payload)
|
||||
if not valid_session(session) then
|
||||
return reply('invalid')
|
||||
end
|
||||
local current = decode_table(redis.call('HGET', sessions_key, session.workerId))
|
||||
if current and valid_session(current) and current.sessionId == session.sessionId and
|
||||
current.instanceId == session.instanceId and type(current.expiresAtMs) == 'number' and
|
||||
current.expiresAtMs > now then
|
||||
local epoch_order = compare_uint(session.ackedOwnershipEpoch, current.ackedOwnershipEpoch)
|
||||
local version_order = compare_uint(session.ackedSnapshotVersion, current.ackedSnapshotVersion)
|
||||
if epoch_order < 0 or (epoch_order == 0 and version_order < 0) then
|
||||
return reply('stale')
|
||||
end
|
||||
if epoch_order > 0 or version_order > 0 then
|
||||
redis.call('HDEL', runtime_key, session.workerId)
|
||||
redis.call('ZREM', runtime_expiry_key, session.workerId)
|
||||
end
|
||||
else
|
||||
redis.call('HDEL', runtime_key, session.workerId)
|
||||
redis.call('ZREM', runtime_expiry_key, session.workerId)
|
||||
end
|
||||
session.expiresAtMs = now + ttl_ms
|
||||
redis.call('HSET', sessions_key, session.workerId, cjson.encode(session))
|
||||
redis.call('ZADD', session_expiry_key, session.expiresAtMs, session.workerId)
|
||||
return reply('ok')
|
||||
end
|
||||
|
||||
if operation == 'replace_report' then
|
||||
if not ttl_ms or ttl_ms <= 0 or type(digest) ~= 'string' or digest == '' then
|
||||
return reply('invalid')
|
||||
end
|
||||
local report = decode_table(payload)
|
||||
if not report or report.version ~= 1 or type(report.workerId) ~= 'string' or report.workerId == '' or
|
||||
type(report.sessionId) ~= 'string' or report.sessionId == '' or not valid_uint(report.sequence) or
|
||||
not valid_uint(report.snapshotVersion) or not valid_uint(report.ownershipEpoch) or
|
||||
type(report.observedAtMs) ~= 'number' or type(report.counters) ~= 'table' then
|
||||
return reply('invalid')
|
||||
end
|
||||
local session = decode_table(redis.call('HGET', sessions_key, report.workerId))
|
||||
if not valid_session(session) or session.workerId ~= report.workerId or session.sessionId ~= report.sessionId or
|
||||
type(session.expiresAtMs) ~= 'number' or session.expiresAtMs <= now then
|
||||
return reply('unavailable')
|
||||
end
|
||||
if report.snapshotVersion ~= session.ackedSnapshotVersion or
|
||||
report.ownershipEpoch ~= session.ackedOwnershipEpoch then
|
||||
return reply('stale')
|
||||
end
|
||||
local current = decode_table(redis.call('HGET', runtime_key, report.workerId))
|
||||
if current and current.sessionId == report.sessionId and valid_uint(current.sequence) then
|
||||
local ordering = compare_uint(report.sequence, current.sequence)
|
||||
if ordering < 0 then
|
||||
return reply('stale')
|
||||
end
|
||||
if ordering == 0 then
|
||||
if current.digest == digest then
|
||||
return reply('ok')
|
||||
end
|
||||
return reply('conflict')
|
||||
end
|
||||
end
|
||||
local seen = {}
|
||||
for _, counter in pairs(report.counters) do
|
||||
if type(counter) ~= 'table' or type(counter.proxyId) ~= 'string' or counter.proxyId == '' or
|
||||
type(counter.active) ~= 'number' or counter.active < 0 or counter.active ~= math.floor(counter.active) or
|
||||
type(counter.reserved) ~= 'number' or counter.reserved < 0 or counter.reserved ~= math.floor(counter.reserved) or
|
||||
type(counter.draining) ~= 'boolean' or seen[counter.proxyId] then
|
||||
return reply('invalid')
|
||||
end
|
||||
seen[counter.proxyId] = true
|
||||
local owner = decode_table(redis.call('HGET', owners_key, counter.proxyId))
|
||||
if not valid_owner(owner, report.workerId, report.ownershipEpoch, now) then
|
||||
return reply('stale')
|
||||
end
|
||||
end
|
||||
report.digest = digest
|
||||
report.expiresAtMs = now + ttl_ms
|
||||
redis.call('HSET', runtime_key, report.workerId, cjson.encode(report))
|
||||
redis.call('ZADD', runtime_expiry_key, report.expiresAtMs, report.workerId)
|
||||
session.expiresAtMs = now + ttl_ms
|
||||
redis.call('HSET', sessions_key, report.workerId, cjson.encode(session))
|
||||
redis.call('ZADD', session_expiry_key, session.expiresAtMs, report.workerId)
|
||||
return reply('ok')
|
||||
end
|
||||
|
||||
if operation == 'read' then
|
||||
local queries = decode_table(payload)
|
||||
if not queries then
|
||||
return reply('invalid')
|
||||
end
|
||||
if next(queries) == nil then
|
||||
return '{"status":"ok","snapshots":[]}'
|
||||
end
|
||||
local snapshots = cjson.decode('[]')
|
||||
local cache = {}
|
||||
for _, query in ipairs(queries) do
|
||||
if type(query) ~= 'table' or type(query.proxyId) ~= 'string' or query.proxyId == '' or
|
||||
type(query.workerId) ~= 'string' or query.workerId == '' or not valid_uint(query.ownershipEpoch) then
|
||||
return reply('invalid')
|
||||
end
|
||||
local snapshot = {proxyId = query.proxyId, active = 0, reserved = 0, draining = false, fresh = false}
|
||||
local owner = decode_table(redis.call('HGET', owners_key, query.proxyId))
|
||||
if valid_owner(owner, query.workerId, query.ownershipEpoch, now) and
|
||||
compare_uint(query.ownershipEpoch, tostring(owner.epoch)) == 0 then
|
||||
local cached = cache[query.workerId]
|
||||
if not cached then
|
||||
local session = decode_table(redis.call('HGET', sessions_key, query.workerId))
|
||||
local report = decode_table(redis.call('HGET', runtime_key, query.workerId))
|
||||
cached = {fresh = false, counters = {}}
|
||||
if valid_session(session) and session.workerId == query.workerId and report and
|
||||
report.workerId == query.workerId and report.sessionId == session.sessionId and
|
||||
type(session.expiresAtMs) == 'number' and session.expiresAtMs > now and
|
||||
type(report.expiresAtMs) == 'number' and report.expiresAtMs > now and
|
||||
valid_uint(report.ownershipEpoch) and
|
||||
report.snapshotVersion == session.ackedSnapshotVersion and
|
||||
report.ownershipEpoch == session.ackedOwnershipEpoch then
|
||||
cached.fresh = true
|
||||
cached.ownershipEpoch = report.ownershipEpoch
|
||||
if type(report.counters) == 'table' then
|
||||
for _, counter in pairs(report.counters) do
|
||||
if type(counter) == 'table' and type(counter.proxyId) == 'string' then
|
||||
cached.counters[counter.proxyId] = counter
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
cache[query.workerId] = cached
|
||||
end
|
||||
if cached.fresh and compare_uint(cached.ownershipEpoch, query.ownershipEpoch) >= 0 then
|
||||
snapshot.fresh = true
|
||||
local counter = cached.counters[query.proxyId]
|
||||
if counter then
|
||||
snapshot.active = counter.active
|
||||
snapshot.reserved = counter.reserved
|
||||
snapshot.draining = counter.draining
|
||||
end
|
||||
end
|
||||
end
|
||||
snapshots[#snapshots + 1] = snapshot
|
||||
end
|
||||
return reply('ok', snapshots)
|
||||
end
|
||||
|
||||
return reply('invalid')
|
||||
@ -73,6 +73,12 @@ local function remove_available(proxy_id, record)
|
||||
end
|
||||
end
|
||||
|
||||
local function remove_owned(proxy_id, record)
|
||||
if record and type(record.ownerIndexKey) == 'string' and record.ownerIndexKey ~= '' then
|
||||
redis.call('ZREM', record.ownerIndexKey, proxy_id)
|
||||
end
|
||||
end
|
||||
|
||||
local function remove_proxy(proxy_id)
|
||||
local raw = redis.call('HGET', records_key, proxy_id)
|
||||
local record = nil
|
||||
@ -80,6 +86,7 @@ local function remove_proxy(proxy_id)
|
||||
local decoded
|
||||
decoded, record = pcall(cjson.decode, raw)
|
||||
remove_available(proxy_id, decoded and record or nil)
|
||||
remove_owned(proxy_id, decoded and record or nil)
|
||||
if decoded and type(record) == 'table' and is_managed(record.state) then
|
||||
decrement_inventory(record.sourceUpstream)
|
||||
end
|
||||
|
||||
@ -70,12 +70,19 @@ local function remove_available(proxy_id, record)
|
||||
end
|
||||
end
|
||||
|
||||
local function remove_owned(proxy_id, record)
|
||||
if record and type(record.ownerIndexKey) == 'string' and record.ownerIndexKey ~= '' then
|
||||
redis.call('ZREM', record.ownerIndexKey, proxy_id)
|
||||
end
|
||||
end
|
||||
|
||||
local function remove_proxy(proxy_id)
|
||||
local raw = redis.call('HGET', records_key, proxy_id)
|
||||
local record = nil
|
||||
if raw then
|
||||
record = cjson.decode(raw)
|
||||
remove_available(proxy_id, record)
|
||||
remove_owned(proxy_id, record)
|
||||
if is_managed(record.state) then
|
||||
decrement_inventory(record.sourceUpstream)
|
||||
end
|
||||
@ -123,6 +130,16 @@ local function add_available(proxy_id, record)
|
||||
end
|
||||
end
|
||||
|
||||
local function sync_owned(proxy_id, record)
|
||||
if record.state == 'AVAILABLE' and record.ownerWorkerId and record.ownerWorkerId ~= '' and
|
||||
tonumber(record.usableUntilMs) > now_ms then
|
||||
redis.call('ZADD', record.ownerIndexKey, record.usableUntilMs, proxy_id)
|
||||
touch(record.ownerIndexKey, tonumber(record.expiresAtMs))
|
||||
else
|
||||
redis.call('ZREM', record.ownerIndexKey, proxy_id)
|
||||
end
|
||||
end
|
||||
|
||||
local function finish(reply)
|
||||
local encoded = cjson.encode(reply)
|
||||
redis.call('SET', operation_key, encoded, 'PX', operation_ttl_ms)
|
||||
@ -132,6 +149,11 @@ end
|
||||
cleanup_expired()
|
||||
|
||||
for _, candidate in ipairs(candidates) do
|
||||
local decoded, incoming = pcall(cjson.decode, candidate.record)
|
||||
if not decoded or type(incoming) ~= 'table' or
|
||||
type(incoming.ownerIndexKey) ~= 'string' or incoming.ownerIndexKey == '' then
|
||||
return finish({status = 'invalid', accepted = 0, inserted = 0, refreshed = 0, dropped = 0})
|
||||
end
|
||||
local mapped = redis.call('HGET', idkeys_key, candidate.proxyId)
|
||||
if mapped and mapped ~= candidate.uniqueDigest then
|
||||
return finish({status = 'invalid', accepted = 0, inserted = 0, refreshed = 0, dropped = 0})
|
||||
@ -174,6 +196,7 @@ for _, candidate in ipairs(candidates) do
|
||||
local encoded = cjson.encode(incoming)
|
||||
redis.call('HSET', records_key, incumbent_id, encoded)
|
||||
redis.call('ZADD', expiry_key, incoming.expiresAtMs, incumbent_id)
|
||||
sync_owned(incumbent_id, incoming)
|
||||
add_available(incumbent_id, incoming)
|
||||
if tonumber(incoming.expiresAtMs) > max_expiry_ms then
|
||||
max_expiry_ms = tonumber(incoming.expiresAtMs)
|
||||
@ -192,6 +215,7 @@ for _, candidate in ipairs(candidates) do
|
||||
redis.call('HSET', unique_key, candidate.uniqueDigest, candidate.proxyId)
|
||||
redis.call('HSET', idkeys_key, candidate.proxyId, candidate.uniqueDigest)
|
||||
redis.call('ZADD', expiry_key, incoming.expiresAtMs, candidate.proxyId)
|
||||
sync_owned(candidate.proxyId, incoming)
|
||||
if is_managed(incoming.state) then
|
||||
redis.call('HINCRBY', inventory_key, candidate.upstream, 1)
|
||||
end
|
||||
|
||||
@ -38,6 +38,10 @@ func TestRedisFixtureUsesIsolatedNamespace(t *testing.T) {
|
||||
}
|
||||
|
||||
func newRedisTestFixture(t *testing.T) redisTestFixture {
|
||||
return newRedisFixtureWithCredentialCapacity(t, 10_000)
|
||||
}
|
||||
|
||||
func newRedisFixtureWithCredentialCapacity(t *testing.T, credentialCapacity int) redisTestFixture {
|
||||
t.Helper()
|
||||
redisURL := os.Getenv("PROXY_POOL_TEST_REDIS_URL")
|
||||
if redisURL == "" {
|
||||
@ -54,7 +58,7 @@ func newRedisTestFixture(t *testing.T) redisTestFixture {
|
||||
_ = client.Close()
|
||||
t.Fatalf("ping test Redis: %v", err)
|
||||
}
|
||||
credentialStore, err := credentials.NewMemoryStore(10_000)
|
||||
credentialStore, err := credentials.NewMemoryStore(credentialCapacity)
|
||||
if err != nil {
|
||||
_ = client.Close()
|
||||
t.Fatalf("NewMemoryStore(): %v", err)
|
||||
|
||||
@ -15,7 +15,10 @@ import (
|
||||
"proxy-pool/internal/platform/credentials"
|
||||
)
|
||||
|
||||
const maxUpsertScriptBatch = 256
|
||||
const (
|
||||
maxUpsertScriptBatch = 256
|
||||
transientCredentialReleaseTimeout = time.Second
|
||||
)
|
||||
|
||||
type upsertCandidate struct {
|
||||
ProxyID string `json:"proxyId"`
|
||||
@ -28,6 +31,9 @@ var _ activitypool.Upserter = (*Adapter)(nil)
|
||||
|
||||
func (a *Adapter) UpsertFetched(ctx context.Context, upstreamID string, batch activitypool.FetchedBatch) (activitypool.UpsertResult, error) {
|
||||
var result activitypool.UpsertResult
|
||||
if a != nil {
|
||||
defer a.releaseTransientCredentials(batch.Proxies)
|
||||
}
|
||||
if ctx == nil {
|
||||
return result, activitypool.ErrInvalidBatch
|
||||
}
|
||||
@ -84,6 +90,26 @@ func (a *Adapter) UpsertFetched(ctx context.Context, upstreamID string, batch ac
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func (a *Adapter) releaseTransientCredentials(proxies []proxyDomain.Proxy) {
|
||||
if a.credentialReleaser == nil {
|
||||
return
|
||||
}
|
||||
releaseCtx, cancel := context.WithTimeout(context.Background(), transientCredentialReleaseTimeout)
|
||||
defer cancel()
|
||||
for _, candidate := range proxies {
|
||||
if releaseCtx.Err() != nil {
|
||||
return
|
||||
}
|
||||
if candidate.SecretRef == "" || candidate.CredentialVersion == "" {
|
||||
continue
|
||||
}
|
||||
_ = a.credentialReleaser.Release(releaseCtx, credentials.Reference{
|
||||
SecretRef: candidate.SecretRef,
|
||||
CredentialVersion: candidate.CredentialVersion,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func (a *Adapter) prepareUpsertCandidate(
|
||||
ctx context.Context,
|
||||
upstreamID string,
|
||||
@ -139,7 +165,8 @@ func (a *Adapter) prepareUpsertCandidate(
|
||||
CreatedAtMS: candidate.CreatedAt.UnixMilli(), ExpiresAtMS: expiresAt.UnixMilli(),
|
||||
UsableUntilMS: usableUntil.UnixMilli(), LatencyNS: int64(candidate.Latency),
|
||||
MaxConcurrency: candidate.MaxConcurrency, State: string(candidate.State),
|
||||
Tags: cloneTags(candidate.Tags), IndexKeys: a.availableIndexKeys(candidate),
|
||||
Tags: cloneTags(candidate.Tags), OwnerIndexKey: a.keys.owned(upstreamID),
|
||||
IndexKeys: a.availableIndexKeys(candidate),
|
||||
}
|
||||
if candidate.LastCheckedAt != nil {
|
||||
record.LastCheckedAtMS = candidate.LastCheckedAt.UnixMilli()
|
||||
|
||||
@ -173,6 +173,31 @@ func TestRedisUpsertResolvesCredentialsBeforeCommit(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestRedisUpsertReleasesTransientCredentialCapacity(t *testing.T) {
|
||||
fixture := newRedisFixtureWithCredentialCapacity(t, 1)
|
||||
first := testProxy("proxy-a", "192.0.2.10")
|
||||
firstReference, err := fixture.Credentials.Put(context.Background(), "first", credentials.Value{
|
||||
Username: "first", Password: "first-password",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Put(first credential): %v", err)
|
||||
}
|
||||
first.SecretRef = firstReference.SecretRef
|
||||
first.CredentialVersion = firstReference.CredentialVersion
|
||||
if _, err := fixture.Adapter.UpsertFetched(context.Background(), "provider-a", activitypool.FetchedBatch{
|
||||
ObservedAt: time.Now().UTC(), ConfiguredTTL: time.Minute, MaxSize: 2,
|
||||
Proxies: []proxyDomain.Proxy{first},
|
||||
}); err != nil {
|
||||
t.Fatalf("UpsertFetched(first): %v", err)
|
||||
}
|
||||
|
||||
if _, err := fixture.Credentials.Put(context.Background(), "second", credentials.Value{
|
||||
Username: "second", Password: "second-password",
|
||||
}); err != nil {
|
||||
t.Fatalf("Put(second credential after upsert): %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRedisHealthTransitionsAreMonotonicAndIdempotent(t *testing.T) {
|
||||
fixture := newRedisTestFixture(t)
|
||||
now := redisTestNow()
|
||||
|
||||
@ -55,6 +55,8 @@ func (adapter *Adapter) RunLeader(
|
||||
) error {
|
||||
if ctx == nil || adapter == nil || work == nil || strings.TrimSpace(upstreamID) != upstreamID || upstreamID == "" ||
|
||||
limits.RequestInterval < 0 || limits.MaxInFlight <= 0 || limits.MaxAttemptDuration <= 0 ||
|
||||
limits.MaxTotal < 0 || limits.MaxTotal > controllerProvider.MaximumCoordinationInteger ||
|
||||
int64(limits.MaxInFlight) > controllerProvider.MaximumCoordinationInteger ||
|
||||
limits.MaxAttemptDuration > time.Duration(math.MaxInt64)-adapter.options.PermitGrace {
|
||||
return controllerProvider.ErrInvalidCoordination
|
||||
}
|
||||
@ -217,9 +219,10 @@ func (session *leaderSession) Fence() controllerProvider.Fence {
|
||||
return controllerProvider.Fence{Generation: session.generation, Epoch: session.epoch}
|
||||
}
|
||||
|
||||
func (session *leaderSession) AcquireFetch(ctx context.Context) (controllerProvider.RequestPermit, error) {
|
||||
if ctx == nil || session == nil || session.adapter == nil || session.ctx == nil {
|
||||
return nil, controllerProvider.ErrInvalidCoordination
|
||||
func (session *leaderSession) AcquireFetch(ctx context.Context, expected int) (controllerProvider.RequestPermit, bool, error) {
|
||||
if ctx == nil || session == nil || session.adapter == nil || session.ctx == nil ||
|
||||
expected <= 0 || int64(expected) > controllerProvider.MaximumCoordinationInteger {
|
||||
return nil, false, controllerProvider.ErrInvalidCoordination
|
||||
}
|
||||
operationCtx, cancel := context.WithCancel(ctx)
|
||||
stop := context.AfterFunc(session.ctx, cancel)
|
||||
@ -229,32 +232,37 @@ func (session *leaderSession) AcquireFetch(ctx context.Context) (controllerProvi
|
||||
}()
|
||||
permitToken, err := randomToken()
|
||||
if err != nil {
|
||||
return nil, errors.Join(controllerProvider.ErrCoordinationUnavailable, err)
|
||||
return nil, false, errors.Join(controllerProvider.ErrCoordinationUnavailable, err)
|
||||
}
|
||||
permitTTL := session.limits.MaxAttemptDuration + session.adapter.options.PermitGrace
|
||||
for operationCtx.Err() == nil {
|
||||
reply, scriptErr := runScript(operationCtx, session.adapter.client, session.keys,
|
||||
"acquire_fetch", session.generation, session.holderID, session.token, session.epoch,
|
||||
permitToken, durationMillis(session.limits.RequestInterval), session.limits.MaxInFlight,
|
||||
durationMillis(permitTTL),
|
||||
durationMillis(permitTTL), expected, session.limits.MaxTotal,
|
||||
)
|
||||
if scriptErr != nil {
|
||||
if session.ctx.Err() != nil {
|
||||
return nil, controllerProvider.ErrLeadershipLost
|
||||
return nil, false, controllerProvider.ErrLeadershipLost
|
||||
}
|
||||
if operationCtx.Err() != nil {
|
||||
return nil, operationCtx.Err()
|
||||
return nil, false, operationCtx.Err()
|
||||
}
|
||||
if err := wait(operationCtx, session.adapter.options.RetryInterval); err != nil {
|
||||
return nil, err
|
||||
return nil, false, err
|
||||
}
|
||||
continue
|
||||
}
|
||||
switch reply.Status {
|
||||
case "ok":
|
||||
return &requestPermit{adapter: session.adapter, keys: session.keys, token: permitToken}, nil
|
||||
return &requestPermit{
|
||||
adapter: session.adapter, keys: session.keys, token: permitToken,
|
||||
settlementTTL: permitTTL,
|
||||
}, true, nil
|
||||
case "quota_exhausted":
|
||||
return nil, false, nil
|
||||
case "stale":
|
||||
return nil, controllerProvider.ErrLeadershipLost
|
||||
return nil, false, controllerProvider.ErrLeadershipLost
|
||||
case "rate_limited", "at_capacity":
|
||||
delay := time.Duration(reply.WaitMS) * time.Millisecond
|
||||
if delay <= 0 {
|
||||
@ -262,30 +270,42 @@ func (session *leaderSession) AcquireFetch(ctx context.Context) (controllerProvi
|
||||
}
|
||||
if err := wait(operationCtx, delay); err != nil {
|
||||
if session.ctx.Err() != nil {
|
||||
return nil, controllerProvider.ErrLeadershipLost
|
||||
return nil, false, controllerProvider.ErrLeadershipLost
|
||||
}
|
||||
return nil, err
|
||||
return nil, false, err
|
||||
}
|
||||
default:
|
||||
return nil, controllerProvider.ErrCoordinationUnavailable
|
||||
return nil, false, controllerProvider.ErrCoordinationUnavailable
|
||||
}
|
||||
}
|
||||
if session.ctx.Err() != nil {
|
||||
return nil, controllerProvider.ErrLeadershipLost
|
||||
return nil, false, controllerProvider.ErrLeadershipLost
|
||||
}
|
||||
return nil, operationCtx.Err()
|
||||
return nil, false, operationCtx.Err()
|
||||
}
|
||||
|
||||
type requestPermit struct {
|
||||
adapter *Adapter
|
||||
keys upstreamKeys
|
||||
token string
|
||||
mu sync.Mutex
|
||||
done bool
|
||||
adapter *Adapter
|
||||
keys upstreamKeys
|
||||
token string
|
||||
settlementTTL time.Duration
|
||||
mu sync.Mutex
|
||||
done bool
|
||||
}
|
||||
|
||||
func (permit *requestPermit) Release(ctx context.Context) error {
|
||||
if ctx == nil || permit == nil || permit.adapter == nil || permit.token == "" {
|
||||
func (permit *requestPermit) Complete(ctx context.Context, fetched int) error {
|
||||
if fetched < 0 || int64(fetched) > controllerProvider.MaximumCoordinationInteger {
|
||||
return controllerProvider.ErrInvalidCoordination
|
||||
}
|
||||
return permit.finish(ctx, "complete_fetch", fetched)
|
||||
}
|
||||
|
||||
func (permit *requestPermit) Cancel(ctx context.Context) error {
|
||||
return permit.finish(ctx, "cancel_fetch", 0)
|
||||
}
|
||||
|
||||
func (permit *requestPermit) finish(ctx context.Context, operation string, fetched int) error {
|
||||
if ctx == nil || permit == nil || permit.adapter == nil || permit.token == "" || permit.settlementTTL <= 0 {
|
||||
return controllerProvider.ErrInvalidCoordination
|
||||
}
|
||||
permit.mu.Lock()
|
||||
@ -293,15 +313,22 @@ func (permit *requestPermit) Release(ctx context.Context) error {
|
||||
if permit.done {
|
||||
return nil
|
||||
}
|
||||
reply, err := runScript(ctx, permit.adapter.client, permit.keys, "release_fetch", permit.token)
|
||||
if err != nil {
|
||||
return err
|
||||
for ctx.Err() == nil {
|
||||
reply, err := runScript(ctx, permit.adapter.client, permit.keys,
|
||||
operation, permit.token, fetched, durationMillis(permit.settlementTTL))
|
||||
if err != nil {
|
||||
if waitErr := wait(ctx, permit.adapter.options.RetryInterval); waitErr != nil {
|
||||
return waitErr
|
||||
}
|
||||
continue
|
||||
}
|
||||
if reply.Status != "ok" {
|
||||
return controllerProvider.ErrCoordinationUnavailable
|
||||
}
|
||||
permit.done = true
|
||||
return nil
|
||||
}
|
||||
if reply.Status != "ok" {
|
||||
return controllerProvider.ErrCoordinationUnavailable
|
||||
}
|
||||
permit.done = true
|
||||
return nil
|
||||
return ctx.Err()
|
||||
}
|
||||
|
||||
func randomToken() (string, error) {
|
||||
|
||||
@ -89,6 +89,27 @@ func TestRunLeaderRejectsInvalidCalls(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunLeaderRejectsNegativeFetchQuota(t *testing.T) {
|
||||
t.Parallel()
|
||||
client := redis.NewClient(&redis.Options{Addr: "127.0.0.1:0"})
|
||||
t.Cleanup(func() { _ = client.Close() })
|
||||
adapter, err := New(client, Options{
|
||||
Namespace: "controller", HolderID: "controller-a", LeaseTTL: 3 * time.Second,
|
||||
RenewEvery: time.Second, RetryInterval: 50 * time.Millisecond,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("New(): %v", err)
|
||||
}
|
||||
limits := controllerProvider.CoordinationLimits{
|
||||
MaxInFlight: 1, MaxAttemptDuration: time.Second, MaxTotal: -1,
|
||||
}
|
||||
err = adapter.RunLeader(context.Background(), "provider-a", limits,
|
||||
func(context.Context, controllerProvider.LeaderSession) error { return nil })
|
||||
if !errors.Is(err, controllerProvider.ErrInvalidCoordination) {
|
||||
t.Fatalf("RunLeader() error = %v, want ErrInvalidCoordination", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLeaderWorkResultPrefersParentCancellation(t *testing.T) {
|
||||
t.Parallel()
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
|
||||
@ -103,36 +103,201 @@ func TestRedisLeaderSessionEnforcesGlobalIntervalAndInFlightLimit(t *testing.T)
|
||||
})
|
||||
}()
|
||||
session := receiveSession(t, sessions)
|
||||
first, err := session.AcquireFetch(context.Background())
|
||||
if err != nil {
|
||||
t.Fatalf("first AcquireFetch(): %v", err)
|
||||
first, available, err := session.AcquireFetch(context.Background(), 1)
|
||||
if err != nil || !available {
|
||||
t.Fatalf("first AcquireFetch() = (%v, %t, %v)", first, available, err)
|
||||
}
|
||||
startedAt := time.Now()
|
||||
secondResult := make(chan permitResultFixture, 1)
|
||||
go func() {
|
||||
permit, acquireErr := session.AcquireFetch(context.Background())
|
||||
secondResult <- permitResultFixture{permit: permit, err: acquireErr}
|
||||
permit, permitAvailable, acquireErr := session.AcquireFetch(context.Background(), 1)
|
||||
secondResult <- permitResultFixture{permit: permit, available: permitAvailable, err: acquireErr}
|
||||
}()
|
||||
select {
|
||||
case result := <-secondResult:
|
||||
t.Fatalf("second AcquireFetch() returned before release: %+v", result)
|
||||
case <-time.After(100 * time.Millisecond):
|
||||
}
|
||||
if err := first.Release(context.Background()); err != nil {
|
||||
t.Fatalf("first Release(): %v", err)
|
||||
if err := first.Complete(context.Background(), 1); err != nil {
|
||||
t.Fatalf("first Complete(): %v", err)
|
||||
}
|
||||
result := receivePermit(t, secondResult)
|
||||
if result.err != nil || result.permit == nil {
|
||||
t.Fatalf("second AcquireFetch() = (%v, %v)", result.permit, result.err)
|
||||
if result.err != nil || !result.available || result.permit == nil {
|
||||
t.Fatalf("second AcquireFetch() = (%v, %t, %v)", result.permit, result.available, result.err)
|
||||
}
|
||||
if elapsed := time.Since(startedAt); elapsed < 200*time.Millisecond {
|
||||
t.Fatalf("global request interval = %s, want at least 200ms", elapsed)
|
||||
}
|
||||
if err := result.permit.Release(context.Background()); err != nil {
|
||||
t.Fatalf("second Release(): %v", err)
|
||||
if err := result.permit.Cancel(context.Background()); err != nil {
|
||||
t.Fatalf("second Cancel(): %v", err)
|
||||
}
|
||||
if err := result.permit.Release(context.Background()); err != nil {
|
||||
t.Fatalf("idempotent second Release(): %v", err)
|
||||
if err := result.permit.Cancel(context.Background()); err != nil {
|
||||
t.Fatalf("idempotent second Cancel(): %v", err)
|
||||
}
|
||||
cancel()
|
||||
waitRunner(t, done)
|
||||
}
|
||||
|
||||
func TestRedisLeaderSessionPreservesFetchQuotaAcrossFailover(t *testing.T) {
|
||||
fixture := newRedisFixture(t)
|
||||
limits := controllerProvider.CoordinationLimits{
|
||||
MaxInFlight: 1, MaxAttemptDuration: time.Second, MaxTotal: 2,
|
||||
}
|
||||
|
||||
firstCtx, cancelFirst := context.WithCancel(context.Background())
|
||||
firstSessions := make(chan controllerProvider.LeaderSession, 1)
|
||||
firstDone := make(chan error, 1)
|
||||
go func() {
|
||||
firstDone <- fixture.coordinator(t, "controller-a").RunLeader(
|
||||
firstCtx, "provider-a", limits,
|
||||
func(workCtx context.Context, session controllerProvider.LeaderSession) error {
|
||||
firstSessions <- session
|
||||
<-workCtx.Done()
|
||||
return nil
|
||||
},
|
||||
)
|
||||
}()
|
||||
firstSession := receiveSession(t, firstSessions)
|
||||
firstPermit, available, err := firstSession.AcquireFetch(context.Background(), 2)
|
||||
if err != nil || !available || firstPermit == nil {
|
||||
t.Fatalf("first AcquireFetch() = (%v, %t, %v)", firstPermit, available, err)
|
||||
}
|
||||
firstFence := firstSession.Fence()
|
||||
cancelFirst()
|
||||
waitRunner(t, firstDone)
|
||||
if err := firstPermit.Complete(context.Background(), 1); err != nil {
|
||||
t.Fatalf("Complete() after leadership loss: %v", err)
|
||||
}
|
||||
|
||||
secondCtx, cancelSecond := context.WithCancel(context.Background())
|
||||
defer cancelSecond()
|
||||
secondSessions := make(chan controllerProvider.LeaderSession, 1)
|
||||
secondDone := make(chan error, 1)
|
||||
go func() {
|
||||
secondDone <- fixture.coordinator(t, "controller-b").RunLeader(
|
||||
secondCtx, "provider-a", limits,
|
||||
func(workCtx context.Context, session controllerProvider.LeaderSession) error {
|
||||
secondSessions <- session
|
||||
<-workCtx.Done()
|
||||
return nil
|
||||
},
|
||||
)
|
||||
}()
|
||||
secondSession := receiveSession(t, secondSessions)
|
||||
secondFence := secondSession.Fence()
|
||||
if secondFence.Generation != firstFence.Generation || secondFence.Epoch <= firstFence.Epoch {
|
||||
t.Fatalf("second fence = %+v, first = %+v", secondFence, firstFence)
|
||||
}
|
||||
|
||||
secondPermit, available, err := secondSession.AcquireFetch(context.Background(), 1)
|
||||
if err != nil || !available || secondPermit == nil {
|
||||
t.Fatalf("second AcquireFetch() = (%v, %t, %v)", secondPermit, available, err)
|
||||
}
|
||||
if err := secondPermit.Complete(context.Background(), 1); err != nil {
|
||||
t.Fatalf("second Complete(): %v", err)
|
||||
}
|
||||
exhaustedPermit, available, err := secondSession.AcquireFetch(context.Background(), 1)
|
||||
if err != nil || available || exhaustedPermit != nil {
|
||||
t.Fatalf("exhausted AcquireFetch() = (%v, %t, %v), want unavailable", exhaustedPermit, available, err)
|
||||
}
|
||||
cancelSecond()
|
||||
waitRunner(t, secondDone)
|
||||
}
|
||||
|
||||
func TestRedisFetchQuotaSettlementIsIdempotentAndCancellationRefundsReservation(t *testing.T) {
|
||||
fixture := newRedisFixture(t)
|
||||
limits := controllerProvider.CoordinationLimits{
|
||||
MaxInFlight: 2, MaxAttemptDuration: time.Second, MaxTotal: 2,
|
||||
}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
sessions := make(chan controllerProvider.LeaderSession, 1)
|
||||
done := make(chan error, 1)
|
||||
go func() {
|
||||
done <- fixture.coordinator(t, "controller-a").RunLeader(
|
||||
ctx, "provider-a", limits,
|
||||
func(workCtx context.Context, session controllerProvider.LeaderSession) error {
|
||||
sessions <- session
|
||||
<-workCtx.Done()
|
||||
return nil
|
||||
},
|
||||
)
|
||||
}()
|
||||
session := receiveSession(t, sessions)
|
||||
|
||||
cancelled, available, err := session.AcquireFetch(context.Background(), 2)
|
||||
if err != nil || !available || cancelled == nil {
|
||||
t.Fatalf("cancelled AcquireFetch() = (%v, %t, %v)", cancelled, available, err)
|
||||
}
|
||||
if err := cancelled.Cancel(context.Background()); err != nil {
|
||||
t.Fatalf("Cancel(): %v", err)
|
||||
}
|
||||
if err := cancelled.Cancel(context.Background()); err != nil {
|
||||
t.Fatalf("idempotent Cancel(): %v", err)
|
||||
}
|
||||
|
||||
completed, available, err := session.AcquireFetch(context.Background(), 2)
|
||||
if err != nil || !available || completed == nil {
|
||||
t.Fatalf("completed AcquireFetch() = (%v, %t, %v)", completed, available, err)
|
||||
}
|
||||
if err := completed.Complete(context.Background(), 1); err != nil {
|
||||
t.Fatalf("Complete(): %v", err)
|
||||
}
|
||||
if err := completed.Complete(context.Background(), 1); err != nil {
|
||||
t.Fatalf("idempotent Complete(): %v", err)
|
||||
}
|
||||
if err := completed.Cancel(context.Background()); err != nil {
|
||||
t.Fatalf("Cancel() after Complete(): %v", err)
|
||||
}
|
||||
|
||||
last, available, err := session.AcquireFetch(context.Background(), 1)
|
||||
if err != nil || !available || last == nil {
|
||||
t.Fatalf("last AcquireFetch() = (%v, %t, %v)", last, available, err)
|
||||
}
|
||||
if err := last.Complete(context.Background(), 1); err != nil {
|
||||
t.Fatalf("last Complete(): %v", err)
|
||||
}
|
||||
exhausted, available, err := session.AcquireFetch(context.Background(), 1)
|
||||
if err != nil || available || exhausted != nil {
|
||||
t.Fatalf("exhausted AcquireFetch() = (%v, %t, %v)", exhausted, available, err)
|
||||
}
|
||||
cancel()
|
||||
waitRunner(t, done)
|
||||
}
|
||||
|
||||
func TestRedisExpiredFetchReservationIsConservativelyCharged(t *testing.T) {
|
||||
fixture := newRedisFixture(t)
|
||||
coordinator, err := New(fixture.client, Options{
|
||||
Namespace: fixture.namespace, HolderID: "controller-a", LeaseTTL: 600 * time.Millisecond,
|
||||
RenewEvery: 150 * time.Millisecond, RetryInterval: 10 * time.Millisecond, PermitGrace: 10 * time.Millisecond,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("New(): %v", err)
|
||||
}
|
||||
limits := controllerProvider.CoordinationLimits{
|
||||
MaxInFlight: 1, MaxAttemptDuration: 40 * time.Millisecond, MaxTotal: 1,
|
||||
}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
sessions := make(chan controllerProvider.LeaderSession, 1)
|
||||
done := make(chan error, 1)
|
||||
go func() {
|
||||
done <- coordinator.RunLeader(ctx, "provider-a", limits,
|
||||
func(workCtx context.Context, session controllerProvider.LeaderSession) error {
|
||||
sessions <- session
|
||||
<-workCtx.Done()
|
||||
return nil
|
||||
})
|
||||
}()
|
||||
session := receiveSession(t, sessions)
|
||||
abandoned, available, err := session.AcquireFetch(context.Background(), 1)
|
||||
if err != nil || !available || abandoned == nil {
|
||||
t.Fatalf("abandoned AcquireFetch() = (%v, %t, %v)", abandoned, available, err)
|
||||
}
|
||||
time.Sleep(80 * time.Millisecond)
|
||||
exhausted, available, err := session.AcquireFetch(context.Background(), 1)
|
||||
if err != nil || available || exhausted != nil {
|
||||
t.Fatalf("post-expiry AcquireFetch() = (%v, %t, %v), want charged quota", exhausted, available, err)
|
||||
}
|
||||
cancel()
|
||||
waitRunner(t, done)
|
||||
@ -280,8 +445,9 @@ func receivePermit(t *testing.T, values <-chan permitResultFixture) permitResult
|
||||
}
|
||||
|
||||
type permitResultFixture struct {
|
||||
permit controllerProvider.RequestPermit
|
||||
err error
|
||||
permit controllerProvider.RequestPermit
|
||||
available bool
|
||||
err error
|
||||
}
|
||||
|
||||
func waitRunner(t *testing.T, done <-chan error) {
|
||||
|
||||
@ -14,11 +14,15 @@ type keyBuilder struct {
|
||||
}
|
||||
|
||||
type upstreamKeys struct {
|
||||
generation string
|
||||
epoch string
|
||||
leader string
|
||||
next string
|
||||
inflight string
|
||||
generation string
|
||||
epoch string
|
||||
leader string
|
||||
next string
|
||||
inflight string
|
||||
fetchedTotal string
|
||||
pendingTotal string
|
||||
permits string
|
||||
permitExpiry string
|
||||
}
|
||||
|
||||
func (builder keyBuilder) forUpstream(upstreamID string) (upstreamKeys, error) {
|
||||
@ -28,16 +32,23 @@ func (builder keyBuilder) forUpstream(upstreamID string) (upstreamKeys, error) {
|
||||
digest := digestParts(upstreamID)
|
||||
prefix := "pp:" + builder.namespace + ":{provider:" + digest + "}"
|
||||
return upstreamKeys{
|
||||
generation: prefix + ":generation",
|
||||
epoch: prefix + ":epoch",
|
||||
leader: prefix + ":leader",
|
||||
next: prefix + ":next-request",
|
||||
inflight: prefix + ":inflight",
|
||||
generation: prefix + ":generation",
|
||||
epoch: prefix + ":epoch",
|
||||
leader: prefix + ":leader",
|
||||
next: prefix + ":next-request",
|
||||
inflight: prefix + ":inflight",
|
||||
fetchedTotal: prefix + ":fetched-total",
|
||||
pendingTotal: prefix + ":pending-total",
|
||||
permits: prefix + ":permits",
|
||||
permitExpiry: prefix + ":permit-expiry",
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (keys upstreamKeys) all() []string {
|
||||
return []string{keys.generation, keys.epoch, keys.leader, keys.next, keys.inflight}
|
||||
return []string{
|
||||
keys.generation, keys.epoch, keys.leader, keys.next, keys.inflight,
|
||||
keys.fetchedTotal, keys.pendingTotal, keys.permits, keys.permitExpiry,
|
||||
}
|
||||
}
|
||||
|
||||
func digestParts(values ...string) string {
|
||||
|
||||
@ -1,4 +1,5 @@
|
||||
local operation = ARGV[1]
|
||||
local max_safe_integer = 9007199254740991
|
||||
|
||||
local function now_ms()
|
||||
local value = redis.call('TIME')
|
||||
@ -33,6 +34,63 @@ local function same_leader(value, generation, holder_id, token, epoch)
|
||||
value.token == token and value.epoch == tostring(epoch)
|
||||
end
|
||||
|
||||
local function read_counter(key)
|
||||
local encoded = redis.call('GET', key)
|
||||
if not encoded then
|
||||
return 0, nil
|
||||
end
|
||||
local value = tonumber(encoded)
|
||||
if not value or value < 0 or value ~= math.floor(value) then
|
||||
return nil, 'invalid'
|
||||
end
|
||||
return value, nil
|
||||
end
|
||||
|
||||
local function read_permit(token)
|
||||
local encoded = redis.call('HGET', KEYS[8], token)
|
||||
if not encoded then
|
||||
return nil, nil
|
||||
end
|
||||
local ok, value = pcall(cjson.decode, encoded)
|
||||
if not ok or type(value) ~= 'table' or type(value.state) ~= 'string' or
|
||||
type(value.expected) ~= 'number' or value.expected <= 0 or
|
||||
value.expected ~= math.floor(value.expected) then
|
||||
return nil, 'invalid'
|
||||
end
|
||||
return value, nil
|
||||
end
|
||||
|
||||
local function cleanup_expired(now)
|
||||
local expired = redis.call('ZRANGEBYSCORE', KEYS[9], '-inf', now, 'LIMIT', 0, 256)
|
||||
if #expired == 0 then
|
||||
return nil
|
||||
end
|
||||
local fetched, fetched_error = read_counter(KEYS[6])
|
||||
local pending, pending_error = read_counter(KEYS[7])
|
||||
if fetched_error or pending_error then
|
||||
return 'invalid'
|
||||
end
|
||||
for _, permit_token in ipairs(expired) do
|
||||
local permit, permit_error = read_permit(permit_token)
|
||||
if permit_error then
|
||||
return 'invalid'
|
||||
end
|
||||
if permit and permit.state == 'reserved' then
|
||||
if pending < permit.expected or fetched > max_safe_integer - permit.expected then
|
||||
return 'invalid'
|
||||
end
|
||||
pending = pending - permit.expected
|
||||
fetched = fetched + permit.expected
|
||||
end
|
||||
redis.call('HDEL', KEYS[8], permit_token)
|
||||
redis.call('ZREM', KEYS[5], permit_token)
|
||||
redis.call('ZREM', KEYS[9], permit_token)
|
||||
end
|
||||
redis.call('SET', KEYS[6], fetched)
|
||||
redis.call('SET', KEYS[7], pending)
|
||||
return nil
|
||||
end
|
||||
|
||||
if operation == 'acquire_leader' then
|
||||
local generation_candidate = ARGV[2]
|
||||
local holder_id = ARGV[3]
|
||||
@ -41,7 +99,10 @@ if operation == 'acquire_leader' then
|
||||
if not lease_ttl or lease_ttl <= 0 then
|
||||
return reply('invalid', '', 0, 0)
|
||||
end
|
||||
redis.call('SET', KEYS[1], generation_candidate, 'NX')
|
||||
local created = redis.call('SET', KEYS[1], generation_candidate, 'NX')
|
||||
if created then
|
||||
redis.call('DEL', KEYS[2], KEYS[3], KEYS[4], KEYS[5], KEYS[6], KEYS[7], KEYS[8], KEYS[9])
|
||||
end
|
||||
local generation = redis.call('GET', KEYS[1])
|
||||
local current, current_error = read_leader()
|
||||
if current_error then
|
||||
@ -112,6 +173,13 @@ if operation == 'acquire_fetch' then
|
||||
local request_interval = tonumber(ARGV[7])
|
||||
local max_in_flight = tonumber(ARGV[8])
|
||||
local permit_ttl = tonumber(ARGV[9])
|
||||
local expected = tonumber(ARGV[10])
|
||||
local max_total = tonumber(ARGV[11])
|
||||
if not expected or expected <= 0 or expected ~= math.floor(expected) or
|
||||
expected > max_safe_integer or not max_total or max_total < 0 or
|
||||
max_total > max_safe_integer or max_total ~= math.floor(max_total) then
|
||||
return reply('invalid', generation, epoch or 0, 0)
|
||||
end
|
||||
local current, current_error = read_leader()
|
||||
if current_error then
|
||||
return reply('unavailable', generation, epoch or 0, 0)
|
||||
@ -120,11 +188,27 @@ if operation == 'acquire_fetch' then
|
||||
return reply('stale', generation, epoch or 0, 0)
|
||||
end
|
||||
local now = now_ms()
|
||||
redis.call('ZREMRANGEBYSCORE', KEYS[5], '-inf', now)
|
||||
local existing = redis.call('ZSCORE', KEYS[5], permit_token)
|
||||
if existing then
|
||||
if cleanup_expired(now) then
|
||||
return reply('unavailable', generation, epoch, 0)
|
||||
end
|
||||
local existing, existing_error = read_permit(permit_token)
|
||||
if existing_error then
|
||||
return reply('unavailable', generation, epoch, 0)
|
||||
end
|
||||
if existing and existing.state == 'reserved' then
|
||||
return reply('ok', generation, epoch, 0)
|
||||
end
|
||||
if existing then
|
||||
return reply('unavailable', generation, epoch, 0)
|
||||
end
|
||||
local fetched, fetched_error = read_counter(KEYS[6])
|
||||
local pending, pending_error = read_counter(KEYS[7])
|
||||
if fetched_error or pending_error then
|
||||
return reply('unavailable', generation, epoch, 0)
|
||||
end
|
||||
if max_total > 0 and fetched + pending + expected > max_total then
|
||||
return reply('quota_exhausted', generation, epoch, 0)
|
||||
end
|
||||
local next_request = redis.call('GET', KEYS[4])
|
||||
if next_request and not tonumber(next_request) then
|
||||
return reply('unavailable', generation, epoch, 0)
|
||||
@ -141,7 +225,13 @@ if operation == 'acquire_fetch' then
|
||||
return reply('at_capacity', generation, epoch, wait_ms)
|
||||
end
|
||||
redis.call('ZADD', KEYS[5], now + permit_ttl, permit_token)
|
||||
redis.call('PEXPIRE', KEYS[5], permit_ttl + 1000)
|
||||
redis.call('ZADD', KEYS[9], now + permit_ttl, permit_token)
|
||||
redis.call('HSET', KEYS[8], permit_token, cjson.encode({
|
||||
version = 1,
|
||||
state = 'reserved',
|
||||
expected = expected
|
||||
}))
|
||||
redis.call('SET', KEYS[7], pending + expected)
|
||||
if request_interval > 0 then
|
||||
redis.call('SET', KEYS[4], now + request_interval, 'PX', request_interval)
|
||||
else
|
||||
@ -150,8 +240,43 @@ if operation == 'acquire_fetch' then
|
||||
return reply('ok', generation, epoch, 0)
|
||||
end
|
||||
|
||||
if operation == 'release_fetch' then
|
||||
redis.call('ZREM', KEYS[5], ARGV[2])
|
||||
if operation == 'complete_fetch' or operation == 'cancel_fetch' then
|
||||
local permit_token = ARGV[2]
|
||||
local fetched_count = tonumber(ARGV[3])
|
||||
local settlement_ttl = tonumber(ARGV[4])
|
||||
if not fetched_count or fetched_count < 0 or fetched_count ~= math.floor(fetched_count) or
|
||||
fetched_count > max_safe_integer or not settlement_ttl or settlement_ttl <= 0 then
|
||||
return reply('invalid', '', 0, 0)
|
||||
end
|
||||
local now = now_ms()
|
||||
if cleanup_expired(now) then
|
||||
return reply('unavailable', '', 0, 0)
|
||||
end
|
||||
local permit, permit_error = read_permit(permit_token)
|
||||
if permit_error then
|
||||
return reply('unavailable', '', 0, 0)
|
||||
end
|
||||
if not permit or permit.state ~= 'reserved' then
|
||||
return reply('ok', '', 0, 0)
|
||||
end
|
||||
local fetched, fetched_error = read_counter(KEYS[6])
|
||||
local pending, pending_error = read_counter(KEYS[7])
|
||||
if fetched_error or pending_error or pending < permit.expected or
|
||||
fetched > max_safe_integer - fetched_count then
|
||||
return reply('unavailable', '', 0, 0)
|
||||
end
|
||||
pending = pending - permit.expected
|
||||
if operation == 'complete_fetch' then
|
||||
fetched = fetched + fetched_count
|
||||
permit.state = 'completed'
|
||||
else
|
||||
permit.state = 'cancelled'
|
||||
end
|
||||
redis.call('SET', KEYS[6], fetched)
|
||||
redis.call('SET', KEYS[7], pending)
|
||||
redis.call('HSET', KEYS[8], permit_token, cjson.encode(permit))
|
||||
redis.call('ZREM', KEYS[5], permit_token)
|
||||
redis.call('ZADD', KEYS[9], now + settlement_ttl, permit_token)
|
||||
return reply('ok', '', 0, 0)
|
||||
end
|
||||
|
||||
|
||||
@ -5,6 +5,12 @@ import (
|
||||
"time"
|
||||
)
|
||||
|
||||
const (
|
||||
MaximumPoolSize = 1_000_000
|
||||
MaximumExactCounter = int64(1<<53 - 1)
|
||||
MaximumUpstreams = 4_096
|
||||
)
|
||||
|
||||
type Duration time.Duration
|
||||
|
||||
func (d *Duration) UnmarshalText(text []byte) error {
|
||||
|
||||
@ -4,6 +4,7 @@ import (
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
@ -553,6 +554,13 @@ func TestValidateRejectsInvalidConfigurationMatrix(t *testing.T) {
|
||||
},
|
||||
want: "estimatedIPsPerCall",
|
||||
},
|
||||
{
|
||||
name: "pool exceeds runtime scan bound",
|
||||
mutate: func(cfg *Config) {
|
||||
updateUpstream(cfg, func(upstream *Upstream) { upstream.Pool.MaxSize = MaximumPoolSize + 1 })
|
||||
},
|
||||
want: "pool.maxSize",
|
||||
},
|
||||
{
|
||||
name: "zero refill interval",
|
||||
mutate: func(cfg *Config) {
|
||||
@ -655,6 +663,58 @@ func TestValidateRejectsInvalidConfigurationMatrix(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateRejectsUnboundedUpstreamCardinality(t *testing.T) {
|
||||
cfg := mustLoadValidConfig(t)
|
||||
for index := len(cfg.Upstreams); index <= MaximumUpstreams; index++ {
|
||||
cfg.Upstreams[fmt.Sprintf("disabled-%d", index)] = Upstream{}
|
||||
}
|
||||
if err := Validate(cfg); err == nil || !strings.Contains(err.Error(), "upstream count") {
|
||||
t.Fatalf("Validate(too many upstreams) error = %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateRejectsCountersOutsideRedisExactRange(t *testing.T) {
|
||||
if strconv.IntSize < 64 {
|
||||
t.Skip("64-bit int is required for values above the Redis exact range")
|
||||
}
|
||||
tests := []struct {
|
||||
name string
|
||||
mutate func(*Upstream)
|
||||
}{
|
||||
{name: "fetch total", mutate: func(upstream *Upstream) {
|
||||
upstream.Fetch.MaxTotal = int(MaximumExactCounter) + 1
|
||||
}},
|
||||
{name: "proxy concurrency", mutate: func(upstream *Upstream) {
|
||||
upstream.Capacity.MaxConcurrencyPerProxy = int(MaximumExactCounter) + 1
|
||||
}},
|
||||
{name: "minimum slots", mutate: func(upstream *Upstream) {
|
||||
upstream.Refill.MinimumAvailableSlots = MaximumExactCounter + 1
|
||||
upstream.Refill.TargetAvailableSlots = MaximumExactCounter + 2
|
||||
}},
|
||||
{name: "target slots", mutate: func(upstream *Upstream) {
|
||||
upstream.Capacity.MaxConcurrencyPerProxy = int(MaximumExactCounter)
|
||||
upstream.Pool.MaxSize = 1
|
||||
upstream.Refill.MinimumAvailableSlots = MaximumExactCounter
|
||||
upstream.Refill.TargetAvailableSlots = MaximumExactCounter + 1
|
||||
}},
|
||||
{name: "theoretical slots", mutate: func(upstream *Upstream) {
|
||||
upstream.Pool.MaxSize = 2
|
||||
upstream.Capacity.MaxConcurrencyPerProxy = int(MaximumExactCounter/2 + 1)
|
||||
upstream.Refill.MinimumAvailableSlots = 1
|
||||
upstream.Refill.TargetAvailableSlots = 2
|
||||
}},
|
||||
}
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
cfg := mustLoadValidConfig(t)
|
||||
updateUpstream(cfg, test.mutate)
|
||||
if err := Validate(cfg); err == nil || !strings.Contains(err.Error(), "exact counter range") {
|
||||
t.Fatalf("Validate(inexact counter) error = %v", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolvedConfigFormattingRedactsSecrets(t *testing.T) {
|
||||
configured := strings.Replace(validConfig, ` auth:
|
||||
mode: none`, ` auth:
|
||||
|
||||
30
internal/config/fingerprint.go
Normal file
30
internal/config/fingerprint.go
Normal file
@ -0,0 +1,30 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"crypto/hmac"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
)
|
||||
|
||||
const MinimumFingerprintKeyBytes = 32
|
||||
|
||||
var ErrInvalidFingerprint = errors.New("invalid configuration fingerprint")
|
||||
|
||||
// Fingerprint returns a keyed digest of one fully resolved configuration.
|
||||
// Only the digest is persisted; the resolved configuration and key remain local.
|
||||
func Fingerprint(configuration *Config, key []byte) (string, error) {
|
||||
if configuration == nil || len(key) < MinimumFingerprintKeyBytes {
|
||||
return "", ErrInvalidFingerprint
|
||||
}
|
||||
encoded, err := json.Marshal(configuration)
|
||||
if err != nil {
|
||||
return "", errors.Join(ErrInvalidFingerprint, err)
|
||||
}
|
||||
digest := hmac.New(sha256.New, key)
|
||||
if _, err := digest.Write(encoded); err != nil {
|
||||
return "", errors.Join(ErrInvalidFingerprint, err)
|
||||
}
|
||||
return hex.EncodeToString(digest.Sum(nil)), nil
|
||||
}
|
||||
68
internal/config/fingerprint_test.go
Normal file
68
internal/config/fingerprint_test.go
Normal file
@ -0,0 +1,68 @@
|
||||
package config
|
||||
|
||||
import "testing"
|
||||
|
||||
var testFingerprintKey = []byte("0123456789abcdef0123456789abcdef")
|
||||
|
||||
func TestFingerprintIsStableAndTracksSecretRotation(t *testing.T) {
|
||||
first := storeTestConfig("provider-a")
|
||||
upstream := first.Upstreams["provider-a"]
|
||||
upstream.ProxyAuth.Password = "secret-a"
|
||||
first.Upstreams["provider-a"] = upstream
|
||||
|
||||
stable, err := Fingerprint(first, testFingerprintKey)
|
||||
if err != nil {
|
||||
t.Fatalf("Fingerprint(first): %v", err)
|
||||
}
|
||||
again, err := Fingerprint(first, testFingerprintKey)
|
||||
if err != nil || again != stable {
|
||||
t.Fatalf("Fingerprint(stable) = %q, %v; want %q", again, err, stable)
|
||||
}
|
||||
|
||||
rotated := storeTestConfig("provider-a")
|
||||
upstream = rotated.Upstreams["provider-a"]
|
||||
upstream.ProxyAuth.Password = "secret-b"
|
||||
rotated.Upstreams["provider-a"] = upstream
|
||||
changed, err := Fingerprint(rotated, testFingerprintKey)
|
||||
if err != nil {
|
||||
t.Fatalf("Fingerprint(rotated): %v", err)
|
||||
}
|
||||
if changed == stable {
|
||||
t.Fatal("Fingerprint did not change after secret rotation")
|
||||
}
|
||||
if len(changed) != 64 {
|
||||
t.Fatalf("Fingerprint length = %d, want 64", len(changed))
|
||||
}
|
||||
}
|
||||
|
||||
func TestFingerprintChangesWithIndependentKey(t *testing.T) {
|
||||
configuration := storeTestConfig("provider-a")
|
||||
first, err := Fingerprint(configuration, testFingerprintKey)
|
||||
if err != nil {
|
||||
t.Fatalf("Fingerprint(first key): %v", err)
|
||||
}
|
||||
second, err := Fingerprint(configuration, []byte("fedcba9876543210fedcba9876543210"))
|
||||
if err != nil {
|
||||
t.Fatalf("Fingerprint(second key): %v", err)
|
||||
}
|
||||
if first == second {
|
||||
t.Fatal("Fingerprint did not change with independent key")
|
||||
}
|
||||
}
|
||||
|
||||
func TestFingerprintRejectsInvalidInputs(t *testing.T) {
|
||||
for name, test := range map[string]struct {
|
||||
configuration *Config
|
||||
key []byte
|
||||
}{
|
||||
"nil configuration": {key: testFingerprintKey},
|
||||
"missing key": {configuration: storeTestConfig("provider-a")},
|
||||
"short key": {configuration: storeTestConfig("provider-a"), key: []byte("too-short")},
|
||||
} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
if _, err := Fingerprint(test.configuration, test.key); err == nil {
|
||||
t.Fatal("Fingerprint() succeeded")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@ -9,7 +9,12 @@ var ErrInvalidStore = errors.New("invalid configuration store")
|
||||
|
||||
// Store publishes complete validated configurations with one atomic pointer swap.
|
||||
type Store struct {
|
||||
current atomic.Pointer[Config]
|
||||
current atomic.Pointer[publishedConfiguration]
|
||||
}
|
||||
|
||||
type publishedConfiguration struct {
|
||||
value Config
|
||||
revision uint64
|
||||
}
|
||||
|
||||
func NewStore(initial *Config) (*Store, error) {
|
||||
@ -17,7 +22,7 @@ func NewStore(initial *Config) (*Store, error) {
|
||||
return nil, errors.Join(ErrInvalidStore, err)
|
||||
}
|
||||
store := &Store{}
|
||||
store.Publish(initial)
|
||||
store.current.Store(&publishedConfiguration{value: cloneConfig(*initial)})
|
||||
return store, nil
|
||||
}
|
||||
|
||||
@ -25,19 +30,38 @@ func (store *Store) Current() *Config {
|
||||
if store == nil {
|
||||
return nil
|
||||
}
|
||||
current := store.current.Load()
|
||||
if current == nil {
|
||||
published := store.current.Load()
|
||||
if published == nil {
|
||||
return nil
|
||||
}
|
||||
cloned := cloneConfig(*current)
|
||||
cloned := cloneConfig(published.value)
|
||||
return &cloned
|
||||
}
|
||||
|
||||
// Publish accepts a non-nil configuration already validated by the caller.
|
||||
func (store *Store) Publish(configuration *Config) {
|
||||
if store == nil || configuration == nil {
|
||||
return
|
||||
func (store *Store) Revision() uint64 {
|
||||
if store == nil {
|
||||
return 0
|
||||
}
|
||||
published := store.current.Load()
|
||||
if published == nil {
|
||||
return 0
|
||||
}
|
||||
return published.revision
|
||||
}
|
||||
|
||||
// PublishRevision publishes only a strictly newer authoritative revision.
|
||||
func (store *Store) PublishRevision(configuration *Config, revision uint64) bool {
|
||||
if store == nil || configuration == nil || revision == 0 {
|
||||
return false
|
||||
}
|
||||
for {
|
||||
current := store.current.Load()
|
||||
if current != nil && revision <= current.revision {
|
||||
return false
|
||||
}
|
||||
next := &publishedConfiguration{value: cloneConfig(*configuration), revision: revision}
|
||||
if store.current.CompareAndSwap(current, next) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
cloned := cloneConfig(*configuration)
|
||||
store.current.Store(&cloned)
|
||||
}
|
||||
|
||||
@ -26,7 +26,9 @@ func TestStorePublishesAndReturnsDetachedConfigurations(t *testing.T) {
|
||||
}
|
||||
|
||||
next := storeTestConfig("provider-b")
|
||||
store.Publish(next)
|
||||
if !store.PublishRevision(next, 1) {
|
||||
t.Fatal("PublishRevision() rejected newer configuration")
|
||||
}
|
||||
next.Routing[0].Upstreams[0] = "mutated"
|
||||
if got := store.Current().Routing[0].Upstreams[0]; got != "provider-b" {
|
||||
t.Fatalf("Publish() retained caller state: %q", got)
|
||||
@ -56,7 +58,7 @@ func TestStoreSupportsConcurrentReadersAndPublishers(t *testing.T) {
|
||||
if index%2 == 1 {
|
||||
name = "provider-b"
|
||||
}
|
||||
store.Publish(storeTestConfig(name))
|
||||
store.PublishRevision(storeTestConfig(name), uint64(index+1))
|
||||
}(index)
|
||||
go func() {
|
||||
defer wait.Done()
|
||||
@ -69,6 +71,29 @@ func TestStoreSupportsConcurrentReadersAndPublishers(t *testing.T) {
|
||||
wait.Wait()
|
||||
}
|
||||
|
||||
func TestStoreRejectsOutOfOrderRevisionPublication(t *testing.T) {
|
||||
t.Parallel()
|
||||
store, err := NewStore(storeTestConfig("provider-a"))
|
||||
if err != nil {
|
||||
t.Fatalf("NewStore() error = %v", err)
|
||||
}
|
||||
if published := store.PublishRevision(storeTestConfig("provider-b"), 2); !published {
|
||||
t.Fatal("PublishRevision(newer) rejected")
|
||||
}
|
||||
if published := store.PublishRevision(storeTestConfig("provider-c"), 1); published {
|
||||
t.Fatal("PublishRevision(stale) succeeded")
|
||||
}
|
||||
if published := store.PublishRevision(storeTestConfig("provider-c"), 2); published {
|
||||
t.Fatal("PublishRevision(equal) succeeded")
|
||||
}
|
||||
if got := store.Current().Routing[0].Upstreams[0]; got != "provider-b" {
|
||||
t.Fatalf("Current() upstream = %q, want provider-b", got)
|
||||
}
|
||||
if got := store.Revision(); got != 2 {
|
||||
t.Fatalf("Revision() = %d, want 2", got)
|
||||
}
|
||||
}
|
||||
|
||||
func storeTestConfig(upstreamName string) *Config {
|
||||
return &Config{
|
||||
Version: 1,
|
||||
|
||||
@ -46,6 +46,9 @@ func Validate(cfg *Config) error {
|
||||
if err := validateCheck("defaults.check", cfg.Defaults.Check); err != nil {
|
||||
return err
|
||||
}
|
||||
if len(cfg.Upstreams) > MaximumUpstreams {
|
||||
return fmt.Errorf("validate configuration: upstream count exceeds %d", MaximumUpstreams)
|
||||
}
|
||||
enabledUpstreams := 0
|
||||
for name, upstream := range cfg.Upstreams {
|
||||
if upstream.Enabled {
|
||||
@ -330,28 +333,46 @@ func validateUpstream(name string, upstream Upstream) error {
|
||||
if err := requirePositive(scope+" pool.maxSize", upstream.Pool.MaxSize); err != nil {
|
||||
return err
|
||||
}
|
||||
if upstream.Pool.MaxSize > MaximumPoolSize {
|
||||
return fmt.Errorf("validate %s pool.maxSize: exceeds %d", scope, MaximumPoolSize)
|
||||
}
|
||||
if err := requireNonNegative(scope+" fetch.maxTotal", upstream.Fetch.MaxTotal); err != nil {
|
||||
return err
|
||||
}
|
||||
if int64(upstream.Fetch.MaxTotal) > MaximumExactCounter {
|
||||
return fmt.Errorf("validate %s fetch.maxTotal: exceeds exact counter range", scope)
|
||||
}
|
||||
if upstream.Fetch.MaxTotal > 0 && upstream.Fetch.MaxTotal < upstream.Pool.MaxSize {
|
||||
return fmt.Errorf("validate %s fetch.maxTotal: cannot be lower than pool.maxSize", scope)
|
||||
}
|
||||
if err := requirePositive(scope+" capacity.maxConcurrencyPerProxy", upstream.Capacity.MaxConcurrencyPerProxy); err != nil {
|
||||
return err
|
||||
}
|
||||
if int64(upstream.Capacity.MaxConcurrencyPerProxy) > MaximumExactCounter {
|
||||
return fmt.Errorf("validate %s capacity.maxConcurrencyPerProxy: exceeds exact counter range", scope)
|
||||
}
|
||||
if err := requirePositive(scope+" refill.reconcileInterval", upstream.Refill.ReconcileInterval); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := requirePositive(scope+" refill.minimumAvailableSlots", upstream.Refill.MinimumAvailableSlots); err != nil {
|
||||
return err
|
||||
}
|
||||
if upstream.Refill.MinimumAvailableSlots > MaximumExactCounter {
|
||||
return fmt.Errorf("validate %s refill.minimumAvailableSlots: exceeds exact counter range", scope)
|
||||
}
|
||||
if upstream.Refill.TargetAvailableSlots <= upstream.Refill.MinimumAvailableSlots {
|
||||
return fmt.Errorf("validate %s refill.targetAvailableSlots: must be greater than minimumAvailableSlots", scope)
|
||||
}
|
||||
if upstream.Refill.TargetAvailableSlots > MaximumExactCounter {
|
||||
return fmt.Errorf("validate %s refill.targetAvailableSlots: exceeds exact counter range", scope)
|
||||
}
|
||||
if int64(upstream.Pool.MaxSize) > math.MaxInt64/int64(upstream.Capacity.MaxConcurrencyPerProxy) {
|
||||
return fmt.Errorf("validate %s refill.targetAvailableSlots: theoretical capacity overflows int64", scope)
|
||||
}
|
||||
theoreticalSlots := int64(upstream.Pool.MaxSize) * int64(upstream.Capacity.MaxConcurrencyPerProxy)
|
||||
if theoreticalSlots > MaximumExactCounter {
|
||||
return fmt.Errorf("validate %s refill.targetAvailableSlots: theoretical capacity exceeds exact counter range", scope)
|
||||
}
|
||||
if upstream.Refill.TargetAvailableSlots > theoreticalSlots {
|
||||
return fmt.Errorf("validate %s refill.targetAvailableSlots: exceeds theoretical capacity", scope)
|
||||
}
|
||||
@ -415,6 +436,9 @@ func validateFetch(scope string, fetch Fetch) error {
|
||||
if err := requirePositive(scope+".maxInFlight", fetch.MaxInFlight); err != nil {
|
||||
return err
|
||||
}
|
||||
if int64(fetch.MaxInFlight) > MaximumExactCounter {
|
||||
return fmt.Errorf("validate %s.maxInFlight: exceeds exact counter range", scope)
|
||||
}
|
||||
if err := requireNonNegative(scope+".maxTotal", fetch.MaxTotal); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@ -2,9 +2,6 @@ package admin
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"sort"
|
||||
"strings"
|
||||
@ -29,9 +26,15 @@ type ConfigurationLoader interface {
|
||||
LoadConfiguration(context.Context) (LoadedConfiguration, error)
|
||||
}
|
||||
|
||||
// ConfigurationPublisher must atomically publish an already validated configuration.
|
||||
// ConfigurationPublisher publishes only a newer authoritative configuration revision.
|
||||
type ConfigurationPublisher interface {
|
||||
Publish(*config.Config)
|
||||
PublishRevision(*config.Config, uint64) bool
|
||||
}
|
||||
|
||||
type RuntimeController interface {
|
||||
Notify()
|
||||
ValidateConfiguration(context.Context, *config.Config) error
|
||||
ValidateUpstream(context.Context, string) error
|
||||
}
|
||||
|
||||
var _ ConfigurationPublisher = (*config.Store)(nil)
|
||||
@ -41,10 +44,12 @@ type ApplicationDependencies struct {
|
||||
Operations OperationalStatusReader
|
||||
Configuration ConfigurationLoader
|
||||
Publisher ConfigurationPublisher
|
||||
Runtime RuntimeController
|
||||
}
|
||||
|
||||
type ApplicationOptions struct {
|
||||
Now func() time.Time
|
||||
Now func() time.Time
|
||||
FingerprintKey []byte
|
||||
}
|
||||
|
||||
type OperationalStatus struct {
|
||||
@ -70,30 +75,40 @@ type LoadedConfiguration struct {
|
||||
}
|
||||
|
||||
type ApplicationService struct {
|
||||
state StateRepository
|
||||
operations OperationalStatusReader
|
||||
configuration ConfigurationLoader
|
||||
publisher ConfigurationPublisher
|
||||
now func() time.Time
|
||||
state StateRepository
|
||||
operations OperationalStatusReader
|
||||
configuration ConfigurationLoader
|
||||
publisher ConfigurationPublisher
|
||||
runtime RuntimeController
|
||||
now func() time.Time
|
||||
fingerprintKey []byte
|
||||
}
|
||||
|
||||
var _ Service = (*ApplicationService)(nil)
|
||||
|
||||
func NewApplicationService(dependencies ApplicationDependencies, options ApplicationOptions) (*ApplicationService, error) {
|
||||
if nilInterface(dependencies.State) || nilInterface(dependencies.Operations) || nilInterface(dependencies.Configuration) ||
|
||||
nilInterface(dependencies.Publisher) || options.Now == nil {
|
||||
nilInterface(dependencies.Publisher) || options.Now == nil ||
|
||||
len(options.FingerprintKey) < config.MinimumFingerprintKeyBytes {
|
||||
return nil, ErrInvalidApplicationService
|
||||
}
|
||||
return &ApplicationService{
|
||||
state: dependencies.State,
|
||||
operations: dependencies.Operations,
|
||||
configuration: dependencies.Configuration,
|
||||
publisher: dependencies.Publisher,
|
||||
now: options.Now,
|
||||
state: dependencies.State,
|
||||
operations: dependencies.Operations,
|
||||
configuration: dependencies.Configuration,
|
||||
publisher: dependencies.Publisher,
|
||||
runtime: dependencies.Runtime,
|
||||
now: options.Now,
|
||||
fingerprintKey: append([]byte(nil), options.FingerprintKey...),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (service *ApplicationService) SetUpstreamEnabled(ctx context.Context, command SetUpstreamCommand) (MutationResult, error) {
|
||||
if command.Enabled && service.runtime != nil {
|
||||
if err := service.runtime.ValidateUpstream(ctx, command.Name); err != nil {
|
||||
return MutationResult{RequestID: command.RequestID}, errors.Join(ErrInvalidConfiguration, err)
|
||||
}
|
||||
}
|
||||
result, err := service.state.SetUpstreamEnabled(ctx, adminstate.SetUpstreamCommand{
|
||||
RequestID: command.RequestID,
|
||||
Actor: adminstate.Actor{ID: command.ActorID, SourceIP: command.SourceIP},
|
||||
@ -101,6 +116,9 @@ func (service *ApplicationService) SetUpstreamEnabled(ctx context.Context, comma
|
||||
Name: command.Name,
|
||||
Enabled: command.Enabled,
|
||||
})
|
||||
if err == nil && service.runtime != nil {
|
||||
service.runtime.Notify()
|
||||
}
|
||||
return mutationResult(result), mapAdminStateError(err)
|
||||
}
|
||||
|
||||
@ -202,14 +220,16 @@ func (service *ApplicationService) ApplyConfiguration(
|
||||
if err := config.Validate(loaded.Value); err != nil {
|
||||
return MutationResult{RequestID: command.RequestID}, errors.Join(ErrInvalidConfiguration, err)
|
||||
}
|
||||
if service.runtime != nil {
|
||||
if err := service.runtime.ValidateConfiguration(ctx, loaded.Value); err != nil {
|
||||
return MutationResult{RequestID: command.RequestID}, errors.Join(ErrInvalidConfiguration, err)
|
||||
}
|
||||
}
|
||||
|
||||
managementView := loaded.Value.Redacted()
|
||||
encoded, err := json.Marshal(managementView)
|
||||
checksum, err := config.Fingerprint(loaded.Value, service.fingerprintKey)
|
||||
if err != nil {
|
||||
return MutationResult{RequestID: command.RequestID}, errors.Join(ErrInvalidConfiguration, err)
|
||||
}
|
||||
digest := sha256.Sum256(encoded)
|
||||
checksum := hex.EncodeToString(digest[:])
|
||||
|
||||
current, err := service.state.Snapshot(ctx)
|
||||
if err != nil {
|
||||
@ -229,7 +249,13 @@ func (service *ApplicationService) ApplyConfiguration(
|
||||
if err != nil {
|
||||
return mutationResult(result), mapAdminStateError(err)
|
||||
}
|
||||
service.publisher.Publish(loaded.Value)
|
||||
if result.Revision == 0 {
|
||||
return mutationResult(result), errors.Join(ErrUnavailable, ErrInvalidApplicationService)
|
||||
}
|
||||
published := service.publisher.PublishRevision(loaded.Value, result.Revision)
|
||||
if published && service.runtime != nil {
|
||||
service.runtime.Notify()
|
||||
}
|
||||
return mutationResult(result), nil
|
||||
}
|
||||
|
||||
|
||||
@ -3,6 +3,7 @@ package admin
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
@ -10,6 +11,12 @@ import (
|
||||
"proxy-pool/internal/domain/adminstate"
|
||||
)
|
||||
|
||||
var applicationTestFingerprintKey = []byte("0123456789abcdef0123456789abcdef")
|
||||
|
||||
func applicationTestOptions(now func() time.Time) ApplicationOptions {
|
||||
return ApplicationOptions{Now: now, FingerprintKey: applicationTestFingerprintKey}
|
||||
}
|
||||
|
||||
func TestApplicationServiceMapsUpstreamMutationToAdminState(t *testing.T) {
|
||||
t.Parallel()
|
||||
now := time.Date(2026, 7, 29, 10, 0, 0, 0, time.UTC)
|
||||
@ -21,12 +28,14 @@ func TestApplicationServiceMapsUpstreamMutationToAdminState(t *testing.T) {
|
||||
Message: "enabled",
|
||||
},
|
||||
}
|
||||
runtime := &recordingRuntimeNotifier{}
|
||||
service, err := NewApplicationService(ApplicationDependencies{
|
||||
State: state,
|
||||
Operations: staticOperationalStatusReader{},
|
||||
Configuration: staticConfigurationLoader{},
|
||||
Publisher: &recordingConfigurationPublisher{},
|
||||
}, ApplicationOptions{Now: func() time.Time { return now }})
|
||||
Runtime: runtime,
|
||||
}, applicationTestOptions(func() time.Time { return now }))
|
||||
if err != nil {
|
||||
t.Fatalf("NewApplicationService() error = %v", err)
|
||||
}
|
||||
@ -53,6 +62,34 @@ func TestApplicationServiceMapsUpstreamMutationToAdminState(t *testing.T) {
|
||||
}) {
|
||||
t.Fatalf("admin state command = %+v", state.lastUpstream)
|
||||
}
|
||||
if runtime.notifications != 1 {
|
||||
t.Fatalf("runtime notifications = %d, want 1", runtime.notifications)
|
||||
}
|
||||
}
|
||||
|
||||
func TestApplicationServicePreflightsProviderRuntimeBeforeMutation(t *testing.T) {
|
||||
t.Parallel()
|
||||
wantErr := errors.New("invalid Provider template")
|
||||
state := &recordingAdminState{}
|
||||
runtime := &recordingRuntimeNotifier{validationErr: wantErr}
|
||||
service, err := NewApplicationService(ApplicationDependencies{
|
||||
State: state, Operations: staticOperationalStatusReader{},
|
||||
Configuration: staticConfigurationLoader{}, Publisher: &recordingConfigurationPublisher{},
|
||||
Runtime: runtime,
|
||||
}, applicationTestOptions(time.Now))
|
||||
if err != nil {
|
||||
t.Fatalf("NewApplicationService(): %v", err)
|
||||
}
|
||||
|
||||
_, err = service.SetUpstreamEnabled(context.Background(), SetUpstreamCommand{
|
||||
RequestID: "req-enable", Name: "provider-a", Enabled: true,
|
||||
})
|
||||
if !errors.Is(err, ErrInvalidConfiguration) || !errors.Is(err, wantErr) {
|
||||
t.Fatalf("SetUpstreamEnabled() error = %v", err)
|
||||
}
|
||||
if state.lastUpstream != (adminstate.SetUpstreamCommand{}) {
|
||||
t.Fatalf("state mutated before runtime preflight: %+v", state.lastUpstream)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewApplicationServiceRejectsMissingDependencies(t *testing.T) {
|
||||
@ -68,11 +105,12 @@ func TestNewApplicationServiceRejectsMissingDependencies(t *testing.T) {
|
||||
dependencies ApplicationDependencies
|
||||
options ApplicationOptions
|
||||
}{
|
||||
{name: "state", dependencies: func() ApplicationDependencies { value := valid; value.State = nil; return value }(), options: ApplicationOptions{Now: time.Now}},
|
||||
{name: "operations", dependencies: func() ApplicationDependencies { value := valid; value.Operations = nil; return value }(), options: ApplicationOptions{Now: time.Now}},
|
||||
{name: "configuration", dependencies: func() ApplicationDependencies { value := valid; value.Configuration = nil; return value }(), options: ApplicationOptions{Now: time.Now}},
|
||||
{name: "publisher", dependencies: func() ApplicationDependencies { value := valid; value.Publisher = nil; return value }(), options: ApplicationOptions{Now: time.Now}},
|
||||
{name: "state", dependencies: func() ApplicationDependencies { value := valid; value.State = nil; return value }(), options: applicationTestOptions(time.Now)},
|
||||
{name: "operations", dependencies: func() ApplicationDependencies { value := valid; value.Operations = nil; return value }(), options: applicationTestOptions(time.Now)},
|
||||
{name: "configuration", dependencies: func() ApplicationDependencies { value := valid; value.Configuration = nil; return value }(), options: applicationTestOptions(time.Now)},
|
||||
{name: "publisher", dependencies: func() ApplicationDependencies { value := valid; value.Publisher = nil; return value }(), options: applicationTestOptions(time.Now)},
|
||||
{name: "clock", dependencies: valid},
|
||||
{name: "fingerprint key", dependencies: valid, options: ApplicationOptions{Now: time.Now}},
|
||||
}
|
||||
for _, test := range tests {
|
||||
test := test
|
||||
@ -99,7 +137,7 @@ func TestNewApplicationServiceRejectsTypedNilDependencies(t *testing.T) {
|
||||
func() ApplicationDependencies { value := valid; value.State = state; return value }(),
|
||||
func() ApplicationDependencies { value := valid; value.Publisher = publisher; return value }(),
|
||||
} {
|
||||
if _, err := NewApplicationService(dependencies, ApplicationOptions{Now: time.Now}); !errors.Is(err, ErrInvalidApplicationService) {
|
||||
if _, err := NewApplicationService(dependencies, applicationTestOptions(time.Now)); !errors.Is(err, ErrInvalidApplicationService) {
|
||||
t.Fatalf("NewApplicationService(typed nil) error = %v, want %v", err, ErrInvalidApplicationService)
|
||||
}
|
||||
}
|
||||
@ -109,7 +147,7 @@ func TestApplicationServiceMapsRoutingSwitchAndDomainErrors(t *testing.T) {
|
||||
t.Parallel()
|
||||
now := time.Date(2026, 7, 29, 11, 0, 0, 0, time.FixedZone("test", 8*60*60))
|
||||
state := &recordingAdminState{mutation: adminstate.MutationResult{RequestID: "req-switch", Changed: true, Revision: 21}}
|
||||
service := mustApplicationService(t, state, ApplicationOptions{Now: func() time.Time { return now }})
|
||||
service := mustApplicationService(t, state, applicationTestOptions(func() time.Time { return now }))
|
||||
|
||||
result, err := service.SwitchRouting(context.Background(), SwitchCommand{
|
||||
RequestID: "req-switch", ActorID: "admin:bob", SourceIP: "198.51.100.7",
|
||||
@ -174,7 +212,7 @@ func TestApplicationServiceBuildsStatusFromAuthoritativeAndOperationalSnapshots(
|
||||
service, err := NewApplicationService(ApplicationDependencies{
|
||||
State: state, Operations: operations, Configuration: staticConfigurationLoader{},
|
||||
Publisher: &recordingConfigurationPublisher{},
|
||||
}, ApplicationOptions{Now: time.Now})
|
||||
}, applicationTestOptions(time.Now))
|
||||
if err != nil {
|
||||
t.Fatalf("NewApplicationService() error = %v", err)
|
||||
}
|
||||
@ -205,7 +243,7 @@ func TestApplicationServiceMapsStatusDependencyFailures(t *testing.T) {
|
||||
service, err := NewApplicationService(ApplicationDependencies{
|
||||
State: &recordingAdminState{err: stateFailure}, Operations: staticOperationalStatusReader{},
|
||||
Configuration: staticConfigurationLoader{}, Publisher: &recordingConfigurationPublisher{},
|
||||
}, ApplicationOptions{Now: time.Now})
|
||||
}, applicationTestOptions(time.Now))
|
||||
if err != nil {
|
||||
t.Fatalf("NewApplicationService() error = %v", err)
|
||||
}
|
||||
@ -217,7 +255,7 @@ func TestApplicationServiceMapsStatusDependencyFailures(t *testing.T) {
|
||||
service, err = NewApplicationService(ApplicationDependencies{
|
||||
State: &recordingAdminState{}, Operations: staticOperationalStatusReader{err: operationsFailure},
|
||||
Configuration: staticConfigurationLoader{}, Publisher: &recordingConfigurationPublisher{},
|
||||
}, ApplicationOptions{Now: time.Now})
|
||||
}, applicationTestOptions(time.Now))
|
||||
if err != nil {
|
||||
t.Fatalf("NewApplicationService() error = %v", err)
|
||||
}
|
||||
@ -228,7 +266,7 @@ func TestApplicationServiceMapsStatusDependencyFailures(t *testing.T) {
|
||||
service, err = NewApplicationService(ApplicationDependencies{
|
||||
State: &recordingAdminState{}, Operations: staticOperationalStatusReader{err: context.Canceled},
|
||||
Configuration: staticConfigurationLoader{}, Publisher: &recordingConfigurationPublisher{},
|
||||
}, ApplicationOptions{Now: time.Now})
|
||||
}, applicationTestOptions(time.Now))
|
||||
if err != nil {
|
||||
t.Fatalf("NewApplicationService() error = %v", err)
|
||||
}
|
||||
@ -242,6 +280,7 @@ func TestApplicationServiceReloadPersistsManagementViewBeforePublishing(t *testi
|
||||
now := time.Date(2026, 7, 29, 12, 0, 0, 0, time.UTC)
|
||||
configuration := validReloadConfiguration()
|
||||
publisher := &recordingConfigurationPublisher{}
|
||||
runtime := &recordingRuntimeNotifier{}
|
||||
state := &recordingAdminState{
|
||||
mutation: adminstate.MutationResult{RequestID: "req-reload", Changed: true, Revision: 42},
|
||||
snapshot: adminstate.Snapshot{Routings: []adminstate.RoutingState{
|
||||
@ -258,8 +297,8 @@ func TestApplicationServiceReloadPersistsManagementViewBeforePublishing(t *testi
|
||||
Configuration: staticConfigurationLoader{loaded: LoadedConfiguration{
|
||||
Value: configuration, Source: "configs/proxy-pool.yaml",
|
||||
}},
|
||||
Publisher: publisher,
|
||||
}, ApplicationOptions{Now: func() time.Time { return now }})
|
||||
Publisher: publisher, Runtime: runtime,
|
||||
}, applicationTestOptions(func() time.Time { return now }))
|
||||
if err != nil {
|
||||
t.Fatalf("NewApplicationService() error = %v", err)
|
||||
}
|
||||
@ -276,6 +315,9 @@ func TestApplicationServiceReloadPersistsManagementViewBeforePublishing(t *testi
|
||||
if len(publisher.published) != 1 || publisher.published[0] != configuration {
|
||||
t.Fatalf("published configurations = %+v", publisher.published)
|
||||
}
|
||||
if runtime.notifications != 1 {
|
||||
t.Fatalf("runtime notifications = %d, want 1", runtime.notifications)
|
||||
}
|
||||
command := state.lastConfig
|
||||
if command.RequestID != "req-reload" || command.Actor != (adminstate.Actor{ID: "admin:alice", SourceIP: "192.0.2.10"}) ||
|
||||
!command.OccurredAt.Equal(now) || command.Source != "configs/proxy-pool.yaml" {
|
||||
@ -306,7 +348,7 @@ func TestApplicationServiceApplyConfigurationUsesProvidedSnapshotWithoutReloadin
|
||||
Operations: staticOperationalStatusReader{},
|
||||
Configuration: forbiddenConfigurationLoader{},
|
||||
Publisher: publisher,
|
||||
}, ApplicationOptions{Now: func() time.Time { return now }})
|
||||
}, applicationTestOptions(func() time.Time { return now }))
|
||||
if err != nil {
|
||||
t.Fatalf("NewApplicationService() error = %v", err)
|
||||
}
|
||||
@ -355,7 +397,7 @@ func TestApplicationServiceReloadDoesNotPublishInvalidOrUncommittedConfiguration
|
||||
service, err := NewApplicationService(ApplicationDependencies{
|
||||
State: state, Operations: staticOperationalStatusReader{},
|
||||
Configuration: staticConfigurationLoader{loaded: test.loaded}, Publisher: publisher,
|
||||
}, ApplicationOptions{Now: time.Now})
|
||||
}, applicationTestOptions(time.Now))
|
||||
if err != nil {
|
||||
t.Fatalf("NewApplicationService() error = %v", err)
|
||||
}
|
||||
@ -381,7 +423,7 @@ func TestApplicationServiceReloadPublishesSuccessfulReplay(t *testing.T) {
|
||||
Configuration: staticConfigurationLoader{loaded: LoadedConfiguration{
|
||||
Value: validReloadConfiguration(), Source: "config.yaml",
|
||||
}}, Publisher: publisher,
|
||||
}, ApplicationOptions{Now: time.Now})
|
||||
}, applicationTestOptions(time.Now))
|
||||
if err != nil {
|
||||
t.Fatalf("NewApplicationService() error = %v", err)
|
||||
}
|
||||
@ -393,13 +435,82 @@ func TestApplicationServiceReloadPublishesSuccessfulReplay(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestApplicationServiceRejectsSuccessfulCommitWithoutRevision(t *testing.T) {
|
||||
publisher := &recordingConfigurationPublisher{}
|
||||
service, err := NewApplicationService(ApplicationDependencies{
|
||||
State: &recordingAdminState{}, Operations: staticOperationalStatusReader{},
|
||||
Configuration: staticConfigurationLoader{loaded: LoadedConfiguration{
|
||||
Value: validReloadConfiguration(), Source: "config.yaml",
|
||||
}}, Publisher: publisher,
|
||||
}, applicationTestOptions(time.Now))
|
||||
if err != nil {
|
||||
t.Fatalf("NewApplicationService(): %v", err)
|
||||
}
|
||||
|
||||
_, err = service.ReloadConfiguration(context.Background(), ReloadCommand{RequestID: "req-zero-revision"})
|
||||
if !errors.Is(err, ErrUnavailable) {
|
||||
t.Fatalf("ReloadConfiguration() error = %v, want unavailable", err)
|
||||
}
|
||||
if len(publisher.published) != 0 {
|
||||
t.Fatalf("published configurations = %d, want 0", len(publisher.published))
|
||||
}
|
||||
}
|
||||
|
||||
func TestApplicationServiceKeepsNewestConfigurationWhenOlderCommitReturnsLater(t *testing.T) {
|
||||
store, err := config.NewStore(configWithOnlyUpstream("provider-a"))
|
||||
if err != nil {
|
||||
t.Fatalf("config.NewStore(): %v", err)
|
||||
}
|
||||
state := &orderedCommitState{
|
||||
firstCommitted: make(chan struct{}),
|
||||
releaseFirst: make(chan struct{}),
|
||||
}
|
||||
service, err := NewApplicationService(ApplicationDependencies{
|
||||
State: state, Operations: staticOperationalStatusReader{},
|
||||
Configuration: staticConfigurationLoader{}, Publisher: store,
|
||||
}, applicationTestOptions(time.Now))
|
||||
if err != nil {
|
||||
t.Fatalf("NewApplicationService(): %v", err)
|
||||
}
|
||||
|
||||
firstDone := make(chan error, 1)
|
||||
go func() {
|
||||
_, applyErr := service.ApplyConfiguration(context.Background(), ReloadCommand{RequestID: "req-old"}, LoadedConfiguration{
|
||||
Value: configWithOnlyUpstream("provider-b"), Source: "old.yaml",
|
||||
})
|
||||
firstDone <- applyErr
|
||||
}()
|
||||
select {
|
||||
case <-state.firstCommitted:
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("first commit did not reach delayed return")
|
||||
}
|
||||
|
||||
if _, err := service.ApplyConfiguration(context.Background(), ReloadCommand{RequestID: "req-new"}, LoadedConfiguration{
|
||||
Value: configWithOnlyUpstream("provider-c"), Source: "new.yaml",
|
||||
}); err != nil {
|
||||
t.Fatalf("ApplyConfiguration(new): %v", err)
|
||||
}
|
||||
close(state.releaseFirst)
|
||||
if err := <-firstDone; err != nil {
|
||||
t.Fatalf("ApplyConfiguration(old): %v", err)
|
||||
}
|
||||
|
||||
if got := store.Current().Routing[0].Upstreams[0]; got != "provider-c" {
|
||||
t.Fatalf("published upstream = %q, want provider-c", got)
|
||||
}
|
||||
if got := store.Revision(); got != 2 {
|
||||
t.Fatalf("published revision = %d, want 2", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestApplicationServiceReloadPreservesCancellation(t *testing.T) {
|
||||
t.Parallel()
|
||||
service, err := NewApplicationService(ApplicationDependencies{
|
||||
State: &recordingAdminState{}, Operations: staticOperationalStatusReader{},
|
||||
Configuration: staticConfigurationLoader{err: context.Canceled},
|
||||
Publisher: &recordingConfigurationPublisher{},
|
||||
}, ApplicationOptions{Now: time.Now})
|
||||
}, applicationTestOptions(time.Now))
|
||||
if err != nil {
|
||||
t.Fatalf("NewApplicationService() error = %v", err)
|
||||
}
|
||||
@ -409,13 +520,16 @@ func TestApplicationServiceReloadPreservesCancellation(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestApplicationServiceUsesSecretFreeManagementChecksum(t *testing.T) {
|
||||
func TestApplicationServiceUsesOpaqueChecksumThatTracksSecretRotation(t *testing.T) {
|
||||
t.Parallel()
|
||||
state := &recordingAdminState{}
|
||||
publisher := &recordingConfigurationPublisher{}
|
||||
var commands []adminstate.CommitConfigCommand
|
||||
state.onCommit = func(command adminstate.CommitConfigCommand) {
|
||||
commands = append(commands, command)
|
||||
state.mutation = adminstate.MutationResult{
|
||||
RequestID: command.RequestID, Changed: true, Revision: uint64(len(commands)),
|
||||
}
|
||||
}
|
||||
for _, secret := range []string{"secret-a", "secret-b"} {
|
||||
configuration := validReloadConfiguration()
|
||||
@ -427,7 +541,7 @@ func TestApplicationServiceUsesSecretFreeManagementChecksum(t *testing.T) {
|
||||
Configuration: staticConfigurationLoader{loaded: LoadedConfiguration{
|
||||
Value: configuration, Source: "config.yaml",
|
||||
}}, Publisher: publisher,
|
||||
}, ApplicationOptions{Now: time.Now})
|
||||
}, applicationTestOptions(time.Now))
|
||||
if err != nil {
|
||||
t.Fatalf("NewApplicationService() error = %v", err)
|
||||
}
|
||||
@ -437,8 +551,8 @@ func TestApplicationServiceUsesSecretFreeManagementChecksum(t *testing.T) {
|
||||
t.Fatalf("ReloadConfiguration() error = %v", err)
|
||||
}
|
||||
}
|
||||
if len(commands) != 2 || commands[0].Checksum != commands[1].Checksum || commands[0].ConfigVersion != commands[1].ConfigVersion {
|
||||
t.Fatalf("secret rotation changed public management digest: %+v", commands)
|
||||
if len(commands) != 2 || commands[0].Checksum == commands[1].Checksum || commands[0].ConfigVersion == commands[1].ConfigVersion {
|
||||
t.Fatalf("secret rotation did not change opaque configuration digest: %+v", commands)
|
||||
}
|
||||
if len(publisher.published) != 2 || publisher.published[1].Upstreams["provider-a"].ProxyAuth.Password != "secret-b" {
|
||||
t.Fatalf("secret rotation was not published: %+v", publisher.published)
|
||||
@ -467,6 +581,16 @@ func validReloadConfiguration() *config.Config {
|
||||
}
|
||||
}
|
||||
|
||||
func configWithOnlyUpstream(name string) *config.Config {
|
||||
configuration := validReloadConfiguration()
|
||||
configuration.Upstreams = map[string]config.Upstream{name: validReloadUpstream("secret")}
|
||||
configuration.Routing = []config.Routing{{
|
||||
Name: "default", Enabled: true, Purpose: "gateway", Upstreams: []string{name},
|
||||
Strategy: config.Strategy{Type: "random"}, OnUnavailable: config.OnUnavailable{Action: "reject"},
|
||||
}}
|
||||
return configuration
|
||||
}
|
||||
|
||||
func validReloadUpstream(secret string) config.Upstream {
|
||||
return config.Upstream{
|
||||
Enabled: true, Exposure: []string{"gateway"},
|
||||
@ -505,6 +629,33 @@ type recordingAdminState struct {
|
||||
onCommit func(adminstate.CommitConfigCommand)
|
||||
}
|
||||
|
||||
type orderedCommitState struct {
|
||||
next atomic.Uint64
|
||||
firstCommitted chan struct{}
|
||||
releaseFirst chan struct{}
|
||||
}
|
||||
|
||||
func (state *orderedCommitState) SetUpstreamEnabled(context.Context, adminstate.SetUpstreamCommand) (adminstate.MutationResult, error) {
|
||||
return adminstate.MutationResult{}, nil
|
||||
}
|
||||
|
||||
func (state *orderedCommitState) SwitchRouting(context.Context, adminstate.SwitchRoutingCommand) (adminstate.MutationResult, error) {
|
||||
return adminstate.MutationResult{}, nil
|
||||
}
|
||||
|
||||
func (state *orderedCommitState) CommitConfig(_ context.Context, command adminstate.CommitConfigCommand) (adminstate.MutationResult, error) {
|
||||
revision := state.next.Add(1)
|
||||
if revision == 1 {
|
||||
close(state.firstCommitted)
|
||||
<-state.releaseFirst
|
||||
}
|
||||
return adminstate.MutationResult{RequestID: command.RequestID, Changed: true, Revision: revision}, nil
|
||||
}
|
||||
|
||||
func (*orderedCommitState) Snapshot(context.Context) (adminstate.Snapshot, error) {
|
||||
return adminstate.Snapshot{}, nil
|
||||
}
|
||||
|
||||
func (state *recordingAdminState) SetUpstreamEnabled(_ context.Context, command adminstate.SetUpstreamCommand) (adminstate.MutationResult, error) {
|
||||
state.lastUpstream = command
|
||||
return state.mutation, state.err
|
||||
@ -553,8 +704,26 @@ func (loader staticConfigurationLoader) LoadConfiguration(context.Context) (Load
|
||||
|
||||
type recordingConfigurationPublisher struct {
|
||||
published []*config.Config
|
||||
revisions []uint64
|
||||
}
|
||||
|
||||
func (publisher *recordingConfigurationPublisher) Publish(configuration *config.Config) {
|
||||
publisher.published = append(publisher.published, configuration)
|
||||
type recordingRuntimeNotifier struct {
|
||||
notifications int
|
||||
validationErr error
|
||||
}
|
||||
|
||||
func (notifier *recordingRuntimeNotifier) Notify() { notifier.notifications++ }
|
||||
|
||||
func (notifier *recordingRuntimeNotifier) ValidateConfiguration(context.Context, *config.Config) error {
|
||||
return notifier.validationErr
|
||||
}
|
||||
|
||||
func (notifier *recordingRuntimeNotifier) ValidateUpstream(context.Context, string) error {
|
||||
return notifier.validationErr
|
||||
}
|
||||
|
||||
func (publisher *recordingConfigurationPublisher) PublishRevision(configuration *config.Config, revision uint64) bool {
|
||||
publisher.published = append(publisher.published, configuration)
|
||||
publisher.revisions = append(publisher.revisions, revision)
|
||||
return true
|
||||
}
|
||||
|
||||
@ -5,6 +5,7 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"reflect"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
@ -15,11 +16,15 @@ import (
|
||||
"proxy-pool/internal/controller/distribution"
|
||||
"proxy-pool/internal/controller/extraction"
|
||||
"proxy-pool/internal/controller/operations"
|
||||
"proxy-pool/internal/controller/pool"
|
||||
"proxy-pool/internal/controller/provider"
|
||||
controllerRuntime "proxy-pool/internal/controller/runtime"
|
||||
"proxy-pool/internal/domain/activitypool"
|
||||
extractionDomain "proxy-pool/internal/domain/extraction"
|
||||
"proxy-pool/internal/platform/admission"
|
||||
"proxy-pool/internal/platform/credentials"
|
||||
"proxy-pool/internal/platform/httpserver"
|
||||
"proxy-pool/internal/platform/lifecycle"
|
||||
platformMetrics "proxy-pool/internal/platform/metrics"
|
||||
)
|
||||
|
||||
@ -29,14 +34,19 @@ var (
|
||||
)
|
||||
|
||||
type Options struct {
|
||||
ConfigPath string
|
||||
Resolver config.Resolver
|
||||
Now func() time.Time
|
||||
HTTP httpserver.Options
|
||||
ConfigPath string
|
||||
Resolver config.Resolver
|
||||
Now func() time.Time
|
||||
HTTP httpserver.Options
|
||||
HolderID string
|
||||
RedisNamespace string
|
||||
FingerprintKey []byte
|
||||
}
|
||||
|
||||
type activityStore interface {
|
||||
extractionDomain.Store
|
||||
activitypool.Upserter
|
||||
pool.InventoryReader
|
||||
activitypool.StateInventoryReader
|
||||
}
|
||||
|
||||
@ -45,6 +55,9 @@ type ports struct {
|
||||
activity activityStore
|
||||
readiness distribution.ReadinessChecker
|
||||
metricsReadiness platformMetrics.ReadinessChecker
|
||||
coordinator provider.Coordinator
|
||||
credentials credentials.Store
|
||||
providerResults provider.ResultRecorder
|
||||
close func() error
|
||||
}
|
||||
|
||||
@ -61,7 +74,9 @@ type runtimeFactory interface {
|
||||
}
|
||||
|
||||
func Run(ctx context.Context, options Options) error {
|
||||
return run(ctx, options, &productionInfrastructure{}, productionRuntimeFactory{})
|
||||
return run(ctx, options, &productionInfrastructure{
|
||||
holderID: options.HolderID, namespace: options.RedisNamespace,
|
||||
}, productionRuntimeFactory{})
|
||||
}
|
||||
|
||||
func run(ctx context.Context, options Options, infrastructure infrastructure, factory runtimeFactory) (resultErr error) {
|
||||
@ -84,6 +99,9 @@ func run(ctx context.Context, options Options, infrastructure infrastructure, fa
|
||||
if err != nil {
|
||||
return fmt.Errorf("%w: load configuration: %w", ErrStartup, err)
|
||||
}
|
||||
if loaded.Value.Admin.Enabled && len(options.FingerprintKey) < config.MinimumFingerprintKeyBytes {
|
||||
return errors.Join(ErrInvalidOptions, config.ErrInvalidFingerprint)
|
||||
}
|
||||
configurationStore, err := config.NewStore(loaded.Value)
|
||||
if err != nil {
|
||||
return fmt.Errorf("%w: initialize configuration store: %w", ErrStartup, err)
|
||||
@ -99,6 +117,31 @@ func run(ctx context.Context, options Options, infrastructure infrastructure, fa
|
||||
defer func() {
|
||||
resultErr = errors.Join(resultErr, opened.close())
|
||||
}()
|
||||
var providerState providerStateReader
|
||||
if loaded.Value.Admin.Enabled {
|
||||
providerState = opened.state
|
||||
}
|
||||
supervisor, err := newProviderSupervisor(
|
||||
configurationStore,
|
||||
providerState,
|
||||
func(name string, upstream config.Upstream) (lifecycle.Runner, error) {
|
||||
buildRuntime, err := providerRuntimeBuilder(opened)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return buildRuntime(name, upstream)
|
||||
},
|
||||
func(ctx context.Context, configuration *config.Config) error {
|
||||
return prepareProviderConfiguration(ctx, configuration, opened.credentials)
|
||||
},
|
||||
func(configuration *config.Config) { retainProviderStats(configuration, opened.providerResults) },
|
||||
loader,
|
||||
options.FingerprintKey,
|
||||
providerSupervisorInterval,
|
||||
)
|
||||
if err != nil {
|
||||
return fmt.Errorf("%w: build Provider supervisor: %w", ErrStartup, err)
|
||||
}
|
||||
|
||||
dependencies := controllerRuntime.Dependencies{}
|
||||
if loaded.Value.Distribution.Enabled {
|
||||
@ -116,13 +159,23 @@ func run(ctx context.Context, options Options, infrastructure infrastructure, fa
|
||||
if nilInterface(opened.state) || nilInterface(opened.activity) {
|
||||
return errors.Join(ErrStartup, ErrInvalidOptions)
|
||||
}
|
||||
statusReader, statusErr := operations.NewReader(configurationStore, opened.activity, options.Now)
|
||||
var providerStats []provider.StatsReader
|
||||
if stats, ok := opened.providerResults.(provider.StatsReader); ok && !nilInterface(stats) {
|
||||
providerStats = append(providerStats, stats)
|
||||
}
|
||||
statusReader, statusErr := operations.NewReader(
|
||||
configurationStore,
|
||||
opened.activity,
|
||||
options.Now,
|
||||
providerStats...,
|
||||
)
|
||||
if statusErr != nil {
|
||||
return fmt.Errorf("%w: build operational status reader: %w", ErrStartup, statusErr)
|
||||
}
|
||||
service, serviceErr := admin.NewApplicationService(admin.ApplicationDependencies{
|
||||
State: opened.state, Operations: statusReader, Configuration: loader, Publisher: configurationStore,
|
||||
}, admin.ApplicationOptions{Now: options.Now})
|
||||
Runtime: supervisor,
|
||||
}, admin.ApplicationOptions{Now: options.Now, FingerprintKey: options.FingerprintKey})
|
||||
if serviceErr != nil {
|
||||
return fmt.Errorf("%w: build admin service: %w", ErrStartup, serviceErr)
|
||||
}
|
||||
@ -146,14 +199,68 @@ func run(ctx context.Context, options Options, infrastructure infrastructure, fa
|
||||
dependencies.MetricsHandler = handler
|
||||
}
|
||||
|
||||
runner, err := factory.New(configurationStore.Current(), dependencies, controllerRuntime.Options{HTTP: options.HTTP})
|
||||
runners := make([]lifecycle.Runner, 0, 2)
|
||||
if hasHTTPRuntime(loaded.Value) {
|
||||
runner, err := factory.New(configurationStore.Current(), dependencies, controllerRuntime.Options{HTTP: options.HTTP})
|
||||
if err != nil {
|
||||
return fmt.Errorf("%w: build HTTP runtime: %w", ErrStartup, err)
|
||||
}
|
||||
if nilInterface(runner) {
|
||||
return errors.Join(ErrStartup, ErrInvalidOptions)
|
||||
}
|
||||
runners = append(runners, runner)
|
||||
}
|
||||
runners = append(runners, supervisor)
|
||||
group, err := lifecycle.NewGroup(runners...)
|
||||
if err != nil {
|
||||
return fmt.Errorf("%w: build HTTP runtime: %w", ErrStartup, err)
|
||||
return fmt.Errorf("%w: build process lifecycle: %w", ErrStartup, err)
|
||||
}
|
||||
if nilInterface(runner) {
|
||||
return errors.Join(ErrStartup, ErrInvalidOptions)
|
||||
return group.Run(ctx)
|
||||
}
|
||||
|
||||
func prepareProviderConfiguration(
|
||||
ctx context.Context,
|
||||
configuration *config.Config,
|
||||
credentialStore credentials.Store,
|
||||
) error {
|
||||
if ctx == nil || configuration == nil {
|
||||
return ErrProviderRuntime
|
||||
}
|
||||
return runner.Run(ctx)
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
if !configuration.Admin.Enabled && !hasEnabledUpstream(configuration) {
|
||||
return nil
|
||||
}
|
||||
ensurer, ok := credentialStore.(credentials.CapacityEnsurer)
|
||||
if !ok || nilInterface(credentialStore) {
|
||||
return ErrProviderRuntime
|
||||
}
|
||||
if err := ensurer.EnsureCapacity(ctx, providerCredentialCapacity(configuration)); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func retainProviderStats(configuration *config.Config, results provider.ResultRecorder) {
|
||||
if configuration == nil {
|
||||
return
|
||||
}
|
||||
retainer, ok := results.(provider.StatsRetainer)
|
||||
if !ok || nilInterface(retainer) {
|
||||
return
|
||||
}
|
||||
names := make([]string, 0, len(configuration.Upstreams))
|
||||
for name := range configuration.Upstreams {
|
||||
names = append(names, name)
|
||||
}
|
||||
sort.Strings(names)
|
||||
retainer.RetainProviderStats(names)
|
||||
}
|
||||
|
||||
func hasHTTPRuntime(configuration *config.Config) bool {
|
||||
return configuration != nil &&
|
||||
(configuration.Distribution.Enabled || configuration.Admin.Enabled || configuration.Metrics.Enabled)
|
||||
}
|
||||
|
||||
func extractionPolicy(configuration *config.Config) extraction.Policy {
|
||||
|
||||
@ -4,16 +4,25 @@ package bootstrap
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/redis/go-redis/v9"
|
||||
|
||||
"proxy-pool/internal/adapters/redisactivity"
|
||||
"proxy-pool/internal/config"
|
||||
"proxy-pool/internal/controller/admin"
|
||||
"proxy-pool/internal/controller/pool"
|
||||
"proxy-pool/internal/controller/provider"
|
||||
controllerRuntime "proxy-pool/internal/controller/runtime"
|
||||
"proxy-pool/internal/platform/credentials"
|
||||
)
|
||||
|
||||
func TestProductionBootstrapOpensBothStoresAndCommitsStartupConfiguration(t *testing.T) {
|
||||
@ -22,18 +31,38 @@ func TestProductionBootstrapOpensBothStoresAndCommitsStartupConfiguration(t *tes
|
||||
if postgresURL == "" || redisURL == "" {
|
||||
t.Skip("PROXY_POOL_TEST_POSTGRES_URL and PROXY_POOL_TEST_REDIS_URL are required")
|
||||
}
|
||||
var providerCalls atomic.Int64
|
||||
providerServer := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, _ *http.Request) {
|
||||
providerCalls.Add(1)
|
||||
_, _ = writer.Write([]byte("http://192.0.2.10:8080"))
|
||||
}))
|
||||
defer providerServer.Close()
|
||||
namespace := "controller-it-" + strconv.FormatInt(time.Now().UnixNano(), 10)
|
||||
inventory := newIntegrationInventoryReader(t, redisURL, namespace)
|
||||
|
||||
source := strings.ReplaceAll(bootstrapTestConfig, "postgres://fixture", postgresURL)
|
||||
source = strings.ReplaceAll(source, "redis://fixture", redisURL)
|
||||
source = strings.ReplaceAll(source, "https://provider.invalid/proxies", providerServer.URL)
|
||||
resolver := &memoryResolver{files: map[string][]byte{"controller.yaml": []byte(source)}}
|
||||
factory := &integrationRuntimeFactory{}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
providerResults := make(chan provider.Result, 8)
|
||||
factory := &integrationRuntimeFactory{
|
||||
cancel: cancel, inventory: inventory, providerResults: providerResults,
|
||||
}
|
||||
infrastructure := &integrationInfrastructure{
|
||||
productionInfrastructure: productionInfrastructure{namespace: namespace},
|
||||
results: providerResultRecorder(func(result provider.Result) { providerResults <- result }),
|
||||
}
|
||||
|
||||
err := run(context.Background(), Options{
|
||||
err := run(ctx, Options{
|
||||
ConfigPath: "controller.yaml", Resolver: resolver, Now: func() time.Time {
|
||||
return time.Date(2026, 7, 30, 13, 0, 0, 0, time.UTC)
|
||||
},
|
||||
}, &productionInfrastructure{}, factory)
|
||||
if err != nil {
|
||||
t.Fatalf("run() error = %v", err)
|
||||
FingerprintKey: bootstrapTestFingerprintKey,
|
||||
}, infrastructure, factory)
|
||||
if !errors.Is(err, context.Canceled) {
|
||||
t.Fatalf("run() error = %v, want context cancellation", err)
|
||||
}
|
||||
if factory.status.ConfigVersion == "" || len(factory.status.Upstreams) != 2 {
|
||||
t.Fatalf("Admin Status = %+v", factory.status)
|
||||
@ -46,13 +75,19 @@ func TestProductionBootstrapOpensBothStoresAndCommitsStartupConfiguration(t *tes
|
||||
!strings.Contains(factory.metricsBody, "go_") {
|
||||
t.Fatalf("Metrics probes = ready:%d metrics:%d body:%q", factory.readyStatus, factory.metricsStatus, factory.metricsBody)
|
||||
}
|
||||
if providerCalls.Load() == 0 {
|
||||
t.Fatal("production Provider HTTP adapter was not called")
|
||||
}
|
||||
}
|
||||
|
||||
type integrationRuntimeFactory struct {
|
||||
status admin.Status
|
||||
readyStatus int
|
||||
metricsStatus int
|
||||
metricsBody string
|
||||
status admin.Status
|
||||
readyStatus int
|
||||
metricsStatus int
|
||||
metricsBody string
|
||||
cancel context.CancelFunc
|
||||
inventory pool.InventoryReader
|
||||
providerResults <-chan provider.Result
|
||||
}
|
||||
|
||||
func (factory *integrationRuntimeFactory) New(
|
||||
@ -73,10 +108,99 @@ func (factory *integrationRuntimeFactory) New(
|
||||
factory.metricsBody = metrics.Body.String()
|
||||
status, err := dependencies.AdminService.Status(ctx)
|
||||
factory.status = status
|
||||
return err
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := waitForProviderInventory(
|
||||
ctx,
|
||||
factory.inventory,
|
||||
factory.providerResults,
|
||||
[]string{"provider-a", "provider-b"},
|
||||
); err != nil {
|
||||
return err
|
||||
}
|
||||
factory.cancel()
|
||||
<-ctx.Done()
|
||||
return ctx.Err()
|
||||
}}, nil
|
||||
}
|
||||
|
||||
func newIntegrationInventoryReader(t *testing.T, redisURL, namespace string) pool.InventoryReader {
|
||||
t.Helper()
|
||||
options, err := redis.ParseURL(redisURL)
|
||||
if err != nil {
|
||||
t.Fatalf("redis.ParseURL(): %v", err)
|
||||
}
|
||||
client := redis.NewClient(options)
|
||||
t.Cleanup(func() { _ = client.Close() })
|
||||
credentialStore, err := credentials.NewMemoryStore(200)
|
||||
if err != nil {
|
||||
t.Fatalf("credentials.NewMemoryStore(): %v", err)
|
||||
}
|
||||
reader, err := redisactivity.New(client, redisactivity.Options{
|
||||
Namespace: namespace, Credentials: credentialStore,
|
||||
OperationTTL: redisOperationTTL, MaxCandidateScan: redisMinimumScan,
|
||||
MaxRuntimeCounters: 200, MaxInventoryScan: 200, CleanupLimit: redisCleanupLimit,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("redisactivity.New(): %v", err)
|
||||
}
|
||||
return reader
|
||||
}
|
||||
|
||||
func waitForProviderInventory(
|
||||
ctx context.Context,
|
||||
inventory pool.InventoryReader,
|
||||
results <-chan provider.Result,
|
||||
upstreamIDs []string,
|
||||
) error {
|
||||
deadline, cancel := context.WithTimeout(ctx, 5*time.Second)
|
||||
defer cancel()
|
||||
ticker := time.NewTicker(10 * time.Millisecond)
|
||||
defer ticker.Stop()
|
||||
var latestResult provider.Result
|
||||
for {
|
||||
var inventoryErr error
|
||||
for _, upstreamID := range upstreamIDs {
|
||||
snapshot, err := inventory.ReadInventory(deadline, upstreamID, 0)
|
||||
inventoryErr = errors.Join(inventoryErr, err)
|
||||
if err == nil && snapshot.Managed > 0 {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
select {
|
||||
case <-deadline.Done():
|
||||
return errors.Join(
|
||||
errors.New("wait for Provider Redis inventory"),
|
||||
deadline.Err(),
|
||||
inventoryErr,
|
||||
latestResult.Err,
|
||||
)
|
||||
case result := <-results:
|
||||
latestResult = result
|
||||
case <-ticker.C:
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
type integrationInfrastructure struct {
|
||||
productionInfrastructure
|
||||
results provider.ResultRecorder
|
||||
}
|
||||
|
||||
func (infrastructure *integrationInfrastructure) Open(
|
||||
ctx context.Context,
|
||||
configuration *config.Config,
|
||||
) (ports, error) {
|
||||
opened, err := infrastructure.productionInfrastructure.Open(ctx, configuration)
|
||||
opened.providerResults = infrastructure.results
|
||||
return opened, err
|
||||
}
|
||||
|
||||
type providerResultRecorder func(provider.Result)
|
||||
|
||||
func (record providerResultRecorder) Record(result provider.Result) { record(result) }
|
||||
|
||||
type integrationRunner struct {
|
||||
run func(context.Context) error
|
||||
}
|
||||
|
||||
@ -3,32 +3,48 @@ package bootstrap
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"proxy-pool/internal/config"
|
||||
"proxy-pool/internal/controller/admin"
|
||||
"proxy-pool/internal/controller/pool"
|
||||
"proxy-pool/internal/controller/provider"
|
||||
controllerRuntime "proxy-pool/internal/controller/runtime"
|
||||
"proxy-pool/internal/domain/activitypool"
|
||||
"proxy-pool/internal/domain/adminstate"
|
||||
extractionDomain "proxy-pool/internal/domain/extraction"
|
||||
"proxy-pool/internal/domain/upstream"
|
||||
"proxy-pool/internal/platform/credentials"
|
||||
)
|
||||
|
||||
var bootstrapTestFingerprintKey = []byte("0123456789abcdef0123456789abcdef")
|
||||
|
||||
func TestRunLoadsOneSnapshotCommitsItAndClosesInfrastructure(t *testing.T) {
|
||||
t.Parallel()
|
||||
resolver := &memoryResolver{files: map[string][]byte{"controller.yaml": []byte(bootstrapTestConfig)}}
|
||||
state := adminstate.NewMemoryStore()
|
||||
activity := &stubActivityStore{}
|
||||
credentialStore, err := credentials.NewMemoryStore(10)
|
||||
if err != nil {
|
||||
t.Fatalf("NewMemoryStore(): %v", err)
|
||||
}
|
||||
closeErr := errors.New("close failed")
|
||||
infrastructure := &stubInfrastructure{ports: ports{
|
||||
state: state, activity: activity, readiness: readyStub{}, metricsReadiness: readyStub{},
|
||||
coordinator: coordinatorStub{}, credentials: credentialStore,
|
||||
close: func() error { return closeErr },
|
||||
}}
|
||||
runErr := errors.New("runtime failed")
|
||||
factory := &recordingRuntimeFactory{runner: runnerStub{err: runErr}}
|
||||
now := time.Date(2026, 7, 30, 11, 0, 0, 0, time.UTC)
|
||||
|
||||
err := run(context.Background(), Options{
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
|
||||
defer cancel()
|
||||
err = run(ctx, Options{
|
||||
ConfigPath: "controller.yaml", Resolver: resolver, Now: func() time.Time { return now },
|
||||
FingerprintKey: bootstrapTestFingerprintKey,
|
||||
}, infrastructure, factory)
|
||||
if !errors.Is(err, runErr) || !errors.Is(err, closeErr) {
|
||||
t.Fatalf("run() error = %v, want runtime and close errors", err)
|
||||
@ -74,6 +90,147 @@ func TestRunRejectsInvalidOptionsBeforeIO(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunRejectsMissingAdminFingerprintKeyBeforeOpeningInfrastructure(t *testing.T) {
|
||||
for name, key := range map[string][]byte{
|
||||
"missing": nil,
|
||||
"short": []byte("too-short"),
|
||||
} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
infrastructure := &stubInfrastructure{}
|
||||
err := run(context.Background(), Options{
|
||||
ConfigPath: "controller.yaml",
|
||||
Resolver: &memoryResolver{files: map[string][]byte{"controller.yaml": []byte(bootstrapTestConfig)}},
|
||||
Now: time.Now,
|
||||
FingerprintKey: key,
|
||||
}, infrastructure, &recordingRuntimeFactory{})
|
||||
if !errors.Is(err, ErrInvalidOptions) || !errors.Is(err, config.ErrInvalidFingerprint) {
|
||||
t.Fatalf("run() error = %v", err)
|
||||
}
|
||||
if infrastructure.opens != 0 {
|
||||
t.Fatalf("infrastructure opens = %d, want 0", infrastructure.opens)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunSupportsProviderOnlyConfigurationWithoutHTTPRuntime(t *testing.T) {
|
||||
source := strings.ReplaceAll(bootstrapTestConfig, "distribution:\n enabled: true", "distribution:\n enabled: false")
|
||||
source = strings.ReplaceAll(source, "admin:\n enabled: true", "admin:\n enabled: false")
|
||||
source = strings.ReplaceAll(source, "metrics:\n enabled: true", "metrics:\n enabled: false")
|
||||
credentialStore, err := credentials.NewMemoryStore(10)
|
||||
if err != nil {
|
||||
t.Fatalf("NewMemoryStore(): %v", err)
|
||||
}
|
||||
infrastructure := &stubInfrastructure{ports: ports{
|
||||
activity: &stubActivityStore{}, coordinator: coordinatorStub{}, credentials: credentialStore,
|
||||
close: func() error { return nil },
|
||||
}}
|
||||
factory := &recordingRuntimeFactory{}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
time.AfterFunc(20*time.Millisecond, cancel)
|
||||
err = run(ctx, Options{
|
||||
ConfigPath: "controller.yaml",
|
||||
Resolver: &memoryResolver{files: map[string][]byte{"controller.yaml": []byte(source)}},
|
||||
Now: time.Now,
|
||||
}, infrastructure, factory)
|
||||
if !errors.Is(err, context.Canceled) {
|
||||
t.Fatalf("run(provider only) error = %v, want context cancellation", err)
|
||||
}
|
||||
if factory.configuration != nil {
|
||||
t.Fatal("HTTP runtime factory was called for Provider-only configuration")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunAdminDisableStopsActiveProviderRuntime(t *testing.T) {
|
||||
state := adminstate.NewMemoryStore()
|
||||
credentialStore, err := credentials.NewMemoryStore(200)
|
||||
if err != nil {
|
||||
t.Fatalf("NewMemoryStore(): %v", err)
|
||||
}
|
||||
started := make(chan string, 2)
|
||||
stopped := make(chan string, 2)
|
||||
infrastructure := &stubInfrastructure{ports: ports{
|
||||
state: state, activity: &stubActivityStore{}, readiness: readyStub{}, metricsReadiness: readyStub{},
|
||||
coordinator: coordinatorFunc(func(ctx context.Context, upstreamID string) error {
|
||||
started <- upstreamID
|
||||
<-ctx.Done()
|
||||
stopped <- upstreamID
|
||||
return ctx.Err()
|
||||
}),
|
||||
credentials: credentialStore,
|
||||
close: func() error { return nil },
|
||||
}}
|
||||
wantErr := errors.New("test HTTP runtime stopped")
|
||||
factory := runtimeFactoryFunc(func(
|
||||
_ *config.Config,
|
||||
dependencies controllerRuntime.Dependencies,
|
||||
_ controllerRuntime.Options,
|
||||
) (controllerRunner, error) {
|
||||
return runnerFunc(func(ctx context.Context) error {
|
||||
for {
|
||||
select {
|
||||
case upstreamID := <-started:
|
||||
if upstreamID != "provider-a" {
|
||||
continue
|
||||
}
|
||||
if _, err := dependencies.AdminService.SetUpstreamEnabled(ctx, admin.SetUpstreamCommand{
|
||||
RequestID: "req-disable", ActorID: "admin:test", Name: "provider-a", Enabled: false,
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
for {
|
||||
select {
|
||||
case stoppedID := <-stopped:
|
||||
if stoppedID == "provider-a" {
|
||||
return wantErr
|
||||
}
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
}
|
||||
}
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
}
|
||||
}
|
||||
}), nil
|
||||
})
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
|
||||
defer cancel()
|
||||
err = run(ctx, Options{
|
||||
ConfigPath: "controller.yaml",
|
||||
Resolver: &memoryResolver{files: map[string][]byte{"controller.yaml": []byte(bootstrapTestConfig)}},
|
||||
Now: time.Now,
|
||||
FingerprintKey: bootstrapTestFingerprintKey,
|
||||
}, infrastructure, factory)
|
||||
if !errors.Is(err, wantErr) {
|
||||
t.Fatalf("run() error = %v, want %v", err, wantErr)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRetainProviderStatsKeepsAllConfiguredProviders(t *testing.T) {
|
||||
configuration, err := config.Load(strings.NewReader(bootstrapTestConfig))
|
||||
if err != nil {
|
||||
t.Fatalf("config.Load(): %v", err)
|
||||
}
|
||||
disabled := configuration.Upstreams["provider-b"]
|
||||
disabled.Enabled = false
|
||||
configuration.Upstreams["provider-b"] = disabled
|
||||
stats, err := provider.NewStatsRecorder(2)
|
||||
if err != nil {
|
||||
t.Fatalf("provider.NewStatsRecorder(): %v", err)
|
||||
}
|
||||
stats.Record(provider.Result{UpstreamID: "removed", Class: upstream.FetchError})
|
||||
stats.Record(provider.Result{UpstreamID: "provider-b", Class: upstream.FetchError})
|
||||
|
||||
retainProviderStats(configuration, stats)
|
||||
stats.Record(provider.Result{UpstreamID: "provider-a", Class: upstream.FetchError})
|
||||
got := stats.ReadProviderStats([]string{"removed", "provider-a", "provider-b"})
|
||||
if got[0].FetchErrorCount != 0 || got[1].FetchErrorCount != 1 || got[2].FetchErrorCount != 1 {
|
||||
t.Fatalf("Provider stats after retention = %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
type memoryResolver struct {
|
||||
files map[string][]byte
|
||||
reads int
|
||||
@ -127,6 +284,24 @@ type runnerStub struct{ err error }
|
||||
|
||||
func (runner runnerStub) Run(context.Context) error { return runner.err }
|
||||
|
||||
type runnerFunc func(context.Context) error
|
||||
|
||||
func (run runnerFunc) Run(ctx context.Context) error { return run(ctx) }
|
||||
|
||||
type runtimeFactoryFunc func(
|
||||
*config.Config,
|
||||
controllerRuntime.Dependencies,
|
||||
controllerRuntime.Options,
|
||||
) (controllerRunner, error)
|
||||
|
||||
func (factory runtimeFactoryFunc) New(
|
||||
configuration *config.Config,
|
||||
dependencies controllerRuntime.Dependencies,
|
||||
options controllerRuntime.Options,
|
||||
) (controllerRunner, error) {
|
||||
return factory(configuration, dependencies, options)
|
||||
}
|
||||
|
||||
type readyStub struct{}
|
||||
|
||||
func (readyStub) Ready(context.Context) error { return nil }
|
||||
@ -149,6 +324,45 @@ func (*stubActivityStore) ReadStateInventory(
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func (*stubActivityStore) UpsertFetched(
|
||||
_ context.Context,
|
||||
_ string,
|
||||
batch activitypool.FetchedBatch,
|
||||
) (activitypool.UpsertResult, error) {
|
||||
return activitypool.UpsertResult{Accepted: len(batch.Proxies), Inserted: len(batch.Proxies)}, nil
|
||||
}
|
||||
|
||||
func (*stubActivityStore) ReadInventory(
|
||||
context.Context,
|
||||
string,
|
||||
time.Duration,
|
||||
) (pool.InventorySnapshot, error) {
|
||||
return pool.InventorySnapshot{Managed: 100, AvailableSlots: 1_000}, nil
|
||||
}
|
||||
|
||||
type coordinatorStub struct{}
|
||||
|
||||
func (coordinatorStub) RunLeader(
|
||||
ctx context.Context,
|
||||
_ string,
|
||||
_ provider.CoordinationLimits,
|
||||
_ func(context.Context, provider.LeaderSession) error,
|
||||
) error {
|
||||
<-ctx.Done()
|
||||
return ctx.Err()
|
||||
}
|
||||
|
||||
type coordinatorFunc func(context.Context, string) error
|
||||
|
||||
func (run coordinatorFunc) RunLeader(
|
||||
ctx context.Context,
|
||||
upstreamID string,
|
||||
_ provider.CoordinationLimits,
|
||||
_ func(context.Context, provider.LeaderSession) error,
|
||||
) error {
|
||||
return run(ctx, upstreamID)
|
||||
}
|
||||
|
||||
const bootstrapTestConfig = `
|
||||
version: 1
|
||||
security:
|
||||
|
||||
@ -2,6 +2,8 @@ package bootstrap
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"strings"
|
||||
"time"
|
||||
@ -11,16 +13,23 @@ import (
|
||||
|
||||
"proxy-pool/internal/adapters/postgresadmin"
|
||||
"proxy-pool/internal/adapters/redisactivity"
|
||||
"proxy-pool/internal/adapters/redisprovider"
|
||||
"proxy-pool/internal/config"
|
||||
controllerProvider "proxy-pool/internal/controller/provider"
|
||||
"proxy-pool/internal/platform/credentials"
|
||||
platformMetrics "proxy-pool/internal/platform/metrics"
|
||||
)
|
||||
|
||||
const (
|
||||
redisNamespace = "controller"
|
||||
redisOperationTTL = 30 * time.Second
|
||||
redisMinimumScan = 4_096
|
||||
redisCleanupLimit = 1_024
|
||||
redisNamespace = "controller"
|
||||
redisOperationTTL = 30 * time.Second
|
||||
redisMinimumScan = 4_096
|
||||
redisMaximumScan = config.MaximumPoolSize
|
||||
redisCleanupLimit = 1_024
|
||||
providerLeaseTTL = 15 * time.Second
|
||||
providerRenewEvery = 3 * time.Second
|
||||
providerRetryInterval = 100 * time.Millisecond
|
||||
providerPermitGrace = 5 * time.Second
|
||||
)
|
||||
|
||||
var (
|
||||
@ -30,15 +39,22 @@ var (
|
||||
ErrRedisUnavailable = errors.New("Redis unavailable")
|
||||
)
|
||||
|
||||
type productionInfrastructure struct{}
|
||||
type productionInfrastructure struct {
|
||||
holderID string
|
||||
namespace string
|
||||
}
|
||||
|
||||
func (*productionInfrastructure) Open(
|
||||
func (infrastructure *productionInfrastructure) Open(
|
||||
ctx context.Context,
|
||||
configuration *config.Config,
|
||||
) (_ ports, resultErr error) {
|
||||
if ctx == nil || configuration == nil {
|
||||
return ports{}, ErrInvalidOptions
|
||||
}
|
||||
namespace, err := resolveRedisNamespace(infrastructure.namespace)
|
||||
if err != nil {
|
||||
return ports{}, err
|
||||
}
|
||||
var postgresPool *pgxpool.Pool
|
||||
var redisClient *redis.Client
|
||||
closeResources := func() error {
|
||||
@ -83,7 +99,8 @@ func (*productionInfrastructure) Open(
|
||||
}
|
||||
}
|
||||
|
||||
if configuration.Distribution.Enabled || configuration.Admin.Enabled {
|
||||
providersEnabled := hasEnabledUpstream(configuration)
|
||||
if configuration.Distribution.Enabled || configuration.Admin.Enabled || providersEnabled {
|
||||
if strings.TrimSpace(configuration.Storage.RedisURL) == "" {
|
||||
return ports{}, ErrRedisConfiguration
|
||||
}
|
||||
@ -95,22 +112,44 @@ func (*productionInfrastructure) Open(
|
||||
if err = redisClient.Ping(ctx).Err(); err != nil {
|
||||
return ports{}, contextOr(ctx, ErrRedisUnavailable)
|
||||
}
|
||||
credentialStore, err := credentials.NewMemoryStore(credentialCapacity(configuration))
|
||||
credentialStore, err := credentials.NewMemoryStore(providerCredentialCapacity(configuration))
|
||||
if err != nil {
|
||||
return ports{}, err
|
||||
}
|
||||
adapter, err := redisactivity.New(redisClient, redisactivity.Options{
|
||||
Namespace: redisNamespace,
|
||||
Credentials: credentialStore,
|
||||
OperationTTL: redisOperationTTL,
|
||||
MaxCandidateScan: candidateScan(configuration),
|
||||
CleanupLimit: redisCleanupLimit,
|
||||
Namespace: namespace,
|
||||
Credentials: credentialStore,
|
||||
OperationTTL: redisOperationTTL,
|
||||
MaxCandidateScan: candidateScan(configuration),
|
||||
MaxRuntimeCounters: credentialCapacity(configuration),
|
||||
MaxInventoryScan: maxInventoryScan(configuration),
|
||||
CleanupLimit: redisCleanupLimit,
|
||||
})
|
||||
if err != nil {
|
||||
return ports{}, err
|
||||
}
|
||||
opened.activity = adapter
|
||||
opened.readiness = redisReadiness{client: redisClient}
|
||||
opened.credentials = credentialStore
|
||||
if providersEnabled {
|
||||
stats, statsErr := controllerProvider.NewStatsRecorder(config.MaximumUpstreams)
|
||||
if statsErr != nil {
|
||||
return ports{}, statsErr
|
||||
}
|
||||
opened.providerResults = stats
|
||||
holderID, holderErr := resolveHolderID(infrastructure.holderID)
|
||||
if holderErr != nil {
|
||||
return ports{}, holderErr
|
||||
}
|
||||
opened.coordinator, err = redisprovider.New(redisClient, redisprovider.Options{
|
||||
Namespace: namespace, HolderID: holderID,
|
||||
LeaseTTL: providerLeaseTTL, RenewEvery: providerRenewEvery,
|
||||
RetryInterval: providerRetryInterval, PermitGrace: providerPermitGrace,
|
||||
})
|
||||
if err != nil {
|
||||
return ports{}, err
|
||||
}
|
||||
}
|
||||
}
|
||||
if configuration.Metrics.Enabled {
|
||||
opened.metricsReadiness = selectMetricsReadiness(
|
||||
@ -122,11 +161,21 @@ func (*productionInfrastructure) Open(
|
||||
return opened, nil
|
||||
}
|
||||
|
||||
func resolveRedisNamespace(configured string) (string, error) {
|
||||
if strings.TrimSpace(configured) != configured {
|
||||
return "", ErrInvalidOptions
|
||||
}
|
||||
if configured == "" {
|
||||
return redisNamespace, nil
|
||||
}
|
||||
return configured, nil
|
||||
}
|
||||
|
||||
func selectMetricsReadiness(
|
||||
configuration *config.Config,
|
||||
admin, activity platformMetrics.ReadinessChecker,
|
||||
) platformMetrics.ReadinessChecker {
|
||||
if configuration.Distribution.Enabled {
|
||||
if configuration.Distribution.Enabled || hasEnabledUpstream(configuration) {
|
||||
return activity
|
||||
}
|
||||
if configuration.Admin.Enabled {
|
||||
@ -184,7 +233,7 @@ func credentialCapacity(configuration *config.Config) int {
|
||||
capacity := 0
|
||||
maximum := int(^uint(0) >> 1)
|
||||
for _, upstream := range configuration.Upstreams {
|
||||
if upstream.Pool.MaxSize <= 0 {
|
||||
if !upstream.Enabled || upstream.Pool.MaxSize <= 0 {
|
||||
continue
|
||||
}
|
||||
if capacity > maximum-upstream.Pool.MaxSize {
|
||||
@ -198,6 +247,62 @@ func credentialCapacity(configuration *config.Config) int {
|
||||
return capacity
|
||||
}
|
||||
|
||||
func providerCredentialCapacity(configuration *config.Config) int {
|
||||
capacity := 0
|
||||
maximum := int(^uint(0) >> 1)
|
||||
for _, upstream := range configuration.Upstreams {
|
||||
if upstream.Pool.MaxSize <= 0 {
|
||||
continue
|
||||
}
|
||||
maxInFlight := upstream.Fetch.MaxInFlight
|
||||
if maxInFlight <= 0 {
|
||||
maxInFlight = 1
|
||||
}
|
||||
if upstream.Pool.MaxSize > maximum/maxInFlight {
|
||||
return maximum
|
||||
}
|
||||
leases := upstream.Pool.MaxSize * maxInFlight
|
||||
if capacity > maximum-leases {
|
||||
return maximum
|
||||
}
|
||||
capacity += leases
|
||||
}
|
||||
if capacity == 0 {
|
||||
return 1
|
||||
}
|
||||
return capacity
|
||||
}
|
||||
|
||||
func maxInventoryScan(_ *config.Config) int {
|
||||
return config.MaximumPoolSize
|
||||
}
|
||||
|
||||
func hasEnabledUpstream(configuration *config.Config) bool {
|
||||
if configuration == nil {
|
||||
return false
|
||||
}
|
||||
for _, upstream := range configuration.Upstreams {
|
||||
if upstream.Enabled {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func resolveHolderID(configured string) (string, error) {
|
||||
if strings.TrimSpace(configured) != configured {
|
||||
return "", ErrInvalidOptions
|
||||
}
|
||||
if configured != "" {
|
||||
return configured, nil
|
||||
}
|
||||
var entropy [16]byte
|
||||
if _, err := rand.Read(entropy[:]); err != nil {
|
||||
return "", errors.Join(ErrStartup, err)
|
||||
}
|
||||
return "controller-" + hex.EncodeToString(entropy[:]), nil
|
||||
}
|
||||
|
||||
func candidateScan(configuration *config.Config) int {
|
||||
configured := configuration.Distribution.Extraction
|
||||
if configured.MaxCountPerRequest > int(^uint(0)>>1)-configured.ReserveForGateway {
|
||||
|
||||
@ -79,13 +79,17 @@ func TestProductionRedisSizingUsesConfigurationBounds(t *testing.T) {
|
||||
MaxCountPerRequest: 100, ReserveForGateway: 5_000,
|
||||
}},
|
||||
Upstreams: map[string]config.Upstream{
|
||||
"provider-a": {Pool: config.Pool{MaxSize: 3_000}},
|
||||
"provider-b": {Pool: config.Pool{MaxSize: 2_000}},
|
||||
"provider-a": {Enabled: true, Pool: config.Pool{MaxSize: 3_000}, Fetch: config.Fetch{MaxInFlight: 2}},
|
||||
"provider-b": {Enabled: true, Pool: config.Pool{MaxSize: 2_000}, Fetch: config.Fetch{MaxInFlight: 1}},
|
||||
"disabled": {Pool: config.Pool{MaxSize: 50_000}, Fetch: config.Fetch{MaxInFlight: 3}},
|
||||
},
|
||||
}
|
||||
if got := credentialCapacity(configuration); got != 5_000 {
|
||||
t.Fatalf("credentialCapacity() = %d, want 5000", got)
|
||||
}
|
||||
if got := providerCredentialCapacity(configuration); got != 158_000 {
|
||||
t.Fatalf("providerCredentialCapacity() = %d, want 158000", got)
|
||||
}
|
||||
if got := candidateScan(configuration); got != 5_100 {
|
||||
t.Fatalf("candidateScan() = %d, want 5100", got)
|
||||
}
|
||||
@ -94,3 +98,14 @@ func TestProductionRedisSizingUsesConfigurationBounds(t *testing.T) {
|
||||
t.Fatalf("candidateScan(minimum) = %d, want %d", got, redisMinimumScan)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMaxInventoryScanSupportsPoolGrowthAfterReload(t *testing.T) {
|
||||
t.Parallel()
|
||||
configuration := &config.Config{Upstreams: map[string]config.Upstream{
|
||||
"provider-a": {Enabled: true, Pool: config.Pool{MaxSize: 100}},
|
||||
}}
|
||||
|
||||
if got := maxInventoryScan(configuration); got != config.MaximumPoolSize {
|
||||
t.Fatalf("maxInventoryScan(initial small pool) = %d, want %d", got, config.MaximumPoolSize)
|
||||
}
|
||||
}
|
||||
|
||||
126
internal/controller/bootstrap/provider.go
Normal file
126
internal/controller/bootstrap/provider.go
Normal file
@ -0,0 +1,126 @@
|
||||
package bootstrap
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"math"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"proxy-pool/internal/adapters/providerapi"
|
||||
"proxy-pool/internal/config"
|
||||
"proxy-pool/internal/controller/pool"
|
||||
controllerProvider "proxy-pool/internal/controller/provider"
|
||||
)
|
||||
|
||||
var ErrProviderRuntime = errors.New("invalid Provider runtime configuration")
|
||||
|
||||
func newProviderFleet(configuration *config.Config, opened ports) (*controllerProvider.Fleet, error) {
|
||||
if configuration == nil {
|
||||
return nil, ErrProviderRuntime
|
||||
}
|
||||
names := make([]string, 0, len(configuration.Upstreams))
|
||||
for name, upstream := range configuration.Upstreams {
|
||||
if upstream.Enabled {
|
||||
names = append(names, name)
|
||||
}
|
||||
}
|
||||
if len(names) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
builder, err := providerRuntimeBuilder(opened)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
sort.Strings(names)
|
||||
runtimes := make([]*controllerProvider.UpstreamRuntime, 0, len(names))
|
||||
for _, name := range names {
|
||||
runtime, err := builder(name, configuration.Upstreams[name])
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
runtimes = append(runtimes, runtime)
|
||||
}
|
||||
return controllerProvider.NewFleet(runtimes...)
|
||||
}
|
||||
|
||||
type buildUpstreamRuntime func(string, config.Upstream) (*controllerProvider.UpstreamRuntime, error)
|
||||
|
||||
func providerRuntimeBuilder(opened ports) (buildUpstreamRuntime, error) {
|
||||
if nilInterface(opened.coordinator) || nilInterface(opened.activity) || nilInterface(opened.credentials) {
|
||||
return nil, ErrProviderRuntime
|
||||
}
|
||||
results := opened.providerResults
|
||||
if nilInterface(results) {
|
||||
results = discardProviderResults{}
|
||||
}
|
||||
return func(name string, upstream config.Upstream) (*controllerProvider.UpstreamRuntime, error) {
|
||||
upstream.Enabled = true
|
||||
mapped, err := providerRuntimeConfig(name, upstream)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
adapter, err := providerapi.NewHTTPAdapter(upstream.API, upstream.Fetch, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
parser, err := providerapi.NewTemplateParser(name, upstream, opened.credentials)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return controllerProvider.NewUpstreamRuntime(mapped, controllerProvider.UpstreamRuntimeDependencies{
|
||||
Coordinator: opened.coordinator,
|
||||
Inventory: opened.activity,
|
||||
Adapter: adapter,
|
||||
Parser: parser,
|
||||
Activity: opened.activity,
|
||||
Results: results,
|
||||
})
|
||||
}, nil
|
||||
}
|
||||
|
||||
func providerRuntimeConfig(
|
||||
upstreamID string,
|
||||
upstream config.Upstream,
|
||||
) (controllerProvider.UpstreamRuntimeConfig, error) {
|
||||
if strings.TrimSpace(upstreamID) != upstreamID || upstreamID == "" || !upstream.Enabled ||
|
||||
upstream.Fetch.EstimatedIPsPerCall <= 0 || upstream.Capacity.MaxConcurrencyPerProxy <= 0 ||
|
||||
upstream.Pool.MaxSize > config.MaximumPoolSize ||
|
||||
int64(upstream.Fetch.EstimatedIPsPerCall) > controllerProvider.MaximumCoordinationInteger ||
|
||||
int64(upstream.Fetch.MaxInFlight) > controllerProvider.MaximumCoordinationInteger ||
|
||||
int64(upstream.Fetch.MaxTotal) > controllerProvider.MaximumCoordinationInteger ||
|
||||
int64(upstream.Fetch.EstimatedIPsPerCall) > math.MaxInt64/int64(upstream.Capacity.MaxConcurrencyPerProxy) {
|
||||
return controllerProvider.UpstreamRuntimeConfig{}, ErrProviderRuntime
|
||||
}
|
||||
expectedSlots := int64(upstream.Fetch.EstimatedIPsPerCall) * int64(upstream.Capacity.MaxConcurrencyPerProxy)
|
||||
return controllerProvider.UpstreamRuntimeConfig{
|
||||
Provider: controllerProvider.Config{
|
||||
UpstreamID: upstreamID,
|
||||
RequestInterval: time.Duration(upstream.Fetch.RequestInterval),
|
||||
Timeout: time.Duration(upstream.Fetch.Timeout),
|
||||
MaxAttempts: upstream.Fetch.MaxAttempts,
|
||||
MaxInFlight: upstream.Fetch.MaxInFlight,
|
||||
MaxTotal: int64(upstream.Fetch.MaxTotal),
|
||||
MaxSize: upstream.Pool.MaxSize,
|
||||
TTL: time.Duration(upstream.Lifecycle.TTL),
|
||||
AllocationSafetyMargin: time.Duration(upstream.Lifecycle.AllocationSafetyMargin),
|
||||
Retry: controllerProvider.RetryConfig{
|
||||
Initial: time.Duration(upstream.Fetch.Retry.Initial),
|
||||
Max: time.Duration(upstream.Fetch.Retry.Max),
|
||||
Jitter: upstream.Fetch.Retry.Jitter,
|
||||
},
|
||||
},
|
||||
ReconcilePolicy: pool.ReconcilePolicy{
|
||||
MinimumAvailableSlots: upstream.Refill.MinimumAvailableSlots,
|
||||
TargetAvailableSlots: upstream.Refill.TargetAvailableSlots,
|
||||
ExpectedPerFetch: upstream.Fetch.EstimatedIPsPerCall,
|
||||
ExpectedSlotsPerFetch: expectedSlots,
|
||||
SafetyMargin: time.Duration(upstream.Lifecycle.AllocationSafetyMargin),
|
||||
},
|
||||
ReconcileInterval: time.Duration(upstream.Refill.ReconcileInterval),
|
||||
}, nil
|
||||
}
|
||||
|
||||
type discardProviderResults struct{}
|
||||
|
||||
func (discardProviderResults) Record(controllerProvider.Result) {}
|
||||
375
internal/controller/bootstrap/provider_supervisor.go
Normal file
375
internal/controller/bootstrap/provider_supervisor.go
Normal file
@ -0,0 +1,375 @@
|
||||
package bootstrap
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"reflect"
|
||||
"sort"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"proxy-pool/internal/config"
|
||||
"proxy-pool/internal/controller/admin"
|
||||
"proxy-pool/internal/domain/adminstate"
|
||||
"proxy-pool/internal/platform/lifecycle"
|
||||
)
|
||||
|
||||
const providerSupervisorInterval = time.Second
|
||||
|
||||
var (
|
||||
ErrProviderSupervisor = errors.New("invalid Provider supervisor")
|
||||
errProviderManagementStateUnavailable = errors.New("Provider management state unavailable")
|
||||
errProviderManagementSnapshotStale = errors.New("Provider management snapshot stale")
|
||||
errProviderConfigurationPending = errors.New("Provider configuration synchronization pending")
|
||||
)
|
||||
|
||||
type providerConfigurationStore interface {
|
||||
Current() *config.Config
|
||||
Revision() uint64
|
||||
PublishRevision(*config.Config, uint64) bool
|
||||
}
|
||||
|
||||
type providerConfigurationSource interface {
|
||||
LoadConfiguration(context.Context) (admin.LoadedConfiguration, error)
|
||||
}
|
||||
|
||||
type providerStateReader interface {
|
||||
Snapshot(context.Context) (adminstate.Snapshot, error)
|
||||
}
|
||||
|
||||
type providerRunnerBuilder func(string, config.Upstream) (lifecycle.Runner, error)
|
||||
type providerConfigurationPreparer func(context.Context, *config.Config) error
|
||||
type providerConfigurationObserver func(*config.Config)
|
||||
|
||||
type providerSupervisor struct {
|
||||
configuration providerConfigurationStore
|
||||
state providerStateReader
|
||||
build providerRunnerBuilder
|
||||
prepare providerConfigurationPreparer
|
||||
observe providerConfigurationObserver
|
||||
source providerConfigurationSource
|
||||
fingerprintKey []byte
|
||||
interval time.Duration
|
||||
notify chan struct{}
|
||||
}
|
||||
|
||||
type runningProvider struct {
|
||||
configuration config.Upstream
|
||||
cancel context.CancelFunc
|
||||
done chan struct{}
|
||||
}
|
||||
|
||||
func newProviderSupervisor(
|
||||
configuration providerConfigurationStore,
|
||||
state providerStateReader,
|
||||
build providerRunnerBuilder,
|
||||
prepare providerConfigurationPreparer,
|
||||
observe providerConfigurationObserver,
|
||||
source providerConfigurationSource,
|
||||
fingerprintKey []byte,
|
||||
interval time.Duration,
|
||||
) (*providerSupervisor, error) {
|
||||
if nilInterface(configuration) || build == nil || interval <= 0 ||
|
||||
(!nilInterface(state) && len(fingerprintKey) < config.MinimumFingerprintKeyBytes) {
|
||||
return nil, ErrProviderSupervisor
|
||||
}
|
||||
return &providerSupervisor{
|
||||
configuration: configuration,
|
||||
state: state,
|
||||
build: build,
|
||||
prepare: prepare,
|
||||
observe: observe,
|
||||
source: source,
|
||||
fingerprintKey: append([]byte(nil), fingerprintKey...),
|
||||
interval: interval,
|
||||
notify: make(chan struct{}, 1),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (supervisor *providerSupervisor) Notify() {
|
||||
if supervisor == nil || supervisor.notify == nil {
|
||||
return
|
||||
}
|
||||
select {
|
||||
case supervisor.notify <- struct{}{}:
|
||||
default:
|
||||
}
|
||||
}
|
||||
|
||||
func (supervisor *providerSupervisor) ValidateConfiguration(ctx context.Context, configuration *config.Config) error {
|
||||
if supervisor == nil || ctx == nil || configuration == nil || supervisor.build == nil {
|
||||
return ErrProviderSupervisor
|
||||
}
|
||||
if err := supervisor.validateConfiguration(ctx, configuration); err != nil {
|
||||
return errors.Join(ErrProviderSupervisor, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (supervisor *providerSupervisor) validateConfiguration(ctx context.Context, configuration *config.Config) error {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := config.Validate(configuration); err != nil {
|
||||
return err
|
||||
}
|
||||
if supervisor.prepare != nil {
|
||||
if err := supervisor.prepare(ctx, configuration); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
names := make([]string, 0, len(configuration.Upstreams))
|
||||
for name, upstream := range configuration.Upstreams {
|
||||
if upstream.Enabled {
|
||||
names = append(names, name)
|
||||
}
|
||||
}
|
||||
sort.Strings(names)
|
||||
for _, name := range names {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := supervisor.build(name, configuration.Upstreams[name]); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (supervisor *providerSupervisor) ValidateUpstream(ctx context.Context, name string) error {
|
||||
if supervisor == nil || ctx == nil || name == "" || supervisor.build == nil {
|
||||
return ErrProviderSupervisor
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
configuration := supervisor.configuration.Current()
|
||||
if configuration == nil {
|
||||
return ErrProviderSupervisor
|
||||
}
|
||||
if supervisor.prepare != nil {
|
||||
if err := supervisor.prepare(ctx, configuration); err != nil {
|
||||
return errors.Join(ErrProviderSupervisor, err)
|
||||
}
|
||||
}
|
||||
upstream, exists := configuration.Upstreams[name]
|
||||
if !exists {
|
||||
return ErrProviderSupervisor
|
||||
}
|
||||
upstream.Enabled = true
|
||||
if _, err := supervisor.build(name, upstream); err != nil {
|
||||
return errors.Join(ErrProviderSupervisor, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (supervisor *providerSupervisor) Run(ctx context.Context) error {
|
||||
if supervisor == nil || ctx == nil || nilInterface(supervisor.configuration) ||
|
||||
supervisor.build == nil || supervisor.interval <= 0 || supervisor.notify == nil {
|
||||
return ErrProviderSupervisor
|
||||
}
|
||||
active := make(map[string]*runningProvider)
|
||||
failures := make(chan error, 1)
|
||||
defer stopAllProviders(active)
|
||||
ticker := time.NewTicker(supervisor.interval)
|
||||
defer ticker.Stop()
|
||||
|
||||
for {
|
||||
if err := supervisor.reconcile(ctx, active, failures); err != nil {
|
||||
return err
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case err := <-failures:
|
||||
return err
|
||||
case <-supervisor.notify:
|
||||
case <-ticker.C:
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (supervisor *providerSupervisor) reconcile(
|
||||
ctx context.Context,
|
||||
active map[string]*runningProvider,
|
||||
failures chan<- error,
|
||||
) error {
|
||||
desired, err := supervisor.desired(ctx)
|
||||
if err != nil {
|
||||
if errors.Is(err, errProviderManagementStateUnavailable) ||
|
||||
errors.Is(err, errProviderManagementSnapshotStale) {
|
||||
return nil
|
||||
}
|
||||
if errors.Is(err, errProviderConfigurationPending) {
|
||||
stopAllProviders(active)
|
||||
clear(active)
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
names := make([]string, 0, len(desired))
|
||||
prepared := make(map[string]lifecycle.Runner)
|
||||
for name, upstream := range desired {
|
||||
names = append(names, name)
|
||||
current := active[name]
|
||||
if current != nil && reflect.DeepEqual(current.configuration, upstream) {
|
||||
continue
|
||||
}
|
||||
runner, buildErr := supervisor.build(name, upstream)
|
||||
if buildErr != nil || nilInterface(runner) {
|
||||
return errors.Join(ErrProviderSupervisor, buildErr)
|
||||
}
|
||||
prepared[name] = runner
|
||||
}
|
||||
sort.Strings(names)
|
||||
|
||||
for name, current := range active {
|
||||
if _, keep := desired[name]; !keep {
|
||||
stopProvider(current)
|
||||
delete(active, name)
|
||||
}
|
||||
}
|
||||
for _, name := range names {
|
||||
runner := prepared[name]
|
||||
if runner == nil {
|
||||
continue
|
||||
}
|
||||
if current := active[name]; current != nil {
|
||||
stopProvider(current)
|
||||
}
|
||||
active[name] = startProvider(ctx, name, desired[name], runner, failures)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (supervisor *providerSupervisor) desired(ctx context.Context) (map[string]config.Upstream, error) {
|
||||
configuration := supervisor.configuration.Current()
|
||||
if configuration == nil {
|
||||
return nil, ErrProviderSupervisor
|
||||
}
|
||||
enabled := make(map[string]bool, len(configuration.Upstreams))
|
||||
var snapshot adminstate.Snapshot
|
||||
if !nilInterface(supervisor.state) {
|
||||
var err error
|
||||
snapshot, err = supervisor.state.Snapshot(ctx)
|
||||
if err != nil {
|
||||
return nil, errors.Join(errProviderManagementStateUnavailable, err)
|
||||
}
|
||||
configuration, err = supervisor.synchronizeConfiguration(ctx, configuration, snapshot)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, upstream := range snapshot.Upstreams {
|
||||
enabled[upstream.Name] = upstream.Enabled
|
||||
}
|
||||
}
|
||||
if supervisor.prepare != nil {
|
||||
if err := supervisor.prepare(ctx, configuration); err != nil {
|
||||
return nil, errors.Join(ErrProviderSupervisor, err)
|
||||
}
|
||||
}
|
||||
if supervisor.observe != nil {
|
||||
supervisor.observe(configuration)
|
||||
}
|
||||
desired := make(map[string]config.Upstream)
|
||||
for name, upstream := range configuration.Upstreams {
|
||||
isEnabled := upstream.Enabled
|
||||
if !nilInterface(supervisor.state) {
|
||||
isEnabled = enabled[name]
|
||||
}
|
||||
if isEnabled {
|
||||
upstream.Enabled = true
|
||||
desired[name] = upstream
|
||||
}
|
||||
}
|
||||
return desired, nil
|
||||
}
|
||||
|
||||
func (supervisor *providerSupervisor) synchronizeConfiguration(
|
||||
ctx context.Context,
|
||||
current *config.Config,
|
||||
snapshot adminstate.Snapshot,
|
||||
) (*config.Config, error) {
|
||||
if snapshot.Config == nil || snapshot.Config.Checksum == "" {
|
||||
return current, nil
|
||||
}
|
||||
if supervisor.configuration.Revision() > snapshot.Config.Revision {
|
||||
return nil, errProviderManagementSnapshotStale
|
||||
}
|
||||
checksum, err := config.Fingerprint(current, supervisor.fingerprintKey)
|
||||
if err != nil {
|
||||
return nil, errors.Join(ErrProviderSupervisor, err)
|
||||
}
|
||||
if checksum == snapshot.Config.Checksum {
|
||||
supervisor.configuration.PublishRevision(current, snapshot.Config.Revision)
|
||||
return current, nil
|
||||
}
|
||||
if nilInterface(supervisor.source) {
|
||||
return nil, errProviderConfigurationPending
|
||||
}
|
||||
loaded, err := supervisor.source.LoadConfiguration(ctx)
|
||||
if err != nil || loaded.Value == nil {
|
||||
return nil, errors.Join(errProviderConfigurationPending, err)
|
||||
}
|
||||
checksum, err = config.Fingerprint(loaded.Value, supervisor.fingerprintKey)
|
||||
if err != nil || checksum != snapshot.Config.Checksum {
|
||||
return nil, errors.Join(errProviderConfigurationPending, err)
|
||||
}
|
||||
if err := supervisor.validateConfiguration(ctx, loaded.Value); err != nil {
|
||||
return nil, errors.Join(errProviderConfigurationPending, err)
|
||||
}
|
||||
if supervisor.configuration.PublishRevision(loaded.Value, snapshot.Config.Revision) {
|
||||
return loaded.Value, nil
|
||||
}
|
||||
if supervisor.configuration.Revision() > snapshot.Config.Revision {
|
||||
return nil, errProviderManagementSnapshotStale
|
||||
}
|
||||
return nil, errProviderConfigurationPending
|
||||
}
|
||||
|
||||
func startProvider(
|
||||
ctx context.Context,
|
||||
name string,
|
||||
configuration config.Upstream,
|
||||
runner lifecycle.Runner,
|
||||
failures chan<- error,
|
||||
) *runningProvider {
|
||||
runCtx, cancel := context.WithCancel(ctx)
|
||||
running := &runningProvider{configuration: configuration, cancel: cancel, done: make(chan struct{})}
|
||||
go func() {
|
||||
defer close(running.done)
|
||||
err := runner.Run(runCtx)
|
||||
if runCtx.Err() != nil {
|
||||
return
|
||||
}
|
||||
if err == nil {
|
||||
err = lifecycle.ErrRunnerStopped
|
||||
}
|
||||
select {
|
||||
case failures <- fmt.Errorf("Provider %s runtime: %w", name, err):
|
||||
default:
|
||||
}
|
||||
}()
|
||||
return running
|
||||
}
|
||||
|
||||
func stopProvider(running *runningProvider) {
|
||||
if running == nil {
|
||||
return
|
||||
}
|
||||
running.cancel()
|
||||
<-running.done
|
||||
}
|
||||
|
||||
func stopAllProviders(active map[string]*runningProvider) {
|
||||
var wait sync.WaitGroup
|
||||
for _, running := range active {
|
||||
wait.Add(1)
|
||||
go func(current *runningProvider) {
|
||||
defer wait.Done()
|
||||
stopProvider(current)
|
||||
}(running)
|
||||
}
|
||||
wait.Wait()
|
||||
}
|
||||
376
internal/controller/bootstrap/provider_supervisor_test.go
Normal file
376
internal/controller/bootstrap/provider_supervisor_test.go
Normal file
@ -0,0 +1,376 @@
|
||||
package bootstrap
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"proxy-pool/internal/config"
|
||||
"proxy-pool/internal/controller/admin"
|
||||
"proxy-pool/internal/domain/adminstate"
|
||||
"proxy-pool/internal/platform/lifecycle"
|
||||
)
|
||||
|
||||
func TestProviderSupervisorAppliesDisableAndConfigurationReplacement(t *testing.T) {
|
||||
configuration, err := config.Load(strings.NewReader(bootstrapTestConfig))
|
||||
if err != nil {
|
||||
t.Fatalf("config.Load(): %v", err)
|
||||
}
|
||||
store, err := config.NewStore(configuration)
|
||||
if err != nil {
|
||||
t.Fatalf("config.NewStore(): %v", err)
|
||||
}
|
||||
state := &mutableProviderState{enabled: map[string]bool{"provider-a": true, "provider-b": true}}
|
||||
started := make(chan providerRuntimeEvent, 4)
|
||||
stopped := make(chan providerRuntimeEvent, 4)
|
||||
supervisor, err := newProviderSupervisor(store, state, func(name string, upstream config.Upstream) (lifecycle.Runner, error) {
|
||||
event := providerRuntimeEvent{name: name, url: upstream.API.URL}
|
||||
return supervisorRunnerFunc(func(ctx context.Context) error {
|
||||
started <- event
|
||||
<-ctx.Done()
|
||||
stopped <- event
|
||||
return ctx.Err()
|
||||
}), nil
|
||||
}, nil, nil, nil, bootstrapTestFingerprintKey, time.Hour)
|
||||
if err != nil {
|
||||
t.Fatalf("newProviderSupervisor(): %v", err)
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
done := make(chan error, 1)
|
||||
go func() { done <- supervisor.Run(ctx) }()
|
||||
waitForRuntimeEvents(t, started, 2)
|
||||
|
||||
state.set("provider-a", false)
|
||||
supervisor.Notify()
|
||||
if event := waitForRuntimeEvent(t, stopped); event.name != "provider-a" {
|
||||
t.Fatalf("stopped Provider = %s, want provider-a", event.name)
|
||||
}
|
||||
|
||||
updated := store.Current()
|
||||
providerB := updated.Upstreams["provider-b"]
|
||||
providerB.API.URL = "https://replacement.invalid/proxies"
|
||||
updated.Upstreams["provider-b"] = providerB
|
||||
if !store.PublishRevision(updated, 1) {
|
||||
t.Fatal("PublishRevision() rejected updated configuration")
|
||||
}
|
||||
supervisor.Notify()
|
||||
if event := waitForRuntimeEvent(t, stopped); event.name != "provider-b" {
|
||||
t.Fatalf("replaced Provider = %s, want provider-b", event.name)
|
||||
}
|
||||
if event := waitForRuntimeEvent(t, started); event.name != "provider-b" || event.url != providerB.API.URL {
|
||||
t.Fatalf("replacement Provider = %+v", event)
|
||||
}
|
||||
|
||||
cancel()
|
||||
if err := <-done; !errors.Is(err, context.Canceled) {
|
||||
t.Fatalf("Run() error = %v, want context cancellation", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProviderSupervisorPropagatesUnexpectedRuntimeFailure(t *testing.T) {
|
||||
configuration, err := config.Load(strings.NewReader(bootstrapTestConfig))
|
||||
if err != nil {
|
||||
t.Fatalf("config.Load(): %v", err)
|
||||
}
|
||||
store, err := config.NewStore(configuration)
|
||||
if err != nil {
|
||||
t.Fatalf("config.NewStore(): %v", err)
|
||||
}
|
||||
wantErr := errors.New("coordination stopped")
|
||||
supervisor, err := newProviderSupervisor(store, nil, func(string, config.Upstream) (lifecycle.Runner, error) {
|
||||
return supervisorRunnerFunc(func(context.Context) error { return wantErr }), nil
|
||||
}, nil, nil, nil, bootstrapTestFingerprintKey, time.Hour)
|
||||
if err != nil {
|
||||
t.Fatalf("newProviderSupervisor(): %v", err)
|
||||
}
|
||||
if err := supervisor.Run(context.Background()); !errors.Is(err, wantErr) {
|
||||
t.Fatalf("Run() error = %v, want %v", err, wantErr)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProviderSupervisorRetainsRuntimesWhileManagementStateIsUnavailable(t *testing.T) {
|
||||
configuration, err := config.Load(strings.NewReader(bootstrapTestConfig))
|
||||
if err != nil {
|
||||
t.Fatalf("config.Load(): %v", err)
|
||||
}
|
||||
store, err := config.NewStore(configuration)
|
||||
if err != nil {
|
||||
t.Fatalf("config.NewStore(): %v", err)
|
||||
}
|
||||
state := &mutableProviderState{enabled: map[string]bool{"provider-a": true, "provider-b": true}}
|
||||
started := make(chan providerRuntimeEvent, 2)
|
||||
stopped := make(chan providerRuntimeEvent, 2)
|
||||
supervisor, err := newProviderSupervisor(store, state, func(name string, upstream config.Upstream) (lifecycle.Runner, error) {
|
||||
event := providerRuntimeEvent{name: name, url: upstream.API.URL}
|
||||
return supervisorRunnerFunc(func(ctx context.Context) error {
|
||||
started <- event
|
||||
<-ctx.Done()
|
||||
stopped <- event
|
||||
return ctx.Err()
|
||||
}), nil
|
||||
}, nil, nil, nil, bootstrapTestFingerprintKey, time.Hour)
|
||||
if err != nil {
|
||||
t.Fatalf("newProviderSupervisor(): %v", err)
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
done := make(chan error, 1)
|
||||
go func() { done <- supervisor.Run(ctx) }()
|
||||
waitForRuntimeEvents(t, started, 2)
|
||||
|
||||
state.setError(errors.New("PostgreSQL temporarily unavailable"))
|
||||
supervisor.Notify()
|
||||
select {
|
||||
case err := <-done:
|
||||
t.Fatalf("Supervisor stopped during transient state failure: %v", err)
|
||||
case event := <-stopped:
|
||||
t.Fatalf("Provider stopped during transient state failure: %+v", event)
|
||||
case <-time.After(50 * time.Millisecond):
|
||||
}
|
||||
|
||||
state.setError(nil)
|
||||
state.set("provider-a", false)
|
||||
supervisor.Notify()
|
||||
if event := waitForRuntimeEvent(t, stopped); event.name != "provider-a" {
|
||||
t.Fatalf("stopped Provider = %s, want provider-a", event.name)
|
||||
}
|
||||
|
||||
cancel()
|
||||
if err := <-done; !errors.Is(err, context.Canceled) {
|
||||
t.Fatalf("Run() error = %v, want context cancellation", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProviderSupervisorStopsStaleRuntimesAndLoadsAuthoritativeConfiguration(t *testing.T) {
|
||||
initial, err := config.Load(strings.NewReader(bootstrapTestConfig))
|
||||
if err != nil {
|
||||
t.Fatalf("config.Load(): %v", err)
|
||||
}
|
||||
store, err := config.NewStore(initial)
|
||||
if err != nil {
|
||||
t.Fatalf("config.NewStore(): %v", err)
|
||||
}
|
||||
initialChecksum, err := config.Fingerprint(initial, bootstrapTestFingerprintKey)
|
||||
if err != nil {
|
||||
t.Fatalf("config.Fingerprint(initial): %v", err)
|
||||
}
|
||||
state := &mutableProviderState{
|
||||
enabled: map[string]bool{"provider-a": true, "provider-b": true},
|
||||
checksum: initialChecksum,
|
||||
revision: 1,
|
||||
}
|
||||
source := &mutableProviderConfigurationSource{configuration: initial}
|
||||
started := make(chan providerRuntimeEvent, 4)
|
||||
stopped := make(chan providerRuntimeEvent, 4)
|
||||
supervisor, err := newProviderSupervisor(store, state, func(name string, upstream config.Upstream) (lifecycle.Runner, error) {
|
||||
event := providerRuntimeEvent{name: name, url: upstream.API.URL}
|
||||
return supervisorRunnerFunc(func(ctx context.Context) error {
|
||||
started <- event
|
||||
<-ctx.Done()
|
||||
stopped <- event
|
||||
return ctx.Err()
|
||||
}), nil
|
||||
}, nil, nil, source, bootstrapTestFingerprintKey, time.Hour)
|
||||
if err != nil {
|
||||
t.Fatalf("newProviderSupervisor(): %v", err)
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
done := make(chan error, 1)
|
||||
go func() { done <- supervisor.Run(ctx) }()
|
||||
waitForRuntimeEvents(t, started, 2)
|
||||
|
||||
updated := store.Current()
|
||||
providerA := updated.Upstreams["provider-a"]
|
||||
providerA.API.URL = "https://replacement.invalid/proxies"
|
||||
updated.Upstreams["provider-a"] = providerA
|
||||
updatedChecksum, err := config.Fingerprint(updated, bootstrapTestFingerprintKey)
|
||||
if err != nil {
|
||||
t.Fatalf("config.Fingerprint(updated): %v", err)
|
||||
}
|
||||
state.setChecksum(updatedChecksum)
|
||||
supervisor.Notify()
|
||||
waitForRuntimeEvents(t, stopped, 2)
|
||||
select {
|
||||
case err := <-done:
|
||||
t.Fatalf("Supervisor stopped for stale local configuration: %v", err)
|
||||
case <-time.After(50 * time.Millisecond):
|
||||
}
|
||||
|
||||
source.set(updated)
|
||||
supervisor.Notify()
|
||||
events := []providerRuntimeEvent{waitForRuntimeEvent(t, started), waitForRuntimeEvent(t, started)}
|
||||
foundReplacement := false
|
||||
for _, event := range events {
|
||||
if event.name == "provider-a" && event.url == providerA.API.URL {
|
||||
foundReplacement = true
|
||||
}
|
||||
}
|
||||
if !foundReplacement {
|
||||
t.Fatalf("started Provider runtimes = %+v, replacement missing", events)
|
||||
}
|
||||
if currentChecksum, err := config.Fingerprint(store.Current(), bootstrapTestFingerprintKey); err != nil || currentChecksum != updatedChecksum {
|
||||
t.Fatalf("published checksum = %q, %v; want %q", currentChecksum, err, updatedChecksum)
|
||||
}
|
||||
|
||||
cancel()
|
||||
if err := <-done; !errors.Is(err, context.Canceled) {
|
||||
t.Fatalf("Run() error = %v, want context cancellation", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProviderSupervisorDoesNotPublishConfigurationOlderThanLocalRevision(t *testing.T) {
|
||||
initial, err := config.Load(strings.NewReader(bootstrapTestConfig))
|
||||
if err != nil {
|
||||
t.Fatalf("config.Load(): %v", err)
|
||||
}
|
||||
store, err := config.NewStore(initial)
|
||||
if err != nil {
|
||||
t.Fatalf("config.NewStore(): %v", err)
|
||||
}
|
||||
store.PublishRevision(initial, 1)
|
||||
|
||||
candidate := store.Current()
|
||||
providerA := candidate.Upstreams["provider-a"]
|
||||
providerA.API.URL = "https://candidate.invalid/proxies"
|
||||
candidate.Upstreams["provider-a"] = providerA
|
||||
candidateChecksum, err := config.Fingerprint(candidate, bootstrapTestFingerprintKey)
|
||||
if err != nil {
|
||||
t.Fatalf("config.Fingerprint(candidate): %v", err)
|
||||
}
|
||||
newest := store.Current()
|
||||
providerA = newest.Upstreams["provider-a"]
|
||||
providerA.API.URL = "https://newest.invalid/proxies"
|
||||
newest.Upstreams["provider-a"] = providerA
|
||||
|
||||
source := providerConfigurationSourceFunc(func(context.Context) (admin.LoadedConfiguration, error) {
|
||||
if !store.PublishRevision(newest, 3) {
|
||||
t.Fatal("failed to publish simulated concurrent revision")
|
||||
}
|
||||
return admin.LoadedConfiguration{Value: candidate, Source: "controller.yaml"}, nil
|
||||
})
|
||||
supervisor, err := newProviderSupervisor(
|
||||
store,
|
||||
&mutableProviderState{},
|
||||
func(string, config.Upstream) (lifecycle.Runner, error) {
|
||||
return supervisorRunnerFunc(func(context.Context) error { return nil }), nil
|
||||
},
|
||||
nil,
|
||||
nil,
|
||||
source,
|
||||
bootstrapTestFingerprintKey,
|
||||
time.Hour,
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("newProviderSupervisor(): %v", err)
|
||||
}
|
||||
|
||||
_, err = supervisor.synchronizeConfiguration(context.Background(), initial, adminstate.Snapshot{
|
||||
Config: &adminstate.ConfigRevision{Revision: 2, Checksum: candidateChecksum},
|
||||
})
|
||||
if !errors.Is(err, errProviderManagementSnapshotStale) {
|
||||
t.Fatalf("synchronizeConfiguration() error = %v, want stale snapshot", err)
|
||||
}
|
||||
if got := store.Revision(); got != 3 {
|
||||
t.Fatalf("configuration revision = %d, want 3", got)
|
||||
}
|
||||
}
|
||||
|
||||
type providerRuntimeEvent struct {
|
||||
name string
|
||||
url string
|
||||
}
|
||||
|
||||
type supervisorRunnerFunc func(context.Context) error
|
||||
|
||||
func (run supervisorRunnerFunc) Run(ctx context.Context) error { return run(ctx) }
|
||||
|
||||
type mutableProviderState struct {
|
||||
mu sync.Mutex
|
||||
enabled map[string]bool
|
||||
err error
|
||||
checksum string
|
||||
revision uint64
|
||||
}
|
||||
|
||||
func (state *mutableProviderState) Snapshot(context.Context) (adminstate.Snapshot, error) {
|
||||
state.mu.Lock()
|
||||
defer state.mu.Unlock()
|
||||
if state.err != nil {
|
||||
return adminstate.Snapshot{}, state.err
|
||||
}
|
||||
snapshot := adminstate.Snapshot{Upstreams: make([]adminstate.UpstreamState, 0, len(state.enabled))}
|
||||
if state.checksum != "" {
|
||||
snapshot.Config = &adminstate.ConfigRevision{
|
||||
Revision: state.revision, ConfigVersion: "cfg-" + state.checksum, Checksum: state.checksum,
|
||||
}
|
||||
}
|
||||
for name, enabled := range state.enabled {
|
||||
snapshot.Upstreams = append(snapshot.Upstreams, adminstate.UpstreamState{Name: name, Enabled: enabled})
|
||||
}
|
||||
return snapshot, nil
|
||||
}
|
||||
|
||||
func (state *mutableProviderState) setError(err error) {
|
||||
state.mu.Lock()
|
||||
defer state.mu.Unlock()
|
||||
state.err = err
|
||||
}
|
||||
|
||||
func (state *mutableProviderState) setChecksum(checksum string) {
|
||||
state.mu.Lock()
|
||||
defer state.mu.Unlock()
|
||||
state.checksum = checksum
|
||||
state.revision++
|
||||
}
|
||||
|
||||
type mutableProviderConfigurationSource struct {
|
||||
mu sync.Mutex
|
||||
configuration *config.Config
|
||||
}
|
||||
|
||||
type providerConfigurationSourceFunc func(context.Context) (admin.LoadedConfiguration, error)
|
||||
|
||||
func (source providerConfigurationSourceFunc) LoadConfiguration(ctx context.Context) (admin.LoadedConfiguration, error) {
|
||||
return source(ctx)
|
||||
}
|
||||
|
||||
func (source *mutableProviderConfigurationSource) LoadConfiguration(context.Context) (admin.LoadedConfiguration, error) {
|
||||
source.mu.Lock()
|
||||
defer source.mu.Unlock()
|
||||
return admin.LoadedConfiguration{Value: source.configuration, Source: "controller.yaml"}, nil
|
||||
}
|
||||
|
||||
func (source *mutableProviderConfigurationSource) set(configuration *config.Config) {
|
||||
source.mu.Lock()
|
||||
defer source.mu.Unlock()
|
||||
source.configuration = configuration
|
||||
}
|
||||
|
||||
func (state *mutableProviderState) set(name string, enabled bool) {
|
||||
state.mu.Lock()
|
||||
defer state.mu.Unlock()
|
||||
state.enabled[name] = enabled
|
||||
}
|
||||
|
||||
func waitForRuntimeEvents(t *testing.T, events <-chan providerRuntimeEvent, count int) {
|
||||
t.Helper()
|
||||
for range count {
|
||||
_ = waitForRuntimeEvent(t, events)
|
||||
}
|
||||
}
|
||||
|
||||
func waitForRuntimeEvent(t *testing.T, events <-chan providerRuntimeEvent) providerRuntimeEvent {
|
||||
t.Helper()
|
||||
select {
|
||||
case event := <-events:
|
||||
return event
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("timed out waiting for Provider runtime event")
|
||||
return providerRuntimeEvent{}
|
||||
}
|
||||
}
|
||||
104
internal/controller/bootstrap/provider_test.go
Normal file
104
internal/controller/bootstrap/provider_test.go
Normal file
@ -0,0 +1,104 @@
|
||||
package bootstrap
|
||||
|
||||
import (
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"proxy-pool/internal/config"
|
||||
"proxy-pool/internal/controller/provider"
|
||||
"proxy-pool/internal/platform/credentials"
|
||||
)
|
||||
|
||||
func TestProviderRuntimeConfigMapsValidatedUpstreamOnce(t *testing.T) {
|
||||
configuration, err := config.Load(strings.NewReader(bootstrapTestConfig))
|
||||
if err != nil {
|
||||
t.Fatalf("config.Load(): %v", err)
|
||||
}
|
||||
upstream := configuration.Upstreams["provider-a"]
|
||||
mapped, err := providerRuntimeConfig("provider-a", upstream)
|
||||
if err != nil {
|
||||
t.Fatalf("providerRuntimeConfig(): %v", err)
|
||||
}
|
||||
|
||||
if mapped.Provider.UpstreamID != "provider-a" ||
|
||||
mapped.Provider.RequestInterval != time.Second ||
|
||||
mapped.Provider.Timeout != 3*time.Second ||
|
||||
mapped.Provider.MaxAttempts != 3 || mapped.Provider.MaxInFlight != 1 ||
|
||||
mapped.Provider.MaxTotal != 1_000 || mapped.Provider.MaxSize != 100 ||
|
||||
mapped.Provider.TTL != 2*time.Minute ||
|
||||
mapped.Provider.AllocationSafetyMargin != 10*time.Second {
|
||||
t.Fatalf("Provider config = %+v", mapped.Provider)
|
||||
}
|
||||
if mapped.ReconcileInterval != time.Second ||
|
||||
mapped.ReconcilePolicy.MinimumAvailableSlots != 100 ||
|
||||
mapped.ReconcilePolicy.TargetAvailableSlots != 200 ||
|
||||
mapped.ReconcilePolicy.ExpectedPerFetch != 10 ||
|
||||
mapped.ReconcilePolicy.ExpectedSlotsPerFetch != 100 ||
|
||||
mapped.ReconcilePolicy.SafetyMargin != 10*time.Second {
|
||||
t.Fatalf("Reconcile config = %+v interval=%s", mapped.ReconcilePolicy, mapped.ReconcileInterval)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewProviderFleetBuildsEnabledUpstreamsInStableOrder(t *testing.T) {
|
||||
configuration, err := config.Load(strings.NewReader(bootstrapTestConfig))
|
||||
if err != nil {
|
||||
t.Fatalf("config.Load(): %v", err)
|
||||
}
|
||||
configuration.Upstreams["disabled"] = config.Upstream{}
|
||||
credentialStore, err := credentials.NewMemoryStore(10)
|
||||
if err != nil {
|
||||
t.Fatalf("NewMemoryStore(): %v", err)
|
||||
}
|
||||
fleet, err := newProviderFleet(configuration, ports{
|
||||
activity: &stubActivityStore{}, coordinator: coordinatorStub{}, credentials: credentialStore,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("newProviderFleet(): %v", err)
|
||||
}
|
||||
if got := fleet.IDs(); len(got) != 2 || got[0] != "provider-a" || got[1] != "provider-b" {
|
||||
t.Fatalf("Fleet IDs = %v, want [provider-a provider-b]", got)
|
||||
}
|
||||
|
||||
configuration.Upstreams["provider-a"] = config.Upstream{}
|
||||
configuration.Upstreams["provider-b"] = config.Upstream{}
|
||||
fleet, err = newProviderFleet(configuration, ports{})
|
||||
if err != nil || fleet != nil {
|
||||
t.Fatalf("newProviderFleet(no enabled) = (%v, %v), want nil fleet", fleet, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProviderRuntimeConfigRejectsDisabledAndOverflowingInputs(t *testing.T) {
|
||||
if _, err := providerRuntimeConfig("provider-a", config.Upstream{}); err == nil {
|
||||
t.Fatal("providerRuntimeConfig(disabled) error = nil")
|
||||
}
|
||||
overflowing := config.Upstream{
|
||||
Enabled: true,
|
||||
Pool: config.Pool{MaxSize: int(^uint(0) >> 1)},
|
||||
Capacity: config.Capacity{
|
||||
MaxConcurrencyPerProxy: int(^uint(0) >> 1),
|
||||
},
|
||||
Fetch: config.Fetch{EstimatedIPsPerCall: int(^uint(0) >> 1)},
|
||||
}
|
||||
if _, err := providerRuntimeConfig("provider-a", overflowing); err == nil {
|
||||
t.Fatal("providerRuntimeConfig(overflow) error = nil")
|
||||
}
|
||||
configuration, err := config.Load(strings.NewReader(bootstrapTestConfig))
|
||||
if err != nil {
|
||||
t.Fatalf("config.Load(): %v", err)
|
||||
}
|
||||
tooLarge := configuration.Upstreams["provider-a"]
|
||||
tooLarge.Pool.MaxSize = redisMaximumScan + 1
|
||||
if _, err := providerRuntimeConfig("provider-a", tooLarge); err == nil {
|
||||
t.Fatal("providerRuntimeConfig(oversized inventory) error = nil")
|
||||
}
|
||||
tooLarge = configuration.Upstreams["provider-a"]
|
||||
tooLarge.Fetch.MaxTotal = int(provider.MaximumCoordinationInteger)
|
||||
if strconv.IntSize == 64 {
|
||||
tooLarge.Fetch.MaxTotal++
|
||||
if _, err := providerRuntimeConfig("provider-a", tooLarge); err == nil {
|
||||
t.Fatal("providerRuntimeConfig(inexact Redis integer) error = nil")
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -9,6 +9,7 @@ import (
|
||||
|
||||
"proxy-pool/internal/config"
|
||||
"proxy-pool/internal/controller/admin"
|
||||
"proxy-pool/internal/controller/provider"
|
||||
"proxy-pool/internal/domain/activitypool"
|
||||
)
|
||||
|
||||
@ -24,6 +25,7 @@ type ConfigurationReader interface {
|
||||
type Reader struct {
|
||||
configuration ConfigurationReader
|
||||
inventory activitypool.StateInventoryReader
|
||||
providerStats provider.StatsReader
|
||||
now func() time.Time
|
||||
}
|
||||
|
||||
@ -33,11 +35,17 @@ func NewReader(
|
||||
configuration ConfigurationReader,
|
||||
inventory activitypool.StateInventoryReader,
|
||||
now func() time.Time,
|
||||
stats ...provider.StatsReader,
|
||||
) (*Reader, error) {
|
||||
if nilInterface(configuration) || nilInterface(inventory) || now == nil {
|
||||
if nilInterface(configuration) || nilInterface(inventory) || now == nil || len(stats) > 1 ||
|
||||
(len(stats) == 1 && nilInterface(stats[0])) {
|
||||
return nil, ErrInvalidReader
|
||||
}
|
||||
return &Reader{configuration: configuration, inventory: inventory, now: now}, nil
|
||||
reader := &Reader{configuration: configuration, inventory: inventory, now: now}
|
||||
if len(stats) == 1 {
|
||||
reader.providerStats = stats[0]
|
||||
}
|
||||
return reader, nil
|
||||
}
|
||||
|
||||
func (reader *Reader) ReadOperationalStatus(ctx context.Context) (admin.OperationalStatus, error) {
|
||||
@ -69,17 +77,27 @@ func (reader *Reader) ReadOperationalStatus(ctx context.Context) (admin.Operatio
|
||||
}
|
||||
|
||||
status := admin.OperationalStatus{Upstreams: make([]admin.UpstreamActivity, len(inventories))}
|
||||
providerStats := make([]provider.Stats, len(upstreamIDs))
|
||||
if reader.providerStats != nil {
|
||||
providerStats = reader.providerStats.ReadProviderStats(upstreamIDs)
|
||||
if len(providerStats) != len(upstreamIDs) {
|
||||
return admin.OperationalStatus{}, ErrUnavailable
|
||||
}
|
||||
}
|
||||
for index, inventory := range inventories {
|
||||
if inventory.UpstreamID != upstreamIDs[index] || invalidInventory(inventory) {
|
||||
if inventory.UpstreamID != upstreamIDs[index] || invalidInventory(inventory) ||
|
||||
providerStats[index].UpstreamID != "" && providerStats[index].UpstreamID != upstreamIDs[index] {
|
||||
return admin.OperationalStatus{}, ErrUnavailable
|
||||
}
|
||||
status.Upstreams[index] = admin.UpstreamActivity{
|
||||
Name: inventory.UpstreamID,
|
||||
Available: inventory.Available,
|
||||
Checking: inventory.Checking,
|
||||
Suspect: inventory.Suspect,
|
||||
Draining: inventory.Draining,
|
||||
Extracted: inventory.Extracted,
|
||||
Name: inventory.UpstreamID,
|
||||
Available: inventory.Available,
|
||||
Checking: inventory.Checking,
|
||||
Suspect: inventory.Suspect,
|
||||
Draining: inventory.Draining,
|
||||
Extracted: inventory.Extracted,
|
||||
ConsecutiveEmptyFetch: providerStats[index].ConsecutiveEmptyFetch,
|
||||
FetchErrorCount: providerStats[index].FetchErrorCount,
|
||||
}
|
||||
}
|
||||
return status, nil
|
||||
|
||||
@ -7,6 +7,7 @@ import (
|
||||
"time"
|
||||
|
||||
"proxy-pool/internal/config"
|
||||
"proxy-pool/internal/controller/provider"
|
||||
"proxy-pool/internal/domain/activitypool"
|
||||
)
|
||||
|
||||
@ -19,7 +20,10 @@ func TestReaderMapsCurrentUpstreamsToAdminOperationalStatus(t *testing.T) {
|
||||
}}
|
||||
reader, err := NewReader(staticConfigurationReader{configuration: &config.Config{
|
||||
Upstreams: map[string]config.Upstream{"provider-b": {}, "provider-a": {}},
|
||||
}}, inventory, func() time.Time { return now })
|
||||
}}, inventory, func() time.Time { return now }, staticProviderStatsReader{result: []provider.Stats{
|
||||
{UpstreamID: "provider-a", ConsecutiveEmptyFetch: 3, FetchErrorCount: 4},
|
||||
{UpstreamID: "provider-b", FetchErrorCount: 1},
|
||||
}})
|
||||
if err != nil {
|
||||
t.Fatalf("NewReader() error = %v", err)
|
||||
}
|
||||
@ -37,7 +41,7 @@ func TestReaderMapsCurrentUpstreamsToAdminOperationalStatus(t *testing.T) {
|
||||
}
|
||||
first := status.Upstreams[0]
|
||||
if first.Name != "provider-a" || first.Available != 11 || first.Checking != 2 || first.Suspect != 1 ||
|
||||
first.Draining != 4 || first.Extracted != 8 {
|
||||
first.Draining != 4 || first.Extracted != 8 || first.ConsecutiveEmptyFetch != 3 || first.FetchErrorCount != 4 {
|
||||
t.Fatalf("first upstream = %+v", first)
|
||||
}
|
||||
if status.Upstreams[1].Name != "provider-b" || status.Upstreams[1].Available != 7 {
|
||||
@ -107,6 +111,14 @@ type recordingStateInventoryReader struct {
|
||||
now time.Time
|
||||
}
|
||||
|
||||
type staticProviderStatsReader struct {
|
||||
result []provider.Stats
|
||||
}
|
||||
|
||||
func (reader staticProviderStatsReader) ReadProviderStats([]string) []provider.Stats {
|
||||
return append([]provider.Stats(nil), reader.result...)
|
||||
}
|
||||
|
||||
func (reader *recordingStateInventoryReader) ReadStateInventory(
|
||||
_ context.Context,
|
||||
upstreamIDs []string,
|
||||
|
||||
@ -20,48 +20,38 @@ var (
|
||||
type FetchBudgetConfig struct {
|
||||
UpstreamID string
|
||||
MaxSize int
|
||||
MaxTotal int64
|
||||
ExpectedPerFetch int
|
||||
Managed int
|
||||
FetchedTotal int64
|
||||
}
|
||||
|
||||
type FetchBudgetSnapshot struct {
|
||||
Managed int
|
||||
PendingExpected int
|
||||
FetchedTotal int64
|
||||
}
|
||||
|
||||
// FetchBudget owns both current-inventory and cumulative-fetch accounting for
|
||||
// one upstream. Reserving the expected response before I/O closes the race
|
||||
// between concurrent provider calls.
|
||||
// FetchBudget owns current-inventory accounting for one upstream. Reserving
|
||||
// the expected response before I/O closes the race between concurrent calls;
|
||||
// distributed cumulative quota belongs to the Provider coordination permit.
|
||||
type FetchBudget struct {
|
||||
mu sync.Mutex
|
||||
|
||||
upstreamID string
|
||||
maxSize int
|
||||
maxTotal int64
|
||||
expected int
|
||||
usage FetchBudgetSnapshot
|
||||
}
|
||||
|
||||
func NewFetchBudget(config FetchBudgetConfig) (*FetchBudget, error) {
|
||||
if config.UpstreamID == "" || config.MaxSize <= 0 || config.ExpectedPerFetch <= 0 ||
|
||||
config.ExpectedPerFetch > config.MaxSize || config.MaxTotal < 0 ||
|
||||
config.Managed < 0 || config.FetchedTotal < 0 {
|
||||
return nil, ErrInvalidFetchBudget
|
||||
}
|
||||
if config.MaxTotal > 0 && int64(config.ExpectedPerFetch) > config.MaxTotal {
|
||||
config.ExpectedPerFetch > config.MaxSize || config.Managed < 0 {
|
||||
return nil, ErrInvalidFetchBudget
|
||||
}
|
||||
return &FetchBudget{
|
||||
upstreamID: config.UpstreamID,
|
||||
maxSize: config.MaxSize,
|
||||
maxTotal: config.MaxTotal,
|
||||
expected: config.ExpectedPerFetch,
|
||||
usage: FetchBudgetSnapshot{
|
||||
Managed: config.Managed,
|
||||
FetchedTotal: config.FetchedTotal,
|
||||
Managed: config.Managed,
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
@ -95,14 +85,7 @@ func (b *FetchBudget) FetchAllowance() int {
|
||||
|
||||
func (b *FetchBudget) canReserveLocked() bool {
|
||||
poolRoom := b.maxSize - b.usage.Managed - b.usage.PendingExpected
|
||||
if poolRoom < b.expected {
|
||||
return false
|
||||
}
|
||||
if b.maxTotal > 0 {
|
||||
totalRoom := b.maxTotal - b.usage.FetchedTotal - int64(b.usage.PendingExpected)
|
||||
return totalRoom >= int64(b.expected)
|
||||
}
|
||||
return true
|
||||
return poolRoom >= b.expected
|
||||
}
|
||||
|
||||
func (b *FetchBudget) Snapshot() FetchBudgetSnapshot {
|
||||
@ -115,8 +98,8 @@ func (b *FetchBudget) Snapshot() FetchBudgetSnapshot {
|
||||
}
|
||||
|
||||
// SynchronizeManaged replaces the local current-inventory count with the
|
||||
// authoritative activity-store observation. Pending requests and cumulative
|
||||
// fetch usage remain owned by this budget.
|
||||
// authoritative activity-store observation. Pending reservations remain owned
|
||||
// by this budget.
|
||||
func (b *FetchBudget) SynchronizeManaged(managed int) error {
|
||||
if b == nil || managed < 0 {
|
||||
return ErrInvalidManagedSynchronization
|
||||
@ -158,11 +141,11 @@ func (p *fetchPermit) Expected() int {
|
||||
return p.expected
|
||||
}
|
||||
|
||||
func (p *fetchPermit) Complete(fetched, retained int) error {
|
||||
func (p *fetchPermit) Complete(retained int) error {
|
||||
if p == nil || p.budget == nil {
|
||||
return ErrFetchPermitFinished
|
||||
}
|
||||
if fetched < 0 || retained < 0 || retained > fetched || retained > p.expected {
|
||||
if retained < 0 || retained > p.expected {
|
||||
return ErrInvalidFetchCompletion
|
||||
}
|
||||
p.budget.mu.Lock()
|
||||
@ -173,7 +156,6 @@ func (p *fetchPermit) Complete(fetched, retained int) error {
|
||||
p.finished = true
|
||||
p.budget.usage.PendingExpected -= p.expected
|
||||
p.budget.usage.Managed += retained
|
||||
p.budget.usage.FetchedTotal += int64(fetched)
|
||||
return nil
|
||||
}
|
||||
|
||||
|
||||
@ -6,14 +6,12 @@ import (
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestFetchBudgetReservesExpectedCapacityAndSeparatesCounters(t *testing.T) {
|
||||
func TestFetchBudgetReservesExpectedPoolCapacity(t *testing.T) {
|
||||
budget, err := NewFetchBudget(FetchBudgetConfig{
|
||||
UpstreamID: "provider-a",
|
||||
MaxSize: 10,
|
||||
MaxTotal: 20,
|
||||
ExpectedPerFetch: 4,
|
||||
Managed: 2,
|
||||
FetchedTotal: 3,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("NewFetchBudget(): %v", err)
|
||||
@ -31,21 +29,21 @@ func TestFetchBudgetReservesExpectedCapacityAndSeparatesCounters(t *testing.T) {
|
||||
t.Fatalf("third ReserveFetch() = (_, %v, %v), want no capacity", ok, err)
|
||||
}
|
||||
|
||||
if err := first.Complete(4, 3); err != nil {
|
||||
if err := first.Complete(3); err != nil {
|
||||
t.Fatalf("first.Complete(): %v", err)
|
||||
}
|
||||
if err := second.Cancel(); err != nil {
|
||||
t.Fatalf("second.Cancel(): %v", err)
|
||||
}
|
||||
usage := budget.Snapshot()
|
||||
if usage.Managed != 5 || usage.PendingExpected != 0 || usage.FetchedTotal != 7 {
|
||||
t.Fatalf("Snapshot() = %+v, want managed=5 pending=0 fetched=7", usage)
|
||||
if usage.Managed != 5 || usage.PendingExpected != 0 {
|
||||
t.Fatalf("Snapshot() = %+v, want managed=5 pending=0", usage)
|
||||
}
|
||||
if err := budget.ReleaseManaged(2); err != nil {
|
||||
t.Fatalf("ReleaseManaged(): %v", err)
|
||||
}
|
||||
if usage := budget.Snapshot(); usage.Managed != 3 || usage.FetchedTotal != 7 {
|
||||
t.Fatalf("Snapshot() after release = %+v, want managed=3 fetched=7", usage)
|
||||
if usage := budget.Snapshot(); usage.Managed != 3 {
|
||||
t.Fatalf("Snapshot() after release = %+v, want managed=3", usage)
|
||||
}
|
||||
}
|
||||
|
||||
@ -63,7 +61,7 @@ func TestFetchBudgetRejectsManagedCounterUnderflow(t *testing.T) {
|
||||
|
||||
func TestFetchBudgetSynchronizesAuthoritativeManagedInventory(t *testing.T) {
|
||||
budget, err := NewFetchBudget(FetchBudgetConfig{
|
||||
UpstreamID: "a", MaxSize: 5, ExpectedPerFetch: 2, Managed: 3, FetchedTotal: 7,
|
||||
UpstreamID: "a", MaxSize: 5, ExpectedPerFetch: 2, Managed: 3,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("NewFetchBudget(): %v", err)
|
||||
@ -77,8 +75,8 @@ func TestFetchBudgetSynchronizesAuthoritativeManagedInventory(t *testing.T) {
|
||||
t.Fatalf("SynchronizeManaged() error = %v, want ErrManagedSynchronizationInFlight", err)
|
||||
}
|
||||
usage := budget.Snapshot()
|
||||
if usage.Managed != 3 || usage.PendingExpected != 2 || usage.FetchedTotal != 7 {
|
||||
t.Fatalf("Snapshot() = %+v, want managed=3 pending=2 fetched=7", usage)
|
||||
if usage.Managed != 3 || usage.PendingExpected != 2 {
|
||||
t.Fatalf("Snapshot() = %+v, want managed=3 pending=2", usage)
|
||||
}
|
||||
if err := permit.Cancel(); err != nil {
|
||||
t.Fatalf("Cancel(): %v", err)
|
||||
@ -86,8 +84,8 @@ func TestFetchBudgetSynchronizesAuthoritativeManagedInventory(t *testing.T) {
|
||||
if err := budget.SynchronizeManaged(1); err != nil {
|
||||
t.Fatalf("SynchronizeManaged() after cancel: %v", err)
|
||||
}
|
||||
if usage := budget.Snapshot(); usage.Managed != 1 || usage.PendingExpected != 0 || usage.FetchedTotal != 7 {
|
||||
t.Fatalf("Snapshot() after synchronization = %+v, want managed=1 pending=0 fetched=7", usage)
|
||||
if usage := budget.Snapshot(); usage.Managed != 1 || usage.PendingExpected != 0 {
|
||||
t.Fatalf("Snapshot() after synchronization = %+v, want managed=1 pending=0", usage)
|
||||
}
|
||||
}
|
||||
|
||||
@ -103,35 +101,15 @@ func TestFetchBudgetRejectsNegativeManagedSynchronization(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestFetchBudgetRequiresWholeExpectedBatchToFitLimits(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
config FetchBudgetConfig
|
||||
}{
|
||||
{
|
||||
name: "pool size",
|
||||
config: FetchBudgetConfig{
|
||||
UpstreamID: "a", MaxSize: 10, ExpectedPerFetch: 4, Managed: 7,
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "cumulative total",
|
||||
config: FetchBudgetConfig{
|
||||
UpstreamID: "a", MaxSize: 10, MaxTotal: 5, ExpectedPerFetch: 4, FetchedTotal: 2,
|
||||
},
|
||||
},
|
||||
func TestFetchBudgetRequiresWholeExpectedBatchToFitPool(t *testing.T) {
|
||||
budget, err := NewFetchBudget(FetchBudgetConfig{
|
||||
UpstreamID: "a", MaxSize: 10, ExpectedPerFetch: 4, Managed: 7,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("NewFetchBudget(): %v", err)
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
budget, err := NewFetchBudget(tt.config)
|
||||
if err != nil {
|
||||
t.Fatalf("NewFetchBudget(): %v", err)
|
||||
}
|
||||
if _, ok, err := budget.ReserveFetch("a"); err != nil || ok {
|
||||
t.Fatalf("ReserveFetch() = (_, %v, %v), want no capacity", ok, err)
|
||||
}
|
||||
})
|
||||
if _, ok, err := budget.ReserveFetch("a"); err != nil || ok {
|
||||
t.Fatalf("ReserveFetch() = (_, %v, %v), want no capacity", ok, err)
|
||||
}
|
||||
}
|
||||
|
||||
@ -185,7 +163,7 @@ func TestFetchPermitRejectsDoubleFinishAndInvalidCounts(t *testing.T) {
|
||||
if err != nil || !ok {
|
||||
t.Fatalf("ReserveFetch() = (_, %v, %v), want permit", ok, err)
|
||||
}
|
||||
if err := permit.Complete(1, 2); !errors.Is(err, ErrInvalidFetchCompletion) {
|
||||
if err := permit.Complete(3); !errors.Is(err, ErrInvalidFetchCompletion) {
|
||||
t.Fatalf("Complete() error = %v, want ErrInvalidFetchCompletion", err)
|
||||
}
|
||||
if err := permit.Cancel(); err != nil {
|
||||
|
||||
17
internal/controller/pool/inventory.go
Normal file
17
internal/controller/pool/inventory.go
Normal file
@ -0,0 +1,17 @@
|
||||
package pool
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
)
|
||||
|
||||
type InventorySnapshot struct {
|
||||
Managed int
|
||||
AvailableSlots int64
|
||||
}
|
||||
|
||||
type InventoryReader interface {
|
||||
// ReadInventory returns a bounded, authoritative aggregate. Unknown Worker
|
||||
// runtime must reduce capacity rather than being treated as idle.
|
||||
ReadInventory(context.Context, string, time.Duration) (InventorySnapshot, error)
|
||||
}
|
||||
@ -26,7 +26,6 @@ type FetchNotifier interface {
|
||||
type ReconcileDecision struct {
|
||||
AvailableSlots int64
|
||||
PendingExpected int
|
||||
FetchedTotal int64
|
||||
FetchAllowance int
|
||||
EffectiveSlots int64
|
||||
Triggered bool
|
||||
@ -62,15 +61,28 @@ func NewReconciler(policy ReconcilePolicy, budget *FetchBudget, notifier FetchNo
|
||||
// Reconcile centralizes the cold-path decision. The notifier may coalesce many
|
||||
// calls; Provider Reconciler atomically reserves the budget before doing I/O.
|
||||
func (r *Reconciler) Reconcile(now time.Time, inventory upstream.Inventory) ReconcileDecision {
|
||||
return r.reconcileSnapshot(
|
||||
InventorySnapshot{AvailableSlots: inventory.AvailableSlots(now, r.policy.SafetyMargin)},
|
||||
false,
|
||||
)
|
||||
}
|
||||
|
||||
func (r *Reconciler) ReconcileSnapshot(inventory InventorySnapshot) ReconcileDecision {
|
||||
return r.reconcileSnapshot(inventory, true)
|
||||
}
|
||||
|
||||
func (r *Reconciler) reconcileSnapshot(inventory InventorySnapshot, synchronizeManaged bool) ReconcileDecision {
|
||||
usage := r.budget.Snapshot()
|
||||
availableSlots := inventory.AvailableSlots(now, r.policy.SafetyMargin)
|
||||
if synchronizeManaged && inventory.Managed >= 0 && usage.PendingExpected == 0 {
|
||||
_ = r.budget.SynchronizeManaged(inventory.Managed)
|
||||
usage = r.budget.Snapshot()
|
||||
}
|
||||
pendingSlots := saturatingMultiply(int64(usage.PendingExpected), r.slotsPerProxy)
|
||||
decision := ReconcileDecision{
|
||||
AvailableSlots: availableSlots,
|
||||
AvailableSlots: inventory.AvailableSlots,
|
||||
PendingExpected: usage.PendingExpected,
|
||||
FetchedTotal: usage.FetchedTotal,
|
||||
FetchAllowance: r.budget.FetchAllowance(),
|
||||
EffectiveSlots: saturatingAdd(availableSlots, pendingSlots),
|
||||
EffectiveSlots: saturatingAdd(inventory.AvailableSlots, pendingSlots),
|
||||
}
|
||||
r.mu.Lock()
|
||||
if r.refilling {
|
||||
|
||||
@ -10,7 +10,7 @@ import (
|
||||
|
||||
func TestPoolReconcilerSignalsOnlyWhenSlotsAreLowAndWholeFetchFits(t *testing.T) {
|
||||
budget, err := NewFetchBudget(FetchBudgetConfig{
|
||||
UpstreamID: "provider-a", MaxSize: 10, MaxTotal: 20, ExpectedPerFetch: 2,
|
||||
UpstreamID: "provider-a", MaxSize: 10, ExpectedPerFetch: 2,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("NewFetchBudget(): %v", err)
|
||||
@ -31,8 +31,7 @@ func TestPoolReconcilerSignalsOnlyWhenSlotsAreLowAndWholeFetchFits(t *testing.T)
|
||||
Proxies: []upstream.ProxyCapacity{{
|
||||
State: proxyDomain.StateAvailable, ExpiresAt: now.Add(time.Minute), Max: 4, Active: 3,
|
||||
}},
|
||||
MaxSize: 10,
|
||||
MaxTotal: 20,
|
||||
MaxSize: 10,
|
||||
}
|
||||
|
||||
decision := reconciler.Reconcile(now, inventory)
|
||||
@ -44,9 +43,9 @@ func TestPoolReconcilerSignalsOnlyWhenSlotsAreLowAndWholeFetchFits(t *testing.T)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPoolReconcilerUsesBudgetPendingAndCumulativeCounters(t *testing.T) {
|
||||
func TestPoolReconcilerUsesPendingPoolReservation(t *testing.T) {
|
||||
budget, err := NewFetchBudget(FetchBudgetConfig{
|
||||
UpstreamID: "provider-a", MaxSize: 2, MaxTotal: 2, ExpectedPerFetch: 2,
|
||||
UpstreamID: "provider-a", MaxSize: 2, ExpectedPerFetch: 2,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("NewFetchBudget(): %v", err)
|
||||
@ -65,7 +64,7 @@ func TestPoolReconcilerUsesBudgetPendingAndCumulativeCounters(t *testing.T) {
|
||||
t.Fatalf("NewReconciler(): %v", err)
|
||||
}
|
||||
|
||||
decision := reconciler.Reconcile(time.Now(), upstream.Inventory{MaxSize: 2, MaxTotal: 2})
|
||||
decision := reconciler.Reconcile(time.Now(), upstream.Inventory{MaxSize: 2})
|
||||
if decision.Triggered || decision.PendingExpected != 2 || decision.FetchAllowance != 0 {
|
||||
t.Fatalf("Reconcile() = %+v, want pending fetch to suppress signal", decision)
|
||||
}
|
||||
@ -133,7 +132,7 @@ func TestPoolReconcilerPendingEstimatePausesWithoutEndingRefillEpisode(t *testin
|
||||
if decision := reconciler.Reconcile(now, inventoryWithSlots(now, 2)); decision.Triggered || decision.EffectiveSlots != 8 {
|
||||
t.Fatalf("pending Reconcile() = %+v, want paused at target estimate", decision)
|
||||
}
|
||||
if err := permit.Complete(2, 1); err != nil {
|
||||
if err := permit.Complete(1); err != nil {
|
||||
t.Fatalf("Complete(): %v", err)
|
||||
}
|
||||
if decision := reconciler.Reconcile(now, inventoryWithSlots(now, 5)); !decision.Triggered {
|
||||
@ -141,6 +140,53 @@ func TestPoolReconcilerPendingEstimatePausesWithoutEndingRefillEpisode(t *testin
|
||||
}
|
||||
}
|
||||
|
||||
func TestPoolReconcilerConsumesAuthoritativeInventorySnapshot(t *testing.T) {
|
||||
budget, err := NewFetchBudget(FetchBudgetConfig{
|
||||
UpstreamID: "provider-a", MaxSize: 10, ExpectedPerFetch: 2,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("NewFetchBudget(): %v", err)
|
||||
}
|
||||
notifier := &recordingFetchNotifier{}
|
||||
reconciler, err := NewReconciler(ReconcilePolicy{
|
||||
MinimumAvailableSlots: 3, TargetAvailableSlots: 8,
|
||||
ExpectedPerFetch: 2, ExpectedSlotsPerFetch: 2,
|
||||
}, budget, notifier)
|
||||
if err != nil {
|
||||
t.Fatalf("NewReconciler(): %v", err)
|
||||
}
|
||||
decision := reconciler.ReconcileSnapshot(InventorySnapshot{Managed: 9, AvailableSlots: 2})
|
||||
if decision.Triggered || decision.AvailableSlots != 2 || decision.FetchAllowance != 0 {
|
||||
t.Fatalf("ReconcileSnapshot() = %+v, want authoritative managed inventory to close budget", decision)
|
||||
}
|
||||
if usage := budget.Snapshot(); usage.Managed != 9 {
|
||||
t.Fatalf("FetchBudget.Managed = %d, want 9", usage.Managed)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPoolReconcilerLegacyInventoryDoesNotClearManagedBudget(t *testing.T) {
|
||||
budget, err := NewFetchBudget(FetchBudgetConfig{
|
||||
UpstreamID: "provider-a", MaxSize: 10, ExpectedPerFetch: 2, Managed: 9,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("NewFetchBudget(): %v", err)
|
||||
}
|
||||
reconciler, err := NewReconciler(ReconcilePolicy{
|
||||
MinimumAvailableSlots: 3, TargetAvailableSlots: 8,
|
||||
ExpectedPerFetch: 2, ExpectedSlotsPerFetch: 2,
|
||||
}, budget, &recordingFetchNotifier{})
|
||||
if err != nil {
|
||||
t.Fatalf("NewReconciler(): %v", err)
|
||||
}
|
||||
decision := reconciler.Reconcile(time.Now(), upstream.Inventory{})
|
||||
if decision.Triggered || decision.FetchAllowance != 0 {
|
||||
t.Fatalf("Reconcile(legacy) = %+v, want preserved managed budget", decision)
|
||||
}
|
||||
if usage := budget.Snapshot(); usage.Managed != 9 {
|
||||
t.Fatalf("FetchBudget.Managed = %d, want 9", usage.Managed)
|
||||
}
|
||||
}
|
||||
|
||||
func inventoryWithSlots(now time.Time, slots int64) upstream.Inventory {
|
||||
return upstream.Inventory{Proxies: []upstream.ProxyCapacity{{
|
||||
State: proxyDomain.StateAvailable, ExpiresAt: now.Add(time.Minute), Max: slots,
|
||||
|
||||
@ -13,10 +13,13 @@ var (
|
||||
ErrLeaderWorkStopped = errors.New("provider leader work stopped")
|
||||
)
|
||||
|
||||
const MaximumCoordinationInteger = int64(1<<53 - 1)
|
||||
|
||||
type CoordinationLimits struct {
|
||||
RequestInterval time.Duration
|
||||
MaxInFlight int
|
||||
MaxAttemptDuration time.Duration
|
||||
MaxTotal int64
|
||||
}
|
||||
|
||||
type Fence struct {
|
||||
@ -32,10 +35,14 @@ type Coordinator interface {
|
||||
|
||||
type LeaderSession interface {
|
||||
Fence() Fence
|
||||
AcquireFetch(context.Context) (RequestPermit, error)
|
||||
AcquireFetch(context.Context, int) (RequestPermit, bool, error)
|
||||
}
|
||||
|
||||
type RequestPermit interface {
|
||||
// Release is idempotent. A failed release expires automatically in storage.
|
||||
Release(context.Context) error
|
||||
// Complete atomically releases in-flight capacity and charges the actual
|
||||
// successful candidate count. Repeated calls are idempotent.
|
||||
Complete(context.Context, int) error
|
||||
// Cancel releases a reservation that is known not to have consumed Provider
|
||||
// quota. A crashed or abandoned reservation is conservatively charged.
|
||||
Cancel(context.Context) error
|
||||
}
|
||||
|
||||
64
internal/controller/provider/fleet.go
Normal file
64
internal/controller/provider/fleet.go
Normal file
@ -0,0 +1,64 @@
|
||||
package provider
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"sort"
|
||||
|
||||
"proxy-pool/internal/platform/lifecycle"
|
||||
)
|
||||
|
||||
var ErrInvalidFleet = errors.New("invalid Provider fleet")
|
||||
|
||||
type Fleet struct {
|
||||
runtimes []*UpstreamRuntime
|
||||
group *lifecycle.Group
|
||||
}
|
||||
|
||||
func NewFleet(runtimes ...*UpstreamRuntime) (*Fleet, error) {
|
||||
if len(runtimes) == 0 {
|
||||
return nil, ErrInvalidFleet
|
||||
}
|
||||
seen := make(map[string]struct{}, len(runtimes))
|
||||
owned := make([]*UpstreamRuntime, len(runtimes))
|
||||
for index, runtime := range runtimes {
|
||||
if runtime == nil || runtime.config.Provider.UpstreamID == "" {
|
||||
return nil, ErrInvalidFleet
|
||||
}
|
||||
if _, exists := seen[runtime.config.Provider.UpstreamID]; exists {
|
||||
return nil, ErrInvalidFleet
|
||||
}
|
||||
seen[runtime.config.Provider.UpstreamID] = struct{}{}
|
||||
owned[index] = runtime
|
||||
}
|
||||
sort.Slice(owned, func(left, right int) bool {
|
||||
return owned[left].ID() < owned[right].ID()
|
||||
})
|
||||
runners := make([]lifecycle.Runner, len(owned))
|
||||
for index, runtime := range owned {
|
||||
runners[index] = runtime
|
||||
}
|
||||
group, err := lifecycle.NewGroup(runners...)
|
||||
if err != nil {
|
||||
return nil, errors.Join(ErrInvalidFleet, err)
|
||||
}
|
||||
return &Fleet{runtimes: owned, group: group}, nil
|
||||
}
|
||||
|
||||
func (fleet *Fleet) Run(ctx context.Context) error {
|
||||
if fleet == nil || ctx == nil || len(fleet.runtimes) == 0 || fleet.group == nil {
|
||||
return ErrInvalidFleet
|
||||
}
|
||||
return fleet.group.Run(ctx)
|
||||
}
|
||||
|
||||
func (fleet *Fleet) IDs() []string {
|
||||
if fleet == nil {
|
||||
return nil
|
||||
}
|
||||
ids := make([]string, len(fleet.runtimes))
|
||||
for index, runtime := range fleet.runtimes {
|
||||
ids[index] = runtime.ID()
|
||||
}
|
||||
return ids
|
||||
}
|
||||
101
internal/controller/provider/fleet_test.go
Normal file
101
internal/controller/provider/fleet_test.go
Normal file
@ -0,0 +1,101 @@
|
||||
package provider
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"proxy-pool/internal/controller/pool"
|
||||
"proxy-pool/internal/domain/activitypool"
|
||||
proxyDomain "proxy-pool/internal/domain/proxy"
|
||||
)
|
||||
|
||||
func TestFleetRunsAllUpstreamsAndCancelsSiblingsOnFailure(t *testing.T) {
|
||||
started := make(chan string, 2)
|
||||
siblingCancelled := make(chan struct{}, 1)
|
||||
first := newFleetRuntime(t, "provider-a", coordinatorFunc(func(
|
||||
context.Context, string, CoordinationLimits, func(context.Context, LeaderSession) error,
|
||||
) error {
|
||||
started <- "provider-a"
|
||||
return errors.New("provider-a stopped")
|
||||
}))
|
||||
second := newFleetRuntime(t, "provider-b", coordinatorFunc(func(
|
||||
ctx context.Context, _ string, _ CoordinationLimits, _ func(context.Context, LeaderSession) error,
|
||||
) error {
|
||||
started <- "provider-b"
|
||||
<-ctx.Done()
|
||||
siblingCancelled <- struct{}{}
|
||||
return ctx.Err()
|
||||
}))
|
||||
fleet, err := NewFleet(first, second)
|
||||
if err != nil {
|
||||
t.Fatalf("NewFleet(): %v", err)
|
||||
}
|
||||
|
||||
err = fleet.Run(context.Background())
|
||||
if err == nil || err.Error() != "provider-a stopped" {
|
||||
t.Fatalf("Run() error = %v, want provider-a failure", err)
|
||||
}
|
||||
seen := map[string]bool{<-started: true, <-started: true}
|
||||
if !seen["provider-a"] || !seen["provider-b"] {
|
||||
t.Fatalf("started upstreams = %v", seen)
|
||||
}
|
||||
select {
|
||||
case <-siblingCancelled:
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("sibling runtime was not cancelled")
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewFleetRejectsEmptyNilAndDuplicateUpstreams(t *testing.T) {
|
||||
valid := newFleetRuntime(t, "provider-a", coordinatorFunc(func(
|
||||
ctx context.Context, _ string, _ CoordinationLimits, _ func(context.Context, LeaderSession) error,
|
||||
) error {
|
||||
return ctx.Err()
|
||||
}))
|
||||
tests := []struct {
|
||||
name string
|
||||
runtimes []*UpstreamRuntime
|
||||
}{
|
||||
{name: "empty"},
|
||||
{name: "nil", runtimes: []*UpstreamRuntime{nil}},
|
||||
{name: "duplicate", runtimes: []*UpstreamRuntime{valid, valid}},
|
||||
}
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
fleet, err := NewFleet(test.runtimes...)
|
||||
if err == nil || fleet != nil {
|
||||
t.Fatalf("NewFleet() = (%v, %v), want invalid fleet", fleet, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func newFleetRuntime(t *testing.T, upstreamID string, coordinator Coordinator) *UpstreamRuntime {
|
||||
t.Helper()
|
||||
runtime, err := NewUpstreamRuntime(UpstreamRuntimeConfig{
|
||||
Provider: runtimeProviderConfig(upstreamID),
|
||||
ReconcilePolicy: runtimeReconcilePolicy(),
|
||||
ReconcileInterval: time.Second,
|
||||
}, UpstreamRuntimeDependencies{
|
||||
Coordinator: coordinator,
|
||||
Inventory: inventoryReaderFunc(func(context.Context, string, time.Duration) (pool.InventorySnapshot, error) {
|
||||
return pool.InventorySnapshot{}, nil
|
||||
}),
|
||||
Adapter: adapterFunc(func(context.Context) (FetchResponse, error) {
|
||||
return FetchResponse{Body: []byte("fixture")}, nil
|
||||
}),
|
||||
Parser: parserFunc(func(context.Context, []byte) ([]proxyDomain.Proxy, error) {
|
||||
return []proxyDomain.Proxy{{ID: "proxy-1"}}, nil
|
||||
}),
|
||||
Activity: activitySinkFunc(func(context.Context, string, activitypool.FetchedBatch) (activitypool.UpsertResult, error) {
|
||||
return activitypool.UpsertResult{Accepted: 1, Inserted: 1}, nil
|
||||
}),
|
||||
Results: resultRecorderFunc(func(Result) {}),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("NewUpstreamRuntime(%s): %v", upstreamID, err)
|
||||
}
|
||||
return runtime
|
||||
}
|
||||
@ -27,9 +27,11 @@ type RetryableError interface {
|
||||
Retryable() bool
|
||||
}
|
||||
|
||||
// Parser must be safe for concurrent calls and must honor context cancellation.
|
||||
// Parser owns any transient resources attached to parsed candidates. It must
|
||||
// be safe for concurrent calls and must honor context cancellation.
|
||||
type Parser interface {
|
||||
Parse(context.Context, []byte) ([]proxyDomain.Proxy, error)
|
||||
ReleaseCandidates([]proxyDomain.Proxy)
|
||||
}
|
||||
|
||||
type Result struct {
|
||||
@ -42,7 +44,8 @@ type Result struct {
|
||||
}
|
||||
|
||||
type ResultRecorder interface {
|
||||
// Record may be called concurrently and must not retain mutable result data.
|
||||
// Record may be called concurrently, must return promptly, and must not
|
||||
// retain mutable result data.
|
||||
Record(Result)
|
||||
}
|
||||
|
||||
|
||||
@ -19,6 +19,7 @@ type Config struct {
|
||||
Timeout time.Duration
|
||||
MaxAttempts int
|
||||
MaxInFlight int
|
||||
MaxTotal int64
|
||||
MaxSize int
|
||||
TTL time.Duration
|
||||
AllocationSafetyMargin time.Duration
|
||||
@ -31,6 +32,8 @@ type RetryConfig struct {
|
||||
Jitter int
|
||||
}
|
||||
|
||||
const requestPermitSettlementTimeout = 5 * time.Second
|
||||
|
||||
type Reconciler struct {
|
||||
config Config
|
||||
ports Ports
|
||||
@ -64,7 +67,7 @@ func NewReconciler(config Config, ports Ports, runtimes ...Runtime) (*Reconciler
|
||||
return nil, fmt.Errorf("new provider reconciler: upstream ID is required")
|
||||
}
|
||||
if config.RequestInterval < 0 || config.Timeout <= 0 || config.MaxAttempts <= 0 ||
|
||||
config.MaxInFlight <= 0 || config.MaxSize <= 0 {
|
||||
config.MaxInFlight <= 0 || config.MaxTotal < 0 || config.MaxSize <= 0 {
|
||||
return nil, fmt.Errorf("new provider reconciler: fetch limits must be positive")
|
||||
}
|
||||
if config.TTL < 0 || config.AllocationSafetyMargin < 0 ||
|
||||
@ -112,7 +115,10 @@ func (r *Reconciler) Notify() {
|
||||
r.signal.Notify()
|
||||
}
|
||||
|
||||
func (r *Reconciler) Run(ctx context.Context) error {
|
||||
func (r *Reconciler) RunLeader(ctx context.Context, session LeaderSession) error {
|
||||
if ctx == nil || session == nil {
|
||||
return ErrInvalidCoordination
|
||||
}
|
||||
var workers sync.WaitGroup
|
||||
defer workers.Wait()
|
||||
for {
|
||||
@ -131,17 +137,57 @@ func (r *Reconciler) Run(ctx context.Context) error {
|
||||
go func() {
|
||||
defer workers.Done()
|
||||
defer func() { <-r.inFlight }()
|
||||
r.reconcile(ctx)
|
||||
r.reconcile(ctx, session)
|
||||
}()
|
||||
}
|
||||
}
|
||||
|
||||
func (r *Reconciler) reconcile(ctx context.Context) {
|
||||
func (r *Reconciler) reconcile(ctx context.Context, session LeaderSession) {
|
||||
permit, available, err := r.ports.Capacity.ReserveFetch(r.config.UpstreamID)
|
||||
if err != nil {
|
||||
r.ports.Results.Record(Result{
|
||||
UpstreamID: r.config.UpstreamID,
|
||||
Class: upstream.FetchError,
|
||||
Err: fmt.Errorf("reserve fetch capacity: %w", err),
|
||||
Attempt: 1,
|
||||
})
|
||||
return
|
||||
}
|
||||
if !available {
|
||||
return
|
||||
}
|
||||
if permit == nil || permit.Expected() <= 0 {
|
||||
if permit != nil {
|
||||
_ = permit.Cancel()
|
||||
}
|
||||
r.ports.Results.Record(Result{
|
||||
UpstreamID: r.config.UpstreamID,
|
||||
Class: upstream.FetchError,
|
||||
Err: fmt.Errorf("reserve fetch capacity: invalid permit"),
|
||||
Attempt: 1,
|
||||
})
|
||||
return
|
||||
}
|
||||
permitFinished := false
|
||||
defer func() {
|
||||
if !permitFinished {
|
||||
_ = permit.Cancel()
|
||||
}
|
||||
}()
|
||||
|
||||
for attempt := 1; attempt <= r.config.MaxAttempts; attempt++ {
|
||||
response, result, retryable, ok := r.fetchAttempt(ctx, attempt)
|
||||
response, result, retryable, ok := r.fetchAttempt(ctx, session, permit, attempt)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
if result.Class != upstream.FetchError {
|
||||
if capacityErr := permit.Complete(result.NewCount); capacityErr != nil {
|
||||
result.Class = upstream.FetchError
|
||||
result.Err = errors.Join(result.Err, capacityErr)
|
||||
} else {
|
||||
permitFinished = true
|
||||
}
|
||||
}
|
||||
r.ports.Results.Record(result)
|
||||
if result.Class != upstream.FetchError || !retryable || attempt == r.config.MaxAttempts {
|
||||
return
|
||||
@ -157,38 +203,42 @@ func (r *Reconciler) reconcile(ctx context.Context) {
|
||||
}
|
||||
}
|
||||
|
||||
func (r *Reconciler) fetchAttempt(ctx context.Context, attempt int) (FetchResponse, Result, bool, bool) {
|
||||
func (r *Reconciler) fetchAttempt(
|
||||
ctx context.Context,
|
||||
session LeaderSession,
|
||||
permit upstream.FetchPermit,
|
||||
attempt int,
|
||||
) (FetchResponse, Result, bool, bool) {
|
||||
if err := r.waitForRequestSlot(ctx); err != nil {
|
||||
return FetchResponse{}, Result{}, false, false
|
||||
}
|
||||
permit, available, err := r.ports.Capacity.ReserveFetch(r.config.UpstreamID)
|
||||
requestPermit, available, err := session.AcquireFetch(ctx, permit.Expected())
|
||||
if err != nil {
|
||||
resultErr := fmt.Errorf("reserve fetch capacity: %w", err)
|
||||
if ctx.Err() != nil || errors.Is(err, ErrLeadershipLost) {
|
||||
return FetchResponse{}, Result{}, false, false
|
||||
}
|
||||
return FetchResponse{}, Result{
|
||||
UpstreamID: r.config.UpstreamID,
|
||||
Class: upstream.FetchError,
|
||||
Err: resultErr,
|
||||
Err: fmt.Errorf("acquire distributed fetch capacity: %w", err),
|
||||
Attempt: attempt,
|
||||
}, false, true
|
||||
}
|
||||
if !available {
|
||||
return FetchResponse{}, Result{}, false, false
|
||||
}
|
||||
if permit == nil || permit.Expected() <= 0 {
|
||||
if permit != nil {
|
||||
_ = permit.Cancel()
|
||||
}
|
||||
if requestPermit == nil {
|
||||
return FetchResponse{}, Result{
|
||||
UpstreamID: r.config.UpstreamID,
|
||||
Class: upstream.FetchError,
|
||||
Err: fmt.Errorf("reserve fetch capacity: invalid permit"),
|
||||
Err: fmt.Errorf("acquire distributed fetch capacity: invalid permit"),
|
||||
Attempt: attempt,
|
||||
}, false, true
|
||||
}
|
||||
permitFinished := false
|
||||
requestSettlementAttempted := false
|
||||
defer func() {
|
||||
if !permitFinished {
|
||||
_ = permit.Cancel()
|
||||
if !requestSettlementAttempted {
|
||||
_ = r.settleRequestPermit(requestPermit, false, 0)
|
||||
}
|
||||
}()
|
||||
|
||||
@ -200,13 +250,23 @@ func (r *Reconciler) fetchAttempt(ctx context.Context, attempt int) (FetchRespon
|
||||
defer cancel()
|
||||
response, callErr := r.ports.Adapter.Fetch(callCtx)
|
||||
|
||||
var parseErr, candidateErr, capacityErr error
|
||||
var parseErr, candidateErr, coordinationErr error
|
||||
var validCount, newCount int
|
||||
if callErr == nil {
|
||||
if callErr != nil {
|
||||
requestSettlementAttempted = true
|
||||
coordinationErr = r.settleRequestPermit(requestPermit, true, permit.Expected())
|
||||
} else {
|
||||
candidates, err := r.ports.Parser.Parse(callCtx, response.Body)
|
||||
defer r.ports.Parser.ReleaseCandidates(candidates)
|
||||
parseErr = err
|
||||
validCount = len(candidates)
|
||||
if parseErr == nil && validCount > 0 {
|
||||
charged := validCount
|
||||
if parseErr != nil {
|
||||
charged = permit.Expected()
|
||||
}
|
||||
requestSettlementAttempted = true
|
||||
coordinationErr = r.settleRequestPermit(requestPermit, true, charged)
|
||||
if parseErr == nil && coordinationErr == nil && validCount > 0 {
|
||||
retained := candidates
|
||||
if expected := permit.Expected(); expected < len(retained) {
|
||||
retained = retained[:expected]
|
||||
@ -222,12 +282,13 @@ func (r *Reconciler) fetchAttempt(ctx context.Context, attempt int) (FetchRespon
|
||||
newCount = upserted.Inserted
|
||||
}
|
||||
}
|
||||
if callErr == nil && parseErr == nil && candidateErr == nil {
|
||||
capacityErr = permit.Complete(validCount, newCount)
|
||||
permitFinished = capacityErr == nil
|
||||
}
|
||||
resultErr := errors.Join(callErr, parseErr, candidateErr, capacityErr)
|
||||
class := upstream.ClassifyFetchResult(callErr, errors.Join(parseErr, candidateErr, capacityErr), validCount, newCount)
|
||||
resultErr := errors.Join(callErr, parseErr, candidateErr, coordinationErr)
|
||||
class := upstream.ClassifyFetchResult(
|
||||
callErr,
|
||||
errors.Join(parseErr, candidateErr, coordinationErr),
|
||||
validCount,
|
||||
newCount,
|
||||
)
|
||||
return response, Result{
|
||||
UpstreamID: r.config.UpstreamID,
|
||||
Class: class,
|
||||
@ -238,6 +299,15 @@ func (r *Reconciler) fetchAttempt(ctx context.Context, attempt int) (FetchRespon
|
||||
}, isRetryable(callErr) || parseErr != nil, true
|
||||
}
|
||||
|
||||
func (r *Reconciler) settleRequestPermit(permit RequestPermit, complete bool, fetched int) error {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), requestPermitSettlementTimeout)
|
||||
defer cancel()
|
||||
if complete {
|
||||
return permit.Complete(ctx, fetched)
|
||||
}
|
||||
return permit.Cancel(ctx)
|
||||
}
|
||||
|
||||
func isRetryable(err error) bool {
|
||||
if err == nil {
|
||||
return false
|
||||
|
||||
@ -8,6 +8,7 @@ import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
controllerPool "proxy-pool/internal/controller/pool"
|
||||
"proxy-pool/internal/domain/activitypool"
|
||||
proxyDomain "proxy-pool/internal/domain/proxy"
|
||||
"proxy-pool/internal/domain/upstream"
|
||||
@ -89,7 +90,7 @@ func TestReconcilerCoalescesConcurrentNotifications(t *testing.T) {
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
done := make(chan error, 1)
|
||||
go func() { done <- reconciler.Run(ctx) }()
|
||||
go func() { done <- reconciler.RunLeader(ctx, unlimitedLeaderSession{}) }()
|
||||
|
||||
select {
|
||||
case got := <-result:
|
||||
@ -130,7 +131,7 @@ func TestReconcilerEnforcesRequestInterval(t *testing.T) {
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
done := make(chan error, 1)
|
||||
go func() { done <- reconciler.Run(ctx) }()
|
||||
go func() { done <- reconciler.RunLeader(ctx, unlimitedLeaderSession{}) }()
|
||||
|
||||
reconciler.Notify()
|
||||
<-results
|
||||
@ -183,7 +184,7 @@ func TestReconcilerRetriesErrorsWithExponentialBackoffAndJitter(t *testing.T) {
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
done := make(chan error, 1)
|
||||
go func() { done <- reconciler.Run(ctx) }()
|
||||
go func() { done <- reconciler.RunLeader(ctx, unlimitedLeaderSession{}) }()
|
||||
reconciler.Notify()
|
||||
|
||||
wantClasses := []upstream.FetchClass{
|
||||
@ -291,7 +292,7 @@ func TestReconcilerHonorsRetryAfterBeforeBackoff(t *testing.T) {
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
done := make(chan error, 1)
|
||||
go func() { done <- reconciler.Run(ctx) }()
|
||||
go func() { done <- reconciler.RunLeader(ctx, unlimitedLeaderSession{}) }()
|
||||
reconciler.Notify()
|
||||
<-results
|
||||
<-results
|
||||
@ -332,7 +333,7 @@ func TestReconcilerCapsRetryAfter(t *testing.T) {
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
done := make(chan error, 1)
|
||||
go func() { done <- reconciler.Run(ctx) }()
|
||||
go func() { done <- reconciler.RunLeader(ctx, unlimitedLeaderSession{}) }()
|
||||
reconciler.Notify()
|
||||
<-results
|
||||
<-results
|
||||
@ -417,7 +418,7 @@ func TestReconcilerDropsNotificationFanoutWhileFetchIsInFlight(t *testing.T) {
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
done := make(chan error, 1)
|
||||
go func() { done <- reconciler.Run(ctx) }()
|
||||
go func() { done <- reconciler.RunLeader(ctx, unlimitedLeaderSession{}) }()
|
||||
reconciler.Notify()
|
||||
<-started
|
||||
|
||||
@ -473,8 +474,8 @@ func TestReconcilerEnforcesMaxInFlightAcrossRunConsumers(t *testing.T) {
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
done := make(chan error, 2)
|
||||
go func() { done <- reconciler.Run(ctx) }()
|
||||
go func() { done <- reconciler.Run(ctx) }()
|
||||
go func() { done <- reconciler.RunLeader(ctx, unlimitedLeaderSession{}) }()
|
||||
go func() { done <- reconciler.RunLeader(ctx, unlimitedLeaderSession{}) }()
|
||||
|
||||
reconciler.Notify()
|
||||
<-started
|
||||
@ -519,7 +520,7 @@ func TestReconcilerUsesConfiguredMaxInFlight(t *testing.T) {
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
done := make(chan error, 1)
|
||||
go func() { done <- reconciler.Run(ctx) }()
|
||||
go func() { done <- reconciler.RunLeader(ctx, unlimitedLeaderSession{}) }()
|
||||
reconciler.Notify()
|
||||
<-started
|
||||
reconciler.Notify()
|
||||
@ -545,6 +546,7 @@ func TestReconcilerDoesNotRefetchWhenActivitySinkFails(t *testing.T) {
|
||||
clock := newFakeClock(time.Date(2026, 7, 28, 12, 0, 0, 0, time.UTC))
|
||||
sleeper := &fakeSleeper{clock: clock}
|
||||
results := make(chan Result, 3)
|
||||
globalCompleted := make(chan int, 1)
|
||||
var calls atomic.Int64
|
||||
ports := successfulPorts(func() { calls.Add(1) }, results)
|
||||
ports.Activity = activitySinkFunc(func(context.Context, string, activitypool.FetchedBatch) (activitypool.UpsertResult, error) {
|
||||
@ -562,13 +564,20 @@ func TestReconcilerDoesNotRefetchWhenActivitySinkFails(t *testing.T) {
|
||||
t.Fatalf("NewReconciler(): %v", err)
|
||||
}
|
||||
|
||||
result := runSingleReconcile(t, reconciler, results)
|
||||
result := runSingleReconcile(t, reconciler, results, leaderSessionFunc(
|
||||
func(context.Context, int) (RequestPermit, bool, error) {
|
||||
return &recordingRequestPermit{completed: globalCompleted}, true, nil
|
||||
},
|
||||
))
|
||||
if result.Class != upstream.FetchError {
|
||||
t.Fatalf("result class = %q, want %q", result.Class, upstream.FetchError)
|
||||
}
|
||||
if got := calls.Load(); got != 1 {
|
||||
t.Fatalf("ProviderAdapter.Fetch() calls = %d, want 1", got)
|
||||
}
|
||||
if got := <-globalCompleted; got != 1 {
|
||||
t.Fatalf("global charged count = %d, want fetched=1", got)
|
||||
}
|
||||
if got := sleeper.Durations(); len(got) != 0 {
|
||||
t.Fatalf("Sleep durations = %v, want no retry backoff", got)
|
||||
}
|
||||
@ -623,7 +632,7 @@ func TestReconcilerDoesNotCallProviderWithoutFetchCapacity(t *testing.T) {
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
done := make(chan error, 1)
|
||||
go func() { done <- reconciler.Run(ctx) }()
|
||||
go func() { done <- reconciler.RunLeader(ctx, unlimitedLeaderSession{}) }()
|
||||
reconciler.Notify()
|
||||
time.Sleep(20 * time.Millisecond)
|
||||
cancel()
|
||||
@ -640,13 +649,116 @@ func TestReconcilerDoesNotCallProviderWithoutFetchCapacity(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestReconcilerCompletesFetchPermitWithFetchedAndRetainedCounts(t *testing.T) {
|
||||
func TestReconcilerDoesNotCallProviderWhenDistributedQuotaIsExhausted(t *testing.T) {
|
||||
results := make(chan Result, 1)
|
||||
completed := make(chan fetchCompletion, 1)
|
||||
var calls atomic.Int64
|
||||
ports := successfulPorts(func() { calls.Add(1) }, results)
|
||||
ports.Capacity = fetchCapacityFunc(func(string) (upstream.FetchPermit, bool, error) {
|
||||
return &recordingFetchPermit{expected: 2}, true, nil
|
||||
})
|
||||
reconciler, err := NewReconciler(Config{
|
||||
UpstreamID: "provider-a", Timeout: time.Second, MaxAttempts: 1, MaxInFlight: 1, MaxSize: 100,
|
||||
}, ports)
|
||||
if err != nil {
|
||||
t.Fatalf("NewReconciler(): %v", err)
|
||||
}
|
||||
session := leaderSessionFunc(func(_ context.Context, expected int) (RequestPermit, bool, error) {
|
||||
if expected != 2 {
|
||||
t.Errorf("distributed expected = %d, want 2", expected)
|
||||
}
|
||||
return nil, false, nil
|
||||
})
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
done := make(chan error, 1)
|
||||
go func() { done <- reconciler.RunLeader(ctx, session) }()
|
||||
reconciler.Notify()
|
||||
time.Sleep(20 * time.Millisecond)
|
||||
cancel()
|
||||
if err := <-done; err != nil {
|
||||
t.Fatalf("RunLeader(): %v", err)
|
||||
}
|
||||
if got := calls.Load(); got != 0 {
|
||||
t.Fatalf("ProviderAdapter.Fetch() calls = %d, want 0", got)
|
||||
}
|
||||
select {
|
||||
case result := <-results:
|
||||
t.Fatalf("unexpected fetch result: %+v", result)
|
||||
default:
|
||||
}
|
||||
}
|
||||
|
||||
func TestReconcilerChargesExpectedWhenSuccessfulResponseCannotBeParsed(t *testing.T) {
|
||||
results := make(chan Result, 1)
|
||||
globalCompleted := make(chan int, 1)
|
||||
localCancelled := make(chan struct{}, 1)
|
||||
ports := successfulPorts(func() {}, results)
|
||||
ports.Parser = parserFunc(func(context.Context, []byte) ([]proxyDomain.Proxy, error) {
|
||||
return []proxyDomain.Proxy{{ID: "p1"}, {ID: "p2"}, {ID: "p3"}}, nil
|
||||
return nil, errors.New("invalid provider payload")
|
||||
})
|
||||
ports.Capacity = fetchCapacityFunc(func(string) (upstream.FetchPermit, bool, error) {
|
||||
return &recordingFetchPermit{expected: 2, cancelled: localCancelled}, true, nil
|
||||
})
|
||||
reconciler, err := NewReconciler(Config{
|
||||
UpstreamID: "provider-a", Timeout: time.Second, MaxAttempts: 1, MaxInFlight: 1, MaxSize: 100,
|
||||
}, ports)
|
||||
if err != nil {
|
||||
t.Fatalf("NewReconciler(): %v", err)
|
||||
}
|
||||
session := leaderSessionFunc(func(context.Context, int) (RequestPermit, bool, error) {
|
||||
return &recordingRequestPermit{completed: globalCompleted}, true, nil
|
||||
})
|
||||
|
||||
result := runSingleReconcile(t, reconciler, results, session)
|
||||
if result.Class != upstream.FetchError {
|
||||
t.Fatalf("result class = %q, want %q", result.Class, upstream.FetchError)
|
||||
}
|
||||
if got := <-globalCompleted; got != 2 {
|
||||
t.Fatalf("global charged count = %d, want expected=2", got)
|
||||
}
|
||||
select {
|
||||
case <-localCancelled:
|
||||
default:
|
||||
t.Fatal("local pool reservation was not cancelled")
|
||||
}
|
||||
}
|
||||
|
||||
func TestReconcilerConservativelyChargesExpectedWhenProviderCallOutcomeIsUnknown(t *testing.T) {
|
||||
results := make(chan Result, 1)
|
||||
globalCompleted := make(chan int, 1)
|
||||
ports := successfulPorts(func() {}, results)
|
||||
ports.Adapter = adapterFunc(func(context.Context) (FetchResponse, error) {
|
||||
return FetchResponse{}, errors.New("provider connection failed")
|
||||
})
|
||||
reconciler, err := NewReconciler(Config{
|
||||
UpstreamID: "provider-a", Timeout: time.Second, MaxAttempts: 1, MaxInFlight: 1, MaxSize: 100,
|
||||
}, ports)
|
||||
if err != nil {
|
||||
t.Fatalf("NewReconciler(): %v", err)
|
||||
}
|
||||
session := leaderSessionFunc(func(context.Context, int) (RequestPermit, bool, error) {
|
||||
return &recordingRequestPermit{completed: globalCompleted}, true, nil
|
||||
})
|
||||
|
||||
result := runSingleReconcile(t, reconciler, results, session)
|
||||
if result.Class != upstream.FetchError {
|
||||
t.Fatalf("result class = %q, want %q", result.Class, upstream.FetchError)
|
||||
}
|
||||
if got := <-globalCompleted; got <= 0 {
|
||||
t.Fatalf("global charged count = %d, want conservative expected count", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestReconcilerChargesFetchedGloballyAndCompletesRetainedLocally(t *testing.T) {
|
||||
results := make(chan Result, 1)
|
||||
localCompleted := make(chan fetchCompletion, 1)
|
||||
globalCompleted := make(chan int, 1)
|
||||
released := make(chan []proxyDomain.Proxy, 1)
|
||||
ports := successfulPorts(func() {}, results)
|
||||
ports.Parser = &recordingCandidateParser{
|
||||
candidates: []proxyDomain.Proxy{{ID: "p1"}, {ID: "p2"}, {ID: "p3"}},
|
||||
released: released,
|
||||
}
|
||||
ports.Activity = activitySinkFunc(func(_ context.Context, _ string, batch activitypool.FetchedBatch) (activitypool.UpsertResult, error) {
|
||||
if len(batch.Proxies) != 2 {
|
||||
t.Errorf("activity batch proxies = %d, want permit limit 2", len(batch.Proxies))
|
||||
@ -654,7 +766,7 @@ func TestReconcilerCompletesFetchPermitWithFetchedAndRetainedCounts(t *testing.T
|
||||
return activitypool.UpsertResult{Accepted: 2, Inserted: 1}, nil
|
||||
})
|
||||
ports.Capacity = fetchCapacityFunc(func(string) (upstream.FetchPermit, bool, error) {
|
||||
return &recordingFetchPermit{expected: 2, completed: completed}, true, nil
|
||||
return &recordingFetchPermit{expected: 2, completed: localCompleted}, true, nil
|
||||
})
|
||||
reconciler, err := NewReconciler(Config{
|
||||
UpstreamID: "provider-a", Timeout: time.Second, MaxAttempts: 1, MaxInFlight: 1, MaxSize: 100,
|
||||
@ -663,12 +775,30 @@ func TestReconcilerCompletesFetchPermitWithFetchedAndRetainedCounts(t *testing.T
|
||||
t.Fatalf("NewReconciler(): %v", err)
|
||||
}
|
||||
|
||||
result := runSingleReconcile(t, reconciler, results)
|
||||
result := runSingleReconcile(t, reconciler, results, leaderSessionFunc(
|
||||
func(_ context.Context, expected int) (RequestPermit, bool, error) {
|
||||
if expected != 2 {
|
||||
t.Errorf("distributed expected = %d, want 2", expected)
|
||||
}
|
||||
return &recordingRequestPermit{completed: globalCompleted}, true, nil
|
||||
},
|
||||
))
|
||||
if result.ValidCount != 3 || result.NewCount != 1 {
|
||||
t.Fatalf("result = %+v, want valid=3 new=1", result)
|
||||
}
|
||||
if got := <-completed; got.fetched != 3 || got.retained != 1 {
|
||||
t.Fatalf("fetch completion = %+v, want fetched=3 retained=1", got)
|
||||
if got := <-globalCompleted; got != 3 {
|
||||
t.Fatalf("global fetch completion = %d, want fetched=3", got)
|
||||
}
|
||||
if got := <-localCompleted; got.retained != 1 {
|
||||
t.Fatalf("local fetch completion = %+v, want retained=1", got)
|
||||
}
|
||||
select {
|
||||
case got := <-released:
|
||||
if len(got) != 3 {
|
||||
t.Fatalf("released candidates = %d, want all 3 parsed candidates", len(got))
|
||||
}
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("parsed candidates were not released")
|
||||
}
|
||||
}
|
||||
|
||||
@ -699,7 +829,7 @@ func TestReconcilerRetriesParserErrorsAsFetchErrors(t *testing.T) {
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
done := make(chan error, 1)
|
||||
go func() { done <- reconciler.Run(ctx) }()
|
||||
go func() { done <- reconciler.RunLeader(ctx, unlimitedLeaderSession{}) }()
|
||||
reconciler.Notify()
|
||||
first, second := <-results, <-results
|
||||
cancel()
|
||||
@ -717,6 +847,53 @@ func TestReconcilerRetriesParserErrorsAsFetchErrors(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestReconcilerKeepsLocalPoolReservationAcrossRetryBackoff(t *testing.T) {
|
||||
results := make(chan Result, 2)
|
||||
budget, err := controllerPool.NewFetchBudget(controllerPool.FetchBudgetConfig{
|
||||
UpstreamID: "provider-a", MaxSize: 10, ExpectedPerFetch: 1,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("NewFetchBudget(): %v", err)
|
||||
}
|
||||
var parses atomic.Int64
|
||||
ports := successfulPorts(func() {}, results)
|
||||
ports.Capacity = budget
|
||||
ports.Parser = parserFunc(func(context.Context, []byte) ([]proxyDomain.Proxy, error) {
|
||||
if parses.Add(1) == 1 {
|
||||
return nil, errors.New("temporary parser failure")
|
||||
}
|
||||
return []proxyDomain.Proxy{{ID: "proxy-1"}}, nil
|
||||
})
|
||||
reconciler, err := NewReconciler(Config{
|
||||
UpstreamID: "provider-a", Timeout: time.Second, MaxAttempts: 2,
|
||||
MaxInFlight: 1, MaxSize: 10,
|
||||
Retry: RetryConfig{Initial: time.Millisecond, Max: time.Millisecond},
|
||||
}, ports, Runtime{Sleeper: sleeperFunc(func(context.Context, time.Duration) error {
|
||||
if got := budget.Snapshot().PendingExpected; got != 1 {
|
||||
t.Errorf("PendingExpected during retry backoff = %d, want 1", got)
|
||||
}
|
||||
return nil
|
||||
})})
|
||||
if err != nil {
|
||||
t.Fatalf("NewReconciler(): %v", err)
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
done := make(chan error, 1)
|
||||
go func() { done <- reconciler.RunLeader(ctx, unlimitedLeaderSession{}) }()
|
||||
reconciler.Notify()
|
||||
<-results
|
||||
<-results
|
||||
cancel()
|
||||
if err := <-done; err != nil {
|
||||
t.Fatalf("RunLeader(): %v", err)
|
||||
}
|
||||
usage := budget.Snapshot()
|
||||
if usage.PendingExpected != 0 || usage.Managed != 1 {
|
||||
t.Fatalf("FetchBudget snapshot = %+v, want pending=0 managed=1", usage)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewReconcilerRejectsInvalidSchedulingConfiguration(t *testing.T) {
|
||||
results := make(chan Result, 1)
|
||||
ports := successfulPorts(func() {}, results)
|
||||
@ -744,11 +921,23 @@ func TestNewReconcilerRejectsInvalidSchedulingConfiguration(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func runSingleReconcile(t *testing.T, reconciler *Reconciler, results <-chan Result) Result {
|
||||
func runSingleReconcile(
|
||||
t *testing.T,
|
||||
reconciler *Reconciler,
|
||||
results <-chan Result,
|
||||
sessions ...LeaderSession,
|
||||
) Result {
|
||||
t.Helper()
|
||||
session := LeaderSession(unlimitedLeaderSession{})
|
||||
if len(sessions) > 1 {
|
||||
t.Fatal("runSingleReconcile accepts at most one LeaderSession")
|
||||
}
|
||||
if len(sessions) == 1 {
|
||||
session = sessions[0]
|
||||
}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
done := make(chan error, 1)
|
||||
go func() { done <- reconciler.Run(ctx) }()
|
||||
go func() { done <- reconciler.RunLeader(ctx, session) }()
|
||||
reconciler.Notify()
|
||||
var result Result
|
||||
select {
|
||||
@ -821,6 +1010,12 @@ func (s *errorSleeper) Sleep(context.Context, time.Duration) error {
|
||||
return errors.New("unexpected sleep")
|
||||
}
|
||||
|
||||
type sleeperFunc func(context.Context, time.Duration) error
|
||||
|
||||
func (f sleeperFunc) Sleep(ctx context.Context, duration time.Duration) error {
|
||||
return f(ctx, duration)
|
||||
}
|
||||
|
||||
func (s *fakeSleeper) Sleep(ctx context.Context, duration time.Duration) error {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
@ -848,6 +1043,21 @@ func (f parserFunc) Parse(ctx context.Context, body []byte) ([]proxyDomain.Proxy
|
||||
return f(ctx, body)
|
||||
}
|
||||
|
||||
func (parserFunc) ReleaseCandidates([]proxyDomain.Proxy) {}
|
||||
|
||||
type recordingCandidateParser struct {
|
||||
candidates []proxyDomain.Proxy
|
||||
released chan<- []proxyDomain.Proxy
|
||||
}
|
||||
|
||||
func (parser *recordingCandidateParser) Parse(context.Context, []byte) ([]proxyDomain.Proxy, error) {
|
||||
return append([]proxyDomain.Proxy(nil), parser.candidates...), nil
|
||||
}
|
||||
|
||||
func (parser *recordingCandidateParser) ReleaseCandidates(candidates []proxyDomain.Proxy) {
|
||||
parser.released <- append([]proxyDomain.Proxy(nil), candidates...)
|
||||
}
|
||||
|
||||
type activitySinkFunc func(context.Context, string, activitypool.FetchedBatch) (activitypool.UpsertResult, error)
|
||||
|
||||
func (f activitySinkFunc) UpsertFetched(ctx context.Context, upstreamID string, batch activitypool.FetchedBatch) (activitypool.UpsertResult, error) {
|
||||
@ -867,26 +1077,66 @@ func (unlimitedFetchCapacity) ReserveFetch(string) (upstream.FetchPermit, bool,
|
||||
}
|
||||
|
||||
type fetchCompletion struct {
|
||||
fetched int
|
||||
retained int
|
||||
}
|
||||
|
||||
type recordingFetchPermit struct {
|
||||
expected int
|
||||
completed chan<- fetchCompletion
|
||||
cancelled chan<- struct{}
|
||||
}
|
||||
|
||||
func (p *recordingFetchPermit) Expected() int { return p.expected }
|
||||
|
||||
func (p *recordingFetchPermit) Complete(fetched, retained int) error {
|
||||
func (p *recordingFetchPermit) Complete(retained int) error {
|
||||
if p.completed != nil {
|
||||
p.completed <- fetchCompletion{fetched: fetched, retained: retained}
|
||||
p.completed <- fetchCompletion{retained: retained}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (*recordingFetchPermit) Cancel() error { return nil }
|
||||
func (p *recordingFetchPermit) Cancel() error {
|
||||
if p.cancelled != nil {
|
||||
p.cancelled <- struct{}{}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
type resultRecorderFunc func(Result)
|
||||
|
||||
func (f resultRecorderFunc) Record(result Result) { f(result) }
|
||||
|
||||
type unlimitedLeaderSession struct{}
|
||||
|
||||
func (unlimitedLeaderSession) Fence() Fence { return Fence{Generation: "test", Epoch: 1} }
|
||||
|
||||
func (unlimitedLeaderSession) AcquireFetch(context.Context, int) (RequestPermit, bool, error) {
|
||||
return &recordingRequestPermit{}, true, nil
|
||||
}
|
||||
|
||||
type leaderSessionFunc func(context.Context, int) (RequestPermit, bool, error)
|
||||
|
||||
func (leaderSessionFunc) Fence() Fence { return Fence{Generation: "test", Epoch: 1} }
|
||||
|
||||
func (f leaderSessionFunc) AcquireFetch(ctx context.Context, expected int) (RequestPermit, bool, error) {
|
||||
return f(ctx, expected)
|
||||
}
|
||||
|
||||
type recordingRequestPermit struct {
|
||||
completed chan<- int
|
||||
cancelled chan<- struct{}
|
||||
}
|
||||
|
||||
func (p *recordingRequestPermit) Complete(_ context.Context, fetched int) error {
|
||||
if p.completed != nil {
|
||||
p.completed <- fetched
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (p *recordingRequestPermit) Cancel(context.Context) error {
|
||||
if p.cancelled != nil {
|
||||
p.cancelled <- struct{}{}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
103
internal/controller/provider/stats.go
Normal file
103
internal/controller/provider/stats.go
Normal file
@ -0,0 +1,103 @@
|
||||
package provider
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"math"
|
||||
"sync"
|
||||
|
||||
"proxy-pool/internal/domain/upstream"
|
||||
)
|
||||
|
||||
var ErrInvalidStatsRecorder = errors.New("invalid Provider stats recorder")
|
||||
|
||||
type Stats struct {
|
||||
UpstreamID string
|
||||
ConsecutiveEmptyFetch int64
|
||||
FetchErrorCount int64
|
||||
}
|
||||
|
||||
type StatsReader interface {
|
||||
ReadProviderStats([]string) []Stats
|
||||
}
|
||||
|
||||
type StatsRetainer interface {
|
||||
RetainProviderStats([]string)
|
||||
}
|
||||
|
||||
type StatsRecorder struct {
|
||||
mu sync.Mutex
|
||||
maximum int
|
||||
byID map[string]Stats
|
||||
}
|
||||
|
||||
func NewStatsRecorder(maximum int) (*StatsRecorder, error) {
|
||||
if maximum <= 0 {
|
||||
return nil, ErrInvalidStatsRecorder
|
||||
}
|
||||
return &StatsRecorder{maximum: maximum, byID: make(map[string]Stats)}, nil
|
||||
}
|
||||
|
||||
func (recorder *StatsRecorder) Record(result Result) {
|
||||
if recorder == nil || result.UpstreamID == "" {
|
||||
return
|
||||
}
|
||||
recorder.mu.Lock()
|
||||
defer recorder.mu.Unlock()
|
||||
stats, exists := recorder.byID[result.UpstreamID]
|
||||
if !exists {
|
||||
if len(recorder.byID) >= recorder.maximum {
|
||||
return
|
||||
}
|
||||
stats.UpstreamID = result.UpstreamID
|
||||
}
|
||||
switch result.Class {
|
||||
case upstream.FetchEmpty:
|
||||
if stats.ConsecutiveEmptyFetch < math.MaxInt64 {
|
||||
stats.ConsecutiveEmptyFetch++
|
||||
}
|
||||
case upstream.FetchValid, upstream.FetchDuplicateOnly:
|
||||
stats.ConsecutiveEmptyFetch = 0
|
||||
case upstream.FetchError:
|
||||
if stats.FetchErrorCount < math.MaxInt64 {
|
||||
stats.FetchErrorCount++
|
||||
}
|
||||
default:
|
||||
return
|
||||
}
|
||||
recorder.byID[result.UpstreamID] = stats
|
||||
}
|
||||
|
||||
func (recorder *StatsRecorder) ReadProviderStats(upstreamIDs []string) []Stats {
|
||||
result := make([]Stats, len(upstreamIDs))
|
||||
if recorder == nil {
|
||||
return result
|
||||
}
|
||||
recorder.mu.Lock()
|
||||
defer recorder.mu.Unlock()
|
||||
for index, upstreamID := range upstreamIDs {
|
||||
result[index] = recorder.byID[upstreamID]
|
||||
result[index].UpstreamID = upstreamID
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
// RetainProviderStats removes observations for upstreams no longer present in
|
||||
// the complete configuration. Disabled but configured upstreams must be kept.
|
||||
func (recorder *StatsRecorder) RetainProviderStats(upstreamIDs []string) {
|
||||
if recorder == nil {
|
||||
return
|
||||
}
|
||||
retained := make(map[string]struct{}, len(upstreamIDs))
|
||||
for _, upstreamID := range upstreamIDs {
|
||||
if upstreamID != "" {
|
||||
retained[upstreamID] = struct{}{}
|
||||
}
|
||||
}
|
||||
recorder.mu.Lock()
|
||||
defer recorder.mu.Unlock()
|
||||
for upstreamID := range recorder.byID {
|
||||
if _, keep := retained[upstreamID]; !keep {
|
||||
delete(recorder.byID, upstreamID)
|
||||
}
|
||||
}
|
||||
}
|
||||
83
internal/controller/provider/stats_test.go
Normal file
83
internal/controller/provider/stats_test.go
Normal file
@ -0,0 +1,83 @@
|
||||
package provider
|
||||
|
||||
import (
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"proxy-pool/internal/domain/upstream"
|
||||
)
|
||||
|
||||
func TestStatsRecorderTracksEmptyResetAndErrors(t *testing.T) {
|
||||
recorder, err := NewStatsRecorder(2)
|
||||
if err != nil {
|
||||
t.Fatalf("NewStatsRecorder(): %v", err)
|
||||
}
|
||||
for _, class := range []upstream.FetchClass{
|
||||
upstream.FetchEmpty,
|
||||
upstream.FetchEmpty,
|
||||
upstream.FetchError,
|
||||
} {
|
||||
recorder.Record(Result{UpstreamID: "provider-a", Class: class})
|
||||
}
|
||||
stats := recorder.ReadProviderStats([]string{"provider-a"})[0]
|
||||
if stats.ConsecutiveEmptyFetch != 2 || stats.FetchErrorCount != 1 {
|
||||
t.Fatalf("stats = %+v, want empty=2 errors=1", stats)
|
||||
}
|
||||
recorder.Record(Result{UpstreamID: "provider-a", Class: upstream.FetchDuplicateOnly})
|
||||
if got := recorder.ReadProviderStats([]string{"provider-a"})[0].ConsecutiveEmptyFetch; got != 0 {
|
||||
t.Fatalf("consecutive empty after duplicate = %d, want 0", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStatsRecorderIsBoundedAndConcurrent(t *testing.T) {
|
||||
recorder, err := NewStatsRecorder(1)
|
||||
if err != nil {
|
||||
t.Fatalf("NewStatsRecorder(): %v", err)
|
||||
}
|
||||
const workers = 100
|
||||
var wait sync.WaitGroup
|
||||
for range workers {
|
||||
wait.Add(1)
|
||||
go func() {
|
||||
defer wait.Done()
|
||||
recorder.Record(Result{UpstreamID: "provider-a", Class: upstream.FetchError})
|
||||
}()
|
||||
}
|
||||
wait.Wait()
|
||||
recorder.Record(Result{UpstreamID: "provider-b", Class: upstream.FetchError})
|
||||
stats := recorder.ReadProviderStats([]string{"provider-a", "provider-b"})
|
||||
if stats[0].FetchErrorCount != workers || stats[1].FetchErrorCount != 0 {
|
||||
t.Fatalf("stats = %+v, want bounded provider-a errors", stats)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStatsRecorderRetainsConfiguredProvidersAndReusesCapacity(t *testing.T) {
|
||||
recorder, err := NewStatsRecorder(2)
|
||||
if err != nil {
|
||||
t.Fatalf("NewStatsRecorder(): %v", err)
|
||||
}
|
||||
recorder.Record(Result{UpstreamID: "removed-a", Class: upstream.FetchError})
|
||||
recorder.Record(Result{UpstreamID: "provider-b", Class: upstream.FetchError})
|
||||
|
||||
const workers = 100
|
||||
var wait sync.WaitGroup
|
||||
for range workers {
|
||||
wait.Add(2)
|
||||
go func() {
|
||||
defer wait.Done()
|
||||
recorder.Record(Result{UpstreamID: "provider-b", Class: upstream.FetchError})
|
||||
}()
|
||||
go func() {
|
||||
defer wait.Done()
|
||||
recorder.RetainProviderStats([]string{"provider-a", "provider-b"})
|
||||
}()
|
||||
}
|
||||
wait.Wait()
|
||||
recorder.RetainProviderStats([]string{"provider-a", "provider-b"})
|
||||
recorder.Record(Result{UpstreamID: "provider-a", Class: upstream.FetchError})
|
||||
|
||||
stats := recorder.ReadProviderStats([]string{"removed-a", "provider-a", "provider-b"})
|
||||
if stats[0].FetchErrorCount != 0 || stats[1].FetchErrorCount != 1 || stats[2].FetchErrorCount == 0 {
|
||||
t.Fatalf("stats after retention = %+v", stats)
|
||||
}
|
||||
}
|
||||
219
internal/controller/provider/upstream_runtime.go
Normal file
219
internal/controller/provider/upstream_runtime.go
Normal file
@ -0,0 +1,219 @@
|
||||
package provider
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"hash/fnv"
|
||||
"reflect"
|
||||
"time"
|
||||
|
||||
"proxy-pool/internal/controller/pool"
|
||||
"proxy-pool/internal/domain/activitypool"
|
||||
)
|
||||
|
||||
var (
|
||||
ErrInvalidUpstreamRuntime = errors.New("invalid Provider upstream runtime")
|
||||
ErrUpstreamRuntimeStopped = errors.New("Provider upstream runtime stopped")
|
||||
)
|
||||
|
||||
type UpstreamRuntimeConfig struct {
|
||||
Provider Config
|
||||
ReconcilePolicy pool.ReconcilePolicy
|
||||
ReconcileInterval time.Duration
|
||||
}
|
||||
|
||||
type UpstreamRuntimeDependencies struct {
|
||||
Coordinator Coordinator
|
||||
Inventory pool.InventoryReader
|
||||
Adapter ProviderAdapter
|
||||
Parser Parser
|
||||
Activity activitypool.Upserter
|
||||
Results ResultRecorder
|
||||
}
|
||||
|
||||
// UpstreamRuntime owns every leader-scoped object for one Upstream. A fresh
|
||||
// local budget and coalescing signal are created for each leadership term.
|
||||
type UpstreamRuntime struct {
|
||||
config UpstreamRuntimeConfig
|
||||
dependencies UpstreamRuntimeDependencies
|
||||
sleeper Sleeper
|
||||
}
|
||||
|
||||
func NewUpstreamRuntime(
|
||||
config UpstreamRuntimeConfig,
|
||||
dependencies UpstreamRuntimeDependencies,
|
||||
) (*UpstreamRuntime, error) {
|
||||
if config.ReconcileInterval <= 0 || nilRuntimeDependency(dependencies.Coordinator) ||
|
||||
nilRuntimeDependency(dependencies.Inventory) || nilRuntimeDependency(dependencies.Adapter) ||
|
||||
nilRuntimeDependency(dependencies.Parser) || nilRuntimeDependency(dependencies.Activity) ||
|
||||
nilRuntimeDependency(dependencies.Results) {
|
||||
return nil, ErrInvalidUpstreamRuntime
|
||||
}
|
||||
runtime := &UpstreamRuntime{
|
||||
config: config, dependencies: dependencies, sleeper: timerSleeper{},
|
||||
}
|
||||
if _, err := runtime.newLeaderTerm(); err != nil {
|
||||
return nil, errors.Join(ErrInvalidUpstreamRuntime, err)
|
||||
}
|
||||
return runtime, nil
|
||||
}
|
||||
|
||||
func (runtime *UpstreamRuntime) Run(ctx context.Context) error {
|
||||
if runtime == nil || ctx == nil {
|
||||
return ErrInvalidUpstreamRuntime
|
||||
}
|
||||
limits := CoordinationLimits{
|
||||
RequestInterval: runtime.config.Provider.RequestInterval,
|
||||
MaxInFlight: runtime.config.Provider.MaxInFlight,
|
||||
MaxAttemptDuration: runtime.config.Provider.Timeout,
|
||||
MaxTotal: runtime.config.Provider.MaxTotal,
|
||||
}
|
||||
err := runtime.dependencies.Coordinator.RunLeader(
|
||||
ctx,
|
||||
runtime.config.Provider.UpstreamID,
|
||||
limits,
|
||||
func(leaderCtx context.Context, session LeaderSession) error {
|
||||
term, buildErr := runtime.newLeaderTerm()
|
||||
if buildErr != nil {
|
||||
return buildErr
|
||||
}
|
||||
return term.run(leaderCtx, session)
|
||||
},
|
||||
)
|
||||
if ctx.Err() != nil {
|
||||
return ctx.Err()
|
||||
}
|
||||
if err == nil {
|
||||
return ErrUpstreamRuntimeStopped
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
func (runtime *UpstreamRuntime) ID() string {
|
||||
if runtime == nil {
|
||||
return ""
|
||||
}
|
||||
return runtime.config.Provider.UpstreamID
|
||||
}
|
||||
|
||||
func (runtime *UpstreamRuntime) newLeaderTerm() (*upstreamLeaderTerm, error) {
|
||||
budget, err := pool.NewFetchBudget(pool.FetchBudgetConfig{
|
||||
UpstreamID: runtime.config.Provider.UpstreamID,
|
||||
MaxSize: runtime.config.Provider.MaxSize,
|
||||
ExpectedPerFetch: runtime.config.ReconcilePolicy.ExpectedPerFetch,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
providerConfig := runtime.config.Provider
|
||||
providerConfig.RequestInterval = 0
|
||||
providerReconciler, err := NewReconciler(providerConfig, Ports{
|
||||
Adapter: runtime.dependencies.Adapter, Parser: runtime.dependencies.Parser,
|
||||
Activity: runtime.dependencies.Activity, Results: runtime.dependencies.Results,
|
||||
Capacity: budget,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
poolReconciler, err := pool.NewReconciler(
|
||||
runtime.config.ReconcilePolicy,
|
||||
budget,
|
||||
providerReconciler,
|
||||
)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &upstreamLeaderTerm{
|
||||
upstreamID: runtime.config.Provider.UpstreamID,
|
||||
interval: runtime.config.ReconcileInterval,
|
||||
safetyMargin: runtime.config.ReconcilePolicy.SafetyMargin,
|
||||
inventory: runtime.dependencies.Inventory,
|
||||
pool: poolReconciler,
|
||||
provider: providerReconciler,
|
||||
sleeper: runtime.sleeper,
|
||||
}, nil
|
||||
}
|
||||
|
||||
type upstreamLeaderTerm struct {
|
||||
upstreamID string
|
||||
interval time.Duration
|
||||
safetyMargin time.Duration
|
||||
inventory pool.InventoryReader
|
||||
pool *pool.Reconciler
|
||||
provider *Reconciler
|
||||
sleeper Sleeper
|
||||
}
|
||||
|
||||
func (term *upstreamLeaderTerm) run(ctx context.Context, session LeaderSession) error {
|
||||
termCtx, cancel := context.WithCancel(ctx)
|
||||
defer cancel()
|
||||
done := make(chan error, 2)
|
||||
go func() { done <- term.provider.RunLeader(termCtx, session) }()
|
||||
go func() { done <- term.reconcileInventory(termCtx) }()
|
||||
|
||||
first := <-done
|
||||
cancel()
|
||||
second := <-done
|
||||
if ctx.Err() != nil {
|
||||
return nil
|
||||
}
|
||||
if first == nil {
|
||||
first = ErrUpstreamRuntimeStopped
|
||||
}
|
||||
if second != nil && !errors.Is(second, context.Canceled) {
|
||||
return errors.Join(first, second)
|
||||
}
|
||||
return first
|
||||
}
|
||||
|
||||
func (term *upstreamLeaderTerm) reconcileInventory(ctx context.Context) error {
|
||||
if delay := initialReconcileDelay(term.upstreamID, term.interval); delay > 0 {
|
||||
if err := term.sleeper.Sleep(ctx, delay); err != nil {
|
||||
if ctx.Err() != nil {
|
||||
return nil
|
||||
}
|
||||
return fmt.Errorf("wait for initial Provider inventory reconciliation: %w", err)
|
||||
}
|
||||
}
|
||||
for ctx.Err() == nil {
|
||||
inventory, err := term.inventory.ReadInventory(
|
||||
ctx,
|
||||
term.upstreamID,
|
||||
term.safetyMargin,
|
||||
)
|
||||
if err == nil && inventory.Managed >= 0 && inventory.AvailableSlots >= 0 {
|
||||
term.pool.ReconcileSnapshot(inventory)
|
||||
}
|
||||
if err := term.sleeper.Sleep(ctx, term.interval); err != nil {
|
||||
if ctx.Err() != nil {
|
||||
return nil
|
||||
}
|
||||
return fmt.Errorf("wait for Provider inventory reconciliation: %w", err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func initialReconcileDelay(upstreamID string, interval time.Duration) time.Duration {
|
||||
window := min(interval/4, 250*time.Millisecond)
|
||||
if window <= 1 {
|
||||
return 0
|
||||
}
|
||||
digest := fnv.New64a()
|
||||
_, _ = digest.Write([]byte(upstreamID))
|
||||
return time.Duration(digest.Sum64() % uint64(window))
|
||||
}
|
||||
|
||||
func nilRuntimeDependency(value any) bool {
|
||||
if value == nil {
|
||||
return true
|
||||
}
|
||||
reflected := reflect.ValueOf(value)
|
||||
switch reflected.Kind() {
|
||||
case reflect.Chan, reflect.Func, reflect.Interface, reflect.Map, reflect.Pointer, reflect.Slice:
|
||||
return reflected.IsNil()
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
247
internal/controller/provider/upstream_runtime_test.go
Normal file
247
internal/controller/provider/upstream_runtime_test.go
Normal file
@ -0,0 +1,247 @@
|
||||
package provider
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"proxy-pool/internal/controller/pool"
|
||||
"proxy-pool/internal/domain/activitypool"
|
||||
proxyDomain "proxy-pool/internal/domain/proxy"
|
||||
)
|
||||
|
||||
func TestUpstreamRuntimeReadsInventoryAndFetchesOnlyInsideLeaderTerm(t *testing.T) {
|
||||
var inventoryReads atomic.Int64
|
||||
fetched := make(chan struct{}, 1)
|
||||
runtime, err := NewUpstreamRuntime(UpstreamRuntimeConfig{
|
||||
Provider: runtimeProviderConfig("provider-a"),
|
||||
ReconcilePolicy: runtimeReconcilePolicy(),
|
||||
ReconcileInterval: 10 * time.Millisecond,
|
||||
}, UpstreamRuntimeDependencies{
|
||||
Coordinator: coordinatorFunc(func(
|
||||
ctx context.Context,
|
||||
upstreamID string,
|
||||
limits CoordinationLimits,
|
||||
work func(context.Context, LeaderSession) error,
|
||||
) error {
|
||||
if upstreamID != "provider-a" || limits.MaxTotal != 10 || limits.MaxInFlight != 1 {
|
||||
t.Errorf("coordination = (%q, %+v)", upstreamID, limits)
|
||||
}
|
||||
return work(ctx, unlimitedLeaderSession{})
|
||||
}),
|
||||
Inventory: inventoryReaderFunc(func(context.Context, string, time.Duration) (pool.InventorySnapshot, error) {
|
||||
inventoryReads.Add(1)
|
||||
return pool.InventorySnapshot{}, nil
|
||||
}),
|
||||
Adapter: adapterFunc(func(context.Context) (FetchResponse, error) {
|
||||
fetched <- struct{}{}
|
||||
return FetchResponse{Body: []byte("fixture")}, nil
|
||||
}),
|
||||
Parser: parserFunc(func(context.Context, []byte) ([]proxyDomain.Proxy, error) {
|
||||
return []proxyDomain.Proxy{{ID: "proxy-1"}}, nil
|
||||
}),
|
||||
Activity: activitySinkFunc(func(context.Context, string, activitypool.FetchedBatch) (activitypool.UpsertResult, error) {
|
||||
return activitypool.UpsertResult{Accepted: 1, Inserted: 1}, nil
|
||||
}),
|
||||
Results: resultRecorderFunc(func(Result) {}),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("NewUpstreamRuntime(): %v", err)
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
done := make(chan error, 1)
|
||||
go func() { done <- runtime.Run(ctx) }()
|
||||
select {
|
||||
case <-fetched:
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("timed out waiting for leader fetch")
|
||||
}
|
||||
cancel()
|
||||
if err := <-done; !errors.Is(err, context.Canceled) {
|
||||
t.Fatalf("Run() error = %v, want context cancellation", err)
|
||||
}
|
||||
if got := inventoryReads.Load(); got == 0 {
|
||||
t.Fatal("inventory was not read inside leader term")
|
||||
}
|
||||
}
|
||||
|
||||
func TestUpstreamRuntimeFailsClosedUntilInventoryReadRecovers(t *testing.T) {
|
||||
var inventoryReads atomic.Int64
|
||||
var providerCalls atomic.Int64
|
||||
fetched := make(chan struct{}, 1)
|
||||
runtime, err := NewUpstreamRuntime(UpstreamRuntimeConfig{
|
||||
Provider: runtimeProviderConfig("provider-a"),
|
||||
ReconcilePolicy: runtimeReconcilePolicy(),
|
||||
ReconcileInterval: 10 * time.Millisecond,
|
||||
}, UpstreamRuntimeDependencies{
|
||||
Coordinator: coordinatorFunc(func(
|
||||
ctx context.Context,
|
||||
_ string,
|
||||
_ CoordinationLimits,
|
||||
work func(context.Context, LeaderSession) error,
|
||||
) error {
|
||||
return work(ctx, unlimitedLeaderSession{})
|
||||
}),
|
||||
Inventory: inventoryReaderFunc(func(context.Context, string, time.Duration) (pool.InventorySnapshot, error) {
|
||||
if inventoryReads.Add(1) == 1 {
|
||||
return pool.InventorySnapshot{}, errors.New("inventory unavailable")
|
||||
}
|
||||
return pool.InventorySnapshot{}, nil
|
||||
}),
|
||||
Adapter: adapterFunc(func(context.Context) (FetchResponse, error) {
|
||||
providerCalls.Add(1)
|
||||
fetched <- struct{}{}
|
||||
return FetchResponse{Body: []byte("fixture")}, nil
|
||||
}),
|
||||
Parser: parserFunc(func(context.Context, []byte) ([]proxyDomain.Proxy, error) {
|
||||
return []proxyDomain.Proxy{{ID: "proxy-1"}}, nil
|
||||
}),
|
||||
Activity: activitySinkFunc(func(context.Context, string, activitypool.FetchedBatch) (activitypool.UpsertResult, error) {
|
||||
return activitypool.UpsertResult{Accepted: 1, Inserted: 1}, nil
|
||||
}),
|
||||
Results: resultRecorderFunc(func(Result) {}),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("NewUpstreamRuntime(): %v", err)
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
done := make(chan error, 1)
|
||||
go func() { done <- runtime.Run(ctx) }()
|
||||
select {
|
||||
case <-fetched:
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("timed out waiting for recovered inventory fetch")
|
||||
}
|
||||
cancel()
|
||||
<-done
|
||||
if got := inventoryReads.Load(); got < 2 {
|
||||
t.Fatalf("inventory reads = %d, want recovery retry", got)
|
||||
}
|
||||
if got := providerCalls.Load(); got != 1 {
|
||||
t.Fatalf("provider calls = %d, want one call after recovery", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestUpstreamRuntimeDelegatesRequestIntervalOnlyToCoordinator(t *testing.T) {
|
||||
providerConfig := runtimeProviderConfig("provider-a")
|
||||
providerConfig.RequestInterval = 500 * time.Millisecond
|
||||
fetched := make(chan struct{}, 2)
|
||||
runtime, err := NewUpstreamRuntime(UpstreamRuntimeConfig{
|
||||
Provider: providerConfig,
|
||||
ReconcilePolicy: runtimeReconcilePolicy(),
|
||||
ReconcileInterval: 5 * time.Millisecond,
|
||||
}, UpstreamRuntimeDependencies{
|
||||
Coordinator: coordinatorFunc(func(
|
||||
ctx context.Context,
|
||||
_ string,
|
||||
limits CoordinationLimits,
|
||||
work func(context.Context, LeaderSession) error,
|
||||
) error {
|
||||
if limits.RequestInterval != 500*time.Millisecond {
|
||||
t.Errorf("distributed RequestInterval = %s, want 500ms", limits.RequestInterval)
|
||||
}
|
||||
return work(ctx, unlimitedLeaderSession{})
|
||||
}),
|
||||
Inventory: inventoryReaderFunc(func(context.Context, string, time.Duration) (pool.InventorySnapshot, error) {
|
||||
return pool.InventorySnapshot{}, nil
|
||||
}),
|
||||
Adapter: adapterFunc(func(context.Context) (FetchResponse, error) {
|
||||
fetched <- struct{}{}
|
||||
return FetchResponse{Body: []byte("fixture")}, nil
|
||||
}),
|
||||
Parser: parserFunc(func(context.Context, []byte) ([]proxyDomain.Proxy, error) {
|
||||
return []proxyDomain.Proxy{{ID: "proxy-1"}}, nil
|
||||
}),
|
||||
Activity: activitySinkFunc(func(context.Context, string, activitypool.FetchedBatch) (activitypool.UpsertResult, error) {
|
||||
return activitypool.UpsertResult{Accepted: 1, Inserted: 1}, nil
|
||||
}),
|
||||
Results: resultRecorderFunc(func(Result) {}),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("NewUpstreamRuntime(): %v", err)
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
done := make(chan error, 1)
|
||||
go func() { done <- runtime.Run(ctx) }()
|
||||
for range 2 {
|
||||
select {
|
||||
case <-fetched:
|
||||
case <-time.After(150 * time.Millisecond):
|
||||
cancel()
|
||||
<-done
|
||||
t.Fatal("local Provider reconciler duplicated the distributed request interval")
|
||||
}
|
||||
}
|
||||
cancel()
|
||||
<-done
|
||||
}
|
||||
|
||||
func TestNewUpstreamRuntimeRejectsInvalidDependencies(t *testing.T) {
|
||||
validConfig := UpstreamRuntimeConfig{
|
||||
Provider: runtimeProviderConfig("provider-a"),
|
||||
ReconcilePolicy: runtimeReconcilePolicy(),
|
||||
ReconcileInterval: time.Second,
|
||||
}
|
||||
if runtime, err := NewUpstreamRuntime(validConfig, UpstreamRuntimeDependencies{}); err == nil || runtime != nil {
|
||||
t.Fatalf("NewUpstreamRuntime() = (%v, %v), want invalid dependencies", runtime, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestInitialReconcileDelayIsStableAndBounded(t *testing.T) {
|
||||
const interval = time.Second
|
||||
first := initialReconcileDelay("provider-a", interval)
|
||||
if first != initialReconcileDelay("provider-a", interval) {
|
||||
t.Fatal("initial reconcile delay is not stable")
|
||||
}
|
||||
if first < 0 || first >= 250*time.Millisecond {
|
||||
t.Fatalf("initial reconcile delay = %s, want [0, 250ms)", first)
|
||||
}
|
||||
if other := initialReconcileDelay("provider-b", interval); other == first {
|
||||
t.Fatalf("different upstreams have the same initial delay %s", first)
|
||||
}
|
||||
}
|
||||
|
||||
func runtimeProviderConfig(upstreamID string) Config {
|
||||
return Config{
|
||||
UpstreamID: upstreamID, Timeout: time.Second, MaxAttempts: 1,
|
||||
MaxInFlight: 1, MaxTotal: 10, MaxSize: 10,
|
||||
}
|
||||
}
|
||||
|
||||
func runtimeReconcilePolicy() pool.ReconcilePolicy {
|
||||
return pool.ReconcilePolicy{
|
||||
MinimumAvailableSlots: 1, TargetAvailableSlots: 2,
|
||||
ExpectedPerFetch: 1, ExpectedSlotsPerFetch: 1,
|
||||
}
|
||||
}
|
||||
|
||||
type coordinatorFunc func(
|
||||
context.Context,
|
||||
string,
|
||||
CoordinationLimits,
|
||||
func(context.Context, LeaderSession) error,
|
||||
) error
|
||||
|
||||
func (f coordinatorFunc) RunLeader(
|
||||
ctx context.Context,
|
||||
upstreamID string,
|
||||
limits CoordinationLimits,
|
||||
work func(context.Context, LeaderSession) error,
|
||||
) error {
|
||||
return f(ctx, upstreamID, limits, work)
|
||||
}
|
||||
|
||||
type inventoryReaderFunc func(context.Context, string, time.Duration) (pool.InventorySnapshot, error)
|
||||
|
||||
func (f inventoryReaderFunc) ReadInventory(
|
||||
ctx context.Context,
|
||||
upstreamID string,
|
||||
safetyMargin time.Duration,
|
||||
) (pool.InventorySnapshot, error) {
|
||||
return f(ctx, upstreamID, safetyMargin)
|
||||
}
|
||||
@ -13,19 +13,46 @@ var (
|
||||
)
|
||||
|
||||
type Capacity struct {
|
||||
max atomic.Uint32
|
||||
counters atomic.Uint64
|
||||
max atomic.Uint32
|
||||
counters atomic.Uint64
|
||||
configuredObserver *activityObserver
|
||||
observer atomic.Pointer[activityObserver]
|
||||
}
|
||||
|
||||
type activityObserver struct{ notify func(bool) }
|
||||
|
||||
func NewCapacity(max int64) *Capacity {
|
||||
return NewCapacityWithActivityObserver(max, nil)
|
||||
}
|
||||
|
||||
// NewCapacityWithActivityObserver reports successful zero-to-nonzero and
|
||||
// nonzero-to-zero transitions. The observer must tolerate concurrent calls.
|
||||
func NewCapacityWithActivityObserver(max int64, observer func(nonzero bool)) *Capacity {
|
||||
capacity := &Capacity{}
|
||||
if max < 0 || max > int64(counterMask) {
|
||||
max = 0
|
||||
}
|
||||
capacity.max.Store(uint32(max))
|
||||
if observer != nil {
|
||||
capacity.configuredObserver = &activityObserver{notify: observer}
|
||||
capacity.observer.Store(capacity.configuredObserver)
|
||||
}
|
||||
return capacity
|
||||
}
|
||||
|
||||
// SetActivityObservationEnabled lets snapshot ownership disable callbacks for
|
||||
// current Proxies and enable them only while a runtime is retired and draining.
|
||||
func (c *Capacity) SetActivityObservationEnabled(enabled bool) {
|
||||
if c == nil || c.configuredObserver == nil {
|
||||
return
|
||||
}
|
||||
if enabled {
|
||||
c.observer.Store(c.configuredObserver)
|
||||
return
|
||||
}
|
||||
c.observer.Store(nil)
|
||||
}
|
||||
|
||||
func (c *Capacity) SetMax(max int64) bool {
|
||||
if max < 0 || max > int64(counterMask) {
|
||||
return false
|
||||
@ -36,6 +63,15 @@ func (c *Capacity) SetMax(max int64) bool {
|
||||
|
||||
func (c *Capacity) Max() int64 { return int64(c.max.Load()) }
|
||||
|
||||
func (c *Capacity) Counters() (active, reserved, maximum int64) {
|
||||
if c == nil {
|
||||
return 0, 0, 0
|
||||
}
|
||||
packed := c.counters.Load()
|
||||
activeCounter, reservedCounter := unpack(packed)
|
||||
return int64(activeCounter), int64(reservedCounter), int64(c.max.Load())
|
||||
}
|
||||
|
||||
func (c *Capacity) Reserve() (*Reservation, bool) {
|
||||
for {
|
||||
current := c.counters.Load()
|
||||
@ -45,6 +81,11 @@ func (c *Capacity) Reserve() (*Reservation, bool) {
|
||||
}
|
||||
next := pack(active, reserved+1)
|
||||
if c.counters.CompareAndSwap(current, next) {
|
||||
if active+reserved == 0 {
|
||||
if observer := c.observer.Load(); observer != nil {
|
||||
observer.notify(true)
|
||||
}
|
||||
}
|
||||
return &Reservation{capacity: c}, true
|
||||
}
|
||||
}
|
||||
@ -77,7 +118,16 @@ func (c *Capacity) cancel() {
|
||||
for {
|
||||
current := c.counters.Load()
|
||||
active, reserved := unpack(current)
|
||||
if reserved == 0 || c.counters.CompareAndSwap(current, pack(active, reserved-1)) {
|
||||
if reserved == 0 {
|
||||
return
|
||||
}
|
||||
next := pack(active, reserved-1)
|
||||
if c.counters.CompareAndSwap(current, next) {
|
||||
if active+reserved == 1 {
|
||||
if observer := c.observer.Load(); observer != nil {
|
||||
observer.notify(false)
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
}
|
||||
@ -87,7 +137,16 @@ func (c *Capacity) release() {
|
||||
for {
|
||||
current := c.counters.Load()
|
||||
active, reserved := unpack(current)
|
||||
if active == 0 || c.counters.CompareAndSwap(current, pack(active-1, reserved)) {
|
||||
if active == 0 {
|
||||
return
|
||||
}
|
||||
next := pack(active-1, reserved)
|
||||
if c.counters.CompareAndSwap(current, next) {
|
||||
if active+reserved == 1 {
|
||||
if observer := c.observer.Load(); observer != nil {
|
||||
observer.notify(false)
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
@ -2,6 +2,7 @@ package proxy
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"reflect"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
@ -38,6 +39,56 @@ func TestReservationCancelReleasesReservedCapacity(t *testing.T) {
|
||||
assertCapacityCounters(t, capacity, 0, 0)
|
||||
}
|
||||
|
||||
func TestCapacityCountersReadsOnePackedSnapshot(t *testing.T) {
|
||||
capacity := NewCapacity(3)
|
||||
first, ok := capacity.Reserve()
|
||||
if !ok {
|
||||
t.Fatal("first Reserve() = false")
|
||||
}
|
||||
second, ok := capacity.Reserve()
|
||||
if !ok {
|
||||
t.Fatal("second Reserve() = false")
|
||||
}
|
||||
if err := first.Commit(); err != nil {
|
||||
t.Fatalf("Commit(): %v", err)
|
||||
}
|
||||
|
||||
active, reserved, maximum := capacity.Counters()
|
||||
if active != 1 || reserved != 1 || maximum != 3 {
|
||||
t.Fatalf("Counters() = (%d, %d, %d), want (1, 1, 3)", active, reserved, maximum)
|
||||
}
|
||||
if err := first.Release(); err != nil {
|
||||
t.Fatalf("Release(): %v", err)
|
||||
}
|
||||
if err := second.Cancel(); err != nil {
|
||||
t.Fatalf("Cancel(): %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCapacityActivityObserverTracksOnlyNonzeroTransitions(t *testing.T) {
|
||||
var transitions []bool
|
||||
capacity := NewCapacityWithActivityObserver(2, func(nonzero bool) {
|
||||
transitions = append(transitions, nonzero)
|
||||
})
|
||||
first, ok := capacity.Reserve()
|
||||
if !ok {
|
||||
t.Fatal("Reserve(first) = false")
|
||||
}
|
||||
second, ok := capacity.Reserve()
|
||||
if !ok {
|
||||
t.Fatal("Reserve(second) = false")
|
||||
}
|
||||
if err := first.Cancel(); err != nil {
|
||||
t.Fatalf("Cancel(first): %v", err)
|
||||
}
|
||||
if err := second.Cancel(); err != nil {
|
||||
t.Fatalf("Cancel(second): %v", err)
|
||||
}
|
||||
if !reflect.DeepEqual(transitions, []bool{true, false}) {
|
||||
t.Fatalf("transitions = %v, want [true false]", transitions)
|
||||
}
|
||||
}
|
||||
|
||||
func TestReservationCommitAndReleaseAreSingleUse(t *testing.T) {
|
||||
capacity := NewCapacity(1)
|
||||
reservation, ok := capacity.Reserve()
|
||||
|
||||
@ -8,6 +8,6 @@ type FetchCapacity interface {
|
||||
|
||||
type FetchPermit interface {
|
||||
Expected() int
|
||||
Complete(fetched, retained int) error
|
||||
Complete(retained int) error
|
||||
Cancel() error
|
||||
}
|
||||
|
||||
@ -17,9 +17,7 @@ type ProxyCapacity struct {
|
||||
type Inventory struct {
|
||||
Proxies []ProxyCapacity
|
||||
PendingExpected int
|
||||
FetchedTotal int64
|
||||
MaxSize int
|
||||
MaxTotal int64
|
||||
}
|
||||
|
||||
func (i Inventory) AvailableSlots(now time.Time, safetyMargin time.Duration) int64 {
|
||||
@ -55,15 +53,6 @@ func (i Inventory) FetchAllowance(requested int) int {
|
||||
return 0
|
||||
}
|
||||
allowed := min(requested, max(i.MaxSize-i.ManagedCount(), 0))
|
||||
if i.MaxTotal > 0 {
|
||||
remaining := i.MaxTotal - i.FetchedTotal
|
||||
if remaining <= 0 {
|
||||
return 0
|
||||
}
|
||||
if int64(allowed) > remaining {
|
||||
allowed = int(remaining)
|
||||
}
|
||||
}
|
||||
return allowed
|
||||
}
|
||||
|
||||
|
||||
@ -21,7 +21,7 @@ func TestInventoryAvailableSlotsUsesOnlyAllocatableCapacity(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestInventoryFetchAllowanceSeparatesPoolAndCumulativeLimits(t *testing.T) {
|
||||
func TestInventoryFetchAllowanceUsesManagedPoolCapacity(t *testing.T) {
|
||||
inventory := Inventory{
|
||||
Proxies: []ProxyCapacity{
|
||||
{State: proxyDomain.StateFetched},
|
||||
@ -32,20 +32,13 @@ func TestInventoryFetchAllowanceSeparatesPoolAndCumulativeLimits(t *testing.T) {
|
||||
{State: proxyDomain.StateExtracted},
|
||||
},
|
||||
PendingExpected: 2,
|
||||
FetchedTotal: 98,
|
||||
MaxSize: 10,
|
||||
MaxTotal: 100,
|
||||
}
|
||||
|
||||
if got := inventory.ManagedCount(); got != 7 {
|
||||
t.Fatalf("ManagedCount() = %d, want 7", got)
|
||||
}
|
||||
if got := inventory.FetchAllowance(10); got != 2 {
|
||||
t.Fatalf("FetchAllowance() = %d, want 2 from cumulative quota", got)
|
||||
}
|
||||
|
||||
inventory.MaxTotal = 0
|
||||
if got := inventory.FetchAllowance(10); got != 3 {
|
||||
t.Fatalf("FetchAllowance() with unlimited cumulative quota = %d, want 3 from pool size", got)
|
||||
t.Fatalf("FetchAllowance() = %d, want 3 from pool size", got)
|
||||
}
|
||||
}
|
||||
|
||||
211
internal/domain/workerruntime/memory.go
Normal file
211
internal/domain/workerruntime/memory.go
Normal file
@ -0,0 +1,211 @@
|
||||
package workerruntime
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/json"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
type MemoryStore struct {
|
||||
mu sync.Mutex
|
||||
now func() time.Time
|
||||
sessions map[string]memorySession
|
||||
reports map[string]memoryReport
|
||||
}
|
||||
|
||||
type memorySession struct {
|
||||
value Session
|
||||
expiresAt time.Time
|
||||
}
|
||||
|
||||
type memoryReport struct {
|
||||
value Report
|
||||
digest [sha256.Size]byte
|
||||
expiresAt time.Time
|
||||
counters map[string]Counter
|
||||
}
|
||||
|
||||
var (
|
||||
_ SessionWriter = (*MemoryStore)(nil)
|
||||
_ ReportWriter = (*MemoryStore)(nil)
|
||||
_ RuntimeReader = (*MemoryStore)(nil)
|
||||
)
|
||||
|
||||
func NewMemoryStore(now func() time.Time) (*MemoryStore, error) {
|
||||
if now == nil {
|
||||
return nil, ErrInvalidStore
|
||||
}
|
||||
return &MemoryStore{
|
||||
now: now, sessions: make(map[string]memorySession), reports: make(map[string]memoryReport),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (store *MemoryStore) ReplaceSession(ctx context.Context, session Session, ttl time.Duration) error {
|
||||
if ctx == nil || store == nil || !validSession(session) || ttl <= 0 {
|
||||
return ErrInvalidSession
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
now := store.now().UTC()
|
||||
if now.IsZero() {
|
||||
return ErrInvalidStore
|
||||
}
|
||||
store.mu.Lock()
|
||||
defer store.mu.Unlock()
|
||||
current, exists := store.sessions[session.WorkerID]
|
||||
identityChanged := exists && (current.value.SessionID != session.SessionID || current.value.InstanceID != session.InstanceID)
|
||||
expired := exists && !current.expiresAt.After(now)
|
||||
if exists && !identityChanged && !expired && sessionBefore(session, current.value) {
|
||||
return ErrStaleSession
|
||||
}
|
||||
ackAdvanced := exists && !identityChanged && !expired && sessionAfter(session, current.value)
|
||||
if identityChanged || expired || ackAdvanced {
|
||||
delete(store.reports, session.WorkerID)
|
||||
}
|
||||
store.sessions[session.WorkerID] = memorySession{value: session, expiresAt: now.Add(ttl)}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (store *MemoryStore) ReplaceRuntime(ctx context.Context, report Report, ttl time.Duration) error {
|
||||
if ctx == nil || store == nil || ttl <= 0 {
|
||||
return ErrInvalidReport
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
normalized, counterIndex, err := normalizeReport(report)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
payload, err := json.Marshal(normalized)
|
||||
if err != nil {
|
||||
return ErrInvalidReport
|
||||
}
|
||||
digest := sha256.Sum256(payload)
|
||||
now := store.now().UTC()
|
||||
if now.IsZero() {
|
||||
return ErrInvalidStore
|
||||
}
|
||||
store.mu.Lock()
|
||||
defer store.mu.Unlock()
|
||||
session, exists := store.sessions[report.WorkerID]
|
||||
if !exists || !session.expiresAt.After(now) || session.value.SessionID != report.SessionID {
|
||||
return ErrStaleSession
|
||||
}
|
||||
if report.SnapshotVersion != session.value.AckedSnapshotVersion ||
|
||||
report.OwnershipEpoch != session.value.AckedOwnershipEpoch {
|
||||
return ErrStaleReport
|
||||
}
|
||||
if current, exists := store.reports[report.WorkerID]; exists && current.value.SessionID == report.SessionID {
|
||||
switch {
|
||||
case normalized.Sequence < current.value.Sequence:
|
||||
return ErrStaleReport
|
||||
case normalized.Sequence == current.value.Sequence && digest != current.digest:
|
||||
return ErrConflictingReport
|
||||
case normalized.Sequence == current.value.Sequence:
|
||||
return nil
|
||||
}
|
||||
}
|
||||
store.reports[report.WorkerID] = memoryReport{
|
||||
value: normalized, digest: digest, expiresAt: now.Add(ttl), counters: counterIndex,
|
||||
}
|
||||
session.expiresAt = now.Add(ttl)
|
||||
store.sessions[report.WorkerID] = session
|
||||
return nil
|
||||
}
|
||||
|
||||
func (store *MemoryStore) ReadRuntime(ctx context.Context, proxies []OwnedProxy) ([]Snapshot, error) {
|
||||
if ctx == nil || store == nil {
|
||||
return nil, ErrInvalidQuery
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
seen := make(map[string]struct{}, len(proxies))
|
||||
for _, proxy := range proxies {
|
||||
if !clean(proxy.ProxyID) || !clean(proxy.WorkerID) || proxy.OwnershipEpoch == 0 {
|
||||
return nil, ErrInvalidQuery
|
||||
}
|
||||
key := proxy.WorkerID + "\x00" + proxy.ProxyID
|
||||
if _, exists := seen[key]; exists {
|
||||
return nil, ErrInvalidQuery
|
||||
}
|
||||
seen[key] = struct{}{}
|
||||
}
|
||||
now := store.now().UTC()
|
||||
if now.IsZero() {
|
||||
return nil, ErrInvalidStore
|
||||
}
|
||||
store.mu.Lock()
|
||||
defer store.mu.Unlock()
|
||||
result := make([]Snapshot, len(proxies))
|
||||
for index, proxy := range proxies {
|
||||
result[index].ProxyID = proxy.ProxyID
|
||||
session, sessionExists := store.sessions[proxy.WorkerID]
|
||||
report, reportExists := store.reports[proxy.WorkerID]
|
||||
if !sessionExists || !reportExists || !session.expiresAt.After(now) || !report.expiresAt.After(now) ||
|
||||
report.value.SessionID != session.value.SessionID ||
|
||||
report.value.SnapshotVersion != session.value.AckedSnapshotVersion ||
|
||||
report.value.OwnershipEpoch != session.value.AckedOwnershipEpoch ||
|
||||
report.value.OwnershipEpoch < proxy.OwnershipEpoch {
|
||||
continue
|
||||
}
|
||||
result[index].Fresh = true
|
||||
if counter, exists := report.counters[proxy.ProxyID]; exists {
|
||||
result[index].Active = counter.Active
|
||||
result[index].Reserved = counter.Reserved
|
||||
result[index].Draining = counter.Draining
|
||||
}
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func normalizeReport(report Report) (Report, map[string]Counter, error) {
|
||||
if !clean(report.WorkerID) || !clean(report.SessionID) || report.Sequence == 0 ||
|
||||
report.SnapshotVersion == 0 || report.OwnershipEpoch == 0 || report.ObservedAt.IsZero() {
|
||||
return Report{}, nil, ErrInvalidReport
|
||||
}
|
||||
normalized := report
|
||||
normalized.ObservedAt = report.ObservedAt.UTC()
|
||||
normalized.Counters = append([]Counter(nil), report.Counters...)
|
||||
sort.Slice(normalized.Counters, func(left, right int) bool {
|
||||
return normalized.Counters[left].ProxyID < normalized.Counters[right].ProxyID
|
||||
})
|
||||
index := make(map[string]Counter, len(normalized.Counters))
|
||||
for _, counter := range normalized.Counters {
|
||||
if !clean(counter.ProxyID) || counter.Active < 0 || counter.Reserved < 0 {
|
||||
return Report{}, nil, ErrInvalidReport
|
||||
}
|
||||
if _, exists := index[counter.ProxyID]; exists {
|
||||
return Report{}, nil, ErrInvalidReport
|
||||
}
|
||||
index[counter.ProxyID] = counter
|
||||
}
|
||||
return normalized, index, nil
|
||||
}
|
||||
|
||||
func validSession(session Session) bool {
|
||||
return clean(session.WorkerID) && clean(session.InstanceID) && clean(session.SessionID) &&
|
||||
session.AckedSnapshotVersion > 0 && session.AckedOwnershipEpoch > 0
|
||||
}
|
||||
|
||||
func sessionBefore(left, right Session) bool {
|
||||
return left.AckedOwnershipEpoch < right.AckedOwnershipEpoch ||
|
||||
(left.AckedOwnershipEpoch == right.AckedOwnershipEpoch &&
|
||||
left.AckedSnapshotVersion < right.AckedSnapshotVersion)
|
||||
}
|
||||
|
||||
func sessionAfter(left, right Session) bool {
|
||||
return left.AckedOwnershipEpoch > right.AckedOwnershipEpoch ||
|
||||
(left.AckedOwnershipEpoch == right.AckedOwnershipEpoch &&
|
||||
left.AckedSnapshotVersion > right.AckedSnapshotVersion)
|
||||
}
|
||||
|
||||
func clean(value string) bool {
|
||||
return value != "" && strings.TrimSpace(value) == value
|
||||
}
|
||||
156
internal/domain/workerruntime/memory_test.go
Normal file
156
internal/domain/workerruntime/memory_test.go
Normal file
@ -0,0 +1,156 @@
|
||||
package workerruntime
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestMemoryStoreReplacesSparseRuntimeAndClearsMissingCounters(t *testing.T) {
|
||||
now := time.Date(2026, 7, 30, 15, 0, 0, 0, time.UTC)
|
||||
store := newRuntimeStore(t, &now)
|
||||
ctx := context.Background()
|
||||
registerRuntimeSession(t, store, "session-a", time.Minute)
|
||||
report := Report{
|
||||
WorkerID: "worker-a", SessionID: "session-a", Sequence: 1,
|
||||
SnapshotVersion: 3, OwnershipEpoch: 9, ObservedAt: now,
|
||||
Counters: []Counter{{ProxyID: "proxy-a", Active: 2, Reserved: 1}},
|
||||
}
|
||||
if err := store.ReplaceRuntime(ctx, report, time.Minute); err != nil {
|
||||
t.Fatalf("ReplaceRuntime(first): %v", err)
|
||||
}
|
||||
got, err := store.ReadRuntime(ctx, []OwnedProxy{{ProxyID: "proxy-a", WorkerID: "worker-a", OwnershipEpoch: 9}})
|
||||
if err != nil || len(got) != 1 || got[0] != (Snapshot{ProxyID: "proxy-a", Active: 2, Reserved: 1, Fresh: true}) {
|
||||
t.Fatalf("ReadRuntime(first) = %+v, %v", got, err)
|
||||
}
|
||||
|
||||
report.Sequence = 2
|
||||
report.ObservedAt = now.Add(time.Second)
|
||||
report.Counters = nil
|
||||
if err := store.ReplaceRuntime(ctx, report, time.Minute); err != nil {
|
||||
t.Fatalf("ReplaceRuntime(empty): %v", err)
|
||||
}
|
||||
got, err = store.ReadRuntime(ctx, []OwnedProxy{{ProxyID: "proxy-a", WorkerID: "worker-a", OwnershipEpoch: 9}})
|
||||
if err != nil || len(got) != 1 || got[0] != (Snapshot{ProxyID: "proxy-a", Fresh: true}) {
|
||||
t.Fatalf("ReadRuntime(empty) = %+v, %v", got, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMemoryStoreFencesSessionsAndReportSequence(t *testing.T) {
|
||||
now := time.Date(2026, 7, 30, 15, 0, 0, 0, time.UTC)
|
||||
store := newRuntimeStore(t, &now)
|
||||
ctx := context.Background()
|
||||
registerRuntimeSession(t, store, "session-a", time.Minute)
|
||||
report := Report{
|
||||
WorkerID: "worker-a", SessionID: "session-a", Sequence: 2,
|
||||
SnapshotVersion: 3, OwnershipEpoch: 9, ObservedAt: now,
|
||||
Counters: []Counter{{ProxyID: "proxy-a", Active: 1}},
|
||||
}
|
||||
if err := store.ReplaceRuntime(ctx, report, time.Minute); err != nil {
|
||||
t.Fatalf("ReplaceRuntime(first): %v", err)
|
||||
}
|
||||
if err := store.ReplaceRuntime(ctx, report, time.Minute); err != nil {
|
||||
t.Fatalf("ReplaceRuntime(idempotent): %v", err)
|
||||
}
|
||||
conflict := report
|
||||
conflict.Counters = []Counter{{ProxyID: "proxy-a", Active: 2}}
|
||||
if err := store.ReplaceRuntime(ctx, conflict, time.Minute); !errors.Is(err, ErrConflictingReport) {
|
||||
t.Fatalf("ReplaceRuntime(conflict) error = %v", err)
|
||||
}
|
||||
stale := report
|
||||
stale.Sequence = 1
|
||||
if err := store.ReplaceRuntime(ctx, stale, time.Minute); !errors.Is(err, ErrStaleReport) {
|
||||
t.Fatalf("ReplaceRuntime(stale) error = %v", err)
|
||||
}
|
||||
registerRuntimeSession(t, store, "session-b", time.Minute)
|
||||
if err := store.ReplaceRuntime(ctx, report, time.Minute); !errors.Is(err, ErrStaleSession) {
|
||||
t.Fatalf("ReplaceRuntime(old session) error = %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMemoryStoreFailsClosedForExpiredOrOlderOwnershipReport(t *testing.T) {
|
||||
now := time.Date(2026, 7, 30, 15, 0, 0, 0, time.UTC)
|
||||
store := newRuntimeStore(t, &now)
|
||||
ctx := context.Background()
|
||||
registerRuntimeSession(t, store, "session-a", time.Minute)
|
||||
if err := store.ReplaceRuntime(ctx, Report{
|
||||
WorkerID: "worker-a", SessionID: "session-a", Sequence: 1,
|
||||
SnapshotVersion: 3, OwnershipEpoch: 9, ObservedAt: now,
|
||||
}, time.Minute); err != nil {
|
||||
t.Fatalf("ReplaceRuntime(): %v", err)
|
||||
}
|
||||
queries := []OwnedProxy{
|
||||
{ProxyID: "proxy-a", WorkerID: "worker-a", OwnershipEpoch: 10},
|
||||
{ProxyID: "proxy-b", WorkerID: "worker-a", OwnershipEpoch: 9},
|
||||
}
|
||||
got, err := store.ReadRuntime(ctx, queries)
|
||||
if err != nil || got[0].Fresh || !got[1].Fresh {
|
||||
t.Fatalf("ReadRuntime(ownership fence) = %+v, %v", got, err)
|
||||
}
|
||||
now = now.Add(time.Minute)
|
||||
got, err = store.ReadRuntime(ctx, queries[1:])
|
||||
if err != nil || len(got) != 1 || got[0].Fresh {
|
||||
t.Fatalf("ReadRuntime(expired) = %+v, %v", got, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMemoryStoreRejectsRuntimeBeyondAcknowledgedSnapshot(t *testing.T) {
|
||||
now := time.Date(2026, 7, 30, 15, 0, 0, 0, time.UTC)
|
||||
store := newRuntimeStore(t, &now)
|
||||
registerRuntimeSession(t, store, "session-a", time.Minute)
|
||||
report := Report{
|
||||
WorkerID: "worker-a", SessionID: "session-a", Sequence: 1,
|
||||
SnapshotVersion: 4, OwnershipEpoch: 9, ObservedAt: now,
|
||||
}
|
||||
if err := store.ReplaceRuntime(context.Background(), report, time.Minute); !errors.Is(err, ErrStaleReport) {
|
||||
t.Fatalf("ReplaceRuntime(ahead snapshot) error = %v, want ErrStaleReport", err)
|
||||
}
|
||||
report.SnapshotVersion = 3
|
||||
report.OwnershipEpoch = 10
|
||||
if err := store.ReplaceRuntime(context.Background(), report, time.Minute); !errors.Is(err, ErrStaleReport) {
|
||||
t.Fatalf("ReplaceRuntime(ahead epoch) error = %v, want ErrStaleReport", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMemoryStoreExpiredSameIdentitySessionDoesNotReactivateOldReport(t *testing.T) {
|
||||
now := time.Date(2026, 7, 30, 15, 0, 0, 0, time.UTC)
|
||||
store := newRuntimeStore(t, &now)
|
||||
registerRuntimeSession(t, store, "session-a", time.Second)
|
||||
if err := store.ReplaceRuntime(context.Background(), Report{
|
||||
WorkerID: "worker-a", SessionID: "session-a", Sequence: 1,
|
||||
SnapshotVersion: 3, OwnershipEpoch: 9, ObservedAt: now,
|
||||
}, time.Minute); err != nil {
|
||||
t.Fatalf("ReplaceRuntime(): %v", err)
|
||||
}
|
||||
session := store.sessions["worker-a"]
|
||||
session.expiresAt = now.Add(time.Second)
|
||||
store.sessions["worker-a"] = session
|
||||
now = now.Add(2 * time.Second)
|
||||
registerRuntimeSession(t, store, "session-a", time.Minute)
|
||||
got, err := store.ReadRuntime(context.Background(), []OwnedProxy{{
|
||||
ProxyID: "proxy-a", WorkerID: "worker-a", OwnershipEpoch: 9,
|
||||
}})
|
||||
if err != nil || len(got) != 1 || got[0].Fresh {
|
||||
t.Fatalf("ReadRuntime(after re-register) = %+v, %v; want stale", got, err)
|
||||
}
|
||||
}
|
||||
|
||||
func newRuntimeStore(t *testing.T, now *time.Time) *MemoryStore {
|
||||
t.Helper()
|
||||
store, err := NewMemoryStore(func() time.Time { return *now })
|
||||
if err != nil {
|
||||
t.Fatalf("NewMemoryStore(): %v", err)
|
||||
}
|
||||
return store
|
||||
}
|
||||
|
||||
func registerRuntimeSession(t *testing.T, store *MemoryStore, sessionID string, ttl time.Duration) {
|
||||
t.Helper()
|
||||
if err := store.ReplaceSession(context.Background(), Session{
|
||||
WorkerID: "worker-a", InstanceID: "instance-a", SessionID: sessionID,
|
||||
AckedSnapshotVersion: 3, AckedOwnershipEpoch: 9,
|
||||
}, ttl); err != nil {
|
||||
t.Fatalf("ReplaceSession(): %v", err)
|
||||
}
|
||||
}
|
||||
70
internal/domain/workerruntime/runtime.go
Normal file
70
internal/domain/workerruntime/runtime.go
Normal file
@ -0,0 +1,70 @@
|
||||
package workerruntime
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"time"
|
||||
)
|
||||
|
||||
var (
|
||||
ErrInvalidStore = errors.New("invalid worker runtime store")
|
||||
ErrInvalidSession = errors.New("invalid worker runtime session")
|
||||
ErrInvalidReport = errors.New("invalid worker runtime report")
|
||||
ErrInvalidQuery = errors.New("invalid worker runtime query")
|
||||
ErrStaleSession = errors.New("stale worker runtime session")
|
||||
ErrStaleReport = errors.New("stale worker runtime report")
|
||||
ErrConflictingReport = errors.New("conflicting worker runtime report")
|
||||
)
|
||||
|
||||
type Session struct {
|
||||
WorkerID string
|
||||
InstanceID string
|
||||
SessionID string
|
||||
AckedSnapshotVersion uint64
|
||||
AckedOwnershipEpoch uint64
|
||||
}
|
||||
|
||||
type Counter struct {
|
||||
ProxyID string
|
||||
Active int64
|
||||
Reserved int64
|
||||
Draining bool
|
||||
}
|
||||
|
||||
// Report is a complete sparse replacement. Missing counters are zero for the
|
||||
// reported Worker snapshot; callers must increase Sequence for every update.
|
||||
type Report struct {
|
||||
WorkerID string
|
||||
SessionID string
|
||||
Sequence uint64
|
||||
SnapshotVersion uint64
|
||||
OwnershipEpoch uint64
|
||||
ObservedAt time.Time
|
||||
Counters []Counter
|
||||
}
|
||||
|
||||
type OwnedProxy struct {
|
||||
ProxyID string
|
||||
WorkerID string
|
||||
OwnershipEpoch uint64
|
||||
}
|
||||
|
||||
type Snapshot struct {
|
||||
ProxyID string
|
||||
Active int64
|
||||
Reserved int64
|
||||
Draining bool
|
||||
Fresh bool
|
||||
}
|
||||
|
||||
type SessionWriter interface {
|
||||
ReplaceSession(context.Context, Session, time.Duration) error
|
||||
}
|
||||
|
||||
type ReportWriter interface {
|
||||
ReplaceRuntime(context.Context, Report, time.Duration) error
|
||||
}
|
||||
|
||||
type RuntimeReader interface {
|
||||
ReadRuntime(context.Context, []OwnedProxy) ([]Snapshot, error)
|
||||
}
|
||||
@ -7,19 +7,90 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
proxyDomain "proxy-pool/internal/domain/proxy"
|
||||
"proxy-pool/internal/domain/workerruntime"
|
||||
)
|
||||
|
||||
var (
|
||||
ErrWrongTarget = errors.New("snapshot targets another cluster or worker")
|
||||
ErrResyncRequired = errors.New("snapshot sequence requires a full resync")
|
||||
ErrChecksumMismatch = errors.New("snapshot checksum mismatch")
|
||||
ErrWrongTarget = errors.New("snapshot targets another cluster or worker")
|
||||
ErrResyncRequired = errors.New("snapshot sequence requires a full resync")
|
||||
ErrChecksumMismatch = errors.New("snapshot checksum mismatch")
|
||||
ErrInvalidRuntimeReport = errors.New("invalid worker runtime report")
|
||||
ErrInvalidRuntimeLimit = errors.New("invalid snapshot runtime limit")
|
||||
ErrRuntimeLimitExceeded = errors.New("snapshot runtime limit exceeded")
|
||||
)
|
||||
|
||||
const defaultRuntimeLimit = 1_000_000
|
||||
|
||||
const activeRuntimeShardCount = 64
|
||||
|
||||
type activeRuntimeShard struct {
|
||||
mu sync.Mutex
|
||||
entries map[string]*proxyDomain.Capacity
|
||||
}
|
||||
|
||||
type activeRuntimeIndex [activeRuntimeShardCount]activeRuntimeShard
|
||||
|
||||
func (index *activeRuntimeIndex) track(proxyID string, runtime *proxyDomain.Capacity, nonzero bool) {
|
||||
shard := &index[activeRuntimeShardIndex(proxyID)]
|
||||
shard.mu.Lock()
|
||||
defer shard.mu.Unlock()
|
||||
if nonzero {
|
||||
if shard.entries == nil {
|
||||
shard.entries = make(map[string]*proxyDomain.Capacity)
|
||||
}
|
||||
shard.entries[proxyID] = runtime
|
||||
return
|
||||
}
|
||||
active, reserved, _ := runtime.Counters()
|
||||
if active == 0 && reserved == 0 && shard.entries[proxyID] == runtime {
|
||||
delete(shard.entries, proxyID)
|
||||
}
|
||||
}
|
||||
|
||||
func (index *activeRuntimeIndex) remove(proxyID string, runtime *proxyDomain.Capacity) {
|
||||
shard := &index[activeRuntimeShardIndex(proxyID)]
|
||||
shard.mu.Lock()
|
||||
defer shard.mu.Unlock()
|
||||
if shard.entries[proxyID] == runtime {
|
||||
delete(shard.entries, proxyID)
|
||||
}
|
||||
}
|
||||
|
||||
func (index *activeRuntimeIndex) rangeEntries(visit func(string, *proxyDomain.Capacity)) {
|
||||
for shardIndex := range index {
|
||||
shard := &index[shardIndex]
|
||||
shard.mu.Lock()
|
||||
for proxyID, runtime := range shard.entries {
|
||||
visit(proxyID, runtime)
|
||||
}
|
||||
shard.mu.Unlock()
|
||||
}
|
||||
}
|
||||
|
||||
func activeRuntimeShardIndex(proxyID string) uint64 {
|
||||
const (
|
||||
offset = uint64(14695981039346656037)
|
||||
prime = uint64(1099511628211)
|
||||
)
|
||||
hash := offset
|
||||
for index := 0; index < len(proxyID); index++ {
|
||||
hash ^= uint64(proxyID[index])
|
||||
hash *= prime
|
||||
}
|
||||
return hash % activeRuntimeShardCount
|
||||
}
|
||||
|
||||
type runtimeRegistration struct {
|
||||
capacity *proxyDomain.Capacity
|
||||
current atomic.Bool
|
||||
}
|
||||
|
||||
type Envelope struct {
|
||||
ClusterID string
|
||||
WorkerID string
|
||||
@ -70,17 +141,29 @@ type Store struct {
|
||||
current atomic.Pointer[View]
|
||||
|
||||
mu sync.Mutex
|
||||
runtimes map[string]*proxyDomain.Capacity
|
||||
runtimes map[string]*runtimeRegistration
|
||||
active activeRuntimeIndex
|
||||
limit int
|
||||
}
|
||||
|
||||
func NewStore(clusterID, workerID string) *Store {
|
||||
return &Store{
|
||||
clusterID: clusterID,
|
||||
workerID: workerID,
|
||||
runtimes: make(map[string]*proxyDomain.Capacity),
|
||||
runtimes: make(map[string]*runtimeRegistration),
|
||||
limit: defaultRuntimeLimit,
|
||||
}
|
||||
}
|
||||
|
||||
func NewStoreWithRuntimeLimit(clusterID, workerID string, limit int) (*Store, error) {
|
||||
if limit <= 0 {
|
||||
return nil, ErrInvalidRuntimeLimit
|
||||
}
|
||||
store := NewStore(clusterID, workerID)
|
||||
store.limit = limit
|
||||
return store, nil
|
||||
}
|
||||
|
||||
func (s *Store) Current() *View {
|
||||
if s == nil {
|
||||
return nil
|
||||
@ -88,6 +171,49 @@ func (s *Store) Current() *View {
|
||||
return s.current.Load()
|
||||
}
|
||||
|
||||
func (s *Store) RuntimeReport(sessionID string, sequence uint64, observedAt time.Time) (workerruntime.Report, error) {
|
||||
if s == nil || strings.TrimSpace(sessionID) != sessionID || sessionID == "" || sequence == 0 || observedAt.IsZero() {
|
||||
return workerruntime.Report{}, ErrInvalidRuntimeReport
|
||||
}
|
||||
current := s.current.Load()
|
||||
if current == nil {
|
||||
return workerruntime.Report{}, ErrInvalidRuntimeReport
|
||||
}
|
||||
visible := make(map[string]struct{}, len(current.Entries))
|
||||
counters := make([]workerruntime.Counter, 0)
|
||||
for _, entry := range current.Entries {
|
||||
visible[entry.Proxy.ID] = struct{}{}
|
||||
active, reserved, _ := entry.Runtime.Counters()
|
||||
if active == 0 && reserved == 0 {
|
||||
continue
|
||||
}
|
||||
counters = append(counters, workerruntime.Counter{
|
||||
ProxyID: entry.Proxy.ID, Active: active, Reserved: reserved,
|
||||
Draining: entry.Proxy.State == proxyDomain.StateDraining,
|
||||
})
|
||||
}
|
||||
s.active.rangeEntries(func(proxyID string, runtime *proxyDomain.Capacity) {
|
||||
if _, currentProxy := visible[proxyID]; currentProxy {
|
||||
return
|
||||
}
|
||||
active, reserved, _ := runtime.Counters()
|
||||
if active == 0 && reserved == 0 {
|
||||
return
|
||||
}
|
||||
counters = append(counters, workerruntime.Counter{
|
||||
ProxyID: proxyID, Active: active, Reserved: reserved, Draining: true,
|
||||
})
|
||||
})
|
||||
sort.Slice(counters, func(left, right int) bool {
|
||||
return counters[left].ProxyID < counters[right].ProxyID
|
||||
})
|
||||
return workerruntime.Report{
|
||||
WorkerID: s.workerID, SessionID: sessionID, Sequence: sequence,
|
||||
SnapshotVersion: current.Version, OwnershipEpoch: current.Epoch,
|
||||
ObservedAt: observedAt.UTC(), Counters: counters,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (s *Store) Apply(envelope Envelope) error {
|
||||
if s == nil {
|
||||
return fmt.Errorf("apply snapshot: nil store")
|
||||
@ -118,19 +244,47 @@ func (s *Store) Apply(envelope Envelope) error {
|
||||
}
|
||||
|
||||
proxies := cloneAndSort(envelope.Proxies)
|
||||
newRuntimeCount := 0
|
||||
for _, descriptor := range proxies {
|
||||
if s.runtimes[descriptor.ID] == nil {
|
||||
newRuntimeCount++
|
||||
}
|
||||
}
|
||||
if len(s.runtimes)+newRuntimeCount > s.limit {
|
||||
return ErrRuntimeLimitExceeded
|
||||
}
|
||||
if current != nil {
|
||||
for _, entry := range current.Entries {
|
||||
registration := s.runtimes[entry.Proxy.ID]
|
||||
registration.capacity.SetActivityObservationEnabled(true)
|
||||
registration.current.Store(false)
|
||||
active, reserved, _ := registration.capacity.Counters()
|
||||
s.active.track(entry.Proxy.ID, registration.capacity, active+reserved > 0)
|
||||
}
|
||||
}
|
||||
entries := make([]Entry, 0, len(proxies))
|
||||
for _, descriptor := range proxies {
|
||||
runtime := s.runtimes[descriptor.ID]
|
||||
if runtime == nil {
|
||||
runtime = proxyDomain.NewCapacity(descriptor.MaxConcurrency)
|
||||
registration := s.runtimes[descriptor.ID]
|
||||
if registration == nil {
|
||||
proxyID := descriptor.ID
|
||||
registration = &runtimeRegistration{}
|
||||
registration.capacity = proxyDomain.NewCapacityWithActivityObserver(descriptor.MaxConcurrency, func(nonzero bool) {
|
||||
if registration.current.Load() {
|
||||
return
|
||||
}
|
||||
s.active.track(proxyID, registration.capacity, nonzero)
|
||||
})
|
||||
} else {
|
||||
runtime.SetMax(descriptor.MaxConcurrency)
|
||||
registration.capacity.SetMax(descriptor.MaxConcurrency)
|
||||
}
|
||||
registration.current.Store(true)
|
||||
registration.capacity.SetActivityObservationEnabled(false)
|
||||
s.active.remove(descriptor.ID, registration.capacity)
|
||||
// Keep runtimes for temporarily absent IDs. Old immutable views may still
|
||||
// hold in-flight leases, so reclaiming here could reset active capacity if
|
||||
// the same Proxy reappears in a later snapshot.
|
||||
s.runtimes[descriptor.ID] = runtime
|
||||
entries = append(entries, Entry{Proxy: descriptor, Runtime: runtime})
|
||||
s.runtimes[descriptor.ID] = registration
|
||||
entries = append(entries, Entry{Proxy: descriptor, Runtime: registration.capacity})
|
||||
}
|
||||
|
||||
next := &View{
|
||||
|
||||
@ -8,6 +8,7 @@ import (
|
||||
"time"
|
||||
|
||||
proxyDomain "proxy-pool/internal/domain/proxy"
|
||||
"proxy-pool/internal/domain/workerruntime"
|
||||
)
|
||||
|
||||
func TestStoreAppliesCompleteSnapshotsInOrder(t *testing.T) {
|
||||
@ -266,6 +267,165 @@ func TestStoreReusesRuntimeWhenProxyDisappearsAndReappears(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestStoreRuntimeReportKeepsRemovedActiveProxyUntilRelease(t *testing.T) {
|
||||
store := NewStore("cluster-a", "worker-a")
|
||||
now := time.Date(2026, 7, 30, 15, 0, 0, 0, time.UTC)
|
||||
proxy := proxyDomain.Proxy{
|
||||
ID: "proxy-a", Scheme: proxyDomain.SchemeHTTP, SourceUpstream: "provider-a",
|
||||
State: proxyDomain.StateAvailable, MaxConcurrency: 1,
|
||||
}
|
||||
initial := Envelope{
|
||||
ClusterID: "cluster-a", WorkerID: "worker-a", Epoch: 1, Version: 1, Full: true,
|
||||
Proxies: []proxyDomain.Proxy{proxy},
|
||||
}
|
||||
initial.Checksum = Checksum(initial.Proxies)
|
||||
if err := store.Apply(initial); err != nil {
|
||||
t.Fatalf("Apply(initial): %v", err)
|
||||
}
|
||||
runtime := store.Current().Entries[0].Runtime
|
||||
reservation, ok := runtime.Reserve()
|
||||
if !ok {
|
||||
t.Fatal("Reserve() = false")
|
||||
}
|
||||
if err := reservation.Commit(); err != nil {
|
||||
t.Fatalf("Commit(): %v", err)
|
||||
}
|
||||
removed := Envelope{ClusterID: "cluster-a", WorkerID: "worker-a", Epoch: 1, Version: 2, Full: true}
|
||||
removed.Checksum = Checksum(nil)
|
||||
if err := store.Apply(removed); err != nil {
|
||||
t.Fatalf("Apply(remove): %v", err)
|
||||
}
|
||||
if got := store.activeRuntimeCount(); got != 1 {
|
||||
t.Fatalf("activeRuntimeCount(removed) = %d, want 1", got)
|
||||
}
|
||||
|
||||
report, err := store.RuntimeReport("session-a", 7, now)
|
||||
if err != nil {
|
||||
t.Fatalf("RuntimeReport(): %v", err)
|
||||
}
|
||||
if report.WorkerID != "worker-a" || report.SessionID != "session-a" || report.Sequence != 7 ||
|
||||
report.SnapshotVersion != 2 || report.OwnershipEpoch != 1 || !report.ObservedAt.Equal(now) ||
|
||||
len(report.Counters) != 1 || report.Counters[0] != (workerruntime.Counter{
|
||||
ProxyID: "proxy-a", Active: 1, Draining: true,
|
||||
}) {
|
||||
t.Fatalf("RuntimeReport() = %+v", report)
|
||||
}
|
||||
if err := reservation.Release(); err != nil {
|
||||
t.Fatalf("Release(): %v", err)
|
||||
}
|
||||
if got := store.activeRuntimeCount(); got != 0 {
|
||||
t.Fatalf("activeRuntimeCount(released) = %d, want 0", got)
|
||||
}
|
||||
report, err = store.RuntimeReport("session-a", 8, now.Add(time.Second))
|
||||
if err != nil || len(report.Counters) != 0 {
|
||||
t.Fatalf("RuntimeReport(after release) = %+v, %v", report, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStoreRuntimeReportMarksCurrentDrainingProxy(t *testing.T) {
|
||||
store := NewStore("cluster-a", "worker-a")
|
||||
now := time.Date(2026, 7, 30, 15, 0, 0, 0, time.UTC)
|
||||
proxy := proxyDomain.Proxy{
|
||||
ID: "proxy-a", Scheme: proxyDomain.SchemeHTTP,
|
||||
State: proxyDomain.StateDraining, MaxConcurrency: 1,
|
||||
}
|
||||
envelope := Envelope{
|
||||
ClusterID: "cluster-a", WorkerID: "worker-a", Epoch: 1, Version: 1, Full: true,
|
||||
Proxies: []proxyDomain.Proxy{proxy},
|
||||
}
|
||||
envelope.Checksum = Checksum(envelope.Proxies)
|
||||
if err := store.Apply(envelope); err != nil {
|
||||
t.Fatalf("Apply(): %v", err)
|
||||
}
|
||||
reservation, ok := store.Current().Entries[0].Runtime.Reserve()
|
||||
if !ok {
|
||||
t.Fatal("Reserve() = false")
|
||||
}
|
||||
report, err := store.RuntimeReport("session-a", 1, now)
|
||||
if err != nil || len(report.Counters) != 1 || !report.Counters[0].Draining {
|
||||
t.Fatalf("RuntimeReport() = %+v, %v", report, err)
|
||||
}
|
||||
if err := reservation.Cancel(); err != nil {
|
||||
t.Fatalf("Cancel(): %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStoreBoundsHistoricalRuntimeRegistryAndKeepsApplyTransactional(t *testing.T) {
|
||||
store, err := NewStoreWithRuntimeLimit("cluster-a", "worker-a", 1)
|
||||
if err != nil {
|
||||
t.Fatalf("NewStoreWithRuntimeLimit(): %v", err)
|
||||
}
|
||||
first := proxyDomain.Proxy{
|
||||
ID: "proxy-a", Scheme: proxyDomain.SchemeHTTP,
|
||||
State: proxyDomain.StateAvailable, MaxConcurrency: 1,
|
||||
}
|
||||
envelope := Envelope{
|
||||
ClusterID: "cluster-a", WorkerID: "worker-a", Epoch: 1, Version: 1,
|
||||
Full: true, Proxies: []proxyDomain.Proxy{first},
|
||||
}
|
||||
envelope.Checksum = Checksum(envelope.Proxies)
|
||||
if err := store.Apply(envelope); err != nil {
|
||||
t.Fatalf("Apply(first): %v", err)
|
||||
}
|
||||
removed := Envelope{
|
||||
ClusterID: "cluster-a", WorkerID: "worker-a", Epoch: 1, Version: 2, Full: true,
|
||||
}
|
||||
removed.Checksum = Checksum(nil)
|
||||
if err := store.Apply(removed); err != nil {
|
||||
t.Fatalf("Apply(removed): %v", err)
|
||||
}
|
||||
second := first
|
||||
second.ID = "proxy-b"
|
||||
overLimit := Envelope{
|
||||
ClusterID: "cluster-a", WorkerID: "worker-a", Epoch: 1, Version: 3,
|
||||
Full: true, Proxies: []proxyDomain.Proxy{second},
|
||||
}
|
||||
overLimit.Checksum = Checksum(overLimit.Proxies)
|
||||
if err := store.Apply(overLimit); !errors.Is(err, ErrRuntimeLimitExceeded) {
|
||||
t.Fatalf("Apply(over limit) error = %v, want ErrRuntimeLimitExceeded", err)
|
||||
}
|
||||
if current := store.Current(); current.Version != 2 || len(current.Entries) != 0 {
|
||||
t.Fatalf("Current() after rejected apply = version %d entries %d", current.Version, len(current.Entries))
|
||||
}
|
||||
}
|
||||
|
||||
func TestStoreRuntimeActiveIndexDropsZeroCounters(t *testing.T) {
|
||||
store := NewStore("cluster-a", "worker-a")
|
||||
proxy := proxyDomain.Proxy{
|
||||
ID: "proxy-a", Scheme: proxyDomain.SchemeHTTP,
|
||||
State: proxyDomain.StateAvailable, MaxConcurrency: 1,
|
||||
}
|
||||
envelope := Envelope{
|
||||
ClusterID: "cluster-a", WorkerID: "worker-a", Epoch: 1, Version: 1,
|
||||
Full: true, Proxies: []proxyDomain.Proxy{proxy},
|
||||
}
|
||||
envelope.Checksum = Checksum(envelope.Proxies)
|
||||
if err := store.Apply(envelope); err != nil {
|
||||
t.Fatalf("Apply(): %v", err)
|
||||
}
|
||||
reservation, ok := store.Current().Entries[0].Runtime.Reserve()
|
||||
if !ok {
|
||||
t.Fatal("Reserve() = false")
|
||||
}
|
||||
if got := store.activeRuntimeCount(); got != 0 {
|
||||
t.Fatalf("activeRuntimeCount(current) = %d, want 0", got)
|
||||
}
|
||||
if err := reservation.Cancel(); err != nil {
|
||||
t.Fatalf("Cancel(): %v", err)
|
||||
}
|
||||
if got := store.activeRuntimeCount(); got != 0 {
|
||||
t.Fatalf("activeRuntimeCount() = %d, want 0", got)
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Store) activeRuntimeCount() int {
|
||||
count := 0
|
||||
s.active.rangeEntries(func(_ string, _ *proxyDomain.Capacity) {
|
||||
count++
|
||||
})
|
||||
return count
|
||||
}
|
||||
|
||||
func collectSelectionIDs(selection Selection) []string {
|
||||
ids := make([]string, 0, selection.Len())
|
||||
for index := 0; index < selection.Len(); index++ {
|
||||
|
||||
@ -48,18 +48,35 @@ type Store interface {
|
||||
Resolve(context.Context, Reference) (Value, error)
|
||||
}
|
||||
|
||||
// Releaser removes transient credential material after the consumer has copied
|
||||
// it into its authoritative storage. Release is idempotent and version fenced.
|
||||
type Releaser interface {
|
||||
Release(context.Context, Reference) error
|
||||
}
|
||||
|
||||
type CapacityEnsurer interface {
|
||||
EnsureCapacity(context.Context, int) error
|
||||
}
|
||||
|
||||
type entry struct {
|
||||
scope *scopeState
|
||||
value Value
|
||||
version uint64
|
||||
reference Reference
|
||||
}
|
||||
|
||||
type scopeState struct {
|
||||
name string
|
||||
value Value
|
||||
version uint64
|
||||
leases int
|
||||
}
|
||||
|
||||
// MemoryStore keeps credentials in process memory and serializes access with a
|
||||
// context-aware lock.
|
||||
type MemoryStore struct {
|
||||
lock chan struct{}
|
||||
capacity int
|
||||
byScope map[string]*entry
|
||||
byScope map[string]*scopeState
|
||||
byRef map[string]*entry
|
||||
}
|
||||
|
||||
@ -82,7 +99,7 @@ func NewMemoryStore(capacity int) (*MemoryStore, error) {
|
||||
return &MemoryStore{
|
||||
lock: lock,
|
||||
capacity: capacity,
|
||||
byScope: make(map[string]*entry),
|
||||
byScope: make(map[string]*scopeState),
|
||||
byRef: make(map[string]*entry),
|
||||
}, nil
|
||||
}
|
||||
@ -100,36 +117,38 @@ func (s *MemoryStore) Put(ctx context.Context, scope string, value Value) (Refer
|
||||
if err := s.acquire(ctx); err != nil {
|
||||
return Reference{}, err
|
||||
}
|
||||
defer s.release()
|
||||
defer s.unlock()
|
||||
if err := ctx.Err(); err != nil {
|
||||
return Reference{}, err
|
||||
}
|
||||
|
||||
if current, ok := s.byScope[scope]; ok {
|
||||
if current.value == value {
|
||||
return current.reference, nil
|
||||
}
|
||||
current.value = value
|
||||
current.version++
|
||||
current.reference.CredentialVersion = versionString(current.version)
|
||||
return current.reference, nil
|
||||
if len(s.byRef) >= s.capacity {
|
||||
return Reference{}, ErrCapacityExceeded
|
||||
}
|
||||
if len(s.byScope) >= s.capacity {
|
||||
|
||||
current, exists := s.byScope[scope]
|
||||
if !exists && len(s.byScope) >= s.capacity {
|
||||
return Reference{}, ErrCapacityExceeded
|
||||
}
|
||||
secretRef, err := s.newUniqueSecretRef()
|
||||
if err != nil {
|
||||
return Reference{}, err
|
||||
}
|
||||
if !exists {
|
||||
current = &scopeState{name: scope, value: value, version: 1}
|
||||
s.byScope[scope] = current
|
||||
} else if current.value != value {
|
||||
current.value = value
|
||||
current.version++
|
||||
}
|
||||
created := &entry{
|
||||
value: value,
|
||||
version: 1,
|
||||
scope: current,
|
||||
value: value,
|
||||
reference: Reference{
|
||||
SecretRef: secretRef,
|
||||
CredentialVersion: versionString(1),
|
||||
CredentialVersion: versionString(current.version),
|
||||
},
|
||||
}
|
||||
s.byScope[scope] = created
|
||||
current.leases++
|
||||
s.byRef[secretRef] = created
|
||||
return created.reference, nil
|
||||
}
|
||||
@ -147,7 +166,7 @@ func (s *MemoryStore) Resolve(ctx context.Context, reference Reference) (Value,
|
||||
if err := s.acquire(ctx); err != nil {
|
||||
return Value{}, err
|
||||
}
|
||||
defer s.release()
|
||||
defer s.unlock()
|
||||
if err := ctx.Err(); err != nil {
|
||||
return Value{}, err
|
||||
}
|
||||
@ -162,6 +181,57 @@ func (s *MemoryStore) Resolve(ctx context.Context, reference Reference) (Value,
|
||||
return current.value, nil
|
||||
}
|
||||
|
||||
func (s *MemoryStore) Release(ctx context.Context, reference Reference) error {
|
||||
if err := contextError(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
if !s.valid() {
|
||||
return ErrInvalidStore
|
||||
}
|
||||
if reference.SecretRef == "" || !validVersion(reference.CredentialVersion) {
|
||||
return ErrInvalidReference
|
||||
}
|
||||
if err := s.acquire(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
defer s.unlock()
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
current, ok := s.byRef[reference.SecretRef]
|
||||
if !ok || current.reference != reference {
|
||||
return nil
|
||||
}
|
||||
delete(s.byRef, reference.SecretRef)
|
||||
current.scope.leases--
|
||||
if current.scope.leases == 0 {
|
||||
delete(s.byScope, current.scope.name)
|
||||
current.scope.value = Value{}
|
||||
}
|
||||
current.value = Value{}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *MemoryStore) EnsureCapacity(ctx context.Context, minimum int) error {
|
||||
if err := contextError(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
if !s.valid() {
|
||||
return ErrInvalidStore
|
||||
}
|
||||
if minimum <= 0 {
|
||||
return ErrInvalidCapacity
|
||||
}
|
||||
if err := s.acquire(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
defer s.unlock()
|
||||
if minimum > s.capacity {
|
||||
s.capacity = minimum
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *MemoryStore) acquire(ctx context.Context) error {
|
||||
if err := contextError(ctx); err != nil {
|
||||
return err
|
||||
@ -174,7 +244,7 @@ func (s *MemoryStore) acquire(ctx context.Context) error {
|
||||
}
|
||||
}
|
||||
|
||||
func (s *MemoryStore) release() {
|
||||
func (s *MemoryStore) unlock() {
|
||||
s.lock <- struct{}{}
|
||||
}
|
||||
|
||||
|
||||
@ -11,7 +11,7 @@ import (
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestMemoryStorePutIsIdempotentForUnchangedScope(t *testing.T) {
|
||||
func TestMemoryStorePutCreatesIndependentLeasesForUnchangedScope(t *testing.T) {
|
||||
store, err := NewMemoryStore(2)
|
||||
if err != nil {
|
||||
t.Fatalf("NewMemoryStore(): %v", err)
|
||||
@ -26,10 +26,10 @@ func TestMemoryStorePutIsIdempotentForUnchangedScope(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("Put(second): %v", err)
|
||||
}
|
||||
if first != second {
|
||||
t.Fatalf("second reference = %#v, want %#v", second, first)
|
||||
if first == second || first.SecretRef == second.SecretRef {
|
||||
t.Fatalf("references = %#v and %#v, want independent leases", first, second)
|
||||
}
|
||||
if first.SecretRef == "" || first.CredentialVersion != "v1" {
|
||||
if first.SecretRef == "" || first.CredentialVersion != "v1" || second.CredentialVersion != "v1" {
|
||||
t.Fatalf("first reference = %#v, want opaque ref at v1", first)
|
||||
}
|
||||
for _, plaintext := range []string{"provider-a", value.Username, value.Password} {
|
||||
@ -45,6 +45,12 @@ func TestMemoryStorePutIsIdempotentForUnchangedScope(t *testing.T) {
|
||||
if got != value {
|
||||
t.Fatalf("Resolve() = %#v, want %#v", got, value)
|
||||
}
|
||||
if err := store.Release(context.Background(), first); err != nil {
|
||||
t.Fatalf("Release(first lease): %v", err)
|
||||
}
|
||||
if got, err := store.Resolve(context.Background(), second); err != nil || got != value {
|
||||
t.Fatalf("Resolve(second lease) = %#v, %v", got, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCredentialFormattingRedactsSensitiveMaterial(t *testing.T) {
|
||||
@ -89,8 +95,8 @@ func TestCredentialFormattingRedactsSensitiveMaterial(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestMemoryStorePutIncrementsVersionAndRejectsStaleReference(t *testing.T) {
|
||||
store, err := NewMemoryStore(1)
|
||||
func TestMemoryStorePutIncrementsVersionWithoutRevokingActiveLease(t *testing.T) {
|
||||
store, err := NewMemoryStore(2)
|
||||
if err != nil {
|
||||
t.Fatalf("NewMemoryStore(): %v", err)
|
||||
}
|
||||
@ -106,14 +112,15 @@ func TestMemoryStorePutIncrementsVersionAndRejectsStaleReference(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("Put(new): %v", err)
|
||||
}
|
||||
if newReference.SecretRef != oldReference.SecretRef {
|
||||
t.Fatalf("new SecretRef changed across versions")
|
||||
if newReference.SecretRef == oldReference.SecretRef {
|
||||
t.Fatalf("new SecretRef reused an active lease")
|
||||
}
|
||||
if newReference.CredentialVersion != "v2" {
|
||||
t.Fatalf("new CredentialVersion = %q, want v2", newReference.CredentialVersion)
|
||||
}
|
||||
if _, err := store.Resolve(context.Background(), oldReference); !errors.Is(err, ErrCredentialVersionMismatch) {
|
||||
t.Fatalf("Resolve(stale) error = %v, want ErrCredentialVersionMismatch", err)
|
||||
old, err := store.Resolve(context.Background(), oldReference)
|
||||
if err != nil || old.Password != "old-password" {
|
||||
t.Fatalf("Resolve(active old lease) = %#v, %v", old, err)
|
||||
}
|
||||
got, err := store.Resolve(context.Background(), newReference)
|
||||
if err != nil {
|
||||
@ -148,8 +155,80 @@ func TestMemoryStoreEnforcesCapacityWithoutChangingExistingCredentials(t *testin
|
||||
if got != want {
|
||||
t.Fatalf("Resolve(existing) returned changed credentials")
|
||||
}
|
||||
if _, err := store.Put(context.Background(), "provider-a", want); !errors.Is(err, ErrCapacityExceeded) {
|
||||
t.Fatalf("Put(second lease at capacity) error = %v, want ErrCapacityExceeded", err)
|
||||
}
|
||||
if err := store.Release(context.Background(), reference); err != nil {
|
||||
t.Fatalf("Release(first lease): %v", err)
|
||||
}
|
||||
if _, err := store.Put(context.Background(), "provider-a", want); err != nil {
|
||||
t.Fatalf("Put(idempotent at capacity): %v", err)
|
||||
t.Fatalf("Put(after lease release): %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMemoryStoreReleaseMakesCapacityReusable(t *testing.T) {
|
||||
store, err := NewMemoryStore(1)
|
||||
if err != nil {
|
||||
t.Fatalf("NewMemoryStore(): %v", err)
|
||||
}
|
||||
first, err := store.Put(context.Background(), "provider-a", Value{Password: "first-password"})
|
||||
if err != nil {
|
||||
t.Fatalf("Put(first): %v", err)
|
||||
}
|
||||
if err := store.Release(context.Background(), first); err != nil {
|
||||
t.Fatalf("Release(first): %v", err)
|
||||
}
|
||||
if _, err := store.Resolve(context.Background(), first); !errors.Is(err, ErrCredentialMissing) {
|
||||
t.Fatalf("Resolve(released) error = %v, want ErrCredentialMissing", err)
|
||||
}
|
||||
if _, err := store.Put(context.Background(), "provider-b", Value{Password: "second-password"}); err != nil {
|
||||
t.Fatalf("Put(after release): %v", err)
|
||||
}
|
||||
if err := store.Release(context.Background(), first); err != nil {
|
||||
t.Fatalf("Release(idempotent): %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMemoryStoreReleaseOfStaleReferencePreservesCurrentVersion(t *testing.T) {
|
||||
store, err := NewMemoryStore(2)
|
||||
if err != nil {
|
||||
t.Fatalf("NewMemoryStore(): %v", err)
|
||||
}
|
||||
stale, err := store.Put(context.Background(), "provider-a", Value{Password: "old-password"})
|
||||
if err != nil {
|
||||
t.Fatalf("Put(old): %v", err)
|
||||
}
|
||||
current, err := store.Put(context.Background(), "provider-a", Value{Password: "new-password"})
|
||||
if err != nil {
|
||||
t.Fatalf("Put(new): %v", err)
|
||||
}
|
||||
if err := store.Release(context.Background(), stale); err != nil {
|
||||
t.Fatalf("Release(stale): %v", err)
|
||||
}
|
||||
if _, err := store.Resolve(context.Background(), current); err != nil {
|
||||
t.Fatalf("Resolve(current): %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMemoryStoreEnsureCapacityOnlyGrowsLimit(t *testing.T) {
|
||||
store, err := NewMemoryStore(1)
|
||||
if err != nil {
|
||||
t.Fatalf("NewMemoryStore(): %v", err)
|
||||
}
|
||||
if err := store.EnsureCapacity(context.Background(), 2); err != nil {
|
||||
t.Fatalf("EnsureCapacity(2): %v", err)
|
||||
}
|
||||
if _, err := store.Put(context.Background(), "provider-a", Value{}); err != nil {
|
||||
t.Fatalf("Put(provider-a): %v", err)
|
||||
}
|
||||
if _, err := store.Put(context.Background(), "provider-b", Value{}); err != nil {
|
||||
t.Fatalf("Put(provider-b): %v", err)
|
||||
}
|
||||
if err := store.EnsureCapacity(context.Background(), 1); err != nil {
|
||||
t.Fatalf("EnsureCapacity(shrink request): %v", err)
|
||||
}
|
||||
if _, err := store.Put(context.Background(), "provider-c", Value{}); !errors.Is(err, ErrCapacityExceeded) {
|
||||
t.Fatalf("Put(provider-c) error = %v, want retained capacity 2", err)
|
||||
}
|
||||
}
|
||||
|
||||
@ -211,13 +290,13 @@ func TestMemoryStoreRejectsNilAndZeroValueStores(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestMemoryStoreIsConcurrencySafeAndIdempotent(t *testing.T) {
|
||||
store, err := NewMemoryStore(1)
|
||||
func TestMemoryStoreCreatesIndependentConcurrentLeases(t *testing.T) {
|
||||
const workers = 100
|
||||
store, err := NewMemoryStore(workers)
|
||||
if err != nil {
|
||||
t.Fatalf("NewMemoryStore(): %v", err)
|
||||
}
|
||||
want := Value{Username: "alice", Password: "shared-password"}
|
||||
const workers = 100
|
||||
references := make(chan Reference, workers)
|
||||
errorsSeen := make(chan error, workers)
|
||||
var wait sync.WaitGroup
|
||||
@ -239,29 +318,29 @@ func TestMemoryStoreIsConcurrencySafeAndIdempotent(t *testing.T) {
|
||||
for err := range errorsSeen {
|
||||
t.Errorf("concurrent Put(): %v", err)
|
||||
}
|
||||
var first Reference
|
||||
secretRefs := make(map[string]struct{}, workers)
|
||||
for reference := range references {
|
||||
if first == (Reference{}) {
|
||||
first = reference
|
||||
if reference.CredentialVersion != "v1" {
|
||||
t.Errorf("concurrent version = %q, want v1", reference.CredentialVersion)
|
||||
}
|
||||
if reference != first {
|
||||
t.Errorf("concurrent Put() reference differs from first")
|
||||
secretRefs[reference.SecretRef] = struct{}{}
|
||||
}
|
||||
if len(secretRefs) != workers {
|
||||
t.Fatalf("unique concurrent leases = %d, want %d", len(secretRefs), workers)
|
||||
}
|
||||
for secretRef := range secretRefs {
|
||||
got, err := store.Resolve(context.Background(), Reference{
|
||||
SecretRef: secretRef, CredentialVersion: "v1",
|
||||
})
|
||||
if err != nil || got != want {
|
||||
t.Fatalf("Resolve(concurrent lease) = %#v, %v", got, err)
|
||||
}
|
||||
}
|
||||
if first.CredentialVersion != "v1" {
|
||||
t.Fatalf("concurrent version = %q, want v1", first.CredentialVersion)
|
||||
}
|
||||
got, err := store.Resolve(context.Background(), first)
|
||||
if err != nil {
|
||||
t.Fatalf("Resolve(): %v", err)
|
||||
}
|
||||
if got != want {
|
||||
t.Fatalf("Resolve() returned unexpected credentials")
|
||||
}
|
||||
}
|
||||
|
||||
func TestMemoryStoreSerializesConcurrentCredentialChanges(t *testing.T) {
|
||||
store, err := NewMemoryStore(1)
|
||||
const workers = 100
|
||||
store, err := NewMemoryStore(workers)
|
||||
if err != nil {
|
||||
t.Fatalf("NewMemoryStore(): %v", err)
|
||||
}
|
||||
@ -270,7 +349,6 @@ func TestMemoryStoreSerializesConcurrentCredentialChanges(t *testing.T) {
|
||||
reference Reference
|
||||
err error
|
||||
}
|
||||
const workers = 100
|
||||
results := make(chan result, workers)
|
||||
var wait sync.WaitGroup
|
||||
for index := range workers {
|
||||
@ -286,18 +364,13 @@ func TestMemoryStoreSerializesConcurrentCredentialChanges(t *testing.T) {
|
||||
close(results)
|
||||
|
||||
versions := make(map[string]struct{}, workers)
|
||||
var secretRef string
|
||||
secretRefs := make(map[string]struct{}, workers)
|
||||
var latest result
|
||||
for current := range results {
|
||||
if current.err != nil {
|
||||
t.Fatalf("concurrent Put(): %v", current.err)
|
||||
}
|
||||
if secretRef == "" {
|
||||
secretRef = current.reference.SecretRef
|
||||
}
|
||||
if current.reference.SecretRef != secretRef {
|
||||
t.Fatal("SecretRef changed across concurrent updates")
|
||||
}
|
||||
secretRefs[current.reference.SecretRef] = struct{}{}
|
||||
versions[current.reference.CredentialVersion] = struct{}{}
|
||||
if current.reference.CredentialVersion == "v100" {
|
||||
latest = current
|
||||
@ -306,6 +379,9 @@ func TestMemoryStoreSerializesConcurrentCredentialChanges(t *testing.T) {
|
||||
if len(versions) != workers {
|
||||
t.Fatalf("unique versions = %d, want %d", len(versions), workers)
|
||||
}
|
||||
if len(secretRefs) != workers {
|
||||
t.Fatalf("unique leases = %d, want %d", len(secretRefs), workers)
|
||||
}
|
||||
if latest.reference == (Reference{}) {
|
||||
t.Fatal("highest version v100 was not returned")
|
||||
}
|
||||
|
||||
72
internal/platform/lifecycle/group.go
Normal file
72
internal/platform/lifecycle/group.go
Normal file
@ -0,0 +1,72 @@
|
||||
package lifecycle
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"reflect"
|
||||
)
|
||||
|
||||
var (
|
||||
ErrInvalidGroup = errors.New("invalid lifecycle group")
|
||||
ErrRunnerStopped = errors.New("lifecycle runner stopped")
|
||||
)
|
||||
|
||||
type Runner interface {
|
||||
Run(context.Context) error
|
||||
}
|
||||
|
||||
type Group struct {
|
||||
runners []Runner
|
||||
}
|
||||
|
||||
func NewGroup(runners ...Runner) (*Group, error) {
|
||||
if len(runners) == 0 {
|
||||
return nil, ErrInvalidGroup
|
||||
}
|
||||
owned := make([]Runner, len(runners))
|
||||
for index, runner := range runners {
|
||||
if isNilRunner(runner) {
|
||||
return nil, ErrInvalidGroup
|
||||
}
|
||||
owned[index] = runner
|
||||
}
|
||||
return &Group{runners: owned}, nil
|
||||
}
|
||||
|
||||
func (group *Group) Run(ctx context.Context) error {
|
||||
if group == nil || ctx == nil || len(group.runners) == 0 {
|
||||
return ErrInvalidGroup
|
||||
}
|
||||
groupCtx, cancel := context.WithCancel(ctx)
|
||||
defer cancel()
|
||||
done := make(chan error, len(group.runners))
|
||||
for _, runner := range group.runners {
|
||||
go func() { done <- runner.Run(groupCtx) }()
|
||||
}
|
||||
|
||||
first := <-done
|
||||
cancel()
|
||||
for range len(group.runners) - 1 {
|
||||
<-done
|
||||
}
|
||||
if ctx.Err() != nil {
|
||||
return ctx.Err()
|
||||
}
|
||||
if first == nil {
|
||||
return ErrRunnerStopped
|
||||
}
|
||||
return first
|
||||
}
|
||||
|
||||
func isNilRunner(runner Runner) bool {
|
||||
if runner == nil {
|
||||
return true
|
||||
}
|
||||
reflected := reflect.ValueOf(runner)
|
||||
switch reflected.Kind() {
|
||||
case reflect.Chan, reflect.Func, reflect.Interface, reflect.Map, reflect.Pointer, reflect.Slice:
|
||||
return reflected.IsNil()
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
87
internal/platform/lifecycle/group_test.go
Normal file
87
internal/platform/lifecycle/group_test.go
Normal file
@ -0,0 +1,87 @@
|
||||
package lifecycle
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestGroupStartsAllRunnersAndCancelsSiblingsOnFirstError(t *testing.T) {
|
||||
started := make(chan struct{}, 2)
|
||||
cancelled := make(chan struct{}, 1)
|
||||
wantErr := errors.New("runner failed")
|
||||
group, err := NewGroup(
|
||||
runnerFunc(func(context.Context) error {
|
||||
started <- struct{}{}
|
||||
return wantErr
|
||||
}),
|
||||
runnerFunc(func(ctx context.Context) error {
|
||||
started <- struct{}{}
|
||||
<-ctx.Done()
|
||||
cancelled <- struct{}{}
|
||||
return ctx.Err()
|
||||
}),
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("NewGroup(): %v", err)
|
||||
}
|
||||
if err := group.Run(context.Background()); !errors.Is(err, wantErr) {
|
||||
t.Fatalf("Run() error = %v, want %v", err, wantErr)
|
||||
}
|
||||
<-started
|
||||
<-started
|
||||
select {
|
||||
case <-cancelled:
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("sibling runner was not cancelled")
|
||||
}
|
||||
}
|
||||
|
||||
func TestGroupWaitsForEveryRunnerBeforeReturning(t *testing.T) {
|
||||
release := make(chan struct{})
|
||||
var exited atomic.Bool
|
||||
group, err := NewGroup(
|
||||
runnerFunc(func(context.Context) error { return errors.New("failed") }),
|
||||
runnerFunc(func(ctx context.Context) error {
|
||||
<-ctx.Done()
|
||||
<-release
|
||||
exited.Store(true)
|
||||
return nil
|
||||
}),
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("NewGroup(): %v", err)
|
||||
}
|
||||
done := make(chan error, 1)
|
||||
go func() { done <- group.Run(context.Background()) }()
|
||||
select {
|
||||
case <-done:
|
||||
t.Fatal("Run() returned before sibling exited")
|
||||
case <-time.After(20 * time.Millisecond):
|
||||
}
|
||||
close(release)
|
||||
<-done
|
||||
if !exited.Load() {
|
||||
t.Fatal("sibling exit was not observed")
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewGroupRejectsEmptyAndTypedNilRunners(t *testing.T) {
|
||||
var typedNil *nilRunner
|
||||
for _, runners := range [][]Runner{nil, {typedNil}} {
|
||||
group, err := NewGroup(runners...)
|
||||
if err == nil || group != nil {
|
||||
t.Fatalf("NewGroup() = (%v, %v), want invalid group", group, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
type runnerFunc func(context.Context) error
|
||||
|
||||
func (f runnerFunc) Run(ctx context.Context) error { return f(ctx) }
|
||||
|
||||
type nilRunner struct{}
|
||||
|
||||
func (*nilRunner) Run(context.Context) error { return nil }
|
||||
20
progress.md
20
progress.md
@ -2,6 +2,26 @@
|
||||
|
||||
## 2026-07-30
|
||||
|
||||
- Gateway `Capacity` 新增一次打包原子读取,`snapshot.Store` 可生成完整稀疏
|
||||
Active/Reserved 运行态报告;当前快照已移除但仍有连接的 Proxy 会持续以
|
||||
draining 上报,归零后从后续报告消失。
|
||||
- 新增公用 `workerruntime` session/report/read seam 与并发安全 MemoryStore;
|
||||
完整替换、空报告清零、session fencing、单调 sequence、同内容幂等重放、
|
||||
冲突/倒序拒绝和 TTL fail-closed 均已有单测。
|
||||
- 生产 `redisactivity` Adapter 新增 Worker session/运行态 Lua 和权威
|
||||
`pool.InventoryReader`;Managed/Available Slots 原子计入状态、TTL safety、
|
||||
MaxConcurrency、ownership 及 Active/Reserved,未知或过期运行态贡献零容量。
|
||||
- 真实 Redis 8.2 已覆盖 ACK snapshot/epoch、旧 session、空报告、超前 epoch、
|
||||
报告过期、扫描预算耗尽及容量聚合。Managed 使用已有权威计数,Available
|
||||
Slots 只扫描目标 Upstream 的未分配/已分配可用索引;Gateway 报告扫描当前
|
||||
Snapshot,并用分片索引补充已移除但仍非零的 runtime,历史注册表设置硬上限。
|
||||
WorkerControlPlane 接收端、Provider
|
||||
Fleet/bootstrap、全局 `fetch.maxTotal` 和健康执行链仍待完成,总验收计数保持
|
||||
51/73。
|
||||
- Gateway 当前 Proxy 禁用活跃索引回调,只有移出 Snapshot 后才开启分片追踪;
|
||||
本机 100k Proxy `Acquire` 三轮 1 秒基准为 893.9-1047 ns/op、256 B/op、
|
||||
2 allocs/op。该数据只证明本地调度微基准,不代表 100k QPS 集群验收。
|
||||
|
||||
- 新增公用 Provider `Coordinator.RunLeader` / `LeaderSession` 深 seam 和独立
|
||||
`redisprovider` Adapter;Redis Lua 原子维护 generation、epoch、Leader 租约、
|
||||
全局 requestInterval 与带 TTL 的 maxInFlight Permit,异常时 fail-closed。
|
||||
|
||||
@ -35,7 +35,8 @@
|
||||
机器契约和文档类滞后勾选已按仓库证据校正
|
||||
13. [进行中] 落地 `proxy-controller` 进程装配;配置单次加载、PostgreSQL 迁移、
|
||||
Redis 活动池、低基数状态聚合、Distribution/Admin/Metrics 启动与关闭已完成,
|
||||
双存储 bootstrap 和探针集成已通过,Provider、业务指标与完整容器进程链仍待实现
|
||||
双存储 bootstrap 和探针集成已通过;Worker 运行态存储与权威容量读取原语已
|
||||
完成,WorkerControlPlane、Provider、业务指标与完整容器进程链仍待实现
|
||||
|
||||
## 串并行关系
|
||||
|
||||
@ -57,7 +58,8 @@
|
||||
- Docker Compose 配置与 Kubernetes Kustomize 已完成静态渲染验证;Redis 8.2
|
||||
与 PostgreSQL 18 的隔离 Adapter fixture 已运行,完整目标运行拓扑尚未启动。
|
||||
- `cmd/proxy-controller` 已实现 Admin/Distribution/Metrics 与双存储启动装配;
|
||||
Gateway、Checker、Loadgen、Provider Leader/分布式限流、业务指标、Redis
|
||||
故障转移验证与代表性集群压测属于后续实施范围。
|
||||
Provider 分布式协调和 Worker 运行态 Redis 原语已完成,但 WorkerControlPlane
|
||||
接收端、Provider Fleet、Gateway、Checker、Loadgen、业务指标、Redis 故障
|
||||
转移验证与代表性集群压测属于后续实施范围。
|
||||
- `implementation-plan.md` 当前按 73 个验收项统计;已校正为 51 项完成,
|
||||
验收项完成率约 69.9%,不等同于生产就绪度。
|
||||
|
||||
Loading…
Reference in New Issue
Block a user